1210 files changed, 208785 insertions, 84577 deletions

diff --git a/.appveyor.yml b/.appveyor.yml
new file mode 100644
index 0000000000..818e074a4e
--- /dev/null
+++ b/.appveyor.yml
@@ -0,0 +1,131 @@
+version: 1.0.{build}
+
+clone_depth: 50
+
+# Appveyor images are named after the Visual Studio version they contain.
+# But we compile using MinGW, not Visual Studio.
+# We use these images because they have different Windows versions.
+image:
+  # Windows Server 2019
+  - Visual Studio 2019
+  # Windows Server 2012 R2
+  - Visual Studio 2015
+
+environment:
+  compiler: mingw
+
+  matrix:
+  - target: i686-w64-mingw32
+    compiler_path: mingw32
+    mingw_prefix: mingw-w64-i686
+    hardening:
+  - target: x86_64-w64-mingw32
+    compiler_path: mingw64
+    mingw_prefix: mingw-w64-x86_64
+    # hardening doesn't work with mingw-w64-x86_64-gcc, because it's gcc 8
+    hardening: --disable-gcc-hardening
+
+matrix:
+  # Don't keep building failing jobs
+  fast_finish: true
+  # Skip the 32-bit Windows Server 2019 job, and the 64-bit Windows Server
+  # 2012 R2 job, to speed up the build.
+  # The environment variables must be listed without the 'environment' tag.
+  exclude:
+    - image: Visual Studio 2019
+      target: i686-w64-mingw32
+      compiler_path: mingw32
+      mingw_prefix: mingw-w64-i686
+      hardening:
+    - image: Visual Studio 2015
+      target: x86_64-w64-mingw32
+      compiler_path: mingw64
+      mingw_prefix: mingw-w64-x86_64
+      # hardening doesn't work with mingw-w64-x86_64-gcc, because it's gcc 8
+      hardening: --disable-gcc-hardening
+
+install:
+- ps: >-
+    Function Execute-Command ($commandPath)
+    {
+        & $commandPath $args 2>&1
+        if ( $LastExitCode -ne 0 ) {
+            $host.SetShouldExit( $LastExitCode )
+        }
+    }
+    Function Execute-Bash ()
+    {
+        Execute-Command 'c:\msys64\usr\bin\bash' '-e' '-c' $args
+    }
+    <# mingw packages start with ${env:mingw_prefix}
+     # unprefixed packages are from MSYS2, which is like Cygwin. Avoid them.
+     #
+     # Use pacman --debug to show package downloads and install locations
+     #>
+    Execute-Command "C:\msys64\usr\bin\pacman" -Sy --verbose --needed --noconfirm ${env:mingw_prefix}-libevent ${env:mingw_prefix}-openssl ${env:mingw_prefix}-pkg-config ${env:mingw_prefix}-xz ${env:mingw_prefix}-zstd ;
+
+build_script:
+- ps: >-
+    if ($env:compiler -eq "mingw") {
+            <# use the MSYS2 compiler and user binaries to build and install #>
+            $oldpath = ${env:Path} -split ';'
+            $buildpath = @("C:\msys64\${env:compiler_path}\bin", "C:\msys64\usr\bin") + $oldpath
+            $env:Path = @($buildpath) -join ';'
+            $env:build = @("${env:APPVEYOR_BUILD_FOLDER}", $env:target) -join '\'
+            Set-Location "${env:APPVEYOR_BUILD_FOLDER}"
+            Execute-Bash 'autoreconf -i'
+            mkdir "${env:build}"
+            Set-Location "${env:build}"
+            Execute-Bash "which ${env:target}-gcc"
+            Execute-Bash "${env:target}-gcc --version"
+            <# compile for mingw
+             # mingw zstd doesn't come with a pkg-config file, so we manually
+             # configure its flags. liblzma just works.
+             #>
+            Execute-Bash "ZSTD_CFLAGS='-L/${env:compiler_path}/include' ZSTD_LIBS='-L/${env:compiler_path}/lib -lzstd' ../configure --prefix=/${env:compiler_path} --build=${env:target} --host=${env:target} --with-openssl-dir=/${env:compiler_path} --disable-asciidoc --enable-fatal-warnings ${env:hardening}"
+            Execute-Bash "V=1 make -k -j2"
+            Execute-Bash "V=1 make -k -j2 install"
+     }
+
+test_script:
+- ps: >-
+    if ($env:compiler -eq "mingw") {
+            <# use the MSYS2 compiler binaries to make check #>
+            $oldpath = ${env:Path} -split ';'
+            $buildpath = @("C:\msys64\${env:compiler_path}\bin") + $oldpath
+            $env:Path = $buildpath -join ';'
+            Set-Location "${env:build}"
+            Execute-Bash "VERBOSE=1 make -k -j2 check"
+    }
+
+on_finish:
+- ps: >-
+    <# if we failed before install:, these functions won't be defined #>
+    Function Execute-Command ($commandPath)
+    {
+        & $commandPath $args 2>&1
+        if ( $LastExitCode -ne 0 ) {
+            $host.SetShouldExit( $LastExitCode )
+        }
+    }
+    Function Execute-Bash ()
+    {
+        Execute-Command 'c:\msys64\usr\bin\bash' '-e' '-c' $args
+    }
+    if ($env:compiler -eq "mingw") {
+            <# use the MSYS2 user binaries to archive failures #>
+            $oldpath = ${env:Path} -split ';'
+            $buildpath = @("C:\msys64\usr\bin") + $oldpath
+            $env:Path = @($buildpath) -join ';'
+            Set-Location "${env:build}"
+            <# store logs as appveyor artifacts: see the artifacts tab #>
+            Execute-Bash "7z a logs.zip config.log || true"
+            Execute-Bash "7z a logs.zip test-suite.log || true"
+            Execute-Bash "appveyor PushArtifact logs.zip || true"
+            Execute-Bash "tail -1000 config.log || true"
+            Execute-Bash "cat test-suite.log || true"
+    }
+
+# notify the IRC channel of any failures
+on_failure:
+- cmd: C:\Python27\python.exe %APPVEYOR_BUILD_FOLDER%\scripts\test\appveyor-irc-notify.py irc.oftc.net:6697 tor-ci failure
diff --git a/.gitignore b/.gitignore
index dc6738c079..f1ce903a11 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,7 @@
 .#*
 *~
 *.swp
+*.swo
 # C stuff
 *.o
 *.obj
@@ -18,6 +19,8 @@
 .dirstamp
 *.trs
 *.log
+# Calltool stuff
+.*.graph
 # Stuff made by our makefiles
 *.bak
 # Python droppings
@@ -38,9 +41,11 @@ uptime-*.json
 /Makefile
 /Makefile.in
 /aclocal.m4
+/ar-lib
 /autom4te.cache
 /build-stamp
 /compile
+/config.rust
 /configure
 /Doxyfile
 /orconfig.h
@@ -51,6 +56,7 @@ uptime-*.json
 /config.guess
 /config.sub
 /conftest*
+/link_rust.sh
 /micro-revision.*
 /patch-stamp
 /stamp-h
@@ -66,7 +72,11 @@ uptime-*.json
 /mkinstalldirs
 /Tor*Bundle.dmg
 /tor-*-win32.exe
+/warning_flags
+
 /coverage_html/
+/callgraph/
+
 
 # /contrib/
 /contrib/dist/tor.sh
@@ -94,11 +104,6 @@ uptime-*.json
 /doc/tor.html
 /doc/tor.html.in
 /doc/tor.1.xml
-/doc/tor-fw-helper.1
-/doc/tor-fw-helper.1.in
-/doc/tor-fw-helper.html
-/doc/tor-fw-helper.html.in
-/doc/tor-fw-helper.1.xml
 /doc/tor-gencert.1
 /doc/tor-gencert.1.in
 /doc/tor-gencert.html
@@ -114,6 +119,11 @@ uptime-*.json
 /doc/torify.html
 /doc/torify.html.in
 /doc/torify.1.xml
+/doc/tor-print-ed-signing-cert.1
+/doc/tor-print-ed-signing-cert.1.in
+/doc/tor-print-ed-signing-cert.html
+/doc/tor-print-ed-signing-cert.html.in
+/doc/tor-print-ed-signing-cert.1.xml
 
 # /doc/spec/
 /doc/spec/Makefile
@@ -127,23 +137,8 @@ uptime-*.json
 /src/Makefile
 /src/Makefile.in
 
-# /src/common/
-/src/common/Makefile
-/src/common/Makefile.in
-/src/common/libor.a
-/src/common/libor-testing.a
-/src/common/libor.lib
-/src/common/libor-ctime.a
-/src/common/libor-ctime-testing.a
-/src/common/libor-ctime.lib
-/src/common/libor-crypto.a
-/src/common/libor-crypto-testing.a
-/src/common/libor-crypto.lib
-/src/common/libor-event.a
-/src/common/libor-event-testing.a
-/src/common/libor-event.lib
-/src/common/libcurve25519_donna.a
-/src/common/libcurve25519_donna.lib
+# /src/trace
+/src/trace/libor-trace.a
 
 # /src/config/
 /src/config/Makefile
@@ -161,16 +156,81 @@ uptime-*.json
 /src/ext/keccak-tiny/libkeccak-tiny.a
 /src/ext/keccak-tiny/libkeccak-tiny.lib
 
-# /src/or/
-/src/or/Makefile
-/src/or/Makefile.in
-/src/or/tor
-/src/or/tor.exe
-/src/or/tor-cov
-/src/or/tor-cov.exe
-/src/or/libtor.a
-/src/or/libtor-testing.a
-/src/or/libtor.lib
+# /src/lib
+/src/lib/libcurve25519_donna.a
+/src/lib/libtor-compress.a
+/src/lib/libtor-compress-testing.a
+/src/lib/libtor-container.a
+/src/lib/libtor-container-testing.a
+/src/lib/libtor-crypt-ops.a
+/src/lib/libtor-crypt-ops-testing.a
+/src/lib/libtor-ctime.a
+/src/lib/libtor-ctime-testing.a
+/src/lib/libtor-encoding.a
+/src/lib/libtor-encoding-testing.a
+/src/lib/libtor-evloop.a
+/src/lib/libtor-evloop-testing.a
+/src/lib/libtor-err.a
+/src/lib/libtor-err-testing.a
+/src/lib/libtor-fdio.a
+/src/lib/libtor-fdio-testing.a
+/src/lib/libtor-fs.a
+/src/lib/libtor-fs-testing.a
+/src/lib/libtor-geoip.a
+/src/lib/libtor-geoip-testing.a
+/src/lib/libtor-intmath.a
+/src/lib/libtor-intmath-testing.a
+/src/lib/libtor-lock.a
+/src/lib/libtor-lock-testing.a
+/src/lib/libtor-log.a
+/src/lib/libtor-log-testing.a
+/src/lib/libtor-malloc.a
+/src/lib/libtor-malloc-testing.a
+/src/lib/libtor-math.a
+/src/lib/libtor-math-testing.a
+/src/lib/libtor-memarea.a
+/src/lib/libtor-memarea-testing.a
+/src/lib/libtor-meminfo.a
+/src/lib/libtor-meminfo-testing.a
+/src/lib/libtor-net.a
+/src/lib/libtor-net-testing.a
+/src/lib/libtor-osinfo.a
+/src/lib/libtor-osinfo-testing.a
+/src/lib/libtor-process.a
+/src/lib/libtor-process-testing.a
+/src/lib/libtor-sandbox.a
+/src/lib/libtor-sandbox-testing.a
+/src/lib/libtor-string.a
+/src/lib/libtor-string-testing.a
+/src/lib/libtor-smartlist-core.a
+/src/lib/libtor-smartlist-core-testing.a
+/src/lib/libtor-term.a
+/src/lib/libtor-term-testing.a
+/src/lib/libtor-thread.a
+/src/lib/libtor-thread-testing.a
+/src/lib/libtor-time.a
+/src/lib/libtor-time-testing.a
+/src/lib/libtor-tls.a
+/src/lib/libtor-tls-testing.a
+/src/lib/libtor-trace.a
+/src/lib/libtor-wallclock.a
+/src/lib/libtor-wallclock-testing.a
+
+# /src/tor
+/src/core/libtor-app.a
+/src/core/libtor-app-testing.a
+
+# /src/app
+/src/app/tor
+/src/app/tor.exe
+/src/app/tor-cov
+/src/app/tor-cov.exe
+
+# /src/rust
+/src/rust/.cargo/config
+/src/rust/.cargo/registry
+/src/rust/target
+/src/rust/registry
 
 # /src/test
 /src/test/Makefile
@@ -183,6 +243,7 @@ uptime-*.json
 /src/test/test-child
 /src/test/test-memwipe
 /src/test/test-ntor-cl
+/src/test/test-hs-ntor-cl
 /src/test/test-switch-id
 /src/test/test-timers
 /src/test/test_workqueue
@@ -191,16 +252,24 @@ uptime-*.json
 /src/test/test-bt-cl.exe
 /src/test/test-child.exe
 /src/test/test-ntor-cl.exe
+/src/test/test-hs-ntor-cl.exe
 /src/test/test-memwipe.exe
 /src/test/test-switch-id.exe
 /src/test/test-timers.exe
 /src/test/test_workqueue.exe
 
+# /src/test/fuzz
+/src/test/fuzz/fuzz-*
+/src/test/fuzz/lf-fuzz-*
+
 # /src/tools/
+/src/tools/libtorrunner.a
 /src/tools/tor-checkkey
 /src/tools/tor-resolve
 /src/tools/tor-cov-resolve
 /src/tools/tor-gencert
+/src/tools/tor-print-ed-signing-cert
+/src/tools/tor-print-ed-signing-cert.exe
 /src/tools/tor-cov-gencert
 /src/tools/tor-checkkey.exe
 /src/tools/tor-resolve.exe
@@ -214,12 +283,6 @@ uptime-*.json
 /src/trunnel/libor-trunnel-testing.a
 /src/trunnel/libor-trunnel.a
 
-# /src/tools/tor-fw-helper/
-/src/tools/tor-fw-helper/tor-fw-helper
-/src/tools/tor-fw-helper/tor-fw-helper.exe
-/src/tools/tor-fw-helper/Makefile
-/src/tools/tor-fw-helper/Makefile.in
-
 # /src/win32/
 /src/win32/Makefile
 /src/win32/Makefile.in
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000000..d2d0d55dd4
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,45 @@
+before_script:
+    - apt-get update -qq
+    - apt-get upgrade -qy
+
+build:
+  script:
+    - apt-get install -qy --fix-missing automake build-essential
+      libevent-dev libssl-dev zlib1g-dev
+      libseccomp-dev liblzma-dev libscrypt-dev
+    - ./autogen.sh
+    - ./configure --disable-asciidoc --enable-fatal-warnings
+      --disable-silent-rules
+    - make check || (e=$?; cat test-suite.log; exit $e)
+    - make install
+
+update:
+  only:
+    - schedules
+  script: 
+    - "apt-get install -y --fix-missing git openssh-client"
+    
+    # Run ssh-agent (inside the build environment)
+    - eval $(ssh-agent -s)
+
+    # Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store
+    - ssh-add <(echo "$DEPLOY_KEY")
+
+    # For Docker builds disable host key checking. Be aware that by adding that
+    # you are susceptible to man-in-the-middle attacks.
+    # WARNING: Use this only with the Docker executor, if you use it with shell
+    # you will overwrite your user's SSH config.
+    - mkdir -p ~/.ssh
+    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
+    # In order to properly check the server's host key, assuming you created the
+    # SSH_SERVER_HOSTKEYS variable previously, uncomment the following two lines
+    # instead.
+    - mkdir -p ~/.ssh
+    - '[[ -f /.dockerenv ]] && echo "$SSH_SERVER_HOSTKEYS" > ~/.ssh/known_hosts'
+    - echo "merging from torgit"
+    - git config --global user.email "labadmin@oniongit.eu"
+    - git config --global user.name "gitadmin"
+    - "mkdir tor"
+    - "cd tor" 
+    - git clone --bare https://git.torproject.org/tor.git
+    - git push --mirror git@oniongit.eu:network/tor.git
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000000..8f80405650
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "src/ext/rust"]
+	path = src/ext/rust
+	url = https://git.torproject.org/tor-rust-dependencies
diff --git a/.travis.yml b/.travis.yml
index 6004727b6d..98bb8cce80 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,6 +2,13 @@ language: c
 
 cache:
   ccache: true
+  ## cargo: true
+  directories:
+    - $HOME/.cargo
+    ## caching CARGO_TARGET_DIR actually slows down the build over time,
+    ## because old build products are never deleted.
+    ## where we point CARGO_TARGET_DIR in all our cargo invocations
+    #- $TRAVIS_BUILD_DIR/src/rust/target
 
 compiler:
   - gcc
@@ -34,10 +41,14 @@ matrix:
       os: osx
       ## Turn off some newer features, turn on clang's -Wtypedef-redefinition
       env: C_DIALECT_OPTIONS="-std=gnu99"
-    ## We run chutney on Linux, because it's faster than chutney on macOS
+    ## We check NSS
     ## Use -std=gnu99 to turn off some newer features, and maybe turn on some
     ## extra gcc warnings?
-    - env: CHUTNEY="yes" CHUTNEY_ALLOW_FAILURES="2" SKIP_MAKE_CHECK="yes" C_DIALECT_OPTIONS="-std=gnu99"
+    - env: NSS_OPTIONS="--enable-nss" C_DIALECT_OPTIONS="-std=gnu99"
+    ## We run chutney on Linux, because it's faster than chutney on macOS
+    ## Chutney is a fast job, clang is slower on Linux, so we do Chutney clang
+    - env: CHUTNEY="yes" CHUTNEY_ALLOW_FAILURES="2" SKIP_MAKE_CHECK="yes"
+      compiler: clang
       ## (Linux only) Use an older Linux image (Ubuntu Trusty)
       ## The Xenial and Bionic images cause permissions issues for chutney,
       ## this is a workaround, until we fix #32240.
@@ -48,9 +59,25 @@ matrix:
       compiler: clang
     ## We include a single coverage build with the best options for coverage
     - env: COVERAGE_OPTIONS="--enable-coverage" HARDENING_OPTIONS=""
+    ## We run rust on Linux, because it's faster than rust on macOS
+    ## We check rust offline
+    - env: RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true
+    ## We check asciidoc with distcheck, to make sure we remove doc products
+    - env: DISTCHECK="yes" ASCIIDOC_OPTIONS="" SKIP_MAKE_CHECK="yes"
+    ## We check disable module dirauth
+    - env: MODULES_OPTIONS="--disable-module-dirauth"
+    ## macOS builds are very slow, and we have a limited number of
+    ## concurrent macOS jobs. We're not actively developing Rust, so it is
+    ## the lowest priority.
+    ## We run rust on macOS, because we have seen macOS rust failures before
+    #- env: RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
+    #  compiler: clang
+    #  os: osx
     ## We run chutney on macOS, because macOS Travis has IPv6
     - env: CHUTNEY="yes" CHUTNEY_ALLOW_FAILURES="2" SKIP_MAKE_CHECK="yes"
       os: osx
+    ## We clone our stem repo and run `make test-stem`
+    - env: TEST_STEM="yes" SKIP_MAKE_CHECK="yes"
 
   ## Allow the build to report success (with non-required sub-builds
   ## continuing to run) if all required sub-builds have succeeded.
@@ -59,11 +86,16 @@ matrix:
   ## Careful! We use global envs, which makes it hard to allow failures by env:
   ## https://docs.travis-ci.com/user/customizing-the-build#matching-jobs-with-allow_failures
   allow_failures:
-    ## macOS chutney is very slow, so we let the build finish before it's done
-    ## We'd like to fast finish, but still eventually show failures.
-    ## But Travis doesn't have that option.
+    ## macOS rust and chutney are very slow, so we let the build finish before
+    ## they are done.  We'd like to fast finish, but still eventually show
+    ## any failures in the build status. But Travis doesn't have that ability.
+    - env: RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
+      compiler: clang
+      os: osx
     - env: CHUTNEY="yes" CHUTNEY_ALLOW_FAILURES="2" SKIP_MAKE_CHECK="yes"
       os: osx
+    ## test-stem sometimes hangs on Travis
+    - env: TEST_STEM="yes" SKIP_MAKE_CHECK="yes"
 
 ## (Linux only) Use a recent Linux image (Ubuntu Bionic)
 dist: bionic
@@ -80,14 +112,21 @@ addons:
       - zlib1g-dev
       ## Optional dependencies
       - libcap-dev
+      - liblzma-dev
+      - libnss3-dev
       - libscrypt-dev
       - libseccomp-dev
+      ## zstd doesn't exist in Ubuntu Trusty
+      #- libzstd
       ## Conditional build dependencies
       ## Always installed, so we don't need sudo
       - asciidoc
       - docbook-xsl
       - docbook-xml
       - xmlto
+      ## Utilities
+      ## preventing or diagnosing hangs
+      - timelimit
   ## (OSX only)
   homebrew:
     packages:
@@ -100,6 +139,8 @@ addons:
       #- zlib
       ## Optional dependencies
       - libscrypt
+      - xz
+      - zstd
       ## Required build dependencies
       ## Tor needs pkg-config to find some dependencies at build time
       - pkg-config
@@ -109,6 +150,9 @@ addons:
       ## Always installed, because manual brew installs are hard to get right
       - asciidoc
       - xmlto
+      ## Utilities
+      ## preventing or diagnosing hangs
+      - timelimit
 
 ## (OSX only) Use a recent macOS image
 ## See https://docs.travis-ci.com/user/reference/osx#os-x-version
@@ -116,6 +160,10 @@ addons:
 ## Recent is Xcode 11.2 on macOS 10.14 as of October 2019
 osx_image: xcode11.2
 
+before_install:
+  ## Create empty rust directories for non-Rust builds, so caching succeeds
+  - if [[ "$RUST_OPTIONS" == "" ]]; then mkdir -p $HOME/.cargo $TRAVIS_BUILD_DIR/src/rust/target; fi
+
 install:
   ## If we're on OSX, configure ccache (ccache is automatically installed and configured on Linux)
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export PATH="/usr/local/opt/ccache/libexec:$PATH"; fi
@@ -126,25 +174,49 @@ install:
   - if [[ "$COVERAGE_OPTIONS" != "" ]]; then pip install --user cpp-coveralls; fi
   ## If we're on OSX, and using asciidoc, configure asciidoc
   - if [[ "$ASCIIDOC_OPTIONS" == "" ]] && [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export XML_CATALOG_FILES="/usr/local/etc/xml/catalog"; fi
+  ## If we're using Rust, download rustup
+  - if [[ "$RUST_OPTIONS" != "" ]]; then curl -Ssf -o rustup.sh https://sh.rustup.rs; fi
+  ## Install the nightly channels of rustc and cargo and setup our toolchain environment
+  - if [[ "$RUST_OPTIONS" != "" ]]; then sh rustup.sh -y --default-toolchain nightly; fi
+  - if [[ "$RUST_OPTIONS" != "" ]]; then source $HOME/.cargo/env; fi
+  ## If we're testing rust builds in offline-mode, then set up our vendored dependencies
+  - if [[ "$TOR_RUST_DEPENDENCIES" == "true" ]]; then export TOR_RUST_DEPENDENCIES=$PWD/src/ext/rust/crates; fi
   ## If we're running chutney, install it.
   - if [[ "$CHUTNEY" != "" ]]; then git clone --depth 1 https://github.com/torproject/chutney.git ; export CHUTNEY_PATH="$(pwd)/chutney"; fi
+  ## If we're running stem, install it.
+  - if [[ "$TEST_STEM" != "" ]]; then git clone --depth 1 https://github.com/torproject/stem.git ; export STEM_SOURCE_DIR=`pwd`/stem; fi
+  ##
   ## Finally, list installed package versions
   - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then dpkg-query --show; fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew list --versions; fi
+  ## Get some info about rustup, rustc and cargo
+  - if [[ "$RUST_OPTIONS" != "" ]]; then which rustup; fi
+  - if [[ "$RUST_OPTIONS" != "" ]]; then which rustc; fi
+  - if [[ "$RUST_OPTIONS" != "" ]]; then which cargo; fi
+  - if [[ "$RUST_OPTIONS" != "" ]]; then rustup --version; fi
+  - if [[ "$RUST_OPTIONS" != "" ]]; then rustc --version; fi
+  - if [[ "$RUST_OPTIONS" != "" ]]; then cargo --version; fi
   ## Get python version
   - python --version
   ## If we're running chutney, show the chutney commit
   - if [[ "$CHUTNEY" != "" ]]; then pushd "$CHUTNEY_PATH"; git log -1 ; popd ; fi
+  ## If we're running stem, show the stem version and commit
+  - if [[ "$TEST_STEM" != "" ]]; then pushd stem; python -c "from stem import stem; print(stem.__version__);"; git log -1; popd; fi
 
 script:
+  # Skip test_rebind on macOS
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export TOR_SKIP_TEST_REBIND=true; fi
   - ./autogen.sh
-  - CONFIGURE_FLAGS="$ASCIIDOC_OPTIONS $COVERAGE_OPTIONS $HARDENING_OPTIONS $OPENSSL_OPTIONS --enable-fatal-warnings --disable-silent-rules"
+  - CONFIGURE_FLAGS="$ASCIIDOC_OPTIONS $COVERAGE_OPTIONS $HARDENING_OPTIONS $MODULES_OPTIONS $NSS_OPTIONS $OPENSSL_OPTIONS $RUST_OPTIONS --enable-fatal-warnings --disable-silent-rules"
   - echo "Configure flags are $CONFIGURE_FLAGS CC=\"$CC $C_DIALECT_OPTIONS\""
   - ./configure $CONFIGURE_FLAGS CC="$CC $C_DIALECT_OPTIONS"
   ## We run `make check` because that's what https://jenkins.torproject.org does.
   - if [[ "$SKIP_MAKE_CHECK" == "" ]]; then make check; fi
   - if [[ "$DISTCHECK" != "" ]]; then make distcheck DISTCHECK_CONFIGURE_FLAGS="$CONFIGURE_FLAGS"; fi
   - if [[ "$CHUTNEY" != "" ]]; then make test-network-all; fi
+  ## Diagnostic for bug 29437: kill stem if it hangs for 9.5 minutes
+  ## Travis will kill the job after 10 minutes with no output
+  - if [[ "$TEST_STEM" != "" ]]; then make src/app/tor; timelimit -p -t 540 -s USR1 -T 30 -S ABRT python3 "$STEM_SOURCE_DIR"/run_tests.py --tor src/app/tor --integ --test control.controller --test control.base_controller --test process --log TRACE --log-file stem.log; fi
   ## If this build was one that produced coverage, upload it.
   - if [[ "$COVERAGE_OPTIONS" != "" ]]; then coveralls -b . --exclude src/test --exclude src/trunnel --gcov-options '\-p' || echo "Coverage failed"; fi
 
@@ -156,11 +228,17 @@ after_failure:
   - if [[ "$SKIP_MAKE_CHECK" == "" ]]; then cat test-suite.log || echo "cat failed"; fi
   ## `make distcheck` puts it somewhere different.
   - if [[ "$DISTCHECK" != "" ]]; then make show-distdir-testlog || echo "make failed"; fi
+  - if [[ "$DISTCHECK" != "" ]]; then make show-distdir-core || echo "make failed"; fi
   - if [[ "$CHUTNEY" != "" ]]; then ls test_network_log || echo "ls failed"; cat test_network_log/* || echo "cat failed"; fi
+  - if [[ "$TEST_STEM" != "" ]]; then tail -1000 "$STEM_SOURCE_DIR"/test/data/tor_log || echo "tail failed"; fi
+  - if [[ "$TEST_STEM" != "" ]]; then grep -v "SocketClosed" stem.log | tail -1000 || echo "grep | tail failed"; fi
 
 before_cache:
   ## Delete all gcov files.
   - if [[ "$COVERAGE_OPTIONS" != "" ]]; then make reset-gcov; fi
+  ## Delete the cargo registry before caching .cargo, because it's cheaper to
+  ## download the registry and throw it away, rather than caching it
+  - rm -rf $HOME/.cargo/registry
 
 notifications:
   irc:
diff --git a/CODE_OF_CONDUCT b/CODE_OF_CONDUCT
new file mode 100644
index 0000000000..d8d160a22a
--- /dev/null
+++ b/CODE_OF_CONDUCT
@@ -0,0 +1,7 @@
+The Tor Project is committed to fostering a inclusive community
+where people feel safe to engage, share their points of view, and
+participate. For the latest version of our Code of Conduct, please
+see
+
+https://gitweb.torproject.org/community/policies.git/plain/code_of_conduct.txt
+
diff --git a/CONTRIBUTING b/CONTRIBUTING
new file mode 100644
index 0000000000..3569f45a88
--- /dev/null
+++ b/CONTRIBUTING
@@ -0,0 +1,39 @@
+Contributing to Tor
+-------------------
+
+### Getting started
+
+Welcome!
+
+We have a bunch of documentation about how to develop Tor in the
+doc/HACKING/ directory.  We recommend that you start with
+doc/HACKING/README.1st.md , and then go from there.  It will tell
+you how to find your way around the source code, how to get
+involved with the Tor community, how to write patches, and much
+more!
+
+You don't have to be a C developer to help with Tor: have a look
+at https://www.torproject.org/getinvolved/volunteer !
+
+The Tor Project is committed to fostering a inclusive community
+where people feel safe to engage, share their points of view, and
+participate. For the latest version of our Code of Conduct, please
+see
+
+https://gitweb.torproject.org/community/policies.git/plain/code_of_conduct.txt
+
+
+
+### License issues
+
+Tor is distributed under the license terms in the LICENSE -- in
+brief, the "3-clause BSD license".  If you send us code to
+distribute with Tor, it needs to be code that we can distribute
+under those terms.  Please don't send us patches unless you agree
+to allow this.
+
+Some compatible licenses include:
+
+  - 3-clause BSD
+  - 2-clause BSD
+  - CC0 Public Domain Dedication
diff --git a/ChangeLog b/ChangeLog
index 7a10a7ce96..cdf7249059 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,9009 @@
-Changes in version 0.2.9.5-rc - 2016-1?-??
+Changes in version 0.3.5.4-alpha - 2018-11-08
+  Tor 0.3.5.4-alpha includes numerous bugfixes on earlier versions and
+  improves our continuous integration support. It continues our attempts
+  to stabilize this alpha branch and build it into a foundation for an
+  acceptable long-term-support release.
+
+  o Major bugfixes (compilation, rust):
+    - Rust tests can now build and run successfully with the
+      --enable-fragile-hardening option enabled. Doing this currently
+      requires the rust beta channel; it will be possible with stable
+      rust once Rust version 1.31 is released. Patch from Alex Crichton.
+      Fixes bugs 27272, 27273, and 27274. Bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (embedding, main loop):
+    - When DisableNetwork becomes set, actually disable periodic events
+      that are already enabled. (Previously, we would refrain from
+      enabling new ones, but we would leave the old ones turned on.)
+      Fixes bug 28348; bugfix on 0.3.4.1-alpha.
+
+  o Minor features (continuous integration):
+    - Add a Travis CI build for --enable-nss on Linux gcc. Closes
+      ticket 27751.
+    - Add new CI job to Travis configuration to run stem-based
+      integration tests. Closes ticket 27913.
+
+  o Minor features (Windows, continuous integration):
+    - Build tor on Windows Server 2012 R2 and Windows Server 2016 using
+      Appveyor's CI. Closes ticket 28318.
+
+  o Minor bugfixes (C correctness, also in 0.3.4.9):
+    - Avoid undefined behavior in an end-of-string check when parsing
+      the BEGIN line in a directory object. Fixes bug 28202; bugfix
+      on 0.2.0.3-alpha.
+
+  o Minor bugfixes (compilation):
+    - Fix a pair of missing headers on OpenBSD. Fixes bug 28303; bugfix
+      on 0.3.5.1-alpha. Patch from Kris Katterjohn.
+
+  o Minor bugfixes (compilation, OpenSolaris):
+    - Fix compilation on OpenSolaris and its descendants by adding a
+      missing include to compat_pthreads.c. Fixes bug 27963; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (configuration):
+    - Refuse to start with relative file paths and RunAsDaemon set
+      (regression from the fix for bug 22731). Fixes bug 28298; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (directory authority, also in 0.3.4.9):
+    - Log additional info when we get a relay that shares an ed25519 ID
+      with a different relay, instead of a BUG() warning with a
+      backtrace. Fixes bug 27800; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion service v3):
+    - Build the service descriptor's signing key certificate before
+      uploading, so we always have a fresh one: leaving no chances for
+      it to expire service side. Fixes bug 27838; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion service v3, client authorization):
+    - Fix an assert() when adding a client authorization for the first
+      time and then sending a HUP signal to the service. Before that,
+      Tor would stop abruptly. Fixes bug 27995; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (onion services):
+    - Unless we have explicitly set HiddenServiceVersion, detect the
+      onion service version and then look for invalid options.
+      Previously, we did the reverse, but that broke existing configs
+      which were pointed to a v2 service and had options like
+      HiddenServiceAuthorizeClient set. Fixes bug 28127; bugfix on
+      0.3.5.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (portability):
+    - Make the OPE code (which is used for v3 onion services) run
+      correctly on big-endian platforms. Fixes bug 28115; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (protover, rust):
+    - Reject extra commas in version strings. Fixes bug 27197; bugfix
+      on 0.3.3.3-alpha.
+
+  o Minor bugfixes (relay shutdown, systemd):
+    - Notify systemd of ShutdownWaitLength so it can be set to longer
+      than systemd's TimeoutStopSec. In Tor's systemd service file, set
+      TimeoutSec to 60 seconds to allow Tor some time to shut down.
+      Fixes bug 28113; bugfix on 0.2.6.2-alpha.
+
+  o Minor bugfixes (rust, also in 0.3.4.9):
+    - Fix a potential null dereference in protover_all_supported(). Add
+      a test for it. Fixes bug 27804; bugfix on 0.3.3.1-alpha.
+    - Return a string that can be safely freed by C code, not one
+      created by the rust allocator, in protover_all_supported(). Fixes
+      bug 27740; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (rust, directory authority, also in 0.3.4.9):
+    - Fix an API mismatch in the rust implementation of
+      protover_compute_vote(). This bug could have caused crashes on any
+      directory authorities running Tor with Rust (which we do not yet
+      recommend). Fixes bug 27741; bugfix on 0.3.3.6.
+
+  o Minor bugfixes (testing):
+    - Avoid hangs and race conditions in test_rebind.py. Fixes bug
+      27968; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (testing, also in 0.3.4.9):
+    - Treat backtrace test failures as expected on BSD-derived systems
+      (NetBSD, OpenBSD, and macOS/Darwin) until we solve bug 17808.
+      (FreeBSD failures have been treated as expected since 18204 in
+      0.2.8.) Fixes bug 27948; bugfix on 0.2.5.2-alpha.
+
+  o Documentation (onion service manpage):
+    - Improve HSv3 client authorization by making some options more
+      explicit and detailed. Closes ticket 28026. Patch by Mike Tigas.
+
+
+Changes in version 0.3.4.9 - 2018-11-02
+  Tor 0.3.4.9 is the second stable release in its series; it backports
+  numerous fixes, including a fix for a bandwidth management bug that
+  was causing memory exhaustion on relays. Anyone running an earlier
+  version of Tor 0.3.4.9 should upgrade.
+
+  o Major bugfixes (compilation, backport from 0.3.5.3-alpha):
+    - Fix compilation on ARM (and other less-used CPUs) when compiling
+      with OpenSSL before 1.1. Fixes bug 27781; bugfix on 0.3.4.1-alpha.
+
+  o Major bugfixes (mainloop, bootstrap, backport from 0.3.5.3-alpha):
+    - Make sure Tor bootstraps and works properly if only the
+      ControlPort is set. Prior to this fix, Tor would only bootstrap
+      when a client port was set (Socks, Trans, NATD, DNS or HTTPTunnel
+      port). Fixes bug 27849; bugfix on 0.3.4.1-alpha.
+
+  o Major bugfixes (relay, backport from 0.3.5.3-alpha):
+    - When our write bandwidth limit is exhausted, stop writing on the
+      connection. Previously, we had a typo in the code that would make
+      us stop reading instead, leading to relay connections being stuck
+      indefinitely and consuming kernel RAM. Fixes bug 28089; bugfix
+      on 0.3.4.1-alpha.
+
+  o Major bugfixes (restart-in-process, backport from 0.3.5.1-alpha):
+    - Fix a use-after-free error that could be caused by passing Tor an
+      impossible set of options that would fail during options_act().
+      Fixes bug 27708; bugfix on 0.3.3.1-alpha.
+
+  o Minor features (continuous integration, backport from 0.3.5.1-alpha):
+    - Don't do a distcheck with --disable-module-dirauth in Travis.
+      Implements ticket 27252.
+    - Only run one online rust build in Travis, to reduce network
+      errors. Skip offline rust builds on Travis for Linux gcc, because
+      they're redundant. Implements ticket 27252.
+    - Skip gcc on OSX in Travis CI, because it's rarely used. Skip a
+      duplicate hardening-off build in Travis on Tor 0.2.9. Skip gcc on
+      Linux with default settings, because all the non-default builds
+      use gcc on Linux. Implements ticket 27252.
+
+  o Minor features (continuous integration, backport from 0.3.5.3-alpha):
+    - Use the Travis Homebrew addon to install packages on macOS during
+      Travis CI. The package list is the same, but the Homebrew addon
+      does not do a `brew update` by default. Implements ticket 27738.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 9 2018 Maxmind GeoLite2
+      Country database. Closes ticket 27991.
+
+  o Minor bugfixes (32-bit OSX and iOS, timing, backport from 0.3.5.2-alpha):
+    - Fix an integer overflow bug in our optimized 32-bit millisecond-
+      difference algorithm for 32-bit Apple platforms. Previously, it
+      would overflow when calculating the difference between two times
+      more than 47 days apart. Fixes part of bug 27139; bugfix
+      on 0.3.4.1-alpha.
+    - Improve the precision of our 32-bit millisecond difference
+      algorithm for 32-bit Apple platforms. Fixes part of bug 27139;
+      bugfix on 0.3.4.1-alpha.
+    - Relax the tolerance on the mainloop/update_time_jumps test when
+      running on 32-bit Apple platforms. Fixes part of bug 27139; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (C correctness, to appear in 0.3.5.4-alpha):
+    - Avoid undefined behavior in an end-of-string check when parsing
+      the BEGIN line in a directory object. Fixes bug 28202; bugfix
+      on 0.2.0.3-alpha.
+
+  o Minor bugfixes (CI, appveyor, to appear in 0.3.5.4-alpha):
+    - Only install the necessary mingw packages during our appveyor
+      builds. This change makes the build a little faster, and prevents
+      a conflict with a preinstalled mingw openssl that appveyor now
+      ships. Fixes bugs 27943 and 27765; bugfix on 0.3.4.2-alpha.
+
+  o Minor bugfixes (code safety, backport from 0.3.5.3-alpha):
+    - Rewrite our assertion macros so that they no longer suppress the
+      compiler's -Wparentheses warnings. Fixes bug 27709; bugfix
+
+  o Minor bugfixes (continuous integration, backport from 0.3.5.1-alpha):
+    - Stop reinstalling identical packages in our Windows CI. Fixes bug
+      27464; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (directory authority, to appear in 0.3.5.4-alpha):
+    - Log additional info when we get a relay that shares an ed25519 ID
+      with a different relay, instead making a BUG() warning. Fixes bug
+      27800; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (directory connection shutdown, backport from 0.3.5.1-alpha):
+    - Avoid a double-close when shutting down a stalled directory
+      connection. Fixes bug 26896; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (HTTP tunnel, backport from 0.3.5.1-alpha):
+    - Fix a bug warning when closing an HTTP tunnel connection due to an
+      HTTP request we couldn't handle. Fixes bug 26470; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (netflow padding, backport from 0.3.5.1-alpha):
+    - Ensure circuitmux queues are empty before scheduling or sending
+      padding. Fixes bug 25505; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.1-alpha):
+    - When the onion service directory can't be created or has the wrong
+      permissions, do not log a stack trace. Fixes bug 27335; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.2-alpha):
+    - Close all SOCKS request (for the same .onion) if the newly fetched
+      descriptor is unusable. Before that, we would close only the first
+      one leaving the other hanging and let to time out by themselves.
+      Fixes bug 27410; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.3-alpha):
+    - When selecting a v3 rendezvous point, don't only look at the
+      protover, but also check whether the curve25519 onion key is
+      present. This way we avoid picking a relay that supports the v3
+      rendezvous but for which we don't have the microdescriptor. Fixes
+      bug 27797; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (protover, backport from 0.3.5.3-alpha):
+    - Reject protocol names containing bytes other than alphanumeric
+      characters and hyphens ([A-Za-z0-9-]). Fixes bug 27316; bugfix
+      on 0.2.9.4-alpha.
+
+  o Minor bugfixes (rust, backport from 0.3.5.1-alpha):
+    - Compute protover votes correctly in the rust version of the
+      protover code. Previously, the protover rewrite in 24031 allowed
+      repeated votes from the same voter for the same protocol version
+      to be counted multiple times in protover_compute_vote(). Fixes bug
+      27649; bugfix on 0.3.3.5-rc.
+    - Reject protover names that contain invalid characters. Fixes bug
+      27687; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (rust, backport from 0.3.5.2-alpha):
+    - protover_all_supported() would attempt to allocate up to 16GB on
+      some inputs, leading to a potential memory DoS. Fixes bug 27206;
+      bugfix on 0.3.3.5-rc.
+
+  o Minor bugfixes (rust, directory authority, to appear in 0.3.5.4-alpha):
+    - Fix an API mismatch in the rust implementation of
+      protover_compute_vote(). This bug could have caused crashes on any
+      directory authorities running Tor with Rust (which we do not yet
+      recommend). Fixes bug 27741; bugfix on 0.3.3.6.
+
+  o Minor bugfixes (rust, to appear in 0.3.5.4-alpha):
+    - Fix a potential null dereference in protover_all_supported(). Add
+      a test for it. Fixes bug 27804; bugfix on 0.3.3.1-alpha.
+    - Return a string that can be safely freed by C code, not one
+      created by the rust allocator, in protover_all_supported(). Fixes
+      bug 27740; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.5.1-alpha):
+    - If a unit test running in a subprocess exits abnormally or with a
+      nonzero status code, treat the test as having failed, even if the
+      test reported success. Without this fix, memory leaks don't cause
+      the tests to fail, even with LeakSanitizer. Fixes bug 27658;
+      bugfix on 0.2.2.4-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.5.3-alpha):
+    - Make the hs_service tests use the same time source when creating
+      the introduction point and when testing it. Now tests work better
+      on very slow systems like ARM or Travis. Fixes bug 27810; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (testing, to appear in 0.3.5.4-alpha):
+    - Treat backtrace test failures as expected on BSD-derived systems
+      (NetBSD, OpenBSD, and macOS/Darwin) until we solve bug 17808.
+      (FreeBSD failures have been treated as expected since 18204 in
+      0.2.8.) Fixes bug 27948; bugfix on 0.2.5.2-alpha.
+
+
+Changes in version 0.3.5.3-alpha - 2018-10-17
+  Tor 0.3.5.3-alpha fixes several bugs, mostly from previous 0.3.5.x
+  versions. One important fix for relays addresses a problem with rate-
+  limiting code from back in 0.3.4.x: If the fix works out, we'll be
+  backporting it soon. This release is still an alpha, but we hope it's
+  getting closer and closer to stability.
+
+  o Major features (onion services):
+    - Version 3 onion services can now use the per-service
+      HiddenServiceExportCircuitID option to differentiate client
+      circuits. It communicates with the service by using the HAProxy
+      protocol to assign virtual IP addresses to inbound client
+      circuits. Closes ticket 4700. Patch by Mahrud Sayrafi.
+
+  o Major bugfixes (compilation):
+    - Fix compilation on ARM (and other less-used CPUs) when compiling
+      with OpenSSL before 1.1. Fixes bug 27781; bugfix on 0.3.4.1-alpha.
+
+  o Major bugfixes (initialization, crash):
+    - Fix an assertion crash that would stop Tor from starting up if it
+      tried to activate a periodic event too early. Fixes bug 27861;
+      bugfix on 0.3.5.1-alpha.
+
+  o Major bugfixes (mainloop, bootstrap):
+    - Make sure Tor bootstraps and works properly if only the
+      ControlPort is set. Prior to this fix, Tor would only bootstrap
+      when a client port was set (Socks, Trans, NATD, DNS or HTTPTunnel
+      port). Fixes bug 27849; bugfix on 0.3.4.1-alpha.
+
+  o Major bugfixes (relay):
+    - When our write bandwidth limit is exhausted, stop writing on the
+      connection. Previously, we had a typo in the code that would make
+      us stop reading instead, leading to relay connections being stuck
+      indefinitely and consuming kernel RAM. Fixes bug 28089; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor features (continuous integration):
+    - Use the Travis Homebrew addon to install packages on macOS during
+      Travis CI. The package list is the same, but the Homebrew addon
+      does not do a `brew update` by default. Implements ticket 27738.
+    - Report what program produced the mysterious core file that we
+      occasionally see on Travis CI during make distcheck. Closes
+      ticket 28024.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 9 2018 Maxmind GeoLite2
+      Country database. Closes ticket 27991.
+
+  o Minor bugfixes (code safety):
+    - Rewrite our assertion macros so that they no longer suppress the
+      compiler's -Wparentheses warnings. Fixes bug 27709; bugfix
+      on 0.0.6.
+
+  o Minor bugfixes (compilation):
+    - Compile the ed25519-donna code with a correct declaration of
+      crypto_strongest_rand(). Previously, we built it with one type,
+      but linked it against another in the unit tests, which caused
+      compilation failures with LTO enabled. This could have caused
+      other undefined behavior in the tests. Fixes bug 27728; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (compilation, netbsd):
+    - Add a missing include back into procmon.c. Fixes bug 27990; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (continuous integration, appveyor):
+    - Install only the necessary mingw packages during our appveyor
+      builds. This change makes the build a little faster, and prevents
+      a conflict with a preinstalled mingw openssl that appveyor now
+      ships. Fixes bugs 27765 and 27943; bugfix on 0.3.4.2-alpha.
+
+  o Minor bugfixes (directory permissions):
+    - When a user requests a group-readable DataDirectory, give it to
+      them. Previously, when the DataDirectory and the CacheDirectory
+      were the same, the default setting (0) for
+      CacheDirectoryGroupReadable would override the setting for
+      DataDirectoryGroupReadable. Fixes bug 26913; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (memory leaks):
+    - Fix a small memory leak when calling Tor with --dump-config. Fixes
+      bug 27893; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (networking):
+    - In retry_listeners_ports(), make sure that we're removing a member
+      of old_conns smartlist at most once. Fixes bug 27808; bugfix
+      on 0.3.5.1-alpha.
+    - Refrain from attempting socket rebinding when old and new
+      listeners are in different address families. Fixes bug 27928;
+      bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (onion service v3):
+    - Stop dumping a stack trace when trying to connect to an intro
+      point without having a descriptor for it. Fixes bug 27774; bugfix
+      on 0.3.2.1-alpha.
+    - Don't warn so loudly when Tor is unable to decode an onion
+      descriptor. This can now happen as a normal use case if a client
+      gets a descriptor with client authorization but the client is not
+      authorized. Fixes bug 27550; bugfix on 0.3.5.1-alpha.
+    - When selecting a v3 rendezvous point, don't only look at the
+      protover, but also check whether the curve25519 onion key is
+      present. This way we avoid picking a relay that supports the v3
+      rendezvous but for which we don't have the microdescriptor. Fixes
+      bug 27797; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (protover):
+    - Reject protocol names containing bytes other than alphanumeric
+      characters and hyphens ([A-Za-z0-9-]). Fixes bug 27316; bugfix
+      on 0.2.9.4-alpha.
+
+  o Minor bugfixes (testing):
+    - Make the hs_service tests use the same time source when creating
+      the introduction point and when testing it. Now tests work better
+      on very slow systems like ARM or Travis. Fixes bug 27810; bugfix
+      on 0.3.2.1-alpha.
+    - In test_rebind.py, check if the Python version is in the supported
+      range. Fixes bug 27675; bugfix on 0.3.5.1-alpha.
+
+  o Code simplification and refactoring:
+    - Divide more large Tor source files -- especially ones that span
+      multiple areas of functionality -- into smaller parts, including
+      onion.c and main.c. Closes ticket 26747.
+    - Divide the "routerparse.c" module into separate modules for each
+      group of parsed objects. Closes ticket 27924.
+    - Move protover_rust.c to the same place protover.c was moved to.
+      Closes ticket 27814.
+    - Split directory.c into separate pieces for client, server, and
+      common functionality. Closes ticket 26744.
+    - Split the non-statistics-related parts from the rephist.c and
+      geoip.c modules. Closes ticket 27892.
+    - Split the router.c file into relay-only and shared components, to
+      help with future modularization. Closes ticket 27864.
+
+  o Documentation:
+    - In the tor-resolve(1) manpage, fix the reference to socks-
+      extensions.txt by adding a web URL. Resolves ticket 27853.
+    - Mention that we require Python to be 2.7 or newer for some
+      integration tests that we ship with Tor. Resolves ticket 27677.
+
+
+Changes in version 0.3.5.2-alpha - 2018-09-21
+  Tor 0.3.5.2-alpha fixes several bugs in 0.3.5.1-alpha, including one
+  that made Tor think it had run out of sockets. Anybody running a relay
+  or an onion service on 0.3.5.1-alpha should upgrade.
+
+  o Major bugfixes (relay bandwidth statistics):
+    - When we close relayed circuits, report the data in the circuit
+      queues as being written in our relay bandwidth stats. This
+      mitigates guard discovery and other attacks that close circuits
+      for the explicit purpose of noticing this discrepancy in
+      statistics. Fixes bug 23512; bugfix on 0.0.8pre3.
+
+  o Major bugfixes (socket accounting):
+    - In our socket accounting code, count a socket as closed even when
+      it is closed indirectly by the TLS layer. Previously, we would
+      count these sockets as still in use, and incorrectly believe that
+      we had run out of sockets. Fixes bug 27795; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (32-bit OSX and iOS, timing):
+    - Fix an integer overflow bug in our optimized 32-bit millisecond-
+      difference algorithm for 32-bit Apple platforms. Previously, it
+      would overflow when calculating the difference between two times
+      more than 47 days apart. Fixes part of bug 27139; bugfix
+      on 0.3.4.1-alpha.
+    - Improve the precision of our 32-bit millisecond difference
+      algorithm for 32-bit Apple platforms. Fixes part of bug 27139;
+      bugfix on 0.3.4.1-alpha.
+    - Relax the tolerance on the mainloop/update_time_jumps test when
+      running on 32-bit Apple platforms. Fixes part of bug 27139; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (onion service v3):
+    - Close all SOCKS request (for the same .onion) if the newly fetched
+      descriptor is unusable. Before that, we would close only the first
+      one leaving the other hanging and let to time out by themselves.
+      Fixes bug 27410; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (memory leak):
+    - Fix an unlikely memory leak when trying to read a private key from
+      a ridiculously large file. Fixes bug 27764; bugfix on
+      0.3.5.1-alpha. This is CID 1439488.
+
+  o Minor bugfixes (NSS):
+    - Correctly detect failure to open a dummy TCP socket when stealing
+      ownership of an fd from the NSS layer. Fixes bug 27782; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (rust):
+    - protover_all_supported() would attempt to allocate up to 16GB on
+      some inputs, leading to a potential memory DoS. Fixes bug 27206;
+      bugfix on 0.3.3.5-rc.
+
+  o Minor bugfixes (testing):
+    - Revise the "conditionvar_timeout" test so that it succeeds even on
+      heavily loaded systems where the test threads are not scheduled
+      within 200 msec. Fixes bug 27073; bugfix on 0.2.6.3-alpha.
+
+  o Code simplification and refactoring:
+    - Divide the routerlist.c and dirserv.c modules into smaller parts.
+      Closes ticket 27799.
+
+
+Changes in version 0.3.5.1-alpha - 2018-09-18
+  Tor 0.3.5.1-alpha is the first release of the 0.3.5.x series. It adds
+  client authorization for modern (v3) onion services, improves
+  bootstrap reporting, begins reorganizing Tor's codebase, adds optional
+  support for NSS in place of OpenSSL, and much more.
+
+  o Major features (onion services, UI change):
+    - For a newly created onion service, the default version is now 3.
+      Tor still supports existing version 2 services, but the operator
+      now needs to set "HiddenServiceVersion 2" in order to create a new
+      version 2 service. For existing services, Tor now learns the
+      version by reading the key file. Closes ticket 27215.
+
+  o Major features (relay, UI change):
+    - Relays no longer run as exits by default. If the "ExitRelay"
+      option is auto (or unset), and no exit policy is specified with
+      ExitPolicy or ReducedExitPolicy, we now treat ExitRelay as 0.
+      Previously in this case, we allowed exit traffic and logged a
+      warning message. Closes ticket 21530. Patch by Neel Chauhan.
+    - Tor now validates that the ContactInfo config option is valid UTF-
+      8 when parsing torrc. Closes ticket 27428.
+
+  o Major features (bootstrap):
+    - Don't report directory progress until after a connection to a
+      relay or bridge has succeeded. Previously, we'd report 80%
+      progress based on cached directory information when we couldn't
+      even connect to the network. Closes ticket 27169.
+
+  o Major features (new code layout):
+    - Nearly all of Tor's source code has been moved around into more
+      logical places. The "common" directory is now divided into a set
+      of libraries in "lib", and files in the "or" directory have been
+      split into "core" (logic absolutely needed for onion routing),
+      "feature" (independent modules in Tor), and "app" (to configure
+      and invoke the rest of Tor). See doc/HACKING/CodeStructure.md for
+      more information. Closes ticket 26481.
+
+      This refactoring is not complete: although the libraries have been
+      refactored to be acyclic, the main body of Tor is still too
+      interconnected. We will attempt to improve this in the future.
+
+  o Major features (onion services v3):
+    - Implement onion service client authorization at the descriptor
+      level: only authorized clients can decrypt a service's descriptor
+      to find out how to contact it. A new torrc option was added to
+      control this client side: ClientOnionAuthDir <path>. On the
+      service side, if the "authorized_clients/" directory exists in the
+      onion service directory path, client configurations are read from
+      the files within. See the manpage for more details. Closes ticket
+      27547. Patch done by Suphanat Chunhapanya (haxxpop).
+    - Improve revision counter generation in next-gen onion services.
+      Onion services can now scale by hosting multiple instances on
+      different hosts without synchronization between them, which was
+      previously impossible because descriptors would get rejected by
+      HSDirs. Addresses ticket 25552.
+
+  o Major features (portability, cryptography, experimental, TLS):
+    - Tor now has the option to compile with the NSS library instead of
+      OpenSSL. This feature is experimental, and we expect that bugs may
+      remain. It is mainly intended for environments where Tor's
+      performance is not CPU-bound, and where NSS is already known to be
+      installed. To try it out, configure Tor with the --enable-nss
+      flag. Closes tickets 26631, 26815, and 26816.
+
+      If you are experimenting with this option and using an old cached
+      consensus, Tor may fail to start. To solve this, delete your
+      "cached-consensus" and "cached-microdesc-consensus" files,
+      (if present), and restart Tor.
+
+  o Major bugfixes (directory authority):
+    - Actually check that the address we get from DirAuthority
+      configuration line is valid IPv4. Explicitly disallow DirAuthority
+      address to be a DNS hostname. Fixes bug 26488; bugfix
+      on 0.1.2.10-rc.
+
+  o Major bugfixes (restart-in-process):
+    - Fix a use-after-free error that could be caused by passing Tor an
+      impossible set of options that would fail during options_act().
+      Fixes bug 27708; bugfix on 0.3.3.1-alpha.
+
+  o Minor features (admin tools):
+    - Add a new --key-expiration option to print the expiration date of
+      the signing cert in an ed25519_signing_cert file. Resolves
+      issue 19506.
+
+  o Minor features (build):
+    - If you pass the "--enable-pic" option to configure, Tor will try
+      to tell the compiler to build position-independent code suitable
+      to link into a dynamic library. (The default remains -fPIE, for
+      code suitable for a relocatable executable.) Closes ticket 23846.
+
+  o Minor features (code correctness, testing):
+    - Tor's build process now includes a "check-includes" make target to
+      verify that no module of Tor relies on any headers from a higher-
+      level module. We hope to use this feature over time to help
+      refactor our codebase. Closes ticket 26447.
+
+  o Minor features (code layout):
+    - We have a new "lowest-level" error-handling API for use by code
+      invoked from within the logging module. With this interface, the
+      logging code is no longer at risk of calling into itself if a
+      failure occurs while it is trying to log something. Closes
+      ticket 26427.
+
+  o Minor features (compilation):
+    - Tor's configure script now supports a --with-malloc= option to
+      select your malloc implementation. Supported options are
+      "tcmalloc", "jemalloc", "openbsd" (deprecated), and "system" (the
+      default). Addresses part of ticket 20424. Based on a patch from
+      Alex Xu.
+
+  o Minor features (config):
+    - The "auto" keyword in torrc is now case-insensitive. Closes
+      ticket 26663.
+
+  o Minor features (continuous integration):
+    - Don't do a distcheck with --disable-module-dirauth in Travis.
+      Implements ticket 27252.
+    - Install libcap-dev and libseccomp2-dev so these optional
+      dependencies get tested on Travis CI. Closes ticket 26560.
+    - Only run one online rust build in Travis, to reduce network
+      errors. Skip offline rust builds on Travis for Linux gcc, because
+      they're redundant. Implements ticket 27252.
+    - Skip gcc on OSX in Travis CI, because it's rarely used. Skip a
+      duplicate hardening-off build in Travis on Tor 0.2.9. Skip gcc on
+      Linux with default settings, because all the non-default builds
+      use gcc on Linux. Implements ticket 27252.
+
+  o Minor features (controller):
+    - Emit CIRC_BW events as soon as we detect that we processed an
+      invalid or otherwise dropped cell on a circuit. This allows
+      vanguards and other controllers to react more quickly to dropped
+      cells. Closes ticket 27678.
+    - For purposes of CIRC_BW-based dropped cell detection, track half-
+      closed stream ids, and allow their ENDs, SENDMEs, DATA and path
+      bias check cells to arrive without counting it as dropped until
+      either the END arrives, or the windows are empty. Closes
+      ticket 25573.
+    - Implement a 'GETINFO md/all' controller command to enable getting
+      all known microdescriptors. Closes ticket 8323.
+    - The GETINFO command now support an "uptime" argument, to return
+      Tor's uptime in seconds. Closes ticket 25132.
+
+  o Minor features (denial-of-service avoidance):
+    - Make our OOM handler aware of the DNS cache so that it doesn't
+      fill up the memory. This check is important for our DoS mitigation
+      subsystem. Closes ticket 18642. Patch by Neel Chauhan.
+
+  o Minor features (development):
+    - Tor's makefile now supports running the "clippy" Rust style tool
+      on our Rust code. Closes ticket 22156.
+
+  o Minor features (directory authority):
+    - There is no longer an artificial upper limit on the length of
+      bandwidth lines. Closes ticket 26223.
+    - When a bandwidth file is used to obtain the bandwidth measurements,
+      include this bandwidth file headers in the votes. Closes
+      ticket 3723.
+    - Improved support for networks with only a single authority or a
+      single fallback directory. Patch from Gabriel Somlo. Closes
+      ticket 25928.
+
+  o Minor features (embedding API):
+    - The Tor controller API now supports a function to launch Tor with
+      a preconstructed owning controller FD, so that embedding
+      applications don't need to manage controller ports and
+      authentication. Closes ticket 24204.
+    - The Tor controller API now has a function that returns the name
+      and version of the backend implementing the API. Closes
+      ticket 26947.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the September 6 2018 Maxmind GeoLite2
+      Country database. Closes ticket 27631.
+
+  o Minor features (memory management):
+    - Get Libevent to use the same memory allocator as Tor, by calling
+      event_set_mem_functions() during initialization. Resolves
+      ticket 8415.
+
+  o Minor features (memory usage):
+    - When not using them, store legacy TAP public onion keys in DER-
+      encoded format, rather than as expanded public keys. This should
+      save several megabytes on typical clients. Closes ticket 27246.
+
+  o Minor features (OpenSSL):
+    - When possible, use RFC5869 HKDF implementation from OpenSSL rather
+      than our own. Resolves ticket 19979.
+
+  o Minor features (Rust, code quality):
+    - Improve rust code quality in the rust protover implementation by
+      making it more idiomatic. Includes changing an internal API to
+      take &str instead of &String. Closes ticket 26492.
+
+  o Minor features (testing):
+    - Add scripts/test/chutney-git-bisect.sh, for bisecting using
+      chutney. Implements ticket 27211.
+
+  o Minor features (tor-resolve):
+    - The tor-resolve utility can now be used with IPv6 SOCKS proxies.
+      Side-effect of the refactoring for ticket 26526.
+
+  o Minor features (UI):
+    - Log each included configuration file or directory as we read it,
+      to provide more visibility about where Tor is reading from. Patch
+      from Unto Sten; closes ticket 27186.
+    - Lower log level of "Scheduler type KIST has been enabled" to INFO.
+      Closes ticket 26703.
+
+  o Minor bugfixes (bootstrap):
+    - Try harder to get descriptors in non-exit test networks, by using
+      the mid weight for the third hop when there are no exits. Fixes
+      bug 27237; bugfix on 0.2.6.2-alpha.
+
+  o Minor bugfixes (C correctness):
+    - Avoid casting smartlist index to int implicitly, as it may trigger
+      a warning (-Wshorten-64-to-32). Fixes bug 26282; bugfix on
+      0.2.3.13-alpha, 0.2.7.1-alpha and 0.2.1.1-alpha.
+    - Use time_t for all values in
+      predicted_ports_prediction_time_remaining(). Rework the code that
+      computes difference between durations/timestamps. Fixes bug 27165;
+      bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (client, memory usage):
+    - When not running as a directory cache, there is no need to store
+      the text of the current consensus networkstatus in RAM.
+      Previously, however, clients would store it anyway, at a cost of
+      over 5 MB. Now, they do not. Fixes bug 27247; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (client, reachableaddresses):
+    - Instead of adding a "reject *:*" line to ReachableAddresses when
+      loading the configuration, add one to the policy after parsing it
+      in parse_reachable_addresses(). This prevents extra "reject *.*"
+      lines from accumulating on reloads. Fixes bug 20874; bugfix on
+      0.1.1.5-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (code quality):
+    - Rename sandbox_getaddrinfo() and other functions to no longer
+      misleadingly suggest that they are sandbox-only. Fixes bug 26525;
+      bugfix on 0.2.7.1-alpha.
+
+  o Minor bugfixes (configuration, Onion Services):
+    - In rend_service_parse_port_config(), disallow any input to remain
+      after address-port pair was parsed. This will catch address and
+      port being whitespace-separated by mistake of the user. Fixes bug
+      27044; bugfix on 0.2.9.10.
+
+  o Minor bugfixes (continuous integration):
+    - Stop reinstalling identical packages in our Windows CI. Fixes bug
+      27464; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (controller):
+    - Consider all routerinfo errors other than "not a server" to be
+      transient for the purpose of "GETINFO exit-policy/*" controller
+      request. Print stacktrace in the unlikely case of failing to
+      recompute routerinfo digest. Fixes bug 27034; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (directory connection shutdown):
+    - Avoid a double-close when shutting down a stalled directory
+      connection. Fixes bug 26896; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (HTTP tunnel):
+    - Fix a bug warning when closing an HTTP tunnel connection due to an
+      HTTP request we couldn't handle. Fixes bug 26470; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (ipv6):
+    - In addrs_in_same_network_family(), we choose the subnet size based
+      on the IP version (IPv4 or IPv6). Previously, we chose a fixed
+      subnet size of /16 for both IPv4 and IPv6 addresses. Fixes bug
+      15518; bugfix on 0.2.3.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (logging):
+    - As a precaution, do an early return from log_addr_has_changed() if
+      Tor is running as client. Also, log a stack trace for debugging as
+      this function should only be called when Tor runs as server. Fixes
+      bug 26892; bugfix on 0.1.1.9-alpha.
+    - Refrain from mentioning bug 21018 in the logs, as it is already
+      fixed. Fixes bug 25477; bugfix on 0.2.9.8.
+
+  o Minor bugfixes (logging, documentation):
+    - When SafeLogging is enabled, scrub IP address in
+      channel_tls_process_netinfo_cell(). Also, add a note to manpage
+      that scrubbing is not guaranteed on loglevels below Notice. Fixes
+      bug 26882; bugfix on 0.2.4.10-alpha.
+
+  o Minor bugfixes (netflow padding):
+    - Ensure circuitmux queues are empty before scheduling or sending
+      padding. Fixes bug 25505; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (onion service v2):
+    - Log at level "info", not "warning", in the case that we do not
+      have a consensus when a .onion request comes in. This can happen
+      normally while bootstrapping. Fixes bug 27040; bugfix
+      on 0.2.8.2-alpha.
+
+  o Minor bugfixes (onion service v3):
+    - When the onion service directory can't be created or has the wrong
+      permissions, do not log a stack trace. Fixes bug 27335; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (OS compatibility):
+    - Properly handle configuration changes that move a listener to/from
+      wildcard IP address. If the first attempt to bind a socket fails,
+      close the old listener and try binding the socket again. Fixes bug
+      17873; bugfix on 0.0.8pre-1.
+
+  o Minor bugfixes (performance)::
+    - Rework node_is_a_configured_bridge() to no longer call
+      node_get_all_orports(), which was performing too many memory
+      allocations. Fixes bug 27224; bugfix on 0.2.3.9.
+
+  o Minor bugfixes (relay statistics):
+    - Update relay descriptor on bandwidth changes only when the uptime
+      is smaller than 24h, in order to reduce the efficiency of guard
+      discovery attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha.
+
+  o Minor bugfixes (relays):
+    - Consider the fact that we'll be making direct connections to our
+      entry and guard nodes when computing the fraction of nodes that
+      have their descriptors. Also, if we are using bridges and there is
+      at least one bridge with a full descriptor, treat the fraction of
+      guards available as 100%. Fixes bug 25886; bugfix on 0.2.4.10-alpha.
+      Patch by Neel Chauhan.
+    - Update the message logged on relays when DirCache is disabled.
+      Since 0.3.3.5-rc, authorities require DirCache (V2Dir) for the
+      Guard flag. Fixes bug 24312; bugfix on 0.3.3.5-rc.
+
+  o Minor bugfixes (rust, protover):
+    - Compute protover votes correctly in the rust version of the
+      protover code. Previously, the protover rewrite in 24031 allowed
+      repeated votes from the same voter for the same protocol version
+      to be counted multiple times in protover_compute_vote(). Fixes bug
+      27649; bugfix on 0.3.3.5-rc.
+    - Reject protover names that contain invalid characters. Fixes bug
+      27687; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Fix two unit tests to work when HOME environment variable is not
+      set. Fixes bug 27096; bugfix on 0.2.8.1-alpha.
+    - If a unit test running in a subprocess exits abnormally or with a
+      nonzero status code, treat the test as having failed, even if the
+      test reported success. Without this fix, memory leaks don't cause
+      the tests to fail, even with LeakSanitizer. Fixes bug 27658;
+      bugfix on 0.2.2.4-alpha.
+    - When logging a version mismatch in our openssl_version tests,
+      report the actual offending version strings. Fixes bug 26152;
+      bugfix on 0.2.9.1-alpha.
+    - Fix forking tests on Windows when there is a space somewhere in
+      the path. Fixes bug 26437; bugfix on 0.2.2.4-alpha.
+
+  o Code simplification and refactoring:
+    - 'updateFallbackDirs.py' now ignores the blacklist file, as it's not
+      longer needed. Closes ticket 26502.
+    - Include paths to header files within Tor are now qualified by
+      directory within the top-level src directory.
+    - Many structures have been removed from the centralized "or.h"
+      header, and moved into their own headers. This will allow us to
+      reduce the number of places in the code that rely on each
+      structure's contents and layout. Closes ticket 26383.
+    - Remove ATTR_NONNULL macro from codebase. Resolves ticket 26527.
+    - Remove GetAdaptersAddresses_fn_t. The code that used it was
+      removed as part of the 26481 refactor. Closes ticket 27467.
+    - Rework Tor SOCKS server code to use Trunnel and benefit from
+      autogenerated functions for parsing and generating SOCKS wire
+      format. New implementation is cleaner, more maintainable and
+      should be less prone to heartbleed-style vulnerabilities.
+      Implements a significant fraction of ticket 3569.
+    - Split sampled_guards_update_from_consensus() and
+      select_entry_guard_for_circuit() into subfunctions. In
+      entry_guards_update_primary() unite three smartlist enumerations
+      into one and move smartlist comparison code out of the function.
+      Closes ticket 21349.
+    - Tor now assumes that you have standards-conformant stdint.h and
+      inttypes.h headers when compiling. Closes ticket 26626.
+    - Unify our bloom filter logic. Previously we had two copies of this
+      code: one for routerlist filtering, and one for address set
+      calculations. Closes ticket 26510.
+    - Use the simpler strcmpstart() helper in
+      rend_parse_v2_service_descriptor instead of strncmp(). Closes
+      ticket 27630.
+    - Utility functions that can perform a DNS lookup are now wholly
+      separated from those that can't, in separate headers and C
+      modules. Closes ticket 26526.
+
+  o Documentation:
+    - Copy paragraph and URL to Tor's code of conduct document from
+      CONTRIBUTING to new CODE_OF_CONDUCT file. Resolves ticket 26638.
+    - Remove old instructions from INSTALL document. Closes ticket 26588.
+    - Warn users that they should not include MyFamily line(s) in their
+      torrc when running Tor bridge. Closes ticket 26908.
+
+  o Removed features:
+    - Tor no longer supports building with the dmalloc library. For
+      debugging memory issues, we suggest using gperftools or msan
+      instead. Closes ticket 26426.
+    - Tor no longer attempts to run on Windows environments without the
+      GetAdaptersAddresses() function. This function has existed since
+      Windows XP, which is itself already older than we support.
+    - Remove Tor2web functionality for version 2 onion services. The
+      Tor2webMode and Tor2webRendezvousPoints options are now obsolete.
+      (This feature was never shipped in vanilla Tor and it was only
+      possible to use this feature by building the support at compile
+      time. Tor2webMode is not implemented for version 3 onion services.)
+      Closes ticket 26367.
+
+
+Changes in version 0.2.9.17 - 2018-09-10
+  Tor 0.2.9.17 backports numerous bugfixes from later versions of Tor.
+
+  o Minor features (compatibility, backport from 0.3.4.8):
+    - Tell OpenSSL to maintain backward compatibility with previous
+      RSA1024/DH1024 users in Tor. With OpenSSL 1.1.1-pre6, these
+      ciphers are disabled by default. Closes ticket 27344.
+
+  o Minor features (continuous integration, backport from 0.3.4.7-rc):
+    - Enable macOS builds in our Travis CI configuration. Closes
+      ticket 24629.
+    - Install libcap-dev and libseccomp2-dev so these optional
+      dependencies get tested on Travis CI. Closes ticket 26560.
+    - Run asciidoc during Travis CI. Implements ticket 27087.
+    - Use ccache in our Travis CI configuration. Closes ticket 26952.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the August 7 2018 Maxmind GeoLite2
+      Country database. Closes ticket 27089.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.6-rc):
+    - When compiling with --enable-openbsd-malloc or --enable-tcmalloc,
+      tell the compiler not to include the system malloc implementation.
+      Fixes bug 20424; bugfix on 0.2.0.20-rc.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.7-rc):
+    - Silence a spurious compiler warning on the GetAdaptersAddresses
+      function pointer cast. This issue is already fixed by 26481 in
+      0.3.5 and later, by removing the lookup and cast. Fixes bug 27465;
+      bugfix on 0.2.3.11-alpha.
+    - Stop calling SetProcessDEPPolicy() on 64-bit Windows. It is not
+      supported, and always fails. Some compilers warn about the
+      function pointer cast on 64-bit Windows. Fixes bug 27461; bugfix
+      on 0.2.2.23-alpha.
+
+  o Minor bugfixes (compilation, windows, backport from 0.3.4.7-rc):
+    - Don't link or search for pthreads when building for Windows, even
+      if we are using build environment (like mingw) that provides a
+      pthreads library. Fixes bug 27081; bugfix on 0.1.0.1-rc.
+
+  o Minor bugfixes (continuous integration, backport from 0.3.4.6-rc):
+    - Skip a pair of unreliable key generation tests on Windows, until
+      the underlying issue in bug 26076 is resolved. Fixes bug 26830 and
+      bug 26853; bugfix on 0.2.7.3-rc and 0.3.2.1-alpha respectively.
+
+  o Minor bugfixes (continuous integration, backport from 0.3.4.7-rc):
+    - Pass the module flags to distcheck configure, and log the flags
+      before running configure. (Backported to 0.2.9 and later as a
+      precaution.) Fixes bug 27088; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (continuous integration, backport from 0.3.4.8):
+    - When a Travis build fails, and showing a log fails, keep trying to
+      show the other logs. Fixes bug 27453; bugfix on 0.3.4.7-rc.
+    - When we use echo in Travis, don't pass a --flag as the first
+      argument. Fixes bug 27418; bugfix on 0.3.4.7-rc.
+
+  o Minor bugfixes (directory authority, backport from 0.3.4.6-rc):
+    - When voting for recommended versions, make sure that all of the
+      versions are well-formed and parsable. Fixes bug 26485; bugfix
+      on 0.1.1.6-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.4.7-rc):
+    - Fix a bug in out sandboxing rules for the openat() syscall.
+      Previously, no openat() call would be permitted, which would break
+      filesystem operations on recent glibc versions. Fixes bug 25440;
+      bugfix on 0.2.9.15. Diagnosis and patch from Daniel Pinto.
+
+  o Minor bugfixes (onion services, backport from 0.3.4.8):
+    - Silence a spurious compiler warning in
+      rend_client_send_introduction(). Fixes bug 27463; bugfix
+      on 0.1.1.2-alpha.
+
+  o Minor bugfixes (single onion services, Tor2web, backport from 0.3.4.6-rc):
+    - Log a protocol warning when single onion services or Tor2web clients
+      fail to authenticate direct connections to relays.
+      Fixes bug 26924; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.4.6-rc):
+    - Disable core dumps in test_bt.sh, to avoid failures in "make
+      distcheck". Fixes bug 26787; bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (testing, chutney, backport from 0.3.4.8):
+    - Before running make test-network-all, delete old logs and test
+      result files, to avoid spurious failures. Fixes bug 27295; bugfix
+      on 0.2.7.3-rc.
+
+  o Minor bugfixes (testing, openssl compatibility, backport from 0.3.4.7-rc):
+    - Our "tortls/cert_matches_key" unit test no longer relies on
+      OpenSSL internals. Previously, it relied on unsupported OpenSSL
+      behavior in a way that caused it to crash with OpenSSL 1.0.2p.
+      Fixes bug 27226; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (Windows, compilation, backport from 0.3.4.7-rc):
+    - Silence a compilation warning on MSVC 2017 and clang-cl. Fixes bug
+      27185; bugfix on 0.2.2.2-alpha.
+
+
+Changes in version 0.3.2.12 - 2018-09-10
+  Tor 0.3.2.12 backport numerous fixes from later versions of Tor.
+
+  o Minor features (compatibility, backport from 0.3.4.8):
+    - Tell OpenSSL to maintain backward compatibility with previous
+      RSA1024/DH1024 users in Tor. With OpenSSL 1.1.1-pre6, these
+      ciphers are disabled by default. Closes ticket 27344.
+
+  o Minor features (continuous integration, backport from 0.3.4.7-rc):
+    - Enable macOS builds in our Travis CI configuration. Closes
+      ticket 24629.
+    - Install libcap-dev and libseccomp2-dev so these optional
+      dependencies get tested on Travis CI. Closes ticket 26560.
+    - Run asciidoc during Travis CI. Implements ticket 27087.
+    - Use ccache in our Travis CI configuration. Closes ticket 26952.
+
+  o Minor features (continuous integration, rust, backport from 0.3.4.7-rc):
+    - Use cargo cache in our Travis CI configuration. Closes
+      ticket 26952.
+
+  o Minor features (controller, backport from 0.3.4.6-rc):
+    - The control port now exposes the list of HTTPTunnelPorts and
+      ExtOrPorts via GETINFO net/listeners/httptunnel and
+      net/listeners/extor respectively. Closes ticket 26647.
+
+  o Minor features (directory authorities, backport from 0.3.4.7-rc):
+    - Authorities no longer vote to make the subprotocol version
+      "LinkAuth=1" a requirement: it is unsupportable with NSS, and
+      hasn't been needed since Tor 0.3.0.1-alpha. Closes ticket 27286.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the August 7 2018 Maxmind GeoLite2
+      Country database. Closes ticket 27089.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.6-rc):
+    - When compiling with --enable-openbsd-malloc or --enable-tcmalloc,
+      tell the compiler not to include the system malloc implementation.
+      Fixes bug 20424; bugfix on 0.2.0.20-rc.
+    - Don't try to use a pragma to temporarily disable the
+      -Wunused-const-variable warning if the compiler doesn't support
+      it. Fixes bug 26785; bugfix on 0.3.2.11.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.7-rc):
+    - Silence a spurious compiler warning on the GetAdaptersAddresses
+      function pointer cast. This issue is already fixed by 26481 in
+      0.3.5 and later, by removing the lookup and cast. Fixes bug 27465;
+      bugfix on 0.2.3.11-alpha.
+    - Stop calling SetProcessDEPPolicy() on 64-bit Windows. It is not
+      supported, and always fails. Some compilers warn about the
+      function pointer cast on 64-bit Windows. Fixes bug 27461; bugfix
+      on 0.2.2.23-alpha.
+
+  o Minor bugfixes (compilation, windows, backport from 0.3.4.7-rc):
+    - Don't link or search for pthreads when building for Windows, even
+      if we are using build environment (like mingw) that provides a
+      pthreads library. Fixes bug 27081; bugfix on 0.1.0.1-rc.
+
+  o Minor bugfixes (continuous integration, backport from 0.3.4.6-rc):
+    - Skip a pair of unreliable key generation tests on Windows, until
+      the underlying issue in bug 26076 is resolved. Fixes bug 26830 and
+      bug 26853; bugfix on 0.2.7.3-rc and 0.3.2.1-alpha respectively.
+
+  o Minor bugfixes (continuous integration, backport from 0.3.4.7-rc):
+    - Build with zstd on macOS. Fixes bug 27090; bugfix on 0.3.1.5-alpha.
+    - Pass the module flags to distcheck configure, and log the flags
+      before running configure. (Backported to 0.2.9 and later as a
+      precaution.) Fixes bug 27088; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (continuous integration, backport from 0.3.4.8):
+    - When a Travis build fails, and showing a log fails, keep trying to
+      show the other logs. Fixes bug 27453; bugfix on 0.3.4.7-rc.
+    - When we use echo in Travis, don't pass a --flag as the first
+      argument. Fixes bug 27418; bugfix on 0.3.4.7-rc.
+
+  o Minor bugfixes (directory authority, backport from 0.3.4.6-rc):
+    - When voting for recommended versions, make sure that all of the
+      versions are well-formed and parsable. Fixes bug 26485; bugfix
+      on 0.1.1.6-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.4.7-rc):
+    - Fix a bug in out sandboxing rules for the openat() syscall.
+      Previously, no openat() call would be permitted, which would break
+      filesystem operations on recent glibc versions. Fixes bug 25440;
+      bugfix on 0.2.9.15. Diagnosis and patch from Daniel Pinto.
+
+  o Minor bugfixes (logging, backport from 0.3.4.6-rc):
+    - Improve the log message when connection initiators fail to
+      authenticate direct connections to relays. Fixes bug 26927; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.3.4.7-rc):
+    - Fix bug that causes services to not ever rotate their descriptors
+      if they were getting SIGHUPed often. Fixes bug 26932; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.3.4.8):
+    - Silence a spurious compiler warning in
+      rend_client_send_introduction(). Fixes bug 27463; bugfix
+      on 0.1.1.2-alpha.
+
+  o Minor bugfixes (rust, backport from 0.3.4.7-rc):
+    - Backport test_rust.sh from master. Fixes bug 26497; bugfix
+      on 0.3.1.5-alpha.
+    - Consistently use ../../.. as a fallback for $abs_top_srcdir in
+      test_rust.sh. Fixes bug 27093; bugfix on 0.3.4.3-alpha.
+    - Stop setting $CARGO_HOME. cargo will use the user's $CARGO_HOME, or
+      $HOME/.cargo by default. Fixes bug 26497; bugfix on 0.3.1.5-alpha.
+
+  o Minor bugfixes (single onion services, Tor2web, backport from 0.3.4.6-rc):
+    - Log a protocol warning when single onion services or Tor2web clients
+      fail to authenticate direct connections to relays.
+      Fixes bug 26924; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.4.6-rc):
+    - Disable core dumps in test_bt.sh, to avoid failures in "make
+      distcheck". Fixes bug 26787; bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (testing, chutney, backport from 0.3.4.8):
+    - When running make test-network-all, use the mixed+hs-v2 network.
+      (A previous fix to chutney removed v3 onion services from the
+      mixed+hs-v23 network, so seeing "mixed+hs-v23" in tests is
+      confusing.) Fixes bug 27345; bugfix on 0.3.2.1-alpha.
+    - Before running make test-network-all, delete old logs and test
+      result files, to avoid spurious failures. Fixes bug 27295; bugfix
+      on 0.2.7.3-rc.
+
+  o Minor bugfixes (testing, openssl compatibility):
+    - Our "tortls/cert_matches_key" unit test no longer relies on OpenSSL
+      internals.  Previously, it relied on unsupported OpenSSL behavior in
+      a way that caused it to crash with OpenSSL 1.0.2p. Fixes bug 27226;
+      bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (testing, openssl compatibility, backport from 0.3.4.7-rc):
+    - Our "tortls/cert_matches_key" unit test no longer relies on
+      OpenSSL internals. Previously, it relied on unsupported OpenSSL
+      behavior in a way that caused it to crash with OpenSSL 1.0.2p.
+      Fixes bug 27226; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (Windows, compilation, backport from 0.3.4.7-rc):
+    - Silence a compilation warning on MSVC 2017 and clang-cl. Fixes bug
+      27185; bugfix on 0.2.2.2-alpha.
+
+
+Changes in version 0.3.3.10 - 2018-09-10
+  Tor 0.3.3.10 backports numerous fixes from later versions of Tor.
+
+  o Minor features (bug workaround, backport from 0.3.4.7-rc):
+    - Compile correctly on systems that provide the C11 stdatomic.h
+      header, but where C11 atomic functions don't actually compile.
+      Closes ticket 26779; workaround for Debian issue 903709.
+
+  o Minor features (compatibility, backport from 0.3.4.8):
+    - Tell OpenSSL to maintain backward compatibility with previous
+      RSA1024/DH1024 users in Tor. With OpenSSL 1.1.1-pre6, these
+      ciphers are disabled by default. Closes ticket 27344.
+
+  o Minor features (continuous integration, backport from 0.3.4.7-rc):
+    - Backport Travis rust distcheck to 0.3.3. Closes ticket 24629.
+    - Enable macOS builds in our Travis CI configuration. Closes
+      ticket 24629.
+    - Install libcap-dev and libseccomp2-dev so these optional
+      dependencies get tested on Travis CI. Closes ticket 26560.
+    - Run asciidoc during Travis CI. Implements ticket 27087.
+    - Use ccache in our Travis CI configuration. Closes ticket 26952.
+
+  o Minor features (continuous integration, rust, backport from 0.3.4.7-rc):
+    - Use cargo cache in our Travis CI configuration. Closes
+      ticket 26952.
+
+  o Minor features (controller, backport from 0.3.4.6-rc):
+    - The control port now exposes the list of HTTPTunnelPorts and
+      ExtOrPorts via GETINFO net/listeners/httptunnel and
+      net/listeners/extor respectively. Closes ticket 26647.
+
+  o Minor features (directory authorities, backport from 0.3.4.7-rc):
+    - Authorities no longer vote to make the subprotocol version
+      "LinkAuth=1" a requirement: it is unsupportable with NSS, and
+      hasn't been needed since Tor 0.3.0.1-alpha. Closes ticket 27286.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the August 7 2018 Maxmind GeoLite2
+      Country database. Closes ticket 27089.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.6-rc):
+    - When compiling with --enable-openbsd-malloc or --enable-tcmalloc,
+      tell the compiler not to include the system malloc implementation.
+      Fixes bug 20424; bugfix on 0.2.0.20-rc.
+    - Don't try to use a pragma to temporarily disable the
+      -Wunused-const-variable warning if the compiler doesn't support
+      it. Fixes bug 26785; bugfix on 0.3.2.11.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.7-rc):
+    - Silence a spurious compiler warning on the GetAdaptersAddresses
+      function pointer cast. This issue is already fixed by 26481 in
+      0.3.5 and later, by removing the lookup and cast. Fixes bug 27465;
+      bugfix on 0.2.3.11-alpha.
+    - Stop calling SetProcessDEPPolicy() on 64-bit Windows. It is not
+      supported, and always fails. Some compilers warn about the
+      function pointer cast on 64-bit Windows. Fixes bug 27461; bugfix
+      on 0.2.2.23-alpha.
+
+  o Minor bugfixes (compilation, windows, backport from 0.3.4.7-rc):
+    - Don't link or search for pthreads when building for Windows, even
+      if we are using build environment (like mingw) that provides a
+      pthreads library. Fixes bug 27081; bugfix on 0.1.0.1-rc.
+
+  o Minor bugfixes (continuous integration, backport from 0.3.4.6-rc):
+    - Skip a pair of unreliable key generation tests on Windows, until
+      the underlying issue in bug 26076 is resolved. Fixes bug 26830 and
+      bug 26853; bugfix on 0.2.7.3-rc and 0.3.2.1-alpha respectively.
+
+  o Minor bugfixes (continuous integration, backport from 0.3.4.7-rc):
+    - Build with zstd on macOS. Fixes bug 27090; bugfix on 0.3.1.5-alpha.
+    - Pass the module flags to distcheck configure, and log the flags
+      before running configure. (Backported to 0.2.9 and later as a
+      precaution.) Fixes bug 27088; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (continuous integration, backport from 0.3.4.8):
+    - When a Travis build fails, and showing a log fails, keep trying to
+      show the other logs. Fixes bug 27453; bugfix on 0.3.4.7-rc.
+    - When we use echo in Travis, don't pass a --flag as the first
+      argument. Fixes bug 27418; bugfix on 0.3.4.7-rc.
+
+  o Minor bugfixes (directory authority, backport from 0.3.4.6-rc):
+    - When voting for recommended versions, make sure that all of the
+      versions are well-formed and parsable. Fixes bug 26485; bugfix
+      on 0.1.1.6-alpha.
+
+  o Minor bugfixes (in-process restart, backport from 0.3.4.7-rc):
+    - Always call tor_free_all() when leaving tor_run_main(). When we
+      did not, restarting tor in-process would cause an assertion
+      failure. Fixes bug 26948; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.4.7-rc):
+    - Fix a bug in our sandboxing rules for the openat() syscall.
+      Previously, no openat() call would be permitted, which would break
+      filesystem operations on recent glibc versions. Fixes bug 25440;
+      bugfix on 0.2.9.15. Diagnosis and patch from Daniel Pinto.
+
+  o Minor bugfixes (logging, backport from 0.3.4.6-rc):
+    - Improve the log message when connection initiators fail to
+      authenticate direct connections to relays. Fixes bug 26927; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.3.4.7-rc):
+    - Fix bug that causes services to not ever rotate their descriptors
+      if they were getting SIGHUPed often. Fixes bug 26932; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.3.4.8):
+    - Silence a spurious compiler warning in
+      rend_client_send_introduction(). Fixes bug 27463; bugfix
+      on 0.1.1.2-alpha.
+
+  o Minor bugfixes (portability, backport from 0.3.4.6-rc):
+    - Work around two different bugs in the OS X 10.10 and later SDKs
+      that would prevent us from successfully targeting earlier versions
+      of OS X. Fixes bug 26876; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (portability, backport from 0.3.4.7-rc):
+    - Fix compilation of the unit tests on GNU/Hurd, which does not
+      define PATH_MAX. Fixes bug 26873; bugfix on 0.3.3.1-alpha. Patch
+      from "paulusASol".
+
+  o Minor bugfixes (rust, backport from 0.3.4.7-rc):
+    - Backport test_rust.sh from master. Fixes bug 26497; bugfix
+      on 0.3.1.5-alpha.
+    - Consistently use ../../.. as a fallback for $abs_top_srcdir in
+      test_rust.sh. Fixes bug 27093; bugfix on 0.3.4.3-alpha.
+    - Protover parsing was accepting the presence of whitespace in
+      version strings, which the C implementation would choke on, e.g.
+      "Desc=1\t,2". Fixes bug 27177; bugfix on 0.3.3.5-rc.
+    - Protover parsing was ignoring a 2nd hyphen and everything after
+      it, accepting entries like "Link=1-5-foo". Fixes bug 27164; bugfix
+      on 0.3.3.1-alpha.
+    - Stop setting $CARGO_HOME. cargo will use the user's $CARGO_HOME, or
+      $HOME/.cargo by default. Fixes bug 26497; bugfix on 0.3.1.5-alpha.
+    - cd to ${abs_top_builddir}/src/rust before running cargo in
+      src/test/test_rust.sh. This makes the working directory consistent
+      between builds and tests. Fixes bug 26497; bugfix on 0.3.3.2-alpha.
+
+  o Minor bugfixes (single onion services, Tor2web, backport from 0.3.4.6-rc):
+    - Log a protocol warning when single onion services or Tor2web clients
+      fail to authenticate direct connections to relays.
+      Fixes bug 26924; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.4.6-rc):
+    - Disable core dumps in test_bt.sh, to avoid failures in "make
+      distcheck". Fixes bug 26787; bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (testing, chutney, backport from 0.3.4.8):
+    - When running make test-network-all, use the mixed+hs-v2 network.
+      (A previous fix to chutney removed v3 onion services from the
+      mixed+hs-v23 network, so seeing "mixed+hs-v23" in tests is
+      confusing.) Fixes bug 27345; bugfix on 0.3.2.1-alpha.
+    - Before running make test-network-all, delete old logs and test
+      result files, to avoid spurious failures. Fixes bug 27295; bugfix
+      on 0.2.7.3-rc.
+
+  o Minor bugfixes (testing, openssl compatibility, backport from 0.3.4.7-rc):
+    - Our "tortls/cert_matches_key" unit test no longer relies on
+      OpenSSL internals. Previously, it relied on unsupported OpenSSL
+      behavior in a way that caused it to crash with OpenSSL 1.0.2p.
+      Fixes bug 27226; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (v3 onion services, backport from 0.3.4.6-rc):
+    - Stop sending ed25519 link specifiers in v3 onion service introduce
+      cells and descriptors, when the rendezvous or introduction point
+      doesn't support ed25519 link authentication. Fixes bug 26627;
+      bugfix on 0.3.2.4-alpha.
+
+  o Minor bugfixes (Windows, compilation, backport from 0.3.4.7-rc):
+    - Silence a compilation warning on MSVC 2017 and clang-cl. Fixes bug
+      27185; bugfix on 0.2.2.2-alpha.
+
+
+Changes in version 0.3.4.8 - 2018-09-10
+  Tor 0.3.4.8 is the first stable release in its series; it includes
+  compilation and portability fixes.
+
+  The Tor 0.3.4 series includes improvements for running Tor in
+  low-power and embedded environments, which should help performance in
+  general. We've begun work on better modularity, and included preliminary
+  changes on the directory authority side to accommodate a new bandwidth
+  measurement system.  We've also integrated more continuous-integration
+  systems into our development process, and made corresponding changes to
+  Tor's testing infrastructure.  Finally, we've continued to refine
+  our anti-denial-of-service code.
+
+  Below are the changes since 0.3.4.7-rc.  For a complete list of changes
+  since 0.3.3.9, see the ReleaseNotes file.
+
+  o Minor features (compatibility):
+    - Tell OpenSSL to maintain backward compatibility with previous
+      RSA1024/DH1024 users in Tor. With OpenSSL 1.1.1-pre6, these
+      ciphers are disabled by default. Closes ticket 27344.
+
+  o Minor features (continuous integration):
+    - Log the compiler path and version during Appveyor builds.
+      Implements ticket 27449.
+    - Show config.log and test-suite.log after failed Appveyor builds.
+      Also upload the zipped full logs as a build artifact. Implements
+      ticket 27430.
+
+  o Minor bugfixes (compilation):
+    - Silence a spurious compiler warning on the GetAdaptersAddresses
+      function pointer cast. This issue is already fixed by 26481 in
+      0.3.5 and later, by removing the lookup and cast. Fixes bug 27465;
+      bugfix on 0.2.3.11-alpha.
+    - Stop calling SetProcessDEPPolicy() on 64-bit Windows. It is not
+      supported, and always fails. Some compilers warn about the
+      function pointer cast on 64-bit Windows. Fixes bug 27461; bugfix
+      on 0.2.2.23-alpha.
+
+  o Minor bugfixes (continuous integration):
+    - Disable gcc hardening in Appveyor Windows 64-bit builds. As of
+      August 29 2018, Appveyor images come with gcc 8.2.0 by default.
+      Executables compiled for 64-bit Windows with this version of gcc
+      crash when Tor's --enable-gcc-hardening flag is set. Fixes bug
+      27460; bugfix on 0.3.4.1-alpha.
+    - When a Travis build fails, and showing a log fails, keep trying to
+      show the other logs. Fixes bug 27453; bugfix on 0.3.4.7-rc.
+    - When we use echo in Travis, don't pass a --flag as the first
+      argument. Fixes bug 27418; bugfix on 0.3.4.7-rc.
+
+  o Minor bugfixes (onion services):
+    - Silence a spurious compiler warning in
+      rend_client_send_introduction(). Fixes bug 27463; bugfix
+      on 0.1.1.2-alpha.
+
+  o Minor bugfixes (testing, chutney):
+    - When running make test-network-all, use the mixed+hs-v2 network.
+      (A previous fix to chutney removed v3 onion services from the
+      mixed+hs-v23 network, so seeing "mixed+hs-v23" in tests is
+      confusing.) Fixes bug 27345; bugfix on 0.3.2.1-alpha.
+    - Before running make test-network-all, delete old logs and test
+      result files, to avoid spurious failures. Fixes bug 27295; bugfix
+      on 0.2.7.3-rc.
+
+Changes in version 0.3.4.7-rc - 2018-08-24
+  Tor 0.3.4.7-rc fixes several small compilation, portability, and
+  correctness issues in previous versions of Tor. This version is a
+  release candidate: if no serious bugs are found, we expect that the
+  stable 0.3.4 release will be (almost) the same as this release.
+
+  o Minor features (bug workaround):
+    - Compile correctly on systems that provide the C11 stdatomic.h
+      header, but where C11 atomic functions don't actually compile.
+      Closes ticket 26779; workaround for Debian issue 903709.
+
+  o Minor features (continuous integration):
+    - Backport Travis rust distcheck to 0.3.3. Closes ticket 24629.
+    - Enable macOS builds in our Travis CI configuration. Closes
+      ticket 24629.
+    - Install libcap-dev and libseccomp2-dev so these optional
+      dependencies get tested on Travis CI. Closes ticket 26560.
+    - Only post Appveyor IRC notifications when the build fails.
+      Implements ticket 27275.
+    - Run asciidoc during Travis CI. Implements ticket 27087.
+    - Use ccache in our Travis CI configuration. Closes ticket 26952.
+
+  o Minor features (continuous integration, rust):
+    - Use cargo cache in our Travis CI configuration. Closes
+      ticket 26952.
+
+  o Minor features (directory authorities):
+    - Authorities no longer vote to make the subprotocol version
+      "LinkAuth=1" a requirement: it is unsupportable with NSS, and
+      hasn't been needed since Tor 0.3.0.1-alpha. Closes ticket 27286.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the August 7 2018 Maxmind GeoLite2
+      Country database. Closes ticket 27089.
+
+  o Minor bugfixes (compilation, windows):
+    - Don't link or search for pthreads when building for Windows, even
+      if we are using build environment (like mingw) that provides a
+      pthreads library. Fixes bug 27081; bugfix on 0.1.0.1-rc.
+
+  o Minor bugfixes (continuous integration):
+    - Improve Appveyor CI IRC logging. Generate correct branches and
+      URLs for pull requests and tags. Use unambiguous short commits.
+      Fixes bug 26979; bugfix on master.
+    - Build with zstd on macOS. Fixes bug 27090; bugfix on 0.3.1.5-alpha.
+    - Pass the module flags to distcheck configure, and log the flags
+      before running configure. (Backported to 0.2.9 and later as a
+      precaution.) Fixes bug 27088; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (in-process restart):
+    - Always call tor_free_all() when leaving tor_run_main(). When we
+      did not, restarting tor in-process would cause an assertion
+      failure. Fixes bug 26948; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox):
+    - Fix a bug in out sandboxing rules for the openat() syscall.
+      Previously, no openat() call would be permitted, which would break
+      filesystem operations on recent glibc versions. Fixes bug 25440;
+      bugfix on 0.2.9.15. Diagnosis and patch from Daniel Pinto.
+
+  o Minor bugfixes (onion services):
+    - Fix bug that causes services to not ever rotate their descriptors
+      if they were getting SIGHUPed often. Fixes bug 26932; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (portability):
+    - Fix compilation of the unit tests on GNU/Hurd, which does not
+      define PATH_MAX. Fixes bug 26873; bugfix on 0.3.3.1-alpha. Patch
+      from "paulusASol".
+
+  o Minor bugfixes (rust):
+    - Backport test_rust.sh from master. Fixes bug 26497; bugfix
+      on 0.3.1.5-alpha.
+    - Consistently use ../../.. as a fallback for $abs_top_srcdir in
+      test_rust.sh. Fixes bug 27093; bugfix on 0.3.4.3-alpha.
+    - Protover parsing was accepting the presence of whitespace in
+      version strings, which the C implementation would choke on, e.g.
+      "Desc=1\t,2". Fixes bug 27177; bugfix on 0.3.3.5-rc.
+    - Protover parsing was ignoring a 2nd hyphen and everything after
+      it, accepting entries like "Link=1-5-foo". Fixes bug 27164; bugfix
+      on 0.3.3.1-alpha.
+    - Stop setting $CARGO_HOME. cargo will use the user's $CARGO_HOME, or
+      $HOME/.cargo by default. Fixes bug 26497; bugfix on 0.3.1.5-alpha.
+    - cd to ${abs_top_builddir}/src/rust before running cargo in
+      src/test/test_rust.sh. This makes the working directory consistent
+      between builds and tests. Fixes bug 26497; bugfix on 0.3.3.2-alpha.
+
+  o Minor bugfixes (testing, bootstrap):
+    - When calculating bootstrap progress, check exit policies and the
+      exit flag. Previously, Tor would only check the exit flag, which
+      caused race conditions in small and fast networks like chutney.
+      Fixes bug 27236; bugfix on 0.2.6.3-alpha.
+
+  o Minor bugfixes (testing, openssl compatibility):
+    - Our "tortls/cert_matches_key" unit test no longer relies on
+      OpenSSL internals. Previously, it relied on unsupported OpenSSL
+      behavior in a way that caused it to crash with OpenSSL 1.0.2p.
+      Fixes bug 27226; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (Windows, compilation):
+    - Silence a compilation warning on MSVC 2017 and clang-cl. Fixes bug
+      27185; bugfix on 0.2.2.2-alpha.
+
+
+Changes in version 0.3.4.6-rc - 2018-08-06
+  Tor 0.3.4.6-rc fixes several small compilation, portability, and
+  correctness issues in previous versions of Tor. This version is a
+  release candidate: if no serious bugs are found, we expect that the
+  stable 0.3.4 release will be (almost) the same as this release.
+
+  o Major bugfixes (event scheduler):
+    - When we enable a periodic event, schedule it in the event loop
+      rather than running it immediately. Previously, we would re-run
+      periodic events immediately in the middle of (for example)
+      changing our options, with unpredictable effects. Fixes bug 27003;
+      bugfix on 0.3.4.1-alpha.
+
+  o Minor features (compilation):
+    - When building Tor, prefer to use Python 3 over Python 2, and more
+      recent (contemplated) versions over older ones. Closes
+      ticket 26372.
+    - When compiling with --enable-openbsd-malloc or --enable-tcmalloc,
+      tell the compiler not to include the system malloc implementation.
+      Fixes bug 20424; bugfix on 0.2.0.20-rc.
+    - Don't try to use a pragma to temporarily disable the
+      -Wunused-const-variable warning if the compiler doesn't support
+      it. Fixes bug 26785; bugfix on 0.3.2.11.
+
+  o Minor bugfixes (continuous integration):
+    - Skip a pair of unreliable key generation tests on Windows, until
+      the underlying issue in bug 26076 is resolved. Fixes bug 26830 and
+      bug 26853; bugfix on 0.2.7.3-rc and 0.3.2.1-alpha respectively.
+
+  o Minor features (controller):
+    - The control port now exposes the list of HTTPTunnelPorts and
+      ExtOrPorts via GETINFO net/listeners/httptunnel and
+      net/listeners/extor respectively. Closes ticket 26647.
+
+  o Minor bugfixes (directory authority):
+    - When voting for recommended versions, make sure that all of the
+      versions are well-formed and parsable. Fixes bug 26485; bugfix
+      on 0.1.1.6-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the July 3 2018 Maxmind GeoLite2
+      Country database. Closes ticket 26674.
+
+  o Minor features (Rust, portability):
+    - Rust cross-compilation is now supported. Closes ticket 25895.
+
+  o Minor bugfixes (compilation):
+    - Update build system so that tor builds again with --disable-unittests
+      after recent refactoring. Fixes bug 26789; bugfix on 0.3.4.3-alpha.
+    - Fix a compilation warning on some versions of GCC when building
+      code that calls routerinfo_get_my_routerinfo() twice, assuming
+      that the second call will succeed if the first one did. Fixes bug
+      26269; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (controller):
+    - Report the port correctly when a port is configured to bind to
+      "auto". Fixes bug 26568; bugfix on 0.3.4.1-alpha.
+    - Parse the "HSADDRESS=" parameter in HSPOST commands properly.
+      Previously, it was misparsed and ignored. Fixes bug 26523; bugfix
+      on 0.3.3.1-alpha. Patch by "akwizgran".
+
+  o Minor bugfixes (correctness, flow control):
+    - Upon receiving a stream-level SENDME cell, verify that our window
+      has not grown too large. Fixes bug 26214; bugfix on svn
+      r54 (pre-0.0.1).
+
+  o Minor bugfixes (memory, correctness):
+    - Fix a number of small memory leaks identified by coverity. Fixes
+      bug 26467; bugfix on numerous Tor versions.
+
+  o Minor bugfixes (logging):
+    - Improve the log message when connection initiators fail to
+      authenticate direct connections to relays. Fixes bug 26927; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (portability):
+    - Avoid a compilation error in test_bwmgt.c on Solaris 10. Fixes bug
+      26994; bugfix on 0.3.4.1-alpha.
+    - Work around two different bugs in the OS X 10.10 and later SDKs
+      that would prevent us from successfully targeting earlier versions
+      of OS X. Fixes bug 26876; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (single onion services, Tor2web):
+    - Log a protocol warning when single onion services or Tor2web
+      clients fail to authenticate direct connections to relays. Fixes
+      bug 26924; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Disable core dumps in test_bt.sh, to avoid failures in "make
+      distcheck". Fixes bug 26787; bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (testing, compatibility):
+    - When running the ntor_ref.py and hs_ntor_ref.py tests, make sure
+      only to pass strings (rather than "bytes" objects) to the Python
+      subprocess module. Python 3 on Windows seems to require this.
+      Fixes bug 26535; bugfix on 0.2.5.5-alpha (for ntor_ref.py) and
+      0.3.1.1-alpha (for hs_ntor_ref.py).
+
+  o Minor bugfixes (v3 onion services):
+    - Stop sending ed25519 link specifiers in v3 onion service introduce
+      cells and descriptors, when the rendezvous or introduction point
+      doesn't support ed25519 link authentication. Fixes bug 26627;
+      bugfix on 0.3.2.4-alpha.
+
+
+Changes in version 0.3.4.5-rc - 2018-07-13
+  Tor 0.3.4.5-rc moves to a new bridge authority, meaning people running
+  bridge relays should upgrade.
+
+  o Directory authority changes:
+    - The "Bifroest" bridge authority has been retired; the new bridge
+      authority is "Serge", and it is operated by George from the
+      TorBSD project. Closes ticket 26771.
+
+
+Changes in version 0.3.3.9 - 2018-07-13
+  Tor 0.3.3.9 moves to a new bridge authority, meaning people running
+  bridge relays should upgrade.
+
+  o Directory authority changes:
+    - The "Bifroest" bridge authority has been retired; the new bridge
+      authority is "Serge", and it is operated by George from the
+      TorBSD project. Closes ticket 26771.
+
+
+Changes in version 0.3.2.11 - 2018-07-13
+  Tor 0.3.2.11 moves to a new bridge authority, meaning people running
+  bridge relays should upgrade. We also take this opportunity to backport
+  other minor fixes.
+
+  o Directory authority changes:
+    - The "Bifroest" bridge authority has been retired; the new bridge
+      authority is "Serge", and it is operated by George from the
+      TorBSD project. Closes ticket 26771.
+
+  o Directory authority changes (backport from 0.3.3.7):
+    - Add an IPv6 address for the "dannenberg" directory authority.
+      Closes ticket 26343.
+
+  o Major bugfixes (directory authorities, backport from 0.3.4.1-alpha):
+    - When directory authorities read a zero-byte bandwidth file, they
+      would previously log a warning with the contents of an
+      uninitialised buffer. They now log a warning about the empty file
+      instead. Fixes bug 26007; bugfix on 0.2.2.1-alpha.
+
+  o Major bugfixes (onion service, backport from 0.3.4.1-alpha):
+    - Correctly detect when onion services get disabled after HUP. Fixes
+      bug 25761; bugfix on 0.3.2.1.
+
+  o Minor features (sandbox, backport from 0.3.3.4-alpha):
+    - Explicitly permit the poll() system call when the Linux
+      seccomp2-based sandbox is enabled: apparently, some versions of
+      libc use poll() when calling getpwnam(). Closes ticket 25313.
+
+  o Minor feature (continuous integration, backport from 0.3.3.5-rc):
+    - Update the Travis CI configuration to use the stable Rust channel,
+      now that we have decided to require that. Closes ticket 25714.
+
+  o Minor features (continuous integration, backport from 0.3.4.1-alpha):
+    - Our .travis.yml configuration now includes support for testing the
+      results of "make distcheck". (It's not uncommon for "make check"
+      to pass but "make distcheck" to fail.) Closes ticket 25814.
+    - Our Travis CI configuration now integrates with the Coveralls
+      coverage analysis tool. Closes ticket 25818.
+
+  o Minor features (relay, diagnostic, backport from 0.3.4.3-alpha):
+    - Add several checks to detect whether Tor relays are uploading
+      their descriptors without specifying why they regenerated them.
+      Diagnostic for ticket 25686.
+
+  o Minor features (compilation, backport from 0.3.4.4-rc):
+    - When building Tor, prefer to use Python 3 over Python 2, and more
+      recent (contemplated) versions over older ones. Closes
+      ticket 26372.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the July 3 2018 Maxmind GeoLite2
+      Country database. Closes ticket 26674.
+
+  o Minor bugfixes (correctness, client, backport from 0.3.4.1-alpha):
+    - Upon receiving a malformed connected cell, stop processing the
+      cell immediately. Previously we would mark the connection for
+      close, but continue processing the cell as if the connection were
+      open. Fixes bug 26072; bugfix on 0.2.4.7-alpha.
+
+  o Minor bugfixes (Linux seccomp2 sandbox, backport from 0.3.4.1-alpha):
+    - Allow the nanosleep() system call, which glibc uses to implement
+      sleep() and usleep(). Fixes bug 24969; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (testing, compatibility, backport from 0.3.4.4-rc):
+    - When running the hs_ntor_ref.py test, make sure only to pass
+      strings (rather than "bytes" objects) to the Python subprocess
+      module. Python 3 on Windows seems to require this. Fixes bug
+      26535; bugfix on 0.3.1.1-alpha.
+    - When running the ntor_ref.py test, make sure only to pass strings
+      (rather than "bytes" objects) to the Python subprocess module.
+      Python 3 on Windows seems to require this. Fixes bug 26535; bugfix
+      on 0.2.5.5-alpha.
+
+  o Minor bugfixes (compatibility, openssl, backport from 0.3.4.2-alpha):
+    - Work around a change in OpenSSL 1.1.1 where return values that
+      would previously indicate "no password" now indicate an empty
+      password. Without this workaround, Tor instances running with
+      OpenSSL 1.1.1 would accept descriptors that other Tor instances
+      would reject. Fixes bug 26116; bugfix on 0.2.5.16.
+
+  o Minor bugfixes (documentation, backport from 0.3.3.5-rc):
+    - Document that the PerConnBW{Rate,Burst} options will fall back to
+      their corresponding consensus parameters only if those parameters
+      are set. Previously we had claimed that these values would always
+      be set in the consensus. Fixes bug 25296; bugfix on 0.2.2.7-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.4-rc):
+    - Fix a compilation warning on some versions of GCC when building
+      code that calls routerinfo_get_my_routerinfo() twice, assuming
+      that the second call will succeed if the first one did. Fixes bug
+      26269; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (client, backport from 0.3.4.1-alpha):
+    - Don't consider Tor running as a client if the ControlPort is open,
+      but no actual client ports are open. Fixes bug 26062; bugfix
+      on 0.2.9.4-alpha.
+
+  o Minor bugfixes (hardening, backport from 0.3.4.2-alpha):
+    - Prevent a possible out-of-bounds smartlist read in
+      protover_compute_vote(). Fixes bug 26196; bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (C correctness, backport from 0.3.3.4-alpha):
+    - Fix a very unlikely (impossible, we believe) null pointer
+      dereference. Fixes bug 25629; bugfix on 0.2.9.15. Found by
+      Coverity; this is CID 1430932.
+
+  o Minor bugfixes (onion service, backport from 0.3.4.1-alpha):
+    - Fix a memory leak when a v3 onion service is configured and gets a
+      SIGHUP signal. Fixes bug 25901; bugfix on 0.3.2.1-alpha.
+    - When parsing the descriptor signature, look for the token plus an
+      extra white-space at the end. This is more correct but also will
+      allow us to support new fields that might start with "signature".
+      Fixes bug 26069; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (relay, backport from 0.3.4.3-alpha):
+    - Relays now correctly block attempts to re-extend to the previous
+      relay by Ed25519 identity. Previously they would warn in this
+      case, but not actually reject the attempt. Fixes bug 26158; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (relay, crash, backport from 0.3.4.1-alpha):
+    - Avoid a crash when running with DirPort set but ORPort turned off.
+      Fixes a case of bug 23693; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.2-alpha):
+    - Silence unused-const-variable warnings in zstd.h with some GCC
+      versions. Fixes bug 26272; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.3.4-alpha):
+    - Avoid intermittent test failures due to a test that had relied on
+      onion service introduction point creation finishing within 5
+      seconds of real clock time. Fixes bug 25450; bugfix
+      on 0.3.1.3-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.3.4-alpha):
+    - Fix a C99 compliance issue in our configuration script that caused
+      compilation issues when compiling Tor with certain versions of
+      xtools. Fixes bug 25474; bugfix on 0.3.2.5-alpha.
+
+  o Minor bugfixes (memory, correctness, backport from 0.3.4.4-rc):
+    - Fix a number of small memory leaks identified by coverity. Fixes
+      bug 26467; bugfix on numerous Tor versions.
+
+  o Code simplification and refactoring (backport from 0.3.3.5-rc):
+    - Move the list of default directory authorities to its own file.
+      Closes ticket 24854. Patch by "beastr0".
+
+
+Changes in version 0.2.9.16 - 2018-07-13
+  Tor 0.2.9.16 moves to a new bridge authority, meaning people running
+  bridge relays should upgrade. We also take this opportunity to backport
+  other minor fixes.
+
+  o Directory authority changes:
+    - The "Bifroest" bridge authority has been retired; the new bridge
+      authority is "Serge", and it is operated by George from the
+      TorBSD project. Closes ticket 26771.
+
+  o Directory authority changes (backport from 0.3.3.7):
+    - Add an IPv6 address for the "dannenberg" directory authority.
+      Closes ticket 26343.
+
+  o Major bugfixes (directory authorities, backport from 0.3.4.1-alpha):
+    - When directory authorities read a zero-byte bandwidth file, they
+      would previously log a warning with the contents of an
+      uninitialised buffer. They now log a warning about the empty file
+      instead. Fixes bug 26007; bugfix on 0.2.2.1-alpha.
+
+  o Minor features (sandbox, backport from 0.3.3.4-alpha):
+    - Explicitly permit the poll() system call when the Linux
+      seccomp2-based sandbox is enabled: apparently, some versions of
+      libc use poll() when calling getpwnam(). Closes ticket 25313.
+
+  o Minor features (continuous integration, backport from 0.3.4.1-alpha):
+    - Our .travis.yml configuration now includes support for testing the
+      results of "make distcheck". (It's not uncommon for "make check"
+      to pass but "make distcheck" to fail.) Closes ticket 25814.
+    - Our Travis CI configuration now integrates with the Coveralls
+      coverage analysis tool. Closes ticket 25818.
+
+  o Minor features (compilation, backport from 0.3.4.4-rc):
+    - When building Tor, prefer to use Python 3 over Python 2, and more
+      recent (contemplated) versions over older ones. Closes
+      ticket 26372.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the July 3 2018 Maxmind GeoLite2
+      Country database. Closes ticket 26674.
+
+  o Minor bugfixes (correctness, client, backport from 0.3.4.1-alpha):
+    - Upon receiving a malformed connected cell, stop processing the
+      cell immediately. Previously we would mark the connection for
+      close, but continue processing the cell as if the connection were
+      open. Fixes bug 26072; bugfix on 0.2.4.7-alpha.
+
+  o Minor bugfixes (Linux seccomp2 sandbox, backport from 0.3.4.1-alpha):
+    - Allow the nanosleep() system call, which glibc uses to implement
+      sleep() and usleep(). Fixes bug 24969; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (testing, compatibility, backport from 0.3.4.4-rc):
+    - When running the ntor_ref.py test, make sure only to pass strings
+      (rather than "bytes" objects) to the Python subprocess module.
+      Python 3 on Windows seems to require this. Fixes bug 26535; bugfix
+      on 0.2.5.5-alpha.
+
+  o Minor bugfixes (compatibility, openssl, backport from 0.3.4.2-alpha):
+    - Work around a change in OpenSSL 1.1.1 where return values that
+      would previously indicate "no password" now indicate an empty
+      password. Without this workaround, Tor instances running with
+      OpenSSL 1.1.1 would accept descriptors that other Tor instances
+      would reject. Fixes bug 26116; bugfix on 0.2.5.16.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.4-rc):
+    - Fix a compilation warning on some versions of GCC when building
+      code that calls routerinfo_get_my_routerinfo() twice, assuming
+      that the second call will succeed if the first one did. Fixes bug
+      26269; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (client, backport from 0.3.4.1-alpha):
+    - Don't consider Tor running as a client if the ControlPort is open,
+      but no actual client ports are open. Fixes bug 26062; bugfix
+      on 0.2.9.4-alpha.
+
+  o Minor bugfixes (hardening, backport from 0.3.4.2-alpha):
+    - Prevent a possible out-of-bounds smartlist read in
+      protover_compute_vote(). Fixes bug 26196; bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (C correctness, backport from 0.3.3.4-alpha):
+    - Fix a very unlikely (impossible, we believe) null pointer
+      dereference. Fixes bug 25629; bugfix on 0.2.9.15. Found by
+      Coverity; this is CID 1430932.
+
+  o Minor bugfixes (memory, correctness, backport from 0.3.4.4-rc):
+    - Fix a number of small memory leaks identified by coverity. Fixes
+      bug 26467; bugfix on numerous Tor versions.
+
+  o Code simplification and refactoring (backport from 0.3.3.5-rc):
+    - Move the list of default directory authorities to its own file.
+      Closes ticket 24854. Patch by "beastr0".
+
+
+Changes in version 0.3.4.4-rc - 2018-07-09
+  Tor 0.3.4.4-rc fixes several small compilation, portability, and
+  correctness issues in previous versions of Tor. This version is a
+  release candidate: if no serious bugs are found, we expect that the
+  stable 0.3.4 release will be (almost) the same as this release.
+
+  o Minor features (compilation):
+    - When building Tor, prefer to use Python 3 over Python 2, and more
+      recent (contemplated) versions over older ones. Closes
+      ticket 26372.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the July 3 2018 Maxmind GeoLite2
+      Country database. Closes ticket 26674.
+
+  o Minor features (Rust, portability):
+    - Rust cross-compilation is now supported. Closes ticket 25895.
+
+  o Minor bugfixes (compilation):
+    - Fix a compilation warning on some versions of GCC when building
+      code that calls routerinfo_get_my_routerinfo() twice, assuming
+      that the second call will succeed if the first one did. Fixes bug
+      26269; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (control port):
+    - Report the port correctly when a port is configured to bind to
+      "auto". Fixes bug 26568; bugfix on 0.3.4.1-alpha.
+    - Handle the HSADDRESS= argument to the HSPOST command properly.
+      (Previously, this argument was misparsed and thus ignored.) Fixes
+      bug 26523; bugfix on 0.3.3.1-alpha. Patch by "akwizgran".
+
+  o Minor bugfixes (correctness, flow control):
+    - Upon receiving a stream-level SENDME cell, verify that our window
+      has not grown too large. Fixes bug 26214; bugfix on svn
+      r54 (pre-0.0.1).
+
+  o Minor bugfixes (memory, correctness):
+    - Fix a number of small memory leaks identified by coverity. Fixes
+      bug 26467; bugfix on numerous Tor versions.
+
+  o Minor bugfixes (testing, compatibility):
+    - When running the hs_ntor_ref.py test, make sure only to pass
+      strings (rather than "bytes" objects) to the Python subprocess
+      module. Python 3 on Windows seems to require this. Fixes bug
+      26535; bugfix on 0.3.1.1-alpha.
+    - When running the ntor_ref.py test, make sure only to pass strings
+      (rather than "bytes" objects) to the Python subprocess module.
+      Python 3 on Windows seems to require this. Fixes bug 26535; bugfix
+      on 0.2.5.5-alpha.
+
+
+Changes in version 0.3.3.8 - 2018-07-09
+  Tor 0.3.3.8 backports several changes from the 0.3.4.x series, including
+  fixes for a memory leak affecting directory authorities.
+
+  o Major bugfixes (directory authority, backport from 0.3.4.3-alpha):
+    - Stop leaking memory on directory authorities when planning to
+      vote. This bug was crashing authorities by exhausting their
+      memory. Fixes bug 26435; bugfix on 0.3.3.6.
+
+  o Major bugfixes (rust, testing, backport from 0.3.4.3-alpha):
+    - Make sure that failing tests in Rust will actually cause the build
+      to fail: previously, they were ignored. Fixes bug 26258; bugfix
+      on 0.3.3.4-alpha.
+
+  o Minor features (compilation, backport from 0.3.4.4-rc):
+    - When building Tor, prefer to use Python 3 over Python 2, and more
+      recent (contemplated) versions over older ones. Closes
+      ticket 26372.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the July 3 2018 Maxmind GeoLite2
+      Country database. Closes ticket 26674.
+
+  o Minor features (relay, diagnostic, backport from 0.3.4.3-alpha):
+    - Add several checks to detect whether Tor relays are uploading
+      their descriptors without specifying why they regenerated them.
+      Diagnostic for ticket 25686.
+
+  o Minor bugfixes (circuit path selection, backport from 0.3.4.1-alpha):
+    - Don't count path selection failures as circuit build failures.
+      This change should eliminate cases where Tor blames its guard or
+      the network for situations like insufficient microdescriptors
+      and/or overly restrictive torrc settings. Fixes bug 25705; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.4-rc):
+    - Fix a compilation warning on some versions of GCC when building
+      code that calls routerinfo_get_my_routerinfo() twice, assuming
+      that the second call will succeed if the first one did. Fixes bug
+      26269; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (control port, backport from 0.3.4.4-rc):
+    - Handle the HSADDRESS= argument to the HSPOST command properly.
+      (Previously, this argument was misparsed and thus ignored.) Fixes
+      bug 26523; bugfix on 0.3.3.1-alpha. Patch by "akwizgran".
+
+  o Minor bugfixes (memory, correctness, backport from 0.3.4.4-rc):
+    - Fix a number of small memory leaks identified by coverity. Fixes
+      bug 26467; bugfix on numerous Tor versions.
+
+  o Minor bugfixes (relay, backport from 0.3.4.3-alpha):
+    - Relays now correctly block attempts to re-extend to the previous
+      relay by Ed25519 identity. Previously they would warn in this
+      case, but not actually reject the attempt. Fixes bug 26158; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (restart-in-process, backport from 0.3.4.1-alpha):
+    - When shutting down, Tor now clears all the flags in the control.c
+      module. This should prevent a bug where authentication cookies are
+      not generated on restart. Fixes bug 25512; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (testing, compatibility, backport from 0.3.4.4-rc):
+    - When running the hs_ntor_ref.py test, make sure only to pass
+      strings (rather than "bytes" objects) to the Python subprocess
+      module. Python 3 on Windows seems to require this. Fixes bug
+      26535; bugfix on 0.3.1.1-alpha.
+    - When running the ntor_ref.py test, make sure only to pass strings
+      (rather than "bytes" objects) to the Python subprocess module.
+      Python 3 on Windows seems to require this. Fixes bug 26535; bugfix
+      on 0.2.5.5-alpha.
+
+
+Changes in version 0.3.4.3-alpha - 2018-06-26
+  Tor 0.3.4.3-alpha fixes several bugs in earlier versions, including
+  one that was causing stability issues on directory authorities.
+
+  o Major bugfixes (directory authority):
+    - Stop leaking memory on directory authorities when planning to
+      vote. This bug was crashing authorities by exhausting their
+      memory. Fixes bug 26435; bugfix on 0.3.3.6.
+
+  o Major bugfixes (rust, testing):
+    - Make sure that failing tests in Rust will actually cause the build
+      to fail: previously, they were ignored. Fixes bug 26258; bugfix
+      on 0.3.3.4-alpha.
+
+  o Minor feature (directory authorities):
+    - Stop warning about incomplete bw lines before the first complete
+      bw line has been found, so that additional header lines can be
+      ignored. Fixes bug 25960; bugfix on 0.2.2.1-alpha
+
+  o Minor features (relay, diagnostic):
+    - Add several checks to detect whether Tor relays are uploading
+      their descriptors without specifying why they regenerated them.
+      Diagnostic for ticket 25686.
+
+  o Minor features (unit tests):
+    - Test complete bandwidth measurements files, and test that
+      incomplete bandwidth lines only give warnings when the end of the
+      header has not been detected. Fixes bug 25947; bugfix
+      on 0.2.2.1-alpha
+
+  o Minor bugfixes (compilation):
+    - Refrain from compiling unit testing related object files when
+      --disable-unittests is set to configure script. Fixes bug 24891;
+      bugfix on 0.2.5.1-alpha.
+    - When linking the libtor_testing.a library, only include the
+      dirauth object files once. Previously, they were getting added
+      twice. Fixes bug 26402; bugfix on 0.3.4.1-alpha.
+    - The --enable-fatal-warnings flag now affects Rust code as well.
+      Closes ticket 26245.
+
+  o Minor bugfixes (onion services):
+    - Recompute some consensus information after detecting a clock jump,
+      or after transitioning from a non-live consensus to a live
+      consensus. We do this to avoid having an outdated state, and
+      miscalculating the index for next-generation onion services. Fixes
+      bug 24977; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (relay):
+    - Relays now correctly block attempts to re-extend to the previous
+      relay by Ed25519 identity. Previously they would warn in this
+      case, but not actually reject the attempt. Fixes bug 26158; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Fix compilation of the doctests in the Rust crypto crate. Fixes
+      bug 26415; bugfix on 0.3.4.1-alpha.
+    - Instead of trying to read the geoip configuration files from
+      within the unit tests, instead create our own ersatz files with
+      just enough geoip data in the format we expect. Trying to read
+      from the source directory created problems on Windows with mingw,
+      where the build system's paths are not the same as the platform's
+      paths. Fixes bug 25787; bugfix on 0.3.4.1-alpha.
+    - Refrain from trying to get an item from an empty smartlist in
+      test_bridges_clear_bridge_list. Set DEBUG_SMARTLIST in unit tests
+      to catch improper smartlist usage. Furthermore, enable
+      DEBUG_SMARTLIST globally when build is configured with fragile
+      hardening. Fixes bug 26196; bugfix on 0.3.4.1-alpha.
+
+
+Changes in version 0.3.3.7 - 2018-06-12
+  Tor 0.3.3.7 backports several changes from the 0.3.4.x series, including
+  fixes for bugs affecting compatibility and stability.
+
+  o Directory authority changes:
+    - Add an IPv6 address for the "dannenberg" directory authority.
+      Closes ticket 26343.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the June 7 2018 Maxmind GeoLite2
+      Country database. Closes ticket 26351.
+
+  o Minor bugfixes (compatibility, openssl, backport from 0.3.4.2-alpha):
+    - Work around a change in OpenSSL 1.1.1 where return values that
+      would previously indicate "no password" now indicate an empty
+      password. Without this workaround, Tor instances running with
+      OpenSSL 1.1.1 would accept descriptors that other Tor instances
+      would reject. Fixes bug 26116; bugfix on 0.2.5.16.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.2-alpha):
+    - Silence unused-const-variable warnings in zstd.h with some GCC
+      versions. Fixes bug 26272; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (controller, backport from 0.3.4.2-alpha):
+    - Improve accuracy of the BUILDTIMEOUT_SET control port event's
+      TIMEOUT_RATE and CLOSE_RATE fields. (We were previously
+      miscounting the total number of circuits for these field values.)
+      Fixes bug 26121; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (hardening, backport from 0.3.4.2-alpha):
+    - Prevent a possible out-of-bounds smartlist read in
+      protover_compute_vote(). Fixes bug 26196; bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (path selection, backport from 0.3.4.1-alpha):
+    - Only select relays when they have the descriptors we prefer to use
+      for them. This change fixes a bug where we could select a relay
+      because it had _some_ descriptor, but reject it later with a
+      nonfatal assertion error because it didn't have the exact one we
+      wanted. Fixes bugs 25691 and 25692; bugfix on 0.3.3.4-alpha.
+
+
+Changes in version 0.3.4.2-alpha - 2018-06-12
+  Tor 0.3.4.2-alpha fixes several minor bugs in the previous alpha
+  release, and forward-ports an authority-only security fix from 0.3.3.6.
+
+  o Directory authority changes:
+    - Add an IPv6 address for the "dannenberg" directory authority.
+      Closes ticket 26343.
+
+  o Major bugfixes (security, directory authority, denial-of-service, also in 0.3.3.6):
+    - Fix a bug that could have allowed an attacker to force a directory
+      authority to use up all its RAM by passing it a maliciously
+      crafted protocol versions string. Fixes bug 25517; bugfix on
+      0.2.9.4-alpha. This issue is also tracked as TROVE-2018-005.
+
+  o Minor features (continuous integration):
+    - Add the necessary configuration files for continuous integration
+      testing on Windows, via the Appveyor platform. Closes ticket
+      25549. Patches from Marcin Cieślak and Isis Lovecruft.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the June 7 2018 Maxmind GeoLite2
+      Country database. Closes ticket 26351.
+
+  o Minor bugfixes (compatibility, openssl):
+    - Work around a change in OpenSSL 1.1.1 where return values that
+      would previously indicate "no password" now indicate an empty
+      password. Without this workaround, Tor instances running with
+      OpenSSL 1.1.1 would accept descriptors that other Tor instances
+      would reject. Fixes bug 26116; bugfix on 0.2.5.16.
+
+  o Minor bugfixes (compilation):
+    - Silence unused-const-variable warnings in zstd.h with some GCC
+      versions. Fixes bug 26272; bugfix on 0.3.1.1-alpha.
+    - Fix compilation when using OpenSSL 1.1.0 with the "no-deprecated"
+      flag enabled. Fixes bug 26156; bugfix on 0.3.4.1-alpha.
+    - Avoid a compiler warning when casting the return value of
+      smartlist_len() to double with DEBUG_SMARTLIST enabled. Fixes bug
+      26283; bugfix on 0.2.4.10-alpha.
+
+  o Minor bugfixes (control port):
+    - Do not count 0-length RELAY_COMMAND_DATA cells as valid data in
+      CIRC_BW events. Previously, such cells were counted entirely in
+      the OVERHEAD field. Now they are not. Fixes bug 26259; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (controller):
+    - Improve accuracy of the BUILDTIMEOUT_SET control port event's
+      TIMEOUT_RATE and CLOSE_RATE fields. (We were previously
+      miscounting the total number of circuits for these field values.)
+      Fixes bug 26121; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (hardening):
+    - Prevent a possible out-of-bounds smartlist read in
+      protover_compute_vote(). Fixes bug 26196; bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (onion services):
+    - Fix a bug that blocked the creation of ephemeral v3 onion
+      services. Fixes bug 25939; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (test coverage tools):
+    - Update our "cov-diff" script to handle output from the latest
+      version of gcov, and to remove extraneous timestamp information
+      from its output. Fixes bugs 26101 and 26102; bugfix
+      on 0.2.5.1-alpha.
+
+
+Changes in version 0.3.3.6 - 2018-05-22
+  Tor 0.3.3.6 is the first stable release in the 0.3.3 series. It
+  backports several important fixes from the 0.3.4.1-alpha.
+
+  The Tor 0.3.3 series includes controller support and other
+  improvements for v3 onion services, official support for embedding Tor
+  within other applications, and our first non-trivial module written in
+  the Rust programming language. (Rust is still not enabled by default
+  when building Tor.) And as usual, there are numerous other smaller
+  bugfixes, features, and improvements.
+
+  Below are the changes since 0.3.3.5-rc. For a list of all changes
+  since 0.3.2.10, see the ReleaseNotes file.
+
+  o Major bugfixes (directory authorities, security, backport from 0.3.4.1-alpha):
+    - When directory authorities read a zero-byte bandwidth file, they
+      would previously log a warning with the contents of an
+      uninitialised buffer. They now log a warning about the empty file
+      instead. Fixes bug 26007; bugfix on 0.2.2.1-alpha.
+
+  o Major bugfixes (security, directory authority, denial-of-service):
+    - Fix a bug that could have allowed an attacker to force a directory
+      authority to use up all its RAM by passing it a maliciously
+      crafted protocol versions string. Fixes bug 25517; bugfix on
+      0.2.9.4-alpha. This issue is also tracked as TROVE-2018-005.
+
+  o Major bugfixes (crash, backport from 0.3.4.1-alpha):
+    - Avoid a rare assertion failure in the circuit build timeout code
+      if we fail to allow any circuits to actually complete. Fixes bug
+      25733; bugfix on 0.2.2.2-alpha.
+
+  o Major bugfixes (directory authorities, backport from 0.3.4.1-alpha):
+    - Avoid a crash when testing router reachability on a router that
+      could have an ed25519 ID, but which does not. Fixes bug 25415;
+      bugfix on 0.3.3.2-alpha.
+
+  o Major bugfixes (onion service, backport from 0.3.4.1-alpha):
+    - Correctly detect when onion services get disabled after HUP. Fixes
+      bug 25761; bugfix on 0.3.2.1.
+
+  o Major bugfixes (relay, denial of service, backport from 0.3.4.1-alpha):
+    - Impose a limit on circuit cell queue size. The limit can be
+      controlled by a consensus parameter. Fixes bug 25226; bugfix
+      on 0.2.4.14-alpha.
+
+  o Minor features (compatibility, backport from 0.3.4.1-alpha):
+    - Avoid some compilation warnings with recent versions of LibreSSL.
+      Closes ticket 26006.
+
+  o Minor features (continuous integration, backport from 0.3.4.1-alpha):
+    - Our .travis.yml configuration now includes support for testing the
+      results of "make distcheck". (It's not uncommon for "make check"
+      to pass but "make distcheck" to fail.) Closes ticket 25814.
+    - Our Travis CI configuration now integrates with the Coveralls
+      coverage analysis tool. Closes ticket 25818.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 1 2018 Maxmind GeoLite2 Country
+      database. Closes ticket 26104.
+
+  o Minor bugfixes (client, backport from 0.3.4.1-alpha):
+    - Don't consider Tor running as a client if the ControlPort is open,
+      but no actual client ports are open. Fixes bug 26062; bugfix
+      on 0.2.9.4-alpha.
+
+  o Minor bugfixes (correctness, client, backport from 0.3.4.1-alpha):
+    - Upon receiving a malformed connected cell, stop processing the
+      cell immediately. Previously we would mark the connection for
+      close, but continue processing the cell as if the connection were
+      open. Fixes bug 26072; bugfix on 0.2.4.7-alpha.
+
+  o Minor bugfixes (documentation, backport from 0.3.4.1-alpha):
+    - Stop saying in the manual that clients cache ipv4 dns answers from
+      exit relays. We haven't used them since 0.2.6.3-alpha, and in
+      ticket 24050 we stopped even caching them as of 0.3.2.6-alpha, but
+      we forgot to say so in the man page. Fixes bug 26052; bugfix
+      on 0.3.2.6-alpha.
+
+  o Minor bugfixes (Linux seccomp2 sandbox, backport from 0.3.4.1-alpha):
+    - Allow the nanosleep() system call, which glibc uses to implement
+      sleep() and usleep(). Fixes bug 24969; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (onion service, backport from 0.3.4.1-alpha):
+    - Fix a memory leak when a v3 onion service is configured and gets a
+      SIGHUP signal. Fixes bug 25901; bugfix on 0.3.2.1-alpha.
+    - When parsing the descriptor signature, look for the token plus an
+      extra white-space at the end. This is more correct but also will
+      allow us to support new fields that might start with "signature".
+      Fixes bug 26069; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (relay, crash, backport from 0.3.4.1-alpha):
+    - Avoid a crash when running with DirPort set but ORPort turned off.
+      Fixes a case of bug 23693; bugfix on 0.3.1.1-alpha.
+
+  o Documentation (backport from 0.3.4.1-alpha):
+    - Correct an IPv6 error in the documentation for ExitPolicy. Closes
+      ticket 25857. Patch from "CTassisF".
+
+
+Changes in version 0.3.4.1-alpha - 2018-05-17
+  Tor 0.3.4.1-alpha is the first release in the 0.3.4.x series. It
+  includes refactoring to begin reducing Tor's binary size and idle CPU
+  usage on mobile, along with prep work for new bandwidth scanners,
+  improvements to the experimental "vanguards" feature, and numerous
+  other small features and bugfixes.
+
+  o New system requirements:
+    - Tor no longer tries to support old operating systems without
+      mmap() or some local equivalent. Apparently, compilation on such
+      systems has been broken for some time, without anybody noticing or
+      complaining. Closes ticket 25398.
+
+  o Major feature (directory authority, modularization):
+    - The directory authority subsystem has been modularized. The code
+      is now located in src/or/dirauth/, and is compiled in by default.
+      To disable the module, the configure option
+      --disable-module-dirauth has been added. This module may be
+      disabled by default in some future release. Closes ticket 25610.
+
+  o Major features (main loop, CPU usage):
+    - When Tor is disabled (via DisableNetwork or via hibernation), it
+      no longer needs to run any per-second events. This change should
+      make it easier for mobile applications to disable Tor while the
+      device is sleeping, or Tor is not running. Closes ticket 26063.
+    - Tor no longer enables all of its periodic events by default.
+      Previously, Tor would enable all possible main loop events,
+      regardless of whether it needed them. Furthermore, many of these
+      events are now disabled with Tor is hibernating or DisableNetwork
+      is set. This is a big step towards reducing client CPU usage by
+      reducing the amount of wake-ups the daemon does. Closes ticket
+      25376 and 25762.
+    - The bandwidth-limitation logic has been refactored so that
+      bandwidth calculations are performed on-demand, rather than every
+      TokenBucketRefillInterval milliseconds. This change should improve
+      the granularity of our bandwidth calculations, and limit the
+      number of times that the Tor process needs to wake up when it is
+      idle. Closes ticket 25373.
+    - Move responsibility for many operations from a once-per-second
+      callback to a callback that is only scheduled as needed. Moving
+      this functionality has allowed us to disable the callback when
+      Tor's network is disabled. Once enough items are removed from our
+      once-per-second callback, we can eliminate it entirely to conserve
+      CPU when idle. The functionality removed includes: closing
+      connections, circuits, and channels (ticket 25932); consensus
+      voting (25937); flushing log callbacks (25951); honoring delayed
+      SIGNEWNYM requests (25949); rescanning the consensus cache
+      (25931); saving the state file to disk (25948); warning relay
+      operators about unreachable ports (25952); and keeping track of
+      Tor's uptime (26009).
+
+  o Major bugfixes (directory authorities, security):
+    - When directory authorities read a zero-byte bandwidth file, they
+      would previously log a warning with the contents of an
+      uninitialised buffer. They now log a warning about the empty file
+      instead. Fixes bug 26007; bugfix on 0.2.2.1-alpha.
+
+  o Major bugfixes (crash):
+    - Avoid a rare assertion failure in the circuit build timeout code
+      if we fail to allow any circuits to actually complete. Fixes bug
+      25733; bugfix on 0.2.2.2-alpha.
+
+  o Major bugfixes (directory authority):
+    - Avoid a crash when testing router reachability on a router that
+      could have an ed25519 ID, but which does not. Fixes bug 25415;
+      bugfix on 0.3.3.2-alpha.
+
+  o Major bugfixes (onion service):
+    - Correctly detect when onion services get disabled after HUP. Fixes
+      bug 25761; bugfix on 0.3.2.1.
+
+  o Major bugfixes (protover, voting):
+    - Revise Rust implementation of protover to use a more memory-
+      efficient voting algorithm and corresponding data structures, thus
+      avoiding a potential (but small impact) DoS attack where specially
+      crafted protocol strings would expand to several potential
+      megabytes in memory. In the process, several portions of code were
+      revised to be methods on new, custom types, rather than functions
+      taking interchangeable types, thus increasing type safety of the
+      module. Custom error types and handling were added as well, in
+      order to facilitate better error dismissal/handling in outside
+      crates and avoid mistakenly passing an internal error string to C
+      over the FFI boundary. Many tests were added, and some previous
+      differences between the C and Rust implementations have been
+      remedied. Fixes bug 24031; bugfix on 0.3.3.1-alpha.
+
+  o Major bugfixes (relay, denial of service):
+    - Impose a limit on circuit cell queue size. The limit can be
+      controlled by a consensus parameter. Fixes bug 25226; bugfix
+      on 0.2.4.14-alpha.
+
+  o Minor features (accounting):
+    - When Tor becomes dormant, it now uses a scheduled event to wake up
+      at the right time. Previously, we would use the per-second timer
+      to check whether to wake up, but we no longer have any per-second
+      timers enabled when the network is disabled. Closes ticket 26064.
+
+  o Minor features (code quality):
+    - Add optional spell-checking for the Tor codebase, using the
+      "misspell" program. To use this feature, run "make check-typos".
+      Closes ticket 25024.
+
+  o Minor features (compatibility):
+    - Tor now detects versions of OpenSSL 1.1.0 and later compiled with
+      the no-deprecated option, and builds correctly with them. Closes
+      tickets 19429, 19981, and 25353.
+    - Avoid some compilation warnings with recent versions of LibreSSL.
+      Closes ticket 26006.
+
+  o Minor features (compression, zstd):
+    - When running with zstd, Tor now considers using advanced functions
+      that the zstd maintainers have labeled as potentially unstable. To
+      prevent breakage, Tor will only use this functionality when the
+      runtime version of the zstd library matches the version with which
+      Tor was compiled. Closes ticket 25162.
+
+  o Minor features (configuration):
+    - The "DownloadSchedule" options have been renamed to end with
+      "DownloadInitialDelay". The old names are still allowed, but will
+      produce a warning. Comma-separated lists are still permitted for
+      these options, but all values after the first are ignored (as they
+      have been since 0.2.9). Closes ticket 23354.
+
+  o Minor features (continuous integration):
+    - Our .travis.yml configuration now includes support for testing the
+      results of "make distcheck". (It's not uncommon for "make check"
+      to pass but "make distcheck" to fail.) Closes ticket 25814.
+    - Our Travis CI configuration now integrates with the Coveralls
+      coverage analysis tool. Closes ticket 25818.
+
+  o Minor features (control port):
+    - Introduce GETINFO "current-time/{local,utc}" to return the local
+      and UTC times respectively in ISO format. This helps a controller
+      like Tor Browser detect a time-related error. Closes ticket 25511.
+      Patch by Neel Chauhan.
+    - Introduce new fields to the CIRC_BW event. There are two new
+      fields in each of the read and written directions. The DELIVERED
+      fields report the total valid data on the circuit, as measured by
+      the payload sizes of verified and error-checked relay command
+      cells. The OVERHEAD fields report the total unused bytes in each
+      of these cells. Closes ticket 25903.
+
+  o Minor features (directory authority):
+    - Directory authorities now open their key-pinning files as O_SYNC,
+      to limit their chances of accidentally writing partial lines.
+      Closes ticket 23909.
+
+  o Minor features (directory authority, forward compatibility):
+    - Make the lines of the measured bandwidth file able to contain
+      their entries in any order. Previously, the node_id entry needed
+      to come first. Closes ticket 26004.
+
+  o Minor features (entry guards):
+    - Introduce a new torrc option NumPrimaryGuards for controlling the
+      number of primary guards. Closes ticket 25843.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 1 2018 Maxmind GeoLite2 Country
+      database. Closes ticket 26104.
+
+  o Minor features (performance):
+    - Avoid a needless call to malloc() when processing an incoming
+      relay cell. Closes ticket 24914.
+    - Make our timing-wheel code run a tiny bit faster on 32-bit
+      platforms, by preferring 32-bit math to 64-bit. Closes
+      ticket 24688.
+    - Avoid a needless malloc()/free() pair every time we handle an ntor
+      handshake. Closes ticket 25150.
+
+  o Minor features (testing):
+    - Add a unit test for voting_schedule_get_start_of_next_interval().
+      Closes ticket 26014, and helps make unit test coverage
+      more deterministic.
+    - A new unittests module specifically for testing the functions in
+      the (new-ish) bridges.c module has been created with new
+      unittests, raising the code coverage percentages. Closes 25425.
+    - We now have improved testing for addressmap_get_virtual_address()
+      function. This should improve our test coverage, and make our test
+      coverage more deterministic. Closes ticket 25993.
+
+  o Minor features (timekeeping, circuit scheduling):
+    - When keeping track of how busy each circuit have been recently on
+      a given connection, use coarse-grained monotonic timers rather
+      than gettimeofday(). This change should marginally increase
+      accuracy and performance. Implements part of ticket 25927.
+
+  o Minor bugfixes (bandwidth management):
+    - Consider ourselves "low on write bandwidth" if we have exhausted
+      our write bandwidth some time in the last second. This was the
+      documented behavior before, but the actual behavior was to change
+      this value every TokenBucketRefillInterval. Fixes bug 25828;
+      bugfix on 0.2.3.5-alpha.
+
+  o Minor bugfixes (C correctness):
+    - Add a missing lock acquisition in the shutdown code of the control
+      subsystem. Fixes bug 25675; bugfix on 0.2.7.3-rc. Found by
+      Coverity; this is CID 1433643.
+
+  o Minor bugfixes (circuit path selection):
+    - Don't count path selection failures as circuit build failures.
+      This change should eliminate cases where Tor blames its guard or
+      the network for situations like insufficient microdescriptors
+      and/or overly restrictive torrc settings. Fixes bug 25705; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (client):
+    - Don't consider Tor running as a client if the ControlPort is open,
+      but no actual client ports are open. Fixes bug 26062; bugfix
+      on 0.2.9.4-alpha.
+
+  o Minor bugfixes (code style):
+    - Fixed multiple includes of transports.h in src/or/connection.c
+      Fixes bug 25261; bugfix on 0.2.5.1-alpha.
+    - Remove the unused variable n_possible from the function
+      channel_get_for_extend(). Fixes bug 25645; bugfix on 0.2.4.4-alpha
+
+  o Minor bugfixes (control interface):
+    - Respond with more human-readable error messages to GETINFO exit-
+      policy/* requests. Also, let controller know if an error is
+      transient (response code 551) or not (response code 552). Fixes
+      bug 25852; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (controller):
+    - Make CIRC_BW event reflect the total of all data sent on a
+      circuit, including padding and dropped cells. Also fix a mis-
+      counting bug when STREAM_BW events were enabled. Fixes bug 25400;
+      bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (correctness, client):
+    - Upon receiving a malformed connected cell, stop processing the cell
+      immediately.  Previously we would mark the connection for close, but
+      continue processing the cell as if the connection were open. Fixes bug
+      26072; bugfix on 0.2.4.7-alpha.
+
+  o Minor bugfixes (directory client):
+    - When unverified-consensus is verified, rename it to cached-
+      consenus. Fixes bug 4187; bugfix on 0.2.0.3-alpha.
+    - Fixed launching a certificate fetch always during the scheduled
+      periodic consensus fetch by fetching only in those cases when
+      consensus are waiting for certs. Fixes bug 24740; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (documentation):
+    - Stop saying in the manual that clients cache ipv4 dns answers from
+      exit relays. We haven't used them since 0.2.6.3-alpha, and in
+      ticket 24050 we stopped even caching them as of 0.3.2.6-alpha, but
+      we forgot to say so in the man page. Fixes bug 26052; bugfix
+      on 0.3.2.6-alpha.
+
+  o Minor bugfixes (error reporting):
+    - Improve tolerance for directory authorities with skewed clocks.
+      Previously, an authority with a clock more than 60 seconds ahead
+      could cause a client with a correct clock to warn that the
+      client's clock was behind. Now the clocks of a majority of
+      directory authorities have to be ahead of the client before this
+      warning will occur. Fixes bug 25756; bugfix on 0.2.2.25-alpha.
+
+  o Minor bugfixes (Linux seccomp2 sandbox):
+    - Allow the nanosleep() system call, which glibc uses to implement
+      sleep() and usleep(). Fixes bug 24969; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (onion service):
+    - Fix a memory leak when a v3 onion service is configured and gets a
+      SIGHUP signal. Fixes bug 25901; bugfix on 0.3.2.1-alpha.
+    - When parsing the descriptor signature, look for the token plus an
+      extra white-space at the end. This is more correct but also will
+      allow us to support new fields that might start with "signature".
+      Fixes bug 26069; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (path selection):
+    - Only select relays when they have the descriptors we prefer to use
+      for them. This change fixes a bug where we could select a relay
+      because it had _some_ descriptor, but reject it later with a
+      nonfatal assertion error because it didn't have the exact one we
+      wanted. Fixes bugs 25691 and 25692; bugfix on 0.3.3.4-alpha.
+
+  o Minor bugfixes (portability):
+    - Do not align mmap length, as it is not required by POSIX, and the
+      getpagesize function is deprecated. Fixes bug 25399; bugfix
+      on 0.1.1.23.
+
+  o Minor bugfixes (portability, FreeBSD):
+    - In have_enough_mem_for_dircache(), the variable DIRCACHE_MIN_MEM_MB
+      does not stringify on FreeBSD, so we switch to tor_asprintf().
+      Fixes bug 20887; bugfix on 0.2.8.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (relay statistics):
+    - When a relay is collecting internal statistics about how many
+      create cell requests it has seen of each type, accurately count
+      the requests from relays that temporarily fall out of the
+      consensus. (To be extra conservative, we were already ignoring
+      requests from clients in our counts, and we continue ignoring them
+      here.) Fixes bug 24910; bugfix on 0.2.4.17-rc.
+
+  o Minor bugfixes (relay, crash):
+    - Avoid a crash when running with DirPort set but ORPort turned off.
+      Fixes a case of bug 23693; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (restart-in-process):
+    - When shutting down, Tor now clears all the flags in the control.c
+      module. This should prevent a bug where authentication cookies are
+      not generated on restart. Fixes bug 25512; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (testing):
+    - When testing workqueue event-cancellation, make sure that we
+      actually cancel an event, and that cancel each event with equal
+      probability. (It was previously possible, though extremely
+      unlikely, for our event-canceling test not to cancel any events.)
+      Fixes bug 26008; bugfix on 0.2.6.3-alpha.
+    - Repeat part of the test in test_client_pick_intro() a number of
+      times, to give it consistent coverage. Fixes bug 25996; bugfix
+      on 0.3.2.1-alpha.
+    - Remove randomness from the hs_common/responsible_hsdirs test, so
+      that it always takes the same path through the function it tests.
+      Fixes bug 25997; bugfix on 0.3.2.1-alpha.
+    - Change the behavior of the "channel/outbound" test so that it
+      never causes a 10-second rollover for the EWMA circuitmux code.
+      Previously, this behavior would happen randomly, and result in
+      fluctuating test coverage. Fixes bug 25994; bugfix
+      on 0.3.3.1-alpha.
+    - Use X509_new() to allocate certificates that will be freed later
+      with X509_free(). Previously, some parts of the unit tests had
+      used tor_malloc_zero(), which is incorrect, and which caused test
+      failures on Windows when they were built with extra hardening.
+      Fixes bugs 25943 and 25944; bugfix on 0.2.8.1-alpha. Patch by
+      Marcin Cieślak.
+    - While running the circuit_timeout test, fix the PRNG to a
+      deterministic AES stream, so that the test coverage from this test
+      will itself be deterministic. Fixes bug 25995; bugfix
+      on 0.2.2.2-alpha.
+
+  o Minor bugfixes (vanguards):
+    - Allow the last hop in a vanguard circuit to be the same as our
+      first, to prevent the adversary from influencing guard node choice
+      by choice of last hop. Also prevent the creation of A - B - A
+      paths, or A - A paths, which are forbidden by relays. Fixes bug
+      25870; bugfix on 0.3.3.1-alpha.
+
+  o Code simplification and refactoring:
+    - Remove duplicate code in parse_{c,s}method_line and bootstrap
+      their functionalities into a single function. Fixes bug 6236;
+      bugfix on 0.2.3.6-alpha.
+    - We remove the PortForwsrding and PortForwardingHelper options,
+      related functions, and the port_forwarding tests. These options
+      were used by the now-deprecated Vidalia to help ordinary users
+      become Tor relays or bridges. Closes ticket 25409. Patch by
+      Neel Chauhan.
+    - In order to make the OR and dir checking function in router.c less
+      confusing we renamed some functions and
+      consider_testing_reachability() has been split into
+      router_should_check_reachability() and
+      router_do_reachability_checks(). Also we improved the documentation
+      in some functions. Closes ticket 18918.
+    - Initial work to isolate Libevent usage to a handful of modules in
+      our codebase, to simplify our call structure, and so that we can
+      more easily change event loops in the future if needed. Closes
+      ticket 23750.
+    - Introduce a function to call getsockname() and return tor_addr_t,
+      to save a little complexity throughout the codebase. Closes
+      ticket 18105.
+    - Make hsdir_index in node_t a hsdir_index_t rather than a pointer
+      as hsdir_index is always present. Also, we move hsdir_index_t into
+      or.h. Closes ticket 23094. Patch by Neel Chauhan.
+    - Merge functions used for describing nodes and suppress the
+      functions that do not allocate memory for the output buffer
+      string. NODE_DESC_BUF_LEN constant and format_node_description()
+      function cannot be used externally from router.c module anymore.
+      Closes ticket 25432. Patch by valentecaio.
+    - Our main loop has been simplified so that all important operations
+      happen inside events. Previously, some operations had to happen
+      outside the event loop, to prevent infinite sequences of event
+      activations. Closes ticket 25374.
+    - Put a SHA1 public key digest in hs_service_intro_point_t, and use
+      it in register_intro_circ() and service_intro_point_new(). This
+      prevents the digest from being re-calculated each time. Closes
+      ticket 23107. Patch by Neel Chauhan.
+    - Refactor token-bucket implementations to use a common backend.
+      Closes ticket 25766.
+    - Remove extern declaration of stats_n_seconds_working variable from
+      main, protecting its accesses with get_uptime() and reset_uptime()
+      functions. Closes ticket 25081, patch by “valentecaio”.
+    - Remove our previous logic for "cached gettimeofday()" -- our
+      coarse monotonic timers are fast enough for this purpose, and far
+      less error-prone. Implements part of ticket 25927.
+    - Remove the return value for fascist_firewall_choose_address_base(),
+      and sister functions such as fascist_firewall_choose_address_node()
+      and fascist_firewall_choose_address_rs(). Also, while we're here,
+      initialize the ap argument as leaving it uninitialized can pose a
+      security hazard. Closes ticket 24734. Patch by Neel Chauhan.
+    - Rename two fields of connection_t struct. timestamp_lastwritten is
+      renamed to timestamp_last_write_allowed and timestamp_lastread is
+      renamed to timestamp_last_read_allowed. Closes ticket 24714, patch
+      by "valentecaio".
+    - Since Tor requires C99, remove our old workaround code for libc
+      implementations where free(NULL) doesn't work. Closes ticket 24484.
+    - Use our standard rate-limiting code to deal with excessive
+      libevent failures, rather than the hand-rolled logic we had
+      before. Closes ticket 26016.
+    - We remove the return value of node_get_prim_orport() and
+      node_get_prim_dirport(), and introduce node_get_prim_orport() in
+      node_ipv6_or_preferred() and node_ipv6_dir_preferred() in order to
+      check for a null address. Closes ticket 23873. Patch by
+      Neel Chauhan.
+    - We switch to should_record_bridge_info() in
+      geoip_note_client_seen() and options_need_geoip_info() instead of
+      accessing the configuration values directly. Fixes bug 25290;
+      bugfix on 0.2.1.6-alpha. Patch by Neel Chauhan.
+
+  o Deprecated features:
+    - As we are not recommending 0.2.5 anymore, we require relays that
+      once had an ed25519 key associated with their RSA key to always
+      have that key, instead of allowing them to drop back to a version
+      that didn't support ed25519. This means they need to use a new RSA
+      key if they want to downgrade to an older version of tor without
+      ed25519. Closes ticket 20522.
+
+  o Documentation:
+    - Correct an IPv6 error in the documentation for ExitPolicy. Closes
+      ticket 25857. Patch from "CTassisF".
+
+  o Removed features:
+    - Directory authorities will no longer support voting according to
+      any consensus method before consensus method 25. This keeps
+      authorities compatible with all authorities running 0.2.9.8 and
+      later, and does not break any clients or relays. Implements ticket
+      24378 and proposal 290.
+    - The PortForwarding and PortForwardingHelper features have been
+      removed. The reasoning is, given that implementations of NAT
+      traversal protocols within common consumer grade routers are
+      frequently buggy, and that the target audience for a NAT punching
+      feature is a perhaps less-technically-inclined relay operator,
+      when the helper fails to setup traversal the problems are usually
+      deep, ugly, and very router specific, making them horrendously
+      impossible for technical support to reliable assist with, and thus
+      resulting in frustration all around. Unfortunately, relay
+      operators who would like to run relays behind NATs will need to
+      become more familiar with the port forwarding configurations on
+      their local router. Closes 25409.
+    - The TestingEnableTbEmptyEvent option has been removed. It was used
+      in testing simulations to measure how often connection buckets
+      were emptied, in order to improve our scheduling, but it has not
+      been actively used in years. Closes ticket 25760.
+    - The old "round-robin" circuit multiplexer (circuitmux)
+      implementation has been removed, along with a fairly large set of
+      code that existed to support it. It has not been the default
+      circuitmux since we introduced the "EWMA" circuitmux in 0.2.4.x,
+      but it still required an unreasonable amount of memory and CPU.
+      Closes ticket 25268.
+
+
+Changes in version 0.3.3.5-rc - 2018-04-15
+  Tor 0.3.3.5-rc fixes various bugs in earlier versions of Tor,
+  including some that could affect reliability or correctness.
+
+  This is the first release candidate in the 0.3.3 series. If we find no
+  new bugs or regression here, then the first stable 0.3.3 release will
+  be nearly identical to this one.
+
+  o Major bugfixes (security, protover, voting):
+    - Revise Rust implementation of protover to use a more memory-
+      efficient voting algorithm and corresponding data structures, thus
+      avoiding a potential memory-based DoS attack where specially
+      crafted protocol strings would expand to fill available memory.
+      Fixes bug 24031; bugfix on 0.3.3.1-alpha.
+
+  o Major bugfixes (performance, load balancing):
+    - Directory authorities no longer vote in favor of the Guard flag
+      for relays without directory support. Starting in Tor
+      0.3.0.1-alpha, clients have been avoiding using such relays in the
+      Guard position, leading to increasingly broken load balancing for
+      the 5%-or-so of Guards that don't advertise directory support.
+      Fixes bug 22310; bugfix on 0.3.0.6.
+
+  o Minor feature (continuous integration):
+    - Update the Travis CI configuration to use the stable Rust channel,
+      now that we have decided to require that. Closes ticket 25714.
+
+  o Minor features (config options):
+    - Change the way the default value for MaxMemInQueues is calculated.
+      We now use 40% of the hardware RAM if the system has 8 GB RAM or
+      more. Otherwise we use the former value of 75%. Closes
+      ticket 24782.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the April 3 2018 Maxmind GeoLite2
+      Country database. Closes ticket 25718.
+
+  o Minor bugfixes (client):
+    - When using a listed relay as a bridge, and also using
+      microdescriptors, and considering that relay as a non-bridge in a
+      circuit, treat its microdescriptor as a valid source of
+      information about that relay. This change should prevent a non-
+      fatal assertion error. Fixes bug 25691; bugfix on 0.3.3.4-alpha.
+
+  o Minor bugfixes (controller):
+    - Restore the correct operation of the RESOLVE command, which had
+      been broken since we added the ability to enable/disable DNS on
+      specific listener ports. Fixes bug 25617; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (distribution, compilation, rust):
+    - Build correctly when the rust dependencies submodule is loaded,
+      but the TOR_RUST_DEPENDENCIES environment variable is not set.
+      Fixes bug 25679; bugfix on 0.3.3.1-alpha.
+    - Actually include all of our Rust source in our source
+      distributions. (Previously, a few of the files were accidentally
+      omitted.) Fixes bug 25732; bugfix on 0.3.3.2-alpha.
+
+  o Minor bugfixes (documentation):
+    - Document that the PerConnBW{Rate,Burst} options will fall back to
+      their corresponding consensus parameters only if those parameters
+      are set. Previously we had claimed that these values would always
+      be set in the consensus. Fixes bug 25296; bugfix on 0.2.2.7-alpha.
+    - Revert a misformatting issue in the ExitPolicy documentation.
+      Fixes bug 25582; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (exit relay DNS retries):
+    - Re-attempt timed-out DNS queries 3 times before failure, since our
+      timeout is 5 seconds for them, but clients wait 10-15. Also allow
+      slightly more timeouts per resolver when an exit has multiple
+      resolvers configured. Fixes bug 21394; bugfix on 0.3.1.9.
+
+  o Minor bugfixes (onion services):
+    - Re-instate counting the client HSDir fetch circuits against the
+      MaxClientCircuitsPending rate limit. Fixes bug 24989; bugfix
+      on 0.3.3.1-alpha.
+    - Remove underscores from the _HSLayer{2,3}Nodes options. This
+      expert-user configuration can now be enabled as HSLayer{2,3}Nodes.
+      Fixes bug 25581; bugfix on 0.3.3.1-alpha
+
+  o Code simplification and refactoring:
+    - Move the list of default directory authorities to its own file.
+      Closes ticket 24854. Patch by "beastr0".
+
+  o Documentation (manpage, denial of service):
+    - Provide more detail about the denial-of-service options, by
+      listing each mitigation and explaining how they relate. Closes
+      ticket 25248.
+
+
+Changes in version 0.3.3.4-alpha - 2018-03-29
+  Tor 0.3.3.4-alpha includes various bugfixes for issues found during
+  the alpha testing of earlier releases in its series. We are
+  approaching a stable 0.3.3.4-alpha release: more testing is welcome!
+
+  o New system requirements:
+    - When built with Rust, Tor now depends on version 0.2.39 of the
+      libc crate. Closes tickets 25310 and 25664.
+
+  o Major bugfixes (relay, connection):
+    - If we have failed to connect to a relay and received a connection
+      refused, timeout, or similar error (at the TCP level), do not try
+      that same address/port again for 60 seconds after the failure has
+      occurred. Fixes bug 24767; bugfix on 0.0.6.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the March 8 2018 Maxmind GeoLite2
+      Country database. Closes ticket 25469.
+
+  o Minor features (log messages):
+    - Improve log message in the out-of-memory handler to include
+      information about memory usage from the different compression
+      backends. Closes ticket 25372.
+
+  o Minor features (sandbox):
+    - Explicitly permit the poll() system call when the Linux
+      seccomp2-based sandbox is enabled: apparently, some versions of
+      libc use poll() when calling getpwnam(). Closes ticket 25313.
+
+  o Minor bugfixes (C correctness):
+    - Fix a very unlikely (impossible, we believe) null pointer
+      dereference. Fixes bug 25629; bugfix on 0.2.9.15. Found by
+      Coverity; this is CID 1430932.
+
+  o Minor bugfixes (channel, client):
+    - Better identify client connection when reporting to the geoip
+      client cache. Fixes bug 24904; bugfix on 0.3.1.7.
+
+  o Minor bugfixes (compilation):
+    - Fix a C99 compliance issue in our configuration script that caused
+      compilation issues when compiling Tor with certain versions of
+      xtools. Fixes bug 25474; bugfix on 0.3.2.5-alpha.
+
+  o Minor bugfixes (controller, reliability):
+    - Avoid a (nonfatal) assertion failure when extending a one-hop
+      circuit from the controller to become a multihop circuit. Fixes
+      bug 24903; bugfix on 0.2.5.2-alpha.
+
+  o Major bugfixes (networking):
+    - Tor will no longer reject IPv6 address strings from Tor Browser
+      when they are passed as hostnames in SOCKS5 requests. Fixes bug
+      25036, bugfix on Tor 0.3.1.2.
+
+  o Minor bugfixes (networking):
+    - string_is_valid_hostname() will not consider IP strings to be
+      valid hostnames. Fixes bug 25055; bugfix on Tor 0.2.5.5.
+
+  o Minor bugfixes (onion service v3):
+    - Avoid an assertion failure when the next onion service
+      descriptor rotation type is out of sync with the consensus's
+      valid-after time. Instead, log a warning message with extra
+      information, so we can better hunt down the cause of this
+      assertion. Fixes bug 25306; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Avoid intermittent test failures due to a test that had relied on
+      onion service introduction point creation finishing within 5
+      seconds of real clock time. Fixes bug 25450; bugfix
+      on 0.3.1.3-alpha.
+    - Rust crates are now automatically detected and tested. Previously,
+      some crates were not tested by `make test-rust` due to a static
+      string in the `src/test/test_rust.sh` script specifying which
+      crates to test. Fixes bug 25560; bugfix on 0.3.3.3-alpha.
+
+  o Minor bugfixes (testing, benchmarks):
+    - Fix a crash when running benchmark tests on win32 systems. The
+      crash was due to a mutex that wasn't initialized before logging
+      and options were initialized. Fixes bug 25479; bugfix
+      on 0.3.3.3-alpha.
+
+  o Minor bugfixes (warnings, ipv6):
+    - Avoid a bug warning that could occur when trying to connect to a
+      relay over IPv6. This warning would occur on a Tor instance that
+      downloads router descriptors, but prefers to use microdescriptors.
+      Fixes bug 25213; bugfix on 0.3.3.1-alpha.
+
+  o Code simplification and refactoring:
+    - Remove the old (deterministic) directory retry logic entirely:
+      We've used exponential backoff exclusively for some time. Closes
+      ticket 23814.
+
+  o Documentation:
+    - Improved the documentation of AccountingStart parameter. Closes
+      ticket 23635.
+    - Update the documentation for "Log" to include the current list of
+      logging domains. Closes ticket 25378.
+
+
+Changes in version 0.3.1.10 - 2018-03-03
+  Tor 0.3.1.10 backports a number of bugfixes, including important fixes for
+  security issues.
+
+  It includes an important security fix for a remote crash attack
+  against directory authorities, tracked as TROVE-2018-001.
+
+  This release also backports our new system for improved resistance to
+  denial-of-service attacks against relays.
+
+  This release also fixes several minor bugs and annoyances from
+  earlier releases.
+
+  All directory authorities should upgrade to one of the versions
+  released today. Relays running 0.3.1.x may wish to update to one of
+  the versions released today, for the DoS mitigations.
+
+  Please note: according to our release calendar, Tor 0.3.1 will no
+  longer be supported after 1 July 2018. If you will be running Tor
+  after that date, you should make sure to plan to upgrade to the latest
+  stable version, or downgrade to 0.2.9 (which will receive long-term
+  support).
+
+  o Major bugfixes (denial-of-service, directory authority, backport from 0.3.3.3-alpha):
+    - Fix a protocol-list handling bug that could be used to remotely crash
+      directory authorities with a null-pointer exception. Fixes bug 25074;
+      bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001 and
+      CVE-2018-0490.
+
+  o Major features (denial-of-service mitigation, backport from 0.3.3.2-alpha):
+    - Give relays some defenses against the recent network overload. We
+      start with three defenses (default parameters in parentheses).
+      First: if a single client address makes too many concurrent
+      connections (>100), hang up on further connections. Second: if a
+      single client address makes circuits too quickly (more than 3 per
+      second, with an allowed burst of 90) while also having too many
+      connections open (3), refuse new create cells for the next while
+      (1-2 hours). Third: if a client asks to establish a rendezvous
+      point to you directly, ignore the request. These defenses can be
+      manually controlled by new torrc options, but relays will also
+      take guidance from consensus parameters, so there's no need to
+      configure anything manually. Implements ticket 24902.
+
+  o Minor features (linux seccomp2 sandbox, backport from 0.3.2.5-alpha):
+    - Update the sandbox rules so that they should now work correctly
+      with Glibc 2.26. Closes ticket 24315.
+
+  o Major bugfixes (onion services, retry behavior, backport from 0.3.3.1-alpha):
+    - Fix an "off by 2" error in counting rendezvous failures on the
+      onion service side. While we thought we would stop the rendezvous
+      attempt after one failed circuit, we were actually making three
+      circuit attempts before giving up. Now switch to a default of 2,
+      and allow the consensus parameter "hs_service_max_rdv_failures" to
+      override. Fixes bug 24895; bugfix on 0.0.6.
+
+  o Major bugfixes (protocol versions, backport from 0.3.3.2-alpha):
+    - Add Link protocol version 5 to the supported protocols list. Fixes
+      bug 25070; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (relay, backport from 0.3.3.1-alpha):
+    - Fix a set of false positives where relays would consider
+      connections to other relays as being client-only connections (and
+      thus e.g. deserving different link padding schemes) if those
+      relays fell out of the consensus briefly. Now we look only at the
+      initial handshake and whether the connection authenticated as a
+      relay. Fixes bug 24898; bugfix on 0.3.1.1-alpha.
+
+  o Minor features (denial-of-service avoidance, backport from 0.3.3.2-alpha):
+    - Make our OOM handler aware of the geoip client history cache so it
+      doesn't fill up the memory. This check is important for IPv6 and
+      our DoS mitigation subsystem. Closes ticket 25122.
+
+  o Minor feature (relay statistics, backport from 0.3.2.6-alpha):
+    - Change relay bandwidth reporting stats interval from 4 hours to 24
+      hours in order to reduce the efficiency of guard discovery
+      attacks. Fixes ticket 23856.
+
+  o Minor features (compatibility, OpenSSL, backport from 0.3.3.3-alpha):
+    - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released.
+      Previous versions of Tor would not have worked with OpenSSL 1.1.1,
+      since they neither disabled TLS 1.3 nor enabled any of the
+      ciphersuites it requires. Now we enable the TLS 1.3 ciphersuites.
+      Closes ticket 24978.
+
+  o Minor features (fallback directory mirrors, backport from 0.3.2.9):
+    - The fallback directory list has been re-generated based on the
+      current status of the network. Tor uses fallback directories to
+      bootstrap when it doesn't yet have up-to-date directory
+      information. Closes ticket 24801.
+    - Make the default DirAuthorityFallbackRate 0.1, so that clients
+      prefer to bootstrap from fallback directory mirrors. This is a
+      follow-up to 24679, which removed weights from the default
+      fallbacks. Implements ticket 24681.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 7 2018 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfix (channel connection, backport from 0.3.3.2-alpha):
+    - Use the actual observed address of an incoming relay connection,
+      not the canonical address of the relay from its descriptor, when
+      making decisions about how to handle the incoming connection.
+      Fixes bug 24952; bugfix on 0.2.4.11-alpha. Patch by "ffmancera".
+
+  o Minor bugfix (directory authority, backport from 0.3.3.2-alpha):
+    - Directory authorities, when refusing a descriptor from a rejected
+      relay, now explicitly tell the relay (in its logs) to set a valid
+      ContactInfo address and contact the bad-relays@ mailing list.
+      Fixes bug 25170; bugfix on 0.2.9.1.
+
+  o Minor bugfixes (address selection, backport from 0.3.2.9):
+    - When the fascist_firewall_choose_address_ functions don't find a
+      reachable address, set the returned address to the null address
+      and port. This is a precautionary measure, because some callers do
+      not check the return value. Fixes bug 24736; bugfix
+      on 0.2.8.2-alpha.
+
+  o Major bugfixes (bootstrapping, backport from 0.3.2.5-alpha):
+    - Fetch descriptors aggressively whenever we lack enough to build
+      circuits, regardless of how many descriptors we are missing.
+      Previously, we would delay launching the fetch when we had fewer
+      than 15 missing descriptors, even if some of those descriptors
+      were blocking circuits from building. Fixes bug 23985; bugfix on
+      0.1.1.11-alpha. The effects of this bug became worse in
+      0.3.0.3-alpha, when we began treating missing descriptors from our
+      primary guards as a reason to delay circuits.
+    - Don't try fetching microdescriptors from relays that have failed
+      to deliver them in the past. Fixes bug 23817; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.2.7-rc):
+    - Fix a signed/unsigned comparison warning introduced by our fix to
+      TROVE-2017-009. Fixes bug 24480; bugfix on 0.2.5.16.
+
+  o Minor bugfixes (control port, linux seccomp2 sandbox, backport from 0.3.2.5-alpha):
+    - Avoid a crash when attempting to use the seccomp2 sandbox together
+      with the OwningControllerProcess feature. Fixes bug 24198; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (denial-of-service, backport from 0.3.3.3-alpha):
+    - Fix a possible crash on malformed consensus. If a consensus had
+      contained an unparseable protocol line, it could have made clients
+      and relays crash with a null-pointer exception. To exploit this
+      issue, however, an attacker would need to be able to subvert the
+      directory authority system. Fixes bug 25251; bugfix on
+      0.2.9.4-alpha. Also tracked as TROVE-2018-004.
+
+  o Minor bugfixes (directory cache, backport from 0.3.2.5-alpha):
+    - Recover better from empty or corrupt files in the consensus cache
+      directory. Fixes bug 24099; bugfix on 0.3.1.1-alpha.
+    - When a consensus diff calculation is only partially successful,
+      only record the successful parts as having succeeded. Partial
+      success can happen if (for example) one compression method fails
+      but the others succeed. Previously we misrecorded all the
+      calculations as having succeeded, which would later cause a
+      nonfatal assertion failure. Fixes bug 24086; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (entry guards, backport from 0.3.2.3-alpha):
+    - Tor now updates its guard state when it reads a consensus
+      regardless of whether it's missing descriptors. That makes tor use
+      its primary guards to fetch descriptors in some edge cases where
+      it would previously have used fallback directories. Fixes bug
+      23862; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (logging, backport from 0.3.3.2-alpha):
+    - Don't treat inability to store a cached consensus object as a bug:
+      it can happen normally when we are out of disk space. Fixes bug
+      24859; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (memory usage, backport from 0.3.2.8-rc):
+    - When queuing DESTROY cells on a channel, only queue the circuit-id
+      and reason fields: not the entire 514-byte cell. This fix should
+      help mitigate any bugs or attacks that fill up these queues, and
+      free more RAM for other uses. Fixes bug 24666; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (network layer, backport from 0.3.2.5-alpha):
+    - When closing a connection via close_connection_immediately(), we
+      mark it as "not blocked on bandwidth", to prevent later calls from
+      trying to unblock it, and give it permission to read. This fixes a
+      backtrace warning that can happen on relays under various
+      circumstances. Fixes bug 24167; bugfix on 0.1.0.1-rc.
+
+  o Minor bugfixes (path selection, backport from 0.3.2.4-alpha):
+    - When selecting relays by bandwidth, avoid a rounding error that
+      could sometimes cause load to be imbalanced incorrectly.
+      Previously, we would always round upwards; now, we round towards
+      the nearest integer. This had the biggest effect when a relay's
+      weight adjustments should have given it weight 0, but it got
+      weight 1 instead. Fixes bug 23318; bugfix on 0.2.4.3-alpha.
+    - When calculating the fraction of nodes that have descriptors, and
+      all nodes in the network have zero bandwidths, count the number of
+      nodes instead. Fixes bug 23318; bugfix on 0.2.4.10-alpha.
+    - Actually log the total bandwidth in compute_weighted_bandwidths().
+      Fixes bug 24170; bugfix on 0.2.4.3-alpha.
+
+  o Minor bugfixes (performance, fragile-hardening, backport from 0.3.3.1-alpha):
+    - Improve the performance of our consensus-diff application code
+      when Tor is built with the --enable-fragile-hardening option set.
+      Fixes bug 24826; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (OSX, backport from 0.3.3.1-alpha):
+    - Don't exit the Tor process if setrlimit() fails to change the file
+      limit (which can happen sometimes on some versions of OSX). Fixes
+      bug 21074; bugfix on 0.0.9pre5.
+
+  o Minor bugfixes (portability, msvc, backport from 0.3.2.9):
+    - Fix a bug in the bit-counting parts of our timing-wheel code on
+      MSVC. (Note that MSVC is still not a supported build platform, due
+      to cyptographic timing channel risks.) Fixes bug 24633; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (relay, partial backport):
+    - Make the internal channel_is_client() function look at what sort
+      of connection handshake the other side used, rather than whether
+      the other side ever sent a create_fast cell to us. Backports part
+      of the fixes from bugs 22805 and 24898.
+
+  o Minor bugfixes (spec conformance, backport from 0.3.3.3-alpha):
+    - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on
+      0.2.9.4-alpha.
+    - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249;
+      bugfix on 0.2.9.4-alpha.
+
+  o Code simplification and refactoring (backport from 0.3.3.3-alpha):
+    - Update the "rust dependencies" submodule to be a project-level
+      repository, rather than a user repository. Closes ticket 25323.
+
+
+Changes in version 0.2.9.15 - 2018-03-03
+  Tor 0.2.9.15 backports important security and stability bugfixes from
+  later Tor releases.
+
+  It includes an important security fix for a remote crash attack
+  against directory authorities, tracked as TROVE-2018-001.
+
+  This release also backports our new system for improved resistance to
+  denial-of-service attacks against relays.
+
+  This release also fixes several minor bugs and annoyances from
+  earlier releases.
+
+  All directory authorities should upgrade to one of the versions
+  released today. Relays running 0.2.9.x may wish to update to one of
+  the versions released today, for the DoS mitigations.
+
+  o Major bugfixes (denial-of-service, directory authority, backport from 0.3.3.3-alpha):
+    - Fix a protocol-list handling bug that could be used to remotely crash
+      directory authorities with a null-pointer exception. Fixes bug 25074;
+      bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001 and
+      CVE-2018-0490.
+
+  o Major features (denial-of-service mitigation):
+    - Give relays some defenses against the recent network overload. We
+      start with three defenses (default parameters in parentheses).
+      First: if a single client address makes too many concurrent
+      connections (>100), hang up on further connections. Second: if a
+      single client address makes circuits too quickly (more than 3 per
+      second, with an allowed burst of 90) while also having too many
+      connections open (3), refuse new create cells for the next while
+      (1-2 hours). Third: if a client asks to establish a rendezvous
+      point to you directly, ignore the request. These defenses can be
+      manually controlled by new torrc options, but relays will also
+      take guidance from consensus parameters, so there's no need to
+      configure anything manually. Implements ticket 24902.
+
+  o Major bugfixes (bootstrapping):
+    - Fetch descriptors aggressively whenever we lack enough to build
+      circuits, regardless of how many descriptors we are missing.
+      Previously, we would delay launching the fetch when we had fewer
+      than 15 missing descriptors, even if some of those descriptors
+      were blocking circuits from building. Fixes bug 23985; bugfix on
+      0.1.1.11-alpha. The effects of this bug became worse in
+      0.3.0.3-alpha, when we began treating missing descriptors from our
+      primary guards as a reason to delay circuits.
+
+  o Major bugfixes (onion services, retry behavior):
+    - Fix an "off by 2" error in counting rendezvous failures on the
+      onion service side. While we thought we would stop the rendezvous
+      attempt after one failed circuit, we were actually making three
+      circuit attempts before giving up. Now switch to a default of 2,
+      and allow the consensus parameter "hs_service_max_rdv_failures" to
+      override. Fixes bug 24895; bugfix on 0.0.6.
+
+  o Minor feature (relay statistics):
+    - Change relay bandwidth reporting stats interval from 4 hours to 24
+      hours in order to reduce the efficiency of guard discovery
+      attacks. Fixes ticket 23856.
+
+  o Minor features (compatibility, OpenSSL):
+    - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released.
+      Previous versions of Tor would not have worked with OpenSSL 1.1.1,
+      since they neither disabled TLS 1.3 nor enabled any of the
+      ciphersuites it requires. Now we enable the TLS 1.3 ciphersuites.
+      Closes ticket 24978.
+
+  o Minor features (denial-of-service avoidance):
+    - Make our OOM handler aware of the geoip client history cache so it
+      doesn't fill up the memory. This check is important for IPv6 and
+      our DoS mitigation subsystem. Closes ticket 25122.
+
+  o Minor features (fallback directory mirrors):
+    - The fallback directory list has been re-generated based on the
+      current status of the network. Tor uses fallback directories to
+      bootstrap when it doesn't yet have up-to-date directory
+      information. Closes ticket 24801.
+    - Make the default DirAuthorityFallbackRate 0.1, so that clients
+      prefer to bootstrap from fallback directory mirrors. This is a
+      follow-up to 24679, which removed weights from the default
+      fallbacks. Implements ticket 24681.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 7 2018 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (linux seccomp2 sandbox):
+    - Update the sandbox rules so that they should now work correctly
+      with Glibc 2.26. Closes ticket 24315.
+
+  o Minor bugfix (channel connection):
+    - Use the actual observed address of an incoming relay connection,
+      not the canonical address of the relay from its descriptor, when
+      making decisions about how to handle the incoming connection.
+      Fixes bug 24952; bugfix on 0.2.4.11-alpha. Patch by "ffmancera".
+
+  o Minor bugfix (directory authority):
+    - Directory authorities, when refusing a descriptor from a rejected
+      relay, now explicitly tell the relay (in its logs) to set a valid
+      ContactInfo address and contact the bad-relays@ mailing list.
+      Fixes bug 25170; bugfix on 0.2.9.1.
+
+  o Minor bugfixes (address selection):
+    - When the fascist_firewall_choose_address_ functions don't find a
+      reachable address, set the returned address to the null address
+      and port. This is a precautionary measure, because some callers do
+      not check the return value. Fixes bug 24736; bugfix
+      on 0.2.8.2-alpha.
+
+  o Minor bugfixes (compilation):
+    - Fix a signed/unsigned comparison warning introduced by our fix to
+      TROVE-2017-009. Fixes bug 24480; bugfix on 0.2.5.16.
+
+  o Minor bugfixes (control port, linux seccomp2 sandbox):
+    - Avoid a crash when attempting to use the seccomp2 sandbox together
+      with the OwningControllerProcess feature. Fixes bug 24198; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (denial-of-service, backport from 0.3.3.3-alpha):
+    - Fix a possible crash on malformed consensus. If a consensus had
+      contained an unparseable protocol line, it could have made clients
+      and relays crash with a null-pointer exception. To exploit this
+      issue, however, an attacker would need to be able to subvert the
+      directory authority system. Fixes bug 25251; bugfix on
+      0.2.9.4-alpha. Also tracked as TROVE-2018-004.
+
+  o Minor bugfixes (memory usage):
+    - When queuing DESTROY cells on a channel, only queue the circuit-id
+      and reason fields: not the entire 514-byte cell. This fix should
+      help mitigate any bugs or attacks that fill up these queues, and
+      free more RAM for other uses. Fixes bug 24666; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (network layer):
+    - When closing a connection via close_connection_immediately(), we
+      mark it as "not blocked on bandwidth", to prevent later calls from
+      trying to unblock it, and give it permission to read. This fixes a
+      backtrace warning that can happen on relays under various
+      circumstances. Fixes bug 24167; bugfix on 0.1.0.1-rc.
+
+  o Minor bugfixes (OSX):
+    - Don't exit the Tor process if setrlimit() fails to change the file
+      limit (which can happen sometimes on some versions of OSX). Fixes
+      bug 21074; bugfix on 0.0.9pre5.
+
+  o Minor bugfixes (path selection):
+    - When selecting relays by bandwidth, avoid a rounding error that
+      could sometimes cause load to be imbalanced incorrectly.
+      Previously, we would always round upwards; now, we round towards
+      the nearest integer. This had the biggest effect when a relay's
+      weight adjustments should have given it weight 0, but it got
+      weight 1 instead. Fixes bug 23318; bugfix on 0.2.4.3-alpha.
+    - When calculating the fraction of nodes that have descriptors, and
+      all nodes in the network have zero bandwidths, count the number of
+      nodes instead. Fixes bug 23318; bugfix on 0.2.4.10-alpha.
+    - Actually log the total bandwidth in compute_weighted_bandwidths().
+      Fixes bug 24170; bugfix on 0.2.4.3-alpha.
+
+  o Minor bugfixes (portability, msvc):
+    - Fix a bug in the bit-counting parts of our timing-wheel code on
+      MSVC. (Note that MSVC is still not a supported build platform, due
+      to cryptographic timing channel risks.) Fixes bug 24633; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (relay):
+    - Make the internal channel_is_client() function look at what sort
+      of connection handshake the other side used, rather than whether
+      the other side ever sent a create_fast cell to us. Backports part
+      of the fixes from bugs 22805 and 24898.
+
+  o Minor bugfixes (spec conformance, backport from 0.3.3.3-alpha):
+    - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on
+      0.2.9.4-alpha.
+    - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249;
+      bugfix on 0.2.9.4-alpha.
+
+
+Changes in version 0.3.2.10 - 2018-03-03
+  Tor 0.3.2.10 is the second stable release in the 0.3.2 series. It
+  backports a number of bugfixes, including important fixes for security
+  issues.
+
+  It includes an important security fix for a remote crash attack
+  against directory authorities, tracked as TROVE-2018-001.
+
+  Additionally, it backports a fix for a bug whose severity we have
+  upgraded: Bug 24700, which was fixed in 0.3.3.2-alpha, can be remotely
+  triggered in order to crash relays with a use-after-free pattern. As
+  such, we are now tracking that bug as TROVE-2018-002 and
+  CVE-2018-0491, and backporting it to earlier releases. This bug
+  affected versions 0.3.2.1-alpha through 0.3.2.9, as well as version
+  0.3.3.1-alpha.
+
+  This release also backports our new system for improved resistance to
+  denial-of-service attacks against relays.
+
+  This release also fixes several minor bugs and annoyances from
+  earlier releases.
+
+  Relays running 0.3.2.x SHOULD upgrade to one of the versions released
+  today, for the fix to TROVE-2018-002. Directory authorities should
+  also upgrade. (Relays on earlier versions might want to update too for
+  the DoS mitigations.)
+
+  o Major bugfixes (denial-of-service, directory authority, backport from 0.3.3.3-alpha):
+    - Fix a protocol-list handling bug that could be used to remotely crash
+      directory authorities with a null-pointer exception. Fixes bug 25074;
+      bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001 and
+      CVE-2018-0490.
+
+  o Major bugfixes (scheduler, KIST, denial-of-service, backport from 0.3.3.2-alpha):
+    - Avoid adding the same channel twice in the KIST scheduler pending
+      list, which could lead to remote denial-of-service use-after-free
+      attacks against relays. Fixes bug 24700; bugfix on 0.3.2.1-alpha.
+
+  o Major features (denial-of-service mitigation, backport from 0.3.3.2-alpha):
+    - Give relays some defenses against the recent network overload. We
+      start with three defenses (default parameters in parentheses).
+      First: if a single client address makes too many concurrent
+      connections (>100), hang up on further connections. Second: if a
+      single client address makes circuits too quickly (more than 3 per
+      second, with an allowed burst of 90) while also having too many
+      connections open (3), refuse new create cells for the next while
+      (1-2 hours). Third: if a client asks to establish a rendezvous
+      point to you directly, ignore the request. These defenses can be
+      manually controlled by new torrc options, but relays will also
+      take guidance from consensus parameters, so there's no need to
+      configure anything manually. Implements ticket 24902.
+
+  o Major bugfixes (onion services, retry behavior, backport from 0.3.3.1-alpha):
+    - Fix an "off by 2" error in counting rendezvous failures on the
+      onion service side. While we thought we would stop the rendezvous
+      attempt after one failed circuit, we were actually making three
+      circuit attempts before giving up. Now switch to a default of 2,
+      and allow the consensus parameter "hs_service_max_rdv_failures" to
+      override. Fixes bug 24895; bugfix on 0.0.6.
+    - New-style (v3) onion services now obey the "max rendezvous circuit
+      attempts" logic. Previously they would make as many rendezvous
+      circuit attempts as they could fit in the MAX_REND_TIMEOUT second
+      window before giving up. Fixes bug 24894; bugfix on 0.3.2.1-alpha.
+
+  o Major bugfixes (protocol versions, backport from 0.3.3.2-alpha):
+    - Add Link protocol version 5 to the supported protocols list. Fixes
+      bug 25070; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (relay, backport from 0.3.3.1-alpha):
+    - Fix a set of false positives where relays would consider
+      connections to other relays as being client-only connections (and
+      thus e.g. deserving different link padding schemes) if those
+      relays fell out of the consensus briefly. Now we look only at the
+      initial handshake and whether the connection authenticated as a
+      relay. Fixes bug 24898; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (scheduler, consensus, backport from 0.3.3.2-alpha):
+    - The scheduler subsystem was failing to promptly notice changes in
+      consensus parameters, making it harder to switch schedulers
+      network-wide. Fixes bug 24975; bugfix on 0.3.2.1-alpha.
+
+  o Minor features (denial-of-service avoidance, backport from 0.3.3.2-alpha):
+    - Make our OOM handler aware of the geoip client history cache so it
+      doesn't fill up the memory. This check is important for IPv6 and
+      our DoS mitigation subsystem. Closes ticket 25122.
+
+  o Minor features (compatibility, OpenSSL, backport from 0.3.3.3-alpha):
+    - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released.
+      Previous versions of Tor would not have worked with OpenSSL 1.1.1,
+      since they neither disabled TLS 1.3 nor enabled any of the
+      ciphersuites it requires. Now we enable the TLS 1.3 ciphersuites.
+      Closes ticket 24978.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 7 2018 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (logging, diagnostic, backport from 0.3.3.2-alpha):
+    - When logging a failure to create an onion service's descriptor,
+      also log what the problem with the descriptor was. Diagnostic
+      for ticket 24972.
+
+  o Minor bugfix (channel connection, backport from 0.3.3.2-alpha):
+    - Use the actual observed address of an incoming relay connection,
+      not the canonical address of the relay from its descriptor, when
+      making decisions about how to handle the incoming connection.
+      Fixes bug 24952; bugfix on 0.2.4.11-alpha. Patch by "ffmancera".
+
+  o Minor bugfixes (denial-of-service, backport from 0.3.3.3-alpha):
+    - Fix a possible crash on malformed consensus. If a consensus had
+      contained an unparseable protocol line, it could have made clients
+      and relays crash with a null-pointer exception. To exploit this
+      issue, however, an attacker would need to be able to subvert the
+      directory authority system. Fixes bug 25251; bugfix on
+      0.2.9.4-alpha. Also tracked as TROVE-2018-004.
+
+  o Minor bugfix (directory authority, backport from 0.3.3.2-alpha):
+    - Directory authorities, when refusing a descriptor from a rejected
+      relay, now explicitly tell the relay (in its logs) to set a valid
+      ContactInfo address and contact the bad-relays@ mailing list.
+      Fixes bug 25170; bugfix on 0.2.9.1.
+
+  o Minor bugfixes (build, rust, backport from 0.3.3.1-alpha):
+    - When building with Rust on OSX, link against libresolv, to work
+      around the issue at https://github.com/rust-lang/rust/issues/46797.
+      Fixes bug 24652; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.3.3.2-alpha):
+    - Remove a BUG() statement when a client fetches an onion descriptor
+      that has a lower revision counter than the one in its cache. This
+      can happen in normal circumstances due to HSDir desync. Fixes bug
+      24976; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (logging, backport from 0.3.3.2-alpha):
+    - Don't treat inability to store a cached consensus object as a bug:
+      it can happen normally when we are out of disk space. Fixes bug
+      24859; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (performance, fragile-hardening, backport from 0.3.3.1-alpha):
+    - Improve the performance of our consensus-diff application code
+      when Tor is built with the --enable-fragile-hardening option set.
+      Fixes bug 24826; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (OSX, backport from 0.3.3.1-alpha):
+    - Don't exit the Tor process if setrlimit() fails to change the file
+      limit (which can happen sometimes on some versions of OSX). Fixes
+      bug 21074; bugfix on 0.0.9pre5.
+
+  o Minor bugfixes (spec conformance, backport from 0.3.3.3-alpha):
+    - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on
+      0.2.9.4-alpha.
+    - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249;
+      bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.3.1-alpha):
+    - Fix a memory leak in the scheduler/loop_kist unit test. Fixes bug
+      25005; bugfix on 0.3.2.7-rc.
+
+  o Minor bugfixes (v3 onion services, backport from 0.3.3.2-alpha):
+    - Look at the "HSRend" protocol version, not the "HSDir" protocol
+      version, when deciding whether a consensus entry can support the
+      v3 onion service protocol as a rendezvous point. Fixes bug 25105;
+      bugfix on 0.3.2.1-alpha.
+
+  o Code simplification and refactoring (backport from 0.3.3.3-alpha):
+    - Update the "rust dependencies" submodule to be a project-level
+      repository, rather than a user repository. Closes ticket 25323.
+
+  o Documentation (backport from 0.3.3.1-alpha)
+    - Document that operators who run more than one relay or bridge are
+      expected to set MyFamily and ContactInfo correctly. Closes
+      ticket 24526.
+
+
+Changes in version 0.3.3.3-alpha - 2018-03-03
+  Tor 0.3.3.3-alpha is the third alpha release for the 0.3.3.x series.
+  It includes an important security fix for a remote crash attack
+  against directory authorities tracked as TROVE-2018-001.
+
+  Additionally, with this release, we are upgrading the severity of a
+  bug fixed in 0.3.3.2-alpha. Bug 24700, which was fixed in
+  0.3.3.2-alpha, can be remotely triggered in order to crash relays with
+  a use-after-free pattern. As such, we are now tracking that bug as
+  TROVE-2018-002 and CVE-2018-0491. This bug affected versions
+  0.3.2.1-alpha through 0.3.2.9, as well as 0.3.3.1-alpha.
+
+  This release also fixes several minor bugs and annoyances from
+  earlier releases.
+
+  Relays running 0.3.2.x should upgrade to one of the versions released
+  today, for the fix to TROVE-2018-002. Directory authorities should
+  also upgrade. (Relays on earlier versions might want to update too for
+  the DoS mitigations.)
+
+  o Major bugfixes (denial-of-service, directory authority):
+    - Fix a protocol-list handling bug that could be used to remotely crash
+      directory authorities with a null-pointer exception. Fixes bug 25074;
+      bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001 and
+      CVE-2018-0490.
+
+  o Minor features (compatibility, OpenSSL):
+    - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released.
+      Previous versions of Tor would not have worked with OpenSSL 1.1.1,
+      since they neither disabled TLS 1.3 nor enabled any of the
+      ciphersuites it requires. Now we enable the TLS 1.3 ciphersuites.
+      Closes ticket 24978.
+
+  o Minor features (logging):
+    - Clarify the log messages produced when getrandom() or a related
+      entropy-generation mechanism gives an error. Closes ticket 25120.
+
+  o Minor features (testing):
+    - Add a "make test-rust" target to run the rust tests only. Closes
+      ticket 25071.
+
+  o Minor bugfixes (denial-of-service):
+    - Fix a possible crash on malformed consensus. If a consensus had
+      contained an unparseable protocol line, it could have made clients
+      and relays crash with a null-pointer exception. To exploit this
+      issue, however, an attacker would need to be able to subvert the
+      directory authority system. Fixes bug 25251; bugfix on
+      0.2.9.4-alpha. Also tracked as TROVE-2018-004.
+
+  o Minor bugfixes (DoS mitigation):
+    - Add extra safety checks when refilling the circuit creation bucket
+      to ensure we never set a value above the allowed maximum burst.
+      Fixes bug 25202; bugfix on 0.3.3.2-alpha.
+    - When a new consensus arrives, don't update our DoS-mitigation
+      parameters if we aren't a public relay. Fixes bug 25223; bugfix
+      on 0.3.3.2-alpha.
+
+  o Minor bugfixes (man page, SocksPort):
+    - Remove dead code from the old "SocksSocket" option, and rename
+      SocksSocketsGroupWritable to UnixSocksGroupWritable. The old option
+      still works, but is deprecated. Fixes bug 24343; bugfix on 0.2.6.3.
+
+  o Minor bugfixes (performance):
+    - Reduce the number of circuits that will be opened at once during
+      the circuit build timeout phase. This is done by increasing the
+      idle timeout to 3 minutes, and lowering the maximum number of
+      concurrent learning circuits to 10. Fixes bug 24769; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (spec conformance):
+    - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on
+      0.2.9.4-alpha.
+    - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249;
+      bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (spec conformance, rust):
+    - Resolve a denial-of-service issue caused by an infinite loop in
+      the rust protover code. Fixes bug 25250, bugfix on 0.3.3.1-alpha.
+      Also tracked as TROVE-2018-003.
+
+  o Code simplification and refactoring:
+    - Update the "rust dependencies" submodule to be a project-level
+      repository, rather than a user repository. Closes ticket 25323.
+
+
+Changes in version 0.3.3.2-alpha - 2018-02-10
+  Tor 0.3.3.2-alpha is the second alpha in the 0.3.3.x series. It
+  introduces a mechanism to handle the high loads that many relay
+  operators have been reporting recently. It also fixes several bugs in
+  older releases. If this new code proves reliable, we plan to backport
+  it to older supported release series.
+
+  o Major features (denial-of-service mitigation):
+    - Give relays some defenses against the recent network overload. We
+      start with three defenses (default parameters in parentheses).
+      First: if a single client address makes too many concurrent
+      connections (>100), hang up on further connections. Second: if a
+      single client address makes circuits too quickly (more than 3 per
+      second, with an allowed burst of 90) while also having too many
+      connections open (3), refuse new create cells for the next while
+      (1-2 hours). Third: if a client asks to establish a rendezvous
+      point to you directly, ignore the request. These defenses can be
+      manually controlled by new torrc options, but relays will also
+      take guidance from consensus parameters, so there's no need to
+      configure anything manually. Implements ticket 24902.
+
+  o Major bugfixes (netflow padding):
+    - Stop adding unneeded channel padding right after we finish
+      flushing to a connection that has been trying to flush for many
+      seconds. Instead, treat all partial or complete flushes as
+      activity on the channel, which will defer the time until we need
+      to add padding. This fix should resolve confusing and scary log
+      messages like "Channel padding timeout scheduled 221453ms in the
+      past." Fixes bug 22212; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (protocol versions):
+    - Add Link protocol version 5 to the supported protocols list. Fixes
+      bug 25070; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (scheduler, consensus):
+    - The scheduler subsystem was failing to promptly notice changes in
+      consensus parameters, making it harder to switch schedulers
+      network-wide. Fixes bug 24975; bugfix on 0.3.2.1-alpha.
+
+  o Minor features (denial-of-service avoidance):
+    - Make our OOM handler aware of the geoip client history cache so it
+      doesn't fill up the memory. This check is important for IPv6 and
+      our DoS mitigation subsystem. Closes ticket 25122.
+
+  o Minor features (directory authority):
+    - When directory authorities are unable to add signatures to a
+      pending consensus, log the reason why. Closes ticket 24849.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 7 2018 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (logging, diagnostic):
+    - When logging a failure to create an onion service's descriptor,
+      also log what the problem with the descriptor was. Diagnostic for
+      ticket 24972.
+
+  o Minor bugfix (channel connection):
+    - Use the actual observed address of an incoming relay connection,
+      not the canonical address of the relay from its descriptor, when
+      making decisions about how to handle the incoming connection.
+      Fixes bug 24952; bugfix on 0.2.4.11-alpha. Patch by "ffmancera".
+
+  o Minor bugfix (directory authority):
+    - Directory authorities, when refusing a descriptor from a rejected
+      relay, now explicitly tell the relay (in its logs) to set a valid
+      ContactInfo address and contact the bad-relays@ mailing list.
+      Fixes bug 25170; bugfix on 0.2.9.1.
+
+  o Minor bugfixes (all versions of Tor):
+    - Use the "misspell" tool to detect and fix typos throughout the
+      source code. Fixes bug 23650; bugfix on various versions of Tor.
+      Patch from Deepesh Pathak.
+
+  o Minor bugfixes (circuit, cannibalization):
+    - Don't cannibalize preemptively-built circuits if we no longer
+      recognize their first hop. This situation can happen if our Guard
+      relay went off the consensus after the circuit was created. Fixes
+      bug 24469; bugfix on 0.0.6.
+
+  o Minor bugfixes (correctness):
+    - Remove a nonworking, unnecessary check to see whether a circuit
+      hop's identity digest was set when the circuit failed. Fixes bug
+      24927; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (logging):
+    - Don't treat inability to store a cached consensus object as a bug:
+      it can happen normally when we are out of disk space. Fixes bug
+      24859; bugfix on 0.3.1.1-alpha.
+    - Fix a (mostly harmless) race condition when invoking
+      LOG_PROTOCOL_WARN message from a subthread while the torrc options
+      are changing. Fixes bug 23954; bugfix on 0.1.1.9-alpha.
+
+  o Minor bugfixes (onion services):
+    - Remove a BUG() statement when a client fetches an onion descriptor
+      that has a lower revision counter than the one in its cache. This
+      can happen in normal circumstances due to HSDir desync. Fixes bug
+      24976; bugfix on 0.3.2.1-alpha.
+    - If we are configured to offer a single onion service, don't log
+      long-term established one hop rendezvous points in the heartbeat.
+      Fixes bug 25116; bugfix on 0.2.9.6-rc.
+
+  o Minor bugfixes (performance):
+    - Avoid calling protocol_list_supports_protocol() from inside tight
+      loops when running with cached routerinfo_t objects. Instead,
+      summarize the relevant protocols as flags in the routerinfo_t, as
+      we do for routerstatus_t objects. This change simplifies our code
+      a little, and saves a large amount of short-term memory allocation
+      operations. Fixes bug 25008; bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (Rust FFI):
+    - Fix a minor memory leak which would happen whenever the C code
+      would call the Rust implementation of
+      protover_get_supported_protocols(). This was due to the C version
+      returning a static string, whereas the Rust version newly allocated
+      a CString to pass across the FFI boundary. Consequently, the C
+      code was not expecting to need to free() what it was given. Fixes
+      bug 25127; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (scheduler, KIST):
+    - Avoid adding the same channel twice in the KIST scheduler pending
+      list, which would waste CPU cycles. Fixes bug 24700; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (unit test, monotonic time):
+    - Increase a constant (1msec to 10msec) in the monotonic time test
+      that makes sure the nsec/usec/msec times read are synchronized.
+      This change was needed to accommodate slow systems like armel or
+      when the clock_gettime() is not a VDSO on the running kernel.
+      Fixes bug 25113; bugfix on 0.2.9.1.
+
+  o Minor bugfixes (v3 onion services):
+    - Look at the "HSRend" protocol version, not the "HSDir" protocol
+      version, when deciding whether a consensus entry can support the
+      v3 onion service protocol as a rendezvous point. Fixes bug 25105;
+      bugfix on 0.3.2.1-alpha.
+
+  o Code simplification and refactoring:
+    - Remove the unused nodelist_recompute_all_hsdir_indices(). Closes
+      ticket 25108.
+    - Remove a series of counters used to track circuit extend attempts
+      and connection status but that in reality we aren't using for
+      anything other than stats logged by a SIGUSR1 signal. Closes
+      ticket 25163.
+
+  o Documentation (man page):
+    - The HiddenServiceVersion torrc option accepts only one number:
+      either version 2 or 3. Closes ticket 25026; bugfix
+      on 0.3.2.2-alpha.
+
+
+Changes in version 0.3.3.1-alpha - 2018-01-25
+  Tor 0.3.3.1-alpha is the first release in the 0.3.3.x series. It adds
+  several new features to Tor, including several improvements to
+  bootstrapping, and support for an experimental "vanguards" feature to
+  resist guard discovery attacks. This series also includes better
+  support for applications that need to embed Tor or manage v3
+  onion services.
+
+  o Major features (embedding):
+    - There is now a documented stable API for programs that need to
+      embed Tor. See tor_api.h for full documentation and known bugs.
+      Closes ticket 23684.
+    - Tor now has support for restarting in the same process.
+      Controllers that run Tor using the "tor_api.h" interface can now
+      restart Tor after Tor has exited. This support is incomplete,
+      however: we fixed crash bugs that prevented it from working at
+      all, but many bugs probably remain, including a possibility of
+      security issues. Implements ticket 24581.
+
+  o Major features (IPv6, directory documents):
+    - Add consensus method 27, which adds IPv6 ORPorts to the microdesc
+      consensus. This information makes it easier for IPv6 clients to
+      bootstrap and choose reachable entry guards. Implements ticket 23826.
+    - Add consensus method 28, which removes IPv6 ORPorts from
+      microdescriptors. Now that the consensus contains IPv6 ORPorts, they
+      are redundant in microdescs. This change will be used by Tor clients
+      on 0.2.8.x and later. (That is to say, with all Tor clients that
+      have IPv6 bootstrap and guard support.) Implements ticket 23828.
+    - Expand the documentation for AuthDirHasIPv6Connectivity when it is
+      set by different numbers of authorities. Fixes 23870
+      on 0.2.4.1-alpha.
+
+  o Major features (onion service v3, control port):
+    - The control port now supports commands and events for v3 onion
+      services. It is now possible to create ephemeral v3 services using
+      ADD_ONION. Additionally, several events (HS_DESC, HS_DESC_CONTENT,
+      CIRC and CIRC_MINOR) and commands (GETINFO, HSPOST, ADD_ONION and
+      DEL_ONION) have been extended to support v3 onion services. Closes
+      ticket 20699; implements proposal 284.
+
+  o Major features (onion services):
+    - Provide torrc options to pin the second and third hops of onion
+      service circuits to a list of nodes. The option HSLayer2Guards
+      pins the second hop, and the option HSLayer3Guards pins the third
+      hop. These options are for use in conjunction with experiments
+      with "vanguards" for preventing guard enumeration attacks. Closes
+      ticket 13837.
+
+  o Major features (rust, portability, experimental):
+    - Tor now ships with an optional implementation of one of its
+      smaller modules (protover.c) in the Rust programming language. To
+      try it out, install a Rust build environment, and configure Tor
+      with "--enable-rust --enable-cargo-online-mode". This should not
+      cause any user-visible changes, but should help us gain more
+      experience with Rust, and plan future Rust integration work.
+      Implementation by Chelsea Komlo. Closes ticket 22840.
+
+  o Minor features (storage, configuration):
+    - Users can store cached directory documents somewhere other than
+      the DataDirectory by using the CacheDirectory option. Similarly,
+      the storage location for relay's keys can be overridden with the
+      KeyDirectory option. Closes ticket 22703.
+
+  o Major features (v3 onion services, ipv6):
+    - When v3 onion service clients send introduce cells, they now
+      include the IPv6 address of the rendezvous point, if it has one.
+      Current v3 onion services running 0.3.2 ignore IPv6 addresses, but
+      in future Tor versions, IPv6-only v3 single onion services will be
+      able to use IPv6 addresses to connect directly to the rendezvous
+      point. Closes ticket 23577. Patch by Neel Chauhan.
+
+  o Major bugfixes (onion services, retry behavior):
+    - Fix an "off by 2" error in counting rendezvous failures on the
+      onion service side. While we thought we would stop the rendezvous
+      attempt after one failed circuit, we were actually making three
+      circuit attempts before giving up. Now switch to a default of 2,
+      and allow the consensus parameter "hs_service_max_rdv_failures" to
+      override. Fixes bug 24895; bugfix on 0.0.6.
+    - New-style (v3) onion services now obey the "max rendezvous circuit
+      attempts" logic. Previously they would make as many rendezvous
+      circuit attempts as they could fit in the MAX_REND_TIMEOUT second
+      window before giving up. Fixes bug 24894; bugfix on 0.3.2.1-alpha.
+
+  o Major bugfixes (relays):
+    - Fix a set of false positives where relays would consider
+      connections to other relays as being client-only connections (and
+      thus e.g. deserving different link padding schemes) if those
+      relays fell out of the consensus briefly. Now we look only at the
+      initial handshake and whether the connection authenticated as a
+      relay. Fixes bug 24898; bugfix on 0.3.1.1-alpha.
+
+  o Minor feature (IPv6):
+    - Make IPv6-only clients wait for microdescs for relays, even if we
+      were previously using descriptors (or were using them as a bridge)
+      and have a cached descriptor for them. Implements ticket 23827.
+    - When a consensus has IPv6 ORPorts, make IPv6-only clients use
+      them, rather than waiting to download microdescriptors.
+      Implements ticket 23827.
+
+  o Minor features (cleanup):
+    - Tor now deletes the CookieAuthFile and ExtORPortCookieAuthFile
+      when it stops. Closes ticket 23271.
+
+  o Minor features (defensive programming):
+    - Most of the functions in Tor that free objects have been replaced
+      with macros that free the objects and set the corresponding
+      pointers to NULL. This change should help prevent a large class of
+      dangling pointer bugs. Closes ticket 24337.
+    - Where possible, the tor_free() macro now only evaluates its input
+      once. Part of ticket 24337.
+    - Check that microdesc ed25519 ids are non-zero in
+      node_get_ed25519_id() before returning them. Implements ticket
+      24001, patch by "aruna1234".
+
+  o Minor features (embedding):
+    - Tor can now start with a preauthenticated control connection
+      created by the process that launched it. This feature is meant for
+      use by programs that want to launch and manage a Tor process
+      without allowing other programs to manage it as well. For more
+      information, see the __OwningControllerFD option documented in
+      control-spec.txt. Closes ticket 23900.
+    - On most errors that would cause Tor to exit, it now tries to
+      return from the tor_main() function, rather than calling the
+      system exit() function. Most users won't notice a difference here,
+      but it should be significant for programs that run Tor inside
+      a separate thread: they should now be able to survive Tor's exit
+      conditions rather than having Tor shut down the entire process.
+      Closes ticket 23848.
+    - Applications that want to embed Tor can now tell Tor not to
+      register any of its own POSIX signal handlers, using the
+      __DisableSignalHandlers option. Closes ticket 24588.
+
+  o Minor features (fallback directory list):
+    - Avoid selecting fallbacks that change their IP addresses too
+      often. Select more fallbacks by ignoring the Guard flag, and
+      allowing lower cutoffs for the Running and V2Dir flags. Also allow
+      a lower bandwidth, and a higher number of fallbacks per operator
+      (5% of the list). Implements ticket 24785.
+    - Update the fallback whitelist and blacklist based on opt-ins and
+      relay changes. Closes tickets 22321, 24678, 22527, 24135,
+      and 24695.
+
+  o Minor features (fallback directory mirror configuration):
+    - Add a nickname to each fallback in a C comment. This makes it
+      easier for operators to find their relays, and allows stem to use
+      nicknames to identify fallbacks. Implements ticket 24600.
+    - Add a type and version header to the fallback directory mirror
+      file. Also add a delimiter to the end of each fallback entry. This
+      helps external parsers like stem and Relay Search. Implements
+      ticket 24725.
+    - Add an extrainfo cache flag for each fallback in a C comment. This
+      allows stem to use fallbacks to fetch extra-info documents, rather
+      than using authorities. Implements ticket 22759.
+    - Add the generateFallbackDirLine.py script for automatically
+      generating fallback directory mirror lines from relay fingerprints.
+      No more typos! Add the lookupFallbackDirContact.py script for
+      automatically looking up operator contact info from relay
+      fingerprints. Implements ticket 24706, patch by teor and atagar.
+    - Reject any fallback directory mirror that serves an expired
+      consensus. Implements ticket 20942, patch by "minik".
+    - Remove commas and equals signs from external string inputs to the
+      fallback list. This avoids format confusion attacks. Implements
+      ticket 24726.
+    - Remove the "weight=10" line from fallback directory mirror
+      entries. Ticket 24681 will maintain the current fallback weights
+      by changing Tor's default fallback weight to 10. Implements
+      ticket 24679.
+    - Stop logging excessive information about fallback netblocks.
+      Implements ticket 24791.
+
+  o Minor features (forward-compatibility):
+    - If a relay supports some link authentication protocol that we do
+      not recognize, then include that relay's ed25519 key when telling
+      other relays to extend to it. Previously, we treated future
+      versions as if they were too old to support ed25519 link
+      authentication. Closes ticket 20895.
+
+  o Minor features (heartbeat):
+    - Add onion service information to our heartbeat logs, displaying
+      stats about the activity of configured onion services. Closes
+      ticket 24896.
+
+  o Minor features (instrumentation, development):
+    - Add the MainloopStats option to allow developers to get
+      instrumentation information from the main event loop via the
+      heartbeat messages. We hope to use this to improve Tor's behavior
+      when it's trying to sleep. Closes ticket 24605.
+
+  o Minor features (log messages):
+    - Improve a warning message that happens when we fail to re-parse an
+      old router because of an expired certificate. Closes ticket 20020.
+    - Make the log more quantitative when we hit MaxMemInQueues
+      threshold exposing some values. Closes ticket 24501.
+
+  o Minor features (logging, android):
+    - Added support for the Android logging subsystem. Closes
+      ticket 24362.
+
+  o Minor features (performance):
+    - Support predictive circuit building for onion service circuits
+      with multiple layers of guards. Closes ticket 23101.
+    - Use stdatomic.h where available, rather than mutexes, to implement
+      atomic_counter_t. Closes ticket 23953.
+
+  o Minor features (performance, 32-bit):
+    - Improve performance on 32-bit systems by avoiding 64-bit division
+      when calculating the timestamp in milliseconds for channel padding
+      computations. Implements ticket 24613.
+    - Improve performance on 32-bit systems by avoiding 64-bit division
+      when timestamping cells and buffer chunks for OOM calculations.
+      Implements ticket 24374.
+
+  o Minor features (performance, OSX, iOS):
+    - Use the mach_approximate_time() function (when available) to
+      implement coarse monotonic time. Having a coarse time function
+      should avoid a large number of system calls, and improve
+      performance slightly, especially under load. Closes ticket 24427.
+
+  o Minor features (performance, windows):
+    - Improve performance on Windows Vista and Windows 7 by adjusting
+      TCP send window size according to the recommendation from
+      SIO_IDEAL_SEND_BACKLOG_QUERY. Closes ticket 22798. Patch
+      from Vort.
+
+  o Major features (relay):
+    - Implement an option, ReducedExitPolicy, to allow an Tor exit relay
+      operator to use a more reasonable ("reduced") exit policy, rather
+      than the default one. If you want to run an exit node without
+      thinking too hard about which ports to allow, this one is for you.
+      Closes ticket 13605. Patch from Neel Chauhan.
+
+  o Minor features (testing, debugging, embedding):
+    - For development purposes, Tor now has a mode in which it runs for
+      a few seconds, then stops, and starts again without exiting the
+      process. This mode is meant to help us debug various issues with
+      ticket 23847. To use this feature, compile with
+      --enable-restart-debugging, and set the TOR_DEBUG_RESTART
+      environment variable. This is expected to crash a lot, and is
+      really meant for developers only. It will likely be removed in a
+      future release. Implements ticket 24583.
+
+  o Minor bugfix (network IPv6 test):
+    - Tor's test scripts now check if "ping -6 ::1" works when the user
+      runs "make test-network-all". Fixes bug 24677; bugfix on
+      0.2.9.3-alpha. Patch by "ffmancera".
+
+  o Minor bugfixes (build, rust):
+    - Fix output of autoconf checks to display success messages for Rust
+      dependencies and a suitable rustc compiler version. Fixes bug
+      24612; bugfix on 0.3.1.3-alpha.
+    - When building with Rust on OSX, link against libresolv, to work
+      around the issue at https://github.com/rust-lang/rust/issues/46797.
+      Fixes bug 24652; bugfix on 0.3.1.1-alpha.
+    - Don't pass the --quiet option to cargo: it seems to suppress some
+      errors, which is not what we want to do when building. Fixes bug
+      24518; bugfix on 0.3.1.7.
+    - Build correctly when building from outside Tor's source tree with
+      the TOR_RUST_DEPENDENCIES option set. Fixes bug 22768; bugfix
+      on 0.3.1.7.
+
+  o Minor bugfixes (directory authorities, IPv6):
+    - When creating a routerstatus (vote) from a routerinfo (descriptor),
+      set the IPv6 address to the unspecified IPv6 address, and
+      explicitly initialize the port to zero. Fixes bug 24488; bugfix
+      on 0.2.4.1-alpha.
+
+  o Minor bugfixes (fallback directory mirrors):
+    - Make updateFallbackDirs.py search harder for python. (Some OSs
+      don't put it in /usr/bin.) Fixes bug 24708; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (hibernation, bandwidth accounting, shutdown):
+    - When hibernating, close connections normally and allow them to
+      flush. Fixes bug 23571; bugfix on 0.2.4.7-alpha. Also fixes
+      bug 7267.
+    - Do not attempt to launch self-reachability tests when entering
+      hibernation. Fixes a case of bug 12062; bugfix on 0.0.9pre5.
+    - Resolve several bugs related to descriptor fetching on bridge
+      clients with bandwidth accounting enabled. (This combination is
+      not recommended!) Fixes a case of bug 12062; bugfix
+      on 0.2.0.3-alpha.
+    - When hibernating, do not attempt to launch DNS checks. Fixes a
+      case of bug 12062; bugfix on 0.1.2.2-alpha.
+    - When hibernating, do not try to upload or download descriptors.
+      Fixes a case of bug 12062; bugfix on 0.0.9pre5.
+
+  o Minor bugfixes (IPv6, bridges):
+    - Tor now always sets IPv6 preferences for bridges. Fixes bug 24573;
+      bugfix on 0.2.8.2-alpha.
+    - Tor now sets IPv6 address in the routerstatus as well as in the
+      router descriptors when updating addresses for a bridge. Closes
+      ticket 24572; bugfix on 0.2.4.5-alpha. Patch by "ffmancera".
+
+  o Minor bugfixes (linux seccomp2 sandbox):
+    - When running with the sandbox enabled, reload configuration files
+      correctly even when %include was used. Previously we would crash.
+      Fixes bug 22605; bugfix on 0.3.1. Patch from Daniel Pinto.
+
+  o Minor bugfixes (memory leaks):
+    - Avoid possible at-exit memory leaks related to use of Libevent's
+      event_base_once() function. (This function tends to leak memory if
+      the event_base is closed before the event fires.) Fixes bug 24584;
+      bugfix on 0.2.8.1-alpha.
+    - Fix a harmless memory leak in tor-resolve. Fixes bug 24582; bugfix
+      on 0.2.1.1-alpha.
+
+  o Minor bugfixes (OSX):
+    - Don't exit the Tor process if setrlimit() fails to change the file
+      limit (which can happen sometimes on some versions of OSX). Fixes
+      bug 21074; bugfix on 0.0.9pre5.
+
+  o Minor bugfixes (performance, fragile-hardening):
+    - Improve the performance of our consensus-diff application code
+      when Tor is built with the --enable-fragile-hardening option set.
+      Fixes bug 24826; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (performance, timeouts):
+    - Consider circuits for timeout as soon as they complete a hop. This
+      is more accurate than applying the timeout in
+      circuit_expire_building() because that function is only called
+      once per second, which is now too slow for typical timeouts on the
+      current network. Fixes bug 23114; bugfix on 0.2.2.2-alpha.
+    - Use onion service circuits (and other circuits longer than 3 hops)
+      to calculate a circuit build timeout. Previously, Tor only
+      calculated its build timeout based on circuits that planned to be
+      exactly 3 hops long. With this change, we include measurements
+      from all circuits at the point where they complete their third
+      hop. Fixes bug 23100; bugfix on 0.2.2.2-alpha.
+
+  o Minor bugfixes (testing):
+    - Give out Exit flags in bootstrapping networks. Fixes bug 24137;
+      bugfix on 0.2.3.1-alpha.
+    - Fix a memory leak in the scheduler/loop_kist unit test. Fixes bug
+      25005; bugfix on 0.3.2.7-rc.
+
+  o Code simplification and refactoring:
+    - Remove /usr/athena from search path in configure.ac. Closes
+      ticket 24363.
+    - Remove duplicate code in node_has_curve25519_onion_key() and
+      node_get_curve25519_onion_key(), and add a check for a zero
+      microdesc curve25519 onion key. Closes ticket 23966, patch by
+      "aruna1234" and teor.
+    - Rewrite channel_rsa_id_group_set_badness to reduce temporary
+      memory allocations with large numbers of OR connections (e.g.
+      relays). Closes ticket 24119.
+    - Separate the function that deletes ephemeral files when Tor
+      stops gracefully.
+    - Small changes to Tor's buf_t API to make it suitable for use as a
+      general-purpose safe string constructor. Closes ticket 22342.
+    - Switch -Wnormalized=id to -Wnormalized=nfkc in configure.ac to
+      avoid source code identifier confusion. Closes ticket 24467.
+    - The tor_git_revision[] constant no longer needs to be redeclared
+      by everything that links against the rest of Tor. Done as part of
+      ticket 23845, to simplify our external API.
+    - We make extend_info_from_node() use node_get_curve25519_onion_key()
+      introduced in ticket 23577 to access the curve25519 public keys
+      rather than accessing it directly. Closes ticket 23760. Patch by
+      Neel Chauhan.
+    - Add a function to log channels' scheduler state changes to aid
+      debugging efforts. Closes ticket 24531.
+
+  o Documentation:
+    - Add documentation on how to build tor with Rust dependencies
+      without having to be online. Closes ticket 22907; bugfix
+      on 0.3.0.3-alpha.
+    - Clarify the behavior of RelayBandwidth{Rate,Burst} with client
+      traffic. Closes ticket 24318.
+    - Document that OutboundBindAddress doesn't apply to DNS requests.
+      Closes ticket 22145. Patch from Aruna Maurya.
+    - Document that operators who run more than one relay or bridge are
+      expected to set MyFamily and ContactInfo correctly. Closes
+      ticket 24526.
+
+  o Code simplification and refactoring (channels):
+    - Remove the incoming and outgoing channel queues. These were never
+      used, but still took up a step in our fast path.
+    - The majority of the channel unit tests have been rewritten and the
+      code coverage has now been raised to 83.6% for channel.c. Closes
+      ticket 23709.
+    - Remove other dead code from the channel subsystem: All together,
+      this cleanup has removed more than 1500 lines of code overall and
+      adding very little except for unit test.
+
+  o Code simplification and refactoring (circuit rendezvous):
+    - Split the client-side rendezvous circuit lookup into two
+      functions: one that returns only established circuits and another
+      that returns all kinds of circuits. Closes ticket 23459.
+
+  o Code simplification and refactoring (controller):
+    - Make most of the variables in networkstatus_getinfo_by_purpose()
+      const. Implements ticket 24489.
+
+
+Changes in version 0.3.2.9 - 2018-01-09
+  Tor 0.3.2.9 is the first stable release in the 0.3.2 series.
+
+  The 0.3.2 series includes our long-anticipated new onion service
+  design, with numerous security features. (For more information, see
+  our blog post at https://blog.torproject.org/fall-harvest.) We also
+  have a new circuit scheduler algorithm for improved performance on
+  relays everywhere (see https://blog.torproject.org/kist-and-tell),
+  along with many smaller features and bugfixes.
+
+  Per our stable release policy, we plan to support each stable release
+  series for at least the next nine months, or for three months after
+  the first stable release of the next series: whichever is longer. If
+  you need a release with long-term support, we recommend that you stay
+  with the 0.2.9 series.
+
+  Below is a list of the changes since 0.3.2.8-rc. For a list of all
+  changes since 0.3.1, see the ReleaseNotes file.
+
+  o Minor features (fallback directory mirrors):
+    - The fallback directory list has been re-generated based on the
+      current status of the network. Tor uses fallback directories to
+      bootstrap when it doesn't yet have up-to-date directory
+      information. Closes ticket 24801.
+    - Make the default DirAuthorityFallbackRate 0.1, so that clients
+      prefer to bootstrap from fallback directory mirrors. This is a
+      follow-up to 24679, which removed weights from the default
+      fallbacks. Implements ticket 24681.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the January 5 2018 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (address selection):
+    - When the fascist_firewall_choose_address_ functions don't find a
+      reachable address, set the returned address to the null address
+      and port. This is a precautionary measure, because some callers do
+      not check the return value. Fixes bug 24736; bugfix
+      on 0.2.8.2-alpha.
+
+  o Minor bugfixes (compilation):
+    - Resolve a few shadowed-variable warnings in the onion service
+      code. Fixes bug 24634; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (portability, msvc):
+    - Fix a bug in the bit-counting parts of our timing-wheel code on
+      MSVC. (Note that MSVC is still not a supported build platform, due
+      to cryptographic timing channel risks.) Fixes bug 24633; bugfix
+      on 0.2.9.1-alpha.
+
+
+Changes in version 0.3.2.8-rc - 2017-12-21
+  Tor 0.3.2.8-rc fixes a pair of bugs in the KIST and KISTLite
+  schedulers that had led servers under heavy load to overload their
+  outgoing connections. All relay operators running earlier 0.3.2.x
+  versions should upgrade. This version also includes a mitigation for
+  over-full DESTROY queues leading to out-of-memory conditions: if it
+  works, we will soon backport it to earlier release series.
+
+  This is the second release candidate in the 0.3.2 series. If we find
+  no new bugs or regression here, then the first stable 0.3.2 release
+  will be nearly identical to this.
+
+  o Major bugfixes (KIST, scheduler):
+    - The KIST scheduler did not correctly account for data already
+      enqueued in each connection's send socket buffer, particularly in
+      cases when the TCP/IP congestion window was reduced between
+      scheduler calls. This situation lead to excessive per-connection
+      buffering in the kernel, and a potential memory DoS. Fixes bug
+      24665; bugfix on 0.3.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the December 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (hidden service v3):
+    - Bump hsdir_spread_store parameter from 3 to 4 in order to increase
+      the probability of reaching a service for a client missing
+      microdescriptors. Fixes bug 24425; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (memory usage):
+    - When queuing DESTROY cells on a channel, only queue the circuit-id
+      and reason fields: not the entire 514-byte cell. This fix should
+      help mitigate any bugs or attacks that fill up these queues, and
+      free more RAM for other uses. Fixes bug 24666; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (scheduler, KIST):
+    - Use a sane write limit for KISTLite when writing onto a connection
+      buffer instead of using INT_MAX and shoving as much as it can.
+      Because the OOM handler cleans up circuit queues, we are better
+      off at keeping them in that queue instead of the connection's
+      buffer. Fixes bug 24671; bugfix on 0.3.2.1-alpha.
+
+
+Changes in version 0.3.2.7-rc - 2017-12-14
+  Tor 0.3.2.7-rc fixes various bugs in earlier versions of Tor,
+  including some that could affect reliability or correctness.
+
+  This is the first release candidate in the 0.3.2 series. If we find no
+  new bugs or regression here, then the first stable 0.3.2. release will
+  be nearly identical to this.
+
+  o Major bugfixes (circuit prediction):
+    - Fix circuit prediction logic so that a client doesn't treat a port
+      as being "handled" by a circuit if that circuit already has
+      isolation settings on it. This change should make Tor clients more
+      responsive by improving their chances of having a pre-created
+      circuit ready for use when a request arrives. Fixes bug 18859;
+      bugfix on 0.2.3.3-alpha.
+
+  o Minor features (logging):
+    - Provide better warnings when the getrandom() syscall fails. Closes
+      ticket 24500.
+
+  o Minor features (portability):
+    - Tor now compiles correctly on arm64 with libseccomp-dev installed.
+      (It doesn't yet work with the sandbox enabled.) Closes
+      ticket 24424.
+
+  o Minor bugfixes (bridge clients, bootstrap):
+    - Retry directory downloads when we get our first bridge descriptor
+      during bootstrap or while reconnecting to the network. Keep
+      retrying every time we get a bridge descriptor, until we have a
+      reachable bridge. Fixes part of bug 24367; bugfix on 0.2.0.3-alpha.
+    - Stop delaying bridge descriptor fetches when we have cached bridge
+      descriptors. Instead, only delay bridge descriptor fetches when we
+      have at least one reachable bridge. Fixes part of bug 24367;
+      bugfix on 0.2.0.3-alpha.
+    - Stop delaying directory fetches when we have cached bridge
+      descriptors. Instead, only delay bridge descriptor fetches when
+      all our bridges are definitely unreachable. Fixes part of bug
+      24367; bugfix on 0.2.0.3-alpha.
+
+  o Minor bugfixes (compilation):
+    - Fix a signed/unsigned comparison warning introduced by our fix to
+      TROVE-2017-009. Fixes bug 24480; bugfix on 0.2.5.16.
+
+  o Minor bugfixes (correctness):
+    - Fix several places in our codebase where a C compiler would be
+      likely to eliminate a check, based on assuming that undefined
+      behavior had not happened elsewhere in the code. These cases are
+      usually a sign of redundant checking or dubious arithmetic. Found
+      by Georg Koppen using the "STACK" tool from Wang, Zeldovich,
+      Kaashoek, and Solar-Lezama. Fixes bug 24423; bugfix on various
+      Tor versions.
+
+  o Minor bugfixes (onion service v3):
+    - Fix a race where an onion service would launch a new intro circuit
+      after closing an old one, but fail to register it before freeing
+      the previously closed circuit. This bug was making the service
+      unable to find the established intro circuit and thus not upload
+      its descriptor, thus making a service unavailable for up to 24
+      hours. Fixes bug 23603; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (scheduler, KIST):
+    - Properly set the scheduler state of an unopened channel in the
+      KIST scheduler main loop. This prevents a harmless but annoying
+      log warning. Fixes bug 24502; bugfix on 0.3.2.4-alpha.
+    - Avoid a possible integer overflow when computing the available
+      space on the TCP buffer of a channel. This had no security
+      implications; but could make KIST allow too many cells on a
+      saturated connection. Fixes bug 24590; bugfix on 0.3.2.1-alpha.
+    - Downgrade to "info" a harmless warning about the monotonic time
+      moving backwards: This can happen on platform not supporting
+      monotonic time. Fixes bug 23696; bugfix on 0.3.2.1-alpha.
+
+
+Changes in version 0.3.2.6-alpha - 2017-12-01
+  This version of Tor is the latest in the 0.3.2 alpha series. It
+  includes fixes for several important security issues. All Tor users
+  should upgrade to this release, or to one of the other releases coming
+  out today.
+
+  o Major bugfixes (security):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - Fix a denial of service issue where an attacker could crash a
+      directory authority using a malformed router descriptor. Fixes bug
+      24245; bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2017-010
+      and CVE-2017-8820.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, onion service v2):
+    - Fix a use-after-free error that could crash v2 Tor onion services
+      when they failed to open circuits while expiring introduction
+      points. Fixes bug 24313; bugfix on 0.2.7.2-alpha. This issue is
+      also tracked as TROVE-2017-013 and CVE-2017-8823.
+
+  o Major bugfixes (security, relay):
+    - When running as a relay, make sure that we never build a path
+      through ourselves, even in the case where we have somehow lost the
+      version of our descriptor appearing in the consensus. Fixes part
+      of bug 21534; bugfix on 0.2.0.1-alpha. This issue is also tracked
+      as TROVE-2017-012 and CVE-2017-8822.
+    - When running as a relay, make sure that we never choose ourselves
+      as a guard. Fixes part of bug 21534; bugfix on 0.3.0.1-alpha. This
+      issue is also tracked as TROVE-2017-012 and CVE-2017-8822.
+
+  o Minor feature (relay statistics):
+    - Change relay bandwidth reporting stats interval from 4 hours to 24
+      hours in order to reduce the efficiency of guard discovery
+      attacks. Fixes ticket 23856.
+
+  o Minor features (directory authority):
+    - Add an IPv6 address for the "bastet" directory authority. Closes
+      ticket 24394.
+
+  o Minor bugfixes (client):
+    - By default, do not enable storage of client-side DNS values. These
+      values were unused by default previously, but they should not have
+      been cached at all. Fixes bug 24050; bugfix on 0.2.6.3-alpha.
+
+
+Changes in version 0.3.1.9 - 2017-12-01:
+  Tor 0.3.1.9 backports important security and stability fixes from the
+  0.3.2 development series. All Tor users should upgrade to this
+  release, or to another of the releases coming out today.
+
+  o Major bugfixes (security, backport from 0.3.2.6-alpha):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - Fix a denial of service issue where an attacker could crash a
+      directory authority using a malformed router descriptor. Fixes bug
+      24245; bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2017-010
+      and CVE-2017-8820.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, onion service v2, backport from 0.3.2.6-alpha):
+    - Fix a use-after-free error that could crash v2 Tor onion services
+      when they failed to open circuits while expiring introduction
+      points. Fixes bug 24313; bugfix on 0.2.7.2-alpha. This issue is
+      also tracked as TROVE-2017-013 and CVE-2017-8823.
+
+  o Major bugfixes (security, relay, backport from 0.3.2.6-alpha):
+    - When running as a relay, make sure that we never build a path
+      through ourselves, even in the case where we have somehow lost the
+      version of our descriptor appearing in the consensus. Fixes part
+      of bug 21534; bugfix on 0.2.0.1-alpha. This issue is also tracked
+      as TROVE-2017-012 and CVE-2017-8822.
+    - When running as a relay, make sure that we never choose ourselves
+      as a guard. Fixes part of bug 21534; bugfix on 0.3.0.1-alpha. This
+      issue is also tracked as TROVE-2017-012 and CVE-2017-8822.
+
+  o Major bugfixes (exit relays, DNS, backport from 0.3.2.4-alpha):
+    - Fix an issue causing DNS to fail on high-bandwidth exit nodes,
+      making them nearly unusable. Fixes bugs 21394 and 18580; bugfix on
+      0.1.2.2-alpha, which introduced eventdns. Thanks to Dhalgren for
+      identifying and finding a workaround to this bug and to Moritz,
+      Arthur Edelstein, and Roger for helping to track it down and
+      analyze it.
+
+  o Minor features (bridge):
+    - Bridges now include notice in their descriptors that they are
+      bridges, and notice of their distribution status, based on their
+      publication settings. Implements ticket 18329. For more fine-
+      grained control of how a bridge is distributed, upgrade to 0.3.2.x
+      or later.
+
+  o Minor features (directory authority, backport from 0.3.2.6-alpha):
+    - Add an IPv6 address for the "bastet" directory authority. Closes
+      ticket 24394.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfix (relay address resolution, backport from 0.3.2.1-alpha):
+    - Avoid unnecessary calls to directory_fetches_from_authorities() on
+      relays, to prevent spurious address resolutions and descriptor
+      rebuilds. This is a mitigation for bug 21789. Fixes bug 23470;
+      bugfix on in 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.2.1-alpha):
+    - Fix unused variable warnings in donna's Curve25519 SSE2 code.
+      Fixes bug 22895; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (logging, relay shutdown, annoyance, backport from 0.3.2.2-alpha):
+    - When a circuit is marked for close, do not attempt to package any
+      cells for channels on that circuit. Previously, we would detect
+      this condition lower in the call stack, when we noticed that the
+      circuit had no attached channel, and log an annoying message.
+      Fixes bug 8185; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (onion service, backport from 0.3.2.5-alpha):
+    - Rename the consensus parameter "hsdir-interval" to "hsdir_interval"
+      so it matches dir-spec.txt. Fixes bug 24262; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (relay, crash, backport from 0.3.2.4-alpha):
+    - Avoid a crash when transitioning from client mode to bridge mode.
+      Previously, we would launch the worker threads whenever our
+      "public server" mode changed, but not when our "server" mode
+      changed. Fixes bug 23693; bugfix on 0.2.6.3-alpha.
+
+
+Changes in version 0.3.0.13 - 2017-12-01
+  Tor 0.3.0.13 backports important security and stability bugfixes from
+  later Tor releases. All Tor users should upgrade to this release, or
+  to another of the releases coming out today.
+
+  Note: the Tor 0.3.0 series will no longer be supported after 26 Jan
+  2018. If you need a release with long-term support, please stick with
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Major bugfixes (security, backport from 0.3.2.6-alpha):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - Fix a denial of service issue where an attacker could crash a
+      directory authority using a malformed router descriptor. Fixes bug
+      24245; bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2017-010
+      and CVE-2017-8820.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, onion service v2, backport from 0.3.2.6-alpha):
+    - Fix a use-after-free error that could crash v2 Tor onion services
+      when they failed to open circuits while expiring introduction
+      points. Fixes bug 24313; bugfix on 0.2.7.2-alpha. This issue is
+      also tracked as TROVE-2017-013 and CVE-2017-8823.
+
+  o Major bugfixes (security, relay, backport from 0.3.2.6-alpha):
+    - When running as a relay, make sure that we never build a path
+      through ourselves, even in the case where we have somehow lost the
+      version of our descriptor appearing in the consensus. Fixes part
+      of bug 21534; bugfix on 0.2.0.1-alpha. This issue is also tracked
+      as TROVE-2017-012 and CVE-2017-8822.
+    - When running as a relay, make sure that we never choose ourselves
+      as a guard. Fixes part of bug 21534; bugfix on 0.3.0.1-alpha. This
+      issue is also tracked as TROVE-2017-012 and CVE-2017-8822.
+
+  o Major bugfixes (exit relays, DNS, backport from 0.3.2.4-alpha):
+    - Fix an issue causing DNS to fail on high-bandwidth exit nodes,
+      making them nearly unusable. Fixes bugs 21394 and 18580; bugfix on
+      0.1.2.2-alpha, which introduced eventdns. Thanks to Dhalgren for
+      identifying and finding a workaround to this bug and to Moritz,
+      Arthur Edelstein, and Roger for helping to track it down and
+      analyze it.
+
+  o Minor features (security, windows, backport from 0.3.1.1-alpha):
+    - Enable a couple of pieces of Windows hardening: one
+      (HeapEnableTerminationOnCorruption) that has been on-by-default
+      since Windows 8, and unavailable before Windows 7; and one
+      (PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION) which we believe doesn't
+      affect us, but shouldn't do any harm. Closes ticket 21953.
+
+  o Minor features (bridge, backport from 0.3.1.9):
+    - Bridges now include notice in their descriptors that they are
+      bridges, and notice of their distribution status, based on their
+      publication settings. Implements ticket 18329. For more fine-
+      grained control of how a bridge is distributed, upgrade to 0.3.2.x
+      or later.
+
+  o Minor features (directory authority, backport from 0.3.2.6-alpha):
+    - Add an IPv6 address for the "bastet" directory authority. Closes
+      ticket 24394.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfix (relay address resolution, backport from 0.3.2.1-alpha):
+    - Avoid unnecessary calls to directory_fetches_from_authorities() on
+      relays, to prevent spurious address resolutions and descriptor
+      rebuilds. This is a mitigation for bug 21789. Fixes bug 23470;
+      bugfix on in 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.2.1-alpha):
+    - Fix unused variable warnings in donna's Curve25519 SSE2 code.
+      Fixes bug 22895; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (logging, relay shutdown, annoyance, backport from 0.3.2.2-alpha):
+    - When a circuit is marked for close, do not attempt to package any
+      cells for channels on that circuit. Previously, we would detect
+      this condition lower in the call stack, when we noticed that the
+      circuit had no attached channel, and log an annoying message.
+      Fixes bug 8185; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (relay, crash, backport from 0.3.2.4-alpha):
+    - Avoid a crash when transitioning from client mode to bridge mode.
+      Previously, we would launch the worker threads whenever our
+      "public server" mode changed, but not when our "server" mode
+      changed. Fixes bug 23693; bugfix on 0.2.6.3-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.1.6-rc):
+    - Fix an undersized buffer in test-memwipe.c. Fixes bug 23291;
+      bugfix on 0.2.7.2-alpha. Found and patched by Ties Stuij.
+
+
+Changes in version 0.2.9.14 - 2017-12-01
+  Tor 0.3.0.13 backports important security and stability bugfixes from
+  later Tor releases. All Tor users should upgrade to this release, or
+  to another of the releases coming out today.
+
+  o Major bugfixes (exit relays, DNS, backport from 0.3.2.4-alpha):
+    - Fix an issue causing DNS to fail on high-bandwidth exit nodes,
+      making them nearly unusable. Fixes bugs 21394 and 18580; bugfix on
+      0.1.2.2-alpha, which introduced eventdns. Thanks to Dhalgren for
+      identifying and finding a workaround to this bug and to Moritz,
+      Arthur Edelstein, and Roger for helping to track it down and
+      analyze it.
+
+  o Major bugfixes (security, backport from 0.3.2.6-alpha):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - Fix a denial of service issue where an attacker could crash a
+      directory authority using a malformed router descriptor. Fixes bug
+      24245; bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2017-010
+      and CVE-2017-8820.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, onion service v2, backport from 0.3.2.6-alpha):
+    - Fix a use-after-free error that could crash v2 Tor onion services
+      when they failed to open circuits while expiring introduction
+      points. Fixes bug 24313; bugfix on 0.2.7.2-alpha. This issue is
+      also tracked as TROVE-2017-013 and CVE-2017-8823.
+
+  o Major bugfixes (security, relay, backport from 0.3.2.6-alpha):
+    - When running as a relay, make sure that we never build a path
+      through ourselves, even in the case where we have somehow lost the
+      version of our descriptor appearing in the consensus. Fixes part
+      of bug 21534; bugfix on 0.2.0.1-alpha. This issue is also tracked
+      as TROVE-2017-012 and CVE-2017-8822.
+
+  o Minor features (bridge, backport from 0.3.1.9):
+    - Bridges now include notice in their descriptors that they are
+      bridges, and notice of their distribution status, based on their
+      publication settings. Implements ticket 18329. For more fine-
+      grained control of how a bridge is distributed, upgrade to 0.3.2.x
+      or later.
+
+  o Minor features (directory authority, backport from 0.3.2.6-alpha):
+    - Add an IPv6 address for the "bastet" directory authority. Closes
+      ticket 24394.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (security, windows, backport from 0.3.1.1-alpha):
+    - Enable a couple of pieces of Windows hardening: one
+      (HeapEnableTerminationOnCorruption) that has been on-by-default
+      since Windows 8, and unavailable before Windows 7; and one
+      (PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION) which we believe doesn't
+      affect us, but shouldn't do any harm. Closes ticket 21953.
+
+  o Minor bugfix (relay address resolution, backport from 0.3.2.1-alpha):
+    - Avoid unnecessary calls to directory_fetches_from_authorities() on
+      relays, to prevent spurious address resolutions and descriptor
+      rebuilds. This is a mitigation for bug 21789. Fixes bug 23470;
+      bugfix on in 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.2.1-alpha):
+    - Fix unused variable warnings in donna's Curve25519 SSE2 code.
+      Fixes bug 22895; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (logging, relay shutdown, annoyance, backport from 0.3.2.2-alpha):
+    - When a circuit is marked for close, do not attempt to package any
+      cells for channels on that circuit. Previously, we would detect
+      this condition lower in the call stack, when we noticed that the
+      circuit had no attached channel, and log an annoying message.
+      Fixes bug 8185; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (relay, crash, backport from 0.3.2.4-alpha):
+    - Avoid a crash when transitioning from client mode to bridge mode.
+      Previously, we would launch the worker threads whenever our
+      "public server" mode changed, but not when our "server" mode
+      changed. Fixes bug 23693; bugfix on 0.2.6.3-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.1.6-rc):
+    - Fix an undersized buffer in test-memwipe.c. Fixes bug 23291;
+      bugfix on 0.2.7.2-alpha. Found and patched by Ties Stuij.
+
+
+Changes in version 0.2.8.17 - 2017-12-01
+  Tor 0.2.8.17 backports important security and stability bugfixes from
+  later Tor releases. All Tor users should upgrade to this release, or
+  to another of the releases coming out today.
+
+  Note: the Tor 0.2.8 series will no longer be supported after 1 Jan
+  2018. If you need a release with long-term support, please upgrade with
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Major bugfixes (security, backport from 0.3.2.6-alpha):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, onion service v2, backport from 0.3.2.6-alpha):
+    - Fix a use-after-free error that could crash v2 Tor onion services
+      when they failed to open circuits while expiring introduction
+      points. Fixes bug 24313; bugfix on 0.2.7.2-alpha. This issue is
+      also tracked as TROVE-2017-013 and CVE-2017-8823.
+
+  o Major bugfixes (security, relay, backport from 0.3.2.6-alpha):
+    - When running as a relay, make sure that we never build a path through
+      ourselves, even in the case where we have somehow lost the version of
+      our descriptor appearing in the consensus. Fixes part of bug 21534;
+      bugfix on 0.2.0.1-alpha. This issue is also tracked as TROVE-2017-012
+      and CVE-2017-8822.
+
+  o Minor features (bridge, backport from 0.3.1.9):
+    - Bridges now include notice in their descriptors that they are
+      bridges, and notice of their distribution status, based on their
+      publication settings. Implements ticket 18329. For more fine-
+      grained control of how a bridge is distributed, upgrade to 0.3.2.x
+      or later.
+
+  o Minor features (directory authority, backport from 0.3.2.6-alpha):
+    - Add an IPv6 address for the "bastet" directory authority. Closes
+      ticket 24394.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (testing, backport from 0.3.1.6-rc):
+    - Fix an undersized buffer in test-memwipe.c. Fixes bug 23291;
+      bugfix on 0.2.7.2-alpha. Found and patched by Ties Stuij.
+
+
+Changes in version 0.2.5.16 - 2017-12-01
+  Tor 0.2.5.13 backports important security and stability bugfixes from
+  later Tor releases. All Tor users should upgrade to this release, or
+  to another of the releases coming out today.
+
+  Note: the Tor 0.2.5 series will no longer be supported after 1 May
+  2018. If you need a release with long-term support, please upgrade to
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Major bugfixes (security, backport from 0.3.2.6-alpha):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, relay, backport from 0.3.2.6-alpha):
+    - When running as a relay, make sure that we never build a path
+      through ourselves, even in the case where we have somehow lost the
+      version of our descriptor appearing in the consensus. Fixes part
+      of bug 21534; bugfix on 0.2.0.1-alpha. This issue is also tracked
+      as TROVE-2017-012 and CVE-2017-8822.
+
+  o Minor features (bridge, backport from 0.3.1.9):
+    - Bridges now include notice in their descriptors that they are
+      bridges, and notice of their distribution status, based on their
+      publication settings. Implements ticket 18329. For more fine-
+      grained control of how a bridge is distributed, upgrade to 0.3.2.x
+      or later.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.3.2.5-alpha - 2017-11-22
+  Tor 0.3.2.5-alpha is the fifth alpha release in the 0.3.2.x series. It
+  fixes several stability and reliability bugs, including a fix for
+  intermittent bootstrapping failures that some people have been seeing
+  since the 0.3.0.x series.
+
+  Please test this alpha out -- many of these fixes will soon be
+  backported to stable Tor versions if no additional bugs are found
+  in them.
+
+  o Major bugfixes (bootstrapping):
+    - Fetch descriptors aggressively whenever we lack enough to build
+      circuits, regardless of how many descriptors we are missing.
+      Previously, we would delay launching the fetch when we had fewer
+      than 15 missing descriptors, even if some of those descriptors
+      were blocking circuits from building. Fixes bug 23985; bugfix on
+      0.1.1.11-alpha. The effects of this bug became worse in
+      0.3.0.3-alpha, when we began treating missing descriptors from our
+      primary guards as a reason to delay circuits.
+    - Don't try fetching microdescriptors from relays that have failed
+      to deliver them in the past. Fixes bug 23817; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor features (directory authority):
+    - Make the "Exit" flag assignment only depend on whether the exit
+      policy allows connections to ports 80 and 443. Previously relays
+      would get the Exit flag if they allowed connections to one of
+      these ports and also port 6667. Resolves ticket 23637.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (linux seccomp2 sandbox):
+    - Update the sandbox rules so that they should now work correctly
+      with Glibc 2.26. Closes ticket 24315.
+
+  o Minor features (logging):
+    - Downgrade a pair of log messages that could occur when an exit's
+      resolver gave us an unusual (but not forbidden) response. Closes
+      ticket 24097.
+    - Improve the message we log when re-enabling circuit build timeouts
+      after having received a consensus. Closes ticket 20963.
+
+  o Minor bugfixes (compilation):
+    - Fix a memory leak warning in one of the libevent-related
+      configuration tests that could occur when manually specifying
+      -fsanitize=address. Fixes bug 24279; bugfix on 0.3.0.2-alpha.
+      Found and patched by Alex Xu.
+    - When detecting OpenSSL on Windows from our configure script, make
+      sure to try linking with the ws2_32 library. Fixes bug 23783;
+      bugfix on 0.3.2.2-alpha.
+
+  o Minor bugfixes (control port, linux seccomp2 sandbox):
+    - Avoid a crash when attempting to use the seccomp2 sandbox together
+      with the OwningControllerProcess feature. Fixes bug 24198; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (control port, onion services):
+    - Report "FAILED" instead of "UPLOAD_FAILED" "FAILED" for the
+      HS_DESC event when a service is not able to upload a descriptor.
+      Fixes bug 24230; bugfix on 0.2.7.1-alpha.
+
+  o Minor bugfixes (directory cache):
+    - Recover better from empty or corrupt files in the consensus cache
+      directory. Fixes bug 24099; bugfix on 0.3.1.1-alpha.
+    - When a consensus diff calculation is only partially successful,
+      only record the successful parts as having succeeded. Partial
+      success can happen if (for example) one compression method fails
+      but the others succeed. Previously we misrecorded all the
+      calculations as having succeeded, which would later cause a
+      nonfatal assertion failure. Fixes bug 24086; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Only log once if we notice that KIST support is gone. Fixes bug
+      24158; bugfix on 0.3.2.1-alpha.
+    - Suppress a log notice when relay descriptors arrive. We already
+      have a bootstrap progress for this so no need to log notice
+      everytime tor receives relay descriptors. Microdescriptors behave
+      the same. Fixes bug 23861; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (network layer):
+    - When closing a connection via close_connection_immediately(), we
+      mark it as "not blocked on bandwidth", to prevent later calls from
+      trying to unblock it, and give it permission to read. This fixes a
+      backtrace warning that can happen on relays under various
+      circumstances. Fixes bug 24167; bugfix on 0.1.0.1-rc.
+
+  o Minor bugfixes (onion services):
+    - The introduction circuit was being timed out too quickly while
+      waiting for the rendezvous circuit to complete. Keep the intro
+      circuit around longer instead of timing out and reopening new ones
+      constantly. Fixes bug 23681; bugfix on 0.2.4.8-alpha.
+    - Rename the consensus parameter "hsdir-interval" to "hsdir_interval"
+      so it matches dir-spec.txt. Fixes bug 24262; bugfix
+      on 0.3.1.1-alpha.
+    - Silence a warning about failed v3 onion descriptor uploads that
+      can happen naturally under certain edge cases. Fixes part of bug
+      23662; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (tests):
+    - Fix a memory leak in one of the bridge-distribution test cases.
+      Fixes bug 24345; bugfix on 0.3.2.3-alpha.
+    - Fix a bug in our fuzzing mock replacement for crypto_pk_checksig(),
+      to correctly handle cases where a caller gives it an RSA key of
+      under 160 bits. (This is not actually a bug in Tor itself, but
+      rather in our fuzzing code.) Fixes bug 24247; bugfix on
+      0.3.0.3-alpha. Found by OSS-Fuzz as issue 4177.
+
+  o Documentation:
+    - Add notes in man page regarding OS support for the various
+      scheduler types. Attempt to use less jargon in the scheduler
+      section. Closes ticket 24254.
+
+
+Changes in version 0.3.2.4-alpha - 2017-11-08
+  Tor 0.3.2.4-alpha is the fourth alpha release in the 0.3.2.x series.
+  It fixes several stability and reliability bugs, especially including
+  a major reliability issue that has been plaguing fast exit relays in
+  recent months.
+
+  o Major bugfixes (exit relays, DNS):
+    - Fix an issue causing DNS to fail on high-bandwidth exit nodes,
+      making them nearly unusable. Fixes bugs 21394 and 18580; bugfix on
+      0.1.2.2-alpha, which introduced eventdns. Thanks to Dhalgren for
+      identifying and finding a workaround to this bug and to Moritz,
+      Arthur Edelstein, and Roger for helping to track it down and
+      analyze it.
+
+  o Major bugfixes (scheduler, channel):
+    - Stop processing scheduled channels if they closed while flushing
+      cells. This can happen if the write on the connection fails
+      leading to the channel being closed while in the scheduler loop.
+      Fixes bug 23751; bugfix on 0.3.2.1-alpha.
+
+  o Minor features (logging, scheduler):
+    - Introduce a SCHED_BUG() function to log extra information about
+      the scheduler state if we ever catch a bug in the scheduler.
+      Closes ticket 23753.
+
+  o Minor features (removed deprecations):
+    - The ClientDNSRejectInternalAddresses flag can once again be set in
+      non-testing Tor networks, so long as they do not use the default
+      directory authorities. This change also removes the deprecation of
+      this flag from 0.2.9.2-alpha. Closes ticket 21031.
+
+  o Minor features (testing):
+    - Our fuzzing tests now test the encrypted portions of v3 onion
+      service descriptors. Implements more of 21509.
+
+  o Minor bugfixes (directory client):
+    - On failure to download directory information, delay retry attempts
+      by a random amount based on the "decorrelated jitter" algorithm.
+      Our previous delay algorithm tended to produce extra-long delays
+      too easily. Fixes bug 23816; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (IPv6, v3 single onion services):
+    - Remove buggy code for IPv6-only v3 single onion services, and
+      reject attempts to configure them. This release supports IPv4,
+      dual-stack, and IPv6-only v3 onion services; and IPv4 and dual-
+      stack v3 single onion services. Fixes bug 23820; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (logging, relay):
+    - Give only a protocol warning when the ed25519 key is not
+      consistent between the descriptor and microdescriptor of a relay.
+      This can happen, for instance, if the relay has been flagged
+      NoEdConsensus. Fixes bug 24025; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (manpage, onion service):
+    - Document that the HiddenServiceNumIntroductionPoints option is
+      0-10 for v2 services and 0-20 for v3 services. Fixes bug 24115;
+      bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (memory leaks):
+    - Fix a minor memory leak at exit in the KIST scheduler. This bug
+      should have no user-visible impact. Fixes bug 23774; bugfix
+      on 0.3.2.1-alpha.
+    - Fix a memory leak when decrypting a badly formatted v3 onion
+      service descriptor. Fixes bug 24150; bugfix on 0.3.2.1-alpha.
+      Found by OSS-Fuzz; this is OSS-Fuzz issue 3994.
+
+  o Minor bugfixes (onion services):
+    - Cache some needed onion service client information instead of
+      constantly computing it over and over again. Fixes bug 23623;
+      bugfix on 0.3.2.1-alpha.
+    - Properly retry HSv3 descriptor fetches when missing required
+      directory information. Fixes bug 23762; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (path selection):
+    - When selecting relays by bandwidth, avoid a rounding error that
+      could sometimes cause load to be imbalanced incorrectly.
+      Previously, we would always round upwards; now, we round towards
+      the nearest integer. This had the biggest effect when a relay's
+      weight adjustments should have given it weight 0, but it got
+      weight 1 instead. Fixes bug 23318; bugfix on 0.2.4.3-alpha.
+    - When calculating the fraction of nodes that have descriptors, and
+      all nodes in the network have zero bandwidths, count the number of
+      nodes instead. Fixes bug 23318; bugfix on 0.2.4.10-alpha.
+    - Actually log the total bandwidth in compute_weighted_bandwidths().
+      Fixes bug 24170; bugfix on 0.2.4.3-alpha.
+
+  o Minor bugfixes (relay, crash):
+    - Avoid a crash when transitioning from client mode to bridge mode.
+      Previously, we would launch the worker threads whenever our
+      "public server" mode changed, but not when our "server" mode
+      changed. Fixes bug 23693; bugfix on 0.2.6.3-alpha.
+
+  o Minor bugfixes (testing):
+    - Fix a spurious fuzzing-only use of an uninitialized value. Found
+      by Brian Carpenter. Fixes bug 24082; bugfix on 0.3.0.3-alpha.
+    - Test that IPv6-only clients can use microdescriptors when running
+      "make test-network-all". Requires chutney master 61c28b9 or later.
+      Closes ticket 24109.
+
+
+Changes in version 0.3.2.3-alpha - 2017-10-27
+  Tor 0.3.2.3-alpha is the third release in the 0.3.2 series. It fixes
+  numerous small bugs in earlier versions of 0.3.2.x, and adds a new
+  directory authority, Bastet.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Minor features (bridge):
+    - Bridge relays can now set the BridgeDistribution config option to
+      add a "bridge-distribution-request" line to their bridge
+      descriptor, which tells BridgeDB how they'd like their bridge
+      address to be given out. (Note that as of Oct 2017, BridgeDB does
+      not yet implement this feature.) As a side benefit, this feature
+      provides a way to distinguish bridge descriptors from non-bridge
+      descriptors. Implements tickets 18329.
+
+  o Minor features (client, entry guards):
+    - Improve log messages when missing descriptors for primary guards.
+      Resolves ticket 23670.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (bridge):
+    - Overwrite the bridge address earlier in the process of retrieving
+      its descriptor, to make sure we reach it on the configured
+      address. Fixes bug 20532; bugfix on 0.2.0.10-alpha.
+
+  o Minor bugfixes (documentation):
+    - Document better how to read gcov, and what our gcov postprocessing
+      scripts do. Fixes bug 23739; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (entry guards):
+    - Tor now updates its guard state when it reads a consensus
+      regardless of whether it's missing descriptors. That makes tor use
+      its primary guards to fetch descriptors in some edge cases where
+      it would previously have used fallback directories. Fixes bug
+      23862; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (hidden service client):
+    - When handling multiple SOCKS request for the same .onion address,
+      only fetch the service descriptor once.
+    - When a descriptor fetch fails with a non-recoverable error, close
+      all pending SOCKS requests for that .onion. Fixes bug 23653;
+      bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (hidden service):
+    - Always regenerate missing hidden service public key files. Prior
+      to this, if the public key was deleted from disk, it wouldn't get
+      recreated. Fixes bug 23748; bugfix on 0.3.2.2-alpha. Patch
+      from "cathugger".
+    - Make sure that we have a usable ed25519 key when the intro point
+      relay supports ed25519 link authentication. Fixes bug 24002;
+      bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (hidden service, v2):
+    - When reloading configured hidden services, copy all information
+      from the old service object. Previously, some data was omitted,
+      causing delays in descriptor upload, and other bugs. Fixes bug
+      23790; bugfix on 0.2.1.9-alpha.
+
+  o Minor bugfixes (memory safety, defensive programming):
+    - Clear the target address when node_get_prim_orport() returns
+      early. Fixes bug 23874; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (relay):
+    - Avoid a BUG warning when receiving a dubious CREATE cell while an
+      option transition is in progress. Fixes bug 23952; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Adjust the GitLab CI configuration to more closely match that of
+      Travis CI. Fixes bug 23757; bugfix on 0.3.2.2-alpha.
+    - Prevent scripts/test/coverage from attempting to move gcov output
+      to the root directory. Fixes bug 23741; bugfix on 0.2.5.1-alpha.
+    - When running unit tests as root, skip a test that would fail
+      because it expects a permissions error. This affects some
+      continuous integration setups. Fixes bug 23758; bugfix
+      on 0.3.2.2-alpha.
+    - Stop unconditionally mirroring the tor repository in GitLab CI.
+      This prevented developers from enabling GitLab CI on master. Fixes
+      bug 23755; bugfix on 0.3.2.2-alpha.
+    - Fix the hidden service v3 descriptor decoding fuzzing to use the
+      latest decoding API correctly. Fixes bug 21509; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (warnings):
+    - When we get an HTTP request on a SOCKS port, tell the user about
+      the new HTTPTunnelPort option. Previously, we would give a "Tor is
+      not an HTTP Proxy" message, which stopped being true when
+      HTTPTunnelPort was introduced. Fixes bug 23678; bugfix
+      on 0.3.2.1-alpha.
+
+
+Changes in version 0.2.5.15 - 2017-10-25
+  Tor 0.2.5.15 backports a collection of bugfixes from later Tor release
+  series. It also adds a new directory authority, Bastet.
+
+  Note: the Tor 0.2.5 series will no longer be supported after 1 May
+  2018. If you need a release with long-term support, please upgrade to
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Major bugfixes (openbsd, denial-of-service, backport from 0.3.1.5-alpha):
+    - Avoid an assertion failure bug affecting our implementation of
+      inet_pton(AF_INET6) on certain OpenBSD systems whose strtol()
+      handling of "0xx" differs from what we had expected. Fixes bug
+      22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (defensive programming, undefined behavior, backport from 0.3.1.4-alpha):
+    - Fix a memset() off the end of an array when packing cells. This
+      bug should be harmless in practice, since the corrupted bytes are
+      still in the same structure, and are always padding bytes,
+      ignored, or immediately overwritten, depending on compiler
+      behavior. Nevertheless, because the memset()'s purpose is to make
+      sure that any other cell-handling bugs can't expose bytes to the
+      network, we need to fix it. Fixes bug 22737; bugfix on
+      0.2.4.11-alpha. Fixes CID 1401591.
+
+  o Build features (backport from 0.3.1.5-alpha):
+    - Tor's repository now includes a Travis Continuous Integration (CI)
+      configuration file (.travis.yml). This is meant to help new
+      developers and contributors who fork Tor to a Github repository be
+      better able to test their changes, and understand what we expect
+      to pass. To use this new build feature, you must fork Tor to your
+      Github account, then go into the "Integrations" menu in the
+      repository settings for your fork and enable Travis, then push
+      your changes. Closes ticket 22636.
+
+
+Changes in version 0.2.8.16 - 2017-10-25
+  Tor 0.2.8.16 backports a collection of bugfixes from later Tor release
+  series, including a bugfix for a crash issue that had affected relays
+  under memory pressure. It also adds a new directory authority, Bastet.
+
+  Note: the Tor 0.2.8 series will no longer be supported after 1 Jan
+  2018. If you need a release with long-term support, please stick with
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Major bugfixes (relay, crash, assertion failure, backport from 0.3.2.2-alpha):
+    - Fix a timing-based assertion failure that could occur when the
+      circuit out-of-memory handler freed a connection's output buffer.
+      Fixes bug 23690; bugfix on 0.2.6.1-alpha.
+
+  o Minor features (directory authorities, backport from 0.3.2.2-alpha):
+    - Remove longclaw's IPv6 address, as it will soon change. Authority
+      IPv6 addresses were originally added in 0.2.8.1-alpha. This leaves
+      3/8 directory authorities with IPv6 addresses, but there are also
+      52 fallback directory mirrors with IPv6 addresses. Resolves 19760.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.9.13 - 2017-10-25
+  Tor 0.2.9.13 backports a collection of bugfixes from later Tor release
+  series, including a bugfix for a crash issue that had affected relays
+  under memory pressure. It also adds a new directory authority, Bastet.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Major bugfixes (relay, crash, assertion failure, backport from 0.3.2.2-alpha):
+    - Fix a timing-based assertion failure that could occur when the
+      circuit out-of-memory handler freed a connection's output buffer.
+      Fixes bug 23690; bugfix on 0.2.6.1-alpha.
+
+  o Minor features (directory authorities, backport from 0.3.2.2-alpha):
+    - Remove longclaw's IPv6 address, as it will soon change. Authority
+      IPv6 addresses were originally added in 0.2.8.1-alpha. This leaves
+      3/8 directory authorities with IPv6 addresses, but there are also
+      52 fallback directory mirrors with IPv6 addresses. Resolves 19760.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (directory authority, backport from 0.3.1.5-alpha):
+    - When a directory authority rejects a descriptor or extrainfo with
+      a given digest, mark that digest as undownloadable, so that we do
+      not attempt to download it again over and over. We previously
+      tried to avoid downloading such descriptors by other means, but we
+      didn't notice if we accidentally downloaded one anyway. This
+      behavior became problematic in 0.2.7.2-alpha, when authorities
+      began pinning Ed25519 keys. Fixes bug 22349; bugfix
+      on 0.2.1.19-alpha.
+
+  o Minor bugfixes (memory safety, backport from 0.3.2.3-alpha):
+    - Clear the address when node_get_prim_orport() returns early.
+      Fixes bug 23874; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (Windows service, backport from 0.3.1.6-rc):
+    - When running as a Windows service, set the ID of the main thread
+      correctly. Failure to do so made us fail to send log messages to
+      the controller in 0.2.1.16-rc, slowed down controller event
+      delivery in 0.2.7.3-rc and later, and crash with an assertion
+      failure in 0.3.1.1-alpha. Fixes bug 23081; bugfix on 0.2.1.6-alpha.
+      Patch and diagnosis from "Vort".
+
+
+Changes in version 0.3.0.12 - 2017-10-25
+  Tor 0.3.0.12 backports a collection of bugfixes from later Tor release
+  series, including a bugfix for a crash issue that had affected relays
+  under memory pressure. It also adds a new directory authority, Bastet.
+
+  Note: the Tor 0.3.0 series will no longer be supported after 26 Jan
+  2018. If you need a release with long-term support, please stick with
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Major bugfixes (relay, crash, assertion failure, backport from 0.3.2.2-alpha):
+    - Fix a timing-based assertion failure that could occur when the
+      circuit out-of-memory handler freed a connection's output buffer.
+      Fixes bug 23690; bugfix on 0.2.6.1-alpha.
+
+  o Minor features (directory authorities, backport from 0.3.2.2-alpha):
+    - Remove longclaw's IPv6 address, as it will soon change. Authority
+      IPv6 addresses were originally added in 0.2.8.1-alpha. This leaves
+      3/8 directory authorities with IPv6 addresses, but there are also
+      52 fallback directory mirrors with IPv6 addresses. Resolves 19760.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (directory authority, backport from 0.3.1.5-alpha):
+    - When a directory authority rejects a descriptor or extrainfo with
+      a given digest, mark that digest as undownloadable, so that we do
+      not attempt to download it again over and over. We previously
+      tried to avoid downloading such descriptors by other means, but we
+      didn't notice if we accidentally downloaded one anyway. This
+      behavior became problematic in 0.2.7.2-alpha, when authorities
+      began pinning Ed25519 keys. Fixes bug 22349; bugfix
+      on 0.2.1.19-alpha.
+
+  o Minor bugfixes (hidden service, relay, backport from 0.3.2.2-alpha):
+    - Avoid a possible double close of a circuit by the intro point on
+      error of sending the INTRO_ESTABLISHED cell. Fixes bug 23610;
+      bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (memory safety, backport from 0.3.2.3-alpha):
+    - Clear the address when node_get_prim_orport() returns early.
+      Fixes bug 23874; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (Windows service, backport from 0.3.1.6-rc):
+    - When running as a Windows service, set the ID of the main thread
+      correctly. Failure to do so made us fail to send log messages to
+      the controller in 0.2.1.16-rc, slowed down controller event
+      delivery in 0.2.7.3-rc and later, and crash with an assertion
+      failure in 0.3.1.1-alpha. Fixes bug 23081; bugfix on 0.2.1.6-alpha.
+      Patch and diagnosis from "Vort".
+
+
+Changes in version 0.3.1.8 - 2017-10-25
+  Tor 0.3.1.8 is the second stable release in the 0.3.1 series.
+  It includes several bugfixes, including a bugfix for a crash issue
+  that had affected relays under memory pressure. It also adds
+  a new directory authority, Bastet.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Major bugfixes (relay, crash, assertion failure, backport from 0.3.2.2-alpha):
+    - Fix a timing-based assertion failure that could occur when the
+      circuit out-of-memory handler freed a connection's output buffer.
+      Fixes bug 23690; bugfix on 0.2.6.1-alpha.
+
+  o Minor features (directory authorities, backport from 0.3.2.2-alpha):
+    - Remove longclaw's IPv6 address, as it will soon change. Authority
+      IPv6 addresses were originally added in 0.2.8.1-alpha. This leaves
+      3/8 directory authorities with IPv6 addresses, but there are also
+      52 fallback directory mirrors with IPv6 addresses. Resolves 19760.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.3.2.2-alpha):
+    - Fix a compilation warning when building with zstd support on
+      32-bit platforms. Fixes bug 23568; bugfix on 0.3.1.1-alpha. Found
+      and fixed by Andreas Stieger.
+
+  o Minor bugfixes (compression, backport from 0.3.2.2-alpha):
+    - Handle a pathological case when decompressing Zstandard data when
+      the output buffer size is zero. Fixes bug 23551; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (directory authority, backport from 0.3.2.1-alpha):
+    - Remove the length limit on HTTP status lines that authorities can
+      send in their replies. Fixes bug 23499; bugfix on 0.3.1.6-rc.
+
+  o Minor bugfixes (hidden service, relay, backport from 0.3.2.2-alpha):
+    - Avoid a possible double close of a circuit by the intro point on
+      error of sending the INTRO_ESTABLISHED cell. Fixes bug 23610;
+      bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (memory safety, backport from 0.3.2.3-alpha):
+    - Clear the address when node_get_prim_orport() returns early.
+      Fixes bug 23874; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (unit tests, backport from 0.3.2.2-alpha):
+    - Fix additional channelpadding unit test failures by using mocked
+      time instead of actual time for all tests. Fixes bug 23608; bugfix
+      on 0.3.1.1-alpha.
+
+
+Changes in version 0.3.2.2-alpha - 2017-09-29
+  Tor 0.3.2.2-alpha is the second release in the 0.3.2 series. This
+  release fixes several minor bugs in the new scheduler and next-
+  generation onion services; both features were newly added in the 0.3.2
+  series. Other fixes in this alpha include several fixes for non-fatal
+  tracebacks which would appear in logs.
+
+  With the aim to stabilise the 0.3.2 series by 15 December 2017, this
+  alpha does not contain any substantial new features. Minor features
+  include better testing and logging.
+
+  The following comprises the complete list of changes included
+  in 0.3.2.2-alpha:
+
+  o Major bugfixes (relay, crash, assertion failure):
+    - Fix a timing-based assertion failure that could occur when the
+      circuit out-of-memory handler freed a connection's output buffer.
+      Fixes bug 23690; bugfix on 0.2.6.1-alpha.
+
+  o Major bugfixes (scheduler):
+    - If a channel is put into the scheduler's pending list, then it
+      starts closing, and then if the scheduler runs before it finishes
+      closing, the scheduler will get stuck trying to flush its cells
+      while the lower layers refuse to cooperate. Fix that race
+      condition by giving the scheduler an escape method. Fixes bug
+      23676; bugfix on 0.3.2.1-alpha.
+
+  o Minor features (build, compilation):
+    - The "check-changes" feature is now part of the "make check" tests;
+      we'll use it to try to prevent misformed changes files from
+      accumulating. Closes ticket 23564.
+    - Tor builds should now fail if there are any mismatches between the
+      C type representing a configuration variable and the C type the
+      data-driven parser uses to store a value there. Previously, we
+      needed to check these by hand, which sometimes led to mistakes.
+      Closes ticket 23643.
+
+  o Minor features (directory authorities):
+    - Remove longclaw's IPv6 address, as it will soon change. Authority
+      IPv6 addresses were originally added in 0.2.8.1-alpha. This leaves
+      3/8 directory authorities with IPv6 addresses, but there are also
+      52 fallback directory mirrors with IPv6 addresses. Resolves 19760.
+
+  o Minor features (hidden service, circuit, logging):
+    - Improve logging of many callsite in the circuit subsystem to print
+      the circuit identifier(s).
+    - Log when we cleanup an intro point from a service so we know when
+      and for what reason it happened. Closes ticket 23604.
+
+  o Minor features (logging):
+    - Log more circuit information whenever we are about to try to
+      package a relay cell on a circuit with a nonexistent n_chan.
+      Attempt to diagnose ticket 8185.
+    - Improve info-level log identification of particular circuits, to
+      help with debugging. Closes ticket 23645.
+
+  o Minor features (relay):
+    - When choosing which circuits can be expired as unused, consider
+      circuits from clients even if those clients used regular CREATE
+      cells to make them; and do not consider circuits from relays even
+      if they were made with CREATE_FAST. Part of ticket 22805.
+
+  o Minor features (robustness):
+    - Change several fatal assertions when flushing buffers into non-
+      fatal assertions, to prevent any recurrence of 23690.
+
+  o Minor features (spec conformance, bridge, diagnostic):
+    - When handling the USERADDR command on an ExtOrPort, warn when the
+      transports provides a USERADDR with no port. In a future version,
+      USERADDR commands of this format may be rejected. Detects problems
+      related to ticket 23080.
+
+  o Minor features (testing):
+    - Add a unit test to make sure that our own generated platform
+      string will be accepted by directory authorities. Closes
+      ticket 22109.
+
+  o Minor bugfixes (bootstrapping):
+    - When warning about state file clock skew, report the correct
+      direction for the detected skew. Fixes bug 23606; bugfix
+      on 0.2.8.1-alpha.
+    - Avoid an assertion failure when logging a state file clock skew
+      very early in bootstrapping. Fixes bug 23607; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (build, compilation):
+    - Fix a compilation warning when building with zstd support on
+      32-bit platforms. Fixes bug 23568; bugfix on 0.3.1.1-alpha. Found
+      and fixed by Andreas Stieger.
+    - When searching for OpenSSL, don't accept any OpenSSL library that
+      lacks TLSv1_1_method(): Tor doesn't build with those versions.
+      Additionally, look in /usr/local/opt/openssl, if it's present.
+      These changes together repair the default build on OSX systems
+      with Homebrew installed. Fixes bug 23602; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (compression):
+    - Handle a pathological case when decompressing Zstandard data when
+      the output buffer size is zero. Fixes bug 23551; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (documentation):
+    - Fix manpage to not refer to the obsolete (and misspelled)
+      UseEntryGuardsAsDirectoryGuards parameter in the description of
+      NumDirectoryGuards. Fixes bug 23611; bugfix on 0.2.4.8-alpha.
+
+  o Minor bugfixes (hidden service v3):
+    - Don't log an assertion failure when we can't find the right
+      information to extend to an introduction point. In rare cases,
+      this could happen, causing a warning, even though tor would
+      recover gracefully. Fixes bug 23159; bugfix on 0.3.2.1-alpha.
+    - Pad RENDEZVOUS cell up to the size of the legacy cell which is
+      much bigger so the rendezvous point can't distinguish which hidden
+      service protocol is being used. Fixes bug 23420; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (hidden service, relay):
+    - Avoid a possible double close of a circuit by the intro point on
+      error of sending the INTRO_ESTABLISHED cell. Fixes bug 23610;
+      bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (logging, relay shutdown, annoyance):
+    - When a circuit is marked for close, do not attempt to package any
+      cells for channels on that circuit. Previously, we would detect
+      this condition lower in the call stack, when we noticed that the
+      circuit had no attached channel, and log an annoying message.
+      Fixes bug 8185; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (scheduler):
+    - When switching schedulers due to a consensus change, we didn't
+      give the new scheduler a chance to react to the consensus. Fix
+      that. Fixes bug 23537; bugfix on 0.3.2.1-alpha.
+    - Make the KISTSchedRunInterval option a non negative value. With
+      this, the way to disable KIST through the consensus is to set it
+      to 0. Fixes bug 23539; bugfix on 0.3.2.1-alpha.
+    - Only notice log the selected scheduler when we switch scheduler
+      types. Fixes bug 23552; bugfix on 0.3.2.1-alpha.
+    - Avoid a compilation warning on macOS in scheduler_ev_add() caused
+      by a different tv_usec data type. Fixes bug 23575; bugfix
+      on 0.3.2.1-alpha.
+    - Make a hard exit if tor is unable to pick a scheduler which can
+      happen if the user specifies a scheduler type that is not
+      supported and not other types in Schedulers. Fixes bug 23581;
+      bugfix on 0.3.2.1-alpha.
+    - Properly initialize the scheduler last run time counter so it is
+      not 0 at the first tick. Fixes bug 23696; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Capture and detect several "Result does not fit" warnings in unit
+      tests on platforms with 32-bit time_t. Fixes bug 21800; bugfix
+      on 0.2.9.3-alpha.
+    - Fix additional channelpadding unit test failures by using mocked
+      time instead of actual time for all tests. Fixes bug 23608; bugfix
+      on 0.3.1.1-alpha.
+    - The removal of some old scheduler options caused some tests to
+      fail on BSD systems. Assume current behavior is correct and make
+      the tests pass again. Fixes bug 23566; bugfix on 0.3.2.1-alpha.
+
+  o Code simplification and refactoring:
+    - Remove various ways of testing circuits and connections for
+      "clientness"; instead, favor channel_is_client(). Part of
+      ticket 22805.
+
+  o Deprecated features:
+    - The ReachableDirAddresses and ClientPreferIPv6DirPort options are
+      now deprecated; they do not apply to relays, and they have had no
+      effect on clients since 0.2.8.x. Closes ticket 19704.
+
+  o Documentation:
+    - HiddenServiceVersion man page entry wasn't mentioning the now
+      supported version 3. Fixes ticket 23580; bugfix on 0.3.2.1-alpha.
+    - Clarify that the Address option is entirely about setting an
+      advertised IPv4 address. Closes ticket 18891.
+    - Clarify the manpage's use of the term "address" to clarify what
+      kind of address is intended. Closes ticket 21405.
+    - Document that onion service subdomains are allowed, and ignored.
+      Closes ticket 18736.
+
+
+Changes in version 0.3.2.1-alpha - 2017-09-18
+  Tor 0.3.2.1-alpha is the first release in the 0.3.2.x series. It
+  includes support for our next-generation ("v3") onion service
+  protocol, and adds a new circuit scheduler for more responsive
+  forwarding decisions from relays. There are also numerous other small
+  features and bugfixes here.
+
+  Below are the changes since Tor 0.3.1.7.
+
+  o Major feature (scheduler, channel):
+    - Tor now uses new schedulers to decide which circuits should
+      deliver cells first, in order to improve congestion at relays. The
+      first type is called "KIST" ("Kernel Informed Socket Transport"),
+      and is only available on Linux-like systems: it uses feedback from
+      the kernel to prevent the kernel's TCP buffers from growing too
+      full. The second new scheduler type is called "KISTLite": it
+      behaves the same as KIST, but runs on systems without kernel
+      support for inspecting TCP implementation details. The old
+      scheduler is still available, under the name "Vanilla". To change
+      the default scheduler preference order, use the new "Schedulers"
+      option. (The default preference order is "KIST,KISTLite,Vanilla".)
+
+      Matt Traudt implemented KIST, based on research by Rob Jansen,
+      John Geddes, Christ Wacek, Micah Sherr, and Paul Syverson. For
+      more information, see the design paper at
+      http://www.robgjansen.com/publications/kist-sec2014.pdf and the
+      followup implementation paper at https://arxiv.org/abs/1709.01044.
+      Closes ticket 12541.
+
+  o Major features (next-generation onion services):
+    - Tor now supports the next-generation onion services protocol for
+      clients and services! As part of this release, the core of
+      proposal 224 has been implemented and is available for
+      experimentation and testing by our users. This newer version of
+      onion services ("v3") features many improvements over the legacy
+      system, including:
+
+      a) Better crypto (replaced SHA1/DH/RSA1024
+      with SHA3/ed25519/curve25519)
+
+      b) Improved directory protocol, leaking much less information to
+      directory servers.
+
+      c) Improved directory protocol, with smaller surface for
+      targeted attacks.
+
+      d) Better onion address security against impersonation.
+
+      e) More extensible introduction/rendezvous protocol.
+
+      f) A cleaner and more modular codebase.
+
+      You can identify a next-generation onion address by its length:
+      they are 56 characters long, as in
+      "4acth47i6kxnvkewtm6q7ib2s3ufpo5sqbsnzjpbi7utijcltosqemad.onion".
+
+      In the future, we will release more options and features for v3
+      onion services, but we first need a testing period, so that the
+      current codebase matures and becomes more robust. Planned features
+      include: offline keys, advanced client authorization, improved
+      guard algorithms, and statistics. For full details, see
+      proposal 224.
+
+      Legacy ("v2") onion services will still work for the foreseeable
+      future, and will remain the default until this new codebase gets
+      tested and hardened. Service operators who want to experiment with
+      the new system can use the 'HiddenServiceVersion 3' torrc
+      directive along with the regular onion service configuration
+      options. We will publish a blog post about this new feature
+      soon! Enjoy!
+
+  o Major bugfixes (usability, control port):
+    - Report trusted clock skew indications as bootstrap errors, so
+      controllers can more easily alert users when their clocks are
+      wrong. Fixes bug 23506; bugfix on 0.1.2.6-alpha.
+
+  o Minor features (bug detection):
+    - Log a warning message with a stack trace for any attempt to call
+      get_options() during option validation. This pattern has caused
+      subtle bugs in the past. Closes ticket 22281.
+
+  o Minor features (client):
+    - You can now use Tor as a tunneled HTTP proxy: use the new
+      HTTPTunnelPort option to open a port that accepts HTTP CONNECT
+      requests. Closes ticket 22407.
+    - Add an extra check to make sure that we always use the newer guard
+      selection code for picking our guards. Closes ticket 22779.
+    - When downloading (micro)descriptors, don't split the list into
+      multiple requests unless we want at least 32 descriptors.
+      Previously, we split at 4, not 32, which led to significant
+      overhead in HTTP request size and degradation in compression
+      performance. Closes ticket 23220.
+
+  o Minor features (command line):
+    - Add a new commandline option, --key-expiration, which prints when
+      the current signing key is going to expire. Implements ticket
+      17639; patch by Isis Lovecruft.
+
+  o Minor features (control port):
+    - If an application tries to use the control port as an HTTP proxy,
+      respond with a meaningful "This is the Tor control port" message,
+      and log the event. Closes ticket 1667. Patch from Ravi
+      Chandra Padmala.
+    - Provide better error message for GETINFO desc/(id|name) when not
+      fetching router descriptors. Closes ticket 5847. Patch by
+      Kevin Butler.
+    - Add GETINFO "{desc,md}/download-enabled", to inform the controller
+      whether Tor will try to download router descriptors and
+      microdescriptors respectively. Closes ticket 22684.
+    - Added new GETINFO targets "ip-to-country/{ipv4,ipv6}-available",
+      so controllers can tell whether the geoip databases are loaded.
+      Closes ticket 23237.
+    - Adds a timestamp field to the CIRC_BW and STREAM_BW bandwidth
+      events. Closes ticket 19254. Patch by "DonnchaC".
+
+  o Minor features (development support):
+    - Developers can now generate a call-graph for Tor using the
+      "calltool" python program, which post-processes object dumps. It
+      should work okay on many Linux and OSX platforms, and might work
+      elsewhere too. To run it, install calltool from
+      https://gitweb.torproject.org/user/nickm/calltool.git and run
+      "make callgraph". Closes ticket 19307.
+
+  o Minor features (ed25519):
+    - Add validation function to checks for torsion components in
+      ed25519 public keys, used by prop224 client-side code. Closes
+      ticket 22006. Math help by Ian Goldberg.
+
+  o Minor features (exit relay, DNS):
+    - Improve the clarity and safety of the log message from evdns when
+      receiving an apparently spoofed DNS reply. Closes ticket 3056.
+
+  o Minor features (integration, hardening):
+    - Add a new NoExec option to prevent Tor from running other
+      programs. When this option is set to 1, Tor will never try to run
+      another program, regardless of the settings of
+      PortForwardingHelper, ClientTransportPlugin, or
+      ServerTransportPlugin. Once NoExec is set, it cannot be disabled
+      without restarting Tor. Closes ticket 22976.
+
+  o Minor features (logging):
+    - Improve the warning message for specifying a relay by nickname.
+      The previous message implied that nickname registration was still
+      part of the Tor network design, which it isn't. Closes
+      ticket 20488.
+    - If the sandbox filter fails to load, suggest to the user that
+      their kernel might not support seccomp2. Closes ticket 23090.
+
+  o Minor features (portability):
+    - Check at configure time whether uint8_t is the same type as
+      unsigned char. Lots of existing code already makes this
+      assumption, and there could be strict aliasing issues if the
+      assumption is violated. Closes ticket 22410.
+
+  o Minor features (relay, configuration):
+    - Reject attempts to use relative file paths when RunAsDaemon is
+      set. Previously, Tor would accept these, but the directory-
+      changing step of RunAsDaemon would give strange and/or confusing
+      results. Closes ticket 22731.
+
+  o Minor features (startup, safety):
+    - When configured to write a PID file, Tor now exits if it is unable
+      to do so. Previously, it would warn and continue. Closes
+      ticket 20119.
+
+  o Minor features (static analysis):
+    - The BUG() macro has been changed slightly so that Coverity no
+      longer complains about dead code if the bug is impossible. Closes
+      ticket 23054.
+
+  o Minor features (testing):
+    - The default chutney network tests now include tests for the v3
+      hidden service design. Make sure you have the latest version of
+      chutney if you want to run these. Closes ticket 22437.
+    - Add a unit test to verify that we can parse a hardcoded v2 hidden
+      service descriptor. Closes ticket 15554.
+
+  o Minor bugfixes (certificate handling):
+    - Fix a time handling bug in Tor certificates set to expire after
+      the year 2106. Fixes bug 23055; bugfix on 0.3.0.1-alpha. Found by
+      Coverity as CID 1415728.
+
+  o Minor bugfixes (client, usability):
+    - Refrain from needlessly rejecting SOCKS5-with-hostnames and
+      SOCKS4a requests that contain IP address strings, even when
+      SafeSocks in enabled, as this prevents user from connecting to
+      known IP addresses without relying on DNS for resolving. SafeSocks
+      still rejects SOCKS connections that connect to IP addresses when
+      those addresses are _not_ encoded as hostnames. Fixes bug 22461;
+      bugfix on Tor 0.2.6.2-alpha.
+
+  o Minor bugfixes (code correctness):
+    - Call htons() in extend_cell_format() for encoding a 16-bit value.
+      Previously we used ntohs(), which happens to behave the same on
+      all the platforms we support, but which isn't really correct.
+      Fixes bug 23106; bugfix on 0.2.4.8-alpha.
+    - For defense-in-depth, make the controller's write_escaped_data()
+      function robust to extremely long inputs. Fixes bug 19281; bugfix
+      on 0.1.1.1-alpha. Reported by Guido Vranken.
+
+  o Minor bugfixes (compilation):
+    - Fix unused-variable warnings in donna's Curve25519 SSE2 code.
+      Fixes bug 22895; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (consensus expiry):
+    - Check for adequate directory information correctly. Previously, Tor
+      would reconsider whether it had sufficient directory information
+      every 2 minutes. Fixes bug 23091; bugfix on 0.2.0.19-alpha.
+
+  o Minor bugfixes (directory protocol):
+    - Directory servers now include a "Date:" http header for response
+      codes other than 200. Clients starting with a skewed clock and a
+      recent consensus were getting "304 Not modified" responses from
+      directory authorities, so without the Date header, the client
+      would never hear about a wrong clock. Fixes bug 23499; bugfix
+      on 0.0.8rc1.
+    - Make clients wait for 6 seconds before trying to download a
+      consensus from an authority. Fixes bug 17750; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (DoS-resistance):
+    - If future code asks if there are any running bridges, without
+      checking if bridges are enabled, log a BUG warning rather than
+      crashing. Fixes bug 23524; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (format strictness):
+    - Restrict several data formats to decimal. Previously, the
+      BuildTimeHistogram entries in the state file, the "bw=" entries in
+      the bandwidth authority file, and the process IDs passed to the
+      __OwningControllerProcess option could all be specified in hex or
+      octal as well as in decimal. This was not an intentional feature.
+      Fixes bug 22802; bugfixes on 0.2.2.1-alpha, 0.2.2.2-alpha,
+      and 0.2.2.28-beta.
+
+  o Minor bugfixes (heartbeat):
+    - If we fail to write a heartbeat message, schedule a retry for the
+      minimum heartbeat interval number of seconds in the future. Fixes
+      bug 19476; bugfix on 0.2.3.1-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox, logging):
+    - Fix some messages on unexpected errors from the seccomp2 library.
+      Fixes bug 22750; bugfix on 0.2.5.1-alpha. Patch from "cypherpunks".
+
+  o Minor bugfixes (logging):
+    - Remove duplicate log messages regarding opening non-local
+      SocksPorts upon parsing config and opening listeners at startup.
+      Fixes bug 4019; bugfix on 0.2.3.3-alpha.
+    - Use a more comprehensible log message when telling the user
+      they've excluded every running exit node. Fixes bug 7890; bugfix
+      on 0.2.2.25-alpha.
+    - When logging the number of descriptors we intend to download per
+      directory request, do not log a number higher than then the number
+      of descriptors we're fetching in total. Fixes bug 19648; bugfix
+      on 0.1.1.8-alpha.
+    - When warning about a directory owned by the wrong user, log the
+      actual name of the user owning the directory. Previously, we'd log
+      the name of the process owner twice. Fixes bug 23487; bugfix
+      on 0.2.9.1-alpha.
+    - The tor specification says hop counts are 1-based, so fix two log
+      messages that mistakenly logged 0-based hop counts. Fixes bug
+      18982; bugfix on 0.2.6.2-alpha and 0.2.4.5-alpha. Patch by teor.
+      Credit to Xiaofan Li for reporting this issue.
+
+  o Minor bugfixes (portability):
+    - Stop using the PATH_MAX variable, which is not defined on GNU
+      Hurd. Fixes bug 23098; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (relay):
+    - When uploading our descriptor for the first time after startup,
+      report the reason for uploading as "Tor just started" rather than
+      leaving it blank. Fixes bug 22885; bugfix on 0.2.3.4-alpha.
+    - Avoid unnecessary calls to directory_fetches_from_authorities() on
+      relays, to prevent spurious address resolutions and descriptor
+      rebuilds. This is a mitigation for bug 21789. Fixes bug 23470;
+      bugfix on in 0.2.8.1-alpha.
+
+  o Minor bugfixes (tests):
+    - Fix a broken unit test for the OutboundAddress option: the parsing
+      function was never returning an error on failure. Fixes bug 23366;
+      bugfix on 0.3.0.3-alpha.
+    - Fix a signed-integer overflow in the unit tests for
+      dir/download_status_random_backoff, which was untriggered until we
+      fixed bug 17750. Fixes bug 22924; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (usability, control port):
+    - Stop making an unnecessary routerlist check in NETINFO clock skew
+      detection; this was preventing clients from reporting NETINFO clock
+      skew to controllers. Fixes bug 23532; bugfix on 0.2.4.4-alpha.
+
+  o Code simplification and refactoring:
+    - Extract the code for handling newly-open channels into a separate
+      function from the general code to handle channel state
+      transitions. This change simplifies our callgraph, reducing the
+      size of the largest strongly connected component by roughly a
+      factor of two. Closes ticket 22608.
+    - Remove dead code for largely unused statistics on the number of
+      times we've attempted various public key operations. Fixes bug
+      19871; bugfix on 0.1.2.4-alpha. Fix by Isis Lovecruft.
+    - Remove several now-obsolete functions for asking about old
+      variants directory authority status. Closes ticket 22311; patch
+      from "huyvq".
+    - Remove some of the code that once supported "Named" and "Unnamed"
+      routers. Authorities no longer vote for these flags. Closes
+      ticket 22215.
+    - Rename the obsolete malleable hybrid_encrypt functions used in TAP
+      and old hidden services, to indicate that they aren't suitable for
+      new protocols or formats. Closes ticket 23026.
+    - Replace our STRUCT_OFFSET() macro with offsetof(). Closes ticket
+      22521. Patch from Neel Chauhan.
+    - Split the enormous circuit_send_next_onion_skin() function into
+      multiple subfunctions. Closes ticket 22804.
+    - Split the portions of the buffer.c module that handle particular
+      protocols into separate modules. Part of ticket 23149.
+    - Use our test macros more consistently, to produce more useful
+      error messages when our unit tests fail. Add coccinelle patches to
+      allow us to re-check for test macro uses. Closes ticket 22497.
+
+  o Deprecated features:
+    - Deprecate HTTPProxy/HTTPProxyAuthenticator config options. They
+      only applies to direct unencrypted HTTP connections to your
+      directory server, which your Tor probably isn't using. Closes
+      ticket 20575.
+
+  o Documentation:
+    - Clarify in the manual that "Sandbox 1" is only supported on Linux
+      kernels. Closes ticket 22677.
+    - Document all values of PublishServerDescriptor in the manpage.
+      Closes ticket 15645.
+    - Improve the documentation for the directory port part of the
+      DirAuthority line. Closes ticket 20152.
+    - Restore documentation for the authorities' "approved-routers"
+      file. Closes ticket 21148.
+
+  o Removed features:
+    - The AllowDotExit option has been removed as unsafe. It has been
+      deprecated since 0.2.9.2-alpha. Closes ticket 23426.
+    - The ClientDNSRejectInternalAddresses flag can no longer be set on
+      non-testing networks. It has been deprecated since 0.2.9.2-alpha.
+      Closes ticket 21031.
+    - The controller API no longer includes an AUTHDIR_NEWDESCS event:
+      nobody was using it any longer. Closes ticket 22377.
+
+
+Changes in version 0.2.8.15 - 2017-09-18
+  Tor 0.2.8.15 backports a collection of bugfixes from later
+  Tor series.
+
+  Most significantly, it includes a fix for TROVE-2017-008, a
+  security bug that affects hidden services running with the
+  SafeLogging option disabled. For more information, see
+  https://trac.torproject.org/projects/tor/ticket/23490
+
+  Note that Tor 0.2.8.x will no longer be supported after 1 Jan
+  2018.  We suggest that you upgrade to the latest stable release if
+  possible.  If you can't, we recommend that you upgrade at least to
+  0.2.9, which will be supported until 2020.
+
+  o Major bugfixes (openbsd, denial-of-service, backport from 0.3.1.5-alpha):
+    - Avoid an assertion failure bug affecting our implementation of
+      inet_pton(AF_INET6) on certain OpenBSD systems whose strtol()
+      handling of "0xx" differs from what we had expected. Fixes bug
+      22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007.
+
+  o Minor features:
+    - Update geoip and geoip6 to the September 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, mingw, backport from 0.3.1.1-alpha):
+    - Backport a fix for an "unused variable" warning that appeared
+      in some versions of mingw. Fixes bug 22838; bugfix on
+      0.2.8.1-alpha.
+
+  o Minor bugfixes (defensive programming, undefined behavior, backport from 0.3.1.4-alpha):
+    - Fix a memset() off the end of an array when packing cells. This
+      bug should be harmless in practice, since the corrupted bytes are
+      still in the same structure, and are always padding bytes,
+      ignored, or immediately overwritten, depending on compiler
+      behavior. Nevertheless, because the memset()'s purpose is to make
+      sure that any other cell-handling bugs can't expose bytes to the
+      network, we need to fix it. Fixes bug 22737; bugfix on
+      0.2.4.11-alpha. Fixes CID 1401591.
+
+  o Build features (backport from 0.3.1.5-alpha):
+    - Tor's repository now includes a Travis Continuous Integration (CI)
+      configuration file (.travis.yml). This is meant to help new
+      developers and contributors who fork Tor to a Github repository be
+      better able to test their changes, and understand what we expect
+      to pass. To use this new build feature, you must fork Tor to your
+      Github account, then go into the "Integrations" menu in the
+      repository settings for your fork and enable Travis, then push
+      your changes. Closes ticket 22636.
+
+
+Changes in version 0.2.9.12 - 2017-09-18
+  Tor 0.2.9.12 backports a collection of bugfixes from later
+  Tor series.
+
+  Most significantly, it includes a fix for TROVE-2017-008, a
+  security bug that affects hidden services running with the
+  SafeLogging option disabled. For more information, see
+  https://trac.torproject.org/projects/tor/ticket/23490
+
+  o Major features (security, backport from 0.3.0.2-alpha):
+    - Change the algorithm used to decide DNS TTLs on client and server
+      side, to better resist DNS-based correlation attacks like the
+      DefecTor attack of Greschbach, Pulls, Roberts, Winter, and
+      Feamster. Now relays only return one of two possible DNS TTL
+      values, and clients are willing to believe DNS TTL values up to 3
+      hours long. Closes ticket 19769.
+
+  o Major bugfixes (crash, directory connections, backport from 0.3.0.5-rc):
+    - Fix a rare crash when sending a begin cell on a circuit whose
+      linked directory connection had already been closed. Fixes bug
+      21576; bugfix on 0.2.9.3-alpha. Reported by Alec Muffett.
+
+  o Major bugfixes (DNS, backport from 0.3.0.2-alpha):
+    - Fix a bug that prevented exit nodes from caching DNS records for
+      more than 60 seconds. Fixes bug 19025; bugfix on 0.2.4.7-alpha.
+
+  o Major bugfixes (linux TPROXY support, backport from 0.3.1.1-alpha):
+    - Fix a typo that had prevented TPROXY-based transparent proxying
+      from working under Linux. Fixes bug 18100; bugfix on 0.2.6.3-alpha.
+      Patch from "d4fq0fQAgoJ".
+
+  o Major bugfixes (openbsd, denial-of-service, backport from 0.3.1.5-alpha):
+    - Avoid an assertion failure bug affecting our implementation of
+      inet_pton(AF_INET6) on certain OpenBSD systems whose strtol()
+      handling of "0xx" differs from what we had expected. Fixes bug
+      22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007.
+
+  o Minor features (code style, backport from 0.3.1.3-alpha):
+    - Add "Falls through" comments to our codebase, in order to silence
+      GCC 7's -Wimplicit-fallthrough warnings. Patch from Andreas
+      Stieger. Closes ticket 22446.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the September 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (bandwidth accounting, backport from 0.3.1.1-alpha):
+    - Roll over monthly accounting at the configured hour and minute,
+      rather than always at 00:00. Fixes bug 22245; bugfix on 0.0.9rc1.
+      Found by Andrey Karpov with PVS-Studio.
+
+  o Minor bugfixes (compilation, backport from 0.3.1.5-alpha):
+    - Suppress -Wdouble-promotion warnings with clang 4.0. Fixes bug 22915;
+      bugfix on 0.2.8.1-alpha.
+    - Fix warnings when building with libscrypt and openssl scrypt support
+      on Clang. Fixes bug 22916; bugfix on 0.2.7.2-alpha.
+    - When building with certain versions the mingw C header files, avoid
+      float-conversion warnings when calling the C functions isfinite(),
+      isnan(), and signbit(). Fixes bug 22801; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.1.7):
+    - Avoid compiler warnings in the unit tests for running tor_sscanf()
+      with wide string outputs. Fixes bug 15582; bugfix on 0.2.6.2-alpha.
+
+  o Minor bugfixes (compilation, mingw, backport from 0.3.1.1-alpha):
+    - Backport a fix for an "unused variable" warning that appeared
+      in some versions of mingw. Fixes bug 22838; bugfix on
+      0.2.8.1-alpha.
+
+  o Minor bugfixes (controller, backport from 0.3.1.7):
+    - Do not crash when receiving a HSPOST command with an empty body.
+      Fixes part of bug 22644; bugfix on 0.2.7.1-alpha.
+    - Do not crash when receiving a POSTDESCRIPTOR command with an
+      empty body. Fixes part of bug 22644; bugfix on 0.2.0.1-alpha.
+
+  o Minor bugfixes (coverity build support, backport from 0.3.1.5-alpha):
+    - Avoid Coverity build warnings related to our BUG() macro. By
+      default, Coverity treats BUG() as the Linux kernel does: an
+      instant abort(). We need to override that so our BUG() macro
+      doesn't prevent Coverity from analyzing functions that use it.
+      Fixes bug 23030; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (defensive programming, undefined behavior, backport from 0.3.1.4-alpha):
+    - Fix a memset() off the end of an array when packing cells. This
+      bug should be harmless in practice, since the corrupted bytes are
+      still in the same structure, and are always padding bytes,
+      ignored, or immediately overwritten, depending on compiler
+      behavior. Nevertheless, because the memset()'s purpose is to make
+      sure that any other cell-handling bugs can't expose bytes to the
+      network, we need to fix it. Fixes bug 22737; bugfix on
+      0.2.4.11-alpha. Fixes CID 1401591.
+
+  o Minor bugfixes (file limits, osx, backport from 0.3.1.5-alpha):
+    - When setting the maximum number of connections allowed by the OS,
+      always allow some extra file descriptors for other files. Fixes
+      bug 22797; bugfix on 0.2.0.10-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.1.5-alpha):
+    - Avoid a sandbox failure when trying to re-bind to a socket and
+      mark it as IPv6-only. Fixes bug 20247; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.1.4-alpha):
+    - Permit the fchmod system call, to avoid crashing on startup when
+      starting with the seccomp2 sandbox and an unexpected set of
+      permissions on the data directory or its contents. Fixes bug
+      22516; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (relay, backport from 0.3.0.5-rc):
+    - Avoid a double-marked-circuit warning that could happen when we
+      receive DESTROY cells under heavy load. Fixes bug 20059; bugfix
+      on 0.1.0.1-rc.
+
+  o Minor bugfixes (voting consistency, backport from 0.3.1.1-alpha):
+    - Reject version numbers with non-numeric prefixes (such as +, -, or
+      whitespace). Disallowing whitespace prevents differential version
+      parsing between POSIX-based and Windows platforms. Fixes bug 21507
+      and part of 21508; bugfix on 0.0.8pre1.
+
+  o Build features (backport from 0.3.1.5-alpha):
+    - Tor's repository now includes a Travis Continuous Integration (CI)
+      configuration file (.travis.yml). This is meant to help new
+      developers and contributors who fork Tor to a Github repository be
+      better able to test their changes, and understand what we expect
+      to pass. To use this new build feature, you must fork Tor to your
+      Github account, then go into the "Integrations" menu in the
+      repository settings for your fork and enable Travis, then push
+      your changes. Closes ticket 22636.
+
+
+Changes in version 0.3.0.11 - 2017-09-18
+  Tor 0.3.0.11 backports a collection of bugfixes from Tor the 0.3.1
+  series.
+
+  Most significantly, it includes a fix for TROVE-2017-008, a
+  security bug that affects hidden services running with the
+  SafeLogging option disabled. For more information, see
+  https://trac.torproject.org/projects/tor/ticket/23490
+
+  o Minor features (code style, backport from 0.3.1.7):
+    - Add "Falls through" comments to our codebase, in order to silence
+      GCC 7's -Wimplicit-fallthrough warnings. Patch from Andreas
+      Stieger. Closes ticket 22446.
+
+  o Minor features:
+    - Update geoip and geoip6 to the September 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.3.1.7):
+    - Avoid compiler warnings in the unit tests for calling tor_sscanf()
+      with wide string outputs. Fixes bug 15582; bugfix on 0.2.6.2-alpha.
+
+  o Minor bugfixes (controller, backport from 0.3.1.7):
+    - Do not crash when receiving a HSPOST command with an empty body.
+      Fixes part of bug 22644; bugfix on 0.2.7.1-alpha.
+    - Do not crash when receiving a POSTDESCRIPTOR command with an empty
+      body. Fixes part of bug 22644; bugfix on 0.2.0.1-alpha.
+
+  o Minor bugfixes (file limits, osx, backport from 0.3.1.5-alpha):
+    - When setting the maximum number of connections allowed by the OS,
+      always allow some extra file descriptors for other files. Fixes
+      bug 22797; bugfix on 0.2.0.10-alpha.
+
+  o Minor bugfixes (logging, relay, backport from 0.3.1.6-rc):
+    - Remove a forgotten debugging message when an introduction point
+      successfully establishes a hidden service prop224 circuit with
+      a client.
+    - Change three other log_warn() for an introduction point to
+      protocol warnings, because they can be failure from the network
+      and are not relevant to the operator. Fixes bug 23078; bugfix on
+      0.3.0.1-alpha and 0.3.0.2-alpha.
+
+
+Changes in version 0.3.1.7 - 2017-09-18
+  Tor 0.3.1.7 is the first stable release in the 0.3.1 series.
+
+  With the 0.3.1 series, Tor now serves and downloads directory
+  information in more compact formats, to save on bandwidth overhead. It
+  also contains a new padding system to resist netflow-based traffic
+  analysis, and experimental support for building parts of Tor in Rust
+  (though no parts of Tor are in Rust yet). There are also numerous
+  small features, bugfixes on earlier release series, and groundwork for
+  the hidden services revamp of 0.3.2.
+
+  This release also includes a fix for TROVE-2017-008, a security bug
+  that affects hidden services running with the SafeLogging option
+  disabled. For more information, see
+  https://trac.torproject.org/projects/tor/ticket/23490
+
+  Per our stable release policy, we plan to support each stable release
+  series for at least the next nine months, or for three months after
+  the first stable release of the next series: whichever is longer. If
+  you need a release with long-term support, we recommend that you stay
+  with the 0.2.9 series.
+
+  Below is a list of the changes since 0.3.1.6-rc. For a list of all
+  changes since 0.3.0, see the ReleaseNotes file.
+
+  o Major bugfixes (security, hidden services, loggging):
+    - Fix a bug where we could log uninitialized stack when a certain
+      hidden service error occurred while SafeLogging was disabled.
+      Fixes bug #23490; bugfix on 0.2.7.2-alpha. This is also tracked as
+      TROVE-2017-008 and CVE-2017-0380.
+
+  o Minor features (defensive programming):
+    - Create a pair of consensus parameters, nf_pad_tor2web and
+      nf_pad_single_onion, to disable netflow padding in the consensus
+      for non-anonymous connections in case the overhead is high. Closes
+      ticket 17857.
+
+  o Minor features (diagnostic):
+    - Add a stack trace to the bug warnings that can be logged when
+      trying to send an outgoing relay cell with n_chan == 0. Diagnostic
+      attempt for bug 23105.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the September 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation):
+    - Avoid compiler warnings in the unit tests for calling tor_sscanf()
+      with wide string outputs. Fixes bug 15582; bugfix on 0.2.6.2-alpha.
+
+  o Minor bugfixes (controller):
+    - Do not crash when receiving a HSPOST command with an empty body.
+      Fixes part of bug 22644; bugfix on 0.2.7.1-alpha.
+    - Do not crash when receiving a POSTDESCRIPTOR command with an empty
+      body. Fixes part of bug 22644; bugfix on 0.2.0.1-alpha.
+
+  o Minor bugfixes (relay):
+    - Inform the geoip and rephist modules about all requests, even on
+      relays that are only fetching microdescriptors. Fixes a bug
+      related to 21585; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (unit tests):
+    - Fix a channelpadding unit test failure on slow systems by using
+      mocked time instead of actual time. Fixes bug 23077; bugfix
+      on 0.3.1.1-alpha.
+
+
+Changes in version 0.3.1.6-rc - 2017-09-05
+  Tor 0.3.1.6-rc fixes a few small bugs and annoyances in the 0.3.1
+  release series, including a bug that produced weird behavior on
+  Windows directory caches.
+
+  This is the first release candidate in the Tor 0.3.1 series. If we
+  find no new bugs or regressions here, the first stable 0.3.1 release
+  will be nearly identical to it.
+
+  o Major bugfixes (windows, directory cache):
+    - On Windows, do not try to delete cached consensus documents and
+      diffs before they are unmapped from memory--Windows won't allow
+      that. Instead, allow the consensus cache directory to grow larger,
+      to hold files that might need to stay around longer. Fixes bug
+      22752; bugfix on 0.3.1.1-alpha.
+
+  o Minor features (directory authority):
+    - Improve the message that authorities report to relays that present
+      RSA/Ed25519 keypairs that conflict with previously pinned keys.
+      Closes ticket 22348.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the August 3 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (testing):
+    - Add more tests for compression backend initialization. Closes
+      ticket 22286.
+
+  o Minor bugfixes (directory cache):
+    - Fix a memory leak when recovering space in the consensus cache.
+      Fixes bug 23139; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (hidden service):
+    - Increase the number of circuits that a service is allowed to
+      open over a specific period of time. The value was lower than it
+      should be (8 vs 12) in the normal case of 3 introduction points.
+      Fixes bug 22159; bugfix on 0.3.0.5-rc.
+    - Fix a BUG warning during HSv3 descriptor decoding that could be
+      cause by a specially crafted descriptor. Fixes bug 23233; bugfix
+      on 0.3.0.1-alpha. Bug found by "haxxpop".
+    - Rate-limit the log messages if we exceed the maximum number of
+      allowed intro circuits. Fixes bug 22159; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (logging, relay):
+    - Remove a forgotten debugging message when an introduction point
+      successfully establishes a hidden service prop224 circuit with
+      a client.
+    - Change three other log_warn() for an introduction point to
+      protocol warnings, because they can be failure from the network
+      and are not relevant to the operator. Fixes bug 23078; bugfix on
+      0.3.0.1-alpha and 0.3.0.2-alpha.
+
+  o Minor bugfixes (relay):
+    - When a relay is not running as a directory cache, it will no
+      longer generate compressed consensuses and consensus diff
+      information. Previously, this was a waste of disk and CPU. Fixes
+      bug 23275; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (robustness, error handling):
+    - Improve our handling of the cases where OpenSSL encounters a
+      memory error while encoding keys and certificates. We haven't
+      observed these errors in the wild, but if they do happen, we now
+      detect and respond better. Fixes bug 19418; bugfix on all versions
+      of Tor. Reported by Guido Vranken.
+
+  o Minor bugfixes (stability):
+    - Avoid crashing on a double-free when unable to load or process an
+      included file. Fixes bug 23155; bugfix on 0.3.1.1-alpha. Found
+      with the clang static analyzer.
+
+  o Minor bugfixes (testing):
+    - Fix an undersized buffer in test-memwipe.c. Fixes bug 23291;
+      bugfix on 0.2.7.2-alpha. Found and patched by Ties Stuij.
+    - Port the hs_ntor handshake test to work correctly with recent
+      versions of the pysha3 module. Fixes bug 23071; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (Windows service):
+    - When running as a Windows service, set the ID of the main thread
+      correctly. Failure to do so made us fail to send log messages to
+      the controller in 0.2.1.16-rc, slowed down controller event
+      delivery in 0.2.7.3-rc and later, and crash with an assertion
+      failure in 0.3.1.1-alpha. Fixes bug 23081; bugfix on 0.2.1.6-alpha.
+      Patch and diagnosis from "Vort".
+
+
+Changes in version 0.3.0.10 - 2017-08-02
+   Tor 0.3.0.10 backports a collection of small-to-medium bugfixes
+   from the current Tor alpha series. OpenBSD users and TPROXY users
+   should upgrade; others are probably okay sticking with 0.3.0.9.
+
+  o Major features (build system, continuous integration, backport from 0.3.1.5-alpha):
+    - Tor's repository now includes a Travis Continuous Integration (CI)
+      configuration file (.travis.yml). This is meant to help new
+      developers and contributors who fork Tor to a Github repository be
+      better able to test their changes, and understand what we expect
+      to pass. To use this new build feature, you must fork Tor to your
+      Github account, then go into the "Integrations" menu in the
+      repository settings for your fork and enable Travis, then push
+      your changes. Closes ticket 22636.
+
+  o Major bugfixes (linux TPROXY support, backport from 0.3.1.1-alpha):
+    - Fix a typo that had prevented TPROXY-based transparent proxying
+      from working under Linux. Fixes bug 18100; bugfix on 0.2.6.3-alpha.
+      Patch from "d4fq0fQAgoJ".
+
+  o Major bugfixes (openbsd, denial-of-service, backport from 0.3.1.5-alpha):
+    - Avoid an assertion failure bug affecting our implementation of
+      inet_pton(AF_INET6) on certain OpenBSD systems whose strtol()
+      handling of "0xbar" differs from what we had expected. Fixes bug
+      22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007.
+
+  o Minor features (backport from 0.3.1.5-alpha):
+    - Update geoip and geoip6 to the July 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (bandwidth accounting, backport from 0.3.1.2-alpha):
+    - Roll over monthly accounting at the configured hour and minute,
+      rather than always at 00:00. Fixes bug 22245; bugfix on 0.0.9rc1.
+      Found by Andrey Karpov with PVS-Studio.
+
+  o Minor bugfixes (compilation warnings, backport from 0.3.1.5-alpha):
+    - Suppress -Wdouble-promotion warnings with clang 4.0. Fixes bug 22915;
+      bugfix on 0.2.8.1-alpha.
+    - Fix warnings when building with libscrypt and openssl scrypt
+      support on Clang. Fixes bug 22916; bugfix on 0.2.7.2-alpha.
+    - When building with certain versions of the mingw C header files,
+      avoid float-conversion warnings when calling the C functions
+      isfinite(), isnan(), and signbit(). Fixes bug 22801; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation, mingw, backport from 0.3.1.1-alpha):
+    - Backport a fix for an "unused variable" warning that appeared
+      in some versions of mingw. Fixes bug 22838; bugfix on
+      0.2.8.1-alpha.
+
+  o Minor bugfixes (coverity build support, backport from 0.3.1.5-alpha):
+    - Avoid Coverity build warnings related to our BUG() macro. By
+      default, Coverity treats BUG() as the Linux kernel does: an
+      instant abort(). We need to override that so our BUG() macro
+      doesn't prevent Coverity from analyzing functions that use it.
+      Fixes bug 23030; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (directory authority, backport from 0.3.1.1-alpha):
+    - When rejecting a router descriptor for running an obsolete version
+      of Tor without ntor support, warn about the obsolete tor version,
+      not the missing ntor key. Fixes bug 20270; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.1.5-alpha):
+    - Avoid a sandbox failure when trying to re-bind to a socket and
+      mark it as IPv6-only. Fixes bug 20247; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (unit tests, backport from 0.3.1.5-alpha)
+    - Fix a memory leak in the link-handshake/certs_ok_ed25519 test.
+      Fixes bug 22803; bugfix on 0.3.0.1-alpha.
+
+
+Changes in version 0.3.1.5-alpha - 2017-08-01
+  Tor 0.3.1.5-alpha improves the performance of consensus diff
+  calculation, fixes a crash bug on older versions of OpenBSD, and fixes
+  several other bugs. If no serious bugs are found in this version, the
+  next version will be a release candidate.
+
+  This release also marks the end of support for the Tor 0.2.4.x,
+  0.2.6.x, and 0.2.7.x release series. Those releases will receive no
+  further bug or security fixes. Anyone still running or distributing
+  one of those versions should upgrade.
+
+  o Major features (build system, continuous integration):
+    - Tor's repository now includes a Travis Continuous Integration (CI)
+      configuration file (.travis.yml). This is meant to help new
+      developers and contributors who fork Tor to a Github repository be
+      better able to test their changes, and understand what we expect
+      to pass. To use this new build feature, you must fork Tor to your
+      Github account, then go into the "Integrations" menu in the
+      repository settings for your fork and enable Travis, then push
+      your changes. Closes ticket 22636.
+
+  o Major bugfixes (openbsd, denial-of-service):
+    - Avoid an assertion failure bug affecting our implementation of
+      inet_pton(AF_INET6) on certain OpenBSD systems whose strtol()
+      handling of "0xbar" differs from what we had expected. Fixes bug
+      22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007.
+
+  o Major bugfixes (relay, performance):
+    - Perform circuit handshake operations at a higher priority than we
+      use for consensus diff creation and compression. This should
+      prevent circuits from starving when a relay or bridge receives a
+      new consensus, especially on lower-powered machines. Fixes bug
+      22883; bugfix on 0.3.1.1-alpha.
+
+  o Minor features (bridge authority):
+    - Add "fingerprint" lines to the networkstatus-bridges file produced
+      by bridge authorities. Closes ticket 22207.
+
+  o Minor features (directory cache, consensus diff):
+    - Add a new MaxConsensusAgeForDiffs option to allow directory cache
+      operators with low-resource environments to adjust the number of
+      consensuses they'll store and generate diffs from. Most cache
+      operators should leave it unchanged. Helps to work around
+      bug 22883.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the July 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (relay, performance):
+    - Always start relays with at least two worker threads, to prevent
+      priority inversion on slow tasks. Part of the fix for bug 22883.
+    - Allow background work to be queued with different priorities, so
+      that a big pile of slow low-priority jobs will not starve out
+      higher priority jobs. This lays the groundwork for a fix for
+      bug 22883.
+
+  o Minor bugfixes (build system, rust):
+    - Fix a problem where Rust toolchains were not being found when
+      building without --enable-cargo-online-mode, due to setting the
+      $HOME environment variable instead of $CARGO_HOME. Fixes bug
+      22830; bugfix on 0.3.1.1-alpha. Fix by Chelsea Komlo.
+
+  o Minor bugfixes (compatibility, zstd):
+    - Write zstd epilogues correctly when the epilogue requires
+      reallocation of the output buffer, even with zstd 1.3.0.
+      (Previously, we worked on 1.2.0 and failed with 1.3.0). Fixes bug
+      22927; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (compilation warnings):
+    - Suppress -Wdouble-promotion warnings with clang 4.0. Fixes bug
+      22915; bugfix on 0.2.8.1-alpha.
+    - Fix warnings when building with libscrypt and openssl scrypt
+      support on Clang. Fixes bug 22916; bugfix on 0.2.7.2-alpha.
+    - Compile correctly when both openssl 1.1.0 and libscrypt are
+      detected. Previously this would cause an error. Fixes bug 22892;
+      bugfix on 0.3.1.1-alpha.
+    - When building with certain versions of the mingw C header files,
+      avoid float-conversion warnings when calling the C functions
+      isfinite(), isnan(), and signbit(). Fixes bug 22801; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (coverity build support):
+    - Avoid Coverity build warnings related to our BUG() macro. By
+      default, Coverity treats BUG() as the Linux kernel does: an
+      instant abort(). We need to override that so our BUG() macro
+      doesn't prevent Coverity from analyzing functions that use it.
+      Fixes bug 23030; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (directory authority):
+    - When a directory authority rejects a descriptor or extrainfo with
+      a given digest, mark that digest as undownloadable, so that we do
+      not attempt to download it again over and over. We previously
+      tried to avoid downloading such descriptors by other means, but we
+      didn't notice if we accidentally downloaded one anyway. This
+      behavior became problematic in 0.2.7.2-alpha, when authorities
+      began pinning Ed25519 keys. Fixes bug 22349; bugfix
+      on 0.2.1.19-alpha.
+
+  o Minor bugfixes (error reporting, windows):
+    - When formatting Windows error messages, use the English format to
+      avoid codepage issues. Fixes bug 22520; bugfix on 0.1.2.8-alpha.
+      Patch from "Vort".
+
+  o Minor bugfixes (file limits, osx):
+    - When setting the maximum number of connections allowed by the OS,
+      always allow some extra file descriptors for other files. Fixes
+      bug 22797; bugfix on 0.2.0.10-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox):
+    - Avoid a sandbox failure when trying to re-bind to a socket and
+      mark it as IPv6-only. Fixes bug 20247; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (memory leaks):
+    - Fix a small memory leak when validating a configuration that uses
+      two or more AF_UNIX sockets for the same port type. Fixes bug
+      23053; bugfix on 0.2.6.3-alpha. This is CID 1415725.
+
+  o Minor bugfixes (unit tests):
+    - test_consdiff_base64cmp would fail on OS X because while OS X
+      follows the standard of (less than zero/zero/greater than zero),
+      it doesn't follow the convention of (-1/0/+1). Make the test
+      comply with the standard. Fixes bug 22870; bugfix on 0.3.1.1-alpha.
+    - Fix a memory leak in the link-handshake/certs_ok_ed25519 test.
+      Fixes bug 22803; bugfix on 0.3.0.1-alpha.
+
+
+Changes in version 0.3.1.4-alpha - 2017-06-29
+  Tor 0.3.1.4-alpha fixes a path selection bug that would allow a client
+  to use a guard that was in the same network family as a chosen exit
+  relay. This is a security regression; all clients running earlier
+  versions of 0.3.0.x or 0.3.1.x should upgrade to 0.3.0.9
+  or 0.3.1.4-alpha.
+
+  This release also fixes several other bugs introduced in 0.3.0.x
+  and 0.3.1.x, including others that can affect bandwidth usage
+  and correctness.
+
+  o New dependencies:
+    - To build with zstd and lzma support, Tor now requires the
+      pkg-config tool at build time. (This requirement was new in
+      0.3.1.1-alpha, but was not noted at the time. Noting it here to
+      close ticket 22623.)
+
+  o Major bugfixes (path selection, security):
+    - When choosing which guard to use for a circuit, avoid the exit's
+      family along with the exit itself. Previously, the new guard
+      selection logic avoided the exit, but did not consider its family.
+      Fixes bug 22753; bugfix on 0.3.0.1-alpha. Tracked as TROVE-2017-
+      006 and CVE-2017-0377.
+
+  o Major bugfixes (compression, zstd):
+    - Correctly detect a full buffer when decompressing a large zstd-
+      compressed input. Previously, we would sometimes treat a full
+      buffer as an error. Fixes bug 22628; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (directory protocol):
+    - Ensure that we send "304 Not modified" as HTTP status code when a
+      client is attempting to fetch a consensus or consensus diff, and
+      the best one we can send them is one they already have. Fixes bug
+      22702; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (entry guards):
+    - When starting with an old consensus, do not add new entry guards
+      unless the consensus is "reasonably live" (under 1 day old). Fixes
+      one root cause of bug 22400; bugfix on 0.3.0.1-alpha.
+
+  o Minor features (bug mitigation, diagnostics, logging):
+    - Avoid an assertion failure, and log a better error message, when
+      unable to remove a file from the consensus cache on Windows.
+      Attempts to mitigate and diagnose bug 22752.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the June 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compression):
+    - When compressing or decompressing a buffer, check for a failure to
+      create a compression object. Fixes bug 22626; bugfix
+      on 0.3.1.1-alpha.
+    - When decompressing a buffer, check for extra data after the end of
+      the compressed data. Fixes bug 22629; bugfix on 0.3.1.1-alpha.
+    - When decompressing an object received over an anonymous directory
+      connection, if we have already decompressed it using an acceptable
+      compression method, do not reject it for looking like an
+      unacceptable compression method. Fixes part of bug 22670; bugfix
+      on 0.3.1.1-alpha.
+    - When serving directory votes compressed with zlib, do not claim to
+      have compressed them with zstd. Fixes bug 22669; bugfix
+      on 0.3.1.1-alpha.
+    - When spooling compressed data to an output buffer, don't try to
+      spool more data when there is no more data to spool and we are not
+      trying to flush the input. Previously, we would sometimes launch
+      compression requests with nothing to do, which interferes with our
+      22672 checks. Fixes bug 22719; bugfix on 0.2.0.16-alpha.
+
+  o Minor bugfixes (defensive programming):
+    - Detect and break out of infinite loops in our compression code. We
+      don't think that any such loops exist now, but it's best to be
+      safe. Closes ticket 22672.
+    - Fix a memset() off the end of an array when packing cells. This
+      bug should be harmless in practice, since the corrupted bytes are
+      still in the same structure, and are always padding bytes,
+      ignored, or immediately overwritten, depending on compiler
+      behavior. Nevertheless, because the memset()'s purpose is to make
+      sure that any other cell-handling bugs can't expose bytes to the
+      network, we need to fix it. Fixes bug 22737; bugfix on
+      0.2.4.11-alpha. Fixes CID 1401591.
+
+  o Minor bugfixes (linux seccomp2 sandbox):
+    - Permit the fchmod system call, to avoid crashing on startup when
+      starting with the seccomp2 sandbox and an unexpected set of
+      permissions on the data directory or its contents. Fixes bug
+      22516; bugfix on 0.2.5.4-alpha.
+    - Fix a crash in the LZMA module, when the sandbox was enabled, and
+      liblzma would allocate more than 16 MB of memory. We solve this by
+      bumping the mprotect() limit in the sandbox module from 16 MB to
+      20 MB. Fixes bug 22751; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (logging):
+    - When decompressing, do not warn if we fail to decompress using a
+      compression method that we merely guessed. Fixes part of bug
+      22670; bugfix on 0.1.1.14-alpha.
+    - When decompressing, treat mismatch between content-encoding and
+      actual compression type as a protocol warning. Fixes part of bug
+      22670; bugfix on 0.1.1.9-alpha.
+    - Downgrade "assigned_to_cpuworker failed" message to info-level
+      severity. In every case that can reach it, either a better warning
+      has already been logged, or no warning is warranted. Fixes bug
+      22356; bugfix on 0.2.6.3-alpha.
+    - Demote a warn that was caused by libevent delays to info if
+      netflow padding is less than 4.5 seconds late, or to notice
+      if it is more (4.5 seconds is the amount of time that a netflow
+      record might be emitted after, if we chose the maximum timeout).
+      Fixes bug 22212; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (process behavior):
+    - When exiting because of an error, always exit with a nonzero exit
+      status. Previously, we would fail to report an error in our exit
+      status in cases related to __OwningControllerProcess failure,
+      lockfile contention, and Ed25519 key initialization. Fixes bug
+      22720; bugfix on versions 0.2.1.6-alpha, 0.2.2.28-beta, and
+      0.2.7.2-alpha respectively. Reported by "f55jwk4f"; patch
+      from "huyvq".
+
+  o Documentation:
+    - Add a manpage description for the key-pinning-journal file. Closes
+      ticket 22347.
+    - Correctly note that bandwidth accounting values are stored in the
+      state file, and the bw_accounting file is now obsolete. Closes
+      ticket 16082.
+    - Document more of the files in the Tor data directory, including
+      cached-extrainfo, secret_onion_key{,_ntor}.old, hidserv-stats,
+      approved-routers, sr-random, and diff-cache. Found while fixing
+      ticket 22347.
+
+
+Changes in version 0.3.0.9 - 2017-06-29
+  Tor 0.3.0.9 fixes a path selection bug that would allow a client
+  to use a guard that was in the same network family as a chosen exit
+  relay. This is a security regression; all clients running earlier
+  versions of 0.3.0.x or 0.3.1.x should upgrade to 0.3.0.9 or
+  0.3.1.4-alpha.
+
+  This release also backports several other bugfixes from the 0.3.1.x
+  series.
+
+  o Major bugfixes (path selection, security, backport from 0.3.1.4-alpha):
+    - When choosing which guard to use for a circuit, avoid the exit's
+      family along with the exit itself. Previously, the new guard
+      selection logic avoided the exit, but did not consider its family.
+      Fixes bug 22753; bugfix on 0.3.0.1-alpha. Tracked as TROVE-2017-
+      006 and CVE-2017-0377.
+
+  o Major bugfixes (entry guards, backport from 0.3.1.1-alpha):
+    - Don't block bootstrapping when a primary bridge is offline and we
+      can't get its descriptor. Fixes bug 22325; fixes one case of bug
+      21969; bugfix on 0.3.0.3-alpha.
+
+  o Major bugfixes (entry guards, backport from 0.3.1.4-alpha):
+    - When starting with an old consensus, do not add new entry guards
+      unless the consensus is "reasonably live" (under 1 day old). Fixes
+      one root cause of bug 22400; bugfix on 0.3.0.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the June 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (voting consistency, backport from 0.3.1.1-alpha):
+    - Reject version numbers with non-numeric prefixes (such as +, -, or
+      whitespace). Disallowing whitespace prevents differential version
+      parsing between POSIX-based and Windows platforms. Fixes bug 21507
+      and part of 21508; bugfix on 0.0.8pre1.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.1.4-alpha):
+    - Permit the fchmod system call, to avoid crashing on startup when
+      starting with the seccomp2 sandbox and an unexpected set of
+      permissions on the data directory or its contents. Fixes bug
+      22516; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (defensive programming, backport from 0.3.1.4-alpha):
+    - Fix a memset() off the end of an array when packing cells. This
+      bug should be harmless in practice, since the corrupted bytes are
+      still in the same structure, and are always padding bytes,
+      ignored, or immediately overwritten, depending on compiler
+      behavior. Nevertheless, because the memset()'s purpose is to make
+      sure that any other cell-handling bugs can't expose bytes to the
+      network, we need to fix it. Fixes bug 22737; bugfix on
+      0.2.4.11-alpha. Fixes CID 1401591.
+
+
+Changes in version 0.3.1.3-alpha - 2017-06-08
+  Tor 0.3.1.3-alpha fixes a pair of bugs that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-004 and TROVE-2017-005.
+
+  Tor 0.3.1.3-alpha also includes fixes for several key management bugs
+  that sometimes made relays unreliable, as well as several other
+  bugfixes described below.
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure when a hidden service
+      handles a malformed BEGIN cell. Fixes bug 22493, tracked as
+      TROVE-2017-004 and as CVE-2017-0375; bugfix on 0.3.0.1-alpha.
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Major bugfixes (relay, link handshake):
+    - When performing the v3 link handshake on a TLS connection, report
+      that we have the x509 certificate that we actually used on that
+      connection, even if we have changed certificates since that
+      connection was first opened. Previously, we would claim to have
+      used our most recent x509 link certificate, which would sometimes
+      make the link handshake fail. Fixes one case of bug 22460; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (relays, key management):
+    - Regenerate link and authentication certificates whenever the key
+      that signs them changes; also, regenerate link certificates
+      whenever the signed key changes. Previously, these processes were
+      only weakly coupled, and we relays could (for minutes to hours)
+      wind up with an inconsistent set of keys and certificates, which
+      other relays would not accept. Fixes two cases of bug 22460;
+      bugfix on 0.3.0.1-alpha.
+    - When sending an Ed25519 signing->link certificate in a CERTS cell,
+      send the certificate that matches the x509 certificate that we
+      used on the TLS connection. Previously, there was a race condition
+      if the TLS context rotated after we began the TLS handshake but
+      before we sent the CERTS cell. Fixes a case of bug 22460; bugfix
+      on 0.3.0.1-alpha.
+
+  o Major bugfixes (torrc, crash):
+    - Fix a crash bug when using %include in torrc. Fixes bug 22417;
+      bugfix on 0.3.1.1-alpha. Patch by Daniel Pinto.
+
+  o Minor features (code style):
+    - Add "Falls through" comments to our codebase, in order to silence
+      GCC 7's -Wimplicit-fallthrough warnings. Patch from Andreas
+      Stieger. Closes ticket 22446.
+
+  o Minor features (diagnostic):
+    - Add logging messages to try to diagnose a rare bug that seems to
+      generate RSA->Ed25519 cross-certificates dated in the 1970s. We
+      think this is happening because of incorrect system clocks, but
+      we'd like to know for certain. Diagnostic for bug 22466.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+  o Minor bugfixes (directory protocol):
+    - Check for libzstd >= 1.1, because older versions lack the
+      necessary streaming API. Fixes bug 22413; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (link handshake):
+    - Lower the lifetime of the RSA->Ed25519 cross-certificate to six
+      months, and regenerate it when it is within one month of expiring.
+      Previously, we had generated this certificate at startup with a
+      ten-year lifetime, but that could lead to weird behavior when Tor
+      was started with a grossly inaccurate clock. Mitigates bug 22466;
+      mitigation on 0.3.0.1-alpha.
+
+  o Minor bugfixes (storage directories):
+    - Always check for underflows in the cached storage directory usage.
+      If the usage does underflow, re-calculate it. Also, avoid a
+      separate underflow when the usage is not known. Fixes bug 22424;
+      bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (unit tests):
+    - The unit tests now pass on systems where localhost is misconfigured
+      to some IPv4 address other than 127.0.0.1. Fixes bug 6298; bugfix
+      on 0.0.9pre2.
+
+  o Documentation:
+    - Clarify the manpage for the (deprecated) torify script. Closes
+      ticket 6892.
+
+Changes in version 0.3.0.8 - 2017-06-08
+  Tor 0.3.0.8 fixes a pair of bugs that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-004 and TROVE-2017-005.
+
+  Tor 0.3.0.8 also includes fixes for several key management bugs
+  that sometimes made relays unreliable, as well as several other
+  bugfixes described below.
+
+  o Major bugfixes (hidden service, relay, security, backport
+    from 0.3.1.3-alpha):
+    - Fix a remotely triggerable assertion failure when a hidden service
+      handles a malformed BEGIN cell. Fixes bug 22493, tracked as
+      TROVE-2017-004 and as CVE-2017-0375; bugfix on 0.3.0.1-alpha.
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Major bugfixes (relay, link handshake, backport from 0.3.1.3-alpha):
+    - When performing the v3 link handshake on a TLS connection, report
+      that we have the x509 certificate that we actually used on that
+      connection, even if we have changed certificates since that
+      connection was first opened. Previously, we would claim to have
+      used our most recent x509 link certificate, which would sometimes
+      make the link handshake fail. Fixes one case of bug 22460; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (relays, key management, backport from 0.3.1.3-alpha):
+    - Regenerate link and authentication certificates whenever the key
+      that signs them changes; also, regenerate link certificates
+      whenever the signed key changes. Previously, these processes were
+      only weakly coupled, and we relays could (for minutes to hours)
+      wind up with an inconsistent set of keys and certificates, which
+      other relays would not accept. Fixes two cases of bug 22460;
+      bugfix on 0.3.0.1-alpha.
+    - When sending an Ed25519 signing->link certificate in a CERTS cell,
+      send the certificate that matches the x509 certificate that we
+      used on the TLS connection. Previously, there was a race condition
+      if the TLS context rotated after we began the TLS handshake but
+      before we sent the CERTS cell. Fixes a case of bug 22460; bugfix
+      on 0.3.0.1-alpha.
+
+  o Major bugfixes (hidden service v3, backport from 0.3.1.1-alpha):
+    - Stop rejecting v3 hidden service descriptors because their size
+      did not match an old padding rule. Fixes bug 22447; bugfix on
+      0.3.0.1-alpha.
+
+  o Minor features (fallback directory list, backport from 0.3.1.3-alpha):
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor bugfixes (configuration, backport from 0.3.1.1-alpha):
+    - Do not crash when starting with LearnCircuitBuildTimeout 0. Fixes
+      bug 22252; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (correctness, backport from 0.3.1.3-alpha):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+  o Minor bugfixes (link handshake, backport from 0.3.1.3-alpha):
+    - Lower the lifetime of the RSA->Ed25519 cross-certificate to six
+      months, and regenerate it when it is within one month of expiring.
+      Previously, we had generated this certificate at startup with a
+      ten-year lifetime, but that could lead to weird behavior when Tor
+      was started with a grossly inaccurate clock. Mitigates bug 22466;
+      mitigation on 0.3.0.1-alpha.
+
+  o Minor bugfixes (memory leak, directory authority, backport from
+    0.3.1.2-alpha):
+    - When directory authorities reject a router descriptor due to
+      keypinning, free the router descriptor rather than leaking the
+      memory. Fixes bug 22370; bugfix on 0.2.7.2-alpha.
+
+
+Changes in version 0.2.9.11 - 2017-06-08
+  Tor 0.2.9.11 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005. (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  Tor 0.2.9.11 also backports fixes for several key management bugs
+  that sometimes made relays unreliable, as well as several other
+  bugfixes described below.
+
+  o Major bugfixes (hidden service, relay, security, backport
+    from 0.3.1.3-alpha):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Major bugfixes (relay, link handshake, backport from 0.3.1.3-alpha):
+    - When performing the v3 link handshake on a TLS connection, report
+      that we have the x509 certificate that we actually used on that
+      connection, even if we have changed certificates since that
+      connection was first opened. Previously, we would claim to have
+      used our most recent x509 link certificate, which would sometimes
+      make the link handshake fail. Fixes one case of bug 22460; bugfix
+      on 0.2.3.6-alpha.
+
+  o Minor features (fallback directory list, backport from 0.3.1.3-alpha):
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor features (future-proofing, backport from 0.3.0.7):
+    - Tor no longer refuses to download microdescriptors or descriptors if
+      they are listed as "published in the future".  This change will
+      eventually allow us to stop listing meaningful "published" dates
+      in microdescriptor consensuses, and thereby allow us to reduce the
+      resources required to download consensus diffs by over 50%.
+      Implements part of ticket 21642; implements part of proposal 275.
+
+  o Minor features (directory authorities, backport from 0.3.0.4-rc)
+    - Directory authorities now reject relays running versions
+      0.2.9.1-alpha through 0.2.9.4-alpha, because those relays
+      suffer from bug 20499 and don't keep their consensus cache
+      up-to-date. Resolves ticket 20509.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (control port, backport from 0.3.0.6):
+    - The GETINFO extra-info/digest/<digest> command was broken because
+      of a wrong base16 decode return value check, introduced when
+      refactoring that API. Fixes bug 22034; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (correctness, backport from 0.3.1.3-alpha):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+  o Minor bugfixes (Linux seccomp2 sandbox, backport from 0.3.0.7):
+    - The getpid() system call is now permitted under the Linux seccomp2
+      sandbox, to avoid crashing with versions of OpenSSL (and other
+      libraries) that attempt to learn the process's PID by using the
+      syscall rather than the VDSO code. Fixes bug 21943; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (memory leak, directory authority, backport
+    from 0.3.1.2-alpha):
+    - When directory authorities reject a router descriptor due to
+      keypinning, free the router descriptor rather than leaking the
+      memory. Fixes bug 22370; bugfix on 0.2.7.2-alpha.
+
+Changes in version 0.2.8.14 - 2017-06-08
+  Tor 0.2.7.8 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (fallback directory list, backport from 0.3.1.3-alpha):
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+Changes in version 0.2.7.8 - 2017-06-08
+  Tor 0.2.7.8 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+
+Changes in version 0.2.6.12 - 2017-06-08
+  Tor 0.2.6.12 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+Changes in version 0.2.5.14 - 2017-06-08
+  Tor 0.2.5.14 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+Changes in version 0.2.4.29 - 2017-06-08
+  Tor 0.2.4.29 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+
+Changes in version 0.3.1.2-alpha - 2017-05-26
+  Tor 0.3.1.2-alpha is the second release in the 0.3.1.x series. It
+  fixes a few bugs found while testing 0.3.1.1-alpha, including a
+  memory corruption bug that affected relay stability.
+
+  o Major bugfixes (crash, relay):
+    - Fix a memory-corruption bug in relays that set MyFamily.
+      Previously, they would double-free MyFamily elements when making
+      the next descriptor or when changing their configuration. Fixes
+      bug 22368; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Log a better message when a directory authority replies to an
+      upload with an unexpected status code. Fixes bug 11121; bugfix
+      on 0.1.0.1-rc.
+
+  o Minor bugfixes (memory leak, directory authority):
+    - When directory authorities reject a router descriptor due to
+      keypinning, free the router descriptor rather than leaking the
+      memory. Fixes bug 22370; bugfix on 0.2.7.2-alpha.
+
+
+Changes in version 0.3.1.1-alpha - 2017-05-22
+  Tor 0.3.1.1-alpha is the first release in the 0.3.1.x series. It
+  reduces the bandwidth usage for Tor's directory protocol, adds some
+  basic padding to resist netflow-based traffic analysis and to serve as
+  the basis of other padding in the future, and adds rust support to the
+  build system.
+
+  It also contains numerous other small features and improvements to
+  security, correctness, and performance.
+
+  Below are the changes since 0.3.0.7.
+
+  o Major features (directory protocol):
+    - Tor relays and authorities can now serve clients an abbreviated
+      version of the consensus document, containing only the changes
+      since an older consensus document that the client holds. Clients
+      now request these documents when available. When both client and
+      server use this new protocol, they will use far less bandwidth (up
+      to 94% less) to keep the client's consensus up-to-date. Implements
+      proposal 140; closes ticket 13339. Based on work by Daniel Martí.
+    - Tor can now compress directory traffic with lzma or with zstd
+      compression algorithms, which can deliver better bandwidth
+      performance. Because lzma is computationally expensive, it's only
+      used for documents that can be compressed once and served many
+      times. Support for these algorithms requires that tor is built
+      with the libzstd and/or liblzma libraries available. Implements
+      proposal 278; closes ticket 21662.
+    - Relays now perform the more expensive compression operations, and
+      consensus diff generation, in worker threads. This separation
+      avoids delaying the main thread when a new consensus arrives.
+
+  o Major features (experimental):
+    - Tor can now build modules written in Rust. To turn this on, pass
+      the "--enable-rust" flag to the configure script. It's not time to
+      get excited yet: currently, there is no actual Rust functionality
+      beyond some simple glue code, and a notice at startup to tell you
+      that Rust is running. Still, we hope that programmers and
+      packagers will try building Tor with Rust support, so that we can
+      find issues and solve portability problems. Closes ticket 22106.
+
+  o Major features (traffic analysis resistance):
+    - Connections between clients and relays now send a padding cell in
+      each direction every 1.5 to 9.5 seconds (tunable via consensus
+      parameters). This padding will not resist specialized
+      eavesdroppers, but it should be enough to make many ISPs' routine
+      network flow logging less useful in traffic analysis against
+      Tor users.
+
+      Padding is negotiated using Tor's link protocol, so both relays
+      and clients must upgrade for this to take effect. Clients may
+      still send padding despite the relay's version by setting
+      ConnectionPadding 1 in torrc, and may disable padding by setting
+      ConnectionPadding 0 in torrc. Padding may be minimized for mobile
+      users with the torrc option ReducedConnectionPadding. Implements
+      Proposal 251 and Section 2 of Proposal 254; closes ticket 16861.
+    - Relays will publish 24 hour totals of padding and non-padding cell
+      counts to their extra-info descriptors, unless PaddingStatistics 0
+      is set in torrc. These 24 hour totals are also rounded to
+      multiples of 10000.
+
+  o Major bugfixes (connection usage):
+    - We use NETINFO cells to try to determine if both relays involved
+      in a connection will agree on the canonical status of that
+      connection. We prefer the connections where this is the case for
+      extend cells, and try to close connections where relays disagree
+      on their canonical status early. Also, we now prefer the oldest
+      valid connection for extend cells. These two changes should reduce
+      the number of long-term connections that are kept open between
+      relays. Fixes bug 17604; bugfix on 0.2.5.5-alpha.
+    - Relays now log hourly statistics (look for
+      "channel_check_for_duplicates" lines) on the total number of
+      connections to other relays. If the number of connections per
+      relay is unexpectedly large, this log message is at notice level.
+      Otherwise it is at info.
+
+  o Major bugfixes (entry guards):
+    - Don't block bootstrapping when a primary bridge is offline and we
+      can't get its descriptor. Fixes bug 22325; fixes one case of bug
+      21969; bugfix on 0.3.0.3-alpha.
+
+  o Major bugfixes (linux TPROXY support):
+    - Fix a typo that had prevented TPROXY-based transparent proxying
+      from working under Linux. Fixes bug 18100; bugfix on 0.2.6.3-alpha.
+      Patch from "d4fq0fQAgoJ".
+
+  o Minor features (security, windows):
+    - Enable a couple of pieces of Windows hardening: one
+      (HeapEnableTerminationOnCorruption) that has been on-by-default
+      since Windows 8, and unavailable before Windows 7; and one
+      (PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION) which we believe doesn't
+      affect us, but shouldn't do any harm. Closes ticket 21953.
+
+  o Minor features (config options):
+    - Allow "%include" directives in torrc configuration files. These
+      directives import the settings from other files, or from all the
+      files in a directory. Closes ticket 1922. Code by Daniel Pinto.
+    - Make SAVECONF return an error when overwriting a torrc that has
+      includes. Using SAVECONF with the FORCE option will allow it to
+      overwrite torrc even if includes are used. Related to ticket 1922.
+    - Add "GETINFO config-can-saveconf" to tell controllers if SAVECONF
+      will work without the FORCE option. Related to ticket 1922.
+
+  o Minor features (controller):
+    - Warn the first time that a controller requests data in the long-
+      deprecated 'GETINFO network-status' format. Closes ticket 21703.
+
+  o Minor features (defaults):
+    - The default value for UseCreateFast is now 0: clients which
+      haven't yet received a consensus document will now use a proper
+      ntor handshake to talk to their directory servers whenever they
+      can. Closes ticket 21407.
+    - Onion key rotation and expiry intervals are now defined as a
+      network consensus parameter, per proposal 274. The default
+      lifetime of an onion key is increased from 7 to 28 days. Old onion
+      keys will expire after 7 days by default. This change will make
+      consensus diffs much smaller, and save significant bandwidth.
+      Closes ticket 21641.
+
+  o Minor features (fallback directory list):
+    - Update the fallback directory mirror whitelist and blacklist based
+      on operator emails. Closes task 21121.
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor features (hidden services, logging):
+    - Log a message when a hidden service descriptor has fewer
+      introduction points than specified in
+      HiddenServiceNumIntroductionPoints. Closes tickets 21598.
+    - Log a message when a hidden service reaches its introduction point
+      circuit limit, and when that limit is reset. Follow up to ticket
+      21594; closes ticket 21622.
+    - Warn user if multiple entries in EntryNodes and at least one
+      HiddenService are used together. Pinning EntryNodes along with a
+      hidden service can be possibly harmful; for instance see ticket
+      14917 or 21155. Closes ticket 21155.
+
+  o Minor features (linux seccomp2 sandbox):
+    - We now have a document storage backend compatible with the Linux
+      seccomp2 sandbox. This backend is used for consensus documents and
+      diffs between them; in the long term, we'd like to use it for
+      unparseable directory material too. Closes ticket 21645
+    - Increase the maximum allowed size passed to mprotect(PROT_WRITE)
+      from 1MB to 16MB. This was necessary with the glibc allocator in
+      order to allow worker threads to allocate more memory -- which in
+      turn is necessary because of our new use of worker threads for
+      compression. Closes ticket 22096.
+
+  o Minor features (logging):
+    - Log files are no longer created world-readable by default.
+      (Previously, most distributors would store the logs in a non-
+      world-readable location to prevent inappropriate access. This
+      change is an extra precaution.) Closes ticket 21729; patch
+      from toralf.
+
+  o Minor features (performance):
+    - Our Keccak (SHA-3) implementation now accesses memory more
+      efficiently, especially on little-endian systems. Closes
+      ticket 21737.
+    - Add an O(1) implementation of channel_find_by_global_id(), to
+      speed some controller functions.
+
+  o Minor features (relay, configuration):
+    - The MyFamily option may now be repeated as many times as desired,
+      for relays that want to configure large families. Closes ticket
+      4998; patch by Daniel Pinto.
+
+  o Minor features (safety):
+    - Add an explicit check to extrainfo_parse_entry_from_string() for
+      NULL inputs. We don't believe this can actually happen, but it may
+      help silence a warning from the Clang analyzer. Closes
+      ticket 21496.
+
+  o Minor features (testing):
+    - Add a "--disable-memory-sentinels" feature to help with fuzzing.
+      When Tor is compiled with this option, we disable a number of
+      redundant memory-safety failsafes that are intended to stop bugs
+      from becoming security issues. This makes it easier to hunt for
+      bugs that would be security issues without the failsafes turned
+      on. Closes ticket 21439.
+    - Add a general event-tracing instrumentation support to Tor. This
+      subsystem will enable developers and researchers to add fine-
+      grained instrumentation to their Tor instances, for use when
+      examining Tor network performance issues. There are no trace
+      events yet, and event-tracing is off by default unless enabled at
+      compile time. Implements ticket 13802.
+    - Improve our version parsing tests: add tests for typical version
+      components, add tests for invalid versions, including numeric
+      range and non-numeric prefixes. Unit tests 21278, 21450, and
+      21507. Partially implements 21470.
+
+  o Minor bugfixes (bandwidth accounting):
+    - Roll over monthly accounting at the configured hour and minute,
+      rather than always at 00:00. Fixes bug 22245; bugfix on 0.0.9rc1.
+      Found by Andrey Karpov with PVS-Studio.
+
+  o Minor bugfixes (code correctness):
+    - Accurately identify client connections by their lack of peer
+      authentication. This means that we bail out earlier if asked to
+      extend to a client. Follow-up to 21407. Fixes bug 21406; bugfix
+      on 0.2.4.23.
+
+  o Minor bugfixes (configuration):
+    - Do not crash when starting with LearnCircuitBuildTimeout 0. Fixes
+      bug 22252; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (connection lifespan):
+    - Allow more control over how long TLS connections are kept open:
+      unify CircuitIdleTimeout and PredictedPortsRelevanceTime into a
+      single option called CircuitsAvailableTimeout. Also, allow the
+      consensus to control the default values for both this preference
+      and the lifespan of relay-to-relay connections. Fixes bug 17592;
+      bugfix on 0.2.5.5-alpha.
+    - Increase the initial circuit build timeout testing frequency, to
+      help ensure that ReducedConnectionPadding clients finish learning
+      a timeout before their orconn would expire. The initial testing
+      rate was set back in the days of TAP and before the Tor Browser
+      updater, when we had to be much more careful about new clients
+      making lots of circuits. With this change, a circuit build timeout
+      is learned in about 15-20 minutes, instead of 100-120 minutes.
+
+  o Minor bugfixes (controller):
+    - GETINFO onions/current and onions/detached no longer respond with
+      551 on empty lists. Fixes bug 21329; bugfix on 0.2.7.1-alpha.
+    - Trigger HS descriptor events on the control port when the client
+      fails to pick a hidden service directory for a hidden service.
+      This can happen if all the hidden service directories are in
+      ExcludeNodes, or they have all been queried within the last 15
+      minutes. Fixes bug 22042; bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (directory authority):
+    - When rejecting a router descriptor for running an obsolete version
+      of Tor without ntor support, warn about the obsolete tor version,
+      not the missing ntor key. Fixes bug 20270; bugfix on 0.2.9.3-alpha.
+    - Prevent the shared randomness subsystem from asserting when
+      initialized by a bridge authority with an incomplete configuration
+      file. Fixes bug 21586; bugfix on 0.2.9.8.
+
+  o Minor bugfixes (exit-side DNS):
+    - Fix an untriggerable assertion that checked the output of a
+      libevent DNS error, so that the assertion actually behaves as
+      expected. Fixes bug 22244; bugfix on 0.2.0.20-rc. Found by Andrey
+      Karpov using PVS-Studio.
+
+  o Minor bugfixes (fallback directories):
+    - Make the usage example in updateFallbackDirs.py actually work, and
+      explain what it does. Fixes bug 22270; bugfix on 0.3.0.3-alpha.
+    - Decrease the guard flag average required to be a fallback. This
+      allows us to keep relays that have their guard flag removed when
+      they restart. Fixes bug 20913; bugfix on 0.2.8.1-alpha.
+    - Decrease the minimum number of fallbacks to 100. Fixes bug 20913;
+      bugfix on 0.2.8.1-alpha.
+    - Make sure fallback directory mirrors have the same address, port,
+      and relay identity key for at least 30 days before they are
+      selected. Fixes bug 20913; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (hidden services):
+    - Stop printing a cryptic warning when a hidden service gets a
+      request to connect to a virtual port that it hasn't configured.
+      Fixes bug 16706; bugfix on 0.2.6.3-alpha.
+    - Simplify hidden service descriptor creation by using an existing
+      flag to check if an introduction point is established. Fixes bug
+      21599; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (memory leak):
+    - Fix a small memory leak at exit from the backtrace handler code.
+      Fixes bug 21788; bugfix on 0.2.5.2-alpha. Patch from Daniel Pinto.
+
+  o Minor bugfixes (protocol, logging):
+    - Downgrade a log statement about unexpected relay cells from "bug"
+      to "protocol warning", because there is at least one use case
+      where it can be triggered by a buggy tor implementation. Fixes bug
+      21293; bugfix on 0.1.1.14-alpha.
+
+  o Minor bugfixes (testing):
+    - Use unbuffered I/O for utility functions around the
+      process_handle_t type. This fixes unit test failures reported on
+      OpenBSD and FreeBSD. Fixes bug 21654; bugfix on 0.2.3.1-alpha.
+    - Make display of captured unit test log messages consistent. Fixes
+      bug 21510; bugfix on 0.2.9.3-alpha.
+    - Make test-network.sh always call chutney's test-network.sh.
+      Previously, this only worked on systems which had bash installed,
+      due to some bash-specific code in the script. Fixes bug 19699;
+      bugfix on 0.3.0.4-rc. Follow-up to ticket 21581.
+
+  o Minor bugfixes (voting consistency):
+    - Reject version numbers with non-numeric prefixes (such as +, -, or
+      whitespace). Disallowing whitespace prevents differential version
+      parsing between POSIX-based and Windows platforms. Fixes bug 21507
+      and part of 21508; bugfix on 0.0.8pre1.
+
+  o Minor bugfixes (windows, relay):
+    - Resolve "Failure from drain_fd: No error" warnings on Windows
+      relays. Fixes bug 21540; bugfix on 0.2.6.3-alpha.
+
+  o Code simplification and refactoring:
+    - Break up the 630-line function connection_dir_client_reached_eof()
+      into a dozen smaller functions. This change should help
+      maintainability and readability of the client directory code.
+    - Isolate our use of the openssl headers so that they are only
+      included from our crypto wrapper modules, and from tests that
+      examine those modules' internals. Closes ticket 21841.
+    - Simplify our API to launch directory requests, making it more
+      extensible and less error-prone. Now it's easier to add extra
+      headers to directory requests. Closes ticket 21646.
+    - Our base64 decoding functions no longer overestimate the output
+      space that they need when parsing unpadded inputs. Closes
+      ticket 17868.
+    - Remove unused "ROUTER_ADDED_NOTIFY_GENERATOR" internal value.
+      Resolves ticket 22213.
+    - The logic that directory caches use to spool request to clients,
+      serving them one part at a time so as not to allocate too much
+      memory, has been refactored for consistency. Previously there was
+      a separate spooling implementation per type of spoolable data. Now
+      there is one common spooling implementation, with extensible data
+      types. Closes ticket 21651.
+    - Tor's compression module now supports multiple backends. Part of
+      the implementation for proposal 278; closes ticket 21663.
+
+  o Documentation:
+    - Clarify the behavior of the KeepAliveIsolateSOCKSAuth sub-option.
+      Closes ticket 21873.
+    - Correct documentation about the default DataDirectory value.
+      Closes ticket 21151.
+    - Document the default behavior of NumEntryGuards and
+      NumDirectoryGuards correctly. Fixes bug 21715; bugfix
+      on 0.3.0.1-alpha.
+    - Document key=value pluggable transport arguments for Bridge lines
+      in torrc. Fixes bug 20341; bugfix on 0.2.5.1-alpha.
+    - Note that bandwidth-limiting options don't affect TCP headers or
+      DNS. Closes ticket 17170.
+
+  o Removed features (configuration options, all in ticket 22060):
+    - These configuration options are now marked Obsolete, and no longer
+      have any effect: AllowInvalidNodes, AllowSingleHopCircuits,
+      AllowSingleHopExits, ExcludeSingleHopRelays, FastFirstHopPK,
+      TLSECGroup, WarnUnsafeSocks. They were first marked as deprecated
+      in 0.2.9.2-alpha and have now been removed. The previous default
+      behavior is now always chosen; the previous (less secure) non-
+      default behavior is now unavailable.
+    - CloseHSClientCircuitsImmediatelyOnTimeout and
+      CloseHSServiceRendCircuitsImmediatelyOnTimeout were deprecated in
+      0.2.9.2-alpha and now have been removed. HS circuits never close
+      on circuit build timeout; they have a longer timeout period.
+    - {Control,DNS,Dir,Socks,Trans,NATD,OR}ListenAddress were deprecated
+      in 0.2.9.2-alpha and now have been removed. Use the ORPort option
+      (and others) to configure listen-only and advertise-only addresses.
+
+  o Removed features (tools):
+    - We've removed the tor-checkkey tool from src/tools. Long ago, we
+      used it to help people detect RSA keys that were generated by
+      versions of Debian affected by CVE-2008-0166. But those keys have
+      been out of circulation for ages, and this tool is no longer
+      required. Closes ticket 21842.
+
+
+Changes in version 0.3.0.7 - 2017-05-15
+  Tor 0.3.0.7 fixes a medium-severity security bug in earlier versions
+  of Tor 0.3.0.x, where an attacker could cause a Tor relay process
+  to exit. Relays running earlier versions of Tor 0.3.0.x should upgrade;
+  clients are not affected.
+
+  o Major bugfixes (hidden service directory, security):
+    - Fix an assertion failure in the hidden service directory code, which
+      could be used by an attacker to remotely cause a Tor relay process to
+      exit. Relays running earlier versions of Tor 0.3.0.x should upgrade.
+      should upgrade. This security issue is tracked as TROVE-2017-002.
+      Fixes bug 22246; bugfix on 0.3.0.1-alpha.
+
+  o Minor features:
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (future-proofing):
+    - Tor no longer refuses to download microdescriptors or descriptors
+      if they are listed as "published in the future". This change will
+      eventually allow us to stop listing meaningful "published" dates
+      in microdescriptor consensuses, and thereby allow us to reduce the
+      resources required to download consensus diffs by over 50%.
+      Implements part of ticket 21642; implements part of proposal 275.
+
+  o Minor bugfixes (Linux seccomp2 sandbox):
+    - The getpid() system call is now permitted under the Linux seccomp2
+      sandbox, to avoid crashing with versions of OpenSSL (and other
+      libraries) that attempt to learn the process's PID by using the
+      syscall rather than the VDSO code. Fixes bug 21943; bugfix
+      on 0.2.5.1-alpha.
+
+
+Changes in version 0.3.0.6 - 2017-04-26
+  Tor 0.3.0.6 is the first stable release of the Tor 0.3.0 series.
+
+  With the 0.3.0 series, clients and relays now use Ed25519 keys to
+  authenticate their link connections to relays, rather than the old
+  RSA1024 keys that they used before. (Circuit crypto has been
+  Curve25519-authenticated since 0.2.4.8-alpha.) We have also replaced
+  the guard selection and replacement algorithm to behave more robustly
+  in the presence of unreliable networks, and to resist guard-
+  capture attacks.
+
+  This series also includes numerous other small features and bugfixes,
+  along with more groundwork for the upcoming hidden-services revamp.
+
+  Per our stable release policy, we plan to support the Tor 0.3.0
+  release series for at least the next nine months, or for three months
+  after the first stable release of the 0.3.1 series: whichever is
+  longer. If you need a release with long-term support, we recommend
+  that you stay with the 0.2.9 series.
+
+  Below are the changes since 0.3.0.5-rc. For a list of all changes
+  since 0.2.9, see the ReleaseNotes file.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the April 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (control port):
+    - The GETINFO extra-info/digest/<digest> command was broken because
+      of a wrong base16 decode return value check, introduced when
+      refactoring that API. Fixes bug 22034; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (crash prevention):
+    - Fix a (currently untriggerable, but potentially dangerous) crash
+      bug when base32-encoding inputs whose sizes are not a multiple of
+      5. Fixes bug 21894; bugfix on 0.2.9.1-alpha.
+
+
+Changes in version 0.3.0.5-rc - 2017-04-05
+  Tor 0.3.0.5-rc fixes a few remaining bugs, large and small, in the
+  0.3.0 release series.
+
+  This is the second release candidate in the Tor 0.3.0 series, and has
+  much fewer changes than the first. If we find no new bugs or
+  regressions here, the first stable 0.3.0 release will be nearly
+  identical to it.
+
+  o Major bugfixes (crash, directory connections):
+    - Fix a rare crash when sending a begin cell on a circuit whose
+      linked directory connection had already been closed. Fixes bug
+      21576; bugfix on 0.2.9.3-alpha. Reported by Alec Muffett.
+
+  o Major bugfixes (guard selection):
+    - Fix a guard selection bug where Tor would refuse to bootstrap in
+      some cases if the user swapped a bridge for another bridge in
+      their configuration file. Fixes bug 21771; bugfix on 0.3.0.1-alpha.
+      Reported by "torvlnt33r".
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the March 7 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfix (compilation):
+    - Fix a warning when compiling hs_service.c. Previously, it had no
+      exported symbols when compiled for libor.a, resulting in a
+      compilation warning from clang. Fixes bug 21825; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (hidden services):
+    - Make hidden services check for failed intro point connections,
+      even when they have exceeded their intro point creation limit.
+      Fixes bug 21596; bugfix on 0.2.7.2-alpha. Reported by Alec Muffett.
+    - Make hidden services with 8 to 10 introduction points check for
+      failed circuits immediately after startup. Previously, they would
+      wait for 5 minutes before performing their first checks. Fixes bug
+      21594; bugfix on 0.2.3.9-alpha. Reported by Alec Muffett.
+
+  o Minor bugfixes (memory leaks):
+    - Fix a memory leak when using GETCONF on a port option. Fixes bug
+      21682; bugfix on 0.3.0.3-alpha.
+
+  o Minor bugfixes (relay):
+    - Avoid a double-marked-circuit warning that could happen when we
+      receive DESTROY cells under heavy load. Fixes bug 20059; bugfix
+      on 0.1.0.1-rc.
+
+  o Minor bugfixes (tests):
+    - Run the entry_guard_parse_from_state_full() test with the time set
+      to a specific date. (The guard state that this test was parsing
+      contained guards that had expired since the test was first
+      written.) Fixes bug 21799; bugfix on 0.3.0.1-alpha.
+
+  o Documentation:
+    - Update the description of the directory server options in the
+      manual page, to clarify that a relay no longer needs to set
+      DirPort in order to be a directory cache. Closes ticket 21720.
+
+
+
+Changes in version 0.2.8.13 - 2017-03-03
+  Tor 0.2.8.13 backports a security fix from later Tor
+  releases.  Anybody running Tor 0.2.8.12 or earlier should upgrade to this
+  this release, if for some reason they cannot upgrade to a later
+  release series, and if they build Tor with the --enable-expensive-hardening
+  option.
+
+  Note that support for Tor 0.2.8.x is ending next year: we will not issue
+  any fixes for the Tor 0.2.8.x series after 1 Jan 2018.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.7.7 - 2017-03-03
+  Tor 0.2.7.7 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.7.6 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.7.x is ending this year: we will not issue
+  any fixes for the Tor 0.2.7.x series after 1 August 2017.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, client, DNS proxy, backport from 0.2.8.3-alpha):
+    - Stop a crash that could occur when a client running with DNSPort
+      received a query with multiple address types, and the first
+      address type was not supported. Found and fixed by Scott Dial.
+      Fixes bug 18710; bugfix on 0.2.5.4-alpha.
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.6.11 - 2017-03-03
+  Tor 0.2.6.11 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.6.10 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.6.x is ending this year: we will not issue
+  any fixes for the Tor 0.2.6.x series after 1 August 2017.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major features (security fixes, backport from 0.2.9.4-alpha):
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, client, DNS proxy, backport from 0.2.8.3-alpha):
+    - Stop a crash that could occur when a client running with DNSPort
+      received a query with multiple address types, and the first
+      address type was not supported. Found and fixed by Scott Dial.
+      Fixes bug 18710; bugfix on 0.2.5.4-alpha.
+
+  o Major bugfixes (security, correctness, backport from 0.2.7.4-rc):
+    - Fix an error that could cause us to read 4 bytes before the
+      beginning of an openssl string. This bug could be used to cause
+      Tor to crash on systems with unusual malloc implementations, or
+      systems with unusual hardening installed. Fixes bug 17404; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (guard selection, backport from 0.2.7.6):
+    - Actually look at the Guard flag when selecting a new directory
+      guard. When we implemented the directory guard design, we
+      accidentally started treating all relays as if they have the Guard
+      flag during guard selection, leading to weaker anonymity and worse
+      performance. Fixes bug 17772; bugfix on 0.2.4.8-alpha. Discovered
+      by Mohsen Imani.
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.2.7.6):
+    - Fix a compilation warning with Clang 3.6: Do not check the
+      presence of an address which can never be NULL. Fixes bug 17781.
+
+
+Changes in version 0.2.5.13 - 2017-03-03
+  Tor 0.2.5.13 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.5.13 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.5.x is ending next year: we will not issue
+  any fixes for the Tor 0.2.5.x series after 1 May 2018.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major features (security fixes, backport from 0.2.9.4-alpha):
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, client, DNS proxy, backport from 0.2.8.3-alpha):
+    - Stop a crash that could occur when a client running with DNSPort
+      received a query with multiple address types, and the first
+      address type was not supported. Found and fixed by Scott Dial.
+      Fixes bug 18710; bugfix on 0.2.5.4-alpha.
+
+  o Major bugfixes (security, correctness, backport from 0.2.7.4-rc):
+    - Fix an error that could cause us to read 4 bytes before the
+      beginning of an openssl string. This bug could be used to cause
+      Tor to crash on systems with unusual malloc implementations, or
+      systems with unusual hardening installed. Fixes bug 17404; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (guard selection, backport from 0.2.7.6):
+    - Actually look at the Guard flag when selecting a new directory
+      guard. When we implemented the directory guard design, we
+      accidentally started treating all relays as if they have the Guard
+      flag during guard selection, leading to weaker anonymity and worse
+      performance. Fixes bug 17772; bugfix on 0.2.4.8-alpha. Discovered
+      by Mohsen Imani.
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.2.7.6):
+    - Fix a compilation warning with Clang 3.6: Do not check the
+      presence of an address which can never be NULL. Fixes bug 17781.
+
+  o Minor bugfixes (crypto error-handling, backport from 0.2.7.2-alpha):
+    - Check for failures from crypto_early_init, and refuse to continue.
+      A previous typo meant that we could keep going with an
+      uninitialized crypto library, and would have OpenSSL initialize
+      its own PRNG. Fixes bug 16360; bugfix on 0.2.5.2-alpha, introduced
+      when implementing ticket 4900. Patch by "teor".
+
+  o Minor bugfixes (hidden service, backport from 0.2.7.1-alpha):
+    - Fix an out-of-bounds read when parsing invalid INTRODUCE2 cells on
+      a client authorized hidden service. Fixes bug 15823; bugfix
+      on 0.2.1.6-alpha.
+
+
+Changes in version 0.2.4.28 - 2017-03-03
+  Tor 0.2.4.28 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.4.27 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.4.x is ending soon: we will not issue
+  any fixes for the Tor 0.2.4.x series after 1 August 2017.  If you need
+  a Tor release series with long-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major features (security fixes, backport from 0.2.9.4-alpha):
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, correctness, backport from 0.2.7.4-rc):
+    - Fix an error that could cause us to read 4 bytes before the
+      beginning of an openssl string. This bug could be used to cause
+      Tor to crash on systems with unusual malloc implementations, or
+      systems with unusual hardening installed. Fixes bug 17404; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (guard selection, backport from 0.2.7.6):
+    - Actually look at the Guard flag when selecting a new directory
+      guard. When we implemented the directory guard design, we
+      accidentally started treating all relays as if they have the Guard
+      flag during guard selection, leading to weaker anonymity and worse
+      performance. Fixes bug 17772; bugfix on 0.2.4.8-alpha. Discovered
+      by Mohsen Imani.
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (DoS-resistance, backport from 0.2.7.1-alpha):
+    - Make it harder for attackers to overload hidden services with
+      introductions, by blocking multiple introduction requests on the
+      same circuit. Resolves ticket 15515.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.2.7.6):
+    - Fix a compilation warning with Clang 3.6: Do not check the
+      presence of an address which can never be NULL. Fixes bug 17781.
+
+  o Minor bugfixes (hidden service, backport from 0.2.7.1-alpha):
+    - Fix an out-of-bounds read when parsing invalid INTRODUCE2 cells on
+      a client authorized hidden service. Fixes bug 15823; bugfix
+      on 0.2.1.6-alpha.
+
+
+Changes in version 0.3.0.4-rc - 2017-03-01
+  Tor 0.3.0.4-rc fixes some remaining bugs, large and small, in the
+  0.3.0 release series, and introduces a few reliability features to
+  keep them from coming back.
+
+  This is the first release candidate in the Tor 0.3.0 series. If we
+  find no new bugs or regressions here, the first stable 0.3.0 release
+  will be nearly identical to it.
+
+  o Major bugfixes (bridges):
+    - When the same bridge is configured multiple times with the same
+      identity, but at different address:port combinations, treat those
+      bridge instances as separate guards. This fix restores the ability
+      of clients to configure the same bridge with multiple pluggable
+      transports. Fixes bug 21027; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (hidden service directory v3):
+    - Stop crashing on a failed v3 hidden service descriptor lookup
+      failure. Fixes bug 21471; bugfixes on 0.3.0.1-alpha.
+
+  o Major bugfixes (parsing):
+    - When parsing a malformed content-length field from an HTTP
+      message, do not read off the end of the buffer. This bug was a
+      potential remote denial-of-service attack against Tor clients and
+      relays. A workaround was released in October 2016, to prevent this
+      bug from crashing Tor. This is a fix for the underlying issue,
+      which should no longer matter (if you applied the earlier patch).
+      Fixes bug 20894; bugfix on 0.2.0.16-alpha. Bug found by fuzzing
+      using AFL (http://lcamtuf.coredump.cx/afl/).
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor feature (protocol versioning):
+    - Add new protocol version for proposal 224. HSIntro now advertises
+      version "3-4" and HSDir version "1-2". Fixes ticket 20656.
+
+  o Minor features (directory authorities):
+    - Directory authorities now reject descriptors that claim to be
+      malformed versions of Tor. Helps prevent exploitation of
+      bug 21278.
+    - Reject version numbers with components that exceed INT32_MAX.
+      Otherwise 32-bit and 64-bit platforms would behave inconsistently.
+      Fixes bug 21450; bugfix on 0.0.8pre1.
+    - Directory authorities now reject relays running versions
+      0.2.9.1-alpha through 0.2.9.4-alpha, because those relays
+      suffer from bug 20499 and don't keep their consensus cache
+      up-to-date. Resolves ticket 20509.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (reliability, crash):
+    - Try better to detect problems in buffers where they might grow (or
+      think they have grown) over 2 GB in size. Diagnostic for
+      bug 21369.
+
+  o Minor features (testing):
+    - During 'make test-network-all', if tor logs any warnings, ask
+      chutney to output them. Requires a recent version of chutney with
+      the 21572 patch. Implements 21570.
+
+  o Minor bugfixes (certificate expiration time):
+    - Avoid using link certificates that don't become valid till some
+      time in the future. Fixes bug 21420; bugfix on 0.2.4.11-alpha
+
+  o Minor bugfixes (code correctness):
+    - Repair a couple of (unreachable or harmless) cases of the risky
+      comparison-by-subtraction pattern that caused bug 21278.
+    - Remove a redundant check for the UseEntryGuards option from the
+      options_transition_affects_guards() function. Fixes bug 21492;
+      bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (directory mirrors):
+    - Allow relays to use directory mirrors without a DirPort: these
+      relays need to be contacted over their ORPorts using a begindir
+      connection. Fixes one case of bug 20711; bugfix on 0.2.8.2-alpha.
+    - Clarify the message logged when a remote relay is unexpectedly
+      missing an ORPort or DirPort: users were confusing this with a
+      local port. Fixes another case of bug 20711; bugfix
+      on 0.2.8.2-alpha.
+
+  o Minor bugfixes (guards):
+    - Don't warn about a missing guard state on timeout-measurement
+      circuits: they aren't supposed to be using guards. Fixes an
+      instance of bug 21007; bugfix on 0.3.0.1-alpha.
+    - Silence a BUG() warning when attempting to use a guard whose
+      descriptor we don't know, and make this scenario less likely to
+      happen. Fixes bug 21415; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (hidden service):
+    - Pass correct buffer length when encoding legacy ESTABLISH_INTRO
+      cells. Previously, we were using sizeof() on a pointer, instead of
+      the real destination buffer. Fortunately, that value was only used
+      to double-check that there was enough room--which was already
+      enforced elsewhere. Fixes bug 21553; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Fix Raspbian build issues related to missing socket errno in
+      test_util.c. Fixes bug 21116; bugfix on 0.2.8.2. Patch
+      by "hein".
+    - Rename "make fuzz" to "make test-fuzz-corpora", since it doesn't
+      actually fuzz anything. Fixes bug 21447; bugfix on 0.3.0.3-alpha.
+    - Use bash in src/test/test-network.sh. This ensures we reliably
+      call chutney's newer tools/test-network.sh when available. Fixes
+      bug 21562; bugfix on 0.2.9.1-alpha.
+
+  o Documentation:
+    - Small fixes to the fuzzing documentation. Closes ticket 21472.
+
+
+Changes in version 0.2.9.10 - 2017-03-01
+  Tor 0.2.9.10 backports a security fix from later Tor release.  It also
+  includes fixes for some major issues affecting directory authorities,
+  LibreSSL compatibility, and IPv6 correctness.
+
+  The Tor 0.2.9.x release series is now marked as a long-term-support
+  series.  We intend to backport security fixes to 0.2.9.x until at
+  least January of 2020.
+
+  o Major bugfixes (directory authority, 0.3.0.3-alpha):
+    - During voting, when marking a relay as a probable sybil, do not
+      clear its BadExit flag: sybils can still be bad in other ways
+      too. (We still clear the other flags.) Fixes bug 21108; bugfix
+      on 0.2.0.13-alpha.
+
+  o Major bugfixes (IPv6 Exits, backport from 0.3.0.3-alpha):
+    - Stop rejecting all IPv6 traffic on Exits whose exit policy rejects
+      any IPv6 addresses. Instead, only reject a port over IPv6 if the
+      exit policy rejects that port on more than an IPv6 /16 of
+      addresses. This bug was made worse by 17027 in 0.2.8.1-alpha,
+      which rejected a relay's own IPv6 address by default. Fixes bug
+      21357; bugfix on commit 004f3f4e53 in 0.2.4.7-alpha.
+
+  o Major bugfixes (parsing, also in 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (directory authorities, also in 0.3.0.4-rc):
+    - Directory authorities now reject descriptors that claim to be
+      malformed versions of Tor. Helps prevent exploitation of
+      bug 21278.
+    - Reject version numbers with components that exceed INT32_MAX.
+      Otherwise 32-bit and 64-bit platforms would behave inconsistently.
+      Fixes bug 21450; bugfix on 0.0.8pre1.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (portability, compilation, backport from 0.3.0.3-alpha):
+    - Autoconf now checks to determine if OpenSSL structures are opaque,
+      instead of explicitly checking for OpenSSL version numbers. Part
+      of ticket 21359.
+    - Support building with recent LibreSSL code that uses opaque
+      structures. Closes ticket 21359.
+
+  o Minor bugfixes (code correctness, also in 0.3.0.4-rc):
+    - Repair a couple of (unreachable or harmless) cases of the risky
+      comparison-by-subtraction pattern that caused bug 21278.
+
+  o Minor bugfixes (tor-resolve, backport from 0.3.0.3-alpha):
+    - The tor-resolve command line tool now rejects hostnames over 255
+      characters in length. Previously, it would silently truncate them,
+      which could lead to bugs. Fixes bug 21280; bugfix on 0.0.9pre5.
+      Patch by "junglefowl".
+
+
+Changes in version 0.3.0.3-alpha - 2017-02-03
+  Tor 0.3.0.3-alpha fixes a few significant bugs introduced over the
+  0.3.0.x development series, including some that could cause
+  authorities to behave badly. There is also a fix for a longstanding
+  bug that could prevent IPv6 exits from working. Tor 0.3.0.3-alpha also
+  includes some smaller features and bugfixes.
+
+  The Tor 0.3.0.x release series is now in patch-freeze: no additional
+  features will be considered for inclusion in 0.3.0.x. We suspect that
+  some bugs will probably remain, however, and we encourage people to
+  test this release.
+
+  o Major bugfixes (directory authority):
+    - During voting, when marking a relay as a probable sybil, do not
+      clear its BadExit flag: sybils can still be bad in other ways
+      too. (We still clear the other flags.) Fixes bug 21108; bugfix
+      on 0.2.0.13-alpha.
+    - When deciding whether we have just found a router to be reachable,
+      do not penalize it for not having performed an Ed25519 link
+      handshake if it does not claim to support an Ed25519 handshake.
+      Previously, we would treat such relays as non-running. Fixes bug
+      21107; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (entry guards):
+    - Stop trying to build circuits through entry guards for which we
+      have no descriptor. Also, stop crashing in the case that we *do*
+      accidentally try to build a circuit in such a state. Fixes bug
+      21242; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (IPv6 Exits):
+    - Stop rejecting all IPv6 traffic on Exits whose exit policy rejects
+      any IPv6 addresses. Instead, only reject a port over IPv6 if the
+      exit policy rejects that port on more than an IPv6 /16 of
+      addresses. This bug was made worse by 17027 in 0.2.8.1-alpha,
+      which rejected a relay's own IPv6 address by default. Fixes bug
+      21357; bugfix on commit 004f3f4e53 in 0.2.4.7-alpha.
+
+  o Minor feature (client):
+    - Enable IPv6 traffic on the SocksPort by default. To disable this,
+      a user will have to specify "NoIPv6Traffic". Closes ticket 21269.
+
+  o Minor feature (fallback scripts):
+    - Add a check_existing mode to updateFallbackDirs.py, which checks
+      if fallbacks in the hard-coded list are working. Closes ticket
+      20174. Patch by haxxpop.
+
+  o Minor features (ciphersuite selection):
+    - Clients now advertise a list of ciphersuites closer to the ones
+      preferred by Firefox. Closes part of ticket 15426.
+    - Allow relays to accept a wider range of ciphersuites, including
+      chacha20-poly1305 and AES-CCM. Closes the other part of 15426.
+
+  o Minor features (controller, configuration):
+    - Each of the *Port options, such as SocksPort, ORPort, ControlPort,
+      and so on, now comes with a __*Port variant that will not be saved
+      to the torrc file by the controller's SAVECONF command. This
+      change allows TorBrowser to set up a single-use domain socket for
+      each time it launches Tor. Closes ticket 20956.
+    - The GETCONF command can now query options that may only be
+      meaningful in context-sensitive lists. This allows the controller
+      to query the mixed SocksPort/__SocksPort style options introduced
+      in feature 20956. Implements ticket 21300.
+
+  o Minor features (portability, compilation):
+    - Autoconf now checks to determine if OpenSSL structures are opaque,
+      instead of explicitly checking for OpenSSL version numbers. Part
+      of ticket 21359.
+    - Support building with recent LibreSSL code that uses opaque
+      structures. Closes ticket 21359.
+
+  o Minor features (relay):
+    - We now allow separation of exit and relay traffic to different
+      source IP addresses, using the OutboundBindAddressExit and
+      OutboundBindAddressOR options respectively. Closes ticket 17975.
+      Written by Michael Sonntag.
+
+  o Minor bugfix (logging):
+    - Don't recommend the use of Tor2web in non-anonymous mode.
+      Recommending Tor2web is a bad idea because the client loses all
+      anonymity. Tor2web should only be used in specific cases by users
+      who *know* and understand the issues. Fixes bug 21294; bugfix
+      on 0.2.9.3-alpha.
+
+  o Minor bugfixes (client):
+    - Always recover from failures in extend_info_from_node(), in an
+      attempt to prevent any recurrence of bug 21242. Fixes bug 21372;
+      bugfix on 0.2.3.1-alpha.
+
+  o Minor bugfixes (client, entry guards):
+    - Fix a bug warning (with backtrace) when we fail a channel that
+      circuits to fallback directories on it. Fixes bug 21128; bugfix
+      on 0.3.0.1-alpha.
+    - Fix a spurious bug warning (with backtrace) when removing an
+      expired entry guard. Fixes bug 21129; bugfix on 0.3.0.1-alpha.
+    - Fix a bug of the new guard algorithm where tor could stall for up
+      to 10 minutes before retrying a guard after a long period of no
+      network. Fixes bug 21052; bugfix on 0.3.0.1-alpha.
+    - Do not try to build circuits until we have descriptors for our
+      primary entry guards. Related to fix for bug 21242.
+
+  o Minor bugfixes (configure, autoconf):
+    - Rename the configure option --enable-expensive-hardening to
+      --enable-fragile-hardening. Expensive hardening makes the tor
+      daemon abort when some kinds of issues are detected. Thus, it
+      makes tor more at risk of remote crashes but safer against RCE or
+      heartbleed bug category. We now try to explain this issue in a
+      message from the configure script. Fixes bug 21290; bugfix
+      on 0.2.5.4-alpha.
+
+  o Minor bugfixes (controller):
+    - Restore the (deprecated) DROPGUARDS controller command. Fixes bug
+      20824; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (hidden service):
+    - Clean up the code for expiring intro points with no associated
+      circuits. It was causing, rarely, a service with some expiring
+      introduction points to not open enough additional introduction
+      points. Fixes part of bug 21302; bugfix on 0.2.7.2-alpha.
+    - Stop setting the torrc option HiddenServiceStatistics to "0" just
+      because we're not a bridge or relay. Instead, we preserve whatever
+      value the user set (or didn't set). Fixes bug 21150; bugfix
+      on 0.2.6.2-alpha.
+    - Resolve two possible underflows which could lead to creating and
+      closing a lot of introduction point circuits in a non-stop loop.
+      Fixes bug 21302; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (portability):
+    - Use "OpenBSD" compiler macro instead of "OPENBSD" or "__OpenBSD__".
+      It is supported by OpenBSD itself, and also by most OpenBSD
+      variants (such as Bitrig). Fixes bug 20980; bugfix
+      on 0.1.2.1-alpha.
+    - When mapping a file of length greater than SIZE_MAX, do not
+      silently truncate its contents. This issue could occur on 32 bit
+      systems with large file support and files which are larger than 4
+      GB. Fixes bug 21134; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (tor-resolve):
+    - The tor-resolve command line tool now rejects hostnames over 255
+      characters in length. Previously, it would silently truncate them,
+      which could lead to bugs. Fixes bug 21280; bugfix on 0.0.9pre5.
+      Patch by "junglefowl".
+
+  o Minor bugfixes (Windows services):
+    - Be sure to initialize the monotonic time subsystem before using
+      it, even when running as an NT service. Fixes bug 21356; bugfix
+      on 0.2.9.1-alpha.
+
+
+Changes in version 0.3.0.2-alpha - 2017-01-23
+  Tor 0.3.0.2-alpha fixes a denial-of-service bug where an attacker could
+  cause relays and clients to crash, even if they were not built with
+  the --enable-expensive-hardening option. This bug affects all 0.2.9.x
+  versions, and also affects 0.3.0.1-alpha: all relays running an affected
+  version should upgrade.
+
+  Tor 0.3.0.2-alpha also improves how exit relays and clients handle DNS
+  time-to-live values, makes directory authorities enforce the 1-to-1
+  mapping of relay RSA identity keys to ED25519 identity keys, fixes a
+  client-side onion service reachability bug, does better at selecting
+  the set of fallback directories, and more.
+
+  o Major bugfixes (security, also in 0.2.9.9):
+    - Downgrade the "-ftrapv" option from "always on" to "only on when
+      --enable-expensive-hardening is provided."  This hardening option, like
+      others, can turn survivable bugs into crashes--and having it on by
+      default made a (relatively harmless) integer overflow bug into a
+      denial-of-service bug. Fixes bug 21278 (TROVE-2017-001); bugfix on
+      0.2.9.1-alpha.
+
+  o Major features (security):
+    - Change the algorithm used to decide DNS TTLs on client and server
+      side, to better resist DNS-based correlation attacks like the
+      DefecTor attack of Greschbach, Pulls, Roberts, Winter, and
+      Feamster. Now relays only return one of two possible DNS TTL
+      values, and clients are willing to believe DNS TTL values up to 3
+      hours long. Closes ticket 19769.
+
+  o Major features (directory authority, security):
+    - The default for AuthDirPinKeys is now 1: directory authorities
+      will reject relays where the RSA identity key matches a previously
+      seen value, but the Ed25519 key has changed. Closes ticket 18319.
+
+  o Major bugfixes (client, guard, crash):
+    - In circuit_get_global_origin_list(), return the actual list of
+      origin circuits. The previous version of this code returned the
+      list of all the circuits, and could have caused strange bugs,
+      including possible crashes. Fixes bug 21118; bugfix
+      on 0.3.0.1-alpha.
+
+  o Major bugfixes (client, onion service, also in 0.2.9.9):
+    - Fix a client-side onion service reachability bug, where multiple
+      socks requests to an onion service (or a single slow request)
+      could cause us to mistakenly mark some of the service's
+      introduction points as failed, and we cache that failure so
+      eventually we run out and can't reach the service. Also resolves a
+      mysterious "Remote server sent bogus reason code 65021" log
+      warning. The bug was introduced in ticket 17218, where we tried to
+      remember the circuit end reason as a uint16_t, which mangled
+      negative values. Partially fixes bug 21056 and fixes bug 20307;
+      bugfix on 0.2.8.1-alpha.
+
+  o Major bugfixes (DNS):
+    - Fix a bug that prevented exit nodes from caching DNS records for
+      more than 60 seconds. Fixes bug 19025; bugfix on 0.2.4.7-alpha.
+
+  o Minor features (controller):
+    - Add "GETINFO sr/current" and "GETINFO sr/previous" keys, to expose
+      shared-random values to the controller. Closes ticket 19925.
+
+  o Minor features (entry guards):
+    - Add UseEntryGuards to TEST_OPTIONS_DEFAULT_VALUES in order to not
+      break regression tests.
+    - Require UseEntryGuards when UseBridges is set, in order to make
+      sure bridges aren't bypassed. Resolves ticket 20502.
+
+  o Minor features (fallback directories):
+    - Select 200 fallback directories for each release. Closes
+      ticket 20881.
+    - Allow 3 fallback relays per operator, which is safe now that we
+      are choosing 200 fallback relays. Closes ticket 20912.
+    - Exclude relays affected by bug 20499 from the fallback list.
+      Exclude relays from the fallback list if they are running versions
+      known to be affected by bug 20499, or if in our tests they deliver
+      a stale consensus (i.e. one that expired more than 24 hours ago).
+      Closes ticket 20539.
+    - Reduce the minimum fallback bandwidth to 1 MByte/s. Part of
+      ticket 18828.
+    - Require fallback directories to have the same address and port for
+      7 days (now that we have enough relays with this stability).
+      Relays whose OnionOO stability timer is reset on restart by bug
+      18050 should upgrade to Tor 0.2.8.7 or later, which has a fix for
+      this issue. Closes ticket 20880; maintains short-term fix
+      in 0.2.8.2-alpha.
+    - Require fallbacks to have flags for 90% of the time (weighted
+      decaying average), rather than 95%. This allows at least 73% of
+      clients to bootstrap in the first 5 seconds without contacting an
+      authority. Part of ticket 18828.
+    - Annotate updateFallbackDirs.py with the bandwidth and consensus
+      weight for each candidate fallback. Closes ticket 20878.
+    - Make it easier to change the output sort order of fallbacks.
+      Closes ticket 20822.
+    - Display the relay fingerprint when downloading consensuses from
+      fallbacks. Closes ticket 20908.
+
+  o Minor features (geoip, also in 0.2.9.9):
+    - Update geoip and geoip6 to the January 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (next-gen onion service directories):
+    - Remove the "EnableOnionServicesV3" consensus parameter that we
+      introduced in 0.3.0.1-alpha: relays are now always willing to act
+      as v3 onion service directories. Resolves ticket 19899.
+
+  o Minor features (linting):
+    - Enhance the changes file linter to warn on Tor versions that are
+      prefixed with "tor-". Closes ticket 21096.
+
+  o Minor features (logging):
+    - In several places, describe unset ed25519 keys as "<unset>",
+      rather than the scary "AAAAAAAA...AAA". Closes ticket 21037.
+
+  o Minor bugfix (control protocol):
+    - The reply to a "GETINFO config/names" request via the control
+      protocol now spells the type "Dependent" correctly. This is a
+      breaking change in the control protocol. (The field seems to be
+      ignored by the most common known controllers.) Fixes bug 18146;
+      bugfix on 0.1.1.4-alpha.
+
+  o Minor bugfixes (bug resilience):
+    - Fix an unreachable size_t overflow in base64_decode(). Fixes bug
+      19222; bugfix on 0.2.0.9-alpha. Found by Guido Vranken; fixed by
+      Hans Jerry Illikainen.
+
+  o Minor bugfixes (build):
+    - Replace obsolete Autoconf macros with their modern equivalent and
+      prevent similar issues in the future. Fixes bug 20990; bugfix
+      on 0.1.0.1-rc.
+
+  o Minor bugfixes (client, guards):
+    - Fix bug where Tor would think that there are circuits waiting for
+      better guards even though those circuits have been freed. Fixes
+      bug 21142; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (config):
+    - Don't assert on startup when trying to get the options list and
+      LearnCircuitBuildTimeout is set to 0: we are currently parsing the
+      options so of course they aren't ready yet. Fixes bug 21062;
+      bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (controller):
+    - Make the GETINFO interface for inquiring about entry guards
+      support the new guards backend. Fixes bug 20823; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (dead code):
+    - Remove a redundant check for PidFile changes at runtime in
+      options_transition_allowed(): this check is already performed
+      regardless of whether the sandbox is active. Fixes bug 21123;
+      bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (documentation):
+    - Update the tor manual page to document every option that can not
+      be changed while tor is running. Fixes bug 21122.
+
+  o Minor bugfixes (fallback directories):
+    - Stop failing when a relay has no uptime data in
+      updateFallbackDirs.py. Fixes bug 20945; bugfix on 0.2.8.1-alpha.
+    - Avoid checking fallback candidates' DirPorts if they are down in
+      OnionOO. When a relay operator has multiple relays, this
+      prioritizes relays that are up over relays that are down. Fixes
+      bug 20926; bugfix on 0.2.8.3-alpha.
+    - Stop failing when OUTPUT_COMMENTS is True in updateFallbackDirs.py.
+      Fixes bug 20877; bugfix on 0.2.8.3-alpha.
+
+  o Minor bugfixes (guards, bootstrapping):
+    - When connecting to a directory guard during bootstrap, do not mark
+      the guard as successful until we receive a good-looking directory
+      response from it. Fixes bug 20974; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (onion services):
+    - Fix the config reload pruning of old vs new services so it
+      actually works when both ephemeral and non-ephemeral services are
+      configured. Fixes bug 21054; bugfix on 0.3.0.1-alpha.
+    - Allow the number of introduction points to be as low as 0, rather
+      than as low as 3. Fixes bug 21033; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (IPv6):
+    - Make IPv6-using clients try harder to find an IPv6 directory
+      server. Fixes bug 20999; bugfix on 0.2.8.2-alpha.
+    - When IPv6 addresses have not been downloaded yet (microdesc
+      consensus documents don't list relay IPv6 addresses), use hard-
+      coded addresses for authorities, fallbacks, and configured
+      bridges. Now IPv6-only clients can use microdescriptors. Fixes bug
+      20996; bugfix on b167e82 from 19608 in 0.2.8.5-alpha.
+
+  o Minor bugfixes (memory leaks):
+    - Fix a memory leak when configuring hidden services. Fixes bug
+      20987; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (portability, also in 0.2.9.9):
+    - Avoid crashing when Tor is built using headers that contain
+      CLOCK_MONOTONIC_COARSE, but then tries to run on an older kernel
+      without CLOCK_MONOTONIC_COARSE. Fixes bug 21035; bugfix
+      on 0.2.9.1-alpha.
+    - Fix Libevent detection on platforms without Libevent 1 headers
+      installed. Fixes bug 21051; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (relay):
+    - Honor DataDirectoryGroupReadable when tor is a relay. Previously,
+      initializing the keys would reset the DataDirectory to 0700
+      instead of 0750 even if DataDirectoryGroupReadable was set to 1.
+      Fixes bug 19953; bugfix on 0.0.2pre16. Patch by "redfish".
+
+  o Minor bugfixes (testing):
+    - Remove undefined behavior from the backtrace generator by removing
+      its signal handler. Fixes bug 21026; bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (unit tests):
+    - Allow the unit tests to pass even when DNS lookups of bogus
+      addresses do not fail as expected. Fixes bug 20862 and 20863;
+      bugfix on unit tests introduced in 0.2.8.1-alpha
+      through 0.2.9.4-alpha.
+
+  o Code simplification and refactoring:
+    - Refactor code to manipulate global_origin_circuit_list into
+      separate functions. Closes ticket 20921.
+
+  o Documentation (formatting):
+    - Clean up formatting of tor.1 man page and HTML doc, where <pre>
+      blocks were incorrectly appearing. Closes ticket 20885.
+
+  o Documentation (man page):
+    - Clarify many options in tor.1 and add some min/max values for
+      HiddenService options. Closes ticket 21058.
+
+
+Changes in version 0.2.9.9 - 2017-01-23
+  Tor 0.2.9.9 fixes a denial-of-service bug where an attacker could
+  cause relays and clients to crash, even if they were not built with
+  the --enable-expensive-hardening option. This bug affects all 0.2.9.x
+  versions, and also affects 0.3.0.1-alpha: all relays running an affected
+  version should upgrade.
+
+  This release also resolves a client-side onion service reachability
+  bug, and resolves a pair of small portability issues.
+
+  o Major bugfixes (security):
+    - Downgrade the "-ftrapv" option from "always on" to "only on when
+      --enable-expensive-hardening is provided." This hardening option,
+      like others, can turn survivable bugs into crashes -- and having
+      it on by default made a (relatively harmless) integer overflow bug
+      into a denial-of-service bug. Fixes bug 21278 (TROVE-2017-001);
+      bugfix on 0.2.9.1-alpha.
+
+  o Major bugfixes (client, onion service):
+    - Fix a client-side onion service reachability bug, where multiple
+      socks requests to an onion service (or a single slow request)
+      could cause us to mistakenly mark some of the service's
+      introduction points as failed, and we cache that failure so
+      eventually we run out and can't reach the service. Also resolves a
+      mysterious "Remote server sent bogus reason code 65021" log
+      warning. The bug was introduced in ticket 17218, where we tried to
+      remember the circuit end reason as a uint16_t, which mangled
+      negative values. Partially fixes bug 21056 and fixes bug 20307;
+      bugfix on 0.2.8.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the January 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (portability):
+    - Avoid crashing when Tor is built using headers that contain
+      CLOCK_MONOTONIC_COARSE, but then tries to run on an older kernel
+      without CLOCK_MONOTONIC_COARSE. Fixes bug 21035; bugfix
+      on 0.2.9.1-alpha.
+    - Fix Libevent detection on platforms without Libevent 1 headers
+      installed. Fixes bug 21051; bugfix on 0.2.9.1-alpha.
+
+
+Changes in version 0.3.0.1-alpha - 2016-12-19
+  Tor 0.3.0.1-alpha is the first alpha release in the 0.3.0 development
+  series. It strengthens Tor's link and circuit handshakes by
+  identifying relays by their Ed25519 keys, improves the algorithm that
+  clients use to choose and maintain their list of guards, and includes
+  additional backend support for the next-generation hidden service
+  design. It also contains numerous other small features and
+  improvements to security, correctness, and performance.
+
+  Below are the changes since 0.2.9.8.
+
+  o Major features (guard selection algorithm):
+    - Tor's guard selection algorithm has been redesigned from the
+      ground up, to better support unreliable networks and restrictive
+      sets of entry nodes, and to better resist guard-capture attacks by
+      hostile local networks. Implements proposal 271; closes
+      ticket 19877.
+
+  o Major features (next-generation hidden services):
+    - Relays can now handle v3 ESTABLISH_INTRO cells as specified by
+      prop224 aka "Next Generation Hidden Services". Service and clients
+      don't use this functionality yet. Closes ticket 19043. Based on
+      initial code by Alec Heifetz.
+    - Relays now support the HSDir version 3 protocol, so that they can
+      can store and serve v3 descriptors. This is part of the next-
+      generation onion service work detailed in proposal 224. Closes
+      ticket 17238.
+
+  o Major features (protocol, ed25519 identity keys):
+    - Relays now use Ed25519 to prove their Ed25519 identities and to
+      one another, and to clients. This algorithm is faster and more
+      secure than the RSA-based handshake we've been doing until now.
+      Implements the second big part of proposal 220; Closes
+      ticket 15055.
+    - Clients now support including Ed25519 identity keys in the EXTEND2
+      cells they generate. By default, this is controlled by a consensus
+      parameter, currently disabled. You can turn this feature on for
+      testing by setting ExtendByEd25519ID in your configuration. This
+      might make your traffic appear different than the traffic
+      generated by other users, however. Implements part of ticket
+      15056; part of proposal 220.
+    - Relays now understand requests to extend to other relays by their
+      Ed25519 identity keys. When an Ed25519 identity key is included in
+      an EXTEND2 cell, the relay will only extend the circuit if the
+      other relay can prove ownership of that identity. Implements part
+      of ticket 15056; part of proposal 220.
+
+  o Major bugfixes (scheduler):
+    - Actually compare circuit policies in ewma_cmp_cmux(). This bug
+      caused the channel scheduler to behave more or less randomly,
+      rather than preferring channels with higher-priority circuits.
+      Fixes bug 20459; bugfix on 0.2.6.2-alpha.
+
+  o Minor features (controller):
+    - When HSFETCH arguments cannot be parsed, say "Invalid argument"
+      rather than "unrecognized." Closes ticket 20389; patch from
+      Ivan Markin.
+
+  o Minor features (diagnostic, directory client):
+    - Warn when we find an unexpected inconsistency in directory
+      download status objects. Prevents some negative consequences of
+      bug 20593.
+
+  o Minor features (directory authority):
+    - Add a new authority-only AuthDirTestEd25519LinkKeys option (on by
+      default) to control whether authorities should try to probe relays
+      by their Ed25519 link keys. This option will go away in a few
+      releases--unless we encounter major trouble in our ed25519 link
+      protocol rollout, in which case it will serve as a safety option.
+
+  o Minor features (directory cache):
+    - Relays and bridges will now refuse to serve the consensus they
+      have if they know it is too old for a client to use. Closes
+      ticket 20511.
+
+  o Minor features (ed25519 link handshake):
+    - Advertise support for the ed25519 link handshake using the
+      subprotocol-versions mechanism, so that clients can tell which
+      relays can identity themselves by Ed25519 ID. Closes ticket 20552.
+
+  o Minor features (fingerprinting resistance, authentication):
+    - Extend the length of RSA keys used for TLS link authentication to
+      2048 bits. (These weren't used for forward secrecy; for forward
+      secrecy, we used P256.) Closes ticket 13752.
+
+  o Minor features (infrastructure):
+    - Implement smartlist_add_strdup() function. Replaces the use of
+      smartlist_add(sl, tor_strdup(str)). Closes ticket 20048.
+
+  o Minor bugfixes (client):
+    - When clients that use bridges start up with a cached consensus on
+      disk, they were ignoring it and downloading a new one. Now they
+      use the cached one. Fixes bug 20269; bugfix on 0.2.3.12-alpha.
+
+  o Minor bugfixes (configuration):
+    - Accept non-space whitespace characters after the severity level in
+      the `Log` option. Fixes bug 19965; bugfix on 0.2.1.1-alpha.
+    - Support "TByte" and "TBytes" units in options given in bytes.
+      "TB", "terabyte(s)", "TBit(s)" and "terabit(s)" were already
+      supported. Fixes bug 20622; bugfix on 0.2.0.14-alpha.
+
+  o Minor bugfixes (consensus weight):
+    - Add new consensus method that initializes bw weights to 1 instead
+      of 0. This prevents a zero weight from making it all the way to
+      the end (happens in small testing networks) and causing an error.
+      Fixes bug 14881; bugfix on 0.2.2.17-alpha.
+
+  o Minor bugfixes (descriptors):
+    - Correctly recognise downloaded full descriptors as valid, even
+      when using microdescriptors as circuits. This affects clients with
+      FetchUselessDescriptors set, and may affect directory authorities.
+      Fixes bug 20839; bugfix on 0.2.3.2-alpha.
+
+  o Minor bugfixes (directory system):
+    - Download all consensus flavors, descriptors, and authority
+      certificates when FetchUselessDescriptors is set, regardless of
+      whether tor is a directory cache or not. Fixes bug 20667; bugfix
+      on all recent tor versions.
+    - Bridges and relays now use microdescriptors (like clients do)
+      rather than old-style router descriptors. Now bridges will blend
+      in with clients in terms of the circuits they build. Fixes bug
+      6769; bugfix on 0.2.3.2-alpha.
+
+  o Minor bugfixes (ed25519 certificates):
+    - Correctly interpret ed25519 certificates that would expire some
+      time after 19 Jan 2038. Fixes bug 20027; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (hidden services):
+    - Stop ignoring misconfigured hidden services. Instead, refuse to
+      start tor until the misconfigurations have been corrected. Fixes
+      bug 20559; bugfix on multiple commits in 0.2.7.1-alpha
+      and earlier.
+
+  o Minor bugfixes (memory leak at exit):
+    - Fix a small harmless memory leak at exit of the previously unused
+      RSA->Ed identity cross-certificate. Fixes bug 17779; bugfix
+      on 0.2.7.2-alpha.
+
+  o Minor bugfixes (util):
+    - When finishing writing a file to disk, if we were about to replace
+      the file with the temporary file created before and we fail to
+      replace it, remove the temporary file so it doesn't stay on disk.
+      Fixes bug 20646; bugfix on 0.2.0.7-alpha. Patch by fk.
+
+  o Minor bugfixes (Windows):
+    - Check for getpagesize before using it to mmap files. This fixes
+      compilation in some MinGW environments. Fixes bug 20530; bugfix on
+      0.1.2.1-alpha. Reported by "ice".
+
+  o Code simplification and refactoring:
+    - Abolish all global guard context in entrynodes.c; replace with new
+      guard_selection_t structure as preparation for proposal 271.
+      Closes ticket 19858.
+    - Introduce rend_service_is_ephemeral() that tells if given onion
+      service is ephemeral. Replace unclear NULL-checkings for service
+      directory with this function. Closes ticket 20526.
+    - Extract magic numbers in circuituse.c into defined variables.
+    - Refactor circuit_is_available_for_use to remove unnecessary check.
+    - Refactor circuit_predict_and_launch_new for readability and
+      testability. Closes ticket 18873.
+    - Refactor large if statement in purpose_needs_anonymity to use
+      switch statement instead. Closes part of ticket 20077.
+    - Refactor the hashing API to return negative values for errors, as
+      is done as throughout the codebase. Closes ticket 20717.
+    - Remove data structures that were used to index or_connection
+      objects by their RSA identity digests. These structures are fully
+      redundant with the similar structures used in the
+      channel abstraction.
+    - Remove duplicate code in the channel_write_*cell() functions.
+      Closes ticket 13827; patch from Pingl.
+    - Remove redundant behavior of is_sensitive_dir_purpose, refactor to
+      use only purpose_needs_anonymity. Closes part of ticket 20077.
+    - The code to generate and parse EXTEND and EXTEND2 cells has been
+      replaced with code automatically generated by the
+      "trunnel" utility.
+
+  o Documentation:
+    - Include the "TBits" unit in Tor's man page. Fixes part of bug
+      20622; bugfix on 0.2.5.1-alpha.
+    - Change '1' to 'weight_scale' in consensus bw weights calculation
+      comments, as that is reality. Closes ticket 20273. Patch
+      from pastly.
+    - Correct the value for AuthDirGuardBWGuarantee in the manpage, from
+      250 KBytes to 2 MBytes. Fixes bug 20435; bugfix
+      on 0.2.5.6-alpha.
+    - Stop the man page from incorrectly stating that HiddenServiceDir
+      must already exist. Fixes 20486.
+    - Clarify that when ClientRejectInternalAddresses is enabled (which
+      is the default), multicast DNS hostnames for machines on the local
+      network (of the form *.local) are also rejected. Closes
+      ticket 17070.
+
+  o Removed features:
+    - The AuthDirMaxServersPerAuthAddr option no longer exists: The same
+      limit for relays running on a single IP applies to authority IP
+      addresses as well as to non-authority IP addresses. Closes
+      ticket 20960.
+    - The UseDirectoryGuards torrc option no longer exists: all users
+      that use entry guards will also use directory guards. Related to
+      proposal 271; implements part of ticket 20831.
+
+  o Testing:
+    - New unit tests for tor_htonll(). Closes ticket 19563. Patch
+      from "overcaffeinated".
+    - Perform the coding style checks when running the tests and fail
+      when coding style violations are found. Closes ticket 5500.
+    - Add tests for networkstatus_compute_bw_weights_v10.
+    - Add unit tests circuit_predict_and_launch_new.
+    - Extract dummy_origin_circuit_new so it can be used by other
+      test functions.
+
+
+Changes in version 0.2.8.12 - 2016-12-19
+  Tor 0.2.8.12 backports a fix for a medium-severity issue (bug 21018
+  below) where Tor clients could crash when attempting to visit a
+  hostile hidden service. Clients are recommended to upgrade as packages
+  become available for their systems.
+
+  It also includes an updated list of fallback directories, backported
+  from 0.2.9.
+
+  Now that the Tor 0.2.9 series is stable, only major bugfixes will be
+  backported to 0.2.8 in the future.
+
+  o Major bugfixes (parsing, security, backported from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Minor features (fallback directory list, backported from 0.2.9.8):
+    - Replace the 81 remaining fallbacks of the 100 originally
+      introduced in Tor 0.2.8.3-alpha in March 2016, with a list of 177
+      fallbacks (123 new, 54 existing, 27 removed) generated in December
+      2016. Resolves ticket 20170.
+
+  o Minor features (geoip, backported from 0.2.9.7-rc):
+    - Update geoip and geoip6 to the December 7 2016 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.9.8 - 2016-12-19
+  Tor 0.2.9.8 is the first stable release of the Tor 0.2.9 series.
+
+  The Tor 0.2.9 series makes mandatory a number of security features
+  that were formerly optional. It includes support for a new shared-
+  randomness protocol that will form the basis for next generation
+  hidden services, includes a single-hop hidden service mode for
+  optimizing .onion services that don't actually want to be hidden,
+  tries harder not to overload the directory authorities with excessive
+  downloads, and supports a better protocol versioning scheme for
+  improved compatibility with other implementations of the Tor protocol.
+
+  And of course, there are numerous other bugfixes and improvements.
+
+  This release also includes a fix for a medium-severity issue (bug
+  21018 below) where Tor clients could crash when attempting to visit a
+  hostile hidden service. Clients are recommended to upgrade as packages
+  become available for their systems.
+
+  Below are the changes since 0.2.9.7-rc. For a list of all changes
+  since 0.2.8, see the ReleaseNotes file.
+
+  o Major bugfixes (parsing, security):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Minor features (fallback directory list):
+    - Replace the 81 remaining fallbacks of the 100 originally
+      introduced in Tor 0.2.8.3-alpha in March 2016, with a list of 177
+      fallbacks (123 new, 54 existing, 27 removed) generated in December
+      2016. Resolves ticket 20170.
+
+
+Changes in version 0.2.9.7-rc - 2016-12-12
+  Tor 0.2.9.7-rc fixes a few small bugs remaining in Tor 0.2.9.6-rc,
+  including a few that had prevented tests from passing on
+  some platforms.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the December 7 2016 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfix (build):
+    - The current Git revision when building from a local repository is
+      now detected correctly when using git worktrees. Fixes bug 20492;
+      bugfix on 0.2.3.9-alpha.
+
+  o Minor bugfixes (directory authority):
+    - When computing old Tor protocol line version in protover, we were
+      looking at 0.2.7.5 twice instead of a specific case for
+      0.2.9.1-alpha. Fixes bug 20810; bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (download scheduling):
+    - Resolve a "bug" warning when considering a download schedule whose
+      delay had approached INT_MAX. Fixes 20875; bugfix on 0.2.9.5-alpha.
+
+  o Minor bugfixes (logging):
+    - Downgrade a harmless log message about the
+      pending_entry_connections list from "warn" to "info". Mitigates
+      bug 19926.
+
+  o Minor bugfixes (memory leak):
+    - Fix a small memory leak when receiving AF_UNIX connections on a
+      SocksPort. Fixes bug 20716; bugfix on 0.2.6.3-alpha.
+    - When moving a signed descriptor object from a source to an
+      existing destination, free the allocated memory inside that
+      destination object. Fixes bug 20715; bugfix on 0.2.8.3-alpha.
+
+  o Minor bugfixes (memory leak, use-after-free, linux seccomp2 sandbox):
+    - Fix a memory leak and use-after-free error when removing entries
+      from the sandbox's getaddrinfo() cache. Fixes bug 20710; bugfix on
+      0.2.5.5-alpha. Patch from "cypherpunks".
+
+  o Minor bugfixes (portability):
+    - Use the correct spelling of MAC_OS_X_VERSION_10_12 on configure.ac
+      Fixes bug 20935; bugfix on 0.2.9.6-rc.
+
+  o Minor bugfixes (unit tests):
+    - Stop expecting NetBSD unit tests to report success for ipfw. Part
+      of a fix for bug 19960; bugfix on 0.2.9.5-alpha.
+    - Fix tolerances in unit tests for monotonic time comparisons
+      between nanoseconds and microseconds. Previously, we accepted a 10
+      us difference only, which is not realistic on every platform's
+      clock_gettime(). Fixes bug 19974; bugfix on 0.2.9.1-alpha.
+    - Remove a double-free in the single onion service unit test. Stop
+      ignoring a return value. Make future changes less error-prone.
+      Fixes bug 20864; bugfix on 0.2.9.6-rc.
+
+
+Changes in version 0.2.8.11 - 2016-12-08
+  Tor 0.2.8.11 backports fixes for additional portability issues that
+  could prevent Tor from building correctly on OSX Sierra, or with
+  OpenSSL 1.1. Affected users should upgrade; others can safely stay
+  with 0.2.8.10.
+
+  o Minor bugfixes (portability):
+    - Avoid compilation errors when building on OSX Sierra. Sierra began
+      to support the getentropy() and clock_gettime() APIs, but created
+      a few problems in doing so. Tor 0.2.9 has a more thorough set of
+      workarounds; in 0.2.8, we are just using the /dev/urandom and mach
+      monotonic time interfaces. Fixes bug 20865. Bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (portability, backport from 0.2.9.5-alpha):
+    - Fix compilation with OpenSSL 1.1 and less commonly-used CPU
+      architectures. Closes ticket 20588.
+
+
+Changes in version 0.2.8.10 - 2016-12-02
+  Tor 0.2.8.10 backports a fix for a bug that would sometimes make clients
+  unusable after they left standby mode. It also backports fixes for
+  a few portability issues and a small but problematic memory leak.
+
+  o Major bugfixes (client reliability, backport from 0.2.9.5-alpha):
+    - When Tor leaves standby because of a new application request, open
+      circuits as needed to serve that request. Previously, we would
+      potentially wait a very long time. Fixes part of bug 19969; bugfix
+      on 0.2.8.1-alpha.
+
+  o Major bugfixes (client performance, backport from 0.2.9.5-alpha):
+    - Clients now respond to new application stream requests immediately
+      when they arrive, rather than waiting up to one second before
+      starting to handle them. Fixes part of bug 19969; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (portability, backport from 0.2.9.6-rc):
+    - Work around a bug in the OSX 10.12 SDK that would prevent us from
+      successfully targeting earlier versions of OSX. Resolves
+      ticket 20235.
+
+  o Minor bugfixes (portability, backport from 0.2.9.5-alpha):
+    - Fix implicit conversion warnings under OpenSSL 1.1. Fixes bug
+      20551; bugfix on 0.2.1.1-alpha.
+
+  o Minor bugfixes (relay, backport from 0.2.9.5-alpha):
+    - Work around a memory leak in OpenSSL 1.1 when encoding public
+      keys. Fixes bug 20553; bugfix on 0.0.2pre8.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 3 2016 Maxmind GeoLite2
+      Country database.
+
+Changes in version 0.2.9.6-rc - 2016-12-02
+  Tor 0.2.9.6-rc fixes a few remaining bugs found in the previous alpha
+  version. We hope that it will be ready to become stable soon, and we
+  encourage everyone to test this release. If no showstopper bugs are
+  found here, the next 0.2.9 release will be stable.
+
+  o Major bugfixes (relay, resolver, logging):
+    - For relays that don't know their own address, avoid attempting a
+      local hostname resolve for each descriptor we download. This
+      will cut down on the number of "Success: chose address 'x.x.x.x'"
+      log lines, and also avoid confusing clock jumps if the resolver
+      is slow. Fixes bugs 20423 and 20610; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (client, fascistfirewall):
+    - Avoid spurious warnings when ReachableAddresses or FascistFirewall
+      is set. Fixes bug 20306; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (hidden services):
+    - Stop ignoring the anonymity status of saved keys for hidden
+      services and single onion services when first starting tor.
+      Instead, refuse to start tor if any hidden service key has been
+      used in a different hidden service anonymity mode. Fixes bug
+      20638; bugfix on 17178 in 0.2.9.3-alpha; reported by ahf.
+
+  o Minor bugfixes (portability):
+    - Work around a bug in the OSX 10.12 SDK that would prevent us from
+      successfully targeting earlier versions of OSX. Resolves
+      ticket 20235.
+    - Run correctly when built on Windows build environments that
+      require _vcsprintf(). Fixes bug 20560; bugfix on 0.2.2.11-alpha.
+
+  o Minor bugfixes (single onion services, Tor2web):
+    - Stop complaining about long-term one-hop circuits deliberately
+      created by single onion services and Tor2web. These log messages
+      are intended to diagnose issue 8387, which relates to circuits
+      hanging around forever for no reason. Fixes bug 20613; bugfix on
+      0.2.9.1-alpha. Reported by "pastly".
+
+  o Minor bugfixes (unit tests):
+    - Stop spurious failures in the local interface address discovery
+      unit tests. Fixes bug 20634; bugfix on 0.2.8.1-alpha; patch by
+      Neel Chauhan.
+
+  o Documentation:
+    - Correct the minimum bandwidth value in torrc.sample, and queue a
+      corresponding change for torrc.minimal. Closes ticket 20085.
+
+
+Changes in version 0.2.9.5-alpha - 2016-11-08
+  Tor 0.2.9.5-alpha fixes numerous bugs discovered in the previous alpha
+  version. We believe one or two probably remain, and we encourage
+  everyone to test this release.
+
+  o Major bugfixes (client performance):
+    - Clients now respond to new application stream requests immediately
+      when they arrive, rather than waiting up to one second before
+      starting to handle them. Fixes part of bug 19969; bugfix
+      on 0.2.8.1-alpha.
+
+  o Major bugfixes (client reliability):
+    - When Tor leaves standby because of a new application request, open
+      circuits as needed to serve that request. Previously, we would
+      potentially wait a very long time. Fixes part of bug 19969; bugfix
+      on 0.2.8.1-alpha.
+
+  o Major bugfixes (download scheduling):
+    - When using an exponential backoff schedule, do not give up on
+      downloading just because we have failed a bunch of times. Since
+      each delay is longer than the last, retrying indefinitely won't
+      hurt. Fixes bug 20536; bugfix on 0.2.9.1-alpha.
+    - If a consensus expires while we are waiting for certificates to
+      download, stop waiting for certificates.
+    - If we stop waiting for certificates less than a minute after we
+      started downloading them, do not consider the certificate download
+      failure a separate failure. Fixes bug 20533; bugfix
+      on 0.2.0.9-alpha.
+    - Remove the maximum delay on exponential-backoff scheduling. Since
+      we now allow an infinite number of failures (see ticket 20536), we
+      must now allow the time to grow longer on each failure. Fixes part
+      of bug 20534; bugfix on 0.2.9.1-alpha.
+    - Make our initial download delays closer to those from 0.2.8. Fixes
+      another part of bug 20534; bugfix on 0.2.9.1-alpha.
+    - When determining when to download a directory object, handle times
+      after 2038 if the operating system supports them. (Someday this
+      will be important!) Fixes bug 20587; bugfix on 0.2.8.1-alpha.
+    - When using exponential backoff in test networks, use a lower
+      exponent, so the delays do not vary as much. This helps test
+      networks bootstrap consistently. Fixes bug 20597; bugfix on 20499.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 3 2016 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (client directory scheduling):
+    - Treat "relay too busy to answer request" as a failed request and a
+      reason to back off on our retry frequency. This is safe now that
+      exponential backoffs retry indefinitely, and avoids a bug where we
+      would reset our download schedule erroneously. Fixes bug 20593;
+      bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (client, logging):
+    - Remove a BUG warning in circuit_pick_extend_handshake(). Instead,
+      assume all nodes support EXTEND2. Use ntor whenever a key is
+      available. Fixes bug 20472; bugfix on 0.2.9.3-alpha.
+    - On DNSPort, stop logging a BUG warning on a failed hostname
+      lookup. Fixes bug 19869; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (hidden services):
+    - When configuring hidden services, check every hidden service
+      directory's permissions. Previously, we only checked the last
+      hidden service. Fixes bug 20529; bugfix the work to fix 13942
+      in 0.2.6.2-alpha.
+
+  o Minor bugfixes (portability):
+    - Fix compilation with OpenSSL 1.1 and less commonly-used CPU
+      architectures. Closes ticket 20588.
+    - Use ECDHE ciphers instead of ECDH in tortls tests. LibreSSL has
+      removed the ECDH ciphers which caused the tests to fail on
+      platforms which use it. Fixes bug 20460; bugfix on 0.2.8.1-alpha.
+    - Fix implicit conversion warnings under OpenSSL 1.1. Fixes bug
+      20551; bugfix on 0.2.1.1-alpha.
+
+  o Minor bugfixes (relay bootstrap):
+    - Ensure relays don't make multiple connections during bootstrap.
+      Fixes bug 20591; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (relay):
+    - Work around a memory leak in OpenSSL 1.1 when encoding public
+      keys. Fixes bug 20553; bugfix on 0.0.2pre8.
+    - Avoid a small memory leak when informing worker threads about
+      rotated onion keys. Fixes bug 20401; bugfix on 0.2.6.3-alpha.
+    - Do not try to parallelize workers more than 16x without the user
+      explicitly configuring us to do so, even if we do detect more than
+      16 CPU cores. Fixes bug 19968; bugfix on 0.2.3.1-alpha.
+
+  o Minor bugfixes (single onion services):
+    - Start correctly when creating a single onion service in a
+      directory that did not previously exist. Fixes bug 20484; bugfix
+      on 0.2.9.3-alpha.
+
+  o Minor bugfixes (testing):
+    - Avoid a unit test failure on systems with over 16 detectable CPU
+      cores. Fixes bug 19968; bugfix on 0.2.3.1-alpha.
+
+  o Documentation:
+    - Clarify that setting HiddenServiceNonAnonymousMode requires you to
+      also set "SOCKSPort 0". Fixes bug 20487; bugfix on 0.2.9.3-alpha.
+    - Module-level documentation for several more modules. Closes
+      tickets 19287 and 19290.
 
 
 Changes in version 0.2.8.9 - 2016-10-17
@@ -123,7 +9128,7 @@ Changes in version 0.2.9.4-alpha - 2016-10-17
   o Minor bugfixes (getpass):
     - Defensively fix a non-triggerable heap corruption at do_getpass()
       to protect ourselves from mistakes in the future. Fixes bug
-      #19223; bugfix on 0.2.7.3-rc. Bug found by Guido Vranken, patch
+      19223; bugfix on 0.2.7.3-rc. Bug found by Guido Vranken, patch
       by nherring.
 
   o Minor bugfixes (hidden service):
@@ -194,7 +9199,7 @@ Changes in version 0.2.9.3-alpha - 2016-09-23
 
   o Major bugfixes (circuit building):
     - Hidden service client-to-intro-point and service-to-rendezvous-
-      point cicruits use the TAP key supplied by the protocol, to avoid
+      point circuits use the TAP key supplied by the protocol, to avoid
       epistemic attacks. Fixes bug 19163; bugfix on 0.2.4.18-rc.
 
   o Major bugfixes (compilation, OpenBSD):
@@ -216,8 +9221,9 @@ Changes in version 0.2.9.3-alpha - 2016-09-23
       OpenSSL 0.9.7 or later since 2009. Closes ticket 19998.
 
   o Minor feature (fallback directories):
-    - Remove broken entries from the hard-coded fallback directory list.
-      Closes ticket 20190; patch by teor.
+    - Remove 8 fallbacks that are no longer suitable, leaving 81 of the
+      100 fallbacks originally introduced in Tor 0.2.8.2-alpha in March
+      2016. Closes ticket 20190; patch by teor.
 
   o Minor features (geoip, also in 0.2.8.8):
     - Update geoip and geoip6 to the September 6 2016 Maxmind GeoLite2
@@ -312,7 +9318,7 @@ Changes in version 0.2.9.3-alpha - 2016-09-23
     - Document the default PathsNeededToBuildCircuits value that's used
       by clients when the directory authorities don't set
       min_paths_for_circs_pct. Fixes bug 20117; bugfix on 02c320916e02
-      in tor-0.2.4.10-alpha. Patch by teor, reported by Jesse V.
+      in 0.2.4.10-alpha. Patch by teor, reported by Jesse V.
     - Fix manual for the User option: it takes a username, not a UID.
       Fixes bug 19122; bugfix on 0.0.2pre16 (the first version to have
       a manpage!).
@@ -351,12 +9357,12 @@ Changes in version 0.2.9.3-alpha - 2016-09-23
 
   o Minor bugfixes (options):
     - Check the consistency of UseEntryGuards and EntryNodes more
-      reliably. Fixes bug 20074; bugfix on tor- 0.2.4.12-alpha. Patch
+      reliably. Fixes bug 20074; bugfix on 0.2.4.12-alpha. Patch
       by teor.
     - Stop changing the configured value of UseEntryGuards on
       authorities and Tor2web clients. Fixes bug 20074; bugfix on
-      commits 51fc6799 in tor-0.1.1.16-rc and acda1735 in tor-0.2.4.3-
-      alpha. Patch by teor.
+      commits 51fc6799 in 0.1.1.16-rc and acda1735 in 0.2.4.3-alpha.
+      Patch by teor.
 
   o Minor bugfixes (Tor2web):
     - Prevent Tor2web clients running hidden services, these services
@@ -395,8 +9401,9 @@ Changes in version 0.2.8.8 - 2016-09-23
       this one.
 
   o Minor feature (fallback directories):
-    - Remove broken fallbacks from the hard-coded fallback directory
-      list. Closes ticket 20190; patch by teor.
+    - Remove 8 fallbacks that are no longer suitable, leaving 81 of the
+      100 fallbacks originally introduced in Tor 0.2.8.2-alpha in March
+      2016. Closes ticket 20190; patch by teor.
 
   o Minor features (geoip):
     - Update geoip and geoip6 to the September 6 2016 Maxmind GeoLite2
@@ -478,7 +9485,7 @@ Changes in version 0.2.9.2-alpha - 2016-08-24
   o Deprecated features:
     - A number of DNS-cache-related sub-options for client ports are now
       deprecated for security reasons, and may be removed in a future
-      version of Tor. (We believe that client-side DNS cacheing is a bad
+      version of Tor. (We believe that client-side DNS caching is a bad
       idea for anonymity, and you should not turn it on.) The options
       are: CacheDNS, CacheIPv4DNS, CacheIPv6DNS, UseDNSCache,
       UseIPv4Cache, and UseIPv6Cache.
@@ -850,8 +9857,9 @@ Changes in version 0.2.8.6 - 2016-08-02
       is signed. Fixes bug 19682; bugfix on 0.2.8.1-alpha.
 
   o Minor bugfixes (fallback directories):
-    - Remove a fallback that was on the hardcoded list, then opted-out.
-      Fixes bug 19782; update to fallback list from 0.2.8.2-alpha.
+    - Remove 1 fallback that was on the hardcoded list, then opted-out,
+      leaving 89 of the 100 fallbacks originally introduced in Tor
+      0.2.8.2-alpha in March 2016. Closes ticket 19782; patch by teor.
 
   o Minor bugfixes (Linux seccomp2 sandbox):
     - Allow more syscalls when running with "Sandbox 1" enabled:
@@ -875,7 +9883,7 @@ Changes in version 0.2.8.5-rc - 2016-07-07
   o Major bugfixes (heartbeat):
     - Fix a regression that would crash Tor when the periodic
       "heartbeat" log messages were disabled. Fixes bug 19454; bugfix on
-      tor-0.2.8.1-alpha. Reported by "kubaku".
+      0.2.8.1-alpha. Reported by "kubaku".
 
   o Minor features (build):
     - Tor now again builds with the recent OpenSSL 1.1 development
@@ -923,8 +9931,9 @@ Changes in version 0.2.8.5-rc - 2016-07-07
     - Update fallback whitelist and blacklist based on relay operator
       emails. Blacklist unsuitable (non-working, over-volatile)
       fallbacks. Resolves ticket 19071. Patch by teor.
-    - Update hard-coded fallback list to remove unsuitable fallbacks.
-      Resolves ticket 19071. Patch by teor.
+    - Remove 10 unsuitable fallbacks, leaving 90 of the 100 fallbacks
+      originally introduced in Tor 0.2.8.2-alpha in March 2016. Closes
+      ticket 19071; patch by teor.
 
 
 Changes in version 0.2.8.4-rc - 2016-06-15
@@ -1021,7 +10030,7 @@ Changes in version 0.2.8.3-alpha - 2016-05-26
 
   o Major bugfixes (testing):
     - Fix a bug that would block 'make test-network-all' on systems where
-      IPv6 packets were lost. Fixes bug 19008; bugfix on tor-0.2.7.3-rc.
+      IPv6 packets were lost. Fixes bug 19008; bugfix on 0.2.7.3-rc.
     - Avoid "WSANOTINITIALISED" warnings in the unit tests. Fixes bug 18668;
       bugfix on 0.2.8.1-alpha.
 
@@ -1034,10 +10043,12 @@ Changes in version 0.2.8.3-alpha - 2016-05-26
     - Give each fallback the same weight for client selection; restrict
       fallbacks to one per operator; report fallback directory detail
       changes when rebuilding list; add new fallback directory mirrors
-      to the whitelist; update fallback directories based on the latest
-      OnionOO data; and any other minor simplifications and fixes.
-      Closes tasks 17158, 17905, 18749, bug 18689, and fixes part of bug
-      18812 on 0.2.8.1-alpha; patch by "teor".
+      to the whitelist; and many other minor simplifications and fixes.
+      Closes tasks 17905, 18749, bug 18689, and fixes part of bug 18812 on
+      0.2.8.1-alpha; patch by "teor".
+    - Replace the 21 fallbacks generated in January 2016 and included in
+      Tor 0.2.8.1-alpha, with a list of 100 fallbacks generated in March
+      2016. Closes task 17158; patch by "teor".
 
   o Minor features (geoip):
     - Update geoip and geoip6 to the May 4 2016 Maxmind GeoLite2
@@ -1307,7 +10318,7 @@ Changes in version 0.2.8.2-alpha - 2016-03-28
       testing versions of the static libraries. Fixes bug 18490; bugfix
       on 0.2.7.1-alpha.
     - Avoid spurious failures from configure files related to calling
-      exit(0) in TOR_SEARCH_LIBRARY. Fixes bug 18625; bugfix on
+      exit(0) in TOR_SEARCH_LIBRARY. Fixes bug 18626; bugfix on
       0.2.0.1-alpha. Patch from "cypherpunks".
     - Silence spurious clang-scan warnings in the ed25519_donna code by
       explicitly initializing some objects. Fixes bug 18384; bugfix on
@@ -1481,10 +10492,11 @@ Changes in version 0.2.8.1-alpha - 2016-02-04
       should reduces failures due to fallback churn. Implements ticket
       4483. Patch by "teor". Implements IPv4 portions of proposal 210 by
       "mikeperry" and "teor".
-    - Include a trial list of default fallback directories, based on an
-      opt-in survey of suitable relays. Doing this should make clients
-      bootstrap more quickly and reliably, and reduce the load on the
-      directory authorities. Closes ticket 15775. Patch by "teor".
+    - Include a trial list of 21 default fallback directories, generated
+      in January 2016, based on an opt-in survey of suitable relays.
+      Doing this should make clients bootstrap more quickly and reliably,
+      and reduce the load on the directory authorities. Closes ticket
+      15775. Patch by "teor".
       Candidates identified using an OnionOO script by "weasel", "teor",
       "gsathya", and "karsten".
     - Previously only relays that explicitly opened a directory port
@@ -2596,7 +11608,7 @@ Changes in version 0.2.6.8 - 2015-05-21
 
   o Major bugfixes (hidden services, backport from 0.2.7.1-alpha):
     - Revert commit that made directory authorities assign the HSDir
-      flag to relay without a DirPort; this was bad because such relays
+      flag to relays without a DirPort; this was bad because such relays
       can't handle BEGIN_DIR cells. Fixes bug 15850; bugfix
       on 0.2.6.3-alpha.
 
@@ -2637,7 +11649,7 @@ Changes in version 0.2.7.1-alpha - 2015-05-12
 
   o Major bugfixes (hidden services):
     - Revert commit that made directory authorities assign the HSDir
-      flag to relay without a DirPort; this was bad because such relays
+      flag to relays without a DirPort; this was bad because such relays
       can't handle BEGIN_DIR cells. Fixes bug 15850; bugfix
       on 0.2.6.3-alpha.
 
@@ -2943,7 +11955,7 @@ Changes in version 0.2.6.5-rc - 2015-03-18
   o Major bugfixes (pluggable transports):
     - Initialize the extended OR Port authentication cookie before
       launching pluggable transports. This prevents a race condition
-      that occured when server-side pluggable transports would cache the
+      that occurred when server-side pluggable transports would cache the
       authentication cookie before it has been (re)generated. Fixes bug
       15240; bugfix on 0.2.5.1-alpha.
 
@@ -3686,7 +12698,7 @@ Changes in version 0.2.6.2-alpha - 2014-12-31
       some bugs where we would look at (but fortunately, not reveal)
       uninitialized memory on the stack. Fixes bug 14013; bugfix on all
       versions of Tor.
-    - Clear all memory targetted by tor_addr_{to,from}_sockaddr(), not
+    - Clear all memory targeted by tor_addr_{to,from}_sockaddr(), not
       just the part that's used. This makes it harder for data leak bugs
       to occur in the event of other programming failures. Resolves
       ticket 14041.
@@ -4829,7 +13841,7 @@ Changes in version 0.2.5.5-alpha - 2014-06-18
       directory authority options, remove the documentation for a
       V2-directory fetching option that no longer exists. Resolves
       ticket 11634.
-    - Correct the documenation so that it lists the correct directory
+    - Correct the documentation so that it lists the correct directory
       for the stats files. (They are in a subdirectory called "stats",
       not "status".)
     - In the manpage, move more authority-only options into the
@@ -6114,7 +15126,7 @@ Changes in version 0.2.5.1-alpha - 2013-10-02
       from Arlo Breault.
     - Remove instances of strcpy() from the unit tests. They weren't
       hurting anything, since they were only in the unit tests, but it's
-      embarassing to have strcpy() in the code at all, and some analysis
+      embarrassing to have strcpy() in the code at all, and some analysis
       tools don't like it. Fixes bug 8790; bugfix on 0.2.3.6-alpha and
       0.2.3.8-alpha. Patch from Arlo Breault.
 
@@ -6215,7 +15227,7 @@ Changes in version 0.2.4.16-rc - 2013-08-10
       0.2.4.15-rc. Found by stem integration tests.
 
   o Minor bugfixes:
-    - Fix an invalid memory read that occured when a pluggable
+    - Fix an invalid memory read that occurred when a pluggable
       transport proxy failed its configuration protocol.
       Fixes bug 9288; bugfix on 0.2.4.1-alpha.
     - When evaluating whether to use a connection that we haven't
@@ -6473,7 +15485,7 @@ Changes in version 0.2.4.12-alpha - 2013-04-18
       0.2.0.10-alpha. Reported pseudonymously.
     - Make the format and order of STREAM events for DNS lookups
       consistent among the various ways to launch DNS lookups. Fixes
-      bug 8203; bugfix on 0.2.0.24-rc. Patch by "Desoxy."
+      bug 8203; bugfix on 0.2.0.24-rc. Patch by "Desoxy".
     - Correct our check for which versions of Tor support the EXTEND2
       cell. We had been willing to send it to Tor 0.2.4.7-alpha and
       later, when support was really added in version 0.2.4.8-alpha.
@@ -8562,7 +17574,7 @@ Changes in version 0.2.2.36 - 2012-05-24
       issue 4788.
     - Update to the May 1 2012 Maxmind GeoLite Country database.
 
-  - Feature removal:
+  o Feature removal:
     - When sending or relaying a RELAY_EARLY cell, we used to convert
       it to a RELAY cell if the connection was using the v1 link
       protocol. This was a workaround for older versions of Tor, which
@@ -9006,7 +18018,7 @@ Changes in version 0.2.3.11-alpha - 2012-01-22
       CloseHSServiceRendCircuitsImmediatelyOnTimeout option. Fixes the
       remaining part of bug 1297; bugfix on 0.2.2.2-alpha.
     - Make sure we never mark the wrong rendezvous circuit as having
-      had its introduction cell acknowleged by the introduction-point
+      had its introduction cell acknowledged by the introduction-point
       relay. Previously, when we received an INTRODUCE_ACK cell on a
       client-side hidden-service introduction circuit, we might have
       marked a rendezvous circuit other than the one we specified in
@@ -10313,7 +19325,7 @@ Changes in version 0.2.3.3-alpha - 2011-09-01
       raised by bug 3898.
     - The "--quiet" and "--hush" options now apply not only to Tor's
       behavior before logs are configured, but also to Tor's behavior in
-      the absense of configured logs. Fixes bug 3550; bugfix on
+      the absence of configured logs. Fixes bug 3550; bugfix on
       0.2.0.10-alpha.
 
   o Minor bugfixes (also part of 0.2.2.31-rc):
@@ -12814,7 +21826,7 @@ Changes in version 0.2.2.8-alpha - 2010-01-26
   please upgrade.
 
   o Major bugfixes:
-    - Fix a memory corruption bug on bridges that occured during the
+    - Fix a memory corruption bug on bridges that occurred during the
       inclusion of stats data in extra-info descriptors. Also fix the
       interface for geoip_get_bridge_stats* to prevent similar bugs in
       the future. Diagnosis by Tas, patch by Karsten and Sebastian.
@@ -13228,7 +22240,7 @@ Changes in version 0.2.2.2-alpha - 2009-09-21
       to EDGE and find out if the build-time data in the .tor/state gets
       reset without loss of Tor usability. You should also see a notice
       log message telling you that Tor has reset its timeout.
-    - Directory authorities can now vote on arbitary integer values as
+    - Directory authorities can now vote on arbitrary integer values as
       part of the consensus process. This is designed to help set
       network-wide parameters. Implements proposal 167.
     - Tor now reads the "circwindow" parameter out of the consensus,
@@ -16256,7 +25268,7 @@ Changes in version 0.2.0.9-alpha - 2007-10-24
     - Distinguish between detached signatures for the wrong period, and
       detached signatures for a divergent vote.
     - Fix a small memory leak when computing a consensus.
-    - When there's no concensus, we were forming a vote every 30
+    - When there's no consensus, we were forming a vote every 30
       minutes, but writing the "valid-after" line in our vote based
       on our configured V3AuthVotingInterval: so unless the intervals
       matched up, we immediately rejected our own vote because it didn't
@@ -19643,7 +28655,7 @@ Changes in version 0.1.0.9-rc - 2005-06-09
       KeepalivePeriod, ClientOnly, NoPublish, HttpProxy, HttpsProxy,
       HttpProxyAuthenticator
     - Stop warning about sigpipes in the logs. We're going to
-      pretend that getting these occassionally is normal and fine.
+      pretend that getting these occasionally is normal and fine.
     - Resolve OS X installer bugs: stop claiming to be 0.0.9.2 in
       certain
       installer screens; and don't put stuff into StartupItems unless
@@ -20108,7 +29120,7 @@ Changes in version 0.1.0.1-rc - 2005-03-28
 
 Changes in version 0.0.9.6 - 2005-03-24
   o Bugfixes on 0.0.9.x (crashes and asserts):
-    - Add new end stream reasons to maintainance branch. Fix bug where
+    - Add new end stream reasons to maintenance branch. Fix bug where
       reason (8) could trigger an assert. Prevent bug from recurring.
     - Apparently win32 stat wants paths to not end with a slash.
     - Fix assert triggers in assert_cpath_layer_ok(), where we were
@@ -21498,4 +30510,3 @@ Changes in version 0.0.2pre13 - 2003-10-19
     - If --DebugLogFile is specified, log to it at -l debug
     - If --LogFile is specified, use it instead of commandline
     - If --RunAsDaemon is set, tor forks and backgrounds on startup
-
diff --git a/Doxyfile.in b/Doxyfile.in
index a39348f2cb..4caf421097 100644
--- a/Doxyfile.in
+++ b/Doxyfile.in
@@ -14,211 +14,211 @@
 # Project related configuration options
 #---------------------------------------------------------------------------
 
-# This tag specifies the encoding used for all characters in the config file 
-# that follow. The default is UTF-8 which is also the encoding used for all 
-# text before the first occurrence of this tag. Doxygen uses libiconv (or the 
-# iconv built into libc) for the transcoding. See 
+# This tag specifies the encoding used for all characters in the config file
+# that follow. The default is UTF-8 which is also the encoding used for all
+# text before the first occurrence of this tag. Doxygen uses libiconv (or the
+# iconv built into libc) for the transcoding. See
 # http://www.gnu.org/software/libiconv for the list of possible encodings.
 
 DOXYFILE_ENCODING      = UTF-8
 
-# The PROJECT_NAME tag is a single word (or a sequence of words surrounded 
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded
 # by quotes) that should identify the project.
 
 PROJECT_NAME           = tor
 
-# The PROJECT_NUMBER tag can be used to enter a project or revision number. 
-# This could be handy for archiving the generated documentation or 
+# The PROJECT_NUMBER tag can be used to enter a project or revision number.
+# This could be handy for archiving the generated documentation or
 # if some version control system is used.
 
 PROJECT_NUMBER         = @VERSION@
 
-# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) 
-# base path where the generated documentation will be put. 
-# If a relative path is entered, it will be relative to the location 
+# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute)
+# base path where the generated documentation will be put.
+# If a relative path is entered, it will be relative to the location
 # where doxygen was started. If left blank the current directory will be used.
 
 OUTPUT_DIRECTORY       = @top_builddir@/doc/doxygen
 
-# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create 
-# 4096 sub-directories (in 2 levels) under the output directory of each output 
-# format and will distribute the generated files over these directories. 
-# Enabling this option can be useful when feeding doxygen a huge amount of 
-# source files, where putting all generated files in the same directory would 
+# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create
+# 4096 sub-directories (in 2 levels) under the output directory of each output
+# format and will distribute the generated files over these directories.
+# Enabling this option can be useful when feeding doxygen a huge amount of
+# source files, where putting all generated files in the same directory would
 # otherwise cause performance problems for the file system.
 
 CREATE_SUBDIRS         = NO
 
-# The OUTPUT_LANGUAGE tag is used to specify the language in which all 
-# documentation generated by doxygen is written. Doxygen will use this 
-# information to generate all constant output in the proper language. 
-# The default language is English, other supported languages are: 
-# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, 
-# Croatian, Czech, Danish, Dutch, Farsi, Finnish, French, German, Greek, 
-# Hungarian, Italian, Japanese, Japanese-en (Japanese with English messages), 
-# Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian, Polish, 
-# Portuguese, Romanian, Russian, Serbian, Slovak, Slovene, Spanish, Swedish, 
+# The OUTPUT_LANGUAGE tag is used to specify the language in which all
+# documentation generated by doxygen is written. Doxygen will use this
+# information to generate all constant output in the proper language.
+# The default language is English, other supported languages are:
+# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional,
+# Croatian, Czech, Danish, Dutch, Farsi, Finnish, French, German, Greek,
+# Hungarian, Italian, Japanese, Japanese-en (Japanese with English messages),
+# Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian, Polish,
+# Portuguese, Romanian, Russian, Serbian, Slovak, Slovene, Spanish, Swedish,
 # and Ukrainian.
 
 OUTPUT_LANGUAGE        = English
 
-# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will 
-# include brief member descriptions after the members that are listed in 
-# the file and class documentation (similar to JavaDoc). 
+# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will
+# include brief member descriptions after the members that are listed in
+# the file and class documentation (similar to JavaDoc).
 # Set to NO to disable this.
 
 BRIEF_MEMBER_DESC      = NO
 
-# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend 
-# the brief description of a member or function before the detailed description. 
-# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the 
+# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend
+# the brief description of a member or function before the detailed description.
+# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the
 # brief descriptions will be completely suppressed.
 
 REPEAT_BRIEF           = YES
 
-# This tag implements a quasi-intelligent brief description abbreviator 
-# that is used to form the text in various listings. Each string 
-# in this list, if found as the leading text of the brief description, will be 
-# stripped from the text and the result after processing the whole list, is 
-# used as the annotated text. Otherwise, the brief description is used as-is. 
-# If left blank, the following values are used ("$name" is automatically 
-# replaced with the name of the entity): "The $name class" "The $name widget" 
-# "The $name file" "is" "provides" "specifies" "contains" 
+# This tag implements a quasi-intelligent brief description abbreviator
+# that is used to form the text in various listings. Each string
+# in this list, if found as the leading text of the brief description, will be
+# stripped from the text and the result after processing the whole list, is
+# used as the annotated text. Otherwise, the brief description is used as-is.
+# If left blank, the following values are used ("$name" is automatically
+# replaced with the name of the entity): "The $name class" "The $name widget"
+# "The $name file" "is" "provides" "specifies" "contains"
 # "represents" "a" "an" "the"
 
-ABBREVIATE_BRIEF       = 
+ABBREVIATE_BRIEF       =
 
-# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then 
-# Doxygen will generate a detailed section even if there is only a brief 
+# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then
+# Doxygen will generate a detailed section even if there is only a brief
 # description.
 
 ALWAYS_DETAILED_SEC    = NO
 
-# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all 
-# inherited members of a class in the documentation of that class as if those 
-# members were ordinary class members. Constructors, destructors and assignment 
+# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all
+# inherited members of a class in the documentation of that class as if those
+# members were ordinary class members. Constructors, destructors and assignment
 # operators of the base classes will not be shown.
 
 INLINE_INHERITED_MEMB  = NO
 
-# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full 
-# path before files name in the file list and in the header files. If set 
+# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full
+# path before files name in the file list and in the header files. If set
 # to NO the shortest path that makes the file name unique will be used.
 
 FULL_PATH_NAMES        = NO
 
-# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag 
-# can be used to strip a user-defined part of the path. Stripping is 
-# only done if one of the specified strings matches the left-hand part of 
-# the path. The tag can be used to show relative paths in the file list. 
-# If left blank the directory from which doxygen is run is used as the 
+# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag
+# can be used to strip a user-defined part of the path. Stripping is
+# only done if one of the specified strings matches the left-hand part of
+# the path. The tag can be used to show relative paths in the file list.
+# If left blank the directory from which doxygen is run is used as the
 # path to strip.
 
-STRIP_FROM_PATH        = 
+STRIP_FROM_PATH        =
 
-# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of 
-# the path mentioned in the documentation of a class, which tells 
-# the reader which header file to include in order to use a class. 
-# If left blank only the name of the header file containing the class 
-# definition is used. Otherwise one should specify the include paths that 
+# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of
+# the path mentioned in the documentation of a class, which tells
+# the reader which header file to include in order to use a class.
+# If left blank only the name of the header file containing the class
+# definition is used. Otherwise one should specify the include paths that
 # are normally passed to the compiler using the -I flag.
 
-STRIP_FROM_INC_PATH    = 
+STRIP_FROM_INC_PATH    =
 
-# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter 
-# (but less readable) file names. This can be useful is your file systems 
+# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter
+# (but less readable) file names. This can be useful is your file systems
 # doesn't support long names like on DOS, Mac, or CD-ROM.
 
 SHORT_NAMES            = NO
 
-# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen 
-# will interpret the first line (until the first dot) of a JavaDoc-style 
-# comment as the brief description. If set to NO, the JavaDoc 
-# comments will behave just like regular Qt-style comments 
+# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen
+# will interpret the first line (until the first dot) of a JavaDoc-style
+# comment as the brief description. If set to NO, the JavaDoc
+# comments will behave just like regular Qt-style comments
 # (thus requiring an explicit @brief command for a brief description.)
 
 JAVADOC_AUTOBRIEF      = NO
 
-# If the QT_AUTOBRIEF tag is set to YES then Doxygen will 
-# interpret the first line (until the first dot) of a Qt-style 
-# comment as the brief description. If set to NO, the comments 
-# will behave just like regular Qt-style comments (thus requiring 
+# If the QT_AUTOBRIEF tag is set to YES then Doxygen will
+# interpret the first line (until the first dot) of a Qt-style
+# comment as the brief description. If set to NO, the comments
+# will behave just like regular Qt-style comments (thus requiring
 # an explicit \brief command for a brief description.)
 
 QT_AUTOBRIEF           = NO
 
-# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen 
-# treat a multi-line C++ special comment block (i.e. a block of //! or /// 
-# comments) as a brief description. This used to be the default behaviour. 
-# The new default is to treat a multi-line C++ comment block as a detailed 
+# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen
+# treat a multi-line C++ special comment block (i.e. a block of //! or ///
+# comments) as a brief description. This used to be the default behaviour.
+# The new default is to treat a multi-line C++ comment block as a detailed
 # description. Set this tag to YES if you prefer the old behaviour instead.
 
 MULTILINE_CPP_IS_BRIEF = NO
 
-# If the DETAILS_AT_TOP tag is set to YES then Doxygen 
+# If the DETAILS_AT_TOP tag is set to YES then Doxygen
 # will output the detailed description near the top, like JavaDoc.
-# If set to NO, the detailed description appears after the member 
+# If set to NO, the detailed description appears after the member
 # documentation.
 
 # DETAILS_AT_TOP         = NO
 
-# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented 
-# member inherits the documentation from any documented member that it 
+# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented
+# member inherits the documentation from any documented member that it
 # re-implements.
 
 INHERIT_DOCS           = YES
 
-# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce 
-# a new page for each member. If set to NO, the documentation of a member will 
+# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce
+# a new page for each member. If set to NO, the documentation of a member will
 # be part of the file/class/namespace that contains it.
 
 SEPARATE_MEMBER_PAGES  = NO
 
-# The TAB_SIZE tag can be used to set the number of spaces in a tab. 
+# The TAB_SIZE tag can be used to set the number of spaces in a tab.
 # Doxygen uses this value to replace tabs by spaces in code fragments.
 
 TAB_SIZE               = 8
 
-# This tag can be used to specify a number of aliases that acts 
-# as commands in the documentation. An alias has the form "name=value". 
-# For example adding "sideeffect=\par Side Effects:\n" will allow you to 
-# put the command \sideeffect (or @sideeffect) in the documentation, which 
-# will result in a user-defined paragraph with heading "Side Effects:". 
+# This tag can be used to specify a number of aliases that acts
+# as commands in the documentation. An alias has the form "name=value".
+# For example adding "sideeffect=\par Side Effects:\n" will allow you to
+# put the command \sideeffect (or @sideeffect) in the documentation, which
+# will result in a user-defined paragraph with heading "Side Effects:".
 # You can put \n's in the value part of an alias to insert newlines.
 
-ALIASES                = 
+ALIASES                =
 
-# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C 
-# sources only. Doxygen will then generate output that is more tailored for C. 
-# For instance, some of the names that are used will be different. The list 
+# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C
+# sources only. Doxygen will then generate output that is more tailored for C.
+# For instance, some of the names that are used will be different. The list
 # of all members will be omitted, etc.
 
 OPTIMIZE_OUTPUT_FOR_C  = YES
 
-# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java 
-# sources only. Doxygen will then generate output that is more tailored for 
-# Java. For instance, namespaces will be presented as packages, qualified 
+# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java
+# sources only. Doxygen will then generate output that is more tailored for
+# Java. For instance, namespaces will be presented as packages, qualified
 # scopes will look different, etc.
 
 OPTIMIZE_OUTPUT_JAVA   = NO
 
-# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran 
-# sources only. Doxygen will then generate output that is more tailored for 
+# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran
+# sources only. Doxygen will then generate output that is more tailored for
 # Fortran.
 
 OPTIMIZE_FOR_FORTRAN   = NO
 
-# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL 
-# sources. Doxygen will then generate output that is tailored for 
+# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL
+# sources. Doxygen will then generate output that is tailored for
 # VHDL.
 
 OPTIMIZE_OUTPUT_VHDL   = NO
 
-# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want 
-# to include (a tag file for) the STL sources as input, then you should 
-# set this tag to YES in order to let doxygen match functions declarations and 
-# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. 
-# func(std::string) {}). This also make the inheritance and collaboration 
+# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want
+# to include (a tag file for) the STL sources as input, then you should
+# set this tag to YES in order to let doxygen match functions declarations and
+# definitions whose arguments contain STL classes (e.g. func(std::string); v.s.
+# func(std::string) {}). This also make the inheritance and collaboration
 # diagrams that involve STL classes more complete and accurate.
 
 BUILTIN_STL_SUPPORT    = NO
@@ -228,42 +228,42 @@ BUILTIN_STL_SUPPORT    = NO
 
 CPP_CLI_SUPPORT        = NO
 
-# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. 
-# Doxygen will parse them like normal C++ but will assume all classes use public 
+# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only.
+# Doxygen will parse them like normal C++ but will assume all classes use public
 # instead of private inheritance when no explicit protection keyword is present.
 
 SIP_SUPPORT            = NO
 
-# For Microsoft's IDL there are propget and propput attributes to indicate getter 
-# and setter methods for a property. Setting this option to YES (the default) 
-# will make doxygen to replace the get and set methods by a property in the 
-# documentation. This will only work if the methods are indeed getting or 
-# setting a simple type. If this is not the case, or you want to show the 
+# For Microsoft's IDL there are propget and propput attributes to indicate getter
+# and setter methods for a property. Setting this option to YES (the default)
+# will make doxygen to replace the get and set methods by a property in the
+# documentation. This will only work if the methods are indeed getting or
+# setting a simple type. If this is not the case, or you want to show the
 # methods anyway, you should set this option to NO.
 
 IDL_PROPERTY_SUPPORT   = NO
 
-# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC 
-# tag is set to YES, then doxygen will reuse the documentation of the first 
-# member in the group (if any) for the other members of the group. By default 
+# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC
+# tag is set to YES, then doxygen will reuse the documentation of the first
+# member in the group (if any) for the other members of the group. By default
 # all members of a group must be documented explicitly.
 
 DISTRIBUTE_GROUP_DOC   = NO
 
-# Set the SUBGROUPING tag to YES (the default) to allow class member groups of 
-# the same type (for instance a group of public functions) to be put as a 
-# subgroup of that type (e.g. under the Public Functions section). Set it to 
-# NO to prevent subgrouping. Alternatively, this can be done per class using 
+# Set the SUBGROUPING tag to YES (the default) to allow class member groups of
+# the same type (for instance a group of public functions) to be put as a
+# subgroup of that type (e.g. under the Public Functions section). Set it to
+# NO to prevent subgrouping. Alternatively, this can be done per class using
 # the \nosubgrouping command.
 
 SUBGROUPING            = YES
 
-# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum 
-# is documented as struct, union, or enum with the name of the typedef. So 
-# typedef struct TypeS {} TypeT, will appear in the documentation as a struct 
-# with name TypeT. When disabled the typedef will appear as a member of a file, 
-# namespace, or class. And the struct will be named TypeS. This can typically 
-# be useful for C code in case the coding convention dictates that all compound 
+# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum
+# is documented as struct, union, or enum with the name of the typedef. So
+# typedef struct TypeS {} TypeT, will appear in the documentation as a struct
+# with name TypeT. When disabled the typedef will appear as a member of a file,
+# namespace, or class. And the struct will be named TypeS. This can typically
+# be useful for C code in case the coding convention dictates that all compound
 # types are typedef'ed and only the typedef is referenced, never the tag name.
 
 TYPEDEF_HIDES_STRUCT   = NO
@@ -272,370 +272,366 @@ TYPEDEF_HIDES_STRUCT   = NO
 # Build related configuration options
 #---------------------------------------------------------------------------
 
-# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in 
-# documentation are documented, even if no documentation was available. 
-# Private class members and static file members will be hidden unless 
+# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in
+# documentation are documented, even if no documentation was available.
+# Private class members and static file members will be hidden unless
 # the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES
 
 EXTRACT_ALL            = NO
 
-# If the EXTRACT_PRIVATE tag is set to YES all private members of a class 
+# If the EXTRACT_PRIVATE tag is set to YES all private members of a class
 # will be included in the documentation.
 
 EXTRACT_PRIVATE        = NO
 
-# If the EXTRACT_STATIC tag is set to YES all static members of a file 
+# If the EXTRACT_STATIC tag is set to YES all static members of a file
 # will be included in the documentation.
 
 EXTRACT_STATIC         = YES
 
-# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) 
-# defined locally in source files will be included in the documentation. 
+# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs)
+# defined locally in source files will be included in the documentation.
 # If set to NO only classes defined in header files are included.
 
 EXTRACT_LOCAL_CLASSES  = YES
 
-# This flag is only useful for Objective-C code. When set to YES local 
-# methods, which are defined in the implementation section but not in 
-# the interface are included in the documentation. 
+# This flag is only useful for Objective-C code. When set to YES local
+# methods, which are defined in the implementation section but not in
+# the interface are included in the documentation.
 # If set to NO (the default) only methods in the interface are included.
 
 EXTRACT_LOCAL_METHODS  = NO
 
-# If this flag is set to YES, the members of anonymous namespaces will be 
-# extracted and appear in the documentation as a namespace called 
-# 'anonymous_namespace{file}', where file will be replaced with the base 
-# name of the file that contains the anonymous namespace. By default 
+# If this flag is set to YES, the members of anonymous namespaces will be
+# extracted and appear in the documentation as a namespace called
+# 'anonymous_namespace{file}', where file will be replaced with the base
+# name of the file that contains the anonymous namespace. By default
 # anonymous namespace are hidden.
 
 EXTRACT_ANON_NSPACES   = NO
 
-# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all 
-# undocumented members of documented classes, files or namespaces. 
-# If set to NO (the default) these members will be included in the 
-# various overviews, but no documentation section is generated. 
+# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all
+# undocumented members of documented classes, files or namespaces.
+# If set to NO (the default) these members will be included in the
+# various overviews, but no documentation section is generated.
 # This option has no effect if EXTRACT_ALL is enabled.
 
 HIDE_UNDOC_MEMBERS     = NO
 
-# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all 
-# undocumented classes that are normally visible in the class hierarchy. 
-# If set to NO (the default) these classes will be included in the various 
+# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all
+# undocumented classes that are normally visible in the class hierarchy.
+# If set to NO (the default) these classes will be included in the various
 # overviews. This option has no effect if EXTRACT_ALL is enabled.
 
 HIDE_UNDOC_CLASSES     = NO
 
-# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all 
-# friend (class|struct|union) declarations. 
-# If set to NO (the default) these declarations will be included in the 
+# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all
+# friend (class|struct|union) declarations.
+# If set to NO (the default) these declarations will be included in the
 # documentation.
 
 HIDE_FRIEND_COMPOUNDS  = NO
 
-# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any 
-# documentation blocks found inside the body of a function. 
-# If set to NO (the default) these blocks will be appended to the 
+# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any
+# documentation blocks found inside the body of a function.
+# If set to NO (the default) these blocks will be appended to the
 # function's detailed documentation block.
 
 HIDE_IN_BODY_DOCS      = NO
 
-# The INTERNAL_DOCS tag determines if documentation 
-# that is typed after a \internal command is included. If the tag is set 
-# to NO (the default) then the documentation will be excluded. 
+# The INTERNAL_DOCS tag determines if documentation
+# that is typed after a \internal command is included. If the tag is set
+# to NO (the default) then the documentation will be excluded.
 # Set it to YES to include the internal documentation.
 
 INTERNAL_DOCS          = NO
 
-# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate 
-# file names in lower-case letters. If set to YES upper-case letters are also 
-# allowed. This is useful if you have classes or files whose names only differ 
-# in case and if your file system supports case sensitive file names. Windows 
+# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate
+# file names in lower-case letters. If set to YES upper-case letters are also
+# allowed. This is useful if you have classes or files whose names only differ
+# in case and if your file system supports case sensitive file names. Windows
 # and Mac users are advised to set this option to NO.
 
 CASE_SENSE_NAMES       = YES
 
-# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen 
-# will show members with their full class and namespace scopes in the 
+# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen
+# will show members with their full class and namespace scopes in the
 # documentation. If set to YES the scope will be hidden.
 
 HIDE_SCOPE_NAMES       = NO
 
-# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen 
-# will put a list of the files that are included by a file in the documentation 
+# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen
+# will put a list of the files that are included by a file in the documentation
 # of that file.
 
 SHOW_INCLUDE_FILES     = YES
 
-# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] 
+# If the INLINE_INFO tag is set to YES (the default) then a tag [inline]
 # is inserted in the documentation for inline members.
 
 INLINE_INFO            = YES
 
-# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen 
-# will sort the (detailed) documentation of file and class members 
-# alphabetically by member name. If set to NO the members will appear in 
+# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen
+# will sort the (detailed) documentation of file and class members
+# alphabetically by member name. If set to NO the members will appear in
 # declaration order.
 
 SORT_MEMBER_DOCS       = YES
 
-# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the 
-# brief documentation of file, namespace and class members alphabetically 
-# by member name. If set to NO (the default) the members will appear in 
+# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the
+# brief documentation of file, namespace and class members alphabetically
+# by member name. If set to NO (the default) the members will appear in
 # declaration order.
 
 SORT_BRIEF_DOCS        = NO
 
-# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the 
-# hierarchy of group names into alphabetical order. If set to NO (the default) 
+# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the
+# hierarchy of group names into alphabetical order. If set to NO (the default)
 # the group names will appear in their defined order.
 
 SORT_GROUP_NAMES       = NO
 
-# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be 
-# sorted by fully-qualified names, including namespaces. If set to 
-# NO (the default), the class list will be sorted only by class name, 
-# not including the namespace part. 
+# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be
+# sorted by fully-qualified names, including namespaces. If set to
+# NO (the default), the class list will be sorted only by class name,
+# not including the namespace part.
 # Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES.
-# Note: This option applies only to the class list, not to the 
+# Note: This option applies only to the class list, not to the
 # alphabetical list.
 
 SORT_BY_SCOPE_NAME     = NO
 
-# The GENERATE_TODOLIST tag can be used to enable (YES) or 
-# disable (NO) the todo list. This list is created by putting \todo 
+# The GENERATE_TODOLIST tag can be used to enable (YES) or
+# disable (NO) the todo list. This list is created by putting \todo
 # commands in the documentation.
 
 GENERATE_TODOLIST      = YES
 
-# The GENERATE_TESTLIST tag can be used to enable (YES) or 
-# disable (NO) the test list. This list is created by putting \test 
+# The GENERATE_TESTLIST tag can be used to enable (YES) or
+# disable (NO) the test list. This list is created by putting \test
 # commands in the documentation.
 
 GENERATE_TESTLIST      = YES
 
-# The GENERATE_BUGLIST tag can be used to enable (YES) or 
-# disable (NO) the bug list. This list is created by putting \bug 
+# The GENERATE_BUGLIST tag can be used to enable (YES) or
+# disable (NO) the bug list. This list is created by putting \bug
 # commands in the documentation.
 
 GENERATE_BUGLIST       = YES
 
-# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or 
-# disable (NO) the deprecated list. This list is created by putting 
+# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or
+# disable (NO) the deprecated list. This list is created by putting
 # \deprecated commands in the documentation.
 
 GENERATE_DEPRECATEDLIST= YES
 
-# The ENABLED_SECTIONS tag can be used to enable conditional 
+# The ENABLED_SECTIONS tag can be used to enable conditional
 # documentation sections, marked by \if sectionname ... \endif.
 
-ENABLED_SECTIONS       = 
+ENABLED_SECTIONS       =
 
-# The MAX_INITIALIZER_LINES tag determines the maximum number of lines 
-# the initial value of a variable or define consists of for it to appear in 
-# the documentation. If the initializer consists of more lines than specified 
-# here it will be hidden. Use a value of 0 to hide initializers completely. 
-# The appearance of the initializer of individual variables and defines in the 
-# documentation can be controlled using \showinitializer or \hideinitializer 
+# The MAX_INITIALIZER_LINES tag determines the maximum number of lines
+# the initial value of a variable or define consists of for it to appear in
+# the documentation. If the initializer consists of more lines than specified
+# here it will be hidden. Use a value of 0 to hide initializers completely.
+# The appearance of the initializer of individual variables and defines in the
+# documentation can be controlled using \showinitializer or \hideinitializer
 # command in the documentation regardless of this setting.
 
 MAX_INITIALIZER_LINES  = 30
 
-# Set the SHOW_USED_FILES tag to NO to disable the list of files generated 
-# at the bottom of the documentation of classes and structs. If set to YES the 
+# Set the SHOW_USED_FILES tag to NO to disable the list of files generated
+# at the bottom of the documentation of classes and structs. If set to YES the
 # list will mention the files that were used to generate the documentation.
 
 SHOW_USED_FILES        = YES
 
-# If the sources in your project are distributed over multiple directories 
-# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy 
-# in the documentation. The default is NO.
-
-SHOW_DIRECTORIES       = NO
-
 # Set the SHOW_FILES tag to NO to disable the generation of the Files page.
-# This will remove the Files entry from the Quick Index and from the 
+# This will remove the Files entry from the Quick Index and from the
 # Folder Tree View (if specified). The default is YES.
 
 SHOW_FILES             = YES
 
-# Set the SHOW_NAMESPACES tag to NO to disable the generation of the 
+# Set the SHOW_NAMESPACES tag to NO to disable the generation of the
 # Namespaces page.  This will remove the Namespaces entry from the Quick Index
 # and from the Folder Tree View (if specified). The default is YES.
 
 SHOW_NAMESPACES        = YES
 
-# The FILE_VERSION_FILTER tag can be used to specify a program or script that 
-# doxygen should invoke to get the current version for each file (typically from 
-# the version control system). Doxygen will invoke the program by executing (via 
-# popen()) the command <command> <input-file>, where <command> is the value of 
-# the FILE_VERSION_FILTER tag, and <input-file> is the name of an input file 
-# provided by doxygen. Whatever the program writes to standard output 
+# The FILE_VERSION_FILTER tag can be used to specify a program or script that
+# doxygen should invoke to get the current version for each file (typically from
+# the version control system). Doxygen will invoke the program by executing (via
+# popen()) the command <command> <input-file>, where <command> is the value of
+# the FILE_VERSION_FILTER tag, and <input-file> is the name of an input file
+# provided by doxygen. Whatever the program writes to standard output
 # is used as the file version. See the manual for examples.
 
-FILE_VERSION_FILTER    = 
+FILE_VERSION_FILTER    =
 
 #---------------------------------------------------------------------------
 # configuration options related to warning and progress messages
 #---------------------------------------------------------------------------
 
-# The QUIET tag can be used to turn on/off the messages that are generated 
+# The QUIET tag can be used to turn on/off the messages that are generated
 # by doxygen. Possible values are YES and NO. If left blank NO is used.
 
 QUIET                  = NO
 
-# The WARNINGS tag can be used to turn on/off the warning messages that are 
-# generated by doxygen. Possible values are YES and NO. If left blank 
+# The WARNINGS tag can be used to turn on/off the warning messages that are
+# generated by doxygen. Possible values are YES and NO. If left blank
 # NO is used.
 
 WARNINGS               = YES
 
-# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings 
-# for undocumented members. If EXTRACT_ALL is set to YES then this flag will 
+# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings
+# for undocumented members. If EXTRACT_ALL is set to YES then this flag will
 # automatically be disabled.
 
 WARN_IF_UNDOCUMENTED   = YES
 
-# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for 
-# potential errors in the documentation, such as not documenting some 
-# parameters in a documented function, or documenting parameters that 
+# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for
+# potential errors in the documentation, such as not documenting some
+# parameters in a documented function, or documenting parameters that
 # don't exist or using markup commands wrongly.
 
 WARN_IF_DOC_ERROR      = YES
 
-# This WARN_NO_PARAMDOC option can be abled to get warnings for 
-# functions that are documented, but have no documentation for their parameters 
-# or return value. If set to NO (the default) doxygen will only warn about 
-# wrong or incomplete parameter documentation, but not about the absence of 
+# This WARN_NO_PARAMDOC option can be abled to get warnings for
+# functions that are documented, but have no documentation for their parameters
+# or return value. If set to NO (the default) doxygen will only warn about
+# wrong or incomplete parameter documentation, but not about the absence of
 # documentation.
 
 WARN_NO_PARAMDOC       = NO
 
-# The WARN_FORMAT tag determines the format of the warning messages that 
-# doxygen can produce. The string should contain the $file, $line, and $text 
-# tags, which will be replaced by the file and line number from which the 
-# warning originated and the warning text. Optionally the format may contain 
-# $version, which will be replaced by the version of the file (if it could 
+# The WARN_FORMAT tag determines the format of the warning messages that
+# doxygen can produce. The string should contain the $file, $line, and $text
+# tags, which will be replaced by the file and line number from which the
+# warning originated and the warning text. Optionally the format may contain
+# $version, which will be replaced by the version of the file (if it could
 # be obtained via FILE_VERSION_FILTER)
 
 WARN_FORMAT            = "$file:$line: $text"
 
-# The WARN_LOGFILE tag can be used to specify a file to which warning 
-# and error messages should be written. If left blank the output is written 
+# The WARN_LOGFILE tag can be used to specify a file to which warning
+# and error messages should be written. If left blank the output is written
 # to stderr.
 
-WARN_LOGFILE           = 
+WARN_LOGFILE           =
 
 #---------------------------------------------------------------------------
 # configuration options related to the input files
 #---------------------------------------------------------------------------
 
-# The INPUT tag can be used to specify the files and/or directories that contain 
-# documented source files. You may enter file names like "myfile.cpp" or 
-# directories like "/usr/src/myproject". Separate the files or directories 
+# The INPUT tag can be used to specify the files and/or directories that contain
+# documented source files. You may enter file names like "myfile.cpp" or
+# directories like "/usr/src/myproject". Separate the files or directories
 # with spaces.
 
-INPUT                  = @top_srcdir@/src/common \
-                         @top_srcdir@/src/or
+INPUT                  = @top_srcdir@/src/lib \
+		         @top_srcdir@/src/core \
+ 		         @top_srcdir@/src/feature \
+ 		         @top_srcdir@/src/app
 
-# This tag can be used to specify the character encoding of the source files 
-# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is 
-# also the default input encoding. Doxygen uses libiconv (or the iconv built 
-# into libc) for the transcoding. See http://www.gnu.org/software/libiconv for 
+# This tag can be used to specify the character encoding of the source files
+# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is
+# also the default input encoding. Doxygen uses libiconv (or the iconv built
+# into libc) for the transcoding. See http://www.gnu.org/software/libiconv for
 # the list of possible encodings.
 
 INPUT_ENCODING         = UTF-8
 
-# If the value of the INPUT tag contains directories, you can use the 
-# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp 
-# and *.h) to filter out the source-files in the directories. If left 
-# blank the following patterns are tested: 
-# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx 
+# If the value of the INPUT tag contains directories, you can use the
+# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp
+# and *.h) to filter out the source-files in the directories. If left
+# blank the following patterns are tested:
+# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx
 # *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90
 
 FILE_PATTERNS          = *.c \
                          *.h
 
-# The RECURSIVE tag can be used to turn specify whether or not subdirectories 
-# should be searched for input files as well. Possible values are YES and NO. 
+# The RECURSIVE tag can be used to turn specify whether or not subdirectories
+# should be searched for input files as well. Possible values are YES and NO.
 # If left blank NO is used.
 
-RECURSIVE              = NO
+RECURSIVE              = YES
 
-# The EXCLUDE tag can be used to specify files and/or directories that should 
-# excluded from the INPUT source files. This way you can easily exclude a 
+# The EXCLUDE tag can be used to specify files and/or directories that should
+# excluded from the INPUT source files. This way you can easily exclude a
 # subdirectory from a directory tree whose root is specified with the INPUT tag.
 
 EXCLUDE                = tree.h
 
-# The EXCLUDE_SYMLINKS tag can be used select whether or not files or 
-# directories that are symbolic links (a Unix filesystem feature) are excluded 
+# The EXCLUDE_SYMLINKS tag can be used select whether or not files or
+# directories that are symbolic links (a Unix filesystem feature) are excluded
 # from the input.
 
 EXCLUDE_SYMLINKS       = NO
 
-# If the value of the INPUT tag contains directories, you can use the 
-# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude 
-# certain files from those directories. Note that the wildcards are matched 
-# against the file with absolute path, so to exclude all test directories 
+# If the value of the INPUT tag contains directories, you can use the
+# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude
+# certain files from those directories. Note that the wildcards are matched
+# against the file with absolute path, so to exclude all test directories
 # for example use the pattern */test/*
 
-EXCLUDE_PATTERNS       = 
+EXCLUDE_PATTERNS       =
 
-# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names 
-# (namespaces, classes, functions, etc.) that should be excluded from the 
-# output. The symbol name can be a fully qualified name, a word, or if the 
-# wildcard * is used, a substring. Examples: ANamespace, AClass, 
+# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
+# (namespaces, classes, functions, etc.) that should be excluded from the
+# output. The symbol name can be a fully qualified name, a word, or if the
+# wildcard * is used, a substring. Examples: ANamespace, AClass,
 # AClass::ANamespace, ANamespace::*Test
 
-EXCLUDE_SYMBOLS        = 
+EXCLUDE_SYMBOLS        =
 
-# The EXAMPLE_PATH tag can be used to specify one or more files or 
-# directories that contain example code fragments that are included (see 
+# The EXAMPLE_PATH tag can be used to specify one or more files or
+# directories that contain example code fragments that are included (see
 # the \include command).
 
-EXAMPLE_PATH           = 
+EXAMPLE_PATH           =
 
-# If the value of the EXAMPLE_PATH tag contains directories, you can use the 
-# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp 
-# and *.h) to filter out the source-files in the directories. If left 
+# If the value of the EXAMPLE_PATH tag contains directories, you can use the
+# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp
+# and *.h) to filter out the source-files in the directories. If left
 # blank all files are included.
 
-EXAMPLE_PATTERNS       = 
+EXAMPLE_PATTERNS       =
 
-# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be 
-# searched for input files to be used with the \include or \dontinclude 
-# commands irrespective of the value of the RECURSIVE tag. 
+# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be
+# searched for input files to be used with the \include or \dontinclude
+# commands irrespective of the value of the RECURSIVE tag.
 # Possible values are YES and NO. If left blank NO is used.
 
 EXAMPLE_RECURSIVE      = NO
 
-# The IMAGE_PATH tag can be used to specify one or more files or 
-# directories that contain image that are included in the documentation (see 
+# The IMAGE_PATH tag can be used to specify one or more files or
+# directories that contain image that are included in the documentation (see
 # the \image command).
 
-IMAGE_PATH             = 
+IMAGE_PATH             =
 
-# The INPUT_FILTER tag can be used to specify a program that doxygen should 
-# invoke to filter for each input file. Doxygen will invoke the filter program 
-# by executing (via popen()) the command <filter> <input-file>, where <filter> 
-# is the value of the INPUT_FILTER tag, and <input-file> is the name of an 
-# input file. Doxygen will then use the output that the filter program writes 
-# to standard output.  If FILTER_PATTERNS is specified, this tag will be 
+# The INPUT_FILTER tag can be used to specify a program that doxygen should
+# invoke to filter for each input file. Doxygen will invoke the filter program
+# by executing (via popen()) the command <filter> <input-file>, where <filter>
+# is the value of the INPUT_FILTER tag, and <input-file> is the name of an
+# input file. Doxygen will then use the output that the filter program writes
+# to standard output.  If FILTER_PATTERNS is specified, this tag will be
 # ignored.
 
-INPUT_FILTER           = 
+INPUT_FILTER           =
 
-# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern 
-# basis.  Doxygen will compare the file name with each pattern and apply the 
-# filter if there is a match.  The filters are a list of the form: 
-# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further 
-# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER 
+# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern
+# basis.  Doxygen will compare the file name with each pattern and apply the
+# filter if there is a match.  The filters are a list of the form:
+# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further
+# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER
 # is applied to all files.
 
-FILTER_PATTERNS        = 
+FILTER_PATTERNS        =
 
-# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using 
-# INPUT_FILTER) will be used to filter the input files when producing source 
+# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using
+# INPUT_FILTER) will be used to filter the input files when producing source
 # files to browse (i.e. when SOURCE_BROWSER is set to YES).
 
 FILTER_SOURCE_FILES    = NO
@@ -644,32 +640,32 @@ FILTER_SOURCE_FILES    = NO
 # configuration options related to source browsing
 #---------------------------------------------------------------------------
 
-# If the SOURCE_BROWSER tag is set to YES then a list of source files will 
-# be generated. Documented entities will be cross-referenced with these sources. 
-# Note: To get rid of all source code in the generated output, make sure also 
+# If the SOURCE_BROWSER tag is set to YES then a list of source files will
+# be generated. Documented entities will be cross-referenced with these sources.
+# Note: To get rid of all source code in the generated output, make sure also
 # VERBATIM_HEADERS is set to NO.
 
 SOURCE_BROWSER         = YES
 
-# Setting the INLINE_SOURCES tag to YES will include the body 
+# Setting the INLINE_SOURCES tag to YES will include the body
 # of functions and classes directly in the documentation.
 
 INLINE_SOURCES         = NO
 
-# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct 
-# doxygen to hide any special comment blocks from generated source code 
+# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct
+# doxygen to hide any special comment blocks from generated source code
 # fragments. Normal C and C++ comments will always remain visible.
 
 STRIP_CODE_COMMENTS    = YES
 
-# If the REFERENCED_BY_RELATION tag is set to YES 
-# then for each documented function all documented 
+# If the REFERENCED_BY_RELATION tag is set to YES
+# then for each documented function all documented
 # functions referencing it will be listed.
 
 REFERENCED_BY_RELATION = YES
 
-# If the REFERENCES_RELATION tag is set to YES 
-# then for each documented function all documented entities 
+# If the REFERENCES_RELATION tag is set to YES
+# then for each documented function all documented entities
 # called/used by that function will be listed.
 
 REFERENCES_RELATION    = YES
@@ -681,16 +677,16 @@ REFERENCES_RELATION    = YES
 
 REFERENCES_LINK_SOURCE = YES
 
-# If the USE_HTAGS tag is set to YES then the references to source code 
-# will point to the HTML generated by the htags(1) tool instead of doxygen 
-# built-in source browser. The htags tool is part of GNU's global source 
-# tagging system (see http://www.gnu.org/software/global/global.html). You 
+# If the USE_HTAGS tag is set to YES then the references to source code
+# will point to the HTML generated by the htags(1) tool instead of doxygen
+# built-in source browser. The htags tool is part of GNU's global source
+# tagging system (see http://www.gnu.org/software/global/global.html). You
 # will need version 4.8.6 or higher.
 
 USE_HTAGS              = NO
 
-# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen 
-# will generate a verbatim copy of the header file for each class for 
+# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen
+# will generate a verbatim copy of the header file for each class for
 # which an include is specified. Set to NO to disable this.
 
 VERBATIM_HEADERS       = YES
@@ -699,129 +695,123 @@ VERBATIM_HEADERS       = YES
 # configuration options related to the alphabetical class index
 #---------------------------------------------------------------------------
 
-# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index 
-# of all compounds will be generated. Enable this if the project 
+# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index
+# of all compounds will be generated. Enable this if the project
 # contains a lot of classes, structs, unions or interfaces.
 
 ALPHABETICAL_INDEX     = NO
 
-# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then 
-# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns 
+# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then
+# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns
 # in which this list will be split (can be a number in the range [1..20])
 
 COLS_IN_ALPHA_INDEX    = 5
 
-# In case all classes in a project start with a common prefix, all 
-# classes will be put under the same header in the alphabetical index. 
-# The IGNORE_PREFIX tag can be used to specify one or more prefixes that 
+# In case all classes in a project start with a common prefix, all
+# classes will be put under the same header in the alphabetical index.
+# The IGNORE_PREFIX tag can be used to specify one or more prefixes that
 # should be ignored while generating the index headers.
 
-IGNORE_PREFIX          = 
+IGNORE_PREFIX          =
 
 #---------------------------------------------------------------------------
 # configuration options related to the HTML output
 #---------------------------------------------------------------------------
 
-# If the GENERATE_HTML tag is set to YES (the default) Doxygen will 
+# If the GENERATE_HTML tag is set to YES (the default) Doxygen will
 # generate HTML output.
 
 GENERATE_HTML          = YES
 
-# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. 
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be 
+# The HTML_OUTPUT tag is used to specify where the HTML docs will be put.
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be
 # put in front of it. If left blank `html' will be used as the default path.
 
 HTML_OUTPUT            = html
 
-# The HTML_FILE_EXTENSION tag can be used to specify the file extension for 
-# each generated HTML page (for example: .htm,.php,.asp). If it is left blank 
+# The HTML_FILE_EXTENSION tag can be used to specify the file extension for
+# each generated HTML page (for example: .htm,.php,.asp). If it is left blank
 # doxygen will generate files with .html extension.
 
 HTML_FILE_EXTENSION    = .html
 
-# The HTML_HEADER tag can be used to specify a personal HTML header for 
-# each generated HTML page. If it is left blank doxygen will generate a 
+# The HTML_HEADER tag can be used to specify a personal HTML header for
+# each generated HTML page. If it is left blank doxygen will generate a
 # standard header.
 
-HTML_HEADER            = 
+HTML_HEADER            =
 
-# The HTML_FOOTER tag can be used to specify a personal HTML footer for 
-# each generated HTML page. If it is left blank doxygen will generate a 
+# The HTML_FOOTER tag can be used to specify a personal HTML footer for
+# each generated HTML page. If it is left blank doxygen will generate a
 # standard footer.
 
-HTML_FOOTER            = 
+HTML_FOOTER            =
 
-# The HTML_STYLESHEET tag can be used to specify a user-defined cascading 
-# style sheet that is used by each HTML page. It can be used to 
-# fine-tune the look of the HTML output. If the tag is left blank doxygen 
-# will generate a default style sheet. Note that doxygen will try to copy 
-# the style sheet file to the HTML output directory, so don't put your own 
+# The HTML_STYLESHEET tag can be used to specify a user-defined cascading
+# style sheet that is used by each HTML page. It can be used to
+# fine-tune the look of the HTML output. If the tag is left blank doxygen
+# will generate a default style sheet. Note that doxygen will try to copy
+# the style sheet file to the HTML output directory, so don't put your own
 # stylesheet in the HTML output directory as well, or it will be erased!
 
-HTML_STYLESHEET        = 
-
-# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, 
-# files or namespaces will be aligned in HTML using tables. If set to 
-# NO a bullet list will be used.
-
-HTML_ALIGN_MEMBERS     = YES
+HTML_STYLESHEET        =
 
-# If the GENERATE_HTMLHELP tag is set to YES, additional index files 
-# will be generated that can be used as input for tools like the 
-# Microsoft HTML help workshop to generate a compiled HTML help file (.chm) 
+# If the GENERATE_HTMLHELP tag is set to YES, additional index files
+# will be generated that can be used as input for tools like the
+# Microsoft HTML help workshop to generate a compiled HTML help file (.chm)
 # of the generated HTML documentation.
 
 GENERATE_HTMLHELP      = NO
 
-# If the GENERATE_DOCSET tag is set to YES, additional index files 
-# will be generated that can be used as input for Apple's Xcode 3 
-# integrated development environment, introduced with OSX 10.5 (Leopard). 
-# To create a documentation set, doxygen will generate a Makefile in the 
-# HTML output directory. Running make will produce the docset in that 
-# directory and running "make install" will install the docset in 
-# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find 
+# If the GENERATE_DOCSET tag is set to YES, additional index files
+# will be generated that can be used as input for Apple's Xcode 3
+# integrated development environment, introduced with OSX 10.5 (Leopard).
+# To create a documentation set, doxygen will generate a Makefile in the
+# HTML output directory. Running make will produce the docset in that
+# directory and running "make install" will install the docset in
+# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find
 # it at startup.
 
 GENERATE_DOCSET        = NO
 
-# When GENERATE_DOCSET tag is set to YES, this tag determines the name of the 
-# feed. A documentation feed provides an umbrella under which multiple 
-# documentation sets from a single provider (such as a company or product suite) 
+# When GENERATE_DOCSET tag is set to YES, this tag determines the name of the
+# feed. A documentation feed provides an umbrella under which multiple
+# documentation sets from a single provider (such as a company or product suite)
 # can be grouped.
 
 DOCSET_FEEDNAME        = "Doxygen generated docs for Tor"
 
-# When GENERATE_DOCSET tag is set to YES, this tag specifies a string that 
-# should uniquely identify the documentation set bundle. This should be a 
-# reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen 
+# When GENERATE_DOCSET tag is set to YES, this tag specifies a string that
+# should uniquely identify the documentation set bundle. This should be a
+# reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen
 # will append .docset to the name.
 
 DOCSET_BUNDLE_ID       = org.torproject.Tor
 
-# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML 
-# documentation will contain sections that can be hidden and shown after the 
-# page has loaded. For this to work a browser that supports 
-# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox 
+# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
+# documentation will contain sections that can be hidden and shown after the
+# page has loaded. For this to work a browser that supports
+# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox
 # Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari).
 
 HTML_DYNAMIC_SECTIONS  = NO
 
-# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can 
-# be used to specify the file name of the resulting .chm file. You 
-# can add a path in front of the file if the result should not be 
+# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can
+# be used to specify the file name of the resulting .chm file. You
+# can add a path in front of the file if the result should not be
 # written to the html output directory.
 
-CHM_FILE               = 
+CHM_FILE               =
 
-# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can 
-# be used to specify the location (absolute path including file name) of 
-# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run 
+# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can
+# be used to specify the location (absolute path including file name) of
+# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run
 # the HTML help compiler on the generated index.hhp.
 
-HHC_LOCATION           = 
+HHC_LOCATION           =
 
-# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag 
-# controls if a separate .chi index file is generated (YES) or that 
+# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag
+# controls if a separate .chi index file is generated (YES) or that
 # it should be included in the master .chm file (NO).
 
 GENERATE_CHI           = NO
@@ -830,26 +820,26 @@ GENERATE_CHI           = NO
 # is used to encode HtmlHelp index (hhk), content (hhc) and project file
 # content.
 
-CHM_INDEX_ENCODING     = 
+CHM_INDEX_ENCODING     =
 
-# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag 
-# controls whether a binary table of contents is generated (YES) or a 
+# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag
+# controls whether a binary table of contents is generated (YES) or a
 # normal table of contents (NO) in the .chm file.
 
 BINARY_TOC             = NO
 
-# The TOC_EXPAND flag can be set to YES to add extra items for group members 
+# The TOC_EXPAND flag can be set to YES to add extra items for group members
 # to the contents of the HTML help documentation and to the tree view.
 
 TOC_EXPAND             = NO
 
-# The DISABLE_INDEX tag can be used to turn on/off the condensed index at 
-# top of each HTML page. The value NO (the default) enables the index and 
+# The DISABLE_INDEX tag can be used to turn on/off the condensed index at
+# top of each HTML page. The value NO (the default) enables the index and
 # the value YES disables it.
 
 DISABLE_INDEX          = NO
 
-# This tag can be used to set the number of enum values (range [1..20]) 
+# This tag can be used to set the number of enum values (range [1..20])
 # that doxygen will group on one line in the generated HTML documentation.
 
 ENUM_VALUES_PER_LINE   = 4
@@ -857,11 +847,11 @@ ENUM_VALUES_PER_LINE   = 4
 # The GENERATE_TREEVIEW tag is used to specify whether a tree-like index
 # structure should be generated to display hierarchical information.
 # If the tag value is set to FRAME, a side panel will be generated
-# containing a tree-like index structure (just like the one that 
-# is generated for HTML Help). For this to work a browser that supports 
-# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, 
-# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are 
-# probably better off using the HTML help feature. Other possible values 
+# containing a tree-like index structure (just like the one that
+# is generated for HTML Help). For this to work a browser that supports
+# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+,
+# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are
+# probably better off using the HTML help feature. Other possible values
 # for this tag are: HIERARCHIES, which will generate the Groups, Directories,
 # and Class Hiererachy pages using a tree view instead of an ordered list;
 # ALL, which combines the behavior of FRAME and HIERARCHIES; and NONE, which
@@ -871,16 +861,16 @@ ENUM_VALUES_PER_LINE   = 4
 
 GENERATE_TREEVIEW      = NO
 
-# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be 
-# used to set the initial width (in pixels) of the frame in which the tree 
+# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be
+# used to set the initial width (in pixels) of the frame in which the tree
 # is shown.
 
 TREEVIEW_WIDTH         = 250
 
-# Use this tag to change the font size of Latex formulas included 
-# as images in the HTML documentation. The default is 10. Note that 
-# when you change the font size after a successful doxygen run you need 
-# to manually remove any form_*.png images from the HTML output directory 
+# Use this tag to change the font size of Latex formulas included
+# as images in the HTML documentation. The default is 10. Note that
+# when you change the font size after a successful doxygen run you need
+# to manually remove any form_*.png images from the HTML output directory
 # to force them to be regenerated.
 
 FORMULA_FONTSIZE       = 10
@@ -889,74 +879,74 @@ FORMULA_FONTSIZE       = 10
 # configuration options related to the LaTeX output
 #---------------------------------------------------------------------------
 
-# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will 
+# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will
 # generate Latex output.
 
 GENERATE_LATEX         = YES
 
-# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. 
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be 
+# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put.
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be
 # put in front of it. If left blank `latex' will be used as the default path.
 
 LATEX_OUTPUT           = latex
 
-# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be 
+# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be
 # invoked. If left blank `latex' will be used as the default command name.
 
 LATEX_CMD_NAME         = latex
 
-# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to 
-# generate index for LaTeX. If left blank `makeindex' will be used as the 
+# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to
+# generate index for LaTeX. If left blank `makeindex' will be used as the
 # default command name.
 
 MAKEINDEX_CMD_NAME     = makeindex
 
-# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact 
-# LaTeX documents. This may be useful for small projects and may help to 
+# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact
+# LaTeX documents. This may be useful for small projects and may help to
 # save some trees in general.
 
 COMPACT_LATEX          = NO
 
-# The PAPER_TYPE tag can be used to set the paper type that is used 
-# by the printer. Possible values are: a4, a4wide, letter, legal and 
+# The PAPER_TYPE tag can be used to set the paper type that is used
+# by the printer. Possible values are: a4, a4wide, letter, legal and
 # executive. If left blank a4wide will be used.
 
 PAPER_TYPE             = a4wide
 
-# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX 
+# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX
 # packages that should be included in the LaTeX output.
 
-EXTRA_PACKAGES         = 
+EXTRA_PACKAGES         =
 
-# The LATEX_HEADER tag can be used to specify a personal LaTeX header for 
-# the generated latex document. The header should contain everything until 
-# the first chapter. If it is left blank doxygen will generate a 
+# The LATEX_HEADER tag can be used to specify a personal LaTeX header for
+# the generated latex document. The header should contain everything until
+# the first chapter. If it is left blank doxygen will generate a
 # standard header. Notice: only use this tag if you know what you are doing!
 
-LATEX_HEADER           = 
+LATEX_HEADER           =
 
-# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated 
-# is prepared for conversion to pdf (using ps2pdf). The pdf file will 
-# contain links (just like the HTML output) instead of page references 
+# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated
+# is prepared for conversion to pdf (using ps2pdf). The pdf file will
+# contain links (just like the HTML output) instead of page references
 # This makes the output suitable for online browsing using a pdf viewer.
 
 PDF_HYPERLINKS         = NO
 
-# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of 
-# plain latex in the generated Makefile. Set this option to YES to get a 
+# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of
+# plain latex in the generated Makefile. Set this option to YES to get a
 # higher quality PDF documentation.
 
 USE_PDFLATEX           = NO
 
-# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. 
-# command to the generated LaTeX files. This will instruct LaTeX to keep 
-# running if errors occur, instead of asking the user for help. 
+# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode.
+# command to the generated LaTeX files. This will instruct LaTeX to keep
+# running if errors occur, instead of asking the user for help.
 # This option is also used when generating formulas in HTML.
 
 LATEX_BATCHMODE        = NO
 
-# If LATEX_HIDE_INDICES is set to YES then doxygen will not 
-# include the index chapters (such as File Index, Compound Index, etc.) 
+# If LATEX_HIDE_INDICES is set to YES then doxygen will not
+# include the index chapters (such as File Index, Compound Index, etc.)
 # in the output.
 
 LATEX_HIDE_INDICES     = NO
@@ -965,68 +955,68 @@ LATEX_HIDE_INDICES     = NO
 # configuration options related to the RTF output
 #---------------------------------------------------------------------------
 
-# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output 
-# The RTF output is optimized for Word 97 and may not look very pretty with 
+# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output
+# The RTF output is optimized for Word 97 and may not look very pretty with
 # other RTF readers or editors.
 
 GENERATE_RTF           = NO
 
-# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. 
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be 
+# The RTF_OUTPUT tag is used to specify where the RTF docs will be put.
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be
 # put in front of it. If left blank `rtf' will be used as the default path.
 
 RTF_OUTPUT             = rtf
 
-# If the COMPACT_RTF tag is set to YES Doxygen generates more compact 
-# RTF documents. This may be useful for small projects and may help to 
+# If the COMPACT_RTF tag is set to YES Doxygen generates more compact
+# RTF documents. This may be useful for small projects and may help to
 # save some trees in general.
 
 COMPACT_RTF            = NO
 
-# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated 
-# will contain hyperlink fields. The RTF file will 
-# contain links (just like the HTML output) instead of page references. 
-# This makes the output suitable for online browsing using WORD or other 
-# programs which support those fields. 
+# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated
+# will contain hyperlink fields. The RTF file will
+# contain links (just like the HTML output) instead of page references.
+# This makes the output suitable for online browsing using WORD or other
+# programs which support those fields.
 # Note: wordpad (write) and others do not support links.
 
 RTF_HYPERLINKS         = NO
 
-# Load stylesheet definitions from file. Syntax is similar to doxygen's 
-# config file, i.e. a series of assignments. You only have to provide 
+# Load stylesheet definitions from file. Syntax is similar to doxygen's
+# config file, i.e. a series of assignments. You only have to provide
 # replacements, missing definitions are set to their default value.
 
-RTF_STYLESHEET_FILE    = 
+RTF_STYLESHEET_FILE    =
 
-# Set optional variables used in the generation of an rtf document. 
+# Set optional variables used in the generation of an rtf document.
 # Syntax is similar to doxygen's config file.
 
-RTF_EXTENSIONS_FILE    = 
+RTF_EXTENSIONS_FILE    =
 
 #---------------------------------------------------------------------------
 # configuration options related to the man page output
 #---------------------------------------------------------------------------
 
-# If the GENERATE_MAN tag is set to YES (the default) Doxygen will 
+# If the GENERATE_MAN tag is set to YES (the default) Doxygen will
 # generate man pages
 
 GENERATE_MAN           = NO
 
-# The MAN_OUTPUT tag is used to specify where the man pages will be put. 
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be 
+# The MAN_OUTPUT tag is used to specify where the man pages will be put.
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be
 # put in front of it. If left blank `man' will be used as the default path.
 
 MAN_OUTPUT             = man
 
-# The MAN_EXTENSION tag determines the extension that is added to 
+# The MAN_EXTENSION tag determines the extension that is added to
 # the generated man pages (default is the subroutine's section .3)
 
 MAN_EXTENSION          = .3
 
-# If the MAN_LINKS tag is set to YES and Doxygen generates man output, 
-# then it will generate one additional man file for each entity 
-# documented in the real man page(s). These additional files 
-# only source the real man page, but without them the man command 
+# If the MAN_LINKS tag is set to YES and Doxygen generates man output,
+# then it will generate one additional man file for each entity
+# documented in the real man page(s). These additional files
+# only source the real man page, but without them the man command
 # would be unable to find the correct page. The default is NO.
 
 MAN_LINKS              = NO
@@ -1035,33 +1025,21 @@ MAN_LINKS              = NO
 # configuration options related to the XML output
 #---------------------------------------------------------------------------
 
-# If the GENERATE_XML tag is set to YES Doxygen will 
-# generate an XML file that captures the structure of 
+# If the GENERATE_XML tag is set to YES Doxygen will
+# generate an XML file that captures the structure of
 # the code including all documentation.
 
 GENERATE_XML           = NO
 
-# The XML_OUTPUT tag is used to specify where the XML pages will be put. 
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be 
+# The XML_OUTPUT tag is used to specify where the XML pages will be put.
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be
 # put in front of it. If left blank `xml' will be used as the default path.
 
 XML_OUTPUT             = xml
 
-# The XML_SCHEMA tag can be used to specify an XML schema, 
-# which can be used by a validating XML parser to check the 
-# syntax of the XML files.
-
-XML_SCHEMA             = 
-
-# The XML_DTD tag can be used to specify an XML DTD, 
-# which can be used by a validating XML parser to check the 
-# syntax of the XML files.
-
-XML_DTD                = 
-
-# If the XML_PROGRAMLISTING tag is set to YES Doxygen will 
-# dump the program listings (including syntax highlighting 
-# and cross-referencing information) to the XML output. Note that 
+# If the XML_PROGRAMLISTING tag is set to YES Doxygen will
+# dump the program listings (including syntax highlighting
+# and cross-referencing information) to the XML output. Note that
 # enabling this will significantly increase the size of the XML output.
 
 XML_PROGRAMLISTING     = YES
@@ -1070,10 +1048,10 @@ XML_PROGRAMLISTING     = YES
 # configuration options for the AutoGen Definitions output
 #---------------------------------------------------------------------------
 
-# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will 
-# generate an AutoGen Definitions (see autogen.sf.net) file 
-# that captures the structure of the code including all 
-# documentation. Note that this feature is still experimental 
+# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will
+# generate an AutoGen Definitions (see autogen.sf.net) file
+# that captures the structure of the code including all
+# documentation. Note that this feature is still experimental
 # and incomplete at the moment.
 
 GENERATE_AUTOGEN_DEF   = NO
@@ -1082,338 +1060,338 @@ GENERATE_AUTOGEN_DEF   = NO
 # configuration options related to the Perl module output
 #---------------------------------------------------------------------------
 
-# If the GENERATE_PERLMOD tag is set to YES Doxygen will 
-# generate a Perl module file that captures the structure of 
-# the code including all documentation. Note that this 
-# feature is still experimental and incomplete at the 
+# If the GENERATE_PERLMOD tag is set to YES Doxygen will
+# generate a Perl module file that captures the structure of
+# the code including all documentation. Note that this
+# feature is still experimental and incomplete at the
 # moment.
 
 GENERATE_PERLMOD       = NO
 
-# If the PERLMOD_LATEX tag is set to YES Doxygen will generate 
-# the necessary Makefile rules, Perl scripts and LaTeX code to be able 
+# If the PERLMOD_LATEX tag is set to YES Doxygen will generate
+# the necessary Makefile rules, Perl scripts and LaTeX code to be able
 # to generate PDF and DVI output from the Perl module output.
 
 PERLMOD_LATEX          = NO
 
-# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be 
-# nicely formatted so it can be parsed by a human reader.  This is useful 
-# if you want to understand what is going on.  On the other hand, if this 
-# tag is set to NO the size of the Perl module output will be much smaller 
+# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be
+# nicely formatted so it can be parsed by a human reader.  This is useful
+# if you want to understand what is going on.  On the other hand, if this
+# tag is set to NO the size of the Perl module output will be much smaller
 # and Perl will parse it just the same.
 
 PERLMOD_PRETTY         = YES
 
-# The names of the make variables in the generated doxyrules.make file 
-# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. 
-# This is useful so different doxyrules.make files included by the same 
+# The names of the make variables in the generated doxyrules.make file
+# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX.
+# This is useful so different doxyrules.make files included by the same
 # Makefile don't overwrite each other's variables.
 
-PERLMOD_MAKEVAR_PREFIX = 
+PERLMOD_MAKEVAR_PREFIX =
 
 #---------------------------------------------------------------------------
-# Configuration options related to the preprocessor   
+# Configuration options related to the preprocessor
 #---------------------------------------------------------------------------
 
-# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will 
-# evaluate all C-preprocessor directives found in the sources and include 
+# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will
+# evaluate all C-preprocessor directives found in the sources and include
 # files.
 
 ENABLE_PREPROCESSING   = YES
 
-# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro 
-# names in the source code. If set to NO (the default) only conditional 
-# compilation will be performed. Macro expansion can be done in a controlled 
+# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro
+# names in the source code. If set to NO (the default) only conditional
+# compilation will be performed. Macro expansion can be done in a controlled
 # way by setting EXPAND_ONLY_PREDEF to YES.
 
 MACRO_EXPANSION        = NO
 
-# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES 
-# then the macro expansion is limited to the macros specified with the 
+# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES
+# then the macro expansion is limited to the macros specified with the
 # PREDEFINED and EXPAND_AS_DEFINED tags.
 
 EXPAND_ONLY_PREDEF     = NO
 
-# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files 
+# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files
 # in the INCLUDE_PATH (see below) will be search if a #include is found.
 
 SEARCH_INCLUDES        = YES
 
-# The INCLUDE_PATH tag can be used to specify one or more directories that 
-# contain include files that are not input files but should be processed by 
+# The INCLUDE_PATH tag can be used to specify one or more directories that
+# contain include files that are not input files but should be processed by
 # the preprocessor.
 
-INCLUDE_PATH           = 
+INCLUDE_PATH           =
 
-# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard 
-# patterns (like *.h and *.hpp) to filter out the header-files in the 
-# directories. If left blank, the patterns specified with FILE_PATTERNS will 
+# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard
+# patterns (like *.h and *.hpp) to filter out the header-files in the
+# directories. If left blank, the patterns specified with FILE_PATTERNS will
 # be used.
 
-INCLUDE_FILE_PATTERNS  = 
+INCLUDE_FILE_PATTERNS  =
 
-# The PREDEFINED tag can be used to specify one or more macro names that 
-# are defined before the preprocessor is started (similar to the -D option of 
-# gcc). The argument of the tag is a list of macros of the form: name 
-# or name=definition (no spaces). If the definition and the = are 
-# omitted =1 is assumed. To prevent a macro definition from being 
-# undefined via #undef or recursively expanded use the := operator 
+# The PREDEFINED tag can be used to specify one or more macro names that
+# are defined before the preprocessor is started (similar to the -D option of
+# gcc). The argument of the tag is a list of macros of the form: name
+# or name=definition (no spaces). If the definition and the = are
+# omitted =1 is assumed. To prevent a macro definition from being
+# undefined via #undef or recursively expanded use the := operator
 # instead of the = operator.
 
-PREDEFINED             = 
+PREDEFINED             =
 
-# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then 
-# this tag can be used to specify a list of macro names that should be expanded. 
-# The macro definition that is found in the sources will be used. 
+# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then
+# this tag can be used to specify a list of macro names that should be expanded.
+# The macro definition that is found in the sources will be used.
 # Use the PREDEFINED tag if you want to use a different macro definition.
 
-EXPAND_AS_DEFINED      = 
+EXPAND_AS_DEFINED      =
 
-# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then 
-# doxygen's preprocessor will remove all function-like macros that are alone 
-# on a line, have an all uppercase name, and do not end with a semicolon. Such 
-# function macros are typically used for boiler-plate code, and will confuse 
+# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then
+# doxygen's preprocessor will remove all function-like macros that are alone
+# on a line, have an all uppercase name, and do not end with a semicolon. Such
+# function macros are typically used for boiler-plate code, and will confuse
 # the parser if not removed.
 
 SKIP_FUNCTION_MACROS   = YES
 
 #---------------------------------------------------------------------------
-# Configuration::additions related to external references   
+# Configuration::additions related to external references
 #---------------------------------------------------------------------------
 
-# The TAGFILES option can be used to specify one or more tagfiles. 
-# Optionally an initial location of the external documentation 
-# can be added for each tagfile. The format of a tag file without 
-# this location is as follows: 
-#   TAGFILES = file1 file2 ... 
-# Adding location for the tag files is done as follows: 
-#   TAGFILES = file1=loc1 "file2 = loc2" ... 
-# where "loc1" and "loc2" can be relative or absolute paths or 
-# URLs. If a location is present for each tag, the installdox tool 
+# The TAGFILES option can be used to specify one or more tagfiles.
+# Optionally an initial location of the external documentation
+# can be added for each tagfile. The format of a tag file without
+# this location is as follows:
+#   TAGFILES = file1 file2 ...
+# Adding location for the tag files is done as follows:
+#   TAGFILES = file1=loc1 "file2 = loc2" ...
+# where "loc1" and "loc2" can be relative or absolute paths or
+# URLs. If a location is present for each tag, the installdox tool
 # does not have to be run to correct the links.
 # Note that each tag file must have a unique name
 # (where the name does NOT include the path)
-# If a tag file is not located in the directory in which doxygen 
+# If a tag file is not located in the directory in which doxygen
 # is run, you must also specify the path to the tagfile here.
 
-TAGFILES               = 
+TAGFILES               =
 
-# When a file name is specified after GENERATE_TAGFILE, doxygen will create 
+# When a file name is specified after GENERATE_TAGFILE, doxygen will create
 # a tag file that is based on the input files it reads.
 
-GENERATE_TAGFILE       = 
+GENERATE_TAGFILE       =
 
-# If the ALLEXTERNALS tag is set to YES all external classes will be listed 
-# in the class index. If set to NO only the inherited external classes 
+# If the ALLEXTERNALS tag is set to YES all external classes will be listed
+# in the class index. If set to NO only the inherited external classes
 # will be listed.
 
 ALLEXTERNALS           = NO
 
-# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed 
-# in the modules index. If set to NO, only the current project's groups will 
+# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed
+# in the modules index. If set to NO, only the current project's groups will
 # be listed.
 
 EXTERNAL_GROUPS        = YES
 
-# The PERL_PATH should be the absolute path and name of the perl script 
+# The PERL_PATH should be the absolute path and name of the perl script
 # interpreter (i.e. the result of `which perl').
 
 PERL_PATH              = /usr/bin/perl
 
 #---------------------------------------------------------------------------
-# Configuration options related to the dot tool   
+# Configuration options related to the dot tool
 #---------------------------------------------------------------------------
 
-# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will 
-# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base 
-# or super classes. Setting the tag to NO turns the diagrams off. Note that 
-# this option is superseded by the HAVE_DOT option below. This is only a 
-# fallback. It is recommended to install and use dot, since it yields more 
+# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will
+# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base
+# or super classes. Setting the tag to NO turns the diagrams off. Note that
+# this option is superseded by the HAVE_DOT option below. This is only a
+# fallback. It is recommended to install and use dot, since it yields more
 # powerful graphs.
 
 CLASS_DIAGRAMS         = YES
 
-# You can define message sequence charts within doxygen comments using the \msc 
-# command. Doxygen will then run the mscgen tool (see 
-# http://www.mcternan.me.uk/mscgen/) to produce the chart and insert it in the 
-# documentation. The MSCGEN_PATH tag allows you to specify the directory where 
-# the mscgen tool resides. If left empty the tool is assumed to be found in the 
+# You can define message sequence charts within doxygen comments using the \msc
+# command. Doxygen will then run the mscgen tool (see
+# http://www.mcternan.me.uk/mscgen/) to produce the chart and insert it in the
+# documentation. The MSCGEN_PATH tag allows you to specify the directory where
+# the mscgen tool resides. If left empty the tool is assumed to be found in the
 # default search path.
 
-MSCGEN_PATH            = 
+MSCGEN_PATH            =
 
-# If set to YES, the inheritance and collaboration graphs will hide 
-# inheritance and usage relations if the target is undocumented 
+# If set to YES, the inheritance and collaboration graphs will hide
+# inheritance and usage relations if the target is undocumented
 # or is not a class.
 
 HIDE_UNDOC_RELATIONS   = YES
 
-# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is 
-# available from the path. This tool is part of Graphviz, a graph visualization 
-# toolkit from AT&T and Lucent Bell Labs. The other options in this section 
+# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is
+# available from the path. This tool is part of Graphviz, a graph visualization
+# toolkit from AT&T and Lucent Bell Labs. The other options in this section
 # have no effect if this option is set to NO (the default)
 
 HAVE_DOT               = NO
 
-# By default doxygen will write a font called FreeSans.ttf to the output 
-# directory and reference it in all dot files that doxygen generates. This 
-# font does not include all possible unicode characters however, so when you need 
-# these (or just want a differently looking font) you can specify the font name 
-# using DOT_FONTNAME. You need need to make sure dot is able to find the font, 
-# which can be done by putting it in a standard location or by setting the 
-# DOTFONTPATH environment variable or by setting DOT_FONTPATH to the directory 
+# By default doxygen will write a font called FreeSans.ttf to the output
+# directory and reference it in all dot files that doxygen generates. This
+# font does not include all possible unicode characters however, so when you need
+# these (or just want a differently looking font) you can specify the font name
+# using DOT_FONTNAME. You need need to make sure dot is able to find the font,
+# which can be done by putting it in a standard location or by setting the
+# DOTFONTPATH environment variable or by setting DOT_FONTPATH to the directory
 # containing the font.
 
-DOT_FONTNAME           = FreeSans
+DOT_FONTNAME           =
 
-# By default doxygen will tell dot to use the output directory to look for the 
-# FreeSans.ttf font (which doxygen will put there itself). If you specify a 
-# different font using DOT_FONTNAME you can set the path where dot 
+# By default doxygen will tell dot to use the output directory to look for the
+# FreeSans.ttf font (which doxygen will put there itself). If you specify a
+# different font using DOT_FONTNAME you can set the path where dot
 # can find it using this tag.
 
-DOT_FONTPATH           = 
+DOT_FONTPATH           =
 
-# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen 
-# will generate a graph for each documented class showing the direct and 
-# indirect inheritance relations. Setting this tag to YES will force the 
+# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen
+# will generate a graph for each documented class showing the direct and
+# indirect inheritance relations. Setting this tag to YES will force the
 # the CLASS_DIAGRAMS tag to NO.
 
 CLASS_GRAPH            = YES
 
-# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen 
-# will generate a graph for each documented class showing the direct and 
-# indirect implementation dependencies (inheritance, containment, and 
+# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen
+# will generate a graph for each documented class showing the direct and
+# indirect implementation dependencies (inheritance, containment, and
 # class references variables) of the class with other documented classes.
 
 COLLABORATION_GRAPH    = YES
 
-# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen 
+# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen
 # will generate a graph for groups, showing the direct groups dependencies
 
 GROUP_GRAPHS           = YES
 
-# If the UML_LOOK tag is set to YES doxygen will generate inheritance and 
-# collaboration diagrams in a style similar to the OMG's Unified Modeling 
+# If the UML_LOOK tag is set to YES doxygen will generate inheritance and
+# collaboration diagrams in a style similar to the OMG's Unified Modeling
 # Language.
 
 UML_LOOK               = NO
 
-# If set to YES, the inheritance and collaboration graphs will show the 
+# If set to YES, the inheritance and collaboration graphs will show the
 # relations between templates and their instances.
 
 TEMPLATE_RELATIONS     = YES
 
-# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT 
-# tags are set to YES then doxygen will generate a graph for each documented 
-# file showing the direct and indirect include dependencies of the file with 
+# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT
+# tags are set to YES then doxygen will generate a graph for each documented
+# file showing the direct and indirect include dependencies of the file with
 # other documented files.
 
 INCLUDE_GRAPH          = YES
 
-# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and 
-# HAVE_DOT tags are set to YES then doxygen will generate a graph for each 
-# documented header file showing the documented files that directly or 
+# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and
+# HAVE_DOT tags are set to YES then doxygen will generate a graph for each
+# documented header file showing the documented files that directly or
 # indirectly include this file.
 
 INCLUDED_BY_GRAPH      = YES
 
-# If the CALL_GRAPH and HAVE_DOT options are set to YES then 
-# doxygen will generate a call dependency graph for every global function 
-# or class method. Note that enabling this option will significantly increase 
-# the time of a run. So in most cases it will be better to enable call graphs 
+# If the CALL_GRAPH and HAVE_DOT options are set to YES then
+# doxygen will generate a call dependency graph for every global function
+# or class method. Note that enabling this option will significantly increase
+# the time of a run. So in most cases it will be better to enable call graphs
 # for selected functions only using the \callgraph command.
 
 CALL_GRAPH             = NO
 
-# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then 
-# doxygen will generate a caller dependency graph for every global function 
-# or class method. Note that enabling this option will significantly increase 
-# the time of a run. So in most cases it will be better to enable caller 
+# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then
+# doxygen will generate a caller dependency graph for every global function
+# or class method. Note that enabling this option will significantly increase
+# the time of a run. So in most cases it will be better to enable caller
 # graphs for selected functions only using the \callergraph command.
 
 CALLER_GRAPH           = NO
 
-# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen 
+# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen
 # will graphical hierarchy of all classes instead of a textual one.
 
 GRAPHICAL_HIERARCHY    = YES
 
-# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES 
-# then doxygen will show the dependencies a directory has on other directories 
+# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES
+# then doxygen will show the dependencies a directory has on other directories
 # in a graphical way. The dependency relations are determined by the #include
 # relations between the files in the directories.
 
 DIRECTORY_GRAPH        = YES
 
-# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images 
+# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images
 # generated by dot. Possible values are png, jpg, or gif
 # If left blank png will be used.
 
 DOT_IMAGE_FORMAT       = png
 
-# The tag DOT_PATH can be used to specify the path where the dot tool can be 
+# The tag DOT_PATH can be used to specify the path where the dot tool can be
 # found. If left blank, it is assumed the dot tool can be found in the path.
 
-DOT_PATH               = 
+DOT_PATH               =
 
-# The DOTFILE_DIRS tag can be used to specify one or more directories that 
-# contain dot files that are included in the documentation (see the 
+# The DOTFILE_DIRS tag can be used to specify one or more directories that
+# contain dot files that are included in the documentation (see the
 # \dotfile command).
 
-DOTFILE_DIRS           = 
+DOTFILE_DIRS           =
 
-# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of 
-# nodes that will be shown in the graph. If the number of nodes in a graph 
-# becomes larger than this value, doxygen will truncate the graph, which is 
-# visualized by representing a node as a red box. Note that doxygen if the 
-# number of direct children of the root node in a graph is already larger than 
-# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note 
+# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of
+# nodes that will be shown in the graph. If the number of nodes in a graph
+# becomes larger than this value, doxygen will truncate the graph, which is
+# visualized by representing a node as a red box. Note that doxygen if the
+# number of direct children of the root node in a graph is already larger than
+# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note
 # that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH.
 
 DOT_GRAPH_MAX_NODES    = 50
 
-# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the 
-# graphs generated by dot. A depth value of 3 means that only nodes reachable 
-# from the root by following a path via at most 3 edges will be shown. Nodes 
-# that lay further from the root node will be omitted. Note that setting this 
-# option to 1 or 2 may greatly reduce the computation time needed for large 
-# code bases. Also note that the size of a graph can be further restricted by 
+# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the
+# graphs generated by dot. A depth value of 3 means that only nodes reachable
+# from the root by following a path via at most 3 edges will be shown. Nodes
+# that lay further from the root node will be omitted. Note that setting this
+# option to 1 or 2 may greatly reduce the computation time needed for large
+# code bases. Also note that the size of a graph can be further restricted by
 # DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction.
 
 MAX_DOT_GRAPH_DEPTH    = 0
 
-# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent 
-# background. This is enabled by default, which results in a transparent 
-# background. Warning: Depending on the platform used, enabling this option 
-# may lead to badly anti-aliased labels on the edges of a graph (i.e. they 
+# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent
+# background. This is enabled by default, which results in a transparent
+# background. Warning: Depending on the platform used, enabling this option
+# may lead to badly anti-aliased labels on the edges of a graph (i.e. they
 # become hard to read).
 
 DOT_TRANSPARENT        = NO
 
-# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output 
-# files in one run (i.e. multiple -o and -T options on the command line). This 
-# makes dot run faster, but since only newer versions of dot (>1.8.10) 
+# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output
+# files in one run (i.e. multiple -o and -T options on the command line). This
+# makes dot run faster, but since only newer versions of dot (>1.8.10)
 # support this, this feature is disabled by default.
 
 DOT_MULTI_TARGETS      = NO
 
-# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will 
-# generate a legend page explaining the meaning of the various boxes and 
+# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will
+# generate a legend page explaining the meaning of the various boxes and
 # arrows in the dot generated graphs.
 
 GENERATE_LEGEND        = YES
 
-# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will 
-# remove the intermediate dot files that are used to generate 
+# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will
+# remove the intermediate dot files that are used to generate
 # the various graphs.
 
 DOT_CLEANUP            = YES
 
 #---------------------------------------------------------------------------
-# Configuration::additions related to the search engine   
+# Configuration::additions related to the search engine
 #---------------------------------------------------------------------------
 
-# The SEARCHENGINE tag specifies whether or not a search engine should be 
+# The SEARCHENGINE tag specifies whether or not a search engine should be
 # used. If set to NO the values of all tags below this one will be ignored.
 
 SEARCHENGINE           = NO
diff --git a/INSTALL b/INSTALL
index ddb790b0b1..f493f4ed3d 100644
--- a/INSTALL
+++ b/INSTALL
@@ -16,37 +16,3 @@ If it doesn't build for you:
     ./configure --with-libevent-dir=/usr/local
   rather than simply ./configure.
 
-  If you have mysterious autoconf failures while linking openssl,
-  consider setting your LD_LIBRARY_PATH to the openssl lib directory.
-  For example, "setenv LD_LIBRARY_PATH /usr/athena/lib".
-
-  Lastly, check out
-  https://www.torproject.org/docs/faq#DoesntWork
-
-How to do static builds of tor:
-
-Tor supports linking each of the libraries it needs statically. Use the
---enable-static-X ./configure option in conjunction with the --with-X-dir
-option for libevent, zlib, and openssl. For this to work sanely, libevent
-should be built with --disable-shared --enable-static --with-pic, and
-OpenSSL should be built with no-shared no-dso.
-
-If you need to build tor so that system libraries are also statically linked,
-use the --enable-static-tor ./configure option. This won't work on OS X
-unless you build the required crt0.o yourself. It is also incompatible with
-the --enable-gcc-hardening option.
-
-An example of how to build a mostly static tor:
-./configure --enable-static-libevent \
-            --enable-static-openssl  \
-            --enable-static-zlib     \
-            --with-libevent-dir=/tmp/static-tor/libevent-1.4.14b-stable \
-            --with-openssl-dir=/tmp/static-tor/openssl-0.9.8r/ \
-            --with-zlib-dir=/tmp/static-tor/zlib-1.2.5
-
-An example of how to build an entirely static tor:
-./configure --enable-static-tor \
-            --with-libevent-dir=/tmp/static-tor/libevent-1.4.14b-stable \
-            --with-openssl-dir=/tmp/static-tor/openssl-0.9.8r/ \
-            --with-zlib-dir=/tmp/static-tor/zlib-1.2.5
-
diff --git a/LICENSE b/LICENSE
index 56d1002251..cc27668427 100644
--- a/LICENSE
+++ b/LICENSE
@@ -13,7 +13,7 @@ Tor is distributed under this license:
 
 Copyright (c) 2001-2004, Roger Dingledine
 Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
-Copyright (c) 2007-2016, The Tor Project, Inc.
+Copyright (c) 2007-2019, The Tor Project, Inc.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are
@@ -156,6 +156,36 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 ===============================================================================
+getdelim.c is distributed under this license:
+
+ Copyright (c) 2011 The NetBSD Foundation, Inc.
+ All rights reserved.
+
+ This code is derived from software contributed to The NetBSD Foundation
+ by Christos Zoulas.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ POSSIBILITY OF SUCH DAMAGE.
+
+===============================================================================
 src/config/geoip is licensed under the following license:
 
 OPEN DATA LICENSE (GeoLite Country and GeoLite City databases)
diff --git a/Makefile.am b/Makefile.am
index 48d3df2654..5d18666edc 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,6 +1,6 @@
 # Copyright (c) 2001-2004, Roger Dingledine
 # Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
-# Copyright (c) 2007-2015, The Tor Project, Inc.
+# Copyright (c) 2007-2019, The Tor Project, Inc.
 # See LICENSE for licensing information
 
 ACLOCAL_AMFLAGS = -I m4
@@ -9,32 +9,157 @@ noinst_LIBRARIES=
 EXTRA_DIST=
 noinst_HEADERS=
 bin_PROGRAMS=
+EXTRA_PROGRAMS=
 CLEANFILES=
 TESTS=
 noinst_PROGRAMS=
 DISTCLEANFILES=
 bin_SCRIPTS=
-AM_CPPFLAGS=
-AM_CFLAGS=@TOR_SYSTEMD_CFLAGS@ @CFLAGS_BUGTRAP@
+AM_CPPFLAGS=\
+	-I$(top_srcdir)/src \
+	-I$(top_srcdir)/src/ext \
+	-I$(top_srcdir)/src/ext/trunnel \
+	-I$(top_srcdir)/src/trunnel
+
+AM_CFLAGS=@TOR_SYSTEMD_CFLAGS@ @CFLAGS_BUGTRAP@ @TOR_LZMA_CFLAGS@ @TOR_ZSTD_CFLAGS@
 SHELL=@SHELL@
 
 if COVERAGE_ENABLED
-TESTING_TOR_BINARY=$(top_builddir)/src/or/tor-cov$(EXEEXT)
+TESTING_TOR_BINARY=$(top_builddir)/src/app/tor-cov$(EXEEXT)
+else
+TESTING_TOR_BINARY=$(top_builddir)/src/app/tor$(EXEEXT)
+endif
+
+if USE_RUST
+## this MUST be $(), otherwise am__DEPENDENCIES will not track it
+rust_ldadd=$(top_builddir)/$(TOR_RUST_LIB_PATH) \
+	$(TOR_RUST_EXTRA_LIBS)
 else
-TESTING_TOR_BINARY=$(top_builddir)/src/or/tor$(EXEEXT)
+rust_ldadd=
+endif
+
+# "Common" libraries used to link tor's utility code.
+TOR_UTIL_LIBS = \
+        src/lib/libtor-geoip.a \
+	src/lib/libtor-process.a \
+	src/lib/libtor-time.a \
+	src/lib/libtor-fs.a \
+	src/lib/libtor-encoding.a \
+	src/lib/libtor-sandbox.a \
+	src/lib/libtor-container.a \
+	src/lib/libtor-net.a \
+	src/lib/libtor-thread.a \
+	src/lib/libtor-memarea.a \
+	src/lib/libtor-math.a \
+	src/lib/libtor-meminfo.a \
+	src/lib/libtor-osinfo.a \
+	src/lib/libtor-log.a \
+	src/lib/libtor-lock.a \
+	src/lib/libtor-fdio.a \
+	src/lib/libtor-string.a \
+	src/lib/libtor-term.a \
+	src/lib/libtor-smartlist-core.a \
+	src/lib/libtor-malloc.a \
+	src/lib/libtor-wallclock.a \
+	src/lib/libtor-err.a \
+	src/lib/libtor-intmath.a \
+	src/lib/libtor-ctime.a
+
+# Variants of the above for linking the testing variant of tor (for coverage
+# and tests)
+if UNITTESTS_ENABLED
+TOR_UTIL_TESTING_LIBS = \
+        src/lib/libtor-geoip-testing.a \
+	src/lib/libtor-process-testing.a \
+	src/lib/libtor-time-testing.a \
+	src/lib/libtor-fs-testing.a \
+	src/lib/libtor-encoding-testing.a \
+	src/lib/libtor-sandbox-testing.a \
+	src/lib/libtor-container-testing.a \
+	src/lib/libtor-net-testing.a \
+	src/lib/libtor-thread-testing.a \
+	src/lib/libtor-memarea-testing.a \
+	src/lib/libtor-math-testing.a \
+	src/lib/libtor-meminfo-testing.a \
+	src/lib/libtor-osinfo-testing.a \
+	src/lib/libtor-term-testing.a \
+	src/lib/libtor-log-testing.a \
+	src/lib/libtor-lock-testing.a \
+	src/lib/libtor-fdio-testing.a \
+	src/lib/libtor-string-testing.a \
+	src/lib/libtor-smartlist-core-testing.a \
+	src/lib/libtor-malloc-testing.a \
+	src/lib/libtor-wallclock-testing.a \
+	src/lib/libtor-err-testing.a \
+	src/lib/libtor-intmath.a \
+	src/lib/libtor-ctime-testing.a
+endif
+
+# Internal crypto libraries used in Tor
+TOR_CRYPTO_LIBS = \
+	src/lib/libtor-tls.a \
+	src/lib/libtor-crypt-ops.a \
+	$(LIBKECCAK_TINY) \
+	$(LIBDONNA)
+
+# Variants of the above for linking the testing variant of tor (for coverage
+# and tests)
+if UNITTESTS_ENABLED
+TOR_CRYPTO_TESTING_LIBS = \
+	src/lib/libtor-tls-testing.a \
+	src/lib/libtor-crypt-ops-testing.a \
+	$(LIBKECCAK_TINY) \
+	$(LIBDONNA)
+endif
+
+# All static libraries used to link tor.
+TOR_INTERNAL_LIBS = \
+	src/core/libtor-app.a \
+	src/lib/libtor-compress.a \
+	src/lib/libtor-evloop.a \
+	$(TOR_CRYPTO_LIBS) \
+	$(TOR_UTIL_LIBS) \
+	src/trunnel/libor-trunnel.a \
+	src/lib/libtor-trace.a
+
+# Variants of the above for linking the testing variant of tor (for coverage
+# and tests)
+if UNITTESTS_ENABLED
+TOR_INTERNAL_TESTING_LIBS = \
+	src/core/libtor-app-testing.a \
+	src/lib/libtor-compress-testing.a \
+	src/lib/libtor-evloop-testing.a \
+	$(TOR_CRYPTO_TESTING_LIBS) \
+	$(TOR_UTIL_TESTING_LIBS) \
+	src/trunnel/libor-trunnel-testing.a \
+	src/lib/libtor-trace.a
+endif
+
+TOR_LDFLAGS_CRYPTLIB=@TOR_LDFLAGS_openssl@
+TOR_LIBS_CRYPTLIB=@TOR_OPENSSL_LIBS@
+TOR_CFLAGS_CRYPTLIB=
+if USE_NSS
+TOR_CFLAGS_CRYPTLIB+=@NSS_CFLAGS@
+TOR_LIBS_CRYPTLIB+=@NSS_LIBS@
 endif
 
+# All libraries used to link tor-cov
+
 include src/include.am
 include doc/include.am
 include contrib/include.am
 
 EXTRA_DIST+= \
 	ChangeLog					\
+	CONTRIBUTING					\
+	CODE_OF_CONDUCT                                 \
 	INSTALL						\
 	LICENSE						\
 	Makefile.nmake					\
 	README						\
-	ReleaseNotes
+	ReleaseNotes					\
+	scripts/maint/checkIncludes.py                  \
+	scripts/maint/checkSpace.pl
 
 ## This tells etags how to find mockable function definitions.
 AM_ETAGSFLAGS=--regex='{c}/MOCK_IMPL([^,]+,\W*\([a-zA-Z0-9_]+\)\W*,/\1/s'
@@ -42,16 +167,22 @@ AM_ETAGSFLAGS=--regex='{c}/MOCK_IMPL([^,]+,\W*\([a-zA-Z0-9_]+\)\W*,/\1/s'
 if COVERAGE_ENABLED
 TEST_CFLAGS=-fno-inline -fprofile-arcs -ftest-coverage
 if DISABLE_ASSERTS_IN_UNIT_TESTS
-TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE -DDISABLE_ASSERTS_IN_UNIT_TESTS
+TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE -DDISABLE_ASSERTS_IN_UNIT_TESTS @TOR_MODULES_ALL_ENABLED@
 else
-TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE
+TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE @TOR_MODULES_ALL_ENABLED@
 endif
 TEST_NETWORK_FLAGS=--coverage --hs-multi-client 1
 else
 TEST_CFLAGS=
-TEST_CPPFLAGS=-DTOR_UNIT_TESTS
+TEST_CPPFLAGS=-DTOR_UNIT_TESTS @TOR_MODULES_ALL_ENABLED@
 TEST_NETWORK_FLAGS=--hs-multi-client 1
 endif
+TEST_NETWORK_WARNING_FLAGS=--quiet --only-warnings
+
+if LIBFUZZER_ENABLED
+TEST_CFLAGS += -fsanitize-coverage=trace-pc-guard,trace-cmp,trace-div
+# not "edge"
+endif
 
 TEST_NETWORK_ALL_LOG_DIR=$(top_builddir)/test_network_log
 TEST_NETWORK_ALL_DRIVER_FLAGS=--color-tests yes
@@ -82,6 +213,8 @@ doxygen:
 test: all
 	$(top_builddir)/src/test/test
 
+check-local: check-spaces check-changes check-includes
+
 need-chutney-path:
 	@if test ! -d "$$CHUTNEY_PATH"; then \
 		echo '$$CHUTNEY_PATH was not set.'; \
@@ -101,18 +234,20 @@ test-network: need-chutney-path $(TESTING_TOR_BINARY) src/tools/tor-gencert
 
 # Run all available tests using automake's test-driver
 # only run IPv6 tests if we can ping6 ::1 (localhost)
+# only run IPv6 tests if we can ping ::1 (localhost)
 # some IPv6 tests will fail without an IPv6 DNS server (see #16971 and #17011)
 # only run mixed tests if we have a tor-stable binary
-# Try both the BSD and the Linux ping6 syntax, because they're incompatible
+# Try the syntax for BSD ping6, Linux ping6, and Linux ping -6,
+# because they're incompatible
 test-network-all: need-chutney-path test-driver $(TESTING_TOR_BINARY) src/tools/tor-gencert
 	mkdir -p $(TEST_NETWORK_ALL_LOG_DIR)
 	rm -f $(TEST_NETWORK_ALL_LOG_DIR)/*.log $(TEST_NETWORK_ALL_LOG_DIR)/*.trs
 	@flavors="$(TEST_CHUTNEY_FLAVORS)"; \
-	if ping6 -q -c 1 -o ::1 >/dev/null 2>&1 || ping6 -q -c 1 -W 1 ::1 >/dev/null 2>&1; then \
-		echo "ping6 ::1 succeeded, running IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
+	if ping6 -q -c 1 -o ::1 >/dev/null 2>&1 || ping6 -q -c 1 -W 1 ::1 >/dev/null 2>&1 || ping -6 -c 1 -W 1 ::1 >/dev/null 2>&1; then \
+		echo "ping6 ::1 or ping ::1 succeeded, running IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
 		flavors="$$flavors $(TEST_CHUTNEY_FLAVORS_IPV6)"; \
 	else \
-		echo "ping6 ::1 failed, skipping IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
+		echo "ping6 ::1 and ping ::1 failed, skipping IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
 		skip_flavors="$$skip_flavors $(TEST_CHUTNEY_FLAVORS_IPV6)"; \
 	fi; \
 	if command -v tor-stable >/dev/null 2>&1; then \
@@ -127,6 +262,7 @@ test-network-all: need-chutney-path test-driver $(TESTING_TOR_BINARY) src/tools/
 	done; \
 	for f in $$flavors; do \
 		$(SHELL) $(top_srcdir)/test-driver --test-name $$f --log-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.log --trs-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.trs $(TEST_NETWORK_ALL_DRIVER_FLAGS) $(top_srcdir)/src/test/test-network.sh --flavor $$f $(TEST_NETWORK_FLAGS); \
+		$(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_WARNING_FLAGS); \
 	done; \
 	echo "Log and result files are available in $(TEST_NETWORK_ALL_LOG_DIR)."; \
 	! grep -q FAIL $(TEST_NETWORK_ALL_LOG_DIR)/*.trs
@@ -139,7 +275,7 @@ need-stem-path:
 	fi
 
 test-stem: need-stem-path $(TESTING_TOR_BINARY)
-	@$(PYTHON) "$$STEM_SOURCE_DIR"/run_tests.py --tor "$(TESTING_TOR_BINARY)" --all --log notice --target RUN_ALL;
+	@$(PYTHON) "$$STEM_SOURCE_DIR"/run_tests.py --tor "$(TESTING_TOR_BINARY)" --integ --test control.controller --test control.base_controller --test process --log notice;
 
 test-stem-full: need-stem-path $(TESTING_TOR_BINARY)
 	@$(PYTHON) "$$STEM_SOURCE_DIR"/run_tests.py --tor "$(TESTING_TOR_BINARY)" --all --log notice --target RUN_ALL,ONLINE -v;
@@ -184,11 +320,21 @@ coverage-html-full: all
 # Avoid strlcpy.c, strlcat.c, aes.c, OpenBSD_malloc_Linux.c, sha256.c,
 # tinytest*.[ch]
 check-spaces:
-	$(top_srcdir)/scripts/maint/checkSpace.pl -C \
-		$(top_srcdir)/src/common/*.[ch] \
-		$(top_srcdir)/src/or/*.[ch] \
+if USE_PERL
+	$(PERL) $(top_srcdir)/scripts/maint/checkSpace.pl -C \
+		$(top_srcdir)/src/lib/*/*.[ch] \
+		$(top_srcdir)/src/core/*/*.[ch] \
+		$(top_srcdir)/src/feature/*/*.[ch] \
+		$(top_srcdir)/src/app/*/*.[ch] \
 		$(top_srcdir)/src/test/*.[ch] \
+		$(top_srcdir)/src/test/*/*.[ch] \
 		$(top_srcdir)/src/tools/*.[ch]
+endif
+
+check-includes:
+if USEPYTHON
+	$(PYTHON) $(top_srcdir)/scripts/maint/checkIncludes.py
+endif
 
 check-docs: all
 	$(PERL) $(top_builddir)/scripts/maint/checkOptionDocs.pl
@@ -197,16 +343,87 @@ check-logs:
 	$(top_srcdir)/scripts/maint/checkLogs.pl \
 		$(top_srcdir)/src/*/*.[ch] | sort -n
 
+.PHONY: check-typos
+check-typos:
+	@if test -x "`which misspell 2>&1;true`"; then \
+		echo "Checking for Typos ..."; \
+		(misspell \
+			$(top_srcdir)/src/[^e]*/*.[ch] \
+			$(top_srcdir)/doc \
+			$(top_srcdir)/contrib \
+			$(top_srcdir)/scripts \
+			$(top_srcdir)/README \
+			$(top_srcdir)/ChangeLog \
+			$(top_srcdir)/INSTALL \
+			$(top_srcdir)/ReleaseNotes \
+			$(top_srcdir)/LICENSE); \
+	else \
+		echo "Tor can use misspell to check for typos."; \
+		echo "It seems that you don't have misspell installed."; \
+		echo "You can install the latest version of misspell here: https://github.com/client9/misspell#install"; \
+	fi
+
+.PHONY: rustfmt
+rustfmt:
+if USE_RUST
+	@if test -x "`which cargo-fmt 2>&1;true`"; then \
+		echo "Formatting Rust code ..."; \
+		(cd "$(top_srcdir)/src/rust" && cargo fmt --all --); \
+	else \
+		echo "Tor uses rustfmt (via cargo-fmt) to format Rust code."; \
+		echo "However, it seems that you don't have rustfmt installed."; \
+		printf "You can install rustfmt by following the directions here:"; \
+		echo " https://github.com/rust-lang-nursery/rustfmt"; \
+	fi
+endif
+
+.PHONY: check-rustfmt
+check-rustfmt:
+if USE_RUST
+	@if test -x "`which cargo-fmt 2>&1;true`"; then \
+		printf "Running rustfmt..."; \
+		(cd "$(top_srcdir)/src/rust" && cargo fmt --all -- --check && echo "done.") || \
+		(echo "**************** check-rustfmt failed. ****************"; \
+		 echo "   Run \`make rustfmt\` to apply the above changes."; \
+		 exit 1); \
+	else \
+		echo "Tor uses rustfmt (via cargo-fmt) to format Rust code."; \
+		echo "However, it seems that you don't have rustfmt installed."; \
+		printf "You can install rustfmt by following the directions here:"; \
+		echo " https://github.com/rust-lang-nursery/rustfmt"; \
+	fi
+endif
+
+.PHONY: clippy
+clippy:
+if USE_RUST
+	@if test -x "`which cargo-clippy 2>&1;true`"; then \
+		echo "Running cargo clippy ..."; \
+		echo "Prepare yourself for the onslaught of suggestions ..."; \
+		(cd "$(top_srcdir)/src/rust" && cargo clippy); \
+	else \
+		echo "Tor can use clippy to lint Rust code."; \
+		echo "However, it seems that you don't have clippy installed."; \
+		echo "You can install the latest version of clippy by following the directions here: https://github.com/rust-lang-nursery/rust-clippy"; \
+	fi
+endif
+
 .PHONY: check-changes
 check-changes:
+if USEPYTHON
 	@if test -d "$(top_srcdir)/changes"; then \
-		$(PYTHON) $(top_srcdir)/scripts/maint/lintChanges.py $(top_srcdir)/changes/*; \
+		$(PYTHON) $(top_srcdir)/scripts/maint/lintChanges.py $(top_srcdir)/changes; \
 		fi
+endif
 
 .PHONY: update-versions
 update-versions:
 	$(PERL) $(top_builddir)/scripts/maint/updateVersions.pl
 
+.PHONY: callgraph
+callgraph:
+	$(top_builddir)/scripts/maint/run_calltool.sh
+
 version:
 	@echo "Tor @VERSION@"
 	@if test -d "$(top_srcdir)/.git" && test -x "`which git 2>&1;true`"; then \
@@ -220,6 +437,14 @@ mostlyclean-local:
 	rm -rf $(top_builddir)/doc/doxygen
 	rm -rf $(TEST_NETWORK_ALL_LOG_DIR)
 
+clean-local:
+	rm -rf $(top_builddir)/src/rust/target
+	rm -rf $(top_builddir)/src/rust/.cargo/registry
+
+if USE_RUST
+distclean-local: distclean-rust
+endif
+
 # This relies on some internal details of how automake implements
 # distcheck.  We check two directories because automake-1.15 changed
 # from $(distdir)/_build to $(distdir)/_build/sub.
@@ -228,3 +453,18 @@ show-distdir-testlog:
 	  cat $(distdir)/_build/sub/$(TEST_SUITE_LOG); \
 	else \
 	  cat $(distdir)/_build/$(TEST_SUITE_LOG); fi
+
+# Similarly, this relies on automake internals to run file on an
+# intermittent core file whose provenance is not known to us.  See
+# ticket 26787.
+show-distdir-core:
+	@if test -d "$(distdir)/_build/sub"; then \
+	  file $(distdir)/_build/sub/core ; \
+	else \
+	  file $(distdir)/_build/core; fi
+
+show-libs:
+	@echo $(TOR_INTERNAL_LIBS)
+
+show-testing-libs:
+	@echo $(TOR_INTERNAL_TESTING_LIBS)
diff --git a/README b/README
index d246a6930e..9e0f4b8241 100644
--- a/README
+++ b/README
@@ -27,3 +27,6 @@ Frequently Asked Questions:
 
 To get started working on Tor development:
         See the doc/HACKING directory.
+
+Release timeline:
+         https://trac.torproject.org/projects/tor/wiki/org/teams/NetworkTeam/CoreTorReleases
diff --git a/ReleaseNotes b/ReleaseNotes
index af61a4d739..e0a25a74b7 100644
--- a/ReleaseNotes
+++ b/ReleaseNotes
@@ -1,6 +1,7294 @@
-This document summarizes new features and bugfixes in each stable release
-of Tor. If you want to see more detailed descriptions of the changes in
-each development snapshot, see the ChangeLog file.
+This document summarizes new features and bugfixes in each stable
+release of Tor. If you want to see more detailed descriptions of the
+changes in each development snapshot, see the ChangeLog file.
+
+Changes in version 0.3.4.9 - 2018-11-02
+  Tor 0.3.4.9 is the second stable release in its series; it backports
+  numerous fixes, including a fix for a bandwidth management bug that
+  was causing memory exhaustion on relays. Anyone running an earlier
+  version of Tor 0.3.4.9 should upgrade.
+
+  o Major bugfixes (compilation, backport from 0.3.5.3-alpha):
+    - Fix compilation on ARM (and other less-used CPUs) when compiling
+      with OpenSSL before 1.1. Fixes bug 27781; bugfix on 0.3.4.1-alpha.
+
+  o Major bugfixes (mainloop, bootstrap, backport from 0.3.5.3-alpha):
+    - Make sure Tor bootstraps and works properly if only the
+      ControlPort is set. Prior to this fix, Tor would only bootstrap
+      when a client port was set (Socks, Trans, NATD, DNS or HTTPTunnel
+      port). Fixes bug 27849; bugfix on 0.3.4.1-alpha.
+
+  o Major bugfixes (relay, backport from 0.3.5.3-alpha):
+    - When our write bandwidth limit is exhausted, stop writing on the
+      connection. Previously, we had a typo in the code that would make
+      us stop reading instead, leading to relay connections being stuck
+      indefinitely and consuming kernel RAM. Fixes bug 28089; bugfix
+      on 0.3.4.1-alpha.
+
+  o Major bugfixes (restart-in-process, backport from 0.3.5.1-alpha):
+    - Fix a use-after-free error that could be caused by passing Tor an
+      impossible set of options that would fail during options_act().
+      Fixes bug 27708; bugfix on 0.3.3.1-alpha.
+
+  o Minor features (continuous integration, backport from 0.3.5.1-alpha):
+    - Don't do a distcheck with --disable-module-dirauth in Travis.
+      Implements ticket 27252.
+    - Only run one online rust build in Travis, to reduce network
+      errors. Skip offline rust builds on Travis for Linux gcc, because
+      they're redundant. Implements ticket 27252.
+    - Skip gcc on OSX in Travis CI, because it's rarely used. Skip a
+      duplicate hardening-off build in Travis on Tor 0.2.9. Skip gcc on
+      Linux with default settings, because all the non-default builds
+      use gcc on Linux. Implements ticket 27252.
+
+  o Minor features (continuous integration, backport from 0.3.5.3-alpha):
+    - Use the Travis Homebrew addon to install packages on macOS during
+      Travis CI. The package list is the same, but the Homebrew addon
+      does not do a `brew update` by default. Implements ticket 27738.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 9 2018 Maxmind GeoLite2
+      Country database. Closes ticket 27991.
+
+  o Minor bugfixes (32-bit OSX and iOS, timing, backport from 0.3.5.2-alpha):
+    - Fix an integer overflow bug in our optimized 32-bit millisecond-
+      difference algorithm for 32-bit Apple platforms. Previously, it
+      would overflow when calculating the difference between two times
+      more than 47 days apart. Fixes part of bug 27139; bugfix
+      on 0.3.4.1-alpha.
+    - Improve the precision of our 32-bit millisecond difference
+      algorithm for 32-bit Apple platforms. Fixes part of bug 27139;
+      bugfix on 0.3.4.1-alpha.
+    - Relax the tolerance on the mainloop/update_time_jumps test when
+      running on 32-bit Apple platforms. Fixes part of bug 27139; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (C correctness, to appear in 0.3.5.4-alpha):
+    - Avoid undefined behavior in an end-of-string check when parsing
+      the BEGIN line in a directory object. Fixes bug 28202; bugfix
+      on 0.2.0.3-alpha.
+
+  o Minor bugfixes (CI, appveyor, to appear in 0.3.5.4-alpha):
+    - Only install the necessary mingw packages during our appveyor
+      builds. This change makes the build a little faster, and prevents
+      a conflict with a preinstalled mingw openssl that appveyor now
+      ships. Fixes bugs 27943 and 27765; bugfix on 0.3.4.2-alpha.
+
+  o Minor bugfixes (code safety, backport from 0.3.5.3-alpha):
+    - Rewrite our assertion macros so that they no longer suppress the
+      compiler's -Wparentheses warnings. Fixes bug 27709; bugfix
+
+  o Minor bugfixes (continuous integration, backport from 0.3.5.1-alpha):
+    - Stop reinstalling identical packages in our Windows CI. Fixes bug
+      27464; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (directory authority, to appear in 0.3.5.4-alpha):
+    - Log additional info when we get a relay that shares an ed25519 ID
+      with a different relay, instead making a BUG() warning. Fixes bug
+      27800; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (directory connection shutdown, backport from 0.3.5.1-alpha):
+    - Avoid a double-close when shutting down a stalled directory
+      connection. Fixes bug 26896; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (HTTP tunnel, backport from 0.3.5.1-alpha):
+    - Fix a bug warning when closing an HTTP tunnel connection due to an
+      HTTP request we couldn't handle. Fixes bug 26470; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (netflow padding, backport from 0.3.5.1-alpha):
+    - Ensure circuitmux queues are empty before scheduling or sending
+      padding. Fixes bug 25505; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.1-alpha):
+    - When the onion service directory can't be created or has the wrong
+      permissions, do not log a stack trace. Fixes bug 27335; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.2-alpha):
+    - Close all SOCKS request (for the same .onion) if the newly fetched
+      descriptor is unusable. Before that, we would close only the first
+      one leaving the other hanging and let to time out by themselves.
+      Fixes bug 27410; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.3-alpha):
+    - When selecting a v3 rendezvous point, don't only look at the
+      protover, but also check whether the curve25519 onion key is
+      present. This way we avoid picking a relay that supports the v3
+      rendezvous but for which we don't have the microdescriptor. Fixes
+      bug 27797; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (protover, backport from 0.3.5.3-alpha):
+    - Reject protocol names containing bytes other than alphanumeric
+      characters and hyphens ([A-Za-z0-9-]). Fixes bug 27316; bugfix
+      on 0.2.9.4-alpha.
+
+  o Minor bugfixes (rust, backport from 0.3.5.1-alpha):
+    - Compute protover votes correctly in the rust version of the
+      protover code. Previously, the protover rewrite in 24031 allowed
+      repeated votes from the same voter for the same protocol version
+      to be counted multiple times in protover_compute_vote(). Fixes bug
+      27649; bugfix on 0.3.3.5-rc.
+    - Reject protover names that contain invalid characters. Fixes bug
+      27687; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (rust, backport from 0.3.5.2-alpha):
+    - protover_all_supported() would attempt to allocate up to 16GB on
+      some inputs, leading to a potential memory DoS. Fixes bug 27206;
+      bugfix on 0.3.3.5-rc.
+
+  o Minor bugfixes (rust, directory authority, to appear in 0.3.5.4-alpha):
+    - Fix an API mismatch in the rust implementation of
+      protover_compute_vote(). This bug could have caused crashes on any
+      directory authorities running Tor with Rust (which we do not yet
+      recommend). Fixes bug 27741; bugfix on 0.3.3.6.
+
+  o Minor bugfixes (rust, to appear in 0.3.5.4-alpha):
+    - Fix a potential null dereference in protover_all_supported(). Add
+      a test for it. Fixes bug 27804; bugfix on 0.3.3.1-alpha.
+    - Return a string that can be safely freed by C code, not one
+      created by the rust allocator, in protover_all_supported(). Fixes
+      bug 27740; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.5.1-alpha):
+    - If a unit test running in a subprocess exits abnormally or with a
+      nonzero status code, treat the test as having failed, even if the
+      test reported success. Without this fix, memory leaks don't cause
+      the tests to fail, even with LeakSanitizer. Fixes bug 27658;
+      bugfix on 0.2.2.4-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.5.3-alpha):
+    - Make the hs_service tests use the same time source when creating
+      the introduction point and when testing it. Now tests work better
+      on very slow systems like ARM or Travis. Fixes bug 27810; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (testing, to appear in 0.3.5.4-alpha):
+    - Treat backtrace test failures as expected on BSD-derived systems
+      (NetBSD, OpenBSD, and macOS/Darwin) until we solve bug 17808.
+      (FreeBSD failures have been treated as expected since 18204 in
+      0.2.8.) Fixes bug 27948; bugfix on 0.2.5.2-alpha.
+
+
+Changes in version 0.2.9.17 - 2018-09-10
+  Tor 0.2.9.17 backports numerous bugfixes from later versions of Tor.
+
+  o Minor features (compatibility, backport from 0.3.4.8):
+    - Tell OpenSSL to maintain backward compatibility with previous
+      RSA1024/DH1024 users in Tor. With OpenSSL 1.1.1-pre6, these
+      ciphers are disabled by default. Closes ticket 27344.
+
+  o Minor features (continuous integration, backport from 0.3.4.7-rc):
+    - Enable macOS builds in our Travis CI configuration. Closes
+      ticket 24629.
+    - Install libcap-dev and libseccomp2-dev so these optional
+      dependencies get tested on Travis CI. Closes ticket 26560.
+    - Run asciidoc during Travis CI. Implements ticket 27087.
+    - Use ccache in our Travis CI configuration. Closes ticket 26952.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the August 7 2018 Maxmind GeoLite2
+      Country database. Closes ticket 27089.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.6-rc):
+    - When compiling with --enable-openbsd-malloc or --enable-tcmalloc,
+      tell the compiler not to include the system malloc implementation.
+      Fixes bug 20424; bugfix on 0.2.0.20-rc.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.7-rc):
+    - Silence a spurious compiler warning on the GetAdaptersAddresses
+      function pointer cast. This issue is already fixed by 26481 in
+      0.3.5 and later, by removing the lookup and cast. Fixes bug 27465;
+      bugfix on 0.2.3.11-alpha.
+    - Stop calling SetProcessDEPPolicy() on 64-bit Windows. It is not
+      supported, and always fails. Some compilers warn about the
+      function pointer cast on 64-bit Windows. Fixes bug 27461; bugfix
+      on 0.2.2.23-alpha.
+
+  o Minor bugfixes (compilation, windows, backport from 0.3.4.7-rc):
+    - Don't link or search for pthreads when building for Windows, even
+      if we are using build environment (like mingw) that provides a
+      pthreads library. Fixes bug 27081; bugfix on 0.1.0.1-rc.
+
+  o Minor bugfixes (continuous integration, backport from 0.3.4.6-rc):
+    - Skip a pair of unreliable key generation tests on Windows, until
+      the underlying issue in bug 26076 is resolved. Fixes bug 26830 and
+      bug 26853; bugfix on 0.2.7.3-rc and 0.3.2.1-alpha respectively.
+
+  o Minor bugfixes (continuous integration, backport from 0.3.4.7-rc):
+    - Pass the module flags to distcheck configure, and log the flags
+      before running configure. (Backported to 0.2.9 and later as a
+      precaution.) Fixes bug 27088; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (continuous integration, backport from 0.3.4.8):
+    - When a Travis build fails, and showing a log fails, keep trying to
+      show the other logs. Fixes bug 27453; bugfix on 0.3.4.7-rc.
+    - When we use echo in Travis, don't pass a --flag as the first
+      argument. Fixes bug 27418; bugfix on 0.3.4.7-rc.
+
+  o Minor bugfixes (directory authority, backport from 0.3.4.6-rc):
+    - When voting for recommended versions, make sure that all of the
+      versions are well-formed and parsable. Fixes bug 26485; bugfix
+      on 0.1.1.6-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.4.7-rc):
+    - Fix a bug in out sandboxing rules for the openat() syscall.
+      Previously, no openat() call would be permitted, which would break
+      filesystem operations on recent glibc versions. Fixes bug 25440;
+      bugfix on 0.2.9.15. Diagnosis and patch from Daniel Pinto.
+
+  o Minor bugfixes (onion services, backport from 0.3.4.8):
+    - Silence a spurious compiler warning in
+      rend_client_send_introduction(). Fixes bug 27463; bugfix
+      on 0.1.1.2-alpha.
+
+  o Minor bugfixes (single onion services, Tor2web, backport from 0.3.4.6-rc):
+    - Log a protocol warning when single onion services or Tor2web clients
+      fail to authenticate direct connections to relays.
+      Fixes bug 26924; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.4.6-rc):
+    - Disable core dumps in test_bt.sh, to avoid failures in "make
+      distcheck". Fixes bug 26787; bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (testing, chutney, backport from 0.3.4.8):
+    - Before running make test-network-all, delete old logs and test
+      result files, to avoid spurious failures. Fixes bug 27295; bugfix
+      on 0.2.7.3-rc.
+
+  o Minor bugfixes (testing, openssl compatibility, backport from 0.3.4.7-rc):
+    - Our "tortls/cert_matches_key" unit test no longer relies on
+      OpenSSL internals. Previously, it relied on unsupported OpenSSL
+      behavior in a way that caused it to crash with OpenSSL 1.0.2p.
+      Fixes bug 27226; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (Windows, compilation, backport from 0.3.4.7-rc):
+    - Silence a compilation warning on MSVC 2017 and clang-cl. Fixes bug
+      27185; bugfix on 0.2.2.2-alpha.
+
+
+Changes in version 0.3.2.12 - 2018-09-10
+  Tor 0.3.2.12 backport numerous fixes from later versions of Tor.
+
+  o Minor features (compatibility, backport from 0.3.4.8):
+    - Tell OpenSSL to maintain backward compatibility with previous
+      RSA1024/DH1024 users in Tor. With OpenSSL 1.1.1-pre6, these
+      ciphers are disabled by default. Closes ticket 27344.
+
+  o Minor features (continuous integration, backport from 0.3.4.7-rc):
+    - Enable macOS builds in our Travis CI configuration. Closes
+      ticket 24629.
+    - Install libcap-dev and libseccomp2-dev so these optional
+      dependencies get tested on Travis CI. Closes ticket 26560.
+    - Run asciidoc during Travis CI. Implements ticket 27087.
+    - Use ccache in our Travis CI configuration. Closes ticket 26952.
+
+  o Minor features (continuous integration, rust, backport from 0.3.4.7-rc):
+    - Use cargo cache in our Travis CI configuration. Closes
+      ticket 26952.
+
+  o Minor features (controller, backport from 0.3.4.6-rc):
+    - The control port now exposes the list of HTTPTunnelPorts and
+      ExtOrPorts via GETINFO net/listeners/httptunnel and
+      net/listeners/extor respectively. Closes ticket 26647.
+
+  o Minor features (directory authorities, backport from 0.3.4.7-rc):
+    - Authorities no longer vote to make the subprotocol version
+      "LinkAuth=1" a requirement: it is unsupportable with NSS, and
+      hasn't been needed since Tor 0.3.0.1-alpha. Closes ticket 27286.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the August 7 2018 Maxmind GeoLite2
+      Country database. Closes ticket 27089.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.6-rc):
+    - When compiling with --enable-openbsd-malloc or --enable-tcmalloc,
+      tell the compiler not to include the system malloc implementation.
+      Fixes bug 20424; bugfix on 0.2.0.20-rc.
+    - Don't try to use a pragma to temporarily disable the
+      -Wunused-const-variable warning if the compiler doesn't support
+      it. Fixes bug 26785; bugfix on 0.3.2.11.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.7-rc):
+    - Silence a spurious compiler warning on the GetAdaptersAddresses
+      function pointer cast. This issue is already fixed by 26481 in
+      0.3.5 and later, by removing the lookup and cast. Fixes bug 27465;
+      bugfix on 0.2.3.11-alpha.
+    - Stop calling SetProcessDEPPolicy() on 64-bit Windows. It is not
+      supported, and always fails. Some compilers warn about the
+      function pointer cast on 64-bit Windows. Fixes bug 27461; bugfix
+      on 0.2.2.23-alpha.
+
+  o Minor bugfixes (compilation, windows, backport from 0.3.4.7-rc):
+    - Don't link or search for pthreads when building for Windows, even
+      if we are using build environment (like mingw) that provides a
+      pthreads library. Fixes bug 27081; bugfix on 0.1.0.1-rc.
+
+  o Minor bugfixes (continuous integration, backport from 0.3.4.6-rc):
+    - Skip a pair of unreliable key generation tests on Windows, until
+      the underlying issue in bug 26076 is resolved. Fixes bug 26830 and
+      bug 26853; bugfix on 0.2.7.3-rc and 0.3.2.1-alpha respectively.
+
+  o Minor bugfixes (continuous integration, backport from 0.3.4.7-rc):
+    - Build with zstd on macOS. Fixes bug 27090; bugfix on 0.3.1.5-alpha.
+    - Pass the module flags to distcheck configure, and log the flags
+      before running configure. (Backported to 0.2.9 and later as a
+      precaution.) Fixes bug 27088; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (continuous integration, backport from 0.3.4.8):
+    - When a Travis build fails, and showing a log fails, keep trying to
+      show the other logs. Fixes bug 27453; bugfix on 0.3.4.7-rc.
+    - When we use echo in Travis, don't pass a --flag as the first
+      argument. Fixes bug 27418; bugfix on 0.3.4.7-rc.
+
+  o Minor bugfixes (directory authority, backport from 0.3.4.6-rc):
+    - When voting for recommended versions, make sure that all of the
+      versions are well-formed and parsable. Fixes bug 26485; bugfix
+      on 0.1.1.6-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.4.7-rc):
+    - Fix a bug in out sandboxing rules for the openat() syscall.
+      Previously, no openat() call would be permitted, which would break
+      filesystem operations on recent glibc versions. Fixes bug 25440;
+      bugfix on 0.2.9.15. Diagnosis and patch from Daniel Pinto.
+
+  o Minor bugfixes (logging, backport from 0.3.4.6-rc):
+    - Improve the log message when connection initiators fail to
+      authenticate direct connections to relays. Fixes bug 26927; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.3.4.7-rc):
+    - Fix bug that causes services to not ever rotate their descriptors
+      if they were getting SIGHUPed often. Fixes bug 26932; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.3.4.8):
+    - Silence a spurious compiler warning in
+      rend_client_send_introduction(). Fixes bug 27463; bugfix
+      on 0.1.1.2-alpha.
+
+  o Minor bugfixes (rust, backport from 0.3.4.7-rc):
+    - Backport test_rust.sh from master. Fixes bug 26497; bugfix
+      on 0.3.1.5-alpha.
+    - Consistently use ../../.. as a fallback for $abs_top_srcdir in
+      test_rust.sh. Fixes bug 27093; bugfix on 0.3.4.3-alpha.
+    - Stop setting $CARGO_HOME. cargo will use the user's $CARGO_HOME, or
+      $HOME/.cargo by default. Fixes bug 26497; bugfix on 0.3.1.5-alpha.
+
+  o Minor bugfixes (single onion services, Tor2web, backport from 0.3.4.6-rc):
+    - Log a protocol warning when single onion services or Tor2web clients
+      fail to authenticate direct connections to relays.
+      Fixes bug 26924; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.4.6-rc):
+    - Disable core dumps in test_bt.sh, to avoid failures in "make
+      distcheck". Fixes bug 26787; bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (testing, chutney, backport from 0.3.4.8):
+    - When running make test-network-all, use the mixed+hs-v2 network.
+      (A previous fix to chutney removed v3 onion services from the
+      mixed+hs-v23 network, so seeing "mixed+hs-v23" in tests is
+      confusing.) Fixes bug 27345; bugfix on 0.3.2.1-alpha.
+    - Before running make test-network-all, delete old logs and test
+      result files, to avoid spurious failures. Fixes bug 27295; bugfix
+      on 0.2.7.3-rc.
+
+  o Minor bugfixes (testing, openssl compatibility):
+    - Our "tortls/cert_matches_key" unit test no longer relies on OpenSSL
+      internals.  Previously, it relied on unsupported OpenSSL behavior in
+      a way that caused it to crash with OpenSSL 1.0.2p. Fixes bug 27226;
+      bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (testing, openssl compatibility, backport from 0.3.4.7-rc):
+    - Our "tortls/cert_matches_key" unit test no longer relies on
+      OpenSSL internals. Previously, it relied on unsupported OpenSSL
+      behavior in a way that caused it to crash with OpenSSL 1.0.2p.
+      Fixes bug 27226; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (Windows, compilation, backport from 0.3.4.7-rc):
+    - Silence a compilation warning on MSVC 2017 and clang-cl. Fixes bug
+      27185; bugfix on 0.2.2.2-alpha.
+
+
+Changes in version 0.3.3.10 - 2018-09-10
+  Tor 0.3.3.10 backports numerous fixes from later versions of Tor.
+
+  o Minor features (bug workaround, backport from 0.3.4.7-rc):
+    - Compile correctly on systems that provide the C11 stdatomic.h
+      header, but where C11 atomic functions don't actually compile.
+      Closes ticket 26779; workaround for Debian issue 903709.
+
+  o Minor features (compatibility, backport from 0.3.4.8):
+    - Tell OpenSSL to maintain backward compatibility with previous
+      RSA1024/DH1024 users in Tor. With OpenSSL 1.1.1-pre6, these
+      ciphers are disabled by default. Closes ticket 27344.
+
+  o Minor features (continuous integration, backport from 0.3.4.7-rc):
+    - Backport Travis rust distcheck to 0.3.3. Closes ticket 24629.
+    - Enable macOS builds in our Travis CI configuration. Closes
+      ticket 24629.
+    - Install libcap-dev and libseccomp2-dev so these optional
+      dependencies get tested on Travis CI. Closes ticket 26560.
+    - Run asciidoc during Travis CI. Implements ticket 27087.
+    - Use ccache in our Travis CI configuration. Closes ticket 26952.
+
+  o Minor features (continuous integration, rust, backport from 0.3.4.7-rc):
+    - Use cargo cache in our Travis CI configuration. Closes
+      ticket 26952.
+
+  o Minor features (controller, backport from 0.3.4.6-rc):
+    - The control port now exposes the list of HTTPTunnelPorts and
+      ExtOrPorts via GETINFO net/listeners/httptunnel and
+      net/listeners/extor respectively. Closes ticket 26647.
+
+  o Minor features (directory authorities, backport from 0.3.4.7-rc):
+    - Authorities no longer vote to make the subprotocol version
+      "LinkAuth=1" a requirement: it is unsupportable with NSS, and
+      hasn't been needed since Tor 0.3.0.1-alpha. Closes ticket 27286.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the August 7 2018 Maxmind GeoLite2
+      Country database. Closes ticket 27089.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.6-rc):
+    - When compiling with --enable-openbsd-malloc or --enable-tcmalloc,
+      tell the compiler not to include the system malloc implementation.
+      Fixes bug 20424; bugfix on 0.2.0.20-rc.
+    - Don't try to use a pragma to temporarily disable the
+      -Wunused-const-variable warning if the compiler doesn't support
+      it. Fixes bug 26785; bugfix on 0.3.2.11.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.7-rc):
+    - Silence a spurious compiler warning on the GetAdaptersAddresses
+      function pointer cast. This issue is already fixed by 26481 in
+      0.3.5 and later, by removing the lookup and cast. Fixes bug 27465;
+      bugfix on 0.2.3.11-alpha.
+    - Stop calling SetProcessDEPPolicy() on 64-bit Windows. It is not
+      supported, and always fails. Some compilers warn about the
+      function pointer cast on 64-bit Windows. Fixes bug 27461; bugfix
+      on 0.2.2.23-alpha.
+
+  o Minor bugfixes (compilation, windows, backport from 0.3.4.7-rc):
+    - Don't link or search for pthreads when building for Windows, even
+      if we are using build environment (like mingw) that provides a
+      pthreads library. Fixes bug 27081; bugfix on 0.1.0.1-rc.
+
+  o Minor bugfixes (continuous integration, backport from 0.3.4.6-rc):
+    - Skip a pair of unreliable key generation tests on Windows, until
+      the underlying issue in bug 26076 is resolved. Fixes bug 26830 and
+      bug 26853; bugfix on 0.2.7.3-rc and 0.3.2.1-alpha respectively.
+
+  o Minor bugfixes (continuous integration, backport from 0.3.4.7-rc):
+    - Build with zstd on macOS. Fixes bug 27090; bugfix on 0.3.1.5-alpha.
+    - Pass the module flags to distcheck configure, and log the flags
+      before running configure. (Backported to 0.2.9 and later as a
+      precaution.) Fixes bug 27088; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (continuous integration, backport from 0.3.4.8):
+    - When a Travis build fails, and showing a log fails, keep trying to
+      show the other logs. Fixes bug 27453; bugfix on 0.3.4.7-rc.
+    - When we use echo in Travis, don't pass a --flag as the first
+      argument. Fixes bug 27418; bugfix on 0.3.4.7-rc.
+
+  o Minor bugfixes (directory authority, backport from 0.3.4.6-rc):
+    - When voting for recommended versions, make sure that all of the
+      versions are well-formed and parsable. Fixes bug 26485; bugfix
+      on 0.1.1.6-alpha.
+
+  o Minor bugfixes (in-process restart, backport from 0.3.4.7-rc):
+    - Always call tor_free_all() when leaving tor_run_main(). When we
+      did not, restarting tor in-process would cause an assertion
+      failure. Fixes bug 26948; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.4.7-rc):
+    - Fix a bug in our sandboxing rules for the openat() syscall.
+      Previously, no openat() call would be permitted, which would break
+      filesystem operations on recent glibc versions. Fixes bug 25440;
+      bugfix on 0.2.9.15. Diagnosis and patch from Daniel Pinto.
+
+  o Minor bugfixes (logging, backport from 0.3.4.6-rc):
+    - Improve the log message when connection initiators fail to
+      authenticate direct connections to relays. Fixes bug 26927; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.3.4.7-rc):
+    - Fix bug that causes services to not ever rotate their descriptors
+      if they were getting SIGHUPed often. Fixes bug 26932; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.3.4.8):
+    - Silence a spurious compiler warning in
+      rend_client_send_introduction(). Fixes bug 27463; bugfix
+      on 0.1.1.2-alpha.
+
+  o Minor bugfixes (portability, backport from 0.3.4.6-rc):
+    - Work around two different bugs in the OS X 10.10 and later SDKs
+      that would prevent us from successfully targeting earlier versions
+      of OS X. Fixes bug 26876; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (portability, backport from 0.3.4.7-rc):
+    - Fix compilation of the unit tests on GNU/Hurd, which does not
+      define PATH_MAX. Fixes bug 26873; bugfix on 0.3.3.1-alpha. Patch
+      from "paulusASol".
+
+  o Minor bugfixes (rust, backport from 0.3.4.7-rc):
+    - Backport test_rust.sh from master. Fixes bug 26497; bugfix
+      on 0.3.1.5-alpha.
+    - Consistently use ../../.. as a fallback for $abs_top_srcdir in
+      test_rust.sh. Fixes bug 27093; bugfix on 0.3.4.3-alpha.
+    - Protover parsing was accepting the presence of whitespace in
+      version strings, which the C implementation would choke on, e.g.
+      "Desc=1\t,2". Fixes bug 27177; bugfix on 0.3.3.5-rc.
+    - Protover parsing was ignoring a 2nd hyphen and everything after
+      it, accepting entries like "Link=1-5-foo". Fixes bug 27164; bugfix
+      on 0.3.3.1-alpha.
+    - Stop setting $CARGO_HOME. cargo will use the user's $CARGO_HOME, or
+      $HOME/.cargo by default. Fixes bug 26497; bugfix on 0.3.1.5-alpha.
+    - cd to ${abs_top_builddir}/src/rust before running cargo in
+      src/test/test_rust.sh. This makes the working directory consistent
+      between builds and tests. Fixes bug 26497; bugfix on 0.3.3.2-alpha.
+
+  o Minor bugfixes (single onion services, Tor2web, backport from 0.3.4.6-rc):
+    - Log a protocol warning when single onion services or Tor2web clients
+      fail to authenticate direct connections to relays.
+      Fixes bug 26924; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.4.6-rc):
+    - Disable core dumps in test_bt.sh, to avoid failures in "make
+      distcheck". Fixes bug 26787; bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (testing, chutney, backport from 0.3.4.8):
+    - When running make test-network-all, use the mixed+hs-v2 network.
+      (A previous fix to chutney removed v3 onion services from the
+      mixed+hs-v23 network, so seeing "mixed+hs-v23" in tests is
+      confusing.) Fixes bug 27345; bugfix on 0.3.2.1-alpha.
+    - Before running make test-network-all, delete old logs and test
+      result files, to avoid spurious failures. Fixes bug 27295; bugfix
+      on 0.2.7.3-rc.
+
+  o Minor bugfixes (testing, openssl compatibility, backport from 0.3.4.7-rc):
+    - Our "tortls/cert_matches_key" unit test no longer relies on
+      OpenSSL internals. Previously, it relied on unsupported OpenSSL
+      behavior in a way that caused it to crash with OpenSSL 1.0.2p.
+      Fixes bug 27226; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (v3 onion services, backport from 0.3.4.6-rc):
+    - Stop sending ed25519 link specifiers in v3 onion service introduce
+      cells and descriptors, when the rendezvous or introduction point
+      doesn't support ed25519 link authentication. Fixes bug 26627;
+      bugfix on 0.3.2.4-alpha.
+
+  o Minor bugfixes (Windows, compilation, backport from 0.3.4.7-rc):
+    - Silence a compilation warning on MSVC 2017 and clang-cl. Fixes bug
+      27185; bugfix on 0.2.2.2-alpha.
+
+
+Changes in version 0.3.4.8 - 2018-09-10
+  Tor 0.3.4.8 is the first stable release in its series; it includes
+  compilation and portability fixes.
+
+  The Tor 0.3.4 series includes improvements for running Tor in
+  low-power and embedded environments, which should help performance in
+  general. We've begun work on better modularity, and included preliminary
+  changes on the directory authority side to accommodate a new bandwidth
+  measurement system.  We've also integrated more continuous-integration
+  systems into our development process, and made corresponding changes to
+  Tor's testing infrastructure.  Finally, we've continued to refine
+  our anti-denial-of-service code.
+
+  Below are the changes since 0.3.3.9. For a list of only the changes
+  since 0.3.4.7-rc, see the ChangeLog file.
+
+  o New system requirements:
+    - Tor no longer tries to support old operating systems without
+      mmap() or some local equivalent. Apparently, compilation on such
+      systems has been broken for some time, without anybody noticing or
+      complaining. Closes ticket 25398.
+
+  o Major features (directory authority, modularization):
+    - The directory authority subsystem has been modularized. The code
+      is now located in src/or/dirauth/, and is compiled in by default.
+      To disable the module, the configure option
+      --disable-module-dirauth has been added. This module may be
+      disabled by default in some future release. Closes ticket 25610.
+
+  o Major features (main loop, CPU usage):
+    - When Tor is disabled (via DisableNetwork or via hibernation), it
+      no longer needs to run any per-second events. This change should
+      make it easier for mobile applications to disable Tor while the
+      device is sleeping, or Tor is not running. Closes ticket 26063.
+    - Tor no longer enables all of its periodic events by default.
+      Previously, Tor would enable all possible main loop events,
+      regardless of whether it needed them. Furthermore, many of these
+      events are now disabled when Tor is hibernating or DisableNetwork
+      is set. This is a big step towards reducing client CPU usage by
+      reducing the amount of wake-ups the daemon does. Closes tickets
+      25376 and 25762.
+    - The bandwidth-limitation logic has been refactored so that
+      bandwidth calculations are performed on-demand, rather than every
+      TokenBucketRefillInterval milliseconds. This change should improve
+      the granularity of our bandwidth calculations, and limit the
+      number of times that the Tor process needs to wake up when it is
+      idle. Closes ticket 25373.
+    - Move responsibility for many operations from a once-per-second
+      callback to a callback that is only scheduled as needed. Moving
+      this functionality has allowed us to disable the callback when
+      Tor's network is disabled. Once enough items are removed from our
+      once-per-second callback, we can eliminate it entirely to conserve
+      CPU when idle. The functionality removed includes: closing
+      connections, circuits, and channels (ticket 25932); consensus
+      voting (25937); flushing log callbacks (25951); honoring delayed
+      SIGNEWNYM requests (25949); rescanning the consensus cache
+      (25931); saving the state file to disk (25948); warning relay
+      operators about unreachable ports (25952); and keeping track of
+      Tor's uptime (26009).
+
+  o Minor features (accounting):
+    - When Tor becomes dormant, it now uses a scheduled event to wake up
+      at the right time. Previously, we would use the per-second timer
+      to check whether to wake up, but we no longer have any per-second
+      timers enabled when the network is disabled. Closes ticket 26064.
+
+  o Minor features (bug workaround):
+    - Compile correctly on systems that provide the C11 stdatomic.h
+      header, but where C11 atomic functions don't actually compile.
+      Closes ticket 26779; workaround for Debian issue 903709.
+
+  o Minor features (code quality):
+    - Add optional spell-checking for the Tor codebase, using the
+      "misspell" program. To use this feature, run "make check-typos".
+      Closes ticket 25024.
+
+  o Minor features (compatibility):
+    - Tell OpenSSL to maintain backward compatibility with previous
+      RSA1024/DH1024 users in Tor. With OpenSSL 1.1.1-pre6, these
+      ciphers are disabled by default. Closes ticket 27344.
+    - Tor now detects versions of OpenSSL 1.1.0 and later compiled with
+      the no-deprecated option, and builds correctly with them. Closes
+      tickets 19429, 19981, and 25353.
+
+  o Minor features (compilation):
+    - When compiling with --enable-openbsd-malloc or --enable-tcmalloc,
+      tell the compiler not to include the system malloc implementation.
+      Fixes bug 20424; bugfix on 0.2.0.20-rc.
+    - Don't try to use a pragma to temporarily disable the
+      -Wunused-const-variable warning if the compiler doesn't support
+      it. Fixes bug 26785; bugfix on 0.3.2.11.
+    - When building Tor, prefer to use Python 3 over Python 2, and more
+      recent (contemplated) versions over older ones. Closes
+      ticket 26372.
+
+  o Minor features (compression, zstd):
+    - When running with zstd, Tor now considers using advanced functions
+      that the zstd maintainers have labeled as potentially unstable. To
+      prevent breakage, Tor will only use this functionality when the
+      runtime version of the zstd library matches the version with which
+      Tor was compiled. Closes ticket 25162.
+
+  o Minor features (configuration):
+    - The "DownloadSchedule" options have been renamed to end with
+      "DownloadInitialDelay". The old names are still allowed, but will
+      produce a warning. Comma-separated lists are still permitted for
+      these options, but all values after the first are ignored (as they
+      have been since 0.2.9). Closes ticket 23354.
+
+  o Minor features (continuous integration):
+    - Log the compiler path and version during Appveyor builds.
+      Implements ticket 27449.
+    - Show config.log and test-suite.log after failed Appveyor builds.
+      Also upload the zipped full logs as a build artifact. Implements
+      ticket 27430.
+    - Backport Travis rust distcheck to 0.3.3. Closes ticket 24629.
+    - Enable macOS builds in our Travis CI configuration. Closes
+      ticket 24629.
+    - Install libcap-dev and libseccomp2-dev so these optional
+      dependencies get tested on Travis CI. Closes ticket 26560.
+    - Only post Appveyor IRC notifications when the build fails.
+      Implements ticket 27275.
+    - Run asciidoc during Travis CI. Implements ticket 27087.
+    - Use ccache in our Travis CI configuration. Closes ticket 26952.
+    - Add the necessary configuration files for continuous integration
+      testing on Windows, via the Appveyor platform. Closes ticket
+      25549. Patches from Marcin Cieślak and Isis Lovecruft.
+
+  o Minor features (continuous integration, rust):
+    - Use cargo cache in our Travis CI configuration. Closes
+      ticket 26952.
+
+  o Minor features (control port):
+    - Introduce GETINFO "current-time/{local,utc}" to return the local
+      and UTC times respectively in ISO format. This helps a controller
+      like Tor Browser detect a time-related error. Closes ticket 25511.
+      Patch by Neel Chauhan.
+    - Introduce new fields to the CIRC_BW event. There are two new
+      fields in each of the read and written directions. The DELIVERED
+      fields report the total valid data on the circuit, as measured by
+      the payload sizes of verified and error-checked relay command
+      cells. The OVERHEAD fields report the total unused bytes in each
+      of these cells. Closes ticket 25903.
+
+  o Minor features (controller):
+    - The control port now exposes the list of HTTPTunnelPorts and
+      ExtOrPorts via GETINFO net/listeners/httptunnel and
+      net/listeners/extor respectively. Closes ticket 26647.
+
+  o Minor features (directory authorities):
+    - Stop warning about incomplete bw lines before the first complete
+      bw line has been found, so that additional header lines can be
+      ignored. Fixes bug 25960; bugfix on 0.2.2.1-alpha
+    - Authorities no longer vote to make the subprotocol version
+      "LinkAuth=1" a requirement: it is unsupportable with NSS, and
+      hasn't been needed since Tor 0.3.0.1-alpha. Closes ticket 27286.
+
+  o Minor features (directory authority):
+    - Directory authorities now open their key-pinning files as O_SYNC,
+      to limit their chances of accidentally writing partial lines.
+      Closes ticket 23909.
+
+  o Minor features (directory authority, forward compatibility):
+    - Make the lines of the measured bandwidth file able to contain
+      their entries in any order. Previously, the node_id entry needed
+      to come first. Closes ticket 26004.
+
+  o Minor features (entry guards):
+    - Introduce a new torrc option NumPrimaryGuards for controlling the
+      number of primary guards. Closes ticket 25843.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the August 7 2018 Maxmind GeoLite2
+      Country database. Closes ticket 27089.
+
+  o Minor features (performance):
+    - Avoid a needless call to malloc() when processing an incoming
+      relay cell. Closes ticket 24914.
+    - Make our timing-wheel code run a tiny bit faster on 32-bit
+      platforms, by preferring 32-bit math to 64-bit. Closes
+      ticket 24688.
+    - Avoid a needless malloc()/free() pair every time we handle an ntor
+      handshake. Closes ticket 25150.
+
+  o Minor features (Rust, portability):
+    - Rust cross-compilation is now supported. Closes ticket 25895.
+
+  o Minor features (testing):
+    - Add a unit test for voting_schedule_get_start_of_next_interval().
+      Closes ticket 26014, and helps make unit test coverage
+      more deterministic.
+    - A new unittests module specifically for testing the functions in
+      the (new-ish) bridges.c module has been created with new
+      unittests, raising the code coverage percentages. Closes 25425.
+    - We now have improved testing for addressmap_get_virtual_address()
+      function. This should improve our test coverage, and make our test
+      coverage more deterministic. Closes ticket 25993.
+
+  o Minor features (timekeeping, circuit scheduling):
+    - When keeping track of how busy each circuit have been recently on
+      a given connection, use coarse-grained monotonic timers rather
+      than gettimeofday(). This change should marginally increase
+      accuracy and performance. Implements part of ticket 25927.
+
+  o Minor features (unit tests):
+    - Test complete bandwidth measurements files, and test that
+      incomplete bandwidth lines only give warnings when the end of the
+      header has not been detected. Fixes bug 25947; bugfix
+      on 0.2.2.1-alpha
+
+  o Minor bugfixes (bandwidth management):
+    - Consider ourselves "low on write bandwidth" if we have exhausted
+      our write bandwidth some time in the last second. This was the
+      documented behavior before, but the actual behavior was to change
+      this value every TokenBucketRefillInterval. Fixes bug 25828;
+      bugfix on 0.2.3.5-alpha.
+
+  o Minor bugfixes (C correctness):
+    - Add a missing lock acquisition in the shutdown code of the control
+      subsystem. Fixes bug 25675; bugfix on 0.2.7.3-rc. Found by
+      Coverity; this is CID 1433643.
+
+  o Minor bugfixes (code style):
+    - Fixed multiple includes of transports.h in src/or/connection.c
+      Fixes bug 25261; bugfix on 0.2.5.1-alpha.
+    - Remove the unused variable n_possible from the function
+      channel_get_for_extend(). Fixes bug 25645; bugfix on 0.2.4.4-alpha
+
+  o Minor bugfixes (compilation):
+    - Silence a spurious compiler warning on the GetAdaptersAddresses
+      function pointer cast. This issue is already fixed by 26481 in
+      0.3.5 and later, by removing the lookup and cast. Fixes bug 27465;
+      bugfix on 0.2.3.11-alpha.
+    - Stop calling SetProcessDEPPolicy() on 64-bit Windows. It is not
+      supported, and always fails. Some compilers warn about the
+      function pointer cast on 64-bit Windows. Fixes bug 27461; bugfix
+      on 0.2.2.23-alpha.
+    - Fix a compilation warning on some versions of GCC when building
+      code that calls routerinfo_get_my_routerinfo() twice, assuming
+      that the second call will succeed if the first one did. Fixes bug
+      26269; bugfix on 0.2.8.2-alpha.
+    - Refrain from compiling unit testing related object files when
+      --disable-unittests is set to configure script. Fixes bug 24891;
+      bugfix on 0.2.5.1-alpha.
+    - The --enable-fatal-warnings flag now affects Rust code as well.
+      Closes ticket 26245.
+    - Avoid a compiler warning when casting the return value of
+      smartlist_len() to double with DEBUG_SMARTLIST enabled. Fixes bug
+      26283; bugfix on 0.2.4.10-alpha.
+
+  o Minor bugfixes (compilation, windows):
+    - Don't link or search for pthreads when building for Windows, even
+      if we are using build environment (like mingw) that provides a
+      pthreads library. Fixes bug 27081; bugfix on 0.1.0.1-rc.
+
+  o Minor bugfixes (continuous integration):
+    - Build with zstd on macOS. Fixes bug 27090; bugfix on 0.3.1.5-alpha.
+    - Skip a pair of unreliable key generation tests on Windows, until
+      the underlying issue in bug 26076 is resolved. Fixes bug 26830 and
+      bug 26853; bugfix on 0.2.7.3-rc and 0.3.2.1-alpha respectively.
+
+  o Minor bugfixes (control port):
+    - Respond with more human-readable error messages to GETINFO exit-
+      policy/* requests. Also, let controller know if an error is
+      transient (response code 551) or not (response code 552). Fixes
+      bug 25852; bugfix on 0.2.8.1-alpha.
+    - Parse the "HSADDRESS=" parameter in HSPOST commands properly.
+      Previously, it was misparsed and ignored. Fixes bug 26523; bugfix
+      on 0.3.3.1-alpha. Patch by "akwizgran".
+    - Make CIRC_BW event reflect the total of all data sent on a
+      circuit, including padding and dropped cells. Also fix a mis-
+      counting bug when STREAM_BW events were enabled. Fixes bug 25400;
+      bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (correctness, flow control):
+    - Upon receiving a stream-level SENDME cell, verify that our window
+      has not grown too large. Fixes bug 26214; bugfix on svn
+      r54 (pre-0.0.1).
+
+  o Minor bugfixes (directory authority):
+    - When voting for recommended versions, make sure that all of the
+      versions are well-formed and parsable. Fixes bug 26485; bugfix
+      on 0.1.1.6-alpha.
+
+  o Minor bugfixes (directory client):
+    - When unverified-consensus is verified, rename it to cached-
+      consenus. Fixes bug 4187; bugfix on 0.2.0.3-alpha.
+    - Fixed launching a certificate fetch always during the scheduled
+      periodic consensus fetch by fetching only in those cases when
+      consensus are waiting for certs. Fixes bug 24740; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (error reporting):
+    - Improve tolerance for directory authorities with skewed clocks.
+      Previously, an authority with a clock more than 60 seconds ahead
+      could cause a client with a correct clock to warn that the
+      client's clock was behind. Now the clocks of a majority of
+      directory authorities have to be ahead of the client before this
+      warning will occur. Fixes bug 25756; bugfix on 0.2.2.25-alpha.
+
+  o Minor bugfixes (in-process restart):
+    - Always call tor_free_all() when leaving tor_run_main(). When we
+      did not, restarting tor in-process would cause an assertion
+      failure. Fixes bug 26948; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (Linux seccomp2 sandbox):
+    - Fix a bug in our sandboxing rules for the openat() syscall.
+      Previously, no openat() call would be permitted, which would break
+      filesystem operations on recent glibc versions. Fixes bug 25440;
+      bugfix on 0.2.9.15. Diagnosis and patch from Daniel Pinto.
+
+  o Minor bugfixes (logging):
+    - Improve the log message when connection initiators fail to
+      authenticate direct connections to relays. Fixes bug 26927; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (onion services):
+    - Silence a spurious compiler warning in
+      rend_client_send_introduction(). Fixes bug 27463; bugfix
+      on 0.1.1.2-alpha.
+    - Fix bug that causes services to not ever rotate their descriptors
+      if they were getting SIGHUPed often. Fixes bug 26932; bugfix
+      on 0.3.2.1-alpha.
+    - Recompute some consensus information after detecting a clock jump,
+      or after transitioning from a non-live consensus to a live
+      consensus. We do this to avoid having an outdated state, and
+      miscalculating the index for next-generation onion services. Fixes
+      bug 24977; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (portability):
+    - Fix compilation of the unit tests on GNU/Hurd, which does not
+      define PATH_MAX. Fixes bug 26873; bugfix on 0.3.3.1-alpha. Patch
+      from "paulusASol".
+    - Work around two different bugs in the OS X 10.10 and later SDKs
+      that would prevent us from successfully targeting earlier versions
+      of OS X. Fixes bug 26876; bugfix on 0.3.3.1-alpha.
+    - Do not align mmap length, as it is not required by POSIX, and the
+      getpagesize function is deprecated. Fixes bug 25399; bugfix
+      on 0.1.1.23.
+
+  o Minor bugfixes (portability, FreeBSD):
+    - In have_enough_mem_for_dircache(), the variable DIRCACHE_MIN_MEM_MB
+      does not stringify on FreeBSD, so we switch to tor_asprintf().
+      Fixes bug 20887; bugfix on 0.2.8.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (relay statistics):
+    - When a relay is collecting internal statistics about how many
+      create cell requests it has seen of each type, accurately count
+      the requests from relays that temporarily fall out of the
+      consensus. (To be extra conservative, we were already ignoring
+      requests from clients in our counts, and we continue ignoring them
+      here.) Fixes bug 24910; bugfix on 0.2.4.17-rc.
+
+  o Minor bugfixes (rust):
+    - Backport test_rust.sh from master. Fixes bug 26497; bugfix
+      on 0.3.1.5-alpha.
+    - Protover parsing was accepting the presence of whitespace in
+      version strings, which the C implementation would choke on, e.g.
+      "Desc=1\t,2". Fixes bug 27177; bugfix on 0.3.3.5-rc.
+    - Protover parsing was ignoring a 2nd hyphen and everything after
+      it, accepting entries like "Link=1-5-foo". Fixes bug 27164; bugfix
+      on 0.3.3.1-alpha.
+    - Stop setting $CARGO_HOME. cargo will use the user's $CARGO_HOME, or
+      $HOME/.cargo by default. Fixes bug 26497; bugfix on 0.3.1.5-alpha.
+    - cd to ${abs_top_builddir}/src/rust before running cargo in
+      src/test/test_rust.sh. This makes the working directory consistent
+      between builds and tests. Fixes bug 26497; bugfix on 0.3.3.2-alpha.
+
+  o Minor bugfixes (single onion services, Tor2web):
+    - Log a protocol warning when single onion services or Tor2web
+      clients fail to authenticate direct connections to relays. Fixes
+      bug 26924; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (test coverage tools):
+    - Update our "cov-diff" script to handle output from the latest
+      version of gcov, and to remove extraneous timestamp information
+      from its output. Fixes bugs 26101 and 26102; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Disable core dumps in test_bt.sh, to avoid failures in "make
+      distcheck". Fixes bug 26787; bugfix on 0.2.5.2-alpha.
+    - When testing workqueue event-cancellation, make sure that we
+      actually cancel an event, and that cancel each event with equal
+      probability. (It was previously possible, though extremely
+      unlikely, for our event-canceling test not to cancel any events.)
+      Fixes bug 26008; bugfix on 0.2.6.3-alpha.
+    - Repeat part of the test in test_client_pick_intro() a number of
+      times, to give it consistent coverage. Fixes bug 25996; bugfix
+      on 0.3.2.1-alpha.
+    - Remove randomness from the hs_common/responsible_hsdirs test, so
+      that it always takes the same path through the function it tests.
+      Fixes bug 25997; bugfix on 0.3.2.1-alpha.
+    - Change the behavior of the "channel/outbound" test so that it
+      never causes a 10-second rollover for the EWMA circuitmux code.
+      Previously, this behavior would happen randomly, and result in
+      fluctuating test coverage. Fixes bug 25994; bugfix
+      on 0.3.3.1-alpha.
+    - Use X509_new() to allocate certificates that will be freed later
+      with X509_free(). Previously, some parts of the unit tests had
+      used tor_malloc_zero(), which is incorrect, and which caused test
+      failures on Windows when they were built with extra hardening.
+      Fixes bugs 25943 and 25944; bugfix on 0.2.8.1-alpha. Patch by
+      Marcin Cieślak.
+    - While running the circuit_timeout test, fix the PRNG to a
+      deterministic AES stream, so that the test coverage from this test
+      will itself be deterministic. Fixes bug 25995; bugfix
+      on 0.2.2.2-alpha.
+
+  o Minor bugfixes (testing, bootstrap):
+    - When calculating bootstrap progress, check exit policies and the
+      exit flag. Previously, Tor would only check the exit flag, which
+      caused race conditions in small and fast networks like chutney.
+      Fixes bug 27236; bugfix on 0.2.6.3-alpha.
+
+  o Minor bugfixes (testing, chutney):
+    - When running make test-network-all, use the mixed+hs-v2 network.
+      (A previous fix to chutney removed v3 onion services from the
+      mixed+hs-v23 network, so seeing "mixed+hs-v23" in tests is
+      confusing.) Fixes bug 27345; bugfix on 0.3.2.1-alpha.
+    - Before running make test-network-all, delete old logs and test
+      result files, to avoid spurious failures. Fixes bug 27295; bugfix
+      on 0.2.7.3-rc.
+
+  o Minor bugfixes (testing, openssl compatibility):
+    - Our "tortls/cert_matches_key" unit test no longer relies on
+      OpenSSL internals. Previously, it relied on unsupported OpenSSL
+      behavior in a way that caused it to crash with OpenSSL 1.0.2p.
+      Fixes bug 27226; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (v3 onion services):
+    - Stop sending ed25519 link specifiers in v3 onion service introduce
+      cells and descriptors, when the rendezvous or introduction point
+      doesn't support ed25519 link authentication. Fixes bug 26627;
+      bugfix on 0.3.2.4-alpha.
+
+  o Minor bugfixes (vanguards):
+    - Allow the last hop in a vanguard circuit to be the same as our
+      first, to prevent the adversary from influencing guard node choice
+      by choice of last hop. Also prevent the creation of A - B - A
+      paths, or A - A paths, which are forbidden by relays. Fixes bug
+      25870; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (Windows, compilation):
+    - Silence a compilation warning on MSVC 2017 and clang-cl. Fixes bug
+      27185; bugfix on 0.2.2.2-alpha.
+
+  o Code simplification and refactoring:
+    - Remove duplicate code in parse_{c,s}method_line and bootstrap
+      their functionalities into a single function. Fixes bug 6236;
+      bugfix on 0.2.3.6-alpha.
+    - We remove the PortForwsrding and PortForwardingHelper options,
+      related functions, and the port_forwarding tests. These options
+      were used by the now-deprecated Vidalia to help ordinary users
+      become Tor relays or bridges. Closes ticket 25409. Patch by
+      Neel Chauhan.
+    - In order to make the OR and dir checking function in router.c less
+      confusing we renamed some functions and
+      consider_testing_reachability() has been split into
+      router_should_check_reachability() and
+      router_do_reachability_checks(). Also we improved the documentation
+      in some functions. Closes ticket 18918.
+    - Initial work to isolate Libevent usage to a handful of modules in
+      our codebase, to simplify our call structure, and so that we can
+      more easily change event loops in the future if needed. Closes
+      ticket 23750.
+    - Introduce a function to call getsockname() and return tor_addr_t,
+      to save a little complexity throughout the codebase. Closes
+      ticket 18105.
+    - Make hsdir_index in node_t a hsdir_index_t rather than a pointer
+      as hsdir_index is always present. Also, we move hsdir_index_t into
+      or.h. Closes ticket 23094. Patch by Neel Chauhan.
+    - Merge functions used for describing nodes and suppress the
+      functions that do not allocate memory for the output buffer
+      string. NODE_DESC_BUF_LEN constant and format_node_description()
+      function cannot be used externally from router.c module anymore.
+      Closes ticket 25432. Patch by valentecaio.
+    - Our main loop has been simplified so that all important operations
+      happen inside events. Previously, some operations had to happen
+      outside the event loop, to prevent infinite sequences of event
+      activations. Closes ticket 25374.
+    - Put a SHA1 public key digest in hs_service_intro_point_t, and use
+      it in register_intro_circ() and service_intro_point_new(). This
+      prevents the digest from being re-calculated each time. Closes
+      ticket 23107. Patch by Neel Chauhan.
+    - Refactor token-bucket implementations to use a common backend.
+      Closes ticket 25766.
+    - Remove extern declaration of stats_n_seconds_working variable from
+      main, protecting its accesses with get_uptime() and reset_uptime()
+      functions. Closes ticket 25081, patch by “valentecaio”.
+    - Remove our previous logic for "cached gettimeofday()" -- our
+      coarse monotonic timers are fast enough for this purpose, and far
+      less error-prone. Implements part of ticket 25927.
+    - Remove the return value for fascist_firewall_choose_address_base(),
+      and sister functions such as fascist_firewall_choose_address_node()
+      and fascist_firewall_choose_address_rs(). Also, while we're here,
+      initialize the ap argument as leaving it uninitialized can pose a
+      security hazard. Closes ticket 24734. Patch by Neel Chauhan.
+    - Rename two fields of connection_t struct. timestamp_lastwritten is
+      renamed to timestamp_last_write_allowed and timestamp_lastread is
+      renamed to timestamp_last_read_allowed. Closes ticket 24714, patch
+      by "valentecaio".
+    - Since Tor requires C99, remove our old workaround code for libc
+      implementations where free(NULL) doesn't work. Closes ticket 24484.
+    - Use our standard rate-limiting code to deal with excessive
+      libevent failures, rather than the hand-rolled logic we had
+      before. Closes ticket 26016.
+    - We remove the return value of node_get_prim_orport() and
+      node_get_prim_dirport(), and introduce node_get_prim_orport() in
+      node_ipv6_or_preferred() and node_ipv6_dir_preferred() in order to
+      check for a null address. Closes ticket 23873. Patch by
+      Neel Chauhan.
+    - We switch to should_record_bridge_info() in
+      geoip_note_client_seen() and options_need_geoip_info() instead of
+      accessing the configuration values directly. Fixes bug 25290;
+      bugfix on 0.2.1.6-alpha. Patch by Neel Chauhan.
+
+  o Deprecated features:
+    - As we are not recommending 0.2.5 anymore, we require relays that
+      once had an ed25519 key associated with their RSA key to always
+      have that key, instead of allowing them to drop back to a version
+      that didn't support ed25519. This means they need to use a new RSA
+      key if they want to downgrade to an older version of tor without
+      ed25519. Closes ticket 20522.
+
+  o Removed features:
+    - Directory authorities will no longer support voting according to
+      any consensus method before consensus method 25. This keeps
+      authorities compatible with all authorities running 0.2.9.8 and
+      later, and does not break any clients or relays. Implements ticket
+      24378 and proposal 290.
+    - The PortForwarding and PortForwardingHelper features have been
+      removed. The reasoning is, given that implementations of NAT
+      traversal protocols within common consumer grade routers are
+      frequently buggy, and that the target audience for a NAT punching
+      feature is a perhaps less-technically-inclined relay operator,
+      when the helper fails to setup traversal the problems are usually
+      deep, ugly, and very router specific, making them horrendously
+      impossible for technical support to reliable assist with, and thus
+      resulting in frustration all around. Unfortunately, relay
+      operators who would like to run relays behind NATs will need to
+      become more familiar with the port forwarding configurations on
+      their local router. Closes 25409.
+    - The TestingEnableTbEmptyEvent option has been removed. It was used
+      in testing simulations to measure how often connection buckets
+      were emptied, in order to improve our scheduling, but it has not
+      been actively used in years. Closes ticket 25760.
+    - The old "round-robin" circuit multiplexer (circuitmux)
+      implementation has been removed, along with a fairly large set of
+      code that existed to support it. It has not been the default
+      circuitmux since we introduced the "EWMA" circuitmux in 0.2.4.x,
+      but it still required an unreasonable amount of memory and CPU.
+      Closes ticket 25268.
+
+
+Changes in version 0.3.3.9 - 2018-07-13
+  Tor 0.3.3.9 moves to a new bridge authority, meaning people running
+  bridge relays should upgrade.
+
+  o Directory authority changes:
+    - The "Bifroest" bridge authority has been retired; the new bridge
+      authority is "Serge", and it is operated by George from the
+      TorBSD project. Closes ticket 26771.
+
+
+Changes in version 0.3.2.11 - 2018-07-13
+  Tor 0.3.2.11 moves to a new bridge authority, meaning people running
+  bridge relays should upgrade. We also take this opportunity to backport
+  other minor fixes.
+
+  o Directory authority changes:
+    - The "Bifroest" bridge authority has been retired; the new bridge
+      authority is "Serge", and it is operated by George from the
+      TorBSD project. Closes ticket 26771.
+
+  o Directory authority changes (backport from 0.3.3.7):
+    - Add an IPv6 address for the "dannenberg" directory authority.
+      Closes ticket 26343.
+
+  o Major bugfixes (directory authorities, backport from 0.3.4.1-alpha):
+    - When directory authorities read a zero-byte bandwidth file, they
+      would previously log a warning with the contents of an
+      uninitialised buffer. They now log a warning about the empty file
+      instead. Fixes bug 26007; bugfix on 0.2.2.1-alpha.
+
+  o Major bugfixes (onion service, backport from 0.3.4.1-alpha):
+    - Correctly detect when onion services get disabled after HUP. Fixes
+      bug 25761; bugfix on 0.3.2.1.
+
+  o Minor features (sandbox, backport from 0.3.3.4-alpha):
+    - Explicitly permit the poll() system call when the Linux
+      seccomp2-based sandbox is enabled: apparently, some versions of
+      libc use poll() when calling getpwnam(). Closes ticket 25313.
+
+  o Minor feature (continuous integration, backport from 0.3.3.5-rc):
+    - Update the Travis CI configuration to use the stable Rust channel,
+      now that we have decided to require that. Closes ticket 25714.
+
+  o Minor features (continuous integration, backport from 0.3.4.1-alpha):
+    - Our .travis.yml configuration now includes support for testing the
+      results of "make distcheck". (It's not uncommon for "make check"
+      to pass but "make distcheck" to fail.) Closes ticket 25814.
+    - Our Travis CI configuration now integrates with the Coveralls
+      coverage analysis tool. Closes ticket 25818.
+
+  o Minor features (relay, diagnostic, backport from 0.3.4.3-alpha):
+    - Add several checks to detect whether Tor relays are uploading
+      their descriptors without specifying why they regenerated them.
+      Diagnostic for ticket 25686.
+
+  o Minor features (compilation, backport from 0.3.4.4-rc):
+    - When building Tor, prefer to use Python 3 over Python 2, and more
+      recent (contemplated) versions over older ones. Closes
+      ticket 26372.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the July 3 2018 Maxmind GeoLite2
+      Country database. Closes ticket 26674.
+
+  o Minor bugfixes (correctness, client, backport from 0.3.4.1-alpha):
+    - Upon receiving a malformed connected cell, stop processing the
+      cell immediately. Previously we would mark the connection for
+      close, but continue processing the cell as if the connection were
+      open. Fixes bug 26072; bugfix on 0.2.4.7-alpha.
+
+  o Minor bugfixes (Linux seccomp2 sandbox, backport from 0.3.4.1-alpha):
+    - Allow the nanosleep() system call, which glibc uses to implement
+      sleep() and usleep(). Fixes bug 24969; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (testing, compatibility, backport from 0.3.4.4-rc):
+    - When running the hs_ntor_ref.py test, make sure only to pass
+      strings (rather than "bytes" objects) to the Python subprocess
+      module. Python 3 on Windows seems to require this. Fixes bug
+      26535; bugfix on 0.3.1.1-alpha.
+    - When running the ntor_ref.py test, make sure only to pass strings
+      (rather than "bytes" objects) to the Python subprocess module.
+      Python 3 on Windows seems to require this. Fixes bug 26535; bugfix
+      on 0.2.5.5-alpha.
+
+  o Minor bugfixes (compatibility, openssl, backport from 0.3.4.2-alpha):
+    - Work around a change in OpenSSL 1.1.1 where return values that
+      would previously indicate "no password" now indicate an empty
+      password. Without this workaround, Tor instances running with
+      OpenSSL 1.1.1 would accept descriptors that other Tor instances
+      would reject. Fixes bug 26116; bugfix on 0.2.5.16.
+
+  o Minor bugfixes (documentation, backport from 0.3.3.5-rc):
+    - Document that the PerConnBW{Rate,Burst} options will fall back to
+      their corresponding consensus parameters only if those parameters
+      are set. Previously we had claimed that these values would always
+      be set in the consensus. Fixes bug 25296; bugfix on 0.2.2.7-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.4-rc):
+    - Fix a compilation warning on some versions of GCC when building
+      code that calls routerinfo_get_my_routerinfo() twice, assuming
+      that the second call will succeed if the first one did. Fixes bug
+      26269; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (client, backport from 0.3.4.1-alpha):
+    - Don't consider Tor running as a client if the ControlPort is open,
+      but no actual client ports are open. Fixes bug 26062; bugfix
+      on 0.2.9.4-alpha.
+
+  o Minor bugfixes (hardening, backport from 0.3.4.2-alpha):
+    - Prevent a possible out-of-bounds smartlist read in
+      protover_compute_vote(). Fixes bug 26196; bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (C correctness, backport from 0.3.3.4-alpha):
+    - Fix a very unlikely (impossible, we believe) null pointer
+      dereference. Fixes bug 25629; bugfix on 0.2.9.15. Found by
+      Coverity; this is CID 1430932.
+
+  o Minor bugfixes (onion service, backport from 0.3.4.1-alpha):
+    - Fix a memory leak when a v3 onion service is configured and gets a
+      SIGHUP signal. Fixes bug 25901; bugfix on 0.3.2.1-alpha.
+    - When parsing the descriptor signature, look for the token plus an
+      extra white-space at the end. This is more correct but also will
+      allow us to support new fields that might start with "signature".
+      Fixes bug 26069; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (relay, backport from 0.3.4.3-alpha):
+    - Relays now correctly block attempts to re-extend to the previous
+      relay by Ed25519 identity. Previously they would warn in this
+      case, but not actually reject the attempt. Fixes bug 26158; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (relay, crash, backport from 0.3.4.1-alpha):
+    - Avoid a crash when running with DirPort set but ORPort turned off.
+      Fixes a case of bug 23693; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.2-alpha):
+    - Silence unused-const-variable warnings in zstd.h with some GCC
+      versions. Fixes bug 26272; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.3.4-alpha):
+    - Avoid intermittent test failures due to a test that had relied on
+      onion service introduction point creation finishing within 5
+      seconds of real clock time. Fixes bug 25450; bugfix
+      on 0.3.1.3-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.3.4-alpha):
+    - Fix a C99 compliance issue in our configuration script that caused
+      compilation issues when compiling Tor with certain versions of
+      xtools. Fixes bug 25474; bugfix on 0.3.2.5-alpha.
+
+  o Minor bugfixes (memory, correctness, backport from 0.3.4.4-rc):
+    - Fix a number of small memory leaks identified by coverity. Fixes
+      bug 26467; bugfix on numerous Tor versions.
+
+  o Code simplification and refactoring (backport from 0.3.3.5-rc):
+    - Move the list of default directory authorities to its own file.
+      Closes ticket 24854. Patch by "beastr0".
+
+
+Changes in version 0.2.9.16 - 2018-07-13
+  Tor 0.2.9.16 moves to a new bridge authority, meaning people running
+  bridge relays should upgrade. We also take this opportunity to backport
+  other minor fixes.
+
+  o Directory authority changes:
+    - The "Bifroest" bridge authority has been retired; the new bridge
+      authority is "Serge", and it is operated by George from the
+      TorBSD project. Closes ticket 26771.
+
+  o Directory authority changes (backport from 0.3.3.7):
+    - Add an IPv6 address for the "dannenberg" directory authority.
+      Closes ticket 26343.
+
+  o Major bugfixes (directory authorities, backport from 0.3.4.1-alpha):
+    - When directory authorities read a zero-byte bandwidth file, they
+      would previously log a warning with the contents of an
+      uninitialised buffer. They now log a warning about the empty file
+      instead. Fixes bug 26007; bugfix on 0.2.2.1-alpha.
+
+  o Minor features (sandbox, backport from 0.3.3.4-alpha):
+    - Explicitly permit the poll() system call when the Linux
+      seccomp2-based sandbox is enabled: apparently, some versions of
+      libc use poll() when calling getpwnam(). Closes ticket 25313.
+
+  o Minor features (continuous integration, backport from 0.3.4.1-alpha):
+    - Our .travis.yml configuration now includes support for testing the
+      results of "make distcheck". (It's not uncommon for "make check"
+      to pass but "make distcheck" to fail.) Closes ticket 25814.
+    - Our Travis CI configuration now integrates with the Coveralls
+      coverage analysis tool. Closes ticket 25818.
+
+  o Minor features (compilation, backport from 0.3.4.4-rc):
+    - When building Tor, prefer to use Python 3 over Python 2, and more
+      recent (contemplated) versions over older ones. Closes
+      ticket 26372.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the July 3 2018 Maxmind GeoLite2
+      Country database. Closes ticket 26674.
+
+  o Minor bugfixes (correctness, client, backport from 0.3.4.1-alpha):
+    - Upon receiving a malformed connected cell, stop processing the
+      cell immediately. Previously we would mark the connection for
+      close, but continue processing the cell as if the connection were
+      open. Fixes bug 26072; bugfix on 0.2.4.7-alpha.
+
+  o Minor bugfixes (Linux seccomp2 sandbox, backport from 0.3.4.1-alpha):
+    - Allow the nanosleep() system call, which glibc uses to implement
+      sleep() and usleep(). Fixes bug 24969; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (testing, compatibility, backport from 0.3.4.4-rc):
+    - When running the ntor_ref.py test, make sure only to pass strings
+      (rather than "bytes" objects) to the Python subprocess module.
+      Python 3 on Windows seems to require this. Fixes bug 26535; bugfix
+      on 0.2.5.5-alpha.
+
+  o Minor bugfixes (compatibility, openssl, backport from 0.3.4.2-alpha):
+    - Work around a change in OpenSSL 1.1.1 where return values that
+      would previously indicate "no password" now indicate an empty
+      password. Without this workaround, Tor instances running with
+      OpenSSL 1.1.1 would accept descriptors that other Tor instances
+      would reject. Fixes bug 26116; bugfix on 0.2.5.16.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.4-rc):
+    - Fix a compilation warning on some versions of GCC when building
+      code that calls routerinfo_get_my_routerinfo() twice, assuming
+      that the second call will succeed if the first one did. Fixes bug
+      26269; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (client, backport from 0.3.4.1-alpha):
+    - Don't consider Tor running as a client if the ControlPort is open,
+      but no actual client ports are open. Fixes bug 26062; bugfix
+      on 0.2.9.4-alpha.
+
+  o Minor bugfixes (hardening, backport from 0.3.4.2-alpha):
+    - Prevent a possible out-of-bounds smartlist read in
+      protover_compute_vote(). Fixes bug 26196; bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (C correctness, backport from 0.3.3.4-alpha):
+    - Fix a very unlikely (impossible, we believe) null pointer
+      dereference. Fixes bug 25629; bugfix on 0.2.9.15. Found by
+      Coverity; this is CID 1430932.
+
+  o Minor bugfixes (memory, correctness, backport from 0.3.4.4-rc):
+    - Fix a number of small memory leaks identified by coverity. Fixes
+      bug 26467; bugfix on numerous Tor versions.
+
+  o Code simplification and refactoring (backport from 0.3.3.5-rc):
+    - Move the list of default directory authorities to its own file.
+      Closes ticket 24854. Patch by "beastr0".
+
+
+Changes in version 0.3.3.8 - 2018-07-09
+  Tor 0.3.3.8 backports several changes from the 0.3.4.x series, including
+  fixes for a memory leak affecting directory authorities.
+
+  o Major bugfixes (directory authority, backport from 0.3.4.3-alpha):
+    - Stop leaking memory on directory authorities when planning to
+      vote. This bug was crashing authorities by exhausting their
+      memory. Fixes bug 26435; bugfix on 0.3.3.6.
+
+  o Major bugfixes (rust, testing, backport from 0.3.4.3-alpha):
+    - Make sure that failing tests in Rust will actually cause the build
+      to fail: previously, they were ignored. Fixes bug 26258; bugfix
+      on 0.3.3.4-alpha.
+
+  o Minor features (compilation, backport from 0.3.4.4-rc):
+    - When building Tor, prefer to use Python 3 over Python 2, and more
+      recent (contemplated) versions over older ones. Closes
+      ticket 26372.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the July 3 2018 Maxmind GeoLite2
+      Country database. Closes ticket 26674.
+
+  o Minor features (relay, diagnostic, backport from 0.3.4.3-alpha):
+    - Add several checks to detect whether Tor relays are uploading
+      their descriptors without specifying why they regenerated them.
+      Diagnostic for ticket 25686.
+
+  o Minor bugfixes (circuit path selection, backport from 0.3.4.1-alpha):
+    - Don't count path selection failures as circuit build failures.
+      This change should eliminate cases where Tor blames its guard or
+      the network for situations like insufficient microdescriptors
+      and/or overly restrictive torrc settings. Fixes bug 25705; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.4-rc):
+    - Fix a compilation warning on some versions of GCC when building
+      code that calls routerinfo_get_my_routerinfo() twice, assuming
+      that the second call will succeed if the first one did. Fixes bug
+      26269; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (control port, backport from 0.3.4.4-rc):
+    - Handle the HSADDRESS= argument to the HSPOST command properly.
+      (Previously, this argument was misparsed and thus ignored.) Fixes
+      bug 26523; bugfix on 0.3.3.1-alpha. Patch by "akwizgran".
+
+  o Minor bugfixes (memory, correctness, backport from 0.3.4.4-rc):
+    - Fix a number of small memory leaks identified by coverity. Fixes
+      bug 26467; bugfix on numerous Tor versions.
+
+  o Minor bugfixes (relay, backport from 0.3.4.3-alpha):
+    - Relays now correctly block attempts to re-extend to the previous
+      relay by Ed25519 identity. Previously they would warn in this
+      case, but not actually reject the attempt. Fixes bug 26158; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (restart-in-process, backport from 0.3.4.1-alpha):
+    - When shutting down, Tor now clears all the flags in the control.c
+      module. This should prevent a bug where authentication cookies are
+      not generated on restart. Fixes bug 25512; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (testing, compatibility, backport from 0.3.4.4-rc):
+    - When running the hs_ntor_ref.py test, make sure only to pass
+      strings (rather than "bytes" objects) to the Python subprocess
+      module. Python 3 on Windows seems to require this. Fixes bug
+      26535; bugfix on 0.3.1.1-alpha.
+    - When running the ntor_ref.py test, make sure only to pass strings
+      (rather than "bytes" objects) to the Python subprocess module.
+      Python 3 on Windows seems to require this. Fixes bug 26535; bugfix
+      on 0.2.5.5-alpha.
+
+
+Changes in version 0.3.3.7 - 2018-06-12
+  Tor 0.3.3.7 backports several changes from the 0.3.4.x series, including
+  fixes for bugs affecting compatibility and stability.
+
+  o Directory authority changes:
+    - Add an IPv6 address for the "dannenberg" directory authority.
+      Closes ticket 26343.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the June 7 2018 Maxmind GeoLite2
+      Country database. Closes ticket 26351.
+
+  o Minor bugfixes (compatibility, openssl, backport from 0.3.4.2-alpha):
+    - Work around a change in OpenSSL 1.1.1 where return values that
+      would previously indicate "no password" now indicate an empty
+      password. Without this workaround, Tor instances running with
+      OpenSSL 1.1.1 would accept descriptors that other Tor instances
+      would reject. Fixes bug 26116; bugfix on 0.2.5.16.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.2-alpha):
+    - Silence unused-const-variable warnings in zstd.h with some GCC
+      versions. Fixes bug 26272; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (controller, backport from 0.3.4.2-alpha):
+    - Improve accuracy of the BUILDTIMEOUT_SET control port event's
+      TIMEOUT_RATE and CLOSE_RATE fields. (We were previously
+      miscounting the total number of circuits for these field values.)
+      Fixes bug 26121; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (hardening, backport from 0.3.4.2-alpha):
+    - Prevent a possible out-of-bounds smartlist read in
+      protover_compute_vote(). Fixes bug 26196; bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (path selection, backport from 0.3.4.1-alpha):
+    - Only select relays when they have the descriptors we prefer to use
+      for them. This change fixes a bug where we could select a relay
+      because it had _some_ descriptor, but reject it later with a
+      nonfatal assertion error because it didn't have the exact one we
+      wanted. Fixes bugs 25691 and 25692; bugfix on 0.3.3.4-alpha.
+
+
+Changes in version 0.3.3.6 - 2018-05-22
+  Tor 0.3.3.6 is the first stable release in the 0.3.3 series. It
+  backports several important fixes from the 0.3.4.1-alpha.
+
+  The Tor 0.3.3 series includes controller support and other
+  improvements for v3 onion services, official support for embedding Tor
+  within other applications, and our first non-trivial module written in
+  the Rust programming language. (Rust is still not enabled by default
+  when building Tor.) And as usual, there are numerous other smaller
+  bugfixes, features, and improvements.
+
+  Below are the changes since 0.3.2.10. For a list of only the changes
+  since 0.3.3.5-rc, see the ChangeLog file.
+
+  o New system requirements:
+    - When built with Rust, Tor now depends on version 0.2.39 of the
+      libc crate. Closes tickets 25310 and 25664.
+
+  o Major features (embedding):
+    - There is now a documented stable API for programs that need to
+      embed Tor. See tor_api.h for full documentation and known bugs.
+      Closes ticket 23684.
+    - Tor now has support for restarting in the same process.
+      Controllers that run Tor using the "tor_api.h" interface can now
+      restart Tor after Tor has exited. This support is incomplete,
+      however: we fixed crash bugs that prevented it from working at
+      all, but many bugs probably remain, including a possibility of
+      security issues. Implements ticket 24581.
+
+  o Major features (IPv6, directory documents):
+    - Add consensus method 27, which adds IPv6 ORPorts to the microdesc
+      consensus. This information makes it easier for IPv6 clients to
+      bootstrap and choose reachable entry guards. Implements
+      ticket 23826.
+    - Add consensus method 28, which removes IPv6 ORPorts from
+      microdescriptors. Now that the consensus contains IPv6 ORPorts,
+      they are redundant in microdescs. This change will be used by Tor
+      clients on 0.2.8.x and later. (That is to say, with all Tor
+      clients that have IPv6 bootstrap and guard support.) Implements
+      ticket 23828.
+    - Expand the documentation for AuthDirHasIPv6Connectivity when it is
+      set by different numbers of authorities. Fixes 23870
+      on 0.2.4.1-alpha.
+
+  o Major features (onion service v3, control port):
+    - The control port now supports commands and events for v3 onion
+      services. It is now possible to create ephemeral v3 services using
+      ADD_ONION. Additionally, several events (HS_DESC, HS_DESC_CONTENT,
+      CIRC and CIRC_MINOR) and commands (GETINFO, HSPOST, ADD_ONION and
+      DEL_ONION) have been extended to support v3 onion services. Closes
+      ticket 20699; implements proposal 284.
+
+  o Major features (onion services):
+    - Provide torrc options to pin the second and third hops of onion
+      service circuits to a list of nodes. The option HSLayer2Guards
+      pins the second hop, and the option HSLayer3Guards pins the third
+      hop. These options are for use in conjunction with experiments
+      with "vanguards" for preventing guard enumeration attacks. Closes
+      ticket 13837.
+    - When v3 onion service clients send introduce cells, they now
+      include the IPv6 address of the rendezvous point, if it has one.
+      Current v3 onion services running 0.3.2 ignore IPv6 addresses, but
+      in future Tor versions, IPv6-only v3 single onion services will be
+      able to use IPv6 addresses to connect directly to the rendezvous
+      point. Closes ticket 23577. Patch by Neel Chauhan.
+
+  o Major features (relay):
+    - Implement an option, ReducedExitPolicy, to allow an Tor exit relay
+      operator to use a more reasonable ("reduced") exit policy, rather
+      than the default one. If you want to run an exit node without
+      thinking too hard about which ports to allow, this one is for you.
+      Closes ticket 13605. Patch from Neel Chauhan.
+
+  o Major features (rust, portability, experimental):
+    - Tor now ships with an optional implementation of one of its
+      smaller modules (protover.c) in the Rust programming language. To
+      try it out, install a Rust build environment, and configure Tor
+      with "--enable-rust --enable-cargo-online-mode". This should not
+      cause any user-visible changes, but should help us gain more
+      experience with Rust, and plan future Rust integration work.
+      Implementation by Chelsea Komlo. Closes ticket 22840.
+
+  o Major bugfixes (directory authorities, security, backport from 0.3.4.1-alpha):
+    - When directory authorities read a zero-byte bandwidth file, they
+      would previously log a warning with the contents of an
+      uninitialised buffer. They now log a warning about the empty file
+      instead. Fixes bug 26007; bugfix on 0.2.2.1-alpha.
+
+  o Major bugfixes (security, directory authority, denial-of-service):
+    - Fix a bug that could have allowed an attacker to force a directory
+      authority to use up all its RAM by passing it a maliciously
+      crafted protocol versions string. Fixes bug 25517; bugfix on
+      0.2.9.4-alpha. This issue is also tracked as TROVE-2018-005.
+
+  o Major bugfixes (crash, backport from 0.3.4.1-alpha):
+    - Avoid a rare assertion failure in the circuit build timeout code
+      if we fail to allow any circuits to actually complete. Fixes bug
+      25733; bugfix on 0.2.2.2-alpha.
+
+  o Major bugfixes (netflow padding):
+    - Stop adding unneeded channel padding right after we finish
+      flushing to a connection that has been trying to flush for many
+      seconds. Instead, treat all partial or complete flushes as
+      activity on the channel, which will defer the time until we need
+      to add padding. This fix should resolve confusing and scary log
+      messages like "Channel padding timeout scheduled 221453ms in the
+      past." Fixes bug 22212; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (networking):
+    - Tor will no longer reject IPv6 address strings from Tor Browser
+      when they are passed as hostnames in SOCKS5 requests. Fixes bug
+      25036, bugfix on Tor 0.3.1.2.
+
+  o Major bugfixes (onion service, backport from 0.3.4.1-alpha):
+    - Correctly detect when onion services get disabled after HUP. Fixes
+      bug 25761; bugfix on 0.3.2.1.
+
+  o Major bugfixes (performance, load balancing):
+    - Directory authorities no longer vote in favor of the Guard flag
+      for relays without directory support. Starting in Tor
+      0.3.0.1-alpha, clients have been avoiding using such relays in the
+      Guard position, leading to increasingly broken load balancing for
+      the 5%-or-so of Guards that don't advertise directory support.
+      Fixes bug 22310; bugfix on 0.3.0.6.
+
+  o Major bugfixes (relay):
+    - If we have failed to connect to a relay and received a connection
+      refused, timeout, or similar error (at the TCP level), do not try
+      that same address/port again for 60 seconds after the failure has
+      occurred. Fixes bug 24767; bugfix on 0.0.6.
+
+  o Major bugfixes (relay, denial of service, backport from 0.3.4.1-alpha):
+    - Impose a limit on circuit cell queue size. The limit can be
+      controlled by a consensus parameter. Fixes bug 25226; bugfix
+      on 0.2.4.14-alpha.
+
+  o Minor features (cleanup):
+    - Tor now deletes the CookieAuthFile and ExtORPortCookieAuthFile
+      when it stops. Closes ticket 23271.
+
+  o Minor features (compatibility, backport from 0.3.4.1-alpha):
+    - Avoid some compilation warnings with recent versions of LibreSSL.
+      Closes ticket 26006.
+
+  o Minor features (config options):
+    - Change the way the default value for MaxMemInQueues is calculated.
+      We now use 40% of the hardware RAM if the system has 8 GB RAM or
+      more. Otherwise we use the former value of 75%. Closes
+      ticket 24782.
+
+  o Minor features (continuous integration):
+    - Update the Travis CI configuration to use the stable Rust channel,
+      now that we have decided to require that. Closes ticket 25714.
+
+  o Minor features (continuous integration, backport from 0.3.4.1-alpha):
+    - Our .travis.yml configuration now includes support for testing the
+      results of "make distcheck". (It's not uncommon for "make check"
+      to pass but "make distcheck" to fail.) Closes ticket 25814.
+    - Our Travis CI configuration now integrates with the Coveralls
+      coverage analysis tool. Closes ticket 25818.
+
+  o Minor features (defensive programming):
+    - Most of the functions in Tor that free objects have been replaced
+      with macros that free the objects and set the corresponding
+      pointers to NULL. This change should help prevent a large class of
+      dangling pointer bugs. Closes ticket 24337.
+    - Where possible, the tor_free() macro now only evaluates its input
+      once. Part of ticket 24337.
+    - Check that microdesc ed25519 ids are non-zero in
+      node_get_ed25519_id() before returning them. Implements ticket
+      24001, patch by "aruna1234".
+
+  o Minor features (directory authority):
+    - When directory authorities are unable to add signatures to a
+      pending consensus, log the reason why. Closes ticket 24849.
+
+  o Minor features (embedding):
+    - Tor can now start with a preauthenticated control connection
+      created by the process that launched it. This feature is meant for
+      use by programs that want to launch and manage a Tor process
+      without allowing other programs to manage it as well. For more
+      information, see the __OwningControllerFD option documented in
+      control-spec.txt. Closes ticket 23900.
+    - On most errors that would cause Tor to exit, it now tries to
+      return from the tor_main() function, rather than calling the
+      system exit() function. Most users won't notice a difference here,
+      but it should be significant for programs that run Tor inside a
+      separate thread: they should now be able to survive Tor's exit
+      conditions rather than having Tor shut down the entire process.
+      Closes ticket 23848.
+    - Applications that want to embed Tor can now tell Tor not to
+      register any of its own POSIX signal handlers, using the
+      __DisableSignalHandlers option. Closes ticket 24588.
+
+  o Minor features (fallback directory list):
+    - Avoid selecting fallbacks that change their IP addresses too
+      often. Select more fallbacks by ignoring the Guard flag, and
+      allowing lower cutoffs for the Running and V2Dir flags. Also allow
+      a lower bandwidth, and a higher number of fallbacks per operator
+      (5% of the list). Implements ticket 24785.
+    - Update the fallback whitelist and blacklist based on opt-ins and
+      relay changes. Closes tickets 22321, 24678, 22527, 24135,
+      and 24695.
+
+  o Minor features (fallback directory mirror configuration):
+    - Add a nickname to each fallback in a C comment. This makes it
+      easier for operators to find their relays, and allows stem to use
+      nicknames to identify fallbacks. Implements ticket 24600.
+    - Add a type and version header to the fallback directory mirror
+      file. Also add a delimiter to the end of each fallback entry. This
+      helps external parsers like stem and Relay Search. Implements
+      ticket 24725.
+    - Add an extrainfo cache flag for each fallback in a C comment. This
+      allows stem to use fallbacks to fetch extra-info documents, rather
+      than using authorities. Implements ticket 22759.
+    - Add the generateFallbackDirLine.py script for automatically
+      generating fallback directory mirror lines from relay fingerprints.
+      No more typos! Add the lookupFallbackDirContact.py script for
+      automatically looking up operator contact info from relay
+      fingerprints. Implements ticket 24706, patch by teor and atagar.
+    - Reject any fallback directory mirror that serves an expired
+      consensus. Implements ticket 20942, patch by "minik".
+    - Remove commas and equals signs from external string inputs to the
+      fallback list. This avoids format confusion attacks. Implements
+      ticket 24726.
+    - Remove the "weight=10" line from fallback directory mirror
+      entries. Ticket 24681 will maintain the current fallback weights
+      by changing Tor's default fallback weight to 10. Implements
+      ticket 24679.
+    - Stop logging excessive information about fallback netblocks.
+      Implements ticket 24791.
+
+  o Minor features (forward-compatibility):
+    - If a relay supports some link authentication protocol that we do
+      not recognize, then include that relay's ed25519 key when telling
+      other relays to extend to it. Previously, we treated future
+      versions as if they were too old to support ed25519 link
+      authentication. Closes ticket 20895.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 1 2018 Maxmind GeoLite2 Country
+      database. Closes ticket 26104.
+
+  o Minor features (heartbeat):
+    - Add onion service information to our heartbeat logs, displaying
+      stats about the activity of configured onion services. Closes
+      ticket 24896.
+
+  o Minor features (instrumentation, development):
+    - Add the MainloopStats option to allow developers to get
+      instrumentation information from the main event loop via the
+      heartbeat messages. We hope to use this to improve Tor's behavior
+      when it's trying to sleep. Closes ticket 24605.
+
+  o Minor features (IPv6):
+    - Make IPv6-only clients wait for microdescs for relays, even if we
+      were previously using descriptors (or were using them as a bridge)
+      and have a cached descriptor for them. Implements ticket 23827.
+    - When a consensus has IPv6 ORPorts, make IPv6-only clients use
+      them, rather than waiting to download microdescriptors. Implements
+      ticket 23827.
+
+  o Minor features (log messages):
+    - Improve log message in the out-of-memory handler to include
+      information about memory usage from the different compression
+      backends. Closes ticket 25372.
+    - Improve a warning message that happens when we fail to re-parse an
+      old router because of an expired certificate. Closes ticket 20020.
+    - Make the log more quantitative when we hit MaxMemInQueues
+      threshold exposing some values. Closes ticket 24501.
+
+  o Minor features (logging):
+    - Clarify the log messages produced when getrandom() or a related
+      entropy-generation mechanism gives an error. Closes ticket 25120.
+    - Added support for the Android logging subsystem. Closes
+      ticket 24362.
+
+  o Minor features (performance):
+    - Support predictive circuit building for onion service circuits
+      with multiple layers of guards. Closes ticket 23101.
+    - Use stdatomic.h where available, rather than mutexes, to implement
+      atomic_counter_t. Closes ticket 23953.
+
+  o Minor features (performance, 32-bit):
+    - Improve performance on 32-bit systems by avoiding 64-bit division
+      when calculating the timestamp in milliseconds for channel padding
+      computations. Implements ticket 24613.
+    - Improve performance on 32-bit systems by avoiding 64-bit division
+      when timestamping cells and buffer chunks for OOM calculations.
+      Implements ticket 24374.
+
+  o Minor features (performance, OSX, iOS):
+    - Use the mach_approximate_time() function (when available) to
+      implement coarse monotonic time. Having a coarse time function
+      should avoid a large number of system calls, and improve
+      performance slightly, especially under load. Closes ticket 24427.
+
+  o Minor features (performance, windows):
+    - Improve performance on Windows Vista and Windows 7 by adjusting
+      TCP send window size according to the recommendation from
+      SIO_IDEAL_SEND_BACKLOG_QUERY. Closes ticket 22798. Patch
+      from Vort.
+
+  o Minor features (sandbox):
+    - Explicitly permit the poll() system call when the Linux
+      seccomp2-based sandbox is enabled: apparently, some versions of
+      libc use poll() when calling getpwnam(). Closes ticket 25313.
+
+  o Minor features (storage, configuration):
+    - Users can store cached directory documents somewhere other than
+      the DataDirectory by using the CacheDirectory option. Similarly,
+      the storage location for relay's keys can be overridden with the
+      KeyDirectory option. Closes ticket 22703.
+
+  o Minor features (testing):
+    - Add a "make test-rust" target to run the rust tests only. Closes
+      ticket 25071.
+
+  o Minor features (testing, debugging, embedding):
+    - For development purposes, Tor now has a mode in which it runs for
+      a few seconds, then stops, and starts again without exiting the
+      process. This mode is meant to help us debug various issues with
+      ticket 23847. To use this feature, compile with
+      --enable-restart-debugging, and set the TOR_DEBUG_RESTART
+      environment variable. This is expected to crash a lot, and is
+      really meant for developers only. It will likely be removed in a
+      future release. Implements ticket 24583.
+
+  o Minor bugfixes (build, rust):
+    - Fix output of autoconf checks to display success messages for Rust
+      dependencies and a suitable rustc compiler version. Fixes bug
+      24612; bugfix on 0.3.1.3-alpha.
+    - Don't pass the --quiet option to cargo: it seems to suppress some
+      errors, which is not what we want to do when building. Fixes bug
+      24518; bugfix on 0.3.1.7.
+    - Build correctly when building from outside Tor's source tree with
+      the TOR_RUST_DEPENDENCIES option set. Fixes bug 22768; bugfix
+      on 0.3.1.7.
+
+  o Minor bugfixes (C correctness):
+    - Fix a very unlikely (impossible, we believe) null pointer
+      dereference. Fixes bug 25629; bugfix on 0.2.9.15. Found by
+      Coverity; this is CID 1430932.
+
+  o Minor bugfixes (channel, client):
+    - Better identify client connection when reporting to the geoip
+      client cache. Fixes bug 24904; bugfix on 0.3.1.7.
+
+  o Minor bugfixes (circuit, cannibalization):
+    - Don't cannibalize preemptively-built circuits if we no longer
+      recognize their first hop. This situation can happen if our Guard
+      relay went off the consensus after the circuit was created. Fixes
+      bug 24469; bugfix on 0.0.6.
+
+  o Minor bugfixes (client, backport from 0.3.4.1-alpha):
+    - Don't consider Tor running as a client if the ControlPort is open,
+      but no actual client ports are open. Fixes bug 26062; bugfix
+      on 0.2.9.4-alpha.
+
+  o Minor bugfixes (compilation):
+    - Fix a C99 compliance issue in our configuration script that caused
+      compilation issues when compiling Tor with certain versions of
+      xtools. Fixes bug 25474; bugfix on 0.3.2.5-alpha.
+
+  o Minor bugfixes (controller):
+    - Restore the correct operation of the RESOLVE command, which had
+      been broken since we added the ability to enable/disable DNS on
+      specific listener ports. Fixes bug 25617; bugfix on 0.2.9.3-alpha.
+    - Avoid a (nonfatal) assertion failure when extending a one-hop
+      circuit from the controller to become a multihop circuit. Fixes
+      bug 24903; bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (correctness):
+    - Remove a nonworking, unnecessary check to see whether a circuit
+      hop's identity digest was set when the circuit failed. Fixes bug
+      24927; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (correctness, client, backport from 0.3.4.1-alpha):
+    - Upon receiving a malformed connected cell, stop processing the
+      cell immediately. Previously we would mark the connection for
+      close, but continue processing the cell as if the connection were
+      open. Fixes bug 26072; bugfix on 0.2.4.7-alpha.
+
+  o Minor bugfixes (directory authorities, IPv6):
+    - When creating a routerstatus (vote) from a routerinfo (descriptor),
+      set the IPv6 address to the unspecified IPv6 address, and
+      explicitly initialize the port to zero. Fixes bug 24488; bugfix
+      on 0.2.4.1-alpha.
+
+  o Minor bugfixes (documentation):
+    - Document that the PerConnBW{Rate,Burst} options will fall back to
+      their corresponding consensus parameters only if those parameters
+      are set. Previously we had claimed that these values would always
+      be set in the consensus. Fixes bug 25296; bugfix on 0.2.2.7-alpha.
+
+  o Minor bugfixes (documentation, backport from 0.3.4.1-alpha):
+    - Stop saying in the manual that clients cache ipv4 dns answers from
+      exit relays. We haven't used them since 0.2.6.3-alpha, and in
+      ticket 24050 we stopped even caching them as of 0.3.2.6-alpha, but
+      we forgot to say so in the man page. Fixes bug 26052; bugfix
+      on 0.3.2.6-alpha.
+
+  o Minor bugfixes (exit relay DNS retries):
+    - Re-attempt timed-out DNS queries 3 times before failure, since our
+      timeout is 5 seconds for them, but clients wait 10-15. Also allow
+      slightly more timeouts per resolver when an exit has multiple
+      resolvers configured. Fixes bug 21394; bugfix on 0.3.1.9.
+
+  o Minor bugfixes (fallback directory mirrors):
+    - Make updateFallbackDirs.py search harder for python. (Some OSs
+      don't put it in /usr/bin.) Fixes bug 24708; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (hibernation, bandwidth accounting, shutdown):
+    - When hibernating, close connections normally and allow them to
+      flush. Fixes bug 23571; bugfix on 0.2.4.7-alpha. Also fixes
+      bug 7267.
+    - Do not attempt to launch self-reachability tests when entering
+      hibernation. Fixes a case of bug 12062; bugfix on 0.0.9pre5.
+    - Resolve several bugs related to descriptor fetching on bridge
+      clients with bandwidth accounting enabled. (This combination is
+      not recommended!) Fixes a case of bug 12062; bugfix
+      on 0.2.0.3-alpha.
+    - When hibernating, do not attempt to launch DNS checks. Fixes a
+      case of bug 12062; bugfix on 0.1.2.2-alpha.
+    - When hibernating, do not try to upload or download descriptors.
+      Fixes a case of bug 12062; bugfix on 0.0.9pre5.
+
+  o Minor bugfixes (IPv6, bridges):
+    - Tor now always sets IPv6 preferences for bridges. Fixes bug 24573;
+      bugfix on 0.2.8.2-alpha.
+    - Tor now sets IPv6 address in the routerstatus as well as in the
+      router descriptors when updating addresses for a bridge. Closes
+      ticket 24572; bugfix on 0.2.4.5-alpha. Patch by "ffmancera".
+
+  o Minor bugfixes (Linux seccomp2 sandbox):
+    - When running with the sandbox enabled, reload configuration files
+      correctly even when %include was used. Previously we would crash.
+      Fixes bug 22605; bugfix on 0.3.1. Patch from Daniel Pinto.
+
+  o Minor bugfixes (Linux seccomp2 sandbox, backport from 0.3.4.1-alpha):
+    - Allow the nanosleep() system call, which glibc uses to implement
+      sleep() and usleep(). Fixes bug 24969; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Fix a (mostly harmless) race condition when invoking
+      LOG_PROTOCOL_WARN message from a subthread while the torrc options
+      are changing. Fixes bug 23954; bugfix on 0.1.1.9-alpha.
+
+  o Minor bugfixes (man page, SocksPort):
+    - Remove dead code from the old "SocksSocket" option, and rename
+      SocksSocketsGroupWritable to UnixSocksGroupWritable. The old
+      option still works, but is deprecated. Fixes bug 24343; bugfix
+      on 0.2.6.3.
+
+  o Minor bugfixes (memory leaks):
+    - Avoid possible at-exit memory leaks related to use of Libevent's
+      event_base_once() function. (This function tends to leak memory if
+      the event_base is closed before the event fires.) Fixes bug 24584;
+      bugfix on 0.2.8.1-alpha.
+    - Fix a harmless memory leak in tor-resolve. Fixes bug 24582; bugfix
+      on 0.2.1.1-alpha.
+
+  o Minor bugfixes (network IPv6 test):
+    - Tor's test scripts now check if "ping -6 ::1" works when the user
+      runs "make test-network-all". Fixes bug 24677; bugfix on
+      0.2.9.3-alpha. Patch by "ffmancera".
+
+  o Minor bugfixes (networking):
+    - string_is_valid_hostname() will not consider IP strings to be
+      valid hostnames. Fixes bug 25055; bugfix on Tor 0.2.5.5.
+
+  o Minor bugfixes (onion service v3):
+    - Avoid an assertion failure when the next onion service descriptor
+      rotation type is out of sync with the consensus's valid-after
+      time. Instead, log a warning message with extra information, so we
+      can better hunt down the cause of this assertion. Fixes bug 25306;
+      bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion service, backport from 0.3.4.1-alpha):
+    - Fix a memory leak when a v3 onion service is configured and gets a
+      SIGHUP signal. Fixes bug 25901; bugfix on 0.3.2.1-alpha.
+    - When parsing the descriptor signature, look for the token plus an
+      extra white-space at the end. This is more correct but also will
+      allow us to support new fields that might start with "signature".
+      Fixes bug 26069; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (onion services):
+    - If we are configured to offer a single onion service, don't log
+      long-term established one hop rendezvous points in the heartbeat.
+      Fixes bug 25116; bugfix on 0.2.9.6-rc.
+
+  o Minor bugfixes (performance):
+    - Reduce the number of circuits that will be opened at once during
+      the circuit build timeout phase. This is done by increasing the
+      idle timeout to 3 minutes, and lowering the maximum number of
+      concurrent learning circuits to 10. Fixes bug 24769; bugfix
+      on 0.3.1.1-alpha.
+    - Avoid calling protocol_list_supports_protocol() from inside tight
+      loops when running with cached routerinfo_t objects. Instead,
+      summarize the relevant protocols as flags in the routerinfo_t, as
+      we do for routerstatus_t objects. This change simplifies our code
+      a little, and saves a large amount of short-term memory allocation
+      operations. Fixes bug 25008; bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (performance, timeouts):
+    - Consider circuits for timeout as soon as they complete a hop. This
+      is more accurate than applying the timeout in
+      circuit_expire_building() because that function is only called
+      once per second, which is now too slow for typical timeouts on the
+      current network. Fixes bug 23114; bugfix on 0.2.2.2-alpha.
+    - Use onion service circuits (and other circuits longer than 3 hops)
+      to calculate a circuit build timeout. Previously, Tor only
+      calculated its build timeout based on circuits that planned to be
+      exactly 3 hops long. With this change, we include measurements
+      from all circuits at the point where they complete their third
+      hop. Fixes bug 23100; bugfix on 0.2.2.2-alpha.
+
+  o Minor bugfixes (relay, crash, backport from 0.3.4.1-alpha):
+    - Avoid a crash when running with DirPort set but ORPort turned off.
+      Fixes a case of bug 23693; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (Rust FFI):
+    - Fix a minor memory leak which would happen whenever the C code
+      would call the Rust implementation of
+      protover_get_supported_protocols(). This was due to the C version
+      returning a static string, whereas the Rust version newly allocated
+      a CString to pass across the FFI boundary. Consequently, the C
+      code was not expecting to need to free() what it was given. Fixes
+      bug 25127; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (spelling):
+    - Use the "misspell" tool to detect and fix typos throughout the
+      source code. Fixes bug 23650; bugfix on various versions of Tor.
+      Patch from Deepesh Pathak.
+
+  o Minor bugfixes (testing):
+    - Avoid intermittent test failures due to a test that had relied on
+      onion service introduction point creation finishing within 5
+      seconds of real clock time. Fixes bug 25450; bugfix
+      on 0.3.1.3-alpha.
+    - Give out Exit flags in bootstrapping networks. Fixes bug 24137;
+      bugfix on 0.2.3.1-alpha.
+
+  o Minor bugfixes (unit test, monotonic time):
+    - Increase a constant (1msec to 10msec) in the monotonic time test
+      that makes sure the nsec/usec/msec times read are synchronized.
+      This change was needed to accommodate slow systems like armel or
+      when the clock_gettime() is not a VDSO on the running kernel.
+      Fixes bug 25113; bugfix on 0.2.9.1.
+
+  o Code simplification and refactoring:
+    - Move the list of default directory authorities to its own file.
+      Closes ticket 24854. Patch by "beastr0".
+    - Remove the old (deterministic) directory retry logic entirely:
+      We've used exponential backoff exclusively for some time. Closes
+      ticket 23814.
+    - Remove the unused nodelist_recompute_all_hsdir_indices(). Closes
+      ticket 25108.
+    - Remove a series of counters used to track circuit extend attempts
+      and connection status but that in reality we aren't using for
+      anything other than stats logged by a SIGUSR1 signal. Closes
+      ticket 25163.
+    - Remove /usr/athena from search path in configure.ac. Closes
+      ticket 24363.
+    - Remove duplicate code in node_has_curve25519_onion_key() and
+      node_get_curve25519_onion_key(), and add a check for a zero
+      microdesc curve25519 onion key. Closes ticket 23966, patch by
+      "aruna1234" and teor.
+    - Rewrite channel_rsa_id_group_set_badness to reduce temporary
+      memory allocations with large numbers of OR connections (e.g.
+      relays). Closes ticket 24119.
+    - Separate the function that deletes ephemeral files when Tor
+      stops gracefully.
+    - Small changes to Tor's buf_t API to make it suitable for use as a
+      general-purpose safe string constructor. Closes ticket 22342.
+    - Switch -Wnormalized=id to -Wnormalized=nfkc in configure.ac to
+      avoid source code identifier confusion. Closes ticket 24467.
+    - The tor_git_revision[] constant no longer needs to be redeclared
+      by everything that links against the rest of Tor. Done as part of
+      ticket 23845, to simplify our external API.
+    - We make extend_info_from_node() use node_get_curve25519_onion_key()
+      introduced in ticket 23577 to access the curve25519 public keys
+      rather than accessing it directly. Closes ticket 23760. Patch by
+      Neel Chauhan.
+    - Add a function to log channels' scheduler state changes to aid
+      debugging efforts. Closes ticket 24531.
+
+  o Documentation:
+    - Improved the documentation of AccountingStart parameter. Closes
+      ticket 23635.
+    - Update the documentation for "Log" to include the current list of
+      logging domains. Closes ticket 25378.
+    - Add documentation on how to build tor with Rust dependencies
+      without having to be online. Closes ticket 22907; bugfix
+      on 0.3.0.3-alpha.
+    - Clarify the behavior of RelayBandwidth{Rate,Burst} with client
+      traffic. Closes ticket 24318.
+    - Document that OutboundBindAddress doesn't apply to DNS requests.
+      Closes ticket 22145. Patch from Aruna Maurya.
+
+  o Code simplification and refactoring (channels):
+    - Remove the incoming and outgoing channel queues. These were never
+      used, but still took up a step in our fast path.
+    - The majority of the channel unit tests have been rewritten and the
+      code coverage has now been raised to 83.6% for channel.c. Closes
+      ticket 23709.
+    - Remove other dead code from the channel subsystem: All together,
+      this cleanup has removed more than 1500 lines of code overall and
+      adding very little except for unit test.
+
+  o Code simplification and refactoring (circuit rendezvous):
+    - Split the client-side rendezvous circuit lookup into two
+      functions: one that returns only established circuits and another
+      that returns all kinds of circuits. Closes ticket 23459.
+
+  o Code simplification and refactoring (controller):
+    - Make most of the variables in networkstatus_getinfo_by_purpose()
+      const. Implements ticket 24489.
+
+  o Documentation (backport from 0.3.4.1-alpha):
+    - Correct an IPv6 error in the documentation for ExitPolicy. Closes
+      ticket 25857. Patch from "CTassisF".
+
+  o Documentation (man page):
+    - The HiddenServiceVersion torrc option accepts only one number:
+      either version 2 or 3. Closes ticket 25026; bugfix
+      on 0.3.2.2-alpha.
+
+  o Documentation (manpage, denial of service):
+    - Provide more detail about the denial-of-service options, by
+      listing each mitigation and explaining how they relate. Closes
+      ticket 25248.
+
+
+Changes in version 0.3.1.10 - 2018-03-03
+  Tor 0.3.1.10 backports a number of bugfixes, including important fixes for
+  security issues.
+
+  It includes an important security fix for a remote crash attack
+  against directory authorities, tracked as TROVE-2018-001.
+
+  This release also backports our new system for improved resistance to
+  denial-of-service attacks against relays.
+
+  This release also fixes several minor bugs and annoyances from
+  earlier releases.
+
+  All directory authorities should upgrade to one of the versions
+  released today. Relays running 0.3.1.x may wish to update to one of
+  the versions released today, for the DoS mitigations.
+
+  Please note: according to our release calendar, Tor 0.3.1 will no
+  longer be supported after 1 July 2018. If you will be running Tor
+  after that date, you should make sure to plan to upgrade to the latest
+  stable version, or downgrade to 0.2.9 (which will receive long-term
+  support).
+
+  o Major bugfixes (denial-of-service, directory authority, backport from 0.3.3.3-alpha):
+    - Fix a protocol-list handling bug that could be used to remotely crash
+      directory authorities with a null-pointer exception. Fixes bug 25074;
+      bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001 and
+      CVE-2018-0490.
+
+  o Major features (denial-of-service mitigation, backport from 0.3.3.2-alpha):
+    - Give relays some defenses against the recent network overload. We
+      start with three defenses (default parameters in parentheses).
+      First: if a single client address makes too many concurrent
+      connections (>100), hang up on further connections. Second: if a
+      single client address makes circuits too quickly (more than 3 per
+      second, with an allowed burst of 90) while also having too many
+      connections open (3), refuse new create cells for the next while
+      (1-2 hours). Third: if a client asks to establish a rendezvous
+      point to you directly, ignore the request. These defenses can be
+      manually controlled by new torrc options, but relays will also
+      take guidance from consensus parameters, so there's no need to
+      configure anything manually. Implements ticket 24902.
+
+  o Minor features (linux seccomp2 sandbox, backport from 0.3.2.5-alpha):
+    - Update the sandbox rules so that they should now work correctly
+      with Glibc 2.26. Closes ticket 24315.
+
+  o Major bugfixes (onion services, retry behavior, backport from 0.3.3.1-alpha):
+    - Fix an "off by 2" error in counting rendezvous failures on the
+      onion service side. While we thought we would stop the rendezvous
+      attempt after one failed circuit, we were actually making three
+      circuit attempts before giving up. Now switch to a default of 2,
+      and allow the consensus parameter "hs_service_max_rdv_failures" to
+      override. Fixes bug 24895; bugfix on 0.0.6.
+
+  o Major bugfixes (protocol versions, backport from 0.3.3.2-alpha):
+    - Add Link protocol version 5 to the supported protocols list. Fixes
+      bug 25070; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (relay, backport from 0.3.3.1-alpha):
+    - Fix a set of false positives where relays would consider
+      connections to other relays as being client-only connections (and
+      thus e.g. deserving different link padding schemes) if those
+      relays fell out of the consensus briefly. Now we look only at the
+      initial handshake and whether the connection authenticated as a
+      relay. Fixes bug 24898; bugfix on 0.3.1.1-alpha.
+
+  o Minor features (denial-of-service avoidance, backport from 0.3.3.2-alpha):
+    - Make our OOM handler aware of the geoip client history cache so it
+      doesn't fill up the memory. This check is important for IPv6 and
+      our DoS mitigation subsystem. Closes ticket 25122.
+
+  o Minor feature (relay statistics, backport from 0.3.2.6-alpha):
+    - Change relay bandwidth reporting stats interval from 4 hours to 24
+      hours in order to reduce the efficiency of guard discovery
+      attacks. Fixes ticket 23856.
+
+  o Minor features (compatibility, OpenSSL, backport from 0.3.3.3-alpha):
+    - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released.
+      Previous versions of Tor would not have worked with OpenSSL 1.1.1,
+      since they neither disabled TLS 1.3 nor enabled any of the
+      ciphersuites it requires. Now we enable the TLS 1.3 ciphersuites.
+      Closes ticket 24978.
+
+  o Minor features (fallback directory mirrors, backport from 0.3.2.9):
+    - The fallback directory list has been re-generated based on the
+      current status of the network. Tor uses fallback directories to
+      bootstrap when it doesn't yet have up-to-date directory
+      information. Closes ticket 24801.
+    - Make the default DirAuthorityFallbackRate 0.1, so that clients
+      prefer to bootstrap from fallback directory mirrors. This is a
+      follow-up to 24679, which removed weights from the default
+      fallbacks. Implements ticket 24681.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 7 2018 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfix (channel connection, backport from 0.3.3.2-alpha):
+    - Use the actual observed address of an incoming relay connection,
+      not the canonical address of the relay from its descriptor, when
+      making decisions about how to handle the incoming connection.
+      Fixes bug 24952; bugfix on 0.2.4.11-alpha. Patch by "ffmancera".
+
+  o Minor bugfix (directory authority, backport from 0.3.3.2-alpha):
+    - Directory authorities, when refusing a descriptor from a rejected
+      relay, now explicitly tell the relay (in its logs) to set a valid
+      ContactInfo address and contact the bad-relays@ mailing list.
+      Fixes bug 25170; bugfix on 0.2.9.1.
+
+  o Minor bugfixes (address selection, backport from 0.3.2.9):
+    - When the fascist_firewall_choose_address_ functions don't find a
+      reachable address, set the returned address to the null address
+      and port. This is a precautionary measure, because some callers do
+      not check the return value. Fixes bug 24736; bugfix
+      on 0.2.8.2-alpha.
+
+  o Major bugfixes (bootstrapping, backport from 0.3.2.5-alpha):
+    - Fetch descriptors aggressively whenever we lack enough to build
+      circuits, regardless of how many descriptors we are missing.
+      Previously, we would delay launching the fetch when we had fewer
+      than 15 missing descriptors, even if some of those descriptors
+      were blocking circuits from building. Fixes bug 23985; bugfix on
+      0.1.1.11-alpha. The effects of this bug became worse in
+      0.3.0.3-alpha, when we began treating missing descriptors from our
+      primary guards as a reason to delay circuits.
+    - Don't try fetching microdescriptors from relays that have failed
+      to deliver them in the past. Fixes bug 23817; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.2.7-rc):
+    - Fix a signed/unsigned comparison warning introduced by our fix to
+      TROVE-2017-009. Fixes bug 24480; bugfix on 0.2.5.16.
+
+  o Minor bugfixes (control port, linux seccomp2 sandbox, backport from 0.3.2.5-alpha):
+    - Avoid a crash when attempting to use the seccomp2 sandbox together
+      with the OwningControllerProcess feature. Fixes bug 24198; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (denial-of-service, backport from 0.3.3.3-alpha):
+    - Fix a possible crash on malformed consensus. If a consensus had
+      contained an unparseable protocol line, it could have made clients
+      and relays crash with a null-pointer exception. To exploit this
+      issue, however, an attacker would need to be able to subvert the
+      directory authority system. Fixes bug 25251; bugfix on
+      0.2.9.4-alpha. Also tracked as TROVE-2018-004.
+
+  o Minor bugfixes (directory cache, backport from 0.3.2.5-alpha):
+    - Recover better from empty or corrupt files in the consensus cache
+      directory. Fixes bug 24099; bugfix on 0.3.1.1-alpha.
+    - When a consensus diff calculation is only partially successful,
+      only record the successful parts as having succeeded. Partial
+      success can happen if (for example) one compression method fails
+      but the others succeed. Previously we misrecorded all the
+      calculations as having succeeded, which would later cause a
+      nonfatal assertion failure. Fixes bug 24086; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (entry guards, backport from 0.3.2.3-alpha):
+    - Tor now updates its guard state when it reads a consensus
+      regardless of whether it's missing descriptors. That makes tor use
+      its primary guards to fetch descriptors in some edge cases where
+      it would previously have used fallback directories. Fixes bug
+      23862; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (logging, backport from 0.3.3.2-alpha):
+    - Don't treat inability to store a cached consensus object as a bug:
+      it can happen normally when we are out of disk space. Fixes bug
+      24859; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (memory usage, backport from 0.3.2.8-rc):
+    - When queuing DESTROY cells on a channel, only queue the circuit-id
+      and reason fields: not the entire 514-byte cell. This fix should
+      help mitigate any bugs or attacks that fill up these queues, and
+      free more RAM for other uses. Fixes bug 24666; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (network layer, backport from 0.3.2.5-alpha):
+    - When closing a connection via close_connection_immediately(), we
+      mark it as "not blocked on bandwidth", to prevent later calls from
+      trying to unblock it, and give it permission to read. This fixes a
+      backtrace warning that can happen on relays under various
+      circumstances. Fixes bug 24167; bugfix on 0.1.0.1-rc.
+
+  o Minor bugfixes (path selection, backport from 0.3.2.4-alpha):
+    - When selecting relays by bandwidth, avoid a rounding error that
+      could sometimes cause load to be imbalanced incorrectly.
+      Previously, we would always round upwards; now, we round towards
+      the nearest integer. This had the biggest effect when a relay's
+      weight adjustments should have given it weight 0, but it got
+      weight 1 instead. Fixes bug 23318; bugfix on 0.2.4.3-alpha.
+    - When calculating the fraction of nodes that have descriptors, and
+      all nodes in the network have zero bandwidths, count the number of
+      nodes instead. Fixes bug 23318; bugfix on 0.2.4.10-alpha.
+    - Actually log the total bandwidth in compute_weighted_bandwidths().
+      Fixes bug 24170; bugfix on 0.2.4.3-alpha.
+
+  o Minor bugfixes (performance, fragile-hardening, backport from 0.3.3.1-alpha):
+    - Improve the performance of our consensus-diff application code
+      when Tor is built with the --enable-fragile-hardening option set.
+      Fixes bug 24826; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (OSX, backport from 0.3.3.1-alpha):
+    - Don't exit the Tor process if setrlimit() fails to change the file
+      limit (which can happen sometimes on some versions of OSX). Fixes
+      bug 21074; bugfix on 0.0.9pre5.
+
+  o Minor bugfixes (portability, msvc, backport from 0.3.2.9):
+    - Fix a bug in the bit-counting parts of our timing-wheel code on
+      MSVC. (Note that MSVC is still not a supported build platform, due
+      to cyptographic timing channel risks.) Fixes bug 24633; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (relay, partial backport):
+    - Make the internal channel_is_client() function look at what sort
+      of connection handshake the other side used, rather than whether
+      the other side ever sent a create_fast cell to us. Backports part
+      of the fixes from bugs 22805 and 24898.
+
+  o Minor bugfixes (spec conformance, backport from 0.3.3.3-alpha):
+    - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on
+      0.2.9.4-alpha.
+    - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249;
+      bugfix on 0.2.9.4-alpha.
+
+  o Code simplification and refactoring (backport from 0.3.3.3-alpha):
+    - Update the "rust dependencies" submodule to be a project-level
+      repository, rather than a user repository. Closes ticket 25323.
+
+
+Changes in version 0.2.9.15 - 2018-03-03
+  Tor 0.2.9.15 backports important security and stability bugfixes from
+  later Tor releases.
+
+  It includes an important security fix for a remote crash attack
+  against directory authorities, tracked as TROVE-2018-001.
+
+  This release also backports our new system for improved resistance to
+  denial-of-service attacks against relays.
+
+  This release also fixes several minor bugs and annoyances from
+  earlier releases.
+
+  All directory authorities should upgrade to one of the versions
+  released today. Relays running 0.2.9.x may wish to update to one of
+  the versions released today, for the DoS mitigations.
+
+  o Major bugfixes (denial-of-service, directory authority, backport from 0.3.3.3-alpha):
+    - Fix a protocol-list handling bug that could be used to remotely crash
+      directory authorities with a null-pointer exception. Fixes bug 25074;
+      bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001 and
+      CVE-2018-0490.
+
+  o Major features (denial-of-service mitigation):
+    - Give relays some defenses against the recent network overload. We
+      start with three defenses (default parameters in parentheses).
+      First: if a single client address makes too many concurrent
+      connections (>100), hang up on further connections. Second: if a
+      single client address makes circuits too quickly (more than 3 per
+      second, with an allowed burst of 90) while also having too many
+      connections open (3), refuse new create cells for the next while
+      (1-2 hours). Third: if a client asks to establish a rendezvous
+      point to you directly, ignore the request. These defenses can be
+      manually controlled by new torrc options, but relays will also
+      take guidance from consensus parameters, so there's no need to
+      configure anything manually. Implements ticket 24902.
+
+  o Major bugfixes (bootstrapping):
+    - Fetch descriptors aggressively whenever we lack enough to build
+      circuits, regardless of how many descriptors we are missing.
+      Previously, we would delay launching the fetch when we had fewer
+      than 15 missing descriptors, even if some of those descriptors
+      were blocking circuits from building. Fixes bug 23985; bugfix on
+      0.1.1.11-alpha. The effects of this bug became worse in
+      0.3.0.3-alpha, when we began treating missing descriptors from our
+      primary guards as a reason to delay circuits.
+
+  o Major bugfixes (onion services, retry behavior):
+    - Fix an "off by 2" error in counting rendezvous failures on the
+      onion service side. While we thought we would stop the rendezvous
+      attempt after one failed circuit, we were actually making three
+      circuit attempts before giving up. Now switch to a default of 2,
+      and allow the consensus parameter "hs_service_max_rdv_failures" to
+      override. Fixes bug 24895; bugfix on 0.0.6.
+
+  o Minor feature (relay statistics):
+    - Change relay bandwidth reporting stats interval from 4 hours to 24
+      hours in order to reduce the efficiency of guard discovery
+      attacks. Fixes ticket 23856.
+
+  o Minor features (compatibility, OpenSSL):
+    - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released.
+      Previous versions of Tor would not have worked with OpenSSL 1.1.1,
+      since they neither disabled TLS 1.3 nor enabled any of the
+      ciphersuites it requires. Now we enable the TLS 1.3 ciphersuites.
+      Closes ticket 24978.
+
+  o Minor features (denial-of-service avoidance):
+    - Make our OOM handler aware of the geoip client history cache so it
+      doesn't fill up the memory. This check is important for IPv6 and
+      our DoS mitigation subsystem. Closes ticket 25122.
+
+  o Minor features (fallback directory mirrors):
+    - The fallback directory list has been re-generated based on the
+      current status of the network. Tor uses fallback directories to
+      bootstrap when it doesn't yet have up-to-date directory
+      information. Closes ticket 24801.
+    - Make the default DirAuthorityFallbackRate 0.1, so that clients
+      prefer to bootstrap from fallback directory mirrors. This is a
+      follow-up to 24679, which removed weights from the default
+      fallbacks. Implements ticket 24681.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 7 2018 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (linux seccomp2 sandbox):
+    - Update the sandbox rules so that they should now work correctly
+      with Glibc 2.26. Closes ticket 24315.
+
+  o Minor bugfix (channel connection):
+    - Use the actual observed address of an incoming relay connection,
+      not the canonical address of the relay from its descriptor, when
+      making decisions about how to handle the incoming connection.
+      Fixes bug 24952; bugfix on 0.2.4.11-alpha. Patch by "ffmancera".
+
+  o Minor bugfix (directory authority):
+    - Directory authorities, when refusing a descriptor from a rejected
+      relay, now explicitly tell the relay (in its logs) to set a valid
+      ContactInfo address and contact the bad-relays@ mailing list.
+      Fixes bug 25170; bugfix on 0.2.9.1.
+
+  o Minor bugfixes (address selection):
+    - When the fascist_firewall_choose_address_ functions don't find a
+      reachable address, set the returned address to the null address
+      and port. This is a precautionary measure, because some callers do
+      not check the return value. Fixes bug 24736; bugfix
+      on 0.2.8.2-alpha.
+
+  o Minor bugfixes (compilation):
+    - Fix a signed/unsigned comparison warning introduced by our fix to
+      TROVE-2017-009. Fixes bug 24480; bugfix on 0.2.5.16.
+
+  o Minor bugfixes (control port, linux seccomp2 sandbox):
+    - Avoid a crash when attempting to use the seccomp2 sandbox together
+      with the OwningControllerProcess feature. Fixes bug 24198; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (denial-of-service, backport from 0.3.3.3-alpha):
+    - Fix a possible crash on malformed consensus. If a consensus had
+      contained an unparseable protocol line, it could have made clients
+      and relays crash with a null-pointer exception. To exploit this
+      issue, however, an attacker would need to be able to subvert the
+      directory authority system. Fixes bug 25251; bugfix on
+      0.2.9.4-alpha. Also tracked as TROVE-2018-004.
+
+  o Minor bugfixes (memory usage):
+    - When queuing DESTROY cells on a channel, only queue the circuit-id
+      and reason fields: not the entire 514-byte cell. This fix should
+      help mitigate any bugs or attacks that fill up these queues, and
+      free more RAM for other uses. Fixes bug 24666; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (network layer):
+    - When closing a connection via close_connection_immediately(), we
+      mark it as "not blocked on bandwidth", to prevent later calls from
+      trying to unblock it, and give it permission to read. This fixes a
+      backtrace warning that can happen on relays under various
+      circumstances. Fixes bug 24167; bugfix on 0.1.0.1-rc.
+
+  o Minor bugfixes (OSX):
+    - Don't exit the Tor process if setrlimit() fails to change the file
+      limit (which can happen sometimes on some versions of OSX). Fixes
+      bug 21074; bugfix on 0.0.9pre5.
+
+  o Minor bugfixes (path selection):
+    - When selecting relays by bandwidth, avoid a rounding error that
+      could sometimes cause load to be imbalanced incorrectly.
+      Previously, we would always round upwards; now, we round towards
+      the nearest integer. This had the biggest effect when a relay's
+      weight adjustments should have given it weight 0, but it got
+      weight 1 instead. Fixes bug 23318; bugfix on 0.2.4.3-alpha.
+    - When calculating the fraction of nodes that have descriptors, and
+      all nodes in the network have zero bandwidths, count the number of
+      nodes instead. Fixes bug 23318; bugfix on 0.2.4.10-alpha.
+    - Actually log the total bandwidth in compute_weighted_bandwidths().
+      Fixes bug 24170; bugfix on 0.2.4.3-alpha.
+
+  o Minor bugfixes (portability, msvc):
+    - Fix a bug in the bit-counting parts of our timing-wheel code on
+      MSVC. (Note that MSVC is still not a supported build platform, due
+      to cryptographic timing channel risks.) Fixes bug 24633; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (relay):
+    - Make the internal channel_is_client() function look at what sort
+      of connection handshake the other side used, rather than whether
+      the other side ever sent a create_fast cell to us. Backports part
+      of the fixes from bugs 22805 and 24898.
+
+  o Minor bugfixes (spec conformance, backport from 0.3.3.3-alpha):
+    - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on
+      0.2.9.4-alpha.
+    - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249;
+      bugfix on 0.2.9.4-alpha.
+
+
+Changes in version 0.3.2.10 - 2018-03-03
+  Tor 0.3.2.10 is the second stable release in the 0.3.2 series. It
+  backports a number of bugfixes, including important fixes for security
+  issues.
+
+  It includes an important security fix for a remote crash attack
+  against directory authorities, tracked as TROVE-2018-001.
+
+  Additionally, it backports a fix for a bug whose severity we have
+  upgraded: Bug 24700, which was fixed in 0.3.3.2-alpha, can be remotely
+  triggered in order to crash relays with a use-after-free pattern. As
+  such, we are now tracking that bug as TROVE-2018-002 and
+  CVE-2018-0491, and backporting it to earlier releases. This bug
+  affected versions 0.3.2.1-alpha through 0.3.2.9, as well as version
+  0.3.3.1-alpha.
+
+  This release also backports our new system for improved resistance to
+  denial-of-service attacks against relays.
+
+  This release also fixes several minor bugs and annoyances from
+  earlier releases.
+
+  Relays running 0.3.2.x SHOULD upgrade to one of the versions released
+  today, for the fix to TROVE-2018-002. Directory authorities should
+  also upgrade. (Relays on earlier versions might want to update too for
+  the DoS mitigations.)
+
+  o Major bugfixes (denial-of-service, directory authority, backport from 0.3.3.3-alpha):
+    - Fix a protocol-list handling bug that could be used to remotely crash
+      directory authorities with a null-pointer exception. Fixes bug 25074;
+      bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001 and
+      CVE-2018-0490.
+
+  o Major bugfixes (scheduler, KIST, denial-of-service, backport from 0.3.3.2-alpha):
+    - Avoid adding the same channel twice in the KIST scheduler pending
+      list, which could lead to remote denial-of-service use-after-free
+      attacks against relays. Fixes bug 24700; bugfix on 0.3.2.1-alpha.
+
+  o Major features (denial-of-service mitigation, backport from 0.3.3.2-alpha):
+    - Give relays some defenses against the recent network overload. We
+      start with three defenses (default parameters in parentheses).
+      First: if a single client address makes too many concurrent
+      connections (>100), hang up on further connections. Second: if a
+      single client address makes circuits too quickly (more than 3 per
+      second, with an allowed burst of 90) while also having too many
+      connections open (3), refuse new create cells for the next while
+      (1-2 hours). Third: if a client asks to establish a rendezvous
+      point to you directly, ignore the request. These defenses can be
+      manually controlled by new torrc options, but relays will also
+      take guidance from consensus parameters, so there's no need to
+      configure anything manually. Implements ticket 24902.
+
+  o Major bugfixes (onion services, retry behavior, backport from 0.3.3.1-alpha):
+    - Fix an "off by 2" error in counting rendezvous failures on the
+      onion service side. While we thought we would stop the rendezvous
+      attempt after one failed circuit, we were actually making three
+      circuit attempts before giving up. Now switch to a default of 2,
+      and allow the consensus parameter "hs_service_max_rdv_failures" to
+      override. Fixes bug 24895; bugfix on 0.0.6.
+    - New-style (v3) onion services now obey the "max rendezvous circuit
+      attempts" logic. Previously they would make as many rendezvous
+      circuit attempts as they could fit in the MAX_REND_TIMEOUT second
+      window before giving up. Fixes bug 24894; bugfix on 0.3.2.1-alpha.
+
+  o Major bugfixes (protocol versions, backport from 0.3.3.2-alpha):
+    - Add Link protocol version 5 to the supported protocols list. Fixes
+      bug 25070; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (relay, backport from 0.3.3.1-alpha):
+    - Fix a set of false positives where relays would consider
+      connections to other relays as being client-only connections (and
+      thus e.g. deserving different link padding schemes) if those
+      relays fell out of the consensus briefly. Now we look only at the
+      initial handshake and whether the connection authenticated as a
+      relay. Fixes bug 24898; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (scheduler, consensus, backport from 0.3.3.2-alpha):
+    - The scheduler subsystem was failing to promptly notice changes in
+      consensus parameters, making it harder to switch schedulers
+      network-wide. Fixes bug 24975; bugfix on 0.3.2.1-alpha.
+
+  o Minor features (denial-of-service avoidance, backport from 0.3.3.2-alpha):
+    - Make our OOM handler aware of the geoip client history cache so it
+      doesn't fill up the memory. This check is important for IPv6 and
+      our DoS mitigation subsystem. Closes ticket 25122.
+
+  o Minor features (compatibility, OpenSSL, backport from 0.3.3.3-alpha):
+    - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released.
+      Previous versions of Tor would not have worked with OpenSSL 1.1.1,
+      since they neither disabled TLS 1.3 nor enabled any of the
+      ciphersuites it requires. Now we enable the TLS 1.3 ciphersuites.
+      Closes ticket 24978.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 7 2018 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (logging, diagnostic, backport from 0.3.3.2-alpha):
+    - When logging a failure to create an onion service's descriptor,
+      also log what the problem with the descriptor was. Diagnostic
+      for ticket 24972.
+
+  o Minor bugfix (channel connection, backport from 0.3.3.2-alpha):
+    - Use the actual observed address of an incoming relay connection,
+      not the canonical address of the relay from its descriptor, when
+      making decisions about how to handle the incoming connection.
+      Fixes bug 24952; bugfix on 0.2.4.11-alpha. Patch by "ffmancera".
+
+  o Minor bugfixes (denial-of-service, backport from 0.3.3.3-alpha):
+    - Fix a possible crash on malformed consensus. If a consensus had
+      contained an unparseable protocol line, it could have made clients
+      and relays crash with a null-pointer exception. To exploit this
+      issue, however, an attacker would need to be able to subvert the
+      directory authority system. Fixes bug 25251; bugfix on
+      0.2.9.4-alpha. Also tracked as TROVE-2018-004.
+
+  o Minor bugfix (directory authority, backport from 0.3.3.2-alpha):
+    - Directory authorities, when refusing a descriptor from a rejected
+      relay, now explicitly tell the relay (in its logs) to set a valid
+      ContactInfo address and contact the bad-relays@ mailing list.
+      Fixes bug 25170; bugfix on 0.2.9.1.
+
+  o Minor bugfixes (build, rust, backport from 0.3.3.1-alpha):
+    - When building with Rust on OSX, link against libresolv, to work
+      around the issue at https://github.com/rust-lang/rust/issues/46797.
+      Fixes bug 24652; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.3.3.2-alpha):
+    - Remove a BUG() statement when a client fetches an onion descriptor
+      that has a lower revision counter than the one in its cache. This
+      can happen in normal circumstances due to HSDir desync. Fixes bug
+      24976; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (logging, backport from 0.3.3.2-alpha):
+    - Don't treat inability to store a cached consensus object as a bug:
+      it can happen normally when we are out of disk space. Fixes bug
+      24859; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (performance, fragile-hardening, backport from 0.3.3.1-alpha):
+    - Improve the performance of our consensus-diff application code
+      when Tor is built with the --enable-fragile-hardening option set.
+      Fixes bug 24826; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (OSX, backport from 0.3.3.1-alpha):
+    - Don't exit the Tor process if setrlimit() fails to change the file
+      limit (which can happen sometimes on some versions of OSX). Fixes
+      bug 21074; bugfix on 0.0.9pre5.
+
+  o Minor bugfixes (spec conformance, backport from 0.3.3.3-alpha):
+    - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on
+      0.2.9.4-alpha.
+    - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249;
+      bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.3.1-alpha):
+    - Fix a memory leak in the scheduler/loop_kist unit test. Fixes bug
+      25005; bugfix on 0.3.2.7-rc.
+
+  o Minor bugfixes (v3 onion services, backport from 0.3.3.2-alpha):
+    - Look at the "HSRend" protocol version, not the "HSDir" protocol
+      version, when deciding whether a consensus entry can support the
+      v3 onion service protocol as a rendezvous point. Fixes bug 25105;
+      bugfix on 0.3.2.1-alpha.
+
+  o Code simplification and refactoring (backport from 0.3.3.3-alpha):
+    - Update the "rust dependencies" submodule to be a project-level
+      repository, rather than a user repository. Closes ticket 25323.
+
+  o Documentation (backport from 0.3.3.1-alpha)
+    - Document that operators who run more than one relay or bridge are
+      expected to set MyFamily and ContactInfo correctly. Closes
+      ticket 24526.
+
+
+Changes in version 0.3.2.9 - 2018-01-09
+  Tor 0.3.2.9 is the first stable release in the 0.3.2 series.
+
+  The 0.3.2 series includes our long-anticipated new onion service
+  design, with numerous security features. (For more information, see
+  our blog post at https://blog.torproject.org/fall-harvest.) We also
+  have a new circuit scheduler algorithm for improved performance on
+  relays everywhere (see https://blog.torproject.org/kist-and-tell),
+  along with many smaller features and bugfixes.
+
+  Per our stable release policy, we plan to support each stable release
+  series for at least the next nine months, or for three months after
+  the first stable release of the next series: whichever is longer. If
+  you need a release with long-term support, we recommend that you stay
+  with the 0.2.9 series.
+
+  Below is a list of the changes since 0.3.1.7. For a list of all
+  changes since 0.3.2.8-rc, see the ChangeLog file.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+    - Remove longclaw's IPv6 address, as it will soon change. Authority
+      IPv6 addresses were originally added in 0.2.8.1-alpha. This leaves
+      3/8 directory authorities with IPv6 addresses, but there are also
+      52 fallback directory mirrors with IPv6 addresses. Resolves 19760.
+    - Add an IPv6 address for the "bastet" directory authority. Closes
+      ticket 24394.
+
+  o Major features (next-generation onion services):
+    - Tor now supports the next-generation onion services protocol for
+      clients and services! As part of this release, the core of
+      proposal 224 has been implemented and is available for
+      experimentation and testing by our users. This newer version of
+      onion services ("v3") features many improvements over the legacy
+      system, including:
+
+      a) Better crypto (replaced SHA1/DH/RSA1024
+      with SHA3/ed25519/curve25519)
+
+      b) Improved directory protocol, leaking much less information to
+      directory servers.
+
+      c) Improved directory protocol, with smaller surface for
+      targeted attacks.
+
+      d) Better onion address security against impersonation.
+
+      e) More extensible introduction/rendezvous protocol.
+
+      f) A cleaner and more modular codebase.
+
+      You can identify a next-generation onion address by its length:
+      they are 56 characters long, as in
+      "4acth47i6kxnvkewtm6q7ib2s3ufpo5sqbsnzjpbi7utijcltosqemad.onion".
+
+      In the future, we will release more options and features for v3
+      onion services, but we first need a testing period, so that the
+      current codebase matures and becomes more robust. Planned features
+      include: offline keys, advanced client authorization, improved
+      guard algorithms, and statistics. For full details, see
+      proposal 224.
+
+      Legacy ("v2") onion services will still work for the foreseeable
+      future, and will remain the default until this new codebase gets
+      tested and hardened. Service operators who want to experiment with
+      the new system can use the 'HiddenServiceVersion 3' torrc
+      directive along with the regular onion service configuration
+      options. For more information, see our blog post at
+      "https://blog.torproject.org/fall-harvest". Enjoy!
+
+  o Major feature (scheduler, channel):
+    - Tor now uses new schedulers to decide which circuits should
+      deliver cells first, in order to improve congestion at relays. The
+      first type is called "KIST" ("Kernel Informed Socket Transport"),
+      and is only available on Linux-like systems: it uses feedback from
+      the kernel to prevent the kernel's TCP buffers from growing too
+      full. The second new scheduler type is called "KISTLite": it
+      behaves the same as KIST, but runs on systems without kernel
+      support for inspecting TCP implementation details. The old
+      scheduler is still available, under the name "Vanilla". To change
+      the default scheduler preference order, use the new "Schedulers"
+      option. (The default preference order is "KIST,KISTLite,Vanilla".)
+
+      Matt Traudt implemented KIST, based on research by Rob Jansen,
+      John Geddes, Christ Wacek, Micah Sherr, and Paul Syverson. For
+      more information, see the design paper at
+      http://www.robgjansen.com/publications/kist-sec2014.pdf and the
+      followup implementation paper at https://arxiv.org/abs/1709.01044.
+      Closes ticket 12541. For more information, see our blog post at
+      "https://blog.torproject.org/kist-and-tell".
+
+  o Major bugfixes (security, general):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+
+  o Major bugfixes (security, directory authority):
+    - Fix a denial of service issue where an attacker could crash a
+      directory authority using a malformed router descriptor. Fixes bug
+      24245; bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2017-010
+      and CVE-2017-8820.
+
+  o Major bugfixes (security, onion service v2):
+    - Fix a use-after-free error that could crash v2 Tor onion services
+      when they failed to open circuits while expiring introduction
+      points. Fixes bug 24313; bugfix on 0.2.7.2-alpha. This issue is
+      also tracked as TROVE-2017-013 and CVE-2017-8823.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, relay):
+    - When running as a relay, make sure that we never build a path
+      through ourselves, even in the case where we have somehow lost the
+      version of our descriptor appearing in the consensus. Fixes part
+      of bug 21534; bugfix on 0.2.0.1-alpha. This issue is also tracked
+      as TROVE-2017-012 and CVE-2017-8822.
+    - When running as a relay, make sure that we never choose ourselves
+      as a guard. Fixes part of bug 21534; bugfix on 0.3.0.1-alpha. This
+      issue is also tracked as TROVE-2017-012 and CVE-2017-8822.
+
+  o Major bugfixes (bootstrapping):
+    - Fetch descriptors aggressively whenever we lack enough to build
+      circuits, regardless of how many descriptors we are missing.
+      Previously, we would delay launching the fetch when we had fewer
+      than 15 missing descriptors, even if some of those descriptors
+      were blocking circuits from building. Fixes bug 23985; bugfix on
+      0.1.1.11-alpha. The effects of this bug became worse in
+      0.3.0.3-alpha, when we began treating missing descriptors from our
+      primary guards as a reason to delay circuits.
+    - Don't try fetching microdescriptors from relays that have failed
+      to deliver them in the past. Fixes bug 23817; bugfix
+      on 0.3.0.1-alpha.
+
+  o Major bugfixes (circuit prediction):
+    - Fix circuit prediction logic so that a client doesn't treat a port
+      as being "handled" by a circuit if that circuit already has
+      isolation settings on it. This change should make Tor clients more
+      responsive by improving their chances of having a pre-created
+      circuit ready for use when a request arrives. Fixes bug 18859;
+      bugfix on 0.2.3.3-alpha.
+
+  o Major bugfixes (exit relays, DNS):
+    - Fix an issue causing DNS to fail on high-bandwidth exit nodes,
+      making them nearly unusable. Fixes bugs 21394 and 18580; bugfix on
+      0.1.2.2-alpha, which introduced eventdns. Thanks to Dhalgren for
+      identifying and finding a workaround to this bug and to Moritz,
+      Arthur Edelstein, and Roger for helping to track it down and
+      analyze it.
+
+  o Major bugfixes (relay, crash, assertion failure):
+    - Fix a timing-based assertion failure that could occur when the
+      circuit out-of-memory handler freed a connection's output buffer.
+      Fixes bug 23690; bugfix on 0.2.6.1-alpha.
+
+  o Major bugfixes (usability, control port):
+    - Report trusted clock skew indications as bootstrap errors, so
+      controllers can more easily alert users when their clocks are
+      wrong. Fixes bug 23506; bugfix on 0.1.2.6-alpha.
+
+  o Minor features (bridge):
+    - Bridge relays can now set the BridgeDistribution config option to
+      add a "bridge-distribution-request" line to their bridge
+      descriptor, which tells BridgeDB how they'd like their bridge
+      address to be given out. (Note that as of Oct 2017, BridgeDB does
+      not yet implement this feature.) As a side benefit, this feature
+      provides a way to distinguish bridge descriptors from non-bridge
+      descriptors. Implements tickets 18329.
+    - When handling the USERADDR command on an ExtOrPort, warn when the
+      transports provides a USERADDR with no port. In a future version,
+      USERADDR commands of this format may be rejected. Detects problems
+      related to ticket 23080.
+
+  o Minor features (bug detection):
+    - Log a warning message with a stack trace for any attempt to call
+      get_options() during option validation. This pattern has caused
+      subtle bugs in the past. Closes ticket 22281.
+
+  o Minor features (build, compilation):
+    - The "check-changes" feature is now part of the "make check" tests;
+      we'll use it to try to prevent misformed changes files from
+      accumulating. Closes ticket 23564.
+    - Tor builds should now fail if there are any mismatches between the
+      C type representing a configuration variable and the C type the
+      data-driven parser uses to store a value there. Previously, we
+      needed to check these by hand, which sometimes led to mistakes.
+      Closes ticket 23643.
+
+  o Minor features (client):
+    - You can now use Tor as a tunneled HTTP proxy: use the new
+      HTTPTunnelPort option to open a port that accepts HTTP CONNECT
+      requests. Closes ticket 22407.
+    - Add an extra check to make sure that we always use the newer guard
+      selection code for picking our guards. Closes ticket 22779.
+    - When downloading (micro)descriptors, don't split the list into
+      multiple requests unless we want at least 32 descriptors.
+      Previously, we split at 4, not 32, which led to significant
+      overhead in HTTP request size and degradation in compression
+      performance. Closes ticket 23220.
+    - Improve log messages when missing descriptors for primary guards.
+      Resolves ticket 23670.
+
+  o Minor features (command line):
+    - Add a new commandline option, --key-expiration, which prints when
+      the current signing key is going to expire. Implements ticket
+      17639; patch by Isis Lovecruft.
+
+  o Minor features (control port):
+    - If an application tries to use the control port as an HTTP proxy,
+      respond with a meaningful "This is the Tor control port" message,
+      and log the event. Closes ticket 1667. Patch from Ravi
+      Chandra Padmala.
+    - Provide better error message for GETINFO desc/(id|name) when not
+      fetching router descriptors. Closes ticket 5847. Patch by
+      Kevin Butler.
+    - Add GETINFO "{desc,md}/download-enabled", to inform the controller
+      whether Tor will try to download router descriptors and
+      microdescriptors respectively. Closes ticket 22684.
+    - Added new GETINFO targets "ip-to-country/{ipv4,ipv6}-available",
+      so controllers can tell whether the geoip databases are loaded.
+      Closes ticket 23237.
+    - Adds a timestamp field to the CIRC_BW and STREAM_BW bandwidth
+      events. Closes ticket 19254. Patch by "DonnchaC".
+
+  o Minor features (development support):
+    - Developers can now generate a call-graph for Tor using the
+      "calltool" python program, which post-processes object dumps. It
+      should work okay on many Linux and OSX platforms, and might work
+      elsewhere too. To run it, install calltool from
+      https://gitweb.torproject.org/user/nickm/calltool.git and run
+      "make callgraph". Closes ticket 19307.
+
+  o Minor features (directory authority):
+    - Make the "Exit" flag assignment only depend on whether the exit
+      policy allows connections to ports 80 and 443. Previously relays
+      would get the Exit flag if they allowed connections to one of
+      these ports and also port 6667. Resolves ticket 23637.
+
+  o Minor features (ed25519):
+    - Add validation function to checks for torsion components in
+      ed25519 public keys, used by prop224 client-side code. Closes
+      ticket 22006. Math help by Ian Goldberg.
+
+  o Minor features (exit relay, DNS):
+    - Improve the clarity and safety of the log message from evdns when
+      receiving an apparently spoofed DNS reply. Closes ticket 3056.
+
+  o Minor features (fallback directory mirrors):
+    - The fallback directory list has been re-generated based on the
+      current status of the network. Tor uses fallback directories to
+      bootstrap when it doesn't yet have up-to-date directory
+      information. Closes ticket 24801.
+    - Make the default DirAuthorityFallbackRate 0.1, so that clients
+      prefer to bootstrap from fallback directory mirrors. This is a
+      follow-up to 24679, which removed weights from the default
+      fallbacks. Implements ticket 24681.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the January 5 2018 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (integration, hardening):
+    - Add a new NoExec option to prevent Tor from running other
+      programs. When this option is set to 1, Tor will never try to run
+      another program, regardless of the settings of
+      PortForwardingHelper, ClientTransportPlugin, or
+      ServerTransportPlugin. Once NoExec is set, it cannot be disabled
+      without restarting Tor. Closes ticket 22976.
+
+  o Minor features (linux seccomp2 sandbox):
+    - Update the sandbox rules so that they should now work correctly
+      with Glibc 2.26. Closes ticket 24315.
+
+  o Minor features (logging):
+    - Provide better warnings when the getrandom() syscall fails. Closes
+      ticket 24500.
+    - Downgrade a pair of log messages that could occur when an exit's
+      resolver gave us an unusual (but not forbidden) response. Closes
+      ticket 24097.
+    - Improve the message we log when re-enabling circuit build timeouts
+      after having received a consensus. Closes ticket 20963.
+    - Log more circuit information whenever we are about to try to
+      package a relay cell on a circuit with a nonexistent n_chan.
+      Attempt to diagnose ticket 8185.
+    - Improve info-level log identification of particular circuits, to
+      help with debugging. Closes ticket 23645.
+    - Improve the warning message for specifying a relay by nickname.
+      The previous message implied that nickname registration was still
+      part of the Tor network design, which it isn't. Closes
+      ticket 20488.
+    - If the sandbox filter fails to load, suggest to the user that
+      their kernel might not support seccomp2. Closes ticket 23090.
+
+  o Minor features (onion service, circuit, logging):
+    - Improve logging of many callsite in the circuit subsystem to print
+      the circuit identifier(s).
+    - Log when we cleanup an intro point from a service so we know when
+      and for what reason it happened. Closes ticket 23604.
+
+  o Minor features (portability):
+    - Tor now compiles correctly on arm64 with libseccomp-dev installed.
+      (It doesn't yet work with the sandbox enabled.) Closes
+      ticket 24424.
+    - Check at configure time whether uint8_t is the same type as
+      unsigned char. Lots of existing code already makes this
+      assumption, and there could be strict aliasing issues if the
+      assumption is violated. Closes ticket 22410.
+
+  o Minor features (relay):
+    - When choosing which circuits can be expired as unused, consider
+      circuits from clients even if those clients used regular CREATE
+      cells to make them; and do not consider circuits from relays even
+      if they were made with CREATE_FAST. Part of ticket 22805.
+    - Reject attempts to use relative file paths when RunAsDaemon is
+      set. Previously, Tor would accept these, but the directory-
+      changing step of RunAsDaemon would give strange and/or confusing
+      results. Closes ticket 22731.
+
+  o Minor features (relay statistics):
+    - Change relay bandwidth reporting stats interval from 4 hours to 24
+      hours in order to reduce the efficiency of guard discovery
+      attacks. Fixes ticket 23856.
+
+  o Minor features (reverted deprecations):
+    - The ClientDNSRejectInternalAddresses flag can once again be set in
+      non-testing Tor networks, so long as they do not use the default
+      directory authorities. This change also removes the deprecation of
+      this flag from 0.2.9.2-alpha. Closes ticket 21031.
+
+  o Minor features (robustness):
+    - Change several fatal assertions when flushing buffers into non-
+      fatal assertions, to prevent any recurrence of 23690.
+
+  o Minor features (startup, safety):
+    - When configured to write a PID file, Tor now exits if it is unable
+      to do so. Previously, it would warn and continue. Closes
+      ticket 20119.
+
+  o Minor features (static analysis):
+    - The BUG() macro has been changed slightly so that Coverity no
+      longer complains about dead code if the bug is impossible. Closes
+      ticket 23054.
+
+  o Minor features (testing):
+    - Our fuzzing tests now test the encrypted portions of v3 onion
+      service descriptors. Implements more of 21509.
+    - Add a unit test to make sure that our own generated platform
+      string will be accepted by directory authorities. Closes
+      ticket 22109.
+    - The default chutney network tests now include tests for the v3
+      onion service design. Make sure you have the latest version of
+      chutney if you want to run these. Closes ticket 22437.
+    - Add a unit test to verify that we can parse a hardcoded v2 onion
+      service descriptor. Closes ticket 15554.
+
+  o Minor bugfixes (address selection):
+    - When the fascist_firewall_choose_address_ functions don't find a
+      reachable address, set the returned address to the null address
+      and port. This is a precautionary measure, because some callers do
+      not check the return value. Fixes bug 24736; bugfix
+      on 0.2.8.2-alpha.
+
+  o Minor bugfixes (bootstrapping):
+    - When warning about state file clock skew, report the correct
+      direction for the detected skew. Fixes bug 23606; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (bridge clients, bootstrap):
+    - Retry directory downloads when we get our first bridge descriptor
+      during bootstrap or while reconnecting to the network. Keep
+      retrying every time we get a bridge descriptor, until we have a
+      reachable bridge. Fixes part of bug 24367; bugfix on 0.2.0.3-alpha.
+    - Stop delaying bridge descriptor fetches when we have cached bridge
+      descriptors. Instead, only delay bridge descriptor fetches when we
+      have at least one reachable bridge. Fixes part of bug 24367;
+      bugfix on 0.2.0.3-alpha.
+    - Stop delaying directory fetches when we have cached bridge
+      descriptors. Instead, only delay bridge descriptor fetches when
+      all our bridges are definitely unreachable. Fixes part of bug
+      24367; bugfix on 0.2.0.3-alpha.
+
+  o Minor bugfixes (bridge):
+    - Overwrite the bridge address earlier in the process of retrieving
+      its descriptor, to make sure we reach it on the configured
+      address. Fixes bug 20532; bugfix on 0.2.0.10-alpha.
+
+  o Minor bugfixes (build, compilation):
+    - Fix a compilation warning when building with zstd support on
+      32-bit platforms. Fixes bug 23568; bugfix on 0.3.1.1-alpha. Found
+      and fixed by Andreas Stieger.
+    - When searching for OpenSSL, don't accept any OpenSSL library that
+      lacks TLSv1_1_method(): Tor doesn't build with those versions.
+      Additionally, look in /usr/local/opt/openssl, if it's present.
+      These changes together repair the default build on OSX systems
+      with Homebrew installed. Fixes bug 23602; bugfix on 0.2.7.2-alpha.
+    - Fix a signed/unsigned comparison warning introduced by our fix to
+      TROVE-2017-009. Fixes bug 24480; bugfix on 0.2.5.16.
+    - Fix a memory leak warning in one of the libevent-related
+      configuration tests that could occur when manually specifying
+      -fsanitize=address. Fixes bug 24279; bugfix on 0.3.0.2-alpha.
+      Found and patched by Alex Xu.
+    - Fix unused-variable warnings in donna's Curve25519 SSE2 code.
+      Fixes bug 22895; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (certificate handling):
+    - Fix a time handling bug in Tor certificates set to expire after
+      the year 2106. Fixes bug 23055; bugfix on 0.3.0.1-alpha. Found by
+      Coverity as CID 1415728.
+
+  o Minor bugfixes (client):
+    - By default, do not enable storage of client-side DNS values. These
+      values were unused by default previously, but they should not have
+      been cached at all. Fixes bug 24050; bugfix on 0.2.6.3-alpha.
+
+  o Minor bugfixes (client, usability):
+    - Refrain from needlessly rejecting SOCKS5-with-hostnames and
+      SOCKS4a requests that contain IP address strings, even when
+      SafeSocks in enabled, as this prevents user from connecting to
+      known IP addresses without relying on DNS for resolving. SafeSocks
+      still rejects SOCKS connections that connect to IP addresses when
+      those addresses are _not_ encoded as hostnames. Fixes bug 22461;
+      bugfix on Tor 0.2.6.2-alpha.
+
+  o Minor bugfixes (code correctness):
+    - Call htons() in extend_cell_format() for encoding a 16-bit value.
+      Previously we used ntohs(), which happens to behave the same on
+      all the platforms we support, but which isn't really correct.
+      Fixes bug 23106; bugfix on 0.2.4.8-alpha.
+    - For defense-in-depth, make the controller's write_escaped_data()
+      function robust to extremely long inputs. Fixes bug 19281; bugfix
+      on 0.1.1.1-alpha. Reported by Guido Vranken.
+    - Fix several places in our codebase where a C compiler would be
+      likely to eliminate a check, based on assuming that undefined
+      behavior had not happened elsewhere in the code. These cases are
+      usually a sign of redundant checking or dubious arithmetic. Found
+      by Georg Koppen using the "STACK" tool from Wang, Zeldovich,
+      Kaashoek, and Solar-Lezama. Fixes bug 24423; bugfix on various
+      Tor versions.
+
+  o Minor bugfixes (compression):
+    - Handle a pathological case when decompressing Zstandard data when
+      the output buffer size is zero. Fixes bug 23551; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (consensus expiry):
+    - Check for adequate directory information correctly. Previously, Tor
+      would reconsider whether it had sufficient directory information
+      every 2 minutes. Fixes bug 23091; bugfix on 0.2.0.19-alpha.
+
+  o Minor bugfixes (control port, linux seccomp2 sandbox):
+    - Avoid a crash when attempting to use the seccomp2 sandbox together
+      with the OwningControllerProcess feature. Fixes bug 24198; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (control port, onion services):
+    - Report "FAILED" instead of "UPLOAD_FAILED" "FAILED" for the
+      HS_DESC event when a service is not able to upload a descriptor.
+      Fixes bug 24230; bugfix on 0.2.7.1-alpha.
+
+  o Minor bugfixes (directory cache):
+    - Recover better from empty or corrupt files in the consensus cache
+      directory. Fixes bug 24099; bugfix on 0.3.1.1-alpha.
+    - When a consensus diff calculation is only partially successful,
+      only record the successful parts as having succeeded. Partial
+      success can happen if (for example) one compression method fails
+      but the others succeed. Previously we misrecorded all the
+      calculations as having succeeded, which would later cause a
+      nonfatal assertion failure. Fixes bug 24086; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (directory client):
+    - On failure to download directory information, delay retry attempts
+      by a random amount based on the "decorrelated jitter" algorithm.
+      Our previous delay algorithm tended to produce extra-long delays
+      too easily. Fixes bug 23816; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (directory protocol):
+    - Directory servers now include a "Date:" http header for response
+      codes other than 200. Clients starting with a skewed clock and a
+      recent consensus were getting "304 Not modified" responses from
+      directory authorities, so without the Date header, the client
+      would never hear about a wrong clock. Fixes bug 23499; bugfix
+      on 0.0.8rc1.
+    - Make clients wait for 6 seconds before trying to download a
+      consensus from an authority. Fixes bug 17750; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (documentation):
+    - Document better how to read gcov, and what our gcov postprocessing
+      scripts do. Fixes bug 23739; bugfix on 0.2.9.1-alpha.
+    - Fix manpage to not refer to the obsolete (and misspelled)
+      UseEntryGuardsAsDirectoryGuards parameter in the description of
+      NumDirectoryGuards. Fixes bug 23611; bugfix on 0.2.4.8-alpha.
+
+  o Minor bugfixes (DoS-resistance):
+    - If future code asks if there are any running bridges, without
+      checking if bridges are enabled, log a BUG warning rather than
+      crashing. Fixes bug 23524; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (entry guards):
+    - Tor now updates its guard state when it reads a consensus
+      regardless of whether it's missing descriptors. That makes tor use
+      its primary guards to fetch descriptors in some edge cases where
+      it would previously have used fallback directories. Fixes bug
+      23862; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (format strictness):
+    - Restrict several data formats to decimal. Previously, the
+      BuildTimeHistogram entries in the state file, the "bw=" entries in
+      the bandwidth authority file, and the process IDs passed to the
+      __OwningControllerProcess option could all be specified in hex or
+      octal as well as in decimal. This was not an intentional feature.
+      Fixes bug 22802; bugfixes on 0.2.2.1-alpha, 0.2.2.2-alpha,
+      and 0.2.2.28-beta.
+
+  o Minor bugfixes (heartbeat):
+    - If we fail to write a heartbeat message, schedule a retry for the
+      minimum heartbeat interval number of seconds in the future. Fixes
+      bug 19476; bugfix on 0.2.3.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Suppress a log notice when relay descriptors arrive. We already
+      have a bootstrap progress for this so no need to log notice
+      everytime tor receives relay descriptors. Microdescriptors behave
+      the same. Fixes bug 23861; bugfix on 0.2.8.2-alpha.
+    - Remove duplicate log messages regarding opening non-local
+      SocksPorts upon parsing config and opening listeners at startup.
+      Fixes bug 4019; bugfix on 0.2.3.3-alpha.
+    - Use a more comprehensible log message when telling the user
+      they've excluded every running exit node. Fixes bug 7890; bugfix
+      on 0.2.2.25-alpha.
+    - When logging the number of descriptors we intend to download per
+      directory request, do not log a number higher than then the number
+      of descriptors we're fetching in total. Fixes bug 19648; bugfix
+      on 0.1.1.8-alpha.
+    - When warning about a directory owned by the wrong user, log the
+      actual name of the user owning the directory. Previously, we'd log
+      the name of the process owner twice. Fixes bug 23487; bugfix
+      on 0.2.9.1-alpha.
+    - Fix some messages on unexpected errors from the seccomp2 library.
+      Fixes bug 22750; bugfix on 0.2.5.1-alpha. Patch from "cypherpunks".
+    - The tor specification says hop counts are 1-based, so fix two log
+      messages that mistakenly logged 0-based hop counts. Fixes bug
+      18982; bugfix on 0.2.6.2-alpha and 0.2.4.5-alpha. Patch by teor.
+      Credit to Xiaofan Li for reporting this issue.
+
+  o Minor bugfixes (logging, relay shutdown, annoyance):
+    - When a circuit is marked for close, do not attempt to package any
+      cells for channels on that circuit. Previously, we would detect
+      this condition lower in the call stack, when we noticed that the
+      circuit had no attached channel, and log an annoying message.
+      Fixes bug 8185; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (memory safety, defensive programming):
+    - Clear the target address when node_get_prim_orport() returns
+      early. Fixes bug 23874; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (memory usage):
+    - When queuing DESTROY cells on a channel, only queue the circuit-id
+      and reason fields: not the entire 514-byte cell. This fix should
+      help mitigate any bugs or attacks that fill up these queues, and
+      free more RAM for other uses. Fixes bug 24666; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (network layer):
+    - When closing a connection via close_connection_immediately(), we
+      mark it as "not blocked on bandwidth", to prevent later calls from
+      trying to unblock it, and give it permission to read. This fixes a
+      backtrace warning that can happen on relays under various
+      circumstances. Fixes bug 24167; bugfix on 0.1.0.1-rc.
+
+  o Minor bugfixes (onion services):
+    - The introduction circuit was being timed out too quickly while
+      waiting for the rendezvous circuit to complete. Keep the intro
+      circuit around longer instead of timing out and reopening new ones
+      constantly. Fixes bug 23681; bugfix on 0.2.4.8-alpha.
+    - Rename the consensus parameter "hsdir-interval" to "hsdir_interval"
+      so it matches dir-spec.txt. Fixes bug 24262; bugfix
+      on 0.3.1.1-alpha.
+    - When handling multiple SOCKS request for the same .onion address,
+      only fetch the service descriptor once.
+    - Avoid a possible double close of a circuit by the intro point on
+      error of sending the INTRO_ESTABLISHED cell. Fixes bug 23610;
+      bugfix on 0.3.0.1-alpha.
+    - When reloading configured onion services, copy all information
+      from the old service object. Previously, some data was omitted,
+      causing delays in descriptor upload, and other bugs. Fixes bug
+      23790; bugfix on 0.2.1.9-alpha.
+
+  o Minor bugfixes (path selection):
+    - When selecting relays by bandwidth, avoid a rounding error that
+      could sometimes cause load to be imbalanced incorrectly.
+      Previously, we would always round upwards; now, we round towards
+      the nearest integer. This had the biggest effect when a relay's
+      weight adjustments should have given it weight 0, but it got
+      weight 1 instead. Fixes bug 23318; bugfix on 0.2.4.3-alpha.
+    - When calculating the fraction of nodes that have descriptors, and
+      all nodes in the network have zero bandwidths, count the number of
+      nodes instead. Fixes bug 23318; bugfix on 0.2.4.10-alpha.
+    - Actually log the total bandwidth in compute_weighted_bandwidths().
+      Fixes bug 24170; bugfix on 0.2.4.3-alpha.
+
+  o Minor bugfixes (portability):
+    - Stop using the PATH_MAX variable, which is not defined on GNU
+      Hurd. Fixes bug 23098; bugfix on 0.3.1.1-alpha.
+    - Fix a bug in the bit-counting parts of our timing-wheel code on
+      MSVC. (Note that MSVC is still not a supported build platform, due
+      to cryptographic timing channel risks.) Fixes bug 24633; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (relay):
+    - When uploading our descriptor for the first time after startup,
+      report the reason for uploading as "Tor just started" rather than
+      leaving it blank. Fixes bug 22885; bugfix on 0.2.3.4-alpha.
+    - Avoid unnecessary calls to directory_fetches_from_authorities() on
+      relays, to prevent spurious address resolutions and descriptor
+      rebuilds. This is a mitigation for bug 21789. Fixes bug 23470;
+      bugfix on in 0.2.8.1-alpha.
+    - Avoid a crash when transitioning from client mode to bridge mode.
+      Previously, we would launch the worker threads whenever our
+      "public server" mode changed, but not when our "server" mode
+      changed. Fixes bug 23693; bugfix on 0.2.6.3-alpha.
+
+  o Minor bugfixes (testing):
+    - Fix a spurious fuzzing-only use of an uninitialized value. Found
+      by Brian Carpenter. Fixes bug 24082; bugfix on 0.3.0.3-alpha.
+    - Test that IPv6-only clients can use microdescriptors when running
+      "make test-network-all". Requires chutney master 61c28b9 or later.
+      Closes ticket 24109.
+    - Prevent scripts/test/coverage from attempting to move gcov output
+      to the root directory. Fixes bug 23741; bugfix on 0.2.5.1-alpha.
+    - Capture and detect several "Result does not fit" warnings in unit
+      tests on platforms with 32-bit time_t. Fixes bug 21800; bugfix
+      on 0.2.9.3-alpha.
+    - Fix additional channelpadding unit test failures by using mocked
+      time instead of actual time for all tests. Fixes bug 23608; bugfix
+      on 0.3.1.1-alpha.
+    - Fix a bug in our fuzzing mock replacement for crypto_pk_checksig(),
+      to correctly handle cases where a caller gives it an RSA key of
+      under 160 bits. (This is not actually a bug in Tor itself, but
+      rather in our fuzzing code.) Fixes bug 24247; bugfix on
+      0.3.0.3-alpha. Found by OSS-Fuzz as issue 4177.
+    - Fix a broken unit test for the OutboundAddress option: the parsing
+      function was never returning an error on failure. Fixes bug 23366;
+      bugfix on 0.3.0.3-alpha.
+    - Fix a signed-integer overflow in the unit tests for
+      dir/download_status_random_backoff, which was untriggered until we
+      fixed bug 17750. Fixes bug 22924; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (usability, control port):
+    - Stop making an unnecessary routerlist check in NETINFO clock skew
+      detection; this was preventing clients from reporting NETINFO clock
+      skew to controllers. Fixes bug 23532; bugfix on 0.2.4.4-alpha.
+
+  o Code simplification and refactoring:
+    - Remove various ways of testing circuits and connections for
+      "clientness"; instead, favor channel_is_client(). Part of
+      ticket 22805.
+    - Extract the code for handling newly-open channels into a separate
+      function from the general code to handle channel state
+      transitions. This change simplifies our callgraph, reducing the
+      size of the largest strongly connected component by roughly a
+      factor of two. Closes ticket 22608.
+    - Remove dead code for largely unused statistics on the number of
+      times we've attempted various public key operations. Fixes bug
+      19871; bugfix on 0.1.2.4-alpha. Fix by Isis Lovecruft.
+    - Remove several now-obsolete functions for asking about old
+      variants directory authority status. Closes ticket 22311; patch
+      from "huyvq".
+    - Remove some of the code that once supported "Named" and "Unnamed"
+      routers. Authorities no longer vote for these flags. Closes
+      ticket 22215.
+    - Rename the obsolete malleable hybrid_encrypt functions used in TAP
+      and old hidden services, to indicate that they aren't suitable for
+      new protocols or formats. Closes ticket 23026.
+    - Replace our STRUCT_OFFSET() macro with offsetof(). Closes ticket
+      22521. Patch from Neel Chauhan.
+    - Split the enormous circuit_send_next_onion_skin() function into
+      multiple subfunctions. Closes ticket 22804.
+    - Split the portions of the buffer.c module that handle particular
+      protocols into separate modules. Part of ticket 23149.
+    - Use our test macros more consistently, to produce more useful
+      error messages when our unit tests fail. Add coccinelle patches to
+      allow us to re-check for test macro uses. Closes ticket 22497.
+
+  o Deprecated features:
+    - The ReachableDirAddresses and ClientPreferIPv6DirPort options are
+      now deprecated; they do not apply to relays, and they have had no
+      effect on clients since 0.2.8.x. Closes ticket 19704.
+    - Deprecate HTTPProxy/HTTPProxyAuthenticator config options. They
+      only applies to direct unencrypted HTTP connections to your
+      directory server, which your Tor probably isn't using. Closes
+      ticket 20575.
+
+  o Documentation:
+    - Add notes in man page regarding OS support for the various
+      scheduler types. Attempt to use less jargon in the scheduler
+      section. Closes ticket 24254.
+    - Clarify that the Address option is entirely about setting an
+      advertised IPv4 address. Closes ticket 18891.
+    - Clarify the manpage's use of the term "address" to clarify what
+      kind of address is intended. Closes ticket 21405.
+    - Document that onion service subdomains are allowed, and ignored.
+      Closes ticket 18736.
+    - Clarify in the manual that "Sandbox 1" is only supported on Linux
+      kernels. Closes ticket 22677.
+    - Document all values of PublishServerDescriptor in the manpage.
+      Closes ticket 15645.
+    - Improve the documentation for the directory port part of the
+      DirAuthority line. Closes ticket 20152.
+    - Restore documentation for the authorities' "approved-routers"
+      file. Closes ticket 21148.
+
+  o Removed features:
+    - The AllowDotExit option has been removed as unsafe. It has been
+      deprecated since 0.2.9.2-alpha. Closes ticket 23426.
+    - The ClientDNSRejectInternalAddresses flag can no longer be set on
+      non-testing networks. It has been deprecated since 0.2.9.2-alpha.
+      Closes ticket 21031.
+    - The controller API no longer includes an AUTHDIR_NEWDESCS event:
+      nobody was using it any longer. Closes ticket 22377.
+
+
+Changes in version 0.3.1.9 - 2017-12-01:
+  Tor 0.3.1.9 backports important security and stability fixes from the
+  0.3.2 development series. All Tor users should upgrade to this
+  release, or to another of the releases coming out today.
+
+  o Major bugfixes (security, backport from 0.3.2.6-alpha):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - Fix a denial of service issue where an attacker could crash a
+      directory authority using a malformed router descriptor. Fixes bug
+      24245; bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2017-010
+      and CVE-2017-8820.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, onion service v2, backport from 0.3.2.6-alpha):
+    - Fix a use-after-free error that could crash v2 Tor onion services
+      when they failed to open circuits while expiring introduction
+      points. Fixes bug 24313; bugfix on 0.2.7.2-alpha. This issue is
+      also tracked as TROVE-2017-013 and CVE-2017-8823.
+
+  o Major bugfixes (security, relay, backport from 0.3.2.6-alpha):
+    - When running as a relay, make sure that we never build a path
+      through ourselves, even in the case where we have somehow lost the
+      version of our descriptor appearing in the consensus. Fixes part
+      of bug 21534; bugfix on 0.2.0.1-alpha. This issue is also tracked
+      as TROVE-2017-012 and CVE-2017-8822.
+    - When running as a relay, make sure that we never choose ourselves
+      as a guard. Fixes part of bug 21534; bugfix on 0.3.0.1-alpha. This
+      issue is also tracked as TROVE-2017-012 and CVE-2017-8822.
+
+  o Major bugfixes (exit relays, DNS, backport from 0.3.2.4-alpha):
+    - Fix an issue causing DNS to fail on high-bandwidth exit nodes,
+      making them nearly unusable. Fixes bugs 21394 and 18580; bugfix on
+      0.1.2.2-alpha, which introduced eventdns. Thanks to Dhalgren for
+      identifying and finding a workaround to this bug and to Moritz,
+      Arthur Edelstein, and Roger for helping to track it down and
+      analyze it.
+
+  o Minor features (bridge):
+    - Bridges now include notice in their descriptors that they are
+      bridges, and notice of their distribution status, based on their
+      publication settings. Implements ticket 18329. For more fine-
+      grained control of how a bridge is distributed, upgrade to 0.3.2.x
+      or later.
+
+  o Minor features (directory authority, backport from 0.3.2.6-alpha):
+    - Add an IPv6 address for the "bastet" directory authority. Closes
+      ticket 24394.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfix (relay address resolution, backport from 0.3.2.1-alpha):
+    - Avoid unnecessary calls to directory_fetches_from_authorities() on
+      relays, to prevent spurious address resolutions and descriptor
+      rebuilds. This is a mitigation for bug 21789. Fixes bug 23470;
+      bugfix on in 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.2.1-alpha):
+    - Fix unused variable warnings in donna's Curve25519 SSE2 code.
+      Fixes bug 22895; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (logging, relay shutdown, annoyance, backport from 0.3.2.2-alpha):
+    - When a circuit is marked for close, do not attempt to package any
+      cells for channels on that circuit. Previously, we would detect
+      this condition lower in the call stack, when we noticed that the
+      circuit had no attached channel, and log an annoying message.
+      Fixes bug 8185; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (onion service, backport from 0.3.2.5-alpha):
+    - Rename the consensus parameter "hsdir-interval" to "hsdir_interval"
+      so it matches dir-spec.txt. Fixes bug 24262; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (relay, crash, backport from 0.3.2.4-alpha):
+    - Avoid a crash when transitioning from client mode to bridge mode.
+      Previously, we would launch the worker threads whenever our
+      "public server" mode changed, but not when our "server" mode
+      changed. Fixes bug 23693; bugfix on 0.2.6.3-alpha.
+
+
+Changes in version 0.3.0.13 - 2017-12-01
+  Tor 0.3.0.13 backports important security and stability bugfixes from
+  later Tor releases. All Tor users should upgrade to this release, or
+  to another of the releases coming out today.
+
+  Note: the Tor 0.3.0 series will no longer be supported after 26 Jan
+  2018. If you need a release with long-term support, please stick with
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Major bugfixes (security, backport from 0.3.2.6-alpha):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - Fix a denial of service issue where an attacker could crash a
+      directory authority using a malformed router descriptor. Fixes bug
+      24245; bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2017-010
+      and CVE-2017-8820.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, onion service v2, backport from 0.3.2.6-alpha):
+    - Fix a use-after-free error that could crash v2 Tor onion services
+      when they failed to open circuits while expiring introduction
+      points. Fixes bug 24313; bugfix on 0.2.7.2-alpha. This issue is
+      also tracked as TROVE-2017-013 and CVE-2017-8823.
+
+  o Major bugfixes (security, relay, backport from 0.3.2.6-alpha):
+    - When running as a relay, make sure that we never build a path
+      through ourselves, even in the case where we have somehow lost the
+      version of our descriptor appearing in the consensus. Fixes part
+      of bug 21534; bugfix on 0.2.0.1-alpha. This issue is also tracked
+      as TROVE-2017-012 and CVE-2017-8822.
+    - When running as a relay, make sure that we never choose ourselves
+      as a guard. Fixes part of bug 21534; bugfix on 0.3.0.1-alpha. This
+      issue is also tracked as TROVE-2017-012 and CVE-2017-8822.
+
+  o Major bugfixes (exit relays, DNS, backport from 0.3.2.4-alpha):
+    - Fix an issue causing DNS to fail on high-bandwidth exit nodes,
+      making them nearly unusable. Fixes bugs 21394 and 18580; bugfix on
+      0.1.2.2-alpha, which introduced eventdns. Thanks to Dhalgren for
+      identifying and finding a workaround to this bug and to Moritz,
+      Arthur Edelstein, and Roger for helping to track it down and
+      analyze it.
+
+  o Minor features (security, windows, backport from 0.3.1.1-alpha):
+    - Enable a couple of pieces of Windows hardening: one
+      (HeapEnableTerminationOnCorruption) that has been on-by-default
+      since Windows 8, and unavailable before Windows 7; and one
+      (PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION) which we believe doesn't
+      affect us, but shouldn't do any harm. Closes ticket 21953.
+
+  o Minor features (bridge, backport from 0.3.1.9):
+    - Bridges now include notice in their descriptors that they are
+      bridges, and notice of their distribution status, based on their
+      publication settings. Implements ticket 18329. For more fine-
+      grained control of how a bridge is distributed, upgrade to 0.3.2.x
+      or later.
+
+  o Minor features (directory authority, backport from 0.3.2.6-alpha):
+    - Add an IPv6 address for the "bastet" directory authority. Closes
+      ticket 24394.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfix (relay address resolution, backport from 0.3.2.1-alpha):
+    - Avoid unnecessary calls to directory_fetches_from_authorities() on
+      relays, to prevent spurious address resolutions and descriptor
+      rebuilds. This is a mitigation for bug 21789. Fixes bug 23470;
+      bugfix on in 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.2.1-alpha):
+    - Fix unused variable warnings in donna's Curve25519 SSE2 code.
+      Fixes bug 22895; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (logging, relay shutdown, annoyance, backport from 0.3.2.2-alpha):
+    - When a circuit is marked for close, do not attempt to package any
+      cells for channels on that circuit. Previously, we would detect
+      this condition lower in the call stack, when we noticed that the
+      circuit had no attached channel, and log an annoying message.
+      Fixes bug 8185; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (relay, crash, backport from 0.3.2.4-alpha):
+    - Avoid a crash when transitioning from client mode to bridge mode.
+      Previously, we would launch the worker threads whenever our
+      "public server" mode changed, but not when our "server" mode
+      changed. Fixes bug 23693; bugfix on 0.2.6.3-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.1.6-rc):
+    - Fix an undersized buffer in test-memwipe.c. Fixes bug 23291;
+      bugfix on 0.2.7.2-alpha. Found and patched by Ties Stuij.
+
+
+Changes in version 0.2.9.14 - 2017-12-01
+  Tor 0.3.0.13 backports important security and stability bugfixes from
+  later Tor releases. All Tor users should upgrade to this release, or
+  to another of the releases coming out today.
+
+  o Major bugfixes (exit relays, DNS, backport from 0.3.2.4-alpha):
+    - Fix an issue causing DNS to fail on high-bandwidth exit nodes,
+      making them nearly unusable. Fixes bugs 21394 and 18580; bugfix on
+      0.1.2.2-alpha, which introduced eventdns. Thanks to Dhalgren for
+      identifying and finding a workaround to this bug and to Moritz,
+      Arthur Edelstein, and Roger for helping to track it down and
+      analyze it.
+
+  o Major bugfixes (security, backport from 0.3.2.6-alpha):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - Fix a denial of service issue where an attacker could crash a
+      directory authority using a malformed router descriptor. Fixes bug
+      24245; bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2017-010
+      and CVE-2017-8820.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, onion service v2, backport from 0.3.2.6-alpha):
+    - Fix a use-after-free error that could crash v2 Tor onion services
+      when they failed to open circuits while expiring introduction
+      points. Fixes bug 24313; bugfix on 0.2.7.2-alpha. This issue is
+      also tracked as TROVE-2017-013 and CVE-2017-8823.
+
+  o Major bugfixes (security, relay, backport from 0.3.2.6-alpha):
+    - When running as a relay, make sure that we never build a path
+      through ourselves, even in the case where we have somehow lost the
+      version of our descriptor appearing in the consensus. Fixes part
+      of bug 21534; bugfix on 0.2.0.1-alpha. This issue is also tracked
+      as TROVE-2017-012 and CVE-2017-8822.
+
+  o Minor features (bridge, backport from 0.3.1.9):
+    - Bridges now include notice in their descriptors that they are
+      bridges, and notice of their distribution status, based on their
+      publication settings. Implements ticket 18329. For more fine-
+      grained control of how a bridge is distributed, upgrade to 0.3.2.x
+      or later.
+
+  o Minor features (directory authority, backport from 0.3.2.6-alpha):
+    - Add an IPv6 address for the "bastet" directory authority. Closes
+      ticket 24394.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (security, windows, backport from 0.3.1.1-alpha):
+    - Enable a couple of pieces of Windows hardening: one
+      (HeapEnableTerminationOnCorruption) that has been on-by-default
+      since Windows 8, and unavailable before Windows 7; and one
+      (PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION) which we believe doesn't
+      affect us, but shouldn't do any harm. Closes ticket 21953.
+
+  o Minor bugfix (relay address resolution, backport from 0.3.2.1-alpha):
+    - Avoid unnecessary calls to directory_fetches_from_authorities() on
+      relays, to prevent spurious address resolutions and descriptor
+      rebuilds. This is a mitigation for bug 21789. Fixes bug 23470;
+      bugfix on in 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.2.1-alpha):
+    - Fix unused variable warnings in donna's Curve25519 SSE2 code.
+      Fixes bug 22895; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (logging, relay shutdown, annoyance, backport from 0.3.2.2-alpha):
+    - When a circuit is marked for close, do not attempt to package any
+      cells for channels on that circuit. Previously, we would detect
+      this condition lower in the call stack, when we noticed that the
+      circuit had no attached channel, and log an annoying message.
+      Fixes bug 8185; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (relay, crash, backport from 0.3.2.4-alpha):
+    - Avoid a crash when transitioning from client mode to bridge mode.
+      Previously, we would launch the worker threads whenever our
+      "public server" mode changed, but not when our "server" mode
+      changed. Fixes bug 23693; bugfix on 0.2.6.3-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.1.6-rc):
+    - Fix an undersized buffer in test-memwipe.c. Fixes bug 23291;
+      bugfix on 0.2.7.2-alpha. Found and patched by Ties Stuij.
+
+
+Changes in version 0.2.8.17 - 2017-12-01
+  Tor 0.2.8.17 backports important security and stability bugfixes from
+  later Tor releases. All Tor users should upgrade to this release, or
+  to another of the releases coming out today.
+
+  Note: the Tor 0.2.8 series will no longer be supported after 1 Jan
+  2018. If you need a release with long-term support, please upgrade with
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Major bugfixes (security, backport from 0.3.2.6-alpha):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, onion service v2, backport from 0.3.2.6-alpha):
+    - Fix a use-after-free error that could crash v2 Tor onion services
+      when they failed to open circuits while expiring introduction
+      points. Fixes bug 24313; bugfix on 0.2.7.2-alpha. This issue is
+      also tracked as TROVE-2017-013 and CVE-2017-8823.
+
+  o Major bugfixes (security, relay, backport from 0.3.2.6-alpha):
+    - When running as a relay, make sure that we never build a path through
+      ourselves, even in the case where we have somehow lost the version of
+      our descriptor appearing in the consensus. Fixes part of bug 21534;
+      bugfix on 0.2.0.1-alpha. This issue is also tracked as TROVE-2017-012
+      and CVE-2017-8822.
+
+  o Minor features (bridge, backport from 0.3.1.9):
+    - Bridges now include notice in their descriptors that they are
+      bridges, and notice of their distribution status, based on their
+      publication settings. Implements ticket 18329. For more fine-
+      grained control of how a bridge is distributed, upgrade to 0.3.2.x
+      or later.
+
+  o Minor features (directory authority, backport from 0.3.2.6-alpha):
+    - Add an IPv6 address for the "bastet" directory authority. Closes
+      ticket 24394.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (testing, backport from 0.3.1.6-rc):
+    - Fix an undersized buffer in test-memwipe.c. Fixes bug 23291;
+      bugfix on 0.2.7.2-alpha. Found and patched by Ties Stuij.
+
+
+Changes in version 0.2.5.16 - 2017-12-01
+  Tor 0.2.5.13 backports important security and stability bugfixes from
+  later Tor releases. All Tor users should upgrade to this release, or
+  to another of the releases coming out today.
+
+  Note: the Tor 0.2.5 series will no longer be supported after 1 May
+  2018. If you need a release with long-term support, please upgrade to
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Major bugfixes (security, backport from 0.3.2.6-alpha):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, relay, backport from 0.3.2.6-alpha):
+    - When running as a relay, make sure that we never build a path
+      through ourselves, even in the case where we have somehow lost the
+      version of our descriptor appearing in the consensus. Fixes part
+      of bug 21534; bugfix on 0.2.0.1-alpha. This issue is also tracked
+      as TROVE-2017-012 and CVE-2017-8822.
+
+  o Minor features (bridge, backport from 0.3.1.9):
+    - Bridges now include notice in their descriptors that they are
+      bridges, and notice of their distribution status, based on their
+      publication settings. Implements ticket 18329. For more fine-
+      grained control of how a bridge is distributed, upgrade to 0.3.2.x
+      or later.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.5.15 - 2017-10-25
+  Tor 0.2.5.15 backports a collection of bugfixes from later Tor release
+  series. It also adds a new directory authority, Bastet.
+
+  Note: the Tor 0.2.5 series will no longer be supported after 1 May
+  2018. If you need a release with long-term support, please upgrade to
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Major bugfixes (openbsd, denial-of-service, backport from 0.3.1.5-alpha):
+    - Avoid an assertion failure bug affecting our implementation of
+      inet_pton(AF_INET6) on certain OpenBSD systems whose strtol()
+      handling of "0xx" differs from what we had expected. Fixes bug
+      22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (defensive programming, undefined behavior, backport from 0.3.1.4-alpha):
+    - Fix a memset() off the end of an array when packing cells. This
+      bug should be harmless in practice, since the corrupted bytes are
+      still in the same structure, and are always padding bytes,
+      ignored, or immediately overwritten, depending on compiler
+      behavior. Nevertheless, because the memset()'s purpose is to make
+      sure that any other cell-handling bugs can't expose bytes to the
+      network, we need to fix it. Fixes bug 22737; bugfix on
+      0.2.4.11-alpha. Fixes CID 1401591.
+
+  o Build features (backport from 0.3.1.5-alpha):
+    - Tor's repository now includes a Travis Continuous Integration (CI)
+      configuration file (.travis.yml). This is meant to help new
+      developers and contributors who fork Tor to a Github repository be
+      better able to test their changes, and understand what we expect
+      to pass. To use this new build feature, you must fork Tor to your
+      Github account, then go into the "Integrations" menu in the
+      repository settings for your fork and enable Travis, then push
+      your changes. Closes ticket 22636.
+
+
+Changes in version 0.2.8.16 - 2017-10-25
+  Tor 0.2.8.16 backports a collection of bugfixes from later Tor release
+  series, including a bugfix for a crash issue that had affected relays
+  under memory pressure. It also adds a new directory authority, Bastet.
+
+  Note: the Tor 0.2.8 series will no longer be supported after 1 Jan
+  2018. If you need a release with long-term support, please stick with
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Major bugfixes (relay, crash, assertion failure, backport from 0.3.2.2-alpha):
+    - Fix a timing-based assertion failure that could occur when the
+      circuit out-of-memory handler freed a connection's output buffer.
+      Fixes bug 23690; bugfix on 0.2.6.1-alpha.
+
+  o Minor features (directory authorities, backport from 0.3.2.2-alpha):
+    - Remove longclaw's IPv6 address, as it will soon change. Authority
+      IPv6 addresses were originally added in 0.2.8.1-alpha. This leaves
+      3/8 directory authorities with IPv6 addresses, but there are also
+      52 fallback directory mirrors with IPv6 addresses. Resolves 19760.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.9.13 - 2017-10-25
+  Tor 0.2.9.13 backports a collection of bugfixes from later Tor release
+  series, including a bugfix for a crash issue that had affected relays
+  under memory pressure. It also adds a new directory authority, Bastet.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Major bugfixes (relay, crash, assertion failure, backport from 0.3.2.2-alpha):
+    - Fix a timing-based assertion failure that could occur when the
+      circuit out-of-memory handler freed a connection's output buffer.
+      Fixes bug 23690; bugfix on 0.2.6.1-alpha.
+
+  o Minor features (directory authorities, backport from 0.3.2.2-alpha):
+    - Remove longclaw's IPv6 address, as it will soon change. Authority
+      IPv6 addresses were originally added in 0.2.8.1-alpha. This leaves
+      3/8 directory authorities with IPv6 addresses, but there are also
+      52 fallback directory mirrors with IPv6 addresses. Resolves 19760.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (directory authority, backport from 0.3.1.5-alpha):
+    - When a directory authority rejects a descriptor or extrainfo with
+      a given digest, mark that digest as undownloadable, so that we do
+      not attempt to download it again over and over. We previously
+      tried to avoid downloading such descriptors by other means, but we
+      didn't notice if we accidentally downloaded one anyway. This
+      behavior became problematic in 0.2.7.2-alpha, when authorities
+      began pinning Ed25519 keys. Fixes bug 22349; bugfix
+      on 0.2.1.19-alpha.
+
+  o Minor bugfixes (memory safety, backport from 0.3.2.3-alpha):
+    - Clear the address when node_get_prim_orport() returns early.
+      Fixes bug 23874; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (Windows service, backport from 0.3.1.6-rc):
+    - When running as a Windows service, set the ID of the main thread
+      correctly. Failure to do so made us fail to send log messages to
+      the controller in 0.2.1.16-rc, slowed down controller event
+      delivery in 0.2.7.3-rc and later, and crash with an assertion
+      failure in 0.3.1.1-alpha. Fixes bug 23081; bugfix on 0.2.1.6-alpha.
+      Patch and diagnosis from "Vort".
+
+
+Changes in version 0.3.0.12 - 2017-10-25
+  Tor 0.3.0.12 backports a collection of bugfixes from later Tor release
+  series, including a bugfix for a crash issue that had affected relays
+  under memory pressure. It also adds a new directory authority, Bastet.
+
+  Note: the Tor 0.3.0 series will no longer be supported after 26 Jan
+  2018. If you need a release with long-term support, please stick with
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Major bugfixes (relay, crash, assertion failure, backport from 0.3.2.2-alpha):
+    - Fix a timing-based assertion failure that could occur when the
+      circuit out-of-memory handler freed a connection's output buffer.
+      Fixes bug 23690; bugfix on 0.2.6.1-alpha.
+
+  o Minor features (directory authorities, backport from 0.3.2.2-alpha):
+    - Remove longclaw's IPv6 address, as it will soon change. Authority
+      IPv6 addresses were originally added in 0.2.8.1-alpha. This leaves
+      3/8 directory authorities with IPv6 addresses, but there are also
+      52 fallback directory mirrors with IPv6 addresses. Resolves 19760.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (directory authority, backport from 0.3.1.5-alpha):
+    - When a directory authority rejects a descriptor or extrainfo with
+      a given digest, mark that digest as undownloadable, so that we do
+      not attempt to download it again over and over. We previously
+      tried to avoid downloading such descriptors by other means, but we
+      didn't notice if we accidentally downloaded one anyway. This
+      behavior became problematic in 0.2.7.2-alpha, when authorities
+      began pinning Ed25519 keys. Fixes bug 22349; bugfix
+      on 0.2.1.19-alpha.
+
+  o Minor bugfixes (hidden service, relay, backport from 0.3.2.2-alpha):
+    - Avoid a possible double close of a circuit by the intro point on
+      error of sending the INTRO_ESTABLISHED cell. Fixes bug 23610;
+      bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (memory safety, backport from 0.3.2.3-alpha):
+    - Clear the address when node_get_prim_orport() returns early.
+      Fixes bug 23874; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (Windows service, backport from 0.3.1.6-rc):
+    - When running as a Windows service, set the ID of the main thread
+      correctly. Failure to do so made us fail to send log messages to
+      the controller in 0.2.1.16-rc, slowed down controller event
+      delivery in 0.2.7.3-rc and later, and crash with an assertion
+      failure in 0.3.1.1-alpha. Fixes bug 23081; bugfix on 0.2.1.6-alpha.
+      Patch and diagnosis from "Vort".
+
+
+Changes in version 0.3.1.8 - 2017-10-25
+  Tor 0.3.1.8 is the second stable release in the 0.3.1 series.
+  It includes several bugfixes, including a bugfix for a crash issue
+  that had affected relays under memory pressure. It also adds
+  a new directory authority, Bastet.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Major bugfixes (relay, crash, assertion failure, backport from 0.3.2.2-alpha):
+    - Fix a timing-based assertion failure that could occur when the
+      circuit out-of-memory handler freed a connection's output buffer.
+      Fixes bug 23690; bugfix on 0.2.6.1-alpha.
+
+  o Minor features (directory authorities, backport from 0.3.2.2-alpha):
+    - Remove longclaw's IPv6 address, as it will soon change. Authority
+      IPv6 addresses were originally added in 0.2.8.1-alpha. This leaves
+      3/8 directory authorities with IPv6 addresses, but there are also
+      52 fallback directory mirrors with IPv6 addresses. Resolves 19760.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.3.2.2-alpha):
+    - Fix a compilation warning when building with zstd support on
+      32-bit platforms. Fixes bug 23568; bugfix on 0.3.1.1-alpha. Found
+      and fixed by Andreas Stieger.
+
+  o Minor bugfixes (compression, backport from 0.3.2.2-alpha):
+    - Handle a pathological case when decompressing Zstandard data when
+      the output buffer size is zero. Fixes bug 23551; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (directory authority, backport from 0.3.2.1-alpha):
+    - Remove the length limit on HTTP status lines that authorities can
+      send in their replies. Fixes bug 23499; bugfix on 0.3.1.6-rc.
+
+  o Minor bugfixes (hidden service, relay, backport from 0.3.2.2-alpha):
+    - Avoid a possible double close of a circuit by the intro point on
+      error of sending the INTRO_ESTABLISHED cell. Fixes bug 23610;
+      bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (memory safety, backport from 0.3.2.3-alpha):
+    - Clear the address when node_get_prim_orport() returns early.
+      Fixes bug 23874; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (unit tests, backport from 0.3.2.2-alpha):
+    - Fix additional channelpadding unit test failures by using mocked
+      time instead of actual time for all tests. Fixes bug 23608; bugfix
+      on 0.3.1.1-alpha.
+
+
+Changes in version 0.2.8.15 - 2017-09-18
+  Tor 0.2.8.15 backports a collection of bugfixes from later
+  Tor series.
+
+  Most significantly, it includes a fix for TROVE-2017-008, a
+  security bug that affects hidden services running with the
+  SafeLogging option disabled. For more information, see
+  https://trac.torproject.org/projects/tor/ticket/23490
+
+  Note that Tor 0.2.8.x will no longer be supported after 1 Jan
+  2018.  We suggest that you upgrade to the latest stable release if
+  possible.  If you can't, we recommend that you upgrade at least to
+  0.2.9, which will be supported until 2020.
+
+  o Major bugfixes (openbsd, denial-of-service, backport from 0.3.1.5-alpha):
+    - Avoid an assertion failure bug affecting our implementation of
+      inet_pton(AF_INET6) on certain OpenBSD systems whose strtol()
+      handling of "0xx" differs from what we had expected. Fixes bug
+      22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007.
+
+  o Minor features:
+    - Update geoip and geoip6 to the September 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, mingw, backport from 0.3.1.1-alpha):
+    - Backport a fix for an "unused variable" warning that appeared
+      in some versions of mingw. Fixes bug 22838; bugfix on
+      0.2.8.1-alpha.
+
+  o Minor bugfixes (defensive programming, undefined behavior, backport from 0.3.1.4-alpha):
+    - Fix a memset() off the end of an array when packing cells. This
+      bug should be harmless in practice, since the corrupted bytes are
+      still in the same structure, and are always padding bytes,
+      ignored, or immediately overwritten, depending on compiler
+      behavior. Nevertheless, because the memset()'s purpose is to make
+      sure that any other cell-handling bugs can't expose bytes to the
+      network, we need to fix it. Fixes bug 22737; bugfix on
+      0.2.4.11-alpha. Fixes CID 1401591.
+
+  o Build features (backport from 0.3.1.5-alpha):
+    - Tor's repository now includes a Travis Continuous Integration (CI)
+      configuration file (.travis.yml). This is meant to help new
+      developers and contributors who fork Tor to a Github repository be
+      better able to test their changes, and understand what we expect
+      to pass. To use this new build feature, you must fork Tor to your
+      Github account, then go into the "Integrations" menu in the
+      repository settings for your fork and enable Travis, then push
+      your changes. Closes ticket 22636.
+
+
+Changes in version 0.2.9.12 - 2017-09-18
+  Tor 0.2.9.12 backports a collection of bugfixes from later
+  Tor series.
+
+  Most significantly, it includes a fix for TROVE-2017-008, a
+  security bug that affects hidden services running with the
+  SafeLogging option disabled. For more information, see
+  https://trac.torproject.org/projects/tor/ticket/23490
+
+  o Major features (security, backport from 0.3.0.2-alpha):
+    - Change the algorithm used to decide DNS TTLs on client and server
+      side, to better resist DNS-based correlation attacks like the
+      DefecTor attack of Greschbach, Pulls, Roberts, Winter, and
+      Feamster. Now relays only return one of two possible DNS TTL
+      values, and clients are willing to believe DNS TTL values up to 3
+      hours long. Closes ticket 19769.
+
+  o Major bugfixes (crash, directory connections, backport from 0.3.0.5-rc):
+    - Fix a rare crash when sending a begin cell on a circuit whose
+      linked directory connection had already been closed. Fixes bug
+      21576; bugfix on 0.2.9.3-alpha. Reported by Alec Muffett.
+
+  o Major bugfixes (DNS, backport from 0.3.0.2-alpha):
+    - Fix a bug that prevented exit nodes from caching DNS records for
+      more than 60 seconds. Fixes bug 19025; bugfix on 0.2.4.7-alpha.
+
+  o Major bugfixes (linux TPROXY support, backport from 0.3.1.1-alpha):
+    - Fix a typo that had prevented TPROXY-based transparent proxying
+      from working under Linux. Fixes bug 18100; bugfix on 0.2.6.3-alpha.
+      Patch from "d4fq0fQAgoJ".
+
+  o Major bugfixes (openbsd, denial-of-service, backport from 0.3.1.5-alpha):
+    - Avoid an assertion failure bug affecting our implementation of
+      inet_pton(AF_INET6) on certain OpenBSD systems whose strtol()
+      handling of "0xx" differs from what we had expected. Fixes bug
+      22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007.
+
+  o Minor features (code style, backport from 0.3.1.3-alpha):
+    - Add "Falls through" comments to our codebase, in order to silence
+      GCC 7's -Wimplicit-fallthrough warnings. Patch from Andreas
+      Stieger. Closes ticket 22446.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the September 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (bandwidth accounting, backport from 0.3.1.1-alpha):
+    - Roll over monthly accounting at the configured hour and minute,
+      rather than always at 00:00. Fixes bug 22245; bugfix on 0.0.9rc1.
+      Found by Andrey Karpov with PVS-Studio.
+
+  o Minor bugfixes (compilation, backport from 0.3.1.5-alpha):
+    - Suppress -Wdouble-promotion warnings with clang 4.0. Fixes bug 22915;
+      bugfix on 0.2.8.1-alpha.
+    - Fix warnings when building with libscrypt and openssl scrypt support
+      on Clang. Fixes bug 22916; bugfix on 0.2.7.2-alpha.
+    - When building with certain versions the mingw C header files, avoid
+      float-conversion warnings when calling the C functions isfinite(),
+      isnan(), and signbit(). Fixes bug 22801; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.1.7):
+    - Avoid compiler warnings in the unit tests for running tor_sscanf()
+      with wide string outputs. Fixes bug 15582; bugfix on 0.2.6.2-alpha.
+
+  o Minor bugfixes (compilation, mingw, backport from 0.3.1.1-alpha):
+    - Backport a fix for an "unused variable" warning that appeared
+      in some versions of mingw. Fixes bug 22838; bugfix on
+      0.2.8.1-alpha.
+
+  o Minor bugfixes (controller, backport from 0.3.1.7):
+    - Do not crash when receiving a HSPOST command with an empty body.
+      Fixes part of bug 22644; bugfix on 0.2.7.1-alpha.
+    - Do not crash when receiving a POSTDESCRIPTOR command with an
+      empty body. Fixes part of bug 22644; bugfix on 0.2.0.1-alpha.
+
+  o Minor bugfixes (coverity build support, backport from 0.3.1.5-alpha):
+    - Avoid Coverity build warnings related to our BUG() macro. By
+      default, Coverity treats BUG() as the Linux kernel does: an
+      instant abort(). We need to override that so our BUG() macro
+      doesn't prevent Coverity from analyzing functions that use it.
+      Fixes bug 23030; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (defensive programming, undefined behavior, backport from 0.3.1.4-alpha):
+    - Fix a memset() off the end of an array when packing cells. This
+      bug should be harmless in practice, since the corrupted bytes are
+      still in the same structure, and are always padding bytes,
+      ignored, or immediately overwritten, depending on compiler
+      behavior. Nevertheless, because the memset()'s purpose is to make
+      sure that any other cell-handling bugs can't expose bytes to the
+      network, we need to fix it. Fixes bug 22737; bugfix on
+      0.2.4.11-alpha. Fixes CID 1401591.
+
+  o Minor bugfixes (file limits, osx, backport from 0.3.1.5-alpha):
+    - When setting the maximum number of connections allowed by the OS,
+      always allow some extra file descriptors for other files. Fixes
+      bug 22797; bugfix on 0.2.0.10-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.1.5-alpha):
+    - Avoid a sandbox failure when trying to re-bind to a socket and
+      mark it as IPv6-only. Fixes bug 20247; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.1.4-alpha):
+    - Permit the fchmod system call, to avoid crashing on startup when
+      starting with the seccomp2 sandbox and an unexpected set of
+      permissions on the data directory or its contents. Fixes bug
+      22516; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (relay, backport from 0.3.0.5-rc):
+    - Avoid a double-marked-circuit warning that could happen when we
+      receive DESTROY cells under heavy load. Fixes bug 20059; bugfix
+      on 0.1.0.1-rc.
+
+  o Minor bugfixes (voting consistency, backport from 0.3.1.1-alpha):
+    - Reject version numbers with non-numeric prefixes (such as +, -, or
+      whitespace). Disallowing whitespace prevents differential version
+      parsing between POSIX-based and Windows platforms. Fixes bug 21507
+      and part of 21508; bugfix on 0.0.8pre1.
+
+  o Build features (backport from 0.3.1.5-alpha):
+    - Tor's repository now includes a Travis Continuous Integration (CI)
+      configuration file (.travis.yml). This is meant to help new
+      developers and contributors who fork Tor to a Github repository be
+      better able to test their changes, and understand what we expect
+      to pass. To use this new build feature, you must fork Tor to your
+      Github account, then go into the "Integrations" menu in the
+      repository settings for your fork and enable Travis, then push
+      your changes. Closes ticket 22636.
+
+
+Changes in version 0.3.0.11 - 2017-09-18
+  Tor 0.3.0.11 backports a collection of bugfixes from Tor the 0.3.1
+  series.
+
+  Most significantly, it includes a fix for TROVE-2017-008, a
+  security bug that affects hidden services running with the
+  SafeLogging option disabled. For more information, see
+  https://trac.torproject.org/projects/tor/ticket/23490
+
+  o Minor features (code style, backport from 0.3.1.7):
+    - Add "Falls through" comments to our codebase, in order to silence
+      GCC 7's -Wimplicit-fallthrough warnings. Patch from Andreas
+      Stieger. Closes ticket 22446.
+
+  o Minor features:
+    - Update geoip and geoip6 to the September 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.3.1.7):
+    - Avoid compiler warnings in the unit tests for calling tor_sscanf()
+      with wide string outputs. Fixes bug 15582; bugfix on 0.2.6.2-alpha.
+
+  o Minor bugfixes (controller, backport from 0.3.1.7):
+    - Do not crash when receiving a HSPOST command with an empty body.
+      Fixes part of bug 22644; bugfix on 0.2.7.1-alpha.
+    - Do not crash when receiving a POSTDESCRIPTOR command with an empty
+      body. Fixes part of bug 22644; bugfix on 0.2.0.1-alpha.
+
+  o Minor bugfixes (file limits, osx, backport from 0.3.1.5-alpha):
+    - When setting the maximum number of connections allowed by the OS,
+      always allow some extra file descriptors for other files. Fixes
+      bug 22797; bugfix on 0.2.0.10-alpha.
+
+  o Minor bugfixes (logging, relay, backport from 0.3.1.6-rc):
+    - Remove a forgotten debugging message when an introduction point
+      successfully establishes a hidden service prop224 circuit with
+      a client.
+    - Change three other log_warn() for an introduction point to
+      protocol warnings, because they can be failure from the network
+      and are not relevant to the operator. Fixes bug 23078; bugfix on
+      0.3.0.1-alpha and 0.3.0.2-alpha.
+
+
+Changes in version 0.3.1.7 - 2017-09-18
+  Tor 0.3.1.7 is the first stable release in the 0.3.1 series.
+
+  With the 0.3.1 series, Tor now serves and downloads directory
+  information in more compact formats, to save on bandwidth overhead. It
+  also contains a new padding system to resist netflow-based traffic
+  analysis, and experimental support for building parts of Tor in Rust
+  (though no parts of Tor are in Rust yet). There are also numerous
+  small features, bugfixes on earlier release series, and groundwork for
+  the hidden services revamp of 0.3.2.
+
+  This release also includes a fix for TROVE-2017-008, a security bug
+  that affects hidden services running with the SafeLogging option
+  disabled. For more information, see
+  https://trac.torproject.org/projects/tor/ticket/23490
+
+  Per our stable release policy, we plan to support each stable release
+  series for at least the next nine months, or for three months after
+  the first stable release of the next series: whichever is longer. If
+  you need a release with long-term support, we recommend that you stay
+  with the 0.2.9 series.
+
+  Below is a list of the changes since 0.3.0. For a list of all
+  changes since 0.3.1.6-rc, see the ChangeLog file.
+
+  o New dependencies:
+    - To build with zstd and lzma support, Tor now requires the
+      pkg-config tool at build time.
+
+  o Major bugfixes (security, hidden services, loggging):
+    - Fix a bug where we could log uninitialized stack when a certain
+      hidden service error occurred while SafeLogging was disabled.
+      Fixes bug #23490; bugfix on 0.2.7.2-alpha.
+      This is also tracked as TROVE-2017-008 and CVE-2017-0380.
+
+  o Major features (build system, continuous integration):
+    - Tor's repository now includes a Travis Continuous Integration (CI)
+      configuration file (.travis.yml). This is meant to help new
+      developers and contributors who fork Tor to a Github repository be
+      better able to test their changes, and understand what we expect
+      to pass. To use this new build feature, you must fork Tor to your
+      Github account, then go into the "Integrations" menu in the
+      repository settings for your fork and enable Travis, then push
+      your changes. Closes ticket 22636.
+
+  o Major features (directory protocol):
+    - Tor relays and authorities can now serve clients an abbreviated
+      version of the consensus document, containing only the changes
+      since an older consensus document that the client holds. Clients
+      now request these documents when available. When both client and
+      server use this new protocol, they will use far less bandwidth (up
+      to 94% less) to keep the client's consensus up-to-date. Implements
+      proposal 140; closes ticket 13339. Based on work by Daniel Martí.
+    - Tor can now compress directory traffic with lzma or with zstd
+      compression algorithms, which can deliver better bandwidth
+      performance. Because lzma is computationally expensive, it's only
+      used for documents that can be compressed once and served many
+      times. Support for these algorithms requires that tor is built
+      with the libzstd and/or liblzma libraries available. Implements
+      proposal 278; closes ticket 21662.
+    - Relays now perform the more expensive compression operations, and
+      consensus diff generation, in worker threads. This separation
+      avoids delaying the main thread when a new consensus arrives.
+
+  o Major features (experimental):
+    - Tor can now build modules written in Rust. To turn this on, pass
+      the "--enable-rust" flag to the configure script. It's not time to
+      get excited yet: currently, there is no actual Rust functionality
+      beyond some simple glue code, and a notice at startup to tell you
+      that Rust is running. Still, we hope that programmers and
+      packagers will try building Tor with Rust support, so that we can
+      find issues and solve portability problems. Closes ticket 22106.
+
+  o Major features (traffic analysis resistance):
+    - Connections between clients and relays now send a padding cell in
+      each direction every 1.5 to 9.5 seconds (tunable via consensus
+      parameters). This padding will not resist specialized
+      eavesdroppers, but it should be enough to make many ISPs' routine
+      network flow logging less useful in traffic analysis against
+      Tor users.
+
+      Padding is negotiated using Tor's link protocol, so both relays
+      and clients must upgrade for this to take effect. Clients may
+      still send padding despite the relay's version by setting
+      ConnectionPadding 1 in torrc, and may disable padding by setting
+      ConnectionPadding 0 in torrc. Padding may be minimized for mobile
+      users with the torrc option ReducedConnectionPadding. Implements
+      Proposal 251 and Section 2 of Proposal 254; closes ticket 16861.
+    - Relays will publish 24 hour totals of padding and non-padding cell
+      counts to their extra-info descriptors, unless PaddingStatistics 0
+      is set in torrc. These 24 hour totals are also rounded to
+      multiples of 10000.
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure when a hidden service
+      handles a malformed BEGIN cell. Fixes bug 22493, tracked as
+      TROVE-2017-004 and as CVE-2017-0375; bugfix on 0.3.0.1-alpha.
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Major bugfixes (path selection, security):
+    - When choosing which guard to use for a circuit, avoid the exit's
+      family along with the exit itself. Previously, the new guard
+      selection logic avoided the exit, but did not consider its family.
+      Fixes bug 22753; bugfix on 0.3.0.1-alpha. Tracked as TROVE-2017-
+      006 and CVE-2017-0377.
+
+  o Major bugfixes (connection usage):
+    - We use NETINFO cells to try to determine if both relays involved
+      in a connection will agree on the canonical status of that
+      connection. We prefer the connections where this is the case for
+      extend cells, and try to close connections where relays disagree
+      on their canonical status early. Also, we now prefer the oldest
+      valid connection for extend cells. These two changes should reduce
+      the number of long-term connections that are kept open between
+      relays. Fixes bug 17604; bugfix on 0.2.5.5-alpha.
+    - Relays now log hourly statistics (look for
+      "channel_check_for_duplicates" lines) on the total number of
+      connections to other relays. If the number of connections per
+      relay is unexpectedly large, this log message is at notice level.
+      Otherwise it is at info.
+
+  o Major bugfixes (entry guards):
+    - When starting with an old consensus, do not add new entry guards
+      unless the consensus is "reasonably live" (under 1 day old). Fixes
+      one root cause of bug 22400; bugfix on 0.3.0.1-alpha.
+    - Don't block bootstrapping when a primary bridge is offline and we
+      can't get its descriptor. Fixes bug 22325; fixes one case of bug
+      21969; bugfix on 0.3.0.3-alpha.
+
+  o Major bugfixes (linux TPROXY support):
+    - Fix a typo that had prevented TPROXY-based transparent proxying
+      from working under Linux. Fixes bug 18100; bugfix on 0.2.6.3-alpha.
+      Patch from "d4fq0fQAgoJ".
+
+  o Major bugfixes (openbsd, denial-of-service):
+    - Avoid an assertion failure bug affecting our implementation of
+      inet_pton(AF_INET6) on certain OpenBSD systems whose strtol()
+      handling of "0xx" differs from what we had expected. Fixes bug
+      22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007.
+
+  o Major bugfixes (relay, link handshake):
+    - When performing the v3 link handshake on a TLS connection, report
+      that we have the x509 certificate that we actually used on that
+      connection, even if we have changed certificates since that
+      connection was first opened. Previously, we would claim to have
+      used our most recent x509 link certificate, which would sometimes
+      make the link handshake fail. Fixes one case of bug 22460; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (relays, key management):
+    - Regenerate link and authentication certificates whenever the key
+      that signs them changes; also, regenerate link certificates
+      whenever the signed key changes. Previously, these processes were
+      only weakly coupled, and we relays could (for minutes to hours)
+      wind up with an inconsistent set of keys and certificates, which
+      other relays would not accept. Fixes two cases of bug 22460;
+      bugfix on 0.3.0.1-alpha.
+    - When sending an Ed25519 signing->link certificate in a CERTS cell,
+      send the certificate that matches the x509 certificate that we
+      used on the TLS connection. Previously, there was a race condition
+      if the TLS context rotated after we began the TLS handshake but
+      before we sent the CERTS cell. Fixes a case of bug 22460; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor features (security, windows):
+    - Enable a couple of pieces of Windows hardening: one
+      (HeapEnableTerminationOnCorruption) that has been on-by-default
+      since Windows 8, and unavailable before Windows 7; and one
+      (PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION) which we believe doesn't
+      affect us, but shouldn't do any harm. Closes ticket 21953.
+
+  o Minor features (bridge authority):
+    - Add "fingerprint" lines to the networkstatus-bridges file produced
+      by bridge authorities. Closes ticket 22207.
+
+  o Minor features (code style):
+    - Add "Falls through" comments to our codebase, in order to silence
+      GCC 7's -Wimplicit-fallthrough warnings. Patch from Andreas
+      Stieger. Closes ticket 22446.
+
+  o Minor features (config options):
+    - Allow "%include" directives in torrc configuration files. These
+      directives import the settings from other files, or from all the
+      files in a directory. Closes ticket 1922. Code by Daniel Pinto.
+    - Make SAVECONF return an error when overwriting a torrc that has
+      includes. Using SAVECONF with the FORCE option will allow it to
+      overwrite torrc even if includes are used. Related to ticket 1922.
+    - Add "GETINFO config-can-saveconf" to tell controllers if SAVECONF
+      will work without the FORCE option. Related to ticket 1922.
+
+  o Minor features (controller):
+    - Warn the first time that a controller requests data in the long-
+      deprecated 'GETINFO network-status' format. Closes ticket 21703.
+
+  o Minor features (defaults):
+    - The default value for UseCreateFast is now 0: clients which
+      haven't yet received a consensus document will now use a proper
+      ntor handshake to talk to their directory servers whenever they
+      can. Closes ticket 21407.
+    - Onion key rotation and expiry intervals are now defined as a
+      network consensus parameter, per proposal 274. The default
+      lifetime of an onion key is increased from 7 to 28 days. Old onion
+      keys will expire after 7 days by default. This change will make
+      consensus diffs much smaller, and save significant bandwidth.
+      Closes ticket 21641.
+
+  o Minor features (defensive programming):
+    - Create a pair of consensus parameters, nf_pad_tor2web and
+      nf_pad_single_onion, to disable netflow padding in the consensus
+      for non-anonymous connections in case the overhead is high. Closes
+      ticket 17857.
+
+  o Minor features (diagnostic):
+    - Add a stack trace to the bug warnings that can be logged when
+      trying to send an outgoing relay cell with n_chan == 0. Diagnostic
+      attempt for bug 23105.
+    - Add logging messages to try to diagnose a rare bug that seems to
+      generate RSA->Ed25519 cross-certificates dated in the 1970s. We
+      think this is happening because of incorrect system clocks, but
+      we'd like to know for certain. Diagnostic for bug 22466.
+    - Avoid an assertion failure, and log a better error message, when
+      unable to remove a file from the consensus cache on Windows.
+      Attempts to mitigate and diagnose bug 22752.
+
+  o Minor features (directory authority):
+    - Improve the message that authorities report to relays that present
+      RSA/Ed25519 keypairs that conflict with previously pinned keys.
+      Closes ticket 22348.
+
+  o Minor features (directory cache, consensus diff):
+    - Add a new MaxConsensusAgeForDiffs option to allow directory cache
+      operators with low-resource environments to adjust the number of
+      consensuses they'll store and generate diffs from. Most cache
+      operators should leave it unchanged. Helps to work around
+      bug 22883.
+
+  o Minor features (fallback directory list):
+    - Update the fallback directory mirror whitelist and blacklist based
+      on operator emails. Closes task 21121.
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the September 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (hidden services, logging):
+    - Log a message when a hidden service descriptor has fewer
+      introduction points than specified in
+      HiddenServiceNumIntroductionPoints. Closes tickets 21598.
+    - Log a message when a hidden service reaches its introduction point
+      circuit limit, and when that limit is reset. Follow up to ticket
+      21594; closes ticket 21622.
+    - Warn user if multiple entries in EntryNodes and at least one
+      HiddenService are used together. Pinning EntryNodes along with a
+      hidden service can be possibly harmful; for instance see ticket
+      14917 or 21155. Closes ticket 21155.
+
+  o Minor features (linux seccomp2 sandbox):
+    - We now have a document storage backend compatible with the Linux
+      seccomp2 sandbox. This backend is used for consensus documents and
+      diffs between them; in the long term, we'd like to use it for
+      unparseable directory material too. Closes ticket 21645
+    - Increase the maximum allowed size passed to mprotect(PROT_WRITE)
+      from 1MB to 16MB. This was necessary with the glibc allocator in
+      order to allow worker threads to allocate more memory -- which in
+      turn is necessary because of our new use of worker threads for
+      compression. Closes ticket 22096.
+
+  o Minor features (logging):
+    - Log files are no longer created world-readable by default.
+      (Previously, most distributors would store the logs in a non-
+      world-readable location to prevent inappropriate access. This
+      change is an extra precaution.) Closes ticket 21729; patch
+      from toralf.
+
+  o Minor features (performance):
+    - Our Keccak (SHA-3) implementation now accesses memory more
+      efficiently, especially on little-endian systems. Closes
+      ticket 21737.
+    - Add an O(1) implementation of channel_find_by_global_id(), to
+      speed some controller functions.
+
+  o Minor features (relay, configuration):
+    - The MyFamily option may now be repeated as many times as desired,
+      for relays that want to configure large families. Closes ticket
+      4998; patch by Daniel Pinto.
+
+  o Minor features (relay, performance):
+    - Always start relays with at least two worker threads, to prevent
+      priority inversion on slow tasks. Part of the fix for bug 22883.
+    - Allow background work to be queued with different priorities, so
+      that a big pile of slow low-priority jobs will not starve out
+      higher priority jobs. This lays the groundwork for a fix for
+      bug 22883.
+
+  o Minor features (safety):
+    - Add an explicit check to extrainfo_parse_entry_from_string() for
+      NULL inputs. We don't believe this can actually happen, but it may
+      help silence a warning from the Clang analyzer. Closes
+      ticket 21496.
+
+  o Minor features (testing):
+    - Add more tests for compression backend initialization. Closes
+      ticket 22286.
+    - Add a "--disable-memory-sentinels" feature to help with fuzzing.
+      When Tor is compiled with this option, we disable a number of
+      redundant memory-safety failsafes that are intended to stop bugs
+      from becoming security issues. This makes it easier to hunt for
+      bugs that would be security issues without the failsafes turned
+      on. Closes ticket 21439.
+    - Add a general event-tracing instrumentation support to Tor. This
+      subsystem will enable developers and researchers to add fine-
+      grained instrumentation to their Tor instances, for use when
+      examining Tor network performance issues. There are no trace
+      events yet, and event-tracing is off by default unless enabled at
+      compile time. Implements ticket 13802.
+    - Improve our version parsing tests: add tests for typical version
+      components, add tests for invalid versions, including numeric
+      range and non-numeric prefixes. Unit tests 21278, 21450, and
+      21507. Partially implements 21470.
+
+  o Minor bugfixes (bandwidth accounting):
+    - Roll over monthly accounting at the configured hour and minute,
+      rather than always at 00:00. Fixes bug 22245; bugfix on 0.0.9rc1.
+      Found by Andrey Karpov with PVS-Studio.
+
+  o Minor bugfixes (code correctness):
+    - Accurately identify client connections by their lack of peer
+      authentication. This means that we bail out earlier if asked to
+      extend to a client. Follow-up to 21407. Fixes bug 21406; bugfix
+      on 0.2.4.23.
+
+  o Minor bugfixes (compilation warnings):
+    - Suppress -Wdouble-promotion warnings with clang 4.0. Fixes bug
+      22915; bugfix on 0.2.8.1-alpha.
+    - Fix warnings when building with libscrypt and openssl scrypt
+      support on Clang. Fixes bug 22916; bugfix on 0.2.7.2-alpha.
+    - When building with certain versions of the mingw C header files,
+      avoid float-conversion warnings when calling the C functions
+      isfinite(), isnan(), and signbit(). Fixes bug 22801; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation):
+    - Avoid compiler warnings in the unit tests for calling tor_sscanf()
+      with wide string outputs. Fixes bug 15582; bugfix on 0.2.6.2-alpha.
+
+  o Minor bugfixes (compression):
+    - When spooling compressed data to an output buffer, don't try to
+      spool more data when there is no more data to spool and we are not
+      trying to flush the input. Previously, we would sometimes launch
+      compression requests with nothing to do, which interferes with our
+      22672 checks. Fixes bug 22719; bugfix on 0.2.0.16-alpha.
+
+  o Minor bugfixes (configuration):
+    - Do not crash when starting with LearnCircuitBuildTimeout 0. Fixes
+      bug 22252; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (connection lifespan):
+    - Allow more control over how long TLS connections are kept open:
+      unify CircuitIdleTimeout and PredictedPortsRelevanceTime into a
+      single option called CircuitsAvailableTimeout. Also, allow the
+      consensus to control the default values for both this preference
+      and the lifespan of relay-to-relay connections. Fixes bug 17592;
+      bugfix on 0.2.5.5-alpha.
+    - Increase the initial circuit build timeout testing frequency, to
+      help ensure that ReducedConnectionPadding clients finish learning
+      a timeout before their orconn would expire. The initial testing
+      rate was set back in the days of TAP and before the Tor Browser
+      updater, when we had to be much more careful about new clients
+      making lots of circuits. With this change, a circuit build timeout
+      is learned in about 15-20 minutes, instead of 100-120 minutes.
+
+  o Minor bugfixes (controller):
+    - Do not crash when receiving a HSPOST command with an empty body.
+      Fixes part of bug 22644; bugfix on 0.2.7.1-alpha.
+    - Do not crash when receiving a POSTDESCRIPTOR command with an empty
+      body. Fixes part of bug 22644; bugfix on 0.2.0.1-alpha.
+    - GETINFO onions/current and onions/detached no longer respond with
+      551 on empty lists. Fixes bug 21329; bugfix on 0.2.7.1-alpha.
+    - Trigger HS descriptor events on the control port when the client
+      fails to pick a hidden service directory for a hidden service.
+      This can happen if all the hidden service directories are in
+      ExcludeNodes, or they have all been queried within the last 15
+      minutes. Fixes bug 22042; bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+  o Minor bugfixes (coverity build support):
+    - Avoid Coverity build warnings related to our BUG() macro. By
+      default, Coverity treats BUG() as the Linux kernel does: an
+      instant abort(). We need to override that so our BUG() macro
+      doesn't prevent Coverity from analyzing functions that use it.
+      Fixes bug 23030; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (defensive programming):
+    - Detect and break out of infinite loops in our compression code. We
+      don't think that any such loops exist now, but it's best to be
+      safe. Closes ticket 22672.
+    - Fix a memset() off the end of an array when packing cells. This
+      bug should be harmless in practice, since the corrupted bytes are
+      still in the same structure, and are always padding bytes,
+      ignored, or immediately overwritten, depending on compiler
+      behavior. Nevertheless, because the memset()'s purpose is to make
+      sure that any other cell-handling bugs can't expose bytes to the
+      network, we need to fix it. Fixes bug 22737; bugfix on
+      0.2.4.11-alpha. Fixes CID 1401591.
+
+  o Minor bugfixes (directory authority):
+    - When a directory authority rejects a descriptor or extrainfo with
+      a given digest, mark that digest as undownloadable, so that we do
+      not attempt to download it again over and over. We previously
+      tried to avoid downloading such descriptors by other means, but we
+      didn't notice if we accidentally downloaded one anyway. This
+      behavior became problematic in 0.2.7.2-alpha, when authorities
+      began pinning Ed25519 keys. Fixes bug 22349; bugfix
+      on 0.2.1.19-alpha.
+    - When rejecting a router descriptor for running an obsolete version
+      of Tor without ntor support, warn about the obsolete tor version,
+      not the missing ntor key. Fixes bug 20270; bugfix on 0.2.9.3-alpha.
+    - Prevent the shared randomness subsystem from asserting when
+      initialized by a bridge authority with an incomplete configuration
+      file. Fixes bug 21586; bugfix on 0.2.9.8.
+
+  o Minor bugfixes (error reporting, windows):
+    - When formatting Windows error messages, use the English format to
+      avoid codepage issues. Fixes bug 22520; bugfix on 0.1.2.8-alpha.
+      Patch from "Vort".
+
+  o Minor bugfixes (exit-side DNS):
+    - Fix an untriggerable assertion that checked the output of a
+      libevent DNS error, so that the assertion actually behaves as
+      expected. Fixes bug 22244; bugfix on 0.2.0.20-rc. Found by Andrey
+      Karpov using PVS-Studio.
+
+  o Minor bugfixes (fallback directories):
+    - Make the usage example in updateFallbackDirs.py actually work, and
+      explain what it does. Fixes bug 22270; bugfix on 0.3.0.3-alpha.
+    - Decrease the guard flag average required to be a fallback. This
+      allows us to keep relays that have their guard flag removed when
+      they restart. Fixes bug 20913; bugfix on 0.2.8.1-alpha.
+    - Decrease the minimum number of fallbacks to 100. Fixes bug 20913;
+      bugfix on 0.2.8.1-alpha.
+    - Make sure fallback directory mirrors have the same address, port,
+      and relay identity key for at least 30 days before they are
+      selected. Fixes bug 20913; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (file limits, osx):
+    - When setting the maximum number of connections allowed by the OS,
+      always allow some extra file descriptors for other files. Fixes
+      bug 22797; bugfix on 0.2.0.10-alpha.
+
+  o Minor bugfixes (hidden services):
+    - Increase the number of circuits that a service is allowed to
+      open over a specific period of time. The value was lower than it
+      should be (8 vs 12) in the normal case of 3 introduction points.
+      Fixes bug 22159; bugfix on 0.3.0.5-rc.
+    - Fix a BUG warning during HSv3 descriptor decoding that could be
+      cause by a specially crafted descriptor. Fixes bug 23233; bugfix
+      on 0.3.0.1-alpha. Bug found by "haxxpop".
+    - Stop printing a cryptic warning when a hidden service gets a
+      request to connect to a virtual port that it hasn't configured.
+      Fixes bug 16706; bugfix on 0.2.6.3-alpha.
+    - Simplify hidden service descriptor creation by using an existing
+      flag to check if an introduction point is established. Fixes bug
+      21599; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (link handshake):
+    - Lower the lifetime of the RSA->Ed25519 cross-certificate to six
+      months, and regenerate it when it is within one month of expiring.
+      Previously, we had generated this certificate at startup with a
+      ten-year lifetime, but that could lead to weird behavior when Tor
+      was started with a grossly inaccurate clock. Mitigates bug 22466;
+      mitigation on 0.3.0.1-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox):
+    - Avoid a sandbox failure when trying to re-bind to a socket and
+      mark it as IPv6-only. Fixes bug 20247; bugfix on 0.2.5.1-alpha.
+    - Permit the fchmod system call, to avoid crashing on startup when
+      starting with the seccomp2 sandbox and an unexpected set of
+      permissions on the data directory or its contents. Fixes bug
+      22516; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (logging):
+    - When decompressing, do not warn if we fail to decompress using a
+      compression method that we merely guessed. Fixes part of bug
+      22670; bugfix on 0.1.1.14-alpha.
+    - When decompressing, treat mismatch between content-encoding and
+      actual compression type as a protocol warning. Fixes part of bug
+      22670; bugfix on 0.1.1.9-alpha.
+    - Downgrade "assigned_to_cpuworker failed" message to info-level
+      severity. In every case that can reach it, either a better warning
+      has already been logged, or no warning is warranted. Fixes bug
+      22356; bugfix on 0.2.6.3-alpha.
+    - Log a better message when a directory authority replies to an
+      upload with an unexpected status code. Fixes bug 11121; bugfix
+      on 0.1.0.1-rc.
+    - Downgrade a log statement about unexpected relay cells from "bug"
+      to "protocol warning", because there is at least one use case
+      where it can be triggered by a buggy tor implementation. Fixes bug
+      21293; bugfix on 0.1.1.14-alpha.
+
+  o Minor bugfixes (logging, relay):
+    - Remove a forgotten debugging message when an introduction point
+      successfully establishes a hidden service prop224 circuit with
+      a client.
+    - Change three other log_warn() for an introduction point to
+      protocol warnings, because they can be failure from the network
+      and are not relevant to the operator. Fixes bug 23078; bugfix on
+      0.3.0.1-alpha and 0.3.0.2-alpha.
+
+  o Minor bugfixes (relay):
+    - Inform the geoip and rephist modules about all requests, even on
+      relays that are only fetching microdescriptors. Fixes a bug
+      related to 21585; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (memory leaks):
+    - Fix a small memory leak at exit from the backtrace handler code.
+      Fixes bug 21788; bugfix on 0.2.5.2-alpha. Patch from Daniel Pinto.
+    - When directory authorities reject a router descriptor due to
+      keypinning, free the router descriptor rather than leaking the
+      memory. Fixes bug 22370; bugfix on 0.2.7.2-alpha.
+    - Fix a small memory leak when validating a configuration that uses
+      two or more AF_UNIX sockets for the same port type. Fixes bug
+      23053; bugfix on 0.2.6.3-alpha. This is CID 1415725.
+
+  o Minor bugfixes (process behavior):
+    - When exiting because of an error, always exit with a nonzero exit
+      status. Previously, we would fail to report an error in our exit
+      status in cases related to __OwningControllerProcess failure,
+      lockfile contention, and Ed25519 key initialization. Fixes bug
+      22720; bugfix on versions 0.2.1.6-alpha, 0.2.2.28-beta, and
+      0.2.7.2-alpha respectively. Reported by "f55jwk4f"; patch
+      from "huyvq".
+
+  o Minor bugfixes (robustness, error handling):
+    - Improve our handling of the cases where OpenSSL encounters a
+      memory error while encoding keys and certificates. We haven't
+      observed these errors in the wild, but if they do happen, we now
+      detect and respond better. Fixes bug 19418; bugfix on all versions
+      of Tor. Reported by Guido Vranken.
+
+  o Minor bugfixes (testing):
+    - Fix an undersized buffer in test-memwipe.c. Fixes bug 23291;
+      bugfix on 0.2.7.2-alpha. Found and patched by Ties Stuij.
+    - Use unbuffered I/O for utility functions around the
+      process_handle_t type. This fixes unit test failures reported on
+      OpenBSD and FreeBSD. Fixes bug 21654; bugfix on 0.2.3.1-alpha.
+    - Make display of captured unit test log messages consistent. Fixes
+      bug 21510; bugfix on 0.2.9.3-alpha.
+    - Make test-network.sh always call chutney's test-network.sh.
+      Previously, this only worked on systems which had bash installed,
+      due to some bash-specific code in the script. Fixes bug 19699;
+      bugfix on 0.3.0.4-rc. Follow-up to ticket 21581.
+    - Fix a memory leak in the link-handshake/certs_ok_ed25519 test.
+      Fixes bug 22803; bugfix on 0.3.0.1-alpha.
+    - The unit tests now pass on systems where localhost is misconfigured
+      to some IPv4 address other than 127.0.0.1. Fixes bug 6298; bugfix
+      on 0.0.9pre2.
+
+  o Minor bugfixes (voting consistency):
+    - Reject version numbers with non-numeric prefixes (such as +, -, or
+      whitespace). Disallowing whitespace prevents differential version
+      parsing between POSIX-based and Windows platforms. Fixes bug 21507
+      and part of 21508; bugfix on 0.0.8pre1.
+
+  o Minor bugfixes (Windows service):
+    - When running as a Windows service, set the ID of the main thread
+      correctly. Failure to do so made us fail to send log messages to
+      the controller in 0.2.1.16-rc, slowed down controller event
+      delivery in 0.2.7.3-rc and later, and crash with an assertion
+      failure in 0.3.1.1-alpha. Fixes bug 23081; bugfix on 0.2.1.6-alpha.
+      Patch and diagnosis from "Vort".
+
+  o Minor bugfixes (windows, relay):
+    - Resolve "Failure from drain_fd: No error" warnings on Windows
+      relays. Fixes bug 21540; bugfix on 0.2.6.3-alpha.
+
+  o Code simplification and refactoring:
+    - Break up the 630-line function connection_dir_client_reached_eof()
+      into a dozen smaller functions. This change should help
+      maintainability and readability of the client directory code.
+    - Isolate our use of the openssl headers so that they are only
+      included from our crypto wrapper modules, and from tests that
+      examine those modules' internals. Closes ticket 21841.
+    - Simplify our API to launch directory requests, making it more
+      extensible and less error-prone. Now it's easier to add extra
+      headers to directory requests. Closes ticket 21646.
+    - Our base64 decoding functions no longer overestimate the output
+      space that they need when parsing unpadded inputs. Closes
+      ticket 17868.
+    - Remove unused "ROUTER_ADDED_NOTIFY_GENERATOR" internal value.
+      Resolves ticket 22213.
+    - The logic that directory caches use to spool request to clients,
+      serving them one part at a time so as not to allocate too much
+      memory, has been refactored for consistency. Previously there was
+      a separate spooling implementation per type of spoolable data. Now
+      there is one common spooling implementation, with extensible data
+      types. Closes ticket 21651.
+    - Tor's compression module now supports multiple backends. Part of
+      the implementation for proposal 278; closes ticket 21663.
+
+  o Documentation:
+    - Add a manpage description for the key-pinning-journal file. Closes
+      ticket 22347.
+    - Correctly note that bandwidth accounting values are stored in the
+      state file, and the bw_accounting file is now obsolete. Closes
+      ticket 16082.
+    - Document more of the files in the Tor data directory, including
+      cached-extrainfo, secret_onion_key{,_ntor}.old, hidserv-stats,
+      approved-routers, sr-random, and diff-cache. Found while fixing
+      ticket 22347.
+    - Clarify the manpage for the (deprecated) torify script. Closes
+      ticket 6892.
+    - Clarify the behavior of the KeepAliveIsolateSOCKSAuth sub-option.
+      Closes ticket 21873.
+    - Correct documentation about the default DataDirectory value.
+      Closes ticket 21151.
+    - Document the default behavior of NumEntryGuards and
+      NumDirectoryGuards correctly. Fixes bug 21715; bugfix
+      on 0.3.0.1-alpha.
+    - Document key=value pluggable transport arguments for Bridge lines
+      in torrc. Fixes bug 20341; bugfix on 0.2.5.1-alpha.
+    - Note that bandwidth-limiting options don't affect TCP headers or
+      DNS. Closes ticket 17170.
+
+  o Removed features (configuration options, all in ticket 22060):
+    - These configuration options are now marked Obsolete, and no longer
+      have any effect: AllowInvalidNodes, AllowSingleHopCircuits,
+      AllowSingleHopExits, ExcludeSingleHopRelays, FastFirstHopPK,
+      TLSECGroup, WarnUnsafeSocks. They were first marked as deprecated
+      in 0.2.9.2-alpha and have now been removed. The previous default
+      behavior is now always chosen; the previous (less secure) non-
+      default behavior is now unavailable.
+    - CloseHSClientCircuitsImmediatelyOnTimeout and
+      CloseHSServiceRendCircuitsImmediatelyOnTimeout were deprecated in
+      0.2.9.2-alpha and now have been removed. HS circuits never close
+      on circuit build timeout; they have a longer timeout period.
+    - {Control,DNS,Dir,Socks,Trans,NATD,OR}ListenAddress were deprecated
+      in 0.2.9.2-alpha and now have been removed. Use the ORPort option
+      (and others) to configure listen-only and advertise-only addresses.
+
+  o Removed features (tools):
+    - We've removed the tor-checkkey tool from src/tools. Long ago, we
+      used it to help people detect RSA keys that were generated by
+      versions of Debian affected by CVE-2008-0166. But those keys have
+      been out of circulation for ages, and this tool is no longer
+      required. Closes ticket 21842.
+
+
+Changes in version 0.3.0.10 - 2017-08-02
+   Tor 0.3.0.10 backports a collection of small-to-medium bugfixes
+   from the current Tor alpha series. OpenBSD users and TPROXY users
+   should upgrade; others are probably okay sticking with 0.3.0.9.
+
+  o Major features (build system, continuous integration, backport from 0.3.1.5-alpha):
+    - Tor's repository now includes a Travis Continuous Integration (CI)
+      configuration file (.travis.yml). This is meant to help new
+      developers and contributors who fork Tor to a Github repository be
+      better able to test their changes, and understand what we expect
+      to pass. To use this new build feature, you must fork Tor to your
+      Github account, then go into the "Integrations" menu in the
+      repository settings for your fork and enable Travis, then push
+      your changes. Closes ticket 22636.
+
+  o Major bugfixes (linux TPROXY support, backport from 0.3.1.1-alpha):
+    - Fix a typo that had prevented TPROXY-based transparent proxying
+      from working under Linux. Fixes bug 18100; bugfix on 0.2.6.3-alpha.
+      Patch from "d4fq0fQAgoJ".
+
+  o Major bugfixes (openbsd, denial-of-service, backport from 0.3.1.5-alpha):
+    - Avoid an assertion failure bug affecting our implementation of
+      inet_pton(AF_INET6) on certain OpenBSD systems whose strtol()
+      handling of "0xbar" differs from what we had expected. Fixes bug
+      22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007.
+
+  o Minor features (backport from 0.3.1.5-alpha):
+    - Update geoip and geoip6 to the July 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (bandwidth accounting, backport from 0.3.1.2-alpha):
+    - Roll over monthly accounting at the configured hour and minute,
+      rather than always at 00:00. Fixes bug 22245; bugfix on 0.0.9rc1.
+      Found by Andrey Karpov with PVS-Studio.
+
+  o Minor bugfixes (compilation warnings, backport from 0.3.1.5-alpha):
+    - Suppress -Wdouble-promotion warnings with clang 4.0. Fixes bug 22915;
+      bugfix on 0.2.8.1-alpha.
+    - Fix warnings when building with libscrypt and openssl scrypt
+      support on Clang. Fixes bug 22916; bugfix on 0.2.7.2-alpha.
+    - When building with certain versions of the mingw C header files,
+      avoid float-conversion warnings when calling the C functions
+      isfinite(), isnan(), and signbit(). Fixes bug 22801; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation, mingw, backport from 0.3.1.1-alpha):
+    - Backport a fix for an "unused variable" warning that appeared
+      in some versions of mingw. Fixes bug 22838; bugfix on
+      0.2.8.1-alpha.
+
+  o Minor bugfixes (coverity build support, backport from 0.3.1.5-alpha):
+    - Avoid Coverity build warnings related to our BUG() macro. By
+      default, Coverity treats BUG() as the Linux kernel does: an
+      instant abort(). We need to override that so our BUG() macro
+      doesn't prevent Coverity from analyzing functions that use it.
+      Fixes bug 23030; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (directory authority, backport from 0.3.1.1-alpha):
+    - When rejecting a router descriptor for running an obsolete version
+      of Tor without ntor support, warn about the obsolete tor version,
+      not the missing ntor key. Fixes bug 20270; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.1.5-alpha):
+    - Avoid a sandbox failure when trying to re-bind to a socket and
+      mark it as IPv6-only. Fixes bug 20247; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (unit tests, backport from 0.3.1.5-alpha)
+    - Fix a memory leak in the link-handshake/certs_ok_ed25519 test.
+      Fixes bug 22803; bugfix on 0.3.0.1-alpha.
+
+
+Changes in version 0.3.0.9 - 2017-06-29
+  Tor 0.3.0.9 fixes a path selection bug that would allow a client
+  to use a guard that was in the same network family as a chosen exit
+  relay. This is a security regression; all clients running earlier
+  versions of 0.3.0.x or 0.3.1.x should upgrade to 0.3.0.9 or
+  0.3.1.4-alpha.
+
+  This release also backports several other bugfixes from the 0.3.1.x
+  series.
+
+  o Major bugfixes (path selection, security, backport from 0.3.1.4-alpha):
+    - When choosing which guard to use for a circuit, avoid the exit's
+      family along with the exit itself. Previously, the new guard
+      selection logic avoided the exit, but did not consider its family.
+      Fixes bug 22753; bugfix on 0.3.0.1-alpha. Tracked as TROVE-2017-
+      006 and CVE-2017-0377.
+
+  o Major bugfixes (entry guards, backport from 0.3.1.1-alpha):
+    - Don't block bootstrapping when a primary bridge is offline and we
+      can't get its descriptor. Fixes bug 22325; fixes one case of bug
+      21969; bugfix on 0.3.0.3-alpha.
+
+  o Major bugfixes (entry guards, backport from 0.3.1.4-alpha):
+    - When starting with an old consensus, do not add new entry guards
+      unless the consensus is "reasonably live" (under 1 day old). Fixes
+      one root cause of bug 22400; bugfix on 0.3.0.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the June 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (voting consistency, backport from 0.3.1.1-alpha):
+    - Reject version numbers with non-numeric prefixes (such as +, -, or
+      whitespace). Disallowing whitespace prevents differential version
+      parsing between POSIX-based and Windows platforms. Fixes bug 21507
+      and part of 21508; bugfix on 0.0.8pre1.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.1.4-alpha):
+    - Permit the fchmod system call, to avoid crashing on startup when
+      starting with the seccomp2 sandbox and an unexpected set of
+      permissions on the data directory or its contents. Fixes bug
+      22516; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (defensive programming, backport from 0.3.1.4-alpha):
+    - Fix a memset() off the end of an array when packing cells. This
+      bug should be harmless in practice, since the corrupted bytes are
+      still in the same structure, and are always padding bytes,
+      ignored, or immediately overwritten, depending on compiler
+      behavior. Nevertheless, because the memset()'s purpose is to make
+      sure that any other cell-handling bugs can't expose bytes to the
+      network, we need to fix it. Fixes bug 22737; bugfix on
+      0.2.4.11-alpha. Fixes CID 1401591.
+
+
+Changes in version 0.3.0.8 - 2017-06-08
+  Tor 0.3.0.8 fixes a pair of bugs that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-004 and TROVE-2017-005.
+
+  Tor 0.3.0.8 also includes fixes for several key management bugs
+  that sometimes made relays unreliable, as well as several other
+  bugfixes described below.
+
+  o Major bugfixes (hidden service, relay, security, backport
+    from 0.3.1.3-alpha):
+    - Fix a remotely triggerable assertion failure when a hidden service
+      handles a malformed BEGIN cell. Fixes bug 22493, tracked as
+      TROVE-2017-004 and as CVE-2017-0375; bugfix on 0.3.0.1-alpha.
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Major bugfixes (relay, link handshake, backport from 0.3.1.3-alpha):
+    - When performing the v3 link handshake on a TLS connection, report
+      that we have the x509 certificate that we actually used on that
+      connection, even if we have changed certificates since that
+      connection was first opened. Previously, we would claim to have
+      used our most recent x509 link certificate, which would sometimes
+      make the link handshake fail. Fixes one case of bug 22460; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (relays, key management, backport from 0.3.1.3-alpha):
+    - Regenerate link and authentication certificates whenever the key
+      that signs them changes; also, regenerate link certificates
+      whenever the signed key changes. Previously, these processes were
+      only weakly coupled, and we relays could (for minutes to hours)
+      wind up with an inconsistent set of keys and certificates, which
+      other relays would not accept. Fixes two cases of bug 22460;
+      bugfix on 0.3.0.1-alpha.
+    - When sending an Ed25519 signing->link certificate in a CERTS cell,
+      send the certificate that matches the x509 certificate that we
+      used on the TLS connection. Previously, there was a race condition
+      if the TLS context rotated after we began the TLS handshake but
+      before we sent the CERTS cell. Fixes a case of bug 22460; bugfix
+      on 0.3.0.1-alpha.
+
+  o Major bugfixes (hidden service v3, backport from 0.3.1.1-alpha):
+    - Stop rejecting v3 hidden service descriptors because their size
+      did not match an old padding rule. Fixes bug 22447; bugfix on
+      tor-0.3.0.1-alpha.
+
+  o Minor features (fallback directory list, backport from 0.3.1.3-alpha):
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor bugfixes (configuration, backport from 0.3.1.1-alpha):
+    - Do not crash when starting with LearnCircuitBuildTimeout 0. Fixes
+      bug 22252; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (correctness, backport from 0.3.1.3-alpha):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+  o Minor bugfixes (link handshake, backport from 0.3.1.3-alpha):
+    - Lower the lifetime of the RSA->Ed25519 cross-certificate to six
+      months, and regenerate it when it is within one month of expiring.
+      Previously, we had generated this certificate at startup with a
+      ten-year lifetime, but that could lead to weird behavior when Tor
+      was started with a grossly inaccurate clock. Mitigates bug 22466;
+      mitigation on 0.3.0.1-alpha.
+
+  o Minor bugfixes (memory leak, directory authority, backport from
+    0.3.1.2-alpha):
+    - When directory authorities reject a router descriptor due to
+      keypinning, free the router descriptor rather than leaking the
+      memory. Fixes bug 22370; bugfix on 0.2.7.2-alpha.
+
+
+Changes in version 0.2.9.11 - 2017-06-08
+  Tor 0.2.9.11 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005. (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  Tor 0.2.9.11 also backports fixes for several key management bugs
+  that sometimes made relays unreliable, as well as several other
+  bugfixes described below.
+
+  o Major bugfixes (hidden service, relay, security, backport
+    from 0.3.1.3-alpha):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Major bugfixes (relay, link handshake, backport from 0.3.1.3-alpha):
+    - When performing the v3 link handshake on a TLS connection, report
+      that we have the x509 certificate that we actually used on that
+      connection, even if we have changed certificates since that
+      connection was first opened. Previously, we would claim to have
+      used our most recent x509 link certificate, which would sometimes
+      make the link handshake fail. Fixes one case of bug 22460; bugfix
+      on 0.2.3.6-alpha.
+
+  o Minor features (fallback directory list, backport from 0.3.1.3-alpha):
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor features (future-proofing, backport from 0.3.0.7):
+    - Tor no longer refuses to download microdescriptors or descriptors if
+      they are listed as "published in the future".  This change will
+      eventually allow us to stop listing meaningful "published" dates
+      in microdescriptor consensuses, and thereby allow us to reduce the
+      resources required to download consensus diffs by over 50%.
+      Implements part of ticket 21642; implements part of proposal 275.
+
+  o Minor features (directory authorities, backport from 0.3.0.4-rc)
+    - Directory authorities now reject relays running versions
+      0.2.9.1-alpha through 0.2.9.4-alpha, because those relays
+      suffer from bug 20499 and don't keep their consensus cache
+      up-to-date. Resolves ticket 20509.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (control port, backport from 0.3.0.6):
+    - The GETINFO extra-info/digest/<digest> command was broken because
+      of a wrong base16 decode return value check, introduced when
+      refactoring that API. Fixes bug 22034; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (correctness, backport from 0.3.1.3-alpha):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+  o Minor bugfixes (Linux seccomp2 sandbox, backport from 0.3.0.7):
+    - The getpid() system call is now permitted under the Linux seccomp2
+      sandbox, to avoid crashing with versions of OpenSSL (and other
+      libraries) that attempt to learn the process's PID by using the
+      syscall rather than the VDSO code. Fixes bug 21943; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (memory leak, directory authority, backport
+    from 0.3.1.2-alpha):
+    - When directory authorities reject a router descriptor due to
+      keypinning, free the router descriptor rather than leaking the
+      memory. Fixes bug 22370; bugfix on 0.2.7.2-alpha.
+
+Changes in version 0.2.8.14 - 2017-06-08
+  Tor 0.2.7.8 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (fallback directory list, backport from 0.3.1.3-alpha):
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+Changes in version 0.2.7.8 - 2017-06-08
+  Tor 0.2.7.8 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+
+Changes in version 0.2.6.12 - 2017-06-08
+  Tor 0.2.6.12 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+Changes in version 0.2.5.14 - 2017-06-08
+  Tor 0.2.5.14 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+Changes in version 0.2.4.29 - 2017-06-08
+  Tor 0.2.4.29 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+
+Changes in version 0.3.0.7 - 2017-05-15
+  Tor 0.3.0.7 fixes a medium-severity security bug in earlier versions
+  of Tor 0.3.0.x, where an attacker could cause a Tor relay process
+  to exit. Relays running earlier versions of Tor 0.3.0.x should upgrade;
+  clients are not affected.
+
+  o Major bugfixes (hidden service directory, security):
+    - Fix an assertion failure in the hidden service directory code, which
+      could be used by an attacker to remotely cause a Tor relay process to
+      exit. Relays running earlier versions of Tor 0.3.0.x should upgrade.
+      should upgrade. This security issue is tracked as TROVE-2017-002.
+      Fixes bug 22246; bugfix on 0.3.0.1-alpha.
+
+  o Minor features:
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (future-proofing):
+    - Tor no longer refuses to download microdescriptors or descriptors
+      if they are listed as "published in the future". This change will
+      eventually allow us to stop listing meaningful "published" dates
+      in microdescriptor consensuses, and thereby allow us to reduce the
+      resources required to download consensus diffs by over 50%.
+      Implements part of ticket 21642; implements part of proposal 275.
+
+  o Minor bugfixes (Linux seccomp2 sandbox):
+    - The getpid() system call is now permitted under the Linux seccomp2
+      sandbox, to avoid crashing with versions of OpenSSL (and other
+      libraries) that attempt to learn the process's PID by using the
+      syscall rather than the VDSO code. Fixes bug 21943; bugfix
+      on 0.2.5.1-alpha.
+
+
+Changes in version 0.3.0.6 - 2017-04-26
+  Tor 0.3.0.6 is the first stable release of the Tor 0.3.0 series.
+
+  With the 0.3.0 series, clients and relays now use Ed25519 keys to
+  authenticate their link connections to relays, rather than the old
+  RSA1024 keys that they used before. (Circuit crypto has been
+  Curve25519-authenticated since 0.2.4.8-alpha.) We have also replaced
+  the guard selection and replacement algorithm to behave more robustly
+  in the presence of unreliable networks, and to resist guard-
+  capture attacks.
+
+  This series also includes numerous other small features and bugfixes,
+  along with more groundwork for the upcoming hidden-services revamp.
+
+  Per our stable release policy, we plan to support the Tor 0.3.0
+  release series for at least the next nine months, or for three months
+  after the first stable release of the 0.3.1 series: whichever is
+  longer. If you need a release with long-term support, we recommend
+  that you stay with the 0.2.9 series.
+
+  Below are the changes since 0.2.9.10. For a list of only the changes
+  since 0.3.0.5-rc, see the ChangeLog file.
+
+  o Major features (directory authority, security):
+    - The default for AuthDirPinKeys is now 1: directory authorities
+      will reject relays where the RSA identity key matches a previously
+      seen value, but the Ed25519 key has changed. Closes ticket 18319.
+
+  o Major features (guard selection algorithm):
+    - Tor's guard selection algorithm has been redesigned from the
+      ground up, to better support unreliable networks and restrictive
+      sets of entry nodes, and to better resist guard-capture attacks by
+      hostile local networks. Implements proposal 271; closes
+      ticket 19877.
+
+  o Major features (next-generation hidden services):
+    - Relays can now handle v3 ESTABLISH_INTRO cells as specified by
+      prop224 aka "Next Generation Hidden Services". Service and clients
+      don't use this functionality yet. Closes ticket 19043. Based on
+      initial code by Alec Heifetz.
+    - Relays now support the HSDir version 3 protocol, so that they can
+      can store and serve v3 descriptors. This is part of the next-
+      generation onion service work detailed in proposal 224. Closes
+      ticket 17238.
+
+  o Major features (protocol, ed25519 identity keys):
+    - Clients now support including Ed25519 identity keys in the EXTEND2
+      cells they generate. By default, this is controlled by a consensus
+      parameter, currently disabled. You can turn this feature on for
+      testing by setting ExtendByEd25519ID in your configuration. This
+      might make your traffic appear different than the traffic
+      generated by other users, however. Implements part of ticket
+      15056; part of proposal 220.
+    - Relays now understand requests to extend to other relays by their
+      Ed25519 identity keys. When an Ed25519 identity key is included in
+      an EXTEND2 cell, the relay will only extend the circuit if the
+      other relay can prove ownership of that identity. Implements part
+      of ticket 15056; part of proposal 220.
+    - Relays now use Ed25519 to prove their Ed25519 identities and to
+      one another, and to clients. This algorithm is faster and more
+      secure than the RSA-based handshake we've been doing until now.
+      Implements the second big part of proposal 220; Closes
+      ticket 15055.
+
+  o Major features (security):
+    - Change the algorithm used to decide DNS TTLs on client and server
+      side, to better resist DNS-based correlation attacks like the
+      DefecTor attack of Greschbach, Pulls, Roberts, Winter, and
+      Feamster. Now relays only return one of two possible DNS TTL
+      values, and clients are willing to believe DNS TTL values up to 3
+      hours long. Closes ticket 19769.
+
+  o Major bugfixes (client, onion service, also in 0.2.9.9):
+    - Fix a client-side onion service reachability bug, where multiple
+      socks requests to an onion service (or a single slow request)
+      could cause us to mistakenly mark some of the service's
+      introduction points as failed, and we cache that failure so
+      eventually we run out and can't reach the service. Also resolves a
+      mysterious "Remote server sent bogus reason code 65021" log
+      warning. The bug was introduced in ticket 17218, where we tried to
+      remember the circuit end reason as a uint16_t, which mangled
+      negative values. Partially fixes bug 21056 and fixes bug 20307;
+      bugfix on 0.2.8.1-alpha.
+
+  o Major bugfixes (crash, directory connections):
+    - Fix a rare crash when sending a begin cell on a circuit whose
+      linked directory connection had already been closed. Fixes bug
+      21576; bugfix on 0.2.9.3-alpha. Reported by Alec Muffett.
+
+  o Major bugfixes (directory authority):
+    - During voting, when marking a relay as a probable sybil, do not
+      clear its BadExit flag: sybils can still be bad in other ways
+      too. (We still clear the other flags.) Fixes bug 21108; bugfix
+      on 0.2.0.13-alpha.
+
+  o Major bugfixes (DNS):
+    - Fix a bug that prevented exit nodes from caching DNS records for
+      more than 60 seconds. Fixes bug 19025; bugfix on 0.2.4.7-alpha.
+
+  o Major bugfixes (IPv6 Exits):
+    - Stop rejecting all IPv6 traffic on Exits whose exit policy rejects
+      any IPv6 addresses. Instead, only reject a port over IPv6 if the
+      exit policy rejects that port on more than an IPv6 /16 of
+      addresses. This bug was made worse by 17027 in 0.2.8.1-alpha,
+      which rejected a relay's own IPv6 address by default. Fixes bug
+      21357; bugfix on commit 004f3f4e53 in 0.2.4.7-alpha.
+
+  o Major bugfixes (parsing):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+    - When parsing a malformed content-length field from an HTTP
+      message, do not read off the end of the buffer. This bug was a
+      potential remote denial-of-service attack against Tor clients and
+      relays. A workaround was released in October 2016, to prevent this
+      bug from crashing Tor. This is a fix for the underlying issue,
+      which should no longer matter (if you applied the earlier patch).
+      Fixes bug 20894; bugfix on 0.2.0.16-alpha. Bug found by fuzzing
+      using AFL (http://lcamtuf.coredump.cx/afl/).
+
+  o Major bugfixes (scheduler):
+    - Actually compare circuit policies in ewma_cmp_cmux(). This bug
+      caused the channel scheduler to behave more or less randomly,
+      rather than preferring channels with higher-priority circuits.
+      Fixes bug 20459; bugfix on 0.2.6.2-alpha.
+
+  o Major bugfixes (security, also in 0.2.9.9):
+    - Downgrade the "-ftrapv" option from "always on" to "only on when
+      --enable-expensive-hardening is provided." This hardening option,
+      like others, can turn survivable bugs into crashes--and having it
+      on by default made a (relatively harmless) integer overflow bug
+      into a denial-of-service bug. Fixes bug 21278 (TROVE-2017-001);
+      bugfix on 0.2.9.1-alpha.
+
+  o Minor feature (client):
+    - Enable IPv6 traffic on the SocksPort by default. To disable this,
+      a user will have to specify "NoIPv6Traffic". Closes ticket 21269.
+
+  o Minor feature (fallback scripts):
+    - Add a check_existing mode to updateFallbackDirs.py, which checks
+      if fallbacks in the hard-coded list are working. Closes ticket
+      20174. Patch by haxxpop.
+
+  o Minor feature (protocol versioning):
+    - Add new protocol version for proposal 224. HSIntro now advertises
+      version "3-4" and HSDir version "1-2". Fixes ticket 20656.
+
+  o Minor features (ciphersuite selection):
+    - Allow relays to accept a wider range of ciphersuites, including
+      chacha20-poly1305 and AES-CCM. Closes the other part of 15426.
+    - Clients now advertise a list of ciphersuites closer to the ones
+      preferred by Firefox. Closes part of ticket 15426.
+
+  o Minor features (controller):
+    - Add "GETINFO sr/current" and "GETINFO sr/previous" keys, to expose
+      shared-random values to the controller. Closes ticket 19925.
+    - When HSFETCH arguments cannot be parsed, say "Invalid argument"
+      rather than "unrecognized." Closes ticket 20389; patch from
+      Ivan Markin.
+
+  o Minor features (controller, configuration):
+    - Each of the *Port options, such as SocksPort, ORPort, ControlPort,
+      and so on, now comes with a __*Port variant that will not be saved
+      to the torrc file by the controller's SAVECONF command. This
+      change allows TorBrowser to set up a single-use domain socket for
+      each time it launches Tor. Closes ticket 20956.
+    - The GETCONF command can now query options that may only be
+      meaningful in context-sensitive lists. This allows the controller
+      to query the mixed SocksPort/__SocksPort style options introduced
+      in feature 20956. Implements ticket 21300.
+
+  o Minor features (diagnostic, directory client):
+    - Warn when we find an unexpected inconsistency in directory
+      download status objects. Prevents some negative consequences of
+      bug 20593.
+
+  o Minor features (directory authorities):
+    - Directory authorities now reject descriptors that claim to be
+      malformed versions of Tor. Helps prevent exploitation of
+      bug 21278.
+    - Reject version numbers with components that exceed INT32_MAX.
+      Otherwise 32-bit and 64-bit platforms would behave inconsistently.
+      Fixes bug 21450; bugfix on 0.0.8pre1.
+
+  o Minor features (directory authority):
+    - Add a new authority-only AuthDirTestEd25519LinkKeys option (on by
+      default) to control whether authorities should try to probe relays
+      by their Ed25519 link keys. This option will go away in a few
+      releases--unless we encounter major trouble in our ed25519 link
+      protocol rollout, in which case it will serve as a safety option.
+
+  o Minor features (directory cache):
+    - Relays and bridges will now refuse to serve the consensus they
+      have if they know it is too old for a client to use. Closes
+      ticket 20511.
+
+  o Minor features (ed25519 link handshake):
+    - Advertise support for the ed25519 link handshake using the
+      subprotocol-versions mechanism, so that clients can tell which
+      relays can identity themselves by Ed25519 ID. Closes ticket 20552.
+
+  o Minor features (entry guards):
+    - Add UseEntryGuards to TEST_OPTIONS_DEFAULT_VALUES in order to not
+      break regression tests.
+    - Require UseEntryGuards when UseBridges is set, in order to make
+      sure bridges aren't bypassed. Resolves ticket 20502.
+
+  o Minor features (fallback directories):
+    - Allow 3 fallback relays per operator, which is safe now that we
+      are choosing 200 fallback relays. Closes ticket 20912.
+    - Annotate updateFallbackDirs.py with the bandwidth and consensus
+      weight for each candidate fallback. Closes ticket 20878.
+    - Display the relay fingerprint when downloading consensuses from
+      fallbacks. Closes ticket 20908.
+    - Exclude relays affected by bug 20499 from the fallback list.
+      Exclude relays from the fallback list if they are running versions
+      known to be affected by bug 20499, or if in our tests they deliver
+      a stale consensus (i.e. one that expired more than 24 hours ago).
+      Closes ticket 20539.
+    - Make it easier to change the output sort order of fallbacks.
+      Closes ticket 20822.
+    - Reduce the minimum fallback bandwidth to 1 MByte/s. Part of
+      ticket 18828.
+    - Require fallback directories to have the same address and port for
+      7 days (now that we have enough relays with this stability).
+      Relays whose OnionOO stability timer is reset on restart by bug
+      18050 should upgrade to Tor 0.2.8.7 or later, which has a fix for
+      this issue. Closes ticket 20880; maintains short-term fix
+      in 0.2.8.2-alpha.
+    - Require fallbacks to have flags for 90% of the time (weighted
+      decaying average), rather than 95%. This allows at least 73% of
+      clients to bootstrap in the first 5 seconds without contacting an
+      authority. Part of ticket 18828.
+    - Select 200 fallback directories for each release. Closes
+      ticket 20881.
+
+  o Minor features (fingerprinting resistance, authentication):
+    - Extend the length of RSA keys used for TLS link authentication to
+      2048 bits. (These weren't used for forward secrecy; for forward
+      secrecy, we used P256.) Closes ticket 13752.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the April 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (geoip, also in 0.2.9.9):
+    - Update geoip and geoip6 to the January 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (infrastructure):
+    - Implement smartlist_add_strdup() function. Replaces the use of
+      smartlist_add(sl, tor_strdup(str)). Closes ticket 20048.
+
+  o Minor features (linting):
+    - Enhance the changes file linter to warn on Tor versions that are
+      prefixed with "tor-". Closes ticket 21096.
+
+  o Minor features (logging):
+    - In several places, describe unset ed25519 keys as "<unset>",
+      rather than the scary "AAAAAAAA...AAA". Closes ticket 21037.
+
+  o Minor features (portability, compilation):
+    - Autoconf now checks to determine if OpenSSL structures are opaque,
+      instead of explicitly checking for OpenSSL version numbers. Part
+      of ticket 21359.
+    - Support building with recent LibreSSL code that uses opaque
+      structures. Closes ticket 21359.
+
+  o Minor features (relay):
+    - We now allow separation of exit and relay traffic to different
+      source IP addresses, using the OutboundBindAddressExit and
+      OutboundBindAddressOR options respectively. Closes ticket 17975.
+      Written by Michael Sonntag.
+
+  o Minor features (reliability, crash):
+    - Try better to detect problems in buffers where they might grow (or
+      think they have grown) over 2 GB in size. Diagnostic for
+      bug 21369.
+
+  o Minor features (testing):
+    - During 'make test-network-all', if tor logs any warnings, ask
+      chutney to output them. Requires a recent version of chutney with
+      the 21572 patch. Implements 21570.
+
+  o Minor bugfix (control protocol):
+    - The reply to a "GETINFO config/names" request via the control
+      protocol now spells the type "Dependent" correctly. This is a
+      breaking change in the control protocol. (The field seems to be
+      ignored by the most common known controllers.) Fixes bug 18146;
+      bugfix on 0.1.1.4-alpha.
+    - The GETINFO extra-info/digest/<digest> command was broken because
+      of a wrong base16 decode return value check, introduced when
+      refactoring that API. Fixes bug 22034; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfix (logging):
+    - Don't recommend the use of Tor2web in non-anonymous mode.
+      Recommending Tor2web is a bad idea because the client loses all
+      anonymity. Tor2web should only be used in specific cases by users
+      who *know* and understand the issues. Fixes bug 21294; bugfix
+      on 0.2.9.3-alpha.
+
+  o Minor bugfixes (bug resilience):
+    - Fix an unreachable size_t overflow in base64_decode(). Fixes bug
+      19222; bugfix on 0.2.0.9-alpha. Found by Guido Vranken; fixed by
+      Hans Jerry Illikainen.
+
+  o Minor bugfixes (build):
+    - Replace obsolete Autoconf macros with their modern equivalent and
+      prevent similar issues in the future. Fixes bug 20990; bugfix
+      on 0.1.0.1-rc.
+
+  o Minor bugfixes (certificate expiration time):
+    - Avoid using link certificates that don't become valid till some
+      time in the future. Fixes bug 21420; bugfix on 0.2.4.11-alpha
+
+  o Minor bugfixes (client):
+    - Always recover from failures in extend_info_from_node(), in an
+      attempt to prevent any recurrence of bug 21242. Fixes bug 21372;
+      bugfix on 0.2.3.1-alpha.
+    - When clients that use bridges start up with a cached consensus on
+      disk, they were ignoring it and downloading a new one. Now they
+      use the cached one. Fixes bug 20269; bugfix on 0.2.3.12-alpha.
+
+  o Minor bugfixes (code correctness):
+    - Repair a couple of (unreachable or harmless) cases of the risky
+      comparison-by-subtraction pattern that caused bug 21278.
+
+  o Minor bugfixes (config):
+    - Don't assert on startup when trying to get the options list and
+      LearnCircuitBuildTimeout is set to 0: we are currently parsing the
+      options so of course they aren't ready yet. Fixes bug 21062;
+      bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (configuration):
+    - Accept non-space whitespace characters after the severity level in
+      the `Log` option. Fixes bug 19965; bugfix on 0.2.1.1-alpha.
+    - Support "TByte" and "TBytes" units in options given in bytes.
+      "TB", "terabyte(s)", "TBit(s)" and "terabit(s)" were already
+      supported. Fixes bug 20622; bugfix on 0.2.0.14-alpha.
+
+  o Minor bugfixes (configure, autoconf):
+    - Rename the configure option --enable-expensive-hardening to
+      --enable-fragile-hardening. Expensive hardening makes the tor
+      daemon abort when some kinds of issues are detected. Thus, it
+      makes tor more at risk of remote crashes but safer against RCE or
+      heartbleed bug category. We now try to explain this issue in a
+      message from the configure script. Fixes bug 21290; bugfix
+      on 0.2.5.4-alpha.
+
+  o Minor bugfixes (consensus weight):
+    - Add new consensus method that initializes bw weights to 1 instead
+      of 0. This prevents a zero weight from making it all the way to
+      the end (happens in small testing networks) and causing an error.
+      Fixes bug 14881; bugfix on 0.2.2.17-alpha.
+
+  o Minor bugfixes (crash prevention):
+    - Fix an (currently untriggerable, but potentially dangerous) crash
+      bug when base32-encoding inputs whose sizes are not a multiple of
+      5. Fixes bug 21894; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (dead code):
+    - Remove a redundant check for PidFile changes at runtime in
+      options_transition_allowed(): this check is already performed
+      regardless of whether the sandbox is active. Fixes bug 21123;
+      bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (descriptors):
+    - Correctly recognise downloaded full descriptors as valid, even
+      when using microdescriptors as circuits. This affects clients with
+      FetchUselessDescriptors set, and may affect directory authorities.
+      Fixes bug 20839; bugfix on 0.2.3.2-alpha.
+
+  o Minor bugfixes (directory mirrors):
+    - Allow relays to use directory mirrors without a DirPort: these
+      relays need to be contacted over their ORPorts using a begindir
+      connection. Fixes one case of bug 20711; bugfix on 0.2.8.2-alpha.
+    - Clarify the message logged when a remote relay is unexpectedly
+      missing an ORPort or DirPort: users were confusing this with a
+      local port. Fixes another case of bug 20711; bugfix
+      on 0.2.8.2-alpha.
+
+  o Minor bugfixes (directory system):
+    - Bridges and relays now use microdescriptors (like clients do)
+      rather than old-style router descriptors. Now bridges will blend
+      in with clients in terms of the circuits they build. Fixes bug
+      6769; bugfix on 0.2.3.2-alpha.
+    - Download all consensus flavors, descriptors, and authority
+      certificates when FetchUselessDescriptors is set, regardless of
+      whether tor is a directory cache or not. Fixes bug 20667; bugfix
+      on all recent tor versions.
+
+  o Minor bugfixes (documentation):
+    - Update the tor manual page to document every option that can not
+      be changed while tor is running. Fixes bug 21122.
+
+  o Minor bugfixes (ed25519 certificates):
+    - Correctly interpret ed25519 certificates that would expire some
+      time after 19 Jan 2038. Fixes bug 20027; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (fallback directories):
+    - Avoid checking fallback candidates' DirPorts if they are down in
+      OnionOO. When a relay operator has multiple relays, this
+      prioritizes relays that are up over relays that are down. Fixes
+      bug 20926; bugfix on 0.2.8.3-alpha.
+    - Stop failing when OUTPUT_COMMENTS is True in updateFallbackDirs.py.
+      Fixes bug 20877; bugfix on 0.2.8.3-alpha.
+    - Stop failing when a relay has no uptime data in
+      updateFallbackDirs.py. Fixes bug 20945; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (hidden service):
+    - Clean up the code for expiring intro points with no associated
+      circuits. It was causing, rarely, a service with some expiring
+      introduction points to not open enough additional introduction
+      points. Fixes part of bug 21302; bugfix on 0.2.7.2-alpha.
+    - Resolve two possible underflows which could lead to creating and
+      closing a lot of introduction point circuits in a non-stop loop.
+      Fixes bug 21302; bugfix on 0.2.7.2-alpha.
+    - Stop setting the torrc option HiddenServiceStatistics to "0" just
+      because we're not a bridge or relay. Instead, we preserve whatever
+      value the user set (or didn't set). Fixes bug 21150; bugfix
+      on 0.2.6.2-alpha.
+
+  o Minor bugfixes (hidden services):
+    - Make hidden services check for failed intro point connections,
+      even when they have exceeded their intro point creation limit.
+      Fixes bug 21596; bugfix on 0.2.7.2-alpha. Reported by Alec Muffett.
+    - Make hidden services with 8 to 10 introduction points check for
+      failed circuits immediately after startup. Previously, they would
+      wait for 5 minutes before performing their first checks. Fixes bug
+      21594; bugfix on 0.2.3.9-alpha. Reported by Alec Muffett.
+    - Stop ignoring misconfigured hidden services. Instead, refuse to
+      start tor until the misconfigurations have been corrected. Fixes
+      bug 20559; bugfix on multiple commits in 0.2.7.1-alpha
+      and earlier.
+
+  o Minor bugfixes (IPv6):
+    - Make IPv6-using clients try harder to find an IPv6 directory
+      server. Fixes bug 20999; bugfix on 0.2.8.2-alpha.
+    - When IPv6 addresses have not been downloaded yet (microdesc
+      consensus documents don't list relay IPv6 addresses), use hard-
+      coded addresses for authorities, fallbacks, and configured
+      bridges. Now IPv6-only clients can use microdescriptors. Fixes bug
+      20996; bugfix on b167e82 from 19608 in 0.2.8.5-alpha.
+
+  o Minor bugfixes (memory leak at exit):
+    - Fix a small harmless memory leak at exit of the previously unused
+      RSA->Ed identity cross-certificate. Fixes bug 17779; bugfix
+      on 0.2.7.2-alpha.
+
+  o Minor bugfixes (onion services):
+    - Allow the number of introduction points to be as low as 0, rather
+      than as low as 3. Fixes bug 21033; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (portability):
+    - Use "OpenBSD" compiler macro instead of "OPENBSD" or "__OpenBSD__".
+      It is supported by OpenBSD itself, and also by most OpenBSD
+      variants (such as Bitrig). Fixes bug 20980; bugfix
+      on 0.1.2.1-alpha.
+
+  o Minor bugfixes (portability, also in 0.2.9.9):
+    - Avoid crashing when Tor is built using headers that contain
+      CLOCK_MONOTONIC_COARSE, but then tries to run on an older kernel
+      without CLOCK_MONOTONIC_COARSE. Fixes bug 21035; bugfix
+      on 0.2.9.1-alpha.
+    - Fix Libevent detection on platforms without Libevent 1 headers
+      installed. Fixes bug 21051; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (relay):
+    - Avoid a double-marked-circuit warning that could happen when we
+      receive DESTROY cells under heavy load. Fixes bug 20059; bugfix
+      on 0.1.0.1-rc.
+    - Honor DataDirectoryGroupReadable when tor is a relay. Previously,
+      initializing the keys would reset the DataDirectory to 0700
+      instead of 0750 even if DataDirectoryGroupReadable was set to 1.
+      Fixes bug 19953; bugfix on 0.0.2pre16. Patch by "redfish".
+
+  o Minor bugfixes (testing):
+    - Fix Raspbian build issues related to missing socket errno in
+      test_util.c. Fixes bug 21116; bugfix on 0.2.8.2. Patch by "hein".
+    - Remove undefined behavior from the backtrace generator by removing
+      its signal handler. Fixes bug 21026; bugfix on 0.2.5.2-alpha.
+    - Use bash in src/test/test-network.sh. This ensures we reliably
+      call chutney's newer tools/test-network.sh when available. Fixes
+      bug 21562; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (tor-resolve):
+    - The tor-resolve command line tool now rejects hostnames over 255
+      characters in length. Previously, it would silently truncate them,
+      which could lead to bugs. Fixes bug 21280; bugfix on 0.0.9pre5.
+      Patch by "junglefowl".
+
+  o Minor bugfixes (unit tests):
+    - Allow the unit tests to pass even when DNS lookups of bogus
+      addresses do not fail as expected. Fixes bug 20862 and 20863;
+      bugfix on unit tests introduced in 0.2.8.1-alpha
+      through 0.2.9.4-alpha.
+
+  o Minor bugfixes (util):
+    - When finishing writing a file to disk, if we were about to replace
+      the file with the temporary file created before and we fail to
+      replace it, remove the temporary file so it doesn't stay on disk.
+      Fixes bug 20646; bugfix on 0.2.0.7-alpha. Patch by fk.
+
+  o Minor bugfixes (Windows services):
+    - Be sure to initialize the monotonic time subsystem before using
+      it, even when running as an NT service. Fixes bug 21356; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (Windows):
+    - Check for getpagesize before using it to mmap files. This fixes
+      compilation in some MinGW environments. Fixes bug 20530; bugfix on
+      0.1.2.1-alpha. Reported by "ice".
+
+  o Code simplification and refactoring:
+    - Abolish all global guard context in entrynodes.c; replace with new
+      guard_selection_t structure as preparation for proposal 271.
+      Closes ticket 19858.
+    - Extract magic numbers in circuituse.c into defined variables.
+    - Introduce rend_service_is_ephemeral() that tells if given onion
+      service is ephemeral. Replace unclear NULL-checkings for service
+      directory with this function. Closes ticket 20526.
+    - Refactor circuit_is_available_for_use to remove unnecessary check.
+    - Refactor circuit_predict_and_launch_new for readability and
+      testability. Closes ticket 18873.
+    - Refactor code to manipulate global_origin_circuit_list into
+      separate functions. Closes ticket 20921.
+    - Refactor large if statement in purpose_needs_anonymity to use
+      switch statement instead. Closes part of ticket 20077.
+    - Refactor the hashing API to return negative values for errors, as
+      is done as throughout the codebase. Closes ticket 20717.
+    - Remove data structures that were used to index or_connection
+      objects by their RSA identity digests. These structures are fully
+      redundant with the similar structures used in the
+      channel abstraction.
+    - Remove duplicate code in the channel_write_*cell() functions.
+      Closes ticket 13827; patch from Pingl.
+    - Remove redundant behavior of is_sensitive_dir_purpose, refactor to
+      use only purpose_needs_anonymity. Closes part of ticket 20077.
+    - The code to generate and parse EXTEND and EXTEND2 cells has been
+      replaced with code automatically generated by the
+      "trunnel" utility.
+
+  o Documentation (formatting):
+    - Clean up formatting of tor.1 man page and HTML doc, where <pre>
+      blocks were incorrectly appearing. Closes ticket 20885.
+
+  o Documentation (man page):
+    - Clarify many options in tor.1 and add some min/max values for
+      HiddenService options. Closes ticket 21058.
+
+  o Documentation:
+    - Change '1' to 'weight_scale' in consensus bw weights calculation
+      comments, as that is reality. Closes ticket 20273. Patch
+      from pastly.
+    - Clarify that when ClientRejectInternalAddresses is enabled (which
+      is the default), multicast DNS hostnames for machines on the local
+      network (of the form *.local) are also rejected. Closes
+      ticket 17070.
+    - Correct the value for AuthDirGuardBWGuarantee in the manpage, from
+      250 KBytes to 2 MBytes. Fixes bug 20435; bugfix on 0.2.5.6-alpha.
+    - Include the "TBits" unit in Tor's man page. Fixes part of bug
+      20622; bugfix on 0.2.5.1-alpha.
+    - Small fixes to the fuzzing documentation. Closes ticket 21472.
+    - Stop the man page from incorrectly stating that HiddenServiceDir
+      must already exist. Fixes 20486.
+    - Update the description of the directory server options in the
+      manual page, to clarify that a relay no longer needs to set
+      DirPort in order to be a directory cache. Closes ticket 21720.
+
+  o Removed features:
+    - The AuthDirMaxServersPerAuthAddr option no longer exists: The same
+      limit for relays running on a single IP applies to authority IP
+      addresses as well as to non-authority IP addresses. Closes
+      ticket 20960.
+    - The UseDirectoryGuards torrc option no longer exists: all users
+      that use entry guards will also use directory guards. Related to
+      proposal 271; implements part of ticket 20831.
+
+  o Testing:
+    - Add tests for networkstatus_compute_bw_weights_v10.
+    - Add unit tests circuit_predict_and_launch_new.
+    - Extract dummy_origin_circuit_new so it can be used by other
+      test functions.
+    - New unit tests for tor_htonll(). Closes ticket 19563. Patch
+      from "overcaffeinated".
+    - Perform the coding style checks when running the tests and fail
+      when coding style violations are found. Closes ticket 5500.
+
+
+Changes in version 0.2.8.13 - 2017-03-03
+  Tor 0.2.8.13 backports a security fix from later Tor
+  releases.  Anybody running Tor 0.2.8.12 or earlier should upgrade to this
+  this release, if for some reason they cannot upgrade to a later
+  release series, and if they build Tor with the --enable-expensive-hardening
+  option.
+
+  Note that support for Tor 0.2.8.x is ending next year: we will not issue
+  any fixes for the Tor 0.2.8.x series after 1 Jan 2018.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.7.7 - 2017-03-03
+  Tor 0.2.7.7 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.7.6 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.7.x is ending this year: we will not issue
+  any fixes for the Tor 0.2.7.x series after 1 August 2017.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, client, DNS proxy, backport from 0.2.8.3-alpha):
+    - Stop a crash that could occur when a client running with DNSPort
+      received a query with multiple address types, and the first
+      address type was not supported. Found and fixed by Scott Dial.
+      Fixes bug 18710; bugfix on 0.2.5.4-alpha.
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.6.11 - 2017-03-03
+  Tor 0.2.6.11 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.6.10 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.6.x is ending this year: we will not issue
+  any fixes for the Tor 0.2.6.x series after 1 August 2017.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major features (security fixes, backport from 0.2.9.4-alpha):
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, client, DNS proxy, backport from 0.2.8.3-alpha):
+    - Stop a crash that could occur when a client running with DNSPort
+      received a query with multiple address types, and the first
+      address type was not supported. Found and fixed by Scott Dial.
+      Fixes bug 18710; bugfix on 0.2.5.4-alpha.
+
+  o Major bugfixes (security, correctness, backport from 0.2.7.4-rc):
+    - Fix an error that could cause us to read 4 bytes before the
+      beginning of an openssl string. This bug could be used to cause
+      Tor to crash on systems with unusual malloc implementations, or
+      systems with unusual hardening installed. Fixes bug 17404; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (guard selection, backport from 0.2.7.6):
+    - Actually look at the Guard flag when selecting a new directory
+      guard. When we implemented the directory guard design, we
+      accidentally started treating all relays as if they have the Guard
+      flag during guard selection, leading to weaker anonymity and worse
+      performance. Fixes bug 17772; bugfix on 0.2.4.8-alpha. Discovered
+      by Mohsen Imani.
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.2.7.6):
+    - Fix a compilation warning with Clang 3.6: Do not check the
+      presence of an address which can never be NULL. Fixes bug 17781.
+
+
+Changes in version 0.2.5.13 - 2017-03-03
+  Tor 0.2.5.13 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.5.13 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.5.x is ending next year: we will not issue
+  any fixes for the Tor 0.2.5.x series after 1 May 2018.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major features (security fixes, backport from 0.2.9.4-alpha):
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, client, DNS proxy, backport from 0.2.8.3-alpha):
+    - Stop a crash that could occur when a client running with DNSPort
+      received a query with multiple address types, and the first
+      address type was not supported. Found and fixed by Scott Dial.
+      Fixes bug 18710; bugfix on 0.2.5.4-alpha.
+
+  o Major bugfixes (security, correctness, backport from 0.2.7.4-rc):
+    - Fix an error that could cause us to read 4 bytes before the
+      beginning of an openssl string. This bug could be used to cause
+      Tor to crash on systems with unusual malloc implementations, or
+      systems with unusual hardening installed. Fixes bug 17404; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (guard selection, backport from 0.2.7.6):
+    - Actually look at the Guard flag when selecting a new directory
+      guard. When we implemented the directory guard design, we
+      accidentally started treating all relays as if they have the Guard
+      flag during guard selection, leading to weaker anonymity and worse
+      performance. Fixes bug 17772; bugfix on 0.2.4.8-alpha. Discovered
+      by Mohsen Imani.
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.2.7.6):
+    - Fix a compilation warning with Clang 3.6: Do not check the
+      presence of an address which can never be NULL. Fixes bug 17781.
+
+  o Minor bugfixes (crypto error-handling, backport from 0.2.7.2-alpha):
+    - Check for failures from crypto_early_init, and refuse to continue.
+      A previous typo meant that we could keep going with an
+      uninitialized crypto library, and would have OpenSSL initialize
+      its own PRNG. Fixes bug 16360; bugfix on 0.2.5.2-alpha, introduced
+      when implementing ticket 4900. Patch by "teor".
+
+  o Minor bugfixes (hidden service, backport from 0.2.7.1-alpha):
+    - Fix an out-of-bounds read when parsing invalid INTRODUCE2 cells on
+      a client authorized hidden service. Fixes bug 15823; bugfix
+      on 0.2.1.6-alpha.
+
+
+Changes in version 0.2.4.28 - 2017-03-03
+  Tor 0.2.4.28 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.4.27 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.4.x is ending soon: we will not issue
+  any fixes for the Tor 0.2.4.x series after 1 August 2017.  If you need
+  a Tor release series with long-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major features (security fixes, backport from 0.2.9.4-alpha):
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, correctness, backport from 0.2.7.4-rc):
+    - Fix an error that could cause us to read 4 bytes before the
+      beginning of an openssl string. This bug could be used to cause
+      Tor to crash on systems with unusual malloc implementations, or
+      systems with unusual hardening installed. Fixes bug 17404; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (guard selection, backport from 0.2.7.6):
+    - Actually look at the Guard flag when selecting a new directory
+      guard. When we implemented the directory guard design, we
+      accidentally started treating all relays as if they have the Guard
+      flag during guard selection, leading to weaker anonymity and worse
+      performance. Fixes bug 17772; bugfix on 0.2.4.8-alpha. Discovered
+      by Mohsen Imani.
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (DoS-resistance, backport from 0.2.7.1-alpha):
+    - Make it harder for attackers to overload hidden services with
+      introductions, by blocking multiple introduction requests on the
+      same circuit. Resolves ticket 15515.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.2.7.6):
+    - Fix a compilation warning with Clang 3.6: Do not check the
+      presence of an address which can never be NULL. Fixes bug 17781.
+
+  o Minor bugfixes (hidden service, backport from 0.2.7.1-alpha):
+    - Fix an out-of-bounds read when parsing invalid INTRODUCE2 cells on
+      a client authorized hidden service. Fixes bug 15823; bugfix
+      on 0.2.1.6-alpha.
+
+
+Changes in version 0.2.9.10 - 2017-03-01
+  Tor 0.2.9.10 backports a security fix from later Tor release.  It also
+  includes fixes for some major issues affecting directory authorities,
+  LibreSSL compatibility, and IPv6 correctness.
+
+  The Tor 0.2.9.x release series is now marked as a long-term-support
+  series.  We intend to backport security fixes to 0.2.9.x until at
+  least January of 2020.
+
+  o Major bugfixes (directory authority, 0.3.0.3-alpha):
+    - During voting, when marking a relay as a probable sybil, do not
+      clear its BadExit flag: sybils can still be bad in other ways
+      too. (We still clear the other flags.) Fixes bug 21108; bugfix
+      on 0.2.0.13-alpha.
+
+  o Major bugfixes (IPv6 Exits, backport from 0.3.0.3-alpha):
+    - Stop rejecting all IPv6 traffic on Exits whose exit policy rejects
+      any IPv6 addresses. Instead, only reject a port over IPv6 if the
+      exit policy rejects that port on more than an IPv6 /16 of
+      addresses. This bug was made worse by 17027 in 0.2.8.1-alpha,
+      which rejected a relay's own IPv6 address by default. Fixes bug
+      21357; bugfix on commit 004f3f4e53 in 0.2.4.7-alpha.
+
+  o Major bugfixes (parsing, also in 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (directory authorities, also in 0.3.0.4-rc):
+    - Directory authorities now reject descriptors that claim to be
+      malformed versions of Tor. Helps prevent exploitation of
+      bug 21278.
+    - Reject version numbers with components that exceed INT32_MAX.
+      Otherwise 32-bit and 64-bit platforms would behave inconsistently.
+      Fixes bug 21450; bugfix on 0.0.8pre1.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (portability, compilation, backport from 0.3.0.3-alpha):
+    - Autoconf now checks to determine if OpenSSL structures are opaque,
+      instead of explicitly checking for OpenSSL version numbers. Part
+      of ticket 21359.
+    - Support building with recent LibreSSL code that uses opaque
+      structures. Closes ticket 21359.
+
+  o Minor bugfixes (code correctness, also in 0.3.0.4-rc):
+    - Repair a couple of (unreachable or harmless) cases of the risky
+      comparison-by-subtraction pattern that caused bug 21278.
+
+  o Minor bugfixes (tor-resolve, backport from 0.3.0.3-alpha):
+    - The tor-resolve command line tool now rejects hostnames over 255
+      characters in length. Previously, it would silently truncate them,
+      which could lead to bugs. Fixes bug 21280; bugfix on 0.0.9pre5.
+      Patch by "junglefowl".
+
+
+Changes in version 0.2.9.9 - 2017-01-23
+  Tor 0.2.9.9 fixes a denial-of-service bug where an attacker could
+  cause relays and clients to crash, even if they were not built with
+  the --enable-expensive-hardening option. This bug affects all 0.2.9.x
+  versions, and also affects 0.3.0.1-alpha: all relays running an affected
+  version should upgrade.
+
+  This release also resolves a client-side onion service reachability
+  bug, and resolves a pair of small portability issues.
+
+  o Major bugfixes (security):
+    - Downgrade the "-ftrapv" option from "always on" to "only on when
+      --enable-expensive-hardening is provided." This hardening option,
+      like others, can turn survivable bugs into crashes -- and having
+      it on by default made a (relatively harmless) integer overflow bug
+      into a denial-of-service bug. Fixes bug 21278 (TROVE-2017-001);
+      bugfix on 0.2.9.1-alpha.
+
+  o Major bugfixes (client, onion service):
+    - Fix a client-side onion service reachability bug, where multiple
+      socks requests to an onion service (or a single slow request)
+      could cause us to mistakenly mark some of the service's
+      introduction points as failed, and we cache that failure so
+      eventually we run out and can't reach the service. Also resolves a
+      mysterious "Remote server sent bogus reason code 65021" log
+      warning. The bug was introduced in ticket 17218, where we tried to
+      remember the circuit end reason as a uint16_t, which mangled
+      negative values. Partially fixes bug 21056 and fixes bug 20307;
+      bugfix on 0.2.8.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the January 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (portability):
+    - Avoid crashing when Tor is built using headers that contain
+      CLOCK_MONOTONIC_COARSE, but then tries to run on an older kernel
+      without CLOCK_MONOTONIC_COARSE. Fixes bug 21035; bugfix
+      on 0.2.9.1-alpha.
+    - Fix Libevent detection on platforms without Libevent 1 headers
+      installed. Fixes bug 21051; bugfix on 0.2.9.1-alpha.
+
+
+Changes in version 0.2.8.12 - 2016-12-19
+  Tor 0.2.8.12 backports a fix for a medium-severity issue (bug 21018
+  below) where Tor clients could crash when attempting to visit a
+  hostile hidden service. Clients are recommended to upgrade as packages
+  become available for their systems.
+
+  It also includes an updated list of fallback directories, backported
+  from 0.2.9.
+
+  Now that the Tor 0.2.9 series is stable, only major bugfixes will be
+  backported to 0.2.8 in the future.
+
+  o Major bugfixes (parsing, security, backported from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Minor features (fallback directory list, backported from 0.2.9.8):
+    - Replace the 81 remaining fallbacks of the 100 originally
+      introduced in Tor 0.2.8.3-alpha in March 2016, with a list of 177
+      fallbacks (123 new, 54 existing, 27 removed) generated in December
+      2016. Resolves ticket 20170.
+
+  o Minor features (geoip, backported from 0.2.9.7-rc):
+    - Update geoip and geoip6 to the December 7 2016 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.9.8 - 2016-12-19
+  Tor 0.2.9.8 is the first stable release of the Tor 0.2.9 series.
+
+  The Tor 0.2.9 series makes mandatory a number of security features
+  that were formerly optional. It includes support for a new shared-
+  randomness protocol that will form the basis for next generation
+  hidden services, includes a single-hop hidden service mode for
+  optimizing .onion services that don't actually want to be hidden,
+  tries harder not to overload the directory authorities with excessive
+  downloads, and supports a better protocol versioning scheme for
+  improved compatibility with other implementations of the Tor protocol.
+
+  And of course, there are numerous other bugfixes and improvements.
+
+  This release also includes a fix for a medium-severity issue (bug
+  21018 below) where Tor clients could crash when attempting to visit a
+  hostile hidden service. Clients are recommended to upgrade as packages
+  become available for their systems.
+
+  Below are listed the changes since Tor 0.2.8.11.  For a list of
+  changes since 0.2.9.7-rc, see the ChangeLog file.
+
+  o New system requirements:
+    - When building with OpenSSL, Tor now requires version 1.0.1 or
+      later. OpenSSL 1.0.0 and earlier are no longer supported by the
+      OpenSSL team, and should not be used. Closes ticket 20303.
+    - Tor now requires Libevent version 2.0.10-stable or later. Older
+      versions of Libevent have less efficient backends for several
+      platforms, and lack the DNS code that we use for our server-side
+      DNS support. This implements ticket 19554.
+    - Tor now requires zlib version 1.2 or later, for security,
+      efficiency, and (eventually) gzip support. (Back when we started,
+      zlib 1.1 and zlib 1.0 were still found in the wild. 1.2 was
+      released in 2003. We recommend the latest version.)
+
+  o Deprecated features:
+    - A number of DNS-cache-related sub-options for client ports are now
+      deprecated for security reasons, and may be removed in a future
+      version of Tor. (We believe that client-side DNS caching is a bad
+      idea for anonymity, and you should not turn it on.) The options
+      are: CacheDNS, CacheIPv4DNS, CacheIPv6DNS, UseDNSCache,
+      UseIPv4Cache, and UseIPv6Cache.
+    - A number of options are deprecated for security reasons, and may
+      be removed in a future version of Tor. The options are:
+      AllowDotExit, AllowInvalidNodes, AllowSingleHopCircuits,
+      AllowSingleHopExits, ClientDNSRejectInternalAddresses,
+      CloseHSClientCircuitsImmediatelyOnTimeout,
+      CloseHSServiceRendCircuitsImmediatelyOnTimeout,
+      ExcludeSingleHopRelays, FastFirstHopPK, TLSECGroup,
+      UseNTorHandshake, and WarnUnsafeSocks.
+    - The *ListenAddress options are now deprecated as unnecessary: the
+      corresponding *Port options should be used instead. These options
+      may someday be removed. The affected options are:
+      ControlListenAddress, DNSListenAddress, DirListenAddress,
+      NATDListenAddress, ORListenAddress, SocksListenAddress,
+      and TransListenAddress.
+
+  o Major bugfixes (parsing, security, new since 0.2.9.7-rc):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major features (build, hardening):
+    - Tor now builds with -ftrapv by default on compilers that support
+      it. This option detects signed integer overflow (which C forbids),
+      and turns it into a hard-failure. We do not apply this option to
+      code that needs to run in constant time to avoid side-channels;
+      instead, we use -fwrapv in that code. Closes ticket 17983.
+    - When --enable-expensive-hardening is selected, stop applying the
+      clang/gcc sanitizers to code that needs to run in constant time.
+      Although we are aware of no introduced side-channels, we are not
+      able to prove that there are none. Related to ticket 17983.
+
+  o Major features (circuit building, security):
+    - Authorities, relays, and clients now require ntor keys in all
+      descriptors, for all hops (except for rare hidden service protocol
+      cases), for all circuits, and for all other roles. Part of
+      ticket 19163.
+    - Authorities, relays, and clients only use ntor, except for
+      rare cases in the hidden service protocol. Part of ticket 19163.
+
+  o Major features (compilation):
+    - Our big list of extra GCC warnings is now enabled by default when
+      building with GCC (or with anything like Clang that claims to be
+      GCC-compatible). To make all warnings into fatal compilation
+      errors, pass --enable-fatal-warnings to configure. Closes
+      ticket 19044.
+    - Use the Autoconf macro AC_USE_SYSTEM_EXTENSIONS to automatically
+      turn on C and POSIX extensions. (Previously, we attempted to do
+      this on an ad hoc basis.) Closes ticket 19139.
+
+  o Major features (directory authorities, hidden services):
+    - Directory authorities can now perform the shared randomness
+      protocol specified by proposal 250. Using this protocol, directory
+      authorities generate a global fresh random value every day. In the
+      future, this value will be used by hidden services to select
+      HSDirs. This release implements the directory authority feature;
+      the hidden service side will be implemented in the future as part
+      of proposal 224. Resolves ticket 16943; implements proposal 250.
+
+  o Major features (downloading, random exponential backoff):
+    - When we fail to download an object from a directory service, wait
+      for an (exponentially increasing) randomized amount of time before
+      retrying, rather than a fixed interval as we did before. This
+      prevents a group of Tor instances from becoming too synchronized,
+      or a single Tor instance from becoming too predictable, in its
+      download schedule. Closes ticket 15942.
+
+  o Major features (resource management):
+    - Tor can now notice it is about to run out of sockets, and
+      preemptively close connections of lower priority. (This feature is
+      off by default for now, since the current prioritizing method is
+      yet not mature enough. You can enable it by setting
+      "DisableOOSCheck 0", but watch out: it might close some sockets
+      you would rather have it keep.) Closes ticket 18640.
+
+  o Major features (single-hop "hidden" services):
+    - Add experimental HiddenServiceSingleHopMode and
+      HiddenServiceNonAnonymousMode options. When both are set to 1,
+      every hidden service on that Tor instance becomes a non-anonymous
+      Single Onion Service. Single Onions make one-hop (direct)
+      connections to their introduction and rendezvous points. One-hop
+      circuits make Single Onion servers easily locatable, but clients
+      remain location-anonymous. This is compatible with the existing
+      hidden service implementation, and works on the current Tor
+      network without any changes to older relays or clients. Implements
+      proposal 260, completes ticket 17178. Patch by teor and asn.
+
+  o Major features (subprotocol versions):
+    - Tor directory authorities now vote on a set of recommended
+      "subprotocol versions", and on a set of required subprotocol
+      versions. Clients and relays that lack support for a _required_
+      subprotocol version will not start; those that lack support for a
+      _recommended_ subprotocol version will warn the user to upgrade.
+      This change allows compatible implementations of the Tor protocol(s)
+      to exist without pretending to be 100% bug-compatible with
+      particular releases of Tor itself. Closes ticket 19958; implements
+      part of proposal 264.
+
+  o Major bugfixes (circuit building):
+    - Hidden service client-to-intro-point and service-to-rendezvous-
+      point circuits use the TAP key supplied by the protocol, to avoid
+      epistemic attacks. Fixes bug 19163; bugfix on 0.2.4.18-rc.
+
+  o Major bugfixes (download scheduling):
+    - Avoid resetting download status for consensuses hourly, since we
+      already have another, smarter retry mechanism. Fixes bug 8625;
+      bugfix on 0.2.0.9-alpha.
+    - If a consensus expires while we are waiting for certificates to
+      download, stop waiting for certificates.
+    - If we stop waiting for certificates less than a minute after we
+      started downloading them, do not consider the certificate download
+      failure a separate failure. Fixes bug 20533; bugfix
+      on 0.2.0.9-alpha.
+    - When using exponential backoff in test networks, use a lower
+      exponent, so the delays do not vary as much. This helps test
+      networks bootstrap consistently. Fixes bug 20597; bugfix on 20499.
+
+  o Major bugfixes (exit policies):
+    - Avoid disclosing exit outbound bind addresses, configured port
+      bind addresses, and local interface addresses in relay descriptors
+      by default under ExitPolicyRejectPrivate. Instead, only reject
+      these (otherwise unlisted) addresses if
+      ExitPolicyRejectLocalInterfaces is set. Fixes bug 18456; bugfix on
+      0.2.7.2-alpha. Patch by teor.
+
+  o Major bugfixes (hidden services):
+    - Allow Tor clients with appropriate controllers to work with
+      FetchHidServDescriptors set to 0. Previously, this option also
+      disabled descriptor cache lookup, thus breaking hidden services
+      entirely. Fixes bug 18704; bugfix on 0.2.0.20-rc. Patch by "twim".
+    - Clients now require hidden services to include the TAP keys for
+      their intro points in the hidden service descriptor. This prevents
+      an inadvertent upgrade to ntor, which a malicious hidden service
+      could use to distinguish clients by consensus version. Fixes bug
+      20012; bugfix on 0.2.4.8-alpha. Patch by teor.
+
+  o Major bugfixes (relay, resolver, logging):
+    - For relays that don't know their own address, avoid attempting a
+      local hostname resolve for each descriptor we download. This
+      will cut down on the number of "Success: chose address 'x.x.x.x'"
+      log lines, and also avoid confusing clock jumps if the resolver
+      is slow. Fixes bugs 20423 and 20610; bugfix on 0.2.8.1-alpha.
+
+  o Minor features (port flags):
+    - Add new flags to the *Port options to give finer control over which
+      requests are allowed. The flags are NoDNSRequest, NoOnionTraffic,
+      and the synthetic flag OnionTrafficOnly, which is equivalent to
+      NoDNSRequest, NoIPv4Traffic, and NoIPv6Traffic. Closes enhancement
+      18693; patch by "teor".
+
+  o Minor features (build, hardening):
+    - Detect and work around a libclang_rt problem that would prevent
+      clang from finding __mulodi4() on some 32-bit platforms, and thus
+      keep -ftrapv from linking on those systems. Closes ticket 19079.
+    - When building on a system without runtime support for the runtime
+      hardening options, try to log a useful warning at configuration
+      time, rather than an incomprehensible warning at link time. If
+      expensive hardening was requested, this warning becomes an error.
+      Closes ticket 18895.
+
+  o Minor features (client, directory):
+    - Since authorities now omit all routers that lack the Running and
+      Valid flags, we assume that any relay listed in the consensus must
+      have those flags. Closes ticket 20001; implements part of
+      proposal 272.
+
+  o Minor features (code safety):
+    - In our integer-parsing functions, ensure that the maximum value we
+      allow is no smaller than the minimum value. Closes ticket 19063;
+      patch from "U+039b".
+
+  o Minor features (compilation, portability):
+    - Compile correctly on MacOS 10.12 (aka "Sierra"). Closes
+      ticket 20241.
+
+  o Minor features (config):
+    - Warn users when descriptor and port addresses are inconsistent.
+      Mitigates bug 13953; patch by teor.
+
+  o Minor features (controller):
+    - Allow controllers to configure basic client authorization on
+      hidden services when they create them with the ADD_ONION controller
+      command. Implements ticket 15588. Patch by "special".
+    - Fire a STATUS_SERVER controller event whenever the hibernation
+      status changes between "awake"/"soft"/"hard". Closes ticket 18685.
+    - Implement new GETINFO queries for all downloads that use
+      download_status_t to schedule retries. This allows controllers to
+      examine the schedule for pending downloads. Closes ticket 19323.
+
+  o Minor features (development tools, etags):
+    - Teach the "make tags" Makefile target how to correctly find
+      "MOCK_IMPL" function definitions. Patch from nherring; closes
+      ticket 16869.
+
+  o Minor features (directory authority):
+    - After voting, if the authorities decide that a relay is not
+      "Valid", they no longer include it in the consensus at all. Closes
+      ticket 20002; implements part of proposal 272.
+    - Directory authorities now only give the Guard flag to a relay if
+      they are also giving it the Stable flag. This change allows us to
+      simplify path selection for clients. It should have minimal effect
+      in practice, since >99% of Guards already have the Stable flag.
+      Implements ticket 18624.
+    - Directory authorities now write their v3-status-votes file out to
+      disk earlier in the consensus process, so we have a record of the
+      votes even if we abort the consensus process. Resolves
+      ticket 19036.
+
+  o Minor features (fallback directory list, new since 0.2.9.7-rc):
+    - Replace the 81 remaining fallbacks of the 100 originally
+      introduced in Tor 0.2.8.3-alpha in March 2016, with a list of 177
+      fallbacks (123 new, 54 existing, 27 removed) generated in December
+      2016. Resolves ticket 20170.
+
+  o Minor features (hidden service):
+    - Stop being so strict about the payload length of "rendezvous1"
+      cells. We used to be locked in to the "TAP" handshake length, and
+      now we can handle better handshakes like "ntor". Resolves
+      ticket 18998.
+
+  o Minor features (infrastructure, time):
+    - Tor now includes an improved timer backend, so that we can
+      efficiently support tens or hundreds of thousands of concurrent
+      timers, as will be needed for some of our planned anti-traffic-
+      analysis work. This code is based on William Ahern's "timeout.c"
+      project, which implements a "tickless hierarchical timing wheel".
+      Closes ticket 18365.
+    - Tor now uses the operating system's monotonic timers (where
+      available) for internal fine-grained timing. Previously we would
+      look at the system clock, and then attempt to compensate for the
+      clock running backwards. Closes ticket 18908.
+
+  o Minor features (logging):
+    - Add a set of macros to check nonfatal assertions, for internal
+      use. Migrating more of our checks to these should help us avoid
+      needless crash bugs. Closes ticket 18613.
+    - Provide a more useful warning message when configured with an
+      invalid Nickname. Closes ticket 18300; patch from "icanhasaccount".
+    - When dumping unparseable router descriptors, optionally store them
+      in separate files, named by digest, up to a configurable size
+      limit. You can change the size limit by setting the
+      MaxUnparseableDescSizeToLog option, and disable this feature by
+      setting that option to 0. Closes ticket 18322.
+
+  o Minor features (performance):
+    - Change the "optimistic data" extension from "off by default" to
+      "on by default". The default was ordinarily overridden by a
+      consensus option, but when clients were bootstrapping for the
+      first time, they would not have a consensus to get the option
+      from. Changing this default saves a round-trip during startup.
+      Closes ticket 18815.
+
+  o Minor features (relay, usability):
+    - When the directory authorities refuse a bad relay's descriptor,
+      encourage the relay operator to contact us. Many relay operators
+      won't notice this line in their logs, but it's a win if even a few
+      learn why we don't like what their relay was doing. Resolves
+      ticket 18760.
+
+  o Minor features (security, TLS):
+    - Servers no longer support clients that lack AES ciphersuites.
+      (3DES is no longer considered an acceptable cipher.) We believe
+      that no such Tor clients currently exist, since Tor has required
+      OpenSSL 0.9.7 or later since 2009. Closes ticket 19998.
+
+  o Minor features (testing):
+    - Disable memory protections on OpenBSD when performing our unit
+      tests for memwipe(). The test deliberately invokes undefined
+      behavior, and the OpenBSD protections interfere with this. Patch
+      from "rubiate". Closes ticket 20066.
+    - Move the test-network.sh script to chutney, and modify tor's test-
+      network.sh to call the (newer) chutney version when available.
+      Resolves ticket 19116. Patch by teor.
+    - Use the lcov convention for marking lines as unreachable, so that
+      we don't count them when we're generating test coverage data.
+      Update our coverage tools to understand this convention. Closes
+      ticket 16792.
+    - Our link-handshake unit tests now check that when invalid
+      handshakes fail, they fail with the error messages we expected.
+    - Our unit testing code that captures log messages no longer
+      prevents them from being written out if the user asked for them
+      (by passing --debug or --info or --notice or --warn to the "test"
+      binary). This change prevents us from missing unexpected log
+      messages simply because we were looking for others. Related to
+      ticket 19999.
+    - The unit tests now log all warning messages with the "BUG" flag.
+      Previously, they only logged errors by default. This change will
+      help us make our testing code more correct, and make sure that we
+      only hit this code when we mean to. In the meantime, however,
+      there will be more warnings in the unit test logs than before.
+      This is preparatory work for ticket 19999.
+    - The unit tests now treat any failure of a "tor_assert_nonfatal()"
+      assertion as a test failure.
+    - We've done significant work to make the unit tests run faster.
+
+  o Minor features (testing, ipv6):
+    - Add the hs-ipv6 chutney target to make test-network-all's IPv6
+      tests. Remove bridges+hs, as it's somewhat redundant. This
+      requires a recent chutney version that supports IPv6 clients,
+      relays, and authorities. Closes ticket 20069; patch by teor.
+    - Add the single-onion and single-onion-ipv6 chutney targets to
+      "make test-network-all". This requires a recent chutney version
+      with the single onion network flavors (git c72a652 or later).
+      Closes ticket 20072; patch by teor.
+
+  o Minor features (Tor2web):
+    - Make Tor2web clients respect ReachableAddresses. This feature was
+      inadvertently enabled in 0.2.8.6, then removed by bugfix 19973 on
+      0.2.8.7. Implements feature 20034. Patch by teor.
+
+  o Minor features (unix domain sockets):
+    - When configuring a unix domain socket for a SocksPort,
+      ControlPort, or Hidden service, you can now wrap the address in
+      quotes, using C-style escapes inside the quotes. This allows unix
+      domain socket paths to contain spaces. Resolves ticket 18753.
+
+  o Minor features (user interface):
+    - Tor now supports the ability to declare options deprecated, so
+      that we can recommend that people stop using them. Previously, this
+      was done in an ad-hoc way. There is a new --list-deprecated-options
+      command-line option to list all of the deprecated options. Closes
+      ticket 19820.
+
+  o Minor features (virtual addresses):
+    - Increase the maximum number of bits for the IPv6 virtual network
+      prefix from 16 to 104. In this way, the condition for address
+      allocation is less restrictive. Closes ticket 20151; feature
+      on 0.2.4.7-alpha.
+
+  o Minor bug fixes (circuits):
+    - Use the CircuitBuildTimeout option whenever
+      LearnCircuitBuildTimeout is disabled. Previously, we would respect
+      the option when a user disabled it, but not when it was disabled
+      because some other option was set. Fixes bug 20073; bugfix on
+      0.2.4.12-alpha. Patch by teor.
+
+  o Minor bugfixes (build):
+    - The current Git revision when building from a local repository is
+      now detected correctly when using git worktrees. Fixes bug 20492;
+      bugfix on 0.2.3.9-alpha.
+
+  o Minor bugfixes (relay address discovery):
+    - Stop reordering IP addresses returned by the OS. This makes it
+      more likely that Tor will guess the same relay IP address every
+      time. Fixes issue 20163; bugfix on 0.2.7.1-alpha, ticket 17027.
+      Reported by René Mayrhofer, patch by "cypherpunks".
+
+  o Minor bugfixes (memory allocation):
+    - Change how we allocate memory for large chunks on buffers, to
+      avoid a (currently impossible) integer overflow, and to waste less
+      space when allocating unusually large chunks. Fixes bug 20081;
+      bugfix on 0.2.0.16-alpha. Issue identified by Guido Vranken.
+
+  o Minor bugfixes (bootstrap):
+    - Remember the directory server we fetched the consensus or previous
+      certificates from, and use it to fetch future authority
+      certificates. This change improves bootstrapping performance.
+      Fixes bug 18963; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (circuits):
+    - Make sure extend_info_from_router() is only called on servers.
+      Fixes bug 19639; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (client, fascistfirewall):
+    - Avoid spurious warnings when ReachableAddresses or FascistFirewall
+      is set. Fixes bug 20306; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (client, unix domain sockets):
+    - Disable IsolateClientAddr when using AF_UNIX backed SocksPorts as
+      the client address is meaningless. Fixes bug 20261; bugfix
+      on 0.2.6.3-alpha.
+
+  o Minor bugfixes (code style):
+    - Fix an integer signedness conversion issue in the case conversion
+      tables. Fixes bug 19168; bugfix on 0.2.1.11-alpha.
+
+  o Minor bugfixes (compilation):
+    - Build correctly on versions of libevent2 without support for
+      evutil_secure_rng_add_bytes(). Fixes bug 19904; bugfix
+      on 0.2.5.4-alpha.
+    - When building with Clang, use a full set of GCC warnings.
+      (Previously, we included only a subset, because of the way we
+      detected them.) Fixes bug 19216; bugfix on 0.2.0.1-alpha.
+    - Detect Libevent2 functions correctly on systems that provide
+      libevent2, but where libevent1 is linked with -levent. Fixes bug
+      19904; bugfix on 0.2.2.24-alpha. Patch from Rubiate.
+    - Run correctly when built on Windows build environments that
+      require _vcsprintf(). Fixes bug 20560; bugfix on 0.2.2.11-alpha.
+
+  o Minor bugfixes (configuration):
+    - When parsing quoted configuration values from the torrc file,
+      handle Windows line endings correctly. Fixes bug 19167; bugfix on
+      0.2.0.16-alpha. Patch from "Pingl".
+
+  o Minor bugfixes (directory authority):
+    - Authorities now sort the "package" lines in their votes, for ease
+      of debugging. (They are already sorted in consensus documents.)
+      Fixes bug 18840; bugfix on 0.2.6.3-alpha.
+    - Die with a more useful error when the operator forgets to place
+      the authority_signing_key file into the keys directory. This
+      avoids an uninformative assert & traceback about having an invalid
+      key. Fixes bug 20065; bugfix on 0.2.0.1-alpha.
+    - When allowing private addresses, mark Exits that only exit to
+      private locations as such. Fixes bug 20064; bugfix
+      on 0.2.2.9-alpha.
+    - When parsing a detached signature, make sure we use the length of
+      the digest algorithm instead of a hardcoded DIGEST256_LEN in
+      order to avoid comparing bytes out-of-bounds with a smaller digest
+      length such as SHA1. Fixes bug 19066; bugfix on 0.2.2.6-alpha.
+
+  o Minor bugfixes (getpass):
+    - Defensively fix a non-triggerable heap corruption at do_getpass()
+      to protect ourselves from mistakes in the future. Fixes bug
+      19223; bugfix on 0.2.7.3-rc. Bug found by Guido Vranken, patch
+      by nherring.
+
+  o Minor bugfixes (guard selection):
+    - Don't mark guards as unreachable if connection_connect() fails.
+      That function fails for local reasons, so it shouldn't reveal
+      anything about the status of the guard. Fixes bug 14334; bugfix
+      on 0.2.3.10-alpha.
+    - Use a single entry guard even if the NumEntryGuards consensus
+      parameter is not provided. Fixes bug 17688; bugfix
+      on 0.2.5.6-alpha.
+
+  o Minor bugfixes (hidden services):
+    - Increase the minimum number of internal circuits we preemptively
+      build from 2 to 3, so a circuit is available when a client
+      connects to another onion service. Fixes bug 13239; bugfix
+      on 0.1.0.1-rc.
+    - Allow hidden services to run on IPv6 addresses even when the
+      IPv6Exit option is not set. Fixes bug 18357; bugfix
+      on 0.2.4.7-alpha.
+    - Stop logging intro point details to the client log on certain
+      error conditions. Fixed as part of bug 20012; bugfix on
+      0.2.4.8-alpha. Patch by teor.
+    - When deleting an ephemeral hidden service, close its intro points
+      even if they are not completely open. Fixes bug 18604; bugfix
+      on 0.2.7.1-alpha.
+    - When configuring hidden services, check every hidden service
+      directory's permissions. Previously, we only checked the last
+      hidden service. Fixes bug 20529; bugfix on 0.2.6.2-alpha.
+
+  o Minor bugfixes (IPv6, testing):
+    - Check for IPv6 correctly on Linux when running test networks.
+      Fixes bug 19905; bugfix on 0.2.7.3-rc; patch by teor.
+
+  o Minor bugfixes (Linux seccomp2 sandbox):
+    - Add permission to run the sched_yield() and sigaltstack() system
+      calls, in order to support versions of Tor compiled with asan or
+      ubsan code that use these calls. Now "sandbox 1" and
+      "--enable-expensive-hardening" should be compatible on more
+      systems. Fixes bug 20063; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Downgrade a harmless log message about the
+      pending_entry_connections list from "warn" to "info". Mitigates
+      bug 19926.
+    - Log a more accurate message when we fail to dump a microdescriptor.
+      Fixes bug 17758; bugfix on 0.2.2.8-alpha. Patch from Daniel Pinto.
+    - When logging a directory ownership mismatch, log the owning
+      username correctly. Fixes bug 19578; bugfix on 0.2.2.29-beta.
+    - When we are unable to remove the bw_accounting file, do not warn
+      if the reason we couldn't remove it was that it didn't exist.
+      Fixes bug 19964; bugfix on 0.2.5.4-alpha. Patch from pastly.
+
+  o Minor bugfixes (memory leak):
+    - Fix a series of slow memory leaks related to parsing torrc files
+      and options. Fixes bug 19466; bugfix on 0.2.1.6-alpha.
+    - Avoid a small memory leak when informing worker threads about
+      rotated onion keys. Fixes bug 20401; bugfix on 0.2.6.3-alpha.
+    - Fix a small memory leak when receiving AF_UNIX connections on a
+      SocksPort. Fixes bug 20716; bugfix on 0.2.6.3-alpha.
+    - When moving a signed descriptor object from a source to an
+      existing destination, free the allocated memory inside that
+      destination object. Fixes bug 20715; bugfix on 0.2.8.3-alpha.
+    - Fix a memory leak and use-after-free error when removing entries
+      from the sandbox's getaddrinfo() cache. Fixes bug 20710; bugfix on
+      0.2.5.5-alpha. Patch from "cypherpunks".
+    - Fix a small, uncommon memory leak that could occur when reading a
+      truncated ed25519 key file. Fixes bug 18956; bugfix
+      on 0.2.6.1-alpha.
+
+  o Minor bugfixes (option parsing):
+    - Count unix sockets when counting client listeners (SOCKS, Trans,
+      NATD, and DNS). This has no user-visible behavior changes: these
+      options are set once, and never read. Required for correct
+      behavior in ticket 17178. Fixes bug 19677; bugfix on
+      0.2.6.3-alpha. Patch by teor.
+
+  o Minor bugfixes (options):
+    - Check the consistency of UseEntryGuards and EntryNodes more
+      reliably. Fixes bug 20074; bugfix on 0.2.4.12-alpha. Patch
+      by teor.
+    - Stop changing the configured value of UseEntryGuards on
+      authorities and Tor2web clients. Fixes bug 20074; bugfix on
+      commits 51fc6799 in 0.1.1.16-rc and acda1735 in 0.2.4.3-alpha.
+      Patch by teor.
+
+  o Minor bugfixes (relay):
+    - Ensure relays don't make multiple connections during bootstrap.
+      Fixes bug 20591; bugfix on 0.2.8.1-alpha.
+    - Do not try to parallelize workers more than 16x without the user
+      explicitly configuring us to do so, even if we do detect more than
+      16 CPU cores. Fixes bug 19968; bugfix on 0.2.3.1-alpha.
+
+  o Minor bugfixes (testing):
+    - The test-stem and test-network makefile targets now depend only on
+      the tor binary that they are testing. Previously, they depended on
+      "make all". Fixes bug 18240; bugfix on 0.2.8.2-alpha. Based on a
+      patch from "cypherpunks".
+    - Allow clients to retry HSDirs much faster in test networks. Fixes
+      bug 19702; bugfix on 0.2.7.1-alpha. Patch by teor.
+    - Avoid a unit test failure on systems with over 16 detectable CPU
+      cores. Fixes bug 19968; bugfix on 0.2.3.1-alpha.
+    - Let backtrace tests work correctly under AddressSanitizer:
+      disable ASAN's detection of segmentation faults while running
+      test_bt.sh, so that we can make sure that our own backtrace
+      generation code works. Fixes bug 18934; bugfix
+      on 0.2.5.2-alpha. Patch from "cypherpunks".
+    - Fix the test-network-all target on out-of-tree builds by using the
+      correct path to the test driver script. Fixes bug 19421; bugfix
+      on 0.2.7.3-rc.
+    - Stop spurious failures in the local interface address discovery
+      unit tests. Fixes bug 20634; bugfix on 0.2.8.1-alpha; patch by
+      Neel Chauhan.
+    - Use ECDHE ciphers instead of ECDH in tortls tests. LibreSSL has
+      removed the ECDH ciphers which caused the tests to fail on
+      platforms which use it. Fixes bug 20460; bugfix on 0.2.8.1-alpha.
+    - The tor_tls_server_info_callback unit test no longer crashes when
+      debug-level logging is turned on. Fixes bug 20041; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (time):
+    - Improve overflow checks in tv_udiff and tv_mdiff. Fixes bug 19483;
+      bugfix on all released tor versions.
+    - When computing the difference between two times in milliseconds,
+      we now round to the nearest millisecond correctly. Previously, we
+      could sometimes round in the wrong direction. Fixes bug 19428;
+      bugfix on 0.2.2.2-alpha.
+
+  o Minor bugfixes (Tor2web):
+    - Prevent Tor2web clients from running hidden services: these services
+      are not anonymous due to the one-hop client paths. Fixes bug
+      19678. Patch by teor.
+
+  o Minor bugfixes (user interface):
+    - Display a more accurate number of suppressed messages in the log
+      rate-limiter. Previously, there was a potential integer overflow
+      in the counter. Now, if the number of messages hits a maximum, the
+      rate-limiter doesn't count any further. Fixes bug 19435; bugfix
+      on 0.2.4.11-alpha.
+    - Fix a typo in the passphrase prompt for the ed25519 identity key.
+      Fixes bug 19503; bugfix on 0.2.7.2-alpha.
+
+  o Code simplification and refactoring:
+    - Remove redundant declarations of the MIN macro. Closes
+      ticket 18889.
+    - Rename tor_dup_addr() to tor_addr_to_str_dup() to avoid confusion.
+      Closes ticket 18462; patch from "icanhasaccount".
+    - Split the 600-line directory_handle_command_get function into
+      separate functions for different URL types. Closes ticket 16698.
+
+  o Documentation:
+    - Add module-level internal documentation for 36 C files that
+      previously didn't have a high-level overview. Closes ticket 20385.
+    - Correct the IPv6 syntax in our documentation for the
+      VirtualAddrNetworkIPv6 torrc option. Closes ticket 19743.
+    - Correct the minimum bandwidth value in torrc.sample, and queue a
+      corresponding change for torrc.minimal. Closes ticket 20085.
+    - Fix spelling of "--enable-tor2web-mode" in the manpage. Closes
+      ticket 19153. Patch from "U+039b".
+    - Module-level documentation for several more modules. Closes
+      tickets 19287 and 19290.
+    - Document the --passphrase-fd option in the tor manpage. Fixes bug
+      19504; bugfix on 0.2.7.3-rc.
+    - Document the default PathsNeededToBuildCircuits value that's used
+      by clients when the directory authorities don't set
+      min_paths_for_circs_pct. Fixes bug 20117; bugfix on 0.2.4.10-alpha.
+      Patch by teor, reported by Jesse V.
+    - Fix manual for the User option: it takes a username, not a UID.
+      Fixes bug 19122; bugfix on 0.0.2pre16 (the first version to have
+      a manpage!).
+    - Fix the description of the --passphrase-fd option in the
+      tor-gencert manpage. The option is used to pass the number of a
+      file descriptor to read the passphrase from, not to read the file
+      descriptor from. Fixes bug 19505; bugfix on 0.2.0.20-alpha.
+
+  o Removed code:
+    - We no longer include the (dead, deprecated) bufferevent code in
+      Tor. Closes ticket 19450. Based on a patch from "U+039b".
+
+  o Removed features:
+    - Remove support for "GET /tor/bytes.txt" DirPort request, and
+      "GETINFO dir-usage" controller request, which were only available
+      via a compile-time option in Tor anyway. Feature was added in
+      0.2.2.1-alpha. Resolves ticket 19035.
+    - There is no longer a compile-time option to disable support for
+      TransPort. (If you don't want TransPort, just don't use it.) Patch
+      from "U+039b". Closes ticket 19449.
+
+  o Testing:
+    - Run more workqueue tests as part of "make check". These had
+      previously been implemented, but you needed to know special
+      command-line options to enable them.
+    - We now have unit tests for our code to reject zlib "compression
+      bombs". (Fortunately, the code works fine.)
+
+
+Changes in version 0.2.8.11 - 2016-12-08
+  Tor 0.2.8.11 backports fixes for additional portability issues that
+  could prevent Tor from building correctly on OSX Sierra, or with
+  OpenSSL 1.1. Affected users should upgrade; others can safely stay
+  with 0.2.8.10.
+
+  o Minor bugfixes (portability):
+    - Avoid compilation errors when building on OSX Sierra. Sierra began
+      to support the getentropy() and clock_gettime() APIs, but created
+      a few problems in doing so. Tor 0.2.9 has a more thorough set of
+      workarounds; in 0.2.8, we are just using the /dev/urandom and mach
+      monotonic time interfaces. Fixes bug 20865. Bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (portability, backport from 0.2.9.5-alpha):
+    - Fix compilation with OpenSSL 1.1 and less commonly-used CPU
+      architectures. Closes ticket 20588.
+
+
+Changes in version 0.2.8.10 - 2016-12-02
+  Tor 0.2.8.10 backports a fix for a bug that would sometimes make clients
+  unusable after they left standby mode. It also backports fixes for
+  a few portability issues and a small but problematic memory leak.
+
+  o Major bugfixes (client reliability, backport from 0.2.9.5-alpha):
+    - When Tor leaves standby because of a new application request, open
+      circuits as needed to serve that request. Previously, we would
+      potentially wait a very long time. Fixes part of bug 19969; bugfix
+      on 0.2.8.1-alpha.
+
+  o Major bugfixes (client performance, backport from 0.2.9.5-alpha):
+    - Clients now respond to new application stream requests immediately
+      when they arrive, rather than waiting up to one second before
+      starting to handle them. Fixes part of bug 19969; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (portability, backport from 0.2.9.6-rc):
+    - Work around a bug in the OSX 10.12 SDK that would prevent us from
+      successfully targeting earlier versions of OSX. Resolves
+      ticket 20235.
+
+  o Minor bugfixes (portability, backport from 0.2.9.5-alpha):
+    - Fix implicit conversion warnings under OpenSSL 1.1. Fixes bug
+      20551; bugfix on 0.2.1.1-alpha.
+
+  o Minor bugfixes (relay, backport from 0.2.9.5-alpha):
+    - Work around a memory leak in OpenSSL 1.1 when encoding public
+      keys. Fixes bug 20553; bugfix on 0.0.2pre8.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 3 2016 Maxmind GeoLite2
+      Country database.
+
 
 Changes in version 0.2.8.9 - 2016-10-17
   Tor 0.2.8.9 backports a fix for a security hole in previous versions
@@ -449,7 +7737,7 @@ Changes in version 0.2.8.6 - 2016-08-02
 
   o Minor bugfixes (build):
     - Avoid spurious failures from configure files related to calling
-      exit(0) in TOR_SEARCH_LIBRARY. Fixes bug 18625; bugfix on
+      exit(0) in TOR_SEARCH_LIBRARY. Fixes bug 18626; bugfix on
       0.2.0.1-alpha. Patch from "cypherpunks".
     - Do not link the unit tests against both the testing and non-
       testing versions of the static libraries. Fixes bug 18490; bugfix
@@ -991,7 +8279,7 @@ Changes in version 0.2.7.5 - 2015-11-20
 
   o Major bugfixes (hidden services):
     - Revert commit that made directory authorities assign the HSDir
-      flag to relay without a DirPort; this was bad because such relays
+      flag to relays without a DirPort; this was bad because such relays
       can't handle BEGIN_DIR cells. Fixes bug 15850; bugfix
       on 0.2.6.3-alpha.
     - When cannibalizing a circuit for an introduction point, always
@@ -1606,7 +8894,7 @@ Changes in version 0.2.6.8 - 2015-05-21
 
   o Major bugfixes (hidden services, backport from 0.2.7.1-alpha):
     - Revert commit that made directory authorities assign the HSDir
-      flag to relay without a DirPort; this was bad because such relays
+      flag to relays without a DirPort; this was bad because such relays
       can't handle BEGIN_DIR cells. Fixes bug 15850; bugfix
       on 0.2.6.3-alpha.
 
@@ -1905,7 +9193,7 @@ Changes in version 0.2.6.6 - 2015-03-24
   o Major bugfixes (pluggable transports):
     - Initialize the extended OR Port authentication cookie before
       launching pluggable transports. This prevents a race condition
-      that occured when server-side pluggable transports would cache the
+      that occurred when server-side pluggable transports would cache the
       authentication cookie before it has been (re)generated. Fixes bug
       15240; bugfix on 0.2.5.1-alpha.
 
@@ -2378,7 +9666,7 @@ Changes in version 0.2.6.6 - 2015-03-24
       some bugs where we would look at (but fortunately, not reveal)
       uninitialized memory on the stack. Fixes bug 14013; bugfix on all
       versions of Tor.
-    - Clear all memory targetted by tor_addr_{to,from}_sockaddr(), not
+    - Clear all memory targeted by tor_addr_{to,from}_sockaddr(), not
       just the part that's used. This makes it harder for data leak bugs
       to occur in the event of other programming failures. Resolves
       ticket 14041.
@@ -3555,7 +10843,7 @@ Changes in version 0.2.5.10 - 2014-10-24
       from Arlo Breault.
     - Remove instances of strcpy() from the unit tests. They weren't
       hurting anything, since they were only in the unit tests, but it's
-      embarassing to have strcpy() in the code at all, and some analysis
+      embarrassing to have strcpy() in the code at all, and some analysis
       tools don't like it. Fixes bug 8790; bugfix on 0.2.3.6-alpha and
       0.2.3.8-alpha. Patch from Arlo Breault.
     - Remove is_internal_IP() function. Resolves ticket 4645.
@@ -3579,7 +10867,7 @@ Changes in version 0.2.5.10 - 2014-10-24
       directory authority options, remove the documentation for a
       V2-directory fetching option that no longer exists. Resolves
       ticket 11634.
-    - Correct the documenation so that it lists the correct directory
+    - Correct the documentation so that it lists the correct directory
       for the stats files. (They are in a subdirectory called "stats",
       not "status".)
     - In the manpage, move more authority-only options into the
@@ -5722,7 +13010,7 @@ Changes in version 0.2.3.25 - 2012-11-19
       bugfix on 0.2.0.3-alpha.
     - The "--quiet" and "--hush" options now apply not only to Tor's
       behavior before logs are configured, but also to Tor's behavior in
-      the absense of configured logs. Fixes bug 3550; bugfix on
+      the absence of configured logs. Fixes bug 3550; bugfix on
       0.2.0.10-alpha.
     - Change the AllowDotExit rules so they should actually work.
       We now enforce AllowDotExit only immediately after receiving an
@@ -6577,7 +13865,7 @@ Changes in version 0.2.1.31 - 2011-10-26
       circuit EXTEND request. Now relays can protect clients from the
       CVE-2011-2768 issue even if the clients haven't upgraded yet.
     - Bridges now refuse CREATE or CREATE_FAST cells on OR connections
-      that they initiated. Relays could distinguish incoming bridge 
+      that they initiated. Relays could distinguish incoming bridge
       connections from client connections, creating another avenue for
       enumerating bridges. Fixes CVE-2011-2769. Bugfix on 0.2.0.3-alpha.
       Found by "frosty_un".
@@ -6835,7 +14123,7 @@ Changes in version 0.2.2.32 - 2011-08-27
       algorithms for signatures and resource selection. Newer formats
       are signed with SHA256, with a possibility for moving to a better
       hash algorithm in the future.
-    - Directory authorities can now vote on arbitary integer values as
+    - Directory authorities can now vote on arbitrary integer values as
       part of the consensus process. This is designed to help set
       network-wide parameters. Implements proposal 167.
 
@@ -12024,7 +19312,7 @@ Changes in version 0.1.0.10 - 2005-06-14
     - Use correct errno on win32 if libevent fails.
     - Check and warn about known-bad/slow libevent versions.
     - Stop warning about sigpipes in the logs. We're going to
-      pretend that getting these occassionally is normal and fine.
+      pretend that getting these occasionally is normal and fine.
 
   o New contrib scripts:
     - New experimental script tor/contrib/exitlist: a simple python
@@ -12143,7 +19431,7 @@ Changes in version 0.0.9.7 - 2005-04-01
 
 Changes in version 0.0.9.6 - 2005-03-24
   o Bugfixes on 0.0.9.x (crashes and asserts):
-    - Add new end stream reasons to maintainance branch. Fix bug where
+    - Add new end stream reasons to maintenance branch. Fix bug where
       reason (8) could trigger an assert.  Prevent bug from recurring.
     - Apparently win32 stat wants paths to not end with a slash.
     - Fix assert triggers in assert_cpath_layer_ok(), where we were
@@ -13196,4 +20484,3 @@ Changes in version 0.0.2pre13 - 2003-10-19
     - If --DebugLogFile is specified, log to it at -l debug
     - If --LogFile is specified, use it instead of commandline
     - If --RunAsDaemon is set, tor forks and backgrounds on startup
-
diff --git a/acinclude.m4 b/acinclude.m4
index ab12317139..5ecdf1d5c2 100644
--- a/acinclude.m4
+++ b/acinclude.m4
@@ -2,7 +2,7 @@ dnl Helper macros for Tor configure.ac
 dnl Copyright (c) 2001-2004, Roger Dingledine
 dnl Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
 dnl Copyright (c) 2007-2008, Roger Dingledine, Nick Mathewson
-dnl Copyright (c) 2007-2015, The Tor Project, Inc.
+dnl Copyright (c) 2007-2019, The Tor Project, Inc.
 dnl See LICENSE for licensing information
 
 AC_DEFUN([TOR_EXTEND_CODEPATH],
@@ -51,12 +51,12 @@ AC_DEFUN([TOR_TRY_COMPILE_WITH_CFLAGS], [
   AC_CACHE_CHECK([whether the compiler accepts $1], VAR, [
     tor_saved_CFLAGS="$CFLAGS"
     CFLAGS="$CFLAGS -pedantic -Werror $1"
-    AC_TRY_COMPILE([], [return 0;],
+    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[]])],
                    [AS_VAR_SET(VAR,yes)],
                    [AS_VAR_SET(VAR,no)])
     if test x$2 != x; then
       AS_VAR_PUSHDEF([can_link],[tor_can_link_$1])
-      AC_TRY_LINK([], [return 0;],
+      AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])],
                   [AS_VAR_SET(can_link,yes)],
                   [AS_VAR_SET(can_link,no)])
       AS_VAR_POPDEF([can_link])
@@ -93,7 +93,7 @@ AC_DEFUN([TOR_CHECK_LDFLAGS], [
     AC_RUN_IFELSE([AC_LANG_PROGRAM([#include <stdio.h>], [fputs("", stdout)])],
                   [AS_VAR_SET(VAR,yes)],
                   [AS_VAR_SET(VAR,no)],
-	          [AC_TRY_LINK([], [return 0;],
+                  [AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])],
                                    [AS_VAR_SET(VAR,yes)],
                                    [AS_VAR_SET(VAR,no)])])
     CFLAGS="$tor_saved_CFLAGS"
@@ -113,21 +113,21 @@ if test x$2 = xdevpkg; then
   h=" headers for"
 fi
 if test -f /etc/debian_version && test x"$tor_$1_$2_debian" != x; then
-  AC_WARN([On Debian, you can install$h $1 using "apt-get install $tor_$1_$2_debian"])
+  AC_MSG_WARN([On Debian, you can install$h $1 using "apt-get install $tor_$1_$2_debian"])
   if test x"$tor_$1_$2_debian" != x"$tor_$1_devpkg_debian"; then 
-    AC_WARN([   You will probably need $tor_$1_devpkg_debian too.])
+    AC_MSG_WARN([   You will probably need $tor_$1_devpkg_debian too.])
   fi 
 fi
 if test -f /etc/fedora-release && test x"$tor_$1_$2_redhat" != x; then
-  AC_WARN([On Fedora, you can install$h $1 using "dnf install $tor_$1_$2_redhat"])
+  AC_MSG_WARN([On Fedora, you can install$h $1 using "dnf install $tor_$1_$2_redhat"])
   if test x"$tor_$1_$2_redhat" != x"$tor_$1_devpkg_redhat"; then 
-    AC_WARN([   You will probably need to install $tor_$1_devpkg_redhat too.])
+    AC_MSG_WARN([   You will probably need to install $tor_$1_devpkg_redhat too.])
   fi 
 else
   if test -f /etc/redhat-release && test x"$tor_$1_$2_redhat" != x; then
-    AC_WARN([On most Redhat-based systems, you can get$h $1 by installing the $tor_$1_$2_redhat RPM package])
+    AC_MSG_WARN([On most Redhat-based systems, you can get$h $1 by installing the $tor_$1_$2_redhat RPM package])
     if test x"$tor_$1_$2_redhat" != x"$tor_$1_devpkg_redhat"; then 
-      AC_WARN([   You will probably need to install $tor_$1_devpkg_redhat too.])
+      AC_MSG_WARN([   You will probably need to install $tor_$1_devpkg_redhat too.])
     fi 
   fi
 fi
@@ -245,7 +245,10 @@ if test "$cross_compiling" != yes; then
        LDFLAGS="$tor_tryextra $orig_LDFLAGS"
      fi
      AC_RUN_IFELSE([AC_LANG_PROGRAM([$5], [$6])],
-                   [runnable=yes], [runnable=no])
+                   [runnable=yes], [runnable=no],
+                   [AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])],
+                                   [runnable=yes],
+                                   [runnable=no])])
      if test "$runnable" = yes; then
         tor_cv_library_$1_linker_option=$tor_tryextra
         break
diff --git a/autogen.sh b/autogen.sh
index 8c43a9798a..276dd4047c 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -1,12 +1,12 @@
 #!/bin/sh
 
 if [ -x "`which autoreconf 2>/dev/null`" ] ; then
-  opt="-if"
+  opt="-i -f -W all,error"
 
   for i in $@; do
     case "$i" in
       -v)
-        opt=$opt"v"
+        opt="${opt} -v"
         ;;
     esac
   done
diff --git a/changes/29241_diagnostic b/changes/29241_diagnostic
new file mode 100644
index 0000000000..1e38654957
--- /dev/null
+++ b/changes/29241_diagnostic
@@ -0,0 +1,4 @@
+  o Minor features (NSS, diagnostic):
+    - Try to log an error from NSS (if there is any) and a more useful
+      description of our situation if we are using NSS and a call to
+      SSL_ExportKeyingMaterial() fails.  Diagnostic for ticket 29241.
diff --git a/changes/bug12399 b/changes/bug12399
new file mode 100644
index 0000000000..922c08c5e3
--- /dev/null
+++ b/changes/bug12399
@@ -0,0 +1,3 @@
+  o Minor bugfixes (logging):
+    - Change log level of message "Hash of session info was not as expected"
+      to LOG_PROTOCOL_WARN. Fixes bug 12399; bugfix on 0.1.1.10-alpha.
diff --git a/changes/bug13221 b/changes/bug13221
new file mode 100644
index 0000000000..13935a1921
--- /dev/null
+++ b/changes/bug13221
@@ -0,0 +1,5 @@
+  o Minor bugfixes (logging):
+    - Correct a misleading error message when IPv4Only or IPv6Only
+      is used but the resolved address can not be interpreted as an
+      address of the specified IP version.  Fixes bug 13221; bugfix
+      on 0.2.3.9-alpha.  Patch from Kris Katterjohn.
diff --git a/changes/bug21394.2 b/changes/bug21394.2
deleted file mode 100644
index b580d2a786..0000000000
--- a/changes/bug21394.2
+++ /dev/null
@@ -1,7 +0,0 @@
-  o Minor bugfix (Exit node DNS retries):
-    - Re-attempt timed-out DNS queries 3 times before failure, since our
-      timeout is 5 seconds for them, but clients wait 10-15. Also allow
-      slightly more timeouts per resolver before giving up on it in the
-      case where an exit has multiple resolvers configured. Fixes bug 21394;
-      bugfix on 0.3.1.9.
-
diff --git a/changes/bug22619 b/changes/bug22619
new file mode 100644
index 0000000000..9c71996f5b
--- /dev/null
+++ b/changes/bug22619
@@ -0,0 +1,3 @@
+  o Minor bugfixes (circuit isolation):
+    - Fix a logic error that prevented the SessionGroup sub-option from
+      being accepted. Fixes bug 22619; bugfix on 0.2.7.2-alpha.
diff --git a/changes/bug23507 b/changes/bug23507
new file mode 100644
index 0000000000..de18273fdb
--- /dev/null
+++ b/changes/bug23507
@@ -0,0 +1,5 @@
+  o Minor bugfixes (v3 single onion services):
+    - Make v3 single onion services fall back to a 3-hop intro, when there
+      all intro points are unreachable via a 1-hop path. Previously, v3
+      single onion services failed when all intro nodes were unreachable
+      via a 1-hop path. Fixes bug 23507; bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug23512 b/changes/bug23512
deleted file mode 100644
index 91b2786de4..0000000000
--- a/changes/bug23512
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Major bugfix (Relay bandwidth statistics):
-    - When we close relayed circuits, report the data in the circuit queues
-      as being written in our relay bandwidth stats. This mitigates guard
-      discovery and other attacks that close circuits for the explicit purpose
-      of noticing this discrepancy in statistics. Fixes bug 23512; bugfix
-      on 0.0.8pre3.
diff --git a/changes/bug23681 b/changes/bug23681
deleted file mode 100644
index e317f36d50..0000000000
--- a/changes/bug23681
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (hidden service client):
-    - The introduction circuit was being timed out too quickly while waiting
-      for the rendezvous circuit to complete. Keep the intro circuit around
-      longer instead of timing out and reopening new ones constantly. Fixes
-      bug 23681; bugfix on 0.2.4.8-alpha.
diff --git a/changes/bug23790 b/changes/bug23790
deleted file mode 100644
index 4aaf616e4d..0000000000
--- a/changes/bug23790
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (hidden service v2):
-    - When reloading tor (HUP) configured with hidden service(s), some
-      information weren't copy to the new service object. One problem with this
-      was that tor would wait at least the RendPostPeriod time before uploading
-      the descriptor if the reload happened before the descriptor needed to be
-      published. Fixes bug 23790; bugfix on 0.2.1.9-alpha.
diff --git a/changes/bug23818_v2 b/changes/bug23818_v2
new file mode 100644
index 0000000000..0219a20f49
--- /dev/null
+++ b/changes/bug23818_v2
@@ -0,0 +1,6 @@
+  o Minor bugfixes (v2 single onion services):
+    - Always retry v2 single onion service intro and rend circuits with a
+      3-hop path. Previously, v2 single onion services used a 3-hop path
+      when rend circuits were retried after a remote or delayed failure,
+      but a 1-hop path for immediate retries. Fixes bug 23818;
+      bugfix on 0.2.9.3-alpha.
diff --git a/changes/bug23818_v3 b/changes/bug23818_v3
new file mode 100644
index 0000000000..c430144d81
--- /dev/null
+++ b/changes/bug23818_v3
@@ -0,0 +1,6 @@
+  o Minor bugfixes (v3 single onion services):
+    - Always retry v3 single onion service intro and rend circuits with a
+      3-hop path. Previously, v3 single onion services used a 3-hop path
+      when rend circuits were retried after a remote or delayed failure,
+      but a 1-hop path for immediate retries. Fixes bug 23818;
+      bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug24104 b/changes/bug24104
deleted file mode 100644
index ca2a3537fa..0000000000
--- a/changes/bug24104
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfix (relay statistics):
-    - Update relay descriptor on bandwidth changes only when the uptime is
-      smaller than 24h in order to reduce the efficiency of guard discovery
-      attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha.
diff --git a/changes/bug24661 b/changes/bug24661
new file mode 100644
index 0000000000..a915a93e0e
--- /dev/null
+++ b/changes/bug24661
@@ -0,0 +1,3 @@
+  o Minor bugfixes (client, guard selection):
+    - When Tor's consensus has expired, but is still reasonably live, use it
+      to select guards. Fixes bug 24661; bugfix on 0.3.0.1-alpha.
diff --git a/changes/bug24903 b/changes/bug24903
deleted file mode 100644
index 01c9b53f23..0000000000
--- a/changes/bug24903
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (controller, reliability):
-    - Avoid a (nonfatal) assertion failure when extending a one-hop circuit
-      from the controller to become a multihop circuit. Fixes bug 24903;
-      bugfix on 0.2.5.2-alpha.
-
diff --git a/changes/bug25113 b/changes/bug25113
deleted file mode 100644
index 4a020b784d..0000000000
--- a/changes/bug25113
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (unit test, monotonic time):
-    - Bump a gap of 1msec to 10msec used in the monotonic time test that makes
-      sure the nsec/usec/msec time read are synchronized. This change was
-      needed to accommodate slow system like armel or when the clock_gettime()
-      is not a VDSO on the running kernel. Fixes bug 25113; bugfix on 0.2.9.1.
diff --git a/changes/bug25116 b/changes/bug25116
deleted file mode 100644
index b3e73feeaa..0000000000
--- a/changes/bug25116
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (hidden service, heartbeat):
-    - Don't log in the heartbeat any long term established one hop rendezvous
-      points if tor is a single onion service. Fixes bug 25116; bugfix on
-      0.2.9.6-rc;
diff --git a/changes/bug25733 b/changes/bug25733
deleted file mode 100644
index 775c1ae00e..0000000000
--- a/changes/bug25733
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (Assert crash):
-    - Avoid an assert in the circuit build timeout code if we fail to
-      allow any circuits to actually complete. Fixes bug 25733;
-      bugfix on 0.2.2.2-alpha.
diff --git a/changes/bug27073 b/changes/bug27073
deleted file mode 100644
index 851e7f0df6..0000000000
--- a/changes/bug27073
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (testing):
-    - Revise the "conditionvar_timeout" test so that it succeeds even
-      on heavily loaded systems where the test threads are not scheduled
-      within 200 msec. Fixes bug 27073; bugfix on 0.2.6.3-alpha.
diff --git a/changes/bug27197 b/changes/bug27197
new file mode 100644
index 0000000000..e389f85065
--- /dev/null
+++ b/changes/bug27197
@@ -0,0 +1,3 @@
+  o Minor bugfixes (protover, rust):
+    - Reject extra commas in version string. Fixes bug 27197; bugfix on
+      0.3.3.3-alpha.
diff --git a/changes/bug27199 b/changes/bug27199
new file mode 100644
index 0000000000..f9d2a422f9
--- /dev/null
+++ b/changes/bug27199
@@ -0,0 +1,3 @@
+  o Minor bugfixes (rust):
+    - Abort on panic in all build profiles, instead of potentially unwinding
+      into C code. Fixes bug 27199; bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug27316 b/changes/bug27316
deleted file mode 100644
index cec9348912..0000000000
--- a/changes/bug27316
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (protover):
-    - Reject protocol names containing bytes other than alphanumeric characters
-      and hyphens ([A-Za-z0-9-]). Fixes bug 27316; bugfix on 0.2.9.4-alpha.
diff --git a/changes/bug27658 b/changes/bug27658
deleted file mode 100644
index 8cc0aa4714..0000000000
--- a/changes/bug27658
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (testing):
-    - If a unit test running in a subprocess exits abnormally or with a
-      nonzero status code, treat the test as having failed, even if
-      the test reported success. Without this fix, memory leaks don't cause
-      cause the tests to fail, even with LeakSanitizer. Fixes bug 27658;
-      bugfix on 0.2.2.4-alpha.
diff --git a/changes/bug27709 b/changes/bug27709
deleted file mode 100644
index 49e87cbb0a..0000000000
--- a/changes/bug27709
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (code safety):
-    - Rewrite our assertion macros so that they no longer suppress
-      the compiler's -Wparentheses warnings on their inputs. Fixes bug 27709;
-      bugfix on 0.0.6.
diff --git a/changes/bug27740 b/changes/bug27740
new file mode 100644
index 0000000000..76a17b7dda
--- /dev/null
+++ b/changes/bug27740
@@ -0,0 +1,4 @@
+  o Minor bugfixes (rust):
+    - Return a string that can be safely freed by C code, not one created by
+      the rust allocator, in protover_all_supported(). Fixes bug 27740; bugfix
+      on 0.3.3.1-alpha.
diff --git a/changes/bug27741 b/changes/bug27741
new file mode 100644
index 0000000000..531e264b63
--- /dev/null
+++ b/changes/bug27741
@@ -0,0 +1,5 @@
+  o Minor bugfixes (rust, directory authority):
+    - Fix an API mismatch in the rust implementation of
+      protover_compute_vote(). This bug could have caused crashes on any
+      directory authorities running Tor with Rust (which we do not yet
+      recommend). Fixes bug 27741; bugfix on 0.3.3.6.
diff --git a/changes/bug27750 b/changes/bug27750
new file mode 100644
index 0000000000..c234788b1c
--- /dev/null
+++ b/changes/bug27750
@@ -0,0 +1,6 @@
+  o Minor bugfixes (connection, relay):
+    - Avoid a wrong BUG() stacktrace in case a closing connection is being held
+      open because the write side is rate limited but not the read side. Now,
+      the connection read side is simply shutdown instead of kept open until tor
+      is able to flush the connection and then fully close it. Fixes bug 27750;
+      bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug27800 b/changes/bug27800
new file mode 100644
index 0000000000..63d5dbc681
--- /dev/null
+++ b/changes/bug27800
@@ -0,0 +1,4 @@
+  o Minor bugfixes (directory authority):
+    - Log additional info when we get a relay that shares an ed25519
+      ID with a different relay, instead making a BUG() warning.
+      Fixes bug 27800; bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug27804 b/changes/bug27804
new file mode 100644
index 0000000000..fa7fec0bc5
--- /dev/null
+++ b/changes/bug27804
@@ -0,0 +1,3 @@
+  o Minor bugfixes (rust):
+    - Fix a potential null dereference in protover_all_supported().
+      Add a test for it. Fixes bug 27804; bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug27841 b/changes/bug27841
new file mode 100644
index 0000000000..9cd1da7275
--- /dev/null
+++ b/changes/bug27841
@@ -0,0 +1,7 @@
+  o Minor bugfixes (onion services):
+    - On an intro point for a version 3 onion service, we do not close
+      an introduction circuit on an NACK. This lets the client decide
+      whether to reuse the circuit or discard it. Previously, we closed
+      intro circuits on NACKs. Fixes bug 27841; bugfix on 0.3.2.1-alpha.
+      Patch by Neel Chaunan
+
diff --git a/changes/bug27963_timeradd b/changes/bug27963_timeradd
new file mode 100644
index 0000000000..34b361cf8d
--- /dev/null
+++ b/changes/bug27963_timeradd
@@ -0,0 +1,4 @@
+  o Minor bugfixes (compilation, opensolaris):
+    - Add a missing include to compat_pthreads.c, to fix compilation
+      on OpenSolaris and its descendants. Fixes bug 27963; bugfix
+      on 0.3.5.1-alpha.
diff --git a/changes/bug27968 b/changes/bug27968
new file mode 100644
index 0000000000..78c8eee33a
--- /dev/null
+++ b/changes/bug27968
@@ -0,0 +1,3 @@
+  o Minor bugfixes (testing):
+    - Avoid hangs and race conditions in test_rebind.py.
+      Fixes bug 27968; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug28115 b/changes/bug28115
new file mode 100644
index 0000000000..e3e29968eb
--- /dev/null
+++ b/changes/bug28115
@@ -0,0 +1,3 @@
+  o Minor bugfixes (portability):
+    - Make the OPE code (which is used for v3 onion services) run correctly
+      on big-endian platforms. Fixes bug 28115; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug28127 b/changes/bug28127
new file mode 100644
index 0000000000..541128c88e
--- /dev/null
+++ b/changes/bug28127
@@ -0,0 +1,7 @@
+  o Minor bugfixes (onion services):
+    - Unless we have explicitly set HiddenServiceVersion, detect the onion
+      service version and then look for invalid options. Previously, we
+      did the reverse, but that broke existing configs which were pointed
+      to a v2 hidden service and had options like HiddenServiceAuthorizeClient
+      set Fixes bug 28127; bugfix on 0.3.5.1-alpha. Patch by Neel Chauhan.
+
diff --git a/changes/bug28183 b/changes/bug28183
new file mode 100644
index 0000000000..8d35dcdc01
--- /dev/null
+++ b/changes/bug28183
@@ -0,0 +1,4 @@
+  o Minor bugfixes (Linux seccomp2 sandbox):
+    - Permit the "shutdown()" system call, which is apparently
+      used by OpenSSL under some circumstances. Fixes bug 28183;
+      bugfix on 0.2.5.1-alpha.
diff --git a/changes/bug28298 b/changes/bug28298
new file mode 100644
index 0000000000..8db340f3df
--- /dev/null
+++ b/changes/bug28298
@@ -0,0 +1,4 @@
+  o Minor bugfixes (configuration):
+    - Resume refusing to start with relative file paths and RunAsDaemon
+      set (regression from the fix for bug 22731). Fixes bug 28298;
+      bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug28303 b/changes/bug28303
new file mode 100644
index 0000000000..80f1302e5e
--- /dev/null
+++ b/changes/bug28303
@@ -0,0 +1,3 @@
+  o Minor bugfixes (compilation):
+    - Fix a pair of missing headers on OpenBSD. Fixes bug 28303;
+      bugfix on 0.3.5.1-alpha. Patch from  Kris Katterjohn.
diff --git a/changes/bug28348_034 b/changes/bug28348_034
new file mode 100644
index 0000000000..3913c03a4c
--- /dev/null
+++ b/changes/bug28348_034
@@ -0,0 +1,5 @@
+  o Major bugfixes (embedding, main loop):
+    - When DisableNetwork becomes set, actually disable periodic events that
+      are already enabled. (Previously, we would refrain from enabling new
+      ones, but we would leave the old ones turned on.)
+      Fixes bug 28348; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug28399 b/changes/bug28399
new file mode 100644
index 0000000000..9096db70b0
--- /dev/null
+++ b/changes/bug28399
@@ -0,0 +1,4 @@
+  o Minor bugfixes (continuous integration, Windows):
+    - Stop using an external OpenSSL install, and stop installing MSYS2
+      packages, when building using mingw on Appveyor Windows CI.
+      Fixes bug 28399; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug28419 b/changes/bug28419
new file mode 100644
index 0000000000..52ceb0a2a7
--- /dev/null
+++ b/changes/bug28419
@@ -0,0 +1,3 @@
+  o Minor bugfixes (memory leaks):
+    - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419;
+      bugfix on 0.3.3.1-alpha.  Patch from Martin Kepplinger.
\ No newline at end of file
diff --git a/changes/bug28435 b/changes/bug28435
new file mode 100644
index 0000000000..2a886cb8b7
--- /dev/null
+++ b/changes/bug28435
@@ -0,0 +1,3 @@
+  o Minor bugfixes (documentation):
+    - Make Doxygen work again after the 0.3.5 source tree moves.
+      Fixes bug 28435; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug28441 b/changes/bug28441
new file mode 100644
index 0000000000..d259b9f742
--- /dev/null
+++ b/changes/bug28441
@@ -0,0 +1,4 @@
+  o Minor bugfixes (logging):
+    - Stop talking about the Named flag in log messages. Clients have
+      ignored the Named flag since 0.3.2. Fixes bug 28441;
+      bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug28454 b/changes/bug28454
new file mode 100644
index 0000000000..ca46ae2777
--- /dev/null
+++ b/changes/bug28454
@@ -0,0 +1,4 @@
+  o Minor bugfixes (continuous integration, Windows):
+    - Manually configure the zstd compiler options, when building using
+      mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does not
+      come with a pkg-config file. Fixes bug 28454; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug28485 b/changes/bug28485
new file mode 100644
index 0000000000..a8309ae21f
--- /dev/null
+++ b/changes/bug28485
@@ -0,0 +1,3 @@
+  o Minor bugfixes (compilation):
+    - Add missing dependency on libgdi32.dll for tor-print-ed-signing-cert.exe
+      on Windows. Fixes bug 28485; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug28524 b/changes/bug28524
new file mode 100644
index 0000000000..1cad700422
--- /dev/null
+++ b/changes/bug28524
@@ -0,0 +1,4 @@
+  o Minor bugfixes (restart-in-process, boostrap):
+    - Add missing resets of bootstrap tracking state when shutting
+      down (regression caused by ticket 27169).  Fixes bug 28524;
+      bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug28554 b/changes/bug28554
new file mode 100644
index 0000000000..9a0b281406
--- /dev/null
+++ b/changes/bug28554
@@ -0,0 +1,3 @@
+  o Minor bugfixes (unit tests, guard selection):
+    - Stop leaking memory in an entry guard unit test. Fixes bug 28554;
+      bugfix on 0.3.0.1-alpha.
diff --git a/changes/bug28562 b/changes/bug28562
new file mode 100644
index 0000000000..e14362164d
--- /dev/null
+++ b/changes/bug28562
@@ -0,0 +1,5 @@
+  o Minor bugfixes (testing):
+    - Use a separate DataDirectory for the test_rebind script.
+      Previously, this script would run using the default DataDirectory,
+      and sometimes fail. Fixes bug 28562; bugfix on 0.3.5.1-alpha.
+      Patch from Taylor R Campbell.
diff --git a/changes/bug28568 b/changes/bug28568
new file mode 100644
index 0000000000..919ec08903
--- /dev/null
+++ b/changes/bug28568
@@ -0,0 +1,4 @@
+  o Minor bugfixes (testing):
+    - Stop running stem's unit tests as part of "make test-stem". But continue
+      to run stem's unit and online tests during "make test-stem-full".
+      Fixes bug 28568; bugfix on 0.2.6.3-alpha.
diff --git a/changes/bug28569 b/changes/bug28569
new file mode 100644
index 0000000000..45a57a80ae
--- /dev/null
+++ b/changes/bug28569
@@ -0,0 +1,3 @@
+  o Minor bugfixes (unit tests, directory clients):
+    - Mark outdated dirservers when Tor only has a reasonably live consensus.
+      Fixes bug 28569; bugfix on 0.3.2.5-alpha.
diff --git a/changes/bug28612 b/changes/bug28612
new file mode 100644
index 0000000000..559f254234
--- /dev/null
+++ b/changes/bug28612
@@ -0,0 +1,4 @@
+  o Minor bugfixes (windows services):
+    - Make Tor start correctly as an NT service again: previously it
+      was broken by refactoring.  Fixes bug 28612; bugfix on 0.3.5.3-alpha.
+      
diff --git a/changes/bug28619 b/changes/bug28619
new file mode 100644
index 0000000000..86be8cb2fb
--- /dev/null
+++ b/changes/bug28619
@@ -0,0 +1,6 @@
+  o Minor bugfixes (hidden service v3):
+    - When deleting an ephemeral onion service (DEL_ONION), do not close any
+      rendezvous circuits in order to let the existing client connections
+      finish by themselves or closed by the application. The HS v2 is doing
+      that already so now we have the same behavior for all versions. Fixes
+      bug 28619; bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug28656 b/changes/bug28656
new file mode 100644
index 0000000000..d3a13d196c
--- /dev/null
+++ b/changes/bug28656
@@ -0,0 +1,3 @@
+  o Minor bugfixes (logging):
+    - Stop logging a BUG() warning when tor is waiting for exit descriptors.
+      Fixes bug 28656; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug28698 b/changes/bug28698
new file mode 100644
index 0000000000..716aa0c552
--- /dev/null
+++ b/changes/bug28698
@@ -0,0 +1,3 @@
+  o Minor bugfix (logging):
+    - Avoid logging about relaxing circuits when their time is fixed.
+      Fixes bug 28698; bugfix on 0.2.4.7-alpha
diff --git a/changes/bug28895 b/changes/bug28895
new file mode 100644
index 0000000000..25fb167b2e
--- /dev/null
+++ b/changes/bug28895
@@ -0,0 +1,5 @@
+  o Minor bugfixes (usability):
+    - Stop saying "Your Guard ..." in pathbias_measure_{use,close}_rate()
+      as that confusingly suggests that mentioned guard node is under control
+      and responsibility of end user, which it is not. Fixes bug 28895;
+      bugfix on Tor 0.3.0.1-alpha.
diff --git a/changes/bug28920 b/changes/bug28920
new file mode 100644
index 0000000000..e698686a6d
--- /dev/null
+++ b/changes/bug28920
@@ -0,0 +1,6 @@
+  o Minor bugfixes (logging):
+    - Rework rep_hist_log_link_protocol_counts() to iterate through all link
+      protocol versions when logging incoming/outgoing connection counts. Tor
+      no longer skips version 5 and we don't have to remember to update this
+      function when new link protocol version is developed. Fixes bug 28920;
+      bugfix on 0.2.6.10.
diff --git a/changes/bug28938 b/changes/bug28938
new file mode 100644
index 0000000000..de6c5f7b79
--- /dev/null
+++ b/changes/bug28938
@@ -0,0 +1,4 @@
+  o Minor bugfixes (compilation):
+    - Fix missing headers required for proper detection of
+      OpenBSD.  Fixes bug 28938; bugfix on 0.3.5.1-alpha.
+      Patch from Kris Katterjohn.
diff --git a/changes/bug28974 b/changes/bug28974
new file mode 100644
index 0000000000..2d74f5674f
--- /dev/null
+++ b/changes/bug28974
@@ -0,0 +1,3 @@
+  o Minor bugfixes (compilation):
+    - Fix compilation for Android by adding a missing header to
+      freespace.c.  Fixes bug 28974; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug28979 b/changes/bug28979
new file mode 100644
index 0000000000..0625fd5d25
--- /dev/null
+++ b/changes/bug28979
@@ -0,0 +1,4 @@
+  o Minor bugfixes (documentation):
+    - Describe the contents of the v3 onion service client authorization
+      files correctly: They hold public keys, not private keys. Fixes bug
+      28979; bugfix on 0.3.5.1-alpha. Spotted by "Felixix".
diff --git a/changes/bug28981 b/changes/bug28981
new file mode 100644
index 0000000000..c0ea92ab35
--- /dev/null
+++ b/changes/bug28981
@@ -0,0 +1,5 @@
+  o Minor bugfixes (misc):
+    - The amount of total available physical memory is now determined
+      using the sysctl identifier HW_PHYSMEM (rather than HW_USERMEM)
+      when it is defined and a 64-bit variant is not available.  Fixes
+      bug 28981; bugfix on 0.2.5.4-alpha.  Patch from Kris Katterjohn.
diff --git a/changes/bug28995 b/changes/bug28995
new file mode 100644
index 0000000000..f76b6a085a
--- /dev/null
+++ b/changes/bug28995
@@ -0,0 +1,5 @@
+  o Minor bugfix (IPv6):
+    Fix tor_ersatz_socketpair on IPv6-only systems.  Previously,
+    the IPv6 socket was bound using an address family of AF_INET
+    instead of AF_INET6.  Fixes bug 28995; bugfix on 0.3.5.1-alpha.
+    Patch from Kris Katterjohn.
diff --git a/changes/bug29017 b/changes/bug29017
new file mode 100644
index 0000000000..5c4a53c43f
--- /dev/null
+++ b/changes/bug29017
@@ -0,0 +1,4 @@
+  o Minor bugfixes (stats):
+    - When ExtraInfoStatistics is 0, stop including PaddingStatistics in
+      relay and bridge extra-info documents. Fixes bug 29017;
+      bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug29034 b/changes/bug29034
new file mode 100644
index 0000000000..e7aa9af00b
--- /dev/null
+++ b/changes/bug29034
@@ -0,0 +1,5 @@
+  o Major bugfixes (Onion service reachability):
+    - Properly clean up the introduction point map when circuits change purpose
+      from onion service circuits to pathbias, measurement, or other circuit types.
+      This should fix some service-side instances of introduction point failure.
+      Fixes bug 29034; bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug29040 b/changes/bug29040
new file mode 100644
index 0000000000..0662aaa8a5
--- /dev/null
+++ b/changes/bug29040
@@ -0,0 +1,4 @@
+  o Minor bugfixes (onion services):
+    - Avoid crashing if ClientOnionAuthDir (incorrectly) contains
+      more than one private key for a hidden service. Fixes bug 29040;
+      bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug29042 b/changes/bug29042
new file mode 100644
index 0000000000..8d76939cea
--- /dev/null
+++ b/changes/bug29042
@@ -0,0 +1,5 @@
+  o Minor bugfixes (logging):
+    - Log more information at "warning" level when unable to read a private
+      key; log more information ad "info" level when unable to read a public
+      key. We had warnings here before, but they were lost during our
+      NSS work. Fixes bug 29042; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug29135 b/changes/bug29135
new file mode 100644
index 0000000000..fd7b1ae80e
--- /dev/null
+++ b/changes/bug29135
@@ -0,0 +1,5 @@
+  o Minor bugfixes (onion services, logging):
+    - In hs_cache_store_as_client() log an HSDesc we failed to parse at Debug
+      loglevel. Tor used to log it at Warning loglevel, which caused
+      very long log lines to appear for some users. Fixes bug 29135; bugfix on
+      0.3.2.1-alpha.
diff --git a/changes/bug29144 b/changes/bug29144
new file mode 100644
index 0000000000..5801224f14
--- /dev/null
+++ b/changes/bug29144
@@ -0,0 +1,5 @@
+  o Minor bugfixes (logging):
+    - Log the correct port number for listening sockets when "auto" is
+      used to let Tor pick the port number.  Previously, port 0 was
+      logged instead of the actual port number.  Fixes bug 29144;
+      bugfix on 0.3.5.1-alpha.  Patch from Kris Katterjohn.
diff --git a/changes/bug29161 b/changes/bug29161
new file mode 100644
index 0000000000..39a638acf6
--- /dev/null
+++ b/changes/bug29161
@@ -0,0 +1,3 @@
+  o Minor bugfixes (tests):
+    - Detect and suppress "bug" warnings from the util/time test on Windows.
+      Fixes bug 29161; bugfix on 0.2.9.3-alpha.
diff --git a/changes/bug29175_035 b/changes/bug29175_035
new file mode 100644
index 0000000000..134c1d9529
--- /dev/null
+++ b/changes/bug29175_035
@@ -0,0 +1,4 @@
+  o Major bugfixes (networking):
+    - Gracefully handle empty username/password fields in SOCKS5
+      username/password auth messsage and allow SOCKS5 handshake to
+      continue. Fixes bug 29175; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug29241 b/changes/bug29241
new file mode 100644
index 0000000000..7f25e154d1
--- /dev/null
+++ b/changes/bug29241
@@ -0,0 +1,6 @@
+  o Major bugfixes (NSS, relay):
+    - When running with NSS, disable TLS 1.2 ciphersuites that use SHA384
+      for their PRF. Due to an NSS bug, the TLS key exporters for these
+      ciphersuites don't work -- which caused relays to fail to handshake
+      with one another when these ciphersuites were enabled.
+      Fixes bug 29241; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug29244 b/changes/bug29244
new file mode 100644
index 0000000000..6206a95463
--- /dev/null
+++ b/changes/bug29244
@@ -0,0 +1,4 @@
+  o Minor bugfixes (build, compatibility):
+    - Update Cargo.lock file to match the version made by the latest
+      version of Rust, so that "make distcheck" will pass again.
+      Fixes bug 29244; bugfix on 0.3.3.4-alpha.
diff --git a/changes/bug29530_035 b/changes/bug29530_035
new file mode 100644
index 0000000000..6dfcd51e7b
--- /dev/null
+++ b/changes/bug29530_035
@@ -0,0 +1,5 @@
+  o Minor bugfixes (testing):
+    - Downgrade some LOG_ERR messages in the address/* tests to warnings.
+      The LOG_ERR messages were occurring when we had no configured network.
+      We were failing the unit tests, because we backported 28668 to 0.3.5.8,
+      but did not backport 29530. Fixes bug 29530; bugfix on 0.3.5.8.
diff --git a/changes/bug29601 b/changes/bug29601
new file mode 100644
index 0000000000..c4ba5fbc8b
--- /dev/null
+++ b/changes/bug29601
@@ -0,0 +1,6 @@
+  o Minor bugfixes (Windows, CI):
+    - Skip the Appveyor 32-bit Windows Server 2016 job, and 64-bit Windows
+      Server 2012 R2 job. The remaining 2 jobs still provide coverage of
+      64/32-bit, and Windows Server 2016/2012 R2. Also set fast_finish, so
+      failed jobs terminate the build immediately.
+      Fixes bug 29601; bugfix on 0.3.5.4-alpha.
diff --git a/changes/bug29670 b/changes/bug29670
new file mode 100644
index 0000000000..00b0c33327
--- /dev/null
+++ b/changes/bug29670
@@ -0,0 +1,4 @@
+  o Minor bugfixes (configuration, proxies):
+    - Fix a bug that prevented us from supporting SOCKS5 proxies that want
+      authentication along with configued (but unused!)
+      ClientTransportPlugins. Fixes bug 29670; bugfix on 0.2.6.1-alpha.
diff --git a/changes/bug29875 b/changes/bug29875
new file mode 100644
index 0000000000..58a1c871cd
--- /dev/null
+++ b/changes/bug29875
@@ -0,0 +1,11 @@
+  o Major bugfixes (bridges):
+    - Do not count previously configured working bridges towards our total of
+      working bridges. Previously, when Tor's list of bridges changed, it
+      would think that the old bridges were still usable, and delay fetching
+      router descriptors for the new ones.  Fixes part of bug 29875; bugfix
+      on 0.3.0.1-alpha.
+    - Consider our directory information to have changed when our list of
+      bridges changes. Previously, Tor would not re-compute the status of its
+      directory information when bridges changed, and therefore would not
+      realize that it was no longer able to build circuits. Fixes part of bug
+      29875.
diff --git a/changes/bug29922 b/changes/bug29922
new file mode 100644
index 0000000000..dacb951097
--- /dev/null
+++ b/changes/bug29922
@@ -0,0 +1,4 @@
+  o Minor bugfixes (testing, windows):
+    - Fix a test failure caused by an unexpected bug warning in
+      our test for tor_gmtime_r(-1). Fixes bug 29922;
+      bugfix on 0.2.9.3-alpha.
diff --git a/changes/bug30011 b/changes/bug30011
new file mode 100644
index 0000000000..4c9069e291
--- /dev/null
+++ b/changes/bug30011
@@ -0,0 +1,4 @@
+  o Minor bugfixes (CI):
+    - Terminate test-stem if it takes more than 9.5 minutes to run.
+      (Travis terminates the job after 10 minutes of no output.)
+      Diagnostic for 29437. Fixes bug 30011; bugfix on 0.3.5.4-alpha.
diff --git a/changes/bug30040 b/changes/bug30040
new file mode 100644
index 0000000000..7d80528a10
--- /dev/null
+++ b/changes/bug30040
@@ -0,0 +1,9 @@
+  o Minor bugfixes (security):
+    - Fix a potential double free bug when reading huge bandwidth files. The
+      issue is not exploitable in the current Tor network because the
+      vulnerable code is only reached when directory authorities read bandwidth
+      files, but bandwidth files come from a trusted source (usually the
+      authorities themselves). Furthermore, the issue is only exploitable in
+      rare (non-POSIX) 32-bit architectures which are not used by any of the
+      current authorities. Fixes bug 30040; bugfix on 0.3.5.1-alpha. Bug found
+      and fixed by Tobias Stoeckmann.
diff --git a/changes/bug30148 b/changes/bug30148
new file mode 100644
index 0000000000..7d0257e3fe
--- /dev/null
+++ b/changes/bug30148
@@ -0,0 +1,4 @@
+  o Minor bugfixes (memory leak):
+    - Avoid a minor memory leak that could occur on relays when
+      creating a keys directory failed. Fixes bug 30148; bugfix on
+      0.3.3.1-alpha.
diff --git a/changes/bug30189 b/changes/bug30189
new file mode 100644
index 0000000000..f8c932a5f9
--- /dev/null
+++ b/changes/bug30189
@@ -0,0 +1,4 @@
+  o Minor bugfixes (compilation, unusual configuration):
+    - Avoid failures when building with ALL_BUGS_ARE_FAILED due to
+      missing declarations of abort(), and prevent other such failures
+      in the future. Fixes bug 30189; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug30190 b/changes/bug30190
new file mode 100644
index 0000000000..e2352c3b9c
--- /dev/null
+++ b/changes/bug30190
@@ -0,0 +1,3 @@
+  o Minor bugfixes (lib):
+    do not log a warning for OpenSSL versions that should be compatible
+    Fixes bug 30190; bugfix on 0.2.4.2-alpha
diff --git a/changes/bug30316 b/changes/bug30316
new file mode 100644
index 0000000000..3e396318ad
--- /dev/null
+++ b/changes/bug30316
@@ -0,0 +1,4 @@
+  o Minor bugfixes (directory authority):
+    - Move the "bandwidth-file-headers" line in directory authority votes
+      so that it conforms to dir-spec.txt. Fixes bug 30316; bugfix on
+      0.3.5.1-alpha.
diff --git a/changes/bug30344 b/changes/bug30344
new file mode 100644
index 0000000000..37561bf944
--- /dev/null
+++ b/changes/bug30344
@@ -0,0 +1,4 @@
+  o Minor bugfixes (connection):
+    - Avoid reading data from closed connections, which can cause needless
+      loops in libevent and infinite loops in Shadow. Fixes bug 30344; bugfix
+      on 0.1.1.1-alpha.
diff --git a/changes/bug30452 b/changes/bug30452
new file mode 100644
index 0000000000..2bb401d87d
--- /dev/null
+++ b/changes/bug30452
@@ -0,0 +1,3 @@
+  o Minor features (compile-time modules):
+    - Add a --list-modules command to print a list of which compile-time
+      modules are enabled. Closes ticket 30452.
diff --git a/changes/bug30475 b/changes/bug30475
new file mode 100644
index 0000000000..839597b885
--- /dev/null
+++ b/changes/bug30475
@@ -0,0 +1,4 @@
+  o Minor bugfixes ():
+    - Avoid a GCC 9.1.1 warning (and possible crash depending on libc
+      implemenation) when failing to load a hidden service client authorization
+      file.  Fixes bug 30475; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug30713 b/changes/bug30713
new file mode 100644
index 0000000000..e00b98da65
--- /dev/null
+++ b/changes/bug30713
@@ -0,0 +1,5 @@
+  o Minor bugfixes (testing):
+    - Skip test_rebind when the TOR_SKIP_TEST_REBIND environmental variable is
+      set. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
+    - Skip test_rebind on macOS in Travis, because it is unreliable on
+      macOS on Travis. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug30744 b/changes/bug30744
new file mode 100644
index 0000000000..9f07d4855f
--- /dev/null
+++ b/changes/bug30744
@@ -0,0 +1,3 @@
+  o Minor bugfixes (continuous integration):
+    - Allow the test-stem job to fail in Travis, because it sometimes hangs.
+      Fixes bug 30744; bugfix on 0.3.5.4-alpha.
diff --git a/changes/bug30894 b/changes/bug30894
new file mode 100644
index 0000000000..64c14c4e6d
--- /dev/null
+++ b/changes/bug30894
@@ -0,0 +1,4 @@
+  o Minor bugfixes (memory leaks):
+    - Fix a trivial memory leak when parsing an invalid value
+      from a download schedule in the configuration. Fixes bug
+      30894; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug30916 b/changes/bug30916
new file mode 100644
index 0000000000..b006bfc75d
--- /dev/null
+++ b/changes/bug30916
@@ -0,0 +1,4 @@
+  o Minor bugfixes (relay):
+    - Avoid crashing when starting with a corrupt keys directory where
+      the old ntor key and the new ntor key are identical. Fixes bug 30916;
+      bugfix on 0.2.4.8-alpha.
diff --git a/changes/bug31003 b/changes/bug31003
new file mode 100644
index 0000000000..6c75163380
--- /dev/null
+++ b/changes/bug31003
@@ -0,0 +1,4 @@
+  o Minor bugfixes (crash on exit):
+    - Avoid a set of possible code paths that could use try to use freed memory
+      in routerlist_free() while Tor was exiting.  Fixes bug 31003; bugfix on
+      0.1.2.2-alpha.
diff --git a/changes/bug31408 b/changes/bug31408
new file mode 100644
index 0000000000..3e4ffa927d
--- /dev/null
+++ b/changes/bug31408
@@ -0,0 +1,5 @@
+  o Major bugfixes (torrc):
+    - Stop ignoring torrc options after an %include directive, when the
+      included directory ends with a file that does not contain any config
+      options. (But does contain comments or whitespace.)
+      Fixes bug 31408; bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug31463 b/changes/bug31463
new file mode 100644
index 0000000000..d85c0887c3
--- /dev/null
+++ b/changes/bug31463
@@ -0,0 +1,3 @@
+  o Minor bugfixes (rust):
+    - Correctly exclude a redundant rust build job in Travis. Fixes bug 31463;
+      bugfix on 0.3.5.4-alpha.
diff --git a/changes/bug31571 b/changes/bug31571
new file mode 100644
index 0000000000..86de3537ba
--- /dev/null
+++ b/changes/bug31571
@@ -0,0 +1,7 @@
+  o Minor bugfixes (error handling):
+    - Report the tor version whenever an assertion fails. Previously, we only
+      reported the Tor version on some crashes, and some non-fatal assertions.
+      Fixes bug 31571; bugfix on 0.3.5.1-alpha.
+    - On abort, try harder to flush the output buffers of log messages. On
+      some platforms (macOS), log messages can be discarded when the process
+      terminates. Fixes bug 31571; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug31657 b/changes/bug31657
new file mode 100644
index 0000000000..08e9d95fdf
--- /dev/null
+++ b/changes/bug31657
@@ -0,0 +1,5 @@
+  o Minor bugfixes (guards):
+    - When tor is missing descriptors for some primary entry guards, make the
+      log message less alarming. It's normal for descriptors to expire, as long
+      as tor fetches new ones soon after. Fixes bug 31657;
+      bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug31837 b/changes/bug31837
new file mode 100644
index 0000000000..0f976edfe0
--- /dev/null
+++ b/changes/bug31837
@@ -0,0 +1,5 @@
+  o Minor bugfixes (testing):
+    - When testing port rebinding, don't busy-wait for tor to log. Instead,
+      actually sleep for a short time before polling again. Also improve the
+      formatting of control commands and log messages.
+      Fixes bug 31837; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug31884 b/changes/bug31884
new file mode 100644
index 0000000000..ddb6c50d74
--- /dev/null
+++ b/changes/bug31884
@@ -0,0 +1,3 @@
+  o Minor bugfixes (Appveyor CI):
+    - Avoid spurious errors when Appveyor CI fails before the install step.
+      Fixes bug 31884; bugfix on 0.3.4.2-alpha.
diff --git a/changes/bug31939 b/changes/bug31939
new file mode 100644
index 0000000000..a36ea495d6
--- /dev/null
+++ b/changes/bug31939
@@ -0,0 +1,3 @@
+  o Minor bugfixes (tls, logging):
+    - Log TLS read buffer length bugs once, rather than filling the logs
+      with similar warnings. Fixes bug 31939; bugfix on 0.3.0.4-rc.
diff --git a/changes/bug32449 b/changes/bug32449
new file mode 100644
index 0000000000..213d8a1014
--- /dev/null
+++ b/changes/bug32449
@@ -0,0 +1,5 @@
+  o Minor bugfixes (CI, appveyor):
+    - Install the mingw OpenSSL package in Appveyor. This makes sure that
+      the OpenSSL headers and libraries match in Tor's Appveyor builds.
+      (This bug was triggered by an Appveyor image update.)
+      Fixes bug 32449; bugfix on 0.3.5.6-rc.
diff --git a/changes/cid1444119 b/changes/cid1444119
new file mode 100644
index 0000000000..bb6854e66f
--- /dev/null
+++ b/changes/cid1444119
@@ -0,0 +1,3 @@
+  o Minor bugfixes (C correctness):
+    - Fix an unlikely memory leak in consensus_diff_apply(). Fixes bug 29824;
+      bugfix on 0.3.1.1-alpha. This is Coverity warning CID 1444119.
diff --git a/changes/geoip-2018-09-06 b/changes/geoip-2018-09-06
deleted file mode 100644
index 851ec46e25..0000000000
--- a/changes/geoip-2018-09-06
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the September 6 2018 Maxmind GeoLite2
-      Country database. Closes ticket 27631.
-
diff --git a/changes/geoip-2018-10-09 b/changes/geoip-2018-10-09
deleted file mode 100644
index 9b8e621852..0000000000
--- a/changes/geoip-2018-10-09
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the October 9 2018 Maxmind GeoLite2
-      Country database. Closes ticket 27991.
-
diff --git a/changes/rust_asan b/changes/rust_asan
new file mode 100644
index 0000000000..1ca7ae6888
--- /dev/null
+++ b/changes/rust_asan
@@ -0,0 +1,8 @@
+  o Major bugfixes (compilation, rust):
+    - Rust tests can now build and run successfully with the
+      --enable-fragile-hardening option enabled.
+      Doing this currently requires the rust beta channel; it will
+      be possible with stable rust as of rust version 1.31 is out.
+      Patch from Alex Crichton.
+      Fixes bugs 27272, 27273, and 27274.
+      Bugfix on 0.3.1.1-alpha.
diff --git a/changes/ticket19566 b/changes/ticket19566
new file mode 100644
index 0000000000..bf7071e660
--- /dev/null
+++ b/changes/ticket19566
@@ -0,0 +1,6 @@
+  o Code simplification and refactoring (shared random, dirauth):
+    - Change many tor_assert() to use BUG() instead. The idea is to not crash
+      a dirauth but rather scream loudly with a stacktrace and let it continue
+      run. The shared random subsystem is very resilient and if anything wrong
+      happens with it, at worst a non coherent value will be put in the vote
+      and discarded by the other authorities. Closes ticket 19566.
diff --git a/changes/ticket27252 b/changes/ticket27252
deleted file mode 100644
index 410ddef8c0..0000000000
--- a/changes/ticket27252
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor features (continuous integration):
-    - Skip gcc on OSX in Travis CI, it's rarely used.
-      Skip a duplicate hardening-off build in Travis on Tor 0.2.9.
-      Skip gcc on Linux with default settings, because all the non-default
-      builds use gcc on Linux.
-      Implements ticket 27252.
diff --git a/changes/ticket27471 b/changes/ticket27471
new file mode 100644
index 0000000000..ffe77d268e
--- /dev/null
+++ b/changes/ticket27471
@@ -0,0 +1,5 @@
+  o Minor bugfixes (hidden service v3, client):
+    - When replacing a descriptor in the client cache with a newer descriptor,
+      make sure to close all client introduction circuits of the old
+      descriptor so we don't end up with unusable leftover circuits. Fixes bug
+      27471; bugfix on 0.3.2.1-alpha.
diff --git a/changes/ticket27738 b/changes/ticket27738
deleted file mode 100644
index f23bfb019e..0000000000
--- a/changes/ticket27738
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (continuous integration):
-    - Use the Travis Homebrew addon to install packages on macOS. The package
-      list is the same, but the Homebrew addon does not do a `brew update` by
-      default. Implements ticket 27738.
diff --git a/changes/ticket27751 b/changes/ticket27751
new file mode 100644
index 0000000000..593c473b61
--- /dev/null
+++ b/changes/ticket27751
@@ -0,0 +1,2 @@
+  o Minor features (continuous integration):
+    - Add a Travis CI build for --enable-nss on Linux gcc. Closes ticket 27751.
diff --git a/changes/ticket27838 b/changes/ticket27838
new file mode 100644
index 0000000000..1699730d7a
--- /dev/null
+++ b/changes/ticket27838
@@ -0,0 +1,4 @@
+  o Minor bugfixes (hidden service v3):
+    - Build the service descriptor signing key certificate before uploading so
+      we always have a fresh one leaving no chances for it to expire service
+      side. Fixes bug 27838; bugfix on 0.3.2.1-alpha.
diff --git a/changes/ticket27913 b/changes/ticket27913
new file mode 100644
index 0000000000..81ce725932
--- /dev/null
+++ b/changes/ticket27913
@@ -0,0 +1,3 @@
+  o Testing:
+    - Add new CI job to Travis configuration that runs stem-based
+      integration tests. Closes ticket 27913.
diff --git a/changes/ticket27995 b/changes/ticket27995
new file mode 100644
index 0000000000..8c75425749
--- /dev/null
+++ b/changes/ticket27995
@@ -0,0 +1,4 @@
+  o Minor bugfixes (hidden service v3, client authorization):
+    - Fix an assert() when adding a client authorization for the first time
+      and then sending a HUP signal to the service. Before that, tor would
+      stop abruptly. Fixes bug 27995; bugfix on 0.3.5.1-alpha.
diff --git a/changes/ticket28026 b/changes/ticket28026
new file mode 100644
index 0000000000..a6911c2cab
--- /dev/null
+++ b/changes/ticket28026
@@ -0,0 +1,3 @@
+  o Documentation (hidden service manpage):
+    - Improve HSv3 client authorization by making some options more explicit
+      and detailed. Closes ticket 28026. Patch by "mtigas".
diff --git a/changes/ticket28113 b/changes/ticket28113
new file mode 100644
index 0000000000..30dd825a9b
--- /dev/null
+++ b/changes/ticket28113
@@ -0,0 +1,5 @@
+  o Minor bugfixes (relay shutdown, systemd):
+    - Notify systemd of ShutdownWaitLength so it can be set to longer than
+      systemd's TimeoutStopSec. In tor's systemd service file, set
+      TimeoutSec to 60 seconds, to allow tor some time to shut down.
+      Fixes bug 28113; bugfix on 0.2.6.2-alpha.
diff --git a/changes/ticket28128 b/changes/ticket28128
new file mode 100644
index 0000000000..6d08c74242
--- /dev/null
+++ b/changes/ticket28128
@@ -0,0 +1,4 @@
+  o Documentation (hidden service manpage, sandbox):
+    - Document in the man page that changing ClientOnionAuthDir value or
+      adding a new file in the directory will not work at runtime upon sending
+      a HUP if Sandbox 1. Closes ticket 28128.
diff --git a/changes/ticket28229_diag b/changes/ticket28229_diag
new file mode 100644
index 0000000000..cd02b81faa
--- /dev/null
+++ b/changes/ticket28229_diag
@@ -0,0 +1,3 @@
+  o Testing:
+    - Increase logging and tag all log entries with timestamps
+      in test_rebind.py. Provides diagnostics for issue 28229.
diff --git a/changes/ticket28275 b/changes/ticket28275
new file mode 100644
index 0000000000..eadca86b7b
--- /dev/null
+++ b/changes/ticket28275
@@ -0,0 +1,4 @@
+  o Documentation (hidden service v3, man page):
+    - Note in the man page that the only real way to fully revoke an onion
+      service v3 client authorization is by restarting the tor process. Closes
+      ticket 28275.
diff --git a/changes/ticket28318 b/changes/ticket28318
new file mode 100644
index 0000000000..24dc1e9580
--- /dev/null
+++ b/changes/ticket28318
@@ -0,0 +1,3 @@
+  o Minor features (Windows, continuous integration):
+    - Build tor on Windows Server 2012 R2 and Windows Server 2016 using
+      Appveyor's CI. Closes ticket 28318.
diff --git a/changes/ticket28459 b/changes/ticket28459
new file mode 100644
index 0000000000..6b5839b52b
--- /dev/null
+++ b/changes/ticket28459
@@ -0,0 +1,4 @@
+  o Minor features (continuous integration, Windows):
+    - Always show the configure and test logs, and upload them as build
+      artifacts, when building for Windows using Appveyor CI.
+      Implements 28459.
diff --git a/changes/ticket28574 b/changes/ticket28574
new file mode 100644
index 0000000000..562810f511
--- /dev/null
+++ b/changes/ticket28574
@@ -0,0 +1,4 @@
+  o Minor bugfixes (continuous integration, Windows):
+    - Explicitly specify the path to the OpenSSL library and do not download
+      OpenSSL from Pacman, but instead use the library that is already provided
+      by AppVeyor. Fixes bug 28574; bugfix on master.
diff --git a/changes/ticket28668 b/changes/ticket28668
new file mode 100644
index 0000000000..6386e0051f
--- /dev/null
+++ b/changes/ticket28668
@@ -0,0 +1,3 @@
+  o Minor features (testing):
+    - Treat all unexpected ERR and BUG messages as test failures.
+      Closes ticket 28668.
diff --git a/changes/ticket28669 b/changes/ticket28669
new file mode 100644
index 0000000000..32c6114ffc
--- /dev/null
+++ b/changes/ticket28669
@@ -0,0 +1,6 @@
+  o Minor bugfix (hidden service v3, client):
+    - Avoid a BUG() stacktrace in case a SOCKS connection is found waiting for
+      the descriptor while we do have it in the cache. There is a rare case
+      when this can happen. Now, tor will recover and retry the descriptor.
+      Fixes bug 28669; bugfix on 0.3.2.4-alpha.
+    
diff --git a/changes/ticket28838 b/changes/ticket28838
new file mode 100644
index 0000000000..6c290bf82b
--- /dev/null
+++ b/changes/ticket28838
@@ -0,0 +1,8 @@
+  o Minor features (performance):
+    - Remove about 96% of the work from the function that we run at
+      startup to test our curve25519_basepoint implementation.  Since
+      this function has yet to find an actual failure, we'll only
+      run it for 8 iterations instead of 200. Based on our profile
+      information, this change should save around 8% of our startup
+      time on typical desktops, and may have a similar effect on
+      other platforms.  Closes ticket 28838.
diff --git a/changes/ticket28851 b/changes/ticket28851
new file mode 100644
index 0000000000..bab0318662
--- /dev/null
+++ b/changes/ticket28851
@@ -0,0 +1,4 @@
+  o Minor features (performance):
+    - Stop re-validating our hardcoded Diffie-Hellman parameters on every
+      startup. Doing this wasted time and cycles, especially on low-powered
+      devices. Closes ticket 28851.
diff --git a/changes/ticket28879 b/changes/ticket28879
new file mode 100644
index 0000000000..126420f6ca
--- /dev/null
+++ b/changes/ticket28879
@@ -0,0 +1,5 @@
+  o Minor bugfixes (correctness):
+    - Fix an unreached code-path where we checked the value of "hostname"
+      inside send_resolved_hostnam_cell(). Previously, we used it before
+      checking it; now we check it first. Fixes bug 28879; bugfix on
+      0.1.2.7-alpha.
diff --git a/changes/ticket28881 b/changes/ticket28881
new file mode 100644
index 0000000000..1b015a6c37
--- /dev/null
+++ b/changes/ticket28881
@@ -0,0 +1,4 @@
+  o Code simplification and refactoring:
+    -  When parsing a port configuration, make it more
+       obvious to static analyzer tools that we will always initialize the
+       address. Closes ticket 28881.
diff --git a/changes/ticket28883 b/changes/ticket28883
new file mode 100644
index 0000000000..1d8b6cb416
--- /dev/null
+++ b/changes/ticket28883
@@ -0,0 +1,4 @@
+  o Minor bugfixes (testing):
+    - Make sure that test_rebind.py actually obeys its timeout, even
+      when it receives a large number of log messages. Fixes bug 28883;
+      bugfix on 0.3.5.4-alpha.
diff --git a/changes/ticket28912 b/changes/ticket28912
new file mode 100644
index 0000000000..4119b778bc
--- /dev/null
+++ b/changes/ticket28912
@@ -0,0 +1,6 @@
+  o Major bugfixes (relay, directory):
+    - A connection serving directory information wouldn't get reactivated after
+      the first chunk of data was sent (usually 32KB). Tor now always activate
+      the main loop event that goes through these connections as long as at
+      least one connection is still active. Fixes bug 28912; bugfix on
+      0.3.4.1-alpha. Patch by "cypherpunks3".
diff --git a/changes/ticket28924 b/changes/ticket28924
new file mode 100644
index 0000000000..055a6cf285
--- /dev/null
+++ b/changes/ticket28924
@@ -0,0 +1,4 @@
+  o Minor features (compilation):
+    - When possible, place our warning flags in a separate file, to avoid
+      flooding verbose build logs to an unacceptable amount. Closes ticket
+      28924.
diff --git a/changes/ticket28970 b/changes/ticket28970
new file mode 100644
index 0000000000..138c575fcc
--- /dev/null
+++ b/changes/ticket28970
@@ -0,0 +1,6 @@
+  o Minor bugfixes (clietn, hidden service v3):
+    - Fix a BUG() assertion that occurs within a very small race window between
+      a client intro circuit opens and its descriptor that gets cleaned up from
+      the cache. The circuit is now closed which will trigger a re-fetch of the
+      descriptor and continue the HS connection. Fixes bug 28970; bugfix on
+      0.3.2.1-alpha.
diff --git a/changes/ticket28973 b/changes/ticket28973
new file mode 100644
index 0000000000..b1d208ee51
--- /dev/null
+++ b/changes/ticket28973
@@ -0,0 +1,6 @@
+  o Minor features (OpenSSL bug workaround):
+    - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3
+      key export function from handling long labels.  When this bug
+      is detected, Tor will disable TLS 1.3.  We recommend upgrading to
+      a version of OpenSSL without this bug when it becomes available.
+      Closes ticket 28973.
diff --git a/changes/ticket29026 b/changes/ticket29026
new file mode 100644
index 0000000000..1db873dfcf
--- /dev/null
+++ b/changes/ticket29026
@@ -0,0 +1,4 @@
+  o Minor features (compilation):
+    - Compile correctly when OpenSSL is built with engine support
+      disabled, or with deprecated APIs disabled. Closes ticket
+      29026. Patches from "Mangix".
diff --git a/changes/ticket29160 b/changes/ticket29160
new file mode 100644
index 0000000000..8e11183064
--- /dev/null
+++ b/changes/ticket29160
@@ -0,0 +1,4 @@
+  o Minor bugfixes (tests):
+    - Do not log an error-level message if we fail to find an IPv6
+      network interface from the unit tests. Fixes bug 29160; bugfix on
+      0.2.7.3-rc.
diff --git a/changes/ticket29168 b/changes/ticket29168
new file mode 100644
index 0000000000..65c5232f65
--- /dev/null
+++ b/changes/ticket29168
@@ -0,0 +1,5 @@
+  o Major bugfixes (cell scheduler, KIST):
+    - Make KIST to always take into account the outbuf length when computing
+      what we can actually put in the outbuf. This could lead to the outbuf
+      being filled up and thus a possible memory DoS vector. TROVE-2019-001.
+      Fixes bug 29168; bugfix on 0.3.2.1-alpha.
diff --git a/changes/ticket29435 b/changes/ticket29435
new file mode 100644
index 0000000000..d48ae98e4b
--- /dev/null
+++ b/changes/ticket29435
@@ -0,0 +1,3 @@
+  o Minor bugfixes (testing):
+    - Fix our gcov wrapper script to look for object files at the
+      correct locations. Fixes bug 29435; bugfix on 0.3.5.1-alpha.
diff --git a/changes/ticket29617 b/changes/ticket29617
new file mode 100644
index 0000000000..4d50ea9627
--- /dev/null
+++ b/changes/ticket29617
@@ -0,0 +1,4 @@
+  o Minor bugfixes (out-of-memory handler):
+    - When purging the DNS cache because of an out-of-memory condition,
+      try purging just the older entries at first.  Previously, we would
+      purge the whole thing. Fixes bug 29617; bugfix on 0.3.5.1-alpha.
diff --git a/changes/ticket29702 b/changes/ticket29702
new file mode 100644
index 0000000000..e1cc1f867b
--- /dev/null
+++ b/changes/ticket29702
@@ -0,0 +1,4 @@
+  o Testing:
+    - Specify torrc paths (with empty files) when launching tor in
+      integration tests; refrain from reading user and system torrcs.
+      Resolves issue 29702.
diff --git a/changes/ticket29806 b/changes/ticket29806
new file mode 100644
index 0000000000..6afefd4c04
--- /dev/null
+++ b/changes/ticket29806
@@ -0,0 +1,7 @@
+  o Minor features (bandwidth authority):
+    - Make bandwidth authorities to ignore relays that are reported in the
+      bandwidth file with the key-value "vote=0".
+      This change allows to report the relays that were not measured due
+      some failure and diagnose the reasons without the bandwidth being included in the
+      bandwidth authorities vote.
+      Closes ticket 29806.
diff --git a/changes/ticket29962 b/changes/ticket29962
new file mode 100644
index 0000000000..e36cc0cf9a
--- /dev/null
+++ b/changes/ticket29962
@@ -0,0 +1,3 @@
+  o Minor features (continuous integration):
+    - On Travis Rust builds, cleanup Rust registry and refrain from caching
+      target/ directory to speed up builds. Resolves issue 29962.
diff --git a/changes/ticket30117 b/changes/ticket30117
new file mode 100644
index 0000000000..5b6e6dabf7
--- /dev/null
+++ b/changes/ticket30117
@@ -0,0 +1,4 @@
+  o Testing (continuous integration):
+    - In Travis, tell timelimit to use stem's backtrace signals. And launch
+      python directly from timelimit, so python receives the signals from
+      timelimit, rather than make. Closes ticket 30117.
diff --git a/changes/ticket30234 b/changes/ticket30234
new file mode 100644
index 0000000000..5a0076bad2
--- /dev/null
+++ b/changes/ticket30234
@@ -0,0 +1,2 @@
+  o Testing (continuous integration):
+    - In Travis, show stem's tor log after failure. Closes ticket 30234.
diff --git a/changes/ticket30454 b/changes/ticket30454
new file mode 100644
index 0000000000..77c45d0feb
--- /dev/null
+++ b/changes/ticket30454
@@ -0,0 +1,10 @@
+  o Major bugfixes (hidden service v3):
+    - An intro point could try to send an INTRODUCE_ACK with a status code
+      that it wasn't able to encode leading to a hard assert() of the relay.
+      Fortunately, that specific code path can not be reached thus this issue
+      can't be triggered. We've consolidated the ABI values into trunnel now.
+      Fixes bug 30454; bugfix on 0.3.0.1-alpha.
+    - HSv3 client will now be able to properly handle unknown status code from
+      a INTRODUCE_ACK cell (nack) even if they do not know it. The NACK
+      behavior will stay the same. This will allow us to extend status code if
+      we want in the future without breaking the normal client behavior.
diff --git a/changes/ticket30591 b/changes/ticket30591
new file mode 100644
index 0000000000..f97c024009
--- /dev/null
+++ b/changes/ticket30591
@@ -0,0 +1,3 @@
+  o Testing (continuous integration):
+    - In Travis, make stem log a controller trace to the console. And tail
+      stem's tor log after failure. Closes ticket 30591.
diff --git a/changes/ticket30694 b/changes/ticket30694
new file mode 100644
index 0000000000..70dbf6481a
--- /dev/null
+++ b/changes/ticket30694
@@ -0,0 +1,3 @@
+  o Testing (continuous integration):
+    - In Travis, only run the stem tests that use a tor binary.
+      Closes ticket 30694.
diff --git a/changes/ticket30871 b/changes/ticket30871
new file mode 100644
index 0000000000..81c076bb02
--- /dev/null
+++ b/changes/ticket30871
@@ -0,0 +1,6 @@
+  o Major bugfixes (circuit build, guard):
+    - When considering upgrading circuits from "waiting for guard" to "open",
+      always ignore the ones that are mark for close. Else, we can end up in
+      the situation where a subsystem is notified of that circuit opening but
+      still marked for close leading to undesirable behavior. Fixes bug 30871;
+      bugfix on 0.3.0.1-alpha.
diff --git a/changes/ticket31001 b/changes/ticket31001
deleted file mode 100644
index 2ce1cbdf34..0000000000
--- a/changes/ticket31001
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (compatibility, standards compliance):
-    - Fix a bug that would invoke undefined behavior on certain operating
-      systems when trying to asprintf() a string exactly INT_MAX bytes
-      long. We don't believe this is exploitable, but it's better
-      to fix it anyway. Fixes bug 31001; bugfix on 0.2.2.11-alpha.
-      Found and fixed by Tobias Stoeckmann.
diff --git a/changes/ticket31372_appveyor b/changes/ticket31372_appveyor
new file mode 100644
index 0000000000..e7bb03182e
--- /dev/null
+++ b/changes/ticket31372_appveyor
@@ -0,0 +1,4 @@
+  o Minor features (continuous integration):
+    - When building on Appveyor, pass the "-k" flag to make, so that
+      we are informed of all compilation failures, not just the first
+      one or two. Closes part of ticket 31372.
diff --git a/changes/ticket31548 b/changes/ticket31548
new file mode 100644
index 0000000000..fef0b5d01f
--- /dev/null
+++ b/changes/ticket31548
@@ -0,0 +1,7 @@
+  o Major bugfixes (hidden service v3):
+    - Make onion service always use the exact amount of configured intro points
+      (or less due to node exlusion). Before, a service could sometimes pick
+      more intro points than configured with the
+      HiddenServiceNumIntroductionPoints option. Fixes bug 31548; bugfix on
+      0.3.2.1-alpha.
+
diff --git a/changes/ticket31554 b/changes/ticket31554
new file mode 100644
index 0000000000..73f4159ff3
--- /dev/null
+++ b/changes/ticket31554
@@ -0,0 +1,4 @@
+  o Minor features (stem tests):
+    - Change "make test-stem" so it only runs the stem tests that use tor.
+      This change makes test-stem faster and more reliable.
+      Closes ticket 31554.
diff --git a/changes/ticket31687_1 b/changes/ticket31687_1
new file mode 100644
index 0000000000..2f4d440974
--- /dev/null
+++ b/changes/ticket31687_1
@@ -0,0 +1,4 @@
+  o Minor bugfixes (compilation):
+    - Suppress spurious float-conversion warnings from GCC when calling
+      floating-point classifier functions on FreeBSD.  Fixes part of bug
+      31687; bugfix on 0.3.1.5-alpha.
diff --git a/changes/ticket31687_2 b/changes/ticket31687_2
new file mode 100644
index 0000000000..eadc698275
--- /dev/null
+++ b/changes/ticket31687_2
@@ -0,0 +1,5 @@
+  o Minor bugfixes (FreeBSD, PF-based proxy, IPv6):
+    - When extracting an IPv6 address from a PF-based proxy, verify
+      that we are actually configured to receive an IPv6 address,
+      and log an internal error if not. Fixes part of bug 31687;
+      bugfix on 0.2.3.4-alpha.
diff --git a/changes/ticket32058 b/changes/ticket32058
new file mode 100644
index 0000000000..b40bcda416
--- /dev/null
+++ b/changes/ticket32058
@@ -0,0 +1,5 @@
+  o Minor bugfixes (mainloop, periodic events):
+    - Periodic events enabled flag was not unset properly when shutting down tor
+      cleanly. This had the side effect to not re-enable periodic events when
+      tor_api.h is used to relaunch tor after a shutdown. Fixes bug 32058;
+      bugfix on 0.3.3.1-alpha.
diff --git a/changes/ticket32086 b/changes/ticket32086
new file mode 100644
index 0000000000..b9312c2bea
--- /dev/null
+++ b/changes/ticket32086
@@ -0,0 +1,3 @@
+  o Testing:
+    - Use Windows Server 2019 instead of Windows Server 2016 in our
+      Appveyor builds. Closes ticket 32086.
diff --git a/config.rust.in b/config.rust.in
new file mode 100644
index 0000000000..11b671b980
--- /dev/null
+++ b/config.rust.in
@@ -0,0 +1,24 @@
+# Used by our cargo build.rs script to get variables from autoconf.
+#
+# The "configure" script will generate "config.rust" from "config.rust.in",
+# and then build.rs will read "config.rust".
+
+BUILDDIR=@BUILDDIR@
+TOR_LDFLAGS_zlib=@TOR_LDFLAGS_zlib@
+TOR_LDFLAGS_nss=@TOR_LDFLAGS_nss@
+TOR_LDFLAGS_openssl=@TOR_LDFLAGS_openssl@
+TOR_LDFLAGS_libevent=@TOR_LDFLAGS_libevent@
+TOR_ZLIB_LIBS=@TOR_ZLIB_LIBS@
+TOR_LIB_MATH=@TOR_LIB_MATH@
+TOR_LIBEVENT_LIBS=@TOR_LIBEVENT_LIBS@
+TOR_OPENSSL_LIBS=@TOR_OPENSSL_LIBS@
+TOR_LIB_WS32=@TOR_LIB_WS32@
+TOR_LIB_GDI=@TOR_LIB_GDI@
+TOR_LIB_USERENV=@TOR_LIB_USERENV@
+CURVE25519_LIBS=@CURVE25519_LIBS@
+TOR_SYSTEMD_LIBS=@TOR_SYSTEMD_LIBS@
+TOR_LZMA_LIBS=@TOR_LZMA_LIBS@
+TOR_ZSTD_LIBS=@TOR_ZSTD_LIBS@
+LIBS=@LIBS@
+LDFLAGS=@LDFLAGS@
+NSS_LIBS=@NSS_LIBS@
\ No newline at end of file
diff --git a/configure.ac b/configure.ac
index 771769422e..834fea073a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,17 +1,17 @@
 dnl Copyright (c) 2001-2004, Roger Dingledine
 dnl Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
-dnl Copyright (c) 2007-2015, The Tor Project, Inc.
+dnl Copyright (c) 2007-2019, The Tor Project, Inc.
 dnl See LICENSE for licensing information
 
 AC_PREREQ([2.63])
-AC_INIT([tor],[0.2.9.17-dev])
-AC_CONFIG_SRCDIR([src/or/main.c])
+AC_INIT([tor],[0.3.5.9-dev])
+AC_CONFIG_SRCDIR([src/app/main/tor_main.c])
 AC_CONFIG_MACRO_DIR([m4])
 
 # "foreign" means we don't follow GNU package layout standards
 # "1.11" means we require automake version 1.11 or newer
 # "subdir-objects" means put .o files in the same directory as the .c files
-AM_INIT_AUTOMAKE([foreign 1.11 subdir-objects])
+AM_INIT_AUTOMAKE([foreign 1.11 subdir-objects -Wall -Werror])
 
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 AC_CONFIG_HEADERS([orconfig.h])
@@ -21,18 +21,8 @@ AC_CANONICAL_HOST
 
 PKG_PROG_PKG_CONFIG
 
-if test -f "/etc/redhat-release"; then
-  if test -f "/usr/kerberos/include"; then
-    CPPFLAGS="$CPPFLAGS -I/usr/kerberos/include"
-  fi
-fi
-
-# Not a no-op; we want to make sure that CPPFLAGS is set before we use
-# the += operator on it in src/or/Makefile.am
-CPPFLAGS="$CPPFLAGS -I\${top_srcdir}/src/common"
-
 AC_ARG_ENABLE(openbsd-malloc,
-   AS_HELP_STRING(--enable-openbsd-malloc, [use malloc code from OpenBSD.  Linux only]))
+   AS_HELP_STRING(--enable-openbsd-malloc, [use malloc code from OpenBSD.  Linux only. Deprecated: see --with-malloc]))
 AC_ARG_ENABLE(static-openssl,
    AS_HELP_STRING(--enable-static-openssl, [link against a static openssl library. Requires --with-openssl-dir]))
 AC_ARG_ENABLE(static-libevent,
@@ -49,6 +39,24 @@ AC_ARG_ENABLE(asserts-in-tests,
    AS_HELP_STRING(--disable-asserts-in-tests, [disable tor_assert() calls in the unit tests, for branch coverage]))
 AC_ARG_ENABLE(system-torrc,
    AS_HELP_STRING(--disable-system-torrc, [don't look for a system-wide torrc file]))
+AC_ARG_ENABLE(libfuzzer,
+   AS_HELP_STRING(--enable-libfuzzer, [build extra fuzzers based on 'libfuzzer']))
+AC_ARG_ENABLE(oss-fuzz,
+   AS_HELP_STRING(--enable-oss-fuzz, [build extra fuzzers based on 'oss-fuzz' environment]))
+AC_ARG_ENABLE(memory-sentinels,
+   AS_HELP_STRING(--disable-memory-sentinels, [disable code that tries to prevent some kinds of memory access bugs. For fuzzing only.]))
+AC_ARG_ENABLE(rust,
+   AS_HELP_STRING(--enable-rust, [enable rust integration]))
+AC_ARG_ENABLE(cargo-online-mode,
+   AS_HELP_STRING(--enable-cargo-online-mode, [Allow cargo to make network requests to fetch crates. For builds with rust only.]))
+AC_ARG_ENABLE(restart-debugging,
+   AS_HELP_STRING(--enable-restart-debugging, [Build Tor with support for debugging in-process restart. Developers only.]))
+AC_ARG_ENABLE(zstd-advanced-apis,
+   AS_HELP_STRING(--disable-zstd-advanced-apis, [Build without support for zstd's "static-only" APIs.]))
+AC_ARG_ENABLE(nss,
+   AS_HELP_STRING(--enable-nss, [Use Mozilla's NSS TLS library. (EXPERIMENTAL)]))
+AC_ARG_ENABLE(pic,
+   AS_HELP_STRING(--enable-pic, [Build Tor's binaries as position-independent code, suitable to link as a library.]))
 
 if test "x$enable_coverage" != "xyes" -a "x$enable_asserts_in_tests" = "xno" ; then
     AC_MSG_ERROR([Can't disable assertions outside of coverage build])
@@ -57,6 +65,19 @@ fi
 AM_CONDITIONAL(UNITTESTS_ENABLED, test "x$enable_unittests" != "xno")
 AM_CONDITIONAL(COVERAGE_ENABLED, test "x$enable_coverage" = "xyes")
 AM_CONDITIONAL(DISABLE_ASSERTS_IN_UNIT_TESTS, test "x$enable_asserts_in_tests" = "xno")
+AM_CONDITIONAL(LIBFUZZER_ENABLED, test "x$enable_libfuzzer" = "xyes")
+AM_CONDITIONAL(OSS_FUZZ_ENABLED, test "x$enable_oss_fuzz" = "xyes")
+AM_CONDITIONAL(USE_RUST, test "x$enable_rust" = "xyes")
+AM_CONDITIONAL(USE_NSS, test "x$enable_nss" = "xyes")
+AM_CONDITIONAL(USE_OPENSSL, test "x$enable_nss" != "xyes")
+
+if test "x$enable_nss" = "xyes"; then
+  AC_DEFINE(ENABLE_NSS, 1,
+  	    [Defined if we're building with NSS.])
+else
+  AC_DEFINE(ENABLE_OPENSSL, 1,
+            [Defined if we're building with OpenSSL or LibreSSL])
+fi
 
 if test "$enable_static_tor" = "yes"; then
   enable_static_libevent="yes";
@@ -70,7 +91,10 @@ if test "$enable_system_torrc" = "no"; then
             [Defined if we're not going to look for a torrc in SYSCONF])
 fi
 
-AM_CONDITIONAL(USE_OPENBSD_MALLOC, test "x$enable_openbsd_malloc" = "xyes")
+if test "$enable_memory_sentinels" = "no"; then
+  AC_DEFINE(DISABLE_MEMORY_SENTINELS, 1,
+           [Defined if we're turning off memory safety code to look for bugs])
+fi
 
 AC_ARG_ENABLE(asciidoc,
      AS_HELP_STRING(--disable-asciidoc, [don't use asciidoc (disables building of manpages)]),
@@ -89,7 +113,15 @@ AC_ARG_ENABLE(systemd,
         * ) AC_MSG_ERROR(bad value for --enable-systemd) ;;
       esac], [systemd=auto])
 
+if test "$enable_restart_debugging" = "yes"; then
+  AC_DEFINE(ENABLE_RESTART_DEBUGGING, 1,
+            [Defined if we're building with support for in-process restart debugging.])
+fi
 
+if test "$enable_zstd_advanced_apis" != "no"; then
+   AC_DEFINE(ENABLE_ZSTD_ADVANCED_APIS, 1,
+             [Defined if we're going to try to use zstd's "static-only" APIs.])
+fi
 
 # systemd support
 if test "x$enable_systemd" = "xno"; then
@@ -139,8 +171,15 @@ dnl Others suggest '/gs /safeseh /nxcompat /dynamicbase' for non-gcc on Windows
 AC_ARG_ENABLE(gcc-hardening,
     AS_HELP_STRING(--disable-gcc-hardening, [disable compiler security checks]))
 
+dnl Deprecated --enable-expensive-hardening but keep it for now for backward compat.
 AC_ARG_ENABLE(expensive-hardening,
-    AS_HELP_STRING(--enable-expensive-hardening, [enable more expensive compiler hardening; makes Tor slower]))
+    AS_HELP_STRING(--enable-expensive-hardening, [enable more fragile and expensive compiler hardening; makes Tor slower]))
+AC_ARG_ENABLE(fragile-hardening,
+    AS_HELP_STRING(--enable-fragile-hardening, [enable more fragile and expensive compiler hardening; makes Tor slower]))
+if test "x$enable_expensive_hardening" = "xyes" || test "x$enable_fragile_hardening" = "xyes"; then
+  fragile_hardening="yes"
+  AC_DEFINE(DEBUG_SMARTLIST, 1, [Enable smartlist debugging])
+fi
 
 dnl Linker hardening options
 dnl Currently these options are ELF specific - you can't use this with MacOSX
@@ -154,13 +193,6 @@ if test "$enable_local_appdata" = "yes"; then
             [Defined if we default to host local appdata paths on Windows])
 fi
 
-# Tor2web mode flag
-AC_ARG_ENABLE(tor2web-mode,
-     AS_HELP_STRING(--enable-tor2web-mode, [support tor2web non-anonymous mode]),
-[if test "x$enableval" = "xyes"; then
-    CFLAGS="$CFLAGS -D ENABLE_TOR2WEB_MODE=1"
-fi])
-
 AC_ARG_ENABLE(tool-name-check,
      AS_HELP_STRING(--disable-tool-name-check, [check for sanely named toolchain when cross-compiling]))
 
@@ -170,11 +202,64 @@ AC_ARG_ENABLE(seccomp,
 AC_ARG_ENABLE(libscrypt,
      AS_HELP_STRING(--disable-libscrypt, [do not attempt to use libscrypt]))
 
-dnl check for the correct "ar" when cross-compiling
-AN_MAKEVAR([AR], [AC_PROG_AR])
-AN_PROGRAM([ar], [AC_PROG_AR])
-AC_DEFUN([AC_PROG_AR], [AC_CHECK_TOOL([AR], [ar], [ar])])
-AC_PROG_AR
+dnl Enable event tracing which are transformed to debug log statement.
+AC_ARG_ENABLE(event-tracing-debug,
+     AS_HELP_STRING(--enable-event-tracing-debug, [build with event tracing to debug log]))
+AM_CONDITIONAL([USE_EVENT_TRACING_DEBUG], [test "x$enable_event_tracing_debug" = "xyes"])
+
+if test x$enable_event_tracing_debug = xyes; then
+  AC_DEFINE([USE_EVENT_TRACING_DEBUG], [1], [Tracing framework to log debug])
+  AC_DEFINE([TOR_EVENT_TRACING_ENABLED], [1], [Compile the event tracing instrumentation])
+fi
+
+dnl Enable Android only features.
+AC_ARG_ENABLE(android,
+     AS_HELP_STRING(--enable-android, [build with Android features enabled]))
+AM_CONDITIONAL([USE_ANDROID], [test "x$enable_android" = "xyes"])
+
+if test "x$enable_android" = "xyes"; then
+  AC_DEFINE([USE_ANDROID], [1], [Compile with Android specific features enabled])
+
+  dnl Check if the Android log library is available.
+  AC_CHECK_HEADERS([android/log.h])
+  AC_SEARCH_LIBS(__android_log_write, [log])
+
+fi
+
+dnl ---
+dnl Tor modules options. These options are namespaced with --disable-module-XXX
+dnl ---
+
+dnl All our modules.
+m4_define(MODULES, dirauth)
+
+dnl Directory Authority module.
+AC_ARG_ENABLE([module-dirauth],
+              AS_HELP_STRING([--disable-module-dirauth],
+                             [Do not build tor with the dirauth module]),
+              [], dnl Action if-given
+              AC_DEFINE([HAVE_MODULE_DIRAUTH], [1],
+                        [Compile with Directory Authority feature support]))
+AM_CONDITIONAL(BUILD_MODULE_DIRAUTH, [test "x$enable_module_dirauth" != "xno"])
+
+dnl Helper variables.
+TOR_MODULES_ALL_ENABLED=
+AC_DEFUN([ADD_MODULE], [
+    MODULE=m4_toupper($1)
+    TOR_MODULES_ALL_ENABLED="${TOR_MODULES_ALL_ENABLED} -DHAVE_MODULE_${MODULE}=1"
+])
+m4_foreach_w([module], MODULES, [ADD_MODULE([module])])
+AC_SUBST(TOR_MODULES_ALL_ENABLED)
+
+dnl check for the correct "ar" when cross-compiling.
+dnl   (AM_PROG_AR was new in automake 1.11.2, which we do not yet require,
+dnl    so kludge up a replacement for the case where it isn't there yet.)
+m4_ifdef([AM_PROG_AR],
+         [AM_PROG_AR],
+         [AN_MAKEVAR([AR], [AC_PROG_AR])
+          AN_PROGRAM([ar], [AC_PROG_AR])
+          AC_DEFUN([AC_PROG_AR], [AC_CHECK_TOOL([AR], [ar], [:])])
+          AC_PROG_AR])
 
 dnl Check whether the above macro has settled for a simply named tool even
 dnl though we're cross compiling. We must do this before running AC_PROG_CC,
@@ -194,11 +279,11 @@ AC_PROG_CC
 AC_PROG_CPP
 AC_PROG_MAKE_SET
 AC_PROG_RANLIB
+AC_PROG_SED
 
-AC_PATH_PROG([PERL], [perl])
-
-dnl autoconf 2.59 appears not to support AC_PROG_SED
-AC_CHECK_PROG([SED],[sed],[sed],[/bin/false])
+AC_ARG_VAR([PERL], [path to Perl binary])
+AC_CHECK_PROGS([PERL], [perl])
+AM_CONDITIONAL(USE_PERL, [test "x$ac_cv_prog_PERL" != "x"])
 
 dnl check for asciidoc and a2x
 AC_PATH_PROG([ASCIIDOC], [asciidoc], none)
@@ -206,9 +291,6 @@ AC_PATH_PROGS([A2X], [a2x a2x.py], none)
 
 AM_CONDITIONAL(USE_ASCIIDOC, test "x$asciidoc" = "xtrue")
 
-AM_CONDITIONAL(USE_FW_HELPER, test "x$natpmp" = "xtrue" || test "x$upnp" = "xtrue")
-AM_CONDITIONAL(NAT_PMP, test "x$natpmp" = "xtrue")
-AM_CONDITIONAL(MINIUPNPC, test "x$upnp" = "xtrue")
 AM_PROG_CC_C_O
 AC_PROG_CC_C99
 
@@ -223,6 +305,13 @@ if test "x$PYTHON" = "x"; then
 fi
 AM_CONDITIONAL(USEPYTHON, [test "x$PYTHON" != "x"])
 
+dnl List all external rust crates we depend on here. Include the version
+rust_crates=" \
+    digest-0.7.2 \
+    libc-0.2.39 \
+"
+AC_SUBST(rust_crates)
+
 ifdef([AC_C_FLEXIBLE_ARRAY_MEMBER], [
 AC_C_FLEXIBLE_ARRAY_MEMBER
 ], [
@@ -338,8 +427,9 @@ AH_BOTTOM([
 #endif
 ])
 
-
+AM_CONDITIONAL(WIN32, test "x$bwin32" = "xtrue")
 AM_CONDITIONAL(BUILD_NT_SERVICES, test "x$bwin32" = "xtrue")
+AM_CONDITIONAL(BUILD_LIBTORRUNNER, test "x$bwin32" != "xtrue")
 
 dnl Enable C99 when compiling with MIPSpro
 AC_MSG_CHECKING([for MIPSpro compiler])
@@ -358,6 +448,116 @@ fi
 
 AC_C_BIGENDIAN
 
+AC_ARG_VAR([TOR_RUST_TARGET], [Rust target, must be specified when cross-compiling (HOST != BUILD). example: i686-pc-windows-gnu])
+
+if test "x$enable_rust" = "xyes"; then
+  AC_ARG_VAR([RUSTC], [path to the rustc binary])
+  AC_CHECK_PROG([RUSTC], [rustc], [rustc],[no])
+  if test "x$RUSTC" = "xno"; then
+    AC_MSG_ERROR([rustc unavailable but rust integration requested.])
+  fi
+
+  AC_ARG_VAR([CARGO], [path to the cargo binary])
+  AC_CHECK_PROG([CARGO], [cargo], [cargo],[no])
+  if test "x$CARGO" = "xno"; then
+    AC_MSG_ERROR([cargo unavailable but rust integration requested.])
+  fi
+
+  AC_DEFINE([HAVE_RUST], 1, [have Rust])
+  if test "x$enable_fatal_warnings" = "xyes"; then
+    RUST_WARN=
+  else
+    RUST_WARN=#
+  fi
+  if test "x$enable_cargo_online_mode" = "xyes"; then
+    CARGO_ONLINE=
+    RUST_DL=#
+  else
+    CARGO_ONLINE=--frozen
+    RUST_DL=
+
+    dnl When we're not allowed to touch the network, we need crate dependencies
+    dnl locally available.
+    AC_MSG_CHECKING([rust crate dependencies])
+    AC_ARG_VAR([TOR_RUST_DEPENDENCIES], [path to directory with local crate mirror])
+    if test "x$TOR_RUST_DEPENDENCIES" = "x"; then
+      TOR_RUST_DEPENDENCIES="${srcdir}/src/ext/rust/crates"
+    fi
+    dnl Check whether the path exists before we try to cd into it.
+    if test ! -d "$TOR_RUST_DEPENDENCIES"; then
+      AC_MSG_ERROR([Rust dependency directory $TOR_RUST_DEPENDENCIES does not exist. Specify a dependency directory using the TOR_RUST_DEPENDENCIES variable or allow cargo to fetch crates using --enable-cargo-online-mode.])
+      ERRORED=1
+    fi
+    dnl Make the path absolute, since we'll be using it from within a
+    dnl subdirectory.
+    TOR_RUST_DEPENDENCIES=$(cd "$TOR_RUST_DEPENDENCIES" ; pwd)
+
+    for dep in $rust_crates; do
+      if test ! -d "$TOR_RUST_DEPENDENCIES"/"$dep"; then
+        AC_MSG_ERROR([Failure to find rust dependency $TOR_RUST_DEPENDENCIES/$dep. Specify a dependency directory using the TOR_RUST_DEPENDENCIES variable or allow cargo to fetch crates using --enable-cargo-online-mode.])
+        ERRORED=1
+      fi
+    done
+    if test "x$ERRORED" = "x"; then
+      AC_MSG_RESULT([yes])
+    fi
+  fi
+
+  dnl This is a workaround for #46797
+  dnl (a.k.a https://github.com/rust-lang/rust/issues/46797 ).  Once the
+  dnl upstream bug is fixed, we can remove this workaround.
+  case "$host_os" in
+      darwin*)
+        TOR_RUST_EXTRA_LIBS="-lresolv"
+	;;
+  esac
+
+  dnl For now both MSVC and MinGW rust libraries will output static libs with
+  dnl the MSVC naming convention.
+  if test "$bwin32" = "true"; then
+    tor_rust_static_name=tor_rust.lib
+  else
+    tor_rust_static_name=libtor_rust.a
+  fi
+
+  AC_CANONICAL_BUILD
+
+  if test -n "$TOR_RUST_TARGET"; then
+    if test "$host" = "$build"; then
+      AC_MSG_ERROR([HOST = BUILD is invalid if TOR_RUST_TARGET is specified, see configure --help for more information.])
+    fi
+    RUST_TARGET_PROP="target = '$TOR_RUST_TARGET'"
+    TOR_RUST_LIB_PATH="src/rust/target/$TOR_RUST_TARGET/release/$tor_rust_static_name"
+  else
+    if test "$host" != "$build"; then
+      AC_MSG_ERROR([TOR_RUST_TARGET must be specified when cross-compiling with Rust enabled.])
+    fi
+    RUST_TARGET_PROP=
+    TOR_RUST_LIB_PATH="src/rust/target/release/$tor_rust_static_name"
+  fi
+
+  AC_SUBST(RUST_TARGET_PROP)
+  AC_SUBST(TOR_RUST_LIB_PATH)
+  AC_SUBST(CARGO_ONLINE)
+  AC_SUBST(RUST_WARN)
+  AC_SUBST(RUST_DL)
+
+  dnl Let's check the rustc version, too
+  AC_MSG_CHECKING([rust version])
+  RUSTC_VERSION=`$RUSTC --version`
+  RUSTC_VERSION_MAJOR=`$RUSTC --version | cut -d ' ' -f 2 | cut -d '.' -f 1`
+  RUSTC_VERSION_MINOR=`$RUSTC --version | cut -d ' ' -f 2 | cut -d '.' -f 2`
+  if test "x$RUSTC_VERSION_MAJOR" = "x" -o "x$RUSTC_VERSION_MINOR" = "x"; then
+    AC_MSG_ERROR([rustc version couldn't be identified])
+  fi
+  if test "$RUSTC_VERSION_MAJOR" -lt 2 -a "$RUSTC_VERSION_MINOR" -lt 14; then
+    AC_MSG_ERROR([rustc must be at least version 1.14])
+  fi
+  AC_MSG_RESULT([$RUSTC_VERSION])
+fi
+
+AC_SUBST(TOR_RUST_EXTRA_LIBS)
+
 AC_SEARCH_LIBS(socket, [socket network])
 AC_SEARCH_LIBS(gethostbyname, [nsl])
 AC_SEARCH_LIBS(dlopen, [dl])
@@ -379,62 +579,95 @@ AM_CONDITIONAL(THREADS_WIN32, test "$bwin32" = "true")
 AM_CONDITIONAL(THREADS_PTHREADS, test "$bwin32" = "false")
 
 AC_CHECK_FUNCS(
-        _NSGetEnviron \
+	_NSGetEnviron \
 	RtlSecureZeroMemory \
 	SecureZeroMemory \
-        accept4 \
-        backtrace \
-        backtrace_symbols_fd \
+	accept4 \
+	backtrace \
+	backtrace_symbols_fd \
 	eventfd \
 	explicit_bzero \
 	timingsafe_memcmp \
-        flock \
-        ftime \
-        getaddrinfo \
-        getifaddrs \
-        getpass \
-        getrlimit \
-        gettimeofday \
-        gmtime_r \
+	flock \
+	ftime \
+	get_current_dir_name \
+	getaddrinfo \
+	getdelim \
+	getifaddrs \
+	getline \
+	getpass \
+	getrlimit \
+	gettimeofday \
+	gmtime_r \
 	gnu_get_libc_version \
 	htonll \
-        inet_aton \
-        ioctl \
-        issetugid \
-        llround \
-        localtime_r \
-        lround \
-        memmem \
-        memset_s \
+	inet_aton \
+	ioctl \
+	issetugid \
+	llround \
+	localtime_r \
+	lround \
+	memmem \
+	memset_s \
+	mmap \
 	pipe \
 	pipe2 \
-        prctl \
+	prctl \
 	readpassphrase \
-        rint \
-        sigaction \
-        socketpair \
+	rint \
+	sigaction \
+	socketpair \
 	statvfs \
-        strlcat \
-        strlcpy \
+	strncasecmp \
+	strcasecmp \
+	strlcat \
+	strlcpy \
 	strnlen \
-        strptime \
-        strtok_r \
-        strtoull \
-        sysconf \
+	strptime \
+	strtok_r \
+	strtoull \
+	sysconf \
 	sysctl \
 	truncate \
-        uname \
+	uname \
 	usleep \
-        vasprintf \
+	vasprintf \
 	_vscprintf
 )
 
-# Apple messed up when they added two functions functions in Sierra: they
+# Apple messed up when they added some functions: they
 # forgot to decorate them with appropriate AVAILABLE_MAC_OS_VERSION
-# checks. So we should only probe for those functions if we are sure that we
-# are not targetting OSX 10.11 or earlier.
+# checks.
+
+# We should only probe for these functions if we are sure that we
+# are not targeting OS X 10.9 or earlier.
+AC_MSG_CHECKING([for a pre-Yosemite OS X build target])
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
+#ifdef __APPLE__
+#  include <AvailabilityMacros.h>
+#  ifndef MAC_OS_X_VERSION_10_10
+#    define MAC_OS_X_VERSION_10_10 101000
+#  endif
+#  if defined(MAC_OS_X_VERSION_MIN_REQUIRED)
+#    if MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_10
+#      error "Running on Mac OS X 10.9 or earlier"
+#    endif
+#  endif
+#endif
+]], [[]])],
+   [on_macos_pre_10_10=no ; AC_MSG_RESULT([no])],
+   [on_macos_pre_10_10=yes; AC_MSG_RESULT([yes])])
+
+if test "$on_macos_pre_10_10" = "no"; then
+  AC_CHECK_FUNCS(
+        mach_approximate_time \
+  )
+fi
+
+# We should only probe for these functions if we are sure that we
+# are not targeting OSX 10.11 or earlier.
 AC_MSG_CHECKING([for a pre-Sierra OSX build target])
-AC_TRY_COMPILE([
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #ifdef __APPLE__
 #  include <AvailabilityMacros.h>
 #  ifndef MAC_OS_X_VERSION_10_12
@@ -446,7 +679,7 @@ AC_TRY_COMPILE([
 #    endif
 #  endif
 #endif
-], [],
+]], [[]])],
    [on_macos_pre_10_12=no ; AC_MSG_RESULT([no])],
    [on_macos_pre_10_12=yes; AC_MSG_RESULT([yes])])
 
@@ -474,6 +707,21 @@ fi
 AM_CONDITIONAL(BUILD_READPASSPHRASE_C,
   test "x$ac_cv_func_readpassphrase" = "xno" && test "$bwin32" = "false")
 
+AC_MSG_CHECKING([whether free(NULL) works])
+AC_RUN_IFELSE([AC_LANG_PROGRAM([
+  #include <stdlib.h>
+], [
+char *p = NULL;
+free(p);
+])],
+[free_null_ok=true; AC_MSG_RESULT(yes)],
+[free_null_ok=false; AC_MSG_RESULT(no)],
+[free_null_ok=cross; AC_MSG_RESULT(cross)])
+
+if test "$free_null_ok" = "false"; then
+   AC_MSG_ERROR([Your libc implementation doesn't allow free(NULL), as required by C99.])
+fi
+
 dnl ------------------------------------------------------
 dnl Where do you live, libevent?  And how do we call you?
 
@@ -483,13 +731,16 @@ if test "$bwin32" = "true"; then
   # Some of the cargo-cults recommend -lwsock32 as well, but I don't
   # think it's actually necessary.
   TOR_LIB_GDI=-lgdi32
+  TOR_LIB_USERENV=-luserenv
 else
   TOR_LIB_WS32=
   TOR_LIB_GDI=
+  TOR_LIB_USERENV=
 fi
 AC_SUBST(TOR_LIB_WS32)
 AC_SUBST(TOR_LIB_GDI)
 AC_SUBST(TOR_LIB_IPHLPAPI)
+AC_SUBST(TOR_LIB_USERENV)
 
 tor_libevent_pkg_redhat="libevent"
 tor_libevent_pkg_debian="libevent-dev"
@@ -516,12 +767,13 @@ TOR_SEARCH_LIBRARY(libevent, $trylibeventdir, [-levent $STATIC_LIBEVENT_FLAGS $T
 #include <winsock2.h>
 #endif
 struct event_base;
-struct event_base *event_base_new(void);],
+struct event_base *event_base_new(void);
+void event_base_free(struct event_base *);],
     [
 #ifdef _WIN32
 {WSADATA d; WSAStartup(0x101,&d); }
 #endif
-event_base_new();
+event_base_free(event_base_new());
 ], [--with-libevent-dir], [/opt/libevent])
 
 dnl Determine the incantation needed to link libevent.
@@ -603,8 +855,21 @@ LIBS="$save_LIBS"
 AC_SUBST(TOR_LIB_MATH)
 
 dnl ------------------------------------------------------
+dnl Hello, NSS.  You're new around here.
+if test "x$enable_nss" = "xyes"; then
+  PKG_CHECK_MODULES(NSS,
+     [nss],
+     [have_nss=yes],
+     [have_nss=no; AC_MSG_ERROR([You asked for NSS but I can't find it.])])
+  AC_SUBST(NSS_CFLAGS)
+  AC_SUBST(NSS_LIBS)
+fi
+
+dnl ------------------------------------------------------
 dnl Where do you live, openssl?  And how do we call you?
 
+if test "x$enable_nss" != "xyes"; then
+
 tor_openssl_pkg_redhat="openssl"
 tor_openssl_pkg_debian="libssl-dev"
 tor_openssl_devpkg_redhat="openssl-devel"
@@ -619,11 +884,21 @@ AC_ARG_WITH(ssl-dir,
       fi
   ])
 
-TOR_SEARCH_LIBRARY(openssl, $tryssldir, [-lssl -lcrypto $TOR_LIB_GDI],
-    [#include <openssl/rand.h>],
-    [void RAND_add(const void *buf, int num, double entropy);],
-    [RAND_add((void*)0,0,0);], [],
-    [/usr/local/openssl /usr/lib/openssl /usr/local/ssl /usr/lib/ssl /usr/local /usr/athena /opt/openssl])
+AC_MSG_NOTICE([Now, we'll look for OpenSSL >= 1.0.1])
+TOR_SEARCH_LIBRARY(openssl, $tryssldir, [-lssl -lcrypto $TOR_LIB_GDI $TOR_LIB_WS32],
+    [#include <openssl/ssl.h>
+     char *getenv(const char *);],
+    [struct ssl_cipher_st;
+     unsigned SSL_CIPHER_get_id(const struct ssl_cipher_st *);
+     char *getenv(const char *);],
+    dnl This funny-looking test program calls getenv, so that the compiler
+    dnl will neither make code that call SSL_CIPHER_get_id(NULL) [producing
+    dnl a crash], nor optimize out the call to SSL_CIPHER_get_id().
+    dnl We look for SSL_cipher_get_id() because it is present in
+    dnl OpenSSL >=1.0.1, because it is not deprecated, and because Tor
+    dnl depends on it.
+    [if (getenv("THIS_SHOULDNT_BE_SET_X201803")) SSL_CIPHER_get_id((void *)0);], [],
+    [/usr/local/opt/openssl /usr/local/openssl /usr/lib/openssl /usr/local/ssl /usr/lib/ssl /usr/local /opt/openssl])
 
 dnl XXXX check for OPENSSL_VERSION_NUMBER == SSLeay()
 
@@ -646,16 +921,16 @@ LIBS="$TOR_OPENSSL_LIBS $LIBS"
 LDFLAGS="$TOR_LDFLAGS_openssl $LDFLAGS"
 CPPFLAGS="$TOR_CPPFLAGS_openssl $CPPFLAGS"
 
-AC_TRY_COMPILE([
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #include <openssl/opensslv.h>
 #if !defined(LIBRESSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER < 0x1000100fL
 #error "too old"
 #endif
-   ], [],
+   ]], [[]])],
    [ : ],
-   [ AC_ERROR([OpenSSL is too old. We require 1.0.1 or later. You can specify a path to a newer one with --with-openssl-dir.]) ])
+   [ AC_MSG_ERROR([OpenSSL is too old. We require 1.0.1 or later. You can specify a path to a newer one with --with-openssl-dir.]) ])
 
-AC_TRY_COMPILE([
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #include <openssl/opensslv.h>
 #include <openssl/evp.h>
 #if defined(OPENSSL_NO_EC) || defined(OPENSSL_NO_ECDH) || defined(OPENSSL_NO_ECDSA)
@@ -664,15 +939,16 @@ AC_TRY_COMPILE([
 #if !defined(NID_X9_62_prime256v1) || !defined(NID_secp224r1)
 #error "curves unavailable"
 #endif
-   ], [],
+   ]], [[]])],
    [ : ],
-   [ AC_ERROR([OpenSSL is built without full ECC support, including curves P256 and P224. You can specify a path to one with ECC support with --with-openssl-dir.]) ])
+   [ AC_MSG_ERROR([OpenSSL is built without full ECC support, including curves P256 and P224. You can specify a path to one with ECC support with --with-openssl-dir.]) ])
 
 AC_CHECK_MEMBERS([struct ssl_method_st.get_cipher_by_char], , ,
 [#include <openssl/ssl.h>
 ])
 
 AC_CHECK_FUNCS([ \
+		ERR_load_KDF_strings \
 		SSL_SESSION_get_master_key \
 		SSL_get_server_random \
                 SSL_get_client_ciphers \
@@ -691,6 +967,43 @@ AC_CHECK_MEMBERS([SSL.state], , ,
 [#include <openssl/ssl.h>
 ])
 
+AC_CHECK_SIZEOF(SHA_CTX, , [AC_INCLUDES_DEFAULT()
+#include <openssl/sha.h>
+])
+
+fi # enable_nss
+
+dnl ======================================================================
+dnl Can we use KIST?
+
+dnl Define the set of checks for KIST scheduler support.
+AC_DEFUN([CHECK_KIST_SUPPORT],[
+  dnl KIST needs struct tcp_info and for certain members to exist.
+  AC_CHECK_MEMBERS(
+    [struct tcp_info.tcpi_unacked, struct tcp_info.tcpi_snd_mss],
+    , ,[[#include <netinet/tcp.h>]])
+  dnl KIST needs SIOCOUTQNSD to exist for an ioctl call.
+  AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [
+                     #include <linux/sockios.h>
+                     #ifndef SIOCOUTQNSD
+                     #error
+                     #endif
+                     ])], have_siocoutqnsd=yes, have_siocoutqnsd=no)
+  if test "x$have_siocoutqnsd" = "xyes"; then
+    if test "x$ac_cv_member_struct_tcp_info_tcpi_unacked" = "xyes"; then
+      if test "x$ac_cv_member_struct_tcp_info_tcpi_snd_mss" = "xyes"; then
+        have_kist_support=yes
+      fi
+    fi
+  fi
+])
+dnl Now, trigger the check.
+CHECK_KIST_SUPPORT
+AS_IF([test "x$have_kist_support" = "xyes"],
+      [AC_DEFINE(HAVE_KIST_SUPPORT, 1, [Defined if KIST scheduler is supported
+                                        on this system])],
+      [AC_MSG_NOTICE([KIST scheduler can't be used. Missing support.])])
+
 LIBS="$save_LIBS"
 LDFLAGS="$save_LDFLAGS"
 CPPFLAGS="$save_CPPFLAGS"
@@ -721,6 +1034,80 @@ else
 fi
 AC_SUBST(TOR_ZLIB_LIBS)
 
+dnl ------------------------------------------------------
+dnl Where we do we find lzma?
+
+AC_ARG_ENABLE(lzma,
+      AS_HELP_STRING(--enable-lzma, [enable support for the LZMA compression scheme.]),
+      [case "${enableval}" in
+        "yes") lzma=true ;;
+        "no")  lzma=false ;;
+        * ) AC_MSG_ERROR(bad value for --enable-lzma) ;;
+      esac], [lzma=auto])
+
+if test "x$enable_lzma" = "xno"; then
+    have_lzma=no;
+else
+    PKG_CHECK_MODULES([LZMA],
+                      [liblzma],
+                      have_lzma=yes,
+                      have_lzma=no)
+
+    if test "x$have_lzma" = "xno" ; then
+        AC_MSG_WARN([Unable to find liblzma.])
+    fi
+fi
+
+if test "x$have_lzma" = "xyes"; then
+    AC_DEFINE(HAVE_LZMA,1,[Have LZMA])
+    TOR_LZMA_CFLAGS="${LZMA_CFLAGS}"
+    TOR_LZMA_LIBS="${LZMA_LIBS}"
+fi
+AC_SUBST(TOR_LZMA_CFLAGS)
+AC_SUBST(TOR_LZMA_LIBS)
+
+dnl ------------------------------------------------------
+dnl Where we do we find zstd?
+
+AC_ARG_ENABLE(zstd,
+      AS_HELP_STRING(--enable-zstd, [enable support for the Zstandard compression scheme.]),
+      [case "${enableval}" in
+        "yes") zstd=true ;;
+        "no")  zstd=false ;;
+        * ) AC_MSG_ERROR(bad value for --enable-zstd) ;;
+      esac], [zstd=auto])
+
+if test "x$enable_zstd" = "xno"; then
+    have_zstd=no;
+else
+    PKG_CHECK_MODULES([ZSTD],
+                      [libzstd >= 1.1],
+                      have_zstd=yes,
+                      have_zstd=no)
+
+    if test "x$have_zstd" = "xno" ; then
+        AC_MSG_WARN([Unable to find libzstd.])
+    fi
+fi
+
+if test "x$have_zstd" = "xyes"; then
+    AC_DEFINE(HAVE_ZSTD,1,[Have Zstd])
+    TOR_ZSTD_CFLAGS="${ZSTD_CFLAGS}"
+    TOR_ZSTD_LIBS="${ZSTD_LIBS}"
+
+    dnl now check for zstd functions
+    save_LIBS="$LIBS"
+    save_CFLAGS="$CFLAGS"
+    LIBS="$LIBS $ZSTD_LIBS"
+    CFLAGS="$CFLAGS $ZSTD_CFLAGS"
+    AC_CHECK_FUNCS(ZSTD_estimateCStreamSize \
+                   ZSTD_estimateDCtxSize)
+    LIBS="$save_LIBS"
+    CFLAGS="$save_CFLAGS"
+fi
+AC_SUBST(TOR_ZSTD_CFLAGS)
+AC_SUBST(TOR_ZSTD_LIBS)
+
 dnl ----------------------------------------------------------------------
 dnl Check if libcap is available for capabilities.
 
@@ -741,18 +1128,23 @@ dnl since sometimes the linker will like an option but not be willing to
 dnl use it with a build of a library.
 
 all_ldflags_for_check="$TOR_LDFLAGS_zlib $TOR_LDFLAGS_openssl $TOR_LDFLAGS_libevent"
-all_libs_for_check="$TOR_ZLIB_LIBS $TOR_LIB_MATH $TOR_LIBEVENT_LIBS $TOR_OPENSSL_LIBS $TOR_SYSTEMD_LIBS $TOR_LIB_WS32 $TOR_LIB_GDI $TOR_CAP_LIBS"
+all_libs_for_check="$TOR_ZLIB_LIBS $TOR_LIB_MATH $TOR_LIBEVENT_LIBS $TOR_OPENSSL_LIBS $TOR_SYSTEMD_LIBS $TOR_LIB_WS32 $TOR_LIB_GDI $TOR_LIB_USERENV $TOR_CAP_LIBS"
 
 CFLAGS_FTRAPV=
 CFLAGS_FWRAPV=
 CFLAGS_ASAN=
 CFLAGS_UBSAN=
 
+
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [
 #if !defined(__clang__)
 #error
 #endif])], have_clang=yes, have_clang=no)
 
+if test "x$enable_pic" = "xyes"; then
+    TOR_CHECK_CFLAGS(-fPIC)
+fi
+
 if test "x$enable_gcc_hardening" != "xno"; then
     CFLAGS="$CFLAGS -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2"
     if test "x$have_clang" = "xyes"; then
@@ -771,36 +1163,69 @@ m4_ifdef([AS_VAR_IF],[
     AS_VAR_POPDEF([can_compile])
     TOR_CHECK_CFLAGS(-Wstack-protector)
     TOR_CHECK_CFLAGS(--param ssp-buffer-size=1)
-    if test "$bwin32" = "false"; then
-       TOR_CHECK_CFLAGS(-fPIE)
+    if test "$bwin32" = "false" && test "$enable_libfuzzer" != "yes" && test "$enable_oss_fuzz" != "yes"; then
+       if test "$enable_pic" != "yes"; then
+           # If we have already enabled -fPIC, then we don't also need to
+           # compile with -fPIE...
+           TOR_CHECK_CFLAGS(-fPIE)
+       fi
+       # ... but we want to link our executables with -pie in any case, since
+       # they're executables, not a library.
        TOR_CHECK_LDFLAGS(-pie, "$all_ldflags_for_check", "$all_libs_for_check")
     fi
     TOR_TRY_COMPILE_WITH_CFLAGS(-fwrapv, also_link, CFLAGS_FWRAPV="-fwrapv", true)
 fi
 
-if test "x$enable_expensive_hardening" = "xyes"; then
+if test "$fragile_hardening" = "yes"; then
     TOR_TRY_COMPILE_WITH_CFLAGS(-ftrapv, also_link, CFLAGS_FTRAPV="-ftrapv", true)
    if test "$tor_cv_cflags__ftrapv" = "yes" && test "$tor_can_link__ftrapv" != "yes"; then
       AC_MSG_WARN([The compiler supports -ftrapv, but for some reason I was not able to link with -ftrapv. Are you missing run-time support? Run-time hardening will not work as well as it should.])
    fi
 
    if test "$tor_cv_cflags__ftrapv" != "yes"; then
-     AC_MSG_ERROR([You requested expensive hardening, but the compiler does not seem to support -ftrapv.])
+     AC_MSG_ERROR([You requested fragile hardening, but the compiler does not seem to support -ftrapv.])
    fi
 
    TOR_TRY_COMPILE_WITH_CFLAGS([-fsanitize=address], also_link, CFLAGS_ASAN="-fsanitize=address", true)
     if test "$tor_cv_cflags__fsanitize_address" = "yes" && test "$tor_can_link__fsanitize_address" != "yes"; then
-      AC_MSG_ERROR([The compiler supports -fsanitize=address, but for some reason I was not able to link when using it. Are you missing run-time support? With GCC you need libubsan.so, and with Clang you need libclang_rt.ubsan*])
+      AC_MSG_ERROR([The compiler supports -fsanitize=address, but for some reason I was not able to link when using it. Are you missing run-time support? With GCC you need libubsan.*, and with Clang you need libclang_rt.ubsan*])
     fi
 
    TOR_TRY_COMPILE_WITH_CFLAGS([-fsanitize=undefined], also_link, CFLAGS_UBSAN="-fsanitize=undefined", true)
     if test "$tor_cv_cflags__fsanitize_address" = "yes" && test "$tor_can_link__fsanitize_address" != "yes"; then
-      AC_MSG_ERROR([The compiler supports -fsanitize=undefined, but for some reason I was not able to link when using it. Are you missing run-time support? With GCC you need libasan.so, and with Clang you need libclang_rt.ubsan*])
+      AC_MSG_ERROR([The compiler supports -fsanitize=undefined, but for some reason I was not able to link when using it. Are you missing run-time support? With GCC you need libasan.*, and with Clang you need libclang_rt.ubsan*])
     fi
 
 TOR_CHECK_CFLAGS([-fno-omit-frame-pointer])
 fi
 
+dnl Find the correct libraries to add in order to use the sanitizers.
+dnl
+dnl When building Rust, Cargo will run the linker with the -nodefaultlibs
+dnl option, which will prevent the compiler from linking the sanitizer
+dnl libraries it needs.  We need to specify them manually.
+dnl
+dnl What's more, we need to specify them in a linker script rather than
+dnl from build.rs: these options aren't allowed in the cargo:rustc-flags
+dnl variable.
+RUST_LINKER_OPTIONS=""
+if test "x$have_clang" = "xyes"; then
+	if test "x$CFLAGS_ASAN" != "x"; then
+		RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS -Clink-arg=$CFLAGS_ASAN -Cdefault-linker-libraries"
+	fi
+	if test "x$CFLAGS_UBSAN" != "x"; then
+        	RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS -Clink-arg=$CFLAGS_UBSAN -Cdefault-linker-libraries"
+	fi
+else
+	if test "x$CFLAGS_ASAN" != "x"; then
+		RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS -Clink-arg=-fsanitize=address -Cdefault-linker-libraries"
+	fi
+	if test "x$CFLAGS_UBSAN" != "x"; then
+        	RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS -Clink-arg=-fsanitize=undefined -Cdefault-linker-libraries"
+	fi
+fi
+AC_SUBST(RUST_LINKER_OPTIONS)
+
 CFLAGS_BUGTRAP="$CFLAGS_FTRAPV $CFLAGS_ASAN $CFLAGS_UBSAN"
 CFLAGS_CONSTTIME="$CFLAGS_FWRAPV"
 
@@ -867,7 +1292,7 @@ saved_CFLAGS="$CFLAGS"
 TOR_CHECK_CFLAGS(-fomit-frame-pointer)
 F_OMIT_FRAME_POINTER=''
 if test "$saved_CFLAGS" != "$CFLAGS"; then
-  if test "x$enable_expensive_hardening" != "xyes"; then
+  if test "$fragile_hardening" = "yes"; then
     F_OMIT_FRAME_POINTER='-fomit-frame-pointer'
   fi
 fi
@@ -1005,59 +1430,58 @@ AC_SUBST(CURVE25519_LIBS)
 dnl Make sure to enable support for large off_t if available.
 AC_SYS_LARGEFILE
 
-AC_CHECK_HEADERS([assert.h \
-                  errno.h \
-                  fcntl.h \
-                  signal.h \
-                  string.h \
-                  sys/capability.h \
-                  sys/fcntl.h \
-                  sys/stat.h \
-                  sys/time.h \
-                  sys/types.h \
-                  time.h \
-                  unistd.h \
-                  arpa/inet.h \
-                  crt_externs.h \
-                  execinfo.h \
-                  gnu/libc-version.h \
-                  grp.h \
-                  ifaddrs.h \
-                  inttypes.h \
-                  limits.h \
-                  linux/types.h \
-                  machine/limits.h \
-                  malloc.h \
-                  malloc/malloc.h \
-                  malloc_np.h \
-                  netdb.h \
-                  netinet/in.h \
-                  netinet/in6.h \
-                  pwd.h \
-                  readpassphrase.h \
-                  stdint.h \
-                  sys/eventfd.h \
-                  sys/file.h \
-                  sys/ioctl.h \
-                  sys/limits.h \
-                  sys/mman.h \
-                  sys/param.h \
-                  sys/prctl.h \
+AC_CHECK_HEADERS([errno.h \
+		  fcntl.h \
+		  signal.h \
+		  string.h \
+		  sys/capability.h \
+		  sys/fcntl.h \
+		  sys/stat.h \
+		  sys/time.h \
+		  sys/types.h \
+		  time.h \
+		  unistd.h \
+		  arpa/inet.h \
+		  crt_externs.h \
+		  execinfo.h \
+		  gnu/libc-version.h \
+		  grp.h \
+		  ifaddrs.h \
+		  inttypes.h \
+		  limits.h \
+		  linux/types.h \
+		  machine/limits.h \
+		  malloc.h \
+		  malloc/malloc.h \
+		  malloc_np.h \
+		  netdb.h \
+		  netinet/in.h \
+		  netinet/in6.h \
+		  pwd.h \
+		  readpassphrase.h \
+		  stdatomic.h \
+		  sys/eventfd.h \
+		  sys/file.h \
+		  sys/ioctl.h \
+		  sys/limits.h \
+		  sys/mman.h \
+		  sys/param.h \
+		  sys/prctl.h \
 		  sys/random.h \
-                  sys/resource.h \
-                  sys/select.h \
-                  sys/socket.h \
-                  sys/statvfs.h \
-                  sys/syscall.h \
-                  sys/sysctl.h \
-                  sys/syslimits.h \
-                  sys/time.h \
-                  sys/types.h \
-                  sys/un.h \
-                  sys/utime.h \
-                  sys/wait.h \
-                  syslog.h \
-                  utime.h])
+		  sys/resource.h \
+		  sys/select.h \
+		  sys/socket.h \
+		  sys/statvfs.h \
+		  sys/syscall.h \
+		  sys/sysctl.h \
+		  sys/syslimits.h \
+		  sys/time.h \
+		  sys/types.h \
+		  sys/un.h \
+		  sys/utime.h \
+		  sys/wait.h \
+		  syslog.h \
+		  utime.h])
 
 AC_CHECK_HEADERS(sys/param.h)
 
@@ -1146,10 +1570,6 @@ if test "x$linux_netfilter_ipv6_ip6_tables" = "x1"; then
 fi
 if test "x$transparent_ok" = "x1"; then
   AC_DEFINE(USE_TRANSPARENT, 1, "Define to enable transparent proxy support")
-  case "$host" in
-    *-*-openbsd* | *-*-bitrig*)
-      AC_DEFINE(OPENBSD, 1, "Define to handle pf on OpenBSD properly") ;;
-  esac
 else
   AC_MSG_NOTICE([Transparent proxy support enabled, but missing headers.])
 fi
@@ -1162,22 +1582,6 @@ AC_CHECK_MEMBERS([struct timeval.tv_sec], , ,
 #include <sys/time.h>
 #endif])
 
-dnl In case we aren't given a working stdint.h, we'll need to grow our own.
-dnl Watch out.
-
-AC_CHECK_SIZEOF(int8_t)
-AC_CHECK_SIZEOF(int16_t)
-AC_CHECK_SIZEOF(int32_t)
-AC_CHECK_SIZEOF(int64_t)
-AC_CHECK_SIZEOF(uint8_t)
-AC_CHECK_SIZEOF(uint16_t)
-AC_CHECK_SIZEOF(uint32_t)
-AC_CHECK_SIZEOF(uint64_t)
-AC_CHECK_SIZEOF(intptr_t)
-AC_CHECK_SIZEOF(uintptr_t)
-
-dnl AC_CHECK_TYPES([int8_t, int16_t, int32_t, int64_t, uint8_t, uint16_t, uint32_t, uint64_t, intptr_t, uintptr_t])
-
 AC_CHECK_SIZEOF(char)
 AC_CHECK_SIZEOF(short)
 AC_CHECK_SIZEOF(int)
@@ -1293,6 +1697,26 @@ AC_CHECK_SIZEOF(socklen_t, , [AC_INCLUDES_DEFAULT()
 
 AC_CHECK_SIZEOF(cell_t)
 
+# Let's see if stdatomic works. (There are some debian clangs that screw it
+# up; see Tor bug #26779 and debian bug 903709.)
+AC_CACHE_CHECK([whether C11 stdatomic.h actually works],
+               tor_cv_stdatomic_works,
+[AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
+#include <stdatomic.h>
+struct x { atomic_size_t y; };
+void try_atomic_init(struct x *xx)
+{
+  atomic_init(&xx->y, 99);
+  atomic_fetch_add(&xx->y, 1);
+}
+]])], [tor_cv_stdatomic_works=yes], [tor_cv_stdatomic_works=no])])
+
+if test "$tor_cv_stdatomic_works" = "yes"; then
+   AC_DEFINE(STDATOMIC_WORKS, 1, [Set to 1 if we can compile a simple stdatomic example.])
+elif test "$ac_cv_header_stdatomic_h" = "yes"; then
+   AC_MSG_WARN([Your compiler provides the stdatomic.h header, but it doesn't seem to work.  I'll pretend it isn't there. If you are using Clang on Debian, maybe this is because of https://bugs.debian.org/903709 ])
+fi
+
 # Now make sure that NULL can be represented as zero bytes.
 AC_CACHE_CHECK([whether memset(0) sets pointers to NULL], tor_cv_null_is_zero,
 [AC_RUN_IFELSE([AC_LANG_SOURCE(
@@ -1404,43 +1828,93 @@ if test "$tor_cv_sign_extend" != "no"; then
             [Define to 1 iff right-shifting a negative value performs sign-extension])
 fi
 
-# Whether we should use the dmalloc memory allocation debugging library.
-AC_MSG_CHECKING(whether to use dmalloc (debug memory allocation library))
-AC_ARG_WITH(dmalloc,
-AS_HELP_STRING(--with-dmalloc, [use debug memory allocation library]),
-[if [[ "$withval" = "yes" ]]; then
-  dmalloc=1
-  AC_MSG_RESULT(yes)
-else
-  dmalloc=1
-  AC_MSG_RESULT(no)
-fi], [ dmalloc=0; AC_MSG_RESULT(no) ]
-)
+# Is uint8_t the same type as unsigned char?
+AC_CACHE_CHECK([whether uint8_t is the same type as unsigned char], tor_cv_uint8_uchar,
+[AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
+#include <stdint.h>
+extern uint8_t c;
+unsigned char c;]])],
+       [tor_cv_uint8_uchar=yes],
+       [tor_cv_uint8_uchar=no],
+       [tor_cv_uint8_uchar=cross])])
+
+if test "$tor_cv_uint8_uchar" = "cross"; then
+  AC_MSG_NOTICE([Cross-compiling: we'll assume that uint8_t is the same type as unsigned char])
+fi
 
-if [[ $dmalloc -eq 1 ]]; then
-  AC_CHECK_HEADERS(dmalloc.h, , AC_MSG_ERROR(dmalloc header file not found. Do you have the development files for dmalloc installed?))
-  AC_SEARCH_LIBS(dmalloc_malloc, [dmallocth dmalloc], , AC_MSG_ERROR(Libdmalloc library not found. If you enable it you better have it installed.))
-  AC_DEFINE(USE_DMALLOC, 1, [Debug memory allocation library])
-  AC_CHECK_FUNCS(dmalloc_strdup dmalloc_strndup)
+if test "$tor_cv_uint8_uchar" = "no"; then
+  AC_MSG_ERROR([We assume that uint8_t is the same type as unsigned char, but your compiler disagrees.])
 fi
 
 AC_ARG_WITH(tcmalloc,
-AS_HELP_STRING(--with-tcmalloc, [use tcmalloc memory allocation library]),
+AS_HELP_STRING(--with-tcmalloc, [use tcmalloc memory allocation library. Deprecated; see --with-malloc]),
 [ tcmalloc=yes ], [ tcmalloc=no ])
 
-if test "x$tcmalloc" = "xyes"; then
-   LDFLAGS="-ltcmalloc $LDFLAGS"
-fi
+default_malloc=system
 
-using_custom_malloc=no
-if test "x$enable_openbsd_malloc" = "xyes"; then
-   using_custom_malloc=yes
+if test "x$enable_openbsd_malloc" = "xyes" ; then
+  AC_MSG_NOTICE([The --enable-openbsd-malloc argument is deprecated; use --with-malloc=openbsd instead.])
+  default_malloc=openbsd
 fi
+
 if test "x$tcmalloc" = "xyes"; then
-   using_custom_malloc=yes
+  AC_MSG_NOTICE([The --with-tcmalloc argument is deprecated; use --with-malloc=tcmalloc instead.])
+  default_malloc=tcmalloc
 fi
-if test "$using_custom_malloc" = "no"; then
-   AC_CHECK_FUNCS(mallinfo)
+
+AC_ARG_WITH(malloc,
+   AS_HELP_STRING([--with-malloc=[system,jemalloc,tcmalloc,openbsd]],
+                  [select special malloc implementation [system]]),
+   [ malloc="$with_malloc" ], [ malloc="$default_malloc" ])
+
+AS_CASE([$malloc],
+  [tcmalloc], [
+      PKG_CHECK_MODULES([TCMALLOC],
+			[libtcmalloc],
+			have_tcmalloc=yes,
+			have_tcmalloc=no)
+
+      if test "x$have_tcmalloc" = "xno" ; then
+          AC_MSG_ERROR([Unable to find tcmalloc requested by --with-malloc.])
+      fi
+
+      CFLAGS="$CFLAGS $TCMALLOC_CFLAGS"
+      LIBS="$TCMALLOC_LIBS $LIBS"
+  ],
+
+  [jemalloc], [
+      PKG_CHECK_MODULES([JEMALLOC],
+		        [jemalloc],
+			have_jemalloc=yes,
+			have_jemalloc=no)
+
+      if test "x$have_tcmalloc" = "xno" ; then
+          AC_MSG_ERROR([Unable to find jemalloc requested by --with-malloc.])
+      fi
+
+      CFLAGS="$CFLAGS $JEMALLOC_CFLAGS"
+      LIBS="$JEMALLOC_LIBS $LIBS"
+      using_custom_malloc=yes
+  ],
+
+  [openbsd], [
+    AC_MSG_WARN([The openbsd malloc port is deprecated in Tor 0.3.5 and will be removed in a future version.])
+    enable_openbsd_malloc=yes
+  ],
+
+  [system], [
+     # handle this later, including the jemalloc fallback
+    AC_CHECK_FUNCS(mallinfo)
+  ],
+
+  [AC_MSG_ERROR([--with-malloc=`$with_malloc' not supported, see --help])
+])
+
+AM_CONDITIONAL(USE_OPENBSD_MALLOC, test "x$enable_openbsd_malloc" = "xyes")
+
+if test "$malloc" != "system"; then
+  # Tell the C compiler not to use the system allocator functions.
+  TOR_CHECK_CFLAGS([-fno-builtin-malloc -fno-builtin-realloc -fno-builtin-calloc -fno-builtin-free])
 fi
 if test "$using_custom_malloc" = "yes"; then
   # Tell the C compiler not to use the system allocator functions.
@@ -1476,9 +1950,9 @@ AC_CHECK_FUNC(gethostbyname_r, [
   AC_MSG_CHECKING([how many arguments gethostbyname_r() wants])
   OLD_CFLAGS=$CFLAGS
   CFLAGS="$CFLAGS $MY_CPPFLAGS $MY_THREAD_CPPFLAGS $MY_CFLAGS"
-  AC_COMPILE_IFELSE([AC_LANG_PROGRAM([
+  AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #include <netdb.h>
-  ], [[
+  ]], [[
     char *cp1, *cp2;
     struct hostent *h1, *h2;
     int i1, i2;
@@ -1489,27 +1963,27 @@ AC_CHECK_FUNC(gethostbyname_r, [
      [Define this if gethostbyname_r takes 6 arguments])
     AC_MSG_RESULT(6)
   ], [
-    AC_TRY_COMPILE([
+    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #include <netdb.h>
-    ], [
+    ]], [[
       char *cp1, *cp2;
       struct hostent *h1;
       int i1, i2;
       (void)gethostbyname_r(cp1,h1,cp2,i1,&i2);
-    ], [
+    ]])], [
       AC_DEFINE(HAVE_GETHOSTBYNAME_R)
       AC_DEFINE(HAVE_GETHOSTBYNAME_R_5_ARG, 1,
         [Define this if gethostbyname_r takes 5 arguments])
       AC_MSG_RESULT(5)
    ], [
-      AC_TRY_COMPILE([
+      AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #include <netdb.h>
-     ], [
+     ]], [[
        char *cp1;
        struct hostent *h1;
        struct hostent_data hd;
        (void) gethostbyname_r(cp1,h1,&hd);
-     ], [
+     ]])], [
        AC_DEFINE(HAVE_GETHOSTBYNAME_R)
        AC_DEFINE(HAVE_GETHOSTBYNAME_R_3_ARG, 1,
          [Define this if gethostbyname_r takes 3 arguments])
@@ -1594,6 +2068,12 @@ AC_SUBST(BUILDDIR)
 AH_TEMPLATE([BUILDDIR],[tor's build directory])
 AC_DEFINE_UNQUOTED(BUILDDIR,"$BUILDDIR")
 
+if test "x$SRCDIR" = "x"; then
+  SRCDIR=$(cd "$srcdir"; pwd)
+fi
+AH_TEMPLATE([SRCDIR],[tor's sourcedir directory])
+AC_DEFINE_UNQUOTED(SRCDIR,"$SRCDIR")
+
 if test "x$CONFDIR" = "x"; then
   CONFDIR=`eval echo $sysconfdir/tor`
 fi
@@ -1649,6 +2129,8 @@ case "$host_os" in
     LDFLAGS="$LDFLAGS -dead_strip" ;;
 esac
 
+TOR_WARNING_FLAGS=""
+
 # Add some more warnings which we use in development but not in the
 # released versions.  (Some relevant gcc versions can't handle these.)
 #
@@ -1666,6 +2148,8 @@ if test "x$enable_gcc_warnings_advisory" != "xno"; then
       CFLAGS="$CFLAGS -Wno-system-headers" ;;
   esac
 
+  CFLAGS_NOWARNINGS="$CFLAGS"
+
   # GCC4.3 users once report trouble with -Wstrict-overflow=5.  GCC5 users
   # have it work better.
   # CFLAGS="$CFLAGS -Wstrict-overflow=1"
@@ -1775,7 +2259,6 @@ if test "x$enable_gcc_warnings_advisory" != "xno"; then
      -Winvalid-source-encoding
      -Winvalid-token-paste
      -Wknr-promoted-parameter
-     -Wlanguage-extension-token
      -Wlarge-by-value-copy
      -Wliteral-conversion
      -Wliteral-range
@@ -1801,7 +2284,7 @@ if test "x$enable_gcc_warnings_advisory" != "xno"; then
      -Wnon-literal-null-conversion
      -Wnon-pod-varargs
      -Wnonportable-cfstrings
-     -Wnormalized=id
+     -Wnormalized=nfkc
      -Wnull-arithmetic
      -Wnull-character
      -Wnull-conversion
@@ -1882,7 +2365,9 @@ if test "x$enable_gcc_warnings_advisory" != "xno"; then
      -Wvisibility
      -Wvla-extension
      -Wzero-length-array
-  ], [ TOR_CHECK_CFLAGS([warning_flag]) ])
+  ], [ TOR_TRY_COMPILE_WITH_CFLAGS(warning_flag, [],
+              [TOR_WARNING_FLAGS="$TOR_WARNING_FLAGS warning_flag" CFLAGS="$CFLAGS warning_flag"], true)
+     ])
 
 dnl    We should re-enable this in some later version.  Clang doesn't
 dnl    mind, but it causes trouble with GCC.
@@ -1896,16 +2381,19 @@ dnl     -Wthread-safety-attributes
 dnl     -Wthread-safety-beta
 dnl     -Wthread-safety-precise
 
-  CFLAGS="$CFLAGS -W -Wfloat-equal -Wundef -Wpointer-arith"
-  CFLAGS="$CFLAGS -Wstrict-prototypes -Wmissing-prototypes -Wwrite-strings"
-  CFLAGS="$CFLAGS -Wredundant-decls -Wchar-subscripts -Wcomment -Wformat=2"
-  CFLAGS="$CFLAGS -Wwrite-strings"
-  CFLAGS="$CFLAGS -Wnested-externs -Wbad-function-cast -Wswitch-enum"
-  CFLAGS="$CFLAGS -Waggregate-return -Wpacked -Wunused"
-  CFLAGS="$CFLAGS -Wunused-parameter "
+  W_FLAGS="$W_FLAGS -W -Wfloat-equal -Wundef -Wpointer-arith"
+  W_FLAGS="$W_FLAGS -Wstrict-prototypes -Wmissing-prototypes -Wwrite-strings"
+  W_FLAGS="$W_FLAGS -Wredundant-decls -Wchar-subscripts -Wcomment -Wformat=2"
+  W_FLAGS="$W_FLAGS -Wwrite-strings"
+  W_FLAGS="$W_FLAGS -Wnested-externs -Wbad-function-cast -Wswitch-enum"
+  W_FLAGS="$W_FLAGS -Waggregate-return -Wpacked -Wunused"
+  W_FLAGS="$W_FLAGS -Wunused-parameter "
   # These interfere with building main() { return 0; }, which autoconf
   # likes to use as its default program.
-  CFLAGS="$CFLAGS -Wold-style-definition -Wmissing-declarations"
+  W_FLAGS="$W_FLAGS -Wold-style-definition -Wmissing-declarations"
+
+  TOR_WARNING_FLAGS="$TOR_WARNING_FLAGS $W_FLAGS"
+  CFLAGS="$CFLAGS $W_FLAGS"
 
   if test "$tor_cv_cflags__Wnull_dereference" = "yes"; then
     AC_DEFINE([HAVE_CFLAG_WNULL_DEREFERENCE], 1, [True if we have -Wnull-dereference])
@@ -1913,6 +2401,11 @@ dnl     -Wthread-safety-precise
   if test "$tor_cv_cflags__Woverlength_strings" = "yes"; then
     AC_DEFINE([HAVE_CFLAG_WOVERLENGTH_STRINGS], 1, [True if we have -Woverlength-strings])
   fi
+  if test "$tor_cv_cflags__warn_unused_const_variable_2" = "yes"; then
+    AC_DEFINE([HAVE_CFLAG_WUNUSED_CONST_VARIABLE], 1, [True if we have -Wunused-const-variable])
+  fi
+
+  CFLAGS="$CFLAGS_NOWARNINGS"
 
   if test "x$enable_fatal_warnings" = "xyes"; then
     # I'd like to use TOR_CHECK_CFLAGS here, but I can't, since the
@@ -1922,6 +2415,14 @@ dnl     -Wthread-safety-precise
 
 fi
 
+AC_SUBST(TOR_WARNING_FLAGS)
+
+echo "$TOR_WARNING_FLAGS">warning_flags
+
+TOR_TRY_COMPILE_WITH_CFLAGS([@warning_flags], [],
+                            CFLAGS="$CFLAGS @warning_flags",
+                            CFLAGS="$CFLAGS $TOR_WARNING_FLAGS")
+
 if test "$enable_coverage" = "yes" && test "$have_clang" = "no"; then
    case "$host_os" in
     darwin*)
@@ -1932,17 +2433,20 @@ fi
 CPPFLAGS="$CPPFLAGS $TOR_CPPFLAGS_libevent $TOR_CPPFLAGS_openssl $TOR_CPPFLAGS_zlib"
 
 AC_CONFIG_FILES([
-        Doxyfile
-        Makefile
-        contrib/dist/suse/tor.sh
-        contrib/operator-tools/tor.logrotate
-        contrib/dist/tor.sh
-        contrib/dist/torctl
-        contrib/dist/tor.service
-        src/config/torrc.sample
-        src/config/torrc.minimal
-        scripts/maint/checkOptionDocs.pl
-        scripts/maint/updateVersions.pl
+	Doxyfile
+	Makefile
+	config.rust
+	contrib/dist/suse/tor.sh
+	contrib/operator-tools/tor.logrotate
+	contrib/dist/tor.sh
+	contrib/dist/torctl
+	contrib/dist/tor.service
+	src/config/torrc.sample
+	src/config/torrc.minimal
+	src/rust/.cargo/config
+	scripts/maint/checkOptionDocs.pl
+	scripts/maint/updateVersions.pl
+        warning_flags
 ])
 
 if test "x$asciidoc" = "xtrue" && test "$ASCIIDOC" = "none"; then
@@ -1963,4 +2467,19 @@ if test "x$asciidoc" = "xtrue" && test "$ASCIIDOC" = "none"; then
   done
 fi
 
+if test "$fragile_hardening" = "yes"; then
+  AC_MSG_WARN([
+
+============
+Warning!  Building Tor with --enable-fragile-hardening (also known as
+--enable-expensive-hardening) makes some kinds of attacks harder, but makes
+other kinds of attacks easier. A Tor instance build with this option will be
+somewhat less vulnerable to remote code execution, arithmetic overflow, or
+out-of-bounds read/writes... but at the cost of becoming more vulnerable to
+denial of service attacks. For more information, see
+https://trac.torproject.org/projects/tor/wiki/doc/TorFragileHardening
+============
+  ])
+fi
+
 AC_OUTPUT
diff --git a/contrib/dist/tor.service.in b/contrib/dist/tor.service.in
index 9c1a255b2e..e857a8664e 100644
--- a/contrib/dist/tor.service.in
+++ b/contrib/dist/tor.service.in
@@ -15,7 +15,7 @@ ExecStartPre=@BINDIR@/tor -f @CONFDIR@/torrc --verify-config
 ExecStart=@BINDIR@/tor -f @CONFDIR@/torrc
 ExecReload=/bin/kill -HUP ${MAINPID}
 KillSignal=SIGINT
-TimeoutSec=30
+TimeoutSec=60
 Restart=on-failure
 WatchdogSec=1m
 LimitNOFILE=32768
diff --git a/contrib/include.am b/contrib/include.am
index 5d5f216490..a23e82d6da 100644
--- a/contrib/include.am
+++ b/contrib/include.am
@@ -10,7 +10,6 @@ EXTRA_DIST+= \
 	contrib/operator-tools/linux-tor-prio.sh	\
 	contrib/operator-tools/tor-exit-notice.html	\
 	contrib/or-tools/exitlist			\
-	contrib/win32build/package_nsis-mingw.sh	\
 	contrib/win32build/tor-mingw.nsi.in		\
 	contrib/win32build/tor.ico			\
 	contrib/win32build/tor.nsi.in
diff --git a/contrib/operator-tools/linux-tor-prio.sh b/contrib/operator-tools/linux-tor-prio.sh
index ea9e0ddaa5..30ea5fc659 100644
--- a/contrib/operator-tools/linux-tor-prio.sh
+++ b/contrib/operator-tools/linux-tor-prio.sh
@@ -87,7 +87,7 @@ RATE_UP=5000
 # machine does any other network activity. That is not very fun.
 RATE_UP_TOR=1500
 
-# RATE_UP_TOR_CEIL is the maximum rate allowed for all Tor trafic in
+# RATE_UP_TOR_CEIL is the maximum rate allowed for all Tor traffic in
 # kbits/sec.
 RATE_UP_TOR_CEIL=5000
 
diff --git a/contrib/win32build/package_nsis-mingw.sh b/contrib/win32build/package_nsis-mingw.sh
deleted file mode 100644
index cae862b919..0000000000
--- a/contrib/win32build/package_nsis-mingw.sh
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/bin/sh
-#
-# ===============================================================================
-# package_nsis-ming.sh is distributed under this license:
-
-# Copyright (c) 2006-2007 Andrew Lewman
-# Copyright (c) 2008 The Tor Project, Inc.
-
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-
-#     * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-
-#     * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-
-#     * Neither the names of the copyright owners nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-# ===============================================================================
-
-# Script to package a Tor installer on win32.  This script assumes that
-# you have already built Tor, that you are running msys/mingw, and that
-# you know what you are doing.
-
-# Start in the tor source directory after you've compiled tor.exe
-# This means start as ./contrib/win32build/package_nsis-mingw.sh
-
-rm -rf win_tmp
-mkdir win_tmp
-mkdir win_tmp/bin
-mkdir win_tmp/contrib
-mkdir win_tmp/doc
-mkdir win_tmp/doc/spec
-mkdir win_tmp/doc/design-paper
-mkdir win_tmp/doc/contrib
-mkdir win_tmp/src
-mkdir win_tmp/src/config
-mkdir win_tmp/tmp
-
-cp src/or/tor.exe win_tmp/bin/
-cp src/tools/tor-resolve.exe win_tmp/bin/
-cp contrib/win32build/tor.ico win_tmp/bin/
-cp src/config/geoip win_tmp/bin/
-strip win_tmp/bin/*.exe
-
-# There is no man2html in mingw.  
-# Maybe we should add this into make dist instead.
-# One has to do this manually and cp it do the tor-source/doc dir
-#man2html doc/tor.1.in > win_tmp/tmp/tor-reference.html
-#man2html doc/tor-resolve.1 > win_tmp/tmp/tor-resolve.html
-
-clean_newlines() {
-    perl -pe 's/^\n$/\r\n/mg; s/([^\r])\n$/\1\r\n/mg;' $1 >$2
-}
-
-clean_localstatedir() {
-    perl -pe 's/^\n$/\r\n/mg; s/([^\r])\n$/\1\r\n/mg; s{\@LOCALSTATEDIR\@/(lib|log)/tor/}{C:\\Documents and Settings\\Application Data\\Tor\\}' $1 >$2
-}
-
-for fn in address-spec.txt bridges-spec.txt control-spec.txt dir-spec.txt path-spec.txt rend-spec.txt socks-extensions.txt tor-spec.txt version-spec.txt; do
-    clean_newlines doc/spec/$fn win_tmp/doc/spec/$fn
-done
-
-for fn in HACKING tor-gencert.html tor.html torify.html tor-resolve.html; do
-    clean_newlines doc/$fn win_tmp/doc/$fn
-done
-
-for fn in README ChangeLog LICENSE; do
-    clean_newlines $fn win_tmp/$fn
-done
-
-clean_localstatedir src/config/torrc.sample.in win_tmp/src/config/torrc.sample
-
-cp contrib/win32build/tor-mingw.nsi.in win_tmp/contrib/
-
-cd win_tmp
-makensis.exe contrib/tor-mingw.nsi.in
-
diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in
index 452720516e..9eeb328223 100644
--- a/contrib/win32build/tor-mingw.nsi.in
+++ b/contrib/win32build/tor-mingw.nsi.in
@@ -8,7 +8,7 @@
 !include "LogicLib.nsh"
 !include "FileFunc.nsh"
 !insertmacro GetParameters
-!define VERSION "0.2.9.17-dev"
+!define VERSION "0.3.5.9-dev"
 !define INSTALLER "tor-${VERSION}-win32.exe"
 !define WEBSITE "https://www.torproject.org/"
 !define LICENSE "LICENSE"
diff --git a/doc/HACKING/CodeStructure.md b/doc/HACKING/CodeStructure.md
new file mode 100644
index 0000000000..736d6cd484
--- /dev/null
+++ b/doc/HACKING/CodeStructure.md
@@ -0,0 +1,129 @@
+
+TODO: revise this to talk about how things are, rather than how things
+have changed.
+
+TODO: Make this into good markdown.
+
+
+
+For quite a while now, the program "tor" has been built from source
+code in just two directories: src/common and src/or.
+
+This has become more-or-less untenable, for a few reasons -- most
+notably of which is that it has led our code to become more
+spaghetti-ish than I can endorse with a clean conscience.
+
+So to fix that, we've gone and done a huge code movement in our git
+master branch, which will land in a release once Tor 0.3.5.1-alpha is
+out.
+
+Here's what we did:
+
+  * src/common has been turned into a set of static libraries.  These
+all live in the "src/lib/*" directories.  The dependencies between
+these libraries should have no cycles.  The libraries are:
+
+    arch -- Headers to handle architectural differences
+    cc -- headers to handle differences among compilers
+    compress -- wraps zlib, zstd, lzma
+    container -- high-level container types
+    crypt_ops -- Cryptographic operations. Planning to split this into
+a higher and lower level library
+    ctime -- Operations that need to run in constant-time. (Properly,
+data-invariant time)
+    defs -- miscelaneous definitions needed throughout Tor.
+    encoding -- transforming one data type into another, and various
+data types into strings.
+    err -- lowest-level error handling, in cases where we can't use
+the logs because something that the logging system needs has broken.
+    evloop -- Generic event-loop handling logic
+    fdio -- Low-level IO wrapper functions for file descriptors.
+    fs -- Operations on the filesystem
+    intmath -- low-level integer math and misc bit-twiddling hacks
+    lock -- low-level locking code
+    log -- Tor's logging module.  This library sits roughly halfway up
+the library dependency diagram, since everything it depends on has to
+be carefully crafted to *not* log.
+    malloc -- Low-level wrappers for the platform memory allocation functions.
+    math -- Higher-level mathematical functions, and floating-point math
+    memarea -- An arena allocator
+    meminfo -- Functions for querying the current process's memory
+status and resources
+    net -- Networking compatibility and convenience code
+    osinfo -- Querying information about the operating system
+    process -- Launching and querying the status of other processes
+    sandbox -- Backend for the linux seccomp2 sandbox
+    smartlist_core -- The lowest-level of the smartlist_t data type.
+Separated from the rest of the containers library because the logging
+subsystem depends on it.
+    string -- Compatibility and convenience functions for manipulating
+C strings.
+    term -- Terminal-related functions (currently limited to a getpass
+function).
+    testsupport -- Macros for mocking, unit tests, etc.
+    thread -- Higher-level thread compatibility code
+    time -- Higher-level time management code, including format
+conversions and monotonic time
+    tls -- Our wrapper around our TLS library
+    trace -- Formerly src/trace -- a generic event tracing API
+    wallclock -- Low-level time code, used by the log module.
+
+  * To ensure that the dependency graph in src/common remains under
+control, there is a tool that you can run called "make
+check-includes".  It verifies that each module in Tor only includes
+the headers that it is permitted to include, using a per-directory
+".may_include" file.
+
+  * The src/or/or.h header has been split into numerous smaller
+headers.  Notably, many important structures are now declared in a
+header called foo_st.h, where "foo" is the name of the structure.
+
+  * The src/or directory, which had most of Tor's code, had been split
+up into several directories.  This is still a work in progress:  This
+code has not itself been refactored, and its dependency graph is still
+a tangled web.  I hope we'll be working on that over the coming
+releases, but it will take a while to do.
+
+    The new top-level source directories are:
+
+     src/core -- Code necessary to actually perform or use onion routing.
+     src/feature -- Code used only by some onion routing
+configurations, or only for a special purpose.
+     src/app -- Top-level code to run, invoke, and configure the
+lower-level code
+
+   The new second-level source directories are:
+     src/core/crypto -- High-level cryptographic protocols used in Tor
+     src/core/mainloop -- Tor's event loop, connection-handling, and
+traffic-routing code.
+     src/core/or -- Parts related to handling onion routing itself
+     src/core/proto -- support for encoding and decoding different
+wire protocols
+
+     src/feature/api -- Support for making Tor embeddable
+     src/feature/client -- Functionality which only Tor clients need
+     src/feature/control -- Controller implementation
+     src/feature/dirauth -- Directory authority
+     src/feature/dircache -- Directory cache
+     src/feature/dirclient -- Directory client
+     src/feature/dircommon -- Shared code between the other directory modules
+     src/feature/hibernate -- Hibernating when Tor is out of bandwidth
+or shutting down
+     src/feature/hs -- v3 onion service implementation
+     src/feature/hs_common -- shared code between both onion service
+implementations
+     src/feature/nodelist -- storing and accessing the list of relays on
+the network.
+     src/feature/relay -- code that only relay servers and exit servers need.
+     src/feature/rend -- v2 onion service implementation
+     src/feature/stats -- statistics and history
+
+     src/app/config -- configuration and state for Tor
+     src/app/main -- Top-level functions to invoke the rest or Tor.
+
+  * The "tor" executable is now built in src/app/tor rather than src/or/tor.
+
+  * There are more static libraries than before that you need to build
+into your application if you want to embed Tor.  Rather than
+maintaining this list yourself, I recommend that you run "make
+show-libs" to have Tor emit a list of what you need to link.
diff --git a/doc/HACKING/CodingStandards.md b/doc/HACKING/CodingStandards.md
index f1c65850a4..4f229348e4 100644
--- a/doc/HACKING/CodingStandards.md
+++ b/doc/HACKING/CodingStandards.md
@@ -4,9 +4,10 @@ Coding conventions for Tor
 tl;dr:
 
    - Run configure with `--enable-fatal-warnings`
-   - Run `make check-spaces` to catch whitespace errors
    - Document your functions
    - Write unit tests
+   - Run `make check` before submitting a patch
+   - Run `make distcheck` if you have made changes to build system components
    - Add a file in `changes` for your branch.
 
 Patch checklist
@@ -22,13 +23,42 @@ preference)
 Did you remember...
 
    - To build your code while configured with `--enable-fatal-warnings`?
-   - To run `make check-spaces` on your code?
    - To run `make check-docs` to see whether all new options are on
      the manpage?
    - To write unit tests, as possible?
+   - To run `make test-full` to test against all unit and integration tests (or
+     `make test-full-online` if you have a working connection to the internet)?
+   - To test that the distribution will actually work via `make distcheck`?
    - To base your code on the appropriate branch?
    - To include a file in the `changes` directory as appropriate?
 
+If you are submitting a major patch or new feature, or want to in the future...
+
+   - Set up Chutney and Stem, see HACKING/WritingTests.md
+   - Run `make test-full` to test against all unit and integration tests.
+
+If you have changed build system components:
+   - Please run `make distcheck`
+   - For example, if you have changed Makefiles, autoconf files, or anything
+     else that affects the build system.
+
+License issues
+==============
+
+Tor is distributed under the license terms in the LICENSE -- in
+brief, the "3-clause BSD license".  If you send us code to
+distribute with Tor, it needs to be code that we can distribute
+under those terms.  Please don't send us patches unless you agree
+to allow this.
+
+Some compatible licenses include:
+
+  - 3-clause BSD
+  - 2-clause BSD
+  - CC0 Public Domain Dedication
+
+
+
 How we use Git branches
 =======================
 
@@ -49,8 +79,17 @@ before it gets merged into maint, but that's rare.
 
 If you're working on a bugfix for a bug that occurs in a particular version,
 base your bugfix branch on the "maint" branch for the first supported series
-that has that bug.  (As of June 2013, we're supporting 0.2.3 and later.) If
-you're working on a new feature, base it on the master branch.
+that has that bug.  (As of June 2013, we're supporting 0.2.3 and later.)
+
+If you're working on a new feature, base it on the master branch. If you're
+working on a new feature and it will take a while to implement and/or you'd
+like to avoid the possibility of unrelated bugs in Tor while you're
+implementing your feature, consider branching off of the latest maint- branch.
+_Never_ branch off a relase- branch. Don't branch off a tag either: they come
+from release branches. Doing so will likely produce a nightmare of merge
+conflicts in the ChangeLog when it comes time to merge your branch into Tor.
+Best advice: don't try to keep an independent branch forked for more than 6
+months and expect it to merge cleanly. Try to merge pieces early and often.
 
 
 How we log changes
@@ -74,17 +113,34 @@ you can use `git describe --contains <sha1 of commit>`.
 If at all possible, try to create this file in the same commit where you are
 making the change.  Please give it a distinctive name that no other branch will
 use for the lifetime of your change. To verify the format of the changes file,
-you can use `make check-changes`.
+you can use `make check-changes`.  This is run automatically as part of
+`make check` -- if it fails, we must fix it before we release.  These
+checks are implemented in `scripts/maint/lintChanges.py`.
+
+Changes file style guide:
+  * Changes files begin with "  o Header (subheading):".  The header
+    should usually be "Minor/Major bugfixes/features". The subheading is a
+    particular area within Tor.  See the ChangeLog for examples.
+
+  * Make everything terse.
+
+  * Write from the user's point of view: describe the user-visible changes
+    right away.
+
+  * Mention configuration options by name.  If they're rare or unusual,
+    remind people what they're for.
+
+  * Describe changes in the present tense and in the imperative: not past.
+
+  * Every bugfix should have a sentence of the form "Fixes bug 1234; bugfix
+    on 0.1.2.3-alpha", describing what bug was fixed and where it came from.
+
+  * "Relays", not "servers", "nodes", or "Tor relays".
 
 When we go to make a release, we will concatenate all the entries
 in changes to make a draft changelog, and clear the directory. We'll
 then edit the draft changelog into a nice readable format.
 
-To make sure that stuff is in the right format, we use
-scripts/maint/lintChanges.py to check the changes files for
-(superficial) validity.  You can run this script on your own changes
-files!
-
 What needs a changes file?
 
    * A not-exhaustive list: Anything that might change user-visible
@@ -93,6 +149,10 @@ What needs a changes file?
    rewrites.  Anything about which somebody might plausibly wonder "when
    did that happen, and/or why did we do that" 6 months down the line.
 
+What does not need a changes file?
+
+   * Bugfixes for code that hasn't shipped in any released version of Tor
+
 Why use changes files instead of Git commit messages?
 
    * Git commit messages are written for developers, not users, and they
@@ -112,7 +172,6 @@ deviations from our C whitespace style.  Generally, we use:
    - Unix-style line endings
    - K&R-style indentation
    - No space before newlines
-   - A blank line at the end of each file
    - Never more than one blank line in a row
    - Always spaces, never tabs
    - No more than 79-columns per line.
@@ -125,6 +184,9 @@ deviations from our C whitespace style.  Generally, we use:
      `puts (x)`.
    - Function declarations at the start of the line.
 
+If you use an editor that has plugins for editorconfig.org, the file
+`.editorconfig` will help you to conform this coding style.
+
 We try hard to build without warnings everywhere.  In particular, if
 you're using gcc, you should invoke the configure script with the
 option `--enable-fatal-warnings`.  This will tell the compiler
@@ -138,8 +200,8 @@ We have some wrapper functions like `tor_malloc`, `tor_free`, `tor_strdup`, and
 always succeed or exit.)
 
 You can get a full list of the compatibility functions that Tor provides by
-looking through `src/common/util*.h` and `src/common/compat*.h`.  You can see the
-available containers in `src/common/containers*.h`.  You should probably
+looking through `src/lib/*/*.h`.  You can see the
+available containers in `src/lib/containers/*.h`.  You should probably
 familiarize yourself with these modules before you write too much code, or
 else you'll wind up reinventing the wheel.
 
@@ -149,6 +211,97 @@ old C functions.  Use `strlcat`, `strlcpy`, or `tor_snprintf/tor_asprintf` inste
 We don't call `memcmp()` directly.  Use `fast_memeq()`, `fast_memneq()`,
 `tor_memeq()`, or `tor_memneq()` for most purposes.
 
+Also see a longer list of functions to avoid in:
+https://people.torproject.org/~nickm/tor-auto/internal/this-not-that.html
+
+What code can use what other code?
+----------------------------------
+
+We're trying to simplify Tor's structure over time.  In the long run, we want
+Tor to be structured as a set of modules with *no circular dependencies*.
+
+This property is currently provided by the modules in src/lib, but not
+throughout the rest of Tor.  In general, higher-level libraries may use
+lower-level libraries, but never the reverse.
+
+To prevent new circular dependencies from landing, we have a tool that
+you can invoke with `make check-includes`, and which is run
+automatically as part of `make check`.  This tool will verify that, for
+every source directory with a `.may_include` file, no local headers are
+included except those specifically permitted by the `.may_include` file.
+When editing one of these files, please make sure that you are not
+introducing any cycles into Tor's dependency graph.
+
+Floating point math is hard
+---------------------------
+
+Floating point arithmetic as typically implemented by computers is
+very counterintuitive.  Failure to adequately analyze floating point
+usage can result in surprising behavior and even security
+vulnerabilities!
+
+General advice:
+
+   - Don't use floating point.
+   - If you must use floating point, document how the limits of
+     floating point precision and calculation accuracy affect function
+     outputs.
+   - Try to do as much as possible of your calculations using integers
+     (possibly acting as fixed-point numbers) and convert to floating
+     point for display.
+   - If you must send floating point numbers on the wire, serialize
+     them in a platform-independent way.  Tor avoids exchanging
+     floating-point values, but when it does, it uses ASCII numerals,
+     with a decimal point (".").
+   - Binary fractions behave very differently from decimal fractions.
+     Make sure you understand how these differences affect your
+     calculations.
+   - Every floating point arithmetic operation is an opportunity to
+     lose precision, overflow, underflow, or otherwise produce
+     undesired results.  Addition and subtraction tend to be worse
+     than multiplication and division (due to things like catastrophic
+     cancellation).  Try to arrange your calculations to minimize such
+     effects.
+   - Changing the order of operations changes the results of many
+     floating-point calculations.  Be careful when you simplify
+     calculations!  If the order is significant, document it using a
+     code comment.
+   - Comparing most floating point values for equality is unreliable.
+     Avoid using `==`, instead, use `>=` or `<=`.  If you use an
+     epsilon value, make sure it's appropriate for the ranges in
+     question.
+   - Different environments (including compiler flags and per-thread
+     state on a single platform!) can get different results from the
+     same floating point calculations.  This means you can't use
+     floats in anything that needs to be deterministic, like consensus
+     generation.  This also makes reliable unit tests of
+     floating-point outputs hard to write.
+
+For additional useful advice (and a little bit of background), see
+[What Every Programmer Should Know About Floating-Point
+Arithmetic](http://floating-point-gui.de/).
+
+A list of notable (and surprising) facts about floating point
+arithmetic is at [Floating-point
+complexities](https://randomascii.wordpress.com/2012/04/05/floating-point-complexities/).
+Most of that [series of posts on floating
+point](https://randomascii.wordpress.com/category/floating-point/) is
+helpful.
+
+For more detailed (and math-intensive) background, see [What Every
+Computer Scientist Should Know About Floating-Point
+Arithmetic](https://docs.oracle.com/cd/E19957-01/806-3568/ncg_goldberg.html).
+
+Other C conventions
+-------------------
+
+The `a ? b : c` trinary operator only goes inside other expressions;
+don't use it as a replacement for if. (You can ignore this inside macro
+definitions when necessary.)
+
+Assignment operators shouldn't nest inside other expressions.  (You can
+ignore this inside macro definitions when necessary.)
+
 Functions not to write
 ----------------------
 
@@ -210,6 +363,64 @@ end-users that they aren't expected to understand the message (perhaps
 with a string like "internal error"). Option (A) is to be preferred to
 option (B).
 
+Assertions In Tor
+-----------------
+
+Assertions should be used for bug-detection only.  Don't use assertions to
+detect bad user inputs, network errors, resource exhaustion, or similar
+issues.
+
+Tor is always built with assertions enabled, so try to only use
+`tor_assert()` for cases where you are absolutely sure that crashing is the
+least bad option.  Many bugs have been caused by use of `tor_assert()` when
+another kind of check would have been safer.
+
+If you're writing an assertion to test for a bug that you _can_ recover from,
+use `tor_assert_nonfatal()` in place of `tor_assert()`.  If you'd like to
+write a conditional that incorporates a nonfatal assertion, use the `BUG()`
+macro, as in:
+
+	if (BUG(ptr == NULL))
+		return -1;
+
+Allocator conventions
+---------------------
+
+By convention, any tor type with a name like `abc_t` should be allocated
+by a function named `abc_new()`.  This function should never return
+NULL.
+
+Also, a type named `abc_t` should be freed by a function named `abc_free_()`.
+Don't call this `abc_free_()` function directly -- instead, wrap it in a
+macro called `abc_free()`, using the `FREE_AND_NULL` macro:
+
+    void abc_free_(abc_t *obj);
+    #define abc_free(obj) FREE_AND_NULL(abc_t, abc_free_, (obj))
+
+This macro will free the underlying `abc_t` object, and will also set
+the object pointer to NULL.
+
+You should define all `abc_free_()` functions to accept NULL inputs:
+
+    void
+    abc_free_(abc_t *obj)
+    {
+      if (!obj)
+        return;
+      tor_free(obj->name);
+      thing_free(obj->thing);
+      tor_free(obj);
+    }
+
+If you need a free function that takes a `void *` argument (for example,
+to use it as a function callback), define it with a name like
+`abc_free_void()`:
+
+    static void
+    abc_free_void_(void *obj)
+    {
+      abc_free_(obj);
+    }
 
 
 Doxygen comment conventions
diff --git a/doc/HACKING/CodingStandardsRust.md b/doc/HACKING/CodingStandardsRust.md
new file mode 100644
index 0000000000..fc562816db
--- /dev/null
+++ b/doc/HACKING/CodingStandardsRust.md
@@ -0,0 +1,523 @@
+
+ Rust Coding Standards
+=======================
+
+You MUST follow the standards laid out in `.../doc/HACKING/CodingStandards.md`,
+where applicable.
+
+ Module/Crate Declarations
+---------------------------
+
+Each Tor C module which is being rewritten MUST be in its own crate.
+See the structure of `.../src/rust` for examples.
+
+In your crate, you MUST use `lib.rs` ONLY for pulling in external
+crates (e.g. `extern crate libc;`) and exporting public objects from
+other Rust modules (e.g. `pub use mymodule::foo;`).  For example, if
+you create a crate in `.../src/rust/yourcrate`, your Rust code should
+live in `.../src/rust/yourcrate/yourcode.rs` and the public interface
+to it should be exported in `.../src/rust/yourcrate/lib.rs`.
+
+If your code is to be called from Tor C code, you MUST define a safe
+`ffi.rs`.  See the "Safety" section further down for more details.
+
+For example, in a hypothetical `tor_addition` Rust module:
+
+In `.../src/rust/tor_addition/addition.rs`:
+
+    pub fn get_sum(a: i32, b: i32) -> i32 {
+        a + b
+    }
+
+In `.../src/rust/tor_addition/lib.rs`:
+
+    pub use addition::*;
+
+In `.../src/rust/tor_addition/ffi.rs`:
+
+    #[no_mangle]
+    pub extern "C" fn tor_get_sum(a: c_int, b: c_int) -> c_int {
+        get_sum(a, b)
+    }
+
+If your Rust code must call out to parts of Tor's C code, you must
+declare the functions you are calling in the `external` crate, located
+at `.../src/rust/external`.
+
+<!-- XXX get better examples of how to declare these externs, when/how they -->
+<!-- XXX are unsafe, what they are expected to do —isis                     -->
+
+Modules should strive to be below 500 lines (tests excluded). Single
+responsibility and limited dependencies should be a guiding standard.
+
+If you have any external modules as dependencies (e.g. `extern crate
+libc;`), you MUST declare them in your crate's `lib.rs` and NOT in any
+other module.
+
+ Dependencies and versions
+---------------------------
+
+In general, we use modules from only the Rust standard library
+whenever possible. We will review including external crates on a
+case-by-case basis.
+
+If a crate only contains traits meant for compatibility between Rust
+crates, such as [the digest crate](https://crates.io/crates/digest) or
+[the failure crate](https://crates.io/crates/failure), it is very likely
+permissible to add it as a dependency.  However, a brief review should
+be conducted as to the usefulness of implementing external traits
+(i.e. how widespread is the usage, how many other crates either
+implement the traits or have trait bounds based upon them), as well as
+the stability of the traits (i.e. if the trait is going to change, we'll
+potentially have to re-do all our implementations of it).
+
+For large external libraries, especially which implement features which
+would be labour-intensive to reproduce/maintain ourselves, such as
+cryptographic or mathematical/statistics libraries, only crates which
+have stabilised to 1.0.0 should be considered, however, again, we may
+make exceptions on a case-by-case basis.
+
+Currently, Tor requires that you use the latest stable Rust version. At
+some point in the future, we will freeze on a given stable Rust version,
+to ensure backward compatibility with stable distributions that ship it.
+
+ Updating/Adding Dependencies
+------------------------------
+
+To add/remove/update dependencies, first add your dependencies,
+exactly specifying their versions, into the appropriate *crate-level*
+`Cargo.toml` in `src/rust/` (i.e. *not* `/src/rust/Cargo.toml`, but
+instead the one for your crate).  Also, investigate whether your
+dependency has any optional dependencies which are unnecessary but are
+enabled by default.  If so, you'll likely be able to enable/disable
+them via some feature, e.g.:
+
+```toml
+[dependencies]
+foo = { version = "1.0.0", default-features = false }
+```
+
+Next, run `/scripts/maint/updateRustDependencies.sh`.  Then, go into
+`src/ext/rust` and commit the changes to the `tor-rust-dependencies`
+repo.
+
+ Documentation
+---------------
+
+You MUST include `#![deny(missing_docs)]` in your crate.
+
+For function/method comments, you SHOULD include a one-sentence, "first person"
+description of function behaviour (see requirements for documentation as
+described in `.../src/HACKING/CodingStandards.md`), then an `# Inputs` section
+for inputs or initialisation values, a `# Returns` section for return
+values/types, a `# Warning` section containing warnings for unsafe behaviours or
+panics that could happen.  For publicly accessible
+types/constants/objects/functions/methods, you SHOULD also include an
+`# Examples` section with runnable doctests.
+
+You MUST document your module with _module docstring_ comments,
+i.e. `//!` at the beginning of each line.
+
+ Style
+-------
+
+You SHOULD consider breaking up large literal numbers with `_` when it makes it
+more human readable to do so, e.g. `let x: u64 = 100_000_000_000`.
+
+ Testing
+---------
+
+All code MUST be unittested and integration tested.
+
+Public functions/objects exported from a crate SHOULD include doctests
+describing how the function/object is expected to be used.
+
+Integration tests SHOULD go into a `tests/` directory inside your
+crate.  Unittests SHOULD go into their own module inside the module
+they are testing, e.g. in `.../src/rust/tor_addition/addition.rs` you
+should put:
+
+    #[cfg(test)]
+    mod test {
+        use super::*;
+
+        #[test]
+        fn addition_with_zero() {
+            let sum: i32 = get_sum(5i32, 0i32);
+            assert_eq!(sum, 5);
+        }
+    }
+
+ Benchmarking
+--------------
+
+The external `test` crate can be used for most benchmarking.  However, using
+this crate requires nightly Rust.  Since we may want to switch to a more
+stable Rust compiler eventually, we shouldn't do things which will automatically
+break builds for stable compilers.  Therefore, you MUST feature-gate your
+benchmarks in the following manner.
+
+If you wish to benchmark some of your Rust code, you MUST put the
+following in the `[features]` section of your crate's `Cargo.toml`:
+
+    [features]
+    bench = []
+
+Next, in your crate's `lib.rs` you MUST put:
+
+    #[cfg(all(test, feature = "bench"))]
+    extern crate test;
+
+This ensures that the external crate `test`, which contains utilities
+for basic benchmarks, is only used when running benchmarks via `cargo
+bench --features bench`.
+
+Finally, to write your benchmark code, in
+`.../src/rust/tor_addition/addition.rs` you SHOULD put:
+
+    #[cfg(all(test, features = "bench"))]
+    mod bench {
+        use test::Bencher;
+        use super::*;
+
+        #[bench]
+        fn addition_small_integers(b: &mut Bencher) {
+            b.iter(| | get_sum(5i32, 0i32));
+        }
+    }
+
+ Fuzzing
+---------
+
+If you wish to fuzz parts of your code, please see the
+[`cargo fuzz`](https://github.com/rust-fuzz/cargo-fuzz) crate, which uses
+[libfuzzer-sys](https://github.com/rust-fuzz/libfuzzer-sys).
+
+ Whitespace & Formatting
+-------------------------
+
+You MUST run `rustfmt` (https://github.com/rust-lang-nursery/rustfmt)
+on your code before your code will be merged.  You can install rustfmt
+by doing `cargo install rustfmt-nightly` and then run it with `cargo
+fmt`.
+
+ Safety
+--------
+
+You SHOULD read [the nomicon](https://doc.rust-lang.org/nomicon/) before writing
+Rust FFI code.  It is *highly advised* that you read and write normal Rust code
+before attempting to write FFI or any other unsafe code.
+
+Here are some additional bits of advice and rules:
+
+0. Any behaviours which Rust considers to be undefined are forbidden
+
+   From https://doc.rust-lang.org/reference/behavior-considered-undefined.html:
+
+   > Behavior considered undefined
+   >
+   > The following is a list of behavior which is forbidden in all Rust code,
+   > including within unsafe blocks and unsafe functions. Type checking provides the
+   > guarantee that these issues are never caused by safe code.
+   > 
+   > * Data races
+   > * Dereferencing a null/dangling raw pointer
+   > * Reads of [undef](http://llvm.org/docs/LangRef.html#undefined-values)
+   >   (uninitialized) memory
+   > * Breaking the
+   >   [pointer aliasing rules](http://llvm.org/docs/LangRef.html#pointer-aliasing-rules)
+   >   with raw pointers (a subset of the rules used by C)
+   > * `&mut T` and `&T` follow LLVM’s scoped noalias model, except if the `&T`
+   >   contains an `UnsafeCell<U>`. Unsafe code must not violate these aliasing
+   >   guarantees.
+   > * Mutating non-mutable data (that is, data reached through a shared
+   >   reference or data owned by a `let` binding), unless that data is
+   >   contained within an `UnsafeCell<U>`.
+   > * Invoking undefined behavior via compiler intrinsics:
+   >     - Indexing outside of the bounds of an object with
+   >       `std::ptr::offset` (`offset` intrinsic), with the exception of
+   >       one byte past the end which is permitted.
+   >     - Using `std::ptr::copy_nonoverlapping_memory` (`memcpy32`/`memcpy64`
+   >       intrinsics) on overlapping buffers
+   > * Invalid values in primitive types, even in private fields/locals:
+   >     - Dangling/null references or boxes
+   >     - A value other than `false` (0) or `true` (1) in a `bool`
+   >     - A discriminant in an `enum` not included in the type definition
+   >     - A value in a `char` which is a surrogate or above `char::MAX`
+   >     - Non-UTF-8 byte sequences in a `str`
+   > * Unwinding into Rust from foreign code or unwinding from Rust into foreign
+   >   code. Rust's failure system is not compatible with exception handling in other
+   >   languages. Unwinding must be caught and handled at FFI boundaries.
+
+1. `unwrap()`
+
+   If you call `unwrap()`, anywhere, even in a test, you MUST include
+   an inline comment stating how the unwrap will either 1) never fail,
+   or 2) should fail (i.e. in a unittest).
+
+   You SHOULD NOT use `unwrap()` anywhere in which it is possible to handle the
+   potential error with either `expect()` or the eel operator, `?`.
+   For example, consider a function which parses a string into an integer:
+
+        fn parse_port_number(config_string: &str) -> u16 {
+            u16::from_str_radix(config_string, 10).unwrap()
+        }
+
+   There are numerous ways this can fail, and the `unwrap()` will cause the
+   whole program to byte the dust!  Instead, either you SHOULD use `expect()`
+   (or another equivalent function which will return an `Option` or a `Result`)
+   and change the return type to be compatible:
+
+        fn parse_port_number(config_string: &str) -> Option<u16> {
+            u16::from_str_radix(config_string, 10).expect("Couldn't parse port into a u16")
+        }
+
+   or you SHOULD use `or()` (or another similar method):
+
+        fn parse_port_number(config_string: &str) -> Option<u16> {
+            u16::from_str_radix(config_string, 10).or(Err("Couldn't parse port into a u16")
+        }
+
+   Using methods like `or()` can be particularly handy when you must do
+   something afterwards with the data, for example, if we wanted to guarantee
+   that the port is high.  Combining these methods with the eel operator (`?`)
+   makes this even easier:
+
+        fn parse_port_number(config_string: &str) -> Result<u16, Err> {
+            let port = u16::from_str_radix(config_string, 10).or(Err("Couldn't parse port into a u16"))?;
+
+            if port > 1024 {
+                return Ok(port);
+            } else {
+                return Err("Low ports not allowed");
+            }
+        }
+
+2. `unsafe`
+
+   If you use `unsafe`, you MUST describe a contract in your
+   documentation which describes how and when the unsafe code may
+   fail, and what expectations are made w.r.t. the interfaces to
+   unsafe code.  This is also REQUIRED for major pieces of FFI between
+   C and Rust.
+
+   When creating an FFI in Rust for C code to call, it is NOT REQUIRED
+   to declare the entire function `unsafe`.  For example, rather than doing:
+
+        #[no_mangle]
+        pub unsafe extern "C" fn increment_and_combine_numbers(mut numbers: [u8; 4]) -> u32 {
+            for number in &mut numbers {
+                *number += 1;
+            }
+            std::mem::transmute::<[u8; 4], u32>(numbers)
+        }
+
+   You SHOULD instead do:
+
+        #[no_mangle]
+        pub extern "C" fn increment_and_combine_numbers(mut numbers: [u8; 4]) -> u32 {
+            for index in 0..numbers.len() {
+                numbers[index] += 1;
+            }
+            unsafe {
+                std::mem::transmute::<[u8; 4], u32>(numbers)
+            }
+        }
+
+3. Pass only C-compatible primitive types and bytes over the boundary
+
+   Rust's C-compatible primitive types are integers and floats.
+   These types are declared in the [libc crate](https://doc.rust-lang.org/libc/x86_64-unknown-linux-gnu/libc/index.html#types).
+   Most Rust objects have different [representations](https://doc.rust-lang.org/libc/x86_64-unknown-linux-gnu/libc/index.html#types)
+   in C and Rust, so they can't be passed using FFI.
+
+   Tor currently uses the following Rust primitive types from libc for FFI:
+   * defined-size integers: `uint32_t`
+   * native-sized integers: `c_int`
+   * native-sized floats: `c_double`
+   * native-sized raw pointers: `* c_void`, `* c_char`, `** c_char`
+
+   TODO: C smartlist to Stringlist conversion using FFI
+
+   The only non-primitive type which may cross the FFI boundary is
+   bytes, e.g. `&[u8]`.  This SHOULD be done on the Rust side by
+   passing a pointer (`*mut libc::c_char`). The length can be passed
+   explicitly (`libc::size_t`), or the string can be NUL-byte terminated
+   C string.
+
+   One might be tempted to do this via doing
+   `CString::new("blah").unwrap().into_raw()`. This has several problems:
+
+   a) If you do `CString::new("bl\x00ah")` then the unwrap() will fail
+      due to the additional NULL terminator, causing a dangling
+      pointer to be returned (as well as a potential use-after-free).
+
+   b) Returning the raw pointer will cause the CString to run its deallocator,
+      which causes any C code which tries to access the contents to dereference a
+      NULL pointer.
+
+   c) If we were to do `as_raw()` this would result in a potential double-free
+      since the Rust deallocator would run and possibly Tor's deallocator.
+
+   d) Calling `into_raw()` without later using the same pointer in Rust to call
+      `from_raw()` and then deallocate in Rust can result in a
+      [memory leak](https://doc.rust-lang.org/std/ffi/struct.CString.html#method.into_raw).
+
+      [It was determined](https://github.com/rust-lang/rust/pull/41074) that this
+      is safe to do if you use the same allocator in C and Rust and also specify
+      the memory alignment for CString (except that there is no way to specify
+      the alignment for CString).  It is believed that the alignment is always 1,
+      which would mean it's safe to dealloc the resulting `*mut c_char` in Tor's
+      C code.  However, the Rust developers are not willing to guarantee the
+      stability of, or a contract for, this behaviour, citing concerns that this
+      is potentially extremely and subtly unsafe.
+
+4. Perform an allocation on the other side of the boundary
+
+   After crossing the boundary, the other side MUST perform an
+   allocation to copy the data and is therefore responsible for
+   freeing that memory later.
+
+5. No touching other language's enums
+
+   Rust enums should never be touched from C (nor can they be safely
+   `#[repr(C)]`) nor vice versa:
+
+   >  "The chosen size is the default enum size for the target platform's C
+   >  ABI. Note that enum representation in C is implementation defined, so this is
+   >  really a "best guess". In particular, this may be incorrect when the C code
+   >  of interest is compiled with certain flags."
+
+   (from https://gankro.github.io/nomicon/other-reprs.html)
+
+6. Type safety
+
+   Wherever possible and sensical, you SHOULD create new types in a
+   manner which prevents type confusion or misuse.  For example,
+   rather than using an untyped mapping between strings and integers
+   like so:
+
+        use std::collections::HashMap;
+
+        pub fn get_elements_with_over_9000_points(map: &HashMap<String, usize>) -> Vec<String> {
+            ...
+        }
+
+   It would be safer to define a new type, such that some other usage
+   of `HashMap<String, usize>` cannot be confused for this type:
+
+        pub struct DragonBallZPowers(pub HashMap<String, usize>);
+
+        impl DragonBallZPowers {
+            pub fn over_nine_thousand<'a>(&'a self) -> Vec<&'a String> {
+                let mut powerful_enough: Vec<&'a String> = Vec::with_capacity(5);
+
+                for (character, power) in &self.0 {
+                    if *power > 9000 {
+                        powerful_enough.push(character);
+                    }
+                }
+                powerful_enough
+            }
+        }
+
+   Note the following code, which uses Rust's type aliasing, is valid
+   but it does NOT meet the desired type safety goals:
+
+        pub type Power = usize;
+
+        pub fn over_nine_thousand(power: &Power) -> bool {
+            if *power > 9000 {
+                return true;
+            }
+            false
+        }
+
+        // We can still do the following:
+        let his_power: usize = 9001;
+        over_nine_thousand(&his_power);
+
+7. Unsafe mucking around with lifetimes
+
+   Because lifetimes are technically, in type theory terms, a kind, i.e. a
+   family of types, individual lifetimes can be treated as types.  For example,
+   one can arbitrarily extend and shorten lifetime using `std::mem::transmute`:
+
+        struct R<'a>(&'a i32);
+
+        unsafe fn extend_lifetime<'b>(r: R<'b>) -> R<'static> {
+            std::mem::transmute::<R<'b>, R<'static>>(r)
+        }
+
+        unsafe fn shorten_invariant_lifetime<'b, 'c>(r: &'b mut R<'static>) -> &'b mut R<'c> {
+            std::mem::transmute::<&'b mut R<'static>, &'b mut R<'c>>(r)
+        }
+
+   Calling `extend_lifetime()` would cause an `R` passed into it to live forever
+   for the life of the program (the `'static` lifetime).  Similarly,
+   `shorten_invariant_lifetime()` could be used to take something meant to live
+   forever, and cause it to disappear!  This is incredibly unsafe.  If you're
+   going to be mucking around with lifetimes like this, first, you better have
+   an extremely good reason, and second, you may as be honest and explicit about
+   it, and for ferris' sake just use a raw pointer.
+
+   In short, just because lifetimes can be treated like types doesn't mean you
+   should do it.
+
+8. Doing excessively unsafe things when there's a safer alternative
+
+   Similarly to #7, often there are excessively unsafe ways to do a task and a
+   simpler, safer way.  You MUST choose the safer option where possible.
+
+   For example, `std::mem::transmute` can be abused in ways where casting with
+   `as` would be both simpler and safer:
+
+        // Don't do this
+        let ptr = &0;
+        let ptr_num_transmute = unsafe { std::mem::transmute::<&i32, usize>(ptr)};
+
+        // Use an `as` cast instead
+        let ptr_num_cast = ptr as *const i32 as usize;
+
+   In fact, using `std::mem::transmute` for *any* reason is a code smell and as
+   such SHOULD be avoided.
+
+9. Casting integers with `as`
+
+   This is generally fine to do, but it has some behaviours which you should be
+   aware of.  Casting down chops off the high bits, e.g.:
+
+        let x: u32 = 4294967295;
+        println!("{}", x as u16); // prints 65535
+
+   Some cases which you MUST NOT do include:
+
+   * Casting an `u128` down to an `f32` or vice versa (e.g.
+     `u128::MAX as f32` but this isn't only a problem with overflowing
+     as it is also undefined behaviour for `42.0f32 as u128`),
+
+   * Casting between integers and floats when the thing being cast
+     cannot fit into the type it is being casted into, e.g.:
+
+         println!("{}", 42949.0f32 as u8); // prints 197 in debug mode and 0 in release
+         println!("{}", 1.04E+17 as u8);   // prints 0 in both modes
+         println!("{}", (0.0/0.0) as i64); // prints whatever the heck LLVM wants
+
+     Because this behaviour is undefined, it can even produce segfaults in
+     safe Rust code.  For example, the following program built in release
+     mode segfaults:
+
+         #[inline(never)]
+         pub fn trigger_ub(sl: &[u8; 666]) -> &[u8] {
+             // Note that the float is out of the range of `usize`, invoking UB when casting.
+             let idx = 1e99999f64 as usize;
+             &sl[idx..] // The bound check is elided due to `idx` being of an undefined value.
+         }
+
+         fn main() {
+             println!("{}", trigger_ub(&[1; 666])[999999]); // ~ out of bound
+         }
+
+      And in debug mode panics with:
+
+         thread 'main' panicked at 'slice index starts at 140721821254240 but ends at 666', /checkout/src/libcore/slice/mod.rs:754:4
diff --git a/doc/HACKING/Fuzzing.md b/doc/HACKING/Fuzzing.md
new file mode 100644
index 0000000000..2039d6a4c0
--- /dev/null
+++ b/doc/HACKING/Fuzzing.md
@@ -0,0 +1,123 @@
+= Fuzzing Tor
+
+== The simple version (no fuzzing, only tests)
+
+Check out fuzzing-corpora, and set TOR_FUZZ_CORPORA to point to the place
+where you checked it out.
+
+To run the fuzzing test cases in a deterministic fashion, use:
+      make test-fuzz-corpora
+
+This won't actually fuzz Tor!  It will just run all the fuzz binaries
+on our existing set of testcases for the fuzzer.
+
+
+== Different kinds of fuzzing
+
+Right now we support three different kinds of fuzzer.
+
+First, there's American Fuzzy Lop (AFL), a fuzzer that works by forking
+a target binary and passing it lots of different inputs on stdin.  It's the
+trickiest one to set up, so I'll be describing it more below.
+
+Second, there's libFuzzer, a llvm-based fuzzer that you link in as a library,
+and it runs a target function over and over.  To use this one, you'll need to
+have a reasonably recent clang and libfuzzer installed.  At that point, you
+just build with --enable-expensive-hardening and --enable-libfuzzer.  That
+will produce a set of binaries in src/test/fuzz/lf-fuzz-* .  These programs
+take as input a series of directories full of fuzzing examples.  For more
+information on libfuzzer, see http://llvm.org/docs/LibFuzzer.html
+
+Third, there's Google's OSS-Fuzz infrastructure, which expects to get all of
+its.  For more on this, see https://github.com/google/oss-fuzz and the
+projects/tor subdirectory.  You'll need to mess around with Docker a bit to
+test this one out; it's meant to run on Google's infrastructure.
+
+In all cases, you'll need some starting examples to give the fuzzer when it
+starts out.  There's a set in the "fuzzing-corpora" git repository.  Try
+setting TOR_FUZZ_CORPORA to point to a checkout of that repository
+
+== Writing Tor fuzzers
+
+A tor fuzzing harness should have:
+* a fuzz_init() function to set up any necessary global state.
+* a fuzz_main() function to receive input and pass it to a parser.
+* a fuzz_cleanup() function to clear global state.
+
+Most fuzzing frameworks will produce many invalid inputs - a tor fuzzing
+harness should rejecting invalid inputs without crashing or behaving badly.
+
+But the fuzzing harness should crash if tor fails an assertion, triggers a
+bug, or accesses memory it shouldn't. This helps fuzzing frameworks detect
+"interesting" cases.
+
+
+== Guided Fuzzing with AFL
+
+There is no HTTPS, hash, or signature for American Fuzzy Lop's source code, so
+its integrity can't be verified. That said, you really shouldn't fuzz on a
+machine you care about, anyway.
+
+To Build:
+  Get AFL from http://lcamtuf.coredump.cx/afl/ and unpack it
+  cd afl
+  make
+  cd ../tor
+  PATH=$PATH:../afl/ CC="../afl/afl-gcc" ./configure --enable-expensive-hardening
+  AFL_HARDEN=1 make clean fuzzers
+
+To Find The ASAN Memory Limit: (64-bit only)
+
+On 64-bit platforms, afl needs to know how much memory ASAN uses,
+because ASAN tends to allocate a ridiculous amount of virtual memory,
+and then not actually use it.
+
+Read afl/docs/notes_for_asan.txt for more details.
+
+  Download recidivm from http://jwilk.net/software/recidivm
+  Download the signature
+  Check the signature
+  tar xvzf recidivm*.tar.gz
+  cd recidivm*
+  make
+  /path/to/recidivm -v src/test/fuzz/fuzz-http
+  Use the final "ok" figure as the input to -m when calling afl-fuzz
+  (Normally, recidivm would output a figure automatically, but in some cases,
+  the fuzzing harness will hang when the memory limit is too small.)
+
+You could also just say "none" instead of the memory limit below, if you
+don't care about memory limits.
+
+
+To Run:
+  mkdir -p src/test/fuzz/fuzz_http_findings
+  ../afl/afl-fuzz -i ${TOR_FUZZ_CORPORA}/http -o src/test/fuzz/fuzz_http_findings -m <asan-memory-limit> -- src/test/fuzz/fuzz-http
+
+
+AFL has a multi-core mode, check the documentation for details.
+You might find the included fuzz-multi.sh script useful for this.
+
+macOS (OS X) requires slightly more preparation, including:
+* using afl-clang (or afl-clang-fast from the llvm directory)
+* disabling external crash reporting (AFL will guide you through this step)
+
+== Triaging Issues
+
+Crashes are usually interesting, particularly if using AFL_HARDEN=1 and --enable-expensive-hardening. Sometimes crashes are due to bugs in the harness code.
+
+Hangs might be interesting, but they might also be spurious machine slowdowns.
+Check if a hang is reproducible before reporting it. Sometimes, processing
+valid inputs may take a second or so, particularly with the fuzzer and
+sanitizers enabled.
+
+To see what fuzz-http is doing with a test case, call it like this:
+  src/test/fuzz/fuzz-http --debug < /path/to/test.case
+
+(Logging is disabled while fuzzing to increase fuzzing speed.)
+
+== Reporting Issues
+
+Please report any issues discovered using the process in Tor's security issue
+policy:
+
+https://trac.torproject.org/projects/tor/wiki/org/meetings/2016SummerDevMeeting/Notes/SecurityIssuePolicy
diff --git a/doc/HACKING/GettingStarted.md b/doc/HACKING/GettingStarted.md
index 0295adc1ff..0c42404634 100644
--- a/doc/HACKING/GettingStarted.md
+++ b/doc/HACKING/GettingStarted.md
@@ -11,8 +11,9 @@ whole Tor ecosystem.)
 
 
 If you are looking for a more bare-bones, less user-friendly information
-dump of important information, you might like reading doc/HACKING
-instead.  You should probably read it before you write your first patch.
+dump of important information, you might like reading the "torguts"
+documents linked to below. You should probably read it before you write
+your first patch.
 
 
 Required background
diff --git a/doc/HACKING/GettingStartedRust.md b/doc/HACKING/GettingStartedRust.md
new file mode 100644
index 0000000000..aa29c097da
--- /dev/null
+++ b/doc/HACKING/GettingStartedRust.md
@@ -0,0 +1,183 @@
+
+ Hacking on Rust in Tor
+========================
+
+ Getting Started
+-----------------
+
+Please read or review our documentation on Rust coding standards
+(`.../doc/HACKING/CodingStandardsRust.md`) before doing anything.
+
+Please also read
+[the Rust Code of Conduct](https://www.rust-lang.org/en-US/conduct.html). We
+aim to follow the good example set by the Rust community and be
+excellent to one another.  Let's be careful with each other, so we can
+be memory-safe together!
+
+Next, please contact us before rewriting anything!  Rust in Tor is still
+an experiment.  It is an experiment that we very much want to see
+succeed, so we're going slowly and carefully.  For the moment, it's also
+a completely volunteer-driven effort: while many, if not most, of us are
+paid to work on Tor, we are not yet funded to write Rust code for Tor.
+Please be patient with the other people who are working on getting more
+Rust code into Tor, because they are graciously donating their free time
+to contribute to this effort.
+
+ Resources for learning Rust
+-----------------------------
+
+**Beginning resources**
+
+The primary resource for learning Rust is
+[The Book](https://doc.rust-lang.org/book/).  If you'd like to start writing
+Rust immediately, without waiting for anything to install, there is
+[an interactive browser-based playground](https://play.rust-lang.org/).
+
+**Advanced resources**
+
+If you're interested in playing with various Rust compilers and viewing
+a very nicely displayed output of the generated assembly, there is
+[the Godbolt compiler explorer](https://rust.godbolt.org/)
+
+For learning how to write unsafe Rust, read
+[The Rustonomicon](https://doc.rust-lang.org/nomicon/).
+
+For learning everything you ever wanted to know about Rust macros, there
+is
+[The Little Book of Rust Macros](https://danielkeep.github.io/tlborm/book/index.html).
+
+For learning more about FFI and Rust, see Jake Goulding's
+[Rust FFI Omnibus](http://jakegoulding.com/rust-ffi-omnibus/).
+
+ Compiling Tor with Rust enabled
+---------------------------------
+
+You will need to run the `configure` script with the `--enable-rust`
+flag to explicitly build with Rust. Additionally, you will need to
+specify where to fetch Rust dependencies, as we allow for either
+fetching dependencies from Cargo or specifying a local directory.
+
+**Fetch dependencies from Cargo**
+
+    ./configure --enable-rust --enable-cargo-online-mode
+
+**Using a local dependency cache**
+
+You'll need the following Rust dependencies (as of this writing):
+
+    libc==0.2.39
+
+We vendor our Rust dependencies in a separate repo using
+[cargo-vendor](https://github.com/alexcrichton/cargo-vendor).  To use
+them, do:
+
+    git submodule init
+    git submodule update
+
+To specify the local directory containing the dependencies, (assuming
+you are in the top level of the repository) configure tor with:
+
+    TOR_RUST_DEPENDENCIES='path_to_dependencies_directory' ./configure --enable-rust
+
+(Note that TOR_RUST_DEPENDENCIES must be the full path to the directory; it
+cannot be relative.)
+
+Assuming you used the above `git submodule` commands and you're in the
+topmost directory of the repository, this would be:
+
+    TOR_RUST_DEPENDENCIES=`pwd`/src/ext/rust/crates ./configure --enable-rust
+
+
+ Identifying which modules to rewrite
+======================================
+
+The places in the Tor codebase that are good candidates for porting to
+Rust are:
+
+1. loosely coupled to other Tor submodules,
+2. have high test coverage, and
+3. would benefit from being implemented in a memory safe language.
+
+Help in either identifying places such as this, or working to improve
+existing areas of the C codebase by adding regression tests and
+simplifying dependencies, would be really helpful.
+
+Furthermore, as submodules in C are implemented in Rust, this is a good
+opportunity to refactor, add more tests, and split modules into smaller
+areas of responsibility.
+
+A good first step is to build a module-level callgraph to understand how
+interconnected your target module is.
+
+    git clone https://git.torproject.org/user/nickm/calltool.git
+    cd tor
+    CFLAGS=0 ./configure
+    ../calltool/src/main.py module_callgraph
+
+The output will tell you each module name, along with a set of every module that
+the module calls.  Modules which call fewer other modules are better targets.
+
+ Writing your Rust module
+==========================
+
+Strive to change the C API as little as possible.
+
+We are currently targetting Rust stable. (See CodingStandardsRust.md for more
+details.)
+
+It is on our TODO list to try to cultivate good
+standing with various distro maintainers of `rustc` and `cargo`, in
+order to ensure that whatever version we solidify on is readily
+available.
+
+If parts of your Rust code needs to stay in sync with C code (such as
+handling enums across the FFI boundary), annonotate these places in a
+comment structured as follows:
+
+  /// C_RUST_COUPLED: <path_to_file> `<name_of_c_object>`
+
+Where <name_of_c_object> can be an enum, struct, constant, etc.  Then,
+do the same in the C code, to note that rust will need to be changed
+when the C does.
+
+
+ Adding your Rust module to Tor's build system
+-----------------------------------------------
+
+0. Your translation of the C module should live in its own crate(s)
+   in the `.../tor/src/rust/` directory.
+1. Add your crate to `.../tor/src/rust/Cargo.toml`, in the
+   `[workspace.members]` section.
+2. Add your crate's files to src/rust/include.am
+
+If your crate should be available to C (rather than just being included as a
+dependency of other Rust modules):
+0. Declare the crate as a dependency of tor_rust in
+   `src/rust/tor_util/Cargo.toml` and include it in
+   `src/rust/tor_rust/lib.rs`
+
+ How to test your Rust code
+----------------------------
+
+Everything should be tested full stop.  Even non-public functionality.
+
+Be sure to edit `.../tor/src/test/test_rust.sh` to add the name of your
+crate to the `crates` variable! This will ensure that `cargo test` is
+run on your crate.
+
+Configure Tor's build system to build with Rust enabled:
+
+    ./configure --enable-fatal-warnings --enable-rust --enable-cargo-online-mode
+
+Tor's test should be run by doing:
+
+    make check
+
+Tor's integration tests should also pass:
+
+    make test-stem
+
+ Submitting a patch
+=====================
+
+Please follow the instructions in `.../doc/HACKING/GettingStarted.md`.
diff --git a/doc/HACKING/HelpfulTools.md b/doc/HACKING/HelpfulTools.md
index a7f36e6c7e..d499238526 100644
--- a/doc/HACKING/HelpfulTools.md
+++ b/doc/HACKING/HelpfulTools.md
@@ -4,25 +4,49 @@ Useful tools
 These aren't strictly necessary for hacking on Tor, but they can help track
 down bugs.
 
+Travis/Appveyor CI
+------------------
+It's CI.
+
+Looks like this:
+* https://travis-ci.org/torproject/tor
+* https://ci.appveyor.com/project/torproject/tor
+
+Travis builds and runs tests on Linux, and eventually macOS (#24629).
+Appveyor builds and runs tests on Windows (using Windows Services for Linux).
+
+Runs automatically on Pull Requests sent to torproject/tor. You can set it up
+for your fork to build commits outside of PRs too:
+
+1. sign up for GitHub: https://github.com/join
+2. fork https://github.com/torproject/tor:
+   https://help.github.com/articles/fork-a-repo/
+3. follow https://docs.travis-ci.com/user/getting-started/#To-get-started-with-Travis-CI.
+   skip steps involving `.travis.yml` (we already have one).
+4. go to https://ci.appveyor.com/login , log in with github, and select
+   "NEW PROJECT"
+
+Builds should show up on the web at travis-ci.com and on IRC at #tor-ci on
+OFTC. If they don't, ask #tor-dev (also on OFTC).
+
 Jenkins
 -------
 
-    https://jenkins.torproject.org
+It's CI/builders. Looks like this: https://jenkins.torproject.org
 
-Dmalloc
--------
+Runs automatically on commits merged to git.torproject.org. We CI the
+master branch and all supported tor versions. We also build nightly debian
+packages from master.
 
-The dmalloc library will keep track of memory allocation, so you can find out
-if we're leaking memory, doing any double-frees, or so on.
+Builds Linux and Windows cross-compilation. Runs Linux tests.
 
-    dmalloc -l -/dmalloc.log
-    (run the commands it tells you)
-    ./configure --with-dmalloc
+Builds should show up on the web at jenkins.torproject.org and on IRC at
+#tor-bots on OFTC. If they don't, ask #tor-dev (also on OFTC).
 
 Valgrind
 --------
 
-    valgrind --leak-check=yes --error-limit=no --show-reachable=yes src/or/tor
+    valgrind --leak-check=yes --error-limit=no --show-reachable=yes src/app/tor
 
 (Note that if you get a zillion openssl warnings, you will also need to
 pass `--undef-value-errors=no` to valgrind, or rebuild your openssl
@@ -111,15 +135,19 @@ Running gcov for unit test coverage
 
 (On OSX, you'll need to start with `--enable-coverage CC=clang`.)
 
-Then, look at the .gcov files in `coverage-output`.  '-' before a line means
-that the compiler generated no code for that line.  '######' means that the
-line was never reached.  Lines with numbers were called that number of times.
-
 If that doesn't work:
 
    * Try configuring Tor with `--disable-gcc-hardening`
    * You might need to run `make clean` after you run `./configure`.
 
+Then, look at the .gcov files in `coverage-output`.  '-' before a line means
+that the compiler generated no code for that line.  '######' means that the
+line was never reached.  Lines with numbers were called that number of times.
+
+For more details about how to read gcov output, see the [Invoking
+gcov](https://gcc.gnu.org/onlinedocs/gcc/Invoking-Gcov.html) chapter
+of the GCC manual.
+
 If you make changes to Tor and want to get another set of coverage results,
 you can run `make reset-gcov` to clear the intermediary gcov output.
 
@@ -128,9 +156,13 @@ a meaningful diff between them, you can run:
 
     ./scripts/test/cov-diff coverage-output1 coverage-output2 | less
 
-In this diff, any lines that were visited at least once will have coverage
-"1".  This lets you inspect what you (probably) really want to know: which
-untested lines were changed?  Are there any new untested lines?
+In this diff, any lines that were visited at least once will have coverage "1",
+and line numbers are deleted.  This lets you inspect what you (probably) really
+want to know: which untested lines were changed?  Are there any new untested
+lines?
+
+If you run ./scripts/test/cov-exclude, it marks excluded unreached
+lines with 'x', and excluded reached lines with '!!!'.
 
 Running integration tests
 -------------------------
@@ -142,6 +174,12 @@ run `make test-network`.
 We also have scripts to run integration tests using Stem.  To try them, set
 `STEM_SOURCE_DIR` to your Stem source directory, and run `test-stem`.
 
+Profiling Tor
+-------------
+
+Ongoing notes about Tor profiling can be found at
+https://pad.riseup.net/p/profiling-tor
+
 Profiling Tor with oprofile
 ---------------------------
 
@@ -168,20 +206,62 @@ Here are some basic instructions
    * `opreport -l that_dir/*`
  - Profit
 
+Profiling Tor with perf
+-----------------------
+
+This works with a running Tor, and requires root.
+
+1. Decide how long you want to profile for. Start with (say) 30 seconds. If that
+   works, try again with longer times.
+
+2. Find the PID of your running tor process.
+
+3. Run `perf record --call-graph dwarf -p <PID> sleep <SECONDS>`
+
+   (You may need to do this as root.)
+
+   You might need to add `-e cpu-clock` as an option to the perf record line
+   above, if you are on an older CPU without access to hardware profiling
+   events, or in a VM, or something.
+
+4. Now you have a perf.data file. Have a look at it with `perf report
+   --no-children --sort symbol,dso` or `perf report --no-children --sort
+   symbol,dso --stdio --header`. How does it look?
+
+5a. Once you have a nice big perf.data file, you can compress it, encrypt it,
+    and send it to your favorite Tor developers.
+
+5b. Or maybe you'd rather not send a nice big perf.data file. Who knows what's
+    in that!? It's kinda scary. To generate a less scary file, you can use `perf
+    report -g > <FILENAME>.out`. Then you can compress that and put it somewhere
+    public.
+
+Profiling Tor with gperftools aka Google-performance-tools
+----------------------------------------------------------
+
+This should work on nearly any unixy system. It doesn't seem to be compatible
+with RunAsDaemon though.
+
+Beforehand, install google-perftools.
+
+1. You need to rebuild Tor, hack the linking steps to add `-lprofiler` to the
+   libs. You can do this by adding `LIBS=-lprofiler` when you call `./configure`.
+
+Now you can run Tor with profiling enabled, and use the pprof utility to look at
+performance! See the gperftools manual for more info, but basically:
+
+2. Run `env CPUPROFILE=/tmp/profile src/app/tor -f <path/torrc>`. The profile file
+   is not written to until Tor finishes execuction.
+
+3. Run `pprof src/app/tor /tm/profile` to start the REPL.
+
 Generating and analyzing a callgraph
 ------------------------------------
 
-1. Run `./scripts/maint/generate_callgraph.sh`.  This will generate a
-   bunch of files in a new ./callgraph directory.
-
-2. Run `./scripts/maint/analyze_callgraph.py callgraph/src/*/*`.  This
-   will do a lot of graph operations and then dump out a new
-   `callgraph.pkl` file, containing data in Python's 'pickle' format.
+0. Build Tor on linux or mac, ideally with -O0 or -fno-inline.
 
-3. Run `./scripts/maint/display_callgraph.py`.  It will display:
-    - the number of functions reachable from each function.
-    - all strongly-connnected components in the Tor callgraph
-    - the largest bottlenecks in the largest SCC in the Tor callgraph.
+1. Clone 'https://gitweb.torproject.org/user/nickm/calltool.git/' .
+   Follow the README in that repository.
 
 Note that currently the callgraph generator can't detect calls that pass
 through function pointers.
diff --git a/doc/HACKING/HowToReview.md b/doc/HACKING/HowToReview.md
index d53318942f..2d1f3d1c9e 100644
--- a/doc/HACKING/HowToReview.md
+++ b/doc/HACKING/HowToReview.md
@@ -19,6 +19,8 @@ Top-level smell-checks
 
 - Does `make check-spaces` pass?
 
+- Does `make check-changes` pass?
+
 - Does it have a reasonable amount of tests?  Do they pass?  Do they leak
   memory?
 
@@ -32,6 +34,7 @@ Top-level smell-checks
 
 - If this changes Tor's behavior on the wire, is there a design proposal?
 
+- If this changes anything in the code, is there a "changes" file?
 
 
 Let's look at the code!
diff --git a/doc/HACKING/Module.md b/doc/HACKING/Module.md
new file mode 100644
index 0000000000..9cf36090b4
--- /dev/null
+++ b/doc/HACKING/Module.md
@@ -0,0 +1,111 @@
+# Modules in Tor #
+
+This document describes the build system and coding standards when writing a
+module in Tor.
+
+## What is a module? ##
+
+In the context of the tor code base, a module is a subsystem that we can
+selectively enable or disable, at `configure` time.
+
+Currently, there is only one module:
+
+  - Directory Authority subsystem (dirauth)
+
+It is located in its own directory in `src/feature/dirauth/`. To disable it,
+one need to pass `--disable-module-dirauth` at configure time. All modules
+are currently enabled by default.
+
+## Build System ##
+
+The changes to the build system are pretty straightforward.
+
+1. Locate in the `configure.ac` file this define: `m4_define(MODULES`. It
+   contains a list (white-space separated) of the module in tor. Add yours to
+   the list.
+
+2. Use the `AC_ARG_ENABLE([module-dirauth]` template for your new module. We
+   use the "disable module" approach instead of enabling them one by one. So,
+   by default, tor will build all the modules.
+
+   This will define the `HAVE_MODULE_<name>` statement which can be used in
+   the C code to conditionally compile things for your module. And the
+   `BUILD_MODULE_<name>` is also defined for automake files (e.g: include.am).
+
+3. In the `src/core/include.am` file, locate the `MODULE_DIRAUTH_SOURCES`
+   value.  You need to create your own `_SOURCES` variable for your module
+   and then conditionally add the it to `LIBTOR_A_SOURCES` if you should
+   build the module.
+
+   It is then **very** important to add your SOURCES variable to
+   `src_or_libtor_testing_a_SOURCES` so the tests can build it.
+
+4. Do the same for header files, locate `ORHEADERS +=` which always add all
+   headers of all modules so the symbol can be found for the module entry
+   points.
+
+Finally, your module will automatically be included in the
+`TOR_MODULES_ALL_ENABLED` variable which is used to build the unit tests. They
+always build everything in order to tests everything.
+
+## Coding ##
+
+As mentioned above, a module must be isolated in its own directory (name of
+the module) in `src/feature/`.
+
+There are couples of "rules" you want to follow:
+
+* Minimize as much as you can the number of entry points into your module.
+  Less is always better but of course that doesn't work out for every use
+  case. However, it is a good thing to always keep that in mind.
+
+* Do **not** use the `HAVE_MODULE_<name>` define outside of the module code
+  base. Every entry point should have a second definition if the module is
+  disabled. For instance:
+
+  ```
+  #ifdef HAVE_MODULE_DIRAUTH
+
+  int sr_init(int save_to_disk);
+
+  #else /* HAVE_MODULE_DIRAUTH */
+
+  static inline int
+  sr_init(int save_to_disk)
+  {
+    (void) save_to_disk;
+    return 0;
+  }
+
+  #endif /* HAVE_MODULE_DIRAUTH */
+
+  ```
+
+  The main reason for this approach is to avoid having conditional code
+  everywhere in the code base. It should be centralized as much as possible
+  which helps maintainability but also avoids conditional spaghetti code
+  making the code much more difficult to follow/understand.
+
+* It is possible that you end up with code that needs to be used by the rest
+  of the code base but is still part of your module. As a good example, if
+  you look at `src/feature/shared_random_client.c`: it contains code needed
+  by the hidden service subsystem but mainly related to the shared random
+  subsystem very specific to the dirauth module.
+
+  This is fine but try to keep it as lean as possible and never use the same
+  filename as the one in the module. For example, this is a bad idea and
+  should never be done:
+
+    - `src/feature/dirclient/shared_random.c`
+    - `src/feature/dirauth/shared_random.c`
+
+* When you include headers from the module, **always** use the full module
+  path in your statement. Example:
+
+  `#include "feature/dirauth/dirvote.h"`
+
+  The main reason is that we do **not** add the module include path by default
+  so it needs to be specified. But also, it helps our human brain understand
+  which part comes from a module or not.
+
+  Even **in** the module itself, use the full include path like above.
diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md
index 7595398241..55a40fc89b 100644
--- a/doc/HACKING/ReleasingTor.md
+++ b/doc/HACKING/ReleasingTor.md
@@ -7,9 +7,16 @@ new Tor release:
 
 === 0. Preliminaries
 
-1. Get at least three of weasel/arma/Sebastian/Sina to put the new
-   version number in their approved versions list.
+1. Get at least two of weasel/arma/Sebastian to put the new
+   version number in their approved versions list.  Give them a few
+   days to do this if you can.
 
+2. If this is going to be an important security release, give the packagers
+   some advance warning: See this list of packagers in IV.3 below.
+
+3. Given the release date for Tor, ask the TB team about the likely release
+   date of a TB that contains it.  See note below in "commit, upload,
+   announce".
 
 === I. Make sure it works
 
@@ -18,6 +25,7 @@ new Tor release:
    resolve those.
 
    As applicable, merge the `maint-X` branch into the `release-X` branch.
+   But you've been doing that all along, right?
 
 2. Are all of the jenkins builders happy?  See jenkins.torproject.org.
 
@@ -26,39 +34,37 @@ new Tor release:
 
    What about Coverity Scan?
 
-   Is make check-spaces happy?
+   What about clang scan-build?
 
-   Does 'make distcheck' compain?
+   Does 'make distcheck' complain?
 
-   How about 'make test-stem' and 'make test-network'?
+   How about 'make test-stem' and 'make test-network' and
+   `make test-network-full`?
 
        - Are all those tests still happy with --enable-expensive-hardening ?
 
    Any memory leaks?
 
 
-=== II. Write a changelog.
+=== II. Write a changelog
+
 
+1a. (Alpha release variant)
 
-1. Gather the `changes/*` files into a changelog entry, rewriting many
+   Gather the `changes/*` files into a changelog entry, rewriting many
    of them and reordering to focus on what users and funders would find
    interesting and understandable.
 
-   1. Make sure that everything that wants a bug number has one.
-      Make sure that everything which is a bugfix says what version
-      it was a bugfix on.
-
-   2. Concatenate them.
-
-   3. Sort them by section. Within each section, sort by "version it's
-      a bugfix on", else by numerical ticket order.
+   To do this, first run `./scripts/maint/lintChanges.py changes/*` and
+   fix as many warnings as you can.  Then run `./scripts/maint/sortChanges.py
+   changes/* > changelog.in` to combine headings and sort the entries.
+   After that, it's time to hand-edit and fix the issues that lintChanges
+   can't find:
 
-   4. Clean them up:
+   1. Within each section, sort by "version it's a bugfix on", else by
+      numerical ticket order.
 
-      Standard idioms:
-      `Fixes bug 9999; bugfix on 0.3.3.3-alpha.`
-
-      One space after a period.
+   2. Clean them up:
 
       Make stuff very terse
 
@@ -86,19 +92,32 @@ new Tor release:
       maint-0.2.1), be sure to make the stanzas identical (so people can
       distinguish if these are the same change).
 
-   5. Merge them in.
+   3. Clean everything one last time.
+
+   4. Run `./scripts/maint/format_changelog.py --inplace` to make it prettier
 
-   6. Clean everything one last time.
+1b. (old-stable release variant)
 
-   7. Run `./scripts/maint/format_changelog.py` to make it prettier.
+   For stable releases that backport things from later, we try to compose
+   their releases, we try to make sure that we keep the changelog entries
+   identical to their original versions, with a 'backport from 0.x.y.z'
+   note added to each section.  So in this case, once you have the items
+   from the changes files copied together, don't use them to build a new
+   changelog: instead, look up the corrected versions that were merged
+   into ChangeLog in the master branch, and use those.
 
 2. Compose a short release blurb to highlight the user-facing
    changes. Insert said release blurb into the ChangeLog stanza. If it's
    a stable release, add it to the ReleaseNotes file too. If we're adding
-   to a release-0.2.x branch, manually commit the changelogs to the later
+   to a release-* branch, manually commit the changelogs to the later
    git branches too.
 
-3.  If you're doing the first stable release in a series, you need to
+3. If there are changes that require or suggest operator intervention
+   before or during the update, mail operators (either dirauth or relays
+   list) with a headline that indicates that an action is required or
+   appreciated.
+
+4. If you're doing the first stable release in a series, you need to
    create a ReleaseNotes for the series as a whole.  To get started
    there, copy all of the Changelog entries from the series into a new
    file, and run `./scripts/maint/sortChanges.py` on it.  That will
@@ -111,51 +130,61 @@ new Tor release:
 
 === III. Making the source release.
 
-1. In `maint-0.2.x`, bump the version number in `configure.ac` and run
-   `scripts/maint/updateVersions.pl` to update version numbers in other
-   places, and commit.  Then merge `maint-0.2.x` into `release-0.2.x`.
+1. In `maint-0.?.x`, bump the version number in `configure.ac` and run
+   `perl scripts/maint/updateVersions.pl` to update version numbers in other
+   places, and commit.  Then merge `maint-0.?.x` into `release-0.?.x`.
 
    (NOTE: To bump the version number, edit `configure.ac`, and then run
    either `make`, or `perl scripts/maint/updateVersions.pl`, depending on
    your version.)
 
-2. Make distcheck, put the tarball up somewhere, and tell `#tor` about
-   it. Wait a while to see if anybody has problems building it. Try to
-   get Sebastian or somebody to try building it on Windows.
+   When you merge the maint branch forward to the next maint branch, or into
+   master, merge it with "-s ours" to avoid a needless version bump.
+
+2. Make distcheck, put the tarball up in somewhere (how about your
+   homedir on your homedir on people.torproject.org?) , and tell `#tor`
+   about it. Wait a while to see if anybody has problems building it.
+   (Though jenkins is usually pretty good about catching these things.)
 
 === IV. Commit, upload, announce
 
 1. Sign the tarball, then sign and push the git tag:
 
         gpg -ba <the_tarball>
-        git tag -u <keyid> tor-0.2.x.y-status
-        git push origin tag tor-0.2.x.y-status
+        git tag -u <keyid> tor-0.3.x.y-status
+        git push origin tag tor-0.3.x.y-status
+
+   (You must do this before you update the website: it relies on finding
+   the version by tag.)
 
 2. scp the tarball and its sig to the dist website, i.e.
    `/srv/dist-master.torproject.org/htdocs/` on dist-master. When you want
    it to go live, you run "static-update-component dist.torproject.org"
    on dist-master.
 
-   Edit `include/versions.wmi` and `Makefile` to note the new version.
+   In the webwml.git repository, `include/versions.wmi` and `Makefile`
+   to note the new version.
 
    (NOTE: Due to #17805, there can only be one stable version listed at
    once.  Nonetheless, do not call your version "alpha" if it is stable,
    or people will get confused.)
 
-3. Email the packagers (cc'ing tor-assistants) that a new tarball is up.
+3. Email the packagers (cc'ing tor-team) that a new tarball is up.
    The current list of packagers is:
 
        - {weasel,gk,mikeperry} at torproject dot org
        - {blueness} at gentoo dot org
        - {paul} at invizbox dot io
+       - {vincent} at invizbox dot com
        - {lfleischer} at archlinux dot org
        - {Nathan} at freitas dot net
        - {mike} at tig dot as
-       - {tails-rm} at boum dot org (for pre-release announcments)
-
-
-       - {tails-dev} at boum dot org (for at-release announcements)
+       - {tails-rm} at boum dot org
+       - {simon} at sdeziel.info
+       - {yuri} at freebsd.org
+       - {mh+tor} at scrit.ch
 
+   Also, email tor-packagers@lists.torproject.org.
 
 4. Add the version number to Trac.  To do this, go to Trac, log in,
     select "Admin" near the top of the screen, then select "Versions" from
@@ -164,22 +193,30 @@ new Tor release:
     0.2.2.23-alpha" (or whatever the version is), and we select the date as
     the date in the ChangeLog.
 
-5. Wait up to a day or two (for a development release), or until most
-    packages are up (for a stable release), and mail the release blurb and
-    changelog to tor-talk or tor-announce.
+5. Double-check: did the version get recommended in the consensus yet?  Is
+   the website updated?  If not, don't announce until they have the
+   up-to-date versions, or people will get confused.
+
+6. Mail the release blurb and ChangeLog to tor-talk (development release) or
+   tor-announce (stable).
+
+   Post the changelog on the blog as well. You can generate a
+   blog-formatted version of the changelog with the -B option to
+   format-changelog.
 
-   (We might be moving to faster announcements, but don't announce until
-   the website is at least updated.)
+   When you post, include an estimate of when the next TorBrowser
+   releases will come out that include this Tor release.  This will
+   usually track https://wiki.mozilla.org/RapidRelease/Calendar , but it
+   can vary.
 
 
 === V. Aftermath and cleanup
 
-1. If it's a stable release, bump the version number in the `maint-x.y.z`
-    branch to "newversion-dev", and do a `merge -s ours` merge to avoid
-    taking that change into master.  Do a similar `merge -s theirs`
-    merge to get the change (and only that change) into release.  (Some
-    of the build scripts require that maint merge cleanly into release.)
+1. If it's a stable release, bump the version number in the
+    `maint-x.y.z` branch to "newversion-dev", and do a `merge -s ours`
+    merge to avoid taking that change into master.
 
 2. Forward-port the ChangeLog (and ReleaseNotes if appropriate).
 
+3. Keep an eye on the blog post, to moderate comments and answer questions.
 
diff --git a/doc/HACKING/Tracing.md b/doc/HACKING/Tracing.md
new file mode 100644
index 0000000000..24fa761310
--- /dev/null
+++ b/doc/HACKING/Tracing.md
@@ -0,0 +1,91 @@
+# Tracing #
+
+This document describes how the event tracing subsystem works in tor so
+developers can add events to the code base but also hook them to an event
+tracing framework.
+
+## Basics ###
+
+Event tracing is separated in two concepts, trace events and a tracer. The
+tracing subsystem can be found in `src/trace`. The `events.h` header file is
+the main file that maps the different tracers to trace events.
+
+### Events ###
+
+A trace event is basically a function from which we can pass any data that
+we want to collect. In addition, we specify a context for the event such as
+a subsystem and an event name.
+
+A trace event in tor has the following standard format:
+
+	tor_trace(subsystem, event\_name, args...)
+
+The `subsystem` parameter is the name of the subsytem the trace event is in.
+For example that could be "scheduler" or "vote" or "hs". The idea is to add
+some context to the event so when we collect them we know where it's coming
+from. The `event_name` is the name of the event which helps a lot with
+adding some semantic to the event. Finally, `args` is any number of
+arguments we want to collect.
+
+Here is an example of a possible tracepoint in main():
+
+	tor_trace(main, init_phase, argc)
+
+The above is a tracepoint in the `main` subsystem with `init_phase` as the
+event name and the `int argc` is passed to the event as well.
+
+How `argc` is collected or used has nothing to do with the instrumentation
+(adding trace events to the code). It is the work of the tracer so this is why
+the trace events and collection framework (tracer) are decoupled. You _can_
+have trace events without a tracer.
+
+### Tracer ###
+
+In `src/trace/events.h`, we map the `tor_trace()` function to the right
+tracer. A tracer support is only enabled at compile time. For instance, the
+file `src/trace/debug.h` contains the mapping of the generic tracing function
+`tor_trace()` to the `log_debug()` function. More specialized function can be
+mapped depending on the tracepoint.
+
+## Build System ##
+
+This section describes how it is integrated into the build system of tor.
+
+By default, every tracing events are disabled in tor that is `tor_trace()`
+is a NOP.
+
+To enable a tracer, there is a configure option on the form of:
+
+	--enable-tracing-<tracer>
+
+We have an option that will send every trace events to a `log_debug()` (as
+mentionned above) which will print you the subsystem and name of the event but
+not the arguments for technical reasons. This is useful if you want to quickly
+see if your trace event is being hit or well written. To do so, use this
+configure option:
+
+	--enable-tracing-debug
+
+## Instrument Tor ##
+
+This is pretty easy. Let's say you want to add a trace event in
+`src/feature/rend/rendcache.c`, you only have to add this include statement:
+
+	#include "trace/events.h"
+
+Once done, you can add as many as you want `tor_trace()` that you need.
+Please use the right subsystem (here it would be `hs`) and a unique name that
+tells what the event is for. For example:
+
+	tor_trace(hs, store_desc_as_client, desc, desc_id);
+
+If you look in `src/trace/events.h`, you'll see that if tracing is enabled it
+will be mapped to a function called:
+
+	tor_trace_hs_store_desc_as_client(desc, desc_id)
+
+And the point of all this is for that function to be defined in a new file
+that you might want to add named `src/trace/hs.{c|h}` which would defined how
+to collect the data for the `tor_trace_hs_store_desc_as_client()` function
+like for instance sending it to a `log_debug()` or do more complex operations
+or use a userspace tracer like LTTng (https://lttng.org).
diff --git a/doc/HACKING/WritingTests.md b/doc/HACKING/WritingTests.md
index de80bbdef2..05de8e0be8 100644
--- a/doc/HACKING/WritingTests.md
+++ b/doc/HACKING/WritingTests.md
@@ -7,8 +7,8 @@ keep from introducing bugs.  The major ones are:
 
    1. Unit tests written in C and shipped with the Tor distribution.
 
-   2. Integration tests written in Python and shipped with the Tor
-      distribution.
+   2. Integration tests written in Python 2 (>= 2.7) or Python 3
+      (>= 3.1) and shipped with the Tor distribution.
 
    3. Integration tests written in Python and shipped with the Stem
       library.  Some of these use the Tor controller protocol.
@@ -48,7 +48,7 @@ isolation, you just run `./src/test/test-memwipe`.
 To run tests within the unit test programs, you can specify the name
 of the test.  The string ".." can be used as a wildcard at the end of the
 test name.  For example, to run all the cell format tests, enter
-`./src/test/test cellfmt/..`.  To run
+`./src/test/test cellfmt/..`.
 
 Many tests that need to mess with global state run in forked subprocesses in
 order to keep from contaminating one another.  But when debugging a failing test,
@@ -91,6 +91,9 @@ coverage percentage.
 For a summary of the test coverage for each _function_, run
 `./scripts/test/cov-display -f ${TMPDIR}/*`.
 
+For more details on using gcov, including the helper scripts in
+scripts/test, see HelpfulTools.md.
+
 ### Comparing test coverage
 
 Sometimes it's useful to compare test coverage for a branch you're writing to
@@ -117,7 +120,8 @@ with LCOV_EXCL_START... LCOV_EXCL_STOP.  Note that older versions of
 lcov don't understand these lines.
 
 You can post-process .gcov files to make these lines 'unreached' by
-running ./scripts/test/cov-exclude on them.
+running ./scripts/test/cov-exclude on them.  It marks excluded
+unreached lines with 'x', and excluded reached lines with '!!!'.
 
 Note: you should never do this unless the line is meant to 100%
 unreachable by actual code.
diff --git a/doc/HACKING/android/Simpleperf.md b/doc/HACKING/android/Simpleperf.md
new file mode 100644
index 0000000000..25f39a3d23
--- /dev/null
+++ b/doc/HACKING/android/Simpleperf.md
@@ -0,0 +1,98 @@
+# Using `simpleperf` to collect CPU profiling on Android
+
+This document describes how you can use Android's `simpleperf`
+command-line tool to get CPU profiling information from Tor via the
+Orbot application. The tool is particularly useful for Tor development
+because it is able to profile native applications on the platform
+whereas a lot of the normal tooling for the Android platform is only
+able to collect information from Java-based applications.
+
+## Prerequisites
+
+Before using `simpleperf` there is a couple of steps that must be
+followed. You should make sure you have both a recent installation of
+the Android Software Development Kit (SDK) and Native Development Kit
+(NDK) installed. These can be found on the Android Developers website.
+
+1. Follow the build instructions from the `BUILD` file in the Orbot
+   repository and build an Orbot APK (Android Package) file with
+   debugging enabled. Make sure that when you build the native content of
+   the Orbot application that you run the `make -C external` command with
+   an additional `DEBUG=1` as parameter to ensure that the Orbot build
+   process does not strip the debug symbols from the Tor binary.
+
+2. (Optional) Uninstall and clean-up your old Orbot installation that
+   is most likely downloaded from Google's Play Store or via fdroid:
+
+       $ adb shell pm clear org.torproject.android
+       $ adb uninstall org.torproject.android
+
+3. Install the Android Package you generated in step 1:
+
+       $ adb install /path/to/your/app-fullperm-debug.apk
+
+4. Check on your device that the newly installed Orbot actually works
+   and behaves in the way you expect it to.
+
+## Profiling using `simpleperf`
+
+The `simpleperf` tool can be found in the `simpleperf/` directory in
+the directory where you installed the Android NDK to. In this
+directory there is a set of Python files that will help you deploy the
+tool to a device and collect the measurement data such that you can
+analyze the results on your computer rather than on your phone.
+
+1. Change directory to the location of the `simpleperf` directory.
+2. Open the `app_profiler.config` file and change
+   `app_package_name` to `org.torproject.android`, `apk_file_path` to
+   the path of your Orbot Android Package (APK file).
+3. Optionally change the duration parameter in the `record_options`
+   variable in `app_profiler.config` to the duration which you would like
+   to collect samples in. The value is specified in seconds.
+4. Run the app profiler using `python app_profiler.py`. This helper
+   script will push the `simpleperf` tool to your device, start the
+   profiler, and once it has completed copy the generated `perf.data`
+   file over to your computer with the results.
+
+### Analyzing the results
+
+You can inspect your resulting `perf.data` file via a simple GUI
+program `python report.py` or via the command-line tool `simpleperf
+report`. I've found the GUI tool to be easier to navigate around with
+than the command-line tool.
+
+The `-g` option can be passed to the command line `simpleperf report`
+tool allows you to see the call graph of functions and how much time
+was spend on the call.
+
+## Tips & Tricks
+
+- When you have installed Orbot the first time, you will notice that
+  if you get a shell on the Android device that there is no Tor binary
+  available. This is because Orbot unpacks the Tor binary first time it
+  is executed and places it under the `app_bin/` directory on the
+  device.
+
+  To access binaries, `torrc` files, and other useful information on
+  the device do the following:
+
+      $ adb shell
+      (device):/ $ run-as org.torproject.android
+      (device):/data/data/org.torproject.android $ ls
+      app_bin app_data cache databases files lib shared_prefs
+
+  Descriptors, control authentication cookie, state, and other files can be
+  found in the `app_data` directory. The `torrc` can be found in the `app_bin/`
+  directory.
+
+- You can enable logging in Tor via the syslog (or android) log
+  mechanism with:
+
+      $ adb shell
+      (device):/ $ run-as org.torproject.android
+      (device):/data/data/org.torproject.android $ echo -e "\nLog info syslog" >> app_bin/torrc
+
+  Start Tor the normal way via Orbot and collect the logs from your computer using
+
+      $ adb logcat
+
diff --git a/doc/include.am b/doc/include.am
index 7164a4b2a0..0a123aae11 100644
--- a/doc/include.am
+++ b/doc/include.am
@@ -12,17 +12,11 @@
 # part of the source distribution, so that people without asciidoc can
 # just use the .1 and .html files.
 
-base_mans = doc/tor doc/tor-gencert doc/tor-resolve doc/torify
-all_mans = $(base_mans)
-if USE_FW_HELPER
-install_mans = $(all_mans)
-else
-install_mans = $(base_mans)
-endif
+all_mans = doc/tor doc/tor-gencert doc/tor-resolve doc/torify doc/tor-print-ed-signing-cert
 
 if USE_ASCIIDOC
-nodist_man1_MANS = $(install_mans:=.1)
-doc_DATA = $(install_mans:=.html)
+nodist_man1_MANS = $(all_mans:=.1)
+doc_DATA = $(all_mans:=.html)
 html_in = $(all_mans:=.html.in)
 man_in = $(all_mans:=.1.in)
 txt_in = $(all_mans:=.1.txt)
@@ -35,16 +29,22 @@ doc_DATA =
 endif
 
 EXTRA_DIST+= doc/asciidoc-helper.sh			\
-             $(html_in) $(man_in) $(txt_in)			\
-             doc/state-contents.txt				\
-             doc/torrc_format.txt 				\
+	     $(html_in) $(man_in) $(txt_in)			\
+	     doc/state-contents.txt				\
+	     doc/torrc_format.txt 				\
 	     doc/TUNING						\
 	     doc/HACKING/README.1st.md				\
 	     doc/HACKING/CodingStandards.md 			\
+	     doc/HACKING/CodingStandardsRust.md			\
+	     doc/HACKING/CodeStructure.md			\
+	     doc/HACKING/Fuzzing.md				\
 	     doc/HACKING/GettingStarted.md 			\
+	     doc/HACKING/GettingStartedRust.md 			\
 	     doc/HACKING/HelpfulTools.md 			\
 	     doc/HACKING/HowToReview.md  			\
+	     doc/HACKING/Module.md				\
 	     doc/HACKING/ReleasingTor.md                        \
+	     doc/HACKING/Tracing.md				\
 	     doc/HACKING/WritingTests.md
 
 docdir = @docdir@
@@ -65,11 +65,13 @@ doc/tor.1.in: doc/tor.1.txt
 doc/torify.1.in: doc/torify.1.txt
 doc/tor-gencert.1.in: doc/tor-gencert.1.txt
 doc/tor-resolve.1.in: doc/tor-resolve.1.txt
+doc/tor-print-ed-signing-cert.1.in: doc/tor-print-ed-signing-cert.1.txt
 
 doc/tor.html.in: doc/tor.1.txt
 doc/torify.html.in: doc/torify.1.txt
 doc/tor-gencert.html.in: doc/tor-gencert.1.txt
 doc/tor-resolve.html.in: doc/tor-resolve.1.txt
+doc/tor-print-ed-signing-cert.html.in: doc/tor-print-ed-signing-cert.1.txt
 
 # use config.status to swap all machine-specific magic strings
 # in the asciidoc with their replacements.
@@ -83,11 +85,13 @@ $(asciidoc_product) :
 doc/tor.html: doc/tor.html.in
 doc/tor-gencert.html: doc/tor-gencert.html.in
 doc/tor-resolve.html: doc/tor-resolve.html.in
+doc/tor-print-ed-signing-cert.html: doc/tor-print-ed-signing-cert.html.in
 doc/torify.html: doc/torify.html.in
 
 doc/tor.1: doc/tor.1.in
 doc/tor-gencert.1: doc/tor-gencert.1.in
 doc/tor-resolve.1: doc/tor-resolve.1.in
+doc/tor-print-ed-signing-cert.1: doc/tor-print-ed-signing-cert.1.in
 doc/torify.1: doc/torify.1.in
 
 CLEANFILES+= $(asciidoc_product)
diff --git a/doc/tor-print-ed-signing-cert.1.txt b/doc/tor-print-ed-signing-cert.1.txt
new file mode 100644
index 0000000000..1a3109df95
--- /dev/null
+++ b/doc/tor-print-ed-signing-cert.1.txt
@@ -0,0 +1,32 @@
+// Copyright (c) The Tor Project, Inc.
+// See LICENSE for licensing information
+// This is an asciidoc file used to generate the manpage/html reference.
+// Learn asciidoc on http://www.methods.co.nz/asciidoc/userguide.html
+:man source:   Tor
+:man manual:   Tor Manual
+tor-print-ed-signing-cert(1)
+============================
+Tor Project, Inc.
+
+NAME
+----
+tor-print-ed-signing-cert - print expiration date of ed25519 signing certificate
+
+SYNOPSIS
+--------
+**tor-print-ed-signing-cert** __<path to ed25519_signing_cert file>__
+
+DESCRIPTION
+-----------
+**tor-print-ed-signing-cert** is utility program for Tor relay operators to
+check expiration date of ed25519 signing certificate.
+
+SEE ALSO
+--------
+**tor**(1) +
+
+https://spec.torproject.org/cert-spec
+
+AUTHORS
+-------
+Roger Dingledine <arma@mit.edu>, Nick Mathewson <nickm@alum.mit.edu>.
diff --git a/doc/tor-resolve.1.txt b/doc/tor-resolve.1.txt
index 30e16d5daa..f1f8f77a42 100644
--- a/doc/tor-resolve.1.txt
+++ b/doc/tor-resolve.1.txt
@@ -47,7 +47,7 @@ SEE ALSO
 --------
 **tor**(1), **torify**(1). +
 
-See doc/socks-extensions.txt in the Tor package for protocol details.
+For protocol details, see: https://spec.torproject.org/socks-extensions
 
 AUTHORS
 -------
diff --git a/doc/tor.1.txt b/doc/tor.1.txt
index a7ee7d11ca..21b482802e 100644
--- a/doc/tor.1.txt
+++ b/doc/tor.1.txt
@@ -88,6 +88,10 @@ COMMAND-LINE OPTIONS
     List all valid options that are scheduled to become obsolete in a
     future version. (This is a warning, not a promise.)
 
+[[opt-list-modules]] **--list-modules**::
+    For each optional module, list whether or not it has been compiled
+    into Tor.  (Any module not listed is not optional in this version of Tor.)
+
 [[opt-version]] **--version**::
     Display Tor version and exit.
 
@@ -128,6 +132,16 @@ COMMAND-LINE OPTIONS
     the passphrase, including any trailing newlines.
     Default: read from the terminal.
 
+[[opt-key-expiration]] **--key-expiration** [**purpose**]::
+    The **purpose** specifies which type of key certificate to determine
+    the expiration of.  The only currently recognised **purpose** is
+    "sign". +
+ +
+    Running "tor --key-expiration sign" will attempt to find your signing
+    key certificate and will output, both in the logs as well as to stdout,
+    the signing key certificate's expiration time in ISO-8601 format.
+    For example, the output sent to stdout will be of the form:
+    "signing-cert-expiry: 2017-07-25 08:30:15 UTC"
 
 Other options can be specified on the command-line in the format "--option
 value", in the format "option value", or in a configuration file.  For
@@ -153,6 +167,13 @@ values. To split one configuration entry into multiple lines, use a single
 backslash character (\) before the end of the line.  Comments can be used in
 such multiline entries, but they must start at the beginning of a line.
 
+Configuration options can be imported from files or folders using the %include
+option with the value being a path. If the path is a file, the options from the
+file will be parsed as if they were written where the %include option is. If
+the path is a folder, all files on that folder will be parsed following lexical
+order. Files starting with a dot are ignored. Files on subfolders are ignored.
+The %include option can be used recursively.
+
 By default, an option on the command line overrides an option found in the
 configuration file, and an option in a configuration file overrides one in
 the defaults file.
@@ -176,7 +197,7 @@ forward slash (/) in the configuration file and on the command line.
 GENERAL OPTIONS
 ---------------
 
-[[BandwidthRate]] **BandwidthRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[BandwidthRate]] **BandwidthRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     A token bucket limits the average incoming bandwidth usage on this node
     to the specified number of bytes per second, and the average outgoing
     bandwidth usage to that same value.  If you want to run a relay in the
@@ -185,6 +206,9 @@ GENERAL OPTIONS
     course, more is better; we recommend at least 250 KBytes (2 mbits) if
     possible.  (Default: 1 GByte) +
  +
+    Note that this option, and other bandwidth-limiting options, apply to TCP
+    data only: They do not count TCP headers or DNS traffic. +
+ +
     With this option, and in other options that take arguments in bytes,
     KBytes, and so on, other formats are also supported. Notably, "KBytes" can
     also be written as "kilobytes" or "kb"; "MBytes" can be written as
@@ -195,43 +219,48 @@ GENERAL OPTIONS
     To avoid confusion, we recommend writing "bytes" or "bits" explicitly,
     since it's easy to forget that "B" means bytes, not bits.
 
-[[BandwidthBurst]] **BandwidthBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[BandwidthBurst]] **BandwidthBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     Limit the maximum token bucket size (also known as the burst) to the given
     number of bytes in each direction. (Default: 1 GByte)
 
-[[MaxAdvertisedBandwidth]] **MaxAdvertisedBandwidth** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[MaxAdvertisedBandwidth]] **MaxAdvertisedBandwidth** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     If set, we will not advertise more than this amount of bandwidth for our
     BandwidthRate. Server operators who want to reduce the number of clients
     who ask to build circuits through them (since this is proportional to
     advertised bandwidth rate) can thus reduce the CPU demands on their server
     without impacting network performance.
 
-[[RelayBandwidthRate]] **RelayBandwidthRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[RelayBandwidthRate]] **RelayBandwidthRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     If not 0, a separate token bucket limits the average incoming bandwidth
     usage for \_relayed traffic_ on this node to the specified number of bytes
     per second, and the average outgoing bandwidth usage to that same value.
     Relayed traffic currently is calculated to include answers to directory
-    requests, but that may change in future versions. (Default: 0)
+    requests, but that may change in future versions. They do not include directory
+    fetches by the relay (from authority or other relays), because that is considered
+    "client" activity.  (Default: 0)
 
-[[RelayBandwidthBurst]] **RelayBandwidthBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[RelayBandwidthBurst]] **RelayBandwidthBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     If not 0, limit the maximum token bucket size (also known as the burst) for
     \_relayed traffic_ to the given number of bytes in each direction.
-    (Default: 0)
+    They do not include directory fetches by the relay (from authority
+    or other relays), because that is considered "client" activity. (Default: 0)
 
-[[PerConnBWRate]] **PerConnBWRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
-    If set, do separate rate limiting for each connection from a non-relay.
-    You should never need to change this value, since a network-wide value is
-    published in the consensus and your relay will use that value. (Default: 0)
+[[PerConnBWRate]] **PerConnBWRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
+    If this option is set manually, or via the "perconnbwrate" consensus
+    field, Tor will use it for separate rate limiting for each connection
+    from a non-relay. (Default: 0)
 
-[[PerConnBWBurst]] **PerConnBWBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
-    If set, do separate rate limiting for each connection from a non-relay.
-    You should never need to change this value, since a network-wide value is
-    published in the consensus and your relay will use that value. (Default: 0)
+[[PerConnBWBurst]] **PerConnBWBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
+    If this option is set manually, or via the "perconnbwburst" consensus
+    field, Tor will use it for separate rate limiting for each connection
+    from a non-relay. (Default: 0)
 
 [[ClientTransportPlugin]] **ClientTransportPlugin** __transport__ socks4|socks5 __IP__:__PORT__::
 **ClientTransportPlugin** __transport__ exec __path-to-binary__ [options]::
     In its first form, when set along with a corresponding Bridge line, the Tor
-    client forwards its traffic to a SOCKS-speaking proxy on "IP:PORT". It's the
+    client forwards its traffic to a SOCKS-speaking proxy on "IP:PORT".
+    (IPv4 addresses should written as-is; IPv6 addresses should be wrapped in
+    square brackets.) It's the
     duty of that proxy to properly forward the traffic to the bridge. +
  +
     In its second form, when set along with a corresponding Bridge line, the Tor
@@ -248,7 +277,8 @@ GENERAL OPTIONS
 [[ServerTransportListenAddr]] **ServerTransportListenAddr** __transport__ __IP__:__PORT__::
     When this option is set, Tor will suggest __IP__:__PORT__ as the
     listening address of any pluggable transport proxy that tries to
-    launch __transport__.
+    launch __transport__. (IPv4 addresses should written as-is; IPv6
+    addresses should be wrapped in square brackets.)
 
 [[ServerTransportOptions]] **ServerTransportOptions** __transport__ __k=v__ __k=v__ ...::
     When this option is set, Tor will pass the __k=v__ parameters to
@@ -277,15 +307,24 @@ GENERAL OPTIONS
     descriptors as the OS will allow (you can find this by "ulimit -H -n").
     If this number is less than ConnLimit, then Tor will refuse to start. +
  +
-    You probably don't need to adjust this. It has no effect on Windows
-    since that platform lacks getrlimit(). (Default: 1000)
+    Tor relays need thousands of sockets, to connect to every other relay.
+    If you are running a private bridge, you can reduce the number of sockets
+    that Tor uses. For example, to limit Tor to 500 sockets, run
+    "ulimit -n 500" in a shell. Then start tor in the same shell, with
+    **ConnLimit 500**. You may also need to set **DisableOOSCheck 0**. +
+ +
+    Unless you have severely limited sockets, you probably don't need to
+    adjust **ConnLimit** itself. It has no effect on Windows, since that
+    platform lacks getrlimit(). (Default: 1000)
 
 [[DisableNetwork]] **DisableNetwork** **0**|**1**::
     When this option is set, we don't listen for or accept any connections
     other than controller connections, and we close (and don't reattempt)
     any outbound
     connections.  Controllers sometimes use this option to avoid using
-    the network until Tor is fully configured. (Default: 0)
+    the network until Tor is fully configured.  Tor will make still certain
+    network-related calls (like DNS lookups) as a part of its configuration
+    process, even if DisableNetwork is set. (Default: 0)
 
 [[ConstrainedSockets]] **ConstrainedSockets** **0**|**1**::
     If set, Tor will tell the kernel to attempt to shrink the buffers for all
@@ -338,17 +377,10 @@ GENERAL OPTIONS
         Unix domain sockets only: Do not insist that the directory
         that holds the socket be read-restricted.
 
-[[ControlListenAddress]] **ControlListenAddress** __IP__[:__PORT__]::
-    Bind the controller listener to this address. If you specify a port, bind
-    to this port rather than the one specified in ControlPort. We strongly
-    recommend that you leave this alone unless you know what you're doing,
-    since giving attackers access to your control listener is really
-    dangerous. This directive can be specified multiple
-    times to bind to multiple addresses/ports.  (Default: 127.0.0.1)
-
 [[ControlSocket]] **ControlSocket** __Path__::
     Like ControlPort, but listens on a Unix domain socket, rather than a TCP
-    socket. '0' disables ControlSocket (Unix and Unix-like systems only.)
+    socket. '0' disables ControlSocket. (Unix and Unix-like systems only.)
+    (Default: 0)
 
 [[ControlSocketsGroupWritable]] **ControlSocketsGroupWritable** **0**|**1**::
     If this option is set to 0, don't allow the filesystem group to read and
@@ -390,14 +422,29 @@ GENERAL OPTIONS
     file readable by the default GID. (Default: 0)
 
 [[DataDirectory]] **DataDirectory** __DIR__::
-    Store working data in DIR (Default: @LOCALSTATEDIR@/lib/tor)
+    Store working data in DIR. Can not be changed while tor is running.
+    (Default: ~/.tor if your home directory is not /; otherwise,
+    @LOCALSTATEDIR@/lib/tor.  On Windows, the default is
+    your ApplicationData folder.)
 
 [[DataDirectoryGroupReadable]] **DataDirectoryGroupReadable** **0**|**1**::
     If this option is set to 0, don't allow the filesystem group to read the
     DataDirectory. If the option is set to 1, make the DataDirectory readable
     by the default GID. (Default: 0)
 
-[[FallbackDir]] **FallbackDir** __address__:__port__ orport=__port__ id=__fingerprint__ [weight=__num__] [ipv6=__address__:__orport__]::
+[[CacheDirectory]] **CacheDirectory** __DIR__::
+    Store cached directory data in DIR. Can not be changed while tor is
+    running.
+    (Default: uses the value of DataDirectory.)
+
+[[CacheDirectoryGroupReadable]] **CacheDirectoryGroupReadable** **0**|**1**|**auto**::
+    If this option is set to 0, don't allow the filesystem group to read the
+    CacheDirectory. If the option is set to 1, make the CacheDirectory readable
+    by the default GID. If the option is "auto", then we use the
+    setting for DataDirectoryGroupReadable when the CacheDirectory is the
+    same as the DataDirectory, and 0 otherwise. (Default: auto)
+
+[[FallbackDir]] **FallbackDir** __ipv4address__:__port__ orport=__port__ id=__fingerprint__ [weight=__num__] [ipv6=**[**__ipv6address__**]**:__orport__]::
     When we're unable to connect to any directory cache for directory info
     (usually because we don't know about any yet) we try a directory authority.
     Clients also simultaneously try a FallbackDir, to avoid hangs on client
@@ -413,7 +460,7 @@ GENERAL OPTIONS
     FallbackDir line is present, it replaces the hard-coded FallbackDirs,
     regardless of the value of UseDefaultFallbackDirs.) (Default: 1)
 
-[[DirAuthority]] **DirAuthority** [__nickname__] [**flags**] __address__:__port__ __fingerprint__::
+[[DirAuthority]] **DirAuthority** [__nickname__] [**flags**] __ipv4address__:__port__ __fingerprint__::
     Use a nonstandard authoritative directory server at the provided address
     and port, with the specified key fingerprint. This option can be repeated
     many times, for multiple authoritative directory servers. Flags are
@@ -427,13 +474,16 @@ GENERAL OPTIONS
     with probability proportional to that weight (default 1.0). If a
     flag "v3ident=**fp**" is given, the dirserver is a v3 directory authority
     whose v3 long-term signing key has the fingerprint **fp**. Lastly,
-    if an "ipv6=__address__:__orport__" flag is present, then the directory
+    if an "ipv6=**[**__ipv6address__**]**:__orport__" flag is present, then
+    the directory
     authority is listening for IPv6 connections on the indicated IPv6 address
     and OR Port. +
  +
-    Tor will contact the authority at __address__:__port__ (the DirPort) to
-    download directory documents. If an IPv6 address is supplied, Tor will
-    also download directory documents at the IPv6 address on the DirPort. +
+    Tor will contact the authority at __ipv4address__ to
+    download directory documents. The provided __port__ value is a dirport;
+    clients ignore this in favor of the specified "orport=" value.  If an
+    IPv6 ORPort is supplied, Tor will
+    also download directory documents at the IPv6 ORPort. +
  +
     If no **DirAuthority** line is given, Tor will use the default directory
     authorities. NOTE: this option is intended for setting up a private Tor
@@ -448,9 +498,9 @@ GENERAL OPTIONS
     should be 1.0 or less. The default is less than 1, to reduce load on
     authorities. (Default: 0.1)
 
-[[AlternateDirAuthority]] **AlternateDirAuthority** [__nickname__] [**flags**] __address__:__port__ __fingerprint__ +
+[[AlternateDirAuthority]] **AlternateDirAuthority** [__nickname__] [**flags**] __ipv4address__:__port__ __fingerprint__ +
 
-[[AlternateBridgeAuthority]] **AlternateBridgeAuthority** [__nickname__] [**flags**] __address__:__port__ __ fingerprint__::
+[[AlternateBridgeAuthority]] **AlternateBridgeAuthority** [__nickname__] [**flags**] __ipv4address__:__port__ __ fingerprint__::
     These options behave as DirAuthority, but they replace fewer of the
     default directory authorities. Using
     AlternateDirAuthority replaces the default Tor directory authorities, but
@@ -465,7 +515,8 @@ GENERAL OPTIONS
     not supported. We believe that this feature works on modern Gnu/Linux
     distributions, and that it should work on *BSD systems (untested). This
     option requires that you start your Tor as root, and you should use the
-    **User** option to properly reduce Tor's privileges. (Default: 0)
+    **User** option to properly reduce Tor's privileges.
+    Can not be changed while tor is running. (Default: 0)
 
 [[DisableDebuggerAttachment]] **DisableDebuggerAttachment** **0**|**1**::
    If set to 1, Tor will attempt to prevent basic debugging attachment attempts
@@ -505,22 +556,33 @@ GENERAL OPTIONS
     (Default: 1)
 
 [[FetchUselessDescriptors]] **FetchUselessDescriptors** **0**|**1**::
-    If set to 1, Tor will fetch every non-obsolete descriptor from the
-    authorities that it hears about. Otherwise, it will avoid fetching useless
-    descriptors, for example for routers that are not running. This option is
-    useful if you're using the contributed "exitlist" script to enumerate Tor
-    nodes that exit to certain addresses. (Default: 0)
+    If set to 1, Tor will fetch every consensus flavor, and all server
+    descriptors and authority certificates referenced by those consensuses,
+    except for extra info descriptors. When this option is 1, Tor will also
+    keep fetching descriptors, even when idle.
+    If set to 0, Tor will avoid fetching useless descriptors: flavors that it
+    is not using to build circuits, and authority certificates it does not
+    trust. When Tor hasn't built any application circuits, it will go idle,
+    and stop fetching descriptors. This option is useful if you're using a
+    tor client with an external parser that uses a full consensus.
+    This option fetches all documents except extrainfo descriptors,
+    **DirCache** fetches and serves all documents except extrainfo
+    descriptors, **DownloadExtraInfo*** fetches extrainfo documents, and serves
+    them if **DirCache** is on, and **UseMicrodescriptors** changes the
+    flavour of consensues and descriptors that is fetched and used for
+    building circuits. (Default: 0)
 
 [[HTTPProxy]] **HTTPProxy** __host__[:__port__]::
     Tor will make all its directory requests through this host:port (or host:80
     if port is not specified), rather than connecting directly to any directory
-    servers.
+    servers. (DEPRECATED: As of 0.3.1.0-alpha you should use HTTPSProxy.)
 
 [[HTTPProxyAuthenticator]] **HTTPProxyAuthenticator** __username:password__::
     If defined, Tor will use this username:password for Basic HTTP proxy
     authentication, as in RFC 2617. This is currently the only form of HTTP
     proxy authentication that Tor supports; feel free to submit a patch if you
-    want it to support others.
+    want it to support others. (DEPRECATED: As of 0.3.1.0-alpha you should use
+    HTTPSProxyAuthenticator.)
 
 [[HTTPSProxy]] **HTTPSProxy** __host__[:__port__]::
     Tor will make all its OR (SSL) connections through this host:port (or
@@ -538,7 +600,23 @@ GENERAL OPTIONS
 [[Sandbox]] **Sandbox** **0**|**1**::
     If set to 1, Tor will run securely through the use of a syscall sandbox.
     Otherwise the sandbox will be disabled. The option is currently an
-    experimental feature. (Default: 0)
+    experimental feature. It only works on Linux-based operating systems,
+    and only when Tor has been built with the libseccomp library. This option
+    can not be changed while tor is running.
+ +
+    When the Sandbox is 1, the following options can not be changed when tor
+    is running:
+    Address
+    ConnLimit
+    CookieAuthFile
+    DirPortFrontPage
+    ExtORPortCookieAuthFile
+    Logs
+    ServerDNSResolvConfFile
+    Tor must remain in client or server mode (some changes to ClientOnly and
+    ORPort are not allowed).
+    ClientOnionAuthDir and any files in it won't reload on HUP signal.
+    (Default: 0)
 
 [[Socks4Proxy]] **Socks4Proxy** __host__[:__port__]::
     Tor will make all OR connections through the SOCKS 4 proxy at host:port
@@ -555,16 +633,14 @@ GENERAL OPTIONS
     in accordance to RFC 1929. Both username and password must be between 1 and
     255 characters.
 
-[[SocksSocketsGroupWritable]] **SocksSocketsGroupWritable** **0**|**1**::
+[[UnixSocksGroupWritable]] **UnixSocksGroupWritable** **0**|**1**::
     If this option is set to 0, don't allow the filesystem group to read and
-    write unix sockets (e.g. SocksSocket). If the option is set to 1, make
-    the SocksSocket socket readable and writable by the default GID. (Default: 0)
+    write unix sockets (e.g. SocksPort unix:). If the option is set to 1, make
+    the Unix socket readable and writable by the default GID. (Default: 0)
 
 [[KeepalivePeriod]] **KeepalivePeriod** __NUM__::
     To keep firewalls from expiring connections, send a padding keepalive cell
-    every NUM seconds on open connections that are in use. If the connection
-    has no open circuits, it will instead be closed after NUM seconds of
-    idleness. (Default: 5 minutes)
+    every NUM seconds on open connections that are in use. (Default: 5 minutes)
 
 [[Log]] **Log** __minSeverity__[-__maxSeverity__] **stderr**|**stdout**|**syslog**::
     Send all messages between __minSeverity__ and __maxSeverity__ to the standard
@@ -597,7 +673,8 @@ GENERAL OPTIONS
  +
     The currently recognized domains are: general, crypto, net, config, fs,
     protocol, mm, http, app, control, circ, rend, bug, dir, dirserv, or, edge,
-    acct, hist, and handshake.  Domain names are case-insensitive. +
+    acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, and dos.
+    Domain names are case-insensitive. +
  +
     For example, "`Log [handshake]debug [~net,~mm]info notice stdout`" sends
     to stdout: all handshake messages of any severity, all info-and-higher
@@ -609,7 +686,7 @@ GENERAL OPTIONS
     message currently has at least one domain; most currently have exactly
     one.  This doesn't affect controller log messages. (Default: 0)
 
-[[MaxUnparseableDescSizeToLog]] **MaxUnparseableDescSizeToLog** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**::
+[[MaxUnparseableDescSizeToLog]] **MaxUnparseableDescSizeToLog** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**::
     Unparseable descriptors (e.g. for votes, consensuses, routers) are logged
     in separate files by hash, up to the specified size in total.  Note that
     only files logged during the lifetime of this Tor process count toward the
@@ -621,27 +698,41 @@ GENERAL OPTIONS
     is only useful when you have multiple network interfaces, and you want all
     of Tor's outgoing connections to use a single one. This option may
     be used twice, once with an IPv4 address and once with an IPv6 address.
+    IPv6 addresses should be wrapped in square brackets.
     This setting will be ignored for connections to the loopback addresses
-    (127.0.0.0/8 and ::1).
+    (127.0.0.0/8 and ::1), and is not used for DNS requests as well.
+
+[[OutboundBindAddressOR]] **OutboundBindAddressOR** __IP__::
+    Make all outbound non-exit (relay and other) connections
+    originate from the IP address specified. This option overrides
+    **OutboundBindAddress** for the same IP version. This option may
+    be used twice, once with an IPv4 address and once with an IPv6
+    address. IPv6 addresses should be wrapped in square brackets.
+    This setting will be ignored for connections to the loopback
+    addresses (127.0.0.0/8 and ::1).
+
+[[OutboundBindAddressExit]] **OutboundBindAddressExit** __IP__::
+    Make all outbound exit connections originate from the IP address
+    specified. This option overrides **OutboundBindAddress** for the
+    same IP version. This option may be used twice, once with an IPv4
+    address and once with an IPv6 address.
+    IPv6 addresses should be wrapped in square brackets.
+    This setting will be ignored
+    for connections to the loopback addresses (127.0.0.0/8 and ::1).
 
 [[PidFile]] **PidFile** __FILE__::
     On startup, write our PID to FILE. On clean shutdown, remove
-    FILE.
+    FILE. Can not be changed while tor is running.
 
 [[ProtocolWarnings]] **ProtocolWarnings** **0**|**1**::
     If 1, Tor will log with severity \'warn' various cases of other parties not
     following the Tor specification. Otherwise, they are logged with severity
     \'info'. (Default: 0)
 
-[[PredictedPortsRelevanceTime]] **PredictedPortsRelevanceTime** __NUM__::
-    Set how long, after the client has made an anonymized connection to a
-    given port, we will try to make sure that we build circuits to
-    exits that support that port. The maximum value for this option is 1
-    hour. (Default: 1 hour)
-
 [[RunAsDaemon]] **RunAsDaemon** **0**|**1**::
     If 1, Tor forks and daemonizes to the background. This option has no effect
     on Windows; instead you should use the --service command-line option.
+    Can not be changed while tor is running.
     (Default: 0)
 
 [[LogTimeGranularity]] **LogTimeGranularity** __NUM__::
@@ -658,7 +749,13 @@ GENERAL OPTIONS
 
 [[SyslogIdentityTag]] **SyslogIdentityTag** __tag__::
     When logging to syslog, adds a tag to the syslog identity such that
-    log entries are marked with "Tor-__tag__".  (Default: none)
+    log entries are marked with "Tor-__tag__". Can not be changed while tor is
+    running. (Default: none)
+
+[[AndroidIdentityTag]] **AndroidIdentityTag** __tag__::
+    When logging to Android's logging subsystem, adds a tag to the log identity
+    such that log entries are marked with "Tor-__tag__". Can not be changed while
+    tor is running. (Default: none)
 
 [[SafeLogging]] **SafeLogging** **0**|**1**|**relay**::
     Tor can scrub potentially sensitive strings from log messages (e.g.
@@ -669,10 +766,13 @@ GENERAL OPTIONS
     If this option is set to 0, Tor will not perform any scrubbing, if it is
     set to 1, all potentially sensitive strings are replaced. If it is set to
     relay, all log messages generated when acting as a relay are sanitized, but
-    all messages generated when acting as a client are not. (Default: 1)
+    all messages generated when acting as a client are not.
+    Note: Tor may not heed this option when logging at log levels below Notice.
+    (Default: 1)
 
 [[User]] **User** __Username__::
     On startup, setuid to this user and setgid to their primary group.
+    Can not be changed while tor is running.
 
 [[KeepBindCapabilities]] **KeepBindCapabilities** **0**|**1**|**auto**::
     On Linux, when we are started as root and we switch our identity using
@@ -680,37 +780,38 @@ GENERAL OPTIONS
     try to retain our ability to bind to low ports.  If this value is 1, we
     try to keep the capability; if it is 0 we do not; and if it is **auto**,
     we keep the capability only if we are configured to listen on a low port.
+    Can not be changed while tor is running.
     (Default: auto.)
 
 [[HardwareAccel]] **HardwareAccel** **0**|**1**::
     If non-zero, try to use built-in (static) crypto hardware acceleration when
-    available. (Default: 0)
+    available. Can not be changed while tor is running. (Default: 0)
 
 [[AccelName]] **AccelName** __NAME__::
     When using OpenSSL hardware crypto acceleration attempt to load the dynamic
     engine of this name. This must be used for any dynamic hardware engine.
-    Names can be verified with the openssl engine command.
+    Names can be verified with the openssl engine command. Can not be changed
+    while tor is running.
 
 [[AccelDir]] **AccelDir** __DIR__::
     Specify this option if using dynamic hardware acceleration and the engine
     implementation library resides somewhere other than the OpenSSL default.
+    Can not be changed while tor is running.
 
 [[AvoidDiskWrites]] **AvoidDiskWrites** **0**|**1**::
     If non-zero, try to write to disk less frequently than we would otherwise.
     This is useful when running on flash memory or other media that support
     only a limited number of writes. (Default: 0)
 
-[[CircuitPriorityHalflife]] **CircuitPriorityHalflife** __NUM1__::
+[[CircuitPriorityHalflife]] **CircuitPriorityHalflife** __NUM__::
     If this value is set, we override the default algorithm for choosing which
-    circuit's cell to deliver or relay next. When the value is 0, we
-    round-robin between the active circuits on a connection, delivering one
-    cell from each in turn. When the value is positive, we prefer delivering
-    cells from whichever connection has the lowest weighted cell count, where
-    cells are weighted exponentially according to the supplied
-    CircuitPriorityHalflife value (in seconds). If this option is not set at
-    all, we use the behavior recommended in the current consensus
-    networkstatus. This is an advanced option; you generally shouldn't have
-    to mess with it. (Default: not set)
+    circuit's cell to deliver or relay next. It is delivered first to the
+    circuit that has the lowest weighted cell count, where cells are weighted
+    exponentially according to this value (in seconds). If the value is -1, it
+    is taken from the consensus if possible else it will fallback to the
+    default value of 30. Minimum: 1, Maximum: 2147483647. This can be defined
+    as a float value. This is an advanced option; you generally shouldn't have
+    to mess with it. (Default: -1)
 
 [[CountPrivateBandwidth]] **CountPrivateBandwidth** **0**|**1**::
     If this option is set, then Tor's rate-limiting applies not only to
@@ -718,27 +819,63 @@ GENERAL OPTIONS
     127.0.0.1 or 10.0.0.1.  This is mostly useful for debugging
     rate-limiting.  (Default: 0)
 
+[[ExtendByEd25519ID]] **ExtendByEd25519ID** **0**|**1**|**auto**::
+    If this option is set to 1, we always try to include a relay's Ed25519 ID
+    when telling the proceeding relay in a circuit to extend to it.
+    If this option is set to 0, we never include Ed25519 IDs when extending
+    circuits.  If the option is set to "default", we obey a
+    parameter in the consensus document. (Default: auto)
+
+[[NoExec]] **NoExec** **0**|**1**::
+    If this option is set to 1, then Tor will never launch another
+    executable, regardless of the settings of ClientTransportPlugin
+    or ServerTransportPlugin.  Once this option has been set to 1,
+    it cannot be set back to 0 without restarting Tor. (Default: 0)
+
+[[Schedulers]] **Schedulers** **KIST**|**KISTLite**|**Vanilla**::
+    Specify the scheduler type that tor should use. The scheduler is
+    responsible for moving data around within a Tor process. This is an ordered
+    list by priority which means that the first value will be tried first and if
+    unavailable, the second one is tried and so on. It is possible to change
+    these values at runtime. This option mostly effects relays, and most
+    operators should leave it set to its default value.
+    (Default: KIST,KISTLite,Vanilla)
+ +
+    The possible scheduler types are:
+ +
+    **KIST**: Kernel-Informed Socket Transport. Tor will use TCP information
+    from the kernel to make informed decisions regarding how much data to send
+    and when to send it. KIST also handles traffic in batches (see
+    KISTSchedRunInterval) in order to improve traffic prioritization decisions.
+    As implemented, KIST will only work on Linux kernel version 2.6.39 or
+    higher.
+ +
+    **KISTLite**: Same as KIST but without kernel support. Tor will use all
+    the same mechanics as with KIST, including the batching, but its decisions
+    regarding how much data to send will not be as good. KISTLite will work on
+    all kernels and operating systems, and the majority of the benefits of KIST
+    are still realized with KISTLite.
+ +
+    **Vanilla**: The scheduler that Tor used before KIST was implemented. It
+    sends as much data as possible, as soon as possible. Vanilla will work on
+    all kernels and operating systems.
+
+[[KISTSchedRunInterval]] **KISTSchedRunInterval** __NUM__ **msec**::
+    If KIST or KISTLite is used in the Schedulers option, this controls at which
+    interval the scheduler tick is. If the value is 0 msec, the value is taken
+    from the consensus if possible else it will fallback to the default 10
+    msec. Maximum possible value is 100 msec. (Default: 0 msec)
+
+[[KISTSockBufSizeFactor]] **KISTSockBufSizeFactor** __NUM__::
+    If KIST is used in Schedulers, this is a multiplier of the per-socket
+    limit calculation of the KIST algorithm. (Default: 1.0)
+
 CLIENT OPTIONS
 --------------
 
 The following options are useful only for clients (that is, if
-**SocksPort**, **TransPort**, **DNSPort**, or **NATDPort** is non-zero):
-
-[[AllowInvalidNodes]] **AllowInvalidNodes** **entry**|**exit**|**middle**|**introduction**|**rendezvous**|**...**::
-    If some Tor servers are obviously not working right, the directory
-    authorities can manually mark them as invalid, meaning that it's not
-    recommended you use them for entry or exit positions in your circuits. You
-    can opt to use them in some circuit positions, though. The default is
-    "middle,rendezvous", and other choices are not advised.
-
-[[ExcludeSingleHopRelays]] **ExcludeSingleHopRelays** **0**|**1**::
-    This option controls whether circuits built by Tor will include relays with
-    the AllowSingleHopExits flag set to true. If ExcludeSingleHopRelays is set
-    to 0, these relays will be included. Note that these relays might be at
-    higher risk of being seized or observed, so they are not normally
-    included.  Also note that relatively few clients turn off this option,
-    so using these relays might make your client stand out.
-    (Default: 1)
+**SocksPort**, **HTTPTunnelPort**, **TransPort**, **DNSPort**, or
+**NATDPort** is non-zero):
 
 [[Bridge]] **Bridge** [__transport__] __IP__:__ORPort__ [__fingerprint__]::
     When set along with UseBridges, instructs Tor to use the relay at
@@ -753,7 +890,12 @@ The following options are useful only for clients (that is, if
     rather than connecting to the bridge directly. Some transports use a
     transport-specific method to work out the remote address to connect to.
     These transports typically ignore the "IP:ORPort" specified in the bridge
-    line.
+    line.  +
+ +
+    Tor passes any "key=val" settings to the pluggable transport proxy as
+    per-connection arguments when connecting to the bridge.  Consult
+    the documentation of the pluggable transport for details of what
+    arguments it supports.
 
 [[LearnCircuitBuildTimeout]] **LearnCircuitBuildTimeout** **0**|**1**::
     If 0, CircuitBuildTimeout adaptive learning is disabled. (Default: 1)
@@ -766,13 +908,15 @@ The following options are useful only for clients (that is, if
     LearnCircuitBuildTimeout is 0, this value is the only value used.
     (Default: 60 seconds)
 
-[[CircuitIdleTimeout]] **CircuitIdleTimeout** __NUM__::
-    If we have kept a clean (never used) circuit around for NUM seconds, then
-    close it. This way when the Tor client is entirely idle, it can expire all
-    of its circuits, and then expire its TLS connections. Also, if we end up
-    making a circuit that is not useful for exiting any of the requests we're
-    receiving, it won't forever take up a slot in the circuit list. (Default: 1
-    hour)
+[[CircuitsAvailableTimeout]] **CircuitsAvailableTimeout** __NUM__::
+    Tor will attempt to keep at least one open, unused circuit available for
+    this amount of time. This option governs how long idle circuits are kept
+    open, as well as the amount of time Tor will keep a circuit open to each
+    of the recently used ports. This way when the Tor client is entirely
+    idle, it can expire all of its circuits, and then expire its TLS
+    connections. Note that the actual timeout value is uniformly randomized
+    from the specified value to twice that amount. (Default: 30 minutes;
+    Max: 24 hours)
 
 [[CircuitStreamTimeout]] **CircuitStreamTimeout** __NUM__::
     If non-zero, this option overrides our internal timeout schedule for how
@@ -789,6 +933,22 @@ The following options are useful only for clients (that is, if
     and fast enough. The current behavior is simply that Tor is a client
     unless ORPort, ExtORPort, or DirPort are configured.) (Default: 0)
 
+[[ConnectionPadding]] **ConnectionPadding** **0**|**1**|**auto**::
+    This option governs Tor's use of padding to defend against some forms of
+    traffic analysis. If it is set to 'auto', Tor will send padding only
+    if both the client and the relay support it. If it is set to 0, Tor will
+    not send any padding cells. If it is set to 1, Tor will still send padding
+    for client connections regardless of relay support. Only clients may set
+    this option. This option should be offered via the UI to mobile users
+    for use where bandwidth may be expensive.
+    (Default: auto)
+
+[[ReducedConnectionPadding]] **ReducedConnectionPadding** **0**|**1**::
+    If set to 1, Tor will not not hold OR connections open for very long,
+    and will send less padding on these connections. Only clients may set
+    this option. This option should be offered via the UI to mobile users
+    for use where bandwidth may be expensive. (Default: 0)
+
 [[ExcludeNodes]] **ExcludeNodes** __node__,__node__,__...__::
     A list of identity fingerprints, country codes, and address
     patterns of nodes to avoid when building a circuit. Country codes are
@@ -817,7 +977,7 @@ The following options are useful only for clients (that is, if
 [[ExcludeExitNodes]] **ExcludeExitNodes** __node__,__node__,__...__::
     A list of identity fingerprints, country codes, and address
     patterns of nodes to never use when picking an exit node---that is, a
-    node that delivers traffic for you outside the Tor network.   Note that any
+    node that delivers traffic for you *outside* the Tor network.   Note that any
     node listed in ExcludeNodes is automatically considered to be part of this
     list too.  See
     the **ExcludeNodes** option for more information on how to specify
@@ -834,7 +994,7 @@ The following options are useful only for clients (that is, if
 [[ExitNodes]] **ExitNodes** __node__,__node__,__...__::
     A list of identity fingerprints, country codes, and address
     patterns of nodes to use as exit node---that is, a
-    node that delivers traffic for you outside the Tor network.  See
+    node that delivers traffic for you *outside* the Tor network.  See
     the **ExcludeNodes** option for more information on how to specify nodes. +
  +
     Note that if you list too few nodes here, or if you exclude too many exit
@@ -842,7 +1002,7 @@ The following options are useful only for clients (that is, if
     if none of the exits you list allows traffic on port 80 or 443, you won't
     be able to browse the web. +
  +
-    Note also that not every circuit is used to deliver traffic outside of
+    Note also that not every circuit is used to deliver traffic *outside* of
     the Tor network.  It is normal to see non-exit circuits (such as those
     used to connect to hidden services, those that do directory fetches,
     those used for relay reachability self-tests, and so on) that end
@@ -852,7 +1012,7 @@ The following options are useful only for clients (that is, if
     The ExcludeNodes option overrides this option: any node listed in both
     ExitNodes and ExcludeNodes is treated as excluded. +
  +
-    The .exit address notation, if enabled via AllowDotExit, overrides
+    The .exit address notation, if enabled via MapAddress, overrides
     this option.
 
 [[EntryNodes]] **EntryNodes** __node__,__node__,__...__::
@@ -868,16 +1028,16 @@ The following options are useful only for clients (that is, if
     the **ExcludeNodes** option for more information on how to specify nodes.
 
 [[StrictNodes]] **StrictNodes** **0**|**1**::
-    If StrictNodes is set to 1, Tor will treat the ExcludeNodes option as a
-    requirement to follow for all the circuits you generate, even if doing so
-    will break functionality for you.  If StrictNodes is set to 0, Tor will
+    If StrictNodes is set to 1, Tor will treat solely the ExcludeNodes option
+    as a requirement to follow for all the circuits you generate, even if
+    doing so will break functionality for you (StrictNodes applies to neither
+    ExcludeExitNodes nor to ExitNodes).  If StrictNodes is set to 0, Tor will
     still try to avoid nodes in the ExcludeNodes list, but it will err on the
-    side of avoiding unexpected errors.  Specifically, StrictNodes 0 tells
-    Tor that it is okay to use an excluded node when it is *necessary* to
-    perform relay reachability self-tests, connect to
-    a hidden service, provide a hidden service to a client, fulfill a .exit
-    request, upload directory information, or download directory information.
-    (Default: 0)
+    side of avoiding unexpected errors.  Specifically, StrictNodes 0 tells Tor
+    that it is okay to use an excluded node when it is *necessary* to perform
+    relay reachability self-tests, connect to a hidden service, provide a
+    hidden service to a client, fulfill a .exit request, upload directory
+    information, or download directory information.  (Default: 0)
 
 [[FascistFirewall]] **FascistFirewall** **0**|**1**::
     If 1, Tor will only create outgoing connections to ORs running on ports
@@ -892,7 +1052,7 @@ The following options are useful only for clients (that is, if
     **FascistFirewall** is set. This option is deprecated; use ReachableAddresses
     instead. (Default: 80, 443)
 
-[[ReachableAddresses]] **ReachableAddresses** __ADDR__[/__MASK__][:__PORT__]...::
+[[ReachableAddresses]] **ReachableAddresses** __IP__[/__MASK__][:__PORT__]...::
     A comma-separated list of IP addresses and ports that your firewall allows
     you to connect to. The format is as for the addresses in ExitPolicy, except
     that "accept" is understood unless "reject" is explicitly provided. For
@@ -901,14 +1061,15 @@ The following options are useful only for clients (that is, if
     99, rejects port 80 connections to net 18, and accepts connections to port
     80 otherwise. (Default: \'accept \*:*'.)
 
-[[ReachableDirAddresses]] **ReachableDirAddresses** __ADDR__[/__MASK__][:__PORT__]...::
+[[ReachableDirAddresses]] **ReachableDirAddresses** __IP__[/__MASK__][:__PORT__]...::
     Like **ReachableAddresses**, a list of addresses and ports. Tor will obey
     these restrictions when fetching directory information, using standard HTTP
     GET requests. If not set explicitly then the value of
     **ReachableAddresses** is used. If **HTTPProxy** is set then these
-    connections will go through that proxy.
+    connections will go through that proxy.   (DEPRECATED: This option has
+    had no effect for some time.)
 
-[[ReachableORAddresses]] **ReachableORAddresses** __ADDR__[/__MASK__][:__PORT__]...::
+[[ReachableORAddresses]] **ReachableORAddresses** __IP__[/__MASK__][:__PORT__]...::
     Like **ReachableAddresses**, a list of addresses and ports. Tor will obey
     these restrictions when connecting to Onion Routers, using TLS/SSL. If not
     set explicitly then the value of **ReachableAddresses** is used. If
@@ -931,23 +1092,17 @@ The following options are useful only for clients (that is, if
     services can be configured to require authorization using the
     **HiddenServiceAuthorizeClient** option.
 
-[[CloseHSClientCircuitsImmediatelyOnTimeout]] **CloseHSClientCircuitsImmediatelyOnTimeout** **0**|**1**::
-    If 1, Tor will close unfinished hidden service client circuits
-    which have not moved closer to connecting to their destination
-    hidden service when their internal state has not changed for the
-    duration of the current circuit-build timeout.  Otherwise, such
-    circuits will be left open, in the hope that they will finish
-    connecting to their destination hidden services.  In either case,
-    another set of introduction and rendezvous circuits for the same
-    destination hidden service will be launched. (Default: 0)
-
-[[CloseHSServiceRendCircuitsImmediatelyOnTimeout]] **CloseHSServiceRendCircuitsImmediatelyOnTimeout** **0**|**1**::
-    If 1, Tor will close unfinished hidden-service-side rendezvous
-    circuits after the current circuit-build timeout.  Otherwise, such
-    circuits will be left open, in the hope that they will finish
-    connecting to their destinations.  In either case, another
-    rendezvous circuit for the same destination client will be
-    launched. (Default: 0)
+[[ClientOnionAuthDir]] **ClientOnionAuthDir** __path__::
+    Path to the directory containing v3 hidden service authorization files.
+    Each file is for a single onion address, and the files MUST have the suffix
+    ".auth_private" (i.e. "bob_onion.auth_private"). The content format MUST be:
+ +
+      <onion-address>:descriptor:x25519:<base32-encoded-privkey>
+ +
+    The <onion-address> MUST NOT have the ".onion" suffix. The
+    <base32-encoded-privkey> is the base32 representation of the raw key bytes
+    only (32 bytes for x25519). See Appendix G in the rend-spec-v3.txt file of
+    https://spec.torproject.org/[torspec] for more information.
 
 [[LongLivedPorts]] **LongLivedPorts** __PORTS__::
     A list of ports for services that tend to have long-running connections
@@ -1007,7 +1162,8 @@ The following options are useful only for clients (that is, if
     but never attach a new stream to a circuit that is too old.  For hidden
     services, this applies to the __last__ time a circuit was used, not the
     first. Circuits with streams constructed with SOCKS authentication via
-    SocksPorts that have **KeepAliveIsolateSOCKSAuth** ignore this value.
+    SocksPorts that have **KeepAliveIsolateSOCKSAuth** also remain alive
+    for MaxCircuitDirtiness seconds after carrying the last such stream.
     (Default: 10 minutes)
 
 [[MaxClientCircuitsPending]] **MaxClientCircuitsPending** __NUM__::
@@ -1056,7 +1212,9 @@ The following options are useful only for clients (that is, if
         Unsupported and force-disabled when using Unix domain sockets.)
     **IsolateSOCKSAuth**;;
         Don't share circuits with streams for which different
-        SOCKS authentication was provided. (On by default;
+        SOCKS authentication was provided. (For HTTPTunnelPort
+        connections, this option looks at the Proxy-Authorization and
+        X-Tor-Stream-Isolation headers. On by default;
         you can disable it with **NoIsolateSOCKSAuth**.)
     **IsolateClientProtocol**;;
         Don't share circuits with streams using a different protocol.
@@ -1069,8 +1227,9 @@ The following options are useful only for clients (that is, if
         Don't share circuits with streams targeting a different
         destination address.
     **KeepAliveIsolateSOCKSAuth**;;
-        If **IsolateSOCKSAuth** is enabled, keep alive circuits that have
-        streams with SOCKS authentication set indefinitely.
+        If **IsolateSOCKSAuth** is enabled, keep alive circuits while they have
+        at least one stream with SOCKS authentication active. After such a circuit
+        is idle for more than MaxCircuitDirtiness seconds, it can be closed.
     **SessionGroup=**__INT__;;
         If no other isolation rules would prevent it, allow streams
         on this port to share circuits with streams from every other
@@ -1078,6 +1237,7 @@ The following options are useful only for clients (that is, if
         on different SocksPorts, TransPorts, etc are always isolated from one
         another. This option overrides that behavior.)
 
+// Anchor only for formatting, not visible in the man page.
 [[OtherSocksPortFlags]]::
     Other recognized __flags__ for a SocksPort are:
     **NoIPv4Traffic**;;
@@ -1103,7 +1263,7 @@ The following options are useful only for clients (that is, if
         flag is not supported.
     **CacheIPv4DNS**;;
         Tells the client to remember IPv4 DNS answers we receive from exit
-        nodes via this connection. (On by default.)
+        nodes via this connection.
     **CacheIPv6DNS**;;
         Tells the client to remember IPv6 DNS answers we receive from exit
         nodes via this connection.
@@ -1118,8 +1278,8 @@ The following options are useful only for clients (that is, if
         nodes via this connection.
     **UseIPv4Cache**;;
         Tells the client to use any cached IPv4 DNS answers we have when making
-        requests via this connection. (NOTE: This option, along UseIPv6Cache
-        and UseDNSCache, can harm your anonymity, and probably
+        requests via this connection. (NOTE: This option, or UseIPv6Cache
+        or UseDNSCache, can harm your anonymity, and probably
         won't help performance as much as you might expect. Use with care!)
     **UseIPv6Cache**;;
         Tells the client to use any cached IPv6 DNS answers we have when making
@@ -1142,20 +1302,12 @@ The following options are useful only for clients (that is, if
         authentication" when IsolateSOCKSAuth is disabled, or when this
         option is set.
 
+// Anchor only for formatting, not visible in the man page.
+[[SocksPortFlagsMisc]]::
     Flags are processed left to right. If flags conflict, the last flag on the
     line is used, and all earlier flags are ignored. No error is issued for
     conflicting flags.
 
-[[SocksListenAddress]] **SocksListenAddress** __IP__[:__PORT__]::
-    Bind to this address to listen for connections from Socks-speaking
-    applications. (Default: 127.0.0.1) You can also specify a port (e.g.
-    192.168.0.1:9100). This directive can be specified multiple times to bind
-    to multiple addresses/ports.  (DEPRECATED: As of 0.2.3.x-alpha, you can
-    now use multiple SocksPort entries, and provide addresses for SocksPort
-    entries, so SocksListenAddress no longer has a purpose.  For backward
-    compatibility, SocksListenAddress is only allowed when SocksPort is just
-    a port number.)
-
 [[SocksPolicy]] **SocksPolicy** __policy__,__policy__,__...__::
     Set an entrance policy for this server, to limit who can connect to the
     SocksPort and DNSPort ports. The policies have the same form as exit
@@ -1168,11 +1320,14 @@ The following options are useful only for clients (that is, if
     2 minutes)
 
 [[TokenBucketRefillInterval]] **TokenBucketRefillInterval** __NUM__ [**msec**|**second**]::
-    Set the refill interval of Tor's token bucket to NUM milliseconds.
-    NUM must be between 1 and 1000, inclusive.  Note that the configured
-    bandwidth limits are still expressed in bytes per second: this
+    Set the refill delay interval of Tor's token bucket to NUM milliseconds.
+    NUM must be between 1 and 1000, inclusive.  When Tor is out of bandwidth,
+    on a connection or globally, it will wait up to this long before it tries
+    to use that connection again.
+    Note that bandwidth limits are still expressed in bytes per second: this
     option only affects the frequency with which Tor checks to see whether
-    previously exhausted connections may read again. (Default: 100 msec)
+    previously exhausted connections may read again.
+    Can not be changed while tor is running. (Default: 100 msec)
 
 [[TrackHostExits]] **TrackHostExits** __host__,__.domain__,__...__::
     For each value in the comma separated list, Tor will track recent
@@ -1206,17 +1361,8 @@ The following options are useful only for clients (that is, if
     to stick with them. This is desirable because constantly changing servers
     increases the odds that an adversary who owns some servers will observe a
     fraction of your paths. Entry Guards can not be used by Directory
-    Authorities, Single Onion Services, and Tor2web clients. In these cases,
-    the this option is ignored. (Default: 1)
-
-[[UseEntryGuardsAsDirGuards]] **UseEntryGuardsAsDirGuards** **0**|**1**::
-    If this option is set to 1, and UseEntryGuards is also set to 1,
-    we try to use our entry guards as directory
-    guards, and failing that, pick more nodes to act as our directory guards.
-    This helps prevent an adversary from enumerating clients. It's only
-    available for clients (non-relay, non-bridge) that aren't configured to
-    download any non-default directory material.  It doesn't currently
-    do anything when we lack a live consensus. (Default: 1)
+    Authorities or Single Onion Services. In these cases,
+    this option is ignored. (Default: 1)
 
 [[GuardfractionFile]] **GuardfractionFile** __FILENAME__::
     V3 authoritative directories only. Configures the location of the
@@ -1224,23 +1370,29 @@ The following options are useful only for clients (that is, if
     have been guards. (Default: unset)
 
 [[UseGuardFraction]] **UseGuardFraction** **0**|**1**|**auto**::
-    This torrc option specifies whether clients should use the
+    This option specifies whether clients should use the
     guardfraction information found in the consensus during path
     selection. If it's set to 'auto', clients will do what the
     UseGuardFraction consensus parameter tells them to do. (Default: auto)
 
 [[NumEntryGuards]] **NumEntryGuards** __NUM__::
     If UseEntryGuards is set to 1, we will try to pick a total of NUM routers
-    as long-term entries for our circuits. If NUM is 0, we try to learn
-    the number from the NumEntryGuards consensus parameter, and default
-    to 3 if the consensus parameter isn't set. (Default: 0)
+    as long-term entries for our circuits. If NUM is 0, we try to learn the
+    number from the guard-n-primary-guards-to-use consensus parameter, and
+    default to 1 if the consensus parameter isn't set. (Default: 0)
+
+[[NumPrimaryGuards]] **NumPrimaryGuards** __NUM__::
+    If UseEntryGuards is set to 1, we will try to pick NUM routers for our
+    primary guard list, which is the set of routers we strongly prefer when
+    connecting to the Tor network. If NUM is 0, we try to learn the number from
+    the guard-n-primary-guards consensus parameter, and default to 3 if the
+    consensus parameter isn't set. (Default: 0)
 
 [[NumDirectoryGuards]] **NumDirectoryGuards** __NUM__::
-    If UseEntryGuardsAsDirectoryGuards is enabled, we try to make sure we
-    have at least NUM routers to use as directory guards. If this option
-    is set to 0, use the value from the NumDirectoryGuards consensus
-    parameter, falling back to the value from NumEntryGuards if the
-    consensus parameter is 0 or isn't set. (Default: 0)
+    If UseEntryGuards is set to 1, we try to make sure we have at least NUM
+    routers to use as directory guards. If this option is set to 0, use the
+    value from the guard-n-primary-dir-guards-to-use consensus parameter, and
+    default to 3 if the consensus parameter isn't set.  (Default: 0)
 
 [[GuardLifetime]] **GuardLifetime**  __N__ **days**|**weeks**|**months**::
     If nonzero, and UseEntryGuards is set, minimum time to keep a guard before
@@ -1262,15 +1414,9 @@ The following options are useful only for clients (that is, if
     helps to determine whether an application using Tor is possibly leaking
     DNS requests. (Default: 0)
 
-[[WarnUnsafeSocks]] **WarnUnsafeSocks** **0**|**1**::
-    When this option is enabled, Tor will warn whenever a request is
-    received that only contains an IP address instead of a hostname. Allowing
-    applications to do DNS resolves themselves is usually a bad idea and
-    can leak your location to attackers. (Default: 1)
+[[VirtualAddrNetworkIPv4]] **VirtualAddrNetworkIPv4** __IPv4Address__/__bits__ +
 
-[[VirtualAddrNetworkIPv4]] **VirtualAddrNetworkIPv4** __Address__/__bits__ +
-
-[[VirtualAddrNetworkIPv6]] **VirtualAddrNetworkIPv6** [__Address__]/__bits__::
+[[VirtualAddrNetworkIPv6]] **VirtualAddrNetworkIPv6** [__IPv6Address__]/__bits__::
     When Tor needs to assign a virtual (unused) address because of a MAPADDRESS
     command from the controller or the AutomapHostsOnResolve feature, Tor
     picks an unassigned address from this range. (Defaults:
@@ -1293,23 +1439,13 @@ The following options are useful only for clients (that is, if
     resolved. This helps trap accidental attempts to resolve URLs and so on.
     (Default: 0)
 
-[[AllowDotExit]] **AllowDotExit** **0**|**1**::
-    If enabled, we convert "www.google.com.foo.exit" addresses on the
-    SocksPort/TransPort/NATDPort into "www.google.com" addresses that exit from
-    the node "foo". Disabled by default since attacking websites and exit
-    relays can use it to manipulate your path selection. (Default: 0)
-
-[[FastFirstHopPK]] **FastFirstHopPK** **0**|**1**|**auto**::
-    When this option is disabled, Tor uses the public key step for the first
-    hop of creating circuits. Skipping it is generally safe since we have
-    already used TLS to authenticate the relay and to establish forward-secure
-    keys. Turning this option off makes circuit building a little
-    slower. Setting this option to "auto" takes advice from the authorities
-    in the latest consensus about whether to use this feature.  +
- +
-    Note that Tor will always use the public key step for the first hop if it's
-    operating as a relay, and it will never use the public key step if it
-    doesn't yet know the onion key of the first hop. (Default: auto)
+[[HTTPTunnelPort]] **HTTPTunnelPort**  \['address':]__port__|**auto** [_isolation flags_]::
+    Open this port to listen for proxy connections using the "HTTP CONNECT"
+    protocol instead of SOCKS. Set this to
+    0 if you don't want to allow "HTTP CONNECT" connections. Set the port
+    to "auto" to have Tor pick a port for you. This directive can be
+    specified multiple times to bind to multiple addresses/ports.  See
+    SOCKSPort for an explanation of isolation flags. (Default: 0)
 
 [[TransPort]] **TransPort**  \['address':]__port__|**auto** [_isolation flags_]::
     Open this port to listen for transparent proxy connections.  Set this to
@@ -1321,43 +1457,31 @@ The following options are useful only for clients (that is, if
     TransPort requires OS support for transparent proxies, such as BSDs' pf or
     Linux's IPTables. If you're planning to use Tor as a transparent proxy for
     a network, you'll want to examine and change VirtualAddrNetwork from the
-    default setting. You'll also want to set the TransListenAddress option for
-    the network you'd like to proxy. (Default: 0)
-
-[[TransListenAddress]] **TransListenAddress** __IP__[:__PORT__]::
-    Bind to this address to listen for transparent proxy connections. (Default:
-    127.0.0.1). This is useful for exporting a transparent proxy server to an
-    entire network. (DEPRECATED: As of 0.2.3.x-alpha, you can
-    now use multiple TransPort entries, and provide addresses for TransPort
-    entries, so TransListenAddress no longer has a purpose.  For backward
-    compatibility, TransListenAddress is only allowed when TransPort is just
-    a port number.)
+    default setting. (Default: 0)
 
 [[TransProxyType]] **TransProxyType** **default**|**TPROXY**|**ipfw**|**pf-divert**::
     TransProxyType may only be enabled when there is transparent proxy listener
-    enabled.
+    enabled. +
  +
     Set this to "TPROXY" if you wish to be able to use the TPROXY Linux module
     to transparently proxy connections that are configured using the TransPort
-    option. This setting lets the listener on the TransPort accept connections
-    for all addresses, even when the TransListenAddress is configured for an
-    internal address. Detailed information on how to configure the TPROXY
+    option. Detailed information on how to configure the TPROXY
     feature can be found in the Linux kernel source tree in the file
-    Documentation/networking/tproxy.txt.
+    Documentation/networking/tproxy.txt. +
  +
-    Set this option to "ipfw" to use the FreeBSD ipfw interface.
+    Set this option to "ipfw" to use the FreeBSD ipfw interface. +
  +
     On *BSD operating systems when using pf, set this to "pf-divert" to take
     advantage of +divert-to+ rules, which do not modify the packets like
     +rdr-to+ rules do. Detailed information on how to configure pf to use
     +divert-to+ rules can be found in the pf.conf(5) manual page. On OpenBSD,
     +divert-to+ is available to use on versions greater than or equal to
-    OpenBSD 4.4.
+    OpenBSD 4.4. +
  +
     Set this to "default", or leave it unconfigured, to use regular IPTables
-    on Linux, or to use pf +rdr-to+ rules on *BSD systems.
+    on Linux, or to use pf +rdr-to+ rules on *BSD systems. +
  +
-    (Default: "default".)
+    (Default: "default")
 
 [[NATDPort]] **NATDPort** \['address':]__port__|**auto** [_isolation flags_]::
     Open this port to listen for connections from old versions of ipfw (as
@@ -1369,13 +1493,6 @@ The following options are useful only for clients (that is, if
  +
     This option is only for people who cannot use TransPort. (Default: 0)
 
-[[NATDListenAddress]] **NATDListenAddress** __IP__[:__PORT__]::
-    Bind to this address to listen for NATD connections. (DEPRECATED: As of
-    0.2.3.x-alpha, you can now use multiple NATDPort entries, and provide
-    addresses for NATDPort entries, so NATDListenAddress no longer has a
-    purpose.  For backward compatibility, NATDListenAddress is only allowed
-    when NATDPort is just a port number.)
-
 [[AutomapHostsOnResolve]] **AutomapHostsOnResolve** **0**|**1**::
     When this option is enabled, and we get a request to resolve an address
     that ends with one of the suffixes in **AutomapHostsSuffixes**, we map an
@@ -1396,24 +1513,18 @@ The following options are useful only for clients (that is, if
     addresses/ports. See SocksPort for an explanation of isolation
     flags. (Default: 0)
 
-[[DNSListenAddress]] **DNSListenAddress** __IP__[:__PORT__]::
-    Bind to this address to listen for DNS connections. (DEPRECATED: As of
-    0.2.3.x-alpha, you can now use multiple DNSPort entries, and provide
-    addresses for DNSPort entries, so DNSListenAddress no longer has a
-    purpose.  For backward compatibility, DNSListenAddress is only allowed
-    when DNSPort is just a port number.)
-
 [[ClientDNSRejectInternalAddresses]] **ClientDNSRejectInternalAddresses** **0**|**1**::
     If true, Tor does not believe any anonymously retrieved DNS answer that
     tells it that an address resolves to an internal address (like 127.0.0.1 or
-    192.168.0.1). This option prevents certain browser-based attacks; don't
-    turn it off unless you know what you're doing. (Default: 1)
+    192.168.0.1). This option prevents certain browser-based attacks; it
+    is not allowed to be set on the default network. (Default: 1)
 
 [[ClientRejectInternalAddresses]] **ClientRejectInternalAddresses** **0**|**1**::
     If true, Tor does not try to fulfill requests to connect to an internal
-    address (like 127.0.0.1 or 192.168.0.1) __unless a exit node is
+    address (like 127.0.0.1 or 192.168.0.1) __unless an exit node is
     specifically requested__ (for example, via a .exit hostname, or a
-    controller request).  (Default: 1)
+    controller request).  If true, multicast DNS hostnames for machines on the
+    local network (of the form *.local) are also rejected.  (Default: 1)
 
 [[DownloadExtraInfo]] **DownloadExtraInfo** **0**|**1**::
     If true, Tor downloads and caches "extra-info" documents. These documents
@@ -1431,11 +1542,6 @@ The following options are useful only for clients (that is, if
     Like WarnPlaintextPorts, but instead of warning about risky port uses, Tor
     will instead refuse to make the connection. (Default: None)
 
-[[AllowSingleHopCircuits]] **AllowSingleHopCircuits** **0**|**1**::
-    When this option is set, the attached Tor controller can use relays
-    that have the **AllowSingleHopExits** option turned on to build
-    one-hop Tor connections.  (Default: 0)
-
 [[OptimisticData]] **OptimisticData** **0**|**1**|**auto**::
     When this option is set, and Tor is using an exit node that supports
     the feature, it will try optimistically to send data to the exit node
@@ -1445,40 +1551,123 @@ The following options are useful only for clients (that is, if
     Tor will look at the UseOptimisticData parameter in the networkstatus.
     (Default: auto)
 
-[[Tor2webMode]] **Tor2webMode** **0**|**1**::
-    When this option is set, Tor connects to hidden services
-    **non-anonymously**.  This option also disables client connections to
-    non-hidden-service hostnames through Tor. It **must only** be used when
-    running a tor2web Hidden Service web proxy.
-    To enable this option the compile time flag --enable-tor2web-mode must be
-    specified. Since Tor2webMode is non-anonymous, you can not run an
-    anonymous Hidden Service on a tor version compiled with Tor2webMode.
-    (Default: 0)
-
-[[Tor2webRendezvousPoints]] **Tor2webRendezvousPoints** __node__,__node__,__...__::
-    A list of identity fingerprints, nicknames, country codes and
-    address patterns of nodes that are allowed to be used as RPs
-    in HS circuits; any other nodes will not be used as RPs.
-    (Example:
-    Tor2webRendezvousPoints Fastyfasty, ABCD1234CDEF5678ABCD1234CDEF5678ABCD1234, \{cc}, 255.254.0.0/8) +
- +
-    This feature can only be used if Tor2webMode is also enabled.
+[[HSLayer2Nodes]] **HSLayer2Nodes** __node__,__node__,__...__::
+    A list of identity fingerprints, nicknames, country codes, and
+    address patterns of nodes that are allowed to be used as the
+    second hop in all client or service-side Onion Service circuits.
+    This option mitigates attacks where the adversary runs middle nodes
+    and induces your client or service to create many circuits, in order
+    to discover your primary guard node.
+    (Default: Any node in the network may be used in the second hop.)
  +
-    ExcludeNodes have higher priority than Tor2webRendezvousPoints,
+    (Example:
+    HSLayer2Nodes ABCD1234CDEF5678ABCD1234CDEF5678ABCD1234, \{cc}, 255.254.0.0/8) +
+ +
+    When this is set, the resulting hidden service paths will
+    look like:
+ +
+        C - G - L2 - M - Rend +
+        C - G - L2 - M - HSDir +
+        C - G - L2 - M - Intro +
+        S - G - L2 - M - Rend +
+        S - G - L2 - M - HSDir +
+        S - G - L2 - M - Intro +
+ +
+    where C is this client, S is the service, G is the Guard node,
+    L2 is a node from this option, and M is a random middle node.
+    Rend, HSDir, and Intro point selection is not affected by this
+    option.
+ +
+    This option may be combined with HSLayer3Nodes to create
+    paths of the form:
+ +
+        C - G - L2 - L3 - Rend +
+        C - G - L2 - L3 - M - HSDir +
+        C - G - L2 - L3 - M - Intro +
+        S - G - L2 - L3 - M - Rend +
+        S - G - L2 - L3 - HSDir +
+        S - G - L2 - L3 - Intro +
+ +
+    ExcludeNodes have higher priority than HSLayer2Nodes,
     which means that nodes specified in ExcludeNodes will not be
-    picked as RPs.
+    picked.
+ +
+    When either this option or HSLayer3Nodes are set, the /16 subnet
+    and node family restrictions are removed for hidden service
+    circuits. Additionally, we allow the guard node to be present
+    as the Rend, HSDir, and IP node, and as the hop before it. This
+    is done to prevent the adversary from inferring information
+    about our guard, layer2, and layer3 node choices at later points
+    in the path.
+ +
+    This option is meant to be managed by a Tor controller such as
+    https://github.com/mikeperry-tor/vanguards that selects and
+    updates this set of nodes for you. Hence it does not do load
+    balancing if fewer than 20 nodes are selected, and if no nodes in
+    HSLayer2Nodes are currently available for use, Tor will not work.
+    Please use extreme care if you are setting this option manually.
+
+[[HSLayer3Nodes]] **HSLayer3Nodes** __node__,__node__,__...__::
+    A list of identity fingerprints, nicknames, country codes, and
+    address patterns of nodes that are allowed to be used as the
+    third hop in all client and service-side Onion Service circuits.
+    This option mitigates attacks where the adversary runs middle nodes
+    and induces your client or service to create many circuits, in order
+    to discover your primary or Layer2 guard nodes.
+    (Default: Any node in the network may be used in the third hop.)
  +
-    If no nodes in Tor2webRendezvousPoints are currently available for
-    use, Tor will choose a random node when building HS circuits.
+    (Example:
+    HSLayer3Nodes ABCD1234CDEF5678ABCD1234CDEF5678ABCD1234, \{cc}, 255.254.0.0/8) +
+ +
+    When this is set by itself, the resulting hidden service paths
+    will look like: +
+        C - G - M - L3 - Rend +
+        C - G - M - L3 - M - HSDir +
+        C - G - M - L3 - M - Intro +
+        S - G - M - L3 - M - Rend +
+        S - G - M - L3 - HSDir +
+        S - G - M - L3 - Intro +
+    where C is this client, S is the service, G is the Guard node,
+    L2 is a node from this option, and M is a random middle node.
+    Rend, HSDir, and Intro point selection is not affected by this
+    option.
+ +
+    While it is possible to use this option by itself, it should be
+    combined with HSLayer2Nodes to create paths of the form:
+ +
+        C - G - L2 - L3 - Rend +
+        C - G - L2 - L3 - M - HSDir +
+        C - G - L2 - L3 - M - Intro +
+        S - G - L2 - L3 - M - Rend +
+        S - G - L2 - L3 - HSDir +
+        S - G - L2 - L3 - Intro +
+ +
+    ExcludeNodes have higher priority than HSLayer3Nodes,
+    which means that nodes specified in ExcludeNodes will not be
+    picked.
+ +
+    When either this option or HSLayer2Nodes are set, the /16 subnet
+    and node family restrictions are removed for hidden service
+    circuits. Additionally, we allow the guard node to be present
+    as the Rend, HSDir, and IP node, and as the hop before it. This
+    is done to prevent the adversary from inferring information
+    about our guard, layer2, and layer3 node choices at later points
+    in the path.
+  +
+    This option is meant to be managed by a Tor controller such as
+    https://github.com/mikeperry-tor/vanguards that selects and
+    updates this set of nodes for you. Hence it does not do load
+    balancing if fewer than 20 nodes are selected, and if no nodes in
+    HSLayer3Nodes are currently available for use, Tor will not work.
+    Please use extreme care if you are setting this option manually.
 
 [[UseMicrodescriptors]] **UseMicrodescriptors** **0**|**1**|**auto**::
     Microdescriptors are a smaller version of the information that Tor needs
     in order to build its circuits.  Using microdescriptors makes Tor clients
     download less directory information, thus saving bandwidth.  Directory
     caches need to fetch regular descriptors and microdescriptors, so this
-    option doesn't save any bandwidth for them.  If this option is set to
-    "auto" (recommended) then it is on for all clients that do not set
-    FetchUselessDescriptors. (Default: auto)
+    option doesn't save any bandwidth for them.  For legacy reasons, auto is
+    accepted, but it has the same effect as 1. (Default: auto)
 
 [[PathBiasCircThreshold]] **PathBiasCircThreshold** __NUM__ +
 
@@ -1494,7 +1683,7 @@ The following options are useful only for clients (that is, if
     These options override the default behavior of Tor's (**currently
     experimental**) path bias detection algorithm. To try to find broken or
     misbehaving guard nodes, Tor looks for nodes where more than a certain
-    fraction of circuits through that guard fail to get built.
+    fraction of circuits through that guard fail to get built. +
  +
     The PathBiasCircThreshold option controls how many circuits we need to build
     through a guard before we make these checks.  The PathBiasNoticeRate,
@@ -1520,14 +1709,14 @@ The following options are useful only for clients (that is, if
 
 [[PathBiasScaleUseThreshold]] **PathBiasScaleUseThreshold** __NUM__::
     Similar to the above options, these options override the default behavior
-    of Tor's (**currently experimental**) path use bias detection algorithm.
+    of Tor's (**currently experimental**) path use bias detection algorithm. +
  +
     Where as the path bias parameters govern thresholds for successfully
     building circuits, these four path use bias parameters govern thresholds
     only for circuit usage. Circuits which receive no stream usage
     are not counted by this detection algorithm. A used circuit is considered
     successful if it is capable of carrying streams or otherwise receiving
-    well-formed responses to RELAY cells.
+    well-formed responses to RELAY cells. +
  +
     By default, or if a negative value is provided for one of these options,
     Tor uses reasonable defaults from the networkstatus consensus document.
@@ -1542,9 +1731,10 @@ The following options are useful only for clients (that is, if
 
 [[ClientUseIPv6]] **ClientUseIPv6** **0**|**1**::
     If this option is set to 1, Tor might connect to directory servers or
-    entry nodes over IPv6. Note that clients configured with an IPv6 address
-    in a **Bridge**, proxy, or pluggable transport line will try connecting
-    over IPv6 even if **ClientUseIPv6** is set to 0. (Default: 0)
+    entry nodes over IPv6. For IPv6 only hosts, you need to also set
+    **ClientUseIPv4** to 0 to disable IPv4. Note that clients configured with
+    an IPv6 address in a **Bridge**, proxy, or pluggable transportline will
+    try connecting over IPv6 even if **ClientUseIPv6** is set to 0. (Default: 0)
 
 [[ClientPreferIPv6DirPort]] **ClientPreferIPv6DirPort** **0**|**1**|**auto**::
     If this option is set to 1, Tor prefers a directory port with an IPv6
@@ -1552,7 +1742,8 @@ The following options are useful only for clients (that is, if
     server has both. (Tor also prefers an IPv6 DirPort if IPv4Client is set to
     0.) If this option is set to auto, clients prefer IPv4. Other things may
     influence the choice. This option breaks a tie to the favor of IPv6.
-    (Default: auto)
+    (Default: auto)  (DEPRECATED: This option has had no effect for some
+    time.)
 
 [[ClientPreferIPv6ORPort]] **ClientPreferIPv6ORPort** **0**|**1**|**auto**::
     If this option is set to 1, Tor prefers an OR port with an IPv6
@@ -1573,46 +1764,35 @@ The following options are useful only for clients (that is, if
     prevent your Tor client from bootstrapping.  If this option is negative,
     Tor will use a default value chosen by the directory authorities. If the
     directory authorities do not choose a value, Tor will default to 0.6.
-    (Default: -1.)
+    (Default: -1)
 
-[[ClientBootstrapConsensusAuthorityDownloadSchedule]] **ClientBootstrapConsensusAuthorityDownloadSchedule** __N__,__N__,__...__::
-    Schedule for when clients should download consensuses from authorities
+[[ClientBootstrapConsensusAuthorityDownloadInitialDelay]] **ClientBootstrapConsensusAuthorityDownloadInitialDelay** __N__::
+    Initial delay in seconds for when clients should download consensuses from authorities
     if they are bootstrapping (that is, they don't have a usable, reasonably
     live consensus). Only used by clients fetching from a list of fallback
     directory mirrors. This schedule is advanced by (potentially concurrent)
     connection attempts, unlike other schedules, which are advanced by
-    connection failures. (Default: 10, 11, 3600, 10800, 25200, 54000,
-    111600, 262800)
+    connection failures. (Default: 6)
 
-[[ClientBootstrapConsensusFallbackDownloadSchedule]] **ClientBootstrapConsensusFallbackDownloadSchedule** __N__,__N__,__...__::
-    Schedule for when clients should download consensuses from fallback
+[[ClientBootstrapConsensusFallbackDownloadInitialDelay]] **ClientBootstrapConsensusFallbackDownloadInitialDelay** __N__::
+    Initial delay in seconds for when clients should download consensuses from fallback
     directory mirrors if they are bootstrapping (that is, they don't have a
     usable, reasonably live consensus). Only used by clients fetching from a
     list of fallback directory mirrors. This schedule is advanced by
     (potentially concurrent) connection attempts, unlike other schedules,
-    which are advanced by connection failures. (Default: 0, 1, 4, 11, 3600,
-    10800, 25200, 54000, 111600, 262800)
+    which are advanced by connection failures. (Default: 0)
 
-[[ClientBootstrapConsensusAuthorityOnlyDownloadSchedule]] **ClientBootstrapConsensusAuthorityOnlyDownloadSchedule** __N__,__N__,__...__::
-    Schedule for when clients should download consensuses from authorities
+[[ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay]] **ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay** __N__::
+    Initial delay in seconds for when clients should download consensuses from authorities
     if they are bootstrapping (that is, they don't have a usable, reasonably
     live consensus). Only used by clients which don't have or won't fetch
     from a list of fallback directory mirrors. This schedule is advanced by
     (potentially concurrent) connection attempts, unlike other schedules,
-    which are advanced by connection failures. (Default: 0, 3, 7, 3600,
-    10800, 25200, 54000, 111600, 262800)
-
-[[ClientBootstrapConsensusMaxDownloadTries]] **ClientBootstrapConsensusMaxDownloadTries** __NUM__::
-    Try this many times to download a consensus while bootstrapping using
-    fallback directory mirrors before giving up. (Default: 7)
-
-[[ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries]] **ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries** __NUM__::
-    Try this many times to download a consensus while bootstrapping using
-    authorities before giving up. (Default: 4)
+    which are advanced by connection failures. (Default: 0)
 
 [[ClientBootstrapConsensusMaxInProgressTries]] **ClientBootstrapConsensusMaxInProgressTries** __NUM__::
     Try this many simultaneous connections to download a consensus before
-    waiting for one to complete, timeout, or error out. (Default: 4)
+    waiting for one to complete, timeout, or error out. (Default: 3)
 
 SERVER OPTIONS
 --------------
@@ -1621,19 +1801,13 @@ The following options are useful only for servers (that is, if ORPort
 is non-zero):
 
 [[Address]] **Address** __address__::
-    The IP address or fully qualified domain name of this server (e.g.
-    moria.mit.edu). You can leave this unset, and Tor will guess your IP
-    address.  This IP address is the one used to tell clients and other
-    servers where to find your Tor server; it doesn't affect the IP that your
-    Tor client binds to.  To bind to a different address, use the
-    *ListenAddress and OutboundBindAddress options.
-
-[[AllowSingleHopExits]] **AllowSingleHopExits** **0**|**1**::
-    This option controls whether clients can use this server as a single hop
-    proxy. If set to 1, clients can use this server as an exit even if it is
-    the only hop in the circuit.  Note that most clients will refuse to use
-    servers that set this option, since most clients have
-    ExcludeSingleHopRelays set.  (Default: 0)
+    The IPv4 address of this server, or a fully qualified domain name of
+    this server that resolves to an IPv4 address.  You can leave this
+    unset, and Tor will try to guess your IPv4 address.  This IPv4
+    address is the one used to tell clients and other servers where to
+    find your Tor server; it doesn't affect the address that your server
+    binds to.  To bind to a different address, use the ORPort and
+    OutboundBindAddress options.
 
 [[AssumeReachable]] **AssumeReachable** **0**|**1**::
     This option is used when bootstrapping a new Tor network. If set to 1,
@@ -1646,7 +1820,21 @@ is non-zero):
     Sets the relay to act as a "bridge" with respect to relaying connections
     from bridge users to the Tor network. It mainly causes Tor to publish a
     server descriptor to the bridge database, rather than
-    to the public directory authorities.
+    to the public directory authorities. +
+ +
+    Note: make sure that no MyFamily lines are present in your torrc when
+    relay is configured in bridge mode.
+
+[[BridgeDistribution]] **BridgeDistribution** __string__::
+    If set along with BridgeRelay, Tor will include a new line in its
+    bridge descriptor which indicates to the BridgeDB service how it
+    would like its bridge address to be given out. Set it to "none" if
+    you want BridgeDB to avoid distributing your bridge address, or "any" to
+    let BridgeDB decide. (Default: any)
+ +
+    Note: as of Oct 2017, the BridgeDB part of this option is not yet
+    implemented.  Until BridgeDB is updated to obey this option, your
+    bridge will make this request, but it will not (yet) be obeyed.
 
 [[ContactInfo]] **ContactInfo** __email_address__::
     Administrative contact information for this relay or bridge. This line
@@ -1655,28 +1843,34 @@ is non-zero):
     descriptors containing these lines and that Google indexes them, so
     spammers might also collect them. You may want to obscure the fact
     that it's an email address and/or generate a new address for this
-    purpose.
+    purpose. +
+ +
+    ContactInfo **must** be set to a working address if you run more than one
+    relay or bridge.  (Really, everybody running a relay or bridge should set
+    it.)
+
 
 [[ExitRelay]] **ExitRelay** **0**|**1**|**auto**::
     Tells Tor whether to run as an exit relay.  If Tor is running as a
     non-bridge server, and ExitRelay is set to 1, then Tor allows traffic to
-    exit according to the ExitPolicy option (or the default ExitPolicy if
-    none is specified).
+    exit according to the ExitPolicy option, the ReducedExitPolicy option,
+    or the default ExitPolicy (if no other exit policy option is specified). +
  +
     If ExitRelay is set to 0, no traffic is allowed to
-    exit, and the ExitPolicy option is ignored. +
+    exit, and the ExitPolicy and ReducedExitPolicy options are ignored. +
  +
-    If ExitRelay is set to "auto", then Tor behaves as if it were set to 1, but
-    warns the user if this would cause traffic to exit.  In a future version,
-    the default value will be 0. (Default: auto)
+    If ExitRelay is set to "auto", then Tor checks the ExitPolicy and
+    ReducedExitPolicy options. If either is set, Tor behaves as if ExitRelay
+    were set to 1. If neither exit policy option is set, Tor behaves as if
+    ExitRelay were set to 0. (Default: auto)
 
 [[ExitPolicy]] **ExitPolicy** __policy__,__policy__,__...__::
     Set an exit policy for this server. Each policy is of the form
     "**accept[6]**|**reject[6]**  __ADDR__[/__MASK__][:__PORT__]". If /__MASK__ is
     omitted then this policy just applies to the host given. Instead of giving
     a host or network you can also use "\*" to denote the universe (0.0.0.0/0
-    and ::/128), or \*4 to denote all IPv4 addresses, and \*6 to denote all
-    IPv6 addresses.
+    and ::/0), or \*4 to denote all IPv4 addresses, and \*6 to denote all IPv6
+    addresses.
     __PORT__ can be a single port number, an interval of ports
     "__FROM_PORT__-__TO_PORT__", or "\*". If __PORT__ is omitted, that means
     "\*". +
@@ -1724,8 +1918,15 @@ is non-zero):
     write your IPv6 rules using accept6/reject6 \*6, and your IPv4 rules using
     accept/reject \*4. If you want to \_replace_ the default exit policy, end
     your exit policy with either a reject \*:* or an accept \*:*. Otherwise,
-    you're \_augmenting_ (prepending to) the default exit policy. The default
-    exit policy is: +
+    you're \_augmenting_ (prepending to) the default exit policy. +
+ +
+    If you want to use a reduced exit policy rather than the default exit
+    policy, set "ReducedExitPolicy 1". If you want to _replace_ the default
+    exit policy with your custom exit policy, end your exit policy with either
+    a reject *:* or an accept *:*. Otherwise, you're _augmenting_ (prepending
+    to) the default or reduced exit policy. +
+ +
+    The default exit policy is:
 
        reject *:25
        reject *:119
@@ -1739,6 +1940,8 @@ is non-zero):
        reject *:6881-6999
        accept *:*
 
+// Anchor only for formatting, not visible in the man page.
+[[ExitPolicyDefault]]::
     Since the default exit policy uses accept/reject *, it applies to both
     IPv4 and IPv6 addresses.
 
@@ -1760,6 +1963,99 @@ is non-zero):
     to disclose.
     (Default: 0)
 
+[[ReducedExitPolicy]] **ReducedExitPolicy** **0**|**1**::
+    If set, use a reduced exit policy rather than the default one. +
+ +
+    The reduced exit policy is an alternative to the default exit policy. It
+    allows as many Internet services as possible while still blocking the
+    majority of TCP ports. Currently, the policy allows approximately 65 ports.
+    This reduces the odds that your node will be used for peer-to-peer
+    applications. +
+ +
+    The reduced exit policy is:
+
+        accept *:20-21
+        accept *:22
+        accept *:23
+        accept *:43
+        accept *:53
+        accept *:79
+        accept *:80-81
+        accept *:88
+        accept *:110
+        accept *:143
+        accept *:194
+        accept *:220
+        accept *:389
+        accept *:443
+        accept *:464
+        accept *:465
+        accept *:531
+        accept *:543-544
+        accept *:554
+        accept *:563
+        accept *:587
+        accept *:636
+        accept *:706
+        accept *:749
+        accept *:873
+        accept *:902-904
+        accept *:981
+        accept *:989-990
+        accept *:991
+        accept *:992
+        accept *:993
+        accept *:994
+        accept *:995
+        accept *:1194
+        accept *:1220
+        accept *:1293
+        accept *:1500
+        accept *:1533
+        accept *:1677
+        accept *:1723
+        accept *:1755
+        accept *:1863
+        accept *:2082
+        accept *:2083
+        accept *:2086-2087
+        accept *:2095-2096
+        accept *:2102-2104
+        accept *:3128
+        accept *:3389
+        accept *:3690
+        accept *:4321
+        accept *:4643
+        accept *:5050
+        accept *:5190
+        accept *:5222-5223
+        accept *:5228
+        accept *:5900
+        accept *:6660-6669
+        accept *:6679
+        accept *:6697
+        accept *:8000
+        accept *:8008
+        accept *:8074
+        accept *:8080
+        accept *:8082
+        accept *:8087-8088
+        accept *:8232-8233
+        accept *:8332-8333
+        accept *:8443
+        accept *:8888
+        accept *:9418
+        accept *:9999
+        accept *:10000
+        accept *:11371
+        accept *:19294
+        accept *:19638
+        accept *:50002
+        accept *:64738
+        reject *:*
+
+    (Default: 0)
+
 [[IPv6Exit]] **IPv6Exit** **0**|**1**::
     If set, and we are an exit node, allow clients to use us for IPv6
     traffic. (Default: 0)
@@ -1768,21 +2064,33 @@ is non-zero):
     If we have more onionskins queued for processing than we can process in
     this amount of time, reject new ones. (Default: 1750 msec)
 
-[[MyFamily]] **MyFamily** __node__,__node__,__...__::
-    Declare that this Tor server is controlled or administered by a group or
-    organization identical or similar to that of the other servers, defined by
-    their identity fingerprints. When two servers both declare
-    that they are in the same \'family', Tor clients will not use them in the
-    same circuit. (Each server only needs to list the other servers in its
-    family; it doesn't need to list itself, but it won't hurt.) Do not list
-    any bridge relay as it would compromise its concealment.
+[[MyFamily]] **MyFamily** __fingerprint__,__fingerprint__,...::
+    Declare that this Tor relay is controlled or administered by a group or
+    organization identical or similar to that of the other relays, defined by
+    their (possibly $-prefixed) identity fingerprints.
+    This option can be repeated many times, for
+    convenience in defining large families: all fingerprints in all MyFamily
+    lines are merged into one list.
+    When two relays both declare that they are in the
+    same \'family', Tor clients will not use them in the same circuit. (Each
+    relay only needs to list the other servers in its family; it doesn't need to
+    list itself, but it won't hurt if it does.) Do not list any bridge relay as it would
+    compromise its concealment. +
  +
     When listing a node, it's better to list it by fingerprint than by
-    nickname: fingerprints are more reliable.
+    nickname: fingerprints are more reliable. +
+ +
+    If you run more than one relay, the MyFamily option on each relay
+    **must** list all other relays, as described above. +
+ +
+    Note: do not use MyFamily when configuring your Tor instance as a
+    brigde.
 
 [[Nickname]] **Nickname** __name__::
     Set the server's nickname to \'name'. Nicknames must be between 1 and 19
     characters inclusive, and must contain only the characters [a-zA-Z0-9].
+    If not set, **Unnamed** will be used. Relays can always be uniquely identified
+    by their identity fingerprints.
 
 [[NumCPUs]] **NumCPUs** __num__::
     How many processes to use at once for decrypting onionskins and other
@@ -1793,62 +2101,45 @@ is non-zero):
     Advertise this port to listen for connections from Tor clients and
     servers.  This option is required to be a Tor server.
     Set it to "auto" to have Tor pick a port for you. Set it to 0 to not
-    run an ORPort at all. This option can occur more than once. (Default: 0)
-+
+    run an ORPort at all. This option can occur more than once. (Default: 0) +
+ +
     Tor recognizes these flags on each ORPort:
-    **NoAdvertise**::
+    **NoAdvertise**;;
         By default, we bind to a port and tell our users about it. If
         NoAdvertise is specified, we don't advertise, but listen anyway.  This
         can be useful if the port everybody will be connecting to (for
         example, one that's opened on our firewall) is somewhere else.
-    **NoListen**::
+    **NoListen**;;
         By default, we bind to a port and tell our users about it. If
         NoListen is specified, we don't bind, but advertise anyway.  This
         can be useful if something else  (for example, a firewall's port
         forwarding configuration) is causing connections to reach us.
-    **IPv4Only**::
+    **IPv4Only**;;
         If the address is absent, or resolves to both an IPv4 and an IPv6
         address, only listen to the IPv4 address.
-    **IPv6Only**::
+    **IPv6Only**;;
         If the address is absent, or resolves to both an IPv4 and an IPv6
         address, only listen to the IPv6 address.
-+
+
+// Anchor only for formatting, not visible in the man page.
+[[ORPortFlagsExclusive]]::
     For obvious reasons, NoAdvertise and NoListen are mutually exclusive, and
     IPv4Only and IPv6Only are mutually exclusive.
 
-[[ORListenAddress]] **ORListenAddress** __IP__[:__PORT__]::
-    Bind to this IP address to listen for connections from Tor clients and
-    servers. If you specify a port, bind to this port rather than the one
-    specified in ORPort. (Default: 0.0.0.0) This directive can be specified
-    multiple times to bind to multiple addresses/ports.
-+
-    This option is deprecated; you can get the same behavior with ORPort now
-    that it supports NoAdvertise and explicit addresses.
-
-[[PortForwarding]] **PortForwarding** **0**|**1**::
-    Attempt to automatically forward the DirPort and ORPort on a NAT router
-    connecting this Tor server to the Internet. If set, Tor will try both
-    NAT-PMP (common on Apple routers) and UPnP (common on routers from other
-    manufacturers). (Default: 0)
-
-[[PortForwardingHelper]] **PortForwardingHelper** __filename__|__pathname__::
-    If PortForwarding is set, use this executable to configure the forwarding.
-    If set to a filename, the system path will be searched for the executable.
-    If set to a path, only the specified path will be executed.
-    (Default: tor-fw-helper)
-
 [[PublishServerDescriptor]] **PublishServerDescriptor** **0**|**1**|**v3**|**bridge**,**...**::
     This option specifies which descriptors Tor will publish when acting as
     a relay. You can
-    choose multiple arguments, separated by commas.
+    choose multiple arguments, separated by commas. +
  +
     If this option is set to 0, Tor will not publish its
     descriptors to any directories. (This is useful if you're testing
-    out your server, or if you're using a Tor controller that handles directory
-    publishing for you.) Otherwise, Tor will publish its descriptors of all
-    type(s) specified. The default is "1",
-    which means "if running as a server, publish the
-    appropriate descriptors to the authorities".
+    out your server, or if you're using a Tor controller that handles
+    directory publishing for you.) Otherwise, Tor will publish its
+    descriptors of all type(s) specified. The default is "1", which
+    means "if running as a relay or bridge, publish descriptors to the
+    appropriate authorities". Other possibilities are "v3", meaning
+    "publish as if you're a relay", and "bridge", meaning "publish as
+    if you're a bridge".
 
 [[ShutdownWaitLength]] **ShutdownWaitLength** __NUM__::
     When we get a SIGINT and we're a server, we begin shutting down:
@@ -1868,7 +2159,12 @@ is non-zero):
     to 0 will disable the heartbeat. Otherwise, it must be at least 30
     minutes. (Default: 6 hours)
 
-[[AccountingMax]] **AccountingMax** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**|**TBytes**::
+[[MainloopStats]] **MainloopStats** **0**|**1**::
+    Log main loop statistics every **HeartbeatPeriod** seconds. This is a log
+    level __notice__ message designed to help developers instrumenting Tor's
+    main event loop. (Default: 0)
+
+[[AccountingMax]] **AccountingMax** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     Limits the max number of bytes sent and received within a set time period
     using a given calculation rule (see: AccountingStart, AccountingRule).
     Useful if you need to stay under a specific bandwidth. By default, the
@@ -1898,15 +2194,16 @@ is non-zero):
     (Default: max)
 
 [[AccountingStart]] **AccountingStart** **day**|**week**|**month** [__day__] __HH:MM__::
-    Specify how long accounting periods last. If **month** is given, each
-    accounting period runs from the time __HH:MM__ on the __dayth__ day of one
-    month to the same day and time of the next. (The day must be between 1 and
-    28.) If **week** is given, each accounting period runs from the time __HH:MM__
-    of the __dayth__ day of one week to the same day and time of the next week,
-    with Monday as day 1 and Sunday as day 7. If **day** is given, each
-    accounting period runs from the time __HH:MM__ each day to the same time on
-    the next day. All times are local, and given in 24-hour time. (Default:
-    "month 1 0:00")
+    Specify how long accounting periods last. If **month** is given,
+    each accounting period runs from the time __HH:MM__ on the __dayth__ day of one
+    month to the same day and time of the next. The relay will go at full speed,
+    use all the quota you specify, then hibernate for the rest of the period. (The
+    day must be between 1 and 28.) If **week** is given, each accounting period
+    runs from the time __HH:MM__ of the __dayth__ day of one week to the same day
+    and time of the next week, with Monday as day 1 and Sunday as day 7. If **day**
+    is given, each accounting period runs from the time __HH:MM__ each day to the
+    same time on the next day. All times are local, and given in 24-hour time.
+    (Default: "month 1 0:00")
 
 [[RefuseUnknownExits]] **RefuseUnknownExits** **0**|**1**|**auto**::
     Prevent nodes that don't appear in the consensus from exiting using this
@@ -1942,7 +2239,7 @@ is non-zero):
     correct this. This option only affects name lookups that your server does
     on behalf of clients. (Default: 1)
 
-[[ServerDNSTestAddresses]] **ServerDNSTestAddresses** __address__,__address__,__...__::
+[[ServerDNSTestAddresses]] **ServerDNSTestAddresses** __hostname__,__hostname__,__...__::
     When we're detecting DNS hijacking, make sure that these __valid__ addresses
     aren't getting redirected. If they are, then our DNS is completely useless,
     and we'll reset our exit policy to "reject \*:*". This option only affects
@@ -1976,12 +2273,6 @@ is non-zero):
 [[GeoIPv6File]] **GeoIPv6File** __filename__::
     A filename containing IPv6 GeoIP data, for use with by-country statistics.
 
-[[TLSECGroup]] **TLSECGroup** **P224**|**P256**::
-    What EC group should we try to use for incoming TLS connections?
-    P224 is faster, but makes us stand out more. Has no effect if
-    we're a client, or if our OpenSSL version lacks support for ECDHE.
-    (Default: P256)
-
 [[CellStatistics]] **CellStatistics** **0**|**1**::
     Relays only.
     When this option is enabled, Tor collects statistics about cell
@@ -1992,6 +2283,15 @@ is non-zero):
     If ExtraInfoStatistics is enabled, it will published as part of
     extra-info document. (Default: 0)
 
+[[PaddingStatistics]] **PaddingStatistics** **0**|**1**::
+    Relays and bridges only.
+    When this option is enabled, Tor collects statistics for padding cells
+    sent and received by this relay, in addition to total cell counts.
+    These statistics are rounded, and omitted if traffic is low. This
+    information is important for load balancing decisions related to padding.
+    If ExtraInfoStatistics is enabled, it will be published
+    as a part of extra-info document. (Default: 1)
+
 [[DirReqStatistics]] **DirReqStatistics** **0**|**1**::
     Relays and bridges only.
     When this option is enabled, a Tor directory writes statistics on the
@@ -2079,11 +2379,28 @@ is non-zero):
     ed25519 master identity key, as well as the corresponding temporary
     signing keys and certificates. (Default: 0)
 
+[[KeyDirectory]] **KeyDirectory** __DIR__::
+    Store secret keys in DIR. Can not be changed while tor is
+    running.
+    (Default: the "keys" subdirectory of DataDirectory.)
+
+[[KeyDirectoryGroupReadable]] **KeyDirectoryGroupReadable** **0**|**1**::
+    If this option is set to 0, don't allow the filesystem group to read the
+    KeywDirectory. If the option is set to 1, make the KeyDirectory readable
+    by the default GID. (Default: 0)
+
+[[RephistTrackTime]] **RephistTrackTime** __N__ **seconds**|**minutes**|**hours**|**days**|**weeks**::
+    Tells an authority, or other node tracking node reliability and history,
+    that fine-grained information about nodes can be discarded when it hasn't
+    changed for a given amount of time.  (Default: 24 hours)
+
+
 DIRECTORY SERVER OPTIONS
 ------------------------
 
-The following options are useful only for directory servers (that is,
-if DirPort is non-zero):
+The following options are useful only for directory servers. (Relays with
+enough bandwidth automatically become directory servers; see DirCache for
+details.)
 
 [[DirPortFrontPage]] **DirPortFrontPage** __FILENAME__::
     When this option is set, it takes an HTML file and publishes it as "/" on
@@ -2095,19 +2412,10 @@ if DirPort is non-zero):
     If this option is nonzero, advertise the directory service on this port.
     Set it to "auto" to have Tor pick a port for you.  This option can occur
     more than once, but only one advertised DirPort is supported: all
-    but one DirPort must have the **NoAdvertise** flag set. (Default: 0)
-+
+    but one DirPort must have the **NoAdvertise** flag set. (Default: 0) +
+ +
     The same flags are supported here as are supported by ORPort.
 
-[[DirListenAddress]] **DirListenAddress** __IP__[:__PORT__]::
-    Bind the directory service to this address. If you specify a port, bind to
-    this port rather than the one specified in DirPort.  (Default: 0.0.0.0)
-    This directive can be specified multiple times  to bind to multiple
-    addresses/ports.
-+
-    This option is deprecated; you can get the same behavior with DirPort now
-    that it supports NoAdvertise and explicit addresses.
-
 [[DirPolicy]] **DirPolicy** __policy__,__policy__,__...__::
     Set an entrance policy for this server, to limit who can connect to the
     directory ports. The policies have the same form as exit policies above,
@@ -2115,10 +2423,152 @@ if DirPort is non-zero):
     some entry in the policy is accepted.
 
 [[DirCache]] **DirCache** **0**|**1**::
-    When this option is set, Tor caches all current directory documents and
-    accepts client requests for them. Setting DirPort is not required for this,
-    because clients connect via the ORPort by default. Setting either DirPort
-    or BridgeRelay and setting DirCache to 0 is not supported.  (Default: 1)
+    When this option is set, Tor caches all current directory documents except
+    extra info documents, and accepts client requests for them. If
+    **DownloadExtraInfo** is set, cached extra info documents are also cached.
+    Setting **DirPort** is not required for **DirCache**, because clients
+    connect via the ORPort by default. Setting either DirPort or BridgeRelay
+    and setting DirCache to 0 is not supported.  (Default: 1)
+
+[[MaxConsensusAgeForDiffs]] **MaxConsensusAgeForDiffs**  __N__ **minutes**|**hours**|**days**|**weeks**::
+    When this option is nonzero, Tor caches will not try to generate
+    consensus diffs for any consensus older than this amount of time.
+    If this option is set to zero, Tor will pick a reasonable default from
+    the current networkstatus document.  You should not set this
+    option unless your cache is severely low on disk space or CPU.
+    If you need to set it, keeping it above 3 or 4 hours will help clients
+    much more than setting it to zero.
+    (Default: 0)
+
+
+DENIAL OF SERVICE MITIGATION OPTIONS
+------------------------------------
+
+Tor has three built-in mitigation options that can be individually
+enabled/disabled and fine-tuned, but by default Tor directory authorities will
+define reasonable values for relays and no explicit configuration is required
+to make use of these protections. The mitigations take place at relays,
+and are as follows:
+
+  1. If a single client address makes too many concurrent connections (this is
+     configurable via DoSConnectionMaxConcurrentCount), hang up on further
+     connections.
+ +
+  2. If a single client IP address (v4 or v6) makes circuits too quickly
+     (default values are more than 3 per second, with an allowed burst of 90,
+     see DoSCircuitCreationRate and DoSCircuitCreationBurst) while also having
+     too many connections open (default is 3, see
+     DoSCircuitCreationMinConnections), tor will refuse any new circuit (CREATE
+     cells) for the next while (random value between 1 and 2 hours).
+ +
+  3. If a client asks to establish a rendezvous point to you directly (ex:
+     Tor2Web client), ignore the request.
+
+These defenses can be manually controlled by torrc options, but relays will
+also take guidance from consensus parameters using these same names, so there's
+no need to configure anything manually. In doubt, do not change those values.
+
+The values set by the consensus, if any, can be found here:
+https://consensus-health.torproject.org/#consensusparams
+
+If any of the DoS mitigations are enabled, a heartbeat message will appear in
+your log at NOTICE level which looks like:
+
+    DoS mitigation since startup: 429042 circuits rejected, 17 marked addresses.
+    2238 connections closed. 8052 single hop clients refused.
+
+The following options are useful only for a public relay. They control the
+Denial of Service mitigation subsystem described above.
+
+[[DoSCircuitCreationEnabled]] **DoSCircuitCreationEnabled** **0**|**1**|**auto**::
+
+    Enable circuit creation DoS mitigation. If set to 1 (enabled), tor will
+    cache client IPs along with statistics in order to detect circuit DoS
+    attacks. If an address is positively identified, tor will activate
+    defenses against the address. See the DoSCircuitCreationDefenseType option
+    for more details.  This is a client to relay detection only. "auto" means
+    use the consensus parameter. If not defined in the consensus, the value is 0.
+    (Default: auto)
+
+[[DoSCircuitCreationMinConnections]] **DoSCircuitCreationMinConnections** __NUM__::
+
+    Minimum threshold of concurrent connections before a client address can be
+    flagged as executing a circuit creation DoS. In other words, once a client
+    address reaches the circuit rate and has a minimum of NUM concurrent
+    connections, a detection is positive. "0" means use the consensus
+    parameter. If not defined in the consensus, the value is 3.
+    (Default: 0)
+
+[[DoSCircuitCreationRate]] **DoSCircuitCreationRate** __NUM__::
+
+    The allowed circuit creation rate per second applied per client IP
+    address. If this option is 0, it obeys a consensus parameter. If not
+    defined in the consensus, the value is 3.
+    (Default: 0)
+
+[[DoSCircuitCreationBurst]] **DoSCircuitCreationBurst** __NUM__::
+
+    The allowed circuit creation burst per client IP address. If the circuit
+    rate and the burst are reached, a client is marked as executing a circuit
+    creation DoS. "0" means use the consensus parameter. If not defined in the
+    consensus, the value is 90.
+    (Default: 0)
+
+[[DoSCircuitCreationDefenseType]] **DoSCircuitCreationDefenseType** __NUM__::
+
+    This is the type of defense applied to a detected client address. The
+    possible values are:
+ +
+      1: No defense.
+ +
+      2: Refuse circuit creation for the DoSCircuitCreationDefenseTimePeriod period of time.
+ +
+    "0" means use the consensus parameter. If not defined in the consensus, the value is 2.
+    (Default: 0)
+
+[[DoSCircuitCreationDefenseTimePeriod]] **DoSCircuitCreationDefenseTimePeriod** __N__ **seconds**|**minutes**|**hours**::
+
+    The base time period in seconds that the DoS defense is activated for. The
+    actual value is selected randomly for each activation from N+1 to 3/2 * N.
+    "0" means use the consensus parameter. If not defined in the consensus,
+    the value is 3600 seconds (1 hour).
+    (Default: 0)
+
+[[DoSConnectionEnabled]] **DoSConnectionEnabled** **0**|**1**|**auto**::
+
+    Enable the connection DoS mitigation. If set to 1 (enabled), for client
+    address only, this allows tor to mitigate against large number of
+    concurrent connections made by a single IP address. "auto" means use the
+    consensus parameter. If not defined in the consensus, the value is 0.
+    (Default: auto)
+
+[[DoSConnectionMaxConcurrentCount]] **DoSConnectionMaxConcurrentCount** __NUM__::
+
+    The maximum threshold of concurrent connection from a client IP address.
+    Above this limit, a defense selected by DoSConnectionDefenseType is
+    applied. "0" means use the consensus parameter. If not defined in the
+    consensus, the value is 100.
+    (Default: 0)
+
+[[DoSConnectionDefenseType]] **DoSConnectionDefenseType** __NUM__::
+
+    This is the type of defense applied to a detected client address for the
+    connection mitigation. The possible values are:
+ +
+      1: No defense.
+ +
+      2: Immediately close new connections.
+ +
+    "0" means use the consensus parameter. If not defined in the consensus, the value is 2.
+    (Default: 0)
+
+[[DoSRefuseSingleHopClientRendezvous]] **DoSRefuseSingleHopClientRendezvous** **0**|**1**|**auto**::
+
+    Refuse establishment of rendezvous points for single hop clients. In other
+    words, if a client directly connects to the relay and sends an
+    ESTABLISH_RENDEZVOUS cell, it is silently dropped. "auto" means use the
+    consensus parameter. If not defined in the consensus, the value is 0.
+    (Default: auto)
 
 
 DIRECTORY AUTHORITY SERVER OPTIONS
@@ -2199,7 +2649,7 @@ on the public Tor network.
 [[AuthDirBadExit]] **AuthDirBadExit** __AddressPattern...__::
     Authoritative directories only. A set of address patterns for servers that
     will be listed as bad exits in any network status document this authority
-    publishes, if **AuthDirListBadExits** is set.
+    publishes, if **AuthDirListBadExits** is set. +
  +
     (The address pattern syntax here and in the options below
     is the same as for exit policies, except that you don't need to say
@@ -2237,26 +2687,22 @@ on the public Tor network.
     list as acceptable on a single IP address. Set this to "0" for "no limit".
     (Default: 2)
 
-[[AuthDirMaxServersPerAuthAddr]] **AuthDirMaxServersPerAuthAddr** __NUM__::
-    Authoritative directories only. Like AuthDirMaxServersPerAddr, but applies
-    to addresses shared with directory authorities. (Default: 5)
-
-[[AuthDirFastGuarantee]] **AuthDirFastGuarantee** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[AuthDirFastGuarantee]] **AuthDirFastGuarantee** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     Authoritative directories only. If non-zero, always vote the
     Fast flag for any relay advertising this amount of capacity or
     more. (Default: 100 KBytes)
 
-[[AuthDirGuardBWGuarantee]] **AuthDirGuardBWGuarantee** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[AuthDirGuardBWGuarantee]] **AuthDirGuardBWGuarantee** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     Authoritative directories only. If non-zero, this advertised capacity
     or more is always sufficient to satisfy the bandwidth requirement
-    for the Guard flag. (Default: 250 KBytes)
+    for the Guard flag. (Default: 2 MBytes)
 
 [[AuthDirPinKeys]] **AuthDirPinKeys** **0**|**1**::
     Authoritative directories only. If non-zero, do not allow any relay to
     publish a descriptor if any other relay has reserved its <Ed25519,RSA>
     identity keypair. In all cases, Tor records every keypair it accepts
     in a journal if it is new, or if it differs from the most recently
-    accepted pinning for one of the keys it contains. (Default: 0)
+    accepted pinning for one of the keys it contains. (Default: 1)
 
 [[AuthDirSharedRandomness]] **AuthDirSharedRandomness** **0**|**1**::
     Authoritative directories only. Switch for the shared random protocol.
@@ -2264,6 +2710,13 @@ on the public Tor network.
     (default), the flag "shared-rand-participate" is added to the authority
     vote indicating participation in the protocol. (Default: 1)
 
+[[AuthDirTestEd25519LinkKeys]] **AuthDirTestEd25519LinkKeys**  **0**|**1**::
+    Authoritative directories only. If this option is set to 0, then we treat
+    relays as "Running" if their RSA key is correct when we probe them,
+    regardless of their Ed25519 key. We should only ever set this option to 0
+    if there is some major bug in Ed25519 link authentication that causes us
+    to label all the relays as not Running.  (Default: 1)
+
 [[BridgePassword]] **BridgePassword** __Password__::
     If set, contains an HTTP authenticator that tells a bridge authority to
     serve all requested bridge information. Used by the (only partially
@@ -2302,7 +2755,9 @@ on the public Tor network.
 [[V3BandwidthsFile]] **V3BandwidthsFile** __FILENAME__::
     V3 authoritative directories only. Configures the location of the
     bandwidth-authority generated file storing information on relays' measured
-    bandwidth capacities. (Default: unset)
+    bandwidth capacities. To avoid inconsistent reads, bandwidth data should
+    be written to temporary file, then renamed to the configured filename.
+    (Default: unset)
 
 [[V3AuthUseLegacyKey]] **V3AuthUseLegacyKey** **0**|**1**::
     If set, the directory authority will sign consensuses not only with its
@@ -2310,16 +2765,31 @@ on the public Tor network.
     different identity.  This feature is used to migrate directory authority
     keys in the event of a compromise.  (Default: 0)
 
-[[RephistTrackTime]] **RephistTrackTime** __N__ **seconds**|**minutes**|**hours**|**days**|**weeks**::
-    Tells an authority, or other node tracking node reliability and history,
-    that fine-grained information about nodes can be discarded when it hasn't
-    changed for a given amount of time.  (Default: 24 hours)
-
 [[AuthDirHasIPv6Connectivity]] **AuthDirHasIPv6Connectivity** **0**|**1**::
     Authoritative directories only. When set to 0, OR ports with an
-    IPv6 address are being accepted without reachability testing.
-    When set to 1, IPv6 OR ports are being tested just like IPv4 OR
-    ports. (Default: 0)
+    IPv6 address are not included in the authority's votes. When set to 1,
+    IPv6 OR ports are tested for reachability like IPv4 OR ports. If the
+    reachability test succeeds, the authority votes for the IPv6 ORPort, and
+    votes Running for the relay. If the reachability test fails, the authority
+    does not vote for the IPv6 ORPort, and does not vote Running (Default: 0)  +
++
+    The content of the consensus depends on the number of voting authorities
+    that set AuthDirHasIPv6Connectivity:
+
+      If no authorities set AuthDirHasIPv6Connectivity 1, there will be no
+      IPv6 ORPorts in the consensus.
+
+      If a minority of authorities set AuthDirHasIPv6Connectivity 1,
+      unreachable IPv6 ORPorts will be removed from the consensus. But the
+      majority of IPv4-only authorities will still vote the relay as Running.
+      Reachable IPv6 ORPort lines will be included in the consensus
+
+      If a majority of voting authorities set AuthDirHasIPv6Connectivity 1,
+      relays with unreachable IPv6 ORPorts will not be listed as Running.
+      Reachable IPv6 ORPort lines will be included in the consensus
+      (To ensure that any valid majority will vote relays with unreachable
+      IPv6 ORPorts not Running, 75% of authorities must set
+      AuthDirHasIPv6Connectivity 1.)
 
 [[MinMeasuredBWsForAuthToIgnoreAdvertised]] **MinMeasuredBWsForAuthToIgnoreAdvertised** __N__::
     A total value, in abstract bandwidth units, describing how much
@@ -2335,9 +2805,9 @@ The following options are used to configure a hidden service.
 [[HiddenServiceDir]] **HiddenServiceDir** __DIRECTORY__::
     Store data files for a hidden service in DIRECTORY. Every hidden service
     must have a separate directory. You may use this option  multiple times to
-    specify multiple services. DIRECTORY must be an existing directory.
+    specify multiple services. If DIRECTORY does not exist, Tor will create it.
     (Note: in current versions of Tor, if DIRECTORY is a relative path,
-    it will be relative to current
+    it will be relative to the current
     working directory of Tor instance, not to its DataDirectory.  Do not
     rely on this behavior; it is not guaranteed to remain the same in future
     versions.)
@@ -2352,7 +2822,7 @@ The following options are used to configure a hidden service.
     paths may be quoted, and may use standard C escapes.)
     You may also have multiple lines with  the same VIRTPORT: when a user
     connects to that VIRTPORT, one of the TARGETs from those lines will be
-    chosen at random.
+    chosen at random. Note that address-port pairs have to be comma-separated.
 
 [[PublishHidServDescriptors]] **PublishHidServDescriptors** **0**|**1**::
     If set to 0, Tor will run any hidden services you configure, but it won't
@@ -2360,9 +2830,9 @@ The following options are used to configure a hidden service.
     you're using a Tor controller that handles hidserv publishing for you.
     (Default: 1)
 
-[[HiddenServiceVersion]] **HiddenServiceVersion** __version__,__version__,__...__::
+[[HiddenServiceVersion]] **HiddenServiceVersion** **2**|**3**::
     A list of rendezvous service descriptor versions to publish for the hidden
-    service. Currently, only version 2 is supported. (Default: 2)
+    service. Currently, versions 2 and 3 are supported. (Default: 3)
 
 [[HiddenServiceAuthorizeClient]] **HiddenServiceAuthorizeClient** __auth-type__ __client-name__,__client-name__,__...__::
     If configured, the hidden service is accessible for authorized clients
@@ -2374,7 +2844,9 @@ The following options are used to configure a hidden service.
     spaces). If this option is set, the hidden service is not accessible for
     clients without authorization any more. Generated authorization data can be
     found in the hostname file. Clients need to put this authorization data in
-    their configuration file using **HidServAuth**.
+    their configuration file using **HidServAuth**. This option is only for v2
+    services; v3 services configure client authentication in a subdirectory of
+    HiddenServiceDir instead (see the **Client Authorization** section).
 
 [[HiddenServiceAllowUnknownPorts]] **HiddenServiceAllowUnknownPorts** **0**|**1**::
    If set to 1, then connections to unrecognized ports do not cause the
@@ -2382,10 +2854,37 @@ The following options are used to configure a hidden service.
    not an authorization mechanism; it is instead meant to be a mild
    inconvenience to port-scanners.) (Default: 0)
 
+[[HiddenServiceExportCircuitID]] **HiddenServiceExportCircuitID** __protocol__::
+   The onion service will use the given protocol to expose the global circuit
+   identifier of each inbound client circuit via the selected protocol. The only
+   protocol supported right now \'haproxy'. This option is only for v3
+   services. (Default: none) +
+ +
+   The haproxy option works in the following way: when the feature is
+   enabled, the Tor process will write a header line when a client is connecting
+   to the onion service. The header will look like this: +
+ +
+   "PROXY TCP6 fc00:dead:beef:4dad::ffff:ffff ::1 65535 42\r\n" +
+ +
+   We encode the "global circuit identifier" as the last 32-bits of the first
+   IPv6 address. All other values in the header can safely be ignored. You can
+   compute the global circuit identifier using the following formula given the
+   IPv6 address "fc00:dead:beef:4dad::AABB:CCDD": +
+ +
+      global_circuit_id = (0xAA << 24) + (0xBB << 16) + (0xCC << 8) + 0xDD; +
+ +
+   In the case above, where the last 32-bit is 0xffffffff, the global circuit
+   identifier would be 4294967295. You can use this value together with Tor's
+   control port where it is possible to terminate a circuit given the global
+   circuit identifier. For more information about this see controls-spec.txt. +
+ +
+   The HAProxy version 1 proxy protocol is described in detail at
+   https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
+
 [[HiddenServiceMaxStreams]] **HiddenServiceMaxStreams** __N__::
    The maximum number of simultaneous streams (connections) per rendezvous
-   circuit. (Setting this to 0 will allow an unlimited number of simultanous
-   streams.) (Default: 0)
+   circuit. The maximum value allowed is 65535. (Setting this to 0 will allow
+   an unlimited number of simultaneous streams.) (Default: 0)
 
 [[HiddenServiceMaxStreamsCloseCircuit]] **HiddenServiceMaxStreamsCloseCircuit** **0**|**1**::
    If set to 1, then exceeding **HiddenServiceMaxStreams** will cause the
@@ -2394,8 +2893,10 @@ The following options are used to configure a hidden service.
 
 [[RendPostPeriod]] **RendPostPeriod** __N__ **seconds**|**minutes**|**hours**|**days**|**weeks**::
     Every time the specified period elapses, Tor uploads any rendezvous
-    service descriptors to the directory servers. This information  is also
-    uploaded whenever it changes. (Default: 1 hour)
+    service descriptors to the directory servers. This information is also
+    uploaded whenever it changes. Minimum value allowed is 10 minutes and
+    maximum is 3.5 days. This option is only for v2 services.
+    (Default: 1 hour)
 
 [[HiddenServiceDirGroupReadable]] **HiddenServiceDirGroupReadable** **0**|**1**::
     If this option is set to 1, allow the filesystem group to read the
@@ -2405,7 +2906,7 @@ The following options are used to configure a hidden service.
 
 [[HiddenServiceNumIntroductionPoints]] **HiddenServiceNumIntroductionPoints** __NUM__::
     Number of introduction points the hidden service will have. You can't
-    have more than 10. (Default: 3)
+    have more than 10 for v2 service and 20 for v3. (Default: 3)
 
 [[HiddenServiceSingleHopMode]] **HiddenServiceSingleHopMode** **0**|**1**::
     **Experimental - Non Anonymous** Hidden Services on a tor instance in
@@ -2417,20 +2918,20 @@ The following options are used to configure a hidden service.
     Single Onion Service. One-hop circuits make Single Onion servers easily
     locatable, but clients remain location-anonymous. However, the fact that a
     client is accessing a Single Onion rather than a Hidden Service may be
-    statistically distinguishable.
-
+    statistically distinguishable. +
+ +
     **WARNING:** Once a hidden service directory has been used by a tor
     instance in HiddenServiceSingleHopMode, it can **NEVER** be used again for
     a hidden service. It is best practice to create a new hidden service
     directory, key, and address for each new Single Onion Service and Hidden
     Service. It is not possible to run Single Onion Services and Hidden
     Services from the same tor instance: they should be run on different
-    servers with different IP addresses.
-
+    servers with different IP addresses. +
+ +
     HiddenServiceSingleHopMode requires HiddenServiceNonAnonymousMode to be set
     to 1. Since a Single Onion service is non-anonymous, you can not configure
     a SOCKSPort on a tor instance that is running in
-    **HiddenServiceSingleHopMode**.
+    **HiddenServiceSingleHopMode**. Can not be changed while tor is running.
     (Default: 0)
 
 [[HiddenServiceNonAnonymousMode]] **HiddenServiceNonAnonymousMode** **0**|**1**::
@@ -2438,103 +2939,39 @@ The following options are used to configure a hidden service.
     non-anonymous HiddenServiceSingleHopMode. Enables direct connections in the
     server-side hidden service protocol.  If you are using this option,
     you need to disable all client-side services on your Tor instance,
-    including setting SOCKSPort to "0".
-    (Default: 0)
+    including setting SOCKSPort to "0". Can not be changed while tor is
+    running. (Default: 0)
 
-DENIAL OF SERVICE MITIGATION OPTIONS
-------------------------------------
-
-The following options are useful only for a public relay. They control the
-Denial of Service mitigation subsystem.
-
-[[DoSCircuitCreationEnabled]] **DoSCircuitCreationEnabled** **0**|**1**|**auto**::
-
-    Enable circuit creation DoS mitigation. If enabled, tor will cache client
-    IPs along with statistics in order to detect circuit DoS attacks. If an
-    address is positively identified, tor will activate defenses against the
-    address. See the DoSCircuitCreationDefenseType option for more details.
-    This is a client to relay detection only. "auto" means use the consensus
-    parameter. If not defined in the consensus, the value is 0.
-    (Default: auto)
-
-[[DoSCircuitCreationMinConnections]] **DoSCircuitCreationMinConnections** __NUM__::
-
-    Minimum threshold of concurrent connections before a client address can be
-    flagged as executing a circuit creation DoS. In other words, once a client
-    address reaches the circuit rate and has a minimum of NUM concurrent
-    connections, a detection is positive. "0" means use the consensus
-    parameter. If not defined in the consensus, the value is 3.
-    (Default: 0)
-
-[[DoSCircuitCreationRate]] **DoSCircuitCreationRate** __NUM__::
-
-    The allowed circuit creation rate per second applied per client IP
-    address. If this option is 0, it obeys a consensus parameter. If not
-    defined in the consensus, the value is 3.
-    (Default: 0)
-
-[[DoSCircuitCreationBurst]] **DoSCircuitCreationBurst** __NUM__::
-
-    The allowed circuit creation burst per client IP address. If the circuit
-    rate and the burst are reached, a client is marked as executing a circuit
-    creation DoS. "0" means use the consensus parameter. If not defined in the
-    consensus, the value is 90.
-    (Default: 0)
-
-[[DoSCircuitCreationDefenseType]] **DoSCircuitCreationDefenseType** __NUM__::
-
-    This is the type of defense applied to a detected client address. The
-    possible values are:
-
-      1: No defense.
-      2: Refuse circuit creation for the DoSCircuitCreationDefenseTimePeriod period of time.
-+
-    "0" means use the consensus parameter. If not defined in the consensus,
-    the value is 2.
-    (Default: 0)
-
-[[DoSCircuitCreationDefenseTimePeriod]] **DoSCircuitCreationDefenseTimePeriod** __N__ **seconds**|**minutes**|**hours**::
-
-    The base time period in seconds that the DoS defense is activated for. The
-    actual value is selected randomly for each activation from N+1 to 3/2 * N.
-    "0" means use the consensus parameter. If not defined in the consensus,
-    the value is 3600 seconds (1 hour).  (Default: 0)
-
-[[DoSConnectionEnabled]] **DoSConnectionEnabled** **0**|**1**|**auto**::
+Client Authorization
+--------------------
 
-    Enable the connection DoS mitigation. For client address only, this allows
-    tor to mitigate against large number of concurrent connections made by a
-    single IP address. "auto" means use the consensus parameter. If not
-    defined in the consensus, the value is 0.
-    (Default: auto)
+(Version 3 only)
 
-[[DoSConnectionMaxConcurrentCount]] **DoSConnectionMaxConcurrentCount** __NUM__::
+To configure client authorization on the service side, the
+"<HiddenServiceDir>/authorized_clients/" directory needs to exist. Each file
+in that directory should be suffixed with ".auth" (i.e. "alice.auth"; the
+file name is irrelevant) and its content format MUST be:
 
-    The maximum threshold of concurrent connection from a client IP address.
-    Above this limit, a defense selected by DoSConnectionDefenseType is
-    applied. "0" means use the consensus parameter. If not defined in the
-    consensus, the value is 100.
-    (Default: 0)
+        <auth-type>:<key-type>:<base32-encoded-public-key>
 
-[[DoSConnectionDefenseType]] **DoSConnectionDefenseType** __NUM__::
+The supported <auth-type> are: "descriptor". The supported <key-type> are:
+"x25519". The <base32-encoded-public-key> is the base32 representation of
+the raw key bytes only (32 bytes for x25519).
 
-    This is the type of defense applied to a detected client address for the
-    connection mitigation. The possible values are:
+Each file MUST contain one line only. Any malformed file will be
+ignored. Client authorization will only be enabled for the service if tor
+successfully loads at least one authorization file.
 
-      1: No defense.
-      2: Immediately close new connections.
-+
-    "0" means use the consensus parameter. If not defined in the consensus,
-    the value is 2.
-    (Default: 0)
+Note that once you've configured client authorization, anyone else with the
+address won't be able to access it from this point on. If no authorization is
+configured, the service will be accessible to anyone with the onion address.
 
-[[DoSRefuseSingleHopClientRendezvous]] **DoSRefuseSingleHopClientRendezvous** **0**|**1**|**auto**::
+Revoking a client can be done by removing their ".auth" file, however the
+revocation will be in effect only after the tor process gets restarted even if
+a SIGHUP takes place.
 
-    Refuse establishment of rendezvous points for single hop clients. In other
-    words, if a client directly connects to the relay and sends an
-    ESTABLISH_RENDEZVOUS cell, it is silently dropped. "auto" means use the
-    consensus parameter. If not defined in the consensus, the value is 0.
-    (Default: auto)
+See the Appendix G in the rend-spec-v3.txt file of
+https://spec.torproject.org/[torspec] for more information.
 
 TESTING NETWORK OPTIONS
 -----------------------
@@ -2554,14 +2991,9 @@ The following options are used for running a testing Tor network.
        AssumeReachable 1
        AuthDirMaxServersPerAddr 0
        AuthDirMaxServersPerAuthAddr 0
-       ClientBootstrapConsensusAuthorityDownloadSchedule 0, 2,
-          4 (for 40 seconds), 8, 16, 32, 60
-       ClientBootstrapConsensusFallbackDownloadSchedule 0, 1,
-          4 (for 40 seconds), 8, 16, 32, 60
-       ClientBootstrapConsensusAuthorityOnlyDownloadSchedule 0, 1,
-          4 (for 40 seconds), 8, 16, 32, 60
-       ClientBootstrapConsensusMaxDownloadTries 80
-       ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries 80
+       ClientBootstrapConsensusAuthorityDownloadInitialDelay 0
+       ClientBootstrapConsensusFallbackDownloadInitialDelay 0
+       ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay 0
        ClientDNSRejectInternalAddresses 0
        ClientRejectInternalAddresses 0
        CountPrivateBandwidth 1
@@ -2576,20 +3008,16 @@ The following options are used for running a testing Tor network.
        TestingV3AuthInitialDistDelay 20 seconds
        TestingAuthDirTimeToLearnReachability 0 minutes
        TestingEstimatedDescriptorPropagationTime 0 minutes
-       TestingServerDownloadSchedule 0, 0, 0, 5, 10, 15, 20, 30, 60
-       TestingClientDownloadSchedule 0, 0, 5, 10, 15, 20, 30, 60
-       TestingServerConsensusDownloadSchedule 0, 0, 5, 10, 15, 20, 30, 60
-       TestingClientConsensusDownloadSchedule 0, 0, 5, 10, 15, 20, 30, 60
-       TestingBridgeDownloadSchedule 60, 30, 30, 60
+       TestingServerDownloadInitialDelay 0
+       TestingClientDownloadInitialDelay 0
+       TestingServerConsensusDownloadInitialDelay 0
+       TestingClientConsensusDownloadInitialDelay 0
+       TestingBridgeDownloadInitialDelay 10
+       TestingBridgeBootstrapDownloadInitialDelay 0
        TestingClientMaxIntervalWithoutRequest 5 seconds
        TestingDirConnectionMaxStall 30 seconds
-       TestingConsensusMaxDownloadTries 80
-       TestingDescriptorMaxDownloadTries 80
-       TestingMicrodescMaxDownloadTries 80
-       TestingCertMaxDownloadTries 80
        TestingEnableConnBwEvent 1
        TestingEnableCellStatsEvent 1
-       TestingEnableTbEmptyEvent 1
 
 [[TestingV3AuthInitialVotingInterval]] **TestingV3AuthInitialVotingInterval** __N__ **minutes**|**hours**::
     Like V3AuthVotingInterval, but for initial voting interval before the first
@@ -2620,33 +3048,35 @@ The following options are used for running a testing Tor network.
     time. Changing this requires that **TestingTorNetwork** is set. (Default:
     10 minutes)
 
-[[TestingMinFastFlagThreshold]] **TestingMinFastFlagThreshold** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[TestingMinFastFlagThreshold]] **TestingMinFastFlagThreshold** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     Minimum value for the Fast flag.  Overrides the ordinary minimum taken
     from the consensus when TestingTorNetwork is set. (Default: 0.)
 
-[[TestingServerDownloadSchedule]] **TestingServerDownloadSchedule** __N__,__N__,__...__::
-    Schedule for when servers should download things in general. Changing this
-    requires that **TestingTorNetwork** is set. (Default: 0, 0, 0, 60, 60, 120,
-    300, 900, 2147483647)
+[[TestingServerDownloadInitialDelay]] **TestingServerDownloadInitialDelay** __N__::
+    Initial delay in seconds for when servers should download things in general. Changing this
+    requires that **TestingTorNetwork** is set. (Default: 0)
 
-[[TestingClientDownloadSchedule]] **TestingClientDownloadSchedule** __N__,__N__,__...__::
-    Schedule for when clients should download things in general. Changing this
-    requires that **TestingTorNetwork** is set. (Default: 0, 0, 60, 300, 600,
-    2147483647)
+[[TestingClientDownloadInitialDelay]] **TestingClientDownloadInitialDelay** __N__::
+    Initial delay in seconds for when clients should download things in general. Changing this
+    requires that **TestingTorNetwork** is set. (Default: 0)
 
-[[TestingServerConsensusDownloadSchedule]] **TestingServerConsensusDownloadSchedule** __N__,__N__,__...__::
-    Schedule for when servers should download consensuses. Changing this
-    requires that **TestingTorNetwork** is set. (Default: 0, 0, 60, 300, 600,
-    1800, 1800, 1800, 1800, 1800, 3600, 7200)
+[[TestingServerConsensusDownloadInitialDelay]] **TestingServerConsensusDownloadInitialDelay** __N__::
+    Initial delay in seconds for when servers should download consensuses. Changing this
+    requires that **TestingTorNetwork** is set. (Default: 0)
 
-[[TestingClientConsensusDownloadSchedule]] **TestingClientConsensusDownloadSchedule** __N__,__N__,__...__::
-    Schedule for when clients should download consensuses. Changing this
-    requires that **TestingTorNetwork** is set. (Default: 0, 0, 60, 300, 600,
-    1800, 3600, 3600, 3600, 10800, 21600, 43200)
+[[TestingClientConsensusDownloadInitialDelay]] **TestingClientConsensusDownloadInitialDelay** __N__::
+    Initial delay in seconds for when clients should download consensuses. Changing this
+    requires that **TestingTorNetwork** is set. (Default: 0)
 
-[[TestingBridgeDownloadSchedule]] **TestingBridgeDownloadSchedule** __N__,__N__,__...__::
-    Schedule for when clients should download bridge descriptors. Changing this
-    requires that **TestingTorNetwork** is set. (Default: 3600, 900, 900, 3600)
+[[TestingBridgeDownloadInitialDelay]] **TestingBridgeDownloadInitialDelay** __N__::
+    Initial delay in seconds for when clients should download each bridge descriptor when they
+    know that one or more of their configured bridges are running. Changing
+    this requires that **TestingTorNetwork** is set. (Default: 10800)
+
+[[TestingBridgeBootstrapDownloadInitialDelay]] **TestingBridgeBootstrapDownloadInitialDelay** __N__::
+    Initial delay in seconds for when clients should download each bridge descriptor when they
+    have just started, or when they can not contact any of their bridges.
+    Changing this requires that **TestingTorNetwork** is set. (Default: 0)
 
 [[TestingClientMaxIntervalWithoutRequest]] **TestingClientMaxIntervalWithoutRequest** __N__ **seconds**|**minutes**::
     When directory clients have only a few descriptors to request, they batch
@@ -2659,27 +3089,11 @@ The following options are used for running a testing Tor network.
     Changing this requires that **TestingTorNetwork** is set. (Default:
     5 minutes)
 
-[[TestingConsensusMaxDownloadTries]] **TestingConsensusMaxDownloadTries** __NUM__::
-    Try this many times to download a consensus before giving up. Changing
-    this requires that **TestingTorNetwork** is set. (Default: 8)
-
-[[TestingDescriptorMaxDownloadTries]] **TestingDescriptorMaxDownloadTries** __NUM__::
-    Try this often to download a server descriptor before giving up.
-    Changing this requires that **TestingTorNetwork** is set. (Default: 8)
-
-[[TestingMicrodescMaxDownloadTries]] **TestingMicrodescMaxDownloadTries** __NUM__::
-    Try this often to download a microdesc descriptor before giving up.
-    Changing this requires that **TestingTorNetwork** is set. (Default: 8)
-
-[[TestingCertMaxDownloadTries]] **TestingCertMaxDownloadTries** __NUM__::
-    Try this often to download a v3 authority certificate before giving up.
-    Changing this requires that **TestingTorNetwork** is set. (Default: 8)
-
 [[TestingDirAuthVoteExit]] **TestingDirAuthVoteExit** __node__,__node__,__...__::
     A list of identity fingerprints, country codes, and
     address patterns of nodes to vote Exit for regardless of their
     uptime, bandwidth, or exit policy. See the **ExcludeNodes**
-    option for more information on how to specify nodes.
+    option for more information on how to specify nodes. +
  +
     In order for this option to have any effect, **TestingTorNetwork**
     has to be set. See the **ExcludeNodes** option for more
@@ -2688,7 +3102,7 @@ The following options are used for running a testing Tor network.
 [[TestingDirAuthVoteExitIsStrict]] **TestingDirAuthVoteExitIsStrict** **0**|**1** ::
     If True (1), a node will never receive the Exit flag unless it is specified
     in the **TestingDirAuthVoteExit** list, regardless of its uptime, bandwidth,
-    or exit policy.
+    or exit policy. +
  +
     In order for this option to have any effect, **TestingTorNetwork**
     has to be set.
@@ -2697,14 +3111,14 @@ The following options are used for running a testing Tor network.
     A list of identity fingerprints and country codes and
     address patterns of nodes to vote Guard for regardless of their
     uptime and bandwidth. See the **ExcludeNodes** option for more
-    information on how to specify nodes.
+    information on how to specify nodes. +
  +
     In order for this option to have any effect, **TestingTorNetwork**
     has to be set.
 
 [[TestingDirAuthVoteGuardIsStrict]] **TestingDirAuthVoteGuardIsStrict** **0**|**1** ::
     If True (1), a node will never receive the Guard flag unless it is specified
-    in the **TestingDirAuthVoteGuard** list, regardless of its uptime and bandwidth.
+    in the **TestingDirAuthVoteGuard** list, regardless of its uptime and bandwidth. +
  +
     In order for this option to have any effect, **TestingTorNetwork**
     has to be set.
@@ -2713,14 +3127,14 @@ The following options are used for running a testing Tor network.
     A list of identity fingerprints and country codes and
     address patterns of nodes to vote HSDir for regardless of their
     uptime and DirPort. See the **ExcludeNodes** option for more
-    information on how to specify nodes.
+    information on how to specify nodes. +
  +
     In order for this option to have any effect, **TestingTorNetwork**
     must be set.
 
 [[TestingDirAuthVoteHSDirIsStrict]] **TestingDirAuthVoteHSDirIsStrict** **0**|**1** ::
     If True (1), a node will never receive the HSDir flag unless it is specified
-    in the **TestingDirAuthVoteHSDir** list, regardless of its uptime and DirPort.
+    in the **TestingDirAuthVoteHSDir** list, regardless of its uptime and DirPort. +
  +
     In order for this option to have any effect, **TestingTorNetwork**
     has to be set.
@@ -2735,12 +3149,7 @@ The following options are used for running a testing Tor network.
     events.  Changing this requires that **TestingTorNetwork** is set.
     (Default: 0)
 
-[[TestingEnableTbEmptyEvent]] **TestingEnableTbEmptyEvent** **0**|**1**::
-    If this option is set, then Tor controllers may register for TB_EMPTY
-    events.  Changing this requires that **TestingTorNetwork** is set.
-    (Default: 0)
-
-[[TestingMinExitFlagThreshold]] **TestingMinExitFlagThreshold**  __N__ **KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[TestingMinExitFlagThreshold]] **TestingMinExitFlagThreshold**  __N__ **KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     Sets a lower-bound for assigning an exit flag when running as an
     authority on a testing network. Overrides the usual default lower bound
     of 4 KB. (Default: 0)
@@ -2764,6 +3173,19 @@ The following options are used for running a testing Tor network.
     we replace it and issue a new key?
     (Default: 3 hours for link and auth; 1 day for signing.)
 
+NON-PERSISTENT OPTIONS
+----------------------
+
+These options are not saved to the torrc file by the "SAVECONF" controller
+command.  Other options of this type are documented in control-spec.txt,
+section 5.4. End-users should mostly ignore them.
+
+[[UnderscorePorts]] **\_\_ControlPort**, **\_\_DirPort**, **\_\_DNSPort**, **\_\_ExtORPort**, **\_\_NATDPort**, **\_\_ORPort**, **\_\_SocksPort**, **\_\_TransPort**::
+    These underscore-prefixed options are variants of the regular Port
+    options.  They behave the same, except they are not saved to the
+    torrc file by the controller's SAVECONF command.
+
+
 SIGNALS
 -------
 
@@ -2810,32 +3232,35 @@ FILES
 **@LOCALSTATEDIR@/lib/tor/**::
     The tor process stores keys and other data here.
 
-__DataDirectory__**/cached-status/**::
-    The most recently downloaded network status document for each authority.
-    Each file holds one such document; the filenames are the hexadecimal
-    identity key fingerprints of the directory authorities.  Mostly obsolete.
 
-__DataDirectory__**/cached-certs**::
+__CacheDirectory__**/cached-certs**::
     This file holds downloaded directory key certificates that are used to
     verify authenticity of documents generated by Tor directory authorities.
 
-__DataDirectory__**/cached-consensus** and/or **cached-microdesc-consensus**::
+__CacheDirectory__**/cached-consensus** and/or **cached-microdesc-consensus**::
     The most recent consensus network status document we've downloaded.
 
-__DataDirectory__**/cached-descriptors** and **cached-descriptors.new**::
+__CacheDirectory__**/cached-descriptors** and **cached-descriptors.new**::
     These files hold downloaded router statuses. Some routers may appear more
     than once; if so, the most recently published descriptor is used. Lines
     beginning with @-signs are annotations that contain more information about
     a given router. The ".new" file is an append-only journal; when it gets
     too large, all entries are merged into a new cached-descriptors file.
 
-__DataDirectory__**/cached-microdescs** and **cached-microdescs.new**::
+__CacheDirectory__**/cached-extrainfo** and **cached-extrainfo.new**::
+   As "cached-descriptors", but holds optionally-downloaded "extra-info"
+   documents. Relays use these documents to send inessential information
+   about statistics, bandwidth history, and network health to the
+   authorities. They aren't fetched by default; see the DownloadExtraInfo
+   option for more info.
+
+__CacheDirectory__**/cached-microdescs** and **cached-microdescs.new**::
     These files hold downloaded microdescriptors.  Lines beginning with
     @-signs are annotations that contain more information about a given
     router. The ".new" file is an append-only journal; when it gets too
     large, all entries are merged into a new cached-microdescs file.
 
-__DataDirectory__**/cached-routers** and **cached-routers.new**::
+__CacheDirectory__**/cached-routers** and **cached-routers.new**::
     Obsolete versions of cached-descriptors and cached-descriptors.new. When
     Tor can't find the newer files, it looks here instead.
 
@@ -2843,18 +3268,27 @@ __DataDirectory__**/state**::
     A set of persistent key-value mappings. These are documented in
     the file. These include:
             - The current entry guards and their status.
-            - The current bandwidth accounting values (unused so far; see
-            below).
+            - The current bandwidth accounting values.
             - When the file was last written
             - What version of Tor generated the state file
             - A short history of bandwidth usage, as produced in the server
             descriptors.
 
+__DataDirectory__**/sr-state**::
+    Authority only. State file used to record information about the current
+    status of the shared-random-value voting state.
+
+__CacheDirectory__**/diff-cache**::
+    Directory cache only. Holds older consensuses, and diffs from older
+    consensuses to the most recent consensus of each type, compressed
+    in various ways. Each file contains a set of key-value arguments
+    describing its contents, followed by a single NUL byte, followed by the
+    main file contents.
+
 __DataDirectory__**/bw_accounting**::
     Used to track bandwidth accounting values (when the current period starts
     and ends; how much has been read and written so far this period). This file
-    is obsolete, and the data is now stored in the \'state' file as well. Only
-    used when bandwidth accounting is enabled.
+    is obsolete, and the data is now stored in the \'state' file instead.
 
 __DataDirectory__**/control_auth_cookie**::
     Used for cookie authentication with the controller. Location can be
@@ -2867,63 +3301,71 @@ __DataDirectory__**/lock**::
     directory. If access to this file is locked, data directory is already
     in use by Tor.
 
-__DataDirectory__**/keys/***::
-    Only used by servers. Holds identity keys and onion keys.
+__DataDirectory__**/key-pinning-journal**::
+    Used by authorities. A line-based file that records mappings between
+    RSA1024 identity keys and Ed25519 identity keys. Authorities enforce
+    these mappings, so that once a relay has picked an Ed25519 key, stealing
+    or factoring the RSA1024 key will no longer let an attacker impersonate
+    the relay.
 
-__DataDirectory__**/keys/authority_identity_key**::
+__KeyDirectory__**/authority_identity_key**::
     A v3 directory authority's master identity key, used to authenticate its
     signing key. Tor doesn't use this while it's running. The tor-gencert
     program uses this. If you're running an authority, you should keep this
     key offline, and not actually put it here.
 
-__DataDirectory__**/keys/authority_certificate**::
+__KeyDirectory__**/authority_certificate**::
     A v3 directory authority's certificate, which authenticates the authority's
     current vote- and consensus-signing key using its master identity key.
     Only directory authorities use this file.
 
-__DataDirectory__**/keys/authority_signing_key**::
+__KeyDirectory__**/authority_signing_key**::
     A v3 directory authority's signing key, used to sign votes and consensuses.
     Only directory authorities use this file.  Corresponds to the
     **authority_certificate** cert.
 
-__DataDirectory__**/keys/legacy_certificate**::
+__KeyDirectory__**/legacy_certificate**::
     As authority_certificate: used only when V3AuthUseLegacyKey is set.
     See documentation for V3AuthUseLegacyKey.
 
-__DataDirectory__**/keys/legacy_signing_key**::
+__KeyDirectory__**/legacy_signing_key**::
     As authority_signing_key: used only when V3AuthUseLegacyKey is set.
     See documentation for V3AuthUseLegacyKey.
 
-__DataDirectory__**/keys/secret_id_key**::
+__KeyDirectory__**/secret_id_key**::
     A relay's RSA1024 permanent identity key, including private and public
     components.  Used to sign router descriptors, and to sign other keys.
 
-__DataDirectory__**/keys/ed25519_master_id_public_key**::
+__KeyDirectory__**/ed25519_master_id_public_key**::
     The public part of a relay's Ed25519 permanent identity key.
 
-__DataDirectory__**/keys/ed25519_master_id_secret_key**::
+__KeyDirectory__**/ed25519_master_id_secret_key**::
     The private part of a relay's Ed25519 permanent identity key.  This key
     is used to sign the medium-term ed25519 signing key.  This file can be
     kept offline, or kept encrypted. If so, Tor will not be able to generate
     new signing keys itself; you'll need to use tor --keygen yourself to do
     so.
 
-__DataDirectory__**/keys/ed25519_signing_secret_key**::
+__KeyDirectory__**/ed25519_signing_secret_key**::
     The private and public components of a relay's medium-term Ed25519 signing
     key. This key is authenticated by the Ed25519 master key, in turn
     authenticates other keys (and router descriptors).
 
-__DataDirectory__**/keys/ed25519_signing_cert**::
+__KeyDirectory__**/ed25519_signing_cert**::
     The certificate which authenticates "ed25519_signing_secret_key" as
     having been signed by the Ed25519 master key.
 
-__DataDirectory__**/keys/secret_onion_key**::
+__KeyDirectory__**/secret_onion_key** and **secret_onion_key.old**::
     A relay's RSA1024 short-term onion key. Used to decrypt old-style ("TAP")
-    circuit extension requests.
+    circuit extension requests. The ".old" file holds the previously
+    generated key, which the relay uses to handle any requests that were
+    made by clients that didn't have the new one.
 
-__DataDirectory__**/keys/secret_onion_key_ntor**::
+__KeyDirectory__**/secret_onion_key_ntor** and **secret_onion_key_ntor.old**::
     A relay's Curve25519 short-term onion key. Used to handle modern ("ntor")
-    circuit extension requests.
+    circuit extension requests. The ".old" file holds the previously
+    generated key, which the relay uses to handle any requests that were
+    made by clients that didn't have the new one.
 
 __DataDirectory__**/fingerprint**::
     Only used by servers. Holds the fingerprint of the server's identity key.
@@ -2932,15 +3374,25 @@ __DataDirectory__**/hashed-fingerprint**::
     Only used by bridges. Holds the hashed fingerprint of the bridge's
     identity key. (That is, the hash of the hash of the identity key.)
 
+__DataDirectory__**/approved-routers**::
+    Only used by authoritative directory servers. This file lists
+    the status of routers by their identity fingerprint.
+    Each line lists a status and a fingerprint separated by
+    whitespace. See your **fingerprint** file in the __DataDirectory__ for an
+    example line. If the status is **!reject** then descriptors from the
+    given identity (fingerprint) are rejected by this server. If it is
+    **!invalid** then descriptors are accepted but marked in the directory as
+    not valid, that is, not recommended.
+
 __DataDirectory__**/v3-status-votes**::
     Only for v3 authoritative directory servers. This file contains
     status votes from all the authoritative directory servers.
 
-__DataDirectory__**/unverified-consensus**::
+__CacheDirectory__**/unverified-consensus**::
     This file contains a network consensus document that has been downloaded,
     but which we didn't have the right certificates to check yet.
 
-__DataDirectory__**/unverified-microdesc-consensus**::
+__CacheDirectory__**/unverified-microdesc-consensus**::
     This file contains a microdescriptor-flavored network consensus document
     that has been downloaded, but which we didn't have the right certificates
     to check yet.
@@ -2978,15 +3430,29 @@ __DataDirectory__**/stats/conn-stats**::
     Only used by servers. This file is used to collect approximate connection
     history (number of active connections over time).
 
+__DataDirectory__**/stats/hidserv-stats**::
+    Only used by servers. This file is used to collect approximate counts
+    of what fraction of the traffic is hidden service rendezvous traffic, and
+    approximately how many hidden services the relay has seen.
+
 __DataDirectory__**/networkstatus-bridges**::
     Only used by authoritative bridge directories. Contains information
     about bridges that have self-reported themselves to the bridge
     authority.
 
+__DataDirectory__**/approved-routers**::
+    Authorities only. This file is used to configure which relays are
+    known to be valid, invalid, and so forth.
+
 __HiddenServiceDirectory__**/hostname**::
     The <base32-encoded-fingerprint>.onion domain name for this hidden service.
     If the hidden service is restricted to authorized clients only, this file
     also contains authorization data for all clients.
+ +
+    Note that clients will ignore any extra subdomains prepended to a hidden
+    service hostname. So if you have "xyz.onion" as your hostname, you
+    can tell clients to connect to "www.xyz.onion" or "irc.xyz.onion"
+    for virtual-hosting purposes.
 
 __HiddenServiceDirectory__**/private_key**::
     The private key for this hidden service.
diff --git a/doc/torify.1.txt b/doc/torify.1.txt
index 8dca901317..7e49081cfc 100644
--- a/doc/torify.1.txt
+++ b/doc/torify.1.txt
@@ -17,25 +17,23 @@ SYNOPSIS
 
 DESCRIPTION
 -----------
-**torify** is a simple wrapper that attempts to find the best underlying Tor
-wrapper available on a system. It calls torsocks with a tor specific
-configuration file. +
+**torify** is a simple wrapper that calls torsocks with a tor-specific
+configuration file.
 
-torsocks is an improved wrapper that explicitly rejects UDP, safely resolves DNS
-lookups and properly socksifies your TCP connections. +
-
-Please note that since both method use LD_PRELOAD, torify cannot be applied to
-suid binaries.
+It is provided for backward compatibility; instead you should use torsocks.
 
 WARNING
 -------
-When used with torsocks, torify should not leak DNS requests or UDP data. +
+When used with torsocks, torify should not leak DNS requests or UDP data.
+
+torify can leak ICMP data.
 
-Both will leak ICMP data.
+torify will not ensure that different requests are processed on
+different circuits.
 
 SEE ALSO
 --------
-**tor**(1), **tor-resolve**(1), **torsocks**(1)
+**tor**(1), **torsocks**(1)
 
 AUTHORS
 -------
diff --git a/doc/torrc_format.txt b/doc/torrc_format.txt
index 7a809901d9..7a4a92a663 100644
--- a/doc/torrc_format.txt
+++ b/doc/torrc_format.txt
@@ -18,9 +18,10 @@ does, not what it should do.
    ; specified in RFC5234.
 
    ; A file is interpreted as every Entry in the file, in order.
-   TorrcFile = *Line
+   TorrcFile = *Line [ UnterminatedLine ]
 
-   Line = BlankLine / Entry
+   Line = BlankLine LF / Entry LF
+   UnterminatedLine = BlankLine / Entry
 
    BlankLine =  *WSP OptComment LF
    BlankLine =/ *WSP LF
@@ -69,6 +70,12 @@ does, not what it should do.
    ; Anything besides NUL and LF
    NonLF = %x01-%x09 / %x0b - %xff
 
+   ; Note that on windows, we open our configuration files in "text" mode,
+   ; which causes CRLF pairs to be interpreted as LF.  So, on windows:
+   ;         LF = [ %x0d ] %x0a
+   ; but everywhere else,
+   LF = %0x0a
+
    OCTDIG = '0' - '7'
 
    KC = Any character except an isspace() character or '#' or NUL
@@ -175,7 +182,7 @@ and\
 friends
 
 # Backslashes in the middle of a line are included as-is.  The key of
-# this one is "Too" and the value is "Many\\Backsl\ashes here" (with
+# this one is "Too" and the value is "Many\\Backsl\ashes \here" (with
 # backslashes in that last string as-is)
 Too \
 Many\\\
@@ -185,7 +192,7 @@ here
 # And here's the really yucky part. If a comment appears in a multi-line
 # entry, the entry is still able to continue on the next line, as in the
 # following, where the key is "This" and the value is
-# "entry   and some        are  silly"
+# "entry        and some        are  silly"
 This entry      \
  # has comments \
  and some       \
diff --git a/m4/ax_check_sign.m4 b/m4/ax_check_sign.m4
index 104b17014c..d67e114dba 100644
--- a/m4/ax_check_sign.m4
+++ b/m4/ax_check_sign.m4
@@ -41,8 +41,8 @@ AU_ALIAS([VL_CHECK_SIGN], [AX_CHECK_SIGN])
 AC_DEFUN([AX_CHECK_SIGN], [
  typename=`echo $1 | sed "s/@<:@^a-zA-Z0-9_@:>@/_/g"`
  AC_CACHE_CHECK([whether $1 is signed], ax_cv_decl_${typename}_signed, [
-   AC_TRY_COMPILE([$4],
-     [ int foo @<:@ 1 - 2 * !((($1) -1) < 0) @:>@ ],
+   AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$4]],
+     [[ int foo @<:@ 1 - 2 * !((($1) -1) < 0) @:>@ ]])],
      [ eval "ax_cv_decl_${typename}_signed=\"yes\"" ],
      [ eval "ax_cv_decl_${typename}_signed=\"no\"" ])])
  symbolname=`echo $1 | sed "s/@<:@^a-zA-Z0-9_@:>@/_/g" | tr "a-z" "A-Z"`
diff --git a/m4/pc_from_ucontext.m4 b/m4/pc_from_ucontext.m4
index 8a9dc459e6..9b66bf752c 100644
--- a/m4/pc_from_ucontext.m4
+++ b/m4/pc_from_ucontext.m4
@@ -79,29 +79,29 @@ AC_DEFUN([AC_PC_FROM_UCONTEXT],
      if ! $pc_field_found; then
        # Prefer sys/ucontext.h to ucontext.h, for OS X's sake.
        if test "x$ac_cv_header_cygwin_signal_h" = xyes; then
-         AC_TRY_COMPILE([#include <cygwin/signal.h>],
-                        [ucontext_t u; return u.$pc_field == 0;],
+         AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <cygwin/signal.h>]],
+                        [[ucontext_t u; return u.$pc_field == 0;]])],
                         AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field,
                                            How to access the PC from a struct ucontext)
                         AC_MSG_RESULT([$pc_field])
                         pc_field_found=true)
        elif test "x$ac_cv_header_sys_ucontext_h" = xyes; then
-         AC_TRY_COMPILE([#include <sys/ucontext.h>],
-                        [ucontext_t u; return u.$pc_field == 0;],
+         AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/ucontext.h>]],
+                        [[ucontext_t u; return u.$pc_field == 0;]])],
                         AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field,
                                            How to access the PC from a struct ucontext)
                         AC_MSG_RESULT([$pc_field])
                         pc_field_found=true)
        elif test "x$ac_cv_header_ucontext_h" = xyes; then
-         AC_TRY_COMPILE([#include <ucontext.h>],
-                        [ucontext_t u; return u.$pc_field == 0;],
+         AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <ucontext.h>]],
+                        [[ucontext_t u; return u.$pc_field == 0;]])],
                         AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field,
                                            How to access the PC from a struct ucontext)
                         AC_MSG_RESULT([$pc_field])
                         pc_field_found=true)
        else     # hope some standard header gives it to us
-         AC_TRY_COMPILE([],
-                        [ucontext_t u; return u.$pc_field == 0;],
+         AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]],
+                        [[ucontext_t u; return u.$pc_field == 0;]])],
                         AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field,
                                            How to access the PC from a struct ucontext)
                         AC_MSG_RESULT([$pc_field])
@@ -114,8 +114,8 @@ AC_DEFUN([AC_PC_FROM_UCONTEXT],
      pc_fields="$pc_fields sc_rip"  # OpenBSD (x86_64)
      for pc_field in $pc_fields; do
        if ! $pc_field_found; then
-         AC_TRY_COMPILE([#include <signal.h>],
-                        [ucontext_t u; return u.$pc_field == 0;],
+         AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <signal.h>]],
+                        [[ucontext_t u; return u.$pc_field == 0;]])],
                         AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field,
                                            How to access the PC from a struct ucontext)
                         AC_MSG_RESULT([$pc_field])
diff --git a/scripts/README b/scripts/README
index 02faabe06b..9cd6e74ac7 100644
--- a/scripts/README
+++ b/scripts/README
@@ -30,6 +30,12 @@ orconfig.h files.
 Testing scripts
 ---------------
 
+test/chutney-git-bisect.sh -- a git bisect run script that bisects using
+chutney. The script builds tor and tor-gencert, then runs chutney. The script
+takes optional arguments for out-of-tree builds, and specific chutney network
+flavours. You should copy this script before using it with git bisect, so that
+it doesn't change (or disappear) during bisection.
+
 test/cov-blame -- Mash up the results of gcov with git blame.  Mainly useful
 to find out who has been writing untested code.
 
diff --git a/scripts/coccinelle/ceil_div.cocci b/scripts/coccinelle/ceil_div.cocci
new file mode 100644
index 0000000000..00843e82c3
--- /dev/null
+++ b/scripts/coccinelle/ceil_div.cocci
@@ -0,0 +1,6 @@
+@@
+expression n, d;
+@@
+
+- (((n) + (d) - 1) / (d))
++ CEIL_DIV(n, d)
diff --git a/scripts/coccinelle/test-operator-cleanup b/scripts/coccinelle/test-operator-cleanup
new file mode 100755
index 0000000000..e7822542a4
--- /dev/null
+++ b/scripts/coccinelle/test-operator-cleanup
@@ -0,0 +1,11 @@
+#!/usr/bin/perl -w -p -i
+
+next if m#^ */\*# or m#^ *\* #;
+
+s/<([,)])/OP_LT$1/;
+s/(?<=[\s,])>([,)])/OP_GT$1/;
+#s/>([,)])/OP_GT$1/;
+s/==([,)])/OP_EQ$1/;
+s/>=([,)])/OP_GE$1/;
+s/<=([,)])/OP_LE$1/;
+s/!=([,)])/OP_NE$1/;
diff --git a/scripts/coccinelle/test_assert_int.cocci b/scripts/coccinelle/test_assert_int.cocci
new file mode 100644
index 0000000000..80e86b4f3c
--- /dev/null
+++ b/scripts/coccinelle/test_assert_int.cocci
@@ -0,0 +1,49 @@
+@@
+int e;
+constant c;
+@@
+
+(
+- tt_assert(e == c)
++ tt_int_op(e, OP_EQ, c)
+|
+- tt_assert(e != c)
++ tt_int_op(e, OP_NE, c)
+|
+- tt_assert(e < c)
++ tt_int_op(e, OP_LT, c)
+|
+- tt_assert(e <= c)
++ tt_int_op(e, OP_LE, c)
+|
+- tt_assert(e > c)
++ tt_int_op(e, OP_GT, c)
+|
+- tt_assert(e >= c)
++ tt_int_op(e, OP_GE, c)
+)
+
+@@
+unsigned int e;
+constant c;
+@@
+
+(
+- tt_assert(e == c)
++ tt_uint_op(e, OP_EQ, c)
+|
+- tt_assert(e != c)
++ tt_uint_op(e, OP_NE, c)
+|
+- tt_assert(e < c)
++ tt_uint_op(e, OP_LT, c)
+|
+- tt_assert(e <= c)
++ tt_uint_op(e, OP_LE, c)
+|
+- tt_assert(e > c)
++ tt_uint_op(e, OP_GT, c)
+|
+- tt_assert(e >= c)
++ tt_uint_op(e, OP_GE, c)
+)
diff --git a/scripts/coccinelle/test_assert_null.cocci b/scripts/coccinelle/test_assert_null.cocci
new file mode 100644
index 0000000000..3d66e1ee0b
--- /dev/null
+++ b/scripts/coccinelle/test_assert_null.cocci
@@ -0,0 +1,11 @@
+@@
+expression * e;
+@@
+
+(
+- tt_assert(e != NULL)
++ tt_ptr_op(e, OP_NE, NULL)
+|
+- tt_assert(e == NULL)
++ tt_ptr_op(e, OP_EQ, NULL)
+)
diff --git a/scripts/coccinelle/test_assert_zero.cocci b/scripts/coccinelle/test_assert_zero.cocci
new file mode 100644
index 0000000000..09feaa5fb4
--- /dev/null
+++ b/scripts/coccinelle/test_assert_zero.cocci
@@ -0,0 +1,5 @@
+@@
+@@
+
+- tt_assert(0)
++ tt_abort()
diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py
new file mode 100755
index 0000000000..3c948d87cf
--- /dev/null
+++ b/scripts/codegen/fuzzing_include_am.py
@@ -0,0 +1,154 @@
+#!/usr/bin/python
+
+FUZZERS = """
+	consensus
+	descriptor
+	diff
+	diff-apply
+	extrainfo
+	hsdescv2
+	hsdescv3
+	http
+	http-connect
+	iptsv2
+	microdesc
+	socks
+	vrs
+"""
+
+
+PREAMBLE = r"""
+FUZZING_CPPFLAGS = \
+	$(src_test_AM_CPPFLAGS) $(TEST_CPPFLAGS)
+FUZZING_CFLAGS = \
+	$(AM_CFLAGS) $(TEST_CFLAGS)
+FUZZING_LDFLAG = \
+	@TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@
+FUZZING_LIBS = \
+	$(TOR_INTERNAL_TESTING_LIBS) \
+	$(rust_ldadd) \
+	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
+	@TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \
+	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \
+	@TOR_SYSTEMD_LIBS@ \
+	@TOR_LZMA_LIBS@ \
+	@TOR_ZSTD_LIBS@
+
+oss-fuzz-prereqs: \
+    $(TOR_INTERNAL_TESTING_LIBS)
+
+noinst_HEADERS += \
+	src/test/fuzz/fuzzing.h
+
+LIBFUZZER = -lFuzzer
+LIBFUZZER_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ
+LIBFUZZER_CFLAGS = $(FUZZING_CFLAGS)
+LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG)
+LIBFUZZER_LIBS = $(FUZZING_LIBS) $(LIBFUZZER) -lstdc++
+
+LIBOSS_FUZZ_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ
+LIBOSS_FUZZ_CFLAGS = $(FUZZING_CFLAGS)
+"""
+
+POSTAMBLE = r"""
+noinst_PROGRAMS += $(FUZZERS) $(LIBFUZZER_FUZZERS)
+noinst_LIBRARIES += $(OSS_FUZZ_FUZZERS)
+oss-fuzz-fuzzers:  oss-fuzz-prereqs $(OSS_FUZZ_FUZZERS)
+fuzzers: $(FUZZERS) $(LIBFUZZER_FUZZERS)
+
+test-fuzz-corpora: $(FUZZERS)
+	$(top_srcdir)/src/test/fuzz_static_testcases.sh
+"""
+
+########### No user serviceable parts will follow.
+
+PREAMBLE = PREAMBLE.strip()
+POSTAMBLE = POSTAMBLE.strip()  # If I use it, it's a word!
+FUZZERS = FUZZERS.split()
+FUZZERS.sort()
+
+WARNING = """
+# This file was generated by fuzzing_include_am.py; do not hand-edit unless
+# you enjoy having your changes erased.
+""".strip()
+
+print(WARNING)
+
+print(PREAMBLE)
+
+print("\n# ===== AFL fuzzers")
+
+def get_id_name(s):
+    return s.replace("-", "_")
+
+for fuzzer in FUZZERS:
+    idname = get_id_name(fuzzer)
+    print("""\
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_{name}_SOURCES = \\
+	src/test/fuzz/fuzzing_common.c \\
+	src/test/fuzz/fuzz_{name}.c
+src_test_fuzz_fuzz_{name}_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_{name}_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_{name}_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_{name}_LDADD = $(FUZZING_LIBS)
+endif
+""".format(name=idname))
+
+print("if UNITTESTS_ENABLED")
+print("FUZZERS = \\")
+print(" \\\n".join("\tsrc/test/fuzz/fuzz-{name}".format(name=fuzzer)
+                   for fuzzer in FUZZERS))
+print("endif")
+
+print("\n# ===== libfuzzer")
+print("\nif LIBFUZZER_ENABLED")
+
+for fuzzer in FUZZERS:
+    idname = get_id_name(fuzzer)
+    print("""\
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_{name}_SOURCES = \\
+	$(src_test_fuzz_fuzz_{name}_SOURCES)
+src_test_fuzz_lf_fuzz_{name}_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_{name}_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_{name}_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_{name}_LDADD = $(LIBFUZZER_LIBS)
+endif
+""".format(name=idname))
+
+print("LIBFUZZER_FUZZERS = \\")
+print(" \\\n".join("\tsrc/test/fuzz/lf-fuzz-{name}".format(name=fuzzer)
+                   for fuzzer in FUZZERS))
+
+print("""
+else
+LIBFUZZER_FUZZERS =
+endif""")
+
+print("\n# ===== oss-fuzz\n")
+print("if OSS_FUZZ_ENABLED")
+
+for fuzzer in FUZZERS:
+    idname = get_id_name(fuzzer)
+    print("""\
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_{name}_a_SOURCES = \\
+	$(src_test_fuzz_fuzz_{name}_SOURCES)
+src_test_fuzz_liboss_fuzz_{name}_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_{name}_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+""".format(name=idname))
+
+print("OSS_FUZZ_FUZZERS = \\")
+print(" \\\n".join("\tsrc/test/fuzz/liboss-fuzz-{name}.a".format(name=fuzzer)
+                   for fuzzer in FUZZERS))
+
+print("""
+else
+OSS_FUZZ_FUZZERS =
+endif""")
+
+print("")
+
+print(POSTAMBLE)
diff --git a/scripts/codegen/gen_server_ciphers.py b/scripts/codegen/gen_server_ciphers.py
index 0dca8a6734..5d326f8b9e 100755
--- a/scripts/codegen/gen_server_ciphers.py
+++ b/scripts/codegen/gen_server_ciphers.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright 2014-2015, The Tor Project, Inc
+# Copyright 2014-2019, The Tor Project, Inc
 # See LICENSE for licensing information
 
 # This script parses openssl headers to find ciphersuite names, determines
@@ -13,13 +13,13 @@ import sys
 
 EPHEMERAL_INDICATORS = [ "_EDH_", "_DHE_", "_ECDHE_" ]
 BAD_STUFF = [ "_DES_40_", "MD5", "_RC4_", "_DES_64_",
-              "_SEED_", "_CAMELLIA_", "_NULL" ]
+              "_SEED_", "_CAMELLIA_", "_NULL",
+              "_CCM_8", "_DES_", ]
 
 # these never get #ifdeffed.
 MANDATORY = [
     "TLS1_TXT_DHE_RSA_WITH_AES_256_SHA",
     "TLS1_TXT_DHE_RSA_WITH_AES_128_SHA",
-    "SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA",
 ]
 
 def find_ciphers(filename):
@@ -48,15 +48,23 @@ def usable_cipher(ciph):
 # All fields we sort on, in order of priority.
 FIELDS = [ 'cipher', 'fwsec', 'mode',  'digest', 'bitlength' ]
 # Map from sorted fields to recognized value in descending order of goodness
-FIELD_VALS = { 'cipher' : [ 'AES', 'DES'],
+FIELD_VALS = { 'cipher' : [ 'AES', 'CHACHA20' ],
                'fwsec' : [ 'ECDHE', 'DHE' ],
-               'mode' : [ 'GCM', 'CBC' ],
-               'digest' : [ 'SHA384', 'SHA256', 'SHA' ],
+               'mode' : [ 'POLY1305', 'GCM', 'CCM', 'CBC', ],
+               'digest' : [ 'n/a', 'SHA384', 'SHA256', 'SHA', ],
                'bitlength' : [ '256', '128', '192' ],
 }
 
 class Ciphersuite(object):
     def __init__(self, name, fwsec, cipher, bitlength, mode, digest):
+        if fwsec == 'EDH':
+            fwsec = 'DHE'
+
+        if mode in [ '_CBC3', '_CBC', '' ]:
+            mode = 'CBC'
+        elif mode == '_GCM':
+            mode = 'GCM'
+
         self.name = name
         self.fwsec = fwsec
         self.cipher = cipher
@@ -74,42 +82,50 @@ class Ciphersuite(object):
 def parse_cipher(ciph):
     m = re.match('(?:TLS1|SSL3)_TXT_(EDH|DHE|ECDHE)_RSA(?:_WITH)?_(AES|DES)_(256|128|192)(|_CBC|_CBC3|_GCM)_(SHA|SHA256|SHA384)$', ciph)
 
-    if not m:
-        print "/* Couldn't parse %s ! */"%ciph
-        return None
+    if m:
+        fwsec, cipher, bits, mode, digest = m.groups()
+        return Ciphersuite(ciph, fwsec, cipher, bits, mode, digest)
 
-    fwsec, cipher, bits, mode, digest = m.groups()
-    if fwsec == 'EDH':
-        fwsec = 'DHE'
+    m = re.match('(?:TLS1|SSL3)_TXT_(EDH|DHE|ECDHE)_RSA(?:_WITH)?_(AES|DES)_(256|128|192)_CCM', ciph)
+    if m:
+        fwsec, cipher, bits = m.groups()
+        return Ciphersuite(ciph, fwsec, cipher, bits, "CCM", "n/a")
 
-    if mode in [ '_CBC3', '_CBC', '' ]:
-        mode = 'CBC'
-    elif mode == '_GCM':
-        mode = 'GCM'
+    m = re.match('(?:TLS1|SSL3)_TXT_(EDH|DHE|ECDHE)_RSA(?:_WITH)?_CHACHA20_POLY1305', ciph)
+    if m:
+        fwsec, = m.groups()
+        return Ciphersuite(ciph, fwsec, "CHACHA20", "256", "POLY1305", "n/a")
+
+    print "/* Couldn't parse %s ! */"%ciph
+    return None
 
-    return Ciphersuite(ciph, fwsec, cipher, bits, mode, digest)
 
 ALL_CIPHERS = []
 
 for fname in sys.argv[1:]:
-    ALL_CIPHERS += (parse_cipher(c)
-                           for c in find_ciphers(fname)
-                           if usable_cipher(c) )
+    for c in find_ciphers(fname):
+        if usable_cipher(c):
+            parsed = parse_cipher(c)
+            if parsed != None:
+                ALL_CIPHERS.append(parsed)
 
 ALL_CIPHERS.sort(key=Ciphersuite.sort_key)
 
+indent = " "*7
+
 for c in ALL_CIPHERS:
     if c is ALL_CIPHERS[-1]:
-        colon = ';'
+        colon = ''
     else:
         colon = ' ":"'
 
     if c.name in MANDATORY:
-        print "       /* Required */"
-        print '       %s%s'%(c.name,colon)
+        print "%s/* Required */"%indent
+        print '%s%s%s'%(indent,c.name,colon)
     else:
         print "#ifdef %s"%c.name
-        print '       %s%s'%(c.name,colon)
+        print '%s%s%s'%(indent,c.name,colon)
         print "#endif"
 
+print '%s;'%indent
 
diff --git a/scripts/codegen/get_mozilla_ciphers.py b/scripts/codegen/get_mozilla_ciphers.py
index e673ec7dc6..f23f2f1e6f 100644..100755
--- a/scripts/codegen/get_mozilla_ciphers.py
+++ b/scripts/codegen/get_mozilla_ciphers.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python
 # coding=utf-8
-# Copyright 2011-2015, The Tor Project, Inc
+# Copyright 2011-2019, The Tor Project, Inc
 # original version by Arturo Filastò
 # See LICENSE for licensing information
 
@@ -127,9 +127,9 @@ for k, v in enabled_ciphers.items():
 #oSSLinclude = ('/usr/include/openssl/ssl3.h', '/usr/include/openssl/ssl.h',
 #               '/usr/include/openssl/ssl2.h', '/usr/include/openssl/ssl23.h',
 #               '/usr/include/openssl/tls1.h')
-oSSLinclude = ('ssl/ssl3.h', 'ssl/ssl.h',
-               'ssl/ssl2.h', 'ssl/ssl23.h',
-               'ssl/tls1.h')
+oSSLinclude = ['ssl3.h', 'ssl.h'
+               'ssl2.h', 'ssl23.h',
+               'tls1.h']
 
 #####
 # This reads the hex code for the ciphers that are used by firefox.
@@ -155,9 +155,12 @@ for x in used_ciphers:
 openssl_macro_by_hex = {}
 all_openssl_macros = {}
 for fl in oSSLinclude:
-    fp = open(ossl(fl), 'r')
+    fname = ossl("include/openssl/"+fl)
+    if not os.path.exists(fname):
+        continue
+    fp = open(fname, 'r')
     for line in fp.readlines():
-        m = re.match('#define\s+(\S+)\s+(\S+)', line)
+        m = re.match('# *define\s+(\S+)\s+(\S+)', line)
         if m:
             value,key = m.groups()
             if key.startswith('0x') and "_CK_" in value:
diff --git a/scripts/codegen/makedesc.py b/scripts/codegen/makedesc.py
index d4ba21efae..efca4dda9a 100644
--- a/scripts/codegen/makedesc.py
+++ b/scripts/codegen/makedesc.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright 2014-2015, The Tor Project, Inc.
+# Copyright 2014-2019, The Tor Project, Inc.
 # See LICENSE for license information
 
 # This is a kludgey python script that uses ctypes and openssl to sign
diff --git a/scripts/codegen/run_trunnel.sh b/scripts/codegen/run_trunnel.sh
index d2669931e9..428804342a 100755
--- a/scripts/codegen/run_trunnel.sh
+++ b/scripts/codegen/run_trunnel.sh
@@ -5,7 +5,13 @@ if test "x$TRUNNEL_PATH" != "x"; then
   export PYTHONPATH
 fi
 
-python -m trunnel --require-version=1.4 ./src/trunnel/*.trunnel
+OPTIONS="--require-version=1.5.1"
 
-python -m trunnel --require-version=1.4 --write-c-files --target-dir=./src/ext/trunnel/
+# Get all .trunnel files recursively from that directory so we can support
+# multiple sub-directories.
+for file in `find ./src/trunnel/ -name '*.trunnel'`; do
+  python -m trunnel ${OPTIONS} $file
+done
+
+python -m trunnel ${OPTIONS} --write-c-files --target-dir=./src/ext/trunnel/
 
diff --git a/scripts/maint/analyze_callgraph.py b/scripts/maint/analyze_callgraph.py
deleted file mode 100755
index 8ce5827f07..0000000000
--- a/scripts/maint/analyze_callgraph.py
+++ /dev/null
@@ -1,259 +0,0 @@
-#!/usr/bin/python
-
-import re
-import sys
-import copy
-import cPickle
-import os
-
-class Parser:
-  def __init__(self):
-    self.calls = {}
-    self.definedIn = {}
-
-  def enter_func(self, name):
-    if self.infunc and not self.extern and self.calledfns:
-      if self.infunc in self.definedIn:
-        #print "{}: {} or {}?".format(
-        #  self.infunc, self.definedIn[self.infunc], self.module)
-        self.definedIn[self.infunc] = 'nil'
-      else:
-        self.definedIn[self.infunc] = self.module
-      self.calls.setdefault(self.infunc, set()).update( self.calledfns )
-
-    self.calledfns = set()
-    self.infunc = name
-    self.extern = False
-
-  def parse_callgraph_file(self, inp, module):
-    self.infunc = None
-    self.extern = False
-    self.calledfns = set()
-    self.module = module
-
-    for line in inp:
-       m = re.match(r"Call graph node for function: '([^']+)'", line)
-       if m:
-           self.enter_func(m.group(1))
-           continue
-       m = re.match(r"  CS<[^>]+> calls external node", line)
-       if m:
-           self.extern = True
-       m = re.match(r"  CS<[^>]+> calls function '([^']+)'", line)
-       if m:
-           self.calledfns.add(m.group(1))
-    self.enter_func(None)
-
-  def extract_callgraph(self):
-    c = self.calls
-    self.calls = {}
-    return c
-
-
-def transitive_closure(g):
-    passno = 0
-    changed = True
-    g = copy.deepcopy(g)
-    import random
-    while changed:
-      passno += 1
-      changed = False
-      keys = g.keys()
-      idx = 0
-      for k in keys:
-         idx += 1
-         print "Pass %d/?: %d/%d\r" %(passno, idx, len(keys)),
-         sys.stdout.flush()
-         newset = g[k].copy()
-         for fn in g[k]:
-            newset.update(g.get(fn, set()))
-         if len(newset) != len(g[k]):
-            g[k].update( newset )
-            changed = True
-
-      print
-
-    return g
-
-def strongly_connected_components(g):
-  # From https://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm, done stupidly.
-  index_of = {}
-  index = [ 0 ]
-  lowlink = {}
-  S = []
-  onStack = set()
-
-  all_sccs = []
-
-  def strongconnect(fn):
-    index_of[fn] = index[0]
-    lowlink[fn] = index[0]
-    index[0] += 1
-    S.append(fn)
-    onStack.add(fn)
-
-    for w in g.get(fn, []):
-      if w not in index_of:
-        strongconnect(w)
-        lowlink[fn] = min(lowlink[fn], lowlink[w])
-      elif w in onStack:
-        lowlink[fn] = min(lowlink[fn], index_of[w])
-
-    if lowlink[fn] == index_of[fn]:
-      this_scc = []
-      all_sccs.append(this_scc)
-      while True:
-        w = S.pop()
-        onStack.remove(w)
-        this_scc.append(w)
-        if w == fn:
-          break
-
-  for v in g.keys():
-    if v not in index_of:
-      strongconnect(v)
-
-  return all_sccs
-
-def biggest_component(sccs):
-  return max(len(c) for c in sccs)
-
-def connection_bottlenecks(callgraph):
-
-  callers = {}
-  for fn in callgraph:
-    for fn2 in callgraph[fn]:
-      callers.setdefault(fn2, set()).add(fn)
-
-  components = strongly_connected_components(callgraph)
-  components.sort(key=len)
-  big_component_fns = components[-1]
-  size = len(big_component_fns)
-
-  function_bottlenecks = fn_results = []
-
-  total = len(big_component_fns)
-  idx = 0
-  for fn in big_component_fns:
-    idx += 1
-    print "Pass 1/3: %d/%d\r"%(idx, total),
-    sys.stdout.flush()
-    cg2 = copy.deepcopy(callgraph)
-    del cg2[fn]
-
-    fn_results.append( (size - biggest_component(strongly_connected_components(cg2)), fn) )
-
-  print
-  bcf_set = set(big_component_fns)
-
-  call_bottlenecks = fn_results = []
-  result_set = set()
-  total = len(big_component_fns)
-  idx = 0
-  for fn in big_component_fns:
-    fn_callers = callers[fn].intersection(bcf_set)
-    idx += 1
-    if len(fn_callers) != 1:
-      continue
-
-    print "Pass 2/3: %d/%d\r"%(idx, total),
-    sys.stdout.flush()
-
-    caller = fn_callers.pop()
-    assert len(fn_callers) == 0
-    cg2 = copy.deepcopy(callgraph)
-    cg2[caller].remove(fn)
-
-    fn_results.append( (size - biggest_component(strongly_connected_components(cg2)), fn, "called by", caller) )
-    result_set.add( (caller, fn) )
-
-  print
-
-  total = len(big_component_fns)
-  idx = 0
-  for fn in big_component_fns:
-    fn_calls = callgraph[fn].intersection(bcf_set)
-    idx += 1
-    if len(fn_calls) != 1:
-      continue
-
-    print "Pass 3/3: %d/%d\r"%(idx, total),
-    sys.stdout.flush()
-
-    callee = fn_calls.pop()
-    if (fn, callee) in result_set:
-      continue
-
-    assert len(fn_calls) == 0
-    cg2 = copy.deepcopy(callgraph)
-    cg2[fn].remove(callee)
-
-    fn_results.append( (size - biggest_component(strongly_connected_components(cg2)), callee, "called by", fn) )
-
-  print
-
-
-  return (function_bottlenecks, call_bottlenecks)
-
-if __name__ == '__main__':
-    p = Parser()
-    for fname in sys.argv[1:]:
-      modname = re.sub(r'.*/', '', fname).replace('.callgraph', '.c')
-      with open(fname, 'r') as f:
-        p.parse_callgraph_file(f, modname)
-
-    sys.stdout.flush()
-
-    print "Building callgraph"
-    callgraph = p.extract_callgraph()
-    inModule = p.definedIn
-
-    print "Deriving module callgraph"
-    modCallgraph = {}
-    for fn in callgraph:
-      fnMod = inModule[fn]
-      for called in callgraph[fn]:
-        try:
-          calledMod = inModule[called]
-        except KeyError:
-            continue
-        modCallgraph.setdefault(fnMod, set()).add(calledMod)
-    del modCallgraph['nil']
-
-    print "Finding strongly connected components"
-    sccs = strongly_connected_components(callgraph)
-
-    print "Finding the transitive closure of the callgraph.."
-    closure = transitive_closure(callgraph)
-
-    print "Finding bottlenecks..."
-    bottlenecks = connection_bottlenecks(callgraph)
-
-    print "Finding module SCCs"
-    modSCCS = strongly_connected_components(modCallgraph)
-
-    print "Finding module TC"
-    modTC = transitive_closure(modCallgraph)
-
-    print "Finding module bottlenecks"
-    modB = connection_bottlenecks(modCallgraph)
-
-    data = {
-      'callgraph' : callgraph,
-      'sccs' : sccs,
-      'closure' : closure,
-      'bottlenecks' : bottlenecks,
-      'modules' : p.definedIn,
-      'modItems' : {
-        'callgraph' : modCallgraph,
-        'sccs' : modSCCS,
-        'closure' : modTC,
-        'bottlenecks' : modB,
-      }
-    }
-
-    with open('callgraph.pkl', 'w') as f:
-      cPickle.dump(data, f)
-
-
-
diff --git a/scripts/maint/annotate_ifdef_directives b/scripts/maint/annotate_ifdef_directives
new file mode 100755
index 0000000000..ca267a865e
--- /dev/null
+++ b/scripts/maint/annotate_ifdef_directives
@@ -0,0 +1,74 @@
+#!/usr/bin/python
+# Copyright (c) 2017-2019, The Tor Project, Inc.
+# See LICENSE for licensing information
+
+import re
+
+LINE_OBVIOUSNESS_LIMIT = 4
+
+class Problem(Exception):
+    pass
+
+def uncomment(s):
+    s = re.sub(r'//.*','',s)
+    s = re.sub(r'/\*.*','',s)
+    return s.strip()
+
+def translate(f_in, f_out):
+    whole_file = []
+    stack = []
+    cur_level = whole_file
+    lineno = 0
+    for line in f_in:
+        lineno += 1
+        m = re.match(r'\s*#\s*(if|ifdef|ifndef|else|endif|elif)\b\s*(.*)',
+                     line)
+        if not m:
+            f_out.write(line)
+            continue
+        command,rest = m.groups()
+        if command in ("if", "ifdef", "ifndef"):
+            # The #if directive pushes us one level lower on the stack.
+            if command == 'ifdef':
+                rest = "defined(%s)"%uncomment(rest)
+            elif command == 'ifndef':
+                rest = "!defined(%s)"%uncomment(rest)
+            elif rest.endswith("\\"):
+                rest = rest[:-1]+"..."
+
+            rest = uncomment(rest)
+
+            new_level = [ (command, rest, lineno) ]
+            stack.append(cur_level)
+            cur_level = new_level
+            f_out.write(line)
+        elif command in ("else", "elif"):
+            if len(cur_level) == 0 or cur_level[-1][0] == 'else':
+                raise Problem("Unexpected #%s on %d"% (command,lineno))
+            if (len(cur_level) == 1 and command == 'else' and
+                lineno > cur_level[0][2] + LINE_OBVIOUSNESS_LIMIT):
+                f_out.write("#else /* !(%s) */\n"%cur_level[0][1])
+            else:
+                f_out.write(line)
+            cur_level.append((command, rest, lineno))
+        else:
+            assert command == 'endif'
+            if len(stack) == 0:
+                raise Problem("Unmatched #%s on %s"% (command,lineno))
+            if lineno <= cur_level[0][2] + LINE_OBVIOUSNESS_LIMIT:
+                f_out.write(line)
+            elif len(cur_level) == 1 or (
+                    len(cur_level) == 2 and cur_level[1][0] == 'else'):
+                f_out.write("#endif /* %s */\n"%cur_level[0][1])
+            else:
+                f_out.write("#endif /* %s || ... */\n"%cur_level[0][1])
+            cur_level = stack.pop()
+    if len(stack) or cur_level != whole_file:
+        raise Problem("Missing #endif")
+
+import sys,os
+for fn in sys.argv[1:]:
+    with open(fn+"_OUT", 'w') as output_file:
+        translate(open(fn, 'r'), output_file)
+    os.rename(fn+"_OUT", fn)
+
diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py
new file mode 100755
index 0000000000..46a3f39638
--- /dev/null
+++ b/scripts/maint/checkIncludes.py
@@ -0,0 +1,115 @@
+#!/usr/bin/python3
+# Copyright 2018 The Tor Project, Inc.  See LICENSE file for licensing info.
+
+"""This script looks through all the directories for files matching *.c or
+   *.h, and checks their #include directives to make sure that only "permitted"
+   headers are included.
+
+   Any #include directives with angle brackets (like #include <stdio.h>) are
+   ignored -- only directives with quotes (like #include "foo.h") are
+   considered.
+
+   To decide what includes are permitted, this script looks at a .may_include
+   file in each directory.  This file contains empty lines, #-prefixed
+   comments, filenames (like "lib/foo/bar.h") and file globs (like lib/*/*.h)
+   for files that are permitted.
+"""
+
+
+from __future__ import print_function
+
+import fnmatch
+import os
+import re
+import sys
+
+# Global: Have there been any errors?
+trouble = False
+
+if sys.version_info[0] <= 2:
+    def open_file(fname):
+        return open(fname, 'r')
+else:
+    def open_file(fname):
+        return open(fname, 'r', encoding='utf-8')
+
+def err(msg):
+    """ Declare that an error has happened, and remember that there has
+        been an error. """
+    global trouble
+    trouble = True
+    print(msg, file=sys.stderr)
+
+def fname_is_c(fname):
+    """ Return true iff 'fname' is the name of a file that we should
+        search for possibly disallowed #include directives. """
+    return fname.endswith(".h") or fname.endswith(".c")
+
+INCLUDE_PATTERN = re.compile(r'\s*#\s*include\s+"([^"]*)"')
+RULES_FNAME = ".may_include"
+
+class Rules(object):
+    """ A 'Rules' object is the parsed version of a .may_include file. """
+    def __init__(self, dirpath):
+        self.dirpath = dirpath
+        self.patterns = []
+        self.usedPatterns = set()
+
+    def addPattern(self, pattern):
+        self.patterns.append(pattern)
+
+    def includeOk(self, path):
+        for pattern in self.patterns:
+            if fnmatch.fnmatchcase(path, pattern):
+                self.usedPatterns.add(pattern)
+                return True
+        return False
+
+    def applyToLines(self, lines, context=""):
+        lineno = 0
+        for line in lines:
+            lineno += 1
+            m = INCLUDE_PATTERN.match(line)
+            if m:
+                include = m.group(1)
+                if not self.includeOk(include):
+                    err("Forbidden include of {} on line {}{}".format(
+                        include, lineno, context))
+
+    def applyToFile(self, fname):
+        with open_file(fname) as f:
+            #print(fname)
+            self.applyToLines(iter(f), " of {}".format(fname))
+
+    def noteUnusedRules(self):
+        for p in self.patterns:
+            if p not in self.usedPatterns:
+                print("Pattern {} in {} was never used.".format(p, self.dirpath))
+
+def load_include_rules(fname):
+    """ Read a rules file from 'fname', and return it as a Rules object. """
+    result = Rules(os.path.split(fname)[0])
+    with open_file(fname) as f:
+        for line in f:
+            line = line.strip()
+            if line.startswith("#") or not line:
+                continue
+            result.addPattern(line)
+    return result
+
+list_unused = False
+
+for dirpath, dirnames, fnames in os.walk("src"):
+    if ".may_include" in fnames:
+        rules = load_include_rules(os.path.join(dirpath, RULES_FNAME))
+        for fname in fnames:
+            if fname_is_c(fname):
+                rules.applyToFile(os.path.join(dirpath,fname))
+        if list_unused:
+            rules.noteUnusedRules()
+
+if trouble:
+    err(
+"""To change which includes are allowed in a C file, edit the {}
+files in its enclosing directory.""".format(RULES_FNAME))
+    sys.exit(1)
diff --git a/scripts/maint/checkOptionDocs.pl.in b/scripts/maint/checkOptionDocs.pl.in
index 1f53adf099..6533c762c5 100644
--- a/scripts/maint/checkOptionDocs.pl.in
+++ b/scripts/maint/checkOptionDocs.pl.in
@@ -7,7 +7,7 @@ my %torrcSampleOptions = ();
 my %manPageOptions = ();
 
 # Load the canonical list as actually accepted by Tor.
-open(F, "@abs_top_builddir@/src/or/tor --list-torrc-options |") or die;
+open(F, "@abs_top_builddir@/src/app/tor --list-torrc-options |") or die;
 while (<F>) {
     next if m!\[notice\] Tor v0\.!;
     if (m!^([A-Za-z0-9_]+)!) {
diff --git a/scripts/maint/checkSpace.pl b/scripts/maint/checkSpace.pl
index 5af95a27e5..633b47e314 100755
--- a/scripts/maint/checkSpace.pl
+++ b/scripts/maint/checkSpace.pl
@@ -1,48 +1,68 @@
-#!/usr/bin/perl -w
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+my $found = 0;
+sub msg {
+  $found = 1;
+  print "$_[0]";
+}
+
+my $C = 0;
 
 if ($ARGV[0] =~ /^-/) {
-    $lang = shift @ARGV;
+    my $lang = shift @ARGV;
     $C = ($lang eq '-C');
-#    $TXT = ($lang eq '-txt');
 }
 
-for $fn (@ARGV) {
+our %basenames = ();
+
+for my $fn (@ARGV) {
     open(F, "$fn");
-    $lastnil = 0;
-    $lastline = "";
-    $incomment = 0;
+    my $lastnil = 0;
+    my $lastline = "";
+    my $incomment = 0;
+    my $in_func_head = 0;
+    my $basename = $fn;
+    $basename =~ s#.*/##;
+    if ($basenames{$basename}) {
+        msg "Duplicate fnames: $fn and $basenames{$basename}.\n";
+    } else {
+        $basenames{$basename} = $fn;
+    }
     while (<F>) {
         ## Warn about windows-style newlines.
-	#    (We insist on lines that end with a single LF character, not
-	#    CR LF.)
+        #    (We insist on lines that end with a single LF character, not
+        #    CR LF.)
         if (/\r/) {
-            print "       CR:$fn:$.\n";
+            msg "       CR:$fn:$.\n";
         }
         ## Warn about tabs.
-	#    (We only use spaces)
+        #    (We only use spaces)
         if (/\t/) {
-            print "      TAB:$fn:$.\n";
+            msg "      TAB:$fn:$.\n";
         }
         ## Warn about labels that don't have a space in front of them
-	#    (We indent every label at least one space)
+        #    (We indent every label at least one space)
         if (/^[a-zA-Z_][a-zA-Z_0-9]*:/) {
-            print "nosplabel:$fn:$.\n";
+            msg "nosplabel:$fn:$.\n";
         }
         ## Warn about trailing whitespace.
-	#    (We don't allow whitespace at the end of the line; make your
-	#    editor highlight it for you so you can stop adding it in.)
+        #    (We don't allow whitespace at the end of the line; make your
+        #    editor highlight it for you so you can stop adding it in.)
         if (/ +$/) {
-            print "Space\@EOL:$fn:$.\n";
+            msg "Space\@EOL:$fn:$.\n";
         }
         ## Warn about control keywords without following space.
-	#    (We put a space after every 'if', 'while', 'for', 'switch', etc)
+        #    (We put a space after every 'if', 'while', 'for', 'switch', etc)
         if ($C && /\s(?:if|while|for|switch)\(/) {
-            print "      KW(:$fn:$.\n";
+            msg "      KW(:$fn:$.\n";
         }
         ## Warn about #else #if instead of #elif.
         #    (We only allow #elif)
         if (($lastline =~ /^\# *else/) and ($_ =~ /^\# *if/)) {
-            print " #else#if:$fn:$.\n";
+            msg " #else#if:$fn:$.\n";
         }
         ## Warn about some K&R violations
         #    (We use K&R-style C, where open braces go on the same line as
@@ -53,23 +73,23 @@ for $fn (@ARGV) {
         #            other stuff;
         #          }
         if (/^\s+\{/ and $lastline =~ /^\s*(if|while|for|else if)/ and
-	    $lastline !~ /\{$/) {
-            print "non-K&R {:$fn:$.\n";
-	}
+            $lastline !~ /\{$/) {
+            msg "non-K&R {:$fn:$.\n";
+        }
         if (/^\s*else/ and $lastline =~ /\}$/) {
-	    print "  }\\nelse:$fn:$.\n";
-	}
+            msg "  }\\nelse:$fn:$.\n";
+        }
         $lastline = $_;
         ## Warn about unnecessary empty lines.
         #   (Don't put an empty line before a line that contains nothing
         #   but a closing brace.)
         if ($lastnil && /^\s*}\n/) {
-            print "  UnnecNL:$fn:$.\n";
+            msg "  UnnecNL:$fn:$.\n";
         }
         ## Warn about multiple empty lines.
         #   (At most one blank line in a row.)
         if ($lastnil && /^$/) {
-            print " DoubleNL:$fn:$.\n";
+            msg " DoubleNL:$fn:$.\n";
         } elsif (/^$/) {
             $lastnil = 1;
         } else {
@@ -79,7 +99,7 @@ for $fn (@ARGV) {
         ## accept double-line lines.
         #   (Don't make lines wider than 80 characters, including newline.)
         if (/^.{80}/) {
-            print "     Wide:$fn:$.\n";
+            msg "     Wide:$fn:$.\n";
         }
         ### Juju to skip over comments and strings, since the tests
         ### we're about to do are okay there.
@@ -102,48 +122,52 @@ for $fn (@ARGV) {
             s!"(?:[^\"]+|\\.)*"!"X"!g;
             next if /^\#/;
             ## Warn about C++-style comments.
-	    #   (Use C style comments only.)
+            #   (Use C style comments only.)
             if (m!//!) {
-                #    print "       //:$fn:$.\n";
+                #    msg "       //:$fn:$.\n";
                 s!//.*!!;
             }
             ## Warn about unquoted braces preceded by non-space.
-	    #   (No character except a space should come before a {)
+            #   (No character except a space should come before a {)
             if (/([^\s'])\{/) {
-                print "       $1\{:$fn:$.\n";
+                msg "       $1\{:$fn:$.\n";
+            }
+            ## Warn about double semi-colons at the end of a line.
+            if (/;;$/) {
+                msg "       double semi-colons at the end of $. in $fn\n"
             }
             ## Warn about multiple internal spaces.
             #if (/[^\s,:]\s{2,}[^\s\\=]/) {
-            #    print "     X  X:$fn:$.\n";
+            #    msg "     X  X:$fn:$.\n";
             #}
             ## Warn about { with stuff after.
             #s/\s+$//;
             #if (/\{[^\}\\]+$/) {
-            #    print "     {X:$fn:$.\n";
+            #    msg "     {X:$fn:$.\n";
             #}
             ## Warn about function calls with space before parens.
-	    #   (Don't put a space between the name of a function and its
-	    #   arguments.)
+            #   (Don't put a space between the name of a function and its
+            #   arguments.)
             if (/(\w+)\s\(([A-Z]*)/) {
                 if ($1 ne "if" and $1 ne "while" and $1 ne "for" and
                     $1 ne "switch" and $1 ne "return" and $1 ne "int" and
                     $1 ne "elsif" and $1 ne "WINAPI" and $2 ne "WINAPI" and
                     $1 ne "void" and $1 ne "__attribute__" and $1 ne "op" and
-                    $1 ne "size_t" and $1 ne "double" and
+                    $1 ne "size_t" and $1 ne "double" and $1 ne "uint64_t" and
                     $1 ne "workqueue_reply_t") {
-                    print "     fn ():$fn:$.\n";
+                    msg "     fn ():$fn:$.\n";
                 }
             }
             ## Warn about functions not declared at start of line.
-	    #    (When you're declaring functions, put "static" and "const"
-	    #    and the return type on one line, and the function name at
-	    #    the start of a new line.)
+            #    (When you're declaring functions, put "static" and "const"
+            #    and the return type on one line, and the function name at
+            #    the start of a new line.)
             if ($in_func_head ||
                 ($fn !~ /\.h$/ && /^[a-zA-Z0-9_]/ &&
                  ! /^(?:const |static )*(?:typedef|struct|union)[^\(]*$/ &&
                  ! /= *\{$/ && ! /;$/)) {
                 if (/.\{$/){
-                    print "fn() {:$fn:$.\n";
+                    msg "fn() {:$fn:$.\n";
                     $in_func_head = 0;
                 } elsif (/^\S[^\(]* +\**[a-zA-Z0-9_]+\(/) {
                     $in_func_head = -1; # started with tp fn
@@ -151,31 +175,33 @@ for $fn (@ARGV) {
                     $in_func_head = 0;
                 } elsif (/\{/) {
                     if ($in_func_head == -1) {
-                        print "tp fn():$fn:$.\n";
+                        msg "tp fn():$fn:$.\n";
                     }
                     $in_func_head = 0;
                 }
             }
 
-	    ## Check for forbidden functions except when they are
-	    # explicitly permitted
-	    if (/\bassert\(/ && not /assert OK/) {
-		print "assert :$fn:$.   (use tor_assert)\n";
-	    }
-	    if (/\bmemcmp\(/ && not /memcmp OK/) {
-		print "memcmp :$fn:$.   (use {tor,fast}_mem{eq,neq,cmp}\n";
-	    }
-	    # always forbidden.
-	    if (not / OVERRIDE /) {
-		if (/\bstrcat\(/ or /\bstrcpy\(/ or /\bsprintf\(/) {
-		    print "$& :$fn:$.\n";
-		}
-		if (/\bmalloc\(/ or /\bfree\(/ or /\brealloc\(/ or
-		    /\bstrdup\(/ or /\bstrndup\(/ or /\bcalloc\(/) {
-		    print "$& :$fn:$.    (use tor_malloc, tor_free, etc)\n";
-		}
-	    }
+            ## Check for forbidden functions except when they are
+            # explicitly permitted
+            if (/\bassert\(/ && not /assert OK/) {
+                msg "assert :$fn:$.   (use tor_assert)\n";
+            }
+            if (/\bmemcmp\(/ && not /memcmp OK/) {
+                msg "memcmp :$fn:$.   (use {tor,fast}_mem{eq,neq,cmp}\n";
+            }
+            # always forbidden.
+            if (not /\ OVERRIDE\ /) {
+                if (/\bstrcat\(/ or /\bstrcpy\(/ or /\bsprintf\(/) {
+                    msg "$& :$fn:$.\n";
+                }
+                if (/\bmalloc\(/ or /\bfree\(/ or /\brealloc\(/ or
+                    /\bstrdup\(/ or /\bstrndup\(/ or /\bcalloc\(/) {
+                    msg "$& :$fn:$.    (use tor_malloc, tor_free, etc)\n";
+                }
+            }
         }
     }
     close(F);
 }
+
+exit $found;
diff --git a/scripts/maint/display_callgraph.py b/scripts/maint/display_callgraph.py
deleted file mode 100755
index c9001c6d96..0000000000
--- a/scripts/maint/display_callgraph.py
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/usr/bin/python
-
-import cPickle
-
-data = cPickle.load(open("callgraph.pkl"))
-
-# data = data['modItems']
-
-callgraph = data['callgraph']
-closure = data['closure']
-sccs = data['sccs']
-fn_bottle, call_bottle = data['bottlenecks']
-
-for n_reachable, fn in sorted(list((len(r), fn) for fn, r in closure.iteritems())):
-   print "%s can reach %s other functions." %(fn, n_reachable)
-
-
-c = [ (len(component), component) for component in sccs ]
-c.sort()
-
-print "\n================================"
-
-for n, component in c:
-   if n < 2:
-      continue
-   print "Strongly connected component of size %d:"%n
-   print component
-
-
-print "\n================================"
-
-print "====== Number of functions pulled into blob, by function in blob."
-fn_bottle.sort()
-for n, fn in fn_bottle[-30:]:
-   print "%3d: %s"%(n, fn)
-
-print "====== Number of functions pulled into blob, by call in blob."
-call_bottle.sort()
-for n, fn1, _, fn2 in call_bottle[-30:]:
-   print "%3d: %s -> %s "%(n, fn2, fn1)
-
diff --git a/scripts/maint/fallback.blacklist b/scripts/maint/fallback.blacklist
deleted file mode 100644
index c9fd8a9236..0000000000
--- a/scripts/maint/fallback.blacklist
+++ /dev/null
@@ -1,229 +0,0 @@
-# updateFallbackDirs.py directory mirror blacklist
-#
-# Format:
-# [ IPv4[:DirPort] ] [ orport=<ORPort> ] [ id=<ID> ] ...
-#   [ ipv6=<IPv6>[:<IPv6 ORPort>] ]
-#
-# If a sufficiently specific group of attributes matches, the directory mirror
-# will be excluded: (each group is listed on its own line)
-# <IPv4>, <DirPort>
-# <IPv4>, <ORPort>
-# <ID>
-# <IPv6>, <DirPort>
-# <IPv6>, <IPv6 ORPort>
-# If DirPort and ORPort are not present, the entire IP address is blacklisted.
-# (The blacklist overrides the whitelist.)
-
-# If a relay operator doesn't want their relay to be a FallbackDir,
-# enter the following information here:
-# <IPv4>:<DirPort> orport=<ORPort> id=<ID> ipv6=<IPv6>:<IPv6 ORPort>
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008364.html
-87.181.248.227:9030 orport=443 id=8827944C4BDCBDAC9079803F47823403C11A9B7A
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008368.html
-149.18.2.82:9030 orport=9001 id=953DB709F2A2DECC8D7560661F934E64411444F7
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008384.html
-80.82.215.199:80 orport=443 id=3BEFAB76461B6B99DCF34C285E933562F5712AE4 ipv6=[2001:4ba0:cafe:a18::1]:443
-
-# https://lists.torproject.org/pipermail/tor-relays/2016-January/008515.html
-# later opt-out in
-# https://lists.torproject.org/pipermail/tor-relays/2016-January/008521.html
-5.9.158.75:80 orport=443 id=F1BE15429B3CE696D6807F4D4A58B1BFEC45C822 ipv6=[2a01:4f8:190:514a::2]:443
-
-# Email sent directly to teor, verified using relay contact info
-5.34.183.168:80 orport=443 id=601C92108A568742A7A6D9473FE3A414F7149070
-217.12.199.208:8080 orport=22 id=BCFB0933367D626715DA32A147F417194A5D48D6
-
-# https://lists.torproject.org/pipermail/tor-relays/2016-January/008555.html
-62.210.207.124:9030 orport=9001 id=58938B1A5C4029B4415D38A4F36B7724273F4755 ipv6=[2001:bc8:31eb:100::1]:9001
-62.210.207.124:9130 orport=9101 id=338D0AB6DBAB7B529B9C91B2FD770658000693C4 ipv6=[2001:bc8:31eb:100::1]:9101
-
-# these fallback candidates fail the consensus download test in a way that
-# causes stem to hang (and not respond to ^C, at least on OS X)
-# (Is something sending weird responses to DirPort traffic?)
-#217.23.14.190:1194
-#151.80.164.147:80
-#148.251.255.92:80
-#78.142.19.59:80
-
-# Email sent directly to teor, verified using relay contact info
-216.17.99.183:80 orport=443 id=D52CD431CEF28E01B11F545A84347EE45524BCA7
-216.17.99.183:8080 orport=9001 id=EE21F83AB6F76E3B3FFCBA5C2496F789CB84E7C6
-65.19.167.130:80 orport=443 id=890E2EA65455FBF0FAAB4159FAC4412BDCB24295
-65.19.167.131:80 orport=443 id=0DA9BD201766EDB19F57F49F1A013A8A5432C008
-65.19.167.132:80 orport=443 id=12B80ABF019354A9D25EE8BE85EB3C0AD8F7DFC1
-65.19.167.133:80 orport=443 id=C170AE5A886C5A09D6D1CF5CF284653632EEF25D
-
-# Email sent directly to teor, verified using relay contact info
-213.136.83.225:80 orport=443 id=B411027C926A9BFFCF7DA91E3CAF1856A321EFFD
-195.154.126.78:80 orport=443 id=F6556156E2B3837248E03FDB770441CF64DBBFBE
-
-# Email sent directly to teor, verified using relay contact info
-178.63.198.113:80 orport=443 id=872B18761953254914F77C71846E8A2623C52591
-
-# Email sent directly to teor, verified using relay contact info
-63.141.226.34:80 orport=9001 id=5EF131C0C82270F40B756987FDB5D54D9C966238
-185.75.56.103:80 orport=9001 id=3763CE5C3F574670D4296573744F821C0FFFB98E
-
-# Email sent directly to teor, verified using relay contact info
-81.7.14.227:9030 orport=9001 id=BCA197C43A44B7B9D14509637F96A45B13C233D0
-
-# Email sent directly to teor, verified using relay contact info
-84.245.32.195:9030 orport=9001 id=4CD4DFFEF3971C902A22100D911CAC639BE2EF5C
-
-# Email sent directly to teor, verified using relay contact info
-185.21.217.10:9030 orport=9001 id=41537E1D3DD3CAE86F5A3F0882F1C647FE8FC0A0
-
-# Email sent directly to teor, verified using relay contact info
-185.21.216.140:9030 orport=9001 id=921DA852C95141F8964B359F774B35502E489869
-
-# Email sent directly to teor, verified using relay contact info
-62.210.82.44:143 orport=21 id=1C90D3AEADFF3BCD079810632C8B85637924A58E ipv6=[2001:bc8:3d7c::]:21
-
-# Email sent directly to teor, verified using relay contact info
-46.101.220.161:80 orport=443 id=7DDFE5B2C306B19A79832FBE581EAA245BAE90C6 ipv6=[2a03:b0c0:3:d0::8b:3001]:443
-
-# Email sent directly to teor, verified using relay contact info
-195.154.107.23:80 orport=443 id=A1F89F26E82209169E4037B035AE7B6C94A49AEB ipv6=[2001:bc8:3829:300::1]:443
-195.154.92.70:80 orport=443 id=E7FF4ECEEFCFE3A40A6D3594898A4A3DE018BBF5 ipv6=[2001:bc8:3829:500::1]:443
-195.154.113.200:80 orport=443 id=D1A4763FA0BD71978901B1951FEE1DC29777F95A ipv6=[2001:bc8:3829:600::1]:443
-195.154.92.155:110 orport=993 id=4477D3466FE136B7FE6F7FF8EBD0D6E2FFE3288B ipv6=[2001:bc8:3829:100::1]:993
-195.154.117.182:110 orport=993 id=B1A0F1143789466AADD5FAE5948C8138548EECEC ipv6=[2001:bc8:3829:400::1]:993
-195.154.97.163:80 orport=443 id=8A2994A63B20813B7724817A8FB8C444D10BA2E2
-
-# Email sent directly to teor, verified using relay contact info
-5.135.154.206:9030 orport=9001 id=7D67B342DC1158F4CFFEE8BC530A2448848026E3
-
-# Email sent directly to teor, verified using relay contact info
-85.24.215.117:9030 orport=9001 id=5989521A85C94EE101E88B8DB2E68321673F9405 ipv6=[2001:9b0:20:2106:21a:4aff:fea5:ad05]:9001
-
-# Email sent directly to teor, verified using relay contact info
-62.210.137.230:8888 orport=8843 id=CD6B850159CFF4C068A8D0F1BA5296AE4EDCAB39 ipv6=[2001:bc8:31d3:100::1]:3443
-62.210.137.230:8080 orport=8443 id=F596E1B1EF98E1DDBBDC934DB722AF54069868F6 ipv6=[2001:bc8:31d3:100::1]:8443
-
-# Email sent directly to teor, verified using relay contact info
-195.154.99.80:80 orport=443 id=6E7CB6E783C1B67B79D0EBBE7D48BC09BD441201
-195.154.127.60:80 orport=443 id=D74ABE34845190E242EC74BA28B8C89B0A480D4B
-
-# Email sent directly to teor, verified using relay contact info
-212.51.143.20:80 orport=443 id=62DA0256BBC28992D41CBAFB549FFD7C9B846A99
-
-# Email sent directly to teor, verified using relay contact info
-195.154.90.122:80 orport=443 id=3A0D88024A30152E6F6372CFDF8F9B725F984362
-
-# Email sent directly to teor, verified using relay contact info
-188.166.118.215:9030 orport=443 id=FB5FF60F5EBA010F8A45AC6ED31A4393718A2C31 ipv6=[2a03:b0c0:2:d0::72:9001]:443
-
-# Email sent directly to teor, verified using relay contact info
-185.87.185.245:40001 orport=40000 id=2A499AEEA95FB10F07544383D562368E49BE32CA
-
-# Email sent directly to teor, verified using relay contact info
-82.161.109.71:9030 orport=9001 id=BD9CE352648B940E788A8E45393C5400CC3E87E7
-
-# Email sent directly to teor, verified using relay contact info
-212.83.40.239:9030 orport=9001 id=6DC5616BD3FC463329DCE87DD7AAAEA112C264B5
-
-# Email sent directly to teor, verified using relay contact info
-178.32.53.53:80 orport=443 id=10582C360E972EE76B0DB1C246F4E892A6BF5465
-
-# Email sent directly to teor, verified using relay contact info
-85.114.135.20:9030 orport=9001 id=ED8A9291A3139E34BBD35037B082081EC6C26C80 ipv6=[2001:4ba0:fff5:2d::8]:9001
-148.251.128.156:9030 orport=9001 id=E382042E06A0A68AFC533E5AD5FB6867A12DF9FF ipv6=[2a01:4f8:210:238a::8]:9001
-62.210.115.147:9030 orport=9001 id=7F1D94E2C36F8CC595C2AB00022A5AE38171D50B ipv6=[2001:bc8:3182:101::8]:9001
-212.47.250.24:9030 orport=9001 id=33DA0CAB7C27812EFF2E22C9705630A54D101FEB
-
-# Email sent directly to teor, verified using relay contact info
-74.208.220.222:60000 orport=59999 id=4AA22235F0E9B3795A33930343CBB3EDAC60C5B0
-
-# Email sent directly to teor, verified using relay contact info
-89.163.140.168:9030 orport=9001 id=839C1212DB15723263BE96C83DA7E1B24FA395E8
-
-# Email sent directly to teor, verified using relay contact info
-212.47.246.211:9030 orport=9001 id=AA34219475E41282095DD3C088009EE562AF14E5
-
-# Email sent directly to teor, verified using relay contact info
-85.195.235.148:9030 orport=9001 id=103336165A0D2EFCAD3605339843A0A7710B8B92
-85.195.235.148:19030 orport=19001 id=713235638AB6C64715EAFD1B4772685E38AFD52A
-
-# Email sent directly to teor, verified using relay contact info
-163.172.7.30:9030 orport=9001 id=E2EACD4752B2583202F374A34CACC844A3AECAC4
-
-# Email sent directly to teor, verified using relay contact info
-178.62.90.111:22 orport=25 id=3254D1DC1F1531D9C07C535E4991F38EE99B99E1
-
-# Email sent directly to teor, verified using relay contact info
-213.200.106.131:9030 orport=4443 id=B07CE79FD215129C381F6645B16E76DCA0845CAB
-
-# Email sent directly to teor, verified using relay contact info
-198.51.75.165:80 orport=9001 id=DABCB84A524A22FDDD3AFCB090E3090CC12D9770
-
-# Email sent directly to teor, verified using relay contact info
-204.194.29.4:80 orport=9001 id=78C7C299DB4C4BD119A22B87B57D5AF5F3741A79
-
-# Email sent directly to teor, verified using relay contact info
-104.207.132.109:9030 orport=9001 id=12D5737383C23E756A7AA1A90BB24413BA428DA7 ipv6=[2001:19f0:300:2261::1]:9001
-
-# Email sent directly to teor, verified using relay contact info
-46.252.25.249:9030 orport=443 id=80DCBB6EF4E86A7CD4FBCBDEE64979645509A610
-
-# Email sent directly to teor, verified using relay contact info
-176.10.99.200:8080 orport=443 id=2B44FD1742D26E4F28D4CACF1F0CF8A686270E45
-176.10.99.200:8000 orport=22 id=EB79F07792A065D3C534063773E83268E069F5EB
-176.10.99.201:667 orport=666 id=3EAAAB35932610411E24FA4317603CB5780B80BC
-176.10.99.201:990 orport=989 id=7C3A4CFF09C1981D41173CDE2A2ADD4A5CA109FD
-176.10.99.202:992 orport=991 id=615EBC4B48F03858FA50A3E23E5AF569D0D2308A
-176.10.99.202:994 orport=993 id=E34E25D958D46DDE5092385B14117C9B301DC0E9
-176.10.99.203:1194 orport=995 id=AD368442E9FF33C08C7407DF2DA7DB958F406CE2
-176.10.99.203:43 orport=53 id=79CF377F0ACEC5F0002D85335E4192B34202A269
-176.10.99.204:1755 orport=1723 id=69DF3CDA1CDA460C17ECAD9D6F0C117A42384FA0
-176.10.99.204:1293 orport=4321 id=3F061400B6FB1F55E7F19BB3C713884D677E55B7
-176.10.99.205:426 orport=425 id=C30B284784BF11D0D58C6A250240EE58D2084AD0
-176.10.99.205:109 orport=110 id=12D17D9F9E30FA901DE68806950A0EA278716CED
-176.10.99.206:24 orport=23 id=2C804AAB0C02F971A4386B3A1F2AC00F9E080679
-176.10.99.206:20 orport=21 id=237588726AB6BEA37FF23CA00F5BD178586CA68E
-176.10.99.207:3390 orport=3389 id=A838D5B8890B10172429ECE92EB5677DF93DC4DD
-176.10.99.207:1415 orport=1414 id=377E5E817A84FAE0F4DC3427805DB2E8A6CBBFC0
-176.10.99.208:390 orport=389 id=7C288587BA0D99CC6B8537CDC2C4639FA827B907
-176.10.99.208:3307 orport=3306 id=1F0D2A44C56F42816DED2022EFD631878C29905B
-176.10.99.209:1434 orport=1433 id=BDA7A91FF3806DE5109FDAE74CFEFB3BABB9E10F
-176.10.99.209:220 orport=219 id=B8C2030001D832066A648269CFBA94171951D34B
-
-# Email sent directly to teor, verified using relay contact info
-78.193.40.205:8080 orport=8443 id=C91450840E75AC1B654A3096744338A573A239C6
-
-# Email sent directly to teor, verified using relay contact info
-37.187.22.172:9030 orport=9035 id=335E4117BD9A4966403C2AFA31CFDD1BC13BD46A
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008367.html
-# Email sent directly to teor to opt-out
-88.198.38.226:22 orport=443 id=4B9E2C56FB42B891794FE2CD2FCAD08A320CC3BB ipv6=[2a01:4f8:a0:1351::2]:80
-213.239.210.204:22 orport=443 id=5BFDECCE9B4A23AE14EC767C5A2C1E10558B00B9 ipv6=[2a01:4f8:a0:9474::2]:80
-213.239.220.25:22 orport=443 id=BEE2317AE127EB681C5AE1551C1EA0630580638A ipv6=[2a01:4f8:a0:710c::2]:80
-85.10.201.38:22 orport=443 id=F6279A203C1950ACF592322A235647A05BFBCF91 ipv6=[2a01:4f8:a0:43cc::2]:80
-
-# Email sent directly to teor, verified using relay contact info
-88.190.208.4:30555 orport=30556 id=030A6EB24725C05D8E0FCE21923CBA5223E75E0E
-
-# Fallback was on 0.2.8.2-alpha list, but changed fingerprint before 0.2.8.5
-46.101.102.71:80 orport=443 id=9504CB22EEB25D344DE63CB7A6F2C46F895C3686 ipv6=[2a03:b0c0:3:d0::2ed:7001]:9050
-# Also blacklist anything with the new fingerprint
-id=9C8A123081EFBE022EF795630F447839DDFDDDEC
-
-# Fallbacks were on 0.2.8.2-alpha list, but downloads were slow before 0.2.8.5
-185.96.88.29:80 orport=443 id=86C281AD135058238D7A337D546C902BE8505DDE
-178.62.36.64:9030 orport=9001 id=B87C84E38DAECFFFFDE98E5AEE5786AFDC748F2C
-
-# Fallback was on 0.2.8.2-alpha list, but changed address before 0.2.8.5
-84.219.173.60:9030 orport=443 id=855BC2DABE24C861CD887DB9B2E950424B49FC34
-# Also blacklist anything with the new address
-84.216.235.55:9030 orport=443
-
-# Fallbacks were on 0.2.8.2-alpha list, but disappeared before 0.2.8.5
-81.7.17.171:80 orport=443 id=CFECDDCA990E3EF7B7EC958B22441386B6B8D820 ipv6=[2a02:180:1:1::517:11ab]:443
-51.254.215.121:80 orport=443 id=262B66AD25C79588AD1FC8ED0E966395B47E5C1D
-185.100.85.138:80 orport=46356 id=5C4DF16A0029CC4F67D3E127356E68F219269859
-
-# Fallback was on 0.2.8.2-alpha list, but opted-out before 0.2.8.6
-37.187.1.149:9030 orport=9001 id=08DC0F3C6E3D9C527C1FC8745D35DD1B0DE1875D ipv6=[2001:41d0:a:195::1]:9001
diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist
index c801e46b15..79551948c6 100644
--- a/scripts/maint/fallback.whitelist
+++ b/scripts/maint/fallback.whitelist
@@ -2,6 +2,8 @@
 #
 # Format:
 # IPv4:DirPort orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ]
+# or use:
+# scripts/maint/generateFallbackDirLine.py fingerprint ...
 #
 # All attributes must match for the directory mirror to be included.
 # If the fallback has an ipv6 key, the whitelist line must also have
@@ -10,7 +12,8 @@
 
 # To replace this list with the hard-coded fallback list (for testing), use
 # a command similar to:
-#   cat src/or/fallback_dirs.inc | grep \" | grep -v weight | tr -d '\n' | \
+#   cat src/app/config/fallback_dirs.inc | grep \" | grep -v weight | \
+#   tr -d '\n' | \
 #   sed 's/"" / /g' | sed 's/""/"/g' | tr \" '\n' | grep -v '^$' \
 #   > scripts/maint/fallback.whitelist
 #
@@ -28,8 +31,9 @@
 # <IPv4>:<DirPort> orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ]
 
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008362.html
-78.47.18.110:443 orport=80 id=F8D27B163B9247B232A2EEE68DD8B698695C28DE
-131.188.40.188:443 orport=80 id=EBE718E1A49EE229071702964F8DB1F318075FF8
+# https://trac.torproject.org/projects/tor/ticket/22321#comment:22
+78.47.18.110:443 orport=80 id=F8D27B163B9247B232A2EEE68DD8B698695C28DE ipv6=[2a01:4f8:120:4023::110]:80 # fluxe3
+131.188.40.188:1443 orport=80 id=EBE718E1A49EE229071702964F8DB1F318075FF8 ipv6=[2001:638:a000:4140::ffff:188]:80 # fluxe4
 
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008366.html
 5.39.88.19:9030 orport=9001 id=7CB8C31432A796731EA7B6BF4025548DFEB25E0C ipv6=[2001:41d0:8:9a13::1]:9050
@@ -37,8 +41,6 @@
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008370.html
 # https://lists.torproject.org/pipermail/tor-relays/2016-January/008517.html
 # https://lists.torproject.org/pipermail/tor-relays/2016-January/008555.html
-62.210.124.124:9030 orport=9001 id=86E78DD3720C78DA8673182EF96C54B162CD660C ipv6=[2001:bc8:3f23:100::1]:9001
-62.210.124.124:9130 orport=9101 id=2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E ipv6=[2001:bc8:3f23:100::1]:9101
 212.47.237.95:9030 orport=9001 id=3F5D8A879C58961BB45A3D26AC41B543B40236D6
 212.47.237.95:9130 orport=9101 id=6FB38EB22E57EF7ED5EF00238F6A48E553735D88
 
@@ -49,27 +51,26 @@
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008373.html
 167.114.35.28:9030 orport=9001 id=E65D300F11E1DB12C534B0146BDAB6972F1A8A48
 
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008374.html
-170.130.1.7:9030 orport=9001 id=FA3415659444AE006E7E9E5375E82F29700CFDFD
-
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008378.html
 144.76.14.145:110 orport=143 id=14419131033443AE6E21DA82B0D307F7CAE42BDB ipv6=[2a01:4f8:190:9490::dead]:443
 
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008379.html
 # Email sent directly to teor, verified using relay contact info
 91.121.84.137:4951 orport=4051 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008380.html
-5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33
+91.121.84.137:4952 orport=4052 id=9FBEB75E8BC142565F12CBBE078D63310236A334
 
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008381.html
-# Sent additional email to teor with more relays
-178.254.44.135:9030 orport=9001 id=8FA37B93397015B2BC5A525C908485260BE9F422
-178.254.20.134:80 orport=443 id=9F5068310818ED7C70B0BC4087AB55CB12CB4377
-178.254.20.134:9030 orport=9001 id=2CE96A8A1DA032664C90F574AFFBECE18A6E8DFC
-178.254.44.135:80 orport=443 id=AE6A8C18E7499B586CD36246AC4BCAFFBBF93AB2
-178.254.13.126:80 orport=443 id=F9246DEF2B653807236DA134F2AEAB103D58ABFE
-178.254.13.126:9030 orport=9001 id=0C475BA4D3AA3C289B716F95954CAD616E50C4E5
+# Sent additional emails to teor with updated relays
+81.7.11.96:9030 orport=9001 id=8FA37B93397015B2BC5A525C908485260BE9F422 # Doedel22
+# 9F5068310818ED7C70B0BC4087AB55CB12CB4377 not found in current consensus
+178.254.19.101:80 orport=443 id=F9246DEF2B653807236DA134F2AEAB103D58ABFE # Freebird31
+178.254.19.101:9030 orport=9001 id=0C475BA4D3AA3C289B716F95954CAD616E50C4E5 # Freebird32
+81.7.14.253:9001 orport=443 id=1AE039EE0B11DB79E4B4B29CBA9F752864A0259E # Ichotolot60
+81.7.11.186:1080 orport=443 id=B86137AE9681701901C6720E55C16805B46BD8E3 # BeastieJoy60
+85.25.213.211:465 orport=80 id=CE47F0356D86CF0A1A2008D97623216D560FB0A8 # BeastieJoy61
+85.25.159.65:995 orport=80 id=52BFADA8BEAA01BA46C8F767F83C18E2FE50C1B9 # BeastieJoy63
+81.7.3.67:993 orport=443 id=A2E6BB5C391CD46B38C55B4329C35304540771F1 # BeastieJoy62
+81.7.14.31:9001 orport=443 id=7600680249A22080ECC6173FBBF64D6FCF330A61 # Ichotolot62
 
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008382.html
 51.255.33.237:9091 orport=9001 id=A360C21FA87FFA2046D92C17086A6B47E5C68109
@@ -98,33 +99,28 @@
 # https://lists.torproject.org/pipermail/tor-relays/2016-January/008542.html
 178.62.199.226:80 orport=443 id=CBEFF7BA4A4062045133C053F2D70524D8BBE5BE ipv6=[2a03:b0c0:2:d0::b7:5001]:443
 
-# Emails sent directly to teor, verified using relay contact info
-217.12.199.208:80 orport=443 id=DF3AED4322B1824BF5539AE54B2D1B38E080FF05
-
 # Email sent directly to teor, verified using relay contact info
 94.23.204.175:9030 orport=9001 id=5665A3904C89E22E971305EE8C1997BCA4123C69
 
-# https://twitter.com/binarytenshi/status/717952514327453697
-94.126.23.174:9030 orport=9001 id=6FC6F08270D565BE89B7C819DD8E2D487397C073
-
 # Email sent directly to teor, verified using relay contact info
-171.25.193.78:80 orport=443 id=A478E421F83194C114F41E94F95999672AED51FE ipv6=[2001:67c:289c:3::78]:443
 171.25.193.77:80 orport=443 id=A10C4F666D27364036B562823E5830BC448E046A ipv6=[2001:67c:289c:3::77]:443
-171.25.193.131:80 orport=443 id=79861CF8522FC637EF046F7688F5289E49D94576
+171.25.193.78:80 orport=443 id=A478E421F83194C114F41E94F95999672AED51FE ipv6=[2001:67c:289c:3::78]:443
 171.25.193.20:80 orport=443 id=DD8BD7307017407FCC36F8D04A688F74A0774C02 ipv6=[2001:67c:289c::20]:443
-# OK, but same machine as 79861CF8522FC637EF046F7688F5289E49D94576
-#171.25.193.132:80 orport=443 id=01C67E0CA8F97111E652C7564CB3204361FFFAB8
-# OK, but same machine as DD8BD7307017407FCC36F8D04A688F74A0774C02
-#171.25.193.25:80 orport=443 id=185663B7C12777F052B2C2D23D7A239D8DA88A0F ipv6=[2001:67c:289c::25]:443
+# same machine as DD8BD7307017407FCC36F8D04A688F74A0774C02
+171.25.193.25:80 orport=443 id=185663B7C12777F052B2C2D23D7A239D8DA88A0F ipv6=[2001:67c:289c::25]:443
 
 # Email sent directly to teor, verified using relay contact info
-212.47.229.2:9030 orport=9001 id=20462CBA5DA4C2D963567D17D0B7249718114A68
+212.47.229.2:9030 orport=9001 id=20462CBA5DA4C2D963567D17D0B7249718114A68 ipv6=[2001:bc8:4400:2100::f03]:9001
 93.115.97.242:9030 orport=9001 id=B5212DB685A2A0FCFBAE425738E478D12361710D
-46.28.109.231:9030 orport=9001 id=F70B7C5CD72D74C7F9F2DC84FA9D20D51BA13610 ipv6=[2a02:2b88:2:1::4205:42]:9001
+46.28.109.231:9030 orport=9001 id=F70B7C5CD72D74C7F9F2DC84FA9D20D51BA13610 ipv6=[2a02:2b88:2:1::4205:1]:9001
 
 # Email sent directly to teor, verified using relay contact info
-85.235.250.88:80 orport=443 id=72B2B12A3F60408BDBC98C6DF53988D3A0B3F0EE
-185.96.180.29:80 orport=443 id=F93D8F37E35C390BCAD9F9069E13085B745EC216
+85.235.250.88:80 orport=443 id=72B2B12A3F60408BDBC98C6DF53988D3A0B3F0EE # TykRelay01
+185.96.88.29:80 orport=443 id=86C281AD135058238D7A337D546C902BE8505DDE # TykRelay051
+# This fallback opted-in in previous releases, then changed its details,
+# and so we blacklisted it. Now we want to whitelist changes.
+# Assume details update is permanent
+185.96.180.29:80 orport=443 id=F93D8F37E35C390BCAD9F9069E13085B745EC216 # TykRelay06
 
 # Email sent directly to teor, verified using relay contact info
 185.11.180.67:80 orport=9001 id=794D8EA8343A4E820320265D05D4FA83AB6D1778
@@ -154,31 +150,20 @@
 178.16.208.59:80 orport=443 id=136F9299A5009A4E0E96494E723BDB556FB0A26B ipv6=[2a00:1c20:4089:1234:bff6:e1bb:1ce3:8dc6]:443
 
 # Email sent directly to teor, verified using relay contact info
-195.154.8.111:80 orport=443 id=FCB6695F8F2DC240E974510A4B3A0F2B12AB5B64
-51.255.235.246:80 orport=443 id=9B99C72B02AF8E3E5BE3596964F9CACD0090D132
 5.39.76.158:80 orport=443 id=C41F60F8B00E7FEF5CCC5BC6BB514CA1B8AAB651
 
 # Email sent directly to teor, verified using relay contact info
-109.163.234.5:80 orport=443 id=5C84C35936B7100B949AC75764EEF1352550550B
-109.163.234.7:80 orport=443 id=C46524E586E1B997329703D356C07EE12B28C722
-109.163.234.9:80 orport=443 id=5714542DCBEE1DD9864824723638FD44B2122CEA
-77.247.181.162:80 orport=443 id=7BB160A8F54BD74F3DA5F2CE701E8772B841859D
-109.163.234.4:80 orport=443 id=6B1E001929AF4DDBB747D02EC28340792B7724A6
-77.247.181.164:80 orport=443 id=10E13E340651D0EF66B4DEBF610B3C0981168107
-109.163.234.8:80 orport=443 id=20B0038D7A2FD73C696922551B8344CB0893D1F8
-77.247.181.166:80 orport=443 id=06E123865C590189B3181114F23F0F13A7BC0E69
-109.163.234.2:80 orport=443 id=B4F883DB3D478C7AE569C9F6CB766FD58650DC6A
-62.102.148.67:80 orport=443 id=4A0C3E177AF684581EF780981AEAF51A98A6B5CF
-109.163.234.5:80 orport=443 id=5C84C35936B7100B949AC75764EEF1352550550B
-109.163.234.7:80 orport=443 id=C46524E586E1B997329703D356C07EE12B28C722
-109.163.234.9:80 orport=443 id=5714542DCBEE1DD9864824723638FD44B2122CEA
-77.247.181.162:80 orport=443 id=7BB160A8F54BD74F3DA5F2CE701E8772B841859D
-109.163.234.4:80 orport=443 id=6B1E001929AF4DDBB747D02EC28340792B7724A6
-77.247.181.164:80 orport=443 id=10E13E340651D0EF66B4DEBF610B3C0981168107
-109.163.234.8:80 orport=443 id=20B0038D7A2FD73C696922551B8344CB0893D1F8
-77.247.181.166:80 orport=443 id=06E123865C590189B3181114F23F0F13A7BC0E69
-109.163.234.2:80 orport=443 id=B4F883DB3D478C7AE569C9F6CB766FD58650DC6A
+109.163.234.2:80 orport=443 id=14F92FF956105932E9DEC5B82A7778A0B1BD9A52
+109.163.234.4:80 orport=443 id=4888770464F0E900EFEF1BA181EA873D13F7713C
+109.163.234.5:80 orport=443 id=5EB8D862E70981B8690DEDEF546789E26AB2BD24
+109.163.234.7:80 orport=443 id=23038A7F2845EBA2234ECD6651BD4A7762F51B18
+109.163.234.8:80 orport=443 id=0818DAE0E2DDF795AEDEAC60B15E71901084F281
+109.163.234.9:80 orport=443 id=ABF7FBF389C9A747938B639B20E80620B460B2A9
 62.102.148.67:80 orport=443 id=4A0C3E177AF684581EF780981AEAF51A98A6B5CF
+# Assume details update is permanent
+77.247.181.166:80 orport=443 id=77131D7E2EC1CA9B8D737502256DA9103599CE51 # CriticalMass
+77.247.181.164:80 orport=443 id=204DFD2A2C6A0DC1FA0EACB495218E0B661704FD # HaveHeart
+77.247.181.162:80 orport=443 id=7BFB908A3AA5B491DA4CA72CCBEE0E1F2A939B55 # sofia
 
 # https://twitter.com/biotimylated/status/718994247500718080
 212.47.252.149:9030 orport=9001 id=2CAC39BAA996791CEFAADC9D4754D65AF5EB77C0
@@ -215,9 +200,7 @@
 
 # Email sent directly to teor, verified using relay contact info
 86.59.119.88:80 orport=443 id=ACD889D86E02EDDAB1AFD81F598C0936238DC6D0
-
-# Email sent directly to teor, verified using relay contact info
-144.76.73.140:9030 orport=9001 id=6A640018EABF3DA9BAD9321AA37C2C87BBE1F907
+86.59.119.83:80 orport=443 id=FC9AC8EA0160D88BCCFDE066940D7DD9FA45495B
 
 # Email sent directly to teor, verified using relay contact info
 193.11.164.243:9030 orport=9001 id=FFA72BD683BC2FCF988356E6BEC1E490F313FB07 ipv6=[2001:6b0:7:125::243]:9001
@@ -232,20 +215,19 @@
 # Email sent directly to teor, verified using relay contact info
 89.187.142.208:80 orport=443 id=64186650FFE4469EBBE52B644AE543864D32F43C
 
-# Email sent directly to teor, verified using relay contact info
-212.51.134.123:9030 orport=9001 id=50586E25BE067FD1F739998550EDDCB1A14CA5B2 ipv6=[2a02:168:6e00:0:3a60:77ff:fe9c:8bd1]:9001
+# Email sent directly to teor
+# Assume details update is permanent
+212.51.134.123:9030 orport=9001 id=50586E25BE067FD1F739998550EDDCB1A14CA5B2 # Jans
 
 # Email sent directly to teor, verified using relay contact info
 46.101.143.173:80 orport=443 id=F960DF50F0FD4075AC9B505C1D4FFC8384C490FB
 
 # Email sent directly to teor, verified using relay contact info
-217.79.190.25:9030 orport=9090 id=361D33C96D0F161275EE67E2C91EE10B276E778B
-
-# Email sent directly to teor, verified using relay contact info
 193.171.202.146:9030 orport=9001 id=01A9258A46E97FF8B2CAC7910577862C14F2C524
 
 # Email sent directly to teor, verified using relay contact info
-197.231.221.211:9030 orport=9001 id=BC630CBBB518BE7E9F4E09712AB0269E9DC7D626
+# Assume details update is permanent
+197.231.221.211:9030 orport=443 id=BC630CBBB518BE7E9F4E09712AB0269E9DC7D626 # IPredator
 
 # Email sent directly to teor, verified using relay contact info
 185.61.138.18:8080 orport=4443 id=2541759BEC04D37811C2209A88E863320271EC9C
@@ -256,7 +238,7 @@
 193.11.114.46:9032 orport=9003 id=B83DC1558F0D34353BB992EF93AFEAFDB226A73E
 
 # Email sent directly to teor, verified using relay contact info
-144.76.26.175:9012 orport=9011 id=2BA2C8E96B2590E1072AECE2BDB5C48921BF8510
+138.201.250.33:9012 orport=9011 id=2BA2C8E96B2590E1072AECE2BDB5C48921BF8510
 
 # Email sent directly to teor, verified using relay contact info
 37.221.162.226:9030 orport=9001 id=D64366987CB39F61AD21DBCF8142FA0577B92811
@@ -274,12 +256,12 @@
 134.119.3.164:9030 orport=9001 id=D1B8AAA98C65F3DF7D8BB3AF881CAEB84A33D8EE
 
 # Email sent directly to teor, verified using relay contact info
-81.7.10.93:31336 orport=31337 id=99E246DB480B313A3012BC3363093CC26CD209C7
+173.212.254.192:31336 orport=31337 id=99E246DB480B313A3012BC3363093CC26CD209C7
 
 # Email sent directly to teor, verified using relay contact info
 178.62.22.36:80 orport=443 id=A0766C0D3A667A3232C7D569DE94A28F9922FCB1 ipv6=[2a03:b0c0:1:d0::174:1]:9050
-188.166.23.127:80 orport=443 id=3771A8154DEA98D551607806C80A209CDAA74535 ipv6=[2a03:b0c0:2:d0::27b:7001]:9050
-198.199.64.217:80 orport=443 id=FAD306BAA59F6A02783F8606BDAA431F5FF7D1EA ipv6=[2604:a880:400:d0::1a9:b001]:9050
+188.166.23.127:80 orport=443 id=8672E8A01B4D3FA4C0BBE21C740D4506302EA487 ipv6=[2a03:b0c0:2:d0::27b:7001]:9050
+198.199.64.217:80 orport=443 id=B1D81825CFD7209BD1B4520B040EF5653C204A23 ipv6=[2604:a880:400:d0::1a9:b001]:9050
 159.203.32.149:80 orport=443 id=55C7554AFCEC1062DCBAC93E67B2E03C6F330EFC ipv6=[2604:a880:cad:d0::105:f001]:9050
 
 # Email sent directly to teor, verified using relay contact info
@@ -301,9 +283,6 @@
 212.47.230.49:9030 orport=9001 id=3D6D0771E54056AEFC28BB1DE816951F11826E97
 
 # Email sent directly to teor, verified using relay contact info
-176.31.180.157:143 orport=22 id=E781F4EC69671B3F1864AE2753E0890351506329 ipv6=[2001:41d0:8:eb9d::1]:22
-
-# Email sent directly to teor, verified using relay contact info
 192.99.55.69:80 orport=443 id=0682DE15222A4A4A0D67DBA72A8132161992C023
 192.99.59.140:80 orport=443 id=3C9148DA49F20654730FAC83FFF693A4D49D0244
 51.254.215.13:80 orport=443 id=73C30C8ABDD6D9346C822966DE73B9F82CB6178A
@@ -318,7 +297,7 @@
 151.80.42.103:9030 orport=9001 id=9007C1D8E4F03D506A4A011B907A9E8D04E3C605 ipv6=[2001:41d0:e:f67::114]:9001
 
 # Email sent directly to teor, verified using relay contact info
-5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6
+5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6 ipv6=[2001:41d0:8:b1c7::1]:443
 
 # Email sent directly to teor, verified using relay contact info
 176.31.159.231:80 orport=443 id=D5DBCC0B4F029F80C7B8D33F20CF7D97F0423BB1
@@ -328,17 +307,15 @@
 46.148.18.74:8080 orport=443 id=6CACF0B5F03C779672F3C5C295F37C8D234CA3F7
 
 # Email sent directly to teor, verified using relay contact info
-37.187.102.108:9090 orport=5550 id=F4263275CF54A6836EE7BD527B1328836A6F06E1
-212.47.241.21:80 orport=443 id=892F941915F6A0C6E0958E52E0A9685C190CF45C
+37.187.102.108:80 orport=443 id=F4263275CF54A6836EE7BD527B1328836A6F06E1 ipv6=[2001:41d0:a:266c::1]:443 # EvilMoe
+212.47.241.21:80 orport=443 id=892F941915F6A0C6E0958E52E0A9685C190CF45C # EvilMoe
 
 # Email sent directly to teor, verified using relay contact info
-195.191.233.221:80 orport=443 id=DE134FC8E5CC4EC8A5DE66934E70AC9D70267197
+212.129.38.254:9030 orport=9001 id=FDF845FC159C0020E2BDDA120C30C5C5038F74B4
 
-# Email sent directly to teor, verified using relay contact info
-62.210.238.33:9030 orport=9001 id=FDF845FC159C0020E2BDDA120C30C5C5038F74B4
-
-# Email sent directly to teor, verified using relay contact info
-37.157.195.87:8030 orport=443 id=12FD624EE73CEF37137C90D38B2406A66F68FAA2
+# Email sent directly to teor
+37.157.195.87:8030 orport=443 id=12FD624EE73CEF37137C90D38B2406A66F68FAA2 # thanatosCZ
+5.189.169.190:8030 orport=8080 id=8D79F73DCD91FC4F5017422FAC70074D6DB8DD81 # thanatosDE
 
 # Email sent directly to teor, verified using relay contact info
 37.187.7.74:80 orport=443 id=AEA43CB1E47BE5F8051711B2BF01683DB1568E05 ipv6=[2001:41d0:a:74a::1]:443
@@ -356,9 +333,6 @@
 37.187.102.186:9030 orport=9001 id=489D94333DF66D57FFE34D9D59CC2D97E2CB0053 ipv6=[2001:41d0:a:26ba::1]:9001
 
 # Email sent directly to teor, verified using relay contact info
-5.35.251.247:9030 orport=9001 id=9B1F5187DFBA89DC24B37EA7BF896C12B43A27AE
-
-# Email sent directly to teor, verified using relay contact info
 198.96.155.3:8080 orport=5001 id=BCEDF6C193AA687AE471B8A22EBF6BC57C2D285E
 
 # Email sent directly to teor, verified using relay contact info
@@ -405,14 +379,6 @@
 91.219.237.229:80 orport=443 id=1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7
 
 # Email sent directly to teor, verified using relay contact info
-# Suitable, check with operator before adding
-#212.47.240.10:82 orport=443 id=2A4C448784F5A83AFE6C78DA357D5E31F7989DEB
-212.47.240.10:81 orport=993 id=72527E3242CB15AADE28374AE0D35833FC083F60
-163.172.131.88:80 orport=443 id=AD253B49E303C6AB1E048B014392AC569E8A7DAE ipv6=[2001:bc8:4400:2100::2:1009]:443
-# Suitable, check with operator before adding
-#163.172.131.88:81 orport=993 id=D5F3FB17504744FB7ECEF46F4B1D155258A6D942 ipv6=D5F3FB17504744FB7ECEF46F4B1D155258A6D942
-
-# Email sent directly to teor, verified using relay contact info
 46.101.151.222:80 orport=443 id=1DBAED235E3957DE1ABD25B4206BE71406FB61F8
 178.62.60.37:80 orport=443 id=175921396C7C426309AB03775A9930B6F611F794
 
@@ -440,11 +406,8 @@
 # Email sent directly to teor, verified using relay contact info
 5.199.142.236:9030 orport=9001 id=F4C0EDAA0BF0F7EC138746F8FEF1CE26C7860265
 
-# Email sent directly to teor, verified using relay contact info
-188.166.133.133:9030 orport=9001 id=774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7 ipv6=[2a03:b0c0:2:d0::5:f001]:9001
-
-# Email sent directly to teor, verified using relay contact info
-5.196.88.122:9030 orport=9001 id=0C2C599AFCB26F5CFC2C7592435924C1D63D9484
+# Email sent directly to teor
+188.166.133.133:9030 orport=9001 id=774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7 ipv6=[2a03:b0c0:2:d0::26c0:1]:9001 # dropsy
 
 # Email sent directly to teor, verified using relay contact info
 46.8.249.10:80 orport=443 id=31670150090A7C3513CB7914B9610E786391A95D
@@ -456,12 +419,10 @@
 46.4.24.161:9030 orport=9001 id=DB4C76A3AD7E234DA0F00D6F1405D8AFDF4D8DED
 46.4.24.161:9031 orport=9002 id=7460F3D12EBE861E4EE073F6233047AACFE46AB4
 46.38.51.132:9030 orport=9001 id=810DEFA7E90B6C6C383C063028EC397A71D7214A
-163.172.194.53:9030 orport=9001 id=8C00FA7369A7A308F6A137600F0FA07990D9D451
+163.172.194.53:9030 orport=9001 id=8C00FA7369A7A308F6A137600F0FA07990D9D451 ipv6=[2001:bc8:225f:142:6c69:7461:7669:73]:9001
 
 # Email sent directly to teor, verified using relay contact info
 176.10.107.180:9030 orport=9001 id=3D7E274A87D9A89AF064C13D1EE4CA1F184F2600
-195.154.75.84:9030 orport=9001 id=F80FDE27EFCB3F6A7B4E2CC517133DBFFA78BA2D
-195.154.127.246:9030 orport=9001 id=4FEE77AFFD157BBCF2D896AE417FBF647860466C
 
 # Email sent directly to teor, verified using relay contact info
 46.28.207.19:80 orport=443 id=5B92FA5C8A49D46D235735504C72DBB3472BA321
@@ -485,11 +446,10 @@
 5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11
 
 # Email sent directly to teor, verified using relay contact info
-167.114.152.100:9030 orport=443 id=0EF5E5FFC5D1EABCBDA1AFF6F6D6325C5756B0B2 ipv6=[2607:5300:100:200::1608]:443
-
-# Email sent directly to teor, verified using relay contact info
-192.99.168.102:80 orport=443 id=230A8B2A8BA861210D9B4BA97745AEC217A94207
-167.114.153.21:80 orport=443 id=0B85617241252517E8ECF2CFC7F4C1A32DCD153F
+# Updated details from atlas based on ticket #20010
+163.172.176.167:80 orport=443 id=230A8B2A8BA861210D9B4BA97745AEC217A94207
+163.172.149.155:80 orport=443 id=0B85617241252517E8ECF2CFC7F4C1A32DCD153F
+163.172.149.122:80 orport=443 id=A9406A006D6E7B5DA30F2C6D4E42A338B5E340B2
 
 # Email sent directly to teor, verified using relay contact info
 204.11.50.131:9030 orport=9001 id=185F2A57B0C4620582602761097D17DB81654F70
@@ -498,13 +458,10 @@
 151.236.222.217:44607 orport=9001 id=94D58704C2589C130C9C39ED148BD8EA468DBA54
 
 # Email sent directly to teor, verified using relay contact info
-194.150.168.79:11112 orport=11111 id=29F1020B94BE25E6BE1AD13E93CE19D2131B487C
-
-# Email sent directly to teor, verified using relay contact info
 185.35.202.221:9030 orport=9001 id=C13B91384CDD52A871E3ECECE4EF74A7AC7DCB08 ipv6=[2a02:ed06::221]:9001
 
 # Email sent directly to teor, verified using relay contact info
-5.9.151.241:9030 orport=4223 id=9BF04559224F0F1C3C953D641F1744AF0192543A
+5.9.151.241:9030 orport=4223 id=9BF04559224F0F1C3C953D641F1744AF0192543A ipv6=[2a01:4f8:190:34f0::2]:4223
 
 # Email sent directly to teor, verified using relay contact info
 89.40.71.149:8081 orport=8080 id=EC639EDAA5121B47DBDF3D6B01A22E48A8CB6CC7
@@ -513,7 +470,7 @@
 92.222.20.130:80 orport=443 id=0639612FF149AA19DF3BCEA147E5B8FED6F3C87C
 
 # Email sent directly to teor, verified using relay contact info
-80.112.155.100:9030 orport=9001 id=1163378F239C36CA1BDC730AC50BF4F2976141F5 ipv6=[2001:470:7b02::38]:9001
+80.112.155.100:9030 orport=9001 id=53B000310984CD86AF47E5F3CD0BFF184E34B383 ipv6=[2001:470:7b02::38]:9001
 
 # Email sent directly to teor, verified using relay contact info
 83.212.99.68:80 orport=443 id=DDBB2A38252ADDA53E4492DDF982CA6CC6E10EC0 ipv6=[2001:648:2ffc:1225:a800:bff:fe3d:67b5]:443
@@ -522,9 +479,6 @@
 95.130.11.147:9030 orport=443 id=6B697F3FF04C26123466A5C0E5D1F8D91925967A
 
 # Email sent directly to teor, verified using relay contact info
-176.31.191.26:9030 orport=9001 id=7350AB9ED7568F22745198359373C04AC783C37C
-
-# Email sent directly to teor, verified using relay contact info
 128.199.55.207:9030 orport=9001 id=BCEF908195805E03E92CCFE669C48738E556B9C5 ipv6=[2a03:b0c0:2:d0::158:3001]:9001
 
 # Email sent directly to teor, verified using relay contact info
@@ -540,16 +494,17 @@
 80.240.139.111:80 orport=443 id=DD3BE7382C221F31723C7B294310EF9282B9111B
 
 # Email sent directly to teor, verified using relay contact info
-185.97.32.18:9030 orport=9001 id=3BAB316CAAEC47E71905EB6C65584636D5689A8A
+185.97.32.18:9030 orport=9001 id=04250C3835019B26AA6764E85D836088BE441088
 
-# Email sent directly to teor, verified using relay contact info
-149.56.45.200:9030 orport=9001 id=FE296180018833AF03A8EACD5894A614623D3F76
+# Email sent directly to teor
+149.56.45.200:9030 orport=9001 id=FE296180018833AF03A8EACD5894A614623D3F76 ipv6=[2607:5300:201:3000::17d3]:9002 # PiotrTorpotkinOne
 
 # Email sent directly to teor, verified using relay contact info
-81.2.209.10:443 orport=80 id=B6904ADD4C0D10CDA7179E051962350A69A63243
+81.2.209.10:443 orport=80 id=B6904ADD4C0D10CDA7179E051962350A69A63243 ipv6=[2001:15e8:201:1::d10a]:80
 
 # Email sent directly to teor, verified using relay contact info
-195.154.164.243:80 orport=443 id=AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C ipv6=[2001:bc8:399f:f000::1]:993
+# IPv6 address unreliable
+195.154.164.243:80 orport=443 id=AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C #ipv6=[2001:bc8:399f:f000::1]:993
 138.201.26.2:80 orport=443 id=6D3A3ED5671E4E3F58D4951438B10AE552A5FA0F
 81.7.16.182:80 orport=443 id=51E1CF613FD6F9F11FE24743C91D6F9981807D82 ipv6=[2a02:180:1:1::517:10b6]:993
 134.119.36.135:80 orport=443 id=763C9556602BD6207771A7A3D958091D44C43228 ipv6=[2a00:1158:3::2a8]:993
@@ -563,7 +518,7 @@
 217.12.208.117:80 orport=443 id=E6E18151300F90C235D3809F90B31330737CEB43 ipv6=[2a00:1ca8:a7::1bb]:993
 81.7.10.251:80 orport=443 id=8073670F8F852971298F8AF2C5B23AE012645901 ipv6=[2a02:180:1:1::517:afb]:993
 46.36.39.50:80 orport=443 id=ED4B0DBA79AEF5521564FA0231455DCFDDE73BB6 ipv6=[2a02:25b0:aaaa:aaaa:8d49:b692:4852:0]:995
-91.194.90.103:80 orport=443 id=75C4495F4D80522CA6F6A3FB349F1B009563F4B7 ipv6=[2a02:c200:0:10:3:0:5449:1]:993
+91.194.90.103:80 orport=443 id=75C4495F4D80522CA6F6A3FB349F1B009563F4B7 ipv6=[2a02:c205:3000:5449::1]:993
 163.172.25.118:80 orport=22 id=0CF8F3E6590F45D50B70F2F7DA6605ECA6CD408F
 188.138.88.42:80 orport=443 id=70C55A114C0EF3DC5784A4FAEE64388434A3398F
 81.7.13.84:80 orport=443 id=0C1E7DD9ED0676C788933F68A9985ED853CA5812 ipv6=[2a02:180:1:1::5b8f:538c]:993
@@ -580,18 +535,17 @@
 # Email sent directly to teor, verified using relay contact info
 212.238.208.48:9030 orport=9001 id=F406219CDD339026D160E53FCA0EF6857C70F109 ipv6=[2001:984:a8fb:1:ba27:ebff:feac:c109]:9001
 
-# Email sent directly to teor, verified using relay contact info
-176.158.132.12:9030 orport=9001 id=DC163DDEF4B6F0C6BC226F9F6656A5A30C5C5686
+# Email sent directly to teor
+176.158.236.102:9030 orport=9001 id=DC163DDEF4B6F0C6BC226F9F6656A5A30C5C5686 # Underworld
 
 # Email sent directly to teor, verified using relay contact info
 91.229.20.27:9030 orport=9001 id=9A0D54D3A6D2E0767596BF1515E6162A75B3293F
 
 # Email sent directly to teor, verified using relay contact info
-# Awaiting confirmation of new ORPort from relay operator
 80.127.137.19:80 orport=443 id=6EF897645B79B6CB35E853B32506375014DE3621 ipv6=[2001:981:47c1:1::6]:443
 
-# Email sent directly to teor, verified using relay contact info
-163.172.138.22:80 orport=443 id=8664DC892540F3C789DB37008236C096C871734D
+# Email sent directly to teor
+163.172.138.22:80 orport=443 id=16102E458460349EE45C0901DAA6C30094A9BBEA ipv6=[2001:bc8:4400:2100::1:3]:443 # mkultra
 
 # Email sent directly to teor, verified using relay contact info
 97.74.237.196:9030 orport=9001 id=2F0F32AB1E5B943CA7D062C03F18960C86E70D94
@@ -603,10 +557,11 @@
 178.62.98.160:9030 orport=9001 id=8B92044763E880996A988831B15B2B0E5AD1544A
 
 # Email sent directly to teor, verified using relay contact info
-195.154.15.227:9030 orport=9001 id=6C3E3AB2F5F03CD71B637D433BAD924A1ECC5796
+163.172.217.50:9030 orport=9001 id=02ECD99ECD596013A8134D46531560816ECC4BE6
 
 # Email sent directly to teor, verified using relay contact info
 185.100.86.100:80 orport=443 id=0E8C0C8315B66DB5F703804B3889A1DD66C67CE0
+185.100.84.82:80 orport=443 id=7D05A38E39FC5D29AFE6BE487B9B4DC9E635D09E
 
 # Email sent directly to teor, verified using relay contact info
 164.132.77.175:9030 orport=9001 id=3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B
@@ -617,13 +572,15 @@
 178.62.86.96:9030 orport=9001 id=439D0447772CB107B886F7782DBC201FA26B92D1 ipv6=[2a03:b0c0:1:d0::3cf:7001]:9050
 
 # Email sent directly to teor, verified using relay contact info
-91.233.106.121:80 orport=443 id=896364B7996F5DFBA0E15D1A2E06D0B98B555DD6
+# Very low bandwidth, stale consensues, excluded to cut down on warnings
+#91.233.106.121:80 orport=443 id=896364B7996F5DFBA0E15D1A2E06D0B98B555DD6
 
 # Email sent directly to teor, verified using relay contact info
-167.114.113.48:9030 orport=443 id=2EC0C66EA700C44670444280AABAB1EC78B722A0
+167.114.113.48:9030 orport=403 id=2EC0C66EA700C44670444280AABAB1EC78B722A0
 
 # Email sent directly to teor, verified using relay contact info
-79.120.16.42:9030 orport=9001 id=BD552C165E2ED2887D3F1CCE9CFF155DDA2D86E6
+# Assume details update is permanent
+213.141.138.174:9030 orport=9001 id=BD552C165E2ED2887D3F1CCE9CFF155DDA2D86E6 # Schakalium
 
 # Email sent directly to teor, verified using relay contact info
 95.128.43.164:80 orport=443 id=616081EC829593AF4232550DE6FFAA1D75B37A90 ipv6=[2a02:ec0:209:10::4]:443
@@ -652,10 +609,13 @@
 # Email sent directly to teor, verified using relay contact info
 31.31.78.49:80 orport=443 id=46791D156C9B6C255C2665D4D8393EC7DBAA7798
 
-# Email sent directly to teor, verified using relay contact info
-96.47.231.214:9030 orport=8080 id=F843CB5729575D76FF1FFBB2179BDCF52C0C6387
-192.99.246.48:9030 orport=9001 id=CD6B149BED1BB254EF6DFF9D75DDB11E7F8A38A4 ipv6=[2607:5300:100:200::de3]:9002
-192.160.102.164:80 orport=9001 id=823AA81E277F366505545522CEDC2F529CE4DC3F ipv6=[2605:e200:d00c:c01d::1111]:9002
+# Email sent directly to teor
+192.160.102.169:80 orport=9001 id=C0192FF43E777250084175F4E59AC1BA2290CE38 ipv6=[2620:132:300c:c01d::9]:9002 # manipogo
+192.160.102.166:80 orport=9001 id=547DA56F6B88B6C596B3E3086803CDA4F0EF8F21 ipv6=[2620:132:300c:c01d::6]:9002 # chaucer
+192.160.102.170:80 orport=9001 id=557ACEC850F54EEE65839F83CACE2B0825BE811E ipv6=[2620:132:300c:c01d::a]:9002 # ogopogo
+192.160.102.164:80 orport=9001 id=823AA81E277F366505545522CEDC2F529CE4DC3F ipv6=[2620:132:300c:c01d::4]:9002 # snowfall
+192.160.102.165:80 orport=9001 id=C90CA3B7FE01A146B8268D56977DC4A2C024B9EA ipv6=[2620:132:300c:c01d::5]:9002 # cowcat
+192.160.102.168:80 orport=9001 id=F6A358DD367B3282D6EF5824C9D45E1A19C7E815 ipv6=[2620:132:300c:c01d::8]:9002 # prawksi
 
 # Email sent directly to teor, verified using relay contact info
 136.243.214.137:80 orport=443 id=B291D30517D23299AD7CEE3E60DFE60D0E3A4664
@@ -666,8 +626,8 @@
 
 # Email sent directly to teor, verified using relay contact info
 192.87.28.28:9030 orport=9001 id=ED2338CAC2711B3E331392E1ED2831219B794024
-# OK, but same machine as ED2338CAC2711B3E331392E1ED2831219B794024
-#192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7
+# same machine as ED2338CAC2711B3E331392E1ED2831219B794024
+192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7
 
 # https://twitter.com/kosjoli/status/719507270904758272
 85.10.202.87:9030 orport=9001 id=971AFB23C168DCD8EDA17473C1C452B359DE3A5A
@@ -675,20 +635,18 @@
 46.4.111.124:9030 orport=9001 id=D9065F9E57899B3D272AA212317AF61A9B14D204
 
 # Email sent directly to teor, verified using relay contact info
-78.46.164.129:9030 orport=9001 id=52AEA31188331F421B2EDB494DB65CD181E5B257
-
-# Email sent directly to teor, verified using relay contact info
 185.100.85.61:80 orport=443 id=025B66CEBC070FCB0519D206CF0CF4965C20C96E
 
 # Email sent directly to teor, verified using relay contact info
 108.166.168.158:80 orport=443 id=CDAB3AE06A8C9C6BF817B3B0F1877A4B91465699
 
 # Email sent directly to teor, verified using relay contact info
-91.219.236.222:80 orport=443 id=EC413181CEB1C8EDC17608BBB177CD5FD8535E99
+91.219.236.222:80 orport=443 id=20704E7DD51501DC303FA51B738D7B7E61397CF6
 
 # Email sent directly to teor, verified using relay contact info
 185.14.185.240:9030 orport=443 id=D62FB817B0288085FAC38A6DC8B36DCD85B70260
 192.34.63.137:9030 orport=443 id=ABCB4965F1FEE193602B50A365425105C889D3F8
+128.199.197.16:9030 orport=443 id=DEE5298B3BA18CDE651421CD2DCB34A4A69F224D
 
 # Email sent directly to teor, verified using relay contact info
 185.13.38.75:9030 orport=9001 id=D2A1703758A0FBBA026988B92C2F88BAB59F9361
@@ -719,13 +677,14 @@
 166.70.207.2:9030 orport=9001 id=E3DB2E354B883B59E8DC56B3E7A353DDFD457812
 
 # Emails sent directly to teor, verified using relay contact info
-#69.162.139.9:9030 orport=9001 id=4791FC0692EAB60DF2BCCAFF940B95B74E7654F6 ipv6=[2607:f128:40:1212::45a2:8b09]:9001
+69.162.139.9:9030 orport=9001 id=4791FC0692EAB60DF2BCCAFF940B95B74E7654F6 ipv6=[2607:f128:40:1212::45a2:8b09]:9001
 
 # Email sent directly to teor, verified using relay contact info
 213.239.217.18:1338 orport=1337 id=C37BC191AC389179674578C3E6944E925FE186C2 ipv6=[2a01:4f8:a0:746a:101:1:1:1]:1337
 
 # Email sent directly to teor, verified using relay contact info
-188.40.128.246:9030 orport=9001 id=AD19490C7DBB26D3A68EFC824F67E69B0A96E601
+# Assume details update is permanent
+188.40.128.246:9030 orport=9001 id=AD19490C7DBB26D3A68EFC824F67E69B0A96E601 ipv6=[2a01:4f8:221:1ac1:dead:beef:7005:9001]:9001 # sputnik
 
 # Email sent directly to teor, verified using relay contact info
 88.198.253.13:9030 orport=9001 id=DF924196D69AAE3C00C115A9CCDF7BB62A175310 ipv6=[2a01:4f8:11a:b1f::2]:9001
@@ -743,20 +702,44 @@
 # Email sent directly to teor, verified using relay contact info
 107.170.101.39:9030 orport=443 id=30973217E70AF00EBE51797FF6D9AA720A902EAA
 
-# Email sent directly to teor, verified using relay contact info
-192.99.212.139:80 orport=443 id=F10BDE279AE71515DDCCCC61DC19AC8765F8A3CC
-
-# Email sent directly to teor, verified using relay contact info
-163.172.35.249:80 orport=443 id=C08DE49658E5B3CFC6F2A952B453C4B608C9A16A
-163.172.35.247:80 orport=443 id=71AB4726D830FAE776D74AEF790CF04D8E0151B4
-163.172.13.124:80 orport=443 id=B771AA877687F88E6F1CA5354756DF6C8A7B6B24
+# Email sent directly to teor
+193.70.112.165:80 orport=443 id=F10BDE279AE71515DDCCCC61DC19AC8765F8A3CC # ParkBenchInd001
+
+# Email sent directly to teor
+185.220.101.6:10006 orport=20006 id=C08DE49658E5B3CFC6F2A952B453C4B608C9A16A # niftyvolcanorabbit
+185.220.101.13:10013 orport=20013 id=71AB4726D830FAE776D74AEF790CF04D8E0151B4 # niftycottontail
+185.220.101.5:10005 orport=20005 id=1084200B44021D308EA4253F256794671B1D099A # niftyhedgehog
+185.220.101.9:10009 orport=20009 id=14877C6384A9E793F422C8D1DDA447CACA4F7C4B # niftywoodmouse
+185.220.101.8:10008 orport=20008 id=24E91955D969AEA1D80413C64FE106FAE7FD2EA9 # niftymouse
+185.220.101.1:10001 orport=20001 id=28F4F392F8F19E3FBDE09616D9DB8143A1E2DDD3 # niftycottonmouse
+185.220.101.21:10021 orport=20021 id=348B89013EDDD99E4755951D1EC284D9FED71226 # niftysquirrel
+185.220.101.10:10010 orport=20010 id=4031460683AE9E0512D3620C2758D98758AC6C93 # niftyeuropeanrabbit
+185.220.101.34:10034 orport=20034 id=47C42E2094EE482E7C9B586B10BABFB67557030B # niftyquokka
+185.220.101.18:10018 orport=20018 id=5D5006E4992F2F97DF4F8B926C3688870EB52BD8 # niftyplagiodontia
+185.220.101.28:10028 orport=20028 id=609E598FB6A00BCF7872906B602B705B64541C50 # niftychipmunk
+185.220.101.20:10020 orport=20020 id=619349D82424C601CAEB94161A4CF778993DAEE7 # niftytucotuco
+185.220.101.17:10017 orport=20017 id=644DECC5A1879C0FE23DE927DD7049F58BBDF349 # niftyhutia
+185.220.101.0:10000 orport=20000 id=6E94866ED8CA098BACDFD36D4E8E2B459B8A734E # niftybeaver
+185.220.101.30:10030 orport=20030 id=71CFDEB4D9E00CCC3E31EC4E8A29E109BBC1FB36 # niftypedetidae
+185.220.101.29:10029 orport=20029 id=7DC52AE6667A30536BA2383CD102CFC24F20AD71 # niftyllipika
+185.220.101.41:10041 orport=20041 id=7E281CD2C315C4F7A84BC7C8721C3BC974DDBFA3 # niftyporcupine
+185.220.101.25:10025 orport=20025 id=8EE0534532EA31AA5172B1892F53B2F25C76EB02 # niftyjerboa
+185.220.101.33:10033 orport=20033 id=906DCB390F2BA987AE258D745E60BAAABAD31DE8 # niftyquokka
+185.220.101.26:10026 orport=20026 id=92A6085EABAADD928B6F8E871540A1A41CBC08BA # niftypedetes
+185.220.101.40:10040 orport=20040 id=9A857254F379194D1CD76F4A79A20D2051BEDA3F # niftynutria
+185.220.101.42:10042 orport=20042 id=9B816A5B3EB20B8E4E9B9D1FBA299BD3F40F0320 # niftypygmyjerboa
+185.220.101.2:10002 orport=20002 id=B740BCECC4A9569232CDD45C0E1330BA0D030D33 # niftybunny
+185.220.101.32:10032 orport=20032 id=B771AA877687F88E6F1CA5354756DF6C8A7B6B24 # niftypika
+185.220.101.12:10012 orport=20012 id=BC82F2190DE2E97DE65F49B4A95572374BDC0789 # niftycapybara
+185.220.101.22:10022 orport=20022 id=CA37CD46799449D83B6B98B8C22C649906307888 # niftyjackrabbit
+185.220.101.4:10004 orport=20004 id=CDA2EA326E2272C57ACB26773D7252C211795B78 # niftygerbil
+185.220.101.14:10014 orport=20014 id=E7EBA5D8A4E09684D11A1DF24F75362817333768 # niftyhare
+185.220.101.16:10016 orport=20016 id=EC1997D51892E4607C68E800549A1E7E4694005A # niftyguineapig
+185.220.101.24:10024 orport=20024 id=FDA70EC93DB01E3CB418CB6943B0C68464B18B4C # niftyrat
 
 # Email sent directly to teor, verified using relay contact info
 64.113.32.29:9030 orport=9001 id=30C19B81981F450C402306E2E7CFB6C3F79CB6B2
 
-# Email sent directly to teor, verified using relay contact info
-212.51.156.193:995 orport=110 id=32E7AAF1F602814D699BEF6761AD03E387758D49 ipv6=[2a02:168:4a01::49]:110
-
 # Emails sent directly to teor, verified using relay contact info
 51.254.101.242:9002 orport=9001 id=4CC9CC9195EC38645B699A33307058624F660CCF
 
@@ -768,3 +751,247 @@
 
 # Email sent directly to teor, verified using relay contact info
 62.216.5.120:9030 orport=9001 id=D032D4D617140D6B828FC7C4334860E45E414FBE
+
+# Email sent directly to teor, verified using relay contact info
+51.254.136.195:80 orport=443 id=7BB70F8585DFC27E75D692970C0EEB0F22983A63
+
+# Email sent directly to teor, verified using relay contact info
+163.172.13.165:9030 orport=9001 id=33DA0CAB7C27812EFF2E22C9705630A54D101FEB ipv6=[2001:bc8:38cb:201::8]:9001
+
+# Email sent directly to teor, verified using relay contact info
+5.196.88.122:9030 orport=9001 id=0C2C599AFCB26F5CFC2C7592435924C1D63D9484 ipv6=[2001:41d0:a:fb7a::1]:9001
+
+# Email sent directly to teor, verified using relay contact info
+5.9.158.75:80 orport=443 id=1AF72E8906E6C49481A791A6F8F84F8DFEBBB2BA ipv6=[2a01:4f8:190:514a::2]:443
+
+# Email sent directly to teor, verified using relay contact info
+46.101.169.151:9030 orport=9001 id=D760C5B436E42F93D77EF2D969157EEA14F9B39C ipv6=[2a03:b0c0:3:d0::74f:a001]:9001
+
+# Email sent directly to teor, verified using relay contact info
+199.249.223.81:80 orport=443 id=F7447E99EB5CBD4D5EB913EE0E35AC642B5C1EF3
+199.249.223.79:80 orport=443 id=D33292FEDE24DD40F2385283E55C87F85C0943B6
+199.249.223.78:80 orport=443 id=EC15DB62D9101481F364DE52EB8313C838BDDC29
+199.249.223.77:80 orport=443 id=CC4A3AE960E3617F49BF9887B79186C14CBA6813
+199.249.223.76:80 orport=443 id=43209F6D50C657A56FE79AF01CA69F9EF19BD338
+199.249.223.75:80 orport=443 id=60D3667F56AEC5C69CF7E8F557DB21DDF6C36060
+199.249.223.74:80 orport=443 id=5F4CD12099AF20FAF9ADFDCEC65316A376D0201C
+199.249.223.73:80 orport=443 id=5649CB2158DA94FB747415F26628BEC07FA57616
+199.249.223.72:80 orport=443 id=B028707969D8ED84E6DEA597A884F78AAD471971
+199.249.223.71:80 orport=443 id=B6320E44A230302C7BF9319E67597A9B87882241
+199.249.223.60:80 orport=443 id=B7047FBDE9C53C39011CA84E5CB2A8E3543066D0
+199.249.223.61:80 orport=443 id=40E7D6CE5085E4CDDA31D51A29D1457EB53F12AD
+199.249.223.62:80 orport=443 id=0077BCBA7244DB3E6A5ED2746E86170066684887
+199.249.223.63:80 orport=443 id=1DB25DF59DAA01B5BE3D3CEB8AFED115940EBE8B
+199.249.223.64:80 orport=443 id=9F2856F6D2B89AD4EF6D5723FAB167DB5A53519A
+199.249.223.65:80 orport=443 id=9D21F034C3BFF4E7737D08CF775DC1745706801F
+199.249.223.66:80 orport=443 id=C5A53BCC174EF8FD0DCB223E4AA929FA557DEDB2
+199.249.223.67:80 orport=443 id=155D6F57425F16C0624D77777641E4EB1B47C6F0
+199.249.223.68:80 orport=443 id=DF20497E487A979995D851A5BCEC313DF7E5BC51
+199.249.223.69:80 orport=443 id=7FA8E7E44F1392A4E40FFC3B69DB3B00091B7FD3
+
+# https://lists.torproject.org/pipermail/tor-relays/2016-December/011114.html
+86.105.212.130:9030 orport=443 id=9C900A7F6F5DD034CFFD192DAEC9CCAA813DB022
+
+# Email sent directly to teor, verified using relay contact info
+178.33.183.251:80 orport=443 id=DD823AFB415380A802DCAEB9461AE637604107FB ipv6=[2001:41d0:2:a683::251]:443
+
+# Email sent directly to teor, verified using relay contact info
+31.185.104.19:80 orport=443 id=9EAD5B2D3DBD96DBC80DCE423B0C345E920A758D
+# same machine as 9EAD5B2D3DBD96DBC80DCE423B0C345E920A758D
+31.185.104.20:80 orport=443 id=ADB2C26629643DBB9F8FE0096E7D16F9414B4F8D
+31.185.104.21:80 orport=443 id=C2AAB088555850FC434E68943F551072042B85F1
+31.185.104.22:80 orport=443 id=5BA3A52760A0EABF7E7C3ED3048A77328FF0F148
+
+# Email sent directly to teor, verified using relay contact info
+185.34.60.114:80 orport=443 id=7F7A695DF6F2B8640A70B6ADD01105BC2EBC5135
+
+# https://lists.torproject.org/pipermail/tor-relays/2017-December/013939.html
+94.142.242.84:80 orport=443 id=AA0D167E03E298F9A8CD50F448B81FBD7FA80D56 ipv6=[2a02:898:24:84::1]:443 # rejozenger
+
+# Email sent directly to teor, verified using relay contact info
+185.129.62.62:9030 orport=9001 id=ACDD9E85A05B127BA010466C13C8C47212E8A38F ipv6=[2a06:d380:0:3700::62]:9001
+
+# Email sent directly to teor, verified using relay contact info
+# The e84 part of the IPv6 address does not have a leading 0 in the consensus
+81.30.158.213:9030 orport=9001 id=789EA6C9AE9ADDD8760903171CFA9AC5741B0C70 ipv6=[2001:4ba0:cafe:e84::1]:9001
+
+# https://lists.torproject.org/pipermail/tor-relays/2016-December/011209.html
+5.9.159.14:9030 orport=9001 id=0F100F60C7A63BED90216052324D29B08CFCF797
+
+# Email sent directly to teor, verified using relay contact info
+45.62.255.25:80 orport=443 id=3473ED788D9E63361D1572B7E82EC54338953D2A
+
+# Email sent directly to teor, verified using relay contact info
+217.79.179.177:9030 orport=9001 id=3E53D3979DB07EFD736661C934A1DED14127B684 ipv6=[2001:4ba0:fff9:131:6c4f::90d3]:9001
+
+# Email sent directly to teor, verified using relay contact info
+212.47.244.38:8080 orport=443 id=E81EF60A73B3809F8964F73766B01BAA0A171E20
+163.172.157.213:8080 orport=443 id=4623A9EC53BFD83155929E56D6F7B55B5E718C24
+163.172.139.104:8080 orport=443 id=68F175CCABE727AA2D2309BCD8789499CEE36ED7
+
+# Email sent directly to teor, verified using relay contact info
+163.172.223.200:80 orport=443 id=998BF3ED7F70E33D1C307247B9626D9E7573C438
+195.154.122.54:80 orport=443 id=64E99CB34C595A02A3165484BD1215E7389322C6
+
+# Email sent directly to teor, verified using relay contact info
+185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883
+185.100.85.101:9030 orport=9001 id=4061C553CA88021B8302F0814365070AAE617270
+31.171.155.108:9030 orport=9001 id=D3E5EDDBE5159388704D6785BE51930AAFACEC6F
+
+# Email sent directly to teor, verified using relay contact info
+89.163.247.43:9030 orport=9001 id=BC7ACFAC04854C77167C7D66B7E471314ED8C410 ipv6=[2001:4ba0:fff7:25::5]:9001
+
+# Email sent directly to teor, verified using relay contact info
+95.85.8.226:80 orport=443 id=1211AC1BBB8A1AF7CBA86BCE8689AA3146B86423
+
+# Email sent directly to teor, verified using relay contact info
+85.214.151.72:9030 orport=9001 id=722D365140C8C52DBB3C9FF6986E3CEFFE2BA812
+
+# email sent directly to teor
+72.52.75.27:9030 orport=9001 id=8567AD0A6369ED08527A8A8533A5162AC00F7678 # piecoopdotnet
+
+# Email sent directly to teor, verified using relay contact info
+5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11
+5.9.159.14:9030 orport=9001 id=0F100F60C7A63BED90216052324D29B08CFCF797
+
+# Email sent directly to teor, verified using relay contact info
+# Assume details update is permanent
+5.9.147.226:9030 orport=9001 id=B0553175AADB0501E5A61FC61CEA3970BE130FF2 ipv6=[2a01:4f8:190:30e1::2]:9001 # zwiubel
+
+# https://trac.torproject.org/projects/tor/ticket/22527#comment:1
+199.184.246.250:80 orport=443 id=1F6ABD086F40B890A33C93CC4606EE68B31C9556 ipv6=[2620:124:1009:1::171]:443
+
+# https://trac.torproject.org/projects/tor/ticket/24695
+163.172.53.84:143 orport=21 id=1C90D3AEADFF3BCD079810632C8B85637924A58E ipv6=[2001:bc8:24f8::]:21 # Multivac
+
+# Email sent directly to teor
+54.36.237.163:80 orport=443 id=DB2682153AC0CCAECD2BD1E9EBE99C6815807A1E # GermanCraft2
+
+# Email sent directly to teor
+62.138.7.171:9030 orport=9001 id=9844B981A80B3E4B50897098E2D65167E6AEF127 # 0x3d004
+62.138.7.171:8030 orport=8001 id=9285B22F7953D7874604EEE2B470609AD81C74E9 # 0x3d005
+91.121.23.100:9030 orport=9001 id=3711E80B5B04494C971FB0459D4209AB7F2EA799 # 0x3d002
+91.121.23.100:8030 orport=8001 id=CFBBA0D858F02E40B1432A65F6D13C9BDFE7A46B # 0x3d001
+51.15.13.245:9030 orport=9001 id=CED527EAC230E7B56E5B363F839671829C3BA01B # 0x3d006
+51.15.13.245:8030 orport=8001 id=8EBB8D1CF48FE2AB95C451DA8F10DB6235F40F8A # 0x3d007
+
+# Email sent directly to teor
+104.192.5.248:9030 orport=9001 id=BF735F669481EE1CCC348F0731551C933D1E2278 # Freeway11
+
+# Email sent directly to teor
+# https://lists.torproject.org/pipermail/tor-relays/2017-December/013961.html
+178.17.174.14:9030 orport=9001 id=B06F093A3D4DFAD3E923F4F28A74901BD4F74EB1 # TorExitMoldova
+178.17.170.156:9030 orport=9001 id=41C59606AFE1D1AA6EC6EF6719690B856F0B6587 # TorExitMoldova2
+
+# Email sent directly to teor
+163.172.221.44:59030 orport=59001 id=164604F5C86FC8CC9C0288BD9C02311958427597 # altego
+
+# Email sent directly to teor
+46.38.237.221:9030 orport=9001 id=D30E9D4D639068611D6D96861C95C2099140B805 # mine
+
+# https://lists.torproject.org/pipermail/tor-relays/2017-December/013911.html
+# https://lists.torproject.org/pipermail/tor-relays/2017-December/013912.html
+199.249.223.62:80 orport=443 id=0077BCBA7244DB3E6A5ED2746E86170066684887 # Quintex13
+199.249.224.45:80 orport=443 id=041646640AB306EA74B001966E86169B04CC88D2 # QuintexAirVPN26
+199.249.223.67:80 orport=443 id=155D6F57425F16C0624D77777641E4EB1B47C6F0 # Quintex18
+199.249.223.45:80 orport=443 id=1AE949967F82BBE7534A3D6BA77A7EBE1CED4369 # Quintex36
+199.249.223.63:80 orport=443 id=1DB25DF59DAA01B5BE3D3CEB8AFED115940EBE8B # Quintex14
+199.249.224.63:80 orport=443 id=1E5136DDC52FAE1219208F0A6BADB0BA62587EE6 # Quintex43
+199.249.224.46:80 orport=443 id=2ED4D25766973713EB8C56A290BF07E06B85BF12 # QuintexAirVPN27
+199.249.223.42:80 orport=443 id=3687FEC7E73F61AC66F7AE251E7DEE6BBD8C0252 # Quintex33
+199.249.223.49:80 orport=443 id=36D68478366CB8627866757EBCE7FB3C17FC1CB8 # Quintex40
+199.249.224.49:80 orport=443 id=3CA0D15567024D2E0B557DC0CF3E962B37999A79 # QuintexAirVPN30
+199.249.223.61:80 orport=443 id=40E7D6CE5085E4CDDA31D51A29D1457EB53F12AD # Quintex12
+199.249.223.76:80 orport=443 id=43209F6D50C657A56FE79AF01CA69F9EF19BD338 # QuintexAirVPN5
+199.249.224.41:80 orport=443 id=54A4820B46E65509BF3E2B892E66930A41759DE9 # QuintexAirVPN22
+199.249.223.73:80 orport=443 id=5649CB2158DA94FB747415F26628BEC07FA57616 # QuintexAirVPN8
+199.249.223.74:80 orport=443 id=5F4CD12099AF20FAF9ADFDCEC65316A376D0201C # QuintexAirVPN7
+199.249.223.75:80 orport=443 id=60D3667F56AEC5C69CF7E8F557DB21DDF6C36060 # QuintexAirVPN6
+199.249.223.46:80 orport=443 id=66E19E8C4773086F669A1E06A3F8C23B6C079129 # Quintex37
+199.249.224.65:80 orport=443 id=764BF8A03868F84C8F323C1A676AA254B80DC3BF # Quintex45
+199.249.223.48:80 orport=443 id=7A3DD280EA4CD4DD16EF8C67B93D9BDE184D1A81 # Quintex39
+199.249.224.68:80 orport=443 id=7E6E9A6FDDB8DC7C92F0CFCC3CBE76C29F061799 # Quintex48
+199.249.223.69:80 orport=443 id=7FA8E7E44F1392A4E40FFC3B69DB3B00091B7FD3 # Quintex20
+199.249.223.44:80 orport=443 id=8B80169BEF71450FC4069A190853523B7AEA45E1 # Quintex35
+199.249.224.60:80 orport=443 id=9314BD9503B9014261A65C221D77E57389DBCCC1 # Quintex50
+199.249.224.40:80 orport=443 id=9C1E7D92115D431385B8CAEA6A7C15FB89CE236B # QuintexAirVPN21
+199.249.223.65:80 orport=443 id=9D21F034C3BFF4E7737D08CF775DC1745706801F # Quintex16
+199.249.224.67:80 orport=443 id=9E2D7C6981269404AA1970B53891701A20424EF8 # Quintex47
+199.249.223.64:80 orport=443 id=9F2856F6D2B89AD4EF6D5723FAB167DB5A53519A # Quintex15
+199.249.224.48:80 orport=443 id=A0DB820FEC87C0405F7BF05DEE5E4ADED2BB9904 # QuintexAirVPN29
+199.249.224.64:80 orport=443 id=A4A393FEF48640961AACE92D041934B55348CEF9 # Quintex44
+199.249.223.72:80 orport=443 id=B028707969D8ED84E6DEA597A884F78AAD471971 # QuintexAirVPN9
+199.249.223.40:80 orport=443 id=B0CD9F9B5B60651ADC5919C0F1EAA87DBA1D9249 # Quintex31
+199.249.224.61:80 orport=443 id=B2197C23A4FF5D1C49EE45BA7688BA8BCCD89A0B # Quintex41
+199.249.223.71:80 orport=443 id=B6320E44A230302C7BF9319E67597A9B87882241 # QuintexAirVPN10
+199.249.223.60:80 orport=443 id=B7047FBDE9C53C39011CA84E5CB2A8E3543066D0 # Quintex11
+199.249.224.66:80 orport=443 id=C78AFFEEE320EA0F860961763E613FD2FAC855F5 # Quintex46
+199.249.224.44:80 orport=443 id=CB7C0D841FE376EF43F7845FF201B0290C0A239E # QuintexAirVPN25
+199.249.223.47:80 orport=443 id=CC14C97F1D23EE97766828FC8ED8582E21E11665 # Quintex38
+199.249.223.77:80 orport=443 id=CC4A3AE960E3617F49BF9887B79186C14CBA6813 # QuintexAirVPN4
+199.249.223.41:80 orport=443 id=D25210CE07C49F2A4F2BC7A506EB0F5EA7F5E2C2 # Quintex32
+199.249.223.79:80 orport=443 id=D33292FEDE24DD40F2385283E55C87F85C0943B6 # QuintexAirVPN2
+199.249.224.47:80 orport=443 id=D6FF2697CEA5C0C7DA84797C2E71163814FC2466 # QuintexAirVPN28
+199.249.223.68:80 orport=443 id=DF20497E487A979995D851A5BCEC313DF7E5BC51 # Quintex19
+199.249.223.43:80 orport=443 id=E480D577F58E782A5BC4FA6F49A6650E9389302F # Quintex34
+199.249.224.69:80 orport=443 id=EABC2DD0D47B5DB11F2D37EB3C60C2A4D91C10F2 # Quintex49
+199.249.223.78:80 orport=443 id=EC15DB62D9101481F364DE52EB8313C838BDDC29 # QuintexAirVPN3
+199.249.224.42:80 orport=443 id=F21DE9C7DE31601D9716781E17E24380887883D1 # QuintexAirVPN23
+199.249.223.81:80 orport=443 id=F7447E99EB5CBD4D5EB913EE0E35AC642B5C1EF3 # QuintexAirVPN1
+199.249.224.43:80 orport=443 id=FDD700C791CC6BB0AC1C2099A82CBC367AD4B764 # QuintexAirVPN24
+199.249.224.62:80 orport=443 id=FE00A3A835680E67FBBC895A724E2657BB253E97 # Quintex42
+199.249.223.66:80 orport=443 id=C5A53BCC174EF8FD0DCB223E4AA929FA557DEDB2 # Quintex17
+
+# https://lists.torproject.org/pipermail/tor-relays/2017-December/013914.html
+5.196.23.64:9030 orport=9001 id=775B0FAFDE71AADC23FFC8782B7BEB1D5A92733E # Aerodynamik01
+217.182.75.181:9030 orport=9001 id=EFEACD781604EB80FBC025EDEDEA2D523AEAAA2F # Aerodynamik02
+193.70.43.76:9030 orport=9001 id=484A10BA2B8D48A5F0216674C8DD50EF27BC32F3 # Aerodynamik03
+149.56.141.138:9030 orport=9001 id=1938EBACBB1A7BFA888D9623C90061130E63BB3F # Aerodynamik04
+
+# https://lists.torproject.org/pipermail/tor-relays/2017-December/013917.html
+104.200.20.46:80 orport=9001 id=78E2BE744A53631B4AAB781468E94C52AB73968B # bynumlawtor
+
+# https://lists.torproject.org/pipermail/tor-relays/2017-December/013929.html
+139.99.130.178:80 orport=443 id=867B95CACD64653FEEC4D2CEFC5C49B4620307A7 # coffswifi2
+
+# https://lists.torproject.org/pipermail/tor-relays/2017-December/013946.html
+172.98.193.43:80 orport=443 id=5E56738E7F97AA81DEEF59AF28494293DFBFCCDF # Backplane
+
+# Email sent directly to teor
+62.210.254.132:80 orport=443 id=8456DFA94161CDD99E480C2A2992C366C6564410 # turingmachine
+
+# Email sent directly to teor
+80.127.117.180:80 orport=443 id=328E54981C6DDD7D89B89E418724A4A7881E3192 ipv6=[2001:985:e77:10::4]:443 # sjc01
+
+# https://lists.torproject.org/pipermail/tor-relays/2017-December/013960.html
+51.15.205.214:9030 orport=9001 id=8B6556601612F1E2AFCE2A12FFFAF8482A76DD1F ipv6=[2001:bc8:4400:2500::5:b07]:9001 # titania1
+51.15.205.214:9031 orport=9002 id=5E363D72488276160D062DDD2DFA25CFEBAF5EA9 ipv6=[2001:bc8:4400:2500::5:b07]:9002 # titania2
+
+# Email sent directly to teor
+185.129.249.124:9030 orport=9001 id=1FA8F638298645BE58AC905276680889CB795A94 # treadstone
+
+# https://lists.torproject.org/pipermail/tor-relays/2017-December/014000.html
+24.117.231.229:34175 orport=45117 id=CE24412AD69444954B4015E293AE53DDDAFEA3D6 # Anosognosia
+
+# https://lists.torproject.org/pipermail/tor-relays/2018-January/014012.html
+128.31.0.13:80 orport=443 id=A53C46F5B157DD83366D45A8E99A244934A14C46 # csailmitexit
+
+# Email sent directly to teor
+82.247.103.117:110 orport=995 id=C9B3C1661A9577BA24C1C2C6123918921A495509 # Casper01
+109.238.2.79:110 orport=995 id=7520892E3DD133D0B0464D01A158B54B8E2A8B75 # Casper02
+51.15.179.153:110 orport=995 id=BB60F5BA113A0B8B44B7B37DE3567FE561E92F78 # Casper04
+
+# Email sent directly to teor
+80.127.107.179:80 orport=443 id=BC6B2E2F62ACC5EDECBABE64DA1E48F84DD98B78 ipv6=[2001:981:4a22:c::6]:443 # TVISION02
+
+# https://lists.torproject.org/pipermail/tor-relays/2018-January/014020.html
+37.120.174.249:80 orport=443 id=11DF0017A43AF1F08825CD5D973297F81AB00FF3 ipv6=[2a03:4000:6:724c:df98:15f9:b34d:443]:443 # gGDHjdcC6zAlM8k08lX
+
+# These fallbacks opted-in in previous releases, then changed their details,
+# and so we blacklisted them. Now we want to whitelist changes.
+# Assume details update is permanent
+85.230.184.93:9030 orport=443 id=855BC2DABE24C861CD887DB9B2E950424B49FC34 # Logforme
+176.31.180.157:143 orport=22 id=E781F4EC69671B3F1864AE2753E0890351506329 ipv6=[2001:41d0:8:eb9d::1]:22 # armbrust
+
+# https://lists.torproject.org/pipermail/tor-relays/2018-January/014024.html
+82.161.212.209:9030 orport=9001 id=4E8CE6F5651E7342C1E7E5ED031E82078134FB0D ipv6=[2001:980:d7ed:1:ff:b0ff:fe00:d0b]:9001 # ymkeo
diff --git a/scripts/maint/format_changelog.py b/scripts/maint/format_changelog.py
index e909fc550a..8dce4b6e51 100755
--- a/scripts/maint/format_changelog.py
+++ b/scripts/maint/format_changelog.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright (c) 2014-2015, The Tor Project, Inc.
+# Copyright (c) 2014-2019, The Tor Project, Inc.
 # See LICENSE for licensing information
 #
 # This script reformats a section of the changelog to wrap everything to
@@ -205,6 +205,8 @@ def head_score(s):
         score = -300
     elif lw.startswith("deprecated version"):
         score = -200
+    elif lw.startswith("directory auth"):
+        score = -150
     elif (('new' in lw and 'requirement' in lw) or
           ('new' in lw and 'dependenc' in lw) or
           ('build' in lw and 'requirement' in lw) or
diff --git a/scripts/maint/generateFallbackDirLine.py b/scripts/maint/generateFallbackDirLine.py
new file mode 100755
index 0000000000..b856c938bf
--- /dev/null
+++ b/scripts/maint/generateFallbackDirLine.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+# Generate a fallback directory whitelist/blacklist line for every fingerprint
+# passed as an argument.
+#
+# Usage:
+# generateFallbackDirLine.py fingerprint ...
+
+import sys
+import urllib2
+
+import stem.descriptor.remote
+import stem.util.tor_tools
+
+if len(sys.argv) <= 1:
+  print('Usage: %s fingerprint ...' % sys.argv[0])
+  sys.exit(1)
+
+for fingerprint in sys.argv[1:]:
+  if not stem.util.tor_tools.is_valid_fingerprint(fingerprint):
+    print("'%s' isn't a valid relay fingerprint" % fingerprint)
+    sys.exit(1)
+
+  try:
+    desc = stem.descriptor.remote.get_server_descriptors(fingerprint).run()[0]
+  except urllib2.HTTPError as exc:
+    if exc.code == 404:
+      print('# %s not found in recent descriptors' % fingerprint)
+      continue
+    else:
+      raise
+
+  if not desc.dir_port:
+    print("# %s needs a DirPort" % fingerprint)
+  else:
+    ipv6_addresses = [(address, port) for address, port, is_ipv6 in desc.or_addresses if is_ipv6]
+    ipv6_field = ' ipv6=[%s]:%s' % ipv6_addresses[0] if ipv6_addresses else ''
+    print('%s:%s orport=%s id=%s%s # %s' % (desc.address, desc.dir_port, desc.or_port, fingerprint, ipv6_field, desc.nickname))
diff --git a/scripts/maint/generate_callgraph.sh b/scripts/maint/generate_callgraph.sh
deleted file mode 100755
index c6b33c0aea..0000000000
--- a/scripts/maint/generate_callgraph.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/sh
-
-C_FILES=`echo src/common/*.c src/or/*.c src/tools/*.c`
-CFLAGS="-Isrc/ext/trunnel -Isrc/trunnel -I. -Isrc/ext -Isrc/common -DLOCALSTATEDIR=\"\" -DSHARE_DATADIR=\"\" -Dinline="
-
-mkdir -p callgraph/src/common
-mkdir -p callgraph/src/or
-mkdir -p callgraph/src/tools
-
-for fn in $C_FILES; do
-  echo $fn
-  clang $CFLAGS  -S -emit-llvm -fno-inline -o - $fn  | \
-    opt -analyze -print-callgraph >/dev/null 2> "callgraph/${fn}allgraph"
-done
diff --git a/scripts/maint/lintChanges.py b/scripts/maint/lintChanges.py
index b63a4eb3a1..d5b8fcae5c 100755
--- a/scripts/maint/lintChanges.py
+++ b/scripts/maint/lintChanges.py
@@ -7,7 +7,7 @@ import re
 import os
 
 
-KNOWN_GROUPS=set([
+KNOWN_GROUPS = set([
     "Minor bugfix",
     "Minor bugfixes",
     "Major bugfix",
@@ -20,7 +20,20 @@ KNOWN_GROUPS=set([
     "Testing",
     "Documentation",
     "Code simplification and refactoring",
-    "Removed features"])
+    "Removed features",
+    "Deprecated features",
+    "Directory authority changes"])
+
+NEEDS_SUBCATEGORIES = set([
+    "Minor bugfix",
+    "Minor bugfixes",
+    "Major bugfix",
+    "Major bugfixes",
+    "Minor feature",
+    "Minor features",
+    "Major feature",
+    "Major features",
+    ])
 
 def lintfile(fname):
     have_warned = []
@@ -43,17 +56,13 @@ def lintfile(fname):
     if bugnum and bugnum not in contents:
         warn("bug number {} does not appear".format(bugnum))
 
-    lines = contents.split("\n")
-
     m = re.match(r'^[ ]{2}o ([^\(:]*)([^:]*):', contents)
     if not m:
-        warn("header not in format expected")
+        warn("Header not in format expected. ('  o Foo:' or '  o Foo (Bar):')")
     elif m.group(1).strip() not in KNOWN_GROUPS:
-        warn("Weird header: %r"%m.group(1))
-    elif ( ("bugfix" in m.group(1) or "feature" in m.group(1)) and
-           ("Removed" not in m.group(1)) and
-           '(' not in m.group(2)):
-        warn("Missing subcategory on %s"%m.group(1))
+        warn("Unrecognized header: %r" % m.group(1))
+    elif (m.group(1) in NEEDS_SUBCATEGORIES and '(' not in m.group(2)):
+        warn("Missing subcategory on %r" % m.group(1))
 
     if m:
         isBug = ("bug" in m.group(1).lower() or "fix" in m.group(1).lower())
@@ -63,23 +72,46 @@ def lintfile(fname):
     contents = " ".join(contents.split())
 
     if re.search(r'\#\d{2,}', contents):
-        warn("don't use a # before ticket numbers")
+        warn("Don't use a # before ticket numbers. ('bug 1234' not '#1234')")
 
     if isBug and not re.search(r'(\d+)', contents):
-        warn("bugfix does not mention a number")
-    elif isBug and not re.search(r'Fixes ([a-z ]*)bug (\d+)', contents):
-        warn("bugfix does not say 'Fixes bug XXX'")
+        warn("Ticket marked as bugfix, but does not mention a number.")
+    elif isBug and not re.search(r'Fixes ([a-z ]*)bugs? (\d+)', contents):
+        warn("Ticket marked as bugfix, but does not say 'Fixes bug XXX'")
 
     if re.search(r'[bB]ug (\d+)', contents):
         if not re.search(r'[Bb]ugfix on ', contents):
-            warn("bugfix does not say 'bugfix on X.Y.Z'")
-        elif not re.search('[fF]ixes ([a-z ]*)bug (\d+); bugfix on ',
+            warn("Bugfix does not say 'bugfix on X.Y.Z'")
+        elif not re.search('[fF]ixes ([a-z ]*)bugs? (\d+)((, \d+)* and \d+)?; bugfix on ',
                            contents):
-            warn("bugfix incant is not semicoloned")
-
+            warn("Bugfix does not say 'Fixes bug X; bugfix on Y'")
+        elif re.search('tor-([0-9]+)', contents):
+            warn("Do not prefix versions with 'tor-'. ('0.1.2', not 'tor-0.1.2'.)")
+
+    return have_warned != []
+
+def files(args):
+    """Walk through the arguments: for directories, yield their contents;
+       for files, just yield the files. Only search one level deep, because
+       that's how the changes directory is laid out."""
+    for f in args:
+        if os.path.isdir(f):
+            for item in os.listdir(f):
+                if item.startswith("."): #ignore dotfiles
+                    continue
+                yield os.path.join(f, item)
+        else:
+            yield f
 
 if __name__ == '__main__':
-    for fname in sys.argv[1:]:
+    problems = 0
+    for fname in files(sys.argv[1:]):
         if fname.endswith("~"):
             continue
-        lintfile(fname)
+        if lintfile(fname):
+            problems += 1
+
+    if problems:
+        sys.exit(1)
+    else:
+        sys.exit(0)
diff --git a/scripts/maint/lookupFallbackDirContact.py b/scripts/maint/lookupFallbackDirContact.py
new file mode 100755
index 0000000000..14c53d1282
--- /dev/null
+++ b/scripts/maint/lookupFallbackDirContact.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+
+# Lookup fallback directory contact lines for every fingerprint passed as an
+# argument.
+#
+# Usage:
+# lookupFallbackDirContact.py fingerprint ...
+
+import sys
+
+import stem.descriptor.remote as remote
+
+if len(sys.argv) <= 1:
+  print "Usage: {} fingerprint ...".format(sys.argv[0])
+  sys.exit(-1)
+
+# we need descriptors, because the consensus does not have contact infos
+descriptor_list = remote.get_server_descriptors(fingerprints=sys.argv[1:]).run()
+
+descriptor_list_fingerprints = []
+for d in descriptor_list:
+  assert d.fingerprint in sys.argv[1:]
+  descriptor_list_fingerprints.append(d.fingerprint)
+  print "{} {}".format(d.fingerprint, d.contact)
+
+for fingerprint in sys.argv[1:]:
+  if fingerprint not in descriptor_list_fingerprints:
+    print "{} not found in current descriptors".format(fingerprint)
diff --git a/scripts/maint/rectify_include_paths.py b/scripts/maint/rectify_include_paths.py
new file mode 100755
index 0000000000..401fadae6d
--- /dev/null
+++ b/scripts/maint/rectify_include_paths.py
@@ -0,0 +1,60 @@
+#!/usr/bin/python3
+
+import os
+import os.path
+import re
+
+# Find all the include files, map them to their real names.
+
+def exclude(paths, dirnames):
+    for p in paths:
+        if p in dirnames:
+            dirnames.remove(p)
+
+def get_include_map():
+    includes = { }
+
+    for dirpath,dirnames,fnames in os.walk("src"):
+        exclude(["ext", "win32"], dirnames)
+
+        for fname in fnames:
+            if fname.endswith(".h"):
+                assert fname not in includes
+                include = os.path.join(dirpath, fname)
+                assert include.startswith("src/")
+                includes[fname] = include[4:]
+
+    return includes
+
+INCLUDE_PAT = re.compile(r'( *# *include +")([^"]+)(".*)')
+
+def get_base_header_name(hdr):
+    return os.path.split(hdr)[1]
+
+def fix_includes(inp, out, mapping):
+    for line in inp:
+        m = INCLUDE_PAT.match(line)
+        if m:
+            include,hdr,rest = m.groups()
+            basehdr = get_base_header_name(hdr)
+            if basehdr in mapping:
+                out.write('{}{}{}\n'.format(include,mapping[basehdr],rest))
+                continue
+
+        out.write(line)
+
+incs = get_include_map()
+
+for dirpath,dirnames,fnames in os.walk("src"):
+    exclude(["trunnel"], dirnames)
+
+    for fname in fnames:
+        if fname.endswith(".c") or fname.endswith(".h"):
+            fname = os.path.join(dirpath, fname)
+            tmpfile = fname+".tmp"
+            f_in = open(fname, 'r')
+            f_out = open(tmpfile, 'w')
+            fix_includes(f_in, f_out, incs)
+            f_in.close()
+            f_out.close()
+            os.rename(tmpfile, fname)
diff --git a/scripts/maint/redox.py b/scripts/maint/redox.py
index 43f5b6eb16..203cce0107 100755
--- a/scripts/maint/redox.py
+++ b/scripts/maint/redox.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python
 #
-#  Copyright (c) 2008-2015, The Tor Project, Inc.
+#  Copyright (c) 2008-2019, The Tor Project, Inc.
 #  See LICENSE for licensing information.
 #
 # Hi!
@@ -101,7 +101,7 @@ def read():
 
 def findline(lines, lineno, ident):
     """Given a list of all the lines in the file (adjusted so 1-indexing works),
-       a line number that ident is alledgedly on, and ident, I figure out
+       a line number that ident is allegedly on, and ident, I figure out
        the line where ident was really declared."""
     lno = lineno
     for lineno in xrange(lineno, 0, -1):
diff --git a/scripts/maint/run_calltool.sh b/scripts/maint/run_calltool.sh
new file mode 100755
index 0000000000..efb8706fea
--- /dev/null
+++ b/scripts/maint/run_calltool.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# You can find calltool at https://gitweb.torproject.org/user/nickm/calltool.git
+
+set -e
+
+if test "x$CALLTOOL_PATH" != "x"; then
+  PYTHONPATH="${CALLTOOL_PATH}:${PYTHONPATH}"
+  export PYTHONPATH
+fi
+
+mkdir -p callgraph
+
+SUBITEMS="fn_graph fn_invgraph fn_scc fn_scc_weaklinks module_graph module_invgraph module_scc module_scc_weaklinks"
+
+for calculation in $SUBITEMS; do
+  echo "======== $calculation"
+  python -m calltool $calculation > callgraph/$calculation
+done
+
+echo <<EOF > callgraph/README
+This directory holds output from calltool, as run on Tor.  For more
+information about each of these files, see the NOTES and README files in
+the calltool distribution.
+
+You can find calltool at
+    https://gitweb.torproject.org/user/nickm/calltool.git
+EOF
+
diff --git a/scripts/maint/sortChanges.py b/scripts/maint/sortChanges.py
index d6ec0e269d..986b94b025 100755
--- a/scripts/maint/sortChanges.py
+++ b/scripts/maint/sortChanges.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright (c) 2014-2015, The Tor Project, Inc.
+# Copyright (c) 2014-2019, The Tor Project, Inc.
 # See LICENSE for licensing information
 
 """This script sorts a bunch of changes files listed on its command
diff --git a/scripts/maint/updateCopyright.pl b/scripts/maint/updateCopyright.pl
index 8bd6a18210..bd24377d38 100755
--- a/scripts/maint/updateCopyright.pl
+++ b/scripts/maint/updateCopyright.pl
@@ -1,7 +1,7 @@
 #!/usr/bin/perl -i -w -p
 
-$NEWYEAR=2016;
+$NEWYEAR=2018;
 
-s/Copyright(.*) (201[^6]), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/;
+s/Copyright(.*) (201[^8]), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/;
 
 s/Copyright(.*)-(20..), The Tor Project/Copyright$1-${NEWYEAR}, The Tor Project/;
diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py
index 110ecda64c..0ea3992d8f 100755
--- a/scripts/maint/updateFallbackDirs.py
+++ b/scripts/maint/updateFallbackDirs.py
@@ -1,20 +1,25 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
-# Usage: scripts/maint/updateFallbackDirs.py > src/or/fallback_dirs.inc
+# Usage:
+#
+# Regenerate the list:
+# scripts/maint/updateFallbackDirs.py > src/app/config/fallback_dirs.inc 2> fallback_dirs.log
+#
+# Check the existing list:
+# scripts/maint/updateFallbackDirs.py check_existing > fallback_dirs.inc.ok 2> fallback_dirs.log
+# mv fallback_dirs.inc.ok src/app/config/fallback_dirs.inc
 #
 # This script should be run from a stable, reliable network connection,
 # with no other network activity (and not over tor).
 # If this is not possible, please disable:
 # PERFORM_IPV4_DIRPORT_CHECKS and PERFORM_IPV6_DIRPORT_CHECKS
 #
-# Needs dateutil (and potentially other python packages)
-# Needs stem available in your PYTHONPATH, or just ln -s ../stem/stem .
+# Needs dateutil, stem, and potentially other python packages.
 # Optionally uses ipaddress (python 3 builtin) or py2-ipaddress (package)
-# for netblock analysis, in PYTHONPATH, or just
-# ln -s ../py2-ipaddress-3.4.1/ipaddress.py .
+# for netblock analysis.
 #
 # Then read the logs to make sure the fallbacks aren't dominated by a single
-# netblock or port
+# netblock or port.
 
 # Script by weasel, April 2015
 # Portions by gsathya & karsten, 2013
@@ -37,16 +42,13 @@ import dateutil.parser
 # bson_lazy provides bson
 #from bson import json_util
 import copy
+import re
 
-from stem.descriptor.remote import DescriptorDownloader
+from stem.descriptor import DocumentHandler
+from stem.descriptor.remote import get_consensus, get_server_descriptors, MAX_FINGERPRINTS
 
 import logging
-# INFO tells you why each relay was included or excluded
-# WARN tells you about potential misconfigurations and relay detail changes
-logging.basicConfig(level=logging.WARNING)
 logging.root.name = ''
-# INFO tells you about each consensus download attempt
-logging.getLogger('stem').setLevel(logging.WARNING)
 
 HAVE_IPADDRESS = False
 try:
@@ -64,6 +66,17 @@ except ImportError:
 
 ## Top-Level Configuration
 
+# We use semantic versioning: https://semver.org
+# In particular:
+# * major changes include removing a mandatory field, or anything else that
+#   would break an appropriately tolerant parser,
+# * minor changes include adding a field,
+# * patch changes include changing header comments or other unstructured
+#   content
+FALLBACK_FORMAT_VERSION = '2.0.0'
+SECTION_SEPARATOR_BASE = '====='
+SECTION_SEPARATOR_COMMENT = '/* ' + SECTION_SEPARATOR_BASE + ' */'
+
 # Output all candidate fallbacks, or only output selected fallbacks?
 OUTPUT_CANDIDATES = False
 
@@ -80,13 +93,39 @@ PERFORM_IPV4_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else True
 # Don't check ~1000 candidates when OUTPUT_CANDIDATES is True
 PERFORM_IPV6_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else False
 
-# Output fallback name, flags, and ContactInfo in a C comment?
+# Must relays be running now?
+MUST_BE_RUNNING_NOW = (PERFORM_IPV4_DIRPORT_CHECKS
+                       or PERFORM_IPV6_DIRPORT_CHECKS)
+
+# Clients have been using microdesc consensuses by default for a while now
+DOWNLOAD_MICRODESC_CONSENSUS = True
+
+# If a relay delivers an expired consensus, if it expired less than this many
+# seconds ago, we still allow the relay. This should never be less than -90,
+# as all directory mirrors should have downloaded a consensus 90 minutes
+# before it expires. It should never be more than 24 hours, because clients
+# reject consensuses that are older than REASONABLY_LIVE_TIME.
+# For the consensus expiry check to be accurate, the machine running this
+# script needs an accurate clock.
+#
+# Relays on 0.3.0 and later return a 404 when they are about to serve an
+# expired consensus. This makes them fail the download check.
+# We use a tolerance of 0, so that 0.2.x series relays also fail the download
+# check if they serve an expired consensus.
+CONSENSUS_EXPIRY_TOLERANCE = 0
+
+# Output fallback name, flags, bandwidth, and ContactInfo in a C comment?
 OUTPUT_COMMENTS = True if OUTPUT_CANDIDATES else False
 
-# Output matching ContactInfo in fallbacks list or the blacklist?
+# Output matching ContactInfo in fallbacks list?
 # Useful if you're trying to contact operators
 CONTACT_COUNT = True if OUTPUT_CANDIDATES else False
-CONTACT_BLACKLIST_COUNT = True if OUTPUT_CANDIDATES else False
+
+# How the list should be sorted:
+# fingerprint: is useful for stable diffs of fallback lists
+# measured_bandwidth: is useful when pruning the list based on bandwidth
+# contact: is useful for contacting operators once the list has been pruned
+OUTPUT_SORT_FIELD = 'contact' if OUTPUT_CANDIDATES else 'fingerprint'
 
 ## OnionOO Settings
 
@@ -101,44 +140,39 @@ LOCAL_FILES_ONLY = False
 
 # The whitelist contains entries that are included if all attributes match
 # (IPv4, dirport, orport, id, and optionally IPv6 and IPv6 orport)
-# The blacklist contains (partial) entries that are excluded if any
-# sufficiently specific group of attributes matches:
-# IPv4 & DirPort
-# IPv4 & ORPort
-# ID
-# IPv6 & DirPort
-# IPv6 & IPv6 ORPort
-# If neither port is included in the blacklist, the entire IP address is
-# blacklisted.
-
-# What happens to entries in neither list?
+
+# What happens to entries not in whitelist?
 # When True, they are included, when False, they are excluded
 INCLUDE_UNLISTED_ENTRIES = True if OUTPUT_CANDIDATES else False
 
-# If an entry is in both lists, what happens?
-# When True, it is excluded, when False, it is included
-BLACKLIST_EXCLUDES_WHITELIST_ENTRIES = True
-
 WHITELIST_FILE_NAME = 'scripts/maint/fallback.whitelist'
-BLACKLIST_FILE_NAME = 'scripts/maint/fallback.blacklist'
+FALLBACK_FILE_NAME  = 'src/app/config/fallback_dirs.inc'
 
 # The number of bytes we'll read from a filter file before giving up
 MAX_LIST_FILE_SIZE = 1024 * 1024
 
 ## Eligibility Settings
 
-# Reduced due to a bug in tor where a relay submits a 0 DirPort when restarted
-# This causes OnionOO to (correctly) reset its stability timer
-# This issue will be fixed in 0.2.7.7 and 0.2.8.2
-# Until then, the CUTOFFs below ensure a decent level of stability.
-ADDRESS_AND_PORT_STABLE_DAYS = 7
-# What time-weighted-fraction of these flags must FallbackDirs
-# Equal or Exceed?
-CUTOFF_RUNNING = .95
-CUTOFF_V2DIR = .95
-CUTOFF_GUARD = .95
-# What time-weighted-fraction of these flags must FallbackDirs
-# Equal or Fall Under?
+# Require fallbacks to have the same address and port for a set amount of time
+# We used to have this at 1 week, but that caused many fallback failures, which
+# meant that we had to rebuild the list more often. We want fallbacks to be
+# stable for 2 years, so we set it to a few months.
+#
+# If a relay changes address or port, that's it, it's not useful any more,
+# because clients can't find it
+ADDRESS_AND_PORT_STABLE_DAYS = 90
+# We ignore relays that have been down for more than this period
+MAX_DOWNTIME_DAYS = 0 if MUST_BE_RUNNING_NOW else 7
+# FallbackDirs must have a time-weighted-fraction that is greater than or
+# equal to:
+# Mirrors that are down half the time are still useful half the time
+CUTOFF_RUNNING = .50
+CUTOFF_V2DIR = .50
+# Guard flags are removed for some time after a relay restarts, so we ignore
+# the guard flag.
+CUTOFF_GUARD = .00
+# FallbackDirs must have a time-weighted-fraction that is less than or equal
+# to:
 # .00 means no bad exits
 PERMITTED_BADEXIT = .00
 
@@ -155,28 +189,41 @@ ONIONOO_SCALE_ONE = 999.
 _FB_POG = 0.2
 FALLBACK_PROPORTION_OF_GUARDS = None if OUTPUT_CANDIDATES else _FB_POG
 
-# We want exactly 100 fallbacks for the initial release
-# This gives us scope to add extra fallbacks to the list as needed
 # Limit the number of fallbacks (eliminating lowest by advertised bandwidth)
-MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 100
-# Emit a C #error if the number of fallbacks is below
-MIN_FALLBACK_COUNT = 100
+MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 200
+# Emit a C #error if the number of fallbacks is less than expected
+MIN_FALLBACK_COUNT = 0 if OUTPUT_CANDIDATES else MAX_FALLBACK_COUNT*0.5
+
+# The maximum number of fallbacks on the same address, contact, or family
+#
+# With 150 fallbacks, this means each operator sees 5% of client bootstraps.
+# For comparison:
+#  - We try to limit guard and exit operators to 5% of the network
+#  - The directory authorities used to see 11% of client bootstraps each
+#
+# We also don't want too much of the list to go down if a single operator
+# has to move all their relays.
+MAX_FALLBACKS_PER_IP = 1
+MAX_FALLBACKS_PER_IPV4 = MAX_FALLBACKS_PER_IP
+MAX_FALLBACKS_PER_IPV6 = MAX_FALLBACKS_PER_IP
+MAX_FALLBACKS_PER_CONTACT = 7
+MAX_FALLBACKS_PER_FAMILY = 7
 
 ## Fallback Bandwidth Requirements
 
-# Any fallback with the Exit flag has its bandwidth multipled by this fraction
+# Any fallback with the Exit flag has its bandwidth multiplied by this fraction
 # to make sure we aren't further overloading exits
 # (Set to 1.0, because we asked that only lightly loaded exits opt-in,
 # and the extra load really isn't that much for large relays.)
 EXIT_BANDWIDTH_FRACTION = 1.0
 
 # If a single fallback's bandwidth is too low, it's pointless adding it
-# We expect fallbacks to handle an extra 30 kilobytes per second of traffic
-# Make sure they can support a hundred times the expected extra load
-# (Use 102.4 to make it come out nicely in MB/s)
+# We expect fallbacks to handle an extra 10 kilobytes per second of traffic
+# Make sure they can support fifty times the expected extra load
+#
 # We convert this to a consensus weight before applying the filter,
 # because all the bandwidth amounts are specified by the relay
-MIN_BANDWIDTH = 102.4 * 30.0 * 1024.0
+MIN_BANDWIDTH = 50.0 * 10.0 * 1024.0
 
 # Clients will time out after 30 seconds trying to download a consensus
 # So allow fallback directories half that to deliver a consensus
@@ -188,21 +235,6 @@ CONSENSUS_DOWNLOAD_SPEED_MAX = 15.0
 # This avoids delisting a relay due to transient network conditions
 CONSENSUS_DOWNLOAD_RETRY = True
 
-## Fallback Weights for Client Selection
-
-# All fallback weights are equal, and set to the value below
-# Authorities are weighted 1.0 by default
-# Clients use these weights to select fallbacks and authorities at random
-# If there are 100 fallbacks and 9 authorities:
-#  - each fallback is chosen with probability 10.0/(10.0*100 + 1.0*9) ~= 0.99%
-#  - each authority is chosen with probability 1.0/(10.0*100 + 1.0*9) ~= 0.09%
-# A client choosing a bootstrap directory server will choose a fallback for
-# 10.0/(10.0*100 + 1.0*9) * 100 = 99.1% of attempts, and an authority for
-# 1.0/(10.0*100 + 1.0*9) * 9 = 0.9% of attempts.
-# (This disregards the bootstrap schedules, where clients start by choosing
-# from fallbacks & authoritites, then later choose from only authorities.)
-FALLBACK_OUTPUT_WEIGHT = 10.0
-
 ## Parsing Functions
 
 def parse_ts(t):
@@ -242,6 +274,10 @@ def cleanse_c_multiline_comment(raw_string):
   bad_char_list = '*/'
   # Prevent a malicious string from using C nulls
   bad_char_list += '\0'
+  # Avoid confusing parsers by making sure there is only one comma per fallback
+  bad_char_list += ','
+  # Avoid confusing parsers by making sure there is only one equals per field
+  bad_char_list += '='
   # Be safer by removing bad characters entirely
   cleansed_string = remove_bad_chars(cleansed_string, bad_char_list)
   # Some compilers may further process the content of comments
@@ -262,6 +298,10 @@ def cleanse_c_string(raw_string):
   bad_char_list += '\\'
   # Prevent a malicious string from using C nulls
   bad_char_list += '\0'
+  # Avoid confusing parsers by making sure there is only one comma per fallback
+  bad_char_list += ','
+  # Avoid confusing parsers by making sure there is only one equals per field
+  bad_char_list += '='
   # Be safer by removing bad characters entirely
   cleansed_string = remove_bad_chars(cleansed_string, bad_char_list)
   # Some compilers may further process the content of strings
@@ -329,6 +369,15 @@ def read_from_file(file_name, max_len):
                  )
   return None
 
+def parse_fallback_file(file_name):
+  file_data = read_from_file(file_name, MAX_LIST_FILE_SIZE)
+  file_data = cleanse_unprintable(file_data)
+  file_data = remove_bad_chars(file_data, '\n"\0')
+  file_data = re.sub('/\*.*?\*/', '', file_data)
+  file_data = file_data.replace(',', '\n')
+  file_data = file_data.replace(' weight=10', '')
+  return file_data
+
 def load_possibly_compressed_response_json(response):
     if response.info().get('Content-Encoding') == 'gzip':
       buf = StringIO.StringIO( response.read() )
@@ -367,8 +416,8 @@ def onionoo_fetch(what, **kwargs):
   params = kwargs
   params['type'] = 'relay'
   #params['limit'] = 10
-  params['first_seen_days'] = '%d-'%(ADDRESS_AND_PORT_STABLE_DAYS,)
-  params['last_seen_days'] = '-7'
+  params['first_seen_days'] = '%d-'%(ADDRESS_AND_PORT_STABLE_DAYS)
+  params['last_seen_days'] = '-%d'%(MAX_DOWNTIME_DAYS)
   params['flag'] = 'V2Dir'
   url = ONIONOO + what + '?' + urllib.urlencode(params)
 
@@ -491,12 +540,14 @@ class Candidate(object):
       details['flags'] = []
     if (not 'advertised_bandwidth' in details
         or details['advertised_bandwidth'] is None):
-      # relays without advertised bandwdith have it calculated from their
+      # relays without advertised bandwidth have it calculated from their
       # consensus weight
       details['advertised_bandwidth'] = 0
     if (not 'effective_family' in details
         or details['effective_family'] is None):
       details['effective_family'] = []
+    if not 'platform' in details:
+      details['platform'] = None
     details['last_changed_address_or_port'] = parse_ts(
                                       details['last_changed_address_or_port'])
     self._data = details
@@ -511,6 +562,8 @@ class Candidate(object):
     self._compute_ipv6addr()
     if not self.has_ipv6():
       logging.debug("Failed to get an ipv6 address for %s."%(self._fpr,))
+    self._compute_version()
+    self._extra_info_cache = None
 
   def _stable_sort_or_addresses(self):
     # replace self._data['or_addresses'] with a stable ordering,
@@ -623,6 +676,59 @@ class Candidate(object):
         self.ipv6orport = int(port)
         return
 
+  def _compute_version(self):
+    # parse the version out of the platform string
+    # The platform looks like: "Tor 0.2.7.6 on Linux"
+    self._data['version'] = None
+    if self._data['platform'] is None:
+      return
+    # be tolerant of weird whitespacing, use a whitespace split
+    tokens = self._data['platform'].split()
+    for token in tokens:
+      vnums = token.split('.')
+      # if it's at least a.b.c.d, with potentially an -alpha-dev, -alpha, -rc
+      if (len(vnums) >= 4 and vnums[0].isdigit() and vnums[1].isdigit() and
+          vnums[2].isdigit()):
+        self._data['version'] = token
+        return
+
+  # From #20509
+  # bug #20499 affects versions from 0.2.9.1-alpha-dev to 0.2.9.4-alpha-dev
+  # and version 0.3.0.0-alpha-dev
+  # Exhaustive lists are hard to get wrong
+  STALE_CONSENSUS_VERSIONS = ['0.2.9.1-alpha-dev',
+                              '0.2.9.2-alpha',
+                              '0.2.9.2-alpha-dev',
+                              '0.2.9.3-alpha',
+                              '0.2.9.3-alpha-dev',
+                              '0.2.9.4-alpha',
+                              '0.2.9.4-alpha-dev',
+                              '0.3.0.0-alpha-dev'
+                              ]
+
+  def is_valid_version(self):
+    # call _compute_version before calling this
+    # is the version of the relay a version we want as a fallback?
+    # checks both recommended versions and bug #20499 / #20509
+    #
+    # if the relay doesn't have a recommended version field, exclude the relay
+    if not self._data.has_key('recommended_version'):
+      log_excluded('%s not a candidate: no recommended_version field',
+                   self._fpr)
+      return False
+    if not self._data['recommended_version']:
+      log_excluded('%s not a candidate: version not recommended', self._fpr)
+      return False
+    # if the relay doesn't have version field, exclude the relay
+    if not self._data.has_key('version'):
+      log_excluded('%s not a candidate: no version field', self._fpr)
+      return False
+    if self._data['version'] in Candidate.STALE_CONSENSUS_VERSIONS:
+      logging.warning('%s not a candidate: version delivers stale consensuses',
+                      self._fpr)
+      return False
+    return True
+
   @staticmethod
   def _extract_generic_history(history, which='unknown'):
     # given a tree like this:
@@ -767,41 +873,42 @@ class Candidate(object):
       self._badexit = self._avg_generic_history(badexit) / ONIONOO_SCALE_ONE
 
   def is_candidate(self):
-    must_be_running_now = (PERFORM_IPV4_DIRPORT_CHECKS
-                           or PERFORM_IPV6_DIRPORT_CHECKS)
-    if (must_be_running_now and not self.is_running()):
-      logging.info('%s not a candidate: not running now, unable to check ' +
-                   'DirPort consensus download', self._fpr)
-      return False
-    if (self._data['last_changed_address_or_port'] >
-        self.CUTOFF_ADDRESS_AND_PORT_STABLE):
-      logging.info('%s not a candidate: changed address/port recently (%s)',
-                   self._fpr, self._data['last_changed_address_or_port'])
-      return False
-    if self._running < CUTOFF_RUNNING:
-      logging.info('%s not a candidate: running avg too low (%lf)',
-                   self._fpr, self._running)
-      return False
-    if self._v2dir < CUTOFF_V2DIR:
-      logging.info('%s not a candidate: v2dir avg too low (%lf)',
-                   self._fpr, self._v2dir)
-      return False
-    if self._badexit is not None and self._badexit > PERMITTED_BADEXIT:
-      logging.info('%s not a candidate: badexit avg too high (%lf)',
-                   self._fpr, self._badexit)
-      return False
-    # if the relay doesn't report a version, also exclude the relay
-    if (not self._data.has_key('recommended_version')
-        or not self._data['recommended_version']):
-      logging.info('%s not a candidate: version not recommended', self._fpr)
-      return False
-    if self._guard < CUTOFF_GUARD:
-      logging.info('%s not a candidate: guard avg too low (%lf)',
-                   self._fpr, self._guard)
-      return False
-    if (not self._data.has_key('consensus_weight')
-        or self._data['consensus_weight'] < 1):
-      logging.info('%s not a candidate: consensus weight invalid', self._fpr)
+    try:
+      if (MUST_BE_RUNNING_NOW and not self.is_running()):
+        log_excluded('%s not a candidate: not running now, unable to check ' +
+                     'DirPort consensus download', self._fpr)
+        return False
+      if (self._data['last_changed_address_or_port'] >
+          self.CUTOFF_ADDRESS_AND_PORT_STABLE):
+        log_excluded('%s not a candidate: changed address/port recently (%s)',
+                     self._fpr, self._data['last_changed_address_or_port'])
+        return False
+      if self._running < CUTOFF_RUNNING:
+        log_excluded('%s not a candidate: running avg too low (%lf)',
+                     self._fpr, self._running)
+        return False
+      if self._v2dir < CUTOFF_V2DIR:
+        log_excluded('%s not a candidate: v2dir avg too low (%lf)',
+                     self._fpr, self._v2dir)
+        return False
+      if self._badexit is not None and self._badexit > PERMITTED_BADEXIT:
+        log_excluded('%s not a candidate: badexit avg too high (%lf)',
+                     self._fpr, self._badexit)
+        return False
+      # this function logs a message depending on which check fails
+      if not self.is_valid_version():
+        return False
+      if self._guard < CUTOFF_GUARD:
+        log_excluded('%s not a candidate: guard avg too low (%lf)',
+                     self._fpr, self._guard)
+        return False
+      if (not self._data.has_key('consensus_weight')
+          or self._data['consensus_weight'] < 1):
+        log_excluded('%s not a candidate: consensus weight invalid', self._fpr)
+        return False
+    except BaseException as e:
+      logging.warning("Exception %s when checking if fallback is a candidate",
+                      str(e))
       return False
     return True
 
@@ -862,78 +969,6 @@ class Candidate(object):
       return True
     return False
 
-  def is_in_blacklist(self, relaylist):
-    """ A fallback matches a blacklist line if a sufficiently specific group
-        of attributes matches:
-          ipv4 & dirport
-          ipv4 & orport
-          id
-          ipv6 & dirport
-          ipv6 & ipv6 orport
-        If the fallback and the blacklist line both have an ipv6 key,
-        their values will be compared, otherwise, they will be ignored.
-        If there is no dirport and no orport, the entry matches all relays on
-        that ip. """
-    for entry in relaylist:
-      for key in entry:
-        value = entry[key]
-        if key == 'id' and value == self._fpr:
-          logging.info('%s is in the blacklist: fingerprint matches',
-                       self._fpr)
-          return True
-        if key == 'ipv4' and value == self.dirip:
-          # if the dirport is present, check it too
-          if entry.has_key('dirport'):
-            if int(entry['dirport']) == self.dirport:
-              logging.info('%s is in the blacklist: IPv4 (%s) and ' +
-                           'DirPort (%d) match', self._fpr, self.dirip,
-                           self.dirport)
-              return True
-          # if the orport is present, check it too
-          elif entry.has_key('orport'):
-            if int(entry['orport']) == self.orport:
-              logging.info('%s is in the blacklist: IPv4 (%s) and ' +
-                           'ORPort (%d) match', self._fpr, self.dirip,
-                           self.orport)
-              return True
-          else:
-            logging.info('%s is in the blacklist: IPv4 (%s) matches, and ' +
-                         'entry has no DirPort or ORPort', self._fpr,
-                         self.dirip)
-            return True
-        ipv6 = None
-        if self.has_ipv6():
-          ipv6 = '%s:%d'%(self.ipv6addr, self.ipv6orport)
-        if (key == 'ipv6' and self.has_ipv6()):
-        # if both entry and fallback have an ipv6 address, compare them,
-        # otherwise, disregard ipv6 addresses
-          if value == ipv6:
-            # if the dirport is present, check it too
-            if entry.has_key('dirport'):
-              if int(entry['dirport']) == self.dirport:
-                logging.info('%s is in the blacklist: IPv6 (%s) and ' +
-                             'DirPort (%d) match', self._fpr, ipv6,
-                             self.dirport)
-                return True
-            # we've already checked the ORPort, it's part of entry['ipv6']
-            else:
-              logging.info('%s is in the blacklist: IPv6 (%s) matches, and' +
-                           'entry has no DirPort', self._fpr, ipv6)
-              return True
-        elif (key == 'ipv6' or self.has_ipv6()):
-          # only log if the fingerprint matches but the IPv6 doesn't
-          if entry.has_key('id') and entry['id'] == self._fpr:
-            logging.info('%s skipping IPv6 blacklist comparison: relay ' +
-                         'has%s IPv6%s, but entry has%s IPv6%s', self._fpr,
-                         '' if self.has_ipv6() else ' no',
-                         (' (' + ipv6 + ')') if self.has_ipv6() else  '',
-                         '' if key == 'ipv6' else ' no',
-                         (' (' + value + ')') if key == 'ipv6' else '')
-            logging.warning('Has %s %s IPv6 address %s?', self._fpr,
-                        'gained an' if self.has_ipv6() else 'lost its former',
-                        ipv6 if self.has_ipv6() else value)
-    return False
-
   def cw_to_bw_factor(self):
     # any relays with a missing or zero consensus weight are not candidates
     # any relays with a missing advertised bandwidth have it set to zero
@@ -1062,42 +1097,63 @@ class Candidate(object):
         return True
     return False
 
-  # report how long it takes to download a consensus from dirip:dirport
+  # log how long it takes to download a consensus from dirip:dirport
+  # returns True if the download failed, False if it succeeded within max_time
   @staticmethod
-  def fallback_consensus_download_speed(dirip, dirport, nickname, max_time):
+  def fallback_consensus_download_speed(dirip, dirport, nickname, fingerprint,
+                                        max_time):
     download_failed = False
-    downloader = DescriptorDownloader()
-    start = datetime.datetime.utcnow()
     # some directory mirrors respond to requests in ways that hang python
     # sockets, which is why we log this line here
-    logging.info('Initiating consensus download from %s (%s:%d).', nickname,
-                 dirip, dirport)
+    logging.info('Initiating %sconsensus download from %s (%s:%d) %s.',
+                 'microdesc ' if DOWNLOAD_MICRODESC_CONSENSUS else '',
+                 nickname, dirip, dirport, fingerprint)
     # there appears to be about 1 second of overhead when comparing stem's
     # internal trace time and the elapsed time calculated here
     TIMEOUT_SLOP = 1.0
+    start = datetime.datetime.utcnow()
     try:
-      downloader.get_consensus(endpoints = [(dirip, dirport)],
-                               timeout = (max_time + TIMEOUT_SLOP),
-                               validate = True,
-                               retries = 0,
-                               fall_back_to_authority = False).run()
+      consensus = get_consensus(
+                              endpoints = [(dirip, dirport)],
+                              timeout = (max_time + TIMEOUT_SLOP),
+                              validate = True,
+                              retries = 0,
+                              fall_back_to_authority = False,
+                              document_handler = DocumentHandler.BARE_DOCUMENT,
+                              microdescriptor = DOWNLOAD_MICRODESC_CONSENSUS
+                                ).run()[0]
+      end = datetime.datetime.utcnow()
+      time_since_expiry = (end - consensus.valid_until).total_seconds()
     except Exception, stem_error:
-      logging.info('Unable to retrieve a consensus from %s: %s', nickname,
+      end = datetime.datetime.utcnow()
+      log_excluded('Unable to retrieve a consensus from %s: %s', nickname,
                     stem_error)
       status = 'error: "%s"' % (stem_error)
       level = logging.WARNING
       download_failed = True
-    elapsed = (datetime.datetime.utcnow() - start).total_seconds()
-    if elapsed > max_time:
+    elapsed = (end - start).total_seconds()
+    if download_failed:
+      # keep the error failure status, and avoid using the variables
+      pass
+    elif elapsed > max_time:
       status = 'too slow'
       level = logging.WARNING
       download_failed = True
+    elif (time_since_expiry > 0):
+      status = 'outdated consensus, expired %ds ago'%(int(time_since_expiry))
+      if time_since_expiry <= CONSENSUS_EXPIRY_TOLERANCE:
+        status += ', tolerating up to %ds'%(CONSENSUS_EXPIRY_TOLERANCE)
+        level = logging.INFO
+      else:
+        status += ', invalid'
+        level = logging.WARNING
+        download_failed = True
     else:
       status = 'ok'
       level = logging.DEBUG
-    logging.log(level, 'Consensus download: %0.1fs %s from %s (%s:%d), ' +
+    logging.log(level, 'Consensus download: %0.1fs %s from %s (%s:%d) %s, ' +
                  'max download time %0.1fs.', elapsed, status, nickname,
-                 dirip, dirport, max_time)
+                 dirip, dirport, fingerprint, max_time)
     return download_failed
 
   # does this fallback download the consensus fast enough?
@@ -1109,12 +1165,14 @@ class Candidate(object):
       ipv4_failed = Candidate.fallback_consensus_download_speed(self.dirip,
                                                 self.dirport,
                                                 self._data['nickname'],
+                                                self._fpr,
                                                 CONSENSUS_DOWNLOAD_SPEED_MAX)
     if self.has_ipv6() and PERFORM_IPV6_DIRPORT_CHECKS:
       # Clients assume the IPv6 DirPort is the same as the IPv4 DirPort
       ipv6_failed = Candidate.fallback_consensus_download_speed(self.ipv6addr,
                                                 self.dirport,
                                                 self._data['nickname'],
+                                                self._fpr,
                                                 CONSENSUS_DOWNLOAD_SPEED_MAX)
     return ((not ipv4_failed) and (not ipv6_failed))
 
@@ -1151,6 +1209,7 @@ class Candidate(object):
     # /*
     # nickname
     # flags
+    # adjusted bandwidth, consensus weight
     # [contact]
     # [identical contact counts]
     # */
@@ -1162,27 +1221,21 @@ class Candidate(object):
     s += 'Flags: '
     s += cleanse_c_multiline_comment(' '.join(sorted(self._data['flags'])))
     s += '\n'
+    # this is an adjusted bandwidth, see calculate_measured_bandwidth()
+    bandwidth = self._data['measured_bandwidth']
+    weight = self._data['consensus_weight']
+    s += 'Bandwidth: %.1f MByte/s, Consensus Weight: %d'%(
+        bandwidth/(1024.0*1024.0),
+        weight)
+    s += '\n'
     if self._data['contact'] is not None:
       s += cleanse_c_multiline_comment(self._data['contact'])
-      if CONTACT_COUNT or CONTACT_BLACKLIST_COUNT:
+      if CONTACT_COUNT:
         fallback_count = len([f for f in fallbacks
                               if f._data['contact'] == self._data['contact']])
         if fallback_count > 1:
           s += '\n'
           s += '%d identical contacts listed' % (fallback_count)
-      if CONTACT_BLACKLIST_COUNT:
-        prefilter_count = len([f for f in prefilter_fallbacks
-                               if f._data['contact'] == self._data['contact']])
-        filter_count = prefilter_count - fallback_count
-        if filter_count > 0:
-          if fallback_count > 1:
-            s += ' '
-          else:
-            s += '\n'
-          s += '%d blacklisted' % (filter_count)
-      s += '\n'
-    s += '*/'
-    s += '\n'
 
   # output the fallback info C string for this fallback
   # this is the text that would go after FallbackDir in a torrc
@@ -1190,8 +1243,14 @@ class Candidate(object):
   # comment-out the returned string
   def fallbackdir_info(self, dl_speed_ok):
     # "address:dirport orport=port id=fingerprint"
+    # (insert additional madatory fields here)
     # "[ipv6=addr:orport]"
-    # "weight=FALLBACK_OUTPUT_WEIGHT",
+    # (insert additional optional fields here)
+    # /* nickname=name */
+    # /* extrainfo={0,1} */
+    # (insert additional comment fields here)
+    # /* ===== */
+    # ,
     #
     # Do we want a C string, or a commented-out string?
     c_string = dl_speed_ok
@@ -1212,10 +1271,34 @@ class Candidate(object):
             self.orport,
             cleanse_c_string(self._fpr))
     s += '\n'
+    # (insert additional madatory fields here)
     if self.has_ipv6():
       s += '" ipv6=%s:%d"'%(cleanse_c_string(self.ipv6addr), self.ipv6orport)
       s += '\n'
-    s += '" weight=%d",'%(FALLBACK_OUTPUT_WEIGHT)
+    # (insert additional optional fields here)
+    if not comment_string:
+      s += '/* '
+    s += 'nickname=%s'%(cleanse_c_string(self._data['nickname']))
+    if not comment_string:
+      s += ' */'
+    s += '\n'
+    # if we know that the fallback is an extrainfo cache, flag it
+    # and if we don't know, assume it is not
+    if not comment_string:
+      s += '/* '
+    s += 'extrainfo=%d'%(1 if self._extra_info_cache else 0)
+    if not comment_string:
+      s += ' */'
+    s += '\n'
+    # (insert additional comment fields here)
+    # The terminator and comma must be the last line in each fallback entry
+    if not comment_string:
+      s += '/* '
+    s += SECTION_SEPARATOR_BASE
+    if not comment_string:
+      s += ' */'
+    s += '\n'
+    s += ','
     if comment_string:
       s += '\n'
       s += '*/'
@@ -1251,7 +1334,8 @@ class CandidateList(dict):
     d = fetch('details',
         fields=('fingerprint,nickname,contact,last_changed_address_or_port,' +
                 'consensus_weight,advertised_bandwidth,or_addresses,' +
-                'dir_address,recommended_version,flags,effective_family'))
+                'dir_address,recommended_version,flags,effective_family,' +
+                'platform'))
     logging.debug('Loading details document done.')
 
     if not 'relays' in d: raise Exception("No relays found in document.")
@@ -1297,13 +1381,12 @@ class CandidateList(dict):
     self.fallbacks.sort(key=lambda f: f._data['measured_bandwidth'],
                         reverse=True)
 
-  # sort fallbacks by their fingerprint, lowest to highest
-  # this is useful for stable diffs of fallback lists
-  def sort_fallbacks_by_fingerprint(self):
-    self.fallbacks.sort(key=lambda f: f._fpr)
+  # sort fallbacks by the data field data_field, lowest to highest
+  def sort_fallbacks_by(self, data_field):
+    self.fallbacks.sort(key=lambda f: f._data[data_field])
 
   @staticmethod
-  def load_relaylist(file_name):
+  def load_relaylist(file_obj):
     """ Read each line in the file, and parse it like a FallbackDir line:
         an IPv4 address and optional port:
           <IPv4 address>:<port>
@@ -1318,8 +1401,9 @@ class CandidateList(dict):
         (of string -> string key/value pairs),
         and these dictionaries are placed in an array.
         comments start with # and are ignored """
+    file_data = file_obj['data']
+    file_name = file_obj['name']
     relaylist = []
-    file_data = read_from_file(file_name, MAX_LIST_FILE_SIZE)
     if file_data is None:
       return relaylist
     for line in file_data.split('\n'):
@@ -1359,52 +1443,36 @@ class CandidateList(dict):
       relaylist.append(relay_entry)
     return relaylist
 
-  # apply the fallback whitelist and blacklist
-  def apply_filter_lists(self):
+  # apply the fallback whitelist
+  def apply_filter_lists(self, whitelist_obj):
     excluded_count = 0
-    logging.debug('Applying whitelist and blacklist.')
-    # parse the whitelist and blacklist
-    whitelist = self.load_relaylist(WHITELIST_FILE_NAME)
-    blacklist = self.load_relaylist(BLACKLIST_FILE_NAME)
+    logging.debug('Applying whitelist')
+    # parse the whitelist
+    whitelist = self.load_relaylist(whitelist_obj)
     filtered_fallbacks = []
     for f in self.fallbacks:
       in_whitelist = f.is_in_whitelist(whitelist)
-      in_blacklist = f.is_in_blacklist(blacklist)
-      if in_whitelist and in_blacklist:
-        if BLACKLIST_EXCLUDES_WHITELIST_ENTRIES:
-          # exclude
-          excluded_count += 1
-          logging.warning('Excluding %s: in both blacklist and whitelist.',
-                          f._fpr)
-        else:
-          # include
-          filtered_fallbacks.append(f)
-      elif in_whitelist:
+      if in_whitelist:
         # include
         filtered_fallbacks.append(f)
-      elif in_blacklist:
-        # exclude
-        excluded_count += 1
-        logging.info('Excluding %s: in blacklist.', f._fpr)
-      else:
-        if INCLUDE_UNLISTED_ENTRIES:
+      elif INCLUDE_UNLISTED_ENTRIES:
           # include
           filtered_fallbacks.append(f)
-        else:
+      else:
           # exclude
           excluded_count += 1
-          logging.info('Excluding %s: in neither blacklist nor whitelist.',
+          log_excluded('Excluding %s: not in whitelist.',
                        f._fpr)
     self.fallbacks = filtered_fallbacks
     return excluded_count
 
   @staticmethod
   def summarise_filters(initial_count, excluded_count):
-    return '/* Whitelist & blacklist excluded %d of %d candidates. */'%(
+    return '/* Whitelist excluded %d of %d candidates. */'%(
                                                 excluded_count, initial_count)
 
   # calculate each fallback's measured bandwidth based on the median
-  # consensus weight to advertised bandwdith ratio
+  # consensus weight to advertised bandwidth ratio
   def calculate_measured_bandwidth(self):
     self.sort_fallbacks_by_cw_to_bw_factor()
     median_fallback = self.fallback_median(True)
@@ -1429,8 +1497,8 @@ class CandidateList(dict):
         # the bandwidth we log here is limited by the relay's consensus weight
         # as well as its adverttised bandwidth. See set_measured_bandwidth
         # for details
-        logging.info('%s not a candidate: bandwidth %.1fMB/s too low, must ' +
-                     'be at least %.1fMB/s', f._fpr,
+        log_excluded('%s not a candidate: bandwidth %.1fMByte/s too low, ' +
+                     'must be at least %.1fMByte/s', f._fpr,
                      f._data['measured_bandwidth']/(1024.0*1024.0),
                      MIN_BANDWIDTH/(1024.0*1024.0))
     self.fallbacks = above_min_bw_fallbacks
@@ -1470,49 +1538,85 @@ class CandidateList(dict):
     else:
       return None
 
-  # does exclusion_list contain attribute?
+  # return a new bag suitable for storing attributes
+  @staticmethod
+  def attribute_new():
+    return dict()
+
+  # get the count of attribute in attribute_bag
+  # if attribute is None or the empty string, return 0
+  @staticmethod
+  def attribute_count(attribute, attribute_bag):
+    if attribute is None or attribute == '':
+      return 0
+    if attribute not in attribute_bag:
+      return 0
+    return attribute_bag[attribute]
+
+  # does attribute_bag contain more than max_count instances of attribute?
   # if so, return False
   # if not, return True
-  # if attribute is None or the empty string, always return True
+  # if attribute is None or the empty string, or max_count is invalid,
+  # always return True
   @staticmethod
-  def allow(attribute, exclusion_list):
-    if attribute is None or attribute == '':
+  def attribute_allow(attribute, attribute_bag, max_count=1):
+    if attribute is None or attribute == '' or max_count <= 0:
       return True
-    elif attribute in exclusion_list:
+    elif CandidateList.attribute_count(attribute, attribute_bag) >= max_count:
       return False
     else:
       return True
 
-  # make sure there is only one fallback per IPv4 address, and per IPv6 address
+  # add attribute to attribute_bag, incrementing the count if it is already
+  # present
+  # if attribute is None or the empty string, or count is invalid,
+  # do nothing
+  @staticmethod
+  def attribute_add(attribute, attribute_bag, count=1):
+    if attribute is None or attribute == '' or count <= 0:
+      pass
+    attribute_bag.setdefault(attribute, 0)
+    attribute_bag[attribute] += count
+
+  # make sure there are only MAX_FALLBACKS_PER_IP fallbacks per IPv4 address,
+  # and per IPv6 address
   # there is only one IPv4 address on each fallback: the IPv4 DirPort address
   # (we choose the IPv4 ORPort which is on the same IPv4 as the DirPort)
   # there is at most one IPv6 address on each fallback: the IPv6 ORPort address
   # we try to match the IPv4 ORPort, but will use any IPv6 address if needed
-  # (clients assume the IPv6 DirPort is the same as the IPv4 DirPort, but
-  # typically only use the IPv6 ORPort)
+  # (clients only use the IPv6 ORPort)
   # if there is no IPv6 address, only the IPv4 address is checked
   # return the number of candidates we excluded
   def limit_fallbacks_same_ip(self):
     ip_limit_fallbacks = []
-    ip_list = []
+    ip_list = CandidateList.attribute_new()
     for f in self.fallbacks:
-      if (CandidateList.allow(f.dirip, ip_list)
-          and CandidateList.allow(f.ipv6addr, ip_list)):
+      if (CandidateList.attribute_allow(f.dirip, ip_list,
+                                        MAX_FALLBACKS_PER_IPV4)
+          and CandidateList.attribute_allow(f.ipv6addr, ip_list,
+                                            MAX_FALLBACKS_PER_IPV6)):
         ip_limit_fallbacks.append(f)
-        ip_list.append(f.dirip)
+        CandidateList.attribute_add(f.dirip, ip_list)
         if f.has_ipv6():
-          ip_list.append(f.ipv6addr)
-      elif not CandidateList.allow(f.dirip, ip_list):
-        logging.info('Eliminated %s: already have fallback on IPv4 %s'%(
-                                                          f._fpr, f.dirip))
-      elif f.has_ipv6() and not CandidateList.allow(f.ipv6addr, ip_list):
-        logging.info('Eliminated %s: already have fallback on IPv6 %s'%(
-                                                          f._fpr, f.ipv6addr))
+          CandidateList.attribute_add(f.ipv6addr, ip_list)
+      elif not CandidateList.attribute_allow(f.dirip, ip_list,
+                                             MAX_FALLBACKS_PER_IPV4):
+        log_excluded('Eliminated %s: already have %d fallback(s) on IPv4 %s'
+                     %(f._fpr, CandidateList.attribute_count(f.dirip, ip_list),
+                       f.dirip))
+      elif (f.has_ipv6() and
+            not CandidateList.attribute_allow(f.ipv6addr, ip_list,
+                                              MAX_FALLBACKS_PER_IPV6)):
+        log_excluded('Eliminated %s: already have %d fallback(s) on IPv6 %s'
+                     %(f._fpr, CandidateList.attribute_count(f.ipv6addr,
+                                                             ip_list),
+                       f.ipv6addr))
     original_count = len(self.fallbacks)
     self.fallbacks = ip_limit_fallbacks
     return original_count - len(self.fallbacks)
 
-  # make sure there is only one fallback per ContactInfo
+  # make sure there are only MAX_FALLBACKS_PER_CONTACT fallbacks for each
+  # ContactInfo
   # if there is no ContactInfo, allow the fallback
   # this check can be gamed by providing no ContactInfo, or by setting the
   # ContactInfo to match another fallback
@@ -1520,41 +1624,96 @@ class CandidateList(dict):
   # go down at similar times, its usefulness outweighs the risk
   def limit_fallbacks_same_contact(self):
     contact_limit_fallbacks = []
-    contact_list = []
+    contact_list = CandidateList.attribute_new()
     for f in self.fallbacks:
-      if CandidateList.allow(f._data['contact'], contact_list):
+      if CandidateList.attribute_allow(f._data['contact'], contact_list,
+                                       MAX_FALLBACKS_PER_CONTACT):
         contact_limit_fallbacks.append(f)
-        contact_list.append(f._data['contact'])
+        CandidateList.attribute_add(f._data['contact'], contact_list)
       else:
-        logging.info(('Eliminated %s: already have fallback on ' +
-                       'ContactInfo %s')%(f._fpr, f._data['contact']))
+        log_excluded(
+          'Eliminated %s: already have %d fallback(s) on ContactInfo %s'
+          %(f._fpr, CandidateList.attribute_count(f._data['contact'],
+                                                  contact_list),
+            f._data['contact']))
     original_count = len(self.fallbacks)
     self.fallbacks = contact_limit_fallbacks
     return original_count - len(self.fallbacks)
 
-  # make sure there is only one fallback per effective family
+  # make sure there are only MAX_FALLBACKS_PER_FAMILY fallbacks per effective
+  # family
   # if there is no family, allow the fallback
-  # this check can't be gamed, because we use effective family, which ensures
-  # mutual family declarations
+  # we use effective family, which ensures mutual family declarations
+  # but the check can be gamed by not declaring a family at all
   # if any indirect families exist, the result depends on the order in which
   # fallbacks are sorted in the list
   def limit_fallbacks_same_family(self):
     family_limit_fallbacks = []
-    fingerprint_list = []
+    fingerprint_list = CandidateList.attribute_new()
     for f in self.fallbacks:
-      if CandidateList.allow(f._fpr, fingerprint_list):
+      if CandidateList.attribute_allow(f._fpr, fingerprint_list,
+                                       MAX_FALLBACKS_PER_FAMILY):
         family_limit_fallbacks.append(f)
-        fingerprint_list.append(f._fpr)
-        fingerprint_list.extend(f._data['effective_family'])
+        CandidateList.attribute_add(f._fpr, fingerprint_list)
+        for family_fingerprint in f._data['effective_family']:
+          CandidateList.attribute_add(family_fingerprint, fingerprint_list)
       else:
-        # technically, we already have a fallback with this fallback in its
-        # effective family
-        logging.info('Eliminated %s: already have fallback in effective ' +
-                      'family'%(f._fpr))
+        # we already have a fallback with this fallback in its effective
+        # family
+        log_excluded(
+          'Eliminated %s: already have %d fallback(s) in effective family'
+          %(f._fpr, CandidateList.attribute_count(f._fpr, fingerprint_list)))
     original_count = len(self.fallbacks)
     self.fallbacks = family_limit_fallbacks
     return original_count - len(self.fallbacks)
 
+  # try once to get the descriptors for fingerprint_list using stem
+  # returns an empty list on exception
+  @staticmethod
+  def get_fallback_descriptors_once(fingerprint_list):
+    desc_list = get_server_descriptors(fingerprints=fingerprint_list).run(suppress=True)
+    return desc_list
+
+  # try up to max_retries times to get the descriptors for fingerprint_list
+  # using stem. Stops retrying when all descriptors have been retrieved.
+  # returns a list containing the descriptors that were retrieved
+  @staticmethod
+  def get_fallback_descriptors(fingerprint_list, max_retries=5):
+    # we can't use stem's retries=, because we want to support more than 96
+    # descriptors
+    #
+    # add an attempt for every MAX_FINGERPRINTS (or part thereof) in the list
+    max_retries += (len(fingerprint_list) + MAX_FINGERPRINTS - 1) / MAX_FINGERPRINTS
+    remaining_list = fingerprint_list
+    desc_list = []
+    for _ in xrange(max_retries):
+      if len(remaining_list) == 0:
+        break
+      new_desc_list = CandidateList.get_fallback_descriptors_once(remaining_list[0:MAX_FINGERPRINTS])
+      for d in new_desc_list:
+        try:
+          remaining_list.remove(d.fingerprint)
+        except ValueError:
+          # warn and ignore if a directory mirror returned a bad descriptor
+          logging.warning("Directory mirror returned unwanted descriptor %s, ignoring",
+                          d.fingerprint)
+          continue
+        desc_list.append(d)
+    return desc_list
+
+  # find the fallbacks that cache extra-info documents
+  # Onionoo doesn't know this, so we have to use stem
+  def mark_extra_info_caches(self):
+    fingerprint_list = [ f._fpr for f in self.fallbacks ]
+    logging.info("Downloading fallback descriptors to find extra-info caches")
+    desc_list = CandidateList.get_fallback_descriptors(fingerprint_list)
+    for d in desc_list:
+      self[d.fingerprint]._extra_info_cache = d.extra_info_cache
+    missing_descriptor_list = [ f._fpr for f in self.fallbacks
+                                if f._extra_info_cache is None ]
+    for f in missing_descriptor_list:
+      logging.warning("No descriptor for {}. Assuming extrainfo=0.".format(f))
+
   # try a download check on each fallback candidate in order
   # stop after max_count successful downloads
   # but don't remove any candidates from the array
@@ -1714,7 +1873,7 @@ class CandidateList(dict):
    # this doesn't actually tell us anything useful
    #self.describe_fallback_ipv4_netblock_mask(8)
    self.describe_fallback_ipv4_netblock_mask(16)
-   self.describe_fallback_ipv4_netblock_mask(24)
+   #self.describe_fallback_ipv4_netblock_mask(24)
 
   # log a message about the proportion of fallbacks in each IPv6 /12 (RIR),
   # /23 (smaller RIR blocks), /32 (LIR), /48 (Customer), and /64 (Host)
@@ -1724,7 +1883,7 @@ class CandidateList(dict):
     #self.describe_fallback_ipv6_netblock_mask(12)
     #self.describe_fallback_ipv6_netblock_mask(23)
     self.describe_fallback_ipv6_netblock_mask(32)
-    self.describe_fallback_ipv6_netblock_mask(48)
+    #self.describe_fallback_ipv6_netblock_mask(48)
     self.describe_fallback_ipv6_netblock_mask(64)
 
   # log a message about the proportion of fallbacks in each IPv4 and IPv6
@@ -1802,6 +1961,18 @@ class CandidateList(dict):
                     CandidateList.describe_percentage(dir_count,
                                                       fallback_count)))
 
+  # return a list of fallbacks which cache extra-info documents
+  def fallbacks_with_extra_info_cache(self):
+    return filter(lambda x: x._extra_info_cache, self.fallbacks)
+
+  # log a message about the proportion of fallbacks that cache extra-info docs
+  def describe_fallback_extra_info_caches(self):
+    extra_info_falback_count = len(self.fallbacks_with_extra_info_cache())
+    fallback_count = len(self.fallbacks)
+    logging.warning('%s of fallbacks cache extra-info documents'%(
+                    CandidateList.describe_percentage(extra_info_falback_count,
+                                                      fallback_count)))
+
   # return a list of fallbacks which have the Exit flag
   def fallbacks_with_exit(self):
     return filter(lambda x: x.is_exit(), self.fallbacks)
@@ -1829,10 +2000,6 @@ class CandidateList(dict):
   def summarise_fallbacks(self, eligible_count, operator_count, failed_count,
                           guard_count, target_count):
     s = ''
-    s += '/* To comment-out entries in this file, use C comments, and add *'
-    s += ' to the start of each line. (stem finds fallback entries using "'
-    s += ' at the start of a line.) */'
-    s += '\n'
     # Report:
     #  whether we checked consensus download times
     #  the number of fallback directories (and limits/exclusions, if relevant)
@@ -1878,8 +2045,8 @@ class CandidateList(dict):
     min_bw = min_fb._data['measured_bandwidth']
     max_fb = self.fallback_max()
     max_bw = max_fb._data['measured_bandwidth']
-    s += 'Bandwidth Range: %.1f - %.1f MB/s'%(min_bw/(1024.0*1024.0),
-                                              max_bw/(1024.0*1024.0))
+    s += 'Bandwidth Range: %.1f - %.1f MByte/s'%(min_bw/(1024.0*1024.0),
+                                                 max_bw/(1024.0*1024.0))
     s += '\n'
     s += '*/'
     if fallback_count < MIN_FALLBACK_COUNT:
@@ -1892,12 +2059,53 @@ class CandidateList(dict):
       s += 'or setting INCLUDE_UNLISTED_ENTRIES = True.'
     return s
 
+def process_existing():
+  logging.basicConfig(level=logging.INFO)
+  logging.getLogger('stem').setLevel(logging.INFO)
+  whitelist = {'data': parse_fallback_file(FALLBACK_FILE_NAME),
+               'name': FALLBACK_FILE_NAME}
+  list_fallbacks(whitelist)
+
+def process_default():
+  logging.basicConfig(level=logging.WARNING)
+  logging.getLogger('stem').setLevel(logging.WARNING)
+  whitelist = {'data': read_from_file(WHITELIST_FILE_NAME, MAX_LIST_FILE_SIZE),
+               'name': WHITELIST_FILE_NAME}
+  list_fallbacks(whitelist)
+
 ## Main Function
+def main():
+  if get_command() == 'check_existing':
+    process_existing()
+  else:
+    process_default()
 
-def list_fallbacks():
+def get_command():
+  if len(sys.argv) == 2:
+    return sys.argv[1]
+  else:
+    return None
+
+def log_excluded(msg, *args):
+  if get_command() == 'check_existing':
+    logging.warning(msg, *args)
+  else:
+    logging.info(msg, *args)
+
+def list_fallbacks(whitelist):
   """ Fetches required onionoo documents and evaluates the
       fallback directory criteria for each of the relays """
 
+  print "/* type=fallback */"
+  print ("/* version={} */"
+         .format(cleanse_c_multiline_comment(FALLBACK_FORMAT_VERSION)))
+  now = datetime.datetime.utcnow()
+  timestamp = now.strftime('%Y%m%d%H%M%S')
+  print ("/* timestamp={} */"
+         .format(cleanse_c_multiline_comment(timestamp)))
+  # end the header with a separator, to make it easier for parsers
+  print SECTION_SEPARATOR_COMMENT
+
   logging.warning('Downloading and parsing Onionoo data. ' +
                   'This may take some time.')
   # find relays that could be fallbacks
@@ -1921,13 +2129,13 @@ def list_fallbacks():
   candidates.compute_fallbacks()
   prefilter_fallbacks = copy.copy(candidates.fallbacks)
 
-  # filter with the whitelist and blacklist
+  # filter with the whitelist
   # if a relay has changed IPv4 address or ports recently, it will be excluded
   # as ineligible before we call apply_filter_lists, and so there will be no
   # warning that the details have changed from those in the whitelist.
   # instead, there will be an info-level log during the eligibility check.
   initial_count = len(candidates.fallbacks)
-  excluded_count = candidates.apply_filter_lists()
+  excluded_count = candidates.apply_filter_lists(whitelist)
   print candidates.summarise_filters(initial_count, excluded_count)
   eligible_count = len(candidates.fallbacks)
 
@@ -1963,6 +2171,9 @@ def list_fallbacks():
                     'This may take some time.')
   failed_count = candidates.perform_download_consensus_checks(max_count)
 
+  # work out which fallbacks cache extra-infos
+  candidates.mark_extra_info_caches()
+
   # analyse and log interesting diversity metrics
   # like netblock, ports, exit, IPv4-only
   # (we can't easily analyse AS, and it's hard to accurately analyse country)
@@ -1971,6 +2182,7 @@ def list_fallbacks():
   if HAVE_IPADDRESS:
     candidates.describe_fallback_netblocks()
   candidates.describe_fallback_ports()
+  candidates.describe_fallback_extra_info_caches()
   candidates.describe_fallback_exit_flag()
 
   # output C comments summarising the fallback selection process
@@ -1985,15 +2197,20 @@ def list_fallbacks():
   for s in fetch_source_list():
     print describe_fetch_source(s)
 
+  # start the list with a separator, to make it easy for parsers
+  print SECTION_SEPARATOR_COMMENT
+
+  # sort the list differently depending on why we've created it:
   # if we're outputting the final fallback list, sort by fingerprint
   # this makes diffs much more stable
-  # otherwise, leave sorted by bandwidth, which allows operators to be
-  # contacted in priority order
-  if not OUTPUT_CANDIDATES:
-    candidates.sort_fallbacks_by_fingerprint()
+  # otherwise, if we're trying to find a bandwidth cutoff, or we want to
+  # contact operators in priority order, sort by bandwidth (not yet
+  # implemented)
+  # otherwise, if we're contacting operators, sort by contact
+  candidates.sort_fallbacks_by(OUTPUT_SORT_FIELD)
 
   for x in candidates.fallbacks:
     print x.fallbackdir_line(candidates.fallbacks, prefilter_fallbacks)
 
 if __name__ == "__main__":
-  list_fallbacks()
+  main()
diff --git a/scripts/maint/updateRustDependencies.sh b/scripts/maint/updateRustDependencies.sh
new file mode 100755
index 0000000000..a5a92579d3
--- /dev/null
+++ b/scripts/maint/updateRustDependencies.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018 The Tor Project, Inc.
+# Copyright (c) 2018 isis agora lovecruft
+# See LICENSE for license information
+#
+# updateRustDependencies.sh
+# -------------------------
+# Update our vendored Rust dependencies, either adding/removing
+# dependencies and/or upgrading current dependencies to newer
+# versions.
+#
+# To use this script, first add your dependencies, exactly specifying
+# their versions, into the appropriate *crate-level* Cargo.toml in
+# src/rust/ (i.e. *not* /src/rust/Cargo.toml, but instead the one for
+# your crate).
+#
+# Next, run this script.  Then, go into src/ext/rust and commit the
+# changes to the tor-rust-dependencies repo.
+
+set -e
+
+HERE=`dirname $(realpath $0)`
+TOPLEVEL=`dirname $(dirname $HERE)`
+TOML="$TOPLEVEL/src/rust/Cargo.toml"
+VENDORED="$TOPLEVEL/src/ext/rust/crates"
+CARGO=`which cargo`
+
+if ! test -f "$TOML"  ; then
+    printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\n" "$TOML"
+fi
+
+if ! test -d "$VENDORED" ; then
+    printf "Error: Couldn't find directory for Rust dependencies! Expected location: %s\n" "$VENDORED"
+fi
+
+if test -z "$CARGO" ; then
+    printf "Error: cargo must be installed and in your \$PATH\n"
+fi
+
+if test -z `cargo --list | grep vendor` ; then
+    printf "Error: cargo-vendor not installed\n"
+fi
+
+$CARGO vendor -v --locked --explicit-version --no-delete --sync $TOML $VENDORED
diff --git a/scripts/test/appveyor-irc-notify.py b/scripts/test/appveyor-irc-notify.py
new file mode 100644
index 0000000000..cfe0afe7ae
--- /dev/null
+++ b/scripts/test/appveyor-irc-notify.py
@@ -0,0 +1,219 @@
+# coding=utf8
+# Copyright (C) 2015-2016 Christopher R. Wood
+# Copyright (c) 2018 The Tor Project
+# Copyright (c) 2018 isis agora lovecruft
+#
+# From: https://raw.githubusercontent.com/gridsync/gridsync/def54f8166089b733d166665fdabcad4cdc526d8/misc/irc-notify.py
+# and: https://github.com/gridsync/gridsync
+#
+# Modified by nexB on October 2016:
+#  - rework the handling of environment variables.
+#  - made the script use functions
+#  - support only Appveyor loading its environment variable to craft IRC notices.
+#
+# Modified by isis agora lovecruft <isis@torproject.org> in 2018:
+#  - Make IRC server configurable.
+#  - Make bot IRC nick deterministic.
+#  - Make bot join the channel rather than sending NOTICE messages externally.
+#  - Fix a bug which always caused sys.exit() to be logged as a traceback.
+#  - Actually reset the IRC colour codes after printing.
+#
+# Modified by Marcin Cieślak in 2018:
+#  - Accept UTF-8
+#  - only guess github URLs
+#  - stop using ANSI colors
+#
+# Modified by teor in 2018:
+#  - fix github provider detection ('gitHub' or 'gitHubEnterprise', apparently)
+#  - make short commits 10 hexdigits long (that's what git does for tor)
+#  - generate correct branches and URLs for pull requests and tags
+#  - switch to one URL per line
+
+# This program is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software Foundation;
+# either version 2 of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with this
+# program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street,
+# Fifth Floor, Boston, MA 02110-1301 USA.
+
+"""Simple AppVeyor IRC notification script.
+
+The first argument is an IRC server and port; the second is the channel. Other
+arguments passed to the script will be sent as notice messages content and any
+{var}-formatted environment variables will be expanded automatically, replaced
+with a corresponding Appveyor environment variable value. Use commas to
+delineate multiple messages.
+
+
+Example:
+export APPVEYOR_ACCOUNT_NAME=isislovecruft
+export APPVEYOR_BUILD_VERSION=1
+export APPVEYOR_PROJECT_NAME=tor
+export APPVEYOR_PULL_REQUEST_NUMBER=pull_request_number
+export APPVEYOR_PULL_REQUEST_TITLE=pull_request_title
+export APPVEYOR_REPO_BRANCH=repo_branch
+export APPVEYOR_REPO_COMMIT=22c95b72e29248dc4de9b85e590ee18f6f587de8
+export APPVEYOR_REPO_COMMIT_AUTHOR=isislovecruft
+export APPVEYOR_REPO_COMMIT_MESSAGE="some IRC test"
+export APPVEYOR_REPO_COMMIT_TIMESTAMP=2018-04-23
+export APPVEYOR_REPO_NAME=isislovecruft/tor
+export APPVEYOR_REPO_PROVIDER=github
+export APPVEYOR_URL=https://ci.appveyor.com
+python ./appveyor-irc-notify.py irc.oftc.net:6697 tor-ci '{repo_name} {repo_branch} {short_commit} - {repo_commit_author}: {repo_commit_message}','Build #{build_version} passed. Details: {build_url} |  Commit: {commit_url}
+
+See also https://github.com/gridsync/gridsync/blob/master/appveyor.yml for examples
+in Appveyor's YAML:
+
+    on_success:
+      - "python scripts/test/appveyor-irc-notify.py irc.oftc.net:6697 tor-ci success
+    on_failure:
+      - "python scripts/test/appveyor-irc-notify.py irc.oftc.net:6697 tor-ci failure
+"""
+
+from __future__ import print_function
+from __future__ import absolute_import
+
+import os
+import random
+import socket
+import ssl
+import sys
+import time
+
+
+def appveyor_vars():
+    """
+    Return a dict of key value crafted from appveyor environment variables.
+    """
+
+    vars = dict([
+            (
+                v.replace('APPVEYOR_', '').lower(),
+                os.getenv(v, '').decode('utf-8')
+            ) for v in [
+                'APPVEYOR_ACCOUNT_NAME',
+                'APPVEYOR_BUILD_VERSION',
+                'APPVEYOR_PROJECT_NAME',
+                'APPVEYOR_PULL_REQUEST_HEAD_COMMIT',
+                'APPVEYOR_PULL_REQUEST_HEAD_REPO_BRANCH',
+                'APPVEYOR_PULL_REQUEST_HEAD_REPO_NAME',
+                'APPVEYOR_PULL_REQUEST_NUMBER',
+                'APPVEYOR_PULL_REQUEST_TITLE',
+                'APPVEYOR_REPO_BRANCH',
+                'APPVEYOR_REPO_COMMIT',
+                'APPVEYOR_REPO_COMMIT_AUTHOR',
+                'APPVEYOR_REPO_COMMIT_AUTHOR_EMAIL',
+                'APPVEYOR_REPO_COMMIT_MESSAGE',
+                'APPVEYOR_REPO_COMMIT_MESSAGE_EXTENDED',
+                'APPVEYOR_REPO_COMMIT_TIMESTAMP',
+                'APPVEYOR_REPO_NAME',
+                'APPVEYOR_REPO_PROVIDER',
+                'APPVEYOR_REPO_TAG_NAME',
+                'APPVEYOR_URL',
+            ]
+    ])
+
+    BUILD_FMT = u'{url}/project/{account_name}/{project_name}/build/{build_version}'
+
+    if vars["repo_tag_name"]:
+        BRANCH_FMT = u'{repo_name} {repo_tag_name} {short_commit}'
+    else:
+        BRANCH_FMT = u'{repo_name} {repo_branch} {short_commit}'
+
+    vars.update(head_commit=vars["repo_commit"])
+
+    if vars["repo_provider"].lower().startswith('github'):
+        COMMIT_FMT = u'https://github.com/{repo_name}/commit/{repo_commit}'
+        if vars["pull_request_number"]:
+            vars.update(head_commit=vars["pull_request_head_commit"])
+            BRANCH_FMT = u'{repo_name} {repo_branch} pull {pull_request_head_repo_name} {pull_request_head_repo_branch} {short_commit}'
+            COMMIT_FMT = u'https://github.com/{pull_request_head_repo_name}/commit/{pull_request_head_commit}'
+            PULL_FMT = u'https://github.com/{repo_name}/pull/{pull_request_number}'
+            vars.update(pull_url=PULL_FMT.format(**vars))
+        vars.update(commit_url=COMMIT_FMT.format(**vars))
+
+    vars.update(short_commit=vars["head_commit"][:10])
+
+    vars.update(
+        build_url=BUILD_FMT.format(**vars),
+        branch_detail=BRANCH_FMT.format(**vars),
+    )
+    return vars
+
+
+def notify():
+    """
+    Send IRC notification
+    """
+    apvy_vars = appveyor_vars()
+
+    server, port = sys.argv[1].rsplit(":", 1)
+    channel = sys.argv[2]
+    success = sys.argv[3] == "success"
+    failure = sys.argv[3] == "failure"
+
+    if success or failure:
+        messages = []
+        messages.append(u"{branch_detail} - {repo_commit_author}: {repo_commit_message}")
+
+        if success:
+            messages.append(u"Build #{build_version} passed. Details: {build_url}")
+        if failure:
+            messages.append(u"Build #{build_version} failed. Details: {build_url}")
+
+        if "commit_url" in apvy_vars:
+            messages.append(u"Commit: {commit_url}")
+
+        if "pull_url" in apvy_vars:
+            messages.append(u"Pull: {pull_url}")
+
+    else:
+        messages = sys.argv[3:]
+        messages = ' '.join(messages)
+        messages = messages.decode("utf-8").split(',')
+
+    print(repr(apvy_vars))
+    messages = [msg.format(**apvy_vars).strip() for msg in messages]
+
+    irc_username = 'appveyor-ci'
+    irc_nick = irc_username
+
+    # establish connection
+    irc_sock = ssl.wrap_socket(socket.socket(socket.AF_INET, socket.SOCK_STREAM))
+    irc_sock.connect((socket.gethostbyname(server), int(port)))
+    irc_sock.send('NICK {0}\r\nUSER {0} * 0 :{0}\r\n'.format(irc_username).encode())
+    irc_sock.send('JOIN #{0}\r\n'.format(channel).encode())
+    irc_file = irc_sock.makefile()
+
+    while irc_file:
+        line = irc_file.readline()
+        print(line.rstrip())
+        response = line.split()
+
+        if response[0] == 'PING':
+            irc_file.send('PONG {}\r\n'.format(response[1]).encode())
+
+        elif response[1] == '433':
+            irc_sock.send('NICK {}\r\n'.format(irc_nick).encode())
+
+        elif response[1] == '001':
+            time.sleep(5)
+            # send notification
+            for msg in messages:
+                print(u'PRIVMSG #{} :{}'.format(channel, msg).encode("utf-8"))
+                irc_sock.send(u'PRIVMSG #{} :{}\r\n'.format(channel, msg).encode("utf-8"))
+            time.sleep(5)
+            return
+
+
+if __name__ == '__main__':
+    try:
+        notify()
+    except:
+        import traceback
+        print('ERROR: Failed to send notification: \n' + traceback.format_exc())
diff --git a/scripts/test/chutney-git-bisect.sh b/scripts/test/chutney-git-bisect.sh
new file mode 100755
index 0000000000..8a3f2c70c8
--- /dev/null
+++ b/scripts/test/chutney-git-bisect.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+# Compile tor and run chutney to find out if the current commit works
+#
+# Usage:
+# # Copy the script, so it doesn't change during bisection
+# cp scripts/test/chutney-git-bisect.sh .
+# git bisect run \
+#   ./chutney-git-bisect.sh [tries [build-dir [flavour [skip-flavour]]]]
+#
+# Runs chutney up to <tries> times (default 3), because some bugs involve race
+# conditions.
+# Changes to <build-dir> (default no cd) before running tests.
+# Runs chutney network <flavour> (default make test-network-all) as the test.
+# Skips the test if <skip-flavour> fails (default no skip).
+
+CHUTNEY_TRIES=3
+if [ ! -z "$1" ]; then
+    CHUTNEY_TRIES="$1"
+fi
+
+if [ ! -z "$2" ]; then
+    cd "$2"
+fi
+
+CHUTNEY_TEST_CMD="make test-network-all"
+if [ ! -z "$3" ]; then
+    CHUTNEY_TEST_CMD="$CHUTNEY_PATH/tools/test-network.sh --flavour $3"
+fi
+
+CHUTNEY_SKIP_ON_FAIL_CMD="true"
+if [ ! -z "$4" ]; then
+    CHUTNEY_SKIP_ON_FAIL_CMD="$CHUTNEY_PATH/tools/test-network.sh --flavour $4"
+fi
+
+CHUTNEY_BUILD_CMD_OR="make src/or/tor src/tools/tor-gencert"
+CHUTNEY_BUILD_CMD_APP="make src/app/tor src/tools/tor-gencert"
+if ! ( $CHUTNEY_BUILD_CMD_APP || $CHUTNEY_BUILD_CMD_OR ) ; then
+    echo "building '$CHUTNEY_BUILD_CMD_APP || $CHUTNEY_BUILD_CMD_OR' failed, skip"
+    exit 125
+fi
+
+if ! $CHUTNEY_SKIP_ON_FAIL_CMD ; then
+    echo "pre-condition '$CHUTNEY_SKIP_ON_FAIL_CMD' failed, skip"
+    exit 125
+fi
+
+i=1
+while [ "$i" -le "$CHUTNEY_TRIES" ]; do
+    echo
+    echo "Round $i/$CHUTNEY_TRIES:"
+    echo
+    if $CHUTNEY_TEST_CMD ; then
+        echo "test '$CHUTNEY_TEST_CMD' succeeded after $i/$CHUTNEY_TRIES attempts, good"
+        exit 0
+    fi
+    i=$[$i+1]
+done
+
+i=$[$i-1]
+echo "test '$CHUTNEY_TEST_CMD' failed $i/$CHUTNEY_TRIES attempts, bad"
+exit 1
diff --git a/scripts/test/cov-diff b/scripts/test/cov-diff
index 7da7f0be9d..6179dff63e 100755
--- a/scripts/test/cov-diff
+++ b/scripts/test/cov-diff
@@ -7,11 +7,15 @@
 DIRA="$1"
 DIRB="$2"
 
-for A in $DIRA/*; do
-  B=$DIRB/`basename $A`
-  perl -pe 's/^\s*\!*\d+:/        1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$A" > "$A.tmp"
-  perl -pe 's/^\s*\!*\d+:/        1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$B" > "$B.tmp"
-  diff -u "$A.tmp" "$B.tmp"
+for B in $DIRB/*; do
+  A=$DIRA/`basename $B`
+  if [ -f $A ]; then
+      perl -pe 's/^\s*\!*\d+(\*?):/        1$1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$A" > "$A.tmp"
+  else
+      cat /dev/null > "$A.tmp"
+  fi
+  perl -pe 's/^\s*\!*\d+(\*?):/        1$1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$B" > "$B.tmp"
+  diff -u "$A.tmp" "$B.tmp" |perl -pe 's/^((?:\+\+\+|---)(?:.*tmp))\s+.*/$1/;'
   rm "$A.tmp" "$B.tmp"
 done
 
diff --git a/scripts/test/cov-exclude b/scripts/test/cov-exclude
index 5117f11ec4..5cb9b1282d 100755
--- a/scripts/test/cov-exclude
+++ b/scripts/test/cov-exclude
@@ -26,3 +26,9 @@ if ($excluding or $exclude_this) {
   s{^\s*\#\#+:}{        x:};
   s{^   (\s*)(\d+):}{$1!!!$2:};
 }
+
+if (eof and $excluding) {
+    warn "Runaway LCOV_EXCL_START in $ARGV";
+    $excluding = 0;
+}
+
diff --git a/scripts/test/coverage b/scripts/test/coverage
index f4ae475828..b6e17abe25 100755
--- a/scripts/test/coverage
+++ b/scripts/test/coverage
@@ -7,14 +7,14 @@
 
 dst=$1
 
-for fn in src/or/*.c src/common/*.c; do
+for fn in src/core/*/*.c src/feature/*/*.c src/app/*/*.c src/lib/*/*.c; do
     BN=`basename $fn`
     DN=`dirname $fn`
     F=`echo $BN | sed -e 's/\.c$//;'`
     GC="${BN}.gcov"
     # Figure out the object file names
-    ONS=`echo ${DN}/src_*-${F}.o`
-    ONS_WILDCARD_LITERAL="${DN}/src_*-${F}.o"
+    ONS=$(echo "${DN}"/*testing_a-"${F}".o)
+    ONS_WILDCARD_LITERAL="${DN}/*testing_a-${F}.o"
     # If the wildcard didn't expand, no files
     if [ "$ONS" != "${ONS_WILDCARD_LITERAL}" ]
     then
@@ -29,7 +29,7 @@ for fn in src/or/*.c src/common/*.c; do
           gcov -o $on $fn
           if [ -e $GC ]
           then
-            if [ -n $dst ]
+            if [ -d "$dst" ]
             then
               mv $GC $dst/$GC
             fi
diff --git a/scripts/test/scan-build.sh b/scripts/test/scan-build.sh
index 36e69e6d00..8d126cbcee 100644..100755
--- a/scripts/test/scan-build.sh
+++ b/scripts/test/scan-build.sh
@@ -5,37 +5,76 @@
 # This script is used for running a bunch of clang scan-build checkers
 # on Tor.
 
+# These don't seem to cause false positives in our code, so let's turn
+# them on.
 CHECKERS="\
-    -disable-checker deadcode.DeadStores \
-    -enable-checker alpha.core.CastSize \
+    -enable-checker alpha.core.CallAndMessageUnInitRefArg \
     -enable-checker alpha.core.CastToStruct \
+    -enable-checker alpha.core.Conversion \
+    -enable-checker alpha.core.FixedAddr \
     -enable-checker alpha.core.IdenticalExpr \
+    -enable-checker alpha.core.PointerArithm \
     -enable-checker alpha.core.SizeofPtr \
-    -enable-checker alpha.security.ArrayBoundV2 \
+    -enable-checker alpha.core.TestAfterDivZero \
     -enable-checker alpha.security.MallocOverflow \
     -enable-checker alpha.security.ReturnPtrRange \
-    -enable-checker alpha.unix.SimpleStream
+    -enable-checker alpha.unix.BlockInCriticalSection \
+    -enable-checker alpha.unix.Chroot \
+    -enable-checker alpha.unix.PthreadLock \
+    -enable-checker alpha.unix.PthreadLock \
+    -enable-checker alpha.unix.SimpleStream \
+    -enable-checker alpha.unix.Stream \
     -enable-checker alpha.unix.cstring.BufferOverlap \
     -enable-checker alpha.unix.cstring.NotNullTerminated \
-    -enable-checker alpha.unix.cstring.OutOfBounds \
-    -enable-checker alpha.core.FixedAddr \
+    -enable-checker valist.CopyToSelf \
+    -enable-checker valist.Uninitialized \
+    -enable-checker valist.Unterminated \
+    -enable-checker security.FloatLoopCounter \
     -enable-checker security.insecureAPI.strcpy \
-    -enable-checker alpha.unix.PthreadLock \
-    -enable-checker alpha.core.PointerArithm \
-    -enable-checker alpha.core.TestAfterDivZero \
 "
 
+# These have high false-positive rates.
+EXTRA_CHECKERS="\
+    -enable-checker alpha.security.ArrayBoundV2 \
+    -enable-checker alpha.unix.cstring.OutOfBounds \
+    -enable-checker alpha.core.CastSize \
+"
+
+# These don't seem to generate anything useful
+NOISY_CHECKERS="\
+    -enable-checker alpha.clone.CloneChecker \
+    -enable-checker alpha.deadcode.UnreachableCode \
+"
+
+if test "x$SCAN_BUILD_OUTPUT" != "x"; then
+   OUTPUTARG="-o $SCAN_BUILD_OUTPUT"
+else
+   OUTPUTARG=""
+fi
+
 scan-build \
     $CHECKERS \
     ./configure
 
 scan-build \
+    make clean
+
+# Make this not get scanned for dead assignments, since it has lots of
+# dead assignments we don't care about.
+scan-build \
     $CHECKERS \
-    make -j2 -k
+    -disable-checker deadcode.DeadStores \
+    make -j5 -k ./src/ext/ed25519/ref10/libed25519_ref10.a
 
+scan-build \
+    $CHECKERS $OUTPUTARG \
+    make -j5 -k
+
+CHECKERS="\
+"
 
 # This one gives a false positive on every strcmp.
 #    -enable-checker alpha.core.PointerSub
 
 # Needs work
-#  alpha.unix.MallocWithAnnotations ??
+#    -enable-checker alpha.unix.MallocWithAnnotations
diff --git a/src/or/auth_dirs.inc b/src/app/config/auth_dirs.inc
index 278f08bfcf..278f08bfcf 100644
--- a/src/or/auth_dirs.inc
+++ b/src/app/config/auth_dirs.inc
diff --git a/src/or/config.c b/src/app/config/config.c
index 810f1e9a7a..0b1b758d96 100644
--- a/src/or/config.c
+++ b/src/app/config/config.c
@@ -1,61 +1,163 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file config.c
- * \brief Code to parse and interpret configuration files.
+ * \brief Code to interpret the user's configuration of Tor.
+ *
+ * This module handles torrc configuration file, including parsing it,
+ * combining it with torrc.defaults and the command line, allowing
+ * user changes to it (via editing and SIGHUP or via the control port),
+ * writing it back to disk (because of SAVECONF from the control port),
+ * and -- most importantly, acting on it.
+ *
+ * The module additionally has some tools for manipulating and
+ * inspecting values that are calculated as a result of the
+ * configured options.
+ *
+ * <h3>How to add new options</h3>
+ *
+ * To add new items to the torrc, there are a minimum of three places to edit:
+ * <ul>
+ *   <li>The or_options_t structure in or.h, where the options are stored.
+ *   <li>The option_vars_ array below in this module, which configures
+ *       the names of the torrc options, their types, their multiplicities,
+ *       and their mappings to fields in or_options_t.
+ *   <li>The manual in doc/tor.1.txt, to document what the new option
+ *       is, and how it works.
+ * </ul>
+ *
+ * Additionally, you might need to edit these places too:
+ * <ul>
+ *   <li>options_validate() below, in case you want to reject some possible
+ *       values of the new configuration option.
+ *   <li>options_transition_allowed() below, in case you need to
+ *       forbid some or all changes in the option while Tor is
+ *       running.
+ *   <li>options_transition_affects_workers(), in case changes in the option
+ *       might require Tor to relaunch or reconfigure its worker threads.
+ *   <li>options_transition_affects_descriptor(), in case changes in the
+ *       option might require a Tor relay to build and publish a new server
+ *       descriptor.
+ *   <li>options_act() and/or options_act_reversible(), in case there's some
+ *       action that needs to be taken immediately based on the option's
+ *       value.
+ * </ul>
+ *
+ * <h3>Changing the value of an option</h3>
+ *
+ * Because of the SAVECONF command from the control port, it's a bad
+ * idea to change the value of any user-configured option in the
+ * or_options_t.  If you want to sometimes do this anyway, we recommend
+ * that you create a secondary field in or_options_t; that you have the
+ * user option linked only to the secondary field; that you use the
+ * secondary field to initialize the one that Tor actually looks at; and that
+ * you use the one Tor looks as the one that you modify.
  **/
 
 #define CONFIG_PRIVATE
-#include "or.h"
-#include "compat.h"
-#include "addressmap.h"
-#include "channel.h"
-#include "circuitbuild.h"
-#include "circuitlist.h"
-#include "circuitmux.h"
-#include "circuitmux_ewma.h"
-#include "circuitstats.h"
-#include "config.h"
-#include "connection.h"
-#include "connection_edge.h"
-#include "connection_or.h"
-#include "control.h"
-#include "confparse.h"
-#include "cpuworker.h"
-#include "dirserv.h"
-#include "dirvote.h"
-#include "dns.h"
-#include "dos.h"
-#include "entrynodes.h"
-#include "geoip.h"
-#include "hibernate.h"
-#include "main.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "policies.h"
-#include "relay.h"
-#include "rendclient.h"
-#include "rendservice.h"
-#include "rephist.h"
-#include "router.h"
-#include "sandbox.h"
-#include "util.h"
-#include "routerlist.h"
-#include "routerset.h"
-#include "scheduler.h"
-#include "statefile.h"
-#include "transports.h"
-#include "ext_orport.h"
-#include "torgzip.h"
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "app/config/confparse.h"
+#include "app/config/statefile.h"
+#include "app/main/main.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/cpuworker.h"
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
+#include "core/or/channel.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitmux.h"
+#include "core/or/circuitmux_ewma.h"
+#include "core/or/circuitstats.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "core/or/dos.h"
+#include "core/or/policies.h"
+#include "core/or/relay.h"
+#include "core/or/scheduler.h"
+#include "feature/client/addressmap.h"
+#include "feature/client/bridges.h"
+#include "feature/client/entrynodes.h"
+#include "feature/client/transports.h"
+#include "feature/control/control.h"
+#include "feature/dirauth/bwauth.h"
+#include "feature/dirauth/guardfraction.h"
+#include "feature/dircache/consdiffmgr.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dircommon/voting_schedule.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/hs/hs_config.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nickname.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/relay/dns.h"
+#include "feature/relay/ext_orport.h"
+#include "feature/relay/routermode.h"
+#include "feature/rend/rendclient.h"
+#include "feature/rend/rendservice.h"
+#include "lib/geoip/geoip.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/stats/predict_ports.h"
+#include "feature/stats/rephist.h"
+#include "lib/compress/compress.h"
+#include "lib/crypt_ops/crypto_init.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/encoding/confline.h"
+#include "lib/log/git_revision.h"
+#include "lib/net/resolve.h"
+#include "lib/sandbox/sandbox.h"
+
+#ifdef ENABLE_NSS
+#include "lib/crypt_ops/crypto_nss_mgt.h"
+#else
+#include "lib/crypt_ops/crypto_openssl_mgt.h"
+#endif
+
 #ifdef _WIN32
 #include <shlobj.h>
 #endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#include "lib/meminfo/meminfo.h"
+#include "lib/osinfo/uname.h"
+#include "lib/process/daemon.h"
+#include "lib/process/pidfile.h"
+#include "lib/process/restrict.h"
+#include "lib/process/setuid.h"
+#include "lib/process/subprocess.h"
+#include "lib/net/gethostname.h"
+#include "lib/thread/numcpus.h"
 
-#include "procmon.h"
+#include "lib/encoding/keyval.h"
+#include "lib/fs/conffile.h"
+#include "lib/evloop/procmon.h"
+
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/recommend_pkg.h"
+#include "feature/dirauth/authmode.h"
+
+#include "core/or/connection_st.h"
+#include "core/or/port_cfg_st.h"
 
 #ifdef HAVE_SYSTEMD
 #   if defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__)
@@ -63,9 +165,9 @@
  * Coverity. Here's a kludge to unconfuse it.
  */
 #   define __INCLUDE_LEVEL__ 2
-#   endif
+#endif /* defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) */
 #include <systemd/sd-daemon.h>
-#endif
+#endif /* defined(HAVE_SYSTEMD) */
 
 /* Prefix used to indicate a Unix socket in a FooPort configuration. */
 static const char unix_socket_prefix[] = "unix:";
@@ -73,6 +175,15 @@ static const char unix_socket_prefix[] = "unix:";
  * configuration. */
 static const char unix_q_socket_prefix[] = "unix:\"";
 
+/* limits for TCP send and recv buffer size used for constrained sockets */
+#define MIN_CONSTRAINED_TCP_BUFFER 2048
+#define MAX_CONSTRAINED_TCP_BUFFER 262144  /* 256k */
+
+/** macro to help with the bulk rename of *DownloadSchedule to
+ * *DowloadInitialDelay . */
+#define DOWNLOAD_SCHEDULE(name) \
+  { #name "DownloadSchedule", #name "DownloadInitialDelay", 0, 1 }
+
 /** A list of abbreviations and aliases to map command-line options, obsolete
  * option names, or alternative option names, to their current values. */
 static config_abbrev_t option_abbrevs_[] = {
@@ -82,7 +193,6 @@ static config_abbrev_t option_abbrevs_[] = {
   PLURAL(AuthDirRejectCC),
   PLURAL(EntryNode),
   PLURAL(ExcludeNode),
-  PLURAL(Tor2webRendezvousPoint),
   PLURAL(FirewallPort),
   PLURAL(LongLivedPort),
   PLURAL(HiddenServiceNode),
@@ -118,24 +228,58 @@ static config_abbrev_t option_abbrevs_[] = {
   { "BridgeAuthoritativeDirectory", "BridgeAuthoritativeDir", 0, 0},
   { "HashedControlPassword", "__HashedControlSessionPassword", 1, 0},
   { "VirtualAddrNetwork", "VirtualAddrNetworkIPv4", 0, 0},
+  { "SocksSocketsGroupWritable", "UnixSocksGroupWritable", 0, 1},
+  { "_HSLayer2Nodes", "HSLayer2Nodes", 0, 1 },
+  { "_HSLayer3Nodes", "HSLayer3Nodes", 0, 1 },
+
+  DOWNLOAD_SCHEDULE(ClientBootstrapConsensusAuthority),
+  DOWNLOAD_SCHEDULE(ClientBootstrapConsensusAuthorityOnly),
+  DOWNLOAD_SCHEDULE(ClientBootstrapConsensusFallback),
+  DOWNLOAD_SCHEDULE(TestingBridge),
+  DOWNLOAD_SCHEDULE(TestingBridgeBootstrap),
+  DOWNLOAD_SCHEDULE(TestingClient),
+  DOWNLOAD_SCHEDULE(TestingClientConsensus),
+  DOWNLOAD_SCHEDULE(TestingServer),
+  DOWNLOAD_SCHEDULE(TestingServerConsensus),
+
   { NULL, NULL, 0, 0},
 };
 
+/** dummy instance of or_options_t, used for type-checking its
+ * members with CONF_CHECK_VAR_TYPE. */
+DUMMY_TYPECHECK_INSTANCE(or_options_t);
+
 /** An entry for config_vars: "The option <b>name</b> has type
  * CONFIG_TYPE_<b>conftype</b>, and corresponds to
  * or_options_t.<b>member</b>"
  */
 #define VAR(name,conftype,member,initvalue)                             \
-  { name, CONFIG_TYPE_ ## conftype, STRUCT_OFFSET(or_options_t, member), \
-      initvalue }
+  { name, CONFIG_TYPE_ ## conftype, offsetof(or_options_t, member),     \
+      initvalue CONF_TEST_MEMBERS(or_options_t, conftype, member) }
 /** As VAR, but the option name and member name are the same. */
 #define V(member,conftype,initvalue)                                    \
   VAR(#member, conftype, member, initvalue)
 /** An entry for config_vars: "The option <b>name</b> is obsolete." */
+#ifdef TOR_UNIT_TESTS
+#define OBSOLETE(name) { name, CONFIG_TYPE_OBSOLETE, 0, NULL, {.INT=NULL} }
+#else
 #define OBSOLETE(name) { name, CONFIG_TYPE_OBSOLETE, 0, NULL }
+#endif
+
+/**
+ * Macro to declare *Port options.  Each one comes in three entries.
+ * For example, most users should use "SocksPort" to configure the
+ * socks port, but TorBrowser wants to use __SocksPort so that it
+ * isn't stored by SAVECONF.  The SocksPortLines virtual option is
+ * used to query both options from the controller.
+ */
+#define VPORT(member)                                           \
+  VAR(#member "Lines", LINELIST_V, member ## _lines, NULL),     \
+  VAR(#member, LINELIST_S, member ## _lines, NULL),             \
+  VAR("__" #member, LINELIST_S, member ## _lines, NULL)
 
-#define VPORT(member,conftype,initvalue)                                    \
-  VAR(#member, conftype, member ## _lines, initvalue)
+/** UINT64_MAX as a decimal string */
+#define UINT64_MAX_STRING "18446744073709551615"
 
 /** Array of configuration options.  Until we disallow nonstandard
  * abbreviations, order is significant, since the first matching option will
@@ -146,11 +290,11 @@ static config_var_t option_vars_[] = {
   VAR("AccountingRule",          STRING,   AccountingRule_option,  "max"),
   V(AccountingStart,             STRING,   NULL),
   V(Address,                     STRING,   NULL),
-  V(AllowDotExit,                BOOL,     "0"),
-  V(AllowInvalidNodes,           CSV,      "middle,rendezvous"),
+  OBSOLETE("AllowDotExit"),
+  OBSOLETE("AllowInvalidNodes"),
   V(AllowNonRFC953Hostnames,     BOOL,     "0"),
-  V(AllowSingleHopCircuits,      BOOL,     "0"),
-  V(AllowSingleHopExits,         BOOL,     "0"),
+  OBSOLETE("AllowSingleHopCircuits"),
+  OBSOLETE("AllowSingleHopExits"),
   V(AlternateBridgeAuthority,    LINELIST, NULL),
   V(AlternateDirAuthority,       LINELIST, NULL),
   OBSOLETE("AlternateHSAuthority"),
@@ -163,14 +307,14 @@ static config_var_t option_vars_[] = {
   V(AuthDirInvalidCCs,           CSV,      ""),
   V(AuthDirFastGuarantee,        MEMUNIT,  "100 KB"),
   V(AuthDirGuardBWGuarantee,     MEMUNIT,  "2 MB"),
-  V(AuthDirPinKeys,              BOOL,     "0"),
+  V(AuthDirPinKeys,              BOOL,     "1"),
   V(AuthDirReject,               LINELIST, NULL),
   V(AuthDirRejectCCs,            CSV,      ""),
   OBSOLETE("AuthDirRejectUnlisted"),
   OBSOLETE("AuthDirListBadDirs"),
   V(AuthDirListBadExits,         BOOL,     "0"),
   V(AuthDirMaxServersPerAddr,    UINT,     "2"),
-  V(AuthDirMaxServersPerAuthAddr,UINT,     "5"),
+  OBSOLETE("AuthDirMaxServersPerAuthAddr"),
   V(AuthDirHasIPv6Connectivity,  BOOL,     "0"),
   VAR("AuthoritativeDirectory",  BOOL, AuthoritativeDir,    "0"),
   V(AutomapHostsOnResolve,       BOOL,     "0"),
@@ -183,12 +327,17 @@ static config_var_t option_vars_[] = {
   V(BridgePassword,              STRING,   NULL),
   V(BridgeRecordUsageByCountry,  BOOL,     "1"),
   V(BridgeRelay,                 BOOL,     "0"),
+  V(BridgeDistribution,          STRING,   NULL),
+  VAR("CacheDirectory",          FILENAME, CacheDirectory_option, NULL),
+  V(CacheDirectoryGroupReadable, AUTOBOOL,     "auto"),
   V(CellStatistics,              BOOL,     "0"),
+  V(PaddingStatistics,           BOOL,     "1"),
   V(LearnCircuitBuildTimeout,    BOOL,     "1"),
   V(CircuitBuildTimeout,         INTERVAL, "0"),
-  V(CircuitIdleTimeout,          INTERVAL, "1 hour"),
+  OBSOLETE("CircuitIdleTimeout"),
+  V(CircuitsAvailableTimeout,    INTERVAL, "0"),
   V(CircuitStreamTimeout,        INTERVAL, "0"),
-  V(CircuitPriorityHalflife,     DOUBLE,  "-100.0"), /*negative:'Use default'*/
+  V(CircuitPriorityHalflife,     DOUBLE,  "-1.0"), /*negative:'Use default'*/
   V(ClientDNSRejectInternalAddresses, BOOL,"1"),
   V(ClientOnly,                  BOOL,     "0"),
   V(ClientPreferIPv6ORPort,      AUTOBOOL, "auto"),
@@ -203,26 +352,26 @@ static config_var_t option_vars_[] = {
   V(ConstrainedSockets,          BOOL,     "0"),
   V(ConstrainedSockSize,         MEMUNIT,  "8192"),
   V(ContactInfo,                 STRING,   NULL),
-  V(ControlListenAddress,        LINELIST, NULL),
-  VPORT(ControlPort,                 LINELIST, NULL),
+  OBSOLETE("ControlListenAddress"),
+  VPORT(ControlPort),
   V(ControlPortFileGroupReadable,BOOL,     "0"),
   V(ControlPortWriteToFile,      FILENAME, NULL),
   V(ControlSocket,               LINELIST, NULL),
   V(ControlSocketsGroupWritable, BOOL,     "0"),
-  V(SocksSocketsGroupWritable,   BOOL,     "0"),
+  V(UnixSocksGroupWritable,    BOOL,     "0"),
   V(CookieAuthentication,        BOOL,     "0"),
   V(CookieAuthFileGroupReadable, BOOL,     "0"),
   V(CookieAuthFile,              STRING,   NULL),
   V(CountPrivateBandwidth,       BOOL,     "0"),
-  V(DataDirectory,               FILENAME, NULL),
+  VAR("DataDirectory",           FILENAME, DataDirectory_option, NULL),
   V(DataDirectoryGroupReadable,  BOOL,     "0"),
   V(DisableOOSCheck,             BOOL,     "1"),
   V(DisableNetwork,              BOOL,     "0"),
   V(DirAllowPrivateAddresses,    BOOL,     "0"),
   V(TestingAuthDirTimeToLearnReachability, INTERVAL, "30 minutes"),
-  V(DirListenAddress,            LINELIST, NULL),
+  OBSOLETE("DirListenAddress"),
   V(DirPolicy,                   LINELIST, NULL),
-  VPORT(DirPort,                     LINELIST, NULL),
+  VPORT(DirPort),
   V(DirPortFrontPage,            FILENAME, NULL),
   VAR("DirReqStatistics",        BOOL,     DirReqStatistics_option, "1"),
   VAR("DirAuthority",            LINELIST, DirAuthorities, NULL),
@@ -240,8 +389,8 @@ static config_var_t option_vars_[] = {
   OBSOLETE("DisableIOCP"),
   OBSOLETE("DisableV2DirectoryInfo_"),
   OBSOLETE("DynamicDHGroups"),
-  VPORT(DNSPort,                     LINELIST, NULL),
-  V(DNSListenAddress,            LINELIST, NULL),
+  VPORT(DNSPort),
+  OBSOLETE("DNSListenAddress"),
   /* DoS circuit creation options. */
   V(DoSCircuitCreationEnabled,   AUTOBOOL, "auto"),
   V(DoSCircuitCreationMinConnections,      UINT, "0"),
@@ -258,14 +407,14 @@ static config_var_t option_vars_[] = {
   V(DownloadExtraInfo,           BOOL,     "0"),
   V(TestingEnableConnBwEvent,    BOOL,     "0"),
   V(TestingEnableCellStatsEvent, BOOL,     "0"),
-  V(TestingEnableTbEmptyEvent,   BOOL,     "0"),
+  OBSOLETE("TestingEnableTbEmptyEvent"),
   V(EnforceDistinctSubnets,      BOOL,     "1"),
   V(EntryNodes,                  ROUTERSET,   NULL),
   V(EntryStatistics,             BOOL,     "0"),
   V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "10 minutes"),
   V(ExcludeNodes,                ROUTERSET, NULL),
   V(ExcludeExitNodes,            ROUTERSET, NULL),
-  V(ExcludeSingleHopRelays,      BOOL,     "1"),
+  OBSOLETE("ExcludeSingleHopRelays"),
   V(ExitNodes,                   ROUTERSET, NULL),
   V(ExitPolicy,                  LINELIST, NULL),
   V(ExitPolicyRejectPrivate,     BOOL,     "1"),
@@ -273,17 +422,19 @@ static config_var_t option_vars_[] = {
   V(ExitPortStatistics,          BOOL,     "0"),
   V(ExtendAllowPrivateAddresses, BOOL,     "0"),
   V(ExitRelay,                   AUTOBOOL, "auto"),
-  VPORT(ExtORPort,               LINELIST, NULL),
+  VPORT(ExtORPort),
   V(ExtORPortCookieAuthFile,     STRING,   NULL),
   V(ExtORPortCookieAuthFileGroupReadable, BOOL, "0"),
   V(ExtraInfoStatistics,         BOOL,     "1"),
+  V(ExtendByEd25519ID,           AUTOBOOL, "auto"),
   V(FallbackDir,                 LINELIST, NULL),
+
   V(UseDefaultFallbackDirs,      BOOL,     "1"),
 
   OBSOLETE("FallbackNetworkstatusFile"),
   V(FascistFirewall,             BOOL,     "0"),
   V(FirewallPorts,               CSV,      ""),
-  V(FastFirstHopPK,              AUTOBOOL, "auto"),
+  OBSOLETE("FastFirstHopPK"),
   V(FetchDirInfoEarly,           BOOL,     "0"),
   V(FetchDirInfoExtraEarly,      BOOL,     "0"),
   V(FetchServerDescriptors,      BOOL,     "1"),
@@ -299,11 +450,12 @@ static config_var_t option_vars_[] = {
     SHARE_DATADIR PATH_SEPARATOR "tor" PATH_SEPARATOR "geoip"),
   V(GeoIPv6File,                 FILENAME,
     SHARE_DATADIR PATH_SEPARATOR "tor" PATH_SEPARATOR "geoip6"),
-#endif
+#endif /* defined(_WIN32) */
   OBSOLETE("Group"),
   V(GuardLifetime,               INTERVAL, "0 minutes"),
   V(HardwareAccel,               BOOL,     "0"),
   V(HeartbeatPeriod,             INTERVAL, "6 hours"),
+  V(MainloopStats,               BOOL,     "0"),
   V(AccelName,                   STRING,   NULL),
   V(AccelDir,                    FILENAME, NULL),
   V(HashedControlPassword,       LINELIST, NULL),
@@ -318,16 +470,19 @@ static config_var_t option_vars_[] = {
   VAR("HiddenServiceMaxStreams",LINELIST_S, RendConfigLines, NULL),
   VAR("HiddenServiceMaxStreamsCloseCircuit",LINELIST_S, RendConfigLines, NULL),
   VAR("HiddenServiceNumIntroductionPoints", LINELIST_S, RendConfigLines, NULL),
-  V(HiddenServiceStatistics,     BOOL,     "1"),
+  VAR("HiddenServiceExportCircuitID", LINELIST_S,  RendConfigLines, NULL),
+  VAR("HiddenServiceStatistics", BOOL, HiddenServiceStatistics_option, "1"),
   V(HidServAuth,                 LINELIST, NULL),
-  V(CloseHSClientCircuitsImmediatelyOnTimeout, BOOL, "0"),
-  V(CloseHSServiceRendCircuitsImmediatelyOnTimeout, BOOL, "0"),
+  V(ClientOnionAuthDir,          FILENAME, NULL),
+  OBSOLETE("CloseHSClientCircuitsImmediatelyOnTimeout"),
+  OBSOLETE("CloseHSServiceRendCircuitsImmediatelyOnTimeout"),
   V(HiddenServiceSingleHopMode,  BOOL,     "0"),
   V(HiddenServiceNonAnonymousMode,BOOL,    "0"),
   V(HTTPProxy,                   STRING,   NULL),
   V(HTTPProxyAuthenticator,      STRING,   NULL),
   V(HTTPSProxy,                  STRING,   NULL),
   V(HTTPSProxyAuthenticator,     STRING,   NULL),
+  VPORT(HTTPTunnelPort),
   V(IPv6Exit,                    BOOL,     "0"),
   VAR("ServerTransportPlugin",   LINELIST, ServerTransportPlugin,  NULL),
   V(ServerTransportListenAddr,   LINELIST, NULL),
@@ -337,6 +492,10 @@ static config_var_t option_vars_[] = {
   V(Socks5Proxy,                 STRING,   NULL),
   V(Socks5ProxyUsername,         STRING,   NULL),
   V(Socks5ProxyPassword,         STRING,   NULL),
+  VAR("KeyDirectory",            FILENAME, KeyDirectory_option, NULL),
+  V(KeyDirectoryGroupReadable,   BOOL,     "0"),
+  VAR("HSLayer2Nodes",           ROUTERSET,  HSLayer2Nodes,  NULL),
+  VAR("HSLayer3Nodes",           ROUTERSET,  HSLayer3Nodes,  NULL),
   V(KeepalivePeriod,             INTERVAL, "5 minutes"),
   V(KeepBindCapabilities,            AUTOBOOL, "auto"),
   VAR("Log",                     LINELIST, Logs,             NULL),
@@ -344,33 +503,39 @@ static config_var_t option_vars_[] = {
   V(LogTimeGranularity,          MSEC_INTERVAL, "1 second"),
   V(TruncateLogFile,             BOOL,     "0"),
   V(SyslogIdentityTag,           STRING,   NULL),
+  V(AndroidIdentityTag,          STRING,   NULL),
   V(LongLivedPorts,              CSV,
         "21,22,706,1863,5050,5190,5222,5223,6523,6667,6697,8300"),
   VAR("MapAddress",              LINELIST, AddressMap,           NULL),
   V(MaxAdvertisedBandwidth,      MEMUNIT,  "1 GB"),
   V(MaxCircuitDirtiness,         INTERVAL, "10 minutes"),
   V(MaxClientCircuitsPending,    UINT,     "32"),
+  V(MaxConsensusAgeForDiffs,     INTERVAL, "0 seconds"),
   VAR("MaxMemInQueues",          MEMUNIT,   MaxMemInQueues_raw, "0"),
   OBSOLETE("MaxOnionsPending"),
   V(MaxOnionQueueDelay,          MSEC_INTERVAL, "1750 msec"),
   V(MaxUnparseableDescSizeToLog, MEMUNIT, "10 MB"),
   V(MinMeasuredBWsForAuthToIgnoreAdvertised, INT, "500"),
-  V(MyFamily,                    STRING,   NULL),
+  VAR("MyFamily",                LINELIST, MyFamily_lines,       NULL),
   V(NewCircuitPeriod,            INTERVAL, "30 seconds"),
   OBSOLETE("NamingAuthoritativeDirectory"),
-  V(NATDListenAddress,           LINELIST, NULL),
-  VPORT(NATDPort,                    LINELIST, NULL),
+  OBSOLETE("NATDListenAddress"),
+  VPORT(NATDPort),
   V(Nickname,                    STRING,   NULL),
-  V(PredictedPortsRelevanceTime,  INTERVAL, "1 hour"),
-  V(WarnUnsafeSocks,              BOOL,     "1"),
+  OBSOLETE("PredictedPortsRelevanceTime"),
+  OBSOLETE("WarnUnsafeSocks"),
   VAR("NodeFamily",              LINELIST, NodeFamilies,         NULL),
+  V(NoExec,                      BOOL,     "0"),
   V(NumCPUs,                     UINT,     "0"),
   V(NumDirectoryGuards,          UINT,     "0"),
   V(NumEntryGuards,              UINT,     "0"),
+  V(NumPrimaryGuards,            UINT,     "0"),
   V(OfflineMasterKey,            BOOL,     "0"),
-  V(ORListenAddress,             LINELIST, NULL),
-  VPORT(ORPort,                      LINELIST, NULL),
+  OBSOLETE("ORListenAddress"),
+  VPORT(ORPort),
   V(OutboundBindAddress,         LINELIST,   NULL),
+  V(OutboundBindAddressOR,       LINELIST,   NULL),
+  V(OutboundBindAddressExit,     LINELIST,   NULL),
 
   OBSOLETE("PathBiasDisableRate"),
   V(PathBiasCircThreshold,       INT,      "-1"),
@@ -403,8 +568,8 @@ static config_var_t option_vars_[] = {
   V(TestingSigningKeySlop,           INTERVAL, "1 day"),
 
   V(OptimisticData,              AUTOBOOL, "auto"),
-  V(PortForwarding,              BOOL,     "0"),
-  V(PortForwardingHelper,        FILENAME, "tor-fw-helper"),
+  OBSOLETE("PortForwarding"),
+  OBSOLETE("PortForwardingHelper"),
   OBSOLETE("PreferTunneledDirConns"),
   V(ProtocolWarnings,            BOOL,     "0"),
   V(PublishServerDescriptor,     CSV,      "1"),
@@ -416,6 +581,8 @@ static config_var_t option_vars_[] = {
   V(RecommendedClientVersions,   LINELIST, NULL),
   V(RecommendedServerVersions,   LINELIST, NULL),
   V(RecommendedPackages,         LINELIST, NULL),
+  V(ReducedConnectionPadding,    BOOL,     "0"),
+  V(ConnectionPadding,           AUTOBOOL, "auto"),
   V(RefuseUnknownExits,          AUTOBOOL, "auto"),
   V(RejectPlaintextPorts,        CSV,      ""),
   V(RelayBandwidthBurst,         MEMUNIT,  "0"),
@@ -423,6 +590,7 @@ static config_var_t option_vars_[] = {
   V(RendPostPeriod,              INTERVAL, "1 hour"),
   V(RephistTrackTime,            INTERVAL, "24 hours"),
   V(RunAsDaemon,                 BOOL,     "0"),
+  V(ReducedExitPolicy,           BOOL,     "0"),
   OBSOLETE("RunTesting"), // currently unused
   V(Sandbox,                     BOOL,     "0"),
   V(SafeLogging,                 STRING,   "1"),
@@ -435,13 +603,16 @@ static config_var_t option_vars_[] = {
   V(ServerDNSSearchDomains,      BOOL,     "0"),
   V(ServerDNSTestAddresses,      CSV,
       "www.google.com,www.mit.edu,www.yahoo.com,www.slashdot.org"),
-  V(SchedulerLowWaterMark__,     MEMUNIT,  "100 MB"),
-  V(SchedulerHighWaterMark__,    MEMUNIT,  "101 MB"),
-  V(SchedulerMaxFlushCells__,    UINT,     "1000"),
+  OBSOLETE("SchedulerLowWaterMark__"),
+  OBSOLETE("SchedulerHighWaterMark__"),
+  OBSOLETE("SchedulerMaxFlushCells__"),
+  V(KISTSchedRunInterval,        MSEC_INTERVAL, "0 msec"),
+  V(KISTSockBufSizeFactor,       DOUBLE,   "1.0"),
+  V(Schedulers,                  CSV,      "KIST,KISTLite,Vanilla"),
   V(ShutdownWaitLength,          INTERVAL, "30 seconds"),
-  V(SocksListenAddress,          LINELIST, NULL),
+  OBSOLETE("SocksListenAddress"),
   V(SocksPolicy,                 LINELIST, NULL),
-  VPORT(SocksPort,                   LINELIST, NULL),
+  VPORT(SocksPort),
   V(SocksTimeout,                INTERVAL, "2 minutes"),
   V(SSLKeyLifetime,              INTERVAL, "0"),
   OBSOLETE("StrictEntryNodes"),
@@ -450,25 +621,26 @@ static config_var_t option_vars_[] = {
   OBSOLETE("Support022HiddenServices"),
   V(TestSocks,                   BOOL,     "0"),
   V(TokenBucketRefillInterval,   MSEC_INTERVAL, "100 msec"),
-  V(Tor2webMode,                 BOOL,     "0"),
-  V(Tor2webRendezvousPoints,      ROUTERSET, NULL),
-  V(TLSECGroup,                  STRING,   NULL),
+  OBSOLETE("Tor2webMode"),
+  OBSOLETE("Tor2webRendezvousPoints"),
+  OBSOLETE("TLSECGroup"),
   V(TrackHostExits,              CSV,      NULL),
   V(TrackHostExitsExpire,        INTERVAL, "30 minutes"),
-  V(TransListenAddress,          LINELIST, NULL),
-  VPORT(TransPort,                   LINELIST, NULL),
+  OBSOLETE("TransListenAddress"),
+  VPORT(TransPort),
   V(TransProxyType,              STRING,   "default"),
   OBSOLETE("TunnelDirConns"),
   V(UpdateBridgesFromAuthority,  BOOL,     "0"),
   V(UseBridges,                  BOOL,     "0"),
   VAR("UseEntryGuards",          BOOL,     UseEntryGuards_option, "1"),
-  V(UseEntryGuardsAsDirGuards,   BOOL,     "1"),
+  OBSOLETE("UseEntryGuardsAsDirGuards"),
   V(UseGuardFraction,            AUTOBOOL, "auto"),
   V(UseMicrodescriptors,         AUTOBOOL, "auto"),
   OBSOLETE("UseNTorHandshake"),
   V(User,                        STRING,   NULL),
   OBSOLETE("UserspaceIOCPBuffers"),
   V(AuthDirSharedRandomness,     BOOL,     "1"),
+  V(AuthDirTestEd25519LinkKeys,  BOOL,     "1"),
   OBSOLETE("V1AuthoritativeDirectory"),
   OBSOLETE("V2AuthoritativeDirectory"),
   VAR("V3AuthoritativeDirectory",BOOL, V3AuthoritativeDir,   "0"),
@@ -493,58 +665,58 @@ static config_var_t option_vars_[] = {
   VAR("__ReloadTorrcOnSIGHUP",   BOOL,  ReloadTorrcOnSIGHUP,      "1"),
   VAR("__AllDirActionsPrivate",  BOOL,  AllDirActionsPrivate,     "0"),
   VAR("__DisablePredictedCircuits",BOOL,DisablePredictedCircuits, "0"),
+  VAR("__DisableSignalHandlers", BOOL,  DisableSignalHandlers,    "0"),
   VAR("__LeaveStreamsUnattached",BOOL,  LeaveStreamsUnattached,   "0"),
   VAR("__HashedControlSessionPassword", LINELIST, HashedControlSessionPassword,
       NULL),
   VAR("__OwningControllerProcess",STRING,OwningControllerProcess, NULL),
+  VAR("__OwningControllerFD", UINT64, OwningControllerFD, UINT64_MAX_STRING),
   V(MinUptimeHidServDirectoryV2, INTERVAL, "96 hours"),
-  V(TestingServerDownloadSchedule, CSV_INTERVAL, "0, 0, 0, 60, 60, 120, "
-                                 "300, 900, 2147483647"),
-  V(TestingClientDownloadSchedule, CSV_INTERVAL, "0, 0, 60, 300, 600, "
-                                 "2147483647"),
-  V(TestingServerConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 60, "
-                                 "300, 600, 1800, 1800, 1800, 1800, "
-                                 "1800, 3600, 7200"),
-  V(TestingClientConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 60, "
-                                 "300, 600, 1800, 3600, 3600, 3600, "
-                                 "10800, 21600, 43200"),
+  V(TestingServerDownloadInitialDelay, CSV_INTERVAL, "0"),
+  V(TestingClientDownloadInitialDelay, CSV_INTERVAL, "0"),
+  V(TestingServerConsensusDownloadInitialDelay, CSV_INTERVAL, "0"),
+  V(TestingClientConsensusDownloadInitialDelay, CSV_INTERVAL, "0"),
   /* With the ClientBootstrapConsensus*Download* below:
    * Clients with only authorities will try:
-   *  - 3 authorities over 10 seconds, then wait 60 minutes.
+   *  - at least 3 authorities over 10 seconds, then exponentially backoff,
+   *    with the next attempt 3-21 seconds later,
    * Clients with authorities and fallbacks will try:
-   *  - 2 authorities and 4 fallbacks over 21 seconds, then wait 60 minutes.
+   *  - at least 2 authorities and 4 fallbacks over 21 seconds, then
+   *    exponentially backoff, with the next attempts 4-33 seconds later,
    * Clients will also retry when an application request arrives.
-   * After a number of failed reqests, clients retry every 3 days + 1 hour.
+   * After a number of failed requests, clients retry every 3 days + 1 hour.
    *
    * Clients used to try 2 authorities over 10 seconds, then wait for
    * 60 minutes or an application request.
    *
    * When clients have authorities and fallbacks available, they use these
    * schedules: (we stagger the times to avoid thundering herds) */
-  V(ClientBootstrapConsensusAuthorityDownloadSchedule, CSV_INTERVAL,
-    "6, 11, 3600, 10800, 25200, 54000, 111600, 262800" /* 3 days + 1 hour */),
-  V(ClientBootstrapConsensusFallbackDownloadSchedule, CSV_INTERVAL,
-    "0, 1, 4, 11, 3600, 10800, 25200, 54000, 111600, 262800"),
+  V(ClientBootstrapConsensusAuthorityDownloadInitialDelay, CSV_INTERVAL, "6"),
+  V(ClientBootstrapConsensusFallbackDownloadInitialDelay, CSV_INTERVAL, "0"),
   /* When clients only have authorities available, they use this schedule: */
-  V(ClientBootstrapConsensusAuthorityOnlyDownloadSchedule, CSV_INTERVAL,
-    "0, 3, 7, 3600, 10800, 25200, 54000, 111600, 262800"),
+  V(ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay, CSV_INTERVAL,
+    "0"),
   /* We don't want to overwhelm slow networks (or mirrors whose replies are
    * blocked), but we also don't want to fail if only some mirrors are
    * blackholed. Clients will try 3 directories simultaneously.
    * (Relays never use simultaneous connections.) */
   V(ClientBootstrapConsensusMaxInProgressTries, UINT, "3"),
-  V(TestingBridgeDownloadSchedule, CSV_INTERVAL, "1200, 900, 900, 3600"),
+  /* When a client has any running bridges, check each bridge occasionally,
+    * whether or not that bridge is actually up. */
+  V(TestingBridgeDownloadInitialDelay, CSV_INTERVAL,"10800"),
+  /* When a client is just starting, or has no running bridges, check each
+   * bridge a few times quickly, and then try again later. These schedules
+   * are much longer than the other schedules, because we try each and every
+   * configured bridge with this schedule. */
+  V(TestingBridgeBootstrapDownloadInitialDelay, CSV_INTERVAL, "0"),
   V(TestingClientMaxIntervalWithoutRequest, INTERVAL, "10 minutes"),
   V(TestingDirConnectionMaxStall, INTERVAL, "5 minutes"),
-  V(TestingConsensusMaxDownloadTries, UINT, "8"),
-  /* Since we try connections rapidly and simultaneously, we can afford
-   * to give up earlier. (This protects against overloading directories.) */
-  V(ClientBootstrapConsensusMaxDownloadTries, UINT, "7"),
-  /* We want to give up much earlier if we're only using authorities. */
-  V(ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries, UINT, "4"),
-  V(TestingDescriptorMaxDownloadTries, UINT, "8"),
-  V(TestingMicrodescMaxDownloadTries, UINT, "8"),
-  V(TestingCertMaxDownloadTries, UINT, "8"),
+  OBSOLETE("TestingConsensusMaxDownloadTries"),
+  OBSOLETE("ClientBootstrapConsensusMaxDownloadTries"),
+  OBSOLETE("ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries"),
+  OBSOLETE("TestingDescriptorMaxDownloadTries"),
+  OBSOLETE("TestingMicrodescMaxDownloadTries"),
+  OBSOLETE("TestingCertMaxDownloadTries"),
   V(TestingDirAuthVoteExit, ROUTERSET, NULL),
   V(TestingDirAuthVoteExitIsStrict,  BOOL,     "0"),
   V(TestingDirAuthVoteGuard, ROUTERSET, NULL),
@@ -553,27 +725,21 @@ static config_var_t option_vars_[] = {
   V(TestingDirAuthVoteHSDirIsStrict,  BOOL,     "0"),
   VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "0"),
 
-  { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL }
+  END_OF_CONFIG_VARS
 };
 
 /** Override default values with these if the user sets the TestingTorNetwork
  * option. */
 static const config_var_t testing_tor_network_defaults[] = {
-  V(ServerDNSAllowBrokenConfig,  BOOL,     "1"),
   V(DirAllowPrivateAddresses,    BOOL,     "1"),
   V(EnforceDistinctSubnets,      BOOL,     "0"),
   V(AssumeReachable,             BOOL,     "1"),
   V(AuthDirMaxServersPerAddr,    UINT,     "0"),
-  V(AuthDirMaxServersPerAuthAddr,UINT,     "0"),
-  V(ClientBootstrapConsensusAuthorityDownloadSchedule, CSV_INTERVAL,
-    "0, 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 8, 16, 32, 60"),
-  V(ClientBootstrapConsensusFallbackDownloadSchedule, CSV_INTERVAL,
-    "0, 1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 8, 16, 32, 60"),
-  V(ClientBootstrapConsensusAuthorityOnlyDownloadSchedule, CSV_INTERVAL,
-    "0, 1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 8, 16, 32, 60"),
-  V(ClientBootstrapConsensusMaxDownloadTries, UINT, "80"),
-  V(ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries, UINT, "80"),
-  V(ClientDNSRejectInternalAddresses, BOOL,"0"), // deprecated in 0.2.9.2-alpha
+  V(ClientBootstrapConsensusAuthorityDownloadInitialDelay, CSV_INTERVAL, "0"),
+  V(ClientBootstrapConsensusFallbackDownloadInitialDelay, CSV_INTERVAL, "0"),
+  V(ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay, CSV_INTERVAL,
+    "0"),
+  V(ClientDNSRejectInternalAddresses, BOOL,"0"),
   V(ClientRejectInternalAddresses, BOOL,   "0"),
   V(CountPrivateBandwidth,       BOOL,     "1"),
   V(ExitPolicyRejectPrivate,     BOOL,     "0"),
@@ -584,32 +750,23 @@ static const config_var_t testing_tor_network_defaults[] = {
   V(TestingV3AuthInitialVotingInterval, INTERVAL, "150 seconds"),
   V(TestingV3AuthInitialVoteDelay, INTERVAL, "20 seconds"),
   V(TestingV3AuthInitialDistDelay, INTERVAL, "20 seconds"),
-  V(TestingV3AuthVotingStartOffset, INTERVAL, "0"),
   V(TestingAuthDirTimeToLearnReachability, INTERVAL, "0 minutes"),
   V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "0 minutes"),
   V(MinUptimeHidServDirectoryV2, INTERVAL, "0 minutes"),
-  V(TestingServerDownloadSchedule, CSV_INTERVAL, "0, 0, 0, 5, 10, 15, "
-                                 "20, 30, 60"),
-  V(TestingClientDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, 15, 20, "
-                                 "30, 60"),
-  V(TestingServerConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, "
-                                 "15, 20, 30, 60"),
-  V(TestingClientConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, "
-                                 "15, 20, 30, 60"),
-  V(TestingBridgeDownloadSchedule, CSV_INTERVAL, "60, 30, 30, 60"),
+  V(TestingServerDownloadInitialDelay, CSV_INTERVAL, "0"),
+  V(TestingClientDownloadInitialDelay, CSV_INTERVAL, "0"),
+  V(TestingServerConsensusDownloadInitialDelay, CSV_INTERVAL, "0"),
+  V(TestingClientConsensusDownloadInitialDelay, CSV_INTERVAL, "0"),
+  V(TestingBridgeDownloadInitialDelay, CSV_INTERVAL, "10"),
+  V(TestingBridgeBootstrapDownloadInitialDelay, CSV_INTERVAL, "0"),
   V(TestingClientMaxIntervalWithoutRequest, INTERVAL, "5 seconds"),
   V(TestingDirConnectionMaxStall, INTERVAL, "30 seconds"),
-  V(TestingConsensusMaxDownloadTries, UINT, "80"),
-  V(TestingDescriptorMaxDownloadTries, UINT, "80"),
-  V(TestingMicrodescMaxDownloadTries, UINT, "80"),
-  V(TestingCertMaxDownloadTries, UINT, "80"),
   V(TestingEnableConnBwEvent,    BOOL,     "1"),
   V(TestingEnableCellStatsEvent, BOOL,     "1"),
-  V(TestingEnableTbEmptyEvent,   BOOL,     "1"),
   VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "1"),
   V(RendPostPeriod,              INTERVAL, "2 minutes"),
 
-  { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL }
+  END_OF_CONFIG_VARS
 };
 
 #undef VAR
@@ -617,39 +774,19 @@ static const config_var_t testing_tor_network_defaults[] = {
 #undef OBSOLETE
 
 static const config_deprecation_t option_deprecation_notes_[] = {
-  /* Deprecated since 0.2.9.2-alpha... */
-  { "AllowDotExit", "Unrestricted use of the .exit notation can be used for "
-    "a wide variety of application-level attacks." },
-  { "AllowInvalidNodes", "There is no reason to enable this option; at best "
-    "it will make you easier to track." },
-  { "AllowSingleHopCircuits", "Almost no relays actually allow single-hop "
-    "exits, making this option pointless." },
-  { "AllowSingleHopExits", "Turning this on will make your relay easier "
-    "to abuse." },
-  { "ClientDNSRejectInternalAddresses", "Turning this on makes your client "
-    "easier to fingerprint, and may open you to esoteric attacks." },
-  { "ExcludeSingleHopRelays", "Turning it on makes your client easier to "
-    "fingerprint." },
-  { "FastFirstHopPK", "Changing this option does not make your client more "
-    "secure, but does make it easier to fingerprint." },
-  { "CloseHSClientCircuitsImmediatelyOnTimeout", "This option makes your "
-    "client easier to fingerprint." },
-  { "CloseHSServiceRendCircuitsImmediatelyOnTimeout", "This option makes "
-    "your hidden services easier to fingerprint." },
-  { "WarnUnsafeSocks", "Changing this option makes it easier for you "
-    "to accidentally lose your anonymity by leaking DNS information" },
-  { "TLSECGroup", "The default is a nice secure choice; the other option "
-    "is less secure." },
-  { "ControlListenAddress", "Use ControlPort instead." },
-  { "DirListenAddress", "Use DirPort instead, possibly with the "
-    "NoAdvertise sub-option" },
-  { "DNSListenAddress", "Use DNSPort instead." },
-  { "SocksListenAddress", "Use SocksPort instead." },
-  { "TransListenAddress", "Use TransPort instead." },
-  { "NATDListenAddress", "Use NATDPort instead." },
-  { "ORListenAddress", "Use ORPort instead, possibly with the "
-    "NoAdvertise sub-option" },
-  /* End of options deprecated since 0.2.9.2-alpha. */
+  /* Deprecated since 0.3.2.0-alpha. */
+  { "HTTPProxy", "It only applies to direct unencrypted HTTP connections "
+    "to your directory server, which your Tor probably wasn't using." },
+  { "HTTPProxyAuthenticator", "HTTPProxy is deprecated in favor of HTTPSProxy "
+    "which should be used with HTTPSProxyAuthenticator." },
+  /* End of options deprecated since 0.3.2.1-alpha */
+
+  /* Options deprecated since 0.3.2.2-alpha */
+  { "ReachableDirAddresses", "It has no effect on relays, and has had no "
+    "effect on clients since 0.2.8." },
+  { "ClientPreferIPv6DirPort", "It has no effect on relays, and has had no "
+    "effect on clients since 0.2.8." },
+  /* End of options deprecated since 0.3.2.2-alpha. */
 
   { NULL, NULL }
 };
@@ -665,7 +802,11 @@ static int options_transition_affects_workers(
       const or_options_t *old_options, const or_options_t *new_options);
 static int options_transition_affects_descriptor(
       const or_options_t *old_options, const or_options_t *new_options);
-static int check_nickname_list(char **lst, const char *name, char **msg);
+static int options_transition_affects_dirauth_timing(
+      const or_options_t *old_options, const or_options_t *new_options);
+static int normalize_nickname_list(config_line_t **normalized_out,
+                                   const config_line_t *lst, const char *name,
+                                   char **msg);
 static char *get_bindaddr_from_transport_listen_line(const char *line,
                                                      const char *transport);
 static int parse_ports(or_options_t *options, int validate_only,
@@ -674,8 +815,7 @@ static int parse_ports(or_options_t *options, int validate_only,
 static int check_server_ports(const smartlist_t *ports,
                               const or_options_t *options,
                               int *num_low_ports_out);
-
-static int validate_data_directory(or_options_t *options);
+static int validate_data_directories(or_options_t *options);
 static int write_configuration_file(const char *fname,
                                     const or_options_t *options);
 static int options_init_logs(const or_options_t *old_options,
@@ -690,8 +830,9 @@ static void config_maybe_load_geoip_files_(const or_options_t *options,
 static int options_validate_cb(void *old_options, void *options,
                                void *default_options,
                                int from_setconf, char **msg);
-static uint64_t compute_real_max_mem_in_queues(const uint64_t val,
-                                               int log_guess);
+static void options_free_cb(void *options);
+static void cleanup_protocol_warning_severity_level(void);
+static void set_protocol_warning_severity_level(int warning_severity);
 
 /** Magic value for or_options_t. */
 #define OR_OPTIONS_MAGIC 9090909
@@ -700,11 +841,12 @@ static uint64_t compute_real_max_mem_in_queues(const uint64_t val,
 STATIC config_format_t options_format = {
   sizeof(or_options_t),
   OR_OPTIONS_MAGIC,
-  STRUCT_OFFSET(or_options_t, magic_),
+  offsetof(or_options_t, magic_),
   option_abbrevs_,
   option_deprecation_notes_,
   option_vars_,
   options_validate_cb,
+  options_free_cb,
   NULL
 };
 
@@ -720,7 +862,7 @@ static or_options_t *global_default_options = NULL;
 /** Name of most recently read torrc file. */
 static char *torrc_fname = NULL;
 /** Name of the most recently read torrc-defaults file.*/
-static char *torrc_defaults_fname;
+static char *torrc_defaults_fname = NULL;
 /** Configuration options set by command line. */
 static config_line_t *global_cmdline_options = NULL;
 /** Non-configuration options set by the command line */
@@ -731,6 +873,11 @@ static int have_parsed_cmdline = 0;
 static char *global_dirfrontpagecontents = NULL;
 /** List of port_cfg_t for all configured ports. */
 static smartlist_t *configured_ports = NULL;
+/** True iff we're currently validating options, and any calls to
+ * get_options() are likely to be bugs. */
+static int in_option_validation = 0;
+/* True iff we've initialized libevent */
+static int libevent_initialized = 0;
 
 /** Return the contents of our frontpage string, or NULL if not configured. */
 MOCK_IMPL(const char*,
@@ -744,6 +891,7 @@ MOCK_IMPL(or_options_t *,
 get_options_mutable, (void))
 {
   tor_assert(global_options);
+  tor_assert_nonfatal(! in_option_validation);
   return global_options;
 }
 
@@ -774,9 +922,13 @@ set_options(or_options_t *new_val, char **msg)
     return -1;
   }
   if (options_act(old_options) < 0) { /* acting on the options failed. die. */
-    log_err(LD_BUG,
-            "Acting on config options left us in a broken state. Dying.");
-    exit(1);
+    if (! tor_event_loop_shutdown_is_pending()) {
+      log_err(LD_BUG,
+              "Acting on config options left us in a broken state. Dying.");
+      tor_shutdown_event_loop_and_exit(1);
+    }
+    global_options = old_options;
+    return -1;
   }
   /* Issues a CONF_CHANGED event to notify controller of the change. If Tor is
    * just starting up then the old_options will be undefined. */
@@ -802,7 +954,7 @@ set_options(or_options_t *new_val, char **msg)
             tor_free(line);
           }
         } else {
-          smartlist_add(elements, tor_strdup(options_format.vars[i].name));
+          smartlist_add_strdup(elements, options_format.vars[i].name);
           smartlist_add(elements, NULL);
         }
       }
@@ -812,14 +964,17 @@ set_options(or_options_t *new_val, char **msg)
     smartlist_free(elements);
   }
 
-  if (old_options != global_options)
+  if (old_options != global_options) {
     or_options_free(old_options);
+    /* If we are here it means we've successfully applied the new options and
+     * that the global options have been changed to the new values. We'll
+     * check if we need to remove or add periodic events. */
+    periodic_events_on_new_options(global_options);
+  }
 
   return 0;
 }
 
-extern const char tor_git_revision[]; /* from tor_main.c */
-
 /** The version of this Tor process, as parsed. */
 static char *the_tor_version = NULL;
 /** A shorter version of this Tor process's version, for export in our router
@@ -859,7 +1014,7 @@ get_short_version(void)
 /** Release additional memory allocated in options
  */
 STATIC void
-or_options_free(or_options_t *options)
+or_options_free_(or_options_t *options)
 {
   if (!options)
     return;
@@ -870,9 +1025,21 @@ or_options_free(or_options_t *options)
                       rs, routerset_free(rs));
     smartlist_free(options->NodeFamilySets);
   }
+  if (options->SchedulerTypes_) {
+    SMARTLIST_FOREACH(options->SchedulerTypes_, int *, i, tor_free(i));
+    smartlist_free(options->SchedulerTypes_);
+  }
+  if (options->FilesOpenedByIncludes) {
+    SMARTLIST_FOREACH(options->FilesOpenedByIncludes, char *, f, tor_free(f));
+    smartlist_free(options->FilesOpenedByIncludes);
+  }
+  tor_free(options->DataDirectory);
+  tor_free(options->CacheDirectory);
+  tor_free(options->KeyDirectory);
   tor_free(options->BridgePassword_AuthDigest_);
   tor_free(options->command_arg);
   tor_free(options->master_key_fname);
+  config_free_lines(options->MyFamily);
   config_free(&options_format, options);
 }
 
@@ -905,6 +1072,11 @@ config_free_all(void)
 
   tor_free(the_short_tor_version);
   tor_free(the_tor_version);
+
+  cleanup_protocol_warning_severity_level();
+
+  have_parsed_cmdline = 0;
+  libevent_initialized = 0;
 }
 
 /** Make <b>address</b> -- a piece of information related to our operation as
@@ -914,10 +1086,14 @@ config_free_all(void)
  * (We return "[scrubbed]" if SafeLogging is "1", and address otherwise.)
  */
 const char *
-safe_str_client(const char *address)
+safe_str_client_opts(const or_options_t *options, const char *address)
 {
   tor_assert(address);
-  if (get_options()->SafeLogging_ == SAFELOG_SCRUB_ALL)
+  if (!options) {
+    options = get_options();
+  }
+
+  if (options->SafeLogging_ == SAFELOG_SCRUB_ALL)
     return "[scrubbed]";
   else
     return address;
@@ -931,10 +1107,14 @@ safe_str_client(const char *address)
  * otherwise.)
  */
 const char *
-safe_str(const char *address)
+safe_str_opts(const or_options_t *options, const char *address)
 {
   tor_assert(address);
-  if (get_options()->SafeLogging_ != SAFELOG_SCRUB_NONE)
+  if (!options) {
+    options = get_options();
+  }
+
+  if (options->SafeLogging_ != SAFELOG_SCRUB_NONE)
     return "[scrubbed]";
   else
     return address;
@@ -964,6 +1144,52 @@ escaped_safe_str(const char *address)
     return escaped(address);
 }
 
+/**
+ * The severity level that should be used for warnings of severity
+ * LOG_PROTOCOL_WARN.
+ *
+ * We keep this outside the options, and we use an atomic_counter_t, in case
+ * one thread needs to use LOG_PROTOCOL_WARN while an option transition is
+ * happening in the main thread.
+ */
+static atomic_counter_t protocol_warning_severity_level;
+
+/** Return the severity level that should be used for warnings of severity
+ * LOG_PROTOCOL_WARN. */
+int
+get_protocol_warning_severity_level(void)
+{
+  return (int) atomic_counter_get(&protocol_warning_severity_level);
+}
+
+/** Set the protocol warning severity level to <b>severity</b>. */
+static void
+set_protocol_warning_severity_level(int warning_severity)
+{
+  atomic_counter_exchange(&protocol_warning_severity_level,
+                          warning_severity);
+}
+
+/**
+ * Initialize the log warning severity level for protocol warnings. Call
+ * only once at startup.
+ */
+void
+init_protocol_warning_severity_level(void)
+{
+  atomic_counter_init(&protocol_warning_severity_level);
+  set_protocol_warning_severity_level(LOG_WARN);
+}
+
+/**
+ * Tear down protocol_warning_severity_level.
+ */
+static void
+cleanup_protocol_warning_severity_level(void)
+{
+   atomic_counter_destroy(&protocol_warning_severity_level);
+}
+
 /** List of default directory authorities */
 
 static const char *default_authorities[] = {
@@ -1129,6 +1355,69 @@ consider_adding_dir_servers(const or_options_t *options,
   return 0;
 }
 
+/**
+ * Make sure that <b>directory</b> exists, with appropriate ownership and
+ * permissions (as modified by <b>group_readable</b>). If <b>create</b>,
+ * create the directory if it is missing. Return 0 on success.
+ * On failure, return -1 and set *<b>msg_out</b>.
+ */
+static int
+check_and_create_data_directory(int create,
+                                const char *directory,
+                                int group_readable,
+                                const char *owner,
+                                char **msg_out)
+{
+  cpd_check_t cpd_opts = create ? CPD_CREATE : CPD_CHECK;
+  if (group_readable)
+      cpd_opts |= CPD_GROUP_READ;
+  if (check_private_dir(directory,
+                        cpd_opts,
+                        owner) < 0) {
+    tor_asprintf(msg_out,
+                 "Couldn't %s private data directory \"%s\"",
+                 create ? "create" : "access",
+                 directory);
+    return -1;
+  }
+
+#ifndef _WIN32
+  if (group_readable) {
+    /* Only new dirs created get new opts, also enforce group read. */
+    if (chmod(directory, 0750)) {
+      log_warn(LD_FS,"Unable to make %s group-readable: %s",
+               directory, strerror(errno));
+    }
+  }
+#endif /* !defined(_WIN32) */
+
+  return 0;
+}
+
+/**
+ * Ensure that our keys directory exists, with appropriate permissions.
+ * Return 0 on success, -1 on failure.
+ */
+int
+create_keys_directory(const or_options_t *options)
+{
+  /* Make sure DataDirectory exists, and is private. */
+  cpd_check_t cpd_opts = CPD_CREATE;
+  if (options->DataDirectoryGroupReadable)
+    cpd_opts |= CPD_GROUP_READ;
+  if (check_private_dir(options->DataDirectory, cpd_opts, options->User)) {
+    log_err(LD_OR, "Can't create/check datadirectory %s",
+            options->DataDirectory);
+    return -1;
+  }
+
+  /* Check the key directory. */
+  if (check_private_dir(options->KeyDirectory, CPD_CREATE, options->User)) {
+    return -1;
+  }
+  return 0;
+}
+
 /* Helps determine flags to pass to switch_id. */
 static int have_low_ports = -1;
 
@@ -1142,8 +1431,6 @@ static int
 options_act_reversible(const or_options_t *old_options, char **msg)
 {
   smartlist_t *new_listeners = smartlist_new();
-  smartlist_t *replaced_listeners = smartlist_new();
-  static int libevent_initialized = 0;
   or_options_t *options = get_options_mutable();
   int running_tor = options->command == CMD_RUN_TOR;
   int set_conn_limit = 0;
@@ -1155,8 +1442,11 @@ options_act_reversible(const or_options_t *old_options, char **msg)
    * the subprocess.  Libevent bases can't be reliably inherited across
    * processes. */
   if (running_tor && options->RunAsDaemon) {
+    if (! start_daemon_has_been_called())
+      crypto_prefork();
     /* No need to roll back, since you can't change the value. */
-    start_daemon();
+    if (start_daemon())
+      crypto_postfork();
   }
 
 #ifdef HAVE_SYSTEMD
@@ -1170,13 +1460,13 @@ options_act_reversible(const or_options_t *old_options, char **msg)
                       "on this OS/with this build.");
     goto rollback;
   }
-#else
+#else /* !(!defined(HAVE_SYS_UN_H)) */
   if (options->ControlSocketsGroupWritable && !options->ControlSocket) {
     *msg = tor_strdup("Setting ControlSocketGroupWritable without setting"
                       "a ControlSocket makes no sense.");
     goto rollback;
   }
-#endif
+#endif /* !defined(HAVE_SYS_UN_H) */
 
   if (running_tor) {
     int n_ports=0;
@@ -1223,12 +1513,11 @@ options_act_reversible(const or_options_t *old_options, char **msg)
     consider_hibernation(time(NULL));
 
     /* Launch the listeners.  (We do this before we setuid, so we can bind to
-     * ports under 1024.)  We don't want to rebind if we're hibernating. If
-     * networking is disabled, this will close all but the control listeners,
-     * but disable those. */
+     * ports under 1024.)  We don't want to rebind if we're hibernating or
+     * shutting down. If networking is disabled, this will close all but the
+     * control listeners, but disable those. */
     if (!we_are_hibernating()) {
-      if (retry_all_listeners(replaced_listeners, new_listeners,
-                              options->DisableNetwork) < 0) {
+      if (retry_all_listeners(new_listeners, options->DisableNetwork) < 0) {
         *msg = tor_strdup("Failed to bind one of the listener ports.");
         goto rollback;
       }
@@ -1253,7 +1542,7 @@ options_act_reversible(const or_options_t *old_options, char **msg)
       goto rollback;
     }
   }
-#endif
+#endif /* defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) */
 
   /* Attempt to lock all current and future memory with mlockall() only once */
   if (options->DisableAllSwap) {
@@ -1283,29 +1572,47 @@ options_act_reversible(const or_options_t *old_options, char **msg)
   }
 
   /* Ensure data directory is private; create if possible. */
-  cpd_check_t cpd_opts = running_tor ? CPD_CREATE : CPD_CHECK;
-  if (options->DataDirectoryGroupReadable)
-      cpd_opts |= CPD_GROUP_READ;
-  if (check_private_dir(options->DataDirectory,
-                        cpd_opts,
-                        options->User)<0) {
-    tor_asprintf(msg,
-              "Couldn't access/create private data directory \"%s\"",
-              options->DataDirectory);
-
+  /* It's okay to do this in "options_act_reversible()" even though it isn't
+   * actually reversible, since you can't change the DataDirectory while
+   * Tor is running. */
+  if (check_and_create_data_directory(running_tor /* create */,
+                                      options->DataDirectory,
+                                      options->DataDirectoryGroupReadable,
+                                      options->User,
+                                      msg) < 0) {
+    goto done;
+  }
+  if (check_and_create_data_directory(running_tor /* create */,
+                                      options->KeyDirectory,
+                                      options->KeyDirectoryGroupReadable,
+                                      options->User,
+                                      msg) < 0) {
     goto done;
-    /* No need to roll back, since you can't change the value. */
   }
 
-#ifndef _WIN32
-  if (options->DataDirectoryGroupReadable) {
-    /* Only new dirs created get new opts, also enforce group read. */
-    if (chmod(options->DataDirectory, 0750)) {
-      log_warn(LD_FS,"Unable to make %s group-readable: %s",
-               options->DataDirectory, strerror(errno));
-    }
+  /* We need to handle the group-readable flag for the cache directory
+   * specially, since the directory defaults to being the same as the
+   * DataDirectory. */
+  int cache_dir_group_readable;
+  if (options->CacheDirectoryGroupReadable != -1) {
+    /* If the user specified a value, use their setting */
+    cache_dir_group_readable = options->CacheDirectoryGroupReadable;
+  } else if (!strcmp(options->CacheDirectory, options->DataDirectory)) {
+    /* If the user left the value as "auto", and the cache is the same as the
+     * datadirectory, use the datadirectory setting.
+     */
+    cache_dir_group_readable = options->DataDirectoryGroupReadable;
+  } else {
+    /* Otherwise, "auto" means "not group readable". */
+    cache_dir_group_readable = 0;
+  }
+  if (check_and_create_data_directory(running_tor /* create */,
+                                      options->CacheDirectory,
+                                      cache_dir_group_readable,
+                                      options->User,
+                                      msg) < 0) {
+    goto done;
   }
-#endif
 
   /* Bail out at this point if we're not going to be a client or server:
    * we don't run Tor itself. */
@@ -1328,6 +1635,7 @@ options_act_reversible(const or_options_t *old_options, char **msg)
       tor_malloc_zero(sizeof(log_severity_list_t));
     close_temp_logs();
     add_callback_log(severity, control_event_logmsg);
+    logs_set_pending_callback_callback(control_event_logmsg_pending);
     control_adjust_event_log_severity();
     tor_free(severity);
     tor_log_update_sigsafe_err_fds();
@@ -1362,17 +1670,6 @@ options_act_reversible(const or_options_t *old_options, char **msg)
                "Overwrite the log afterwards.", badness);
   }
 
-  SMARTLIST_FOREACH(replaced_listeners, connection_t *, conn,
-  {
-    int marked = conn->marked_for_close;
-    log_notice(LD_NET, "Closing old %s on %s:%d",
-               conn_type_to_string(conn->type), conn->address, conn->port);
-    connection_close_immediate(conn);
-    if (!marked) {
-      connection_mark_for_close(conn);
-    }
-  });
-
   if (set_conn_limit) {
     /*
      * If we adjusted the conn limit, recompute the OOS threshold too
@@ -1426,7 +1723,6 @@ options_act_reversible(const or_options_t *old_options, char **msg)
 
  done:
   smartlist_free(new_listeners);
-  smartlist_free(replaced_listeners);
   return r;
 }
 
@@ -1435,14 +1731,14 @@ options_act_reversible(const or_options_t *old_options, char **msg)
 int
 options_need_geoip_info(const or_options_t *options, const char **reason_out)
 {
-  int bridge_usage =
-    options->BridgeRelay && options->BridgeRecordUsageByCountry;
+  int bridge_usage = should_record_bridge_info(options);
   int routerset_usage =
     routerset_needs_geoip(options->EntryNodes) ||
     routerset_needs_geoip(options->ExitNodes) ||
     routerset_needs_geoip(options->ExcludeExitNodes) ||
     routerset_needs_geoip(options->ExcludeNodes) ||
-    routerset_needs_geoip(options->Tor2webRendezvousPoints);
+    routerset_needs_geoip(options->HSLayer2Nodes) ||
+    routerset_needs_geoip(options->HSLayer3Nodes);
 
   if (routerset_usage && reason_out) {
     *reason_out = "We've been configured to use (or avoid) nodes in certain "
@@ -1482,19 +1778,70 @@ get_effective_bwburst(const or_options_t *options)
   return (uint32_t)bw;
 }
 
-/** Return True if any changes from <b>old_options</b> to
- * <b>new_options</b> needs us to refresh our TLS context. */
+/* Used in the various options_transition_affects* functions. */
+#define YES_IF_CHANGED_BOOL(opt) \
+  if (!CFG_EQ_BOOL(old_options, new_options, opt)) return 1;
+#define YES_IF_CHANGED_INT(opt) \
+  if (!CFG_EQ_INT(old_options, new_options, opt)) return 1;
+#define YES_IF_CHANGED_STRING(opt) \
+  if (!CFG_EQ_STRING(old_options, new_options, opt)) return 1;
+#define YES_IF_CHANGED_LINELIST(opt) \
+  if (!CFG_EQ_LINELIST(old_options, new_options, opt)) return 1;
+#define YES_IF_CHANGED_SMARTLIST(opt) \
+  if (!CFG_EQ_SMARTLIST(old_options, new_options, opt)) return 1;
+#define YES_IF_CHANGED_ROUTERSET(opt) \
+  if (!CFG_EQ_ROUTERSET(old_options, new_options, opt)) return 1;
+
+/**
+ * Return true if changing the configuration from <b>old</b> to <b>new</b>
+ * affects the guard subsystem.
+ */
 static int
-options_transition_requires_fresh_tls_context(const or_options_t *old_options,
-                                              const or_options_t *new_options)
+options_transition_affects_guards(const or_options_t *old_options,
+                                  const or_options_t *new_options)
 {
+  /* NOTE: Make sure this function stays in sync with
+   * node_passes_guard_filter */
+  tor_assert(old_options);
   tor_assert(new_options);
 
-  if (!old_options)
-    return 0;
+  YES_IF_CHANGED_BOOL(UseEntryGuards);
+  YES_IF_CHANGED_BOOL(UseBridges);
+  YES_IF_CHANGED_BOOL(ClientUseIPv4);
+  YES_IF_CHANGED_BOOL(ClientUseIPv6);
+  YES_IF_CHANGED_BOOL(FascistFirewall);
+  YES_IF_CHANGED_ROUTERSET(ExcludeNodes);
+  YES_IF_CHANGED_ROUTERSET(EntryNodes);
+  YES_IF_CHANGED_SMARTLIST(FirewallPorts);
+  YES_IF_CHANGED_LINELIST(Bridges);
+  YES_IF_CHANGED_LINELIST(ReachableORAddresses);
+  YES_IF_CHANGED_LINELIST(ReachableDirAddresses);
+
+  return 0;
+}
+
+/**
+ * Return true if changing the configuration from <b>old</b> to <b>new</b>
+ * affects the timing of the voting subsystem
+ */
+static int
+options_transition_affects_dirauth_timing(const or_options_t *old_options,
+                                          const or_options_t *new_options)
+{
+  tor_assert(old_options);
+  tor_assert(new_options);
 
-  if (!opt_streq(old_options->TLSECGroup, new_options->TLSECGroup))
+  if (authdir_mode_v3(old_options) != authdir_mode_v3(new_options))
     return 1;
+  if (! authdir_mode_v3(new_options))
+    return 0;
+  YES_IF_CHANGED_INT(V3AuthVotingInterval);
+  YES_IF_CHANGED_INT(V3AuthVoteDelay);
+  YES_IF_CHANGED_INT(V3AuthDistDelay);
+  YES_IF_CHANGED_INT(TestingV3AuthInitialVotingInterval);
+  YES_IF_CHANGED_INT(TestingV3AuthInitialVoteDelay);
+  YES_IF_CHANGED_INT(TestingV3AuthInitialDistDelay);
+  YES_IF_CHANGED_INT(TestingV3AuthVotingStartOffset);
 
   return 0;
 }
@@ -1517,7 +1864,12 @@ options_act(const or_options_t *old_options)
   char *msg=NULL;
   const int transition_affects_workers =
     old_options && options_transition_affects_workers(old_options, options);
-  int old_ewma_enabled;
+  const int transition_affects_guards =
+    old_options && options_transition_affects_guards(old_options, options);
+
+  if (options->NoExec || options->Sandbox) {
+    tor_disable_spawning_background_processes();
+  }
 
   /* disable ptrace and later, other basic debugging techniques */
   {
@@ -1553,33 +1905,22 @@ options_act(const or_options_t *old_options)
       return -1;
   }
 
-  if (consider_adding_dir_servers(options, old_options) < 0)
+  {
+    int warning_severity = options->ProtocolWarnings ? LOG_WARN : LOG_INFO;
+    set_protocol_warning_severity_level(warning_severity);
+  }
+
+  if (consider_adding_dir_servers(options, old_options) < 0) {
+    // XXXX This should get validated earlier, and committed here, to
+    // XXXX lower opportunities for reaching an error case.
     return -1;
+  }
 
   if (rend_non_anonymous_mode_enabled(options)) {
     log_warn(LD_GENERAL, "This copy of Tor was compiled or configured to run "
              "in a non-anonymous mode. It will provide NO ANONYMITY.");
   }
 
-#ifdef ENABLE_TOR2WEB_MODE
-/* LCOV_EXCL_START */
-  if (!options->Tor2webMode) {
-    log_err(LD_CONFIG, "This copy of Tor was compiled to run in "
-            "'tor2web mode'. It can only be run with the Tor2webMode torrc "
-            "option enabled.");
-    return -1;
-  }
-/* LCOV_EXCL_STOP */
-#else
-  if (options->Tor2webMode) {
-    log_err(LD_CONFIG, "This copy of Tor was not compiled to run in "
-            "'tor2web mode'. It cannot be run with the Tor2webMode torrc "
-            "option enabled. To enable Tor2webMode recompile with the "
-            "--enable-tor2web-mode option.");
-    return -1;
-  }
-#endif
-
   /* If we are a bridge with a pluggable transport proxy but no
      Extended ORPort, inform the user that they are missing out. */
   if (server_mode(options) && options->ServerTransportPlugin &&
@@ -1598,25 +1939,44 @@ options_act(const or_options_t *old_options)
     for (cl = options->Bridges; cl; cl = cl->next) {
       bridge_line_t *bridge_line = parse_bridge_line(cl->value);
       if (!bridge_line) {
+        // LCOV_EXCL_START
         log_warn(LD_BUG,
                  "Previously validated Bridge line could not be added!");
         return -1;
+        // LCOV_EXCL_STOP
       }
       bridge_add_from_config(bridge_line);
     }
     sweep_bridge_list();
   }
 
-  if (running_tor && rend_config_services(options, 0)<0) {
+  if (running_tor && hs_config_service_all(options, 0)<0) {
+    // LCOV_EXCL_START
     log_warn(LD_BUG,
        "Previously validated hidden services line could not be added!");
     return -1;
+    // LCOV_EXCL_STOP
   }
 
-  if (running_tor && rend_parse_service_authorization(options, 0) < 0) {
+  if (running_tor && hs_config_client_auth_all(options, 0) < 0) {
+    // LCOV_EXCL_START
     log_warn(LD_BUG, "Previously validated client authorization for "
                      "hidden services could not be added!");
     return -1;
+    // LCOV_EXCL_STOP
+  }
+
+  if (running_tor && !old_options &&
+      options->OwningControllerFD != UINT64_MAX) {
+    const unsigned ctrl_flags =
+      CC_LOCAL_FD_IS_OWNER |
+      CC_LOCAL_FD_IS_AUTHENTICATED;
+    tor_socket_t ctrl_sock = (tor_socket_t)options->OwningControllerFD;
+    if (control_connection_add_local_fd(ctrl_sock, ctrl_flags) < 0) {
+      log_warn(LD_CONFIG, "Could not add local controller connection with "
+               "given FD.");
+      return -1;
+    }
   }
 
   /* Load state */
@@ -1639,10 +1999,12 @@ options_act(const or_options_t *old_options)
     if (options->ClientTransportPlugin) {
       for (cl = options->ClientTransportPlugin; cl; cl = cl->next) {
         if (parse_transport_line(options, cl->value, 0, 0) < 0) {
+          // LCOV_EXCL_START
           log_warn(LD_BUG,
                    "Previously validated ClientTransportPlugin line "
                    "could not be added!");
           return -1;
+          // LCOV_EXCL_STOP
         }
       }
     }
@@ -1650,10 +2012,12 @@ options_act(const or_options_t *old_options)
     if (options->ServerTransportPlugin && server_mode(options)) {
       for (cl = options->ServerTransportPlugin; cl; cl = cl->next) {
         if (parse_transport_line(options, cl->value, 0, 1) < 0) {
+          // LCOV_EXCL_START
           log_warn(LD_BUG,
                    "Previously validated ServerTransportPlugin line "
                    "could not be added!");
           return -1;
+          // LCOV_EXCL_STOP
         }
       }
     }
@@ -1678,6 +2042,9 @@ options_act(const or_options_t *old_options)
     finish_daemon(options->DataDirectory);
   }
 
+  /* See whether we need to enable/disable our once-a-second timer. */
+  reschedule_per_second_timer();
+
   /* We want to reinit keys as needed before we do much of anything else:
      keys are important, and other things can depend on them. */
   if (transition_affects_workers ||
@@ -1687,19 +2054,16 @@ options_act(const or_options_t *old_options)
       log_warn(LD_BUG,"Error initializing keys; exiting");
       return -1;
     }
-  } else if (old_options &&
-             options_transition_requires_fresh_tls_context(old_options,
-                                                           options)) {
-    if (router_initialize_tls_context() < 0) {
-      log_warn(LD_BUG,"Error initializing TLS context.");
-      return -1;
-    }
   }
 
   /* Write our PID to the PID file. If we do not have write permissions we
-   * will log a warning */
+   * will log a warning and exit. */
   if (options->PidFile && !sandbox_is_active()) {
-    write_pidfile(options->PidFile);
+    if (write_pidfile(options->PidFile) < 0) {
+      log_err(LD_CONFIG, "Unable to write PIDFile %s",
+              escaped(options->PidFile));
+      return -1;
+    }
   }
 
   /* Register addressmap directives */
@@ -1714,6 +2078,15 @@ options_act(const or_options_t *old_options)
     return -1;
   }
 
+  if (server_mode(options)) {
+    static int cdm_initialized = 0;
+    if (cdm_initialized == 0) {
+      cdm_initialized = 1;
+      consdiffmgr_configure(NULL);
+      consdiffmgr_validate();
+    }
+  }
+
   if (init_control_cookie_authentication(options->CookieAuthentication) < 0) {
     log_warn(LD_CONFIG,"Error creating control cookie authentication file.");
     return -1;
@@ -1722,35 +2095,27 @@ options_act(const or_options_t *old_options)
   monitor_owning_controller_process(options->OwningControllerProcess);
 
   /* reload keys as needed for rendezvous services. */
-  if (rend_service_load_all_keys(NULL)<0) {
+  if (hs_service_load_all_keys() < 0) {
     log_warn(LD_GENERAL,"Error loading rendezvous service keys");
     return -1;
   }
 
-  /* Set up scheduler thresholds */
-  scheduler_set_watermarks((uint32_t)options->SchedulerLowWaterMark__,
-                           (uint32_t)options->SchedulerHighWaterMark__,
-                           (options->SchedulerMaxFlushCells__ > 0) ?
-                           options->SchedulerMaxFlushCells__ : 1000);
+  /* Inform the scheduler subsystem that a configuration changed happened. It
+   * might be a change of scheduler or parameter. */
+  scheduler_conf_changed();
 
   /* Set up accounting */
   if (accounting_parse_options(options, 0)<0) {
-    log_warn(LD_CONFIG,"Error in accounting options");
+    // LCOV_EXCL_START
+    log_warn(LD_BUG,"Error in previously validated accounting options");
     return -1;
+    // LCOV_EXCL_STOP
   }
   if (accounting_is_enabled(options))
     configure_accounting(time(NULL));
 
-  old_ewma_enabled = cell_ewma_enabled();
   /* Change the cell EWMA settings */
-  cell_ewma_set_scale_factor(options, networkstatus_get_latest_consensus());
-  /* If we just enabled ewma, set the cmux policy on all active channels */
-  if (cell_ewma_enabled() && !old_ewma_enabled) {
-    channel_set_cmux_policy_everywhere(&ewma_policy);
-  } else if (!cell_ewma_enabled() && old_ewma_enabled) {
-    /* Turn it off everywhere */
-    channel_set_cmux_policy_everywhere(NULL);
-  }
+  cmux_ewma_set_options(options, networkstatus_get_latest_consensus());
 
   /* Update the BridgePassword's hashed version as needed.  We store this as a
    * digest so that we can do side-channel-proof comparisons on it.
@@ -1759,6 +2124,7 @@ options_act(const or_options_t *old_options)
     char *http_authenticator;
     http_authenticator = alloc_http_authenticator(options->BridgePassword);
     if (!http_authenticator) {
+      // XXXX This should get validated in options_validate().
       log_warn(LD_BUG, "Unable to allocate HTTP authenticator. Not setting "
                "BridgePassword.");
       return -1;
@@ -1771,9 +2137,12 @@ options_act(const or_options_t *old_options)
   }
 
   if (parse_outbound_addresses(options, 0, &msg) < 0) {
-    log_warn(LD_BUG, "Failed parsing outbound bind addresses: %s", msg);
+    // LCOV_EXCL_START
+    log_warn(LD_BUG, "Failed parsing previously validated outbound "
+             "bind addresses: %s", msg);
     tor_free(msg);
     return -1;
+    // LCOV_EXCL_STOP
   }
 
   config_maybe_load_geoip_files_(options, old_options);
@@ -1794,6 +2163,7 @@ options_act(const or_options_t *old_options)
   if (old_options) {
     int revise_trackexithosts = 0;
     int revise_automap_entries = 0;
+    int abandon_circuits = 0;
     if ((options->UseEntryGuards && !old_options->UseEntryGuards) ||
         options->UseBridges != old_options->UseBridges ||
         (options->UseBridges &&
@@ -1803,13 +2173,25 @@ options_act(const or_options_t *old_options)
                          options->ExcludeExitNodes) ||
         !routerset_equal(old_options->EntryNodes, options->EntryNodes) ||
         !routerset_equal(old_options->ExitNodes, options->ExitNodes) ||
-        !routerset_equal(old_options->Tor2webRendezvousPoints,
-                         options->Tor2webRendezvousPoints) ||
+        !routerset_equal(old_options->HSLayer2Nodes,
+                         options->HSLayer2Nodes) ||
+        !routerset_equal(old_options->HSLayer3Nodes,
+                         options->HSLayer3Nodes) ||
         options->StrictNodes != old_options->StrictNodes) {
       log_info(LD_CIRC,
                "Changed to using entry guards or bridges, or changed "
                "preferred or excluded node lists. "
                "Abandoning previous circuits.");
+      abandon_circuits = 1;
+    }
+
+    if (transition_affects_guards) {
+      if (guards_update_all()) {
+        abandon_circuits = 1;
+      }
+    }
+
+    if (abandon_circuits) {
       circuit_mark_all_unused_circs();
       circuit_mark_all_dirty_circs_as_unusable();
       revise_trackexithosts = 1;
@@ -1840,7 +2222,7 @@ options_act(const or_options_t *old_options)
       addressmap_clear_invalid_automaps(options);
 
 /* How long should we delay counting bridge stats after becoming a bridge?
- * We use this so we don't count people who used our bridge thinking it is
+ * We use this so we don't count clients who used our bridge thinking it is
  * a relay. If you change this, don't forget to change the log message
  * below. It's 4 hours (the time it takes to stop being used by clients)
  * plus some extra time for clock skew. */
@@ -1868,9 +2250,16 @@ options_act(const or_options_t *old_options)
     if (transition_affects_workers) {
       log_info(LD_GENERAL,
                "Worker-related options changed. Rotating workers.");
+      const int server_mode_turned_on =
+        server_mode(options) && !server_mode(old_options);
+      const int dir_server_mode_turned_on =
+        dir_server_mode(options) && !dir_server_mode(old_options);
 
-      if (server_mode(options) && !server_mode(old_options)) {
+      if (server_mode_turned_on || dir_server_mode_turned_on) {
         cpu_init();
+      }
+
+      if (server_mode_turned_on) {
         ip_address_changed(0);
         if (have_completed_a_circuit() || !any_predicted_circuits(time(NULL)))
           inform_testing_reachability();
@@ -1886,11 +2275,23 @@ options_act(const or_options_t *old_options)
     if (options->PerConnBWRate != old_options->PerConnBWRate ||
         options->PerConnBWBurst != old_options->PerConnBWBurst)
       connection_or_update_token_buckets(get_connection_array(), options);
+
+    if (options->BandwidthRate != old_options->BandwidthRate ||
+        options->BandwidthBurst != old_options->BandwidthBurst ||
+        options->RelayBandwidthRate != old_options->RelayBandwidthRate ||
+        options->RelayBandwidthBurst != old_options->RelayBandwidthBurst)
+      connection_bucket_adjust(options);
+
+    if (options->MainloopStats != old_options->MainloopStats) {
+      reset_main_loop_counters();
+    }
   }
 
   /* Only collect directory-request statistics on relays and bridges. */
   options->DirReqStatistics = options->DirReqStatistics_option &&
     server_mode(options);
+  options->HiddenServiceStatistics =
+    options->HiddenServiceStatistics_option && server_mode(options);
 
   if (options->CellStatistics || options->DirReqStatistics ||
       options->EntryStatistics || options->ExitPortStatistics ||
@@ -1905,7 +2306,6 @@ options_act(const or_options_t *old_options)
       options->CellStatistics = 0;
       options->EntryStatistics = 0;
       options->ConnDirectionStatistics = 0;
-      options->HiddenServiceStatistics = 0;
       options->ExitPortStatistics = 0;
     }
 
@@ -1931,6 +2331,11 @@ options_act(const or_options_t *old_options)
     }
     if ((!old_options || !old_options->EntryStatistics) &&
         options->EntryStatistics && !should_record_bridge_info(options)) {
+      /* If we get here, we've started recording bridge info when we didn't
+       * do so before.  Note that "should_record_bridge_info()" will
+       * always be false at this point, because of the earlier block
+       * that cleared EntryStatistics when public_server_mode() was false.
+       * We're leaving it in as defensive programming. */
       if (geoip_is_loaded(AF_INET) || geoip_is_loaded(AF_INET6)) {
         geoip_entry_stats_init(now);
         print_notice = 1;
@@ -1991,13 +2396,6 @@ options_act(const or_options_t *old_options)
       !options->BridgeAuthoritativeDir)
     rep_hist_desc_stats_term();
 
-  /* Check if we need to parse and add the EntryNodes config option. */
-  if (options->EntryNodes &&
-      (!old_options ||
-       !routerset_equal(old_options->EntryNodes,options->EntryNodes) ||
-       !routerset_equal(old_options->ExcludeNodes,options->ExcludeNodes)))
-    entry_nodes_should_be_added();
-
   /* Since our options changed, we might need to regenerate and upload our
    * server descriptor.
    */
@@ -2007,12 +2405,15 @@ options_act(const or_options_t *old_options)
 
   /* We may need to reschedule some directory stuff if our status changed. */
   if (old_options) {
-    if (authdir_mode_v3(options) && !authdir_mode_v3(old_options))
-      dirvote_recalculate_timing(options, time(NULL));
+    if (options_transition_affects_dirauth_timing(old_options, options)) {
+      voting_schedule_recalculate_timing(options, time(NULL));
+      reschedule_dirvote(options);
+    }
     if (!bool_eq(directory_fetches_dir_info_early(options),
                  directory_fetches_dir_info_early(old_options)) ||
         !bool_eq(directory_fetches_dir_info_later(options),
-                 directory_fetches_dir_info_later(old_options))) {
+                 directory_fetches_dir_info_later(old_options)) ||
+        !config_lines_eq(old_options->Bridges, options->Bridges)) {
       /* Make sure update_router_have_minimum_dir_info() gets called. */
       router_dir_info_changed();
       /* We might need to download a new consensus status later or sooner than
@@ -2065,6 +2466,7 @@ static const struct {
   { "--dump-config",          ARGUMENT_OPTIONAL },
   { "--list-fingerprint",     TAKES_NO_ARGUMENT },
   { "--keygen",               TAKES_NO_ARGUMENT },
+  { "--key-expiration",       ARGUMENT_OPTIONAL },
   { "--newpass",              TAKES_NO_ARGUMENT },
   { "--no-passphrase",        TAKES_NO_ARGUMENT },
   { "--passphrase-fd",        ARGUMENT_NECESSARY },
@@ -2073,6 +2475,7 @@ static const struct {
   { "--quiet",                TAKES_NO_ARGUMENT },
   { "--hush",                 TAKES_NO_ARGUMENT },
   { "--version",              TAKES_NO_ARGUMENT },
+  { "--list-modules",         TAKES_NO_ARGUMENT },
   { "--library-versions",     TAKES_NO_ARGUMENT },
   { "-h",                     TAKES_NO_ARGUMENT },
   { "--help",                 TAKES_NO_ARGUMENT },
@@ -2225,24 +2628,36 @@ options_trial_assign(config_line_t *list, unsigned flags, char **msg)
     return r;
   }
 
-  if (options_validate(get_options_mutable(), trial_options,
+  setopt_err_t rv;
+  or_options_t *cur_options = get_options_mutable();
+
+  in_option_validation = 1;
+
+  if (options_validate(cur_options, trial_options,
                        global_default_options, 1, msg) < 0) {
     or_options_free(trial_options);
-    return SETOPT_ERR_PARSE; /*XXX make this a separate return value. */
+    rv = SETOPT_ERR_PARSE; /*XXX make this a separate return value. */
+    goto done;
   }
 
-  if (options_transition_allowed(get_options(), trial_options, msg) < 0) {
+  if (options_transition_allowed(cur_options, trial_options, msg) < 0) {
     or_options_free(trial_options);
-    return SETOPT_ERR_TRANSITION;
+    rv = SETOPT_ERR_TRANSITION;
+    goto done;
   }
+  in_option_validation = 0;
 
   if (set_options(trial_options, msg)<0) {
     or_options_free(trial_options);
-    return SETOPT_ERR_SETTING;
+    rv = SETOPT_ERR_SETTING;
+    goto done;
   }
 
   /* we liked it. put it in place. */
-  return SETOPT_OK;
+  rv = SETOPT_OK;
+ done:
+  in_option_validation = 0;
+  return rv;
 }
 
 /** Print a usage message for tor. */
@@ -2252,7 +2667,7 @@ print_usage(void)
   printf(
 "Copyright (c) 2001-2004, Roger Dingledine\n"
 "Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson\n"
-"Copyright (c) 2007-2016, The Tor Project, Inc.\n\n"
+"Copyright (c) 2007-2019, The Tor Project, Inc.\n\n"
 "tor -f <torrc> [args]\n"
 "See man page for options, or https://www.torproject.org/ for "
 "documentation.\n");
@@ -2282,6 +2697,13 @@ list_deprecated_options(void)
   }
 }
 
+/** Print all compile-time modules and their enabled/disabled status. */
+static void
+list_enabled_modules(void)
+{
+  printf("%s: %s\n", "dirauth", have_module_dirauth() ? "yes" : "no");
+}
+
 /** Last value actually set by resolve_my_address. */
 static uint32_t last_resolved_addr = 0;
 
@@ -2333,8 +2755,8 @@ using_default_dir_authorities(const or_options_t *options)
  * Fail if one or more of the following is true:
  *   - DNS name in <b>options-\>Address</b> cannot be resolved.
  *   - <b>options-\>Address</b> is a local host address.
- *   - Attempt to getting local hostname fails.
- *   - Attempt to getting network interface address fails.
+ *   - Attempt at getting local hostname fails.
+ *   - Attempt at getting network interface address fails.
  *
  * Return 0 if all is well, or -1 if we can't find a suitable
  * public IP address.
@@ -2659,8 +3081,8 @@ ensure_bandwidth_cap(uint64_t *value, const char *desc, char **msg)
     --*value;
   }
   if (*value > ROUTER_MAX_DECLARED_BANDWIDTH) {
-    tor_asprintf(msg, "%s ("U64_FORMAT") must be at most %d",
-                 desc, U64_PRINTF_ARG(*value),
+    tor_asprintf(msg, "%s (%"PRIu64") must be at most %d",
+                 desc, (*value),
                  ROUTER_MAX_DECLARED_BANDWIDTH);
     return -1;
   }
@@ -2714,13 +3136,13 @@ compute_publishserverdescriptor(or_options_t *options)
 #define MIN_REND_POST_PERIOD (10*60)
 #define MIN_REND_POST_PERIOD_TESTING (5)
 
-/** Higest allowable value for PredictedPortsRelevanceTime; if this is
- * too high, our selection of exits will decrease for an extended
- * period of time to an uncomfortable level .*/
-#define MAX_PREDICTED_CIRCS_RELEVANCE (60*60)
+/** Highest allowable value for CircuitsAvailableTimeout.
+ * If this is too large, client connections will stay open for too long,
+ * incurring extra padding overhead. */
+#define MAX_CIRCS_AVAILABLE_TIME (24*60*60)
 
 /** Highest allowable value for RendPostPeriod. */
-#define MAX_DIR_PERIOD (MIN_ONION_KEY_LIFETIME/2)
+#define MAX_DIR_PERIOD ((7*24*60*60)/2)
 
 /** Lowest allowable value for MaxCircuitDirtiness; if this is too low, Tor
  * will generate too many circuits and potentially overload the network. */
@@ -2734,10 +3156,6 @@ compute_publishserverdescriptor(or_options_t *options)
  * will generate too many circuits and potentially overload the network. */
 #define MIN_CIRCUIT_STREAM_TIMEOUT 10
 
-/** Lowest allowable value for HeartbeatPeriod; if this is too low, we might
- * expose more information than we're comfortable with. */
-#define MIN_HEARTBEAT_PERIOD (30*60)
-
 /** Lowest recommended value for CircuitBuildTimeout; if it is set too low
  * and LearnCircuitBuildTimeout is off, the failure rate for circuit
  * construction may be very high.  In that case, if it is set below this
@@ -2749,8 +3167,18 @@ static int
 options_validate_cb(void *old_options, void *options, void *default_options,
                     int from_setconf, char **msg)
 {
-  return options_validate(old_options, options, default_options,
+  in_option_validation = 1;
+  int rv = options_validate(old_options, options, default_options,
                           from_setconf, msg);
+  in_option_validation = 0;
+  return rv;
+}
+
+/** Callback to free an or_options_t */
+static void
+options_free_cb(void *options)
+{
+  or_options_free_(options);
 }
 
 #define REJECT(arg) \
@@ -2761,15 +3189,17 @@ options_validate_cb(void *old_options, void *options, void *default_options,
 #else
 #define COMPLAIN(args, ...)                                     \
   STMT_BEGIN log_warn(LD_CONFIG, args, ##__VA_ARGS__); STMT_END
-#endif
+#endif /* defined(__GNUC__) && __GNUC__ <= 3 */
 
 /** Log a warning message iff <b>filepath</b> is not absolute.
  * Warning message must contain option name <b>option</b> and
  * an absolute path that <b>filepath</b> will resolve to.
  *
  * In case <b>filepath</b> is absolute, do nothing.
+ *
+ * Return 1 if there were relative paths; 0 otherwise.
  */
-static void
+static int
 warn_if_option_path_is_relative(const char *option,
                                 char *filepath)
 {
@@ -2778,39 +3208,102 @@ warn_if_option_path_is_relative(const char *option,
     COMPLAIN("Path for %s (%s) is relative and will resolve to %s."
              " Is this what you wanted?", option, filepath, abs_path);
     tor_free(abs_path);
+    return 1;
   }
+  return 0;
 }
 
-/** Scan <b>options</b> for occurances of relative file/directory
+/** Scan <b>options</b> for occurrences of relative file/directory
  * path and log a warning whenever it is found.
+ *
+ * Return 1 if there were relative paths; 0 otherwise.
  */
-static void
+static int
 warn_about_relative_paths(or_options_t *options)
 {
   tor_assert(options);
+  int n = 0;
 
-  warn_if_option_path_is_relative("CookieAuthFile",
-                                  options->CookieAuthFile);
-  warn_if_option_path_is_relative("ExtORPortCookieAuthFile",
-                                  options->ExtORPortCookieAuthFile);
-  warn_if_option_path_is_relative("DirPortFrontPage",
-                                  options->DirPortFrontPage);
-  warn_if_option_path_is_relative("V3BandwidthsFile",
-                                  options->V3BandwidthsFile);
-  warn_if_option_path_is_relative("ControlPortWriteToFile",
-                                  options->ControlPortWriteToFile);
-  warn_if_option_path_is_relative("GeoIPFile",options->GeoIPFile);
-  warn_if_option_path_is_relative("GeoIPv6File",options->GeoIPv6File);
-  warn_if_option_path_is_relative("Log",options->DebugLogFile);
-  warn_if_option_path_is_relative("AccelDir",options->AccelDir);
-  warn_if_option_path_is_relative("DataDirectory",options->DataDirectory);
-  warn_if_option_path_is_relative("PidFile",options->PidFile);
+  n += warn_if_option_path_is_relative("CookieAuthFile",
+                                       options->CookieAuthFile);
+  n += warn_if_option_path_is_relative("ExtORPortCookieAuthFile",
+                                       options->ExtORPortCookieAuthFile);
+  n += warn_if_option_path_is_relative("DirPortFrontPage",
+                                       options->DirPortFrontPage);
+  n += warn_if_option_path_is_relative("V3BandwidthsFile",
+                                       options->V3BandwidthsFile);
+  n += warn_if_option_path_is_relative("ControlPortWriteToFile",
+                                       options->ControlPortWriteToFile);
+  n += warn_if_option_path_is_relative("GeoIPFile",options->GeoIPFile);
+  n += warn_if_option_path_is_relative("GeoIPv6File",options->GeoIPv6File);
+  n += warn_if_option_path_is_relative("Log",options->DebugLogFile);
+  n += warn_if_option_path_is_relative("AccelDir",options->AccelDir);
+  n += warn_if_option_path_is_relative("DataDirectory",options->DataDirectory);
+  n += warn_if_option_path_is_relative("PidFile",options->PidFile);
+  n += warn_if_option_path_is_relative("ClientOnionAuthDir",
+                                        options->ClientOnionAuthDir);
 
   for (config_line_t *hs_line = options->RendConfigLines; hs_line;
        hs_line = hs_line->next) {
     if (!strcasecmp(hs_line->key, "HiddenServiceDir"))
-      warn_if_option_path_is_relative("HiddenServiceDir",hs_line->value);
+      n += warn_if_option_path_is_relative("HiddenServiceDir",hs_line->value);
+  }
+  return n != 0;
+}
+
+/* Validate options related to the scheduler. From the Schedulers list, the
+ * SchedulerTypes_ list is created with int values so once we select the
+ * scheduler, which can happen anytime at runtime, we don't have to parse
+ * strings and thus be quick.
+ *
+ * Return 0 on success else -1 and msg is set with an error message. */
+static int
+options_validate_scheduler(or_options_t *options, char **msg)
+{
+  tor_assert(options);
+  tor_assert(msg);
+
+  if (!options->Schedulers || smartlist_len(options->Schedulers) == 0) {
+    REJECT("Empty Schedulers list. Either remove the option so the defaults "
+           "can be used or set at least one value.");
+  }
+  /* Ok, we do have scheduler types, validate them. */
+  options->SchedulerTypes_ = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(options->Schedulers, const char *, type) {
+    int *sched_type;
+    if (!strcasecmp("KISTLite", type)) {
+      sched_type = tor_malloc_zero(sizeof(int));
+      *sched_type = SCHEDULER_KIST_LITE;
+      smartlist_add(options->SchedulerTypes_, sched_type);
+    } else if (!strcasecmp("KIST", type)) {
+      sched_type = tor_malloc_zero(sizeof(int));
+      *sched_type = SCHEDULER_KIST;
+      smartlist_add(options->SchedulerTypes_, sched_type);
+    } else if (!strcasecmp("Vanilla", type)) {
+      sched_type = tor_malloc_zero(sizeof(int));
+      *sched_type = SCHEDULER_VANILLA;
+      smartlist_add(options->SchedulerTypes_, sched_type);
+    } else {
+      tor_asprintf(msg, "Unknown type %s in option Schedulers. "
+                        "Possible values are KIST, KISTLite and Vanilla.",
+                   escaped(type));
+      return -1;
+    }
+  } SMARTLIST_FOREACH_END(type);
+
+  if (options->KISTSockBufSizeFactor < 0) {
+    REJECT("KISTSockBufSizeFactor must be at least 0");
   }
+
+  /* Don't need to validate that the Interval is less than anything because
+   * zero is valid and all negative values are valid. */
+  if (options->KISTSchedRunInterval > KIST_SCHED_RUN_INTERVAL_MAX) {
+    tor_asprintf(msg, "KISTSchedRunInterval must not be more than %d (ms)",
+                 KIST_SCHED_RUN_INTERVAL_MAX);
+    return -1;
+  }
+
+  return 0;
 }
 
 /* Validate options related to single onion services.
@@ -2841,28 +3334,17 @@ options_validate_single_onion(or_options_t *options, char **msg)
   const int client_port_set = (options->SocksPort_set ||
                                options->TransPort_set ||
                                options->NATDPort_set ||
-                               options->DNSPort_set);
-  if (rend_service_non_anonymous_mode_enabled(options) && client_port_set &&
-      !options->Tor2webMode) {
+                               options->DNSPort_set ||
+                               options->HTTPTunnelPort_set);
+  if (rend_service_non_anonymous_mode_enabled(options) && client_port_set) {
     REJECT("HiddenServiceNonAnonymousMode is incompatible with using Tor as "
            "an anonymous client. Please set Socks/Trans/NATD/DNSPort to 0, or "
-           "HiddenServiceNonAnonymousMode to 0, or use the non-anonymous "
-           "Tor2webMode.");
-  }
-
-  /* If you run a hidden service in non-anonymous mode, the hidden service
-   * loses anonymity, even if SOCKSPort / Tor2web mode isn't used. */
-  if (!rend_service_non_anonymous_mode_enabled(options) &&
-      options->RendConfigLines && options->Tor2webMode) {
-    REJECT("Non-anonymous (Tor2web) mode is incompatible with using Tor as a "
-           "hidden service. Please remove all HiddenServiceDir lines, or use "
-           "a version of tor compiled without --enable-tor2web-mode, or use "
-           " HiddenServiceNonAnonymousMode.");
+           "revert HiddenServiceNonAnonymousMode to 0.");
   }
 
   if (rend_service_allow_non_anonymous_connection(options)
       && options->UseEntryGuards) {
-    /* Single Onion services only use entry guards when uploading descriptors,
+    /* Single Onion services only use entry guards when uploading descriptors;
      * all other connections are one-hop. Further, Single Onions causes the
      * hidden service code to do things which break the path bias
      * detector, and it's far easier to turn off entry guards (and
@@ -2895,7 +3377,6 @@ STATIC int
 options_validate(or_options_t *old_options, or_options_t *options,
                  or_options_t *default_options, int from_setconf, char **msg)
 {
-  int i;
   config_line_t *cl;
   const char *uname = get_uname();
   int n_ports=0;
@@ -2904,14 +3385,16 @@ options_validate(or_options_t *old_options, or_options_t *options,
   tor_assert(msg);
   *msg = NULL;
 
+  if (parse_ports(options, 1, msg, &n_ports,
+                  &world_writable_control_socket) < 0)
+    return -1;
+
   /* Set UseEntryGuards from the configured value, before we check it below.
-   * We change UseEntryGuards whenn it's incompatible with other options,
+   * We change UseEntryGuards when it's incompatible with other options,
    * but leave UseEntryGuards_option with the original value.
    * Always use the value of UseEntryGuards, not UseEntryGuards_option. */
   options->UseEntryGuards = options->UseEntryGuards_option;
 
-  warn_about_relative_paths(options);
-
   if (server_mode(options) &&
       (!strcmpstart(uname, "Windows 95") ||
        !strcmpstart(uname, "Windows 98") ||
@@ -2922,19 +3405,23 @@ options_validate(or_options_t *old_options, or_options_t *options,
         "for details.", uname);
   }
 
-  if (parse_ports(options, 1, msg, &n_ports,
-                  &world_writable_control_socket) < 0)
-    return -1;
-
   if (parse_outbound_addresses(options, 1, msg) < 0)
     return -1;
 
-  if (validate_data_directory(options)<0)
+  if (validate_data_directories(options)<0)
     REJECT("Invalid DataDirectory");
 
+  /* need to check for relative paths after we populate
+   * options->DataDirectory (just above). */
+  if (warn_about_relative_paths(options) && options->RunAsDaemon) {
+    REJECT("You have specified at least one relative path (see above) "
+           "with the RunAsDaemon option. RunAsDaemon is not compatible "
+           "with relative paths.");
+  }
+
   if (options->Nickname == NULL) {
     if (server_mode(options)) {
-        options->Nickname = tor_strdup(UNNAMED_ROUTER_NICKNAME);
+      options->Nickname = tor_strdup(UNNAMED_ROUTER_NICKNAME);
     }
   } else {
     if (!is_legal_nickname(options->Nickname)) {
@@ -2950,13 +3437,16 @@ options_validate(or_options_t *old_options, or_options_t *options,
     log_notice(LD_CONFIG, "Your ContactInfo config option is not set. "
         "Please consider setting it, so we can contact you if your server is "
         "misconfigured or something else goes wrong.");
+  const char *ContactInfo = options->ContactInfo;
+  if (ContactInfo && !string_is_utf8(ContactInfo, strlen(ContactInfo)))
+    REJECT("ContactInfo config option must be UTF-8.");
 
   /* Special case on first boot if no Log options are given. */
   if (!options->Logs && !options->RunAsDaemon && !from_setconf) {
     if (quiet_level == 0)
-        config_line_append(&options->Logs, "Log", "notice stdout");
+      config_line_append(&options->Logs, "Log", "notice stdout");
     else if (quiet_level == 1)
-        config_line_append(&options->Logs, "Log", "warn stdout");
+      config_line_append(&options->Logs, "Log", "warn stdout");
   }
 
   /* Validate the tor_log(s) */
@@ -2991,13 +3481,13 @@ options_validate(or_options_t *old_options, or_options_t *options,
     if (!strcasecmp(options->TransProxyType, "default")) {
       options->TransProxyType_parsed = TPT_DEFAULT;
     } else if (!strcasecmp(options->TransProxyType, "pf-divert")) {
-#if !defined(__OpenBSD__) && !defined( DARWIN )
+#if !defined(OpenBSD) && !defined( DARWIN )
       /* Later versions of OS X have pf */
       REJECT("pf-divert is a OpenBSD-specific "
              "and OS X/Darwin-specific feature.");
 #else
       options->TransProxyType_parsed = TPT_PF_DIVERT;
-#endif
+#endif /* !defined(OpenBSD) && !defined( DARWIN ) */
     } else if (!strcasecmp(options->TransProxyType, "tproxy")) {
 #if !defined(__linux__)
       REJECT("TPROXY is a Linux-specific feature.");
@@ -3011,22 +3501,20 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "and OS X/Darwin-specific feature.");
 #else
       options->TransProxyType_parsed = TPT_IPFW;
-#endif
+#endif /* !defined(KERNEL_MAY_SUPPORT_IPFW) */
     } else {
       REJECT("Unrecognized value for TransProxyType");
     }
 
     if (strcasecmp(options->TransProxyType, "default") &&
         !options->TransPort_set) {
-      REJECT("Cannot use TransProxyType without any valid TransPort or "
-             "TransListenAddress.");
+      REJECT("Cannot use TransProxyType without any valid TransPort.");
     }
   }
-#else
+#else /* !(defined(USE_TRANSPARENT)) */
   if (options->TransPort_set)
-    REJECT("TransPort and TransListenAddress are disabled "
-           "in this build.");
-#endif
+    REJECT("TransPort is disabled in this build.");
+#endif /* defined(USE_TRANSPARENT) */
 
   if (options->TokenBucketRefillInterval <= 0
       || options->TokenBucketRefillInterval > 1000) {
@@ -3039,17 +3527,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
     routerset_union(options->ExcludeExitNodesUnion_,options->ExcludeNodes);
   }
 
-  if (options->SchedulerLowWaterMark__ == 0 ||
-      options->SchedulerLowWaterMark__ > UINT32_MAX) {
-    log_warn(LD_GENERAL, "Bad SchedulerLowWaterMark__ option");
-    return -1;
-  } else if (options->SchedulerHighWaterMark__ <=
-             options->SchedulerLowWaterMark__ ||
-             options->SchedulerHighWaterMark__ > UINT32_MAX) {
-    log_warn(LD_GENERAL, "Bad SchedulerHighWaterMark option");
-    return -1;
-  }
-
   if (options->NodeFamilies) {
     options->NodeFamilySets = smartlist_new();
     for (cl = options->NodeFamilies; cl; cl = cl->next) {
@@ -3062,15 +3539,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
     }
   }
 
-  if (options->TLSECGroup && (strcasecmp(options->TLSECGroup, "P256") &&
-                              strcasecmp(options->TLSECGroup, "P224"))) {
-    COMPLAIN("Unrecognized TLSECGroup: Falling back to the default.");
-    tor_free(options->TLSECGroup);
-  }
-  if (!evaluate_ecgroup_for_tls(options->TLSECGroup)) {
-    REJECT("Unsupported TLSECGroup.");
-  }
-
   if (options->ExcludeNodes && options->StrictNodes) {
     COMPLAIN("You have asked to exclude certain relays from all positions "
              "in your circuits. Expect hidden services and other Tor "
@@ -3099,19 +3567,21 @@ options_validate(or_options_t *old_options, or_options_t *options,
       REJECT("Versioning authoritative dir servers must set "
              "Recommended*Versions.");
 
+#ifdef HAVE_MODULE_DIRAUTH
     char *t;
     /* Call these functions to produce warnings only. */
     t = format_recommended_version_list(options->RecommendedClientVersions, 1);
     tor_free(t);
     t = format_recommended_version_list(options->RecommendedServerVersions, 1);
     tor_free(t);
+#endif
 
     if (options->UseEntryGuards) {
       log_info(LD_CONFIG, "Authoritative directory servers can't set "
                "UseEntryGuards. Disabling.");
       options->UseEntryGuards = 0;
     }
-    if (!options->DownloadExtraInfo && authdir_mode_any_main(options)) {
+    if (!options->DownloadExtraInfo && authdir_mode_v3(options)) {
       log_info(LD_CONFIG, "Authoritative directories always try to download "
                "extra-info documents. Setting DownloadExtraInfo.");
       options->DownloadExtraInfo = 1;
@@ -3122,7 +3592,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "(Bridge/V3)AuthoritativeDir is set.");
     /* If we have a v3bandwidthsfile and it's broken, complain on startup */
     if (options->V3BandwidthsFile && !old_options) {
-      dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL);
+      dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL);
     }
     /* same for guardfraction file */
     if (options->GuardfractionFile && !old_options) {
@@ -3232,30 +3702,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
     }
   }
 
-  /* Terminate Reachable*Addresses with reject *
-   */
-  for (i=0; i<3; i++) {
-    config_line_t **linep =
-      (i==0) ? &options->ReachableAddresses :
-        (i==1) ? &options->ReachableORAddresses :
-                 &options->ReachableDirAddresses;
-    if (!*linep)
-      continue;
-    /* We need to end with a reject *:*, not an implicit accept *:* */
-    for (;;) {
-      linep = &((*linep)->next);
-      if (!*linep) {
-        *linep = tor_malloc_zero(sizeof(config_line_t));
-        (*linep)->key = tor_strdup(
-          (i==0) ?  "ReachableAddresses" :
-            (i==1) ? "ReachableORAddresses" :
-                     "ReachableDirAddresses");
-        (*linep)->value = tor_strdup("reject *:*");
-        break;
-      }
-    }
-  }
-
   if ((options->ReachableAddresses ||
        options->ReachableORAddresses ||
        options->ReachableDirAddresses ||
@@ -3265,23 +3711,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
            "of the Internet, so they must not set Reachable*Addresses "
            "or FascistFirewall or FirewallPorts or ClientUseIPv4 0.");
 
-  /* We check if Reachable*Addresses blocks all addresses in
-   * parse_reachable_addresses(). */
-
-#define WARN_PLEASE_USE_IPV6_LOG_MSG \
-        "ClientPreferIPv6%sPort 1 is ignored unless tor is using IPv6. " \
-        "Please set ClientUseIPv6 1, ClientUseIPv4 0, or configure bridges."
-
-  if (!fascist_firewall_use_ipv6(options)
-      && options->ClientPreferIPv6ORPort == 1)
-    log_warn(LD_CONFIG, WARN_PLEASE_USE_IPV6_LOG_MSG, "OR");
-
-  if (!fascist_firewall_use_ipv6(options)
-      && options->ClientPreferIPv6DirPort == 1)
-    log_warn(LD_CONFIG, WARN_PLEASE_USE_IPV6_LOG_MSG, "Dir");
-
-#undef WARN_PLEASE_USE_IPV6_LOG_MSG
-
   if (options->UseBridges &&
       server_mode(options))
     REJECT("Servers must be able to freely connect to the rest "
@@ -3293,33 +3722,16 @@ options_validate(or_options_t *old_options, or_options_t *options,
   if (options->UseBridges && options->EntryNodes)
     REJECT("You cannot set both UseBridges and EntryNodes.");
 
+  /* If we have UseBridges as 1 and UseEntryGuards as 0, we end up bypassing
+   * the use of bridges */
+  if (options->UseBridges && !options->UseEntryGuards)
+    REJECT("Setting UseBridges requires also setting UseEntryGuards.");
+
   options->MaxMemInQueues =
     compute_real_max_mem_in_queues(options->MaxMemInQueues_raw,
                                    server_mode(options));
   options->MaxMemInQueues_low_threshold = (options->MaxMemInQueues / 4) * 3;
 
-  options->AllowInvalid_ = 0;
-
-  if (options->AllowInvalidNodes) {
-    SMARTLIST_FOREACH_BEGIN(options->AllowInvalidNodes, const char *, cp) {
-        if (!strcasecmp(cp, "entry"))
-          options->AllowInvalid_ |= ALLOW_INVALID_ENTRY;
-        else if (!strcasecmp(cp, "exit"))
-          options->AllowInvalid_ |= ALLOW_INVALID_EXIT;
-        else if (!strcasecmp(cp, "middle"))
-          options->AllowInvalid_ |= ALLOW_INVALID_MIDDLE;
-        else if (!strcasecmp(cp, "introduction"))
-          options->AllowInvalid_ |= ALLOW_INVALID_INTRODUCTION;
-        else if (!strcasecmp(cp, "rendezvous"))
-          options->AllowInvalid_ |= ALLOW_INVALID_RENDEZVOUS;
-        else {
-          tor_asprintf(msg,
-              "Unrecognized value '%s' in AllowInvalidNodes", cp);
-          return -1;
-        }
-    } SMARTLIST_FOREACH_END(cp);
-  }
-
   if (!options->SafeLogging ||
       !strcasecmp(options->SafeLogging, "0")) {
     options->SafeLogging_ = SAFELOG_SCRUB_NONE;
@@ -3355,6 +3767,23 @@ options_validate(or_options_t *old_options, or_options_t *options,
     options->DirPort_set = 0;
   }
 
+  if (server_mode(options) && options->ConnectionPadding != -1) {
+    REJECT("Relays must use 'auto' for the ConnectionPadding setting.");
+  }
+
+  if (server_mode(options) && options->ReducedConnectionPadding != 0) {
+    REJECT("Relays cannot set ReducedConnectionPadding. ");
+  }
+
+  if (options->BridgeDistribution) {
+    if (!options->BridgeRelay) {
+      REJECT("You set BridgeDistribution, but you didn't set BridgeRelay!");
+    }
+    if (check_bridge_distribution_setting(options->BridgeDistribution) < 0) {
+      REJECT("Invalid BridgeDistribution value.");
+    }
+  }
+
   if (options->MinUptimeHidServDirectoryV2 < 0) {
     log_warn(LD_CONFIG, "MinUptimeHidServDirectoryV2 option must be at "
                         "least 0 seconds. Changing to 0.");
@@ -3367,7 +3796,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
   if (options->RendPostPeriod < min_rendpostperiod) {
     log_warn(LD_CONFIG, "RendPostPeriod option is too short; "
              "raising to %d seconds.", min_rendpostperiod);
-    options->RendPostPeriod = min_rendpostperiod;;
+    options->RendPostPeriod = min_rendpostperiod;
   }
 
   if (options->RendPostPeriod > MAX_DIR_PERIOD) {
@@ -3376,35 +3805,15 @@ options_validate(or_options_t *old_options, or_options_t *options,
     options->RendPostPeriod = MAX_DIR_PERIOD;
   }
 
-  if (options->PredictedPortsRelevanceTime >
-      MAX_PREDICTED_CIRCS_RELEVANCE) {
-    log_warn(LD_CONFIG, "PredictedPortsRelevanceTime is too large; "
-             "clipping to %ds.", MAX_PREDICTED_CIRCS_RELEVANCE);
-    options->PredictedPortsRelevanceTime = MAX_PREDICTED_CIRCS_RELEVANCE;
-  }
-
   /* Check the Single Onion Service options */
   if (options_validate_single_onion(options, msg) < 0)
     return -1;
 
-#ifdef ENABLE_TOR2WEB_MODE
-  if (options->Tor2webMode && options->UseEntryGuards) {
-    /* tor2web mode clients do not (and should not) use entry guards
-     * in any meaningful way.  Further, tor2web mode causes the hidden
-     * service client code to do things which break the path bias
-     * detector, and it's far easier to turn off entry guards (and
-     * thus the path bias detector with it) than to figure out how to
-     * make a piece of code which cannot possibly help tor2web mode
-     * users compatible with tor2web mode.
-     */
-    log_notice(LD_CONFIG,
-               "Tor2WebMode is enabled; disabling UseEntryGuards.");
-    options->UseEntryGuards = 0;
-  }
-#endif
-
-  if (options->Tor2webRendezvousPoints && !options->Tor2webMode) {
-    REJECT("Tor2webRendezvousPoints cannot be set without Tor2webMode.");
+  if (options->CircuitsAvailableTimeout > MAX_CIRCS_AVAILABLE_TIME) {
+    // options_t is immutable for new code (the above code is older),
+    // so just make the user fix the value themselves rather than
+    // silently keep a shadow value lower than what they asked for.
+    REJECT("CircuitsAvailableTimeout is too large. Max is 24 hours.");
   }
 
   if (options->EntryNodes && !options->UseEntryGuards) {
@@ -3421,6 +3830,11 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "http://freehaven.net/anonbib/#hs-attack06 for details.");
   }
 
+  if (options->NumPrimaryGuards && options->NumEntryGuards &&
+      options->NumEntryGuards > options->NumPrimaryGuards) {
+    REJECT("NumEntryGuards must not be greater than NumPrimaryGuards.");
+  }
+
   if (options->EntryNodes &&
       routerset_is_list(options->EntryNodes) &&
       (routerset_len(options->EntryNodes) == 1) &&
@@ -3436,6 +3850,20 @@ options_validate(or_options_t *old_options, or_options_t *options,
     return -1;
   }
 
+  /* Inform the hidden service operator that pinning EntryNodes can possibly
+   * be harmful for the service anonymity. */
+  if (options->EntryNodes &&
+      routerset_is_list(options->EntryNodes) &&
+      (options->RendConfigLines != NULL)) {
+    log_warn(LD_CONFIG,
+             "EntryNodes is set with multiple entries and at least one "
+             "hidden service is configured. Pinning entry nodes can possibly "
+             "be harmful to the service anonymity. Because of this, we "
+             "recommend you either don't do that or make sure you know what "
+             "you are doing. For more details, please look at "
+             "https://trac.torproject.org/projects/tor/ticket/21155.");
+  }
+
   /* Single Onion Services: non-anonymous hidden services */
   if (rend_service_non_anonymous_mode_enabled(options)) {
     log_warn(LD_CONFIG,
@@ -3461,7 +3889,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
     int severity = LOG_NOTICE;
     /* Be a little quieter if we've deliberately disabled
      * LearnCircuitBuildTimeout. */
-    if (circuit_build_times_disabled()) {
+    if (circuit_build_times_disabled_(options, 1)) {
       severity = LOG_INFO;
     }
     log_fn(severity, LD_CONFIG, "You disabled LearnCircuitBuildTimeout, but "
@@ -3529,11 +3957,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
   if (options->KeepalivePeriod < 1)
     REJECT("KeepalivePeriod option must be positive.");
 
-  if (options->PortForwarding && options->Sandbox) {
-    REJECT("PortForwarding is not compatible with Sandbox; at most one can "
-           "be set");
-  }
-
   if (ensure_bandwidth_cap(&options->BandwidthRate,
                            "BandwidthRate", msg) < 0)
     return -1;
@@ -3791,13 +4214,14 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "have it group-readable.");
   }
 
-  if (options->MyFamily && options->BridgeRelay) {
+  if (options->MyFamily_lines && options->BridgeRelay) {
     log_warn(LD_CONFIG, "Listing a family for a bridge relay is not "
              "supported: it can reveal bridge fingerprints to censors. "
              "You should also make sure you aren't listing this bridge's "
              "fingerprint in any other MyFamily.");
   }
-  if (check_nickname_list(&options->MyFamily, "MyFamily", msg))
+  if (normalize_nickname_list(&options->MyFamily,
+                              options->MyFamily_lines, "MyFamily", msg))
     return -1;
   for (cl = options->NodeFamilies; cl; cl = cl->next) {
     routerset_t *rs = routerset_new();
@@ -3970,11 +4394,11 @@ options_validate(or_options_t *old_options, or_options_t *options,
     COMPLAIN("V3AuthVotingInterval does not divide evenly into 24 hours.");
   }
 
-  if (rend_config_services(options, 1) < 0)
+  if (hs_config_service_all(options, 1) < 0)
     REJECT("Failed to configure rendezvous options. See logs for details.");
 
   /* Parse client-side authorization for hidden services. */
-  if (rend_parse_service_authorization(options, 1) < 0)
+  if (hs_config_client_auth_all(options, 1) < 0)
     REJECT("Failed to configure client authorization for hidden services. "
            "See logs for details.");
 
@@ -3994,13 +4418,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
            "AlternateDirAuthority and AlternateBridgeAuthority configured.");
   }
 
-  if (options->AllowSingleHopExits && !options->DirAuthorities) {
-    COMPLAIN("You have set AllowSingleHopExits; now your relay will allow "
-             "others to make one-hop exits. However, since by default most "
-             "clients avoid relays that set this option, most clients will "
-             "ignore you.");
-  }
-
 #define CHECK_DEFAULT(arg)                                              \
   STMT_BEGIN                                                            \
     if (!options->TestingTorNetwork &&                                  \
@@ -4016,17 +4433,14 @@ options_validate(or_options_t *old_options, or_options_t *options,
   CHECK_DEFAULT(TestingV3AuthVotingStartOffset);
   CHECK_DEFAULT(TestingAuthDirTimeToLearnReachability);
   CHECK_DEFAULT(TestingEstimatedDescriptorPropagationTime);
-  CHECK_DEFAULT(TestingServerDownloadSchedule);
-  CHECK_DEFAULT(TestingClientDownloadSchedule);
-  CHECK_DEFAULT(TestingServerConsensusDownloadSchedule);
-  CHECK_DEFAULT(TestingClientConsensusDownloadSchedule);
-  CHECK_DEFAULT(TestingBridgeDownloadSchedule);
+  CHECK_DEFAULT(TestingServerDownloadInitialDelay);
+  CHECK_DEFAULT(TestingClientDownloadInitialDelay);
+  CHECK_DEFAULT(TestingServerConsensusDownloadInitialDelay);
+  CHECK_DEFAULT(TestingClientConsensusDownloadInitialDelay);
+  CHECK_DEFAULT(TestingBridgeDownloadInitialDelay);
+  CHECK_DEFAULT(TestingBridgeBootstrapDownloadInitialDelay);
   CHECK_DEFAULT(TestingClientMaxIntervalWithoutRequest);
   CHECK_DEFAULT(TestingDirConnectionMaxStall);
-  CHECK_DEFAULT(TestingConsensusMaxDownloadTries);
-  CHECK_DEFAULT(TestingDescriptorMaxDownloadTries);
-  CHECK_DEFAULT(TestingMicrodescMaxDownloadTries);
-  CHECK_DEFAULT(TestingCertMaxDownloadTries);
   CHECK_DEFAULT(TestingAuthKeyLifetime);
   CHECK_DEFAULT(TestingLinkCertLifetime);
   CHECK_DEFAULT(TestingSigningKeySlop);
@@ -4034,6 +4448,10 @@ options_validate(or_options_t *old_options, or_options_t *options,
   CHECK_DEFAULT(TestingLinkKeySlop);
 #undef CHECK_DEFAULT
 
+  if (!options->ClientDNSRejectInternalAddresses &&
+      !(options->DirAuthorities ||
+        (options->AlternateDirAuthority && options->AlternateBridgeAuthority)))
+    REJECT("ClientDNSRejectInternalAddresses used for default network.");
   if (options->SigningKeyLifetime < options->TestingSigningKeySlop*2)
     REJECT("SigningKeyLifetime is too short.");
   if (options->TestingLinkCertLifetime < options->TestingAuthKeySlop*2)
@@ -4097,33 +4515,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
     COMPLAIN("TestingDirConnectionMaxStall is insanely high.");
   }
 
-  if (options->TestingConsensusMaxDownloadTries < 2) {
-    REJECT("TestingConsensusMaxDownloadTries must be greater than 2.");
-  } else if (options->TestingConsensusMaxDownloadTries > 800) {
-    COMPLAIN("TestingConsensusMaxDownloadTries is insanely high.");
-  }
-
-  if (options->ClientBootstrapConsensusMaxDownloadTries < 2) {
-    REJECT("ClientBootstrapConsensusMaxDownloadTries must be greater "
-           "than 2."
-           );
-  } else if (options->ClientBootstrapConsensusMaxDownloadTries > 800) {
-    COMPLAIN("ClientBootstrapConsensusMaxDownloadTries is insanely "
-             "high.");
-  }
-
-  if (options->ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries
-      < 2) {
-    REJECT("ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries must "
-           "be greater than 2."
-           );
-  } else if (
-        options->ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries
-        > 800) {
-    COMPLAIN("ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries is "
-             "insanely high.");
-  }
-
   if (options->ClientBootstrapConsensusMaxInProgressTries < 1) {
     REJECT("ClientBootstrapConsensusMaxInProgressTries must be greater "
            "than 0.");
@@ -4133,24 +4524,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "high.");
   }
 
-  if (options->TestingDescriptorMaxDownloadTries < 2) {
-    REJECT("TestingDescriptorMaxDownloadTries must be greater than 1.");
-  } else if (options->TestingDescriptorMaxDownloadTries > 800) {
-    COMPLAIN("TestingDescriptorMaxDownloadTries is insanely high.");
-  }
-
-  if (options->TestingMicrodescMaxDownloadTries < 2) {
-    REJECT("TestingMicrodescMaxDownloadTries must be greater than 1.");
-  } else if (options->TestingMicrodescMaxDownloadTries > 800) {
-    COMPLAIN("TestingMicrodescMaxDownloadTries is insanely high.");
-  }
-
-  if (options->TestingCertMaxDownloadTries < 2) {
-    REJECT("TestingCertMaxDownloadTries must be greater than 1.");
-  } else if (options->TestingCertMaxDownloadTries > 800) {
-    COMPLAIN("TestingCertMaxDownloadTries is insanely high.");
-  }
-
   if (options->TestingEnableConnBwEvent &&
       !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) {
     REJECT("TestingEnableConnBwEvent may only be changed in testing "
@@ -4163,12 +4536,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
            "Tor networks!");
   }
 
-  if (options->TestingEnableTbEmptyEvent &&
-      !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) {
-    REJECT("TestingEnableTbEmptyEvent may only be changed in testing "
-           "Tor networks!");
-  }
-
   if (options->TestingTorNetwork) {
     log_warn(LD_CONFIG, "TestingTorNetwork is set. This will make your node "
                         "almost unusable in the public Tor network, and is "
@@ -4197,6 +4564,10 @@ options_validate(or_options_t *old_options, or_options_t *options,
       REJECT("BridgeRelay is 1, ORPort is not set. This is an invalid "
              "combination.");
 
+  if (options_validate_scheduler(options, msg) < 0) {
+    return -1;
+  }
+
   return 0;
 }
 
@@ -4206,19 +4577,14 @@ options_validate(or_options_t *old_options, or_options_t *options,
 /* Given the value that the user has set for MaxMemInQueues, compute the
  * actual maximum value.  We clip this value if it's too low, and autodetect
  * it if it's set to 0. */
-static uint64_t
+STATIC uint64_t
 compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
 {
   uint64_t result;
 
   if (val == 0) {
-#define ONE_GIGABYTE (U64_LITERAL(1) << 30)
-#define ONE_MEGABYTE (U64_LITERAL(1) << 20)
-#if SIZEOF_VOID_P >= 8
-#define MAX_DEFAULT_MAXMEM (8*ONE_GIGABYTE)
-#else
-#define MAX_DEFAULT_MAXMEM (2*ONE_GIGABYTE)
-#endif
+#define ONE_GIGABYTE (UINT64_C(1) << 30)
+#define ONE_MEGABYTE (UINT64_C(1) << 20)
     /* The user didn't pick a memory limit.  Choose a very large one
      * that is still smaller than the system memory */
     static int notice_sent = 0;
@@ -4231,16 +4597,40 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
 #else
       /* (presumably) 32-bit system. Let's hope for 1 GB. */
       result = ONE_GIGABYTE;
-#endif
+#endif /* SIZEOF_VOID_P >= 8 */
     } else {
-      /* We detected it, so let's pick 3/4 of the total RAM as our limit. */
-      const uint64_t avail = (ram / 4) * 3;
+      /* We detected the amount of memory available. */
+      uint64_t avail = 0;
+
+#if SIZEOF_SIZE_T > 4
+/* On a 64-bit platform, we consider 8GB "very large". */
+#define RAM_IS_VERY_LARGE(x) ((x) >= (8 * ONE_GIGABYTE))
+#else
+/* On a 32-bit platform, we can't have 8GB of ram. */
+#define RAM_IS_VERY_LARGE(x) (0)
+#endif
+
+      if (RAM_IS_VERY_LARGE(ram)) {
+        /* If we have 8 GB, or more, RAM available, we set the MaxMemInQueues
+         * to 0.4 * RAM. The idea behind this value is that the amount of RAM
+         * is more than enough for a single relay and should allow the relay
+         * operator to run two relays if they have additional bandwidth
+         * available.
+         */
+        avail = (ram / 5) * 2;
+      } else {
+        /* If we have less than 8 GB of RAM available, we use the "old" default
+         * for MaxMemInQueues of 0.75 * RAM.
+         */
+        avail = (ram / 4) * 3;
+      }
 
-      /* Make sure it's in range from 0.25 GB to 8 GB. */
-      if (avail > MAX_DEFAULT_MAXMEM) {
+      /* Make sure it's in range from 0.25 GB to 8 GB for 64-bit and 0.25 to 2
+       * GB for 32-bit. */
+      if (avail > MAX_DEFAULT_MEMORY_QUEUE_SIZE) {
         /* If you want to use more than this much RAM, you need to configure
            it yourself */
-        result = MAX_DEFAULT_MAXMEM;
+        result = MAX_DEFAULT_MEMORY_QUEUE_SIZE;
       } else if (avail < ONE_GIGABYTE / 4) {
         result = ONE_GIGABYTE / 4;
       } else {
@@ -4248,10 +4638,10 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
       }
     }
     if (log_guess && ! notice_sent) {
-      log_notice(LD_CONFIG, "%sMaxMemInQueues is set to "U64_FORMAT" MB. "
+      log_notice(LD_CONFIG, "%sMaxMemInQueues is set to %"PRIu64" MB. "
                  "You can override this by setting MaxMemInQueues by hand.",
                  ram ? "Based on detected system memory, " : "",
-                 U64_PRINTF_ARG(result / ONE_MEGABYTE));
+                 (result / ONE_MEGABYTE));
       notice_sent = 1;
     }
     return result;
@@ -4266,8 +4656,8 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
 }
 
 /* If we have less than 300 MB suggest disabling dircache */
-#define DIRCACHE_MIN_MB_BANDWIDTH 300
-#define DIRCACHE_MIN_BANDWIDTH (DIRCACHE_MIN_MB_BANDWIDTH*ONE_MEGABYTE)
+#define DIRCACHE_MIN_MEM_MB 300
+#define DIRCACHE_MIN_MEM_BYTES (DIRCACHE_MIN_MEM_MB*ONE_MEGABYTE)
 #define STRINGIFY(val) #val
 
 /** Create a warning message for emitting if we are a dircache but may not have
@@ -4287,24 +4677,22 @@ have_enough_mem_for_dircache(const or_options_t *options, size_t total_mem,
     }
   }
   if (options->DirCache) {
-    if (total_mem < DIRCACHE_MIN_BANDWIDTH) {
+    if (total_mem < DIRCACHE_MIN_MEM_BYTES) {
       if (options->BridgeRelay) {
-        *msg = tor_strdup("Running a Bridge with less than "
-                      STRINGIFY(DIRCACHE_MIN_MB_BANDWIDTH) " MB of memory is "
-                      "not recommended.");
+        tor_asprintf(msg, "Running a Bridge with less than %d MB of memory "
+                       "is not recommended.", DIRCACHE_MIN_MEM_MB);
       } else {
-        *msg = tor_strdup("Being a directory cache (default) with less than "
-                      STRINGIFY(DIRCACHE_MIN_MB_BANDWIDTH) " MB of memory is "
-                      "not recommended and may consume most of the available "
-                      "resources, consider disabling this functionality by "
-                      "setting the DirCache option to 0.");
+        tor_asprintf(msg, "Being a directory cache (default) with less than "
+                       "%d MB of memory is not recommended and may consume "
+                       "most of the available resources. Consider disabling "
+                       "this functionality by setting the DirCache option "
+                       "to 0.", DIRCACHE_MIN_MEM_MB);
       }
     }
   } else {
-    if (total_mem >= DIRCACHE_MIN_BANDWIDTH) {
+    if (total_mem >= DIRCACHE_MIN_MEM_BYTES) {
       *msg = tor_strdup("DirCache is disabled and we are configured as a "
-               "relay. This may disqualify us from becoming a guard in the "
-               "future.");
+               "relay. We will not become a Guard.");
     }
   }
   return *msg == NULL ? 0 : -1;
@@ -4328,120 +4716,61 @@ options_transition_allowed(const or_options_t *old,
   if (!old)
     return 0;
 
-  if (!opt_streq(old->PidFile, new_val->PidFile)) {
-    *msg = tor_strdup("PidFile is not allowed to change.");
-    return -1;
-  }
-
-  if (old->RunAsDaemon != new_val->RunAsDaemon) {
-    *msg = tor_strdup("While Tor is running, changing RunAsDaemon "
-                      "is not allowed.");
-    return -1;
-  }
-
-  if (old->Sandbox != new_val->Sandbox) {
-    *msg = tor_strdup("While Tor is running, changing Sandbox "
-                      "is not allowed.");
-    return -1;
-  }
-
-  if (strcmp(old->DataDirectory,new_val->DataDirectory)!=0) {
-    tor_asprintf(msg,
-               "While Tor is running, changing DataDirectory "
-               "(\"%s\"->\"%s\") is not allowed.",
-               old->DataDirectory, new_val->DataDirectory);
-    return -1;
-  }
-
-  if (!opt_streq(old->User, new_val->User)) {
-    *msg = tor_strdup("While Tor is running, changing User is not allowed.");
-    return -1;
-  }
-
-  if (old->KeepBindCapabilities != new_val->KeepBindCapabilities) {
-    *msg = tor_strdup("While Tor is running, changing KeepBindCapabilities is "
-                      "not allowed.");
-    return -1;
-  }
-
-  if (!opt_streq(old->SyslogIdentityTag, new_val->SyslogIdentityTag)) {
-    *msg = tor_strdup("While Tor is running, changing "
-                      "SyslogIdentityTag is not allowed.");
-    return -1;
-  }
-
-  if ((old->HardwareAccel != new_val->HardwareAccel)
-      || !opt_streq(old->AccelName, new_val->AccelName)
-      || !opt_streq(old->AccelDir, new_val->AccelDir)) {
-    *msg = tor_strdup("While Tor is running, changing OpenSSL hardware "
-                      "acceleration engine is not allowed.");
-    return -1;
-  }
-
-  if (old->TestingTorNetwork != new_val->TestingTorNetwork) {
-    *msg = tor_strdup("While Tor is running, changing TestingTorNetwork "
-                      "is not allowed.");
-    return -1;
-  }
-
-  if (old->DisableAllSwap != new_val->DisableAllSwap) {
-    *msg = tor_strdup("While Tor is running, changing DisableAllSwap "
-                      "is not allowed.");
-    return -1;
-  }
-
-  if (old->TokenBucketRefillInterval != new_val->TokenBucketRefillInterval) {
-    *msg = tor_strdup("While Tor is running, changing TokenBucketRefill"
-                      "Interval is not allowed");
-    return -1;
-  }
-
-  if (old->HiddenServiceSingleHopMode != new_val->HiddenServiceSingleHopMode) {
-    *msg = tor_strdup("While Tor is running, changing "
-                      "HiddenServiceSingleHopMode is not allowed.");
-    return -1;
-  }
-
-  if (old->HiddenServiceNonAnonymousMode !=
-      new_val->HiddenServiceNonAnonymousMode) {
-    *msg = tor_strdup("While Tor is running, changing "
-                      "HiddenServiceNonAnonymousMode is not allowed.");
-    return -1;
-  }
-
-  if (old->DisableDebuggerAttachment &&
-      !new_val->DisableDebuggerAttachment) {
-    *msg = tor_strdup("While Tor is running, disabling "
-                      "DisableDebuggerAttachment is not allowed.");
-    return -1;
-  }
+#define BAD_CHANGE_TO(opt, how) do {                                    \
+    *msg = tor_strdup("While Tor is running"how", changing " #opt       \
+                      " is not allowed");                               \
+    return -1;                                                          \
+  } while (0)
+
+#define NO_CHANGE_BOOL(opt) \
+  if (! CFG_EQ_BOOL(old, new_val, opt)) BAD_CHANGE_TO(opt,"")
+#define NO_CHANGE_INT(opt) \
+  if (! CFG_EQ_INT(old, new_val, opt)) BAD_CHANGE_TO(opt,"")
+#define NO_CHANGE_STRING(opt) \
+  if (! CFG_EQ_STRING(old, new_val, opt)) BAD_CHANGE_TO(opt,"")
+
+  NO_CHANGE_STRING(PidFile);
+  NO_CHANGE_BOOL(RunAsDaemon);
+  NO_CHANGE_BOOL(Sandbox);
+  NO_CHANGE_STRING(DataDirectory);
+  NO_CHANGE_STRING(KeyDirectory);
+  NO_CHANGE_STRING(CacheDirectory);
+  NO_CHANGE_STRING(User);
+  NO_CHANGE_BOOL(KeepBindCapabilities);
+  NO_CHANGE_STRING(SyslogIdentityTag);
+  NO_CHANGE_STRING(AndroidIdentityTag);
+  NO_CHANGE_BOOL(HardwareAccel);
+  NO_CHANGE_STRING(AccelName);
+  NO_CHANGE_STRING(AccelDir);
+  NO_CHANGE_BOOL(TestingTorNetwork);
+  NO_CHANGE_BOOL(DisableAllSwap);
+  NO_CHANGE_INT(TokenBucketRefillInterval);
+  NO_CHANGE_BOOL(HiddenServiceSingleHopMode);
+  NO_CHANGE_BOOL(HiddenServiceNonAnonymousMode);
+  NO_CHANGE_BOOL(DisableDebuggerAttachment);
+  NO_CHANGE_BOOL(NoExec);
+  NO_CHANGE_INT(OwningControllerFD);
+  NO_CHANGE_BOOL(DisableSignalHandlers);
 
   if (sandbox_is_active()) {
-#define SB_NOCHANGE_STR(opt)                                            \
-    do {                                                                \
-      if (! opt_streq(old->opt, new_val->opt)) {                        \
-        *msg = tor_strdup("Can't change " #opt " while Sandbox is active"); \
-        return -1;                                                      \
-      }                                                                 \
-    } while (0)
+#define SB_NOCHANGE_STR(opt)                      \
+    if (! CFG_EQ_STRING(old, new_val, opt))       \
+      BAD_CHANGE_TO(opt," with Sandbox active")
+#define SB_NOCHANGE_LINELIST(opt)                  \
+    if (! CFG_EQ_LINELIST(old, new_val, opt))      \
+      BAD_CHANGE_TO(opt," with Sandbox active")
+#define SB_NOCHANGE_INT(opt)                       \
+    if (! CFG_EQ_INT(old, new_val, opt))           \
+      BAD_CHANGE_TO(opt," with Sandbox active")
 
     SB_NOCHANGE_STR(Address);
-    SB_NOCHANGE_STR(PidFile);
     SB_NOCHANGE_STR(ServerDNSResolvConfFile);
     SB_NOCHANGE_STR(DirPortFrontPage);
     SB_NOCHANGE_STR(CookieAuthFile);
     SB_NOCHANGE_STR(ExtORPortCookieAuthFile);
+    SB_NOCHANGE_LINELIST(Logs);
+    SB_NOCHANGE_INT(ConnLimit);
 
-#undef SB_NOCHANGE_STR
-
-    if (! config_lines_eq(old->Logs, new_val->Logs)) {
-      *msg = tor_strdup("Can't change Logs while Sandbox is active");
-      return -1;
-    }
-    if (old->ConnLimit != new_val->ConnLimit) {
-      *msg = tor_strdup("Can't change ConnLimit while Sandbox is active");
-      return -1;
-    }
     if (server_mode(old) != server_mode(new_val)) {
       *msg = tor_strdup("Can't start/stop being a server while "
                         "Sandbox is active");
@@ -4449,6 +4778,13 @@ options_transition_allowed(const or_options_t *old,
     }
   }
 
+#undef SB_NOCHANGE_LINELIST
+#undef SB_NOCHANGE_STR
+#undef SB_NOCHANGE_INT
+#undef BAD_CHANGE_TO
+#undef NO_CHANGE_BOOL
+#undef NO_CHANGE_INT
+#undef NO_CHANGE_STRING
   return 0;
 }
 
@@ -4458,21 +4794,20 @@ static int
 options_transition_affects_workers(const or_options_t *old_options,
                                    const or_options_t *new_options)
 {
-  if (!opt_streq(old_options->DataDirectory, new_options->DataDirectory) ||
-      old_options->NumCPUs != new_options->NumCPUs ||
-      !config_lines_eq(old_options->ORPort_lines, new_options->ORPort_lines) ||
-      old_options->ServerDNSSearchDomains !=
-                                       new_options->ServerDNSSearchDomains ||
-      old_options->SafeLogging_ != new_options->SafeLogging_ ||
-      old_options->ClientOnly != new_options->ClientOnly ||
-      server_mode(old_options) != server_mode(new_options) ||
+  YES_IF_CHANGED_STRING(DataDirectory);
+  YES_IF_CHANGED_INT(NumCPUs);
+  YES_IF_CHANGED_LINELIST(ORPort_lines);
+  YES_IF_CHANGED_BOOL(ServerDNSSearchDomains);
+  YES_IF_CHANGED_BOOL(SafeLogging_);
+  YES_IF_CHANGED_BOOL(ClientOnly);
+  YES_IF_CHANGED_BOOL(LogMessageDomains);
+  YES_IF_CHANGED_LINELIST(Logs);
+
+  if (server_mode(old_options) != server_mode(new_options) ||
       public_server_mode(old_options) != public_server_mode(new_options) ||
-      !config_lines_eq(old_options->Logs, new_options->Logs) ||
-      old_options->LogMessageDomains != new_options->LogMessageDomains)
+      dir_server_mode(old_options) != dir_server_mode(new_options))
     return 1;
 
-  /* Check whether log options match. */
-
   /* Nothing that changed matters. */
   return 0;
 }
@@ -4485,35 +4820,34 @@ options_transition_affects_descriptor(const or_options_t *old_options,
 {
   /* XXX We can be smarter here. If your DirPort isn't being
    * published and you just turned it off, no need to republish. Etc. */
-  if (!opt_streq(old_options->DataDirectory, new_options->DataDirectory) ||
-      !opt_streq(old_options->Nickname,new_options->Nickname) ||
-      !opt_streq(old_options->Address,new_options->Address) ||
-      !config_lines_eq(old_options->ExitPolicy,new_options->ExitPolicy) ||
-      old_options->ExitRelay != new_options->ExitRelay ||
-      old_options->ExitPolicyRejectPrivate !=
-        new_options->ExitPolicyRejectPrivate ||
-      old_options->ExitPolicyRejectLocalInterfaces !=
-        new_options->ExitPolicyRejectLocalInterfaces ||
-      old_options->IPv6Exit != new_options->IPv6Exit ||
-      !config_lines_eq(old_options->ORPort_lines,
-                       new_options->ORPort_lines) ||
-      !config_lines_eq(old_options->DirPort_lines,
-                       new_options->DirPort_lines) ||
-      old_options->ClientOnly != new_options->ClientOnly ||
-      old_options->DisableNetwork != new_options->DisableNetwork ||
-      old_options->PublishServerDescriptor_ !=
-        new_options->PublishServerDescriptor_ ||
-      get_effective_bwrate(old_options) != get_effective_bwrate(new_options) ||
+
+  YES_IF_CHANGED_STRING(DataDirectory);
+  YES_IF_CHANGED_STRING(Nickname);
+  YES_IF_CHANGED_STRING(Address);
+  YES_IF_CHANGED_LINELIST(ExitPolicy);
+  YES_IF_CHANGED_BOOL(ExitRelay);
+  YES_IF_CHANGED_BOOL(ExitPolicyRejectPrivate);
+  YES_IF_CHANGED_BOOL(ExitPolicyRejectLocalInterfaces);
+  YES_IF_CHANGED_BOOL(IPv6Exit);
+  YES_IF_CHANGED_LINELIST(ORPort_lines);
+  YES_IF_CHANGED_LINELIST(DirPort_lines);
+  YES_IF_CHANGED_LINELIST(DirPort_lines);
+  YES_IF_CHANGED_BOOL(ClientOnly);
+  YES_IF_CHANGED_BOOL(DisableNetwork);
+  YES_IF_CHANGED_BOOL(PublishServerDescriptor_);
+  YES_IF_CHANGED_STRING(ContactInfo);
+  YES_IF_CHANGED_STRING(BridgeDistribution);
+  YES_IF_CHANGED_LINELIST(MyFamily);
+  YES_IF_CHANGED_STRING(AccountingStart);
+  YES_IF_CHANGED_INT(AccountingMax);
+  YES_IF_CHANGED_INT(AccountingRule);
+  YES_IF_CHANGED_BOOL(DirCache);
+  YES_IF_CHANGED_BOOL(AssumeReachable);
+
+  if (get_effective_bwrate(old_options) != get_effective_bwrate(new_options) ||
       get_effective_bwburst(old_options) !=
         get_effective_bwburst(new_options) ||
-      !opt_streq(old_options->ContactInfo, new_options->ContactInfo) ||
-      !opt_streq(old_options->MyFamily, new_options->MyFamily) ||
-      !opt_streq(old_options->AccountingStart, new_options->AccountingStart) ||
-      old_options->AccountingMax != new_options->AccountingMax ||
-      old_options->AccountingRule != new_options->AccountingRule ||
-      public_server_mode(old_options) != public_server_mode(new_options) ||
-      old_options->DirCache != new_options->DirCache ||
-      old_options->AssumeReachable != new_options->AssumeReachable)
+      public_server_mode(old_options) != public_server_mode(new_options))
     return 1;
 
   return 0;
@@ -4560,7 +4894,7 @@ get_windows_conf_root(void)
   path[sizeof(path)-1] = '\0';
 #else
   strlcpy(path,tpath,sizeof(path));
-#endif
+#endif /* defined(UNICODE) */
 
   /* Now we need to free the memory that the path-idl was stored in.  In
    * typical Windows fashion, we can't just call 'free()' on it. */
@@ -4576,7 +4910,7 @@ get_windows_conf_root(void)
   is_set = 1;
   return path;
 }
-#endif
+#endif /* defined(_WIN32) */
 
 /** Return the default location for our torrc file (if <b>defaults_file</b> is
  * false), or for the torrc-defaults file (if <b>defaults_file</b> is true). */
@@ -4600,30 +4934,39 @@ get_default_conf_file(int defaults_file)
   }
 #else
   return defaults_file ? CONFDIR "/torrc-defaults" : CONFDIR "/torrc";
-#endif
+#endif /* defined(DISABLE_SYSTEM_TORRC) || ... */
 }
 
-/** Verify whether lst is a string containing valid-looking comma-separated
- * nicknames, or NULL. Will normalise <b>lst</b> to prefix '$' to any nickname
- * or fingerprint that needs it. Return 0 on success.
+/** Verify whether lst is a list of strings containing valid-looking
+ * comma-separated nicknames, or NULL. Will normalise <b>lst</b> to prefix '$'
+ * to any nickname or fingerprint that needs it. Also splits comma-separated
+ * list elements into multiple elements. Return 0 on success.
  * Warn and return -1 on failure.
  */
 static int
-check_nickname_list(char **lst, const char *name, char **msg)
+normalize_nickname_list(config_line_t **normalized_out,
+                        const config_line_t *lst, const char *name,
+                        char **msg)
 {
-  int r = 0;
-  smartlist_t *sl;
-  int changes = 0;
-
-  if (!*lst)
+  if (!lst)
     return 0;
-  sl = smartlist_new();
 
-  smartlist_split_string(sl, *lst, ",",
-    SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK|SPLIT_STRIP_SPACE, 0);
+  config_line_t *new_nicknames = NULL;
+  config_line_t **new_nicknames_next = &new_nicknames;
+
+  const config_line_t *cl;
+  for (cl = lst; cl; cl = cl->next) {
+    const char *line = cl->value;
+    if (!line)
+      continue;
 
-  SMARTLIST_FOREACH_BEGIN(sl, char *, s)
+    int valid_line = 1;
+    smartlist_t *sl = smartlist_new();
+    smartlist_split_string(sl, line, ",",
+      SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK|SPLIT_STRIP_SPACE, 0);
+    SMARTLIST_FOREACH_BEGIN(sl, char *, s)
     {
+      char *normalized = NULL;
       if (!is_legal_nickname_or_hexdigest(s)) {
         // check if first char is dollar
         if (s[0] != '$') {
@@ -4632,36 +4975,45 @@ check_nickname_list(char **lst, const char *name, char **msg)
           tor_asprintf(&prepended, "$%s", s);
 
           if (is_legal_nickname_or_hexdigest(prepended)) {
-            // The nickname is valid when it's prepended, swap the current
-            // version with a prepended one
-            tor_free(s);
-            SMARTLIST_REPLACE_CURRENT(sl, s, prepended);
-            changes = 1;
-            continue;
+            // The nickname is valid when it's prepended, set it as the
+            // normalized version
+            normalized = prepended;
+          } else {
+            // Still not valid, free and fallback to error message
+            tor_free(prepended);
           }
-
-          // Still not valid, free and fallback to error message
-          tor_free(prepended);
         }
 
-        tor_asprintf(msg, "Invalid nickname '%s' in %s line", s, name);
-        r = -1;
-        break;
+        if (!normalized) {
+          tor_asprintf(msg, "Invalid nickname '%s' in %s line", s, name);
+          valid_line = 0;
+          break;
+        }
+      } else {
+        normalized = tor_strdup(s);
       }
-    }
-  SMARTLIST_FOREACH_END(s);
 
-  // Replace the caller's nickname list with a fixed one
-  if (changes && r == 0) {
-    char *newNicknames = smartlist_join_strings(sl, ", ", 0, NULL);
-    tor_free(*lst);
-    *lst = newNicknames;
+      config_line_t *next = tor_malloc_zero(sizeof(*next));
+      next->key = tor_strdup(cl->key);
+      next->value = normalized;
+      next->next = NULL;
+
+      *new_nicknames_next = next;
+      new_nicknames_next = &next->next;
+    } SMARTLIST_FOREACH_END(s);
+
+    SMARTLIST_FOREACH(sl, char *, s, tor_free(s));
+    smartlist_free(sl);
+
+    if (!valid_line) {
+      config_free_lines(new_nicknames);
+      return -1;
+    }
   }
 
-  SMARTLIST_FOREACH(sl, char *, s, tor_free(s));
-  smartlist_free(sl);
+  *normalized_out = new_nicknames;
 
-  return r;
+  return 0;
 }
 
 /** Learn config file name from command line arguments, or use the default.
@@ -4733,9 +5085,9 @@ find_torrc_filename(config_line_t *cmd_arg,
       } else {
         fname = dflt ? tor_strdup(dflt) : NULL;
       }
-#else
+#else /* !(!defined(_WIN32)) */
       fname = dflt ? tor_strdup(dflt) : NULL;
-#endif
+#endif /* !defined(_WIN32) */
     }
   }
   return fname;
@@ -4809,7 +5161,8 @@ load_torrc_from_disk(config_line_t *cmd_arg, int defaults_file)
 /** Read a configuration file into <b>options</b>, finding the configuration
  * file location based on the command line.  After loading the file
  * call options_init_from_string() to load the config.
- * Return 0 if success, -1 if failure. */
+ * Return 0 if success, -1 if failure, and 1 if we succeeded but should exit
+ * anyway. */
 int
 options_init_from_torrc(int argc, char **argv)
 {
@@ -4836,22 +5189,27 @@ options_init_from_torrc(int argc, char **argv)
   if (config_line_find(cmdline_only_options, "-h") ||
       config_line_find(cmdline_only_options, "--help")) {
     print_usage();
-    exit(0);
+    return 1;
   }
   if (config_line_find(cmdline_only_options, "--list-torrc-options")) {
     /* For validating whether we've documented everything. */
     list_torrc_options();
-    exit(0);
+    return 1;
   }
   if (config_line_find(cmdline_only_options, "--list-deprecated-options")) {
     /* For validating whether what we have deprecated really exists. */
     list_deprecated_options();
-    exit(0);
+    return 1;
   }
 
   if (config_line_find(cmdline_only_options, "--version")) {
     printf("Tor version %s.\n",get_version());
-    exit(0);
+    return 1;
+  }
+
+  if (config_line_find(cmdline_only_options, "--list-modules")) {
+    list_enabled_modules();
+    return 1;
   }
 
   if (config_line_find(cmdline_only_options, "--library-versions")) {
@@ -4860,20 +5218,42 @@ options_init_from_torrc(int argc, char **argv)
     printf("Libevent\t\t%-15s\t\t%s\n",
                       tor_libevent_get_header_version_str(),
                       tor_libevent_get_version_str());
+#ifdef ENABLE_OPENSSL
     printf("OpenSSL \t\t%-15s\t\t%s\n",
                       crypto_openssl_get_header_version_str(),
                       crypto_openssl_get_version_str());
-    printf("Zlib    \t\t%-15s\t\t%s\n",
-                      tor_zlib_get_header_version_str(),
-                      tor_zlib_get_version_str());
+#endif
+#ifdef ENABLE_NSS
+    printf("NSS \t\t%-15s\t\t%s\n",
+           crypto_nss_get_header_version_str(),
+           crypto_nss_get_version_str());
+#endif
+    if (tor_compress_supports_method(ZLIB_METHOD)) {
+      printf("Zlib    \t\t%-15s\t\t%s\n",
+                        tor_compress_version_str(ZLIB_METHOD),
+                        tor_compress_header_version_str(ZLIB_METHOD));
+    }
+    if (tor_compress_supports_method(LZMA_METHOD)) {
+      printf("Liblzma \t\t%-15s\t\t%s\n",
+                        tor_compress_version_str(LZMA_METHOD),
+                        tor_compress_header_version_str(LZMA_METHOD));
+    }
+    if (tor_compress_supports_method(ZSTD_METHOD)) {
+      printf("Libzstd \t\t%-15s\t\t%s\n",
+                        tor_compress_version_str(ZSTD_METHOD),
+                        tor_compress_header_version_str(ZSTD_METHOD));
+    }
     //TODO: Hex versions?
-    exit(0);
+    return 1;
   }
 
   command = CMD_RUN_TOR;
   for (p_index = cmdline_only_options; p_index; p_index = p_index->next) {
     if (!strcmp(p_index->key,"--keygen")) {
       command = CMD_KEYGEN;
+    } else if (!strcmp(p_index->key, "--key-expiration")) {
+      command = CMD_KEY_EXPIRATION;
+      command_arg = p_index->value;
     } else if (!strcmp(p_index->key,"--list-fingerprint")) {
       command = CMD_LIST_FINGERPRINT;
     } else if (!strcmp(p_index->key, "--hash-password")) {
@@ -4925,7 +5305,8 @@ options_init_from_torrc(int argc, char **argv)
       get_options_mutable()->keygen_force_passphrase = FORCE_PASSPHRASE_OFF;
     } else {
       log_err(LD_CONFIG, "--no-passphrase specified without --keygen!");
-      exit(1);
+      retval = -1;
+      goto err;
     }
   }
 
@@ -4934,7 +5315,8 @@ options_init_from_torrc(int argc, char **argv)
       get_options_mutable()->change_key_passphrase = 1;
     } else {
       log_err(LD_CONFIG, "--newpass specified without --keygen!");
-      exit(1);
+      retval = -1;
+      goto err;
     }
   }
 
@@ -4944,17 +5326,20 @@ options_init_from_torrc(int argc, char **argv)
     if (fd_line) {
       if (get_options()->keygen_force_passphrase == FORCE_PASSPHRASE_OFF) {
         log_err(LD_CONFIG, "--no-passphrase specified with --passphrase-fd!");
-        exit(1);
+        retval = -1;
+        goto err;
       } else if (command != CMD_KEYGEN) {
         log_err(LD_CONFIG, "--passphrase-fd specified without --keygen!");
-        exit(1);
+        retval = -1;
+        goto err;
       } else {
         const char *v = fd_line->value;
         int ok = 1;
         long fd = tor_parse_long(v, 10, 0, INT_MAX, &ok, NULL);
         if (fd < 0 || ok == 0) {
           log_err(LD_CONFIG, "Invalid --passphrase-fd value %s", escaped(v));
-          exit(1);
+          retval = -1;
+          goto err;
         }
         get_options_mutable()->keygen_passphrase_fd = (int)fd;
         get_options_mutable()->use_keygen_passphrase_fd = 1;
@@ -4969,7 +5354,8 @@ options_init_from_torrc(int argc, char **argv)
     if (key_line) {
       if (command != CMD_KEYGEN) {
         log_err(LD_CONFIG, "--master-key without --keygen!");
-        exit(1);
+        retval = -1;
+        goto err;
       } else {
         get_options_mutable()->master_key_fname = tor_strdup(key_line->value);
       }
@@ -5005,6 +5391,7 @@ options_init_from_string(const char *cf_defaults, const char *cf,
   config_line_t *cl;
   int retval;
   setopt_err_t err = SETOPT_ERR_MISC;
+  int cf_has_include = 0;
   tor_assert(msg);
 
   oldoptions = global_options; /* get_options unfortunately asserts if
@@ -5016,12 +5403,16 @@ options_init_from_string(const char *cf_defaults, const char *cf,
   newoptions->command = command;
   newoptions->command_arg = command_arg ? tor_strdup(command_arg) : NULL;
 
+  smartlist_t *opened_files = smartlist_new();
   for (int i = 0; i < 2; ++i) {
     const char *body = i==0 ? cf_defaults : cf;
     if (!body)
       continue;
+
     /* get config lines, assign them */
-    retval = config_get_lines(body, &cl, 1);
+    retval = config_get_lines_include(body, &cl, 1,
+                                      body == cf ? &cf_has_include : NULL,
+                                      opened_files);
     if (retval < 0) {
       err = SETOPT_ERR_PARSE;
       goto err;
@@ -5049,6 +5440,9 @@ options_init_from_string(const char *cf_defaults, const char *cf,
     goto err;
   }
 
+  newoptions->IncludeUsed = cf_has_include;
+  newoptions->FilesOpenedByIncludes = opened_files;
+
   /* If this is a testing network configuration, change defaults
    * for a list of dependent config options, re-initialize newoptions
    * with the new defaults, and assign all options to it second time. */
@@ -5087,12 +5481,16 @@ options_init_from_string(const char *cf_defaults, const char *cf,
     newoptions->command_arg = command_arg ? tor_strdup(command_arg) : NULL;
 
     /* Assign all options a second time. */
+    opened_files = smartlist_new();
     for (int i = 0; i < 2; ++i) {
       const char *body = i==0 ? cf_defaults : cf;
       if (!body)
         continue;
+
       /* get config lines, assign them */
-      retval = config_get_lines(body, &cl, 1);
+      retval = config_get_lines_include(body, &cl, 1,
+                                        body == cf ? &cf_has_include : NULL,
+                                        opened_files);
       if (retval < 0) {
         err = SETOPT_ERR_PARSE;
         goto err;
@@ -5115,6 +5513,10 @@ options_init_from_string(const char *cf_defaults, const char *cf,
     }
   }
 
+  newoptions->IncludeUsed = cf_has_include;
+  in_option_validation = 1;
+  newoptions->FilesOpenedByIncludes = opened_files;
+
   /* Validate newoptions */
   if (options_validate(oldoptions, newoptions, newdefaultoptions,
                        0, msg) < 0) {
@@ -5126,17 +5528,26 @@ options_init_from_string(const char *cf_defaults, const char *cf,
     err = SETOPT_ERR_TRANSITION;
     goto err;
   }
+  in_option_validation = 0;
 
   if (set_options(newoptions, msg)) {
     err = SETOPT_ERR_SETTING;
     goto err; /* frees and replaces old options */
   }
+
   or_options_free(global_default_options);
   global_default_options = newdefaultoptions;
 
   return SETOPT_OK;
 
  err:
+  in_option_validation = 0;
+  if (opened_files) {
+    SMARTLIST_FOREACH(opened_files, char *, f, tor_free(f));
+    smartlist_free(opened_files);
+  }
+  // may have been set to opened_files, avoid double free
+  newoptions->FilesOpenedByIncludes = NULL;
   or_options_free(newoptions);
   or_options_free(newdefaultoptions);
   if (*msg) {
@@ -5218,39 +5629,56 @@ addressmap_register_auto(const char *from, const char *to,
   int from_wildcard = 0, to_wildcard = 0;
 
   *msg = "whoops, forgot the error message";
-  if (1) {
-    if (!strcmp(to, "*") || !strcmp(from, "*")) {
-      *msg = "can't remap from or to *";
-      return -1;
-    }
-    /* Detect asterisks in expressions of type: '*.example.com' */
-    if (!strncmp(from,"*.",2)) {
-      from += 2;
-      from_wildcard = 1;
-    }
-    if (!strncmp(to,"*.",2)) {
-      to += 2;
-      to_wildcard = 1;
-    }
 
-    if (to_wildcard && !from_wildcard) {
-      *msg =  "can only use wildcard (i.e. '*.') if 'from' address "
-        "uses wildcard also";
-      return -1;
-    }
+  if (!strcmp(to, "*") || !strcmp(from, "*")) {
+    *msg = "can't remap from or to *";
+    return -1;
+  }
+  /* Detect asterisks in expressions of type: '*.example.com' */
+  if (!strncmp(from,"*.",2)) {
+    from += 2;
+    from_wildcard = 1;
+  }
+  if (!strncmp(to,"*.",2)) {
+    to += 2;
+    to_wildcard = 1;
+  }
 
-    if (address_is_invalid_destination(to, 1)) {
-      *msg = "destination is invalid";
-      return -1;
-    }
+  if (to_wildcard && !from_wildcard) {
+    *msg =  "can only use wildcard (i.e. '*.') if 'from' address "
+      "uses wildcard also";
+    return -1;
+  }
 
-    addressmap_register(from, tor_strdup(to), expires, addrmap_source,
-                        from_wildcard, to_wildcard);
+  if (address_is_invalid_destination(to, 1)) {
+    *msg = "destination is invalid";
+    return -1;
   }
+
+  addressmap_register(from, tor_strdup(to), expires, addrmap_source,
+                      from_wildcard, to_wildcard);
+
   return 0;
 }
 
 /**
+ * As add_file_log, but open the file as appropriate.
+ */
+STATIC int
+open_and_add_file_log(const log_severity_list_t *severity,
+                      const char *filename, int truncate_log)
+{
+  int open_flags = O_WRONLY|O_CREAT;
+  open_flags |= truncate_log ? O_TRUNC : O_APPEND;
+
+  int fd = tor_open_cloexec(filename, open_flags, 0640);
+  if (fd < 0)
+    return -1;
+
+  return add_file_log(severity, filename, fd);
+}
+
+/**
  * Initialize the logs based on the configuration file.
  */
 static int
@@ -5313,7 +5741,7 @@ options_init_logs(const or_options_t *old_options, or_options_t *options,
                            SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2);
 
     if (smartlist_len(elts) == 0)
-      smartlist_add(elts, tor_strdup("stdout"));
+      smartlist_add_strdup(elts, "stdout");
 
     if (smartlist_len(elts) == 1 &&
         (!strcasecmp(smartlist_get(elts,0), "stdout") ||
@@ -5332,16 +5760,29 @@ options_init_logs(const or_options_t *old_options, or_options_t *options,
       }
       goto cleanup;
     }
-    if (smartlist_len(elts) == 1 &&
-        !strcasecmp(smartlist_get(elts,0), "syslog")) {
+    if (smartlist_len(elts) == 1) {
+      if (!strcasecmp(smartlist_get(elts,0), "syslog")) {
 #ifdef HAVE_SYSLOG_H
-      if (!validate_only) {
-        add_syslog_log(severity, options->SyslogIdentityTag);
+        if (!validate_only) {
+          add_syslog_log(severity, options->SyslogIdentityTag);
+        }
+#else
+        log_warn(LD_CONFIG, "Syslog is not supported on this system. Sorry.");
+#endif /* defined(HAVE_SYSLOG_H) */
+        goto cleanup;
       }
+
+      if (!strcasecmp(smartlist_get(elts, 0), "android")) {
+#ifdef HAVE_ANDROID_LOG_H
+        if (!validate_only) {
+          add_android_log(severity, options->AndroidIdentityTag);
+        }
 #else
-      log_warn(LD_CONFIG, "Syslog is not supported on this system. Sorry.");
-#endif
-      goto cleanup;
+        log_warn(LD_CONFIG, "Android logging is not supported"
+                            " on this system. Sorry.");
+#endif // HAVE_ANDROID_LOG_H.
+        goto cleanup;
+      }
     }
 
     if (smartlist_len(elts) == 2 &&
@@ -5362,7 +5803,7 @@ options_init_logs(const or_options_t *old_options, or_options_t *options,
               }
           }
         }
-        if (add_file_log(severity, fname, truncate_log) < 0) {
+        if (open_and_add_file_log(severity, fname, truncate_log) < 0) {
           log_warn(LD_CONFIG, "Couldn't open file for 'Log %s': %s",
                    opt->value, strerror(errno));
           ok = 0;
@@ -5427,7 +5868,7 @@ validate_transport_socks_arguments(const smartlist_t *args)
 
 /** Deallocate a bridge_line_t structure. */
 /* private */ void
-bridge_line_free(bridge_line_t *bridge_line)
+bridge_line_free_(bridge_line_t *bridge_line)
 {
   if (!bridge_line)
     return;
@@ -5657,6 +6098,15 @@ parse_transport_line(const or_options_t *options,
     goto err;
   }
 
+  if (is_managed && options->NoExec) {
+    log_warn(LD_CONFIG,
+             "Managed proxies are not compatible with NoExec mode; ignoring."
+             "(%sTransportPlugin line was %s)",
+             server ? "Server" : "Client", escaped(line));
+    r = 0;
+    goto done;
+  }
+
   if (is_managed) {
     /* managed */
 
@@ -5848,7 +6298,7 @@ get_options_from_transport_options_line(const char *line,const char *transport)
     }
 
     /* add it to the options smartlist */
-    smartlist_add(options, tor_strdup(option));
+    smartlist_add_strdup(options, option);
     log_debug(LD_CONFIG, "Added %s to the list of options", escaped(option));
   } SMARTLIST_FOREACH_END(option);
 
@@ -5927,6 +6377,8 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type,
   dirinfo_type_t type = 0;
   double weight = 1.0;
 
+  memset(v3_digest, 0, sizeof(v3_digest));
+
   items = smartlist_new();
   smartlist_split_string(items, line, NULL,
                          SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
@@ -6006,10 +6458,18 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type,
   }
   addrport = smartlist_get(items, 0);
   smartlist_del_keeporder(items, 0);
-  if (addr_port_lookup(LOG_WARN, addrport, &address, NULL, &dir_port)<0) {
-    log_warn(LD_CONFIG, "Error parsing DirAuthority address '%s'", addrport);
+
+  if (tor_addr_port_split(LOG_WARN, addrport, &address, &dir_port) < 0) {
+    log_warn(LD_CONFIG, "Error parsing DirAuthority address '%s'.", addrport);
+    goto err;
+  }
+
+  if (!string_is_valid_ipv4_address(address)) {
+    log_warn(LD_CONFIG, "Error parsing DirAuthority address '%s' "
+                        "(invalid IPv4 address)", address);
     goto err;
   }
+
   if (!dir_port) {
     log_warn(LD_CONFIG, "Missing port in DirAuthority address '%s'",addrport);
     goto err;
@@ -6173,16 +6633,16 @@ port_cfg_new(size_t namelen)
   tor_assert(namelen <= SIZE_T_CEILING - sizeof(port_cfg_t) - 1);
   port_cfg_t *cfg = tor_malloc_zero(sizeof(port_cfg_t) + namelen + 1);
   cfg->entry_cfg.ipv4_traffic = 1;
+  cfg->entry_cfg.ipv6_traffic = 1;
   cfg->entry_cfg.dns_request = 1;
   cfg->entry_cfg.onion_traffic = 1;
-  cfg->entry_cfg.cache_ipv4_answers = 1;
   cfg->entry_cfg.prefer_ipv6_virtaddr = 1;
   return cfg;
 }
 
 /** Free all storage held in <b>port</b> */
 STATIC void
-port_cfg_free(port_cfg_t *port)
+port_cfg_free_(port_cfg_t *port)
 {
   tor_free(port);
 }
@@ -6190,8 +6650,9 @@ port_cfg_free(port_cfg_t *port)
 /** Warn for every port in <b>ports</b> of type <b>listener_type</b> that is
  * on a publicly routable address. */
 static void
-warn_nonlocal_client_ports(const smartlist_t *ports, const char *portname,
-                           int listener_type)
+warn_nonlocal_client_ports(const smartlist_t *ports,
+                           const char *portname,
+                           const int listener_type)
 {
   SMARTLIST_FOREACH_BEGIN(ports, const port_cfg_t *, port) {
     if (port->type != listener_type)
@@ -6348,16 +6809,60 @@ warn_client_dns_cache(const char *option, int disabling)
 }
 
 /**
+ * Validate the configured bridge distribution method from a BridgeDistribution
+ * config line.
+ *
+ * The input <b>bd</b>, is a string taken from the BridgeDistribution config
+ * line (if present).  If the option wasn't set, return 0 immediately.  The
+ * BridgeDistribution option is then validated.  Currently valid, recognised
+ * options are:
+ *
+ * - "none"
+ * - "any"
+ * - "https"
+ * - "email"
+ * - "moat"
+ * - "hyphae"
+ *
+ * If the option string is unrecognised, a warning will be logged and 0 is
+ * returned.  If the option string contains an invalid character, -1 is
+ * returned.
+ **/
+STATIC int
+check_bridge_distribution_setting(const char *bd)
+{
+  if (bd == NULL)
+    return 0;
+
+  const char *RECOGNIZED[] = {
+    "none", "any", "https", "email", "moat", "hyphae"
+  };
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(RECOGNIZED); ++i) {
+    if (!strcmp(bd, RECOGNIZED[i]))
+      return 0;
+  }
+
+  const char *cp = bd;
+  //  Method = (KeywordChar | "_") +
+  while (TOR_ISALNUM(*cp) || *cp == '-' || *cp == '_')
+    ++cp;
+
+  if (*cp == 0) {
+    log_warn(LD_CONFIG, "Unrecognized BridgeDistribution value %s. I'll "
+           "assume you know what you are doing...", escaped(bd));
+    return 0; // we reached the end of the string; all is well
+  } else {
+    return -1; // we found a bad character in the string.
+  }
+}
+
+/**
  * Parse port configuration for a single port type.
  *
- * Read entries of the "FooPort" type from the list <b>ports</b>, and
- * entries of the "FooListenAddress" type from the list
- * <b>listenaddrs</b>.  Two syntaxes are supported: a legacy syntax
- * where FooPort is at most a single entry containing a port number and
- * where FooListenAddress has any number of address:port combinations;
- * and a new syntax where there are no FooListenAddress entries and
- * where FooPort can have any number of entries of the format
- * "[Address:][Port] IsolationOptions".
+ * Read entries of the "FooPort" type from the list <b>ports</b>.  Syntax is
+ * that FooPort can have any number of entries of the format
+ *  "[Address:][Port] IsolationOptions".
  *
  * In log messages, describe the port type as <b>portname</b>.
  *
@@ -6371,9 +6876,6 @@ warn_client_dns_cache(const char *option, int disabling)
  * ports are not on a local address.  If CL_PORT_FORBID_NONLOCAL is set,
  * this is a control port with no password set: don't even allow it.
  *
- * Unless CL_PORT_ALLOW_EXTRA_LISTENADDR is set in <b>flags</b>, warn
- * if FooListenAddress is set but FooPort is 0.
- *
  * If CL_PORT_SERVER_OPTIONS is set in <b>flags</b>, do not allow stream
  * isolation options in the FooPort entries; instead allow the
  * server-port option set.
@@ -6388,7 +6890,6 @@ warn_client_dns_cache(const char *option, int disabling)
 STATIC int
 parse_port_config(smartlist_t *out,
                   const config_line_t *ports,
-                  const config_line_t *listenaddrs,
                   const char *portname,
                   int listener_type,
                   const char *defaultaddr,
@@ -6405,90 +6906,12 @@ parse_port_config(smartlist_t *out,
   const unsigned forbid_nonlocal = flags & CL_PORT_FORBID_NONLOCAL;
   const unsigned default_to_group_writable =
     flags & CL_PORT_DFLT_GROUP_WRITABLE;
-  const unsigned allow_spurious_listenaddr =
-    flags & CL_PORT_ALLOW_EXTRA_LISTENADDR;
   const unsigned takes_hostnames = flags & CL_PORT_TAKES_HOSTNAMES;
   const unsigned is_unix_socket = flags & CL_PORT_IS_UNIXSOCKET;
   int got_zero_port=0, got_nonzero_port=0;
   char *unix_socket_path = NULL;
 
-  /* FooListenAddress is deprecated; let's make it work like it used to work,
-   * though. */
-  if (listenaddrs) {
-    int mainport = defaultport;
-
-    if (ports && ports->next) {
-      log_warn(LD_CONFIG, "%sListenAddress can't be used when there are "
-               "multiple %sPort lines", portname, portname);
-      return -1;
-    } else if (ports) {
-      if (!strcmp(ports->value, "auto")) {
-        mainport = CFG_AUTO_PORT;
-      } else {
-        int ok;
-        mainport = (int)tor_parse_long(ports->value, 10, 0, 65535, &ok, NULL);
-        if (!ok) {
-          log_warn(LD_CONFIG, "%sListenAddress can only be used with a single "
-                   "%sPort with value \"auto\" or 1-65535 and no options set.",
-                   portname, portname);
-          return -1;
-        }
-      }
-    }
-
-    if (mainport == 0) {
-      if (allow_spurious_listenaddr)
-        return 1; /*DOCDOC*/
-      log_warn(LD_CONFIG, "%sPort must be defined if %sListenAddress is used",
-               portname, portname);
-      return -1;
-    }
-
-    if (use_server_options && out) {
-      /* Add a no_listen port. */
-      port_cfg_t *cfg = port_cfg_new(0);
-      cfg->type = listener_type;
-      cfg->port = mainport;
-      tor_addr_make_unspec(&cfg->addr); /* Server ports default to 0.0.0.0 */
-      cfg->server_cfg.no_listen = 1;
-      cfg->server_cfg.bind_ipv4_only = 1;
-      /* cfg->entry_cfg defaults are already set by port_cfg_new */
-      smartlist_add(out, cfg);
-    }
-
-    for (; listenaddrs; listenaddrs = listenaddrs->next) {
-      tor_addr_t addr;
-      uint16_t port = 0;
-      if (tor_addr_port_lookup(listenaddrs->value, &addr, &port) < 0) {
-        log_warn(LD_CONFIG, "Unable to parse %sListenAddress '%s'",
-                 portname, listenaddrs->value);
-        return -1;
-      }
-      if (out) {
-        port_cfg_t *cfg = port_cfg_new(0);
-        cfg->type = listener_type;
-        cfg->port = port ? port : mainport;
-        tor_addr_copy(&cfg->addr, &addr);
-        cfg->entry_cfg.session_group = SESSION_GROUP_UNSET;
-        cfg->entry_cfg.isolation_flags = ISO_DEFAULT;
-        cfg->server_cfg.no_advertise = 1;
-        smartlist_add(out, cfg);
-      }
-    }
-
-    if (warn_nonlocal && out) {
-      if (is_control)
-        warn_nonlocal_controller_ports(out, forbid_nonlocal);
-      else if (is_ext_orport)
-        warn_nonlocal_ext_orports(out, portname);
-      else
-        warn_nonlocal_client_ports(out, portname, listener_type);
-    }
-    return 0;
-  } /* end if (listenaddrs) */
-
-  /* No ListenAddress lines. If there's no FooPort, then maybe make a default
-   * one. */
+  /* If there's no FooPort, then maybe make a default one. */
   if (! ports) {
     if (defaultport && defaultaddr && out) {
       port_cfg_t *cfg = port_cfg_new(is_unix_socket ? strlen(defaultaddr) : 0);
@@ -6515,6 +6938,8 @@ parse_port_config(smartlist_t *out,
 
   for (; ports; ports = ports->next) {
     tor_addr_t addr;
+    tor_addr_make_unspec(&addr);
+
     int port;
     int sessiongroup = SESSION_GROUP_UNSET;
     unsigned isolation = ISO_DEFAULT;
@@ -6526,9 +6951,9 @@ parse_port_config(smartlist_t *out,
     /* This must be kept in sync with port_cfg_new's defaults */
     int no_listen = 0, no_advertise = 0, all_addrs = 0,
       bind_ipv4_only = 0, bind_ipv6_only = 0,
-      ipv4_traffic = 1, ipv6_traffic = 0, prefer_ipv6 = 0, dns_request = 1,
+      ipv4_traffic = 1, ipv6_traffic = 1, prefer_ipv6 = 0, dns_request = 1,
       onion_traffic = 1,
-      cache_ipv4 = 1, use_cached_ipv4 = 0,
+      cache_ipv4 = 0, use_cached_ipv4 = 0,
       cache_ipv6 = 0, use_cached_ipv6 = 0,
       prefer_ipv6_automap = 1, world_writable = 0, group_writable = 0,
       relax_dirmode_check = 0,
@@ -6577,7 +7002,7 @@ parse_port_config(smartlist_t *out,
         port = 0;
       else
         port = 1;
-    } else if (!strcmp(addrport, "auto")) {
+    } else if (!strcasecmp(addrport, "auto")) {
       port = CFG_AUTO_PORT;
       int af = tor_addr_parse(&addr, defaultaddr);
       tor_assert(af >= 0);
@@ -6627,7 +7052,7 @@ parse_port_config(smartlist_t *out,
         } else if (!strcasecmp(elt, "AllAddrs")) {
 
           all_addrs = 1;
-#endif
+#endif /* 0 */
         } else if (!strcasecmp(elt, "IPv4Only")) {
           bind_ipv4_only = 1;
         } else if (!strcasecmp(elt, "IPv6Only")) {
@@ -6650,13 +7075,13 @@ parse_port_config(smartlist_t *out,
                  portname, escaped(ports->value));
         goto err;
       }
-      if (bind_ipv4_only && tor_addr_family(&addr) == AF_INET6) {
-        log_warn(LD_CONFIG, "Could not interpret %sPort address as IPv6",
+      if (bind_ipv4_only && tor_addr_family(&addr) != AF_INET) {
+        log_warn(LD_CONFIG, "Could not interpret %sPort address as IPv4",
                  portname);
         goto err;
       }
-      if (bind_ipv6_only && tor_addr_family(&addr) == AF_INET) {
-        log_warn(LD_CONFIG, "Could not interpret %sPort address as IPv4",
+      if (bind_ipv6_only && tor_addr_family(&addr) != AF_INET6) {
+        log_warn(LD_CONFIG, "Could not interpret %sPort address as IPv6",
                  portname);
         goto err;
       }
@@ -6669,7 +7094,7 @@ parse_port_config(smartlist_t *out,
         if (!strcasecmpstart(elt, "SessionGroup=")) {
           int group = (int)tor_parse_long(elt+strlen("SessionGroup="),
                                           10, 0, INT_MAX, &ok, NULL);
-          if (!ok || !allow_no_stream_options) {
+          if (!ok || allow_no_stream_options) {
             log_warn(LD_CONFIG, "Invalid %sPort option '%s'",
                      portname, escaped(elt));
             goto err;
@@ -6889,6 +7314,7 @@ parse_port_config(smartlist_t *out,
     SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp));
     smartlist_clear(elts);
     tor_free(addrport);
+    tor_free(unix_socket_path);
   }
 
   if (warn_nonlocal && out) {
@@ -6956,39 +7382,48 @@ parse_ports(or_options_t *options, int validate_only,
 
   *n_ports_out = 0;
 
-  const unsigned gw_flag = options->SocksSocketsGroupWritable ?
+  const unsigned gw_flag = options->UnixSocksGroupWritable ?
     CL_PORT_DFLT_GROUP_WRITABLE : 0;
   if (parse_port_config(ports,
-             options->SocksPort_lines, options->SocksListenAddress,
+             options->SocksPort_lines,
              "Socks", CONN_TYPE_AP_LISTENER,
              "127.0.0.1", 9050,
-             CL_PORT_WARN_NONLOCAL|CL_PORT_ALLOW_EXTRA_LISTENADDR|
-             CL_PORT_TAKES_HOSTNAMES|gw_flag) < 0) {
-    *msg = tor_strdup("Invalid SocksPort/SocksListenAddress configuration");
+             ((validate_only ? 0 : CL_PORT_WARN_NONLOCAL)
+              | CL_PORT_TAKES_HOSTNAMES | gw_flag)) < 0) {
+    *msg = tor_strdup("Invalid SocksPort configuration");
     goto err;
   }
   if (parse_port_config(ports,
-                        options->DNSPort_lines, options->DNSListenAddress,
+                        options->DNSPort_lines,
                         "DNS", CONN_TYPE_AP_DNS_LISTENER,
                         "127.0.0.1", 0,
                         CL_PORT_WARN_NONLOCAL|CL_PORT_TAKES_HOSTNAMES) < 0) {
-    *msg = tor_strdup("Invalid DNSPort/DNSListenAddress configuration");
+    *msg = tor_strdup("Invalid DNSPort configuration");
     goto err;
   }
   if (parse_port_config(ports,
-                        options->TransPort_lines, options->TransListenAddress,
+                        options->TransPort_lines,
                         "Trans", CONN_TYPE_AP_TRANS_LISTENER,
                         "127.0.0.1", 0,
                         CL_PORT_WARN_NONLOCAL) < 0) {
-    *msg = tor_strdup("Invalid TransPort/TransListenAddress configuration");
+    *msg = tor_strdup("Invalid TransPort configuration");
     goto err;
   }
   if (parse_port_config(ports,
-                        options->NATDPort_lines, options->NATDListenAddress,
+                        options->NATDPort_lines,
                         "NATD", CONN_TYPE_AP_NATD_LISTENER,
                         "127.0.0.1", 0,
                         CL_PORT_WARN_NONLOCAL) < 0) {
-    *msg = tor_strdup("Invalid NatdPort/NatdListenAddress configuration");
+    *msg = tor_strdup("Invalid NatdPort configuration");
+    goto err;
+  }
+  if (parse_port_config(ports,
+                        options->HTTPTunnelPort_lines,
+                        "HTTP Tunnel", CONN_TYPE_AP_HTTP_CONNECT_LISTENER,
+                        "127.0.0.1", 0,
+                        ((validate_only ? 0 : CL_PORT_WARN_NONLOCAL)
+                         | CL_PORT_TAKES_HOSTNAMES | gw_flag)) < 0) {
+    *msg = tor_strdup("Invalid HTTPTunnelPort configuration");
     goto err;
   }
   {
@@ -7004,16 +7439,14 @@ parse_ports(or_options_t *options, int validate_only,
 
     if (parse_port_config(ports,
                           options->ControlPort_lines,
-                          options->ControlListenAddress,
                           "Control", CONN_TYPE_CONTROL_LISTENER,
                           "127.0.0.1", 0,
                           control_port_flags) < 0) {
-      *msg = tor_strdup("Invalid ControlPort/ControlListenAddress "
-                        "configuration");
+      *msg = tor_strdup("Invalid ControlPort configuration");
       goto err;
     }
 
-    if (parse_port_config(ports, options->ControlSocket, NULL,
+    if (parse_port_config(ports, options->ControlSocket,
                           "ControlSocket",
                           CONN_TYPE_CONTROL_LISTENER, NULL, 0,
                           control_port_flags | CL_PORT_IS_UNIXSOCKET) < 0) {
@@ -7023,15 +7456,15 @@ parse_ports(or_options_t *options, int validate_only,
   }
   if (! options->ClientOnly) {
     if (parse_port_config(ports,
-                          options->ORPort_lines, options->ORListenAddress,
+                          options->ORPort_lines,
                           "OR", CONN_TYPE_OR_LISTENER,
                           "0.0.0.0", 0,
                           CL_PORT_SERVER_OPTIONS) < 0) {
-      *msg = tor_strdup("Invalid ORPort/ORListenAddress configuration");
+      *msg = tor_strdup("Invalid ORPort configuration");
       goto err;
     }
     if (parse_port_config(ports,
-                          options->ExtORPort_lines, NULL,
+                          options->ExtORPort_lines,
                           "ExtOR", CONN_TYPE_EXT_OR_LISTENER,
                           "127.0.0.1", 0,
                           CL_PORT_SERVER_OPTIONS|CL_PORT_WARN_NONLOCAL) < 0) {
@@ -7039,11 +7472,11 @@ parse_ports(or_options_t *options, int validate_only,
       goto err;
     }
     if (parse_port_config(ports,
-                          options->DirPort_lines, options->DirListenAddress,
+                          options->DirPort_lines,
                           "Dir", CONN_TYPE_DIR_LISTENER,
                           "0.0.0.0", 0,
                           CL_PORT_SERVER_OPTIONS) < 0) {
-      *msg = tor_strdup("Invalid DirPort/DirListenAddress configuration");
+      *msg = tor_strdup("Invalid DirPort configuration");
       goto err;
     }
   }
@@ -7070,6 +7503,8 @@ parse_ports(or_options_t *options, int validate_only,
     !! count_real_listeners(ports, CONN_TYPE_AP_TRANS_LISTENER, 1);
   options->NATDPort_set =
     !! count_real_listeners(ports, CONN_TYPE_AP_NATD_LISTENER, 1);
+  options->HTTPTunnelPort_set =
+    !! count_real_listeners(ports, CONN_TYPE_AP_HTTP_CONNECT_LISTENER, 1);
   /* Use options->ControlSocket to test if a control socket is set */
   options->ControlPort_set =
     !! count_real_listeners(ports, CONN_TYPE_CONTROL_LISTENER, 0);
@@ -7188,8 +7623,8 @@ check_server_ports(const smartlist_t *ports,
   }
   if (n_orport_advertised && !n_orport_advertised_ipv4 &&
       !options->BridgeRelay) {
-    log_warn(LD_CONFIG, "Configured non-bridge only to listen on an IPv6 "
-             "address.");
+    log_warn(LD_CONFIG, "Configured public relay to listen only on an IPv6 "
+             "address. Tor needs to listen on an IPv4 address too.");
     r = -1;
   }
 
@@ -7387,60 +7822,81 @@ port_exists_by_type_addr32h_port(int listener_type, uint32_t addr_ipv4h,
                                        check_wildcard);
 }
 
-/** Adjust the value of options->DataDirectory, or fill it in if it's
- * absent. Return 0 on success, -1 on failure. */
-static int
-normalize_data_directory(or_options_t *options)
+/** Allocate and return a good value for the DataDirectory based on
+ *  <b>val</b>, which may be NULL.  Return NULL on failure. */
+static char *
+get_data_directory(const char *val)
 {
 #ifdef _WIN32
-  char *p;
-  if (options->DataDirectory)
-    return 0; /* all set */
-  p = tor_malloc(MAX_PATH);
-  strlcpy(p,get_windows_conf_root(),MAX_PATH);
-  options->DataDirectory = p;
-  return 0;
-#else
-  const char *d = options->DataDirectory;
+  if (val) {
+    return tor_strdup(val);
+  } else {
+    return tor_strdup(get_windows_conf_root());
+  }
+#else /* !(defined(_WIN32)) */
+  const char *d = val;
   if (!d)
     d = "~/.tor";
 
- if (strncmp(d,"~/",2) == 0) {
-   char *fn = expand_filename(d);
-   if (!fn) {
-     log_warn(LD_CONFIG,"Failed to expand filename \"%s\".", d);
-     return -1;
-   }
-   if (!options->DataDirectory && !strcmp(fn,"/.tor")) {
-     /* If our homedir is /, we probably don't want to use it. */
-     /* Default to LOCALSTATEDIR/tor which is probably closer to what we
-      * want. */
-     log_warn(LD_CONFIG,
-              "Default DataDirectory is \"~/.tor\".  This expands to "
-              "\"%s\", which is probably not what you want.  Using "
-              "\"%s"PATH_SEPARATOR"tor\" instead", fn, LOCALSTATEDIR);
-     tor_free(fn);
-     fn = tor_strdup(LOCALSTATEDIR PATH_SEPARATOR "tor");
-   }
-   tor_free(options->DataDirectory);
-   options->DataDirectory = fn;
- }
- return 0;
-#endif
+  if (!strcmpstart(d, "~/")) {
+    char *fn = expand_filename(d);
+    if (!fn) {
+      log_warn(LD_CONFIG,"Failed to expand filename \"%s\".", d);
+      return NULL;
+    }
+    if (!val && !strcmp(fn,"/.tor")) {
+      /* If our homedir is /, we probably don't want to use it. */
+      /* Default to LOCALSTATEDIR/tor which is probably closer to what we
+       * want. */
+      log_warn(LD_CONFIG,
+               "Default DataDirectory is \"~/.tor\".  This expands to "
+               "\"%s\", which is probably not what you want.  Using "
+               "\"%s"PATH_SEPARATOR"tor\" instead", fn, LOCALSTATEDIR);
+      tor_free(fn);
+      fn = tor_strdup(LOCALSTATEDIR PATH_SEPARATOR "tor");
+    }
+    return fn;
+  }
+  return tor_strdup(d);
+#endif /* defined(_WIN32) */
 }
 
-/** Check and normalize the value of options->DataDirectory; return 0 if it
- * is sane, -1 otherwise. */
+/** Check and normalize the values of options->{Key,Data,Cache}Directory;
+ * return 0 if it is sane, -1 otherwise. */
 static int
-validate_data_directory(or_options_t *options)
+validate_data_directories(or_options_t *options)
 {
-  if (normalize_data_directory(options) < 0)
+  tor_free(options->DataDirectory);
+  options->DataDirectory = get_data_directory(options->DataDirectory_option);
+  if (!options->DataDirectory)
     return -1;
-  tor_assert(options->DataDirectory);
   if (strlen(options->DataDirectory) > (512-128)) {
     log_warn(LD_CONFIG, "DataDirectory is too long.");
     return -1;
   }
+
+  tor_free(options->KeyDirectory);
+  if (options->KeyDirectory_option) {
+    options->KeyDirectory = get_data_directory(options->KeyDirectory_option);
+    if (!options->KeyDirectory)
+      return -1;
+  } else {
+    /* Default to the data directory's keys subdir */
+    tor_asprintf(&options->KeyDirectory, "%s"PATH_SEPARATOR"keys",
+                 options->DataDirectory);
+  }
+
+  tor_free(options->CacheDirectory);
+  if (options->CacheDirectory_option) {
+    options->CacheDirectory = get_data_directory(
+                                             options->CacheDirectory_option);
+    if (!options->CacheDirectory)
+      return -1;
+  } else {
+    /* Default to the data directory. */
+    options->CacheDirectory = tor_strdup(options->DataDirectory);
+  }
+
   return 0;
 }
 
@@ -7581,53 +8037,56 @@ init_libevent(const or_options_t *options)
   suppress_libevent_log_msg(NULL);
 }
 
-/** Return a newly allocated string holding a filename relative to the data
- * directory.  If <b>sub1</b> is present, it is the first path component after
+/** Return a newly allocated string holding a filename relative to the
+ * directory in <b>options</b> specified by <b>roottype</b>.
+ * If <b>sub1</b> is present, it is the first path component after
  * the data directory.  If <b>sub2</b> is also present, it is the second path
  * component after the data directory.  If <b>suffix</b> is present, it
  * is appended to the filename.
  *
- * Examples:
- *    get_datadir_fname2_suffix("a", NULL, NULL) -> $DATADIR/a
- *    get_datadir_fname2_suffix("a", NULL, ".tmp") -> $DATADIR/a.tmp
- *    get_datadir_fname2_suffix("a", "b", ".tmp") -> $DATADIR/a/b/.tmp
- *    get_datadir_fname2_suffix("a", "b", NULL) -> $DATADIR/a/b
- *
- * Note: Consider using the get_datadir_fname* macros in or.h.
+ * Note: Consider using macros in config.h that wrap this function;
+ * you should probably never need to call it as-is.
  */
 MOCK_IMPL(char *,
-options_get_datadir_fname2_suffix,(const or_options_t *options,
-                                   const char *sub1, const char *sub2,
-                                   const char *suffix))
+options_get_dir_fname2_suffix,(const or_options_t *options,
+                               directory_root_t roottype,
+                               const char *sub1, const char *sub2,
+                               const char *suffix))
 {
-  char *fname = NULL;
-  size_t len;
   tor_assert(options);
-  tor_assert(options->DataDirectory);
-  tor_assert(sub1 || !sub2); /* If sub2 is present, sub1 must be present. */
-  len = strlen(options->DataDirectory);
-  if (sub1) {
-    len += strlen(sub1)+1;
-    if (sub2)
-      len += strlen(sub2)+1;
-  }
-  if (suffix)
-    len += strlen(suffix);
-  len++;
-  fname = tor_malloc(len);
-  if (sub1) {
-    if (sub2) {
-      tor_snprintf(fname, len, "%s"PATH_SEPARATOR"%s"PATH_SEPARATOR"%s",
-                   options->DataDirectory, sub1, sub2);
-    } else {
-      tor_snprintf(fname, len, "%s"PATH_SEPARATOR"%s",
-                   options->DataDirectory, sub1);
-    }
+
+  const char *rootdir = NULL;
+  switch (roottype) {
+    case DIRROOT_DATADIR:
+      rootdir = options->DataDirectory;
+      break;
+    case DIRROOT_CACHEDIR:
+      rootdir = options->CacheDirectory;
+      break;
+    case DIRROOT_KEYDIR:
+      rootdir = options->KeyDirectory;
+      break;
+    default:
+      tor_assert_unreached();
+      break;
+  }
+  tor_assert(rootdir);
+
+  if (!suffix)
+    suffix = "";
+
+  char *fname = NULL;
+
+  if (sub1 == NULL) {
+    tor_asprintf(&fname, "%s%s", rootdir, suffix);
+    tor_assert(!sub2); /* If sub2 is present, sub1 must be present. */
+  } else if (sub2 == NULL) {
+    tor_asprintf(&fname, "%s"PATH_SEPARATOR"%s%s", rootdir, sub1, suffix);
   } else {
-    strlcpy(fname, options->DataDirectory, len);
+    tor_asprintf(&fname, "%s"PATH_SEPARATOR"%s"PATH_SEPARATOR"%s%s",
+                 rootdir, sub1, sub2, suffix);
   }
-  if (suffix)
-    strlcat(fname, suffix, len);
+
   return fname;
 }
 
@@ -7668,28 +8127,6 @@ write_to_data_subdir(const char* subdir, const char* fname,
   return return_val;
 }
 
-/** Given a file name check to see whether the file exists but has not been
- * modified for a very long time.  If so, remove it. */
-void
-remove_file_if_very_old(const char *fname, time_t now)
-{
-#define VERY_OLD_FILE_AGE (28*24*60*60)
-  struct stat st;
-
-  log_debug(LD_FS, "stat()ing %s", fname);
-  if (stat(sandbox_intern_string(fname), &st)==0 &&
-      st.st_mtime < now-VERY_OLD_FILE_AGE) {
-    char buf[ISO_TIME_LEN+1];
-    format_local_iso_time(buf, st.st_mtime);
-    log_notice(LD_GENERAL, "Obsolete file %s hasn't been modified since %s. "
-               "Removing it.", fname, buf);
-    if (unlink(fname) != 0) {
-      log_warn(LD_FS, "Failed to unlink %s: %s",
-               fname, strerror(errno));
-    }
-  }
-}
-
 /** Return a smartlist of ports that must be forwarded by
  *  tor-fw-helper. The smartlist contains the ports in a string format
  *  that is understandable by tor-fw-helper. */
@@ -7751,6 +8188,7 @@ getinfo_helper_config(control_connection_t *conn,
         case CONFIG_TYPE_STRING: type = "String"; break;
         case CONFIG_TYPE_FILENAME: type = "Filename"; break;
         case CONFIG_TYPE_UINT: type = "Integer"; break;
+        case CONFIG_TYPE_UINT64: type = "Integer"; break;
         case CONFIG_TYPE_INT: type = "SignedInteger"; break;
         case CONFIG_TYPE_PORT: type = "Port"; break;
         case CONFIG_TYPE_INTERVAL: type = "TimeInterval"; break;
@@ -7762,9 +8200,12 @@ getinfo_helper_config(control_connection_t *conn,
         case CONFIG_TYPE_ISOTIME: type = "Time"; break;
         case CONFIG_TYPE_ROUTERSET: type = "RouterList"; break;
         case CONFIG_TYPE_CSV: type = "CommaList"; break;
-        case CONFIG_TYPE_CSV_INTERVAL: type = "TimeIntervalCommaList"; break;
+        /* This type accepts more inputs than TimeInterval, but it ignores
+         * everything after the first entry, so we may as well pretend
+         * it's a TimeInterval. */
+        case CONFIG_TYPE_CSV_INTERVAL: type = "TimeInterval"; break;
         case CONFIG_TYPE_LINELIST: type = "LineList"; break;
-        case CONFIG_TYPE_LINELIST_S: type = "Dependant"; break;
+        case CONFIG_TYPE_LINELIST_S: type = "Dependent"; break;
         case CONFIG_TYPE_LINELIST_V: type = "Virtual"; break;
         default:
         case CONFIG_TYPE_OBSOLETE:
@@ -7846,60 +8287,98 @@ getinfo_helper_config(control_connection_t *conn,
   return 0;
 }
 
-/** Parse outbound bind address option lines. If <b>validate_only</b>
- * is not 0 update OutboundBindAddressIPv4_ and
- * OutboundBindAddressIPv6_ in <b>options</b>. On failure, set
- * <b>msg</b> (if provided) to a newly allocated string containing a
- * description of the problem and return -1. */
+/* Check whether an address has already been set against the options
+ * depending on address family and destination type. Any exsting
+ * value will lead to a fail, even if it is the same value. If not
+ * set and not only validating, copy it into this location too.
+ * Returns 0 on success or -1 if this address is already set.
+ */
 static int
-parse_outbound_addresses(or_options_t *options, int validate_only, char **msg)
+verify_and_store_outbound_address(sa_family_t family, tor_addr_t *addr,
+       outbound_addr_t type, or_options_t *options, int validate_only)
 {
-  const config_line_t *lines = options->OutboundBindAddress;
-  int found_v4 = 0, found_v6 = 0;
-
+  if (type>=OUTBOUND_ADDR_MAX || (family!=AF_INET && family!=AF_INET6)) {
+    return -1;
+  }
+  int fam_index=0;
+  if (family==AF_INET6) {
+    fam_index=1;
+  }
+  tor_addr_t *dest=&options->OutboundBindAddresses[type][fam_index];
+  if (!tor_addr_is_null(dest)) {
+    return -1;
+  }
   if (!validate_only) {
-    memset(&options->OutboundBindAddressIPv4_, 0,
-           sizeof(options->OutboundBindAddressIPv4_));
-    memset(&options->OutboundBindAddressIPv6_, 0,
-           sizeof(options->OutboundBindAddressIPv6_));
+    tor_addr_copy(dest, addr);
   }
+  return 0;
+}
+
+/* Parse a list of address lines for a specific destination type.
+ * Will store them into the options if not validate_only. If a
+ * problem occurs, a suitable error message is store in msg.
+ * Returns 0 on success or -1 if any address is already set.
+ */
+static int
+parse_outbound_address_lines(const config_line_t *lines, outbound_addr_t type,
+           or_options_t *options, int validate_only, char **msg)
+{
+  tor_addr_t addr;
+  sa_family_t family;
   while (lines) {
-    tor_addr_t addr, *dst_addr = NULL;
-    int af = tor_addr_parse(&addr, lines->value);
-    switch (af) {
-    case AF_INET:
-      if (found_v4) {
-        if (msg)
-          tor_asprintf(msg, "Multiple IPv4 outbound bind addresses "
-                       "configured: %s", lines->value);
-        return -1;
-      }
-      found_v4 = 1;
-      dst_addr = &options->OutboundBindAddressIPv4_;
-      break;
-    case AF_INET6:
-      if (found_v6) {
-        if (msg)
-          tor_asprintf(msg, "Multiple IPv6 outbound bind addresses "
-                       "configured: %s", lines->value);
-        return -1;
-      }
-      found_v6 = 1;
-      dst_addr = &options->OutboundBindAddressIPv6_;
-      break;
-    default:
+    family = tor_addr_parse(&addr, lines->value);
+    if (verify_and_store_outbound_address(family, &addr, type,
+                                 options, validate_only)) {
       if (msg)
-        tor_asprintf(msg, "Outbound bind address '%s' didn't parse.",
-                     lines->value);
+        tor_asprintf(msg, "Multiple%s%s outbound bind addresses "
+                     "configured: %s",
+                     family==AF_INET?" IPv4":(family==AF_INET6?" IPv6":""),
+                     type==OUTBOUND_ADDR_OR?" OR":
+                     (type==OUTBOUND_ADDR_EXIT?" exit":""), lines->value);
       return -1;
     }
-    if (!validate_only)
-      tor_addr_copy(dst_addr, &addr);
     lines = lines->next;
   }
   return 0;
 }
 
+/** Parse outbound bind address option lines. If <b>validate_only</b>
+ * is not 0 update OutboundBindAddresses in <b>options</b>.
+ * Only one address can be set for any of these values.
+ * On failure, set <b>msg</b> (if provided) to a newly allocated string
+ * containing a description of the problem and return -1.
+ */
+static int
+parse_outbound_addresses(or_options_t *options, int validate_only, char **msg)
+{
+  if (!validate_only) {
+    memset(&options->OutboundBindAddresses, 0,
+           sizeof(options->OutboundBindAddresses));
+  }
+
+  if (parse_outbound_address_lines(options->OutboundBindAddress,
+                                   OUTBOUND_ADDR_EXIT_AND_OR, options,
+                                   validate_only, msg) < 0) {
+    goto err;
+  }
+
+  if (parse_outbound_address_lines(options->OutboundBindAddressOR,
+                                   OUTBOUND_ADDR_OR, options, validate_only,
+                                   msg) < 0) {
+    goto err;
+  }
+
+  if (parse_outbound_address_lines(options->OutboundBindAddressExit,
+                                   OUTBOUND_ADDR_EXIT, options, validate_only,
+                                   msg)  < 0) {
+    goto err;
+  }
+
+  return 0;
+ err:
+  return -1;
+}
+
 /** Load one of the geoip files, <a>family</a> determining which
  * one. <a>default_fname</a> is used if on Windows and
  * <a>fname</a> equals "<default>". */
@@ -7908,6 +8387,11 @@ config_load_geoip_file_(sa_family_t family,
                         const char *fname,
                         const char *default_fname)
 {
+  const or_options_t *options = get_options();
+  const char *msg = "";
+  int severity = options_need_geoip_info(options, &msg) ? LOG_WARN : LOG_INFO;
+  int r;
+
 #ifdef _WIN32
   char *free_fname = NULL; /* Used to hold any temporary-allocated value */
   /* XXXX Don't use this "<default>" junk; make our filename options
@@ -7917,12 +8401,16 @@ config_load_geoip_file_(sa_family_t family,
     tor_asprintf(&free_fname, "%s\\%s", conf_root, default_fname);
     fname = free_fname;
   }
-  geoip_load_file(family, fname);
+  r = geoip_load_file(family, fname, severity);
   tor_free(free_fname);
-#else
+#else /* !(defined(_WIN32)) */
   (void)default_fname;
-  geoip_load_file(family, fname);
-#endif
+  r = geoip_load_file(family, fname, severity);
+#endif /* defined(_WIN32) */
+
+  if (r < 0 && severity == LOG_WARN) {
+    log_warn(LD_GENERAL, "%s", msg);
+  }
 }
 
 /** Load geoip files for IPv4 and IPv6 if <a>options</a> and
@@ -7936,13 +8424,19 @@ config_maybe_load_geoip_files_(const or_options_t *options,
   if (options->GeoIPFile &&
       ((!old_options || !opt_streq(old_options->GeoIPFile,
                                    options->GeoIPFile))
-       || !geoip_is_loaded(AF_INET)))
+       || !geoip_is_loaded(AF_INET))) {
     config_load_geoip_file_(AF_INET, options->GeoIPFile, "geoip");
+    /* Okay, now we need to maybe change our mind about what is in
+     * which country. We do this for IPv4 only since that's what we
+     * store in node->country. */
+    refresh_all_country_info();
+  }
   if (options->GeoIPv6File &&
       ((!old_options || !opt_streq(old_options->GeoIPv6File,
                                    options->GeoIPv6File))
-       || !geoip_is_loaded(AF_INET6)))
+       || !geoip_is_loaded(AF_INET6))) {
     config_load_geoip_file_(AF_INET6, options->GeoIPv6File, "geoip6");
+  }
 }
 
 /** Initialize cookie authentication (used so far by the ControlPort
@@ -7997,9 +8491,9 @@ init_cookie_authentication(const char *fname, const char *header,
       log_warn(LD_FS,"Unable to make %s group-readable.", escaped(fname));
     }
   }
-#else
+#else /* !(!defined(_WIN32)) */
   (void) group_readable;
-#endif
+#endif /* !defined(_WIN32) */
 
   /* Success! */
   log_info(LD_GENERAL, "Generated auth cookie file in '%s'.", escaped(fname));
@@ -8011,3 +8505,17 @@ init_cookie_authentication(const char *fname, const char *header,
   tor_free(cookie_file_str);
   return retval;
 }
+
+/**
+ * Return true if any option is set in <b>options</b> to make us behave
+ * as a client.
+ */
+int
+options_any_client_port_set(const or_options_t *options)
+{
+  return (options->SocksPort_set ||
+          options->TransPort_set ||
+          options->NATDPort_set ||
+          options->DNSPort_set ||
+          options->HTTPTunnelPort_set);
+}
diff --git a/src/or/config.h b/src/app/config/config.h
index 6645532514..301faf7067 100644
--- a/src/or/config.h
+++ b/src/app/config/config.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,12 +12,24 @@
 #ifndef TOR_CONFIG_H
 #define TOR_CONFIG_H
 
-#include "testsupport.h"
+#include "app/config/or_options_st.h"
+#include "lib/testsupport/testsupport.h"
 
 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(DARWIN)
 #define KERNEL_MAY_SUPPORT_IPFW
 #endif
 
+/** Lowest allowable value for HeartbeatPeriod; if this is too low, we might
+ * expose more information than we're comfortable with. */
+#define MIN_HEARTBEAT_PERIOD (30*60)
+
+/** Maximum default value for MaxMemInQueues, in bytes. */
+#if SIZEOF_VOID_P >= 8
+#define MAX_DEFAULT_MEMORY_QUEUE_SIZE (UINT64_C(8) << 30)
+#else
+#define MAX_DEFAULT_MEMORY_QUEUE_SIZE (UINT64_C(2) << 30)
+#endif
+
 MOCK_DECL(const char*, get_dirportfrontpage, (void));
 MOCK_DECL(const or_options_t *, get_options, (void));
 MOCK_DECL(or_options_t *, get_options_mutable, (void));
@@ -27,9 +39,20 @@ const char *safe_str_client(const char *address);
 const char *safe_str(const char *address);
 const char *escaped_safe_str_client(const char *address);
 const char *escaped_safe_str(const char *address);
+void init_protocol_warning_severity_level(void);
+int get_protocol_warning_severity_level(void);
 const char *get_version(void);
 const char *get_short_version(void);
-setopt_err_t options_trial_assign(config_line_t *list, unsigned flags,
+
+/** An error from options_trial_assign() or options_init_from_string(). */
+typedef enum setopt_err_t {
+  SETOPT_OK = 0,
+  SETOPT_ERR_MISC = -1,
+  SETOPT_ERR_PARSE = -2,
+  SETOPT_ERR_TRANSITION = -3,
+  SETOPT_ERR_SETTING = -4,
+} setopt_err_t;
+setopt_err_t options_trial_assign(struct config_line_t *list, unsigned flags,
                                   char **msg);
 
 uint32_t get_last_resolved_addr(void);
@@ -49,35 +72,91 @@ setopt_err_t options_init_from_string(const char *cf_defaults, const char *cf,
                             int command, const char *command_arg, char **msg);
 int option_is_recognized(const char *key);
 const char *option_get_canonical_name(const char *key);
-config_line_t *option_get_assignment(const or_options_t *options,
+struct config_line_t *option_get_assignment(const or_options_t *options,
                                      const char *key);
 int options_save_current(void);
 const char *get_torrc_fname(int defaults_fname);
+typedef enum {
+  DIRROOT_DATADIR,
+  DIRROOT_CACHEDIR,
+  DIRROOT_KEYDIR
+} directory_root_t;
+
 MOCK_DECL(char *,
-          options_get_datadir_fname2_suffix,
+          options_get_dir_fname2_suffix,
           (const or_options_t *options,
+           directory_root_t roottype,
            const char *sub1, const char *sub2,
            const char *suffix));
+
+/* These macros wrap options_get_dir_fname2_suffix to provide a more
+ * convenient API for finding filenames that Tor uses inside its storage
+ * They are named according to a pattern:
+ *    (options_)?get_(cache|key|data)dir_fname(2)?(_suffix)?
+ *
+ * Macros that begin with options_ take an options argument; the others
+ * work with respect to the global options.
+ *
+ * Each macro works relative to the data directory, the key directory,
+ * or the cache directory, as determined by which one is mentioned.
+ *
+ * Macro variants with "2" in their name take two path components; others
+ * take one.
+ *
+ * Macro variants with "_suffix" at the end take an additional suffix
+ * that gets appended to the end of the file
+ */
+#define options_get_datadir_fname2_suffix(options, sub1, sub2, suffix) \
+  options_get_dir_fname2_suffix((options), DIRROOT_DATADIR, \
+                                (sub1), (sub2), (suffix))
+#define options_get_cachedir_fname2_suffix(options, sub1, sub2, suffix) \
+  options_get_dir_fname2_suffix((options), DIRROOT_CACHEDIR, \
+                                (sub1), (sub2), (suffix))
+#define options_get_keydir_fname2_suffix(options, sub1, sub2, suffix) \
+  options_get_dir_fname2_suffix((options), DIRROOT_KEYDIR, \
+                                (sub1), (sub2), (suffix))
+
+#define options_get_datadir_fname(opts,sub1)                    \
+  options_get_datadir_fname2_suffix((opts),(sub1), NULL, NULL)
+#define options_get_datadir_fname2(opts,sub1,sub2)                      \
+  options_get_datadir_fname2_suffix((opts),(sub1), (sub2), NULL)
+
 #define get_datadir_fname2_suffix(sub1, sub2, suffix) \
   options_get_datadir_fname2_suffix(get_options(), (sub1), (sub2), (suffix))
-/** Return a newly allocated string containing datadir/sub1.  See
- * get_datadir_fname2_suffix.  */
-#define get_datadir_fname(sub1) get_datadir_fname2_suffix((sub1), NULL, NULL)
-/** Return a newly allocated string containing datadir/sub1/sub2.  See
- * get_datadir_fname2_suffix.  */
+#define get_datadir_fname(sub1)                 \
+  get_datadir_fname2_suffix((sub1), NULL, NULL)
 #define get_datadir_fname2(sub1,sub2) \
   get_datadir_fname2_suffix((sub1), (sub2), NULL)
-/** Return a newly allocated string containing datadir/sub1/sub2 relative to
- * opts.  See get_datadir_fname2_suffix.  */
-#define options_get_datadir_fname2(opts,sub1,sub2)                      \
-  options_get_datadir_fname2_suffix((opts),(sub1), (sub2), NULL)
-/** Return a newly allocated string containing datadir/sub1suffix.  See
- * get_datadir_fname2_suffix. */
 #define get_datadir_fname_suffix(sub1, suffix) \
   get_datadir_fname2_suffix((sub1), NULL, (suffix))
 
+/** DOCDOC */
+#define options_get_keydir_fname(options, sub1)  \
+  options_get_keydir_fname2_suffix((options), (sub1), NULL, NULL)
+#define get_keydir_fname_suffix(sub1, suffix)   \
+  options_get_keydir_fname2_suffix(get_options(), (sub1), NULL, suffix)
+#define get_keydir_fname(sub1)                  \
+  options_get_keydir_fname2_suffix(get_options(), (sub1), NULL, NULL)
+
+#define get_cachedir_fname(sub1) \
+  options_get_cachedir_fname2_suffix(get_options(), (sub1), NULL, NULL)
+#define get_cachedir_fname_suffix(sub1, suffix) \
+  options_get_cachedir_fname2_suffix(get_options(), (sub1), NULL, (suffix))
+
+#define safe_str_client(address) \
+  safe_str_client_opts(NULL, address)
+#define safe_str(address) \
+  safe_str_opts(NULL, address)
+
+const char * safe_str_client_opts(const or_options_t *options,
+                                  const char *address);
+const char * safe_str_opts(const or_options_t *options,
+                           const char *address);
+
 int using_default_dir_authorities(const or_options_t *options);
 
+int create_keys_directory(const or_options_t *options);
+
 int check_or_create_data_subdir(const char *subdir);
 int write_to_data_subdir(const char* subdir, const char* fname,
                          const char* str, const char* descr);
@@ -121,8 +200,8 @@ int init_cookie_authentication(const char *fname, const char *header,
 or_options_t *options_new(void);
 
 int config_parse_commandline(int argc, char **argv, int ignore_errors,
-                             config_line_t **result,
-                             config_line_t **cmdline_result);
+                             struct config_line_t **result,
+                             struct config_line_t **cmdline_result);
 
 void config_register_addressmaps(const or_options_t *options);
 /* XXXX move to connection_edge.h */
@@ -147,17 +226,22 @@ typedef struct bridge_line_t {
                                transport proxy. */
 } bridge_line_t;
 
-void bridge_line_free(bridge_line_t *bridge_line);
+void bridge_line_free_(bridge_line_t *bridge_line);
+#define bridge_line_free(line) \
+  FREE_AND_NULL(bridge_line_t, bridge_line_free_, (line))
 bridge_line_t *parse_bridge_line(const char *line);
 smartlist_t *get_options_from_transport_options_line(const char *line,
                                                      const char *transport);
 smartlist_t *get_options_for_server_transport(const char *transport);
 
+/* Port helper functions. */
+int options_any_client_port_set(const or_options_t *options);
+
 #ifdef CONFIG_PRIVATE
 
 #define CL_PORT_NO_STREAM_OPTIONS (1u<<0)
 #define CL_PORT_WARN_NONLOCAL (1u<<1)
-#define CL_PORT_ALLOW_EXTRA_LISTENADDR (1u<<2)
+/* Was CL_PORT_ALLOW_EXTRA_LISTENADDR (1u<<2) */
 #define CL_PORT_SERVER_OPTIONS (1u<<3)
 #define CL_PORT_FORBID_NONLOCAL (1u<<4)
 #define CL_PORT_TAKES_HOSTNAMES (1u<<5)
@@ -170,8 +254,12 @@ extern struct config_format_t options_format;
 #endif
 
 STATIC port_cfg_t *port_cfg_new(size_t namelen);
-STATIC void port_cfg_free(port_cfg_t *port);
-STATIC void or_options_free(or_options_t *options);
+#define port_cfg_free(port) \
+  FREE_AND_NULL(port_cfg_t, port_cfg_free_, (port))
+STATIC void port_cfg_free_(port_cfg_t *port);
+#define or_options_free(opt) \
+  FREE_AND_NULL(or_options_t, or_options_free_, (opt))
+STATIC void or_options_free_(or_options_t *options);
 STATIC int options_validate_single_onion(or_options_t *options,
                                          char **msg);
 STATIC int options_validate(or_options_t *old_options,
@@ -192,14 +280,21 @@ STATIC int parse_dir_fallback_line(const char *line, int validate_only);
 STATIC int have_enough_mem_for_dircache(const or_options_t *options,
                                         size_t total_mem, char **msg);
 STATIC int parse_port_config(smartlist_t *out,
-                  const config_line_t *ports,
-                  const config_line_t *listenaddrs,
+                  const struct config_line_t *ports,
                   const char *portname,
                   int listener_type,
                   const char *defaultaddr,
                   int defaultport,
                   const unsigned flags);
-#endif
 
-#endif
+STATIC int check_bridge_distribution_setting(const char *bd);
+
+STATIC uint64_t compute_real_max_mem_in_queues(const uint64_t val,
+                                               int log_guess);
+STATIC int open_and_add_file_log(const log_severity_list_t *severity,
+                                 const char *fname,
+                                 int truncate_log);
+
+#endif /* defined(CONFIG_PRIVATE) */
 
+#endif /* !defined(TOR_CONFIG_H) */
diff --git a/src/or/confparse.c b/src/app/config/confparse.c
index efcf4f981e..729e7a4478 100644
--- a/src/or/confparse.c
+++ b/src/app/config/confparse.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,11 +9,24 @@
  *
  * \brief Back-end for parsing and generating key-value files, used to
  *   implement the torrc file format and the state file.
+ *
+ * This module is used by config.c to parse and encode torrc
+ * configuration files, and by statefile.c to parse and encode the
+ * $DATADIR/state file.
+ *
+ * To use this module, its callers provide an instance of
+ * config_format_t to describe the mappings from a set of configuration
+ * options to a number of fields in a C structure.  With this mapping,
+ * the functions here can convert back and forth between the C structure
+ * specified, and a linked list of key-value pairs.
  */
 
-#include "or.h"
-#include "confparse.h"
-#include "routerset.h"
+#include "core/or/or.h"
+#include "app/config/confparse.h"
+#include "feature/nodelist/routerset.h"
+
+#include "lib/container/bitarray.h"
+#include "lib/encoding/confline.h"
 
 static uint64_t config_parse_memunit(const char *s, int *ok);
 static int config_parse_msec_interval(const char *s, int *ok);
@@ -67,120 +80,6 @@ config_expand_abbrev(const config_format_t *fmt, const char *option,
   return option;
 }
 
-/** Helper: allocate a new configuration option mapping 'key' to 'val',
- * append it to *<b>lst</b>. */
-void
-config_line_append(config_line_t **lst,
-                   const char *key,
-                   const char *val)
-{
-  config_line_t *newline;
-
-  newline = tor_malloc_zero(sizeof(config_line_t));
-  newline->key = tor_strdup(key);
-  newline->value = tor_strdup(val);
-  newline->next = NULL;
-  while (*lst)
-    lst = &((*lst)->next);
-
-  (*lst) = newline;
-}
-
-/** Return the line in <b>lines</b> whose key is exactly <b>key</b>, or NULL
- * if no such key exists. For handling commandline-only options only; other
- * options should be looked up in the appropriate data structure. */
-const config_line_t *
-config_line_find(const config_line_t *lines,
-                 const char *key)
-{
-  const config_line_t *cl;
-  for (cl = lines; cl; cl = cl->next) {
-    if (!strcmp(cl->key, key))
-      return cl;
-  }
-  return NULL;
-}
-
-/** Helper: parse the config string and strdup into key/value
- * strings. Set *result to the list, or NULL if parsing the string
- * failed.  Return 0 on success, -1 on failure. Warn and ignore any
- * misformatted lines.
- *
- * If <b>extended</b> is set, then treat keys beginning with / and with + as
- * indicating "clear" and "append" respectively. */
-int
-config_get_lines(const char *string, config_line_t **result, int extended)
-{
-  config_line_t *list = NULL, **next;
-  char *k, *v;
-  const char *parse_err;
-
-  next = &list;
-  do {
-    k = v = NULL;
-    string = parse_config_line_from_str_verbose(string, &k, &v, &parse_err);
-    if (!string) {
-      log_warn(LD_CONFIG, "Error while parsing configuration: %s",
-               parse_err?parse_err:"<unknown>");
-      config_free_lines(list);
-      tor_free(k);
-      tor_free(v);
-      return -1;
-    }
-    if (k && v) {
-      unsigned command = CONFIG_LINE_NORMAL;
-      if (extended) {
-        if (k[0] == '+') {
-          char *k_new = tor_strdup(k+1);
-          tor_free(k);
-          k = k_new;
-          command = CONFIG_LINE_APPEND;
-        } else if (k[0] == '/') {
-          char *k_new = tor_strdup(k+1);
-          tor_free(k);
-          k = k_new;
-          tor_free(v);
-          v = tor_strdup("");
-          command = CONFIG_LINE_CLEAR;
-        }
-      }
-      /* This list can get long, so we keep a pointer to the end of it
-       * rather than using config_line_append over and over and getting
-       * n^2 performance. */
-      *next = tor_malloc_zero(sizeof(config_line_t));
-      (*next)->key = k;
-      (*next)->value = v;
-      (*next)->next = NULL;
-      (*next)->command = command;
-      next = &((*next)->next);
-    } else {
-      tor_free(k);
-      tor_free(v);
-    }
-  } while (*string);
-
-  *result = list;
-  return 0;
-}
-
-/**
- * Free all the configuration lines on the linked list <b>front</b>.
- */
-void
-config_free_lines(config_line_t *front)
-{
-  config_line_t *tmp;
-
-  while (front) {
-    tmp = front;
-    front = tmp->next;
-
-    tor_free(tmp->key);
-    tor_free(tmp->value);
-    tor_free(tmp);
-  }
-}
-
 /** If <b>key</b> is a deprecated configuration option, return the message
  * explaining why it is deprecated (which may be an empty string). Return NULL
  * if it is not deprecated. The <b>key</b> field must be fully expanded. */
@@ -265,8 +164,6 @@ config_assign_value(const config_format_t *fmt, void *options,
   int i, ok;
   const config_var_t *var;
   void *lvalue;
-  int *csv_int;
-  smartlist_t *csv_str;
 
   CONFIG_CHECK(fmt, options);
 
@@ -298,6 +195,44 @@ config_assign_value(const config_format_t *fmt, void *options,
     *(int *)lvalue = i;
     break;
 
+  case CONFIG_TYPE_UINT64: {
+    uint64_t u64 = tor_parse_uint64(c->value, 10,
+                                    0, UINT64_MAX, &ok, NULL);
+    if (!ok) {
+      tor_asprintf(msg,
+          "uint64 keyword '%s %s' is malformed or out of bounds.",
+          c->key, c->value);
+      return -1;
+    }
+    *(uint64_t *)lvalue = u64;
+    break;
+  }
+
+  case CONFIG_TYPE_CSV_INTERVAL: {
+    /* We used to have entire smartlists here.  But now that all of our
+     * download schedules use exponential backoff, only the first part
+     * matters. */
+    const char *comma = strchr(c->value, ',');
+    const char *val = c->value;
+    char *tmp = NULL;
+    if (comma) {
+      tmp = tor_strndup(c->value, comma - c->value);
+      val = tmp;
+    }
+
+    i = config_parse_interval(val, &ok);
+    if (!ok) {
+      tor_asprintf(msg,
+          "Interval '%s %s' is malformed or out of bounds.",
+          c->key, c->value);
+      tor_free(tmp);
+      return -1;
+    }
+    *(int *)lvalue = i;
+    tor_free(tmp);
+    break;
+  }
+
   case CONFIG_TYPE_INTERVAL: {
     i = config_parse_interval(c->value, &ok);
     if (!ok) {
@@ -346,7 +281,7 @@ config_assign_value(const config_format_t *fmt, void *options,
     break;
 
   case CONFIG_TYPE_AUTOBOOL:
-    if (!strcmp(c->value, "auto"))
+    if (!strcasecmp(c->value, "auto"))
       *(int *)lvalue = -1;
     else if (!strcmp(c->value, "0"))
       *(int *)lvalue = 0;
@@ -401,36 +336,6 @@ config_assign_value(const config_format_t *fmt, void *options,
                            SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
     break;
 
-  case CONFIG_TYPE_CSV_INTERVAL:
-    if (*(smartlist_t**)lvalue) {
-      SMARTLIST_FOREACH(*(smartlist_t**)lvalue, int *, cp, tor_free(cp));
-      smartlist_clear(*(smartlist_t**)lvalue);
-    } else {
-      *(smartlist_t**)lvalue = smartlist_new();
-    }
-    csv_str = smartlist_new();
-    smartlist_split_string(csv_str, c->value, ",",
-                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-    SMARTLIST_FOREACH_BEGIN(csv_str, char *, str)
-      {
-        i = config_parse_interval(str, &ok);
-        if (!ok) {
-          tor_asprintf(msg,
-              "Interval in '%s %s' is malformed or out of bounds.",
-              c->key, c->value);
-          SMARTLIST_FOREACH(csv_str, char *, cp, tor_free(cp));
-          smartlist_free(csv_str);
-          return -1;
-        }
-        csv_int = tor_malloc_zero(sizeof(int));
-        *csv_int = i;
-        smartlist_add(*(smartlist_t**)lvalue, csv_int);
-      }
-    SMARTLIST_FOREACH_END(str);
-    SMARTLIST_FOREACH(csv_str, char *, cp, tor_free(cp));
-    smartlist_free(csv_str);
-    break;
-
   case CONFIG_TYPE_LINELIST:
   case CONFIG_TYPE_LINELIST_S:
     {
@@ -620,23 +525,6 @@ config_value_needs_escape(const char *value)
   return 0;
 }
 
-/** Return a newly allocated deep copy of the lines in <b>inp</b>. */
-config_line_t *
-config_lines_dup(const config_line_t *inp)
-{
-  config_line_t *result = NULL;
-  config_line_t **next_out = &result;
-  while (inp) {
-    *next_out = tor_malloc_zero(sizeof(config_line_t));
-    (*next_out)->key = tor_strdup(inp->key);
-    (*next_out)->value = tor_strdup(inp->value);
-    inp = inp->next;
-    next_out = &((*next_out)->next);
-  }
-  (*next_out) = NULL;
-  return result;
-}
-
 /** Return newly allocated line or lines corresponding to <b>key</b> in the
  * configuration <b>options</b>.  If <b>escape_val</b> is true and a
  * value needs to be quoted before it's put in a config file, quote and
@@ -648,7 +536,6 @@ config_get_assigned_option(const config_format_t *fmt, const void *options,
   const config_var_t *var;
   const void *value;
   config_line_t *result;
-  smartlist_t *csv_str;
   tor_assert(options && key);
 
   CONFIG_CHECK(fmt, options);
@@ -691,6 +578,7 @@ config_get_assigned_option(const config_format_t *fmt, const void *options,
         break;
       }
       /* fall through */
+    case CONFIG_TYPE_CSV_INTERVAL:
     case CONFIG_TYPE_INTERVAL:
     case CONFIG_TYPE_MSEC_INTERVAL:
     case CONFIG_TYPE_UINT:
@@ -700,9 +588,10 @@ config_get_assigned_option(const config_format_t *fmt, const void *options,
       tor_asprintf(&result->value, "%d", *(int*)value);
       escape_val = 0; /* Can't need escape. */
       break;
+    case CONFIG_TYPE_UINT64: /* Fall through */
     case CONFIG_TYPE_MEMUNIT:
-      tor_asprintf(&result->value, U64_FORMAT,
-                   U64_PRINTF_ARG(*(uint64_t*)value));
+      tor_asprintf(&result->value, "%"PRIu64,
+                   (*(uint64_t*)value));
       escape_val = 0; /* Can't need escape. */
       break;
     case CONFIG_TYPE_DOUBLE:
@@ -731,20 +620,6 @@ config_get_assigned_option(const config_format_t *fmt, const void *options,
       else
         result->value = tor_strdup("");
       break;
-    case CONFIG_TYPE_CSV_INTERVAL:
-      if (*(smartlist_t**)value) {
-        csv_str = smartlist_new();
-        SMARTLIST_FOREACH_BEGIN(*(smartlist_t**)value, int *, i)
-          {
-            smartlist_add_asprintf(csv_str, "%d", *i);
-          }
-        SMARTLIST_FOREACH_END(i);
-        result->value = smartlist_join_strings(csv_str, ",", 0, NULL);
-        SMARTLIST_FOREACH(csv_str, char *, cp, tor_free(cp));
-        smartlist_free(csv_str);
-      } else
-        result->value = tor_strdup("");
-      break;
     case CONFIG_TYPE_OBSOLETE:
       log_fn(LOG_INFO, LD_CONFIG,
              "You asked me for the value of an obsolete config option '%s'.",
@@ -753,11 +628,11 @@ config_get_assigned_option(const config_format_t *fmt, const void *options,
       tor_free(result);
       return NULL;
     case CONFIG_TYPE_LINELIST_S:
-      log_warn(LD_CONFIG,
-               "Can't return context-sensitive '%s' on its own", key);
       tor_free(result->key);
       tor_free(result);
-      return NULL;
+      result = config_lines_dup_and_filter(*(const config_line_t **)value,
+                                           key);
+      break;
     case CONFIG_TYPE_LINELIST:
     case CONFIG_TYPE_LINELIST_V:
       tor_free(result->key);
@@ -909,6 +784,7 @@ config_clear(const config_format_t *fmt, void *options,
     case CONFIG_TYPE_ISOTIME:
       *(time_t*)lvalue = 0;
       break;
+    case CONFIG_TYPE_CSV_INTERVAL:
     case CONFIG_TYPE_INTERVAL:
     case CONFIG_TYPE_MSEC_INTERVAL:
     case CONFIG_TYPE_UINT:
@@ -920,6 +796,7 @@ config_clear(const config_format_t *fmt, void *options,
     case CONFIG_TYPE_AUTOBOOL:
       *(int*)lvalue = -1;
       break;
+    case CONFIG_TYPE_UINT64:
     case CONFIG_TYPE_MEMUNIT:
       *(uint64_t*)lvalue = 0;
       break;
@@ -936,13 +813,6 @@ config_clear(const config_format_t *fmt, void *options,
         *(smartlist_t **)lvalue = NULL;
       }
       break;
-    case CONFIG_TYPE_CSV_INTERVAL:
-      if (*(smartlist_t**)lvalue) {
-        SMARTLIST_FOREACH(*(smartlist_t **)lvalue, int *, cp, tor_free(cp));
-        smartlist_free(*(smartlist_t **)lvalue);
-        *(smartlist_t **)lvalue = NULL;
-      }
-      break;
     case CONFIG_TYPE_LINELIST:
     case CONFIG_TYPE_LINELIST_S:
       config_free_lines(*(config_line_t **)lvalue);
@@ -983,7 +853,7 @@ config_reset(const config_format_t *fmt, void *options,
 
 /** Release storage held by <b>options</b>. */
 void
-config_free(const config_format_t *fmt, void *options)
+config_free_(const config_format_t *fmt, void *options)
 {
   int i;
 
@@ -1002,36 +872,6 @@ config_free(const config_format_t *fmt, void *options)
   tor_free(options);
 }
 
-/** Return true iff a and b contain identical keys and values in identical
- * order. */
-int
-config_lines_eq(config_line_t *a, config_line_t *b)
-{
-  while (a && b) {
-    if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value))
-      return 0;
-    a = a->next;
-    b = b->next;
-  }
-  if (a || b)
-    return 0;
-  return 1;
-}
-
-/** Return the number of lines in <b>a</b> whose key is <b>key</b>. */
-int
-config_count_key(const config_line_t *a, const char *key)
-{
-  int n = 0;
-  while (a) {
-    if (!strcasecmp(a->key, key)) {
-      ++n;
-    }
-    a = a->next;
-  }
-  return n;
-}
-
 /** Return true iff the option <b>name</b> has the same value in <b>o1</b>
  * and <b>o2</b>.  Must not be called for LINELIST_S or OBSOLETE options.
  */
@@ -1148,6 +988,11 @@ config_dump(const config_format_t *fmt, const void *default_options,
       config_get_assigned_option(fmt, options, fmt->vars[i].name, 1);
 
     for (; line; line = line->next) {
+      if (!strcmpstart(line->key, "__")) {
+        /* This check detects "hidden" variables inside LINELIST_V structures.
+         */
+        continue;
+      }
       smartlist_add_asprintf(elements, "%s%s %s\n",
                    comment_option ? "# " : "",
                    line->key, line->value);
@@ -1165,8 +1010,9 @@ config_dump(const config_format_t *fmt, const void *default_options,
   result = smartlist_join_strings(elements, "", 0, NULL);
   SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
   smartlist_free(elements);
-  if (defaults_tmp)
-    config_free(fmt, defaults_tmp);
+  if (defaults_tmp) {
+    fmt->free_fn(defaults_tmp);
+  }
   return result;
 }
 
@@ -1212,13 +1058,15 @@ static struct unit_table_t memory_units[] = {
   { "gigabit",   1<<27 },
   { "gbits",     1<<27 },
   { "gbit",      1<<27 },
-  { "tb",        U64_LITERAL(1)<<40 },
-  { "terabyte",  U64_LITERAL(1)<<40 },
-  { "terabytes", U64_LITERAL(1)<<40 },
-  { "terabits",  U64_LITERAL(1)<<37 },
-  { "terabit",   U64_LITERAL(1)<<37 },
-  { "tbits",     U64_LITERAL(1)<<37 },
-  { "tbit",      U64_LITERAL(1)<<37 },
+  { "tb",        UINT64_C(1)<<40 },
+  { "tbyte",     UINT64_C(1)<<40 },
+  { "tbytes",    UINT64_C(1)<<40 },
+  { "terabyte",  UINT64_C(1)<<40 },
+  { "terabytes", UINT64_C(1)<<40 },
+  { "terabits",  UINT64_C(1)<<37 },
+  { "terabit",   UINT64_C(1)<<37 },
+  { "tbits",     UINT64_C(1)<<37 },
+  { "tbit",      UINT64_C(1)<<37 },
   { NULL, 0 },
 };
 
@@ -1287,7 +1135,7 @@ config_parse_units(const char *val, struct unit_table_t *u, int *ok)
 
   if (!cp) {
     *ok = 1;
-    v = use_float ? DBL_TO_U64(d) :  v;
+    v = use_float ? ((uint64_t)d) :  v;
     goto done;
   }
 
@@ -1333,8 +1181,6 @@ config_parse_msec_interval(const char *s, int *ok)
 {
   uint64_t r;
   r = config_parse_units(s, time_msec_units, ok);
-  if (!ok)
-    return -1;
   if (r > INT_MAX) {
     log_warn(LD_CONFIG, "Msec interval '%s' is too long", s);
     *ok = 0;
@@ -1352,8 +1198,6 @@ config_parse_interval(const char *s, int *ok)
 {
   uint64_t r;
   r = config_parse_units(s, time_units, ok);
-  if (!ok)
-    return -1;
   if (r > INT_MAX) {
     log_warn(LD_CONFIG, "Interval '%s' is too long", s);
     *ok = 0;
@@ -1361,4 +1205,3 @@ config_parse_interval(const char *s, int *ok)
   }
   return (int)r;
 }
-
diff --git a/src/or/confparse.h b/src/app/config/confparse.h
index 8d915d266b..57f1ec1762 100644
--- a/src/or/confparse.h
+++ b/src/app/config/confparse.h
@@ -1,9 +1,15 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * \file confparse.h
+ *
+ * \brief Header for confparse.c.
+ */
+
 #ifndef TOR_CONFPARSE_H
 #define TOR_CONFPARSE_H
 
@@ -13,6 +19,7 @@ typedef enum config_type_t {
   CONFIG_TYPE_FILENAME,     /**< A filename: some prefixes get expanded. */
   CONFIG_TYPE_UINT,         /**< A non-negative integer less than MAX_INT */
   CONFIG_TYPE_INT,          /**< Any integer. */
+  CONFIG_TYPE_UINT64,       /**< A value in range 0..UINT64_MAX */
   CONFIG_TYPE_PORT,         /**< A port from 1...65535, 0 for "not set", or
                              * "auto".  */
   CONFIG_TYPE_INTERVAL,     /**< A number of seconds, with optional units*/
@@ -28,7 +35,9 @@ typedef enum config_type_t {
                               * optional whitespace. */
   CONFIG_TYPE_CSV_INTERVAL, /**< A list of strings, separated by commas and
                               * optional whitespace, representing intervals in
-                              * seconds, with optional units */
+                              * seconds, with optional units.  We allow
+                              * multiple values here for legacy reasons, but
+                              * ignore every value after the first. */
   CONFIG_TYPE_LINELIST,     /**< Uninterpreted config lines */
   CONFIG_TYPE_LINELIST_S,   /**< Uninterpreted, context-sensitive config lines,
                              * mixed with other keywords. */
@@ -40,6 +49,37 @@ typedef enum config_type_t {
   CONFIG_TYPE_OBSOLETE,     /**< Obsolete (ignored) option. */
 } config_type_t;
 
+#ifdef TOR_UNIT_TESTS
+/**
+ * Union used when building in test mode typechecking the members of a type
+ * used with confparse.c.  See CONF_CHECK_VAR_TYPE for a description of how
+ * it is used. */
+typedef union {
+  char **STRING;
+  char **FILENAME;
+  int *UINT; /* yes, really: Even though the confparse type is called
+              * "UINT", it still uses the C int type -- it just enforces that
+              * the values are in range [0,INT_MAX].
+              */
+  uint64_t *UINT64;
+  int *INT;
+  int *PORT;
+  int *INTERVAL;
+  int *MSEC_INTERVAL;
+  uint64_t *MEMUNIT;
+  double *DOUBLE;
+  int *BOOL;
+  int *AUTOBOOL;
+  time_t *ISOTIME;
+  smartlist_t **CSV;
+  int *CSV_INTERVAL;
+  struct config_line_t **LINELIST;
+  struct config_line_t **LINELIST_S;
+  struct config_line_t **LINELIST_V;
+  routerset_t **ROUTERSET;
+} confparse_dummy_values_t;
+#endif /* defined(TOR_UNIT_TESTS) */
+
 /** An abbreviation for a configuration option allowed on the command line. */
 typedef struct config_abbrev_t {
   const char *abbreviated;
@@ -64,13 +104,60 @@ typedef struct config_var_t {
                        * value. */
   off_t var_offset; /**< Offset of the corresponding member of or_options_t. */
   const char *initvalue; /**< String (or null) describing initial value. */
+
+#ifdef TOR_UNIT_TESTS
+  /** Used for compiler-magic to typecheck the corresponding field in the
+   * corresponding struct. Only used in unit test mode, at compile-time. */
+  confparse_dummy_values_t var_ptr_dummy;
+#endif
 } config_var_t;
 
+/* Macros to define extra members inside config_var_t fields, and at the
+ * end of a list of them.
+ */
+#ifdef TOR_UNIT_TESTS
+/* This is a somewhat magic type-checking macro for users of confparse.c.
+ * It initializes a union member "confparse_dummy_values_t.conftype" with
+ * the address of a static member "tp_dummy.member".   This
+ * will give a compiler warning unless the member field is of the correct
+ * type.
+ *
+ * (This warning is mandatory, because a type mismatch here violates the type
+ * compatibility constraint for simple assignment, and requires a diagnostic,
+ * according to the C spec.)
+ *
+ * For example, suppose you say:
+ *     "CONF_CHECK_VAR_TYPE(or_options_t, STRING, Address)".
+ * Then this macro will evaluate to:
+ *     { .STRING = &or_options_t_dummy.Address }
+ * And since confparse_dummy_values_t.STRING has type "char **", that
+ * expression will create a warning unless or_options_t.Address also
+ * has type "char *".
+ */
+#define CONF_CHECK_VAR_TYPE(tp, conftype, member)       \
+  { . conftype = &tp ## _dummy . member }
+#define CONF_TEST_MEMBERS(tp, conftype, member) \
+  , CONF_CHECK_VAR_TYPE(tp, conftype, member)
+#define END_OF_CONFIG_VARS                                      \
+  { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL, { .INT=NULL } }
+#define DUMMY_TYPECHECK_INSTANCE(tp)            \
+  static tp tp ## _dummy
+#else /* !(defined(TOR_UNIT_TESTS)) */
+#define CONF_TEST_MEMBERS(tp, conftype, member)
+#define END_OF_CONFIG_VARS { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL }
+/* Repeatedly declarable incomplete struct to absorb redundant semicolons */
+#define DUMMY_TYPECHECK_INSTANCE(tp)            \
+  struct tor_semicolon_eater
+#endif /* defined(TOR_UNIT_TESTS) */
+
 /** Type of a callback to validate whether a given configuration is
  * well-formed and consistent. See options_trial_assign() for documentation
  * of arguments. */
 typedef int (*validate_fn_t)(void*,void*,void*,int,char**);
 
+/** Callback to free a configuration object. */
+typedef void (*free_cfg_fn_t)(void*);
+
 /** Information on the keys, value types, key-to-struct-member mappings,
  * variable descriptions, validation functions, and abbreviations for a
  * configuration or storage format. */
@@ -85,6 +172,7 @@ typedef struct config_format_t {
   config_var_t *vars; /**< List of variables we recognize, their default
                        * values, and where we stick them in the structure. */
   validate_fn_t validate_fn; /**< Function to validate config. */
+  free_cfg_fn_t free_fn; /**< Function to free the configuration. */
   /** If present, extra is a LINELIST variable for unrecognized
    * lines.  Otherwise, unrecognized lines are an error. */
   config_var_t *extra;
@@ -103,15 +191,13 @@ typedef struct config_format_t {
 #define CAL_WARN_DEPRECATIONS (1u<<2)
 
 void *config_new(const config_format_t *fmt);
-void config_line_append(config_line_t **lst,
-                        const char *key, const char *val);
-config_line_t *config_lines_dup(const config_line_t *inp);
-const config_line_t *config_line_find(const config_line_t *lines,
-                                      const char *key);
-void config_free(const config_format_t *fmt, void *options);
-int config_lines_eq(config_line_t *a, config_line_t *b);
-int config_count_key(const config_line_t *a, const char *key);
-config_line_t *config_get_assigned_option(const config_format_t *fmt,
+void config_free_(const config_format_t *fmt, void *options);
+#define config_free(fmt, options) do {                \
+    config_free_((fmt), (options));                   \
+    (options) = NULL;                                 \
+  } while (0)
+
+struct config_line_t *config_get_assigned_option(const config_format_t *fmt,
                                           const void *options, const char *key,
                                           int escape_val);
 int config_is_same(const config_format_t *fmt,
@@ -123,7 +209,7 @@ char *config_dump(const config_format_t *fmt, const void *default_options,
                   const void *options, int minimal,
                   int comment_defaults);
 int config_assign(const config_format_t *fmt, void *options,
-                  config_line_t *list,
+                  struct config_line_t *list,
                   unsigned flags, char **msg);
 config_var_t *config_find_option_mutable(config_format_t *fmt,
                                          const char *key);
@@ -131,13 +217,17 @@ const char *config_find_deprecation(const config_format_t *fmt,
                                      const char *key);
 const config_var_t *config_find_option(const config_format_t *fmt,
                                        const char *key);
-
-int config_get_lines(const char *string, config_line_t **result, int extended);
-void config_free_lines(config_line_t *front);
 const char *config_expand_abbrev(const config_format_t *fmt,
                                  const char *option,
                                  int command_line, int warn_obsolete);
 void warn_deprecated_option(const char *what, const char *why);
 
-#endif
+/* Helper macros to compare an option across two configuration objects */
+#define CFG_EQ_BOOL(a,b,opt) ((a)->opt == (b)->opt)
+#define CFG_EQ_INT(a,b,opt) ((a)->opt == (b)->opt)
+#define CFG_EQ_STRING(a,b,opt) (!strcmp_opt((a)->opt, (b)->opt))
+#define CFG_EQ_SMARTLIST(a,b,opt) smartlist_strings_eq((a)->opt, (b)->opt)
+#define CFG_EQ_LINELIST(a,b,opt) config_lines_eq((a)->opt, (b)->opt)
+#define CFG_EQ_ROUTERSET(a,b,opt) routerset_equal((a)->opt, (b)->opt)
 
+#endif /* !defined(TOR_CONFPARSE_H) */
diff --git a/src/or/fallback_dirs.inc b/src/app/config/fallback_dirs.inc
index 793f65ce88..793f65ce88 100644
--- a/src/or/fallback_dirs.inc
+++ b/src/app/config/fallback_dirs.inc
diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h
new file mode 100644
index 0000000000..74d2fefa16
--- /dev/null
+++ b/src/app/config/or_options_st.h
@@ -0,0 +1,1077 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file or_options_st.h
+ *
+ * \brief The or_options_t structure, which represents Tor's configuration.
+ */
+
+#ifndef TOR_OR_OPTIONS_ST_H
+#define TOR_OR_OPTIONS_ST_H
+
+#include "lib/cc/torint.h"
+#include "lib/net/address.h"
+
+struct smartlist_t;
+struct config_line_t;
+
+/** Enumeration of outbound address configuration types:
+ * Exit-only, OR-only, or both */
+typedef enum {OUTBOUND_ADDR_EXIT, OUTBOUND_ADDR_OR,
+              OUTBOUND_ADDR_EXIT_AND_OR,
+              OUTBOUND_ADDR_MAX} outbound_addr_t;
+
+/** Configuration options for a Tor process. */
+struct or_options_t {
+  uint32_t magic_;
+
+  /** What should the tor process actually do? */
+  enum {
+    CMD_RUN_TOR=0, CMD_LIST_FINGERPRINT, CMD_HASH_PASSWORD,
+    CMD_VERIFY_CONFIG, CMD_RUN_UNITTESTS, CMD_DUMP_CONFIG,
+    CMD_KEYGEN,
+    CMD_KEY_EXPIRATION,
+  } command;
+  char *command_arg; /**< Argument for command-line option. */
+
+  struct config_line_t *Logs; /**< New-style list of configuration lines
+                        * for logs */
+  int LogTimeGranularity; /**< Log resolution in milliseconds. */
+
+  int LogMessageDomains; /**< Boolean: Should we log the domain(s) in which
+                          * each log message occurs? */
+  int TruncateLogFile; /**< Boolean: Should we truncate the log file
+                            before we start writing? */
+  char *SyslogIdentityTag; /**< Identity tag to add for syslog logging. */
+  char *AndroidIdentityTag; /**< Identity tag to add for Android logging. */
+
+  char *DebugLogFile; /**< Where to send verbose log messages. */
+  char *DataDirectory_option; /**< Where to store long-term data, as
+                               * configured by the user. */
+  char *DataDirectory; /**< Where to store long-term data, as modified. */
+  int DataDirectoryGroupReadable; /**< Boolean: Is the DataDirectory g+r? */
+
+  char *KeyDirectory_option; /**< Where to store keys, as
+                               * configured by the user. */
+  char *KeyDirectory; /**< Where to store keys data, as modified. */
+  int KeyDirectoryGroupReadable; /**< Boolean: Is the KeyDirectory g+r? */
+
+  char *CacheDirectory_option; /**< Where to store cached data, as
+                               * configured by the user. */
+  char *CacheDirectory; /**< Where to store cached data, as modified. */
+  int CacheDirectoryGroupReadable; /**< Boolean: Is the CacheDirectory g+r? */
+
+  char *Nickname; /**< OR only: nickname of this onion router. */
+  char *Address; /**< OR only: configured address for this onion router. */
+  char *PidFile; /**< Where to store PID of Tor process. */
+
+  routerset_t *ExitNodes; /**< Structure containing nicknames, digests,
+                           * country codes and IP address patterns of ORs to
+                           * consider as exits. */
+  routerset_t *EntryNodes;/**< Structure containing nicknames, digests,
+                           * country codes and IP address patterns of ORs to
+                           * consider as entry points. */
+  int StrictNodes; /**< Boolean: When none of our EntryNodes or ExitNodes
+                    * are up, or we need to access a node in ExcludeNodes,
+                    * do we just fail instead? */
+  routerset_t *ExcludeNodes;/**< Structure containing nicknames, digests,
+                             * country codes and IP address patterns of ORs
+                             * not to use in circuits. But see StrictNodes
+                             * above. */
+  routerset_t *ExcludeExitNodes;/**< Structure containing nicknames, digests,
+                                 * country codes and IP address patterns of
+                                 * ORs not to consider as exits. */
+
+  /** Union of ExcludeNodes and ExcludeExitNodes */
+  routerset_t *ExcludeExitNodesUnion_;
+
+  int DisableAllSwap; /**< Boolean: Attempt to call mlockall() on our
+                       * process for all current and future memory. */
+
+  struct config_line_t *ExitPolicy; /**< Lists of exit policy components. */
+  int ExitPolicyRejectPrivate; /**< Should we not exit to reserved private
+                                * addresses, and our own published addresses?
+                                */
+  int ExitPolicyRejectLocalInterfaces; /**< Should we not exit to local
+                                        * interface addresses?
+                                        * Includes OutboundBindAddresses and
+                                        * configured ports. */
+  int ReducedExitPolicy; /**<Should we use the Reduced Exit Policy? */
+  struct config_line_t *SocksPolicy; /**< Lists of socks policy components */
+  struct config_line_t *DirPolicy; /**< Lists of dir policy components */
+  /** Local address to bind outbound sockets */
+  struct config_line_t *OutboundBindAddress;
+  /** Local address to bind outbound relay sockets */
+  struct config_line_t *OutboundBindAddressOR;
+  /** Local address to bind outbound exit sockets */
+  struct config_line_t *OutboundBindAddressExit;
+  /** Addresses derived from the various OutboundBindAddress lines.
+   * [][0] is IPv4, [][1] is IPv6
+   */
+  tor_addr_t OutboundBindAddresses[OUTBOUND_ADDR_MAX][2];
+  /** Directory server only: which versions of
+   * Tor should we tell users to run? */
+  struct config_line_t *RecommendedVersions;
+  struct config_line_t *RecommendedClientVersions;
+  struct config_line_t *RecommendedServerVersions;
+  struct config_line_t *RecommendedPackages;
+  /** Whether dirservers allow router descriptors with private IPs. */
+  int DirAllowPrivateAddresses;
+  /** Whether routers accept EXTEND cells to routers with private IPs. */
+  int ExtendAllowPrivateAddresses;
+  char *User; /**< Name of user to run Tor as. */
+   /** Ports to listen on for OR connections. */
+  struct config_line_t *ORPort_lines;
+  /** Ports to listen on for extended OR connections. */
+  struct config_line_t *ExtORPort_lines;
+  /** Ports to listen on for SOCKS connections. */
+  struct config_line_t *SocksPort_lines;
+  /** Ports to listen on for transparent pf/netfilter connections. */
+  struct config_line_t *TransPort_lines;
+  char *TransProxyType; /**< What kind of transparent proxy
+                         * implementation are we using? */
+  /** Parsed value of TransProxyType. */
+  enum {
+    TPT_DEFAULT,
+    TPT_PF_DIVERT,
+    TPT_IPFW,
+    TPT_TPROXY,
+  } TransProxyType_parsed;
+  /** Ports to listen on for transparent natd connections. */
+  struct config_line_t *NATDPort_lines;
+  /** Ports to listen on for HTTP Tunnel connections. */
+  struct config_line_t *HTTPTunnelPort_lines;
+  struct config_line_t *ControlPort_lines; /**< Ports to listen on for control
+                               * connections. */
+  /** List of Unix Domain Sockets to listen on for control connections. */
+  struct config_line_t *ControlSocket;
+
+  int ControlSocketsGroupWritable; /**< Boolean: Are control sockets g+rw? */
+  int UnixSocksGroupWritable; /**< Boolean: Are SOCKS Unix sockets g+rw? */
+  /** Ports to listen on for directory connections. */
+  struct config_line_t *DirPort_lines;
+  /** Ports to listen on for DNS requests. */
+  struct config_line_t *DNSPort_lines;
+
+  /* MaxMemInQueues value as input by the user. We clean this up to be
+   * MaxMemInQueues. */
+  uint64_t MaxMemInQueues_raw;
+  uint64_t MaxMemInQueues;/**< If we have more memory than this allocated
+                            * for queues and buffers, run the OOM handler */
+  /** Above this value, consider ourselves low on RAM. */
+  uint64_t MaxMemInQueues_low_threshold;
+
+  /** @name port booleans
+   *
+   * Derived booleans: For server ports and ControlPort, true iff there is a
+   * non-listener port on an AF_INET or AF_INET6 address of the given type
+   * configured in one of the _lines options above.
+   * For client ports, also true if there is a unix socket configured.
+   * If you are checking for client ports, you may want to use:
+   *   SocksPort_set || TransPort_set || NATDPort_set || DNSPort_set ||
+   *   HTTPTunnelPort_set
+   * rather than SocksPort_set.
+   *
+   * @{
+   */
+  unsigned int ORPort_set : 1;
+  unsigned int SocksPort_set : 1;
+  unsigned int TransPort_set : 1;
+  unsigned int NATDPort_set : 1;
+  unsigned int ControlPort_set : 1;
+  unsigned int DirPort_set : 1;
+  unsigned int DNSPort_set : 1;
+  unsigned int ExtORPort_set : 1;
+  unsigned int HTTPTunnelPort_set : 1;
+  /**@}*/
+
+  int AssumeReachable; /**< Whether to publish our descriptor regardless. */
+  int AuthoritativeDir; /**< Boolean: is this an authoritative directory? */
+  int V3AuthoritativeDir; /**< Boolean: is this an authoritative directory
+                           * for version 3 directories? */
+  int VersioningAuthoritativeDir; /**< Boolean: is this an authoritative
+                                   * directory that's willing to recommend
+                                   * versions? */
+  int BridgeAuthoritativeDir; /**< Boolean: is this an authoritative directory
+                               * that aggregates bridge descriptors? */
+
+  /** If set on a bridge relay, it will include this value on a new
+   * "bridge-distribution-request" line in its bridge descriptor. */
+  char *BridgeDistribution;
+
+  /** If set on a bridge authority, it will answer requests on its dirport
+   * for bridge statuses -- but only if the requests use this password. */
+  char *BridgePassword;
+  /** If BridgePassword is set, this is a SHA256 digest of the basic http
+   * authenticator for it. Used so we can do a time-independent comparison. */
+  char *BridgePassword_AuthDigest_;
+
+  int UseBridges; /**< Boolean: should we start all circuits with a bridge? */
+  struct config_line_t *Bridges; /**< List of bootstrap bridge addresses. */
+
+  struct config_line_t *ClientTransportPlugin; /**< List of client
+                                           transport plugins. */
+
+  struct config_line_t *ServerTransportPlugin; /**< List of client
+                                           transport plugins. */
+
+  /** List of TCP/IP addresses that transports should listen at. */
+  struct config_line_t *ServerTransportListenAddr;
+
+  /** List of options that must be passed to pluggable transports. */
+  struct config_line_t *ServerTransportOptions;
+
+  int BridgeRelay; /**< Boolean: are we acting as a bridge relay? We make
+                    * this explicit so we can change how we behave in the
+                    * future. */
+
+  /** Boolean: if we know the bridge's digest, should we get new
+   * descriptors from the bridge authorities or from the bridge itself? */
+  int UpdateBridgesFromAuthority;
+
+  int AvoidDiskWrites; /**< Boolean: should we never cache things to disk?
+                        * Not used yet. */
+  int ClientOnly; /**< Boolean: should we never evolve into a server role? */
+
+  int ReducedConnectionPadding; /**< Boolean: Should we try to keep connections
+                                  open shorter and pad them less against
+                                  connection-level traffic analysis? */
+  /** Autobool: if auto, then connection padding will be negotiated by client
+   * and server. If 0, it will be fully disabled. If 1, the client will still
+   * pad to the server regardless of server support. */
+  int ConnectionPadding;
+
+  /** To what authority types do we publish our descriptor? Choices are
+   * "v1", "v2", "v3", "bridge", or "". */
+  struct smartlist_t *PublishServerDescriptor;
+  /** A bitfield of authority types, derived from PublishServerDescriptor. */
+  dirinfo_type_t PublishServerDescriptor_;
+  /** Boolean: do we publish hidden service descriptors to the HS auths? */
+  int PublishHidServDescriptors;
+  int FetchServerDescriptors; /**< Do we fetch server descriptors as normal? */
+  int FetchHidServDescriptors; /**< and hidden service descriptors? */
+
+  int MinUptimeHidServDirectoryV2; /**< As directory authority, accept hidden
+                                    * service directories after what time? */
+
+  int FetchUselessDescriptors; /**< Do we fetch non-running descriptors too? */
+  int AllDirActionsPrivate; /**< Should every directory action be sent
+                             * through a Tor circuit? */
+
+  /** A routerset that should be used when picking middle nodes for HS
+   *  circuits. */
+  routerset_t *HSLayer2Nodes;
+
+  /** A routerset that should be used when picking third-hop nodes for HS
+   *  circuits. */
+  routerset_t *HSLayer3Nodes;
+
+  /** Onion Services in HiddenServiceSingleHopMode make one-hop (direct)
+   * circuits between the onion service server, and the introduction and
+   * rendezvous points. (Onion service descriptors are still posted using
+   * 3-hop paths, to avoid onion service directories blocking the service.)
+   * This option makes every hidden service instance hosted by
+   * this tor instance a Single Onion Service.
+   * HiddenServiceSingleHopMode requires HiddenServiceNonAnonymousMode to be
+   * set to 1.
+   * Use rend_service_allow_non_anonymous_connection() or
+   * rend_service_reveal_startup_time() instead of using this option directly.
+   */
+  int HiddenServiceSingleHopMode;
+  /* Makes hidden service clients and servers non-anonymous on this tor
+   * instance. Allows the non-anonymous HiddenServiceSingleHopMode. Enables
+   * non-anonymous behaviour in the hidden service protocol.
+   * Use rend_service_non_anonymous_mode_enabled() instead of using this option
+   * directly.
+   */
+  int HiddenServiceNonAnonymousMode;
+
+  int ConnLimit; /**< Demanded minimum number of simultaneous connections. */
+  int ConnLimit_; /**< Maximum allowed number of simultaneous connections. */
+  int ConnLimit_high_thresh; /**< start trying to lower socket usage if we
+                              *   have this many. */
+  int ConnLimit_low_thresh; /**< try to get down to here after socket
+                             *   exhaustion. */
+  int RunAsDaemon; /**< If true, run in the background. (Unix only) */
+  int FascistFirewall; /**< Whether to prefer ORs reachable on open ports. */
+  struct smartlist_t *FirewallPorts; /**< Which ports our firewall allows
+                               * (strings). */
+   /** IP:ports our firewall allows. */
+  struct config_line_t *ReachableAddresses;
+  struct config_line_t *ReachableORAddresses; /**< IP:ports for OR conns. */
+  struct config_line_t *ReachableDirAddresses; /**< IP:ports for Dir conns. */
+
+  int ConstrainedSockets; /**< Shrink xmit and recv socket buffers. */
+  uint64_t ConstrainedSockSize; /**< Size of constrained buffers. */
+
+  /** Whether we should drop exit streams from Tors that we don't know are
+   * relays.  One of "0" (never refuse), "1" (always refuse), or "-1" (do
+   * what the consensus says, defaulting to 'refuse' if the consensus says
+   * nothing). */
+  int RefuseUnknownExits;
+
+  /** Application ports that require all nodes in circ to have sufficient
+   * uptime. */
+  struct smartlist_t *LongLivedPorts;
+  /** Application ports that are likely to be unencrypted and
+   * unauthenticated; we reject requests for them to prevent the
+   * user from screwing up and leaking plaintext secrets to an
+   * observer somewhere on the Internet. */
+  struct smartlist_t *RejectPlaintextPorts;
+  /** Related to RejectPlaintextPorts above, except this config option
+   * controls whether we warn (in the log and via a controller status
+   * event) every time a risky connection is attempted. */
+  struct smartlist_t *WarnPlaintextPorts;
+  /** Should we try to reuse the same exit node for a given host */
+  struct smartlist_t *TrackHostExits;
+  int TrackHostExitsExpire; /**< Number of seconds until we expire an
+                             * addressmap */
+  struct config_line_t *AddressMap; /**< List of address map directives. */
+  int AutomapHostsOnResolve; /**< If true, when we get a resolve request for a
+                              * hostname ending with one of the suffixes in
+                              * <b>AutomapHostsSuffixes</b>, map it to a
+                              * virtual address. */
+  /** List of suffixes for <b>AutomapHostsOnResolve</b>.  The special value
+   * "." means "match everything." */
+  struct smartlist_t *AutomapHostsSuffixes;
+  int RendPostPeriod; /**< How often do we post each rendezvous service
+                       * descriptor? Remember to publish them independently. */
+  int KeepalivePeriod; /**< How often do we send padding cells to keep
+                        * connections alive? */
+  int SocksTimeout; /**< How long do we let a socks connection wait
+                     * unattached before we fail it? */
+  int LearnCircuitBuildTimeout; /**< If non-zero, we attempt to learn a value
+                                 * for CircuitBuildTimeout based on timeout
+                                 * history. Use circuit_build_times_disabled()
+                                 * rather than checking this value directly. */
+  int CircuitBuildTimeout; /**< Cull non-open circuits that were born at
+                            * least this many seconds ago. Used until
+                            * adaptive algorithm learns a new value. */
+  int CircuitsAvailableTimeout; /**< Try to have an open circuit for at
+                                     least this long after last activity */
+  int CircuitStreamTimeout; /**< If non-zero, detach streams from circuits
+                             * and try a new circuit if the stream has been
+                             * waiting for this many seconds. If zero, use
+                             * our default internal timeout schedule. */
+  int MaxOnionQueueDelay; /*< DOCDOC */
+  int NewCircuitPeriod; /**< How long do we use a circuit before building
+                         * a new one? */
+  int MaxCircuitDirtiness; /**< Never use circs that were first used more than
+                                this interval ago. */
+  uint64_t BandwidthRate; /**< How much bandwidth, on average, are we willing
+                           * to use in a second? */
+  uint64_t BandwidthBurst; /**< How much bandwidth, at maximum, are we willing
+                            * to use in a second? */
+  uint64_t MaxAdvertisedBandwidth; /**< How much bandwidth are we willing to
+                                    * tell other nodes we have? */
+  uint64_t RelayBandwidthRate; /**< How much bandwidth, on average, are we
+                                 * willing to use for all relayed conns? */
+  uint64_t RelayBandwidthBurst; /**< How much bandwidth, at maximum, will we
+                                 * use in a second for all relayed conns? */
+  uint64_t PerConnBWRate; /**< Long-term bw on a single TLS conn, if set. */
+  uint64_t PerConnBWBurst; /**< Allowed burst on a single TLS conn, if set. */
+  int NumCPUs; /**< How many CPUs should we try to use? */
+  struct config_line_t *RendConfigLines; /**< List of configuration lines
+                                          * for rendezvous services. */
+  struct config_line_t *HidServAuth; /**< List of configuration lines for
+                               * client-side authorizations for hidden
+                               * services */
+  char *ClientOnionAuthDir; /**< Directory to keep client
+                             * onion service authorization secret keys */
+  char *ContactInfo; /**< Contact info to be published in the directory. */
+
+  int HeartbeatPeriod; /**< Log heartbeat messages after this many seconds
+                        * have passed. */
+  int MainloopStats; /**< Log main loop statistics as part of the
+                      * heartbeat messages. */
+
+  char *HTTPProxy; /**< hostname[:port] to use as http proxy, if any. */
+  tor_addr_t HTTPProxyAddr; /**< Parsed IPv4 addr for http proxy, if any. */
+  uint16_t HTTPProxyPort; /**< Parsed port for http proxy, if any. */
+  char *HTTPProxyAuthenticator; /**< username:password string, if any. */
+
+  char *HTTPSProxy; /**< hostname[:port] to use as https proxy, if any. */
+  tor_addr_t HTTPSProxyAddr; /**< Parsed addr for https proxy, if any. */
+  uint16_t HTTPSProxyPort; /**< Parsed port for https proxy, if any. */
+  char *HTTPSProxyAuthenticator; /**< username:password string, if any. */
+
+  char *Socks4Proxy; /**< hostname:port to use as a SOCKS4 proxy, if any. */
+  tor_addr_t Socks4ProxyAddr; /**< Derived from Socks4Proxy. */
+  uint16_t Socks4ProxyPort; /**< Derived from Socks4Proxy. */
+
+  char *Socks5Proxy; /**< hostname:port to use as a SOCKS5 proxy, if any. */
+  tor_addr_t Socks5ProxyAddr; /**< Derived from Sock5Proxy. */
+  uint16_t Socks5ProxyPort; /**< Derived from Socks5Proxy. */
+  char *Socks5ProxyUsername; /**< Username for SOCKS5 authentication, if any */
+  char *Socks5ProxyPassword; /**< Password for SOCKS5 authentication, if any */
+
+  /** List of configuration lines for replacement directory authorities.
+   * If you just want to replace one class of authority at a time,
+   * use the "Alternate*Authority" options below instead. */
+  struct config_line_t *DirAuthorities;
+
+  /** List of fallback directory servers */
+  struct config_line_t *FallbackDir;
+  /** Whether to use the default hard-coded FallbackDirs */
+  int UseDefaultFallbackDirs;
+
+  /** Weight to apply to all directory authority rates if considering them
+   * along with fallbackdirs */
+  double DirAuthorityFallbackRate;
+
+  /** If set, use these main (currently v3) directory authorities and
+   * not the default ones. */
+  struct config_line_t *AlternateDirAuthority;
+
+  /** If set, use these bridge authorities and not the default one. */
+  struct config_line_t *AlternateBridgeAuthority;
+
+  struct config_line_t *MyFamily_lines; /**< Declared family for this OR. */
+  struct config_line_t *MyFamily; /**< Declared family for this OR,
+                                     normalized */
+  struct config_line_t *NodeFamilies; /**< List of config lines for
+                                * node families */
+  /** List of parsed NodeFamilies values. */
+  struct smartlist_t *NodeFamilySets;
+  struct config_line_t *AuthDirBadExit; /**< Address policy for descriptors to
+                                  * mark as bad exits. */
+  struct config_line_t *AuthDirReject; /**< Address policy for descriptors to
+                                 * reject. */
+  struct config_line_t *AuthDirInvalid; /**< Address policy for descriptors to
+                                  * never mark as valid. */
+  /** @name AuthDir...CC
+   *
+   * Lists of country codes to mark as BadExit, or Invalid, or to
+   * reject entirely.
+   *
+   * @{
+   */
+  struct smartlist_t *AuthDirBadExitCCs;
+  struct smartlist_t *AuthDirInvalidCCs;
+  struct smartlist_t *AuthDirRejectCCs;
+  /**@}*/
+
+  int AuthDirListBadExits; /**< True iff we should list bad exits,
+                            * and vote for all other exits as good. */
+  int AuthDirMaxServersPerAddr; /**< Do not permit more than this
+                                 * number of servers per IP address. */
+  int AuthDirHasIPv6Connectivity; /**< Boolean: are we on IPv6?  */
+  int AuthDirPinKeys; /**< Boolean: Do we enforce key-pinning? */
+
+  /** If non-zero, always vote the Fast flag for any relay advertising
+   * this amount of capacity or more. */
+  uint64_t AuthDirFastGuarantee;
+
+  /** If non-zero, this advertised capacity or more is always sufficient
+   * to satisfy the bandwidth requirement for the Guard flag. */
+  uint64_t AuthDirGuardBWGuarantee;
+
+  char *AccountingStart; /**< How long is the accounting interval, and when
+                          * does it start? */
+  uint64_t AccountingMax; /**< How many bytes do we allow per accounting
+                           * interval before hibernation?  0 for "never
+                           * hibernate." */
+  /** How do we determine when our AccountingMax has been reached?
+   * "max" for when in or out reaches AccountingMax
+   * "sum" for when in plus out reaches AccountingMax
+   * "in"  for when in reaches AccountingMax
+   * "out" for when out reaches AccountingMax */
+  char *AccountingRule_option;
+  enum { ACCT_MAX, ACCT_SUM, ACCT_IN, ACCT_OUT } AccountingRule;
+
+  /** Base64-encoded hash of accepted passwords for the control system. */
+  struct config_line_t *HashedControlPassword;
+  /** As HashedControlPassword, but not saved. */
+  struct config_line_t *HashedControlSessionPassword;
+
+  int CookieAuthentication; /**< Boolean: do we enable cookie-based auth for
+                             * the control system? */
+  char *CookieAuthFile; /**< Filesystem location of a ControlPort
+                         *   authentication cookie. */
+  char *ExtORPortCookieAuthFile; /**< Filesystem location of Extended
+                                 *   ORPort authentication cookie. */
+  int CookieAuthFileGroupReadable; /**< Boolean: Is the CookieAuthFile g+r? */
+  int ExtORPortCookieAuthFileGroupReadable; /**< Boolean: Is the
+                                             * ExtORPortCookieAuthFile g+r? */
+  int LeaveStreamsUnattached; /**< Boolean: Does Tor attach new streams to
+                          * circuits itself (0), or does it expect a controller
+                          * to cope? (1) */
+  int DisablePredictedCircuits; /**< Boolean: does Tor preemptively
+                                 * make circuits in the background (0),
+                                 * or not (1)? */
+
+  /** Process specifier for a controller that ‘owns’ this Tor
+   * instance.  Tor will terminate if its owning controller does. */
+  char *OwningControllerProcess;
+  /** FD specifier for a controller that owns this Tor instance. */
+  uint64_t OwningControllerFD;
+
+  int ShutdownWaitLength; /**< When we get a SIGINT and we're a server, how
+                           * long do we wait before exiting? */
+  char *SafeLogging; /**< Contains "relay", "1", "0" (meaning no scrubbing). */
+
+  /* Derived from SafeLogging */
+  enum {
+    SAFELOG_SCRUB_ALL, SAFELOG_SCRUB_RELAY, SAFELOG_SCRUB_NONE
+  } SafeLogging_;
+
+  int Sandbox; /**< Boolean: should sandboxing be enabled? */
+  int SafeSocks; /**< Boolean: should we outright refuse application
+                  * connections that use socks4 or socks5-with-local-dns? */
+  int ProtocolWarnings; /**< Boolean: when other parties screw up the Tor
+                         * protocol, is it a warn or an info in our logs? */
+  int TestSocks; /**< Boolean: when we get a socks connection, do we loudly
+                  * log whether it was DNS-leaking or not? */
+  int HardwareAccel; /**< Boolean: Should we enable OpenSSL hardware
+                      * acceleration where available? */
+  /** Token Bucket Refill resolution in milliseconds. */
+  int TokenBucketRefillInterval;
+  char *AccelName; /**< Optional hardware acceleration engine name. */
+  char *AccelDir; /**< Optional hardware acceleration engine search dir. */
+
+  /** Boolean: Do we try to enter from a smallish number
+   * of fixed nodes? */
+  int UseEntryGuards_option;
+  /** Internal variable to remember whether we're actually acting on
+   * UseEntryGuards_option -- when we're a non-anonymous Single Onion Service,
+   * it is always false, otherwise we use the value of UseEntryGuards_option.
+   * */
+  int UseEntryGuards;
+
+  int NumEntryGuards; /**< How many entry guards do we try to establish? */
+
+  /** If 1, we use any guardfraction information we see in the
+   * consensus.  If 0, we don't.  If -1, let the consensus parameter
+   * decide. */
+  int UseGuardFraction;
+
+  int NumDirectoryGuards; /**< How many dir guards do we try to establish?
+                           * If 0, use value from NumEntryGuards. */
+  int NumPrimaryGuards; /**< How many primary guards do we want? */
+
+  int RephistTrackTime; /**< How many seconds do we keep rephist info? */
+  /** Should we always fetch our dir info on the mirror schedule (which
+   * means directly from the authorities) no matter our other config? */
+  int FetchDirInfoEarly;
+
+  /** Should we fetch our dir info at the start of the consensus period? */
+  int FetchDirInfoExtraEarly;
+
+  int DirCache; /**< Cache all directory documents and accept requests via
+                 * tunnelled dir conns from clients. If 1, enabled (default);
+                 * If 0, disabled. */
+
+  char *VirtualAddrNetworkIPv4; /**< Address and mask to hand out for virtual
+                                 * MAPADDRESS requests for IPv4 addresses */
+  char *VirtualAddrNetworkIPv6; /**< Address and mask to hand out for virtual
+                                 * MAPADDRESS requests for IPv6 addresses */
+  int ServerDNSSearchDomains; /**< Boolean: If set, we don't force exit
+                      * addresses to be FQDNs, but rather search for them in
+                      * the local domains. */
+  int ServerDNSDetectHijacking; /**< Boolean: If true, check for DNS failure
+                                 * hijacking. */
+  int ServerDNSRandomizeCase; /**< Boolean: Use the 0x20-hack to prevent
+                               * DNS poisoning attacks. */
+  char *ServerDNSResolvConfFile; /**< If provided, we configure our internal
+                     * resolver from the file here rather than from
+                     * /etc/resolv.conf (Unix) or the registry (Windows). */
+  char *DirPortFrontPage; /**< This is a full path to a file with an html
+                    disclaimer. This allows a server administrator to show
+                    that they're running Tor and anyone visiting their server
+                    will know this without any specialized knowledge. */
+  int DisableDebuggerAttachment; /**< Currently Linux only specific attempt to
+                                      disable ptrace; needs BSD testing. */
+  /** Boolean: if set, we start even if our resolv.conf file is missing
+   * or broken. */
+  int ServerDNSAllowBrokenConfig;
+  /** Boolean: if set, then even connections to private addresses will get
+   * rate-limited. */
+  int CountPrivateBandwidth;
+  /** A list of addresses that definitely should be resolvable. Used for
+   * testing our DNS server. */
+  struct smartlist_t *ServerDNSTestAddresses;
+  int EnforceDistinctSubnets; /**< If true, don't allow multiple routers in the
+                               * same network zone in the same circuit. */
+  int AllowNonRFC953Hostnames; /**< If true, we allow connections to hostnames
+                                * with weird characters. */
+  /** If true, we try resolving hostnames with weird characters. */
+  int ServerDNSAllowNonRFC953Hostnames;
+
+  /** If true, we try to download extra-info documents (and we serve them,
+   * if we are a cache).  For authorities, this is always true. */
+  int DownloadExtraInfo;
+
+  /** If true, we're configured to collect statistics on clients
+   * requesting network statuses from us as directory. */
+  int DirReqStatistics_option;
+  /** Internal variable to remember whether we're actually acting on
+   * DirReqStatistics_option -- yes if it's set and we're a server, else no. */
+  int DirReqStatistics;
+
+  /** If true, the user wants us to collect statistics on port usage. */
+  int ExitPortStatistics;
+
+  /** If true, the user wants us to collect connection statistics. */
+  int ConnDirectionStatistics;
+
+  /** If true, the user wants us to collect cell statistics. */
+  int CellStatistics;
+
+  /** If true, the user wants us to collect padding statistics. */
+  int PaddingStatistics;
+
+  /** If true, the user wants us to collect statistics as entry node. */
+  int EntryStatistics;
+
+  /** If true, the user wants us to collect statistics as hidden service
+   * directory, introduction point, or rendezvous point. */
+  int HiddenServiceStatistics_option;
+  /** Internal variable to remember whether we're actually acting on
+   * HiddenServiceStatistics_option -- yes if it's set and we're a server,
+   * else no. */
+  int HiddenServiceStatistics;
+
+  /** If true, include statistics file contents in extra-info documents. */
+  int ExtraInfoStatistics;
+
+  /** If true, do not believe anybody who tells us that a domain resolves
+   * to an internal address, or that an internal address has a PTR mapping.
+   * Helps avoid some cross-site attacks. */
+  int ClientDNSRejectInternalAddresses;
+
+  /** If true, do not accept any requests to connect to internal addresses
+   * over randomly chosen exits. */
+  int ClientRejectInternalAddresses;
+
+  /** If true, clients may connect over IPv4. If false, they will avoid
+   * connecting over IPv4. We enforce this for OR and Dir connections. */
+  int ClientUseIPv4;
+  /** If true, clients may connect over IPv6. If false, they will avoid
+   * connecting over IPv4. We enforce this for OR and Dir connections.
+   * Use fascist_firewall_use_ipv6() instead of accessing this value
+   * directly. */
+  int ClientUseIPv6;
+  /** If true, prefer an IPv6 OR port over an IPv4 one for entry node
+   * connections. If auto, bridge clients prefer IPv6, and other clients
+   * prefer IPv4. Use node_ipv6_or_preferred() instead of accessing this value
+   * directly. */
+  int ClientPreferIPv6ORPort;
+  /** If true, prefer an IPv6 directory port over an IPv4 one for direct
+   * directory connections. If auto, bridge clients prefer IPv6, and other
+   * clients prefer IPv4. Use fascist_firewall_prefer_ipv6_dirport() instead of
+   * accessing this value directly.  */
+  int ClientPreferIPv6DirPort;
+
+  /** The length of time that we think a consensus should be fresh. */
+  int V3AuthVotingInterval;
+  /** The length of time we think it will take to distribute votes. */
+  int V3AuthVoteDelay;
+  /** The length of time we think it will take to distribute signatures. */
+  int V3AuthDistDelay;
+  /** The number of intervals we think a consensus should be valid. */
+  int V3AuthNIntervalsValid;
+
+  /** Should advertise and sign consensuses with a legacy key, for key
+   * migration purposes? */
+  int V3AuthUseLegacyKey;
+
+  /** Location of bandwidth measurement file */
+  char *V3BandwidthsFile;
+
+  /** Location of guardfraction file */
+  char *GuardfractionFile;
+
+  /** Authority only: key=value pairs that we add to our networkstatus
+   * consensus vote on the 'params' line. */
+  char *ConsensusParams;
+
+  /** Authority only: minimum number of measured bandwidths we must see
+   * before we only believe measured bandwidths to assign flags. */
+  int MinMeasuredBWsForAuthToIgnoreAdvertised;
+
+  /** The length of time that we think an initial consensus should be fresh.
+   * Only altered on testing networks. */
+  int TestingV3AuthInitialVotingInterval;
+
+  /** The length of time we think it will take to distribute initial votes.
+   * Only altered on testing networks. */
+  int TestingV3AuthInitialVoteDelay;
+
+  /** The length of time we think it will take to distribute initial
+   * signatures.  Only altered on testing networks.*/
+  int TestingV3AuthInitialDistDelay;
+
+  /** Offset in seconds added to the starting time for consensus
+      voting. Only altered on testing networks. */
+  int TestingV3AuthVotingStartOffset;
+
+  /** If an authority has been around for less than this amount of time, it
+   * does not believe its reachability information is accurate.  Only
+   * altered on testing networks. */
+  int TestingAuthDirTimeToLearnReachability;
+
+  /** Clients don't download any descriptor this recent, since it will
+   * probably not have propagated to enough caches.  Only altered on testing
+   * networks. */
+  int TestingEstimatedDescriptorPropagationTime;
+
+  /** Schedule for when servers should download things in general.  Only
+   * altered on testing networks. */
+  int TestingServerDownloadInitialDelay;
+
+  /** Schedule for when clients should download things in general.  Only
+   * altered on testing networks. */
+  int TestingClientDownloadInitialDelay;
+
+  /** Schedule for when servers should download consensuses.  Only altered
+   * on testing networks. */
+  int TestingServerConsensusDownloadInitialDelay;
+
+  /** Schedule for when clients should download consensuses.  Only altered
+   * on testing networks. */
+  int TestingClientConsensusDownloadInitialDelay;
+
+  /** Schedule for when clients should download consensuses from authorities
+   * if they are bootstrapping (that is, they don't have a usable, reasonably
+   * live consensus).  Only used by clients fetching from a list of fallback
+   * directory mirrors.
+   *
+   * This schedule is incremented by (potentially concurrent) connection
+   * attempts, unlike other schedules, which are incremented by connection
+   * failures.  Only altered on testing networks. */
+  int ClientBootstrapConsensusAuthorityDownloadInitialDelay;
+
+  /** Schedule for when clients should download consensuses from fallback
+   * directory mirrors if they are bootstrapping (that is, they don't have a
+   * usable, reasonably live consensus). Only used by clients fetching from a
+   * list of fallback directory mirrors.
+   *
+   * This schedule is incremented by (potentially concurrent) connection
+   * attempts, unlike other schedules, which are incremented by connection
+   * failures.  Only altered on testing networks. */
+  int ClientBootstrapConsensusFallbackDownloadInitialDelay;
+
+  /** Schedule for when clients should download consensuses from authorities
+   * if they are bootstrapping (that is, they don't have a usable, reasonably
+   * live consensus).  Only used by clients which don't have or won't fetch
+   * from a list of fallback directory mirrors.
+   *
+   * This schedule is incremented by (potentially concurrent) connection
+   * attempts, unlike other schedules, which are incremented by connection
+   * failures.  Only altered on testing networks. */
+  int ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay;
+
+  /** Schedule for when clients should download bridge descriptors.  Only
+   * altered on testing networks. */
+  int TestingBridgeDownloadInitialDelay;
+
+  /** Schedule for when clients should download bridge descriptors when they
+   * have no running bridges.  Only altered on testing networks. */
+  int TestingBridgeBootstrapDownloadInitialDelay;
+
+  /** When directory clients have only a few descriptors to request, they
+   * batch them until they have more, or until this amount of time has
+   * passed.  Only altered on testing networks. */
+  int TestingClientMaxIntervalWithoutRequest;
+
+  /** How long do we let a directory connection stall before expiring
+   * it?  Only altered on testing networks. */
+  int TestingDirConnectionMaxStall;
+
+  /** How many simultaneous in-progress connections will we make when trying
+   * to fetch a consensus before we wait for one to complete, timeout, or
+   * error out?  Only altered on testing networks. */
+  int ClientBootstrapConsensusMaxInProgressTries;
+
+  /** If true, we take part in a testing network. Change the defaults of a
+   * couple of other configuration options and allow to change the values
+   * of certain configuration options. */
+  int TestingTorNetwork;
+
+  /** Minimum value for the Exit flag threshold on testing networks. */
+  uint64_t TestingMinExitFlagThreshold;
+
+  /** Minimum value for the Fast flag threshold on testing networks. */
+  uint64_t TestingMinFastFlagThreshold;
+
+  /** Relays in a testing network which should be voted Exit
+   * regardless of exit policy. */
+  routerset_t *TestingDirAuthVoteExit;
+  int TestingDirAuthVoteExitIsStrict;
+
+  /** Relays in a testing network which should be voted Guard
+   * regardless of uptime and bandwidth. */
+  routerset_t *TestingDirAuthVoteGuard;
+  int TestingDirAuthVoteGuardIsStrict;
+
+  /** Relays in a testing network which should be voted HSDir
+   * regardless of uptime and DirPort. */
+  routerset_t *TestingDirAuthVoteHSDir;
+  int TestingDirAuthVoteHSDirIsStrict;
+
+  /** Enable CONN_BW events.  Only altered on testing networks. */
+  int TestingEnableConnBwEvent;
+
+  /** Enable CELL_STATS events.  Only altered on testing networks. */
+  int TestingEnableCellStatsEvent;
+
+  /** If true, and we have GeoIP data, and we're a bridge, keep a per-country
+   * count of how many client addresses have contacted us so that we can help
+   * the bridge authority guess which countries have blocked access to us. */
+  int BridgeRecordUsageByCountry;
+
+  /** Optionally, IPv4 and IPv6 GeoIP data. */
+  char *GeoIPFile;
+  char *GeoIPv6File;
+
+  /** Autobool: if auto, then any attempt to Exclude{Exit,}Nodes a particular
+   * country code will exclude all nodes in ?? and A1.  If true, all nodes in
+   * ?? and A1 are excluded. Has no effect if we don't know any GeoIP data. */
+  int GeoIPExcludeUnknown;
+
+  /** If true, SIGHUP should reload the torrc.  Sometimes controllers want
+   * to make this false. */
+  int ReloadTorrcOnSIGHUP;
+
+  /* The main parameter for picking circuits within a connection.
+   *
+   * If this value is positive, when picking a cell to relay on a connection,
+   * we always relay from the circuit whose weighted cell count is lowest.
+   * Cells are weighted exponentially such that if one cell is sent
+   * 'CircuitPriorityHalflife' seconds before another, it counts for half as
+   * much.
+   *
+   * If this value is zero, we're disabling the cell-EWMA algorithm.
+   *
+   * If this value is negative, we're using the default approach
+   * according to either Tor or a parameter set in the consensus.
+   */
+  double CircuitPriorityHalflife;
+
+  /** Set to true if the TestingTorNetwork configuration option is set.
+   * This is used so that options_validate() has a chance to realize that
+   * the defaults have changed. */
+  int UsingTestNetworkDefaults_;
+
+  /** If 1, we try to use microdescriptors to build circuits.  If 0, we don't.
+   * If -1, Tor decides. */
+  int UseMicrodescriptors;
+
+  /** File where we should write the ControlPort. */
+  char *ControlPortWriteToFile;
+  /** Should that file be group-readable? */
+  int ControlPortFileGroupReadable;
+
+#define MAX_MAX_CLIENT_CIRCUITS_PENDING 1024
+  /** Maximum number of non-open general-purpose origin circuits to allow at
+   * once. */
+  int MaxClientCircuitsPending;
+
+  /** If 1, we always send optimistic data when it's supported.  If 0, we
+   * never use it.  If -1, we do what the consensus says. */
+  int OptimisticData;
+
+  /** If 1, we accept and launch no external network connections, except on
+   * control ports. */
+  int DisableNetwork;
+
+  /**
+   * Parameters for path-bias detection.
+   * @{
+   * These options override the default behavior of Tor's (**currently
+   * experimental**) path bias detection algorithm. To try to find broken or
+   * misbehaving guard nodes, Tor looks for nodes where more than a certain
+   * fraction of circuits through that guard fail to get built.
+   *
+   * The PathBiasCircThreshold option controls how many circuits we need to
+   * build through a guard before we make these checks.  The
+   * PathBiasNoticeRate, PathBiasWarnRate and PathBiasExtremeRate options
+   * control what fraction of circuits must succeed through a guard so we
+   * won't write log messages.  If less than PathBiasExtremeRate circuits
+   * succeed *and* PathBiasDropGuards is set to 1, we disable use of that
+   * guard.
+   *
+   * When we have seen more than PathBiasScaleThreshold circuits through a
+   * guard, we scale our observations by 0.5 (governed by the consensus) so
+   * that new observations don't get swamped by old ones.
+   *
+   * By default, or if a negative value is provided for one of these options,
+   * Tor uses reasonable defaults from the networkstatus consensus document.
+   * If no defaults are available there, these options default to 150, .70,
+   * .50, .30, 0, and 300 respectively.
+   */
+  int PathBiasCircThreshold;
+  double PathBiasNoticeRate;
+  double PathBiasWarnRate;
+  double PathBiasExtremeRate;
+  int PathBiasDropGuards;
+  int PathBiasScaleThreshold;
+  /** @} */
+
+  /**
+   * Parameters for path-bias use detection
+   * @{
+   * Similar to the above options, these options override the default behavior
+   * of Tor's (**currently experimental**) path use bias detection algorithm.
+   *
+   * Where as the path bias parameters govern thresholds for successfully
+   * building circuits, these four path use bias parameters govern thresholds
+   * only for circuit usage. Circuits which receive no stream usage are not
+   * counted by this detection algorithm. A used circuit is considered
+   * successful if it is capable of carrying streams or otherwise receiving
+   * well-formed responses to RELAY cells.
+   *
+   * By default, or if a negative value is provided for one of these options,
+   * Tor uses reasonable defaults from the networkstatus consensus document.
+   * If no defaults are available there, these options default to 20, .80,
+   * .60, and 100, respectively.
+   */
+  int PathBiasUseThreshold;
+  double PathBiasNoticeUseRate;
+  double PathBiasExtremeUseRate;
+  int PathBiasScaleUseThreshold;
+  /** @} */
+
+  int IPv6Exit; /**< Do we support exiting to IPv6 addresses? */
+
+  /** Fraction: */
+  double PathsNeededToBuildCircuits;
+
+  /** What expiry time shall we place on our SSL certs? "0" means we
+   * should guess a suitable value. */
+  int SSLKeyLifetime;
+
+  /** How long (seconds) do we keep a guard before picking a new one? */
+  int GuardLifetime;
+
+  /** Is this an exit node?  This is a tristate, where "1" means "yes, and use
+   * the default exit policy if none is given" and "0" means "no; exit policy
+   * is 'reject *'" and "auto" (-1) means "same as 1, but warn the user."
+   *
+   * XXXX Eventually, the default will be 0. */
+  int ExitRelay;
+
+  /** For how long (seconds) do we declare our signing keys to be valid? */
+  int SigningKeyLifetime;
+  /** For how long (seconds) do we declare our link keys to be valid? */
+  int TestingLinkCertLifetime;
+  /** For how long (seconds) do we declare our auth keys to be valid? */
+  int TestingAuthKeyLifetime;
+
+  /** How long before signing keys expire will we try to make a new one? */
+  int TestingSigningKeySlop;
+  /** How long before link keys expire will we try to make a new one? */
+  int TestingLinkKeySlop;
+  /** How long before auth keys expire will we try to make a new one? */
+  int TestingAuthKeySlop;
+
+  /** Force use of offline master key features: never generate a master
+   * ed25519 identity key except from tor --keygen */
+  int OfflineMasterKey;
+
+  enum {
+    FORCE_PASSPHRASE_AUTO=0,
+    FORCE_PASSPHRASE_ON,
+    FORCE_PASSPHRASE_OFF
+  } keygen_force_passphrase;
+  int use_keygen_passphrase_fd;
+  int keygen_passphrase_fd;
+  int change_key_passphrase;
+  char *master_key_fname;
+
+  /** Autobool: Do we try to retain capabilities if we can? */
+  int KeepBindCapabilities;
+
+  /** Maximum total size of unparseable descriptors to log during the
+   * lifetime of this Tor process.
+   */
+  uint64_t MaxUnparseableDescSizeToLog;
+
+  /** Bool (default: 1): Switch for the shared random protocol. Only
+   * relevant to a directory authority. If off, the authority won't
+   * participate in the protocol. If on (default), a flag is added to the
+   * vote indicating participation. */
+  int AuthDirSharedRandomness;
+
+  /** If 1, we skip all OOS checks. */
+  int DisableOOSCheck;
+
+  /** Autobool: Should we include Ed25519 identities in extend2 cells?
+   * If -1, we should do whatever the consensus parameter says. */
+  int ExtendByEd25519ID;
+
+  /** Bool (default: 1): When testing routerinfos as a directory authority,
+   * do we enforce Ed25519 identity match? */
+  /* NOTE: remove this option someday. */
+  int AuthDirTestEd25519LinkKeys;
+
+  /** Bool (default: 0): Tells if a %include was used on torrc */
+  int IncludeUsed;
+
+  /** The seconds after expiration which we as a relay should keep old
+   * consensuses around so that we can generate diffs from them.  If 0,
+   * use the default. */
+  int MaxConsensusAgeForDiffs;
+
+  /** Bool (default: 0). Tells Tor to never try to exec another program.
+   */
+  int NoExec;
+
+  /** Have the KIST scheduler run every X milliseconds. If less than zero, do
+   * not use the KIST scheduler but use the old vanilla scheduler instead. If
+   * zero, do what the consensus says and fall back to using KIST as if this is
+   * set to "10 msec" if the consensus doesn't say anything. */
+  int KISTSchedRunInterval;
+
+  /** A multiplier for the KIST per-socket limit calculation. */
+  double KISTSockBufSizeFactor;
+
+  /** The list of scheduler type string ordered by priority that is first one
+   * has to be tried first. Default: KIST,KISTLite,Vanilla */
+  struct smartlist_t *Schedulers;
+  /* An ordered list of scheduler_types mapped from Schedulers. */
+  struct smartlist_t *SchedulerTypes_;
+
+  /** List of files that were opened by %include in torrc and torrc-defaults */
+  struct smartlist_t *FilesOpenedByIncludes;
+
+  /** If true, Tor shouldn't install any posix signal handlers, since it is
+   * running embedded inside another process.
+   */
+  int DisableSignalHandlers;
+
+  /** Autobool: Is the circuit creation DoS mitigation subsystem enabled? */
+  int DoSCircuitCreationEnabled;
+  /** Minimum concurrent connection needed from one single address before any
+   * defense is used. */
+  int DoSCircuitCreationMinConnections;
+  /** Circuit rate used to refill the token bucket. */
+  int DoSCircuitCreationRate;
+  /** Maximum allowed burst of circuits. Reaching that value, the address is
+   * detected as malicious and a defense might be used. */
+  int DoSCircuitCreationBurst;
+  /** When an address is marked as malicous, what defense should be used
+   * against it. See the dos_cc_defense_type_t enum. */
+  int DoSCircuitCreationDefenseType;
+  /** For how much time (in seconds) the defense is applicable for a malicious
+   * address. A random time delta is added to the defense time of an address
+   * which will be between 1 second and half of this value. */
+  int DoSCircuitCreationDefenseTimePeriod;
+
+  /** Autobool: Is the DoS connection mitigation subsystem enabled? */
+  int DoSConnectionEnabled;
+  /** Maximum concurrent connection allowed per address. */
+  int DoSConnectionMaxConcurrentCount;
+  /** When an address is reaches the maximum count, what defense should be
+   * used against it. See the dos_conn_defense_type_t enum. */
+  int DoSConnectionDefenseType;
+
+  /** Autobool: Do we refuse single hop client rendezvous? */
+  int DoSRefuseSingleHopClientRendezvous;
+};
+
+#endif
diff --git a/src/app/config/or_state_st.h b/src/app/config/or_state_st.h
new file mode 100644
index 0000000000..5f8214d146
--- /dev/null
+++ b/src/app/config/or_state_st.h
@@ -0,0 +1,92 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file or_state_t
+ *
+ * \brief The or_state_t structure, which represents Tor's state file.
+ */
+
+#ifndef TOR_OR_STATE_ST_H
+#define TOR_OR_STATE_ST_H
+
+#include "lib/cc/torint.h"
+struct smartlist_t;
+
+/** Persistent state for an onion router, as saved to disk. */
+struct or_state_t {
+  uint32_t magic_;
+  /** The time at which we next plan to write the state to the disk.  Equal to
+   * TIME_MAX if there are no savable changes, 0 if there are changes that
+   * should be saved right away. */
+  time_t next_write;
+
+  /** When was the state last written to disk? */
+  time_t LastWritten;
+
+  /** Fields for accounting bandwidth use. */
+  time_t AccountingIntervalStart;
+  uint64_t AccountingBytesReadInInterval;
+  uint64_t AccountingBytesWrittenInInterval;
+  int AccountingSecondsActive;
+  int AccountingSecondsToReachSoftLimit;
+  time_t AccountingSoftLimitHitAt;
+  uint64_t AccountingBytesAtSoftLimit;
+  uint64_t AccountingExpectedUsage;
+
+  /** A list of Entry Guard-related configuration lines. (pre-prop271) */
+  struct config_line_t *EntryGuards;
+
+  /** A list of guard-related configuration lines. (post-prop271) */
+  struct config_line_t *Guard;
+
+  struct config_line_t *TransportProxies;
+
+  /** Cached revision counters for active hidden services on this host */
+  struct config_line_t *HidServRevCounter;
+
+  /** These fields hold information on the history of bandwidth usage for
+   * servers.  The "Ends" fields hold the time when we last updated the
+   * bandwidth usage. The "Interval" fields hold the granularity, in seconds,
+   * of the entries of Values.  The "Values" lists hold decimal string
+   * representations of the number of bytes read or written in each
+   * interval. The "Maxima" list holds decimal strings describing the highest
+   * rate achieved during the interval.
+   */
+  time_t      BWHistoryReadEnds;
+  int         BWHistoryReadInterval;
+  struct smartlist_t *BWHistoryReadValues;
+  struct smartlist_t *BWHistoryReadMaxima;
+  time_t      BWHistoryWriteEnds;
+  int         BWHistoryWriteInterval;
+  struct smartlist_t *BWHistoryWriteValues;
+  struct smartlist_t *BWHistoryWriteMaxima;
+  time_t      BWHistoryDirReadEnds;
+  int         BWHistoryDirReadInterval;
+  struct smartlist_t *BWHistoryDirReadValues;
+  struct smartlist_t *BWHistoryDirReadMaxima;
+  time_t      BWHistoryDirWriteEnds;
+  int         BWHistoryDirWriteInterval;
+  struct smartlist_t *BWHistoryDirWriteValues;
+  struct smartlist_t *BWHistoryDirWriteMaxima;
+
+  /** Build time histogram */
+  struct config_line_t * BuildtimeHistogram;
+  int TotalBuildTimes;
+  int CircuitBuildAbandonedCount;
+
+  /** What version of Tor wrote this state file? */
+  char *TorVersion;
+
+  /** Holds any unrecognized values we found in the state file, in the order
+   * in which we found them. */
+  struct config_line_t *ExtraLines;
+
+  /** When did we last rotate our onion key?  "0" for 'no idea'. */
+  time_t LastRotatedOnionKey;
+};
+
+#endif
diff --git a/src/or/statefile.c b/src/app/config/statefile.c
index 8fa4324b25..89039a05b5 100644
--- a/src/or/statefile.c
+++ b/src/app/config/statefile.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,17 +29,28 @@
  */
 
 #define STATEFILE_PRIVATE
-#include "or.h"
-#include "circuitstats.h"
-#include "config.h"
-#include "confparse.h"
-#include "connection.h"
-#include "entrynodes.h"
-#include "hibernate.h"
-#include "rephist.h"
-#include "router.h"
-#include "sandbox.h"
-#include "statefile.h"
+#include "core/or/or.h"
+#include "core/or/circuitstats.h"
+#include "app/config/config.h"
+#include "app/config/confparse.h"
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/connection.h"
+#include "feature/control/control.h"
+#include "feature/client/entrynodes.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/stats/rephist.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+#include "lib/sandbox/sandbox.h"
+#include "app/config/statefile.h"
+#include "lib/encoding/confline.h"
+#include "lib/net/resolve.h"
+
+#include "app/config/or_state_st.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
 
 /** A list of state-file "abbreviations," for compatibility. */
 static config_abbrev_t state_abbrevs_[] = {
@@ -53,10 +64,14 @@ static config_abbrev_t state_abbrevs_[] = {
   { NULL, NULL, 0, 0},
 };
 
+/** dummy instance of or_state_t, used for type-checking its
+ * members with CONF_CHECK_VAR_TYPE. */
+DUMMY_TYPECHECK_INSTANCE(or_state_t);
+
 /*XXXX these next two are duplicates or near-duplicates from config.c */
 #define VAR(name,conftype,member,initvalue)                             \
-  { name, CONFIG_TYPE_ ## conftype, STRUCT_OFFSET(or_state_t, member),  \
-      initvalue }
+  { name, CONFIG_TYPE_ ## conftype, offsetof(or_state_t, member),       \
+      initvalue CONF_TEST_MEMBERS(or_state_t, conftype, member) }
 /** As VAR, but the option name and member name are the same. */
 #define V(member,conftype,initvalue)                                    \
   VAR(#member, conftype, member, initvalue)
@@ -85,6 +100,8 @@ static config_var_t state_vars_[] = {
   VAR("TransportProxy",               LINELIST_S, TransportProxies, NULL),
   V(TransportProxies,                 LINELIST_V, NULL),
 
+  V(HidServRevCounter,            LINELIST, NULL),
+
   V(BWHistoryReadEnds,                ISOTIME,  NULL),
   V(BWHistoryReadInterval,            UINT,     "900"),
   V(BWHistoryReadValues,              CSV,      ""),
@@ -102,6 +119,8 @@ static config_var_t state_vars_[] = {
   V(BWHistoryDirWriteValues,          CSV,      ""),
   V(BWHistoryDirWriteMaxima,          CSV,      ""),
 
+  V(Guard,                            LINELIST, NULL),
+
   V(TorVersion,                       STRING,   NULL),
 
   V(LastRotatedOnionKey,              ISOTIME,  NULL),
@@ -111,7 +130,8 @@ static config_var_t state_vars_[] = {
   V(CircuitBuildAbandonedCount,       UINT,     "0"),
   VAR("CircuitBuildTimeBin",          LINELIST_S, BuildtimeHistogram, NULL),
   VAR("BuildtimeHistogram",           LINELIST_V, BuildtimeHistogram, NULL),
-  { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL }
+
+  END_OF_CONFIG_VARS
 };
 
 #undef VAR
@@ -123,24 +143,28 @@ static int or_state_validate_cb(void *old_options, void *options,
                                 void *default_options,
                                 int from_setconf, char **msg);
 
+static void or_state_free_cb(void *state);
+
 /** Magic value for or_state_t. */
 #define OR_STATE_MAGIC 0x57A73f57
 
 /** "Extra" variable in the state that receives lines we can't parse. This
  * lets us preserve options from versions of Tor newer than us. */
 static config_var_t state_extra_var = {
-  "__extra", CONFIG_TYPE_LINELIST, STRUCT_OFFSET(or_state_t, ExtraLines), NULL
+  "__extra", CONFIG_TYPE_LINELIST, offsetof(or_state_t, ExtraLines), NULL
+  CONF_TEST_MEMBERS(or_state_t, LINELIST, ExtraLines)
 };
 
 /** Configuration format for or_state_t. */
 static const config_format_t state_format = {
   sizeof(or_state_t),
   OR_STATE_MAGIC,
-  STRUCT_OFFSET(or_state_t, magic_),
+  offsetof(or_state_t, magic_),
   state_abbrevs_,
   NULL,
   state_vars_,
   or_state_validate_cb,
+  or_state_free_cb,
   &state_extra_var,
 };
 
@@ -238,6 +262,12 @@ or_state_validate_cb(void *old_state, void *state, void *default_state,
   return or_state_validate(state, msg);
 }
 
+static void
+or_state_free_cb(void *state)
+{
+  or_state_free_(state);
+}
+
 /** Return 0 if every setting in <b>state</b> is reasonable, and a
  * permissible transition from <b>old_state</b>.  Else warn and return -1.
  * Should have no side effects, except for normalizing the contents of
@@ -400,10 +430,15 @@ or_state_load(void)
     log_info(LD_GENERAL, "Loaded state from \"%s\"", fname);
     /* Warn the user if their clock has been set backwards,
      * they could be tricked into using old consensuses */
-    time_t apparent_skew = new_state->LastWritten - time(NULL);
-    if (apparent_skew > 0)
+    time_t apparent_skew = time(NULL) - new_state->LastWritten;
+    if (apparent_skew < 0) {
+      /* Initialize bootstrap event reporting because we might call
+       * clock_skew_warning() before the bootstrap state is
+       * initialized, causing an assertion failure. */
+      control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0);
       clock_skew_warning(NULL, (long)apparent_skew, 1, LD_GENERAL,
                          "local state file", fname);
+    }
   } else {
     log_info(LD_GENERAL, "Initialized state");
   }
@@ -655,8 +690,6 @@ save_transport_to_state(const char *transport,
     *next = line = tor_malloc_zero(sizeof(config_line_t));
     line->key = tor_strdup("TransportProxy");
     tor_asprintf(&line->value, "%s %s", transport, fmt_addrport(addr, port));
-
-    next = &(line->next);
   }
 
   if (!get_options()->AvoidDiskWrites)
@@ -666,8 +699,20 @@ save_transport_to_state(const char *transport,
   tor_free(transport_addrport);
 }
 
+/** Change the next_write time of <b>state</b> to <b>when</b>, unless the
+ * state is already scheduled to be written to disk earlier than <b>when</b>.
+ */
+void
+or_state_mark_dirty(or_state_t *state, time_t when)
+{
+  if (state->next_write > when) {
+    state->next_write = when;
+    reschedule_or_state_save();
+  }
+}
+
 STATIC void
-or_state_free(or_state_t *state)
+or_state_free_(or_state_t *state)
 {
   if (!state)
     return;
@@ -681,4 +726,3 @@ or_state_free_all(void)
   or_state_free(global_state);
   global_state = NULL;
 }
-
diff --git a/src/or/statefile.h b/src/app/config/statefile.h
index b13743481d..1950078450 100644
--- a/src/or/statefile.h
+++ b/src/app/config/statefile.h
@@ -1,9 +1,15 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * \file statefile.h
+ *
+ * \brief Header for statefile.c
+ */
+
 #ifndef TOR_STATEFILE_H
 #define TOR_STATEFILE_H
 
@@ -17,12 +23,14 @@ char *get_stored_bindaddr_for_server_transport(const char *transport);
 int or_state_load(void);
 int or_state_loaded(void);
 void or_state_free_all(void);
+void or_state_mark_dirty(or_state_t *state, time_t when);
 
 #ifdef STATEFILE_PRIVATE
-STATIC config_line_t *get_transport_in_state_by_name(const char *transport);
-STATIC void or_state_free(or_state_t *state);
+STATIC struct config_line_t *get_transport_in_state_by_name(
+                                                 const char *transport);
+STATIC void or_state_free_(or_state_t *state);
+#define or_state_free(st) FREE_AND_NULL(or_state_t, or_state_free_, (st))
 STATIC or_state_t *or_state_new(void);
 #endif
 
-#endif
-
+#endif /* !defined(TOR_STATEFILE_H) */
diff --git a/src/app/include.am b/src/app/include.am
new file mode 100644
index 0000000000..97d53ec0fd
--- /dev/null
+++ b/src/app/include.am
@@ -0,0 +1,35 @@
+
+bin_PROGRAMS+= src/app/tor
+
+if COVERAGE_ENABLED
+noinst_PROGRAMS+= src/app/tor-cov
+endif
+
+noinst_HEADERS += \
+	src/app/main/ntmain.h
+
+src_app_tor_SOURCES = src/app/main/tor_main.c
+
+# -L flags need to go in LDFLAGS. -l flags need to go in LDADD.
+# This seems to matter nowhere but on windows, but I assure you that it
+# matters a lot there, and is quite hard to debug if you forget to do it.
+
+src_app_tor_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) @TOR_LDFLAGS_libevent@
+src_app_tor_LDADD = $(TOR_INTERNAL_LIBS) \
+	$(rust_ldadd) \
+	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \
+	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
+	@CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \
+	@TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@
+
+if COVERAGE_ENABLED
+src_app_tor_cov_SOURCES = $(src_app_tor_SOURCES)
+src_app_tor_cov_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_app_tor_cov_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+src_app_tor_cov_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) @TOR_LDFLAGS_libevent@
+src_app_tor_cov_LDADD = $(TOR_INTERNAL_TESTING_LIBS) \
+	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \
+	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ \
+	@CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \
+	@TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@
+endif
diff --git a/src/app/main/main.c b/src/app/main/main.c
new file mode 100644
index 0000000000..67f2181cd5
--- /dev/null
+++ b/src/app/main/main.c
@@ -0,0 +1,1519 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file main.c
+ * \brief Invocation module.  Initializes subsystems and runs the main loop.
+ **/
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "app/config/statefile.h"
+#include "app/main/main.h"
+#include "app/main/ntmain.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/cpuworker.h"
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
+#include "core/or/channel.h"
+#include "core/or/channelpadding.h"
+#include "core/or/channeltls.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitmux_ewma.h"
+#include "core/or/command.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "core/or/dos.h"
+#include "core/or/policies.h"
+#include "core/or/protover.h"
+#include "core/or/relay.h"
+#include "core/or/scheduler.h"
+#include "core/or/status.h"
+#include "feature/api/tor_api.h"
+#include "feature/api/tor_api_internal.h"
+#include "feature/client/addressmap.h"
+#include "feature/client/bridges.h"
+#include "feature/client/entrynodes.h"
+#include "feature/client/transports.h"
+#include "feature/control/control.h"
+#include "feature/dirauth/bwauth.h"
+#include "feature/dirauth/keypin.h"
+#include "feature/dirauth/process_descs.h"
+#include "feature/dircache/consdiffmgr.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirparse/routerparse.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/hs/hs_cache.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/dns.h"
+#include "feature/relay/ext_orport.h"
+#include "feature/relay/onion_queue.h"
+#include "feature/relay/routerkeys.h"
+#include "feature/relay/routermode.h"
+#include "feature/rend/rendcache.h"
+#include "feature/rend/rendclient.h"
+#include "feature/rend/rendservice.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/stats/predict_ports.h"
+#include "feature/stats/rephist.h"
+#include "lib/compress/compress.h"
+#include "lib/container/buffers.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_s2k.h"
+#include "lib/err/backtrace.h"
+#include "lib/geoip/geoip.h"
+
+#include "lib/process/waitpid.h"
+
+#include "lib/meminfo/meminfo.h"
+#include "lib/osinfo/uname.h"
+#include "lib/sandbox/sandbox.h"
+#include "lib/fs/lockfile.h"
+#include "lib/net/resolve.h"
+#include "lib/tls/tortls.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/encoding/confline.h"
+#include "lib/evloop/timers.h"
+#include "lib/crypt_ops/crypto_init.h"
+
+#include <event2/event.h>
+
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/authmode.h"
+#include "feature/dirauth/shared_random.h"
+
+#include "core/or/or_connection_st.h"
+#include "core/or/port_cfg_st.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifdef HAVE_SYSTEMD
+#   if defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__)
+/* Systemd's use of gcc's __INCLUDE_LEVEL__ extension macro appears to confuse
+ * Coverity. Here's a kludge to unconfuse it.
+ */
+#   define __INCLUDE_LEVEL__ 2
+#endif /* defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) */
+#include <systemd/sd-daemon.h>
+#endif /* defined(HAVE_SYSTEMD) */
+
+void evdns_shutdown(int);
+
+#ifdef HAVE_RUST
+// helper function defined in Rust to output a log message indicating if tor is
+// running with Rust enabled. See src/rust/tor_util
+void rust_log_welcome_string(void);
+#endif
+
+/********* PROTOTYPES **********/
+
+static void dumpmemusage(int severity);
+static void dumpstats(int severity); /* log stats */
+static void process_signal(int sig);
+
+/********* START VARIABLES **********/
+
+/** Decides our behavior when no logs are configured/before any
+ * logs have been configured.  For 0, we log notice to stdout as normal.
+ * For 1, we log warnings only.  For 2, we log nothing.
+ */
+int quiet_level = 0;
+
+/********* END VARIABLES ************/
+
+/** Called when we get a SIGHUP: reload configuration files and keys,
+ * retry all connections, and so on. */
+static int
+do_hup(void)
+{
+  const or_options_t *options = get_options();
+
+  log_notice(LD_GENERAL,"Received reload signal (hup). Reloading config and "
+             "resetting internal state.");
+  if (accounting_is_enabled(options))
+    accounting_record_bandwidth_usage(time(NULL), get_or_state());
+
+  router_reset_warnings();
+  routerlist_reset_warnings();
+  /* first, reload config variables, in case they've changed */
+  if (options->ReloadTorrcOnSIGHUP) {
+    /* no need to provide argc/v, they've been cached in init_from_config */
+    int init_rv = options_init_from_torrc(0, NULL);
+    if (init_rv < 0) {
+      log_err(LD_CONFIG,"Reading config failed--see warnings above. "
+              "For usage, try -h.");
+      return -1;
+    } else if (BUG(init_rv > 0)) {
+      // LCOV_EXCL_START
+      /* This should be impossible: the only "return 1" cases in
+       * options_init_from_torrc are ones caused by command-line arguments;
+       * but they can't change while Tor is running. */
+      return -1;
+      // LCOV_EXCL_STOP
+    }
+    options = get_options(); /* they have changed now */
+    /* Logs are only truncated the first time they are opened, but were
+       probably intended to be cleaned up on signal. */
+    if (options->TruncateLogFile)
+      truncate_logs();
+  } else {
+    char *msg = NULL;
+    log_notice(LD_GENERAL, "Not reloading config file: the controller told "
+               "us not to.");
+    /* Make stuff get rescanned, reloaded, etc. */
+    if (set_options((or_options_t*)options, &msg) < 0) {
+      if (!msg)
+        msg = tor_strdup("Unknown error");
+      log_warn(LD_GENERAL, "Unable to re-set previous options: %s", msg);
+      tor_free(msg);
+    }
+  }
+  if (authdir_mode(options)) {
+    /* reload the approved-routers file */
+    if (dirserv_load_fingerprint_file() < 0) {
+      /* warnings are logged from dirserv_load_fingerprint_file() directly */
+      log_info(LD_GENERAL, "Error reloading fingerprints. "
+               "Continuing with old list.");
+    }
+  }
+
+  /* Rotate away from the old dirty circuits. This has to be done
+   * after we've read the new options, but before we start using
+   * circuits for directory fetches. */
+  circuit_mark_all_dirty_circs_as_unusable();
+
+  /* retry appropriate downloads */
+  router_reset_status_download_failures();
+  router_reset_descriptor_download_failures();
+  if (!net_is_disabled())
+    update_networkstatus_downloads(time(NULL));
+
+  /* We'll retry routerstatus downloads in about 10 seconds; no need to
+   * force a retry there. */
+
+  if (server_mode(options)) {
+    /* Maybe we've been given a new ed25519 key or certificate?
+     */
+    time_t now = approx_time();
+    int new_signing_key = load_ed_keys(options, now);
+    if (new_signing_key < 0 ||
+        generate_ed_link_cert(options, now, new_signing_key > 0)) {
+      log_warn(LD_OR, "Problem reloading Ed25519 keys; still using old keys.");
+    }
+
+    /* Update cpuworker and dnsworker processes, so they get up-to-date
+     * configuration options. */
+    cpuworkers_rotate_keyinfo();
+    dns_reset();
+  }
+  return 0;
+}
+
+/** Libevent callback: invoked when we get a signal.
+ */
+static void
+signal_callback(evutil_socket_t fd, short events, void *arg)
+{
+  const int *sigptr = arg;
+  const int sig = *sigptr;
+  (void)fd;
+  (void)events;
+
+  update_current_time(time(NULL));
+  process_signal(sig);
+}
+
+/** Do the work of acting on a signal received in <b>sig</b> */
+static void
+process_signal(int sig)
+{
+  switch (sig)
+    {
+    case SIGTERM:
+      log_notice(LD_GENERAL,"Catching signal TERM, exiting cleanly.");
+      tor_shutdown_event_loop_and_exit(0);
+      break;
+    case SIGINT:
+      if (!server_mode(get_options())) { /* do it now */
+        log_notice(LD_GENERAL,"Interrupt: exiting cleanly.");
+        tor_shutdown_event_loop_and_exit(0);
+        return;
+      }
+#ifdef HAVE_SYSTEMD
+      sd_notify(0, "STOPPING=1");
+#endif
+      hibernate_begin_shutdown();
+      break;
+#ifdef SIGPIPE
+    case SIGPIPE:
+      log_debug(LD_GENERAL,"Caught SIGPIPE. Ignoring.");
+      break;
+#endif
+    case SIGUSR1:
+      /* prefer to log it at INFO, but make sure we always see it */
+      dumpstats(get_min_log_level()<LOG_INFO ? get_min_log_level() : LOG_INFO);
+      control_event_signal(sig);
+      break;
+    case SIGUSR2:
+      switch_logs_debug();
+      log_debug(LD_GENERAL,"Caught USR2, going to loglevel debug. "
+                "Send HUP to change back.");
+      control_event_signal(sig);
+      break;
+    case SIGHUP:
+#ifdef HAVE_SYSTEMD
+      sd_notify(0, "RELOADING=1");
+#endif
+      if (do_hup() < 0) {
+        log_warn(LD_CONFIG,"Restart failed (config error?). Exiting.");
+        tor_shutdown_event_loop_and_exit(1);
+        return;
+      }
+#ifdef HAVE_SYSTEMD
+      sd_notify(0, "READY=1");
+#endif
+      control_event_signal(sig);
+      break;
+#ifdef SIGCHLD
+    case SIGCHLD:
+      notify_pending_waitpid_callbacks();
+      break;
+#endif
+    case SIGNEWNYM: {
+      do_signewnym(time(NULL));
+      break;
+    }
+    case SIGCLEARDNSCACHE:
+      addressmap_clear_transient();
+      control_event_signal(sig);
+      break;
+    case SIGHEARTBEAT:
+      log_heartbeat(time(NULL));
+      control_event_signal(sig);
+      break;
+  }
+}
+
+/**
+ * Write current memory usage information to the log.
+ */
+static void
+dumpmemusage(int severity)
+{
+  connection_dump_buffer_mem_stats(severity);
+  tor_log(severity, LD_GENERAL, "In rephist: %"PRIu64" used by %d Tors.",
+      (rephist_total_alloc), rephist_total_num);
+  dump_routerlist_mem_usage(severity);
+  dump_cell_pool_usage(severity);
+  dump_dns_mem_usage(severity);
+  tor_log_mallinfo(severity);
+}
+
+/** Write all statistics to the log, with log level <b>severity</b>. Called
+ * in response to a SIGUSR1. */
+static void
+dumpstats(int severity)
+{
+  time_t now = time(NULL);
+  time_t elapsed;
+  size_t rbuf_cap, wbuf_cap, rbuf_len, wbuf_len;
+
+  tor_log(severity, LD_GENERAL, "Dumping stats:");
+
+  SMARTLIST_FOREACH_BEGIN(get_connection_array(), connection_t *, conn) {
+    int i = conn_sl_idx;
+    tor_log(severity, LD_GENERAL,
+        "Conn %d (socket %d) type %d (%s), state %d (%s), created %d secs ago",
+        i, (int)conn->s, conn->type, conn_type_to_string(conn->type),
+        conn->state, conn_state_to_string(conn->type, conn->state),
+        (int)(now - conn->timestamp_created));
+    if (!connection_is_listener(conn)) {
+      tor_log(severity,LD_GENERAL,
+          "Conn %d is to %s:%d.", i,
+          safe_str_client(conn->address),
+          conn->port);
+      tor_log(severity,LD_GENERAL,
+          "Conn %d: %d bytes waiting on inbuf (len %d, last read %d secs ago)",
+          i,
+          (int)connection_get_inbuf_len(conn),
+          (int)buf_allocation(conn->inbuf),
+          (int)(now - conn->timestamp_last_read_allowed));
+      tor_log(severity,LD_GENERAL,
+          "Conn %d: %d bytes waiting on outbuf "
+          "(len %d, last written %d secs ago)",i,
+          (int)connection_get_outbuf_len(conn),
+          (int)buf_allocation(conn->outbuf),
+          (int)(now - conn->timestamp_last_write_allowed));
+      if (conn->type == CONN_TYPE_OR) {
+        or_connection_t *or_conn = TO_OR_CONN(conn);
+        if (or_conn->tls) {
+          if (tor_tls_get_buffer_sizes(or_conn->tls, &rbuf_cap, &rbuf_len,
+                                       &wbuf_cap, &wbuf_len) == 0) {
+            tor_log(severity, LD_GENERAL,
+                "Conn %d: %d/%d bytes used on OpenSSL read buffer; "
+                "%d/%d bytes used on write buffer.",
+                i, (int)rbuf_len, (int)rbuf_cap, (int)wbuf_len, (int)wbuf_cap);
+          }
+        }
+      }
+    }
+    circuit_dump_by_conn(conn, severity); /* dump info about all the circuits
+                                           * using this conn */
+  } SMARTLIST_FOREACH_END(conn);
+
+  channel_dumpstats(severity);
+  channel_listener_dumpstats(severity);
+
+  tor_log(severity, LD_NET,
+      "Cells processed: %"PRIu64" padding\n"
+      "                 %"PRIu64" create\n"
+      "                 %"PRIu64" created\n"
+      "                 %"PRIu64" relay\n"
+      "                        (%"PRIu64" relayed)\n"
+      "                        (%"PRIu64" delivered)\n"
+      "                 %"PRIu64" destroy",
+      (stats_n_padding_cells_processed),
+      (stats_n_create_cells_processed),
+      (stats_n_created_cells_processed),
+      (stats_n_relay_cells_processed),
+      (stats_n_relay_cells_relayed),
+      (stats_n_relay_cells_delivered),
+      (stats_n_destroy_cells_processed));
+  if (stats_n_data_cells_packaged)
+    tor_log(severity,LD_NET,"Average packaged cell fullness: %2.3f%%",
+        100*(((double)stats_n_data_bytes_packaged) /
+             ((double)stats_n_data_cells_packaged*RELAY_PAYLOAD_SIZE)) );
+  if (stats_n_data_cells_received)
+    tor_log(severity,LD_NET,"Average delivered cell fullness: %2.3f%%",
+        100*(((double)stats_n_data_bytes_received) /
+             ((double)stats_n_data_cells_received*RELAY_PAYLOAD_SIZE)) );
+
+  cpuworker_log_onionskin_overhead(severity, ONION_HANDSHAKE_TYPE_TAP, "TAP");
+  cpuworker_log_onionskin_overhead(severity, ONION_HANDSHAKE_TYPE_NTOR,"ntor");
+
+  if (now - time_of_process_start >= 0)
+    elapsed = now - time_of_process_start;
+  else
+    elapsed = 0;
+
+  if (elapsed) {
+    tor_log(severity, LD_NET,
+        "Average bandwidth: %"PRIu64"/%d = %d bytes/sec reading",
+        (get_bytes_read()),
+        (int)elapsed,
+        (int) (get_bytes_read()/elapsed));
+    tor_log(severity, LD_NET,
+        "Average bandwidth: %"PRIu64"/%d = %d bytes/sec writing",
+        (get_bytes_written()),
+        (int)elapsed,
+        (int) (get_bytes_written()/elapsed));
+  }
+
+  tor_log(severity, LD_NET, "--------------- Dumping memory information:");
+  dumpmemusage(severity);
+
+  rep_hist_dump_stats(now,severity);
+  rend_service_dump_stats(severity);
+}
+
+/** Called by exit() as we shut down the process.
+ */
+static void
+exit_function(void)
+{
+  /* NOTE: If we ever daemonize, this gets called immediately.  That's
+   * okay for now, because we only use this on Windows.  */
+#ifdef _WIN32
+  WSACleanup();
+#endif
+}
+
+#ifdef _WIN32
+#define UNIX_ONLY 0
+#else
+#define UNIX_ONLY 1
+#endif
+
+static struct {
+  /** A numeric code for this signal. Must match the signal value if
+   * try_to_register is true. */
+  int signal_value;
+  /** True if we should try to register this signal with libevent and catch
+   * corresponding posix signals. False otherwise. */
+  int try_to_register;
+  /** Pointer to hold the event object constructed for this signal. */
+  struct event *signal_event;
+} signal_handlers[] = {
+#ifdef SIGINT
+  { SIGINT, UNIX_ONLY, NULL }, /* do a controlled slow shutdown */
+#endif
+#ifdef SIGTERM
+  { SIGTERM, UNIX_ONLY, NULL }, /* to terminate now */
+#endif
+#ifdef SIGPIPE
+  { SIGPIPE, UNIX_ONLY, NULL }, /* otherwise SIGPIPE kills us */
+#endif
+#ifdef SIGUSR1
+  { SIGUSR1, UNIX_ONLY, NULL }, /* dump stats */
+#endif
+#ifdef SIGUSR2
+  { SIGUSR2, UNIX_ONLY, NULL }, /* go to loglevel debug */
+#endif
+#ifdef SIGHUP
+  { SIGHUP, UNIX_ONLY, NULL }, /* to reload config, retry conns, etc */
+#endif
+#ifdef SIGXFSZ
+  { SIGXFSZ, UNIX_ONLY, NULL }, /* handle file-too-big resource exhaustion */
+#endif
+#ifdef SIGCHLD
+  { SIGCHLD, UNIX_ONLY, NULL }, /* handle dns/cpu workers that exit */
+#endif
+  /* These are controller-only */
+  { SIGNEWNYM, 0, NULL },
+  { SIGCLEARDNSCACHE, 0, NULL },
+  { SIGHEARTBEAT, 0, NULL },
+  { -1, -1, NULL }
+};
+
+/** Set up the signal handler events for this process, and register them
+ * with libevent if appropriate. */
+void
+handle_signals(void)
+{
+  int i;
+  const int enabled = !get_options()->DisableSignalHandlers;
+
+  for (i = 0; signal_handlers[i].signal_value >= 0; ++i) {
+    /* Signal handlers are only registered with libevent if they need to catch
+     * real POSIX signals.  We construct these signal handler events in either
+     * case, though, so that controllers can activate them with the SIGNAL
+     * command.
+     */
+    if (enabled && signal_handlers[i].try_to_register) {
+      signal_handlers[i].signal_event =
+        tor_evsignal_new(tor_libevent_get_base(),
+                         signal_handlers[i].signal_value,
+                         signal_callback,
+                         &signal_handlers[i].signal_value);
+      if (event_add(signal_handlers[i].signal_event, NULL))
+        log_warn(LD_BUG, "Error from libevent when adding "
+                 "event for signal %d",
+                 signal_handlers[i].signal_value);
+    } else {
+      signal_handlers[i].signal_event =
+        tor_event_new(tor_libevent_get_base(), -1,
+                      EV_SIGNAL, signal_callback,
+                      &signal_handlers[i].signal_value);
+    }
+  }
+}
+
+/* Cause the signal handler for signal_num to be called in the event loop. */
+void
+activate_signal(int signal_num)
+{
+  int i;
+  for (i = 0; signal_handlers[i].signal_value >= 0; ++i) {
+    if (signal_handlers[i].signal_value == signal_num) {
+      event_active(signal_handlers[i].signal_event, EV_SIGNAL, 1);
+      return;
+    }
+  }
+}
+
+/** Main entry point for the Tor command-line client.  Return 0 on "success",
+ * negative on "failure", and positive on "success and exit".
+ */
+int
+tor_init(int argc, char *argv[])
+{
+  char progname[256];
+  int quiet = 0;
+
+  time_of_process_start = time(NULL);
+  tor_init_connection_lists();
+  /* Have the log set up with our application name. */
+  tor_snprintf(progname, sizeof(progname), "Tor %s", get_version());
+  log_set_application_name(progname);
+
+  /* Set up the crypto nice and early */
+  if (crypto_early_init() < 0) {
+    log_err(LD_GENERAL, "Unable to initialize the crypto subsystem!");
+    return -1;
+  }
+
+  /* Initialize the history structures. */
+  rep_hist_init();
+  /* Initialize the service cache. */
+  rend_cache_init();
+  addressmap_init(); /* Init the client dns cache. Do it always, since it's
+                      * cheap. */
+  /* Initialize the HS subsystem. */
+  hs_init();
+
+  {
+  /* We search for the "quiet" option first, since it decides whether we
+   * will log anything at all to the command line. */
+    config_line_t *opts = NULL, *cmdline_opts = NULL;
+    const config_line_t *cl;
+    (void) config_parse_commandline(argc, argv, 1, &opts, &cmdline_opts);
+    for (cl = cmdline_opts; cl; cl = cl->next) {
+      if (!strcmp(cl->key, "--hush"))
+        quiet = 1;
+      if (!strcmp(cl->key, "--quiet") ||
+          !strcmp(cl->key, "--dump-config"))
+        quiet = 2;
+      /* The following options imply --hush */
+      if (!strcmp(cl->key, "--version") || !strcmp(cl->key, "--digests") ||
+          !strcmp(cl->key, "--list-torrc-options") ||
+          !strcmp(cl->key, "--library-versions") ||
+          !strcmp(cl->key, "--list-modules") ||
+          !strcmp(cl->key, "--hash-password") ||
+          !strcmp(cl->key, "-h") || !strcmp(cl->key, "--help")) {
+        if (quiet < 1)
+          quiet = 1;
+      }
+    }
+    config_free_lines(opts);
+    config_free_lines(cmdline_opts);
+  }
+
+ /* give it somewhere to log to initially */
+  switch (quiet) {
+    case 2:
+      /* no initial logging */
+      break;
+    case 1:
+      add_temp_log(LOG_WARN);
+      break;
+    default:
+      add_temp_log(LOG_NOTICE);
+  }
+  quiet_level = quiet;
+
+  {
+    const char *version = get_version();
+
+    log_notice(LD_GENERAL, "Tor %s running on %s with Libevent %s, "
+               "%s %s, Zlib %s, Liblzma %s, and Libzstd %s.", version,
+               get_uname(),
+               tor_libevent_get_version_str(),
+               crypto_get_library_name(),
+               crypto_get_library_version_string(),
+               tor_compress_supports_method(ZLIB_METHOD) ?
+                 tor_compress_version_str(ZLIB_METHOD) : "N/A",
+               tor_compress_supports_method(LZMA_METHOD) ?
+                 tor_compress_version_str(LZMA_METHOD) : "N/A",
+               tor_compress_supports_method(ZSTD_METHOD) ?
+                 tor_compress_version_str(ZSTD_METHOD) : "N/A");
+
+    log_notice(LD_GENERAL, "Tor can't help you if you use it wrong! "
+               "Learn how to be safe at "
+               "https://www.torproject.org/download/download#warning");
+
+    if (strstr(version, "alpha") || strstr(version, "beta"))
+      log_notice(LD_GENERAL, "This version is not a stable Tor release. "
+                 "Expect more bugs than usual.");
+
+    tor_compress_log_init_warnings();
+  }
+
+#ifdef HAVE_RUST
+  rust_log_welcome_string();
+#endif /* defined(HAVE_RUST) */
+
+  if (network_init()<0) {
+    log_err(LD_BUG,"Error initializing network; exiting.");
+    return -1;
+  }
+  atexit(exit_function);
+
+  int init_rv = options_init_from_torrc(argc,argv);
+  if (init_rv < 0) {
+    log_err(LD_CONFIG,"Reading config failed--see warnings above.");
+    return -1;
+  } else if (init_rv > 0) {
+    // We succeeded, and should exit anyway -- probably the user just said
+    // "--version" or something like that.
+    return 1;
+  }
+
+  /* The options are now initialised */
+  const or_options_t *options = get_options();
+
+  /* Initialize channelpadding parameters to defaults until we get
+   * a consensus */
+  channelpadding_new_consensus_params(NULL);
+
+  /* Initialize predicted ports list after loading options */
+  predicted_ports_init();
+
+#ifndef _WIN32
+  if (geteuid()==0)
+    log_warn(LD_GENERAL,"You are running Tor as root. You don't need to, "
+             "and you probably shouldn't.");
+#endif
+
+  if (crypto_global_init(options->HardwareAccel,
+                         options->AccelName,
+                         options->AccelDir)) {
+    log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting.");
+    return -1;
+  }
+  stream_choice_seed_weak_rng();
+  if (tor_init_libevent_rng() < 0) {
+    log_warn(LD_NET, "Problem initializing libevent RNG.");
+  }
+
+  /* Scan/clean unparseable descriptors; after reading config */
+  routerparse_init();
+
+  return 0;
+}
+
+/** A lockfile structure, used to prevent two Tors from messing with the
+ * data directory at once.  If this variable is non-NULL, we're holding
+ * the lockfile. */
+static tor_lockfile_t *lockfile = NULL;
+
+/** Try to grab the lock file described in <b>options</b>, if we do not
+ * already have it.  If <b>err_if_locked</b> is true, warn if somebody else is
+ * holding the lock, and exit if we can't get it after waiting.  Otherwise,
+ * return -1 if we can't get the lockfile.  Return 0 on success.
+ */
+int
+try_locking(const or_options_t *options, int err_if_locked)
+{
+  if (lockfile)
+    return 0;
+  else {
+    char *fname = options_get_datadir_fname(options, "lock");
+    int already_locked = 0;
+    tor_lockfile_t *lf = tor_lockfile_lock(fname, 0, &already_locked);
+    tor_free(fname);
+    if (!lf) {
+      if (err_if_locked && already_locked) {
+        int r;
+        log_warn(LD_GENERAL, "It looks like another Tor process is running "
+                 "with the same data directory.  Waiting 5 seconds to see "
+                 "if it goes away.");
+#ifndef _WIN32
+        sleep(5);
+#else
+        Sleep(5000);
+#endif
+        r = try_locking(options, 0);
+        if (r<0) {
+          log_err(LD_GENERAL, "No, it's still there.  Exiting.");
+          return -1;
+        }
+        return r;
+      }
+      return -1;
+    }
+    lockfile = lf;
+    return 0;
+  }
+}
+
+/** Return true iff we've successfully acquired the lock file. */
+int
+have_lockfile(void)
+{
+  return lockfile != NULL;
+}
+
+/** If we have successfully acquired the lock file, release it. */
+void
+release_lockfile(void)
+{
+  if (lockfile) {
+    tor_lockfile_unlock(lockfile);
+    lockfile = NULL;
+  }
+}
+
+/** Free all memory that we might have allocated somewhere.
+ * If <b>postfork</b>, we are a worker process and we want to free
+ * only the parts of memory that we won't touch. If !<b>postfork</b>,
+ * Tor is shutting down and we should free everything.
+ *
+ * Helps us find the real leaks with sanitizers and the like. Also valgrind
+ * should then report 0 reachable in its leak report (in an ideal world --
+ * in practice libevent, SSL, libc etc never quite free everything). */
+void
+tor_free_all(int postfork)
+{
+  if (!postfork) {
+    evdns_shutdown(1);
+  }
+  geoip_free_all();
+  geoip_stats_free_all();
+  dirvote_free_all();
+  routerlist_free_all();
+  networkstatus_free_all();
+  addressmap_free_all();
+  dirserv_free_fingerprint_list();
+  dirserv_free_all();
+  dirserv_clear_measured_bw_cache();
+  rend_cache_free_all();
+  rend_service_authorization_free_all();
+  rep_hist_free_all();
+  dns_free_all();
+  clear_pending_onions();
+  circuit_free_all();
+  entry_guards_free_all();
+  pt_free_all();
+  channel_tls_free_all();
+  channel_free_all();
+  connection_free_all();
+  connection_edge_free_all();
+  scheduler_free_all();
+  nodelist_free_all();
+  microdesc_free_all();
+  routerparse_free_all();
+  ext_orport_free_all();
+  control_free_all();
+  tor_free_getaddrinfo_cache();
+  protover_free_all();
+  bridges_free_all();
+  consdiffmgr_free_all();
+  hs_free_all();
+  dos_free_all();
+  circuitmux_ewma_free_all();
+  accounting_free_all();
+
+  if (!postfork) {
+    config_free_all();
+    or_state_free_all();
+    router_free_all();
+    routerkeys_free_all();
+    policies_free_all();
+  }
+  if (!postfork) {
+    tor_tls_free_all();
+#ifndef _WIN32
+    tor_getpwnam(NULL);
+#endif
+  }
+  /* stuff in main.c */
+
+  tor_mainloop_free_all();
+
+  if (!postfork) {
+    release_lockfile();
+  }
+  tor_libevent_free_all();
+  /* Stuff in util.c and address.c*/
+  if (!postfork) {
+    escaped(NULL);
+    esc_router_info(NULL);
+    clean_up_backtrace_handler();
+    logs_free_all(); /* free log strings. do this last so logs keep working. */
+  }
+}
+
+/**
+ * Remove the specified file, and log a warning if the operation fails for
+ * any reason other than the file not existing. Ignores NULL filenames.
+ */
+void
+tor_remove_file(const char *filename)
+{
+  if (filename && tor_unlink(filename) != 0 && errno != ENOENT) {
+    log_warn(LD_FS, "Couldn't unlink %s: %s",
+               filename, strerror(errno));
+  }
+}
+
+/** Do whatever cleanup is necessary before shutting Tor down. */
+void
+tor_cleanup(void)
+{
+  const or_options_t *options = get_options();
+  if (options->command == CMD_RUN_TOR) {
+    time_t now = time(NULL);
+    /* Remove our pid file. We don't care if there was an error when we
+     * unlink, nothing we could do about it anyways. */
+    tor_remove_file(options->PidFile);
+    /* Remove control port file */
+    tor_remove_file(options->ControlPortWriteToFile);
+    /* Remove cookie authentication file */
+    {
+      char *cookie_fname = get_controller_cookie_file_name();
+      tor_remove_file(cookie_fname);
+      tor_free(cookie_fname);
+    }
+    /* Remove Extended ORPort cookie authentication file */
+    {
+      char *cookie_fname = get_ext_or_auth_cookie_file_name();
+      tor_remove_file(cookie_fname);
+      tor_free(cookie_fname);
+    }
+    if (accounting_is_enabled(options))
+      accounting_record_bandwidth_usage(now, get_or_state());
+    or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */
+    or_state_save(now);
+    if (authdir_mode(options)) {
+      sr_save_and_cleanup();
+    }
+    if (authdir_mode_tests_reachability(options))
+      rep_hist_record_mtbf_data(now, 0);
+    keypin_close_journal();
+  }
+
+  timers_shutdown();
+
+  tor_free_all(0); /* We could move tor_free_all back into the ifdef below
+                      later, if it makes shutdown unacceptably slow.  But for
+                      now, leave it here: it's helped us catch bugs in the
+                      past. */
+  crypto_global_cleanup();
+}
+
+/** Read/create keys as needed, and echo our fingerprint to stdout. */
+static int
+do_list_fingerprint(void)
+{
+  char buf[FINGERPRINT_LEN+1];
+  crypto_pk_t *k;
+  const char *nickname = get_options()->Nickname;
+  sandbox_disable_getaddrinfo_cache();
+  if (!server_mode(get_options())) {
+    log_err(LD_GENERAL,
+            "Clients don't have long-term identity keys. Exiting.");
+    return -1;
+  }
+  tor_assert(nickname);
+  if (init_keys() < 0) {
+    log_err(LD_GENERAL,"Error initializing keys; exiting.");
+    return -1;
+  }
+  if (!(k = get_server_identity_key())) {
+    log_err(LD_GENERAL,"Error: missing identity key.");
+    return -1;
+  }
+  if (crypto_pk_get_fingerprint(k, buf, 1)<0) {
+    log_err(LD_BUG, "Error computing fingerprint");
+    return -1;
+  }
+  printf("%s %s\n", nickname, buf);
+  return 0;
+}
+
+/** Entry point for password hashing: take the desired password from
+ * the command line, and print its salted hash to stdout. **/
+static void
+do_hash_password(void)
+{
+
+  char output[256];
+  char key[S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN];
+
+  crypto_rand(key, S2K_RFC2440_SPECIFIER_LEN-1);
+  key[S2K_RFC2440_SPECIFIER_LEN-1] = (uint8_t)96; /* Hash 64 K of data. */
+  secret_to_key_rfc2440(key+S2K_RFC2440_SPECIFIER_LEN, DIGEST_LEN,
+                get_options()->command_arg, strlen(get_options()->command_arg),
+                key);
+  base16_encode(output, sizeof(output), key, sizeof(key));
+  printf("16:%s\n",output);
+}
+
+/** Entry point for configuration dumping: write the configuration to
+ * stdout. */
+static int
+do_dump_config(void)
+{
+  const or_options_t *options = get_options();
+  const char *arg = options->command_arg;
+  int how;
+  char *opts;
+
+  if (!strcmp(arg, "short")) {
+    how = OPTIONS_DUMP_MINIMAL;
+  } else if (!strcmp(arg, "non-builtin")) {
+    how = OPTIONS_DUMP_DEFAULTS;
+  } else if (!strcmp(arg, "full")) {
+    how = OPTIONS_DUMP_ALL;
+  } else {
+    fprintf(stderr, "No valid argument to --dump-config found!\n");
+    fprintf(stderr, "Please select 'short', 'non-builtin', or 'full'.\n");
+
+    return -1;
+  }
+
+  opts = options_dump(options, how);
+  printf("%s", opts);
+  tor_free(opts);
+
+  return 0;
+}
+
+static void
+init_addrinfo(void)
+{
+  if (! server_mode(get_options()) ||
+      (get_options()->Address && strlen(get_options()->Address) > 0)) {
+    /* We don't need to seed our own hostname, because we won't be calling
+     * resolve_my_address on it.
+     */
+    return;
+  }
+  char hname[256];
+
+  // host name to sandbox
+  gethostname(hname, sizeof(hname));
+  tor_add_addrinfo(hname);
+}
+
+static sandbox_cfg_t*
+sandbox_init_filter(void)
+{
+  const or_options_t *options = get_options();
+  sandbox_cfg_t *cfg = sandbox_cfg_new();
+  int i;
+
+  sandbox_cfg_allow_openat_filename(&cfg,
+      get_cachedir_fname("cached-status"));
+
+#define OPEN(name)                              \
+  sandbox_cfg_allow_open_filename(&cfg, tor_strdup(name))
+
+#define OPEN_DATADIR(name)                      \
+  sandbox_cfg_allow_open_filename(&cfg, get_datadir_fname(name))
+
+#define OPEN_DATADIR2(name, name2)                       \
+  sandbox_cfg_allow_open_filename(&cfg, get_datadir_fname2((name), (name2)))
+
+#define OPEN_DATADIR_SUFFIX(name, suffix) do {  \
+    OPEN_DATADIR(name);                         \
+    OPEN_DATADIR(name suffix);                  \
+  } while (0)
+
+#define OPEN_DATADIR2_SUFFIX(name, name2, suffix) do {  \
+    OPEN_DATADIR2(name, name2);                         \
+    OPEN_DATADIR2(name, name2 suffix);                  \
+  } while (0)
+
+#define OPEN_KEY_DIRECTORY() \
+  sandbox_cfg_allow_open_filename(&cfg, tor_strdup(options->KeyDirectory))
+#define OPEN_CACHEDIR(name)                      \
+  sandbox_cfg_allow_open_filename(&cfg, get_cachedir_fname(name))
+#define OPEN_CACHEDIR_SUFFIX(name, suffix) do {  \
+    OPEN_CACHEDIR(name);                         \
+    OPEN_CACHEDIR(name suffix);                  \
+  } while (0)
+#define OPEN_KEYDIR(name)                      \
+  sandbox_cfg_allow_open_filename(&cfg, get_keydir_fname(name))
+#define OPEN_KEYDIR_SUFFIX(name, suffix) do {    \
+    OPEN_KEYDIR(name);                           \
+    OPEN_KEYDIR(name suffix);                    \
+  } while (0)
+
+  OPEN(options->DataDirectory);
+  OPEN_KEY_DIRECTORY();
+
+  OPEN_CACHEDIR_SUFFIX("cached-certs", ".tmp");
+  OPEN_CACHEDIR_SUFFIX("cached-consensus", ".tmp");
+  OPEN_CACHEDIR_SUFFIX("unverified-consensus", ".tmp");
+  OPEN_CACHEDIR_SUFFIX("unverified-microdesc-consensus", ".tmp");
+  OPEN_CACHEDIR_SUFFIX("cached-microdesc-consensus", ".tmp");
+  OPEN_CACHEDIR_SUFFIX("cached-microdescs", ".tmp");
+  OPEN_CACHEDIR_SUFFIX("cached-microdescs.new", ".tmp");
+  OPEN_CACHEDIR_SUFFIX("cached-descriptors", ".tmp");
+  OPEN_CACHEDIR_SUFFIX("cached-descriptors.new", ".tmp");
+  OPEN_CACHEDIR("cached-descriptors.tmp.tmp");
+  OPEN_CACHEDIR_SUFFIX("cached-extrainfo", ".tmp");
+  OPEN_CACHEDIR_SUFFIX("cached-extrainfo.new", ".tmp");
+  OPEN_CACHEDIR("cached-extrainfo.tmp.tmp");
+
+  OPEN_DATADIR_SUFFIX("state", ".tmp");
+  OPEN_DATADIR_SUFFIX("sr-state", ".tmp");
+  OPEN_DATADIR_SUFFIX("unparseable-desc", ".tmp");
+  OPEN_DATADIR_SUFFIX("v3-status-votes", ".tmp");
+  OPEN_DATADIR("key-pinning-journal");
+  OPEN("/dev/srandom");
+  OPEN("/dev/urandom");
+  OPEN("/dev/random");
+  OPEN("/etc/hosts");
+  OPEN("/proc/meminfo");
+
+  if (options->BridgeAuthoritativeDir)
+    OPEN_DATADIR_SUFFIX("networkstatus-bridges", ".tmp");
+
+  if (authdir_mode(options))
+    OPEN_DATADIR("approved-routers");
+
+  if (options->ServerDNSResolvConfFile)
+    sandbox_cfg_allow_open_filename(&cfg,
+                                tor_strdup(options->ServerDNSResolvConfFile));
+  else
+    sandbox_cfg_allow_open_filename(&cfg, tor_strdup("/etc/resolv.conf"));
+
+  for (i = 0; i < 2; ++i) {
+    if (get_torrc_fname(i)) {
+      sandbox_cfg_allow_open_filename(&cfg, tor_strdup(get_torrc_fname(i)));
+    }
+  }
+
+  SMARTLIST_FOREACH(options->FilesOpenedByIncludes, char *, f, {
+    OPEN(f);
+  });
+
+#define RENAME_SUFFIX(name, suffix)        \
+  sandbox_cfg_allow_rename(&cfg,           \
+      get_datadir_fname(name suffix),      \
+      get_datadir_fname(name))
+
+#define RENAME_SUFFIX2(prefix, name, suffix) \
+  sandbox_cfg_allow_rename(&cfg,                                        \
+                           get_datadir_fname2(prefix, name suffix),     \
+                           get_datadir_fname2(prefix, name))
+
+#define RENAME_CACHEDIR_SUFFIX(name, suffix)        \
+  sandbox_cfg_allow_rename(&cfg,           \
+      get_cachedir_fname(name suffix),      \
+      get_cachedir_fname(name))
+
+#define RENAME_KEYDIR_SUFFIX(name, suffix)    \
+  sandbox_cfg_allow_rename(&cfg,           \
+      get_keydir_fname(name suffix),      \
+      get_keydir_fname(name))
+
+  RENAME_CACHEDIR_SUFFIX("cached-certs", ".tmp");
+  RENAME_CACHEDIR_SUFFIX("cached-consensus", ".tmp");
+  RENAME_CACHEDIR_SUFFIX("unverified-consensus", ".tmp");
+  RENAME_CACHEDIR_SUFFIX("unverified-microdesc-consensus", ".tmp");
+  RENAME_CACHEDIR_SUFFIX("cached-microdesc-consensus", ".tmp");
+  RENAME_CACHEDIR_SUFFIX("cached-microdescs", ".tmp");
+  RENAME_CACHEDIR_SUFFIX("cached-microdescs", ".new");
+  RENAME_CACHEDIR_SUFFIX("cached-microdescs.new", ".tmp");
+  RENAME_CACHEDIR_SUFFIX("cached-descriptors", ".tmp");
+  RENAME_CACHEDIR_SUFFIX("cached-descriptors", ".new");
+  RENAME_CACHEDIR_SUFFIX("cached-descriptors.new", ".tmp");
+  RENAME_CACHEDIR_SUFFIX("cached-extrainfo", ".tmp");
+  RENAME_CACHEDIR_SUFFIX("cached-extrainfo", ".new");
+  RENAME_CACHEDIR_SUFFIX("cached-extrainfo.new", ".tmp");
+
+  RENAME_SUFFIX("state", ".tmp");
+  RENAME_SUFFIX("sr-state", ".tmp");
+  RENAME_SUFFIX("unparseable-desc", ".tmp");
+  RENAME_SUFFIX("v3-status-votes", ".tmp");
+
+  if (options->BridgeAuthoritativeDir)
+    RENAME_SUFFIX("networkstatus-bridges", ".tmp");
+
+#define STAT_DATADIR(name)                      \
+  sandbox_cfg_allow_stat_filename(&cfg, get_datadir_fname(name))
+
+#define STAT_CACHEDIR(name)                                             \
+  sandbox_cfg_allow_stat_filename(&cfg, get_cachedir_fname(name))
+
+#define STAT_DATADIR2(name, name2)                                      \
+  sandbox_cfg_allow_stat_filename(&cfg, get_datadir_fname2((name), (name2)))
+
+#define STAT_KEY_DIRECTORY() \
+  sandbox_cfg_allow_stat_filename(&cfg, tor_strdup(options->KeyDirectory))
+
+  STAT_DATADIR(NULL);
+  STAT_DATADIR("lock");
+  STAT_DATADIR("state");
+  STAT_DATADIR("router-stability");
+
+  STAT_CACHEDIR("cached-extrainfo.new");
+
+  {
+    smartlist_t *files = smartlist_new();
+    tor_log_get_logfile_names(files);
+    SMARTLIST_FOREACH(files, char *, file_name, {
+      /* steals reference */
+      sandbox_cfg_allow_open_filename(&cfg, file_name);
+    });
+    smartlist_free(files);
+  }
+
+  {
+    smartlist_t *files = smartlist_new();
+    smartlist_t *dirs = smartlist_new();
+    hs_service_lists_fnames_for_sandbox(files, dirs);
+    SMARTLIST_FOREACH(files, char *, file_name, {
+      char *tmp_name = NULL;
+      tor_asprintf(&tmp_name, "%s.tmp", file_name);
+      sandbox_cfg_allow_rename(&cfg,
+                               tor_strdup(tmp_name), tor_strdup(file_name));
+      /* steals references */
+      sandbox_cfg_allow_open_filename(&cfg, file_name);
+      sandbox_cfg_allow_open_filename(&cfg, tmp_name);
+    });
+    SMARTLIST_FOREACH(dirs, char *, dir, {
+      /* steals reference */
+      sandbox_cfg_allow_stat_filename(&cfg, dir);
+    });
+    smartlist_free(files);
+    smartlist_free(dirs);
+  }
+
+  {
+    char *fname;
+    if ((fname = get_controller_cookie_file_name())) {
+      sandbox_cfg_allow_open_filename(&cfg, fname);
+    }
+    if ((fname = get_ext_or_auth_cookie_file_name())) {
+      sandbox_cfg_allow_open_filename(&cfg, fname);
+    }
+  }
+
+  SMARTLIST_FOREACH_BEGIN(get_configured_ports(), port_cfg_t *, port) {
+    if (!port->is_unix_addr)
+      continue;
+    /* When we open an AF_UNIX address, we want permission to open the
+     * directory that holds it. */
+    char *dirname = tor_strdup(port->unix_addr);
+    if (get_parent_directory(dirname) == 0) {
+      OPEN(dirname);
+    }
+    tor_free(dirname);
+    sandbox_cfg_allow_chmod_filename(&cfg, tor_strdup(port->unix_addr));
+    sandbox_cfg_allow_chown_filename(&cfg, tor_strdup(port->unix_addr));
+  } SMARTLIST_FOREACH_END(port);
+
+  if (options->DirPortFrontPage) {
+    sandbox_cfg_allow_open_filename(&cfg,
+                                    tor_strdup(options->DirPortFrontPage));
+  }
+
+  // orport
+  if (server_mode(get_options())) {
+
+    OPEN_KEYDIR_SUFFIX("secret_id_key", ".tmp");
+    OPEN_KEYDIR_SUFFIX("secret_onion_key", ".tmp");
+    OPEN_KEYDIR_SUFFIX("secret_onion_key_ntor", ".tmp");
+    OPEN_KEYDIR("secret_id_key.old");
+    OPEN_KEYDIR("secret_onion_key.old");
+    OPEN_KEYDIR("secret_onion_key_ntor.old");
+
+    OPEN_KEYDIR_SUFFIX("ed25519_master_id_secret_key", ".tmp");
+    OPEN_KEYDIR_SUFFIX("ed25519_master_id_secret_key_encrypted", ".tmp");
+    OPEN_KEYDIR_SUFFIX("ed25519_master_id_public_key", ".tmp");
+    OPEN_KEYDIR_SUFFIX("ed25519_signing_secret_key", ".tmp");
+    OPEN_KEYDIR_SUFFIX("ed25519_signing_secret_key_encrypted", ".tmp");
+    OPEN_KEYDIR_SUFFIX("ed25519_signing_public_key", ".tmp");
+    OPEN_KEYDIR_SUFFIX("ed25519_signing_cert", ".tmp");
+
+    OPEN_DATADIR2_SUFFIX("stats", "bridge-stats", ".tmp");
+    OPEN_DATADIR2_SUFFIX("stats", "dirreq-stats", ".tmp");
+
+    OPEN_DATADIR2_SUFFIX("stats", "entry-stats", ".tmp");
+    OPEN_DATADIR2_SUFFIX("stats", "exit-stats", ".tmp");
+    OPEN_DATADIR2_SUFFIX("stats", "buffer-stats", ".tmp");
+    OPEN_DATADIR2_SUFFIX("stats", "conn-stats", ".tmp");
+    OPEN_DATADIR2_SUFFIX("stats", "hidserv-stats", ".tmp");
+
+    OPEN_DATADIR("approved-routers");
+    OPEN_DATADIR_SUFFIX("fingerprint", ".tmp");
+    OPEN_DATADIR_SUFFIX("hashed-fingerprint", ".tmp");
+    OPEN_DATADIR_SUFFIX("router-stability", ".tmp");
+
+    OPEN("/etc/resolv.conf");
+
+    RENAME_SUFFIX("fingerprint", ".tmp");
+    RENAME_KEYDIR_SUFFIX("secret_onion_key_ntor", ".tmp");
+
+    RENAME_KEYDIR_SUFFIX("secret_id_key", ".tmp");
+    RENAME_KEYDIR_SUFFIX("secret_id_key.old", ".tmp");
+    RENAME_KEYDIR_SUFFIX("secret_onion_key", ".tmp");
+    RENAME_KEYDIR_SUFFIX("secret_onion_key.old", ".tmp");
+
+    RENAME_SUFFIX2("stats", "bridge-stats", ".tmp");
+    RENAME_SUFFIX2("stats", "dirreq-stats", ".tmp");
+    RENAME_SUFFIX2("stats", "entry-stats", ".tmp");
+    RENAME_SUFFIX2("stats", "exit-stats", ".tmp");
+    RENAME_SUFFIX2("stats", "buffer-stats", ".tmp");
+    RENAME_SUFFIX2("stats", "conn-stats", ".tmp");
+    RENAME_SUFFIX2("stats", "hidserv-stats", ".tmp");
+    RENAME_SUFFIX("hashed-fingerprint", ".tmp");
+    RENAME_SUFFIX("router-stability", ".tmp");
+
+    RENAME_KEYDIR_SUFFIX("ed25519_master_id_secret_key", ".tmp");
+    RENAME_KEYDIR_SUFFIX("ed25519_master_id_secret_key_encrypted", ".tmp");
+    RENAME_KEYDIR_SUFFIX("ed25519_master_id_public_key", ".tmp");
+    RENAME_KEYDIR_SUFFIX("ed25519_signing_secret_key", ".tmp");
+    RENAME_KEYDIR_SUFFIX("ed25519_signing_cert", ".tmp");
+
+    sandbox_cfg_allow_rename(&cfg,
+             get_keydir_fname("secret_onion_key"),
+             get_keydir_fname("secret_onion_key.old"));
+    sandbox_cfg_allow_rename(&cfg,
+             get_keydir_fname("secret_onion_key_ntor"),
+             get_keydir_fname("secret_onion_key_ntor.old"));
+
+    STAT_KEY_DIRECTORY();
+    OPEN_DATADIR("stats");
+    STAT_DATADIR("stats");
+    STAT_DATADIR2("stats", "dirreq-stats");
+
+    consdiffmgr_register_with_sandbox(&cfg);
+  }
+
+  init_addrinfo();
+
+  return cfg;
+}
+
+int
+run_tor_main_loop(void)
+{
+  handle_signals();
+  monotime_init();
+  timers_initialize();
+  initialize_mainloop_events();
+
+  /* load the private keys, if we're supposed to have them, and set up the
+   * TLS context. */
+  if (! client_identity_key_is_set()) {
+    if (init_keys() < 0) {
+      log_err(LD_OR, "Error initializing keys; exiting");
+      return -1;
+    }
+  }
+
+  /* Set up our buckets */
+  connection_bucket_init();
+
+  /* initialize the bootstrap status events to know we're starting up */
+  control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0);
+
+  /* Initialize the keypinning log. */
+  if (authdir_mode_v3(get_options())) {
+    char *fname = get_datadir_fname("key-pinning-journal");
+    int r = 0;
+    if (keypin_load_journal(fname)<0) {
+      log_err(LD_DIR, "Error loading key-pinning journal: %s",strerror(errno));
+      r = -1;
+    }
+    if (keypin_open_journal(fname)<0) {
+      log_err(LD_DIR, "Error opening key-pinning journal: %s",strerror(errno));
+      r = -1;
+    }
+    tor_free(fname);
+    if (r)
+      return r;
+  }
+  {
+    /* This is the old name for key-pinning-journal.  These got corrupted
+     * in a couple of cases by #16530, so we started over. See #16580 for
+     * the rationale and for other options we didn't take.  We can remove
+     * this code once all the authorities that ran 0.2.7.1-alpha-dev are
+     * upgraded.
+     */
+    char *fname = get_datadir_fname("key-pinning-entries");
+    unlink(fname);
+    tor_free(fname);
+  }
+
+  if (trusted_dirs_reload_certs()) {
+    log_warn(LD_DIR,
+             "Couldn't load all cached v3 certificates. Starting anyway.");
+  }
+  if (router_reload_consensus_networkstatus()) {
+    return -1;
+  }
+  /* load the routers file, or assign the defaults. */
+  if (router_reload_router_list()) {
+    return -1;
+  }
+  /* load the networkstatuses. (This launches a download for new routers as
+   * appropriate.)
+   */
+  const time_t now = time(NULL);
+  directory_info_has_arrived(now, 1, 0);
+
+  if (server_mode(get_options()) || dir_server_mode(get_options())) {
+    /* launch cpuworkers. Need to do this *after* we've read the onion key. */
+    cpu_init();
+  }
+  consdiffmgr_enable_background_compression();
+
+  /* Setup shared random protocol subsystem. */
+  if (authdir_mode_v3(get_options())) {
+    if (sr_init(1) < 0) {
+      return -1;
+    }
+  }
+
+  /* initialize dns resolve map, spawn workers if needed */
+  if (dns_init() < 0) {
+    if (get_options()->ServerDNSAllowBrokenConfig)
+      log_warn(LD_GENERAL, "Couldn't set up any working nameservers. "
+               "Network not up yet?  Will try again soon.");
+    else {
+      log_err(LD_GENERAL,"Error initializing dns subsystem; exiting.  To "
+              "retry instead, set the ServerDNSAllowBrokenResolvConf option.");
+    }
+  }
+
+#ifdef HAVE_SYSTEMD
+  {
+    const int r = sd_notify(0, "READY=1");
+    if (r < 0) {
+      log_warn(LD_GENERAL, "Unable to send readiness to systemd: %s",
+               strerror(r));
+    } else if (r > 0) {
+      log_notice(LD_GENERAL, "Signaled readiness to systemd");
+    } else {
+      log_info(LD_GENERAL, "Systemd NOTIFY_SOCKET not present.");
+    }
+  }
+#endif /* defined(HAVE_SYSTEMD) */
+
+  return do_main_loop();
+}
+
+/* Main entry point for the Tor process.  Called from tor_main(), and by
+ * anybody embedding Tor. */
+int
+tor_run_main(const tor_main_configuration_t *tor_cfg)
+{
+  int result = 0;
+
+#ifdef _WIN32
+#ifndef HeapEnableTerminationOnCorruption
+#define HeapEnableTerminationOnCorruption 1
+#endif
+  /* On heap corruption, just give up; don't try to play along. */
+  HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
+
+  /* SetProcessDEPPolicy is only supported on 32-bit Windows.
+   * (On 64-bit Windows it always fails, and some compilers don't like the
+   * PSETDEP cast.)
+   * 32-bit Windows defines _WIN32.
+   * 64-bit Windows defines _WIN32 and _WIN64. */
+#ifndef _WIN64
+  /* Call SetProcessDEPPolicy to permanently enable DEP.
+     The function will not resolve on earlier versions of Windows,
+     and failure is not dangerous. */
+  HMODULE hMod = GetModuleHandleA("Kernel32.dll");
+  if (hMod) {
+    typedef BOOL (WINAPI *PSETDEP)(DWORD);
+    PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod,
+                           "SetProcessDEPPolicy");
+    if (setdeppolicy) {
+      /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */
+      setdeppolicy(3);
+    }
+  }
+#endif /* !defined(_WIN64) */
+#endif /* defined(_WIN32) */
+
+  {
+    int bt_err = configure_backtrace_handler(get_version());
+    if (bt_err < 0) {
+      log_warn(LD_BUG, "Unable to install backtrace handler: %s",
+               strerror(-bt_err));
+    }
+  }
+
+#ifdef EVENT_SET_MEM_FUNCTIONS_IMPLEMENTED
+  event_set_mem_functions(tor_malloc_, tor_realloc_, tor_free_);
+#endif
+
+  init_protocol_warning_severity_level();
+
+  update_approx_time(time(NULL));
+  tor_threads_init();
+  tor_compress_init();
+  init_logging(0);
+  monotime_init();
+
+  int argc = tor_cfg->argc + tor_cfg->argc_owned;
+  char **argv = tor_calloc(argc, sizeof(char*));
+  memcpy(argv, tor_cfg->argv, tor_cfg->argc*sizeof(char*));
+  if (tor_cfg->argc_owned)
+    memcpy(argv + tor_cfg->argc, tor_cfg->argv_owned,
+           tor_cfg->argc_owned*sizeof(char*));
+
+#ifdef NT_SERVICE
+  {
+     int done = 0;
+     result = nt_service_parse_options(argc, argv, &done);
+     if (done) {
+       goto done;
+     }
+  }
+#endif /* defined(NT_SERVICE) */
+  {
+    int init_rv = tor_init(argc, argv);
+    if (init_rv) {
+      tor_free_all(0);
+      result = (init_rv < 0) ? -1 : 0;
+      goto done;
+    }
+  }
+
+  if (get_options()->Sandbox && get_options()->command == CMD_RUN_TOR) {
+    sandbox_cfg_t* cfg = sandbox_init_filter();
+
+    if (sandbox_init(cfg)) {
+      tor_free(argv);
+      log_err(LD_BUG,"Failed to create syscall sandbox filter");
+      tor_free_all(0);
+      return -1;
+    }
+
+    // registering libevent rng
+#ifdef HAVE_EVUTIL_SECURE_RNG_SET_URANDOM_DEVICE_FILE
+    evutil_secure_rng_set_urandom_device_file(
+        (char*) sandbox_intern_string("/dev/urandom"));
+#endif
+  }
+
+  switch (get_options()->command) {
+  case CMD_RUN_TOR:
+#ifdef NT_SERVICE
+    nt_service_set_state(SERVICE_RUNNING);
+#endif
+    result = run_tor_main_loop();
+    break;
+  case CMD_KEYGEN:
+    result = load_ed_keys(get_options(), time(NULL)) < 0;
+    break;
+  case CMD_KEY_EXPIRATION:
+    init_keys();
+    result = log_cert_expiration();
+    break;
+  case CMD_LIST_FINGERPRINT:
+    result = do_list_fingerprint();
+    break;
+  case CMD_HASH_PASSWORD:
+    do_hash_password();
+    result = 0;
+    break;
+  case CMD_VERIFY_CONFIG:
+    if (quiet_level == 0)
+      printf("Configuration was valid\n");
+    result = 0;
+    break;
+  case CMD_DUMP_CONFIG:
+    result = do_dump_config();
+    break;
+  case CMD_RUN_UNITTESTS: /* only set by test.c */
+  default:
+    log_warn(LD_BUG,"Illegal command number %d: internal error.",
+             get_options()->command);
+    result = -1;
+  }
+  tor_cleanup();
+ done:
+  tor_free(argv);
+  return result;
+}
diff --git a/src/app/main/main.h b/src/app/main/main.h
new file mode 100644
index 0000000000..bbbbf984fb
--- /dev/null
+++ b/src/app/main/main.h
@@ -0,0 +1,31 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file main.h
+ * \brief Header file for main.c.
+ **/
+
+#ifndef TOR_MAIN_H
+#define TOR_MAIN_H
+
+void handle_signals(void);
+void activate_signal(int signal_num);
+
+int try_locking(const or_options_t *options, int err_if_locked);
+int have_lockfile(void);
+void release_lockfile(void);
+
+void tor_remove_file(const char *filename);
+
+void tor_cleanup(void);
+void tor_free_all(int postfork);
+
+int tor_init(int argc, char **argv);
+
+int run_tor_main_loop(void);
+
+#endif /* !defined(TOR_MAIN_H) */
diff --git a/src/or/ntmain.c b/src/app/main/ntmain.c
index 0e6f296d24..05d203b0be 100644
--- a/src/or/ntmain.c
+++ b/src/app/main/ntmain.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,12 +19,15 @@
 
 #ifdef _WIN32
 
-#include "or.h"
-#include "config.h"
-#include "main.h"
-#include "ntmain.h"
+#include "core/or/or.h"
 
-#include <event2/event.h>
+#include "app/config/config.h"
+#include "app/main/main.h"
+#include "app/main/ntmain.h"
+#include "core/mainloop/mainloop.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/fs/winlib.h"
+#include "lib/log/win32err.h"
 
 #include <windows.h>
 #define GENSRV_SERVICENAME  "tor"
@@ -195,7 +198,7 @@ nt_service_loadlibrary(void)
   return;
  err:
   printf("Unable to load library support for NT services: exiting.\n");
-  exit(1);
+  exit(1); // exit ok: ntmain can't read libraries
 }
 
 /** If we're compiled to run as an NT service, and the service wants to
@@ -245,7 +248,8 @@ nt_service_control(DWORD request)
           log_notice(LD_GENERAL,
                      "Got stop/shutdown request; shutting down cleanly.");
           service_status.dwCurrentState = SERVICE_STOP_PENDING;
-          event_base_loopexit(tor_libevent_get_base(), &exit_now);
+          tor_libevent_exit_loop_after_delay(tor_libevent_get_base(),
+                                             &exit_now);
           return;
   }
   service_fns.SetServiceStatus_fn(hStatus, &service_status);
@@ -294,7 +298,7 @@ nt_service_body(int argc, char **argv)
   service_status.dwCurrentState = SERVICE_RUNNING;
   service_fns.SetServiceStatus_fn(hStatus, &service_status);
   set_main_thread();
-  do_main_loop();
+  run_tor_main_loop();
   tor_cleanup();
 }
 
@@ -318,20 +322,21 @@ nt_service_main(void)
     printf("Service error %d : %s\n", (int) result, errmsg);
     tor_free(errmsg);
     if (result == ERROR_FAILED_SERVICE_CONTROLLER_CONNECT) {
-      if (tor_init(backup_argc, backup_argv) < 0)
+      if (tor_init(backup_argc, backup_argv))
         return;
       switch (get_options()->command) {
       case CMD_RUN_TOR:
-        do_main_loop();
+        run_tor_main_loop();
         break;
       case CMD_LIST_FINGERPRINT:
       case CMD_HASH_PASSWORD:
       case CMD_VERIFY_CONFIG:
       case CMD_DUMP_CONFIG:
       case CMD_KEYGEN:
+      case CMD_KEY_EXPIRATION:
         log_err(LD_CONFIG, "Unsupported command (--list-fingerprint, "
-               "--hash-password, --keygen, --dump-config, or --verify-config) "
-                "in NT service.");
+               "--hash-password, --keygen, --dump-config, --verify-config, "
+               "or --key-expiration) in NT service.");
         break;
       case CMD_RUN_UNITTESTS:
       default:
@@ -501,7 +506,7 @@ nt_service_command_line(int *using_default_torrc)
   tor_exe_ascii[sizeof(tor_exe_ascii)-1] = '\0';
 #else
   strlcpy(tor_exe_ascii, tor_exe, sizeof(tor_exe_ascii));
-#endif
+#endif /* defined(UNICODE) */
 
   /* Allocate a string for the NT service command line and */
   /* Format the service command */
@@ -777,5 +782,4 @@ nt_service_parse_options(int argc, char **argv, int *should_exit)
   return 0;
 }
 
-#endif
-
+#endif /* defined(_WIN32) */
diff --git a/src/or/ntmain.h b/src/app/main/ntmain.h
index 31bf38c62c..c39386c054 100644
--- a/src/or/ntmain.h
+++ b/src/app/main/ntmain.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,7 +22,7 @@ int nt_service_is_stopping(void);
 void nt_service_set_state(DWORD state);
 #else
 #define nt_service_is_stopping() 0
-#endif
+#endif /* defined(NT_SERVICE) */
 
-#endif
+#endif /* !defined(TOR_NTMAIN_H) */
 
diff --git a/src/or/tor_main.c b/src/app/main/tor_main.c
index d67eda2ac9..8a887ed269 100644
--- a/src/or/tor_main.c
+++ b/src/app/main/tor_main.c
@@ -1,19 +1,12 @@
 /* Copyright 2001-2004 Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-extern const char tor_git_revision[];
-
-/** String describing which Tor Git repository version the source was
- * built from.  This string is generated by a bit of shell kludging in
- * src/or/include.am, and is usually right.
- */
-const char tor_git_revision[] =
-#ifndef _MSC_VER
-#include "micro-revision.i"
+#include "orconfig.h"
+#ifdef ENABLE_RESTART_DEBUGGING
+#include <stdlib.h>
 #endif
-  "";
 
 /**
  * \file tor_main.c
@@ -26,14 +19,23 @@ const char tor_git_revision[] =
 int tor_main(int argc, char *argv[]);
 
 /** We keep main() in a separate file so that our unit tests can use
- * functions from main.c)
+ * functions from main.c.
  */
 int
 main(int argc, char *argv[])
 {
-  int r = tor_main(argc, argv);
+  int r;
+#ifdef ENABLE_RESTART_DEBUGGING
+  int restart_count = getenv("TOR_DEBUG_RESTART") ? 1 : 0;
+ again:
+#endif
+  r = tor_main(argc, argv);
   if (r < 0 || r > 255)
     return 1;
+#ifdef ENABLE_RESTART_DEBUGGING
+  else if (r == 0 && restart_count--)
+    goto again;
+#endif
   else
     return r;
 }
diff --git a/src/common/Makefile.nmake b/src/common/Makefile.nmake
deleted file mode 100644
index b8c5dd4fea..0000000000
--- a/src/common/Makefile.nmake
+++ /dev/null
@@ -1,28 +0,0 @@
-all: libor.lib libor-crypto.lib libor-event.lib
-
-CFLAGS = /O2 /MT /I ..\win32 /I ..\..\..\build-alpha\include /I ..\common \
-    /I ..\ext
-
-LIBOR_OBJECTS = address.obj backtrace.obj compat.obj container.obj di_ops.obj \
-	log.obj memarea.obj mempool.obj procmon.obj sandbox.obj util.obj \
-	util_codedigest.obj
-
-LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj torgzip.obj tortls.obj \
-	crypto_curve25519.obj curve25519-donna.obj
-
-LIBOR_EVENT_OBJECTS = compat_libevent.obj
-
-curve25519-donna.obj: ..\ext\curve25519_donna\curve25519-donna.c
-	$(CC) $(CFLAGS) /D inline=_inline /c ..\ext\curve25519_donna\curve25519-donna.c
-
-libor.lib: $(LIBOR_OBJECTS)
-	lib $(LIBOR_OBJECTS) /out:libor.lib
-
-libor-crypto.lib: $(LIBOR_CRYPTO_OBJECTS)
-	lib $(LIBOR_CRYPTO_OBJECTS) /out:libor-crypto.lib
-
-libor-event.lib:  $(LIBOR_EVENT_OBJECTS)
-	lib $(LIBOR_EVENT_OBJECTS) /out:libor-event.lib
-
-clean:
-	del *.obj *.lib libor*.lib
diff --git a/src/common/address_set.c b/src/common/address_set.c
deleted file mode 100644
index 4924cb65c2..0000000000
--- a/src/common/address_set.c
+++ /dev/null
@@ -1,129 +0,0 @@
-/* Copyright (c) 2018, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file address_set.c
- * \brief Implementation for a set of addresses.
- *
- * This module was first written on a semi-emergency basis to improve the
- * robustness of the anti-DoS module.  As such, it's written in a pretty
- * conservative way, and should be susceptible to improvement later on.
- **/
-
-#include "orconfig.h"
-#include "address_set.h"
-#include "address.h"
-#include "compat.h"
-#include "container.h"
-#include "crypto.h"
-#include "util.h"
-#include "siphash.h"
-
-/** How many 64-bit siphash values to extract per address */
-#define N_HASHES 2
-/** How many bloom-filter bits we set per address. This is twice the N_HASHES
- * value, since we split the siphash output into two 32-bit values. */
-#define N_BITS_PER_ITEM (N_HASHES * 2)
-
-/* XXXX This code is largely duplicated with digestset_t.  We should merge
- * them together into a common bloom-filter implementation.  I'm keeping
- * them separate for now, though, since this module needs to be backported
- * all the way to 0.2.9.
- *
- * The main difference between digestset_t and this code is that we use
- * independent siphashes rather than messing around with bit-shifts.  The
- * approach here is probably more sound, and we should prefer it if&when we
- * unify the implementations.
- **/
-
-struct address_set_t {
-  /** siphash keys to make N_HASHES independent hashes for each address. */
-  struct sipkey key[N_HASHES];
-  int mask; /**< One less than the number of bits in <b>ba</b>; always one less
-             * than a power of two. */
-  bitarray_t *ba; /**< A bit array to implement the Bloom filter. */
-};
-
-/**
- * Allocate and return an address_set, suitable for holding up to
- * <b>max_address_guess</b> distinct values.
- */
-address_set_t *
-address_set_new(int max_addresses_guess)
-{
-  /* See digestset_new() for rationale on this equation. */
-  int n_bits = 1u << (tor_log2(max_addresses_guess)+5);
-
-  address_set_t *set = tor_malloc_zero(sizeof(address_set_t));
-  set->mask = n_bits - 1;
-  set->ba = bitarray_init_zero(n_bits);
-  crypto_rand((char*) set->key, sizeof(set->key));
-
-  return set;
-}
-
-/**
- * Release all storage associated with <b>set</b>
- */
-void
-address_set_free(address_set_t *set)
-{
-  if (! set)
-    return;
-
-  bitarray_free(set->ba);
-  tor_free(set);
-}
-
-/** Yield the bit index corresponding to 'val' for set. */
-#define BIT(set, val) ((val) & (set)->mask)
-
-/**
- * Add <b>addr</b> to <b>set</b>.
- *
- * All future queries for <b>addr</b> in set will return true. Removing
- * items is not possible.
- */
-void
-address_set_add(address_set_t *set, const struct tor_addr_t *addr)
-{
-  int i;
-  for (i = 0; i < N_HASHES; ++i) {
-    uint64_t h = tor_addr_keyed_hash(&set->key[i], addr);
-    uint32_t high_bits = (uint32_t)(h >> 32);
-    uint32_t low_bits = (uint32_t)(h);
-    bitarray_set(set->ba, BIT(set, high_bits));
-    bitarray_set(set->ba, BIT(set, low_bits));
-  }
-}
-
-/** As address_set_add(), but take an ipv4 address in host order. */
-void
-address_set_add_ipv4h(address_set_t *set, uint32_t addr)
-{
-  tor_addr_t a;
-  tor_addr_from_ipv4h(&a, addr);
-  address_set_add(set, &a);
-}
-
-/**
- * Return true if <b>addr</b> if a member of <b>set</b>.  (And probably,
- * return false if <b>addr</b> is not a member of set.)
- */
-int
-address_set_probably_contains(address_set_t *set,
-                              const struct tor_addr_t *addr)
-{
-  int i, matches = 0;
-  for (i = 0; i < N_HASHES; ++i) {
-    uint64_t h = tor_addr_keyed_hash(&set->key[i], addr);
-    uint32_t high_bits = (uint32_t)(h >> 32);
-    uint32_t low_bits = (uint32_t)(h);
-    // Note that !! is necessary here, since bitarray_is_set does not
-    // necessarily return 1 on true.
-    matches += !! bitarray_is_set(set->ba, BIT(set, high_bits));
-    matches += !! bitarray_is_set(set->ba, BIT(set, low_bits));
-  }
-  return matches == N_BITS_PER_ITEM;
-}
-
diff --git a/src/common/backtrace.h b/src/common/backtrace.h
deleted file mode 100644
index b53fd2c668..0000000000
--- a/src/common/backtrace.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#ifndef TOR_BACKTRACE_H
-#define TOR_BACKTRACE_H
-
-#include "orconfig.h"
-
-void log_backtrace(int severity, int domain, const char *msg);
-int configure_backtrace_handler(const char *tor_version);
-void clean_up_backtrace_handler(void);
-
-#ifdef EXPOSE_CLEAN_BACKTRACE
-#if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \
-  defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION)
-void clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx);
-#endif
-#endif
-
-#endif
-
diff --git a/src/common/compat.c b/src/common/compat.c
deleted file mode 100644
index 4d4a81e1c1..0000000000
--- a/src/common/compat.c
+++ /dev/null
@@ -1,3557 +0,0 @@
-/* Copyright (c) 2003-2004, Roger Dingledine
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file compat.c
- * \brief Wrappers to make calls more portable.  This code defines
- * functions such as tor_snprintf, get/set various data types,
- * renaming, setting socket options, switching user IDs.  It is basically
- * where the non-portable items are conditionally included depending on
- * the platform.
- **/
-
-#define COMPAT_PRIVATE
-#include "compat.h"
-
-#ifdef _WIN32
-#include <winsock2.h>
-#include <windows.h>
-#include <sys/locking.h>
-#endif
-
-#ifdef HAVE_UNAME
-#include <sys/utsname.h>
-#endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
-#endif
-#ifdef HAVE_UTIME_H
-#include <utime.h>
-#endif
-#ifdef HAVE_SYS_UTIME_H
-#include <sys/utime.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SYS_FCNTL_H
-#include <sys/fcntl.h>
-#endif
-#ifdef HAVE_PWD_H
-#include <pwd.h>
-#endif
-#ifdef HAVE_GRP_H
-#include <grp.h>
-#endif
-#ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif
-#ifdef HAVE_ERRNO_H
-#include <errno.h>
-#endif
-#ifdef HAVE_ARPA_INET_H
-#include <arpa/inet.h>
-#endif
-#ifdef HAVE_CRT_EXTERNS_H
-#include <crt_externs.h>
-#endif
-#ifdef HAVE_SYS_STATVFS_H
-#include <sys/statvfs.h>
-#endif
-#ifdef HAVE_SYS_CAPABILITY_H
-#include <sys/capability.h>
-#endif
-
-/* Now deprecated in Linux GLIBC */
-#if defined(HAVE_SYS_SYSCTL_H) && !defined(_WIN32) && !defined(__linux__)
-#include <sys/sysctl.h>
-#endif
-
-#ifdef _WIN32
-#include <conio.h>
-#include <wchar.h>
-/* Some mingw headers lack these. :p */
-#if defined(HAVE_DECL__GETWCH) && !HAVE_DECL__GETWCH
-wint_t _getwch(void);
-#endif
-#ifndef WEOF
-#define WEOF (wchar_t)(0xFFFF)
-#endif
-#if defined(HAVE_DECL_SECUREZEROMEMORY) && !HAVE_DECL_SECUREZEROMEMORY
-static inline void
-SecureZeroMemory(PVOID ptr, SIZE_T cnt)
-{
-  volatile char *vcptr = (volatile char*)ptr;
-  while (cnt--)
-    *vcptr++ = 0;
-}
-#endif
-#elif defined(HAVE_READPASSPHRASE_H)
-#include <readpassphrase.h>
-#else
-#include "tor_readpassphrase.h"
-#endif
-
-/* Includes for the process attaching prevention */
-#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__)
-/* Only use the linux prctl;  the IRIX prctl is totally different */
-#include <sys/prctl.h>
-#elif defined(__APPLE__)
-#include <sys/types.h>
-#include <sys/ptrace.h>
-#endif
-
-#ifdef HAVE_NETDB_H
-#include <netdb.h>
-#endif
-#ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h> /* FreeBSD needs this to know what version it is */
-#endif
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#ifdef HAVE_SIGNAL_H
-#include <signal.h>
-#endif
-#ifdef HAVE_SYS_MMAN_H
-#include <sys/mman.h>
-#endif
-#ifdef HAVE_SYS_SYSLIMITS_H
-#include <sys/syslimits.h>
-#endif
-#ifdef HAVE_SYS_FILE_H
-#include <sys/file.h>
-#endif
-
-#include "torlog.h"
-#include "util.h"
-#include "container.h"
-#include "address.h"
-#include "sandbox.h"
-
-/* Inline the strl functions if the platform doesn't have them. */
-#ifndef HAVE_STRLCPY
-#include "strlcpy.c"
-#endif
-#ifndef HAVE_STRLCAT
-#include "strlcat.c"
-#endif
-
-/* When set_max_file_descriptors() is called, update this with the max file
- * descriptor value so we can use it to check the limit when opening a new
- * socket. Default value is what Debian sets as the default hard limit. */
-static int max_sockets = 1024;
-
-/** As open(path, flags, mode), but return an fd with the close-on-exec mode
- * set. */
-int
-tor_open_cloexec(const char *path, int flags, unsigned mode)
-{
-  int fd;
-  const char *p = sandbox_intern_string(path);
-#ifdef O_CLOEXEC
-  fd = open(p, flags|O_CLOEXEC, mode);
-  if (fd >= 0)
-    return fd;
-  /* If we got an error, see if it is EINVAL. EINVAL might indicate that,
-   * even though we were built on a system with O_CLOEXEC support, we
-   * are running on one without. */
-  if (errno != EINVAL)
-    return -1;
-#endif
-
-  log_debug(LD_FS, "Opening %s with flags %x", p, flags);
-  fd = open(p, flags, mode);
-#ifdef FD_CLOEXEC
-  if (fd >= 0) {
-    if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1) {
-      log_warn(LD_FS,"Couldn't set FD_CLOEXEC: %s", strerror(errno));
-      close(fd);
-      return -1;
-    }
-  }
-#endif
-  return fd;
-}
-
-/** As fopen(path,mode), but ensures that the O_CLOEXEC bit is set on the
- * underlying file handle. */
-FILE *
-tor_fopen_cloexec(const char *path, const char *mode)
-{
-  FILE *result = fopen(path, mode);
-#ifdef FD_CLOEXEC
-  if (result != NULL) {
-    if (fcntl(fileno(result), F_SETFD, FD_CLOEXEC) == -1) {
-      log_warn(LD_FS,"Couldn't set FD_CLOEXEC: %s", strerror(errno));
-      fclose(result);
-      return NULL;
-    }
-  }
-#endif
-  return result;
-}
-
-/** As rename(), but work correctly with the sandbox. */
-int
-tor_rename(const char *path_old, const char *path_new)
-{
-  log_debug(LD_FS, "Renaming %s to %s", path_old, path_new);
-  return rename(sandbox_intern_string(path_old),
-                sandbox_intern_string(path_new));
-}
-
-#if defined(HAVE_SYS_MMAN_H) || defined(RUNNING_DOXYGEN)
-/** Try to create a memory mapping for <b>filename</b> and return it.  On
- * failure, return NULL.  Sets errno properly, using ERANGE to mean
- * "empty file". */
-tor_mmap_t *
-tor_mmap_file(const char *filename)
-{
-  int fd; /* router file */
-  char *string;
-  int page_size, result;
-  tor_mmap_t *res;
-  size_t size, filesize;
-  struct stat st;
-
-  tor_assert(filename);
-
-  fd = tor_open_cloexec(filename, O_RDONLY, 0);
-  if (fd<0) {
-    int save_errno = errno;
-    int severity = (errno == ENOENT) ? LOG_INFO : LOG_WARN;
-    log_fn(severity, LD_FS,"Could not open \"%s\" for mmap(): %s",filename,
-           strerror(errno));
-    errno = save_errno;
-    return NULL;
-  }
-
-  /* Get the size of the file */
-  result = fstat(fd, &st);
-  if (result != 0) {
-    int save_errno = errno;
-    log_warn(LD_FS,
-             "Couldn't fstat opened descriptor for \"%s\" during mmap: %s",
-             filename, strerror(errno));
-    close(fd);
-    errno = save_errno;
-    return NULL;
-  }
-  size = filesize = (size_t)(st.st_size);
-  /*
-   * Should we check for weird crap like mmapping a named pipe here,
-   * or just wait for if (!size) below to fail?
-   */
-  /* ensure page alignment */
-  page_size = getpagesize();
-  size += (size%page_size) ? page_size-(size%page_size) : 0;
-
-  if (!size) {
-    /* Zero-length file. If we call mmap on it, it will succeed but
-     * return NULL, and bad things will happen. So just fail. */
-    log_info(LD_FS,"File \"%s\" is empty. Ignoring.",filename);
-    errno = ERANGE;
-    close(fd);
-    return NULL;
-  }
-
-  string = mmap(0, size, PROT_READ, MAP_PRIVATE, fd, 0);
-  close(fd);
-  if (string == MAP_FAILED) {
-    int save_errno = errno;
-    log_warn(LD_FS,"Could not mmap file \"%s\": %s", filename,
-             strerror(errno));
-    errno = save_errno;
-    return NULL;
-  }
-
-  res = tor_malloc_zero(sizeof(tor_mmap_t));
-  res->data = string;
-  res->size = filesize;
-  res->mapping_size = size;
-
-  return res;
-}
-/** Release storage held for a memory mapping; returns 0 on success,
- * or -1 on failure (and logs a warning). */
-int
-tor_munmap_file(tor_mmap_t *handle)
-{
-  int res;
-
-  if (handle == NULL)
-    return 0;
-
-  res = munmap((char*)handle->data, handle->mapping_size);
-  if (res == 0) {
-    /* munmap() succeeded */
-    tor_free(handle);
-  } else {
-    log_warn(LD_FS, "Failed to munmap() in tor_munmap_file(): %s",
-             strerror(errno));
-    res = -1;
-  }
-
-  return res;
-}
-#elif defined(_WIN32)
-tor_mmap_t *
-tor_mmap_file(const char *filename)
-{
-  TCHAR tfilename[MAX_PATH]= {0};
-  tor_mmap_t *res = tor_malloc_zero(sizeof(tor_mmap_t));
-  int empty = 0;
-  HANDLE file_handle = INVALID_HANDLE_VALUE;
-  DWORD size_low, size_high;
-  uint64_t real_size;
-  res->mmap_handle = NULL;
-#ifdef UNICODE
-  mbstowcs(tfilename,filename,MAX_PATH);
-#else
-  strlcpy(tfilename,filename,MAX_PATH);
-#endif
-  file_handle = CreateFile(tfilename,
-                           GENERIC_READ, FILE_SHARE_READ,
-                           NULL,
-                           OPEN_EXISTING,
-                           FILE_ATTRIBUTE_NORMAL,
-                           0);
-
-  if (file_handle == INVALID_HANDLE_VALUE)
-    goto win_err;
-
-  size_low = GetFileSize(file_handle, &size_high);
-
-  if (size_low == INVALID_FILE_SIZE && GetLastError() != NO_ERROR) {
-    log_warn(LD_FS,"Error getting size of \"%s\".",filename);
-    goto win_err;
-  }
-  if (size_low == 0 && size_high == 0) {
-    log_info(LD_FS,"File \"%s\" is empty. Ignoring.",filename);
-    empty = 1;
-    goto err;
-  }
-  real_size = (((uint64_t)size_high)<<32) | size_low;
-  if (real_size > SIZE_MAX) {
-    log_warn(LD_FS,"File \"%s\" is too big to map; not trying.",filename);
-    goto err;
-  }
-  res->size = real_size;
-
-  res->mmap_handle = CreateFileMapping(file_handle,
-                                       NULL,
-                                       PAGE_READONLY,
-                                       size_high,
-                                       size_low,
-                                       NULL);
-  if (res->mmap_handle == NULL)
-    goto win_err;
-  res->data = (char*) MapViewOfFile(res->mmap_handle,
-                                    FILE_MAP_READ,
-                                    0, 0, 0);
-  if (!res->data)
-    goto win_err;
-
-  CloseHandle(file_handle);
-  return res;
- win_err: {
-    DWORD e = GetLastError();
-    int severity = (e == ERROR_FILE_NOT_FOUND || e == ERROR_PATH_NOT_FOUND) ?
-      LOG_INFO : LOG_WARN;
-    char *msg = format_win32_error(e);
-    log_fn(severity, LD_FS, "Couldn't mmap file \"%s\": %s", filename, msg);
-    tor_free(msg);
-    if (e == ERROR_FILE_NOT_FOUND || e == ERROR_PATH_NOT_FOUND)
-      errno = ENOENT;
-    else
-      errno = EINVAL;
-  }
- err:
-  if (empty)
-    errno = ERANGE;
-  if (file_handle != INVALID_HANDLE_VALUE)
-    CloseHandle(file_handle);
-  tor_munmap_file(res);
-  return NULL;
-}
-
-/* Unmap the file, and return 0 for success or -1 for failure */
-int
-tor_munmap_file(tor_mmap_t *handle)
-{
-  if (handle == NULL)
-    return 0;
-
-  if (handle->data) {
-    /* This is an ugly cast, but without it, "data" in struct tor_mmap_t would
-       have to be redefined as non-const. */
-    BOOL ok = UnmapViewOfFile( (LPVOID) handle->data);
-    if (!ok) {
-      log_warn(LD_FS, "Failed to UnmapViewOfFile() in tor_munmap_file(): %d",
-               (int)GetLastError());
-    }
-  }
-
-  if (handle->mmap_handle != NULL)
-    CloseHandle(handle->mmap_handle);
-  tor_free(handle);
-
-  return 0;
-}
-#else
-tor_mmap_t *
-tor_mmap_file(const char *filename)
-{
-  struct stat st;
-  char *res = read_file_to_str(filename, RFTS_BIN|RFTS_IGNORE_MISSING, &st);
-  tor_mmap_t *handle;
-  if (! res)
-    return NULL;
-  handle = tor_malloc_zero(sizeof(tor_mmap_t));
-  handle->data = res;
-  handle->size = st.st_size;
-  return handle;
-}
-
-/** Unmap the file mapped with tor_mmap_file(), and return 0 for success
- * or -1 for failure.
- */
-
-int
-tor_munmap_file(tor_mmap_t *handle)
-{
-  char *d = NULL;
-  if (handle == NULL)
-    return 0;
-
-  d = (char*)handle->data;
-  tor_free(d);
-  memwipe(handle, 0, sizeof(tor_mmap_t));
-  tor_free(handle);
-
-  /* Can't fail in this mmap()/munmap()-free case */
-  return 0;
-}
-#endif
-
-/** Replacement for snprintf.  Differs from platform snprintf in two
- * ways: First, always NUL-terminates its output.  Second, always
- * returns -1 if the result is truncated.  (Note that this return
- * behavior does <i>not</i> conform to C99; it just happens to be
- * easier to emulate "return -1" with conformant implementations than
- * it is to emulate "return number that would be written" with
- * non-conformant implementations.) */
-int
-tor_snprintf(char *str, size_t size, const char *format, ...)
-{
-  va_list ap;
-  int r;
-  va_start(ap,format);
-  r = tor_vsnprintf(str,size,format,ap);
-  va_end(ap);
-  return r;
-}
-
-/** Replacement for vsnprintf; behavior differs as tor_snprintf differs from
- * snprintf.
- */
-int
-tor_vsnprintf(char *str, size_t size, const char *format, va_list args)
-{
-  int r;
-  if (size == 0)
-    return -1; /* no place for the NUL */
-  if (size > SIZE_T_CEILING)
-    return -1;
-#ifdef _WIN32
-  r = _vsnprintf(str, size, format, args);
-#else
-  r = vsnprintf(str, size, format, args);
-#endif
-  str[size-1] = '\0';
-  if (r < 0 || r >= (ssize_t)size)
-    return -1;
-  return r;
-}
-
-/**
- * Portable asprintf implementation.  Does a printf() into a newly malloc'd
- * string.  Sets *<b>strp</b> to this string, and returns its length (not
- * including the terminating NUL character).
- *
- * You can treat this function as if its implementation were something like
-   <pre>
-     char buf[_INFINITY_];
-     tor_snprintf(buf, sizeof(buf), fmt, args);
-     *strp = tor_strdup(buf);
-     return strlen(*strp):
-   </pre>
- * Where _INFINITY_ is an imaginary constant so big that any string can fit
- * into it.
- */
-int
-tor_asprintf(char **strp, const char *fmt, ...)
-{
-  int r;
-  va_list args;
-  va_start(args, fmt);
-  r = tor_vasprintf(strp, fmt, args);
-  va_end(args);
-  if (!*strp || r < 0) {
-    /* LCOV_EXCL_START */
-    log_err(LD_BUG, "Internal error in asprintf");
-    tor_assert(0);
-    /* LCOV_EXCL_STOP */
-  }
-  return r;
-}
-
-/**
- * Portable vasprintf implementation.  Does a printf() into a newly malloc'd
- * string.  Differs from regular vasprintf in the same ways that
- * tor_asprintf() differs from regular asprintf.
- */
-int
-tor_vasprintf(char **strp, const char *fmt, va_list args)
-{
-  /* use a temporary variable in case *strp is in args. */
-  char *strp_tmp=NULL;
-#ifdef HAVE_VASPRINTF
-  /* If the platform gives us one, use it. */
-  int r = vasprintf(&strp_tmp, fmt, args);
-  if (r < 0)
-    *strp = NULL;
-  else
-    *strp = strp_tmp;
-  return r;
-#elif defined(HAVE__VSCPRINTF)
-  /* On Windows, _vsnprintf won't tell us the length of the string if it
-   * overflows, so we need to use _vcsprintf to tell how much to allocate */
-  int len, r;
-  va_list tmp_args;
-  va_copy(tmp_args, args);
-  len = _vscprintf(fmt, tmp_args);
-  va_end(tmp_args);
-  if (len < 0) {
-    *strp = NULL;
-    return -1;
-  }
-  strp_tmp = tor_malloc((size_t)len + 1);
-  r = _vsnprintf(strp_tmp, (size_t)len+1, fmt, args);
-  if (r != len) {
-    tor_free(strp_tmp);
-    *strp = NULL;
-    return -1;
-  }
-  *strp = strp_tmp;
-  return len;
-#else
-  /* Everywhere else, we have a decent vsnprintf that tells us how many
-   * characters we need.  We give it a try on a short buffer first, since
-   * it might be nice to avoid the second vsnprintf call.
-   */
-  /* XXXX This code spent a number of years broken (see bug 30651). It is
-   * possible that no Tor users actually run on systems without vasprintf() or
-   * _vscprintf(). If so, we should consider removing this code. */
-  char buf[128];
-  int len, r;
-  va_list tmp_args;
-  va_copy(tmp_args, args);
-  /* Use vsnprintf to retrieve needed length.  tor_vsnprintf() is not an
-   * option here because it will simply return -1 if buf is not large enough
-   * to hold the complete string.
-   */
-  len = vsnprintf(buf, sizeof(buf), fmt, tmp_args);
-  va_end(tmp_args);
-  buf[sizeof(buf) - 1] = '\0';
-  if (len < 0) {
-    *strp = NULL;
-    return -1;
-  }
-  if (len < (int)sizeof(buf)) {
-    *strp = tor_strdup(buf);
-    return len;
-  }
-  strp_tmp = tor_malloc((size_t)len+1);
-  /* use of tor_vsnprintf() will ensure string is null terminated */
-  r = tor_vsnprintf(strp_tmp, (size_t)len+1, fmt, args);
-  if (r != len) {
-    tor_free(strp_tmp);
-    *strp = NULL;
-    return -1;
-  }
-  *strp = strp_tmp;
-  return len;
-#endif
-}
-
-/** Given <b>hlen</b> bytes at <b>haystack</b> and <b>nlen</b> bytes at
- * <b>needle</b>, return a pointer to the first occurrence of the needle
- * within the haystack, or NULL if there is no such occurrence.
- *
- * This function is <em>not</em> timing-safe.
- *
- * Requires that <b>nlen</b> be greater than zero.
- */
-const void *
-tor_memmem(const void *_haystack, size_t hlen,
-           const void *_needle, size_t nlen)
-{
-#if defined(HAVE_MEMMEM) && (!defined(__GNUC__) || __GNUC__ >= 2)
-  tor_assert(nlen);
-  return memmem(_haystack, hlen, _needle, nlen);
-#else
-  /* This isn't as fast as the GLIBC implementation, but it doesn't need to
-   * be. */
-  const char *p, *last_possible_start;
-  const char *haystack = (const char*)_haystack;
-  const char *needle = (const char*)_needle;
-  char first;
-  tor_assert(nlen);
-
-  if (nlen > hlen)
-    return NULL;
-
-  p = haystack;
-  /* Last position at which the needle could start. */
-  last_possible_start = haystack + hlen - nlen;
-  first = *(const char*)needle;
-  while ((p = memchr(p, first, last_possible_start + 1 - p))) {
-    if (fast_memeq(p, needle, nlen))
-      return p;
-    if (++p > last_possible_start) {
-      /* This comparison shouldn't be necessary, since if p was previously
-       * equal to last_possible_start, the next memchr call would be
-       * "memchr(p, first, 0)", which will return NULL. But it clarifies the
-       * logic. */
-      return NULL;
-    }
-  }
-  return NULL;
-#endif
-}
-
-/**
- * Tables to implement ctypes-replacement TOR_IS*() functions.  Each table
- * has 256 bits to look up whether a character is in some set or not.  This
- * fails on non-ASCII platforms, but it is hard to find a platform whose
- * character set is not a superset of ASCII nowadays. */
-
-/**@{*/
-const uint32_t TOR_ISALPHA_TABLE[8] =
-  { 0, 0, 0x7fffffe, 0x7fffffe, 0, 0, 0, 0 };
-const uint32_t TOR_ISALNUM_TABLE[8] =
-  { 0, 0x3ff0000, 0x7fffffe, 0x7fffffe, 0, 0, 0, 0 };
-const uint32_t TOR_ISSPACE_TABLE[8] = { 0x3e00, 0x1, 0, 0, 0, 0, 0, 0 };
-const uint32_t TOR_ISXDIGIT_TABLE[8] =
-  { 0, 0x3ff0000, 0x7e, 0x7e, 0, 0, 0, 0 };
-const uint32_t TOR_ISDIGIT_TABLE[8] = { 0, 0x3ff0000, 0, 0, 0, 0, 0, 0 };
-const uint32_t TOR_ISPRINT_TABLE[8] =
-  { 0, 0xffffffff, 0xffffffff, 0x7fffffff, 0, 0, 0, 0x0 };
-const uint32_t TOR_ISUPPER_TABLE[8] = { 0, 0, 0x7fffffe, 0, 0, 0, 0, 0 };
-const uint32_t TOR_ISLOWER_TABLE[8] = { 0, 0, 0, 0x7fffffe, 0, 0, 0, 0 };
-
-/** Upper-casing and lowercasing tables to map characters to upper/lowercase
- * equivalents.  Used by tor_toupper() and tor_tolower(). */
-/**@{*/
-const uint8_t TOR_TOUPPER_TABLE[256] = {
-  0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,
-  16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,
-  32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,
-  48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,
-  64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,
-  80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,
-  96,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,
-  80,81,82,83,84,85,86,87,88,89,90,123,124,125,126,127,
-  128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,
-  144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,
-  160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,
-  176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,
-  192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,
-  208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,
-  224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,
-  240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,
-};
-const uint8_t TOR_TOLOWER_TABLE[256] = {
-  0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,
-  16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,
-  32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,
-  48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,
-  64,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,
-  112,113,114,115,116,117,118,119,120,121,122,91,92,93,94,95,
-  96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,
-  112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,
-  128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,
-  144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,
-  160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,
-  176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,
-  192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,
-  208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,
-  224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,
-  240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,
-};
-/**@}*/
-
-/** Helper for tor_strtok_r_impl: Advances cp past all characters in
- * <b>sep</b>, and returns its new value. */
-static char *
-strtok_helper(char *cp, const char *sep)
-{
-  if (sep[1]) {
-    while (*cp && strchr(sep, *cp))
-      ++cp;
-  } else {
-    while (*cp && *cp == *sep)
-      ++cp;
-  }
-  return cp;
-}
-
-/** Implementation of strtok_r for platforms whose coders haven't figured out
- * how to write one.  Hey, retrograde libc developers!  You can use this code
- * here for free! */
-char *
-tor_strtok_r_impl(char *str, const char *sep, char **lasts)
-{
-  char *cp, *start;
-  tor_assert(*sep);
-  if (str) {
-    str = strtok_helper(str, sep);
-    if (!*str)
-      return NULL;
-    start = cp = *lasts = str;
-  } else if (!*lasts || !**lasts) {
-    return NULL;
-  } else {
-    start = cp = *lasts;
-  }
-
-  if (sep[1]) {
-    while (*cp && !strchr(sep, *cp))
-      ++cp;
-  } else {
-    cp = strchr(cp, *sep);
-  }
-
-  if (!cp || !*cp) {
-    *lasts = NULL;
-  } else {
-    *cp++ = '\0';
-    *lasts = strtok_helper(cp, sep);
-  }
-  return start;
-}
-
-#ifdef _WIN32
-/** Take a filename and return a pointer to its final element.  This
- * function is called on __FILE__ to fix a MSVC nit where __FILE__
- * contains the full path to the file.  This is bad, because it
- * confuses users to find the home directory of the person who
- * compiled the binary in their warning messages.
- */
-const char *
-tor_fix_source_file(const char *fname)
-{
-  const char *cp1, *cp2, *r;
-  cp1 = strrchr(fname, '/');
-  cp2 = strrchr(fname, '\\');
-  if (cp1 && cp2) {
-    r = (cp1<cp2)?(cp2+1):(cp1+1);
-  } else if (cp1) {
-    r = cp1+1;
-  } else if (cp2) {
-    r = cp2+1;
-  } else {
-    r = fname;
-  }
-  return r;
-}
-#endif
-
-/**
- * Read a 16-bit value beginning at <b>cp</b>.  Equivalent to
- * *(uint16_t*)(cp), but will not cause segfaults on platforms that forbid
- * unaligned memory access.
- */
-uint16_t
-get_uint16(const void *cp)
-{
-  uint16_t v;
-  memcpy(&v,cp,2);
-  return v;
-}
-/**
- * Read a 32-bit value beginning at <b>cp</b>.  Equivalent to
- * *(uint32_t*)(cp), but will not cause segfaults on platforms that forbid
- * unaligned memory access.
- */
-uint32_t
-get_uint32(const void *cp)
-{
-  uint32_t v;
-  memcpy(&v,cp,4);
-  return v;
-}
-/**
- * Read a 64-bit value beginning at <b>cp</b>.  Equivalent to
- * *(uint64_t*)(cp), but will not cause segfaults on platforms that forbid
- * unaligned memory access.
- */
-uint64_t
-get_uint64(const void *cp)
-{
-  uint64_t v;
-  memcpy(&v,cp,8);
-  return v;
-}
-
-/**
- * Set a 16-bit value beginning at <b>cp</b> to <b>v</b>. Equivalent to
- * *(uint16_t*)(cp) = v, but will not cause segfaults on platforms that forbid
- * unaligned memory access. */
-void
-set_uint16(void *cp, uint16_t v)
-{
-  memcpy(cp,&v,2);
-}
-/**
- * Set a 32-bit value beginning at <b>cp</b> to <b>v</b>. Equivalent to
- * *(uint32_t*)(cp) = v, but will not cause segfaults on platforms that forbid
- * unaligned memory access. */
-void
-set_uint32(void *cp, uint32_t v)
-{
-  memcpy(cp,&v,4);
-}
-/**
- * Set a 64-bit value beginning at <b>cp</b> to <b>v</b>. Equivalent to
- * *(uint64_t*)(cp) = v, but will not cause segfaults on platforms that forbid
- * unaligned memory access. */
-void
-set_uint64(void *cp, uint64_t v)
-{
-  memcpy(cp,&v,8);
-}
-
-/**
- * Rename the file <b>from</b> to the file <b>to</b>.  On Unix, this is
- * the same as rename(2).  On windows, this removes <b>to</b> first if
- * it already exists.
- * Returns 0 on success.  Returns -1 and sets errno on failure.
- */
-int
-replace_file(const char *from, const char *to)
-{
-#ifndef _WIN32
-  return tor_rename(from, to);
-#else
-  switch (file_status(to))
-    {
-    case FN_NOENT:
-      break;
-    case FN_FILE:
-    case FN_EMPTY:
-      if (unlink(to)) return -1;
-      break;
-    case FN_ERROR:
-      return -1;
-    case FN_DIR:
-      errno = EISDIR;
-      return -1;
-    }
-  return tor_rename(from,to);
-#endif
-}
-
-/** Change <b>fname</b>'s modification time to now. */
-int
-touch_file(const char *fname)
-{
-  if (utime(fname, NULL)!=0)
-    return -1;
-  return 0;
-}
-
-/** Represents a lockfile on which we hold the lock. */
-struct tor_lockfile_t {
-  /** Name of the file */
-  char *filename;
-  /** File descriptor used to hold the file open */
-  int fd;
-};
-
-/** Try to get a lock on the lockfile <b>filename</b>, creating it as
- * necessary.  If someone else has the lock and <b>blocking</b> is true,
- * wait until the lock is available.  Otherwise return immediately whether
- * we succeeded or not.
- *
- * Set *<b>locked_out</b> to true if somebody else had the lock, and to false
- * otherwise.
- *
- * Return a <b>tor_lockfile_t</b> on success, NULL on failure.
- *
- * (Implementation note: because we need to fall back to fcntl on some
- *  platforms, these locks are per-process, not per-thread.  If you want
- *  to do in-process locking, use tor_mutex_t like a normal person.
- *  On Windows, when <b>blocking</b> is true, the maximum time that
- *  is actually waited is 10 seconds, after which NULL is returned
- *  and <b>locked_out</b> is set to 1.)
- */
-tor_lockfile_t *
-tor_lockfile_lock(const char *filename, int blocking, int *locked_out)
-{
-  tor_lockfile_t *result;
-  int fd;
-  *locked_out = 0;
-
-  log_info(LD_FS, "Locking \"%s\"", filename);
-  fd = tor_open_cloexec(filename, O_RDWR|O_CREAT|O_TRUNC, 0600);
-  if (fd < 0) {
-    log_warn(LD_FS,"Couldn't open \"%s\" for locking: %s", filename,
-             strerror(errno));
-    return NULL;
-  }
-
-#ifdef _WIN32
-  _lseek(fd, 0, SEEK_SET);
-  if (_locking(fd, blocking ? _LK_LOCK : _LK_NBLCK, 1) < 0) {
-    if (errno != EACCES && errno != EDEADLOCK)
-      log_warn(LD_FS,"Couldn't lock \"%s\": %s", filename, strerror(errno));
-    else
-      *locked_out = 1;
-    close(fd);
-    return NULL;
-  }
-#elif defined(HAVE_FLOCK)
-  if (flock(fd, LOCK_EX|(blocking ? 0 : LOCK_NB)) < 0) {
-    if (errno != EWOULDBLOCK)
-      log_warn(LD_FS,"Couldn't lock \"%s\": %s", filename, strerror(errno));
-    else
-      *locked_out = 1;
-    close(fd);
-    return NULL;
-  }
-#else
-  {
-    struct flock lock;
-    memset(&lock, 0, sizeof(lock));
-    lock.l_type = F_WRLCK;
-    lock.l_whence = SEEK_SET;
-    if (fcntl(fd, blocking ? F_SETLKW : F_SETLK, &lock) < 0) {
-      if (errno != EACCES && errno != EAGAIN)
-        log_warn(LD_FS, "Couldn't lock \"%s\": %s", filename, strerror(errno));
-      else
-        *locked_out = 1;
-      close(fd);
-      return NULL;
-    }
-  }
-#endif
-
-  result = tor_malloc(sizeof(tor_lockfile_t));
-  result->filename = tor_strdup(filename);
-  result->fd = fd;
-  return result;
-}
-
-/** Release the lock held as <b>lockfile</b>. */
-void
-tor_lockfile_unlock(tor_lockfile_t *lockfile)
-{
-  tor_assert(lockfile);
-
-  log_info(LD_FS, "Unlocking \"%s\"", lockfile->filename);
-#ifdef _WIN32
-  _lseek(lockfile->fd, 0, SEEK_SET);
-  if (_locking(lockfile->fd, _LK_UNLCK, 1) < 0) {
-    log_warn(LD_FS,"Error unlocking \"%s\": %s", lockfile->filename,
-             strerror(errno));
-  }
-#elif defined(HAVE_FLOCK)
-  if (flock(lockfile->fd, LOCK_UN) < 0) {
-    log_warn(LD_FS, "Error unlocking \"%s\": %s", lockfile->filename,
-             strerror(errno));
-  }
-#else
-  /* Closing the lockfile is sufficient. */
-#endif
-
-  close(lockfile->fd);
-  lockfile->fd = -1;
-  tor_free(lockfile->filename);
-  tor_free(lockfile);
-}
-
-/** @{ */
-/** Some old versions of Unix didn't define constants for these values,
- * and instead expect you to say 0, 1, or 2. */
-#ifndef SEEK_SET
-#define SEEK_SET 0
-#endif
-#ifndef SEEK_CUR
-#define SEEK_CUR 1
-#endif
-#ifndef SEEK_END
-#define SEEK_END 2
-#endif
-/** @} */
-
-/** Return the position of <b>fd</b> with respect to the start of the file. */
-off_t
-tor_fd_getpos(int fd)
-{
-#ifdef _WIN32
-  return (off_t) _lseek(fd, 0, SEEK_CUR);
-#else
-  return (off_t) lseek(fd, 0, SEEK_CUR);
-#endif
-}
-
-/** Move <b>fd</b> to the end of the file. Return -1 on error, 0 on success.
- * If the file is a pipe, do nothing and succeed.
- **/
-int
-tor_fd_seekend(int fd)
-{
-#ifdef _WIN32
-  return _lseek(fd, 0, SEEK_END) < 0 ? -1 : 0;
-#else
-  off_t rc = lseek(fd, 0, SEEK_END) < 0 ? -1 : 0;
-#ifdef ESPIPE
-  /* If we get an error and ESPIPE, then it's a pipe or a socket of a fifo:
-   * no need to worry. */
-  if (rc < 0 && errno == ESPIPE)
-    rc = 0;
-#endif
-  return (rc < 0) ? -1 : 0;
-#endif
-}
-
-/** Move <b>fd</b> to position <b>pos</b> in the file. Return -1 on error, 0
- * on success. */
-int
-tor_fd_setpos(int fd, off_t pos)
-{
-#ifdef _WIN32
-  return _lseek(fd, pos, SEEK_SET) < 0 ? -1 : 0;
-#else
-  return lseek(fd, pos, SEEK_SET) < 0 ? -1 : 0;
-#endif
-}
-
-/** Replacement for ftruncate(fd, 0): move to the front of the file and remove
- * all the rest of the file. Return -1 on error, 0 on success. */
-int
-tor_ftruncate(int fd)
-{
-  /* Rumor has it that some versions of ftruncate do not move the file pointer.
-   */
-  if (tor_fd_setpos(fd, 0) < 0)
-    return -1;
-
-#ifdef _WIN32
-  return _chsize(fd, 0);
-#else
-  return ftruncate(fd, 0);
-#endif
-}
-
-#undef DEBUG_SOCKET_COUNTING
-#ifdef DEBUG_SOCKET_COUNTING
-/** A bitarray of all fds that should be passed to tor_socket_close(). Only
- * used if DEBUG_SOCKET_COUNTING is defined. */
-static bitarray_t *open_sockets = NULL;
-/** The size of <b>open_sockets</b>, in bits. */
-static int max_socket = -1;
-#endif
-
-/** Count of number of sockets currently open.  (Undercounts sockets opened by
- * eventdns and libevent.) */
-static int n_sockets_open = 0;
-
-/** Mutex to protect open_sockets, max_socket, and n_sockets_open. */
-static tor_mutex_t *socket_accounting_mutex = NULL;
-
-/** Helper: acquire the socket accounting lock. */
-static inline void
-socket_accounting_lock(void)
-{
-  if (PREDICT_UNLIKELY(!socket_accounting_mutex))
-    socket_accounting_mutex = tor_mutex_new();
-  tor_mutex_acquire(socket_accounting_mutex);
-}
-
-/** Helper: release the socket accounting lock. */
-static inline void
-socket_accounting_unlock(void)
-{
-  tor_mutex_release(socket_accounting_mutex);
-}
-
-/** As close(), but guaranteed to work for sockets across platforms (including
- * Windows, where close()ing a socket doesn't work.  Returns 0 on success and
- * the socket error code on failure. */
-int
-tor_close_socket_simple(tor_socket_t s)
-{
-  int r = 0;
-
-  /* On Windows, you have to call close() on fds returned by open(),
-  * and closesocket() on fds returned by socket().  On Unix, everything
-  * gets close()'d.  We abstract this difference by always using
-  * tor_close_socket to close sockets, and always using close() on
-  * files.
-  */
-  #if defined(_WIN32)
-    r = closesocket(s);
-  #else
-    r = close(s);
-  #endif
-
-  if (r != 0) {
-    int err = tor_socket_errno(-1);
-    log_info(LD_NET, "Close returned an error: %s", tor_socket_strerror(err));
-    return err;
-  }
-
-  return r;
-}
-
-/** As tor_close_socket_simple(), but keeps track of the number
- * of open sockets. Returns 0 on success, -1 on failure. */
-MOCK_IMPL(int,
-tor_close_socket,(tor_socket_t s))
-{
-  int r = tor_close_socket_simple(s);
-
-  socket_accounting_lock();
-#ifdef DEBUG_SOCKET_COUNTING
-  if (s > max_socket || ! bitarray_is_set(open_sockets, s)) {
-    log_warn(LD_BUG, "Closing a socket (%d) that wasn't returned by tor_open_"
-             "socket(), or that was already closed or something.", s);
-  } else {
-    tor_assert(open_sockets && s <= max_socket);
-    bitarray_clear(open_sockets, s);
-  }
-#endif
-  if (r == 0) {
-    --n_sockets_open;
-  } else {
-#ifdef _WIN32
-    if (r != WSAENOTSOCK)
-      --n_sockets_open;
-#else
-    if (r != EBADF)
-      --n_sockets_open; // LCOV_EXCL_LINE -- EIO and EINTR too hard to force.
-#endif
-    r = -1;
-  }
-
-  tor_assert_nonfatal(n_sockets_open >= 0);
-  socket_accounting_unlock();
-  return r;
-}
-
-/** @{ */
-#ifdef DEBUG_SOCKET_COUNTING
-/** Helper: if DEBUG_SOCKET_COUNTING is enabled, remember that <b>s</b> is
- * now an open socket. */
-static inline void
-mark_socket_open(tor_socket_t s)
-{
-  /* XXXX This bitarray business will NOT work on windows: sockets aren't
-     small ints there. */
-  if (s > max_socket) {
-    if (max_socket == -1) {
-      open_sockets = bitarray_init_zero(s+128);
-      max_socket = s+128;
-    } else {
-      open_sockets = bitarray_expand(open_sockets, max_socket, s+128);
-      max_socket = s+128;
-    }
-  }
-  if (bitarray_is_set(open_sockets, s)) {
-    log_warn(LD_BUG, "I thought that %d was already open, but socket() just "
-             "gave it to me!", s);
-  }
-  bitarray_set(open_sockets, s);
-}
-#else
-#define mark_socket_open(s) STMT_NIL
-#endif
-/** @} */
-
-/** As socket(), but counts the number of open sockets. */
-MOCK_IMPL(tor_socket_t,
-tor_open_socket,(int domain, int type, int protocol))
-{
-  return tor_open_socket_with_extensions(domain, type, protocol, 1, 0);
-}
-
-/** Mockable wrapper for connect(). */
-MOCK_IMPL(tor_socket_t,
-tor_connect_socket,(tor_socket_t sock, const struct sockaddr *address,
-                     socklen_t address_len))
-{
-  return connect(sock,address,address_len);
-}
-
-/** As socket(), but creates a nonblocking socket and
- * counts the number of open sockets. */
-tor_socket_t
-tor_open_socket_nonblocking(int domain, int type, int protocol)
-{
-  return tor_open_socket_with_extensions(domain, type, protocol, 1, 1);
-}
-
-/** As socket(), but counts the number of open sockets and handles
- * socket creation with either of SOCK_CLOEXEC and SOCK_NONBLOCK specified.
- * <b>cloexec</b> and <b>nonblock</b> should be either 0 or 1 to indicate
- * if the corresponding extension should be used.*/
-tor_socket_t
-tor_open_socket_with_extensions(int domain, int type, int protocol,
-                                int cloexec, int nonblock)
-{
-  tor_socket_t s;
-
-  /* We are about to create a new file descriptor so make sure we have
-   * enough of them. */
-  if (get_n_open_sockets() >= max_sockets - 1) {
-#ifdef _WIN32
-    WSASetLastError(WSAEMFILE);
-#else
-    errno = EMFILE;
-#endif
-    return TOR_INVALID_SOCKET;
-  }
-
-#if defined(SOCK_CLOEXEC) && defined(SOCK_NONBLOCK)
-  int ext_flags = (cloexec ? SOCK_CLOEXEC : 0) |
-                  (nonblock ? SOCK_NONBLOCK : 0);
-  s = socket(domain, type|ext_flags, protocol);
-  if (SOCKET_OK(s))
-    goto socket_ok;
-  /* If we got an error, see if it is EINVAL. EINVAL might indicate that,
-   * even though we were built on a system with SOCK_CLOEXEC and SOCK_NONBLOCK
-   * support, we are running on one without. */
-  if (errno != EINVAL)
-    return s;
-#endif /* SOCK_CLOEXEC && SOCK_NONBLOCK */
-
-  s = socket(domain, type, protocol);
-  if (! SOCKET_OK(s))
-    return s;
-
-#if defined(FD_CLOEXEC)
-  if (cloexec) {
-    if (fcntl(s, F_SETFD, FD_CLOEXEC) == -1) {
-      log_warn(LD_FS,"Couldn't set FD_CLOEXEC: %s", strerror(errno));
-      tor_close_socket_simple(s);
-      return TOR_INVALID_SOCKET;
-    }
-  }
-#else
-  (void)cloexec;
-#endif
-
-  if (nonblock) {
-    if (set_socket_nonblocking(s) == -1) {
-      tor_close_socket_simple(s);
-      return TOR_INVALID_SOCKET;
-    }
-  }
-
-  goto socket_ok; /* So that socket_ok will not be unused. */
-
- socket_ok:
-  socket_accounting_lock();
-  ++n_sockets_open;
-  mark_socket_open(s);
-  socket_accounting_unlock();
-  return s;
-}
-
-/** As accept(), but counts the number of open sockets. */
-tor_socket_t
-tor_accept_socket(tor_socket_t sockfd, struct sockaddr *addr, socklen_t *len)
-{
-  return tor_accept_socket_with_extensions(sockfd, addr, len, 1, 0);
-}
-
-/** As accept(), but returns a nonblocking socket and
- * counts the number of open sockets. */
-tor_socket_t
-tor_accept_socket_nonblocking(tor_socket_t sockfd, struct sockaddr *addr,
-                              socklen_t *len)
-{
-  return tor_accept_socket_with_extensions(sockfd, addr, len, 1, 1);
-}
-
-/** As accept(), but counts the number of open sockets and handles
- * socket creation with either of SOCK_CLOEXEC and SOCK_NONBLOCK specified.
- * <b>cloexec</b> and <b>nonblock</b> should be either 0 or 1 to indicate
- * if the corresponding extension should be used.*/
-tor_socket_t
-tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr,
-                                 socklen_t *len, int cloexec, int nonblock)
-{
-  tor_socket_t s;
-
-  /* We are about to create a new file descriptor so make sure we have
-   * enough of them. */
-  if (get_n_open_sockets() >= max_sockets - 1) {
-#ifdef _WIN32
-    WSASetLastError(WSAEMFILE);
-#else
-    errno = EMFILE;
-#endif
-    return TOR_INVALID_SOCKET;
-  }
-
-#if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) && defined(SOCK_NONBLOCK)
-  int ext_flags = (cloexec ? SOCK_CLOEXEC : 0) |
-                  (nonblock ? SOCK_NONBLOCK : 0);
-  s = accept4(sockfd, addr, len, ext_flags);
-  if (SOCKET_OK(s))
-    goto socket_ok;
-  /* If we got an error, see if it is ENOSYS. ENOSYS indicates that,
-   * even though we were built on a system with accept4 support, we
-   * are running on one without. Also, check for EINVAL, which indicates that
-   * we are missing SOCK_CLOEXEC/SOCK_NONBLOCK support. */
-  if (errno != EINVAL && errno != ENOSYS)
-    return s;
-#endif
-
-  s = accept(sockfd, addr, len);
-  if (!SOCKET_OK(s))
-    return s;
-
-#if defined(FD_CLOEXEC)
-  if (cloexec) {
-    if (fcntl(s, F_SETFD, FD_CLOEXEC) == -1) {
-      log_warn(LD_NET, "Couldn't set FD_CLOEXEC: %s", strerror(errno));
-      tor_close_socket_simple(s);
-      return TOR_INVALID_SOCKET;
-    }
-  }
-#else
-  (void)cloexec;
-#endif
-
-  if (nonblock) {
-    if (set_socket_nonblocking(s) == -1) {
-      tor_close_socket_simple(s);
-      return TOR_INVALID_SOCKET;
-    }
-  }
-
-  goto socket_ok; /* So that socket_ok will not be unused. */
-
- socket_ok:
-  socket_accounting_lock();
-  ++n_sockets_open;
-  mark_socket_open(s);
-  socket_accounting_unlock();
-  return s;
-}
-
-/** Return the number of sockets we currently have opened. */
-int
-get_n_open_sockets(void)
-{
-  int n;
-  socket_accounting_lock();
-  n = n_sockets_open;
-  socket_accounting_unlock();
-  return n;
-}
-
-/** Mockable wrapper for getsockname(). */
-MOCK_IMPL(int,
-tor_getsockname,(tor_socket_t sock, struct sockaddr *address,
-                 socklen_t *address_len))
-{
-   return getsockname(sock, address, address_len);
-}
-
-/** Turn <b>socket</b> into a nonblocking socket. Return 0 on success, -1
- * on failure.
- */
-int
-set_socket_nonblocking(tor_socket_t sock)
-{
-#if defined(_WIN32)
-  unsigned long nonblocking = 1;
-  ioctlsocket(sock, FIONBIO, (unsigned long*) &nonblocking);
-#else
-  int flags;
-
-  flags = fcntl(sock, F_GETFL, 0);
-  if (flags == -1) {
-    log_warn(LD_NET, "Couldn't get file status flags: %s", strerror(errno));
-    return -1;
-  }
-  flags |= O_NONBLOCK;
-  if (fcntl(sock, F_SETFL, flags) == -1) {
-    log_warn(LD_NET, "Couldn't set file status flags: %s", strerror(errno));
-    return -1;
-  }
-#endif
-
-  return 0;
-}
-
-/**
- * Allocate a pair of connected sockets.  (Like socketpair(family,
- * type,protocol,fd), but works on systems that don't have
- * socketpair.)
- *
- * Currently, only (AF_UNIX, SOCK_STREAM, 0) sockets are supported.
- *
- * Note that on systems without socketpair, this call will fail if
- * localhost is inaccessible (for example, if the networking
- * stack is down). And even if it succeeds, the socket pair will not
- * be able to read while localhost is down later (the socket pair may
- * even close, depending on OS-specific timeouts).
- *
- * Returns 0 on success and -errno on failure; do not rely on the value
- * of errno or WSAGetLastError().
- **/
-/* It would be nicer just to set errno, but that won't work for windows. */
-int
-tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2])
-{
-//don't use win32 socketpairs (they are always bad)
-#if defined(HAVE_SOCKETPAIR) && !defined(_WIN32)
-  int r;
-
-#ifdef SOCK_CLOEXEC
-  r = socketpair(family, type|SOCK_CLOEXEC, protocol, fd);
-  if (r == 0)
-    goto sockets_ok;
-  /* If we got an error, see if it is EINVAL. EINVAL might indicate that,
-   * even though we were built on a system with SOCK_CLOEXEC support, we
-   * are running on one without. */
-  if (errno != EINVAL)
-    return -errno;
-#endif
-
-  r = socketpair(family, type, protocol, fd);
-  if (r < 0)
-    return -errno;
-
-#if defined(FD_CLOEXEC)
-  if (SOCKET_OK(fd[0])) {
-    r = fcntl(fd[0], F_SETFD, FD_CLOEXEC);
-    if (r == -1) {
-      close(fd[0]);
-      close(fd[1]);
-      return -errno;
-    }
-  }
-  if (SOCKET_OK(fd[1])) {
-    r = fcntl(fd[1], F_SETFD, FD_CLOEXEC);
-    if (r == -1) {
-      close(fd[0]);
-      close(fd[1]);
-      return -errno;
-    }
-  }
-#endif
-  goto sockets_ok; /* So that sockets_ok will not be unused. */
-
- sockets_ok:
-  socket_accounting_lock();
-  if (SOCKET_OK(fd[0])) {
-    ++n_sockets_open;
-    mark_socket_open(fd[0]);
-  }
-  if (SOCKET_OK(fd[1])) {
-    ++n_sockets_open;
-    mark_socket_open(fd[1]);
-  }
-  socket_accounting_unlock();
-
-  return 0;
-#else
-  return tor_ersatz_socketpair(family, type, protocol, fd);
-#endif
-}
-
-#ifdef NEED_ERSATZ_SOCKETPAIR
-
-static inline socklen_t
-SIZEOF_SOCKADDR(int domain)
-{
-  switch (domain) {
-    case AF_INET:
-      return sizeof(struct sockaddr_in);
-    case AF_INET6:
-      return sizeof(struct sockaddr_in6);
-    default:
-      return 0;
-  }
-}
-
-/**
- * Helper used to implement socketpair on systems that lack it, by
- * making a direct connection to localhost.
- */
-STATIC int
-tor_ersatz_socketpair(int family, int type, int protocol, tor_socket_t fd[2])
-{
-    /* This socketpair does not work when localhost is down. So
-     * it's really not the same thing at all. But it's close enough
-     * for now, and really, when localhost is down sometimes, we
-     * have other problems too.
-     */
-    tor_socket_t listener = TOR_INVALID_SOCKET;
-    tor_socket_t connector = TOR_INVALID_SOCKET;
-    tor_socket_t acceptor = TOR_INVALID_SOCKET;
-    tor_addr_t listen_tor_addr;
-    struct sockaddr_storage connect_addr_ss, listen_addr_ss;
-    struct sockaddr *listen_addr = (struct sockaddr *) &listen_addr_ss;
-    uint16_t listen_port = 0;
-    tor_addr_t connect_tor_addr;
-    uint16_t connect_port = 0;
-    struct sockaddr *connect_addr = (struct sockaddr *) &connect_addr_ss;
-    socklen_t size;
-    int saved_errno = -1;
-    int ersatz_domain = AF_INET;
-
-    memset(&connect_tor_addr, 0, sizeof(connect_tor_addr));
-    memset(&connect_addr_ss, 0, sizeof(connect_addr_ss));
-    memset(&listen_tor_addr, 0, sizeof(listen_tor_addr));
-    memset(&listen_addr_ss, 0, sizeof(listen_addr_ss));
-
-    if (protocol
-#ifdef AF_UNIX
-        || family != AF_UNIX
-#endif
-        ) {
-#ifdef _WIN32
-      return -WSAEAFNOSUPPORT;
-#else
-      return -EAFNOSUPPORT;
-#endif
-    }
-    if (!fd) {
-      return -EINVAL;
-    }
-
-    listener = tor_open_socket(ersatz_domain, type, 0);
-    if (!SOCKET_OK(listener)) {
-      int first_errno = tor_socket_errno(-1);
-      if (first_errno == SOCK_ERRNO(EPROTONOSUPPORT)
-          && ersatz_domain == AF_INET) {
-        /* Assume we're on an IPv6-only system */
-        ersatz_domain = AF_INET6;
-        listener = tor_open_socket(ersatz_domain, type, 0);
-        if (!SOCKET_OK(listener)) {
-          /* Keep the previous behaviour, which was to return the IPv4 error.
-           * (This may be less informative on IPv6-only systems.)
-           * XX/teor - is there a better way to decide which errno to return?
-           * (I doubt we care much either way, once there is an error.)
-           */
-          return -first_errno;
-        }
-      }
-    }
-    /* If there is no 127.0.0.1 or ::1, this will and must fail. Otherwise, we
-     * risk exposing a socketpair on a routable IP address. (Some BSD jails
-     * use a routable address for localhost. Fortunately, they have the real
-     * AF_UNIX socketpair.) */
-    if (ersatz_domain == AF_INET) {
-      tor_addr_from_ipv4h(&listen_tor_addr, INADDR_LOOPBACK);
-    } else {
-      tor_addr_parse(&listen_tor_addr, "[::1]");
-    }
-    tor_assert(tor_addr_is_loopback(&listen_tor_addr));
-    size = tor_addr_to_sockaddr(&listen_tor_addr,
-                         0 /* kernel chooses port.  */,
-                         listen_addr,
-                         sizeof(listen_addr_ss));
-    if (bind(listener, listen_addr, size) == -1)
-      goto tidy_up_and_fail;
-    if (listen(listener, 1) == -1)
-      goto tidy_up_and_fail;
-
-    connector = tor_open_socket(ersatz_domain, type, 0);
-    if (!SOCKET_OK(connector))
-      goto tidy_up_and_fail;
-    /* We want to find out the port number to connect to.  */
-    size = sizeof(connect_addr_ss);
-    if (getsockname(listener, connect_addr, &size) == -1)
-      goto tidy_up_and_fail;
-    if (size != SIZEOF_SOCKADDR (connect_addr->sa_family))
-      goto abort_tidy_up_and_fail;
-    if (connect(connector, connect_addr, size) == -1)
-      goto tidy_up_and_fail;
-
-    size = sizeof(listen_addr_ss);
-    acceptor = tor_accept_socket(listener, listen_addr, &size);
-    if (!SOCKET_OK(acceptor))
-      goto tidy_up_and_fail;
-    if (size != SIZEOF_SOCKADDR(listen_addr->sa_family))
-      goto abort_tidy_up_and_fail;
-    /* Now check we are talking to ourself by matching port and host on the
-       two sockets.  */
-    if (getsockname(connector, connect_addr, &size) == -1)
-      goto tidy_up_and_fail;
-    /* Set *_tor_addr and *_port to the address and port that was used */
-    tor_addr_from_sockaddr(&listen_tor_addr, listen_addr, &listen_port);
-    tor_addr_from_sockaddr(&connect_tor_addr, connect_addr, &connect_port);
-    if (size != SIZEOF_SOCKADDR (connect_addr->sa_family)
-        || tor_addr_compare(&listen_tor_addr, &connect_tor_addr, CMP_SEMANTIC)
-        || listen_port != connect_port) {
-      goto abort_tidy_up_and_fail;
-    }
-    tor_close_socket(listener);
-    fd[0] = connector;
-    fd[1] = acceptor;
-
-    return 0;
-
-  abort_tidy_up_and_fail:
-#ifdef _WIN32
-    saved_errno = WSAECONNABORTED;
-#else
-    saved_errno = ECONNABORTED; /* I hope this is portable and appropriate.  */
-#endif
-  tidy_up_and_fail:
-    if (saved_errno < 0)
-      saved_errno = errno;
-    if (SOCKET_OK(listener))
-      tor_close_socket(listener);
-    if (SOCKET_OK(connector))
-      tor_close_socket(connector);
-    if (SOCKET_OK(acceptor))
-      tor_close_socket(acceptor);
-    return -saved_errno;
-}
-
-#undef SIZEOF_SOCKADDR
-
-#endif
-
-/* Return the maximum number of allowed sockets. */
-int
-get_max_sockets(void)
-{
-  return max_sockets;
-}
-
-/** Number of extra file descriptors to keep in reserve beyond those that we
- * tell Tor it's allowed to use. */
-#define ULIMIT_BUFFER 32 /* keep 32 extra fd's beyond ConnLimit_ */
-
-/** Learn the maximum allowed number of file descriptors, and tell the
- * system we want to use up to that number. (Some systems have a low soft
- * limit, and let us set it higher.)  We compute this by finding the largest
- * number that we can use.
- *
- * If the limit is below the reserved file descriptor value (ULIMIT_BUFFER),
- * return -1 and <b>max_out</b> is untouched.
- *
- * If we can't find a number greater than or equal to <b>limit</b>, then we
- * fail by returning -1 and <b>max_out</b> is untouched.
- *
- * If we are unable to set the limit value because of setrlimit() failing,
- * return 0 and <b>max_out</b> is set to the current maximum value returned
- * by getrlimit().
- *
- * Otherwise, return 0 and store the maximum we found inside <b>max_out</b>
- * and set <b>max_sockets</b> with that value as well.*/
-int
-set_max_file_descriptors(rlim_t limit, int *max_out)
-{
-  if (limit < ULIMIT_BUFFER) {
-    log_warn(LD_CONFIG,
-             "ConnLimit must be at least %d. Failing.", ULIMIT_BUFFER);
-    return -1;
-  }
-
-  /* Define some maximum connections values for systems where we cannot
-   * automatically determine a limit. Re Cygwin, see
-   * http://archives.seul.org/or/talk/Aug-2006/msg00210.html
-   * For an iPhone, 9999 should work. For Windows and all other unknown
-   * systems we use 15000 as the default. */
-#ifndef HAVE_GETRLIMIT
-#if defined(CYGWIN) || defined(__CYGWIN__)
-  const char *platform = "Cygwin";
-  const unsigned long MAX_CONNECTIONS = 3200;
-#elif defined(_WIN32)
-  const char *platform = "Windows";
-  const unsigned long MAX_CONNECTIONS = 15000;
-#else
-  const char *platform = "unknown platforms with no getrlimit()";
-  const unsigned long MAX_CONNECTIONS = 15000;
-#endif
-  log_fn(LOG_INFO, LD_NET,
-         "This platform is missing getrlimit(). Proceeding.");
-  if (limit > MAX_CONNECTIONS) {
-    log_warn(LD_CONFIG,
-             "We do not support more than %lu file descriptors "
-             "on %s. Tried to raise to %lu.",
-             (unsigned long)MAX_CONNECTIONS, platform, (unsigned long)limit);
-    return -1;
-  }
-  limit = MAX_CONNECTIONS;
-#else /* HAVE_GETRLIMIT */
-  struct rlimit rlim;
-
-  if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) {
-    log_warn(LD_NET, "Could not get maximum number of file descriptors: %s",
-             strerror(errno));
-    return -1;
-  }
-  if (rlim.rlim_max < limit) {
-    log_warn(LD_CONFIG,"We need %lu file descriptors available, and we're "
-             "limited to %lu. Please change your ulimit -n.",
-             (unsigned long)limit, (unsigned long)rlim.rlim_max);
-    return -1;
-  }
-
-  if (rlim.rlim_max > rlim.rlim_cur) {
-    log_info(LD_NET,"Raising max file descriptors from %lu to %lu.",
-             (unsigned long)rlim.rlim_cur, (unsigned long)rlim.rlim_max);
-  }
-  /* Set the current limit value so if the attempt to set the limit to the
-   * max fails at least we'll have a valid value of maximum sockets. */
-  *max_out = max_sockets = (int)rlim.rlim_cur - ULIMIT_BUFFER;
-  rlim.rlim_cur = rlim.rlim_max;
-
-  if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) {
-    int couldnt_set = 1;
-    const int setrlimit_errno = errno;
-#ifdef OPEN_MAX
-    uint64_t try_limit = OPEN_MAX - ULIMIT_BUFFER;
-    if (errno == EINVAL && try_limit < (uint64_t) rlim.rlim_cur) {
-      /* On some platforms, OPEN_MAX is the real limit, and getrlimit() is
-       * full of nasty lies.  I'm looking at you, OSX 10.5.... */
-      rlim.rlim_cur = MIN((rlim_t) try_limit, rlim.rlim_cur);
-      if (setrlimit(RLIMIT_NOFILE, &rlim) == 0) {
-        if (rlim.rlim_cur < (rlim_t)limit) {
-          log_warn(LD_CONFIG, "We are limited to %lu file descriptors by "
-                   "OPEN_MAX (%lu), and ConnLimit is %lu.  Changing "
-                   "ConnLimit; sorry.",
-                   (unsigned long)try_limit, (unsigned long)OPEN_MAX,
-                   (unsigned long)limit);
-        } else {
-          log_info(LD_CONFIG, "Dropped connection limit to %lu based on "
-                   "OPEN_MAX (%lu); Apparently, %lu was too high and rlimit "
-                   "lied to us.",
-                   (unsigned long)try_limit, (unsigned long)OPEN_MAX,
-                   (unsigned long)rlim.rlim_max);
-        }
-        couldnt_set = 0;
-      }
-    }
-#endif /* OPEN_MAX */
-    if (couldnt_set) {
-      log_warn(LD_CONFIG,"Couldn't set maximum number of file descriptors: %s",
-               strerror(setrlimit_errno));
-    }
-  }
-  /* leave some overhead for logs, etc, */
-  limit = rlim.rlim_cur;
-#endif /* HAVE_GETRLIMIT */
-
-  if (limit > INT_MAX)
-    limit = INT_MAX;
-  tor_assert(max_out);
-  *max_out = max_sockets = (int)limit - ULIMIT_BUFFER;
-  return 0;
-}
-
-#ifndef _WIN32
-/** Log details of current user and group credentials. Return 0 on
- * success. Logs and return -1 on failure.
- */
-static int
-log_credential_status(void)
-{
-/** Log level to use when describing non-error UID/GID status. */
-#define CREDENTIAL_LOG_LEVEL LOG_INFO
-  /* Real, effective and saved UIDs */
-  uid_t ruid, euid, suid;
-  /* Read, effective and saved GIDs */
-  gid_t rgid, egid, sgid;
-  /* Supplementary groups */
-  gid_t *sup_gids = NULL;
-  int sup_gids_size;
-  /* Number of supplementary groups */
-  int ngids;
-
-  /* log UIDs */
-#ifdef HAVE_GETRESUID
-  if (getresuid(&ruid, &euid, &suid) != 0 ) {
-    log_warn(LD_GENERAL, "Error getting changed UIDs: %s", strerror(errno));
-    return -1;
-  } else {
-    log_fn(CREDENTIAL_LOG_LEVEL, LD_GENERAL,
-           "UID is %u (real), %u (effective), %u (saved)",
-           (unsigned)ruid, (unsigned)euid, (unsigned)suid);
-  }
-#else
-  /* getresuid is not present on MacOS X, so we can't get the saved (E)UID */
-  ruid = getuid();
-  euid = geteuid();
-  (void)suid;
-
-  log_fn(CREDENTIAL_LOG_LEVEL, LD_GENERAL,
-         "UID is %u (real), %u (effective), unknown (saved)",
-         (unsigned)ruid, (unsigned)euid);
-#endif
-
-  /* log GIDs */
-#ifdef HAVE_GETRESGID
-  if (getresgid(&rgid, &egid, &sgid) != 0 ) {
-    log_warn(LD_GENERAL, "Error getting changed GIDs: %s", strerror(errno));
-    return -1;
-  } else {
-    log_fn(CREDENTIAL_LOG_LEVEL, LD_GENERAL,
-           "GID is %u (real), %u (effective), %u (saved)",
-           (unsigned)rgid, (unsigned)egid, (unsigned)sgid);
-  }
-#else
-  /* getresgid is not present on MacOS X, so we can't get the saved (E)GID */
-  rgid = getgid();
-  egid = getegid();
-  (void)sgid;
-  log_fn(CREDENTIAL_LOG_LEVEL, LD_GENERAL,
-         "GID is %u (real), %u (effective), unknown (saved)",
-         (unsigned)rgid, (unsigned)egid);
-#endif
-
-  /* log supplementary groups */
-  sup_gids_size = 64;
-  sup_gids = tor_calloc(64, sizeof(gid_t));
-  while ((ngids = getgroups(sup_gids_size, sup_gids)) < 0 &&
-         errno == EINVAL &&
-         sup_gids_size < NGROUPS_MAX) {
-    sup_gids_size *= 2;
-    sup_gids = tor_reallocarray(sup_gids, sizeof(gid_t), sup_gids_size);
-  }
-
-  if (ngids < 0) {
-    log_warn(LD_GENERAL, "Error getting supplementary GIDs: %s",
-             strerror(errno));
-    tor_free(sup_gids);
-    return -1;
-  } else {
-    int i, retval = 0;
-    char *s = NULL;
-    smartlist_t *elts = smartlist_new();
-
-    for (i = 0; i<ngids; i++) {
-      smartlist_add_asprintf(elts, "%u", (unsigned)sup_gids[i]);
-    }
-
-    s = smartlist_join_strings(elts, " ", 0, NULL);
-
-    log_fn(CREDENTIAL_LOG_LEVEL, LD_GENERAL, "Supplementary groups are: %s",s);
-
-    tor_free(s);
-    SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp));
-    smartlist_free(elts);
-    tor_free(sup_gids);
-
-    return retval;
-  }
-
-  return 0;
-}
-#endif
-
-#ifndef _WIN32
-/** Cached struct from the last getpwname() call we did successfully. */
-static struct passwd *passwd_cached = NULL;
-
-/** Helper: copy a struct passwd object.
- *
- * We only copy the fields pw_uid, pw_gid, pw_name, pw_dir.  Tor doesn't use
- * any others, and I don't want to run into incompatibilities.
- */
-static struct passwd *
-tor_passwd_dup(const struct passwd *pw)
-{
-  struct passwd *new_pw = tor_malloc_zero(sizeof(struct passwd));
-  if (pw->pw_name)
-    new_pw->pw_name = tor_strdup(pw->pw_name);
-  if (pw->pw_dir)
-    new_pw->pw_dir = tor_strdup(pw->pw_dir);
-  new_pw->pw_uid = pw->pw_uid;
-  new_pw->pw_gid = pw->pw_gid;
-
-  return new_pw;
-}
-
-/** Helper: free one of our cached 'struct passwd' values. */
-static void
-tor_passwd_free(struct passwd *pw)
-{
-  if (!pw)
-    return;
-
-  tor_free(pw->pw_name);
-  tor_free(pw->pw_dir);
-  tor_free(pw);
-}
-
-/** Wrapper around getpwnam() that caches result. Used so that we don't need
- * to give the sandbox access to /etc/passwd.
- *
- * The following fields alone will definitely be copied in the output: pw_uid,
- * pw_gid, pw_name, pw_dir.  Other fields are not present in cached values.
- *
- * When called with a NULL argument, this function clears storage associated
- * with static variables it uses.
- **/
-const struct passwd *
-tor_getpwnam(const char *username)
-{
-  struct passwd *pw;
-
-  if (username == NULL) {
-    tor_passwd_free(passwd_cached);
-    passwd_cached = NULL;
-    return NULL;
-  }
-
-  if ((pw = getpwnam(username))) {
-    tor_passwd_free(passwd_cached);
-    passwd_cached = tor_passwd_dup(pw);
-    log_info(LD_GENERAL, "Caching new entry %s for %s",
-             passwd_cached->pw_name, username);
-    return pw;
-  }
-
-  /* Lookup failed */
-  if (! passwd_cached || ! passwd_cached->pw_name)
-    return NULL;
-
-  if (! strcmp(username, passwd_cached->pw_name))
-    return passwd_cached; // LCOV_EXCL_LINE - would need to make getpwnam flaky
-
-  return NULL;
-}
-
-/** Wrapper around getpwnam() that can use cached result from
- * tor_getpwnam(). Used so that we don't need to give the sandbox access to
- * /etc/passwd.
- *
- * The following fields alone will definitely be copied in the output: pw_uid,
- * pw_gid, pw_name, pw_dir.  Other fields are not present in cached values.
- */
-const struct passwd *
-tor_getpwuid(uid_t uid)
-{
-  struct passwd *pw;
-
-  if ((pw = getpwuid(uid))) {
-    return pw;
-  }
-
-  /* Lookup failed */
-  if (! passwd_cached)
-    return NULL;
-
-  if (uid == passwd_cached->pw_uid)
-    return passwd_cached; // LCOV_EXCL_LINE - would need to make getpwnam flaky
-
-  return NULL;
-}
-#endif
-
-/** Return true iff we were compiled with capability support, and capabilities
- * seem to work. **/
-int
-have_capability_support(void)
-{
-#ifdef HAVE_LINUX_CAPABILITIES
-  cap_t caps = cap_get_proc();
-  if (caps == NULL)
-    return 0;
-  cap_free(caps);
-  return 1;
-#else
-  return 0;
-#endif
-}
-
-#ifdef HAVE_LINUX_CAPABILITIES
-/** Helper. Drop all capabilities but a small set, and set PR_KEEPCAPS as
- * appropriate.
- *
- * If pre_setuid, retain only CAP_NET_BIND_SERVICE, CAP_SETUID, and
- * CAP_SETGID, and use PR_KEEPCAPS to ensure that capabilities persist across
- * setuid().
- *
- * If not pre_setuid, retain only CAP_NET_BIND_SERVICE, and disable
- * PR_KEEPCAPS.
- *
- * Return 0 on success, and -1 on failure.
- */
-static int
-drop_capabilities(int pre_setuid)
-{
-  /* We keep these three capabilities, and these only, as we setuid.
-   * After we setuid, we drop all but the first. */
-  const cap_value_t caplist[] = {
-    CAP_NET_BIND_SERVICE, CAP_SETUID, CAP_SETGID
-  };
-  const char *where = pre_setuid ? "pre-setuid" : "post-setuid";
-  const int n_effective = pre_setuid ? 3 : 1;
-  const int n_permitted = pre_setuid ? 3 : 1;
-  const int n_inheritable = 1;
-  const int keepcaps = pre_setuid ? 1 : 0;
-
-  /* Sets whether we keep capabilities across a setuid. */
-  if (prctl(PR_SET_KEEPCAPS, keepcaps) < 0) {
-    log_warn(LD_CONFIG, "Unable to call prctl() %s: %s",
-             where, strerror(errno));
-    return -1;
-  }
-
-  cap_t caps = cap_get_proc();
-  if (!caps) {
-    log_warn(LD_CONFIG, "Unable to call cap_get_proc() %s: %s",
-             where, strerror(errno));
-    return -1;
-  }
-  cap_clear(caps);
-
-  cap_set_flag(caps, CAP_EFFECTIVE, n_effective, caplist, CAP_SET);
-  cap_set_flag(caps, CAP_PERMITTED, n_permitted, caplist, CAP_SET);
-  cap_set_flag(caps, CAP_INHERITABLE, n_inheritable, caplist, CAP_SET);
-
-  int r = cap_set_proc(caps);
-  cap_free(caps);
-  if (r < 0) {
-    log_warn(LD_CONFIG, "No permission to set capabilities %s: %s",
-             where, strerror(errno));
-    return -1;
-  }
-
-  return 0;
-}
-#endif
-
-/** Call setuid and setgid to run as <b>user</b> and switch to their
- * primary group.  Return 0 on success.  On failure, log and return -1.
- *
- * If SWITCH_ID_KEEP_BINDLOW is set in 'flags', try to use the capability
- * system to retain the abilitity to bind low ports.
- *
- * If SWITCH_ID_WARN_IF_NO_CAPS is set in flags, also warn if we have
- * don't have capability support.
- */
-int
-switch_id(const char *user, const unsigned flags)
-{
-#ifndef _WIN32
-  const struct passwd *pw = NULL;
-  uid_t old_uid;
-  gid_t old_gid;
-  static int have_already_switched_id = 0;
-  const int keep_bindlow = !!(flags & SWITCH_ID_KEEP_BINDLOW);
-  const int warn_if_no_caps = !!(flags & SWITCH_ID_WARN_IF_NO_CAPS);
-
-  tor_assert(user);
-
-  if (have_already_switched_id)
-    return 0;
-
-  /* Log the initial credential state */
-  if (log_credential_status())
-    return -1;
-
-  log_fn(CREDENTIAL_LOG_LEVEL, LD_GENERAL, "Changing user and groups");
-
-  /* Get old UID/GID to check if we changed correctly */
-  old_uid = getuid();
-  old_gid = getgid();
-
-  /* Lookup the user and group information, if we have a problem, bail out. */
-  pw = tor_getpwnam(user);
-  if (pw == NULL) {
-    log_warn(LD_CONFIG, "Error setting configured user: %s not found", user);
-    return -1;
-  }
-
-#ifdef HAVE_LINUX_CAPABILITIES
-  (void) warn_if_no_caps;
-  if (keep_bindlow) {
-    if (drop_capabilities(1))
-      return -1;
-  }
-#else
-  (void) keep_bindlow;
-  if (warn_if_no_caps) {
-    log_warn(LD_CONFIG, "KeepBindCapabilities set, but no capability support "
-             "on this system.");
-  }
-#endif
-
-  /* Properly switch egid,gid,euid,uid here or bail out */
-  if (setgroups(1, &pw->pw_gid)) {
-    log_warn(LD_GENERAL, "Error setting groups to gid %d: \"%s\".",
-             (int)pw->pw_gid, strerror(errno));
-    if (old_uid == pw->pw_uid) {
-      log_warn(LD_GENERAL, "Tor is already running as %s.  You do not need "
-               "the \"User\" option if you are already running as the user "
-               "you want to be.  (If you did not set the User option in your "
-               "torrc, check whether it was specified on the command line "
-               "by a startup script.)", user);
-    } else {
-      log_warn(LD_GENERAL, "If you set the \"User\" option, you must start Tor"
-               " as root.");
-    }
-    return -1;
-  }
-
-  if (setegid(pw->pw_gid)) {
-    log_warn(LD_GENERAL, "Error setting egid to %d: %s",
-             (int)pw->pw_gid, strerror(errno));
-    return -1;
-  }
-
-  if (setgid(pw->pw_gid)) {
-    log_warn(LD_GENERAL, "Error setting gid to %d: %s",
-             (int)pw->pw_gid, strerror(errno));
-    return -1;
-  }
-
-  if (setuid(pw->pw_uid)) {
-    log_warn(LD_GENERAL, "Error setting configured uid to %s (%d): %s",
-             user, (int)pw->pw_uid, strerror(errno));
-    return -1;
-  }
-
-  if (seteuid(pw->pw_uid)) {
-    log_warn(LD_GENERAL, "Error setting configured euid to %s (%d): %s",
-             user, (int)pw->pw_uid, strerror(errno));
-    return -1;
-  }
-
-  /* This is how OpenBSD rolls:
-  if (setgroups(1, &pw->pw_gid) || setegid(pw->pw_gid) ||
-      setgid(pw->pw_gid) || setuid(pw->pw_uid) || seteuid(pw->pw_uid)) {
-      setgid(pw->pw_gid) || seteuid(pw->pw_uid) || setuid(pw->pw_uid)) {
-    log_warn(LD_GENERAL, "Error setting configured UID/GID: %s",
-    strerror(errno));
-    return -1;
-  }
-  */
-
-  /* We've properly switched egid, gid, euid, uid, and supplementary groups if
-   * we're here. */
-#ifdef HAVE_LINUX_CAPABILITIES
-  if (keep_bindlow) {
-    if (drop_capabilities(0))
-      return -1;
-  }
-#endif
-
-#if !defined(CYGWIN) && !defined(__CYGWIN__)
-  /* If we tried to drop privilege to a group/user other than root, attempt to
-   * restore root (E)(U|G)ID, and abort if the operation succeeds */
-
-  /* Only check for privilege dropping if we were asked to be non-root */
-  if (pw->pw_uid) {
-    /* Try changing GID/EGID */
-    if (pw->pw_gid != old_gid &&
-        (setgid(old_gid) != -1 || setegid(old_gid) != -1)) {
-      log_warn(LD_GENERAL, "Was able to restore group credentials even after "
-               "switching GID: this means that the setgid code didn't work.");
-      return -1;
-    }
-
-    /* Try changing UID/EUID */
-    if (pw->pw_uid != old_uid &&
-        (setuid(old_uid) != -1 || seteuid(old_uid) != -1)) {
-      log_warn(LD_GENERAL, "Was able to restore user credentials even after "
-               "switching UID: this means that the setuid code didn't work.");
-      return -1;
-    }
-  }
-#endif
-
-  /* Check what really happened */
-  if (log_credential_status()) {
-    return -1;
-  }
-
-  have_already_switched_id = 1; /* mark success so we never try again */
-
-#if defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && defined(HAVE_PRCTL)
-#ifdef PR_SET_DUMPABLE
-  if (pw->pw_uid) {
-    /* Re-enable core dumps if we're not running as root. */
-    log_info(LD_CONFIG, "Re-enabling coredumps");
-    if (prctl(PR_SET_DUMPABLE, 1)) {
-      log_warn(LD_CONFIG, "Unable to re-enable coredumps: %s",strerror(errno));
-    }
-  }
-#endif
-#endif
-  return 0;
-
-#else
-  (void)user;
-  (void)flags;
-
-  log_warn(LD_CONFIG, "Switching users is unsupported on your OS.");
-  return -1;
-#endif
-}
-
-/* We only use the linux prctl for now. There is no Win32 support; this may
- * also work on various BSD systems and Mac OS X - send testing feedback!
- *
- * On recent Gnu/Linux kernels it is possible to create a system-wide policy
- * that will prevent non-root processes from attaching to other processes
- * unless they are the parent process; thus gdb can attach to programs that
- * they execute but they cannot attach to other processes running as the same
- * user. The system wide policy may be set with the sysctl
- * kernel.yama.ptrace_scope or by inspecting
- * /proc/sys/kernel/yama/ptrace_scope and it is 1 by default on Ubuntu 11.04.
- *
- * This ptrace scope will be ignored on Gnu/Linux for users with
- * CAP_SYS_PTRACE and so it is very likely that root will still be able to
- * attach to the Tor process.
- */
-/** Attempt to disable debugger attachment: return 1 on success, -1 on
- * failure, and 0 if we don't know how to try on this platform. */
-int
-tor_disable_debugger_attach(void)
-{
-  int r, attempted;
-  r = -1;
-  attempted = 0;
-  log_debug(LD_CONFIG,
-            "Attemping to disable debugger attachment to Tor for "
-            "unprivileged users.");
-#if defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && defined(HAVE_PRCTL)
-#ifdef PR_SET_DUMPABLE
-  attempted = 1;
-  r = prctl(PR_SET_DUMPABLE, 0);
-#endif
-#endif
-#if defined(__APPLE__) && defined(PT_DENY_ATTACH)
-  if (r < 0) {
-    attempted = 1;
-    r = ptrace(PT_DENY_ATTACH, 0, 0, 0);
-  }
-#endif
-
-  // XXX: TODO - Mac OS X has dtrace and this may be disabled.
-  // XXX: TODO - Windows probably has something similar
-  if (r == 0 && attempted) {
-    log_debug(LD_CONFIG,"Debugger attachment disabled for "
-              "unprivileged users.");
-    return 1;
-  } else if (attempted) {
-    log_warn(LD_CONFIG, "Unable to disable debugger attaching: %s",
-             strerror(errno));
-  }
-  return r;
-}
-
-#ifdef HAVE_PWD_H
-/** Allocate and return a string containing the home directory for the
- * user <b>username</b>. Only works on posix-like systems. */
-char *
-get_user_homedir(const char *username)
-{
-  const struct passwd *pw;
-  tor_assert(username);
-
-  if (!(pw = tor_getpwnam(username))) {
-    log_err(LD_CONFIG,"User \"%s\" not found.", username);
-    return NULL;
-  }
-  return tor_strdup(pw->pw_dir);
-}
-#endif
-
-/** Modify <b>fname</b> to contain the name of its parent directory.  Doesn't
- * actually examine the filesystem; does a purely syntactic modification.
- *
- * The parent of the root director is considered to be iteself.
- *
- * Path separators are the forward slash (/) everywhere and additionally
- * the backslash (\) on Win32.
- *
- * Cuts off any number of trailing path separators but otherwise ignores
- * them for purposes of finding the parent directory.
- *
- * Returns 0 if a parent directory was successfully found, -1 otherwise (fname
- * did not have any path separators or only had them at the end).
- * */
-int
-get_parent_directory(char *fname)
-{
-  char *cp;
-  int at_end = 1;
-  tor_assert(fname);
-#ifdef _WIN32
-  /* If we start with, say, c:, then don't consider that the start of the path
-   */
-  if (fname[0] && fname[1] == ':') {
-    fname += 2;
-  }
-#endif
-  /* Now we want to remove all path-separators at the end of the string,
-   * and to remove the end of the string starting with the path separator
-   * before the last non-path-separator.  In perl, this would be
-   *   s#[/]*$##; s#/[^/]*$##;
-   * on a unixy platform.
-   */
-  cp = fname + strlen(fname);
-  at_end = 1;
-  while (--cp >= fname) {
-    int is_sep = (*cp == '/'
-#ifdef _WIN32
-                  || *cp == '\\'
-#endif
-                  );
-    if (is_sep) {
-      if (cp == fname) {
-        /* This is the first separator in the file name; don't remove it! */
-        cp[1] = '\0';
-        return 0;
-      }
-      *cp = '\0';
-      if (! at_end)
-        return 0;
-    } else {
-      at_end = 0;
-    }
-  }
-  return -1;
-}
-
-#ifndef _WIN32
-/** Return a newly allocated string containing the output of getcwd(). Return
- * NULL on failure. (We can't just use getcwd() into a PATH_MAX buffer, since
- * Hurd hasn't got a PATH_MAX.)
- */
-static char *
-alloc_getcwd(void)
-{
-#ifdef PATH_MAX
-#define MAX_CWD PATH_MAX
-#else
-#define MAX_CWD 4096
-#endif
-
-  char path_buf[MAX_CWD];
-  char *path = getcwd(path_buf, sizeof(path_buf));
-  return path ? tor_strdup(path) : NULL;
-}
-#endif
-
-/** Expand possibly relative path <b>fname</b> to an absolute path.
- * Return a newly allocated string, possibly equal to <b>fname</b>. */
-char *
-make_path_absolute(char *fname)
-{
-#ifdef _WIN32
-  char *absfname_malloced = _fullpath(NULL, fname, 1);
-
-  /* We don't want to assume that tor_free can free a string allocated
-   * with malloc.  On failure, return fname (it's better than nothing). */
-  char *absfname = tor_strdup(absfname_malloced ? absfname_malloced : fname);
-  if (absfname_malloced) raw_free(absfname_malloced);
-
-  return absfname;
-#else
-  char *absfname = NULL, *path = NULL;
-
-  tor_assert(fname);
-
-  if (fname[0] == '/') {
-    absfname = tor_strdup(fname);
-  } else {
-    path = alloc_getcwd();
-    if (path) {
-      tor_asprintf(&absfname, "%s/%s", path, fname);
-      tor_free(path);
-    } else {
-      /* LCOV_EXCL_START Can't make getcwd fail. */
-      /* If getcwd failed, the best we can do here is keep using the
-       * relative path.  (Perhaps / isn't readable by this UID/GID.) */
-      log_warn(LD_GENERAL, "Unable to find current working directory: %s",
-               strerror(errno));
-      absfname = tor_strdup(fname);
-      /* LCOV_EXCL_STOP */
-    }
-  }
-  return absfname;
-#endif
-}
-
-#ifndef HAVE__NSGETENVIRON
-#ifndef HAVE_EXTERN_ENVIRON_DECLARED
-/* Some platforms declare environ under some circumstances, others don't. */
-#ifndef RUNNING_DOXYGEN
-extern char **environ;
-#endif
-#endif
-#endif
-
-/** Return the current environment. This is a portable replacement for
- * 'environ'. */
-char **
-get_environment(void)
-{
-#ifdef HAVE__NSGETENVIRON
-  /* This is for compatibility between OSX versions.  Otherwise (for example)
-   * when we do a mostly-static build on OSX 10.7, the resulting binary won't
-   * work on OSX 10.6. */
-  return *_NSGetEnviron();
-#else
-  return environ;
-#endif
-}
-
-/** Get name of current host and write it to <b>name</b> array, whose
- * length is specified by <b>namelen</b> argument. Return 0 upon
- * successfull completion; otherwise return return -1. (Currently,
- * this function is merely a mockable wrapper for POSIX gethostname().)
- */
-MOCK_IMPL(int,
-tor_gethostname,(char *name, size_t namelen))
-{
-   return gethostname(name,namelen);
-}
-
-/** Set *addr to the IP address (in dotted-quad notation) stored in *str.
- * Return 1 on success, 0 if *str is badly formatted.
- * (Like inet_aton(str,addr), but works on Windows and Solaris.)
- */
-int
-tor_inet_aton(const char *str, struct in_addr* addr)
-{
-  unsigned a,b,c,d;
-  char more;
-  if (tor_sscanf(str, "%3u.%3u.%3u.%3u%c", &a,&b,&c,&d,&more) != 4)
-    return 0;
-  if (a > 255) return 0;
-  if (b > 255) return 0;
-  if (c > 255) return 0;
-  if (d > 255) return 0;
-  addr->s_addr = htonl((a<<24) | (b<<16) | (c<<8) | d);
-  return 1;
-}
-
-/** Given <b>af</b>==AF_INET and <b>src</b> a struct in_addr, or
- * <b>af</b>==AF_INET6 and <b>src</b> a struct in6_addr, try to format the
- * address and store it in the <b>len</b>-byte buffer <b>dst</b>.  Returns
- * <b>dst</b> on success, NULL on failure.
- *
- * (Like inet_ntop(af,src,dst,len), but works on platforms that don't have it:
- * Tor sometimes needs to format ipv6 addresses even on platforms without ipv6
- * support.) */
-const char *
-tor_inet_ntop(int af, const void *src, char *dst, size_t len)
-{
-  if (af == AF_INET) {
-    if (tor_inet_ntoa(src, dst, len) < 0)
-      return NULL;
-    else
-      return dst;
-  } else if (af == AF_INET6) {
-    const struct in6_addr *addr = src;
-    char buf[64], *cp;
-    int longestGapLen = 0, longestGapPos = -1, i,
-      curGapPos = -1, curGapLen = 0;
-    uint16_t words[8];
-    for (i = 0; i < 8; ++i) {
-      words[i] = (((uint16_t)addr->s6_addr[2*i])<<8) + addr->s6_addr[2*i+1];
-    }
-    if (words[0] == 0 && words[1] == 0 && words[2] == 0 && words[3] == 0 &&
-        words[4] == 0 && ((words[5] == 0 && words[6] && words[7]) ||
-                          (words[5] == 0xffff))) {
-      /* This is an IPv4 address. */
-      if (words[5] == 0) {
-        tor_snprintf(buf, sizeof(buf), "::%d.%d.%d.%d",
-                     addr->s6_addr[12], addr->s6_addr[13],
-                     addr->s6_addr[14], addr->s6_addr[15]);
-      } else {
-        tor_snprintf(buf, sizeof(buf), "::%x:%d.%d.%d.%d", words[5],
-                     addr->s6_addr[12], addr->s6_addr[13],
-                     addr->s6_addr[14], addr->s6_addr[15]);
-      }
-      if ((strlen(buf) + 1) > len) /* +1 for \0 */
-        return NULL;
-      strlcpy(dst, buf, len);
-      return dst;
-    }
-    i = 0;
-    while (i < 8) {
-      if (words[i] == 0) {
-        curGapPos = i++;
-        curGapLen = 1;
-        while (i<8 && words[i] == 0) {
-          ++i; ++curGapLen;
-        }
-        if (curGapLen > longestGapLen) {
-          longestGapPos = curGapPos;
-          longestGapLen = curGapLen;
-        }
-      } else {
-        ++i;
-      }
-    }
-    if (longestGapLen<=1)
-      longestGapPos = -1;
-
-    cp = buf;
-    for (i = 0; i < 8; ++i) {
-      if (words[i] == 0 && longestGapPos == i) {
-        if (i == 0)
-          *cp++ = ':';
-        *cp++ = ':';
-        while (i < 8 && words[i] == 0)
-          ++i;
-        --i; /* to compensate for loop increment. */
-      } else {
-        tor_snprintf(cp, sizeof(buf)-(cp-buf), "%x", (unsigned)words[i]);
-        cp += strlen(cp);
-        if (i != 7)
-          *cp++ = ':';
-      }
-    }
-    *cp = '\0';
-    if ((strlen(buf) + 1) > len) /* +1 for \0 */
-      return NULL;
-    strlcpy(dst, buf, len);
-    return dst;
-  } else {
-    return NULL;
-  }
-}
-
-/** Given <b>af</b>==AF_INET or <b>af</b>==AF_INET6, and a string <b>src</b>
- * encoding an IPv4 address or IPv6 address correspondingly, try to parse the
- * address and store the result in <b>dst</b> (which must have space for a
- * struct in_addr or a struct in6_addr, as appropriate).  Return 1 on success,
- * 0 on a bad parse, and -1 on a bad <b>af</b>.
- *
- * (Like inet_pton(af,src,dst) but works on platforms that don't have it: Tor
- * sometimes needs to format ipv6 addresses even on platforms without ipv6
- * support.) */
-int
-tor_inet_pton(int af, const char *src, void *dst)
-{
-  if (af == AF_INET) {
-    return tor_inet_aton(src, dst);
-  } else if (af == AF_INET6) {
-    struct in6_addr *out = dst;
-    uint16_t words[8];
-    int gapPos = -1, i, setWords=0;
-    const char *dot = strchr(src, '.');
-    const char *eow; /* end of words. */
-    if (dot == src)
-      return 0;
-    else if (!dot)
-      eow = src+strlen(src);
-    else {
-      unsigned byte1,byte2,byte3,byte4;
-      char more;
-      for (eow = dot-1; eow > src && TOR_ISDIGIT(*eow); --eow)
-        ;
-      if (*eow != ':')
-        return 0;
-      ++eow;
-
-      /* We use "scanf" because some platform inet_aton()s are too lax
-       * about IPv4 addresses of the form "1.2.3" */
-      if (tor_sscanf(eow, "%3u.%3u.%3u.%3u%c",
-                     &byte1,&byte2,&byte3,&byte4,&more) != 4)
-        return 0;
-
-      if (byte1 > 255 || byte2 > 255 || byte3 > 255 || byte4 > 255)
-        return 0;
-
-      words[6] = (byte1<<8) | byte2;
-      words[7] = (byte3<<8) | byte4;
-      setWords += 2;
-    }
-
-    i = 0;
-    while (src < eow) {
-      if (i > 7)
-        return 0;
-      if (TOR_ISXDIGIT(*src)) {
-        char *next;
-        ssize_t len;
-        long r = strtol(src, &next, 16);
-        if (next == NULL || next == src) {
-          /* The 'next == src' error case can happen on versions of openbsd
-           * where treats "0xfoo" as an error, rather than as "0" followed by
-           * "xfoo". */
-          return 0;
-        }
-
-        len = *next == '\0' ? eow - src : next - src;
-        if (len > 4)
-          return 0;
-        if (len > 1 && !TOR_ISXDIGIT(src[1]))
-          return 0; /* 0x is not valid */
-
-        tor_assert(r >= 0);
-        tor_assert(r < 65536);
-        words[i++] = (uint16_t)r;
-        setWords++;
-        src = next;
-        if (*src != ':' && src != eow)
-          return 0;
-        ++src;
-      } else if (*src == ':' && i > 0 && gapPos == -1) {
-        gapPos = i;
-        ++src;
-      } else if (*src == ':' && i == 0 && src+1 < eow && src[1] == ':' &&
-                 gapPos == -1) {
-        gapPos = i;
-        src += 2;
-      } else {
-        return 0;
-      }
-    }
-
-    if (setWords > 8 ||
-        (setWords == 8 && gapPos != -1) ||
-        (setWords < 8 && gapPos == -1))
-      return 0;
-
-    if (gapPos >= 0) {
-      int nToMove = setWords - (dot ? 2 : 0) - gapPos;
-      int gapLen = 8 - setWords;
-      tor_assert(nToMove >= 0);
-      memmove(&words[gapPos+gapLen], &words[gapPos],
-              sizeof(uint16_t)*nToMove);
-      memset(&words[gapPos], 0, sizeof(uint16_t)*gapLen);
-    }
-    for (i = 0; i < 8; ++i) {
-      out->s6_addr[2*i  ] = words[i] >> 8;
-      out->s6_addr[2*i+1] = words[i] & 0xff;
-    }
-
-    return 1;
-  } else {
-    return -1;
-  }
-}
-
-/** Similar behavior to Unix gethostbyname: resolve <b>name</b>, and set
- * *<b>addr</b> to the proper IP address, in host byte order.  Returns 0
- * on success, -1 on failure; 1 on transient failure.
- *
- * (This function exists because standard windows gethostbyname
- * doesn't treat raw IP addresses properly.)
- */
-
-MOCK_IMPL(int,
-tor_lookup_hostname,(const char *name, uint32_t *addr))
-{
-  tor_addr_t myaddr;
-  int ret;
-
-  if ((ret = tor_addr_lookup(name, AF_INET, &myaddr)))
-    return ret;
-
-  if (tor_addr_family(&myaddr) == AF_INET) {
-    *addr = tor_addr_to_ipv4h(&myaddr);
-    return ret;
-  }
-
-  return -1;
-}
-
-/** Hold the result of our call to <b>uname</b>. */
-static char uname_result[256];
-/** True iff uname_result is set. */
-static int uname_result_is_set = 0;
-
-/** Return a pointer to a description of our platform.
- */
-MOCK_IMPL(const char *, get_uname, (void))
-{
-#ifdef HAVE_UNAME
-  struct utsname u;
-#endif
-  if (!uname_result_is_set) {
-#ifdef HAVE_UNAME
-    if (uname(&u) != -1) {
-      /* (Linux says 0 is success, Solaris says 1 is success) */
-      strlcpy(uname_result, u.sysname, sizeof(uname_result));
-    } else
-#endif
-      {
-#ifdef _WIN32
-        OSVERSIONINFOEX info;
-        int i;
-        int is_client = 0;
-        int is_server = 0;
-        const char *plat = NULL;
-        static struct {
-          unsigned major; unsigned minor;
-          const char *client_version; const char *server_version;
-        } win_version_table[] = {
-    /* This table must be sorted in descending order.
-     * Sources:
-     *   https://en.wikipedia.org/wiki/List_of_Microsoft_Windows_versions
-     *   https://docs.microsoft.com/en-us/windows/desktop/api/winnt/
-     *     ns-winnt-_osversioninfoexa#remarks
-     */
-    /* Windows Server 2019 is indistinguishable from Windows Server 2016
-     * using GetVersionEx().
-    { 10,  0, NULL,                        "Windows Server 2019" }, */
-    { 10,  0, "Windows 10",                "Windows Server 2016" },
-    {  6,  3, "Windows 8.1",               "Windows Server 2012 R2" },
-    {  6,  2, "Windows 8",                 "Windows Server 2012" },
-    {  6,  1, "Windows 7",                 "Windows Server 2008 R2" },
-    {  6,  0, "Windows Vista",             "Windows Server 2008" },
-    {  5,  2, "Windows XP Professional",   "Windows Server 2003" },
-    /* Windows XP did not have a server version, but we need something here */
-    {  5,  1, "Windows XP",                "Windows XP Server" },
-    {  5,  0, "Windows 2000 Professional", "Windows 2000 Server" },
-    /* Earlier versions are not supported by GetVersionEx(). */
-    {  0,  0, NULL,                        NULL }
-        };
-        memset(&info, 0, sizeof(info));
-        info.dwOSVersionInfoSize = sizeof(info);
-        if (! GetVersionEx((LPOSVERSIONINFO)&info)) {
-          strlcpy(uname_result, "Bizarre version of Windows where GetVersionEx"
-                  " doesn't work.", sizeof(uname_result));
-          uname_result_is_set = 1;
-          return uname_result;
-        }
-#ifdef VER_NT_SERVER
-        if (info.wProductType == VER_NT_SERVER ||
-            info.wProductType == VER_NT_DOMAIN_CONTROLLER) {
-          is_server = 1;
-        } else {
-          is_client = 1;
-        }
-#endif
-        /* Search the version table for a matching version */
-        for (i=0; win_version_table[i].major>0; ++i) {
-          if (win_version_table[i].major == info.dwMajorVersion &&
-              win_version_table[i].minor == info.dwMinorVersion) {
-            if (is_server) {
-              plat = win_version_table[i].server_version;
-            } else {
-              /* Use client versions for clients, and when we don't know if it
-              * is a client or a server. */
-              plat = win_version_table[i].client_version;
-            }
-            break;
-          }
-        }
-        if (plat) {
-          strlcpy(uname_result, plat, sizeof(uname_result));
-        } else {
-          if (info.dwMajorVersion > win_version_table[0].major ||
-              (info.dwMajorVersion == win_version_table[0].major &&
-               info.dwMinorVersion > win_version_table[0].minor))
-            tor_snprintf(uname_result, sizeof(uname_result),
-                         "Very recent version of Windows [major=%d,minor=%d]",
-                         (int)info.dwMajorVersion,(int)info.dwMinorVersion);
-          else
-            tor_snprintf(uname_result, sizeof(uname_result),
-                         "Unrecognized version of Windows [major=%d,minor=%d]",
-                         (int)info.dwMajorVersion,(int)info.dwMinorVersion);
-        }
-        /* Now append extra information to the name.
-         *
-         * Microsoft's API documentation says that on Windows 8.1 and later,
-         * GetVersionEx returns Windows 8 (6.2) for applications without an
-         * app compatibility manifest (including tor's default build).
-         *
-         * But in our testing, we have seen the actual Windows version on
-         * Windows Server 2012 R2, even without a manifest. */
-        if (info.dwMajorVersion > 6 ||
-            (info.dwMajorVersion == 6 && info.dwMinorVersion >= 2)) {
-          /* When GetVersionEx() returns Windows 8, the actual OS may be any
-           * later version. */
-          strlcat(uname_result, " [or later]", sizeof(uname_result));
-        }
-        /* When we don't know if the OS is a client or server version, we use
-         * the client version, and this qualifier. */
-        if (!is_server && !is_client) {
-          strlcat(uname_result, " [client or server]", sizeof(uname_result));
-        }
-#else
-        /* LCOV_EXCL_START -- can't provoke uname failure */
-        strlcpy(uname_result, "Unknown platform", sizeof(uname_result));
-        /* LCOV_EXCL_STOP */
-#endif
-      }
-    uname_result_is_set = 1;
-  }
-  return uname_result;
-}
-
-/*
- *   Process control
- */
-
-/** Implementation logic for compute_num_cpus(). */
-static int
-compute_num_cpus_impl(void)
-{
-#ifdef _WIN32
-  SYSTEM_INFO info;
-  memset(&info, 0, sizeof(info));
-  GetSystemInfo(&info);
-  if (info.dwNumberOfProcessors >= 1 && info.dwNumberOfProcessors < INT_MAX)
-    return (int)info.dwNumberOfProcessors;
-  else
-    return -1;
-#elif defined(HAVE_SYSCONF)
-#ifdef _SC_NPROCESSORS_CONF
-  long cpus_conf = sysconf(_SC_NPROCESSORS_CONF);
-#else
-  long cpus_conf = -1;
-#endif
-#ifdef _SC_NPROCESSORS_ONLN
-  long cpus_onln = sysconf(_SC_NPROCESSORS_ONLN);
-#else
-  long cpus_onln = -1;
-#endif
-  long cpus = -1;
-
-  if (cpus_conf > 0 && cpus_onln < 0) {
-    cpus = cpus_conf;
-  } else if (cpus_onln > 0 && cpus_conf < 0) {
-    cpus = cpus_onln;
-  } else if (cpus_onln > 0 && cpus_conf > 0) {
-    if (cpus_onln < cpus_conf) {
-      log_notice(LD_GENERAL, "I think we have %ld CPUS, but only %ld of them "
-                 "are available. Telling Tor to only use %ld. You can over"
-                 "ride this with the NumCPUs option",
-                 cpus_conf, cpus_onln, cpus_onln);
-    }
-    cpus = cpus_onln;
-  }
-
-  if (cpus >= 1 && cpus < INT_MAX)
-    return (int)cpus;
-  else
-    return -1;
-#else
-  return -1;
-#endif
-}
-
-#define MAX_DETECTABLE_CPUS 16
-
-/** Return how many CPUs we are running with.  We assume that nobody is
- * using hot-swappable CPUs, so we don't recompute this after the first
- * time.  Return -1 if we don't know how to tell the number of CPUs on this
- * system.
- */
-int
-compute_num_cpus(void)
-{
-  static int num_cpus = -2;
-  if (num_cpus == -2) {
-    num_cpus = compute_num_cpus_impl();
-    tor_assert(num_cpus != -2);
-    if (num_cpus > MAX_DETECTABLE_CPUS) {
-      /* LCOV_EXCL_START */
-      log_notice(LD_GENERAL, "Wow!  I detected that you have %d CPUs. I "
-                 "will not autodetect any more than %d, though.  If you "
-                 "want to configure more, set NumCPUs in your torrc",
-                 num_cpus, MAX_DETECTABLE_CPUS);
-      num_cpus = MAX_DETECTABLE_CPUS;
-      /* LCOV_EXCL_STOP */
-    }
-  }
-  return num_cpus;
-}
-
-#if !defined(_WIN32)
-/** Defined iff we need to add locks when defining fake versions of reentrant
- * versions of time-related functions. */
-#define TIME_FNS_NEED_LOCKS
-#endif
-
-/** Helper: Deal with confused or out-of-bounds values from localtime_r and
- * friends.  (On some platforms, they can give out-of-bounds values or can
- * return NULL.)  If <b>islocal</b>, this is a localtime result; otherwise
- * it's from gmtime.  The function returned <b>r</b>, when given <b>timep</b>
- * as its input. If we need to store new results, store them in
- * <b>resultbuf</b>. */
-static struct tm *
-correct_tm(int islocal, const time_t *timep, struct tm *resultbuf,
-           struct tm *r)
-{
-  const char *outcome;
-
-  if (PREDICT_LIKELY(r)) {
-    /* We can't strftime dates after 9999 CE, and we want to avoid dates
-     * before 1 CE (avoiding the year 0 issue and negative years). */
-    if (r->tm_year > 8099) {
-      r->tm_year = 8099;
-      r->tm_mon = 11;
-      r->tm_mday = 31;
-      r->tm_yday = 364;
-      r->tm_wday = 6;
-      r->tm_hour = 23;
-      r->tm_min = 59;
-      r->tm_sec = 59;
-    } else if (r->tm_year < (1-1900)) {
-      r->tm_year = (1-1900);
-      r->tm_mon = 0;
-      r->tm_mday = 1;
-      r->tm_yday = 0;
-      r->tm_wday = 0;
-      r->tm_hour = 0;
-      r->tm_min = 0;
-      r->tm_sec = 0;
-    }
-    return r;
-  }
-
-  /* If we get here, gmtime or localtime returned NULL. It might have done
-   * this because of overrun or underrun, or it might have done it because of
-   * some other weird issue. */
-  if (timep) {
-    if (*timep < 0) {
-      r = resultbuf;
-      r->tm_year = 70; /* 1970 CE */
-      r->tm_mon = 0;
-      r->tm_mday = 1;
-      r->tm_yday = 0;
-      r->tm_wday = 0;
-      r->tm_hour = 0;
-      r->tm_min = 0 ;
-      r->tm_sec = 0;
-      outcome = "Rounding up to 1970";
-      goto done;
-    } else if (*timep >= INT32_MAX) {
-      /* Rounding down to INT32_MAX isn't so great, but keep in mind that we
-       * only do it if gmtime/localtime tells us NULL. */
-      r = resultbuf;
-      r->tm_year = 137; /* 2037 CE */
-      r->tm_mon = 11;
-      r->tm_mday = 31;
-      r->tm_yday = 364;
-      r->tm_wday = 6;
-      r->tm_hour = 23;
-      r->tm_min = 59;
-      r->tm_sec = 59;
-      outcome = "Rounding down to 2037";
-      goto done;
-    }
-  }
-
-  /* If we get here, then gmtime/localtime failed without getting an extreme
-   * value for *timep */
-  /* LCOV_EXCL_START */
-  tor_fragile_assert();
-  r = resultbuf;
-  memset(resultbuf, 0, sizeof(struct tm));
-  outcome="can't recover";
-  /* LCOV_EXCL_STOP */
- done:
-  log_warn(LD_BUG, "%s("I64_FORMAT") failed with error %s: %s",
-           islocal?"localtime":"gmtime",
-           timep?I64_PRINTF_ARG(*timep):0,
-           strerror(errno),
-           outcome);
-  return r;
-}
-
-/** @{ */
-/** As localtime_r, but defined for platforms that don't have it:
- *
- * Convert *<b>timep</b> to a struct tm in local time, and store the value in
- * *<b>result</b>.  Return the result on success, or NULL on failure.
- */
-#ifdef HAVE_LOCALTIME_R
-struct tm *
-tor_localtime_r(const time_t *timep, struct tm *result)
-{
-  struct tm *r;
-  r = localtime_r(timep, result);
-  return correct_tm(1, timep, result, r);
-}
-#elif defined(TIME_FNS_NEED_LOCKS)
-struct tm *
-tor_localtime_r(const time_t *timep, struct tm *result)
-{
-  struct tm *r;
-  static tor_mutex_t *m=NULL;
-  if (!m) { m=tor_mutex_new(); }
-  tor_assert(result);
-  tor_mutex_acquire(m);
-  r = localtime(timep);
-  if (r)
-    memcpy(result, r, sizeof(struct tm));
-  tor_mutex_release(m);
-  return correct_tm(1, timep, result, r);
-}
-#else
-struct tm *
-tor_localtime_r(const time_t *timep, struct tm *result)
-{
-  struct tm *r;
-  tor_assert(result);
-  r = localtime(timep);
-  if (r)
-    memcpy(result, r, sizeof(struct tm));
-  return correct_tm(1, timep, result, r);
-}
-#endif
-/** @} */
-
-/** @{ */
-/** As gmtime_r, but defined for platforms that don't have it:
- *
- * Convert *<b>timep</b> to a struct tm in UTC, and store the value in
- * *<b>result</b>.  Return the result on success, or NULL on failure.
- */
-#ifdef HAVE_GMTIME_R
-struct tm *
-tor_gmtime_r(const time_t *timep, struct tm *result)
-{
-  struct tm *r;
-  r = gmtime_r(timep, result);
-  return correct_tm(0, timep, result, r);
-}
-#elif defined(TIME_FNS_NEED_LOCKS)
-struct tm *
-tor_gmtime_r(const time_t *timep, struct tm *result)
-{
-  struct tm *r;
-  static tor_mutex_t *m=NULL;
-  if (!m) { m=tor_mutex_new(); }
-  tor_assert(result);
-  tor_mutex_acquire(m);
-  r = gmtime(timep);
-  if (r)
-    memcpy(result, r, sizeof(struct tm));
-  tor_mutex_release(m);
-  return correct_tm(0, timep, result, r);
-}
-#else
-struct tm *
-tor_gmtime_r(const time_t *timep, struct tm *result)
-{
-  struct tm *r;
-  tor_assert(result);
-  r = gmtime(timep);
-  if (r)
-    memcpy(result, r, sizeof(struct tm));
-  return correct_tm(0, timep, result, r);
-}
-#endif
-
-#if defined(HAVE_MLOCKALL) && HAVE_DECL_MLOCKALL && defined(RLIMIT_MEMLOCK)
-/** Attempt to raise the current and max rlimit to infinity for our process.
- * This only needs to be done once and can probably only be done when we have
- * not already dropped privileges.
- */
-static int
-tor_set_max_memlock(void)
-{
-  /* Future consideration for Windows is probably SetProcessWorkingSetSize
-   * This is similar to setting the memory rlimit of RLIMIT_MEMLOCK
-   * http://msdn.microsoft.com/en-us/library/ms686234(VS.85).aspx
-   */
-
-  struct rlimit limit;
-
-  /* RLIM_INFINITY is -1 on some platforms. */
-  limit.rlim_cur = RLIM_INFINITY;
-  limit.rlim_max = RLIM_INFINITY;
-
-  if (setrlimit(RLIMIT_MEMLOCK, &limit) == -1) {
-    if (errno == EPERM) {
-      log_warn(LD_GENERAL, "You appear to lack permissions to change memory "
-                           "limits. Are you root?");
-    }
-    log_warn(LD_GENERAL, "Unable to raise RLIMIT_MEMLOCK: %s",
-             strerror(errno));
-    return -1;
-  }
-
-  return 0;
-}
-#endif
-
-/** Attempt to lock all current and all future memory pages.
- * This should only be called once and while we're privileged.
- * Like mlockall() we return 0 when we're successful and -1 when we're not.
- * Unlike mlockall() we return 1 if we've already attempted to lock memory.
- */
-int
-tor_mlockall(void)
-{
-  static int memory_lock_attempted = 0;
-
-  if (memory_lock_attempted) {
-    return 1;
-  }
-
-  memory_lock_attempted = 1;
-
-  /*
-   * Future consideration for Windows may be VirtualLock
-   * VirtualLock appears to implement mlock() but not mlockall()
-   *
-   * http://msdn.microsoft.com/en-us/library/aa366895(VS.85).aspx
-   */
-
-#if defined(HAVE_MLOCKALL) && HAVE_DECL_MLOCKALL && defined(RLIMIT_MEMLOCK)
-  if (tor_set_max_memlock() == 0) {
-    log_debug(LD_GENERAL, "RLIMIT_MEMLOCK is now set to RLIM_INFINITY.");
-  }
-
-  if (mlockall(MCL_CURRENT|MCL_FUTURE) == 0) {
-    log_info(LD_GENERAL, "Insecure OS paging is effectively disabled.");
-    return 0;
-  } else {
-    if (errno == ENOSYS) {
-      /* Apple - it's 2009! I'm looking at you. Grrr. */
-      log_notice(LD_GENERAL, "It appears that mlockall() is not available on "
-                             "your platform.");
-    } else if (errno == EPERM) {
-      log_notice(LD_GENERAL, "It appears that you lack the permissions to "
-                             "lock memory. Are you root?");
-    }
-    log_notice(LD_GENERAL, "Unable to lock all current and future memory "
-                           "pages: %s", strerror(errno));
-    return -1;
-  }
-#else
-  log_warn(LD_GENERAL, "Unable to lock memory pages. mlockall() unsupported?");
-  return -1;
-#endif
-}
-
-/**
- * On Windows, WSAEWOULDBLOCK is not always correct: when you see it,
- * you need to ask the socket for its actual errno.  Also, you need to
- * get your errors from WSAGetLastError, not errno.  (If you supply a
- * socket of -1, we check WSAGetLastError, but don't correct
- * WSAEWOULDBLOCKs.)
- *
- * The upshot of all of this is that when a socket call fails, you
- * should call tor_socket_errno <em>at most once</em> on the failing
- * socket to get the error.
- */
-#if defined(_WIN32)
-int
-tor_socket_errno(tor_socket_t sock)
-{
-  int optval, optvallen=sizeof(optval);
-  int err = WSAGetLastError();
-  if (err == WSAEWOULDBLOCK && SOCKET_OK(sock)) {
-    if (getsockopt(sock, SOL_SOCKET, SO_ERROR, (void*)&optval, &optvallen))
-      return err;
-    if (optval)
-      return optval;
-  }
-  return err;
-}
-#endif
-
-#if defined(_WIN32)
-#define E(code, s) { code, (s " [" #code " ]") }
-struct { int code; const char *msg; } windows_socket_errors[] = {
-  E(WSAEINTR, "Interrupted function call"),
-  E(WSAEACCES, "Permission denied"),
-  E(WSAEFAULT, "Bad address"),
-  E(WSAEINVAL, "Invalid argument"),
-  E(WSAEMFILE, "Too many open files"),
-  E(WSAEWOULDBLOCK,  "Resource temporarily unavailable"),
-  E(WSAEINPROGRESS, "Operation now in progress"),
-  E(WSAEALREADY, "Operation already in progress"),
-  E(WSAENOTSOCK, "Socket operation on nonsocket"),
-  E(WSAEDESTADDRREQ, "Destination address required"),
-  E(WSAEMSGSIZE, "Message too long"),
-  E(WSAEPROTOTYPE, "Protocol wrong for socket"),
-  E(WSAENOPROTOOPT, "Bad protocol option"),
-  E(WSAEPROTONOSUPPORT, "Protocol not supported"),
-  E(WSAESOCKTNOSUPPORT, "Socket type not supported"),
-  /* What's the difference between NOTSUPP and NOSUPPORT? :) */
-  E(WSAEOPNOTSUPP, "Operation not supported"),
-  E(WSAEPFNOSUPPORT,  "Protocol family not supported"),
-  E(WSAEAFNOSUPPORT, "Address family not supported by protocol family"),
-  E(WSAEADDRINUSE, "Address already in use"),
-  E(WSAEADDRNOTAVAIL, "Cannot assign requested address"),
-  E(WSAENETDOWN, "Network is down"),
-  E(WSAENETUNREACH, "Network is unreachable"),
-  E(WSAENETRESET, "Network dropped connection on reset"),
-  E(WSAECONNABORTED, "Software caused connection abort"),
-  E(WSAECONNRESET, "Connection reset by peer"),
-  E(WSAENOBUFS, "No buffer space available"),
-  E(WSAEISCONN, "Socket is already connected"),
-  E(WSAENOTCONN, "Socket is not connected"),
-  E(WSAESHUTDOWN, "Cannot send after socket shutdown"),
-  E(WSAETIMEDOUT, "Connection timed out"),
-  E(WSAECONNREFUSED, "Connection refused"),
-  E(WSAEHOSTDOWN, "Host is down"),
-  E(WSAEHOSTUNREACH, "No route to host"),
-  E(WSAEPROCLIM, "Too many processes"),
-  /* Yes, some of these start with WSA, not WSAE. No, I don't know why. */
-  E(WSASYSNOTREADY, "Network subsystem is unavailable"),
-  E(WSAVERNOTSUPPORTED, "Winsock.dll out of range"),
-  E(WSANOTINITIALISED, "Successful WSAStartup not yet performed"),
-  E(WSAEDISCON, "Graceful shutdown now in progress"),
-#ifdef WSATYPE_NOT_FOUND
-  E(WSATYPE_NOT_FOUND, "Class type not found"),
-#endif
-  E(WSAHOST_NOT_FOUND, "Host not found"),
-  E(WSATRY_AGAIN, "Nonauthoritative host not found"),
-  E(WSANO_RECOVERY, "This is a nonrecoverable error"),
-  E(WSANO_DATA, "Valid name, no data record of requested type)"),
-
-  /* There are some more error codes whose numeric values are marked
-   * <b>OS dependent</b>. They start with WSA_, apparently for the same
-   * reason that practitioners of some craft traditions deliberately
-   * introduce imperfections into their baskets and rugs "to allow the
-   * evil spirits to escape."  If we catch them, then our binaries
-   * might not report consistent results across versions of Windows.
-   * Thus, I'm going to let them all fall through.
-   */
-  { -1, NULL },
-};
-/** There does not seem to be a strerror equivalent for Winsock errors.
- * Naturally, we have to roll our own.
- */
-const char *
-tor_socket_strerror(int e)
-{
-  int i;
-  for (i=0; windows_socket_errors[i].code >= 0; ++i) {
-    if (e == windows_socket_errors[i].code)
-      return windows_socket_errors[i].msg;
-  }
-  return strerror(e);
-}
-#endif
-
-/** Called before we make any calls to network-related functions.
- * (Some operating systems require their network libraries to be
- * initialized.) */
-int
-network_init(void)
-{
-#ifdef _WIN32
-  /* This silly exercise is necessary before windows will allow
-   * gethostbyname to work. */
-  WSADATA WSAData;
-  int r;
-  r = WSAStartup(0x101,&WSAData);
-  if (r) {
-    log_warn(LD_NET,"Error initializing windows network layer: code was %d",r);
-    return -1;
-  }
-  if (sizeof(SOCKET) != sizeof(tor_socket_t)) {
-    log_warn(LD_BUG,"The tor_socket_t type does not match SOCKET in size; Tor "
-             "might not work. (Sizes are %d and %d respectively.)",
-             (int)sizeof(tor_socket_t), (int)sizeof(SOCKET));
-  }
-  /* WSAData.iMaxSockets might show the max sockets we're allowed to use.
-   * We might use it to complain if we're trying to be a server but have
-   * too few sockets available. */
-#endif
-  return 0;
-}
-
-#ifdef _WIN32
-/** Return a newly allocated string describing the windows system error code
- * <b>err</b>.  Note that error codes are different from errno.  Error codes
- * come from GetLastError() when a winapi call fails.  errno is set only when
- * ANSI functions fail.  Whee. */
-char *
-format_win32_error(DWORD err)
-{
-  TCHAR *str = NULL;
-  char *result;
-  DWORD n;
-
-  /* Somebody once decided that this interface was better than strerror(). */
-  n = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER |
-                 FORMAT_MESSAGE_FROM_SYSTEM |
-                 FORMAT_MESSAGE_IGNORE_INSERTS,
-                 NULL, err,
-                 MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
-                 (LPVOID)&str,
-                 0, NULL);
-
-  if (str && n) {
-#ifdef UNICODE
-    size_t len;
-    if (n > 128*1024)
-      len = (128 * 1024) * 2 + 1; /* This shouldn't be possible, but let's
-                                   * make sure. */
-    else
-      len = n * 2 + 1;
-    result = tor_malloc(len);
-    wcstombs(result,str,len);
-    result[len-1] = '\0';
-#else
-    result = tor_strdup(str);
-#endif
-  } else {
-    result = tor_strdup("<unformattable error>");
-  }
-  if (str) {
-    LocalFree(str); /* LocalFree != free() */
-  }
-  return result;
-}
-#endif
-
-#if defined(HW_PHYSMEM64)
-/* This appears to be an OpenBSD thing */
-#define INT64_HW_MEM HW_PHYSMEM64
-#elif defined(HW_MEMSIZE)
-/* OSX defines this one */
-#define INT64_HW_MEM HW_MEMSIZE
-#endif
-
-/**
- * Helper: try to detect the total system memory, and return it. On failure,
- * return 0.
- */
-static uint64_t
-get_total_system_memory_impl(void)
-{
-#if defined(__linux__)
-  /* On linux, sysctl is deprecated. Because proc is so awesome that you
-   * shouldn't _want_ to write portable code, I guess? */
-  unsigned long long result=0;
-  int fd = -1;
-  char *s = NULL;
-  const char *cp;
-  size_t file_size=0;
-  if (-1 == (fd = tor_open_cloexec("/proc/meminfo",O_RDONLY,0)))
-    return 0;
-  s = read_file_to_str_until_eof(fd, 65536, &file_size);
-  if (!s)
-    goto err;
-  cp = strstr(s, "MemTotal:");
-  if (!cp)
-    goto err;
-  /* Use the system sscanf so that space will match a wider number of space */
-  if (sscanf(cp, "MemTotal: %llu kB\n", &result) != 1)
-    goto err;
-
-  close(fd);
-  tor_free(s);
-  return result * 1024;
-
- err:
-  /* LCOV_EXCL_START Can't reach this unless proc is broken. */
-  tor_free(s);
-  close(fd);
-  return 0;
-  /* LCOV_EXCL_STOP */
-#elif defined (_WIN32)
-  /* Windows has MEMORYSTATUSEX; pretty straightforward. */
-  MEMORYSTATUSEX ms;
-  memset(&ms, 0, sizeof(ms));
-  ms.dwLength = sizeof(ms);
-  if (! GlobalMemoryStatusEx(&ms))
-    return 0;
-
-  return ms.ullTotalPhys;
-
-#elif defined(HAVE_SYSCTL) && defined(INT64_HW_MEM)
-  /* On many systems, HW_PYHSMEM is clipped to 32 bits; let's use a better
-   * variant if we know about it. */
-  uint64_t memsize = 0;
-  size_t len = sizeof(memsize);
-  int mib[2] = {CTL_HW, INT64_HW_MEM};
-  if (sysctl(mib,2,&memsize,&len,NULL,0))
-    return 0;
-
-  return memsize;
-
-#elif defined(HAVE_SYSCTL) && defined(HW_PHYSMEM)
-  /* On some systems (like FreeBSD I hope) you can use a size_t with
-   * HW_PHYSMEM. */
-  size_t memsize=0;
-  size_t len = sizeof(memsize);
-  int mib[2] = {CTL_HW, HW_USERMEM};
-  if (sysctl(mib,2,&memsize,&len,NULL,0))
-    return 0;
-
-  return memsize;
-
-#else
-  /* I have no clue. */
-  return 0;
-#endif
-}
-
-/**
- * Try to find out how much physical memory the system has. On success,
- * return 0 and set *<b>mem_out</b> to that value. On failure, return -1.
- */
-int
-get_total_system_memory(size_t *mem_out)
-{
-  static size_t mem_cached=0;
-  uint64_t m = get_total_system_memory_impl();
-  if (0 == m) {
-    /* LCOV_EXCL_START -- can't make this happen without mocking. */
-    /* We couldn't find our memory total */
-    if (0 == mem_cached) {
-      /* We have no cached value either */
-      *mem_out = 0;
-      return -1;
-    }
-
-    *mem_out = mem_cached;
-    return 0;
-    /* LCOV_EXCL_STOP */
-  }
-
-#if SIZE_MAX != UINT64_MAX
-  if (m > SIZE_MAX) {
-    /* I think this could happen if we're a 32-bit Tor running on a 64-bit
-     * system: we could have more system memory than would fit in a
-     * size_t. */
-    m = SIZE_MAX;
-  }
-#endif
-
-  *mem_out = mem_cached = (size_t) m;
-
-  return 0;
-}
-
-/** Emit the password prompt <b>prompt</b>, then read up to <b>buflen</b>
- * bytes of passphrase into <b>output</b>. Return the number of bytes in
- * the passphrase, excluding terminating NUL.
- */
-ssize_t
-tor_getpass(const char *prompt, char *output, size_t buflen)
-{
-  tor_assert(buflen <= SSIZE_MAX);
-  tor_assert(buflen >= 1);
-#if defined(HAVE_READPASSPHRASE)
-  char *pwd = readpassphrase(prompt, output, buflen, RPP_ECHO_OFF);
-  if (pwd == NULL)
-    return -1;
-  return strlen(pwd);
-#elif defined(_WIN32)
-  int r = -1;
-  while (*prompt) {
-    _putch(*prompt++);
-  }
-
-  tor_assert(buflen <= INT_MAX);
-  wchar_t *buf = tor_calloc(buflen, sizeof(wchar_t));
-
-  wchar_t *ptr = buf, *lastch = buf + buflen - 1;
-  while (ptr < lastch) {
-    wint_t ch = _getwch();
-    switch (ch) {
-      case '\r':
-      case '\n':
-      case WEOF:
-        goto done_reading;
-      case 3:
-        goto done; /* Can't actually read ctrl-c this way. */
-      case '\b':
-        if (ptr > buf)
-          --ptr;
-        continue;
-      case 0:
-      case 0xe0:
-        ch = _getwch(); /* Ignore; this is a function or arrow key */
-        break;
-      default:
-        *ptr++ = ch;
-        break;
-    }
-  }
- done_reading:
-  ;
-
-#ifndef WC_ERR_INVALID_CHARS
-#define WC_ERR_INVALID_CHARS 0x80
-#endif
-
-  /* Now convert it to UTF-8 */
-  r = WideCharToMultiByte(CP_UTF8,
-                          WC_NO_BEST_FIT_CHARS|WC_ERR_INVALID_CHARS,
-                          buf, (int)(ptr-buf),
-                          output, (int)(buflen-1),
-                          NULL, NULL);
-  if (r <= 0) {
-    r = -1;
-    goto done;
-  }
-
-  tor_assert(r < (int)buflen);
-
-  output[r] = 0;
-
- done:
-  SecureZeroMemory(buf, sizeof(wchar_t)*buflen);
-  tor_free(buf);
-  return r;
-#else
-#error "No implementation for tor_getpass found!"
-#endif
-}
-
-/** Return the amount of free disk space we have permission to use, in
- * bytes. Return -1 if the amount of free space can't be determined. */
-int64_t
-tor_get_avail_disk_space(const char *path)
-{
-#ifdef HAVE_STATVFS
-  struct statvfs st;
-  int r;
-  memset(&st, 0, sizeof(st));
-
-  r = statvfs(path, &st);
-  if (r < 0)
-    return -1;
-
-  int64_t result = st.f_bavail;
-  if (st.f_frsize) {
-    result *= st.f_frsize;
-  } else if (st.f_bsize) {
-    result *= st.f_bsize;
-  } else {
-    return -1;
-  }
-
-  return result;
-#elif defined(_WIN32)
-  ULARGE_INTEGER freeBytesAvail;
-  BOOL ok;
-
-  ok = GetDiskFreeSpaceEx(path, &freeBytesAvail, NULL, NULL);
-  if (!ok) {
-    return -1;
-  }
-  return (int64_t)freeBytesAvail.QuadPart;
-#else
-  (void)path;
-  errno = ENOSYS;
-  return -1;
-#endif
-}
diff --git a/src/common/compat.h b/src/common/compat.h
deleted file mode 100644
index ee1c9454de..0000000000
--- a/src/common/compat.h
+++ /dev/null
@@ -1,747 +0,0 @@
-/* Copyright (c) 2003-2004, Roger Dingledine
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#ifndef TOR_COMPAT_H
-#define TOR_COMPAT_H
-
-#include "orconfig.h"
-#ifdef _WIN32
-#include <winsock2.h>
-#include <ws2tcpip.h>
-#endif
-#include "torint.h"
-#include "testsupport.h"
-#ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h>
-#endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
-#endif
-#ifdef HAVE_TIME_H
-#include <time.h>
-#endif
-#ifdef HAVE_STRING_H
-#include <string.h>
-#endif
-#include <stdarg.h>
-#ifdef HAVE_SYS_RESOURCE_H
-#include <sys/resource.h>
-#endif
-#ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
-#endif
-#ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
-#endif
-#ifdef HAVE_NETINET6_IN6_H
-#include <netinet6/in6.h>
-#endif
-
-#include "compat_time.h"
-
-#if defined(__has_feature)
-#  if __has_feature(address_sanitizer)
-/* Some of the fancy glibc strcmp() macros include references to memory that
- * clang rejects because it is off the end of a less-than-3. Clang hates this,
- * even though those references never actually happen. */
-#    undef strcmp
-#  endif
-#endif
-
-#include <stdio.h>
-#include <errno.h>
-
-#ifndef NULL_REP_IS_ZERO_BYTES
-#error "It seems your platform does not represent NULL as zero. We can't cope."
-#endif
-
-#ifndef DOUBLE_0_REP_IS_ZERO_BYTES
-#error "It seems your platform does not represent 0.0 as zeros. We can't cope."
-#endif
-
-#if 'a'!=97 || 'z'!=122 || 'A'!=65 || ' '!=32
-#error "It seems that you encode characters in something other than ASCII."
-#endif
-
-/* ===== Compiler compatibility */
-
-/* GCC can check printf and scanf types on arbitrary functions. */
-#ifdef __GNUC__
-#define CHECK_PRINTF(formatIdx, firstArg) \
-   __attribute__ ((format(printf, formatIdx, firstArg)))
-#else
-#define CHECK_PRINTF(formatIdx, firstArg)
-#endif
-#ifdef __GNUC__
-#define CHECK_SCANF(formatIdx, firstArg) \
-   __attribute__ ((format(scanf, formatIdx, firstArg)))
-#else
-#define CHECK_SCANF(formatIdx, firstArg)
-#endif
-
-/* What GCC do we have? */
-#ifdef __GNUC__
-#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#else
-#define GCC_VERSION 0
-#endif
-
-/* Temporarily enable and disable warnings. */
-#ifdef __GNUC__
-#  define PRAGMA_STRINGIFY_(s) #s
-#  define PRAGMA_JOIN_STRINGIFY_(a,b) PRAGMA_STRINGIFY_(a ## b)
-/* Support for macro-generated pragmas (c99) */
-#  define PRAGMA_(x) _Pragma (#x)
-#  ifdef __clang__
-#    define PRAGMA_DIAGNOSTIC_(x) PRAGMA_(clang diagnostic x)
-#  else
-#    define PRAGMA_DIAGNOSTIC_(x) PRAGMA_(GCC diagnostic x)
-#  endif
-#  if defined(__clang__) || GCC_VERSION >= 406
-/* we have push/pop support */
-#    define DISABLE_GCC_WARNING(warningopt) \
-          PRAGMA_DIAGNOSTIC_(push) \
-          PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warningopt))
-#    define ENABLE_GCC_WARNING(warningopt) \
-          PRAGMA_DIAGNOSTIC_(pop)
-#  else
-/* older version of gcc: no push/pop support. */
-#    define DISABLE_GCC_WARNING(warningopt) \
-         PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warningopt))
-#    define ENABLE_GCC_WARNING(warningopt) \
-         PRAGMA_DIAGNOSTIC_(warning PRAGMA_JOIN_STRINGIFY_(-W,warningopt))
-#  endif
-#else /* ifdef __GNUC__ */
-/* not gcc at all */
-# define DISABLE_GCC_WARNING(warning)
-# define ENABLE_GCC_WARNING(warning)
-#endif
-
-/* inline is __inline on windows. */
-#ifdef _WIN32
-#define inline __inline
-#endif
-
-/* Try to get a reasonable __func__ substitute in place. */
-#if defined(_MSC_VER)
-
-#define __func__ __FUNCTION__
-
-#else
-/* For platforms where autoconf works, make sure __func__ is defined
- * sanely. */
-#ifndef HAVE_MACRO__func__
-#ifdef HAVE_MACRO__FUNCTION__
-#define __func__ __FUNCTION__
-#elif HAVE_MACRO__FUNC__
-#define __func__ __FUNC__
-#else
-#define __func__ "???"
-#endif
-#endif /* ifndef MAVE_MACRO__func__ */
-#endif /* if not windows */
-
-#define U64_TO_DBL(x) ((double) (x))
-#define DBL_TO_U64(x) ((uint64_t) (x))
-
-#ifdef ENUM_VALS_ARE_SIGNED
-#define ENUM_BF(t) unsigned
-#else
-/** Wrapper for having a bitfield of an enumerated type. Where possible, we
- * just use the enumerated type (so the compiler can help us and notice
- * problems), but if enumerated types are unsigned, we must use unsigned,
- * so that the loss of precision doesn't make large values negative. */
-#define ENUM_BF(t) t
-#endif
-
-/* GCC has several useful attributes. */
-#if defined(__GNUC__) && __GNUC__ >= 3
-#define ATTR_NORETURN __attribute__((noreturn))
-#define ATTR_CONST __attribute__((const))
-#define ATTR_MALLOC __attribute__((malloc))
-#define ATTR_NORETURN __attribute__((noreturn))
-#define ATTR_WUR __attribute__((warn_unused_result))
-/* Alas, nonnull is not at present a good idea for us.  We'd like to get
- * warnings when we pass NULL where we shouldn't (which nonnull does, albeit
- * spottily), but we don't want to tell the compiler to make optimizations
- * with the assumption that the argument can't be NULL (since this would make
- * many of our checks go away, and make our code less robust against
- * programming errors).  Unfortunately, nonnull currently does both of these
- * things, and there's no good way to split them up.
- *
- * #define ATTR_NONNULL(x) __attribute__((nonnull x)) */
-#define ATTR_NONNULL(x)
-#define ATTR_UNUSED __attribute__ ((unused))
-
-/** Macro: Evaluates to <b>exp</b> and hints the compiler that the value
- * of <b>exp</b> will probably be true.
- *
- * In other words, "if (PREDICT_LIKELY(foo))" is the same as "if (foo)",
- * except that it tells the compiler that the branch will be taken most of the
- * time.  This can generate slightly better code with some CPUs.
- */
-#define PREDICT_LIKELY(exp) __builtin_expect(!!(exp), 1)
-/** Macro: Evaluates to <b>exp</b> and hints the compiler that the value
- * of <b>exp</b> will probably be false.
- *
- * In other words, "if (PREDICT_UNLIKELY(foo))" is the same as "if (foo)",
- * except that it tells the compiler that the branch will usually not be
- * taken.  This can generate slightly better code with some CPUs.
- */
-#define PREDICT_UNLIKELY(exp) __builtin_expect(!!(exp), 0)
-#else
-#define ATTR_NORETURN
-#define ATTR_CONST
-#define ATTR_MALLOC
-#define ATTR_NORETURN
-#define ATTR_NONNULL(x)
-#define ATTR_UNUSED
-#define ATTR_WUR
-#define PREDICT_LIKELY(exp) (exp)
-#define PREDICT_UNLIKELY(exp) (exp)
-#endif
-
-/** Expands to a syntactically valid empty statement.  */
-#define STMT_NIL (void)0
-
-/** Expands to a syntactically valid empty statement, explicitly (void)ing its
- * argument. */
-#define STMT_VOID(a) while (0) { (void)(a); }
-
-#ifdef __GNUC__
-/** STMT_BEGIN and STMT_END are used to wrap blocks inside macros so that
- * the macro can be used as if it were a single C statement. */
-#define STMT_BEGIN (void) ({
-#define STMT_END })
-#elif defined(sun) || defined(__sun__)
-#define STMT_BEGIN if (1) {
-#define STMT_END } else STMT_NIL
-#else
-#define STMT_BEGIN do {
-#define STMT_END } while (0)
-#endif
-
-/* Some tools (like coccinelle) don't like to see operators as macro
- * arguments. */
-#define OP_LT <
-#define OP_GT >
-#define OP_GE >=
-#define OP_LE <=
-#define OP_EQ ==
-#define OP_NE !=
-
-/* ===== String compatibility */
-#ifdef _WIN32
-/* Windows names string functions differently from most other platforms. */
-#define strncasecmp _strnicmp
-#define strcasecmp _stricmp
-#endif
-
-#if defined __APPLE__
-/* On OSX 10.9 and later, the overlap-checking code for strlcat would
- * appear to have a severe bug that can sometimes cause aborts in Tor.
- * Instead, use the non-checking variants.  This is sad.
- *
- * See https://trac.torproject.org/projects/tor/ticket/15205
- */
-#undef strlcat
-#undef strlcpy
-#endif
-
-#ifndef HAVE_STRLCAT
-size_t strlcat(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2));
-#endif
-#ifndef HAVE_STRLCPY
-size_t strlcpy(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2));
-#endif
-
-#ifdef _MSC_VER
-/** Casts the uint64_t value in <b>a</b> to the right type for an argument
- * to printf. */
-#define U64_PRINTF_ARG(a) (a)
-/** Casts the uint64_t* value in <b>a</b> to the right type for an argument
- * to scanf. */
-#define U64_SCANF_ARG(a) (a)
-/** Expands to a literal uint64_t-typed constant for the value <b>n</b>. */
-#define U64_LITERAL(n) (n ## ui64)
-#define I64_PRINTF_ARG(a) (a)
-#define I64_SCANF_ARG(a) (a)
-#define I64_LITERAL(n) (n ## i64)
-#else
-#define U64_PRINTF_ARG(a) ((long long unsigned int)(a))
-#define U64_SCANF_ARG(a) ((long long unsigned int*)(a))
-#define U64_LITERAL(n) (n ## llu)
-#define I64_PRINTF_ARG(a) ((long long signed int)(a))
-#define I64_SCANF_ARG(a) ((long long signed int*)(a))
-#define I64_LITERAL(n) (n ## ll)
-#endif
-
-#if defined(_MSC_VER) || defined(__MINGW32__) || defined(__MINGW64__)
-/** The formatting string used to put a uint64_t value in a printf() or
- * scanf() function.  See also U64_PRINTF_ARG and U64_SCANF_ARG. */
-#define U64_FORMAT "%I64u"
-#define I64_FORMAT "%I64d"
-#else
-#define U64_FORMAT "%llu"
-#define I64_FORMAT "%lld"
-#endif
-
-#if (SIZEOF_INTPTR_T == SIZEOF_INT)
-#define INTPTR_T_FORMAT "%d"
-#define INTPTR_PRINTF_ARG(x) ((int)(x))
-#elif (SIZEOF_INTPTR_T == SIZEOF_LONG)
-#define INTPTR_T_FORMAT "%ld"
-#define INTPTR_PRINTF_ARG(x) ((long)(x))
-#elif (SIZEOF_INTPTR_T == 8)
-#define INTPTR_T_FORMAT I64_FORMAT
-#define INTPTR_PRINTF_ARG(x) I64_PRINTF_ARG(x)
-#else
-#error Unknown: SIZEOF_INTPTR_T
-#endif
-
-/** Represents an mmaped file. Allocated via tor_mmap_file; freed with
- * tor_munmap_file. */
-typedef struct tor_mmap_t {
-  const char *data; /**< Mapping of the file's contents. */
-  size_t size; /**< Size of the file. */
-
-  /* None of the fields below should be accessed from outside compat.c */
-#ifdef HAVE_SYS_MMAN_H
-  size_t mapping_size; /**< Size of the actual mapping. (This is this file
-                        * size, rounded up to the nearest page.) */
-#elif defined _WIN32
-  HANDLE mmap_handle;
-#endif
-
-} tor_mmap_t;
-
-tor_mmap_t *tor_mmap_file(const char *filename) ATTR_NONNULL((1));
-int tor_munmap_file(tor_mmap_t *handle) ATTR_NONNULL((1));
-
-int tor_snprintf(char *str, size_t size, const char *format, ...)
-  CHECK_PRINTF(3,4) ATTR_NONNULL((1,3));
-int tor_vsnprintf(char *str, size_t size, const char *format, va_list args)
-  CHECK_PRINTF(3,0) ATTR_NONNULL((1,3));
-
-int tor_asprintf(char **strp, const char *fmt, ...)
-  CHECK_PRINTF(2,3);
-int tor_vasprintf(char **strp, const char *fmt, va_list args)
-  CHECK_PRINTF(2,0);
-
-const void *tor_memmem(const void *haystack, size_t hlen, const void *needle,
-                       size_t nlen) ATTR_NONNULL((1,3));
-static const void *tor_memstr(const void *haystack, size_t hlen,
-                           const char *needle) ATTR_NONNULL((1,3));
-static inline const void *
-tor_memstr(const void *haystack, size_t hlen, const char *needle)
-{
-  return tor_memmem(haystack, hlen, needle, strlen(needle));
-}
-
-/* Much of the time when we're checking ctypes, we're doing spec compliance,
- * which all assumes we're doing ASCII. */
-#define DECLARE_CTYPE_FN(name)                                          \
-  static int TOR_##name(char c);                                        \
-  extern const uint32_t TOR_##name##_TABLE[];                           \
-  static inline int TOR_##name(char c) {                                \
-    uint8_t u = c;                                                      \
-    return !!(TOR_##name##_TABLE[(u >> 5) & 7] & (1u << (u & 31)));     \
-  }
-DECLARE_CTYPE_FN(ISALPHA)
-DECLARE_CTYPE_FN(ISALNUM)
-DECLARE_CTYPE_FN(ISSPACE)
-DECLARE_CTYPE_FN(ISDIGIT)
-DECLARE_CTYPE_FN(ISXDIGIT)
-DECLARE_CTYPE_FN(ISPRINT)
-DECLARE_CTYPE_FN(ISLOWER)
-DECLARE_CTYPE_FN(ISUPPER)
-extern const uint8_t TOR_TOUPPER_TABLE[];
-extern const uint8_t TOR_TOLOWER_TABLE[];
-#define TOR_TOLOWER(c) (TOR_TOLOWER_TABLE[(uint8_t)c])
-#define TOR_TOUPPER(c) (TOR_TOUPPER_TABLE[(uint8_t)c])
-
-char *tor_strtok_r_impl(char *str, const char *sep, char **lasts);
-#ifdef HAVE_STRTOK_R
-#define tor_strtok_r(str, sep, lasts) strtok_r(str, sep, lasts)
-#else
-#define tor_strtok_r(str, sep, lasts) tor_strtok_r_impl(str, sep, lasts)
-#endif
-
-#ifdef _WIN32
-#define SHORT_FILE__ (tor_fix_source_file(__FILE__))
-const char *tor_fix_source_file(const char *fname);
-#else
-#define SHORT_FILE__ (__FILE__)
-#define tor_fix_source_file(s) (s)
-#endif
-
-/* ===== Time compatibility */
-
-struct tm *tor_localtime_r(const time_t *timep, struct tm *result);
-struct tm *tor_gmtime_r(const time_t *timep, struct tm *result);
-
-#ifndef timeradd
-/** Replacement for timeradd on platforms that do not have it: sets tvout to
- * the sum of tv1 and tv2. */
-#define timeradd(tv1,tv2,tvout) \
-  do {                                                  \
-    (tvout)->tv_sec = (tv1)->tv_sec + (tv2)->tv_sec;    \
-    (tvout)->tv_usec = (tv1)->tv_usec + (tv2)->tv_usec; \
-    if ((tvout)->tv_usec >= 1000000) {                  \
-      (tvout)->tv_usec -= 1000000;                      \
-      (tvout)->tv_sec++;                                \
-    }                                                   \
-  } while (0)
-#endif
-
-#ifndef timersub
-/** Replacement for timersub on platforms that do not have it: sets tvout to
- * tv1 minus tv2. */
-#define timersub(tv1,tv2,tvout) \
-  do {                                                  \
-    (tvout)->tv_sec = (tv1)->tv_sec - (tv2)->tv_sec;    \
-    (tvout)->tv_usec = (tv1)->tv_usec - (tv2)->tv_usec; \
-    if ((tvout)->tv_usec < 0) {                         \
-      (tvout)->tv_usec += 1000000;                      \
-      (tvout)->tv_sec--;                                \
-    }                                                   \
-  } while (0)
-#endif
-
-#ifndef timercmp
-/** Replacement for timersub on platforms that do not have it: returns true
- * iff the relational operator "op" makes the expression tv1 op tv2 true.
- *
- * Note that while this definition should work for all boolean opeators, some
- * platforms' native timercmp definitions do not support >=, <=, or ==.  So
- * don't use those.
- */
-#define timercmp(tv1,tv2,op)                    \
-  (((tv1)->tv_sec == (tv2)->tv_sec) ?           \
-   ((tv1)->tv_usec op (tv2)->tv_usec) :         \
-   ((tv1)->tv_sec op (tv2)->tv_sec))
-#endif
-
-/* ===== File compatibility */
-int tor_open_cloexec(const char *path, int flags, unsigned mode);
-FILE *tor_fopen_cloexec(const char *path, const char *mode);
-int tor_rename(const char *path_old, const char *path_new);
-
-int replace_file(const char *from, const char *to);
-int touch_file(const char *fname);
-
-typedef struct tor_lockfile_t tor_lockfile_t;
-tor_lockfile_t *tor_lockfile_lock(const char *filename, int blocking,
-                                  int *locked_out);
-void tor_lockfile_unlock(tor_lockfile_t *lockfile);
-
-off_t tor_fd_getpos(int fd);
-int tor_fd_setpos(int fd, off_t pos);
-int tor_fd_seekend(int fd);
-int tor_ftruncate(int fd);
-
-int64_t tor_get_avail_disk_space(const char *path);
-
-#ifdef _WIN32
-#define PATH_SEPARATOR "\\"
-#else
-#define PATH_SEPARATOR "/"
-#endif
-
-/* ===== Net compatibility */
-
-#if (SIZEOF_SOCKLEN_T == 0)
-typedef int socklen_t;
-#endif
-
-#ifdef _WIN32
-/* XXX Actually, this should arguably be SOCKET; we use intptr_t here so that
- * any inadvertent checks for the socket being <= 0 or > 0 will probably
- * still work. */
-#define tor_socket_t intptr_t
-#define TOR_SOCKET_T_FORMAT INTPTR_T_FORMAT
-#define SOCKET_OK(s) ((SOCKET)(s) != INVALID_SOCKET)
-#define TOR_INVALID_SOCKET INVALID_SOCKET
-#else
-/** Type used for a network socket. */
-#define tor_socket_t int
-#define TOR_SOCKET_T_FORMAT "%d"
-/** Macro: true iff 's' is a possible value for a valid initialized socket. */
-#define SOCKET_OK(s) ((s) >= 0)
-/** Error/uninitialized value for a tor_socket_t. */
-#define TOR_INVALID_SOCKET (-1)
-#endif
-
-int tor_close_socket_simple(tor_socket_t s);
-MOCK_DECL(int, tor_close_socket, (tor_socket_t s));
-tor_socket_t tor_open_socket_with_extensions(
-                                           int domain, int type, int protocol,
-                                           int cloexec, int nonblock);
-MOCK_DECL(tor_socket_t,
-tor_open_socket,(int domain, int type, int protocol));
-tor_socket_t tor_open_socket_nonblocking(int domain, int type, int protocol);
-tor_socket_t tor_accept_socket(tor_socket_t sockfd, struct sockaddr *addr,
-                                  socklen_t *len);
-tor_socket_t tor_accept_socket_nonblocking(tor_socket_t sockfd,
-                                           struct sockaddr *addr,
-                                           socklen_t *len);
-tor_socket_t tor_accept_socket_with_extensions(tor_socket_t sockfd,
-                                               struct sockaddr *addr,
-                                               socklen_t *len,
-                                               int cloexec, int nonblock);
-MOCK_DECL(tor_socket_t,
-tor_connect_socket,(tor_socket_t socket,const struct sockaddr *address,
-                    socklen_t address_len));
-int get_n_open_sockets(void);
-
-MOCK_DECL(int,
-tor_getsockname,(tor_socket_t socket, struct sockaddr *address,
-                 socklen_t *address_len));
-
-#define tor_socket_send(s, buf, len, flags) send(s, buf, len, flags)
-#define tor_socket_recv(s, buf, len, flags) recv(s, buf, len, flags)
-
-/** Implementation of struct in6_addr for platforms that do not have it.
- * Generally, these platforms are ones without IPv6 support, but we want to
- * have a working in6_addr there anyway, so we can use it to parse IPv6
- * addresses. */
-#if !defined(HAVE_STRUCT_IN6_ADDR)
-struct in6_addr
-{
-  union {
-    uint8_t u6_addr8[16];
-    uint16_t u6_addr16[8];
-    uint32_t u6_addr32[4];
-  } in6_u;
-#define s6_addr   in6_u.u6_addr8
-#define s6_addr16 in6_u.u6_addr16
-#define s6_addr32 in6_u.u6_addr32
-};
-#endif
-
-/** @{ */
-/** Many BSD variants seem not to define these. */
-#if defined(__APPLE__) || defined(__darwin__) || defined(__FreeBSD__) \
-    || defined(__NetBSD__) || defined(__OpenBSD__)
-#ifndef s6_addr16
-#define s6_addr16 __u6_addr.__u6_addr16
-#endif
-#ifndef s6_addr32
-#define s6_addr32 __u6_addr.__u6_addr32
-#endif
-#endif
-/** @} */
-
-#ifndef HAVE_SA_FAMILY_T
-typedef uint16_t sa_family_t;
-#endif
-
-/** @{ */
-/** Apparently, MS and Solaris don't define s6_addr16 or s6_addr32; these
- * macros get you a pointer to s6_addr32 or local equivalent. */
-#ifdef HAVE_STRUCT_IN6_ADDR_S6_ADDR32
-#define S6_ADDR32(x) ((uint32_t*)(x).s6_addr32)
-#else
-#define S6_ADDR32(x) ((uint32_t*)((char*)&(x).s6_addr))
-#endif
-#ifdef HAVE_STRUCT_IN6_ADDR_S6_ADDR16
-#define S6_ADDR16(x) ((uint16_t*)(x).s6_addr16)
-#else
-#define S6_ADDR16(x) ((uint16_t*)((char*)&(x).s6_addr))
-#endif
-/** @} */
-
-/** Implementation of struct sockaddr_in6 on platforms that do not have
- * it. See notes on struct in6_addr. */
-#if !defined(HAVE_STRUCT_SOCKADDR_IN6)
-struct sockaddr_in6 {
-  sa_family_t sin6_family;
-  uint16_t sin6_port;
-  // uint32_t sin6_flowinfo;
-  struct in6_addr sin6_addr;
-  // uint32_t sin6_scope_id;
-};
-#endif
-
-MOCK_DECL(int,tor_gethostname,(char *name, size_t namelen));
-int tor_inet_aton(const char *cp, struct in_addr *addr) ATTR_NONNULL((1,2));
-const char *tor_inet_ntop(int af, const void *src, char *dst, size_t len);
-int tor_inet_pton(int af, const char *src, void *dst);
-MOCK_DECL(int,tor_lookup_hostname,(const char *name, uint32_t *addr));
-int set_socket_nonblocking(tor_socket_t socket);
-int tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]);
-int network_init(void);
-
-/* For stupid historical reasons, windows sockets have an independent
- * set of errnos, and an independent way to get them.  Also, you can't
- * always believe WSAEWOULDBLOCK.  Use the macros below to compare
- * errnos against expected values, and use tor_socket_errno to find
- * the actual errno after a socket operation fails.
- */
-#if defined(_WIN32)
-/** Expands to WSA<b>e</b> on Windows, and to <b>e</b> elsewhere. */
-#define SOCK_ERRNO(e) WSA##e
-/** Return true if e is EAGAIN or the local equivalent. */
-#define ERRNO_IS_EAGAIN(e)           ((e) == EAGAIN || (e) == WSAEWOULDBLOCK)
-/** Return true if e is EINPROGRESS or the local equivalent. */
-#define ERRNO_IS_EINPROGRESS(e)      ((e) == WSAEINPROGRESS)
-/** Return true if e is EINPROGRESS or the local equivalent as returned by
- * a call to connect(). */
-#define ERRNO_IS_CONN_EINPROGRESS(e) \
-  ((e) == WSAEINPROGRESS || (e)== WSAEINVAL || (e) == WSAEWOULDBLOCK)
-/** Return true if e is EAGAIN or another error indicating that a call to
- * accept() has no pending connections to return. */
-#define ERRNO_IS_ACCEPT_EAGAIN(e)    ERRNO_IS_EAGAIN(e)
-/** Return true if e is EMFILE or another error indicating that a call to
- * accept() has failed because we're out of fds or something. */
-#define ERRNO_IS_RESOURCE_LIMIT(e) \
-  ((e) == WSAEMFILE || (e) == WSAENOBUFS)
-/** Return true if e is EADDRINUSE or the local equivalent. */
-#define ERRNO_IS_EADDRINUSE(e)      ((e) == WSAEADDRINUSE)
-/** Return true if e is EINTR  or the local equivalent */
-#define ERRNO_IS_EINTR(e)            ((e) == WSAEINTR || 0)
-int tor_socket_errno(tor_socket_t sock);
-const char *tor_socket_strerror(int e);
-#else
-#define SOCK_ERRNO(e) e
-#if EAGAIN == EWOULDBLOCK
-/* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */
-#define ERRNO_IS_EAGAIN(e)           ((e) == EAGAIN || 0)
-#else
-#define ERRNO_IS_EAGAIN(e)           ((e) == EAGAIN || (e) == EWOULDBLOCK)
-#endif
-#define ERRNO_IS_EINTR(e)            ((e) == EINTR || 0)
-#define ERRNO_IS_EINPROGRESS(e)      ((e) == EINPROGRESS || 0)
-#define ERRNO_IS_CONN_EINPROGRESS(e) ((e) == EINPROGRESS || 0)
-#define ERRNO_IS_ACCEPT_EAGAIN(e) \
-  (ERRNO_IS_EAGAIN(e) || (e) == ECONNABORTED)
-#define ERRNO_IS_RESOURCE_LIMIT(e) \
-  ((e) == EMFILE || (e) == ENFILE || (e) == ENOBUFS || (e) == ENOMEM)
-#define ERRNO_IS_EADDRINUSE(e)       (((e) == EADDRINUSE) || 0)
-#define tor_socket_errno(sock)       (errno)
-#define tor_socket_strerror(e)       strerror(e)
-#endif
-
-/** Specified SOCKS5 status codes. */
-typedef enum {
-  SOCKS5_SUCCEEDED                  = 0x00,
-  SOCKS5_GENERAL_ERROR              = 0x01,
-  SOCKS5_NOT_ALLOWED                = 0x02,
-  SOCKS5_NET_UNREACHABLE            = 0x03,
-  SOCKS5_HOST_UNREACHABLE           = 0x04,
-  SOCKS5_CONNECTION_REFUSED         = 0x05,
-  SOCKS5_TTL_EXPIRED                = 0x06,
-  SOCKS5_COMMAND_NOT_SUPPORTED      = 0x07,
-  SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED = 0x08,
-} socks5_reply_status_t;
-
-/* ===== OS compatibility */
-MOCK_DECL(const char *, get_uname, (void));
-
-uint16_t get_uint16(const void *cp) ATTR_NONNULL((1));
-uint32_t get_uint32(const void *cp) ATTR_NONNULL((1));
-uint64_t get_uint64(const void *cp) ATTR_NONNULL((1));
-void set_uint16(void *cp, uint16_t v) ATTR_NONNULL((1));
-void set_uint32(void *cp, uint32_t v) ATTR_NONNULL((1));
-void set_uint64(void *cp, uint64_t v) ATTR_NONNULL((1));
-
-/* These uint8 variants are defined to make the code more uniform. */
-#define get_uint8(cp) (*(const uint8_t*)(cp))
-static void set_uint8(void *cp, uint8_t v);
-static inline void
-set_uint8(void *cp, uint8_t v)
-{
-  *(uint8_t*)cp = v;
-}
-
-#if !defined(HAVE_RLIM_T)
-typedef unsigned long rlim_t;
-#endif
-int get_max_sockets(void);
-int set_max_file_descriptors(rlim_t limit, int *max);
-int tor_disable_debugger_attach(void);
-
-#if defined(HAVE_SYS_CAPABILITY_H) && defined(HAVE_CAP_SET_PROC)
-#define HAVE_LINUX_CAPABILITIES
-#endif
-
-int have_capability_support(void);
-
-/** Flag for switch_id; see switch_id() for documentation */
-#define SWITCH_ID_KEEP_BINDLOW    (1<<0)
-/** Flag for switch_id; see switch_id() for documentation */
-#define SWITCH_ID_WARN_IF_NO_CAPS (1<<1)
-int switch_id(const char *user, unsigned flags);
-#ifdef HAVE_PWD_H
-char *get_user_homedir(const char *username);
-#endif
-
-#ifndef _WIN32
-const struct passwd *tor_getpwnam(const char *username);
-const struct passwd *tor_getpwuid(uid_t uid);
-#endif
-
-int get_parent_directory(char *fname);
-char *make_path_absolute(char *fname);
-
-char **get_environment(void);
-
-int get_total_system_memory(size_t *mem_out);
-
-int compute_num_cpus(void);
-
-int tor_mlockall(void);
-
-/** Macros for MIN/MAX.  Never use these when the arguments could have
- * side-effects.
- * {With GCC extensions we could probably define a safer MIN/MAX.  But
- * depending on that safety would be dangerous, since not every platform
- * has it.}
- **/
-#ifndef MAX
-#define MAX(a,b) ( ((a)<(b)) ? (b) : (a) )
-#endif
-#ifndef MIN
-#define MIN(a,b) ( ((a)>(b)) ? (b) : (a) )
-#endif
-
-/* Platform-specific helpers. */
-#ifdef _WIN32
-char *format_win32_error(DWORD err);
-#endif
-
-/*for some reason my compiler doesn't have these version flags defined
-  a nice homework assignment for someone one day is to define the rest*/
-//these are the values as given on MSDN
-#ifdef _WIN32
-
-#ifndef VER_SUITE_EMBEDDEDNT
-#define VER_SUITE_EMBEDDEDNT 0x00000040
-#endif
-
-#ifndef VER_SUITE_SINGLEUSERTS
-#define VER_SUITE_SINGLEUSERTS 0x00000100
-#endif
-
-#endif
-
-#ifdef COMPAT_PRIVATE
-#if !defined(HAVE_SOCKETPAIR) || defined(_WIN32) || defined(TOR_UNIT_TESTS)
-#define NEED_ERSATZ_SOCKETPAIR
-STATIC int tor_ersatz_socketpair(int family, int type, int protocol,
-                                   tor_socket_t fd[2]);
-#endif
-#endif
-
-ssize_t tor_getpass(const char *prompt, char *output, size_t buflen);
-
-/* This needs some of the declarations above so we include it here. */
-#include "compat_threads.h"
-
-#endif
-
diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c
deleted file mode 100644
index 4a3b1af922..0000000000
--- a/src/common/compat_libevent.c
+++ /dev/null
@@ -1,285 +0,0 @@
-/* Copyright (c) 2009-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file compat_libevent.c
- * \brief Wrappers and utility functions for Libevent.
- */
-
-#include "orconfig.h"
-#include "compat.h"
-#define COMPAT_LIBEVENT_PRIVATE
-#include "compat_libevent.h"
-
-#include "crypto.h"
-
-#include "util.h"
-#include "torlog.h"
-
-#include <event2/event.h>
-#include <event2/thread.h>
-
-/** A string which, if it appears in a libevent log, should be ignored. */
-static const char *suppress_msg = NULL;
-/** Callback function passed to event_set_log() so we can intercept
- * log messages from libevent. */
-STATIC void
-libevent_logging_callback(int severity, const char *msg)
-{
-  char buf[1024];
-  size_t n;
-  if (suppress_msg && strstr(msg, suppress_msg))
-    return;
-  n = strlcpy(buf, msg, sizeof(buf));
-  if (n && n < sizeof(buf) && buf[n-1] == '\n') {
-    buf[n-1] = '\0';
-  }
-  switch (severity) {
-    case _EVENT_LOG_DEBUG:
-      log_debug(LD_NOCB|LD_NET, "Message from libevent: %s", buf);
-      break;
-    case _EVENT_LOG_MSG:
-      log_info(LD_NOCB|LD_NET, "Message from libevent: %s", buf);
-      break;
-    case _EVENT_LOG_WARN:
-      log_warn(LD_NOCB|LD_GENERAL, "Warning from libevent: %s", buf);
-      break;
-    case _EVENT_LOG_ERR:
-      log_err(LD_NOCB|LD_GENERAL, "Error from libevent: %s", buf);
-      break;
-    default:
-      log_warn(LD_NOCB|LD_GENERAL, "Message [%d] from libevent: %s",
-          severity, buf);
-      break;
-  }
-}
-/** Set hook to intercept log messages from libevent. */
-void
-configure_libevent_logging(void)
-{
-  event_set_log_callback(libevent_logging_callback);
-}
-
-/** Ignore any libevent log message that contains <b>msg</b>. */
-void
-suppress_libevent_log_msg(const char *msg)
-{
-  suppress_msg = msg;
-}
-
-/* Wrapper for event_free() that tolerates tor_event_free(NULL) */
-void
-tor_event_free(struct event *ev)
-{
-  if (ev == NULL)
-    return;
-  event_free(ev);
-}
-
-/** Global event base for use by the main thread. */
-static struct event_base *the_event_base = NULL;
-
-/* This is what passes for version detection on OSX.  We set
- * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before
- * 10.4.0 (aka 1040). */
-#ifdef __APPLE__
-#ifdef __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__
-#define MACOSX_KQUEUE_IS_BROKEN \
-  (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1040)
-#else
-#define MACOSX_KQUEUE_IS_BROKEN 0
-#endif
-#endif
-
-/** Initialize the Libevent library and set up the event base. */
-void
-tor_libevent_initialize(tor_libevent_cfg *torcfg)
-{
-  tor_assert(the_event_base == NULL);
-  /* some paths below don't use torcfg, so avoid unused variable warnings */
-  (void)torcfg;
-
-  {
-    int attempts = 0;
-    struct event_config *cfg;
-
-    ++attempts;
-    cfg = event_config_new();
-    tor_assert(cfg);
-
-    /* Telling Libevent not to try to turn locking on can avoid a needless
-     * socketpair() attempt. */
-    event_config_set_flag(cfg, EVENT_BASE_FLAG_NOLOCK);
-
-    if (torcfg->num_cpus > 0)
-      event_config_set_num_cpus_hint(cfg, torcfg->num_cpus);
-
-    /* We can enable changelist support with epoll, since we don't give
-     * Libevent any dup'd fds.  This lets us avoid some syscalls. */
-    event_config_set_flag(cfg, EVENT_BASE_FLAG_EPOLL_USE_CHANGELIST);
-
-    the_event_base = event_base_new_with_config(cfg);
-
-    event_config_free(cfg);
-  }
-
-  if (!the_event_base) {
-    /* LCOV_EXCL_START */
-    log_err(LD_GENERAL, "Unable to initialize Libevent: cannot continue.");
-    exit(1);
-    /* LCOV_EXCL_STOP */
-  }
-
-  log_info(LD_GENERAL,
-      "Initialized libevent version %s using method %s. Good.",
-      event_get_version(), tor_libevent_get_method());
-}
-
-/** Return the current Libevent event base that we're set up to use. */
-MOCK_IMPL(struct event_base *,
-tor_libevent_get_base, (void))
-{
-  tor_assert(the_event_base != NULL);
-  return the_event_base;
-}
-
-/** Return the name of the Libevent backend we're using. */
-const char *
-tor_libevent_get_method(void)
-{
-  return event_base_get_method(the_event_base);
-}
-
-/** Return a string representation of the version of the currently running
- * version of Libevent. */
-const char *
-tor_libevent_get_version_str(void)
-{
-  return event_get_version();
-}
-
-/** Return a string representation of the version of Libevent that was used
-* at compilation time. */
-const char *
-tor_libevent_get_header_version_str(void)
-{
-  return LIBEVENT_VERSION;
-}
-
-/** Represents a timer that's run every N microseconds by Libevent. */
-struct periodic_timer_t {
-  /** Underlying event used to implement this periodic event. */
-  struct event *ev;
-  /** The callback we'll be invoking whenever the event triggers */
-  void (*cb)(struct periodic_timer_t *, void *);
-  /** User-supplied data for the callback */
-  void *data;
-};
-
-/** Libevent callback to implement a periodic event. */
-static void
-periodic_timer_cb(evutil_socket_t fd, short what, void *arg)
-{
-  periodic_timer_t *timer = arg;
-  (void) what;
-  (void) fd;
-  timer->cb(timer, timer->data);
-}
-
-/** Create and schedule a new timer that will run every <b>tv</b> in
- * the event loop of <b>base</b>.  When the timer fires, it will
- * run the timer in <b>cb</b> with the user-supplied data in <b>data</b>. */
-periodic_timer_t *
-periodic_timer_new(struct event_base *base,
-                   const struct timeval *tv,
-                   void (*cb)(periodic_timer_t *timer, void *data),
-                   void *data)
-{
-  periodic_timer_t *timer;
-  tor_assert(base);
-  tor_assert(tv);
-  tor_assert(cb);
-  timer = tor_malloc_zero(sizeof(periodic_timer_t));
-  if (!(timer->ev = tor_event_new(base, -1, EV_PERSIST,
-                                  periodic_timer_cb, timer))) {
-    tor_free(timer);
-    return NULL;
-  }
-  timer->cb = cb;
-  timer->data = data;
-  event_add(timer->ev, (struct timeval *)tv); /*drop const for old libevent*/
-  return timer;
-}
-
-/** Stop and free a periodic timer */
-void
-periodic_timer_free(periodic_timer_t *timer)
-{
-  if (!timer)
-    return;
-  tor_event_free(timer->ev);
-  tor_free(timer);
-}
-
-int
-tor_init_libevent_rng(void)
-{
-  int rv = 0;
-  char buf[256];
-  if (evutil_secure_rng_init() < 0) {
-    rv = -1;
-  }
-  crypto_rand(buf, 32);
-#ifdef HAVE_EVUTIL_SECURE_RNG_ADD_BYTES
-  evutil_secure_rng_add_bytes(buf, 32);
-#endif
-  evutil_secure_rng_get_bytes(buf, sizeof(buf));
-  return rv;
-}
-
-#if defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER >= V(2,1,1) \
-  && !defined(TOR_UNIT_TESTS)
-void
-tor_gettimeofday_cached(struct timeval *tv)
-{
-  event_base_gettimeofday_cached(the_event_base, tv);
-}
-void
-tor_gettimeofday_cache_clear(void)
-{
-  event_base_update_cache_time(the_event_base);
-}
-#else
-/** Cache the current hi-res time; the cache gets reset when libevent
- * calls us. */
-static struct timeval cached_time_hires = {0, 0};
-
-/** Return a fairly recent view of the current time. */
-void
-tor_gettimeofday_cached(struct timeval *tv)
-{
-  if (cached_time_hires.tv_sec == 0) {
-    tor_gettimeofday(&cached_time_hires);
-  }
-  *tv = cached_time_hires;
-}
-
-/** Reset the cached view of the current time, so that the next time we try
- * to learn it, we will get an up-to-date value. */
-void
-tor_gettimeofday_cache_clear(void)
-{
-  cached_time_hires.tv_sec = 0;
-}
-
-#ifdef TOR_UNIT_TESTS
-/** For testing: force-update the cached time to a given value. */
-void
-tor_gettimeofday_cache_set(const struct timeval *tv)
-{
-  tor_assert(tv);
-  memcpy(&cached_time_hires, tv, sizeof(*tv));
-}
-#endif
-#endif
-
diff --git a/src/common/compat_threads.h b/src/common/compat_threads.h
deleted file mode 100644
index 171a9f93ff..0000000000
--- a/src/common/compat_threads.h
+++ /dev/null
@@ -1,151 +0,0 @@
-/* Copyright (c) 2003-2004, Roger Dingledine
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#ifndef TOR_COMPAT_THREADS_H
-#define TOR_COMPAT_THREADS_H
-
-#include "orconfig.h"
-#include "torint.h"
-#include "testsupport.h"
-
-#if defined(HAVE_PTHREAD_H) && !defined(_WIN32)
-#include <pthread.h>
-#endif
-
-#if defined(_WIN32)
-#define USE_WIN32_THREADS
-#elif defined(HAVE_PTHREAD_H) && defined(HAVE_PTHREAD_CREATE)
-#define USE_PTHREADS
-#else
-#error "No threading system was found"
-#endif
-
-int spawn_func(void (*func)(void *), void *data);
-void spawn_exit(void) ATTR_NORETURN;
-
-/* Because we use threads instead of processes on most platforms (Windows,
- * Linux, etc), we need locking for them.  On platforms with poor thread
- * support or broken gethostbyname_r, these functions are no-ops. */
-
-/** A generic lock structure for multithreaded builds. */
-typedef struct tor_mutex_t {
-#if defined(USE_WIN32_THREADS)
-  /** Windows-only: on windows, we implement locks with CRITICAL_SECTIONS. */
-  CRITICAL_SECTION mutex;
-#elif defined(USE_PTHREADS)
-  /** Pthreads-only: with pthreads, we implement locks with
-   * pthread_mutex_t. */
-  pthread_mutex_t mutex;
-#else
-  /** No-threads only: Dummy variable so that tor_mutex_t takes up space. */
-  int _unused;
-#endif
-} tor_mutex_t;
-
-tor_mutex_t *tor_mutex_new(void);
-tor_mutex_t *tor_mutex_new_nonrecursive(void);
-void tor_mutex_init(tor_mutex_t *m);
-void tor_mutex_init_nonrecursive(tor_mutex_t *m);
-void tor_mutex_acquire(tor_mutex_t *m);
-void tor_mutex_release(tor_mutex_t *m);
-void tor_mutex_free(tor_mutex_t *m);
-void tor_mutex_uninit(tor_mutex_t *m);
-unsigned long tor_get_thread_id(void);
-void tor_threads_init(void);
-
-/** Conditions need nonrecursive mutexes with pthreads. */
-#define tor_mutex_init_for_cond(m) tor_mutex_init_nonrecursive(m)
-
-void set_main_thread(void);
-int in_main_thread(void);
-
-typedef struct tor_cond_t {
-#ifdef USE_PTHREADS
-  pthread_cond_t cond;
-#elif defined(USE_WIN32_THREADS)
-  HANDLE event;
-
-  CRITICAL_SECTION lock;
-  int n_waiting;
-  int n_to_wake;
-  int generation;
-#else
-#error no known condition implementation.
-#endif
-} tor_cond_t;
-
-tor_cond_t *tor_cond_new(void);
-void tor_cond_free(tor_cond_t *cond);
-int tor_cond_init(tor_cond_t *cond);
-void tor_cond_uninit(tor_cond_t *cond);
-int tor_cond_wait(tor_cond_t *cond, tor_mutex_t *mutex,
-                  const struct timeval *tv);
-void tor_cond_signal_one(tor_cond_t *cond);
-void tor_cond_signal_all(tor_cond_t *cond);
-
-/** Helper type used to manage waking up the main thread while it's in
- * the libevent main loop.  Used by the work queue code. */
-typedef struct alert_sockets_s {
-  /* XXXX This structure needs a better name. */
-  /** Socket that the main thread should listen for EV_READ events on.
-   * Note that this socket may be a regular fd on a non-Windows platform.
-   */
-  tor_socket_t read_fd;
-  /** Socket to use when alerting the main thread. */
-  tor_socket_t write_fd;
-  /** Function to alert the main thread */
-  int (*alert_fn)(tor_socket_t write_fd);
-  /** Function to make the main thread no longer alerted. */
-  int (*drain_fn)(tor_socket_t read_fd);
-} alert_sockets_t;
-
-/* Flags to disable one or more alert_sockets backends. */
-#define ASOCKS_NOEVENTFD2   (1u<<0)
-#define ASOCKS_NOEVENTFD    (1u<<1)
-#define ASOCKS_NOPIPE2      (1u<<2)
-#define ASOCKS_NOPIPE       (1u<<3)
-#define ASOCKS_NOSOCKETPAIR (1u<<4)
-
-int alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags);
-void alert_sockets_close(alert_sockets_t *socks);
-
-typedef struct tor_threadlocal_s {
-#ifdef _WIN32
-  DWORD index;
-#else
-  pthread_key_t key;
-#endif
-} tor_threadlocal_t;
-
-/** Initialize a thread-local variable.
- *
- * After you call this function on a tor_threadlocal_t, you can call
- * tor_threadlocal_set to change the current value of this variable for the
- * current thread, and tor_threadlocal_get to retrieve the current value for
- * the current thread.  Each thread has its own value.
- **/
-int tor_threadlocal_init(tor_threadlocal_t *threadlocal);
-/**
- * Release all resource associated with a thread-local variable.
- */
-void tor_threadlocal_destroy(tor_threadlocal_t *threadlocal);
-/**
- * Return the current value of a thread-local variable for this thread.
- *
- * It's undefined behavior to use this function if the threadlocal hasn't
- * been initialized, or has been destroyed.
- */
-void *tor_threadlocal_get(tor_threadlocal_t *threadlocal);
-/**
- * Change the current value of a thread-local variable for this thread to
- * <b>value</b>.
- *
- * It's undefined behavior to use this function if the threadlocal hasn't
- * been initialized, or has been destroyed.
- */
-void tor_threadlocal_set(tor_threadlocal_t *threadlocal, void *value);
-
-#endif
-
diff --git a/src/common/container.c b/src/common/container.c
deleted file mode 100644
index ec59dccf62..0000000000
--- a/src/common/container.c
+++ /dev/null
@@ -1,1517 +0,0 @@
-/* Copyright (c) 2003-2004, Roger Dingledine
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file container.c
- * \brief Implements a smartlist (a resizable array) along
- * with helper functions to use smartlists.  Also includes
- * hash table implementations of a string-to-void* map, and of
- * a digest-to-void* map.
- **/
-
-#include "compat.h"
-#include "util.h"
-#include "torlog.h"
-#include "container.h"
-#include "crypto.h"
-
-#include <stdlib.h>
-#include <string.h>
-#include <assert.h>
-
-#include "ht.h"
-
-/** All newly allocated smartlists have this capacity. */
-#define SMARTLIST_DEFAULT_CAPACITY 16
-
-/** Allocate and return an empty smartlist.
- */
-MOCK_IMPL(smartlist_t *,
-smartlist_new,(void))
-{
-  smartlist_t *sl = tor_malloc(sizeof(smartlist_t));
-  sl->num_used = 0;
-  sl->capacity = SMARTLIST_DEFAULT_CAPACITY;
-  sl->list = tor_calloc(sizeof(void *), sl->capacity);
-  return sl;
-}
-
-/** Deallocate a smartlist.  Does not release storage associated with the
- * list's elements.
- */
-MOCK_IMPL(void,
-smartlist_free,(smartlist_t *sl))
-{
-  if (!sl)
-    return;
-  tor_free(sl->list);
-  tor_free(sl);
-}
-
-/** Remove all elements from the list.
- */
-void
-smartlist_clear(smartlist_t *sl)
-{
-  memset(sl->list, 0, sizeof(void *) * sl->num_used);
-  sl->num_used = 0;
-}
-
-#if SIZE_MAX < INT_MAX
-#error "We don't support systems where size_t is smaller than int."
-#endif
-
-/** Make sure that <b>sl</b> can hold at least <b>size</b> entries. */
-static inline void
-smartlist_ensure_capacity(smartlist_t *sl, size_t size)
-{
-  /* Set MAX_CAPACITY to MIN(INT_MAX, SIZE_MAX / sizeof(void*)) */
-#if (SIZE_MAX/SIZEOF_VOID_P) > INT_MAX
-#define MAX_CAPACITY (INT_MAX)
-#else
-#define MAX_CAPACITY (int)((SIZE_MAX / (sizeof(void*))))
-#endif
-
-  tor_assert(size <= MAX_CAPACITY);
-
-  if (size > (size_t) sl->capacity) {
-    size_t higher = (size_t) sl->capacity;
-    if (PREDICT_UNLIKELY(size > MAX_CAPACITY/2)) {
-      higher = MAX_CAPACITY;
-    } else {
-      while (size > higher)
-        higher *= 2;
-    }
-    sl->list = tor_reallocarray(sl->list, sizeof(void *),
-                                ((size_t)higher));
-    memset(sl->list + sl->capacity, 0,
-           sizeof(void *) * (higher - sl->capacity));
-    sl->capacity = (int) higher;
-  }
-#undef ASSERT_CAPACITY
-#undef MAX_CAPACITY
-}
-
-/** Append element to the end of the list. */
-void
-smartlist_add(smartlist_t *sl, void *element)
-{
-  smartlist_ensure_capacity(sl, ((size_t) sl->num_used)+1);
-  sl->list[sl->num_used++] = element;
-}
-
-/** Append each element from S2 to the end of S1. */
-void
-smartlist_add_all(smartlist_t *s1, const smartlist_t *s2)
-{
-  size_t new_size = (size_t)s1->num_used + (size_t)s2->num_used;
-  tor_assert(new_size >= (size_t) s1->num_used); /* check for overflow. */
-  smartlist_ensure_capacity(s1, new_size);
-  memcpy(s1->list + s1->num_used, s2->list, s2->num_used*sizeof(void*));
-  tor_assert(new_size <= INT_MAX); /* redundant. */
-  s1->num_used = (int) new_size;
-}
-
-/** Remove all elements E from sl such that E==element.  Preserve
- * the order of any elements before E, but elements after E can be
- * rearranged.
- */
-void
-smartlist_remove(smartlist_t *sl, const void *element)
-{
-  int i;
-  if (element == NULL)
-    return;
-  for (i=0; i < sl->num_used; i++)
-    if (sl->list[i] == element) {
-      sl->list[i] = sl->list[--sl->num_used]; /* swap with the end */
-      i--; /* so we process the new i'th element */
-      sl->list[sl->num_used] = NULL;
-    }
-}
-
-/** If <b>sl</b> is nonempty, remove and return the final element.  Otherwise,
- * return NULL. */
-void *
-smartlist_pop_last(smartlist_t *sl)
-{
-  tor_assert(sl);
-  if (sl->num_used) {
-    void *tmp = sl->list[--sl->num_used];
-    sl->list[sl->num_used] = NULL;
-    return tmp;
-  } else
-    return NULL;
-}
-
-/** Reverse the order of the items in <b>sl</b>. */
-void
-smartlist_reverse(smartlist_t *sl)
-{
-  int i, j;
-  void *tmp;
-  tor_assert(sl);
-  for (i = 0, j = sl->num_used-1; i < j; ++i, --j) {
-    tmp = sl->list[i];
-    sl->list[i] = sl->list[j];
-    sl->list[j] = tmp;
-  }
-}
-
-/** If there are any strings in sl equal to element, remove and free them.
- * Does not preserve order. */
-void
-smartlist_string_remove(smartlist_t *sl, const char *element)
-{
-  int i;
-  tor_assert(sl);
-  tor_assert(element);
-  for (i = 0; i < sl->num_used; ++i) {
-    if (!strcmp(element, sl->list[i])) {
-      tor_free(sl->list[i]);
-      sl->list[i] = sl->list[--sl->num_used]; /* swap with the end */
-      i--; /* so we process the new i'th element */
-      sl->list[sl->num_used] = NULL;
-    }
-  }
-}
-
-/** Return true iff some element E of sl has E==element.
- */
-int
-smartlist_contains(const smartlist_t *sl, const void *element)
-{
-  int i;
-  for (i=0; i < sl->num_used; i++)
-    if (sl->list[i] == element)
-      return 1;
-  return 0;
-}
-
-/** Return true iff <b>sl</b> has some element E such that
- * !strcmp(E,<b>element</b>)
- */
-int
-smartlist_contains_string(const smartlist_t *sl, const char *element)
-{
-  int i;
-  if (!sl) return 0;
-  for (i=0; i < sl->num_used; i++)
-    if (strcmp((const char*)sl->list[i],element)==0)
-      return 1;
-  return 0;
-}
-
-/** If <b>element</b> is equal to an element of <b>sl</b>, return that
- * element's index.  Otherwise, return -1. */
-int
-smartlist_string_pos(const smartlist_t *sl, const char *element)
-{
-  int i;
-  if (!sl) return -1;
-  for (i=0; i < sl->num_used; i++)
-    if (strcmp((const char*)sl->list[i],element)==0)
-      return i;
-  return -1;
-}
-
-/** If <b>element</b> is the same pointer as an element of <b>sl</b>, return
- * that element's index.  Otherwise, return -1. */
-int
-smartlist_pos(const smartlist_t *sl, const void *element)
-{
-  int i;
-  if (!sl) return -1;
-  for (i=0; i < sl->num_used; i++)
-    if (element == sl->list[i])
-      return i;
-  return -1;
-}
-
-/** Return true iff <b>sl</b> has some element E such that
- * !strcasecmp(E,<b>element</b>)
- */
-int
-smartlist_contains_string_case(const smartlist_t *sl, const char *element)
-{
-  int i;
-  if (!sl) return 0;
-  for (i=0; i < sl->num_used; i++)
-    if (strcasecmp((const char*)sl->list[i],element)==0)
-      return 1;
-  return 0;
-}
-
-/** Return true iff <b>sl</b> has some element E such that E is equal
- * to the decimal encoding of <b>num</b>.
- */
-int
-smartlist_contains_int_as_string(const smartlist_t *sl, int num)
-{
-  char buf[32]; /* long enough for 64-bit int, and then some. */
-  tor_snprintf(buf,sizeof(buf),"%d", num);
-  return smartlist_contains_string(sl, buf);
-}
-
-/** Return true iff the two lists contain the same strings in the same
- * order, or if they are both NULL. */
-int
-smartlist_strings_eq(const smartlist_t *sl1, const smartlist_t *sl2)
-{
-  if (sl1 == NULL)
-    return sl2 == NULL;
-  if (sl2 == NULL)
-    return 0;
-  if (smartlist_len(sl1) != smartlist_len(sl2))
-    return 0;
-  SMARTLIST_FOREACH(sl1, const char *, cp1, {
-      const char *cp2 = smartlist_get(sl2, cp1_sl_idx);
-      if (strcmp(cp1, cp2))
-        return 0;
-    });
-  return 1;
-}
-
-/** Return true iff the two lists contain the same int pointer values in
- * the same order, or if they are both NULL. */
-int
-smartlist_ints_eq(const smartlist_t *sl1, const smartlist_t *sl2)
-{
-  if (sl1 == NULL)
-    return sl2 == NULL;
-  if (sl2 == NULL)
-    return 0;
-  if (smartlist_len(sl1) != smartlist_len(sl2))
-    return 0;
-  SMARTLIST_FOREACH(sl1, int *, cp1, {
-      int *cp2 = smartlist_get(sl2, cp1_sl_idx);
-      if (*cp1 != *cp2)
-        return 0;
-    });
-  return 1;
-}
-
-/** Return true iff <b>sl</b> has some element E such that
- * tor_memeq(E,<b>element</b>,DIGEST_LEN)
- */
-int
-smartlist_contains_digest(const smartlist_t *sl, const char *element)
-{
-  int i;
-  if (!sl) return 0;
-  for (i=0; i < sl->num_used; i++)
-    if (tor_memeq((const char*)sl->list[i],element,DIGEST_LEN))
-      return 1;
-  return 0;
-}
-
-/** Return true iff some element E of sl2 has smartlist_contains(sl1,E).
- */
-int
-smartlist_overlap(const smartlist_t *sl1, const smartlist_t *sl2)
-{
-  int i;
-  for (i=0; i < sl2->num_used; i++)
-    if (smartlist_contains(sl1, sl2->list[i]))
-      return 1;
-  return 0;
-}
-
-/** Remove every element E of sl1 such that !smartlist_contains(sl2,E).
- * Does not preserve the order of sl1.
- */
-void
-smartlist_intersect(smartlist_t *sl1, const smartlist_t *sl2)
-{
-  int i;
-  for (i=0; i < sl1->num_used; i++)
-    if (!smartlist_contains(sl2, sl1->list[i])) {
-      sl1->list[i] = sl1->list[--sl1->num_used]; /* swap with the end */
-      i--; /* so we process the new i'th element */
-      sl1->list[sl1->num_used] = NULL;
-    }
-}
-
-/** Remove every element E of sl1 such that smartlist_contains(sl2,E).
- * Does not preserve the order of sl1.
- */
-void
-smartlist_subtract(smartlist_t *sl1, const smartlist_t *sl2)
-{
-  int i;
-  for (i=0; i < sl2->num_used; i++)
-    smartlist_remove(sl1, sl2->list[i]);
-}
-
-/** Remove the <b>idx</b>th element of sl; if idx is not the last
- * element, swap the last element of sl into the <b>idx</b>th space.
- */
-void
-smartlist_del(smartlist_t *sl, int idx)
-{
-  tor_assert(sl);
-  tor_assert(idx>=0);
-  tor_assert(idx < sl->num_used);
-  sl->list[idx] = sl->list[--sl->num_used];
-  sl->list[sl->num_used] = NULL;
-}
-
-/** Remove the <b>idx</b>th element of sl; if idx is not the last element,
- * moving all subsequent elements back one space. Return the old value
- * of the <b>idx</b>th element.
- */
-void
-smartlist_del_keeporder(smartlist_t *sl, int idx)
-{
-  tor_assert(sl);
-  tor_assert(idx>=0);
-  tor_assert(idx < sl->num_used);
-  --sl->num_used;
-  if (idx < sl->num_used)
-    memmove(sl->list+idx, sl->list+idx+1, sizeof(void*)*(sl->num_used-idx));
-  sl->list[sl->num_used] = NULL;
-}
-
-/** Insert the value <b>val</b> as the new <b>idx</b>th element of
- * <b>sl</b>, moving all items previously at <b>idx</b> or later
- * forward one space.
- */
-void
-smartlist_insert(smartlist_t *sl, int idx, void *val)
-{
-  tor_assert(sl);
-  tor_assert(idx>=0);
-  tor_assert(idx <= sl->num_used);
-  if (idx == sl->num_used) {
-    smartlist_add(sl, val);
-  } else {
-    smartlist_ensure_capacity(sl, ((size_t) sl->num_used)+1);
-    /* Move other elements away */
-    if (idx < sl->num_used)
-      memmove(sl->list + idx + 1, sl->list + idx,
-              sizeof(void*)*(sl->num_used-idx));
-    sl->num_used++;
-    sl->list[idx] = val;
-  }
-}
-
-/**
- * Split a string <b>str</b> along all occurrences of <b>sep</b>,
- * appending the (newly allocated) split strings, in order, to
- * <b>sl</b>.  Return the number of strings added to <b>sl</b>.
- *
- * If <b>flags</b>&amp;SPLIT_SKIP_SPACE is true, remove initial and
- * trailing space from each entry.
- * If <b>flags</b>&amp;SPLIT_IGNORE_BLANK is true, remove any entries
- * of length 0.
- * If <b>flags</b>&amp;SPLIT_STRIP_SPACE is true, strip spaces from each
- * split string.
- *
- * If <b>max</b>\>0, divide the string into no more than <b>max</b> pieces. If
- * <b>sep</b> is NULL, split on any sequence of horizontal space.
- */
-int
-smartlist_split_string(smartlist_t *sl, const char *str, const char *sep,
-                       int flags, int max)
-{
-  const char *cp, *end, *next;
-  int n = 0;
-
-  tor_assert(sl);
-  tor_assert(str);
-
-  cp = str;
-  while (1) {
-    if (flags&SPLIT_SKIP_SPACE) {
-      while (TOR_ISSPACE(*cp)) ++cp;
-    }
-
-    if (max>0 && n == max-1) {
-      end = strchr(cp,'\0');
-    } else if (sep) {
-      end = strstr(cp,sep);
-      if (!end)
-        end = strchr(cp,'\0');
-    } else {
-      for (end = cp; *end && *end != '\t' && *end != ' '; ++end)
-        ;
-    }
-
-    tor_assert(end);
-
-    if (!*end) {
-      next = NULL;
-    } else if (sep) {
-      next = end+strlen(sep);
-    } else {
-      next = end+1;
-      while (*next == '\t' || *next == ' ')
-        ++next;
-    }
-
-    if (flags&SPLIT_SKIP_SPACE) {
-      while (end > cp && TOR_ISSPACE(*(end-1)))
-        --end;
-    }
-    if (end != cp || !(flags&SPLIT_IGNORE_BLANK)) {
-      char *string = tor_strndup(cp, end-cp);
-      if (flags&SPLIT_STRIP_SPACE)
-        tor_strstrip(string, " ");
-      smartlist_add(sl, string);
-      ++n;
-    }
-    if (!next)
-      break;
-    cp = next;
-  }
-
-  return n;
-}
-
-/** Allocate and return a new string containing the concatenation of
- * the elements of <b>sl</b>, in order, separated by <b>join</b>.  If
- * <b>terminate</b> is true, also terminate the string with <b>join</b>.
- * If <b>len_out</b> is not NULL, set <b>len_out</b> to the length of
- * the returned string. Requires that every element of <b>sl</b> is
- * NUL-terminated string.
- */
-char *
-smartlist_join_strings(smartlist_t *sl, const char *join,
-                       int terminate, size_t *len_out)
-{
-  return smartlist_join_strings2(sl,join,strlen(join),terminate,len_out);
-}
-
-/** As smartlist_join_strings, but instead of separating/terminated with a
- * NUL-terminated string <b>join</b>, uses the <b>join_len</b>-byte sequence
- * at <b>join</b>.  (Useful for generating a sequence of NUL-terminated
- * strings.)
- */
-char *
-smartlist_join_strings2(smartlist_t *sl, const char *join,
-                        size_t join_len, int terminate, size_t *len_out)
-{
-  int i;
-  size_t n = 0;
-  char *r = NULL, *dst, *src;
-
-  tor_assert(sl);
-  tor_assert(join);
-
-  if (terminate)
-    n = join_len;
-
-  for (i = 0; i < sl->num_used; ++i) {
-    n += strlen(sl->list[i]);
-    if (i+1 < sl->num_used) /* avoid double-counting the last one */
-      n += join_len;
-  }
-  dst = r = tor_malloc(n+1);
-  for (i = 0; i < sl->num_used; ) {
-    for (src = sl->list[i]; *src; )
-      *dst++ = *src++;
-    if (++i < sl->num_used) {
-      memcpy(dst, join, join_len);
-      dst += join_len;
-    }
-  }
-  if (terminate) {
-    memcpy(dst, join, join_len);
-    dst += join_len;
-  }
-  *dst = '\0';
-
-  if (len_out)
-    *len_out = dst-r;
-  return r;
-}
-
-/** Sort the members of <b>sl</b> into an order defined by
- * the ordering function <b>compare</b>, which returns less then 0 if a
- * precedes b, greater than 0 if b precedes a, and 0 if a 'equals' b.
- */
-void
-smartlist_sort(smartlist_t *sl, int (*compare)(const void **a, const void **b))
-{
-  if (!sl->num_used)
-    return;
-  qsort(sl->list, sl->num_used, sizeof(void*),
-        (int (*)(const void *,const void*))compare);
-}
-
-/** Given a smartlist <b>sl</b> sorted with the function <b>compare</b>,
- * return the most frequent member in the list.  Break ties in favor of
- * later elements.  If the list is empty, return NULL.  If count_out is
- * non-null, set it to the count of the most frequent member.
- */
-void *
-smartlist_get_most_frequent_(const smartlist_t *sl,
-                             int (*compare)(const void **a, const void **b),
-                             int *count_out)
-{
-  const void *most_frequent = NULL;
-  int most_frequent_count = 0;
-
-  const void *cur = NULL;
-  int i, count=0;
-
-  if (!sl->num_used) {
-    if (count_out)
-      *count_out = 0;
-    return NULL;
-  }
-  for (i = 0; i < sl->num_used; ++i) {
-    const void *item = sl->list[i];
-    if (cur && 0 == compare(&cur, &item)) {
-      ++count;
-    } else {
-      if (cur && count >= most_frequent_count) {
-        most_frequent = cur;
-        most_frequent_count = count;
-      }
-      cur = item;
-      count = 1;
-    }
-  }
-  if (cur && count >= most_frequent_count) {
-    most_frequent = cur;
-    most_frequent_count = count;
-  }
-  if (count_out)
-    *count_out = most_frequent_count;
-  return (void*)most_frequent;
-}
-
-/** Given a sorted smartlist <b>sl</b> and the comparison function used to
- * sort it, remove all duplicate members.  If free_fn is provided, calls
- * free_fn on each duplicate.  Otherwise, just removes them.  Preserves order.
- */
-void
-smartlist_uniq(smartlist_t *sl,
-               int (*compare)(const void **a, const void **b),
-               void (*free_fn)(void *a))
-{
-  int i;
-  for (i=1; i < sl->num_used; ++i) {
-    if (compare((const void **)&(sl->list[i-1]),
-                (const void **)&(sl->list[i])) == 0) {
-      if (free_fn)
-        free_fn(sl->list[i]);
-      smartlist_del_keeporder(sl, i--);
-    }
-  }
-}
-
-/** Assuming the members of <b>sl</b> are in order, return a pointer to the
- * member that matches <b>key</b>.  Ordering and matching are defined by a
- * <b>compare</b> function that returns 0 on a match; less than 0 if key is
- * less than member, and greater than 0 if key is greater then member.
- */
-void *
-smartlist_bsearch(smartlist_t *sl, const void *key,
-                  int (*compare)(const void *key, const void **member))
-{
-  int found, idx;
-  idx = smartlist_bsearch_idx(sl, key, compare, &found);
-  return found ? smartlist_get(sl, idx) : NULL;
-}
-
-/** Assuming the members of <b>sl</b> are in order, return the index of the
- * member that matches <b>key</b>.  If no member matches, return the index of
- * the first member greater than <b>key</b>, or smartlist_len(sl) if no member
- * is greater than <b>key</b>.  Set <b>found_out</b> to true on a match, to
- * false otherwise.  Ordering and matching are defined by a <b>compare</b>
- * function that returns 0 on a match; less than 0 if key is less than member,
- * and greater than 0 if key is greater then member.
- */
-int
-smartlist_bsearch_idx(const smartlist_t *sl, const void *key,
-                      int (*compare)(const void *key, const void **member),
-                      int *found_out)
-{
-  int hi, lo, cmp, mid, len, diff;
-
-  tor_assert(sl);
-  tor_assert(compare);
-  tor_assert(found_out);
-
-  len = smartlist_len(sl);
-
-  /* Check for the trivial case of a zero-length list */
-  if (len == 0) {
-    *found_out = 0;
-    /* We already know smartlist_len(sl) is 0 in this case */
-    return 0;
-  }
-
-  /* Okay, we have a real search to do */
-  tor_assert(len > 0);
-  lo = 0;
-  hi = len - 1;
-
-  /*
-   * These invariants are always true:
-   *
-   * For all i such that 0 <= i < lo, sl[i] < key
-   * For all i such that hi < i <= len, sl[i] > key
-   */
-
-  while (lo <= hi) {
-    diff = hi - lo;
-    /*
-     * We want mid = (lo + hi) / 2, but that could lead to overflow, so
-     * instead diff = hi - lo (non-negative because of loop condition), and
-     * then hi = lo + diff, mid = (lo + lo + diff) / 2 = lo + (diff / 2).
-     */
-    mid = lo + (diff / 2);
-    cmp = compare(key, (const void**) &(sl->list[mid]));
-    if (cmp == 0) {
-      /* sl[mid] == key; we found it */
-      *found_out = 1;
-      return mid;
-    } else if (cmp > 0) {
-      /*
-       * key > sl[mid] and an index i such that sl[i] == key must
-       * have i > mid if it exists.
-       */
-
-      /*
-       * Since lo <= mid <= hi, hi can only decrease on each iteration (by
-       * being set to mid - 1) and hi is initially len - 1, mid < len should
-       * always hold, and this is not symmetric with the left end of list
-       * mid > 0 test below.  A key greater than the right end of the list
-       * should eventually lead to lo == hi == mid == len - 1, and then
-       * we set lo to len below and fall out to the same exit we hit for
-       * a key in the middle of the list but not matching.  Thus, we just
-       * assert for consistency here rather than handle a mid == len case.
-       */
-      tor_assert(mid < len);
-      /* Move lo to the element immediately after sl[mid] */
-      lo = mid + 1;
-    } else {
-      /* This should always be true in this case */
-      tor_assert(cmp < 0);
-
-      /*
-       * key < sl[mid] and an index i such that sl[i] == key must
-       * have i < mid if it exists.
-       */
-
-      if (mid > 0) {
-        /* Normal case, move hi to the element immediately before sl[mid] */
-        hi = mid - 1;
-      } else {
-        /* These should always be true in this case */
-        tor_assert(mid == lo);
-        tor_assert(mid == 0);
-        /*
-         * We were at the beginning of the list and concluded that every
-         * element e compares e > key.
-         */
-        *found_out = 0;
-        return 0;
-      }
-    }
-  }
-
-  /*
-   * lo > hi; we have no element matching key but we have elements falling
-   * on both sides of it.  The lo index points to the first element > key.
-   */
-  tor_assert(lo == hi + 1); /* All other cases should have been handled */
-  tor_assert(lo >= 0);
-  tor_assert(lo <= len);
-  tor_assert(hi >= 0);
-  tor_assert(hi <= len);
-
-  if (lo < len) {
-    cmp = compare(key, (const void **) &(sl->list[lo]));
-    tor_assert(cmp < 0);
-  } else {
-    cmp = compare(key, (const void **) &(sl->list[len-1]));
-    tor_assert(cmp > 0);
-  }
-
-  *found_out = 0;
-  return lo;
-}
-
-/** Helper: compare two const char **s. */
-static int
-compare_string_ptrs_(const void **_a, const void **_b)
-{
-  return strcmp((const char*)*_a, (const char*)*_b);
-}
-
-/** Sort a smartlist <b>sl</b> containing strings into lexically ascending
- * order. */
-void
-smartlist_sort_strings(smartlist_t *sl)
-{
-  smartlist_sort(sl, compare_string_ptrs_);
-}
-
-/** Return the most frequent string in the sorted list <b>sl</b> */
-const char *
-smartlist_get_most_frequent_string(smartlist_t *sl)
-{
-  return smartlist_get_most_frequent(sl, compare_string_ptrs_);
-}
-
-/** Return the most frequent string in the sorted list <b>sl</b>.
- * If <b>count_out</b> is provided, set <b>count_out</b> to the
- * number of times that string appears.
- */
-const char *
-smartlist_get_most_frequent_string_(smartlist_t *sl, int *count_out)
-{
-  return smartlist_get_most_frequent_(sl, compare_string_ptrs_, count_out);
-}
-
-/** Remove duplicate strings from a sorted list, and free them with tor_free().
- */
-void
-smartlist_uniq_strings(smartlist_t *sl)
-{
-  smartlist_uniq(sl, compare_string_ptrs_, tor_free_);
-}
-
-/** Helper: compare two pointers. */
-static int
-compare_ptrs_(const void **_a, const void **_b)
-{
-  const void *a = *_a, *b = *_b;
-  if (a<b)
-    return -1;
-  else if (a==b)
-    return 0;
-  else
-    return 1;
-}
-
-/** Sort <b>sl</b> in ascending order of the pointers it contains. */
-void
-smartlist_sort_pointers(smartlist_t *sl)
-{
-  smartlist_sort(sl, compare_ptrs_);
-}
-
-/* Heap-based priority queue implementation for O(lg N) insert and remove.
- * Recall that the heap property is that, for every index I, h[I] <
- * H[LEFT_CHILD[I]] and h[I] < H[RIGHT_CHILD[I]].
- *
- * For us to remove items other than the topmost item, each item must store
- * its own index within the heap.  When calling the pqueue functions, tell
- * them about the offset of the field that stores the index within the item.
- *
- * Example:
- *
- *   typedef struct timer_t {
- *     struct timeval tv;
- *     int heap_index;
- *   } timer_t;
- *
- *   static int compare(const void *p1, const void *p2) {
- *     const timer_t *t1 = p1, *t2 = p2;
- *     if (t1->tv.tv_sec < t2->tv.tv_sec) {
- *        return -1;
- *     } else if (t1->tv.tv_sec > t2->tv.tv_sec) {
- *        return 1;
- *     } else {
- *        return t1->tv.tv_usec - t2->tv_usec;
- *     }
- *   }
- *
- *   void timer_heap_insert(smartlist_t *heap, timer_t *timer) {
- *      smartlist_pqueue_add(heap, compare, STRUCT_OFFSET(timer_t, heap_index),
- *         timer);
- *   }
- *
- *   void timer_heap_pop(smartlist_t *heap) {
- *      return smartlist_pqueue_pop(heap, compare,
- *         STRUCT_OFFSET(timer_t, heap_index));
- *   }
- */
-
-/** @{ */
-/** Functions to manipulate heap indices to find a node's parent and children.
- *
- * For a 1-indexed array, we would use LEFT_CHILD[x] = 2*x and RIGHT_CHILD[x]
- *   = 2*x + 1.  But this is C, so we have to adjust a little. */
-
-/* MAX_PARENT_IDX is the largest IDX in the smartlist which might have
- * children whose indices fit inside an int.
- * LEFT_CHILD(MAX_PARENT_IDX) == INT_MAX-2;
- * RIGHT_CHILD(MAX_PARENT_IDX) == INT_MAX-1;
- * LEFT_CHILD(MAX_PARENT_IDX + 1) == INT_MAX // impossible, see max list size.
- */
-#define MAX_PARENT_IDX ((INT_MAX - 2) / 2)
-/* If this is true, then i is small enough to potentially have children
- * in the smartlist, and it is save to use LEFT_CHILD/RIGHT_CHILD on it. */
-#define IDX_MAY_HAVE_CHILDREN(i) ((i) <= MAX_PARENT_IDX)
-#define LEFT_CHILD(i)  ( 2*(i) + 1 )
-#define RIGHT_CHILD(i) ( 2*(i) + 2 )
-#define PARENT(i)      ( ((i)-1) / 2 )
-/** }@ */
-
-/** @{ */
-/** Helper macros for heaps: Given a local variable <b>idx_field_offset</b>
- * set to the offset of an integer index within the heap element structure,
- * IDX_OF_ITEM(p) gives you the index of p, and IDXP(p) gives you a pointer to
- * where p's index is stored.  Given additionally a local smartlist <b>sl</b>,
- * UPDATE_IDX(i) sets the index of the element at <b>i</b> to the correct
- * value (that is, to <b>i</b>).
- */
-#define IDXP(p) ((int*)STRUCT_VAR_P(p, idx_field_offset))
-
-#define UPDATE_IDX(i)  do {                            \
-    void *updated = sl->list[i];                       \
-    *IDXP(updated) = i;                                \
-  } while (0)
-
-#define IDX_OF_ITEM(p) (*IDXP(p))
-/** @} */
-
-/** Helper. <b>sl</b> may have at most one violation of the heap property:
- * the item at <b>idx</b> may be greater than one or both of its children.
- * Restore the heap property. */
-static inline void
-smartlist_heapify(smartlist_t *sl,
-                  int (*compare)(const void *a, const void *b),
-                  int idx_field_offset,
-                  int idx)
-{
-  while (1) {
-    if (! IDX_MAY_HAVE_CHILDREN(idx)) {
-      /* idx is so large that it cannot have any children, since doing so
-       * would mean the smartlist was over-capacity. Therefore it cannot
-       * violate the heap property by being greater than a child (since it
-       * doesn't have any). */
-      return;
-    }
-
-    int left_idx = LEFT_CHILD(idx);
-    int best_idx;
-
-    if (left_idx >= sl->num_used)
-      return;
-    if (compare(sl->list[idx],sl->list[left_idx]) < 0)
-      best_idx = idx;
-    else
-      best_idx = left_idx;
-    if (left_idx+1 < sl->num_used &&
-        compare(sl->list[left_idx+1],sl->list[best_idx]) < 0)
-      best_idx = left_idx + 1;
-
-    if (best_idx == idx) {
-      return;
-    } else {
-      void *tmp = sl->list[idx];
-      sl->list[idx] = sl->list[best_idx];
-      sl->list[best_idx] = tmp;
-      UPDATE_IDX(idx);
-      UPDATE_IDX(best_idx);
-
-      idx = best_idx;
-    }
-  }
-}
-
-/** Insert <b>item</b> into the heap stored in <b>sl</b>, where order is
- * determined by <b>compare</b> and the offset of the item in the heap is
- * stored in an int-typed field at position <b>idx_field_offset</b> within
- * item.
- */
-void
-smartlist_pqueue_add(smartlist_t *sl,
-                     int (*compare)(const void *a, const void *b),
-                     int idx_field_offset,
-                     void *item)
-{
-  int idx;
-  smartlist_add(sl,item);
-  UPDATE_IDX(sl->num_used-1);
-
-  for (idx = sl->num_used - 1; idx; ) {
-    int parent = PARENT(idx);
-    if (compare(sl->list[idx], sl->list[parent]) < 0) {
-      void *tmp = sl->list[parent];
-      sl->list[parent] = sl->list[idx];
-      sl->list[idx] = tmp;
-      UPDATE_IDX(parent);
-      UPDATE_IDX(idx);
-      idx = parent;
-    } else {
-      return;
-    }
-  }
-}
-
-/** Remove and return the top-priority item from the heap stored in <b>sl</b>,
- * where order is determined by <b>compare</b> and the item's position is
- * stored at position <b>idx_field_offset</b> within the item.  <b>sl</b> must
- * not be empty. */
-void *
-smartlist_pqueue_pop(smartlist_t *sl,
-                     int (*compare)(const void *a, const void *b),
-                     int idx_field_offset)
-{
-  void *top;
-  tor_assert(sl->num_used);
-
-  top = sl->list[0];
-  *IDXP(top)=-1;
-  if (--sl->num_used) {
-    sl->list[0] = sl->list[sl->num_used];
-    sl->list[sl->num_used] = NULL;
-    UPDATE_IDX(0);
-    smartlist_heapify(sl, compare, idx_field_offset, 0);
-  }
-  sl->list[sl->num_used] = NULL;
-  return top;
-}
-
-/** Remove the item <b>item</b> from the heap stored in <b>sl</b>,
- * where order is determined by <b>compare</b> and the item's position is
- * stored at position <b>idx_field_offset</b> within the item.  <b>sl</b> must
- * not be empty. */
-void
-smartlist_pqueue_remove(smartlist_t *sl,
-                        int (*compare)(const void *a, const void *b),
-                        int idx_field_offset,
-                        void *item)
-{
-  int idx = IDX_OF_ITEM(item);
-  tor_assert(idx >= 0);
-  tor_assert(sl->list[idx] == item);
-  --sl->num_used;
-  *IDXP(item) = -1;
-  if (idx == sl->num_used) {
-    sl->list[sl->num_used] = NULL;
-    return;
-  } else {
-    sl->list[idx] = sl->list[sl->num_used];
-    sl->list[sl->num_used] = NULL;
-    UPDATE_IDX(idx);
-    smartlist_heapify(sl, compare, idx_field_offset, idx);
-  }
-}
-
-/** Assert that the heap property is correctly maintained by the heap stored
- * in <b>sl</b>, where order is determined by <b>compare</b>. */
-void
-smartlist_pqueue_assert_ok(smartlist_t *sl,
-                           int (*compare)(const void *a, const void *b),
-                           int idx_field_offset)
-{
-  int i;
-  for (i = sl->num_used - 1; i >= 0; --i) {
-    if (i>0)
-      tor_assert(compare(sl->list[PARENT(i)], sl->list[i]) <= 0);
-    tor_assert(IDX_OF_ITEM(sl->list[i]) == i);
-  }
-}
-
-/** Helper: compare two DIGEST_LEN digests. */
-static int
-compare_digests_(const void **_a, const void **_b)
-{
-  return tor_memcmp((const char*)*_a, (const char*)*_b, DIGEST_LEN);
-}
-
-/** Sort the list of DIGEST_LEN-byte digests into ascending order. */
-void
-smartlist_sort_digests(smartlist_t *sl)
-{
-  smartlist_sort(sl, compare_digests_);
-}
-
-/** Remove duplicate digests from a sorted list, and free them with tor_free().
- */
-void
-smartlist_uniq_digests(smartlist_t *sl)
-{
-  smartlist_uniq(sl, compare_digests_, tor_free_);
-}
-
-/** Helper: compare two DIGEST256_LEN digests. */
-static int
-compare_digests256_(const void **_a, const void **_b)
-{
-  return tor_memcmp((const char*)*_a, (const char*)*_b, DIGEST256_LEN);
-}
-
-/** Sort the list of DIGEST256_LEN-byte digests into ascending order. */
-void
-smartlist_sort_digests256(smartlist_t *sl)
-{
-  smartlist_sort(sl, compare_digests256_);
-}
-
-/** Return the most frequent member of the sorted list of DIGEST256_LEN
- * digests in <b>sl</b> */
-const uint8_t *
-smartlist_get_most_frequent_digest256(smartlist_t *sl)
-{
-  return smartlist_get_most_frequent(sl, compare_digests256_);
-}
-
-/** Remove duplicate 256-bit digests from a sorted list, and free them with
- * tor_free().
- */
-void
-smartlist_uniq_digests256(smartlist_t *sl)
-{
-  smartlist_uniq(sl, compare_digests256_, tor_free_);
-}
-
-/** Helper: Declare an entry type and a map type to implement a mapping using
- * ht.h.  The map type will be called <b>maptype</b>.  The key part of each
- * entry is declared using the C declaration <b>keydecl</b>.  All functions
- * and types associated with the map get prefixed with <b>prefix</b> */
-#define DEFINE_MAP_STRUCTS(maptype, keydecl, prefix)      \
-  typedef struct prefix ## entry_t {                      \
-    HT_ENTRY(prefix ## entry_t) node;                     \
-    void *val;                                            \
-    keydecl;                                              \
-  } prefix ## entry_t;                                    \
-  struct maptype {                                        \
-    HT_HEAD(prefix ## impl, prefix ## entry_t) head;      \
-  }
-
-DEFINE_MAP_STRUCTS(strmap_t, char *key, strmap_);
-DEFINE_MAP_STRUCTS(digestmap_t, char key[DIGEST_LEN], digestmap_);
-DEFINE_MAP_STRUCTS(digest256map_t, uint8_t key[DIGEST256_LEN], digest256map_);
-
-/** Helper: compare strmap_entry_t objects by key value. */
-static inline int
-strmap_entries_eq(const strmap_entry_t *a, const strmap_entry_t *b)
-{
-  return !strcmp(a->key, b->key);
-}
-
-/** Helper: return a hash value for a strmap_entry_t. */
-static inline unsigned int
-strmap_entry_hash(const strmap_entry_t *a)
-{
-  return (unsigned) siphash24g(a->key, strlen(a->key));
-}
-
-/** Helper: compare digestmap_entry_t objects by key value. */
-static inline int
-digestmap_entries_eq(const digestmap_entry_t *a, const digestmap_entry_t *b)
-{
-  return tor_memeq(a->key, b->key, DIGEST_LEN);
-}
-
-/** Helper: return a hash value for a digest_map_t. */
-static inline unsigned int
-digestmap_entry_hash(const digestmap_entry_t *a)
-{
-  return (unsigned) siphash24g(a->key, DIGEST_LEN);
-}
-
-/** Helper: compare digestmap_entry_t objects by key value. */
-static inline int
-digest256map_entries_eq(const digest256map_entry_t *a,
-                        const digest256map_entry_t *b)
-{
-  return tor_memeq(a->key, b->key, DIGEST256_LEN);
-}
-
-/** Helper: return a hash value for a digest_map_t. */
-static inline unsigned int
-digest256map_entry_hash(const digest256map_entry_t *a)
-{
-  return (unsigned) siphash24g(a->key, DIGEST256_LEN);
-}
-
-HT_PROTOTYPE(strmap_impl, strmap_entry_t, node, strmap_entry_hash,
-             strmap_entries_eq)
-HT_GENERATE2(strmap_impl, strmap_entry_t, node, strmap_entry_hash,
-             strmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
-
-HT_PROTOTYPE(digestmap_impl, digestmap_entry_t, node, digestmap_entry_hash,
-             digestmap_entries_eq)
-HT_GENERATE2(digestmap_impl, digestmap_entry_t, node, digestmap_entry_hash,
-             digestmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
-
-HT_PROTOTYPE(digest256map_impl, digest256map_entry_t, node,
-             digest256map_entry_hash,
-             digest256map_entries_eq)
-HT_GENERATE2(digest256map_impl, digest256map_entry_t, node,
-             digest256map_entry_hash,
-             digest256map_entries_eq, 0.6, tor_reallocarray_, tor_free_)
-
-static inline void
-strmap_entry_free(strmap_entry_t *ent)
-{
-  tor_free(ent->key);
-  tor_free(ent);
-}
-static inline void
-digestmap_entry_free(digestmap_entry_t *ent)
-{
-  tor_free(ent);
-}
-static inline void
-digest256map_entry_free(digest256map_entry_t *ent)
-{
-  tor_free(ent);
-}
-
-static inline void
-strmap_assign_tmp_key(strmap_entry_t *ent, const char *key)
-{
-  ent->key = (char*)key;
-}
-static inline void
-digestmap_assign_tmp_key(digestmap_entry_t *ent, const char *key)
-{
-  memcpy(ent->key, key, DIGEST_LEN);
-}
-static inline void
-digest256map_assign_tmp_key(digest256map_entry_t *ent, const uint8_t *key)
-{
-  memcpy(ent->key, key, DIGEST256_LEN);
-}
-static inline void
-strmap_assign_key(strmap_entry_t *ent, const char *key)
-{
-  ent->key = tor_strdup(key);
-}
-static inline void
-digestmap_assign_key(digestmap_entry_t *ent, const char *key)
-{
-  memcpy(ent->key, key, DIGEST_LEN);
-}
-static inline void
-digest256map_assign_key(digest256map_entry_t *ent, const uint8_t *key)
-{
-  memcpy(ent->key, key, DIGEST256_LEN);
-}
-
-/**
- * Macro: implement all the functions for a map that are declared in
- * container.h by the DECLARE_MAP_FNS() macro.  You must additionally define a
- * prefix_entry_free_() function to free entries (and their keys), a
- * prefix_assign_tmp_key() function to temporarily set a stack-allocated
- * entry to hold a key, and a prefix_assign_key() function to set a
- * heap-allocated entry to hold a key.
- */
-#define IMPLEMENT_MAP_FNS(maptype, keytype, prefix)                     \
-  /** Create and return a new empty map. */                             \
-  MOCK_IMPL(maptype *,                                                  \
-  prefix##_new,(void))                                                  \
-  {                                                                     \
-    maptype *result;                                                    \
-    result = tor_malloc(sizeof(maptype));                               \
-    HT_INIT(prefix##_impl, &result->head);                              \
-    return result;                                                      \
-  }                                                                     \
-                                                                        \
-  /** Return the item from <b>map</b> whose key matches <b>key</b>, or  \
-   * NULL if no such value exists. */                                   \
-  void *                                                                \
-  prefix##_get(const maptype *map, const keytype key)                   \
-  {                                                                     \
-    prefix ##_entry_t *resolve;                                         \
-    prefix ##_entry_t search;                                           \
-    tor_assert(map);                                                    \
-    tor_assert(key);                                                    \
-    prefix ##_assign_tmp_key(&search, key);                             \
-    resolve = HT_FIND(prefix ##_impl, &map->head, &search);             \
-    if (resolve) {                                                      \
-      return resolve->val;                                              \
-    } else {                                                            \
-      return NULL;                                                      \
-    }                                                                   \
-  }                                                                     \
-                                                                        \
-  /** Add an entry to <b>map</b> mapping <b>key</b> to <b>val</b>;      \
-   * return the previous value, or NULL if no such value existed. */     \
-  void *                                                                \
-  prefix##_set(maptype *map, const keytype key, void *val)              \
-  {                                                                     \
-    prefix##_entry_t search;                                            \
-    void *oldval;                                                       \
-    tor_assert(map);                                                    \
-    tor_assert(key);                                                    \
-    tor_assert(val);                                                    \
-    prefix##_assign_tmp_key(&search, key);                              \
-    /* We a lot of our time in this function, so the code below is */   \
-    /* meant to optimize the check/alloc/set cycle by avoiding the two */\
-    /* trips to the hash table that we would do in the unoptimized */   \
-    /* version of this code. (Each of HT_INSERT and HT_FIND calls */     \
-    /* HT_SET_HASH and HT_FIND_P.) */                                   \
-    HT_FIND_OR_INSERT_(prefix##_impl, node, prefix##_entry_hash,        \
-                       &(map->head),                                    \
-                       prefix##_entry_t, &search, ptr,                  \
-                       {                                                \
-                         /* we found an entry. */                       \
-                         oldval = (*ptr)->val;                          \
-                         (*ptr)->val = val;                             \
-                         return oldval;                                 \
-                       },                                               \
-                       {                                                \
-                         /* We didn't find the entry. */                \
-                         prefix##_entry_t *newent =                     \
-                           tor_malloc_zero(sizeof(prefix##_entry_t));   \
-                         prefix##_assign_key(newent, key);              \
-                         newent->val = val;                             \
-                         HT_FOI_INSERT_(node, &(map->head),             \
-                            &search, newent, ptr);                      \
-                         return NULL;                                   \
-    });                                                                 \
-  }                                                                     \
-                                                                        \
-  /** Remove the value currently associated with <b>key</b> from the map. \
-   * Return the value if one was set, or NULL if there was no entry for \
-   * <b>key</b>.                                                        \
-   *                                                                    \
-   * Note: you must free any storage associated with the returned value. \
-   */                                                                   \
-  void *                                                                \
-  prefix##_remove(maptype *map, const keytype key)                      \
-  {                                                                     \
-    prefix##_entry_t *resolve;                                          \
-    prefix##_entry_t search;                                            \
-    void *oldval;                                                       \
-    tor_assert(map);                                                    \
-    tor_assert(key);                                                    \
-    prefix##_assign_tmp_key(&search, key);                              \
-    resolve = HT_REMOVE(prefix##_impl, &map->head, &search);            \
-    if (resolve) {                                                      \
-      oldval = resolve->val;                                            \
-      prefix##_entry_free(resolve);                                     \
-      return oldval;                                                    \
-    } else {                                                            \
-      return NULL;                                                      \
-    }                                                                   \
-  }                                                                     \
-                                                                        \
-  /** Return the number of elements in <b>map</b>. */                   \
-  int                                                                   \
-  prefix##_size(const maptype *map)                                     \
-  {                                                                     \
-    return HT_SIZE(&map->head);                                         \
-  }                                                                     \
-                                                                        \
-  /** Return true iff <b>map</b> has no entries. */                     \
-  int                                                                   \
-  prefix##_isempty(const maptype *map)                                  \
-  {                                                                     \
-    return HT_EMPTY(&map->head);                                        \
-  }                                                                     \
-                                                                        \
-  /** Assert that <b>map</b> is not corrupt. */                         \
-  void                                                                  \
-  prefix##_assert_ok(const maptype *map)                                \
-  {                                                                     \
-    tor_assert(!prefix##_impl_HT_REP_IS_BAD_(&map->head));              \
-  }                                                                     \
-                                                                        \
-  /** Remove all entries from <b>map</b>, and deallocate storage for    \
-   * those entries.  If free_val is provided, invoked it every value in \
-   * <b>map</b>. */                                                     \
-  MOCK_IMPL(void,                                                       \
-  prefix##_free, (maptype *map, void (*free_val)(void*)))               \
-  {                                                                     \
-    prefix##_entry_t **ent, **next, *this;                              \
-    if (!map)                                                           \
-      return;                                                           \
-    for (ent = HT_START(prefix##_impl, &map->head); ent != NULL;        \
-         ent = next) {                                                  \
-      this = *ent;                                                      \
-      next = HT_NEXT_RMV(prefix##_impl, &map->head, ent);               \
-      if (free_val)                                                     \
-        free_val(this->val);                                            \
-      prefix##_entry_free(this);                                        \
-    }                                                                   \
-    tor_assert(HT_EMPTY(&map->head));                                   \
-    HT_CLEAR(prefix##_impl, &map->head);                                \
-    tor_free(map);                                                      \
-  }                                                                     \
-                                                                        \
-  /** return an <b>iterator</b> pointer to the front of a map.          \
-   *                                                                    \
-   * Iterator example:                                                  \
-   *                                                                    \
-   * \code                                                              \
-   * // uppercase values in "map", removing empty values.               \
-   *                                                                    \
-   * strmap_iter_t *iter;                                               \
-   * const char *key;                                                   \
-   * void *val;                                                         \
-   * char *cp;                                                          \
-   *                                                                    \
-   * for (iter = strmap_iter_init(map); !strmap_iter_done(iter); ) {    \
-   *    strmap_iter_get(iter, &key, &val);                              \
-   *    cp = (char*)val;                                                \
-   *    if (!*cp) {                                                     \
-   *       iter = strmap_iter_next_rmv(map,iter);                       \
-   *       free(val);                                                   \
-   *    } else {                                                        \
-   *       for (;*cp;cp++) *cp = TOR_TOUPPER(*cp);                      \
-   */                                                                   \
-  prefix##_iter_t *                                                     \
-  prefix##_iter_init(maptype *map)                                      \
-  {                                                                     \
-    tor_assert(map);                                                    \
-    return HT_START(prefix##_impl, &map->head);                         \
-  }                                                                     \
-                                                                        \
-  /** Advance <b>iter</b> a single step to the next entry, and return   \
-   * its new value. */                                                  \
-  prefix##_iter_t *                                                     \
-  prefix##_iter_next(maptype *map, prefix##_iter_t *iter)               \
-  {                                                                     \
-    tor_assert(map);                                                    \
-    tor_assert(iter);                                                   \
-    return HT_NEXT(prefix##_impl, &map->head, iter);                    \
-  }                                                                     \
-  /** Advance <b>iter</b> a single step to the next entry, removing the \
-   * current entry, and return its new value. */                        \
-  prefix##_iter_t *                                                     \
-  prefix##_iter_next_rmv(maptype *map, prefix##_iter_t *iter)           \
-  {                                                                     \
-    prefix##_entry_t *rmv;                                              \
-    tor_assert(map);                                                    \
-    tor_assert(iter);                                                   \
-    tor_assert(*iter);                                                  \
-    rmv = *iter;                                                        \
-    iter = HT_NEXT_RMV(prefix##_impl, &map->head, iter);                \
-    prefix##_entry_free(rmv);                                           \
-    return iter;                                                        \
-  }                                                                     \
-  /** Set *<b>keyp</b> and *<b>valp</b> to the current entry pointed    \
-   * to by iter. */                                                     \
-  void                                                                  \
-  prefix##_iter_get(prefix##_iter_t *iter, const keytype *keyp,         \
-                    void **valp)                                        \
-  {                                                                     \
-    tor_assert(iter);                                                   \
-    tor_assert(*iter);                                                  \
-    tor_assert(keyp);                                                   \
-    tor_assert(valp);                                                   \
-    *keyp = (*iter)->key;                                               \
-    *valp = (*iter)->val;                                               \
-  }                                                                     \
-  /** Return true iff <b>iter</b> has advanced past the last entry of   \
-   * <b>map</b>. */                                                     \
-  int                                                                   \
-  prefix##_iter_done(prefix##_iter_t *iter)                             \
-  {                                                                     \
-    return iter == NULL;                                                \
-  }
-
-IMPLEMENT_MAP_FNS(strmap_t, char *, strmap)
-IMPLEMENT_MAP_FNS(digestmap_t, char *, digestmap)
-IMPLEMENT_MAP_FNS(digest256map_t, uint8_t *, digest256map)
-
-/** Same as strmap_set, but first converts <b>key</b> to lowercase. */
-void *
-strmap_set_lc(strmap_t *map, const char *key, void *val)
-{
-  /* We could be a little faster by using strcasecmp instead, and a separate
-   * type, but I don't think it matters. */
-  void *v;
-  char *lc_key = tor_strdup(key);
-  tor_strlower(lc_key);
-  v = strmap_set(map,lc_key,val);
-  tor_free(lc_key);
-  return v;
-}
-
-/** Same as strmap_get, but first converts <b>key</b> to lowercase. */
-void *
-strmap_get_lc(const strmap_t *map, const char *key)
-{
-  void *v;
-  char *lc_key = tor_strdup(key);
-  tor_strlower(lc_key);
-  v = strmap_get(map,lc_key);
-  tor_free(lc_key);
-  return v;
-}
-
-/** Same as strmap_remove, but first converts <b>key</b> to lowercase */
-void *
-strmap_remove_lc(strmap_t *map, const char *key)
-{
-  void *v;
-  char *lc_key = tor_strdup(key);
-  tor_strlower(lc_key);
-  v = strmap_remove(map,lc_key);
-  tor_free(lc_key);
-  return v;
-}
-
-/** Declare a function called <b>funcname</b> that acts as a find_nth_FOO
- * function for an array of type <b>elt_t</b>*.
- *
- * NOTE: The implementation kind of sucks: It's O(n log n), whereas finding
- * the kth element of an n-element list can be done in O(n).  Then again, this
- * implementation is not in critical path, and it is obviously correct. */
-#define IMPLEMENT_ORDER_FUNC(funcname, elt_t)                   \
-  static int                                                    \
-  _cmp_ ## elt_t(const void *_a, const void *_b)                \
-  {                                                             \
-    const elt_t *a = _a, *b = _b;                               \
-    if (*a<*b)                                                  \
-      return -1;                                                \
-    else if (*a>*b)                                             \
-      return 1;                                                 \
-    else                                                        \
-      return 0;                                                 \
-  }                                                             \
-  elt_t                                                         \
-  funcname(elt_t *array, int n_elements, int nth)               \
-  {                                                             \
-    tor_assert(nth >= 0);                                       \
-    tor_assert(nth < n_elements);                               \
-    qsort(array, n_elements, sizeof(elt_t), _cmp_ ##elt_t);     \
-    return array[nth];                                          \
-  }
-
-IMPLEMENT_ORDER_FUNC(find_nth_int, int)
-IMPLEMENT_ORDER_FUNC(find_nth_time, time_t)
-IMPLEMENT_ORDER_FUNC(find_nth_double, double)
-IMPLEMENT_ORDER_FUNC(find_nth_uint32, uint32_t)
-IMPLEMENT_ORDER_FUNC(find_nth_int32, int32_t)
-IMPLEMENT_ORDER_FUNC(find_nth_long, long)
-
-/** Return a newly allocated digestset_t, optimized to hold a total of
- * <b>max_elements</b> digests with a reasonably low false positive weight. */
-digestset_t *
-digestset_new(int max_elements)
-{
-  /* The probability of false positives is about P=(1 - exp(-kn/m))^k, where k
-   * is the number of hash functions per entry, m is the bits in the array,
-   * and n is the number of elements inserted.  For us, k==4, n<=max_elements,
-   * and m==n_bits= approximately max_elements*32.  This gives
-   *   P<(1-exp(-4*n/(32*n)))^4 == (1-exp(1/-8))^4 == .00019
-   *
-   * It would be more optimal in space vs false positives to get this false
-   * positive rate by going for k==13, and m==18.5n, but we also want to
-   * conserve CPU, and k==13 is pretty big.
-   */
-  int n_bits = 1u << (tor_log2(max_elements)+5);
-  digestset_t *r = tor_malloc(sizeof(digestset_t));
-  r->mask = n_bits - 1;
-  r->ba = bitarray_init_zero(n_bits);
-  return r;
-}
-
-/** Free all storage held in <b>set</b>. */
-void
-digestset_free(digestset_t *set)
-{
-  if (!set)
-    return;
-  bitarray_free(set->ba);
-  tor_free(set);
-}
-
diff --git a/src/common/container.h b/src/common/container.h
deleted file mode 100644
index 71495b660a..0000000000
--- a/src/common/container.h
+++ /dev/null
@@ -1,725 +0,0 @@
-/* Copyright (c) 2003-2004, Roger Dingledine
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#ifndef TOR_CONTAINER_H
-#define TOR_CONTAINER_H
-
-#include "util.h"
-#include "siphash.h"
-
-/** A resizeable list of pointers, with associated helpful functionality.
- *
- * The members of this struct are exposed only so that macros and inlines can
- * use them; all access to smartlist internals should go through the functions
- * and macros defined here.
- **/
-typedef struct smartlist_t {
-  /** @{ */
-  /** <b>list</b> has enough capacity to store exactly <b>capacity</b> elements
-   * before it needs to be resized.  Only the first <b>num_used</b> (\<=
-   * capacity) elements point to valid data.
-   */
-  void **list;
-  int num_used;
-  int capacity;
-  /** @} */
-} smartlist_t;
-
-MOCK_DECL(smartlist_t *, smartlist_new, (void));
-MOCK_DECL(void, smartlist_free, (smartlist_t *sl));
-void smartlist_clear(smartlist_t *sl);
-void smartlist_add(smartlist_t *sl, void *element);
-void smartlist_add_all(smartlist_t *sl, const smartlist_t *s2);
-void smartlist_remove(smartlist_t *sl, const void *element);
-void *smartlist_pop_last(smartlist_t *sl);
-void smartlist_reverse(smartlist_t *sl);
-void smartlist_string_remove(smartlist_t *sl, const char *element);
-int smartlist_contains(const smartlist_t *sl, const void *element);
-int smartlist_contains_string(const smartlist_t *sl, const char *element);
-int smartlist_pos(const smartlist_t *sl, const void *element);
-int smartlist_string_pos(const smartlist_t *, const char *elt);
-int smartlist_contains_string_case(const smartlist_t *sl, const char *element);
-int smartlist_contains_int_as_string(const smartlist_t *sl, int num);
-int smartlist_strings_eq(const smartlist_t *sl1, const smartlist_t *sl2);
-int smartlist_contains_digest(const smartlist_t *sl, const char *element);
-int smartlist_ints_eq(const smartlist_t *sl1, const smartlist_t *sl2);
-int smartlist_overlap(const smartlist_t *sl1, const smartlist_t *sl2);
-void smartlist_intersect(smartlist_t *sl1, const smartlist_t *sl2);
-void smartlist_subtract(smartlist_t *sl1, const smartlist_t *sl2);
-
-/* smartlist_choose() is defined in crypto.[ch] */
-#ifdef DEBUG_SMARTLIST
-/** Return the number of items in sl.
- */
-static inline int smartlist_len(const smartlist_t *sl);
-static inline int smartlist_len(const smartlist_t *sl) {
-  tor_assert(sl);
-  return (sl)->num_used;
-}
-/** Return the <b>idx</b>th element of sl.
- */
-static inline void *smartlist_get(const smartlist_t *sl, int idx);
-static inline void *smartlist_get(const smartlist_t *sl, int idx) {
-  tor_assert(sl);
-  tor_assert(idx>=0);
-  tor_assert(sl->num_used > idx);
-  return sl->list[idx];
-}
-static inline void smartlist_set(smartlist_t *sl, int idx, void *val) {
-  tor_assert(sl);
-  tor_assert(idx>=0);
-  tor_assert(sl->num_used > idx);
-  sl->list[idx] = val;
-}
-#else
-#define smartlist_len(sl) ((sl)->num_used)
-#define smartlist_get(sl, idx) ((sl)->list[idx])
-#define smartlist_set(sl, idx, val) ((sl)->list[idx] = (val))
-#endif
-
-/** Exchange the elements at indices <b>idx1</b> and <b>idx2</b> of the
- * smartlist <b>sl</b>. */
-static inline void smartlist_swap(smartlist_t *sl, int idx1, int idx2)
-{
-  if (idx1 != idx2) {
-    void *elt = smartlist_get(sl, idx1);
-    smartlist_set(sl, idx1, smartlist_get(sl, idx2));
-    smartlist_set(sl, idx2, elt);
-  }
-}
-
-void smartlist_del(smartlist_t *sl, int idx);
-void smartlist_del_keeporder(smartlist_t *sl, int idx);
-void smartlist_insert(smartlist_t *sl, int idx, void *val);
-void smartlist_sort(smartlist_t *sl,
-                    int (*compare)(const void **a, const void **b));
-void *smartlist_get_most_frequent_(const smartlist_t *sl,
-                    int (*compare)(const void **a, const void **b),
-                    int *count_out);
-#define smartlist_get_most_frequent(sl, compare) \
-  smartlist_get_most_frequent_((sl), (compare), NULL)
-void smartlist_uniq(smartlist_t *sl,
-                    int (*compare)(const void **a, const void **b),
-                    void (*free_fn)(void *elt));
-
-void smartlist_sort_strings(smartlist_t *sl);
-void smartlist_sort_digests(smartlist_t *sl);
-void smartlist_sort_digests256(smartlist_t *sl);
-void smartlist_sort_pointers(smartlist_t *sl);
-
-const char *smartlist_get_most_frequent_string(smartlist_t *sl);
-const char *smartlist_get_most_frequent_string_(smartlist_t *sl,
-                                                int *count_out);
-const uint8_t *smartlist_get_most_frequent_digest256(smartlist_t *sl);
-
-void smartlist_uniq_strings(smartlist_t *sl);
-void smartlist_uniq_digests(smartlist_t *sl);
-void smartlist_uniq_digests256(smartlist_t *sl);
-void *smartlist_bsearch(smartlist_t *sl, const void *key,
-                        int (*compare)(const void *key, const void **member));
-int smartlist_bsearch_idx(const smartlist_t *sl, const void *key,
-                          int (*compare)(const void *key, const void **member),
-                          int *found_out);
-
-void smartlist_pqueue_add(smartlist_t *sl,
-                          int (*compare)(const void *a, const void *b),
-                          int idx_field_offset,
-                          void *item);
-void *smartlist_pqueue_pop(smartlist_t *sl,
-                           int (*compare)(const void *a, const void *b),
-                           int idx_field_offset);
-void smartlist_pqueue_remove(smartlist_t *sl,
-                             int (*compare)(const void *a, const void *b),
-                             int idx_field_offset,
-                             void *item);
-void smartlist_pqueue_assert_ok(smartlist_t *sl,
-                                int (*compare)(const void *a, const void *b),
-                                int idx_field_offset);
-
-#define SPLIT_SKIP_SPACE   0x01
-#define SPLIT_IGNORE_BLANK 0x02
-#define SPLIT_STRIP_SPACE  0x04
-int smartlist_split_string(smartlist_t *sl, const char *str, const char *sep,
-                           int flags, int max);
-char *smartlist_join_strings(smartlist_t *sl, const char *join, int terminate,
-                             size_t *len_out) ATTR_MALLOC;
-char *smartlist_join_strings2(smartlist_t *sl, const char *join,
-                              size_t join_len, int terminate, size_t *len_out)
-  ATTR_MALLOC;
-
-/** Iterate over the items in a smartlist <b>sl</b>, in order.  For each item,
- * assign it to a new local variable of type <b>type</b> named <b>var</b>, and
- * execute the statements inside the loop body.  Inside the loop, the loop
- * index can be accessed as <b>var</b>_sl_idx and the length of the list can
- * be accessed as <b>var</b>_sl_len.
- *
- * NOTE: Do not change the length of the list while the loop is in progress,
- * unless you adjust the _sl_len variable correspondingly.  See second example
- * below.
- *
- * Example use:
- * <pre>
- *   smartlist_t *list = smartlist_split("A:B:C", ":", 0, 0);
- *   SMARTLIST_FOREACH_BEGIN(list, char *, cp) {
- *     printf("%d: %s\n", cp_sl_idx, cp);
- *     tor_free(cp);
- *   } SMARTLIST_FOREACH_END(cp);
- *   smartlist_free(list);
- * </pre>
- *
- * Example use (advanced):
- * <pre>
- *   SMARTLIST_FOREACH_BEGIN(list, char *, cp) {
- *     if (!strcmp(cp, "junk")) {
- *       tor_free(cp);
- *       SMARTLIST_DEL_CURRENT(list, cp);
- *     }
- *   } SMARTLIST_FOREACH_END(cp);
- * </pre>
- */
-/* Note: these macros use token pasting, and reach into smartlist internals.
- * This can make them a little daunting. Here's the approximate unpacking of
- * the above examples, for entertainment value:
- *
- * <pre>
- * smartlist_t *list = smartlist_split("A:B:C", ":", 0, 0);
- * {
- *   int cp_sl_idx, cp_sl_len = smartlist_len(list);
- *   char *cp;
- *   for (cp_sl_idx = 0; cp_sl_idx < cp_sl_len; ++cp_sl_idx) {
- *     cp = smartlist_get(list, cp_sl_idx);
- *     printf("%d: %s\n", cp_sl_idx, cp);
- *     tor_free(cp);
- *   }
- * }
- * smartlist_free(list);
- * </pre>
- *
- * <pre>
- * {
- *   int cp_sl_idx, cp_sl_len = smartlist_len(list);
- *   char *cp;
- *   for (cp_sl_idx = 0; cp_sl_idx < cp_sl_len; ++cp_sl_idx) {
- *     cp = smartlist_get(list, cp_sl_idx);
- *     if (!strcmp(cp, "junk")) {
- *       tor_free(cp);
- *       smartlist_del(list, cp_sl_idx);
- *       --cp_sl_idx;
- *       --cp_sl_len;
- *     }
- *   }
- * }
- * </pre>
- */
-#define SMARTLIST_FOREACH_BEGIN(sl, type, var)  \
-  STMT_BEGIN                                                    \
-    int var ## _sl_idx, var ## _sl_len=(sl)->num_used;          \
-    type var;                                                   \
-    for (var ## _sl_idx = 0; var ## _sl_idx < var ## _sl_len;   \
-         ++var ## _sl_idx) {                                    \
-      var = (sl)->list[var ## _sl_idx];
-
-#define SMARTLIST_FOREACH_END(var)              \
-    var = NULL;                                 \
-  } STMT_END
-
-/**
- * An alias for SMARTLIST_FOREACH_BEGIN and SMARTLIST_FOREACH_END, using
- * <b>cmd</b> as the loop body.  This wrapper is here for convenience with
- * very short loops.
- *
- * By convention, we do not use this for loops which nest, or for loops over
- * 10 lines or so.  Use SMARTLIST_FOREACH_{BEGIN,END} for those.
- */
-#define SMARTLIST_FOREACH(sl, type, var, cmd)                   \
-  SMARTLIST_FOREACH_BEGIN(sl,type,var) {                        \
-    cmd;                                                        \
-  } SMARTLIST_FOREACH_END(var)
-
-/** Helper: While in a SMARTLIST_FOREACH loop over the list <b>sl</b> indexed
- * with the variable <b>var</b>, remove the current element in a way that
- * won't confuse the loop. */
-#define SMARTLIST_DEL_CURRENT(sl, var)          \
-  STMT_BEGIN                                    \
-    smartlist_del(sl, var ## _sl_idx);          \
-    --var ## _sl_idx;                           \
-    --var ## _sl_len;                           \
-  STMT_END
-
-/** Helper: While in a SMARTLIST_FOREACH loop over the list <b>sl</b> indexed
- * with the variable <b>var</b>, remove the current element in a way that
- * won't confuse the loop. */
-#define SMARTLIST_DEL_CURRENT_KEEPORDER(sl, var)          \
-  STMT_BEGIN                                              \
-     smartlist_del_keeporder(sl, var ## _sl_idx);         \
-     --var ## _sl_idx;                                    \
-     --var ## _sl_len;                                    \
-  STMT_END
-
-/** Helper: While in a SMARTLIST_FOREACH loop over the list <b>sl</b> indexed
- * with the variable <b>var</b>, replace the current element with <b>val</b>.
- * Does not deallocate the current value of <b>var</b>.
- */
-#define SMARTLIST_REPLACE_CURRENT(sl, var, val) \
-  STMT_BEGIN                                    \
-    smartlist_set(sl, var ## _sl_idx, val);     \
-  STMT_END
-
-/* Helper: Given two lists of items, possibly of different types, such that
- * both lists are sorted on some common field (as determined by a comparison
- * expression <b>cmpexpr</b>), and such that one list (<b>sl1</b>) has no
- * duplicates on the common field, loop through the lists in lockstep, and
- * execute <b>unmatched_var2</b> on items in var2 that do not appear in
- * var1.
- *
- * WARNING: It isn't safe to add remove elements from either list while the
- * loop is in progress.
- *
- * Example use:
- *  SMARTLIST_FOREACH_JOIN(routerstatus_list, routerstatus_t *, rs,
- *                     routerinfo_list, routerinfo_t *, ri,
- *                    tor_memcmp(rs->identity_digest, ri->identity_digest, 20),
- *                     log_info(LD_GENERAL,"No match for %s", ri->nickname)) {
- *    log_info(LD_GENERAL, "%s matches routerstatus %p", ri->nickname, rs);
- * } SMARTLIST_FOREACH_JOIN_END(rs, ri);
- **/
-/* The example above unpacks (approximately) to:
- *  int rs_sl_idx = 0, rs_sl_len = smartlist_len(routerstatus_list);
- *  int ri_sl_idx, ri_sl_len = smartlist_len(routerinfo_list);
- *  int rs_ri_cmp;
- *  routerstatus_t *rs;
- *  routerinfo_t *ri;
- *  for (; ri_sl_idx < ri_sl_len; ++ri_sl_idx) {
- *    ri = smartlist_get(routerinfo_list, ri_sl_idx);
- *    while (rs_sl_idx < rs_sl_len) {
- *      rs = smartlist_get(routerstatus_list, rs_sl_idx);
- *      rs_ri_cmp = tor_memcmp(rs->identity_digest, ri->identity_digest, 20);
- *      if (rs_ri_cmp > 0) {
- *        break;
- *      } else if (rs_ri_cmp == 0) {
- *        goto matched_ri;
- *      } else {
- *        ++rs_sl_idx;
- *      }
- *    }
- *    log_info(LD_GENERAL,"No match for %s", ri->nickname);
- *    continue;
- *   matched_ri: {
- *    log_info(LD_GENERAL,"%s matches with routerstatus %p",ri->nickname,rs);
- *    }
- *  }
- */
-#define SMARTLIST_FOREACH_JOIN(sl1, type1, var1, sl2, type2, var2,      \
-                                cmpexpr, unmatched_var2)                \
-  STMT_BEGIN                                                            \
-  int var1 ## _sl_idx = 0, var1 ## _sl_len=(sl1)->num_used;             \
-  int var2 ## _sl_idx = 0, var2 ## _sl_len=(sl2)->num_used;             \
-  int var1 ## _ ## var2 ## _cmp;                                        \
-  type1 var1;                                                           \
-  type2 var2;                                                           \
-  for (; var2##_sl_idx < var2##_sl_len; ++var2##_sl_idx) {              \
-    var2 = (sl2)->list[var2##_sl_idx];                                  \
-    while (var1##_sl_idx < var1##_sl_len) {                             \
-      var1 = (sl1)->list[var1##_sl_idx];                                \
-      var1##_##var2##_cmp = (cmpexpr);                                  \
-      if (var1##_##var2##_cmp > 0) {                                    \
-        break;                                                          \
-      } else if (var1##_##var2##_cmp == 0) {                            \
-        goto matched_##var2;                                            \
-      } else {                                                          \
-        ++var1##_sl_idx;                                                \
-      }                                                                 \
-    }                                                                   \
-    /* Ran out of v1, or no match for var2. */                          \
-    unmatched_var2;                                                     \
-    continue;                                                           \
-    matched_##var2: ;                                                   \
-
-#define SMARTLIST_FOREACH_JOIN_END(var1, var2)  \
-  }                                             \
-  STMT_END
-
-#define DECLARE_MAP_FNS(maptype, keytype, prefix)                       \
-  typedef struct maptype maptype;                                       \
-  typedef struct prefix##entry_t *prefix##iter_t;                       \
-  MOCK_DECL(maptype*, prefix##new, (void));                             \
-  void* prefix##set(maptype *map, keytype key, void *val);              \
-  void* prefix##get(const maptype *map, keytype key);                   \
-  void* prefix##remove(maptype *map, keytype key);                      \
-  MOCK_DECL(void, prefix##free, (maptype *map, void (*free_val)(void*))); \
-  int prefix##isempty(const maptype *map);                              \
-  int prefix##size(const maptype *map);                                 \
-  prefix##iter_t *prefix##iter_init(maptype *map);                      \
-  prefix##iter_t *prefix##iter_next(maptype *map, prefix##iter_t *iter); \
-  prefix##iter_t *prefix##iter_next_rmv(maptype *map, prefix##iter_t *iter); \
-  void prefix##iter_get(prefix##iter_t *iter, keytype *keyp, void **valp); \
-  int prefix##iter_done(prefix##iter_t *iter);                          \
-  void prefix##assert_ok(const maptype *map)
-
-/* Map from const char * to void *. Implemented with a hash table. */
-DECLARE_MAP_FNS(strmap_t, const char *, strmap_);
-/* Map from const char[DIGEST_LEN] to void *. Implemented with a hash table. */
-DECLARE_MAP_FNS(digestmap_t, const char *, digestmap_);
-/* Map from const uint8_t[DIGEST256_LEN] to void *. Implemented with a hash
- * table. */
-DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_);
-
-#undef DECLARE_MAP_FNS
-
-/** Iterates over the key-value pairs in a map <b>map</b> in order.
- * <b>prefix</b> is as for DECLARE_MAP_FNS (i.e., strmap_ or digestmap_).
- * The map's keys and values are of type keytype and valtype respectively;
- * each iteration assigns them to keyvar and valvar.
- *
- * Example use:
- *   MAP_FOREACH(digestmap_, m, const char *, k, routerinfo_t *, r) {
- *     // use k and r
- *   } MAP_FOREACH_END.
- */
-/* Unpacks to, approximately:
- * {
- *   digestmap_iter_t *k_iter;
- *   for (k_iter = digestmap_iter_init(m); !digestmap_iter_done(k_iter);
- *        k_iter = digestmap_iter_next(m, k_iter)) {
- *     const char *k;
- *     void *r_voidp;
- *     routerinfo_t *r;
- *     digestmap_iter_get(k_iter, &k, &r_voidp);
- *     r = r_voidp;
- *     // use k and r
- *   }
- * }
- */
-#define MAP_FOREACH(prefix, map, keytype, keyvar, valtype, valvar)      \
-  STMT_BEGIN                                                            \
-    prefix##iter_t *keyvar##_iter;                                      \
-    for (keyvar##_iter = prefix##iter_init(map);                        \
-         !prefix##iter_done(keyvar##_iter);                             \
-         keyvar##_iter = prefix##iter_next(map, keyvar##_iter)) {       \
-      keytype keyvar;                                                   \
-      void *valvar##_voidp;                                             \
-      valtype valvar;                                                   \
-      prefix##iter_get(keyvar##_iter, &keyvar, &valvar##_voidp);        \
-      valvar = valvar##_voidp;
-
-/** As MAP_FOREACH, except allows members to be removed from the map
- * during the iteration via MAP_DEL_CURRENT.  Example use:
- *
- * Example use:
- *   MAP_FOREACH(digestmap_, m, const char *, k, routerinfo_t *, r) {
- *      if (is_very_old(r))
- *       MAP_DEL_CURRENT(k);
- *   } MAP_FOREACH_END.
- **/
-/* Unpacks to, approximately:
- * {
- *   digestmap_iter_t *k_iter;
- *   int k_del=0;
- *   for (k_iter = digestmap_iter_init(m); !digestmap_iter_done(k_iter);
- *        k_iter = k_del ? digestmap_iter_next(m, k_iter)
- *                       : digestmap_iter_next_rmv(m, k_iter)) {
- *     const char *k;
- *     void *r_voidp;
- *     routerinfo_t *r;
- *     k_del=0;
- *     digestmap_iter_get(k_iter, &k, &r_voidp);
- *     r = r_voidp;
- *     if (is_very_old(r)) {
- *       k_del = 1;
- *     }
- *   }
- * }
- */
-#define MAP_FOREACH_MODIFY(prefix, map, keytype, keyvar, valtype, valvar) \
-  STMT_BEGIN                                                            \
-    prefix##iter_t *keyvar##_iter;                                      \
-    int keyvar##_del=0;                                                 \
-    for (keyvar##_iter = prefix##iter_init(map);                        \
-         !prefix##iter_done(keyvar##_iter);                             \
-         keyvar##_iter = keyvar##_del ?                                 \
-           prefix##iter_next_rmv(map, keyvar##_iter) :                  \
-           prefix##iter_next(map, keyvar##_iter)) {                     \
-      keytype keyvar;                                                   \
-      void *valvar##_voidp;                                             \
-      valtype valvar;                                                   \
-      keyvar##_del=0;                                                   \
-      prefix##iter_get(keyvar##_iter, &keyvar, &valvar##_voidp);        \
-      valvar = valvar##_voidp;
-
-/** Used with MAP_FOREACH_MODIFY to remove the currently-iterated-upon
- * member of the map.  */
-#define MAP_DEL_CURRENT(keyvar)                   \
-  STMT_BEGIN                                      \
-    keyvar##_del = 1;                             \
-  STMT_END
-
-/** Used to end a MAP_FOREACH() block. */
-#define MAP_FOREACH_END } STMT_END ;
-
-/** As MAP_FOREACH, but does not require declaration of prefix or keytype.
- * Example use:
- *   DIGESTMAP_FOREACH(m, k, routerinfo_t *, r) {
- *     // use k and r
- *   } DIGESTMAP_FOREACH_END.
- */
-#define DIGESTMAP_FOREACH(map, keyvar, valtype, valvar)                 \
-  MAP_FOREACH(digestmap_, map, const char *, keyvar, valtype, valvar)
-
-/** As MAP_FOREACH_MODIFY, but does not require declaration of prefix or
- * keytype.
- * Example use:
- *   DIGESTMAP_FOREACH_MODIFY(m, k, routerinfo_t *, r) {
- *      if (is_very_old(r))
- *       MAP_DEL_CURRENT(k);
- *   } DIGESTMAP_FOREACH_END.
- */
-#define DIGESTMAP_FOREACH_MODIFY(map, keyvar, valtype, valvar)          \
-  MAP_FOREACH_MODIFY(digestmap_, map, const char *, keyvar, valtype, valvar)
-/** Used to end a DIGESTMAP_FOREACH() block. */
-#define DIGESTMAP_FOREACH_END MAP_FOREACH_END
-
-#define DIGEST256MAP_FOREACH(map, keyvar, valtype, valvar)               \
-  MAP_FOREACH(digest256map_, map, const uint8_t *, keyvar, valtype, valvar)
-#define DIGEST256MAP_FOREACH_MODIFY(map, keyvar, valtype, valvar)       \
-  MAP_FOREACH_MODIFY(digest256map_, map, const uint8_t *,               \
-                     keyvar, valtype, valvar)
-#define DIGEST256MAP_FOREACH_END MAP_FOREACH_END
-
-#define STRMAP_FOREACH(map, keyvar, valtype, valvar)                 \
-  MAP_FOREACH(strmap_, map, const char *, keyvar, valtype, valvar)
-#define STRMAP_FOREACH_MODIFY(map, keyvar, valtype, valvar)          \
-  MAP_FOREACH_MODIFY(strmap_, map, const char *, keyvar, valtype, valvar)
-#define STRMAP_FOREACH_END MAP_FOREACH_END
-
-void* strmap_set_lc(strmap_t *map, const char *key, void *val);
-void* strmap_get_lc(const strmap_t *map, const char *key);
-void* strmap_remove_lc(strmap_t *map, const char *key);
-
-#define DECLARE_TYPED_DIGESTMAP_FNS(prefix, maptype, valtype)           \
-  typedef struct maptype maptype;                                       \
-  typedef struct prefix##iter_t *prefix##iter_t;                        \
-  ATTR_UNUSED static inline maptype*                                    \
-  prefix##new(void)                                                     \
-  {                                                                     \
-    return (maptype*)digestmap_new();                                   \
-  }                                                                     \
-  ATTR_UNUSED static inline digestmap_t*                                \
-  prefix##to_digestmap(maptype *map)                                    \
-  {                                                                     \
-    return (digestmap_t*)map;                                           \
-  }                                                                     \
-  ATTR_UNUSED static inline valtype*                                    \
-  prefix##get(maptype *map, const char *key)     \
-  {                                                                     \
-    return (valtype*)digestmap_get((digestmap_t*)map, key);             \
-  }                                                                     \
-  ATTR_UNUSED static inline valtype*                                    \
-  prefix##set(maptype *map, const char *key, valtype *val)              \
-  {                                                                     \
-    return (valtype*)digestmap_set((digestmap_t*)map, key, val);        \
-  }                                                                     \
-  ATTR_UNUSED static inline valtype*                                    \
-  prefix##remove(maptype *map, const char *key)                         \
-  {                                                                     \
-    return (valtype*)digestmap_remove((digestmap_t*)map, key);          \
-  }                                                                     \
-  ATTR_UNUSED static inline void                                        \
-  prefix##f##ree(maptype *map, void (*free_val)(void*))                 \
-  {                                                                     \
-    digestmap_free((digestmap_t*)map, free_val);                        \
-  }                                                                     \
-  ATTR_UNUSED static inline int                                         \
-  prefix##isempty(maptype *map)                                         \
-  {                                                                     \
-    return digestmap_isempty((digestmap_t*)map);                        \
-  }                                                                     \
-  ATTR_UNUSED static inline int                                         \
-  prefix##size(maptype *map)                                            \
-  {                                                                     \
-    return digestmap_size((digestmap_t*)map);                           \
-  }                                                                     \
-  ATTR_UNUSED static inline                                             \
-  prefix##iter_t *prefix##iter_init(maptype *map)                       \
-  {                                                                     \
-    return (prefix##iter_t*) digestmap_iter_init((digestmap_t*)map);    \
-  }                                                                     \
-  ATTR_UNUSED static inline                                             \
-  prefix##iter_t *prefix##iter_next(maptype *map, prefix##iter_t *iter) \
-  {                                                                     \
-    return (prefix##iter_t*) digestmap_iter_next(                       \
-                       (digestmap_t*)map, (digestmap_iter_t*)iter);     \
-  }                                                                     \
-  ATTR_UNUSED static inline prefix##iter_t*                             \
-  prefix##iter_next_rmv(maptype *map, prefix##iter_t *iter)             \
-  {                                                                     \
-    return (prefix##iter_t*) digestmap_iter_next_rmv(                   \
-                       (digestmap_t*)map, (digestmap_iter_t*)iter);     \
-  }                                                                     \
-  ATTR_UNUSED static inline void                                        \
-  prefix##iter_get(prefix##iter_t *iter,                                \
-                   const char **keyp,                                   \
-                   valtype **valp)                                      \
-  {                                                                     \
-    void *v;                                                            \
-    digestmap_iter_get((digestmap_iter_t*) iter, keyp, &v);             \
-    *valp = v;                                                          \
-  }                                                                     \
-  ATTR_UNUSED static inline int                                         \
-  prefix##iter_done(prefix##iter_t *iter)                               \
-  {                                                                     \
-    return digestmap_iter_done((digestmap_iter_t*)iter);                \
-  }
-
-#if SIZEOF_INT == 4
-#define BITARRAY_SHIFT 5
-#elif SIZEOF_INT == 8
-#define BITARRAY_SHIFT 6
-#else
-#error "int is neither 4 nor 8 bytes. I can't deal with that."
-#endif
-#define BITARRAY_MASK ((1u<<BITARRAY_SHIFT)-1)
-
-/** A random-access array of one-bit-wide elements. */
-typedef unsigned int bitarray_t;
-/** Create a new bit array that can hold <b>n_bits</b> bits. */
-static inline bitarray_t *
-bitarray_init_zero(unsigned int n_bits)
-{
-  /* round up to the next int. */
-  size_t sz = (n_bits+BITARRAY_MASK) >> BITARRAY_SHIFT;
-  return tor_calloc(sz, sizeof(unsigned int));
-}
-/** Expand <b>ba</b> from holding <b>n_bits_old</b> to <b>n_bits_new</b>,
- * clearing all new bits.  Returns a possibly changed pointer to the
- * bitarray. */
-static inline bitarray_t *
-bitarray_expand(bitarray_t *ba,
-                unsigned int n_bits_old, unsigned int n_bits_new)
-{
-  size_t sz_old = (n_bits_old+BITARRAY_MASK) >> BITARRAY_SHIFT;
-  size_t sz_new = (n_bits_new+BITARRAY_MASK) >> BITARRAY_SHIFT;
-  char *ptr;
-  if (sz_new <= sz_old)
-    return ba;
-  ptr = tor_reallocarray(ba, sz_new, sizeof(unsigned int));
-  /* This memset does nothing to the older excess bytes.  But they were
-   * already set to 0 by bitarry_init_zero. */
-  memset(ptr+sz_old*sizeof(unsigned int), 0,
-         (sz_new-sz_old)*sizeof(unsigned int));
-  return (bitarray_t*) ptr;
-}
-/** Free the bit array <b>ba</b>. */
-static inline void
-bitarray_free(bitarray_t *ba)
-{
-  tor_free(ba);
-}
-/** Set the <b>bit</b>th bit in <b>b</b> to 1. */
-static inline void
-bitarray_set(bitarray_t *b, int bit)
-{
-  b[bit >> BITARRAY_SHIFT] |= (1u << (bit & BITARRAY_MASK));
-}
-/** Set the <b>bit</b>th bit in <b>b</b> to 0. */
-static inline void
-bitarray_clear(bitarray_t *b, int bit)
-{
-  b[bit >> BITARRAY_SHIFT] &= ~ (1u << (bit & BITARRAY_MASK));
-}
-/** Return true iff <b>bit</b>th bit in <b>b</b> is nonzero.  NOTE: does
- * not necessarily return 1 on true. */
-static inline unsigned int
-bitarray_is_set(bitarray_t *b, int bit)
-{
-  return b[bit >> BITARRAY_SHIFT] & (1u << (bit & BITARRAY_MASK));
-}
-
-/** A set of digests, implemented as a Bloom filter. */
-typedef struct {
-  int mask; /**< One less than the number of bits in <b>ba</b>; always one less
-             * than a power of two. */
-  bitarray_t *ba; /**< A bit array to implement the Bloom filter. */
-} digestset_t;
-
-#define BIT(n) ((n) & set->mask)
-/** Add the digest <b>digest</b> to <b>set</b>. */
-static inline void
-digestset_add(digestset_t *set, const char *digest)
-{
-  const uint64_t x = siphash24g(digest, 20);
-  const uint32_t d1 = (uint32_t) x;
-  const uint32_t d2 = (uint32_t)( (x>>16) + x);
-  const uint32_t d3 = (uint32_t)( (x>>32) + x);
-  const uint32_t d4 = (uint32_t)( (x>>48) + x);
-  bitarray_set(set->ba, BIT(d1));
-  bitarray_set(set->ba, BIT(d2));
-  bitarray_set(set->ba, BIT(d3));
-  bitarray_set(set->ba, BIT(d4));
-}
-
-/** If <b>digest</b> is in <b>set</b>, return nonzero.  Otherwise,
- * <em>probably</em> return zero. */
-static inline int
-digestset_contains(const digestset_t *set, const char *digest)
-{
-  const uint64_t x = siphash24g(digest, 20);
-  const uint32_t d1 = (uint32_t) x;
-  const uint32_t d2 = (uint32_t)( (x>>16) + x);
-  const uint32_t d3 = (uint32_t)( (x>>32) + x);
-  const uint32_t d4 = (uint32_t)( (x>>48) + x);
-  return bitarray_is_set(set->ba, BIT(d1)) &&
-         bitarray_is_set(set->ba, BIT(d2)) &&
-         bitarray_is_set(set->ba, BIT(d3)) &&
-         bitarray_is_set(set->ba, BIT(d4));
-}
-#undef BIT
-
-digestset_t *digestset_new(int max_elements);
-void digestset_free(digestset_t* set);
-
-/* These functions, given an <b>array</b> of <b>n_elements</b>, return the
- * <b>nth</b> lowest element. <b>nth</b>=0 gives the lowest element;
- * <b>n_elements</b>-1 gives the highest; and (<b>n_elements</b>-1) / 2 gives
- * the median.  As a side effect, the elements of <b>array</b> are sorted. */
-int find_nth_int(int *array, int n_elements, int nth);
-time_t find_nth_time(time_t *array, int n_elements, int nth);
-double find_nth_double(double *array, int n_elements, int nth);
-int32_t find_nth_int32(int32_t *array, int n_elements, int nth);
-uint32_t find_nth_uint32(uint32_t *array, int n_elements, int nth);
-long find_nth_long(long *array, int n_elements, int nth);
-static inline int
-median_int(int *array, int n_elements)
-{
-  return find_nth_int(array, n_elements, (n_elements-1)/2);
-}
-static inline time_t
-median_time(time_t *array, int n_elements)
-{
-  return find_nth_time(array, n_elements, (n_elements-1)/2);
-}
-static inline double
-median_double(double *array, int n_elements)
-{
-  return find_nth_double(array, n_elements, (n_elements-1)/2);
-}
-static inline uint32_t
-median_uint32(uint32_t *array, int n_elements)
-{
-  return find_nth_uint32(array, n_elements, (n_elements-1)/2);
-}
-static inline int32_t
-median_int32(int32_t *array, int n_elements)
-{
-  return find_nth_int32(array, n_elements, (n_elements-1)/2);
-}
-
-static inline uint32_t
-third_quartile_uint32(uint32_t *array, int n_elements)
-{
-  return find_nth_uint32(array, n_elements, (n_elements*3)/4);
-}
-
-#endif
-
diff --git a/src/common/crypto.c b/src/common/crypto.c
deleted file mode 100644
index f8495bb107..0000000000
--- a/src/common/crypto.c
+++ /dev/null
@@ -1,3432 +0,0 @@
-/* Copyright (c) 2001, Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file crypto.c
- * \brief Wrapper functions to present a consistent interface to
- * public-key and symmetric cryptography operations from OpenSSL and
- * other places.
- **/
-
-#include "orconfig.h"
-
-#ifdef _WIN32
-#include <winsock2.h>
-#include <windows.h>
-#include <wincrypt.h>
-/* Windows defines this; so does OpenSSL 0.9.8h and later. We don't actually
- * use either definition. */
-#undef OCSP_RESPONSE
-#endif
-
-#define CRYPTO_PRIVATE
-#include "crypto.h"
-#include "compat_openssl.h"
-#include "crypto_curve25519.h"
-#include "crypto_ed25519.h"
-#include "crypto_format.h"
-
-DISABLE_GCC_WARNING(redundant-decls)
-
-#include <openssl/err.h>
-#include <openssl/rsa.h>
-#include <openssl/pem.h>
-#include <openssl/evp.h>
-#include <openssl/engine.h>
-#include <openssl/rand.h>
-#include <openssl/bn.h>
-#include <openssl/dh.h>
-#include <openssl/conf.h>
-#include <openssl/hmac.h>
-
-ENABLE_GCC_WARNING(redundant-decls)
-
-#if __GNUC__ && GCC_VERSION >= 402
-#if GCC_VERSION >= 406
-#pragma GCC diagnostic pop
-#else
-#pragma GCC diagnostic warning "-Wredundant-decls"
-#endif
-#endif
-
-#ifdef HAVE_CTYPE_H
-#include <ctype.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif
-#ifdef HAVE_SYS_FCNTL_H
-#include <sys/fcntl.h>
-#endif
-#ifdef HAVE_SYS_SYSCALL_H
-#include <sys/syscall.h>
-#endif
-#ifdef HAVE_SYS_RANDOM_H
-#include <sys/random.h>
-#endif
-
-#include "torlog.h"
-#include "torint.h"
-#include "aes.h"
-#include "util.h"
-#include "container.h"
-#include "compat.h"
-#include "sandbox.h"
-#include "util_format.h"
-
-#include "keccak-tiny/keccak-tiny.h"
-
-#ifdef ANDROID
-/* Android's OpenSSL seems to have removed all of its Engine support. */
-#define DISABLE_ENGINES
-#endif
-
-#if OPENSSL_VERSION_NUMBER >= OPENSSL_VER(1,1,0,0,5) && \
-  !defined(LIBRESSL_VERSION_NUMBER)
-/* OpenSSL as of 1.1.0pre4 has an "new" thread API, which doesn't require
- * seting up various callbacks.
- *
- * OpenSSL 1.1.0pre4 has a messed up `ERR_remove_thread_state()` prototype,
- * while the previous one was restored in pre5, and the function made a no-op
- * (along with a deprecated annotation, which produces a compiler warning).
- *
- * While it is possible to support all three versions of the thread API,
- * a version that existed only for one snapshot pre-release is kind of
- * pointless, so let's not.
- */
-#define NEW_THREAD_API
-#endif
-
-/** Longest recognized */
-#define MAX_DNS_LABEL_SIZE 63
-
-/** Largest strong entropy request */
-#define MAX_STRONGEST_RAND_SIZE 256
-
-#ifndef NEW_THREAD_API
-/** A number of preallocated mutexes for use by OpenSSL. */
-static tor_mutex_t **openssl_mutexes_ = NULL;
-/** How many mutexes have we allocated for use by OpenSSL? */
-static int n_openssl_mutexes_ = 0;
-#endif
-
-/** A public key, or a public/private key-pair. */
-struct crypto_pk_t
-{
-  int refs; /**< reference count, so we don't have to copy keys */
-  RSA *key; /**< The key itself */
-};
-
-/** A structure to hold the first half (x, g^x) of a Diffie-Hellman handshake
- * while we're waiting for the second.*/
-struct crypto_dh_t {
-  DH *dh; /**< The openssl DH object */
-};
-
-static int setup_openssl_threading(void);
-static int tor_check_dh_key(int severity, const BIGNUM *bn);
-
-/** Return the number of bytes added by padding method <b>padding</b>.
- */
-static inline int
-crypto_get_rsa_padding_overhead(int padding)
-{
-  switch (padding)
-    {
-    case RSA_PKCS1_OAEP_PADDING: return PKCS1_OAEP_PADDING_OVERHEAD;
-    default: tor_assert(0); return -1; // LCOV_EXCL_LINE
-    }
-}
-
-/** Given a padding method <b>padding</b>, return the correct OpenSSL constant.
- */
-static inline int
-crypto_get_rsa_padding(int padding)
-{
-  switch (padding)
-    {
-    case PK_PKCS1_OAEP_PADDING: return RSA_PKCS1_OAEP_PADDING;
-    default: tor_assert(0); return -1; // LCOV_EXCL_LINE
-    }
-}
-
-/** Boolean: has OpenSSL's crypto been initialized? */
-static int crypto_early_initialized_ = 0;
-
-/** Boolean: has OpenSSL's crypto been initialized? */
-static int crypto_global_initialized_ = 0;
-
-/** Log all pending crypto errors at level <b>severity</b>.  Use
- * <b>doing</b> to describe our current activities.
- */
-static void
-crypto_log_errors(int severity, const char *doing)
-{
-  unsigned long err;
-  const char *msg, *lib, *func;
-  while ((err = ERR_get_error()) != 0) {
-    msg = (const char*)ERR_reason_error_string(err);
-    lib = (const char*)ERR_lib_error_string(err);
-    func = (const char*)ERR_func_error_string(err);
-    if (!msg) msg = "(null)";
-    if (!lib) lib = "(null)";
-    if (!func) func = "(null)";
-    if (BUG(!doing)) doing = "(null)";
-    tor_log(severity, LD_CRYPTO, "crypto error while %s: %s (in %s:%s)",
-              doing, msg, lib, func);
-  }
-}
-
-#ifndef DISABLE_ENGINES
-/** Log any OpenSSL engines we're using at NOTICE. */
-static void
-log_engine(const char *fn, ENGINE *e)
-{
-  if (e) {
-    const char *name, *id;
-    name = ENGINE_get_name(e);
-    id = ENGINE_get_id(e);
-    log_notice(LD_CRYPTO, "Default OpenSSL engine for %s is %s [%s]",
-               fn, name?name:"?", id?id:"?");
-  } else {
-    log_info(LD_CRYPTO, "Using default implementation for %s", fn);
-  }
-}
-#endif
-
-#ifndef DISABLE_ENGINES
-/** Try to load an engine in a shared library via fully qualified path.
- */
-static ENGINE *
-try_load_engine(const char *path, const char *engine)
-{
-  ENGINE *e = ENGINE_by_id("dynamic");
-  if (e) {
-    if (!ENGINE_ctrl_cmd_string(e, "ID", engine, 0) ||
-        !ENGINE_ctrl_cmd_string(e, "DIR_LOAD", "2", 0) ||
-        !ENGINE_ctrl_cmd_string(e, "DIR_ADD", path, 0) ||
-        !ENGINE_ctrl_cmd_string(e, "LOAD", NULL, 0)) {
-      ENGINE_free(e);
-      e = NULL;
-    }
-  }
-  return e;
-}
-#endif
-
-/* Returns a trimmed and human-readable version of an openssl version string
-* <b>raw_version</b>. They are usually in the form of 'OpenSSL 1.0.0b 10
-* May 2012' and this will parse them into a form similar to '1.0.0b' */
-static char *
-parse_openssl_version_str(const char *raw_version)
-{
-  const char *end_of_version = NULL;
-  /* The output should be something like "OpenSSL 1.0.0b 10 May 2012. Let's
-     trim that down. */
-  if (!strcmpstart(raw_version, "OpenSSL ")) {
-    raw_version += strlen("OpenSSL ");
-    end_of_version = strchr(raw_version, ' ');
-  }
-
-  if (end_of_version)
-    return tor_strndup(raw_version,
-                      end_of_version-raw_version);
-  else
-    return tor_strdup(raw_version);
-}
-
-static char *crypto_openssl_version_str = NULL;
-/* Return a human-readable version of the run-time openssl version number. */
-const char *
-crypto_openssl_get_version_str(void)
-{
-  if (crypto_openssl_version_str == NULL) {
-    const char *raw_version = OpenSSL_version(OPENSSL_VERSION);
-    crypto_openssl_version_str = parse_openssl_version_str(raw_version);
-  }
-  return crypto_openssl_version_str;
-}
-
-static char *crypto_openssl_header_version_str = NULL;
-/* Return a human-readable version of the compile-time openssl version
-* number. */
-const char *
-crypto_openssl_get_header_version_str(void)
-{
-  if (crypto_openssl_header_version_str == NULL) {
-    crypto_openssl_header_version_str =
-                        parse_openssl_version_str(OPENSSL_VERSION_TEXT);
-  }
-  return crypto_openssl_header_version_str;
-}
-
-/** Make sure that openssl is using its default PRNG. Return 1 if we had to
- * adjust it; 0 otherwise. */
-STATIC int
-crypto_force_rand_ssleay(void)
-{
-  RAND_METHOD *default_method;
-  default_method = RAND_OpenSSL();
-  if (RAND_get_rand_method() != default_method) {
-    log_notice(LD_CRYPTO, "It appears that one of our engines has provided "
-               "a replacement the OpenSSL RNG. Resetting it to the default "
-               "implementation.");
-    RAND_set_rand_method(default_method);
-    return 1;
-  }
-  return 0;
-}
-
-/** Set up the siphash key if we haven't already done so. */
-int
-crypto_init_siphash_key(void)
-{
-  static int have_seeded_siphash = 0;
-  struct sipkey key;
-  if (have_seeded_siphash)
-    return 0;
-
-  crypto_rand((char*) &key, sizeof(key));
-  siphash_set_global_key(&key);
-  have_seeded_siphash = 1;
-  return 0;
-}
-
-/** Initialize the crypto library.  Return 0 on success, -1 on failure.
- */
-int
-crypto_early_init(void)
-{
-  if (!crypto_early_initialized_) {
-
-    crypto_early_initialized_ = 1;
-
-    ERR_load_crypto_strings();
-    OpenSSL_add_all_algorithms();
-
-    setup_openssl_threading();
-
-    unsigned long version_num = OpenSSL_version_num();
-    const char *version_str = OpenSSL_version(OPENSSL_VERSION);
-    if (version_num == OPENSSL_VERSION_NUMBER &&
-        !strcmp(version_str, OPENSSL_VERSION_TEXT)) {
-      log_info(LD_CRYPTO, "OpenSSL version matches version from headers "
-                 "(%lx: %s).", version_num, version_str);
-    } else {
-      log_warn(LD_CRYPTO, "OpenSSL version from headers does not match the "
-               "version we're running with. If you get weird crashes, that "
-               "might be why. (Compiled with %lx: %s; running with %lx: %s).",
-               (unsigned long)OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_TEXT,
-               version_num, version_str);
-    }
-
-    crypto_force_rand_ssleay();
-
-    if (crypto_seed_rng() < 0)
-      return -1;
-    if (crypto_init_siphash_key() < 0)
-      return -1;
-
-    curve25519_init();
-    ed25519_init();
-  }
-  return 0;
-}
-
-/** Initialize the crypto library.  Return 0 on success, -1 on failure.
- */
-int
-crypto_global_init(int useAccel, const char *accelName, const char *accelDir)
-{
-  if (!crypto_global_initialized_) {
-    if (crypto_early_init() < 0)
-      return -1;
-
-    crypto_global_initialized_ = 1;
-
-    if (useAccel > 0) {
-#ifdef DISABLE_ENGINES
-      (void)accelName;
-      (void)accelDir;
-      log_warn(LD_CRYPTO, "No OpenSSL hardware acceleration support enabled.");
-#else
-      ENGINE *e = NULL;
-
-      log_info(LD_CRYPTO, "Initializing OpenSSL engine support.");
-      ENGINE_load_builtin_engines();
-      ENGINE_register_all_complete();
-
-      if (accelName) {
-        if (accelDir) {
-          log_info(LD_CRYPTO, "Trying to load dynamic OpenSSL engine \"%s\""
-                   " via path \"%s\".", accelName, accelDir);
-          e = try_load_engine(accelName, accelDir);
-        } else {
-          log_info(LD_CRYPTO, "Initializing dynamic OpenSSL engine \"%s\""
-                   " acceleration support.", accelName);
-          e = ENGINE_by_id(accelName);
-        }
-        if (!e) {
-          log_warn(LD_CRYPTO, "Unable to load dynamic OpenSSL engine \"%s\".",
-                   accelName);
-        } else {
-          log_info(LD_CRYPTO, "Loaded dynamic OpenSSL engine \"%s\".",
-                   accelName);
-        }
-      }
-      if (e) {
-        log_info(LD_CRYPTO, "Loaded OpenSSL hardware acceleration engine,"
-                 " setting default ciphers.");
-        ENGINE_set_default(e, ENGINE_METHOD_ALL);
-      }
-      /* Log, if available, the intersection of the set of algorithms
-         used by Tor and the set of algorithms available in the engine */
-      log_engine("RSA", ENGINE_get_default_RSA());
-      log_engine("DH", ENGINE_get_default_DH());
-#ifdef OPENSSL_1_1_API
-      log_engine("EC", ENGINE_get_default_EC());
-#else
-      log_engine("ECDH", ENGINE_get_default_ECDH());
-      log_engine("ECDSA", ENGINE_get_default_ECDSA());
-#endif
-      log_engine("RAND", ENGINE_get_default_RAND());
-      log_engine("RAND (which we will not use)", ENGINE_get_default_RAND());
-      log_engine("SHA1", ENGINE_get_digest_engine(NID_sha1));
-      log_engine("3DES-CBC", ENGINE_get_cipher_engine(NID_des_ede3_cbc));
-      log_engine("AES-128-ECB", ENGINE_get_cipher_engine(NID_aes_128_ecb));
-      log_engine("AES-128-CBC", ENGINE_get_cipher_engine(NID_aes_128_cbc));
-#ifdef NID_aes_128_ctr
-      log_engine("AES-128-CTR", ENGINE_get_cipher_engine(NID_aes_128_ctr));
-#endif
-#ifdef NID_aes_128_gcm
-      log_engine("AES-128-GCM", ENGINE_get_cipher_engine(NID_aes_128_gcm));
-#endif
-      log_engine("AES-256-CBC", ENGINE_get_cipher_engine(NID_aes_256_cbc));
-#ifdef NID_aes_256_gcm
-      log_engine("AES-256-GCM", ENGINE_get_cipher_engine(NID_aes_256_gcm));
-#endif
-
-#endif
-    } else {
-      log_info(LD_CRYPTO, "NOT using OpenSSL engine support.");
-    }
-
-    if (crypto_force_rand_ssleay()) {
-      if (crypto_seed_rng() < 0)
-        return -1;
-    }
-
-    evaluate_evp_for_aes(-1);
-    evaluate_ctr_for_aes();
-  }
-  return 0;
-}
-
-/** Free crypto resources held by this thread. */
-void
-crypto_thread_cleanup(void)
-{
-#ifndef NEW_THREAD_API
-  ERR_remove_thread_state(NULL);
-#endif
-}
-
-/** used internally: quicly validate a crypto_pk_t object as a private key.
- * Return 1 iff the public key is valid, 0 if obviously invalid.
- */
-static int
-crypto_pk_private_ok(const crypto_pk_t *k)
-{
-#ifdef OPENSSL_1_1_API
-  if (!k || !k->key)
-    return 0;
-
-  const BIGNUM *p, *q;
-  RSA_get0_factors(k->key, &p, &q);
-  return p != NULL; /* XXX/yawning: Should we check q? */
-#else
-  return k && k->key && k->key->p;
-#endif
-}
-
-/** used by tortls.c: wrap an RSA* in a crypto_pk_t. */
-crypto_pk_t *
-crypto_new_pk_from_rsa_(RSA *rsa)
-{
-  crypto_pk_t *env;
-  tor_assert(rsa);
-  env = tor_malloc(sizeof(crypto_pk_t));
-  env->refs = 1;
-  env->key = rsa;
-  return env;
-}
-
-/** Helper, used by tor-checkkey.c and tor-gencert.c.  Return the RSA from a
- * crypto_pk_t. */
-RSA *
-crypto_pk_get_rsa_(crypto_pk_t *env)
-{
-  return env->key;
-}
-
-/** used by tortls.c: get an equivalent EVP_PKEY* for a crypto_pk_t.  Iff
- * private is set, include the private-key portion of the key. Return a valid
- * pointer on success, and NULL on failure. */
-MOCK_IMPL(EVP_PKEY *,
-          crypto_pk_get_evp_pkey_,(crypto_pk_t *env, int private))
-{
-  RSA *key = NULL;
-  EVP_PKEY *pkey = NULL;
-  tor_assert(env->key);
-  if (private) {
-    if (!(key = RSAPrivateKey_dup(env->key)))
-      goto error;
-  } else {
-    if (!(key = RSAPublicKey_dup(env->key)))
-      goto error;
-  }
-  if (!(pkey = EVP_PKEY_new()))
-    goto error;
-  if (!(EVP_PKEY_assign_RSA(pkey, key)))
-    goto error;
-  return pkey;
- error:
-  if (pkey)
-    EVP_PKEY_free(pkey);
-  if (key)
-    RSA_free(key);
-  return NULL;
-}
-
-/** Used by tortls.c: Get the DH* from a crypto_dh_t.
- */
-DH *
-crypto_dh_get_dh_(crypto_dh_t *dh)
-{
-  return dh->dh;
-}
-
-/** Allocate and return storage for a public key.  The key itself will not yet
- * be set.
- */
-MOCK_IMPL(crypto_pk_t *,
-          crypto_pk_new,(void))
-{
-  RSA *rsa;
-
-  rsa = RSA_new();
-  tor_assert(rsa);
-  return crypto_new_pk_from_rsa_(rsa);
-}
-
-/** Release a reference to an asymmetric key; when all the references
- * are released, free the key.
- */
-void
-crypto_pk_free(crypto_pk_t *env)
-{
-  if (!env)
-    return;
-
-  if (--env->refs > 0)
-    return;
-  tor_assert(env->refs == 0);
-
-  if (env->key)
-    RSA_free(env->key);
-
-  tor_free(env);
-}
-
-/** Allocate and return a new symmetric cipher using the provided key and iv.
- * The key is <b>bits</b> bits long; the IV is CIPHER_IV_LEN bytes.  Both
- * must be provided. Key length must be 128, 192, or 256 */
-crypto_cipher_t *
-crypto_cipher_new_with_iv_and_bits(const uint8_t *key,
-                                   const uint8_t *iv,
-                                   int bits)
-{
-  tor_assert(key);
-  tor_assert(iv);
-
-  return aes_new_cipher((const uint8_t*)key, (const uint8_t*)iv, bits);
-}
-
-/** Allocate and return a new symmetric cipher using the provided key and iv.
- * The key is CIPHER_KEY_LEN bytes; the IV is CIPHER_IV_LEN bytes.  Both
- * must be provided.
- */
-crypto_cipher_t *
-crypto_cipher_new_with_iv(const char *key, const char *iv)
-{
-  return crypto_cipher_new_with_iv_and_bits((uint8_t*)key, (uint8_t*)iv,
-                                            128);
-}
-
-/** Return a new crypto_cipher_t with the provided <b>key</b> and an IV of all
- * zero bytes and key length <b>bits</b>.  Key length must be 128, 192, or
- * 256. */
-crypto_cipher_t *
-crypto_cipher_new_with_bits(const char *key, int bits)
-{
-  char zeroiv[CIPHER_IV_LEN];
-  memset(zeroiv, 0, sizeof(zeroiv));
-  return crypto_cipher_new_with_iv_and_bits((uint8_t*)key, (uint8_t*)zeroiv,
-                                            bits);
-}
-
-/** Return a new crypto_cipher_t with the provided <b>key</b> (of
- * CIPHER_KEY_LEN bytes) and an IV of all zero bytes.  */
-crypto_cipher_t *
-crypto_cipher_new(const char *key)
-{
-  return crypto_cipher_new_with_bits(key, 128);
-}
-
-/** Free a symmetric cipher.
- */
-void
-crypto_cipher_free(crypto_cipher_t *env)
-{
-  if (!env)
-    return;
-
-  aes_cipher_free(env);
-}
-
-/* public key crypto */
-
-/** Generate a <b>bits</b>-bit new public/private keypair in <b>env</b>.
- * Return 0 on success, -1 on failure.
- */
-MOCK_IMPL(int,
-          crypto_pk_generate_key_with_bits,(crypto_pk_t *env, int bits))
-{
-  tor_assert(env);
-
-  if (env->key) {
-    RSA_free(env->key);
-    env->key = NULL;
-  }
-
-  {
-    BIGNUM *e = BN_new();
-    RSA *r = NULL;
-    if (!e)
-      goto done;
-    if (! BN_set_word(e, 65537))
-      goto done;
-    r = RSA_new();
-    if (!r)
-      goto done;
-    if (RSA_generate_key_ex(r, bits, e, NULL) == -1)
-      goto done;
-
-    env->key = r;
-    r = NULL;
-  done:
-    if (e)
-      BN_clear_free(e);
-    if (r)
-      RSA_free(r);
-  }
-
-  if (!env->key) {
-    crypto_log_errors(LOG_WARN, "generating RSA key");
-    return -1;
-  }
-
-  return 0;
-}
-
-/** A PEM callback that always reports a failure to get a password */
-static int
-pem_no_password_cb(char *buf, int size, int rwflag, void *u)
-{
-  (void)buf;
-  (void)size;
-  (void)rwflag;
-  (void)u;
-  /* The openssl documentation says that a callback "must" return 0 if an
-   * error occurred.  But during the 1.1.1 series (commit c82c3462267afdbbaa5
-   * they changed the interpretation so that 0 indicates an empty password and
-   * -1 indicates an error. We want to reject any encrypted PEM buffers, so we
-   * return -1.  This will work on older OpenSSL versions and LibreSSL too. */
-  return -1;
-}
-
-/** Read a PEM-encoded private key from the <b>len</b>-byte string <b>s</b>
- * into <b>env</b>.  Return 0 on success, -1 on failure.  If len is -1,
- * the string is nul-terminated.
- */
-int
-crypto_pk_read_private_key_from_string(crypto_pk_t *env,
-                                       const char *s, ssize_t len)
-{
-  BIO *b;
-
-  tor_assert(env);
-  tor_assert(s);
-  tor_assert(len < INT_MAX && len < SSIZE_T_CEILING);
-
-  /* Create a read-only memory BIO, backed by the string 's' */
-  b = BIO_new_mem_buf((char*)s, (int)len);
-  if (!b)
-    return -1;
-
-  if (env->key)
-    RSA_free(env->key);
-
-  env->key = PEM_read_bio_RSAPrivateKey(b,NULL,pem_no_password_cb,NULL);
-
-  BIO_free(b);
-
-  if (!env->key) {
-    crypto_log_errors(LOG_WARN, "Error parsing private key");
-    return -1;
-  }
-  return 0;
-}
-
-/** Read a PEM-encoded private key from the file named by
- * <b>keyfile</b> into <b>env</b>.  Return 0 on success, -1 on failure.
- */
-int
-crypto_pk_read_private_key_from_filename(crypto_pk_t *env,
-                                         const char *keyfile)
-{
-  char *contents;
-  int r;
-
-  /* Read the file into a string. */
-  contents = read_file_to_str(keyfile, 0, NULL);
-  if (!contents) {
-    log_warn(LD_CRYPTO, "Error reading private key from \"%s\"", keyfile);
-    return -1;
-  }
-
-  /* Try to parse it. */
-  r = crypto_pk_read_private_key_from_string(env, contents, -1);
-  memwipe(contents, 0, strlen(contents));
-  tor_free(contents);
-  if (r)
-    return -1; /* read_private_key_from_string already warned, so we don't.*/
-
-  /* Make sure it's valid. */
-  if (crypto_pk_check_key(env) <= 0)
-    return -1;
-
-  return 0;
-}
-
-/** Helper function to implement crypto_pk_write_*_key_to_string. Return 0 on
- * success, -1 on failure. */
-static int
-crypto_pk_write_key_to_string_impl(crypto_pk_t *env, char **dest,
-                                   size_t *len, int is_public)
-{
-  BUF_MEM *buf;
-  BIO *b;
-  int r;
-
-  tor_assert(env);
-  tor_assert(env->key);
-  tor_assert(dest);
-
-  b = BIO_new(BIO_s_mem()); /* Create a memory BIO */
-  if (!b)
-    return -1;
-
-  /* Now you can treat b as if it were a file.  Just use the
-   * PEM_*_bio_* functions instead of the non-bio variants.
-   */
-  if (is_public)
-    r = PEM_write_bio_RSAPublicKey(b, env->key);
-  else
-    r = PEM_write_bio_RSAPrivateKey(b, env->key, NULL,NULL,0,NULL,NULL);
-
-  if (!r) {
-    crypto_log_errors(LOG_WARN, "writing RSA key to string");
-    BIO_free(b);
-    return -1;
-  }
-
-  BIO_get_mem_ptr(b, &buf);
-
-  *dest = tor_malloc(buf->length+1);
-  memcpy(*dest, buf->data, buf->length);
-  (*dest)[buf->length] = 0; /* nul terminate it */
-  *len = buf->length;
-
-  BIO_free(b);
-
-  return 0;
-}
-
-/** PEM-encode the public key portion of <b>env</b> and write it to a
- * newly allocated string.  On success, set *<b>dest</b> to the new
- * string, *<b>len</b> to the string's length, and return 0.  On
- * failure, return -1.
- */
-int
-crypto_pk_write_public_key_to_string(crypto_pk_t *env, char **dest,
-                                     size_t *len)
-{
-  return crypto_pk_write_key_to_string_impl(env, dest, len, 1);
-}
-
-/** PEM-encode the private key portion of <b>env</b> and write it to a
- * newly allocated string.  On success, set *<b>dest</b> to the new
- * string, *<b>len</b> to the string's length, and return 0.  On
- * failure, return -1.
- */
-int
-crypto_pk_write_private_key_to_string(crypto_pk_t *env, char **dest,
-                                     size_t *len)
-{
-  return crypto_pk_write_key_to_string_impl(env, dest, len, 0);
-}
-
-/** Read a PEM-encoded public key from the first <b>len</b> characters of
- * <b>src</b>, and store the result in <b>env</b>.  Return 0 on success, -1 on
- * failure.
- */
-int
-crypto_pk_read_public_key_from_string(crypto_pk_t *env, const char *src,
-                                      size_t len)
-{
-  BIO *b;
-
-  tor_assert(env);
-  tor_assert(src);
-  tor_assert(len<INT_MAX);
-
-  b = BIO_new(BIO_s_mem()); /* Create a memory BIO */
-  if (!b)
-    return -1;
-
-  BIO_write(b, src, (int)len);
-
-  if (env->key)
-    RSA_free(env->key);
-  env->key = PEM_read_bio_RSAPublicKey(b, NULL, pem_no_password_cb, NULL);
-  BIO_free(b);
-  if (!env->key) {
-    crypto_log_errors(LOG_WARN, "reading public key from string");
-    return -1;
-  }
-
-  return 0;
-}
-
-/** Write the private key from <b>env</b> into the file named by <b>fname</b>,
- * PEM-encoded.  Return 0 on success, -1 on failure.
- */
-int
-crypto_pk_write_private_key_to_filename(crypto_pk_t *env,
-                                        const char *fname)
-{
-  BIO *bio;
-  char *cp;
-  long len;
-  char *s;
-  int r;
-
-  tor_assert(crypto_pk_private_ok(env));
-
-  if (!(bio = BIO_new(BIO_s_mem())))
-    return -1;
-  if (PEM_write_bio_RSAPrivateKey(bio, env->key, NULL,NULL,0,NULL,NULL)
-      == 0) {
-    crypto_log_errors(LOG_WARN, "writing private key");
-    BIO_free(bio);
-    return -1;
-  }
-  len = BIO_get_mem_data(bio, &cp);
-  tor_assert(len >= 0);
-  s = tor_malloc(len+1);
-  memcpy(s, cp, len);
-  s[len]='\0';
-  r = write_str_to_file(fname, s, 0);
-  BIO_free(bio);
-  memwipe(s, 0, strlen(s));
-  tor_free(s);
-  return r;
-}
-
-/** Return true iff <b>env</b> has a valid key.
- */
-int
-crypto_pk_check_key(crypto_pk_t *env)
-{
-  int r;
-  tor_assert(env);
-
-  r = RSA_check_key(env->key);
-  if (r <= 0)
-    crypto_log_errors(LOG_WARN,"checking RSA key");
-  return r;
-}
-
-/** Return true iff <b>key</b> contains the private-key portion of the RSA
- * key. */
-int
-crypto_pk_key_is_private(const crypto_pk_t *key)
-{
-  tor_assert(key);
-  return crypto_pk_private_ok(key);
-}
-
-/** Return true iff <b>env</b> contains a public key whose public exponent
- * equals 65537.
- */
-int
-crypto_pk_public_exponent_ok(crypto_pk_t *env)
-{
-  tor_assert(env);
-  tor_assert(env->key);
-
-  const BIGNUM *e;
-
-#ifdef OPENSSL_1_1_API
-  const BIGNUM *n, *d;
-  RSA_get0_key(env->key, &n, &e, &d);
-#else
-  e = env->key->e;
-#endif
-  return BN_is_word(e, 65537);
-}
-
-/** Compare the public-key components of a and b.  Return less than 0
- * if a\<b, 0 if a==b, and greater than 0 if a\>b.  A NULL key is
- * considered to be less than all non-NULL keys, and equal to itself.
- *
- * Note that this may leak information about the keys through timing.
- */
-int
-crypto_pk_cmp_keys(const crypto_pk_t *a, const crypto_pk_t *b)
-{
-  int result;
-  char a_is_non_null = (a != NULL) && (a->key != NULL);
-  char b_is_non_null = (b != NULL) && (b->key != NULL);
-  char an_argument_is_null = !a_is_non_null | !b_is_non_null;
-
-  result = tor_memcmp(&a_is_non_null, &b_is_non_null, sizeof(a_is_non_null));
-  if (an_argument_is_null)
-    return result;
-
-  const BIGNUM *a_n, *a_e;
-  const BIGNUM *b_n, *b_e;
-
-#ifdef OPENSSL_1_1_API
-  const BIGNUM *a_d, *b_d;
-  RSA_get0_key(a->key, &a_n, &a_e, &a_d);
-  RSA_get0_key(b->key, &b_n, &b_e, &b_d);
-#else
-  a_n = a->key->n;
-  a_e = a->key->e;
-  b_n = b->key->n;
-  b_e = b->key->e;
-#endif
-
-  tor_assert(a_n != NULL && a_e != NULL);
-  tor_assert(b_n != NULL && b_e != NULL);
-
-  result = BN_cmp(a_n, b_n);
-  if (result)
-    return result;
-  return BN_cmp(a_e, b_e);
-}
-
-/** Compare the public-key components of a and b.  Return non-zero iff
- * a==b.  A NULL key is considered to be distinct from all non-NULL
- * keys, and equal to itself.
- *
- *  Note that this may leak information about the keys through timing.
- */
-int
-crypto_pk_eq_keys(const crypto_pk_t *a, const crypto_pk_t *b)
-{
-  return (crypto_pk_cmp_keys(a, b) == 0);
-}
-
-/** Return the size of the public key modulus in <b>env</b>, in bytes. */
-size_t
-crypto_pk_keysize(const crypto_pk_t *env)
-{
-  tor_assert(env);
-  tor_assert(env->key);
-
-  return (size_t) RSA_size((RSA*)env->key);
-}
-
-/** Return the size of the public key modulus of <b>env</b>, in bits. */
-int
-crypto_pk_num_bits(crypto_pk_t *env)
-{
-  tor_assert(env);
-  tor_assert(env->key);
-
-#ifdef OPENSSL_1_1_API
-  /* It's so stupid that there's no other way to check that n is valid
-   * before calling RSA_bits().
-   */
-  const BIGNUM *n, *e, *d;
-  RSA_get0_key(env->key, &n, &e, &d);
-  tor_assert(n != NULL);
-
-  return RSA_bits(env->key);
-#else
-  tor_assert(env->key->n);
-  return BN_num_bits(env->key->n);
-#endif
-}
-
-/** Increase the reference count of <b>env</b>, and return it.
- */
-crypto_pk_t *
-crypto_pk_dup_key(crypto_pk_t *env)
-{
-  tor_assert(env);
-  tor_assert(env->key);
-
-  env->refs++;
-  return env;
-}
-
-#ifdef TOR_UNIT_TESTS
-/** For testing: replace dest with src.  (Dest must have a refcount
- * of 1) */
-void
-crypto_pk_assign_(crypto_pk_t *dest, const crypto_pk_t *src)
-{
-  tor_assert(dest);
-  tor_assert(dest->refs == 1);
-  tor_assert(src);
-  RSA_free(dest->key);
-  dest->key = RSAPrivateKey_dup(src->key);
-}
-#endif
-
-/** Make a real honest-to-goodness copy of <b>env</b>, and return it.
- * Returns NULL on failure. */
-crypto_pk_t *
-crypto_pk_copy_full(crypto_pk_t *env)
-{
-  RSA *new_key;
-  int privatekey = 0;
-  tor_assert(env);
-  tor_assert(env->key);
-
-  if (crypto_pk_private_ok(env)) {
-    new_key = RSAPrivateKey_dup(env->key);
-    privatekey = 1;
-  } else {
-    new_key = RSAPublicKey_dup(env->key);
-  }
-  if (!new_key) {
-    /* LCOV_EXCL_START
-     *
-     * We can't cause RSA*Key_dup() to fail, so we can't really test this.
-     */
-    log_err(LD_CRYPTO, "Unable to duplicate a %s key: openssl failed.",
-            privatekey?"private":"public");
-    crypto_log_errors(LOG_ERR,
-                      privatekey ? "Duplicating a private key" :
-                      "Duplicating a public key");
-    tor_fragile_assert();
-    return NULL;
-    /* LCOV_EXCL_STOP */
-  }
-
-  return crypto_new_pk_from_rsa_(new_key);
-}
-
-/** Encrypt <b>fromlen</b> bytes from <b>from</b> with the public key
- * in <b>env</b>, using the padding method <b>padding</b>.  On success,
- * write the result to <b>to</b>, and return the number of bytes
- * written.  On failure, return -1.
- *
- * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
- * at least the length of the modulus of <b>env</b>.
- */
-int
-crypto_pk_public_encrypt(crypto_pk_t *env, char *to, size_t tolen,
-                         const char *from, size_t fromlen, int padding)
-{
-  int r;
-  tor_assert(env);
-  tor_assert(from);
-  tor_assert(to);
-  tor_assert(fromlen<INT_MAX);
-  tor_assert(tolen >= crypto_pk_keysize(env));
-
-  r = RSA_public_encrypt((int)fromlen,
-                         (unsigned char*)from, (unsigned char*)to,
-                         env->key, crypto_get_rsa_padding(padding));
-  if (r<0) {
-    crypto_log_errors(LOG_WARN, "performing RSA encryption");
-    return -1;
-  }
-  return r;
-}
-
-/** Decrypt <b>fromlen</b> bytes from <b>from</b> with the private key
- * in <b>env</b>, using the padding method <b>padding</b>.  On success,
- * write the result to <b>to</b>, and return the number of bytes
- * written.  On failure, return -1.
- *
- * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
- * at least the length of the modulus of <b>env</b>.
- */
-int
-crypto_pk_private_decrypt(crypto_pk_t *env, char *to,
-                          size_t tolen,
-                          const char *from, size_t fromlen,
-                          int padding, int warnOnFailure)
-{
-  int r;
-  tor_assert(env);
-  tor_assert(from);
-  tor_assert(to);
-  tor_assert(env->key);
-  tor_assert(fromlen<INT_MAX);
-  tor_assert(tolen >= crypto_pk_keysize(env));
-  if (!crypto_pk_key_is_private(env))
-    /* Not a private key */
-    return -1;
-
-  r = RSA_private_decrypt((int)fromlen,
-                          (unsigned char*)from, (unsigned char*)to,
-                          env->key, crypto_get_rsa_padding(padding));
-
-  if (r<0) {
-    crypto_log_errors(warnOnFailure?LOG_WARN:LOG_DEBUG,
-                      "performing RSA decryption");
-    return -1;
-  }
-  return r;
-}
-
-/** Check the signature in <b>from</b> (<b>fromlen</b> bytes long) with the
- * public key in <b>env</b>, using PKCS1 padding.  On success, write the
- * signed data to <b>to</b>, and return the number of bytes written.
- * On failure, return -1.
- *
- * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
- * at least the length of the modulus of <b>env</b>.
- */
-int
-crypto_pk_public_checksig(const crypto_pk_t *env, char *to,
-                          size_t tolen,
-                          const char *from, size_t fromlen)
-{
-  int r;
-  tor_assert(env);
-  tor_assert(from);
-  tor_assert(to);
-  tor_assert(fromlen < INT_MAX);
-  tor_assert(tolen >= crypto_pk_keysize(env));
-  r = RSA_public_decrypt((int)fromlen,
-                         (unsigned char*)from, (unsigned char*)to,
-                         env->key, RSA_PKCS1_PADDING);
-
-  if (r<0) {
-    crypto_log_errors(LOG_INFO, "checking RSA signature");
-    return -1;
-  }
-  return r;
-}
-
-/** Check a siglen-byte long signature at <b>sig</b> against
- * <b>datalen</b> bytes of data at <b>data</b>, using the public key
- * in <b>env</b>. Return 0 if <b>sig</b> is a correct signature for
- * SHA1(data).  Else return -1.
- */
-int
-crypto_pk_public_checksig_digest(crypto_pk_t *env, const char *data,
-                               size_t datalen, const char *sig, size_t siglen)
-{
-  char digest[DIGEST_LEN];
-  char *buf;
-  size_t buflen;
-  int r;
-
-  tor_assert(env);
-  tor_assert(data);
-  tor_assert(sig);
-  tor_assert(datalen < SIZE_T_CEILING);
-  tor_assert(siglen < SIZE_T_CEILING);
-
-  if (crypto_digest(digest,data,datalen)<0) {
-    log_warn(LD_BUG, "couldn't compute digest");
-    return -1;
-  }
-  buflen = crypto_pk_keysize(env);
-  buf = tor_malloc(buflen);
-  r = crypto_pk_public_checksig(env,buf,buflen,sig,siglen);
-  if (r != DIGEST_LEN) {
-    log_warn(LD_CRYPTO, "Invalid signature");
-    tor_free(buf);
-    return -1;
-  }
-  if (tor_memneq(buf, digest, DIGEST_LEN)) {
-    log_warn(LD_CRYPTO, "Signature mismatched with digest.");
-    tor_free(buf);
-    return -1;
-  }
-  tor_free(buf);
-
-  return 0;
-}
-
-/** Sign <b>fromlen</b> bytes of data from <b>from</b> with the private key in
- * <b>env</b>, using PKCS1 padding.  On success, write the signature to
- * <b>to</b>, and return the number of bytes written.  On failure, return
- * -1.
- *
- * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
- * at least the length of the modulus of <b>env</b>.
- */
-int
-crypto_pk_private_sign(const crypto_pk_t *env, char *to, size_t tolen,
-                       const char *from, size_t fromlen)
-{
-  int r;
-  tor_assert(env);
-  tor_assert(from);
-  tor_assert(to);
-  tor_assert(fromlen < INT_MAX);
-  tor_assert(tolen >= crypto_pk_keysize(env));
-  if (!crypto_pk_key_is_private(env))
-    /* Not a private key */
-    return -1;
-
-  r = RSA_private_encrypt((int)fromlen,
-                          (unsigned char*)from, (unsigned char*)to,
-                          (RSA*)env->key, RSA_PKCS1_PADDING);
-  if (r<0) {
-    crypto_log_errors(LOG_WARN, "generating RSA signature");
-    return -1;
-  }
-  return r;
-}
-
-/** Compute a SHA1 digest of <b>fromlen</b> bytes of data stored at
- * <b>from</b>; sign the data with the private key in <b>env</b>, and
- * store it in <b>to</b>.  Return the number of bytes written on
- * success, and -1 on failure.
- *
- * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
- * at least the length of the modulus of <b>env</b>.
- */
-int
-crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen,
-                              const char *from, size_t fromlen)
-{
-  int r;
-  char digest[DIGEST_LEN];
-  if (crypto_digest(digest,from,fromlen)<0)
-    return -1;
-  r = crypto_pk_private_sign(env,to,tolen,digest,DIGEST_LEN);
-  memwipe(digest, 0, sizeof(digest));
-  return r;
-}
-
-/** Perform a hybrid (public/secret) encryption on <b>fromlen</b>
- * bytes of data from <b>from</b>, with padding type 'padding',
- * storing the results on <b>to</b>.
- *
- * Returns the number of bytes written on success, -1 on failure.
- *
- * The encrypted data consists of:
- *   - The source data, padded and encrypted with the public key, if the
- *     padded source data is no longer than the public key, and <b>force</b>
- *     is false, OR
- *   - The beginning of the source data prefixed with a 16-byte symmetric key,
- *     padded and encrypted with the public key; followed by the rest of
- *     the source data encrypted in AES-CTR mode with the symmetric key.
- */
-int
-crypto_pk_public_hybrid_encrypt(crypto_pk_t *env,
-                                char *to, size_t tolen,
-                                const char *from,
-                                size_t fromlen,
-                                int padding, int force)
-{
-  int overhead, outlen, r;
-  size_t pkeylen, symlen;
-  crypto_cipher_t *cipher = NULL;
-  char *buf = NULL;
-
-  tor_assert(env);
-  tor_assert(from);
-  tor_assert(to);
-  tor_assert(fromlen < SIZE_T_CEILING);
-
-  overhead = crypto_get_rsa_padding_overhead(crypto_get_rsa_padding(padding));
-  pkeylen = crypto_pk_keysize(env);
-
-  if (!force && fromlen+overhead <= pkeylen) {
-    /* It all fits in a single encrypt. */
-    return crypto_pk_public_encrypt(env,to,
-                                    tolen,
-                                    from,fromlen,padding);
-  }
-  tor_assert(tolen >= fromlen + overhead + CIPHER_KEY_LEN);
-  tor_assert(tolen >= pkeylen);
-
-  char key[CIPHER_KEY_LEN];
-  crypto_rand(key, sizeof(key)); /* generate a new key. */
-  cipher = crypto_cipher_new(key);
-
-  buf = tor_malloc(pkeylen+1);
-  memcpy(buf, key, CIPHER_KEY_LEN);
-  memcpy(buf+CIPHER_KEY_LEN, from, pkeylen-overhead-CIPHER_KEY_LEN);
-
-  /* Length of symmetrically encrypted data. */
-  symlen = fromlen-(pkeylen-overhead-CIPHER_KEY_LEN);
-
-  outlen = crypto_pk_public_encrypt(env,to,tolen,buf,pkeylen-overhead,padding);
-  if (outlen!=(int)pkeylen) {
-    goto err;
-  }
-  r = crypto_cipher_encrypt(cipher, to+outlen,
-                            from+pkeylen-overhead-CIPHER_KEY_LEN, symlen);
-
-  if (r<0) goto err;
-  memwipe(buf, 0, pkeylen);
-  memwipe(key, 0, sizeof(key));
-  tor_free(buf);
-  crypto_cipher_free(cipher);
-  tor_assert(outlen+symlen < INT_MAX);
-  return (int)(outlen + symlen);
- err:
-
-  memwipe(buf, 0, pkeylen);
-  memwipe(key, 0, sizeof(key));
-  tor_free(buf);
-  crypto_cipher_free(cipher);
-  return -1;
-}
-
-/** Invert crypto_pk_public_hybrid_encrypt. Returns the number of bytes
- * written on success, -1 on failure. */
-int
-crypto_pk_private_hybrid_decrypt(crypto_pk_t *env,
-                                 char *to,
-                                 size_t tolen,
-                                 const char *from,
-                                 size_t fromlen,
-                                 int padding, int warnOnFailure)
-{
-  int outlen, r;
-  size_t pkeylen;
-  crypto_cipher_t *cipher = NULL;
-  char *buf = NULL;
-
-  tor_assert(fromlen < SIZE_T_CEILING);
-  pkeylen = crypto_pk_keysize(env);
-
-  if (fromlen <= pkeylen) {
-    return crypto_pk_private_decrypt(env,to,tolen,from,fromlen,padding,
-                                     warnOnFailure);
-  }
-
-  buf = tor_malloc(pkeylen);
-  outlen = crypto_pk_private_decrypt(env,buf,pkeylen,from,pkeylen,padding,
-                                     warnOnFailure);
-  if (outlen<0) {
-    log_fn(warnOnFailure?LOG_WARN:LOG_DEBUG, LD_CRYPTO,
-           "Error decrypting public-key data");
-    goto err;
-  }
-  if (outlen < CIPHER_KEY_LEN) {
-    log_fn(warnOnFailure?LOG_WARN:LOG_INFO, LD_CRYPTO,
-           "No room for a symmetric key");
-    goto err;
-  }
-  cipher = crypto_cipher_new(buf);
-  if (!cipher) {
-    goto err;
-  }
-  memcpy(to,buf+CIPHER_KEY_LEN,outlen-CIPHER_KEY_LEN);
-  outlen -= CIPHER_KEY_LEN;
-  tor_assert(tolen - outlen >= fromlen - pkeylen);
-  r = crypto_cipher_decrypt(cipher, to+outlen, from+pkeylen, fromlen-pkeylen);
-  if (r<0)
-    goto err;
-  memwipe(buf,0,pkeylen);
-  tor_free(buf);
-  crypto_cipher_free(cipher);
-  tor_assert(outlen + fromlen < INT_MAX);
-  return (int)(outlen + (fromlen-pkeylen));
- err:
-  memwipe(buf,0,pkeylen);
-  tor_free(buf);
-  crypto_cipher_free(cipher);
-  return -1;
-}
-
-/** ASN.1-encode the public portion of <b>pk</b> into <b>dest</b>.
- * Return -1 on error, or the number of characters used on success.
- */
-int
-crypto_pk_asn1_encode(crypto_pk_t *pk, char *dest, size_t dest_len)
-{
-  int len;
-  unsigned char *buf = NULL;
-
-  len = i2d_RSAPublicKey(pk->key, &buf);
-  if (len < 0 || buf == NULL)
-    return -1;
-
-  if ((size_t)len > dest_len || dest_len > SIZE_T_CEILING) {
-    OPENSSL_free(buf);
-    return -1;
-  }
-  /* We don't encode directly into 'dest', because that would be illegal
-   * type-punning.  (C99 is smarter than me, C99 is smarter than me...)
-   */
-  memcpy(dest,buf,len);
-  OPENSSL_free(buf);
-  return len;
-}
-
-/** Decode an ASN.1-encoded public key from <b>str</b>; return the result on
- * success and NULL on failure.
- */
-crypto_pk_t *
-crypto_pk_asn1_decode(const char *str, size_t len)
-{
-  RSA *rsa;
-  unsigned char *buf;
-  const unsigned char *cp;
-  cp = buf = tor_malloc(len);
-  memcpy(buf,str,len);
-  rsa = d2i_RSAPublicKey(NULL, &cp, len);
-  tor_free(buf);
-  if (!rsa) {
-    crypto_log_errors(LOG_WARN,"decoding public key");
-    return NULL;
-  }
-  return crypto_new_pk_from_rsa_(rsa);
-}
-
-/** Given a private or public key <b>pk</b>, put a SHA1 hash of the
- * public key into <b>digest_out</b> (must have DIGEST_LEN bytes of space).
- * Return 0 on success, -1 on failure.
- */
-int
-crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out)
-{
-  unsigned char *buf = NULL;
-  int len;
-
-  len = i2d_RSAPublicKey((RSA*)pk->key, &buf);
-  if (len < 0 || buf == NULL)
-    return -1;
-  if (crypto_digest(digest_out, (char*)buf, len) < 0) {
-    OPENSSL_free(buf);
-    return -1;
-  }
-  OPENSSL_free(buf);
-  return 0;
-}
-
-/** Compute all digests of the DER encoding of <b>pk</b>, and store them
- * in <b>digests_out</b>.  Return 0 on success, -1 on failure. */
-int
-crypto_pk_get_common_digests(crypto_pk_t *pk, common_digests_t *digests_out)
-{
-  unsigned char *buf = NULL;
-  int len;
-
-  len = i2d_RSAPublicKey(pk->key, &buf);
-  if (len < 0 || buf == NULL)
-    return -1;
-  if (crypto_common_digests(digests_out, (char*)buf, len) < 0) {
-    OPENSSL_free(buf);
-    return -1;
-  }
-  OPENSSL_free(buf);
-  return 0;
-}
-
-/** Copy <b>in</b> to the <b>outlen</b>-byte buffer <b>out</b>, adding spaces
- * every four characters. */
-void
-crypto_add_spaces_to_fp(char *out, size_t outlen, const char *in)
-{
-  int n = 0;
-  char *end = out+outlen;
-  tor_assert(outlen < SIZE_T_CEILING);
-
-  while (*in && out<end) {
-    *out++ = *in++;
-    if (++n == 4 && *in && out<end) {
-      n = 0;
-      *out++ = ' ';
-    }
-  }
-  tor_assert(out<end);
-  *out = '\0';
-}
-
-/** Given a private or public key <b>pk</b>, put a fingerprint of the
- * public key into <b>fp_out</b> (must have at least FINGERPRINT_LEN+1 bytes of
- * space).  Return 0 on success, -1 on failure.
- *
- * Fingerprints are computed as the SHA1 digest of the ASN.1 encoding
- * of the public key, converted to hexadecimal, in upper case, with a
- * space after every four digits.
- *
- * If <b>add_space</b> is false, omit the spaces.
- */
-int
-crypto_pk_get_fingerprint(crypto_pk_t *pk, char *fp_out, int add_space)
-{
-  char digest[DIGEST_LEN];
-  char hexdigest[HEX_DIGEST_LEN+1];
-  if (crypto_pk_get_digest(pk, digest)) {
-    return -1;
-  }
-  base16_encode(hexdigest,sizeof(hexdigest),digest,DIGEST_LEN);
-  if (add_space) {
-    crypto_add_spaces_to_fp(fp_out, FINGERPRINT_LEN+1, hexdigest);
-  } else {
-    strncpy(fp_out, hexdigest, HEX_DIGEST_LEN+1);
-  }
-  return 0;
-}
-
-/** Given a private or public key <b>pk</b>, put a hashed fingerprint of
- * the public key into <b>fp_out</b> (must have at least FINGERPRINT_LEN+1
- * bytes of space).  Return 0 on success, -1 on failure.
- *
- * Hashed fingerprints are computed as the SHA1 digest of the SHA1 digest
- * of the ASN.1 encoding of the public key, converted to hexadecimal, in
- * upper case.
- */
-int
-crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out)
-{
-  char digest[DIGEST_LEN], hashed_digest[DIGEST_LEN];
-  if (crypto_pk_get_digest(pk, digest)) {
-    return -1;
-  }
-  if (crypto_digest(hashed_digest, digest, DIGEST_LEN)) {
-    return -1;
-  }
-  base16_encode(fp_out, FINGERPRINT_LEN + 1, hashed_digest, DIGEST_LEN);
-  return 0;
-}
-
-/** Given a crypto_pk_t <b>pk</b>, allocate a new buffer containing the
- * Base64 encoding of the DER representation of the private key as a NUL
- * terminated string, and return it via <b>priv_out</b>.  Return 0 on
- * sucess, -1 on failure.
- *
- * It is the caller's responsibility to sanitize and free the resulting buffer.
- */
-int
-crypto_pk_base64_encode(const crypto_pk_t *pk, char **priv_out)
-{
-  unsigned char *der = NULL;
-  int der_len;
-  int ret = -1;
-
-  *priv_out = NULL;
-
-  der_len = i2d_RSAPrivateKey(pk->key, &der);
-  if (der_len < 0 || der == NULL)
-    return ret;
-
-  size_t priv_len = base64_encode_size(der_len, 0) + 1;
-  char *priv = tor_malloc_zero(priv_len);
-  if (base64_encode(priv, priv_len, (char *)der, der_len, 0) >= 0) {
-    *priv_out = priv;
-    ret = 0;
-  } else {
-    tor_free(priv);
-  }
-
-  memwipe(der, 0, der_len);
-  OPENSSL_free(der);
-  return ret;
-}
-
-/** Given a string containing the Base64 encoded DER representation of the
- * private key <b>str</b>, decode and return the result on success, or NULL
- * on failure.
- */
-crypto_pk_t *
-crypto_pk_base64_decode(const char *str, size_t len)
-{
-  crypto_pk_t *pk = NULL;
-
-  char *der = tor_malloc_zero(len + 1);
-  int der_len = base64_decode(der, len, str, len);
-  if (der_len <= 0) {
-    log_warn(LD_CRYPTO, "Stored RSA private key seems corrupted (base64).");
-    goto out;
-  }
-
-  const unsigned char *dp = (unsigned char*)der; /* Shut the compiler up. */
-  RSA *rsa = d2i_RSAPrivateKey(NULL, &dp, der_len);
-  if (!rsa) {
-    crypto_log_errors(LOG_WARN, "decoding private key");
-    goto out;
-  }
-
-  pk = crypto_new_pk_from_rsa_(rsa);
-
-  /* Make sure it's valid. */
-  if (crypto_pk_check_key(pk) <= 0) {
-    crypto_pk_free(pk);
-    pk = NULL;
-    goto out;
-  }
-
- out:
-  memwipe(der, 0, len + 1);
-  tor_free(der);
-  return pk;
-}
-
-/* symmetric crypto */
-
-/** Encrypt <b>fromlen</b> bytes from <b>from</b> using the cipher
- * <b>env</b>; on success, store the result to <b>to</b> and return 0.
- * Does not check for failure.
- */
-int
-crypto_cipher_encrypt(crypto_cipher_t *env, char *to,
-                      const char *from, size_t fromlen)
-{
-  tor_assert(env);
-  tor_assert(env);
-  tor_assert(from);
-  tor_assert(fromlen);
-  tor_assert(to);
-  tor_assert(fromlen < SIZE_T_CEILING);
-
-  memcpy(to, from, fromlen);
-  aes_crypt_inplace(env, to, fromlen);
-  return 0;
-}
-
-/** Decrypt <b>fromlen</b> bytes from <b>from</b> using the cipher
- * <b>env</b>; on success, store the result to <b>to</b> and return 0.
- * Does not check for failure.
- */
-int
-crypto_cipher_decrypt(crypto_cipher_t *env, char *to,
-                      const char *from, size_t fromlen)
-{
-  tor_assert(env);
-  tor_assert(from);
-  tor_assert(to);
-  tor_assert(fromlen < SIZE_T_CEILING);
-
-  memcpy(to, from, fromlen);
-  aes_crypt_inplace(env, to, fromlen);
-  return 0;
-}
-
-/** Encrypt <b>len</b> bytes on <b>from</b> using the cipher in <b>env</b>;
- * on success. Does not check for failure.
- */
-void
-crypto_cipher_crypt_inplace(crypto_cipher_t *env, char *buf, size_t len)
-{
-  tor_assert(len < SIZE_T_CEILING);
-  aes_crypt_inplace(env, buf, len);
-}
-
-/** Encrypt <b>fromlen</b> bytes (at least 1) from <b>from</b> with the key in
- * <b>key</b> to the buffer in <b>to</b> of length
- * <b>tolen</b>. <b>tolen</b> must be at least <b>fromlen</b> plus
- * CIPHER_IV_LEN bytes for the initialization vector. On success, return the
- * number of bytes written, on failure, return -1.
- */
-int
-crypto_cipher_encrypt_with_iv(const char *key,
-                              char *to, size_t tolen,
-                              const char *from, size_t fromlen)
-{
-  crypto_cipher_t *cipher;
-  tor_assert(from);
-  tor_assert(to);
-  tor_assert(fromlen < INT_MAX);
-
-  if (fromlen < 1)
-    return -1;
-  if (tolen < fromlen + CIPHER_IV_LEN)
-    return -1;
-
-  char iv[CIPHER_IV_LEN];
-  crypto_rand(iv, sizeof(iv));
-  cipher = crypto_cipher_new_with_iv(key, iv);
-
-  memcpy(to, iv, CIPHER_IV_LEN);
-  crypto_cipher_encrypt(cipher, to+CIPHER_IV_LEN, from, fromlen);
-  crypto_cipher_free(cipher);
-  memwipe(iv, 0, sizeof(iv));
-  return (int)(fromlen + CIPHER_IV_LEN);
-}
-
-/** Decrypt <b>fromlen</b> bytes (at least 1+CIPHER_IV_LEN) from <b>from</b>
- * with the key in <b>key</b> to the buffer in <b>to</b> of length
- * <b>tolen</b>. <b>tolen</b> must be at least <b>fromlen</b> minus
- * CIPHER_IV_LEN bytes for the initialization vector. On success, return the
- * number of bytes written, on failure, return -1.
- */
-int
-crypto_cipher_decrypt_with_iv(const char *key,
-                              char *to, size_t tolen,
-                              const char *from, size_t fromlen)
-{
-  crypto_cipher_t *cipher;
-  tor_assert(key);
-  tor_assert(from);
-  tor_assert(to);
-  tor_assert(fromlen < INT_MAX);
-
-  if (fromlen <= CIPHER_IV_LEN)
-    return -1;
-  if (tolen < fromlen - CIPHER_IV_LEN)
-    return -1;
-
-  cipher = crypto_cipher_new_with_iv(key, from);
-
-  crypto_cipher_encrypt(cipher, to, from+CIPHER_IV_LEN, fromlen-CIPHER_IV_LEN);
-  crypto_cipher_free(cipher);
-  return (int)(fromlen - CIPHER_IV_LEN);
-}
-
-/* SHA-1 */
-
-/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in
- * <b>m</b>.  Write the DIGEST_LEN byte result into <b>digest</b>.
- * Return 0 on success, 1 on failure.
- */
-int
-crypto_digest(char *digest, const char *m, size_t len)
-{
-  tor_assert(m);
-  tor_assert(digest);
-  return (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL);
-}
-
-/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
- * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN256-byte result
- * into <b>digest</b>.  Return 0 on success, 1 on failure. */
-int
-crypto_digest256(char *digest, const char *m, size_t len,
-                 digest_algorithm_t algorithm)
-{
-  tor_assert(m);
-  tor_assert(digest);
-  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
-  if (algorithm == DIGEST_SHA256)
-    return (SHA256((const uint8_t*)m,len,(uint8_t*)digest) == NULL);
-  else
-    return (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len)
-            == -1);
-}
-
-/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
- * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN512-byte result
- * into <b>digest</b>.  Return 0 on success, 1 on failure. */
-int
-crypto_digest512(char *digest, const char *m, size_t len,
-                 digest_algorithm_t algorithm)
-{
-  tor_assert(m);
-  tor_assert(digest);
-  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
-  if (algorithm == DIGEST_SHA512)
-    return (SHA512((const unsigned char*)m,len,(unsigned char*)digest)
-            == NULL);
-  else
-    return (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len)
-            == -1);
-}
-
-/** Set the common_digests_t in <b>ds_out</b> to contain every digest on the
- * <b>len</b> bytes in <b>m</b> that we know how to compute.  Return 0 on
- * success, -1 on failure. */
-int
-crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len)
-{
-  tor_assert(ds_out);
-  memset(ds_out, 0, sizeof(*ds_out));
-  if (crypto_digest(ds_out->d[DIGEST_SHA1], m, len) < 0)
-    return -1;
-  if (crypto_digest256(ds_out->d[DIGEST_SHA256], m, len, DIGEST_SHA256) < 0)
-    return -1;
-
-  return 0;
-}
-
-/** Return the name of an algorithm, as used in directory documents. */
-const char *
-crypto_digest_algorithm_get_name(digest_algorithm_t alg)
-{
-  switch (alg) {
-    case DIGEST_SHA1:
-      return "sha1";
-    case DIGEST_SHA256:
-      return "sha256";
-    case DIGEST_SHA512:
-      return "sha512";
-    case DIGEST_SHA3_256:
-      return "sha3-256";
-    case DIGEST_SHA3_512:
-      return "sha3-512";
-    default:
-      // LCOV_EXCL_START
-      tor_fragile_assert();
-      return "??unknown_digest??";
-      // LCOV_EXCL_STOP
-  }
-}
-
-/** Given the name of a digest algorithm, return its integer value, or -1 if
- * the name is not recognized. */
-int
-crypto_digest_algorithm_parse_name(const char *name)
-{
-  if (!strcmp(name, "sha1"))
-    return DIGEST_SHA1;
-  else if (!strcmp(name, "sha256"))
-    return DIGEST_SHA256;
-  else if (!strcmp(name, "sha512"))
-    return DIGEST_SHA512;
-  else if (!strcmp(name, "sha3-256"))
-    return DIGEST_SHA3_256;
-  else if (!strcmp(name, "sha3-512"))
-    return DIGEST_SHA3_512;
-  else
-    return -1;
-}
-
-/** Given an algorithm, return the digest length in bytes. */
-size_t
-crypto_digest_algorithm_get_length(digest_algorithm_t alg)
-{
-  switch (alg) {
-    case DIGEST_SHA1:
-      return DIGEST_LEN;
-    case DIGEST_SHA256:
-      return DIGEST256_LEN;
-    case DIGEST_SHA512:
-      return DIGEST512_LEN;
-    case DIGEST_SHA3_256:
-      return DIGEST256_LEN;
-    case DIGEST_SHA3_512:
-      return DIGEST512_LEN;
-    default:
-      tor_assert(0);              // LCOV_EXCL_LINE
-      return 0; /* Unreachable */ // LCOV_EXCL_LINE
-  }
-}
-
-/** Intermediate information about the digest of a stream of data. */
-struct crypto_digest_t {
-  digest_algorithm_t algorithm; /**< Which algorithm is in use? */
-   /** State for the digest we're using.  Only one member of the
-    * union is usable, depending on the value of <b>algorithm</b>. Note also
-    * that space for other members might not even be allocated!
-    */
-  union {
-    SHA_CTX sha1; /**< state for SHA1 */
-    SHA256_CTX sha2; /**< state for SHA256 */
-    SHA512_CTX sha512; /**< state for SHA512 */
-    keccak_state sha3; /**< state for SHA3-[256,512] */
-  } d;
-};
-
-/**
- * Return the number of bytes we need to malloc in order to get a
- * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe
- * when we free one.
- */
-static size_t
-crypto_digest_alloc_bytes(digest_algorithm_t alg)
-{
-  /* Helper: returns the number of bytes in the 'f' field of 'st' */
-#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f ))
-  /* Gives the length of crypto_digest_t through the end of the field 'd' */
-#define END_OF_FIELD(f) (STRUCT_OFFSET(crypto_digest_t, f) + \
-                         STRUCT_FIELD_SIZE(crypto_digest_t, f))
-  switch (alg) {
-    case DIGEST_SHA1:
-      return END_OF_FIELD(d.sha1);
-    case DIGEST_SHA256:
-      return END_OF_FIELD(d.sha2);
-    case DIGEST_SHA512:
-      return END_OF_FIELD(d.sha512);
-    case DIGEST_SHA3_256:
-    case DIGEST_SHA3_512:
-      return END_OF_FIELD(d.sha3);
-    default:
-      tor_assert(0); // LCOV_EXCL_LINE
-      return 0;      // LCOV_EXCL_LINE
-  }
-#undef END_OF_FIELD
-#undef STRUCT_FIELD_SIZE
-}
-
-/**
- * Internal function: create and return a new digest object for 'algorithm'.
- * Does not typecheck the algorithm.
- */
-static crypto_digest_t *
-crypto_digest_new_internal(digest_algorithm_t algorithm)
-{
-  crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm));
-  r->algorithm = algorithm;
-
-  switch (algorithm)
-    {
-    case DIGEST_SHA1:
-      SHA1_Init(&r->d.sha1);
-      break;
-    case DIGEST_SHA256:
-      SHA256_Init(&r->d.sha2);
-      break;
-    case DIGEST_SHA512:
-      SHA512_Init(&r->d.sha512);
-      break;
-    case DIGEST_SHA3_256:
-      keccak_digest_init(&r->d.sha3, 256);
-      break;
-    case DIGEST_SHA3_512:
-      keccak_digest_init(&r->d.sha3, 512);
-      break;
-    default:
-      tor_assert_unreached();
-    }
-
-  return r;
-}
-
-/** Allocate and return a new digest object to compute SHA1 digests.
- */
-crypto_digest_t *
-crypto_digest_new(void)
-{
-  return crypto_digest_new_internal(DIGEST_SHA1);
-}
-
-/** Allocate and return a new digest object to compute 256-bit digests
- * using <b>algorithm</b>. */
-crypto_digest_t *
-crypto_digest256_new(digest_algorithm_t algorithm)
-{
-  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
-  return crypto_digest_new_internal(algorithm);
-}
-
-/** Allocate and return a new digest object to compute 512-bit digests
- * using <b>algorithm</b>. */
-crypto_digest_t *
-crypto_digest512_new(digest_algorithm_t algorithm)
-{
-  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
-  return crypto_digest_new_internal(algorithm);
-}
-
-/** Deallocate a digest object.
- */
-void
-crypto_digest_free(crypto_digest_t *digest)
-{
-  if (!digest)
-    return;
-  size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  memwipe(digest, 0, bytes);
-  tor_free(digest);
-}
-
-/** Add <b>len</b> bytes from <b>data</b> to the digest object.
- */
-void
-crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
-                        size_t len)
-{
-  tor_assert(digest);
-  tor_assert(data);
-  /* Using the SHA*_*() calls directly means we don't support doing
-   * SHA in hardware. But so far the delay of getting the question
-   * to the hardware, and hearing the answer, is likely higher than
-   * just doing it ourselves. Hashes are fast.
-   */
-  switch (digest->algorithm) {
-    case DIGEST_SHA1:
-      SHA1_Update(&digest->d.sha1, (void*)data, len);
-      break;
-    case DIGEST_SHA256:
-      SHA256_Update(&digest->d.sha2, (void*)data, len);
-      break;
-    case DIGEST_SHA512:
-      SHA512_Update(&digest->d.sha512, (void*)data, len);
-      break;
-    case DIGEST_SHA3_256: /* FALLSTHROUGH */
-    case DIGEST_SHA3_512:
-      keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len);
-      break;
-    default:
-      /* LCOV_EXCL_START */
-      tor_fragile_assert();
-      break;
-      /* LCOV_EXCL_STOP */
-  }
-}
-
-/** Compute the hash of the data that has been passed to the digest
- * object; write the first out_len bytes of the result to <b>out</b>.
- * <b>out_len</b> must be \<= DIGEST512_LEN.
- */
-void
-crypto_digest_get_digest(crypto_digest_t *digest,
-                         char *out, size_t out_len)
-{
-  unsigned char r[DIGEST512_LEN];
-  crypto_digest_t tmpenv;
-  tor_assert(digest);
-  tor_assert(out);
-  tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm));
-
-  /* The SHA-3 code handles copying into a temporary ctx, and also can handle
-   * short output buffers by truncating appropriately. */
-  if (digest->algorithm == DIGEST_SHA3_256 ||
-      digest->algorithm == DIGEST_SHA3_512) {
-    keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len);
-    return;
-  }
-
-  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  /* memcpy into a temporary ctx, since SHA*_Final clears the context */
-  memcpy(&tmpenv, digest, alloc_bytes);
-  switch (digest->algorithm) {
-    case DIGEST_SHA1:
-      SHA1_Final(r, &tmpenv.d.sha1);
-      break;
-    case DIGEST_SHA256:
-      SHA256_Final(r, &tmpenv.d.sha2);
-      break;
-    case DIGEST_SHA512:
-      SHA512_Final(r, &tmpenv.d.sha512);
-      break;
-//LCOV_EXCL_START
-    case DIGEST_SHA3_256: /* FALLSTHROUGH */
-    case DIGEST_SHA3_512:
-    default:
-      log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm);
-      /* This is fatal, because it should never happen. */
-      tor_assert_unreached();
-      break;
-//LCOV_EXCL_STOP
-  }
-  memcpy(out, r, out_len);
-  memwipe(r, 0, sizeof(r));
-}
-
-/** Allocate and return a new digest object with the same state as
- * <b>digest</b>
- */
-crypto_digest_t *
-crypto_digest_dup(const crypto_digest_t *digest)
-{
-  tor_assert(digest);
-  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  return tor_memdup(digest, alloc_bytes);
-}
-
-/** Replace the state of the digest object <b>into</b> with the state
- * of the digest object <b>from</b>.  Requires that 'into' and 'from'
- * have the same digest type.
- */
-void
-crypto_digest_assign(crypto_digest_t *into,
-                     const crypto_digest_t *from)
-{
-  tor_assert(into);
-  tor_assert(from);
-  tor_assert(into->algorithm == from->algorithm);
-  const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm);
-  memcpy(into,from,alloc_bytes);
-}
-
-/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
- * at <b>digest_out</b> to the hash of the concatenation of those strings,
- * plus the optional string <b>append</b>, computed with the algorithm
- * <b>alg</b>.
- * <b>out_len</b> must be \<= DIGEST512_LEN. */
-void
-crypto_digest_smartlist(char *digest_out, size_t len_out,
-                        const smartlist_t *lst,
-                        const char *append,
-                        digest_algorithm_t alg)
-{
-  crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg);
-}
-
-/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
- * at <b>digest_out</b> to the hash of the concatenation of: the
- * optional string <b>prepend</b>, those strings,
- * and the optional string <b>append</b>, computed with the algorithm
- * <b>alg</b>.
- * <b>len_out</b> must be \<= DIGEST512_LEN. */
-void
-crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
-                        const char *prepend,
-                        const smartlist_t *lst,
-                        const char *append,
-                        digest_algorithm_t alg)
-{
-  crypto_digest_t *d = crypto_digest_new_internal(alg);
-  if (prepend)
-    crypto_digest_add_bytes(d, prepend, strlen(prepend));
-  SMARTLIST_FOREACH(lst, const char *, cp,
-                    crypto_digest_add_bytes(d, cp, strlen(cp)));
-  if (append)
-    crypto_digest_add_bytes(d, append, strlen(append));
-  crypto_digest_get_digest(d, digest_out, len_out);
-  crypto_digest_free(d);
-}
-
-/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using
- * the <b>key</b> of length <b>key_len</b>.  Store the DIGEST256_LEN-byte
- * result in <b>hmac_out</b>. Asserts on failure.
- */
-void
-crypto_hmac_sha256(char *hmac_out,
-                   const char *key, size_t key_len,
-                   const char *msg, size_t msg_len)
-{
-  unsigned char *rv = NULL;
-  /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */
-  tor_assert(key_len < INT_MAX);
-  tor_assert(msg_len < INT_MAX);
-  tor_assert(hmac_out);
-  rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len,
-            (unsigned char*)hmac_out, NULL);
-  tor_assert(rv);
-}
-
-/** Internal state for a eXtendable-Output Function (XOF). */
-struct crypto_xof_t {
-  keccak_state s;
-};
-
-/** Allocate a new XOF object backed by SHAKE-256.  The security level
- * provided is a function of the length of the output used.  Read and
- * understand FIPS-202 A.2 "Additional Consideration for Extendable-Output
- * Functions" before using this construct.
- */
-crypto_xof_t *
-crypto_xof_new(void)
-{
-  crypto_xof_t *xof;
-  xof = tor_malloc(sizeof(crypto_xof_t));
-  keccak_xof_init(&xof->s, 256);
-  return xof;
-}
-
-/** Absorb bytes into a XOF object.  Must not be called after a call to
- * crypto_xof_squeeze_bytes() for the same instance, and will assert
- * if attempted.
- */
-void
-crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len)
-{
-  int i = keccak_xof_absorb(&xof->s, data, len);
-  tor_assert(i == 0);
-}
-
-/** Squeeze bytes out of a XOF object.  Calling this routine will render
- * the XOF instance ineligible to absorb further data.
- */
-void
-crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len)
-{
-  int i = keccak_xof_squeeze(&xof->s, out, len);
-  tor_assert(i == 0);
-}
-
-/** Cleanse and deallocate a XOF object. */
-void
-crypto_xof_free(crypto_xof_t *xof)
-{
-  if (!xof)
-    return;
-  memwipe(xof, 0, sizeof(crypto_xof_t));
-  tor_free(xof);
-}
-
-/* DH */
-
-/** Our DH 'g' parameter */
-#define DH_GENERATOR 2
-
-/** Shared P parameter for our circuit-crypto DH key exchanges. */
-static BIGNUM *dh_param_p = NULL;
-/** Shared P parameter for our TLS DH key exchanges. */
-static BIGNUM *dh_param_p_tls = NULL;
-/** Shared G parameter for our DH key exchanges. */
-static BIGNUM *dh_param_g = NULL;
-
-/** Validate a given set of Diffie-Hellman parameters.  This is moderately
- * computationally expensive (milliseconds), so should only be called when
- * the DH parameters change. Returns 0 on success, * -1 on failure.
- */
-static int
-crypto_validate_dh_params(const BIGNUM *p, const BIGNUM *g)
-{
-  DH *dh = NULL;
-  int ret = -1;
-
-  /* Copy into a temporary DH object, just so that DH_check() can be called. */
-  if (!(dh = DH_new()))
-      goto out;
-#ifdef OPENSSL_1_1_API
-  BIGNUM *dh_p, *dh_g;
-  if (!(dh_p = BN_dup(p)))
-    goto out;
-  if (!(dh_g = BN_dup(g)))
-    goto out;
-  if (!DH_set0_pqg(dh, dh_p, NULL, dh_g))
-    goto out;
-#else
-  if (!(dh->p = BN_dup(p)))
-    goto out;
-  if (!(dh->g = BN_dup(g)))
-    goto out;
-#endif
-
-  /* Perform the validation. */
-  int codes = 0;
-  if (!DH_check(dh, &codes))
-    goto out;
-  if (BN_is_word(g, DH_GENERATOR_2)) {
-    /* Per https://wiki.openssl.org/index.php/Diffie-Hellman_parameters
-     *
-     * OpenSSL checks the prime is congruent to 11 when g = 2; while the
-     * IETF's primes are congruent to 23 when g = 2.
-     */
-    BN_ULONG residue = BN_mod_word(p, 24);
-    if (residue == 11 || residue == 23)
-      codes &= ~DH_NOT_SUITABLE_GENERATOR;
-  }
-  if (codes != 0) /* Specifics on why the params suck is irrelevant. */
-    goto out;
-
-  /* Things are probably not evil. */
-  ret = 0;
-
- out:
-  if (dh)
-    DH_free(dh);
-  return ret;
-}
-
-/** Set the global Diffie-Hellman generator, used for both TLS and internal
- * DH stuff.
- */
-static void
-crypto_set_dh_generator(void)
-{
-  BIGNUM *generator;
-  int r;
-
-  if (dh_param_g)
-    return;
-
-  generator = BN_new();
-  tor_assert(generator);
-
-  r = BN_set_word(generator, DH_GENERATOR);
-  tor_assert(r);
-
-  dh_param_g = generator;
-}
-
-/** Set the global TLS Diffie-Hellman modulus.  Use the Apache mod_ssl DH
- * modulus. */
-void
-crypto_set_tls_dh_prime(void)
-{
-  BIGNUM *tls_prime = NULL;
-  int r;
-
-  /* If the space is occupied, free the previous TLS DH prime */
-  if (BUG(dh_param_p_tls)) {
-    /* LCOV_EXCL_START
-     *
-     * We shouldn't be calling this twice.
-     */
-    BN_clear_free(dh_param_p_tls);
-    dh_param_p_tls = NULL;
-    /* LCOV_EXCL_STOP */
-  }
-
-  tls_prime = BN_new();
-  tor_assert(tls_prime);
-
-  /* This is the 1024-bit safe prime that Apache uses for its DH stuff; see
-   * modules/ssl/ssl_engine_dh.c; Apache also uses a generator of 2 with this
-   * prime.
-   */
-  r = BN_hex2bn(&tls_prime,
-               "D67DE440CBBBDC1936D693D34AFD0AD50C84D239A45F520BB88174CB98"
-               "BCE951849F912E639C72FB13B4B4D7177E16D55AC179BA420B2A29FE324A"
-               "467A635E81FF5901377BEDDCFD33168A461AAD3B72DAE8860078045B07A7"
-               "DBCA7874087D1510EA9FCC9DDD330507DD62DB88AEAA747DE0F4D6E2BD68"
-               "B0E7393E0F24218EB3");
-  tor_assert(r);
-
-  tor_assert(tls_prime);
-
-  dh_param_p_tls = tls_prime;
-  crypto_set_dh_generator();
-  tor_assert(0 == crypto_validate_dh_params(dh_param_p_tls, dh_param_g));
-}
-
-/** Initialize dh_param_p and dh_param_g if they are not already
- * set. */
-static void
-init_dh_param(void)
-{
-  BIGNUM *circuit_dh_prime;
-  int r;
-  if (BUG(dh_param_p && dh_param_g))
-    return; // LCOV_EXCL_LINE This function isn't supposed to be called twice.
-
-  circuit_dh_prime = BN_new();
-  tor_assert(circuit_dh_prime);
-
-  /* This is from rfc2409, section 6.2.  It's a safe prime, and
-     supposedly it equals:
-        2^1024 - 2^960 - 1 + 2^64 * { [2^894 pi] + 129093 }.
-  */
-  r = BN_hex2bn(&circuit_dh_prime,
-                "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E08"
-                "8A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B"
-                "302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9"
-                "A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE6"
-                "49286651ECE65381FFFFFFFFFFFFFFFF");
-  tor_assert(r);
-
-  /* Set the new values as the global DH parameters. */
-  dh_param_p = circuit_dh_prime;
-  crypto_set_dh_generator();
-  tor_assert(0 == crypto_validate_dh_params(dh_param_p, dh_param_g));
-
-  if (!dh_param_p_tls) {
-    crypto_set_tls_dh_prime();
-  }
-}
-
-/** Number of bits to use when choosing the x or y value in a Diffie-Hellman
- * handshake.  Since we exponentiate by this value, choosing a smaller one
- * lets our handhake go faster.
- */
-#define DH_PRIVATE_KEY_BITS 320
-
-/** Allocate and return a new DH object for a key exchange. Returns NULL on
- * failure.
- */
-crypto_dh_t *
-crypto_dh_new(int dh_type)
-{
-  crypto_dh_t *res = tor_malloc_zero(sizeof(crypto_dh_t));
-
-  tor_assert(dh_type == DH_TYPE_CIRCUIT || dh_type == DH_TYPE_TLS ||
-             dh_type == DH_TYPE_REND);
-
-  if (!dh_param_p)
-    init_dh_param();
-
-  if (!(res->dh = DH_new()))
-    goto err;
-
-#ifdef OPENSSL_1_1_API
-  BIGNUM *dh_p = NULL, *dh_g = NULL;
-
-  if (dh_type == DH_TYPE_TLS) {
-    dh_p = BN_dup(dh_param_p_tls);
-  } else {
-    dh_p = BN_dup(dh_param_p);
-  }
-  if (!dh_p)
-    goto err;
-
-  dh_g = BN_dup(dh_param_g);
-  if (!dh_g) {
-    BN_free(dh_p);
-    goto err;
-  }
-
-  if (!DH_set0_pqg(res->dh, dh_p, NULL, dh_g)) {
-    goto err;
-  }
-
-  if (!DH_set_length(res->dh, DH_PRIVATE_KEY_BITS))
-    goto err;
-#else
-  if (dh_type == DH_TYPE_TLS) {
-    if (!(res->dh->p = BN_dup(dh_param_p_tls)))
-      goto err;
-  } else {
-    if (!(res->dh->p = BN_dup(dh_param_p)))
-      goto err;
-  }
-
-  if (!(res->dh->g = BN_dup(dh_param_g)))
-    goto err;
-
-  res->dh->length = DH_PRIVATE_KEY_BITS;
-#endif
-
-  return res;
- err:
-  /* LCOV_EXCL_START
-   * This error condition is only reached when an allocation fails */
-  crypto_log_errors(LOG_WARN, "creating DH object");
-  if (res->dh) DH_free(res->dh); /* frees p and g too */
-  tor_free(res);
-  return NULL;
-  /* LCOV_EXCL_STOP */
-}
-
-/** Return a copy of <b>dh</b>, sharing its internal state. */
-crypto_dh_t *
-crypto_dh_dup(const crypto_dh_t *dh)
-{
-  crypto_dh_t *dh_new = tor_malloc_zero(sizeof(crypto_dh_t));
-  tor_assert(dh);
-  tor_assert(dh->dh);
-  dh_new->dh = dh->dh;
-  DH_up_ref(dh->dh);
-  return dh_new;
-}
-
-/** Return the length of the DH key in <b>dh</b>, in bytes.
- */
-int
-crypto_dh_get_bytes(crypto_dh_t *dh)
-{
-  tor_assert(dh);
-  return DH_size(dh->dh);
-}
-
-/** Generate \<x,g^x\> for our part of the key exchange.  Return 0 on
- * success, -1 on failure.
- */
-int
-crypto_dh_generate_public(crypto_dh_t *dh)
-{
-#ifndef OPENSSL_1_1_API
- again:
-#endif
-  if (!DH_generate_key(dh->dh)) {
-    /* LCOV_EXCL_START
-     * To test this we would need some way to tell openssl to break DH. */
-    crypto_log_errors(LOG_WARN, "generating DH key");
-    return -1;
-    /* LCOV_EXCL_STOP */
-  }
-#ifdef OPENSSL_1_1_API
-  /* OpenSSL 1.1.x doesn't appear to let you regenerate a DH key, without
-   * recreating the DH object.  I have no idea what sort of aliasing madness
-   * can occur here, so do the check, and just bail on failure.
-   */
-  const BIGNUM *pub_key, *priv_key;
-  DH_get0_key(dh->dh, &pub_key, &priv_key);
-  if (tor_check_dh_key(LOG_WARN, pub_key)<0) {
-    log_warn(LD_CRYPTO, "Weird! Our own DH key was invalid.  I guess once-in-"
-             "the-universe chances really do happen.  Treating as a failure.");
-    return -1;
-  }
-#else
-  if (tor_check_dh_key(LOG_WARN, dh->dh->pub_key)<0) {
-    /* LCOV_EXCL_START
-     * If this happens, then openssl's DH implementation is busted. */
-    log_warn(LD_CRYPTO, "Weird! Our own DH key was invalid.  I guess once-in-"
-             "the-universe chances really do happen.  Trying again.");
-    /* Free and clear the keys, so OpenSSL will actually try again. */
-    BN_clear_free(dh->dh->pub_key);
-    BN_clear_free(dh->dh->priv_key);
-    dh->dh->pub_key = dh->dh->priv_key = NULL;
-    goto again;
-    /* LCOV_EXCL_STOP */
-  }
-#endif
-  return 0;
-}
-
-/** Generate g^x as necessary, and write the g^x for the key exchange
- * as a <b>pubkey_len</b>-byte value into <b>pubkey</b>. Return 0 on
- * success, -1 on failure.  <b>pubkey_len</b> must be \>= DH_BYTES.
- */
-int
-crypto_dh_get_public(crypto_dh_t *dh, char *pubkey, size_t pubkey_len)
-{
-  int bytes;
-  tor_assert(dh);
-
-  const BIGNUM *dh_pub;
-
-#ifdef OPENSSL_1_1_API
-  const BIGNUM *dh_priv;
-  DH_get0_key(dh->dh, &dh_pub, &dh_priv);
-#else
-  dh_pub = dh->dh->pub_key;
-#endif
-
-  if (!dh_pub) {
-    if (crypto_dh_generate_public(dh)<0)
-      return -1;
-    else {
-#ifdef OPENSSL_1_1_API
-      DH_get0_key(dh->dh, &dh_pub, &dh_priv);
-#else
-      dh_pub = dh->dh->pub_key;
-#endif
-    }
-  }
-
-  tor_assert(dh_pub);
-  bytes = BN_num_bytes(dh_pub);
-  tor_assert(bytes >= 0);
-  if (pubkey_len < (size_t)bytes) {
-    log_warn(LD_CRYPTO,
-             "Weird! pubkey_len (%d) was smaller than DH_BYTES (%d)",
-             (int) pubkey_len, bytes);
-    return -1;
-  }
-
-  memset(pubkey, 0, pubkey_len);
-  BN_bn2bin(dh_pub, (unsigned char*)(pubkey+(pubkey_len-bytes)));
-
-  return 0;
-}
-
-/** Check for bad Diffie-Hellman public keys (g^x).  Return 0 if the key is
- * okay (in the subgroup [2,p-2]), or -1 if it's bad.
- * See http://www.cl.cam.ac.uk/ftp/users/rja14/psandqs.ps.gz for some tips.
- */
-static int
-tor_check_dh_key(int severity, const BIGNUM *bn)
-{
-  BIGNUM *x;
-  char *s;
-  tor_assert(bn);
-  x = BN_new();
-  tor_assert(x);
-  if (BUG(!dh_param_p))
-    init_dh_param(); //LCOV_EXCL_LINE we already checked whether we did this.
-  BN_set_word(x, 1);
-  if (BN_cmp(bn,x)<=0) {
-    log_fn(severity, LD_CRYPTO, "DH key must be at least 2.");
-    goto err;
-  }
-  BN_copy(x,dh_param_p);
-  BN_sub_word(x, 1);
-  if (BN_cmp(bn,x)>=0) {
-    log_fn(severity, LD_CRYPTO, "DH key must be at most p-2.");
-    goto err;
-  }
-  BN_clear_free(x);
-  return 0;
- err:
-  BN_clear_free(x);
-  s = BN_bn2hex(bn);
-  log_fn(severity, LD_CRYPTO, "Rejecting insecure DH key [%s]", s);
-  OPENSSL_free(s);
-  return -1;
-}
-
-/** Given a DH key exchange object, and our peer's value of g^y (as a
- * <b>pubkey_len</b>-byte value in <b>pubkey</b>) generate
- * <b>secret_bytes_out</b> bytes of shared key material and write them
- * to <b>secret_out</b>.  Return the number of bytes generated on success,
- * or -1 on failure.
- *
- * (We generate key material by computing
- *         SHA1( g^xy || "\x00" ) || SHA1( g^xy || "\x01" ) || ...
- * where || is concatenation.)
- */
-ssize_t
-crypto_dh_compute_secret(int severity, crypto_dh_t *dh,
-                         const char *pubkey, size_t pubkey_len,
-                         char *secret_out, size_t secret_bytes_out)
-{
-  char *secret_tmp = NULL;
-  BIGNUM *pubkey_bn = NULL;
-  size_t secret_len=0, secret_tmp_len=0;
-  int result=0;
-  tor_assert(dh);
-  tor_assert(secret_bytes_out/DIGEST_LEN <= 255);
-  tor_assert(pubkey_len < INT_MAX);
-
-  if (!(pubkey_bn = BN_bin2bn((const unsigned char*)pubkey,
-                              (int)pubkey_len, NULL)))
-    goto error;
-  if (tor_check_dh_key(severity, pubkey_bn)<0) {
-    /* Check for invalid public keys. */
-    log_fn(severity, LD_CRYPTO,"Rejected invalid g^x");
-    goto error;
-  }
-  secret_tmp_len = crypto_dh_get_bytes(dh);
-  secret_tmp = tor_malloc(secret_tmp_len);
-  result = DH_compute_key((unsigned char*)secret_tmp, pubkey_bn, dh->dh);
-  if (result < 0) {
-    log_warn(LD_CRYPTO,"DH_compute_key() failed.");
-    goto error;
-  }
-  secret_len = result;
-  if (crypto_expand_key_material_TAP((uint8_t*)secret_tmp, secret_len,
-                                     (uint8_t*)secret_out, secret_bytes_out)<0)
-    goto error;
-  secret_len = secret_bytes_out;
-
-  goto done;
- error:
-  result = -1;
- done:
-  crypto_log_errors(LOG_WARN, "completing DH handshake");
-  if (pubkey_bn)
-    BN_clear_free(pubkey_bn);
-  if (secret_tmp) {
-    memwipe(secret_tmp, 0, secret_tmp_len);
-    tor_free(secret_tmp);
-  }
-  if (result < 0)
-    return result;
-  else
-    return secret_len;
-}
-
-/** Given <b>key_in_len</b> bytes of negotiated randomness in <b>key_in</b>
- * ("K"), expand it into <b>key_out_len</b> bytes of negotiated key material in
- * <b>key_out</b> by taking the first <b>key_out_len</b> bytes of
- *    H(K | [00]) | H(K | [01]) | ....
- *
- * This is the key expansion algorithm used in the "TAP" circuit extension
- * mechanism; it shouldn't be used for new protocols.
- *
- * Return 0 on success, -1 on failure.
- */
-int
-crypto_expand_key_material_TAP(const uint8_t *key_in, size_t key_in_len,
-                               uint8_t *key_out, size_t key_out_len)
-{
-  int i, r = -1;
-  uint8_t *cp, *tmp = tor_malloc(key_in_len+1);
-  uint8_t digest[DIGEST_LEN];
-
-  /* If we try to get more than this amount of key data, we'll repeat blocks.*/
-  tor_assert(key_out_len <= DIGEST_LEN*256);
-
-  memcpy(tmp, key_in, key_in_len);
-  for (cp = key_out, i=0; cp < key_out+key_out_len;
-       ++i, cp += DIGEST_LEN) {
-    tmp[key_in_len] = i;
-    if (crypto_digest((char*)digest, (const char *)tmp, key_in_len+1))
-      goto exit;
-    memcpy(cp, digest, MIN(DIGEST_LEN, key_out_len-(cp-key_out)));
-  }
-
-  r = 0;
- exit:
-  memwipe(tmp, 0, key_in_len+1);
-  tor_free(tmp);
-  memwipe(digest, 0, sizeof(digest));
-  return r;
-}
-
-/** Expand some secret key material according to RFC5869, using SHA256 as the
- * underlying hash.  The <b>key_in_len</b> bytes at <b>key_in</b> are the
- * secret key material; the <b>salt_in_len</b> bytes at <b>salt_in</b> and the
- * <b>info_in_len</b> bytes in <b>info_in_len</b> are the algorithm's "salt"
- * and "info" parameters respectively.  On success, write <b>key_out_len</b>
- * bytes to <b>key_out</b> and return 0.  Assert on failure.
- */
-int
-crypto_expand_key_material_rfc5869_sha256(
-                                    const uint8_t *key_in, size_t key_in_len,
-                                    const uint8_t *salt_in, size_t salt_in_len,
-                                    const uint8_t *info_in, size_t info_in_len,
-                                    uint8_t *key_out, size_t key_out_len)
-{
-  uint8_t prk[DIGEST256_LEN];
-  uint8_t tmp[DIGEST256_LEN + 128 + 1];
-  uint8_t mac[DIGEST256_LEN];
-  int i;
-  uint8_t *outp;
-  size_t tmp_len;
-
-  crypto_hmac_sha256((char*)prk,
-                     (const char*)salt_in, salt_in_len,
-                     (const char*)key_in, key_in_len);
-
-  /* If we try to get more than this amount of key data, we'll repeat blocks.*/
-  tor_assert(key_out_len <= DIGEST256_LEN * 256);
-  tor_assert(info_in_len <= 128);
-  memset(tmp, 0, sizeof(tmp));
-  outp = key_out;
-  i = 1;
-
-  while (key_out_len) {
-    size_t n;
-    if (i > 1) {
-      memcpy(tmp, mac, DIGEST256_LEN);
-      memcpy(tmp+DIGEST256_LEN, info_in, info_in_len);
-      tmp[DIGEST256_LEN+info_in_len] = i;
-      tmp_len = DIGEST256_LEN + info_in_len + 1;
-    } else {
-      memcpy(tmp, info_in, info_in_len);
-      tmp[info_in_len] = i;
-      tmp_len = info_in_len + 1;
-    }
-    crypto_hmac_sha256((char*)mac,
-                       (const char*)prk, DIGEST256_LEN,
-                       (const char*)tmp, tmp_len);
-    n = key_out_len < DIGEST256_LEN ? key_out_len : DIGEST256_LEN;
-    memcpy(outp, mac, n);
-    key_out_len -= n;
-    outp += n;
-    ++i;
-  }
-
-  memwipe(tmp, 0, sizeof(tmp));
-  memwipe(mac, 0, sizeof(mac));
-  return 0;
-}
-
-/** Free a DH key exchange object.
- */
-void
-crypto_dh_free(crypto_dh_t *dh)
-{
-  if (!dh)
-    return;
-  tor_assert(dh->dh);
-  DH_free(dh->dh);
-  tor_free(dh);
-}
-
-/* random numbers */
-
-/** How many bytes of entropy we add at once.
- *
- * This is how much entropy OpenSSL likes to add right now, so maybe it will
- * work for us too. */
-#define ADD_ENTROPY 32
-
-/** Set the seed of the weak RNG to a random value. */
-void
-crypto_seed_weak_rng(tor_weak_rng_t *rng)
-{
-  unsigned seed;
-  crypto_rand((void*)&seed, sizeof(seed));
-  tor_init_weak_random(rng, seed);
-}
-
-#ifdef TOR_UNIT_TESTS
-int break_strongest_rng_syscall = 0;
-int break_strongest_rng_fallback = 0;
-#endif
-
-/** Try to get <b>out_len</b> bytes of the strongest entropy we can generate,
- * via system calls, storing it into <b>out</b>. Return 0 on success, -1 on
- * failure.  A maximum request size of 256 bytes is imposed.
- */
-static int
-crypto_strongest_rand_syscall(uint8_t *out, size_t out_len)
-{
-  tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE);
-
-#ifdef TOR_UNIT_TESTS
-  if (break_strongest_rng_syscall)
-    return -1;
-#endif
-
-#if defined(_WIN32)
-  static int provider_set = 0;
-  static HCRYPTPROV provider;
-
-  if (!provider_set) {
-    if (!CryptAcquireContext(&provider, NULL, NULL, PROV_RSA_FULL,
-                             CRYPT_VERIFYCONTEXT)) {
-      log_warn(LD_CRYPTO, "Can't get CryptoAPI provider [1]");
-      return -1;
-    }
-    provider_set = 1;
-  }
-  if (!CryptGenRandom(provider, out_len, out)) {
-    log_warn(LD_CRYPTO, "Can't get entropy from CryptoAPI.");
-    return -1;
-  }
-
-  return 0;
-#elif defined(__linux__) && defined(SYS_getrandom)
-  static int getrandom_works = 1; /* Be optimitic about our chances... */
-
-  /* getrandom() isn't as straight foward as getentropy(), and has
-   * no glibc wrapper.
-   *
-   * As far as I can tell from getrandom(2) and the source code, the
-   * requests we issue will always succeed (though it will block on the
-   * call if /dev/urandom isn't seeded yet), since we are NOT specifying
-   * GRND_NONBLOCK and the request is <= 256 bytes.
-   *
-   * The manpage is unclear on what happens if a signal interrupts the call
-   * while the request is blocked due to lack of entropy....
-   *
-   * We optimistically assume that getrandom() is available and functional
-   * because it is the way of the future, and 2 branch mispredicts pale in
-   * comparision to the overheads involved with failing to open
-   * /dev/srandom followed by opening and reading from /dev/urandom.
-   */
-  if (PREDICT_LIKELY(getrandom_works)) {
-    long ret;
-    /* A flag of '0' here means to read from '/dev/urandom', and to
-     * block if insufficient entropy is available to service the
-     * request.
-     */
-    const unsigned int flags = 0;
-    do {
-      ret = syscall(SYS_getrandom, out, out_len, flags);
-    } while (ret == -1 && ((errno == EINTR) ||(errno == EAGAIN)));
-
-    if (PREDICT_UNLIKELY(ret == -1)) {
-      /* LCOV_EXCL_START we can't actually make the syscall fail in testing. */
-      tor_assert(errno != EAGAIN);
-      tor_assert(errno != EINTR);
-
-      /* Probably ENOSYS. */
-      log_warn(LD_CRYPTO, "Can't get entropy from getrandom().");
-      getrandom_works = 0; /* Don't bother trying again. */
-      return -1;
-      /* LCOV_EXCL_STOP */
-    }
-
-    tor_assert(ret == (long)out_len);
-    return 0;
-  }
-
-  return -1; /* getrandom() previously failed unexpectedly. */
-#elif defined(HAVE_GETENTROPY)
-  /* getentropy() is what Linux's getrandom() wants to be when it grows up.
-   * the only gotcha is that requests are limited to 256 bytes.
-   */
-  return getentropy(out, out_len);
-#else
-  (void) out;
-#endif
-
-  /* This platform doesn't have a supported syscall based random. */
-  return -1;
-}
-
-/** Try to get <b>out_len</b> bytes of the strongest entropy we can generate,
- * via the per-platform fallback mechanism, storing it into <b>out</b>.
- * Return 0 on success, -1 on failure.  A maximum request size of 256 bytes
- * is imposed.
- */
-static int
-crypto_strongest_rand_fallback(uint8_t *out, size_t out_len)
-{
-#ifdef TOR_UNIT_TESTS
-  if (break_strongest_rng_fallback)
-    return -1;
-#endif
-
-#ifdef _WIN32
-  /* Windows exclusively uses crypto_strongest_rand_syscall(). */
-  (void)out;
-  (void)out_len;
-  return -1;
-#else
-  static const char *filenames[] = {
-    "/dev/srandom", "/dev/urandom", "/dev/random", NULL
-  };
-  int fd, i;
-  size_t n;
-
-  for (i = 0; filenames[i]; ++i) {
-    log_debug(LD_FS, "Opening %s for entropy", filenames[i]);
-    fd = open(sandbox_intern_string(filenames[i]), O_RDONLY, 0);
-    if (fd<0) continue;
-    log_info(LD_CRYPTO, "Reading entropy from \"%s\"", filenames[i]);
-    n = read_all(fd, (char*)out, out_len, 0);
-    close(fd);
-    if (n != out_len) {
-      /* LCOV_EXCL_START
-       * We can't make /dev/foorandom actually fail. */
-      log_warn(LD_CRYPTO,
-               "Error reading from entropy source (read only %lu bytes).",
-               (unsigned long)n);
-      return -1;
-      /* LCOV_EXCL_STOP */
-    }
-
-    return 0;
-  }
-
-  return -1;
-#endif
-}
-
-/** Try to get <b>out_len</b> bytes of the strongest entropy we can generate,
- * storing it into <b>out</b>. Return 0 on success, -1 on failure.  A maximum
- * request size of 256 bytes is imposed.
- */
-STATIC int
-crypto_strongest_rand_raw(uint8_t *out, size_t out_len)
-{
-  static const size_t sanity_min_size = 16;
-  static const int max_attempts = 3;
-  tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE);
-
-  /* For buffers >= 16 bytes (128 bits), we sanity check the output by
-   * zero filling the buffer and ensuring that it actually was at least
-   * partially modified.
-   *
-   * Checking that any individual byte is non-zero seems like it would
-   * fail too often (p = out_len * 1/256) for comfort, but this is an
-   * "adjust according to taste" sort of check.
-   */
-  memwipe(out, 0, out_len);
-  for (int i = 0; i < max_attempts; i++) {
-    /* Try to use the syscall/OS favored mechanism to get strong entropy. */
-    if (crypto_strongest_rand_syscall(out, out_len) != 0) {
-      /* Try to use the less-favored mechanism to get strong entropy. */
-      if (crypto_strongest_rand_fallback(out, out_len) != 0) {
-        /* Welp, we tried.  Hopefully the calling code terminates the process
-         * since we're basically boned without good entropy.
-         */
-        log_warn(LD_CRYPTO,
-                 "Cannot get strong entropy: no entropy source found.");
-        return -1;
-      }
-    }
-
-    if ((out_len < sanity_min_size) || !tor_mem_is_zero((char*)out, out_len))
-      return 0;
-  }
-
-  /* LCOV_EXCL_START
-   *
-   * We tried max_attempts times to fill a buffer >= 128 bits long,
-   * and each time it returned all '0's.  Either the system entropy
-   * source is busted, or the user should go out and buy a ticket to
-   * every lottery on the planet.
-   */
-  log_warn(LD_CRYPTO, "Strong OS entropy returned all zero buffer.");
-
-  return -1;
-  /* LCOV_EXCL_STOP */
-}
-
-/** Try to get <b>out_len</b> bytes of the strongest entropy we can generate,
- * storing it into <b>out</b>.
- */
-void
-crypto_strongest_rand(uint8_t *out, size_t out_len)
-{
-#define DLEN SHA512_DIGEST_LENGTH
-  /* We're going to hash DLEN bytes from the system RNG together with some
-   * bytes from the openssl PRNG, in order to yield DLEN bytes.
-   */
-  uint8_t inp[DLEN*2];
-  uint8_t tmp[DLEN];
-  tor_assert(out);
-  while (out_len) {
-    crypto_rand((char*) inp, DLEN);
-    if (crypto_strongest_rand_raw(inp+DLEN, DLEN) < 0) {
-      // LCOV_EXCL_START
-      log_err(LD_CRYPTO, "Failed to load strong entropy when generating an "
-              "important key. Exiting.");
-      /* Die with an assertion so we get a stack trace. */
-      tor_assert(0);
-      // LCOV_EXCL_STOP
-    }
-    if (out_len >= DLEN) {
-      SHA512(inp, sizeof(inp), out);
-      out += DLEN;
-      out_len -= DLEN;
-    } else {
-      SHA512(inp, sizeof(inp), tmp);
-      memcpy(out, tmp, out_len);
-      break;
-    }
-  }
-  memwipe(tmp, 0, sizeof(tmp));
-  memwipe(inp, 0, sizeof(inp));
-#undef DLEN
-}
-
-/** Seed OpenSSL's random number generator with bytes from the operating
- * system.  Return 0 on success, -1 on failure.
- */
-int
-crypto_seed_rng(void)
-{
-  int rand_poll_ok = 0, load_entropy_ok = 0;
-  uint8_t buf[ADD_ENTROPY];
-
-  /* OpenSSL has a RAND_poll function that knows about more kinds of
-   * entropy than we do.  We'll try calling that, *and* calling our own entropy
-   * functions.  If one succeeds, we'll accept the RNG as seeded. */
-  rand_poll_ok = RAND_poll();
-  if (rand_poll_ok == 0)
-    log_warn(LD_CRYPTO, "RAND_poll() failed."); // LCOV_EXCL_LINE
-
-  load_entropy_ok = !crypto_strongest_rand_raw(buf, sizeof(buf));
-  if (load_entropy_ok) {
-    RAND_seed(buf, sizeof(buf));
-  }
-
-  memwipe(buf, 0, sizeof(buf));
-
-  if ((rand_poll_ok || load_entropy_ok) && RAND_status() == 1)
-    return 0;
-  else
-    return -1;
-}
-
-/** Write <b>n</b> bytes of strong random data to <b>to</b>. Supports mocking
- * for unit tests.
- *
- * This function is not allowed to fail; if it would fail to generate strong
- * entropy, it must terminate the process instead.
- */
-MOCK_IMPL(void,
-crypto_rand, (char *to, size_t n))
-{
-  crypto_rand_unmocked(to, n);
-}
-
-/** Write <b>n</b> bytes of strong random data to <b>to</b>.  Most callers
- * will want crypto_rand instead.
- *
- * This function is not allowed to fail; if it would fail to generate strong
- * entropy, it must terminate the process instead.
- */
-void
-crypto_rand_unmocked(char *to, size_t n)
-{
-  int r;
-  if (n == 0)
-    return;
-
-  tor_assert(n < INT_MAX);
-  tor_assert(to);
-  r = RAND_bytes((unsigned char*)to, (int)n);
-  /* We consider a PRNG failure non-survivable. Let's assert so that we get a
-   * stack trace about where it happened.
-   */
-  tor_assert(r >= 0);
-}
-
-/** Return a pseudorandom integer, chosen uniformly from the values
- * between 0 and <b>max</b>-1 inclusive.  <b>max</b> must be between 1 and
- * INT_MAX+1, inclusive. */
-int
-crypto_rand_int(unsigned int max)
-{
-  unsigned int val;
-  unsigned int cutoff;
-  tor_assert(max <= ((unsigned int)INT_MAX)+1);
-  tor_assert(max > 0); /* don't div by 0 */
-
-  /* We ignore any values that are >= 'cutoff,' to avoid biasing the
-   * distribution with clipping at the upper end of unsigned int's
-   * range.
-   */
-  cutoff = UINT_MAX - (UINT_MAX%max);
-  while (1) {
-    crypto_rand((char*)&val, sizeof(val));
-    if (val < cutoff)
-      return val % max;
-  }
-}
-
-/** Return a pseudorandom integer, chosen uniformly from the values i such
- * that min <= i < max.
- *
- * <b>min</b> MUST be in range [0, <b>max</b>).
- * <b>max</b> MUST be in range (min, INT_MAX].
- */
-int
-crypto_rand_int_range(unsigned int min, unsigned int max)
-{
-  tor_assert(min < max);
-  tor_assert(max <= INT_MAX);
-
-  /* The overflow is avoided here because crypto_rand_int() returns a value
-   * between 0 and (max - min) inclusive. */
-  return min + crypto_rand_int(max - min);
-}
-
-/** As crypto_rand_int_range, but supports uint64_t. */
-uint64_t
-crypto_rand_uint64_range(uint64_t min, uint64_t max)
-{
-  tor_assert(min < max);
-  return min + crypto_rand_uint64(max - min);
-}
-
-/** As crypto_rand_int_range, but supports time_t. */
-time_t
-crypto_rand_time_range(time_t min, time_t max)
-{
-  tor_assert(min < max);
-  return min + (time_t)crypto_rand_uint64(max - min);
-}
-
-/** Return a pseudorandom 64-bit integer, chosen uniformly from the values
- * between 0 and <b>max</b>-1 inclusive. */
-uint64_t
-crypto_rand_uint64(uint64_t max)
-{
-  uint64_t val;
-  uint64_t cutoff;
-  tor_assert(max < UINT64_MAX);
-  tor_assert(max > 0); /* don't div by 0 */
-
-  /* We ignore any values that are >= 'cutoff,' to avoid biasing the
-   * distribution with clipping at the upper end of unsigned int's
-   * range.
-   */
-  cutoff = UINT64_MAX - (UINT64_MAX%max);
-  while (1) {
-    crypto_rand((char*)&val, sizeof(val));
-    if (val < cutoff)
-      return val % max;
-  }
-}
-
-/** Return a pseudorandom double d, chosen uniformly from the range
- * 0.0 <= d < 1.0.
- */
-double
-crypto_rand_double(void)
-{
-  /* We just use an unsigned int here; we don't really care about getting
-   * more than 32 bits of resolution */
-  unsigned int u;
-  crypto_rand((char*)&u, sizeof(u));
-#if SIZEOF_INT == 4
-#define UINT_MAX_AS_DOUBLE 4294967296.0
-#elif SIZEOF_INT == 8
-#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19
-#else
-#error SIZEOF_INT is neither 4 nor 8
-#endif
-  return ((double)u) / UINT_MAX_AS_DOUBLE;
-}
-
-/** Generate and return a new random hostname starting with <b>prefix</b>,
- * ending with <b>suffix</b>, and containing no fewer than
- * <b>min_rand_len</b> and no more than <b>max_rand_len</b> random base32
- * characters. Does not check for failure.
- *
- * Clip <b>max_rand_len</b> to MAX_DNS_LABEL_SIZE.
- **/
-char *
-crypto_random_hostname(int min_rand_len, int max_rand_len, const char *prefix,
-                       const char *suffix)
-{
-  char *result, *rand_bytes;
-  int randlen, rand_bytes_len;
-  size_t resultlen, prefixlen;
-
-  if (max_rand_len > MAX_DNS_LABEL_SIZE)
-    max_rand_len = MAX_DNS_LABEL_SIZE;
-  if (min_rand_len > max_rand_len)
-    min_rand_len = max_rand_len;
-
-  randlen = crypto_rand_int_range(min_rand_len, max_rand_len+1);
-
-  prefixlen = strlen(prefix);
-  resultlen = prefixlen + strlen(suffix) + randlen + 16;
-
-  rand_bytes_len = ((randlen*5)+7)/8;
-  if (rand_bytes_len % 5)
-    rand_bytes_len += 5 - (rand_bytes_len%5);
-  rand_bytes = tor_malloc(rand_bytes_len);
-  crypto_rand(rand_bytes, rand_bytes_len);
-
-  result = tor_malloc(resultlen);
-  memcpy(result, prefix, prefixlen);
-  base32_encode(result+prefixlen, resultlen-prefixlen,
-                rand_bytes, rand_bytes_len);
-  tor_free(rand_bytes);
-  strlcpy(result+prefixlen+randlen, suffix, resultlen-(prefixlen+randlen));
-
-  return result;
-}
-
-/** Return a randomly chosen element of <b>sl</b>; or NULL if <b>sl</b>
- * is empty. */
-void *
-smartlist_choose(const smartlist_t *sl)
-{
-  int len = smartlist_len(sl);
-  if (len)
-    return smartlist_get(sl,crypto_rand_int(len));
-  return NULL; /* no elements to choose from */
-}
-
-/** Scramble the elements of <b>sl</b> into a random order. */
-void
-smartlist_shuffle(smartlist_t *sl)
-{
-  int i;
-  /* From the end of the list to the front, choose at random from the
-     positions we haven't looked at yet, and swap that position into the
-     current position.  Remember to give "no swap" the same probability as
-     any other swap. */
-  for (i = smartlist_len(sl)-1; i > 0; --i) {
-    int j = crypto_rand_int(i+1);
-    smartlist_swap(sl, i, j);
-  }
-}
-
-/**
- * Destroy the <b>sz</b> bytes of data stored at <b>mem</b>, setting them to
- * the value <b>byte</b>.
- * If <b>mem</b> is NULL or <b>sz</b> is zero, nothing happens.
- *
- * This function is preferable to memset, since many compilers will happily
- * optimize out memset() when they can convince themselves that the data being
- * cleared will never be read.
- *
- * Right now, our convention is to use this function when we are wiping data
- * that's about to become inaccessible, such as stack buffers that are about
- * to go out of scope or structures that are about to get freed.  (In
- * practice, it appears that the compilers we're currently using will optimize
- * out the memset()s for stack-allocated buffers, but not those for
- * about-to-be-freed structures. That could change, though, so we're being
- * wary.)  If there are live reads for the data, then you can just use
- * memset().
- */
-void
-memwipe(void *mem, uint8_t byte, size_t sz)
-{
-  if (sz == 0) {
-    return;
-  }
-  /* If sz is nonzero, then mem must not be NULL. */
-  tor_assert(mem != NULL);
-
-  /* Data this large is likely to be an underflow. */
-  tor_assert(sz < SIZE_T_CEILING);
-
-  /* Because whole-program-optimization exists, we may not be able to just
-   * have this function call "memset".  A smart compiler could inline it, then
-   * eliminate dead memsets, and declare itself to be clever. */
-
-#if defined(SecureZeroMemory) || defined(HAVE_SECUREZEROMEMORY)
-  /* Here's what you do on windows. */
-  SecureZeroMemory(mem,sz);
-#elif defined(HAVE_RTLSECUREZEROMEMORY)
-  RtlSecureZeroMemory(mem,sz);
-#elif defined(HAVE_EXPLICIT_BZERO)
-  /* The BSDs provide this. */
-  explicit_bzero(mem, sz);
-#elif defined(HAVE_MEMSET_S)
-  /* This is in the C99 standard. */
-  memset_s(mem, sz, 0, sz);
-#else
-  /* This is a slow and ugly function from OpenSSL that fills 'mem' with junk
-   * based on the pointer value, then uses that junk to update a global
-   * variable.  It's an elaborate ruse to trick the compiler into not
-   * optimizing out the "wipe this memory" code.  Read it if you like zany
-   * programming tricks! In later versions of Tor, we should look for better
-   * not-optimized-out memory wiping stuff...
-   *
-   * ...or maybe not.  In practice, there are pure-asm implementations of
-   * OPENSSL_cleanse() on most platforms, which ought to do the job.
-   **/
-
-  OPENSSL_cleanse(mem, sz);
-#endif
-
-  /* Just in case some caller of memwipe() is relying on getting a buffer
-   * filled with a particular value, fill the buffer.
-   *
-   * If this function gets inlined, this memset might get eliminated, but
-   * that's okay: We only care about this particular memset in the case where
-   * the caller should have been using memset(), and the memset() wouldn't get
-   * eliminated.  In other words, this is here so that we won't break anything
-   * if somebody accidentally calls memwipe() instead of memset().
-   **/
-  memset(mem, byte, sz);
-}
-
-#ifndef OPENSSL_THREADS
-#error OpenSSL has been built without thread support. Tor requires an \
- OpenSSL library with thread support enabled.
-#endif
-
-#ifndef NEW_THREAD_API
-/** Helper: OpenSSL uses this callback to manipulate mutexes. */
-static void
-openssl_locking_cb_(int mode, int n, const char *file, int line)
-{
-  (void)file;
-  (void)line;
-  if (!openssl_mutexes_)
-    /* This is not a really good fix for the
-     * "release-freed-lock-from-separate-thread-on-shutdown" problem, but
-     * it can't hurt. */
-    return;
-  if (mode & CRYPTO_LOCK)
-    tor_mutex_acquire(openssl_mutexes_[n]);
-  else
-    tor_mutex_release(openssl_mutexes_[n]);
-}
-
-static void
-tor_set_openssl_thread_id(CRYPTO_THREADID *threadid)
-{
-  CRYPTO_THREADID_set_numeric(threadid, tor_get_thread_id());
-}
-#endif
-
-#if 0
-/* This code is disabled, because OpenSSL never actually uses these callbacks.
- */
-
-/** OpenSSL helper type: wraps a Tor mutex so that OpenSSL can use it
- * as a lock. */
-struct CRYPTO_dynlock_value {
-  tor_mutex_t *lock;
-};
-
-/** OpenSSL callback function to allocate a lock: see CRYPTO_set_dynlock_*
- * documentation in OpenSSL's docs for more info. */
-static struct CRYPTO_dynlock_value *
-openssl_dynlock_create_cb_(const char *file, int line)
-{
-  struct CRYPTO_dynlock_value *v;
-  (void)file;
-  (void)line;
-  v = tor_malloc(sizeof(struct CRYPTO_dynlock_value));
-  v->lock = tor_mutex_new();
-  return v;
-}
-
-/** OpenSSL callback function to acquire or release a lock: see
- * CRYPTO_set_dynlock_* documentation in OpenSSL's docs for more info. */
-static void
-openssl_dynlock_lock_cb_(int mode, struct CRYPTO_dynlock_value *v,
-                         const char *file, int line)
-{
-  (void)file;
-  (void)line;
-  if (mode & CRYPTO_LOCK)
-    tor_mutex_acquire(v->lock);
-  else
-    tor_mutex_release(v->lock);
-}
-
-/** OpenSSL callback function to free a lock: see CRYPTO_set_dynlock_*
- * documentation in OpenSSL's docs for more info. */
-static void
-openssl_dynlock_destroy_cb_(struct CRYPTO_dynlock_value *v,
-                            const char *file, int line)
-{
-  (void)file;
-  (void)line;
-  tor_mutex_free(v->lock);
-  tor_free(v);
-}
-#endif
-
-/** @{ */
-/** Helper: Construct mutexes, and set callbacks to help OpenSSL handle being
- * multithreaded. Returns 0. */
-static int
-setup_openssl_threading(void)
-{
-#ifndef NEW_THREAD_API
-  int i;
-  int n = CRYPTO_num_locks();
-  n_openssl_mutexes_ = n;
-  openssl_mutexes_ = tor_calloc(n, sizeof(tor_mutex_t *));
-  for (i=0; i < n; ++i)
-    openssl_mutexes_[i] = tor_mutex_new();
-  CRYPTO_set_locking_callback(openssl_locking_cb_);
-  CRYPTO_THREADID_set_callback(tor_set_openssl_thread_id);
-#endif
-#if 0
-  CRYPTO_set_dynlock_create_callback(openssl_dynlock_create_cb_);
-  CRYPTO_set_dynlock_lock_callback(openssl_dynlock_lock_cb_);
-  CRYPTO_set_dynlock_destroy_callback(openssl_dynlock_destroy_cb_);
-#endif
-  return 0;
-}
-
-/** Uninitialize the crypto library. Return 0 on success. Does not detect
- * failure.
- */
-int
-crypto_global_cleanup(void)
-{
-  EVP_cleanup();
-#ifndef NEW_THREAD_API
-  ERR_remove_thread_state(NULL);
-#endif
-  ERR_free_strings();
-
-  if (dh_param_p)
-    BN_clear_free(dh_param_p);
-  if (dh_param_p_tls)
-    BN_clear_free(dh_param_p_tls);
-  if (dh_param_g)
-    BN_clear_free(dh_param_g);
-
-#ifndef DISABLE_ENGINES
-  ENGINE_cleanup();
-#endif
-
-  CONF_modules_unload(1);
-  CRYPTO_cleanup_all_ex_data();
-
-#ifndef NEW_THREAD_API
-  if (n_openssl_mutexes_) {
-    int n = n_openssl_mutexes_;
-    tor_mutex_t **ms = openssl_mutexes_;
-    int i;
-    openssl_mutexes_ = NULL;
-    n_openssl_mutexes_ = 0;
-    for (i=0;i<n;++i) {
-      tor_mutex_free(ms[i]);
-    }
-    tor_free(ms);
-  }
-#endif
-
-  tor_free(crypto_openssl_version_str);
-  tor_free(crypto_openssl_header_version_str);
-  return 0;
-}
-
-/** @} */
-
diff --git a/src/common/crypto.h b/src/common/crypto.h
deleted file mode 100644
index 116e0a62fd..0000000000
--- a/src/common/crypto.h
+++ /dev/null
@@ -1,340 +0,0 @@
-/* Copyright (c) 2001, Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file crypto.h
- *
- * \brief Headers for crypto.c
- **/
-
-#ifndef TOR_CRYPTO_H
-#define TOR_CRYPTO_H
-
-#include "orconfig.h"
-
-#include <stdio.h>
-#include "torint.h"
-#include "testsupport.h"
-#include "compat.h"
-
-/*
-  Macro to create an arbitrary OpenSSL version number as used by
-  OPENSSL_VERSION_NUMBER or SSLeay(), since the actual numbers are a bit hard
-  to read.
-
-  Don't use this directly, instead use one of the other OPENSSL_V macros
-  below.
-
-  The format is: 4 bits major, 8 bits minor, 8 bits fix, 8 bits patch, 4 bit
-  status.
- */
-#define OPENSSL_VER(a,b,c,d,e)                                \
-  (((a)<<28) |                                                \
-   ((b)<<20) |                                                \
-   ((c)<<12) |                                                \
-   ((d)<< 4) |                                                \
-    (e))
-/** An openssl release number.  For example, OPENSSL_V(0,9,8,'j') is the
- * version for the released version of 0.9.8j */
-#define OPENSSL_V(a,b,c,d) \
-  OPENSSL_VER((a),(b),(c),(d)-'a'+1,0xf)
-/** An openssl release number for the first release in the series.  For
- * example, OPENSSL_V_NOPATCH(1,0,0) is the first released version of OpenSSL
- * 1.0.0. */
-#define OPENSSL_V_NOPATCH(a,b,c) \
-  OPENSSL_VER((a),(b),(c),0,0xf)
-/** The first version that would occur for any alpha or beta in an openssl
- * series. For example, OPENSSL_V_SERIES(0,9,8) is greater than any released
- * 0.9.7, and less than any released 0.9.8. */
-#define OPENSSL_V_SERIES(a,b,c) \
-  OPENSSL_VER((a),(b),(c),0,0)
-
-/** Length of the output of our message digest. */
-#define DIGEST_LEN 20
-/** Length of the output of our second (improved) message digests.  (For now
- * this is just sha256, but it could be any other 256-bit digest.) */
-#define DIGEST256_LEN 32
-/** Length of the output of our 64-bit optimized message digests (SHA512). */
-#define DIGEST512_LEN 64
-/** Length of our symmetric cipher's keys. */
-#define CIPHER_KEY_LEN 16
-/** Length of our symmetric cipher's IV. */
-#define CIPHER_IV_LEN 16
-/** Length of our public keys. */
-#define PK_BYTES (1024/8)
-/** Length of our DH keys. */
-#define DH_BYTES (1024/8)
-
-/** Length of a sha1 message digest when encoded in base64 with trailing =
- * signs removed. */
-#define BASE64_DIGEST_LEN 27
-/** Length of a sha256 message digest when encoded in base64 with trailing =
- * signs removed. */
-#define BASE64_DIGEST256_LEN 43
-/** Length of a sha512 message digest when encoded in base64 with trailing =
- * signs removed. */
-#define BASE64_DIGEST512_LEN 86
-
-/** Constant used to indicate OAEP padding for public-key encryption */
-#define PK_PKCS1_OAEP_PADDING 60002
-
-/** Number of bytes added for PKCS1-OAEP padding. */
-#define PKCS1_OAEP_PADDING_OVERHEAD 42
-
-/** Length of encoded public key fingerprints, including space; but not
- * including terminating NUL. */
-#define FINGERPRINT_LEN 49
-/** Length of hex encoding of SHA1 digest, not including final NUL. */
-#define HEX_DIGEST_LEN 40
-/** Length of hex encoding of SHA256 digest, not including final NUL. */
-#define HEX_DIGEST256_LEN 64
-/** Length of hex encoding of SHA512 digest, not including final NUL. */
-#define HEX_DIGEST512_LEN 128
-
-typedef enum {
-  DIGEST_SHA1 = 0,
-  DIGEST_SHA256 = 1,
-  DIGEST_SHA512 = 2,
-  DIGEST_SHA3_256 = 3,
-  DIGEST_SHA3_512 = 4,
-} digest_algorithm_t;
-#define  N_DIGEST_ALGORITHMS (DIGEST_SHA3_512+1)
-#define  N_COMMON_DIGEST_ALGORITHMS (DIGEST_SHA256+1)
-
-/** A set of all the digests we commonly compute, taken on a single
- * string.  Any digests that are shorter than 512 bits are right-padded
- * with 0 bits.
- *
- * Note that this representation wastes 44 bytes for the SHA1 case, so
- * don't use it for anything where we need to allocate a whole bunch at
- * once.
- **/
-typedef struct {
-  char d[N_COMMON_DIGEST_ALGORITHMS][DIGEST256_LEN];
-} common_digests_t;
-
-typedef struct crypto_pk_t crypto_pk_t;
-typedef struct aes_cnt_cipher crypto_cipher_t;
-typedef struct crypto_digest_t crypto_digest_t;
-typedef struct crypto_xof_t crypto_xof_t;
-typedef struct crypto_dh_t crypto_dh_t;
-
-/* global state */
-const char * crypto_openssl_get_version_str(void);
-const char * crypto_openssl_get_header_version_str(void);
-int crypto_early_init(void) ATTR_WUR;
-int crypto_global_init(int hardwareAccel,
-                       const char *accelName,
-                       const char *accelPath) ATTR_WUR;
-void crypto_thread_cleanup(void);
-int crypto_global_cleanup(void);
-
-/* environment setup */
-MOCK_DECL(crypto_pk_t *,crypto_pk_new,(void));
-void crypto_pk_free(crypto_pk_t *env);
-
-void crypto_set_tls_dh_prime(void);
-crypto_cipher_t *crypto_cipher_new(const char *key);
-crypto_cipher_t *crypto_cipher_new_with_bits(const char *key, int bits);
-crypto_cipher_t *crypto_cipher_new_with_iv(const char *key, const char *iv);
-crypto_cipher_t *crypto_cipher_new_with_iv_and_bits(const uint8_t *key,
-                                                    const uint8_t *iv,
-                                                    int bits);
-void crypto_cipher_free(crypto_cipher_t *env);
-
-/* public key crypto */
-MOCK_DECL(int, crypto_pk_generate_key_with_bits,(crypto_pk_t *env, int bits));
-#define crypto_pk_generate_key(env)                     \
-  crypto_pk_generate_key_with_bits((env), (PK_BYTES*8))
-
-int crypto_pk_read_private_key_from_filename(crypto_pk_t *env,
-                                             const char *keyfile);
-int crypto_pk_write_public_key_to_string(crypto_pk_t *env,
-                                         char **dest, size_t *len);
-int crypto_pk_write_private_key_to_string(crypto_pk_t *env,
-                                          char **dest, size_t *len);
-int crypto_pk_read_public_key_from_string(crypto_pk_t *env,
-                                          const char *src, size_t len);
-int crypto_pk_read_private_key_from_string(crypto_pk_t *env,
-                                           const char *s, ssize_t len);
-int crypto_pk_write_private_key_to_filename(crypto_pk_t *env,
-                                            const char *fname);
-
-int crypto_pk_check_key(crypto_pk_t *env);
-int crypto_pk_cmp_keys(const crypto_pk_t *a, const crypto_pk_t *b);
-int crypto_pk_eq_keys(const crypto_pk_t *a, const crypto_pk_t *b);
-size_t crypto_pk_keysize(const crypto_pk_t *env);
-int crypto_pk_num_bits(crypto_pk_t *env);
-crypto_pk_t *crypto_pk_dup_key(crypto_pk_t *orig);
-crypto_pk_t *crypto_pk_copy_full(crypto_pk_t *orig);
-int crypto_pk_key_is_private(const crypto_pk_t *key);
-int crypto_pk_public_exponent_ok(crypto_pk_t *env);
-
-int crypto_pk_public_encrypt(crypto_pk_t *env, char *to, size_t tolen,
-                             const char *from, size_t fromlen, int padding);
-int crypto_pk_private_decrypt(crypto_pk_t *env, char *to, size_t tolen,
-                              const char *from, size_t fromlen,
-                              int padding, int warnOnFailure);
-int crypto_pk_public_checksig(const crypto_pk_t *env, char *to, size_t tolen,
-                              const char *from, size_t fromlen);
-int crypto_pk_public_checksig_digest(crypto_pk_t *env, const char *data,
-                               size_t datalen, const char *sig, size_t siglen);
-int crypto_pk_private_sign(const crypto_pk_t *env, char *to, size_t tolen,
-                           const char *from, size_t fromlen);
-int crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen,
-                                  const char *from, size_t fromlen);
-int crypto_pk_public_hybrid_encrypt(crypto_pk_t *env, char *to,
-                                    size_t tolen,
-                                    const char *from, size_t fromlen,
-                                    int padding, int force);
-int crypto_pk_private_hybrid_decrypt(crypto_pk_t *env, char *to,
-                                     size_t tolen,
-                                     const char *from, size_t fromlen,
-                                     int padding, int warnOnFailure);
-
-int crypto_pk_asn1_encode(crypto_pk_t *pk, char *dest, size_t dest_len);
-crypto_pk_t *crypto_pk_asn1_decode(const char *str, size_t len);
-int crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out);
-int crypto_pk_get_common_digests(crypto_pk_t *pk,
-                                 common_digests_t *digests_out);
-int crypto_pk_get_fingerprint(crypto_pk_t *pk, char *fp_out,int add_space);
-int crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out);
-
-int crypto_pk_base64_encode(const crypto_pk_t *pk, char **priv_out);
-crypto_pk_t *crypto_pk_base64_decode(const char *str, size_t len);
-
-/* symmetric crypto */
-const char *crypto_cipher_get_key(crypto_cipher_t *env);
-
-int crypto_cipher_encrypt(crypto_cipher_t *env, char *to,
-                          const char *from, size_t fromlen);
-int crypto_cipher_decrypt(crypto_cipher_t *env, char *to,
-                          const char *from, size_t fromlen);
-void crypto_cipher_crypt_inplace(crypto_cipher_t *env, char *d, size_t len);
-
-int crypto_cipher_encrypt_with_iv(const char *key,
-                                  char *to, size_t tolen,
-                                  const char *from, size_t fromlen);
-int crypto_cipher_decrypt_with_iv(const char *key,
-                                  char *to, size_t tolen,
-                                  const char *from, size_t fromlen);
-
-/* SHA-1 and other digests. */
-int crypto_digest(char *digest, const char *m, size_t len);
-int crypto_digest256(char *digest, const char *m, size_t len,
-                     digest_algorithm_t algorithm);
-int crypto_digest512(char *digest, const char *m, size_t len,
-                     digest_algorithm_t algorithm);
-int crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len);
-struct smartlist_t;
-void crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
-                                    const char *prepend,
-                                    const struct smartlist_t *lst,
-                                    const char *append,
-                                    digest_algorithm_t alg);
-void crypto_digest_smartlist(char *digest_out, size_t len_out,
-                             const struct smartlist_t *lst, const char *append,
-                             digest_algorithm_t alg);
-const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg);
-size_t crypto_digest_algorithm_get_length(digest_algorithm_t alg);
-int crypto_digest_algorithm_parse_name(const char *name);
-crypto_digest_t *crypto_digest_new(void);
-crypto_digest_t *crypto_digest256_new(digest_algorithm_t algorithm);
-crypto_digest_t *crypto_digest512_new(digest_algorithm_t algorithm);
-void crypto_digest_free(crypto_digest_t *digest);
-void crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
-                             size_t len);
-void crypto_digest_get_digest(crypto_digest_t *digest,
-                              char *out, size_t out_len);
-crypto_digest_t *crypto_digest_dup(const crypto_digest_t *digest);
-void crypto_digest_assign(crypto_digest_t *into,
-                          const crypto_digest_t *from);
-void crypto_hmac_sha256(char *hmac_out,
-                        const char *key, size_t key_len,
-                        const char *msg, size_t msg_len);
-crypto_xof_t *crypto_xof_new(void);
-void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len);
-void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len);
-void crypto_xof_free(crypto_xof_t *xof);
-
-/* Key negotiation */
-#define DH_TYPE_CIRCUIT 1
-#define DH_TYPE_REND 2
-#define DH_TYPE_TLS 3
-crypto_dh_t *crypto_dh_new(int dh_type);
-crypto_dh_t *crypto_dh_dup(const crypto_dh_t *dh);
-int crypto_dh_get_bytes(crypto_dh_t *dh);
-int crypto_dh_generate_public(crypto_dh_t *dh);
-int crypto_dh_get_public(crypto_dh_t *dh, char *pubkey_out,
-                         size_t pubkey_out_len);
-ssize_t crypto_dh_compute_secret(int severity, crypto_dh_t *dh,
-                             const char *pubkey, size_t pubkey_len,
-                             char *secret_out, size_t secret_out_len);
-void crypto_dh_free(crypto_dh_t *dh);
-
-int crypto_expand_key_material_TAP(const uint8_t *key_in,
-                                   size_t key_in_len,
-                                   uint8_t *key_out, size_t key_out_len);
-int crypto_expand_key_material_rfc5869_sha256(
-                                    const uint8_t *key_in, size_t key_in_len,
-                                    const uint8_t *salt_in, size_t salt_in_len,
-                                    const uint8_t *info_in, size_t info_in_len,
-                                    uint8_t *key_out, size_t key_out_len);
-
-/* random numbers */
-int crypto_seed_rng(void) ATTR_WUR;
-MOCK_DECL(void,crypto_rand,(char *to, size_t n));
-void crypto_rand_unmocked(char *to, size_t n);
-void crypto_strongest_rand(uint8_t *out, size_t out_len);
-int crypto_rand_int(unsigned int max);
-int crypto_rand_int_range(unsigned int min, unsigned int max);
-uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max);
-time_t crypto_rand_time_range(time_t min, time_t max);
-uint64_t crypto_rand_uint64(uint64_t max);
-double crypto_rand_double(void);
-struct tor_weak_rng_t;
-void crypto_seed_weak_rng(struct tor_weak_rng_t *rng);
-int crypto_init_siphash_key(void);
-
-char *crypto_random_hostname(int min_rand_len, int max_rand_len,
-                             const char *prefix, const char *suffix);
-
-struct smartlist_t;
-void *smartlist_choose(const struct smartlist_t *sl);
-void smartlist_shuffle(struct smartlist_t *sl);
-
-/** OpenSSL-based utility functions. */
-void memwipe(void *mem, uint8_t byte, size_t sz);
-
-/* Prototypes for private functions only used by tortls.c, crypto.c, and the
- * unit tests. */
-struct rsa_st;
-struct evp_pkey_st;
-struct dh_st;
-struct rsa_st *crypto_pk_get_rsa_(crypto_pk_t *env);
-crypto_pk_t *crypto_new_pk_from_rsa_(struct rsa_st *rsa);
-MOCK_DECL(struct evp_pkey_st *, crypto_pk_get_evp_pkey_,(crypto_pk_t *env,
-                                                         int private));
-struct dh_st *crypto_dh_get_dh_(crypto_dh_t *dh);
-
-void crypto_add_spaces_to_fp(char *out, size_t outlen, const char *in);
-
-#ifdef CRYPTO_PRIVATE
-STATIC int crypto_force_rand_ssleay(void);
-STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len);
-
-#ifdef TOR_UNIT_TESTS
-extern int break_strongest_rng_syscall;
-extern int break_strongest_rng_fallback;
-#endif
-#endif
-
-#ifdef TOR_UNIT_TESTS
-void crypto_pk_assign_(crypto_pk_t *dest, const crypto_pk_t *src);
-#endif
-
-#endif
-
diff --git a/src/common/include.am b/src/common/include.am
deleted file mode 100644
index cb307e9d5f..0000000000
--- a/src/common/include.am
+++ /dev/null
@@ -1,175 +0,0 @@
-
-noinst_LIBRARIES += \
-	src/common/libor.a \
-	src/common/libor-ctime.a \
-	src/common/libor-crypto.a \
-	src/common/libor-event.a
-
-if UNITTESTS_ENABLED
-noinst_LIBRARIES += \
-	src/common/libor-testing.a \
-	src/common/libor-ctime-testing.a \
-	src/common/libor-crypto-testing.a \
-	src/common/libor-event-testing.a
-endif
-
-EXTRA_DIST += src/common/Makefile.nmake
-
-#CFLAGS  = -Wall -Wpointer-arith -O2
-AM_CPPFLAGS += -I$(srcdir)/src/common -Isrc/common -I$(srcdir)/src/ext/trunnel -I$(srcdir)/src/trunnel
-
-if USE_OPENBSD_MALLOC
-libor_extra_source=src/ext/OpenBSD_malloc_Linux.c
-else
-libor_extra_source=
-endif
-
-src_common_libcurve25519_donna_a_CFLAGS=
-
-if BUILD_CURVE25519_DONNA
-src_common_libcurve25519_donna_a_SOURCES=\
-	src/ext/curve25519_donna/curve25519-donna.c
-# See bug 13538 -- this code is known to have signed overflow issues.
-src_common_libcurve25519_donna_a_CFLAGS+=\
-	@F_OMIT_FRAME_POINTER@ @CFLAGS_CONSTTIME@
-noinst_LIBRARIES+=src/common/libcurve25519_donna.a
-LIBDONNA=src/common/libcurve25519_donna.a
-else
-if BUILD_CURVE25519_DONNA_C64
-src_common_libcurve25519_donna_a_CFLAGS+=@CFLAGS_CONSTTIME@
-src_common_libcurve25519_donna_a_SOURCES=\
-	src/ext/curve25519_donna/curve25519-donna-c64.c
-noinst_LIBRARIES+=src/common/libcurve25519_donna.a
-LIBDONNA=src/common/libcurve25519_donna.a
-else
-LIBDONNA=
-endif
-endif
-
-LIBDONNA += $(LIBED25519_REF10)
-LIBDONNA += $(LIBED25519_DONNA)
-
-if THREADS_PTHREADS
-threads_impl_source=src/common/compat_pthreads.c
-endif
-if THREADS_WIN32
-threads_impl_source=src/common/compat_winthreads.c
-endif
-
-if BUILD_READPASSPHRASE_C
-readpassphrase_source=src/ext/readpassphrase.c
-else
-readpassphrase_source=
-endif
-
-if ADD_MULODI4
-mulodi4_source=src/ext/mulodi/mulodi4.c
-else
-mulodi4_source=
-endif
-
-LIBOR_CTIME_A_SRC = \
-   $(mulodi4_source) \
-   src/ext/csiphash.c   \
-   src/common/di_ops.c
-
-src_common_libor_ctime_a_SOURCES = $(LIBOR_CTIME_A_SRC)
-src_common_libor_ctime_testing_a_SOURCES = $(LIBOR_CTIME_A_SRC)
-src_common_libor_ctime_a_CFLAGS = @CFLAGS_CONSTTIME@
-src_common_libor_ctime_testing_a_CFLAGS = @CFLAGS_CONSTTIME@ $(TEST_CFLAGS)
-
-LIBOR_A_SRC = \
-  src/common/address.c					\
-  src/common/address_set.c				\
-  src/common/backtrace.c				\
-  src/common/compat.c					\
-  src/common/compat_threads.c				\
-  src/common/compat_time.c				\
-  src/common/container.c				\
-  src/common/log.c					\
-  src/common/memarea.c					\
-  src/common/pubsub.c					\
-  src/common/util.c					\
-  src/common/util_bug.c					\
-  src/common/util_format.c				\
-  src/common/util_process.c				\
-  src/common/sandbox.c					\
-  src/common/workqueue.c				\
-  $(libor_extra_source)					\
-  $(threads_impl_source)				\
-  $(readpassphrase_source)
-
-src/common/src_common_libor_testing_a-log.$(OBJEXT) \
-  src/common/log.$(OBJEXT): micro-revision.i
-
-LIBOR_CRYPTO_A_SRC = \
-  src/common/aes.c		\
-  src/common/crypto.c		\
-  src/common/crypto_pwbox.c     \
-  src/common/crypto_s2k.c	\
-  src/common/crypto_format.c	\
-  src/common/torgzip.c		\
-  src/common/tortls.c		\
-  src/common/crypto_curve25519.c \
-  src/common/crypto_ed25519.c
-
-LIBOR_EVENT_A_SRC = \
-	src/common/compat_libevent.c \
-	src/common/procmon.c         \
-	src/common/timers.c          \
-	src/ext/timeouts/timeout.c
-
-src_common_libor_a_SOURCES = $(LIBOR_A_SRC)
-src_common_libor_crypto_a_SOURCES = $(LIBOR_CRYPTO_A_SRC)
-src_common_libor_event_a_SOURCES = $(LIBOR_EVENT_A_SRC)
-
-src_common_libor_testing_a_SOURCES = $(LIBOR_A_SRC)
-src_common_libor_crypto_testing_a_SOURCES = $(LIBOR_CRYPTO_A_SRC)
-src_common_libor_event_testing_a_SOURCES = $(LIBOR_EVENT_A_SRC)
-
-src_common_libor_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
-src_common_libor_crypto_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
-src_common_libor_event_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
-src_common_libor_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
-src_common_libor_crypto_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
-src_common_libor_event_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
-
-COMMONHEADERS = \
-  src/common/address.h				\
-  src/common/address_set.h			\
-  src/common/backtrace.h			\
-  src/common/aes.h				\
-  src/common/ciphers.inc			\
-  src/common/compat.h				\
-  src/common/compat_libevent.h			\
-  src/common/compat_openssl.h			\
-  src/common/compat_threads.h			\
-  src/common/compat_time.h			\
-  src/common/container.h			\
-  src/common/crypto.h				\
-  src/common/crypto_curve25519.h		\
-  src/common/crypto_ed25519.h			\
-  src/common/crypto_format.h			\
-  src/common/crypto_pwbox.h			\
-  src/common/crypto_s2k.h			\
-  src/common/di_ops.h				\
-  src/common/handles.h				\
-  src/common/memarea.h				\
-  src/common/linux_syscalls.inc			\
-  src/common/procmon.h				\
-  src/common/pubsub.h				\
-  src/common/sandbox.h				\
-  src/common/testsupport.h			\
-  src/common/timers.h				\
-  src/common/torgzip.h				\
-  src/common/torint.h				\
-  src/common/torlog.h				\
-  src/common/tortls.h				\
-  src/common/util.h				\
-  src/common/util_bug.h				\
-  src/common/util_format.h			\
-  src/common/util_process.h			\
-  src/common/workqueue.h
-
-noinst_HEADERS+= $(COMMONHEADERS)
-
diff --git a/src/common/pubsub.c b/src/common/pubsub.c
deleted file mode 100644
index b3faf40e00..0000000000
--- a/src/common/pubsub.c
+++ /dev/null
@@ -1,129 +0,0 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file pubsub.c
- *
- * \brief DOCDOC
- */
-
-#include "orconfig.h"
-#include "pubsub.h"
-#include "container.h"
-
-/** Helper: insert <b>s</b> into <b>topic's</b> list of subscribers, keeping
- * them sorted in priority order. */
-static void
-subscriber_insert(pubsub_topic_t *topic, pubsub_subscriber_t *s)
-{
-  int i;
-  smartlist_t *sl = topic->subscribers;
-  for (i = 0; i < smartlist_len(sl); ++i) {
-    pubsub_subscriber_t *other = smartlist_get(sl, i);
-    if (s->priority < other->priority) {
-      break;
-    }
-  }
-  smartlist_insert(sl, i, s);
-}
-
-/**
- * Add a new subscriber to <b>topic</b>, where (when an event is triggered),
- * we'll notify the function <b>fn</b> by passing it <b>subscriber_data</b>.
- * Return a handle to the subscribe which can later be passed to
- * pubsub_unsubscribe_().
- *
- * Functions are called in priority order, from lowest to highest.
- *
- * See pubsub.h for <b>subscribe_flags</b>.
- */
-const pubsub_subscriber_t *
-pubsub_subscribe_(pubsub_topic_t *topic,
-                  pubsub_subscriber_fn_t fn,
-                  void *subscriber_data,
-                  unsigned subscribe_flags,
-                  unsigned priority)
-{
-  tor_assert(! topic->locked);
-  if (subscribe_flags & SUBSCRIBE_ATSTART) {
-    tor_assert(topic->n_events_fired == 0);
-  }
-  pubsub_subscriber_t *r = tor_malloc_zero(sizeof(*r));
-  r->priority = priority;
-  r->subscriber_flags = subscribe_flags;
-  r->fn = fn;
-  r->subscriber_data = subscriber_data;
-  if (topic->subscribers == NULL) {
-    topic->subscribers = smartlist_new();
-  }
-  subscriber_insert(topic, r);
-  return r;
-}
-
-/**
- * Remove the subscriber <b>s</b> from <b>topic</b>.  After calling this
- * function, <b>s</b> may no longer be used.
- */
-int
-pubsub_unsubscribe_(pubsub_topic_t *topic,
-                    const pubsub_subscriber_t *s)
-{
-  tor_assert(! topic->locked);
-  smartlist_t *sl = topic->subscribers;
-  if (sl == NULL)
-    return -1;
-  int i = smartlist_pos(sl, s);
-  if (i == -1)
-    return -1;
-  pubsub_subscriber_t *tmp = smartlist_get(sl, i);
-  tor_assert(tmp == s);
-  smartlist_del_keeporder(sl, i);
-  tor_free(tmp);
-  return 0;
-}
-
-/**
- * For every subscriber s in <b>topic</b>, invoke notify_fn on s and
- * event_data.  Return 0 if there were no nonzero return values, and -1 if
- * there were any.
- */
-int
-pubsub_notify_(pubsub_topic_t *topic, pubsub_notify_fn_t notify_fn,
-               void *event_data, unsigned notify_flags)
-{
-  tor_assert(! topic->locked);
-  (void) notify_flags;
-  smartlist_t *sl = topic->subscribers;
-  int n_bad = 0;
-  ++topic->n_events_fired;
-  if (sl == NULL)
-    return -1;
-  topic->locked = 1;
-  SMARTLIST_FOREACH_BEGIN(sl, pubsub_subscriber_t *, s) {
-    int r = notify_fn(s, event_data);
-    if (r != 0)
-      ++n_bad;
-  } SMARTLIST_FOREACH_END(s);
-  topic->locked = 0;
-  return (n_bad == 0) ? 0 : -1;
-}
-
-/**
- * Release all storage held by <b>topic</b>.
- */
-void
-pubsub_clear_(pubsub_topic_t *topic)
-{
-  tor_assert(! topic->locked);
-
-  smartlist_t *sl = topic->subscribers;
-  if (sl == NULL)
-    return;
-  SMARTLIST_FOREACH_BEGIN(sl, pubsub_subscriber_t *, s) {
-    tor_free(s);
-  } SMARTLIST_FOREACH_END(s);
-  smartlist_free(sl);
-  topic->subscribers = NULL;
-  topic->n_events_fired = 0;
-}
-
diff --git a/src/common/pubsub.h b/src/common/pubsub.h
deleted file mode 100644
index bbb4f02a42..0000000000
--- a/src/common/pubsub.h
+++ /dev/null
@@ -1,179 +0,0 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file pubsub.h
- * \brief Macros to implement publish/subscribe abstractions.
- *
- * To use these macros, call DECLARE_PUBSUB_TOPIC() with an identifier to use
- * as your topic.  Below, I'm going to assume you say DECLARE_PUBSUB_TOPIC(T).
- *
- * Doing this will declare the following types:
- *   typedef struct T_event_data_t T_event_data_t; // you define this struct
- *   typedef struct T_subscriber_data_t T_subscriber_data_t; // this one too.
- *   typedef struct T_subscriber_t T_subscriber_t; // opaque
- *   typedef int (*T_subscriber_fn_t)(T_event_data_t*, T_subscriber_data_t*);
- *
- * and it will declare the following functions:
- *     const T_subscriber_t *T_subscribe(T_subscriber_fn_t,
- *                                       T_subscriber_data_t *,
- *                                       unsigned flags,
- *                                       unsigned priority);
- *     int T_unsubscribe(const T_subscriber_t *)
- *
- * Elsewhere you can say DECLARE_NOTIFY_PUBSUB_TOPIC(static, T), which
- * declares:
- *
- *    static int T_notify(T_event_data_t *, unsigned notify_flags);
- *    static void T_clear(void);
- *
- * And in some C file, you would define these functions with:
- *    IMPLEMENT_PUBSUB_TOPIC(static, T).
- *
- * The implementations will be small typesafe wrappers over generic versions
- * of the above functions.
- *
- * To use the typesafe functions, you add any number of subscribers with
- * T_subscribe().  Each has an associated function pointer, data pointer,
- * and priority. Later, you can invoke T_notify() to declare that the
- * event has occurred. Each of the subscribers will be invoked once.
- **/
-
-#ifndef TOR_PUBSUB_H
-#define TOR_PUBSUB_H
-
-#include "torint.h"
-
-/**
- * Flag for T_subscribe: die with an assertion failure if the event
- * have ever been published before.  Used when a subscriber must absolutely
- * never have missed an event.
- */
-#define SUBSCRIBE_ATSTART (1u<<0)
-
-#define DECLARE_PUBSUB_STRUCT_TYPES(name)                               \
-  /* You define this type. */                                           \
-  typedef struct name ## _event_data_t name ## _event_data_t;           \
-  /* You define this type. */                                           \
-  typedef struct name ## _subscriber_data_t name ## _subscriber_data_t;
-
-#define DECLARE_PUBSUB_TOPIC(name)                                      \
-  /* This type is opaque. */                                            \
-  typedef struct name ## _subscriber_t name ## _subscriber_t;           \
-  /* You declare functions matching this type. */                       \
-  typedef int (*name ## _subscriber_fn_t)(                              \
-                                    name ## _event_data_t *data,        \
-                                    name ## _subscriber_data_t *extra); \
-  /* Call this function to subscribe to a topic. */                     \
-  const name ## _subscriber_t *name ## _subscribe(                      \
-                         name##_subscriber_fn_t subscriber,             \
-                         name##_subscriber_data_t *extra_data,          \
-                         unsigned flags,                                \
-                         unsigned priority);                            \
-  /* Call this function to unsubscribe from a topic. */                 \
-  int name ## _unsubscribe(const name##_subscriber_t *s);
-
-#define DECLARE_NOTIFY_PUBSUB_TOPIC(linkage, name)                          \
-  /* Call this function to notify all subscribers. Flags not yet used. */   \
-  linkage int name ## _notify(name ## _event_data_t *data, unsigned flags); \
-  /* Call this function to release storage held by the topic. */            \
-  linkage void name ## _clear(void);
-
-/**
- * Type used to hold a generic function for a subscriber.
- *
- * [Yes, it is safe to cast to this, so long as we cast back to the original
- * type before calling.  From C99: "A pointer to a function of one type may be
- * converted to a pointer to a function of another type and back again; the
- * result shall compare equal to the original pointer."]
-*/
-typedef int (*pubsub_subscriber_fn_t)(void *, void *);
-
-/**
- * Helper type to implement pubsub abstraction. Don't use this directly.
- * It represents a subscriber.
- */
-typedef struct pubsub_subscriber_t {
-  /** Function to invoke when the event triggers. */
-  pubsub_subscriber_fn_t fn;
-  /** Data associated with this subscriber. */
-  void *subscriber_data;
-  /** Priority for this subscriber. Low priorities happen first. */
-  unsigned priority;
-  /** Flags set on this subscriber. Not yet used.*/
-  unsigned subscriber_flags;
-} pubsub_subscriber_t;
-
-/**
- * Helper type to implement pubsub abstraction. Don't use this directly.
- * It represents a topic, and keeps a record of subscribers.
- */
-typedef struct pubsub_topic_t {
-  /** List of subscribers to this topic. May be NULL. */
-  struct smartlist_t *subscribers;
-  /** Total number of times that pubsub_notify_() has ever been called on this
-   * topic. */
-  uint64_t n_events_fired;
-  /** True iff we're running 'notify' on this topic, and shouldn't allow
-   * any concurrent modifications or events. */
-  unsigned locked;
-} pubsub_topic_t;
-
-const pubsub_subscriber_t *pubsub_subscribe_(pubsub_topic_t *topic,
-                                             pubsub_subscriber_fn_t fn,
-                                             void *subscriber_data,
-                                             unsigned subscribe_flags,
-                                             unsigned priority);
-int pubsub_unsubscribe_(pubsub_topic_t *topic, const pubsub_subscriber_t *sub);
-void pubsub_clear_(pubsub_topic_t *topic);
-typedef int (*pubsub_notify_fn_t)(pubsub_subscriber_t *subscriber,
-                                  void *notify_data);
-int pubsub_notify_(pubsub_topic_t *topic, pubsub_notify_fn_t notify_fn,
-                   void *notify_data, unsigned notify_flags);
-
-#define IMPLEMENT_PUBSUB_TOPIC(notify_linkage, name)                    \
-  static pubsub_topic_t name ## _topic_ = { NULL, 0, 0 };               \
-  const name ## _subscriber_t *                                         \
-  name ## _subscribe(name##_subscriber_fn_t subscriber,                 \
-                     name##_subscriber_data_t *extra_data,              \
-                     unsigned flags,                                    \
-                     unsigned priority)                                 \
-  {                                                                     \
-    const pubsub_subscriber_t *s;                                       \
-    s = pubsub_subscribe_(&name##_topic_,                               \
-                          (pubsub_subscriber_fn_t)subscriber,           \
-                          extra_data,                                   \
-                          flags,                                        \
-                          priority);                                    \
-    return (const name##_subscriber_t *)s;                              \
-  }                                                                     \
-  int                                                                   \
-  name ## _unsubscribe(const name##_subscriber_t *subscriber)           \
-  {                                                                     \
-    return pubsub_unsubscribe_(&name##_topic_,                          \
-                               (const pubsub_subscriber_t *)subscriber); \
-  }                                                                     \
-  static int                                                            \
-  name##_call_the_notify_fn_(pubsub_subscriber_t *subscriber,           \
-                             void *notify_data)                         \
-  {                                                                     \
-    name ## _subscriber_fn_t fn;                                        \
-    fn = (name ## _subscriber_fn_t) subscriber->fn;                     \
-    return fn(notify_data, subscriber->subscriber_data);                \
-  }                                                                     \
-  notify_linkage int                                                    \
-  name ## _notify(name ## _event_data_t *event_data, unsigned flags)    \
-  {                                                                     \
-    return pubsub_notify_(&name##_topic_,                               \
-                          name##_call_the_notify_fn_,                   \
-                          event_data,                                   \
-                          flags);                                       \
-  }                                                                     \
-  notify_linkage void                                                   \
-  name ## _clear(void)                                                  \
-  {                                                                     \
-    pubsub_clear_(&name##_topic_);                                      \
-  }
-
-#endif /* TOR_PUBSUB_H */
-
diff --git a/src/common/torgzip.c b/src/common/torgzip.c
deleted file mode 100644
index c44399aa74..0000000000
--- a/src/common/torgzip.c
+++ /dev/null
@@ -1,586 +0,0 @@
-/* Copyright (c) 2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file torgzip.c
- * \brief A simple in-memory gzip implementation.
- **/
-
-#include "orconfig.h"
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <assert.h>
-#include <string.h>
-#include "torint.h"
-
-#ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
-#endif
-
-#include "util.h"
-#include "torlog.h"
-#include "torgzip.h"
-
-/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros.  Instead of
-   saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory
-   that nobody will care if the compile outputs a no-such-identifier warning.
-
-   Sorry, but we like -Werror over here, so I guess we need to define these.
-   I hope that zlib 1.2.6 doesn't break these too.
-*/
-#ifndef _LARGEFILE64_SOURCE
-#define _LARGEFILE64_SOURCE 0
-#endif
-#ifndef _LFS64_LARGEFILE
-#define _LFS64_LARGEFILE 0
-#endif
-#ifndef _FILE_OFFSET_BITS
-#define _FILE_OFFSET_BITS 0
-#endif
-#ifndef off64_t
-#define off64_t int64_t
-#endif
-
-#include <zlib.h>
-
-#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200
-#error "We require zlib version 1.2 or later."
-#endif
-
-static size_t tor_zlib_state_size_precalc(int inflate,
-                                          int windowbits, int memlevel);
-
-/** Total number of bytes allocated for zlib state */
-static size_t total_zlib_allocation = 0;
-
-/** Return a string representation of the version of the currently running
- * version of zlib. */
-const char *
-tor_zlib_get_version_str(void)
-{
-  return zlibVersion();
-}
-
-/** Return a string representation of the version of the version of zlib
-* used at compilation. */
-const char *
-tor_zlib_get_header_version_str(void)
-{
-  return ZLIB_VERSION;
-}
-
-/** Return the 'bits' value to tell zlib to use <b>method</b>.*/
-static inline int
-method_bits(compress_method_t method, zlib_compression_level_t level)
-{
-  /* Bits+16 means "use gzip" in zlib >= 1.2 */
-  const int flag = method == GZIP_METHOD ? 16 : 0;
-  switch (level) {
-    default:
-    case HIGH_COMPRESSION: return flag + 15;
-    case MEDIUM_COMPRESSION: return flag + 13;
-    case LOW_COMPRESSION: return flag + 11;
-  }
-}
-
-static inline int
-get_memlevel(zlib_compression_level_t level)
-{
-  switch (level) {
-    default:
-    case HIGH_COMPRESSION: return 8;
-    case MEDIUM_COMPRESSION: return 7;
-    case LOW_COMPRESSION: return 6;
-  }
-}
-
-/** @{ */
-/* These macros define the maximum allowable compression factor.  Anything of
- * size greater than CHECK_FOR_COMPRESSION_BOMB_AFTER is not allowed to
- * have an uncompression factor (uncompressed size:compressed size ratio) of
- * any greater than MAX_UNCOMPRESSION_FACTOR.
- *
- * Picking a value for MAX_UNCOMPRESSION_FACTOR is a trade-off: we want it to
- * be small to limit the attack multiplier, but we also want it to be large
- * enough so that no legitimate document --even ones we might invent in the
- * future -- ever compresses by a factor of greater than
- * MAX_UNCOMPRESSION_FACTOR. Within those parameters, there's a reasonably
- * large range of possible values. IMO, anything over 8 is probably safe; IMO
- * anything under 50 is probably sufficient.
- */
-#define MAX_UNCOMPRESSION_FACTOR 25
-#define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64)
-/** @} */
-
-/** Return true if uncompressing an input of size <b>in_size</b> to an input
- * of size at least <b>size_out</b> looks like a compression bomb. */
-static int
-is_compression_bomb(size_t size_in, size_t size_out)
-{
-  if (size_in == 0 || size_out < CHECK_FOR_COMPRESSION_BOMB_AFTER)
-    return 0;
-
-  return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR);
-}
-
-/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly
- * allocated buffer, using the method described in <b>method</b>.  Store the
- * compressed string in *<b>out</b>, and its length in *<b>out_len</b>.
- * Return 0 on success, -1 on failure.
- */
-int
-tor_gzip_compress(char **out, size_t *out_len,
-                  const char *in, size_t in_len,
-                  compress_method_t method)
-{
-  struct z_stream_s *stream = NULL;
-  size_t out_size, old_size;
-  off_t offset;
-
-  tor_assert(out);
-  tor_assert(out_len);
-  tor_assert(in);
-  tor_assert(in_len < UINT_MAX);
-
-  *out = NULL;
-
-  stream = tor_malloc_zero(sizeof(struct z_stream_s));
-  stream->zalloc = Z_NULL;
-  stream->zfree = Z_NULL;
-  stream->opaque = NULL;
-  stream->next_in = (unsigned char*) in;
-  stream->avail_in = (unsigned int)in_len;
-
-  if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED,
-                   method_bits(method, HIGH_COMPRESSION),
-                   get_memlevel(HIGH_COMPRESSION),
-                   Z_DEFAULT_STRATEGY) != Z_OK) {
-    //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments.
-    log_warn(LD_GENERAL, "Error from deflateInit2: %s",
-             stream->msg?stream->msg:"<no message>");
-    goto err;
-    //LCOV_EXCL_STOP
-  }
-
-  /* Guess 50% compression. */
-  out_size = in_len / 2;
-  if (out_size < 1024) out_size = 1024;
-  *out = tor_malloc(out_size);
-  stream->next_out = (unsigned char*)*out;
-  stream->avail_out = (unsigned int)out_size;
-
-  while (1) {
-    switch (deflate(stream, Z_FINISH))
-      {
-      case Z_STREAM_END:
-        goto done;
-      case Z_OK:
-        /* In case zlib doesn't work as I think .... */
-        if (stream->avail_out >= stream->avail_in+16)
-          break;
-        /* Falls through. */
-      case Z_BUF_ERROR:
-        offset = stream->next_out - ((unsigned char*)*out);
-        old_size = out_size;
-        out_size *= 2;
-        if (out_size < old_size) {
-          log_warn(LD_GENERAL, "Size overflow in compression.");
-          goto err;
-        }
-        *out = tor_realloc(*out, out_size);
-        stream->next_out = (unsigned char*)(*out + offset);
-        if (out_size - offset > UINT_MAX) {
-          log_warn(LD_BUG,  "Ran over unsigned int limit of zlib while "
-                   "uncompressing.");
-          goto err;
-        }
-        stream->avail_out = (unsigned int)(out_size - offset);
-        break;
-      default:
-        log_warn(LD_GENERAL, "Gzip compression didn't finish: %s",
-                 stream->msg ? stream->msg : "<no message>");
-        goto err;
-      }
-  }
- done:
-  *out_len = stream->total_out;
-#ifdef OPENBSD
-  /* "Hey Rocky!  Watch me change an unsigned field to a signed field in a
-   *    third-party API!"
-   * "Oh, that trick will just make people do unsafe casts to the unsigned
-   *    type in their cross-platform code!"
-   * "Don't be foolish.  I'm _sure_ they'll have the good sense to make sure
-   *    the newly unsigned field isn't negative." */
-  tor_assert(stream->total_out >= 0);
-#endif
-  if (deflateEnd(stream)!=Z_OK) {
-    // LCOV_EXCL_START -- unreachable if we handled the zlib structure right
-    tor_assert_nonfatal_unreached();
-    log_warn(LD_BUG, "Error freeing gzip structures");
-    goto err;
-    // LCOV_EXCL_STOP
-  }
-  tor_free(stream);
-
-  if (is_compression_bomb(*out_len, in_len)) {
-    log_warn(LD_BUG, "We compressed something and got an insanely high "
-          "compression factor; other Tors would think this was a zlib bomb.");
-    goto err;
-  }
-
-  return 0;
- err:
-  if (stream) {
-    deflateEnd(stream);
-    tor_free(stream);
-  }
-  tor_free(*out);
-  return -1;
-}
-
-/** Given zero or more zlib-compressed or gzip-compressed strings of
- * total length
- * <b>in_len</b> bytes at <b>in</b>, uncompress them into a newly allocated
- * buffer, using the method described in <b>method</b>.  Store the uncompressed
- * string in *<b>out</b>, and its length in *<b>out_len</b>.  Return 0 on
- * success, -1 on failure.
- *
- * If <b>complete_only</b> is true, we consider a truncated input as a
- * failure; otherwise we decompress as much as we can.  Warn about truncated
- * or corrupt inputs at <b>protocol_warn_level</b>.
- */
-int
-tor_gzip_uncompress(char **out, size_t *out_len,
-                    const char *in, size_t in_len,
-                    compress_method_t method,
-                    int complete_only,
-                    int protocol_warn_level)
-{
-  struct z_stream_s *stream = NULL;
-  size_t out_size, old_size;
-  off_t offset;
-  int r;
-
-  tor_assert(out);
-  tor_assert(out_len);
-  tor_assert(in);
-  tor_assert(in_len < UINT_MAX);
-
-  *out = NULL;
-
-  stream = tor_malloc_zero(sizeof(struct z_stream_s));
-  stream->zalloc = Z_NULL;
-  stream->zfree = Z_NULL;
-  stream->opaque = NULL;
-  stream->next_in = (unsigned char*) in;
-  stream->avail_in = (unsigned int)in_len;
-
-  if (inflateInit2(stream,
-                   method_bits(method, HIGH_COMPRESSION)) != Z_OK) {
-    // LCOV_EXCL_START -- can only hit this if we give bad inputs.
-    log_warn(LD_GENERAL, "Error from inflateInit2: %s",
-             stream->msg?stream->msg:"<no message>");
-    goto err;
-    // LCOV_EXCL_STOP
-  }
-
-  out_size = in_len * 2;  /* guess 50% compression. */
-  if (out_size < 1024) out_size = 1024;
-  if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX)
-    goto err;
-
-  *out = tor_malloc(out_size);
-  stream->next_out = (unsigned char*)*out;
-  stream->avail_out = (unsigned int)out_size;
-
-  while (1) {
-    switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH))
-      {
-      case Z_STREAM_END:
-        if (stream->avail_in == 0)
-          goto done;
-        /* There may be more compressed data here. */
-        if ((r = inflateEnd(stream)) != Z_OK) {
-          log_warn(LD_BUG, "Error freeing gzip structures");
-          goto err;
-        }
-        if (inflateInit2(stream,
-                         method_bits(method,HIGH_COMPRESSION)) != Z_OK) {
-          log_warn(LD_GENERAL, "Error from second inflateInit2: %s",
-                   stream->msg?stream->msg:"<no message>");
-          goto err;
-        }
-        break;
-      case Z_OK:
-        if (!complete_only && stream->avail_in == 0)
-          goto done;
-        /* In case zlib doesn't work as I think.... */
-        if (stream->avail_out >= stream->avail_in+16)
-          break;
-        /* Falls through. */
-      case Z_BUF_ERROR:
-        if (stream->avail_out > 0) {
-          log_fn(protocol_warn_level, LD_PROTOCOL,
-                 "possible truncated or corrupt zlib data");
-          goto err;
-        }
-        offset = stream->next_out - (unsigned char*)*out;
-        old_size = out_size;
-        out_size *= 2;
-        if (out_size < old_size) {
-          log_warn(LD_GENERAL, "Size overflow in uncompression.");
-          goto err;
-        }
-        if (is_compression_bomb(in_len, out_size)) {
-          log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; "
-                   "not proceeding.");
-          goto err;
-        }
-        if (out_size >= SIZE_T_CEILING) {
-          log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing.");
-          goto err;
-        }
-        *out = tor_realloc(*out, out_size);
-        stream->next_out = (unsigned char*)(*out + offset);
-        if (out_size - offset > UINT_MAX) {
-          log_warn(LD_BUG,  "Ran over unsigned int limit of zlib while "
-                   "uncompressing.");
-          goto err;
-        }
-        stream->avail_out = (unsigned int)(out_size - offset);
-        break;
-      default:
-        log_warn(LD_GENERAL, "Gzip decompression returned an error: %s",
-                 stream->msg ? stream->msg : "<no message>");
-        goto err;
-      }
-  }
- done:
-  *out_len = stream->next_out - (unsigned char*)*out;
-  r = inflateEnd(stream);
-  tor_free(stream);
-  if (r != Z_OK) {
-    log_warn(LD_BUG, "Error freeing gzip structures");
-    goto err;
-  }
-
-  /* NUL-terminate output. */
-  if (out_size == *out_len)
-    *out = tor_realloc(*out, out_size + 1);
-  (*out)[*out_len] = '\0';
-
-  return 0;
- err:
-  if (stream) {
-    inflateEnd(stream);
-    tor_free(stream);
-  }
-  if (*out) {
-    tor_free(*out);
-  }
-  return -1;
-}
-
-/** Try to tell whether the <b>in_len</b>-byte string in <b>in</b> is likely
- * to be compressed or not.  If it is, return the likeliest compression method.
- * Otherwise, return UNKNOWN_METHOD.
- */
-compress_method_t
-detect_compression_method(const char *in, size_t in_len)
-{
-  if (in_len > 2 && fast_memeq(in, "\x1f\x8b", 2)) {
-    return GZIP_METHOD;
-  } else if (in_len > 2 && (in[0] & 0x0f) == 8 &&
-             (ntohs(get_uint16(in)) % 31) == 0) {
-    return ZLIB_METHOD;
-  } else {
-    return UNKNOWN_METHOD;
-  }
-}
-
-/** Internal state for an incremental zlib compression/decompression.  The
- * body of this struct is not exposed. */
-struct tor_zlib_state_t {
-  struct z_stream_s stream; /**< The zlib stream */
-  int compress; /**< True if we are compressing; false if we are inflating */
-
-  /** Number of bytes read so far.  Used to detect zlib bombs. */
-  size_t input_so_far;
-  /** Number of bytes written so far.  Used to detect zlib bombs. */
-  size_t output_so_far;
-
-  /** Approximate number of bytes allocated for this object. */
-  size_t allocation;
-};
-
-/** Construct and return a tor_zlib_state_t object using <b>method</b>.  If
- * <b>compress</b>, it's for compression; otherwise it's for
- * decompression. */
-tor_zlib_state_t *
-tor_zlib_new(int compress_, compress_method_t method,
-             zlib_compression_level_t compression_level)
-{
-  tor_zlib_state_t *out;
-  int bits, memlevel;
-
- if (! compress_) {
-   /* use this setting for decompression, since we might have the
-    * max number of window bits */
-   compression_level = HIGH_COMPRESSION;
- }
-
- out = tor_malloc_zero(sizeof(tor_zlib_state_t));
- out->stream.zalloc = Z_NULL;
- out->stream.zfree = Z_NULL;
- out->stream.opaque = NULL;
- out->compress = compress_;
- bits = method_bits(method, compression_level);
- memlevel = get_memlevel(compression_level);
- if (compress_) {
-   if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,
-                    bits, memlevel,
-                    Z_DEFAULT_STRATEGY) != Z_OK)
-     goto err; // LCOV_EXCL_LINE
- } else {
-   if (inflateInit2(&out->stream, bits) != Z_OK)
-     goto err; // LCOV_EXCL_LINE
- }
- out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel);
-
- total_zlib_allocation += out->allocation;
-
- return out;
-
- err:
- tor_free(out);
- return NULL;
-}
-
-/** Compress/decompress some bytes using <b>state</b>.  Read up to
- * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
- * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
- * we've reached the end of the input.
- *
- * Return TOR_ZLIB_DONE if we've finished the entire compression/decompression.
- * Return TOR_ZLIB_OK if we're processed everything from the input.
- * Return TOR_ZLIB_BUF_FULL if we're out of space on <b>out</b>.
- * Return TOR_ZLIB_ERR if the stream is corrupt.
- */
-tor_zlib_output_t
-tor_zlib_process(tor_zlib_state_t *state,
-                 char **out, size_t *out_len,
-                 const char **in, size_t *in_len,
-                 int finish)
-{
-  int err;
-  tor_assert(*in_len <= UINT_MAX);
-  tor_assert(*out_len <= UINT_MAX);
-  state->stream.next_in = (unsigned char*) *in;
-  state->stream.avail_in = (unsigned int)*in_len;
-  state->stream.next_out = (unsigned char*) *out;
-  state->stream.avail_out = (unsigned int)*out_len;
-
-  if (state->compress) {
-    err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH);
-  } else {
-    err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH);
-  }
-
-  state->input_so_far += state->stream.next_in - ((unsigned char*)*in);
-  state->output_so_far += state->stream.next_out - ((unsigned char*)*out);
-
-  *out = (char*) state->stream.next_out;
-  *out_len = state->stream.avail_out;
-  *in = (const char *) state->stream.next_in;
-  *in_len = state->stream.avail_in;
-
-  if (! state->compress &&
-      is_compression_bomb(state->input_so_far, state->output_so_far)) {
-    log_warn(LD_DIR, "Possible zlib bomb; abandoning stream.");
-    return TOR_ZLIB_ERR;
-  }
-
-  switch (err)
-    {
-    case Z_STREAM_END:
-      return TOR_ZLIB_DONE;
-    case Z_BUF_ERROR:
-      if (state->stream.avail_in == 0 && !finish)
-        return TOR_ZLIB_OK;
-      return TOR_ZLIB_BUF_FULL;
-    case Z_OK:
-      if (state->stream.avail_out == 0 || finish)
-        return TOR_ZLIB_BUF_FULL;
-      return TOR_ZLIB_OK;
-    default:
-      log_warn(LD_GENERAL, "Gzip returned an error: %s",
-               state->stream.msg ? state->stream.msg : "<no message>");
-      return TOR_ZLIB_ERR;
-    }
-}
-
-/** Deallocate <b>state</b>. */
-void
-tor_zlib_free(tor_zlib_state_t *state)
-{
-  if (!state)
-    return;
-
- total_zlib_allocation -= state->allocation;
-
-  if (state->compress)
-    deflateEnd(&state->stream);
-  else
-    inflateEnd(&state->stream);
-
-  tor_free(state);
-}
-
-/** Return an approximate number of bytes used in RAM to hold a state with
- * window bits <b>windowBits</b> and compression level 'memlevel' */
-static size_t
-tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel)
-{
-  windowbits &= 15;
-
-#define A_FEW_KILOBYTES 2048
-
-  if (inflate_) {
-    /* From zconf.h:
-
-       "The memory requirements for inflate are (in bytes) 1 << windowBits
-       that is, 32K for windowBits=15 (default value) plus a few kilobytes
-       for small objects."
-    */
-    return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) +
-      (1 << 15) + A_FEW_KILOBYTES;
-  } else {
-    /* Also from zconf.h:
-
-       "The memory requirements for deflate are (in bytes):
-            (1 << (windowBits+2)) +  (1 << (memLevel+9))
-        ... plus a few kilobytes for small objects."
-    */
-    return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) +
-      (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES;
-  }
-#undef A_FEW_KILOBYTES
-}
-
-/** Return the approximate number of bytes allocated for <b>state</b>. */
-size_t
-tor_zlib_state_size(const tor_zlib_state_t *state)
-{
-  return state->allocation;
-}
-
-/** Return the approximate number of bytes allocated for all zlib states. */
-size_t
-tor_zlib_get_total_allocation(void)
-{
-  return total_zlib_allocation;
-}
-
diff --git a/src/common/torgzip.h b/src/common/torgzip.h
deleted file mode 100644
index 00f62dcb45..0000000000
--- a/src/common/torgzip.h
+++ /dev/null
@@ -1,72 +0,0 @@
-/* Copyright (c) 2003, Roger Dingledine
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file torgzip.h
- * \brief Headers for torgzip.h
- **/
-
-#ifndef TOR_TORGZIP_H
-#define TOR_TORGZIP_H
-
-/** Enumeration of what kind of compression to use.  Only ZLIB_METHOD is
- * guaranteed to be supported by the compress/uncompress functions here;
- * GZIP_METHOD may be supported if we built against zlib version 1.2 or later
- * and is_gzip_supported() returns true. */
-typedef enum {
-  NO_METHOD=0, GZIP_METHOD=1, ZLIB_METHOD=2, UNKNOWN_METHOD=3
-} compress_method_t;
-
-/**
- * Enumeration to define tradeoffs between memory usage and compression level.
- * HIGH_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most
- * memory.
- **/
-typedef enum {
-  HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION
-} zlib_compression_level_t;
-
-int
-tor_gzip_compress(char **out, size_t *out_len,
-                  const char *in, size_t in_len,
-                  compress_method_t method);
-int
-tor_gzip_uncompress(char **out, size_t *out_len,
-                    const char *in, size_t in_len,
-                    compress_method_t method,
-                    int complete_only,
-                    int protocol_warn_level);
-
-int is_gzip_supported(void);
-
-const char *
-tor_zlib_get_version_str(void);
-
-const char *
-tor_zlib_get_header_version_str(void);
-
-compress_method_t detect_compression_method(const char *in, size_t in_len);
-
-/** Return values from tor_zlib_process; see that function's documentation for
- * details. */
-typedef enum {
-  TOR_ZLIB_OK, TOR_ZLIB_DONE, TOR_ZLIB_BUF_FULL, TOR_ZLIB_ERR
-} tor_zlib_output_t;
-/** Internal state for an incremental zlib compression/decompression. */
-typedef struct tor_zlib_state_t tor_zlib_state_t;
-tor_zlib_state_t *tor_zlib_new(int compress, compress_method_t method,
-                               zlib_compression_level_t level);
-
-tor_zlib_output_t tor_zlib_process(tor_zlib_state_t *state,
-                                   char **out, size_t *out_len,
-                                   const char **in, size_t *in_len,
-                                   int finish);
-void tor_zlib_free(tor_zlib_state_t *state);
-
-size_t tor_zlib_state_size(const tor_zlib_state_t *state);
-size_t tor_zlib_get_total_allocation(void);
-
-#endif
-
diff --git a/src/common/torint.h b/src/common/torint.h
deleted file mode 100644
index 58c30f41a8..0000000000
--- a/src/common/torint.h
+++ /dev/null
@@ -1,367 +0,0 @@
-/* Copyright (c) 2003, Roger Dingledine
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file torint.h
- * \brief Header file to define uint32_t and friends
- **/
-
-#ifndef TOR_TORINT_H
-#define TOR_TORINT_H
-
-#include "orconfig.h"
-
-#ifdef HAVE_STDINT_H
-#include <stdint.h>
-#endif
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#ifdef HAVE_LIMITS_H
-#include <limits.h>
-#endif
-#ifdef HAVE_SYS_LIMITS_H
-#include <sys/limits.h>
-#endif
-#ifdef HAVE_MACHINE_LIMITS_H
-#if !defined(__FreeBSD__) && !defined(__FreeBSD_kernel__)
-  /* FreeBSD has a bug where it complains that this file is obsolete,
-     and I should migrate to using sys/limits. It complains even when
-     I include both.
-     __FreeBSD_kernel__ is defined by Debian GNU/kFreeBSD which
-      does the same thing (but doesn't defined __FreeBSD__).
-     */
-#include <machine/limits.h>
-#endif
-#endif
-#ifdef HAVE_INTTYPES_H
-#include <inttypes.h>
-#endif
-
-#if (SIZEOF_INT8_T != 0)
-#define HAVE_INT8_T
-#endif
-#if (SIZEOF_INT16_T != 0)
-#define HAVE_INT16_T
-#endif
-#if (SIZEOF_INT32_T != 0)
-#define HAVE_INT32_T
-#endif
-#if (SIZEOF_INT64_T != 0)
-#define HAVE_INT64_T
-#endif
-#if (SIZEOF_UINT8_T != 0)
-#define HAVE_UINT8_T
-#endif
-#if (SIZEOF_UINT16_T != 0)
-#define HAVE_UINT16_T
-#endif
-#if (SIZEOF_UINT32_T != 0)
-#define HAVE_UINT32_T
-#endif
-#if (SIZEOF_UINT64_T != 0)
-#define HAVE_UINT64_T
-#endif
-#if (SIZEOF_INTPTR_T != 0)
-#define HAVE_INTPTR_T
-#endif
-#if (SIZEOF_UINTPTR_T != 0)
-#define HAVE_UINTPTR_T
-#endif
-
-#if (SIZEOF_CHAR == 1)
-#ifndef HAVE_INT8_T
-typedef signed char int8_t;
-#define HAVE_INT8_T
-#endif
-#ifndef HAVE_UINT8_T
-typedef unsigned char uint8_t;
-#define HAVE_UINT8_T
-#endif
-#endif
-
-#if (SIZEOF_SHORT == 2)
-#ifndef HAVE_INT16_T
-typedef signed short int16_t;
-#define HAVE_INT16_T
-#endif
-#ifndef HAVE_UINT16_T
-typedef unsigned short uint16_t;
-#define HAVE_UINT16_T
-#endif
-#endif
-
-#if (SIZEOF_INT == 2)
-#ifndef HAVE_INT16_T
-typedef signed int int16_t;
-#define HAVE_INT16_T
-#endif
-#ifndef HAVE_UINT16_T
-typedef unsigned int uint16_t;
-#define HAVE_UINT16_T
-#endif
-#elif (SIZEOF_INT == 4)
-#ifndef HAVE_INT32_T
-typedef signed int int32_t;
-#define HAVE_INT32_T
-#endif
-#ifndef HAVE_UINT32_T
-typedef unsigned int uint32_t;
-#define HAVE_UINT32_T
-#endif
-#ifndef UINT16_MAX
-#define UINT16_MAX 0xffffu
-#endif
-#ifndef INT16_MAX
-#define INT16_MAX 0x7fff
-#endif
-#ifndef INT16_MIN
-#define INT16_MIN (-INT16_MAX-1)
-#endif
-#ifndef UINT32_MAX
-#define UINT32_MAX 0xffffffffu
-#endif
-#ifndef INT32_MAX
-#define INT32_MAX 0x7fffffff
-#endif
-#ifndef INT32_MIN
-#define INT32_MIN (-2147483647-1)
-#endif
-#endif
-
-#if (SIZEOF_LONG == 4)
-#ifndef HAVE_INT32_T
-typedef signed long int32_t;
-#define HAVE_INT32_T
-#endif
-#ifndef HAVE_UINT32_T
-typedef unsigned long uint32_t;
-#define HAVE_UINT32_T
-#ifndef UINT32_MAX
-#define UINT32_MAX 0xfffffffful
-#endif
-#endif
-#elif (SIZEOF_LONG == 8)
-#ifndef HAVE_INT64_T
-typedef signed long int64_t;
-#define HAVE_INT64_T
-#endif
-#ifndef HAVE_UINT32_T
-typedef unsigned long uint64_t;
-#define HAVE_UINT32_T
-#endif
-#ifndef UINT64_MAX
-#define UINT64_MAX 0xfffffffffffffffful
-#endif
-#endif
-
-#if (SIZEOF_LONG_LONG == 8)
-#ifndef HAVE_INT64_T
-typedef signed long long int64_t;
-#define HAVE_INT64_T
-#endif
-#ifndef HAVE_UINT64_T
-typedef unsigned long long uint64_t;
-#define HAVE_UINT64_T
-#endif
-#ifndef UINT64_MAX
-#define UINT64_MAX 0xffffffffffffffffull
-#endif
-#ifndef INT64_MAX
-#define INT64_MAX 0x7fffffffffffffffll
-#endif
-#endif
-
-#if (SIZEOF___INT64 == 8)
-#ifndef HAVE_INT64_T
-typedef signed __int64 int64_t;
-#define HAVE_INT64_T
-#endif
-#ifndef HAVE_UINT64_T
-typedef unsigned __int64 uint64_t;
-#define HAVE_UINT64_T
-#endif
-#ifndef UINT64_MAX
-#define UINT64_MAX 0xffffffffffffffffui64
-#endif
-#ifndef INT64_MAX
-#define INT64_MAX 0x7fffffffffffffffi64
-#endif
-#endif
-
-#ifndef INT64_MIN
-#define INT64_MIN ((- INT64_MAX) - 1)
-#endif
-
-#ifndef SIZE_MAX
-#if SIZEOF_SIZE_T == 8
-#define SIZE_MAX UINT64_MAX
-#elif  SIZEOF_SIZE_T == 4
-#define SIZE_MAX UINT32_MAX
-#else
-#error "Can't define SIZE_MAX"
-#endif
-#endif
-
-#ifndef HAVE_SSIZE_T
-#if SIZEOF_SIZE_T == 8
-typedef int64_t ssize_t;
-#elif SIZEOF_SIZE_T == 4
-typedef int32_t ssize_t;
-#else
-#error "Can't define ssize_t."
-#endif
-#endif
-
-#if (SIZEOF_VOID_P > 4 && SIZEOF_VOID_P <= 8)
-#ifndef HAVE_INTPTR_T
-typedef int64_t intptr_t;
-#define SIZEOF_INTPTR_T 8
-#endif
-#ifndef HAVE_UINTPTR_T
-typedef uint64_t uintptr_t;
-#define SIZEOF_UINTPTR_T 8
-#endif
-#elif (SIZEOF_VOID_P > 2 && SIZEOF_VOID_P <= 4)
-#ifndef HAVE_INTPTR_T
-typedef int32_t intptr_t;
-#define SIZEOF_INTPTR_T 4
-#endif
-#ifndef HAVE_UINTPTR_T
-typedef uint32_t uintptr_t;
-#define SIZEOF_UINTPTR_T 4
-#endif
-#else
-#error "void * is either >8 bytes or <= 2.  In either case, I am confused."
-#endif
-
-#ifndef HAVE_INT8_T
-#error "Missing type int8_t"
-#endif
-#ifndef HAVE_UINT8_T
-#error "Missing type uint8_t"
-#endif
-#ifndef HAVE_INT16_T
-#error "Missing type int16_t"
-#endif
-#ifndef HAVE_UINT16_T
-#error "Missing type uint16_t"
-#endif
-#ifndef HAVE_INT32_T
-#error "Missing type int32_t"
-#endif
-#ifndef HAVE_UINT32_T
-#error "Missing type uint32_t"
-#endif
-#ifndef HAVE_INT64_T
-#error "Missing type int64_t"
-#endif
-#ifndef HAVE_UINT64_T
-#error "Missing type uint64_t"
-#endif
-
-/* This assumes a sane (2's-complement) representation.  But if you
- * aren't 2's complement, and you don't define LONG_MAX, then you're so
- * bizarre that I want nothing to do with you. */
-#ifndef USING_TWOS_COMPLEMENT
-#error "Seems that your platform doesn't use 2's complement arithmetic. Argh."
-#endif
-#ifndef LONG_MAX
-#if (SIZEOF_LONG == 4)
-#define LONG_MAX 0x7fffffffL
-#elif (SIZEOF_LONG == 8)
-#define LONG_MAX 0x7fffffffffffffffL
-#else
-#error "Can't define LONG_MAX"
-#endif
-#endif
-
-#ifndef INT_MAX
-#if (SIZEOF_INT == 4)
-#define INT_MAX 0x7fffffffL
-#elif (SIZEOF_INT == 8)
-#define INT_MAX 0x7fffffffffffffffL
-#else
-#error "Can't define INT_MAX"
-#endif
-#endif
-
-#ifndef UINT_MAX
-#if (SIZEOF_INT == 2)
-#define UINT_MAX 0xffffu
-#elif (SIZEOF_INT == 4)
-#define UINT_MAX 0xffffffffu
-#elif (SIZEOF_INT == 8)
-#define UINT_MAX 0xffffffffffffffffu
-#else
-#error "Can't define UINT_MAX"
-#endif
-#endif
-
-#ifndef SHORT_MAX
-#if (SIZEOF_SHORT == 2)
-#define SHORT_MAX 0x7fff
-#elif (SIZEOF_SHORT == 4)
-#define SHORT_MAX 0x7fffffff
-#else
-#error "Can't define SHORT_MAX"
-#endif
-#endif
-
-#ifndef TIME_MAX
-
-#if (SIZEOF_TIME_T == SIZEOF_INT)
-#define TIME_MAX ((time_t)INT_MAX)
-#elif (SIZEOF_TIME_T == SIZEOF_LONG)
-#define TIME_MAX ((time_t)LONG_MAX)
-#elif (SIZEOF_TIME_T == 8)
-#define TIME_MAX ((time_t)INT64_MAX)
-#else
-#error "Can't define TIME_MAX"
-#endif
-
-#endif /* ifndef(TIME_MAX) */
-
-#ifndef TIME_MIN
-
-#if (SIZEOF_TIME_T == SIZEOF_INT)
-#define TIME_MIN ((time_t)INT_MIN)
-#elif (SIZEOF_TIME_T == SIZEOF_LONG)
-#define TIME_MIN ((time_t)LONG_MIN)
-#elif (SIZEOF_TIME_T == 8)
-#define TIME_MIN ((time_t)INT64_MIN)
-#else
-#error "Can't define TIME_MIN"
-#endif
-
-#endif /* ifndef(TIME_MIN) */
-
-#ifndef SIZE_MAX
-#if (SIZEOF_SIZE_T == 4)
-#define SIZE_MAX UINT32_MAX
-#elif (SIZEOF_SIZE_T == 8)
-#define SIZE_MAX UINT64_MAX
-#else
-#error "Can't define SIZE_MAX"
-#endif
-#endif
-
-#ifndef SSIZE_MAX
-#if (SIZEOF_SIZE_T == 4)
-#define SSIZE_MAX INT32_MAX
-#elif (SIZEOF_SIZE_T == 8)
-#define SSIZE_MAX INT64_MAX
-#else
-#error "Can't define SSIZE_MAX"
-#endif
-#endif
-
-/** Any ssize_t larger than this amount is likely to be an underflow. */
-#define SSIZE_T_CEILING ((ssize_t)(SSIZE_MAX-16))
-/** Any size_t larger than this amount is likely to be an underflow. */
-#define SIZE_T_CEILING  ((size_t)(SSIZE_MAX-16))
-
-#endif /* __TORINT_H */
-
diff --git a/src/common/tortls.h b/src/common/tortls.h
deleted file mode 100644
index f018c45c82..0000000000
--- a/src/common/tortls.h
+++ /dev/null
@@ -1,265 +0,0 @@
-/* Copyright (c) 2003, Roger Dingledine
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#ifndef TOR_TORTLS_H
-#define TOR_TORTLS_H
-
-/**
- * \file tortls.h
- * \brief Headers for tortls.c
- **/
-
-#include "crypto.h"
-#include "compat_openssl.h"
-#include "compat.h"
-#include "testsupport.h"
-
-/* Opaque structure to hold a TLS connection. */
-typedef struct tor_tls_t tor_tls_t;
-
-/* Opaque structure to hold an X509 certificate. */
-typedef struct tor_x509_cert_t tor_x509_cert_t;
-
-/* Possible return values for most tor_tls_* functions. */
-#define MIN_TOR_TLS_ERROR_VAL_     -9
-#define TOR_TLS_ERROR_MISC         -9
-/* Rename to unexpected close or something. XXXX */
-#define TOR_TLS_ERROR_IO           -8
-#define TOR_TLS_ERROR_CONNREFUSED  -7
-#define TOR_TLS_ERROR_CONNRESET    -6
-#define TOR_TLS_ERROR_NO_ROUTE     -5
-#define TOR_TLS_ERROR_TIMEOUT      -4
-#define TOR_TLS_CLOSE              -3
-#define TOR_TLS_WANTREAD           -2
-#define TOR_TLS_WANTWRITE          -1
-#define TOR_TLS_DONE                0
-
-/** Collection of case statements for all TLS errors that are not due to
- * underlying IO failure. */
-#define CASE_TOR_TLS_ERROR_ANY_NONIO            \
-  case TOR_TLS_ERROR_MISC:                      \
-  case TOR_TLS_ERROR_CONNREFUSED:               \
-  case TOR_TLS_ERROR_CONNRESET:                 \
-  case TOR_TLS_ERROR_NO_ROUTE:                  \
-  case TOR_TLS_ERROR_TIMEOUT
-
-/** Use this macro in a switch statement to catch _any_ TLS error.  That way,
- * if more errors are added, your switches will still work. */
-#define CASE_TOR_TLS_ERROR_ANY                  \
-  CASE_TOR_TLS_ERROR_ANY_NONIO:                 \
-  case TOR_TLS_ERROR_IO
-
-#define TOR_TLS_IS_ERROR(rv) ((rv) < TOR_TLS_CLOSE)
-
-#ifdef TORTLS_PRIVATE
-#define TOR_TLS_MAGIC 0x71571571
-
-typedef enum {
-    TOR_TLS_ST_HANDSHAKE, TOR_TLS_ST_OPEN, TOR_TLS_ST_GOTCLOSE,
-    TOR_TLS_ST_SENTCLOSE, TOR_TLS_ST_CLOSED, TOR_TLS_ST_RENEGOTIATE,
-    TOR_TLS_ST_BUFFEREVENT
-} tor_tls_state_t;
-#define tor_tls_state_bitfield_t ENUM_BF(tor_tls_state_t)
-
-/** Holds a SSL_CTX object and related state used to configure TLS
- * connections.
- */
-typedef struct tor_tls_context_t {
-  int refcnt;
-  SSL_CTX *ctx;
-  tor_x509_cert_t *my_link_cert;
-  tor_x509_cert_t *my_id_cert;
-  tor_x509_cert_t *my_auth_cert;
-  crypto_pk_t *link_key;
-  crypto_pk_t *auth_key;
-} tor_tls_context_t;
-
-/** Structure that we use for a single certificate. */
-struct tor_x509_cert_t {
-  X509 *cert;
-  uint8_t *encoded;
-  size_t encoded_len;
-  unsigned pkey_digests_set : 1;
-  common_digests_t cert_digests;
-  common_digests_t pkey_digests;
-};
-
-/** Holds a SSL object and its associated data.  Members are only
- * accessed from within tortls.c.
- */
-struct tor_tls_t {
-  uint32_t magic;
-  tor_tls_context_t *context; /** A link to the context object for this tls. */
-  SSL *ssl; /**< An OpenSSL SSL object. */
-  int socket; /**< The underlying file descriptor for this TLS connection. */
-  char *address; /**< An address to log when describing this connection. */
-  tor_tls_state_bitfield_t state : 3; /**< The current SSL state,
-                                       * depending on which operations
-                                       * have completed successfully. */
-  unsigned int isServer:1; /**< True iff this is a server-side connection */
-  unsigned int wasV2Handshake:1; /**< True iff the original handshake for
-                                  * this connection used the updated version
-                                  * of the connection protocol (client sends
-                                  * different cipher list, server sends only
-                                  * one certificate). */
-  /** True iff we should call negotiated_callback when we're done reading. */
-  unsigned int got_renegotiate:1;
-  /** Return value from tor_tls_classify_client_ciphers, or 0 if we haven't
-   * called that function yet. */
-  int8_t client_cipher_list_type;
-  /** Incremented every time we start the server side of a handshake. */
-  uint8_t server_handshake_count;
-  size_t wantwrite_n; /**< 0 normally, >0 if we returned wantwrite last
-                       * time. */
-  /** Last values retrieved from BIO_number_read()/write(); see
-   * tor_tls_get_n_raw_bytes() for usage.
-   */
-  unsigned long last_write_count;
-  unsigned long last_read_count;
-  /** If set, a callback to invoke whenever the client tries to renegotiate
-   * the handshake. */
-  void (*negotiated_callback)(tor_tls_t *tls, void *arg);
-  /** Argument to pass to negotiated_callback. */
-  void *callback_arg;
-};
-
-STATIC int tor_errno_to_tls_error(int e);
-STATIC int tor_tls_get_error(tor_tls_t *tls, int r, int extra,
-                  const char *doing, int severity, int domain);
-STATIC tor_tls_t *tor_tls_get_by_ssl(const SSL *ssl);
-STATIC void tor_tls_allocate_tor_tls_object_ex_data_index(void);
-STATIC int always_accept_verify_cb(int preverify_ok, X509_STORE_CTX *x509_ctx);
-STATIC int tor_tls_classify_client_ciphers(const SSL *ssl,
-                                           STACK_OF(SSL_CIPHER) *peer_ciphers);
-STATIC int tor_tls_client_is_using_v2_ciphers(const SSL *ssl);
-MOCK_DECL(STATIC void, try_to_extract_certs_from_tls,
-          (int severity, tor_tls_t *tls, X509 **cert_out, X509 **id_cert_out));
-#ifndef HAVE_SSL_SESSION_GET_MASTER_KEY
-STATIC size_t SSL_SESSION_get_master_key(SSL_SESSION *s, uint8_t *out,
-                                         size_t len);
-#endif
-STATIC void tor_tls_debug_state_callback(const SSL *ssl, int type, int val);
-STATIC void tor_tls_server_info_callback(const SSL *ssl, int type, int val);
-STATIC int tor_tls_session_secret_cb(SSL *ssl, void *secret,
-                            int *secret_len,
-                            STACK_OF(SSL_CIPHER) *peer_ciphers,
-                            CONST_IF_OPENSSL_1_1_API SSL_CIPHER **cipher,
-                            void *arg);
-STATIC int find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m,
-                             uint16_t cipher);
-MOCK_DECL(STATIC X509*, tor_tls_create_certificate,(crypto_pk_t *rsa,
-                                                    crypto_pk_t *rsa_sign,
-                                                    const char *cname,
-                                                    const char *cname_sign,
-                                                  unsigned int cert_lifetime));
-STATIC tor_tls_context_t *tor_tls_context_new(crypto_pk_t *identity,
-                   unsigned int key_lifetime, unsigned flags, int is_client);
-MOCK_DECL(STATIC tor_x509_cert_t *, tor_x509_cert_new,(X509 *x509_cert));
-STATIC int tor_tls_context_init_one(tor_tls_context_t **ppcontext,
-                                    crypto_pk_t *identity,
-                                    unsigned int key_lifetime,
-                                    unsigned int flags,
-                                    int is_client);
-STATIC void tls_log_errors(tor_tls_t *tls, int severity, int domain,
-                           const char *doing);
-
-#ifdef TOR_UNIT_TESTS
-extern int tor_tls_object_ex_data_index;
-extern tor_tls_context_t *server_tls_context;
-extern tor_tls_context_t *client_tls_context;
-extern uint16_t v2_cipher_list[];
-extern uint64_t total_bytes_written_over_tls;
-extern uint64_t total_bytes_written_by_tls;
-#endif
-
-#endif /* endif TORTLS_PRIVATE */
-
-const char *tor_tls_err_to_string(int err);
-void tor_tls_get_state_description(tor_tls_t *tls, char *buf, size_t sz);
-
-void tor_tls_free_all(void);
-
-#define TOR_TLS_CTX_IS_PUBLIC_SERVER (1u<<0)
-#define TOR_TLS_CTX_USE_ECDHE_P256   (1u<<1)
-#define TOR_TLS_CTX_USE_ECDHE_P224   (1u<<2)
-
-int tor_tls_context_init(unsigned flags,
-                         crypto_pk_t *client_identity,
-                         crypto_pk_t *server_identity,
-                         unsigned int key_lifetime);
-tor_tls_t *tor_tls_new(int sock, int is_server);
-void tor_tls_set_logged_address(tor_tls_t *tls, const char *address);
-void tor_tls_set_renegotiate_callback(tor_tls_t *tls,
-                                      void (*cb)(tor_tls_t *, void *arg),
-                                      void *arg);
-int tor_tls_is_server(tor_tls_t *tls);
-void tor_tls_free(tor_tls_t *tls);
-int tor_tls_peer_has_cert(tor_tls_t *tls);
-tor_x509_cert_t *tor_x509_cert_dup(const tor_x509_cert_t *cert);
-MOCK_DECL(tor_x509_cert_t *,tor_tls_get_peer_cert,(tor_tls_t *tls));
-MOCK_DECL(tor_x509_cert_t *,tor_tls_get_own_cert,(tor_tls_t *tls));
-int tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity);
-int tor_tls_check_lifetime(int severity,
-                           tor_tls_t *tls, int past_tolerance,
-                           int future_tolerance);
-MOCK_DECL(int, tor_tls_read, (tor_tls_t *tls, char *cp, size_t len));
-int tor_tls_write(tor_tls_t *tls, const char *cp, size_t n);
-int tor_tls_handshake(tor_tls_t *tls);
-int tor_tls_finish_handshake(tor_tls_t *tls);
-void tor_tls_unblock_renegotiation(tor_tls_t *tls);
-void tor_tls_block_renegotiation(tor_tls_t *tls);
-void tor_tls_assert_renegotiation_unblocked(tor_tls_t *tls);
-int tor_tls_shutdown(tor_tls_t *tls);
-int tor_tls_get_pending_bytes(tor_tls_t *tls);
-size_t tor_tls_get_forced_write_size(tor_tls_t *tls);
-
-void tor_tls_get_n_raw_bytes(tor_tls_t *tls,
-                             size_t *n_read, size_t *n_written);
-
-int tor_tls_get_buffer_sizes(tor_tls_t *tls,
-                              size_t *rbuf_capacity, size_t *rbuf_bytes,
-                              size_t *wbuf_capacity, size_t *wbuf_bytes);
-
-MOCK_DECL(double, tls_get_write_overhead_ratio, (void));
-
-int tor_tls_used_v1_handshake(tor_tls_t *tls);
-int tor_tls_get_num_server_handshakes(tor_tls_t *tls);
-int tor_tls_server_got_renegotiate(tor_tls_t *tls);
-MOCK_DECL(int,tor_tls_get_tlssecrets,(tor_tls_t *tls, uint8_t *secrets_out));
-
-/* Log and abort if there are unhandled TLS errors in OpenSSL's error stack.
- */
-#define check_no_tls_errors() check_no_tls_errors_(__FILE__,__LINE__)
-
-void check_no_tls_errors_(const char *fname, int line);
-void tor_tls_log_one_error(tor_tls_t *tls, unsigned long err,
-                           int severity, int domain, const char *doing);
-
-void tor_x509_cert_free(tor_x509_cert_t *cert);
-tor_x509_cert_t *tor_x509_cert_decode(const uint8_t *certificate,
-                            size_t certificate_len);
-void tor_x509_cert_get_der(const tor_x509_cert_t *cert,
-                      const uint8_t **encoded_out, size_t *size_out);
-const common_digests_t *tor_x509_cert_get_id_digests(
-                      const tor_x509_cert_t *cert);
-const common_digests_t *tor_x509_cert_get_cert_digests(
-                      const tor_x509_cert_t *cert);
-int tor_tls_get_my_certs(int server,
-                         const tor_x509_cert_t **link_cert_out,
-                         const tor_x509_cert_t **id_cert_out);
-crypto_pk_t *tor_tls_get_my_client_auth_key(void);
-crypto_pk_t *tor_tls_cert_get_key(tor_x509_cert_t *cert);
-MOCK_DECL(int,tor_tls_cert_matches_key,(const tor_tls_t *tls,
-                                        const tor_x509_cert_t *cert));
-int tor_tls_cert_is_valid(int severity,
-                          const tor_x509_cert_t *cert,
-                          const tor_x509_cert_t *signing_cert,
-                          int check_rsa_1024);
-const char *tor_tls_get_ciphersuite_name(tor_tls_t *tls);
-
-int evaluate_ecgroup_for_tls(const char *ecgroup);
-
-#endif
-
diff --git a/src/common/util.c b/src/common/util.c
deleted file mode 100644
index d2cbacde31..0000000000
--- a/src/common/util.c
+++ /dev/null
@@ -1,5774 +0,0 @@
-/* Copyright (c) 2003, Roger Dingledine
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file util.c
- * \brief Common functions for strings, IO, network, data structures,
- * process control.
- **/
-
-#include "orconfig.h"
-#ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif
-#define UTIL_PRIVATE
-#include "util.h"
-#include "torlog.h"
-#include "crypto.h"
-#include "torint.h"
-#include "container.h"
-#include "address.h"
-#include "sandbox.h"
-#include "backtrace.h"
-#include "util_process.h"
-#include "util_format.h"
-
-#ifdef _WIN32
-#include <io.h>
-#include <direct.h>
-#include <process.h>
-#include <tchar.h>
-#include <winbase.h>
-#else
-#include <dirent.h>
-#include <pwd.h>
-#include <grp.h>
-#endif
-
-/* math.h needs this on Linux */
-#ifndef _USE_ISOC99_
-#define _USE_ISOC99_ 1
-#endif
-#include <math.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <assert.h>
-#include <signal.h>
-
-#ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
-#endif
-#ifdef HAVE_ARPA_INET_H
-#include <arpa/inet.h>
-#endif
-#ifdef HAVE_ERRNO_H
-#include <errno.h>
-#endif
-#ifdef HAVE_SYS_SOCKET_H
-#include <sys/socket.h>
-#endif
-#ifdef HAVE_SYS_TIME_H
-#include <sys/time.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
-#endif
-#ifdef HAVE_SYS_FCNTL_H
-#include <sys/fcntl.h>
-#endif
-#ifdef HAVE_TIME_H
-#include <time.h>
-#endif
-#ifdef HAVE_MALLOC_MALLOC_H
-#include <malloc/malloc.h>
-#endif
-#ifdef HAVE_MALLOC_H
-#if !defined(OPENBSD) && !defined(__FreeBSD__)
-/* OpenBSD has a malloc.h, but for our purposes, it only exists in order to
- * scold us for being so stupid as to autodetect its presence.  To be fair,
- * they've done this since 1996, when autoconf was only 5 years old. */
-#include <malloc.h>
-#endif
-#endif
-#ifdef HAVE_MALLOC_NP_H
-#include <malloc_np.h>
-#endif
-#ifdef HAVE_SYS_WAIT_H
-#include <sys/wait.h>
-#endif
-#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__)
-#include <sys/prctl.h>
-#endif
-
-#ifdef __clang_analyzer__
-#undef MALLOC_ZERO_WORKS
-#endif
-
-/* =====
- * Memory management
- * ===== */
-#ifdef USE_DMALLOC
- #undef strndup
- #include <dmalloc.h>
- /* Macro to pass the extra dmalloc args to another function. */
- #define DMALLOC_FN_ARGS , file, line
-
- #if defined(HAVE_DMALLOC_STRDUP)
- /* the dmalloc_strdup should be fine as defined */
- #elif defined(HAVE_DMALLOC_STRNDUP)
- #define dmalloc_strdup(file, line, string, xalloc_b) \
-         dmalloc_strndup(file, line, (string), -1, xalloc_b)
- #else
- #error "No dmalloc_strdup or equivalent"
- #endif
-
-#else /* not using dmalloc */
-
- #define DMALLOC_FN_ARGS
-#endif
-
-/** Allocate a chunk of <b>size</b> bytes of memory, and return a pointer to
- * result.  On error, log and terminate the process.  (Same as malloc(size),
- * but never returns NULL.)
- *
- * <b>file</b> and <b>line</b> are used if dmalloc is enabled, and
- * ignored otherwise.
- */
-void *
-tor_malloc_(size_t size DMALLOC_PARAMS)
-{
-  void *result;
-
-  tor_assert(size < SIZE_T_CEILING);
-
-#ifndef MALLOC_ZERO_WORKS
-  /* Some libc mallocs don't work when size==0. Override them. */
-  if (size==0) {
-    size=1;
-  }
-#endif
-
-#ifdef USE_DMALLOC
-  result = dmalloc_malloc(file, line, size, DMALLOC_FUNC_MALLOC, 0, 0);
-#else
-  result = raw_malloc(size);
-#endif
-
-  if (PREDICT_UNLIKELY(result == NULL)) {
-    /* LCOV_EXCL_START */
-    log_err(LD_MM,"Out of memory on malloc(). Dying.");
-    /* If these functions die within a worker process, they won't call
-     * spawn_exit, but that's ok, since the parent will run out of memory soon
-     * anyway. */
-    exit(1);
-    /* LCOV_EXCL_STOP */
-  }
-  return result;
-}
-
-/** Allocate a chunk of <b>size</b> bytes of memory, fill the memory with
- * zero bytes, and return a pointer to the result.  Log and terminate
- * the process on error.  (Same as calloc(size,1), but never returns NULL.)
- */
-void *
-tor_malloc_zero_(size_t size DMALLOC_PARAMS)
-{
-  /* You may ask yourself, "wouldn't it be smart to use calloc instead of
-   * malloc+memset?  Perhaps libc's calloc knows some nifty optimization trick
-   * we don't!"  Indeed it does, but its optimizations are only a big win when
-   * we're allocating something very big (it knows if it just got the memory
-   * from the OS in a pre-zeroed state).  We don't want to use tor_malloc_zero
-   * for big stuff, so we don't bother with calloc. */
-  void *result = tor_malloc_(size DMALLOC_FN_ARGS);
-  memset(result, 0, size);
-  return result;
-}
-
-/* The square root of SIZE_MAX + 1.  If a is less than this, and b is less
- * than this, then a*b is less than SIZE_MAX.  (For example, if size_t is
- * 32 bits, then SIZE_MAX is 0xffffffff and this value is 0x10000.  If a and
- * b are less than this, then their product is at most (65535*65535) ==
- * 0xfffe0001. */
-#define SQRT_SIZE_MAX_P1 (((size_t)1) << (sizeof(size_t)*4))
-
-/** Return non-zero if and only if the product of the arguments is exact. */
-static inline int
-size_mul_check(const size_t x, const size_t y)
-{
-  /* This first check is equivalent to
-     (x < SQRT_SIZE_MAX_P1 && y < SQRT_SIZE_MAX_P1)
-
-     Rationale: if either one of x or y is >= SQRT_SIZE_MAX_P1, then it
-     will have some bit set in its most significant half.
-   */
-  return ((x|y) < SQRT_SIZE_MAX_P1 ||
-          y == 0 ||
-          x <= SIZE_MAX / y);
-}
-
-#ifdef TOR_UNIT_TESTS
-/** Exposed for unit tests only */
-int
-size_mul_check__(const size_t x, const size_t y)
-{
-  return size_mul_check(x,y);
-}
-#endif
-
-/** Allocate a chunk of <b>nmemb</b>*<b>size</b> bytes of memory, fill
- * the memory with zero bytes, and return a pointer to the result.
- * Log and terminate the process on error.  (Same as
- * calloc(<b>nmemb</b>,<b>size</b>), but never returns NULL.)
- * The second argument (<b>size</b>) should preferably be non-zero
- * and a compile-time constant.
- */
-void *
-tor_calloc_(size_t nmemb, size_t size DMALLOC_PARAMS)
-{
-  tor_assert(size_mul_check(nmemb, size));
-  return tor_malloc_zero_((nmemb * size) DMALLOC_FN_ARGS);
-}
-
-/** Change the size of the memory block pointed to by <b>ptr</b> to <b>size</b>
- * bytes long; return the new memory block.  On error, log and
- * terminate. (Like realloc(ptr,size), but never returns NULL.)
- */
-void *
-tor_realloc_(void *ptr, size_t size DMALLOC_PARAMS)
-{
-  void *result;
-
-  tor_assert(size < SIZE_T_CEILING);
-
-#ifndef MALLOC_ZERO_WORKS
-  /* Some libc mallocs don't work when size==0. Override them. */
-  if (size==0) {
-    size=1;
-  }
-#endif
-
-#ifdef USE_DMALLOC
-  result = dmalloc_realloc(file, line, ptr, size, DMALLOC_FUNC_REALLOC, 0);
-#else
-  result = raw_realloc(ptr, size);
-#endif
-
-  if (PREDICT_UNLIKELY(result == NULL)) {
-    /* LCOV_EXCL_START */
-    log_err(LD_MM,"Out of memory on realloc(). Dying.");
-    exit(1);
-    /* LCOV_EXCL_STOP */
-  }
-  return result;
-}
-
-/**
- * Try to realloc <b>ptr</b> so that it takes up sz1 * sz2 bytes.  Check for
- * overflow. Unlike other allocation functions, return NULL on overflow.
- */
-void *
-tor_reallocarray_(void *ptr, size_t sz1, size_t sz2 DMALLOC_PARAMS)
-{
-  /* XXXX we can make this return 0, but we would need to check all the
-   * reallocarray users. */
-  tor_assert(size_mul_check(sz1, sz2));
-
-  return tor_realloc(ptr, (sz1 * sz2) DMALLOC_FN_ARGS);
-}
-
-/** Return a newly allocated copy of the NUL-terminated string s. On
- * error, log and terminate.  (Like strdup(s), but never returns
- * NULL.)
- */
-char *
-tor_strdup_(const char *s DMALLOC_PARAMS)
-{
-  char *duplicate;
-  tor_assert(s);
-
-#ifdef USE_DMALLOC
-  duplicate = dmalloc_strdup(file, line, s, 0);
-#else
-  duplicate = raw_strdup(s);
-#endif
-  if (PREDICT_UNLIKELY(duplicate == NULL)) {
-    /* LCOV_EXCL_START */
-    log_err(LD_MM,"Out of memory on strdup(). Dying.");
-    exit(1);
-    /* LCOV_EXCL_STOP */
-  }
-  return duplicate;
-}
-
-/** Allocate and return a new string containing the first <b>n</b>
- * characters of <b>s</b>.  If <b>s</b> is longer than <b>n</b>
- * characters, only the first <b>n</b> are copied.  The result is
- * always NUL-terminated.  (Like strndup(s,n), but never returns
- * NULL.)
- */
-char *
-tor_strndup_(const char *s, size_t n DMALLOC_PARAMS)
-{
-  char *duplicate;
-  tor_assert(s);
-  tor_assert(n < SIZE_T_CEILING);
-  duplicate = tor_malloc_((n+1) DMALLOC_FN_ARGS);
-  /* Performance note: Ordinarily we prefer strlcpy to strncpy.  But
-   * this function gets called a whole lot, and platform strncpy is
-   * much faster than strlcpy when strlen(s) is much longer than n.
-   */
-  strncpy(duplicate, s, n);
-  duplicate[n]='\0';
-  return duplicate;
-}
-
-/** Allocate a chunk of <b>len</b> bytes, with the same contents as the
- * <b>len</b> bytes starting at <b>mem</b>. */
-void *
-tor_memdup_(const void *mem, size_t len DMALLOC_PARAMS)
-{
-  char *duplicate;
-  tor_assert(len < SIZE_T_CEILING);
-  tor_assert(mem);
-  duplicate = tor_malloc_(len DMALLOC_FN_ARGS);
-  memcpy(duplicate, mem, len);
-  return duplicate;
-}
-
-/** As tor_memdup(), but add an extra 0 byte at the end of the resulting
- * memory. */
-void *
-tor_memdup_nulterm_(const void *mem, size_t len DMALLOC_PARAMS)
-{
-  char *duplicate;
-  tor_assert(len < SIZE_T_CEILING+1);
-  tor_assert(mem);
-  duplicate = tor_malloc_(len+1 DMALLOC_FN_ARGS);
-  memcpy(duplicate, mem, len);
-  duplicate[len] = '\0';
-  return duplicate;
-}
-
-/** Helper for places that need to take a function pointer to the right
- * spelling of "free()". */
-void
-tor_free_(void *mem)
-{
-  tor_free(mem);
-}
-
-DISABLE_GCC_WARNING(aggregate-return)
-/** Call the platform malloc info function, and dump the results to the log at
- * level <b>severity</b>.  If no such function exists, do nothing. */
-void
-tor_log_mallinfo(int severity)
-{
-#ifdef HAVE_MALLINFO
-  struct mallinfo mi;
-  memset(&mi, 0, sizeof(mi));
-  mi = mallinfo();
-  tor_log(severity, LD_MM,
-      "mallinfo() said: arena=%d, ordblks=%d, smblks=%d, hblks=%d, "
-      "hblkhd=%d, usmblks=%d, fsmblks=%d, uordblks=%d, fordblks=%d, "
-      "keepcost=%d",
-      mi.arena, mi.ordblks, mi.smblks, mi.hblks,
-      mi.hblkhd, mi.usmblks, mi.fsmblks, mi.uordblks, mi.fordblks,
-      mi.keepcost);
-#else
-  (void)severity;
-#endif
-#ifdef USE_DMALLOC
-  dmalloc_log_changed(0, /* Since the program started. */
-                      1, /* Log info about non-freed pointers. */
-                      0, /* Do not log info about freed pointers. */
-                      0  /* Do not log individual pointers. */
-                      );
-#endif
-}
-ENABLE_GCC_WARNING(aggregate-return)
-
-/* =====
- * Math
- * ===== */
-
-/**
- * Returns the natural logarithm of d base e.  We defined this wrapper here so
- * to avoid conflicts with old versions of tor_log(), which were named log().
- */
-double
-tor_mathlog(double d)
-{
-  return log(d);
-}
-
-/** Return the long integer closest to <b>d</b>. We define this wrapper
- * here so that not all users of math.h need to use the right incantations
- * to get the c99 functions. */
-long
-tor_lround(double d)
-{
-#if defined(HAVE_LROUND)
-  return lround(d);
-#elif defined(HAVE_RINT)
-  return (long)rint(d);
-#else
-  return (long)(d > 0 ? d + 0.5 : ceil(d - 0.5));
-#endif
-}
-
-/** Return the 64-bit integer closest to d.  We define this wrapper here so
- * that not all users of math.h need to use the right incantations to get the
- * c99 functions. */
-int64_t
-tor_llround(double d)
-{
-#if defined(HAVE_LLROUND)
-  return (int64_t)llround(d);
-#elif defined(HAVE_RINT)
-  return (int64_t)rint(d);
-#else
-  return (int64_t)(d > 0 ? d + 0.5 : ceil(d - 0.5));
-#endif
-}
-
-/** Returns floor(log2(u64)).  If u64 is 0, (incorrectly) returns 0. */
-int
-tor_log2(uint64_t u64)
-{
-  int r = 0;
-  if (u64 >= (U64_LITERAL(1)<<32)) {
-    u64 >>= 32;
-    r = 32;
-  }
-  if (u64 >= (U64_LITERAL(1)<<16)) {
-    u64 >>= 16;
-    r += 16;
-  }
-  if (u64 >= (U64_LITERAL(1)<<8)) {
-    u64 >>= 8;
-    r += 8;
-  }
-  if (u64 >= (U64_LITERAL(1)<<4)) {
-    u64 >>= 4;
-    r += 4;
-  }
-  if (u64 >= (U64_LITERAL(1)<<2)) {
-    u64 >>= 2;
-    r += 2;
-  }
-  if (u64 >= (U64_LITERAL(1)<<1)) {
-    u64 >>= 1;
-    r += 1;
-  }
-  return r;
-}
-
-/** Return the power of 2 in range [1,UINT64_MAX] closest to <b>u64</b>.  If
- * there are two powers of 2 equally close, round down. */
-uint64_t
-round_to_power_of_2(uint64_t u64)
-{
-  int lg2;
-  uint64_t low;
-  uint64_t high;
-  if (u64 == 0)
-    return 1;
-
-  lg2 = tor_log2(u64);
-  low = U64_LITERAL(1) << lg2;
-
-  if (lg2 == 63)
-    return low;
-
-  high = U64_LITERAL(1) << (lg2+1);
-  if (high - u64 < u64 - low)
-    return high;
-  else
-    return low;
-}
-
-/** Return the lowest x such that x is at least <b>number</b>, and x modulo
- * <b>divisor</b> == 0.  If no such x can be expressed as an unsigned, return
- * UINT_MAX */
-unsigned
-round_to_next_multiple_of(unsigned number, unsigned divisor)
-{
-  tor_assert(divisor > 0);
-  if (UINT_MAX - divisor + 1 < number)
-    return UINT_MAX;
-  number += divisor - 1;
-  number -= number % divisor;
-  return number;
-}
-
-/** Return the lowest x such that x is at least <b>number</b>, and x modulo
- * <b>divisor</b> == 0. If no such x can be expressed as a uint32_t, return
- * UINT32_MAX */
-uint32_t
-round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor)
-{
-  tor_assert(divisor > 0);
-  if (UINT32_MAX - divisor + 1 < number)
-    return UINT32_MAX;
-
-  number += divisor - 1;
-  number -= number % divisor;
-  return number;
-}
-
-/** Return the lowest x such that x is at least <b>number</b>, and x modulo
- * <b>divisor</b> == 0. If no such x can be expressed as a uint64_t, return
- * UINT64_MAX */
-uint64_t
-round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor)
-{
-  tor_assert(divisor > 0);
-  if (UINT64_MAX - divisor + 1 < number)
-    return UINT64_MAX;
-  number += divisor - 1;
-  number -= number % divisor;
-  return number;
-}
-
-/** Transform a random value <b>p</b> from the uniform distribution in
- * [0.0, 1.0[ into a Laplace distributed value with location parameter
- * <b>mu</b> and scale parameter <b>b</b>. Truncate the final result
- * to be an integer in [INT64_MIN, INT64_MAX]. */
-int64_t
-sample_laplace_distribution(double mu, double b, double p)
-{
-  double result;
-  tor_assert(p >= 0.0 && p < 1.0);
-
-  /* This is the "inverse cumulative distribution function" from:
-   * http://en.wikipedia.org/wiki/Laplace_distribution */
-  if (p <= 0.0) {
-    /* Avoid taking log(0.0) == -INFINITY, as some processors or compiler
-     * options can cause the program to trap. */
-    return INT64_MIN;
-  }
-
-  result = mu - b * (p > 0.5 ? 1.0 : -1.0)
-                  * tor_mathlog(1.0 - 2.0 * fabs(p - 0.5));
-
-  return clamp_double_to_int64(result);
-}
-
-/** Add random noise between INT64_MIN and INT64_MAX coming from a Laplace
- * distribution with mu = 0 and b = <b>delta_f</b>/<b>epsilon</b> to
- * <b>signal</b> based on the provided <b>random</b> value in [0.0, 1.0[.
- * The epsilon value must be between ]0.0, 1.0]. delta_f must be greater
- * than 0. */
-int64_t
-add_laplace_noise(int64_t signal_, double random_, double delta_f,
-                  double epsilon)
-{
-  int64_t noise;
-
-  /* epsilon MUST be between ]0.0, 1.0] */
-  tor_assert(epsilon > 0.0 && epsilon <= 1.0);
-  /* delta_f MUST be greater than 0. */
-  tor_assert(delta_f > 0.0);
-
-  /* Just add noise, no further signal */
-  noise = sample_laplace_distribution(0.0,
-                                      delta_f / epsilon,
-                                      random_);
-
-  /* Clip (signal + noise) to [INT64_MIN, INT64_MAX] */
-  if (noise > 0 && INT64_MAX - noise < signal_)
-    return INT64_MAX;
-  else if (noise < 0 && INT64_MIN - noise > signal_)
-    return INT64_MIN;
-  else
-    return signal_ + noise;
-}
-
-/* Helper: return greatest common divisor of a,b */
-static uint64_t
-gcd64(uint64_t a, uint64_t b)
-{
-  while (b) {
-    uint64_t t = b;
-    b = a % b;
-    a = t;
-  }
-  return a;
-}
-
-/* Given a fraction *<b>numer</b> / *<b>denom</b>, simplify it.
- * Requires that the denominator is greater than 0. */
-void
-simplify_fraction64(uint64_t *numer, uint64_t *denom)
-{
-  tor_assert(denom);
-  uint64_t gcd = gcd64(*numer, *denom);
-  *numer /= gcd;
-  *denom /= gcd;
-}
-
-/** Return the number of bits set in <b>v</b>. */
-int
-n_bits_set_u8(uint8_t v)
-{
-  static const int nybble_table[] = {
-    0, /* 0000 */
-    1, /* 0001 */
-    1, /* 0010 */
-    2, /* 0011 */
-    1, /* 0100 */
-    2, /* 0101 */
-    2, /* 0110 */
-    3, /* 0111 */
-    1, /* 1000 */
-    2, /* 1001 */
-    2, /* 1010 */
-    3, /* 1011 */
-    2, /* 1100 */
-    3, /* 1101 */
-    3, /* 1110 */
-    4, /* 1111 */
-  };
-
-  return nybble_table[v & 15] + nybble_table[v>>4];
-}
-
-/* =====
- * String manipulation
- * ===== */
-
-/** Remove from the string <b>s</b> every character which appears in
- * <b>strip</b>. */
-void
-tor_strstrip(char *s, const char *strip)
-{
-  char *readp = s;
-  while (*readp) {
-    if (strchr(strip, *readp)) {
-      ++readp;
-    } else {
-      *s++ = *readp++;
-    }
-  }
-  *s = '\0';
-}
-
-/** Return a pointer to a NUL-terminated hexadecimal string encoding
- * the first <b>fromlen</b> bytes of <b>from</b>. (fromlen must be \<= 32.) The
- * result does not need to be deallocated, but repeated calls to
- * hex_str will trash old results.
- */
-const char *
-hex_str(const char *from, size_t fromlen)
-{
-  static char buf[65];
-  if (fromlen>(sizeof(buf)-1)/2)
-    fromlen = (sizeof(buf)-1)/2;
-  base16_encode(buf,sizeof(buf),from,fromlen);
-  return buf;
-}
-
-/** Convert all alphabetic characters in the nul-terminated string <b>s</b> to
- * lowercase. */
-void
-tor_strlower(char *s)
-{
-  while (*s) {
-    *s = TOR_TOLOWER(*s);
-    ++s;
-  }
-}
-
-/** Convert all alphabetic characters in the nul-terminated string <b>s</b> to
- * lowercase. */
-void
-tor_strupper(char *s)
-{
-  while (*s) {
-    *s = TOR_TOUPPER(*s);
-    ++s;
-  }
-}
-
-/** Return 1 if every character in <b>s</b> is printable, else return 0.
- */
-int
-tor_strisprint(const char *s)
-{
-  while (*s) {
-    if (!TOR_ISPRINT(*s))
-      return 0;
-    s++;
-  }
-  return 1;
-}
-
-/** Return 1 if no character in <b>s</b> is uppercase, else return 0.
- */
-int
-tor_strisnonupper(const char *s)
-{
-  while (*s) {
-    if (TOR_ISUPPER(*s))
-      return 0;
-    s++;
-  }
-  return 1;
-}
-
-/** As strcmp, except that either string may be NULL.  The NULL string is
- * considered to be before any non-NULL string. */
-int
-strcmp_opt(const char *s1, const char *s2)
-{
-  if (!s1) {
-    if (!s2)
-      return 0;
-    else
-      return -1;
-  } else if (!s2) {
-    return 1;
-  } else {
-    return strcmp(s1, s2);
-  }
-}
-
-/** Compares the first strlen(s2) characters of s1 with s2.  Returns as for
- * strcmp.
- */
-int
-strcmpstart(const char *s1, const char *s2)
-{
-  size_t n = strlen(s2);
-  return strncmp(s1, s2, n);
-}
-
-/** Compare the s1_len-byte string <b>s1</b> with <b>s2</b>,
- * without depending on a terminating nul in s1.  Sorting order is first by
- * length, then lexically; return values are as for strcmp.
- */
-int
-strcmp_len(const char *s1, const char *s2, size_t s1_len)
-{
-  size_t s2_len = strlen(s2);
-  if (s1_len < s2_len)
-    return -1;
-  if (s1_len > s2_len)
-    return 1;
-  return fast_memcmp(s1, s2, s2_len);
-}
-
-/** Compares the first strlen(s2) characters of s1 with s2.  Returns as for
- * strcasecmp.
- */
-int
-strcasecmpstart(const char *s1, const char *s2)
-{
-  size_t n = strlen(s2);
-  return strncasecmp(s1, s2, n);
-}
-
-/** Compares the last strlen(s2) characters of s1 with s2.  Returns as for
- * strcmp.
- */
-int
-strcmpend(const char *s1, const char *s2)
-{
-  size_t n1 = strlen(s1), n2 = strlen(s2);
-  if (n2>n1)
-    return strcmp(s1,s2);
-  else
-    return strncmp(s1+(n1-n2), s2, n2);
-}
-
-/** Compares the last strlen(s2) characters of s1 with s2.  Returns as for
- * strcasecmp.
- */
-int
-strcasecmpend(const char *s1, const char *s2)
-{
-  size_t n1 = strlen(s1), n2 = strlen(s2);
-  if (n2>n1) /* then they can't be the same; figure out which is bigger */
-    return strcasecmp(s1,s2);
-  else
-    return strncasecmp(s1+(n1-n2), s2, n2);
-}
-
-/** Compare the value of the string <b>prefix</b> with the start of the
- * <b>memlen</b>-byte memory chunk at <b>mem</b>.  Return as for strcmp.
- *
- * [As fast_memcmp(mem, prefix, strlen(prefix)) but returns -1 if memlen is
- * less than strlen(prefix).]
- */
-int
-fast_memcmpstart(const void *mem, size_t memlen,
-                const char *prefix)
-{
-  size_t plen = strlen(prefix);
-  if (memlen < plen)
-    return -1;
-  return fast_memcmp(mem, prefix, plen);
-}
-
-/** Return a pointer to the first char of s that is not whitespace and
- * not a comment, or to the terminating NUL if no such character exists.
- */
-const char *
-eat_whitespace(const char *s)
-{
-  tor_assert(s);
-
-  while (1) {
-    switch (*s) {
-    case '\0':
-    default:
-      return s;
-    case ' ':
-    case '\t':
-    case '\n':
-    case '\r':
-      ++s;
-      break;
-    case '#':
-      ++s;
-      while (*s && *s != '\n')
-        ++s;
-    }
-  }
-}
-
-/** Return a pointer to the first char of s that is not whitespace and
- * not a comment, or to the terminating NUL if no such character exists.
- */
-const char *
-eat_whitespace_eos(const char *s, const char *eos)
-{
-  tor_assert(s);
-  tor_assert(eos && s <= eos);
-
-  while (s < eos) {
-    switch (*s) {
-    case '\0':
-    default:
-      return s;
-    case ' ':
-    case '\t':
-    case '\n':
-    case '\r':
-      ++s;
-      break;
-    case '#':
-      ++s;
-      while (s < eos && *s && *s != '\n')
-        ++s;
-    }
-  }
-  return s;
-}
-
-/** Return a pointer to the first char of s that is not a space or a tab
- * or a \\r, or to the terminating NUL if no such character exists. */
-const char *
-eat_whitespace_no_nl(const char *s)
-{
-  while (*s == ' ' || *s == '\t' || *s == '\r')
-    ++s;
-  return s;
-}
-
-/** As eat_whitespace_no_nl, but stop at <b>eos</b> whether we have
- * found a non-whitespace character or not. */
-const char *
-eat_whitespace_eos_no_nl(const char *s, const char *eos)
-{
-  while (s < eos && (*s == ' ' || *s == '\t' || *s == '\r'))
-    ++s;
-  return s;
-}
-
-/** Return a pointer to the first char of s that is whitespace or <b>#</b>,
- * or to the terminating NUL if no such character exists.
- */
-const char *
-find_whitespace(const char *s)
-{
-  /* tor_assert(s); */
-  while (1) {
-    switch (*s)
-    {
-    case '\0':
-    case '#':
-    case ' ':
-    case '\r':
-    case '\n':
-    case '\t':
-      return s;
-    default:
-      ++s;
-    }
-  }
-}
-
-/** As find_whitespace, but stop at <b>eos</b> whether we have found a
- * whitespace or not. */
-const char *
-find_whitespace_eos(const char *s, const char *eos)
-{
-  /* tor_assert(s); */
-  while (s < eos) {
-    switch (*s)
-    {
-    case '\0':
-    case '#':
-    case ' ':
-    case '\r':
-    case '\n':
-    case '\t':
-      return s;
-    default:
-      ++s;
-    }
-  }
-  return s;
-}
-
-/** Return the first occurrence of <b>needle</b> in <b>haystack</b> that
- * occurs at the start of a line (that is, at the beginning of <b>haystack</b>
- * or immediately after a newline).  Return NULL if no such string is found.
- */
-const char *
-find_str_at_start_of_line(const char *haystack, const char *needle)
-{
-  size_t needle_len = strlen(needle);
-
-  do {
-    if (!strncmp(haystack, needle, needle_len))
-      return haystack;
-
-    haystack = strchr(haystack, '\n');
-    if (!haystack)
-      return NULL;
-    else
-      ++haystack;
-  } while (*haystack);
-
-  return NULL;
-}
-
-/** Returns true if <b>string</b> could be a C identifier.
-    A C identifier must begin with a letter or an underscore and the
-    rest of its characters can be letters, numbers or underscores. No
-    length limit is imposed. */
-int
-string_is_C_identifier(const char *string)
-{
-  size_t iter;
-  size_t length = strlen(string);
-  if (!length)
-    return 0;
-
-  for (iter = 0; iter < length ; iter++) {
-    if (iter == 0) {
-      if (!(TOR_ISALPHA(string[iter]) ||
-            string[iter] == '_'))
-        return 0;
-    } else {
-      if (!(TOR_ISALPHA(string[iter]) ||
-            TOR_ISDIGIT(string[iter]) ||
-            string[iter] == '_'))
-        return 0;
-    }
-  }
-
-  return 1;
-}
-
-/** Return true iff the 'len' bytes at 'mem' are all zero. */
-int
-tor_mem_is_zero(const char *mem, size_t len)
-{
-  static const char ZERO[] = {
-    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
-  };
-  while (len >= sizeof(ZERO)) {
-    /* It's safe to use fast_memcmp here, since the very worst thing an
-     * attacker could learn is how many initial bytes of a secret were zero */
-    if (fast_memcmp(mem, ZERO, sizeof(ZERO)))
-      return 0;
-    len -= sizeof(ZERO);
-    mem += sizeof(ZERO);
-  }
-  /* Deal with leftover bytes. */
-  if (len)
-    return fast_memeq(mem, ZERO, len);
-
-  return 1;
-}
-
-/** Return true iff the DIGEST_LEN bytes in digest are all zero. */
-int
-tor_digest_is_zero(const char *digest)
-{
-  static const uint8_t ZERO_DIGEST[] = {
-    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
-  };
-  return tor_memeq(digest, ZERO_DIGEST, DIGEST_LEN);
-}
-
-/** Return true if <b>string</b> is a valid 'key=[value]' string.
- *  "value" is optional, to indicate the empty string. Log at logging
- *  <b>severity</b> if something ugly happens. */
-int
-string_is_key_value(int severity, const char *string)
-{
-  /* position of equal sign in string */
-  const char *equal_sign_pos = NULL;
-
-  tor_assert(string);
-
-  if (strlen(string) < 2) { /* "x=" is shortest args string */
-    tor_log(severity, LD_GENERAL, "'%s' is too short to be a k=v value.",
-            escaped(string));
-    return 0;
-  }
-
-  equal_sign_pos = strchr(string, '=');
-  if (!equal_sign_pos) {
-    tor_log(severity, LD_GENERAL, "'%s' is not a k=v value.", escaped(string));
-    return 0;
-  }
-
-  /* validate that the '=' is not in the beginning of the string. */
-  if (equal_sign_pos == string) {
-    tor_log(severity, LD_GENERAL, "'%s' is not a valid k=v value.",
-            escaped(string));
-    return 0;
-  }
-
-  return 1;
-}
-
-/** Return true if <b>string</b> represents a valid IPv4 adddress in
- * 'a.b.c.d' form.
- */
-int
-string_is_valid_ipv4_address(const char *string)
-{
-  struct in_addr addr;
-
-  return (tor_inet_pton(AF_INET,string,&addr) == 1);
-}
-
-/** Return true if <b>string</b> represents a valid IPv6 address in
- * a form that inet_pton() can parse.
- */
-int
-string_is_valid_ipv6_address(const char *string)
-{
-  struct in6_addr addr;
-
-  return (tor_inet_pton(AF_INET6,string,&addr) == 1);
-}
-
-/** Return true iff <b>string</b> matches a pattern of DNS names
- * that we allow Tor clients to connect to.
- *
- * Note: This allows certain technically invalid characters ('_') to cope
- * with misconfigured zones that have been encountered in the wild.
- */
-int
-string_is_valid_hostname(const char *string)
-{
-  int result = 1;
-  smartlist_t *components;
-
-  components = smartlist_new();
-
-  smartlist_split_string(components,string,".",0,0);
-
-  SMARTLIST_FOREACH_BEGIN(components, char *, c) {
-    if ((c[0] == '-') || (*c == '_')) {
-      result = 0;
-      break;
-    }
-
-    /* Allow a single terminating '.' used rarely to indicate domains
-     * are FQDNs rather than relative. */
-    if ((c_sl_idx > 0) && (c_sl_idx + 1 == c_sl_len) && !*c) {
-      continue;
-    }
-
-    do {
-      if ((*c >= 'a' && *c <= 'z') ||
-          (*c >= 'A' && *c <= 'Z') ||
-          (*c >= '0' && *c <= '9') ||
-          (*c == '-') || (*c == '_'))
-        c++;
-      else
-        result = 0;
-    } while (result && *c);
-
-  } SMARTLIST_FOREACH_END(c);
-
-  SMARTLIST_FOREACH_BEGIN(components, char *, c) {
-    tor_free(c);
-  } SMARTLIST_FOREACH_END(c);
-
-  smartlist_free(components);
-
-  return result;
-}
-
-/** Return true iff the DIGEST256_LEN bytes in digest are all zero. */
-int
-tor_digest256_is_zero(const char *digest)
-{
-  return tor_mem_is_zero(digest, DIGEST256_LEN);
-}
-
-/* Helper: common code to check whether the result of a strtol or strtoul or
- * strtoll is correct. */
-#define CHECK_STRTOX_RESULT()                           \
-  /* Did an overflow occur? */                          \
-  if (errno == ERANGE)                                  \
-    goto err;                                           \
-  /* Was at least one character converted? */           \
-  if (endptr == s)                                      \
-    goto err;                                           \
-  /* Were there unexpected unconverted characters? */   \
-  if (!next && *endptr)                                 \
-    goto err;                                           \
-  /* Illogical (max, min) inputs? */                    \
-  if (BUG(max < min))                                   \
-    goto err;                                           \
-  /* Is r within limits? */                             \
-  if (r < min || r > max)                               \
-    goto err;                                           \
-  if (ok) *ok = 1;                                      \
-  if (next) *next = endptr;                             \
-  return r;                                             \
- err:                                                   \
-  if (ok) *ok = 0;                                      \
-  if (next) *next = endptr;                             \
-  return 0
-
-/** Extract a long from the start of <b>s</b>, in the given numeric
- * <b>base</b>.  If <b>base</b> is 0, <b>s</b> is parsed as a decimal,
- * octal, or hex number in the syntax of a C integer literal.  If
- * there is unconverted data and <b>next</b> is provided, set
- * *<b>next</b> to the first unconverted character.  An error has
- * occurred if no characters are converted; or if there are
- * unconverted characters and <b>next</b> is NULL; or if the parsed
- * value is not between <b>min</b> and <b>max</b>.  When no error
- * occurs, return the parsed value and set *<b>ok</b> (if provided) to
- * 1.  When an error occurs, return 0 and set *<b>ok</b> (if provided)
- * to 0.
- */
-long
-tor_parse_long(const char *s, int base, long min, long max,
-               int *ok, char **next)
-{
-  char *endptr;
-  long r;
-
-  if (base < 0) {
-    if (ok)
-      *ok = 0;
-    return 0;
-  }
-
-  errno = 0;
-  r = strtol(s, &endptr, base);
-  CHECK_STRTOX_RESULT();
-}
-
-/** As tor_parse_long(), but return an unsigned long. */
-unsigned long
-tor_parse_ulong(const char *s, int base, unsigned long min,
-                unsigned long max, int *ok, char **next)
-{
-  char *endptr;
-  unsigned long r;
-
-  if (base < 0) {
-    if (ok)
-      *ok = 0;
-    return 0;
-  }
-
-  errno = 0;
-  r = strtoul(s, &endptr, base);
-  CHECK_STRTOX_RESULT();
-}
-
-/** As tor_parse_long(), but return a double. */
-double
-tor_parse_double(const char *s, double min, double max, int *ok, char **next)
-{
-  char *endptr;
-  double r;
-
-  errno = 0;
-  r = strtod(s, &endptr);
-  CHECK_STRTOX_RESULT();
-}
-
-/** As tor_parse_long, but return a uint64_t.  Only base 10 is guaranteed to
- * work for now. */
-uint64_t
-tor_parse_uint64(const char *s, int base, uint64_t min,
-                 uint64_t max, int *ok, char **next)
-{
-  char *endptr;
-  uint64_t r;
-
-  if (base < 0) {
-    if (ok)
-      *ok = 0;
-    return 0;
-  }
-
-  errno = 0;
-#ifdef HAVE_STRTOULL
-  r = (uint64_t)strtoull(s, &endptr, base);
-#elif defined(_WIN32)
-#if defined(_MSC_VER) && _MSC_VER < 1300
-  tor_assert(base <= 10);
-  r = (uint64_t)_atoi64(s);
-  endptr = (char*)s;
-  while (TOR_ISSPACE(*endptr)) endptr++;
-  while (TOR_ISDIGIT(*endptr)) endptr++;
-#else
-  r = (uint64_t)_strtoui64(s, &endptr, base);
-#endif
-#elif SIZEOF_LONG == 8
-  r = (uint64_t)strtoul(s, &endptr, base);
-#else
-#error "I don't know how to parse 64-bit numbers."
-#endif
-
-  CHECK_STRTOX_RESULT();
-}
-
-/** Allocate and return a new string representing the contents of <b>s</b>,
- * surrounded by quotes and using standard C escapes.
- *
- * Generally, we use this for logging values that come in over the network to
- * keep them from tricking users, and for sending certain values to the
- * controller.
- *
- * We trust values from the resolver, OS, configuration file, and command line
- * to not be maliciously ill-formed.  We validate incoming routerdescs and
- * SOCKS requests and addresses from BEGIN cells as they're parsed;
- * afterwards, we trust them as non-malicious.
- */
-char *
-esc_for_log(const char *s)
-{
-  const char *cp;
-  char *result, *outp;
-  size_t len = 3;
-  if (!s) {
-    return tor_strdup("(null)");
-  }
-
-  for (cp = s; *cp; ++cp) {
-    switch (*cp) {
-      case '\\':
-      case '\"':
-      case '\'':
-      case '\r':
-      case '\n':
-      case '\t':
-        len += 2;
-        break;
-      default:
-        if (TOR_ISPRINT(*cp) && ((uint8_t)*cp)<127)
-          ++len;
-        else
-          len += 4;
-        break;
-    }
-  }
-
-  tor_assert(len <= SSIZE_MAX);
-
-  result = outp = tor_malloc(len);
-  *outp++ = '\"';
-  for (cp = s; *cp; ++cp) {
-    /* This assertion should always succeed, since we will write at least
-     * one char here, and two chars for closing quote and nul later */
-    tor_assert((outp-result) < (ssize_t)len-2);
-    switch (*cp) {
-      case '\\':
-      case '\"':
-      case '\'':
-        *outp++ = '\\';
-        *outp++ = *cp;
-        break;
-      case '\n':
-        *outp++ = '\\';
-        *outp++ = 'n';
-        break;
-      case '\t':
-        *outp++ = '\\';
-        *outp++ = 't';
-        break;
-      case '\r':
-        *outp++ = '\\';
-        *outp++ = 'r';
-        break;
-      default:
-        if (TOR_ISPRINT(*cp) && ((uint8_t)*cp)<127) {
-          *outp++ = *cp;
-        } else {
-          tor_assert((outp-result) < (ssize_t)len-4);
-          tor_snprintf(outp, 5, "\\%03o", (int)(uint8_t) *cp);
-          outp += 4;
-        }
-        break;
-    }
-  }
-
-  tor_assert((outp-result) <= (ssize_t)len-2);
-  *outp++ = '\"';
-  *outp++ = 0;
-
-  return result;
-}
-
-/** Similar to esc_for_log. Allocate and return a new string representing
- * the first n characters in <b>chars</b>, surround by quotes and using
- * standard C escapes. If a NUL character is encountered in <b>chars</b>,
- * the resulting string will be terminated there.
- */
-char *
-esc_for_log_len(const char *chars, size_t n)
-{
-  char *string = tor_strndup(chars, n);
-  char *string_escaped = esc_for_log(string);
-  tor_free(string);
-  return string_escaped;
-}
-
-/** Allocate and return a new string representing the contents of <b>s</b>,
- * surrounded by quotes and using standard C escapes.
- *
- * THIS FUNCTION IS NOT REENTRANT.  Don't call it from outside the main
- * thread.  Also, each call invalidates the last-returned value, so don't
- * try log_warn(LD_GENERAL, "%s %s", escaped(a), escaped(b));
- */
-const char *
-escaped(const char *s)
-{
-  static char *escaped_val_ = NULL;
-  tor_free(escaped_val_);
-
-  if (s)
-    escaped_val_ = esc_for_log(s);
-  else
-    escaped_val_ = NULL;
-
-  return escaped_val_;
-}
-
-/** Return a newly allocated string equal to <b>string</b>, except that every
- * character in <b>chars_to_escape</b> is preceded by a backslash. */
-char *
-tor_escape_str_for_pt_args(const char *string, const char *chars_to_escape)
-{
-  char *new_string = NULL;
-  char *new_cp = NULL;
-  size_t length, new_length;
-
-  tor_assert(string);
-
-  length = strlen(string);
-
-  if (!length) /* If we were given the empty string, return the same. */
-    return tor_strdup("");
-  /* (new_length > SIZE_MAX) => ((length * 2) + 1 > SIZE_MAX) =>
-     (length*2 > SIZE_MAX - 1) => (length > (SIZE_MAX - 1)/2) */
-  if (length > (SIZE_MAX - 1)/2) /* check for overflow */
-    return NULL;
-
-  /* this should be enough even if all characters must be escaped */
-  new_length = (length * 2) + 1;
-
-  new_string = new_cp = tor_malloc(new_length);
-
-  while (*string) {
-    if (strchr(chars_to_escape, *string))
-      *new_cp++ = '\\';
-
-    *new_cp++ = *string++;
-  }
-
-  *new_cp = '\0'; /* NUL-terminate the new string */
-
-  return new_string;
-}
-
-/* =====
- * Time
- * ===== */
-
-#define TOR_USEC_PER_SEC 1000000
-
-/** Return the difference between start->tv_sec and end->tv_sec.
- * Returns INT64_MAX on overflow and underflow.
- */
-static int64_t
-tv_secdiff_impl(const struct timeval *start, const struct timeval *end)
-{
-  const int64_t s = (int64_t)start->tv_sec;
-  const int64_t e = (int64_t)end->tv_sec;
-
-  /* This may not be the most efficient way of implemeting this check,
-   * but it's easy to see that it's correct and doesn't overflow */
-
-  if (s > 0 && e < INT64_MIN + s) {
-    /* s is positive: equivalent to e - s < INT64_MIN, but without any
-     * overflow */
-    return INT64_MAX;
-  } else if (s < 0 && e > INT64_MAX + s) {
-    /* s is negative: equivalent to e - s > INT64_MAX, but without any
-     * overflow */
-    return INT64_MAX;
-  }
-
-  return e - s;
-}
-
-/** Return the number of microseconds elapsed between *start and *end.
- * Returns LONG_MAX on overflow and underflow.
- */
-long
-tv_udiff(const struct timeval *start, const struct timeval *end)
-{
-  /* Sanity check tv_usec */
-  if (start->tv_usec > TOR_USEC_PER_SEC || start->tv_usec < 0) {
-    log_warn(LD_GENERAL, "comparing times on microsecond detail with bad "
-             "start tv_usec: " I64_FORMAT " microseconds",
-             I64_PRINTF_ARG(start->tv_usec));
-    return LONG_MAX;
-  }
-
-  if (end->tv_usec > TOR_USEC_PER_SEC || end->tv_usec < 0) {
-    log_warn(LD_GENERAL, "comparing times on microsecond detail with bad "
-             "end tv_usec: " I64_FORMAT " microseconds",
-             I64_PRINTF_ARG(end->tv_usec));
-    return LONG_MAX;
-  }
-
-  /* Some BSDs have struct timeval.tv_sec 64-bit, but time_t (and long) 32-bit
-   */
-  int64_t udiff;
-  const int64_t secdiff = tv_secdiff_impl(start, end);
-
-  /* end->tv_usec - start->tv_usec can be up to 1 second either way */
-  if (secdiff > (int64_t)(LONG_MAX/1000000 - 1) ||
-      secdiff < (int64_t)(LONG_MIN/1000000 + 1)) {
-    log_warn(LD_GENERAL, "comparing times on microsecond detail too far "
-             "apart: " I64_FORMAT " seconds", I64_PRINTF_ARG(secdiff));
-    return LONG_MAX;
-  }
-
-  /* we'll never get an overflow here, because we check that both usecs are
-   * between 0 and TV_USEC_PER_SEC. */
-  udiff = secdiff*1000000 + ((int64_t)end->tv_usec - (int64_t)start->tv_usec);
-
-  /* Some compilers are smart enough to work out this is a no-op on L64 */
-#if SIZEOF_LONG < 8
-  if (udiff > (int64_t)LONG_MAX || udiff < (int64_t)LONG_MIN) {
-    return LONG_MAX;
-  }
-#endif
-
-  return (long)udiff;
-}
-
-/** Return the number of milliseconds elapsed between *start and *end.
- * If the tv_usec difference is 500, rounds away from zero.
- * Returns LONG_MAX on overflow and underflow.
- */
-long
-tv_mdiff(const struct timeval *start, const struct timeval *end)
-{
-  /* Sanity check tv_usec */
-  if (start->tv_usec > TOR_USEC_PER_SEC || start->tv_usec < 0) {
-    log_warn(LD_GENERAL, "comparing times on millisecond detail with bad "
-             "start tv_usec: " I64_FORMAT " microseconds",
-             I64_PRINTF_ARG(start->tv_usec));
-    return LONG_MAX;
-  }
-
-  if (end->tv_usec > TOR_USEC_PER_SEC || end->tv_usec < 0) {
-    log_warn(LD_GENERAL, "comparing times on millisecond detail with bad "
-             "end tv_usec: " I64_FORMAT " microseconds",
-             I64_PRINTF_ARG(end->tv_usec));
-    return LONG_MAX;
-  }
-
-  /* Some BSDs have struct timeval.tv_sec 64-bit, but time_t (and long) 32-bit
-   */
-  int64_t mdiff;
-  const int64_t secdiff = tv_secdiff_impl(start, end);
-
-  /* end->tv_usec - start->tv_usec can be up to 1 second either way, but the
-   * mdiff calculation may add another temporary second for rounding.
-   * Whether this actually causes overflow depends on the compiler's constant
-   * folding and order of operations. */
-  if (secdiff > (int64_t)(LONG_MAX/1000 - 2) ||
-      secdiff < (int64_t)(LONG_MIN/1000 + 1)) {
-    log_warn(LD_GENERAL, "comparing times on millisecond detail too far "
-             "apart: " I64_FORMAT " seconds", I64_PRINTF_ARG(secdiff));
-    return LONG_MAX;
-  }
-
-  /* Subtract and round */
-  mdiff = secdiff*1000 +
-      /* We add a million usec here to ensure that the result is positive,
-       * so that the round-towards-zero behavior of the division will give
-       * the right result for rounding to the nearest msec. Later we subtract
-       * 1000 in order to get the correct result.
-       * We'll never get an overflow here, because we check that both usecs are
-       * between 0 and TV_USEC_PER_SEC. */
-      ((int64_t)end->tv_usec - (int64_t)start->tv_usec + 500 + 1000000) / 1000
-      - 1000;
-
-  /* Some compilers are smart enough to work out this is a no-op on L64 */
-#if SIZEOF_LONG < 8
-  if (mdiff > (int64_t)LONG_MAX || mdiff < (int64_t)LONG_MIN) {
-    return LONG_MAX;
-  }
-#endif
-
-  return (long)mdiff;
-}
-
-/**
- * Converts timeval to milliseconds.
- */
-int64_t
-tv_to_msec(const struct timeval *tv)
-{
-  int64_t conv = ((int64_t)tv->tv_sec)*1000L;
-  /* Round ghetto-style */
-  conv += ((int64_t)tv->tv_usec+500)/1000L;
-  return conv;
-}
-
-/** Yield true iff <b>y</b> is a leap-year. */
-#define IS_LEAPYEAR(y) (!(y % 4) && ((y % 100) || !(y % 400)))
-/** Helper: Return the number of leap-days between Jan 1, y1 and Jan 1, y2. */
-static int
-n_leapdays(int year1, int year2)
-{
-  --year1;
-  --year2;
-  return (year2/4 - year1/4) - (year2/100 - year1/100)
-    + (year2/400 - year1/400);
-}
-/** Number of days per month in non-leap year; used by tor_timegm and
- * parse_rfc1123_time. */
-static const int days_per_month[] =
-  { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
-
-/** Compute a time_t given a struct tm.  The result is given in UTC, and
- * does not account for leap seconds.  Return 0 on success, -1 on failure.
- */
-int
-tor_timegm(const struct tm *tm, time_t *time_out)
-{
-  /* This is a pretty ironclad timegm implementation, snarfed from Python2.2.
-   * It's way more brute-force than fiddling with tzset().
-   *
-   * We use int64_t rather than time_t to avoid overflow on multiplication on
-   * platforms with 32-bit time_t. Since year is clipped to INT32_MAX, and
-   * since 365 * 24 * 60 * 60 is approximately 31 million, it's not possible
-   * for INT32_MAX years to overflow int64_t when converted to seconds. */
-  int64_t year, days, hours, minutes, seconds;
-  int i, invalid_year, dpm;
-
-  /* Initialize time_out to 0 for now, to avoid bad usage in case this function
-     fails and the caller ignores the return value. */
-  tor_assert(time_out);
-  *time_out = 0;
-
-  /* avoid int overflow on addition */
-  if (tm->tm_year < INT32_MAX-1900) {
-    year = tm->tm_year + 1900;
-  } else {
-    /* clamp year */
-    year = INT32_MAX;
-  }
-  invalid_year = (year < 1970 || tm->tm_year >= INT32_MAX-1900);
-
-  if (tm->tm_mon >= 0 && tm->tm_mon <= 11) {
-    dpm = days_per_month[tm->tm_mon];
-    if (tm->tm_mon == 1 && !invalid_year && IS_LEAPYEAR(tm->tm_year)) {
-      dpm = 29;
-    }
-  } else {
-    /* invalid month - default to 0 days per month */
-    dpm = 0;
-  }
-
-  if (invalid_year ||
-      tm->tm_mon < 0 || tm->tm_mon > 11 ||
-      tm->tm_mday < 1 || tm->tm_mday > dpm ||
-      tm->tm_hour < 0 || tm->tm_hour > 23 ||
-      tm->tm_min < 0 || tm->tm_min > 59 ||
-      tm->tm_sec < 0 || tm->tm_sec > 60) {
-    log_warn(LD_BUG, "Out-of-range argument to tor_timegm");
-    return -1;
-  }
-  days = 365 * (year-1970) + n_leapdays(1970,(int)year);
-  for (i = 0; i < tm->tm_mon; ++i)
-    days += days_per_month[i];
-  if (tm->tm_mon > 1 && IS_LEAPYEAR(year))
-    ++days;
-  days += tm->tm_mday - 1;
-  hours = days*24 + tm->tm_hour;
-
-  minutes = hours*60 + tm->tm_min;
-  seconds = minutes*60 + tm->tm_sec;
-  /* Check that "seconds" will fit in a time_t. On platforms where time_t is
-   * 32-bit, this check will fail for dates in and after 2038.
-   *
-   * We already know that "seconds" can't be negative because "year" >= 1970 */
-#if SIZEOF_TIME_T < 8
-  if (seconds < TIME_MIN || seconds > TIME_MAX) {
-    log_warn(LD_BUG, "Result does not fit in tor_timegm");
-    return -1;
-  }
-#endif
-  *time_out = (time_t)seconds;
-  return 0;
-}
-
-/* strftime is locale-specific, so we need to replace those parts */
-
-/** A c-locale array of 3-letter names of weekdays, starting with Sun. */
-static const char *WEEKDAY_NAMES[] =
-  { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" };
-/** A c-locale array of 3-letter names of months, starting with Jan. */
-static const char *MONTH_NAMES[] =
-  { "Jan", "Feb", "Mar", "Apr", "May", "Jun",
-    "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" };
-
-/** Set <b>buf</b> to the RFC1123 encoding of the UTC value of <b>t</b>.
- * The buffer must be at least RFC1123_TIME_LEN+1 bytes long.
- *
- * (RFC1123 format is "Fri, 29 Sep 2006 15:54:20 GMT". Note the "GMT"
- * rather than "UTC".)
- */
-void
-format_rfc1123_time(char *buf, time_t t)
-{
-  struct tm tm;
-
-  tor_gmtime_r(&t, &tm);
-
-  strftime(buf, RFC1123_TIME_LEN+1, "___, %d ___ %Y %H:%M:%S GMT", &tm);
-  tor_assert(tm.tm_wday >= 0);
-  tor_assert(tm.tm_wday <= 6);
-  memcpy(buf, WEEKDAY_NAMES[tm.tm_wday], 3);
-  tor_assert(tm.tm_mon >= 0);
-  tor_assert(tm.tm_mon <= 11);
-  memcpy(buf+8, MONTH_NAMES[tm.tm_mon], 3);
-}
-
-/** Parse the (a subset of) the RFC1123 encoding of some time (in UTC) from
- * <b>buf</b>, and store the result in *<b>t</b>.
- *
- * Note that we only accept the subset generated by format_rfc1123_time above,
- * not the full range of formats suggested by RFC 1123.
- *
- * Return 0 on success, -1 on failure.
-*/
-int
-parse_rfc1123_time(const char *buf, time_t *t)
-{
-  struct tm tm;
-  char month[4];
-  char weekday[4];
-  int i, m, invalid_year;
-  unsigned tm_mday, tm_year, tm_hour, tm_min, tm_sec;
-  unsigned dpm;
-
-  if (strlen(buf) != RFC1123_TIME_LEN)
-    return -1;
-  memset(&tm, 0, sizeof(tm));
-  if (tor_sscanf(buf, "%3s, %2u %3s %u %2u:%2u:%2u GMT", weekday,
-             &tm_mday, month, &tm_year, &tm_hour,
-             &tm_min, &tm_sec) < 7) {
-    char *esc = esc_for_log(buf);
-    log_warn(LD_GENERAL, "Got invalid RFC1123 time %s", esc);
-    tor_free(esc);
-    return -1;
-  }
-
-  m = -1;
-  for (i = 0; i < 12; ++i) {
-    if (!strcmp(month, MONTH_NAMES[i])) {
-      m = i;
-      break;
-    }
-  }
-  if (m<0) {
-    char *esc = esc_for_log(buf);
-    log_warn(LD_GENERAL, "Got invalid RFC1123 time %s: No such month", esc);
-    tor_free(esc);
-    return -1;
-  }
-  tm.tm_mon = m;
-
-  invalid_year = (tm_year >= INT32_MAX || tm_year < 1970);
-  tor_assert(m >= 0 && m <= 11);
-  dpm = days_per_month[m];
-  if (m == 1 && !invalid_year && IS_LEAPYEAR(tm_year)) {
-    dpm = 29;
-  }
-
-  if (invalid_year || tm_mday < 1 || tm_mday > dpm ||
-      tm_hour > 23 || tm_min > 59 || tm_sec > 60) {
-    char *esc = esc_for_log(buf);
-    log_warn(LD_GENERAL, "Got invalid RFC1123 time %s", esc);
-    tor_free(esc);
-    return -1;
-  }
-  tm.tm_mday = (int)tm_mday;
-  tm.tm_year = (int)tm_year;
-  tm.tm_hour = (int)tm_hour;
-  tm.tm_min = (int)tm_min;
-  tm.tm_sec = (int)tm_sec;
-
-  if (tm.tm_year < 1970) {
-    /* LCOV_EXCL_START
-     * XXXX I think this is dead code; we already checked for
-     *      invalid_year above. */
-    tor_assert_nonfatal_unreached();
-    char *esc = esc_for_log(buf);
-    log_warn(LD_GENERAL,
-             "Got invalid RFC1123 time %s. (Before 1970)", esc);
-    tor_free(esc);
-    return -1;
-    /* LCOV_EXCL_STOP */
-  }
-  tm.tm_year -= 1900;
-
-  return tor_timegm(&tm, t);
-}
-
-/** Set <b>buf</b> to the ISO8601 encoding of the local value of <b>t</b>.
- * The buffer must be at least ISO_TIME_LEN+1 bytes long.
- *
- * (ISO8601 format is 2006-10-29 10:57:20)
- */
-void
-format_local_iso_time(char *buf, time_t t)
-{
-  struct tm tm;
-  strftime(buf, ISO_TIME_LEN+1, "%Y-%m-%d %H:%M:%S", tor_localtime_r(&t, &tm));
-}
-
-/** Set <b>buf</b> to the ISO8601 encoding of the GMT value of <b>t</b>.
- * The buffer must be at least ISO_TIME_LEN+1 bytes long.
- */
-void
-format_iso_time(char *buf, time_t t)
-{
-  struct tm tm;
-  strftime(buf, ISO_TIME_LEN+1, "%Y-%m-%d %H:%M:%S", tor_gmtime_r(&t, &tm));
-}
-
-/** As format_iso_time, but use the yyyy-mm-ddThh:mm:ss format to avoid
- * embedding an internal space. */
-void
-format_iso_time_nospace(char *buf, time_t t)
-{
-  format_iso_time(buf, t);
-  buf[10] = 'T';
-}
-
-/** As format_iso_time_nospace, but include microseconds in decimal
- * fixed-point format.  Requires that buf be at least ISO_TIME_USEC_LEN+1
- * bytes long. */
-void
-format_iso_time_nospace_usec(char *buf, const struct timeval *tv)
-{
-  tor_assert(tv);
-  format_iso_time_nospace(buf, (time_t)tv->tv_sec);
-  tor_snprintf(buf+ISO_TIME_LEN, 8, ".%06d", (int)tv->tv_usec);
-}
-
-/** Given an ISO-formatted UTC time value (after the epoch) in <b>cp</b>,
- * parse it and store its value in *<b>t</b>.  Return 0 on success, -1 on
- * failure.  Ignore extraneous stuff in <b>cp</b> after the end of the time
- * string, unless <b>strict</b> is set. */
-int
-parse_iso_time_(const char *cp, time_t *t, int strict)
-{
-  struct tm st_tm;
-  unsigned int year=0, month=0, day=0, hour=0, minute=0, second=0;
-  int n_fields;
-  char extra_char;
-  n_fields = tor_sscanf(cp, "%u-%2u-%2u %2u:%2u:%2u%c", &year, &month,
-                        &day, &hour, &minute, &second, &extra_char);
-  if (strict ? (n_fields != 6) : (n_fields < 6)) {
-    char *esc = esc_for_log(cp);
-    log_warn(LD_GENERAL, "ISO time %s was unparseable", esc);
-    tor_free(esc);
-    return -1;
-  }
-  if (year < 1970 || month < 1 || month > 12 || day < 1 || day > 31 ||
-          hour > 23 || minute > 59 || second > 60 || year >= INT32_MAX) {
-    char *esc = esc_for_log(cp);
-    log_warn(LD_GENERAL, "ISO time %s was nonsensical", esc);
-    tor_free(esc);
-    return -1;
-  }
-  st_tm.tm_year = (int)year-1900;
-  st_tm.tm_mon = month-1;
-  st_tm.tm_mday = day;
-  st_tm.tm_hour = hour;
-  st_tm.tm_min = minute;
-  st_tm.tm_sec = second;
-  st_tm.tm_wday = 0; /* Should be ignored. */
-
-  if (st_tm.tm_year < 70) {
-    /* LCOV_EXCL_START
-     * XXXX I think this is dead code; we already checked for
-     *      year < 1970 above. */
-    tor_assert_nonfatal_unreached();
-    char *esc = esc_for_log(cp);
-    log_warn(LD_GENERAL, "Got invalid ISO time %s. (Before 1970)", esc);
-    tor_free(esc);
-    return -1;
-    /* LCOV_EXCL_STOP */
-  }
-  return tor_timegm(&st_tm, t);
-}
-
-/** Given an ISO-formatted UTC time value (after the epoch) in <b>cp</b>,
- * parse it and store its value in *<b>t</b>.  Return 0 on success, -1 on
- * failure. Reject the string if any characters are present after the time.
- */
-int
-parse_iso_time(const char *cp, time_t *t)
-{
-  return parse_iso_time_(cp, t, 1);
-}
-
-/** Given a <b>date</b> in one of the three formats allowed by HTTP (ugh),
- * parse it into <b>tm</b>.  Return 0 on success, negative on failure. */
-int
-parse_http_time(const char *date, struct tm *tm)
-{
-  const char *cp;
-  char month[4];
-  char wkday[4];
-  int i;
-  unsigned tm_mday, tm_year, tm_hour, tm_min, tm_sec;
-
-  tor_assert(tm);
-  memset(tm, 0, sizeof(*tm));
-
-  /* First, try RFC1123 or RFC850 format: skip the weekday.  */
-  if ((cp = strchr(date, ','))) {
-    ++cp;
-    if (*cp != ' ')
-      return -1;
-    ++cp;
-    if (tor_sscanf(cp, "%2u %3s %4u %2u:%2u:%2u GMT",
-               &tm_mday, month, &tm_year,
-               &tm_hour, &tm_min, &tm_sec) == 6) {
-      /* rfc1123-date */
-      tm_year -= 1900;
-    } else if (tor_sscanf(cp, "%2u-%3s-%2u %2u:%2u:%2u GMT",
-                      &tm_mday, month, &tm_year,
-                      &tm_hour, &tm_min, &tm_sec) == 6) {
-      /* rfc850-date */
-    } else {
-      return -1;
-    }
-  } else {
-    /* No comma; possibly asctime() format. */
-    if (tor_sscanf(date, "%3s %3s %2u %2u:%2u:%2u %4u",
-               wkday, month, &tm_mday,
-               &tm_hour, &tm_min, &tm_sec, &tm_year) == 7) {
-      tm_year -= 1900;
-    } else {
-      return -1;
-    }
-  }
-  tm->tm_mday = (int)tm_mday;
-  tm->tm_year = (int)tm_year;
-  tm->tm_hour = (int)tm_hour;
-  tm->tm_min = (int)tm_min;
-  tm->tm_sec = (int)tm_sec;
-  tm->tm_wday = 0; /* Leave this unset. */
-
-  month[3] = '\0';
-  /* Okay, now decode the month. */
-  /* set tm->tm_mon to dummy value so the check below fails. */
-  tm->tm_mon = -1;
-  for (i = 0; i < 12; ++i) {
-    if (!strcasecmp(MONTH_NAMES[i], month)) {
-      tm->tm_mon = i;
-    }
-  }
-
-  if (tm->tm_year < 0 ||
-      tm->tm_mon < 0  || tm->tm_mon > 11 ||
-      tm->tm_mday < 1 || tm->tm_mday > 31 ||
-      tm->tm_hour < 0 || tm->tm_hour > 23 ||
-      tm->tm_min < 0  || tm->tm_min > 59 ||
-      tm->tm_sec < 0  || tm->tm_sec > 60)
-    return -1; /* Out of range, or bad month. */
-
-  return 0;
-}
-
-/** Given an <b>interval</b> in seconds, try to write it to the
- * <b>out_len</b>-byte buffer in <b>out</b> in a human-readable form.
- * Return 0 on success, -1 on failure.
- */
-int
-format_time_interval(char *out, size_t out_len, long interval)
-{
-  /* We only report seconds if there's no hours. */
-  long sec = 0, min = 0, hour = 0, day = 0;
-
-  /* -LONG_MIN is LONG_MAX + 1, which causes signed overflow */
-  if (interval < -LONG_MAX)
-    interval = LONG_MAX;
-  else if (interval < 0)
-    interval = -interval;
-
-  if (interval >= 86400) {
-    day = interval / 86400;
-    interval %= 86400;
-  }
-  if (interval >= 3600) {
-    hour = interval / 3600;
-    interval %= 3600;
-  }
-  if (interval >= 60) {
-    min = interval / 60;
-    interval %= 60;
-  }
-  sec = interval;
-
-  if (day) {
-    return tor_snprintf(out, out_len, "%ld days, %ld hours, %ld minutes",
-                        day, hour, min);
-  } else if (hour) {
-    return tor_snprintf(out, out_len, "%ld hours, %ld minutes", hour, min);
-  } else if (min) {
-    return tor_snprintf(out, out_len, "%ld minutes, %ld seconds", min, sec);
-  } else {
-    return tor_snprintf(out, out_len, "%ld seconds", sec);
-  }
-}
-
-/* =====
- * Cached time
- * ===== */
-
-#ifndef TIME_IS_FAST
-/** Cached estimate of the current time.  Updated around once per second;
- * may be a few seconds off if we are really busy.  This is a hack to avoid
- * calling time(NULL) (which not everybody has optimized) on critical paths.
- */
-static time_t cached_approx_time = 0;
-
-/** Return a cached estimate of the current time from when
- * update_approx_time() was last called.  This is a hack to avoid calling
- * time(NULL) on critical paths: please do not even think of calling it
- * anywhere else. */
-time_t
-approx_time(void)
-{
-  return cached_approx_time;
-}
-
-/** Update the cached estimate of the current time.  This function SHOULD be
- * called once per second, and MUST be called before the first call to
- * get_approx_time. */
-void
-update_approx_time(time_t now)
-{
-  cached_approx_time = now;
-}
-#endif
-
-/* =====
- * Rate limiting
- * ===== */
-
-/** If the rate-limiter <b>lim</b> is ready at <b>now</b>, return the number
- * of calls to rate_limit_is_ready (including this one!) since the last time
- * rate_limit_is_ready returned nonzero.  Otherwise return 0.
- * If the call number hits <b>RATELIM_TOOMANY</b> limit, drop a warning
- * about this event and stop counting. */
-static int
-rate_limit_is_ready(ratelim_t *lim, time_t now)
-{
-  if (lim->rate + lim->last_allowed <= now) {
-    int res = lim->n_calls_since_last_time + 1;
-    lim->last_allowed = now;
-    lim->n_calls_since_last_time = 0;
-    return res;
-  } else {
-    if (lim->n_calls_since_last_time <= RATELIM_TOOMANY) {
-      ++lim->n_calls_since_last_time;
-    }
-
-    return 0;
-  }
-}
-
-/** If the rate-limiter <b>lim</b> is ready at <b>now</b>, return a newly
- * allocated string indicating how many messages were suppressed, suitable to
- * append to a log message.  Otherwise return NULL. */
-char *
-rate_limit_log(ratelim_t *lim, time_t now)
-{
-  int n;
-  if ((n = rate_limit_is_ready(lim, now))) {
-    if (n == 1) {
-      return tor_strdup("");
-    } else {
-      char *cp=NULL;
-      const char *opt_over = (n >= RATELIM_TOOMANY) ? "over " : "";
-      /* XXXX this is not exactly correct: the messages could have occurred
-       * any time between the old value of lim->allowed and now. */
-      tor_asprintf(&cp,
-                   " [%s%d similar message(s) suppressed in last %d seconds]",
-                   opt_over, n-1, lim->rate);
-      return cp;
-    }
-  } else {
-    return NULL;
-  }
-}
-
-/* =====
- * File helpers
- * ===== */
-
-/** Write <b>count</b> bytes from <b>buf</b> to <b>fd</b>.  <b>isSocket</b>
- * must be 1 if fd was returned by socket() or accept(), and 0 if fd
- * was returned by open().  Return the number of bytes written, or -1
- * on error.  Only use if fd is a blocking fd.  */
-ssize_t
-write_all(tor_socket_t fd, const char *buf, size_t count, int isSocket)
-{
-  size_t written = 0;
-  ssize_t result;
-  tor_assert(count < SSIZE_MAX);
-
-  while (written != count) {
-    if (isSocket)
-      result = tor_socket_send(fd, buf+written, count-written, 0);
-    else
-      result = write((int)fd, buf+written, count-written);
-    if (result<0)
-      return -1;
-    written += result;
-  }
-  return (ssize_t)count;
-}
-
-/** Read from <b>fd</b> to <b>buf</b>, until we get <b>count</b> bytes
- * or reach the end of the file. <b>isSocket</b> must be 1 if fd
- * was returned by socket() or accept(), and 0 if fd was returned by
- * open().  Return the number of bytes read, or -1 on error. Only use
- * if fd is a blocking fd. */
-ssize_t
-read_all(tor_socket_t fd, char *buf, size_t count, int isSocket)
-{
-  size_t numread = 0;
-  ssize_t result;
-
-  if (count > SIZE_T_CEILING || count > SSIZE_MAX) {
-    errno = EINVAL;
-    return -1;
-  }
-
-  while (numread != count) {
-    if (isSocket)
-      result = tor_socket_recv(fd, buf+numread, count-numread, 0);
-    else
-      result = read((int)fd, buf+numread, count-numread);
-    if (result<0)
-      return -1;
-    else if (result == 0)
-      break;
-    numread += result;
-  }
-  return (ssize_t)numread;
-}
-
-/*
- *    Filesystem operations.
- */
-
-/** Clean up <b>name</b> so that we can use it in a call to "stat".  On Unix,
- * we do nothing.  On Windows, we remove a trailing slash, unless the path is
- * the root of a disk. */
-static void
-clean_name_for_stat(char *name)
-{
-#ifdef _WIN32
-  size_t len = strlen(name);
-  if (!len)
-    return;
-  if (name[len-1]=='\\' || name[len-1]=='/') {
-    if (len == 1 || (len==3 && name[1]==':'))
-      return;
-    name[len-1]='\0';
-  }
-#else
-  (void)name;
-#endif
-}
-
-/** Wrapper for unlink() to make it mockable for the test suite; returns 0
- * if unlinking the file succeeded, -1 and sets errno if unlinking fails.
- */
-
-MOCK_IMPL(int,
-tor_unlink,(const char *pathname))
-{
-  return unlink(pathname);
-}
-
-/** Return:
- * FN_ERROR if filename can't be read, is NULL, or is zero-length,
- * FN_NOENT if it doesn't exist,
- * FN_FILE if it is a non-empty regular file, or a FIFO on unix-like systems,
- * FN_EMPTY for zero-byte regular files,
- * FN_DIR if it's a directory, and
- * FN_ERROR for any other file type.
- * On FN_ERROR and FN_NOENT, sets errno.  (errno is not set when FN_ERROR
- * is returned due to an unhandled file type.) */
-file_status_t
-file_status(const char *fname)
-{
-  struct stat st;
-  char *f;
-  int r;
-  if (!fname || strlen(fname) == 0) {
-    return FN_ERROR;
-  }
-  f = tor_strdup(fname);
-  clean_name_for_stat(f);
-  log_debug(LD_FS, "stat()ing %s", f);
-  r = stat(sandbox_intern_string(f), &st);
-  tor_free(f);
-  if (r) {
-    if (errno == ENOENT) {
-      return FN_NOENT;
-    }
-    return FN_ERROR;
-  }
-  if (st.st_mode & S_IFDIR) {
-    return FN_DIR;
-  } else if (st.st_mode & S_IFREG) {
-    if (st.st_size > 0) {
-      return FN_FILE;
-    } else if (st.st_size == 0) {
-      return FN_EMPTY;
-    } else {
-      return FN_ERROR;
-    }
-#ifndef _WIN32
-  } else if (st.st_mode & S_IFIFO) {
-    return FN_FILE;
-#endif
-  } else {
-    return FN_ERROR;
-  }
-}
-
-/** Check whether <b>dirname</b> exists and is private.  If yes return 0.
- * If <b>dirname</b> does not exist:
- *  - if <b>check</b>&CPD_CREATE, try to create it and return 0 on success.
- *  - if <b>check</b>&CPD_CHECK, and we think we can create it, return 0.
- *  - if <b>check</b>&CPD_CHECK is false, and the directory exists, return 0.
- *  - otherwise, return -1.
- * If CPD_GROUP_OK is set, then it's okay if the directory
- * is group-readable, but in all cases we create the directory mode 0700.
- * If CPD_GROUP_READ is set, existing directory behaves as CPD_GROUP_OK and
- * if the directory is created it will use mode 0750 with group read
- * permission. Group read privileges also assume execute permission
- * as norm for directories. If CPD_CHECK_MODE_ONLY is set, then we don't
- * alter the directory permissions if they are too permissive:
- * we just return -1.
- * When effective_user is not NULL, check permissions against the given user
- * and its primary group.
- */
-MOCK_IMPL(int,
-check_private_dir,(const char *dirname, cpd_check_t check,
-                   const char *effective_user))
-{
-  int r;
-  struct stat st;
-
-  tor_assert(dirname);
-
-#ifndef _WIN32
-  int fd;
-  const struct passwd *pw = NULL;
-  uid_t running_uid;
-  gid_t running_gid;
-
-  /*
-   * Goal is to harden the implementation by removing any
-   * potential for race between stat() and chmod().
-   * chmod() accepts filename as argument. If an attacker can move
-   * the file between stat() and chmod(), a potential race exists.
-   *
-   * Several suggestions taken from:
-   * https://developer.apple.com/library/mac/documentation/
-   *     Security/Conceptual/SecureCodingGuide/Articles/RaceConditions.html
-   */
-
-  /* Open directory.
-   * O_NOFOLLOW to ensure that it does not follow symbolic links */
-  fd = open(sandbox_intern_string(dirname), O_NOFOLLOW);
-
-  /* Was there an error? Maybe the directory does not exist? */
-  if (fd == -1) {
-
-    if (errno != ENOENT) {
-      /* Other directory error */
-      log_warn(LD_FS, "Directory %s cannot be read: %s", dirname,
-               strerror(errno));
-      return -1;
-    }
-
-    /* Received ENOENT: Directory does not exist */
-
-    /* Should we create the directory? */
-    if (check & CPD_CREATE) {
-      log_info(LD_GENERAL, "Creating directory %s", dirname);
-      if (check & CPD_GROUP_READ) {
-        r = mkdir(dirname, 0750);
-      } else {
-        r = mkdir(dirname, 0700);
-      }
-
-      /* check for mkdir() error */
-      if (r) {
-        log_warn(LD_FS, "Error creating directory %s: %s", dirname,
-            strerror(errno));
-        return -1;
-      }
-
-      /* we just created the directory. try to open it again.
-       * permissions on the directory will be checked again below.*/
-      fd = open(sandbox_intern_string(dirname), O_NOFOLLOW);
-
-      if (fd == -1)
-        return -1;
-      else
-        close(fd);
-
-    } else if (!(check & CPD_CHECK)) {
-      log_warn(LD_FS, "Directory %s does not exist.", dirname);
-      return -1;
-    }
-
-    /* XXXX In the case where check==CPD_CHECK, we should look at the
-     * parent directory a little harder. */
-    return 0;
-  }
-
-  tor_assert(fd >= 0);
-
-  //f = tor_strdup(dirname);
-  //clean_name_for_stat(f);
-  log_debug(LD_FS, "stat()ing %s", dirname);
-  //r = stat(sandbox_intern_string(f), &st);
-  r = fstat(fd, &st);
-  if (r == -1) {
-      log_warn(LD_FS, "fstat() on directory %s failed.", dirname);
-      close(fd);
-      return -1;
-  }
-  //tor_free(f);
-
-  /* check that dirname is a directory */
-  if (!(st.st_mode & S_IFDIR)) {
-    log_warn(LD_FS, "%s is not a directory", dirname);
-    close(fd);
-    return -1;
-  }
-
-  if (effective_user) {
-    /* Look up the user and group information.
-     * If we have a problem, bail out. */
-    pw = tor_getpwnam(effective_user);
-    if (pw == NULL) {
-      log_warn(LD_CONFIG, "Error setting configured user: %s not found",
-               effective_user);
-      close(fd);
-      return -1;
-    }
-    running_uid = pw->pw_uid;
-    running_gid = pw->pw_gid;
-  } else {
-    running_uid = getuid();
-    running_gid = getgid();
-  }
-  if (st.st_uid != running_uid) {
-    const struct passwd *pw_uid = NULL;
-    char *process_ownername = NULL;
-
-    pw_uid = tor_getpwuid(running_uid);
-    process_ownername = pw_uid ? tor_strdup(pw_uid->pw_name) :
-      tor_strdup("<unknown>");
-
-    pw_uid = tor_getpwuid(st.st_uid);
-
-    log_warn(LD_FS, "%s is not owned by this user (%s, %d) but by "
-        "%s (%d). Perhaps you are running Tor as the wrong user?",
-                         dirname, process_ownername, (int)running_uid,
-                         pw ? pw->pw_name : "<unknown>", (int)st.st_uid);
-
-    tor_free(process_ownername);
-    close(fd);
-    return -1;
-  }
-  if ( (check & (CPD_GROUP_OK|CPD_GROUP_READ))
-       && (st.st_gid != running_gid) && (st.st_gid != 0)) {
-    struct group *gr;
-    char *process_groupname = NULL;
-    gr = getgrgid(running_gid);
-    process_groupname = gr ? tor_strdup(gr->gr_name) : tor_strdup("<unknown>");
-    gr = getgrgid(st.st_gid);
-
-    log_warn(LD_FS, "%s is not owned by this group (%s, %d) but by group "
-             "%s (%d).  Are you running Tor as the wrong user?",
-             dirname, process_groupname, (int)running_gid,
-             gr ?  gr->gr_name : "<unknown>", (int)st.st_gid);
-
-    tor_free(process_groupname);
-    close(fd);
-    return -1;
-  }
-  unsigned unwanted_bits = 0;
-  if (check & (CPD_GROUP_OK|CPD_GROUP_READ)) {
-    unwanted_bits = 0027;
-  } else {
-    unwanted_bits = 0077;
-  }
-  unsigned check_bits_filter = ~0;
-  if (check & CPD_RELAX_DIRMODE_CHECK) {
-    check_bits_filter = 0022;
-  }
-  if ((st.st_mode & unwanted_bits & check_bits_filter) != 0) {
-    unsigned new_mode;
-    if (check & CPD_CHECK_MODE_ONLY) {
-      log_warn(LD_FS, "Permissions on directory %s are too permissive.",
-               dirname);
-      close(fd);
-      return -1;
-    }
-    log_warn(LD_FS, "Fixing permissions on directory %s", dirname);
-    new_mode = st.st_mode;
-    new_mode |= 0700; /* Owner should have rwx */
-    if (check & CPD_GROUP_READ) {
-      new_mode |= 0050; /* Group should have rx */
-    }
-    new_mode &= ~unwanted_bits; /* Clear the bits that we didn't want set...*/
-    if (fchmod(fd, new_mode)) {
-      log_warn(LD_FS, "Could not chmod directory %s: %s", dirname,
-               strerror(errno));
-      close(fd);
-      return -1;
-    } else {
-      close(fd);
-      return 0;
-    }
-  }
-  close(fd);
-#else
-  /* Win32 case: we can't open() a directory. */
-  (void)effective_user;
-
-  char *f = tor_strdup(dirname);
-  clean_name_for_stat(f);
-  log_debug(LD_FS, "stat()ing %s", f);
-  r = stat(sandbox_intern_string(f), &st);
-  tor_free(f);
-  if (r) {
-    if (errno != ENOENT) {
-      log_warn(LD_FS, "Directory %s cannot be read: %s", dirname,
-               strerror(errno));
-      return -1;
-    }
-    if (check & CPD_CREATE) {
-      log_info(LD_GENERAL, "Creating directory %s", dirname);
-      r = mkdir(dirname);
-      if (r) {
-        log_warn(LD_FS, "Error creating directory %s: %s", dirname,
-                 strerror(errno));
-        return -1;
-      }
-    } else if (!(check & CPD_CHECK)) {
-      log_warn(LD_FS, "Directory %s does not exist.", dirname);
-      return -1;
-    }
-    return 0;
-  }
-  if (!(st.st_mode & S_IFDIR)) {
-    log_warn(LD_FS, "%s is not a directory", dirname);
-    return -1;
-  }
-
-#endif
-  return 0;
-}
-
-/** Create a file named <b>fname</b> with the contents <b>str</b>.  Overwrite
- * the previous <b>fname</b> if possible.  Return 0 on success, -1 on failure.
- *
- * This function replaces the old file atomically, if possible.  This
- * function, and all other functions in util.c that create files, create them
- * with mode 0600.
- */
-MOCK_IMPL(int,
-write_str_to_file,(const char *fname, const char *str, int bin))
-{
-#ifdef _WIN32
-  if (!bin && strchr(str, '\r')) {
-    log_warn(LD_BUG,
-             "We're writing a text string that already contains a CR to %s",
-             escaped(fname));
-  }
-#endif
-  return write_bytes_to_file(fname, str, strlen(str), bin);
-}
-
-/** Represents a file that we're writing to, with support for atomic commit:
- * we can write into a temporary file, and either remove the file on
- * failure, or replace the original file on success. */
-struct open_file_t {
-  char *tempname; /**< Name of the temporary file. */
-  char *filename; /**< Name of the original file. */
-  unsigned rename_on_close:1; /**< Are we using the temporary file or not? */
-  unsigned binary:1; /**< Did we open in binary mode? */
-  int fd; /**< fd for the open file. */
-  FILE *stdio_file; /**< stdio wrapper for <b>fd</b>. */
-};
-
-/** Try to start writing to the file in <b>fname</b>, passing the flags
- * <b>open_flags</b> to the open() syscall, creating the file (if needed) with
- * access value <b>mode</b>.  If the O_APPEND flag is set, we append to the
- * original file.  Otherwise, we open a new temporary file in the same
- * directory, and either replace the original or remove the temporary file
- * when we're done.
- *
- * Return the fd for the newly opened file, and store working data in
- * *<b>data_out</b>.  The caller should not close the fd manually:
- * instead, call finish_writing_to_file() or abort_writing_to_file().
- * Returns -1 on failure.
- *
- * NOTE: When not appending, the flags O_CREAT and O_TRUNC are treated
- * as true and the flag O_EXCL is treated as false.
- *
- * NOTE: Ordinarily, O_APPEND means "seek to the end of the file before each
- * write()".  We don't do that.
- */
-int
-start_writing_to_file(const char *fname, int open_flags, int mode,
-                      open_file_t **data_out)
-{
-  open_file_t *new_file = tor_malloc_zero(sizeof(open_file_t));
-  const char *open_name;
-  int append = 0;
-
-  tor_assert(fname);
-  tor_assert(data_out);
-#if (O_BINARY != 0 && O_TEXT != 0)
-  tor_assert((open_flags & (O_BINARY|O_TEXT)) != 0);
-#endif
-  new_file->fd = -1;
-  new_file->filename = tor_strdup(fname);
-  if (open_flags & O_APPEND) {
-    open_name = fname;
-    new_file->rename_on_close = 0;
-    append = 1;
-    open_flags &= ~O_APPEND;
-  } else {
-    tor_asprintf(&new_file->tempname, "%s.tmp", fname);
-    open_name = new_file->tempname;
-    /* We always replace an existing temporary file if there is one. */
-    open_flags |= O_CREAT|O_TRUNC;
-    open_flags &= ~O_EXCL;
-    new_file->rename_on_close = 1;
-  }
-#if O_BINARY != 0
-  if (open_flags & O_BINARY)
-    new_file->binary = 1;
-#endif
-
-  new_file->fd = tor_open_cloexec(open_name, open_flags, mode);
-  if (new_file->fd < 0) {
-    log_warn(LD_FS, "Couldn't open \"%s\" (%s) for writing: %s",
-        open_name, fname, strerror(errno));
-    goto err;
-  }
-  if (append) {
-    if (tor_fd_seekend(new_file->fd) < 0) {
-      log_warn(LD_FS, "Couldn't seek to end of file \"%s\": %s", open_name,
-               strerror(errno));
-      goto err;
-    }
-  }
-
-  *data_out = new_file;
-
-  return new_file->fd;
-
- err:
-  if (new_file->fd >= 0)
-    close(new_file->fd);
-  *data_out = NULL;
-  tor_free(new_file->filename);
-  tor_free(new_file->tempname);
-  tor_free(new_file);
-  return -1;
-}
-
-/** Given <b>file_data</b> from start_writing_to_file(), return a stdio FILE*
- * that can be used to write to the same file.  The caller should not mix
- * stdio calls with non-stdio calls. */
-FILE *
-fdopen_file(open_file_t *file_data)
-{
-  tor_assert(file_data);
-  if (file_data->stdio_file)
-    return file_data->stdio_file;
-  tor_assert(file_data->fd >= 0);
-  if (!(file_data->stdio_file = fdopen(file_data->fd,
-                                       file_data->binary?"ab":"a"))) {
-    log_warn(LD_FS, "Couldn't fdopen \"%s\" [%d]: %s", file_data->filename,
-             file_data->fd, strerror(errno));
-  }
-  return file_data->stdio_file;
-}
-
-/** Combines start_writing_to_file with fdopen_file(): arguments are as
- * for start_writing_to_file, but  */
-FILE *
-start_writing_to_stdio_file(const char *fname, int open_flags, int mode,
-                            open_file_t **data_out)
-{
-  FILE *res;
-  if (start_writing_to_file(fname, open_flags, mode, data_out)<0)
-    return NULL;
-  if (!(res = fdopen_file(*data_out))) {
-    abort_writing_to_file(*data_out);
-    *data_out = NULL;
-  }
-  return res;
-}
-
-/** Helper function: close and free the underlying file and memory in
- * <b>file_data</b>.  If we were writing into a temporary file, then delete
- * that file (if abort_write is true) or replaces the target file with
- * the temporary file (if abort_write is false). */
-static int
-finish_writing_to_file_impl(open_file_t *file_data, int abort_write)
-{
-  int r = 0;
-
-  tor_assert(file_data && file_data->filename);
-  if (file_data->stdio_file) {
-    if (fclose(file_data->stdio_file)) {
-      log_warn(LD_FS, "Error closing \"%s\": %s", file_data->filename,
-               strerror(errno));
-      abort_write = r = -1;
-    }
-  } else if (file_data->fd >= 0 && close(file_data->fd) < 0) {
-    log_warn(LD_FS, "Error flushing \"%s\": %s", file_data->filename,
-             strerror(errno));
-    abort_write = r = -1;
-  }
-
-  if (file_data->rename_on_close) {
-    tor_assert(file_data->tempname && file_data->filename);
-    if (abort_write) {
-      int res = unlink(file_data->tempname);
-      if (res != 0) {
-        /* We couldn't unlink and we'll leave a mess behind */
-        log_warn(LD_FS, "Failed to unlink %s: %s",
-                 file_data->tempname, strerror(errno));
-        r = -1;
-      }
-    } else {
-      tor_assert(strcmp(file_data->filename, file_data->tempname));
-      if (replace_file(file_data->tempname, file_data->filename)) {
-        log_warn(LD_FS, "Error replacing \"%s\": %s", file_data->filename,
-                 strerror(errno));
-        r = -1;
-      }
-    }
-  }
-
-  tor_free(file_data->filename);
-  tor_free(file_data->tempname);
-  tor_free(file_data);
-
-  return r;
-}
-
-/** Finish writing to <b>file_data</b>: close the file handle, free memory as
- * needed, and if using a temporary file, replace the original file with
- * the temporary file. */
-int
-finish_writing_to_file(open_file_t *file_data)
-{
-  return finish_writing_to_file_impl(file_data, 0);
-}
-
-/** Finish writing to <b>file_data</b>: close the file handle, free memory as
- * needed, and if using a temporary file, delete it. */
-int
-abort_writing_to_file(open_file_t *file_data)
-{
-  return finish_writing_to_file_impl(file_data, 1);
-}
-
-/** Helper: given a set of flags as passed to open(2), open the file
- * <b>fname</b> and write all the sized_chunk_t structs in <b>chunks</b> to
- * the file.  Do so as atomically as possible e.g. by opening temp files and
- * renaming. */
-static int
-write_chunks_to_file_impl(const char *fname, const smartlist_t *chunks,
-                          int open_flags)
-{
-  open_file_t *file = NULL;
-  int fd;
-  ssize_t result;
-  fd = start_writing_to_file(fname, open_flags, 0600, &file);
-  if (fd<0)
-    return -1;
-  SMARTLIST_FOREACH(chunks, sized_chunk_t *, chunk,
-  {
-    result = write_all(fd, chunk->bytes, chunk->len, 0);
-    if (result < 0) {
-      log_warn(LD_FS, "Error writing to \"%s\": %s", fname,
-          strerror(errno));
-      goto err;
-    }
-    tor_assert((size_t)result == chunk->len);
-  });
-
-  return finish_writing_to_file(file);
- err:
-  abort_writing_to_file(file);
-  return -1;
-}
-
-/** Given a smartlist of sized_chunk_t, write them to a file
- * <b>fname</b>, overwriting or creating the file as necessary.
- * If <b>no_tempfile</b> is 0 then the file will be written
- * atomically. */
-int
-write_chunks_to_file(const char *fname, const smartlist_t *chunks, int bin,
-                     int no_tempfile)
-{
-  int flags = OPEN_FLAGS_REPLACE|(bin?O_BINARY:O_TEXT);
-
-  if (no_tempfile) {
-    /* O_APPEND stops write_chunks_to_file from using tempfiles */
-    flags |= O_APPEND;
-  }
-  return write_chunks_to_file_impl(fname, chunks, flags);
-}
-
-/** Write <b>len</b> bytes, starting at <b>str</b>, to <b>fname</b>
-    using the open() flags passed in <b>flags</b>. */
-static int
-write_bytes_to_file_impl(const char *fname, const char *str, size_t len,
-                         int flags)
-{
-  int r;
-  sized_chunk_t c = { str, len };
-  smartlist_t *chunks = smartlist_new();
-  smartlist_add(chunks, &c);
-  r = write_chunks_to_file_impl(fname, chunks, flags);
-  smartlist_free(chunks);
-  return r;
-}
-
-/** As write_str_to_file, but does not assume a NUL-terminated
- * string. Instead, we write <b>len</b> bytes, starting at <b>str</b>. */
-MOCK_IMPL(int,
-write_bytes_to_file,(const char *fname, const char *str, size_t len,
-                     int bin))
-{
-  return write_bytes_to_file_impl(fname, str, len,
-                                  OPEN_FLAGS_REPLACE|(bin?O_BINARY:O_TEXT));
-}
-
-/** As write_bytes_to_file, but if the file already exists, append the bytes
- * to the end of the file instead of overwriting it. */
-int
-append_bytes_to_file(const char *fname, const char *str, size_t len,
-                     int bin)
-{
-  return write_bytes_to_file_impl(fname, str, len,
-                                  OPEN_FLAGS_APPEND|(bin?O_BINARY:O_TEXT));
-}
-
-/** Like write_str_to_file(), but also return -1 if there was a file
-    already residing in <b>fname</b>. */
-int
-write_bytes_to_new_file(const char *fname, const char *str, size_t len,
-                        int bin)
-{
-  return write_bytes_to_file_impl(fname, str, len,
-                                  OPEN_FLAGS_DONT_REPLACE|
-                                  (bin?O_BINARY:O_TEXT));
-}
-
-/**
- * Read the contents of the open file <b>fd</b> presuming it is a FIFO
- * (or similar) file descriptor for which the size of the file isn't
- * known ahead of time. Return NULL on failure, and a NUL-terminated
- * string on success.  On success, set <b>sz_out</b> to the number of
- * bytes read.
- */
-char *
-read_file_to_str_until_eof(int fd, size_t max_bytes_to_read, size_t *sz_out)
-{
-  ssize_t r;
-  size_t pos = 0;
-  char *string = NULL;
-  size_t string_max = 0;
-
-  if (max_bytes_to_read+1 >= SIZE_T_CEILING) {
-    errno = EINVAL;
-    return NULL;
-  }
-
-  do {
-    /* XXXX This "add 1K" approach is a little goofy; if we care about
-     * performance here, we should be doubling.  But in practice we shouldn't
-     * be using this function on big files anyway. */
-    string_max = pos + 1024;
-    if (string_max > max_bytes_to_read)
-      string_max = max_bytes_to_read + 1;
-    string = tor_realloc(string, string_max);
-    r = read(fd, string + pos, string_max - pos - 1);
-    if (r < 0) {
-      int save_errno = errno;
-      tor_free(string);
-      errno = save_errno;
-      return NULL;
-    }
-
-    pos += r;
-  } while (r > 0 && pos < max_bytes_to_read);
-
-  tor_assert(pos < string_max);
-  *sz_out = pos;
-  string[pos] = '\0';
-  return string;
-}
-
-/** Read the contents of <b>filename</b> into a newly allocated
- * string; return the string on success or NULL on failure.
- *
- * If <b>stat_out</b> is provided, store the result of stat()ing the
- * file into <b>stat_out</b>.
- *
- * If <b>flags</b> &amp; RFTS_BIN, open the file in binary mode.
- * If <b>flags</b> &amp; RFTS_IGNORE_MISSING, don't warn if the file
- * doesn't exist.
- */
-/*
- * This function <em>may</em> return an erroneous result if the file
- * is modified while it is running, but must not crash or overflow.
- * Right now, the error case occurs when the file length grows between
- * the call to stat and the call to read_all: the resulting string will
- * be truncated.
- */
-MOCK_IMPL(char *,
-read_file_to_str, (const char *filename, int flags, struct stat *stat_out))
-{
-  int fd; /* router file */
-  struct stat statbuf;
-  char *string;
-  ssize_t r;
-  int bin = flags & RFTS_BIN;
-
-  tor_assert(filename);
-
-  fd = tor_open_cloexec(filename,O_RDONLY|(bin?O_BINARY:O_TEXT),0);
-  if (fd<0) {
-    int severity = LOG_WARN;
-    int save_errno = errno;
-    if (errno == ENOENT && (flags & RFTS_IGNORE_MISSING))
-      severity = LOG_INFO;
-    log_fn(severity, LD_FS,"Could not open \"%s\": %s",filename,
-           strerror(errno));
-    errno = save_errno;
-    return NULL;
-  }
-
-  if (fstat(fd, &statbuf)<0) {
-    int save_errno = errno;
-    close(fd);
-    log_warn(LD_FS,"Could not fstat \"%s\".",filename);
-    errno = save_errno;
-    return NULL;
-  }
-
-#ifndef _WIN32
-/** When we detect that we're reading from a FIFO, don't read more than
- * this many bytes.  It's insane overkill for most uses. */
-#define FIFO_READ_MAX (1024*1024)
-  if (S_ISFIFO(statbuf.st_mode)) {
-    size_t sz = 0;
-    string = read_file_to_str_until_eof(fd, FIFO_READ_MAX, &sz);
-    int save_errno = errno;
-    if (string && stat_out) {
-      statbuf.st_size = sz;
-      memcpy(stat_out, &statbuf, sizeof(struct stat));
-    }
-    close(fd);
-    if (!string)
-      errno = save_errno;
-    return string;
-  }
-#endif
-
-  if ((uint64_t)(statbuf.st_size)+1 >= SIZE_T_CEILING) {
-    close(fd);
-    errno = EINVAL;
-    return NULL;
-  }
-
-  string = tor_malloc((size_t)(statbuf.st_size+1));
-
-  r = read_all(fd,string,(size_t)statbuf.st_size,0);
-  if (r<0) {
-    int save_errno = errno;
-    log_warn(LD_FS,"Error reading from file \"%s\": %s", filename,
-             strerror(errno));
-    tor_free(string);
-    close(fd);
-    errno = save_errno;
-    return NULL;
-  }
-  string[r] = '\0'; /* NUL-terminate the result. */
-
-#if defined(_WIN32) || defined(__CYGWIN__)
-  if (!bin && strchr(string, '\r')) {
-    log_debug(LD_FS, "We didn't convert CRLF to LF as well as we hoped "
-              "when reading %s. Coping.",
-              filename);
-    tor_strstrip(string, "\r");
-    r = strlen(string);
-  }
-  if (!bin) {
-    statbuf.st_size = (size_t) r;
-  } else
-#endif
-    if (r != statbuf.st_size) {
-      /* Unless we're using text mode on win32, we'd better have an exact
-       * match for size. */
-      int save_errno = errno;
-      log_warn(LD_FS,"Could read only %d of %ld bytes of file \"%s\".",
-               (int)r, (long)statbuf.st_size,filename);
-      tor_free(string);
-      close(fd);
-      errno = save_errno;
-      return NULL;
-    }
-  close(fd);
-  if (stat_out) {
-    memcpy(stat_out, &statbuf, sizeof(struct stat));
-  }
-
-  return string;
-}
-
-#define TOR_ISODIGIT(c) ('0' <= (c) && (c) <= '7')
-
-/** Given a c-style double-quoted escaped string in <b>s</b>, extract and
- * decode its contents into a newly allocated string.  On success, assign this
- * string to *<b>result</b>, assign its length to <b>size_out</b> (if
- * provided), and return a pointer to the position in <b>s</b> immediately
- * after the string.  On failure, return NULL.
- */
-const char *
-unescape_string(const char *s, char **result, size_t *size_out)
-{
-  const char *cp;
-  char *out;
-  if (s[0] != '\"')
-    return NULL;
-  cp = s+1;
-  while (1) {
-    switch (*cp) {
-      case '\0':
-      case '\n':
-        return NULL;
-      case '\"':
-        goto end_of_loop;
-      case '\\':
-        if (cp[1] == 'x' || cp[1] == 'X') {
-          if (!(TOR_ISXDIGIT(cp[2]) && TOR_ISXDIGIT(cp[3])))
-            return NULL;
-          cp += 4;
-        } else if (TOR_ISODIGIT(cp[1])) {
-          cp += 2;
-          if (TOR_ISODIGIT(*cp)) ++cp;
-          if (TOR_ISODIGIT(*cp)) ++cp;
-        } else if (cp[1] == 'n' || cp[1] == 'r' || cp[1] == 't' || cp[1] == '"'
-                   || cp[1] == '\\' || cp[1] == '\'') {
-          cp += 2;
-        } else {
-          return NULL;
-        }
-        break;
-      default:
-        ++cp;
-        break;
-    }
-  }
- end_of_loop:
-  out = *result = tor_malloc(cp-s + 1);
-  cp = s+1;
-  while (1) {
-    switch (*cp)
-      {
-      case '\"':
-        *out = '\0';
-        if (size_out) *size_out = out - *result;
-        return cp+1;
-      case '\0':
-        /* LCOV_EXCL_START -- we caught this in parse_config_from_line. */
-        tor_fragile_assert();
-        tor_free(*result);
-        return NULL;
-        /* LCOV_EXCL_STOP */
-      case '\\':
-        switch (cp[1])
-          {
-          case 'n': *out++ = '\n'; cp += 2; break;
-          case 'r': *out++ = '\r'; cp += 2; break;
-          case 't': *out++ = '\t'; cp += 2; break;
-          case 'x': case 'X':
-            {
-              int x1, x2;
-
-              x1 = hex_decode_digit(cp[2]);
-              x2 = hex_decode_digit(cp[3]);
-              if (x1 == -1 || x2 == -1) {
-                /* LCOV_EXCL_START */
-                /* we caught this above in the initial loop. */
-                tor_assert_nonfatal_unreached();
-                tor_free(*result);
-                return NULL;
-                /* LCOV_EXCL_STOP */
-              }
-
-              *out++ = ((x1<<4) + x2);
-              cp += 4;
-            }
-            break;
-          case '0': case '1': case '2': case '3': case '4': case '5':
-          case '6': case '7':
-            {
-              int n = cp[1]-'0';
-              cp += 2;
-              if (TOR_ISODIGIT(*cp)) { n = n*8 + *cp-'0'; cp++; }
-              if (TOR_ISODIGIT(*cp)) { n = n*8 + *cp-'0'; cp++; }
-              if (n > 255) { tor_free(*result); return NULL; }
-              *out++ = (char)n;
-            }
-            break;
-          case '\'':
-          case '\"':
-          case '\\':
-          case '\?':
-            *out++ = cp[1];
-            cp += 2;
-            break;
-          default:
-            /* LCOV_EXCL_START */
-            /* we caught this above in the initial loop. */
-            tor_assert_nonfatal_unreached();
-            tor_free(*result); return NULL;
-            /* LCOV_EXCL_STOP */
-          }
-        break;
-      default:
-        *out++ = *cp++;
-      }
-  }
-}
-
-/** Given a string containing part of a configuration file or similar format,
- * advance past comments and whitespace and try to parse a single line.  If we
- * parse a line successfully, set *<b>key_out</b> to a new string holding the
- * key portion and *<b>value_out</b> to a new string holding the value portion
- * of the line, and return a pointer to the start of the next line.  If we run
- * out of data, return a pointer to the end of the string.  If we encounter an
- * error, return NULL and set *<b>err_out</b> (if provided) to an error
- * message.
- */
-const char *
-parse_config_line_from_str_verbose(const char *line, char **key_out,
-                                   char **value_out,
-                                   const char **err_out)
-{
-  /*
-    See torrc_format.txt for a description of the (silly) format this parses.
-   */
-  const char *key, *val, *cp;
-  int continuation = 0;
-
-  tor_assert(key_out);
-  tor_assert(value_out);
-
-  *key_out = *value_out = NULL;
-  key = val = NULL;
-  /* Skip until the first keyword. */
-  while (1) {
-    while (TOR_ISSPACE(*line))
-      ++line;
-    if (*line == '#') {
-      while (*line && *line != '\n')
-        ++line;
-    } else {
-      break;
-    }
-  }
-
-  if (!*line) { /* End of string? */
-    *key_out = *value_out = NULL;
-    return line;
-  }
-
-  /* Skip until the next space or \ followed by newline. */
-  key = line;
-  while (*line && !TOR_ISSPACE(*line) && *line != '#' &&
-         ! (line[0] == '\\' && line[1] == '\n'))
-    ++line;
-  *key_out = tor_strndup(key, line-key);
-
-  /* Skip until the value. */
-  while (*line == ' ' || *line == '\t')
-    ++line;
-
-  val = line;
-
-  /* Find the end of the line. */
-  if (*line == '\"') { // XXX No continuation handling is done here
-    if (!(line = unescape_string(line, value_out, NULL))) {
-      if (err_out)
-        *err_out = "Invalid escape sequence in quoted string";
-      return NULL;
-    }
-    while (*line == ' ' || *line == '\t')
-      ++line;
-    if (*line == '\r' && *(++line) == '\n')
-      ++line;
-    if (*line && *line != '#' && *line != '\n') {
-      if (err_out)
-        *err_out = "Excess data after quoted string";
-      return NULL;
-    }
-  } else {
-    /* Look for the end of the line. */
-    while (*line && *line != '\n' && (*line != '#' || continuation)) {
-      if (*line == '\\' && line[1] == '\n') {
-        continuation = 1;
-        line += 2;
-      } else if (*line == '#') {
-        do {
-          ++line;
-        } while (*line && *line != '\n');
-        if (*line == '\n')
-          ++line;
-      } else {
-        ++line;
-      }
-    }
-
-    if (*line == '\n') {
-      cp = line++;
-    } else {
-      cp = line;
-    }
-    /* Now back cp up to be the last nonspace character */
-    while (cp>val && TOR_ISSPACE(*(cp-1)))
-      --cp;
-
-    tor_assert(cp >= val);
-
-    /* Now copy out and decode the value. */
-    *value_out = tor_strndup(val, cp-val);
-    if (continuation) {
-      char *v_out, *v_in;
-      v_out = v_in = *value_out;
-      while (*v_in) {
-        if (*v_in == '#') {
-          do {
-            ++v_in;
-          } while (*v_in && *v_in != '\n');
-          if (*v_in == '\n')
-            ++v_in;
-        } else if (v_in[0] == '\\' && v_in[1] == '\n') {
-          v_in += 2;
-        } else {
-          *v_out++ = *v_in++;
-        }
-      }
-      *v_out = '\0';
-    }
-  }
-
-  if (*line == '#') {
-    do {
-      ++line;
-    } while (*line && *line != '\n');
-  }
-  while (TOR_ISSPACE(*line)) ++line;
-
-  return line;
-}
-
-/** Expand any homedir prefix on <b>filename</b>; return a newly allocated
- * string. */
-char *
-expand_filename(const char *filename)
-{
-  tor_assert(filename);
-#ifdef _WIN32
-  /* Might consider using GetFullPathName() as described here:
-   * http://etutorials.org/Programming/secure+programming/
-   *     Chapter+3.+Input+Validation/3.7+Validating+Filenames+and+Paths/
-   */
-  return tor_strdup(filename);
-#else
-  if (*filename == '~') {
-    char *home, *result=NULL;
-    const char *rest;
-
-    if (filename[1] == '/' || filename[1] == '\0') {
-      home = getenv("HOME");
-      if (!home) {
-        log_warn(LD_CONFIG, "Couldn't find $HOME environment variable while "
-                 "expanding \"%s\"; defaulting to \"\".", filename);
-        home = tor_strdup("");
-      } else {
-        home = tor_strdup(home);
-      }
-      rest = strlen(filename)>=2?(filename+2):"";
-    } else {
-#ifdef HAVE_PWD_H
-      char *username, *slash;
-      slash = strchr(filename, '/');
-      if (slash)
-        username = tor_strndup(filename+1,slash-filename-1);
-      else
-        username = tor_strdup(filename+1);
-      if (!(home = get_user_homedir(username))) {
-        log_warn(LD_CONFIG,"Couldn't get homedir for \"%s\"",username);
-        tor_free(username);
-        return NULL;
-      }
-      tor_free(username);
-      rest = slash ? (slash+1) : "";
-#else
-      log_warn(LD_CONFIG, "Couldn't expand homedir on system without pwd.h");
-      return tor_strdup(filename);
-#endif
-    }
-    tor_assert(home);
-    /* Remove trailing slash. */
-    if (strlen(home)>1 && !strcmpend(home,PATH_SEPARATOR)) {
-      home[strlen(home)-1] = '\0';
-    }
-    tor_asprintf(&result,"%s"PATH_SEPARATOR"%s",home,rest);
-    tor_free(home);
-    return result;
-  } else {
-    return tor_strdup(filename);
-  }
-#endif
-}
-
-#define MAX_SCANF_WIDTH 9999
-
-/** Helper: given an ASCII-encoded decimal digit, return its numeric value.
- * NOTE: requires that its input be in-bounds. */
-static int
-digit_to_num(char d)
-{
-  int num = ((int)d) - (int)'0';
-  tor_assert(num <= 9 && num >= 0);
-  return num;
-}
-
-/** Helper: Read an unsigned int from *<b>bufp</b> of up to <b>width</b>
- * characters.  (Handle arbitrary width if <b>width</b> is less than 0.)  On
- * success, store the result in <b>out</b>, advance bufp to the next
- * character, and return 0.  On failure, return -1. */
-static int
-scan_unsigned(const char **bufp, unsigned long *out, int width, unsigned base)
-{
-  unsigned long result = 0;
-  int scanned_so_far = 0;
-  const int hex = base==16;
-  tor_assert(base == 10 || base == 16);
-  if (!bufp || !*bufp || !out)
-    return -1;
-  if (width<0)
-    width=MAX_SCANF_WIDTH;
-
-  while (**bufp && (hex?TOR_ISXDIGIT(**bufp):TOR_ISDIGIT(**bufp))
-         && scanned_so_far < width) {
-    unsigned digit = hex?hex_decode_digit(*(*bufp)++):digit_to_num(*(*bufp)++);
-    // Check for overflow beforehand, without actually causing any overflow
-    // This preserves functionality on compilers that don't wrap overflow
-    // (i.e. that trap or optimise away overflow)
-    // result * base + digit > ULONG_MAX
-    // result * base > ULONG_MAX - digit
-    if (result > (ULONG_MAX - digit)/base)
-      return -1; /* Processing this digit would overflow */
-    result = result * base + digit;
-    ++scanned_so_far;
-  }
-
-  if (!scanned_so_far) /* No actual digits scanned */
-    return -1;
-
-  *out = result;
-  return 0;
-}
-
-/** Helper: Read an signed int from *<b>bufp</b> of up to <b>width</b>
- * characters.  (Handle arbitrary width if <b>width</b> is less than 0.)  On
- * success, store the result in <b>out</b>, advance bufp to the next
- * character, and return 0.  On failure, return -1. */
-static int
-scan_signed(const char **bufp, long *out, int width)
-{
-  int neg = 0;
-  unsigned long result = 0;
-
-  if (!bufp || !*bufp || !out)
-    return -1;
-  if (width<0)
-    width=MAX_SCANF_WIDTH;
-
-  if (**bufp == '-') {
-    neg = 1;
-    ++*bufp;
-    --width;
-  }
-
-  if (scan_unsigned(bufp, &result, width, 10) < 0)
-    return -1;
-
-  if (neg && result > 0) {
-    if (result > ((unsigned long)LONG_MAX) + 1)
-      return -1; /* Underflow */
-    else if (result == ((unsigned long)LONG_MAX) + 1)
-      *out = LONG_MIN;
-    else {
-      /* We once had a far more clever no-overflow conversion here, but
-       * some versions of GCC apparently ran it into the ground.  Now
-       * we just check for LONG_MIN explicitly.
-       */
-      *out = -(long)result;
-    }
-  } else {
-    if (result > LONG_MAX)
-      return -1; /* Overflow */
-    *out = (long)result;
-  }
-
-  return 0;
-}
-
-/** Helper: Read a decimal-formatted double from *<b>bufp</b> of up to
- * <b>width</b> characters.  (Handle arbitrary width if <b>width</b> is less
- * than 0.)  On success, store the result in <b>out</b>, advance bufp to the
- * next character, and return 0.  On failure, return -1. */
-static int
-scan_double(const char **bufp, double *out, int width)
-{
-  int neg = 0;
-  double result = 0;
-  int scanned_so_far = 0;
-
-  if (!bufp || !*bufp || !out)
-    return -1;
-  if (width<0)
-    width=MAX_SCANF_WIDTH;
-
-  if (**bufp == '-') {
-    neg = 1;
-    ++*bufp;
-  }
-
-  while (**bufp && TOR_ISDIGIT(**bufp) && scanned_so_far < width) {
-    const int digit = digit_to_num(*(*bufp)++);
-    result = result * 10 + digit;
-    ++scanned_so_far;
-  }
-  if (**bufp == '.') {
-    double fracval = 0, denominator = 1;
-    ++*bufp;
-    ++scanned_so_far;
-    while (**bufp && TOR_ISDIGIT(**bufp) && scanned_so_far < width) {
-      const int digit = digit_to_num(*(*bufp)++);
-      fracval = fracval * 10 + digit;
-      denominator *= 10;
-      ++scanned_so_far;
-    }
-    result += fracval / denominator;
-  }
-
-  if (!scanned_so_far) /* No actual digits scanned */
-    return -1;
-
-  *out = neg ? -result : result;
-  return 0;
-}
-
-/** Helper: copy up to <b>width</b> non-space characters from <b>bufp</b> to
- * <b>out</b>.  Make sure <b>out</b> is nul-terminated. Advance <b>bufp</b>
- * to the next non-space character or the EOS. */
-static int
-scan_string(const char **bufp, char *out, int width)
-{
-  int scanned_so_far = 0;
-  if (!bufp || !out || width < 0)
-    return -1;
-  while (**bufp && ! TOR_ISSPACE(**bufp) && scanned_so_far < width) {
-    *out++ = *(*bufp)++;
-    ++scanned_so_far;
-  }
-  *out = '\0';
-  return 0;
-}
-
-/** Locale-independent, minimal, no-surprises scanf variant, accepting only a
- * restricted pattern format.  For more info on what it supports, see
- * tor_sscanf() documentation.  */
-int
-tor_vsscanf(const char *buf, const char *pattern, va_list ap)
-{
-  int n_matched = 0;
-
-  while (*pattern) {
-    if (*pattern != '%') {
-      if (*buf == *pattern) {
-        ++buf;
-        ++pattern;
-        continue;
-      } else {
-        return n_matched;
-      }
-    } else {
-      int width = -1;
-      int longmod = 0;
-      ++pattern;
-      if (TOR_ISDIGIT(*pattern)) {
-        width = digit_to_num(*pattern++);
-        while (TOR_ISDIGIT(*pattern)) {
-          width *= 10;
-          width += digit_to_num(*pattern++);
-          if (width > MAX_SCANF_WIDTH)
-            return -1;
-        }
-        if (!width) /* No zero-width things. */
-          return -1;
-      }
-      if (*pattern == 'l') {
-        longmod = 1;
-        ++pattern;
-      }
-      if (*pattern == 'u' || *pattern == 'x') {
-        unsigned long u;
-        const int base = (*pattern == 'u') ? 10 : 16;
-        if (!*buf)
-          return n_matched;
-        if (scan_unsigned(&buf, &u, width, base)<0)
-          return n_matched;
-        if (longmod) {
-          unsigned long *out = va_arg(ap, unsigned long *);
-          *out = u;
-        } else {
-          unsigned *out = va_arg(ap, unsigned *);
-          if (u > UINT_MAX)
-            return n_matched;
-          *out = (unsigned) u;
-        }
-        ++pattern;
-        ++n_matched;
-      } else if (*pattern == 'f') {
-        double *d = va_arg(ap, double *);
-        if (!longmod)
-          return -1; /* float not supported */
-        if (!*buf)
-          return n_matched;
-        if (scan_double(&buf, d, width)<0)
-          return n_matched;
-        ++pattern;
-        ++n_matched;
-      } else if (*pattern == 'd') {
-        long lng=0;
-        if (scan_signed(&buf, &lng, width)<0)
-          return n_matched;
-        if (longmod) {
-          long *out = va_arg(ap, long *);
-          *out = lng;
-        } else {
-          int *out = va_arg(ap, int *);
-#if LONG_MAX > INT_MAX
-          if (lng < INT_MIN || lng > INT_MAX)
-            return n_matched;
-#endif
-          *out = (int)lng;
-        }
-        ++pattern;
-        ++n_matched;
-      } else if (*pattern == 's') {
-        char *s = va_arg(ap, char *);
-        if (longmod)
-          return -1;
-        if (width < 0)
-          return -1;
-        if (scan_string(&buf, s, width)<0)
-          return n_matched;
-        ++pattern;
-        ++n_matched;
-      } else if (*pattern == 'c') {
-        char *ch = va_arg(ap, char *);
-        if (longmod)
-          return -1;
-        if (width != -1)
-          return -1;
-        if (!*buf)
-          return n_matched;
-        *ch = *buf++;
-        ++pattern;
-        ++n_matched;
-      } else if (*pattern == '%') {
-        if (*buf != '%')
-          return n_matched;
-        if (longmod)
-          return -1;
-        ++buf;
-        ++pattern;
-      } else {
-        return -1; /* Unrecognized pattern component. */
-      }
-    }
-  }
-
-  return n_matched;
-}
-
-/** Minimal sscanf replacement: parse <b>buf</b> according to <b>pattern</b>
- * and store the results in the corresponding argument fields.  Differs from
- * sscanf in that:
- * <ul><li>It only handles %u, %lu, %x, %lx, %[NUM]s, %d, %ld, %lf, and %c.
- *     <li>It only handles decimal inputs for %lf. (12.3, not 1.23e1)
- *     <li>It does not handle arbitrarily long widths.
- *     <li>Numbers do not consume any space characters.
- *     <li>It is locale-independent.
- *     <li>%u and %x do not consume any space.
- *     <li>It returns -1 on malformed patterns.</ul>
- *
- * (As with other locale-independent functions, we need this to parse data that
- * is in ASCII without worrying that the C library's locale-handling will make
- * miscellaneous characters look like numbers, spaces, and so on.)
- */
-int
-tor_sscanf(const char *buf, const char *pattern, ...)
-{
-  int r;
-  va_list ap;
-  va_start(ap, pattern);
-  r = tor_vsscanf(buf, pattern, ap);
-  va_end(ap);
-  return r;
-}
-
-/** Append the string produced by tor_asprintf(<b>pattern</b>, <b>...</b>)
- * to <b>sl</b>. */
-void
-smartlist_add_asprintf(struct smartlist_t *sl, const char *pattern, ...)
-{
-  va_list ap;
-  va_start(ap, pattern);
-  smartlist_add_vasprintf(sl, pattern, ap);
-  va_end(ap);
-}
-
-/** va_list-based backend of smartlist_add_asprintf. */
-void
-smartlist_add_vasprintf(struct smartlist_t *sl, const char *pattern,
-                        va_list args)
-{
-  char *str = NULL;
-
-  tor_vasprintf(&str, pattern, args);
-  tor_assert(str != NULL);
-
-  smartlist_add(sl, str);
-}
-
-/** Return a new list containing the filenames in the directory <b>dirname</b>.
- * Return NULL on error or if <b>dirname</b> is not a directory.
- */
-MOCK_IMPL(smartlist_t *,
-tor_listdir, (const char *dirname))
-{
-  smartlist_t *result;
-#ifdef _WIN32
-  char *pattern=NULL;
-  TCHAR tpattern[MAX_PATH] = {0};
-  char name[MAX_PATH*2+1] = {0};
-  HANDLE handle;
-  WIN32_FIND_DATA findData;
-  tor_asprintf(&pattern, "%s\\*", dirname);
-#ifdef UNICODE
-  mbstowcs(tpattern,pattern,MAX_PATH);
-#else
-  strlcpy(tpattern, pattern, MAX_PATH);
-#endif
-  if (INVALID_HANDLE_VALUE == (handle = FindFirstFile(tpattern, &findData))) {
-    tor_free(pattern);
-    return NULL;
-  }
-  result = smartlist_new();
-  while (1) {
-#ifdef UNICODE
-    wcstombs(name,findData.cFileName,MAX_PATH);
-    name[sizeof(name)-1] = '\0';
-#else
-    strlcpy(name,findData.cFileName,sizeof(name));
-#endif
-    if (strcmp(name, ".") &&
-        strcmp(name, "..")) {
-      smartlist_add(result, tor_strdup(name));
-    }
-    if (!FindNextFile(handle, &findData)) {
-      DWORD err;
-      if ((err = GetLastError()) != ERROR_NO_MORE_FILES) {
-        char *errstr = format_win32_error(err);
-        log_warn(LD_FS, "Error reading directory '%s': %s", dirname, errstr);
-        tor_free(errstr);
-      }
-      break;
-    }
-  }
-  FindClose(handle);
-  tor_free(pattern);
-#else
-  const char *prot_dname = sandbox_intern_string(dirname);
-  DIR *d;
-  struct dirent *de;
-  if (!(d = opendir(prot_dname)))
-    return NULL;
-
-  result = smartlist_new();
-  while ((de = readdir(d))) {
-    if (!strcmp(de->d_name, ".") ||
-        !strcmp(de->d_name, ".."))
-      continue;
-    smartlist_add(result, tor_strdup(de->d_name));
-  }
-  closedir(d);
-#endif
-  return result;
-}
-
-/** Return true iff <b>filename</b> is a relative path. */
-int
-path_is_relative(const char *filename)
-{
-  if (filename && filename[0] == '/')
-    return 0;
-#ifdef _WIN32
-  else if (filename && filename[0] == '\\')
-    return 0;
-  else if (filename && strlen(filename)>3 && TOR_ISALPHA(filename[0]) &&
-           filename[1] == ':' && filename[2] == '\\')
-    return 0;
-#endif
-  else
-    return 1;
-}
-
-/* =====
- * Process helpers
- * ===== */
-
-#ifndef _WIN32
-/* Based on code contributed by christian grothoff */
-/** True iff we've called start_daemon(). */
-static int start_daemon_called = 0;
-/** True iff we've called finish_daemon(). */
-static int finish_daemon_called = 0;
-/** Socketpair used to communicate between parent and child process while
- * daemonizing. */
-static int daemon_filedes[2];
-/** Start putting the process into daemon mode: fork and drop all resources
- * except standard fds.  The parent process never returns, but stays around
- * until finish_daemon is called.  (Note: it's safe to call this more
- * than once: calls after the first are ignored.)
- */
-void
-start_daemon(void)
-{
-  pid_t pid;
-
-  if (start_daemon_called)
-    return;
-  start_daemon_called = 1;
-
-  if (pipe(daemon_filedes)) {
-    /* LCOV_EXCL_START */
-    log_err(LD_GENERAL,"pipe failed; exiting. Error was %s", strerror(errno));
-    exit(1);
-    /* LCOV_EXCL_STOP */
-  }
-  pid = fork();
-  if (pid < 0) {
-    /* LCOV_EXCL_START */
-    log_err(LD_GENERAL,"fork failed. Exiting.");
-    exit(1);
-    /* LCOV_EXCL_STOP */
-  }
-  if (pid) {  /* Parent */
-    int ok;
-    char c;
-
-    close(daemon_filedes[1]); /* we only read */
-    ok = -1;
-    while (0 < read(daemon_filedes[0], &c, sizeof(char))) {
-      if (c == '.')
-        ok = 1;
-    }
-    fflush(stdout);
-    if (ok == 1)
-      exit(0);
-    else
-      exit(1); /* child reported error */
-  } else { /* Child */
-    close(daemon_filedes[0]); /* we only write */
-
-    pid = setsid(); /* Detach from controlling terminal */
-    /*
-     * Fork one more time, so the parent (the session group leader) can exit.
-     * This means that we, as a non-session group leader, can never regain a
-     * controlling terminal.   This part is recommended by Stevens's
-     * _Advanced Programming in the Unix Environment_.
-     */
-    if (fork() != 0) {
-      exit(0);
-    }
-    set_main_thread(); /* We are now the main thread. */
-
-    return;
-  }
-}
-
-/** Finish putting the process into daemon mode: drop standard fds, and tell
- * the parent process to exit.  (Note: it's safe to call this more than once:
- * calls after the first are ignored.  Calls start_daemon first if it hasn't
- * been called already.)
- */
-void
-finish_daemon(const char *desired_cwd)
-{
-  int nullfd;
-  char c = '.';
-  if (finish_daemon_called)
-    return;
-  if (!start_daemon_called)
-    start_daemon();
-  finish_daemon_called = 1;
-
-  if (!desired_cwd)
-    desired_cwd = "/";
-   /* Don't hold the wrong FS mounted */
-  if (chdir(desired_cwd) < 0) {
-    log_err(LD_GENERAL,"chdir to \"%s\" failed. Exiting.",desired_cwd);
-    exit(1);
-  }
-
-  nullfd = tor_open_cloexec("/dev/null", O_RDWR, 0);
-  if (nullfd < 0) {
-    /* LCOV_EXCL_START */
-    log_err(LD_GENERAL,"/dev/null can't be opened. Exiting.");
-    exit(1);
-    /* LCOV_EXCL_STOP */
-  }
-  /* close fds linking to invoking terminal, but
-   * close usual incoming fds, but redirect them somewhere
-   * useful so the fds don't get reallocated elsewhere.
-   */
-  if (dup2(nullfd,0) < 0 ||
-      dup2(nullfd,1) < 0 ||
-      dup2(nullfd,2) < 0) {
-    /* LCOV_EXCL_START */
-    log_err(LD_GENERAL,"dup2 failed. Exiting.");
-    exit(1);
-    /* LCOV_EXCL_STOP */
-  }
-  if (nullfd > 2)
-    close(nullfd);
-  /* signal success */
-  if (write(daemon_filedes[1], &c, sizeof(char)) != sizeof(char)) {
-    log_err(LD_GENERAL,"write failed. Exiting.");
-  }
-  close(daemon_filedes[1]);
-}
-#else
-/* defined(_WIN32) */
-void
-start_daemon(void)
-{
-}
-void
-finish_daemon(const char *cp)
-{
-  (void)cp;
-}
-#endif
-
-/** Write the current process ID, followed by NL, into <b>filename</b>.
- */
-void
-write_pidfile(const char *filename)
-{
-  FILE *pidfile;
-
-  if ((pidfile = fopen(filename, "w")) == NULL) {
-    log_warn(LD_FS, "Unable to open \"%s\" for writing: %s", filename,
-             strerror(errno));
-  } else {
-#ifdef _WIN32
-    fprintf(pidfile, "%d\n", (int)_getpid());
-#else
-    fprintf(pidfile, "%d\n", (int)getpid());
-#endif
-    fclose(pidfile);
-  }
-}
-
-#ifdef _WIN32
-HANDLE
-load_windows_system_library(const TCHAR *library_name)
-{
-  TCHAR path[MAX_PATH];
-  unsigned n;
-  n = GetSystemDirectory(path, MAX_PATH);
-  if (n == 0 || n + _tcslen(library_name) + 2 >= MAX_PATH)
-    return 0;
-  _tcscat(path, TEXT("\\"));
-  _tcscat(path, library_name);
-  return LoadLibrary(path);
-}
-#endif
-
-/** Format a single argument for being put on a Windows command line.
- * Returns a newly allocated string */
-static char *
-format_win_cmdline_argument(const char *arg)
-{
-  char *formatted_arg;
-  char need_quotes;
-  const char *c;
-  int i;
-  int bs_counter = 0;
-  /* Backslash we can point to when one is inserted into the string */
-  const char backslash = '\\';
-
-  /* Smartlist of *char */
-  smartlist_t *arg_chars;
-  arg_chars = smartlist_new();
-
-  /* Quote string if it contains whitespace or is empty */
-  need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]);
-
-  /* Build up smartlist of *chars */
-  for (c=arg; *c != '\0'; c++) {
-    if ('"' == *c) {
-      /* Double up backslashes preceding a quote */
-      for (i=0; i<(bs_counter*2); i++)
-        smartlist_add(arg_chars, (void*)&backslash);
-      bs_counter = 0;
-      /* Escape the quote */
-      smartlist_add(arg_chars, (void*)&backslash);
-      smartlist_add(arg_chars, (void*)c);
-    } else if ('\\' == *c) {
-      /* Count backslashes until we know whether to double up */
-      bs_counter++;
-    } else {
-      /* Don't double up slashes preceding a non-quote */
-      for (i=0; i<bs_counter; i++)
-        smartlist_add(arg_chars, (void*)&backslash);
-      bs_counter = 0;
-      smartlist_add(arg_chars, (void*)c);
-    }
-  }
-  /* Don't double up trailing backslashes */
-  for (i=0; i<bs_counter; i++)
-    smartlist_add(arg_chars, (void*)&backslash);
-
-  /* Allocate space for argument, quotes (if needed), and terminator */
-  const size_t formatted_arg_len = smartlist_len(arg_chars) +
-    (need_quotes ? 2 : 0) + 1;
-  formatted_arg = tor_malloc_zero(formatted_arg_len);
-
-  /* Add leading quote */
-  i=0;
-  if (need_quotes)
-    formatted_arg[i++] = '"';
-
-  /* Add characters */
-  SMARTLIST_FOREACH(arg_chars, char*, ch,
-  {
-    formatted_arg[i++] = *ch;
-  });
-
-  /* Add trailing quote */
-  if (need_quotes)
-    formatted_arg[i++] = '"';
-  formatted_arg[i] = '\0';
-
-  smartlist_free(arg_chars);
-  return formatted_arg;
-}
-
-/** Format a command line for use on Windows, which takes the command as a
- * string rather than string array. Follows the rules from "Parsing C++
- * Command-Line Arguments" in MSDN. Algorithm based on list2cmdline in the
- * Python subprocess module. Returns a newly allocated string */
-char *
-tor_join_win_cmdline(const char *argv[])
-{
-  smartlist_t *argv_list;
-  char *joined_argv;
-  int i;
-
-  /* Format each argument and put the result in a smartlist */
-  argv_list = smartlist_new();
-  for (i=0; argv[i] != NULL; i++) {
-    smartlist_add(argv_list, (void *)format_win_cmdline_argument(argv[i]));
-  }
-
-  /* Join the arguments with whitespace */
-  joined_argv = smartlist_join_strings(argv_list, " ", 0, NULL);
-
-  /* Free the newly allocated arguments, and the smartlist */
-  SMARTLIST_FOREACH(argv_list, char *, arg,
-  {
-    tor_free(arg);
-  });
-  smartlist_free(argv_list);
-
-  return joined_argv;
-}
-
-/* As format_{hex,dex}_number_sigsafe, but takes a <b>radix</b> argument
- * in range 2..16 inclusive. */
-static int
-format_number_sigsafe(unsigned long x, char *buf, int buf_len,
-                      unsigned int radix)
-{
-  unsigned long tmp;
-  int len;
-  char *cp;
-
-  /* NOT tor_assert. This needs to be safe to run from within a signal handler,
-   * and from within the 'tor_assert() has failed' code. */
-  if (radix < 2 || radix > 16)
-    return 0;
-
-  /* Count how many digits we need. */
-  tmp = x;
-  len = 1;
-  while (tmp >= radix) {
-    tmp /= radix;
-    ++len;
-  }
-
-  /* Not long enough */
-  if (!buf || len >= buf_len)
-    return 0;
-
-  cp = buf + len;
-  *cp = '\0';
-  do {
-    unsigned digit = (unsigned) (x % radix);
-    tor_assert(cp > buf);
-    --cp;
-    *cp = "0123456789ABCDEF"[digit];
-    x /= radix;
-  } while (x);
-
-  /* NOT tor_assert; see above. */
-  if (cp != buf) {
-    abort(); // LCOV_EXCL_LINE
-  }
-
-  return len;
-}
-
-/**
- * Helper function to output hex numbers from within a signal handler.
- *
- * Writes the nul-terminated hexadecimal digits of <b>x</b> into a buffer
- * <b>buf</b> of size <b>buf_len</b>, and return the actual number of digits
- * written, not counting the terminal NUL.
- *
- * If there is insufficient space, write nothing and return 0.
- *
- * This accepts an unsigned int because format_helper_exit_status() needs to
- * call it with a signed int and an unsigned char, and since the C standard
- * does not guarantee that an int is wider than a char (an int must be at
- * least 16 bits but it is permitted for a char to be that wide as well), we
- * can't assume a signed int is sufficient to accomodate an unsigned char.
- * Thus, format_helper_exit_status() will still need to emit any require '-'
- * on its own.
- *
- * For most purposes, you'd want to use tor_snprintf("%x") instead of this
- * function; it's designed to be used in code paths where you can't call
- * arbitrary C functions.
- */
-int
-format_hex_number_sigsafe(unsigned long x, char *buf, int buf_len)
-{
-  return format_number_sigsafe(x, buf, buf_len, 16);
-}
-
-/** As format_hex_number_sigsafe, but format the number in base 10. */
-int
-format_dec_number_sigsafe(unsigned long x, char *buf, int buf_len)
-{
-  return format_number_sigsafe(x, buf, buf_len, 10);
-}
-
-#ifndef _WIN32
-/** Format <b>child_state</b> and <b>saved_errno</b> as a hex string placed in
- * <b>hex_errno</b>.  Called between fork and _exit, so must be signal-handler
- * safe.
- *
- * <b>hex_errno</b> must have at least HEX_ERRNO_SIZE+1 bytes available.
- *
- * The format of <b>hex_errno</b> is: "CHILD_STATE/ERRNO\n", left-padded
- * with spaces. CHILD_STATE indicates where
- * in the processs of starting the child process did the failure occur (see
- * CHILD_STATE_* macros for definition), and SAVED_ERRNO is the value of
- * errno when the failure occurred.
- *
- * On success return the number of characters added to hex_errno, not counting
- * the terminating NUL; return -1 on error.
- */
-STATIC int
-format_helper_exit_status(unsigned char child_state, int saved_errno,
-                          char *hex_errno)
-{
-  unsigned int unsigned_errno;
-  int written, left;
-  char *cur;
-  size_t i;
-  int res = -1;
-
-  /* Fill hex_errno with spaces, and a trailing newline (memset may
-     not be signal handler safe, so we can't use it) */
-  for (i = 0; i < (HEX_ERRNO_SIZE - 1); i++)
-    hex_errno[i] = ' ';
-  hex_errno[HEX_ERRNO_SIZE - 1] = '\n';
-
-  /* Convert errno to be unsigned for hex conversion */
-  if (saved_errno < 0) {
-    // Avoid overflow on the cast to unsigned int when result is INT_MIN
-    // by adding 1 to the signed int negative value,
-    // then, after it has been negated and cast to unsigned,
-    // adding the original 1 back (the double-addition is intentional).
-    // Otherwise, the cast to signed could cause a temporary int
-    // to equal INT_MAX + 1, which is undefined.
-    unsigned_errno = ((unsigned int) -(saved_errno + 1)) + 1;
-  } else {
-    unsigned_errno = (unsigned int) saved_errno;
-  }
-
-  /*
-   * Count how many chars of space we have left, and keep a pointer into the
-   * current point in the buffer.
-   */
-  left = HEX_ERRNO_SIZE+1;
-  cur = hex_errno;
-
-  /* Emit child_state */
-  written = format_hex_number_sigsafe(child_state, cur, left);
-
-  if (written <= 0)
-    goto err;
-
-  /* Adjust left and cur */
-  left -= written;
-  cur += written;
-  if (left <= 0)
-    goto err;
-
-  /* Now the '/' */
-  *cur = '/';
-
-  /* Adjust left and cur */
-  ++cur;
-  --left;
-  if (left <= 0)
-    goto err;
-
-  /* Need minus? */
-  if (saved_errno < 0) {
-    *cur = '-';
-    ++cur;
-    --left;
-    if (left <= 0)
-      goto err;
-  }
-
-  /* Emit unsigned_errno */
-  written = format_hex_number_sigsafe(unsigned_errno, cur, left);
-
-  if (written <= 0)
-    goto err;
-
-  /* Adjust left and cur */
-  left -= written;
-  cur += written;
-
-  /* Check that we have enough space left for a newline and a NUL */
-  if (left <= 1)
-    goto err;
-
-  /* Emit the newline and NUL */
-  *cur++ = '\n';
-  *cur++ = '\0';
-
-  res = (int)(cur - hex_errno - 1);
-
-  goto done;
-
- err:
-  /*
-   * In error exit, just write a '\0' in the first char so whatever called
-   * this at least won't fall off the end.
-   */
-  *hex_errno = '\0';
-
- done:
-  return res;
-}
-#endif
-
-/* Maximum number of file descriptors, if we cannot get it via sysconf() */
-#define DEFAULT_MAX_FD 256
-
-/** Terminate the process of <b>process_handle</b>, if that process has not
- * already exited.
- *
- * Return 0 if we succeeded in terminating the process (or if the process
- * already exited), and -1 if we tried to kill the process but failed.
- *
- * Based on code originally borrowed from Python's os.kill. */
-int
-tor_terminate_process(process_handle_t *process_handle)
-{
-#ifdef _WIN32
-  if (tor_get_exit_code(process_handle, 0, NULL) == PROCESS_EXIT_RUNNING) {
-    HANDLE handle = process_handle->pid.hProcess;
-
-    if (!TerminateProcess(handle, 0))
-      return -1;
-    else
-      return 0;
-  }
-#else /* Unix */
-  if (process_handle->waitpid_cb) {
-    /* We haven't got a waitpid yet, so we can just kill off the process. */
-    return kill(process_handle->pid, SIGTERM);
-  }
-#endif
-
-  return 0; /* We didn't need to kill the process, so report success */
-}
-
-/** Return the Process ID of <b>process_handle</b>. */
-int
-tor_process_get_pid(process_handle_t *process_handle)
-{
-#ifdef _WIN32
-  return (int) process_handle->pid.dwProcessId;
-#else
-  return (int) process_handle->pid;
-#endif
-}
-
-#ifdef _WIN32
-HANDLE
-tor_process_get_stdout_pipe(process_handle_t *process_handle)
-{
-  return process_handle->stdout_pipe;
-}
-#else
-/* DOCDOC tor_process_get_stdout_pipe */
-FILE *
-tor_process_get_stdout_pipe(process_handle_t *process_handle)
-{
-  return process_handle->stdout_handle;
-}
-#endif
-
-/* DOCDOC process_handle_new */
-static process_handle_t *
-process_handle_new(void)
-{
-  process_handle_t *out = tor_malloc_zero(sizeof(process_handle_t));
-
-#ifdef _WIN32
-  out->stdin_pipe = INVALID_HANDLE_VALUE;
-  out->stdout_pipe = INVALID_HANDLE_VALUE;
-  out->stderr_pipe = INVALID_HANDLE_VALUE;
-#else
-  out->stdin_pipe = -1;
-  out->stdout_pipe = -1;
-  out->stderr_pipe = -1;
-#endif
-
-  return out;
-}
-
-#ifndef _WIN32
-/** Invoked when a process that we've launched via tor_spawn_background() has
- * been found to have terminated.
- */
-static void
-process_handle_waitpid_cb(int status, void *arg)
-{
-  process_handle_t *process_handle = arg;
-
-  process_handle->waitpid_exit_status = status;
-  clear_waitpid_callback(process_handle->waitpid_cb);
-  if (process_handle->status == PROCESS_STATUS_RUNNING)
-    process_handle->status = PROCESS_STATUS_NOTRUNNING;
-  process_handle->waitpid_cb = 0;
-}
-#endif
-
-/**
- * @name child-process states
- *
- * Each of these values represents a possible state that a child process can
- * be in.  They're used to determine what to say when telling the parent how
- * far along we were before failure.
- *
- * @{
- */
-#define CHILD_STATE_INIT 0
-#define CHILD_STATE_PIPE 1
-#define CHILD_STATE_MAXFD 2
-#define CHILD_STATE_FORK 3
-#define CHILD_STATE_DUPOUT 4
-#define CHILD_STATE_DUPERR 5
-#define CHILD_STATE_DUPIN 6
-#define CHILD_STATE_CLOSEFD 7
-#define CHILD_STATE_EXEC 8
-#define CHILD_STATE_FAILEXEC 9
-/** @} */
-/** Start a program in the background. If <b>filename</b> contains a '/', then
- * it will be treated as an absolute or relative path.  Otherwise, on
- * non-Windows systems, the system path will be searched for <b>filename</b>.
- * On Windows, only the current directory will be searched. Here, to search the
- * system path (as well as the application directory, current working
- * directory, and system directories), set filename to NULL.
- *
- * The strings in <b>argv</b> will be passed as the command line arguments of
- * the child program (following convention, argv[0] should normally be the
- * filename of the executable, and this must be the case if <b>filename</b> is
- * NULL). The last element of argv must be NULL. A handle to the child process
- * will be returned in process_handle (which must be non-NULL). Read
- * process_handle.status to find out if the process was successfully launched.
- * For convenience, process_handle.status is returned by this function.
- *
- * Some parts of this code are based on the POSIX subprocess module from
- * Python, and example code from
- * http://msdn.microsoft.com/en-us/library/ms682499%28v=vs.85%29.aspx.
- */
-int
-tor_spawn_background(const char *const filename, const char **argv,
-                     process_environment_t *env,
-                     process_handle_t **process_handle_out)
-{
-#ifdef _WIN32
-  HANDLE stdout_pipe_read = NULL;
-  HANDLE stdout_pipe_write = NULL;
-  HANDLE stderr_pipe_read = NULL;
-  HANDLE stderr_pipe_write = NULL;
-  HANDLE stdin_pipe_read = NULL;
-  HANDLE stdin_pipe_write = NULL;
-  process_handle_t *process_handle;
-  int status;
-
-  STARTUPINFOA siStartInfo;
-  BOOL retval = FALSE;
-
-  SECURITY_ATTRIBUTES saAttr;
-  char *joined_argv;
-
-  saAttr.nLength = sizeof(SECURITY_ATTRIBUTES);
-  saAttr.bInheritHandle = TRUE;
-  /* TODO: should we set explicit security attributes? (#2046, comment 5) */
-  saAttr.lpSecurityDescriptor = NULL;
-
-  /* Assume failure to start process */
-  status = PROCESS_STATUS_ERROR;
-
-  /* Set up pipe for stdout */
-  if (!CreatePipe(&stdout_pipe_read, &stdout_pipe_write, &saAttr, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to create pipe for stdout communication with child process: %s",
-      format_win32_error(GetLastError()));
-    return status;
-  }
-  if (!SetHandleInformation(stdout_pipe_read, HANDLE_FLAG_INHERIT, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to configure pipe for stdout communication with child "
-      "process: %s", format_win32_error(GetLastError()));
-    return status;
-  }
-
-  /* Set up pipe for stderr */
-  if (!CreatePipe(&stderr_pipe_read, &stderr_pipe_write, &saAttr, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to create pipe for stderr communication with child process: %s",
-      format_win32_error(GetLastError()));
-    return status;
-  }
-  if (!SetHandleInformation(stderr_pipe_read, HANDLE_FLAG_INHERIT, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to configure pipe for stderr communication with child "
-      "process: %s", format_win32_error(GetLastError()));
-    return status;
-  }
-
-  /* Set up pipe for stdin */
-  if (!CreatePipe(&stdin_pipe_read, &stdin_pipe_write, &saAttr, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to create pipe for stdin communication with child process: %s",
-      format_win32_error(GetLastError()));
-    return status;
-  }
-  if (!SetHandleInformation(stdin_pipe_write, HANDLE_FLAG_INHERIT, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to configure pipe for stdin communication with child "
-      "process: %s", format_win32_error(GetLastError()));
-    return status;
-  }
-
-  /* Create the child process */
-
-  /* Windows expects argv to be a whitespace delimited string, so join argv up
-   */
-  joined_argv = tor_join_win_cmdline(argv);
-
-  process_handle = process_handle_new();
-  process_handle->status = status;
-
-  ZeroMemory(&(process_handle->pid), sizeof(PROCESS_INFORMATION));
-  ZeroMemory(&siStartInfo, sizeof(STARTUPINFO));
-  siStartInfo.cb = sizeof(STARTUPINFO);
-  siStartInfo.hStdError = stderr_pipe_write;
-  siStartInfo.hStdOutput = stdout_pipe_write;
-  siStartInfo.hStdInput = stdin_pipe_read;
-  siStartInfo.dwFlags |= STARTF_USESTDHANDLES;
-
-  /* Create the child process */
-
-  retval = CreateProcessA(filename,      // module name
-                 joined_argv,   // command line
-  /* TODO: should we set explicit security attributes? (#2046, comment 5) */
-                 NULL,          // process security attributes
-                 NULL,          // primary thread security attributes
-                 TRUE,          // handles are inherited
-  /*(TODO: set CREATE_NEW CONSOLE/PROCESS_GROUP to make GetExitCodeProcess()
-   * work?) */
-                 CREATE_NO_WINDOW,             // creation flags
-                 (env==NULL) ? NULL : env->windows_environment_block,
-                 NULL,          // use parent's current directory
-                 &siStartInfo,  // STARTUPINFO pointer
-                 &(process_handle->pid));  // receives PROCESS_INFORMATION
-
-  tor_free(joined_argv);
-
-  if (!retval) {
-    log_warn(LD_GENERAL,
-      "Failed to create child process %s: %s", filename?filename:argv[0],
-      format_win32_error(GetLastError()));
-    tor_free(process_handle);
-  } else  {
-    /* TODO: Close hProcess and hThread in process_handle->pid? */
-    process_handle->stdout_pipe = stdout_pipe_read;
-    process_handle->stderr_pipe = stderr_pipe_read;
-    process_handle->stdin_pipe = stdin_pipe_write;
-    status = process_handle->status = PROCESS_STATUS_RUNNING;
-  }
-
-  /* TODO: Close pipes on exit */
-  *process_handle_out = process_handle;
-  return status;
-#else // _WIN32
-  pid_t pid;
-  int stdout_pipe[2];
-  int stderr_pipe[2];
-  int stdin_pipe[2];
-  int fd, retval;
-  ssize_t nbytes;
-  process_handle_t *process_handle;
-  int status;
-
-  const char *error_message = SPAWN_ERROR_MESSAGE;
-  size_t error_message_length;
-
-  /* Represents where in the process of spawning the program is;
-     this is used for printing out the error message */
-  unsigned char child_state = CHILD_STATE_INIT;
-
-  char hex_errno[HEX_ERRNO_SIZE + 2]; /* + 1 should be sufficient actually */
-
-  static int max_fd = -1;
-
-  status = PROCESS_STATUS_ERROR;
-
-  /* We do the strlen here because strlen() is not signal handler safe,
-     and we are not allowed to use unsafe functions between fork and exec */
-  error_message_length = strlen(error_message);
-
-  child_state = CHILD_STATE_PIPE;
-
-  /* Set up pipe for redirecting stdout, stderr, and stdin of child */
-  retval = pipe(stdout_pipe);
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-      "Failed to set up pipe for stdout communication with child process: %s",
-       strerror(errno));
-    return status;
-  }
-
-  retval = pipe(stderr_pipe);
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-      "Failed to set up pipe for stderr communication with child process: %s",
-      strerror(errno));
-
-    close(stdout_pipe[0]);
-    close(stdout_pipe[1]);
-
-    return status;
-  }
-
-  retval = pipe(stdin_pipe);
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-      "Failed to set up pipe for stdin communication with child process: %s",
-       strerror(errno));
-
-    close(stdout_pipe[0]);
-    close(stdout_pipe[1]);
-    close(stderr_pipe[0]);
-    close(stderr_pipe[1]);
-
-    return status;
-  }
-
-  child_state = CHILD_STATE_MAXFD;
-
-#ifdef _SC_OPEN_MAX
-  if (-1 == max_fd) {
-    max_fd = (int) sysconf(_SC_OPEN_MAX);
-    if (max_fd == -1) {
-      max_fd = DEFAULT_MAX_FD;
-      log_warn(LD_GENERAL,
-               "Cannot find maximum file descriptor, assuming %d", max_fd);
-    }
-  }
-#else
-  max_fd = DEFAULT_MAX_FD;
-#endif
-
-  child_state = CHILD_STATE_FORK;
-
-  pid = fork();
-  if (0 == pid) {
-    /* In child */
-
-#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__)
-    /* Attempt to have the kernel issue a SIGTERM if the parent
-     * goes away. Certain attributes of the binary being execve()ed
-     * will clear this during the execve() call, but it's better
-     * than nothing.
-     */
-    prctl(PR_SET_PDEATHSIG, SIGTERM);
-#endif
-
-    child_state = CHILD_STATE_DUPOUT;
-
-    /* Link child stdout to the write end of the pipe */
-    retval = dup2(stdout_pipe[1], STDOUT_FILENO);
-    if (-1 == retval)
-        goto error;
-
-    child_state = CHILD_STATE_DUPERR;
-
-    /* Link child stderr to the write end of the pipe */
-    retval = dup2(stderr_pipe[1], STDERR_FILENO);
-    if (-1 == retval)
-        goto error;
-
-    child_state = CHILD_STATE_DUPIN;
-
-    /* Link child stdin to the read end of the pipe */
-    retval = dup2(stdin_pipe[0], STDIN_FILENO);
-    if (-1 == retval)
-      goto error;
-
-    child_state = CHILD_STATE_CLOSEFD;
-
-    close(stderr_pipe[0]);
-    close(stderr_pipe[1]);
-    close(stdout_pipe[0]);
-    close(stdout_pipe[1]);
-    close(stdin_pipe[0]);
-    close(stdin_pipe[1]);
-
-    /* Close all other fds, including the read end of the pipe */
-    /* XXX: We should now be doing enough FD_CLOEXEC setting to make
-     * this needless. */
-    for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) {
-      close(fd);
-    }
-
-    child_state = CHILD_STATE_EXEC;
-
-    /* Call the requested program. We need the cast because
-       execvp doesn't define argv as const, even though it
-       does not modify the arguments */
-    if (env)
-      execve(filename, (char *const *) argv, env->unixoid_environment_block);
-    else {
-      static char *new_env[] = { NULL };
-      execve(filename, (char *const *) argv, new_env);
-    }
-
-    /* If we got here, the exec or open(/dev/null) failed */
-
-    child_state = CHILD_STATE_FAILEXEC;
-
-  error:
-    {
-      /* XXX: are we leaking fds from the pipe? */
-      int n;
-
-      n = format_helper_exit_status(child_state, errno, hex_errno);
-
-      if (n >= 0) {
-        /* Write the error message. GCC requires that we check the return
-           value, but there is nothing we can do if it fails */
-        /* TODO: Don't use STDOUT, use a pipe set up just for this purpose */
-        nbytes = write(STDOUT_FILENO, error_message, error_message_length);
-        nbytes = write(STDOUT_FILENO, hex_errno, n);
-      }
-    }
-
-    (void) nbytes;
-
-    _exit(255);
-    /* Never reached, but avoids compiler warning */
-    return status; // LCOV_EXCL_LINE
-  }
-
-  /* In parent */
-
-  if (-1 == pid) {
-    log_warn(LD_GENERAL, "Failed to fork child process: %s", strerror(errno));
-    close(stdin_pipe[0]);
-    close(stdin_pipe[1]);
-    close(stdout_pipe[0]);
-    close(stdout_pipe[1]);
-    close(stderr_pipe[0]);
-    close(stderr_pipe[1]);
-    return status;
-  }
-
-  process_handle = process_handle_new();
-  process_handle->status = status;
-  process_handle->pid = pid;
-
-  /* TODO: If the child process forked but failed to exec, waitpid it */
-
-  /* Return read end of the pipes to caller, and close write end */
-  process_handle->stdout_pipe = stdout_pipe[0];
-  retval = close(stdout_pipe[1]);
-
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-            "Failed to close write end of stdout pipe in parent process: %s",
-            strerror(errno));
-  }
-
-  process_handle->waitpid_cb = set_waitpid_callback(pid,
-                                                    process_handle_waitpid_cb,
-                                                    process_handle);
-
-  process_handle->stderr_pipe = stderr_pipe[0];
-  retval = close(stderr_pipe[1]);
-
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-            "Failed to close write end of stderr pipe in parent process: %s",
-            strerror(errno));
-  }
-
-  /* Return write end of the stdin pipe to caller, and close the read end */
-  process_handle->stdin_pipe = stdin_pipe[1];
-  retval = close(stdin_pipe[0]);
-
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-            "Failed to close read end of stdin pipe in parent process: %s",
-            strerror(errno));
-  }
-
-  status = process_handle->status = PROCESS_STATUS_RUNNING;
-  /* Set stdin/stdout/stderr pipes to be non-blocking */
-  if (fcntl(process_handle->stdout_pipe, F_SETFL, O_NONBLOCK) < 0 ||
-      fcntl(process_handle->stderr_pipe, F_SETFL, O_NONBLOCK) < 0 ||
-      fcntl(process_handle->stdin_pipe, F_SETFL, O_NONBLOCK) < 0) {
-    log_warn(LD_GENERAL, "Failed to set stderror/stdout/stdin pipes "
-             "nonblocking in parent process: %s", strerror(errno));
-  }
-  /* Open the buffered IO streams */
-  process_handle->stdout_handle = fdopen(process_handle->stdout_pipe, "r");
-  process_handle->stderr_handle = fdopen(process_handle->stderr_pipe, "r");
-  process_handle->stdin_handle = fdopen(process_handle->stdin_pipe, "r");
-
-  *process_handle_out = process_handle;
-  return process_handle->status;
-#endif // _WIN32
-}
-
-/** Destroy all resources allocated by the process handle in
- *  <b>process_handle</b>.
- *  If <b>also_terminate_process</b> is true, also terminate the
- *  process of the process handle. */
-MOCK_IMPL(void,
-tor_process_handle_destroy,(process_handle_t *process_handle,
-                            int also_terminate_process))
-{
-  if (!process_handle)
-    return;
-
-  if (also_terminate_process) {
-    if (tor_terminate_process(process_handle) < 0) {
-      const char *errstr =
-#ifdef _WIN32
-        format_win32_error(GetLastError());
-#else
-        strerror(errno);
-#endif
-      log_notice(LD_GENERAL, "Failed to terminate process with "
-                 "PID '%d' ('%s').", tor_process_get_pid(process_handle),
-                 errstr);
-    } else {
-      log_info(LD_GENERAL, "Terminated process with PID '%d'.",
-               tor_process_get_pid(process_handle));
-    }
-  }
-
-  process_handle->status = PROCESS_STATUS_NOTRUNNING;
-
-#ifdef _WIN32
-  if (process_handle->stdout_pipe)
-    CloseHandle(process_handle->stdout_pipe);
-
-  if (process_handle->stderr_pipe)
-    CloseHandle(process_handle->stderr_pipe);
-
-  if (process_handle->stdin_pipe)
-    CloseHandle(process_handle->stdin_pipe);
-#else
-  if (process_handle->stdout_handle)
-    fclose(process_handle->stdout_handle);
-
-  if (process_handle->stderr_handle)
-    fclose(process_handle->stderr_handle);
-
-  if (process_handle->stdin_handle)
-    fclose(process_handle->stdin_handle);
-
-  clear_waitpid_callback(process_handle->waitpid_cb);
-#endif
-
-  memset(process_handle, 0x0f, sizeof(process_handle_t));
-  tor_free(process_handle);
-}
-
-/** Get the exit code of a process specified by <b>process_handle</b> and store
- * it in <b>exit_code</b>, if set to a non-NULL value.  If <b>block</b> is set
- * to true, the call will block until the process has exited.  Otherwise if
- * the process is still running, the function will return
- * PROCESS_EXIT_RUNNING, and exit_code will be left unchanged. Returns
- * PROCESS_EXIT_EXITED if the process did exit. If there is a failure,
- * PROCESS_EXIT_ERROR will be returned and the contents of exit_code (if
- * non-NULL) will be undefined. N.B. Under *nix operating systems, this will
- * probably not work in Tor, because waitpid() is called in main.c to reap any
- * terminated child processes.*/
-int
-tor_get_exit_code(process_handle_t *process_handle,
-                  int block, int *exit_code)
-{
-#ifdef _WIN32
-  DWORD retval;
-  BOOL success;
-
-  if (block) {
-    /* Wait for the process to exit */
-    retval = WaitForSingleObject(process_handle->pid.hProcess, INFINITE);
-    if (retval != WAIT_OBJECT_0) {
-      log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s",
-              (int)retval, format_win32_error(GetLastError()));
-      return PROCESS_EXIT_ERROR;
-    }
-  } else {
-    retval = WaitForSingleObject(process_handle->pid.hProcess, 0);
-    if (WAIT_TIMEOUT == retval) {
-      /* Process has not exited */
-      return PROCESS_EXIT_RUNNING;
-    } else if (retval != WAIT_OBJECT_0) {
-      log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s",
-               (int)retval, format_win32_error(GetLastError()));
-      return PROCESS_EXIT_ERROR;
-    }
-  }
-
-  if (exit_code != NULL) {
-    success = GetExitCodeProcess(process_handle->pid.hProcess,
-                                 (PDWORD)exit_code);
-    if (!success) {
-      log_warn(LD_GENERAL, "GetExitCodeProcess() failed: %s",
-               format_win32_error(GetLastError()));
-      return PROCESS_EXIT_ERROR;
-    }
-  }
-#else
-  int stat_loc;
-  int retval;
-
-  if (process_handle->waitpid_cb) {
-    /* We haven't processed a SIGCHLD yet. */
-    retval = waitpid(process_handle->pid, &stat_loc, block?0:WNOHANG);
-    if (retval == process_handle->pid) {
-      clear_waitpid_callback(process_handle->waitpid_cb);
-      process_handle->waitpid_cb = NULL;
-      process_handle->waitpid_exit_status = stat_loc;
-    }
-  } else {
-    /* We already got a SIGCHLD for this process, and handled it. */
-    retval = process_handle->pid;
-    stat_loc = process_handle->waitpid_exit_status;
-  }
-
-  if (!block && 0 == retval) {
-    /* Process has not exited */
-    return PROCESS_EXIT_RUNNING;
-  } else if (retval != process_handle->pid) {
-    log_warn(LD_GENERAL, "waitpid() failed for PID %d: %s",
-             (int)process_handle->pid, strerror(errno));
-    return PROCESS_EXIT_ERROR;
-  }
-
-  if (!WIFEXITED(stat_loc)) {
-    log_warn(LD_GENERAL, "Process %d did not exit normally",
-             (int)process_handle->pid);
-    return PROCESS_EXIT_ERROR;
-  }
-
-  if (exit_code != NULL)
-    *exit_code = WEXITSTATUS(stat_loc);
-#endif // _WIN32
-
-  return PROCESS_EXIT_EXITED;
-}
-
-/** Helper: return the number of characters in <b>s</b> preceding the first
- * occurrence of <b>ch</b>. If <b>ch</b> does not occur in <b>s</b>, return
- * the length of <b>s</b>. Should be equivalent to strspn(s, "ch"). */
-static inline size_t
-str_num_before(const char *s, char ch)
-{
-  const char *cp = strchr(s, ch);
-  if (cp)
-    return cp - s;
-  else
-    return strlen(s);
-}
-
-/** Return non-zero iff getenv would consider <b>s1</b> and <b>s2</b>
- * to have the same name as strings in a process's environment. */
-int
-environment_variable_names_equal(const char *s1, const char *s2)
-{
-  size_t s1_name_len = str_num_before(s1, '=');
-  size_t s2_name_len = str_num_before(s2, '=');
-
-  return (s1_name_len == s2_name_len &&
-          tor_memeq(s1, s2, s1_name_len));
-}
-
-/** Free <b>env</b> (assuming it was produced by
- * process_environment_make). */
-void
-process_environment_free(process_environment_t *env)
-{
-  if (env == NULL) return;
-
-  /* As both an optimization hack to reduce consing on Unixoid systems
-   * and a nice way to ensure that some otherwise-Windows-specific
-   * code will always get tested before changes to it get merged, the
-   * strings which env->unixoid_environment_block points to are packed
-   * into env->windows_environment_block. */
-  tor_free(env->unixoid_environment_block);
-  tor_free(env->windows_environment_block);
-
-  tor_free(env);
-}
-
-/** Make a process_environment_t containing the environment variables
- * specified in <b>env_vars</b> (as C strings of the form
- * "NAME=VALUE"). */
-process_environment_t *
-process_environment_make(struct smartlist_t *env_vars)
-{
-  process_environment_t *env = tor_malloc_zero(sizeof(process_environment_t));
-  size_t n_env_vars = smartlist_len(env_vars);
-  size_t i;
-  size_t total_env_length;
-  smartlist_t *env_vars_sorted;
-
-  tor_assert(n_env_vars + 1 != 0);
-  env->unixoid_environment_block = tor_calloc(n_env_vars + 1, sizeof(char *));
-  /* env->unixoid_environment_block is already NULL-terminated,
-   * because we assume that NULL == 0 (and check that during compilation). */
-
-  total_env_length = 1; /* terminating NUL of terminating empty string */
-  for (i = 0; i < n_env_vars; ++i) {
-    const char *s = smartlist_get(env_vars, i);
-    size_t slen = strlen(s);
-
-    tor_assert(slen + 1 != 0);
-    tor_assert(slen + 1 < SIZE_MAX - total_env_length);
-    total_env_length += slen + 1;
-  }
-
-  env->windows_environment_block = tor_malloc_zero(total_env_length);
-  /* env->windows_environment_block is already
-   * (NUL-terminated-empty-string)-terminated. */
-
-  /* Some versions of Windows supposedly require that environment
-   * blocks be sorted.  Or maybe some Windows programs (or their
-   * runtime libraries) fail to look up strings in non-sorted
-   * environment blocks.
-   *
-   * Also, sorting strings makes it easy to find duplicate environment
-   * variables and environment-variable strings without an '=' on all
-   * OSes, and they can cause badness.  Let's complain about those. */
-  env_vars_sorted = smartlist_new();
-  smartlist_add_all(env_vars_sorted, env_vars);
-  smartlist_sort_strings(env_vars_sorted);
-
-  /* Now copy the strings into the environment blocks. */
-  {
-    char *cp = env->windows_environment_block;
-    const char *prev_env_var = NULL;
-
-    for (i = 0; i < n_env_vars; ++i) {
-      const char *s = smartlist_get(env_vars_sorted, i);
-      size_t slen = strlen(s);
-      size_t s_name_len = str_num_before(s, '=');
-
-      if (s_name_len == slen) {
-        log_warn(LD_GENERAL,
-                 "Preparing an environment containing a variable "
-                 "without a value: %s",
-                 s);
-      }
-      if (prev_env_var != NULL &&
-          environment_variable_names_equal(s, prev_env_var)) {
-        log_warn(LD_GENERAL,
-                 "Preparing an environment containing two variables "
-                 "with the same name: %s and %s",
-                 prev_env_var, s);
-      }
-
-      prev_env_var = s;
-
-      /* Actually copy the string into the environment. */
-      memcpy(cp, s, slen+1);
-      env->unixoid_environment_block[i] = cp;
-      cp += slen+1;
-    }
-
-    tor_assert(cp == env->windows_environment_block + total_env_length - 1);
-  }
-
-  smartlist_free(env_vars_sorted);
-
-  return env;
-}
-
-/** Return a newly allocated smartlist containing every variable in
- * this process's environment, as a NUL-terminated string of the form
- * "NAME=VALUE".  Note that on some/many/most/all OSes, the parent
- * process can put strings not of that form in our environment;
- * callers should try to not get crashed by that.
- *
- * The returned strings are heap-allocated, and must be freed by the
- * caller. */
-struct smartlist_t *
-get_current_process_environment_variables(void)
-{
-  smartlist_t *sl = smartlist_new();
-
-  char **environ_tmp; /* Not const char ** ? Really? */
-  for (environ_tmp = get_environment(); *environ_tmp; ++environ_tmp) {
-    smartlist_add(sl, tor_strdup(*environ_tmp));
-  }
-
-  return sl;
-}
-
-/** For each string s in <b>env_vars</b> such that
- * environment_variable_names_equal(s, <b>new_var</b>), remove it; if
- * <b>free_p</b> is non-zero, call <b>free_old</b>(s).  If
- * <b>new_var</b> contains '=', insert it into <b>env_vars</b>. */
-void
-set_environment_variable_in_smartlist(struct smartlist_t *env_vars,
-                                      const char *new_var,
-                                      void (*free_old)(void*),
-                                      int free_p)
-{
-  SMARTLIST_FOREACH_BEGIN(env_vars, const char *, s) {
-    if (environment_variable_names_equal(s, new_var)) {
-      SMARTLIST_DEL_CURRENT(env_vars, s);
-      if (free_p) {
-        free_old((void *)s);
-      }
-    }
-  } SMARTLIST_FOREACH_END(s);
-
-  if (strchr(new_var, '=') != NULL) {
-    smartlist_add(env_vars, (void *)new_var);
-  }
-}
-
-#ifdef _WIN32
-/** Read from a handle <b>h</b> into <b>buf</b>, up to <b>count</b> bytes.  If
- * <b>hProcess</b> is NULL, the function will return immediately if there is
- * nothing more to read. Otherwise <b>hProcess</b> should be set to the handle
- * to the process owning the <b>h</b>. In this case, the function will exit
- * only once the process has exited, or <b>count</b> bytes are read. Returns
- * the number of bytes read, or -1 on error. */
-ssize_t
-tor_read_all_handle(HANDLE h, char *buf, size_t count,
-                    const process_handle_t *process)
-{
-  size_t numread = 0;
-  BOOL retval;
-  DWORD byte_count;
-  BOOL process_exited = FALSE;
-
-  if (count > SIZE_T_CEILING || count > SSIZE_MAX)
-    return -1;
-
-  while (numread != count) {
-    /* Check if there is anything to read */
-    retval = PeekNamedPipe(h, NULL, 0, NULL, &byte_count, NULL);
-    if (!retval) {
-      log_warn(LD_GENERAL,
-        "Failed to peek from handle: %s",
-        format_win32_error(GetLastError()));
-      return -1;
-    } else if (0 == byte_count) {
-      /* Nothing available: process exited or it is busy */
-
-      /* Exit if we don't know whether the process is running */
-      if (NULL == process)
-        break;
-
-      /* The process exited and there's nothing left to read from it */
-      if (process_exited)
-        break;
-
-      /* If process is not running, check for output one more time in case
-         it wrote something after the peek was performed. Otherwise keep on
-         waiting for output */
-      tor_assert(process != NULL);
-      byte_count = WaitForSingleObject(process->pid.hProcess, 0);
-      if (WAIT_TIMEOUT != byte_count)
-        process_exited = TRUE;
-
-      continue;
-    }
-
-    /* There is data to read; read it */
-    retval = ReadFile(h, buf+numread, count-numread, &byte_count, NULL);
-    tor_assert(byte_count + numread <= count);
-    if (!retval) {
-      log_warn(LD_GENERAL, "Failed to read from handle: %s",
-        format_win32_error(GetLastError()));
-      return -1;
-    } else if (0 == byte_count) {
-      /* End of file */
-      break;
-    }
-    numread += byte_count;
-  }
-  return (ssize_t)numread;
-}
-#else
-/** Read from a handle <b>h</b> into <b>buf</b>, up to <b>count</b> bytes.  If
- * <b>process</b> is NULL, the function will return immediately if there is
- * nothing more to read. Otherwise data will be read until end of file, or
- * <b>count</b> bytes are read.  Returns the number of bytes read, or -1 on
- * error. Sets <b>eof</b> to true if <b>eof</b> is not NULL and the end of the
- * file has been reached. */
-ssize_t
-tor_read_all_handle(FILE *h, char *buf, size_t count,
-                    const process_handle_t *process,
-                    int *eof)
-{
-  size_t numread = 0;
-  char *retval;
-
-  if (eof)
-    *eof = 0;
-
-  if (count > SIZE_T_CEILING || count > SSIZE_MAX)
-    return -1;
-
-  while (numread != count) {
-    /* Use fgets because that is what we use in log_from_pipe() */
-    retval = fgets(buf+numread, (int)(count-numread), h);
-    if (NULL == retval) {
-      if (feof(h)) {
-        log_debug(LD_GENERAL, "fgets() reached end of file");
-        if (eof)
-          *eof = 1;
-        break;
-      } else {
-        if (EAGAIN == errno) {
-          if (process)
-            continue;
-          else
-            break;
-        } else {
-          log_warn(LD_GENERAL, "fgets() from handle failed: %s",
-                   strerror(errno));
-          return -1;
-        }
-      }
-    }
-    tor_assert(retval != NULL);
-    tor_assert(strlen(retval) + numread <= count);
-    numread += strlen(retval);
-  }
-
-  log_debug(LD_GENERAL, "fgets() read %d bytes from handle", (int)numread);
-  return (ssize_t)numread;
-}
-#endif
-
-/** Read from stdout of a process until the process exits. */
-ssize_t
-tor_read_all_from_process_stdout(const process_handle_t *process_handle,
-                                 char *buf, size_t count)
-{
-#ifdef _WIN32
-  return tor_read_all_handle(process_handle->stdout_pipe, buf, count,
-                             process_handle);
-#else
-  return tor_read_all_handle(process_handle->stdout_handle, buf, count,
-                             process_handle, NULL);
-#endif
-}
-
-/** Read from stdout of a process until the process exits. */
-ssize_t
-tor_read_all_from_process_stderr(const process_handle_t *process_handle,
-                                 char *buf, size_t count)
-{
-#ifdef _WIN32
-  return tor_read_all_handle(process_handle->stderr_pipe, buf, count,
-                             process_handle);
-#else
-  return tor_read_all_handle(process_handle->stderr_handle, buf, count,
-                             process_handle, NULL);
-#endif
-}
-
-/** Split buf into lines, and add to smartlist. The buffer <b>buf</b> will be
- * modified. The resulting smartlist will consist of pointers to buf, so there
- * is no need to free the contents of sl. <b>buf</b> must be a NUL-terminated
- * string. <b>len</b> should be set to the length of the buffer excluding the
- * NUL. Non-printable characters (including NUL) will be replaced with "." */
-int
-tor_split_lines(smartlist_t *sl, char *buf, int len)
-{
-  /* Index in buf of the start of the current line */
-  int start = 0;
-  /* Index in buf of the current character being processed */
-  int cur = 0;
-  /* Are we currently in a line */
-  char in_line = 0;
-
-  /* Loop over string */
-  while (cur < len) {
-    /* Loop until end of line or end of string */
-    for (; cur < len; cur++) {
-      if (in_line) {
-        if ('\r' == buf[cur] || '\n' == buf[cur]) {
-          /* End of line */
-          buf[cur] = '\0';
-          /* Point cur to the next line */
-          cur++;
-          /* Line starts at start and ends with a nul */
-          break;
-        } else {
-          if (!TOR_ISPRINT(buf[cur]))
-            buf[cur] = '.';
-        }
-      } else {
-        if ('\r' == buf[cur] || '\n' == buf[cur]) {
-          /* Skip leading vertical space */
-          ;
-        } else {
-          in_line = 1;
-          start = cur;
-          if (!TOR_ISPRINT(buf[cur]))
-            buf[cur] = '.';
-        }
-      }
-    }
-    /* We are at the end of the line or end of string. If in_line is true there
-     * is a line which starts at buf+start and ends at a NUL. cur points to
-     * the character after the NUL. */
-    if (in_line)
-      smartlist_add(sl, (void *)(buf+start));
-    in_line = 0;
-  }
-  return smartlist_len(sl);
-}
-
-/** Return a string corresponding to <b>stream_status</b>. */
-const char *
-stream_status_to_string(enum stream_status stream_status)
-{
-  switch (stream_status) {
-    case IO_STREAM_OKAY:
-      return "okay";
-    case IO_STREAM_EAGAIN:
-      return "temporarily unavailable";
-    case IO_STREAM_TERM:
-      return "terminated";
-    case IO_STREAM_CLOSED:
-      return "closed";
-    default:
-      tor_fragile_assert();
-      return "unknown";
-  }
-}
-
-/* DOCDOC */
-static void
-log_portfw_spawn_error_message(const char *buf,
-                               const char *executable, int *child_status)
-{
-  /* Parse error message */
-  int retval, child_state, saved_errno;
-  retval = tor_sscanf(buf, SPAWN_ERROR_MESSAGE "%x/%x",
-                      &child_state, &saved_errno);
-  if (retval == 2) {
-    log_warn(LD_GENERAL,
-             "Failed to start child process \"%s\" in state %d: %s",
-             executable, child_state, strerror(saved_errno));
-    if (child_status)
-      *child_status = 1;
-  } else {
-    /* Failed to parse message from child process, log it as a
-       warning */
-    log_warn(LD_GENERAL,
-             "Unexpected message from port forwarding helper \"%s\": %s",
-             executable, buf);
-  }
-}
-
-#ifdef _WIN32
-
-/** Return a smartlist containing lines outputted from
- *  <b>handle</b>. Return NULL on error, and set
- *  <b>stream_status_out</b> appropriately. */
-MOCK_IMPL(smartlist_t *,
-tor_get_lines_from_handle, (HANDLE *handle,
-                            enum stream_status *stream_status_out))
-{
-  int pos;
-  char stdout_buf[600] = {0};
-  smartlist_t *lines = NULL;
-
-  tor_assert(stream_status_out);
-
-  *stream_status_out = IO_STREAM_TERM;
-
-  pos = tor_read_all_handle(handle, stdout_buf, sizeof(stdout_buf) - 1, NULL);
-  if (pos < 0) {
-    *stream_status_out = IO_STREAM_TERM;
-    return NULL;
-  }
-  if (pos == 0) {
-    *stream_status_out = IO_STREAM_EAGAIN;
-    return NULL;
-  }
-
-  /* End with a null even if there isn't a \r\n at the end */
-  /* TODO: What if this is a partial line? */
-  stdout_buf[pos] = '\0';
-
-  /* Split up the buffer */
-  lines = smartlist_new();
-  tor_split_lines(lines, stdout_buf, pos);
-
-  /* Currently 'lines' is populated with strings residing on the
-     stack. Replace them with their exact copies on the heap: */
-  SMARTLIST_FOREACH(lines, char *, line,
-                    SMARTLIST_REPLACE_CURRENT(lines, line, tor_strdup(line)));
-
-  *stream_status_out = IO_STREAM_OKAY;
-
-  return lines;
-}
-
-/** Read from stream, and send lines to log at the specified log level.
- * Returns -1 if there is a error reading, and 0 otherwise.
- * If the generated stream is flushed more often than on new lines, or
- * a read exceeds 256 bytes, lines will be truncated. This should be fixed,
- * along with the corresponding problem on *nix (see bug #2045).
- */
-static int
-log_from_handle(HANDLE *pipe, int severity)
-{
-  char buf[256];
-  int pos;
-  smartlist_t *lines;
-
-  pos = tor_read_all_handle(pipe, buf, sizeof(buf) - 1, NULL);
-  if (pos < 0) {
-    /* Error */
-    log_warn(LD_GENERAL, "Failed to read data from subprocess");
-    return -1;
-  }
-
-  if (0 == pos) {
-    /* There's nothing to read (process is busy or has exited) */
-    log_debug(LD_GENERAL, "Subprocess had nothing to say");
-    return 0;
-  }
-
-  /* End with a null even if there isn't a \r\n at the end */
-  /* TODO: What if this is a partial line? */
-  buf[pos] = '\0';
-  log_debug(LD_GENERAL, "Subprocess had %d bytes to say", pos);
-
-  /* Split up the buffer */
-  lines = smartlist_new();
-  tor_split_lines(lines, buf, pos);
-
-  /* Log each line */
-  SMARTLIST_FOREACH(lines, char *, line,
-  {
-    log_fn(severity, LD_GENERAL, "Port forwarding helper says: %s", line);
-  });
-  smartlist_free(lines);
-
-  return 0;
-}
-
-#else
-
-/** Return a smartlist containing lines outputted from
- *  <b>handle</b>. Return NULL on error, and set
- *  <b>stream_status_out</b> appropriately. */
-MOCK_IMPL(smartlist_t *,
-tor_get_lines_from_handle, (FILE *handle,
-                            enum stream_status *stream_status_out))
-{
-  enum stream_status stream_status;
-  char stdout_buf[400];
-  smartlist_t *lines = NULL;
-
-  while (1) {
-    memset(stdout_buf, 0, sizeof(stdout_buf));
-
-    stream_status = get_string_from_pipe(handle,
-                                         stdout_buf, sizeof(stdout_buf) - 1);
-    if (stream_status != IO_STREAM_OKAY)
-      goto done;
-
-    if (!lines) lines = smartlist_new();
-    smartlist_add(lines, tor_strdup(stdout_buf));
-  }
-
- done:
-  *stream_status_out = stream_status;
-  return lines;
-}
-
-/** Read from stream, and send lines to log at the specified log level.
- * Returns 1 if stream is closed normally, -1 if there is a error reading, and
- * 0 otherwise. Handles lines from tor-fw-helper and
- * tor_spawn_background() specially.
- */
-static int
-log_from_pipe(FILE *stream, int severity, const char *executable,
-              int *child_status)
-{
-  char buf[256];
-  enum stream_status r;
-
-  for (;;) {
-    r = get_string_from_pipe(stream, buf, sizeof(buf) - 1);
-
-    if (r == IO_STREAM_CLOSED) {
-      return 1;
-    } else if (r == IO_STREAM_EAGAIN) {
-      return 0;
-    } else if (r == IO_STREAM_TERM) {
-      return -1;
-    }
-
-    tor_assert(r == IO_STREAM_OKAY);
-
-    /* Check if buf starts with SPAWN_ERROR_MESSAGE */
-    if (strcmpstart(buf, SPAWN_ERROR_MESSAGE) == 0) {
-      log_portfw_spawn_error_message(buf, executable, child_status);
-    } else {
-      log_fn(severity, LD_GENERAL, "Port forwarding helper says: %s", buf);
-    }
-  }
-
-  /* We should never get here */
-  return -1;
-}
-#endif
-
-/** Reads from <b>stream</b> and stores input in <b>buf_out</b> making
- *  sure it's below <b>count</b> bytes.
- *  If the string has a trailing newline, we strip it off.
- *
- * This function is specifically created to handle input from managed
- * proxies, according to the pluggable transports spec. Make sure it
- * fits your needs before using it.
- *
- * Returns:
- * IO_STREAM_CLOSED: If the stream is closed.
- * IO_STREAM_EAGAIN: If there is nothing to read and we should check back
- *  later.
- * IO_STREAM_TERM: If something is wrong with the stream.
- * IO_STREAM_OKAY: If everything went okay and we got a string
- *  in <b>buf_out</b>. */
-enum stream_status
-get_string_from_pipe(FILE *stream, char *buf_out, size_t count)
-{
-  char *retval;
-  size_t len;
-
-  tor_assert(count <= INT_MAX);
-
-  retval = fgets(buf_out, (int)count, stream);
-
-  if (!retval) {
-    if (feof(stream)) {
-      /* Program has closed stream (probably it exited) */
-      /* TODO: check error */
-      return IO_STREAM_CLOSED;
-    } else {
-      if (EAGAIN == errno) {
-        /* Nothing more to read, try again next time */
-        return IO_STREAM_EAGAIN;
-      } else {
-        /* There was a problem, abandon this child process */
-        return IO_STREAM_TERM;
-      }
-    }
-  } else {
-    len = strlen(buf_out);
-    if (len == 0) {
-      /* this probably means we got a NUL at the start of the string. */
-      return IO_STREAM_EAGAIN;
-    }
-
-    if (buf_out[len - 1] == '\n') {
-      /* Remove the trailing newline */
-      buf_out[len - 1] = '\0';
-    } else {
-      /* No newline; check whether we overflowed the buffer */
-      if (!feof(stream))
-        log_info(LD_GENERAL,
-                 "Line from stream was truncated: %s", buf_out);
-      /* TODO: What to do with this error? */
-    }
-
-    return IO_STREAM_OKAY;
-  }
-
-  /* We should never get here */
-  return IO_STREAM_TERM;
-}
-
-/** Parse a <b>line</b> from tor-fw-helper and issue an appropriate
- *  log message to our user. */
-static void
-handle_fw_helper_line(const char *executable, const char *line)
-{
-  smartlist_t *tokens = smartlist_new();
-  char *message = NULL;
-  char *message_for_log = NULL;
-  const char *external_port = NULL;
-  const char *internal_port = NULL;
-  const char *result = NULL;
-  int port = 0;
-  int success = 0;
-
-  if (strcmpstart(line, SPAWN_ERROR_MESSAGE) == 0) {
-    /* We need to check for SPAWN_ERROR_MESSAGE again here, since it's
-     * possible that it got sent after we tried to read it in log_from_pipe.
-     *
-     * XXX Ideally, we should be using one of stdout/stderr for the real
-     * output, and one for the output of the startup code.  We used to do that
-     * before cd05f35d2c.
-     */
-    int child_status;
-    log_portfw_spawn_error_message(line, executable, &child_status);
-    goto done;
-  }
-
-  smartlist_split_string(tokens, line, NULL,
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
-
-  if (smartlist_len(tokens) < 5)
-    goto err;
-
-  if (strcmp(smartlist_get(tokens, 0), "tor-fw-helper") ||
-      strcmp(smartlist_get(tokens, 1), "tcp-forward"))
-    goto err;
-
-  external_port = smartlist_get(tokens, 2);
-  internal_port = smartlist_get(tokens, 3);
-  result = smartlist_get(tokens, 4);
-
-  if (smartlist_len(tokens) > 5) {
-    /* If there are more than 5 tokens, they are part of [<message>].
-       Let's use a second smartlist to form the whole message;
-       strncat loops suck. */
-    int i;
-    int message_words_n = smartlist_len(tokens) - 5;
-    smartlist_t *message_sl = smartlist_new();
-    for (i = 0; i < message_words_n; i++)
-      smartlist_add(message_sl, smartlist_get(tokens, 5+i));
-
-    tor_assert(smartlist_len(message_sl) > 0);
-    message = smartlist_join_strings(message_sl, " ", 0, NULL);
-
-    /* wrap the message in log-friendly wrapping */
-    tor_asprintf(&message_for_log, " ('%s')", message);
-
-    smartlist_free(message_sl);
-  }
-
-  port = atoi(external_port);
-  if (port < 1 || port > 65535)
-    goto err;
-
-  port = atoi(internal_port);
-  if (port < 1 || port > 65535)
-    goto err;
-
-  if (!strcmp(result, "SUCCESS"))
-    success = 1;
-  else if (!strcmp(result, "FAIL"))
-    success = 0;
-  else
-    goto err;
-
-  if (!success) {
-    log_warn(LD_GENERAL, "Tor was unable to forward TCP port '%s' to '%s'%s. "
-             "Please make sure that your router supports port "
-             "forwarding protocols (like NAT-PMP). Note that if '%s' is "
-             "your ORPort, your relay will be unable to receive inbound "
-             "traffic.", external_port, internal_port,
-             message_for_log ? message_for_log : "",
-             internal_port);
-  } else {
-    log_info(LD_GENERAL,
-             "Tor successfully forwarded TCP port '%s' to '%s'%s.",
-             external_port, internal_port,
-             message_for_log ? message_for_log : "");
-  }
-
-  goto done;
-
- err:
-  log_warn(LD_GENERAL, "tor-fw-helper sent us a string we could not "
-           "parse (%s).", line);
-
- done:
-  SMARTLIST_FOREACH(tokens, char *, cp, tor_free(cp));
-  smartlist_free(tokens);
-  tor_free(message);
-  tor_free(message_for_log);
-}
-
-/** Read what tor-fw-helper has to say in its stdout and handle it
- *  appropriately */
-static int
-handle_fw_helper_output(const char *executable,
-                        process_handle_t *process_handle)
-{
-  smartlist_t *fw_helper_output = NULL;
-  enum stream_status stream_status = 0;
-
-  fw_helper_output =
-    tor_get_lines_from_handle(tor_process_get_stdout_pipe(process_handle),
-                              &stream_status);
-  if (!fw_helper_output) { /* didn't get any output from tor-fw-helper */
-    /* if EAGAIN we should retry in the future */
-    return (stream_status == IO_STREAM_EAGAIN) ? 0 : -1;
-  }
-
-  /* Handle the lines we got: */
-  SMARTLIST_FOREACH_BEGIN(fw_helper_output, char *, line) {
-    handle_fw_helper_line(executable, line);
-    tor_free(line);
-  } SMARTLIST_FOREACH_END(line);
-
-  smartlist_free(fw_helper_output);
-
-  return 0;
-}
-
-/** Spawn tor-fw-helper and ask it to forward the ports in
- *  <b>ports_to_forward</b>. <b>ports_to_forward</b> contains strings
- *  of the form "<external port>:<internal port>", which is the format
- *  that tor-fw-helper expects. */
-void
-tor_check_port_forwarding(const char *filename,
-                          smartlist_t *ports_to_forward,
-                          time_t now)
-{
-/* When fw-helper succeeds, how long do we wait until running it again */
-#define TIME_TO_EXEC_FWHELPER_SUCCESS 300
-/* When fw-helper failed to start, how long do we wait until running it again
- */
-#define TIME_TO_EXEC_FWHELPER_FAIL 60
-
-  /* Static variables are initialized to zero, so child_handle.status=0
-   * which corresponds to it not running on startup */
-  static process_handle_t *child_handle=NULL;
-
-  static time_t time_to_run_helper = 0;
-  int stderr_status, retval;
-  int stdout_status = 0;
-
-  tor_assert(filename);
-
-  /* Start the child, if it is not already running */
-  if ((!child_handle || child_handle->status != PROCESS_STATUS_RUNNING) &&
-      time_to_run_helper < now) {
-    /*tor-fw-helper cli looks like this: tor_fw_helper -p :5555 -p 4555:1111 */
-    const char **argv; /* cli arguments */
-    int args_n, status;
-    int argv_index = 0; /* index inside 'argv' */
-
-    tor_assert(smartlist_len(ports_to_forward) > 0);
-
-    /* check for overflow during 'argv' allocation:
-       (len(ports_to_forward)*2 + 2)*sizeof(char*) > SIZE_MAX ==
-       len(ports_to_forward) > (((SIZE_MAX/sizeof(char*)) - 2)/2) */
-    if ((size_t) smartlist_len(ports_to_forward) >
-        (((SIZE_MAX/sizeof(char*)) - 2)/2)) {
-      log_warn(LD_GENERAL,
-               "Overflow during argv allocation. This shouldn't happen.");
-      return;
-    }
-    /* check for overflow during 'argv_index' increase:
-       ((len(ports_to_forward)*2 + 2) > INT_MAX) ==
-       len(ports_to_forward) > (INT_MAX - 2)/2 */
-    if (smartlist_len(ports_to_forward) > (INT_MAX - 2)/2) {
-      log_warn(LD_GENERAL,
-               "Overflow during argv_index increase. This shouldn't happen.");
-      return;
-    }
-
-    /* Calculate number of cli arguments: one for the filename, two
-       for each smartlist element (one for "-p" and one for the
-       ports), and one for the final NULL. */
-    args_n = 1 + 2*smartlist_len(ports_to_forward) + 1;
-    argv = tor_calloc(args_n, sizeof(char *));
-
-    argv[argv_index++] = filename;
-    SMARTLIST_FOREACH_BEGIN(ports_to_forward, const char *, port) {
-      argv[argv_index++] = "-p";
-      argv[argv_index++] = port;
-    } SMARTLIST_FOREACH_END(port);
-    argv[argv_index] = NULL;
-
-    /* Assume tor-fw-helper will succeed, start it later*/
-    time_to_run_helper = now + TIME_TO_EXEC_FWHELPER_SUCCESS;
-
-    if (child_handle) {
-      tor_process_handle_destroy(child_handle, 1);
-      child_handle = NULL;
-    }
-
-#ifdef _WIN32
-    /* Passing NULL as lpApplicationName makes Windows search for the .exe */
-    status = tor_spawn_background(NULL, argv, NULL, &child_handle);
-#else
-    status = tor_spawn_background(filename, argv, NULL, &child_handle);
-#endif
-
-    tor_free_((void*)argv);
-    argv=NULL;
-
-    if (PROCESS_STATUS_ERROR == status) {
-      log_warn(LD_GENERAL, "Failed to start port forwarding helper %s",
-              filename);
-      time_to_run_helper = now + TIME_TO_EXEC_FWHELPER_FAIL;
-      return;
-    }
-
-    log_info(LD_GENERAL,
-             "Started port forwarding helper (%s) with pid '%d'",
-             filename, tor_process_get_pid(child_handle));
-  }
-
-  /* If child is running, read from its stdout and stderr) */
-  if (child_handle && PROCESS_STATUS_RUNNING == child_handle->status) {
-    /* Read from stdout/stderr and log result */
-    retval = 0;
-#ifdef _WIN32
-    stderr_status = log_from_handle(child_handle->stderr_pipe, LOG_INFO);
-#else
-    stderr_status = log_from_pipe(child_handle->stderr_handle,
-                                  LOG_INFO, filename, &retval);
-#endif
-    if (handle_fw_helper_output(filename, child_handle) < 0) {
-      log_warn(LD_GENERAL, "Failed to handle fw helper output.");
-      stdout_status = -1;
-      retval = -1;
-    }
-
-    if (retval) {
-      /* There was a problem in the child process */
-      time_to_run_helper = now + TIME_TO_EXEC_FWHELPER_FAIL;
-    }
-
-    /* Combine the two statuses in order of severity */
-    if (-1 == stdout_status || -1 == stderr_status)
-      /* There was a failure */
-      retval = -1;
-#ifdef _WIN32
-    else if (!child_handle || tor_get_exit_code(child_handle, 0, NULL) !=
-             PROCESS_EXIT_RUNNING) {
-      /* process has exited or there was an error */
-      /* TODO: Do something with the process return value */
-      /* TODO: What if the process output something since
-       * between log_from_handle and tor_get_exit_code? */
-      retval = 1;
-    }
-#else
-    else if (1 == stdout_status || 1 == stderr_status)
-      /* stdout or stderr was closed, the process probably
-       * exited. It will be reaped by waitpid() in main.c */
-      /* TODO: Do something with the process return value */
-      retval = 1;
-#endif
-    else
-      /* Both are fine */
-      retval = 0;
-
-    /* If either pipe indicates a failure, act on it */
-    if (0 != retval) {
-      if (1 == retval) {
-        log_info(LD_GENERAL, "Port forwarding helper terminated");
-        child_handle->status = PROCESS_STATUS_NOTRUNNING;
-      } else {
-        log_warn(LD_GENERAL, "Failed to read from port forwarding helper");
-        child_handle->status = PROCESS_STATUS_ERROR;
-      }
-
-      /* TODO: The child might not actually be finished (maybe it failed or
-         closed stdout/stderr), so maybe we shouldn't start another? */
-    }
-  }
-}
-
-/** Initialize the insecure RNG <b>rng</b> from a seed value <b>seed</b>. */
-void
-tor_init_weak_random(tor_weak_rng_t *rng, unsigned seed)
-{
-  rng->state = (uint32_t)(seed & 0x7fffffff);
-}
-
-/** Return a randomly chosen value in the range 0..TOR_WEAK_RANDOM_MAX based
- * on the RNG state of <b>rng</b>.  This entropy will not be cryptographically
- * strong; do not rely on it for anything an adversary should not be able to
- * predict. */
-int32_t
-tor_weak_random(tor_weak_rng_t *rng)
-{
-  /* Here's a linear congruential generator. OpenBSD and glibc use these
-   * parameters; they aren't too bad, and should have maximal period over the
-   * range 0..INT32_MAX. We don't want to use the platform rand() or random(),
-   * since some platforms have bad weak RNGs that only return values in the
-   * range 0..INT16_MAX, which just isn't enough. */
-  rng->state = (rng->state * 1103515245 + 12345) & 0x7fffffff;
-  return (int32_t) rng->state;
-}
-
-/** Return a random number in the range [0 , <b>top</b>). {That is, the range
- * of integers i such that 0 <= i < top.}  Chooses uniformly.  Requires that
- * top is greater than 0. This randomness is not cryptographically strong; do
- * not rely on it for anything an adversary should not be able to predict. */
-int32_t
-tor_weak_random_range(tor_weak_rng_t *rng, int32_t top)
-{
-  /* We don't want to just do tor_weak_random() % top, since random() is often
-   * implemented with an LCG whose modulus is a power of 2, and those are
-   * cyclic in their low-order bits. */
-  int divisor, result;
-  tor_assert(top > 0);
-  divisor = TOR_WEAK_RANDOM_MAX / top;
-  do {
-    result = (int32_t)(tor_weak_random(rng) / divisor);
-  } while (result >= top);
-  return result;
-}
-
-/** Cast a given double value to a int64_t. Return 0 if number is NaN.
- * Returns either INT64_MIN or INT64_MAX if number is outside of the int64_t
- * range. */
-int64_t
-clamp_double_to_int64(double number)
-{
-  int exponent;
-
-#if (defined(__MINGW32__) || defined(__MINGW64__)) && GCC_VERSION >= 409
-/*
-  Mingw's math.h uses gcc's __builtin_choose_expr() facility to declare
-  isnan, isfinite, and signbit.  But as implemented in at least some
-  versions of gcc, __builtin_choose_expr() can generate type warnings
-  even from branches that are not taken.  So, suppress those warnings.
-*/
-#define PROBLEMATIC_FLOAT_CONVERSION_WARNING
-DISABLE_GCC_WARNING(float-conversion)
-#endif
-
-/*
-  With clang 4.0 we apparently run into "double promotion" warnings here,
-  since clang thinks we're promoting a double to a long double.
- */
-#if defined(__clang__)
-#if __has_warning("-Wdouble-promotion")
-#define PROBLEMATIC_DOUBLE_PROMOTION_WARNING
-DISABLE_GCC_WARNING(double-promotion)
-#endif
-#endif
-
-  /* NaN is a special case that can't be used with the logic below. */
-  if (isnan(number)) {
-    return 0;
-  }
-
-  /* Time to validate if result can overflows a int64_t value. Fun with
-   * float! Find that exponent exp such that
-   *    number == x * 2^exp
-   * for some x with abs(x) in [0.5, 1.0). Note that this implies that the
-   * magnitude of number is strictly less than 2^exp.
-   *
-   * If number is infinite, the call to frexp is legal but the contents of
-   * are exponent unspecified. */
-  frexp(number, &exponent);
-
-  /* If the magnitude of number is strictly less than 2^63, the truncated
-   * version of number is guaranteed to be representable. The only
-   * representable integer for which this is not the case is INT64_MIN, but
-   * it is covered by the logic below. */
-  if (isfinite(number) && exponent <= 63) {
-    return (int64_t)number;
-  }
-
-  /* Handle infinities and finite numbers with magnitude >= 2^63. */
-  return signbit(number) ? INT64_MIN : INT64_MAX;
-
-#ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING
-ENABLE_GCC_WARNING(double-promotion)
-#endif
-#ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING
-ENABLE_GCC_WARNING(float-conversion)
-#endif
-}
-
-/** Return a uint64_t value from <b>a</b> in network byte order. */
-uint64_t
-tor_htonll(uint64_t a)
-{
-#ifdef WORDS_BIGENDIAN
-  /* Big endian. */
-  return a;
-#else /* WORDS_BIGENDIAN */
-  /* Little endian. The worst... */
-  return htonl((uint32_t)(a>>32)) |
-    (((uint64_t)htonl((uint32_t)a))<<32);
-#endif /* WORDS_BIGENDIAN */
-}
-
-/** Return a uint64_t value from <b>a</b> in host byte order. */
-uint64_t
-tor_ntohll(uint64_t a)
-{
-  return tor_htonll(a);
-}
-
diff --git a/src/common/util.h b/src/common/util.h
deleted file mode 100644
index 479fc8d610..0000000000
--- a/src/common/util.h
+++ /dev/null
@@ -1,561 +0,0 @@
-/* Copyright (c) 2003-2004, Roger Dingledine
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file util.h
- * \brief Headers for util.c
- **/
-
-#ifndef TOR_UTIL_H
-#define TOR_UTIL_H
-
-#include "orconfig.h"
-#include "torint.h"
-#include "compat.h"
-#include "di_ops.h"
-#include "testsupport.h"
-#include <stdio.h>
-#include <stdlib.h>
-#ifdef _WIN32
-/* for the correct alias to struct stat */
-#include <sys/stat.h>
-#endif
-#include "util_bug.h"
-
-#ifndef O_BINARY
-#define O_BINARY 0
-#endif
-#ifndef O_TEXT
-#define O_TEXT 0
-#endif
-#ifndef O_NOFOLLOW
-#define O_NOFOLLOW 0
-#endif
-
-/* If we're building with dmalloc, we want all of our memory allocation
- * functions to take an extra file/line pair of arguments.  If not, not.
- * We define DMALLOC_PARAMS to the extra parameters to insert in the
- * function prototypes, and DMALLOC_ARGS to the extra arguments to add
- * to calls. */
-#ifdef USE_DMALLOC
-#define DMALLOC_PARAMS , const char *file, const int line
-#define DMALLOC_ARGS , SHORT_FILE__, __LINE__
-#else
-#define DMALLOC_PARAMS
-#define DMALLOC_ARGS
-#endif
-
-/* Memory management */
-void *tor_malloc_(size_t size DMALLOC_PARAMS) ATTR_MALLOC;
-void *tor_malloc_zero_(size_t size DMALLOC_PARAMS) ATTR_MALLOC;
-void *tor_calloc_(size_t nmemb, size_t size DMALLOC_PARAMS) ATTR_MALLOC;
-void *tor_realloc_(void *ptr, size_t size DMALLOC_PARAMS);
-void *tor_reallocarray_(void *ptr, size_t size1, size_t size2 DMALLOC_PARAMS);
-char *tor_strdup_(const char *s DMALLOC_PARAMS) ATTR_MALLOC ATTR_NONNULL((1));
-char *tor_strndup_(const char *s, size_t n DMALLOC_PARAMS)
-  ATTR_MALLOC ATTR_NONNULL((1));
-void *tor_memdup_(const void *mem, size_t len DMALLOC_PARAMS)
-  ATTR_MALLOC ATTR_NONNULL((1));
-void *tor_memdup_nulterm_(const void *mem, size_t len DMALLOC_PARAMS)
-  ATTR_MALLOC ATTR_NONNULL((1));
-void tor_free_(void *mem);
-uint64_t tor_htonll(uint64_t a);
-uint64_t tor_ntohll(uint64_t a);
-#ifdef USE_DMALLOC
-extern int dmalloc_free(const char *file, const int line, void *pnt,
-                        const int func_id);
-#define tor_free(p) STMT_BEGIN \
-    if (PREDICT_LIKELY((p)!=NULL)) {                \
-      dmalloc_free(SHORT_FILE__, __LINE__, (p), 0); \
-      (p)=NULL;                                     \
-    }                                               \
-  STMT_END
-#else
-/** Release memory allocated by tor_malloc, tor_realloc, tor_strdup, etc.
- * Unlike the free() function, tor_free() will still work on NULL pointers,
- * and it sets the pointer value to NULL after freeing it.
- *
- * This is a macro.  If you need a function pointer to release memory from
- * tor_malloc(), use tor_free_().
- */
-#define tor_free(p) STMT_BEGIN                                 \
-    if (PREDICT_LIKELY((p)!=NULL)) {                           \
-      raw_free(p);                                             \
-      (p)=NULL;                                                \
-    }                                                          \
-  STMT_END
-#endif
-
-#define tor_malloc(size)       tor_malloc_(size DMALLOC_ARGS)
-#define tor_malloc_zero(size)  tor_malloc_zero_(size DMALLOC_ARGS)
-#define tor_calloc(nmemb,size) tor_calloc_(nmemb, size DMALLOC_ARGS)
-#define tor_realloc(ptr, size) tor_realloc_(ptr, size DMALLOC_ARGS)
-#define tor_reallocarray(ptr, sz1, sz2) \
-  tor_reallocarray_((ptr), (sz1), (sz2) DMALLOC_ARGS)
-#define tor_strdup(s)          tor_strdup_(s DMALLOC_ARGS)
-#define tor_strndup(s, n)      tor_strndup_(s, n DMALLOC_ARGS)
-#define tor_memdup(s, n)       tor_memdup_(s, n DMALLOC_ARGS)
-#define tor_memdup_nulterm(s, n)       tor_memdup_nulterm_(s, n DMALLOC_ARGS)
-
-/* Aliases for the underlying system malloc/realloc/free. Only use
- * them to indicate "I really want the underlying system function, I know
- * what I'm doing." */
-#define raw_malloc  malloc
-#define raw_realloc realloc
-#define raw_free    free
-#define raw_strdup  strdup
-
-void tor_log_mallinfo(int severity);
-
-/** Return the offset of <b>member</b> within the type <b>tp</b>, in bytes */
-#if defined(__GNUC__) && __GNUC__ > 3
-#define STRUCT_OFFSET(tp, member) __builtin_offsetof(tp, member)
-#else
- #define STRUCT_OFFSET(tp, member) \
-   ((off_t) (((char*)&((tp*)0)->member)-(char*)0))
-#endif
-
-/** Macro: yield a pointer to the field at position <b>off</b> within the
- * structure <b>st</b>.  Example:
- * <pre>
- *   struct a { int foo; int bar; } x;
- *   off_t bar_offset = STRUCT_OFFSET(struct a, bar);
- *   int *bar_p = STRUCT_VAR_P(&x, bar_offset);
- *   *bar_p = 3;
- * </pre>
- */
-#define STRUCT_VAR_P(st, off) ((void*) ( ((char*)(st)) + (off) ) )
-
-/** Macro: yield a pointer to an enclosing structure given a pointer to
- * a substructure at offset <b>off</b>. Example:
- * <pre>
- *   struct base { ... };
- *   struct subtype { int x; struct base b; } x;
- *   struct base *bp = &x.base;
- *   struct *sp = SUBTYPE_P(bp, struct subtype, b);
- * </pre>
- */
-#define SUBTYPE_P(p, subtype, basemember) \
-  ((void*) ( ((char*)(p)) - STRUCT_OFFSET(subtype, basemember) ))
-
-/* Logic */
-/** Macro: true if two values have the same boolean value. */
-#define bool_eq(a,b) (!(a)==!(b))
-/** Macro: true if two values have different boolean values. */
-#define bool_neq(a,b) (!(a)!=!(b))
-
-/* Math functions */
-double tor_mathlog(double d) ATTR_CONST;
-long tor_lround(double d) ATTR_CONST;
-int64_t tor_llround(double d) ATTR_CONST;
-int tor_log2(uint64_t u64) ATTR_CONST;
-uint64_t round_to_power_of_2(uint64_t u64);
-unsigned round_to_next_multiple_of(unsigned number, unsigned divisor);
-uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor);
-uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor);
-int64_t sample_laplace_distribution(double mu, double b, double p);
-int64_t add_laplace_noise(int64_t signal, double random, double delta_f,
-                          double epsilon);
-int n_bits_set_u8(uint8_t v);
-int64_t clamp_double_to_int64(double number);
-void simplify_fraction64(uint64_t *numer, uint64_t *denom);
-
-/* Compute the CEIL of <b>a</b> divided by <b>b</b>, for nonnegative <b>a</b>
- * and positive <b>b</b>.  Works on integer types only. Not defined if a+b can
- * overflow. */
-#define CEIL_DIV(a,b) (((a)+(b)-1)/(b))
-
-/* Return <b>v</b> if it's between <b>min</b> and <b>max</b>.  Otherwise
- * return <b>min</b> if <b>v</b> is smaller than <b>min</b>, or <b>max</b> if
- * <b>b</b> is larger than <b>max</b>.
- *
- * Requires that <b>min</b> is no more than <b>max</b>. May evaluate any of
- * its arguments more than once! */
-#define CLAMP(min,v,max)                        \
-  ( ((v) < (min)) ? (min) :                     \
-    ((v) > (max)) ? (max) :                     \
-    (v) )
-
-/* String manipulation */
-
-/** Allowable characters in a hexadecimal string. */
-#define HEX_CHARACTERS "0123456789ABCDEFabcdef"
-void tor_strlower(char *s) ATTR_NONNULL((1));
-void tor_strupper(char *s) ATTR_NONNULL((1));
-int tor_strisprint(const char *s) ATTR_NONNULL((1));
-int tor_strisnonupper(const char *s) ATTR_NONNULL((1));
-int strcmp_opt(const char *s1, const char *s2);
-int strcmpstart(const char *s1, const char *s2) ATTR_NONNULL((1,2));
-int strcmp_len(const char *s1, const char *s2, size_t len) ATTR_NONNULL((1,2));
-int strcasecmpstart(const char *s1, const char *s2) ATTR_NONNULL((1,2));
-int strcmpend(const char *s1, const char *s2) ATTR_NONNULL((1,2));
-int strcasecmpend(const char *s1, const char *s2) ATTR_NONNULL((1,2));
-int fast_memcmpstart(const void *mem, size_t memlen, const char *prefix);
-
-void tor_strstrip(char *s, const char *strip) ATTR_NONNULL((1,2));
-long tor_parse_long(const char *s, int base, long min,
-                    long max, int *ok, char **next);
-unsigned long tor_parse_ulong(const char *s, int base, unsigned long min,
-                              unsigned long max, int *ok, char **next);
-double tor_parse_double(const char *s, double min, double max, int *ok,
-                        char **next);
-uint64_t tor_parse_uint64(const char *s, int base, uint64_t min,
-                         uint64_t max, int *ok, char **next);
-const char *hex_str(const char *from, size_t fromlen) ATTR_NONNULL((1));
-const char *eat_whitespace(const char *s);
-const char *eat_whitespace_eos(const char *s, const char *eos);
-const char *eat_whitespace_no_nl(const char *s);
-const char *eat_whitespace_eos_no_nl(const char *s, const char *eos);
-const char *find_whitespace(const char *s);
-const char *find_whitespace_eos(const char *s, const char *eos);
-const char *find_str_at_start_of_line(const char *haystack,
-                                      const char *needle);
-int string_is_C_identifier(const char *string);
-int string_is_key_value(int severity, const char *string);
-int string_is_valid_hostname(const char *string);
-int string_is_valid_ipv4_address(const char *string);
-int string_is_valid_ipv6_address(const char *string);
-
-int tor_mem_is_zero(const char *mem, size_t len);
-int tor_digest_is_zero(const char *digest);
-int tor_digest256_is_zero(const char *digest);
-char *esc_for_log(const char *string) ATTR_MALLOC;
-char *esc_for_log_len(const char *chars, size_t n) ATTR_MALLOC;
-const char *escaped(const char *string);
-
-char *tor_escape_str_for_pt_args(const char *string,
-                                 const char *chars_to_escape);
-
-struct smartlist_t;
-int tor_vsscanf(const char *buf, const char *pattern, va_list ap) \
-  CHECK_SCANF(2, 0);
-int tor_sscanf(const char *buf, const char *pattern, ...)
-  CHECK_SCANF(2, 3);
-
-void smartlist_add_asprintf(struct smartlist_t *sl, const char *pattern, ...)
-  CHECK_PRINTF(2, 3);
-void smartlist_add_vasprintf(struct smartlist_t *sl, const char *pattern,
-                             va_list args)
-  CHECK_PRINTF(2, 0);
-
-/* Time helpers */
-long tv_udiff(const struct timeval *start, const struct timeval *end);
-long tv_mdiff(const struct timeval *start, const struct timeval *end);
-int64_t tv_to_msec(const struct timeval *tv);
-int tor_timegm(const struct tm *tm, time_t *time_out);
-#define RFC1123_TIME_LEN 29
-void format_rfc1123_time(char *buf, time_t t);
-int parse_rfc1123_time(const char *buf, time_t *t);
-#define ISO_TIME_LEN 19
-#define ISO_TIME_USEC_LEN (ISO_TIME_LEN+7)
-void format_local_iso_time(char *buf, time_t t);
-void format_iso_time(char *buf, time_t t);
-void format_iso_time_nospace(char *buf, time_t t);
-void format_iso_time_nospace_usec(char *buf, const struct timeval *tv);
-int parse_iso_time_(const char *cp, time_t *t, int strict);
-int parse_iso_time(const char *buf, time_t *t);
-int parse_http_time(const char *buf, struct tm *tm);
-int format_time_interval(char *out, size_t out_len, long interval);
-
-/* Cached time */
-#ifdef TIME_IS_FAST
-#define approx_time() time(NULL)
-#define update_approx_time(t) STMT_NIL
-#else
-time_t approx_time(void);
-void update_approx_time(time_t now);
-#endif
-
-/* Rate-limiter */
-
-/** A ratelim_t remembers how often an event is occurring, and how often
- * it's allowed to occur.  Typical usage is something like:
- *
-   <pre>
-    if (possibly_very_frequent_event()) {
-      const int INTERVAL = 300;
-      static ratelim_t warning_limit = RATELIM_INIT(INTERVAL);
-      char *m;
-      if ((m = rate_limit_log(&warning_limit, approx_time()))) {
-        log_warn(LD_GENERAL, "The event occurred!%s", m);
-        tor_free(m);
-      }
-    }
-   </pre>
-
-   As a convenience wrapper for logging, you can replace the above with:
-   <pre>
-   if (possibly_very_frequent_event()) {
-     static ratelim_t warning_limit = RATELIM_INIT(300);
-     log_fn_ratelim(&warning_limit, LOG_WARN, LD_GENERAL,
-                    "The event occurred!");
-   }
-   </pre>
- */
-typedef struct ratelim_t {
-  int rate;
-  time_t last_allowed;
-  int n_calls_since_last_time;
-} ratelim_t;
-
-#define RATELIM_INIT(r) { (r), 0, 0 }
-#define RATELIM_TOOMANY (16*1000*1000)
-
-char *rate_limit_log(ratelim_t *lim, time_t now);
-
-/* File helpers */
-ssize_t write_all(tor_socket_t fd, const char *buf, size_t count,int isSocket);
-ssize_t read_all(tor_socket_t fd, char *buf, size_t count, int isSocket);
-
-/** Status of an I/O stream. */
-enum stream_status {
-  IO_STREAM_OKAY,
-  IO_STREAM_EAGAIN,
-  IO_STREAM_TERM,
-  IO_STREAM_CLOSED
-};
-
-const char *stream_status_to_string(enum stream_status stream_status);
-
-enum stream_status get_string_from_pipe(FILE *stream, char *buf, size_t count);
-
-MOCK_DECL(int,tor_unlink,(const char *pathname));
-
-/** Return values from file_status(); see that function's documentation
- * for details. */
-typedef enum { FN_ERROR, FN_NOENT, FN_FILE, FN_DIR, FN_EMPTY } file_status_t;
-file_status_t file_status(const char *filename);
-
-/** Possible behaviors for check_private_dir() on encountering a nonexistent
- * directory; see that function's documentation for details. */
-typedef unsigned int cpd_check_t;
-#define CPD_NONE                 0
-#define CPD_CREATE               (1u << 0)
-#define CPD_CHECK                (1u << 1)
-#define CPD_GROUP_OK             (1u << 2)
-#define CPD_GROUP_READ           (1u << 3)
-#define CPD_CHECK_MODE_ONLY      (1u << 4)
-#define CPD_RELAX_DIRMODE_CHECK  (1u << 5)
-MOCK_DECL(int, check_private_dir,
-    (const char *dirname, cpd_check_t check,
-     const char *effective_user));
-
-#define OPEN_FLAGS_REPLACE (O_WRONLY|O_CREAT|O_TRUNC)
-#define OPEN_FLAGS_APPEND (O_WRONLY|O_CREAT|O_APPEND)
-#define OPEN_FLAGS_DONT_REPLACE (O_CREAT|O_EXCL|O_APPEND|O_WRONLY)
-typedef struct open_file_t open_file_t;
-int start_writing_to_file(const char *fname, int open_flags, int mode,
-                          open_file_t **data_out);
-FILE *start_writing_to_stdio_file(const char *fname, int open_flags, int mode,
-                                  open_file_t **data_out);
-FILE *fdopen_file(open_file_t *file_data);
-int finish_writing_to_file(open_file_t *file_data);
-int abort_writing_to_file(open_file_t *file_data);
-MOCK_DECL(int,
-write_str_to_file,(const char *fname, const char *str, int bin));
-MOCK_DECL(int,
-write_bytes_to_file,(const char *fname, const char *str, size_t len,
-                     int bin));
-/** An ad-hoc type to hold a string of characters and a count; used by
- * write_chunks_to_file. */
-typedef struct sized_chunk_t {
-  const char *bytes;
-  size_t len;
-} sized_chunk_t;
-int write_chunks_to_file(const char *fname, const struct smartlist_t *chunks,
-                         int bin, int no_tempfile);
-int append_bytes_to_file(const char *fname, const char *str, size_t len,
-                         int bin);
-int write_bytes_to_new_file(const char *fname, const char *str, size_t len,
-                            int bin);
-
-/** Flag for read_file_to_str: open the file in binary mode. */
-#define RFTS_BIN            1
-/** Flag for read_file_to_str: it's okay if the file doesn't exist. */
-#define RFTS_IGNORE_MISSING 2
-
-#ifndef _WIN32
-struct stat;
-#endif
-MOCK_DECL_ATTR(char *, read_file_to_str,
-               (const char *filename, int flags, struct stat *stat_out),
-               ATTR_MALLOC);
-char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read,
-                                 size_t *sz_out)
-  ATTR_MALLOC;
-const char *unescape_string(const char *s, char **result, size_t *size_out);
-const char *parse_config_line_from_str_verbose(const char *line,
-                                       char **key_out, char **value_out,
-                                       const char **err_out);
-char *expand_filename(const char *filename);
-MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname));
-int path_is_relative(const char *filename);
-
-/* Process helpers */
-void start_daemon(void);
-void finish_daemon(const char *desired_cwd);
-void write_pidfile(const char *filename);
-
-/* Port forwarding */
-void tor_check_port_forwarding(const char *filename,
-                               struct smartlist_t *ports_to_forward,
-                               time_t now);
-
-typedef struct process_handle_t process_handle_t;
-typedef struct process_environment_t process_environment_t;
-int tor_spawn_background(const char *const filename, const char **argv,
-                         process_environment_t *env,
-                         process_handle_t **process_handle_out);
-
-#define SPAWN_ERROR_MESSAGE "ERR: Failed to spawn background process - code "
-
-#ifdef _WIN32
-HANDLE load_windows_system_library(const TCHAR *library_name);
-#endif
-
-int environment_variable_names_equal(const char *s1, const char *s2);
-
-/* DOCDOC process_environment_t */
-struct process_environment_t {
-  /** A pointer to a sorted empty-string-terminated sequence of
-   * NUL-terminated strings of the form "NAME=VALUE". */
-  char *windows_environment_block;
-  /** A pointer to a NULL-terminated array of pointers to
-   * NUL-terminated strings of the form "NAME=VALUE". */
-  char **unixoid_environment_block;
-};
-
-process_environment_t *process_environment_make(struct smartlist_t *env_vars);
-void process_environment_free(process_environment_t *env);
-
-struct smartlist_t *get_current_process_environment_variables(void);
-
-void set_environment_variable_in_smartlist(struct smartlist_t *env_vars,
-                                           const char *new_var,
-                                           void (*free_old)(void*),
-                                           int free_p);
-
-/* Values of process_handle_t.status. PROCESS_STATUS_NOTRUNNING must be
- * 0 because tor_check_port_forwarding depends on this being the initial
- * statue of the static instance of process_handle_t */
-#define PROCESS_STATUS_NOTRUNNING 0
-#define PROCESS_STATUS_RUNNING 1
-#define PROCESS_STATUS_ERROR -1
-
-#ifdef UTIL_PRIVATE
-struct waitpid_callback_t;
-/** Structure to represent the state of a process with which Tor is
- * communicating. The contents of this structure are private to util.c */
-struct process_handle_t {
-  /** One of the PROCESS_STATUS_* values */
-  int status;
-#ifdef _WIN32
-  HANDLE stdin_pipe;
-  HANDLE stdout_pipe;
-  HANDLE stderr_pipe;
-  PROCESS_INFORMATION pid;
-#else
-  int stdin_pipe;
-  int stdout_pipe;
-  int stderr_pipe;
-  FILE *stdin_handle;
-  FILE *stdout_handle;
-  FILE *stderr_handle;
-  pid_t pid;
-  /** If the process has not given us a SIGCHLD yet, this has the
-   * waitpid_callback_t that gets invoked once it has. Otherwise this
-   * contains NULL. */
-  struct waitpid_callback_t *waitpid_cb;
-  /** The exit status reported by waitpid. */
-  int waitpid_exit_status;
-#endif // _WIN32
-};
-#endif
-
-/* Return values of tor_get_exit_code() */
-#define PROCESS_EXIT_RUNNING 1
-#define PROCESS_EXIT_EXITED 0
-#define PROCESS_EXIT_ERROR -1
-int tor_get_exit_code(process_handle_t *process_handle,
-                      int block, int *exit_code);
-int tor_split_lines(struct smartlist_t *sl, char *buf, int len);
-#ifdef _WIN32
-ssize_t tor_read_all_handle(HANDLE h, char *buf, size_t count,
-                            const process_handle_t *process);
-#else
-ssize_t tor_read_all_handle(FILE *h, char *buf, size_t count,
-                            const process_handle_t *process,
-                            int *eof);
-#endif
-ssize_t tor_read_all_from_process_stdout(
-    const process_handle_t *process_handle, char *buf, size_t count);
-ssize_t tor_read_all_from_process_stderr(
-    const process_handle_t *process_handle, char *buf, size_t count);
-char *tor_join_win_cmdline(const char *argv[]);
-
-int tor_process_get_pid(process_handle_t *process_handle);
-#ifdef _WIN32
-HANDLE tor_process_get_stdout_pipe(process_handle_t *process_handle);
-#else
-FILE *tor_process_get_stdout_pipe(process_handle_t *process_handle);
-#endif
-
-#ifdef _WIN32
-MOCK_DECL(struct smartlist_t *,
-tor_get_lines_from_handle,(HANDLE *handle,
-                           enum stream_status *stream_status));
-#else
-MOCK_DECL(struct smartlist_t *,
-tor_get_lines_from_handle,(FILE *handle,
-                           enum stream_status *stream_status));
-#endif
-
-int
-tor_terminate_process(process_handle_t *process_handle);
-
-MOCK_DECL(void,
-tor_process_handle_destroy,(process_handle_t *process_handle,
-                            int also_terminate_process));
-
-/* ===== Insecure rng */
-typedef struct tor_weak_rng_t {
-  uint32_t state;
-} tor_weak_rng_t;
-
-#define TOR_WEAK_RNG_INIT {383745623}
-#define TOR_WEAK_RANDOM_MAX (INT_MAX)
-void tor_init_weak_random(tor_weak_rng_t *weak_rng, unsigned seed);
-int32_t tor_weak_random(tor_weak_rng_t *weak_rng);
-int32_t tor_weak_random_range(tor_weak_rng_t *rng, int32_t top);
-/** Randomly return true according to <b>rng</b> with probability 1 in
- * <b>n</b> */
-#define tor_weak_random_one_in_n(rng, n) (0==tor_weak_random_range((rng),(n)))
-
-int format_hex_number_sigsafe(unsigned long x, char *buf, int max_len);
-int format_dec_number_sigsafe(unsigned long x, char *buf, int max_len);
-
-#ifdef UTIL_PRIVATE
-/* Prototypes for private functions only used by util.c (and unit tests) */
-
-#ifndef _WIN32
-STATIC int format_helper_exit_status(unsigned char child_state,
-                              int saved_errno, char *hex_errno);
-
-/* Space for hex values of child state, a slash, saved_errno (with
-   leading minus) and newline (no null) */
-#define HEX_ERRNO_SIZE (sizeof(char) * 2 + 1 + \
-                        1 + sizeof(int) * 2 + 1)
-#endif
-
-#endif
-
-#ifdef TOR_UNIT_TESTS
-int size_mul_check__(const size_t x, const size_t y);
-#endif
-
-#define ARRAY_LENGTH(x) ((sizeof(x)) / sizeof(x[0]))
-
-#endif
-
diff --git a/src/common/util_format.h b/src/common/util_format.h
deleted file mode 100644
index 20ac711d10..0000000000
--- a/src/common/util_format.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/* Copyright (c) 2001, Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#ifndef TOR_UTIL_FORMAT_H
-#define TOR_UTIL_FORMAT_H
-
-#include "testsupport.h"
-#include "torint.h"
-
-#define BASE64_ENCODE_MULTILINE 1
-size_t base64_encode_size(size_t srclen, int flags);
-int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen,
-                  int flags);
-int base64_decode(char *dest, size_t destlen, const char *src, size_t srclen);
-int base64_encode_nopad(char *dest, size_t destlen,
-                        const uint8_t *src, size_t srclen);
-int base64_decode_nopad(uint8_t *dest, size_t destlen,
-                        const char *src, size_t srclen);
-
-/** Characters that can appear (case-insensitively) in a base32 encoding. */
-#define BASE32_CHARS "abcdefghijklmnopqrstuvwxyz234567"
-void base32_encode(char *dest, size_t destlen, const char *src, size_t srclen);
-int base32_decode(char *dest, size_t destlen, const char *src, size_t srclen);
-size_t base32_encoded_size(size_t srclen);
-
-int hex_decode_digit(char c);
-void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen);
-int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen);
-
-#endif
-
diff --git a/src/config/torrc.minimal.in-staging b/src/config/torrc.minimal.in-staging
index c537c51f9b..cb3adca35c 100644
--- a/src/config/torrc.minimal.in-staging
+++ b/src/config/torrc.minimal.in-staging
@@ -1,5 +1,5 @@
 ## Configuration file for a typical Tor user
-## Last updated 22 September 2015 for Tor 0.2.7.3-alpha.
+## Last updated 22 December 2017 for Tor 0.3.2.8-rc.
 ## (may or may not work for much older or much newer versions of Tor.)
 ##
 ## Lines that begin with "## " try to explain what's going on. Lines
@@ -100,7 +100,7 @@
 ## A handle for your relay, so people don't have to refer to it by key.
 ## Nicknames must be between 1 and 19 characters inclusive, and must
 ## contain only the alphanumeric characters (a-z, A-Z, 0-9). No unicode,
-## no emoji.
+## no emoji. If not set, "Unnamed" will be used.
 #Nickname ididnteditheconfig
 
 ## Define these to limit how much relayed traffic you will allow. Your
@@ -132,6 +132,9 @@
 ## spammers might also collect them. You may want to obscure the fact that
 ## it's an email address and/or generate a new address for this purpose.
 ## Notice that "<" and ">" are recommended.
+##
+## If you are running multiple relays, you MUST set this option.
+##
 #ContactInfo Random Person <nobody AT example dot com>
 ## You might also include your PGP or GPG fingerprint if you have one.
 ## Use the full fingerprint, not just a (short) KeyID: KeyIDs are easy
@@ -161,8 +164,16 @@
 ## See https://www.torproject.org/docs/faq#MultipleRelays
 ## However, you should never include a bridge's fingerprint here, as it would
 ## break its concealability and potentially reveal its IP/TCP address.
+##
+## If you are running multiple relays, you MUST set this option.
+##
+## Note: do not use MyFamily on bridge relays.
 #MyFamily $keyid,$keyid,...
 
+## Uncomment this if you want your relay to allow IPv6 exit traffic.
+## (Relays only allow IPv4 exit traffic by default.)
+#IPv6Exit 1
+
 ## A comma-separated list of exit policies. They're considered first
 ## to last, and the first match wins.
 ##
@@ -202,6 +213,9 @@
 ## won't be able to block all the bridges. Also, websites won't treat you
 ## differently because they won't know you're running Tor. If you can
 ## be a real relay, please do; but if not, be a bridge!
+##
+## Warning: when running your Tor as a bridge, make sure than MyFamily is
+## NOT configured.
 #BridgeRelay 1
 ## By default, Tor will advertise your bridge to users through various
 ## mechanisms like https://bridges.torproject.org/. If you want to run
diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in
index 5328206da9..c2ae707e93 100644
--- a/src/config/torrc.sample.in
+++ b/src/config/torrc.sample.in
@@ -1,5 +1,5 @@
 ## Configuration file for a typical Tor user
-## Last updated 22 September 2015 for Tor 0.2.7.3-alpha.
+## Last updated 28 February 2019 for Tor 0.3.5.1-alpha.
 ## (may or may not work for much older or much newer versions of Tor.)
 ##
 ## Lines that begin with "## " try to explain what's going on. Lines
@@ -95,11 +95,18 @@
 
 ## If you have multiple network interfaces, you can specify one for
 ## outgoing traffic to use.
-# OutboundBindAddress 10.0.0.5
+## OutboundBindAddressExit will be used for all exit traffic, while
+## OutboundBindAddressOR will be used for all OR and Dir connections
+## (DNS connections ignore OutboundBindAddress).
+## If you do not wish to differentiate, use OutboundBindAddress to
+## specify the same address for both in a single line.
+#OutboundBindAddressExit 10.0.0.4
+#OutboundBindAddressOR 10.0.0.5
 
 ## A handle for your relay, so people don't have to refer to it by key.
 ## Nicknames must be between 1 and 19 characters inclusive, and must
 ## contain only the characters [a-zA-Z0-9].
+## If not set, "Unnamed" will be used.
 #Nickname ididnteditheconfig
 
 ## Define these to limit how much relayed traffic you will allow. Your
@@ -130,6 +137,9 @@
 ## descriptors containing these lines and that Google indexes them, so
 ## spammers might also collect them. You may want to obscure the fact that
 ## it's an email address and/or generate a new address for this purpose.
+##
+## If you are running multiple relays, you MUST set this option.
+##
 #ContactInfo Random Person <nobody AT example dot com>
 ## You might also include your PGP or GPG fingerprint if you have one:
 #ContactInfo 0xFFFFFFFF Random Person <nobody AT example dot com>
@@ -156,8 +166,31 @@
 ## https://www.torproject.org/docs/faq#MultipleRelays
 ## However, you should never include a bridge's fingerprint here, as it would
 ## break its concealability and potentially reveal its IP/TCP address.
+##
+## If you are running multiple relays, you MUST set this option.
+##
+## Note: do not use MyFamily on bridge relays.
 #MyFamily $keyid,$keyid,...
 
+## Uncomment this if you want your relay to be an exit, with the default
+## exit policy (or whatever exit policy you set below).
+## (If ReducedExitPolicy or ExitPolicy are set, relays are exits.
+## If neither exit policy option is set, relays are non-exits.)
+#ExitRelay 1
+
+## Uncomment this if you want your relay to allow IPv6 exit traffic.
+## You must also set ExitRelay, ReducedExitPolicy, or ExitPolicy to make your
+## relay into an exit.
+## (Relays do not allow any exit traffic by default.)
+#IPv6Exit 1
+
+## Uncomment this if you want your relay to be an exit, with a reduced set
+## of exit ports.
+#ReducedExitPolicy 1
+
+## Uncomment these lines if you want your relay to be an exit, with the
+## specified set of exit IPs and ports.
+##
 ## A comma-separated list of exit policies. They're considered first
 ## to last, and the first match wins.
 ##
@@ -197,6 +230,9 @@
 ## won't be able to block all the bridges. Also, websites won't treat you
 ## differently because they won't know you're running Tor. If you can
 ## be a real relay, please do; but if not, be a bridge!
+##
+## Warning: when running your Tor as a bridge, make sure than MyFamily is
+## NOT configured.
 #BridgeRelay 1
 ## By default, Tor will advertise your bridge to users through various
 ## mechanisms like https://bridges.torproject.org/. If you want to run
@@ -204,3 +240,12 @@
 ## address manually to your friends, uncomment this line:
 #PublishServerDescriptor 0
 
+## Configuration options can be imported from files or folders using the %include
+## option with the value being a path. If the path is a file, the options from the
+## file will be parsed as if they were written where the %include option is. If
+## the path is a folder, all files on that folder will be parsed following lexical
+## order. Files starting with a dot are ignored. Files on subfolders are ignored.
+## The %include option can be used recursively.
+#%include /etc/torrc.d/
+#%include /etc/torrc.custom
+
diff --git a/src/core/crypto/hs_ntor.c b/src/core/crypto/hs_ntor.c
new file mode 100644
index 0000000000..c34073690e
--- /dev/null
+++ b/src/core/crypto/hs_ntor.c
@@ -0,0 +1,620 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/** \file hs_ntor.c
+ *  \brief Implements the ntor variant used in Tor hidden services.
+ *
+ *  \details
+ *  This module handles the variant of the ntor handshake that is documented in
+ *  section [NTOR-WITH-EXTRA-DATA] of rend-spec-ng.txt .
+ *
+ *  The functions in this file provide an API that should be used when sending
+ *  or receiving INTRODUCE1/RENDEZVOUS1 cells to generate the various key
+ *  material required to create and handle those cells.
+ *
+ *  In the case of INTRODUCE1 it provides encryption and MAC keys to
+ *  encode/decode the encrypted blob (see hs_ntor_intro_cell_keys_t). The
+ *  relevant pub functions are hs_ntor_{client,service}_get_introduce1_keys().
+ *
+ *  In the case of RENDEZVOUS1 it calculates the MAC required to authenticate
+ *  the cell, and also provides the key seed that is used to derive the crypto
+ *  material for rendezvous encryption (see hs_ntor_rend_cell_keys_t). The
+ *  relevant pub functions are hs_ntor_{client,service}_get_rendezvous1_keys().
+ *  It also provides a function (hs_ntor_circuit_key_expansion()) that does the
+ *  rendezvous key expansion to setup end-to-end rend circuit keys.
+ */
+
+#include "core/or/or.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "core/crypto/hs_ntor.h"
+
+/* String constants used by the ntor HS protocol */
+#define PROTOID "tor-hs-ntor-curve25519-sha3-256-1"
+#define PROTOID_LEN (sizeof(PROTOID) - 1)
+#define SERVER_STR "Server"
+#define SERVER_STR_LEN (sizeof(SERVER_STR) - 1)
+
+/* Protocol-specific tweaks to our crypto inputs */
+#define T_HSENC PROTOID ":hs_key_extract"
+#define T_HSENC_LEN (sizeof(T_HSENC) - 1)
+#define T_HSVERIFY PROTOID ":hs_verify"
+#define T_HSMAC PROTOID ":hs_mac"
+#define M_HSEXPAND PROTOID ":hs_key_expand"
+#define M_HSEXPAND_LEN (sizeof(M_HSEXPAND) - 1)
+
+/************************* Helper functions: *******************************/
+
+/** Helper macro: copy <b>len</b> bytes from <b>inp</b> to <b>ptr</b> and
+ *advance <b>ptr</b> by the number of bytes copied. Stolen from onion_ntor.c */
+#define APPEND(ptr, inp, len)                   \
+  STMT_BEGIN {                                  \
+    memcpy(ptr, (inp), (len));                  \
+    ptr += len;                                 \
+  } STMT_END
+
+/* Length of EXP(X,y) | EXP(X,b) | AUTH_KEY | B | X | Y | PROTOID */
+#define REND_SECRET_HS_INPUT_LEN (CURVE25519_OUTPUT_LEN * 2 + \
+  ED25519_PUBKEY_LEN + CURVE25519_PUBKEY_LEN * 3 + PROTOID_LEN)
+/* Length of auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server" */
+#define REND_AUTH_INPUT_LEN (DIGEST256_LEN + ED25519_PUBKEY_LEN + \
+  CURVE25519_PUBKEY_LEN * 3 + PROTOID_LEN + SERVER_STR_LEN)
+
+/** Helper function: Compute the last part of the HS ntor handshake which
+ *  derives key material necessary to create and handle RENDEZVOUS1
+ *  cells. Function used by both client and service. The actual calculations is
+ *  as follows:
+ *
+ *    NTOR_KEY_SEED = MAC(rend_secret_hs_input, t_hsenc)
+ *    verify = MAC(rend_secret_hs_input, t_hsverify)
+ *    auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server"
+ *    auth_input_mac = MAC(auth_input, t_hsmac)
+ *
+ *  where in the above, AUTH_KEY is <b>intro_auth_pubkey</b>, B is
+ *  <b>intro_enc_pubkey</b>, Y is <b>service_ephemeral_rend_pubkey</b>, and X
+ *  is <b>client_ephemeral_enc_pubkey</b>. The provided
+ *  <b>rend_secret_hs_input</b> is of size REND_SECRET_HS_INPUT_LEN.
+ *
+ *  The final results of NTOR_KEY_SEED and auth_input_mac are placed in
+ *  <b>hs_ntor_rend_cell_keys_out</b>. Return 0 if everything went fine. */
+static int
+get_rendezvous1_key_material(const uint8_t *rend_secret_hs_input,
+                  const ed25519_public_key_t *intro_auth_pubkey,
+                  const curve25519_public_key_t *intro_enc_pubkey,
+                  const curve25519_public_key_t *service_ephemeral_rend_pubkey,
+                  const curve25519_public_key_t *client_ephemeral_enc_pubkey,
+                  hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out)
+{
+  int bad = 0;
+  uint8_t ntor_key_seed[DIGEST256_LEN];
+  uint8_t ntor_verify[DIGEST256_LEN];
+  uint8_t rend_auth_input[REND_AUTH_INPUT_LEN];
+  uint8_t rend_cell_auth[DIGEST256_LEN];
+  uint8_t *ptr;
+
+  /* Let's build NTOR_KEY_SEED */
+  crypto_mac_sha3_256(ntor_key_seed, sizeof(ntor_key_seed),
+                      rend_secret_hs_input, REND_SECRET_HS_INPUT_LEN,
+                      (const uint8_t *)T_HSENC, strlen(T_HSENC));
+  bad |= safe_mem_is_zero(ntor_key_seed, DIGEST256_LEN);
+
+  /* Let's build ntor_verify */
+  crypto_mac_sha3_256(ntor_verify, sizeof(ntor_verify),
+                      rend_secret_hs_input, REND_SECRET_HS_INPUT_LEN,
+                      (const uint8_t *)T_HSVERIFY, strlen(T_HSVERIFY));
+  bad |= safe_mem_is_zero(ntor_verify, DIGEST256_LEN);
+
+  /* Let's build auth_input: */
+  ptr = rend_auth_input;
+  /* Append ntor_verify */
+  APPEND(ptr, ntor_verify, sizeof(ntor_verify));
+  /* Append AUTH_KEY */
+  APPEND(ptr, intro_auth_pubkey->pubkey, ED25519_PUBKEY_LEN);
+  /* Append B */
+  APPEND(ptr, intro_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append Y */
+  APPEND(ptr,
+         service_ephemeral_rend_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append X */
+  APPEND(ptr,
+         client_ephemeral_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append PROTOID */
+  APPEND(ptr, PROTOID, strlen(PROTOID));
+  /* Append "Server" */
+  APPEND(ptr, SERVER_STR, strlen(SERVER_STR));
+  tor_assert(ptr == rend_auth_input + sizeof(rend_auth_input));
+
+  /* Let's build auth_input_mac that goes in RENDEZVOUS1 cell */
+  crypto_mac_sha3_256(rend_cell_auth, sizeof(rend_cell_auth),
+                      rend_auth_input, sizeof(rend_auth_input),
+                      (const uint8_t *)T_HSMAC, strlen(T_HSMAC));
+  bad |= safe_mem_is_zero(ntor_verify, DIGEST256_LEN);
+
+  { /* Get the computed RENDEZVOUS1 material! */
+    memcpy(&hs_ntor_rend_cell_keys_out->rend_cell_auth_mac,
+           rend_cell_auth, DIGEST256_LEN);
+    memcpy(&hs_ntor_rend_cell_keys_out->ntor_key_seed,
+           ntor_key_seed, DIGEST256_LEN);
+  }
+
+  memwipe(rend_cell_auth, 0, sizeof(rend_cell_auth));
+  memwipe(rend_auth_input, 0, sizeof(rend_auth_input));
+  memwipe(ntor_key_seed, 0, sizeof(ntor_key_seed));
+
+  return bad;
+}
+
+/** Length of secret_input = EXP(B,x) | AUTH_KEY | X | B | PROTOID */
+#define INTRO_SECRET_HS_INPUT_LEN (CURVE25519_OUTPUT_LEN +ED25519_PUBKEY_LEN +\
+  CURVE25519_PUBKEY_LEN + CURVE25519_PUBKEY_LEN + PROTOID_LEN)
+/* Length of info = m_hsexpand | subcredential */
+#define INFO_BLOB_LEN (M_HSEXPAND_LEN + DIGEST256_LEN)
+/* Length of KDF input = intro_secret_hs_input | t_hsenc | info */
+#define KDF_INPUT_LEN (INTRO_SECRET_HS_INPUT_LEN + T_HSENC_LEN + INFO_BLOB_LEN)
+
+/** Helper function: Compute the part of the HS ntor handshake that generates
+ *  key material for creating and handling INTRODUCE1 cells. Function used
+ *  by both client and service. Specifically, calculate the following:
+ *
+ *     info = m_hsexpand | subcredential
+ *     hs_keys = KDF(intro_secret_hs_input | t_hsenc | info, S_KEY_LEN+MAC_LEN)
+ *     ENC_KEY = hs_keys[0:S_KEY_LEN]
+ *     MAC_KEY = hs_keys[S_KEY_LEN:S_KEY_LEN+MAC_KEY_LEN]
+ *
+ *  where intro_secret_hs_input is <b>secret_input</b> (of size
+ *  INTRO_SECRET_HS_INPUT_LEN), and <b>subcredential</b> is of size
+ *  DIGEST256_LEN.
+ *
+ * If everything went well, fill <b>hs_ntor_intro_cell_keys_out</b> with the
+ * necessary key material, and return 0. */
+static void
+get_introduce1_key_material(const uint8_t *secret_input,
+                        const uint8_t *subcredential,
+                        hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out)
+{
+  uint8_t keystream[CIPHER256_KEY_LEN + DIGEST256_LEN];
+  uint8_t info_blob[INFO_BLOB_LEN];
+  uint8_t kdf_input[KDF_INPUT_LEN];
+  crypto_xof_t *xof;
+  uint8_t *ptr;
+
+  /* Let's build info */
+  ptr = info_blob;
+  APPEND(ptr, M_HSEXPAND, strlen(M_HSEXPAND));
+  APPEND(ptr, subcredential, DIGEST256_LEN);
+  tor_assert(ptr == info_blob + sizeof(info_blob));
+
+  /* Let's build the input to the KDF */
+  ptr = kdf_input;
+  APPEND(ptr, secret_input, INTRO_SECRET_HS_INPUT_LEN);
+  APPEND(ptr, T_HSENC, strlen(T_HSENC));
+  APPEND(ptr, info_blob, sizeof(info_blob));
+  tor_assert(ptr == kdf_input + sizeof(kdf_input));
+
+  /* Now we need to run kdf_input over SHAKE-256 */
+  xof = crypto_xof_new();
+  crypto_xof_add_bytes(xof, kdf_input, sizeof(kdf_input));
+  crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream)) ;
+  crypto_xof_free(xof);
+
+  { /* Get the keys */
+    memcpy(&hs_ntor_intro_cell_keys_out->enc_key, keystream,CIPHER256_KEY_LEN);
+    memcpy(&hs_ntor_intro_cell_keys_out->mac_key,
+           keystream+CIPHER256_KEY_LEN, DIGEST256_LEN);
+  }
+
+  memwipe(keystream,  0, sizeof(keystream));
+  memwipe(kdf_input,  0, sizeof(kdf_input));
+}
+
+/** Helper function: Calculate the 'intro_secret_hs_input' element used by the
+ * HS ntor handshake and place it in <b>secret_input_out</b>. This function is
+ * used by both client and service code.
+ *
+ * For the client-side it looks like this:
+ *
+ *         intro_secret_hs_input = EXP(B,x) | AUTH_KEY | X | B | PROTOID
+ *
+ * whereas for the service-side it looks like this:
+ *
+ *         intro_secret_hs_input = EXP(X,b) | AUTH_KEY | X | B | PROTOID
+ *
+ * In this function, <b>dh_result</b> carries the EXP() result (and has size
+ * CURVE25519_OUTPUT_LEN) <b>intro_auth_pubkey</b> is AUTH_KEY,
+ * <b>client_ephemeral_enc_pubkey</b> is X, and <b>intro_enc_pubkey</b> is B.
+ */
+static void
+get_intro_secret_hs_input(const uint8_t *dh_result,
+                    const ed25519_public_key_t *intro_auth_pubkey,
+                    const curve25519_public_key_t *client_ephemeral_enc_pubkey,
+                    const curve25519_public_key_t *intro_enc_pubkey,
+                    uint8_t *secret_input_out)
+{
+  uint8_t *ptr;
+
+  /* Append EXP() */
+  ptr = secret_input_out;
+  APPEND(ptr, dh_result, CURVE25519_OUTPUT_LEN);
+  /* Append AUTH_KEY */
+  APPEND(ptr, intro_auth_pubkey->pubkey, ED25519_PUBKEY_LEN);
+  /* Append X */
+  APPEND(ptr, client_ephemeral_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append B */
+  APPEND(ptr, intro_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append PROTOID */
+  APPEND(ptr, PROTOID, strlen(PROTOID));
+  tor_assert(ptr == secret_input_out + INTRO_SECRET_HS_INPUT_LEN);
+}
+
+/** Calculate the 'rend_secret_hs_input' element used by the HS ntor handshake
+ *  and place it in <b>rend_secret_hs_input_out</b>. This function is used by
+ *  both client and service code.
+ *
+ * The computation on the client side is:
+ *  rend_secret_hs_input = EXP(X,y) | EXP(X,b) | AUTH_KEY | B | X | Y | PROTOID
+ * whereas on the service side it is:
+ *  rend_secret_hs_input = EXP(Y,x) | EXP(B,x) | AUTH_KEY | B | X | Y | PROTOID
+ *
+ * where:
+ * <b>dh_result1</b> and <b>dh_result2</b> carry the two EXP() results (of size
+ * CURVE25519_OUTPUT_LEN)
+ * <b>intro_auth_pubkey</b> is AUTH_KEY,
+ * <b>intro_enc_pubkey</b> is B,
+ * <b>client_ephemeral_enc_pubkey</b> is X, and
+ * <b>service_ephemeral_rend_pubkey</b> is Y.
+ */
+static void
+get_rend_secret_hs_input(const uint8_t *dh_result1, const uint8_t *dh_result2,
+                  const ed25519_public_key_t *intro_auth_pubkey,
+                  const curve25519_public_key_t *intro_enc_pubkey,
+                  const curve25519_public_key_t *client_ephemeral_enc_pubkey,
+                  const curve25519_public_key_t *service_ephemeral_rend_pubkey,
+                  uint8_t *rend_secret_hs_input_out)
+{
+  uint8_t *ptr;
+
+  ptr = rend_secret_hs_input_out;
+  /* Append the first EXP() */
+  APPEND(ptr, dh_result1, CURVE25519_OUTPUT_LEN);
+  /* Append the other EXP() */
+  APPEND(ptr, dh_result2, CURVE25519_OUTPUT_LEN);
+  /* Append AUTH_KEY */
+  APPEND(ptr, intro_auth_pubkey->pubkey, ED25519_PUBKEY_LEN);
+  /* Append B */
+  APPEND(ptr, intro_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append X */
+  APPEND(ptr,
+         client_ephemeral_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append Y */
+  APPEND(ptr,
+         service_ephemeral_rend_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append PROTOID */
+  APPEND(ptr, PROTOID, strlen(PROTOID));
+  tor_assert(ptr == rend_secret_hs_input_out + REND_SECRET_HS_INPUT_LEN);
+}
+
+/************************* Public functions: *******************************/
+
+/* Public function: Do the appropriate ntor calculations and derive the keys
+ * needed to encrypt and authenticate INTRODUCE1 cells. Return 0 and place the
+ * final key material in <b>hs_ntor_intro_cell_keys_out</b> if everything went
+ * well, otherwise return -1;
+ *
+ * The relevant calculations are as follows:
+ *
+ *     intro_secret_hs_input = EXP(B,x) | AUTH_KEY | X | B | PROTOID
+ *     info = m_hsexpand | subcredential
+ *     hs_keys = KDF(intro_secret_hs_input | t_hsenc | info, S_KEY_LEN+MAC_LEN)
+ *     ENC_KEY = hs_keys[0:S_KEY_LEN]
+ *     MAC_KEY = hs_keys[S_KEY_LEN:S_KEY_LEN+MAC_KEY_LEN]
+ *
+ * where:
+ * <b>intro_auth_pubkey</b> is AUTH_KEY (found in HS descriptor),
+ * <b>intro_enc_pubkey</b> is B (also found in HS descriptor),
+ * <b>client_ephemeral_enc_keypair</b> is freshly generated keypair (x,X)
+ * <b>subcredential</b> is the hidden service subcredential (of size
+ * DIGEST256_LEN). */
+int
+hs_ntor_client_get_introduce1_keys(
+                      const ed25519_public_key_t *intro_auth_pubkey,
+                      const curve25519_public_key_t *intro_enc_pubkey,
+                      const curve25519_keypair_t *client_ephemeral_enc_keypair,
+                      const uint8_t *subcredential,
+                      hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out)
+{
+  int bad = 0;
+  uint8_t secret_input[INTRO_SECRET_HS_INPUT_LEN];
+  uint8_t dh_result[CURVE25519_OUTPUT_LEN];
+
+  tor_assert(intro_auth_pubkey);
+  tor_assert(intro_enc_pubkey);
+  tor_assert(client_ephemeral_enc_keypair);
+  tor_assert(subcredential);
+  tor_assert(hs_ntor_intro_cell_keys_out);
+
+  /* Calculate EXP(B,x) */
+  curve25519_handshake(dh_result,
+                       &client_ephemeral_enc_keypair->seckey,
+                       intro_enc_pubkey);
+  bad |= safe_mem_is_zero(dh_result, CURVE25519_OUTPUT_LEN);
+
+  /* Get intro_secret_hs_input */
+  get_intro_secret_hs_input(dh_result, intro_auth_pubkey,
+                            &client_ephemeral_enc_keypair->pubkey,
+                            intro_enc_pubkey, secret_input);
+  bad |= safe_mem_is_zero(secret_input, CURVE25519_OUTPUT_LEN);
+
+  /* Get ENC_KEY and MAC_KEY! */
+  get_introduce1_key_material(secret_input, subcredential,
+                              hs_ntor_intro_cell_keys_out);
+
+  /* Cleanup */
+  memwipe(secret_input,  0, sizeof(secret_input));
+  if (bad) {
+    memwipe(hs_ntor_intro_cell_keys_out, 0, sizeof(hs_ntor_intro_cell_keys_t));
+  }
+
+  return bad ? -1 : 0;
+}
+
+/* Public function: Do the appropriate ntor calculations and derive the keys
+ * needed to verify RENDEZVOUS1 cells and encrypt further rendezvous
+ * traffic. Return 0 and place the final key material in
+ * <b>hs_ntor_rend_cell_keys_out</b> if everything went well, else return -1.
+ *
+ * The relevant calculations are as follows:
+ *
+ *  rend_secret_hs_input = EXP(Y,x) | EXP(B,x) | AUTH_KEY | B | X | Y | PROTOID
+ *  NTOR_KEY_SEED = MAC(rend_secret_hs_input, t_hsenc)
+ *  verify = MAC(rend_secret_hs_input, t_hsverify)
+ *  auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server"
+ *  auth_input_mac = MAC(auth_input, t_hsmac)
+ *
+ * where:
+ * <b>intro_auth_pubkey</b> is AUTH_KEY (found in HS descriptor),
+ * <b>client_ephemeral_enc_keypair</b> is freshly generated keypair (x,X)
+ * <b>intro_enc_pubkey</b> is B (also found in HS descriptor),
+ * <b>service_ephemeral_rend_pubkey</b> is Y (SERVER_PK in RENDEZVOUS1 cell) */
+int
+hs_ntor_client_get_rendezvous1_keys(
+                  const ed25519_public_key_t *intro_auth_pubkey,
+                  const curve25519_keypair_t *client_ephemeral_enc_keypair,
+                  const curve25519_public_key_t *intro_enc_pubkey,
+                  const curve25519_public_key_t *service_ephemeral_rend_pubkey,
+                  hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out)
+{
+  int bad = 0;
+  uint8_t rend_secret_hs_input[REND_SECRET_HS_INPUT_LEN];
+  uint8_t dh_result1[CURVE25519_OUTPUT_LEN];
+  uint8_t dh_result2[CURVE25519_OUTPUT_LEN];
+
+  tor_assert(intro_auth_pubkey);
+  tor_assert(client_ephemeral_enc_keypair);
+  tor_assert(intro_enc_pubkey);
+  tor_assert(service_ephemeral_rend_pubkey);
+  tor_assert(hs_ntor_rend_cell_keys_out);
+
+  /* Compute EXP(Y, x) */
+  curve25519_handshake(dh_result1,
+                       &client_ephemeral_enc_keypair->seckey,
+                       service_ephemeral_rend_pubkey);
+  bad |= safe_mem_is_zero(dh_result1, CURVE25519_OUTPUT_LEN);
+
+  /* Compute EXP(B, x) */
+  curve25519_handshake(dh_result2,
+                       &client_ephemeral_enc_keypair->seckey,
+                       intro_enc_pubkey);
+  bad |= safe_mem_is_zero(dh_result2, CURVE25519_OUTPUT_LEN);
+
+  /* Get rend_secret_hs_input */
+  get_rend_secret_hs_input(dh_result1, dh_result2,
+                           intro_auth_pubkey, intro_enc_pubkey,
+                           &client_ephemeral_enc_keypair->pubkey,
+                           service_ephemeral_rend_pubkey,
+                           rend_secret_hs_input);
+
+  /* Get NTOR_KEY_SEED and the auth_input MAC */
+  bad |= get_rendezvous1_key_material(rend_secret_hs_input,
+                                      intro_auth_pubkey,
+                                      intro_enc_pubkey,
+                                      service_ephemeral_rend_pubkey,
+                                      &client_ephemeral_enc_keypair->pubkey,
+                                      hs_ntor_rend_cell_keys_out);
+
+  memwipe(rend_secret_hs_input, 0, sizeof(rend_secret_hs_input));
+  if (bad) {
+    memwipe(hs_ntor_rend_cell_keys_out, 0, sizeof(hs_ntor_rend_cell_keys_t));
+  }
+
+  return bad ? -1 : 0;
+}
+
+/* Public function: Do the appropriate ntor calculations and derive the keys
+ * needed to decrypt and verify INTRODUCE1 cells. Return 0 and place the final
+ * key material in <b>hs_ntor_intro_cell_keys_out</b> if everything went well,
+ * otherwise return -1;
+ *
+ * The relevant calculations are as follows:
+ *
+ *    intro_secret_hs_input = EXP(X,b) | AUTH_KEY | X | B | PROTOID
+ *    info = m_hsexpand | subcredential
+ *    hs_keys = KDF(intro_secret_hs_input | t_hsenc | info, S_KEY_LEN+MAC_LEN)
+ *    HS_DEC_KEY = hs_keys[0:S_KEY_LEN]
+ *    HS_MAC_KEY = hs_keys[S_KEY_LEN:S_KEY_LEN+MAC_KEY_LEN]
+ *
+ * where:
+ * <b>intro_auth_pubkey</b> is AUTH_KEY (introduction point auth key),
+ * <b>intro_enc_keypair</b> is (b,B) (introduction point encryption keypair),
+ * <b>client_ephemeral_enc_pubkey</b> is X (CLIENT_PK in INTRODUCE2 cell),
+ * <b>subcredential</b> is the HS subcredential (of size DIGEST256_LEN) */
+int
+hs_ntor_service_get_introduce1_keys(
+                    const ed25519_public_key_t *intro_auth_pubkey,
+                    const curve25519_keypair_t *intro_enc_keypair,
+                    const curve25519_public_key_t *client_ephemeral_enc_pubkey,
+                    const uint8_t *subcredential,
+                    hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out)
+{
+  int bad = 0;
+  uint8_t secret_input[INTRO_SECRET_HS_INPUT_LEN];
+  uint8_t dh_result[CURVE25519_OUTPUT_LEN];
+
+  tor_assert(intro_auth_pubkey);
+  tor_assert(intro_enc_keypair);
+  tor_assert(client_ephemeral_enc_pubkey);
+  tor_assert(subcredential);
+  tor_assert(hs_ntor_intro_cell_keys_out);
+
+  /* Compute EXP(X, b) */
+  curve25519_handshake(dh_result,
+                       &intro_enc_keypair->seckey,
+                       client_ephemeral_enc_pubkey);
+  bad |= safe_mem_is_zero(dh_result, CURVE25519_OUTPUT_LEN);
+
+  /* Get intro_secret_hs_input */
+  get_intro_secret_hs_input(dh_result, intro_auth_pubkey,
+                            client_ephemeral_enc_pubkey,
+                            &intro_enc_keypair->pubkey,
+                            secret_input);
+  bad |= safe_mem_is_zero(secret_input, CURVE25519_OUTPUT_LEN);
+
+  /* Get ENC_KEY and MAC_KEY! */
+  get_introduce1_key_material(secret_input, subcredential,
+                              hs_ntor_intro_cell_keys_out);
+
+  memwipe(secret_input,  0, sizeof(secret_input));
+  if (bad) {
+    memwipe(hs_ntor_intro_cell_keys_out, 0, sizeof(hs_ntor_intro_cell_keys_t));
+  }
+
+  return bad ? -1 : 0;
+}
+
+/* Public function: Do the appropriate ntor calculations and derive the keys
+ * needed to create and authenticate RENDEZVOUS1 cells. Return 0 and place the
+ * final key material in <b>hs_ntor_rend_cell_keys_out</b> if all went fine,
+ * return -1 if error happened.
+ *
+ * The relevant calculations are as follows:
+ *
+ *  rend_secret_hs_input = EXP(X,y) | EXP(X,b) | AUTH_KEY | B | X | Y | PROTOID
+ *  NTOR_KEY_SEED = MAC(rend_secret_hs_input, t_hsenc)
+ *  verify = MAC(rend_secret_hs_input, t_hsverify)
+ *  auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server"
+ *  auth_input_mac = MAC(auth_input, t_hsmac)
+ *
+ * where:
+ * <b>intro_auth_pubkey</b> is AUTH_KEY (intro point auth key),
+ * <b>intro_enc_keypair</b> is (b,B) (intro point enc keypair)
+ * <b>service_ephemeral_rend_keypair</b> is a fresh (y,Y) keypair
+ * <b>client_ephemeral_enc_pubkey</b> is X (CLIENT_PK in INTRODUCE2 cell) */
+int
+hs_ntor_service_get_rendezvous1_keys(
+                    const ed25519_public_key_t *intro_auth_pubkey,
+                    const curve25519_keypair_t *intro_enc_keypair,
+                    const curve25519_keypair_t *service_ephemeral_rend_keypair,
+                    const curve25519_public_key_t *client_ephemeral_enc_pubkey,
+                    hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out)
+{
+  int bad = 0;
+  uint8_t rend_secret_hs_input[REND_SECRET_HS_INPUT_LEN];
+  uint8_t dh_result1[CURVE25519_OUTPUT_LEN];
+  uint8_t dh_result2[CURVE25519_OUTPUT_LEN];
+
+  tor_assert(intro_auth_pubkey);
+  tor_assert(intro_enc_keypair);
+  tor_assert(service_ephemeral_rend_keypair);
+  tor_assert(client_ephemeral_enc_pubkey);
+  tor_assert(hs_ntor_rend_cell_keys_out);
+
+  /* Compute EXP(X, y) */
+  curve25519_handshake(dh_result1,
+                       &service_ephemeral_rend_keypair->seckey,
+                       client_ephemeral_enc_pubkey);
+  bad |= safe_mem_is_zero(dh_result1, CURVE25519_OUTPUT_LEN);
+
+  /* Compute EXP(X, b) */
+  curve25519_handshake(dh_result2,
+                       &intro_enc_keypair->seckey,
+                       client_ephemeral_enc_pubkey);
+  bad |= safe_mem_is_zero(dh_result2, CURVE25519_OUTPUT_LEN);
+
+  /* Get rend_secret_hs_input */
+  get_rend_secret_hs_input(dh_result1, dh_result2,
+                           intro_auth_pubkey,
+                           &intro_enc_keypair->pubkey,
+                           client_ephemeral_enc_pubkey,
+                           &service_ephemeral_rend_keypair->pubkey,
+                           rend_secret_hs_input);
+
+  /* Get NTOR_KEY_SEED and AUTH_INPUT_MAC! */
+  bad |= get_rendezvous1_key_material(rend_secret_hs_input,
+                                      intro_auth_pubkey,
+                                      &intro_enc_keypair->pubkey,
+                                      &service_ephemeral_rend_keypair->pubkey,
+                                      client_ephemeral_enc_pubkey,
+                                      hs_ntor_rend_cell_keys_out);
+
+  memwipe(rend_secret_hs_input, 0, sizeof(rend_secret_hs_input));
+  if (bad) {
+    memwipe(hs_ntor_rend_cell_keys_out, 0, sizeof(hs_ntor_rend_cell_keys_t));
+  }
+
+  return bad ? -1 : 0;
+}
+
+/** Given a received RENDEZVOUS2 MAC in <b>mac</b> (of length DIGEST256_LEN),
+ *  and the RENDEZVOUS1 key material in <b>hs_ntor_rend_cell_keys</b>, return 1
+ *  if the MAC is good, otherwise return 0. */
+int
+hs_ntor_client_rendezvous2_mac_is_good(
+                        const hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys,
+                        const uint8_t *rcvd_mac)
+{
+  tor_assert(rcvd_mac);
+  tor_assert(hs_ntor_rend_cell_keys);
+
+  return tor_memeq(hs_ntor_rend_cell_keys->rend_cell_auth_mac,
+                   rcvd_mac, DIGEST256_LEN);
+}
+
+/* Input length to KDF for key expansion */
+#define NTOR_KEY_EXPANSION_KDF_INPUT_LEN (DIGEST256_LEN + M_HSEXPAND_LEN)
+
+/** Given the rendezvous key seed in <b>ntor_key_seed</b> (of size
+ *  DIGEST256_LEN), do the circuit key expansion as specified by section
+ *  '4.2.1. Key expansion' and place the keys in <b>keys_out</b> (which must be
+ *  of size HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN).
+ *
+ * Return 0 if things went well, else return -1. */
+int
+hs_ntor_circuit_key_expansion(const uint8_t *ntor_key_seed, size_t seed_len,
+                              uint8_t *keys_out, size_t keys_out_len)
+{
+  uint8_t *ptr;
+  uint8_t kdf_input[NTOR_KEY_EXPANSION_KDF_INPUT_LEN];
+  crypto_xof_t *xof;
+
+  /* Sanity checks on lengths to make sure we are good */
+  if (BUG(seed_len != DIGEST256_LEN)) {
+    return -1;
+  }
+  if (BUG(keys_out_len != HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN)) {
+    return -1;
+  }
+
+  /* Let's build the input to the KDF */
+  ptr = kdf_input;
+  APPEND(ptr, ntor_key_seed, DIGEST256_LEN);
+  APPEND(ptr, M_HSEXPAND, strlen(M_HSEXPAND));
+  tor_assert(ptr == kdf_input + sizeof(kdf_input));
+
+  /* Generate the keys */
+  xof = crypto_xof_new();
+  crypto_xof_add_bytes(xof, kdf_input, sizeof(kdf_input));
+  crypto_xof_squeeze_bytes(xof, keys_out, HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN);
+  crypto_xof_free(xof);
+
+  return 0;
+}
diff --git a/src/core/crypto/hs_ntor.h b/src/core/crypto/hs_ntor.h
new file mode 100644
index 0000000000..e5a5171915
--- /dev/null
+++ b/src/core/crypto/hs_ntor.h
@@ -0,0 +1,69 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_HS_NTOR_H
+#define TOR_HS_NTOR_H
+
+#include "core/or/or.h"
+struct ed25519_public_key_t;
+struct curve25519_public_key_t;
+struct curve25519_keypair_t;
+
+/* Output length of KDF for key expansion */
+#define HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN \
+  (DIGEST256_LEN*2 + CIPHER256_KEY_LEN*2)
+
+/* Key material needed to encode/decode INTRODUCE1 cells */
+typedef struct {
+  /* Key used for encryption of encrypted INTRODUCE1 blob */
+  uint8_t enc_key[CIPHER256_KEY_LEN];
+  /* MAC key used to protect encrypted INTRODUCE1 blob */
+  uint8_t mac_key[DIGEST256_LEN];
+} hs_ntor_intro_cell_keys_t;
+
+/* Key material needed to encode/decode RENDEZVOUS1 cells */
+typedef struct {
+  /* This is the MAC of the HANDSHAKE_INFO field */
+  uint8_t rend_cell_auth_mac[DIGEST256_LEN];
+  /* This is the key seed used to derive further rendezvous crypto keys as
+   * detailed in section 4.2.1 of rend-spec-ng.txt. */
+  uint8_t ntor_key_seed[DIGEST256_LEN];
+} hs_ntor_rend_cell_keys_t;
+
+int hs_ntor_client_get_introduce1_keys(
+              const struct ed25519_public_key_t *intro_auth_pubkey,
+              const struct curve25519_public_key_t *intro_enc_pubkey,
+              const struct curve25519_keypair_t *client_ephemeral_enc_keypair,
+              const uint8_t *subcredential,
+              hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out);
+
+int hs_ntor_client_get_rendezvous1_keys(
+          const struct ed25519_public_key_t *intro_auth_pubkey,
+          const struct curve25519_keypair_t *client_ephemeral_enc_keypair,
+          const struct curve25519_public_key_t *intro_enc_pubkey,
+          const struct curve25519_public_key_t *service_ephemeral_rend_pubkey,
+          hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out);
+
+int hs_ntor_service_get_introduce1_keys(
+            const struct ed25519_public_key_t *intro_auth_pubkey,
+            const struct curve25519_keypair_t *intro_enc_keypair,
+            const struct curve25519_public_key_t *client_ephemeral_enc_pubkey,
+            const uint8_t *subcredential,
+            hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out);
+
+int hs_ntor_service_get_rendezvous1_keys(
+            const struct ed25519_public_key_t *intro_auth_pubkey,
+            const struct curve25519_keypair_t *intro_enc_keypair,
+            const struct curve25519_keypair_t *service_ephemeral_rend_keypair,
+            const struct curve25519_public_key_t *client_ephemeral_enc_pubkey,
+            hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out);
+
+int hs_ntor_circuit_key_expansion(const uint8_t *ntor_key_seed,
+                                  size_t seed_len,
+                                  uint8_t *keys_out, size_t keys_out_len);
+
+int hs_ntor_client_rendezvous2_mac_is_good(
+                        const hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys,
+                        const uint8_t *rcvd_mac);
+
+#endif /* !defined(TOR_HS_NTOR_H) */
diff --git a/src/core/crypto/onion_crypto.c b/src/core/crypto/onion_crypto.c
new file mode 100644
index 0000000000..56b02e2996
--- /dev/null
+++ b/src/core/crypto/onion_crypto.c
@@ -0,0 +1,311 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file onion_crypto.c
+ * \brief Functions to handle different kinds of circuit extension crypto.
+ *
+ * In this module, we provide a set of abstractions to create a uniform
+ * interface over the three circuit extension handshakes that Tor has used
+ * over the years (TAP, CREATE_FAST, and ntor).  These handshakes are
+ * implemented in onion_tap.c, onion_fast.c, and onion_ntor.c respectively.
+ *
+ * All[*] of these handshakes follow a similar pattern: a client, knowing
+ * some key from the relay it wants to extend through, generates the
+ * first part of a handshake. A relay receives that handshake, and sends
+ * a reply.  Once the client handles the reply, it knows that it is
+ * talking to the right relay, and it shares some freshly negotiated key
+ * material with that relay.
+ *
+ * We sometimes call the client's part of the handshake an "onionskin".
+ * We do this because historically, Onion Routing used a multi-layer
+ * structure called an "onion" to construct circuits. Each layer of the
+ * onion contained key material chosen by the client, the identity of
+ * the next relay in the circuit, and a smaller onion, encrypted with
+ * the key of the next relay.  When we changed Tor to use a telescoping
+ * circuit extension design, it corresponded to sending each layer of the
+ * onion separately -- as a series of onionskins.
+ **/
+
+#include "core/or/or.h"
+#include "core/or/circuitbuild.h"
+#include "core/crypto/onion_crypto.h"
+#include "core/crypto/onion_fast.h"
+#include "core/crypto/onion_ntor.h"
+#include "core/crypto/onion_tap.h"
+#include "feature/relay/router.h"
+#include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_util.h"
+
+#include "core/or/crypt_path_st.h"
+#include "core/or/extend_info_st.h"
+
+/** Return a new server_onion_keys_t object with all of the keys
+ * and other info we might need to do onion handshakes.  (We make a copy of
+ * our keys for each cpuworker to avoid race conditions with the main thread,
+ * and to avoid locking) */
+server_onion_keys_t *
+server_onion_keys_new(void)
+{
+  server_onion_keys_t *keys = tor_malloc_zero(sizeof(server_onion_keys_t));
+  memcpy(keys->my_identity, router_get_my_id_digest(), DIGEST_LEN);
+  dup_onion_keys(&keys->onion_key, &keys->last_onion_key);
+  keys->curve25519_key_map = construct_ntor_key_map();
+  keys->junk_keypair = tor_malloc_zero(sizeof(curve25519_keypair_t));
+  curve25519_keypair_generate(keys->junk_keypair, 0);
+  return keys;
+}
+/** Release all storage held in <b>keys</b>. */
+void
+server_onion_keys_free_(server_onion_keys_t *keys)
+{
+  if (! keys)
+    return;
+
+  crypto_pk_free(keys->onion_key);
+  crypto_pk_free(keys->last_onion_key);
+  ntor_key_map_free(keys->curve25519_key_map);
+  tor_free(keys->junk_keypair);
+  memwipe(keys, 0, sizeof(server_onion_keys_t));
+  tor_free(keys);
+}
+
+/** Release whatever storage is held in <b>state</b>, depending on its
+ * type, and clear its pointer. */
+void
+onion_handshake_state_release(onion_handshake_state_t *state)
+{
+  switch (state->tag) {
+  case ONION_HANDSHAKE_TYPE_TAP:
+    crypto_dh_free(state->u.tap);
+    state->u.tap = NULL;
+    break;
+  case ONION_HANDSHAKE_TYPE_FAST:
+    fast_handshake_state_free(state->u.fast);
+    state->u.fast = NULL;
+    break;
+  case ONION_HANDSHAKE_TYPE_NTOR:
+    ntor_handshake_state_free(state->u.ntor);
+    state->u.ntor = NULL;
+    break;
+  default:
+    /* LCOV_EXCL_START
+     * This state should not even exist. */
+    log_warn(LD_BUG, "called with unknown handshake state type %d",
+             (int)state->tag);
+    tor_fragile_assert();
+    /* LCOV_EXCL_STOP */
+  }
+}
+
+/** Perform the first step of a circuit-creation handshake of type <b>type</b>
+ * (one of ONION_HANDSHAKE_TYPE_*): generate the initial "onion skin" in
+ * <b>onion_skin_out</b>, and store any state information in <b>state_out</b>.
+ * Return -1 on failure, and the length of the onionskin on acceptance.
+ */
+int
+onion_skin_create(int type,
+                  const extend_info_t *node,
+                  onion_handshake_state_t *state_out,
+                  uint8_t *onion_skin_out)
+{
+  int r = -1;
+
+  switch (type) {
+  case ONION_HANDSHAKE_TYPE_TAP:
+    if (!node->onion_key)
+      return -1;
+
+    if (onion_skin_TAP_create(node->onion_key,
+                              &state_out->u.tap,
+                              (char*)onion_skin_out) < 0)
+      return -1;
+
+    r = TAP_ONIONSKIN_CHALLENGE_LEN;
+    break;
+  case ONION_HANDSHAKE_TYPE_FAST:
+    if (fast_onionskin_create(&state_out->u.fast, onion_skin_out) < 0)
+      return -1;
+
+    r = CREATE_FAST_LEN;
+    break;
+  case ONION_HANDSHAKE_TYPE_NTOR:
+    if (!extend_info_supports_ntor(node))
+      return -1;
+    if (onion_skin_ntor_create((const uint8_t*)node->identity_digest,
+                               &node->curve25519_onion_key,
+                               &state_out->u.ntor,
+                               onion_skin_out) < 0)
+      return -1;
+
+    r = NTOR_ONIONSKIN_LEN;
+    break;
+  default:
+    /* LCOV_EXCL_START
+     * We should never try to create an impossible handshake type. */
+    log_warn(LD_BUG, "called with unknown handshake state type %d", type);
+    tor_fragile_assert();
+    r = -1;
+    /* LCOV_EXCL_STOP */
+  }
+
+  if (r > 0)
+    state_out->tag = (uint16_t) type;
+
+  return r;
+}
+
+/* This is the maximum value for keys_out_len passed to
+ * onion_skin_server_handshake, plus 16. We can make it bigger if needed:
+ * It just defines how many bytes to stack-allocate. */
+#define MAX_KEYS_TMP_LEN 128
+
+/** Perform the second (server-side) step of a circuit-creation handshake of
+ * type <b>type</b>, responding to the client request in <b>onion_skin</b>
+ * using the keys in <b>keys</b>.  On success, write our response into
+ * <b>reply_out</b>, generate <b>keys_out_len</b> bytes worth of key material
+ * in <b>keys_out_len</b>, a hidden service nonce to <b>rend_nonce_out</b>,
+ * and return the length of the reply. On failure, return -1.
+ */
+int
+onion_skin_server_handshake(int type,
+                      const uint8_t *onion_skin, size_t onionskin_len,
+                      const server_onion_keys_t *keys,
+                      uint8_t *reply_out,
+                      uint8_t *keys_out, size_t keys_out_len,
+                      uint8_t *rend_nonce_out)
+{
+  int r = -1;
+
+  switch (type) {
+  case ONION_HANDSHAKE_TYPE_TAP:
+    if (onionskin_len != TAP_ONIONSKIN_CHALLENGE_LEN)
+      return -1;
+    if (onion_skin_TAP_server_handshake((const char*)onion_skin,
+                                        keys->onion_key, keys->last_onion_key,
+                                        (char*)reply_out,
+                                        (char*)keys_out, keys_out_len)<0)
+      return -1;
+    r = TAP_ONIONSKIN_REPLY_LEN;
+    memcpy(rend_nonce_out, reply_out+DH1024_KEY_LEN, DIGEST_LEN);
+    break;
+  case ONION_HANDSHAKE_TYPE_FAST:
+    if (onionskin_len != CREATE_FAST_LEN)
+      return -1;
+    if (fast_server_handshake(onion_skin, reply_out, keys_out, keys_out_len)<0)
+      return -1;
+    r = CREATED_FAST_LEN;
+    memcpy(rend_nonce_out, reply_out+DIGEST_LEN, DIGEST_LEN);
+    break;
+  case ONION_HANDSHAKE_TYPE_NTOR:
+    if (onionskin_len < NTOR_ONIONSKIN_LEN)
+      return -1;
+    {
+      size_t keys_tmp_len = keys_out_len + DIGEST_LEN;
+      tor_assert(keys_tmp_len <= MAX_KEYS_TMP_LEN);
+      uint8_t keys_tmp[MAX_KEYS_TMP_LEN];
+
+      if (onion_skin_ntor_server_handshake(
+                                   onion_skin, keys->curve25519_key_map,
+                                   keys->junk_keypair,
+                                   keys->my_identity,
+                                   reply_out, keys_tmp, keys_tmp_len)<0) {
+        /* no need to memwipe here, since the output will never be used */
+        return -1;
+      }
+
+      memcpy(keys_out, keys_tmp, keys_out_len);
+      memcpy(rend_nonce_out, keys_tmp+keys_out_len, DIGEST_LEN);
+      memwipe(keys_tmp, 0, sizeof(keys_tmp));
+      r = NTOR_REPLY_LEN;
+    }
+    break;
+  default:
+    /* LCOV_EXCL_START
+     * We should have rejected this far before this point */
+    log_warn(LD_BUG, "called with unknown handshake state type %d", type);
+    tor_fragile_assert();
+    return -1;
+    /* LCOV_EXCL_STOP */
+  }
+
+  return r;
+}
+
+/** Perform the final (client-side) step of a circuit-creation handshake of
+ * type <b>type</b>, using our state in <b>handshake_state</b> and the
+ * server's response in <b>reply</b>. On success, generate <b>keys_out_len</b>
+ * bytes worth of key material in <b>keys_out_len</b>, set
+ * <b>rend_authenticator_out</b> to the "KH" field that can be used to
+ * establish introduction points at this hop, and return 0. On failure,
+ * return -1, and set *msg_out to an error message if this is worth
+ * complaining to the user about. */
+int
+onion_skin_client_handshake(int type,
+                      const onion_handshake_state_t *handshake_state,
+                      const uint8_t *reply, size_t reply_len,
+                      uint8_t *keys_out, size_t keys_out_len,
+                      uint8_t *rend_authenticator_out,
+                      const char **msg_out)
+{
+  if (handshake_state->tag != type)
+    return -1;
+
+  switch (type) {
+  case ONION_HANDSHAKE_TYPE_TAP:
+    if (reply_len != TAP_ONIONSKIN_REPLY_LEN) {
+      if (msg_out)
+        *msg_out = "TAP reply was not of the correct length.";
+      return -1;
+    }
+    if (onion_skin_TAP_client_handshake(handshake_state->u.tap,
+                                        (const char*)reply,
+                                        (char *)keys_out, keys_out_len,
+                                        msg_out) < 0)
+      return -1;
+
+    memcpy(rend_authenticator_out, reply+DH1024_KEY_LEN, DIGEST_LEN);
+
+    return 0;
+  case ONION_HANDSHAKE_TYPE_FAST:
+    if (reply_len != CREATED_FAST_LEN) {
+      if (msg_out)
+        *msg_out = "TAP reply was not of the correct length.";
+      return -1;
+    }
+    if (fast_client_handshake(handshake_state->u.fast, reply,
+                              keys_out, keys_out_len, msg_out) < 0)
+      return -1;
+
+    memcpy(rend_authenticator_out, reply+DIGEST_LEN, DIGEST_LEN);
+    return 0;
+  case ONION_HANDSHAKE_TYPE_NTOR:
+    if (reply_len < NTOR_REPLY_LEN) {
+      if (msg_out)
+        *msg_out = "ntor reply was not of the correct length.";
+      return -1;
+    }
+    {
+      size_t keys_tmp_len = keys_out_len + DIGEST_LEN;
+      uint8_t *keys_tmp = tor_malloc(keys_tmp_len);
+      if (onion_skin_ntor_client_handshake(handshake_state->u.ntor,
+                                        reply,
+                                        keys_tmp, keys_tmp_len, msg_out) < 0) {
+        tor_free(keys_tmp);
+        return -1;
+      }
+      memcpy(keys_out, keys_tmp, keys_out_len);
+      memcpy(rend_authenticator_out, keys_tmp + keys_out_len, DIGEST_LEN);
+      memwipe(keys_tmp, 0, keys_tmp_len);
+      tor_free(keys_tmp);
+    }
+    return 0;
+  default:
+    log_warn(LD_BUG, "called with unknown handshake state type %d", type);
+    tor_fragile_assert();
+    return -1;
+  }
+}
diff --git a/src/core/crypto/onion_crypto.h b/src/core/crypto/onion_crypto.h
new file mode 100644
index 0000000000..1cddde3610
--- /dev/null
+++ b/src/core/crypto/onion_crypto.h
@@ -0,0 +1,47 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file onion_crypto.h
+ * \brief Header file for onion_crypto.c.
+ **/
+
+#ifndef TOR_ONION_CRYPTO_H
+#define TOR_ONION_CRYPTO_H
+
+typedef struct server_onion_keys_t {
+  uint8_t my_identity[DIGEST_LEN];
+  crypto_pk_t *onion_key;
+  crypto_pk_t *last_onion_key;
+  struct di_digest256_map_t *curve25519_key_map;
+  struct curve25519_keypair_t *junk_keypair;
+} server_onion_keys_t;
+
+void onion_handshake_state_release(onion_handshake_state_t *state);
+
+int onion_skin_create(int type,
+                      const extend_info_t *node,
+                      onion_handshake_state_t *state_out,
+                      uint8_t *onion_skin_out);
+int onion_skin_server_handshake(int type,
+                      const uint8_t *onion_skin, size_t onionskin_len,
+                      const server_onion_keys_t *keys,
+                      uint8_t *reply_out,
+                      uint8_t *keys_out, size_t key_out_len,
+                      uint8_t *rend_nonce_out);
+int onion_skin_client_handshake(int type,
+                      const onion_handshake_state_t *handshake_state,
+                      const uint8_t *reply, size_t reply_len,
+                      uint8_t *keys_out, size_t key_out_len,
+                      uint8_t *rend_authenticator_out,
+                      const char **msg_out);
+
+server_onion_keys_t *server_onion_keys_new(void);
+void server_onion_keys_free_(server_onion_keys_t *keys);
+#define server_onion_keys_free(keys) \
+  FREE_AND_NULL(server_onion_keys_t, server_onion_keys_free_, (keys))
+
+#endif
diff --git a/src/or/onion_fast.c b/src/core/crypto/onion_fast.c
index 8dcbfe22d8..31bd20235f 100644
--- a/src/or/onion_fast.c
+++ b/src/core/crypto/onion_fast.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -23,16 +23,19 @@
  * [*]Actually, it's possible that TAP _was_ a little better than TLS with
  * RSA1024 certificates and EDH1024 for forward secrecy, if you
  * hypothesize an adversary who can compute discrete logarithms on a
- * small number of targetted DH1024 fields, but who can't break all that
+ * small number of targeted DH1024 fields, but who can't break all that
  * many RSA1024 keys.
  **/
 
-#include "or.h"
-#include "onion_fast.h"
+#include "core/or/or.h"
+#include "core/crypto/onion_fast.h"
+#include "lib/crypt_ops/crypto_hkdf.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
 
 /** Release all state held in <b>victim</b>. */
 void
-fast_handshake_state_free(fast_handshake_state_t *victim)
+fast_handshake_state_free_(fast_handshake_state_t *victim)
 {
   if (! victim)
     return;
@@ -139,4 +142,3 @@ fast_client_handshake(const fast_handshake_state_t *handshake_state,
   tor_free(out);
   return r;
 }
-
diff --git a/src/or/onion_fast.h b/src/core/crypto/onion_fast.h
index b9626002c3..0ba8cbbc35 100644
--- a/src/or/onion_fast.h
+++ b/src/core/crypto/onion_fast.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,7 +19,9 @@ typedef struct fast_handshake_state_t {
   uint8_t state[DIGEST_LEN];
 } fast_handshake_state_t;
 
-void fast_handshake_state_free(fast_handshake_state_t *victim);
+void fast_handshake_state_free_(fast_handshake_state_t *victim);
+#define fast_handshake_state_free(st) \
+  FREE_AND_NULL(fast_handshake_state_t, fast_handshake_state_free_, (st))
 
 int fast_onionskin_create(fast_handshake_state_t **handshake_state_out,
                           uint8_t *handshake_out);
@@ -35,5 +37,5 @@ int fast_client_handshake(const fast_handshake_state_t *handshake_state,
                           size_t key_out_len,
                           const char **msg_out);
 
-#endif
+#endif /* !defined(TOR_ONION_FAST_H) */
 
diff --git a/src/or/onion_ntor.c b/src/core/crypto/onion_ntor.c
index ded97ee73d..7087fe1bd7 100644
--- a/src/or/onion_ntor.c
+++ b/src/core/crypto/onion_ntor.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,14 +21,21 @@
 #include "orconfig.h"
 
 #define ONION_NTOR_PRIVATE
-#include "crypto.h"
-#include "onion_ntor.h"
-#include "torlog.h"
-#include "util.h"
+
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_hkdf.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "core/crypto/onion_ntor.h"
+
+#include <string.h>
 
 /** Free storage held in an ntor handshake state. */
 void
-ntor_handshake_state_free(ntor_handshake_state_t *state)
+ntor_handshake_state_free_(ntor_handshake_state_t *state)
 {
   if (!state)
     return;
@@ -332,4 +339,3 @@ onion_skin_ntor_client_handshake(
 
   return bad ? -1 : 0;
 }
-
diff --git a/src/or/onion_ntor.h b/src/core/crypto/onion_ntor.h
index f637b437fd..51e72b4083 100644
--- a/src/or/onion_ntor.h
+++ b/src/core/crypto/onion_ntor.h
@@ -1,12 +1,14 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_ONION_NTOR_H
 #define TOR_ONION_NTOR_H
 
-#include "torint.h"
-#include "crypto_curve25519.h"
-#include "di_ops.h"
+#include "lib/cc/torint.h"
+
+struct di_digest256_map_t;
+struct curve25519_public_key_t;
+struct curve25519_keypair_t;
 
 /** State to be maintained by a client between sending an ntor onionskin
  * and receiving a reply. */
@@ -17,20 +19,22 @@ typedef struct ntor_handshake_state_t ntor_handshake_state_t;
 /** Length of an ntor reply, as sent from server to client. */
 #define NTOR_REPLY_LEN 64
 
-void ntor_handshake_state_free(ntor_handshake_state_t *state);
+void ntor_handshake_state_free_(ntor_handshake_state_t *state);
+#define ntor_handshake_state_free(state) \
+  FREE_AND_NULL(ntor_handshake_state_t, ntor_handshake_state_free_, (state))
 
 int onion_skin_ntor_create(const uint8_t *router_id,
-                           const curve25519_public_key_t *router_key,
+                           const struct curve25519_public_key_t *router_key,
                            ntor_handshake_state_t **handshake_state_out,
                            uint8_t *onion_skin_out);
 
 int onion_skin_ntor_server_handshake(const uint8_t *onion_skin,
-                                 const di_digest256_map_t *private_keys,
-                                 const curve25519_keypair_t *junk_keypair,
-                                 const uint8_t *my_node_id,
-                                 uint8_t *handshake_reply_out,
-                                 uint8_t *key_out,
-                                 size_t key_out_len);
+                           const struct di_digest256_map_t *private_keys,
+                           const struct curve25519_keypair_t *junk_keypair,
+                           const uint8_t *my_node_id,
+                           uint8_t *handshake_reply_out,
+                           uint8_t *key_out,
+                           size_t key_out_len);
 
 int onion_skin_ntor_client_handshake(
                              const ntor_handshake_state_t *handshake_state,
@@ -40,6 +44,7 @@ int onion_skin_ntor_client_handshake(
                              const char **msg_out);
 
 #ifdef ONION_NTOR_PRIVATE
+#include "lib/crypt_ops/crypto_curve25519.h"
 
 /** Storage held by a client while waiting for an ntor reply from a server. */
 struct ntor_handshake_state_t {
@@ -55,7 +60,6 @@ struct ntor_handshake_state_t {
   curve25519_public_key_t pubkey_X;
   /** @} */
 };
-#endif
-
-#endif
+#endif /* defined(ONION_NTOR_PRIVATE) */
 
+#endif /* !defined(TOR_ONION_NTOR_H) */
diff --git a/src/or/onion_tap.c b/src/core/crypto/onion_tap.c
index 2769300945..854889d88d 100644
--- a/src/or/onion_tap.c
+++ b/src/core/crypto/onion_tap.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,10 +27,13 @@
  * invoked from onion.c.
  **/
 
-#include "or.h"
-#include "config.h"
-#include "onion_tap.h"
-#include "rephist.h"
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "core/crypto/onion_tap.h"
+#include "feature/stats/rephist.h"
 
 /*----------------------------------------------------------------------*/
 
@@ -51,7 +54,7 @@ onion_skin_TAP_create(crypto_pk_t *dest_router_key,
                   crypto_dh_t **handshake_state_out,
                   char *onion_skin_out) /* TAP_ONIONSKIN_CHALLENGE_LEN bytes */
 {
-  char challenge[DH_KEY_LEN];
+  char challenge[DH1024_KEY_LEN];
   crypto_dh_t *dh = NULL;
   int dhbytes, pkbytes;
 
@@ -72,12 +75,10 @@ onion_skin_TAP_create(crypto_pk_t *dest_router_key,
   if (crypto_dh_get_public(dh, challenge, dhbytes))
     goto err;
 
-  note_crypto_pk_op(ENC_ONIONSKIN);
-
   /* set meeting point, meeting cookie, etc here. Leave zero for now. */
-  if (crypto_pk_public_hybrid_encrypt(dest_router_key, onion_skin_out,
+  if (crypto_pk_obsolete_public_hybrid_encrypt(dest_router_key, onion_skin_out,
                                       TAP_ONIONSKIN_CHALLENGE_LEN,
-                                      challenge, DH_KEY_LEN,
+                                      challenge, DH1024_KEY_LEN,
                                       PK_PKCS1_OAEP_PADDING, 1)<0)
     goto err;
 
@@ -124,8 +125,7 @@ onion_skin_TAP_server_handshake(
     k = i==0?private_key:prev_private_key;
     if (!k)
       break;
-    note_crypto_pk_op(DEC_ONIONSKIN);
-    len = crypto_pk_private_hybrid_decrypt(k, challenge,
+    len = crypto_pk_obsolete_private_hybrid_decrypt(k, challenge,
                                            TAP_ONIONSKIN_CHALLENGE_LEN,
                                            onion_skin,
                                            TAP_ONIONSKIN_CHALLENGE_LEN,
@@ -137,7 +137,7 @@ onion_skin_TAP_server_handshake(
     log_info(LD_PROTOCOL,
              "Couldn't decrypt onionskin: client may be using old onion key");
     goto err;
-  } else if (len != DH_KEY_LEN) {
+  } else if (len != DH1024_KEY_LEN) {
     log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
            "Unexpected onionskin length after decryption: %ld",
            (long)len);
@@ -153,19 +153,19 @@ onion_skin_TAP_server_handshake(
     goto err;
     /* LCOV_EXCL_STOP */
   }
-  if (crypto_dh_get_public(dh, handshake_reply_out, DH_KEY_LEN)) {
+  if (crypto_dh_get_public(dh, handshake_reply_out, DH1024_KEY_LEN)) {
     /* LCOV_EXCL_START
      * This can only fail if the length of the key we just allocated is too
      * big. That should be impossible. */
     log_info(LD_GENERAL, "crypto_dh_get_public failed.");
     goto err;
-    /* LCOV_EXCP_STOP */
+    /* LCOV_EXCL_STOP */
   }
 
   key_material_len = DIGEST_LEN+key_out_len;
   key_material = tor_malloc(key_material_len);
   len = crypto_dh_compute_secret(LOG_PROTOCOL_WARN, dh, challenge,
-                                 DH_KEY_LEN, key_material,
+                                 DH1024_KEY_LEN, key_material,
                                  key_material_len);
   if (len < 0) {
     log_info(LD_GENERAL, "crypto_dh_compute_secret failed.");
@@ -173,7 +173,7 @@ onion_skin_TAP_server_handshake(
   }
 
   /* send back H(K|0) as proof that we learned K. */
-  memcpy(handshake_reply_out+DH_KEY_LEN, key_material, DIGEST_LEN);
+  memcpy(handshake_reply_out+DH1024_KEY_LEN, key_material, DIGEST_LEN);
 
   /* use the rest of the key material for our shared keys, digests, etc */
   memcpy(key_out, key_material+DIGEST_LEN, key_out_len);
@@ -213,12 +213,12 @@ onion_skin_TAP_client_handshake(crypto_dh_t *handshake_state,
   ssize_t len;
   char *key_material=NULL;
   size_t key_material_len;
-  tor_assert(crypto_dh_get_bytes(handshake_state) == DH_KEY_LEN);
+  tor_assert(crypto_dh_get_bytes(handshake_state) == DH1024_KEY_LEN);
 
   key_material_len = DIGEST_LEN + key_out_len;
   key_material = tor_malloc(key_material_len);
   len = crypto_dh_compute_secret(LOG_PROTOCOL_WARN, handshake_state,
-                                 handshake_reply, DH_KEY_LEN, key_material,
+                                 handshake_reply, DH1024_KEY_LEN, key_material,
                                  key_material_len);
   if (len < 0) {
     if (msg_out)
@@ -226,7 +226,7 @@ onion_skin_TAP_client_handshake(crypto_dh_t *handshake_state,
     goto err;
   }
 
-  if (tor_memneq(key_material, handshake_reply+DH_KEY_LEN, DIGEST_LEN)) {
+  if (tor_memneq(key_material, handshake_reply+DH1024_KEY_LEN, DIGEST_LEN)) {
     /* H(K) does *not* match. Something fishy. */
     if (msg_out)
       *msg_out = "Digest DOES NOT MATCH on onion handshake. Bug or attack.";
@@ -244,4 +244,3 @@ onion_skin_TAP_client_handshake(crypto_dh_t *handshake_state,
   tor_free(key_material);
   return -1;
 }
-
diff --git a/src/or/onion_tap.h b/src/core/crypto/onion_tap.h
index a2880f6e98..0e43b9c8ba 100644
--- a/src/or/onion_tap.h
+++ b/src/core/crypto/onion_tap.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,25 +14,27 @@
 
 #define TAP_ONIONSKIN_CHALLENGE_LEN (PKCS1_OAEP_PADDING_OVERHEAD+\
                                  CIPHER_KEY_LEN+\
-                                 DH_KEY_LEN)
-#define TAP_ONIONSKIN_REPLY_LEN (DH_KEY_LEN+DIGEST_LEN)
+                                 DH1024_KEY_LEN)
+#define TAP_ONIONSKIN_REPLY_LEN (DH1024_KEY_LEN+DIGEST_LEN)
 
-int onion_skin_TAP_create(crypto_pk_t *router_key,
-                      crypto_dh_t **handshake_state_out,
+struct crypto_dh_t;
+struct crypto_pk_t;
+
+int onion_skin_TAP_create(struct crypto_pk_t *router_key,
+                      struct crypto_dh_t **handshake_state_out,
                       char *onion_skin_out);
 
 int onion_skin_TAP_server_handshake(const char *onion_skin,
-                                crypto_pk_t *private_key,
-                                crypto_pk_t *prev_private_key,
+                                struct crypto_pk_t *private_key,
+                                struct crypto_pk_t *prev_private_key,
                                 char *handshake_reply_out,
                                 char *key_out,
                                 size_t key_out_len);
 
-int onion_skin_TAP_client_handshake(crypto_dh_t *handshake_state,
+int onion_skin_TAP_client_handshake(struct crypto_dh_t *handshake_state,
                                 const char *handshake_reply,
                                 char *key_out,
                                 size_t key_out_len,
                                 const char **msg_out);
 
-#endif
-
+#endif /* !defined(TOR_ONION_TAP_H) */
diff --git a/src/core/crypto/relay_crypto.c b/src/core/crypto/relay_crypto.c
new file mode 100644
index 0000000000..0b83b2d0a5
--- /dev/null
+++ b/src/core/crypto/relay_crypto.c
@@ -0,0 +1,332 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "core/or/circuitlist.h"
+#include "app/config/config.h"
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "core/crypto/hs_ntor.h" // for HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN
+#include "core/or/relay.h"
+#include "core/crypto/relay_crypto.h"
+
+#include "core/or/cell_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+
+/** Update digest from the payload of cell. Assign integrity part to
+ * cell.
+ */
+static void
+relay_set_digest(crypto_digest_t *digest, cell_t *cell)
+{
+  char integrity[4];
+  relay_header_t rh;
+
+  crypto_digest_add_bytes(digest, (char*)cell->payload, CELL_PAYLOAD_SIZE);
+  crypto_digest_get_digest(digest, integrity, 4);
+//  log_fn(LOG_DEBUG,"Putting digest of %u %u %u %u into relay cell.",
+//    integrity[0], integrity[1], integrity[2], integrity[3]);
+  relay_header_unpack(&rh, cell->payload);
+  memcpy(rh.integrity, integrity, 4);
+  relay_header_pack(cell->payload, &rh);
+}
+
+/** Does the digest for this circuit indicate that this cell is for us?
+ *
+ * Update digest from the payload of cell (with the integrity part set
+ * to 0). If the integrity part is valid, return 1, else restore digest
+ * and cell to their original state and return 0.
+ */
+static int
+relay_digest_matches(crypto_digest_t *digest, cell_t *cell)
+{
+  uint32_t received_integrity, calculated_integrity;
+  relay_header_t rh;
+  crypto_digest_checkpoint_t backup_digest;
+
+  crypto_digest_checkpoint(&backup_digest, digest);
+
+  relay_header_unpack(&rh, cell->payload);
+  memcpy(&received_integrity, rh.integrity, 4);
+  memset(rh.integrity, 0, 4);
+  relay_header_pack(cell->payload, &rh);
+
+//  log_fn(LOG_DEBUG,"Reading digest of %u %u %u %u from relay cell.",
+//    received_integrity[0], received_integrity[1],
+//    received_integrity[2], received_integrity[3]);
+
+  crypto_digest_add_bytes(digest, (char*) cell->payload, CELL_PAYLOAD_SIZE);
+  crypto_digest_get_digest(digest, (char*) &calculated_integrity, 4);
+
+  int rv = 1;
+
+  if (calculated_integrity != received_integrity) {
+//    log_fn(LOG_INFO,"Recognized=0 but bad digest. Not recognizing.");
+// (%d vs %d).", received_integrity, calculated_integrity);
+    /* restore digest to its old form */
+    crypto_digest_restore(digest, &backup_digest);
+    /* restore the relay header */
+    memcpy(rh.integrity, &received_integrity, 4);
+    relay_header_pack(cell->payload, &rh);
+    rv = 0;
+  }
+
+  memwipe(&backup_digest, 0, sizeof(backup_digest));
+  return rv;
+}
+
+/** Apply <b>cipher</b> to CELL_PAYLOAD_SIZE bytes of <b>in</b>
+ * (in place).
+ *
+ * Note that we use the same operation for encrypting and for decrypting.
+ */
+static void
+relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in)
+{
+  crypto_cipher_crypt_inplace(cipher, (char*) in, CELL_PAYLOAD_SIZE);
+}
+
+/** Do the appropriate en/decryptions for <b>cell</b> arriving on
+ * <b>circ</b> in direction <b>cell_direction</b>.
+ *
+ * If cell_direction == CELL_DIRECTION_IN:
+ *   - If we're at the origin (we're the OP), for hops 1..N,
+ *     decrypt cell. If recognized, stop.
+ *   - Else (we're not the OP), encrypt one hop. Cell is not recognized.
+ *
+ * If cell_direction == CELL_DIRECTION_OUT:
+ *   - decrypt one hop. Check if recognized.
+ *
+ * If cell is recognized, set *recognized to 1, and set
+ * *layer_hint to the hop that recognized it.
+ *
+ * Return -1 to indicate that we should mark the circuit for close,
+ * else return 0.
+ */
+int
+relay_decrypt_cell(circuit_t *circ, cell_t *cell,
+                   cell_direction_t cell_direction,
+                   crypt_path_t **layer_hint, char *recognized)
+{
+  relay_header_t rh;
+
+  tor_assert(circ);
+  tor_assert(cell);
+  tor_assert(recognized);
+  tor_assert(cell_direction == CELL_DIRECTION_IN ||
+             cell_direction == CELL_DIRECTION_OUT);
+
+  if (cell_direction == CELL_DIRECTION_IN) {
+    if (CIRCUIT_IS_ORIGIN(circ)) { /* We're at the beginning of the circuit.
+                                    * We'll want to do layered decrypts. */
+      crypt_path_t *thishop, *cpath = TO_ORIGIN_CIRCUIT(circ)->cpath;
+      thishop = cpath;
+      if (thishop->state != CPATH_STATE_OPEN) {
+        log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+               "Relay cell before first created cell? Closing.");
+        return -1;
+      }
+      do { /* Remember: cpath is in forward order, that is, first hop first. */
+        tor_assert(thishop);
+
+        /* decrypt one layer */
+        relay_crypt_one_payload(thishop->crypto.b_crypto, cell->payload);
+
+        relay_header_unpack(&rh, cell->payload);
+        if (rh.recognized == 0) {
+          /* it's possibly recognized. have to check digest to be sure. */
+          if (relay_digest_matches(thishop->crypto.b_digest, cell)) {
+            *recognized = 1;
+            *layer_hint = thishop;
+            return 0;
+          }
+        }
+
+        thishop = thishop->next;
+      } while (thishop != cpath && thishop->state == CPATH_STATE_OPEN);
+      log_fn(LOG_PROTOCOL_WARN, LD_OR,
+             "Incoming cell at client not recognized. Closing.");
+      return -1;
+    } else {
+      relay_crypto_t *crypto = &TO_OR_CIRCUIT(circ)->crypto;
+      /* We're in the middle. Encrypt one layer. */
+      relay_crypt_one_payload(crypto->b_crypto, cell->payload);
+    }
+  } else /* cell_direction == CELL_DIRECTION_OUT */ {
+    /* We're in the middle. Decrypt one layer. */
+    relay_crypto_t *crypto = &TO_OR_CIRCUIT(circ)->crypto;
+
+    relay_crypt_one_payload(crypto->f_crypto, cell->payload);
+
+    relay_header_unpack(&rh, cell->payload);
+    if (rh.recognized == 0) {
+      /* it's possibly recognized. have to check digest to be sure. */
+      if (relay_digest_matches(crypto->f_digest, cell)) {
+        *recognized = 1;
+        return 0;
+      }
+    }
+  }
+  return 0;
+}
+
+/**
+ * Encrypt a cell <b>cell</b> that we are creating, and sending outbound on
+ * <b>circ</b> until the hop corresponding to <b>layer_hint</b>.
+ *
+ * The integrity field and recognized field of <b>cell</b>'s relay headers
+ * must be set to zero.
+ */
+void
+relay_encrypt_cell_outbound(cell_t *cell,
+                            origin_circuit_t *circ,
+                            crypt_path_t *layer_hint)
+{
+  crypt_path_t *thishop; /* counter for repeated crypts */
+  relay_set_digest(layer_hint->crypto.f_digest, cell);
+
+  thishop = layer_hint;
+  /* moving from farthest to nearest hop */
+  do {
+    tor_assert(thishop);
+    log_debug(LD_OR,"encrypting a layer of the relay cell.");
+    relay_crypt_one_payload(thishop->crypto.f_crypto, cell->payload);
+
+    thishop = thishop->prev;
+  } while (thishop != circ->cpath->prev);
+}
+
+/**
+ * Encrypt a cell <b>cell</b> that we are creating, and sending on
+ * <b>circuit</b> to the origin.
+ *
+ * The integrity field and recognized field of <b>cell</b>'s relay headers
+ * must be set to zero.
+ */
+void
+relay_encrypt_cell_inbound(cell_t *cell,
+                           or_circuit_t *or_circ)
+{
+  relay_set_digest(or_circ->crypto.b_digest, cell);
+  /* encrypt one layer */
+  relay_crypt_one_payload(or_circ->crypto.b_crypto, cell->payload);
+}
+
+/**
+ * Release all storage held inside <b>crypto</b>, but do not free
+ * <b>crypto</b> itself: it lives inside another object.
+ */
+void
+relay_crypto_clear(relay_crypto_t *crypto)
+{
+  if (BUG(!crypto))
+    return;
+  crypto_cipher_free(crypto->f_crypto);
+  crypto_cipher_free(crypto->b_crypto);
+  crypto_digest_free(crypto->f_digest);
+  crypto_digest_free(crypto->b_digest);
+}
+
+/** Initialize <b>crypto</b> from the key material in key_data.
+ *
+ * If <b>is_hs_v3</b> is set, this cpath will be used for next gen hidden
+ * service circuits and <b>key_data</b> must be at least
+ * HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN bytes in length.
+ *
+ * If <b>is_hs_v3</b> is not set, key_data must contain CPATH_KEY_MATERIAL_LEN
+ * bytes, which are used as follows:
+ *   - 20 to initialize f_digest
+ *   - 20 to initialize b_digest
+ *   - 16 to key f_crypto
+ *   - 16 to key b_crypto
+ *
+ * (If 'reverse' is true, then f_XX and b_XX are swapped.)
+ *
+ * Return 0 if init was successful, else -1 if it failed.
+ */
+int
+relay_crypto_init(relay_crypto_t *crypto,
+                  const char *key_data, size_t key_data_len,
+                  int reverse, int is_hs_v3)
+{
+  crypto_digest_t *tmp_digest;
+  crypto_cipher_t *tmp_crypto;
+  size_t digest_len = 0;
+  size_t cipher_key_len = 0;
+
+  tor_assert(crypto);
+  tor_assert(key_data);
+  tor_assert(!(crypto->f_crypto || crypto->b_crypto ||
+             crypto->f_digest || crypto->b_digest));
+
+  /* Basic key size validation */
+  if (is_hs_v3 && BUG(key_data_len != HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN)) {
+    goto err;
+  } else if (!is_hs_v3 && BUG(key_data_len != CPATH_KEY_MATERIAL_LEN)) {
+    goto err;
+  }
+
+  /* If we are using this crypto for next gen onion services use SHA3-256,
+     otherwise use good ol' SHA1 */
+  if (is_hs_v3) {
+    digest_len = DIGEST256_LEN;
+    cipher_key_len = CIPHER256_KEY_LEN;
+    crypto->f_digest = crypto_digest256_new(DIGEST_SHA3_256);
+    crypto->b_digest = crypto_digest256_new(DIGEST_SHA3_256);
+  } else {
+    digest_len = DIGEST_LEN;
+    cipher_key_len = CIPHER_KEY_LEN;
+    crypto->f_digest = crypto_digest_new();
+    crypto->b_digest = crypto_digest_new();
+  }
+
+  tor_assert(digest_len != 0);
+  tor_assert(cipher_key_len != 0);
+  const int cipher_key_bits = (int) cipher_key_len * 8;
+
+  crypto_digest_add_bytes(crypto->f_digest, key_data, digest_len);
+  crypto_digest_add_bytes(crypto->b_digest, key_data+digest_len, digest_len);
+
+  crypto->f_crypto = crypto_cipher_new_with_bits(key_data+(2*digest_len),
+                                                cipher_key_bits);
+  if (!crypto->f_crypto) {
+    log_warn(LD_BUG,"Forward cipher initialization failed.");
+    goto err;
+  }
+
+  crypto->b_crypto = crypto_cipher_new_with_bits(
+                                        key_data+(2*digest_len)+cipher_key_len,
+                                        cipher_key_bits);
+  if (!crypto->b_crypto) {
+    log_warn(LD_BUG,"Backward cipher initialization failed.");
+    goto err;
+  }
+
+  if (reverse) {
+    tmp_digest = crypto->f_digest;
+    crypto->f_digest = crypto->b_digest;
+    crypto->b_digest = tmp_digest;
+    tmp_crypto = crypto->f_crypto;
+    crypto->f_crypto = crypto->b_crypto;
+    crypto->b_crypto = tmp_crypto;
+  }
+
+  return 0;
+ err:
+  relay_crypto_clear(crypto);
+  return -1;
+}
+
+/** Assert that <b>crypto</b> is valid and set. */
+void
+relay_crypto_assert_ok(const relay_crypto_t *crypto)
+{
+  tor_assert(crypto->f_crypto);
+  tor_assert(crypto->b_crypto);
+  tor_assert(crypto->f_digest);
+  tor_assert(crypto->b_digest);
+}
diff --git a/src/core/crypto/relay_crypto.h b/src/core/crypto/relay_crypto.h
new file mode 100644
index 0000000000..45a21d14ab
--- /dev/null
+++ b/src/core/crypto/relay_crypto.h
@@ -0,0 +1,31 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file relay.h
+ * \brief Header file for relay.c.
+ **/
+
+#ifndef TOR_RELAY_CRYPTO_H
+#define TOR_RELAY_CRYPTO_H
+
+int relay_crypto_init(relay_crypto_t *crypto,
+                      const char *key_data, size_t key_data_len,
+                      int reverse, int is_hs_v3);
+
+int relay_decrypt_cell(circuit_t *circ, cell_t *cell,
+                       cell_direction_t cell_direction,
+                       crypt_path_t **layer_hint, char *recognized);
+void relay_encrypt_cell_outbound(cell_t *cell, origin_circuit_t *or_circ,
+                            crypt_path_t *layer_hint);
+void relay_encrypt_cell_inbound(cell_t *cell, or_circuit_t *or_circ);
+
+void relay_crypto_clear(relay_crypto_t *crypto);
+
+void relay_crypto_assert_ok(const relay_crypto_t *crypto);
+
+#endif /* !defined(TOR_RELAY_CRYPTO_H) */
+
diff --git a/src/core/include.am b/src/core/include.am
new file mode 100644
index 0000000000..1b8ef2ac58
--- /dev/null
+++ b/src/core/include.am
@@ -0,0 +1,404 @@
+
+noinst_LIBRARIES += \
+	src/core/libtor-app.a
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += \
+	src/core/libtor-app-testing.a
+endif
+
+LIBTOR_APP_A_SOURCES = 				\
+	src/app/config/config.c			\
+	src/app/config/confparse.c		\
+	src/app/config/statefile.c		\
+	src/app/main/main.c			\
+	src/core/crypto/hs_ntor.c		\
+	src/core/crypto/onion_crypto.c		\
+	src/core/crypto/onion_fast.c		\
+	src/core/crypto/onion_ntor.c		\
+	src/core/crypto/onion_tap.c		\
+	src/core/crypto/relay_crypto.c		\
+	src/core/mainloop/connection.c		\
+	src/core/mainloop/cpuworker.c		\
+	src/core/mainloop/mainloop.c		\
+	src/core/mainloop/netstatus.c		\
+	src/core/mainloop/periodic.c		\
+	src/core/or/address_set.c		\
+	src/core/or/channel.c			\
+	src/core/or/channelpadding.c		\
+	src/core/or/channeltls.c		\
+	src/core/or/circuitbuild.c		\
+	src/core/or/circuitlist.c		\
+	src/core/or/circuitmux.c		\
+	src/core/or/circuitmux_ewma.c		\
+	src/core/or/circuitstats.c		\
+	src/core/or/circuituse.c		\
+	src/core/or/command.c			\
+	src/core/or/connection_edge.c		\
+	src/core/or/connection_or.c		\
+	src/core/or/dos.c			\
+	src/core/or/onion.c			\
+	src/core/or/policies.c			\
+	src/core/or/protover.c			\
+	src/core/or/protover_rust.c		\
+	src/core/or/reasons.c			\
+	src/core/or/relay.c			\
+	src/core/or/scheduler.c			\
+	src/core/or/scheduler_kist.c		\
+	src/core/or/scheduler_vanilla.c		\
+	src/core/or/status.c			\
+	src/core/or/versions.c			\
+	src/core/proto/proto_cell.c		\
+	src/core/proto/proto_control0.c		\
+	src/core/proto/proto_ext_or.c		\
+	src/core/proto/proto_http.c		\
+	src/core/proto/proto_socks.c		\
+	src/feature/api/tor_api.c		\
+	src/feature/client/addressmap.c		\
+	src/feature/client/bridges.c		\
+	src/feature/client/circpathbias.c	\
+	src/feature/client/dnsserv.c		\
+	src/feature/client/entrynodes.c		\
+	src/feature/client/transports.c		\
+	src/feature/control/control.c		\
+	src/feature/control/fmt_serverstatus.c  \
+	src/feature/control/getinfo_geoip.c	\
+	src/feature/dirauth/keypin.c		\
+	src/feature/dircache/conscache.c	\
+	src/feature/dircache/consdiffmgr.c	\
+	src/feature/dircache/dircache.c		\
+	src/feature/dircache/dirserv.c		\
+	src/feature/dirclient/dirclient.c	\
+	src/feature/dirclient/dlstatus.c	\
+	src/feature/dircommon/consdiff.c	\
+	src/feature/dircommon/directory.c	\
+	src/feature/dircommon/fp_pair.c		\
+	src/feature/dircommon/voting_schedule.c	\
+	src/feature/dirparse/authcert_parse.c	\
+	src/feature/dirparse/microdesc_parse.c	\
+	src/feature/dirparse/ns_parse.c		\
+	src/feature/dirparse/parsecommon.c	\
+	src/feature/dirparse/policy_parse.c	\
+	src/feature/dirparse/routerparse.c	\
+	src/feature/dirparse/sigcommon.c	\
+	src/feature/dirparse/signing.c		\
+	src/feature/dirparse/unparseable.c	\
+	src/feature/hibernate/hibernate.c	\
+	src/feature/hs/hs_cache.c		\
+	src/feature/hs/hs_cell.c		\
+	src/feature/hs/hs_circuit.c		\
+	src/feature/hs/hs_circuitmap.c		\
+	src/feature/hs/hs_client.c		\
+	src/feature/hs/hs_common.c		\
+	src/feature/hs/hs_config.c		\
+	src/feature/hs/hs_control.c		\
+	src/feature/hs/hs_descriptor.c		\
+	src/feature/hs/hs_ident.c		\
+	src/feature/hs/hs_intropoint.c		\
+	src/feature/hs/hs_service.c		\
+	src/feature/hs/hs_stats.c		\
+	src/feature/hs_common/replaycache.c	\
+	src/feature/hs_common/shared_random_client.c	\
+	src/feature/keymgt/loadkey.c		\
+	src/feature/dirauth/keypin.c		\
+	src/feature/nodelist/authcert.c		\
+	src/feature/nodelist/describe.c		\
+	src/feature/nodelist/dirlist.c		\
+	src/feature/nodelist/microdesc.c	\
+	src/feature/nodelist/networkstatus.c	\
+	src/feature/nodelist/nickname.c		\
+	src/feature/nodelist/nodelist.c		\
+	src/feature/nodelist/node_select.c	\
+	src/feature/nodelist/routerinfo.c	\
+	src/feature/nodelist/routerlist.c	\
+	src/feature/nodelist/routerset.c	\
+	src/feature/nodelist/fmt_routerstatus.c	\
+	src/feature/nodelist/torcert.c		\
+	src/feature/relay/dns.c			\
+	src/feature/relay/ext_orport.c		\
+	src/feature/relay/onion_queue.c		\
+	src/feature/relay/router.c		\
+	src/feature/relay/routerkeys.c		\
+	src/feature/relay/routermode.c		\
+	src/feature/relay/selftest.c		\
+	src/feature/rend/rendcache.c		\
+	src/feature/rend/rendclient.c		\
+	src/feature/rend/rendcommon.c		\
+	src/feature/rend/rendmid.c		\
+	src/feature/rend/rendparse.c		\
+	src/feature/rend/rendservice.c		\
+	src/feature/stats/geoip_stats.c		\
+	src/feature/stats/rephist.c		\
+	src/feature/stats/predict_ports.c
+
+# These should eventually move into module_dirauth_sources, but for now
+# the separation is only in the code location.
+LIBTOR_APP_A_SOURCES += 			\
+	src/feature/dirauth/bwauth.c		\
+	src/feature/dirauth/dsigs_parse.c	\
+	src/feature/dirauth/guardfraction.c	\
+	src/feature/dirauth/reachability.c	\
+	src/feature/dirauth/recommend_pkg.c	\
+	src/feature/dirauth/process_descs.c	\
+	src/feature/dirauth/voteflags.c
+
+if BUILD_NT_SERVICES
+LIBTOR_APP_A_SOURCES += src/app/main/ntmain.c
+endif
+
+#
+# Modules are conditionnally compiled in tor starting here. We add the C files
+# only if the modules has been enabled at configure time. We always add the
+# source files of every module to libtor-testing.a so we can build the unit
+# tests for everything. See the UNITTESTS_ENABLED branch below.
+#
+LIBTOR_APP_TESTING_A_SOURCES = $(LIBTOR_APP_A_SOURCES)
+
+# The Directory Authority module.
+MODULE_DIRAUTH_SOURCES = 					\
+	src/feature/dirauth/authmode.c				\
+	src/feature/dirauth/dircollate.c			\
+	src/feature/dirauth/dirvote.c				\
+	src/feature/dirauth/shared_random.c			\
+	src/feature/dirauth/shared_random_state.c
+
+if BUILD_MODULE_DIRAUTH
+LIBTOR_APP_A_SOURCES += $(MODULE_DIRAUTH_SOURCES)
+endif
+
+src_core_libtor_app_a_SOURCES = $(LIBTOR_APP_A_SOURCES)
+if UNITTESTS_ENABLED
+
+# Add the sources of the modules that are needed for tests to work here.
+LIBTOR_APP_TESTING_A_SOURCES += $(MODULE_DIRAUTH_SOURCES)
+
+src_core_libtor_app_testing_a_SOURCES = $(LIBTOR_APP_TESTING_A_SOURCES)
+else
+src_core_libtor_app_testing_a_SOURCES =
+endif
+
+AM_CPPFLAGS += -DSHARE_DATADIR="\"$(datadir)\""		\
+		-DLOCALSTATEDIR="\"$(localstatedir)\""	\
+		-DBINDIR="\"$(bindir)\""
+
+src_core_libtor_app_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_core_libtor_app_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/app/config/config.h				\
+	src/app/config/confparse.h			\
+	src/app/config/or_options_st.h			\
+	src/app/config/or_state_st.h			\
+	src/app/config/statefile.h			\
+	src/app/main/main.h				\
+	src/app/main/ntmain.h				\
+	src/core/crypto/hs_ntor.h			\
+	src/core/crypto/onion_crypto.h	        	\
+	src/core/crypto/onion_fast.h			\
+	src/core/crypto/onion_ntor.h			\
+	src/core/crypto/onion_tap.h			\
+	src/core/crypto/relay_crypto.h			\
+	src/core/mainloop/connection.h			\
+	src/core/mainloop/cpuworker.h			\
+	src/core/mainloop/mainloop.h			\
+	src/core/mainloop/netstatus.h			\
+	src/core/mainloop/periodic.h			\
+	src/core/or/addr_policy_st.h			\
+	src/core/or/address_set.h			\
+	src/core/or/cell_queue_st.h			\
+	src/core/or/cell_st.h				\
+	src/core/or/channel.h				\
+	src/core/or/channelpadding.h			\
+	src/core/or/channeltls.h			\
+	src/core/or/circuit_st.h			\
+	src/core/or/circuitbuild.h			\
+	src/core/or/circuitlist.h			\
+	src/core/or/circuitmux.h			\
+	src/core/or/circuitmux_ewma.h			\
+	src/core/or/circuitstats.h			\
+	src/core/or/circuituse.h			\
+	src/core/or/command.h				\
+	src/core/or/connection_edge.h			\
+	src/core/or/connection_or.h			\
+	src/core/or/connection_st.h			\
+	src/core/or/cpath_build_state_st.h		\
+	src/core/or/crypt_path_reference_st.h		\
+	src/core/or/crypt_path_st.h			\
+	src/core/or/destroy_cell_queue_st.h		\
+	src/core/or/dos.h				\
+	src/core/or/edge_connection_st.h		\
+	src/core/or/half_edge_st.h		\
+	src/core/or/entry_connection_st.h		\
+	src/core/or/entry_port_cfg_st.h			\
+	src/core/or/extend_info_st.h			\
+	src/core/or/listener_connection_st.h		\
+	src/core/or/onion.h				\
+	src/core/or/or.h				\
+	src/core/or/or_circuit_st.h			\
+	src/core/or/or_connection_st.h			\
+	src/core/or/or_handshake_certs_st.h		\
+	src/core/or/or_handshake_state_st.h		\
+	src/core/or/origin_circuit_st.h			\
+	src/core/or/policies.h				\
+	src/core/or/port_cfg_st.h			\
+	src/core/or/protover.h				\
+	src/core/or/reasons.h				\
+	src/core/or/relay.h				\
+	src/core/or/relay_crypto_st.h			\
+	src/core/or/scheduler.h				\
+	src/core/or/server_port_cfg_st.h		\
+	src/core/or/socks_request_st.h			\
+	src/core/or/status.h				\
+	src/core/or/tor_version_st.h			\
+	src/core/or/var_cell_st.h			\
+	src/core/or/versions.h				\
+	src/core/proto/proto_cell.h			\
+	src/core/proto/proto_control0.h			\
+	src/core/proto/proto_ext_or.h			\
+	src/core/proto/proto_http.h			\
+	src/core/proto/proto_socks.h			\
+	src/feature/api/tor_api_internal.h		\
+	src/feature/client/addressmap.h			\
+	src/feature/client/bridges.h			\
+	src/feature/client/circpathbias.h		\
+	src/feature/client/dnsserv.h			\
+	src/feature/client/entrynodes.h			\
+	src/feature/client/transports.h			\
+	src/feature/control/control.h			\
+	src/feature/control/control_connection_st.h	\
+	src/feature/control/fmt_serverstatus.h		\
+	src/feature/control/getinfo_geoip.h		\
+	src/feature/dirauth/authmode.h			\
+	src/feature/dirauth/bwauth.h			\
+	src/feature/dirauth/dircollate.h		\
+	src/feature/dirauth/dirvote.h			\
+	src/feature/dirauth/dsigs_parse.h		\
+	src/feature/dirauth/guardfraction.h		\
+	src/feature/dirauth/keypin.h			\
+	src/feature/dirauth/ns_detached_signatures_st.h	\
+	src/feature/dirauth/reachability.h		\
+	src/feature/dirauth/recommend_pkg.h		\
+	src/feature/dirauth/process_descs.h		\
+	src/feature/dirauth/shared_random.h		\
+	src/feature/dirauth/shared_random_state.h	\
+	src/feature/dirauth/vote_microdesc_hash_st.h	\
+	src/feature/dirauth/voteflags.h			\
+	src/feature/dircache/cached_dir_st.h		\
+	src/feature/dircache/conscache.h		\
+	src/feature/dircache/consdiffmgr.h		\
+	src/feature/dircache/dircache.h			\
+	src/feature/dircache/dirserv.h			\
+	src/feature/dirclient/dir_server_st.h		\
+	src/feature/dirclient/dirclient.h		\
+	src/feature/dirclient/dlstatus.h		\
+	src/feature/dirclient/download_status_st.h	\
+	src/feature/dircommon/consdiff.h		\
+	src/feature/dircommon/dir_connection_st.h	\
+	src/feature/dircommon/directory.h		\
+	src/feature/dircommon/fp_pair.h			\
+	src/feature/dircommon/vote_timing_st.h		\
+	src/feature/dircommon/voting_schedule.h		\
+	src/feature/dirparse/authcert_members.i		\
+	src/feature/dirparse/authcert_parse.h		\
+	src/feature/dirparse/microdesc_parse.h		\
+	src/feature/dirparse/ns_parse.h			\
+	src/feature/dirparse/parsecommon.h		\
+	src/feature/dirparse/policy_parse.h		\
+	src/feature/dirparse/routerparse.h		\
+	src/feature/dirparse/sigcommon.h		\
+	src/feature/dirparse/signing.h			\
+	src/feature/dirparse/unparseable.h		\
+	src/feature/hibernate/hibernate.h		\
+	src/feature/hs/hs_cache.h			\
+	src/feature/hs/hs_cell.h			\
+	src/feature/hs/hs_circuit.h			\
+	src/feature/hs/hs_circuitmap.h			\
+	src/feature/hs/hs_client.h			\
+	src/feature/hs/hs_common.h			\
+	src/feature/hs/hs_config.h			\
+	src/feature/hs/hs_control.h			\
+	src/feature/hs/hs_descriptor.h			\
+	src/feature/hs/hs_ident.h			\
+	src/feature/hs/hs_intropoint.h			\
+	src/feature/hs/hs_service.h			\
+	src/feature/hs/hs_stats.h			\
+	src/feature/hs/hsdir_index_st.h			\
+	src/feature/hs_common/replaycache.h		\
+	src/feature/hs_common/shared_random_client.h	\
+	src/feature/keymgt/loadkey.h			\
+	src/feature/nodelist/authcert.h			\
+	src/feature/nodelist/authority_cert_st.h	\
+	src/feature/nodelist/describe.h			\
+	src/feature/nodelist/desc_store_st.h		\
+	src/feature/nodelist/dirlist.h			\
+	src/feature/nodelist/document_signature_st.h	\
+	src/feature/nodelist/extrainfo_st.h		\
+	src/feature/nodelist/microdesc.h		\
+	src/feature/nodelist/microdesc_st.h		\
+	src/feature/nodelist/networkstatus.h		\
+	src/feature/nodelist/networkstatus_sr_info_st.h	\
+	src/feature/nodelist/networkstatus_st.h		\
+	src/feature/nodelist/networkstatus_voter_info_st.h	\
+	src/feature/nodelist/nickname.h			\
+	src/feature/nodelist/node_st.h			\
+	src/feature/nodelist/nodelist.h			\
+	src/feature/nodelist/node_select.h		\
+	src/feature/nodelist/routerinfo.h		\
+	src/feature/nodelist/routerinfo_st.h		\
+	src/feature/nodelist/routerlist.h		\
+	src/feature/nodelist/routerlist_st.h		\
+	src/feature/nodelist/routerset.h		\
+	src/feature/nodelist/fmt_routerstatus.h		\
+	src/feature/nodelist/routerstatus_st.h		\
+	src/feature/nodelist/signed_descriptor_st.h	\
+	src/feature/nodelist/torcert.h			\
+	src/feature/nodelist/vote_routerstatus_st.h	\
+	src/feature/relay/dns.h				\
+	src/feature/relay/dns_structs.h			\
+	src/feature/relay/ext_orport.h			\
+	src/feature/relay/onion_queue.h			\
+	src/feature/relay/router.h			\
+	src/feature/relay/routerkeys.h			\
+	src/feature/relay/routermode.h			\
+	src/feature/relay/selftest.h			\
+	src/feature/rend/rend_authorized_client_st.h	\
+	src/feature/rend/rend_encoded_v2_service_descriptor_st.h	\
+	src/feature/rend/rend_intro_point_st.h		\
+	src/feature/rend/rend_service_descriptor_st.h	\
+	src/feature/rend/rendcache.h			\
+	src/feature/rend/rendclient.h			\
+	src/feature/rend/rendcommon.h			\
+	src/feature/rend/rendmid.h			\
+	src/feature/rend/rendparse.h			\
+	src/feature/rend/rendservice.h			\
+	src/feature/stats/geoip_stats.h			\
+	src/feature/stats/rephist.h			\
+	src/feature/stats/predict_ports.h
+
+noinst_HEADERS +=			\
+	src/app/config/auth_dirs.inc	\
+	src/app/config/fallback_dirs.inc
+
+# This may someday want to be an installed file?
+noinst_HEADERS += src/feature/api/tor_api.h
+
+micro-revision.i: FORCE
+	$(AM_V_at)rm -f micro-revision.tmp; \
+	if test -r "$(top_srcdir)/.git" && \
+			test -x "`which git 2>&1;true`"; then \
+		HASH="`cd "$(top_srcdir)" && git rev-parse --short=16 HEAD`"; \
+		echo \"$$HASH\" > micro-revision.tmp; \
+	fi; \
+	if test ! -f micro-revision.tmp; then \
+		if test ! -f micro-revision.i; then \
+			echo '""' > micro-revision.i; \
+		fi; \
+	elif test ! -f micro-revision.i || \
+			test x"`cat micro-revision.tmp`" != x"`cat micro-revision.i`"; then \
+		mv micro-revision.tmp micro-revision.i; \
+	fi; \
+	rm -f micro-revision.tmp; \
+	true
+
+CLEANFILES+= micro-revision.i micro-revision.tmp
+
+FORCE:
diff --git a/src/or/connection.c b/src/core/mainloop/connection.c
index 4f636eeb8c..3595bba85c 100644
--- a/src/or/connection.c
+++ b/src/core/mainloop/connection.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -34,10 +34,10 @@
  * they become able to read or write register the fact with the event main
  * loop by calling connection_watch_events(), connection_start_reading(), or
  * connection_start_writing().  When they no longer want to read or write,
- * they call connection_stop_reading() or connection_start_writing().
+ * they call connection_stop_reading() or connection_stop_writing().
  *
  * To queue data to be written on a connection, call
- * connection_write_to_buf().  When data arrives, the
+ * connection_buf_add().  When data arrives, the
  * connection_process_inbuf() callback is invoked, which dispatches to a
  * type-specific function (such as connection_edge_process_inbuf() for
  * example). Connection types that need notice of when data has been written
@@ -55,75 +55,125 @@
  **/
 
 #define CONNECTION_PRIVATE
-#include "or.h"
-#include "buffers.h"
+#include "core/or/or.h"
+#include "feature/client/bridges.h"
+#include "lib/container/buffers.h"
+#include "lib/tls/buffers_tls.h"
+#include "lib/err/backtrace.h"
+
 /*
  * Define this so we get channel internal functions, since we're implementing
  * part of a subclass (channel_tls_t).
  */
 #define TOR_CHANNEL_INTERNAL_
 #define CONNECTION_PRIVATE
-#include "backtrace.h"
-#include "channel.h"
-#include "channeltls.h"
-#include "circuitbuild.h"
-#include "circuitlist.h"
-#include "circuituse.h"
-#include "config.h"
-#include "connection.h"
-#include "connection_edge.h"
-#include "connection_or.h"
-#include "control.h"
-#include "directory.h"
-#include "dirserv.h"
-#include "dns.h"
-#include "dnsserv.h"
-#include "dos.h"
-#include "entrynodes.h"
-#include "ext_orport.h"
-#include "geoip.h"
-#include "main.h"
-#include "nodelist.h"
-#include "policies.h"
-#include "reasons.h"
-#include "relay.h"
-#include "rendclient.h"
-#include "rendcommon.h"
-#include "rephist.h"
-#include "router.h"
-#include "routerlist.h"
-#include "transports.h"
-#include "routerparse.h"
-#include "sandbox.h"
-#include "transports.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
+#include "core/or/channel.h"
+#include "core/or/channeltls.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "core/or/dos.h"
+#include "core/or/policies.h"
+#include "core/or/reasons.h"
+#include "core/or/relay.h"
+#include "core/proto/proto_http.h"
+#include "core/proto/proto_socks.h"
+#include "feature/client/dnsserv.h"
+#include "feature/client/entrynodes.h"
+#include "feature/client/transports.h"
+#include "feature/control/control.h"
+#include "feature/dirauth/authmode.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dircommon/directory.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_ident.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/dns.h"
+#include "feature/relay/ext_orport.h"
+#include "feature/relay/routermode.h"
+#include "feature/rend/rendclient.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/stats/rephist.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/geoip/geoip.h"
+
+#include "lib/sandbox/sandbox.h"
+#include "lib/net/buffers_net.h"
+#include "lib/tls/tortls.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/compress/compress.h"
 
 #ifdef HAVE_PWD_H
 #include <pwd.h>
 #endif
 
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
 #ifdef HAVE_SYS_UN_H
 #include <sys/socket.h>
 #include <sys/un.h>
 #endif
 
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/control/control_connection_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/listener_connection_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/port_cfg_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "core/or/socks_request_st.h"
+
+/**
+ * On Windows and Linux we cannot reliably bind() a socket to an
+ * address and port if: 1) There's already a socket bound to wildcard
+ * address (0.0.0.0 or ::) with the same port; 2) We try to bind()
+ * to wildcard address and there's another socket bound to a
+ * specific address and the same port.
+ *
+ * To address this problem on these two platforms we implement a
+ * routine that:
+ * 1) Checks if first attempt to bind() a new socket  failed with
+ * EADDRINUSE.
+ * 2) If so, it will close the appropriate old listener connection and
+ * 3) Attempts bind()'ing the new listener socket again.
+ *
+ * Just to be safe, we are enabling listener rebind code on all platforms,
+ * to account for unexpected cases where it may be needed.
+ */
+#define ENABLE_LISTENER_REBIND
+
 static connection_t *connection_listener_new(
                                const struct sockaddr *listensockaddr,
                                socklen_t listensocklen, int type,
                                const char *address,
-                               const port_cfg_t *portcfg);
+                               const port_cfg_t *portcfg,
+                               int *addr_in_use);
+static connection_t *connection_listener_new_for_port(
+                               const port_cfg_t *port,
+                               int *defer, int *addr_in_use);
 static void connection_init(time_t now, connection_t *conn, int type,
                             int socket_family);
-static int connection_init_accepted_conn(connection_t *conn,
-                          const listener_connection_t *listener);
 static int connection_handle_listener_read(connection_t *conn, int new_type);
-static int connection_bucket_should_increase(int bucket,
-                                             or_connection_t *conn);
 static int connection_finished_flushing(connection_t *conn);
 static int connection_flushed_some(connection_t *conn);
 static int connection_finished_connecting(connection_t *conn);
 static int connection_reached_eof(connection_t *conn);
-static int connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
-                                  int *socket_error);
+static int connection_buf_read_from_socket(connection_t *conn,
+                                           ssize_t *max_to_read,
+                                           int *socket_error);
 static int connection_process_inbuf(connection_t *conn, int package_partial);
 static void client_check_address_changed(tor_socket_t sock);
 static void set_constrained_socket_buffers(tor_socket_t sock, int size);
@@ -132,7 +182,11 @@ static const char *connection_proxy_state_to_string(int state);
 static int connection_read_https_proxy_response(connection_t *conn);
 static void connection_send_socks5_connect(connection_t *conn);
 static const char *proxy_type_to_string(int proxy_type);
-static int get_proxy_type(void);
+static int conn_get_proxy_type(const connection_t *conn);
+const tor_addr_t *conn_get_outbound_address(sa_family_t family,
+                  const or_options_t *options, unsigned int conn_type);
+static void reenable_blocked_connection_init(const or_options_t *options);
+static void reenable_blocked_connection_schedule(void);
 
 /** The last addresses that our network interface seemed to have been
  * binding to.  We use this as one way to detect when our IP changes.
@@ -154,10 +208,32 @@ static smartlist_t *outgoing_addrs = NULL;
     case CONN_TYPE_CONTROL_LISTENER: \
     case CONN_TYPE_AP_TRANS_LISTENER: \
     case CONN_TYPE_AP_NATD_LISTENER: \
-    case CONN_TYPE_AP_DNS_LISTENER
+    case CONN_TYPE_AP_DNS_LISTENER: \
+    case CONN_TYPE_AP_HTTP_CONNECT_LISTENER
 
 /**************************************************************/
 
+/** Convert a connection_t* to an listener_connection_t*; assert if the cast
+ * is invalid. */
+listener_connection_t *
+TO_LISTENER_CONN(connection_t *c)
+{
+  tor_assert(c->magic == LISTENER_CONNECTION_MAGIC);
+  return DOWNCAST(listener_connection_t, c);
+}
+
+size_t
+connection_get_inbuf_len(connection_t *conn)
+{
+  return conn->inbuf ? buf_datalen(conn->inbuf) : 0;
+}
+
+size_t
+connection_get_outbuf_len(connection_t *conn)
+{
+    return conn->outbuf ? buf_datalen(conn->outbuf) : 0;
+}
+
 /**
  * Return the human-readable name for the connection type <b>type</b>
  */
@@ -181,6 +257,7 @@ conn_type_to_string(int type)
     case CONN_TYPE_CONTROL: return "Control";
     case CONN_TYPE_EXT_OR: return "Extended OR";
     case CONN_TYPE_EXT_OR_LISTENER: return "Extended OR listener";
+    case CONN_TYPE_AP_HTTP_CONNECT_LISTENER: return "HTTP tunnel listener";
     default:
       log_warn(LD_BUG, "unknown connection type %d", type);
       tor_snprintf(buf, sizeof(buf), "unknown [%d]", type);
@@ -327,8 +404,6 @@ entry_connection_new(int type, int socket_family)
     entry_conn->entry_cfg.ipv4_traffic = 1;
   else if (socket_family == AF_INET6)
     entry_conn->entry_cfg.ipv6_traffic = 1;
-  else if (socket_family == AF_UNIX)
-    entry_conn->is_socks_socket = 1;
   return entry_conn;
 }
 
@@ -453,8 +528,8 @@ connection_init(time_t now, connection_t *conn, int type, int socket_family)
   }
 
   conn->timestamp_created = now;
-  conn->timestamp_lastread = now;
-  conn->timestamp_lastwritten = now;
+  conn->timestamp_last_read_allowed = now;
+  conn->timestamp_last_write_allowed = now;
 }
 
 /** Create a link between <b>conn_a</b> and <b>conn_b</b>. */
@@ -491,7 +566,7 @@ conn_listener_type_supports_af_unix(int type)
  * if <b>conn</b> is an OR or OP connection.
  */
 STATIC void
-connection_free_(connection_t *conn)
+connection_free_minimal(connection_t *conn)
 {
   void *mem;
   size_t memlen;
@@ -566,21 +641,34 @@ connection_free_(connection_t *conn)
 
   if (connection_speaks_cells(conn)) {
     or_connection_t *or_conn = TO_OR_CONN(conn);
-    tor_tls_free(or_conn->tls);
-    or_conn->tls = NULL;
+    if (or_conn->tls) {
+      if (! SOCKET_OK(conn->s)) {
+        /* The socket has been closed by somebody else; we must tell the
+         * TLS object not to close it. */
+        tor_tls_release_socket(or_conn->tls);
+      } else {
+        /* The tor_tls_free() call below will close the socket; we must tell
+         * the code below not to close it a second time. */
+        tor_release_socket_ownership(conn->s);
+        conn->s = TOR_INVALID_SOCKET;
+      }
+      tor_tls_free(or_conn->tls);
+      or_conn->tls = NULL;
+    }
     or_handshake_state_free(or_conn->handshake_state);
     or_conn->handshake_state = NULL;
     tor_free(or_conn->nickname);
     if (or_conn->chan) {
       /* Owww, this shouldn't happen, but... */
+      channel_t *base_chan = TLS_CHAN_TO_BASE(or_conn->chan);
+      tor_assert(base_chan);
       log_info(LD_CHANNEL,
                "Freeing orconn at %p, saw channel %p with ID "
-               U64_FORMAT " left un-NULLed",
-               or_conn, TLS_CHAN_TO_BASE(or_conn->chan),
-               U64_PRINTF_ARG(
-                 TLS_CHAN_TO_BASE(or_conn->chan)->global_identifier));
-      if (!CHANNEL_FINISHED(TLS_CHAN_TO_BASE(or_conn->chan))) {
-        channel_close_for_error(TLS_CHAN_TO_BASE(or_conn->chan));
+               "%"PRIu64 " left un-NULLed",
+               or_conn, base_chan,
+               base_chan->global_identifier);
+      if (!CHANNEL_FINISHED(base_chan)) {
+        channel_close_for_error(base_chan);
       }
 
       or_conn->chan->conn = NULL;
@@ -602,6 +690,7 @@ connection_free_(connection_t *conn)
   }
   if (CONN_IS_EDGE(conn)) {
     rend_data_free(TO_EDGE_CONN(conn)->rend_data);
+    hs_ident_edge_conn_free(TO_EDGE_CONN(conn)->hs_ident);
   }
   if (conn->type == CONN_TYPE_CONTROL) {
     control_connection_t *control_conn = TO_CONTROL_CONN(conn);
@@ -625,14 +714,20 @@ connection_free_(connection_t *conn)
     dir_connection_t *dir_conn = TO_DIR_CONN(conn);
     tor_free(dir_conn->requested_resource);
 
-    tor_zlib_free(dir_conn->zlib_state);
-    if (dir_conn->fingerprint_stack) {
-      SMARTLIST_FOREACH(dir_conn->fingerprint_stack, char *, cp, tor_free(cp));
-      smartlist_free(dir_conn->fingerprint_stack);
+    tor_compress_free(dir_conn->compress_state);
+    if (dir_conn->spool) {
+      SMARTLIST_FOREACH(dir_conn->spool, spooled_resource_t *, spooled,
+                        spooled_resource_free(spooled));
+      smartlist_free(dir_conn->spool);
     }
 
-    cached_dir_decref(dir_conn->cached_dir);
     rend_data_free(dir_conn->rend_data);
+    hs_ident_dir_conn_free(dir_conn->hs_ident);
+    if (dir_conn->guard_state) {
+      /* Cancel before freeing, if it's still there. */
+      entry_guard_cancel(&dir_conn->guard_state);
+    }
+    circuit_guard_state_free(dir_conn->guard_state);
   }
 
   if (SOCKET_OK(conn->s)) {
@@ -644,7 +739,7 @@ connection_free_(connection_t *conn)
   if (conn->type == CONN_TYPE_OR &&
       !tor_digest_is_zero(TO_OR_CONN(conn)->identity_digest)) {
     log_warn(LD_BUG, "called on OR conn with non-zeroed identity_digest");
-    connection_or_remove_from_identity_map(TO_OR_CONN(conn));
+    connection_or_clear_identity(TO_OR_CONN(conn));
   }
   if (conn->type == CONN_TYPE_OR || conn->type == CONN_TYPE_EXT_OR) {
     connection_or_remove_from_ext_or_id_map(TO_OR_CONN(conn));
@@ -660,7 +755,7 @@ connection_free_(connection_t *conn)
 /** Make sure <b>conn</b> isn't in any of the global conn lists; then free it.
  */
 MOCK_IMPL(void,
-connection_free,(connection_t *conn))
+connection_free_,(connection_t *conn))
 {
   if (!conn)
     return;
@@ -675,7 +770,7 @@ connection_free,(connection_t *conn))
   }
   if (connection_speaks_cells(conn)) {
     if (!tor_digest_is_zero(TO_OR_CONN(conn)->identity_digest)) {
-      connection_or_remove_from_identity_map(TO_OR_CONN(conn));
+      connection_or_clear_identity(TO_OR_CONN(conn));
     }
   }
   if (conn->type == CONN_TYPE_CONTROL) {
@@ -687,7 +782,7 @@ connection_free,(connection_t *conn))
     connection_ap_warn_and_unmark_if_pending_circ(TO_ENTRY_CONN(conn),
                                                   "connection_free");
   }
-#endif
+#endif /* 1 */
 
   /* Notify the circuit creation DoS mitigation subsystem that an OR client
    * connection has been closed. And only do that if we track it. */
@@ -696,7 +791,7 @@ connection_free,(connection_t *conn))
   }
 
   connection_unregister_events(conn);
-  connection_free_(conn);
+  connection_free_minimal(conn);
 }
 
 /**
@@ -761,8 +856,8 @@ connection_close_immediate(connection_t *conn)
   connection_unregister_events(conn);
 
   /* Prevent the event from getting unblocked. */
-  conn->read_blocked_on_bw =
-    conn->write_blocked_on_bw = 0;
+  conn->read_blocked_on_bw = 0;
+  conn->write_blocked_on_bw = 0;
 
   if (SOCKET_OK(conn->s))
     tor_close_socket(conn->s);
@@ -802,13 +897,19 @@ connection_mark_for_close_(connection_t *conn, int line, const char *file)
 }
 
 /** Mark <b>conn</b> to be closed next time we loop through
- * conn_close_if_marked() in main.c; the _internal version bypasses the
- * CONN_TYPE_OR checks; this should be called when you either are sure that
- * if this is an or_connection_t the controlling channel has been notified
- * (e.g. with connection_or_notify_error()), or you actually are the
- * connection_or_close_for_error() or connection_or_close_normally function.
- * For all other cases, use connection_mark_and_flush() instead, which
- * checks for or_connection_t properly, instead.  See below.
+ * conn_close_if_marked() in main.c.
+ *
+ * This _internal version bypasses the CONN_TYPE_OR checks; this should be
+ * called when you either are sure that if this is an or_connection_t the
+ * controlling channel has been notified (e.g. with
+ * connection_or_notify_error()), or you actually are the
+ * connection_or_close_for_error() or connection_or_close_normally() function.
+ * For all other cases, use connection_mark_and_flush() which checks for
+ * or_connection_t properly, instead.  See below.
+ *
+ * We want to keep this function simple and quick, since it can be called from
+ * quite deep in the call chain, and hence it should avoid having side-effects
+ * that interfere with its callers view of the connection.
  */
 MOCK_IMPL(void,
 connection_mark_for_close_internal_, (connection_t *conn,
@@ -845,7 +946,7 @@ connection_mark_for_close_internal_, (connection_t *conn,
   /* in case we're going to be held-open-til-flushed, reset
    * the number of seconds since last successful write, so
    * we get our whole 15 seconds */
-  conn->timestamp_lastwritten = time(NULL);
+  conn->timestamp_last_write_allowed = time(NULL);
 }
 
 /** Find each connection that has hold_open_until_flushed set to
@@ -867,7 +968,7 @@ connection_expire_held_open(void)
      */
     if (conn->hold_open_until_flushed) {
       tor_assert(conn->marked_for_close);
-      if (now - conn->timestamp_lastwritten >= 15) {
+      if (now - conn->timestamp_last_write_allowed >= 15) {
         int severity;
         if (conn->type == CONN_TYPE_EXIT ||
             (conn->type == CONN_TYPE_DIR &&
@@ -922,7 +1023,7 @@ create_unix_sockaddr(const char *listenaddress, char **readable_address,
   *len_out = sizeof(struct sockaddr_un);
   return sockaddr;
 }
-#else
+#else /* !(defined(HAVE_SYS_UN_H) || defined(RUNNING_DOXYGEN)) */
 static struct sockaddr *
 create_unix_sockaddr(const char *listenaddress, char **readable_address,
                      socklen_t *len_out)
@@ -935,7 +1036,7 @@ create_unix_sockaddr(const char *listenaddress, char **readable_address,
   tor_fragile_assert();
   return NULL;
 }
-#endif /* HAVE_SYS_UN_H */
+#endif /* defined(HAVE_SYS_UN_H) || defined(RUNNING_DOXYGEN) */
 
 /** Warn that an accept or a connect has failed because we're running out of
  * TCP sockets we can use on current system.  Rate-limit these warnings so
@@ -1050,7 +1151,7 @@ check_location_for_unix_socket(const or_options_t *options, const char *path,
   tor_free(p);
   return r;
 }
-#endif
+#endif /* defined(HAVE_SYS_UN_H) */
 
 /** Tell the TCP stack that it shouldn't wait for a long time after
  * <b>sock</b> has closed before reusing its port. Return 0 on success,
@@ -1073,7 +1174,7 @@ make_socket_reuseable(tor_socket_t sock)
     return -1;
   }
   return 0;
-#endif
+#endif /* defined(_WIN32) */
 }
 
 #ifdef _WIN32
@@ -1094,12 +1195,12 @@ make_win32_socket_exclusive(tor_socket_t sock)
     return -1;
   }
   return 0;
-#else
+#else /* !(defined(SO_EXCLUSIVEADDRUSE)) */
   (void) sock;
   return 0;
-#endif
+#endif /* defined(SO_EXCLUSIVEADDRUSE) */
 }
-#endif
+#endif /* defined(_WIN32) */
 
 /** Max backlog to pass to listen.  We start at */
 static int listen_limit = INT_MAX;
@@ -1127,12 +1228,16 @@ tor_listen(tor_socket_t fd)
  *
  * <b>address</b> is only used for logging purposes and to add the information
  * to the conn.
+ *
+ * Set <b>addr_in_use</b> to true in case socket binding fails with
+ * EADDRINUSE.
  */
 static connection_t *
 connection_listener_new(const struct sockaddr *listensockaddr,
                         socklen_t socklen,
                         int type, const char *address,
-                        const port_cfg_t *port_cfg)
+                        const port_cfg_t *port_cfg,
+                        int *addr_in_use)
 {
   listener_connection_t *lis_conn;
   connection_t *conn = NULL;
@@ -1148,6 +1253,9 @@ connection_listener_new(const struct sockaddr *listensockaddr,
   tor_addr_t addr;
   int exhaustion = 0;
 
+  if (addr_in_use)
+    *addr_in_use = 0;
+
   if (listensockaddr->sa_family == AF_INET ||
       listensockaddr->sa_family == AF_INET6) {
     int is_stream = (type != CONN_TYPE_AP_DNS_LISTENER);
@@ -1189,7 +1297,7 @@ connection_listener_new(const struct sockaddr *listensockaddr,
                conn_type_to_string(type),
                tor_socket_strerror(errno));
     }
-#endif
+#endif /* defined(_WIN32) */
 
 #if defined(USE_TRANSPARENT) && defined(IP_TRANSPARENT)
     if (options->TransProxyType_parsed == TPT_TPROXY &&
@@ -1206,7 +1314,7 @@ connection_listener_new(const struct sockaddr *listensockaddr,
                  tor_socket_strerror(e), extra);
       }
     }
-#endif
+#endif /* defined(USE_TRANSPARENT) && defined(IP_TRANSPARENT) */
 
 #ifdef IPV6_V6ONLY
     if (listensockaddr->sa_family == AF_INET6) {
@@ -1221,13 +1329,16 @@ connection_listener_new(const struct sockaddr *listensockaddr,
         /* Keep going; probably not harmful. */
       }
     }
-#endif
+#endif /* defined(IPV6_V6ONLY) */
 
     if (bind(s,listensockaddr,socklen) < 0) {
       const char *helpfulhint = "";
       int e = tor_socket_errno(s);
-      if (ERRNO_IS_EADDRINUSE(e))
+      if (ERRNO_IS_EADDRINUSE(e)) {
         helpfulhint = ". Is Tor already running?";
+        if (addr_in_use)
+          *addr_in_use = 1;
+      }
       log_warn(LD_NET, "Could not bind to %s:%u: %s%s", address, usePort,
                tor_socket_strerror(e), helpfulhint);
       goto err;
@@ -1324,7 +1435,7 @@ connection_listener_new(const struct sockaddr *listensockaddr,
         goto err;
       }
     }
-#endif
+#endif /* defined(HAVE_PWD_H) */
 
     {
       unsigned mode;
@@ -1355,7 +1466,7 @@ connection_listener_new(const struct sockaddr *listensockaddr,
                tor_socket_strerror(tor_socket_errno(s)));
       goto err;
     }
-#endif /* HAVE_SYS_UN_H */
+#endif /* defined(HAVE_SYS_UN_H) */
   } else {
     log_err(LD_BUG, "Got unexpected address family %d.",
             listensockaddr->sa_family);
@@ -1417,6 +1528,13 @@ connection_listener_new(const struct sockaddr *listensockaddr,
    */
   connection_check_oos(get_n_open_sockets(), 0);
 
+  if (conn->socket_family == AF_UNIX) {
+    log_notice(LD_NET, "Opened %s on %s",
+               conn_type_to_string(type), conn->address);
+  } else {
+    log_notice(LD_NET, "Opened %s on %s",
+               conn_type_to_string(type), fmt_addrport(&addr, gotPort));
+  }
   return conn;
 
  err:
@@ -1431,6 +1549,70 @@ connection_listener_new(const struct sockaddr *listensockaddr,
   return NULL;
 }
 
+/**
+ * Create a new listener connection for a given <b>port</b>. In case we
+ * for a reason that is not an error condition, set <b>defer</b>
+ * to true. If we cannot bind listening socket because address is already
+ * in use, set <b>addr_in_use</b> to true.
+ */
+static connection_t *
+connection_listener_new_for_port(const port_cfg_t *port,
+                                 int *defer, int *addr_in_use)
+{
+  connection_t *conn;
+  struct sockaddr *listensockaddr;
+  socklen_t listensocklen = 0;
+  char *address=NULL;
+  int real_port = port->port == CFG_AUTO_PORT ? 0 : port->port;
+  tor_assert(real_port <= UINT16_MAX);
+
+  if (defer)
+    *defer = 0;
+
+  if (port->server_cfg.no_listen) {
+    if (defer)
+      *defer = 1;
+    return NULL;
+  }
+
+#ifndef _WIN32
+  /* We don't need to be root to create a UNIX socket, so defer until after
+   * setuid. */
+  const or_options_t *options = get_options();
+  if (port->is_unix_addr && !geteuid() && (options->User) &&
+      strcmp(options->User, "root")) {
+    if (defer)
+      *defer = 1;
+    return NULL;
+  }
+#endif /* !defined(_WIN32) */
+
+  if (port->is_unix_addr) {
+    listensockaddr = (struct sockaddr *)
+      create_unix_sockaddr(port->unix_addr,
+                           &address, &listensocklen);
+  } else {
+    listensockaddr = tor_malloc(sizeof(struct sockaddr_storage));
+    listensocklen = tor_addr_to_sockaddr(&port->addr,
+                                         real_port,
+                                         listensockaddr,
+                                         sizeof(struct sockaddr_storage));
+    address = tor_addr_to_str_dup(&port->addr);
+  }
+
+  if (listensockaddr) {
+    conn = connection_listener_new(listensockaddr, listensocklen,
+                                   port->type, address, port,
+                                   addr_in_use);
+    tor_free(listensockaddr);
+    tor_free(address);
+  } else {
+    conn = NULL;
+  }
+
+  return conn;
+}
+
 /** Do basic sanity checking on a newly received socket. Return 0
  * if it looks ok, else return -1.
  *
@@ -1664,11 +1846,15 @@ connection_handle_listener_read(connection_t *conn, int new_type)
 }
 
 /** Initialize states for newly accepted connection <b>conn</b>.
+ *
  * If conn is an OR, start the TLS handshake.
+ *
  * If conn is a transparent AP, get its original destination
  * and place it in circuit_wait.
+ *
+ * The <b>listener</b> parameter is only used for AP connections.
  */
-static int
+int
 connection_init_accepted_conn(connection_t *conn,
                               const listener_connection_t *listener)
 {
@@ -1710,6 +1896,8 @@ connection_init_accepted_conn(connection_t *conn,
           TO_ENTRY_CONN(conn)->is_transparent_ap = 1;
           conn->state = AP_CONN_STATE_NATD_WAIT;
           break;
+        case CONN_TYPE_AP_HTTP_CONNECT_LISTENER:
+          conn->state = AP_CONN_STATE_HTTP_CONNECT_WAIT;
       }
       break;
     case CONN_TYPE_DIR:
@@ -1744,9 +1932,13 @@ connection_connect_sockaddr,(connection_t *conn,
   tor_assert(sa);
   tor_assert(socket_error);
 
-  if (get_options()->DisableNetwork) {
-    /* We should never even try to connect anyplace if DisableNetwork is set.
-     * Warn if we do, and refuse to make the connection. */
+  if (net_is_completely_disabled()) {
+    /* We should never even try to connect anyplace if the network is
+     * completely shut off.
+     *
+     * (We don't check net_is_disabled() here, since we still sometimes
+     * want to open connections when we're in soft hibernation.)
+     */
     static ratelim_t disablenet_violated = RATELIM_INIT(30*60);
     *socket_error = SOCK_ERRNO(ENETUNREACH);
     log_fn_ratelim(&disablenet_violated, LOG_WARN, LD_BUG,
@@ -1784,7 +1976,7 @@ connection_connect_sockaddr,(connection_t *conn,
 
   /*
    * We've got the socket open; give the OOS handler a chance to check
-   * against configuured maximum socket number, but tell it no exhaustion
+   * against configured maximum socket number, but tell it no exhaustion
    * failure.
    */
   connection_check_oos(get_n_open_sockets(), 0);
@@ -1903,6 +2095,55 @@ connection_connect_log_client_use_ip_version(const connection_t *conn)
   }
 }
 
+/** Retrieve the outbound address depending on the protocol (IPv4 or IPv6)
+ * and the connection type (relay, exit, ...)
+ * Return a socket address or NULL in case nothing is configured.
+ **/
+const tor_addr_t *
+conn_get_outbound_address(sa_family_t family,
+             const or_options_t *options, unsigned int conn_type)
+{
+  const tor_addr_t *ext_addr = NULL;
+
+  int fam_index;
+  switch (family) {
+    case AF_INET:
+      fam_index = 0;
+      break;
+    case AF_INET6:
+      fam_index = 1;
+      break;
+    default:
+      return NULL;
+  }
+
+  // If an exit connection, use the exit address (if present)
+  if (conn_type == CONN_TYPE_EXIT) {
+    if (!tor_addr_is_null(
+        &options->OutboundBindAddresses[OUTBOUND_ADDR_EXIT][fam_index])) {
+      ext_addr = &options->OutboundBindAddresses[OUTBOUND_ADDR_EXIT]
+                 [fam_index];
+    } else if (!tor_addr_is_null(
+                 &options->OutboundBindAddresses[OUTBOUND_ADDR_EXIT_AND_OR]
+                 [fam_index])) {
+      ext_addr = &options->OutboundBindAddresses[OUTBOUND_ADDR_EXIT_AND_OR]
+                 [fam_index];
+    }
+  } else { // All non-exit connections
+    if (!tor_addr_is_null(
+           &options->OutboundBindAddresses[OUTBOUND_ADDR_OR][fam_index])) {
+      ext_addr = &options->OutboundBindAddresses[OUTBOUND_ADDR_OR]
+                 [fam_index];
+    } else if (!tor_addr_is_null(
+                 &options->OutboundBindAddresses[OUTBOUND_ADDR_EXIT_AND_OR]
+                 [fam_index])) {
+      ext_addr = &options->OutboundBindAddresses[OUTBOUND_ADDR_EXIT_AND_OR]
+                 [fam_index];
+    }
+  }
+  return ext_addr;
+}
+
 /** Take conn, make a nonblocking socket; try to connect to
  * addr:port (port arrives in *host order*). If fail, return -1 and if
  * applicable put your best guess about errno into *<b>socket_error</b>.
@@ -1924,26 +2165,15 @@ connection_connect(connection_t *conn, const char *address,
   struct sockaddr *bind_addr = NULL;
   struct sockaddr *dest_addr;
   int dest_addr_len, bind_addr_len = 0;
-  const or_options_t *options = get_options();
-  int protocol_family;
 
   /* Log if we didn't stick to ClientUseIPv4/6 or ClientPreferIPv6OR/DirPort
    */
   connection_connect_log_client_use_ip_version(conn);
 
-  if (tor_addr_family(addr) == AF_INET6)
-    protocol_family = PF_INET6;
-  else
-    protocol_family = PF_INET;
-
   if (!tor_addr_is_loopback(addr)) {
     const tor_addr_t *ext_addr = NULL;
-    if (protocol_family == AF_INET &&
-        !tor_addr_is_null(&options->OutboundBindAddressIPv4_))
-      ext_addr = &options->OutboundBindAddressIPv4_;
-    else if (protocol_family == AF_INET6 &&
-             !tor_addr_is_null(&options->OutboundBindAddressIPv6_))
-      ext_addr = &options->OutboundBindAddressIPv6_;
+    ext_addr = conn_get_outbound_address(tor_addr_family(addr), get_options(),
+                                         conn->type);
     if (ext_addr) {
       memset(&bind_addr_ss, 0, sizeof(bind_addr_ss));
       bind_addr_len = tor_addr_to_sockaddr(ext_addr, 0,
@@ -2036,18 +2266,27 @@ connection_proxy_state_to_string(int state)
   return states[state];
 }
 
-/** Returns the global proxy type used by tor. Use this function for
- *  logging or high-level purposes, don't use it to fill the
+/** Returns the proxy type used by tor for a single connection, for
+ *  logging or high-level purposes. Don't use it to fill the
  *  <b>proxy_type</b> field of or_connection_t; use the actual proxy
  *  protocol instead.*/
 static int
-get_proxy_type(void)
+conn_get_proxy_type(const connection_t *conn)
 {
   const or_options_t *options = get_options();
 
-  if (options->ClientTransportPlugin)
-    return PROXY_PLUGGABLE;
-  else if (options->HTTPSProxy)
+  if (options->ClientTransportPlugin) {
+    /* If we have plugins configured *and* this addr/port is a known bridge
+     * with a transport, then we should be PROXY_PLUGGABLE. */
+    const transport_t *transport = NULL;
+    int r;
+    r = get_transport_by_bridge_addrport(&conn->addr, conn->port, &transport);
+    if (r == 0 && transport)
+      return PROXY_PLUGGABLE;
+  }
+
+  /* In all other cases, we're using a global proxy. */
+  if (options->HTTPSProxy)
     return PROXY_CONNECT;
   else if (options->Socks4Proxy)
     return PROXY_SOCKS4;
@@ -2110,7 +2349,7 @@ connection_proxy_connect(connection_t *conn, int type)
                      fmt_addrport(&conn->addr, conn->port));
       }
 
-      connection_write_to_buf(buf, strlen(buf), conn);
+      connection_buf_add(buf, strlen(buf), conn);
       conn->proxy_state = PROXY_HTTPS_WANT_CONNECT_OK;
       break;
     }
@@ -2134,7 +2373,7 @@ connection_proxy_connect(connection_t *conn, int type)
            arguments to transmit. If we do, compress all arguments to
            a single string in 'socks_args_string': */
 
-        if (get_proxy_type() == PROXY_PLUGGABLE) {
+        if (conn_get_proxy_type(conn) == PROXY_PLUGGABLE) {
           socks_args_string =
             pt_get_socks_args_for_proxy_addrport(&conn->addr, conn->port);
           if (socks_args_string)
@@ -2176,7 +2415,7 @@ connection_proxy_connect(connection_t *conn, int type)
         buf[8] = 0; /* no userid */
       }
 
-      connection_write_to_buf((char *)buf, buf_size, conn);
+      connection_buf_add((char *)buf, buf_size, conn);
       tor_free(buf);
 
       conn->proxy_state = PROXY_SOCKS4_WANT_CONNECT_OK;
@@ -2194,7 +2433,7 @@ connection_proxy_connect(connection_t *conn, int type)
          Socks5ProxyUsername or if we want to pass arguments to our
          pluggable transport proxy: */
       if ((options->Socks5ProxyUsername) ||
-          (get_proxy_type() == PROXY_PLUGGABLE &&
+          (conn_get_proxy_type(conn) == PROXY_PLUGGABLE &&
            (get_socks_args_by_bridge_addrport(&conn->addr, conn->port)))) {
       /* number of auth methods */
         buf[1] = 2;
@@ -2207,7 +2446,7 @@ connection_proxy_connect(connection_t *conn, int type)
         conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_NONE;
       }
 
-      connection_write_to_buf((char *)buf, 2 + buf[1], conn);
+      connection_buf_add((char *)buf, 2 + buf[1], conn);
       break;
     }
 
@@ -2313,7 +2552,7 @@ connection_send_socks5_connect(connection_t *conn)
     memcpy(buf + 20, &port, 2);
   }
 
-  connection_write_to_buf((char *)buf, reqsize, conn);
+  connection_buf_add((char *)buf, reqsize, conn);
 
   conn->proxy_state = PROXY_SOCKS5_WANT_CONNECT_OK;
 }
@@ -2387,16 +2626,16 @@ connection_read_proxy_handshake(connection_t *conn)
         const char *user, *pass;
         char *socks_args_string = NULL;
 
-        if (get_proxy_type() == PROXY_PLUGGABLE) {
+        if (conn_get_proxy_type(conn) == PROXY_PLUGGABLE) {
           socks_args_string =
             pt_get_socks_args_for_proxy_addrport(&conn->addr, conn->port);
           if (!socks_args_string) {
-            log_warn(LD_NET, "Could not create SOCKS args string.");
+            log_warn(LD_NET, "Could not create SOCKS args string for PT.");
             ret = -1;
             break;
           }
 
-          log_debug(LD_NET, "SOCKS5 arguments: %s", socks_args_string);
+          log_debug(LD_NET, "PT SOCKS5 arguments: %s", socks_args_string);
           tor_assert(strlen(socks_args_string) > 0);
           tor_assert(strlen(socks_args_string) <= MAX_SOCKS5_AUTH_SIZE_TOTAL);
 
@@ -2439,7 +2678,7 @@ connection_read_proxy_handshake(connection_t *conn)
         if (socks_args_string)
           tor_free(socks_args_string);
 
-        connection_write_to_buf((char *)buf, reqsize, conn);
+        connection_buf_add((char *)buf, reqsize, conn);
 
         conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_RFC1929_OK;
         ret = 0;
@@ -2499,10 +2738,13 @@ connection_read_proxy_handshake(connection_t *conn)
  *
  * Remove from <b>old_conns</b> every connection that has a corresponding
  * entry in <b>ports</b>.  Add to <b>new_conns</b> new every connection we
- * launch.
+ * launch. If we may need to perform socket rebind when creating new
+ * listener that replaces old one, create a <b>listener_replacement_t</b>
+ * struct for affected pair  and add it to <b>replacements</b>.
  *
  * If <b>control_listeners_only</b> is true, then we only open control
- * listeners, and we do not remove any noncontrol listeners from old_conns.
+ * listeners, and we do not remove any noncontrol listeners from
+ * old_conns.
  *
  * Return 0 on success, -1 on failure.
  **/
@@ -2510,8 +2752,13 @@ static int
 retry_listener_ports(smartlist_t *old_conns,
                      const smartlist_t *ports,
                      smartlist_t *new_conns,
+                     smartlist_t *replacements,
                      int control_listeners_only)
 {
+#ifndef ENABLE_LISTENER_REBIND
+  (void)replacements;
+#endif
+
   smartlist_t *launch = smartlist_new();
   int r = 0;
 
@@ -2530,6 +2777,11 @@ retry_listener_ports(smartlist_t *old_conns,
     const port_cfg_t *found_port = NULL;
 
     /* Okay, so this is a listener.  Is it configured? */
+    /* That is, is it either: 1) exact match - address and port
+     * pair match exactly between old listener and new port; or 2)
+     * wildcard match - port matches exactly, but *one* of the
+     * addresses is wildcard (0.0.0.0 or ::)?
+     */
     SMARTLIST_FOREACH_BEGIN(launch, const port_cfg_t *, wanted) {
       if (conn->type != wanted->type)
         continue;
@@ -2547,16 +2799,45 @@ retry_listener_ports(smartlist_t *old_conns,
           break;
         }
       } else {
-        int port_matches;
-        if (wanted->port == CFG_AUTO_PORT) {
-          port_matches = 1;
-        } else {
-          port_matches = (wanted->port == conn->port);
-        }
+        /* Numeric values of old and new port match exactly. */
+        const int port_matches_exact = (wanted->port == conn->port);
+        /* Ports match semantically - either their specific values
+           match exactly, or new port is 'auto'.
+         */
+        const int port_matches = (wanted->port == CFG_AUTO_PORT ||
+                                  port_matches_exact);
+
         if (port_matches && tor_addr_eq(&wanted->addr, &conn->addr)) {
           found_port = wanted;
           break;
         }
+#ifdef ENABLE_LISTENER_REBIND
+        /* Rebinding may be needed if all of the following are true:
+         * 1) Address family is the same in old and new listeners.
+         * 2) Port number matches exactly (numeric value is the same).
+         * 3) *One* of listeners (either old one or new one) has a
+         *    wildcard IP address (0.0.0.0 or [::]).
+         *
+         * These are the exact conditions for a first bind() syscall
+         * to fail with EADDRINUSE.
+         */
+        const int may_need_rebind =
+          tor_addr_family(&wanted->addr) == tor_addr_family(&conn->addr) &&
+          port_matches_exact && bool_neq(tor_addr_is_null(&wanted->addr),
+                                         tor_addr_is_null(&conn->addr));
+        if (replacements && may_need_rebind) {
+          listener_replacement_t *replacement =
+            tor_malloc(sizeof(listener_replacement_t));
+
+          replacement->old_conn = conn;
+          replacement->new_port = wanted;
+          smartlist_add(replacements, replacement);
+
+          SMARTLIST_DEL_CURRENT(launch, wanted);
+          SMARTLIST_DEL_CURRENT(old_conns, conn);
+          break;
+        }
+#endif
       }
     } SMARTLIST_FOREACH_END(wanted);
 
@@ -2572,52 +2853,13 @@ retry_listener_ports(smartlist_t *old_conns,
 
   /* Now open all the listeners that are configured but not opened. */
   SMARTLIST_FOREACH_BEGIN(launch, const port_cfg_t *, port) {
-    struct sockaddr *listensockaddr;
-    socklen_t listensocklen = 0;
-    char *address=NULL;
-    connection_t *conn;
-    int real_port = port->port == CFG_AUTO_PORT ? 0 : port->port;
-    tor_assert(real_port <= UINT16_MAX);
-    if (port->server_cfg.no_listen)
-      continue;
-
-#ifndef _WIN32
-    /* We don't need to be root to create a UNIX socket, so defer until after
-     * setuid. */
-    const or_options_t *options = get_options();
-    if (port->is_unix_addr && !geteuid() && (options->User) &&
-        strcmp(options->User, "root"))
-      continue;
-#endif
-
-    if (port->is_unix_addr) {
-      listensockaddr = (struct sockaddr *)
-        create_unix_sockaddr(port->unix_addr,
-                             &address, &listensocklen);
-    } else {
-      listensockaddr = tor_malloc(sizeof(struct sockaddr_storage));
-      listensocklen = tor_addr_to_sockaddr(&port->addr,
-                                           real_port,
-                                           listensockaddr,
-                                           sizeof(struct sockaddr_storage));
-      address = tor_addr_to_str_dup(&port->addr);
-    }
-
-    if (listensockaddr) {
-      conn = connection_listener_new(listensockaddr, listensocklen,
-                                     port->type, address, port);
-      tor_free(listensockaddr);
-      tor_free(address);
-    } else {
-      conn = NULL;
-    }
+    int skip = 0;
+    connection_t *conn = connection_listener_new_for_port(port, &skip, NULL);
 
-    if (!conn) {
+    if (conn && new_conns)
+      smartlist_add(new_conns, conn);
+    else if (!skip)
       r = -1;
-    } else {
-      if (new_conns)
-        smartlist_add(new_conns, conn);
-    }
   } SMARTLIST_FOREACH_END(port);
 
   smartlist_free(launch);
@@ -2629,17 +2871,16 @@ retry_listener_ports(smartlist_t *old_conns,
  * listeners who are not already open, and only close listeners we no longer
  * want.
  *
- * Add all old conns that should be closed to <b>replaced_conns</b>.
  * Add all new connections to <b>new_conns</b>.
  *
  * If <b>close_all_noncontrol</b> is true, then we only open control
  * listeners, and we close all other listeners.
  */
 int
-retry_all_listeners(smartlist_t *replaced_conns,
-                    smartlist_t *new_conns, int close_all_noncontrol)
+retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol)
 {
   smartlist_t *listeners = smartlist_new();
+  smartlist_t *replacements = smartlist_new();
   const or_options_t *options = get_options();
   int retval = 0;
   const uint16_t old_or_port = router_get_advertised_or_port(options);
@@ -2655,23 +2896,57 @@ retry_all_listeners(smartlist_t *replaced_conns,
   if (retry_listener_ports(listeners,
                            get_configured_ports(),
                            new_conns,
+                           replacements,
                            close_all_noncontrol) < 0)
     retval = -1;
 
+#ifdef ENABLE_LISTENER_REBIND
+  SMARTLIST_FOREACH_BEGIN(replacements, listener_replacement_t *, r) {
+    int addr_in_use = 0;
+    int skip = 0;
+
+    tor_assert(r->new_port);
+    tor_assert(r->old_conn);
+
+    connection_t *new_conn =
+      connection_listener_new_for_port(r->new_port, &skip, &addr_in_use);
+    connection_t *old_conn = r->old_conn;
+
+    if (skip)
+      continue;
+
+    connection_close_immediate(old_conn);
+    connection_mark_for_close(old_conn);
+
+    if (addr_in_use) {
+      new_conn = connection_listener_new_for_port(r->new_port,
+                                                  &skip, &addr_in_use);
+    }
+
+    tor_assert(new_conn);
+
+    smartlist_add(new_conns, new_conn);
+
+    log_notice(LD_NET, "Closed no-longer-configured %s on %s:%d "
+                       "(replaced by %s:%d)",
+               conn_type_to_string(old_conn->type), old_conn->address,
+               old_conn->port, new_conn->address, new_conn->port);
+  } SMARTLIST_FOREACH_END(r);
+#endif
+
   /* Any members that were still in 'listeners' don't correspond to
    * any configured port.  Kill 'em. */
   SMARTLIST_FOREACH_BEGIN(listeners, connection_t *, conn) {
     log_notice(LD_NET, "Closing no-longer-configured %s on %s:%d",
                conn_type_to_string(conn->type), conn->address, conn->port);
-    if (replaced_conns) {
-      smartlist_add(replaced_conns, conn);
-    } else {
-      connection_close_immediate(conn);
-      connection_mark_for_close(conn);
-    }
+    connection_close_immediate(conn);
+    connection_mark_for_close(conn);
   } SMARTLIST_FOREACH_END(conn);
 
   smartlist_free(listeners);
+  /* Cleanup any remaining listener replacement. */
+  SMARTLIST_FOREACH(replacements, listener_replacement_t *, r, tor_free(r));
+  smartlist_free(replacements);
 
   if (old_or_port != router_get_advertised_or_port(options) ||
       old_or_port_ipv6 != router_get_advertised_or_port_by_af(options,
@@ -2758,10 +3033,10 @@ connection_is_rate_limited(connection_t *conn)
     return 1;
 }
 
-/** Did either global write bucket run dry last second? If so,
- * we are likely to run dry again this second, so be stingy with the
- * tokens we just put in. */
-static int write_buckets_empty_last_second = 0;
+/** When was either global write bucket last empty? If this was recent, then
+ * we're probably low on bandwidth, and we should be stingy with our bandwidth
+ * usage. */
+static time_t write_buckets_last_empty_at = -100;
 
 /** How many seconds of no active local circuits will make the
  * connection revert to the "relayed" bandwidth class? */
@@ -2789,25 +3064,25 @@ connection_counts_as_relayed_traffic(connection_t *conn, time_t now)
  * write many of them or just a few; and <b>conn_bucket</b> (if
  * non-negative) provides an upper limit for our answer. */
 static ssize_t
-connection_bucket_round_robin(int base, int priority,
-                              ssize_t global_bucket, ssize_t conn_bucket)
+connection_bucket_get_share(int base, int priority,
+                            ssize_t global_bucket_val, ssize_t conn_bucket)
 {
   ssize_t at_most;
   ssize_t num_bytes_high = (priority ? 32 : 16) * base;
   ssize_t num_bytes_low = (priority ? 4 : 2) * base;
 
-  /* Do a rudimentary round-robin so one circuit can't hog a connection.
+  /* Do a rudimentary limiting so one circuit can't hog a connection.
    * Pick at most 32 cells, at least 4 cells if possible, and if we're in
    * the middle pick 1/8 of the available bandwidth. */
-  at_most = global_bucket / 8;
+  at_most = global_bucket_val / 8;
   at_most -= (at_most % base); /* round down */
   if (at_most > num_bytes_high) /* 16 KB, or 8 KB for low-priority */
     at_most = num_bytes_high;
   else if (at_most < num_bytes_low) /* 2 KB, or 1 KB for low-priority */
     at_most = num_bytes_low;
 
-  if (at_most > global_bucket)
-    at_most = global_bucket;
+  if (at_most > global_bucket_val)
+    at_most = global_bucket_val;
 
   if (conn_bucket >= 0 && at_most > conn_bucket)
     at_most = conn_bucket;
@@ -2823,13 +3098,13 @@ connection_bucket_read_limit(connection_t *conn, time_t now)
 {
   int base = RELAY_PAYLOAD_SIZE;
   int priority = conn->type != CONN_TYPE_DIR;
-  int conn_bucket = -1;
-  int global_bucket = global_read_bucket;
+  ssize_t conn_bucket = -1;
+  size_t global_bucket_val = token_bucket_rw_get_read(&global_bucket);
 
   if (connection_speaks_cells(conn)) {
     or_connection_t *or_conn = TO_OR_CONN(conn);
     if (conn->state == OR_CONN_STATE_OPEN)
-      conn_bucket = or_conn->read_bucket;
+      conn_bucket = token_bucket_rw_get_read(&or_conn->bucket);
     base = get_cell_network_size(or_conn->wide_circ_ids);
   }
 
@@ -2838,12 +3113,13 @@ connection_bucket_read_limit(connection_t *conn, time_t now)
     return conn_bucket>=0 ? conn_bucket : 1<<14;
   }
 
-  if (connection_counts_as_relayed_traffic(conn, now) &&
-      global_relayed_read_bucket <= global_read_bucket)
-    global_bucket = global_relayed_read_bucket;
+  if (connection_counts_as_relayed_traffic(conn, now)) {
+    size_t relayed = token_bucket_rw_get_read(&global_relayed_bucket);
+    global_bucket_val = MIN(global_bucket_val, relayed);
+  }
 
-  return connection_bucket_round_robin(base, priority,
-                                       global_bucket, conn_bucket);
+  return connection_bucket_get_share(base, priority,
+                                     global_bucket_val, conn_bucket);
 }
 
 /** How many bytes at most can we write onto this connection? */
@@ -2852,8 +3128,8 @@ connection_bucket_write_limit(connection_t *conn, time_t now)
 {
   int base = RELAY_PAYLOAD_SIZE;
   int priority = conn->type != CONN_TYPE_DIR;
-  int conn_bucket = (int)conn->outbuf_flushlen;
-  int global_bucket = global_write_bucket;
+  size_t conn_bucket = conn->outbuf_flushlen;
+  size_t global_bucket_val = token_bucket_rw_get_write(&global_bucket);
 
   if (!connection_is_rate_limited(conn)) {
     /* be willing to write to local conns even if our buckets are empty */
@@ -2861,22 +3137,21 @@ connection_bucket_write_limit(connection_t *conn, time_t now)
   }
 
   if (connection_speaks_cells(conn)) {
-    /* use the per-conn write limit if it's lower, but if it's less
-     * than zero just use zero */
+    /* use the per-conn write limit if it's lower */
     or_connection_t *or_conn = TO_OR_CONN(conn);
     if (conn->state == OR_CONN_STATE_OPEN)
-      if (or_conn->write_bucket < conn_bucket)
-        conn_bucket = or_conn->write_bucket >= 0 ?
-                        or_conn->write_bucket : 0;
+      conn_bucket = MIN(conn_bucket,
+                        token_bucket_rw_get_write(&or_conn->bucket));
     base = get_cell_network_size(or_conn->wide_circ_ids);
   }
 
-  if (connection_counts_as_relayed_traffic(conn, now) &&
-      global_relayed_write_bucket <= global_write_bucket)
-    global_bucket = global_relayed_write_bucket;
+  if (connection_counts_as_relayed_traffic(conn, now)) {
+    size_t relayed = token_bucket_rw_get_write(&global_relayed_bucket);
+    global_bucket_val = MIN(global_bucket_val, relayed);
+  }
 
-  return connection_bucket_round_robin(base, priority,
-                                       global_bucket, conn_bucket);
+  return connection_bucket_get_share(base, priority,
+                                     global_bucket_val, conn_bucket);
 }
 
 /** Return 1 if the global write buckets are low enough that we
@@ -2901,27 +3176,31 @@ connection_bucket_write_limit(connection_t *conn, time_t now)
 int
 global_write_bucket_low(connection_t *conn, size_t attempt, int priority)
 {
-  int smaller_bucket = global_write_bucket < global_relayed_write_bucket ?
-                       global_write_bucket : global_relayed_write_bucket;
+  size_t smaller_bucket =
+    MIN(token_bucket_rw_get_write(&global_bucket),
+        token_bucket_rw_get_write(&global_relayed_bucket));
   if (authdir_mode(get_options()) && priority>1)
     return 0; /* there's always room to answer v2 if we're an auth dir */
 
   if (!connection_is_rate_limited(conn))
     return 0; /* local conns don't get limited */
 
-  if (smaller_bucket < (int)attempt)
+  if (smaller_bucket < attempt)
     return 1; /* not enough space no matter the priority */
 
-  if (write_buckets_empty_last_second)
-    return 1; /* we're already hitting our limits, no more please */
+  {
+    const time_t diff = approx_time() - write_buckets_last_empty_at;
+    if (diff <= 1)
+      return 1; /* we're already hitting our limits, no more please */
+  }
 
   if (priority == 1) { /* old-style v1 query */
     /* Could we handle *two* of these requests within the next two seconds? */
     const or_options_t *options = get_options();
-    int64_t can_write = (int64_t)smaller_bucket
+    size_t can_write = (size_t) (smaller_bucket
       + 2*(options->RelayBandwidthRate ? options->RelayBandwidthRate :
-                                         options->BandwidthRate);
-    if (can_write < 2*(int64_t)attempt)
+           options->BandwidthRate));
+    if (can_write < 2*attempt)
       return 1;
   } else { /* v2 query */
     /* no further constraints yet */
@@ -2929,6 +3208,10 @@ global_write_bucket_low(connection_t *conn, size_t attempt, int priority)
   return 0;
 }
 
+/** When did we last tell the accounting subsystem about transmitted
+ * bandwidth? */
+static time_t last_recorded_accounting_at = 0;
+
 /** Helper: adjusts our bandwidth history and informs the controller as
  * appropriate, given that we have just read <b>num_read</b> bytes and written
  * <b>num_written</b> bytes on <b>conn</b>. */
@@ -2959,59 +3242,22 @@ record_num_bytes_transferred_impl(connection_t *conn,
   }
   if (conn->type == CONN_TYPE_EXIT)
     rep_hist_note_exit_bytes(conn->port, num_written, num_read);
-}
-
-/** Helper: convert given <b>tvnow</b> time value to milliseconds since
- * midnight. */
-static uint32_t
-msec_since_midnight(const struct timeval *tvnow)
-{
-  return (uint32_t)(((tvnow->tv_sec % 86400L) * 1000L) +
-         ((uint32_t)tvnow->tv_usec / (uint32_t)1000L));
-}
 
-/** Helper: return the time in milliseconds since <b>last_empty_time</b>
- * when a bucket ran empty that previously had <b>tokens_before</b> tokens
- * now has <b>tokens_after</b> tokens after refilling at timestamp
- * <b>tvnow</b>, capped at <b>milliseconds_elapsed</b> milliseconds since
- * last refilling that bucket.  Return 0 if the bucket has not been empty
- * since the last refill or has not been refilled. */
-uint32_t
-bucket_millis_empty(int tokens_before, uint32_t last_empty_time,
-                    int tokens_after, int milliseconds_elapsed,
-                    const struct timeval *tvnow)
-{
-  uint32_t result = 0, refilled;
-  if (tokens_before <= 0 && tokens_after > tokens_before) {
-    refilled = msec_since_midnight(tvnow);
-    result = (uint32_t)((refilled + 86400L * 1000L - last_empty_time) %
-             (86400L * 1000L));
-    if (result > (uint32_t)milliseconds_elapsed)
-      result = (uint32_t)milliseconds_elapsed;
-  }
-  return result;
-}
+  /* Remember these bytes towards statistics. */
+  stats_increment_bytes_read_and_written(num_read, num_written);
 
-/** Check if a bucket which had <b>tokens_before</b> tokens and which got
- * <b>tokens_removed</b> tokens removed at timestamp <b>tvnow</b> has run
- * out of tokens, and if so, note the milliseconds since midnight in
- * <b>timestamp_var</b> for the next TB_EMPTY event. */
-void
-connection_buckets_note_empty_ts(uint32_t *timestamp_var,
-                                 int tokens_before, size_t tokens_removed,
-                                 const struct timeval *tvnow)
-{
-  if (tokens_before > 0 && (uint32_t)tokens_before <= tokens_removed)
-    *timestamp_var = msec_since_midnight(tvnow);
+  /* Remember these bytes towards accounting. */
+  if (accounting_is_enabled(get_options())) {
+    if (now > last_recorded_accounting_at && last_recorded_accounting_at) {
+      accounting_add_bytes(num_read, num_written,
+                           (int)(now - last_recorded_accounting_at));
+    } else {
+      accounting_add_bytes(num_read, num_written, 0);
+    }
+    last_recorded_accounting_at = now;
+  }
 }
 
-/** Last time at which the global or relay buckets were emptied in msec
- * since midnight. */
-static uint32_t global_relayed_read_emptied = 0,
-                global_relayed_write_emptied = 0,
-                global_read_emptied = 0,
-                global_write_emptied = 0;
-
 /** We just read <b>num_read</b> and wrote <b>num_written</b> bytes
  * onto <b>conn</b>. Decrement buckets appropriately. */
 static void
@@ -3024,9 +3270,11 @@ connection_buckets_decrement(connection_t *conn, time_t now,
              (unsigned long)num_read, (unsigned long)num_written,
              conn_type_to_string(conn->type),
              conn_state_to_string(conn->type, conn->state));
-    if (num_written >= INT_MAX) num_written = 1;
-    if (num_read >= INT_MAX) num_read = 1;
-    tor_fragile_assert();
+    tor_assert_nonfatal_unreached();
+    if (num_written >= INT_MAX)
+      num_written = 1;
+    if (num_read >= INT_MAX)
+      num_read = 1;
   }
 
   record_num_bytes_transferred_impl(conn, now, num_read, num_written);
@@ -3034,45 +3282,54 @@ connection_buckets_decrement(connection_t *conn, time_t now,
   if (!connection_is_rate_limited(conn))
     return; /* local IPs are free */
 
-  /* If one or more of our token buckets ran dry just now, note the
-   * timestamp for TB_EMPTY events. */
-  if (get_options()->TestingEnableTbEmptyEvent) {
-    struct timeval tvnow;
-    tor_gettimeofday_cached(&tvnow);
-    if (connection_counts_as_relayed_traffic(conn, now)) {
-      connection_buckets_note_empty_ts(&global_relayed_read_emptied,
-                         global_relayed_read_bucket, num_read, &tvnow);
-      connection_buckets_note_empty_ts(&global_relayed_write_emptied,
-                         global_relayed_write_bucket, num_written, &tvnow);
-    }
-    connection_buckets_note_empty_ts(&global_read_emptied,
-                       global_read_bucket, num_read, &tvnow);
-    connection_buckets_note_empty_ts(&global_write_emptied,
-                       global_write_bucket, num_written, &tvnow);
-    if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) {
-      or_connection_t *or_conn = TO_OR_CONN(conn);
-      connection_buckets_note_empty_ts(&or_conn->read_emptied_time,
-                         or_conn->read_bucket, num_read, &tvnow);
-      connection_buckets_note_empty_ts(&or_conn->write_emptied_time,
-                         or_conn->write_bucket, num_written, &tvnow);
-    }
+  unsigned flags = 0;
+  if (connection_counts_as_relayed_traffic(conn, now)) {
+    flags = token_bucket_rw_dec(&global_relayed_bucket, num_read, num_written);
   }
+  flags |= token_bucket_rw_dec(&global_bucket, num_read, num_written);
 
-  if (connection_counts_as_relayed_traffic(conn, now)) {
-    global_relayed_read_bucket -= (int)num_read;
-    global_relayed_write_bucket -= (int)num_written;
+  if (flags & TB_WRITE) {
+    write_buckets_last_empty_at = now;
   }
-  global_read_bucket -= (int)num_read;
-  global_write_bucket -= (int)num_written;
   if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) {
-    TO_OR_CONN(conn)->read_bucket -= (int)num_read;
-    TO_OR_CONN(conn)->write_bucket -= (int)num_written;
+    or_connection_t *or_conn = TO_OR_CONN(conn);
+    token_bucket_rw_dec(&or_conn->bucket, num_read, num_written);
   }
 }
 
+/**
+ * Mark <b>conn</b> as needing to stop reading because bandwidth has been
+ * exhausted.  If <b>is_global_bw</b>, it is closing because global bandwidth
+ * limit has been exhausted.  Otherwise, it is closing because its own
+ * bandwidth limit has been exhausted.
+ */
+void
+connection_read_bw_exhausted(connection_t *conn, bool is_global_bw)
+{
+  (void)is_global_bw;
+  conn->read_blocked_on_bw = 1;
+  connection_stop_reading(conn);
+  reenable_blocked_connection_schedule();
+}
+
+/**
+ * Mark <b>conn</b> as needing to stop reading because write bandwidth has
+ * been exhausted.  If <b>is_global_bw</b>, it is closing because global
+ * bandwidth limit has been exhausted.  Otherwise, it is closing because its
+ * own bandwidth limit has been exhausted.
+*/
+void
+connection_write_bw_exhausted(connection_t *conn, bool is_global_bw)
+{
+  (void)is_global_bw;
+  conn->write_blocked_on_bw = 1;
+  connection_stop_writing(conn);
+  reenable_blocked_connection_schedule();
+}
+
 /** If we have exhausted our global buckets, or the buckets for conn,
  * stop reading. */
-static void
+void
 connection_consider_empty_read_buckets(connection_t *conn)
 {
   const char *reason;
@@ -3080,26 +3337,28 @@ connection_consider_empty_read_buckets(connection_t *conn)
   if (!connection_is_rate_limited(conn))
     return; /* Always okay. */
 
-  if (global_read_bucket <= 0) {
+  int is_global = 1;
+
+  if (token_bucket_rw_get_read(&global_bucket) <= 0) {
     reason = "global read bucket exhausted. Pausing.";
   } else if (connection_counts_as_relayed_traffic(conn, approx_time()) &&
-             global_relayed_read_bucket <= 0) {
+             token_bucket_rw_get_read(&global_relayed_bucket) <= 0) {
     reason = "global relayed read bucket exhausted. Pausing.";
   } else if (connection_speaks_cells(conn) &&
              conn->state == OR_CONN_STATE_OPEN &&
-             TO_OR_CONN(conn)->read_bucket <= 0) {
+             token_bucket_rw_get_read(&TO_OR_CONN(conn)->bucket) <= 0) {
     reason = "connection read bucket exhausted. Pausing.";
+    is_global = false;
   } else
     return; /* all good, no need to stop it */
 
   LOG_FN_CONN(conn, (LOG_DEBUG, LD_NET, "%s", reason));
-  conn->read_blocked_on_bw = 1;
-  connection_stop_reading(conn);
+  connection_read_bw_exhausted(conn, is_global);
 }
 
 /** If we have exhausted our global buckets, or the buckets for conn,
  * stop writing. */
-static void
+void
 connection_consider_empty_write_buckets(connection_t *conn)
 {
   const char *reason;
@@ -3107,238 +3366,171 @@ connection_consider_empty_write_buckets(connection_t *conn)
   if (!connection_is_rate_limited(conn))
     return; /* Always okay. */
 
-  if (global_write_bucket <= 0) {
+  bool is_global = true;
+  if (token_bucket_rw_get_write(&global_bucket) <= 0) {
     reason = "global write bucket exhausted. Pausing.";
   } else if (connection_counts_as_relayed_traffic(conn, approx_time()) &&
-             global_relayed_write_bucket <= 0) {
+             token_bucket_rw_get_write(&global_relayed_bucket) <= 0) {
     reason = "global relayed write bucket exhausted. Pausing.";
   } else if (connection_speaks_cells(conn) &&
              conn->state == OR_CONN_STATE_OPEN &&
-             TO_OR_CONN(conn)->write_bucket <= 0) {
+             token_bucket_rw_get_write(&TO_OR_CONN(conn)->bucket) <= 0) {
     reason = "connection write bucket exhausted. Pausing.";
+    is_global = false;
   } else
     return; /* all good, no need to stop it */
 
   LOG_FN_CONN(conn, (LOG_DEBUG, LD_NET, "%s", reason));
-  conn->write_blocked_on_bw = 1;
-  connection_stop_writing(conn);
+  connection_write_bw_exhausted(conn, is_global);
 }
 
-/** Initialize the global read bucket to options-\>BandwidthBurst. */
+/** Initialize the global buckets to the values configured in the
+ * options */
 void
 connection_bucket_init(void)
 {
   const or_options_t *options = get_options();
-  /* start it at max traffic */
-  global_read_bucket = (int)options->BandwidthBurst;
-  global_write_bucket = (int)options->BandwidthBurst;
+  const uint32_t now_ts = monotime_coarse_get_stamp();
+  token_bucket_rw_init(&global_bucket,
+                    (int32_t)options->BandwidthRate,
+                    (int32_t)options->BandwidthBurst,
+                    now_ts);
   if (options->RelayBandwidthRate) {
-    global_relayed_read_bucket = (int)options->RelayBandwidthBurst;
-    global_relayed_write_bucket = (int)options->RelayBandwidthBurst;
+    token_bucket_rw_init(&global_relayed_bucket,
+                      (int32_t)options->RelayBandwidthRate,
+                      (int32_t)options->RelayBandwidthBurst,
+                      now_ts);
   } else {
-    global_relayed_read_bucket = (int)options->BandwidthBurst;
-    global_relayed_write_bucket = (int)options->BandwidthBurst;
+    token_bucket_rw_init(&global_relayed_bucket,
+                      (int32_t)options->BandwidthRate,
+                      (int32_t)options->BandwidthBurst,
+                      now_ts);
   }
+
+  reenable_blocked_connection_init(options);
 }
 
-/** Refill a single <b>bucket</b> called <b>name</b> with bandwidth rate per
- * second <b>rate</b> and bandwidth burst <b>burst</b>, assuming that
- * <b>milliseconds_elapsed</b> milliseconds have passed since the last
- * call. */
-static void
-connection_bucket_refill_helper(int *bucket, int rate, int burst,
-                                int milliseconds_elapsed,
-                                const char *name)
+/** Update the global connection bucket settings to a new value. */
+void
+connection_bucket_adjust(const or_options_t *options)
 {
-  int starting_bucket = *bucket;
-  if (starting_bucket < burst && milliseconds_elapsed > 0) {
-    int64_t incr = (((int64_t)rate) * milliseconds_elapsed) / 1000;
-    if ((burst - starting_bucket) < incr) {
-      *bucket = burst;  /* We would overflow the bucket; just set it to
-                         * the maximum. */
-    } else {
-      *bucket += (int)incr;
-      if (*bucket > burst || *bucket < starting_bucket) {
-        /* If we overflow the burst, or underflow our starting bucket,
-         * cap the bucket value to burst. */
-        /* XXXX this might be redundant now, but it doesn't show up
-         * in profiles.  Remove it after analysis. */
-        *bucket = burst;
-      }
-    }
-    log_debug(LD_NET,"%s now %d.", name, *bucket);
+  token_bucket_rw_adjust(&global_bucket,
+                      (int32_t)options->BandwidthRate,
+                      (int32_t)options->BandwidthBurst);
+  if (options->RelayBandwidthRate) {
+    token_bucket_rw_adjust(&global_relayed_bucket,
+                        (int32_t)options->RelayBandwidthRate,
+                        (int32_t)options->RelayBandwidthBurst);
+  } else {
+    token_bucket_rw_adjust(&global_relayed_bucket,
+                        (int32_t)options->BandwidthRate,
+                        (int32_t)options->BandwidthBurst);
   }
 }
 
-/** Time has passed; increment buckets appropriately. */
-void
-connection_bucket_refill(int milliseconds_elapsed, time_t now)
+/**
+ * Cached value of the last coarse-timestamp when we refilled the
+ * global buckets.
+ */
+static uint32_t last_refilled_global_buckets_ts=0;
+/**
+ * Refill the token buckets for a single connection <b>conn</b>, and the
+ * global token buckets as appropriate.  Requires that <b>now_ts</b> is
+ * the time in coarse timestamp units.
+ */
+static void
+connection_bucket_refill_single(connection_t *conn, uint32_t now_ts)
 {
-  const or_options_t *options = get_options();
-  smartlist_t *conns = get_connection_array();
-  int bandwidthrate, bandwidthburst, relayrate, relayburst;
+  /* Note that we only check for equality here: the underlying
+   * token bucket functions can handle moving backwards in time if they
+   * need to. */
+  if (now_ts != last_refilled_global_buckets_ts) {
+    token_bucket_rw_refill(&global_bucket, now_ts);
+    token_bucket_rw_refill(&global_relayed_bucket, now_ts);
+    last_refilled_global_buckets_ts = now_ts;
+  }
 
-  int prev_global_read = global_read_bucket;
-  int prev_global_write = global_write_bucket;
-  int prev_relay_read = global_relayed_read_bucket;
-  int prev_relay_write = global_relayed_write_bucket;
-  struct timeval tvnow; /*< Only used if TB_EMPTY events are enabled. */
+  if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) {
+    or_connection_t *or_conn = TO_OR_CONN(conn);
+    token_bucket_rw_refill(&or_conn->bucket, now_ts);
+  }
+}
 
-  bandwidthrate = (int)options->BandwidthRate;
-  bandwidthburst = (int)options->BandwidthBurst;
+/**
+ * Event to re-enable all connections that were previously blocked on read or
+ * write.
+ */
+static mainloop_event_t *reenable_blocked_connections_ev = NULL;
 
-  if (options->RelayBandwidthRate) {
-    relayrate = (int)options->RelayBandwidthRate;
-    relayburst = (int)options->RelayBandwidthBurst;
-  } else {
-    relayrate = bandwidthrate;
-    relayburst = bandwidthburst;
-  }
-
-  tor_assert(milliseconds_elapsed >= 0);
-
-  write_buckets_empty_last_second =
-    global_relayed_write_bucket <= 0 || global_write_bucket <= 0;
-
-  /* refill the global buckets */
-  connection_bucket_refill_helper(&global_read_bucket,
-                                  bandwidthrate, bandwidthburst,
-                                  milliseconds_elapsed,
-                                  "global_read_bucket");
-  connection_bucket_refill_helper(&global_write_bucket,
-                                  bandwidthrate, bandwidthburst,
-                                  milliseconds_elapsed,
-                                  "global_write_bucket");
-  connection_bucket_refill_helper(&global_relayed_read_bucket,
-                                  relayrate, relayburst,
-                                  milliseconds_elapsed,
-                                  "global_relayed_read_bucket");
-  connection_bucket_refill_helper(&global_relayed_write_bucket,
-                                  relayrate, relayburst,
-                                  milliseconds_elapsed,
-                                  "global_relayed_write_bucket");
-
-  /* If buckets were empty before and have now been refilled, tell any
-   * interested controllers. */
-  if (get_options()->TestingEnableTbEmptyEvent) {
-    uint32_t global_read_empty_time, global_write_empty_time,
-             relay_read_empty_time, relay_write_empty_time;
-    tor_gettimeofday_cached(&tvnow);
-    global_read_empty_time = bucket_millis_empty(prev_global_read,
-                             global_read_emptied, global_read_bucket,
-                             milliseconds_elapsed, &tvnow);
-    global_write_empty_time = bucket_millis_empty(prev_global_write,
-                              global_write_emptied, global_write_bucket,
-                              milliseconds_elapsed, &tvnow);
-    control_event_tb_empty("GLOBAL", global_read_empty_time,
-                           global_write_empty_time, milliseconds_elapsed);
-    relay_read_empty_time = bucket_millis_empty(prev_relay_read,
-                            global_relayed_read_emptied,
-                            global_relayed_read_bucket,
-                            milliseconds_elapsed, &tvnow);
-    relay_write_empty_time = bucket_millis_empty(prev_relay_write,
-                             global_relayed_write_emptied,
-                             global_relayed_write_bucket,
-                             milliseconds_elapsed, &tvnow);
-    control_event_tb_empty("RELAY", relay_read_empty_time,
-                           relay_write_empty_time, milliseconds_elapsed);
-  }
-
-  /* refill the per-connection buckets */
-  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
-    if (connection_speaks_cells(conn)) {
-      or_connection_t *or_conn = TO_OR_CONN(conn);
-      int orbandwidthrate = or_conn->bandwidthrate;
-      int orbandwidthburst = or_conn->bandwidthburst;
-
-      int prev_conn_read = or_conn->read_bucket;
-      int prev_conn_write = or_conn->write_bucket;
-
-      if (connection_bucket_should_increase(or_conn->read_bucket, or_conn)) {
-        connection_bucket_refill_helper(&or_conn->read_bucket,
-                                        orbandwidthrate,
-                                        orbandwidthburst,
-                                        milliseconds_elapsed,
-                                        "or_conn->read_bucket");
-      }
-      if (connection_bucket_should_increase(or_conn->write_bucket, or_conn)) {
-        connection_bucket_refill_helper(&or_conn->write_bucket,
-                                        orbandwidthrate,
-                                        orbandwidthburst,
-                                        milliseconds_elapsed,
-                                        "or_conn->write_bucket");
-      }
+/** True iff reenable_blocked_connections_ev is currently scheduled. */
+static int reenable_blocked_connections_is_scheduled = 0;
 
-      /* If buckets were empty before and have now been refilled, tell any
-       * interested controllers. */
-      if (get_options()->TestingEnableTbEmptyEvent) {
-        char *bucket;
-        uint32_t conn_read_empty_time, conn_write_empty_time;
-        tor_asprintf(&bucket, "ORCONN ID="U64_FORMAT,
-                     U64_PRINTF_ARG(or_conn->base_.global_identifier));
-        conn_read_empty_time = bucket_millis_empty(prev_conn_read,
-                               or_conn->read_emptied_time,
-                               or_conn->read_bucket,
-                               milliseconds_elapsed, &tvnow);
-        conn_write_empty_time = bucket_millis_empty(prev_conn_write,
-                                or_conn->write_emptied_time,
-                                or_conn->write_bucket,
-                                milliseconds_elapsed, &tvnow);
-        control_event_tb_empty(bucket, conn_read_empty_time,
-                               conn_write_empty_time,
-                               milliseconds_elapsed);
-        tor_free(bucket);
-      }
-    }
+/** Delay after which to run reenable_blocked_connections_ev. */
+static struct timeval reenable_blocked_connections_delay;
 
-    if (conn->read_blocked_on_bw == 1 /* marked to turn reading back on now */
-        && global_read_bucket > 0 /* and we're allowed to read */
-        && (!connection_counts_as_relayed_traffic(conn, now) ||
-            global_relayed_read_bucket > 0) /* even if we're relayed traffic */
-        && (!connection_speaks_cells(conn) ||
-            conn->state != OR_CONN_STATE_OPEN ||
-            TO_OR_CONN(conn)->read_bucket > 0)) {
-        /* and either a non-cell conn or a cell conn with non-empty bucket */
-      LOG_FN_CONN(conn, (LOG_DEBUG,LD_NET,
-                         "waking up conn (fd %d) for read", (int)conn->s));
-      conn->read_blocked_on_bw = 0;
+/**
+ * Re-enable all connections that were previously blocked on read or write.
+ * This event is scheduled after enough time has elapsed to be sure
+ * that the buckets will refill when the connections have something to do.
+ */
+static void
+reenable_blocked_connections_cb(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  (void)arg;
+  SMARTLIST_FOREACH_BEGIN(get_connection_array(), connection_t *, conn) {
+    if (conn->read_blocked_on_bw == 1) {
       connection_start_reading(conn);
+      conn->read_blocked_on_bw = 0;
     }
-
-    if (conn->write_blocked_on_bw == 1
-        && global_write_bucket > 0 /* and we're allowed to write */
-        && (!connection_counts_as_relayed_traffic(conn, now) ||
-            global_relayed_write_bucket > 0) /* even if it's relayed traffic */
-        && (!connection_speaks_cells(conn) ||
-            conn->state != OR_CONN_STATE_OPEN ||
-            TO_OR_CONN(conn)->write_bucket > 0)) {
-      LOG_FN_CONN(conn, (LOG_DEBUG,LD_NET,
-                         "waking up conn (fd %d) for write", (int)conn->s));
-      conn->write_blocked_on_bw = 0;
+    if (conn->write_blocked_on_bw == 1) {
       connection_start_writing(conn);
+      conn->write_blocked_on_bw = 0;
     }
   } SMARTLIST_FOREACH_END(conn);
+
+  reenable_blocked_connections_is_scheduled = 0;
 }
 
-/** Is the <b>bucket</b> for connection <b>conn</b> low enough that we
- * should add another pile of tokens to it?
+/**
+ * Initialize the mainloop event that we use to wake up connections that
+ * find themselves blocked on bandwidth.
  */
-static int
-connection_bucket_should_increase(int bucket, or_connection_t *conn)
+static void
+reenable_blocked_connection_init(const or_options_t *options)
 {
-  tor_assert(conn);
-
-  if (conn->base_.state != OR_CONN_STATE_OPEN)
-    return 0; /* only open connections play the rate limiting game */
-  if (bucket >= conn->bandwidthburst)
-    return 0;
+  if (! reenable_blocked_connections_ev) {
+    reenable_blocked_connections_ev =
+      mainloop_event_new(reenable_blocked_connections_cb, NULL);
+    reenable_blocked_connections_is_scheduled = 0;
+  }
+  time_t sec = options->TokenBucketRefillInterval / 1000;
+  int msec = (options->TokenBucketRefillInterval % 1000);
+  reenable_blocked_connections_delay.tv_sec = sec;
+  reenable_blocked_connections_delay.tv_usec = msec * 1000;
+}
 
-  return 1;
+/**
+ * Called when we have blocked a connection for being low on bandwidth:
+ * schedule an event to reenable such connections, if it is not already
+ * scheduled.
+ */
+static void
+reenable_blocked_connection_schedule(void)
+{
+  if (reenable_blocked_connections_is_scheduled)
+    return;
+  if (BUG(reenable_blocked_connections_ev == NULL)) {
+    reenable_blocked_connection_init(get_options());
+  }
+  mainloop_event_schedule(reenable_blocked_connections_ev,
+                          &reenable_blocked_connections_delay);
+  reenable_blocked_connections_is_scheduled = 1;
 }
 
 /** Read bytes from conn-\>s and process them.
  *
- * It calls connection_read_to_buf() to bring in any new bytes,
+ * It calls connection_buf_read_from_socket() to bring in any new bytes,
  * and then calls connection_process_inbuf() to process them.
  *
  * Mark the connection and return -1 if you want to close it, else
@@ -3354,7 +3546,9 @@ connection_handle_read_impl(connection_t *conn)
   if (conn->marked_for_close)
     return 0; /* do nothing */
 
-  conn->timestamp_lastread = approx_time();
+  conn->timestamp_last_read_allowed = approx_time();
+
+  connection_bucket_refill_single(conn, monotime_coarse_get_stamp());
 
   switch (conn->type) {
     case CONN_TYPE_OR_LISTENER:
@@ -3364,6 +3558,7 @@ connection_handle_read_impl(connection_t *conn)
     case CONN_TYPE_AP_LISTENER:
     case CONN_TYPE_AP_TRANS_LISTENER:
     case CONN_TYPE_AP_NATD_LISTENER:
+    case CONN_TYPE_AP_HTTP_CONNECT_LISTENER:
       return connection_handle_listener_read(conn, CONN_TYPE_AP);
     case CONN_TYPE_DIR_LISTENER:
       return connection_handle_listener_read(conn, CONN_TYPE_DIR);
@@ -3380,7 +3575,7 @@ connection_handle_read_impl(connection_t *conn)
   tor_assert(!conn->marked_for_close);
 
   before = buf_datalen(conn->inbuf);
-  if (connection_read_to_buf(conn, &max_to_read, &socket_error) < 0) {
+  if (connection_buf_read_from_socket(conn, &max_to_read, &socket_error) < 0) {
     /* There's a read error; kill the connection.*/
     if (conn->type == CONN_TYPE_OR) {
       connection_or_notify_error(TO_OR_CONN(conn),
@@ -3460,8 +3655,7 @@ int
 connection_handle_read(connection_t *conn)
 {
   int res;
-
-  tor_gettimeofday_cache_clear();
+  update_current_time(time(NULL));
   res = connection_handle_read_impl(conn);
   return res;
 }
@@ -3477,7 +3671,7 @@ connection_handle_read(connection_t *conn)
  * Return -1 if we want to break conn, else return 0.
  */
 static int
-connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
+connection_buf_read_from_socket(connection_t *conn, ssize_t *max_to_read,
                        int *socket_error)
 {
   int result;
@@ -3518,7 +3712,7 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
 
     initial_size = buf_datalen(conn->inbuf);
     /* else open, or closing */
-    result = read_to_buf_tls(or_conn->tls, at_most, conn->inbuf);
+    result = buf_read_from_tls(conn->inbuf, or_conn->tls, at_most);
     if (TOR_TLS_IS_ERROR(result) || result == TOR_TLS_CLOSE)
       or_conn->tls_error = result;
     else
@@ -3548,16 +3742,15 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
            * waiting for a TLS renegotiation, the renegotiation started, and
            * SSL_read returned WANTWRITE.  But now SSL_read is saying WANTREAD
            * again.  Stop waiting for write events now, or else we'll
-           * busy-loop until data arrives for us to read. */
+           * busy-loop until data arrives for us to read.
+           * XXX: remove this when v2 handshakes support is dropped. */
           connection_stop_writing(conn);
           if (!connection_is_reading(conn))
             connection_start_reading(conn);
         }
         /* we're already reading, one hopes */
-        result = 0;
         break;
       case TOR_TLS_DONE: /* no data read, so nothing to process */
-        result = 0;
         break; /* so we call bucket_decrement below */
       default:
         break;
@@ -3567,7 +3760,7 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
       /* If we have any pending bytes, we read them now.  This *can*
        * take us over our read allotment, but really we shouldn't be
        * believing that SSL bytes are the same as TCP bytes anyway. */
-      int r2 = read_to_buf_tls(or_conn->tls, pending, conn->inbuf);
+      int r2 = buf_read_from_tls(conn->inbuf, or_conn->tls, pending);
       if (BUG(r2<0)) {
         log_warn(LD_BUG, "apparently, reading pending bytes can fail.");
         return -1;
@@ -3579,7 +3772,7 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
               result, (long)n_read, (long)n_written);
   } else if (conn->linked) {
     if (conn->linked_conn) {
-      result = move_buf_to_buf(conn->inbuf, conn->linked_conn->outbuf,
+      result = buf_move_to_buf(conn->inbuf, conn->linked_conn->outbuf,
                                &conn->linked_conn->outbuf_flushlen);
       if (BUG(result<0)) {
         log_warn(LD_BUG, "reading from linked connection buffer failed.");
@@ -3601,8 +3794,10 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
     /* !connection_speaks_cells, !conn->linked_conn. */
     int reached_eof = 0;
     CONN_LOG_PROTECT(conn,
-        result = read_to_buf(conn->s, at_most, conn->inbuf, &reached_eof,
-                             socket_error));
+                     result = buf_read_from_socket(conn->inbuf, conn->s,
+                                                   at_most,
+                                                   &reached_eof,
+                                                   socket_error));
     if (reached_eof)
       conn->inbuf_reached_eof = 1;
 
@@ -3617,25 +3812,15 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
      /* change *max_to_read */
     *max_to_read = at_most - n_read;
 
-    /* Update edge_conn->n_read and ocirc->n_read_circ_bw */
+    /* Update edge_conn->n_read */
     if (conn->type == CONN_TYPE_AP) {
       edge_connection_t *edge_conn = TO_EDGE_CONN(conn);
-      circuit_t *circ = circuit_get_by_edge_conn(edge_conn);
-      origin_circuit_t *ocirc;
 
       /* Check for overflow: */
       if (PREDICT_LIKELY(UINT32_MAX - edge_conn->n_read > n_read))
         edge_conn->n_read += (int)n_read;
       else
         edge_conn->n_read = UINT32_MAX;
-
-      if (circ && CIRCUIT_IS_ORIGIN(circ)) {
-        ocirc = TO_ORIGIN_CIRCUIT(circ);
-        if (PREDICT_LIKELY(UINT32_MAX - ocirc->n_read_circ_bw > n_read))
-          ocirc->n_read_circ_bw += (int)n_read;
-        else
-          ocirc->n_read_circ_bw = UINT32_MAX;
-      }
     }
 
     /* If CONN_BW events are enabled, update conn->n_read_conn_bw for
@@ -3671,17 +3856,17 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
 
 /** A pass-through to fetch_from_buf. */
 int
-connection_fetch_from_buf(char *string, size_t len, connection_t *conn)
+connection_buf_get_bytes(char *string, size_t len, connection_t *conn)
 {
-  return fetch_from_buf(string, len, conn->inbuf);
+  return buf_get_bytes(conn->inbuf, string, len);
 }
 
-/** As fetch_from_buf_line(), but read from a connection's input buffer. */
+/** As buf_get_line(), but read from a connection's input buffer. */
 int
-connection_fetch_from_buf_line(connection_t *conn, char *data,
+connection_buf_get_line(connection_t *conn, char *data,
                                size_t *data_len)
 {
-  return fetch_from_buf_line(conn->inbuf, data, data_len);
+  return buf_get_line(conn->inbuf, data, data_len);
 }
 
 /** As fetch_from_buf_http, but fetches from a connection's input buffer_t as
@@ -3714,13 +3899,50 @@ connection_outbuf_too_full(connection_t *conn)
   return (conn->outbuf_flushlen > 10*CELL_PAYLOAD_SIZE);
 }
 
+/**
+ * On Windows Vista and Windows 7, tune the send buffer size according to a
+ * hint from the OS.
+ *
+ * This should help fix slow upload rates.
+ */
+static void
+update_send_buffer_size(tor_socket_t sock)
+{
+#ifdef _WIN32
+  /* We only do this on Vista and 7, because earlier versions of Windows
+   * don't have the SIO_IDEAL_SEND_BACKLOG_QUERY functionality, and on
+   * later versions it isn't necessary. */
+  static int isVistaOr7 = -1;
+  if (isVistaOr7 == -1) {
+    isVistaOr7 = 0;
+    OSVERSIONINFO osvi = { 0 };
+    osvi.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
+    GetVersionEx(&osvi);
+    if (osvi.dwMajorVersion == 6 && osvi.dwMinorVersion < 2)
+      isVistaOr7 = 1;
+  }
+  if (!isVistaOr7)
+    return;
+  if (get_options()->ConstrainedSockets)
+    return;
+  ULONG isb = 0;
+  DWORD bytesReturned = 0;
+  if (!WSAIoctl(sock, SIO_IDEAL_SEND_BACKLOG_QUERY, NULL, 0,
+      &isb, sizeof(isb), &bytesReturned, NULL, NULL)) {
+    setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (const char*)&isb, sizeof(isb));
+  }
+#else
+  (void) sock;
+#endif
+}
+
 /** Try to flush more bytes onto <b>conn</b>-\>s.
  *
- * This function gets called either from conn_write_callback() in main.c
- * when libevent tells us that conn wants to write, or below
- * from connection_write_to_buf() when an entire TLS record is ready.
+ * This function is called in connection_handle_write(), which gets
+ * called from conn_write_callback() in main.c when libevent tells us
+ * that <b>conn</b> wants to write.
  *
- * Update <b>conn</b>-\>timestamp_lastwritten to now, and call flush_buf
+ * Update <b>conn</b>-\>timestamp_last_write_allowed to now, and call flush_buf
  * or flush_buf_tls appropriately. If it succeeds and there are no more
  * more bytes on <b>conn</b>-\>outbuf, then call connection_finished_flushing
  * on it too.
@@ -3753,7 +3975,9 @@ connection_handle_write_impl(connection_t *conn, int force)
     return 0;
   }
 
-  conn->timestamp_lastwritten = now;
+  conn->timestamp_last_write_allowed = now;
+
+  connection_bucket_refill_single(conn, monotime_coarse_get_stamp());
 
   /* Sometimes, "writable" means "connected". */
   if (connection_state_is_connecting(conn)) {
@@ -3829,9 +4053,12 @@ connection_handle_write_impl(connection_t *conn, int force)
 
     /* else open, or closing */
     initial_size = buf_datalen(conn->outbuf);
-    result = flush_buf_tls(or_conn->tls, conn->outbuf,
+    result = buf_flush_to_tls(conn->outbuf, or_conn->tls,
                            max_to_write, &conn->outbuf_flushlen);
 
+    if (result >= 0)
+      update_send_buffer_size(conn->s);
+
     /* If we just flushed the last bytes, tell the channel on the
      * or_conn to check if it needs to geoip_change_dirreq_state() */
     /* XXXX move this to flushed_some or finished_flushing -NM */
@@ -3865,8 +4092,7 @@ connection_handle_write_impl(connection_t *conn, int force)
         /* Make sure to avoid a loop if the receive buckets are empty. */
         log_debug(LD_NET,"wanted read.");
         if (!connection_is_reading(conn)) {
-          connection_stop_writing(conn);
-          conn->write_blocked_on_bw = 1;
+          connection_write_bw_exhausted(conn, true);
           /* we'll start reading again when we get more tokens in our
            * read bucket; then we'll start writing again too.
            */
@@ -3892,8 +4118,8 @@ connection_handle_write_impl(connection_t *conn, int force)
     result = (int)(initial_size-buf_datalen(conn->outbuf));
   } else {
     CONN_LOG_PROTECT(conn,
-             result = flush_buf(conn->s, conn->outbuf,
-                                max_to_write, &conn->outbuf_flushlen));
+                     result = buf_flush_to_socket(conn->outbuf, conn->s,
+                                        max_to_write, &conn->outbuf_flushlen));
     if (result < 0) {
       if (CONN_IS_EDGE(conn))
         connection_edge_end_errno(TO_EDGE_CONN(conn));
@@ -3906,27 +4132,18 @@ connection_handle_write_impl(connection_t *conn, int force)
       connection_mark_for_close(conn);
       return -1;
     }
+    update_send_buffer_size(conn->s);
     n_written = (size_t) result;
   }
 
   if (n_written && conn->type == CONN_TYPE_AP) {
     edge_connection_t *edge_conn = TO_EDGE_CONN(conn);
-    circuit_t *circ = circuit_get_by_edge_conn(edge_conn);
-    origin_circuit_t *ocirc;
 
     /* Check for overflow: */
     if (PREDICT_LIKELY(UINT32_MAX - edge_conn->n_written > n_written))
       edge_conn->n_written += (int)n_written;
     else
       edge_conn->n_written = UINT32_MAX;
-
-    if (circ && CIRCUIT_IS_ORIGIN(circ)) {
-      ocirc = TO_ORIGIN_CIRCUIT(circ);
-      if (PREDICT_LIKELY(UINT32_MAX - ocirc->n_written_circ_bw > n_written))
-        ocirc->n_written_circ_bw += (int)n_written;
-      else
-        ocirc->n_written_circ_bw = UINT32_MAX;
-    }
   }
 
   /* If CONN_BW events are enabled, update conn->n_written_conn_bw for
@@ -3986,7 +4203,10 @@ int
 connection_handle_write(connection_t *conn, int force)
 {
     int res;
-    tor_gettimeofday_cache_clear();
+    update_current_time(time(NULL));
+    /* connection_handle_write_impl() might call connection_handle_read()
+     * if we're in the middle of a v2 handshake, in which case it needs this
+     * flag set. */
     conn->in_connection_handle_write = 1;
     res = connection_handle_write_impl(conn, force);
     conn->in_connection_handle_write = 0;
@@ -4008,6 +4228,68 @@ connection_flush(connection_t *conn)
   return connection_handle_write(conn, 1);
 }
 
+/** Helper for connection_write_to_buf_impl and connection_write_buf_to_buf:
+ *
+ * Return true iff it is okay to queue bytes on <b>conn</b>'s outbuf for
+ * writing.
+ */
+static int
+connection_may_write_to_buf(connection_t *conn)
+{
+  /* if it's marked for close, only allow write if we mean to flush it */
+  if (conn->marked_for_close && !conn->hold_open_until_flushed)
+    return 0;
+
+  return 1;
+}
+
+/** Helper for connection_write_to_buf_impl and connection_write_buf_to_buf:
+ *
+ * Called when an attempt to add bytes on <b>conn</b>'s outbuf has failed;
+ * mark the connection and warn as appropriate.
+ */
+static void
+connection_write_to_buf_failed(connection_t *conn)
+{
+  if (CONN_IS_EDGE(conn)) {
+    /* if it failed, it means we have our package/delivery windows set
+       wrong compared to our max outbuf size. close the whole circuit. */
+    log_warn(LD_NET,
+             "write_to_buf failed. Closing circuit (fd %d).", (int)conn->s);
+    circuit_mark_for_close(circuit_get_by_edge_conn(TO_EDGE_CONN(conn)),
+                           END_CIRC_REASON_INTERNAL);
+  } else if (conn->type == CONN_TYPE_OR) {
+    or_connection_t *orconn = TO_OR_CONN(conn);
+    log_warn(LD_NET,
+             "write_to_buf failed on an orconn; notifying of error "
+             "(fd %d)", (int)(conn->s));
+    connection_or_close_for_error(orconn, 0);
+  } else {
+    log_warn(LD_NET,
+             "write_to_buf failed. Closing connection (fd %d).",
+             (int)conn->s);
+    connection_mark_for_close(conn);
+  }
+}
+
+/** Helper for connection_write_to_buf_impl and connection_write_buf_to_buf:
+ *
+ * Called when an attempt to add bytes on <b>conn</b>'s outbuf has succeeded:
+ * record the number of bytes added.
+ */
+static void
+connection_write_to_buf_commit(connection_t *conn, size_t len)
+{
+  /* If we receive optimistic data in the EXIT_CONN_STATE_RESOLVING
+   * state, we don't want to try to write it right away, since
+   * conn->write_event won't be set yet.  Otherwise, write data from
+   * this conn as the socket is available. */
+  if (conn->write_event) {
+    connection_start_writing(conn);
+  }
+  conn->outbuf_flushlen += len;
+}
+
 /** Append <b>len</b> bytes of <b>string</b> onto <b>conn</b>'s
  * outbuf, and ask it to start writing.
  *
@@ -4015,10 +4297,6 @@ connection_flush(connection_t *conn)
  * its contents compressed or decompressed as they're written.  If zlib is
  * negative, this is the last data to be compressed, and the connection's zlib
  * state should be flushed.
- *
- * If it's a local control connection and a 64k chunk is ready, try to flush
- * it all, so we don't end up with many megabytes of controller info queued at
- * once.
  */
 MOCK_IMPL(void,
 connection_write_to_buf_impl_,(const char *string, size_t len,
@@ -4026,58 +4304,91 @@ connection_write_to_buf_impl_,(const char *string, size_t len,
 {
   /* XXXX This function really needs to return -1 on failure. */
   int r;
-  size_t old_datalen;
   if (!len && !(zlib<0))
     return;
-  /* if it's marked for close, only allow write if we mean to flush it */
-  if (conn->marked_for_close && !conn->hold_open_until_flushed)
+
+  if (!connection_may_write_to_buf(conn))
     return;
 
-  old_datalen = buf_datalen(conn->outbuf);
+  size_t written;
+
   if (zlib) {
+    size_t old_datalen = buf_datalen(conn->outbuf);
     dir_connection_t *dir_conn = TO_DIR_CONN(conn);
     int done = zlib < 0;
-    CONN_LOG_PROTECT(conn, r = write_to_buf_zlib(conn->outbuf,
-                                                 dir_conn->zlib_state,
-                                                 string, len, done));
+    CONN_LOG_PROTECT(conn, r = buf_add_compress(conn->outbuf,
+                                                dir_conn->compress_state,
+                                                string, len, done));
+    written = buf_datalen(conn->outbuf) - old_datalen;
   } else {
-    CONN_LOG_PROTECT(conn, r = write_to_buf(string, len, conn->outbuf));
+    CONN_LOG_PROTECT(conn, r = buf_add(conn->outbuf, string, len));
+    written = len;
   }
   if (r < 0) {
-    if (CONN_IS_EDGE(conn)) {
-      /* if it failed, it means we have our package/delivery windows set
-         wrong compared to our max outbuf size. close the whole circuit. */
-      log_warn(LD_NET,
-               "write_to_buf failed. Closing circuit (fd %d).", (int)conn->s);
-      circuit_mark_for_close(circuit_get_by_edge_conn(TO_EDGE_CONN(conn)),
-                             END_CIRC_REASON_INTERNAL);
-    } else if (conn->type == CONN_TYPE_OR) {
-      or_connection_t *orconn = TO_OR_CONN(conn);
-      log_warn(LD_NET,
-               "write_to_buf failed on an orconn; notifying of error "
-               "(fd %d)", (int)(conn->s));
-      connection_or_close_for_error(orconn, 0);
-    } else {
-      log_warn(LD_NET,
-               "write_to_buf failed. Closing connection (fd %d).",
-               (int)conn->s);
-      connection_mark_for_close(conn);
-    }
+    connection_write_to_buf_failed(conn);
     return;
   }
+  connection_write_to_buf_commit(conn, written);
+}
 
-  /* If we receive optimistic data in the EXIT_CONN_STATE_RESOLVING
-   * state, we don't want to try to write it right away, since
-   * conn->write_event won't be set yet.  Otherwise, write data from
-   * this conn as the socket is available. */
-  if (conn->write_event) {
-    connection_start_writing(conn);
-  }
-  if (zlib) {
-    conn->outbuf_flushlen += buf_datalen(conn->outbuf) - old_datalen;
-  } else {
-    conn->outbuf_flushlen += len;
-  }
+void
+connection_buf_add_compress(const char *string, size_t len,
+                            dir_connection_t *conn, int done)
+{
+  connection_write_to_buf_impl_(string, len, TO_CONN(conn), done ? -1 : 1);
+}
+
+/**
+ * Add all bytes from <b>buf</b> to <b>conn</b>'s outbuf, draining them
+ * from <b>buf</b>. (If the connection is marked and will soon be closed,
+ * nothing is drained.)
+ */
+void
+connection_buf_add_buf(connection_t *conn, buf_t *buf)
+{
+  tor_assert(conn);
+  tor_assert(buf);
+  size_t len = buf_datalen(buf);
+  if (len == 0)
+    return;
+
+  if (!connection_may_write_to_buf(conn))
+    return;
+
+  buf_move_all(conn->outbuf, buf);
+  connection_write_to_buf_commit(conn, len);
+}
+
+#define CONN_GET_ALL_TEMPLATE(var, test) \
+  STMT_BEGIN \
+    smartlist_t *conns = get_connection_array();   \
+    smartlist_t *ret_conns = smartlist_new();     \
+    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, var) { \
+      if (var && (test) && !var->marked_for_close) \
+        smartlist_add(ret_conns, var); \
+    } SMARTLIST_FOREACH_END(var);                                            \
+    return ret_conns; \
+  STMT_END
+
+/* Return a list of connections that aren't close and matches the given type
+ * and state. The returned list can be empty and must be freed using
+ * smartlist_free(). The caller does NOT have ownership of the objects in the
+ * list so it must not free them nor reference them as they can disappear. */
+smartlist_t *
+connection_list_by_type_state(int type, int state)
+{
+  CONN_GET_ALL_TEMPLATE(conn, (conn->type == type && conn->state == state));
+}
+
+/* Return a list of connections that aren't close and matches the given type
+ * and purpose. The returned list can be empty and must be freed using
+ * smartlist_free(). The caller does NOT have ownership of the objects in the
+ * list so it must not free them nor reference them as they can disappear. */
+smartlist_t *
+connection_list_by_type_purpose(int type, int purpose)
+{
+  CONN_GET_ALL_TEMPLATE(conn,
+                        (conn->type == type && conn->purpose == purpose));
 }
 
 /** Return a connection_t * from get_connection_array() that satisfies test on
@@ -4153,12 +4464,12 @@ connection_get_by_type_state_rendquery(int type, int state,
          (type == CONN_TYPE_DIR &&
           TO_DIR_CONN(conn)->rend_data &&
           !rend_cmp_service_ids(rendquery,
-                                TO_DIR_CONN(conn)->rend_data->onion_address))
+                    rend_data_get_address(TO_DIR_CONN(conn)->rend_data)))
          ||
               (CONN_IS_EDGE(conn) &&
                TO_EDGE_CONN(conn)->rend_data &&
                !rend_cmp_service_ids(rendquery,
-                            TO_EDGE_CONN(conn)->rend_data->onion_address))
+                    rend_data_get_address(TO_EDGE_CONN(conn)->rend_data)))
          ));
 }
 
@@ -4264,6 +4575,7 @@ connection_is_listener(connection_t *conn)
       conn->type == CONN_TYPE_AP_TRANS_LISTENER ||
       conn->type == CONN_TYPE_AP_DNS_LISTENER ||
       conn->type == CONN_TYPE_AP_NATD_LISTENER ||
+      conn->type == CONN_TYPE_AP_HTTP_CONNECT_LISTENER ||
       conn->type == CONN_TYPE_DIR_LISTENER ||
       conn->type == CONN_TYPE_CONTROL_LISTENER)
     return 1;
@@ -4340,8 +4652,6 @@ alloc_http_authenticator(const char *authenticator)
 static void
 client_check_address_changed(tor_socket_t sock)
 {
-  struct sockaddr_storage out_sockaddr;
-  socklen_t out_addr_len = (socklen_t) sizeof(out_sockaddr);
   tor_addr_t out_addr, iface_addr;
   tor_addr_t **last_interface_ip_ptr;
   sa_family_t family;
@@ -4349,13 +4659,12 @@ client_check_address_changed(tor_socket_t sock)
   if (!outgoing_addrs)
     outgoing_addrs = smartlist_new();
 
-  if (getsockname(sock, (struct sockaddr*)&out_sockaddr, &out_addr_len)<0) {
+  if (tor_addr_from_getsockname(&out_addr, sock) < 0) {
     int e = tor_socket_errno(sock);
     log_warn(LD_NET, "getsockname() to check for address change failed: %s",
              tor_socket_strerror(e));
     return;
   }
-  tor_addr_from_sockaddr(&out_addr, (struct sockaddr*)&out_sockaddr, NULL);
   family = tor_addr_family(&out_addr);
 
   if (family == AF_INET)
@@ -4733,6 +5042,20 @@ kill_conn_list_for_oos, (smartlist_t *conns))
              smartlist_len(conns));
 }
 
+/** Check if a connection is on the way out so the OOS handler doesn't try
+ * to kill more than it needs. */
+int
+connection_is_moribund(connection_t *conn)
+{
+  if (conn != NULL &&
+      (conn->conn_array_index < 0 ||
+       conn->marked_for_close)) {
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
 /** Out-of-Sockets handler; n_socks is the current number of open
  * sockets, and failed is non-zero if a socket exhaustion related
  * error immediately preceded this call.  This is where to do
@@ -4855,16 +5178,16 @@ connection_dump_buffer_mem_stats(int severity)
   }
 
   tor_log(severity, LD_GENERAL,
-     "In buffers for %d connections: "U64_FORMAT" used/"U64_FORMAT" allocated",
+     "In buffers for %d connections: %"PRIu64" used/%"PRIu64" allocated",
       smartlist_len(conns),
-      U64_PRINTF_ARG(total_used), U64_PRINTF_ARG(total_alloc));
+      (total_used), (total_alloc));
   for (i=CONN_TYPE_MIN_; i <= CONN_TYPE_MAX_; ++i) {
     if (!n_conns_by_type[i])
       continue;
     tor_log(severity, LD_GENERAL,
-        "  For %d %s connections: "U64_FORMAT" used/"U64_FORMAT" allocated",
+        "  For %d %s connections: %"PRIu64" used/%"PRIu64" allocated",
         n_conns_by_type[i], conn_type_to_string(i),
-        U64_PRINTF_ARG(used_by_type[i]), U64_PRINTF_ARG(alloc_by_type[i]));
+        (used_by_type[i]), (alloc_by_type[i]));
   }
 }
 
@@ -4931,9 +5254,9 @@ assert_connection_ok(connection_t *conn, time_t now)
 
   /* buffers */
   if (conn->inbuf)
-    assert_buf_ok(conn->inbuf);
+    buf_assert_ok(conn->inbuf);
   if (conn->outbuf)
-    assert_buf_ok(conn->outbuf);
+    buf_assert_ok(conn->outbuf);
 
   if (conn->type == CONN_TYPE_OR) {
     or_connection_t *or_conn = TO_OR_CONN(conn);
@@ -5107,8 +5430,8 @@ proxy_type_to_string(int proxy_type)
   return NULL; /*Unreached*/
 }
 
-/** Call connection_free_() on every connection in our array, and release all
- * storage held by connection.c.
+/** Call connection_free_minimal() on every connection in our array, and
+ * release all storage held by connection.c.
  *
  * Don't do the checks in connection_free(), because they will
  * fail.
@@ -5132,7 +5455,8 @@ connection_free_all(void)
   /* Clear out our list of broken connections */
   clear_broken_connection_map(0);
 
-  SMARTLIST_FOREACH(conns, connection_t *, conn, connection_free_(conn));
+  SMARTLIST_FOREACH(conns, connection_t *, conn,
+                    connection_free_minimal(conn));
 
   if (outgoing_addrs) {
     SMARTLIST_FOREACH(outgoing_addrs, tor_addr_t *, addr, tor_free(addr));
@@ -5142,6 +5466,11 @@ connection_free_all(void)
 
   tor_free(last_interface_ipv4);
   tor_free(last_interface_ipv6);
+  last_recorded_accounting_at = 0;
+
+  mainloop_event_free(reenable_blocked_connections_ev);
+  reenable_blocked_connections_is_scheduled = 0;
+  memset(&reenable_blocked_connections_delay, 0, sizeof(struct timeval));
 }
 
 /** Log a warning, and possibly emit a control event, that <b>received</b> came
@@ -5149,13 +5478,13 @@ connection_free_all(void)
  * that we had more faith in and therefore the warning level should have higher
  * severity.
  */
-void
-clock_skew_warning(const connection_t *conn, long apparent_skew, int trusted,
-                   log_domain_mask_t domain, const char *received,
-                   const char *source)
+MOCK_IMPL(void,
+clock_skew_warning, (const connection_t *conn, long apparent_skew, int trusted,
+                     log_domain_mask_t domain, const char *received,
+                     const char *source))
 {
   char dbuf[64];
-  char *ext_source = NULL;
+  char *ext_source = NULL, *warn = NULL;
   format_time_interval(dbuf, sizeof(dbuf), apparent_skew);
   if (conn)
     tor_asprintf(&ext_source, "%s:%s:%d", source, conn->address, conn->port);
@@ -5169,9 +5498,13 @@ clock_skew_warning(const connection_t *conn, long apparent_skew, int trusted,
          apparent_skew > 0 ? "ahead" : "behind", dbuf,
          apparent_skew > 0 ? "behind" : "ahead",
          (!conn || trusted) ? "" : ", or they are sending us the wrong time");
-  if (trusted)
+  if (trusted) {
     control_event_general_status(LOG_WARN, "CLOCK_SKEW SKEW=%ld SOURCE=%s",
                                  apparent_skew, ext_source);
+    tor_asprintf(&warn, "Clock skew %ld in %s from %s", apparent_skew,
+                 received, source);
+    control_event_bootstrap_problem(warn, "CLOCK_SKEW", conn, 1);
+  }
+  tor_free(warn);
   tor_free(ext_source);
 }
-
diff --git a/src/or/connection.h b/src/core/mainloop/connection.h
index d25e002fa4..8ecdd6f06f 100644
--- a/src/or/connection.h
+++ b/src/core/mainloop/connection.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,8 +12,85 @@
 #ifndef TOR_CONNECTION_H
 #define TOR_CONNECTION_H
 
-/* XXXX For buf_datalen in inline function */
-#include "buffers.h"
+listener_connection_t *TO_LISTENER_CONN(connection_t *);
+
+struct buf_t;
+
+#define CONN_TYPE_MIN_ 3
+/** Type for sockets listening for OR connections. */
+#define CONN_TYPE_OR_LISTENER 3
+/** A bidirectional TLS connection transmitting a sequence of cells.
+ * May be from an OR to an OR, or from an OP to an OR. */
+#define CONN_TYPE_OR 4
+/** A TCP connection from an onion router to a stream's destination. */
+#define CONN_TYPE_EXIT 5
+/** Type for sockets listening for SOCKS connections. */
+#define CONN_TYPE_AP_LISTENER 6
+/** A SOCKS proxy connection from the user application to the onion
+ * proxy. */
+#define CONN_TYPE_AP 7
+/** Type for sockets listening for HTTP connections to the directory server. */
+#define CONN_TYPE_DIR_LISTENER 8
+/** Type for HTTP connections to the directory server. */
+#define CONN_TYPE_DIR 9
+/* Type 10 is unused. */
+/** Type for listening for connections from user interface process. */
+#define CONN_TYPE_CONTROL_LISTENER 11
+/** Type for connections from user interface process. */
+#define CONN_TYPE_CONTROL 12
+/** Type for sockets listening for transparent connections redirected by pf or
+ * netfilter. */
+#define CONN_TYPE_AP_TRANS_LISTENER 13
+/** Type for sockets listening for transparent connections redirected by
+ * natd. */
+#define CONN_TYPE_AP_NATD_LISTENER 14
+/** Type for sockets listening for DNS requests. */
+#define CONN_TYPE_AP_DNS_LISTENER 15
+
+/** Type for connections from the Extended ORPort. */
+#define CONN_TYPE_EXT_OR 16
+/** Type for sockets listening for Extended ORPort connections. */
+#define CONN_TYPE_EXT_OR_LISTENER 17
+/** Type for sockets listening for HTTP CONNECT tunnel connections. */
+#define CONN_TYPE_AP_HTTP_CONNECT_LISTENER 18
+
+#define CONN_TYPE_MAX_ 19
+/* !!!! If _CONN_TYPE_MAX is ever over 31, we must grow the type field in
+ * connection_t. */
+
+/* Proxy client handshake states */
+/* We use a proxy but we haven't even connected to it yet. */
+#define PROXY_INFANT 1
+/* We use an HTTP proxy and we've sent the CONNECT command. */
+#define PROXY_HTTPS_WANT_CONNECT_OK 2
+/* We use a SOCKS4 proxy and we've sent the CONNECT command. */
+#define PROXY_SOCKS4_WANT_CONNECT_OK 3
+/* We use a SOCKS5 proxy and we try to negotiate without
+   any authentication . */
+#define PROXY_SOCKS5_WANT_AUTH_METHOD_NONE 4
+/* We use a SOCKS5 proxy and we try to negotiate with
+   Username/Password authentication . */
+#define PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929 5
+/* We use a SOCKS5 proxy and we just sent our credentials. */
+#define PROXY_SOCKS5_WANT_AUTH_RFC1929_OK 6
+/* We use a SOCKS5 proxy and we just sent our CONNECT command. */
+#define PROXY_SOCKS5_WANT_CONNECT_OK 7
+/* We use a proxy and we CONNECTed successfully!. */
+#define PROXY_CONNECTED 8
+
+/** State for any listener connection. */
+#define LISTENER_STATE_READY 0
+
+/**
+ * This struct associates an old listener connection to be replaced
+ * by new connection described by port configuration. Only used when
+ * moving listeners to/from wildcard IP address.
+ */
+typedef struct
+{
+  connection_t *old_conn; /* Old listener connection to be replaced */
+  const port_cfg_t *new_port; /* New port configuration */
+} listener_replacement_t;
 
 const char *conn_type_to_string(int type);
 const char *conn_state_to_string(int type, int state);
@@ -26,9 +103,12 @@ entry_connection_t *entry_connection_new(int type, int socket_family);
 control_connection_t *control_connection_new(int socket_family);
 listener_connection_t *listener_connection_new(int type, int socket_family);
 connection_t *connection_new(int type, int socket_family);
-
+int connection_init_accepted_conn(connection_t *conn,
+                                  const listener_connection_t *listener);
 void connection_link_connections(connection_t *conn_a, connection_t *conn_b);
-MOCK_DECL(void,connection_free,(connection_t *conn));
+MOCK_DECL(void,connection_free_,(connection_t *conn));
+#define connection_free(conn) \
+  FREE_AND_NULL(connection_t, connection_free_, (conn))
 void connection_free_all(void);
 void connection_about_to_close_connection(connection_t *conn);
 void connection_close_immediate(connection_t *conn);
@@ -109,8 +189,7 @@ void log_failed_proxy_connection(connection_t *conn);
 int get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type,
                        const connection_t *conn);
 
-int retry_all_listeners(smartlist_t *replaced_conns,
-                        smartlist_t *new_conns,
+int retry_all_listeners(smartlist_t *new_conns,
                         int close_all_noncontrol);
 
 void connection_mark_all_noncontrol_listeners(void);
@@ -119,12 +198,18 @@ void connection_mark_all_noncontrol_connections(void);
 ssize_t connection_bucket_write_limit(connection_t *conn, time_t now);
 int global_write_bucket_low(connection_t *conn, size_t attempt, int priority);
 void connection_bucket_init(void);
-void connection_bucket_refill(int seconds_elapsed, time_t now);
+void connection_bucket_adjust(const or_options_t *options);
+void connection_bucket_refill_all(time_t now,
+                                  uint32_t now_ts);
+void connection_read_bw_exhausted(connection_t *conn, bool is_global_bw);
+void connection_write_bw_exhausted(connection_t *conn, bool is_global_bw);
+void connection_consider_empty_read_buckets(connection_t *conn);
+void connection_consider_empty_write_buckets(connection_t *conn);
 
 int connection_handle_read(connection_t *conn);
 
-int connection_fetch_from_buf(char *string, size_t len, connection_t *conn);
-int connection_fetch_from_buf_line(connection_t *conn, char *data,
+int connection_buf_get_bytes(char *string, size_t len, connection_t *conn);
+int connection_buf_get_line(connection_t *conn, char *data,
                                    size_t *data_len);
 int connection_fetch_from_buf_http(connection_t *conn,
                                char **headers_out, size_t max_headerlen,
@@ -139,40 +224,19 @@ int connection_flush(connection_t *conn);
 MOCK_DECL(void, connection_write_to_buf_impl_,
           (const char *string, size_t len, connection_t *conn, int zlib));
 /* DOCDOC connection_write_to_buf */
-static void connection_write_to_buf(const char *string, size_t len,
+static void connection_buf_add(const char *string, size_t len,
                                     connection_t *conn);
-/* DOCDOC connection_write_to_buf_zlib */
-static void connection_write_to_buf_zlib(const char *string, size_t len,
-                                         dir_connection_t *conn, int done);
 static inline void
-connection_write_to_buf(const char *string, size_t len, connection_t *conn)
+connection_buf_add(const char *string, size_t len, connection_t *conn)
 {
   connection_write_to_buf_impl_(string, len, conn, 0);
 }
-static inline void
-connection_write_to_buf_zlib(const char *string, size_t len,
-                             dir_connection_t *conn, int done)
-{
-  connection_write_to_buf_impl_(string, len, TO_CONN(conn), done ? -1 : 1);
-}
-
-/* DOCDOC connection_get_inbuf_len */
-static size_t connection_get_inbuf_len(connection_t *conn);
-/* DOCDOC connection_get_outbuf_len */
-static size_t connection_get_outbuf_len(connection_t *conn);
-
-static inline size_t
-connection_get_inbuf_len(connection_t *conn)
-{
-  return conn->inbuf ? buf_datalen(conn->inbuf) : 0;
-}
-
-static inline size_t
-connection_get_outbuf_len(connection_t *conn)
-{
-    return conn->outbuf ? buf_datalen(conn->outbuf) : 0;
-}
+void connection_buf_add_compress(const char *string, size_t len,
+                                 dir_connection_t *conn, int done);
+void connection_buf_add_buf(connection_t *conn, struct buf_t *buf);
 
+size_t connection_get_inbuf_len(connection_t *conn);
+size_t connection_get_outbuf_len(connection_t *conn);
 connection_t *connection_get_by_global_id(uint64_t id);
 
 connection_t *connection_get_by_type(int type);
@@ -182,6 +246,8 @@ MOCK_DECL(connection_t *,connection_get_by_type_addr_port_purpose,(int type,
 connection_t *connection_get_by_type_state(int type, int state);
 connection_t *connection_get_by_type_state_rendquery(int type, int state,
                                                      const char *rendquery);
+smartlist_t *connection_list_by_type_state(int type, int state);
+smartlist_t *connection_list_by_type_purpose(int type, int purpose);
 smartlist_t *connection_dir_list_by_purpose_and_resource(
                                                   int purpose,
                                                   const char *resource);
@@ -241,39 +307,37 @@ char *alloc_http_authenticator(const char *authenticator);
 void assert_connection_ok(connection_t *conn, time_t now);
 int connection_or_nonopen_was_started_here(or_connection_t *conn);
 void connection_dump_buffer_mem_stats(int severity);
-void remove_file_if_very_old(const char *fname, time_t now);
 
-void clock_skew_warning(const connection_t *conn, long apparent_skew,
-                        int trusted, log_domain_mask_t domain,
-                        const char *received, const char *source);
-
-/** Check if a connection is on the way out so the OOS handler doesn't try
- * to kill more than it needs. */
-static inline int
-connection_is_moribund(connection_t *conn)
-{
-  if (conn != NULL &&
-      (conn->conn_array_index < 0 ||
-       conn->marked_for_close)) {
-    return 1;
-  } else {
-    return 0;
-  }
-}
+MOCK_DECL(void, clock_skew_warning,
+          (const connection_t *conn, long apparent_skew, int trusted,
+           log_domain_mask_t domain, const char *received,
+           const char *source));
 
+int connection_is_moribund(connection_t *conn);
 void connection_check_oos(int n_socks, int failed);
 
+/** Execute the statement <b>stmt</b>, which may log events concerning the
+ * connection <b>conn</b>.  To prevent infinite loops, disable log messages
+ * being sent to controllers if <b>conn</b> is a control connection.
+ *
+ * Stmt must not contain any return or goto statements.
+ */
+#define CONN_LOG_PROTECT(conn, stmt)                                    \
+  STMT_BEGIN                                                            \
+    int _log_conn_is_control;                                           \
+    tor_assert(conn);                                                   \
+    _log_conn_is_control = (conn->type == CONN_TYPE_CONTROL);           \
+    if (_log_conn_is_control)                                           \
+      disable_control_logging();                                        \
+  STMT_BEGIN stmt; STMT_END;                                            \
+    if (_log_conn_is_control)                                           \
+      enable_control_logging();                                         \
+  STMT_END
+
 #ifdef CONNECTION_PRIVATE
-STATIC void connection_free_(connection_t *conn);
+STATIC void connection_free_minimal(connection_t *conn);
 
 /* Used only by connection.c and test*.c */
-uint32_t bucket_millis_empty(int tokens_before, uint32_t last_empty_time,
-                             int tokens_after, int milliseconds_elapsed,
-                             const struct timeval *tvnow);
-void connection_buckets_note_empty_ts(uint32_t *timestamp_var,
-                                      int tokens_before,
-                                      size_t tokens_removed,
-                                      const struct timeval *tvnow);
 MOCK_DECL(STATIC int,connection_connect_sockaddr,
                                             (connection_t *conn,
                                              const struct sockaddr *sa,
@@ -284,7 +348,6 @@ MOCK_DECL(STATIC int,connection_connect_sockaddr,
 MOCK_DECL(STATIC void, kill_conn_list_for_oos, (smartlist_t *conns));
 MOCK_DECL(STATIC smartlist_t *, pick_oos_victims, (int n));
 
-#endif
-
-#endif
+#endif /* defined(CONNECTION_PRIVATE) */
 
+#endif /* !defined(TOR_CONNECTION_H) */
diff --git a/src/or/cpuworker.c b/src/core/mainloop/cpuworker.c
index fd6de6ea7c..e704d55642 100644
--- a/src/or/cpuworker.c
+++ b/src/core/mainloop/cpuworker.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,23 +11,30 @@
  * The multithreading backend for this module is in workqueue.c; this module
  * specializes workqueue.c.
  *
- * Right now, we only use this for processing onionskins, and invoke it mostly
- * from onion.c.
+ * Right now, we use this infrastructure
+ *  <ul><li>for processing onionskins in onion.c
+ *      <li>for compressing consensuses in consdiffmgr.c,
+ *      <li>and for calculating diffs and compressing them in consdiffmgr.c.
+ *  </ul>
  **/
-#include "or.h"
-#include "channel.h"
-#include "circuitbuild.h"
-#include "circuitlist.h"
-#include "connection_or.h"
-#include "config.h"
-#include "cpuworker.h"
-#include "main.h"
-#include "onion.h"
-#include "rephist.h"
-#include "router.h"
-#include "workqueue.h"
-
-#include <event2/event.h>
+#include "core/or/or.h"
+#include "core/or/channel.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/connection_or.h"
+#include "app/config/config.h"
+#include "core/mainloop/cpuworker.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "core/or/onion.h"
+#include "feature/relay/onion_queue.h"
+#include "feature/stats/rephist.h"
+#include "feature/relay/router.h"
+#include "lib/evloop/workqueue.h"
+#include "core/crypto/onion_crypto.h"
+
+#include "core/or/or_circuit_st.h"
+#include "lib/intmath/weakrng.h"
 
 static void queue_pending_tasks(void);
 
@@ -45,32 +52,33 @@ worker_state_new(void *arg)
   ws->onion_keys = server_onion_keys_new();
   return ws;
 }
+
+#define worker_state_free(ws) \
+  FREE_AND_NULL(worker_state_t, worker_state_free_, (ws))
+
 static void
-worker_state_free(void *arg)
+worker_state_free_(worker_state_t *ws)
 {
-  worker_state_t *ws = arg;
+  if (!ws)
+    return;
   server_onion_keys_free(ws->onion_keys);
   tor_free(ws);
 }
 
+static void
+worker_state_free_void(void *arg)
+{
+  worker_state_free_(arg);
+}
+
 static replyqueue_t *replyqueue = NULL;
 static threadpool_t *threadpool = NULL;
-static struct event *reply_event = NULL;
 
 static tor_weak_rng_t request_sample_rng = TOR_WEAK_RNG_INIT;
 
 static int total_pending_tasks = 0;
 static int max_pending_tasks = 128;
 
-static void
-replyqueue_process_cb(evutil_socket_t sock, short events, void *arg)
-{
-  replyqueue_t *rq = arg;
-  (void) sock;
-  (void) events;
-  replyqueue_process(rq);
-}
-
 /** Initialize the cpuworker subsystem. It is OK to call this more than once
  * during Tor's lifetime.
  */
@@ -80,21 +88,25 @@ cpu_init(void)
   if (!replyqueue) {
     replyqueue = replyqueue_new(0);
   }
-  if (!reply_event) {
-    reply_event = tor_event_new(tor_libevent_get_base(),
-                                replyqueue_get_socket(replyqueue),
-                                EV_READ|EV_PERSIST,
-                                replyqueue_process_cb,
-                                replyqueue);
-    event_add(reply_event, NULL);
-  }
   if (!threadpool) {
-    threadpool = threadpool_new(get_num_cpus(get_options()),
+    /*
+      In our threadpool implementation, half the threads are permissive and
+      half are strict (when it comes to running lower-priority tasks). So we
+      always make sure we have at least two threads, so that there will be at
+      least one thread of each kind.
+    */
+    const int n_threads = get_num_cpus(get_options()) + 1;
+    threadpool = threadpool_new(n_threads,
                                 replyqueue,
                                 worker_state_new,
-                                worker_state_free,
+                                worker_state_free_void,
                                 NULL);
+
+    int r = threadpool_register_reply_event(threadpool, NULL);
+
+    tor_assert(r == 0);
   }
+
   /* Total voodoo. Can we make this more sensible? */
   max_pending_tasks = get_num_cpus(get_options()) * 64;
   crypto_seed_weak_rng(&request_sample_rng);
@@ -188,7 +200,7 @@ cpuworkers_rotate_keyinfo(void)
   if (threadpool_queue_update(threadpool,
                               worker_state_new,
                               update_state_threadfn,
-                              worker_state_free,
+                              worker_state_free_void,
                               NULL)) {
     log_warn(LD_OR, "Failed to queue key update for worker threads.");
   }
@@ -229,7 +241,7 @@ should_time_request(uint16_t onionskin_type)
 }
 
 /** Return an estimate of how many microseconds we will need for a single
- * cpuworker to to process <b>n_requests</b> onionskins of type
+ * cpuworker to process <b>n_requests</b> onionskins of type
  * <b>onionskin_type</b>. */
 uint64_t
 estimated_usec_for_onionskins(uint32_t n_requests, uint16_t onionskin_type)
@@ -272,7 +284,7 @@ get_overhead_for_onionskins(uint32_t *usec_out, double *frac_out,
     onionskins_usec_internal[onionskin_type];
 
   *usec_out = (uint32_t)(overhead / onionskins_n_processed[onionskin_type]);
-  *frac_out = U64_TO_DBL(overhead) / onionskins_usec_internal[onionskin_type];
+  *frac_out = ((double)overhead) / onionskins_usec_internal[onionskin_type];
 
   return 0;
 }
@@ -372,7 +384,7 @@ cpuworker_onion_handshake_replyfn(void *work_)
 
   if (onionskin_answer(circ,
                        &rpl.created_cell,
-                       (const char*)rpl.keys,
+                       (const char*)rpl.keys, sizeof(rpl.keys),
                        rpl.rend_auth_material) < 0) {
     log_warn(LD_OR,"onionskin_answer failed. Closing.");
     circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
@@ -474,11 +486,27 @@ queue_pending_tasks(void)
     if (!circ)
       return;
 
-    if (assign_onionskin_to_cpuworker(circ, onionskin))
-      log_warn(LD_OR,"assign_to_cpuworker failed. Ignoring.");
+    if (assign_onionskin_to_cpuworker(circ, onionskin) < 0)
+      log_info(LD_OR,"assign_to_cpuworker failed. Ignoring.");
   }
 }
 
+/** DOCDOC */
+MOCK_IMPL(workqueue_entry_t *,
+cpuworker_queue_work,(workqueue_priority_t priority,
+                      workqueue_reply_t (*fn)(void *, void *),
+                      void (*reply_fn)(void *),
+                      void *arg))
+{
+  tor_assert(threadpool);
+
+  return threadpool_queue_work_priority(threadpool,
+                                        priority,
+                                        fn,
+                                        reply_fn,
+                                        arg);
+}
+
 /** Try to tell a cpuworker to perform the public key operations necessary to
  * respond to <b>onionskin</b> for the circuit <b>circ</b>.
  *
@@ -510,7 +538,7 @@ assign_onionskin_to_cpuworker(or_circuit_t *circ,
     return 0;
   }
 
-  if (connection_or_digest_is_known_relay(circ->p_chan->identity_digest))
+  if (!channel_is_client(circ->p_chan))
     rep_hist_note_circuit_handshake_assigned(onionskin->handshake_type);
 
   should_time = should_time_request(onionskin->handshake_type);
@@ -531,7 +559,8 @@ assign_onionskin_to_cpuworker(or_circuit_t *circ,
   memwipe(&req, 0, sizeof(req));
 
   ++total_pending_tasks;
-  queue_entry = threadpool_queue_work(threadpool,
+  queue_entry = threadpool_queue_work_priority(threadpool,
+                                      WQ_PRI_HIGH,
                                       cpuworker_onion_handshake_threadfn,
                                       cpuworker_onion_handshake_replyfn,
                                       job);
@@ -569,4 +598,3 @@ cpuworker_cancel_circ_handshake(or_circuit_t *circ)
     circ->workqueue_entry = NULL;
   }
 }
-
diff --git a/src/or/cpuworker.h b/src/core/mainloop/cpuworker.h
index 62cf0eb164..77e2c42508 100644
--- a/src/or/cpuworker.h
+++ b/src/core/mainloop/cpuworker.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,6 +14,14 @@
 
 void cpu_init(void);
 void cpuworkers_rotate_keyinfo(void);
+struct workqueue_entry_s;
+enum workqueue_reply_t;
+enum workqueue_priority_t;
+MOCK_DECL(struct workqueue_entry_s *, cpuworker_queue_work, (
+                    enum workqueue_priority_t priority,
+                    enum workqueue_reply_t (*fn)(void *, void *),
+                    void (*reply_fn)(void *),
+                    void *arg));
 
 struct create_cell_t;
 int assign_onionskin_to_cpuworker(or_circuit_t *circ,
@@ -25,5 +33,5 @@ void cpuworker_log_onionskin_overhead(int severity, int onionskin_type,
                                       const char *onionskin_type_name);
 void cpuworker_cancel_circ_handshake(or_circuit_t *circ);
 
-#endif
+#endif /* !defined(TOR_CPUWORKER_H) */
 
diff --git a/src/or/main.c b/src/core/mainloop/mainloop.c
index 55bcb18c4b..4b3c3bf6af 100644
--- a/src/or/main.c
+++ b/src/core/mainloop/mainloop.c
@@ -1,122 +1,140 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file main.c
+ * \file mainloop.c
  * \brief Toplevel module. Handles signals, multiplexes between
- * connections, implements main loop, and drives scheduled events.
+ *     connections, implements main loop, and drives scheduled events.
+ *
+ * For the main loop itself; see run_main_loop_once().  It invokes the rest of
+ * Tor mostly through Libevent callbacks.  Libevent callbacks can happen when
+ * a timer elapses, a signal is received, a socket is ready to read or write,
+ * or an event is manually activated.
+ *
+ * Most events in Tor are driven from these callbacks:
+ *  <ul>
+ *   <li>conn_read_callback() and conn_write_callback() here, which are
+ *     invoked when a socket is ready to read or write respectively.
+ *   <li>signal_callback(), which handles incoming signals.
+ *  </ul>
+ * Other events are used for specific purposes, or for building more complex
+ * control structures.  If you search for usage of tor_libevent_new(), you
+ * will find all the events that we construct in Tor.
+ *
+ * Tor has numerous housekeeping operations that need to happen
+ * regularly. They are handled in different ways:
+ * <ul>
+ *   <li>The most frequent operations are handled after every read or write
+ *    event, at the end of connection_handle_read() and
+ *    connection_handle_write().
+ *
+ *   <li>The next most frequent operations happen after each invocation of the
+ *     main loop, in run_main_loop_once().
+ *
+ *   <li>Once per second, we run all of the operations listed in
+ *     second_elapsed_callback(), and in its child, run_scheduled_events().
+ *
+ *   <li>Once-a-second operations are handled in second_elapsed_callback().
+ *
+ *   <li>More infrequent operations take place based on the periodic event
+ *     driver in periodic.c .  These are stored in the periodic_events[]
+ *     table.
+ * </ul>
+ *
  **/
 
-#define MAIN_PRIVATE
-#include "or.h"
-#include "addressmap.h"
-#include "backtrace.h"
-#include "buffers.h"
-#include "channel.h"
-#include "channeltls.h"
-#include "circuitbuild.h"
-#include "circuitlist.h"
-#include "circuituse.h"
-#include "command.h"
-#include "config.h"
-#include "confparse.h"
-#include "connection.h"
-#include "connection_edge.h"
-#include "connection_or.h"
-#include "control.h"
-#include "cpuworker.h"
-#include "crypto_s2k.h"
-#include "directory.h"
-#include "dirserv.h"
-#include "dirvote.h"
-#include "dns.h"
-#include "dnsserv.h"
-#include "dos.h"
-#include "entrynodes.h"
-#include "geoip.h"
-#include "hibernate.h"
-#include "keypin.h"
-#include "main.h"
-#include "microdesc.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "ntmain.h"
-#include "onion.h"
-#include "periodic.h"
-#include "policies.h"
-#include "protover.h"
-#include "transports.h"
-#include "relay.h"
-#include "rendclient.h"
-#include "rendcommon.h"
-#include "rendservice.h"
-#include "rephist.h"
-#include "router.h"
-#include "routerkeys.h"
-#include "routerlist.h"
-#include "routerparse.h"
-#include "scheduler.h"
-#include "shared_random.h"
-#include "statefile.h"
-#include "status.h"
-#include "util_process.h"
-#include "ext_orport.h"
-#ifdef USE_DMALLOC
-#include <dmalloc.h>
-#include <openssl/crypto.h>
-#endif
-#include "memarea.h"
-#include "sandbox.h"
+#define MAINLOOP_PRIVATE
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "app/config/statefile.h"
+#include "app/main/ntmain.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/cpuworker.h"
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
+#include "core/mainloop/periodic.h"
+#include "core/or/channel.h"
+#include "core/or/channelpadding.h"
+#include "core/or/channeltls.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "core/or/dos.h"
+#include "core/or/status.h"
+#include "feature/client/addressmap.h"
+#include "feature/client/bridges.h"
+#include "feature/client/dnsserv.h"
+#include "feature/client/entrynodes.h"
+#include "feature/client/transports.h"
+#include "feature/control/control.h"
+#include "feature/dirauth/authmode.h"
+#include "feature/dirauth/reachability.h"
+#include "feature/dircache/consdiffmgr.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dircommon/directory.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/hs/hs_cache.h"
+#include "feature/hs/hs_client.h"
+#include "feature/hs/hs_service.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/dns.h"
+#include "feature/relay/routerkeys.h"
+#include "feature/relay/routermode.h"
+#include "feature/relay/selftest.h"
+#include "feature/rend/rendcache.h"
+#include "feature/rend/rendservice.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/stats/predict_ports.h"
+#include "feature/stats/rephist.h"
+#include "lib/container/buffers.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/err/backtrace.h"
+#include "lib/tls/buffers_tls.h"
+
+#include "lib/net/buffers_net.h"
+#include "lib/evloop/compat_libevent.h"
 
 #include <event2/event.h>
 
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/authmode.h"
+
+#include "core/or/cell_st.h"
+#include "core/or/entry_connection_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "core/or/or_connection_st.h"
+#include "app/config/or_state_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "core/or/socks_request_st.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
 #ifdef HAVE_SYSTEMD
 #   if defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__)
 /* Systemd's use of gcc's __INCLUDE_LEVEL__ extension macro appears to confuse
  * Coverity. Here's a kludge to unconfuse it.
  */
 #   define __INCLUDE_LEVEL__ 2
-#   endif
+#endif /* defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) */
 #include <systemd/sd-daemon.h>
-#endif
+#endif /* defined(HAVE_SYSTEMD) */
 
-void evdns_shutdown(int);
+/* Token bucket for all traffic. */
+token_bucket_rw_t global_bucket;
 
-/********* PROTOTYPES **********/
-
-static void dumpmemusage(int severity);
-static void dumpstats(int severity); /* log stats */
-static void conn_read_callback(evutil_socket_t fd, short event, void *_conn);
-static void conn_write_callback(evutil_socket_t fd, short event, void *_conn);
-static void second_elapsed_callback(periodic_timer_t *timer, void *args);
-static int conn_close_if_marked(int i);
-static void connection_start_reading_from_linked_conn(connection_t *conn);
-static int connection_should_read_from_linked_conn(connection_t *conn);
-static int run_main_loop_until_done(void);
-static void process_signal(int sig);
-
-/********* START VARIABLES **********/
-int global_read_bucket; /**< Max number of bytes I can read this second. */
-int global_write_bucket; /**< Max number of bytes I can write this second. */
-
-/** Max number of relayed (bandwidth class 1) bytes I can read this second. */
-int global_relayed_read_bucket;
-/** Max number of relayed (bandwidth class 1) bytes I can write this second. */
-int global_relayed_write_bucket;
-/** What was the read bucket before the last second_elapsed_callback() call?
- * (used to determine how many bytes we've read). */
-static int stats_prev_global_read_bucket;
-/** What was the write bucket before the last second_elapsed_callback() call?
- * (used to determine how many bytes we've written). */
-static int stats_prev_global_write_bucket;
-
-/* DOCDOC stats_prev_n_read */
-static uint64_t stats_prev_n_read = 0;
-/* DOCDOC stats_prev_n_written */
-static uint64_t stats_prev_n_written = 0;
+/* Token bucket for relayed traffic. */
+token_bucket_rw_t global_relayed_bucket;
 
 /* XXX we might want to keep stats about global_relayed_*_bucket too. Or not.*/
 /** How many bytes have we read since we started the process? */
@@ -126,7 +144,13 @@ static uint64_t stats_n_bytes_written = 0;
 /** What time did this process start up? */
 time_t time_of_process_start = 0;
 /** How many seconds have we been running? */
-long stats_n_seconds_working = 0;
+static long stats_n_seconds_working = 0;
+/** How many times have we returned from the main loop successfully? */
+static uint64_t stats_n_main_loop_successes = 0;
+/** How many times have we received an error from the main loop? */
+static uint64_t stats_n_main_loop_errors = 0;
+/** How many times have we returned from the main loop with no events. */
+static uint64_t stats_n_main_loop_idle = 0;
 
 /** How often will we honor SIGNEWNYM requests? */
 #define MAX_SIGNEWNYM_RATE 10
@@ -134,11 +158,13 @@ long stats_n_seconds_working = 0;
 static time_t time_of_last_signewnym = 0;
 /** Is there a signewnym request we're currently waiting to handle? */
 static int signewnym_is_pending = 0;
+/** Mainloop event for the deferred signewnym call. */
+static mainloop_event_t *handle_deferred_signewnym_ev = NULL;
 /** How many times have we called newnym? */
 static unsigned newnym_epoch = 0;
 
 /** Smartlist of all open connections. */
-static smartlist_t *connection_array = NULL;
+STATIC smartlist_t *connection_array = NULL;
 /** List of connections that have been marked for close and need to be freed
  * and removed from connection_array. */
 static smartlist_t *closeable_connection_lst = NULL;
@@ -149,6 +175,14 @@ static smartlist_t *active_linked_connection_lst = NULL;
  * <b>loop_once</b>. If so, there's no need to trigger a loopexit in order
  * to handle linked connections. */
 static int called_loop_once = 0;
+/** Flag: if true, it's time to shut down, so the main loop should exit as
+ * soon as possible.
+ */
+static int main_loop_should_exit = 0;
+/** The return value that the main loop should yield when it exits, if
+ * main_loop_should_exit is true.
+ */
+static int main_loop_exit_value = 0;
 
 /** We set this to 1 when we've opened a circuit, so we can print a log
  * entry to inform the user that Tor is working.  We set it to 0 when
@@ -165,13 +199,15 @@ static int can_complete_circuits = 0;
  * when we have enough directory info? */
 #define LAZY_DESCRIPTOR_RETRY_INTERVAL (60)
 
-/** Decides our behavior when no logs are configured/before any
- * logs have been configured.  For 0, we log notice to stdout as normal.
- * For 1, we log warnings only.  For 2, we log nothing.
- */
-int quiet_level = 0;
-
-/********* END VARIABLES ************/
+static int conn_close_if_marked(int i);
+static int run_main_loop_until_done(void);
+static void connection_start_reading_from_linked_conn(connection_t *conn);
+static int connection_should_read_from_linked_conn(connection_t *conn);
+static void conn_read_callback(evutil_socket_t fd, short event, void *_conn);
+static void conn_write_callback(evutil_socket_t fd, short event, void *_conn);
+static void second_elapsed_callback(periodic_timer_t *timer, void *args);
+static void shutdown_did_not_work_callback(evutil_socket_t fd, short event,
+                                           void *arg) ATTR_NORETURN;
 
 /****************************************************************************
  *
@@ -275,7 +311,7 @@ connection_remove(connection_t *conn)
             smartlist_len(connection_array));
 
   if (conn->type == CONN_TYPE_AP && conn->socket_family == AF_UNIX) {
-    log_info(LD_NET, "Closing SOCKS SocksSocket connection");
+    log_info(LD_NET, "Closing SOCKS Unix socket connection");
   }
 
   control_event_conn_bandwidth(conn);
@@ -326,7 +362,7 @@ connection_unlink(connection_t *conn)
   }
   if (conn->type == CONN_TYPE_OR) {
     if (!tor_digest_is_zero(TO_OR_CONN(conn)->identity_digest))
-      connection_or_remove_from_identity_map(TO_OR_CONN(conn));
+      connection_or_clear_identity(TO_OR_CONN(conn));
     /* connection_unlink() can only get called if the connection
      * was already on the closeable list, and it got there by
      * connection_mark_for_close(), which was called from
@@ -343,10 +379,42 @@ connection_unlink(connection_t *conn)
   connection_free(conn);
 }
 
+/** Event that invokes schedule_active_linked_connections_cb. */
+static mainloop_event_t *schedule_active_linked_connections_event = NULL;
+
+/**
+ * Callback: used to activate read events for all linked connections, so
+ * libevent knows to call their read callbacks.  This callback run as a
+ * postloop event, so that the events _it_ activates don't happen until
+ * Libevent has a chance to check for other events.
+ */
+static void
+schedule_active_linked_connections_cb(mainloop_event_t *event, void *arg)
+{
+  (void)event;
+  (void)arg;
+
+  /* All active linked conns should get their read events activated,
+   * so that libevent knows to run their callbacks. */
+  SMARTLIST_FOREACH(active_linked_connection_lst, connection_t *, conn,
+                    event_active(conn->read_event, EV_READ, 1));
+
+  /* Reactivate the event if we still have connections in the active list.
+   *
+   * A linked connection doesn't get woken up by I/O but rather artificially
+   * by this event callback. It has directory data spooled in it and it is
+   * sent incrementally by small chunks unless spool_eagerly is true. For that
+   * to happen, we need to induce the activation of the read event so it can
+   * be flushed. */
+  if (smartlist_len(active_linked_connection_lst)) {
+    mainloop_event_activate(schedule_active_linked_connections_event);
+  }
+}
+
 /** Initialize the global connection list, closeable connection list,
  * and active connection list. */
-STATIC void
-init_connection_lists(void)
+void
+tor_init_connection_lists(void)
 {
   if (!connection_array)
     connection_array = smartlist_new();
@@ -364,6 +432,7 @@ add_connection_to_closeable_list(connection_t *conn)
   tor_assert(conn->marked_for_close);
   assert_connection_ok(conn, time(NULL));
   smartlist_add(closeable_connection_lst, conn);
+  mainloop_schedule_postloop_cleanup();
 }
 
 /** Return 1 if conn is on the closeable list, else return 0. */
@@ -391,21 +460,37 @@ get_connection_array, (void))
   return connection_array;
 }
 
-/** Provides the traffic read and written over the life of the process. */
-
+/**
+ * Return the amount of network traffic read, in bytes, over the life of this
+ * process.
+ */
 MOCK_IMPL(uint64_t,
 get_bytes_read,(void))
 {
   return stats_n_bytes_read;
 }
 
-/* DOCDOC get_bytes_written */
+/**
+ * Return the amount of network traffic read, in bytes, over the life of this
+ * process.
+ */
 MOCK_IMPL(uint64_t,
 get_bytes_written,(void))
 {
   return stats_n_bytes_written;
 }
 
+/**
+ * Increment the amount of network traffic read and written, over the life of
+ * this process.
+ */
+void
+stats_increment_bytes_read_and_written(uint64_t r, uint64_t w)
+{
+  stats_n_bytes_read += r;
+  stats_n_bytes_written += w;
+}
+
 /** Set the event mask on <b>conn</b> to <b>events</b>.  (The event
  * mask is a bitmask whose bits are READ_EVENT and WRITE_EVENT)
  */
@@ -433,6 +518,57 @@ connection_is_reading(connection_t *conn)
     (conn->read_event && event_pending(conn->read_event, EV_READ, NULL));
 }
 
+/** Reset our main loop counters. */
+void
+reset_main_loop_counters(void)
+{
+  stats_n_main_loop_successes = 0;
+  stats_n_main_loop_errors = 0;
+  stats_n_main_loop_idle = 0;
+}
+
+/** Increment the main loop success counter. */
+static void
+increment_main_loop_success_count(void)
+{
+  ++stats_n_main_loop_successes;
+}
+
+/** Get the main loop success counter. */
+uint64_t
+get_main_loop_success_count(void)
+{
+  return stats_n_main_loop_successes;
+}
+
+/** Increment the main loop error counter. */
+static void
+increment_main_loop_error_count(void)
+{
+  ++stats_n_main_loop_errors;
+}
+
+/** Get the main loop error counter. */
+uint64_t
+get_main_loop_error_count(void)
+{
+  return stats_n_main_loop_errors;
+}
+
+/** Increment the main loop idle counter. */
+static void
+increment_main_loop_idle_count(void)
+{
+  ++stats_n_main_loop_idle;
+}
+
+/** Get the main loop idle counter. */
+uint64_t
+get_main_loop_idle_count(void)
+{
+  return stats_n_main_loop_idle;
+}
+
 /** Check whether <b>conn</b> is correct in having (or not having) a
  * read/write event (passed in <b>ev</b>). On success, return 0. On failure,
  * log a warning and return -1. */
@@ -448,7 +584,7 @@ connection_check_event(connection_t *conn, struct event *ev)
      */
     bad = ev != NULL;
   } else {
-    /* Everytyhing else should have an underlying socket, or a linked
+    /* Everything else should have an underlying socket, or a linked
      * connection (which is also tracked with a read_event/write_event pair).
      */
     bad = ev == NULL;
@@ -592,17 +728,71 @@ connection_should_read_from_linked_conn(connection_t *conn)
   return 0;
 }
 
-/** If we called event_base_loop() and told it to never stop until it
- * runs out of events, now we've changed our mind: tell it we want it to
- * finish. */
+/** Event to run 'shutdown did not work callback'. */
+static struct event *shutdown_did_not_work_event = NULL;
+
+/** Failsafe measure that should never actually be necessary: If
+ * tor_shutdown_event_loop_and_exit() somehow doesn't successfully exit the
+ * event loop, then this callback will kill Tor with an assertion failure
+ * seconds later
+ */
+static void
+shutdown_did_not_work_callback(evutil_socket_t fd, short event, void *arg)
+{
+  // LCOV_EXCL_START
+  (void) fd;
+  (void) event;
+  (void) arg;
+  tor_assert_unreached();
+  // LCOV_EXCL_STOP
+}
+
+#ifdef ENABLE_RESTART_DEBUGGING
+static struct event *tor_shutdown_event_loop_for_restart_event = NULL;
+static void
+tor_shutdown_event_loop_for_restart_cb(
+                      evutil_socket_t fd, short event, void *arg)
+{
+  (void)fd;
+  (void)event;
+  (void)arg;
+  tor_event_free(tor_shutdown_event_loop_for_restart_event);
+  tor_shutdown_event_loop_and_exit(0);
+}
+#endif
+
+/**
+ * After finishing the current callback (if any), shut down the main loop,
+ * clean up the process, and exit with <b>exitcode</b>.
+ */
 void
-tell_event_loop_to_finish(void)
+tor_shutdown_event_loop_and_exit(int exitcode)
 {
-  if (!called_loop_once) {
-    struct timeval tv = { 0, 0 };
-    tor_event_base_loopexit(tor_libevent_get_base(), &tv);
-    called_loop_once = 1; /* hack to avoid adding more exit events */
-  }
+  if (main_loop_should_exit)
+    return; /* Ignore multiple calls to this function. */
+
+  main_loop_should_exit = 1;
+  main_loop_exit_value = exitcode;
+
+  /* Die with an assertion failure in ten seconds, if for some reason we don't
+   * exit normally. */
+  /* XXXX We should consider this code if it's never used. */
+  struct timeval ten_seconds = { 10, 0 };
+  shutdown_did_not_work_event = tor_evtimer_new(
+                  tor_libevent_get_base(),
+                  shutdown_did_not_work_callback, NULL);
+  event_add(shutdown_did_not_work_event, &ten_seconds);
+
+  /* Unlike exit_loop_after_delay(), exit_loop_after_callback
+   * prevents other callbacks from running. */
+  tor_libevent_exit_loop_after_callback(tor_libevent_get_base());
+}
+
+/** Return true iff tor_shutdown_event_loop_and_exit() has been called. */
+int
+tor_event_loop_shutdown_is_pending(void)
+{
+  return main_loop_should_exit;
 }
 
 /** Helper: Tell the main loop to begin reading bytes into <b>conn</b> from
@@ -617,10 +807,7 @@ connection_start_reading_from_linked_conn(connection_t *conn)
   if (!conn->active_on_link) {
     conn->active_on_link = 1;
     smartlist_add(active_linked_connection_lst, conn);
-    /* make sure that the event_base_loop() function exits at
-     * the end of its run through the current connections, so we can
-     * activate read events for linked connections. */
-    tell_event_loop_to_finish();
+    mainloop_event_activate(schedule_active_linked_connections_event);
   } else {
     tor_assert(smartlist_contains(active_linked_connection_lst, conn));
   }
@@ -692,6 +879,16 @@ conn_read_callback(evutil_socket_t fd, short event, void *_conn)
 
   /* assert_connection_ok(conn, time(NULL)); */
 
+  /* Handle marked for close connections early */
+  if (conn->marked_for_close && connection_is_reading(conn)) {
+    /* Libevent says we can read, but we are marked for close so we will never
+     * try to read again. We will try to close the connection below inside of
+     * close_closeable_connections(), but let's make sure not to cause Libevent
+     * to spin on conn_read_callback() while we wait for the socket to let us
+     * flush to it.*/
+    connection_stop_reading(conn);
+  }
+
   if (connection_handle_read(conn) < 0) {
     if (!conn->marked_for_close) {
 #ifndef _WIN32
@@ -699,7 +896,7 @@ conn_read_callback(evutil_socket_t fd, short event, void *_conn)
                "(fd %d); removing",
                conn_type_to_string(conn->type), (int)conn->s);
       tor_fragile_assert();
-#endif
+#endif /* !defined(_WIN32) */
       if (CONN_IS_EDGE(conn))
         connection_edge_end_errno(TO_EDGE_CONN(conn));
       connection_mark_for_close(conn);
@@ -794,7 +991,7 @@ conn_close_if_marked(int i)
                (int)conn->outbuf_flushlen,
                 conn->marked_for_close_file, conn->marked_for_close);
     if (conn->linked_conn) {
-      retval = move_buf_to_buf(conn->linked_conn->inbuf, conn->outbuf,
+      retval = buf_move_to_buf(conn->linked_conn->inbuf, conn->outbuf,
                                &conn->outbuf_flushlen);
       if (retval >= 0) {
         /* The linked conn will notice that it has data when it notices that
@@ -808,12 +1005,13 @@ conn_close_if_marked(int i)
                 connection_wants_to_flush(conn));
     } else if (connection_speaks_cells(conn)) {
       if (conn->state == OR_CONN_STATE_OPEN) {
-        retval = flush_buf_tls(TO_OR_CONN(conn)->tls, conn->outbuf, sz,
+        retval = buf_flush_to_tls(conn->outbuf, TO_OR_CONN(conn)->tls, sz,
                                &conn->outbuf_flushlen);
       } else
         retval = -1; /* never flush non-open broken tls connections */
     } else {
-      retval = flush_buf(conn->s, conn->outbuf, sz, &conn->outbuf_flushlen);
+      retval = buf_flush_to_socket(conn->outbuf, conn->s, sz,
+                                   &conn->outbuf_flushlen);
     }
     if (retval >= 0 && /* Technically, we could survive things like
                           TLS_WANT_WRITE here. But don't bother for now. */
@@ -822,7 +1020,8 @@ conn_close_if_marked(int i)
         LOG_FN_CONN(conn, (LOG_INFO,LD_NET,
                            "Holding conn (fd %d) open for more flushing.",
                            (int)conn->s));
-        conn->timestamp_lastwritten = now; /* reset so we can flush more */
+        conn->timestamp_last_write_allowed = now; /* reset so we can flush
+                                                   * more */
       } else if (sz == 0) {
         /* Also, retval==0.  If we get here, we didn't want to write anything
          * (because of rate-limiting) and we didn't. */
@@ -833,20 +1032,18 @@ conn_close_if_marked(int i)
          * busy Libevent loops where we keep ending up here and returning
          * 0 until we are no longer blocked on bandwidth.
          */
-        if (connection_is_writing(conn)) {
-          conn->write_blocked_on_bw = 1;
-          connection_stop_writing(conn);
-        }
-        if (connection_is_reading(conn)) {
-          /* XXXX+ We should make this code unreachable; if a connection is
-           * marked for close and flushing, there is no point in reading to it
-           * at all. Further, checking at this point is a bit of a hack: it
-           * would make much more sense to react in
-           * connection_handle_read_impl, or to just stop reading in
-           * mark_and_flush */
-          conn->read_blocked_on_bw = 1;
-          connection_stop_reading(conn);
+        connection_consider_empty_write_buckets(conn);
+        /* Make sure that consider_empty_buckets really disabled the
+         * connection: */
+        if (BUG(connection_is_writing(conn))) {
+          connection_write_bw_exhausted(conn, true);
         }
+
+        /* The connection is being held due to write rate limit and thus will
+         * flush its data later. We need to stop reading because this
+         * connection is about to be closed once flushed. It should not
+         * process anything more coming in at this stage. */
+        connection_stop_reading(conn);
       }
       return 0;
     }
@@ -873,9 +1070,8 @@ conn_close_if_marked(int i)
  * reason.
  */
 static void
-directory_all_unreachable_cb(evutil_socket_t fd, short event, void *arg)
+directory_all_unreachable_cb(mainloop_event_t *event, void *arg)
 {
-  (void)fd;
   (void)event;
   (void)arg;
 
@@ -895,7 +1091,7 @@ directory_all_unreachable_cb(evutil_socket_t fd, short event, void *arg)
   control_event_general_error("DIR_ALL_UNREACHABLE");
 }
 
-static struct event *directory_all_unreachable_cb_event = NULL;
+static mainloop_event_t *directory_all_unreachable_cb_event = NULL;
 
 /** We've just tried every dirserver we know about, and none of
  * them were reachable. Assume the network is down. Change state
@@ -908,16 +1104,15 @@ directory_all_unreachable(time_t now)
 {
   (void)now;
 
-  stats_n_seconds_working=0; /* reset it */
+  reset_uptime(); /* reset it */
 
   if (!directory_all_unreachable_cb_event) {
     directory_all_unreachable_cb_event =
-      tor_event_new(tor_libevent_get_base(),
-                    -1, EV_READ, directory_all_unreachable_cb, NULL);
+      mainloop_event_new(directory_all_unreachable_cb, NULL);
     tor_assert(directory_all_unreachable_cb_event);
   }
 
-  event_active(directory_all_unreachable_cb_event, EV_READ, 1);
+  mainloop_event_activate(directory_all_unreachable_cb_event);
 }
 
 /** This function is called whenever we successfully pull down some new
@@ -927,6 +1122,15 @@ directory_info_has_arrived(time_t now, int from_cache, int suppress_logs)
 {
   const or_options_t *options = get_options();
 
+  /* if we have enough dir info, then update our guard status with
+   * whatever we just learned. */
+  int invalidate_circs = guards_update_all();
+
+  if (invalidate_circs) {
+    circuit_mark_all_unused_circs();
+    circuit_mark_all_dirty_circs_as_unusable();
+  }
+
   if (!router_have_minimum_dir_info()) {
     int quiet = suppress_logs || from_cache ||
                 directory_too_idle_to_fetch_descriptors(options, now);
@@ -940,9 +1144,6 @@ directory_info_has_arrived(time_t now, int from_cache, int suppress_logs)
       update_all_descriptor_downloads(now);
     }
 
-    /* if we have enough dir info, then update our guard status with
-     * whatever we just learned. */
-    entry_guards_compute_status(options, now);
     /* Don't even bother trying to get extrainfo until the rest of our
      * directory info is up-to-date */
     if (options->DownloadExtraInfo)
@@ -951,7 +1152,7 @@ directory_info_has_arrived(time_t now, int from_cache, int suppress_logs)
 
   if (server_mode(options) && !net_is_disabled() && !from_cache &&
       (have_completed_a_circuit() || !any_predicted_circuits(now)))
-    consider_testing_reachability(1, 1);
+   router_do_reachability_checks(1, 1);
 }
 
 /** Perform regular maintenance tasks for a single connection.  This
@@ -967,7 +1168,7 @@ run_connection_housekeeping(int i, time_t now)
   channel_t *chan = NULL;
   int have_any_circuits;
   int past_keepalive =
-    now >= conn->timestamp_lastwritten + options->KeepalivePeriod;
+    now >= conn->timestamp_last_write_allowed + options->KeepalivePeriod;
 
   if (conn->outbuf && !connection_get_outbuf_len(conn) &&
       conn->type == CONN_TYPE_OR)
@@ -982,10 +1183,10 @@ run_connection_housekeeping(int i, time_t now)
    * if a server or received if a client) for 5 min */
   if (conn->type == CONN_TYPE_DIR &&
       ((DIR_CONN_IS_SERVER(conn) &&
-        conn->timestamp_lastwritten
+        conn->timestamp_last_write_allowed
             + options->TestingDirConnectionMaxStall < now) ||
        (!DIR_CONN_IS_SERVER(conn) &&
-        conn->timestamp_lastread
+        conn->timestamp_last_read_allowed
             + options->TestingDirConnectionMaxStall < now))) {
     log_info(LD_DIR,"Expiring wedged directory conn (fd %d, purpose %d)",
              (int)conn->s, conn->purpose);
@@ -1005,7 +1206,7 @@ run_connection_housekeeping(int i, time_t now)
   if (!connection_speaks_cells(conn))
     return; /* we're all done here, the rest is just for OR conns */
 
-  /* If we haven't written to an OR connection for a while, then either nuke
+  /* If we haven't flushed to an OR connection for a while, then either nuke
      the connection or send a keepalive, depending. */
 
   or_conn = TO_OR_CONN(conn);
@@ -1043,7 +1244,8 @@ run_connection_housekeeping(int i, time_t now)
   } else if (we_are_hibernating() &&
              ! have_any_circuits &&
              !connection_get_outbuf_len(conn)) {
-    /* We're hibernating, there's no circuits, and nothing to flush.*/
+    /* We're hibernating or shutting down, there's no circuits, and nothing to
+     * flush.*/
     log_info(LD_OR,"Expiring non-used OR connection to fd %d (%s:%d) "
              "[Hibernating or exiting].",
              (int)conn->s,conn->address, conn->port);
@@ -1051,8 +1253,9 @@ run_connection_housekeeping(int i, time_t now)
   } else if (!have_any_circuits &&
              now - or_conn->idle_timeout >=
                                          chan->timestamp_last_had_circuits) {
-    log_info(LD_OR,"Expiring non-used OR connection to fd %d (%s:%d) "
-             "[no circuits for %d; timeout %d; %scanonical].",
+    log_info(LD_OR,"Expiring non-used OR connection %"PRIu64" to fd %d "
+             "(%s:%d) [no circuits for %d; timeout %d; %scanonical].",
+             (chan->global_identifier),
              (int)conn->s, conn->address, conn->port,
              (int)(now - chan->timestamp_last_had_circuits),
              or_conn->idle_timeout,
@@ -1060,13 +1263,14 @@ run_connection_housekeeping(int i, time_t now)
     connection_or_close_normally(TO_OR_CONN(conn), 0);
   } else if (
       now >= or_conn->timestamp_lastempty + options->KeepalivePeriod*10 &&
-      now >= conn->timestamp_lastwritten + options->KeepalivePeriod*10) {
+      now >=
+          conn->timestamp_last_write_allowed + options->KeepalivePeriod*10) {
     log_fn(LOG_PROTOCOL_WARN,LD_PROTOCOL,
            "Expiring stuck OR connection to fd %d (%s:%d). (%d bytes to "
            "flush; %d seconds since last write)",
            (int)conn->s, conn->address, conn->port,
            (int)connection_get_outbuf_len(conn),
-           (int)(now-conn->timestamp_lastwritten));
+           (int)(now-conn->timestamp_last_write_allowed));
     connection_or_close_normally(TO_OR_CONN(conn), 0);
   } else if (past_keepalive && !connection_get_outbuf_len(conn)) {
     /* send a padding cell */
@@ -1075,6 +1279,8 @@ run_connection_housekeeping(int i, time_t now)
     memset(&cell,0,sizeof(cell_t));
     cell.command = CELL_PADDING;
     connection_or_write_cell_to_buf(&cell, or_conn);
+  } else {
+    channelpadding_decide_to_pad_channel(chan);
   }
 }
 
@@ -1092,7 +1298,7 @@ signewnym_impl(time_t now)
 
   circuit_mark_all_dirty_circs_as_unusable();
   addressmap_clear_transient();
-  rend_client_purge_state();
+  hs_client_purge_state();
   time_of_last_signewnym = now;
   signewnym_is_pending = 0;
 
@@ -1101,6 +1307,40 @@ signewnym_impl(time_t now)
   control_event_signal(SIGNEWNYM);
 }
 
+/** Callback: run a deferred signewnym. */
+static void
+handle_deferred_signewnym_cb(mainloop_event_t *event, void *arg)
+{
+  (void)event;
+  (void)arg;
+  log_info(LD_CONTROL, "Honoring delayed NEWNYM request");
+  do_signewnym(time(NULL));
+}
+
+/** Either perform a signewnym or schedule one, depending on rate limiting. */
+void
+do_signewnym(time_t now)
+{
+  if (time_of_last_signewnym + MAX_SIGNEWNYM_RATE > now) {
+    const time_t delay_sec =
+      time_of_last_signewnym + MAX_SIGNEWNYM_RATE - now;
+    if (! signewnym_is_pending) {
+      signewnym_is_pending = 1;
+      if (!handle_deferred_signewnym_ev) {
+        handle_deferred_signewnym_ev =
+          mainloop_event_postloop_new(handle_deferred_signewnym_cb, NULL);
+      }
+      const struct timeval delay_tv = { delay_sec, 0 };
+      mainloop_event_schedule(handle_deferred_signewnym_ev, &delay_tv);
+    }
+    log_notice(LD_CONTROL,
+               "Rate limiting NEWNYM request: delaying by %d second(s)",
+               (int)(delay_sec));
+  } else {
+    signewnym_impl(now);
+  }
+}
+
 /** Return the number of times that signewnym has been called. */
 unsigned
 get_signewnym_epoch(void)
@@ -1116,61 +1356,106 @@ static int periodic_events_initialized = 0;
 #undef CALLBACK
 #define CALLBACK(name) \
   static int name ## _callback(time_t, const or_options_t *)
-CALLBACK(rotate_onion_key);
-CALLBACK(check_ed_keys);
-CALLBACK(launch_descriptor_fetches);
-CALLBACK(rotate_x509_certificate);
 CALLBACK(add_entropy);
-CALLBACK(launch_reachability_tests);
-CALLBACK(downrate_stability);
-CALLBACK(save_stability);
 CALLBACK(check_authority_cert);
+CALLBACK(check_canonical_channels);
+CALLBACK(check_descriptor);
+CALLBACK(check_dns_honesty);
+CALLBACK(check_ed_keys);
 CALLBACK(check_expired_networkstatus);
-CALLBACK(write_stats_file);
-CALLBACK(record_bridge_stats);
+CALLBACK(check_for_reachability_bw);
+CALLBACK(check_onion_keys_expiry_time);
 CALLBACK(clean_caches);
+CALLBACK(clean_consdiffmgr);
+CALLBACK(dirvote);
+CALLBACK(downrate_stability);
+CALLBACK(expire_old_ciruits_serverside);
+CALLBACK(fetch_networkstatus);
+CALLBACK(heartbeat);
+CALLBACK(hs_service);
+CALLBACK(launch_descriptor_fetches);
+CALLBACK(launch_reachability_tests);
+CALLBACK(reachability_warnings);
+CALLBACK(record_bridge_stats);
 CALLBACK(rend_cache_failure_clean);
+CALLBACK(reset_padding_counts);
 CALLBACK(retry_dns);
-CALLBACK(check_descriptor);
-CALLBACK(check_for_reachability_bw);
-CALLBACK(fetch_networkstatus);
 CALLBACK(retry_listeners);
-CALLBACK(expire_old_ciruits_serverside);
-CALLBACK(check_dns_honesty);
+CALLBACK(rotate_onion_key);
+CALLBACK(rotate_x509_certificate);
+CALLBACK(save_stability);
+CALLBACK(save_state);
 CALLBACK(write_bridge_ns);
-CALLBACK(check_fw_helper_app);
-CALLBACK(heartbeat);
+CALLBACK(write_stats_file);
 
 #undef CALLBACK
 
 /* Now we declare an array of periodic_event_item_t for each periodic event */
-#define CALLBACK(name) PERIODIC_EVENT(name)
-
-static periodic_event_item_t periodic_events[] = {
-  CALLBACK(rotate_onion_key),
-  CALLBACK(check_ed_keys),
-  CALLBACK(launch_descriptor_fetches),
-  CALLBACK(rotate_x509_certificate),
-  CALLBACK(add_entropy),
-  CALLBACK(launch_reachability_tests),
-  CALLBACK(downrate_stability),
-  CALLBACK(save_stability),
-  CALLBACK(check_authority_cert),
-  CALLBACK(check_expired_networkstatus),
-  CALLBACK(write_stats_file),
-  CALLBACK(record_bridge_stats),
-  CALLBACK(clean_caches),
-  CALLBACK(rend_cache_failure_clean),
-  CALLBACK(retry_dns),
-  CALLBACK(check_descriptor),
-  CALLBACK(check_for_reachability_bw),
-  CALLBACK(fetch_networkstatus),
-  CALLBACK(retry_listeners),
-  CALLBACK(expire_old_ciruits_serverside),
-  CALLBACK(check_dns_honesty),
-  CALLBACK(write_bridge_ns),
-  CALLBACK(check_fw_helper_app),
-  CALLBACK(heartbeat),
+#define CALLBACK(name, r, f) PERIODIC_EVENT(name, r, f)
+
+STATIC periodic_event_item_t periodic_events[] = {
+  /* Everyone needs to run those. */
+  CALLBACK(add_entropy, PERIODIC_EVENT_ROLE_ALL, 0),
+  CALLBACK(check_expired_networkstatus, PERIODIC_EVENT_ROLE_ALL, 0),
+  CALLBACK(clean_caches, PERIODIC_EVENT_ROLE_ALL, 0),
+  CALLBACK(fetch_networkstatus, PERIODIC_EVENT_ROLE_ALL,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(heartbeat, PERIODIC_EVENT_ROLE_ALL, 0),
+  CALLBACK(launch_descriptor_fetches, PERIODIC_EVENT_ROLE_ALL,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(reset_padding_counts, PERIODIC_EVENT_ROLE_ALL, 0),
+  CALLBACK(retry_listeners, PERIODIC_EVENT_ROLE_ALL,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(save_state, PERIODIC_EVENT_ROLE_ALL, 0),
+  CALLBACK(rotate_x509_certificate, PERIODIC_EVENT_ROLE_ALL, 0),
+  CALLBACK(write_stats_file, PERIODIC_EVENT_ROLE_ALL, 0),
+
+  /* Routers (bridge and relay) only. */
+  CALLBACK(check_descriptor, PERIODIC_EVENT_ROLE_ROUTER,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(check_ed_keys, PERIODIC_EVENT_ROLE_ROUTER, 0),
+  CALLBACK(check_for_reachability_bw, PERIODIC_EVENT_ROLE_ROUTER,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(check_onion_keys_expiry_time, PERIODIC_EVENT_ROLE_ROUTER, 0),
+  CALLBACK(expire_old_ciruits_serverside, PERIODIC_EVENT_ROLE_ROUTER,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(reachability_warnings, PERIODIC_EVENT_ROLE_ROUTER,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(retry_dns, PERIODIC_EVENT_ROLE_ROUTER, 0),
+  CALLBACK(rotate_onion_key, PERIODIC_EVENT_ROLE_ROUTER, 0),
+
+  /* Authorities (bridge and directory) only. */
+  CALLBACK(downrate_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0),
+  CALLBACK(launch_reachability_tests, PERIODIC_EVENT_ROLE_AUTHORITIES,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(save_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0),
+
+  /* Directory authority only. */
+  CALLBACK(check_authority_cert, PERIODIC_EVENT_ROLE_DIRAUTH, 0),
+  CALLBACK(dirvote, PERIODIC_EVENT_ROLE_DIRAUTH, PERIODIC_EVENT_FLAG_NEED_NET),
+
+  /* Relay only. */
+  CALLBACK(check_canonical_channels, PERIODIC_EVENT_ROLE_RELAY,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(check_dns_honesty, PERIODIC_EVENT_ROLE_RELAY,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+
+  /* Hidden Service service only. */
+  CALLBACK(hs_service, PERIODIC_EVENT_ROLE_HS_SERVICE,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+
+  /* Bridge only. */
+  CALLBACK(record_bridge_stats, PERIODIC_EVENT_ROLE_BRIDGE, 0),
+
+  /* Client only. */
+  CALLBACK(rend_cache_failure_clean, PERIODIC_EVENT_ROLE_CLIENT, 0),
+
+  /* Bridge Authority only. */
+  CALLBACK(write_bridge_ns, PERIODIC_EVENT_ROLE_BRIDGEAUTH, 0),
+
+  /* Directory server only. */
+  CALLBACK(clean_consdiffmgr, PERIODIC_EVENT_ROLE_DIRSERVER, 0),
+
   END_OF_PERIODIC_EVENTS
 };
 #undef CALLBACK
@@ -1180,9 +1465,11 @@ static periodic_event_item_t periodic_events[] = {
  * can access them by name.  We also keep them inside periodic_events[]
  * so that we can implement "reset all timers" in a reasonable way. */
 static periodic_event_item_t *check_descriptor_event=NULL;
+static periodic_event_item_t *dirvote_event=NULL;
 static periodic_event_item_t *fetch_networkstatus_event=NULL;
 static periodic_event_item_t *launch_descriptor_fetches_event=NULL;
 static periodic_event_item_t *check_dns_honesty_event=NULL;
+static periodic_event_item_t *save_state_event=NULL;
 
 /** Reset all the periodic events so we'll do all our actions again as if we
  * just started up.
@@ -1212,6 +1499,42 @@ find_periodic_event(const char *name)
   return NULL;
 }
 
+/** Return a bitmask of the roles this tor instance is configured for using
+ * the given options. */
+STATIC int
+get_my_roles(const or_options_t *options)
+{
+  tor_assert(options);
+
+  int roles = 0;
+  int is_bridge = options->BridgeRelay;
+  int is_relay = server_mode(options);
+  int is_dirauth = authdir_mode_v3(options);
+  int is_bridgeauth = authdir_mode_bridge(options);
+  int is_hidden_service = !!hs_service_get_num_services() ||
+                          !!rend_num_services();
+  int is_dirserver = dir_server_mode(options);
+  /* We also consider tor to have the role of a client if the ControlPort is
+   * set because a lot of things can be done over the control port which
+   * requires tor to have basic functionnalities. */
+  int is_client = options_any_client_port_set(options) ||
+                  options->ControlPort_set ||
+                  options->OwningControllerFD != UINT64_MAX;
+
+  if (is_bridge) roles |= PERIODIC_EVENT_ROLE_BRIDGE;
+  if (is_client) roles |= PERIODIC_EVENT_ROLE_CLIENT;
+  if (is_relay) roles |= PERIODIC_EVENT_ROLE_RELAY;
+  if (is_dirauth) roles |= PERIODIC_EVENT_ROLE_DIRAUTH;
+  if (is_bridgeauth) roles |= PERIODIC_EVENT_ROLE_BRIDGEAUTH;
+  if (is_hidden_service) roles |= PERIODIC_EVENT_ROLE_HS_SERVICE;
+  if (is_dirserver) roles |= PERIODIC_EVENT_ROLE_DIRSERVER;
+
+  return roles;
+}
+
+/** Event to run initialize_periodic_events_cb */
+static struct event *initialize_periodic_events_event = NULL;
+
 /** Helper, run one second after setup:
  * Initializes all members of periodic_events and starts them running.
  *
@@ -1223,10 +1546,10 @@ initialize_periodic_events_cb(evutil_socket_t fd, short events, void *data)
   (void) fd;
   (void) events;
   (void) data;
-  int i;
-  for (i = 0; periodic_events[i].name; ++i) {
-    periodic_event_launch(&periodic_events[i]);
-  }
+
+  tor_event_free(initialize_periodic_events_event);
+
+  rescan_periodic_events(get_options());
 }
 
 /** Set up all the members of periodic_events[], and configure them all to be
@@ -1234,9 +1557,12 @@ initialize_periodic_events_cb(evutil_socket_t fd, short events, void *data)
 STATIC void
 initialize_periodic_events(void)
 {
-  tor_assert(periodic_events_initialized == 0);
+  if (periodic_events_initialized)
+    return;
+
   periodic_events_initialized = 1;
 
+  /* Set up all periodic events. We'll launch them by roles. */
   int i;
   for (i = 0; periodic_events[i].name; ++i) {
     periodic_event_setup(&periodic_events[i]);
@@ -1246,14 +1572,17 @@ initialize_periodic_events(void)
   STMT_BEGIN name ## _event = find_periodic_event( #name ); STMT_END
 
   NAMED_CALLBACK(check_descriptor);
+  NAMED_CALLBACK(dirvote);
   NAMED_CALLBACK(fetch_networkstatus);
   NAMED_CALLBACK(launch_descriptor_fetches);
   NAMED_CALLBACK(check_dns_honesty);
+  NAMED_CALLBACK(save_state);
 
   struct timeval one_second = { 1, 0 };
-  event_base_once(tor_libevent_get_base(), -1, EV_TIMEOUT,
-                  initialize_periodic_events_cb, NULL,
-                  &one_second);
+  initialize_periodic_events_event = tor_evtimer_new(
+                  tor_libevent_get_base(),
+                  initialize_periodic_events_cb, NULL);
+  event_add(initialize_periodic_events_event, &one_second);
 }
 
 STATIC void
@@ -1263,6 +1592,60 @@ teardown_periodic_events(void)
   for (i = 0; periodic_events[i].name; ++i) {
     periodic_event_destroy(&periodic_events[i]);
   }
+  periodic_events_initialized = 0;
+}
+
+/** Do a pass at all our periodic events, disable those we don't need anymore
+ * and enable those we need now using the given options. */
+void
+rescan_periodic_events(const or_options_t *options)
+{
+  tor_assert(options);
+
+  /* Avoid scanning the event list if we haven't initialized it yet. This is
+   * particularly useful for unit tests in order to avoid initializing main
+   * loop events everytime. */
+  if (!periodic_events_initialized) {
+    return;
+  }
+
+  int roles = get_my_roles(options);
+
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+
+    int enable = !!(item->roles & roles);
+
+    /* Handle the event flags. */
+    if (net_is_disabled() &&
+        (item->flags & PERIODIC_EVENT_FLAG_NEED_NET)) {
+      enable = 0;
+    }
+
+    /* Enable the event if needed. It is safe to enable an event that was
+     * already enabled. Same goes for disabling it. */
+    if (enable) {
+      log_debug(LD_GENERAL, "Launching periodic event %s", item->name);
+      periodic_event_enable(item);
+    } else {
+      log_debug(LD_GENERAL, "Disabling periodic event %s", item->name);
+      periodic_event_disable(item);
+    }
+  }
+}
+
+/* We just got new options globally set, see if we need to enabled or disable
+ * periodic events. */
+void
+periodic_events_on_new_options(const or_options_t *options)
+{
+  /* Only if we've already initialized the events, rescan the list which will
+   * enable or disable events depending on our roles. This will be called at
+   * bootup and we don't want this function to initialize the events because
+   * they aren't set up at this stage. */
+  if (periodic_events_initialized) {
+    rescan_periodic_events(options);
+  }
 }
 
 /**
@@ -1273,8 +1656,9 @@ teardown_periodic_events(void)
 void
 reschedule_descriptor_update_check(void)
 {
-  tor_assert(check_descriptor_event);
-  periodic_event_reschedule(check_descriptor_event);
+  if (check_descriptor_event) {
+    periodic_event_reschedule(check_descriptor_event);
+  }
 }
 
 /**
@@ -1291,6 +1675,35 @@ reschedule_directory_downloads(void)
   periodic_event_reschedule(launch_descriptor_fetches_event);
 }
 
+/** Mainloop callback: clean up circuits, channels, and connections
+ * that are pending close. */
+static void
+postloop_cleanup_cb(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  (void)arg;
+  circuit_close_all_marked();
+  close_closeable_connections();
+  channel_run_cleanup();
+  channel_listener_run_cleanup();
+}
+
+/** Event to run postloop_cleanup_cb */
+static mainloop_event_t *postloop_cleanup_ev=NULL;
+
+/** Schedule a post-loop event to clean up marked channels, connections, and
+ * circuits. */
+void
+mainloop_schedule_postloop_cleanup(void)
+{
+  if (PREDICT_UNLIKELY(postloop_cleanup_ev == NULL)) {
+    // (It's possible that we can get here if we decide to close a connection
+    // in the earliest stages of our configuration, before we create events.)
+    return;
+  }
+  mainloop_event_activate(postloop_cleanup_ev);
+}
+
 #define LONGEST_TIMER_PERIOD (30 * 86400)
 /** Helper: Return the number of seconds between <b>now</b> and <b>next</b>,
  * clipped to the range [1 second, LONGEST_TIMER_PERIOD]. */
@@ -1328,18 +1741,12 @@ run_scheduled_events(time_t now)
    */
   consider_hibernation(now);
 
-  /* 0b. If we've deferred a signewnym, make sure it gets handled
-   * eventually. */
-  if (signewnym_is_pending &&
-      time_of_last_signewnym + MAX_SIGNEWNYM_RATE <= now) {
-    log_info(LD_CONTROL, "Honoring delayed NEWNYM request");
-    signewnym_impl(now);
-  }
+  /* Maybe enough time elapsed for us to reconsider a circuit. */
+  circuit_upgrade_circuits_from_guard_wait();
 
-  /* 0c. If we've deferred log messages for the controller, handle them now */
-  flush_pending_log_callbacks();
-
-  if (options->UseBridges && !options->DisableNetwork) {
+  if (options->UseBridges && !net_is_disabled()) {
+    /* Note: this check uses net_is_disabled(), not should_delay_dir_fetches()
+     * -- the latter is only for fetching consensus-derived directory info. */
     fetch_bridge_descriptors(options, now);
   }
 
@@ -1347,10 +1754,6 @@ run_scheduled_events(time_t now)
     accounting_run_housekeeping(now);
   }
 
-  if (authdir_mode_v3(options)) {
-    dirvote_act(options, now);
-  }
-
   /* 3a. Every second, we examine pending circuits and prune the
    *    ones which have been pending for more than a few seconds.
    *    We do this before step 4, so it can try building more if
@@ -1359,6 +1762,7 @@ run_scheduled_events(time_t now)
   /* (If our circuit build timeout can ever become lower than a second (which
    * it can't, currently), we should do this more often.) */
   circuit_expire_building();
+  circuit_expire_waiting_for_better_guard();
 
   /* 3b. Also look at pending streams and prune the ones that 'began'
    *     a long time ago but haven't gotten a 'connected' yet.
@@ -1383,58 +1787,30 @@ run_scheduled_events(time_t now)
     circuit_expire_old_circs_as_needed(now);
   }
 
-  if (!net_is_disabled()) {
-    /* This is usually redundant with circuit_build_needed_circs() above,
-     * but it is very fast when there is no work to do. */
-    connection_ap_attach_pending(0);
-  }
-
   /* 5. We do housekeeping for each connection... */
-  connection_or_set_bad_connections(NULL, 0);
+  channel_update_bad_for_new_circs(NULL, 0);
   int i;
   for (i=0;i<smartlist_len(connection_array);i++) {
     run_connection_housekeeping(i, now);
   }
 
-  /* 6. And remove any marked circuits... */
-  circuit_close_all_marked();
-
-  /* 7. And upload service descriptors if necessary. */
-  if (have_completed_a_circuit() && !net_is_disabled()) {
-    rend_consider_services_upload(now);
-    rend_consider_descriptor_republication();
-  }
-
-  /* 8. and blow away any connections that need to die. have to do this now,
-   * because if we marked a conn for close and left its socket -1, then
-   * we'll pass it to poll/select and bad things will happen.
-   */
-  close_closeable_connections();
-
-  /* 8b. And if anything in our state is ready to get flushed to disk, we
-   * flush it. */
-  or_state_save(now);
-
-  /* 8c. Do channel cleanup just like for connections */
-  channel_run_cleanup();
-  channel_listener_run_cleanup();
-
   /* 11b. check pending unconfigured managed proxies */
   if (!net_is_disabled() && pt_proxies_configuration_pending())
     pt_configure_remaining_proxies();
 }
 
+/* Periodic callback: rotate the onion keys after the period defined by the
+ * "onion-key-rotation-days" consensus parameter, shut down and restart all
+ * cpuworkers, and update our descriptor if necessary.
+ */
 static int
 rotate_onion_key_callback(time_t now, const or_options_t *options)
 {
-  /* 1a. Every MIN_ONION_KEY_LIFETIME seconds, rotate the onion keys,
-   *  shut down and restart all cpuworkers, and update the directory if
-   *  necessary.
-   */
   if (server_mode(options)) {
-    time_t rotation_time = get_onion_key_set_at()+MIN_ONION_KEY_LIFETIME;
+    int onion_key_lifetime = get_onion_key_lifetime();
+    time_t rotation_time = get_onion_key_set_at()+onion_key_lifetime;
     if (rotation_time > now) {
-      return safe_timer_diff(now, rotation_time);
+      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
     }
 
     log_info(LD_GENERAL,"Rotating onion key.");
@@ -1443,23 +1819,50 @@ rotate_onion_key_callback(time_t now, const or_options_t *options)
     if (router_rebuild_descriptor(1)<0) {
       log_info(LD_CONFIG, "Couldn't rebuild router descriptor");
     }
-    if (advertised_server_mode() && !options->DisableNetwork)
+    if (advertised_server_mode() && !net_is_disabled())
       router_upload_dir_desc_to_dirservers(0);
-    return MIN_ONION_KEY_LIFETIME;
+    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
   }
   return PERIODIC_EVENT_NO_UPDATE;
 }
 
+/* Period callback: Check if our old onion keys are still valid after the
+ * period of time defined by the consensus parameter
+ * "onion-key-grace-period-days", otherwise expire them by setting them to
+ * NULL.
+ */
+static int
+check_onion_keys_expiry_time_callback(time_t now, const or_options_t *options)
+{
+  if (server_mode(options)) {
+    int onion_key_grace_period = get_onion_key_grace_period();
+    time_t expiry_time = get_onion_key_set_at()+onion_key_grace_period;
+    if (expiry_time > now) {
+      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+    }
+
+    log_info(LD_GENERAL, "Expiring old onion keys.");
+    expire_old_onion_keys();
+    cpuworkers_rotate_keyinfo();
+    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+  }
+
+  return PERIODIC_EVENT_NO_UPDATE;
+}
+
+/* Periodic callback: Every 30 seconds, check whether it's time to make new
+ * Ed25519 subkeys.
+ */
 static int
 check_ed_keys_callback(time_t now, const or_options_t *options)
 {
   if (server_mode(options)) {
     if (should_make_new_ed_keys(options, now)) {
-      if (load_ed_keys(options, now) < 0 ||
-          generate_ed_link_cert(options, now)) {
+      int new_signing_key = load_ed_keys(options, now);
+      if (new_signing_key < 0 ||
+          generate_ed_link_cert(options, now, new_signing_key > 0)) {
         log_err(LD_OR, "Unable to update Ed25519 keys!  Exiting.");
-        tor_cleanup();
-        exit(0);
+        tor_shutdown_event_loop_and_exit(1);
       }
     }
     return 30;
@@ -1467,6 +1870,11 @@ check_ed_keys_callback(time_t now, const or_options_t *options)
   return PERIODIC_EVENT_NO_UPDATE;
 }
 
+/**
+ * Periodic callback: Every {LAZY,GREEDY}_DESCRIPTOR_RETRY_INTERVAL,
+ * see about fetching descriptors, microdescriptors, and extrainfo
+ * documents.
+ */
 static int
 launch_descriptor_fetches_callback(time_t now, const or_options_t *options)
 {
@@ -1481,6 +1889,10 @@ launch_descriptor_fetches_callback(time_t now, const or_options_t *options)
     return GREEDY_DESCRIPTOR_RETRY_INTERVAL;
 }
 
+/**
+ * Periodic event: Rotate our X.509 certificates and TLS keys once every
+ * MAX_SSL_KEY_LIFETIME_INTERNAL.
+ */
 static int
 rotate_x509_certificate_callback(time_t now, const or_options_t *options)
 {
@@ -1499,6 +1911,11 @@ rotate_x509_certificate_callback(time_t now, const or_options_t *options)
     log_err(LD_BUG, "Error reinitializing TLS context");
     tor_assert_unreached();
   }
+  if (generate_ed_link_cert(options, now, 1)) {
+    log_err(LD_OR, "Unable to update Ed25519->TLS link certificate for "
+            "new TLS context.");
+    tor_assert_unreached();
+  }
 
   /* We also make sure to rotate the TLS connections themselves if they've
    * been up for too long -- but that's done via is_bad_for_new_circs in
@@ -1506,6 +1923,10 @@ rotate_x509_certificate_callback(time_t now, const or_options_t *options)
   return MAX_SSL_KEY_LIFETIME_INTERNAL;
 }
 
+/**
+ * Periodic callback: once an hour, grab some more entropy from the
+ * kernel and feed it to our CSPRNG.
+ **/
 static int
 add_entropy_callback(time_t now, const or_options_t *options)
 {
@@ -1522,6 +1943,10 @@ add_entropy_callback(time_t now, const or_options_t *options)
   return ENTROPY_INTERVAL;
 }
 
+/**
+ * Periodic callback: if we're an authority, make sure we test
+ * the routers on the network for reachability.
+ */
 static int
 launch_reachability_tests_callback(time_t now, const or_options_t *options)
 {
@@ -1533,6 +1958,10 @@ launch_reachability_tests_callback(time_t now, const or_options_t *options)
   return REACHABILITY_TEST_INTERVAL;
 }
 
+/**
+ * Periodic callback: if we're an authority, discount the stability
+ * information (and other rephist information) that's older.
+ */
 static int
 downrate_stability_callback(time_t now, const or_options_t *options)
 {
@@ -1544,6 +1973,10 @@ downrate_stability_callback(time_t now, const or_options_t *options)
   return safe_timer_diff(now, next);
 }
 
+/**
+ * Periodic callback: if we're an authority, record our measured stability
+ * information from rephist in an mtbf file.
+ */
 static int
 save_stability_callback(time_t now, const or_options_t *options)
 {
@@ -1556,6 +1989,10 @@ save_stability_callback(time_t now, const or_options_t *options)
   return SAVE_STABILITY_INTERVAL;
 }
 
+/**
+ * Periodic callback: if we're an authority, check on our authority
+ * certificate (the one that authenticates our authority signing key).
+ */
 static int
 check_authority_cert_callback(time_t now, const or_options_t *options)
 {
@@ -1568,18 +2005,55 @@ check_authority_cert_callback(time_t now, const or_options_t *options)
   return CHECK_V3_CERTIFICATE_INTERVAL;
 }
 
+/**
+ * Scheduled callback: Run directory-authority voting functionality.
+ *
+ * The schedule is a bit complicated here, so dirvote_act() manages the
+ * schedule itself.
+ **/
+static int
+dirvote_callback(time_t now, const or_options_t *options)
+{
+  if (!authdir_mode_v3(options)) {
+    tor_assert_nonfatal_unreached();
+    return 3600;
+  }
+
+  time_t next = dirvote_act(options, now);
+  if (BUG(next == TIME_MAX)) {
+    /* This shouldn't be returned unless we called dirvote_act() without
+     * being an authority.  If it happens, maybe our configuration will
+     * fix itself in an hour or so? */
+    return 3600;
+  }
+  return safe_timer_diff(now, next);
+}
+
+/** Reschedule the directory-authority voting event.  Run this whenever the
+ * schedule has changed. */
+void
+reschedule_dirvote(const or_options_t *options)
+{
+  if (periodic_events_initialized && authdir_mode_v3(options)) {
+    periodic_event_reschedule(dirvote_event);
+  }
+}
+
+/**
+ * Periodic callback: If our consensus is too old, recalculate whether
+ * we can actually use it.
+ */
 static int
 check_expired_networkstatus_callback(time_t now, const or_options_t *options)
 {
   (void)options;
-  /* 1f. Check whether our networkstatus has expired.
-   */
+  /* Check whether our networkstatus has expired. */
   networkstatus_t *ns = networkstatus_get_latest_consensus();
   /*XXXX RD: This value needs to be the same as REASONABLY_LIVE_TIME in
    * networkstatus_get_reasonably_live_consensus(), but that value is way
    * way too high.  Arma: is the bridge issue there resolved yet? -NM */
 #define NS_EXPIRY_SLOP (24*60*60)
-  if (ns && ns->valid_until < now+NS_EXPIRY_SLOP &&
+  if (ns && ns->valid_until < (now - NS_EXPIRY_SLOP) &&
       router_have_minimum_dir_info()) {
     router_dir_info_changed();
   }
@@ -1587,6 +2061,37 @@ check_expired_networkstatus_callback(time_t now, const or_options_t *options)
   return CHECK_EXPIRED_NS_INTERVAL;
 }
 
+/**
+ * Scheduled callback: Save the state file to disk if appropriate.
+ */
+static int
+save_state_callback(time_t now, const or_options_t *options)
+{
+  (void) options;
+  (void) or_state_save(now); // only saves if appropriate
+  const time_t next_write = get_or_state()->next_write;
+  if (next_write == TIME_MAX) {
+    return 86400;
+  }
+  return safe_timer_diff(now, next_write);
+}
+
+/** Reschedule the event for saving the state file.
+ *
+ * Run this when the state becomes dirty. */
+void
+reschedule_or_state_save(void)
+{
+  if (save_state_event == NULL) {
+    /* This can happen early on during startup. */
+    return;
+  }
+  periodic_event_reschedule(save_state_event);
+}
+
+/**
+ * Periodic callback: Write statistics to disk if appropriate.
+ */
 static int
 write_stats_file_callback(time_t now, const or_options_t *options)
 {
@@ -1634,11 +2139,36 @@ write_stats_file_callback(time_t now, const or_options_t *options)
   return safe_timer_diff(now, next_time_to_write_stats_files);
 }
 
+#define CHANNEL_CHECK_INTERVAL (60*60)
 static int
-record_bridge_stats_callback(time_t now, const or_options_t *options)
+check_canonical_channels_callback(time_t now, const or_options_t *options)
+{
+  (void)now;
+  if (public_server_mode(options))
+    channel_check_for_duplicates();
+
+  return CHANNEL_CHECK_INTERVAL;
+}
+
+static int
+reset_padding_counts_callback(time_t now, const or_options_t *options)
 {
-  static int should_init_bridge_stats = 1;
+  if (options->PaddingStatistics) {
+    rep_hist_prep_published_padding_counts(now);
+  }
 
+  rep_hist_reset_padding_counts();
+  return REPHIST_CELL_PADDING_COUNTS_INTERVAL;
+}
+
+static int should_init_bridge_stats = 1;
+
+/**
+ * Periodic callback: Write bridge statistics to disk if appropriate.
+ */
+static int
+record_bridge_stats_callback(time_t now, const or_options_t *options)
+{
   /* 1h. Check whether we should write bridge statistics to disk.
    */
   if (should_record_bridge_info(options)) {
@@ -1661,19 +2191,26 @@ record_bridge_stats_callback(time_t now, const or_options_t *options)
   return PERIODIC_EVENT_NO_UPDATE;
 }
 
+/**
+ * Periodic callback: Clean in-memory caches every once in a while
+ */
 static int
 clean_caches_callback(time_t now, const or_options_t *options)
 {
   /* Remove old information from rephist and the rend cache. */
   rep_history_clean(now - options->RephistTrackTime);
-  rend_cache_clean(now, REND_CACHE_TYPE_CLIENT);
   rend_cache_clean(now, REND_CACHE_TYPE_SERVICE);
-  rend_cache_clean_v2_descs_as_dir(now, 0);
+  hs_cache_clean_as_client(now);
+  hs_cache_clean_as_dir(now);
   microdesc_cache_rebuild(NULL, 0);
 #define CLEAN_CACHES_INTERVAL (30*60)
   return CLEAN_CACHES_INTERVAL;
 }
 
+/**
+ * Periodic callback: Clean the cache of failed hidden service lookups
+ * frequently.
+ */
 static int
 rend_cache_failure_clean_callback(time_t now, const or_options_t *options)
 {
@@ -1682,23 +2219,25 @@ rend_cache_failure_clean_callback(time_t now, const or_options_t *options)
    * clean it as soon as we can since we want to make sure the client waits
    * as little as possible for reachability reasons. */
   rend_cache_failure_clean(now);
+  hs_cache_client_intro_state_clean(now);
   return 30;
 }
 
+/**
+ * Periodic callback: If we're a server and initializing dns failed, retry.
+ */
 static int
 retry_dns_callback(time_t now, const or_options_t *options)
 {
   (void)now;
 #define RETRY_DNS_INTERVAL (10*60)
-  /* If we're a server and initializing dns failed, retry periodically. */
   if (server_mode(options) && has_dns_init_failed())
     dns_init();
   return RETRY_DNS_INTERVAL;
 }
 
-  /* 2. Periodically, we consider force-uploading our descriptor
-   * (if we've passed our internal checks). */
-
+/** Periodic callback: consider rebuilding or and re-uploading our descriptor
+ * (if we've passed our internal checks). */
 static int
 check_descriptor_callback(time_t now, const or_options_t *options)
 {
@@ -1707,9 +2246,11 @@ check_descriptor_callback(time_t now, const or_options_t *options)
  * address has changed. */
 #define CHECK_DESCRIPTOR_INTERVAL (60)
 
+  (void)options;
+
   /* 2b. Once per minute, regenerate and upload the descriptor if the old
    * one is inaccurate. */
-  if (!options->DisableNetwork) {
+  if (!net_is_disabled()) {
     check_descriptor_bandwidth_changed(now);
     check_descriptor_ipaddress_changed(now);
     mark_my_descriptor_dirty_if_too_old(now);
@@ -1725,6 +2266,11 @@ check_descriptor_callback(time_t now, const or_options_t *options)
   return CHECK_DESCRIPTOR_INTERVAL;
 }
 
+/**
+ * Periodic callback: check whether we're reachable (as a relay), and
+ * whether our bandwidth has changed enough that we need to
+ * publish a new descriptor.
+ */
 static int
 check_for_reachability_bw_callback(time_t now, const or_options_t *options)
 {
@@ -1736,9 +2282,9 @@ check_for_reachability_bw_callback(time_t now, const or_options_t *options)
    * 20 minutes of our uptime. */
   if (server_mode(options) &&
       (have_completed_a_circuit() || !any_predicted_circuits(now)) &&
-      !we_are_hibernating()) {
-    if (stats_n_seconds_working < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
-      consider_testing_reachability(1, dirport_reachability_count==0);
+      !net_is_disabled()) {
+    if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
+      router_do_reachability_checks(1, dirport_reachability_count==0);
       if (++dirport_reachability_count > 5)
         dirport_reachability_count = 0;
       return 1;
@@ -1761,13 +2307,13 @@ check_for_reachability_bw_callback(time_t now, const or_options_t *options)
   return CHECK_DESCRIPTOR_INTERVAL;
 }
 
+/**
+ * Periodic event: once a minute, (or every second if TestingTorNetwork, or
+ * during client bootstrap), check whether we want to download any
+ * networkstatus documents. */
 static int
 fetch_networkstatus_callback(time_t now, const or_options_t *options)
 {
-  /* 2c. Every minute (or every second if TestingTorNetwork, or during
-   * client bootstrap), check whether we want to download any networkstatus
-   * documents. */
-
   /* How often do we check whether we should download network status
    * documents? */
   const int we_are_bootstrapping = networkstatus_consensus_is_bootstrapping(
@@ -1789,19 +2335,23 @@ fetch_networkstatus_callback(time_t now, const or_options_t *options)
   return networkstatus_dl_check_interval;
 }
 
+/**
+ * Periodic callback: Every 60 seconds, we relaunch listeners if any died. */
 static int
 retry_listeners_callback(time_t now, const or_options_t *options)
 {
   (void)now;
   (void)options;
-  /* 3d. And every 60 seconds, we relaunch listeners if any died. */
   if (!net_is_disabled()) {
-    retry_all_listeners(NULL, NULL, 0);
+    retry_all_listeners(NULL, 0);
     return 60;
   }
   return PERIODIC_EVENT_NO_UPDATE;
 }
 
+/**
+ * Periodic callback: as a server, see if we have any old unused circuits
+ * that should be expired */
 static int
 expire_old_ciruits_serverside_callback(time_t now, const or_options_t *options)
 {
@@ -1811,6 +2361,60 @@ expire_old_ciruits_serverside_callback(time_t now, const or_options_t *options)
   return 11;
 }
 
+/**
+ * Callback: Send warnings if Tor doesn't find its ports reachable.
+ */
+static int
+reachability_warnings_callback(time_t now, const or_options_t *options)
+{
+  (void) now;
+
+  if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
+    return (int)(TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT - get_uptime());
+  }
+
+  if (server_mode(options) &&
+      !net_is_disabled() &&
+      have_completed_a_circuit()) {
+    /* every 20 minutes, check and complain if necessary */
+    const routerinfo_t *me = router_get_my_routerinfo();
+    if (me && !check_whether_orport_reachable(options)) {
+      char *address = tor_dup_ip(me->addr);
+      log_warn(LD_CONFIG,"Your server (%s:%d) has not managed to confirm that "
+               "its ORPort is reachable. Relays do not publish descriptors "
+               "until their ORPort and DirPort are reachable. Please check "
+               "your firewalls, ports, address, /etc/hosts file, etc.",
+               address, me->or_port);
+      control_event_server_status(LOG_WARN,
+                                  "REACHABILITY_FAILED ORADDRESS=%s:%d",
+                                  address, me->or_port);
+      tor_free(address);
+    }
+
+    if (me && !check_whether_dirport_reachable(options)) {
+      char *address = tor_dup_ip(me->addr);
+      log_warn(LD_CONFIG,
+               "Your server (%s:%d) has not managed to confirm that its "
+               "DirPort is reachable. Relays do not publish descriptors "
+               "until their ORPort and DirPort are reachable. Please check "
+               "your firewalls, ports, address, /etc/hosts file, etc.",
+               address, me->dir_port);
+      control_event_server_status(LOG_WARN,
+                                  "REACHABILITY_FAILED DIRADDRESS=%s:%d",
+                                  address, me->dir_port);
+      tor_free(address);
+    }
+  }
+
+  return TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT;
+}
+
+static int dns_honesty_first_time = 1;
+
+/**
+ * Periodic event: if we're an exit, see if our DNS server is telling us
+ * obvious lies.
+ */
 static int
 check_dns_honesty_callback(time_t now, const or_options_t *options)
 {
@@ -1822,10 +2426,9 @@ check_dns_honesty_callback(time_t now, const or_options_t *options)
       router_my_exit_policy_is_reject_star())
     return PERIODIC_EVENT_NO_UPDATE;
 
-  static int first_time = 1;
-  if (first_time) {
+  if (dns_honesty_first_time) {
     /* Don't launch right when we start */
-    first_time = 0;
+    dns_honesty_first_time = 0;
     return crypto_rand_int_range(60, 180);
   }
 
@@ -1833,6 +2436,10 @@ check_dns_honesty_callback(time_t now, const or_options_t *options)
   return 12*3600 + crypto_rand_int(12*3600);
 }
 
+/**
+ * Periodic callback: if we're the bridge authority, write a networkstatus
+ * file to disk.
+ */
 static int
 write_bridge_ns_callback(time_t now, const or_options_t *options)
 {
@@ -1845,54 +2452,168 @@ write_bridge_ns_callback(time_t now, const or_options_t *options)
   return PERIODIC_EVENT_NO_UPDATE;
 }
 
+static int heartbeat_callback_first_time = 1;
+
+/**
+ * Periodic callback: write the heartbeat message in the logs.
+ *
+ * If writing the heartbeat message to the logs fails for some reason, retry
+ * again after <b>MIN_HEARTBEAT_PERIOD</b> seconds.
+ */
 static int
-check_fw_helper_app_callback(time_t now, const or_options_t *options)
+heartbeat_callback(time_t now, const or_options_t *options)
 {
-  if (net_is_disabled() ||
-      ! server_mode(options) ||
-      ! options->PortForwarding) {
+  /* Check if heartbeat is disabled */
+  if (!options->HeartbeatPeriod) {
     return PERIODIC_EVENT_NO_UPDATE;
   }
-  /* 11. check the port forwarding app */
 
-#define PORT_FORWARDING_CHECK_INTERVAL 5
-  smartlist_t *ports_to_forward = get_list_of_ports_to_forward();
-  if (ports_to_forward) {
-    tor_check_port_forwarding(options->PortForwardingHelper,
-                              ports_to_forward,
-                              now);
+  /* Skip the first one. */
+  if (heartbeat_callback_first_time) {
+    heartbeat_callback_first_time = 0;
+    return options->HeartbeatPeriod;
+  }
 
-    SMARTLIST_FOREACH(ports_to_forward, char *, cp, tor_free(cp));
-    smartlist_free(ports_to_forward);
+  /* Write the heartbeat message */
+  if (log_heartbeat(now) == 0) {
+    return options->HeartbeatPeriod;
+  } else {
+    /* If we couldn't write the heartbeat log message, try again in the minimum
+     * interval of time. */
+    return MIN_HEARTBEAT_PERIOD;
   }
-  return PORT_FORWARDING_CHECK_INTERVAL;
 }
 
-/** Callback to write heartbeat message in the logs. */
+#define CDM_CLEAN_CALLBACK_INTERVAL 600
 static int
-heartbeat_callback(time_t now, const or_options_t *options)
+clean_consdiffmgr_callback(time_t now, const or_options_t *options)
 {
-  static int first = 1;
-
-  /* Check if heartbeat is disabled */
-  if (!options->HeartbeatPeriod) {
-    return PERIODIC_EVENT_NO_UPDATE;
+  (void)now;
+  if (dir_server_mode(options)) {
+    consdiffmgr_cleanup();
   }
+  return CDM_CLEAN_CALLBACK_INTERVAL;
+}
 
-  /* Write the heartbeat message */
-  if (first) {
-    first = 0; /* Skip the first one. */
-  } else {
-    log_heartbeat(now);
+/*
+ * Periodic callback: Run scheduled events for HS service. This is called
+ * every second.
+ */
+static int
+hs_service_callback(time_t now, const or_options_t *options)
+{
+  (void) options;
+
+  /* We need to at least be able to build circuits and that we actually have
+   * a working network. */
+  if (!have_completed_a_circuit() || net_is_disabled() ||
+      networkstatus_get_live_consensus(now) == NULL) {
+    goto end;
   }
 
-  return options->HeartbeatPeriod;
+  hs_service_run_scheduled_events(now);
+
+ end:
+  /* Every 1 second. */
+  return 1;
 }
 
 /** Timer: used to invoke second_elapsed_callback() once per second. */
 static periodic_timer_t *second_timer = NULL;
-/** Number of libevent errors in the last second: we die if we get too many. */
-static int n_libevent_errors = 0;
+
+/**
+ * Enable or disable the per-second timer as appropriate, creating it if
+ * necessary.
+ */
+void
+reschedule_per_second_timer(void)
+{
+  struct timeval one_second;
+  one_second.tv_sec = 1;
+  one_second.tv_usec = 0;
+
+  if (! second_timer) {
+    second_timer = periodic_timer_new(tor_libevent_get_base(),
+                                      &one_second,
+                                      second_elapsed_callback,
+                                      NULL);
+    tor_assert(second_timer);
+  }
+
+  const bool run_per_second_events =
+    control_any_per_second_event_enabled() || ! net_is_completely_disabled();
+
+  if (run_per_second_events) {
+    periodic_timer_launch(second_timer, &one_second);
+  } else {
+    periodic_timer_disable(second_timer);
+  }
+}
+
+/** Last time that update_current_time was called. */
+static time_t current_second = 0;
+/** Last time that update_current_time updated current_second. */
+static monotime_coarse_t current_second_last_changed;
+
+/**
+ * Set the current time to "now", which should be the value returned by
+ * time().  Check for clock jumps and track the total number of seconds we
+ * have been running.
+ */
+void
+update_current_time(time_t now)
+{
+  if (PREDICT_LIKELY(now == current_second)) {
+    /* We call this function a lot.  Most frequently, the current second
+     * will not have changed, so we just return. */
+    return;
+  }
+
+  const time_t seconds_elapsed = current_second ? (now - current_second) : 0;
+
+  /* Check the wall clock against the monotonic clock, so we can
+   * better tell idleness from clock jumps and/or other shenanigans. */
+  monotime_coarse_t last_updated;
+  memcpy(&last_updated, &current_second_last_changed, sizeof(last_updated));
+  monotime_coarse_get(&current_second_last_changed);
+
+  /** How much clock jumping do we tolerate? */
+#define NUM_JUMPED_SECONDS_BEFORE_WARN 100
+
+  /** How much idleness do we tolerate? */
+#define NUM_IDLE_SECONDS_BEFORE_WARN 3600
+
+  if (seconds_elapsed < -NUM_JUMPED_SECONDS_BEFORE_WARN) {
+    // moving back in time is always a bad sign.
+    circuit_note_clock_jumped(seconds_elapsed, false);
+  } else if (seconds_elapsed >= NUM_JUMPED_SECONDS_BEFORE_WARN) {
+    /* Compare the monotonic clock to the result of time(). */
+    const int32_t monotime_msec_passed =
+      monotime_coarse_diff_msec32(&last_updated,
+                                  &current_second_last_changed);
+    const int monotime_sec_passed = monotime_msec_passed / 1000;
+    const int discrepancy = monotime_sec_passed - (int)seconds_elapsed;
+    /* If the monotonic clock deviates from time(NULL), we have a couple of
+     * possibilities.  On some systems, this means we have been suspended or
+     * sleeping.  Everywhere, it can mean that the wall-clock time has
+     * been changed -- for example, with settimeofday().
+     *
+     * On the other hand, if the monotonic time matches with the wall-clock
+     * time, we've probably just been idle for a while, with no events firing.
+     * we tolerate much more of that.
+     */
+    const bool clock_jumped = abs(discrepancy) > 2;
+
+    if (clock_jumped || seconds_elapsed >= NUM_IDLE_SECONDS_BEFORE_WARN) {
+      circuit_note_clock_jumped(seconds_elapsed, ! clock_jumped);
+    }
+  } else if (seconds_elapsed > 0) {
+    stats_n_seconds_working += seconds_elapsed;
+  }
+
+  update_approx_time(now);
+  current_second = now;
+}
 
 /** Libevent callback: invoked once every second. */
 static void
@@ -1901,83 +2622,22 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg)
   /* XXXX This could be sensibly refactored into multiple callbacks, and we
    * could use Libevent's timers for this rather than checking the current
    * time against a bunch of timeouts every second. */
-  static time_t current_second = 0;
   time_t now;
-  size_t bytes_written;
-  size_t bytes_read;
-  int seconds_elapsed;
-  const or_options_t *options = get_options();
   (void)timer;
   (void)arg;
 
-  n_libevent_errors = 0;
-
-  /* log_notice(LD_GENERAL, "Tick."); */
   now = time(NULL);
-  update_approx_time(now);
-
-  /* the second has rolled over. check more stuff. */
-  seconds_elapsed = current_second ? (int)(now - current_second) : 0;
-  bytes_read = (size_t)(stats_n_bytes_read - stats_prev_n_read);
-  bytes_written = (size_t)(stats_n_bytes_written - stats_prev_n_written);
-  stats_prev_n_read = stats_n_bytes_read;
-  stats_prev_n_written = stats_n_bytes_written;
-
-  control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written);
-  control_event_stream_bandwidth_used();
-  control_event_conn_bandwidth_used();
-  control_event_circ_bandwidth_used();
-  control_event_circuit_cell_stats();
 
-  if (server_mode(options) &&
-      !net_is_disabled() &&
-      seconds_elapsed > 0 &&
-      have_completed_a_circuit() &&
-      stats_n_seconds_working / TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT !=
-      (stats_n_seconds_working+seconds_elapsed) /
-        TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
-    /* every 20 minutes, check and complain if necessary */
-    const routerinfo_t *me = router_get_my_routerinfo();
-    if (me && !check_whether_orport_reachable(options)) {
-      char *address = tor_dup_ip(me->addr);
-      log_warn(LD_CONFIG,"Your server (%s:%d) has not managed to confirm that "
-               "its ORPort is reachable. Relays do not publish descriptors "
-               "until their ORPort and DirPort are reachable. Please check "
-               "your firewalls, ports, address, /etc/hosts file, etc.",
-               address, me->or_port);
-      control_event_server_status(LOG_WARN,
-                                  "REACHABILITY_FAILED ORADDRESS=%s:%d",
-                                  address, me->or_port);
-      tor_free(address);
-    }
-
-    if (me && !check_whether_dirport_reachable(options)) {
-      char *address = tor_dup_ip(me->addr);
-      log_warn(LD_CONFIG,
-               "Your server (%s:%d) has not managed to confirm that its "
-               "DirPort is reachable. Relays do not publish descriptors "
-               "until their ORPort and DirPort are reachable. Please check "
-               "your firewalls, ports, address, /etc/hosts file, etc.",
-               address, me->dir_port);
-      control_event_server_status(LOG_WARN,
-                                  "REACHABILITY_FAILED DIRADDRESS=%s:%d",
-                                  address, me->dir_port);
-      tor_free(address);
-    }
-  }
+  /* We don't need to do this once-per-second any more: time-updating is
+   * only in this callback _because it is a callback_. It should be fine
+   * to disable this callback, and the time will still get updated.
+   */
+  update_current_time(now);
 
-/** If more than this many seconds have elapsed, probably the clock
- * jumped: doesn't count. */
-#define NUM_JUMPED_SECONDS_BEFORE_WARN 100
-  if (seconds_elapsed < -NUM_JUMPED_SECONDS_BEFORE_WARN ||
-      seconds_elapsed >= NUM_JUMPED_SECONDS_BEFORE_WARN) {
-    circuit_note_clock_jumped(seconds_elapsed);
-  } else if (seconds_elapsed > 0)
-    stats_n_seconds_working += seconds_elapsed;
+  /* Maybe some controller events are ready to fire */
+  control_per_second_events();
 
   run_scheduled_events(now);
-
-  current_second = now; /* remember which second it is, for next time */
 }
 
 #ifdef HAVE_SYSTEMD_209
@@ -1991,71 +2651,7 @@ systemd_watchdog_callback(periodic_timer_t *timer, void *arg)
   (void)arg;
   sd_notify(0, "WATCHDOG=1");
 }
-#endif
-
-/** Timer: used to invoke refill_callback(). */
-static periodic_timer_t *refill_timer = NULL;
-
-/** Libevent callback: invoked periodically to refill token buckets
- * and count r/w bytes. */
-static void
-refill_callback(periodic_timer_t *timer, void *arg)
-{
-  static struct timeval current_millisecond;
-  struct timeval now;
-
-  size_t bytes_written;
-  size_t bytes_read;
-  int milliseconds_elapsed = 0;
-  int seconds_rolled_over = 0;
-
-  const or_options_t *options = get_options();
-
-  (void)timer;
-  (void)arg;
-
-  tor_gettimeofday(&now);
-
-  /* If this is our first time, no time has passed. */
-  if (current_millisecond.tv_sec) {
-    long mdiff = tv_mdiff(&current_millisecond, &now);
-    if (mdiff > INT_MAX)
-      mdiff = INT_MAX;
-    milliseconds_elapsed = (int)mdiff;
-    seconds_rolled_over = (int)(now.tv_sec - current_millisecond.tv_sec);
-  }
-
-  bytes_written = stats_prev_global_write_bucket - global_write_bucket;
-  bytes_read = stats_prev_global_read_bucket - global_read_bucket;
-
-  stats_n_bytes_read += bytes_read;
-  stats_n_bytes_written += bytes_written;
-  if (accounting_is_enabled(options) && milliseconds_elapsed >= 0)
-    accounting_add_bytes(bytes_read, bytes_written, seconds_rolled_over);
-
-  if (milliseconds_elapsed > 0)
-    connection_bucket_refill(milliseconds_elapsed, (time_t)now.tv_sec);
-
-  stats_prev_global_read_bucket = global_read_bucket;
-  stats_prev_global_write_bucket = global_write_bucket;
-
-  current_millisecond = now; /* remember what time it is, for next time */
-}
-
-#ifndef _WIN32
-/** Called when a possibly ignorable libevent error occurs; ensures that we
- * don't get into an infinite loop by ignoring too many errors from
- * libevent. */
-static int
-got_libevent_error(void)
-{
-  if (++n_libevent_errors > 8) {
-    log_err(LD_NET, "Too many libevent errors in one second; dying");
-    return -1;
-  }
-  return 0;
-}
-#endif
+#endif /* defined(HAVE_SYSTEMD_209) */
 
 #define UPTIME_CUTOFF_FOR_NEW_BANDWIDTH_TEST (6*60*60)
 
@@ -2078,9 +2674,9 @@ ip_address_changed(int at_interface)
     }
   } else {
     if (server) {
-      if (stats_n_seconds_working > UPTIME_CUTOFF_FOR_NEW_BANDWIDTH_TEST)
+      if (get_uptime() > UPTIME_CUTOFF_FOR_NEW_BANDWIDTH_TEST)
         reset_bandwidth_test();
-      stats_n_seconds_working = 0;
+      reset_uptime();
       router_reset_reachability();
     }
   }
@@ -2108,202 +2704,34 @@ dns_servers_relaunch_checks(void)
   }
 }
 
-/** Called when we get a SIGHUP: reload configuration files and keys,
- * retry all connections, and so on. */
-static int
-do_hup(void)
+/** Initialize some mainloop_event_t objects that we require. */
+void
+initialize_mainloop_events(void)
 {
-  const or_options_t *options = get_options();
-
-#ifdef USE_DMALLOC
-  dmalloc_log_stats();
-  dmalloc_log_changed(0, 1, 0, 0);
-#endif
+  initialize_periodic_events();
 
-  log_notice(LD_GENERAL,"Received reload signal (hup). Reloading config and "
-             "resetting internal state.");
-  if (accounting_is_enabled(options))
-    accounting_record_bandwidth_usage(time(NULL), get_or_state());
-
-  router_reset_warnings();
-  routerlist_reset_warnings();
-  /* first, reload config variables, in case they've changed */
-  if (options->ReloadTorrcOnSIGHUP) {
-    /* no need to provide argc/v, they've been cached in init_from_config */
-    if (options_init_from_torrc(0, NULL) < 0) {
-      log_err(LD_CONFIG,"Reading config failed--see warnings above. "
-              "For usage, try -h.");
-      return -1;
-    }
-    options = get_options(); /* they have changed now */
-    /* Logs are only truncated the first time they are opened, but were
-       probably intended to be cleaned up on signal. */
-    if (options->TruncateLogFile)
-      truncate_logs();
-  } else {
-    char *msg = NULL;
-    log_notice(LD_GENERAL, "Not reloading config file: the controller told "
-               "us not to.");
-    /* Make stuff get rescanned, reloaded, etc. */
-    if (set_options((or_options_t*)options, &msg) < 0) {
-      if (!msg)
-        msg = tor_strdup("Unknown error");
-      log_warn(LD_GENERAL, "Unable to re-set previous options: %s", msg);
-      tor_free(msg);
-    }
+  if (!schedule_active_linked_connections_event) {
+    schedule_active_linked_connections_event =
+      mainloop_event_postloop_new(schedule_active_linked_connections_cb, NULL);
   }
-  if (authdir_mode_handles_descs(options, -1)) {
-    /* reload the approved-routers file */
-    if (dirserv_load_fingerprint_file() < 0) {
-      /* warnings are logged from dirserv_load_fingerprint_file() directly */
-      log_info(LD_GENERAL, "Error reloading fingerprints. "
-               "Continuing with old list.");
-    }
+  if (!postloop_cleanup_ev) {
+    postloop_cleanup_ev =
+      mainloop_event_postloop_new(postloop_cleanup_cb, NULL);
   }
-
-  /* Rotate away from the old dirty circuits. This has to be done
-   * after we've read the new options, but before we start using
-   * circuits for directory fetches. */
-  circuit_mark_all_dirty_circs_as_unusable();
-
-  /* retry appropriate downloads */
-  router_reset_status_download_failures();
-  router_reset_descriptor_download_failures();
-  if (!options->DisableNetwork)
-    update_networkstatus_downloads(time(NULL));
-
-  /* We'll retry routerstatus downloads in about 10 seconds; no need to
-   * force a retry there. */
-
-  if (server_mode(options)) {
-    /* Maybe we've been given a new ed25519 key or certificate?
-     */
-    time_t now = approx_time();
-    if (load_ed_keys(options, now) < 0 ||
-         generate_ed_link_cert(options, now)) {
-      log_warn(LD_OR, "Problem reloading Ed25519 keys; still using old keys.");
-    }
-
-    /* Update cpuworker and dnsworker processes, so they get up-to-date
-     * configuration options. */
-    cpuworkers_rotate_keyinfo();
-    dns_reset();
-  }
-  return 0;
 }
 
 /** Tor main loop. */
 int
 do_main_loop(void)
 {
-  time_t now;
-
   /* initialize the periodic events first, so that code that depends on the
    * events being present does not assert.
    */
-  if (! periodic_events_initialized) {
-    initialize_periodic_events();
-  }
-
-  /* initialize dns resolve map, spawn workers if needed */
-  if (dns_init() < 0) {
-    if (get_options()->ServerDNSAllowBrokenConfig)
-      log_warn(LD_GENERAL, "Couldn't set up any working nameservers. "
-               "Network not up yet?  Will try again soon.");
-    else {
-      log_err(LD_GENERAL,"Error initializing dns subsystem; exiting.  To "
-              "retry instead, set the ServerDNSAllowBrokenResolvConf option.");
-    }
-  }
-
-  handle_signals(1);
-
-  /* load the private keys, if we're supposed to have them, and set up the
-   * TLS context. */
-  if (! client_identity_key_is_set()) {
-    if (init_keys() < 0) {
-      log_err(LD_OR, "Error initializing keys; exiting");
-      return -1;
-    }
-  }
-
-  /* Set up our buckets */
-  connection_bucket_init();
-  stats_prev_global_read_bucket = global_read_bucket;
-  stats_prev_global_write_bucket = global_write_bucket;
-
-  /* initialize the bootstrap status events to know we're starting up */
-  control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0);
-
-  /* Initialize the keypinning log. */
-  if (authdir_mode_v3(get_options())) {
-    char *fname = get_datadir_fname("key-pinning-journal");
-    int r = 0;
-    if (keypin_load_journal(fname)<0) {
-      log_err(LD_DIR, "Error loading key-pinning journal: %s",strerror(errno));
-      r = -1;
-    }
-    if (keypin_open_journal(fname)<0) {
-      log_err(LD_DIR, "Error opening key-pinning journal: %s",strerror(errno));
-      r = -1;
-    }
-    tor_free(fname);
-    if (r)
-      return r;
-  }
-  {
-    /* This is the old name for key-pinning-journal.  These got corrupted
-     * in a couple of cases by #16530, so we started over. See #16580 for
-     * the rationale and for other options we didn't take.  We can remove
-     * this code once all the authorities that ran 0.2.7.1-alpha-dev are
-     * upgraded.
-     */
-    char *fname = get_datadir_fname("key-pinning-entries");
-    unlink(fname);
-    tor_free(fname);
-  }
-
-  if (trusted_dirs_reload_certs()) {
-    log_warn(LD_DIR,
-             "Couldn't load all cached v3 certificates. Starting anyway.");
-  }
-  if (router_reload_consensus_networkstatus()) {
-    return -1;
-  }
-  /* load the routers file, or assign the defaults. */
-  if (router_reload_router_list()) {
-    return -1;
-  }
-  /* load the networkstatuses. (This launches a download for new routers as
-   * appropriate.)
-   */
-  now = time(NULL);
-  directory_info_has_arrived(now, 1, 0);
-
-  if (server_mode(get_options())) {
-    /* launch cpuworkers. Need to do this *after* we've read the onion key. */
-    cpu_init();
-  }
-
-  /* Setup shared random protocol subsystem. */
-  if (authdir_mode_publishes_statuses(get_options())) {
-    if (sr_init(1) < 0) {
-      return -1;
-    }
-  }
+  initialize_periodic_events();
+  initialize_mainloop_events();
 
   /* set up once-a-second callback. */
-  if (! second_timer) {
-    struct timeval one_second;
-    one_second.tv_sec = 1;
-    one_second.tv_usec = 0;
-
-    second_timer = periodic_timer_new(tor_libevent_get_base(),
-                                      &one_second,
-                                      second_elapsed_callback,
-                                      NULL);
-    tor_assert(second_timer);
-  }
+  reschedule_per_second_timer();
 
 #ifdef HAVE_SYSTEMD_209
   uint64_t watchdog_delay;
@@ -2325,32 +2753,32 @@ do_main_loop(void)
       tor_assert(systemd_watchdog_timer);
     }
   }
-#endif
+#endif /* defined(HAVE_SYSTEMD_209) */
 
-  if (!refill_timer) {
-    struct timeval refill_interval;
-    int msecs = get_options()->TokenBucketRefillInterval;
+  main_loop_should_exit = 0;
+  main_loop_exit_value = 0;
 
-    refill_interval.tv_sec =  msecs/1000;
-    refill_interval.tv_usec = (msecs%1000)*1000;
-
-    refill_timer = periodic_timer_new(tor_libevent_get_base(),
-                                      &refill_interval,
-                                      refill_callback,
-                                      NULL);
-    tor_assert(refill_timer);
-  }
-
-#ifdef HAVE_SYSTEMD
+#ifdef ENABLE_RESTART_DEBUGGING
   {
-    const int r = sd_notify(0, "READY=1");
-    if (r < 0) {
-      log_warn(LD_GENERAL, "Unable to send readiness to systemd: %s",
-               strerror(r));
-    } else if (r > 0) {
-      log_notice(LD_GENERAL, "Signaled readiness to systemd");
-    } else {
-      log_info(LD_GENERAL, "Systemd NOTIFY_SOCKET not present.");
+    static int first_time = 1;
+
+    if (first_time && getenv("TOR_DEBUG_RESTART")) {
+      first_time = 0;
+      const char *sec_str = getenv("TOR_DEBUG_RESTART_AFTER_SECONDS");
+      long sec;
+      int sec_ok=0;
+      if (sec_str &&
+          (sec = tor_parse_long(sec_str, 10, 0, INT_MAX, &sec_ok, NULL)) &&
+          sec_ok) {
+        /* Okay, we parsed the seconds. */
+      } else {
+        sec = 5;
+      }
+      struct timeval restart_after = { (time_t) sec, 0 };
+      tor_shutdown_event_loop_for_restart_event =
+        tor_evtimer_new(tor_libevent_get_base(),
+                        tor_shutdown_event_loop_for_restart_cb, NULL);
+      event_add(tor_shutdown_event_loop_for_restart_event, &restart_after);
     }
   }
 #endif
@@ -2358,6 +2786,11 @@ do_main_loop(void)
   return run_main_loop_until_done();
 }
 
+#ifndef _WIN32
+/** Rate-limiter for EINVAL-type libevent warnings. */
+static ratelim_t libevent_error_ratelim = RATELIM_INIT(10);
+#endif
+
 /**
  * Run the main loop a single time. Return 0 for "exit"; -1 for "exit with
  * error", and 1 for "run this again."
@@ -2370,23 +2803,50 @@ run_main_loop_once(void)
   if (nt_service_is_stopping())
     return 0;
 
+  if (main_loop_should_exit)
+    return 0;
+
 #ifndef _WIN32
   /* Make it easier to tell whether libevent failure is our fault or not. */
   errno = 0;
 #endif
-  /* All active linked conns should get their read events activated. */
-  SMARTLIST_FOREACH(active_linked_connection_lst, connection_t *, conn,
-                    event_active(conn->read_event, EV_READ, 1));
-  called_loop_once = smartlist_len(active_linked_connection_lst) ? 1 : 0;
 
+  if (get_options()->MainloopStats) {
+    /* We always enforce that EVLOOP_ONCE is passed to event_base_loop() if we
+     * are collecting main loop statistics. */
+    called_loop_once = 1;
+  } else {
+    called_loop_once = 0;
+  }
+
+  /* Make sure we know (about) what time it is. */
   update_approx_time(time(NULL));
 
-  /* poll until we have an event, or the second ends, or until we have
-   * some active linked connections to trigger events for. */
-  loop_result = event_base_loop(tor_libevent_get_base(),
-                                called_loop_once ? EVLOOP_ONCE : 0);
+  /* Here it is: the main loop.  Here we tell Libevent to poll until we have
+   * an event, or the second ends, or until we have some active linked
+   * connections to trigger events for.  Libevent will wait till one
+   * of these happens, then run all the appropriate callbacks. */
+  loop_result = tor_libevent_run_event_loop(tor_libevent_get_base(),
+                                            called_loop_once);
+
+  if (get_options()->MainloopStats) {
+    /* Update our main loop counters. */
+    if (loop_result == 0) {
+      // The call was successful.
+      increment_main_loop_success_count();
+    } else if (loop_result == -1) {
+      // The call was erroneous.
+      increment_main_loop_error_count();
+    } else if (loop_result == 1) {
+      // The call didn't have any active or pending events
+      // to handle.
+      increment_main_loop_idle_count();
+    }
+  }
 
-  /* let catch() handle things like ^c, and otherwise don't worry about it */
+  /* Oh, the loop failed.  That might be an error that we need to
+   * catch, but more likely, it's just an interrupted poll() call or something,
+   * and we should try again. */
   if (loop_result < 0) {
     int e = tor_socket_errno(-1);
     /* let the program survive things like ^z */
@@ -2396,10 +2856,13 @@ run_main_loop_once(void)
       return -1;
 #ifndef _WIN32
     } else if (e == EINVAL) {
-      log_warn(LD_NET, "EINVAL from libevent: should you upgrade libevent?");
-      if (got_libevent_error())
+      log_fn_ratelim(&libevent_error_ratelim, LOG_WARN, LD_NET,
+                     "EINVAL from libevent: should you upgrade libevent?");
+      if (libevent_error_ratelim.n_calls_since_last_time > 8) {
+        log_err(LD_NET, "Too many libevent errors, too fast: dying");
         return -1;
-#endif
+      }
+#endif /* !defined(_WIN32) */
     } else {
       tor_assert_nonfatal_once(! ERRNO_IS_EINPROGRESS(e));
       log_debug(LD_NET,"libevent call interrupted.");
@@ -2409,10 +2872,8 @@ run_main_loop_once(void)
     }
   }
 
-  /* This will be pretty fast if nothing new is pending. Note that this gets
-   * called once per libevent loop, which will make it happen once per group
-   * of events that fire, or once per second. */
-  connection_ap_attach_pending(0);
+  if (main_loop_should_exit)
+    return 0;
 
   return 1;
 }
@@ -2429,100 +2890,11 @@ run_main_loop_until_done(void)
   do {
     loop_result = run_main_loop_once();
   } while (loop_result == 1);
-  return loop_result;
-}
 
-/** Libevent callback: invoked when we get a signal.
- */
-static void
-signal_callback(evutil_socket_t fd, short events, void *arg)
-{
-  const int *sigptr = arg;
-  const int sig = *sigptr;
-  (void)fd;
-  (void)events;
-
-  process_signal(sig);
-}
-
-/** Do the work of acting on a signal received in <b>sig</b> */
-static void
-process_signal(int sig)
-{
-  switch (sig)
-    {
-    case SIGTERM:
-      log_notice(LD_GENERAL,"Catching signal TERM, exiting cleanly.");
-      tor_cleanup();
-      exit(0);
-      break;
-    case SIGINT:
-      if (!server_mode(get_options())) { /* do it now */
-        log_notice(LD_GENERAL,"Interrupt: exiting cleanly.");
-        tor_cleanup();
-        exit(0);
-      }
-#ifdef HAVE_SYSTEMD
-      sd_notify(0, "STOPPING=1");
-#endif
-      hibernate_begin_shutdown();
-      break;
-#ifdef SIGPIPE
-    case SIGPIPE:
-      log_debug(LD_GENERAL,"Caught SIGPIPE. Ignoring.");
-      break;
-#endif
-    case SIGUSR1:
-      /* prefer to log it at INFO, but make sure we always see it */
-      dumpstats(get_min_log_level()<LOG_INFO ? get_min_log_level() : LOG_INFO);
-      control_event_signal(sig);
-      break;
-    case SIGUSR2:
-      switch_logs_debug();
-      log_debug(LD_GENERAL,"Caught USR2, going to loglevel debug. "
-                "Send HUP to change back.");
-      control_event_signal(sig);
-      break;
-    case SIGHUP:
-#ifdef HAVE_SYSTEMD
-      sd_notify(0, "RELOADING=1");
-#endif
-      if (do_hup() < 0) {
-        log_warn(LD_CONFIG,"Restart failed (config error?). Exiting.");
-        tor_cleanup();
-        exit(1);
-      }
-#ifdef HAVE_SYSTEMD
-      sd_notify(0, "READY=1");
-#endif
-      control_event_signal(sig);
-      break;
-#ifdef SIGCHLD
-    case SIGCHLD:
-      notify_pending_waitpid_callbacks();
-      break;
-#endif
-    case SIGNEWNYM: {
-      time_t now = time(NULL);
-      if (time_of_last_signewnym + MAX_SIGNEWNYM_RATE > now) {
-        signewnym_is_pending = 1;
-        log_notice(LD_CONTROL,
-            "Rate limiting NEWNYM request: delaying by %d second(s)",
-            (int)(MAX_SIGNEWNYM_RATE+time_of_last_signewnym-now));
-      } else {
-        signewnym_impl(now);
-      }
-      break;
-    }
-    case SIGCLEARDNSCACHE:
-      addressmap_clear_transient();
-      control_event_signal(sig);
-      break;
-    case SIGHEARTBEAT:
-      log_heartbeat(time(NULL));
-      control_event_signal(sig);
-      break;
-  }
+  if (main_loop_should_exit)
+    return main_loop_exit_value;
+  else
+    return loop_result;
 }
 
 /** Returns Tor's uptime. */
@@ -2532,1002 +2904,49 @@ get_uptime,(void))
   return stats_n_seconds_working;
 }
 
-/**
- * Write current memory usage information to the log.
- */
-static void
-dumpmemusage(int severity)
-{
-  connection_dump_buffer_mem_stats(severity);
-  tor_log(severity, LD_GENERAL, "In rephist: "U64_FORMAT" used by %d Tors.",
-      U64_PRINTF_ARG(rephist_total_alloc), rephist_total_num);
-  dump_routerlist_mem_usage(severity);
-  dump_cell_pool_usage(severity);
-  dump_dns_mem_usage(severity);
-  tor_log_mallinfo(severity);
-}
-
-/** Write all statistics to the log, with log level <b>severity</b>. Called
- * in response to a SIGUSR1. */
-static void
-dumpstats(int severity)
-{
-  time_t now = time(NULL);
-  time_t elapsed;
-  size_t rbuf_cap, wbuf_cap, rbuf_len, wbuf_len;
-
-  tor_log(severity, LD_GENERAL, "Dumping stats:");
-
-  SMARTLIST_FOREACH_BEGIN(connection_array, connection_t *, conn) {
-    int i = conn_sl_idx;
-    tor_log(severity, LD_GENERAL,
-        "Conn %d (socket %d) type %d (%s), state %d (%s), created %d secs ago",
-        i, (int)conn->s, conn->type, conn_type_to_string(conn->type),
-        conn->state, conn_state_to_string(conn->type, conn->state),
-        (int)(now - conn->timestamp_created));
-    if (!connection_is_listener(conn)) {
-      tor_log(severity,LD_GENERAL,
-          "Conn %d is to %s:%d.", i,
-          safe_str_client(conn->address),
-          conn->port);
-      tor_log(severity,LD_GENERAL,
-          "Conn %d: %d bytes waiting on inbuf (len %d, last read %d secs ago)",
-          i,
-          (int)connection_get_inbuf_len(conn),
-          (int)buf_allocation(conn->inbuf),
-          (int)(now - conn->timestamp_lastread));
-      tor_log(severity,LD_GENERAL,
-          "Conn %d: %d bytes waiting on outbuf "
-          "(len %d, last written %d secs ago)",i,
-          (int)connection_get_outbuf_len(conn),
-          (int)buf_allocation(conn->outbuf),
-          (int)(now - conn->timestamp_lastwritten));
-      if (conn->type == CONN_TYPE_OR) {
-        or_connection_t *or_conn = TO_OR_CONN(conn);
-        if (or_conn->tls) {
-          if (tor_tls_get_buffer_sizes(or_conn->tls, &rbuf_cap, &rbuf_len,
-                                       &wbuf_cap, &wbuf_len) == 0) {
-            tor_log(severity, LD_GENERAL,
-                "Conn %d: %d/%d bytes used on OpenSSL read buffer; "
-                "%d/%d bytes used on write buffer.",
-                i, (int)rbuf_len, (int)rbuf_cap, (int)wbuf_len, (int)wbuf_cap);
-          }
-        }
-      }
-    }
-    circuit_dump_by_conn(conn, severity); /* dump info about all the circuits
-                                           * using this conn */
-  } SMARTLIST_FOREACH_END(conn);
-
-  channel_dumpstats(severity);
-  channel_listener_dumpstats(severity);
-
-  tor_log(severity, LD_NET,
-      "Cells processed: "U64_FORMAT" padding\n"
-      "                 "U64_FORMAT" create\n"
-      "                 "U64_FORMAT" created\n"
-      "                 "U64_FORMAT" relay\n"
-      "                        ("U64_FORMAT" relayed)\n"
-      "                        ("U64_FORMAT" delivered)\n"
-      "                 "U64_FORMAT" destroy",
-      U64_PRINTF_ARG(stats_n_padding_cells_processed),
-      U64_PRINTF_ARG(stats_n_create_cells_processed),
-      U64_PRINTF_ARG(stats_n_created_cells_processed),
-      U64_PRINTF_ARG(stats_n_relay_cells_processed),
-      U64_PRINTF_ARG(stats_n_relay_cells_relayed),
-      U64_PRINTF_ARG(stats_n_relay_cells_delivered),
-      U64_PRINTF_ARG(stats_n_destroy_cells_processed));
-  if (stats_n_data_cells_packaged)
-    tor_log(severity,LD_NET,"Average packaged cell fullness: %2.3f%%",
-        100*(U64_TO_DBL(stats_n_data_bytes_packaged) /
-             U64_TO_DBL(stats_n_data_cells_packaged*RELAY_PAYLOAD_SIZE)) );
-  if (stats_n_data_cells_received)
-    tor_log(severity,LD_NET,"Average delivered cell fullness: %2.3f%%",
-        100*(U64_TO_DBL(stats_n_data_bytes_received) /
-             U64_TO_DBL(stats_n_data_cells_received*RELAY_PAYLOAD_SIZE)) );
-
-  cpuworker_log_onionskin_overhead(severity, ONION_HANDSHAKE_TYPE_TAP, "TAP");
-  cpuworker_log_onionskin_overhead(severity, ONION_HANDSHAKE_TYPE_NTOR,"ntor");
-
-  if (now - time_of_process_start >= 0)
-    elapsed = now - time_of_process_start;
-  else
-    elapsed = 0;
-
-  if (elapsed) {
-    tor_log(severity, LD_NET,
-        "Average bandwidth: "U64_FORMAT"/%d = %d bytes/sec reading",
-        U64_PRINTF_ARG(stats_n_bytes_read),
-        (int)elapsed,
-        (int) (stats_n_bytes_read/elapsed));
-    tor_log(severity, LD_NET,
-        "Average bandwidth: "U64_FORMAT"/%d = %d bytes/sec writing",
-        U64_PRINTF_ARG(stats_n_bytes_written),
-        (int)elapsed,
-        (int) (stats_n_bytes_written/elapsed));
-  }
-
-  tor_log(severity, LD_NET, "--------------- Dumping memory information:");
-  dumpmemusage(severity);
-
-  rep_hist_dump_stats(now,severity);
-  rend_service_dump_stats(severity);
-  dump_pk_ops(severity);
-  dump_distinct_digest_count(severity);
-}
-
-/** Called by exit() as we shut down the process.
- */
-static void
-exit_function(void)
-{
-  /* NOTE: If we ever daemonize, this gets called immediately.  That's
-   * okay for now, because we only use this on Windows.  */
-#ifdef _WIN32
-  WSACleanup();
-#endif
-}
-
-#ifdef _WIN32
-#define UNIX_ONLY 0
-#else
-#define UNIX_ONLY 1
-#endif
-static struct {
-  int signal_value;
-  int try_to_register;
-  struct event *signal_event;
-} signal_handlers[] = {
-#ifdef SIGINT
-  { SIGINT, UNIX_ONLY, NULL }, /* do a controlled slow shutdown */
-#endif
-#ifdef SIGTERM
-  { SIGTERM, UNIX_ONLY, NULL }, /* to terminate now */
-#endif
-#ifdef SIGPIPE
-  { SIGPIPE, UNIX_ONLY, NULL }, /* otherwise SIGPIPE kills us */
-#endif
-#ifdef SIGUSR1
-  { SIGUSR1, UNIX_ONLY, NULL }, /* dump stats */
-#endif
-#ifdef SIGUSR2
-  { SIGUSR2, UNIX_ONLY, NULL }, /* go to loglevel debug */
-#endif
-#ifdef SIGHUP
-  { SIGHUP, UNIX_ONLY, NULL }, /* to reload config, retry conns, etc */
-#endif
-#ifdef SIGXFSZ
-  { SIGXFSZ, UNIX_ONLY, NULL }, /* handle file-too-big resource exhaustion */
-#endif
-#ifdef SIGCHLD
-  { SIGCHLD, UNIX_ONLY, NULL }, /* handle dns/cpu workers that exit */
-#endif
-  /* These are controller-only */
-  { SIGNEWNYM, 0, NULL },
-  { SIGCLEARDNSCACHE, 0, NULL },
-  { SIGHEARTBEAT, 0, NULL },
-  { -1, -1, NULL }
-};
-
-/** Set up the signal handlers for either parent or child process */
-void
-handle_signals(int is_parent)
-{
-  int i;
-  if (is_parent) {
-    for (i = 0; signal_handlers[i].signal_value >= 0; ++i) {
-      if (signal_handlers[i].try_to_register) {
-        signal_handlers[i].signal_event =
-          tor_evsignal_new(tor_libevent_get_base(),
-                           signal_handlers[i].signal_value,
-                           signal_callback,
-                           &signal_handlers[i].signal_value);
-        if (event_add(signal_handlers[i].signal_event, NULL))
-          log_warn(LD_BUG, "Error from libevent when adding "
-                   "event for signal %d",
-                   signal_handlers[i].signal_value);
-      } else {
-        signal_handlers[i].signal_event =
-          tor_event_new(tor_libevent_get_base(), -1,
-                        EV_SIGNAL, signal_callback,
-                        &signal_handlers[i].signal_value);
-      }
-    }
-  } else {
-#ifndef _WIN32
-    struct sigaction action;
-    action.sa_flags = 0;
-    sigemptyset(&action.sa_mask);
-    action.sa_handler = SIG_IGN;
-    sigaction(SIGINT,  &action, NULL);
-    sigaction(SIGTERM, &action, NULL);
-    sigaction(SIGPIPE, &action, NULL);
-    sigaction(SIGUSR1, &action, NULL);
-    sigaction(SIGUSR2, &action, NULL);
-    sigaction(SIGHUP,  &action, NULL);
-#ifdef SIGXFSZ
-    sigaction(SIGXFSZ, &action, NULL);
-#endif
-#endif
-  }
-}
-
-/* Make sure the signal handler for signal_num will be called. */
-void
-activate_signal(int signal_num)
-{
-  int i;
-  for (i = 0; signal_handlers[i].signal_value >= 0; ++i) {
-    if (signal_handlers[i].signal_value == signal_num) {
-      event_active(signal_handlers[i].signal_event, EV_SIGNAL, 1);
-      return;
-    }
-  }
-}
-
-/** Main entry point for the Tor command-line client.
- */
-int
-tor_init(int argc, char *argv[])
-{
-  char progname[256];
-  int quiet = 0;
-
-  time_of_process_start = time(NULL);
-  init_connection_lists();
-  /* Have the log set up with our application name. */
-  tor_snprintf(progname, sizeof(progname), "Tor %s", get_version());
-  log_set_application_name(progname);
-
-  /* Set up the crypto nice and early */
-  if (crypto_early_init() < 0) {
-    log_err(LD_GENERAL, "Unable to initialize the crypto subsystem!");
-    return -1;
-  }
-
-  /* Initialize the history structures. */
-  rep_hist_init();
-  /* Initialize the service cache. */
-  rend_cache_init();
-  addressmap_init(); /* Init the client dns cache. Do it always, since it's
-                      * cheap. */
-
-  {
-  /* We search for the "quiet" option first, since it decides whether we
-   * will log anything at all to the command line. */
-    config_line_t *opts = NULL, *cmdline_opts = NULL;
-    const config_line_t *cl;
-    (void) config_parse_commandline(argc, argv, 1, &opts, &cmdline_opts);
-    for (cl = cmdline_opts; cl; cl = cl->next) {
-      if (!strcmp(cl->key, "--hush"))
-        quiet = 1;
-      if (!strcmp(cl->key, "--quiet") ||
-          !strcmp(cl->key, "--dump-config"))
-        quiet = 2;
-      /* The following options imply --hush */
-      if (!strcmp(cl->key, "--version") || !strcmp(cl->key, "--digests") ||
-          !strcmp(cl->key, "--list-torrc-options") ||
-          !strcmp(cl->key, "--library-versions") ||
-          !strcmp(cl->key, "--hash-password") ||
-          !strcmp(cl->key, "-h") || !strcmp(cl->key, "--help")) {
-        if (quiet < 1)
-          quiet = 1;
-      }
-    }
-    config_free_lines(opts);
-    config_free_lines(cmdline_opts);
-  }
-
- /* give it somewhere to log to initially */
-  switch (quiet) {
-    case 2:
-      /* no initial logging */
-      break;
-    case 1:
-      add_temp_log(LOG_WARN);
-      break;
-    default:
-      add_temp_log(LOG_NOTICE);
-  }
-  quiet_level = quiet;
-
-  {
-    const char *version = get_version();
-
-    log_notice(LD_GENERAL, "Tor %s running on %s with Libevent %s, "
-               "OpenSSL %s and Zlib %s.", version,
-               get_uname(),
-               tor_libevent_get_version_str(),
-               crypto_openssl_get_version_str(),
-               tor_zlib_get_version_str());
-
-    log_notice(LD_GENERAL, "Tor can't help you if you use it wrong! "
-               "Learn how to be safe at "
-               "https://www.torproject.org/download/download#warning");
-
-    if (strstr(version, "alpha") || strstr(version, "beta"))
-      log_notice(LD_GENERAL, "This version is not a stable Tor release. "
-                 "Expect more bugs than usual.");
-  }
-
-  if (network_init()<0) {
-    log_err(LD_BUG,"Error initializing network; exiting.");
-    return -1;
-  }
-  atexit(exit_function);
-
-  if (options_init_from_torrc(argc,argv) < 0) {
-    log_err(LD_CONFIG,"Reading config failed--see warnings above.");
-    return -1;
-  }
-
-  /* The options are now initialised */
-  const or_options_t *options = get_options();
-
-#ifndef _WIN32
-  if (geteuid()==0)
-    log_warn(LD_GENERAL,"You are running Tor as root. You don't need to, "
-             "and you probably shouldn't.");
-#endif
-
-  if (crypto_global_init(options->HardwareAccel,
-                         options->AccelName,
-                         options->AccelDir)) {
-    log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting.");
-    return -1;
-  }
-  stream_choice_seed_weak_rng();
-  if (tor_init_libevent_rng() < 0) {
-    log_warn(LD_NET, "Problem initializing libevent RNG.");
-  }
-
-  /* Scan/clean unparseable descroptors; after reading config */
-  routerparse_init();
-
-  return 0;
-}
-
-/** A lockfile structure, used to prevent two Tors from messing with the
- * data directory at once.  If this variable is non-NULL, we're holding
- * the lockfile. */
-static tor_lockfile_t *lockfile = NULL;
-
-/** Try to grab the lock file described in <b>options</b>, if we do not
- * already have it.  If <b>err_if_locked</b> is true, warn if somebody else is
- * holding the lock, and exit if we can't get it after waiting.  Otherwise,
- * return -1 if we can't get the lockfile.  Return 0 on success.
- */
-int
-try_locking(const or_options_t *options, int err_if_locked)
-{
-  if (lockfile)
-    return 0;
-  else {
-    char *fname = options_get_datadir_fname2_suffix(options, "lock",NULL,NULL);
-    int already_locked = 0;
-    tor_lockfile_t *lf = tor_lockfile_lock(fname, 0, &already_locked);
-    tor_free(fname);
-    if (!lf) {
-      if (err_if_locked && already_locked) {
-        int r;
-        log_warn(LD_GENERAL, "It looks like another Tor process is running "
-                 "with the same data directory.  Waiting 5 seconds to see "
-                 "if it goes away.");
-#ifndef _WIN32
-        sleep(5);
-#else
-        Sleep(5000);
-#endif
-        r = try_locking(options, 0);
-        if (r<0) {
-          log_err(LD_GENERAL, "No, it's still there.  Exiting.");
-          exit(0);
-        }
-        return r;
-      }
-      return -1;
-    }
-    lockfile = lf;
-    return 0;
-  }
-}
-
-/** Return true iff we've successfully acquired the lock file. */
-int
-have_lockfile(void)
+/** Reset Tor's uptime. */
+MOCK_IMPL(void,
+reset_uptime,(void))
 {
-  return lockfile != NULL;
+  stats_n_seconds_working = 0;
 }
 
-/** If we have successfully acquired the lock file, release it. */
 void
-release_lockfile(void)
+tor_mainloop_free_all(void)
 {
-  if (lockfile) {
-    tor_lockfile_unlock(lockfile);
-    lockfile = NULL;
-  }
-}
-
-/** Free all memory that we might have allocated somewhere.
- * If <b>postfork</b>, we are a worker process and we want to free
- * only the parts of memory that we won't touch. If !<b>postfork</b>,
- * Tor is shutting down and we should free everything.
- *
- * Helps us find the real leaks with dmalloc and the like. Also valgrind
- * should then report 0 reachable in its leak report (in an ideal world --
- * in practice libevent, SSL, libc etc never quite free everything). */
-void
-tor_free_all(int postfork)
-{
-  if (!postfork) {
-    evdns_shutdown(1);
-  }
-  geoip_free_all();
-  dirvote_free_all();
-  routerlist_free_all();
-  networkstatus_free_all();
-  addressmap_free_all();
-  dirserv_free_all();
-  rend_service_free_all();
-  rend_cache_free_all();
-  rend_service_authorization_free_all();
-  rep_hist_free_all();
-  dns_free_all();
-  clear_pending_onions();
-  circuit_free_all();
-  entry_guards_free_all();
-  pt_free_all();
-  channel_tls_free_all();
-  channel_free_all();
-  connection_free_all();
-  connection_edge_free_all();
-  scheduler_free_all();
-  nodelist_free_all();
-  microdesc_free_all();
-  routerparse_free_all();
-  ext_orport_free_all();
-  control_free_all();
-  sandbox_free_getaddrinfo_cache();
-  protover_free_all();
-  dos_free_all();
-  if (!postfork) {
-    config_free_all();
-    or_state_free_all();
-    router_free_all();
-    routerkeys_free_all();
-    policies_free_all();
-  }
-  if (!postfork) {
-    tor_tls_free_all();
-#ifndef _WIN32
-    tor_getpwnam(NULL);
-#endif
-  }
-  /* stuff in main.c */
-
   smartlist_free(connection_array);
   smartlist_free(closeable_connection_lst);
   smartlist_free(active_linked_connection_lst);
   periodic_timer_free(second_timer);
   teardown_periodic_events();
-  periodic_timer_free(refill_timer);
-
-  if (!postfork) {
-    release_lockfile();
-  }
-  /* Stuff in util.c and address.c*/
-  if (!postfork) {
-    escaped(NULL);
-    esc_router_info(NULL);
-    logs_free_all(); /* free log strings. do this last so logs keep working. */
-  }
-}
-
-/** Do whatever cleanup is necessary before shutting Tor down. */
-void
-tor_cleanup(void)
-{
-  const or_options_t *options = get_options();
-  if (options->command == CMD_RUN_TOR) {
-    time_t now = time(NULL);
-    /* Remove our pid file. We don't care if there was an error when we
-     * unlink, nothing we could do about it anyways. */
-    if (options->PidFile) {
-      if (unlink(options->PidFile) != 0) {
-        log_warn(LD_FS, "Couldn't unlink pid file %s: %s",
-                 options->PidFile, strerror(errno));
-      }
-    }
-    if (options->ControlPortWriteToFile) {
-      if (unlink(options->ControlPortWriteToFile) != 0) {
-        log_warn(LD_FS, "Couldn't unlink control port file %s: %s",
-                 options->ControlPortWriteToFile,
-                 strerror(errno));
-      }
-    }
-    if (accounting_is_enabled(options))
-      accounting_record_bandwidth_usage(now, get_or_state());
-    or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */
-    or_state_save(now);
-    if (authdir_mode(options)) {
-      sr_save_and_cleanup();
-    }
-    if (authdir_mode_tests_reachability(options))
-      rep_hist_record_mtbf_data(now, 0);
-    keypin_close_journal();
-  }
-#ifdef USE_DMALLOC
-  dmalloc_log_stats();
-#endif
-  tor_free_all(0); /* We could move tor_free_all back into the ifdef below
-                      later, if it makes shutdown unacceptably slow.  But for
-                      now, leave it here: it's helped us catch bugs in the
-                      past. */
-  crypto_global_cleanup();
-#ifdef USE_DMALLOC
-  dmalloc_log_unfreed();
-  dmalloc_shutdown();
-#endif
-}
-
-/** Read/create keys as needed, and echo our fingerprint to stdout. */
-static int
-do_list_fingerprint(void)
-{
-  char buf[FINGERPRINT_LEN+1];
-  crypto_pk_t *k;
-  const char *nickname = get_options()->Nickname;
-  sandbox_disable_getaddrinfo_cache();
-  if (!server_mode(get_options())) {
-    log_err(LD_GENERAL,
-            "Clients don't have long-term identity keys. Exiting.");
-    return -1;
-  }
-  tor_assert(nickname);
-  if (init_keys() < 0) {
-    log_err(LD_GENERAL,"Error initializing keys; exiting.");
-    return -1;
-  }
-  if (!(k = get_server_identity_key())) {
-    log_err(LD_GENERAL,"Error: missing identity key.");
-    return -1;
-  }
-  if (crypto_pk_get_fingerprint(k, buf, 1)<0) {
-    log_err(LD_BUG, "Error computing fingerprint");
-    return -1;
-  }
-  printf("%s %s\n", nickname, buf);
-  return 0;
-}
-
-/** Entry point for password hashing: take the desired password from
- * the command line, and print its salted hash to stdout. **/
-static void
-do_hash_password(void)
-{
-
-  char output[256];
-  char key[S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN];
-
-  crypto_rand(key, S2K_RFC2440_SPECIFIER_LEN-1);
-  key[S2K_RFC2440_SPECIFIER_LEN-1] = (uint8_t)96; /* Hash 64 K of data. */
-  secret_to_key_rfc2440(key+S2K_RFC2440_SPECIFIER_LEN, DIGEST_LEN,
-                get_options()->command_arg, strlen(get_options()->command_arg),
-                key);
-  base16_encode(output, sizeof(output), key, sizeof(key));
-  printf("16:%s\n",output);
-}
-
-/** Entry point for configuration dumping: write the configuration to
- * stdout. */
-static int
-do_dump_config(void)
-{
-  const or_options_t *options = get_options();
-  const char *arg = options->command_arg;
-  int how;
-  char *opts;
-
-  if (!strcmp(arg, "short")) {
-    how = OPTIONS_DUMP_MINIMAL;
-  } else if (!strcmp(arg, "non-builtin")) {
-    how = OPTIONS_DUMP_DEFAULTS;
-  } else if (!strcmp(arg, "full")) {
-    how = OPTIONS_DUMP_ALL;
-  } else {
-    fprintf(stderr, "No valid argument to --dump-config found!\n");
-    fprintf(stderr, "Please select 'short', 'non-builtin', or 'full'.\n");
-
-    return -1;
-  }
-
-  opts = options_dump(options, how);
-  printf("%s", opts);
-  tor_free(opts);
-
-  return 0;
-}
-
-static void
-init_addrinfo(void)
-{
-  if (! server_mode(get_options()) ||
-      (get_options()->Address && strlen(get_options()->Address) > 0)) {
-    /* We don't need to seed our own hostname, because we won't be calling
-     * resolve_my_address on it.
-     */
-    return;
-  }
-  char hname[256];
-
-  // host name to sandbox
-  gethostname(hname, sizeof(hname));
-  sandbox_add_addrinfo(hname);
-}
-
-static sandbox_cfg_t*
-sandbox_init_filter(void)
-{
-  const or_options_t *options = get_options();
-  sandbox_cfg_t *cfg = sandbox_cfg_new();
-  int i;
-
-  sandbox_cfg_allow_openat_filename(&cfg,
-      get_datadir_fname("cached-status"));
-
-#define OPEN(name)                              \
-  sandbox_cfg_allow_open_filename(&cfg, tor_strdup(name))
-
-#define OPEN_DATADIR(name)                      \
-  sandbox_cfg_allow_open_filename(&cfg, get_datadir_fname(name))
-
-#define OPEN_DATADIR2(name, name2)                       \
-  sandbox_cfg_allow_open_filename(&cfg, get_datadir_fname2((name), (name2)))
-
-#define OPEN_DATADIR_SUFFIX(name, suffix) do {  \
-    OPEN_DATADIR(name);                         \
-    OPEN_DATADIR(name suffix);                  \
-  } while (0)
-
-#define OPEN_DATADIR2_SUFFIX(name, name2, suffix) do {  \
-    OPEN_DATADIR2(name, name2);                         \
-    OPEN_DATADIR2(name, name2 suffix);                  \
-  } while (0)
-
-  OPEN(options->DataDirectory);
-  OPEN_DATADIR("keys");
-  OPEN_DATADIR_SUFFIX("cached-certs", ".tmp");
-  OPEN_DATADIR_SUFFIX("cached-consensus", ".tmp");
-  OPEN_DATADIR_SUFFIX("unverified-consensus", ".tmp");
-  OPEN_DATADIR_SUFFIX("unverified-microdesc-consensus", ".tmp");
-  OPEN_DATADIR_SUFFIX("cached-microdesc-consensus", ".tmp");
-  OPEN_DATADIR_SUFFIX("cached-microdescs", ".tmp");
-  OPEN_DATADIR_SUFFIX("cached-microdescs.new", ".tmp");
-  OPEN_DATADIR_SUFFIX("cached-descriptors", ".tmp");
-  OPEN_DATADIR_SUFFIX("cached-descriptors.new", ".tmp");
-  OPEN_DATADIR("cached-descriptors.tmp.tmp");
-  OPEN_DATADIR_SUFFIX("cached-extrainfo", ".tmp");
-  OPEN_DATADIR_SUFFIX("cached-extrainfo.new", ".tmp");
-  OPEN_DATADIR("cached-extrainfo.tmp.tmp");
-  OPEN_DATADIR_SUFFIX("state", ".tmp");
-  OPEN_DATADIR_SUFFIX("sr-state", ".tmp");
-  OPEN_DATADIR_SUFFIX("unparseable-desc", ".tmp");
-  OPEN_DATADIR_SUFFIX("v3-status-votes", ".tmp");
-  OPEN_DATADIR("key-pinning-journal");
-  OPEN("/dev/srandom");
-  OPEN("/dev/urandom");
-  OPEN("/dev/random");
-  OPEN("/etc/hosts");
-  OPEN("/proc/meminfo");
-
-  if (options->BridgeAuthoritativeDir)
-    OPEN_DATADIR_SUFFIX("networkstatus-bridges", ".tmp");
-
-  if (authdir_mode_handles_descs(options, -1))
-    OPEN_DATADIR("approved-routers");
-
-  if (options->ServerDNSResolvConfFile)
-    sandbox_cfg_allow_open_filename(&cfg,
-                                tor_strdup(options->ServerDNSResolvConfFile));
-  else
-    sandbox_cfg_allow_open_filename(&cfg, tor_strdup("/etc/resolv.conf"));
-
-  for (i = 0; i < 2; ++i) {
-    if (get_torrc_fname(i)) {
-      sandbox_cfg_allow_open_filename(&cfg, tor_strdup(get_torrc_fname(i)));
-    }
-  }
-
-#define RENAME_SUFFIX(name, suffix)        \
-  sandbox_cfg_allow_rename(&cfg,           \
-      get_datadir_fname(name suffix),      \
-      get_datadir_fname(name))
-
-#define RENAME_SUFFIX2(prefix, name, suffix) \
-  sandbox_cfg_allow_rename(&cfg,                                        \
-                           get_datadir_fname2(prefix, name suffix),     \
-                           get_datadir_fname2(prefix, name))
-
-  RENAME_SUFFIX("cached-certs", ".tmp");
-  RENAME_SUFFIX("cached-consensus", ".tmp");
-  RENAME_SUFFIX("unverified-consensus", ".tmp");
-  RENAME_SUFFIX("unverified-microdesc-consensus", ".tmp");
-  RENAME_SUFFIX("cached-microdesc-consensus", ".tmp");
-  RENAME_SUFFIX("cached-microdescs", ".tmp");
-  RENAME_SUFFIX("cached-microdescs", ".new");
-  RENAME_SUFFIX("cached-microdescs.new", ".tmp");
-  RENAME_SUFFIX("cached-descriptors", ".tmp");
-  RENAME_SUFFIX("cached-descriptors", ".new");
-  RENAME_SUFFIX("cached-descriptors.new", ".tmp");
-  RENAME_SUFFIX("cached-extrainfo", ".tmp");
-  RENAME_SUFFIX("cached-extrainfo", ".new");
-  RENAME_SUFFIX("cached-extrainfo.new", ".tmp");
-  RENAME_SUFFIX("state", ".tmp");
-  RENAME_SUFFIX("sr-state", ".tmp");
-  RENAME_SUFFIX("unparseable-desc", ".tmp");
-  RENAME_SUFFIX("v3-status-votes", ".tmp");
-
-  if (options->BridgeAuthoritativeDir)
-    RENAME_SUFFIX("networkstatus-bridges", ".tmp");
-
-#define STAT_DATADIR(name)                      \
-  sandbox_cfg_allow_stat_filename(&cfg, get_datadir_fname(name))
-
-#define STAT_DATADIR2(name, name2)                                      \
-  sandbox_cfg_allow_stat_filename(&cfg, get_datadir_fname2((name), (name2)))
-
-  STAT_DATADIR(NULL);
-  STAT_DATADIR("lock");
-  STAT_DATADIR("state");
-  STAT_DATADIR("router-stability");
-  STAT_DATADIR("cached-extrainfo.new");
-
-  {
-    smartlist_t *files = smartlist_new();
-    tor_log_get_logfile_names(files);
-    SMARTLIST_FOREACH(files, char *, file_name, {
-      /* steals reference */
-      sandbox_cfg_allow_open_filename(&cfg, file_name);
-    });
-    smartlist_free(files);
-  }
-
-  {
-    smartlist_t *files = smartlist_new();
-    smartlist_t *dirs = smartlist_new();
-    rend_services_add_filenames_to_lists(files, dirs);
-    SMARTLIST_FOREACH(files, char *, file_name, {
-      char *tmp_name = NULL;
-      tor_asprintf(&tmp_name, "%s.tmp", file_name);
-      sandbox_cfg_allow_rename(&cfg,
-                               tor_strdup(tmp_name), tor_strdup(file_name));
-      /* steals references */
-      sandbox_cfg_allow_open_filename(&cfg, file_name);
-      sandbox_cfg_allow_open_filename(&cfg, tmp_name);
-    });
-    SMARTLIST_FOREACH(dirs, char *, dir, {
-      /* steals reference */
-      sandbox_cfg_allow_stat_filename(&cfg, dir);
-    });
-    smartlist_free(files);
-    smartlist_free(dirs);
-  }
-
-  {
-    char *fname;
-    if ((fname = get_controller_cookie_file_name())) {
-      sandbox_cfg_allow_open_filename(&cfg, fname);
-    }
-    if ((fname = get_ext_or_auth_cookie_file_name())) {
-      sandbox_cfg_allow_open_filename(&cfg, fname);
-    }
-  }
-
-  SMARTLIST_FOREACH_BEGIN(get_configured_ports(), port_cfg_t *, port) {
-    if (!port->is_unix_addr)
-      continue;
-    /* When we open an AF_UNIX address, we want permission to open the
-     * directory that holds it. */
-    char *dirname = tor_strdup(port->unix_addr);
-    if (get_parent_directory(dirname) == 0) {
-      OPEN(dirname);
-    }
-    tor_free(dirname);
-    sandbox_cfg_allow_chmod_filename(&cfg, tor_strdup(port->unix_addr));
-    sandbox_cfg_allow_chown_filename(&cfg, tor_strdup(port->unix_addr));
-  } SMARTLIST_FOREACH_END(port);
-
-  if (options->DirPortFrontPage) {
-    sandbox_cfg_allow_open_filename(&cfg,
-                                    tor_strdup(options->DirPortFrontPage));
-  }
-
-  // orport
-  if (server_mode(get_options())) {
-
-    OPEN_DATADIR2_SUFFIX("keys", "secret_id_key", ".tmp");
-    OPEN_DATADIR2_SUFFIX("keys", "secret_onion_key", ".tmp");
-    OPEN_DATADIR2_SUFFIX("keys", "secret_onion_key_ntor", ".tmp");
-    OPEN_DATADIR2("keys", "secret_id_key.old");
-    OPEN_DATADIR2("keys", "secret_onion_key.old");
-    OPEN_DATADIR2("keys", "secret_onion_key_ntor.old");
-
-    OPEN_DATADIR2_SUFFIX("keys", "ed25519_master_id_secret_key", ".tmp");
-    OPEN_DATADIR2_SUFFIX("keys", "ed25519_master_id_secret_key_encrypted",
-                         ".tmp");
-    OPEN_DATADIR2_SUFFIX("keys", "ed25519_master_id_public_key", ".tmp");
-    OPEN_DATADIR2_SUFFIX("keys", "ed25519_signing_secret_key", ".tmp");
-    OPEN_DATADIR2_SUFFIX("keys", "ed25519_signing_secret_key_encrypted",
-                         ".tmp");
-    OPEN_DATADIR2_SUFFIX("keys", "ed25519_signing_public_key", ".tmp");
-    OPEN_DATADIR2_SUFFIX("keys", "ed25519_signing_cert", ".tmp");
-
-    OPEN_DATADIR2_SUFFIX("stats", "bridge-stats", ".tmp");
-    OPEN_DATADIR2_SUFFIX("stats", "dirreq-stats", ".tmp");
-
-    OPEN_DATADIR2_SUFFIX("stats", "entry-stats", ".tmp");
-    OPEN_DATADIR2_SUFFIX("stats", "exit-stats", ".tmp");
-    OPEN_DATADIR2_SUFFIX("stats", "buffer-stats", ".tmp");
-    OPEN_DATADIR2_SUFFIX("stats", "conn-stats", ".tmp");
-    OPEN_DATADIR2_SUFFIX("stats", "hidserv-stats", ".tmp");
-
-    OPEN_DATADIR("approved-routers");
-    OPEN_DATADIR_SUFFIX("fingerprint", ".tmp");
-    OPEN_DATADIR_SUFFIX("hashed-fingerprint", ".tmp");
-    OPEN_DATADIR_SUFFIX("router-stability", ".tmp");
-
-    OPEN("/etc/resolv.conf");
-
-    RENAME_SUFFIX("fingerprint", ".tmp");
-    RENAME_SUFFIX2("keys", "secret_onion_key_ntor", ".tmp");
-    RENAME_SUFFIX2("keys", "secret_id_key", ".tmp");
-    RENAME_SUFFIX2("keys", "secret_id_key.old", ".tmp");
-    RENAME_SUFFIX2("keys", "secret_onion_key", ".tmp");
-    RENAME_SUFFIX2("keys", "secret_onion_key.old", ".tmp");
-    RENAME_SUFFIX2("stats", "bridge-stats", ".tmp");
-    RENAME_SUFFIX2("stats", "dirreq-stats", ".tmp");
-    RENAME_SUFFIX2("stats", "entry-stats", ".tmp");
-    RENAME_SUFFIX2("stats", "exit-stats", ".tmp");
-    RENAME_SUFFIX2("stats", "buffer-stats", ".tmp");
-    RENAME_SUFFIX2("stats", "conn-stats", ".tmp");
-    RENAME_SUFFIX2("stats", "hidserv-stats", ".tmp");
-    RENAME_SUFFIX("hashed-fingerprint", ".tmp");
-    RENAME_SUFFIX("router-stability", ".tmp");
-
-    RENAME_SUFFIX2("keys", "ed25519_master_id_secret_key", ".tmp");
-    RENAME_SUFFIX2("keys", "ed25519_master_id_secret_key_encrypted", ".tmp");
-    RENAME_SUFFIX2("keys", "ed25519_master_id_public_key", ".tmp");
-    RENAME_SUFFIX2("keys", "ed25519_signing_secret_key", ".tmp");
-    RENAME_SUFFIX2("keys", "ed25519_signing_cert", ".tmp");
-
-    sandbox_cfg_allow_rename(&cfg,
-             get_datadir_fname2("keys", "secret_onion_key"),
-             get_datadir_fname2("keys", "secret_onion_key.old"));
-    sandbox_cfg_allow_rename(&cfg,
-             get_datadir_fname2("keys", "secret_onion_key_ntor"),
-             get_datadir_fname2("keys", "secret_onion_key_ntor.old"));
-
-    STAT_DATADIR("keys");
-    OPEN_DATADIR("stats");
-    STAT_DATADIR("stats");
-    STAT_DATADIR2("stats", "dirreq-stats");
-  }
-
-  init_addrinfo();
-
-  return cfg;
-}
-
-/** Main entry point for the Tor process.  Called from main(). */
-/* This function is distinct from main() only so we can link main.c into
- * the unittest binary without conflicting with the unittests' main. */
-int
-tor_main(int argc, char *argv[])
-{
-  int result = 0;
-
-#ifdef _WIN32
-#ifndef HeapEnableTerminationOnCorruption
-#define HeapEnableTerminationOnCorruption 1
-#endif
-  /* On heap corruption, just give up; don't try to play along. */
-  HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
-
-  /* SetProcessDEPPolicy is only supported on 32-bit Windows.
-   * (On 64-bit Windows it always fails, and some compilers don't like the
-   * PSETDEP cast.)
-   * 32-bit Windows defines _WIN32.
-   * 64-bit Windows defines _WIN32 and _WIN64. */
-#ifndef _WIN64
-  /* Call SetProcessDEPPolicy to permanently enable DEP.
-     The function will not resolve on earlier versions of Windows,
-     and failure is not dangerous. */
-  HMODULE hMod = GetModuleHandleA("Kernel32.dll");
-  if (hMod) {
-    typedef BOOL (WINAPI *PSETDEP)(DWORD);
-    PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod,
-                           "SetProcessDEPPolicy");
-    if (setdeppolicy) {
-      /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */
-      setdeppolicy(3);
-    }
-  }
-#endif /* !defined(_WIN64) */
-#endif
+  tor_event_free(shutdown_did_not_work_event);
+  tor_event_free(initialize_periodic_events_event);
+  mainloop_event_free(directory_all_unreachable_cb_event);
+  mainloop_event_free(schedule_active_linked_connections_event);
+  mainloop_event_free(postloop_cleanup_ev);
+  mainloop_event_free(handle_deferred_signewnym_ev);
 
-  configure_backtrace_handler(get_version());
-
-  update_approx_time(time(NULL));
-  tor_threads_init();
-  init_logging(0);
-#ifdef USE_DMALLOC
-  {
-    /* Instruct OpenSSL to use our internal wrappers for malloc,
-       realloc and free. */
-    int r = CRYPTO_set_mem_ex_functions(tor_malloc_, tor_realloc_, tor_free_);
-    tor_assert(r);
-  }
-#endif
-#ifdef NT_SERVICE
-  {
-     int done = 0;
-     result = nt_service_parse_options(argc, argv, &done);
-     if (done) return result;
-  }
-#endif
-  if (tor_init(argc, argv)<0)
-    return -1;
-
-  if (get_options()->Sandbox && get_options()->command == CMD_RUN_TOR) {
-    sandbox_cfg_t* cfg = sandbox_init_filter();
-
-    if (sandbox_init(cfg)) {
-      log_err(LD_BUG,"Failed to create syscall sandbox filter");
-      return -1;
-    }
-
-    // registering libevent rng
-#ifdef HAVE_EVUTIL_SECURE_RNG_SET_URANDOM_DEVICE_FILE
-    evutil_secure_rng_set_urandom_device_file(
-        (char*) sandbox_intern_string("/dev/urandom"));
+#ifdef HAVE_SYSTEMD_209
+  periodic_timer_free(systemd_watchdog_timer);
 #endif
-  }
 
-  monotime_init();
+  stats_n_bytes_read = stats_n_bytes_written = 0;
 
-  switch (get_options()->command) {
-  case CMD_RUN_TOR:
-#ifdef NT_SERVICE
-    nt_service_set_state(SERVICE_RUNNING);
-#endif
-    result = do_main_loop();
-    break;
-  case CMD_KEYGEN:
-    result = load_ed_keys(get_options(), time(NULL));
-    break;
-  case CMD_LIST_FINGERPRINT:
-    result = do_list_fingerprint();
-    break;
-  case CMD_HASH_PASSWORD:
-    do_hash_password();
-    result = 0;
-    break;
-  case CMD_VERIFY_CONFIG:
-    if (quiet_level == 0)
-      printf("Configuration was valid\n");
-    result = 0;
-    break;
-  case CMD_DUMP_CONFIG:
-    result = do_dump_config();
-    break;
-  case CMD_RUN_UNITTESTS: /* only set by test.c */
-  default:
-    log_warn(LD_BUG,"Illegal command number %d: internal error.",
-             get_options()->command);
-    result = -1;
-  }
-  tor_cleanup();
-  return result;
+  memset(&global_bucket, 0, sizeof(global_bucket));
+  memset(&global_relayed_bucket, 0, sizeof(global_relayed_bucket));
+  time_of_process_start = 0;
+  time_of_last_signewnym = 0;
+  signewnym_is_pending = 0;
+  newnym_epoch = 0;
+  called_loop_once = 0;
+  main_loop_should_exit = 0;
+  main_loop_exit_value = 0;
+  can_complete_circuits = 0;
+  quiet_level = 0;
+  should_init_bridge_stats = 1;
+  dns_honesty_first_time = 1;
+  heartbeat_callback_first_time = 1;
+  current_second = 0;
+  memset(&current_second_last_changed, 0,
+         sizeof(current_second_last_changed));
 }
-
diff --git a/src/or/main.h b/src/core/mainloop/mainloop.h
index 07b22598b1..c5669fc4e0 100644
--- a/src/or/main.h
+++ b/src/core/mainloop/mainloop.h
@@ -1,16 +1,16 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file main.h
- * \brief Header file for main.c.
+ * \file mainloop.h
+ * \brief Header file for mainloop.c.
  **/
 
-#ifndef TOR_MAIN_H
-#define TOR_MAIN_H
+#ifndef TOR_MAINLOOP_H
+#define TOR_MAINLOOP_H
 
 int have_completed_a_circuit(void);
 void note_that_we_completed_a_circuit(void);
@@ -28,6 +28,7 @@ int connection_is_on_closeable_list(connection_t *conn);
 MOCK_DECL(smartlist_t *, get_connection_array, (void));
 MOCK_DECL(uint64_t,get_bytes_read,(void));
 MOCK_DECL(uint64_t,get_bytes_written,(void));
+void stats_increment_bytes_read_and_written(uint64_t r, uint64_t w);
 
 /** Bitmask for events that we can turn on and off with
  * connection_watch_events. */
@@ -45,7 +46,8 @@ int connection_is_writing(connection_t *conn);
 MOCK_DECL(void,connection_stop_writing,(connection_t *conn));
 MOCK_DECL(void,connection_start_writing,(connection_t *conn));
 
-void tell_event_loop_to_finish(void);
+void tor_shutdown_event_loop_and_exit(int exitcode);
+int tor_event_loop_shutdown_is_pending(void);
 
 void connection_stop_reading_from_linked_conn(connection_t *conn);
 
@@ -59,40 +61,54 @@ void dns_servers_relaunch_checks(void);
 void reset_all_main_loop_timers(void);
 void reschedule_descriptor_update_check(void);
 void reschedule_directory_downloads(void);
+void reschedule_or_state_save(void);
+void reschedule_dirvote(const or_options_t *options);
+void mainloop_schedule_postloop_cleanup(void);
+void rescan_periodic_events(const or_options_t *options);
+
+void update_current_time(time_t now);
 
 MOCK_DECL(long,get_uptime,(void));
+MOCK_DECL(void,reset_uptime,(void));
 
 unsigned get_signewnym_epoch(void);
 
-void handle_signals(int is_parent);
-void activate_signal(int signal_num);
+int do_main_loop(void);
 
-int try_locking(const or_options_t *options, int err_if_locked);
-int have_lockfile(void);
-void release_lockfile(void);
+void reset_main_loop_counters(void);
+uint64_t get_main_loop_success_count(void);
+uint64_t get_main_loop_error_count(void);
+uint64_t get_main_loop_idle_count(void);
 
-void tor_cleanup(void);
-void tor_free_all(int postfork);
+void periodic_events_on_new_options(const or_options_t *options);
+void reschedule_per_second_timer(void);
 
-int tor_main(int argc, char *argv[]);
+void do_signewnym(time_t);
+time_t get_last_signewnym_time(void);
 
-int do_main_loop(void);
-int tor_init(int argc, char **argv);
+void tor_init_connection_lists(void);
+void initialize_mainloop_events(void);
+void tor_mainloop_free_all(void);
+
+struct token_bucket_rw_t;
 
 extern time_t time_of_process_start;
-extern long stats_n_seconds_working;
 extern int quiet_level;
-extern int global_read_bucket;
-extern int global_write_bucket;
-extern int global_relayed_read_bucket;
-extern int global_relayed_write_bucket;
+extern struct token_bucket_rw_t global_bucket;
+extern struct token_bucket_rw_t global_relayed_bucket;
 
-#ifdef MAIN_PRIVATE
-STATIC void init_connection_lists(void);
+#ifdef MAINLOOP_PRIVATE
 STATIC void close_closeable_connections(void);
 STATIC void initialize_periodic_events(void);
 STATIC void teardown_periodic_events(void);
-#endif
+STATIC int get_my_roles(const or_options_t *);
+#ifdef TOR_UNIT_TESTS
+extern smartlist_t *connection_array;
 
+/* We need the periodic_event_item_t definition. */
+#include "core/mainloop/periodic.h"
+extern periodic_event_item_t periodic_events[];
 #endif
+#endif /* defined(MAIN_PRIVATE) */
 
+#endif
diff --git a/src/core/mainloop/netstatus.c b/src/core/mainloop/netstatus.c
new file mode 100644
index 0000000000..1444ca5db2
--- /dev/null
+++ b/src/core/mainloop/netstatus.c
@@ -0,0 +1,28 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "core/mainloop/netstatus.h"
+#include "app/config/config.h"
+#include "feature/hibernate/hibernate.h"
+
+/** Return true iff our network is in some sense disabled or shutting down:
+ * either we're hibernating, entering hibernation, or the network is turned
+ * off with DisableNetwork. */
+int
+net_is_disabled(void)
+{
+  return get_options()->DisableNetwork || we_are_hibernating();
+}
+
+/** Return true iff our network is in some sense "completely disabled" either
+ * we're fully hibernating or the network is turned off with
+ * DisableNetwork. */
+int
+net_is_completely_disabled(void)
+{
+  return get_options()->DisableNetwork || we_are_fully_hibernating();
+}
diff --git a/src/core/mainloop/netstatus.h b/src/core/mainloop/netstatus.h
new file mode 100644
index 0000000000..ae8547b30d
--- /dev/null
+++ b/src/core/mainloop/netstatus.h
@@ -0,0 +1,13 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_NETSTATUS_H
+#define TOR_NETSTATUS_H
+
+int net_is_disabled(void);
+int net_is_completely_disabled(void);
+
+#endif
diff --git a/src/or/periodic.c b/src/core/mainloop/periodic.c
index d02d4a7bbb..2651bbbc89 100644
--- a/src/or/periodic.c
+++ b/src/core/mainloop/periodic.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
+/* Copyright (c) 2015-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,12 +11,12 @@
  * that they fire.  See periodic_events[] in main.c for examples.
  */
 
-#include "or.h"
-#include "compat_libevent.h"
-#include "config.h"
-#include "periodic.h"
-
-#include <event2/event.h>
+#include "core/or/or.h"
+#include "lib/evloop/compat_libevent.h"
+#include "app/config/config.h"
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/periodic.h"
+#include "lib/evloop/compat_libevent.h"
 
 /** We disable any interval greater than this number of seconds, on the
  * grounds that it is probably an absolute time mistakenly passed in as a
@@ -34,24 +34,34 @@ periodic_event_set_interval(periodic_event_item_t *event,
   struct timeval tv;
   tv.tv_sec = next_interval;
   tv.tv_usec = 0;
-  event_add(event->ev, &tv);
+  mainloop_event_schedule(event->ev, &tv);
 }
 
 /** Wraps dispatches for periodic events, <b>data</b> will be a pointer to the
  * event that needs to be called */
 static void
-periodic_event_dispatch(evutil_socket_t fd, short what, void *data)
+periodic_event_dispatch(mainloop_event_t *ev, void *data)
 {
-  (void)fd;
-  (void)what;
   periodic_event_item_t *event = data;
+  tor_assert(ev == event->ev);
+
+  if (BUG(!periodic_event_is_enabled(event))) {
+    return;
+  }
 
   time_t now = time(NULL);
+  update_current_time(now);
   const or_options_t *options = get_options();
 //  log_debug(LD_GENERAL, "Dispatching %s", event->name);
   int r = event->fn(now, options);
   int next_interval = 0;
 
+  if (!periodic_event_is_enabled(event)) {
+    /* The event got disabled from inside its callback; no need to
+     * reschedule. */
+    return;
+  }
+
   /* update the last run time if action was taken */
   if (r==0) {
     log_err(LD_BUG, "Invalid return value for periodic event from %s.",
@@ -74,14 +84,17 @@ periodic_event_dispatch(evutil_socket_t fd, short what, void *data)
 //  log_debug(LD_GENERAL, "Scheduling %s for %d seconds", event->name,
 //           next_interval);
   struct timeval tv = { next_interval , 0 };
-  event_add(event->ev, &tv);
+  mainloop_event_schedule(ev, &tv);
 }
 
 /** Schedules <b>event</b> to run as soon as possible from now. */
 void
 periodic_event_reschedule(periodic_event_item_t *event)
 {
-  periodic_event_set_interval(event, 1);
+  /* Don't reschedule a disabled event. */
+  if (periodic_event_is_enabled(event)) {
+    periodic_event_set_interval(event, 1);
+  }
 }
 
 /** Initializes the libevent backend for a periodic event. */
@@ -93,10 +106,8 @@ periodic_event_setup(periodic_event_item_t *event)
     tor_assert(0);
   }
 
-  event->ev = tor_event_new(tor_libevent_get_base(),
-                            -1, 0,
-                            periodic_event_dispatch,
-                            event);
+  event->ev = mainloop_event_new(periodic_event_dispatch,
+                                 event);
   tor_assert(event->ev);
 }
 
@@ -109,9 +120,15 @@ periodic_event_launch(periodic_event_item_t *event)
     log_err(LD_BUG, "periodic_event_launch without periodic_event_setup");
     tor_assert(0);
   }
+  /* Event already enabled? This is a bug */
+  if (periodic_event_is_enabled(event)) {
+    log_err(LD_BUG, "periodic_event_launch on an already enabled event");
+    tor_assert(0);
+  }
 
   // Initial dispatch
-  periodic_event_dispatch(-1, EV_TIMEOUT, event);
+  event->enabled = 1;
+  periodic_event_dispatch(event->ev, event);
 }
 
 /** Release all storage associated with <b>event</b> */
@@ -120,7 +137,43 @@ periodic_event_destroy(periodic_event_item_t *event)
 {
   if (!event)
     return;
-  tor_event_free(event->ev);
+
+  /* First disable the event so we first cancel the event and set its enabled
+   * flag properly. */
+  periodic_event_disable(event);
+
+  mainloop_event_free(event->ev);
   event->last_action_time = 0;
 }
 
+/** Enable the given event by setting its "enabled" flag and scheduling it to
+ * run immediately in the event loop. This can be called for an event that is
+ * already enabled. */
+void
+periodic_event_enable(periodic_event_item_t *event)
+{
+  tor_assert(event);
+  /* Safely and silently ignore if this event is already enabled. */
+  if (periodic_event_is_enabled(event)) {
+    return;
+  }
+
+  tor_assert(event->ev);
+  event->enabled = 1;
+  mainloop_event_activate(event->ev);
+}
+
+/** Disable the given event which means the event is destroyed and then the
+ * event's enabled flag is unset. This can be called for an event that is
+ * already disabled. */
+void
+periodic_event_disable(periodic_event_item_t *event)
+{
+  tor_assert(event);
+  /* Safely and silently ignore if this event is already disabled. */
+  if (!periodic_event_is_enabled(event)) {
+    return;
+  }
+  mainloop_event_cancel(event->ev);
+  event->enabled = 0;
+}
diff --git a/src/core/mainloop/periodic.h b/src/core/mainloop/periodic.h
new file mode 100644
index 0000000000..e49fafd174
--- /dev/null
+++ b/src/core/mainloop/periodic.h
@@ -0,0 +1,88 @@
+/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PERIODIC_H
+#define TOR_PERIODIC_H
+
+#define PERIODIC_EVENT_NO_UPDATE (-1)
+
+/* Tor roles for which a periodic event item is for. An event can be for
+ * multiple roles, they can be combined. */
+#define PERIODIC_EVENT_ROLE_CLIENT      (1U << 0)
+#define PERIODIC_EVENT_ROLE_RELAY       (1U << 1)
+#define PERIODIC_EVENT_ROLE_BRIDGE      (1U << 2)
+#define PERIODIC_EVENT_ROLE_DIRAUTH     (1U << 3)
+#define PERIODIC_EVENT_ROLE_BRIDGEAUTH  (1U << 4)
+#define PERIODIC_EVENT_ROLE_HS_SERVICE  (1U << 5)
+#define PERIODIC_EVENT_ROLE_DIRSERVER   (1U << 6)
+
+/* Helper macro to make it a bit less annoying to defined groups of roles that
+ * are often used. */
+
+/* Router that is a Bridge or Relay. */
+#define PERIODIC_EVENT_ROLE_ROUTER \
+  (PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_RELAY)
+/* Authorities that is both bridge and directory. */
+#define PERIODIC_EVENT_ROLE_AUTHORITIES \
+  (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_DIRAUTH)
+/* All roles. */
+#define PERIODIC_EVENT_ROLE_ALL \
+  (PERIODIC_EVENT_ROLE_AUTHORITIES | PERIODIC_EVENT_ROLE_CLIENT | \
+   PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_ROUTER)
+
+/*
+ * Event flags which can change the behavior of an event.
+ */
+
+/* Indicate that the event needs the network meaning that if we are in
+ * DisableNetwork or hibernation mode, the event won't be enabled. This obey
+ * the net_is_disabled() check. */
+#define PERIODIC_EVENT_FLAG_NEED_NET  (1U << 0)
+
+/** Callback function for a periodic event to take action.  The return value
+* influences the next time the function will get called.  Return
+* PERIODIC_EVENT_NO_UPDATE to not update <b>last_action_time</b> and be polled
+* again in the next second. If a positive value is returned it will update the
+* interval time. */
+typedef int (*periodic_event_helper_t)(time_t now,
+                                      const or_options_t *options);
+
+struct mainloop_event_t;
+
+/** A single item for the periodic-events-function table. */
+typedef struct periodic_event_item_t {
+  periodic_event_helper_t fn; /**< The function to run the event */
+  time_t last_action_time; /**< The last time the function did something */
+  struct mainloop_event_t *ev; /**< Libevent callback we're using to implement
+                                * this */
+  const char *name; /**< Name of the function -- for debug */
+
+  /* Bitmask of roles define above for which this event applies. */
+  uint32_t roles;
+  /* Bitmask of flags which can change the behavior of the event. */
+  uint32_t flags;
+  /* Indicate that this event has been enabled that is scheduled. */
+  unsigned int enabled : 1;
+} periodic_event_item_t;
+
+/** events will get their interval from first execution */
+#define PERIODIC_EVENT(fn, r, f) { fn##_callback, 0, NULL, #fn, r, f, 0 }
+#define END_OF_PERIODIC_EVENTS { NULL, 0, NULL, NULL, 0, 0, 0 }
+
+/* Return true iff the given event was setup before thus is enabled to be
+ * scheduled. */
+static inline int
+periodic_event_is_enabled(const periodic_event_item_t *item)
+{
+  return item->enabled;
+}
+
+void periodic_event_launch(periodic_event_item_t *event);
+void periodic_event_setup(periodic_event_item_t *event);
+void periodic_event_destroy(periodic_event_item_t *event);
+void periodic_event_reschedule(periodic_event_item_t *event);
+void periodic_event_enable(periodic_event_item_t *event);
+void periodic_event_disable(periodic_event_item_t *event);
+
+#endif /* !defined(TOR_PERIODIC_H) */
+
diff --git a/src/core/or/addr_policy_st.h b/src/core/or/addr_policy_st.h
new file mode 100644
index 0000000000..a75f1a731d
--- /dev/null
+++ b/src/core/or/addr_policy_st.h
@@ -0,0 +1,46 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_ADDR_POLICY_ST_H
+#define TOR_ADDR_POLICY_ST_H
+
+#include "lib/cc/torint.h"
+#include "lib/net/address.h"
+
+/** What action type does an address policy indicate: accept or reject? */
+typedef enum {
+  ADDR_POLICY_ACCEPT=1,
+  ADDR_POLICY_REJECT=2,
+} addr_policy_action_t;
+#define addr_policy_action_bitfield_t ENUM_BF(addr_policy_action_t)
+
+/** A reference-counted address policy rule. */
+struct addr_policy_t {
+  int refcnt; /**< Reference count */
+  /** What to do when the policy matches.*/
+  addr_policy_action_bitfield_t policy_type:2;
+  unsigned int is_private:1; /**< True iff this is the pseudo-address,
+                              * "private". */
+  unsigned int is_canonical:1; /**< True iff this policy is the canonical
+                                * copy (stored in a hash table to avoid
+                                * duplication of common policies) */
+  maskbits_t maskbits; /**< Accept/reject all addresses <b>a</b> such that the
+                 * first <b>maskbits</b> bits of <b>a</b> match
+                 * <b>addr</b>. */
+  /** Base address to accept or reject.
+   *
+   * Note that wildcards are treated
+   * differntly depending on address family. An AF_UNSPEC address means
+   * "All addresses, IPv4 or IPv6." An AF_INET address with maskbits==0 means
+   * "All IPv4 addresses" and an AF_INET6 address with maskbits == 0 means
+   * "All IPv6 addresses".
+  **/
+  tor_addr_t addr;
+  uint16_t prt_min; /**< Lowest port number to accept/reject. */
+  uint16_t prt_max; /**< Highest port number to accept/reject. */
+};
+
+#endif
diff --git a/src/core/or/address_set.c b/src/core/or/address_set.c
new file mode 100644
index 0000000000..758fba4aac
--- /dev/null
+++ b/src/core/or/address_set.c
@@ -0,0 +1,71 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file address_set.c
+ * \brief Implementation for a set of addresses.
+ *
+ * This module was first written on a semi-emergency basis to improve the
+ * robustness of the anti-DoS module.  As such, it's written in a pretty
+ * conservative way, and should be susceptible to improvement later on.
+ **/
+
+#include "orconfig.h"
+#include "core/or/address_set.h"
+#include "lib/net/address.h"
+#include "lib/container/bloomfilt.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "siphash.h"
+
+/* Wrap our hash function to have the signature that the bloom filter
+ * needs. */
+static uint64_t
+bloomfilt_addr_hash(const struct sipkey *key,
+                    const void *item)
+{
+  return tor_addr_keyed_hash(key, item);
+}
+
+/**
+ * Allocate and return an address_set, suitable for holding up to
+ * <b>max_address_guess</b> distinct values.
+ */
+address_set_t *
+address_set_new(int max_addresses_guess)
+{
+  uint8_t k[BLOOMFILT_KEY_LEN];
+  crypto_rand((void*)k, sizeof(k));
+  return bloomfilt_new(max_addresses_guess, bloomfilt_addr_hash, k);
+}
+
+/**
+ * Add <b>addr</b> to <b>set</b>.
+ *
+ * All future queries for <b>addr</b> in set will return true. Removing
+ * items is not possible.
+ */
+void
+address_set_add(address_set_t *set, const struct tor_addr_t *addr)
+{
+  bloomfilt_add(set, addr);
+}
+
+/** As address_set_add(), but take an ipv4 address in host order. */
+void
+address_set_add_ipv4h(address_set_t *set, uint32_t addr)
+{
+  tor_addr_t a;
+  tor_addr_from_ipv4h(&a, addr);
+  address_set_add(set, &a);
+}
+
+/**
+ * Return true if <b>addr</b> is a member of <b>set</b>.  (And probably,
+ * return false if <b>addr</b> is not a member of set.)
+ */
+int
+address_set_probably_contains(const address_set_t *set,
+                              const struct tor_addr_t *addr)
+{
+  return bloomfilt_probably_contains(set, addr);
+}
diff --git a/src/common/address_set.h b/src/core/or/address_set.h
index aedf17fc66..7a9e71628e 100644
--- a/src/common/address_set.h
+++ b/src/core/or/address_set.h
@@ -1,35 +1,31 @@
-/* Copyright (c) 2018, The Tor Project, Inc. */
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file addressset.h
+ * \file address_set.h
  * \brief Types to handle sets of addresses.
- *
- * This module was first written on a semi-emergency basis to improve the
- * robustness of the anti-DoS module.  As such, it's written in a pretty
- * conservative way, and should be susceptible to improvement later on.
  **/
 
 #ifndef TOR_ADDRESS_SET_H
 #define TOR_ADDRESS_SET_H
 
 #include "orconfig.h"
-#include "torint.h"
+#include "lib/cc/torint.h"
+#include "lib/container/bloomfilt.h"
 
 /**
  * An address_set_t represents a set of tor_addr_t values. The implementation
  * is probabilistic: false negatives cannot occur but false positives are
  * possible.
  */
-typedef struct address_set_t address_set_t;
+typedef struct bloomfilt_t address_set_t;
 struct tor_addr_t;
 
 address_set_t *address_set_new(int max_addresses_guess);
-void address_set_free(address_set_t *set);
+#define address_set_free(set) bloomfilt_free(set)
 void address_set_add(address_set_t *set, const struct tor_addr_t *addr);
 void address_set_add_ipv4h(address_set_t *set, uint32_t addr);
-int address_set_probably_contains(address_set_t *set,
+int address_set_probably_contains(const address_set_t *set,
                                   const struct tor_addr_t *addr);
 
 #endif
-
diff --git a/src/core/or/cell_queue_st.h b/src/core/or/cell_queue_st.h
new file mode 100644
index 0000000000..130b95a011
--- /dev/null
+++ b/src/core/or/cell_queue_st.h
@@ -0,0 +1,29 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef PACKED_CELL_ST_H
+#define PACKED_CELL_ST_H
+
+#include "tor_queue.h"
+
+/** A cell as packed for writing to the network. */
+struct packed_cell_t {
+  /** Next cell queued on this circuit. */
+  TOR_SIMPLEQ_ENTRY(packed_cell_t) next;
+  char body[CELL_MAX_NETWORK_SIZE]; /**< Cell as packed for network. */
+  uint32_t inserted_timestamp; /**< Time (in timestamp units) when this cell
+                                * was inserted */
+};
+
+/** A queue of cells on a circuit, waiting to be added to the
+ * or_connection_t's outbuf. */
+struct cell_queue_t {
+  /** Linked list of packed_cell_t*/
+  TOR_SIMPLEQ_HEAD(cell_simpleq, packed_cell_t) head;
+  int n; /**< The number of cells in the queue. */
+};
+
+#endif
diff --git a/src/core/or/cell_st.h b/src/core/or/cell_st.h
new file mode 100644
index 0000000000..7ab7eceb50
--- /dev/null
+++ b/src/core/or/cell_st.h
@@ -0,0 +1,20 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef CELL_ST_H
+#define CELL_ST_H
+
+/** Parsed onion routing cell.  All communication between nodes
+ * is via cells. */
+struct cell_t {
+  circid_t circ_id; /**< Circuit which received the cell. */
+  uint8_t command; /**< Type of the cell: one of CELL_PADDING, CELL_CREATE,
+                    * CELL_DESTROY, etc */
+  uint8_t payload[CELL_PAYLOAD_SIZE]; /**< Cell body. */
+};
+
+#endif
+
diff --git a/src/or/channel.c b/src/core/or/channel.c
index 54e10666d2..fd7bf62789 100644
--- a/src/or/channel.c
+++ b/src/core/or/channel.c
@@ -1,13 +1,13 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+
+/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file channel.c
  *
  * \brief OR/OP-to-OR channel abstraction layer. A channel's job is to
- * transfer cells from Tor instance to Tor instance.
- * Currently, there is only one implementation of the channel abstraction: in
- * channeltls.c.
+ * transfer cells from Tor instance to Tor instance. Currently, there is only
+ * one implementation of the channel abstraction: in channeltls.c.
  *
  * Channels are a higher-level abstraction than or_connection_t: In general,
  * any means that two Tor relays use to exchange cells, or any means that a
@@ -24,45 +24,65 @@
  * connection.
  *
  * Every channel implementation is responsible for being able to transmit
- * cells that are added to it with channel_write_cell() and related functions,
- * and to receive incoming cells with the channel_queue_cell() and related
- * functions.  See the channel_t documentation for more information.
- *
- * When new cells arrive on a channel, they are passed to cell handler
- * functions, which can be set by channel_set_cell_handlers()
- * functions. (Tor's cell handlers are in command.c.)
- *
- * Tor flushes cells to channels from relay.c in
- * channel_flush_from_first_active_circuit().
+ * cells that are passed to it
+ *
+ * For *inbound* cells, the entry point is: channel_process_cell(). It takes a
+ * cell and will pass it to the cell handler set by
+ * channel_set_cell_handlers(). Currently, this is passed back to the command
+ * subsystem which is command_process_cell().
+ *
+ * NOTE: For now, the separation between channels and specialized channels
+ * (like channeltls) is not that well defined. So the channeltls layer calls
+ * channel_process_cell() which originally comes from the connection subsytem.
+ * This should be hopefully be fixed with #23993.
+ *
+ * For *outbound* cells, the entry point is: channel_write_packed_cell().
+ * Only packed cells are dequeued from the circuit queue by the scheduler
+ * which uses channel_flush_from_first_active_circuit() to decide which cells
+ * to flush from which circuit on the channel. They are then passed down to
+ * the channel subsystem. This calls the low layer with the function pointer
+ * .write_packed_cell().
+ *
+ * Each specialized channel (currently only channeltls_t) MUST implement a
+ * series of function found in channel_t. See channel.h for more
+ * documentation.
  **/
 
 /*
  * Define this so channel.h gives us things only channel_t subclasses
  * should touch.
  */
-
 #define TOR_CHANNEL_INTERNAL_
 
 /* This one's for stuff only channel.c and the test suite should see */
 #define CHANNEL_PRIVATE_
 
-#include "or.h"
-#include "channel.h"
-#include "channeltls.h"
-#include "circuitbuild.h"
-#include "circuitlist.h"
-#include "circuitstats.h"
-#include "config.h"
-#include "connection_or.h" /* For var_cell_free() */
-#include "circuitmux.h"
-#include "entrynodes.h"
-#include "geoip.h"
-#include "nodelist.h"
-#include "relay.h"
-#include "rephist.h"
-#include "router.h"
-#include "routerlist.h"
-#include "scheduler.h"
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/channel.h"
+#include "core/or/channelpadding.h"
+#include "core/or/channeltls.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitmux.h"
+#include "core/or/circuitstats.h"
+#include "core/or/connection_or.h" /* For var_cell_free() */
+#include "core/or/dos.h"
+#include "core/or/relay.h"
+#include "core/or/scheduler.h"
+#include "feature/client/entrynodes.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/router.h"
+#include "feature/rend/rendservice.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/stats/rephist.h"
+#include "lib/evloop/timers.h"
+#include "lib/time/compat_time.h"
+
+#include "core/or/cell_queue_st.h"
 
 /* Global lists of channels */
 
@@ -84,61 +104,30 @@ static smartlist_t *active_listeners = NULL;
 /* All channel_listener_t instances in LISTENING state */
 static smartlist_t *finished_listeners = NULL;
 
-/* Counter for ID numbers */
-static uint64_t n_channels_allocated = 0;
-/*
- * Channel global byte/cell counters, for statistics and for scheduler high
- * /low-water marks.
- */
-
-/*
- * Total number of cells ever given to any channel with the
- * channel_write_*_cell() functions.
- */
-
-static uint64_t n_channel_cells_queued = 0;
-
-/*
- * Total number of cells ever passed to a channel lower layer with the
- * write_*_cell() methods.
- */
-
-static uint64_t n_channel_cells_passed_to_lower_layer = 0;
-
-/*
- * Current number of cells in all channel queues; should be
- * n_channel_cells_queued - n_channel_cells_passed_to_lower_layer.
- */
-
-static uint64_t n_channel_cells_in_queues = 0;
-
-/*
- * Total number of bytes for all cells ever queued to a channel and
- * counted in n_channel_cells_queued.
- */
-
-static uint64_t n_channel_bytes_queued = 0;
-
-/*
- * Total number of bytes for all cells ever passed to a channel lower layer
- * and counted in n_channel_cells_passed_to_lower_layer.
- */
-
-static uint64_t n_channel_bytes_passed_to_lower_layer = 0;
+/** Map from channel->global_identifier to channel.  Contains the same
+ * elements as all_channels. */
+static HT_HEAD(channel_gid_map, channel_s) channel_gid_map = HT_INITIALIZER();
 
-/*
- * Current number of bytes in all channel queues; should be
- * n_channel_bytes_queued - n_channel_bytes_passed_to_lower_layer.
- */
-
-static uint64_t n_channel_bytes_in_queues = 0;
+static unsigned
+channel_id_hash(const channel_t *chan)
+{
+  return (unsigned) chan->global_identifier;
+}
+static int
+channel_id_eq(const channel_t *a, const channel_t *b)
+{
+  return a->global_identifier == b->global_identifier;
+}
+HT_PROTOTYPE(channel_gid_map, channel_s, gidmap_node,
+             channel_id_hash, channel_id_eq)
+HT_GENERATE2(channel_gid_map, channel_s, gidmap_node,
+             channel_id_hash, channel_id_eq,
+             0.6, tor_reallocarray_, tor_free_)
 
-/*
- * Current total estimated queue size *including lower layer queues and
- * transmit overhead*
- */
+HANDLE_IMPL(channel, channel_s,)
 
-STATIC uint64_t estimated_total_queue_size = 0;
+/* Counter for ID numbers */
+static uint64_t n_channels_allocated = 0;
 
 /* Digest->channel map
  *
@@ -175,49 +164,23 @@ HT_PROTOTYPE(channel_idmap, channel_idmap_entry_s, node, channel_idmap_hash,
 HT_GENERATE2(channel_idmap, channel_idmap_entry_s, node, channel_idmap_hash,
              channel_idmap_eq, 0.5,  tor_reallocarray_, tor_free_)
 
-static cell_queue_entry_t * cell_queue_entry_dup(cell_queue_entry_t *q);
-#if 0
-static int cell_queue_entry_is_padding(cell_queue_entry_t *q);
-#endif
-static cell_queue_entry_t *
-cell_queue_entry_new_fixed(cell_t *cell);
-static cell_queue_entry_t *
-cell_queue_entry_new_var(var_cell_t *var_cell);
-static int is_destroy_cell(channel_t *chan,
-                           const cell_queue_entry_t *q, circid_t *circid_out);
-
-static void channel_assert_counter_consistency(void);
-
 /* Functions to maintain the digest map */
-static void channel_add_to_digest_map(channel_t *chan);
 static void channel_remove_from_digest_map(channel_t *chan);
 
-/*
- * Flush cells from just the outgoing queue without trying to get them
- * from circuits; used internall by channel_flush_some_cells().
- */
-static ssize_t
-channel_flush_some_cells_from_outgoing_queue(channel_t *chan,
-                                             ssize_t num_cells);
-static void channel_force_free(channel_t *chan);
-static void
-channel_free_list(smartlist_t *channels, int mark_for_close);
-static void
-channel_listener_free_list(smartlist_t *channels, int mark_for_close);
-static void channel_listener_force_free(channel_listener_t *chan_l);
-static size_t channel_get_cell_queue_entry_size(channel_t *chan,
-                                                cell_queue_entry_t *q);
-static void
-channel_write_cell_queue_entry(channel_t *chan, cell_queue_entry_t *q);
+static void channel_force_xfree(channel_t *chan);
+static void channel_free_list(smartlist_t *channels,
+                               int mark_for_close);
+static void channel_listener_free_list(smartlist_t *channels,
+                                        int mark_for_close);
+static void channel_listener_force_xfree(channel_listener_t *chan_l);
 
 /***********************************
  * Channel state utility functions *
  **********************************/
 
 /**
- * Indicate whether a given channel state is valid
+ * Indicate whether a given channel state is valid.
  */
-
 int
 channel_state_is_valid(channel_state_t state)
 {
@@ -241,9 +204,8 @@ channel_state_is_valid(channel_state_t state)
 }
 
 /**
- * Indicate whether a given channel listener state is valid
+ * Indicate whether a given channel listener state is valid.
  */
-
 int
 channel_listener_state_is_valid(channel_listener_state_t state)
 {
@@ -265,13 +227,12 @@ channel_listener_state_is_valid(channel_listener_state_t state)
 }
 
 /**
- * Indicate whether a channel state transition is valid
+ * Indicate whether a channel state transition is valid.
  *
  * This function takes two channel states and indicates whether a
  * transition between them is permitted (see the state definitions and
  * transition table in or.h at the channel_state_t typedef).
  */
-
 int
 channel_state_can_transition(channel_state_t from, channel_state_t to)
 {
@@ -312,13 +273,12 @@ channel_state_can_transition(channel_state_t from, channel_state_t to)
 }
 
 /**
- * Indicate whether a channel listener state transition is valid
+ * Indicate whether a channel listener state transition is valid.
  *
  * This function takes two channel listener states and indicates whether a
  * transition between them is permitted (see the state definitions and
  * transition table in or.h at the channel_listener_state_t typedef).
  */
-
 int
 channel_listener_state_can_transition(channel_listener_state_t from,
                                       channel_listener_state_t to)
@@ -349,9 +309,8 @@ channel_listener_state_can_transition(channel_listener_state_t from,
 }
 
 /**
- * Return a human-readable description for a channel state
+ * Return a human-readable description for a channel state.
  */
-
 const char *
 channel_state_to_string(channel_state_t state)
 {
@@ -385,9 +344,8 @@ channel_state_to_string(channel_state_t state)
 }
 
 /**
- * Return a human-readable description for a channel listenier state
+ * Return a human-readable description for a channel listener state.
  */
-
 const char *
 channel_listener_state_to_string(channel_listener_state_t state)
 {
@@ -419,36 +377,39 @@ channel_listener_state_to_string(channel_listener_state_t state)
  ***************************************/
 
 /**
- * Register a channel
+ * Register a channel.
  *
  * This function registers a newly created channel in the global lists/maps
  * of active channels.
  */
-
 void
 channel_register(channel_t *chan)
 {
   tor_assert(chan);
+  tor_assert(chan->global_identifier);
 
   /* No-op if already registered */
   if (chan->registered) return;
 
   log_debug(LD_CHANNEL,
-            "Registering channel %p (ID " U64_FORMAT ") "
+            "Registering channel %p (ID %"PRIu64 ") "
             "in state %s (%d) with digest %s",
-            chan, U64_PRINTF_ARG(chan->global_identifier),
+            chan, (chan->global_identifier),
             channel_state_to_string(chan->state), chan->state,
             hex_str(chan->identity_digest, DIGEST_LEN));
 
   /* Make sure we have all_channels, then add it */
   if (!all_channels) all_channels = smartlist_new();
   smartlist_add(all_channels, chan);
+  channel_t *oldval = HT_REPLACE(channel_gid_map, &channel_gid_map, chan);
+  tor_assert(! oldval);
 
   /* Is it finished? */
   if (CHANNEL_FINISHED(chan)) {
     /* Put it in the finished list, creating it if necessary */
     if (!finished_channels) finished_channels = smartlist_new();
     smartlist_add(finished_channels, chan);
+    mainloop_schedule_postloop_cleanup();
   } else {
     /* Put it in the active list, creating it if necessary */
     if (!active_channels) active_channels = smartlist_new();
@@ -461,9 +422,9 @@ channel_register(channel_t *chan)
         channel_add_to_digest_map(chan);
       } else {
         log_info(LD_CHANNEL,
-                "Channel %p (global ID " U64_FORMAT ") "
+                "Channel %p (global ID %"PRIu64 ") "
                 "in state %s (%d) registered with no identity digest",
-                chan, U64_PRINTF_ARG(chan->global_identifier),
+                chan, (chan->global_identifier),
                 channel_state_to_string(chan->state), chan->state);
       }
     }
@@ -474,12 +435,11 @@ channel_register(channel_t *chan)
 }
 
 /**
- * Unregister a channel
+ * Unregister a channel.
  *
  * This function removes a channel from the global lists and maps and is used
  * when freeing a closed/errored channel.
  */
-
 void
 channel_unregister(channel_t *chan)
 {
@@ -498,7 +458,9 @@ channel_unregister(channel_t *chan)
   }
 
   /* Get it out of all_channels */
- if (all_channels) smartlist_remove(all_channels, chan);
+  if (all_channels) smartlist_remove(all_channels, chan);
+  channel_t *oldval = HT_REMOVE(channel_gid_map, &channel_gid_map, chan);
+  tor_assert(oldval == NULL || oldval == chan);
 
   /* Mark it as unregistered */
   chan->registered = 0;
@@ -512,12 +474,11 @@ channel_unregister(channel_t *chan)
 }
 
 /**
- * Register a channel listener
+ * Register a channel listener.
  *
- * This function registers a newly created channel listner in the global
+ * This function registers a newly created channel listener in the global
  * lists/maps of active channel listeners.
  */
-
 void
 channel_listener_register(channel_listener_t *chan_l)
 {
@@ -527,13 +488,13 @@ channel_listener_register(channel_listener_t *chan_l)
   if (chan_l->registered) return;
 
   log_debug(LD_CHANNEL,
-            "Registering channel listener %p (ID " U64_FORMAT ") "
+            "Registering channel listener %p (ID %"PRIu64 ") "
             "in state %s (%d)",
-            chan_l, U64_PRINTF_ARG(chan_l->global_identifier),
+            chan_l, (chan_l->global_identifier),
             channel_listener_state_to_string(chan_l->state),
             chan_l->state);
 
-  /* Make sure we have all_channels, then add it */
+  /* Make sure we have all_listeners, then add it */
   if (!all_listeners) all_listeners = smartlist_new();
   smartlist_add(all_listeners, chan_l);
 
@@ -554,12 +515,11 @@ channel_listener_register(channel_listener_t *chan_l)
 }
 
 /**
- * Unregister a channel listener
+ * Unregister a channel listener.
  *
  * This function removes a channel listener from the global lists and maps
  * and is used when freeing a closed/errored channel listener.
  */
-
 void
 channel_listener_unregister(channel_listener_t *chan_l)
 {
@@ -578,7 +538,7 @@ channel_listener_unregister(channel_listener_t *chan_l)
     if (active_listeners) smartlist_remove(active_listeners, chan_l);
   }
 
-  /* Get it out of all_channels */
+  /* Get it out of all_listeners */
  if (all_listeners) smartlist_remove(all_listeners, chan_l);
 
   /* Mark it as unregistered */
@@ -590,14 +550,13 @@ channel_listener_unregister(channel_listener_t *chan_l)
  *********************************/
 
 /**
- * Add a channel to the digest map
+ * Add a channel to the digest map.
  *
  * This function adds a channel to the digest map and inserts it into the
  * correct linked list if channels with that remote endpoint identity digest
  * already exist.
  */
-
-static void
+STATIC void
 channel_add_to_digest_map(channel_t *chan)
 {
   channel_idmap_entry_t *ent, search;
@@ -621,20 +580,19 @@ channel_add_to_digest_map(channel_t *chan)
   TOR_LIST_INSERT_HEAD(&ent->channel_list, chan, next_with_same_id);
 
   log_debug(LD_CHANNEL,
-            "Added channel %p (global ID " U64_FORMAT ") "
+            "Added channel %p (global ID %"PRIu64 ") "
             "to identity map in state %s (%d) with digest %s",
-            chan, U64_PRINTF_ARG(chan->global_identifier),
+            chan, (chan->global_identifier),
             channel_state_to_string(chan->state), chan->state,
             hex_str(chan->identity_digest, DIGEST_LEN));
 }
 
 /**
- * Remove a channel from the digest map
+ * Remove a channel from the digest map.
  *
  * This function removes a channel from the digest map and the linked list of
  * channels for that digest if more than one exists.
  */
-
 static void
 channel_remove_from_digest_map(channel_t *chan)
 {
@@ -645,33 +603,6 @@ channel_remove_from_digest_map(channel_t *chan)
   /* Assert that there is a digest */
   tor_assert(!tor_digest_is_zero(chan->identity_digest));
 
-#if 0
-  /* Make sure we have a map */
-  if (!channel_identity_map) {
-    /*
-     * No identity map, so we can't find it by definition.  This
-     * case is similar to digestmap_get() failing below.
-     */
-    log_warn(LD_BUG,
-             "Trying to remove channel %p (global ID " U64_FORMAT ") "
-             "with digest %s from identity map, but didn't have any identity "
-             "map",
-             chan, U64_PRINTF_ARG(chan->global_identifier),
-             hex_str(chan->identity_digest, DIGEST_LEN));
-    /* Clear out its next/prev pointers */
-    if (chan->next_with_same_id) {
-      chan->next_with_same_id->prev_with_same_id = chan->prev_with_same_id;
-    }
-    if (chan->prev_with_same_id) {
-      chan->prev_with_same_id->next_with_same_id = chan->next_with_same_id;
-    }
-    chan->next_with_same_id = NULL;
-    chan->prev_with_same_id = NULL;
-
-    return;
-  }
-#endif
-
   /* Pull it out of its list, wherever that list is */
   TOR_LIST_REMOVE(chan, next_with_same_id);
 
@@ -688,18 +619,18 @@ channel_remove_from_digest_map(channel_t *chan)
     }
 
     log_debug(LD_CHANNEL,
-              "Removed channel %p (global ID " U64_FORMAT ") from "
+              "Removed channel %p (global ID %"PRIu64 ") from "
               "identity map in state %s (%d) with digest %s",
-              chan, U64_PRINTF_ARG(chan->global_identifier),
+              chan, (chan->global_identifier),
               channel_state_to_string(chan->state), chan->state,
               hex_str(chan->identity_digest, DIGEST_LEN));
   } else {
     /* Shouldn't happen */
     log_warn(LD_BUG,
-             "Trying to remove channel %p (global ID " U64_FORMAT ") with "
+             "Trying to remove channel %p (global ID %"PRIu64 ") with "
              "digest %s from identity map, but couldn't find any with "
              "that digest",
-             chan, U64_PRINTF_ARG(chan->global_identifier),
+             chan, (chan->global_identifier),
              hex_str(chan->identity_digest, DIGEST_LEN));
   }
 }
@@ -709,64 +640,95 @@ channel_remove_from_digest_map(channel_t *chan)
  ***************************/
 
 /**
- * Find channel by global ID
+ * Find channel by global ID.
  *
  * This function searches for a channel by the global_identifier assigned
  * at initialization time.  This identifier is unique for the lifetime of the
  * Tor process.
  */
-
 channel_t *
 channel_find_by_global_id(uint64_t global_identifier)
 {
+  channel_t lookup;
   channel_t *rv = NULL;
 
-  if (all_channels && smartlist_len(all_channels) > 0) {
-    SMARTLIST_FOREACH_BEGIN(all_channels, channel_t *, curr) {
-      if (curr->global_identifier == global_identifier) {
-        rv = curr;
-        break;
-      }
-    } SMARTLIST_FOREACH_END(curr);
+  lookup.global_identifier = global_identifier;
+  rv = HT_FIND(channel_gid_map, &channel_gid_map, &lookup);
+  if (rv) {
+    tor_assert(rv->global_identifier == global_identifier);
   }
 
   return rv;
 }
 
+/** Return true iff <b>chan</b> matches <b>rsa_id_digest</b> and <b>ed_id</b>.
+ * as its identity keys.  If either is NULL, do not check for a match. */
+static int
+channel_remote_identity_matches(const channel_t *chan,
+                                const char *rsa_id_digest,
+                                const ed25519_public_key_t *ed_id)
+{
+  if (BUG(!chan))
+    return 0;
+  if (rsa_id_digest) {
+    if (tor_memneq(rsa_id_digest, chan->identity_digest, DIGEST_LEN))
+      return 0;
+  }
+  if (ed_id) {
+    if (tor_memneq(ed_id->pubkey, chan->ed25519_identity.pubkey,
+                   ED25519_PUBKEY_LEN))
+      return 0;
+  }
+  return 1;
+}
+
 /**
- * Find channel by digest of the remote endpoint
+ * Find channel by RSA/Ed25519 identity of of the remote endpoint.
  *
- * This function looks up a channel by the digest of its remote endpoint in
- * the channel digest map.  It's possible that more than one channel to a
- * given endpoint exists.  Use channel_next_with_digest() to walk the list.
+ * This function looks up a channel by the digest of its remote endpoint's RSA
+ * identity key.  If <b>ed_id</b> is provided and nonzero, only a channel
+ * matching the <b>ed_id</b> will be returned.
+ *
+ * It's possible that more than one channel to a given endpoint exists.  Use
+ * channel_next_with_rsa_identity() to walk the list of channels; make sure
+ * to test for Ed25519 identity match too (as appropriate)
  */
-
 channel_t *
-channel_find_by_remote_digest(const char *identity_digest)
+channel_find_by_remote_identity(const char *rsa_id_digest,
+                                const ed25519_public_key_t *ed_id)
 {
   channel_t *rv = NULL;
   channel_idmap_entry_t *ent, search;
 
-  tor_assert(identity_digest);
+  tor_assert(rsa_id_digest); /* For now, we require that every channel have
+                              * an RSA identity, and that every lookup
+                              * contain an RSA identity */
+  if (ed_id && ed25519_public_key_is_zero(ed_id)) {
+    /* Treat zero as meaning "We don't care about the presence or absence of
+     * an Ed key", not "There must be no Ed key". */
+    ed_id = NULL;
+  }
 
-  memcpy(search.digest, identity_digest, DIGEST_LEN);
+  memcpy(search.digest, rsa_id_digest, DIGEST_LEN);
   ent = HT_FIND(channel_idmap, &channel_identity_map, &search);
   if (ent) {
     rv = TOR_LIST_FIRST(&ent->channel_list);
   }
+  while (rv && ! channel_remote_identity_matches(rv, rsa_id_digest, ed_id)) {
+    rv = channel_next_with_rsa_identity(rv);
+  }
 
   return rv;
 }
 
 /**
- * Get next channel with digest
+ * Get next channel with digest.
  *
  * This function takes a channel and finds the next channel in the list
  * with the same digest.
  */
-
 channel_t *
-channel_next_with_digest(channel_t *chan)
+channel_next_with_rsa_identity(channel_t *chan)
 {
   tor_assert(chan);
 
@@ -774,20 +736,96 @@ channel_next_with_digest(channel_t *chan)
 }
 
 /**
- * Initialize a channel
+ * Relays run this once an hour to look over our list of channels to other
+ * relays. It prints out some statistics if there are multiple connections
+ * to many relays.
+ *
+ * This function is similar to connection_or_set_bad_connections(),
+ * and probably could be adapted to replace it, if it was modified to actually
+ * take action on any of these connections.
+ */
+void
+channel_check_for_duplicates(void)
+{
+  channel_idmap_entry_t **iter;
+  channel_t *chan;
+  int total_relay_connections = 0, total_relays = 0, total_canonical = 0;
+  int total_half_canonical = 0;
+  int total_gt_one_connection = 0, total_gt_two_connections = 0;
+  int total_gt_four_connections = 0;
+
+  HT_FOREACH(iter, channel_idmap, &channel_identity_map) {
+    int connections_to_relay = 0;
+
+    /* Only consider relay connections */
+    if (!connection_or_digest_is_known_relay((char*)(*iter)->digest))
+      continue;
+
+    total_relays++;
+
+    for (chan = TOR_LIST_FIRST(&(*iter)->channel_list); chan;
+        chan = channel_next_with_rsa_identity(chan)) {
+
+      if (CHANNEL_CONDEMNED(chan) || !CHANNEL_IS_OPEN(chan))
+        continue;
+
+      connections_to_relay++;
+      total_relay_connections++;
+
+      if (chan->is_canonical(chan, 0)) total_canonical++;
+
+      if (!chan->is_canonical_to_peer && chan->is_canonical(chan, 0)
+          && chan->is_canonical(chan, 1)) {
+        total_half_canonical++;
+      }
+    }
+
+    if (connections_to_relay > 1) total_gt_one_connection++;
+    if (connections_to_relay > 2) total_gt_two_connections++;
+    if (connections_to_relay > 4) total_gt_four_connections++;
+  }
+
+#define MIN_RELAY_CONNECTIONS_TO_WARN 5
+
+  /* If we average 1.5 or more connections per relay, something is wrong */
+  if (total_relays > MIN_RELAY_CONNECTIONS_TO_WARN &&
+          total_relay_connections >= 1.5*total_relays) {
+    log_notice(LD_OR,
+        "Your relay has a very large number of connections to other relays. "
+        "Is your outbound address the same as your relay address? "
+        "Found %d connections to %d relays. Found %d current canonical "
+        "connections, in %d of which we were a non-canonical peer. "
+        "%d relays had more than 1 connection, %d had more than 2, and "
+        "%d had more than 4 connections.",
+        total_relay_connections, total_relays, total_canonical,
+        total_half_canonical, total_gt_one_connection,
+        total_gt_two_connections, total_gt_four_connections);
+  } else {
+    log_info(LD_OR, "Performed connection pruning. "
+        "Found %d connections to %d relays. Found %d current canonical "
+        "connections, in %d of which we were a non-canonical peer. "
+        "%d relays had more than 1 connection, %d had more than 2, and "
+        "%d had more than 4 connections.",
+        total_relay_connections, total_relays, total_canonical,
+        total_half_canonical, total_gt_one_connection,
+        total_gt_two_connections, total_gt_four_connections);
+  }
+}
+
+/**
+ * Initialize a channel.
  *
  * This function should be called by subclasses to set up some per-channel
  * variables.  I.e., this is the superclass constructor.  Before this, the
  * channel should be allocated with tor_malloc_zero().
  */
-
 void
 channel_init(channel_t *chan)
 {
   tor_assert(chan);
 
   /* Assign an ID and bump the counter */
-  chan->global_identifier = n_channels_allocated++;
+  chan->global_identifier = ++n_channels_allocated;
 
   /* Init timestamp */
   chan->timestamp_last_had_circuits = time(NULL);
@@ -795,10 +833,6 @@ channel_init(channel_t *chan)
   /* Warn about exhausted circuit IDs no more than hourly. */
   chan->last_warned_circ_ids_exhausted.rate = 3600;
 
-  /* Initialize queues. */
-  TOR_SIMPLEQ_INIT(&chan->incoming_queue);
-  TOR_SIMPLEQ_INIT(&chan->outgoing_queue);
-
   /* Initialize list entries. */
   memset(&chan->next_with_same_id, 0, sizeof(chan->next_with_same_id));
 
@@ -810,23 +844,25 @@ channel_init(channel_t *chan)
 
   /* Scheduler state is idle */
   chan->scheduler_state = SCHED_CHAN_IDLE;
+
+  /* Channel is not in the scheduler heap. */
+  chan->sched_heap_idx = -1;
 }
 
 /**
- * Initialize a channel listener
+ * Initialize a channel listener.
  *
  * This function should be called by subclasses to set up some per-channel
  * variables.  I.e., this is the superclass constructor.  Before this, the
  * channel listener should be allocated with tor_malloc_zero().
  */
-
 void
 channel_init_listener(channel_listener_t *chan_l)
 {
   tor_assert(chan_l);
 
   /* Assign an ID and bump the counter */
-  chan_l->global_identifier = n_channels_allocated++;
+  chan_l->global_identifier = ++n_channels_allocated;
 
   /* Timestamp it */
   channel_listener_timestamp_created(chan_l);
@@ -836,9 +872,8 @@ channel_init_listener(channel_listener_t *chan_l)
  * Free a channel; nothing outside of channel.c and subclasses should call
  * this - it frees channels after they have closed and been unregistered.
  */
-
 void
-channel_free(channel_t *chan)
+channel_free_(channel_t *chan)
 {
   if (!chan) return;
 
@@ -849,8 +884,8 @@ channel_free(channel_t *chan)
   tor_assert(!(chan->registered));
 
   log_debug(LD_CHANNEL,
-            "Freeing channel " U64_FORMAT " at %p",
-            U64_PRINTF_ARG(chan->global_identifier), chan);
+            "Freeing channel %"PRIu64 " at %p",
+            (chan->global_identifier), chan);
 
   /* Get this one out of the scheduler */
   scheduler_release_channel(chan);
@@ -863,6 +898,11 @@ channel_free(channel_t *chan)
     circuitmux_set_policy(chan->cmux, NULL);
   }
 
+  /* Remove all timers and associated handle entries now */
+  timer_free(chan->padding_timer);
+  channel_handle_free(chan->timer_handle);
+  channel_handles_clear(chan);
+
   /* Call a free method if there is one */
   if (chan->free_fn) chan->free_fn(chan);
 
@@ -876,8 +916,6 @@ channel_free(channel_t *chan)
     chan->cmux = NULL;
   }
 
-  /* We're in CLOSED or ERROR, so the cell queue is already empty */
-
   tor_free(chan);
 }
 
@@ -886,15 +924,14 @@ channel_free(channel_t *chan)
  * should call this - it frees channel listeners after they have closed and
  * been unregistered.
  */
-
 void
-channel_listener_free(channel_listener_t *chan_l)
+channel_listener_free_(channel_listener_t *chan_l)
 {
   if (!chan_l) return;
 
   log_debug(LD_CHANNEL,
-            "Freeing channel_listener_t " U64_FORMAT " at %p",
-            U64_PRINTF_ARG(chan_l->global_identifier),
+            "Freeing channel_listener_t %"PRIu64 " at %p",
+            (chan_l->global_identifier),
             chan_l);
 
   /* It must be closed or errored */
@@ -906,11 +943,6 @@ channel_listener_free(channel_listener_t *chan_l)
   /* Call a free method if there is one */
   if (chan_l->free_fn) chan_l->free_fn(chan_l);
 
-  /*
-   * We're in CLOSED or ERROR, so the incoming channel queue is already
-   * empty.
-   */
-
   tor_free(chan_l);
 }
 
@@ -919,16 +951,14 @@ channel_listener_free(channel_listener_t *chan_l)
  * use-only function should be called only from channel_free_all() when
  * shutting down the Tor process.
  */
-
 static void
-channel_force_free(channel_t *chan)
+channel_force_xfree(channel_t *chan)
 {
-  cell_queue_entry_t *cell, *cell_tmp;
   tor_assert(chan);
 
   log_debug(LD_CHANNEL,
-            "Force-freeing channel " U64_FORMAT " at %p",
-            U64_PRINTF_ARG(chan->global_identifier), chan);
+            "Force-freeing channel %"PRIu64 " at %p",
+            (chan->global_identifier), chan);
 
   /* Get this one out of the scheduler */
   scheduler_release_channel(chan);
@@ -941,6 +971,11 @@ channel_force_free(channel_t *chan)
     circuitmux_set_policy(chan->cmux, NULL);
   }
 
+  /* Remove all timers and associated handle entries now */
+  timer_free(chan->padding_timer);
+  channel_handle_free(chan->timer_handle);
+  channel_handles_clear(chan);
+
   /* Call a free method if there is one */
   if (chan->free_fn) chan->free_fn(chan);
 
@@ -952,35 +987,22 @@ channel_force_free(channel_t *chan)
     chan->cmux = NULL;
   }
 
-  /* We might still have a cell queue; kill it */
-  TOR_SIMPLEQ_FOREACH_SAFE(cell, &chan->incoming_queue, next, cell_tmp) {
-      cell_queue_entry_free(cell, 0);
-  }
-  TOR_SIMPLEQ_INIT(&chan->incoming_queue);
-
-  /* Outgoing cell queue is similar, but we can have to free packed cells */
-  TOR_SIMPLEQ_FOREACH_SAFE(cell, &chan->outgoing_queue, next, cell_tmp) {
-    cell_queue_entry_free(cell, 0);
-  }
-  TOR_SIMPLEQ_INIT(&chan->outgoing_queue);
-
   tor_free(chan);
 }
 
 /**
- * Free a channel listener and skip the state/reigstration asserts; this
+ * Free a channel listener and skip the state/registration asserts; this
  * internal-use-only function should be called only from channel_free_all()
  * when shutting down the Tor process.
  */
-
 static void
-channel_listener_force_free(channel_listener_t *chan_l)
+channel_listener_force_xfree(channel_listener_t *chan_l)
 {
   tor_assert(chan_l);
 
   log_debug(LD_CHANNEL,
-            "Force-freeing channel_listener_t " U64_FORMAT " at %p",
-            U64_PRINTF_ARG(chan_l->global_identifier),
+            "Force-freeing channel_listener_t %"PRIu64 " at %p",
+            (chan_l->global_identifier),
             chan_l);
 
   /* Call a free method if there is one */
@@ -1005,30 +1027,11 @@ channel_listener_force_free(channel_listener_t *chan_l)
 }
 
 /**
- * Return the current registered listener for a channel listener
- *
- * This function returns a function pointer to the current registered
- * handler for new incoming channels on a channel listener.
- */
-
-channel_listener_fn_ptr
-channel_listener_get_listener_fn(channel_listener_t *chan_l)
-{
-  tor_assert(chan_l);
-
-  if (chan_l->state == CHANNEL_LISTENER_STATE_LISTENING)
-    return chan_l->listener;
-
-  return NULL;
-}
-
-/**
- * Set the listener for a channel listener
+ * Set the listener for a channel listener.
  *
  * This function sets the handler for new incoming channels on a channel
  * listener.
  */
-
 void
 channel_listener_set_listener_fn(channel_listener_t *chan_l,
                                 channel_listener_fn_ptr listener)
@@ -1038,8 +1041,8 @@ channel_listener_set_listener_fn(channel_listener_t *chan_l,
 
   log_debug(LD_CHANNEL,
            "Setting listener callback for channel listener %p "
-           "(global ID " U64_FORMAT ") to %p",
-           chan_l, U64_PRINTF_ARG(chan_l->global_identifier),
+           "(global ID %"PRIu64 ") to %p",
+           chan_l, (chan_l->global_identifier),
            listener);
 
   chan_l->listener = listener;
@@ -1047,12 +1050,11 @@ channel_listener_set_listener_fn(channel_listener_t *chan_l,
 }
 
 /**
- * Return the fixed-length cell handler for a channel
+ * Return the fixed-length cell handler for a channel.
  *
  * This function gets the handler for incoming fixed-length cells installed
  * on a channel.
  */
-
 channel_cell_handler_fn_ptr
 channel_get_cell_handler(channel_t *chan)
 {
@@ -1065,12 +1067,11 @@ channel_get_cell_handler(channel_t *chan)
 }
 
 /**
- * Return the variable-length cell handler for a channel
+ * Return the variable-length cell handler for a channel.
  *
  * This function gets the handler for incoming variable-length cells
  * installed on a channel.
  */
-
 channel_var_cell_handler_fn_ptr
 channel_get_var_cell_handler(channel_t *chan)
 {
@@ -1083,21 +1084,17 @@ channel_get_var_cell_handler(channel_t *chan)
 }
 
 /**
- * Set both cell handlers for a channel
+ * Set both cell handlers for a channel.
  *
  * This function sets both the fixed-length and variable length cell handlers
- * for a channel and processes any incoming cells that had been blocked in the
- * queue because none were available.
+ * for a channel.
  */
-
 void
 channel_set_cell_handlers(channel_t *chan,
                           channel_cell_handler_fn_ptr cell_handler,
                           channel_var_cell_handler_fn_ptr
                             var_cell_handler)
 {
-  int try_again = 0;
-
   tor_assert(chan);
   tor_assert(CHANNEL_CAN_HANDLE_CELLS(chan));
 
@@ -1108,21 +1105,9 @@ channel_set_cell_handlers(channel_t *chan,
            "Setting var_cell_handler callback for channel %p to %p",
            chan, var_cell_handler);
 
-  /* Should we try the queue? */
-  if (cell_handler &&
-      cell_handler != chan->cell_handler) try_again = 1;
-  if (var_cell_handler &&
-      var_cell_handler != chan->var_cell_handler) try_again = 1;
-
   /* Change them */
   chan->cell_handler = cell_handler;
   chan->var_cell_handler = var_cell_handler;
-
-  /* Re-run the queue if we have one and there's any reason to */
-  if (! TOR_SIMPLEQ_EMPTY(&chan->incoming_queue) &&
-      try_again &&
-      (chan->cell_handler ||
-       chan->var_cell_handler)) channel_process_cells(chan);
 }
 
 /*
@@ -1139,13 +1124,12 @@ channel_set_cell_handlers(channel_t *chan,
  */
 
 /**
- * Mark a channel for closure
+ * Mark a channel for closure.
  *
  * This function tries to close a channel_t; it will go into the CLOSING
  * state, and eventually the lower layer should put it into the CLOSED or
  * ERROR state.  Then, channel_run_cleanup() will eventually free it.
  */
-
 void
 channel_mark_for_close(channel_t *chan)
 {
@@ -1157,9 +1141,9 @@ channel_mark_for_close(channel_t *chan)
     return;
 
   log_debug(LD_CHANNEL,
-            "Closing channel %p (global ID " U64_FORMAT ") "
+            "Closing channel %p (global ID %"PRIu64 ") "
             "by request",
-            chan, U64_PRINTF_ARG(chan->global_identifier));
+            chan, (chan->global_identifier));
 
   /* Note closing by request from above */
   chan->reason_for_closing = CHANNEL_CLOSE_REQUESTED;
@@ -1179,13 +1163,12 @@ channel_mark_for_close(channel_t *chan)
 }
 
 /**
- * Mark a channel listener for closure
+ * Mark a channel listener for closure.
  *
  * This function tries to close a channel_listener_t; it will go into the
  * CLOSING state, and eventually the lower layer should put it into the CLOSED
  * or ERROR state.  Then, channel_run_cleanup() will eventually free it.
  */
-
 void
 channel_listener_mark_for_close(channel_listener_t *chan_l)
 {
@@ -1198,9 +1181,9 @@ channel_listener_mark_for_close(channel_listener_t *chan_l)
       chan_l->state == CHANNEL_LISTENER_STATE_ERROR) return;
 
   log_debug(LD_CHANNEL,
-            "Closing channel listener %p (global ID " U64_FORMAT ") "
+            "Closing channel listener %p (global ID %"PRIu64 ") "
             "by request",
-            chan_l, U64_PRINTF_ARG(chan_l->global_identifier));
+            chan_l, (chan_l->global_identifier));
 
   /* Note closing by request from above */
   chan_l->reason_for_closing = CHANNEL_LISTENER_CLOSE_REQUESTED;
@@ -1220,13 +1203,12 @@ channel_listener_mark_for_close(channel_listener_t *chan_l)
 }
 
 /**
- * Close a channel from the lower layer
+ * Close a channel from the lower layer.
  *
  * Notify the channel code that the channel is being closed due to a non-error
  * condition in the lower layer.  This does not call the close() method, since
  * the lower layer already knows.
  */
-
 void
 channel_close_from_lower_layer(channel_t *chan)
 {
@@ -1237,9 +1219,9 @@ channel_close_from_lower_layer(channel_t *chan)
     return;
 
   log_debug(LD_CHANNEL,
-            "Closing channel %p (global ID " U64_FORMAT ") "
+            "Closing channel %p (global ID %"PRIu64 ") "
             "due to lower-layer event",
-            chan, U64_PRINTF_ARG(chan->global_identifier));
+            chan, (chan->global_identifier));
 
   /* Note closing by event from below */
   chan->reason_for_closing = CHANNEL_CLOSE_FROM_BELOW;
@@ -1249,43 +1231,12 @@ channel_close_from_lower_layer(channel_t *chan)
 }
 
 /**
- * Close a channel listener from the lower layer
- *
- * Notify the channel code that the channel listener is being closed due to a
- * non-error condition in the lower layer.  This does not call the close()
- * method, since the lower layer already knows.
- */
-
-void
-channel_listener_close_from_lower_layer(channel_listener_t *chan_l)
-{
-  tor_assert(chan_l != NULL);
-
-  /* If it's already in CLOSING, CLOSED or ERROR, this is a no-op */
-  if (chan_l->state == CHANNEL_LISTENER_STATE_CLOSING ||
-      chan_l->state == CHANNEL_LISTENER_STATE_CLOSED ||
-      chan_l->state == CHANNEL_LISTENER_STATE_ERROR) return;
-
-  log_debug(LD_CHANNEL,
-            "Closing channel listener %p (global ID " U64_FORMAT ") "
-            "due to lower-layer event",
-            chan_l, U64_PRINTF_ARG(chan_l->global_identifier));
-
-  /* Note closing by event from below */
-  chan_l->reason_for_closing = CHANNEL_LISTENER_CLOSE_FROM_BELOW;
-
-  /* Change state to CLOSING */
-  channel_listener_change_state(chan_l, CHANNEL_LISTENER_STATE_CLOSING);
-}
-
-/**
- * Notify that the channel is being closed due to an error condition
+ * Notify that the channel is being closed due to an error condition.
  *
  * This function is called by the lower layer implementing the transport
  * when a channel must be closed due to an error condition.  This does not
  * call the channel's close method, since the lower layer already knows.
  */
-
 void
 channel_close_for_error(channel_t *chan)
 {
@@ -1307,44 +1258,12 @@ channel_close_for_error(channel_t *chan)
 }
 
 /**
- * Notify that the channel listener is being closed due to an error condition
- *
- * This function is called by the lower layer implementing the transport
- * when a channel listener must be closed due to an error condition.  This
- * does not call the channel listener's close method, since the lower layer
- * already knows.
- */
-
-void
-channel_listener_close_for_error(channel_listener_t *chan_l)
-{
-  tor_assert(chan_l != NULL);
-
-  /* If it's already in CLOSING, CLOSED or ERROR, this is a no-op */
-  if (chan_l->state == CHANNEL_LISTENER_STATE_CLOSING ||
-      chan_l->state == CHANNEL_LISTENER_STATE_CLOSED ||
-      chan_l->state == CHANNEL_LISTENER_STATE_ERROR) return;
-
-  log_debug(LD_CHANNEL,
-            "Closing channel listener %p (global ID " U64_FORMAT ") "
-            "due to lower-layer error",
-            chan_l, U64_PRINTF_ARG(chan_l->global_identifier));
-
-  /* Note closing by event from below */
-  chan_l->reason_for_closing = CHANNEL_LISTENER_CLOSE_FOR_ERROR;
-
-  /* Change state to CLOSING */
-  channel_listener_change_state(chan_l, CHANNEL_LISTENER_STATE_CLOSING);
-}
-
-/**
- * Notify that the lower layer is finished closing the channel
+ * Notify that the lower layer is finished closing the channel.
  *
  * This function should be called by the lower layer when a channel
  * is finished closing and it should be regarded as inactive and
  * freed by the channel code.
  */
-
 void
 channel_closed(channel_t *chan)
 {
@@ -1371,39 +1290,11 @@ channel_closed(channel_t *chan)
 }
 
 /**
- * Notify that the lower layer is finished closing the channel listener
- *
- * This function should be called by the lower layer when a channel listener
- * is finished closing and it should be regarded as inactive and
- * freed by the channel code.
- */
-
-void
-channel_listener_closed(channel_listener_t *chan_l)
-{
-  tor_assert(chan_l);
-  tor_assert(chan_l->state == CHANNEL_LISTENER_STATE_CLOSING ||
-             chan_l->state == CHANNEL_LISTENER_STATE_CLOSED ||
-             chan_l->state == CHANNEL_LISTENER_STATE_ERROR);
-
-  /* No-op if already inactive */
-  if (chan_l->state == CHANNEL_LISTENER_STATE_CLOSED ||
-      chan_l->state == CHANNEL_LISTENER_STATE_ERROR) return;
-
-  if (chan_l->reason_for_closing != CHANNEL_LISTENER_CLOSE_FOR_ERROR) {
-    channel_listener_change_state(chan_l, CHANNEL_LISTENER_STATE_CLOSED);
-  } else {
-    channel_listener_change_state(chan_l, CHANNEL_LISTENER_STATE_ERROR);
-  }
-}
-
-/**
- * Clear the identity_digest of a channel
+ * Clear the identity_digest of a channel.
  *
  * This function clears the identity digest of the remote endpoint for a
  * channel; this is intended for use by the lower layer.
  */
-
 void
 channel_clear_identity_digest(channel_t *chan)
 {
@@ -1413,8 +1304,8 @@ channel_clear_identity_digest(channel_t *chan)
 
   log_debug(LD_CHANNEL,
             "Clearing remote endpoint digest on channel %p with "
-            "global ID " U64_FORMAT,
-            chan, U64_PRINTF_ARG(chan->global_identifier));
+            "global ID %"PRIu64,
+            chan, (chan->global_identifier));
 
   state_not_in_map = CHANNEL_CONDEMNED(chan);
 
@@ -1428,15 +1319,15 @@ channel_clear_identity_digest(channel_t *chan)
 }
 
 /**
- * Set the identity_digest of a channel
+ * Set the identity_digest of a channel.
  *
  * This function sets the identity digest of the remote endpoint for a
  * channel; this is intended for use by the lower layer.
  */
-
 void
 channel_set_identity_digest(channel_t *chan,
-                            const char *identity_digest)
+                            const char *identity_digest,
+                            const ed25519_public_key_t *ed_identity)
 {
   int was_in_digest_map, should_be_in_digest_map, state_not_in_map;
 
@@ -1444,8 +1335,8 @@ channel_set_identity_digest(channel_t *chan,
 
   log_debug(LD_CHANNEL,
             "Setting remote endpoint digest on channel %p with "
-            "global ID " U64_FORMAT " to digest %s",
-            chan, U64_PRINTF_ARG(chan->global_identifier),
+            "global ID %"PRIu64 " to digest %s",
+            chan, (chan->global_identifier),
             identity_digest ?
               hex_str(identity_digest, DIGEST_LEN) : "(null)");
 
@@ -1475,6 +1366,11 @@ channel_set_identity_digest(channel_t *chan,
     memset(chan->identity_digest, 0,
            sizeof(chan->identity_digest));
   }
+  if (ed_identity) {
+    memcpy(&chan->ed25519_identity, ed_identity, sizeof(*ed_identity));
+  } else {
+    memset(&chan->ed25519_identity, 0, sizeof(*ed_identity));
+  }
 
   /* Put it in the digest map if we should */
   if (should_be_in_digest_map)
@@ -1482,12 +1378,11 @@ channel_set_identity_digest(channel_t *chan,
 }
 
 /**
- * Clear the remote end metadata (identity_digest/nickname) of a channel
+ * Clear the remote end metadata (identity_digest) of a channel.
  *
  * This function clears all the remote end info from a channel; this is
  * intended for use by the lower layer.
  */
-
 void
 channel_clear_remote_end(channel_t *chan)
 {
@@ -1497,8 +1392,8 @@ channel_clear_remote_end(channel_t *chan)
 
   log_debug(LD_CHANNEL,
             "Clearing remote endpoint identity on channel %p with "
-            "global ID " U64_FORMAT,
-            chan, U64_PRINTF_ARG(chan->global_identifier));
+            "global ID %"PRIu64,
+            chan, (chan->global_identifier));
 
   state_not_in_map = CHANNEL_CONDEMNED(chan);
 
@@ -1509,460 +1404,105 @@ channel_clear_remote_end(channel_t *chan)
 
   memset(chan->identity_digest, 0,
          sizeof(chan->identity_digest));
-  tor_free(chan->nickname);
 }
 
 /**
- * Set the remote end metadata (identity_digest/nickname) of a channel
+ * Write to a channel the given packed cell.
  *
- * This function sets new remote end info on a channel; this is intended
- * for use by the lower layer.
- */
-
-void
-channel_set_remote_end(channel_t *chan,
-                       const char *identity_digest,
-                       const char *nickname)
-{
-  int was_in_digest_map, should_be_in_digest_map, state_not_in_map;
-
-  tor_assert(chan);
-
-  log_debug(LD_CHANNEL,
-            "Setting remote endpoint identity on channel %p with "
-            "global ID " U64_FORMAT " to nickname %s, digest %s",
-            chan, U64_PRINTF_ARG(chan->global_identifier),
-            nickname ? nickname : "(null)",
-            identity_digest ?
-              hex_str(identity_digest, DIGEST_LEN) : "(null)");
-
-  state_not_in_map = CHANNEL_CONDEMNED(chan);
-
-  was_in_digest_map =
-    !state_not_in_map &&
-    chan->registered &&
-    !tor_digest_is_zero(chan->identity_digest);
-  should_be_in_digest_map =
-    !state_not_in_map &&
-    chan->registered &&
-    (identity_digest &&
-     !tor_digest_is_zero(identity_digest));
-
-  if (was_in_digest_map)
-    /* We should always remove it; we'll add it back if we're writing
-     * in a new digest.
-     */
-    channel_remove_from_digest_map(chan);
-
-  if (identity_digest) {
-    memcpy(chan->identity_digest,
-           identity_digest,
-           sizeof(chan->identity_digest));
-
-  } else {
-    memset(chan->identity_digest, 0,
-           sizeof(chan->identity_digest));
-  }
-
-  tor_free(chan->nickname);
-  if (nickname)
-    chan->nickname = tor_strdup(nickname);
-
-  /* Put it in the digest map if we should */
-  if (should_be_in_digest_map)
-    channel_add_to_digest_map(chan);
-}
-
-/**
- * Duplicate a cell queue entry; this is a shallow copy intended for use
- * in channel_write_cell_queue_entry().
- */
-
-static cell_queue_entry_t *
-cell_queue_entry_dup(cell_queue_entry_t *q)
-{
-  cell_queue_entry_t *rv = NULL;
-
-  tor_assert(q);
-
-  rv = tor_malloc(sizeof(*rv));
-  memcpy(rv, q, sizeof(*rv));
-
-  return rv;
-}
-
-/**
- * Free a cell_queue_entry_t; the handed_off parameter indicates whether
- * the contents were passed to the lower layer (it is responsible for
- * them) or not (we should free).
+ * Two possible errors can happen. Either the channel is not opened or the
+ * lower layer (specialized channel) failed to write it. In both cases, it is
+ * the caller responsibility to free the cell.
  */
-
-STATIC void
-cell_queue_entry_free(cell_queue_entry_t *q, int handed_off)
-{
-  if (!q) return;
-
-  if (!handed_off) {
-    /*
-     * If we handed it off, the recipient becomes responsible (or
-     * with packed cells the channel_t subclass calls packed_cell
-     * free after writing out its contents; see, e.g.,
-     * channel_tls_write_packed_cell_method().  Otherwise, we have
-     * to take care of it here if possible.
-     */
-    switch (q->type) {
-      case CELL_QUEUE_FIXED:
-        if (q->u.fixed.cell) {
-          /*
-           * There doesn't seem to be a cell_free() function anywhere in the
-           * pre-channel code; just use tor_free()
-           */
-          tor_free(q->u.fixed.cell);
-        }
-        break;
-      case CELL_QUEUE_PACKED:
-        if (q->u.packed.packed_cell) {
-          packed_cell_free(q->u.packed.packed_cell);
-        }
-        break;
-      case CELL_QUEUE_VAR:
-        if (q->u.var.var_cell) {
-          /*
-           * This one's in connection_or.c; it'd be nice to figure out the
-           * whole flow of cells from one end to the other and factor the
-           * cell memory management functions like this out of the specific
-           * TLS lower layer.
-           */
-          var_cell_free(q->u.var.var_cell);
-        }
-        break;
-      default:
-        /*
-         * Nothing we can do if we don't know the type; this will
-         * have been warned about elsewhere.
-         */
-        break;
-    }
-  }
-  tor_free(q);
-}
-
-#if 0
-/**
- * Check whether a cell queue entry is padding; this is a helper function
- * for channel_write_cell_queue_entry()
- */
-
 static int
-cell_queue_entry_is_padding(cell_queue_entry_t *q)
-{
-  tor_assert(q);
-
-  if (q->type == CELL_QUEUE_FIXED) {
-    if (q->u.fixed.cell) {
-      if (q->u.fixed.cell->command == CELL_PADDING ||
-          q->u.fixed.cell->command == CELL_VPADDING) {
-        return 1;
-      }
-    }
-  } else if (q->type == CELL_QUEUE_VAR) {
-    if (q->u.var.var_cell) {
-      if (q->u.var.var_cell->command == CELL_PADDING ||
-          q->u.var.var_cell->command == CELL_VPADDING) {
-        return 1;
-      }
-    }
-  }
-
-  return 0;
-}
-#endif
-
-/**
- * Allocate a new cell queue entry for a fixed-size cell
- */
-
-static cell_queue_entry_t *
-cell_queue_entry_new_fixed(cell_t *cell)
-{
-  cell_queue_entry_t *q = NULL;
-
-  tor_assert(cell);
-
-  q = tor_malloc(sizeof(*q));
-  q->type = CELL_QUEUE_FIXED;
-  q->u.fixed.cell = cell;
-
-  return q;
-}
-
-/**
- * Allocate a new cell queue entry for a variable-size cell
- */
-
-static cell_queue_entry_t *
-cell_queue_entry_new_var(var_cell_t *var_cell)
-{
-  cell_queue_entry_t *q = NULL;
-
-  tor_assert(var_cell);
-
-  q = tor_malloc(sizeof(*q));
-  q->type = CELL_QUEUE_VAR;
-  q->u.var.var_cell = var_cell;
-
-  return q;
-}
-
-/**
- * Ask how big the cell contained in a cell_queue_entry_t is
- */
-
-static size_t
-channel_get_cell_queue_entry_size(channel_t *chan, cell_queue_entry_t *q)
+write_packed_cell(channel_t *chan, packed_cell_t *cell)
 {
-  size_t rv = 0;
-
-  tor_assert(chan);
-  tor_assert(q);
-
-  switch (q->type) {
-    case CELL_QUEUE_FIXED:
-      rv = get_cell_network_size(chan->wide_circ_ids);
-      break;
-    case CELL_QUEUE_VAR:
-      rv = get_var_cell_header_size(chan->wide_circ_ids) +
-           (q->u.var.var_cell ? q->u.var.var_cell->payload_len : 0);
-      break;
-    case CELL_QUEUE_PACKED:
-      rv = get_cell_network_size(chan->wide_circ_ids);
-      break;
-    default:
-      tor_assert(1);
-  }
-
-  return rv;
-}
-
-/**
- * Write to a channel based on a cell_queue_entry_t
- *
- * Given a cell_queue_entry_t filled out by the caller, try to send the cell
- * and queue it if we can't.
- */
-
-static void
-channel_write_cell_queue_entry(channel_t *chan, cell_queue_entry_t *q)
-{
-  int result = 0, sent = 0;
-  cell_queue_entry_t *tmp = NULL;
+  int ret = -1;
   size_t cell_bytes;
 
   tor_assert(chan);
-  tor_assert(q);
+  tor_assert(cell);
 
   /* Assert that the state makes sense for a cell write */
   tor_assert(CHANNEL_CAN_HANDLE_CELLS(chan));
 
   {
     circid_t circ_id;
-    if (is_destroy_cell(chan, q, &circ_id)) {
+    if (packed_cell_is_destroy(chan, cell, &circ_id)) {
       channel_note_destroy_not_pending(chan, circ_id);
     }
   }
 
   /* For statistical purposes, figure out how big this cell is */
-  cell_bytes = channel_get_cell_queue_entry_size(chan, q);
+  cell_bytes = get_cell_network_size(chan->wide_circ_ids);
 
   /* Can we send it right out?  If so, try */
-  if (TOR_SIMPLEQ_EMPTY(&chan->outgoing_queue) &&
-      CHANNEL_IS_OPEN(chan)) {
-    /* Pick the right write function for this cell type and save the result */
-    switch (q->type) {
-      case CELL_QUEUE_FIXED:
-        tor_assert(chan->write_cell);
-        tor_assert(q->u.fixed.cell);
-        result = chan->write_cell(chan, q->u.fixed.cell);
-        break;
-      case CELL_QUEUE_PACKED:
-        tor_assert(chan->write_packed_cell);
-        tor_assert(q->u.packed.packed_cell);
-        result = chan->write_packed_cell(chan, q->u.packed.packed_cell);
-        break;
-      case CELL_QUEUE_VAR:
-        tor_assert(chan->write_var_cell);
-        tor_assert(q->u.var.var_cell);
-        result = chan->write_var_cell(chan, q->u.var.var_cell);
-        break;
-      default:
-        tor_assert(1);
-    }
-
-    /* Check if we got it out */
-    if (result > 0) {
-      sent = 1;
-      /* Timestamp for transmission */
-      channel_timestamp_xmit(chan);
-      /* If we're here the queue is empty, so it's drained too */
-      channel_timestamp_drained(chan);
-      /* Update the counter */
-      ++(chan->n_cells_xmitted);
-      chan->n_bytes_xmitted += cell_bytes;
-      /* Update global counters */
-      ++n_channel_cells_queued;
-      ++n_channel_cells_passed_to_lower_layer;
-      n_channel_bytes_queued += cell_bytes;
-      n_channel_bytes_passed_to_lower_layer += cell_bytes;
-      channel_assert_counter_consistency();
-    }
-  }
-
-  if (!sent) {
-    /* Not sent, queue it */
-    /*
-     * We have to copy the queue entry passed in, since the caller probably
-     * used the stack.
-     */
-    tmp = cell_queue_entry_dup(q);
-    TOR_SIMPLEQ_INSERT_TAIL(&chan->outgoing_queue, tmp, next);
-    /* Update global counters */
-    ++n_channel_cells_queued;
-    ++n_channel_cells_in_queues;
-    n_channel_bytes_queued += cell_bytes;
-    n_channel_bytes_in_queues += cell_bytes;
-    channel_assert_counter_consistency();
-    /* Update channel queue size */
-    chan->bytes_in_queue += cell_bytes;
-    /* Try to process the queue? */
-    if (CHANNEL_IS_OPEN(chan)) channel_flush_cells(chan);
+  if (!CHANNEL_IS_OPEN(chan)) {
+    goto done;
   }
-}
-
-/**
- * Write a cell to a channel
- *
- * Write a fixed-length cell to a channel using the write_cell() method.
- * This is equivalent to the pre-channels connection_or_write_cell_to_buf();
- * it is called by the transport-independent code to deliver a cell to a
- * channel for transmission.
- */
-
-void
-channel_write_cell(channel_t *chan, cell_t *cell)
-{
-  cell_queue_entry_t q;
-
-  tor_assert(chan);
-  tor_assert(cell);
 
-  if (CHANNEL_IS_CLOSING(chan)) {
-    log_debug(LD_CHANNEL, "Discarding cell_t %p on closing channel %p with "
-              "global ID "U64_FORMAT, cell, chan,
-              U64_PRINTF_ARG(chan->global_identifier));
-    tor_free(cell);
-    return;
+  /* Write the cell on the connection's outbuf. */
+  if (chan->write_packed_cell(chan, cell) < 0) {
+    goto done;
   }
+  /* Timestamp for transmission */
+  channel_timestamp_xmit(chan);
+  /* Update the counter */
+  ++(chan->n_cells_xmitted);
+  chan->n_bytes_xmitted += cell_bytes;
+  /* Successfully sent the cell. */
+  ret = 0;
 
-  log_debug(LD_CHANNEL,
-            "Writing cell_t %p to channel %p with global ID "
-            U64_FORMAT,
-            cell, chan, U64_PRINTF_ARG(chan->global_identifier));
-
-  q.type = CELL_QUEUE_FIXED;
-  q.u.fixed.cell = cell;
-  channel_write_cell_queue_entry(chan, &q);
-
-  /* Update the queue size estimate */
-  channel_update_xmit_queue_size(chan);
+ done:
+  return ret;
 }
 
 /**
- * Write a packed cell to a channel
+ * Write a packed cell to a channel.
  *
  * Write a packed cell to a channel using the write_cell() method.  This is
  * called by the transport-independent code to deliver a packed cell to a
  * channel for transmission.
- */
-
-void
-channel_write_packed_cell(channel_t *chan, packed_cell_t *packed_cell)
-{
-  cell_queue_entry_t q;
-
-  tor_assert(chan);
-  tor_assert(packed_cell);
-
-  if (CHANNEL_IS_CLOSING(chan)) {
-    log_debug(LD_CHANNEL, "Discarding packed_cell_t %p on closing channel %p "
-              "with global ID "U64_FORMAT, packed_cell, chan,
-              U64_PRINTF_ARG(chan->global_identifier));
-    packed_cell_free(packed_cell);
-    return;
-  }
-
-  log_debug(LD_CHANNEL,
-            "Writing packed_cell_t %p to channel %p with global ID "
-            U64_FORMAT,
-            packed_cell, chan,
-            U64_PRINTF_ARG(chan->global_identifier));
-
-  q.type = CELL_QUEUE_PACKED;
-  q.u.packed.packed_cell = packed_cell;
-  channel_write_cell_queue_entry(chan, &q);
-
-  /* Update the queue size estimate */
-  channel_update_xmit_queue_size(chan);
-}
-
-/**
- * Write a variable-length cell to a channel
  *
- * Write a variable-length cell to a channel using the write_cell() method.
- * This is equivalent to the pre-channels
- * connection_or_write_var_cell_to_buf(); it's called by the transport-
- * independent code to deliver a var_cell to a channel for transmission.
+ * Return 0 on success else a negative value. In both cases, the caller should
+ * not access the cell anymore, it is freed both on success and error.
  */
-
-void
-channel_write_var_cell(channel_t *chan, var_cell_t *var_cell)
+int
+channel_write_packed_cell(channel_t *chan, packed_cell_t *cell)
 {
-  cell_queue_entry_t q;
+  int ret = -1;
 
   tor_assert(chan);
-  tor_assert(var_cell);
+  tor_assert(cell);
 
   if (CHANNEL_IS_CLOSING(chan)) {
-    log_debug(LD_CHANNEL, "Discarding var_cell_t %p on closing channel %p "
-              "with global ID "U64_FORMAT, var_cell, chan,
-              U64_PRINTF_ARG(chan->global_identifier));
-    var_cell_free(var_cell);
-    return;
+    log_debug(LD_CHANNEL, "Discarding %p on closing channel %p with "
+              "global ID %"PRIu64, cell, chan,
+              (chan->global_identifier));
+    goto end;
   }
-
   log_debug(LD_CHANNEL,
-            "Writing var_cell_t %p to channel %p with global ID "
-            U64_FORMAT,
-            var_cell, chan,
-            U64_PRINTF_ARG(chan->global_identifier));
+            "Writing %p to channel %p with global ID "
+            "%"PRIu64, cell, chan, (chan->global_identifier));
 
-  q.type = CELL_QUEUE_VAR;
-  q.u.var.var_cell = var_cell;
-  channel_write_cell_queue_entry(chan, &q);
+  ret = write_packed_cell(chan, cell);
 
-  /* Update the queue size estimate */
-  channel_update_xmit_queue_size(chan);
+ end:
+  /* Whatever happens, we free the cell. Either an error occurred or the cell
+   * was put on the connection outbuf, both cases we have ownership of the
+   * cell and we free it. */
+  packed_cell_free(cell);
+  return ret;
 }
 
 /**
- * Change channel state
+ * Change channel state.
  *
  * This internal and subclass use only function is used to change channel
  * state, performing all transition validity checks and whatever actions
  * are appropriate to the state transition in question.
  */
-
-void
-channel_change_state(channel_t *chan, channel_state_t to_state)
+static void
+channel_change_state_(channel_t *chan, channel_state_t to_state)
 {
   channel_state_t from_state;
   unsigned char was_active, is_active;
@@ -1979,9 +1519,9 @@ channel_change_state(channel_t *chan, channel_state_t to_state)
   if (from_state == to_state) {
     log_debug(LD_CHANNEL,
               "Got no-op transition from \"%s\" to itself on channel %p"
-              "(global ID " U64_FORMAT ")",
+              "(global ID %"PRIu64 ")",
               channel_state_to_string(to_state),
-              chan, U64_PRINTF_ARG(chan->global_identifier));
+              chan, (chan->global_identifier));
     return;
   }
 
@@ -1992,20 +1532,11 @@ channel_change_state(channel_t *chan, channel_state_t to_state)
     tor_assert(chan->reason_for_closing != CHANNEL_NOT_CLOSING);
   }
 
-  /*
-   * We need to maintain the queues here for some transitions:
-   * when we enter CHANNEL_STATE_OPEN (especially from CHANNEL_STATE_MAINT)
-   * we may have a backlog of cells to transmit, so drain the queues in
-   * that case, and when going to CHANNEL_STATE_CLOSED the subclass
-   * should have made sure to finish sending things (or gone to
-   * CHANNEL_STATE_ERROR if not possible), so we assert for that here.
-   */
-
   log_debug(LD_CHANNEL,
-            "Changing state of channel %p (global ID " U64_FORMAT
+            "Changing state of channel %p (global ID %"PRIu64
             ") from \"%s\" to \"%s\"",
             chan,
-            U64_PRINTF_ARG(chan->global_identifier),
+            (chan->global_identifier),
             channel_state_to_string(chan->state),
             channel_state_to_string(to_state));
 
@@ -2023,6 +1554,7 @@ channel_change_state(channel_t *chan, channel_state_t to_state)
       if (active_channels) smartlist_remove(active_channels, chan);
       if (!finished_channels) finished_channels = smartlist_new();
       smartlist_add(finished_channels, chan);
+      mainloop_schedule_postloop_cleanup();
     }
     /* Need to put on active list? */
     else if (!was_active && is_active) {
@@ -2057,56 +1589,38 @@ channel_change_state(channel_t *chan, channel_state_t to_state)
   } else if (to_state == CHANNEL_STATE_MAINT) {
     scheduler_channel_doesnt_want_writes(chan);
   }
+}
 
-  /*
-   * If we're closing, this channel no longer counts toward the global
-   * estimated queue size; if we're open, it now does.
-   */
-  if ((to_state == CHANNEL_STATE_CLOSING ||
-       to_state == CHANNEL_STATE_CLOSED ||
-       to_state == CHANNEL_STATE_ERROR) &&
-      (from_state == CHANNEL_STATE_OPEN ||
-       from_state == CHANNEL_STATE_MAINT)) {
-    estimated_total_queue_size -= chan->bytes_in_queue;
-  }
+/**
+ * As channel_change_state_, but change the state to any state but open.
+ */
+void
+channel_change_state(channel_t *chan, channel_state_t to_state)
+{
+  tor_assert(to_state != CHANNEL_STATE_OPEN);
+  channel_change_state_(chan, to_state);
+}
 
-  /*
-   * If we're opening, this channel now does count toward the global
-   * estimated queue size.
-   */
-  if ((to_state == CHANNEL_STATE_OPEN ||
-       to_state == CHANNEL_STATE_MAINT) &&
-      !(from_state == CHANNEL_STATE_OPEN ||
-        from_state == CHANNEL_STATE_MAINT)) {
-    estimated_total_queue_size += chan->bytes_in_queue;
-  }
+/**
+ * As channel_change_state, but change the state to open.
+ */
+void
+channel_change_state_open(channel_t *chan)
+{
+  channel_change_state_(chan, CHANNEL_STATE_OPEN);
 
   /* Tell circuits if we opened and stuff */
-  if (to_state == CHANNEL_STATE_OPEN) {
-    channel_do_open_actions(chan);
-    chan->has_been_open = 1;
-
-    /* Check for queued cells to process */
-    if (! TOR_SIMPLEQ_EMPTY(&chan->incoming_queue))
-      channel_process_cells(chan);
-    if (! TOR_SIMPLEQ_EMPTY(&chan->outgoing_queue))
-      channel_flush_cells(chan);
-  } else if (to_state == CHANNEL_STATE_CLOSED ||
-             to_state == CHANNEL_STATE_ERROR) {
-    /* Assert that all queues are empty */
-    tor_assert(TOR_SIMPLEQ_EMPTY(&chan->incoming_queue));
-    tor_assert(TOR_SIMPLEQ_EMPTY(&chan->outgoing_queue));
-  }
+  channel_do_open_actions(chan);
+  chan->has_been_open = 1;
 }
 
 /**
- * Change channel listener state
+ * Change channel listener state.
  *
  * This internal and subclass use only function is used to change channel
  * listener state, performing all transition validity checks and whatever
  * actions are appropriate to the state transition in question.
  */
-
 void
 channel_listener_change_state(channel_listener_t *chan_l,
                               channel_listener_state_t to_state)
@@ -2125,9 +1639,9 @@ channel_listener_change_state(channel_listener_t *chan_l,
   if (from_state == to_state) {
     log_debug(LD_CHANNEL,
               "Got no-op transition from \"%s\" to itself on channel "
-              "listener %p (global ID " U64_FORMAT ")",
+              "listener %p (global ID %"PRIu64 ")",
               channel_listener_state_to_string(to_state),
-              chan_l, U64_PRINTF_ARG(chan_l->global_identifier));
+              chan_l, (chan_l->global_identifier));
     return;
   }
 
@@ -2138,19 +1652,10 @@ channel_listener_change_state(channel_listener_t *chan_l,
     tor_assert(chan_l->reason_for_closing != CHANNEL_LISTENER_NOT_CLOSING);
   }
 
-  /*
-   * We need to maintain the queues here for some transitions:
-   * when we enter CHANNEL_STATE_OPEN (especially from CHANNEL_STATE_MAINT)
-   * we may have a backlog of cells to transmit, so drain the queues in
-   * that case, and when going to CHANNEL_STATE_CLOSED the subclass
-   * should have made sure to finish sending things (or gone to
-   * CHANNEL_STATE_ERROR if not possible), so we assert for that here.
-   */
-
   log_debug(LD_CHANNEL,
-            "Changing state of channel listener %p (global ID " U64_FORMAT
+            "Changing state of channel listener %p (global ID %"PRIu64
             "from \"%s\" to \"%s\"",
-            chan_l, U64_PRINTF_ARG(chan_l->global_identifier),
+            chan_l, (chan_l->global_identifier),
             channel_listener_state_to_string(chan_l->state),
             channel_listener_state_to_string(to_state));
 
@@ -2168,6 +1673,7 @@ channel_listener_change_state(channel_listener_t *chan_l,
       if (active_listeners) smartlist_remove(active_listeners, chan_l);
       if (!finished_listeners) finished_listeners = smartlist_new();
       smartlist_add(finished_listeners, chan_l);
+      mainloop_schedule_postloop_cleanup();
     }
     /* Need to put on active list? */
     else if (!was_active && is_active) {
@@ -2179,30 +1685,39 @@ channel_listener_change_state(channel_listener_t *chan_l,
 
   if (to_state == CHANNEL_LISTENER_STATE_CLOSED ||
       to_state == CHANNEL_LISTENER_STATE_ERROR) {
-    /* Assert that the queue is empty */
     tor_assert(!(chan_l->incoming_list) ||
                 smartlist_len(chan_l->incoming_list) == 0);
   }
 }
 
-/**
- * Try to flush cells to the lower layer
- *
- * this is called by the lower layer to indicate that it wants more cells;
- * it will try to write up to num_cells cells from the channel's cell queue or
- * from circuits active on that channel, or as many as it has available if
- * num_cells == -1.
- */
-
+/* Maximum number of cells that is allowed to flush at once within
+ * channel_flush_some_cells(). */
 #define MAX_CELLS_TO_GET_FROM_CIRCUITS_FOR_UNLIMITED 256
 
+/**
+ * Try to flush cells of the given channel chan up to a maximum of num_cells.
+ *
+ * This is called by the scheduler when it wants to flush cells from the
+ * channel's circuit queue(s) to the connection outbuf (not yet on the wire).
+ *
+ * If the channel is not in state CHANNEL_STATE_OPEN, this does nothing and
+ * will return 0 meaning no cells were flushed.
+ *
+ * If num_cells is -1, we'll try to flush up to the maximum cells allowed
+ * defined in MAX_CELLS_TO_GET_FROM_CIRCUITS_FOR_UNLIMITED.
+ *
+ * On success, the number of flushed cells are returned and it can never be
+ * above num_cells. If 0 is returned, no cells were flushed either because the
+ * channel was not opened or we had no cells on the channel. A negative number
+ * can NOT be sent back.
+ *
+ * This function is part of the fast path. */
 MOCK_IMPL(ssize_t,
 channel_flush_some_cells, (channel_t *chan, ssize_t num_cells))
 {
   unsigned int unlimited = 0;
   ssize_t flushed = 0;
-  int num_cells_from_circs, clamped_num_cells;
-  int q_len_before, q_len_after;
+  int clamped_num_cells;
 
   tor_assert(chan);
 
@@ -2211,11 +1726,6 @@ channel_flush_some_cells, (channel_t *chan, ssize_t num_cells))
 
   /* If we aren't in CHANNEL_STATE_OPEN, nothing goes through */
   if (CHANNEL_IS_OPEN(chan)) {
-    /* Try to flush as much as we can that's already queued */
-    flushed += channel_flush_some_cells_from_outgoing_queue(chan,
-        (unlimited ? -1 : num_cells - flushed));
-    if (!unlimited && num_cells <= flushed) goto done;
-
     if (circuitmux_num_cells(chan->cmux) > 0) {
       /* Calculate number of cells, including clamp */
       if (unlimited) {
@@ -2229,45 +1739,9 @@ channel_flush_some_cells, (channel_t *chan, ssize_t num_cells))
         }
       }
 
-      /*
-       * Keep track of the change in queue size; we have to count cells
-       * channel_flush_from_first_active_circuit() writes out directly,
-       * but not double-count ones we might get later in
-       * channel_flush_some_cells_from_outgoing_queue()
-       */
-      q_len_before = chan_cell_queue_len(&(chan->outgoing_queue));
-
       /* Try to get more cells from any active circuits */
-      num_cells_from_circs = channel_flush_from_first_active_circuit(
+      flushed = channel_flush_from_first_active_circuit(
           chan, clamped_num_cells);
-
-      q_len_after = chan_cell_queue_len(&(chan->outgoing_queue));
-
-      /*
-       * If it claims we got some, adjust the flushed counter and consider
-       * processing the queue again
-       */
-      if (num_cells_from_circs > 0) {
-        /*
-         * Adjust flushed by the number of cells counted in
-         * num_cells_from_circs that didn't go to the cell queue.
-         */
-
-        if (q_len_after > q_len_before) {
-          num_cells_from_circs -= (q_len_after - q_len_before);
-          if (num_cells_from_circs < 0) num_cells_from_circs = 0;
-        }
-
-        flushed += num_cells_from_circs;
-
-        /* Now process the queue if necessary */
-
-        if ((q_len_after > q_len_before) &&
-            (unlimited || (flushed < num_cells))) {
-          flushed += channel_flush_some_cells_from_outgoing_queue(chan,
-            (unlimited ? -1 : num_cells - flushed));
-        }
-      }
     }
   }
 
@@ -2276,198 +1750,16 @@ channel_flush_some_cells, (channel_t *chan, ssize_t num_cells))
 }
 
 /**
- * Flush cells from just the channel's outgoing cell queue
- *
- * This gets called from channel_flush_some_cells() above to flush cells
- * just from the queue without trying for active_circuits.
- */
-
-static ssize_t
-channel_flush_some_cells_from_outgoing_queue(channel_t *chan,
-                                             ssize_t num_cells)
-{
-  unsigned int unlimited = 0;
-  ssize_t flushed = 0;
-  cell_queue_entry_t *q = NULL;
-  size_t cell_size;
-  int free_q = 0, handed_off = 0;
-
-  tor_assert(chan);
-  tor_assert(chan->write_cell);
-  tor_assert(chan->write_packed_cell);
-  tor_assert(chan->write_var_cell);
-
-  if (num_cells < 0) unlimited = 1;
-  if (!unlimited && num_cells <= flushed) return 0;
-
-  /* If we aren't in CHANNEL_STATE_OPEN, nothing goes through */
-  if (CHANNEL_IS_OPEN(chan)) {
-    while ((unlimited || num_cells > flushed) &&
-           NULL != (q = TOR_SIMPLEQ_FIRST(&chan->outgoing_queue))) {
-      free_q = 0;
-      handed_off = 0;
-
-      if (1) {
-        /* Figure out how big it is for statistical purposes */
-        cell_size = channel_get_cell_queue_entry_size(chan, q);
-        /*
-         * Okay, we have a good queue entry, try to give it to the lower
-         * layer.
-         */
-        switch (q->type) {
-          case CELL_QUEUE_FIXED:
-            if (q->u.fixed.cell) {
-              if (chan->write_cell(chan,
-                    q->u.fixed.cell)) {
-                ++flushed;
-                channel_timestamp_xmit(chan);
-                ++(chan->n_cells_xmitted);
-                chan->n_bytes_xmitted += cell_size;
-                free_q = 1;
-                handed_off = 1;
-              }
-              /* Else couldn't write it; leave it on the queue */
-            } else {
-              /* This shouldn't happen */
-              log_info(LD_CHANNEL,
-                       "Saw broken cell queue entry of type CELL_QUEUE_FIXED "
-                       "with no cell on channel %p "
-                       "(global ID " U64_FORMAT ").",
-                       chan, U64_PRINTF_ARG(chan->global_identifier));
-              /* Throw it away */
-              free_q = 1;
-              handed_off = 0;
-            }
-            break;
-         case CELL_QUEUE_PACKED:
-            if (q->u.packed.packed_cell) {
-              if (chan->write_packed_cell(chan,
-                    q->u.packed.packed_cell)) {
-                ++flushed;
-                channel_timestamp_xmit(chan);
-                ++(chan->n_cells_xmitted);
-                chan->n_bytes_xmitted += cell_size;
-                free_q = 1;
-                handed_off = 1;
-              }
-              /* Else couldn't write it; leave it on the queue */
-            } else {
-              /* This shouldn't happen */
-              log_info(LD_CHANNEL,
-                       "Saw broken cell queue entry of type CELL_QUEUE_PACKED "
-                       "with no cell on channel %p "
-                       "(global ID " U64_FORMAT ").",
-                       chan, U64_PRINTF_ARG(chan->global_identifier));
-              /* Throw it away */
-              free_q = 1;
-              handed_off = 0;
-            }
-            break;
-         case CELL_QUEUE_VAR:
-            if (q->u.var.var_cell) {
-              if (chan->write_var_cell(chan,
-                    q->u.var.var_cell)) {
-                ++flushed;
-                channel_timestamp_xmit(chan);
-                ++(chan->n_cells_xmitted);
-                chan->n_bytes_xmitted += cell_size;
-                free_q = 1;
-                handed_off = 1;
-              }
-              /* Else couldn't write it; leave it on the queue */
-            } else {
-              /* This shouldn't happen */
-              log_info(LD_CHANNEL,
-                       "Saw broken cell queue entry of type CELL_QUEUE_VAR "
-                       "with no cell on channel %p "
-                       "(global ID " U64_FORMAT ").",
-                       chan, U64_PRINTF_ARG(chan->global_identifier));
-              /* Throw it away */
-              free_q = 1;
-              handed_off = 0;
-            }
-            break;
-          default:
-            /* Unknown type, log and free it */
-            log_info(LD_CHANNEL,
-                     "Saw an unknown cell queue entry type %d on channel %p "
-                     "(global ID " U64_FORMAT "; ignoring it."
-                     "  Someone should fix this.",
-                     q->type, chan, U64_PRINTF_ARG(chan->global_identifier));
-            free_q = 1;
-            handed_off = 0;
-        }
-
-        /*
-         * if free_q is set, we used it and should remove the queue entry;
-         * we have to do the free down here so TOR_SIMPLEQ_REMOVE_HEAD isn't
-         * accessing freed memory
-         */
-        if (free_q) {
-          TOR_SIMPLEQ_REMOVE_HEAD(&chan->outgoing_queue, next);
-          /*
-           * ...and we handed a cell off to the lower layer, so we should
-           * update the counters.
-           */
-          ++n_channel_cells_passed_to_lower_layer;
-          --n_channel_cells_in_queues;
-          n_channel_bytes_passed_to_lower_layer += cell_size;
-          n_channel_bytes_in_queues -= cell_size;
-          channel_assert_counter_consistency();
-          /* Update the channel's queue size too */
-          chan->bytes_in_queue -= cell_size;
-          /* Finally, free q */
-          cell_queue_entry_free(q, handed_off);
-          q = NULL;
-        }
-        /* No cell removed from list, so we can't go on any further */
-        else break;
-      }
-    }
-  }
-
-  /* Did we drain the queue? */
-  if (TOR_SIMPLEQ_EMPTY(&chan->outgoing_queue)) {
-    channel_timestamp_drained(chan);
-  }
-
-  /* Update the estimate queue size */
-  channel_update_xmit_queue_size(chan);
-
-  return flushed;
-}
-
-/**
- * Flush as many cells as we possibly can from the queue
+ * Check if any cells are available.
  *
- * This tries to flush as many cells from the queue as the lower layer
- * will take.  It just calls channel_flush_some_cells_from_outgoing_queue()
- * in unlimited mode.
+ * This is used by the scheduler to know if the channel has more to flush
+ * after a scheduling round.
  */
-
-void
-channel_flush_cells(channel_t *chan)
-{
-  channel_flush_some_cells_from_outgoing_queue(chan, -1);
-}
-
-/**
- * Check if any cells are available
- *
- * This gets used from the lower layer to check if any more cells are
- * available.
- */
-
-int
-channel_more_to_flush(channel_t *chan)
+MOCK_IMPL(int,
+channel_more_to_flush, (channel_t *chan))
 {
   tor_assert(chan);
 
-  /* Check if we have any queued */
-  if (! TOR_SIMPLEQ_EMPTY(&chan->incoming_queue))
-      return 1;
-
-  /* Check if any circuits would like to queue some */
   if (circuitmux_num_cells(chan->cmux) > 0) return 1;
 
   /* Else no */
@@ -2475,12 +1767,11 @@ channel_more_to_flush(channel_t *chan)
 }
 
 /**
- * Notify the channel we're done flushing the output in the lower layer
+ * Notify the channel we're done flushing the output in the lower layer.
  *
  * Connection.c will call this when we've flushed the output; there's some
  * dirreq-related maintenance to do.
  */
-
 void
 channel_notify_flushed(channel_t *chan)
 {
@@ -2493,12 +1784,11 @@ channel_notify_flushed(channel_t *chan)
 }
 
 /**
- * Process the queue of incoming channels on a listener
+ * Process the queue of incoming channels on a listener.
  *
  * Use a listener's registered callback to process as many entries in the
  * queue of incoming channels as possible.
  */
-
 void
 channel_listener_process_incoming(channel_listener_t *listener)
 {
@@ -2514,8 +1804,8 @@ channel_listener_process_incoming(channel_listener_t *listener)
 
   log_debug(LD_CHANNEL,
             "Processing queue of incoming connections for channel "
-            "listener %p (global ID " U64_FORMAT ")",
-            listener, U64_PRINTF_ARG(listener->global_identifier));
+            "listener %p (global ID %"PRIu64 ")",
+            listener, (listener->global_identifier));
 
   if (!(listener->incoming_list)) return;
 
@@ -2524,12 +1814,12 @@ channel_listener_process_incoming(channel_listener_t *listener)
     tor_assert(chan);
 
     log_debug(LD_CHANNEL,
-              "Handling incoming channel %p (" U64_FORMAT ") "
-              "for listener %p (" U64_FORMAT ")",
+              "Handling incoming channel %p (%"PRIu64 ") "
+              "for listener %p (%"PRIu64 ")",
               chan,
-              U64_PRINTF_ARG(chan->global_identifier),
+              (chan->global_identifier),
               listener,
-              U64_PRINTF_ARG(listener->global_identifier));
+              (listener->global_identifier));
     /* Make sure this is set correctly */
     channel_mark_incoming(chan);
     listener->listener(listener, chan);
@@ -2540,18 +1830,17 @@ channel_listener_process_incoming(channel_listener_t *listener)
 }
 
 /**
- * Take actions required when a channel becomes open
+ * Take actions required when a channel becomes open.
  *
  * Handle actions we should do when we know a channel is open; a lot of
  * this comes from the old connection_or_set_state_open() of connection_or.c.
  *
  * Because of this mechanism, future channel_t subclasses should take care
- * not to change a channel to from CHANNEL_STATE_OPENING to CHANNEL_STATE_OPEN
+ * not to change a channel from CHANNEL_STATE_OPENING to CHANNEL_STATE_OPEN
  * until there is positive confirmation that the network is operational.
  * In particular, anything UDP-based should not make this transition until a
  * packet is received from the other side.
  */
-
 void
 channel_do_open_actions(channel_t *chan)
 {
@@ -2566,21 +1855,10 @@ channel_do_open_actions(channel_t *chan)
 
   if (started_here) {
     circuit_build_times_network_is_live(get_circuit_build_times_mutable());
-    rep_hist_note_connect_succeeded(chan->identity_digest, now);
-    if (entry_guard_register_connect_status(
-          chan->identity_digest, 1, 0, now) < 0) {
-      /* Close any circuits pending on this channel. We leave it in state
-       * 'open' though, because it didn't actually *fail* -- we just
-       * chose not to use it. */
-      log_debug(LD_OR,
-                "New entry guard was reachable, but closing this "
-                "connection so we can retry the earlier entry guards.");
-      close_origin_circuits = 1;
-    }
     router_set_status(chan->identity_digest, 1);
   } else {
-    /* only report it to the geoip module if it's not a known router */
-    if (!router_get_by_id_digest(chan->identity_digest)) {
+    /* only report it to the geoip module if it's a client */
+    if (channel_is_client(chan)) {
       if (channel_get_addr_if_possible(chan, &remote_addr)) {
         char *transport_name = NULL;
         channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan);
@@ -2600,17 +1878,35 @@ channel_do_open_actions(channel_t *chan)
     }
   }
 
+  /* Disable or reduce padding according to user prefs. */
+  if (chan->padding_enabled || get_options()->ConnectionPadding == 1) {
+    if (!get_options()->ConnectionPadding) {
+      /* Disable if torrc disabled */
+      channelpadding_disable_padding_on_channel(chan);
+    } else if (rend_service_allow_non_anonymous_connection(get_options()) &&
+               !networkstatus_get_param(NULL,
+                                        CHANNELPADDING_SOS_PARAM,
+                                        CHANNELPADDING_SOS_DEFAULT, 0, 1)) {
+      /* Disable if we're using RSOS and the consensus disabled padding
+       * for RSOS */
+      channelpadding_disable_padding_on_channel(chan);
+    } else if (get_options()->ReducedConnectionPadding) {
+      /* Padding can be forced and/or reduced by clients, regardless of if
+       * the channel supports it */
+      channelpadding_reduce_padding_on_channel(chan);
+    }
+  }
+
   circuit_n_chan_done(chan, 1, close_origin_circuits);
 }
 
 /**
- * Queue an incoming channel on a listener
+ * Queue an incoming channel on a listener.
  *
  * Internal and subclass use only function to queue an incoming channel from
  * a listener.  A subclass of channel_listener_t should call this when a new
  * incoming channel is created.
  */
-
 void
 channel_listener_queue_incoming(channel_listener_t *listener,
                                 channel_t *incoming)
@@ -2622,10 +1918,10 @@ channel_listener_queue_incoming(channel_listener_t *listener,
   tor_assert(incoming);
 
   log_debug(LD_CHANNEL,
-            "Queueing incoming channel %p (global ID " U64_FORMAT ") on "
-            "channel listener %p (global ID " U64_FORMAT ")",
-            incoming, U64_PRINTF_ARG(incoming->global_identifier),
-            listener, U64_PRINTF_ARG(listener->global_identifier));
+            "Queueing incoming channel %p (global ID %"PRIu64 ") on "
+            "channel listener %p (global ID %"PRIu64 ")",
+            incoming, (incoming->global_identifier),
+            listener, (listener->global_identifier));
 
   /* Do we need to queue it, or can we just call the listener right away? */
   if (!(listener->listener)) need_to_queue = 1;
@@ -2660,207 +1956,31 @@ channel_listener_queue_incoming(channel_listener_t *listener,
 }
 
 /**
- * Process queued incoming cells
- *
- * Process as many queued cells as we can from the incoming
- * cell queue.
+ * Process a cell from the given channel.
  */
-
 void
-channel_process_cells(channel_t *chan)
+channel_process_cell(channel_t *chan, cell_t *cell)
 {
-  cell_queue_entry_t *q;
   tor_assert(chan);
   tor_assert(CHANNEL_IS_CLOSING(chan) || CHANNEL_IS_MAINT(chan) ||
              CHANNEL_IS_OPEN(chan));
-
-  log_debug(LD_CHANNEL,
-            "Processing as many incoming cells as we can for channel %p",
-            chan);
-
-  /* Nothing we can do if we have no registered cell handlers */
-  if (!(chan->cell_handler ||
-        chan->var_cell_handler)) return;
-  /* Nothing we can do if we have no cells */
-  if (TOR_SIMPLEQ_EMPTY(&chan->incoming_queue)) return;
-
-  /*
-   * Process cells until we're done or find one we have no current handler
-   * for.
-   *
-   * We must free the cells here after calling the handler, since custody
-   * of the buffer was given to the channel layer when they were queued;
-   * see comments on memory management in channel_queue_cell() and in
-   * channel_queue_var_cell() below.
-   */
-  while (NULL != (q = TOR_SIMPLEQ_FIRST(&chan->incoming_queue))) {
-    tor_assert(q);
-    tor_assert(q->type == CELL_QUEUE_FIXED ||
-               q->type == CELL_QUEUE_VAR);
-
-    if (q->type == CELL_QUEUE_FIXED &&
-        chan->cell_handler) {
-      /* Handle a fixed-length cell */
-      TOR_SIMPLEQ_REMOVE_HEAD(&chan->incoming_queue, next);
-      tor_assert(q->u.fixed.cell);
-      log_debug(LD_CHANNEL,
-                "Processing incoming cell_t %p for channel %p (global ID "
-                U64_FORMAT ")",
-                q->u.fixed.cell, chan,
-                U64_PRINTF_ARG(chan->global_identifier));
-      chan->cell_handler(chan, q->u.fixed.cell);
-      tor_free(q->u.fixed.cell);
-      tor_free(q);
-    } else if (q->type == CELL_QUEUE_VAR &&
-               chan->var_cell_handler) {
-      /* Handle a variable-length cell */
-      TOR_SIMPLEQ_REMOVE_HEAD(&chan->incoming_queue, next);
-      tor_assert(q->u.var.var_cell);
-      log_debug(LD_CHANNEL,
-                "Processing incoming var_cell_t %p for channel %p (global ID "
-                U64_FORMAT ")",
-                q->u.var.var_cell, chan,
-                U64_PRINTF_ARG(chan->global_identifier));
-      chan->var_cell_handler(chan, q->u.var.var_cell);
-      tor_free(q->u.var.var_cell);
-      tor_free(q);
-    } else {
-      /* Can't handle this one */
-      break;
-    }
-  }
-}
-
-/**
- * Queue incoming cell
- *
- * This should be called by a channel_t subclass to queue an incoming fixed-
- * length cell for processing, and process it if possible.
- */
-
-void
-channel_queue_cell(channel_t *chan, cell_t *cell)
-{
-  int need_to_queue = 0;
-  cell_queue_entry_t *q;
-  cell_t *cell_copy = NULL;
-
-  tor_assert(chan);
   tor_assert(cell);
-  tor_assert(CHANNEL_IS_OPEN(chan));
 
-  /* Do we need to queue it, or can we just call the handler right away? */
-  if (!(chan->cell_handler)) need_to_queue = 1;
-  if (! TOR_SIMPLEQ_EMPTY(&chan->incoming_queue))
-    need_to_queue = 1;
+  /* Nothing we can do if we have no registered cell handlers */
+  if (!chan->cell_handler)
+    return;
 
   /* Timestamp for receiving */
   channel_timestamp_recv(chan);
-
-  /* Update the counters */
+  /* Update received counter. */
   ++(chan->n_cells_recved);
   chan->n_bytes_recved += get_cell_network_size(chan->wide_circ_ids);
 
-  /* If we don't need to queue we can just call cell_handler */
-  if (!need_to_queue) {
-    tor_assert(chan->cell_handler);
-    log_debug(LD_CHANNEL,
-              "Directly handling incoming cell_t %p for channel %p "
-              "(global ID " U64_FORMAT ")",
-              cell, chan,
-              U64_PRINTF_ARG(chan->global_identifier));
-    chan->cell_handler(chan, cell);
-  } else {
-    /*
-     * Otherwise queue it and then process the queue if possible.
-     *
-     * We queue a copy, not the original pointer - it might have been on the
-     * stack in connection_or_process_cells_from_inbuf() (or another caller
-     * if we ever have a subclass other than channel_tls_t), or be freed
-     * there after we return.  This is the uncommon case; the non-copying
-     * fast path occurs in the if (!need_to_queue) case above when the
-     * upper layer has installed cell handlers.
-     */
-    cell_copy = tor_malloc_zero(sizeof(cell_t));
-    memcpy(cell_copy, cell, sizeof(cell_t));
-    q = cell_queue_entry_new_fixed(cell_copy);
-    log_debug(LD_CHANNEL,
-              "Queueing incoming cell_t %p for channel %p "
-              "(global ID " U64_FORMAT ")",
-              cell, chan,
-              U64_PRINTF_ARG(chan->global_identifier));
-    TOR_SIMPLEQ_INSERT_TAIL(&chan->incoming_queue, q, next);
-    if (chan->cell_handler ||
-        chan->var_cell_handler) {
-      channel_process_cells(chan);
-    }
-  }
-}
-
-/**
- * Queue incoming variable-length cell
- *
- * This should be called by a channel_t subclass to queue an incoming
- * variable-length cell for processing, and process it if possible.
- */
-
-void
-channel_queue_var_cell(channel_t *chan, var_cell_t *var_cell)
-{
-  int need_to_queue = 0;
-  cell_queue_entry_t *q;
-  var_cell_t *cell_copy = NULL;
-
-  tor_assert(chan);
-  tor_assert(var_cell);
-  tor_assert(CHANNEL_IS_OPEN(chan));
-
-  /* Do we need to queue it, or can we just call the handler right away? */
-  if (!(chan->var_cell_handler)) need_to_queue = 1;
-  if (! TOR_SIMPLEQ_EMPTY(&chan->incoming_queue))
-    need_to_queue = 1;
-
-  /* Timestamp for receiving */
-  channel_timestamp_recv(chan);
-
-  /* Update the counter */
-  ++(chan->n_cells_recved);
-  chan->n_bytes_recved += get_var_cell_header_size(chan->wide_circ_ids) +
-    var_cell->payload_len;
-
-  /* If we don't need to queue we can just call cell_handler */
-  if (!need_to_queue) {
-    tor_assert(chan->var_cell_handler);
-    log_debug(LD_CHANNEL,
-              "Directly handling incoming var_cell_t %p for channel %p "
-              "(global ID " U64_FORMAT ")",
-              var_cell, chan,
-              U64_PRINTF_ARG(chan->global_identifier));
-    chan->var_cell_handler(chan, var_cell);
-  } else {
-    /*
-     * Otherwise queue it and then process the queue if possible.
-     *
-     * We queue a copy, not the original pointer - it might have been on the
-     * stack in connection_or_process_cells_from_inbuf() (or another caller
-     * if we ever have a subclass other than channel_tls_t), or be freed
-     * there after we return.  This is the uncommon case; the non-copying
-     * fast path occurs in the if (!need_to_queue) case above when the
-     * upper layer has installed cell handlers.
-     */
-    cell_copy = var_cell_copy(var_cell);
-    q = cell_queue_entry_new_var(cell_copy);
-    log_debug(LD_CHANNEL,
-              "Queueing incoming var_cell_t %p for channel %p "
-              "(global ID " U64_FORMAT ")",
-              var_cell, chan,
-              U64_PRINTF_ARG(chan->global_identifier));
-    TOR_SIMPLEQ_INSERT_TAIL(&chan->incoming_queue, q, next);
-    if (chan->cell_handler ||
-        chan->var_cell_handler) {
-      channel_process_cells(chan);
-    }
-  }
+  log_debug(LD_CHANNEL,
+            "Processing incoming cell_t %p for channel %p (global ID "
+            "%"PRIu64 ")", cell, chan,
+            (chan->global_identifier));
+  chan->cell_handler(chan, cell);
 }
 
 /** If <b>packed_cell</b> on <b>chan</b> is a destroy cell, then set
@@ -2887,59 +2007,20 @@ packed_cell_is_destroy(channel_t *chan,
 }
 
 /**
- * Assert that the global channel stats counters are internally consistent
- */
-
-static void
-channel_assert_counter_consistency(void)
-{
-  tor_assert(n_channel_cells_queued ==
-      (n_channel_cells_in_queues + n_channel_cells_passed_to_lower_layer));
-  tor_assert(n_channel_bytes_queued ==
-      (n_channel_bytes_in_queues + n_channel_bytes_passed_to_lower_layer));
-}
-
-/* DOCDOC */
-static int
-is_destroy_cell(channel_t *chan,
-                const cell_queue_entry_t *q, circid_t *circid_out)
-{
-  *circid_out = 0;
-  switch (q->type) {
-    case CELL_QUEUE_FIXED:
-      if (q->u.fixed.cell->command == CELL_DESTROY) {
-        *circid_out = q->u.fixed.cell->circ_id;
-        return 1;
-      }
-      break;
-    case CELL_QUEUE_VAR:
-      if (q->u.var.var_cell->command == CELL_DESTROY) {
-        *circid_out = q->u.var.var_cell->circ_id;
-        return 1;
-      }
-      break;
-    case CELL_QUEUE_PACKED:
-      return packed_cell_is_destroy(chan, q->u.packed.packed_cell, circid_out);
-  }
-  return 0;
-}
-
-/**
- * Send destroy cell on a channel
+ * Send destroy cell on a channel.
  *
  * Write a destroy cell with circ ID <b>circ_id</b> and reason <b>reason</b>
  * onto channel <b>chan</b>.  Don't perform range-checking on reason:
  * we may want to propagate reasons from other cells.
  */
-
 int
 channel_send_destroy(circid_t circ_id, channel_t *chan, int reason)
 {
   tor_assert(chan);
   if (circ_id == 0) {
     log_warn(LD_BUG, "Attempted to send a destroy cell for circID 0 "
-             "on a channel " U64_FORMAT " at %p in state %s (%d)",
-             U64_PRINTF_ARG(chan->global_identifier),
+             "on a channel %"PRIu64 " at %p in state %s (%d)",
+             (chan->global_identifier),
              chan, channel_state_to_string(chan->state),
              chan->state);
     return 0;
@@ -2951,14 +2032,14 @@ channel_send_destroy(circid_t circ_id, channel_t *chan, int reason)
     circuitmux_append_destroy_cell(chan, chan->cmux, circ_id, reason);
     log_debug(LD_OR,
               "Sending destroy (circID %u) on channel %p "
-              "(global ID " U64_FORMAT ")",
+              "(global ID %"PRIu64 ")",
               (unsigned)circ_id, chan,
-              U64_PRINTF_ARG(chan->global_identifier));
+              (chan->global_identifier));
   } else {
     log_warn(LD_BUG,
              "Someone called channel_send_destroy() for circID %u "
-             "on a channel " U64_FORMAT " at %p in state %s (%d)",
-             (unsigned)circ_id, U64_PRINTF_ARG(chan->global_identifier),
+             "on a channel %"PRIu64 " at %p in state %s (%d)",
+             (unsigned)circ_id, (chan->global_identifier),
              chan, channel_state_to_string(chan->state),
              chan->state);
   }
@@ -2967,30 +2048,16 @@ channel_send_destroy(circid_t circ_id, channel_t *chan, int reason)
 }
 
 /**
- * Dump channel statistics to the log
+ * Dump channel statistics to the log.
  *
  * This is called from dumpstats() in main.c and spams the log with
  * statistics on channels.
  */
-
 void
 channel_dumpstats(int severity)
 {
   if (all_channels && smartlist_len(all_channels) > 0) {
     tor_log(severity, LD_GENERAL,
-        "Channels have queued " U64_FORMAT " bytes in " U64_FORMAT " cells, "
-        "and handed " U64_FORMAT " bytes in " U64_FORMAT " cells to the lower"
-        " layer.",
-        U64_PRINTF_ARG(n_channel_bytes_queued),
-        U64_PRINTF_ARG(n_channel_cells_queued),
-        U64_PRINTF_ARG(n_channel_bytes_passed_to_lower_layer),
-        U64_PRINTF_ARG(n_channel_cells_passed_to_lower_layer));
-    tor_log(severity, LD_GENERAL,
-        "There are currently " U64_FORMAT " bytes in " U64_FORMAT " cells "
-        "in channel queues.",
-        U64_PRINTF_ARG(n_channel_bytes_in_queues),
-        U64_PRINTF_ARG(n_channel_cells_in_queues));
-    tor_log(severity, LD_GENERAL,
         "Dumping statistics about %d channels:",
         smartlist_len(all_channels));
     tor_log(severity, LD_GENERAL,
@@ -3012,12 +2079,11 @@ channel_dumpstats(int severity)
 }
 
 /**
- * Dump channel listener statistics to the log
+ * Dump channel listener statistics to the log.
  *
  * This is called from dumpstats() in main.c and spams the log with
  * statistics on channel listeners.
  */
-
 void
 channel_listener_dumpstats(int severity)
 {
@@ -3044,28 +2110,11 @@ channel_listener_dumpstats(int severity)
 }
 
 /**
- * Set the cmux policy on all active channels
- */
-
-void
-channel_set_cmux_policy_everywhere(circuitmux_policy_t *pol)
-{
-  if (!active_channels) return;
-
-  SMARTLIST_FOREACH_BEGIN(active_channels, channel_t *, curr) {
-    if (curr->cmux) {
-      circuitmux_set_policy(curr->cmux, pol);
-    }
-  } SMARTLIST_FOREACH_END(curr);
-}
-
-/**
- * Clean up channels
+ * Clean up channels.
  *
  * This gets called periodically from run_scheduled_events() in main.c;
  * it cleans up after closed channels.
  */
-
 void
 channel_run_cleanup(void)
 {
@@ -3087,12 +2136,11 @@ channel_run_cleanup(void)
 }
 
 /**
- * Clean up channel listeners
+ * Clean up channel listeners.
  *
  * This gets called periodically from run_scheduled_events() in main.c;
  * it cleans up after closed channel listeners.
  */
-
 void
 channel_listener_run_cleanup(void)
 {
@@ -3114,9 +2162,8 @@ channel_listener_run_cleanup(void)
 }
 
 /**
- * Free a list of channels for channel_free_all()
+ * Free a list of channels for channel_free_all().
  */
-
 static void
 channel_free_list(smartlist_t *channels, int mark_for_close)
 {
@@ -3126,9 +2173,9 @@ channel_free_list(smartlist_t *channels, int mark_for_close)
     /* Deregister and free it */
     tor_assert(curr);
     log_debug(LD_CHANNEL,
-              "Cleaning up channel %p (global ID " U64_FORMAT ") "
+              "Cleaning up channel %p (global ID %"PRIu64 ") "
               "in state %s (%d)",
-              curr, U64_PRINTF_ARG(curr->global_identifier),
+              curr, (curr->global_identifier),
               channel_state_to_string(curr->state), curr->state);
     /* Detach circuits early so they can find the channel */
     if (curr->cmux) {
@@ -3140,15 +2187,14 @@ channel_free_list(smartlist_t *channels, int mark_for_close)
       if (!CHANNEL_CONDEMNED(curr)) {
         channel_mark_for_close(curr);
       }
-      channel_force_free(curr);
+      channel_force_xfree(curr);
     } else channel_free(curr);
   } SMARTLIST_FOREACH_END(curr);
 }
 
 /**
- * Free a list of channel listeners for channel_free_all()
+ * Free a list of channel listeners for channel_free_all().
  */
-
 static void
 channel_listener_free_list(smartlist_t *listeners, int mark_for_close)
 {
@@ -3158,9 +2204,9 @@ channel_listener_free_list(smartlist_t *listeners, int mark_for_close)
     /* Deregister and free it */
     tor_assert(curr);
     log_debug(LD_CHANNEL,
-              "Cleaning up channel listener %p (global ID " U64_FORMAT ") "
+              "Cleaning up channel listener %p (global ID %"PRIu64 ") "
               "in state %s (%d)",
-              curr, U64_PRINTF_ARG(curr->global_identifier),
+              curr, (curr->global_identifier),
               channel_listener_state_to_string(curr->state), curr->state);
     channel_listener_unregister(curr);
     if (mark_for_close) {
@@ -3169,20 +2215,19 @@ channel_listener_free_list(smartlist_t *listeners, int mark_for_close)
             curr->state == CHANNEL_LISTENER_STATE_ERROR)) {
         channel_listener_mark_for_close(curr);
       }
-      channel_listener_force_free(curr);
+      channel_listener_force_xfree(curr);
     } else channel_listener_free(curr);
   } SMARTLIST_FOREACH_END(curr);
 }
 
 /**
- * Close all channels and free everything
+ * Close all channels and free everything.
  *
  * This gets called from tor_free_all() in main.c to clean up on exit.
  * It will close all registered channels and free associated storage,
  * then free the all_channels, active_channels, listening_channels and
  * finished_channels lists and also channel_identity_map.
  */
-
 void
 channel_free_all(void)
 {
@@ -3237,12 +2282,17 @@ channel_free_all(void)
   /* Geez, anything still left over just won't die ... let it leak then */
   HT_CLEAR(channel_idmap, &channel_identity_map);
 
+  /* Same with channel_gid_map */
+  log_debug(LD_CHANNEL,
+            "Freeing channel_gid_map");
+  HT_CLEAR(channel_gid_map, &channel_gid_map);
+
   log_debug(LD_CHANNEL,
             "Done cleaning up after channels");
 }
 
 /**
- * Connect to a given addr/port/digest
+ * Connect to a given addr/port/digest.
  *
  * This sets up a new outgoing channel; in the future if multiple
  * channel_t subclasses are available, this is where the selection policy
@@ -3251,16 +2301,16 @@ channel_free_all(void)
  * single abstract object encapsulating all the protocol details of
  * how to contact an OR.
  */
-
 channel_t *
 channel_connect(const tor_addr_t *addr, uint16_t port,
-                const char *id_digest)
+                const char *id_digest,
+                const ed25519_public_key_t *ed_id)
 {
-  return channel_tls_connect(addr, port, id_digest);
+  return channel_tls_connect(addr, port, id_digest, ed_id);
 }
 
 /**
- * Decide which of two channels to prefer for extending a circuit
+ * Decide which of two channels to prefer for extending a circuit.
  *
  * This function is called while extending a circuit and returns true iff
  * a is 'better' than b.  The most important criterion here is that a
@@ -3269,24 +2319,21 @@ channel_connect(const tor_addr_t *addr, uint16_t port,
  *
  * This is based on the former connection_or_is_better() of connection_or.c
  */
-
 int
-channel_is_better(time_t now, channel_t *a, channel_t *b,
-                  int forgive_new_connections)
+channel_is_better(channel_t *a, channel_t *b)
 {
-  int a_grace, b_grace;
   int a_is_canonical, b_is_canonical;
-  int a_has_circs, b_has_circs;
-
-  /*
-   * Do not definitively deprecate a new channel with no circuits on it
-   * until this much time has passed.
-   */
-#define NEW_CHAN_GRACE_PERIOD (15*60)
 
   tor_assert(a);
   tor_assert(b);
 
+  /* If one channel is bad for new circuits, and the other isn't,
+   * use the one that is still good. */
+  if (!channel_is_bad_for_new_circs(a) && channel_is_bad_for_new_circs(b))
+    return 1;
+  if (channel_is_bad_for_new_circs(a) && !channel_is_bad_for_new_circs(b))
+    return 0;
+
   /* Check if one is canonical and the other isn't first */
   a_is_canonical = channel_is_canonical(a);
   b_is_canonical = channel_is_canonical(b);
@@ -3294,31 +2341,36 @@ channel_is_better(time_t now, channel_t *a, channel_t *b,
   if (a_is_canonical && !b_is_canonical) return 1;
   if (!a_is_canonical && b_is_canonical) return 0;
 
+  /* Check if we suspect that one of the channels will be preferred
+   * by the peer */
+  if (a->is_canonical_to_peer && !b->is_canonical_to_peer) return 1;
+  if (!a->is_canonical_to_peer && b->is_canonical_to_peer) return 0;
+
   /*
-   * Okay, if we're here they tied on canonicity. Next we check if
-   * they have any circuits, and if one does and the other doesn't,
-   * we prefer the one that does, unless we are forgiving and the
-   * one that has no circuits is in its grace period.
+   * Okay, if we're here they tied on canonicity, the prefer the older
+   * connection, so that the adversary can't create a new connection
+   * and try to switch us over to it (which will leak information
+   * about long-lived circuits). Additionally, switching connections
+   * too often makes us more vulnerable to attacks like Torscan and
+   * passive netflow-based equivalents.
+   *
+   * Connections will still only live for at most a week, due to
+   * the check in connection_or_group_set_badness() against
+   * TIME_BEFORE_OR_CONN_IS_TOO_OLD, which marks old connections as
+   * unusable for new circuits after 1 week. That check sets
+   * is_bad_for_new_circs, which is checked in channel_get_for_extend().
+   *
+   * We check channel_is_bad_for_new_circs() above here anyway, for safety.
    */
+  if (channel_when_created(a) < channel_when_created(b)) return 1;
+  else if (channel_when_created(a) > channel_when_created(b)) return 0;
 
-  a_has_circs = (channel_num_circuits(a) > 0);
-  b_has_circs = (channel_num_circuits(b) > 0);
-  a_grace = (forgive_new_connections &&
-             (now < channel_when_created(a) + NEW_CHAN_GRACE_PERIOD));
-  b_grace = (forgive_new_connections &&
-             (now < channel_when_created(b) + NEW_CHAN_GRACE_PERIOD));
-
-  if (a_has_circs && !b_has_circs && !b_grace) return 1;
-  if (!a_has_circs && b_has_circs && !a_grace) return 0;
-
-  /* They tied on circuits too; just prefer whichever is newer */
-
-  if (channel_when_created(a) > channel_when_created(b)) return 1;
+  if (channel_num_circuits(a) > channel_num_circuits(b)) return 1;
   else return 0;
 }
 
 /**
- * Get a channel to extend a circuit
+ * Get a channel to extend a circuit.
  *
  * Pick a suitable channel to extend a circuit to given the desired digest
  * the address we believe is correct for that digest; this tries to see
@@ -3327,29 +2379,28 @@ channel_is_better(time_t now, channel_t *a, channel_t *b,
  * and our next action, and set *launch_out to a boolean indicated whether
  * the caller should try to launch a new channel with channel_connect().
  */
-
 channel_t *
-channel_get_for_extend(const char *digest,
+channel_get_for_extend(const char *rsa_id_digest,
+                       const ed25519_public_key_t *ed_id,
                        const tor_addr_t *target_addr,
                        const char **msg_out,
                        int *launch_out)
 {
   channel_t *chan, *best = NULL;
   int n_inprogress_goodaddr = 0, n_old = 0;
-  int n_noncanonical = 0, n_possible = 0;
-  time_t now = approx_time();
+  int n_noncanonical = 0;
 
   tor_assert(msg_out);
   tor_assert(launch_out);
 
-  chan = channel_find_by_remote_digest(digest);
+  chan = channel_find_by_remote_identity(rsa_id_digest, ed_id);
 
   /* Walk the list, unrefing the old one and refing the new at each
    * iteration.
    */
-  for (; chan; chan = channel_next_with_digest(chan)) {
+  for (; chan; chan = channel_next_with_rsa_identity(chan)) {
     tor_assert(tor_memeq(chan->identity_digest,
-                         digest, DIGEST_LEN));
+                         rsa_id_digest, DIGEST_LEN));
 
    if (CHANNEL_CONDEMNED(chan))
       continue;
@@ -3360,6 +2411,11 @@ channel_get_for_extend(const char *digest,
       continue;
     }
 
+    /* The Ed25519 key has to match too */
+    if (!channel_remote_identity_matches(chan, rsa_id_digest, ed_id)) {
+      continue;
+    }
+
     /* Never return a non-open connection. */
     if (!CHANNEL_IS_OPEN(chan)) {
       /* If the address matches, don't launch a new connection for this
@@ -3395,14 +2451,12 @@ channel_get_for_extend(const char *digest,
       continue;
     }
 
-    ++n_possible;
-
     if (!best) {
       best = chan; /* If we have no 'best' so far, this one is good enough. */
       continue;
     }
 
-    if (channel_is_better(now, chan, best, 0))
+    if (channel_is_better(chan, best))
       best = chan;
   }
 
@@ -3427,12 +2481,11 @@ channel_get_for_extend(const char *digest,
 }
 
 /**
- * Describe the transport subclass for a channel
+ * Describe the transport subclass for a channel.
  *
  * Invoke a method to get a string description of the lower-layer
  * transport for this channel.
  */
-
 const char *
 channel_describe_transport(channel_t *chan)
 {
@@ -3443,12 +2496,11 @@ channel_describe_transport(channel_t *chan)
 }
 
 /**
- * Describe the transport subclass for a channel listener
+ * Describe the transport subclass for a channel listener.
  *
  * Invoke a method to get a string description of the lower-layer
  * transport for this channel listener.
  */
-
 const char *
 channel_listener_describe_transport(channel_listener_t *chan_l)
 {
@@ -3459,24 +2511,10 @@ channel_listener_describe_transport(channel_listener_t *chan_l)
 }
 
 /**
- * Return the number of entries in <b>queue</b>
- */
-STATIC int
-chan_cell_queue_len(const chan_cell_queue_t *queue)
-{
-  int r = 0;
-  cell_queue_entry_t *cell;
-  TOR_SIMPLEQ_FOREACH(cell, queue, next)
-    ++r;
-  return r;
-}
-
-/**
- * Dump channel statistics
+ * Dump channel statistics.
  *
- * Dump statistics for one channel to the log
+ * Dump statistics for one channel to the log.
  */
-
 MOCK_IMPL(void,
 channel_dump_statistics, (channel_t *chan, int severity))
 {
@@ -3491,50 +2529,33 @@ channel_dump_statistics, (channel_t *chan, int severity))
   age = (double)(now - chan->timestamp_created);
 
   tor_log(severity, LD_GENERAL,
-      "Channel " U64_FORMAT " (at %p) with transport %s is in state "
+      "Channel %"PRIu64 " (at %p) with transport %s is in state "
       "%s (%d)",
-      U64_PRINTF_ARG(chan->global_identifier), chan,
+      (chan->global_identifier), chan,
       channel_describe_transport(chan),
       channel_state_to_string(chan->state), chan->state);
   tor_log(severity, LD_GENERAL,
-      " * Channel " U64_FORMAT " was created at " U64_FORMAT
-      " (" U64_FORMAT " seconds ago) "
-      "and last active at " U64_FORMAT " (" U64_FORMAT " seconds ago)",
-      U64_PRINTF_ARG(chan->global_identifier),
-      U64_PRINTF_ARG(chan->timestamp_created),
-      U64_PRINTF_ARG(now - chan->timestamp_created),
-      U64_PRINTF_ARG(chan->timestamp_active),
-      U64_PRINTF_ARG(now - chan->timestamp_active));
-
-  /* Handle digest and nickname */
+      " * Channel %"PRIu64 " was created at %"PRIu64
+      " (%"PRIu64 " seconds ago) "
+      "and last active at %"PRIu64 " (%"PRIu64 " seconds ago)",
+      (chan->global_identifier),
+      (uint64_t)(chan->timestamp_created),
+      (uint64_t)(now - chan->timestamp_created),
+      (uint64_t)(chan->timestamp_active),
+      (uint64_t)(now - chan->timestamp_active));
+
+  /* Handle digest. */
   if (!tor_digest_is_zero(chan->identity_digest)) {
-    if (chan->nickname) {
-      tor_log(severity, LD_GENERAL,
-          " * Channel " U64_FORMAT " says it is connected "
-          "to an OR with digest %s and nickname %s",
-          U64_PRINTF_ARG(chan->global_identifier),
-          hex_str(chan->identity_digest, DIGEST_LEN),
-          chan->nickname);
-    } else {
-      tor_log(severity, LD_GENERAL,
-          " * Channel " U64_FORMAT " says it is connected "
-          "to an OR with digest %s and no known nickname",
-          U64_PRINTF_ARG(chan->global_identifier),
-          hex_str(chan->identity_digest, DIGEST_LEN));
-    }
+    tor_log(severity, LD_GENERAL,
+        " * Channel %"PRIu64 " says it is connected "
+        "to an OR with digest %s",
+        (chan->global_identifier),
+        hex_str(chan->identity_digest, DIGEST_LEN));
   } else {
-    if (chan->nickname) {
-      tor_log(severity, LD_GENERAL,
-          " * Channel " U64_FORMAT " does not know the digest"
-          " of the OR it is connected to, but reports its nickname is %s",
-          U64_PRINTF_ARG(chan->global_identifier),
-          chan->nickname);
-    } else {
-      tor_log(severity, LD_GENERAL,
-          " * Channel " U64_FORMAT " does not know the digest"
-          " or the nickname of the OR it is connected to",
-          U64_PRINTF_ARG(chan->global_identifier));
-    }
+    tor_log(severity, LD_GENERAL,
+        " * Channel %"PRIu64 " does not know the digest"
+        " of the OR it is connected to",
+        (chan->global_identifier));
   }
 
   /* Handle remote address and descriptions */
@@ -3543,10 +2564,10 @@ channel_dump_statistics, (channel_t *chan, int severity))
     char *actual = tor_strdup(channel_get_actual_remote_descr(chan));
     remote_addr_str = tor_addr_to_str_dup(&remote_addr);
     tor_log(severity, LD_GENERAL,
-        " * Channel " U64_FORMAT " says its remote address"
+        " * Channel %"PRIu64 " says its remote address"
         " is %s, and gives a canonical description of \"%s\" and an "
         "actual description of \"%s\"",
-        U64_PRINTF_ARG(chan->global_identifier),
+        (chan->global_identifier),
         safe_str(remote_addr_str),
         safe_str(channel_get_canonical_remote_descr(chan)),
         safe_str(actual));
@@ -3555,10 +2576,10 @@ channel_dump_statistics, (channel_t *chan, int severity))
   } else {
     char *actual = tor_strdup(channel_get_actual_remote_descr(chan));
     tor_log(severity, LD_GENERAL,
-        " * Channel " U64_FORMAT " does not know its remote "
+        " * Channel %"PRIu64 " does not know its remote "
         "address, but gives a canonical description of \"%s\" and an "
         "actual description of \"%s\"",
-        U64_PRINTF_ARG(chan->global_identifier),
+        (chan->global_identifier),
         channel_get_canonical_remote_descr(chan),
         actual);
     tor_free(actual);
@@ -3566,9 +2587,9 @@ channel_dump_statistics, (channel_t *chan, int severity))
 
   /* Handle marks */
   tor_log(severity, LD_GENERAL,
-      " * Channel " U64_FORMAT " has these marks: %s %s %s "
+      " * Channel %"PRIu64 " has these marks: %s %s %s "
       "%s %s %s",
-      U64_PRINTF_ARG(chan->global_identifier),
+      (chan->global_identifier),
       channel_is_bad_for_new_circs(chan) ?
         "bad_for_new_circs" : "!bad_for_new_circs",
       channel_is_canonical(chan) ?
@@ -3583,19 +2604,11 @@ channel_dump_statistics, (channel_t *chan, int severity))
       channel_is_incoming(chan) ?
         "incoming" : "outgoing");
 
-  /* Describe queues */
-  tor_log(severity, LD_GENERAL,
-      " * Channel " U64_FORMAT " has %d queued incoming cells"
-      " and %d queued outgoing cells",
-      U64_PRINTF_ARG(chan->global_identifier),
-      chan_cell_queue_len(&chan->incoming_queue),
-      chan_cell_queue_len(&chan->outgoing_queue));
-
   /* Describe circuits */
   tor_log(severity, LD_GENERAL,
-      " * Channel " U64_FORMAT " has %d active circuits out of"
+      " * Channel %"PRIu64 " has %d active circuits out of"
       " %d in total",
-      U64_PRINTF_ARG(chan->global_identifier),
+      (chan->global_identifier),
       (chan->cmux != NULL) ?
          circuitmux_num_active_circuits(chan->cmux) : 0,
       (chan->cmux != NULL) ?
@@ -3603,84 +2616,78 @@ channel_dump_statistics, (channel_t *chan, int severity))
 
   /* Describe timestamps */
   tor_log(severity, LD_GENERAL,
-      " * Channel " U64_FORMAT " was last used by a "
-      "client at " U64_FORMAT " (" U64_FORMAT " seconds ago)",
-      U64_PRINTF_ARG(chan->global_identifier),
-      U64_PRINTF_ARG(chan->timestamp_client),
-      U64_PRINTF_ARG(now - chan->timestamp_client));
+      " * Channel %"PRIu64 " was last used by a "
+      "client at %"PRIu64 " (%"PRIu64 " seconds ago)",
+      (chan->global_identifier),
+      (uint64_t)(chan->timestamp_client),
+      (uint64_t)(now - chan->timestamp_client));
   tor_log(severity, LD_GENERAL,
-      " * Channel " U64_FORMAT " was last drained at "
-      U64_FORMAT " (" U64_FORMAT " seconds ago)",
-      U64_PRINTF_ARG(chan->global_identifier),
-      U64_PRINTF_ARG(chan->timestamp_drained),
-      U64_PRINTF_ARG(now - chan->timestamp_drained));
+      " * Channel %"PRIu64 " last received a cell "
+      "at %"PRIu64 " (%"PRIu64 " seconds ago)",
+      (chan->global_identifier),
+      (uint64_t)(chan->timestamp_recv),
+      (uint64_t)(now - chan->timestamp_recv));
   tor_log(severity, LD_GENERAL,
-      " * Channel " U64_FORMAT " last received a cell "
-      "at " U64_FORMAT " (" U64_FORMAT " seconds ago)",
-      U64_PRINTF_ARG(chan->global_identifier),
-      U64_PRINTF_ARG(chan->timestamp_recv),
-      U64_PRINTF_ARG(now - chan->timestamp_recv));
-  tor_log(severity, LD_GENERAL,
-      " * Channel " U64_FORMAT " last transmitted a cell "
-      "at " U64_FORMAT " (" U64_FORMAT " seconds ago)",
-      U64_PRINTF_ARG(chan->global_identifier),
-      U64_PRINTF_ARG(chan->timestamp_xmit),
-      U64_PRINTF_ARG(now - chan->timestamp_xmit));
+      " * Channel %"PRIu64 " last transmitted a cell "
+      "at %"PRIu64 " (%"PRIu64 " seconds ago)",
+      (chan->global_identifier),
+      (uint64_t)(chan->timestamp_xmit),
+      (uint64_t)(now - chan->timestamp_xmit));
 
   /* Describe counters and rates */
   tor_log(severity, LD_GENERAL,
-      " * Channel " U64_FORMAT " has received "
-      U64_FORMAT " bytes in " U64_FORMAT " cells and transmitted "
-      U64_FORMAT " bytes in " U64_FORMAT " cells",
-      U64_PRINTF_ARG(chan->global_identifier),
-      U64_PRINTF_ARG(chan->n_bytes_recved),
-      U64_PRINTF_ARG(chan->n_cells_recved),
-      U64_PRINTF_ARG(chan->n_bytes_xmitted),
-      U64_PRINTF_ARG(chan->n_cells_xmitted));
+      " * Channel %"PRIu64 " has received "
+      "%"PRIu64 " bytes in %"PRIu64 " cells and transmitted "
+      "%"PRIu64 " bytes in %"PRIu64 " cells",
+      (chan->global_identifier),
+      (chan->n_bytes_recved),
+      (chan->n_cells_recved),
+      (chan->n_bytes_xmitted),
+      (chan->n_cells_xmitted));
   if (now > chan->timestamp_created &&
       chan->timestamp_created > 0) {
     if (chan->n_bytes_recved > 0) {
       avg = (double)(chan->n_bytes_recved) / age;
       tor_log(severity, LD_GENERAL,
-          " * Channel " U64_FORMAT " has averaged %f "
+          " * Channel %"PRIu64 " has averaged %f "
           "bytes received per second",
-          U64_PRINTF_ARG(chan->global_identifier), avg);
+          (chan->global_identifier), avg);
     }
     if (chan->n_cells_recved > 0) {
       avg = (double)(chan->n_cells_recved) / age;
       if (avg >= 1.0) {
         tor_log(severity, LD_GENERAL,
-            " * Channel " U64_FORMAT " has averaged %f "
+            " * Channel %"PRIu64 " has averaged %f "
             "cells received per second",
-            U64_PRINTF_ARG(chan->global_identifier), avg);
+            (chan->global_identifier), avg);
       } else if (avg >= 0.0) {
         interval = 1.0 / avg;
         tor_log(severity, LD_GENERAL,
-            " * Channel " U64_FORMAT " has averaged %f "
+            " * Channel %"PRIu64 " has averaged %f "
             "seconds between received cells",
-            U64_PRINTF_ARG(chan->global_identifier), interval);
+            (chan->global_identifier), interval);
       }
     }
     if (chan->n_bytes_xmitted > 0) {
       avg = (double)(chan->n_bytes_xmitted) / age;
       tor_log(severity, LD_GENERAL,
-          " * Channel " U64_FORMAT " has averaged %f "
+          " * Channel %"PRIu64 " has averaged %f "
           "bytes transmitted per second",
-          U64_PRINTF_ARG(chan->global_identifier), avg);
+          (chan->global_identifier), avg);
     }
     if (chan->n_cells_xmitted > 0) {
       avg = (double)(chan->n_cells_xmitted) / age;
       if (avg >= 1.0) {
         tor_log(severity, LD_GENERAL,
-            " * Channel " U64_FORMAT " has averaged %f "
+            " * Channel %"PRIu64 " has averaged %f "
             "cells transmitted per second",
-            U64_PRINTF_ARG(chan->global_identifier), avg);
+            (chan->global_identifier), avg);
       } else if (avg >= 0.0) {
         interval = 1.0 / avg;
         tor_log(severity, LD_GENERAL,
-            " * Channel " U64_FORMAT " has averaged %f "
+            " * Channel %"PRIu64 " has averaged %f "
             "seconds between transmitted cells",
-            U64_PRINTF_ARG(chan->global_identifier), interval);
+            (chan->global_identifier), interval);
       }
     }
   }
@@ -3690,11 +2697,10 @@ channel_dump_statistics, (channel_t *chan, int severity))
 }
 
 /**
- * Dump channel listener statistics
+ * Dump channel listener statistics.
  *
- * Dump statistics for one channel listener to the log
+ * Dump statistics for one channel listener to the log.
  */
-
 void
 channel_listener_dump_statistics(channel_listener_t *chan_l, int severity)
 {
@@ -3706,29 +2712,29 @@ channel_listener_dump_statistics(channel_listener_t *chan_l, int severity)
   age = (double)(now - chan_l->timestamp_created);
 
   tor_log(severity, LD_GENERAL,
-      "Channel listener " U64_FORMAT " (at %p) with transport %s is in "
+      "Channel listener %"PRIu64 " (at %p) with transport %s is in "
       "state %s (%d)",
-      U64_PRINTF_ARG(chan_l->global_identifier), chan_l,
+      (chan_l->global_identifier), chan_l,
       channel_listener_describe_transport(chan_l),
       channel_listener_state_to_string(chan_l->state), chan_l->state);
   tor_log(severity, LD_GENERAL,
-      " * Channel listener " U64_FORMAT " was created at " U64_FORMAT
-      " (" U64_FORMAT " seconds ago) "
-      "and last active at " U64_FORMAT " (" U64_FORMAT " seconds ago)",
-      U64_PRINTF_ARG(chan_l->global_identifier),
-      U64_PRINTF_ARG(chan_l->timestamp_created),
-      U64_PRINTF_ARG(now - chan_l->timestamp_created),
-      U64_PRINTF_ARG(chan_l->timestamp_active),
-      U64_PRINTF_ARG(now - chan_l->timestamp_active));
+      " * Channel listener %"PRIu64 " was created at %"PRIu64
+      " (%"PRIu64 " seconds ago) "
+      "and last active at %"PRIu64 " (%"PRIu64 " seconds ago)",
+      (chan_l->global_identifier),
+      (uint64_t)(chan_l->timestamp_created),
+      (uint64_t)(now - chan_l->timestamp_created),
+      (uint64_t)(chan_l->timestamp_active),
+      (uint64_t)(now - chan_l->timestamp_active));
 
   tor_log(severity, LD_GENERAL,
-      " * Channel listener " U64_FORMAT " last accepted an incoming "
-        "channel at " U64_FORMAT " (" U64_FORMAT " seconds ago) "
-        "and has accepted " U64_FORMAT " channels in total",
-        U64_PRINTF_ARG(chan_l->global_identifier),
-        U64_PRINTF_ARG(chan_l->timestamp_accepted),
-        U64_PRINTF_ARG(now - chan_l->timestamp_accepted),
-        U64_PRINTF_ARG(chan_l->n_accepted));
+      " * Channel listener %"PRIu64 " last accepted an incoming "
+        "channel at %"PRIu64 " (%"PRIu64 " seconds ago) "
+        "and has accepted %"PRIu64 " channels in total",
+        (chan_l->global_identifier),
+        (uint64_t)(chan_l->timestamp_accepted),
+        (uint64_t)(now - chan_l->timestamp_accepted),
+        (uint64_t)(chan_l->n_accepted));
 
   /*
    * If it's sensible to do so, get the rate of incoming channels on this
@@ -3740,15 +2746,15 @@ channel_listener_dump_statistics(channel_listener_t *chan_l, int severity)
     avg = (double)(chan_l->n_accepted) / age;
     if (avg >= 1.0) {
       tor_log(severity, LD_GENERAL,
-          " * Channel listener " U64_FORMAT " has averaged %f incoming "
+          " * Channel listener %"PRIu64 " has averaged %f incoming "
           "channels per second",
-          U64_PRINTF_ARG(chan_l->global_identifier), avg);
+          (chan_l->global_identifier), avg);
     } else if (avg >= 0.0) {
       interval = 1.0 / avg;
       tor_log(severity, LD_GENERAL,
-          " * Channel listener " U64_FORMAT " has averaged %f seconds "
+          " * Channel listener %"PRIu64 " has averaged %f seconds "
           "between incoming channels",
-          U64_PRINTF_ARG(chan_l->global_identifier), interval);
+          (chan_l->global_identifier), interval);
     }
   }
 
@@ -3757,11 +2763,10 @@ channel_listener_dump_statistics(channel_listener_t *chan_l, int severity)
 }
 
 /**
- * Invoke transport-specific stats dump for channel
+ * Invoke transport-specific stats dump for channel.
  *
- * If there is a lower-layer statistics dump method, invoke it
+ * If there is a lower-layer statistics dump method, invoke it.
  */
-
 void
 channel_dump_transport_statistics(channel_t *chan, int severity)
 {
@@ -3771,11 +2776,10 @@ channel_dump_transport_statistics(channel_t *chan, int severity)
 }
 
 /**
- * Invoke transport-specific stats dump for channel listener
+ * Invoke transport-specific stats dump for channel listener.
  *
- * If there is a lower-layer statistics dump method, invoke it
+ * If there is a lower-layer statistics dump method, invoke it.
  */
-
 void
 channel_listener_dump_transport_statistics(channel_listener_t *chan_l,
                                            int severity)
@@ -3786,7 +2790,7 @@ channel_listener_dump_transport_statistics(channel_listener_t *chan_l,
 }
 
 /**
- * Return text description of the remote endpoint
+ * Return text description of the remote endpoint.
  *
  * This function return a test provided by the lower layer of the remote
  * endpoint for this channel; it should specify the actual address connected
@@ -3819,7 +2823,7 @@ channel_get_actual_remote_address(channel_t *chan)
 }
 
 /**
- * Return text description of the remote endpoint canonical address
+ * Return text description of the remote endpoint canonical address.
  *
  * This function return a test provided by the lower layer of the remote
  * endpoint for this channel; it should use the known canonical address for
@@ -3858,37 +2862,25 @@ channel_get_addr_if_possible,(channel_t *chan, tor_addr_t *addr_out))
 }
 
 /**
- * Check if there are outgoing queue writes on this channel
- *
- * Indicate if either we have queued cells, or if not, whether the underlying
- * lower-layer transport thinks it has an output queue.
+ * Return true iff the channel has any cells on the connection outbuf waiting
+ * to be sent onto the network.
  */
-
 int
 channel_has_queued_writes(channel_t *chan)
 {
-  int has_writes = 0;
-
   tor_assert(chan);
   tor_assert(chan->has_queued_writes);
 
-  if (! TOR_SIMPLEQ_EMPTY(&chan->outgoing_queue)) {
-    has_writes = 1;
-  } else {
-    /* Check with the lower layer */
-    has_writes = chan->has_queued_writes(chan);
-  }
-
-  return has_writes;
+  /* Check with the lower layer */
+  return chan->has_queued_writes(chan);
 }
 
 /**
- * Check the is_bad_for_new_circs flag
+ * Check the is_bad_for_new_circs flag.
  *
  * This function returns the is_bad_for_new_circs flag of the specified
  * channel.
  */
-
 int
 channel_is_bad_for_new_circs(channel_t *chan)
 {
@@ -3898,11 +2890,10 @@ channel_is_bad_for_new_circs(channel_t *chan)
 }
 
 /**
- * Mark a channel as bad for new circuits
+ * Mark a channel as bad for new circuits.
  *
  * Set the is_bad_for_new_circs_flag on chan.
  */
-
 void
 channel_mark_bad_for_new_circs(channel_t *chan)
 {
@@ -3912,15 +2903,14 @@ channel_mark_bad_for_new_circs(channel_t *chan)
 }
 
 /**
- * Get the client flag
+ * Get the client flag.
  *
  * This returns the client flag of a channel, which will be set if
  * command_process_create_cell() in command.c thinks this is a connection
  * from a client.
  */
-
 int
-channel_is_client(channel_t *chan)
+channel_is_client(const channel_t *chan)
 {
   tor_assert(chan);
 
@@ -3928,11 +2918,10 @@ channel_is_client(channel_t *chan)
 }
 
 /**
- * Set the client flag
+ * Set the client flag.
  *
- * Mark a channel as being from a client
+ * Mark a channel as being from a client.
  */
-
 void
 channel_mark_client(channel_t *chan)
 {
@@ -3942,12 +2931,24 @@ channel_mark_client(channel_t *chan)
 }
 
 /**
- * Get the canonical flag for a channel
+ * Clear the client flag.
+ *
+ * Mark a channel as being _not_ from a client.
+ */
+void
+channel_clear_client(channel_t *chan)
+{
+  tor_assert(chan);
+
+  chan->is_client = 0;
+}
+
+/**
+ * Get the canonical flag for a channel.
  *
  * This returns the is_canonical for a channel; this flag is determined by
  * the lower layer and can't be set in a transport-independent way.
  */
-
 int
 channel_is_canonical(channel_t *chan)
 {
@@ -3958,12 +2959,11 @@ channel_is_canonical(channel_t *chan)
 }
 
 /**
- * Test if the canonical flag is reliable
+ * Test if the canonical flag is reliable.
  *
  * This function asks if the lower layer thinks it's safe to trust the
- * result of channel_is_canonical()
+ * result of channel_is_canonical().
  */
-
 int
 channel_is_canonical_is_reliable(channel_t *chan)
 {
@@ -3974,12 +2974,11 @@ channel_is_canonical_is_reliable(channel_t *chan)
 }
 
 /**
- * Test incoming flag
+ * Test incoming flag.
  *
  * This function gets the incoming flag; this is set when a listener spawns
  * a channel.  If this returns true the channel was remotely initiated.
  */
-
 int
 channel_is_incoming(channel_t *chan)
 {
@@ -3989,12 +2988,11 @@ channel_is_incoming(channel_t *chan)
 }
 
 /**
- * Set the incoming flag
+ * Set the incoming flag.
  *
  * This function is called when a channel arrives on a listening channel
  * to mark it as incoming.
  */
-
 void
 channel_mark_incoming(channel_t *chan)
 {
@@ -4004,7 +3002,7 @@ channel_mark_incoming(channel_t *chan)
 }
 
 /**
- * Test local flag
+ * Test local flag.
  *
  * This function gets the local flag; the lower layer should set this when
  * setting up the channel if is_local_addr() is true for all of the
@@ -4012,7 +3010,6 @@ channel_mark_incoming(channel_t *chan)
  * used to decide whether to declare the network reachable when seeing incoming
  * traffic on the channel.
  */
-
 int
 channel_is_local(channel_t *chan)
 {
@@ -4022,13 +3019,12 @@ channel_is_local(channel_t *chan)
 }
 
 /**
- * Set the local flag
+ * Set the local flag.
  *
  * This internal-only function should be called by the lower layer if the
  * channel is to a local address.  See channel_is_local() above or the
- * description of the is_local bit in channel.h
+ * description of the is_local bit in channel.h.
  */
-
 void
 channel_mark_local(channel_t *chan)
 {
@@ -4038,14 +3034,13 @@ channel_mark_local(channel_t *chan)
 }
 
 /**
- * Mark a channel as remote
+ * Mark a channel as remote.
  *
  * This internal-only function should be called by the lower layer if the
  * channel is not to a local address but has previously been marked local.
  * See channel_is_local() above or the description of the is_local bit in
  * channel.h
  */
-
 void
 channel_mark_remote(channel_t *chan)
 {
@@ -4055,13 +3050,12 @@ channel_mark_remote(channel_t *chan)
 }
 
 /**
- * Test outgoing flag
+ * Test outgoing flag.
  *
  * This function gets the outgoing flag; this is the inverse of the incoming
  * bit set when a listener spawns a channel.  If this returns true the channel
  * was locally initiated.
  */
-
 int
 channel_is_outgoing(channel_t *chan)
 {
@@ -4071,12 +3065,11 @@ channel_is_outgoing(channel_t *chan)
 }
 
 /**
- * Mark a channel as outgoing
+ * Mark a channel as outgoing.
  *
  * This function clears the incoming flag and thus marks a channel as
  * outgoing.
  */
-
 void
 channel_mark_outgoing(channel_t *chan)
 {
@@ -4089,24 +3082,11 @@ channel_mark_outgoing(channel_t *chan)
  * Flow control queries *
  ***********************/
 
-/*
- * Get the latest estimate for the total queue size of all open channels
- */
-
-uint64_t
-channel_get_global_queue_estimate(void)
-{
-  return estimated_total_queue_size;
-}
-
-/*
- * Estimate the number of writeable cells
+/**
+ * Estimate the number of writeable cells.
  *
- * Ask the lower layer for an estimate of how many cells it can accept, and
- * then subtract the length of our outgoing_queue, if any, to produce an
- * estimate of the number of cells this channel can accept for writes.
+ * Ask the lower layer for an estimate of how many cells it can accept.
  */
-
 int
 channel_num_cells_writeable(channel_t *chan)
 {
@@ -4118,8 +3098,6 @@ channel_num_cells_writeable(channel_t *chan)
   if (chan->state == CHANNEL_STATE_OPEN) {
     /* Query lower layer */
     result = chan->num_cells_writeable(chan);
-    /* Subtract cell queue length, if any */
-    result -= chan_cell_queue_len(&chan->outgoing_queue);
     if (result < 0) result = 0;
   } else {
     /* No cells are writeable in any other state */
@@ -4134,12 +3112,11 @@ channel_num_cells_writeable(channel_t *chan)
  ********************/
 
 /**
- * Update the created timestamp for a channel
+ * Update the created timestamp for a channel.
  *
  * This updates the channel's created timestamp and should only be called
  * from channel_init().
  */
-
 void
 channel_timestamp_created(channel_t *chan)
 {
@@ -4151,12 +3128,11 @@ channel_timestamp_created(channel_t *chan)
 }
 
 /**
- * Update the created timestamp for a channel listener
+ * Update the created timestamp for a channel listener.
  *
  * This updates the channel listener's created timestamp and should only be
  * called from channel_init_listener().
  */
-
 void
 channel_listener_timestamp_created(channel_listener_t *chan_l)
 {
@@ -4168,7 +3144,7 @@ channel_listener_timestamp_created(channel_listener_t *chan_l)
 }
 
 /**
- * Update the last active timestamp for a channel
+ * Update the last active timestamp for a channel.
  *
  * This function updates the channel's last active timestamp; it should be
  * called by the lower layer whenever there is activity on the channel which
@@ -4177,21 +3153,23 @@ channel_listener_timestamp_created(channel_listener_t *chan_l)
  * but it should be updated for things like the v3 handshake and stuff that
  * produce activity only visible to the lower layer.
  */
-
 void
 channel_timestamp_active(channel_t *chan)
 {
   time_t now = time(NULL);
 
   tor_assert(chan);
+  monotime_coarse_get(&chan->timestamp_xfer);
 
   chan->timestamp_active = now;
+
+  /* Clear any potential netflow padding timer. We're active */
+  monotime_coarse_zero(&chan->next_padding_time);
 }
 
 /**
- * Update the last active timestamp for a channel listener
+ * Update the last active timestamp for a channel listener.
  */
-
 void
 channel_listener_timestamp_active(channel_listener_t *chan_l)
 {
@@ -4209,7 +3187,6 @@ channel_listener_timestamp_active(channel_listener_t *chan_l)
  * should be called whenever a new incoming channel is accepted on a
  * listener.
  */
-
 void
 channel_listener_timestamp_accepted(channel_listener_t *chan_l)
 {
@@ -4222,12 +3199,11 @@ channel_listener_timestamp_accepted(channel_listener_t *chan_l)
 }
 
 /**
- * Update client timestamp
+ * Update client timestamp.
  *
  * This function is called by relay.c to timestamp a channel that appears to
  * be used as a client.
  */
-
 void
 channel_timestamp_client(channel_t *chan)
 {
@@ -4239,57 +3215,44 @@ channel_timestamp_client(channel_t *chan)
 }
 
 /**
- * Update the last drained timestamp
- *
- * This is called whenever we transmit a cell which leaves the outgoing cell
- * queue completely empty.  It also updates the xmit time and the active time.
- */
-
-void
-channel_timestamp_drained(channel_t *chan)
-{
-  time_t now = time(NULL);
-
-  tor_assert(chan);
-
-  chan->timestamp_active = now;
-  chan->timestamp_drained = now;
-  chan->timestamp_xmit = now;
-}
-
-/**
- * Update the recv timestamp
+ * Update the recv timestamp.
  *
  * This is called whenever we get an incoming cell from the lower layer.
  * This also updates the active timestamp.
  */
-
 void
 channel_timestamp_recv(channel_t *chan)
 {
   time_t now = time(NULL);
-
   tor_assert(chan);
+  monotime_coarse_get(&chan->timestamp_xfer);
 
   chan->timestamp_active = now;
   chan->timestamp_recv = now;
+
+  /* Clear any potential netflow padding timer. We're active */
+  monotime_coarse_zero(&chan->next_padding_time);
 }
 
 /**
- * Update the xmit timestamp
+ * Update the xmit timestamp.
+ *
  * This is called whenever we pass an outgoing cell to the lower layer.  This
  * also updates the active timestamp.
  */
-
 void
 channel_timestamp_xmit(channel_t *chan)
 {
   time_t now = time(NULL);
-
   tor_assert(chan);
 
+  monotime_coarse_get(&chan->timestamp_xfer);
+
   chan->timestamp_active = now;
   chan->timestamp_xmit = now;
+
+  /* Clear any potential netflow padding timer. We're active */
+  monotime_coarse_zero(&chan->next_padding_time);
 }
 
 /***************************************************************
@@ -4297,9 +3260,8 @@ channel_timestamp_xmit(channel_t *chan)
  **************************************************************/
 
 /**
- * Query created timestamp for a channel
+ * Query created timestamp for a channel.
  */
-
 time_t
 channel_when_created(channel_t *chan)
 {
@@ -4309,57 +3271,8 @@ channel_when_created(channel_t *chan)
 }
 
 /**
- * Query created timestamp for a channel listener
- */
-
-time_t
-channel_listener_when_created(channel_listener_t *chan_l)
-{
-  tor_assert(chan_l);
-
-  return chan_l->timestamp_created;
-}
-
-/**
- * Query last active timestamp for a channel
- */
-
-time_t
-channel_when_last_active(channel_t *chan)
-{
-  tor_assert(chan);
-
-  return chan->timestamp_active;
-}
-
-/**
- * Query last active timestamp for a channel listener
- */
-
-time_t
-channel_listener_when_last_active(channel_listener_t *chan_l)
-{
-  tor_assert(chan_l);
-
-  return chan_l->timestamp_active;
-}
-
-/**
- * Query last accepted timestamp for a channel listener
+ * Query client timestamp.
  */
-
-time_t
-channel_listener_when_last_accepted(channel_listener_t *chan_l)
-{
-  tor_assert(chan_l);
-
-  return chan_l->timestamp_accepted;
-}
-
-/**
- * Query client timestamp
- */
-
 time_t
 channel_when_last_client(channel_t *chan)
 {
@@ -4369,33 +3282,8 @@ channel_when_last_client(channel_t *chan)
 }
 
 /**
- * Query drained timestamp
+ * Query xmit timestamp.
  */
-
-time_t
-channel_when_last_drained(channel_t *chan)
-{
-  tor_assert(chan);
-
-  return chan->timestamp_drained;
-}
-
-/**
- * Query recv timestamp
- */
-
-time_t
-channel_when_last_recv(channel_t *chan)
-{
-  tor_assert(chan);
-
-  return chan->timestamp_recv;
-}
-
-/**
- * Query xmit timestamp
- */
-
 time_t
 channel_when_last_xmit(channel_t *chan)
 {
@@ -4405,48 +3293,11 @@ channel_when_last_xmit(channel_t *chan)
 }
 
 /**
- * Query accepted counter
- */
-
-uint64_t
-channel_listener_count_accepted(channel_listener_t *chan_l)
-{
-  tor_assert(chan_l);
-
-  return chan_l->n_accepted;
-}
-
-/**
- * Query received cell counter
- */
-
-uint64_t
-channel_count_recved(channel_t *chan)
-{
-  tor_assert(chan);
-
-  return chan->n_cells_recved;
-}
-
-/**
- * Query transmitted cell counter
- */
-
-uint64_t
-channel_count_xmitted(channel_t *chan)
-{
-  tor_assert(chan);
-
-  return chan->n_cells_xmitted;
-}
-
-/**
- * Check if a channel matches an extend_info_t
+ * Check if a channel matches an extend_info_t.
  *
  * This function calls the lower layer and asks if this channel matches a
  * given extend_info_t.
  */
-
 int
 channel_matches_extend_info(channel_t *chan, extend_info_t *extend_info)
 {
@@ -4463,7 +3314,6 @@ channel_matches_extend_info(channel_t *chan, extend_info_t *extend_info)
  * This function calls into the lower layer and asks if this channel thinks
  * it matches a given target address for circuit extension purposes.
  */
-
 int
 channel_matches_target_addr_for_extend(channel_t *chan,
                                        const tor_addr_t *target)
@@ -4476,12 +3326,11 @@ channel_matches_target_addr_for_extend(channel_t *chan,
 }
 
 /**
- * Return the total number of circuits used by a channel
+ * Return the total number of circuits used by a channel.
  *
  * @param chan Channel to query
  * @return Number of circuits using this as n_chan or p_chan
  */
-
 unsigned int
 channel_num_circuits(channel_t *chan)
 {
@@ -4492,10 +3341,10 @@ channel_num_circuits(channel_t *chan)
 }
 
 /**
- * Set up circuit ID generation
+ * Set up circuit ID generation.
  *
  * This is called when setting up a channel and replaces the old
- * connection_or_set_circid_type()
+ * connection_or_set_circid_type().
  */
 MOCK_IMPL(void,
 channel_set_circid_type,(channel_t *chan,
@@ -4531,87 +3380,97 @@ channel_set_circid_type,(channel_t *chan,
   }
 }
 
-/**
- * Update the estimated number of bytes queued to transmit for this channel,
- * and notify the scheduler.  The estimate includes both the channel queue and
- * the queue size reported by the lower layer, and an overhead estimate
- * optionally provided by the lower layer.
- */
+static int
+channel_sort_by_ed25519_identity(const void **a_, const void **b_)
+{
+  const channel_t *a = *a_,
+                  *b = *b_;
+  return fast_memcmp(&a->ed25519_identity.pubkey,
+                     &b->ed25519_identity.pubkey,
+                     sizeof(a->ed25519_identity.pubkey));
+}
 
-void
-channel_update_xmit_queue_size(channel_t *chan)
+/** Helper for channel_update_bad_for_new_circs(): Perform the
+ * channel_update_bad_for_new_circs operation on all channels in <b>lst</b>,
+ * all of which MUST have the same RSA ID.  (They MAY have different
+ * Ed25519 IDs.) */
+static void
+channel_rsa_id_group_set_badness(struct channel_list_s *lst, int force)
 {
-  uint64_t queued, adj;
-  double overhead;
+  /*XXXX This function should really be about channels. 15056 */
+  channel_t *chan = TOR_LIST_FIRST(lst);
 
-  tor_assert(chan);
-  tor_assert(chan->num_bytes_queued);
+  if (!chan)
+    return;
 
-  /*
-   * First, get the number of bytes we have queued without factoring in
-   * lower-layer overhead.
-   */
-  queued = chan->num_bytes_queued(chan) + chan->bytes_in_queue;
-  /* Next, adjust by the overhead factor, if any is available */
-  if (chan->get_overhead_estimate) {
-    overhead = chan->get_overhead_estimate(chan);
-    if (overhead >= 1.0) {
-      queued = (uint64_t)(queued * overhead);
-    } else {
-      /* Ignore silly overhead factors */
-      log_notice(LD_CHANNEL, "Ignoring silly overhead factor %f", overhead);
+  /* if there is only one channel, don't bother looping */
+  if (PREDICT_LIKELY(!TOR_LIST_NEXT(chan, next_with_same_id))) {
+    connection_or_single_set_badness_(
+            time(NULL), BASE_CHAN_TO_TLS(chan)->conn, force);
+    return;
+  }
+
+  smartlist_t *channels = smartlist_new();
+
+  TOR_LIST_FOREACH(chan, lst, next_with_same_id) {
+    if (BASE_CHAN_TO_TLS(chan)->conn) {
+      smartlist_add(channels, chan);
     }
   }
 
-  /* Now, compare to the previous estimate */
-  if (queued > chan->bytes_queued_for_xmit) {
-    adj = queued - chan->bytes_queued_for_xmit;
-    log_debug(LD_CHANNEL,
-              "Increasing queue size for channel " U64_FORMAT " by " U64_FORMAT
-              " from " U64_FORMAT " to " U64_FORMAT,
-              U64_PRINTF_ARG(chan->global_identifier),
-              U64_PRINTF_ARG(adj),
-              U64_PRINTF_ARG(chan->bytes_queued_for_xmit),
-              U64_PRINTF_ARG(queued));
-    /* Update the channel's estimate */
-    chan->bytes_queued_for_xmit = queued;
-
-    /* Update the global queue size estimate if appropriate */
-    if (chan->state == CHANNEL_STATE_OPEN ||
-        chan->state == CHANNEL_STATE_MAINT) {
-      estimated_total_queue_size += adj;
-      log_debug(LD_CHANNEL,
-                "Increasing global queue size by " U64_FORMAT " for channel "
-                U64_FORMAT ", new size is " U64_FORMAT,
-                U64_PRINTF_ARG(adj), U64_PRINTF_ARG(chan->global_identifier),
-                U64_PRINTF_ARG(estimated_total_queue_size));
-      /* Tell the scheduler we're increasing the queue size */
-      scheduler_adjust_queue_size(chan, 1, adj);
+  smartlist_sort(channels, channel_sort_by_ed25519_identity);
+
+  const ed25519_public_key_t *common_ed25519_identity = NULL;
+  /* it would be more efficient to do a slice, but this case is rare */
+  smartlist_t *or_conns = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(channels, channel_t *, channel) {
+    tor_assert(channel); // Suppresses some compiler warnings.
+
+    if (!common_ed25519_identity)
+      common_ed25519_identity = &channel->ed25519_identity;
+
+    if (! ed25519_pubkey_eq(&channel->ed25519_identity,
+                            common_ed25519_identity)) {
+        connection_or_group_set_badness_(or_conns, force);
+        smartlist_clear(or_conns);
+        common_ed25519_identity = &channel->ed25519_identity;
     }
-  } else if (queued < chan->bytes_queued_for_xmit) {
-    adj = chan->bytes_queued_for_xmit - queued;
-    log_debug(LD_CHANNEL,
-              "Decreasing queue size for channel " U64_FORMAT " by " U64_FORMAT
-              " from " U64_FORMAT " to " U64_FORMAT,
-              U64_PRINTF_ARG(chan->global_identifier),
-              U64_PRINTF_ARG(adj),
-              U64_PRINTF_ARG(chan->bytes_queued_for_xmit),
-              U64_PRINTF_ARG(queued));
-    /* Update the channel's estimate */
-    chan->bytes_queued_for_xmit = queued;
-
-    /* Update the global queue size estimate if appropriate */
-    if (chan->state == CHANNEL_STATE_OPEN ||
-        chan->state == CHANNEL_STATE_MAINT) {
-      estimated_total_queue_size -= adj;
-      log_debug(LD_CHANNEL,
-                "Decreasing global queue size by " U64_FORMAT " for channel "
-                U64_FORMAT ", new size is " U64_FORMAT,
-                U64_PRINTF_ARG(adj), U64_PRINTF_ARG(chan->global_identifier),
-                U64_PRINTF_ARG(estimated_total_queue_size));
-      /* Tell the scheduler we're decreasing the queue size */
-      scheduler_adjust_queue_size(chan, -1, adj);
+
+    smartlist_add(or_conns, BASE_CHAN_TO_TLS(channel)->conn);
+  } SMARTLIST_FOREACH_END(channel);
+
+  connection_or_group_set_badness_(or_conns, force);
+
+  /* XXXX 15056 we may want to do something special with connections that have
+   * no set Ed25519 identity! */
+
+  smartlist_free(or_conns);
+  smartlist_free(channels);
+}
+
+/** Go through all the channels (or if <b>digest</b> is non-NULL, just
+ * the OR connections with that digest), and set the is_bad_for_new_circs
+ * flag based on the rules in connection_or_group_set_badness() (or just
+ * always set it if <b>force</b> is true).
+ */
+void
+channel_update_bad_for_new_circs(const char *digest, int force)
+{
+  if (digest) {
+    channel_idmap_entry_t *ent;
+    channel_idmap_entry_t search;
+    memset(&search, 0, sizeof(search));
+    memcpy(search.digest, digest, DIGEST_LEN);
+    ent = HT_FIND(channel_idmap, &channel_identity_map, &search);
+    if (ent) {
+      channel_rsa_id_group_set_badness(&ent->channel_list, force);
     }
+    return;
   }
-}
 
+  /* no digest; just look at everything. */
+  channel_idmap_entry_t **iter;
+  HT_FOREACH(iter, channel_idmap, &channel_identity_map) {
+    channel_rsa_id_group_set_badness(&(*iter)->channel_list, force);
+  }
+}
diff --git a/src/or/channel.h b/src/core/or/channel.h
index bcd345e8d2..97aa000337 100644
--- a/src/or/channel.h
+++ b/src/core/or/channel.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,17 +9,166 @@
 #ifndef TOR_CHANNEL_H
 #define TOR_CHANNEL_H
 
-#include "or.h"
-#include "circuitmux.h"
+#include "core/or/or.h"
+#include "core/or/circuitmux.h"
+#include "lib/container/handles.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+#include "tor_queue.h"
+
+#define tor_timer_t timeout
+struct tor_timer_t;
 
 /* Channel handler function pointer typedefs */
 typedef void (*channel_listener_fn_ptr)(channel_listener_t *, channel_t *);
 typedef void (*channel_cell_handler_fn_ptr)(channel_t *, cell_t *);
 typedef void (*channel_var_cell_handler_fn_ptr)(channel_t *, var_cell_t *);
 
-struct cell_queue_entry_s;
-TOR_SIMPLEQ_HEAD(chan_cell_queue, cell_queue_entry_s);
-typedef struct chan_cell_queue chan_cell_queue_t;
+/**
+ * This enum is used by channelpadding to decide when to pad channels.
+ * Don't add values to it without updating the checks in
+ * channelpadding_decide_to_pad_channel().
+ */
+typedef enum {
+    CHANNEL_USED_NOT_USED_FOR_FULL_CIRCS = 0,
+    CHANNEL_USED_FOR_FULL_CIRCS,
+    CHANNEL_USED_FOR_USER_TRAFFIC,
+} channel_usage_info_t;
+
+/** Possible rules for generating circuit IDs on an OR connection. */
+typedef enum {
+  CIRC_ID_TYPE_LOWER=0, /**< Pick from 0..1<<15-1. */
+  CIRC_ID_TYPE_HIGHER=1, /**< Pick from 1<<15..1<<16-1. */
+  /** The other side of a connection is an OP: never create circuits to it,
+   * and let it use any circuit ID it wants. */
+  CIRC_ID_TYPE_NEITHER=2
+} circ_id_type_t;
+#define circ_id_type_bitfield_t ENUM_BF(circ_id_type_t)
+
+/* channel states for channel_t */
+
+typedef enum {
+  /*
+   * Closed state - channel is inactive
+   *
+   * Permitted transitions from:
+   *   - CHANNEL_STATE_CLOSING
+   * Permitted transitions to:
+   *   - CHANNEL_STATE_OPENING
+   */
+  CHANNEL_STATE_CLOSED = 0,
+  /*
+   * Opening state - channel is trying to connect
+   *
+   * Permitted transitions from:
+   *   - CHANNEL_STATE_CLOSED
+   * Permitted transitions to:
+   *   - CHANNEL_STATE_CLOSING
+   *   - CHANNEL_STATE_ERROR
+   *   - CHANNEL_STATE_OPEN
+   */
+  CHANNEL_STATE_OPENING,
+  /*
+   * Open state - channel is active and ready for use
+   *
+   * Permitted transitions from:
+   *   - CHANNEL_STATE_MAINT
+   *   - CHANNEL_STATE_OPENING
+   * Permitted transitions to:
+   *   - CHANNEL_STATE_CLOSING
+   *   - CHANNEL_STATE_ERROR
+   *   - CHANNEL_STATE_MAINT
+   */
+  CHANNEL_STATE_OPEN,
+  /*
+   * Maintenance state - channel is temporarily offline for subclass specific
+   *   maintenance activities such as TLS renegotiation.
+   *
+   * Permitted transitions from:
+   *   - CHANNEL_STATE_OPEN
+   * Permitted transitions to:
+   *   - CHANNEL_STATE_CLOSING
+   *   - CHANNEL_STATE_ERROR
+   *   - CHANNEL_STATE_OPEN
+   */
+  CHANNEL_STATE_MAINT,
+  /*
+   * Closing state - channel is shutting down
+   *
+   * Permitted transitions from:
+   *   - CHANNEL_STATE_MAINT
+   *   - CHANNEL_STATE_OPEN
+   * Permitted transitions to:
+   *   - CHANNEL_STATE_CLOSED,
+   *   - CHANNEL_STATE_ERROR
+   */
+  CHANNEL_STATE_CLOSING,
+  /*
+   * Error state - channel has experienced a permanent error
+   *
+   * Permitted transitions from:
+   *   - CHANNEL_STATE_CLOSING
+   *   - CHANNEL_STATE_MAINT
+   *   - CHANNEL_STATE_OPENING
+   *   - CHANNEL_STATE_OPEN
+   * Permitted transitions to:
+   *   - None
+   */
+  CHANNEL_STATE_ERROR,
+  /*
+   * Placeholder for maximum state value
+   */
+  CHANNEL_STATE_LAST
+} channel_state_t;
+
+/* channel listener states for channel_listener_t */
+
+typedef enum {
+  /*
+   * Closed state - channel listener is inactive
+   *
+   * Permitted transitions from:
+   *   - CHANNEL_LISTENER_STATE_CLOSING
+   * Permitted transitions to:
+   *   - CHANNEL_LISTENER_STATE_LISTENING
+   */
+  CHANNEL_LISTENER_STATE_CLOSED = 0,
+  /*
+   * Listening state - channel listener is listening for incoming
+   * connections
+   *
+   * Permitted transitions from:
+   *   - CHANNEL_LISTENER_STATE_CLOSED
+   * Permitted transitions to:
+   *   - CHANNEL_LISTENER_STATE_CLOSING
+   *   - CHANNEL_LISTENER_STATE_ERROR
+   */
+  CHANNEL_LISTENER_STATE_LISTENING,
+  /*
+   * Closing state - channel listener is shutting down
+   *
+   * Permitted transitions from:
+   *   - CHANNEL_LISTENER_STATE_LISTENING
+   * Permitted transitions to:
+   *   - CHANNEL_LISTENER_STATE_CLOSED,
+   *   - CHANNEL_LISTENER_STATE_ERROR
+   */
+  CHANNEL_LISTENER_STATE_CLOSING,
+  /*
+   * Error state - channel listener has experienced a permanent error
+   *
+   * Permitted transitions from:
+   *   - CHANNEL_STATE_CLOSING
+   *   - CHANNEL_STATE_LISTENING
+   * Permitted transitions to:
+   *   - None
+   */
+  CHANNEL_LISTENER_STATE_ERROR,
+  /*
+   * Placeholder for maximum state value
+   */
+  CHANNEL_LISTENER_STATE_LAST
+} channel_listener_state_t;
 
 /**
  * Channel struct; see the channel_t typedef in or.h.  A channel is an
@@ -29,16 +178,21 @@ typedef struct chan_cell_queue chan_cell_queue_t;
  * to a particular node, and once constructed support the abstract operations
  * defined below.
  */
-
 struct channel_s {
   /** Magic number for type-checking cast macros */
   uint32_t magic;
 
+  /** List entry for hashtable for global-identifier lookup. */
+  HT_ENTRY(channel_s) gidmap_node;
+
+  /** Handle entry for handle-based lookup */
+  HANDLE_ENTRY(channel, channel_s);
+
   /** Current channel state */
   channel_state_t state;
 
   /** Globally unique ID number for a channel over the lifetime of a Tor
-   * process.
+   * process.  This may not be 0.
    */
   uint64_t global_identifier;
 
@@ -48,6 +202,60 @@ struct channel_s {
   /** has this channel ever been open? */
   unsigned int has_been_open:1;
 
+  /**
+   * This field indicates if the other side has enabled or disabled
+   * padding via either the link protocol version or
+   * channelpadding_negotiate cells.
+   *
+   * Clients can override this with ConnectionPadding in torrc to
+   * disable or force padding to relays, but relays cannot override the
+   * client's request.
+   */
+  unsigned int padding_enabled:1;
+
+  /** Cached value of our decision to pad (to avoid expensive
+   * checks during critical path statistics counting). */
+  unsigned int currently_padding:1;
+
+  /** Is there a pending netflow padding callback? */
+  unsigned int pending_padding_callback:1;
+
+  /** Is our peer likely to consider this channel canonical? */
+  unsigned int is_canonical_to_peer:1;
+
+  /** Has this channel ever been used for non-directory traffic?
+   * Used to decide what channels to pad, and when. */
+  channel_usage_info_t channel_usage;
+
+  /** When should we send a cell for netflow padding? 0 means no padding is
+   *  scheduled. */
+  monotime_coarse_t next_padding_time;
+
+  /** The callback pointer for the padding callbacks */
+  struct tor_timer_t *padding_timer;
+  /** The handle to this channel (to free on canceled timers) */
+  struct channel_handle_t *timer_handle;
+
+  /**
+   * These two fields specify the minimum and maximum negotiated timeout
+   * values for inactivity (send or receive) before we decide to pad a
+   * channel. These fields can be set either via a PADDING_NEGOTIATE cell,
+   * or the torrc option ReducedConnectionPadding. The consensus parameters
+   * nf_ito_low and nf_ito_high are used to ensure that padding can only be
+   * negotiated to be less frequent than what is specified in the consensus.
+   * (This is done to prevent wingnut clients from requesting excessive
+   * padding).
+   *
+   * The actual timeout value is randomly chosen between these two values
+   * as per the table in channelpadding_get_netflow_inactive_timeout_ms(),
+   * after ensuring that these values do not specify lower timeouts than
+   * the consensus parameters.
+   *
+   * If these are 0, we have not negotiated or specified custom padding
+   * times, and instead use consensus defaults. */
+  uint16_t padding_timeout_low_ms;
+  uint16_t padding_timeout_high_ms;
+
   /** Why did we close?
    */
   enum {
@@ -87,6 +295,18 @@ struct channel_s {
   time_t timestamp_created; /* Channel created */
   time_t timestamp_active; /* Any activity */
 
+  /**
+   * This is a monotonic timestamp that marks when we
+   * believe the channel has actually sent or received data to/from
+   * the wire. Right now, it is used to determine when we should send
+   * a padding cell for channelpadding.
+   *
+   * XXX: Are we setting timestamp_xfer_ms in the right places to
+   * accurately reflect actual network data transfer? Or might this be
+   * very wrong wrt when bytes actually go on the wire?
+   */
+  monotime_coarse_t timestamp_xfer;
+
   /* Methods implemented by the lower layer */
 
   /** Free a channel */
@@ -153,25 +373,32 @@ struct channel_s {
   int (*write_var_cell)(channel_t *, var_cell_t *);
 
   /**
-   * Hash of the public RSA key for the other side's identity key, or
-   * zeroes if the other side hasn't shown us a valid identity key.
+   * Hash of the public RSA key for the other side's RSA identity key -- or
+   * zeroes if we don't have an RSA identity in mind for the other side, and
+   * it hasn't shown us one.
+   *
+   * Note that this is the RSA identity that we hope the other side has -- not
+   * necessarily its true identity.  Don't believe this identity unless
+   * authentication has happened.
    */
   char identity_digest[DIGEST_LEN];
-  /** Nickname of the OR on the other side, or NULL if none. */
-  char *nickname;
+  /**
+   * Ed25519 key for the other side of this channel -- or zeroes if we don't
+   * have an Ed25519 identity in mind for the other side, and it hasn't shown
+   * us one.
+   *
+   * Note that this is the identity that we hope the other side has -- not
+   * necessarily its true identity.  Don't believe this identity unless
+   * authentication has happened.
+   */
+  struct ed25519_public_key_t ed25519_identity;
 
   /**
-   * Linked list of channels with the same identity digest, for the
-   * digest->channel map
+   * Linked list of channels with the same RSA identity digest, for use with
+   * the digest->channel map
    */
   TOR_LIST_ENTRY(channel_s) next_with_same_id;
 
-  /** List of incoming cells to handle */
-  chan_cell_queue_t incoming_queue;
-
-  /** List of queued outgoing cells */
-  chan_cell_queue_t outgoing_queue;
-
   /** Circuit mux for circuits sending on this channel */
   circuitmux_t *cmux;
 
@@ -198,8 +425,8 @@ struct channel_s {
   unsigned int is_bad_for_new_circs:1;
 
   /** True iff we have decided that the other end of this connection
-   * is a client.  Channels with this flag set should never be used
-   * to satisfy an EXTEND request.  */
+   * is a client or bridge relay.  Connections with this flag set should never
+   * be used to satisfy an EXTEND request. */
   unsigned int is_client:1;
 
   /** Set if the channel was initiated remotely (came from a listener) */
@@ -218,7 +445,6 @@ struct channel_s {
 
   /** Channel timestamps for cell channels */
   time_t timestamp_client; /* Client used this, according to relay.c */
-  time_t timestamp_drained; /* Output queue empty */
   time_t timestamp_recv; /* Cell received from lower layer */
   time_t timestamp_xmit; /* Cell sent to lower layer */
 
@@ -235,14 +461,6 @@ struct channel_s {
   /** Channel counters for cell channels */
   uint64_t n_cells_recved, n_bytes_recved;
   uint64_t n_cells_xmitted, n_bytes_xmitted;
-
-  /** Our current contribution to the scheduler's total xmit queue */
-  uint64_t bytes_queued_for_xmit;
-
-  /** Number of bytes in this channel's cell queue; does not include
-   * lower-layer queueing.
-   */
-  uint64_t bytes_in_queue;
 };
 
 struct channel_listener_s {
@@ -310,18 +528,13 @@ channel_listener_state_to_string(channel_listener_state_t state);
 /* Abstract channel operations */
 
 void channel_mark_for_close(channel_t *chan);
-void channel_write_cell(channel_t *chan, cell_t *cell);
-void channel_write_packed_cell(channel_t *chan, packed_cell_t *cell);
-void channel_write_var_cell(channel_t *chan, var_cell_t *cell);
+int channel_write_packed_cell(channel_t *chan, packed_cell_t *cell);
 
 void channel_listener_mark_for_close(channel_listener_t *chan_l);
 
 /* Channel callback registrations */
 
 /* Listener callback */
-channel_listener_fn_ptr
-channel_listener_get_listener_fn(channel_listener_t *chan);
-
 void channel_listener_set_listener_fn(channel_listener_t *chan,
                                       channel_listener_fn_ptr listener);
 
@@ -349,40 +562,13 @@ void channel_free_all(void);
 void channel_dumpstats(int severity);
 void channel_listener_dumpstats(int severity);
 
-/* Set the cmux policy on all active channels */
-void channel_set_cmux_policy_everywhere(circuitmux_policy_t *pol);
-
 #ifdef TOR_CHANNEL_INTERNAL_
 
 #ifdef CHANNEL_PRIVATE_
-/* Cell queue structure (here rather than channel.c for test suite use) */
-
-typedef struct cell_queue_entry_s cell_queue_entry_t;
-struct cell_queue_entry_s {
-  TOR_SIMPLEQ_ENTRY(cell_queue_entry_s) next;
-  enum {
-    CELL_QUEUE_FIXED,
-    CELL_QUEUE_VAR,
-    CELL_QUEUE_PACKED
-  } type;
-  union {
-    struct {
-      cell_t *cell;
-    } fixed;
-    struct {
-      var_cell_t *var_cell;
-    } var;
-    struct {
-      packed_cell_t *packed_cell;
-    } packed;
-  } u;
-};
 
-/* Cell queue functions for benefit of test suite */
-STATIC int chan_cell_queue_len(const chan_cell_queue_t *queue);
+STATIC void channel_add_to_digest_map(channel_t *chan);
 
-STATIC void cell_queue_entry_free(cell_queue_entry_t *q, int handed_off);
-#endif
+#endif /* defined(CHANNEL_PRIVATE_) */
 
 /* Channel operations for subclasses and internal use only */
 
@@ -406,17 +592,17 @@ void channel_close_from_lower_layer(channel_t *chan);
 void channel_close_for_error(channel_t *chan);
 void channel_closed(channel_t *chan);
 
-void channel_listener_close_from_lower_layer(channel_listener_t *chan_l);
-void channel_listener_close_for_error(channel_listener_t *chan_l);
-void channel_listener_closed(channel_listener_t *chan_l);
-
 /* Free a channel */
-void channel_free(channel_t *chan);
-void channel_listener_free(channel_listener_t *chan_l);
+void channel_free_(channel_t *chan);
+#define channel_free(chan) FREE_AND_NULL(channel_t, channel_free_, (chan))
+void channel_listener_free_(channel_listener_t *chan_l);
+#define channel_listener_free(chan_l) \
+  FREE_AND_NULL(channel_listener_t, channel_listener_free_, (chan_l))
 
 /* State/metadata setters */
 
 void channel_change_state(channel_t *chan, channel_state_t to_state);
+void channel_change_state_open(channel_t *chan);
 void channel_clear_identity_digest(channel_t *chan);
 void channel_clear_remote_end(channel_t *chan);
 void channel_mark_local(channel_t *chan);
@@ -424,10 +610,8 @@ void channel_mark_incoming(channel_t *chan);
 void channel_mark_outgoing(channel_t *chan);
 void channel_mark_remote(channel_t *chan);
 void channel_set_identity_digest(channel_t *chan,
-                                 const char *identity_digest);
-void channel_set_remote_end(channel_t *chan,
-                            const char *identity_digest,
-                            const char *nickname);
+                             const char *identity_digest,
+                             const struct ed25519_public_key_t *ed_identity);
 
 void channel_listener_change_state(channel_listener_t *chan_l,
                                    channel_listener_state_t to_state);
@@ -435,7 +619,6 @@ void channel_listener_change_state(channel_listener_t *chan_l,
 /* Timestamp updates */
 void channel_timestamp_created(channel_t *chan);
 void channel_timestamp_active(channel_t *chan);
-void channel_timestamp_drained(channel_t *chan);
 void channel_timestamp_recv(channel_t *chan);
 void channel_timestamp_xmit(channel_t *chan);
 
@@ -449,19 +632,14 @@ void channel_listener_queue_incoming(channel_listener_t *listener,
                                      channel_t *incoming);
 
 /* Incoming cell handling */
-void channel_process_cells(channel_t *chan);
-void channel_queue_cell(channel_t *chan, cell_t *cell);
-void channel_queue_var_cell(channel_t *chan, var_cell_t *var_cell);
-
-/* Outgoing cell handling */
-void channel_flush_cells(channel_t *chan);
+void channel_process_cell(channel_t *chan, cell_t *cell);
 
 /* Request from lower layer for more cells if available */
 MOCK_DECL(ssize_t, channel_flush_some_cells,
           (channel_t *chan, ssize_t num_cells));
 
 /* Query if data available on this channel */
-int channel_more_to_flush(channel_t *chan);
+MOCK_DECL(int, channel_more_to_flush, (channel_t *chan));
 
 /* Notify flushed outgoing for dirreq handling */
 void channel_notify_flushed(channel_t *chan);
@@ -469,11 +647,7 @@ void channel_notify_flushed(channel_t *chan);
 /* Handle stuff we need to do on open like notifying circuits */
 void channel_do_open_actions(channel_t *chan);
 
-#ifdef TOR_UNIT_TESTS
-extern uint64_t estimated_total_queue_size;
-#endif
-
-#endif
+#endif /* defined(TOR_CHANNEL_INTERNAL_) */
 
 /* Helper functions to perform operations on channels */
 
@@ -486,27 +660,29 @@ int channel_send_destroy(circid_t circ_id, channel_t *chan,
  */
 
 channel_t * channel_connect(const tor_addr_t *addr, uint16_t port,
-                            const char *id_digest);
+                            const char *rsa_id_digest,
+                            const struct ed25519_public_key_t *ed_id);
 
-channel_t * channel_get_for_extend(const char *digest,
+channel_t * channel_get_for_extend(const char *rsa_id_digest,
+                                   const struct ed25519_public_key_t *ed_id,
                                    const tor_addr_t *target_addr,
                                    const char **msg_out,
                                    int *launch_out);
 
 /* Ask which of two channels is better for circuit-extension purposes */
-int channel_is_better(time_t now,
-                      channel_t *a, channel_t *b,
-                      int forgive_new_connections);
+int channel_is_better(channel_t *a, channel_t *b);
 
 /** Channel lookups
  */
 
 channel_t * channel_find_by_global_id(uint64_t global_identifier);
-channel_t * channel_find_by_remote_digest(const char *identity_digest);
+channel_t * channel_find_by_remote_identity(const char *rsa_id_digest,
+                                    const struct ed25519_public_key_t *ed_id);
 
 /** For things returned by channel_find_by_remote_digest(), walk the list.
+ * The RSA key will match for all returned elements; the Ed25519 key might not.
  */
-channel_t * channel_next_with_digest(channel_t *chan);
+channel_t * channel_next_with_rsa_identity(channel_t *chan);
 
 /*
  * Helper macros to lookup state of given channel.
@@ -558,11 +734,12 @@ int channel_is_bad_for_new_circs(channel_t *chan);
 void channel_mark_bad_for_new_circs(channel_t *chan);
 int channel_is_canonical(channel_t *chan);
 int channel_is_canonical_is_reliable(channel_t *chan);
-int channel_is_client(channel_t *chan);
+int channel_is_client(const channel_t *chan);
 int channel_is_local(channel_t *chan);
 int channel_is_incoming(channel_t *chan);
 int channel_is_outgoing(channel_t *chan);
 void channel_mark_client(channel_t *chan);
+void channel_clear_client(channel_t *chan);
 int channel_matches_extend_info(channel_t *chan, extend_info_t *extend_info);
 int channel_matches_target_addr_for_extend(channel_t *chan,
                                            const tor_addr_t *target);
@@ -571,39 +748,33 @@ MOCK_DECL(void,channel_set_circid_type,(channel_t *chan,
                                         crypto_pk_t *identity_rcvd,
                                         int consider_identity));
 void channel_timestamp_client(channel_t *chan);
-void channel_update_xmit_queue_size(channel_t *chan);
 
 const char * channel_listener_describe_transport(channel_listener_t *chan_l);
 void channel_listener_dump_statistics(channel_listener_t *chan_l,
                                       int severity);
 void channel_listener_dump_transport_statistics(channel_listener_t *chan_l,
                                                 int severity);
+void channel_check_for_duplicates(void);
+
+void channel_update_bad_for_new_circs(const char *digest, int force);
 
 /* Flow control queries */
-uint64_t channel_get_global_queue_estimate(void);
 int channel_num_cells_writeable(channel_t *chan);
 
 /* Timestamp queries */
 time_t channel_when_created(channel_t *chan);
-time_t channel_when_last_active(channel_t *chan);
 time_t channel_when_last_client(channel_t *chan);
-time_t channel_when_last_drained(channel_t *chan);
-time_t channel_when_last_recv(channel_t *chan);
 time_t channel_when_last_xmit(channel_t *chan);
 
-time_t channel_listener_when_created(channel_listener_t *chan_l);
-time_t channel_listener_when_last_active(channel_listener_t *chan_l);
-time_t channel_listener_when_last_accepted(channel_listener_t *chan_l);
-
 /* Counter queries */
-uint64_t channel_count_recved(channel_t *chan);
-uint64_t channel_count_xmitted(channel_t *chan);
-
-uint64_t channel_listener_count_accepted(channel_listener_t *chan_l);
-
 int packed_cell_is_destroy(channel_t *chan,
                            const packed_cell_t *packed_cell,
                            circid_t *circid_out);
 
-#endif
+/* Declare the handle helpers */
+HANDLE_DECL(channel, channel_s,)
+#define channel_handle_free(h)    \
+  FREE_AND_NULL(channel_handle_t, channel_handle_free_, (h))
+#undef tor_timer_t
 
+#endif /* !defined(TOR_CHANNEL_H) */
diff --git a/src/core/or/channelpadding.c b/src/core/or/channelpadding.c
new file mode 100644
index 0000000000..4a0f0e00da
--- /dev/null
+++ b/src/core/or/channelpadding.c
@@ -0,0 +1,794 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/* TOR_CHANNEL_INTERNAL_ define needed for an O(1) implementation of
+ * channelpadding_channel_to_channelinfo() */
+#define TOR_CHANNEL_INTERNAL_
+
+#include "core/or/or.h"
+#include "core/or/channel.h"
+#include "core/or/channelpadding.h"
+#include "core/or/channeltls.h"
+#include "app/config/config.h"
+#include "feature/nodelist/networkstatus.h"
+#include "core/mainloop/connection.h"
+#include "core/or/connection_or.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "core/mainloop/mainloop.h"
+#include "feature/stats/rephist.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+#include "lib/time/compat_time.h"
+#include "feature/rend/rendservice.h"
+#include "lib/evloop/timers.h"
+
+#include "core/or/cell_st.h"
+#include "core/or/or_connection_st.h"
+
+STATIC int32_t channelpadding_get_netflow_inactive_timeout_ms(
+                                                           const channel_t *);
+STATIC int channelpadding_send_disable_command(channel_t *);
+STATIC int64_t channelpadding_compute_time_until_pad_for_netflow(channel_t *);
+
+/** The total number of pending channelpadding timers */
+static uint64_t total_timers_pending;
+
+/** These are cached consensus parameters for netflow */
+/** The timeout lower bound that is allowed before sending padding */
+static int consensus_nf_ito_low;
+/** The timeout upper bound that is allowed before sending padding */
+static int consensus_nf_ito_high;
+/** The timeout lower bound that is allowed before sending reduced padding */
+static int consensus_nf_ito_low_reduced;
+/** The timeout upper bound that is allowed before sending reduced padding */
+static int consensus_nf_ito_high_reduced;
+/** The connection timeout between relays */
+static int consensus_nf_conntimeout_relays;
+/** The connection timeout for client connections */
+static int consensus_nf_conntimeout_clients;
+/** Should we pad before circuits are actually used for client data? */
+static int consensus_nf_pad_before_usage;
+/** Should we pad relay-to-relay connections? */
+static int consensus_nf_pad_relays;
+/** Should we pad rosos connections? */
+static int consensus_nf_pad_single_onion;
+
+#define TOR_MSEC_PER_SEC 1000
+#define TOR_USEC_PER_MSEC 1000
+
+/**
+ * How often do we get called by the connection housekeeping (ie: once
+ * per second) */
+#define TOR_HOUSEKEEPING_CALLBACK_MSEC 1000
+/**
+ * Additional extra time buffer on the housekeeping callback, since
+ * it can be delayed. This extra slack is used to decide if we should
+ * schedule a timer or wait for the next callback. */
+#define TOR_HOUSEKEEPING_CALLBACK_SLACK_MSEC 100
+
+/**
+ * This macro tells us if either end of the channel is connected to a client.
+ * (If we're not a server, we're definitely a client. If the channel thinks
+ *  it's a client, use that. Then finally verify in the consensus).
+ */
+#define CHANNEL_IS_CLIENT(chan, options) \
+  (!public_server_mode((options)) || channel_is_client(chan) || \
+      !connection_or_digest_is_known_relay((chan)->identity_digest))
+
+/**
+ * This function is called to update cached consensus parameters every time
+ * there is a consensus update. This allows us to move the consensus param
+ * search off of the critical path, so it does not need to be evaluated
+ * for every single connection, every second.
+ */
+void
+channelpadding_new_consensus_params(networkstatus_t *ns)
+{
+#define DFLT_NETFLOW_INACTIVE_KEEPALIVE_LOW 1500
+#define DFLT_NETFLOW_INACTIVE_KEEPALIVE_HIGH 9500
+#define DFLT_NETFLOW_INACTIVE_KEEPALIVE_MIN 0
+#define DFLT_NETFLOW_INACTIVE_KEEPALIVE_MAX 60000
+  consensus_nf_ito_low = networkstatus_get_param(ns, "nf_ito_low",
+      DFLT_NETFLOW_INACTIVE_KEEPALIVE_LOW,
+      DFLT_NETFLOW_INACTIVE_KEEPALIVE_MIN,
+      DFLT_NETFLOW_INACTIVE_KEEPALIVE_MAX);
+  consensus_nf_ito_high = networkstatus_get_param(ns, "nf_ito_high",
+      DFLT_NETFLOW_INACTIVE_KEEPALIVE_HIGH,
+      consensus_nf_ito_low,
+      DFLT_NETFLOW_INACTIVE_KEEPALIVE_MAX);
+
+#define DFLT_NETFLOW_REDUCED_KEEPALIVE_LOW 9000
+#define DFLT_NETFLOW_REDUCED_KEEPALIVE_HIGH 14000
+#define DFLT_NETFLOW_REDUCED_KEEPALIVE_MIN 0
+#define DFLT_NETFLOW_REDUCED_KEEPALIVE_MAX 60000
+  consensus_nf_ito_low_reduced =
+    networkstatus_get_param(ns, "nf_ito_low_reduced",
+        DFLT_NETFLOW_REDUCED_KEEPALIVE_LOW,
+        DFLT_NETFLOW_REDUCED_KEEPALIVE_MIN,
+        DFLT_NETFLOW_REDUCED_KEEPALIVE_MAX);
+
+  consensus_nf_ito_high_reduced =
+    networkstatus_get_param(ns, "nf_ito_high_reduced",
+        DFLT_NETFLOW_REDUCED_KEEPALIVE_HIGH,
+        consensus_nf_ito_low_reduced,
+        DFLT_NETFLOW_REDUCED_KEEPALIVE_MAX);
+
+#define CONNTIMEOUT_RELAYS_DFLT (60*60) // 1 hour
+#define CONNTIMEOUT_RELAYS_MIN 60
+#define CONNTIMEOUT_RELAYS_MAX (7*24*60*60) // 1 week
+  consensus_nf_conntimeout_relays =
+    networkstatus_get_param(ns, "nf_conntimeout_relays",
+        CONNTIMEOUT_RELAYS_DFLT,
+        CONNTIMEOUT_RELAYS_MIN,
+        CONNTIMEOUT_RELAYS_MAX);
+
+#define CIRCTIMEOUT_CLIENTS_DFLT (30*60) // 30 minutes
+#define CIRCTIMEOUT_CLIENTS_MIN 60
+#define CIRCTIMEOUT_CLIENTS_MAX (24*60*60) // 24 hours
+  consensus_nf_conntimeout_clients =
+    networkstatus_get_param(ns, "nf_conntimeout_clients",
+        CIRCTIMEOUT_CLIENTS_DFLT,
+        CIRCTIMEOUT_CLIENTS_MIN,
+        CIRCTIMEOUT_CLIENTS_MAX);
+
+  consensus_nf_pad_before_usage =
+    networkstatus_get_param(ns, "nf_pad_before_usage", 1, 0, 1);
+
+  consensus_nf_pad_relays =
+    networkstatus_get_param(ns, "nf_pad_relays", 0, 0, 1);
+
+  consensus_nf_pad_single_onion =
+    networkstatus_get_param(ns,
+                            CHANNELPADDING_SOS_PARAM,
+                            CHANNELPADDING_SOS_DEFAULT, 0, 1);
+}
+
+/**
+ * Get a random netflow inactive timeout keepalive period in milliseconds,
+ * the range for which is determined by consensus parameters, negotiation,
+ * configuration, or default values. The consensus parameters enforce the
+ * minimum possible value, to avoid excessively frequent padding.
+ *
+ * The ranges for this value were chosen to be low enough to ensure that
+ * routers do not emit a new netflow record for a connection due to it
+ * being idle.
+ *
+ * Specific timeout values for major routers are listed in Proposal 251.
+ * No major router appeared capable of setting an inactive timeout below 10
+ * seconds, so we set the defaults below that value, since we can always
+ * scale back if it ends up being too much padding.
+ *
+ * Returns the next timeout period (in milliseconds) after which we should
+ * send a padding packet, or 0 if padding is disabled.
+ */
+STATIC int32_t
+channelpadding_get_netflow_inactive_timeout_ms(const channel_t *chan)
+{
+  int low_timeout = consensus_nf_ito_low;
+  int high_timeout = consensus_nf_ito_high;
+  int X1, X2;
+
+  if (low_timeout == 0 && low_timeout == high_timeout)
+    return 0; // No padding
+
+  /* If we have negotiated different timeout values, use those, but
+   * don't allow them to be lower than the consensus ones */
+  if (chan->padding_timeout_low_ms && chan->padding_timeout_high_ms) {
+    low_timeout = MAX(low_timeout, chan->padding_timeout_low_ms);
+    high_timeout = MAX(high_timeout, chan->padding_timeout_high_ms);
+  }
+
+  if (low_timeout == high_timeout)
+    return low_timeout; // No randomization
+
+  /*
+   * This MAX() hack is here because we apply the timeout on both the client
+   * and the server. This creates the situation where the total time before
+   * sending a packet in either direction is actually
+   * min(client_timeout,server_timeout).
+   *
+   * If X is a random variable uniform from 0..R-1 (where R=high-low),
+   * then Y=max(X,X) has Prob(Y == i) = (2.0*i + 1)/(R*R).
+   *
+   * If we create a third random variable Z=min(Y,Y), then it turns out that
+   * Exp[Z] ~= Exp[X]. Here's a table:
+   *
+   *    R     Exp[X]    Exp[Z]    Exp[min(X,X)]   Exp[max(X,X)]
+   *  2000     999.5    1066        666.2           1332.8
+   *  3000    1499.5    1599.5      999.5           1999.5
+   *  5000    2499.5    2666       1666.2           3332.8
+   *  6000    2999.5    3199.5     1999.5           3999.5
+   *  7000    3499.5    3732.8     2332.8           4666.2
+   *  8000    3999.5    4266.2     2666.2           5332.8
+   *  10000   4999.5    5328       3332.8           6666.2
+   *  15000   7499.5    7995       4999.5           9999.5
+   *  20000   9900.5    10661      6666.2           13332.8
+   *
+   * In other words, this hack makes it so that when both the client and
+   * the guard are sending this padding, then the averages work out closer
+   * to the midpoint of the range, making the overhead easier to tune.
+   * If only one endpoint is padding (for example: if the relay does not
+   * support padding, but the client has set ConnectionPadding 1; or
+   * if the relay does support padding, but the client has set
+   * ReducedConnectionPadding 1), then the defense will still prevent
+   * record splitting, but with less overhead than the midpoint
+   * (as seen by the Exp[max(X,X)] column).
+   *
+   * To calculate average padding packet frequency (and thus overhead),
+   * index into the table by picking a row based on R = high-low. Then,
+   * use the appropriate column (Exp[Z] for two-sided padding, and
+   * Exp[max(X,X)] for one-sided padding). Finally, take this value
+   * and add it to the low timeout value. This value is the average
+   * frequency which padding packets will be sent.
+   */
+
+  X1 = crypto_rand_int(high_timeout - low_timeout);
+  X2 = crypto_rand_int(high_timeout - low_timeout);
+  return low_timeout + MAX(X1, X2);
+}
+
+/**
+ * Update this channel's padding settings based on the PADDING_NEGOTIATE
+ * contents.
+ *
+ * Returns -1 on error; 1 on success.
+ */
+int
+channelpadding_update_padding_for_channel(channel_t *chan,
+                const channelpadding_negotiate_t *pad_vars)
+{
+  if (pad_vars->version != 0) {
+    static ratelim_t version_limit = RATELIM_INIT(600);
+
+    log_fn_ratelim(&version_limit,LOG_PROTOCOL_WARN,LD_PROTOCOL,
+           "Got a PADDING_NEGOTIATE cell with an unknown version. Ignoring.");
+    return -1;
+  }
+
+  // We should not allow malicious relays to disable or reduce padding for
+  // us as clients. In fact, we should only accept this cell at all if we're
+  // operating as a relay. Bridges should not accept it from relays, either
+  // (only from their clients).
+  if ((get_options()->BridgeRelay &&
+       connection_or_digest_is_known_relay(chan->identity_digest)) ||
+      !get_options()->ORPort_set) {
+    static ratelim_t relay_limit = RATELIM_INIT(600);
+
+    log_fn_ratelim(&relay_limit,LOG_PROTOCOL_WARN,LD_PROTOCOL,
+           "Got a PADDING_NEGOTIATE from relay at %s (%s). "
+           "This should not happen.",
+           chan->get_remote_descr(chan, 0),
+           hex_str(chan->identity_digest, DIGEST_LEN));
+    return -1;
+  }
+
+  chan->padding_enabled = (pad_vars->command == CHANNELPADDING_COMMAND_START);
+
+  /* Min must not be lower than the current consensus parameter
+     nf_ito_low. */
+  chan->padding_timeout_low_ms = MAX(consensus_nf_ito_low,
+                                     pad_vars->ito_low_ms);
+
+  /* Max must not be lower than ito_low_ms */
+  chan->padding_timeout_high_ms = MAX(chan->padding_timeout_low_ms,
+                                      pad_vars->ito_high_ms);
+
+  log_fn(LOG_INFO,LD_OR,
+         "Negotiated padding=%d, lo=%d, hi=%d on %"PRIu64,
+         chan->padding_enabled, chan->padding_timeout_low_ms,
+         chan->padding_timeout_high_ms,
+         (chan->global_identifier));
+
+  return 1;
+}
+
+/**
+ * Sends a CELL_PADDING_NEGOTIATE on the channel to tell the other side not
+ * to send padding.
+ *
+ * Returns -1 on error, 0 on success.
+ */
+STATIC int
+channelpadding_send_disable_command(channel_t *chan)
+{
+  channelpadding_negotiate_t disable;
+  cell_t cell;
+
+  tor_assert(chan);
+  tor_assert(BASE_CHAN_TO_TLS(chan)->conn->link_proto >=
+             MIN_LINK_PROTO_FOR_CHANNEL_PADDING);
+
+  memset(&cell, 0, sizeof(cell_t));
+  memset(&disable, 0, sizeof(channelpadding_negotiate_t));
+  cell.command = CELL_PADDING_NEGOTIATE;
+
+  channelpadding_negotiate_set_command(&disable, CHANNELPADDING_COMMAND_STOP);
+
+  if (channelpadding_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE,
+                                      &disable) < 0)
+    return -1;
+
+  if (chan->write_cell(chan, &cell) == 1)
+    return 0;
+  else
+    return -1;
+}
+
+/**
+ * Sends a CELL_PADDING_NEGOTIATE on the channel to tell the other side to
+ * resume sending padding at some rate.
+ *
+ * Returns -1 on error, 0 on success.
+ */
+int
+channelpadding_send_enable_command(channel_t *chan, uint16_t low_timeout,
+                                   uint16_t high_timeout)
+{
+  channelpadding_negotiate_t enable;
+  cell_t cell;
+
+  tor_assert(chan);
+  tor_assert(BASE_CHAN_TO_TLS(chan)->conn->link_proto >=
+             MIN_LINK_PROTO_FOR_CHANNEL_PADDING);
+
+  memset(&cell, 0, sizeof(cell_t));
+  memset(&enable, 0, sizeof(channelpadding_negotiate_t));
+  cell.command = CELL_PADDING_NEGOTIATE;
+
+  channelpadding_negotiate_set_command(&enable, CHANNELPADDING_COMMAND_START);
+  channelpadding_negotiate_set_ito_low_ms(&enable, low_timeout);
+  channelpadding_negotiate_set_ito_high_ms(&enable, high_timeout);
+
+  if (channelpadding_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE,
+                                      &enable) < 0)
+    return -1;
+
+  if (chan->write_cell(chan, &cell) == 1)
+    return 0;
+  else
+    return -1;
+}
+
+/**
+ * Sends a CELL_PADDING cell on a channel if it has been idle since
+ * our callback was scheduled.
+ *
+ * This function also clears the pending padding timer and the callback
+ * flags.
+ */
+static void
+channelpadding_send_padding_cell_for_callback(channel_t *chan)
+{
+  cell_t cell;
+
+  /* Check that the channel is still valid and open */
+  if (!chan || chan->state != CHANNEL_STATE_OPEN) {
+    if (chan) chan->pending_padding_callback = 0;
+    log_fn(LOG_INFO,LD_OR,
+           "Scheduled a netflow padding cell, but connection already closed.");
+    return;
+  }
+
+  /* We should have a pending callback flag set. */
+  if (BUG(chan->pending_padding_callback == 0))
+    return;
+
+  chan->pending_padding_callback = 0;
+
+  if (monotime_coarse_is_zero(&chan->next_padding_time) ||
+      chan->has_queued_writes(chan) ||
+      (chan->cmux && circuitmux_num_cells(chan->cmux))) {
+    /* We must have been active before the timer fired */
+    monotime_coarse_zero(&chan->next_padding_time);
+    return;
+  }
+
+  {
+    monotime_coarse_t now;
+    monotime_coarse_get(&now);
+
+    log_fn(LOG_INFO,LD_OR,
+        "Sending netflow keepalive on %"PRIu64" to %s (%s) after "
+        "%"PRId64" ms. Delta %"PRId64"ms",
+        (chan->global_identifier),
+        safe_str_client(chan->get_remote_descr(chan, 0)),
+        safe_str_client(hex_str(chan->identity_digest, DIGEST_LEN)),
+        (monotime_coarse_diff_msec(&chan->timestamp_xfer,&now)),
+        (
+                   monotime_coarse_diff_msec(&chan->next_padding_time,&now)));
+  }
+
+  /* Clear the timer */
+  monotime_coarse_zero(&chan->next_padding_time);
+
+  /* Send the padding cell. This will cause the channel to get a
+   * fresh timestamp_active */
+  memset(&cell, 0, sizeof(cell));
+  cell.command = CELL_PADDING;
+  chan->write_cell(chan, &cell);
+}
+
+/**
+ * tor_timer callback function for us to send padding on an idle channel.
+ *
+ * This function just obtains the channel from the callback handle, ensures
+ * it is still valid, and then hands it off to
+ * channelpadding_send_padding_cell_for_callback(), which checks if
+ * the channel is still idle before sending padding.
+ */
+static void
+channelpadding_send_padding_callback(tor_timer_t *timer, void *args,
+                                     const struct monotime_t *when)
+{
+  channel_t *chan = channel_handle_get((struct channel_handle_t*)args);
+  (void)timer; (void)when;
+
+  if (chan && CHANNEL_CAN_HANDLE_CELLS(chan)) {
+    /* Hrmm.. It might be nice to have an equivalent to assert_connection_ok
+     * for channels. Then we could get rid of the channeltls dependency */
+    tor_assert(TO_CONN(BASE_CHAN_TO_TLS(chan)->conn)->magic ==
+               OR_CONNECTION_MAGIC);
+    assert_connection_ok(TO_CONN(BASE_CHAN_TO_TLS(chan)->conn), approx_time());
+
+    channelpadding_send_padding_cell_for_callback(chan);
+  } else {
+     log_fn(LOG_INFO,LD_OR,
+            "Channel closed while waiting for timer.");
+  }
+
+  total_timers_pending--;
+}
+
+/**
+ * Schedules a callback to send padding on a channel in_ms milliseconds from
+ * now.
+ *
+ * Returns CHANNELPADDING_WONTPAD on error, CHANNELPADDING_PADDING_SENT if we
+ * sent the packet immediately without a timer, and
+ * CHANNELPADDING_PADDING_SCHEDULED if we decided to schedule a timer.
+ */
+static channelpadding_decision_t
+channelpadding_schedule_padding(channel_t *chan, int in_ms)
+{
+  struct timeval timeout;
+  tor_assert(!chan->pending_padding_callback);
+
+  if (in_ms <= 0) {
+    chan->pending_padding_callback = 1;
+    channelpadding_send_padding_cell_for_callback(chan);
+    return CHANNELPADDING_PADDING_SENT;
+  }
+
+  timeout.tv_sec = in_ms/TOR_MSEC_PER_SEC;
+  timeout.tv_usec = (in_ms%TOR_USEC_PER_MSEC)*TOR_USEC_PER_MSEC;
+
+  if (!chan->timer_handle) {
+    chan->timer_handle = channel_handle_new(chan);
+  }
+
+  if (chan->padding_timer) {
+    timer_set_cb(chan->padding_timer,
+                 channelpadding_send_padding_callback,
+                 chan->timer_handle);
+  } else {
+    chan->padding_timer = timer_new(channelpadding_send_padding_callback,
+                                    chan->timer_handle);
+  }
+  timer_schedule(chan->padding_timer, &timeout);
+
+  rep_hist_padding_count_timers(++total_timers_pending);
+
+  chan->pending_padding_callback = 1;
+  return CHANNELPADDING_PADDING_SCHEDULED;
+}
+
+/**
+ * Calculates the number of milliseconds from now to schedule a padding cell.
+ *
+ * Returns the number of milliseconds from now (relative) to schedule the
+ * padding callback. If the padding timer is more than 1.1 seconds in the
+ * future, we return -1, to avoid scheduling excessive callbacks. If padding
+ * is disabled in the consensus, we return -2.
+ *
+ * Side-effects: Updates chan->next_padding_time_ms, storing an (absolute, not
+ * relative) millisecond representation of when we should send padding, unless
+ * other activity happens first. This side-effect allows us to avoid
+ * scheduling a libevent callback until we're within 1.1 seconds of the padding
+ * time.
+ */
+#define CHANNELPADDING_TIME_LATER -1
+#define CHANNELPADDING_TIME_DISABLED -2
+STATIC int64_t
+channelpadding_compute_time_until_pad_for_netflow(channel_t *chan)
+{
+  monotime_coarse_t now;
+  monotime_coarse_get(&now);
+
+  if (monotime_coarse_is_zero(&chan->next_padding_time)) {
+    /* If the below line or crypto_rand_int() shows up on a profile,
+     * we can avoid getting a timeout until we're at least nf_ito_lo
+     * from a timeout window. That will prevent us from setting timers
+     * on connections that were active up to 1.5 seconds ago.
+     * Idle connections should only call this once every 5.5s on average
+     * though, so that might be a micro-optimization for little gain. */
+    int32_t padding_timeout =
+        channelpadding_get_netflow_inactive_timeout_ms(chan);
+
+    if (!padding_timeout)
+      return CHANNELPADDING_TIME_DISABLED;
+
+    monotime_coarse_add_msec(&chan->next_padding_time,
+                             &chan->timestamp_xfer,
+                             padding_timeout);
+  }
+
+  const int64_t ms_till_pad =
+    monotime_coarse_diff_msec(&now, &chan->next_padding_time);
+
+  /* If the next padding time is beyond the maximum possible consensus value,
+   * then this indicates a clock jump, so just send padding now. This is
+   * better than using monotonic time because we want to avoid the situation
+   * where we wait around forever for monotonic time to move forward after
+   * a clock jump far into the past.
+   */
+  if (ms_till_pad > DFLT_NETFLOW_INACTIVE_KEEPALIVE_MAX) {
+    tor_fragile_assert();
+    log_warn(LD_BUG,
+        "Channel padding timeout scheduled %"PRId64"ms in the future. "
+        "Did the monotonic clock just jump?",
+        (ms_till_pad));
+    return 0; /* Clock jumped: Send padding now */
+  }
+
+  /* If the timeout will expire before the next time we're called (1000ms
+     from now, plus some slack), then calculate the number of milliseconds
+     from now which we should send padding, so we can schedule a callback
+     then.
+   */
+  if (ms_till_pad < (TOR_HOUSEKEEPING_CALLBACK_MSEC +
+                       TOR_HOUSEKEEPING_CALLBACK_SLACK_MSEC)) {
+    /* If the padding time is in the past, that means that libevent delayed
+     * calling the once-per-second callback due to other work taking too long.
+     * See https://bugs.torproject.org/22212 and
+     * https://bugs.torproject.org/16585. This is a systemic problem
+     * with being single-threaded, but let's emit a notice if this
+     * is long enough in the past that we might have missed a netflow window,
+     * and allowed a router to emit a netflow frame, just so we don't forget
+     * about it entirely.. */
+#define NETFLOW_MISSED_WINDOW (150000 - DFLT_NETFLOW_INACTIVE_KEEPALIVE_HIGH)
+    if (ms_till_pad < 0) {
+      int severity = (ms_till_pad < -NETFLOW_MISSED_WINDOW)
+                      ? LOG_NOTICE : LOG_INFO;
+      log_fn(severity, LD_OR,
+              "Channel padding timeout scheduled %"PRId64"ms in the past. ",
+             (-ms_till_pad));
+      return 0; /* Clock jumped: Send padding now */
+    }
+
+    return ms_till_pad;
+  }
+  return CHANNELPADDING_TIME_LATER;
+}
+
+/**
+ * Returns a randomized value for channel idle timeout in seconds.
+ * The channel idle timeout governs how quickly we close a channel
+ * after its last circuit has disappeared.
+ *
+ * There are three classes of channels:
+ *  1. Client+non-canonical. These live for 3-4.5 minutes
+ *  2. relay to relay. These live for 45-75 min by default
+ *  3. Reduced padding clients. These live for 1.5-2.25 minutes.
+ *
+ * Also allows the default relay-to-relay value to be controlled by the
+ * consensus.
+ */
+unsigned int
+channelpadding_get_channel_idle_timeout(const channel_t *chan,
+                                        int is_canonical)
+{
+  const or_options_t *options = get_options();
+  unsigned int timeout;
+
+  /* Non-canonical and client channels only last for 3-4.5 min when idle */
+  if (!is_canonical || CHANNEL_IS_CLIENT(chan, options)) {
+#define CONNTIMEOUT_CLIENTS_BASE 180 // 3 to 4.5 min
+    timeout = CONNTIMEOUT_CLIENTS_BASE
+        + crypto_rand_int(CONNTIMEOUT_CLIENTS_BASE/2);
+  } else { // Canonical relay-to-relay channels
+    // 45..75min or consensus +/- 25%
+    timeout = consensus_nf_conntimeout_relays;
+    timeout = 3*timeout/4 + crypto_rand_int(timeout/2);
+  }
+
+  /* If ReducedConnectionPadding is set, we want to halve the duration of
+   * the channel idle timeout, since reducing the additional time that
+   * a channel stays open will reduce the total overhead for making
+   * new channels. This reduction in overhead/channel expense
+   * is important for mobile users. The option cannot be set by relays.
+   *
+   * We also don't reduce any values for timeout that the user explicitly
+   * set.
+   */
+  if (options->ReducedConnectionPadding
+      && !options->CircuitsAvailableTimeout) {
+    timeout /= 2;
+  }
+
+  return timeout;
+}
+
+/**
+ * This function controls how long we keep idle circuits open,
+ * and how long we build predicted circuits. This behavior is under
+ * the control of channelpadding because circuit availability is the
+ * dominant factor in channel lifespan, which influences total padding
+ * overhead.
+ *
+ * Returns a randomized number of seconds in a range from
+ * CircuitsAvailableTimeout to 2*CircuitsAvailableTimeout. This value is halved
+ * if ReducedConnectionPadding is set. The default value of
+ * CircuitsAvailableTimeout can be controlled by the consensus.
+ */
+int
+channelpadding_get_circuits_available_timeout(void)
+{
+  const or_options_t *options = get_options();
+  int timeout = options->CircuitsAvailableTimeout;
+
+  if (!timeout) {
+    timeout = consensus_nf_conntimeout_clients;
+
+    /* If ReducedConnectionPadding is set, we want to halve the duration of
+     * the channel idle timeout, since reducing the additional time that
+     * a channel stays open will reduce the total overhead for making
+     * new connections. This reduction in overhead/connection expense
+     * is important for mobile users. The option cannot be set by relays.
+     *
+     * We also don't reduce any values for timeout that the user explicitly
+     * set.
+     */
+    if (options->ReducedConnectionPadding) {
+      // half the value to 15..30min by default
+      timeout /= 2;
+    }
+  }
+
+  // 30..60min by default
+  timeout = timeout + crypto_rand_int(timeout);
+
+  tor_assert(timeout >= 0);
+
+  return timeout;
+}
+
+/**
+ * Calling this function on a channel causes it to tell the other side
+ * not to send padding, and disables sending padding from this side as well.
+ */
+void
+channelpadding_disable_padding_on_channel(channel_t *chan)
+{
+  chan->padding_enabled = 0;
+
+  // Send cell to disable padding on the other end
+  channelpadding_send_disable_command(chan);
+}
+
+/**
+ * Calling this function on a channel causes it to tell the other side
+ * not to send padding, and reduces the rate that padding is sent from
+ * this side.
+ */
+void
+channelpadding_reduce_padding_on_channel(channel_t *chan)
+{
+  /* Padding can be forced and reduced by clients, regardless of if
+   * the channel supports it. So we check for support here before
+   * sending any commands. */
+  if (chan->padding_enabled) {
+    channelpadding_send_disable_command(chan);
+  }
+
+  chan->padding_timeout_low_ms = consensus_nf_ito_low_reduced;
+  chan->padding_timeout_high_ms = consensus_nf_ito_high_reduced;
+
+  log_fn(LOG_INFO,LD_OR,
+         "Reduced padding on channel %"PRIu64": lo=%d, hi=%d",
+         (chan->global_identifier),
+         chan->padding_timeout_low_ms, chan->padding_timeout_high_ms);
+}
+
+/**
+ * This function is called once per second by run_connection_housekeeping(),
+ * but only if the channel is still open, valid, and non-wedged.
+ *
+ * It decides if and when we should send a padding cell, and if needed,
+ * schedules a callback to send that cell at the appropriate time.
+ *
+ * Returns an enum that represents the current padding decision state.
+ * Return value is currently used only by unit tests.
+ */
+channelpadding_decision_t
+channelpadding_decide_to_pad_channel(channel_t *chan)
+{
+  const or_options_t *options = get_options();
+
+  /* Only pad open channels */
+  if (chan->state != CHANNEL_STATE_OPEN)
+    return CHANNELPADDING_WONTPAD;
+
+  if (chan->channel_usage == CHANNEL_USED_FOR_FULL_CIRCS) {
+    if (!consensus_nf_pad_before_usage)
+      return CHANNELPADDING_WONTPAD;
+  } else if (chan->channel_usage != CHANNEL_USED_FOR_USER_TRAFFIC) {
+    return CHANNELPADDING_WONTPAD;
+  }
+
+  if (chan->pending_padding_callback)
+    return CHANNELPADDING_PADDING_ALREADY_SCHEDULED;
+
+  /* Don't pad the channel if we didn't negotiate it, but still
+   * allow clients to force padding if options->ChannelPadding is
+   * explicitly set to 1.
+   */
+  if (!chan->padding_enabled && options->ConnectionPadding != 1) {
+    return CHANNELPADDING_WONTPAD;
+  }
+
+  if (rend_service_allow_non_anonymous_connection(options) &&
+      !consensus_nf_pad_single_onion) {
+    /* If the consensus just changed values, this channel may still
+     * think padding is enabled. Negotiate it off. */
+    if (chan->padding_enabled)
+      channelpadding_disable_padding_on_channel(chan);
+
+    return CHANNELPADDING_WONTPAD;
+  }
+
+  /* There should always be a cmux on the circuit. After that,
+   * only schedule padding if there are no queued writes and no
+   * queued cells in circuitmux queues. */
+  if (chan->cmux && !chan->has_queued_writes(chan) &&
+      !circuitmux_num_cells(chan->cmux)) {
+    int is_client_channel = 0;
+
+    if (CHANNEL_IS_CLIENT(chan, options)) {
+       is_client_channel = 1;
+    }
+
+    /* If nf_pad_relays=1 is set in the consensus, we pad
+     * on *all* idle connections, relay-relay or relay-client.
+     * Otherwise pad only for client+bridge cons */
+    if (is_client_channel || consensus_nf_pad_relays) {
+      int64_t pad_time_ms =
+          channelpadding_compute_time_until_pad_for_netflow(chan);
+
+      if (pad_time_ms == CHANNELPADDING_TIME_DISABLED) {
+        return CHANNELPADDING_WONTPAD;
+      } else if (pad_time_ms == CHANNELPADDING_TIME_LATER) {
+        chan->currently_padding = 1;
+        return CHANNELPADDING_PADLATER;
+      } else {
+        if (BUG(pad_time_ms > INT_MAX)) {
+          pad_time_ms = INT_MAX;
+        }
+       /* We have to schedule a callback because we're called exactly once per
+        * second, but we don't want padding packets to go out exactly on an
+        * integer multiple of seconds. This callback will only be scheduled
+        * if we're within 1.1 seconds of the padding time.
+        */
+        chan->currently_padding = 1;
+        return channelpadding_schedule_padding(chan, (int)pad_time_ms);
+      }
+    } else {
+      chan->currently_padding = 0;
+      return CHANNELPADDING_WONTPAD;
+    }
+  } else {
+    return CHANNELPADDING_PADLATER;
+  }
+}
diff --git a/src/core/or/channelpadding.h b/src/core/or/channelpadding.h
new file mode 100644
index 0000000000..48002eedb7
--- /dev/null
+++ b/src/core/or/channelpadding.h
@@ -0,0 +1,43 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitbuild.h
+ * \brief Header file for circuitbuild.c.
+ **/
+#ifndef TOR_CHANNELPADDING_H
+#define TOR_CHANNELPADDING_H
+
+#include "trunnel/channelpadding_negotiation.h"
+
+#define CHANNELPADDING_SOS_PARAM  "nf_pad_single_onion"
+#define CHANNELPADDING_SOS_DEFAULT 1
+
+typedef enum {
+  CHANNELPADDING_WONTPAD,
+  CHANNELPADDING_PADLATER,
+  CHANNELPADDING_PADDING_SCHEDULED,
+  CHANNELPADDING_PADDING_ALREADY_SCHEDULED,
+  CHANNELPADDING_PADDING_SENT,
+} channelpadding_decision_t;
+
+channelpadding_decision_t channelpadding_decide_to_pad_channel(channel_t
+                                                               *chan);
+int channelpadding_update_padding_for_channel(channel_t *,
+                                              const channelpadding_negotiate_t
+                                              *chan);
+
+void channelpadding_disable_padding_on_channel(channel_t *chan);
+void channelpadding_reduce_padding_on_channel(channel_t *chan);
+int channelpadding_send_enable_command(channel_t *chan, uint16_t low_timeout,
+                                       uint16_t high_timeout);
+
+int channelpadding_get_circuits_available_timeout(void);
+unsigned int channelpadding_get_channel_idle_timeout(const channel_t *, int);
+void channelpadding_new_consensus_params(networkstatus_t *ns);
+
+#endif /* !defined(TOR_CHANNELPADDING_H) */
+
diff --git a/src/or/channeltls.c b/src/core/or/channeltls.c
index 6f4e413dc6..4db283d20e 100644
--- a/src/or/channeltls.c
+++ b/src/core/or/channeltls.c
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -34,27 +34,44 @@
  * Define this so channel.h gives us things only channel_t subclasses
  * should touch.
  */
-
 #define TOR_CHANNEL_INTERNAL_
 
 #define CHANNELTLS_PRIVATE
 
-#include "or.h"
-#include "channel.h"
-#include "channeltls.h"
-#include "circuitmux.h"
-#include "circuitmux_ewma.h"
-#include "command.h"
-#include "config.h"
-#include "connection.h"
-#include "connection_or.h"
-#include "control.h"
-#include "link_handshake.h"
-#include "relay.h"
-#include "rephist.h"
-#include "router.h"
-#include "routerlist.h"
-#include "scheduler.h"
+#include "core/or/or.h"
+#include "core/or/channel.h"
+#include "core/or/channeltls.h"
+#include "core/or/circuitmux.h"
+#include "core/or/circuitmux_ewma.h"
+#include "core/or/command.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/connection_or.h"
+#include "feature/control/control.h"
+#include "feature/client/entrynodes.h"
+#include "trunnel/link_handshake.h"
+#include "core/or/relay.h"
+#include "feature/stats/rephist.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+#include "feature/nodelist/dirlist.h"
+#include "core/or/scheduler.h"
+#include "feature/nodelist/torcert.h"
+#include "feature/nodelist/networkstatus.h"
+#include "trunnel/channelpadding_negotiation.h"
+#include "core/or/channelpadding.h"
+
+#include "core/or/cell_st.h"
+#include "core/or/cell_queue_st.h"
+#include "core/or/extend_info_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/or_handshake_certs_st.h"
+#include "core/or/or_handshake_state_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "core/or/var_cell_st.h"
+
+#include "lib/tls/tortls.h"
+#include "lib/tls/x509.h"
 
 /** How many CELL_PADDING cells have we received, ever? */
 uint64_t stats_n_padding_cells_processed = 0;
@@ -120,12 +137,13 @@ static void channel_tls_process_netinfo_cell(cell_t *cell,
 static int command_allowed_before_handshake(uint8_t command);
 static int enter_v3_handshake_with_cell(var_cell_t *cell,
                                         channel_tls_t *tlschan);
+static void channel_tls_process_padding_negotiate_cell(cell_t *cell,
+                                                       channel_tls_t *chan);
 
 /**
  * Do parts of channel_tls_t initialization common to channel_tls_connect()
  * and channel_tls_handle_incoming().
  */
-
 STATIC void
 channel_tls_common_init(channel_tls_t *tlschan)
 {
@@ -155,22 +173,21 @@ channel_tls_common_init(channel_tls_t *tlschan)
   chan->write_var_cell = channel_tls_write_var_cell_method;
 
   chan->cmux = circuitmux_alloc();
-  if (cell_ewma_enabled()) {
-    circuitmux_set_policy(chan->cmux, &ewma_policy);
-  }
+  /* We only have one policy for now so always set it to EWMA. */
+  circuitmux_set_policy(chan->cmux, &ewma_policy);
 }
 
 /**
- * Start a new TLS channel
+ * Start a new TLS channel.
  *
  * Launch a new OR connection to <b>addr</b>:<b>port</b> and expect to
  * handshake with an OR with identity digest <b>id_digest</b>, and wrap
  * it in a channel_tls_t.
  */
-
 channel_t *
 channel_tls_connect(const tor_addr_t *addr, uint16_t port,
-                    const char *id_digest)
+                    const char *id_digest,
+                    const ed25519_public_key_t *ed_id)
 {
   channel_tls_t *tlschan = tor_malloc_zero(sizeof(*tlschan));
   channel_t *chan = &(tlschan->base_);
@@ -179,26 +196,26 @@ channel_tls_connect(const tor_addr_t *addr, uint16_t port,
 
   log_debug(LD_CHANNEL,
             "In channel_tls_connect() for channel %p "
-            "(global id " U64_FORMAT ")",
+            "(global id %"PRIu64 ")",
             tlschan,
-            U64_PRINTF_ARG(chan->global_identifier));
+            (chan->global_identifier));
 
   if (is_local_addr(addr)) {
     log_debug(LD_CHANNEL,
-              "Marking new outgoing channel " U64_FORMAT " at %p as local",
-              U64_PRINTF_ARG(chan->global_identifier), chan);
+              "Marking new outgoing channel %"PRIu64 " at %p as local",
+              (chan->global_identifier), chan);
     channel_mark_local(chan);
   } else {
     log_debug(LD_CHANNEL,
-              "Marking new outgoing channel " U64_FORMAT " at %p as remote",
-              U64_PRINTF_ARG(chan->global_identifier), chan);
+              "Marking new outgoing channel %"PRIu64 " at %p as remote",
+              (chan->global_identifier), chan);
     channel_mark_remote(chan);
   }
 
   channel_mark_outgoing(chan);
 
   /* Set up or_connection stuff */
-  tlschan->conn = connection_or_connect(addr, port, id_digest, tlschan);
+  tlschan->conn = connection_or_connect(addr, port, id_digest, ed_id, tlschan);
   /* connection_or_connect() will fill in tlschan->conn */
   if (!(tlschan->conn)) {
     chan->reason_for_closing = CHANNEL_CLOSE_FOR_ERROR;
@@ -207,8 +224,8 @@ channel_tls_connect(const tor_addr_t *addr, uint16_t port,
   }
 
   log_debug(LD_CHANNEL,
-            "Got orconn %p for channel with global id " U64_FORMAT,
-            tlschan->conn, U64_PRINTF_ARG(chan->global_identifier));
+            "Got orconn %p for channel with global id %"PRIu64,
+            tlschan->conn, (chan->global_identifier));
 
   goto done;
 
@@ -225,12 +242,11 @@ channel_tls_connect(const tor_addr_t *addr, uint16_t port,
 }
 
 /**
- * Return the current channel_tls_t listener
+ * Return the current channel_tls_t listener.
  *
  * Returns the current channel listener for incoming TLS connections, or
  * NULL if none has been established
  */
-
 channel_listener_t *
 channel_tls_get_listener(void)
 {
@@ -238,12 +254,11 @@ channel_tls_get_listener(void)
 }
 
 /**
- * Start a channel_tls_t listener if necessary
+ * Start a channel_tls_t listener if necessary.
  *
  * Return the current channel_tls_t listener, or start one if we haven't yet,
  * and return that.
  */
-
 channel_listener_t *
 channel_tls_start_listener(void)
 {
@@ -260,8 +275,8 @@ channel_tls_start_listener(void)
     channel_tls_listener = listener;
 
     log_debug(LD_CHANNEL,
-              "Starting TLS channel listener %p with global id " U64_FORMAT,
-              listener, U64_PRINTF_ARG(listener->global_identifier));
+              "Starting TLS channel listener %p with global id %"PRIu64,
+              listener, (listener->global_identifier));
 
     channel_listener_register(listener);
   } else listener = channel_tls_listener;
@@ -270,12 +285,11 @@ channel_tls_start_listener(void)
 }
 
 /**
- * Free everything on shutdown
+ * Free everything on shutdown.
  *
  * Not much to do here, since channel_free_all() takes care of a lot, but let's
  * get rid of the listener.
  */
-
 void
 channel_tls_free_all(void)
 {
@@ -291,9 +305,9 @@ channel_tls_free_all(void)
      */
     old_listener = channel_tls_listener;
     log_debug(LD_CHANNEL,
-              "Closing channel_tls_listener with ID " U64_FORMAT
+              "Closing channel_tls_listener with ID %"PRIu64
               " at %p.",
-              U64_PRINTF_ARG(old_listener->global_identifier),
+              (old_listener->global_identifier),
               old_listener);
     channel_listener_unregister(old_listener);
     channel_listener_mark_for_close(old_listener);
@@ -306,9 +320,8 @@ channel_tls_free_all(void)
 }
 
 /**
- * Create a new channel around an incoming or_connection_t
+ * Create a new channel around an incoming or_connection_t.
  */
-
 channel_t *
 channel_tls_handle_incoming(or_connection_t *orconn)
 {
@@ -326,13 +339,13 @@ channel_tls_handle_incoming(or_connection_t *orconn)
 
   if (is_local_addr(&(TO_CONN(orconn)->addr))) {
     log_debug(LD_CHANNEL,
-              "Marking new incoming channel " U64_FORMAT " at %p as local",
-              U64_PRINTF_ARG(chan->global_identifier), chan);
+              "Marking new incoming channel %"PRIu64 " at %p as local",
+              (chan->global_identifier), chan);
     channel_mark_local(chan);
   } else {
     log_debug(LD_CHANNEL,
-              "Marking new incoming channel " U64_FORMAT " at %p as remote",
-              U64_PRINTF_ARG(chan->global_identifier), chan);
+              "Marking new incoming channel %"PRIu64 " at %p as remote",
+              (chan->global_identifier), chan);
     channel_mark_remote(chan);
   }
 
@@ -351,7 +364,6 @@ channel_tls_handle_incoming(or_connection_t *orconn)
 /**
  * Cast a channel_tls_t to a channel_t.
  */
-
 channel_t *
 channel_tls_to_base(channel_tls_t *tlschan)
 {
@@ -364,7 +376,6 @@ channel_tls_to_base(channel_tls_t *tlschan)
  * Cast a channel_t to a channel_tls_t, with appropriate type-checking
  * asserts.
  */
-
 channel_tls_t *
 channel_tls_from_base(channel_t *chan)
 {
@@ -380,11 +391,10 @@ channel_tls_from_base(channel_t *chan)
  *******************************************/
 
 /**
- * Close a channel_tls_t
+ * Close a channel_tls_t.
  *
- * This implements the close method for channel_tls_t
+ * This implements the close method for channel_tls_t.
  */
-
 static void
 channel_tls_close_method(channel_t *chan)
 {
@@ -403,12 +413,11 @@ channel_tls_close_method(channel_t *chan)
 }
 
 /**
- * Describe the transport for a channel_tls_t
+ * Describe the transport for a channel_tls_t.
  *
  * This returns the string "TLS channel on connection <id>" to the upper
  * layer.
  */
-
 static const char *
 channel_tls_describe_transport_method(channel_t *chan)
 {
@@ -426,8 +435,8 @@ channel_tls_describe_transport_method(channel_t *chan)
 
     if (buf) tor_free(buf);
     tor_asprintf(&buf,
-                 "TLS channel (connection " U64_FORMAT ")",
-                 U64_PRINTF_ARG(id));
+                 "TLS channel (connection %"PRIu64 ")",
+                 (id));
 
     rv = buf;
   } else {
@@ -438,7 +447,7 @@ channel_tls_describe_transport_method(channel_t *chan)
 }
 
 /**
- * Free a channel_tls_t
+ * Free a channel_tls_t.
  *
  * This is called by the generic channel layer when freeing a channel_tls_t;
  * this happens either on a channel which has already reached
@@ -447,7 +456,6 @@ channel_tls_describe_transport_method(channel_t *chan)
  * have an orconn active (which connection_free_all() will get to later),
  * so we should null out its channel pointer now.
  */
-
 static void
 channel_tls_free_method(channel_t *chan)
 {
@@ -462,9 +470,8 @@ channel_tls_free_method(channel_t *chan)
 }
 
 /**
- * Get an estimate of the average TLS overhead for the upper layer
+ * Get an estimate of the average TLS overhead for the upper layer.
  */
-
 static double
 channel_tls_get_overhead_estimate_method(channel_t *chan)
 {
@@ -490,20 +497,19 @@ channel_tls_get_overhead_estimate_method(channel_t *chan)
   }
 
   log_debug(LD_CHANNEL,
-            "Estimated overhead ratio for TLS chan " U64_FORMAT " is %f",
-            U64_PRINTF_ARG(chan->global_identifier), overhead);
+            "Estimated overhead ratio for TLS chan %"PRIu64 " is %f",
+            (chan->global_identifier), overhead);
 
   return overhead;
 }
 
 /**
- * Get the remote address of a channel_tls_t
+ * Get the remote address of a channel_tls_t.
  *
  * This implements the get_remote_addr method for channel_tls_t; copy the
  * remote endpoint of the channel to addr_out and return 1 (always
  * succeeds for this transport).
  */
-
 static int
 channel_tls_get_remote_addr_method(channel_t *chan, tor_addr_t *addr_out)
 {
@@ -527,8 +533,8 @@ channel_tls_get_remote_addr_method(channel_t *chan, tor_addr_t *addr_out)
  * This implements the get_transport_name for channel_tls_t. If the
  * channel uses a pluggable transport, copy its name to
  * <b>transport_out</b> and return 0. If the channel did not use a
- * pluggable transport, return -1. */
-
+ * pluggable transport, return -1.
+ */
 static int
 channel_tls_get_transport_name_method(channel_t *chan, char **transport_out)
 {
@@ -546,14 +552,13 @@ channel_tls_get_transport_name_method(channel_t *chan, char **transport_out)
 }
 
 /**
- * Get endpoint description of a channel_tls_t
+ * Get endpoint description of a channel_tls_t.
  *
  * This implements the get_remote_descr method for channel_tls_t; it returns
  * a text description of the remote endpoint of the channel suitable for use
- * in log messages.  The req parameter is 0 for the canonical address or 1 for
+ * in log messages. The req parameter is 0 for the canonical address or 1 for
  * the actual address seen.
  */
-
 static const char *
 channel_tls_get_remote_descr_method(channel_t *chan, int flags)
 {
@@ -598,7 +603,7 @@ channel_tls_get_remote_descr_method(channel_t *chan, int flags)
         break;
       default:
         /* Something's broken in channel.c */
-        tor_assert(1);
+        tor_assert_nonfatal_unreached_once();
     }
   } else {
     strlcpy(buf, "(No connection)", sizeof(buf));
@@ -609,12 +614,11 @@ channel_tls_get_remote_descr_method(channel_t *chan, int flags)
 }
 
 /**
- * Tell the upper layer if we have queued writes
+ * Tell the upper layer if we have queued writes.
  *
  * This implements the has_queued_writes method for channel_tls t_; it returns
  * 1 iff we have queued writes on the outbuf of the underlying or_connection_t.
  */
-
 static int
 channel_tls_has_queued_writes_method(channel_t *chan)
 {
@@ -625,8 +629,8 @@ channel_tls_has_queued_writes_method(channel_t *chan)
   if (!(tlschan->conn)) {
     log_info(LD_CHANNEL,
              "something called has_queued_writes on a tlschan "
-             "(%p with ID " U64_FORMAT " but no conn",
-             chan, U64_PRINTF_ARG(chan->global_identifier));
+             "(%p with ID %"PRIu64 " but no conn",
+             chan, (chan->global_identifier));
   }
 
   outbuf_len = (tlschan->conn != NULL) ?
@@ -637,13 +641,12 @@ channel_tls_has_queued_writes_method(channel_t *chan)
 }
 
 /**
- * Tell the upper layer if we're canonical
+ * Tell the upper layer if we're canonical.
  *
  * This implements the is_canonical method for channel_tls_t; if req is zero,
  * it returns whether this is a canonical channel, and if it is one it returns
  * whether that can be relied upon.
  */
-
 static int
 channel_tls_is_canonical_method(channel_t *chan, int req)
 {
@@ -667,7 +670,7 @@ channel_tls_is_canonical_method(channel_t *chan, int req)
         break;
       default:
         /* This shouldn't happen; channel.c is broken if it does */
-        tor_assert(1);
+        tor_assert_nonfatal_unreached_once();
     }
   }
   /* else return 0 for tlschan->conn == NULL */
@@ -676,12 +679,11 @@ channel_tls_is_canonical_method(channel_t *chan, int req)
 }
 
 /**
- * Check if we match an extend_info_t
+ * Check if we match an extend_info_t.
  *
  * This implements the matches_extend_info method for channel_tls_t; the upper
  * layer wants to know if this channel matches an extend_info_t.
  */
-
 static int
 channel_tls_matches_extend_info_method(channel_t *chan,
                                        extend_info_t *extend_info)
@@ -695,8 +697,8 @@ channel_tls_matches_extend_info_method(channel_t *chan,
   if (!(tlschan->conn)) {
     log_info(LD_CHANNEL,
              "something called matches_extend_info on a tlschan "
-             "(%p with ID " U64_FORMAT " but no conn",
-             chan, U64_PRINTF_ARG(chan->global_identifier));
+             "(%p with ID %"PRIu64 " but no conn",
+             chan, (chan->global_identifier));
     return 0;
   }
 
@@ -712,7 +714,6 @@ channel_tls_matches_extend_info_method(channel_t *chan,
  * layer wants to know if this channel matches a target address when extending
  * a circuit.
  */
-
 static int
 channel_tls_matches_target_method(channel_t *chan,
                                   const tor_addr_t *target)
@@ -726,11 +727,20 @@ channel_tls_matches_target_method(channel_t *chan,
   if (!(tlschan->conn)) {
     log_info(LD_CHANNEL,
              "something called matches_target on a tlschan "
-             "(%p with ID " U64_FORMAT " but no conn",
-             chan, U64_PRINTF_ARG(chan->global_identifier));
+             "(%p with ID %"PRIu64 " but no conn",
+             chan, (chan->global_identifier));
     return 0;
   }
 
+  /* real_addr is the address this connection came from.
+   * base_.addr is updated by connection_or_init_conn_from_address()
+   * to be the address in the descriptor. It may be tempting to
+   * allow either address to be allowed, but if we did so, it would
+   * enable someone who steals a relay's keys to impersonate/MITM it
+   * from anywhere on the Internet! (Because they could make long-lived
+   * TLS connections from anywhere to all relays, and wait for them to
+   * be used for extends).
+   */
   return tor_addr_eq(&(tlschan->conn->real_addr), target);
 }
 
@@ -738,7 +748,6 @@ channel_tls_matches_target_method(channel_t *chan,
  * Tell the upper layer how many bytes we have queued and not yet
  * sent.
  */
-
 static size_t
 channel_tls_num_bytes_queued_method(channel_t *chan)
 {
@@ -751,13 +760,12 @@ channel_tls_num_bytes_queued_method(channel_t *chan)
 }
 
 /**
- * Tell the upper layer how many cells we can accept to write
+ * Tell the upper layer how many cells we can accept to write.
  *
  * This implements the num_cells_writeable method for channel_tls_t; it
  * returns an estimate of the number of cells we can accept with
  * channel_tls_write_*_cell().
  */
-
 static int
 channel_tls_num_cells_writeable_method(channel_t *chan)
 {
@@ -782,12 +790,11 @@ channel_tls_num_cells_writeable_method(channel_t *chan)
 }
 
 /**
- * Write a cell to a channel_tls_t
+ * Write a cell to a channel_tls_t.
  *
  * This implements the write_cell method for channel_tls_t; given a
  * channel_tls_t and a cell_t, transmit the cell_t.
  */
-
 static int
 channel_tls_write_cell_method(channel_t *chan, cell_t *cell)
 {
@@ -803,20 +810,22 @@ channel_tls_write_cell_method(channel_t *chan, cell_t *cell)
   } else {
     log_info(LD_CHANNEL,
              "something called write_cell on a tlschan "
-             "(%p with ID " U64_FORMAT " but no conn",
-             chan, U64_PRINTF_ARG(chan->global_identifier));
+             "(%p with ID %"PRIu64 " but no conn",
+             chan, (chan->global_identifier));
   }
 
   return written;
 }
 
 /**
- * Write a packed cell to a channel_tls_t
+ * Write a packed cell to a channel_tls_t.
  *
  * This implements the write_packed_cell method for channel_tls_t; given a
  * channel_tls_t and a packed_cell_t, transmit the packed_cell_t.
+ *
+ * Return 0 on success or negative value on error. The caller must free the
+ * packed cell.
  */
-
 static int
 channel_tls_write_packed_cell_method(channel_t *chan,
                                      packed_cell_t *packed_cell)
@@ -824,35 +833,30 @@ channel_tls_write_packed_cell_method(channel_t *chan,
   tor_assert(chan);
   channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan);
   size_t cell_network_size = get_cell_network_size(chan->wide_circ_ids);
-  int written = 0;
 
   tor_assert(tlschan);
   tor_assert(packed_cell);
 
   if (tlschan->conn) {
-    connection_write_to_buf(packed_cell->body, cell_network_size,
+    connection_buf_add(packed_cell->body, cell_network_size,
                             TO_CONN(tlschan->conn));
-
-    /* This is where the cell is finished; used to be done from relay.c */
-    packed_cell_free(packed_cell);
-    ++written;
   } else {
     log_info(LD_CHANNEL,
              "something called write_packed_cell on a tlschan "
-             "(%p with ID " U64_FORMAT " but no conn",
-             chan, U64_PRINTF_ARG(chan->global_identifier));
+             "(%p with ID %"PRIu64 " but no conn",
+             chan, (chan->global_identifier));
+    return -1;
   }
 
-  return written;
+  return 0;
 }
 
 /**
- * Write a variable-length cell to a channel_tls_t
+ * Write a variable-length cell to a channel_tls_t.
  *
  * This implements the write_var_cell method for channel_tls_t; given a
  * channel_tls_t and a var_cell_t, transmit the var_cell_t.
  */
-
 static int
 channel_tls_write_var_cell_method(channel_t *chan, var_cell_t *var_cell)
 {
@@ -868,8 +872,8 @@ channel_tls_write_var_cell_method(channel_t *chan, var_cell_t *var_cell)
   } else {
     log_info(LD_CHANNEL,
              "something called write_var_cell on a tlschan "
-             "(%p with ID " U64_FORMAT " but no conn",
-             chan, U64_PRINTF_ARG(chan->global_identifier));
+             "(%p with ID %"PRIu64 " but no conn",
+             chan, (chan->global_identifier));
   }
 
   return written;
@@ -880,11 +884,10 @@ channel_tls_write_var_cell_method(channel_t *chan, var_cell_t *var_cell)
  ************************************************/
 
 /**
- * Close a channel_listener_t
+ * Close a channel_listener_t.
  *
- * This implements the close method for channel_listener_t
+ * This implements the close method for channel_listener_t.
  */
-
 static void
 channel_tls_listener_close_method(channel_listener_t *chan_l)
 {
@@ -920,12 +923,11 @@ channel_tls_listener_close_method(channel_listener_t *chan_l)
 }
 
 /**
- * Describe the transport for a channel_listener_t
+ * Describe the transport for a channel_listener_t.
  *
  * This returns the string "TLS channel (listening)" to the upper
  * layer.
  */
-
 static const char *
 channel_tls_listener_describe_transport_method(channel_listener_t *chan_l)
 {
@@ -939,12 +941,11 @@ channel_tls_listener_describe_transport_method(channel_listener_t *chan_l)
  ******************************************************/
 
 /**
- * Handle an orconn state change
+ * Handle an orconn state change.
  *
  * This function will be called by connection_or.c when the or_connection_t
  * associated with this channel_tls_t changes state.
  */
-
 void
 channel_tls_handle_state_change_on_orconn(channel_tls_t *chan,
                                           or_connection_t *conn,
@@ -976,7 +977,7 @@ channel_tls_handle_state_change_on_orconn(channel_tls_t *chan,
      * We can go to CHANNEL_STATE_OPEN from CHANNEL_STATE_OPENING or
      * CHANNEL_STATE_MAINT on this.
      */
-    channel_change_state(base_chan, CHANNEL_STATE_OPEN);
+    channel_change_state_open(base_chan);
     /* We might have just become writeable; check and tell the scheduler */
     if (connection_or_num_cells_writeable(conn) > 0) {
       scheduler_channel_wants_writes(base_chan);
@@ -995,13 +996,12 @@ channel_tls_handle_state_change_on_orconn(channel_tls_t *chan,
 #ifdef KEEP_TIMING_STATS
 
 /**
- * Timing states wrapper
+ * Timing states wrapper.
  *
  * This is a wrapper function around the actual function that processes the
  * <b>cell</b> that just arrived on <b>chan</b>. Increment <b>*time</b>
  * by the number of microseconds used by the call to <b>*func(cell, chan)</b>.
  */
-
 static void
 channel_tls_time_process_cell(cell_t *cell, channel_tls_t *chan, int *time,
                               void (*func)(cell_t *, channel_tls_t *))
@@ -1027,10 +1027,10 @@ channel_tls_time_process_cell(cell_t *cell, channel_tls_t *chan, int *time,
 
   *time += time_passed;
 }
-#endif
+#endif /* defined(KEEP_TIMING_STATS) */
 
 /**
- * Handle an incoming cell on a channel_tls_t
+ * Handle an incoming cell on a channel_tls_t.
  *
  * This is called from connection_or.c to handle an arriving cell; it checks
  * for cell types specific to the handshake for this transport protocol and
@@ -1042,7 +1042,6 @@ channel_tls_time_process_cell(cell_t *cell, channel_tls_t *chan, int *time,
  * for copying in the case that it queues; we merely pass pointers through
  * which we get from connection_or_process_cells_from_inbuf().
  */
-
 void
 channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
 {
@@ -1055,9 +1054,9 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
     channel_tls_time_process_cell(cl, cn, & tp ## time ,            \
                              channel_tls_process_ ## tp ## _cell);  \
     } STMT_END
-#else
+#else /* !(defined(KEEP_TIMING_STATS)) */
 #define PROCESS_CELL(tp, cl, cn) channel_tls_process_ ## tp ## _cell(cl, cn)
-#endif
+#endif /* defined(KEEP_TIMING_STATS) */
 
   tor_assert(cell);
   tor_assert(conn);
@@ -1092,8 +1091,19 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
   if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3)
     or_handshake_state_record_cell(conn, conn->handshake_state, cell, 1);
 
+  /* We note that we're on the internet whenever we read a cell. This is
+   * a fast operation. */
+  entry_guards_note_internet_connectivity(get_guard_selection_info());
+  rep_hist_padding_count_read(PADDING_TYPE_TOTAL);
+
+  if (TLS_CHAN_TO_BASE(chan)->currently_padding)
+    rep_hist_padding_count_read(PADDING_TYPE_ENABLED_TOTAL);
+
   switch (cell->command) {
     case CELL_PADDING:
+      rep_hist_padding_count_read(PADDING_TYPE_CELL);
+      if (TLS_CHAN_TO_BASE(chan)->currently_padding)
+        rep_hist_padding_count_read(PADDING_TYPE_ENABLED_CELL);
       ++stats_n_padding_cells_processed;
       /* do nothing */
       break;
@@ -1112,6 +1122,10 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
       ++stats_n_netinfo_cells_processed;
       PROCESS_CELL(netinfo, cell, chan);
       break;
+    case CELL_PADDING_NEGOTIATE:
+      ++stats_n_netinfo_cells_processed;
+      PROCESS_CELL(padding_negotiate, cell, chan);
+      break;
     case CELL_CREATE:
     case CELL_CREATE_FAST:
     case CELL_CREATED:
@@ -1125,7 +1139,7 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
        * These are all transport independent and we pass them up through the
        * channel_t mechanism.  They are ultimately handled in command.c.
        */
-      channel_queue_cell(TLS_CHAN_TO_BASE(chan), cell);
+      channel_process_cell(TLS_CHAN_TO_BASE(chan), cell);
       break;
     default:
       log_fn(LOG_INFO, LD_PROTOCOL,
@@ -1137,7 +1151,7 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
 }
 
 /**
- * Handle an incoming variable-length cell on a channel_tls_t
+ * Handle an incoming variable-length cell on a channel_tls_t.
  *
  * Process a <b>var_cell</b> that was just received on <b>conn</b>. Keep
  * internal statistics about how many of each cell we've processed so far
@@ -1153,7 +1167,6 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
  * caller always frees them after this function returns, so this function
  * should never free var_cell.
  */
-
 void
 channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn)
 {
@@ -1180,7 +1193,7 @@ channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn)
     /* remember which second it is, for next time */
     current_second = now;
   }
-#endif
+#endif /* defined(KEEP_TIMING_STATS) */
 
   tor_assert(var_cell);
   tor_assert(conn);
@@ -1278,6 +1291,10 @@ channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn)
       return;
   }
 
+  /* We note that we're on the internet whenever we read a cell. This is
+   * a fast operation. */
+  entry_guards_note_internet_connectivity(get_guard_selection_info());
+
   /* Now handle the cell */
 
   switch (var_cell->command) {
@@ -1314,7 +1331,7 @@ channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn)
 }
 
 /**
- * Update channel marks after connection_or.c has changed an address
+ * Update channel marks after connection_or.c has changed an address.
  *
  * This is called from connection_or_init_conn_from_address() after the
  * connection's _base.addr or real_addr fields have potentially been changed
@@ -1323,7 +1340,6 @@ channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn)
  * remote router by looking it up in the consensus after we finish the
  * handshake and know an authenticated identity digest.
  */
-
 void
 channel_tls_update_marks(or_connection_t *conn)
 {
@@ -1337,27 +1353,26 @@ channel_tls_update_marks(or_connection_t *conn)
   if (is_local_addr(&(TO_CONN(conn)->addr))) {
     if (!channel_is_local(chan)) {
       log_debug(LD_CHANNEL,
-                "Marking channel " U64_FORMAT " at %p as local",
-                U64_PRINTF_ARG(chan->global_identifier), chan);
+                "Marking channel %"PRIu64 " at %p as local",
+                (chan->global_identifier), chan);
       channel_mark_local(chan);
     }
   } else {
     if (channel_is_local(chan)) {
       log_debug(LD_CHANNEL,
-                "Marking channel " U64_FORMAT " at %p as remote",
-                U64_PRINTF_ARG(chan->global_identifier), chan);
+                "Marking channel %"PRIu64 " at %p as remote",
+                (chan->global_identifier), chan);
       channel_mark_remote(chan);
     }
   }
 }
 
 /**
- * Check if this cell type is allowed before the handshake is finished
+ * Check if this cell type is allowed before the handshake is finished.
  *
  * Return true if <b>command</b> is a cell command that's allowed to start a
  * V3 handshake.
  */
-
 static int
 command_allowed_before_handshake(uint8_t command)
 {
@@ -1372,14 +1387,13 @@ command_allowed_before_handshake(uint8_t command)
 }
 
 /**
- * Start a V3 handshake on an incoming connection
+ * Start a V3 handshake on an incoming connection.
  *
  * Called when we as a server receive an appropriate cell while waiting
  * either for a cell or a TLS handshake.  Set the connection's state to
  * "handshaking_v3', initializes the or_handshake_state field as needed,
  * and add the cell to the hash of incoming cells.)
  */
-
 static int
 enter_v3_handshake_with_cell(var_cell_t *cell, channel_tls_t *chan)
 {
@@ -1420,7 +1434,6 @@ enter_v3_handshake_with_cell(var_cell_t *cell, channel_tls_t *chan)
  * we support, pick the highest version we have in common, and continue the
  * negotiation from there.
  */
-
 static void
 channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan)
 {
@@ -1551,7 +1564,7 @@ channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan)
       connection_or_close_normally(chan->conn, 1);
       return;
     }
-#endif
+#endif /* defined(DISABLE_V3_LINKPROTO_SERVERSIDE) */
 
     if (send_versions) {
       if (connection_or_send_versions(chan->conn, 1) < 0) {
@@ -1561,11 +1574,14 @@ channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan)
       }
     }
 
-    /* We set this after sending the verions cell. */
+    /* We set this after sending the versions cell. */
     /*XXXXX symbolic const.*/
-    chan->base_.wide_circ_ids =
+    TLS_CHAN_TO_BASE(chan)->wide_circ_ids =
       chan->conn->link_proto >= MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS;
-    chan->conn->wide_circ_ids = chan->base_.wide_circ_ids;
+    chan->conn->wide_circ_ids = TLS_CHAN_TO_BASE(chan)->wide_circ_ids;
+
+    TLS_CHAN_TO_BASE(chan)->padding_enabled =
+      chan->conn->link_proto >= MIN_LINK_PROTO_FOR_CHANNEL_PADDING;
 
     if (send_certs) {
       if (connection_or_send_certs_cell(chan->conn) < 0) {
@@ -1592,6 +1608,43 @@ channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan)
 }
 
 /**
+ * Process a 'padding_negotiate' cell.
+ *
+ * This function is called to handle an incoming PADDING_NEGOTIATE cell;
+ * enable or disable padding accordingly, and read and act on its timeout
+ * value contents.
+ */
+static void
+channel_tls_process_padding_negotiate_cell(cell_t *cell, channel_tls_t *chan)
+{
+  channelpadding_negotiate_t *negotiation;
+  tor_assert(cell);
+  tor_assert(chan);
+  tor_assert(chan->conn);
+
+  if (chan->conn->link_proto < MIN_LINK_PROTO_FOR_CHANNEL_PADDING) {
+    log_fn(LOG_PROTOCOL_WARN, LD_OR,
+           "Received a PADDING_NEGOTIATE cell on v%d connection; dropping.",
+           chan->conn->link_proto);
+    return;
+  }
+
+  if (channelpadding_negotiate_parse(&negotiation, cell->payload,
+                                     CELL_PAYLOAD_SIZE) < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_OR,
+          "Received malformed PADDING_NEGOTIATE cell on v%d connection; "
+          "dropping.", chan->conn->link_proto);
+
+    return;
+  }
+
+  channelpadding_update_padding_for_channel(TLS_CHAN_TO_BASE(chan),
+                                            negotiation);
+
+  channelpadding_negotiate_free(negotiation);
+}
+
+/**
  * Helper: compute the absolute value of a time_t.
  *
  * (we need this because labs() doesn't always work for time_t, since
@@ -1609,7 +1662,6 @@ time_abs(time_t val)
  * This function is called to handle an incoming NETINFO cell; read and act
  * on its contents, and set the connection state to "open".
  */
-
 static void
 channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
 {
@@ -1620,9 +1672,12 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
   const uint8_t *cp, *end;
   uint8_t n_other_addrs;
   time_t now = time(NULL);
+  const routerinfo_t *me = router_get_my_routerinfo();
 
   time_t apparent_skew = 0;
   tor_addr_t my_apparent_addr = TOR_ADDR_NULL;
+  int started_here = 0;
+  const char *identity_digest = NULL;
 
   tor_assert(cell);
   tor_assert(chan);
@@ -1642,10 +1697,12 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
   }
   tor_assert(chan->conn->handshake_state &&
              chan->conn->handshake_state->received_versions);
+  started_here = connection_or_nonopen_was_started_here(chan->conn);
+  identity_digest = chan->conn->identity_digest;
 
   if (chan->conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3) {
     tor_assert(chan->conn->link_proto >= 3);
-    if (chan->conn->handshake_state->started_here) {
+    if (started_here) {
       if (!(chan->conn->handshake_state->authenticated)) {
         log_fn(LOG_PROTOCOL_WARN, LD_OR,
                "Got a NETINFO cell from server, "
@@ -1659,7 +1716,10 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
       if (!(chan->conn->handshake_state->authenticated)) {
         tor_assert(tor_digest_is_zero(
                   (const char*)(chan->conn->handshake_state->
-                      authenticated_peer_id)));
+                      authenticated_rsa_peer_id)));
+        tor_assert(tor_mem_is_zero(
+                  (const char*)(chan->conn->handshake_state->
+                                authenticated_ed25519_peer_id.pubkey), 32));
         /* If the client never authenticated, it's a tor client or bridge
          * relay, and we must not use it for EXTEND requests (nor could we, as
          * there are no authenticated peer IDs) */
@@ -1670,8 +1730,10 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
         connection_or_init_conn_from_address(chan->conn,
                   &(chan->conn->base_.addr),
                   chan->conn->base_.port,
+                  /* zero, checked above */
                   (const char*)(chan->conn->handshake_state->
-                   authenticated_peer_id),
+                                authenticated_rsa_peer_id),
+                  NULL, /* Ed25519 ID: Also checked as zero */
                   0);
       }
     }
@@ -1701,8 +1763,20 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
 
   if (my_addr_type == RESOLVED_TYPE_IPV4 && my_addr_len == 4) {
     tor_addr_from_ipv4n(&my_apparent_addr, get_uint32(my_addr_ptr));
+
+    if (!get_options()->BridgeRelay && me &&
+        get_uint32(my_addr_ptr) == htonl(me->addr)) {
+      TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer = 1;
+    }
+
   } else if (my_addr_type == RESOLVED_TYPE_IPV6 && my_addr_len == 16) {
     tor_addr_from_ipv6_bytes(&my_apparent_addr, (const char *) my_addr_ptr);
+
+    if (!get_options()->BridgeRelay && me &&
+        !tor_addr_is_null(&me->ipv6_addr) &&
+        tor_addr_eq(&my_apparent_addr, &me->ipv6_addr)) {
+      TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer = 1;
+    }
   }
 
   n_other_addrs = (uint8_t) *cp++;
@@ -1718,6 +1792,13 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
       connection_or_close_for_error(chan->conn, 0);
       return;
     }
+    /* A relay can connect from anywhere and be canonical, so
+     * long as it tells you from where it came. This may sound a bit
+     * concerning... but that's what "canonical" means: that the
+     * address is one that the relay itself has claimed.  The relay
+     * might be doing something funny, but nobody else is doing a MITM
+     * on the relay's TCP.
+     */
     if (tor_addr_eq(&addr, &(chan->conn->real_addr))) {
       connection_or_set_canonical(chan->conn, 1);
       break;
@@ -1726,11 +1807,27 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
     --n_other_addrs;
   }
 
+  if (me && !TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer &&
+      channel_is_canonical(TLS_CHAN_TO_BASE(chan))) {
+    const char *descr =
+      TLS_CHAN_TO_BASE(chan)->get_remote_descr(TLS_CHAN_TO_BASE(chan), 0);
+    log_info(LD_OR,
+             "We made a connection to a relay at %s (fp=%s) but we think "
+             "they will not consider this connection canonical. They "
+             "think we are at %s, but we think its %s.",
+             safe_str(descr),
+             safe_str(hex_str(identity_digest, DIGEST_LEN)),
+             safe_str(tor_addr_is_null(&my_apparent_addr) ?
+             "<none>" : fmt_and_decorate_addr(&my_apparent_addr)),
+             safe_str(fmt_addr32(me->addr)));
+  }
+
   /* Act on apparent skew. */
   /** Warn when we get a netinfo skew with at least this value. */
 #define NETINFO_NOTICE_SKEW 3600
   if (time_abs(apparent_skew) > NETINFO_NOTICE_SKEW &&
-      router_get_by_id_digest(chan->conn->identity_digest)) {
+      (started_here ||
+       connection_or_digest_is_known_relay(chan->conn->identity_digest))) {
     int trusted = router_digest_is_trusted_dir(chan->conn->identity_digest);
     clock_skew_warning(TO_CONN(chan->conn), apparent_skew, trusted, LD_GENERAL,
                        "NETINFO cell", "OR");
@@ -1764,14 +1861,49 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
              safe_str_client(chan->conn->base_.address),
              chan->conn->base_.port,
              (int)(chan->conn->link_proto),
-             hex_str(TLS_CHAN_TO_BASE(chan)->identity_digest,
-                     DIGEST_LEN),
+             hex_str(identity_digest, DIGEST_LEN),
              tor_addr_is_null(&my_apparent_addr) ?
-             "<none>" : fmt_and_decorate_addr(&my_apparent_addr));
+               "<none>" :
+               safe_str_client(fmt_and_decorate_addr(&my_apparent_addr)));
   }
   assert_connection_ok(TO_CONN(chan->conn),time(NULL));
 }
 
+/** Types of certificates that we know how to parse from CERTS cells.  Each
+ * type corresponds to a different encoding format. */
+typedef enum cert_encoding_t {
+  CERT_ENCODING_UNKNOWN, /**< We don't recognize this. */
+  CERT_ENCODING_X509, /**< It's an RSA key, signed with RSA, encoded in x509.
+                   * (Actually, it might not be RSA. We test that later.) */
+  CERT_ENCODING_ED25519, /**< It's something signed with an Ed25519 key,
+                      * encoded asa a tor_cert_t.*/
+  CERT_ENCODING_RSA_CROSSCERT, /**< It's an Ed key signed with an RSA key. */
+} cert_encoding_t;
+
+/**
+ * Given one of the certificate type codes used in a CERTS cell,
+ * return the corresponding cert_encoding_t that we should use to parse
+ * the certificate.
+ */
+static cert_encoding_t
+certs_cell_typenum_to_cert_type(int typenum)
+{
+  switch (typenum) {
+  case CERTTYPE_RSA1024_ID_LINK:
+  case CERTTYPE_RSA1024_ID_ID:
+  case CERTTYPE_RSA1024_ID_AUTH:
+    return CERT_ENCODING_X509;
+  case CERTTYPE_ED_ID_SIGN:
+  case CERTTYPE_ED_SIGN_LINK:
+  case CERTTYPE_ED_SIGN_AUTH:
+    return CERT_ENCODING_ED25519;
+  case CERTTYPE_RSA1024_ID_EDID:
+    return CERT_ENCODING_RSA_CROSSCERT;
+  default:
+    return CERT_ENCODING_UNKNOWN;
+  }
+}
+
 /**
  * Process a CERTS cell from a channel.
  *
@@ -1787,18 +1919,24 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
  * of the connection, we then authenticate the server or mark the connection.
  * If it's the server side, wait for an AUTHENTICATE cell.
  */
-
 STATIC void
 channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan)
 {
-#define MAX_CERT_TYPE_WANTED OR_CERT_TYPE_AUTH_1024
-  tor_x509_cert_t *certs[MAX_CERT_TYPE_WANTED + 1];
+#define MAX_CERT_TYPE_WANTED CERTTYPE_RSA1024_ID_EDID
+  /* These arrays will be sparse, since a cert type can be at most one
+   * of ed/x509 */
+  tor_x509_cert_t *x509_certs[MAX_CERT_TYPE_WANTED + 1];
+  tor_cert_t *ed_certs[MAX_CERT_TYPE_WANTED + 1];
+  uint8_t *rsa_ed_cc_cert = NULL;
+  size_t rsa_ed_cc_cert_len = 0;
+
   int n_certs, i;
   certs_cell_t *cc = NULL;
 
-  int send_netinfo = 0;
+  int send_netinfo = 0, started_here = 0;
 
-  memset(certs, 0, sizeof(certs));
+  memset(x509_certs, 0, sizeof(x509_certs));
+  memset(ed_certs, 0, sizeof(ed_certs));
   tor_assert(cell);
   tor_assert(chan);
   tor_assert(chan->conn);
@@ -1813,6 +1951,11 @@ channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan)
     goto err;                                                   \
   } while (0)
 
+  /* Can't use connection_or_nonopen_was_started_here(); its conn->tls
+   * check looks like it breaks
+   * test_link_handshake_recv_certs_ok_server().  */
+  started_here = chan->conn->handshake_state->started_here;
+
   if (chan->conn->base_.state != OR_CONN_STATE_OR_HANDSHAKING_V3)
     ERR("We're not doing a v3 handshake!");
   if (chan->conn->link_proto < 3)
@@ -1842,77 +1985,149 @@ channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan)
 
     if (cert_type > MAX_CERT_TYPE_WANTED)
       continue;
+    const cert_encoding_t ct = certs_cell_typenum_to_cert_type(cert_type);
+    switch (ct) {
+      default:
+      case CERT_ENCODING_UNKNOWN:
+        break;
+      case CERT_ENCODING_X509: {
+        tor_x509_cert_t *x509_cert = tor_x509_cert_decode(cert_body, cert_len);
+        if (!x509_cert) {
+          log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+                 "Received undecodable certificate in CERTS cell from %s:%d",
+                 safe_str(chan->conn->base_.address),
+               chan->conn->base_.port);
+        } else {
+          if (x509_certs[cert_type]) {
+            tor_x509_cert_free(x509_cert);
+            ERR("Duplicate x509 certificate");
+          } else {
+            x509_certs[cert_type] = x509_cert;
+          }
+        }
+        break;
+      }
+      case CERT_ENCODING_ED25519: {
+        tor_cert_t *ed_cert = tor_cert_parse(cert_body, cert_len);
+        if (!ed_cert) {
+          log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+                 "Received undecodable Ed certificate "
+                 "in CERTS cell from %s:%d",
+                 safe_str(chan->conn->base_.address),
+               chan->conn->base_.port);
+        } else {
+          if (ed_certs[cert_type]) {
+            tor_cert_free(ed_cert);
+            ERR("Duplicate Ed25519 certificate");
+          } else {
+            ed_certs[cert_type] = ed_cert;
+          }
+        }
+        break;
+      }
 
-    tor_x509_cert_t *cert = tor_x509_cert_decode(cert_body, cert_len);
-    if (!cert) {
-      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-             "Received undecodable certificate in CERTS cell from %s:%d",
-             safe_str(chan->conn->base_.address),
-             chan->conn->base_.port);
-    } else {
-      if (certs[cert_type]) {
-        tor_x509_cert_free(cert);
-        ERR("Duplicate x509 certificate");
-      } else {
-        certs[cert_type] = cert;
+     case CERT_ENCODING_RSA_CROSSCERT: {
+        if (rsa_ed_cc_cert) {
+          ERR("Duplicate RSA->Ed25519 crosscert");
+        } else {
+          rsa_ed_cc_cert = tor_memdup(cert_body, cert_len);
+          rsa_ed_cc_cert_len = cert_len;
+        }
+        break;
       }
     }
   }
 
-  tor_x509_cert_t *id_cert = certs[OR_CERT_TYPE_ID_1024];
-  tor_x509_cert_t *auth_cert = certs[OR_CERT_TYPE_AUTH_1024];
-  tor_x509_cert_t *link_cert = certs[OR_CERT_TYPE_TLS_LINK];
+  /* Move the certificates we (might) want into the handshake_state->certs
+   * structure. */
+  tor_x509_cert_t *id_cert = x509_certs[CERTTYPE_RSA1024_ID_ID];
+  tor_x509_cert_t *auth_cert = x509_certs[CERTTYPE_RSA1024_ID_AUTH];
+  tor_x509_cert_t *link_cert = x509_certs[CERTTYPE_RSA1024_ID_LINK];
+  chan->conn->handshake_state->certs->auth_cert = auth_cert;
+  chan->conn->handshake_state->certs->link_cert = link_cert;
+  chan->conn->handshake_state->certs->id_cert = id_cert;
+  x509_certs[CERTTYPE_RSA1024_ID_ID] =
+    x509_certs[CERTTYPE_RSA1024_ID_AUTH] =
+    x509_certs[CERTTYPE_RSA1024_ID_LINK] = NULL;
+
+  tor_cert_t *ed_id_sign = ed_certs[CERTTYPE_ED_ID_SIGN];
+  tor_cert_t *ed_sign_link = ed_certs[CERTTYPE_ED_SIGN_LINK];
+  tor_cert_t *ed_sign_auth = ed_certs[CERTTYPE_ED_SIGN_AUTH];
+  chan->conn->handshake_state->certs->ed_id_sign = ed_id_sign;
+  chan->conn->handshake_state->certs->ed_sign_link = ed_sign_link;
+  chan->conn->handshake_state->certs->ed_sign_auth = ed_sign_auth;
+  ed_certs[CERTTYPE_ED_ID_SIGN] =
+    ed_certs[CERTTYPE_ED_SIGN_LINK] =
+    ed_certs[CERTTYPE_ED_SIGN_AUTH] = NULL;
+
+  chan->conn->handshake_state->certs->ed_rsa_crosscert = rsa_ed_cc_cert;
+  chan->conn->handshake_state->certs->ed_rsa_crosscert_len =
+    rsa_ed_cc_cert_len;
+  rsa_ed_cc_cert = NULL;
+
+  int severity;
+  /* Note that this warns more loudly about time and validity if we were
+   * _trying_ to connect to an authority, not necessarily if we _did_ connect
+   * to one. */
+  if (started_here &&
+      router_digest_is_trusted_dir(TLS_CHAN_TO_BASE(chan)->identity_digest))
+    severity = LOG_WARN;
+  else
+    severity = LOG_PROTOCOL_WARN;
+
+  const ed25519_public_key_t *checked_ed_id = NULL;
+  const common_digests_t *checked_rsa_id = NULL;
+  or_handshake_certs_check_both(severity,
+                                chan->conn->handshake_state->certs,
+                                chan->conn->tls,
+                                time(NULL),
+                                &checked_ed_id,
+                                &checked_rsa_id);
+
+  if (!checked_rsa_id)
+    ERR("Invalid certificate chain!");
 
-  if (chan->conn->handshake_state->started_here) {
-    int severity;
-    if (! (id_cert && link_cert))
-      ERR("The certs we wanted were missing");
-    /* Okay. We should be able to check the certificates now. */
-    if (! tor_tls_cert_matches_key(chan->conn->tls, link_cert)) {
-      ERR("The link certificate didn't match the TLS public key");
-    }
-    /* Note that this warns more loudly about time and validity if we were
-    * _trying_ to connect to an authority, not necessarily if we _did_ connect
-    * to one. */
-    if (router_digest_is_trusted_dir(
-          TLS_CHAN_TO_BASE(chan)->identity_digest))
-      severity = LOG_WARN;
-    else
-      severity = LOG_PROTOCOL_WARN;
-
-    if (! tor_tls_cert_is_valid(severity, link_cert, id_cert, 0))
-      ERR("The link certificate was not valid");
-    if (! tor_tls_cert_is_valid(severity, id_cert, id_cert, 1))
-      ERR("The ID certificate was not valid");
+  if (started_here) {
+    /* No more information is needed. */
 
     chan->conn->handshake_state->authenticated = 1;
+    chan->conn->handshake_state->authenticated_rsa = 1;
     {
-      const common_digests_t *id_digests =
-        tor_x509_cert_get_id_digests(id_cert);
+      const common_digests_t *id_digests = checked_rsa_id;
       crypto_pk_t *identity_rcvd;
       if (!id_digests)
         ERR("Couldn't compute digests for key in ID cert");
 
       identity_rcvd = tor_tls_cert_get_key(id_cert);
-      if (!identity_rcvd)
-        ERR("Internal error: Couldn't get RSA key from ID cert.");
-      memcpy(chan->conn->handshake_state->authenticated_peer_id,
+      if (!identity_rcvd) {
+        ERR("Couldn't get RSA key from ID cert.");
+      }
+      memcpy(chan->conn->handshake_state->authenticated_rsa_peer_id,
              id_digests->d[DIGEST_SHA1], DIGEST_LEN);
       channel_set_circid_type(TLS_CHAN_TO_BASE(chan), identity_rcvd,
                 chan->conn->link_proto < MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS);
       crypto_pk_free(identity_rcvd);
     }
 
+    if (checked_ed_id) {
+      chan->conn->handshake_state->authenticated_ed25519 = 1;
+      memcpy(&chan->conn->handshake_state->authenticated_ed25519_peer_id,
+             checked_ed_id, sizeof(ed25519_public_key_t));
+    }
+
+    log_debug(LD_HANDSHAKE, "calling client_learned_peer_id from "
+              "process_certs_cell");
+
     if (connection_or_client_learned_peer_id(chan->conn,
-            chan->conn->handshake_state->authenticated_peer_id) < 0)
+                  chan->conn->handshake_state->authenticated_rsa_peer_id,
+                  checked_ed_id) < 0)
       ERR("Problem setting or checking peer id");
 
-    log_info(LD_OR,
-             "Got some good certificates from %s:%d: Authenticated it.",
-             safe_str(chan->conn->base_.address), chan->conn->base_.port);
-
-    chan->conn->handshake_state->id_cert = id_cert;
-    certs[OR_CERT_TYPE_ID_1024] = NULL;
+    log_info(LD_HANDSHAKE,
+             "Got some good certificates from %s:%d: Authenticated it with "
+             "RSA%s",
+             safe_str(chan->conn->base_.address), chan->conn->base_.port,
+             checked_ed_id ? " and Ed25519" : "");
 
     if (!public_server_mode(get_options())) {
       /* If we initiated the connection and we are not a public server, we
@@ -1921,25 +2136,14 @@ channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan)
       send_netinfo = 1;
     }
   } else {
-    if (! (id_cert && auth_cert))
-      ERR("The certs we wanted were missing");
-
-    /* Remember these certificates so we can check an AUTHENTICATE cell */
-    if (! tor_tls_cert_is_valid(LOG_PROTOCOL_WARN, auth_cert, id_cert, 1))
-      ERR("The authentication certificate was not valid");
-    if (! tor_tls_cert_is_valid(LOG_PROTOCOL_WARN, id_cert, id_cert, 1))
-      ERR("The ID certificate was not valid");
-
+    /* We can't call it authenticated till we see an AUTHENTICATE cell. */
     log_info(LD_OR,
-             "Got some good certificates from %s:%d: "
+             "Got some good RSA%s certificates from %s:%d. "
              "Waiting for AUTHENTICATE.",
+             checked_ed_id ? " and Ed25519" : "",
              safe_str(chan->conn->base_.address),
              chan->conn->base_.port);
     /* XXXX check more stuff? */
-
-    chan->conn->handshake_state->id_cert = id_cert;
-    chan->conn->handshake_state->auth_cert = auth_cert;
-    certs[OR_CERT_TYPE_ID_1024] = certs[OR_CERT_TYPE_AUTH_1024] = NULL;
   }
 
   chan->conn->handshake_state->received_certs_cell = 1;
@@ -1953,15 +2157,19 @@ channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan)
   }
 
  err:
-  for (unsigned u = 0; u < ARRAY_LENGTH(certs); ++u) {
-    tor_x509_cert_free(certs[u]);
+  for (unsigned u = 0; u < ARRAY_LENGTH(x509_certs); ++u) {
+    tor_x509_cert_free(x509_certs[u]);
+  }
+  for (unsigned u = 0; u < ARRAY_LENGTH(ed_certs); ++u) {
+    tor_cert_free(ed_certs[u]);
   }
+  tor_free(rsa_ed_cc_cert);
   certs_cell_free(cc);
 #undef ERR
 }
 
 /**
- * Process an AUTH_CHALLENGE cell from a channel_tls_t
+ * Process an AUTH_CHALLENGE cell from a channel_tls_t.
  *
  * This function is called to handle an incoming AUTH_CHALLENGE cell on a
  * channel_tls_t; if we weren't supposed to get one (for example, because we're
@@ -1970,7 +2178,6 @@ channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan)
  * want to authenticate, just drop it.  If the cell is well-formed *and* we
  * want to authenticate, send an AUTHENTICATE cell and then a NETINFO cell.
  */
-
 STATIC void
 channel_tls_process_auth_challenge_cell(var_cell_t *cell, channel_tls_t *chan)
 {
@@ -2012,8 +2219,12 @@ channel_tls_process_auth_challenge_cell(var_cell_t *cell, channel_tls_t *chan)
   /* Now see if there is an authentication type we can use */
   for (i = 0; i < n_types; ++i) {
     uint16_t authtype = auth_challenge_cell_get_methods(ac, i);
-    if (authtype == AUTHTYPE_RSA_SHA256_TLSSECRET)
-      use_type = authtype;
+    if (authchallenge_type_is_supported(authtype)) {
+      if (use_type == -1 ||
+          authchallenge_type_is_better(authtype, use_type)) {
+        use_type = authtype;
+      }
+    }
   }
 
   chan->conn->handshake_state->received_auth_challenge = 1;
@@ -2028,9 +2239,10 @@ channel_tls_process_auth_challenge_cell(var_cell_t *cell, channel_tls_t *chan)
   if (use_type >= 0) {
     log_info(LD_OR,
              "Got an AUTH_CHALLENGE cell from %s:%d: Sending "
-             "authentication",
+             "authentication type %d",
              safe_str(chan->conn->base_.address),
-             chan->conn->base_.port);
+             chan->conn->base_.port,
+             use_type);
 
     if (connection_or_send_authenticate_cell(chan->conn, use_type) < 0) {
       log_warn(LD_OR,
@@ -2059,7 +2271,7 @@ channel_tls_process_auth_challenge_cell(var_cell_t *cell, channel_tls_t *chan)
 }
 
 /**
- * Process an AUTHENTICATE cell from a channel_tls_t
+ * Process an AUTHENTICATE cell from a channel_tls_t.
  *
  * If it's ill-formed or we weren't supposed to get one or we're not doing a
  * v3 handshake, then mark the connection.  If it does not authenticate the
@@ -2067,13 +2279,14 @@ channel_tls_process_auth_challenge_cell(var_cell_t *cell, channel_tls_t *chan)
  * we didn't get a CERTS cell, etc) mark the connection.  Otherwise, accept
  * the identity of the router on the other side of the connection.
  */
-
 STATIC void
 channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan)
 {
-  uint8_t expected[V3_AUTH_FIXED_PART_LEN+256];
+  var_cell_t *expected_cell = NULL;
   const uint8_t *auth;
   int authlen;
+  int authtype;
+  int bodylen;
 
   tor_assert(cell);
   tor_assert(chan);
@@ -2086,6 +2299,7 @@ channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan)
            safe_str(chan->conn->base_.address),                 \
            chan->conn->base_.port, (s));                        \
     connection_or_close_for_error(chan->conn, 0);               \
+    var_cell_free(expected_cell);                               \
     return;                                                     \
   } while (0)
 
@@ -2103,9 +2317,7 @@ channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan)
   }
   if (!(chan->conn->handshake_state->received_certs_cell))
     ERR("We never got a certs cell");
-  if (chan->conn->handshake_state->auth_cert == NULL)
-    ERR("We never got an authentication certificate");
-  if (chan->conn->handshake_state->id_cert == NULL)
+  if (chan->conn->handshake_state->certs->id_cert == NULL)
     ERR("We never got an identity certificate");
   if (cell->payload_len < 4)
     ERR("Cell was way too short");
@@ -2117,8 +2329,9 @@ channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan)
     if (4 + len > cell->payload_len)
       ERR("Authenticator was truncated");
 
-    if (type != AUTHTYPE_RSA_SHA256_TLSSECRET)
+    if (! authchallenge_type_is_supported(type))
       ERR("Authenticator type was not recognized");
+    authtype = type;
 
     auth += 4;
     authlen = len;
@@ -2127,25 +2340,55 @@ channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan)
   if (authlen < V3_AUTH_BODY_LEN + 1)
     ERR("Authenticator was too short");
 
-  ssize_t bodylen =
-    connection_or_compute_authenticate_cell_body(
-                        chan->conn, expected, sizeof(expected), NULL, 1);
-  if (bodylen < 0 || bodylen != V3_AUTH_FIXED_PART_LEN)
+  expected_cell = connection_or_compute_authenticate_cell_body(
+                chan->conn, authtype, NULL, NULL, 1);
+  if (! expected_cell)
     ERR("Couldn't compute expected AUTHENTICATE cell body");
 
-  if (tor_memneq(expected, auth, bodylen))
+  int sig_is_rsa;
+  if (authtype == AUTHTYPE_RSA_SHA256_TLSSECRET ||
+      authtype == AUTHTYPE_RSA_SHA256_RFC5705) {
+    bodylen = V3_AUTH_BODY_LEN;
+    sig_is_rsa = 1;
+  } else {
+    tor_assert(authtype == AUTHTYPE_ED25519_SHA256_RFC5705);
+    /* Our earlier check had better have made sure we had room
+     * for an ed25519 sig (inadvertently) */
+    tor_assert(V3_AUTH_BODY_LEN > ED25519_SIG_LEN);
+    bodylen = authlen - ED25519_SIG_LEN;
+    sig_is_rsa = 0;
+  }
+  if (expected_cell->payload_len != bodylen+4) {
+    ERR("Expected AUTHENTICATE cell body len not as expected.");
+  }
+
+  /* Length of random part. */
+  if (BUG(bodylen < 24)) {
+    // LCOV_EXCL_START
+    ERR("Bodylen is somehow less than 24, which should really be impossible");
+    // LCOV_EXCL_STOP
+  }
+
+  if (tor_memneq(expected_cell->payload+4, auth, bodylen-24))
     ERR("Some field in the AUTHENTICATE cell body was not as expected");
 
-  {
+  if (sig_is_rsa) {
+    if (chan->conn->handshake_state->certs->ed_id_sign != NULL)
+      ERR("RSA-signed AUTHENTICATE response provided with an ED25519 cert");
+
+    if (chan->conn->handshake_state->certs->auth_cert == NULL)
+      ERR("We never got an RSA authentication certificate");
+
     crypto_pk_t *pk = tor_tls_cert_get_key(
-                                   chan->conn->handshake_state->auth_cert);
+                             chan->conn->handshake_state->certs->auth_cert);
     char d[DIGEST256_LEN];
     char *signed_data;
     size_t keysize;
     int signed_len;
 
-    if (!pk)
-      ERR("Internal error: couldn't get RSA key from AUTH cert.");
+    if (! pk) {
+      ERR("Couldn't get RSA key from AUTH cert.");
+    }
     crypto_digest256(d, (char*)auth, V3_AUTH_BODY_LEN, DIGEST_SHA256);
 
     keysize = crypto_pk_keysize(pk);
@@ -2156,7 +2399,7 @@ channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan)
     crypto_pk_free(pk);
     if (signed_len < 0) {
       tor_free(signed_data);
-      ERR("Signature wasn't valid");
+      ERR("RSA signature wasn't valid");
     }
     if (signed_len < DIGEST256_LEN) {
       tor_free(signed_data);
@@ -2169,40 +2412,74 @@ channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan)
       ERR("Signature did not match data to be signed.");
     }
     tor_free(signed_data);
+  } else {
+    if (chan->conn->handshake_state->certs->ed_id_sign == NULL)
+      ERR("We never got an Ed25519 identity certificate.");
+    if (chan->conn->handshake_state->certs->ed_sign_auth == NULL)
+      ERR("We never got an Ed25519 authentication certificate.");
+
+    const ed25519_public_key_t *authkey =
+      &chan->conn->handshake_state->certs->ed_sign_auth->signed_key;
+    ed25519_signature_t sig;
+    tor_assert(authlen > ED25519_SIG_LEN);
+    memcpy(&sig.sig, auth + authlen - ED25519_SIG_LEN, ED25519_SIG_LEN);
+    if (ed25519_checksig(&sig, auth, authlen - ED25519_SIG_LEN, authkey)<0) {
+      ERR("Ed25519 signature wasn't valid.");
+    }
   }
 
   /* Okay, we are authenticated. */
   chan->conn->handshake_state->received_authenticate = 1;
   chan->conn->handshake_state->authenticated = 1;
+  chan->conn->handshake_state->authenticated_rsa = 1;
   chan->conn->handshake_state->digest_received_data = 0;
   {
-    crypto_pk_t *identity_rcvd =
-      tor_tls_cert_get_key(chan->conn->handshake_state->id_cert);
-    const common_digests_t *id_digests =
-      tor_x509_cert_get_id_digests(chan->conn->handshake_state->id_cert);
+    tor_x509_cert_t *id_cert = chan->conn->handshake_state->certs->id_cert;
+    crypto_pk_t *identity_rcvd = tor_tls_cert_get_key(id_cert);
+    const common_digests_t *id_digests = tor_x509_cert_get_id_digests(id_cert);
+    const ed25519_public_key_t *ed_identity_received = NULL;
+
+    if (! sig_is_rsa) {
+      chan->conn->handshake_state->authenticated_ed25519 = 1;
+      ed_identity_received =
+        &chan->conn->handshake_state->certs->ed_id_sign->signing_key;
+      memcpy(&chan->conn->handshake_state->authenticated_ed25519_peer_id,
+             ed_identity_received, sizeof(ed25519_public_key_t));
+    }
 
     /* This must exist; we checked key type when reading the cert. */
     tor_assert(id_digests);
 
-    memcpy(chan->conn->handshake_state->authenticated_peer_id,
+    memcpy(chan->conn->handshake_state->authenticated_rsa_peer_id,
            id_digests->d[DIGEST_SHA1], DIGEST_LEN);
 
     channel_set_circid_type(TLS_CHAN_TO_BASE(chan), identity_rcvd,
                chan->conn->link_proto < MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS);
     crypto_pk_free(identity_rcvd);
 
+    log_debug(LD_HANDSHAKE,
+              "Calling connection_or_init_conn_from_address for %s "
+              " from %s, with%s ed25519 id.",
+              safe_str(chan->conn->base_.address),
+              __func__,
+              ed_identity_received ? "" : "out");
+
     connection_or_init_conn_from_address(chan->conn,
                   &(chan->conn->base_.addr),
                   chan->conn->base_.port,
                   (const char*)(chan->conn->handshake_state->
-                    authenticated_peer_id),
+                    authenticated_rsa_peer_id),
+                  ed_identity_received,
                   0);
 
-    log_info(LD_OR,
-             "Got an AUTHENTICATE cell from %s:%d: Looks good.",
+    log_debug(LD_HANDSHAKE,
+             "Got an AUTHENTICATE cell from %s:%d, type %d: Looks good.",
              safe_str(chan->conn->base_.address),
-             chan->conn->base_.port);
+             chan->conn->base_.port,
+             authtype);
   }
 
+  var_cell_free(expected_cell);
+
 #undef ERR
 }
diff --git a/src/or/channeltls.h b/src/core/or/channeltls.h
index 463f7d928c..a2648ff537 100644
--- a/src/or/channeltls.h
+++ b/src/core/or/channeltls.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,8 +9,11 @@
 #ifndef TOR_CHANNELTLS_H
 #define TOR_CHANNELTLS_H
 
-#include "or.h"
-#include "channel.h"
+#include "core/or/or.h"
+#include "core/or/channel.h"
+
+struct ed25519_public_key_t;
+struct curve25519_public_key_t;
 
 #define TLS_PER_CELL_OVERHEAD 29
 
@@ -28,10 +31,11 @@ struct channel_tls_s {
   or_connection_t *conn;
 };
 
-#endif /* TOR_CHANNEL_INTERNAL_ */
+#endif /* defined(TOR_CHANNEL_INTERNAL_) */
 
 channel_t * channel_tls_connect(const tor_addr_t *addr, uint16_t port,
-                                const char *id_digest);
+                                const char *id_digest,
+                                const struct ed25519_public_key_t *ed_id);
 channel_listener_t * channel_tls_get_listener(void);
 channel_listener_t * channel_tls_start_listener(void);
 channel_t * channel_tls_handle_incoming(or_connection_t *orconn);
@@ -70,7 +74,6 @@ STATIC void channel_tls_process_auth_challenge_cell(var_cell_t *cell,
 STATIC void channel_tls_common_init(channel_tls_t *tlschan);
 STATIC void channel_tls_process_authenticate_cell(var_cell_t *cell,
                                                   channel_tls_t *tlschan);
-#endif
-
-#endif
+#endif /* defined(CHANNELTLS_PRIVATE) */
 
+#endif /* !defined(TOR_CHANNELTLS_H) */
diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h
new file mode 100644
index 0000000000..d4339ff50d
--- /dev/null
+++ b/src/core/or/circuit_st.h
@@ -0,0 +1,182 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef CIRCUIT_ST_H
+#define CIRCUIT_ST_H
+
+#include "core/or/or.h"
+
+#include "core/or/cell_queue_st.h"
+
+struct hs_token_t;
+
+/** "magic" value for an origin_circuit_t */
+#define ORIGIN_CIRCUIT_MAGIC 0x35315243u
+/** "magic" value for an or_circuit_t */
+#define OR_CIRCUIT_MAGIC 0x98ABC04Fu
+/** "magic" value for a circuit that would have been freed by circuit_free,
+ * but which we're keeping around until a cpuworker reply arrives.  See
+ * circuit_free() for more documentation. */
+#define DEAD_CIRCUIT_MAGIC 0xdeadc14c
+
+/**
+ * A circuit is a path over the onion routing
+ * network. Applications can connect to one end of the circuit, and can
+ * create exit connections at the other end of the circuit. AP and exit
+ * connections have only one circuit associated with them (and thus these
+ * connection types are closed when the circuit is closed), whereas
+ * OR connections multiplex many circuits at once, and stay standing even
+ * when there are no circuits running over them.
+ *
+ * A circuit_t structure can fill one of two roles.  First, a or_circuit_t
+ * links two connections together: either an edge connection and an OR
+ * connection, or two OR connections.  (When joined to an OR connection, a
+ * circuit_t affects only cells sent to a particular circID on that
+ * connection.  When joined to an edge connection, a circuit_t affects all
+ * data.)
+
+ * Second, an origin_circuit_t holds the cipher keys and state for sending data
+ * along a given circuit.  At the OP, it has a sequence of ciphers, each
+ * of which is shared with a single OR along the circuit.  Separate
+ * ciphers are used for data going "forward" (away from the OP) and
+ * "backward" (towards the OP).  At the OR, a circuit has only two stream
+ * ciphers: one for data going forward, and one for data going backward.
+ */
+struct circuit_t {
+  uint32_t magic; /**< For memory and type debugging: must equal
+                   * ORIGIN_CIRCUIT_MAGIC or OR_CIRCUIT_MAGIC. */
+
+  /** The channel that is next in this circuit. */
+  channel_t *n_chan;
+
+  /**
+   * The circuit_id used in the next (forward) hop of this circuit;
+   * this is unique to n_chan, but this ordered pair is globally
+   * unique:
+   *
+   * (n_chan->global_identifier, n_circ_id)
+   */
+  circid_t n_circ_id;
+
+  /**
+   * Circuit mux associated with n_chan to which this circuit is attached;
+   * NULL if we have no n_chan.
+   */
+  circuitmux_t *n_mux;
+
+  /** Queue of cells waiting to be transmitted on n_chan */
+  cell_queue_t n_chan_cells;
+
+  /**
+   * The hop to which we want to extend this circuit.  Should be NULL if
+   * the circuit has attached to a channel.
+   */
+  extend_info_t *n_hop;
+
+  /** True iff we are waiting for n_chan_cells to become less full before
+   * allowing p_streams to add any more cells. (Origin circuit only.) */
+  unsigned int streams_blocked_on_n_chan : 1;
+  /** True iff we are waiting for p_chan_cells to become less full before
+   * allowing n_streams to add any more cells. (OR circuit only.) */
+  unsigned int streams_blocked_on_p_chan : 1;
+
+  /** True iff we have queued a delete backwards on this circuit, but not put
+   * it on the output buffer. */
+  unsigned int p_delete_pending : 1;
+  /** True iff we have queued a delete forwards on this circuit, but not put
+   * it on the output buffer. */
+  unsigned int n_delete_pending : 1;
+
+  /** True iff this circuit has received a DESTROY cell in either direction */
+  unsigned int received_destroy : 1;
+
+  uint8_t state; /**< Current status of this circuit. */
+  uint8_t purpose; /**< Why are we creating this circuit? */
+
+  /** How many relay data cells can we package (read from edge streams)
+   * on this circuit before we receive a circuit-level sendme cell asking
+   * for more? */
+  int package_window;
+  /** How many relay data cells will we deliver (write to edge streams)
+   * on this circuit? When deliver_window gets low, we send some
+   * circuit-level sendme cells to indicate that we're willing to accept
+   * more. */
+  int deliver_window;
+
+  /** Temporary field used during circuits_handle_oom. */
+  uint32_t age_tmp;
+
+  /** For storage while n_chan is pending (state CIRCUIT_STATE_CHAN_WAIT). */
+  struct create_cell_t *n_chan_create_cell;
+
+  /** When did circuit construction actually begin (ie send the
+   * CREATE cell or begin cannibalization).
+   *
+   * Note: This timer will get reset if we decide to cannibalize
+   * a circuit. It may also get reset during certain phases of hidden
+   * service circuit use.
+   *
+   * We keep this timestamp with a higher resolution than most so that the
+   * circuit-build-time tracking code can get millisecond resolution.
+   */
+  struct timeval timestamp_began;
+
+  /** This timestamp marks when the init_circuit_base constructor ran. */
+  struct timeval timestamp_created;
+
+  /** When the circuit was first used, or 0 if the circuit is clean.
+   *
+   * XXXX Note that some code will artificially adjust this value backward
+   * in time in order to indicate that a circuit shouldn't be used for new
+   * streams, but that it can stay alive as long as it has streams on it.
+   * That's a kludge we should fix.
+   *
+   * XXX The CBT code uses this field to record when HS-related
+   * circuits entered certain states.  This usage probably won't
+   * interfere with this field's primary purpose, but we should
+   * document it more thoroughly to make sure of that.
+   *
+   * XXX The SocksPort option KeepaliveIsolateSOCKSAuth will artificially
+   * adjust this value forward each time a suitable stream is attached to an
+   * already constructed circuit, potentially keeping the circuit alive
+   * indefinitely.
+   */
+  time_t timestamp_dirty;
+
+  uint16_t marked_for_close; /**< Should we close this circuit at the end of
+                              * the main loop? (If true, holds the line number
+                              * where this circuit was marked.) */
+  const char *marked_for_close_file; /**< For debugging: in which file was this
+                                      * circuit marked for close? */
+  /** For what reason (See END_CIRC_REASON...) is this circuit being closed?
+   * This field is set in circuit_mark_for_close and used later in
+   * circuit_about_to_free. */
+  int marked_for_close_reason;
+  /** As marked_for_close_reason, but reflects the underlying reason for
+   * closing this circuit.
+   */
+  int marked_for_close_orig_reason;
+
+  /** Unique ID for measuring tunneled network status requests. */
+  uint64_t dirreq_id;
+
+  /** Index in smartlist of all circuits (global_circuitlist). */
+  int global_circuitlist_idx;
+
+  /** Various statistics about cells being added to or removed from this
+   * circuit's queues; used only if CELL_STATS events are enabled and
+   * cleared after being sent to control port. */
+  smartlist_t *testing_cell_stats;
+
+  /** If set, points to an HS token that this circuit might be carrying.
+   *  Used by the HS circuitmap.  */
+  struct hs_token_t *hs_token;
+  /** Hashtable node: used to look up the circuit by its HS token using the HS
+      circuitmap. */
+  HT_ENTRY(circuit_t) hs_circuitmap_node;
+};
+
+#endif
diff --git a/src/or/circuitbuild.c b/src/core/or/circuitbuild.c
index cb9c146fb7..21369fb538 100644
--- a/src/or/circuitbuild.c
+++ b/src/core/or/circuitbuild.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,56 +9,94 @@
  *
  * \brief Implements the details of building circuits (by chosing paths,
  * constructing/sending create/extend cells, and so on).
+ *
+ * On the client side, this module handles launching circuits. Circuit
+ * launches are srtarted from circuit_establish_circuit(), called from
+ * circuit_launch_by_extend_info()).  To choose the path the circuit will
+ * take, onion_extend_cpath() calls into a maze of node selection functions.
+ *
+ * Once the circuit is ready to be launched, the first hop is treated as a
+ * special case with circuit_handle_first_hop(), since it might need to open a
+ * channel.  As the channel opens, and later as CREATED and RELAY_EXTENDED
+ * cells arrive, the client will invoke circuit_send_next_onion_skin() to send
+ * CREATE or RELAY_EXTEND cells.
+ *
+ * On the server side, this module also handles the logic of responding to
+ * RELAY_EXTEND requests, using circuit_extend().
  **/
 
 #define CIRCUITBUILD_PRIVATE
 
-#include "or.h"
-#include "channel.h"
-#include "circpathbias.h"
-#define CIRCUITBUILD_PRIVATE
-#include "circuitbuild.h"
-#include "circuitlist.h"
-#include "circuitstats.h"
-#include "circuituse.h"
-#include "command.h"
-#include "config.h"
-#include "confparse.h"
-#include "connection.h"
-#include "connection_edge.h"
-#include "connection_or.h"
-#include "control.h"
-#include "crypto.h"
-#include "directory.h"
-#include "entrynodes.h"
-#include "main.h"
-#include "microdesc.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "onion.h"
-#include "onion_tap.h"
-#include "onion_fast.h"
-#include "policies.h"
-#include "relay.h"
-#include "rendcommon.h"
-#include "rephist.h"
-#include "router.h"
-#include "routerlist.h"
-#include "routerparse.h"
-#include "routerset.h"
-#include "transports.h"
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "app/config/confparse.h"
+#include "core/crypto/hs_ntor.h"
+#include "core/crypto/onion_crypto.h"
+#include "core/crypto/onion_fast.h"
+#include "core/crypto/onion_tap.h"
+#include "core/crypto/relay_crypto.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/channel.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitstats.h"
+#include "core/or/circuituse.h"
+#include "core/or/command.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "core/or/onion.h"
+#include "core/or/policies.h"
+#include "core/or/relay.h"
+#include "feature/client/bridges.h"
+#include "feature/client/circpathbias.h"
+#include "feature/client/entrynodes.h"
+#include "feature/client/transports.h"
+#include "feature/control/control.h"
+#include "feature/dircommon/directory.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nickname.h"
+#include "feature/nodelist/node_select.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+#include "feature/relay/selftest.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/stats/predict_ports.h"
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "core/or/cell_st.h"
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/extend_info_st.h"
+#include "feature/nodelist/node_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "feature/nodelist/microdesc_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
 
 static channel_t * channel_connect_for_circuit(const tor_addr_t *addr,
-                                               uint16_t port,
-                                               const char *id_digest);
+                                            uint16_t port,
+                                            const char *id_digest,
+                                            const ed25519_public_key_t *ed_id);
 static int circuit_deliver_create_cell(circuit_t *circ,
                                        const create_cell_t *create_cell,
                                        int relayed);
-static int onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit);
 static crypt_path_t *onion_next_hop_in_cpath(crypt_path_t *cpath);
-static int onion_extend_cpath(origin_circuit_t *circ);
-static int count_acceptable_nodes(smartlist_t *routers);
-static int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
+STATIC int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
+static int circuit_send_first_onion_skin(origin_circuit_t *circ);
+static int circuit_build_no_more_hops(origin_circuit_t *circ);
+static int circuit_send_intermediate_onion_skin(origin_circuit_t *circ,
+                                                crypt_path_t *hop);
+static const node_t *choose_good_middle_server(uint8_t purpose,
+                          cpath_build_state_t *state,
+                          crypt_path_t *head,
+                          int cur_len);
 
 /** This function tries to get a channel to the specified endpoint,
  * and then calls command_setup_channel() to give it the right
@@ -66,11 +104,12 @@ static int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
  */
 static channel_t *
 channel_connect_for_circuit(const tor_addr_t *addr, uint16_t port,
-                            const char *id_digest)
+                            const char *id_digest,
+                            const ed25519_public_key_t *ed_id)
 {
   channel_t *chan;
 
-  chan = channel_connect(addr, port, id_digest);
+  chan = channel_connect(addr, port, id_digest, ed_id);
   if (chan) command_setup_channel(chan);
 
   return chan;
@@ -167,11 +206,11 @@ get_unique_circ_id_by_chan(channel_t *chan)
                                                               chan->cmux);
 
       log_warn(LD_CIRC, "  Circuitmux on this channel has %u circuits, "
-               "of which %u are active. It says it has "I64_FORMAT
+               "of which %u are active. It says it has %"PRId64
                " destroy cells queued.",
                circuitmux_num_circuits(chan->cmux),
                circuitmux_num_active_circuits(chan->cmux),
-               I64_PRINTF_ARG(queued_destroys));
+               (queued_destroys));
 
       /* Change this into "if (1)" in order to get more information about
        * possible failure modes here.  You'll need to know how to use gdb with
@@ -216,9 +255,8 @@ get_unique_circ_id_by_chan(channel_t *chan)
 /** If <b>verbose</b> is false, allocate and return a comma-separated list of
  * the currently built elements of <b>circ</b>. If <b>verbose</b> is true, also
  * list information about link status in a more verbose format using spaces.
- * If <b>verbose_names</b> is false, give nicknames for Named routers and hex
- * digests for others; if <b>verbose_names</b> is true, use $DIGEST=Name style
- * names.
+ * If <b>verbose_names</b> is false, give hex digests; if <b>verbose_names</b>
+ * is true, use $DIGEST=Name style names.
  */
 static char *
 circuit_list_path_impl(origin_circuit_t *circ, int verbose, int verbose_names)
@@ -268,14 +306,9 @@ circuit_list_path_impl(origin_circuit_t *circ, int verbose, int verbose_names)
         base16_encode(elt+1, HEX_DIGEST_LEN+1, id, DIGEST_LEN);
       }
     } else { /* ! verbose_names */
-      node = node_get_by_id(id);
-      if (node && node_is_named(node)) {
-        elt = tor_strdup(node_get_nickname(node));
-      } else {
-        elt = tor_malloc(HEX_DIGEST_LEN+2);
-        elt[0] = '$';
-        base16_encode(elt+1, HEX_DIGEST_LEN+1, id, DIGEST_LEN);
-      }
+      elt = tor_malloc(HEX_DIGEST_LEN+2);
+      elt[0] = '$';
+      base16_encode(elt+1, HEX_DIGEST_LEN+1, id, DIGEST_LEN);
     }
     tor_assert(elt);
     if (verbose) {
@@ -326,45 +359,6 @@ circuit_log_path(int severity, unsigned int domain, origin_circuit_t *circ)
   tor_free(s);
 }
 
-/** Tell the rep(utation)hist(ory) module about the status of the links
- * in <b>circ</b>.  Hops that have become OPEN are marked as successfully
- * extended; the _first_ hop that isn't open (if any) is marked as
- * unable to extend.
- */
-/* XXXX Someday we should learn from OR circuits too. */
-void
-circuit_rep_hist_note_result(origin_circuit_t *circ)
-{
-  crypt_path_t *hop;
-  const char *prev_digest = NULL;
-  hop = circ->cpath;
-  if (!hop) /* circuit hasn't started building yet. */
-    return;
-  if (server_mode(get_options())) {
-    const routerinfo_t *me = router_get_my_routerinfo();
-    if (!me)
-      return;
-    prev_digest = me->cache_info.identity_digest;
-  }
-  do {
-    const node_t *node = node_get_by_id(hop->extend_info->identity_digest);
-    if (node) { /* Why do we check this?  We know the identity. -NM XXXX */
-      if (prev_digest) {
-        if (hop->state == CPATH_STATE_OPEN)
-          rep_hist_note_extend_succeeded(prev_digest, node->identity);
-        else {
-          rep_hist_note_extend_failed(prev_digest, node->identity);
-          break;
-        }
-      }
-      prev_digest = node->identity;
-    } else {
-      prev_digest = NULL;
-    }
-    hop=hop->next;
-  } while (hop!=circ->cpath);
-}
-
 /** Return 1 iff every node in circ's cpath definitely supports ntor. */
 static int
 circuit_cpath_supports_ntor(const origin_circuit_t *circ)
@@ -417,10 +411,10 @@ onion_populate_cpath(origin_circuit_t *circ)
    * edge cases. */
   tor_assert(circuit_get_cpath_len(circ));
   if (circuit_can_use_tap(circ)) {
-    /* Circuits from clients to intro points, and hidden services to
-     * rend points do not support ntor, because the hidden service protocol
-     * does not include ntor onion keys. This is also true for Tor2web clients
-     * and Single Onion Services. */
+    /* Circuits from clients to intro points, and hidden services to rend
+     * points do not support ntor, because the hidden service protocol does
+     * not include ntor onion keys. This is also true for Single Onion
+     * Services. */
     return 0;
   }
 
@@ -435,7 +429,7 @@ onion_populate_cpath(origin_circuit_t *circ)
                                     circ->cpath->extend_info->identity_digest);
     /* If we don't know the node and its descriptor, we must be bootstrapping.
      */
-    if (!node || !node_has_descriptor(node)) {
+    if (!node || !node_has_preferred_descriptor(node, 1)) {
       return 0;
     }
   }
@@ -484,10 +478,15 @@ circuit_establish_circuit(uint8_t purpose, extend_info_t *exit_ei, int flags)
 {
   origin_circuit_t *circ;
   int err_reason = 0;
+  int is_hs_v3_rp_circuit = 0;
+
+  if (flags & CIRCLAUNCH_IS_V3_RP) {
+    is_hs_v3_rp_circuit = 1;
+  }
 
   circ = origin_circuit_init(purpose, flags);
 
-  if (onion_pick_cpath_exit(circ, exit_ei) < 0 ||
+  if (onion_pick_cpath_exit(circ, exit_ei, is_hs_v3_rp_circuit) < 0 ||
       onion_populate_cpath(circ) < 0) {
     circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_NOPATH);
     return NULL;
@@ -502,6 +501,13 @@ circuit_establish_circuit(uint8_t purpose, extend_info_t *exit_ei, int flags)
   return circ;
 }
 
+/** Return the guard state associated with <b>circ</b>, which may be NULL. */
+circuit_guard_state_t *
+origin_circuit_get_guard_state(origin_circuit_t *circ)
+{
+  return circ->guard_state;
+}
+
 /** Start establishing the first hop of our circuit. Figure out what
  * OR we should connect to, and if necessary start the connection to
  * it. If we're already connected, then send the 'create' cell.
@@ -540,6 +546,7 @@ circuit_handle_first_hop(origin_circuit_t *circ)
                          firsthop->extend_info->port));
 
   n_chan = channel_get_for_extend(firsthop->extend_info->identity_digest,
+                                  &firsthop->extend_info->ed_identity,
                                   &firsthop->extend_info->addr,
                                   &msg,
                                   &should_launch);
@@ -557,7 +564,8 @@ circuit_handle_first_hop(origin_circuit_t *circ)
       n_chan = channel_connect_for_circuit(
           &firsthop->extend_info->addr,
           firsthop->extend_info->port,
-          firsthop->extend_info->identity_digest);
+          firsthop->extend_info->identity_digest,
+          &firsthop->extend_info->ed_identity);
       if (!n_chan) { /* connect failed, forget the whole thing */
         log_info(LD_CIRC,"connect to firsthop failed. Closing.");
         return -END_CIRC_REASON_CONNECTFAILED;
@@ -600,8 +608,7 @@ circuit_n_chan_done(channel_t *chan, int status, int close_origin_circuits)
 
   tor_assert(chan);
 
-  log_debug(LD_CIRC,"chan to %s/%s, status=%d",
-            chan->nickname ? chan->nickname : "NULL",
+  log_debug(LD_CIRC,"chan to %s, status=%d",
             channel_get_canonical_remote_descr(chan), status);
 
   pending_circs = smartlist_new();
@@ -791,24 +798,28 @@ should_use_create_fast_for_circuit(origin_circuit_t *circ)
      * creating on behalf of others. */
     return 0;
   }
-  if (options->FastFirstHopPK == -1) {
-    /* option is "auto", so look at the consensus. */
-    return networkstatus_get_param(NULL, "usecreatefast", 1, 0, 1);
-  }
-
-  return options->FastFirstHopPK;
+  return networkstatus_get_param(NULL, "usecreatefast", 0, 0, 1);
 }
 
-/** Return true if <b>circ</b> is the type of circuit we want to count
- * timeouts from. In particular, we want it to have not completed yet
- * (already completing indicates we cannibalized it), and we want it to
- * have exactly three hops.
+/**
+ * Return true if <b>circ</b> is the type of circuit we want to count
+ * timeouts from.
+ *
+ * In particular, we want to consider any circuit that plans to build
+ * at least 3 hops (but maybe more), but has 3 or fewer hops built
+ * so far.
+ *
+ * We still want to consider circuits before 3 hops, because we need
+ * to decide if we should convert them to a measurement circuit in
+ * circuit_build_times_handle_completed_hop(), rather than letting
+ * slow circuits get killed right away.
  */
 int
-circuit_timeout_want_to_count_circ(origin_circuit_t *circ)
+circuit_timeout_want_to_count_circ(const origin_circuit_t *circ)
 {
   return !circ->has_opened
-          && circ->build_state->desired_path_len == DEFAULT_ROUTE_LEN;
+          && circ->build_state->desired_path_len >= DEFAULT_ROUTE_LEN
+          && circuit_get_cpath_opened_len(circ) <= DEFAULT_ROUTE_LEN;
 }
 
 /** Decide whether to use a TAP or ntor handshake for connecting to <b>ei</b>
@@ -816,7 +827,6 @@ circuit_timeout_want_to_count_circ(origin_circuit_t *circ)
  * accordingly.
  * Note that TAP handshakes in CREATE cells are only used for direct
  * connections:
- *  - from Tor2web to intro points not in the client's consensus, and
  *  - from Single Onions to rend points not in the service's consensus.
  * This is checked in onion_populate_cpath. */
 static void
@@ -866,212 +876,292 @@ circuit_pick_extend_handshake(uint8_t *cell_type_out,
   }
 }
 
+/**
+ * Return true iff <b>purpose</b> is a purpose for a circuit which is
+ * allowed to have no guard configured, even if the circuit is multihop
+ * and guards are enabled.
+ */
+static int
+circuit_purpose_may_omit_guard(int purpose)
+{
+  switch (purpose) {
+    case CIRCUIT_PURPOSE_TESTING:
+    case CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT:
+      /* Testing circuits may omit guards because they're measuring
+       * liveness or performance, and don't want guards to interfere. */
+      return 1;
+    default:
+      /* All other multihop circuits should use guards if guards are
+       * enabled. */
+      return 0;
+  }
+}
+
 /** This is the backbone function for building circuits.
  *
  * If circ's first hop is closed, then we need to build a create
  * cell and send it forward.
  *
- * Otherwise, we need to build a relay extend cell and send it
- * forward.
+ * Otherwise, if circ's cpath still has any non-open hops, we need to
+ * build a relay extend cell and send it forward to the next non-open hop.
+ *
+ * If all hops on the cpath are open, we're done building the circuit
+ * and we should do housekeeping for the newly opened circuit.
  *
  * Return -reason if we want to tear down circ, else return 0.
  */
 int
 circuit_send_next_onion_skin(origin_circuit_t *circ)
 {
-  crypt_path_t *hop;
-  const node_t *node;
-
   tor_assert(circ);
 
   if (circ->cpath->state == CPATH_STATE_CLOSED) {
-    /* This is the first hop. */
-    create_cell_t cc;
-    int fast;
-    int len;
-    log_debug(LD_CIRC,"First skin; sending create cell.");
-    memset(&cc, 0, sizeof(cc));
-    if (circ->build_state->onehop_tunnel)
-      control_event_bootstrap(BOOTSTRAP_STATUS_ONEHOP_CREATE, 0);
-    else
-      control_event_bootstrap(BOOTSTRAP_STATUS_CIRCUIT_CREATE, 0);
+    /* Case one: we're on the first hop. */
+    return circuit_send_first_onion_skin(circ);
+  }
 
-    node = node_get_by_id(circ->base_.n_chan->identity_digest);
-    fast = should_use_create_fast_for_circuit(circ);
-    if (!fast) {
-      /* We are an OR and we know the right onion key: we should
-       * send a create cell.
-       */
-      circuit_pick_create_handshake(&cc.cell_type, &cc.handshake_type,
-                                    circ->cpath->extend_info);
-    } else {
-      /* We are not an OR, and we're building the first hop of a circuit to a
-       * new OR: we can be speedy and use CREATE_FAST to save an RSA operation
-       * and a DH operation. */
-      cc.cell_type = CELL_CREATE_FAST;
-      cc.handshake_type = ONION_HANDSHAKE_TYPE_FAST;
-    }
+  tor_assert(circ->cpath->state == CPATH_STATE_OPEN);
+  tor_assert(circ->base_.state == CIRCUIT_STATE_BUILDING);
 
-    len = onion_skin_create(cc.handshake_type,
-                            circ->cpath->extend_info,
-                            &circ->cpath->handshake_state,
-                            cc.onionskin);
-    if (len < 0) {
-      log_warn(LD_CIRC,"onion_skin_create (first hop) failed.");
-      return - END_CIRC_REASON_INTERNAL;
-    }
-    cc.handshake_len = len;
+  crypt_path_t *hop = onion_next_hop_in_cpath(circ->cpath);
+  circuit_build_times_handle_completed_hop(circ);
+
+  if (hop) {
+    /* Case two: we're on a hop after the first. */
+    return circuit_send_intermediate_onion_skin(circ, hop);
+  }
+
+  /* Case three: the circuit is finished. Do housekeeping tasks on it. */
+  return circuit_build_no_more_hops(circ);
+}
+
+/**
+ * Called from circuit_send_next_onion_skin() when we find ourselves connected
+ * to the first hop in <b>circ</b>: Send a CREATE or CREATE2 or CREATE_FAST
+ * cell to that hop.  Return 0 on success; -reason on failure (if the circuit
+ * should be torn down).
+ */
+static int
+circuit_send_first_onion_skin(origin_circuit_t *circ)
+{
+  int fast;
+  int len;
+  const node_t *node;
+  create_cell_t cc;
+  memset(&cc, 0, sizeof(cc));
 
-    if (circuit_deliver_create_cell(TO_CIRCUIT(circ), &cc, 0) < 0)
-      return - END_CIRC_REASON_RESOURCELIMIT;
+  log_debug(LD_CIRC,"First skin; sending create cell.");
 
-    circ->cpath->state = CPATH_STATE_AWAITING_KEYS;
-    circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_BUILDING);
-    log_info(LD_CIRC,"First hop: finished sending %s cell to '%s'",
-             fast ? "CREATE_FAST" : "CREATE",
-             node ? node_describe(node) : "<unnamed>");
+  if (circ->build_state->onehop_tunnel) {
+    control_event_bootstrap(BOOTSTRAP_STATUS_ONEHOP_CREATE, 0);
   } else {
-    extend_cell_t ec;
-    int len;
-    tor_assert(circ->cpath->state == CPATH_STATE_OPEN);
-    tor_assert(circ->base_.state == CIRCUIT_STATE_BUILDING);
-    log_debug(LD_CIRC,"starting to send subsequent skin.");
-    hop = onion_next_hop_in_cpath(circ->cpath);
-    memset(&ec, 0, sizeof(ec));
-    if (!hop) {
-      /* done building the circuit. whew. */
-      circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN);
-      if (circuit_timeout_want_to_count_circ(circ)) {
-        struct timeval end;
-        long timediff;
-        tor_gettimeofday(&end);
-        timediff = tv_mdiff(&circ->base_.timestamp_began, &end);
-
-        /*
-         * If the circuit build time is much greater than we would have cut
-         * it off at, we probably had a suspend event along this codepath,
-         * and we should discard the value.
-         */
-        if (timediff < 0 ||
-            timediff > 2*get_circuit_build_close_time_ms()+1000) {
-          log_notice(LD_CIRC, "Strange value for circuit build time: %ldmsec. "
-                              "Assuming clock jump. Purpose %d (%s)", timediff,
-                     circ->base_.purpose,
-                     circuit_purpose_to_string(circ->base_.purpose));
-        } else if (!circuit_build_times_disabled()) {
-          /* Only count circuit times if the network is live */
-          if (circuit_build_times_network_check_live(
-              get_circuit_build_times())) {
-            circuit_build_times_add_time(get_circuit_build_times_mutable(),
-                (build_time_t)timediff);
-            circuit_build_times_set_timeout(get_circuit_build_times_mutable());
-          }
-
-          if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
-            circuit_build_times_network_circ_success(
-                get_circuit_build_times_mutable());
-          }
-        }
-      }
-      log_info(LD_CIRC,"circuit built!");
-      circuit_reset_failure_count(0);
+    control_event_bootstrap(BOOTSTRAP_STATUS_CIRCUIT_CREATE, 0);
 
-      if (circ->build_state->onehop_tunnel || circ->has_opened) {
-        control_event_bootstrap(BOOTSTRAP_STATUS_REQUESTING_STATUS, 0);
-      }
+    /* If this is not a one-hop tunnel, the channel is being used
+     * for traffic that wants anonymity and protection from traffic
+     * analysis (such as netflow record retention). That means we want
+     * to pad it.
+     */
+    if (circ->base_.n_chan->channel_usage < CHANNEL_USED_FOR_FULL_CIRCS)
+      circ->base_.n_chan->channel_usage = CHANNEL_USED_FOR_FULL_CIRCS;
+  }
 
-      pathbias_count_build_success(circ);
-      circuit_rep_hist_note_result(circ);
-      circuit_has_opened(circ); /* do other actions as necessary */
-
-      if (!have_completed_a_circuit() && !circ->build_state->onehop_tunnel) {
-        const or_options_t *options = get_options();
-        note_that_we_completed_a_circuit();
-        /* FFFF Log a count of known routers here */
-        log_notice(LD_GENERAL,
-            "Tor has successfully opened a circuit. "
-            "Looks like client functionality is working.");
-        if (control_event_bootstrap(BOOTSTRAP_STATUS_DONE, 0) == 0) {
-          log_notice(LD_GENERAL,
-                     "Tor has successfully opened a circuit. "
-                     "Looks like client functionality is working.");
-        }
-        control_event_client_status(LOG_NOTICE, "CIRCUIT_ESTABLISHED");
-        clear_broken_connection_map(1);
-        if (server_mode(options) && !check_whether_orport_reachable(options)) {
-          inform_testing_reachability();
-          consider_testing_reachability(1, 1);
-        }
-      }
+  node = node_get_by_id(circ->base_.n_chan->identity_digest);
+  fast = should_use_create_fast_for_circuit(circ);
+  if (!fast) {
+    /* We know the right onion key: we should send a create cell. */
+    circuit_pick_create_handshake(&cc.cell_type, &cc.handshake_type,
+                                  circ->cpath->extend_info);
+  } else {
+    /* We don't know an onion key, so we need to fall back to CREATE_FAST. */
+    cc.cell_type = CELL_CREATE_FAST;
+    cc.handshake_type = ONION_HANDSHAKE_TYPE_FAST;
+  }
 
-      /* We're done with measurement circuits here. Just close them */
-      if (circ->base_.purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
-        circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED);
-      }
-      return 0;
-    }
+  len = onion_skin_create(cc.handshake_type,
+                          circ->cpath->extend_info,
+                          &circ->cpath->handshake_state,
+                          cc.onionskin);
+  if (len < 0) {
+    log_warn(LD_CIRC,"onion_skin_create (first hop) failed.");
+    return - END_CIRC_REASON_INTERNAL;
+  }
+  cc.handshake_len = len;
+
+  if (circuit_deliver_create_cell(TO_CIRCUIT(circ), &cc, 0) < 0)
+    return - END_CIRC_REASON_RESOURCELIMIT;
+
+  circ->cpath->state = CPATH_STATE_AWAITING_KEYS;
+  circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_BUILDING);
+  log_info(LD_CIRC,"First hop: finished sending %s cell to '%s'",
+           fast ? "CREATE_FAST" : "CREATE",
+           node ? node_describe(node) : "<unnamed>");
+  return 0;
+}
 
-    if (tor_addr_family(&hop->extend_info->addr) != AF_INET) {
-      log_warn(LD_BUG, "Trying to extend to a non-IPv4 address.");
-      return - END_CIRC_REASON_INTERNAL;
+/**
+ * Called from circuit_send_next_onion_skin() when we find that we have no
+ * more hops: mark the circuit as finished, and perform the necessary
+ * bookkeeping.  Return 0 on success; -reason on failure (if the circuit
+ * should be torn down).
+ */
+static int
+circuit_build_no_more_hops(origin_circuit_t *circ)
+{
+  guard_usable_t r;
+  if (! circ->guard_state) {
+    if (circuit_get_cpath_len(circ) != 1 &&
+        ! circuit_purpose_may_omit_guard(circ->base_.purpose) &&
+        get_options()->UseEntryGuards) {
+      log_warn(LD_BUG, "%d-hop circuit %p with purpose %d has no "
+               "guard state",
+               circuit_get_cpath_len(circ), circ, circ->base_.purpose);
     }
+    r = GUARD_USABLE_NOW;
+  } else {
+    r = entry_guard_succeeded(&circ->guard_state);
+  }
+  const int is_usable_for_streams = (r == GUARD_USABLE_NOW);
+  if (r == GUARD_USABLE_NOW) {
+    circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN);
+  } else if (r == GUARD_MAYBE_USABLE_LATER) {
+    // Wait till either a better guard succeeds, or till
+    // all better guards fail.
+    circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_GUARD_WAIT);
+  } else {
+    tor_assert_nonfatal(r == GUARD_USABLE_NEVER);
+    return - END_CIRC_REASON_INTERNAL;
+  }
+
+  /* XXXX #21422 -- the rest of this branch needs careful thought!
+   * Some of the things here need to happen when a circuit becomes
+   * mechanically open; some need to happen when it is actually usable.
+   * I think I got them right, but more checking would be wise. -NM
+   */
 
-    circuit_pick_extend_handshake(&ec.cell_type,
-                                  &ec.create_cell.cell_type,
-                                  &ec.create_cell.handshake_type,
-                                  hop->extend_info);
-
-    tor_addr_copy(&ec.orport_ipv4.addr, &hop->extend_info->addr);
-    ec.orport_ipv4.port = hop->extend_info->port;
-    tor_addr_make_unspec(&ec.orport_ipv6.addr);
-    memcpy(ec.node_id, hop->extend_info->identity_digest, DIGEST_LEN);
-
-    len = onion_skin_create(ec.create_cell.handshake_type,
-                            hop->extend_info,
-                            &hop->handshake_state,
-                            ec.create_cell.onionskin);
-    if (len < 0) {
-      log_warn(LD_CIRC,"onion_skin_create failed.");
-      return - END_CIRC_REASON_INTERNAL;
+  log_info(LD_CIRC,"circuit built!");
+  circuit_reset_failure_count(0);
+
+  if (circ->build_state->onehop_tunnel || circ->has_opened) {
+    control_event_bootstrap(BOOTSTRAP_STATUS_REQUESTING_STATUS, 0);
+  }
+
+  pathbias_count_build_success(circ);
+  if (is_usable_for_streams)
+    circuit_has_opened(circ); /* do other actions as necessary */
+
+  if (!have_completed_a_circuit() && !circ->build_state->onehop_tunnel) {
+    const or_options_t *options = get_options();
+    note_that_we_completed_a_circuit();
+    /* FFFF Log a count of known routers here */
+    log_info(LD_GENERAL,
+             "Tor has successfully opened a circuit. "
+             "Looks like client functionality is working.");
+    control_event_bootstrap(BOOTSTRAP_STATUS_DONE, 0);
+    control_event_client_status(LOG_NOTICE, "CIRCUIT_ESTABLISHED");
+    clear_broken_connection_map(1);
+    if (server_mode(options) && !check_whether_orport_reachable(options)) {
+      inform_testing_reachability();
+      router_do_reachability_checks(1, 1);
     }
-    ec.create_cell.handshake_len = len;
+  }
 
-    log_info(LD_CIRC,"Sending extend relay cell.");
-    {
-      uint8_t command = 0;
-      uint16_t payload_len=0;
-      uint8_t payload[RELAY_PAYLOAD_SIZE];
-      if (extend_cell_format(&command, &payload_len, payload, &ec)<0) {
-        log_warn(LD_CIRC,"Couldn't format extend cell");
-        return -END_CIRC_REASON_INTERNAL;
-      }
+  /* We're done with measurement circuits here. Just close them */
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED);
+  }
+  return 0;
+}
+
+/**
+ * Called from circuit_send_next_onion_skin() when we find that we have a hop
+ * other than the first that we need to extend to: use <b>hop</b>'s
+ * information to extend the circuit another step. Return 0 on success;
+ * -reason on failure (if the circuit should be torn down).
+ */
+static int
+circuit_send_intermediate_onion_skin(origin_circuit_t *circ,
+                                     crypt_path_t *hop)
+{
+  int len;
+  extend_cell_t ec;
+  memset(&ec, 0, sizeof(ec));
+
+  log_debug(LD_CIRC,"starting to send subsequent skin.");
 
-      /* send it to hop->prev, because it will transfer
-       * it to a create cell and then send to hop */
-      if (relay_send_command_from_edge(0, TO_CIRCUIT(circ),
-                                       command,
-                                       (char*)payload, payload_len,
-                                       hop->prev) < 0)
-        return 0; /* circuit is closed */
+  if (tor_addr_family(&hop->extend_info->addr) != AF_INET) {
+    log_warn(LD_BUG, "Trying to extend to a non-IPv4 address.");
+    return - END_CIRC_REASON_INTERNAL;
+  }
+
+  circuit_pick_extend_handshake(&ec.cell_type,
+                                &ec.create_cell.cell_type,
+                                &ec.create_cell.handshake_type,
+                                hop->extend_info);
+
+  tor_addr_copy(&ec.orport_ipv4.addr, &hop->extend_info->addr);
+  ec.orport_ipv4.port = hop->extend_info->port;
+  tor_addr_make_unspec(&ec.orport_ipv6.addr);
+  memcpy(ec.node_id, hop->extend_info->identity_digest, DIGEST_LEN);
+  /* Set the ED25519 identity too -- it will only get included
+   * in the extend2 cell if we're configured to use it, though. */
+  ed25519_pubkey_copy(&ec.ed_pubkey, &hop->extend_info->ed_identity);
+
+  len = onion_skin_create(ec.create_cell.handshake_type,
+                          hop->extend_info,
+                          &hop->handshake_state,
+                          ec.create_cell.onionskin);
+  if (len < 0) {
+    log_warn(LD_CIRC,"onion_skin_create failed.");
+    return - END_CIRC_REASON_INTERNAL;
+  }
+  ec.create_cell.handshake_len = len;
+
+  log_info(LD_CIRC,"Sending extend relay cell.");
+  {
+    uint8_t command = 0;
+    uint16_t payload_len=0;
+    uint8_t payload[RELAY_PAYLOAD_SIZE];
+    if (extend_cell_format(&command, &payload_len, payload, &ec)<0) {
+      log_warn(LD_CIRC,"Couldn't format extend cell");
+      return -END_CIRC_REASON_INTERNAL;
     }
-    hop->state = CPATH_STATE_AWAITING_KEYS;
+
+    /* send it to hop->prev, because that relay will transfer
+     * it to a create cell and then send to hop */
+    if (relay_send_command_from_edge(0, TO_CIRCUIT(circ),
+                                     command,
+                                     (char*)payload, payload_len,
+                                     hop->prev) < 0)
+      return 0; /* circuit is closed */
   }
+  hop->state = CPATH_STATE_AWAITING_KEYS;
   return 0;
 }
 
-/** Our clock just jumped by <b>seconds_elapsed</b>. Assume
- * something has also gone wrong with our network: notify the user,
- * and abandon all not-yet-used circuits. */
+/** Our clock just jumped by <b>seconds_elapsed</b>. If <b>was_idle</b> is
+ * true, then the monotonic time matches; otherwise it doesn't. Assume
+ * something has also gone wrong with our network: notify the user, and
+ * abandon all not-yet-used circuits. */
 void
-circuit_note_clock_jumped(int seconds_elapsed)
+circuit_note_clock_jumped(int64_t seconds_elapsed, bool was_idle)
 {
   int severity = server_mode(get_options()) ? LOG_WARN : LOG_NOTICE;
-  tor_log(severity, LD_GENERAL, "Your system clock just jumped %d seconds %s; "
-      "assuming established circuits no longer work.",
-      seconds_elapsed >=0 ? seconds_elapsed : -seconds_elapsed,
-      seconds_elapsed >=0 ? "forward" : "backward");
-  control_event_general_status(LOG_WARN, "CLOCK_JUMPED TIME=%d",
-                               seconds_elapsed);
+  if (was_idle) {
+    tor_log(severity, LD_GENERAL, "Tor has been idle for %"PRId64
+            " seconds; assuming established circuits no longer work.",
+            (seconds_elapsed));
+  } else {
+    tor_log(severity, LD_GENERAL,
+            "Your system clock just jumped %"PRId64" seconds %s; "
+            "assuming established circuits no longer work.",
+            (
+                 seconds_elapsed >=0 ? seconds_elapsed : -seconds_elapsed),
+            seconds_elapsed >=0 ? "forward" : "backward");
+  }
+  control_event_general_status(LOG_WARN, "CLOCK_JUMPED TIME=%"PRId64
+                               " IDLE=%d",
+                               (seconds_elapsed), was_idle?1:0);
   /* so we log when it works again */
   note_that_we_maybe_cant_complete_circuits();
   control_event_client_status(severity, "CIRCUIT_NOT_ESTABLISHED REASON=%s",
@@ -1143,7 +1233,7 @@ circuit_extend(cell_t *cell, circuit_t *circ)
 
   /* Check if they asked us for 0000..0000. We support using
    * an empty fingerprint for the first hop (e.g. for a bridge relay),
-   * but we don't want to let people send us extend cells for empty
+   * but we don't want to let clients send us extend cells for empty
    * fingerprints -- a) because it opens the user up to a mitm attack,
    * and b) because it lets an attacker force the relay to hold open a
    * new TLS connection for each extend request. */
@@ -1153,6 +1243,18 @@ circuit_extend(cell_t *cell, circuit_t *circ)
     return -1;
   }
 
+  /* Fill in ed_pubkey if it was not provided and we can infer it from
+   * our networkstatus */
+  if (ed25519_public_key_is_zero(&ec.ed_pubkey)) {
+    const node_t *node = node_get_by_id((const char*)ec.node_id);
+    const ed25519_public_key_t *node_ed_id = NULL;
+    if (node &&
+        node_supports_ed25519_link_authentication(node, 1) &&
+        (node_ed_id = node_get_ed25519_id(node))) {
+      ed25519_pubkey_copy(&ec.ed_pubkey, node_ed_id);
+    }
+  }
+
   /* Next, check if we're being asked to connect to the hop that the
    * extend cell came from. There isn't any reason for that, and it can
    * assist circular-path attacks. */
@@ -1164,7 +1266,18 @@ circuit_extend(cell_t *cell, circuit_t *circ)
     return -1;
   }
 
+  /* Check the previous hop Ed25519 ID too */
+  if (! ed25519_public_key_is_zero(&ec.ed_pubkey) &&
+      ed25519_pubkey_eq(&ec.ed_pubkey,
+                        &TO_OR_CIRCUIT(circ)->p_chan->ed25519_identity)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Client asked me to extend back to the previous hop "
+           "(by Ed25519 ID).");
+    return -1;
+  }
+
   n_chan = channel_get_for_extend((const char*)ec.node_id,
+                                  &ec.ed_pubkey,
                                   &ec.orport_ipv4.addr,
                                   &msg,
                                   &should_launch);
@@ -1176,8 +1289,9 @@ circuit_extend(cell_t *cell, circuit_t *circ)
 
     circ->n_hop = extend_info_new(NULL /*nickname*/,
                                   (const char*)ec.node_id,
-                                  NULL /*onion_key*/,
-                                  NULL /*curve25519_key*/,
+                                  &ec.ed_pubkey,
+                                  NULL, /*onion_key*/
+                                  NULL, /*curve25519_key*/
                                   &ec.orport_ipv4.addr,
                                   ec.orport_ipv4.port);
 
@@ -1190,7 +1304,8 @@ circuit_extend(cell_t *cell, circuit_t *circ)
       /* we should try to open a connection */
       n_chan = channel_connect_for_circuit(&ec.orport_ipv4.addr,
                                            ec.orport_ipv4.port,
-                                           (const char*)ec.node_id);
+                                           (const char*)ec.node_id,
+                                           &ec.ed_pubkey);
       if (!n_chan) {
         log_info(LD_CIRC,"Launching n_chan failed. Closing circuit.");
         circuit_mark_for_close(circ, END_CIRC_REASON_CONNECTFAILED);
@@ -1217,54 +1332,32 @@ circuit_extend(cell_t *cell, circuit_t *circ)
   return 0;
 }
 
-/** Initialize cpath-\>{f|b}_{crypto|digest} from the key material in
- * key_data.  key_data must contain CPATH_KEY_MATERIAL bytes, which are
- * used as follows:
+/** Initialize cpath-\>{f|b}_{crypto|digest} from the key material in key_data.
+ *
+ * If <b>is_hs_v3</b> is set, this cpath will be used for next gen hidden
+ * service circuits and <b>key_data</b> must be at least
+ * HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN bytes in length.
+ *
+ * If <b>is_hs_v3</b> is not set, key_data must contain CPATH_KEY_MATERIAL_LEN
+ * bytes, which are used as follows:
  *   - 20 to initialize f_digest
  *   - 20 to initialize b_digest
  *   - 16 to key f_crypto
  *   - 16 to key b_crypto
  *
  * (If 'reverse' is true, then f_XX and b_XX are swapped.)
+ *
+ * Return 0 if init was successful, else -1 if it failed.
  */
 int
-circuit_init_cpath_crypto(crypt_path_t *cpath, const char *key_data,
-                          int reverse)
+circuit_init_cpath_crypto(crypt_path_t *cpath,
+                          const char *key_data, size_t key_data_len,
+                          int reverse, int is_hs_v3)
 {
-  crypto_digest_t *tmp_digest;
-  crypto_cipher_t *tmp_crypto;
 
   tor_assert(cpath);
-  tor_assert(key_data);
-  tor_assert(!(cpath->f_crypto || cpath->b_crypto ||
-             cpath->f_digest || cpath->b_digest));
-
-  cpath->f_digest = crypto_digest_new();
-  crypto_digest_add_bytes(cpath->f_digest, key_data, DIGEST_LEN);
-  cpath->b_digest = crypto_digest_new();
-  crypto_digest_add_bytes(cpath->b_digest, key_data+DIGEST_LEN, DIGEST_LEN);
-
-  if (!(cpath->f_crypto =
-        crypto_cipher_new(key_data+(2*DIGEST_LEN)))) {
-    log_warn(LD_BUG,"Forward cipher initialization failed.");
-    return -1;
-  }
-  if (!(cpath->b_crypto =
-        crypto_cipher_new(key_data+(2*DIGEST_LEN)+CIPHER_KEY_LEN))) {
-    log_warn(LD_BUG,"Backward cipher initialization failed.");
-    return -1;
-  }
-
-  if (reverse) {
-    tmp_digest = cpath->f_digest;
-    cpath->f_digest = cpath->b_digest;
-    cpath->b_digest = tmp_digest;
-    tmp_crypto = cpath->f_crypto;
-    cpath->f_crypto = cpath->b_crypto;
-    cpath->b_crypto = tmp_crypto;
-  }
-
-  return 0;
+  return relay_crypto_init(&cpath->crypto, key_data, key_data_len, reverse,
+                           is_hs_v3);
 }
 
 /** A "created" cell <b>reply</b> came back to us on circuit <b>circ</b>.
@@ -1316,7 +1409,7 @@ circuit_finish_handshake(origin_circuit_t *circ,
 
   onion_handshake_state_release(&hop->handshake_state);
 
-  if (circuit_init_cpath_crypto(hop, keys, 0)<0) {
+  if (circuit_init_cpath_crypto(hop, keys, sizeof(keys), 0, 0)<0) {
     return -END_CIRC_REASON_TORPROTOCOL;
   }
 
@@ -1335,13 +1428,12 @@ circuit_finish_handshake(origin_circuit_t *circ,
  * just give up: force circ to close, and return 0.
  */
 int
-circuit_truncated(origin_circuit_t *circ, crypt_path_t *layer, int reason)
+circuit_truncated(origin_circuit_t *circ, int reason)
 {
 //  crypt_path_t *victim;
 //  connection_t *stream;
 
   tor_assert(circ);
-  tor_assert(layer);
 
   /* XXX Since we don't send truncates currently, getting a truncated
    *     means that a connection broke or an extend failed. For now,
@@ -1374,7 +1466,7 @@ circuit_truncated(origin_circuit_t *circ, crypt_path_t *layer, int reason)
 
   log_info(LD_CIRC, "finished");
   return 0;
-#endif
+#endif /* 0 */
 }
 
 /** Given a response payload and keys, initialize, then send a created
@@ -1383,11 +1475,12 @@ circuit_truncated(origin_circuit_t *circ, crypt_path_t *layer, int reason)
 int
 onionskin_answer(or_circuit_t *circ,
                  const created_cell_t *created_cell,
-                 const char *keys,
+                 const char *keys, size_t keys_len,
                  const uint8_t *rend_circ_nonce)
 {
   cell_t cell;
-  crypt_path_t *tmp_cpath;
+
+  tor_assert(keys_len == CPATH_KEY_MATERIAL_LEN);
 
   if (created_cell_format(&cell, created_cell) < 0) {
     log_warn(LD_BUG,"couldn't format created cell (type=%d, len=%d)",
@@ -1396,34 +1489,24 @@ onionskin_answer(or_circuit_t *circ,
   }
   cell.circ_id = circ->p_circ_id;
 
-  tmp_cpath = tor_malloc_zero(sizeof(crypt_path_t));
-  tmp_cpath->magic = CRYPT_PATH_MAGIC;
-
   circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN);
 
   log_debug(LD_CIRC,"init digest forward 0x%.8x, backward 0x%.8x.",
             (unsigned int)get_uint32(keys),
             (unsigned int)get_uint32(keys+20));
-  if (circuit_init_cpath_crypto(tmp_cpath, keys, 0)<0) {
+  if (relay_crypto_init(&circ->crypto, keys, keys_len, 0, 0)<0) {
     log_warn(LD_BUG,"Circuit initialization failed");
-    tor_free(tmp_cpath);
     return -1;
   }
-  circ->n_digest = tmp_cpath->f_digest;
-  circ->n_crypto = tmp_cpath->f_crypto;
-  circ->p_digest = tmp_cpath->b_digest;
-  circ->p_crypto = tmp_cpath->b_crypto;
-  tmp_cpath->magic = 0;
-  tor_free(tmp_cpath);
 
   memcpy(circ->rend_circ_nonce, rend_circ_nonce, DIGEST_LEN);
 
-  circ->is_first_hop = (created_cell->cell_type == CELL_CREATED_FAST);
+  int used_create_fast = (created_cell->cell_type == CELL_CREATED_FAST);
 
   append_cell_to_circuit_queue(TO_CIRCUIT(circ),
                                circ->p_chan, &cell, CELL_DIRECTION_IN, 0);
   log_debug(LD_CIRC,"Finished sending '%s' cell.",
-            circ->is_first_hop ? "created_fast" : "created");
+            used_create_fast ? "created_fast" : "created");
 
   /* Ignore the local bit when ExtendAllowPrivateAddresses is set:
    * it violates the assumption that private addresses are local.
@@ -1441,13 +1524,137 @@ onionskin_answer(or_circuit_t *circ,
   return 0;
 }
 
-/** Choose a length for a circuit of purpose <b>purpose</b>: three + the
- * number of endpoints that would give something away about our destination.
+/** Helper for new_route_len().  Choose a circuit length for purpose
+ * <b>purpose</b>: DEFAULT_ROUTE_LEN (+ 1 if someone else chose the
+ * exit).  If someone else chose the exit, they could be colluding
+ * with the exit, so add a randomly selected node to preserve
+ * anonymity.
+ *
+ * Here, "exit node" sometimes means an OR acting as an internal
+ * endpoint, rather than as a relay to an external endpoint.  This
+ * means there need to be at least DEFAULT_ROUTE_LEN routers between
+ * us and the internal endpoint to preserve the same anonymity
+ * properties that we would get when connecting to an external
+ * endpoint.  These internal endpoints can include:
+ *
+ *   - Connections to a directory of hidden services
+ *     (CIRCUIT_PURPOSE_C_GENERAL)
+ *
+ *   - A client connecting to an introduction point, which the hidden
+ *     service picked (CIRCUIT_PURPOSE_C_INTRODUCING, via
+ *     circuit_get_open_circ_or_launch() which rewrites it from
+ *     CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT)
+ *
+ *   - A hidden service connecting to a rendezvous point, which the
+ *     client picked (CIRCUIT_PURPOSE_S_CONNECT_REND, via
+ *     rend_service_receive_introduction() and
+ *     rend_service_relaunch_rendezvous)
+ *
+ * There are currently two situations where we picked the exit node
+ * ourselves, making DEFAULT_ROUTE_LEN a safe circuit length:
+ *
+ *   - We are a hidden service connecting to an introduction point
+ *     (CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, via
+ *     rend_service_launch_establish_intro())
+ *
+ *   - We are a router testing its own reachabiity
+ *     (CIRCUIT_PURPOSE_TESTING, via router_do_reachability_checks())
+ *
+ * onion_pick_cpath_exit() bypasses us (by not calling
+ * new_route_len()) in the one-hop tunnel case, so we don't need to
+ * handle that.
+ */
+int
+route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei)
+{
+  int routelen = DEFAULT_ROUTE_LEN;
+  int known_purpose = 0;
+
+  if (circuit_should_use_vanguards(purpose)) {
+    /* Clients want an extra hop for rends to avoid linkability.
+     * Services want it for intro points to avoid publishing their
+     * layer3 guards. They want it for hsdir posts to use
+     * their full layer3 guard set for those connections.
+     * Ex: C - G - L2 - L3 - R
+     *     S - G - L2 - L3 - HSDIR
+     *     S - G - L2 - L3 - I
+     */
+    if (purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND ||
+        purpose == CIRCUIT_PURPOSE_S_HSDIR_POST ||
+        purpose == CIRCUIT_PURPOSE_HS_VANGUARDS ||
+        purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO)
+      return routelen+1;
+
+    /* If we only have Layer2 vanguards, then we do not need
+     * the extra hop for linkabilty reasons (see below).
+     * This means all hops can be of the form:
+     *   S/C - G - L2 - M - R/HSDir/I
+     */
+    if (get_options()->HSLayer2Nodes && !get_options()->HSLayer3Nodes)
+      return routelen+1;
+
+    /* For connections to hsdirs, clients want two extra hops
+     * when using layer3 guards, to avoid linkability.
+     * Same goes for intro points. Note that the route len
+     * includes the intro point or hsdir, hence the +2.
+     * Ex: C - G - L2 - L3 - M - I
+     *     C - G - L2 - L3 - M - HSDIR
+     *     S - G - L2 - L3 - M - R
+     */
+    if (purpose == CIRCUIT_PURPOSE_S_CONNECT_REND ||
+        purpose == CIRCUIT_PURPOSE_C_HSDIR_GET ||
+        purpose == CIRCUIT_PURPOSE_C_INTRODUCING)
+      return routelen+2;
+  }
+
+  if (!exit_ei)
+    return routelen;
+
+  switch (purpose) {
+    /* These two purposes connect to a router that we chose, so
+     * DEFAULT_ROUTE_LEN is safe. */
+  case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO:
+    /* hidden service connecting to introduction point */
+  case CIRCUIT_PURPOSE_TESTING:
+    /* router reachability testing */
+    known_purpose = 1;
+    break;
+
+    /* These three purposes connect to a router that someone else
+     * might have chosen, so add an extra hop to protect anonymity. */
+  case CIRCUIT_PURPOSE_C_GENERAL:
+  case CIRCUIT_PURPOSE_C_HSDIR_GET:
+  case CIRCUIT_PURPOSE_S_HSDIR_POST:
+    /* connecting to hidden service directory */
+  case CIRCUIT_PURPOSE_C_INTRODUCING:
+    /* client connecting to introduction point */
+  case CIRCUIT_PURPOSE_S_CONNECT_REND:
+    /* hidden service connecting to rendezvous point */
+    known_purpose = 1;
+    routelen++;
+    break;
+
+  default:
+    /* Got a purpose not listed above along with a chosen exit.
+     * Increase the circuit length by one anyway for safety. */
+    routelen++;
+    break;
+  }
+
+  if (BUG(exit_ei && !known_purpose)) {
+    log_warn(LD_BUG, "Unhandled purpose %d with a chosen exit; "
+             "assuming routelen %d.", purpose, routelen);
+  }
+  return routelen;
+}
+
+/** Choose a length for a circuit of purpose <b>purpose</b> and check
+ * if enough routers are available.
  *
  * If the routerlist <b>nodes</b> doesn't have enough routers
  * to handle the desired path length, return -1.
  */
-static int
+STATIC int
 new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes)
 {
   int num_acceptable_routers;
@@ -1455,11 +1662,7 @@ new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes)
 
   tor_assert(nodes);
 
-  routelen = DEFAULT_ROUTE_LEN;
-  if (exit_ei &&
-      purpose != CIRCUIT_PURPOSE_TESTING &&
-      purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO)
-    routelen++;
+  routelen = route_len_for_purpose(purpose, exit_ei);
 
   num_acceptable_routers = count_acceptable_nodes(nodes);
 
@@ -1492,9 +1695,9 @@ circuit_get_unhandled_ports(time_t now)
  * If we're returning 0, set need_uptime and need_capacity to
  * indicate any requirements that the unhandled ports have.
  */
-int
-circuit_all_predicted_ports_handled(time_t now, int *need_uptime,
-                                    int *need_capacity)
+MOCK_IMPL(int,
+circuit_all_predicted_ports_handled, (time_t now, int *need_uptime,
+                                      int *need_capacity))
 {
   int i, enough;
   uint16_t *port;
@@ -1571,7 +1774,7 @@ ap_stream_wants_exit_attention(connection_t *conn)
  * Return NULL if we can't find any suitable routers.
  */
 static const node_t *
-choose_good_exit_server_general(int need_uptime, int need_capacity)
+choose_good_exit_server_general(router_crn_flags_t flags)
 {
   int *n_supported;
   int n_pending_connections = 0;
@@ -1581,6 +1784,9 @@ choose_good_exit_server_general(int need_uptime, int need_capacity)
   const or_options_t *options = get_options();
   const smartlist_t *the_nodes;
   const node_t *selected_node=NULL;
+  const int need_uptime = (flags & CRN_NEED_UPTIME) != 0;
+  const int need_capacity = (flags & CRN_NEED_CAPACITY) != 0;
+  const int direct_conn = (flags & CRN_DIRECT_CONN) != 0;
 
   connections = get_connection_array();
 
@@ -1613,7 +1819,7 @@ choose_good_exit_server_general(int need_uptime, int need_capacity)
        */
       continue;
     }
-    if (!node_has_descriptor(node)) {
+    if (!node_has_preferred_descriptor(node, direct_conn)) {
       n_supported[i] = -1;
       continue;
     }
@@ -1643,15 +1849,16 @@ choose_good_exit_server_general(int need_uptime, int need_capacity)
                  * we'll retry later in this function with need_update and
                  * need_capacity set to 0. */
     }
-    if (!(node->is_valid || options->AllowInvalid_ & ALLOW_INVALID_EXIT)) {
+    if (!(node->is_valid)) {
       /* if it's invalid and we don't want it */
       n_supported[i] = -1;
 //      log_fn(LOG_DEBUG,"Skipping node %s (index %d) -- invalid router.",
 //             router->nickname, i);
       continue; /* skip invalid routers */
     }
-    if (options->ExcludeSingleHopRelays &&
-        node_allows_single_hop_exits(node)) {
+    /* We do not allow relays that allow single hop exits by default. Option
+     * was deprecated in 0.2.9.2-alpha and removed in 0.3.1.0-alpha. */
+    if (node_allows_single_hop_exits(node)) {
       n_supported[i] = -1;
       continue;
     }
@@ -1725,7 +1932,8 @@ choose_good_exit_server_general(int need_uptime, int need_capacity)
                  need_capacity?", fast":"",
                  need_uptime?", stable":"");
         tor_free(n_supported);
-        return choose_good_exit_server_general(0, 0);
+        flags &= ~(CRN_NEED_UPTIME|CRN_NEED_CAPACITY);
+        return choose_good_exit_server_general(flags);
       }
       log_notice(LD_CIRC, "All routers are down or won't exit%s -- "
                  "choosing a doomed exit at random.",
@@ -1766,111 +1974,112 @@ choose_good_exit_server_general(int need_uptime, int need_capacity)
   }
   if (options->ExitNodes) {
     log_warn(LD_CIRC,
-             "No specified %sexit routers seem to be running: "
+             "No exits in ExitNodes%s seem to be running: "
              "can't choose an exit.",
-             options->ExcludeExitNodesUnion_ ? "non-excluded " : "");
+             options->ExcludeExitNodesUnion_ ?
+             ", except possibly those excluded by your configuration, " : "");
   }
   return NULL;
 }
 
-#if defined(ENABLE_TOR2WEB_MODE) || defined(TOR_UNIT_TESTS)
-/* The config option Tor2webRendezvousPoints has been set and we need
- * to pick an RP out of that set. Make sure that the RP we choose is
- * alive, and return it. Return NULL if no usable RP could be found in
- * Tor2webRendezvousPoints. */
-STATIC const node_t *
-pick_tor2web_rendezvous_node(router_crn_flags_t flags,
-                             const or_options_t *options)
+/* Pick a Rendezvous Point for our HS circuits according to <b>flags</b>. */
+static const node_t *
+pick_rendezvous_node(router_crn_flags_t flags)
 {
-  const node_t *rp_node = NULL;
-  const int allow_invalid = (flags & CRN_ALLOW_INVALID) != 0;
-  const int need_desc = (flags & CRN_NEED_DESC) != 0;
-  const int pref_addr = (flags & CRN_PREF_ADDR) != 0;
-  const int direct_conn = (flags & CRN_DIRECT_CONN) != 0;
+  const or_options_t *options = get_options();
+  return router_choose_random_node(NULL, options->ExcludeNodes, flags);
+}
 
-  smartlist_t *whitelisted_live_rps = smartlist_new();
+/*
+ * Helper function to pick a configured restricted middle node
+ * (either HSLayer2Nodes or HSLayer3Nodes).
+ *
+ * Make sure that the node we chose is alive, and not excluded,
+ * and return it.
+ *
+ * The exclude_set is a routerset of nodes that the selected node
+ * must not match, and the exclude_list is a simple list of nodes
+ * that the selected node must not be in. Either or both may be
+ * NULL.
+ *
+ * Return NULL if no usable nodes could be found. */
+static const node_t *
+pick_restricted_middle_node(router_crn_flags_t flags,
+                            const routerset_t *pick_from,
+                            const routerset_t *exclude_set,
+                            const smartlist_t *exclude_list,
+                            int position_hint)
+{
+  const node_t *middle_node = NULL;
+
+  smartlist_t *whitelisted_live_middles = smartlist_new();
   smartlist_t *all_live_nodes = smartlist_new();
 
-  tor_assert(options->Tor2webRendezvousPoints);
+  tor_assert(pick_from);
 
   /* Add all running nodes to all_live_nodes */
   router_add_running_nodes_to_smartlist(all_live_nodes,
-                                        allow_invalid,
-                                        0, 0, 0,
-                                        need_desc,
-                                        pref_addr,
-                                        direct_conn);
-
-  /* Filter all_live_nodes to only add live *and* whitelisted RPs to
-   * the list whitelisted_live_rps. */
+                                        (flags & CRN_NEED_UPTIME) != 0,
+                                        (flags & CRN_NEED_CAPACITY) != 0,
+                                        (flags & CRN_NEED_GUARD) != 0,
+                                        (flags & CRN_NEED_DESC) != 0,
+                                        (flags & CRN_PREF_ADDR) != 0,
+                                        (flags & CRN_DIRECT_CONN) != 0);
+
+  /* Filter all_live_nodes to only add live *and* whitelisted middles
+   * to the list whitelisted_live_middles. */
   SMARTLIST_FOREACH_BEGIN(all_live_nodes, node_t *, live_node) {
-    if (routerset_contains_node(options->Tor2webRendezvousPoints, live_node)) {
-      smartlist_add(whitelisted_live_rps, live_node);
+    if (routerset_contains_node(pick_from, live_node)) {
+      smartlist_add(whitelisted_live_middles, live_node);
     }
   } SMARTLIST_FOREACH_END(live_node);
 
   /* Honor ExcludeNodes */
-  if (options->ExcludeNodes) {
-    routerset_subtract_nodes(whitelisted_live_rps, options->ExcludeNodes);
-  }
-
-  /* Now pick randomly amongst the whitelisted RPs. No need to waste time
-     doing bandwidth load balancing, for most use cases
-     'whitelisted_live_rps' contains a single OR anyway. */
-  rp_node = smartlist_choose(whitelisted_live_rps);
-
-  if (!rp_node) {
-    log_warn(LD_REND, "Could not find a Rendezvous Point that suits "
-             "the purposes of Tor2webRendezvousPoints. Choosing random one.");
+  if (exclude_set) {
+    routerset_subtract_nodes(whitelisted_live_middles, exclude_set);
   }
 
-  smartlist_free(whitelisted_live_rps);
-  smartlist_free(all_live_nodes);
-
-  return rp_node;
-}
-#endif
-
-/* Pick a Rendezvous Point for our HS circuits according to <b>flags</b>. */
-static const node_t *
-pick_rendezvous_node(router_crn_flags_t flags)
-{
-  const or_options_t *options = get_options();
-
-  if (options->AllowInvalid_ & ALLOW_INVALID_RENDEZVOUS)
-    flags |= CRN_ALLOW_INVALID;
-
-#ifdef ENABLE_TOR2WEB_MODE
-  /* We want to connect directly to the node if we can */
-  router_crn_flags_t direct_flags = flags;
-  direct_flags |= CRN_PREF_ADDR;
-  direct_flags |= CRN_DIRECT_CONN;
-
-  /* The user wants us to pick specific RPs. */
-  if (options->Tor2webRendezvousPoints) {
-    const node_t *tor2web_rp = pick_tor2web_rendezvous_node(direct_flags,
-                                                            options);
-    if (tor2web_rp) {
-      return tor2web_rp;
-    }
+  if (exclude_list) {
+    smartlist_subtract(whitelisted_live_middles, exclude_list);
   }
 
-  /* Else, if no direct, preferred tor2web RP was found, fall back to choosing
-   * a random direct node */
-  const node_t *node = router_choose_random_node(NULL, options->ExcludeNodes,
-                                                 direct_flags);
-  /* Return the direct node (if found), or log a message and fall back to an
-   * indirect connection. */
-  if (node) {
-    return node;
+  /**
+   * Max number of restricted nodes before we alert the user and try
+   * to load balance for them.
+   *
+   * The most aggressive vanguard design had 16 nodes at layer3.
+   * Let's give a small ceiling above that. */
+#define MAX_SANE_RESTRICTED_NODES 20
+  /* If the user (or associated tor controller) selected only a few nodes,
+   * assume they took load balancing into account and don't do it for them.
+   *
+   * If there are a lot of nodes in here, assume they did not load balance
+   * and do it for them, but also warn them that they may be Doing It Wrong.
+   */
+  if (smartlist_len(whitelisted_live_middles) <=
+          MAX_SANE_RESTRICTED_NODES) {
+    middle_node = smartlist_choose(whitelisted_live_middles);
   } else {
-    log_info(LD_REND,
-             "Unable to find a random rendezvous point that is reachable via "
-             "a direct connection, falling back to a 3-hop path.");
-  }
-#endif
+    static ratelim_t pinned_notice_limit = RATELIM_INIT(24*3600);
+    log_fn_ratelim(&pinned_notice_limit, LOG_NOTICE, LD_CIRC,
+            "Your _HSLayer%dNodes setting has resulted "
+            "in %d total nodes. This is a lot of nodes. "
+            "You may want to consider using a Tor controller "
+            "to select and update a smaller set of nodes instead.",
+            position_hint, smartlist_len(whitelisted_live_middles));
+
+    /* NO_WEIGHTING here just means don't take node flags into account
+     * (ie: use consensus measurement only). This is done so that
+     * we don't further surprise the user by not using Exits that they
+     * specified at all */
+    middle_node = node_sl_choose_by_bandwidth(whitelisted_live_middles,
+                                              NO_WEIGHTING);
+  }
+
+  smartlist_free(whitelisted_live_middles);
+  smartlist_free(all_live_nodes);
 
-  return router_choose_random_node(NULL, options->ExcludeNodes, flags);
+  return middle_node;
 }
 
 /** Return a pointer to a suitable router to be the exit node for the
@@ -1884,24 +2093,25 @@ pick_rendezvous_node(router_crn_flags_t flags)
  * toward the preferences in 'options'.
  */
 static const node_t *
-choose_good_exit_server(uint8_t purpose,
-                        int need_uptime, int need_capacity, int is_internal)
+choose_good_exit_server(origin_circuit_t *circ,
+                        router_crn_flags_t flags, int is_internal)
 {
   const or_options_t *options = get_options();
-  router_crn_flags_t flags = CRN_NEED_DESC;
-  if (need_uptime)
-    flags |= CRN_NEED_UPTIME;
-  if (need_capacity)
-    flags |= CRN_NEED_CAPACITY;
-
-  switch (purpose) {
+  flags |= CRN_NEED_DESC;
+
+  switch (TO_CIRCUIT(circ)->purpose) {
+    case CIRCUIT_PURPOSE_C_HSDIR_GET:
+    case CIRCUIT_PURPOSE_S_HSDIR_POST:
+    case CIRCUIT_PURPOSE_HS_VANGUARDS:
+      /* For these three, we want to pick the exit like a middle hop,
+       * since it should be random. */
+      tor_assert_nonfatal(is_internal);
+      /* Falls through */
     case CIRCUIT_PURPOSE_C_GENERAL:
-      if (options->AllowInvalid_ & ALLOW_INVALID_MIDDLE)
-        flags |= CRN_ALLOW_INVALID;
       if (is_internal) /* pick it like a middle hop */
         return router_choose_random_node(NULL, options->ExcludeNodes, flags);
       else
-        return choose_good_exit_server_general(need_uptime,need_capacity);
+        return choose_good_exit_server_general(flags);
     case CIRCUIT_PURPOSE_C_ESTABLISH_REND:
       {
         /* Pick a new RP */
@@ -1911,7 +2121,7 @@ choose_good_exit_server(uint8_t purpose,
         return rendezvous_node;
       }
   }
-  log_warn(LD_BUG,"Unhandled purpose %d", purpose);
+  log_warn(LD_BUG,"Unhandled purpose %d", TO_CIRCUIT(circ)->purpose);
   tor_fragile_assert();
   return NULL;
 }
@@ -1941,6 +2151,8 @@ warn_if_last_router_excluded(origin_circuit_t *circ,
                (int)purpose,
                circuit_purpose_to_string(purpose));
       return;
+    case CIRCUIT_PURPOSE_S_HSDIR_POST:
+    case CIRCUIT_PURPOSE_C_HSDIR_GET:
     case CIRCUIT_PURPOSE_C_GENERAL:
       if (circ->build_state->is_internal)
         return;
@@ -1994,9 +2206,15 @@ warn_if_last_router_excluded(origin_circuit_t *circ,
 
 /** Decide a suitable length for circ's cpath, and pick an exit
  * router (or use <b>exit</b> if provided). Store these in the
- * cpath. Return 0 if ok, -1 if circuit should be closed. */
-static int
-onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei)
+ * cpath.
+ *
+ * If <b>is_hs_v3_rp_circuit</b> is set, then this exit should be suitable to
+ * be used as an HS v3 rendezvous point.
+ *
+ * Return 0 if ok, -1 if circuit should be closed. */
+STATIC int
+onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei,
+                      int is_hs_v3_rp_circuit)
 {
   cpath_build_state_t *state = circ->build_state;
 
@@ -2018,15 +2236,24 @@ onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei)
              extend_info_describe(exit_ei));
     exit_ei = extend_info_dup(exit_ei);
   } else { /* we have to decide one */
+    router_crn_flags_t flags = CRN_NEED_DESC;
+    if (state->need_uptime)
+      flags |= CRN_NEED_UPTIME;
+    if (state->need_capacity)
+      flags |= CRN_NEED_CAPACITY;
+    if (is_hs_v3_rp_circuit)
+      flags |= CRN_RENDEZVOUS_V3;
+    if (state->onehop_tunnel)
+      flags |= CRN_DIRECT_CONN;
     const node_t *node =
-      choose_good_exit_server(circ->base_.purpose, state->need_uptime,
-                              state->need_capacity, state->is_internal);
+      choose_good_exit_server(circ, flags, state->is_internal);
     if (!node) {
       log_warn(LD_CIRC,"Failed to choose an exit server");
       return -1;
     }
-    exit_ei = extend_info_from_node(node, 0);
-    tor_assert(exit_ei);
+    exit_ei = extend_info_from_node(node, state->onehop_tunnel);
+    if (BUG(exit_ei == NULL))
+      return -1;
   }
   state->chosen_exit = exit_ei;
   return 0;
@@ -2081,9 +2308,13 @@ circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei)
 
 /** Return the number of routers in <b>routers</b> that are currently up
  * and available for building circuits through.
+ *
+ * (Note that this function may overcount or undercount, if we have
+ * descriptors that are not the type we would prefer to use for some
+ * particular router. See bug #25885.)
  */
-static int
-count_acceptable_nodes(smartlist_t *nodes)
+MOCK_IMPL(STATIC int,
+count_acceptable_nodes, (smartlist_t *nodes))
 {
   int num=0;
 
@@ -2094,14 +2325,10 @@ count_acceptable_nodes(smartlist_t *nodes)
     if (! node->is_running)
 //      log_debug(LD_CIRC,"Nope, the directory says %d is not running.",i);
       continue;
-    /* XXX This clause makes us count incorrectly: if AllowInvalidRouters
-     * allows this node in some places, then we're getting an inaccurate
-     * count. For now, be conservative and don't count it. But later we
-     * should try to be smarter. */
     if (! node->is_valid)
 //      log_debug(LD_CIRC,"Nope, the directory says %d is not valid.",i);
       continue;
-    if (! node_has_descriptor(node))
+    if (! node_has_any_descriptor(node))
       continue;
     /* The node has a descriptor, so we can just check the ntor key directly */
     if (!node_has_curve25519_onion_key(node))
@@ -2131,6 +2358,190 @@ onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop)
   }
 }
 
+#ifdef TOR_UNIT_TESTS
+
+/** Unittest helper function: Count number of hops in cpath linked list. */
+unsigned int
+cpath_get_n_hops(crypt_path_t **head_ptr)
+{
+  unsigned int n_hops = 0;
+  crypt_path_t *tmp;
+
+  if (!*head_ptr) {
+    return 0;
+  }
+
+  tmp = *head_ptr;
+  do {
+    n_hops++;
+    tmp = tmp->next;
+  } while (tmp != *head_ptr);
+
+  return n_hops;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Build the exclude list for vanguard circuits.
+ *
+ * For vanguard circuits we exclude all the already chosen nodes (including the
+ * exit) from being middle hops to prevent the creation of A - B - A subpaths.
+ * We also allow the 4th hop to be the same as the guard node so as to not leak
+ * guard information to RP/IP/HSDirs.
+ *
+ * For vanguard circuits, we don't apply any subnet or family restrictions.
+ * This is to avoid impossible-to-build circuit paths, or just situations where
+ * our earlier guards prevent us from using most of our later ones.
+ *
+ * The alternative is building the circuit in reverse. Reverse calls to
+ * onion_extend_cpath() (ie: select outer hops first) would then have the
+ * property that you don't gain information about inner hops by observing
+ * outer ones. See https://trac.torproject.org/projects/tor/ticket/24487
+ * for this.
+ *
+ * (Note further that we still exclude the exit to prevent A - B - A
+ * at the end of the path. */
+static smartlist_t *
+build_vanguard_middle_exclude_list(uint8_t purpose,
+                                   cpath_build_state_t *state,
+                                   crypt_path_t *head,
+                                   int cur_len)
+{
+  smartlist_t *excluded;
+  const node_t *r;
+  crypt_path_t *cpath;
+  int i;
+
+  (void) purpose;
+
+  excluded = smartlist_new();
+
+  /* Add the exit to the exclude list (note that the exit/last hop is always
+   * chosen first in circuit_establish_circuit()). */
+  if ((r = build_state_get_exit_node(state))) {
+    smartlist_add(excluded, (node_t*)r);
+  }
+
+  /* If we are picking the 4th hop, allow that node to be the guard too.
+   * This prevents us from avoiding the Guard for those hops, which
+   * gives the adversary information about our guard if they control
+   * the RP, IP, or HSDIR. We don't do this check based on purpose
+   * because we also want to allow HS_VANGUARDS pre-build circuits
+   * to use the guard for that last hop.
+   */
+  if (cur_len == DEFAULT_ROUTE_LEN+1) {
+    /* Skip the first hop for the exclude list below */
+    head = head->next;
+    cur_len--;
+  }
+
+  for (i = 0, cpath = head; cpath && i < cur_len; ++i, cpath=cpath->next) {
+    if ((r = node_get_by_id(cpath->extend_info->identity_digest))) {
+      smartlist_add(excluded, (node_t*)r);
+    }
+  }
+
+  return excluded;
+}
+
+/**
+ * Build a list of nodes to exclude from the choice of this middle
+ * hop, based on already chosen nodes.
+ */
+static smartlist_t *
+build_middle_exclude_list(uint8_t purpose,
+                          cpath_build_state_t *state,
+                          crypt_path_t *head,
+                          int cur_len)
+{
+  smartlist_t *excluded;
+  const node_t *r;
+  crypt_path_t *cpath;
+  int i;
+
+  /** Vanguard circuits have their own path selection rules */
+  if (circuit_should_use_vanguards(purpose)) {
+    return build_vanguard_middle_exclude_list(purpose, state, head, cur_len);
+  }
+
+  excluded = smartlist_new();
+
+  /* For non-vanguard circuits, add the exit and its family to the exclude list
+   * (note that the exit/last hop is always chosen first in
+   * circuit_establish_circuit()). */
+  if ((r = build_state_get_exit_node(state))) {
+    nodelist_add_node_and_family(excluded, r);
+  }
+
+  /* also exclude all other already chosen nodes and their family */
+  for (i = 0, cpath = head; cpath && i < cur_len; ++i, cpath=cpath->next) {
+    if ((r = node_get_by_id(cpath->extend_info->identity_digest))) {
+      nodelist_add_node_and_family(excluded, r);
+    }
+  }
+
+  return excluded;
+}
+
+/** Return true if we MUST use vanguards for picking this middle node. */
+static int
+middle_node_must_be_vanguard(const or_options_t *options,
+                             uint8_t purpose, int cur_len)
+{
+  /* If this is not a hidden service circuit, don't use vanguards */
+  if (!circuit_purpose_is_hidden_service(purpose)) {
+    return 0;
+  }
+
+  /* If we have sticky L2 nodes, and this is an L2 pick, use vanguards */
+  if (options->HSLayer2Nodes && cur_len == 1) {
+    return 1;
+  }
+
+  /* If we have sticky L3 nodes, and this is an L3 pick, use vanguards */
+  if (options->HSLayer3Nodes && cur_len == 2) {
+    return 1;
+  }
+
+  return 0;
+}
+
+/** Pick a sticky vanguard middle node or return NULL if not found.
+ *  See doc of pick_restricted_middle_node() for argument details. */
+static const node_t *
+pick_vanguard_middle_node(const or_options_t *options,
+                          router_crn_flags_t flags, int cur_len,
+                          const smartlist_t *excluded)
+{
+  const routerset_t *vanguard_routerset = NULL;
+  const node_t *node = NULL;
+
+  /* Pick the right routerset based on the current hop */
+  if (cur_len == 1) {
+    vanguard_routerset = options->HSLayer2Nodes;
+  } else if (cur_len == 2) {
+    vanguard_routerset = options->HSLayer3Nodes;
+  } else {
+    /* guaranteed by middle_node_should_be_vanguard() */
+    tor_assert_nonfatal_unreached();
+    return NULL;
+  }
+
+  node = pick_restricted_middle_node(flags, vanguard_routerset,
+                                     options->ExcludeNodes, excluded,
+                                     cur_len+1);
+
+  if (!node) {
+    static ratelim_t pinned_warning_limit = RATELIM_INIT(300);
+    log_fn_ratelim(&pinned_warning_limit, LOG_WARN, LD_CIRC,
+            "Could not find a node that matches the configured "
+            "_HSLayer%dNodes set", cur_len+1);
+  }
+
+  return node;
+}
+
 /** A helper function used by onion_extend_cpath(). Use <b>purpose</b>
  * and <b>state</b> and the cpath <b>head</b> (currently populated only
  * to length <b>cur_len</b> to decide a suitable middle hop for a
@@ -2143,33 +2554,31 @@ choose_good_middle_server(uint8_t purpose,
                           crypt_path_t *head,
                           int cur_len)
 {
-  int i;
-  const node_t *r, *choice;
-  crypt_path_t *cpath;
+  const node_t *choice;
   smartlist_t *excluded;
   const or_options_t *options = get_options();
   router_crn_flags_t flags = CRN_NEED_DESC;
   tor_assert(CIRCUIT_PURPOSE_MIN_ <= purpose &&
              purpose <= CIRCUIT_PURPOSE_MAX_);
 
-  log_debug(LD_CIRC, "Contemplating intermediate hop %d: random choice.",
-            cur_len);
-  excluded = smartlist_new();
-  if ((r = build_state_get_exit_node(state))) {
-    nodelist_add_node_and_family(excluded, r);
-  }
-  for (i = 0, cpath = head; i < cur_len; ++i, cpath=cpath->next) {
-    if ((r = node_get_by_id(cpath->extend_info->identity_digest))) {
-      nodelist_add_node_and_family(excluded, r);
-    }
-  }
+  log_debug(LD_CIRC, "Contemplating intermediate hop #%d: random choice.",
+            cur_len+1);
+
+  excluded = build_middle_exclude_list(purpose, state, head, cur_len);
 
   if (state->need_uptime)
     flags |= CRN_NEED_UPTIME;
   if (state->need_capacity)
     flags |= CRN_NEED_CAPACITY;
-  if (options->AllowInvalid_ & ALLOW_INVALID_MIDDLE)
-    flags |= CRN_ALLOW_INVALID;
+
+  /** If a hidden service circuit wants a specific middle node, pin it. */
+  if (middle_node_must_be_vanguard(options, purpose, cur_len)) {
+    log_debug(LD_GENERAL, "Picking a sticky node (cur_len = %d)", cur_len);
+    choice = pick_vanguard_middle_node(options, flags, cur_len, excluded);
+    smartlist_free(excluded);
+    return choice;
+  }
+
   choice = router_choose_random_node(excluded, options->ExcludeNodes, flags);
   smartlist_free(excluded);
   return choice;
@@ -2180,11 +2589,13 @@ choose_good_middle_server(uint8_t purpose,
  * router (if we're an OR), and respect firewall settings; if we're
  * configured to use entry guards, return one.
  *
- * If <b>state</b> is NULL, we're choosing a router to serve as an entry
- * guard, not for any particular circuit.
+ * Set *<b>guard_state_out</b> to information about the guard that
+ * we're selecting, which we'll use later to remember whether the
+ * guard worked or not.
  */
 const node_t *
-choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state)
+choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state,
+                         circuit_guard_state_t **guard_state_out)
 {
   const node_t *choice;
   smartlist_t *excluded;
@@ -2195,11 +2606,17 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state)
                               CRN_DIRECT_CONN);
   const node_t *node;
 
+  /* Once we used this function to select a node to be a guard.  We had
+   * 'state == NULL' be the signal for that.  But we don't do that any more.
+   */
+  tor_assert_nonfatal(state);
+
   if (state && options->UseEntryGuards &&
       (purpose != CIRCUIT_PURPOSE_TESTING || options->BridgeRelay)) {
     /* This request is for an entry server to use for a regular circuit,
      * and we use entry guard nodes.  Just return one of the guard nodes.  */
-    return choose_random_entry(state);
+    tor_assert(guard_state_out);
+    return guards_choose_guard(state, purpose, guard_state_out);
   }
 
   excluded = smartlist_new();
@@ -2209,25 +2626,6 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state)
      * family. */
     nodelist_add_node_and_family(excluded, node);
   }
-  /* and exclude current entry guards and their families,
-   * unless we're in a test network, and excluding guards
-   * would exclude all nodes (i.e. we're in an incredibly small tor network,
-   * or we're using TestingAuthVoteGuard *).
-   * This is an incomplete fix, but is no worse than the previous behaviour,
-   * and only applies to minimal, testing tor networks
-   * (so it's no less secure) */
-  /*XXXX++ use the using_as_guard flag to accomplish this.*/
-  if (options->UseEntryGuards
-      && (!options->TestingTorNetwork ||
-         smartlist_len(nodelist_get_list()) > smartlist_len(get_entry_guards())
-     )) {
-    SMARTLIST_FOREACH(get_entry_guards(), const entry_guard_t *, entry,
-      {
-        if ((node = node_get_by_id(entry->identity))) {
-          nodelist_add_node_and_family(excluded, node);
-        }
-      });
-  }
 
   if (state) {
     if (state->need_uptime)
@@ -2235,8 +2633,6 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state)
     if (state->need_capacity)
       flags |= CRN_NEED_CAPACITY;
   }
-  if (options->AllowInvalid_ & ALLOW_INVALID_ENTRY)
-    flags |= CRN_ALLOW_INVALID;
 
   choice = router_choose_random_node(excluded, options->ExcludeNodes, flags);
   smartlist_free(excluded);
@@ -2257,13 +2653,13 @@ onion_next_hop_in_cpath(crypt_path_t *cpath)
   return NULL;
 }
 
-/** Choose a suitable next hop in the cpath <b>head_ptr</b>,
- * based on <b>state</b>. Append the hop info to head_ptr.
+/** Choose a suitable next hop for the circuit <b>circ</b>.
+ * Append the hop info to circ->cpath.
  *
  * Return 1 if the path is complete, 0 if we successfully added a hop,
  * and -1 on error.
  */
-static int
+STATIC int
 onion_extend_cpath(origin_circuit_t *circ)
 {
   uint8_t purpose = circ->base_.purpose;
@@ -2283,7 +2679,8 @@ onion_extend_cpath(origin_circuit_t *circ)
   if (cur_len == state->desired_path_len - 1) { /* Picking last node */
     info = extend_info_dup(state->chosen_exit);
   } else if (cur_len == 0) { /* picking first node */
-    const node_t *r = choose_good_entry_server(purpose, state);
+    const node_t *r = choose_good_entry_server(purpose, state,
+                                               &circ->guard_state);
     if (r) {
       /* If we're a client, use the preferred address rather than the
          primary address, for potentially connecting to an IPv6 OR
@@ -2291,24 +2688,24 @@ onion_extend_cpath(origin_circuit_t *circ)
       int client = (server_mode(get_options()) == 0);
       info = extend_info_from_node(r, client);
       /* Clients can fail to find an allowed address */
-      tor_assert(info || client);
+      tor_assert_nonfatal(info || client);
     }
   } else {
     const node_t *r =
       choose_good_middle_server(purpose, state, circ->cpath, cur_len);
     if (r) {
       info = extend_info_from_node(r, 0);
-      tor_assert(info);
+      tor_assert_nonfatal(info);
     }
   }
 
   if (!info) {
-    log_warn(LD_CIRC,"Failed to find node for hop %d of our path. Discarding "
-             "this circuit.", cur_len);
+    log_warn(LD_CIRC,"Failed to find node for hop #%d of our path. Discarding "
+             "this circuit.", cur_len+1);
     return -1;
   }
 
-  log_debug(LD_CIRC,"Chose router %s for hop %d (exit is %s)",
+  log_debug(LD_CIRC,"Chose router %s for hop #%d (exit is %s)",
             extend_info_describe(info),
             cur_len+1, build_state_get_exit_nickname(state));
 
@@ -2320,7 +2717,7 @@ onion_extend_cpath(origin_circuit_t *circ)
 /** Create a new hop, annotate it with information about its
  * corresponding router <b>choice</b>, and append it to the
  * end of the cpath <b>head_ptr</b>. */
-static int
+STATIC int
 onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice)
 {
   crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t));
@@ -2341,19 +2738,23 @@ onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice)
 
 /** Allocate a new extend_info object based on the various arguments. */
 extend_info_t *
-extend_info_new(const char *nickname, const char *digest,
+extend_info_new(const char *nickname,
+                const char *rsa_id_digest,
+                const ed25519_public_key_t *ed_id,
                 crypto_pk_t *onion_key,
-                const curve25519_public_key_t *curve25519_key,
+                const curve25519_public_key_t *ntor_key,
                 const tor_addr_t *addr, uint16_t port)
 {
   extend_info_t *info = tor_malloc_zero(sizeof(extend_info_t));
-  memcpy(info->identity_digest, digest, DIGEST_LEN);
+  memcpy(info->identity_digest, rsa_id_digest, DIGEST_LEN);
+  if (ed_id && !ed25519_public_key_is_zero(ed_id))
+    memcpy(&info->ed_identity, ed_id, sizeof(ed25519_public_key_t));
   if (nickname)
     strlcpy(info->nickname, nickname, sizeof(info->nickname));
   if (onion_key)
     info->onion_key = crypto_pk_dup_key(onion_key);
-  if (curve25519_key)
-    memcpy(&info->curve25519_onion_key, curve25519_key,
+  if (ntor_key)
+    memcpy(&info->curve25519_onion_key, ntor_key,
            sizeof(curve25519_public_key_t));
   tor_addr_copy(&info->addr, addr);
   info->port = port;
@@ -2365,27 +2766,30 @@ extend_info_new(const char *nickname, const char *digest,
  * of the node (i.e. its IPv4 address) unless
  * <b>for_direct_connect</b> is true, in which case the preferred
  * address is used instead. May return NULL if there is not enough
- * info about <b>node</b> to extend to it--for example, if there is no
- * routerinfo_t or microdesc_t, or if for_direct_connect is true and none of
- * the node's addresses are allowed by tor's firewall and IP version config.
+ * info about <b>node</b> to extend to it--for example, if the preferred
+ * routerinfo_t or microdesc_t is missing, or if for_direct_connect is
+ * true and none of the node's addresses is allowed by tor's firewall
+ * and IP version config.
  **/
 extend_info_t *
 extend_info_from_node(const node_t *node, int for_direct_connect)
 {
+  crypto_pk_t *rsa_pubkey = NULL;
+  extend_info_t *info = NULL;
   tor_addr_port_t ap;
   int valid_addr = 0;
 
-  if (node->ri == NULL && (node->rs == NULL || node->md == NULL))
+  if (!node_has_preferred_descriptor(node, for_direct_connect)) {
     return NULL;
+  }
 
-  /* Choose a preferred address first, but fall back to an allowed address.
-   * choose_address returns 1 on success, but get_prim_orport returns 0. */
+  /* Choose a preferred address first, but fall back to an allowed address. */
   if (for_direct_connect)
-    valid_addr = fascist_firewall_choose_address_node(node,
-                                                      FIREWALL_OR_CONNECTION,
-                                                      0, &ap);
-  else
-    valid_addr = !node_get_prim_orport(node, &ap);
+    fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0, &ap);
+  else {
+    node_get_prim_orport(node, &ap);
+  }
+  valid_addr = tor_addr_port_is_valid_ap(&ap, 0);
 
   if (valid_addr)
     log_debug(LD_CIRC, "using %s for %s",
@@ -2403,27 +2807,49 @@ extend_info_from_node(const node_t *node, int for_direct_connect)
     return NULL;
   }
 
-  if (valid_addr && node->ri)
-    return extend_info_new(node->ri->nickname,
-                             node->identity,
-                             node->ri->onion_pkey,
-                             node->ri->onion_curve25519_pkey,
-                             &ap.addr,
-                             ap.port);
-  else if (valid_addr && node->rs && node->md)
-    return extend_info_new(node->rs->nickname,
-                             node->identity,
-                             node->md->onion_pkey,
-                             node->md->onion_curve25519_pkey,
-                             &ap.addr,
-                             ap.port);
-  else
-    return NULL;
+  const ed25519_public_key_t *ed_pubkey = NULL;
+
+  /* Don't send the ed25519 pubkey unless the target node actually supports
+   * authenticating with it. */
+  if (node_supports_ed25519_link_authentication(node, 0)) {
+    log_info(LD_CIRC, "Including Ed25519 ID for %s", node_describe(node));
+    ed_pubkey = node_get_ed25519_id(node);
+  } else if (node_get_ed25519_id(node)) {
+    log_info(LD_CIRC, "Not including the ed25519 ID for %s, since it won't "
+             "be able to authenticate it.",
+             node_describe(node));
+  }
+
+  /* Retrieve the curve25519 pubkey. */
+  const curve25519_public_key_t *curve_pubkey =
+    node_get_curve25519_onion_key(node);
+  rsa_pubkey = node_get_rsa_onion_key(node);
+
+  if (valid_addr && node->ri) {
+    info = extend_info_new(node->ri->nickname,
+                           node->identity,
+                           ed_pubkey,
+                           rsa_pubkey,
+                           curve_pubkey,
+                           &ap.addr,
+                           ap.port);
+  } else if (valid_addr && node->rs && node->md) {
+    info = extend_info_new(node->rs->nickname,
+                           node->identity,
+                           ed_pubkey,
+                           rsa_pubkey,
+                           curve_pubkey,
+                           &ap.addr,
+                           ap.port);
+  }
+
+  crypto_pk_free(rsa_pubkey);
+  return info;
 }
 
 /** Release storage held by an extend_info_t struct. */
 void
-extend_info_free(extend_info_t *info)
+extend_info_free_(extend_info_t *info)
 {
   if (!info)
     return;
@@ -2447,8 +2873,8 @@ extend_info_dup(extend_info_t *info)
   return newinfo;
 }
 
-/** Return the routerinfo_t for the chosen exit router in <b>state</b>.
- * If there is no chosen exit, or if we don't know the routerinfo_t for
+/** Return the node_t for the chosen exit router in <b>state</b>.
+ * If there is no chosen exit, or if we don't know the node_t for
  * the chosen exit, return NULL.
  */
 const node_t *
@@ -2459,6 +2885,17 @@ build_state_get_exit_node(cpath_build_state_t *state)
   return node_get_by_id(state->chosen_exit->identity_digest);
 }
 
+/** Return the RSA ID digest for the chosen exit router in <b>state</b>.
+ * If there is no chosen exit, return NULL.
+ */
+const uint8_t *
+build_state_get_exit_rsa_id(cpath_build_state_t *state)
+{
+  if (!state || !state->chosen_exit)
+    return NULL;
+  return (const uint8_t *) state->chosen_exit->identity_digest;
+}
+
 /** Return the nickname for the chosen exit router in <b>state</b>. If
  * there is no chosen exit, or if we don't know the routerinfo_t for the
  * chosen exit, return NULL.
@@ -2551,3 +2988,25 @@ extend_info_has_preferred_onion_key(const extend_info_t* ei)
   return extend_info_supports_ntor(ei);
 }
 
+/** Find the circuits that are waiting to find out whether their guards are
+ * usable, and if any are ready to become usable, mark them open and try
+ * attaching streams as appropriate. */
+void
+circuit_upgrade_circuits_from_guard_wait(void)
+{
+  smartlist_t *to_upgrade =
+    circuit_find_circuits_to_upgrade_from_guard_wait();
+
+  if (to_upgrade == NULL)
+    return;
+
+  log_info(LD_GUARD, "Upgrading %d circuits from 'waiting for better guard' "
+           "to 'open'.", smartlist_len(to_upgrade));
+
+  SMARTLIST_FOREACH_BEGIN(to_upgrade, origin_circuit_t *, circ) {
+    circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN);
+    circuit_has_opened(circ);
+  } SMARTLIST_FOREACH_END(circ);
+
+  smartlist_free(to_upgrade);
+}
diff --git a/src/or/circuitbuild.h b/src/core/or/circuitbuild.h
index 1244601f71..969f5b5cc3 100644
--- a/src/or/circuitbuild.h
+++ b/src/core/or/circuitbuild.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,67 +12,91 @@
 #ifndef TOR_CIRCUITBUILD_H
 #define TOR_CIRCUITBUILD_H
 
+struct ed25519_public_key_t;
+struct curve25519_public_key_t;
+
+int route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei);
 char *circuit_list_path(origin_circuit_t *circ, int verbose);
 char *circuit_list_path_for_controller(origin_circuit_t *circ);
 void circuit_log_path(int severity, unsigned int domain,
                       origin_circuit_t *circ);
-void circuit_rep_hist_note_result(origin_circuit_t *circ);
 origin_circuit_t *origin_circuit_init(uint8_t purpose, int flags);
 origin_circuit_t *circuit_establish_circuit(uint8_t purpose,
                                             extend_info_t *exit,
                                             int flags);
+struct circuit_guard_state_t *origin_circuit_get_guard_state(
+                                            origin_circuit_t *circ);
 int circuit_handle_first_hop(origin_circuit_t *circ);
 void circuit_n_chan_done(channel_t *chan, int status,
                          int close_origin_circuits);
 int inform_testing_reachability(void);
-int circuit_timeout_want_to_count_circ(origin_circuit_t *circ);
+int circuit_timeout_want_to_count_circ(const origin_circuit_t *circ);
 int circuit_send_next_onion_skin(origin_circuit_t *circ);
-void circuit_note_clock_jumped(int seconds_elapsed);
+void circuit_note_clock_jumped(int64_t seconds_elapsed, bool was_idle);
 int circuit_extend(cell_t *cell, circuit_t *circ);
-int circuit_init_cpath_crypto(crypt_path_t *cpath, const char *key_data,
-                              int reverse);
+int circuit_init_cpath_crypto(crypt_path_t *cpath,
+                              const char *key_data, size_t key_data_len,
+                              int reverse, int is_hs_v3);
 struct created_cell_t;
 int circuit_finish_handshake(origin_circuit_t *circ,
                              const struct created_cell_t *created_cell);
-int circuit_truncated(origin_circuit_t *circ, crypt_path_t *layer,
-                      int reason);
+int circuit_truncated(origin_circuit_t *circ, int reason);
 int onionskin_answer(or_circuit_t *circ,
                      const struct created_cell_t *created_cell,
-                     const char *keys,
+                     const char *keys, size_t keys_len,
                      const uint8_t *rend_circ_nonce);
-int circuit_all_predicted_ports_handled(time_t now, int *need_uptime,
-                                        int *need_capacity);
+MOCK_DECL(int, circuit_all_predicted_ports_handled, (time_t now,
+                                                     int *need_uptime,
+                                                     int *need_capacity));
 
 int circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *info);
 int circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *info);
 void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop);
-extend_info_t *extend_info_new(const char *nickname, const char *digest,
+extend_info_t *extend_info_new(const char *nickname,
+                               const char *rsa_id_digest,
+                               const struct ed25519_public_key_t *ed_id,
                                crypto_pk_t *onion_key,
-                               const curve25519_public_key_t *curve25519_key,
+                               const struct curve25519_public_key_t *ntor_key,
                                const tor_addr_t *addr, uint16_t port);
 extend_info_t *extend_info_from_node(const node_t *r, int for_direct_connect);
 extend_info_t *extend_info_dup(extend_info_t *info);
-void extend_info_free(extend_info_t *info);
+void extend_info_free_(extend_info_t *info);
+#define extend_info_free(info) \
+  FREE_AND_NULL(extend_info_t, extend_info_free_, (info))
 int extend_info_addr_is_allowed(const tor_addr_t *addr);
 int extend_info_supports_tap(const extend_info_t* ei);
 int extend_info_supports_ntor(const extend_info_t* ei);
 int circuit_can_use_tap(const origin_circuit_t *circ);
 int circuit_has_usable_onion_key(const origin_circuit_t *circ);
 int extend_info_has_preferred_onion_key(const extend_info_t* ei);
+const uint8_t *build_state_get_exit_rsa_id(cpath_build_state_t *state);
 const node_t *build_state_get_exit_node(cpath_build_state_t *state);
 const char *build_state_get_exit_nickname(cpath_build_state_t *state);
 
+struct circuit_guard_state_t;
+
 const node_t *choose_good_entry_server(uint8_t purpose,
-                                       cpath_build_state_t *state);
+                           cpath_build_state_t *state,
+                           struct circuit_guard_state_t **guard_state_out);
+void circuit_upgrade_circuits_from_guard_wait(void);
 
 #ifdef CIRCUITBUILD_PRIVATE
 STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan);
-#if defined(ENABLE_TOR2WEB_MODE) || defined(TOR_UNIT_TESTS)
-STATIC const node_t *pick_tor2web_rendezvous_node(router_crn_flags_t flags,
-                                                  const or_options_t *options);
-#endif
+STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei,
+                         smartlist_t *nodes);
+MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes));
+
+STATIC int onion_extend_cpath(origin_circuit_t *circ);
+
+STATIC int
+onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei,
+                      int is_hs_v3_rp_circuit);
+
+#if defined(TOR_UNIT_TESTS)
+unsigned int cpath_get_n_hops(crypt_path_t **head_ptr);
 
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
-#endif
+#endif /* defined(CIRCUITBUILD_PRIVATE) */
 
+#endif /* !defined(TOR_CIRCUITBUILD_H) */
diff --git a/src/or/circuitlist.c b/src/core/or/circuitlist.c
index 8cfdd3bb94..145004c71d 100644
--- a/src/or/circuitlist.c
+++ b/src/core/or/circuitlist.c
@@ -1,65 +1,172 @@
 /* Copyright 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file circuitlist.c
  *
- * \brief Manage the global circuit list, and looking up circuits within it.
+ * \brief Manage global structures that list and index circuits, and
+ *   look up circuits within them.
+ *
+ * One of the most frequent operations in Tor occurs every time that
+ * a relay cell arrives on a channel.  When that happens, we need to
+ * find which circuit it is associated with, based on the channel and the
+ * circuit ID in the relay cell.
+ *
+ * To handle that, we maintain a global list of circuits, and a hashtable
+ * mapping [channel,circID] pairs to circuits.  Circuits are added to and
+ * removed from this mapping using circuit_set_p_circid_chan() and
+ * circuit_set_n_circid_chan().  To look up a circuit from this map, most
+ * callers should use circuit_get_by_circid_channel(), though
+ * circuit_get_by_circid_channel_even_if_marked() is appropriate under some
+ * circumstances.
+ *
+ * We also need to allow for the possibility that we have blocked use of a
+ * circuit ID (because we are waiting to send a DESTROY cell), but the
+ * circuit is not there any more.  For that case, we allow placeholder
+ * entries in the table, using channel_mark_circid_unusable().
+ *
+ * To efficiently handle a channel that has just opened, we also maintain a
+ * list of the circuits waiting for channels, so we can attach them as
+ * needed without iterating through the whole list of circuits, using
+ * circuit_get_all_pending_on_channel().
+ *
+ * In this module, we also handle the list of circuits that have been
+ * marked for close elsewhere, and close them as needed.  (We use this
+ * "mark now, close later" pattern here and elsewhere to avoid
+ * unpredictable recursion if we closed every circuit immediately upon
+ * realizing it needed to close.)  See circuit_mark_for_close() for the
+ * mark function, and circuit_close_all_marked() for the close function.
+ *
+ * For hidden services, we need to be able to look up introduction point
+ * circuits and rendezvous circuits by cookie, key, etc.  These are
+ * currently handled with linear searches in
+ * circuit_get_ready_rend_circuit_by_rend_data(),
+ * circuit_get_next_by_pk_and_purpose(), and with hash lookups in
+ * circuit_get_rendezvous() and circuit_get_intro_point().
+ *
+ * This module is also the entry point for our out-of-memory handler
+ * logic, which was originally circuit-focused.
  **/
 #define CIRCUITLIST_PRIVATE
-#include "or.h"
-#include "channel.h"
-#include "channeltls.h"
-#include "circpathbias.h"
-#include "circuitbuild.h"
-#include "circuitlist.h"
-#include "circuituse.h"
-#include "circuitstats.h"
-#include "connection.h"
-#include "config.h"
-#include "connection_edge.h"
-#include "connection_or.h"
-#include "control.h"
-#include "main.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "onion.h"
-#include "onion_fast.h"
-#include "policies.h"
-#include "relay.h"
-#include "rendclient.h"
-#include "rendcommon.h"
-#include "rephist.h"
-#include "routerlist.h"
-#include "routerset.h"
+#include "lib/cc/torint.h"  /* TOR_PRIuSZ */
+
+#include "core/or/or.h"
+#include "core/or/channel.h"
+#include "core/or/channeltls.h"
+#include "feature/client/circpathbias.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/circuitstats.h"
+#include "core/mainloop/connection.h"
+#include "app/config/config.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "feature/control/control.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/crypt_ops/crypto_dh.h"
+#include "feature/dircommon/directory.h"
+#include "feature/client/entrynodes.h"
+#include "core/mainloop/mainloop.h"
+#include "feature/hs/hs_circuit.h"
+#include "feature/hs/hs_circuitmap.h"
+#include "feature/hs/hs_ident.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/relay/onion_queue.h"
+#include "core/crypto/onion_crypto.h"
+#include "core/crypto/onion_fast.h"
+#include "core/or/policies.h"
+#include "core/or/relay.h"
+#include "core/crypto/relay_crypto.h"
+#include "feature/rend/rendclient.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/stats/predict_ports.h"
+#include "feature/stats/rephist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerset.h"
+#include "core/or/channelpadding.h"
+#include "lib/compress/compress.h"
+#include "lib/compress/compress_lzma.h"
+#include "lib/compress/compress_zlib.h"
+#include "lib/compress/compress_zstd.h"
+#include "lib/container/buffers.h"
 
 #include "ht.h"
 
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/crypt_path_reference_st.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "core/or/edge_connection_st.h"
+#include "core/or/half_edge_st.h"
+#include "core/or/extend_info_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+
 /********* START VARIABLES **********/
 
 /** A global list of all circuits at this hop. */
 static smartlist_t *global_circuitlist = NULL;
 
+/** A global list of all origin circuits. Every element of this is also
+ * an element of global_circuitlist. */
+static smartlist_t *global_origin_circuit_list = NULL;
+
 /** A list of all the circuits in CIRCUIT_STATE_CHAN_WAIT. */
 static smartlist_t *circuits_pending_chans = NULL;
 
+/** List of all the (origin) circuits whose state is
+ * CIRCUIT_STATE_GUARD_WAIT. */
+static smartlist_t *circuits_pending_other_guards = NULL;
+
 /** A list of all the circuits that have been marked with
  * circuit_mark_for_close and which are waiting for circuit_about_to_free. */
 static smartlist_t *circuits_pending_close = NULL;
 
 static void circuit_free_cpath_node(crypt_path_t *victim);
 static void cpath_ref_decref(crypt_path_reference_t *cpath_ref);
-//static void circuit_set_rend_token(or_circuit_t *circ, int is_rend_circ,
-//                                   const uint8_t *token);
-static void circuit_clear_rend_token(or_circuit_t *circ);
 static void circuit_about_to_free_atexit(circuit_t *circ);
 static void circuit_about_to_free(circuit_t *circ);
 
+/**
+ * A cached value of the current state of the origin circuit list.  Has the
+ * value 1 if we saw any opened circuits recently (since the last call to
+ * circuit_any_opened_circuits(), which gets called around once a second by
+ * circuit_expire_building). 0 otherwise.
+ */
+static int any_opened_circs_cached_val = 0;
+
 /********* END VARIABLES ************/
 
+or_circuit_t *
+TO_OR_CIRCUIT(circuit_t *x)
+{
+  tor_assert(x->magic == OR_CIRCUIT_MAGIC);
+  return DOWNCAST(or_circuit_t, x);
+}
+const or_circuit_t *
+CONST_TO_OR_CIRCUIT(const circuit_t *x)
+{
+  tor_assert(x->magic == OR_CIRCUIT_MAGIC);
+  return DOWNCAST(or_circuit_t, x);
+}
+origin_circuit_t *
+TO_ORIGIN_CIRCUIT(circuit_t *x)
+{
+  tor_assert(x->magic == ORIGIN_CIRCUIT_MAGIC);
+  return DOWNCAST(origin_circuit_t, x);
+}
+const origin_circuit_t *
+CONST_TO_ORIGIN_CIRCUIT(const circuit_t *x)
+{
+  tor_assert(x->magic == ORIGIN_CIRCUIT_MAGIC);
+  return DOWNCAST(origin_circuit_t, x);
+}
+
 /** A map from channel and circuit ID to circuit.  (Lookup performance is
  * very important here, since we need to do it every time a cell arrives.) */
 typedef struct chan_circid_circuit_map_t {
@@ -310,8 +417,8 @@ channel_note_destroy_pending(channel_t *chan, circid_t id)
 
 /** Called to indicate that a DESTROY is no longer pending on <b>chan</b> with
  * circuit ID <b>id</b> -- typically, because it has been sent. */
-MOCK_IMPL(void, channel_note_destroy_not_pending,
-          (channel_t *chan, circid_t id))
+MOCK_IMPL(void,
+channel_note_destroy_not_pending,(channel_t *chan, circid_t id))
 {
   circuit_t *circ = circuit_get_by_circid_channel_even_if_marked(id,chan);
   if (circ) {
@@ -343,9 +450,6 @@ circuit_set_p_circid_chan(or_circuit_t *or_circ, circid_t id,
   circuit_set_circid_chan_helper(circ, CELL_DIRECTION_IN, id, chan);
 
   if (chan) {
-    tor_assert(bool_eq(or_circ->p_chan_cells.n,
-                       or_circ->next_active_on_p_chan));
-
     chan->timestamp_last_had_circuits = approx_time();
   }
 
@@ -368,8 +472,6 @@ circuit_set_n_circid_chan(circuit_t *circ, circid_t id,
   circuit_set_circid_chan_helper(circ, CELL_DIRECTION_OUT, id, chan);
 
   if (chan) {
-    tor_assert(bool_eq(circ->n_chan_cells.n, circ->next_active_on_n_chan));
-
     chan->timestamp_last_had_circuits = approx_time();
   }
 
@@ -387,8 +489,10 @@ circuit_set_state(circuit_t *circ, uint8_t state)
   tor_assert(circ);
   if (state == circ->state)
     return;
-  if (!circuits_pending_chans)
+  if (PREDICT_UNLIKELY(!circuits_pending_chans))
     circuits_pending_chans = smartlist_new();
+  if (PREDICT_UNLIKELY(!circuits_pending_other_guards))
+    circuits_pending_other_guards = smartlist_new();
   if (circ->state == CIRCUIT_STATE_CHAN_WAIT) {
     /* remove from waiting-circuit list. */
     smartlist_remove(circuits_pending_chans, circ);
@@ -397,7 +501,13 @@ circuit_set_state(circuit_t *circ, uint8_t state)
     /* add to waiting-circuit list. */
     smartlist_add(circuits_pending_chans, circ);
   }
-  if (state == CIRCUIT_STATE_OPEN)
+  if (circ->state == CIRCUIT_STATE_GUARD_WAIT) {
+    smartlist_remove(circuits_pending_other_guards, circ);
+  }
+  if (state == CIRCUIT_STATE_GUARD_WAIT) {
+    smartlist_add(circuits_pending_other_guards, circ);
+  }
+  if (state == CIRCUIT_STATE_GUARD_WAIT || state == CIRCUIT_STATE_OPEN)
     tor_assert(!circ->n_chan_create_cell);
   circ->state = state;
 }
@@ -446,13 +556,45 @@ circuit_count_pending_on_channel(channel_t *chan)
   circuit_get_all_pending_on_channel(sl, chan);
   cnt = smartlist_len(sl);
   smartlist_free(sl);
-  log_debug(LD_CIRC,"or_conn to %s at %s, %d pending circs",
-            chan->nickname ? chan->nickname : "NULL",
+  log_debug(LD_CIRC,"or_conn to %s, %d pending circs",
             channel_get_canonical_remote_descr(chan),
             cnt);
   return cnt;
 }
 
+/** Remove <b>origin_circ</b> from the global list of origin circuits.
+ * Called when we are freeing a circuit.
+ */
+static void
+circuit_remove_from_origin_circuit_list(origin_circuit_t *origin_circ)
+{
+  int origin_idx = origin_circ->global_origin_circuit_list_idx;
+  if (origin_idx < 0)
+    return;
+  origin_circuit_t *c2;
+  tor_assert(origin_idx <= smartlist_len(global_origin_circuit_list));
+  c2 = smartlist_get(global_origin_circuit_list, origin_idx);
+  tor_assert(origin_circ == c2);
+  smartlist_del(global_origin_circuit_list, origin_idx);
+  if (origin_idx < smartlist_len(global_origin_circuit_list)) {
+    origin_circuit_t *replacement =
+      smartlist_get(global_origin_circuit_list, origin_idx);
+    replacement->global_origin_circuit_list_idx = origin_idx;
+  }
+  origin_circ->global_origin_circuit_list_idx = -1;
+}
+
+/** Add <b>origin_circ</b> to the global list of origin circuits. Called
+ * when creating the circuit. */
+static void
+circuit_add_to_origin_circuit_list(origin_circuit_t *origin_circ)
+{
+  tor_assert(origin_circ->global_origin_circuit_list_idx == -1);
+  smartlist_t *lst = circuit_get_global_origin_circuit_list();
+  smartlist_add(lst, origin_circ);
+  origin_circ->global_origin_circuit_list_idx = smartlist_len(lst) - 1;
+}
+
 /** Detach from the global circuit list, and deallocate, all
  * circuits that have been marked for close.
  */
@@ -475,6 +617,11 @@ circuit_close_all_marked(void)
     }
     circ->global_circuitlist_idx = -1;
 
+    /* Remove it from the origin circuit list, if appropriate. */
+    if (CIRCUIT_IS_ORIGIN(circ)) {
+      circuit_remove_from_origin_circuit_list(TO_ORIGIN_CIRCUIT(circ));
+    }
+
     circuit_about_to_free(circ);
     circuit_free(circ);
   } SMARTLIST_FOREACH_END(circ);
@@ -482,7 +629,7 @@ circuit_close_all_marked(void)
   smartlist_clear(circuits_pending_close);
 }
 
-/** Return the head of the global linked list of circuits. */
+/** Return a pointer to the global list of circuits. */
 MOCK_IMPL(smartlist_t *,
 circuit_get_global_list,(void))
 {
@@ -491,6 +638,65 @@ circuit_get_global_list,(void))
   return global_circuitlist;
 }
 
+/** Return a pointer to the global list of origin circuits. */
+smartlist_t *
+circuit_get_global_origin_circuit_list(void)
+{
+  if (NULL == global_origin_circuit_list)
+    global_origin_circuit_list = smartlist_new();
+  return global_origin_circuit_list;
+}
+
+/**
+ * Return true if we have any opened general-purpose 3 hop
+ * origin circuits.
+ *
+ * The result from this function is cached for use by
+ * circuit_any_opened_circuits_cached().
+ */
+int
+circuit_any_opened_circuits(void)
+{
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_origin_circuit_list(),
+          const origin_circuit_t *, next_circ) {
+    if (!TO_CIRCUIT(next_circ)->marked_for_close &&
+        next_circ->has_opened &&
+        TO_CIRCUIT(next_circ)->state == CIRCUIT_STATE_OPEN &&
+        TO_CIRCUIT(next_circ)->purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT &&
+        next_circ->build_state &&
+        next_circ->build_state->desired_path_len == DEFAULT_ROUTE_LEN) {
+      circuit_cache_opened_circuit_state(1);
+      return 1;
+    }
+  } SMARTLIST_FOREACH_END(next_circ);
+
+  circuit_cache_opened_circuit_state(0);
+  return 0;
+}
+
+/**
+ * Cache the "any circuits opened" state, as specified in param
+ * circuits_are_opened. This is a helper function to update
+ * the circuit opened status whenever we happen to look at the
+ * circuit list.
+ */
+void
+circuit_cache_opened_circuit_state(int circuits_are_opened)
+{
+  any_opened_circs_cached_val = circuits_are_opened;
+}
+
+/**
+ * Return true if there were any opened circuits since the last call to
+ * circuit_any_opened_circuits(), or since circuit_expire_building() last
+ * ran (it runs roughly once per second).
+ */
+int
+circuit_any_opened_circuits_cached(void)
+{
+  return any_opened_circs_cached_val;
+}
+
 /** Function to make circ-\>state human-readable */
 const char *
 circuit_state_to_string(int state)
@@ -500,6 +706,8 @@ circuit_state_to_string(int state)
     case CIRCUIT_STATE_BUILDING: return "doing handshakes";
     case CIRCUIT_STATE_ONIONSKIN_PENDING: return "processing the onion";
     case CIRCUIT_STATE_CHAN_WAIT: return "connecting to server";
+    case CIRCUIT_STATE_GUARD_WAIT: return "waiting to see how other "
+      "guards perform";
     case CIRCUIT_STATE_OPEN: return "open";
     default:
       log_warn(LD_BUG, "Unknown circuit state %d", state);
@@ -523,6 +731,10 @@ circuit_purpose_to_controller_string(uint8_t purpose)
 
     case CIRCUIT_PURPOSE_C_GENERAL:
       return "GENERAL";
+
+    case CIRCUIT_PURPOSE_C_HSDIR_GET:
+      return "HS_CLIENT_HSDIR";
+
     case CIRCUIT_PURPOSE_C_INTRODUCING:
     case CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT:
     case CIRCUIT_PURPOSE_C_INTRODUCE_ACKED:
@@ -534,6 +746,9 @@ circuit_purpose_to_controller_string(uint8_t purpose)
     case CIRCUIT_PURPOSE_C_REND_JOINED:
       return "HS_CLIENT_REND";
 
+    case CIRCUIT_PURPOSE_S_HSDIR_POST:
+      return "HS_SERVICE_HSDIR";
+
     case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO:
     case CIRCUIT_PURPOSE_S_INTRO:
       return "HS_SERVICE_INTRO";
@@ -550,6 +765,8 @@ circuit_purpose_to_controller_string(uint8_t purpose)
       return "CONTROLLER";
     case CIRCUIT_PURPOSE_PATH_BIAS_TESTING:
       return "PATH_BIAS_TESTING";
+    case CIRCUIT_PURPOSE_HS_VANGUARDS:
+      return "HS_VANGUARDS";
 
     default:
       tor_snprintf(buf, sizeof(buf), "UNKNOWN_%d", (int)purpose);
@@ -578,6 +795,7 @@ circuit_purpose_to_controller_hs_state_string(uint8_t purpose)
     case CIRCUIT_PURPOSE_TESTING:
     case CIRCUIT_PURPOSE_CONTROLLER:
     case CIRCUIT_PURPOSE_PATH_BIAS_TESTING:
+    case CIRCUIT_PURPOSE_HS_VANGUARDS:
       return NULL;
 
     case CIRCUIT_PURPOSE_INTRO_POINT:
@@ -587,6 +805,7 @@ circuit_purpose_to_controller_hs_state_string(uint8_t purpose)
     case CIRCUIT_PURPOSE_REND_ESTABLISHED:
       return "OR_HS_R_JOINED";
 
+    case CIRCUIT_PURPOSE_C_HSDIR_GET:
     case CIRCUIT_PURPOSE_C_INTRODUCING:
       return "HSCI_CONNECTING";
     case CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT:
@@ -603,6 +822,7 @@ circuit_purpose_to_controller_hs_state_string(uint8_t purpose)
     case CIRCUIT_PURPOSE_C_REND_JOINED:
       return "HSCR_JOINED";
 
+    case CIRCUIT_PURPOSE_S_HSDIR_POST:
     case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO:
       return "HSSI_CONNECTING";
     case CIRCUIT_PURPOSE_S_INTRO:
@@ -628,9 +848,9 @@ circuit_purpose_to_string(uint8_t purpose)
     case CIRCUIT_PURPOSE_INTRO_POINT:
       return "Acting as intro point";
     case CIRCUIT_PURPOSE_REND_POINT_WAITING:
-      return "Acting as rendevous (pending)";
+      return "Acting as rendezvous (pending)";
     case CIRCUIT_PURPOSE_REND_ESTABLISHED:
-      return "Acting as rendevous (established)";
+      return "Acting as rendezvous (established)";
     case CIRCUIT_PURPOSE_C_GENERAL:
       return "General-purpose client";
     case CIRCUIT_PURPOSE_C_INTRODUCING:
@@ -647,6 +867,9 @@ circuit_purpose_to_string(uint8_t purpose)
       return "Hidden service client: Pending rendezvous point (ack received)";
     case CIRCUIT_PURPOSE_C_REND_JOINED:
       return "Hidden service client: Active rendezvous point";
+    case CIRCUIT_PURPOSE_C_HSDIR_GET:
+      return "Hidden service client: Fetching HS descriptor";
+
     case CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT:
       return "Measuring circuit timeout";
 
@@ -658,6 +881,8 @@ circuit_purpose_to_string(uint8_t purpose)
       return "Hidden service: Connecting to rendezvous point";
     case CIRCUIT_PURPOSE_S_REND_JOINED:
       return "Hidden service: Active rendezvous point";
+    case CIRCUIT_PURPOSE_S_HSDIR_POST:
+      return "Hidden service: Uploading HS descriptor";
 
     case CIRCUIT_PURPOSE_TESTING:
       return "Testing circuit";
@@ -668,6 +893,9 @@ circuit_purpose_to_string(uint8_t purpose)
     case CIRCUIT_PURPOSE_PATH_BIAS_TESTING:
       return "Path-bias testing circuit";
 
+    case CIRCUIT_PURPOSE_HS_VANGUARDS:
+      return "Hidden service: Pre-built vanguard circuit";
+
     default:
       tor_snprintf(buf, sizeof(buf), "UNKNOWN_%d", (int)purpose);
       return buf;
@@ -709,6 +937,13 @@ init_circuit_base(circuit_t *circ)
   circ->global_circuitlist_idx = smartlist_len(circuit_get_global_list()) - 1;
 }
 
+/** If we haven't yet decided on a good timeout value for circuit
+ * building, we close idle circuits aggressively so we can get more
+ * data points. These are the default, min, and max consensus values */
+#define DFLT_IDLE_TIMEOUT_WHILE_LEARNING (3*60)
+#define MIN_IDLE_TIMEOUT_WHILE_LEARNING (10)
+#define MAX_IDLE_TIMEOUT_WHILE_LEARNING (1000*60)
+
 /** Allocate space for a new circuit, initializing with <b>p_circ_id</b>
  * and <b>p_conn</b>. Add it to the global circuit list.
  */
@@ -730,8 +965,55 @@ origin_circuit_new(void)
 
   init_circuit_base(TO_CIRCUIT(circ));
 
+  /* Add to origin-list. */
+  circ->global_origin_circuit_list_idx = -1;
+  circuit_add_to_origin_circuit_list(circ);
+
   circuit_build_times_update_last_circ(get_circuit_build_times_mutable());
 
+  if (! circuit_build_times_disabled(get_options()) &&
+      circuit_build_times_needs_circuits(get_circuit_build_times())) {
+    /* Circuits should be shorter lived if we need more of them
+     * for learning a good build timeout */
+    circ->circuit_idle_timeout =
+      networkstatus_get_param(NULL, "cbtlearntimeout",
+                              DFLT_IDLE_TIMEOUT_WHILE_LEARNING,
+                              MIN_IDLE_TIMEOUT_WHILE_LEARNING,
+                              MAX_IDLE_TIMEOUT_WHILE_LEARNING);
+  } else {
+    // This should always be larger than the current port prediction time
+    // remaining, or else we'll end up with the case where a circuit times out
+    // and another one is built, effectively doubling the timeout window.
+    //
+    // We also randomize it by up to 5% more (ie 5% of 0 to 3600 seconds,
+    // depending on how much circuit prediction time is remaining) so that
+    // we don't close a bunch of unused circuits all at the same time.
+    int prediction_time_remaining =
+      predicted_ports_prediction_time_remaining(time(NULL));
+    circ->circuit_idle_timeout = prediction_time_remaining+1+
+        crypto_rand_int(1+prediction_time_remaining/20);
+
+    if (circ->circuit_idle_timeout <= 0) {
+      log_warn(LD_BUG,
+               "Circuit chose a negative idle timeout of %d based on "
+               "%d seconds of predictive building remaining.",
+               circ->circuit_idle_timeout,
+               prediction_time_remaining);
+      circ->circuit_idle_timeout =
+          networkstatus_get_param(NULL, "cbtlearntimeout",
+                  DFLT_IDLE_TIMEOUT_WHILE_LEARNING,
+                  MIN_IDLE_TIMEOUT_WHILE_LEARNING,
+                  MAX_IDLE_TIMEOUT_WHILE_LEARNING);
+    }
+
+    log_info(LD_CIRC,
+              "Circuit %"PRIu32" chose an idle timeout of %d based on "
+              "%d seconds of predictive building remaining.",
+              (circ->global_identifier),
+              circ->circuit_idle_timeout,
+              prediction_time_remaining);
+  }
+
   return circ;
 }
 
@@ -772,21 +1054,42 @@ circuit_clear_testing_cell_stats(circuit_t *circ)
 /** Deallocate space associated with circ.
  */
 STATIC void
-circuit_free(circuit_t *circ)
+circuit_free_(circuit_t *circ)
 {
+  circid_t n_circ_id = 0;
   void *mem;
   size_t memlen;
   int should_free = 1;
   if (!circ)
     return;
 
+  /* We keep a copy of this so we can log its value before it gets unset. */
+  n_circ_id = circ->n_circ_id;
+
   circuit_clear_testing_cell_stats(circ);
 
+  /* Cleanup circuit from anything HS v3 related. We also do this when the
+   * circuit is closed. This is to avoid any code path that free registered
+   * circuits without closing them before. This needs to be done before the
+   * hs identifier is freed. */
+  hs_circ_cleanup(circ);
+
   if (CIRCUIT_IS_ORIGIN(circ)) {
     origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
     mem = ocirc;
     memlen = sizeof(origin_circuit_t);
     tor_assert(circ->magic == ORIGIN_CIRCUIT_MAGIC);
+
+    circuit_remove_from_origin_circuit_list(ocirc);
+
+    if (ocirc->half_streams) {
+      SMARTLIST_FOREACH_BEGIN(ocirc->half_streams, half_edge_t *,
+                              half_conn) {
+        half_edge_free(half_conn);
+      } SMARTLIST_FOREACH_END(half_conn);
+      smartlist_free(ocirc->half_streams);
+    }
+
     if (ocirc->build_state) {
         extend_info_free(ocirc->build_state->chosen_exit);
         circuit_free_cpath_node(ocirc->build_state->pending_final_cpath);
@@ -794,11 +1097,22 @@ circuit_free(circuit_t *circ)
     }
     tor_free(ocirc->build_state);
 
+    /* Cancel before freeing, if we haven't already succeeded or failed. */
+    if (ocirc->guard_state) {
+      entry_guard_cancel(&ocirc->guard_state);
+    }
+    circuit_guard_state_free(ocirc->guard_state);
+
     circuit_clear_cpath(ocirc);
 
     crypto_pk_free(ocirc->intro_key);
     rend_data_free(ocirc->rend_data);
 
+    /* Finally, free the identifier of the circuit and nullify it so multiple
+     * cleanup will work. */
+    hs_ident_circuit_free(ocirc->hs_ident);
+    ocirc->hs_ident = NULL;
+
     tor_free(ocirc->dest_address);
     if (ocirc->socks_username) {
       memwipe(ocirc->socks_username, 0x12, ocirc->socks_username_len);
@@ -820,12 +1134,7 @@ circuit_free(circuit_t *circ)
 
     should_free = (ocirc->workqueue_entry == NULL);
 
-    crypto_cipher_free(ocirc->p_crypto);
-    crypto_digest_free(ocirc->p_digest);
-    crypto_cipher_free(ocirc->n_crypto);
-    crypto_digest_free(ocirc->n_digest);
-
-    circuit_clear_rend_token(ocirc);
+    relay_crypto_clear(&ocirc->crypto);
 
     if (ocirc->rend_splice) {
       or_circuit_t *other = ocirc->rend_splice;
@@ -862,6 +1171,11 @@ circuit_free(circuit_t *circ)
    * "active" checks will be violated. */
   cell_queue_clear(&circ->n_chan_cells);
 
+  log_info(LD_CIRC, "Circuit %u (id: %" PRIu32 ") has been freed.",
+           n_circ_id,
+           CIRCUIT_IS_ORIGIN(circ) ?
+              TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0);
+
   if (should_free) {
     memwipe(mem, 0xAA, memlen); /* poison memory */
     tor_free(mem);
@@ -913,7 +1227,7 @@ circuit_free_all(void)
       while (or_circ->resolving_streams) {
         edge_connection_t *next_conn;
         next_conn = or_circ->resolving_streams->next_stream;
-        connection_free(TO_CONN(or_circ->resolving_streams));
+        connection_free_(TO_CONN(or_circ->resolving_streams));
         or_circ->resolving_streams = next_conn;
       }
     }
@@ -926,12 +1240,18 @@ circuit_free_all(void)
   smartlist_free(lst);
   global_circuitlist = NULL;
 
+  smartlist_free(global_origin_circuit_list);
+  global_origin_circuit_list = NULL;
+
   smartlist_free(circuits_pending_chans);
   circuits_pending_chans = NULL;
 
   smartlist_free(circuits_pending_close);
   circuits_pending_close = NULL;
 
+  smartlist_free(circuits_pending_other_guards);
+  circuits_pending_other_guards = NULL;
+
   {
     chan_circid_circuit_map_t **elt, **next, *c;
     for (elt = HT_START(chan_circid_map, &chan_circid_map);
@@ -954,10 +1274,7 @@ circuit_free_cpath_node(crypt_path_t *victim)
   if (!victim)
     return;
 
-  crypto_cipher_free(victim->f_crypto);
-  crypto_cipher_free(victim->b_crypto);
-  crypto_digest_free(victim->f_digest);
-  crypto_digest_free(victim->b_digest);
+  relay_crypto_clear(&victim->crypto);
   onion_handshake_state_release(&victim->handshake_state);
   crypto_dh_free(victim->rend_dh_handshake_state);
   extend_info_free(victim->extend_info);
@@ -1087,9 +1404,9 @@ circuit_get_by_circid_channel_impl(circid_t circ_id, channel_t *chan,
   if (found && found->circuit) {
     log_debug(LD_CIRC,
               "circuit_get_by_circid_channel_impl() returning circuit %p for"
-              " circ_id %u, channel ID " U64_FORMAT " (%p)",
+              " circ_id %u, channel ID %"PRIu64 " (%p)",
               found->circuit, (unsigned)circ_id,
-              U64_PRINTF_ARG(chan->global_identifier), chan);
+              (chan->global_identifier), chan);
     if (found_entry_out)
       *found_entry_out = 1;
     return found->circuit;
@@ -1097,10 +1414,10 @@ circuit_get_by_circid_channel_impl(circid_t circ_id, channel_t *chan,
 
   log_debug(LD_CIRC,
             "circuit_get_by_circid_channel_impl() found %s for"
-            " circ_id %u, channel ID " U64_FORMAT " (%p)",
+            " circ_id %u, channel ID %"PRIu64 " (%p)",
             found ? "placeholder" : "nothing",
             (unsigned)circ_id,
-            U64_PRINTF_ARG(chan->global_identifier), chan);
+            (chan->global_identifier), chan);
 
   if (found_entry_out)
     *found_entry_out = found ? 1 : 0;
@@ -1263,7 +1580,7 @@ circuit_unlink_all_from_channel(channel_t *chan, int reason)
       smartlist_free(detached_2);
     }
   }
-#endif
+#endif /* defined(DEBUG_CIRCUIT_UNLINK_ALL) */
 
   SMARTLIST_FOREACH_BEGIN(detached, circuit_t *, circ) {
     int mark = 0;
@@ -1312,9 +1629,11 @@ circuit_get_ready_rend_circ_by_rend_data(const rend_data_t *rend_data)
     if (!circ->marked_for_close &&
         circ->purpose == CIRCUIT_PURPOSE_C_REND_READY) {
       origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
-      if (ocirc->rend_data &&
-          !rend_cmp_service_ids(rend_data->onion_address,
-                                ocirc->rend_data->onion_address) &&
+      if (ocirc->rend_data == NULL) {
+        continue;
+      }
+      if (!rend_cmp_service_ids(rend_data_get_address(rend_data),
+                                rend_data_get_address(ocirc->rend_data)) &&
           tor_memeq(ocirc->rend_data->rend_cookie,
                     rend_data->rend_cookie,
                     REND_COOKIE_LEN))
@@ -1325,209 +1644,180 @@ circuit_get_ready_rend_circ_by_rend_data(const rend_data_t *rend_data)
   return NULL;
 }
 
-/** Return the first circuit originating here in global_circuitlist after
- * <b>start</b> whose purpose is <b>purpose</b>, and where
- * <b>digest</b> (if set) matches the rend_pk_digest field. Return NULL if no
- * circuit is found.  If <b>start</b> is NULL, begin at the start of the list.
+/** Return the first introduction circuit originating from the global circuit
+ * list after <b>start</b> or at the start of the list if <b>start</b> is
+ * NULL. Return NULL if no circuit is found.
+ *
+ * If <b>want_client_circ</b> is true, then we are looking for client-side
+ * introduction circuits: A client introduction point circuit has a purpose of
+ * either CIRCUIT_PURPOSE_C_INTRODUCING, CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT
+ * or CIRCUIT_PURPOSE_C_INTRODUCE_ACKED. This does not return a circuit marked
+ * for close, but it returns circuits regardless of their circuit state.
+ *
+ * If <b>want_client_circ</b> is false, then we are looking for service-side
+ * introduction circuits: A service introduction point circuit has a purpose of
+ * either CIRCUIT_PURPOSE_S_ESTABLISH_INTRO or CIRCUIT_PURPOSE_S_INTRO. This
+ * does not return circuits marked for close, or in any state other than open.
  */
 origin_circuit_t *
-circuit_get_next_by_pk_and_purpose(origin_circuit_t *start,
-                                   const char *digest, uint8_t purpose)
+circuit_get_next_intro_circ(const origin_circuit_t *start,
+                            bool want_client_circ)
 {
-  int idx;
+  int idx = 0;
   smartlist_t *lst = circuit_get_global_list();
-  tor_assert(CIRCUIT_PURPOSE_IS_ORIGIN(purpose));
-  if (start == NULL)
-    idx = 0;
-  else
+
+  if (start) {
     idx = TO_CIRCUIT(start)->global_circuitlist_idx + 1;
+  }
 
   for ( ; idx < smartlist_len(lst); ++idx) {
     circuit_t *circ = smartlist_get(lst, idx);
 
-    if (circ->marked_for_close)
-      continue;
-    if (circ->purpose != purpose)
+    /* Ignore a marked for close circuit or if the state is not open. */
+    if (circ->marked_for_close) {
       continue;
-    if (!digest)
-      return TO_ORIGIN_CIRCUIT(circ);
-    else if (TO_ORIGIN_CIRCUIT(circ)->rend_data &&
-             tor_memeq(TO_ORIGIN_CIRCUIT(circ)->rend_data->rend_pk_digest,
-                     digest, DIGEST_LEN))
-      return TO_ORIGIN_CIRCUIT(circ);
+    }
+
+    /* Depending on whether we are looking for client or service circs, skip
+     * circuits with other purposes. */
+    if (want_client_circ) {
+      if (circ->purpose != CIRCUIT_PURPOSE_C_INTRODUCING &&
+          circ->purpose != CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT &&
+          circ->purpose != CIRCUIT_PURPOSE_C_INTRODUCE_ACKED) {
+        continue;
+      }
+    } else { /* we are looking for service-side circs */
+      if (circ->state != CIRCUIT_STATE_OPEN) {
+        continue;
+      }
+      if (circ->purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO &&
+          circ->purpose != CIRCUIT_PURPOSE_S_INTRO) {
+        continue;
+      }
+    }
+
+    /* The purposes we are looking for are only for origin circuits so the
+     * following is valid. */
+    return TO_ORIGIN_CIRCUIT(circ);
   }
+  /* Not found. */
   return NULL;
 }
 
-/** Map from rendezvous cookie to or_circuit_t */
-static digestmap_t *rend_cookie_map = NULL;
-
-/** Map from introduction point digest to or_circuit_t */
-static digestmap_t *intro_digest_map = NULL;
-
-/** Return the OR circuit whose purpose is <b>purpose</b>, and whose
- * rend_token is the REND_TOKEN_LEN-byte <b>token</b>. If <b>is_rend_circ</b>,
- * look for rendezvous point circuits; otherwise look for introduction point
- * circuits. */
-static or_circuit_t *
-circuit_get_by_rend_token_and_purpose(uint8_t purpose, int is_rend_circ,
-                                      const char *token)
+/** Return the first service rendezvous circuit originating from the global
+ * circuit list after <b>start</b> or at the start of the list if <b>start</b>
+ * is NULL. Return NULL if no circuit is found.
+ *
+ * A service rendezvous point circuit has a purpose of either
+ * CIRCUIT_PURPOSE_S_CONNECT_REND or CIRCUIT_PURPOSE_S_REND_JOINED. This does
+ * not return a circuit marked for close and its state must be open. */
+origin_circuit_t *
+circuit_get_next_service_rp_circ(origin_circuit_t *start)
 {
-  or_circuit_t *circ;
-  digestmap_t *map = is_rend_circ ? rend_cookie_map : intro_digest_map;
-
-  if (!map)
-    return NULL;
-
-  circ = digestmap_get(map, token);
-  if (!circ ||
-      circ->base_.purpose != purpose ||
-      circ->base_.marked_for_close)
-    return NULL;
+  int idx = 0;
+  smartlist_t *lst = circuit_get_global_list();
 
-  if (!circ->rendinfo) {
-    char *t = tor_strdup(hex_str(token, REND_TOKEN_LEN));
-    log_warn(LD_BUG, "Wanted a circuit with %s:%d, but lookup returned a "
-             "circuit with no rendinfo set.",
-             safe_str(t), is_rend_circ);
-    tor_free(t);
-    return NULL;
+  if (start) {
+    idx = TO_CIRCUIT(start)->global_circuitlist_idx + 1;
   }
 
-  if (! bool_eq(circ->rendinfo->is_rend_circ, is_rend_circ) ||
-      tor_memneq(circ->rendinfo->rend_token, token, REND_TOKEN_LEN)) {
-    char *t = tor_strdup(hex_str(token, REND_TOKEN_LEN));
-    log_warn(LD_BUG, "Wanted a circuit with %s:%d, but lookup returned %s:%d",
-             safe_str(t), is_rend_circ,
-             safe_str(hex_str(circ->rendinfo->rend_token, REND_TOKEN_LEN)),
-             (int)circ->rendinfo->is_rend_circ);
-    tor_free(t);
-    return NULL;
-  }
+  for ( ; idx < smartlist_len(lst); ++idx) {
+    circuit_t *circ = smartlist_get(lst, idx);
 
-  return circ;
+    /* Ignore a marked for close circuit or purpose not matching a service
+     * intro point or if the state is not open. */
+    if (circ->marked_for_close || circ->state != CIRCUIT_STATE_OPEN ||
+        (circ->purpose != CIRCUIT_PURPOSE_S_CONNECT_REND &&
+         circ->purpose != CIRCUIT_PURPOSE_S_REND_JOINED)) {
+      continue;
+    }
+    /* The purposes we are looking for are only for origin circuits so the
+     * following is valid. */
+    return TO_ORIGIN_CIRCUIT(circ);
+  }
+  /* Not found. */
+  return NULL;
 }
 
-/** Clear the rendezvous cookie or introduction point key digest that's
- * configured on <b>circ</b>, if any, and remove it from any such maps. */
-static void
-circuit_clear_rend_token(or_circuit_t *circ)
+/** Return the first circuit originating here in global_circuitlist after
+ * <b>start</b> whose purpose is <b>purpose</b>, and where <b>digest</b> (if
+ * set) matches the private key digest of the rend data associated with the
+ * circuit. Return NULL if no circuit is found. If <b>start</b> is NULL,
+ * begin at the start of the list.
+ */
+origin_circuit_t *
+circuit_get_next_by_pk_and_purpose(origin_circuit_t *start,
+                                   const uint8_t *digest, uint8_t purpose)
 {
-  or_circuit_t *found_circ;
-  digestmap_t *map;
-
-  if (!circ || !circ->rendinfo)
-    return;
-
-  map = circ->rendinfo->is_rend_circ ? rend_cookie_map : intro_digest_map;
+  int idx;
+  smartlist_t *lst = circuit_get_global_list();
+  tor_assert(CIRCUIT_PURPOSE_IS_ORIGIN(purpose));
+  if (start == NULL)
+    idx = 0;
+  else
+    idx = TO_CIRCUIT(start)->global_circuitlist_idx + 1;
 
-  if (!map) {
-    log_warn(LD_BUG, "Tried to clear rend token on circuit, but found no map");
-    return;
-  }
+  for ( ; idx < smartlist_len(lst); ++idx) {
+    circuit_t *circ = smartlist_get(lst, idx);
+    origin_circuit_t *ocirc;
 
-  found_circ = digestmap_get(map, circ->rendinfo->rend_token);
-  if (found_circ == circ) {
-    /* Great, this is the right one. */
-    digestmap_remove(map, circ->rendinfo->rend_token);
-  } else if (found_circ) {
-    log_warn(LD_BUG, "Tried to clear rend token on circuit, but "
-             "it was already replaced in the map.");
-  } else {
-    log_warn(LD_BUG, "Tried to clear rend token on circuit, but "
-             "it not in the map at all.");
+    if (circ->marked_for_close)
+      continue;
+    if (circ->purpose != purpose)
+      continue;
+    /* At this point we should be able to get a valid origin circuit because
+     * the origin purpose we are looking for matches this circuit. */
+    if (BUG(!CIRCUIT_PURPOSE_IS_ORIGIN(circ->purpose))) {
+      break;
+    }
+    ocirc = TO_ORIGIN_CIRCUIT(circ);
+    if (!digest)
+      return ocirc;
+    if (rend_circuit_pk_digest_eq(ocirc, digest)) {
+      return ocirc;
+    }
   }
-
-  tor_free(circ->rendinfo); /* Sets it to NULL too */
+  return NULL;
 }
 
-/** Set the rendezvous cookie (if is_rend_circ), or the introduction point
- * digest (if ! is_rend_circ) of <b>circ</b> to the REND_TOKEN_LEN-byte value
- * in <b>token</b>, and add it to the appropriate map.  If it previously had a
- * token, clear it.  If another circuit previously had the same
- * cookie/intro-digest, mark that circuit and remove it from the map. */
-static void
-circuit_set_rend_token(or_circuit_t *circ, int is_rend_circ,
-                       const uint8_t *token)
+/** We might cannibalize this circuit: Return true if its last hop can be used
+ *  as a v3 rendezvous point. */
+static int
+circuit_can_be_cannibalized_for_v3_rp(const origin_circuit_t *circ)
 {
-  digestmap_t **map_p, *map;
-  or_circuit_t *found_circ;
-
-  /* Find the right map, creating it as needed */
-  map_p = is_rend_circ ? &rend_cookie_map : &intro_digest_map;
-
-  if (!*map_p)
-    *map_p = digestmap_new();
-
-  map = *map_p;
-
-  /* If this circuit already has a token, we need to remove that. */
-  if (circ->rendinfo)
-    circuit_clear_rend_token(circ);
+  if (!circ->build_state) {
+    return 0;
+  }
 
-  if (token == NULL) {
-    /* We were only trying to remove this token, not set a new one. */
-    return;
+  extend_info_t *chosen_exit = circ->build_state->chosen_exit;
+  if (BUG(!chosen_exit)) {
+    return 0;
   }
 
-  found_circ = digestmap_get(map, (const char *)token);
-  if (found_circ) {
-    tor_assert(found_circ != circ);
-    circuit_clear_rend_token(found_circ);
-    if (! found_circ->base_.marked_for_close) {
-      circuit_mark_for_close(TO_CIRCUIT(found_circ), END_CIRC_REASON_FINISHED);
-      if (is_rend_circ) {
-        log_fn(LOG_PROTOCOL_WARN, LD_REND,
-               "Duplicate rendezvous cookie (%s...) used on two circuits",
-               hex_str((const char*)token, 4)); /* only log first 4 chars */
-      }
+  const node_t *rp_node = node_get_by_id(chosen_exit->identity_digest);
+  if (rp_node) {
+    if (node_supports_v3_rendezvous_point(rp_node)) {
+      return 1;
     }
   }
 
-  /* Now set up the rendinfo */
-  circ->rendinfo = tor_malloc(sizeof(*circ->rendinfo));
-  memcpy(circ->rendinfo->rend_token, token, REND_TOKEN_LEN);
-  circ->rendinfo->is_rend_circ = is_rend_circ ? 1 : 0;
-
-  digestmap_set(map, (const char *)token, circ);
-}
-
-/** Return the circuit waiting for a rendezvous with the provided cookie.
- * Return NULL if no such circuit is found.
- */
-or_circuit_t *
-circuit_get_rendezvous(const uint8_t *cookie)
-{
-  return circuit_get_by_rend_token_and_purpose(
-                                     CIRCUIT_PURPOSE_REND_POINT_WAITING,
-                                     1, (const char*)cookie);
-}
-
-/** Return the circuit waiting for intro cells of the given digest.
- * Return NULL if no such circuit is found.
- */
-or_circuit_t *
-circuit_get_intro_point(const uint8_t *digest)
-{
-  return circuit_get_by_rend_token_and_purpose(
-                                     CIRCUIT_PURPOSE_INTRO_POINT, 0,
-                                     (const char *)digest);
-}
-
-/** Set the rendezvous cookie of <b>circ</b> to <b>cookie</b>.  If another
- * circuit previously had that cookie, mark it. */
-void
-circuit_set_rendezvous_cookie(or_circuit_t *circ, const uint8_t *cookie)
-{
-  circuit_set_rend_token(circ, 1, cookie);
+  return 0;
 }
 
-/** Set the intro point key digest of <b>circ</b> to <b>cookie</b>.  If another
- * circuit previously had that intro point digest, mark it. */
-void
-circuit_set_intro_point_digest(or_circuit_t *circ, const uint8_t *digest)
+/** We are trying to create a circuit of purpose <b>purpose</b> and we are
+ *  looking for cannibalizable circuits. Return the circuit purpose we would be
+ *  willing to cannibalize. */
+static uint8_t
+get_circuit_purpose_needed_to_cannibalize(uint8_t purpose)
 {
-  circuit_set_rend_token(circ, 0, digest);
+  if (circuit_should_use_vanguards(purpose)) {
+    /* If we are using vanguards, then we should only cannibalize vanguard
+     * circuits so that we get the same path construction logic. */
+    return CIRCUIT_PURPOSE_HS_VANGUARDS;
+  } else {
+    /* If no vanguards are used just get a general circuit! */
+    return CIRCUIT_PURPOSE_C_GENERAL;
+  }
 }
 
 /** Return a circuit that is open, is CIRCUIT_PURPOSE_C_GENERAL,
@@ -1535,14 +1825,21 @@ circuit_set_intro_point_digest(or_circuit_t *circ, const uint8_t *digest)
  * flags in <b>flags</b>, and if info is defined, does not already use info
  * as any of its hops; or NULL if no circuit fits this description.
  *
- * The <b>purpose</b> argument (currently ignored) refers to the purpose of
- * the circuit we want to create, not the purpose of the circuit we want to
- * cannibalize.
+ * The <b>purpose</b> argument refers to the purpose of the circuit we want to
+ * create, not the purpose of the circuit we want to cannibalize.
  *
  * If !CIRCLAUNCH_NEED_UPTIME, prefer returning non-uptime circuits.
+ *
+ * To "cannibalize" a circuit means to extend it an extra hop, and use it
+ * for some other purpose than we had originally intended.  We do this when
+ * we want to perform some low-bandwidth task at a specific relay, and we
+ * would like the circuit to complete as soon as possible.  (If we were going
+ * to use a lot of bandwidth, we wouldn't want a circuit with an extra hop.
+ * If we didn't care about circuit completion latency, we would just build
+ * a new circuit.)
  */
 origin_circuit_t *
-circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info,
+circuit_find_to_cannibalize(uint8_t purpose_to_produce, extend_info_t *info,
                             int flags)
 {
   origin_circuit_t *best=NULL;
@@ -1550,29 +1847,53 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info,
   int need_capacity = (flags & CIRCLAUNCH_NEED_CAPACITY) != 0;
   int internal = (flags & CIRCLAUNCH_IS_INTERNAL) != 0;
   const or_options_t *options = get_options();
+  /* We want the circuit we are trying to cannibalize to have this purpose */
+  int purpose_to_search_for;
 
   /* Make sure we're not trying to create a onehop circ by
    * cannibalization. */
   tor_assert(!(flags & CIRCLAUNCH_ONEHOP_TUNNEL));
 
+  purpose_to_search_for = get_circuit_purpose_needed_to_cannibalize(
+                                                  purpose_to_produce);
+
+  tor_assert_nonfatal(purpose_to_search_for == CIRCUIT_PURPOSE_C_GENERAL ||
+                      purpose_to_search_for == CIRCUIT_PURPOSE_HS_VANGUARDS);
+
   log_debug(LD_CIRC,
             "Hunting for a circ to cannibalize: purpose %d, uptime %d, "
             "capacity %d, internal %d",
-            purpose, need_uptime, need_capacity, internal);
+            purpose_to_produce, need_uptime, need_capacity, internal);
 
   SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ_) {
     if (CIRCUIT_IS_ORIGIN(circ_) &&
         circ_->state == CIRCUIT_STATE_OPEN &&
         !circ_->marked_for_close &&
-        circ_->purpose == CIRCUIT_PURPOSE_C_GENERAL &&
+        circ_->purpose == purpose_to_search_for &&
         !circ_->timestamp_dirty) {
       origin_circuit_t *circ = TO_ORIGIN_CIRCUIT(circ_);
+
+      /* Only cannibalize from reasonable length circuits. If we
+       * want C_GENERAL, then only choose 3 hop circs. If we want
+       * HS_VANGUARDS, only choose 4 hop circs.
+       */
+      if (circ->build_state->desired_path_len !=
+          route_len_for_purpose(purpose_to_search_for, NULL)) {
+        goto next;
+      }
+
+      /* Ignore any circuits for which we can't use the Guard. It is possible
+       * that the Guard was removed from the samepled set after the circuit
+       * was created so avoid using it. */
+      if (!entry_guard_could_succeed(circ->guard_state)) {
+        goto next;
+      }
+
       if ((!need_uptime || circ->build_state->need_uptime) &&
           (!need_capacity || circ->build_state->need_capacity) &&
           (internal == circ->build_state->is_internal) &&
           !circ->unusable_for_new_conns &&
           circ->remaining_relay_early_cells &&
-          circ->build_state->desired_path_len == DEFAULT_ROUTE_LEN &&
           !circ->build_state->onehop_tunnel &&
           !circ->isolation_values_set) {
         if (info) {
@@ -1604,6 +1925,14 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info,
             hop = hop->next;
           } while (hop != circ->cpath);
         }
+
+        if ((flags & CIRCLAUNCH_IS_V3_RP) &&
+            !circuit_can_be_cannibalized_for_v3_rp(circ)) {
+          log_debug(LD_GENERAL, "Skipping uncannibalizable circuit for v3 "
+                    "rendezvous point.");
+          goto next;
+        }
+
         if (!best || (best->build_state->need_uptime && !need_uptime))
           best = circ;
       next: ;
@@ -1614,6 +1943,37 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info,
   return best;
 }
 
+/**
+ * Check whether any of the origin circuits that are waiting to see if
+ * their guard is good enough to use can be upgraded to "ready". If so,
+ * return a new smartlist containing them. Otherwise return NULL.
+ */
+smartlist_t *
+circuit_find_circuits_to_upgrade_from_guard_wait(void)
+{
+  /* Only if some circuit is actually waiting on an upgrade should we
+   * run the algorithm. */
+  if (! circuits_pending_other_guards ||
+      smartlist_len(circuits_pending_other_guards)==0)
+    return NULL;
+  /* Only if we have some origin circuits should we run the algorithm. */
+  if (!global_origin_circuit_list)
+    return NULL;
+
+  /* Okay; we can pass our circuit list to entrynodes.c.*/
+  smartlist_t *result = smartlist_new();
+  int circuits_upgraded  = entry_guards_upgrade_waiting_circuits(
+                                                 get_guard_selection_info(),
+                                                 global_origin_circuit_list,
+                                                 result);
+  if (circuits_upgraded && smartlist_len(result)) {
+    return result;
+  } else {
+    smartlist_free(result);
+    return NULL;
+  }
+}
+
 /** Return the number of hops in circuit's path. If circ has no entries,
  * or is NULL, returns 0. */
 int
@@ -1630,6 +1990,25 @@ circuit_get_cpath_len(origin_circuit_t *circ)
   return n;
 }
 
+/** Return the number of opened hops in circuit's path.
+ * If circ has no entries, or is NULL, returns 0. */
+int
+circuit_get_cpath_opened_len(const origin_circuit_t *circ)
+{
+  int n = 0;
+  if (circ && circ->cpath) {
+    crypt_path_t *cpath, *cpath_next = NULL;
+    for (cpath = circ->cpath;
+         cpath->state == CPATH_STATE_OPEN
+           && cpath_next != circ->cpath;
+         cpath = cpath_next) {
+      cpath_next = cpath->next;
+      ++n;
+    }
+  }
+  return n;
+}
+
 /** Return the <b>hopnum</b>th hop in <b>circ</b>->cpath, or NULL if there
  * aren't that many hops in the list. <b>hopnum</b> starts at 1.
  * Returns NULL if <b>hopnum</b> is 0 or negative. */
@@ -1741,8 +2120,7 @@ circuit_synchronize_written_or_bandwidth(const circuit_t *c,
  *   - If state is onionskin_pending, remove circ from the onion_pending
  *     list.
  *   - If circ isn't open yet: call circuit_build_failed() if we're
- *     the origin, and in either case call circuit_rep_hist_note_result()
- *     to note stats.
+ *     the origin.
  *   - If purpose is C_INTRODUCE_ACK_WAIT, report the intro point
  *     failure we just had to the hidden service client module.
  *   - If purpose is C_INTRODUCING and <b>reason</b> isn't TIMEOUT,
@@ -1816,10 +2194,21 @@ circuit_mark_for_close_, (circuit_t *circ, int reason, int line,
     }
   }
 
+  /* Notify the HS subsystem that this circuit is closing. */
+  hs_circ_cleanup(circ);
+
   if (circuits_pending_close == NULL)
     circuits_pending_close = smartlist_new();
 
   smartlist_add(circuits_pending_close, circ);
+  mainloop_schedule_postloop_cleanup();
+
+  log_info(LD_GENERAL, "Circuit %u (id: %" PRIu32 ") marked for close at "
+                       "%s:%d (orig reason: %d, new reason: %d)",
+           circ->n_circ_id,
+           CIRCUIT_IS_ORIGIN(circ) ?
+              TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0,
+           file, line, orig_reason, reason);
 }
 
 /** Called immediately before freeing a marked circuit <b>circ</b> from
@@ -1866,20 +2255,25 @@ circuit_about_to_free(circuit_t *circ)
    * module then.  If it isn't OPEN, we send it there now to remember which
    * links worked and which didn't.
    */
-  if (circ->state != CIRCUIT_STATE_OPEN) {
+  if (circ->state != CIRCUIT_STATE_OPEN &&
+      circ->state != CIRCUIT_STATE_GUARD_WAIT) {
     if (CIRCUIT_IS_ORIGIN(circ)) {
       origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
       circuit_build_failed(ocirc); /* take actions if necessary */
-      circuit_rep_hist_note_result(ocirc);
     }
   }
   if (circ->state == CIRCUIT_STATE_CHAN_WAIT) {
     if (circuits_pending_chans)
       smartlist_remove(circuits_pending_chans, circ);
   }
+  if (circuits_pending_other_guards) {
+    smartlist_remove(circuits_pending_other_guards, circ);
+  }
   if (CIRCUIT_IS_ORIGIN(circ)) {
     control_event_circuit_status(TO_ORIGIN_CIRCUIT(circ),
-     (circ->state == CIRCUIT_STATE_OPEN)?CIRC_EVENT_CLOSED:CIRC_EVENT_FAILED,
+     (circ->state == CIRCUIT_STATE_OPEN ||
+      circ->state == CIRCUIT_STATE_GUARD_WAIT) ?
+                                 CIRC_EVENT_CLOSED:CIRC_EVENT_FAILED,
      orig_reason);
   }
 
@@ -1888,11 +2282,11 @@ circuit_about_to_free(circuit_t *circ)
     int timed_out = (reason == END_CIRC_REASON_TIMEOUT);
     tor_assert(circ->state == CIRCUIT_STATE_OPEN);
     tor_assert(ocirc->build_state->chosen_exit);
-    tor_assert(ocirc->rend_data);
-    if (orig_reason != END_CIRC_REASON_IP_NOW_REDUNDANT) {
+    if (orig_reason != END_CIRC_REASON_IP_NOW_REDUNDANT &&
+        ocirc->rend_data) {
       /* treat this like getting a nack from it */
       log_info(LD_REND, "Failed intro circ %s to %s (awaiting ack). %s",
-          safe_str_client(ocirc->rend_data->onion_address),
+          safe_str_client(rend_data_get_address(ocirc->rend_data)),
           safe_str_client(build_state_get_exit_nickname(ocirc->build_state)),
           timed_out ? "Recording timeout." : "Removing from descriptor.");
       rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit,
@@ -1905,11 +2299,12 @@ circuit_about_to_free(circuit_t *circ)
              reason != END_CIRC_REASON_TIMEOUT) {
     origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
     if (ocirc->build_state->chosen_exit && ocirc->rend_data) {
-      if (orig_reason != END_CIRC_REASON_IP_NOW_REDUNDANT) {
+      if (orig_reason != END_CIRC_REASON_IP_NOW_REDUNDANT &&
+          ocirc->rend_data) {
         log_info(LD_REND, "Failed intro circ %s to %s "
             "(building circuit to intro point). "
             "Marking intro point as possibly unreachable.",
-            safe_str_client(ocirc->rend_data->onion_address),
+            safe_str_client(rend_data_get_address(ocirc->rend_data)),
             safe_str_client(build_state_get_exit_nickname(
                                               ocirc->build_state)));
         rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit,
@@ -2004,10 +2399,10 @@ single_conn_free_bytes(connection_t *conn)
   }
   if (conn->type == CONN_TYPE_DIR) {
     dir_connection_t *dir_conn = TO_DIR_CONN(conn);
-    if (dir_conn->zlib_state) {
-      result += tor_zlib_state_size(dir_conn->zlib_state);
-      tor_zlib_free(dir_conn->zlib_state);
-      dir_conn->zlib_state = NULL;
+    if (dir_conn->compress_state) {
+      result += tor_compress_state_size(dir_conn->compress_state);
+      tor_compress_free(dir_conn->compress_state);
+      dir_conn->compress_state = NULL;
     }
   }
   return result;
@@ -2053,13 +2448,27 @@ n_cells_in_circ_queues(const circuit_t *c)
   return n;
 }
 
+/** Return the number of bytes allocated for <b>c</b>'s half-open streams. */
+static size_t
+circuit_alloc_in_half_streams(const circuit_t *c)
+{
+  if (! CIRCUIT_IS_ORIGIN(c)) {
+    return 0;
+  }
+  const origin_circuit_t *ocirc = CONST_TO_ORIGIN_CIRCUIT(c);
+  if (ocirc->half_streams)
+    return smartlist_len(ocirc->half_streams) * sizeof(half_edge_t);
+  else
+    return 0;
+}
+
 /**
- * Return the age of the oldest cell queued on <b>c</b>, in milliseconds.
+ * Return the age of the oldest cell queued on <b>c</b>, in timestamp units.
  * Return 0 if there are no cells queued on c.  Requires that <b>now</b> be
- * the current time in milliseconds since the epoch, truncated.
+ * the current coarse timestamp.
  *
  * This function will return incorrect results if the oldest cell queued on
- * the circuit is older than 2**32 msec (about 49 days) old.
+ * the circuit is older than about 2**32 msec (about 49 days) old.
  */
 STATIC uint32_t
 circuit_max_queued_cell_age(const circuit_t *c, uint32_t now)
@@ -2068,12 +2477,12 @@ circuit_max_queued_cell_age(const circuit_t *c, uint32_t now)
   packed_cell_t *cell;
 
   if (NULL != (cell = TOR_SIMPLEQ_FIRST(&c->n_chan_cells.head)))
-    age = now - cell->inserted_time;
+    age = now - cell->inserted_timestamp;
 
   if (! CIRCUIT_IS_ORIGIN(c)) {
     const or_circuit_t *orcirc = CONST_TO_OR_CIRCUIT(c);
     if (NULL != (cell = TOR_SIMPLEQ_FIRST(&orcirc->p_chan_cells.head))) {
-      uint32_t age2 = now - cell->inserted_time;
+      uint32_t age2 = now - cell->inserted_timestamp;
       if (age2 > age)
         return age2;
     }
@@ -2081,31 +2490,30 @@ circuit_max_queued_cell_age(const circuit_t *c, uint32_t now)
   return age;
 }
 
-/** Return the age in milliseconds of the oldest buffer chunk on <b>conn</b>,
- * where age is taken in milliseconds before the time <b>now</b> (in truncated
- * absolute monotonic msec).  If the connection has no data, treat
- * it as having age zero.
+/** Return the age of the oldest buffer chunk on <b>conn</b>, where age is
+ * taken in timestamp units before the time <b>now</b>.  If the connection has
+ * no data, treat it as having age zero.
  **/
 static uint32_t
-conn_get_buffer_age(const connection_t *conn, uint32_t now)
+conn_get_buffer_age(const connection_t *conn, uint32_t now_ts)
 {
   uint32_t age = 0, age2;
   if (conn->outbuf) {
-    age2 = buf_get_oldest_chunk_timestamp(conn->outbuf, now);
+    age2 = buf_get_oldest_chunk_timestamp(conn->outbuf, now_ts);
     if (age2 > age)
       age = age2;
   }
   if (conn->inbuf) {
-    age2 = buf_get_oldest_chunk_timestamp(conn->inbuf, now);
+    age2 = buf_get_oldest_chunk_timestamp(conn->inbuf, now_ts);
     if (age2 > age)
       age = age2;
   }
   return age;
 }
 
-/** Return the age in milliseconds of the oldest buffer chunk on any stream in
- * the linked list <b>stream</b>, where age is taken in milliseconds before
- * the time <b>now</b> (in truncated milliseconds since the epoch). */
+/** Return the age in timestamp units of the oldest buffer chunk on any stream
+ * in the linked list <b>stream</b>, where age is taken in timestamp units
+ * before the timestamp <b>now</b>. */
 static uint32_t
 circuit_get_streams_max_data_age(const edge_connection_t *stream, uint32_t now)
 {
@@ -2124,9 +2532,9 @@ circuit_get_streams_max_data_age(const edge_connection_t *stream, uint32_t now)
   return age;
 }
 
-/** Return the age in milliseconds of the oldest buffer chunk on any stream
- * attached to the circuit <b>c</b>, where age is taken in milliseconds before
- * the time <b>now</b> (in truncated milliseconds since the epoch). */
+/** Return the age in timestamp units of the oldest buffer chunk on any stream
+ * attached to the circuit <b>c</b>, where age is taken before the timestamp
+ * <b>now</b>. */
 STATIC uint32_t
 circuit_max_queued_data_age(const circuit_t *c, uint32_t now)
 {
@@ -2140,8 +2548,8 @@ circuit_max_queued_data_age(const circuit_t *c, uint32_t now)
 }
 
 /** Return the age of the oldest cell or stream buffer chunk on the circuit
- * <b>c</b>, where age is taken in milliseconds before the time <b>now</b> (in
- * truncated milliseconds since the epoch). */
+ * <b>c</b>, where age is taken in timestamp units before the timestamp
+ * <b>now</b> */
 STATIC uint32_t
 circuit_max_queued_item_age(const circuit_t *c, uint32_t now)
 {
@@ -2171,7 +2579,7 @@ circuits_compare_by_oldest_queued_item_(const void **a_, const void **b_)
     return -1;
 }
 
-static uint32_t now_ms_for_buf_cmp;
+static uint32_t now_ts_for_buf_cmp;
 
 /** Helper to sort a list of circuit_t by age of oldest item, in descending
  * order. */
@@ -2180,8 +2588,8 @@ conns_compare_by_buffer_age_(const void **a_, const void **b_)
 {
   const connection_t *a = *a_;
   const connection_t *b = *b_;
-  time_t age_a = conn_get_buffer_age(a, now_ms_for_buf_cmp);
-  time_t age_b = conn_get_buffer_age(b, now_ms_for_buf_cmp);
+  time_t age_a = conn_get_buffer_age(a, now_ts_for_buf_cmp);
+  time_t age_b = conn_get_buffer_age(b, now_ts_for_buf_cmp);
 
   if (age_a < age_b)
     return 1;
@@ -2206,10 +2614,22 @@ circuits_handle_oom(size_t current_allocation)
   size_t mem_recovered=0;
   int n_circuits_killed=0;
   int n_dirconns_killed=0;
-  uint32_t now_ms;
-  log_notice(LD_GENERAL, "We're low on memory.  Killing circuits with "
-             "over-long queues. (This behavior is controlled by "
-             "MaxMemInQueues.)");
+  uint32_t now_ts;
+  log_notice(LD_GENERAL, "We're low on memory (cell queues total alloc:"
+             " %"TOR_PRIuSZ" buffer total alloc: %" TOR_PRIuSZ ","
+             " tor compress total alloc: %" TOR_PRIuSZ
+             " (zlib: %" TOR_PRIuSZ ", zstd: %" TOR_PRIuSZ ","
+             " lzma: %" TOR_PRIuSZ "),"
+             " rendezvous cache total alloc: %" TOR_PRIuSZ "). Killing"
+             " circuits withover-long queues. (This behavior is controlled by"
+             " MaxMemInQueues.)",
+             cell_queues_get_total_allocation(),
+             buf_get_total_allocation(),
+             tor_compress_get_total_allocation(),
+             tor_zlib_get_total_allocation(),
+             tor_zstd_get_total_allocation(),
+             tor_lzma_get_total_allocation(),
+             rend_cache_get_total_allocation());
 
   {
     size_t mem_target = (size_t)(get_options()->MaxMemInQueues *
@@ -2219,11 +2639,11 @@ circuits_handle_oom(size_t current_allocation)
     mem_to_recover = current_allocation - mem_target;
   }
 
-  now_ms = (uint32_t)monotime_coarse_absolute_msec();
+  now_ts = monotime_coarse_get_stamp();
 
   circlist = circuit_get_global_list();
   SMARTLIST_FOREACH_BEGIN(circlist, circuit_t *, circ) {
-    circ->age_tmp = circuit_max_queued_item_age(circ, now_ms);
+    circ->age_tmp = circuit_max_queued_item_age(circ, now_ts);
   } SMARTLIST_FOREACH_END(circ);
 
   /* This is O(n log n); there are faster algorithms we could use instead.
@@ -2236,9 +2656,9 @@ circuits_handle_oom(size_t current_allocation)
   } SMARTLIST_FOREACH_END(circ);
 
   /* Now sort the connection array ... */
-  now_ms_for_buf_cmp = now_ms;
+  now_ts_for_buf_cmp = now_ts;
   smartlist_sort(connection_array, conns_compare_by_buffer_age_);
-  now_ms_for_buf_cmp = 0;
+  now_ts_for_buf_cmp = 0;
 
   /* Fix up the connection array to its new order. */
   SMARTLIST_FOREACH_BEGIN(connection_array, connection_t *, conn) {
@@ -2257,7 +2677,7 @@ circuits_handle_oom(size_t current_allocation)
      * data older than this circuit. */
     while (conn_idx < smartlist_len(connection_array)) {
       connection_t *conn = smartlist_get(connection_array, conn_idx);
-      uint32_t conn_age = conn_get_buffer_age(conn, now_ms);
+      uint32_t conn_age = conn_get_buffer_age(conn, now_ts);
       if (conn_age < circ->age_tmp) {
         break;
       }
@@ -2276,6 +2696,7 @@ circuits_handle_oom(size_t current_allocation)
 
     /* Now, kill the circuit. */
     n = n_cells_in_circ_queues(circ);
+    const size_t half_stream_alloc = circuit_alloc_in_half_streams(circ);
     if (! circ->marked_for_close) {
       circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT);
     }
@@ -2285,6 +2706,7 @@ circuits_handle_oom(size_t current_allocation)
     ++n_circuits_killed;
 
     mem_recovered += n * packed_cell_mem_cost();
+    mem_recovered += half_stream_alloc;
     mem_recovered += freed;
 
     if (mem_recovered >= mem_to_recover)
@@ -2293,10 +2715,10 @@ circuits_handle_oom(size_t current_allocation)
 
  done_recovering_mem:
 
-  log_notice(LD_GENERAL, "Removed "U64_FORMAT" bytes by killing %d circuits; "
+  log_notice(LD_GENERAL, "Removed %"TOR_PRIuSZ" bytes by killing %d circuits; "
              "%d circuits remain alive. Also killed %d non-linked directory "
              "connections.",
-             U64_PRINTF_ARG(mem_recovered),
+             mem_recovered,
              n_circuits_killed,
              smartlist_len(circlist) - n_circuits_killed,
              n_dirconns_killed);
@@ -2315,8 +2737,7 @@ assert_cpath_layer_ok(const crypt_path_t *cp)
   switch (cp->state)
     {
     case CPATH_STATE_OPEN:
-      tor_assert(cp->f_crypto);
-      tor_assert(cp->b_crypto);
+      relay_crypto_assert_ok(&cp->crypto);
       /* fall through */
     case CPATH_STATE_CLOSED:
       /*XXXX Assert that there's no handshake_state either. */
@@ -2359,8 +2780,8 @@ assert_cpath_ok(const crypt_path_t *cp)
 /** Verify that circuit <b>c</b> has all of its invariants
  * correct. Trigger an assert if anything is invalid.
  */
-void
-assert_circuit_ok(const circuit_t *c)
+MOCK_IMPL(void,
+assert_circuit_ok,(const circuit_t *c))
 {
   edge_connection_t *conn;
   const or_circuit_t *or_circ = NULL;
@@ -2402,13 +2823,11 @@ assert_circuit_ok(const circuit_t *c)
 
   tor_assert(c->deliver_window >= 0);
   tor_assert(c->package_window >= 0);
-  if (c->state == CIRCUIT_STATE_OPEN) {
+  if (c->state == CIRCUIT_STATE_OPEN ||
+      c->state == CIRCUIT_STATE_GUARD_WAIT) {
     tor_assert(!c->n_chan_create_cell);
     if (or_circ) {
-      tor_assert(or_circ->n_crypto);
-      tor_assert(or_circ->p_crypto);
-      tor_assert(or_circ->n_digest);
-      tor_assert(or_circ->p_digest);
+      relay_crypto_assert_ok(&or_circ->crypto);
     }
   }
   if (c->state == CIRCUIT_STATE_CHAN_WAIT && !c->marked_for_close) {
@@ -2432,4 +2851,3 @@ assert_circuit_ok(const circuit_t *c)
     tor_assert(!or_circ || !or_circ->rend_splice);
   }
 }
-
diff --git a/src/core/or/circuitlist.h b/src/core/or/circuitlist.h
new file mode 100644
index 0000000000..b87c6a3667
--- /dev/null
+++ b/src/core/or/circuitlist.h
@@ -0,0 +1,250 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitlist.h
+ * \brief Header file for circuitlist.c.
+ **/
+
+#ifndef TOR_CIRCUITLIST_H
+#define TOR_CIRCUITLIST_H
+
+#include "lib/testsupport/testsupport.h"
+#include "feature/hs/hs_ident.h"
+
+/** Circuit state: I'm the origin, still haven't done all my handshakes. */
+#define CIRCUIT_STATE_BUILDING 0
+/** Circuit state: Waiting to process the onionskin. */
+#define CIRCUIT_STATE_ONIONSKIN_PENDING 1
+/** Circuit state: I'd like to deliver a create, but my n_chan is still
+ * connecting. */
+#define CIRCUIT_STATE_CHAN_WAIT 2
+/** Circuit state: the circuit is open but we don't want to actually use it
+ * until we find out if a better guard will be available.
+ */
+#define CIRCUIT_STATE_GUARD_WAIT 3
+/** Circuit state: onionskin(s) processed, ready to send/receive cells. */
+#define CIRCUIT_STATE_OPEN 4
+
+#define CIRCUIT_PURPOSE_MIN_ 1
+
+/* these circuits were initiated elsewhere */
+#define CIRCUIT_PURPOSE_OR_MIN_ 1
+/** OR-side circuit purpose: normal circuit, at OR. */
+#define CIRCUIT_PURPOSE_OR 1
+/** OR-side circuit purpose: At OR, from the service, waiting for intro from
+ * clients. */
+#define CIRCUIT_PURPOSE_INTRO_POINT 2
+/** OR-side circuit purpose: At OR, from the client, waiting for the service.
+ */
+#define CIRCUIT_PURPOSE_REND_POINT_WAITING 3
+/** OR-side circuit purpose: At OR, both circuits have this purpose. */
+#define CIRCUIT_PURPOSE_REND_ESTABLISHED 4
+#define CIRCUIT_PURPOSE_OR_MAX_ 4
+
+/* these circuits originate at this node */
+
+/* here's how circ client-side purposes work:
+ *   normal circuits are C_GENERAL.
+ *   circuits that are c_introducing are either on their way to
+ *     becoming open, or they are open and waiting for a
+ *     suitable rendcirc before they send the intro.
+ *   circuits that are c_introduce_ack_wait have sent the intro,
+ *     but haven't gotten a response yet.
+ *   circuits that are c_establish_rend are either on their way
+ *     to becoming open, or they are open and have sent the
+ *     establish_rendezvous cell but haven't received an ack.
+ *   circuits that are c_rend_ready are open and have received a
+ *     rend ack, but haven't heard from the service yet. if they have a
+ *     buildstate->pending_final_cpath then they're expecting a
+ *     cell from the service, else they're not.
+ *   circuits that are c_rend_ready_intro_acked are open, and
+ *     some intro circ has sent its intro and received an ack.
+ *   circuits that are c_rend_joined are open, have heard from
+ *     the service, and are talking to it.
+ */
+/** Client-side circuit purpose: Normal circuit, with cpath. */
+#define CIRCUIT_PURPOSE_C_GENERAL 5
+#define CIRCUIT_PURPOSE_C_HS_MIN_ 6
+/** Client-side circuit purpose: at the client, connecting to intro point. */
+#define CIRCUIT_PURPOSE_C_INTRODUCING 6
+/** Client-side circuit purpose: at the client, sent INTRODUCE1 to intro point,
+ * waiting for ACK/NAK. */
+#define CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT 7
+/** Client-side circuit purpose: at the client, introduced and acked, closing.
+ */
+#define CIRCUIT_PURPOSE_C_INTRODUCE_ACKED 8
+/** Client-side circuit purpose: at the client, waiting for ack. */
+#define CIRCUIT_PURPOSE_C_ESTABLISH_REND 9
+/** Client-side circuit purpose: at the client, waiting for the service. */
+#define CIRCUIT_PURPOSE_C_REND_READY 10
+/** Client-side circuit purpose: at the client, waiting for the service,
+ * INTRODUCE has been acknowledged. */
+#define CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED 11
+/** Client-side circuit purpose: at the client, rendezvous established. */
+#define CIRCUIT_PURPOSE_C_REND_JOINED 12
+/** This circuit is used for getting hsdirs */
+#define CIRCUIT_PURPOSE_C_HSDIR_GET 13
+#define CIRCUIT_PURPOSE_C_HS_MAX_ 13
+/** This circuit is used for build time measurement only */
+#define CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT 14
+#define CIRCUIT_PURPOSE_C_MAX_ 14
+
+#define CIRCUIT_PURPOSE_S_HS_MIN_ 15
+/** Hidden-service-side circuit purpose: at the service, waiting for
+ * introductions. */
+#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 15
+/** Hidden-service-side circuit purpose: at the service, successfully
+ * established intro. */
+#define CIRCUIT_PURPOSE_S_INTRO 16
+/** Hidden-service-side circuit purpose: at the service, connecting to rend
+ * point. */
+#define CIRCUIT_PURPOSE_S_CONNECT_REND 17
+/** Hidden-service-side circuit purpose: at the service, rendezvous
+ * established. */
+#define CIRCUIT_PURPOSE_S_REND_JOINED 18
+/** This circuit is used for uploading hsdirs */
+#define CIRCUIT_PURPOSE_S_HSDIR_POST 19
+#define CIRCUIT_PURPOSE_S_HS_MAX_ 19
+
+/** A testing circuit; not meant to be used for actual traffic. */
+#define CIRCUIT_PURPOSE_TESTING 20
+/** A controller made this circuit and Tor should not use it. */
+#define CIRCUIT_PURPOSE_CONTROLLER 21
+/** This circuit is used for path bias probing only */
+#define CIRCUIT_PURPOSE_PATH_BIAS_TESTING 22
+
+/** This circuit is used for vanguards/restricted paths.
+ *
+ *  This type of circuit is *only* created preemptively and never
+ *  on-demand. When an HS operation needs to take place (e.g. connect to an
+ *  intro point), these circuits are then cannibalized and repurposed to the
+ *  actual needed HS purpose. */
+#define CIRCUIT_PURPOSE_HS_VANGUARDS 23
+
+#define CIRCUIT_PURPOSE_MAX_ 23
+/** A catch-all for unrecognized purposes. Currently we don't expect
+ * to make or see any circuits with this purpose. */
+#define CIRCUIT_PURPOSE_UNKNOWN 255
+
+/** True iff the circuit purpose <b>p</b> is for a circuit that
+ * originated at this node. */
+#define CIRCUIT_PURPOSE_IS_ORIGIN(p) ((p)>CIRCUIT_PURPOSE_OR_MAX_)
+/** True iff the circuit purpose <b>p</b> is for a circuit that originated
+ * here to serve as a client.  (Hidden services don't count here.) */
+#define CIRCUIT_PURPOSE_IS_CLIENT(p)  \
+  ((p)> CIRCUIT_PURPOSE_OR_MAX_ &&    \
+   (p)<=CIRCUIT_PURPOSE_C_MAX_)
+/** True iff the circuit_t <b>c</b> is actually an origin_circuit_t. */
+#define CIRCUIT_IS_ORIGIN(c) (CIRCUIT_PURPOSE_IS_ORIGIN((c)->purpose))
+/** True iff the circuit purpose <b>p</b> is for an established rendezvous
+ * circuit. */
+#define CIRCUIT_PURPOSE_IS_ESTABLISHED_REND(p) \
+  ((p) == CIRCUIT_PURPOSE_C_REND_JOINED ||     \
+   (p) == CIRCUIT_PURPOSE_S_REND_JOINED)
+/** True iff the circuit_t c is actually an or_circuit_t */
+#define CIRCUIT_IS_ORCIRC(c) (((circuit_t *)(c))->magic == OR_CIRCUIT_MAGIC)
+
+/** True iff this circuit purpose should count towards the global
+ * pending rate limit (set by MaxClientCircuitsPending). We count all
+ * general purpose circuits, as well as the first step of client onion
+ * service connections (HSDir gets). */
+#define CIRCUIT_PURPOSE_COUNTS_TOWARDS_MAXPENDING(p) \
+    ((p) == CIRCUIT_PURPOSE_C_GENERAL || \
+     (p) == CIRCUIT_PURPOSE_C_HSDIR_GET)
+
+/** Convert a circuit_t* to a pointer to the enclosing or_circuit_t.  Assert
+ * if the cast is impossible. */
+or_circuit_t *TO_OR_CIRCUIT(circuit_t *);
+const or_circuit_t *CONST_TO_OR_CIRCUIT(const circuit_t *);
+/** Convert a circuit_t* to a pointer to the enclosing origin_circuit_t.
+ * Assert if the cast is impossible. */
+origin_circuit_t *TO_ORIGIN_CIRCUIT(circuit_t *);
+const origin_circuit_t *CONST_TO_ORIGIN_CIRCUIT(const circuit_t *);
+
+MOCK_DECL(smartlist_t *, circuit_get_global_list, (void));
+smartlist_t *circuit_get_global_origin_circuit_list(void);
+int circuit_any_opened_circuits(void);
+int circuit_any_opened_circuits_cached(void);
+void circuit_cache_opened_circuit_state(int circuits_are_opened);
+
+const char *circuit_state_to_string(int state);
+const char *circuit_purpose_to_controller_string(uint8_t purpose);
+const char *circuit_purpose_to_controller_hs_state_string(uint8_t purpose);
+const char *circuit_purpose_to_string(uint8_t purpose);
+void circuit_dump_by_conn(connection_t *conn, int severity);
+void circuit_set_p_circid_chan(or_circuit_t *circ, circid_t id,
+                               channel_t *chan);
+void circuit_set_n_circid_chan(circuit_t *circ, circid_t id,
+                               channel_t *chan);
+void channel_mark_circid_unusable(channel_t *chan, circid_t id);
+void channel_mark_circid_usable(channel_t *chan, circid_t id);
+time_t circuit_id_when_marked_unusable_on_channel(circid_t circ_id,
+                                                  channel_t *chan);
+void circuit_set_state(circuit_t *circ, uint8_t state);
+void circuit_close_all_marked(void);
+int32_t circuit_initial_package_window(void);
+origin_circuit_t *origin_circuit_new(void);
+or_circuit_t *or_circuit_new(circid_t p_circ_id, channel_t *p_chan);
+circuit_t *circuit_get_by_circid_channel(circid_t circ_id,
+                                         channel_t *chan);
+circuit_t *
+circuit_get_by_circid_channel_even_if_marked(circid_t circ_id,
+                                             channel_t *chan);
+int circuit_id_in_use_on_channel(circid_t circ_id, channel_t *chan);
+circuit_t *circuit_get_by_edge_conn(edge_connection_t *conn);
+void circuit_unlink_all_from_channel(channel_t *chan, int reason);
+origin_circuit_t *circuit_get_by_global_id(uint32_t id);
+origin_circuit_t *circuit_get_ready_rend_circ_by_rend_data(
+  const rend_data_t *rend_data);
+origin_circuit_t *circuit_get_next_by_pk_and_purpose(origin_circuit_t *start,
+                                     const uint8_t *digest, uint8_t purpose);
+origin_circuit_t *circuit_get_next_intro_circ(const origin_circuit_t *start,
+                                              bool want_client_circ);
+origin_circuit_t *circuit_get_next_service_rp_circ(origin_circuit_t *start);
+origin_circuit_t *circuit_get_next_service_hsdir_circ(origin_circuit_t *start);
+origin_circuit_t *circuit_find_to_cannibalize(uint8_t purpose,
+                                              extend_info_t *info, int flags);
+void circuit_mark_all_unused_circs(void);
+void circuit_mark_all_dirty_circs_as_unusable(void);
+void circuit_synchronize_written_or_bandwidth(const circuit_t *c,
+                                              circuit_channel_direction_t dir);
+MOCK_DECL(void, circuit_mark_for_close_, (circuit_t *circ, int reason,
+                                          int line, const char *file));
+int circuit_get_cpath_len(origin_circuit_t *circ);
+int circuit_get_cpath_opened_len(const origin_circuit_t *);
+void circuit_clear_cpath(origin_circuit_t *circ);
+crypt_path_t *circuit_get_cpath_hop(origin_circuit_t *circ, int hopnum);
+void circuit_get_all_pending_on_channel(smartlist_t *out,
+                                        channel_t *chan);
+int circuit_count_pending_on_channel(channel_t *chan);
+
+#define circuit_mark_for_close(c, reason)                               \
+  circuit_mark_for_close_((c), (reason), __LINE__, SHORT_FILE__)
+
+void assert_cpath_layer_ok(const crypt_path_t *cp);
+MOCK_DECL(void, assert_circuit_ok,(const circuit_t *c));
+void circuit_free_all(void);
+void circuits_handle_oom(size_t current_allocation);
+
+void circuit_clear_testing_cell_stats(circuit_t *circ);
+
+void channel_note_destroy_pending(channel_t *chan, circid_t id);
+MOCK_DECL(void, channel_note_destroy_not_pending,
+          (channel_t *chan, circid_t id));
+
+smartlist_t *circuit_find_circuits_to_upgrade_from_guard_wait(void);
+
+#ifdef CIRCUITLIST_PRIVATE
+STATIC void circuit_free_(circuit_t *circ);
+#define circuit_free(circ) FREE_AND_NULL(circuit_t, circuit_free_, (circ))
+STATIC size_t n_cells_in_circ_queues(const circuit_t *c);
+STATIC uint32_t circuit_max_queued_data_age(const circuit_t *c, uint32_t now);
+STATIC uint32_t circuit_max_queued_cell_age(const circuit_t *c, uint32_t now);
+STATIC uint32_t circuit_max_queued_item_age(const circuit_t *c, uint32_t now);
+#endif /* defined(CIRCUITLIST_PRIVATE) */
+
+#endif /* !defined(TOR_CIRCUITLIST_H) */
diff --git a/src/or/circuitmux.c b/src/core/or/circuitmux.c
index 036fb9570d..88f9ac7923 100644
--- a/src/or/circuitmux.c
+++ b/src/core/or/circuitmux.c
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -69,11 +69,15 @@
  *     made to attach all existing circuits to the new policy.
  **/
 
-#include "or.h"
-#include "channel.h"
-#include "circuitlist.h"
-#include "circuitmux.h"
-#include "relay.h"
+#include "core/or/or.h"
+#include "core/or/channel.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitmux.h"
+#include "core/or/relay.h"
+
+#include "core/or/cell_queue_st.h"
+#include "core/or/destroy_cell_queue_st.h"
+#include "core/or/or_circuit_st.h"
 
 /*
  * Private typedefs for circuitmux.c
@@ -114,13 +118,6 @@ struct circuitmux_s {
    */
   chanid_circid_muxinfo_map_t *chanid_circid_map;
 
-  /*
-   * Double-linked ring of circuits with queued cells waiting for room to
-   * free up on this connection's outbuf.  Every time we pull cells from
-   * a circuit, we advance this pointer to the next circuit in the ring.
-   */
-  struct circuit_t *active_circuits_head, *active_circuits_tail;
-
   /** List of queued destroy cells */
   destroy_cell_queue_t destroy_cell_queue;
   /** Boolean: True iff the last cell to circuitmux_get_first_active_circuit
@@ -177,17 +174,6 @@ struct chanid_circid_muxinfo_t {
 };
 
 /*
- * Internal-use #defines
- */
-
-#ifdef CMUX_PARANOIA
-#define circuitmux_assert_okay_paranoid(cmux) \
-  circuitmux_assert_okay(cmux)
-#else
-#define circuitmux_assert_okay_paranoid(cmux)
-#endif
-
-/*
  * Static function declarations
  */
 
@@ -199,21 +185,9 @@ chanid_circid_entry_hash(chanid_circid_muxinfo_t *a);
 static chanid_circid_muxinfo_t *
 circuitmux_find_map_entry(circuitmux_t *cmux, circuit_t *circ);
 static void
-circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ,
-                               cell_direction_t direction);
+circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ);
 static void
-circuitmux_make_circuit_inactive(circuitmux_t *cmux, circuit_t *circ,
-                                 cell_direction_t direction);
-static inline void
-circuitmux_move_active_circ_to_tail(circuitmux_t *cmux, circuit_t *circ,
-                                    cell_direction_t direction);
-static inline circuit_t **
-circuitmux_next_active_circ_p(circuitmux_t *cmux, circuit_t *circ);
-static inline circuit_t **
-circuitmux_prev_active_circ_p(circuitmux_t *cmux, circuit_t *circ);
-static void circuitmux_assert_okay_pass_one(circuitmux_t *cmux);
-static void circuitmux_assert_okay_pass_two(circuitmux_t *cmux);
-static void circuitmux_assert_okay_pass_three(circuitmux_t *cmux);
+circuitmux_make_circuit_inactive(circuitmux_t *cmux, circuit_t *circ);
 
 /* Static global variables */
 
@@ -223,121 +197,6 @@ static int64_t global_destroy_ctr = 0;
 /* Function definitions */
 
 /**
- * Linked list helpers
- */
-
-/**
- * Move an active circuit to the tail of the cmux's active circuits list;
- * used by circuitmux_notify_xmit_cells().
- */
-
-static inline void
-circuitmux_move_active_circ_to_tail(circuitmux_t *cmux, circuit_t *circ,
-                                    cell_direction_t direction)
-{
-  circuit_t **next_p = NULL, **prev_p = NULL;
-  circuit_t **next_prev = NULL, **prev_next = NULL;
-  circuit_t **tail_next = NULL;
-  or_circuit_t *or_circ = NULL;
-
-  tor_assert(cmux);
-  tor_assert(circ);
-
-  circuitmux_assert_okay_paranoid(cmux);
-
-  /* Figure out our next_p and prev_p for this cmux/direction */
-  if (direction) {
-    if (direction == CELL_DIRECTION_OUT) {
-      tor_assert(circ->n_mux == cmux);
-      next_p = &(circ->next_active_on_n_chan);
-      prev_p = &(circ->prev_active_on_n_chan);
-    } else {
-      or_circ = TO_OR_CIRCUIT(circ);
-      tor_assert(or_circ->p_mux == cmux);
-      next_p = &(or_circ->next_active_on_p_chan);
-      prev_p = &(or_circ->prev_active_on_p_chan);
-    }
-  } else {
-    if (circ->n_mux == cmux) {
-      next_p = &(circ->next_active_on_n_chan);
-      prev_p = &(circ->prev_active_on_n_chan);
-      direction = CELL_DIRECTION_OUT;
-    } else {
-      or_circ = TO_OR_CIRCUIT(circ);
-      tor_assert(or_circ->p_mux == cmux);
-      next_p = &(or_circ->next_active_on_p_chan);
-      prev_p = &(or_circ->prev_active_on_p_chan);
-      direction = CELL_DIRECTION_IN;
-    }
-  }
-  tor_assert(next_p);
-  tor_assert(prev_p);
-
-  /* Check if this really is an active circuit */
-  if ((*next_p == NULL && *prev_p == NULL) &&
-      !(circ == cmux->active_circuits_head ||
-        circ == cmux->active_circuits_tail)) {
-    /* Not active, no-op */
-    return;
-  }
-
-  /* Check if this is already the tail */
-  if (circ == cmux->active_circuits_tail) return;
-
-  /* Okay, we have to move it; figure out next_prev and prev_next */
-  if (*next_p) next_prev = circuitmux_prev_active_circ_p(cmux, *next_p);
-  if (*prev_p) prev_next = circuitmux_next_active_circ_p(cmux, *prev_p);
-  /* Adjust the previous node's next pointer, if any */
-  if (prev_next) *prev_next = *next_p;
-  /* Otherwise, we were the head */
-  else cmux->active_circuits_head = *next_p;
-  /* Adjust the next node's previous pointer, if any */
-  if (next_prev) *next_prev = *prev_p;
-  /* We're out of the list; now re-attach at the tail */
-  /* Adjust our next and prev pointers */
-  *next_p = NULL;
-  *prev_p = cmux->active_circuits_tail;
-  /* Set the next pointer of the tail, or the head if none */
-  if (cmux->active_circuits_tail) {
-    tail_next = circuitmux_next_active_circ_p(cmux,
-                                              cmux->active_circuits_tail);
-    *tail_next = circ;
-  } else {
-    cmux->active_circuits_head = circ;
-  }
-  /* Set the tail to this circuit */
-  cmux->active_circuits_tail = circ;
-
-  circuitmux_assert_okay_paranoid(cmux);
-}
-
-static inline circuit_t **
-circuitmux_next_active_circ_p(circuitmux_t *cmux, circuit_t *circ)
-{
-  tor_assert(cmux);
-  tor_assert(circ);
-
-  if (circ->n_mux == cmux) return &(circ->next_active_on_n_chan);
-  else {
-    tor_assert(TO_OR_CIRCUIT(circ)->p_mux == cmux);
-    return &(TO_OR_CIRCUIT(circ)->next_active_on_p_chan);
-  }
-}
-
-static inline circuit_t **
-circuitmux_prev_active_circ_p(circuitmux_t *cmux, circuit_t *circ)
-{
-  tor_assert(cmux);
-  tor_assert(circ);
-
-  if (circ->n_mux == cmux) return &(circ->prev_active_on_n_chan);
-  else {
-    tor_assert(TO_OR_CIRCUIT(circ)->p_mux == cmux);
-    return &(TO_OR_CIRCUIT(circ)->prev_active_on_p_chan);
-  }
-}
-
-/**
  * Helper for chanid_circid_cell_count_map_t hash table: compare the channel
  * ID and circuit ID for a and b, and return less than, equal to, or greater
  * than zero appropriately.
@@ -408,11 +267,6 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out)
   circuit_t *circ = NULL;
 
   tor_assert(cmux);
-  /*
-   * Don't circuitmux_assert_okay_paranoid() here; this gets called when
-   * channels are being freed and have already been unregistered, so
-   * the channel ID lookups it does will fail.
-   */
 
   i = HT_START(chanid_circid_muxinfo_map, cmux->chanid_circid_map);
   while (i) {
@@ -437,7 +291,7 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out)
              */
 
             if (to_remove->muxinfo.cell_count > 0) {
-              circuitmux_make_circuit_inactive(cmux, circ, CELL_DIRECTION_OUT);
+              circuitmux_make_circuit_inactive(cmux, circ);
             }
 
             /* Clear n_mux */
@@ -452,7 +306,7 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out)
              */
 
             if (to_remove->muxinfo.cell_count > 0) {
-              circuitmux_make_circuit_inactive(cmux, circ, CELL_DIRECTION_IN);
+              circuitmux_make_circuit_inactive(cmux, circ);
             }
 
             /*
@@ -466,10 +320,10 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out)
           } else {
             /* Complain and move on */
             log_warn(LD_CIRC,
-                     "Circuit %u/channel " U64_FORMAT " had direction == "
+                     "Circuit %u/channel %"PRIu64 " had direction == "
                      "CELL_DIRECTION_IN, but isn't an or_circuit_t",
                      (unsigned)to_remove->circ_id,
-                     U64_PRINTF_ARG(to_remove->chan_id));
+                     (to_remove->chan_id));
           }
 
           /* Free policy-specific data if we have it */
@@ -490,15 +344,15 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out)
         } else {
           /* Complain and move on */
           log_warn(LD_CIRC,
-                   "Couldn't find circuit %u (for channel " U64_FORMAT ")",
+                   "Couldn't find circuit %u (for channel %"PRIu64 ")",
                    (unsigned)to_remove->circ_id,
-                   U64_PRINTF_ARG(to_remove->chan_id));
+                   (to_remove->chan_id));
         }
       } else {
         /* Complain and move on */
         log_warn(LD_CIRC,
-                 "Couldn't find channel " U64_FORMAT " (for circuit id %u)",
-                 U64_PRINTF_ARG(to_remove->chan_id),
+                 "Couldn't find channel %"PRIu64 " (for circuit id %u)",
+                 (to_remove->chan_id),
                  (unsigned)to_remove->circ_id);
       }
 
@@ -539,7 +393,7 @@ circuitmux_mark_destroyed_circids_usable(circuitmux_t *cmux, channel_t *chan)
  */
 
 void
-circuitmux_free(circuitmux_t *cmux)
+circuitmux_free_(circuitmux_t *cmux)
 {
   if (!cmux) return;
 
@@ -574,17 +428,17 @@ circuitmux_free(circuitmux_t *cmux)
     global_destroy_ctr -= cmux->destroy_cell_queue.n;
     log_debug(LD_CIRC,
               "Freeing cmux at %p with %u queued destroys; the last cmux "
-              "destroy balance was "I64_FORMAT", global is "I64_FORMAT,
+              "destroy balance was %"PRId64", global is %"PRId64,
               cmux, cmux->destroy_cell_queue.n,
-              I64_PRINTF_ARG(cmux->destroy_ctr),
-              I64_PRINTF_ARG(global_destroy_ctr));
+              (cmux->destroy_ctr),
+              (global_destroy_ctr));
   } else {
     log_debug(LD_CIRC,
               "Freeing cmux at %p with no queued destroys, the cmux destroy "
-              "balance was "I64_FORMAT", global is "I64_FORMAT,
+              "balance was %"PRId64", global is %"PRId64,
               cmux,
-              I64_PRINTF_ARG(cmux->destroy_ctr),
-              I64_PRINTF_ARG(global_destroy_ctr));
+              (cmux->destroy_ctr),
+              (global_destroy_ctr));
   }
 
   destroy_cell_queue_clear(&cmux->destroy_cell_queue);
@@ -608,9 +462,7 @@ circuitmux_clear_policy(circuitmux_t *cmux)
   tor_assert(cmux);
 
   /* Internally, this is just setting policy to NULL */
-  if (cmux->policy) {
-    circuitmux_set_policy(cmux, NULL);
-  }
+  circuitmux_set_policy(cmux, NULL);
 }
 
 /**
@@ -946,7 +798,6 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
   tor_assert(circ);
   tor_assert(direction == CELL_DIRECTION_IN ||
              direction == CELL_DIRECTION_OUT);
-  circuitmux_assert_okay_paranoid(cmux);
 
   /*
    * Figure out which channel we're using, and get the circuit's current
@@ -984,9 +835,9 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
      * directions match and update the cell count and active circuit count.
      */
     log_info(LD_CIRC,
-             "Circuit %u on channel " U64_FORMAT " was already attached to "
+             "Circuit %u on channel %"PRIu64 " was already attached to "
              "cmux %p (trying to attach to %p)",
-             (unsigned)circ_id, U64_PRINTF_ARG(channel_id),
+             (unsigned)circ_id, (channel_id),
              ((direction == CELL_DIRECTION_OUT) ?
                 circ->n_mux : TO_OR_CIRCUIT(circ)->p_mux),
              cmux);
@@ -1004,10 +855,10 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
      */
     if (hashent->muxinfo.cell_count > 0 && cell_count == 0) {
       --(cmux->n_active_circuits);
-      circuitmux_make_circuit_inactive(cmux, circ, direction);
+      circuitmux_make_circuit_inactive(cmux, circ);
     } else if (hashent->muxinfo.cell_count == 0 && cell_count > 0) {
       ++(cmux->n_active_circuits);
-      circuitmux_make_circuit_active(cmux, circ, direction);
+      circuitmux_make_circuit_active(cmux, circ);
     }
     cmux->n_cells -= hashent->muxinfo.cell_count;
     cmux->n_cells += cell_count;
@@ -1018,8 +869,8 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
      * counts.
      */
     log_debug(LD_CIRC,
-             "Attaching circuit %u on channel " U64_FORMAT " to cmux %p",
-              (unsigned)circ_id, U64_PRINTF_ARG(channel_id), cmux);
+             "Attaching circuit %u on channel %"PRIu64 " to cmux %p",
+              (unsigned)circ_id, (channel_id), cmux);
 
     /*
      * Assert that the circuit doesn't already have a mux for this
@@ -1035,7 +886,7 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
     hashent->muxinfo.cell_count = cell_count;
     hashent->muxinfo.direction = direction;
     /* Allocate policy specific circuit data if we need it */
-    if (cmux->policy && cmux->policy->alloc_circ_data) {
+    if (cmux->policy->alloc_circ_data) {
       /* Assert that we have the means to free policy-specific data */
       tor_assert(cmux->policy->free_circ_data);
       /* Allocate it */
@@ -1055,25 +906,14 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
     if (direction == CELL_DIRECTION_OUT) circ->n_mux = cmux;
     else TO_OR_CIRCUIT(circ)->p_mux = cmux;
 
-    /* Make sure the next/prev pointers are NULL */
-    if (direction == CELL_DIRECTION_OUT) {
-      circ->next_active_on_n_chan = NULL;
-      circ->prev_active_on_n_chan = NULL;
-    } else {
-      TO_OR_CIRCUIT(circ)->next_active_on_p_chan = NULL;
-      TO_OR_CIRCUIT(circ)->prev_active_on_p_chan = NULL;
-    }
-
     /* Update counters */
     ++(cmux->n_circuits);
     if (cell_count > 0) {
       ++(cmux->n_active_circuits);
-      circuitmux_make_circuit_active(cmux, circ, direction);
+      circuitmux_make_circuit_active(cmux, circ);
     }
     cmux->n_cells += cell_count;
   }
-
-  circuitmux_assert_okay_paranoid(cmux);
 }
 
 /**
@@ -1097,7 +937,6 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ))
   tor_assert(cmux);
   tor_assert(cmux->chanid_circid_map);
   tor_assert(circ);
-  circuitmux_assert_okay_paranoid(cmux);
 
   /* See if we have it for n_chan/n_circ_id */
   if (circ->n_chan) {
@@ -1135,7 +974,7 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ))
     if (hashent->muxinfo.cell_count > 0) {
       --(cmux->n_active_circuits);
       /* This does policy notifies, so comes before freeing policy data */
-      circuitmux_make_circuit_inactive(cmux, circ, last_searched_direction);
+      circuitmux_make_circuit_inactive(cmux, circ);
     }
     cmux->n_cells -= hashent->muxinfo.cell_count;
 
@@ -1164,8 +1003,6 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ))
     /* Free the hash entry */
     tor_free(hashent);
   }
-
-  circuitmux_assert_okay_paranoid(cmux);
 }
 
 /**
@@ -1174,94 +1011,22 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ))
  */
 
 static void
-circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ,
-                               cell_direction_t direction)
+circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ)
 {
-  circuit_t **next_active = NULL, **prev_active = NULL, **next_prev = NULL;
-  circuitmux_t *circuit_cmux = NULL;
-  chanid_circid_muxinfo_t *hashent = NULL;
-  channel_t *chan = NULL;
-  circid_t circ_id;
-  int already_active;
-
   tor_assert(cmux);
+  tor_assert(cmux->policy);
   tor_assert(circ);
-  tor_assert(direction == CELL_DIRECTION_OUT ||
-             direction == CELL_DIRECTION_IN);
-  /*
-   * Don't circuitmux_assert_okay_paranoid(cmux) here because the cell count
-   * already got changed and we have to update the list for it to be consistent
-   * again.
-   */
-
-  /* Get the right set of active list links for this direction */
-  if (direction == CELL_DIRECTION_OUT) {
-    next_active = &(circ->next_active_on_n_chan);
-    prev_active = &(circ->prev_active_on_n_chan);
-    circuit_cmux = circ->n_mux;
-    chan = circ->n_chan;
-    circ_id = circ->n_circ_id;
-  } else {
-    next_active = &(TO_OR_CIRCUIT(circ)->next_active_on_p_chan);
-    prev_active = &(TO_OR_CIRCUIT(circ)->prev_active_on_p_chan);
-    circuit_cmux = TO_OR_CIRCUIT(circ)->p_mux;
-    chan = TO_OR_CIRCUIT(circ)->p_chan;
-    circ_id = TO_OR_CIRCUIT(circ)->p_circ_id;
-  }
-
-  /* Assert that it is attached to this mux and a channel */
-  tor_assert(cmux == circuit_cmux);
-  tor_assert(chan != NULL);
-
-  /*
-   * Check if the circuit really was inactive; if it's active, at least one
-   * of the next_active and prev_active pointers will not be NULL, or this
-   * circuit will be either the head or tail of the list for this cmux.
-   */
-  already_active = (*prev_active != NULL || *next_active != NULL ||
-                    cmux->active_circuits_head == circ ||
-                    cmux->active_circuits_tail == circ);
-
-  /* If we're already active, log a warning and finish */
-  if (already_active) {
-    log_warn(LD_CIRC,
-             "Circuit %u on channel " U64_FORMAT " was already active",
-             (unsigned)circ_id, U64_PRINTF_ARG(chan->global_identifier));
-    return;
-  }
-
-  /*
-   * This is going at the head of the list; if the old head is not NULL,
-   * then its prev pointer should point to this.
-   */
-  *next_active = cmux->active_circuits_head; /* Next is old head */
-  *prev_active = NULL; /* Prev is NULL (this will be the head) */
-  if (cmux->active_circuits_head) {
-    /* The list had an old head; update its prev pointer */
-    next_prev =
-      circuitmux_prev_active_circ_p(cmux, cmux->active_circuits_head);
-    tor_assert(next_prev);
-    *next_prev = circ;
-  } else {
-    /* The list was empty; this becomes the tail as well */
-    cmux->active_circuits_tail = circ;
-  }
-  /* This becomes the new head of the list */
-  cmux->active_circuits_head = circ;
 
   /* Policy-specific notification */
-  if (cmux->policy &&
-      cmux->policy->notify_circ_active) {
+  if (cmux->policy->notify_circ_active) {
     /* Okay, we need to check the circuit for policy data now */
-    hashent = circuitmux_find_map_entry(cmux, circ);
+    chanid_circid_muxinfo_t *hashent = circuitmux_find_map_entry(cmux, circ);
     /* We should have found something */
     tor_assert(hashent);
     /* Notify */
     cmux->policy->notify_circ_active(cmux, cmux->policy_data,
                                      circ, hashent->muxinfo.policy_data);
   }
-
-  circuitmux_assert_okay_paranoid(cmux);
 }
 
 /**
@@ -1270,112 +1035,22 @@ circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ,
  */
 
 static void
-circuitmux_make_circuit_inactive(circuitmux_t *cmux, circuit_t *circ,
-                                 cell_direction_t direction)
+circuitmux_make_circuit_inactive(circuitmux_t *cmux, circuit_t *circ)
 {
-  circuit_t **next_active = NULL, **prev_active = NULL;
-  circuit_t **next_prev = NULL, **prev_next = NULL;
-  circuitmux_t *circuit_cmux = NULL;
-  chanid_circid_muxinfo_t *hashent = NULL;
-  channel_t *chan = NULL;
-  circid_t circ_id;
-  int already_inactive;
-
   tor_assert(cmux);
+  tor_assert(cmux->policy);
   tor_assert(circ);
-  tor_assert(direction == CELL_DIRECTION_OUT ||
-             direction == CELL_DIRECTION_IN);
-  /*
-   * Don't circuitmux_assert_okay_paranoid(cmux) here because the cell count
-   * already got changed and we have to update the list for it to be consistent
-   * again.
-   */
-
-  /* Get the right set of active list links for this direction */
-  if (direction == CELL_DIRECTION_OUT) {
-    next_active = &(circ->next_active_on_n_chan);
-    prev_active = &(circ->prev_active_on_n_chan);
-    circuit_cmux = circ->n_mux;
-    chan = circ->n_chan;
-    circ_id = circ->n_circ_id;
-  } else {
-    next_active = &(TO_OR_CIRCUIT(circ)->next_active_on_p_chan);
-    prev_active = &(TO_OR_CIRCUIT(circ)->prev_active_on_p_chan);
-    circuit_cmux = TO_OR_CIRCUIT(circ)->p_mux;
-    chan = TO_OR_CIRCUIT(circ)->p_chan;
-    circ_id = TO_OR_CIRCUIT(circ)->p_circ_id;
-  }
-
-  /* Assert that it is attached to this mux and a channel */
-  tor_assert(cmux == circuit_cmux);
-  tor_assert(chan != NULL);
-
-  /*
-   * Check if the circuit really was active; if it's inactive, the
-   * next_active and prev_active pointers will be NULL and this circuit
-   * will not be the head or tail of the list for this cmux.
-   */
-  already_inactive = (*prev_active == NULL && *next_active == NULL &&
-                      cmux->active_circuits_head != circ &&
-                      cmux->active_circuits_tail != circ);
-
-  /* If we're already inactive, log a warning and finish */
-  if (already_inactive) {
-    log_warn(LD_CIRC,
-             "Circuit %d on channel " U64_FORMAT " was already inactive",
-             (unsigned)circ_id, U64_PRINTF_ARG(chan->global_identifier));
-    return;
-  }
-
-  /* Remove from the list; first get next_prev and prev_next */
-  if (*next_active) {
-    /*
-     * If there's a next circuit, its previous circuit becomes this
-     * circuit's previous circuit.
-     */
-    next_prev = circuitmux_prev_active_circ_p(cmux, *next_active);
-  } else {
-    /* Else, the tail becomes this circuit's previous circuit */
-    next_prev = &(cmux->active_circuits_tail);
-  }
-
-  /* Got next_prev, now prev_next */
-  if (*prev_active) {
-    /*
-     * If there's a previous circuit, its next circuit becomes this circuit's
-     * next circuit.
-     */
-    prev_next = circuitmux_next_active_circ_p(cmux, *prev_active);
-  } else {
-    /* Else, the head becomes this circuit's next circuit */
-    prev_next = &(cmux->active_circuits_head);
-  }
-
-  /* Assert that we got sensible values for the next/prev pointers */
-  tor_assert(next_prev != NULL);
-  tor_assert(prev_next != NULL);
-
-  /* Update the next/prev pointers - this removes circ from the list */
-  *next_prev = *prev_active;
-  *prev_next = *next_active;
-
-  /* Now null out prev_active/next_active */
-  *prev_active = NULL;
-  *next_active = NULL;
 
   /* Policy-specific notification */
-  if (cmux->policy &&
-      cmux->policy->notify_circ_inactive) {
+  if (cmux->policy->notify_circ_inactive) {
     /* Okay, we need to check the circuit for policy data now */
-    hashent = circuitmux_find_map_entry(cmux, circ);
+    chanid_circid_muxinfo_t *hashent = circuitmux_find_map_entry(cmux, circ);
     /* We should have found something */
     tor_assert(hashent);
     /* Notify */
     cmux->policy->notify_circ_inactive(cmux, cmux->policy_data,
                                        circ, hashent->muxinfo.policy_data);
   }
-
-  circuitmux_assert_okay_paranoid(cmux);
 }
 
 /**
@@ -1402,8 +1077,6 @@ circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ,
   tor_assert(cmux);
   tor_assert(circ);
 
-  circuitmux_assert_okay_paranoid(cmux);
-
   /* Search for this circuit's entry */
   hashent = circuitmux_find_map_entry(cmux, circ);
   /* Assert that we found one */
@@ -1414,7 +1087,7 @@ circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ,
   cmux->n_cells += n_cells;
 
   /* Do we need to notify a cmux policy? */
-  if (cmux->policy && cmux->policy->notify_set_n_cells) {
+  if (cmux->policy->notify_set_n_cells) {
     /* Call notify_set_n_cells */
     cmux->policy->notify_set_n_cells(cmux,
                                      cmux->policy_data,
@@ -1430,21 +1103,15 @@ circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ,
   if (hashent->muxinfo.cell_count > 0 && n_cells == 0) {
     --(cmux->n_active_circuits);
     hashent->muxinfo.cell_count = n_cells;
-    circuitmux_make_circuit_inactive(cmux, circ, hashent->muxinfo.direction);
+    circuitmux_make_circuit_inactive(cmux, circ);
   /* Is the old cell count == 0 and the new cell count > 0 ? */
   } else if (hashent->muxinfo.cell_count == 0 && n_cells > 0) {
     ++(cmux->n_active_circuits);
     hashent->muxinfo.cell_count = n_cells;
-    circuitmux_make_circuit_active(cmux, circ, hashent->muxinfo.direction);
+    circuitmux_make_circuit_active(cmux, circ);
   } else {
-    /*
-     * Update the entry cell count like this so we can put a
-     * circuitmux_assert_okay_paranoid inside make_circuit_(in)active() too.
-     */
     hashent->muxinfo.cell_count = n_cells;
   }
-
-  circuitmux_assert_okay_paranoid(cmux);
 }
 
 /*
@@ -1470,6 +1137,9 @@ circuitmux_get_first_active_circuit(circuitmux_t *cmux,
   circuit_t *circ = NULL;
 
   tor_assert(cmux);
+  tor_assert(cmux->policy);
+  /* This callback is mandatory. */
+  tor_assert(cmux->policy->pick_active_circuit);
   tor_assert(destroy_queue_out);
 
   *destroy_queue_out = NULL;
@@ -1488,14 +1158,7 @@ circuitmux_get_first_active_circuit(circuitmux_t *cmux,
     /* We also must have a cell available for this to be the case */
     tor_assert(cmux->n_cells > 0);
     /* Do we have a policy-provided circuit selector? */
-    if (cmux->policy && cmux->policy->pick_active_circuit) {
-      circ = cmux->policy->pick_active_circuit(cmux, cmux->policy_data);
-    }
-    /* Fall back on the head of the active circuits list */
-    if (!circ) {
-      tor_assert(cmux->active_circuits_head);
-      circ = cmux->active_circuits_head;
-    }
+    circ = cmux->policy->pick_active_circuit(cmux, cmux->policy_data);
     cmux->last_cell_was_destroy = 0;
   } else {
     tor_assert(cmux->n_cells == 0);
@@ -1519,7 +1182,6 @@ circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ,
 
   tor_assert(cmux);
   tor_assert(circ);
-  circuitmux_assert_okay_paranoid(cmux);
 
   if (n_cells == 0) return;
 
@@ -1546,17 +1208,11 @@ circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ,
   /* Adjust the mux cell counter */
   cmux->n_cells -= n_cells;
 
-  /* If we aren't making it inactive later, move it to the tail of the list */
-  if (!becomes_inactive) {
-    circuitmux_move_active_circ_to_tail(cmux, circ,
-                                        hashent->muxinfo.direction);
-  }
-
   /*
    * We call notify_xmit_cells() before making the circuit inactive if needed,
    * so the policy can always count on this coming in on an active circuit.
    */
-  if (cmux->policy && cmux->policy->notify_xmit_cells) {
+  if (cmux->policy->notify_xmit_cells) {
     cmux->policy->notify_xmit_cells(cmux, cmux->policy_data, circ,
                                     hashent->muxinfo.policy_data,
                                     n_cells);
@@ -1568,10 +1224,8 @@ circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ,
    */
   if (becomes_inactive) {
     --(cmux->n_active_circuits);
-    circuitmux_make_circuit_inactive(cmux, circ, hashent->muxinfo.direction);
+    circuitmux_make_circuit_inactive(cmux, circ);
   }
-
-  circuitmux_assert_okay_paranoid(cmux);
 }
 
 /**
@@ -1587,291 +1241,11 @@ circuitmux_notify_xmit_destroy(circuitmux_t *cmux)
   --(cmux->destroy_ctr);
   --(global_destroy_ctr);
   log_debug(LD_CIRC,
-            "Cmux at %p sent a destroy, cmux counter is now "I64_FORMAT", "
-            "global counter is now "I64_FORMAT,
+            "Cmux at %p sent a destroy, cmux counter is now %"PRId64", "
+            "global counter is now %"PRId64,
             cmux,
-            I64_PRINTF_ARG(cmux->destroy_ctr),
-            I64_PRINTF_ARG(global_destroy_ctr));
-}
-
-/*
- * Circuitmux consistency checking assertions
- */
-
-/**
- * Check that circuitmux data structures are consistent and fail with an
- * assert if not.
- */
-
-void
-circuitmux_assert_okay(circuitmux_t *cmux)
-{
-  tor_assert(cmux);
-
-  /*
-   * Pass 1: iterate the hash table; for each entry:
-   *  a) Check that the circuit has this cmux for n_mux or p_mux
-   *  b) If the cell_count is > 0, set the mark bit; otherwise clear it
-   *  c) Also check activeness (cell_count > 0 should be active)
-   *  d) Count the number of circuits, active circuits and queued cells
-   *     and at the end check that they match the counters in the cmux.
-   *
-   * Pass 2: iterate the active circuits list; for each entry,
-   *         make sure the circuit is attached to this mux and appears
-   *         in the hash table.  Make sure the mark bit is 1, and clear
-   *         it in the hash table entry.  Consistency-check the linked
-   *         list pointers.
-   *
-   * Pass 3: iterate the hash table again; assert if any active circuits
-   *         (mark bit set to 1) are discovered that weren't cleared in pass 2
-   *         (don't appear in the linked list).
-   */
-
-  circuitmux_assert_okay_pass_one(cmux);
-  circuitmux_assert_okay_pass_two(cmux);
-  circuitmux_assert_okay_pass_three(cmux);
-}
-
-/**
- * Do the first pass of circuitmux_assert_okay(); see the comment in that
- * function.
- */
-
-static void
-circuitmux_assert_okay_pass_one(circuitmux_t *cmux)
-{
-  chanid_circid_muxinfo_t **i = NULL;
-  uint64_t chan_id;
-  channel_t *chan;
-  circid_t circ_id;
-  circuit_t *circ;
-  or_circuit_t *or_circ;
-  unsigned int circ_is_active;
-  circuit_t **next_p, **prev_p;
-  unsigned int n_circuits, n_active_circuits, n_cells;
-
-  tor_assert(cmux);
-  tor_assert(cmux->chanid_circid_map);
-
-  /* Reset the counters */
-  n_circuits = n_active_circuits = n_cells = 0;
-  /* Start iterating the hash table */
-  i = HT_START(chanid_circid_muxinfo_map, cmux->chanid_circid_map);
-  while (i) {
-    /* Assert that the hash table entry isn't null */
-    tor_assert(*i);
-
-    /* Get the channel and circuit id */
-    chan_id = (*i)->chan_id;
-    circ_id = (*i)->circ_id;
-
-    /* Find the channel and circuit, assert that they exist */
-    chan = channel_find_by_global_id(chan_id);
-    tor_assert(chan);
-    circ = circuit_get_by_circid_channel_even_if_marked(circ_id, chan);
-    tor_assert(circ);
-    /* Clear the circ_is_active bit to start */
-    circ_is_active = 0;
-
-    /* Assert that we know which direction this is going */
-    tor_assert((*i)->muxinfo.direction == CELL_DIRECTION_OUT ||
-               (*i)->muxinfo.direction == CELL_DIRECTION_IN);
-
-    if ((*i)->muxinfo.direction == CELL_DIRECTION_OUT) {
-      /* We should be n_mux on this circuit */
-      tor_assert(cmux == circ->n_mux);
-      tor_assert(chan == circ->n_chan);
-      /* Get next and prev for next test */
-      next_p = &(circ->next_active_on_n_chan);
-      prev_p = &(circ->prev_active_on_n_chan);
-    } else {
-      /* This should be an or_circuit_t and we should be p_mux */
-      or_circ = TO_OR_CIRCUIT(circ);
-      tor_assert(cmux == or_circ->p_mux);
-      tor_assert(chan == or_circ->p_chan);
-      /* Get next and prev for next test */
-      next_p = &(or_circ->next_active_on_p_chan);
-      prev_p = &(or_circ->prev_active_on_p_chan);
-    }
-
-    /*
-     * Should this circuit be active?  I.e., does the mux know about > 0
-     * cells on it?
-     */
-    circ_is_active = ((*i)->muxinfo.cell_count > 0);
-
-    /* It should be in the linked list iff it's active */
-    if (circ_is_active) {
-      /* Either we have a next link or we are the tail */
-      tor_assert(*next_p || (circ == cmux->active_circuits_tail));
-      /* Either we have a prev link or we are the head */
-      tor_assert(*prev_p || (circ == cmux->active_circuits_head));
-      /* Increment the active circuits counter */
-      ++n_active_circuits;
-    } else {
-      /* Shouldn't be in list, so no next or prev link */
-      tor_assert(!(*next_p));
-      tor_assert(!(*prev_p));
-      /* And can't be head or tail */
-      tor_assert(circ != cmux->active_circuits_head);
-      tor_assert(circ != cmux->active_circuits_tail);
-    }
-
-    /* Increment the circuits counter */
-    ++n_circuits;
-    /* Adjust the cell counter */
-    n_cells += (*i)->muxinfo.cell_count;
-
-    /* Set the mark bit to circ_is_active */
-    (*i)->muxinfo.mark = circ_is_active;
-
-    /* Advance to the next entry */
-    i = HT_NEXT(chanid_circid_muxinfo_map, cmux->chanid_circid_map, i);
-  }
-
-  /* Now check the counters */
-  tor_assert(n_cells == cmux->n_cells);
-  tor_assert(n_circuits == cmux->n_circuits);
-  tor_assert(n_active_circuits == cmux->n_active_circuits);
-}
-
-/**
- * Do the second pass of circuitmux_assert_okay(); see the comment in that
- * function.
- */
-
-static void
-circuitmux_assert_okay_pass_two(circuitmux_t *cmux)
-{
-  circuit_t *curr_circ, *prev_circ = NULL, *next_circ;
-  or_circuit_t *curr_or_circ;
-  uint64_t curr_chan_id;
-  circid_t curr_circ_id;
-  circuit_t **next_p, **prev_p;
-  channel_t *chan;
-  unsigned int n_active_circuits = 0;
-  cell_direction_t direction;
-  chanid_circid_muxinfo_t search, *hashent = NULL;
-
-  tor_assert(cmux);
-  tor_assert(cmux->chanid_circid_map);
-
-  /*
-   * Walk the linked list of active circuits in cmux; keep track of the
-   * previous circuit seen for consistency checking purposes.  Count them
-   * to make sure the number in the linked list matches
-   * cmux->n_active_circuits.
-   */
-  curr_circ = cmux->active_circuits_head;
-  while (curr_circ) {
-    /* Reset some things */
-    chan = NULL;
-    curr_or_circ = NULL;
-    next_circ = NULL;
-    next_p = prev_p = NULL;
-    direction = 0;
-
-    /* Figure out if this is n_mux or p_mux */
-    if (cmux == curr_circ->n_mux) {
-      /* Get next_p and prev_p */
-      next_p = &(curr_circ->next_active_on_n_chan);
-      prev_p = &(curr_circ->prev_active_on_n_chan);
-      /* Get the channel */
-      chan = curr_circ->n_chan;
-      /* Get the circuit id */
-      curr_circ_id = curr_circ->n_circ_id;
-      /* Remember the direction */
-      direction = CELL_DIRECTION_OUT;
-    } else {
-      /* We must be p_mux and this must be an or_circuit_t */
-      curr_or_circ = TO_OR_CIRCUIT(curr_circ);
-      tor_assert(cmux == curr_or_circ->p_mux);
-      /* Get next_p and prev_p */
-      next_p = &(curr_or_circ->next_active_on_p_chan);
-      prev_p = &(curr_or_circ->prev_active_on_p_chan);
-      /* Get the channel */
-      chan = curr_or_circ->p_chan;
-      /* Get the circuit id */
-      curr_circ_id = curr_or_circ->p_circ_id;
-      /* Remember the direction */
-      direction = CELL_DIRECTION_IN;
-    }
-
-    /* Assert that we got a channel and get the channel ID */
-    tor_assert(chan);
-    curr_chan_id = chan->global_identifier;
-
-    /* Assert that prev_p points to last circuit we saw */
-    tor_assert(*prev_p == prev_circ);
-    /* If that's NULL, assert that we are the head */
-    if (!(*prev_p)) tor_assert(curr_circ == cmux->active_circuits_head);
-
-    /* Get the next circuit */
-    next_circ = *next_p;
-    /* If it's NULL, assert that we are the tail */
-    if (!(*next_p)) tor_assert(curr_circ == cmux->active_circuits_tail);
-
-    /* Now find the hash table entry for this circuit */
-    search.chan_id = curr_chan_id;
-    search.circ_id = curr_circ_id;
-    hashent = HT_FIND(chanid_circid_muxinfo_map, cmux->chanid_circid_map,
-                      &search);
-
-    /* Assert that we have one */
-    tor_assert(hashent);
-
-    /* Assert that the direction matches */
-    tor_assert(direction == hashent->muxinfo.direction);
-
-    /* Assert that the hash entry got marked in pass one */
-    tor_assert(hashent->muxinfo.mark);
-
-    /* Clear the mark */
-    hashent->muxinfo.mark = 0;
-
-    /* Increment the counter */
-    ++n_active_circuits;
-
-    /* Advance to the next active circuit and update prev_circ */
-    prev_circ = curr_circ;
-    curr_circ = next_circ;
-  }
-
-  /* Assert that the counter matches the cmux */
-  tor_assert(n_active_circuits == cmux->n_active_circuits);
-}
-
-/**
- * Do the third pass of circuitmux_assert_okay(); see the comment in that
- * function.
- */
-
-static void
-circuitmux_assert_okay_pass_three(circuitmux_t *cmux)
-{
-  chanid_circid_muxinfo_t **i = NULL;
-
-  tor_assert(cmux);
-  tor_assert(cmux->chanid_circid_map);
-
-  /* Start iterating the hash table */
-  i = HT_START(chanid_circid_muxinfo_map, cmux->chanid_circid_map);
-
-  /* Advance through each entry */
-  while (i) {
-    /* Assert that it isn't null */
-    tor_assert(*i);
-
-    /*
-     * Assert that this entry is not marked - i.e., that either we didn't
-     * think it should be active in pass one or we saw it in the active
-     * circuits linked list.
-     */
-    tor_assert(!((*i)->muxinfo.mark));
-
-    /* Advance to the next entry */
-    i = HT_NEXT(chanid_circid_muxinfo_map, cmux->chanid_circid_map, i);
-  }
+            (cmux->destroy_ctr),
+            (global_destroy_ctr));
 }
 
 /*DOCDOC */
@@ -1888,10 +1262,10 @@ circuitmux_append_destroy_cell(channel_t *chan,
   ++global_destroy_ctr;
   log_debug(LD_CIRC,
             "Cmux at %p queued a destroy for circ %u, cmux counter is now "
-            I64_FORMAT", global counter is now "I64_FORMAT,
+            "%"PRId64", global counter is now %"PRId64,
             cmux, circ_id,
-            I64_PRINTF_ARG(cmux->destroy_ctr),
-            I64_PRINTF_ARG(global_destroy_ctr));
+            (cmux->destroy_ctr),
+            (global_destroy_ctr));
 
   /* XXXX Duplicate code from append_cell_to_circuit_queue */
   if (!channel_has_queued_writes(chan)) {
@@ -1929,12 +1303,12 @@ circuitmux_count_queued_destroy_cells(const channel_t *chan,
       n_destroy_cells != manual_total ||
       n_destroy_cells != manual_total_in_map) {
     log_warn(LD_BUG, "  Discrepancy in counts for queued destroy cells on "
-             "circuitmux. n="I64_FORMAT". queue_size="I64_FORMAT". "
-             "manual_total="I64_FORMAT". manual_total_in_map="I64_FORMAT".",
-             I64_PRINTF_ARG(n_destroy_cells),
-             I64_PRINTF_ARG(destroy_queue_size),
-             I64_PRINTF_ARG(manual_total),
-             I64_PRINTF_ARG(manual_total_in_map));
+             "circuitmux. n=%"PRId64". queue_size=%"PRId64". "
+             "manual_total=%"PRId64". manual_total_in_map=%"PRId64".",
+             (n_destroy_cells),
+             (destroy_queue_size),
+             (manual_total),
+             (manual_total_in_map));
   }
 
   return n_destroy_cells;
diff --git a/src/or/circuitmux.h b/src/core/or/circuitmux.h
index f53abdd8c8..67cd9bcdd8 100644
--- a/src/or/circuitmux.h
+++ b/src/core/or/circuitmux.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,8 +9,8 @@
 #ifndef TOR_CIRCUITMUX_H
 #define TOR_CIRCUITMUX_H
 
-#include "or.h"
-#include "testsupport.h"
+#include "core/or/or.h"
+#include "lib/testsupport/testsupport.h"
 
 typedef struct circuitmux_policy_s circuitmux_policy_t;
 typedef struct circuitmux_policy_data_s circuitmux_policy_data_t;
@@ -104,7 +104,9 @@ void circuitmux_assert_okay(circuitmux_t *cmux);
 circuitmux_t * circuitmux_alloc(void);
 void circuitmux_detach_all_circuits(circuitmux_t *cmux,
                                     smartlist_t *detached_out);
-void circuitmux_free(circuitmux_t *cmux);
+void circuitmux_free_(circuitmux_t *cmux);
+#define circuitmux_free(cmux) \
+  FREE_AND_NULL(circuitmux_t, circuitmux_free_, (cmux))
 
 /* Policy control */
 void circuitmux_clear_policy(circuitmux_t *cmux);
@@ -156,5 +158,5 @@ void circuitmux_mark_destroyed_circids_usable(circuitmux_t *cmux,
 MOCK_DECL(int, circuitmux_compare_muxes,
           (circuitmux_t *cmux_1, circuitmux_t *cmux_2));
 
-#endif /* TOR_CIRCUITMUX_H */
+#endif /* !defined(TOR_CIRCUITMUX_H) */
 
diff --git a/src/or/circuitmux_ewma.c b/src/core/or/circuitmux_ewma.c
index 5c2ebde73b..3f83c3fd5a 100644
--- a/src/or/circuitmux_ewma.c
+++ b/src/core/or/circuitmux_ewma.c
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -28,16 +28,18 @@
  *
  **/
 
-#define TOR_CIRCUITMUX_EWMA_C_
+#define CIRCUITMUX_EWMA_PRIVATE
 
 #include "orconfig.h"
 
 #include <math.h>
 
-#include "or.h"
-#include "circuitmux.h"
-#include "circuitmux_ewma.h"
-#include "networkstatus.h"
+#include "core/or/or.h"
+#include "core/or/circuitmux.h"
+#include "core/or/circuitmux_ewma.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "feature/nodelist/networkstatus.h"
+#include "app/config/or_options_st.h"
 
 /*** EWMA parameter #defines ***/
 
@@ -169,8 +171,6 @@ TO_EWMA_POL_CIRC_DATA(circuitmux_policy_circ_data_t *pol)
 
 static void add_cell_ewma(ewma_policy_data_t *pol, cell_ewma_t *ewma);
 static int compare_cell_ewma_counts(const void *p1, const void *p2);
-static unsigned cell_ewma_tick_from_timeval(const struct timeval *now,
-                                            double *remainder_out);
 static circuit_t * cell_ewma_to_circuit(cell_ewma_t *ewma);
 static inline double get_scale_factor(unsigned from_tick, unsigned to_tick);
 static cell_ewma_t * pop_first_cell_ewma(ewma_policy_data_t *pol);
@@ -223,8 +223,6 @@ ewma_cmp_cmux(circuitmux_t *cmux_1, circuitmux_policy_data_t *pol_data_1,
  * has value ewma_scale_factor ** N.)
  */
 static double ewma_scale_factor = 0.1;
-/* DOCDOC ewma_enabled */
-static int ewma_enabled = 0;
 
 /*** EWMA circuitmux_policy_t method table ***/
 
@@ -241,8 +239,26 @@ circuitmux_policy_t ewma_policy = {
   /*.cmp_cmux =*/ ewma_cmp_cmux
 };
 
+/** Have we initialized the ewma tick-counting logic? */
+static int ewma_ticks_initialized = 0;
+/** At what monotime_coarse_t did the current tick begin? */
+static monotime_coarse_t start_of_current_tick;
+/** What is the number of the current tick? */
+static unsigned current_tick_num;
+
 /*** EWMA method implementations using the below EWMA helper functions ***/
 
+/** Compute and return the current cell_ewma tick. */
+static inline unsigned int
+cell_ewma_get_tick(void)
+{
+  monotime_coarse_t now;
+  monotime_coarse_get(&now);
+  int32_t msec_diff = monotime_coarse_diff_msec32(&start_of_current_tick,
+                                                  &now);
+  return current_tick_num + msec_diff / (1000*EWMA_TICK_LEN);
+}
+
 /**
  * Allocate an ewma_policy_data_t and upcast it to a circuitmux_policy_data_t;
  * this is called when setting the policy on a circuitmux_t to ewma_policy.
@@ -416,8 +432,6 @@ ewma_notify_xmit_cells(circuitmux_t *cmux,
   ewma_policy_circ_data_t *cdata = NULL;
   unsigned int tick;
   double fractional_tick, ewma_increment;
-  /* The current (hi-res) time */
-  struct timeval now_hires;
   cell_ewma_t *cell_ewma, *tmp;
 
   tor_assert(cmux);
@@ -430,8 +444,7 @@ ewma_notify_xmit_cells(circuitmux_t *cmux,
   cdata = TO_EWMA_POL_CIRC_DATA(pol_circ_data);
 
   /* Rescale the EWMAs if needed */
-  tor_gettimeofday_cached(&now_hires);
-  tick = cell_ewma_tick_from_timeval(&now_hires, &fractional_tick);
+  tick = cell_ewma_get_current_tick_and_fraction(&fractional_tick);
 
   if (tick != pol->active_circuit_pqueue_last_recalibrated) {
     scale_active_circuits(pol, tick);
@@ -500,7 +513,7 @@ ewma_cmp_cmux(circuitmux_t *cmux_1, circuitmux_policy_data_t *pol_data_1,
   tor_assert(pol_data_2);
 
   p1 = TO_EWMA_POL_DATA(pol_data_1);
-  p2 = TO_EWMA_POL_DATA(pol_data_1);
+  p2 = TO_EWMA_POL_DATA(pol_data_2);
 
   if (p1 != p2) {
     /* Get the head cell_ewma_t from each queue */
@@ -592,79 +605,122 @@ cell_ewma_to_circuit(cell_ewma_t *ewma)
    rescale.
  */
 
-/** Given a timeval <b>now</b>, compute the cell_ewma tick in which it occurs
- * and the fraction of the tick that has elapsed between the start of the tick
- * and <b>now</b>.  Return the former and store the latter in
- * *<b>remainder_out</b>.
+/**
+ * Initialize the system that tells which ewma tick we are in.
+ */
+STATIC void
+cell_ewma_initialize_ticks(void)
+{
+  if (ewma_ticks_initialized)
+    return;
+  monotime_coarse_get(&start_of_current_tick);
+  crypto_rand((char*)&current_tick_num, sizeof(current_tick_num));
+  ewma_ticks_initialized = 1;
+}
+
+/** Compute the current cell_ewma tick and the fraction of the tick that has
+ * elapsed between the start of the tick and the current time.  Return the
+ * former and store the latter in *<b>remainder_out</b>.
  *
  * These tick values are not meant to be shared between Tor instances, or used
  * for other purposes. */
-
-static unsigned
-cell_ewma_tick_from_timeval(const struct timeval *now,
-                            double *remainder_out)
+STATIC unsigned
+cell_ewma_get_current_tick_and_fraction(double *remainder_out)
 {
-  unsigned res = (unsigned) (now->tv_sec / EWMA_TICK_LEN);
-  /* rem */
-  double rem = (now->tv_sec % EWMA_TICK_LEN) +
-    ((double)(now->tv_usec)) / 1.0e6;
-  *remainder_out = rem / EWMA_TICK_LEN;
-  return res;
+  if (BUG(!ewma_ticks_initialized)) {
+    cell_ewma_initialize_ticks(); // LCOV_EXCL_LINE
+  }
+  monotime_coarse_t now;
+  monotime_coarse_get(&now);
+  int32_t msec_diff = monotime_coarse_diff_msec32(&start_of_current_tick,
+                                                  &now);
+  if (msec_diff > (1000*EWMA_TICK_LEN)) {
+    unsigned ticks_difference = msec_diff / (1000*EWMA_TICK_LEN);
+    monotime_coarse_add_msec(&start_of_current_tick,
+                             &start_of_current_tick,
+                             ticks_difference * 1000 * EWMA_TICK_LEN);
+    current_tick_num += ticks_difference;
+    msec_diff %= 1000*EWMA_TICK_LEN;
+  }
+  *remainder_out = ((double)msec_diff) / (1.0e3 * EWMA_TICK_LEN);
+  return current_tick_num;
 }
 
-/** Tell the caller whether ewma_enabled is set */
-int
-cell_ewma_enabled(void)
+/* Default value for the CircuitPriorityHalflifeMsec consensus parameter in
+ * msec. */
+#define CMUX_PRIORITY_HALFLIFE_MSEC_DEFAULT 30000
+/* Minimum and maximum value for the CircuitPriorityHalflifeMsec consensus
+ * parameter. */
+#define CMUX_PRIORITY_HALFLIFE_MSEC_MIN 1
+#define CMUX_PRIORITY_HALFLIFE_MSEC_MAX INT32_MAX
+
+/* Return the value of the circuit priority halflife from the options if
+ * available or else from the consensus (in that order). If none can be found,
+ * a default value is returned.
+ *
+ * The source_msg points to a string describing from where the value was
+ * picked so it can be used for logging. */
+static double
+get_circuit_priority_halflife(const or_options_t *options,
+                              const networkstatus_t *consensus,
+                              const char **source_msg)
 {
-  return ewma_enabled;
-}
+  int32_t halflife_ms;
+  double halflife;
+  /* Compute the default value now. We might need it. */
+  double halflife_default =
+    ((double) CMUX_PRIORITY_HALFLIFE_MSEC_DEFAULT) / 1000.0;
 
-/** Compute and return the current cell_ewma tick. */
-unsigned int
-cell_ewma_get_tick(void)
-{
-  return ((unsigned)approx_time() / EWMA_TICK_LEN);
+  /* Try to get it from configuration file first. */
+  if (options && options->CircuitPriorityHalflife >= -EPSILON) {
+    halflife = options->CircuitPriorityHalflife;
+    *source_msg = "CircuitPriorityHalflife in configuration";
+    goto end;
+  }
+
+  /* Try to get the msec value from the consensus. */
+  halflife_ms = networkstatus_get_param(consensus,
+                                        "CircuitPriorityHalflifeMsec",
+                                        CMUX_PRIORITY_HALFLIFE_MSEC_DEFAULT,
+                                        CMUX_PRIORITY_HALFLIFE_MSEC_MIN,
+                                        CMUX_PRIORITY_HALFLIFE_MSEC_MAX);
+  halflife = ((double) halflife_ms) / 1000.0;
+  *source_msg = "CircuitPriorityHalflifeMsec in consensus";
+
+ end:
+  /* We should never go below the EPSILON else we would consider it disabled
+   * and we can't have that. */
+  if (halflife < EPSILON) {
+    log_warn(LD_CONFIG, "CircuitPriorityHalflife is too small (%f). "
+                        "Adjusting to the smallest value allowed: %f.",
+             halflife, halflife_default);
+    halflife = halflife_default;
+  }
+  return halflife;
 }
 
 /** Adjust the global cell scale factor based on <b>options</b> */
 void
-cell_ewma_set_scale_factor(const or_options_t *options,
-                           const networkstatus_t *consensus)
+cmux_ewma_set_options(const or_options_t *options,
+                      const networkstatus_t *consensus)
 {
-  int32_t halflife_ms;
   double halflife;
   const char *source;
-  if (options && options->CircuitPriorityHalflife >= -EPSILON) {
-    halflife = options->CircuitPriorityHalflife;
-    source = "CircuitPriorityHalflife in configuration";
-  } else if (consensus && (halflife_ms = networkstatus_get_param(
-                 consensus, "CircuitPriorityHalflifeMsec",
-                 -1, -1, INT32_MAX)) >= 0) {
-    halflife = ((double)halflife_ms)/1000.0;
-    source = "CircuitPriorityHalflifeMsec in consensus";
-  } else {
-    halflife = EWMA_DEFAULT_HALFLIFE;
-    source = "Default value";
-  }
 
-  if (halflife <= EPSILON) {
-    /* The cell EWMA algorithm is disabled. */
-    ewma_scale_factor = 0.1;
-    ewma_enabled = 0;
-    log_info(LD_OR,
-             "Disabled cell_ewma algorithm because of value in %s",
-             source);
-  } else {
-    /* convert halflife into halflife-per-tick. */
-    halflife /= EWMA_TICK_LEN;
-    /* compute per-tick scale factor. */
-    ewma_scale_factor = exp( LOG_ONEHALF / halflife );
-    ewma_enabled = 1;
-    log_info(LD_OR,
-             "Enabled cell_ewma algorithm because of value in %s; "
-             "scale factor is %f per %d seconds",
-             source, ewma_scale_factor, EWMA_TICK_LEN);
-  }
+  cell_ewma_initialize_ticks();
+
+  /* Both options and consensus can be NULL. This assures us to either get a
+   * valid configured value or the default one. */
+  halflife = get_circuit_priority_halflife(options, consensus, &source);
+
+  /* convert halflife into halflife-per-tick. */
+  halflife /= EWMA_TICK_LEN;
+  /* compute per-tick scale factor. */
+  ewma_scale_factor = exp( LOG_ONEHALF / halflife );
+  log_info(LD_OR,
+           "Enabled cell_ewma algorithm because of value in %s; "
+           "scale factor is %f per %d seconds",
+           source, ewma_scale_factor, EWMA_TICK_LEN);
 }
 
 /** Return the multiplier necessary to convert the value of a cell sent in
@@ -731,7 +787,7 @@ add_cell_ewma(ewma_policy_data_t *pol, cell_ewma_t *ewma)
 
   smartlist_pqueue_add(pol->active_circuit_pqueue,
                        compare_cell_ewma_counts,
-                       STRUCT_OFFSET(cell_ewma_t, heap_index),
+                       offsetof(cell_ewma_t, heap_index),
                        ewma);
 }
 
@@ -746,7 +802,7 @@ remove_cell_ewma(ewma_policy_data_t *pol, cell_ewma_t *ewma)
 
   smartlist_pqueue_remove(pol->active_circuit_pqueue,
                           compare_cell_ewma_counts,
-                          STRUCT_OFFSET(cell_ewma_t, heap_index),
+                          offsetof(cell_ewma_t, heap_index),
                           ewma);
 }
 
@@ -760,6 +816,14 @@ pop_first_cell_ewma(ewma_policy_data_t *pol)
 
   return smartlist_pqueue_pop(pol->active_circuit_pqueue,
                               compare_cell_ewma_counts,
-                              STRUCT_OFFSET(cell_ewma_t, heap_index));
+                              offsetof(cell_ewma_t, heap_index));
 }
 
+/**
+ * Drop all resources held by circuitmux_ewma.c, and deinitialize the
+ * module. */
+void
+circuitmux_ewma_free_all(void)
+{
+  ewma_ticks_initialized = 0;
+}
diff --git a/src/core/or/circuitmux_ewma.h b/src/core/or/circuitmux_ewma.h
new file mode 100644
index 0000000000..b45ce1f916
--- /dev/null
+++ b/src/core/or/circuitmux_ewma.h
@@ -0,0 +1,30 @@
+/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitmux_ewma.h
+ * \brief Header file for circuitmux_ewma.c
+ **/
+
+#ifndef TOR_CIRCUITMUX_EWMA_H
+#define TOR_CIRCUITMUX_EWMA_H
+
+#include "core/or/or.h"
+#include "core/or/circuitmux.h"
+
+/* The public EWMA policy callbacks object. */
+extern circuitmux_policy_t ewma_policy;
+
+/* Externally visible EWMA functions */
+void cmux_ewma_set_options(const or_options_t *options,
+                           const networkstatus_t *consensus);
+
+void circuitmux_ewma_free_all(void);
+
+#ifdef CIRCUITMUX_EWMA_PRIVATE
+STATIC unsigned cell_ewma_get_current_tick_and_fraction(double *remainder_out);
+STATIC void cell_ewma_initialize_ticks(void);
+#endif
+
+#endif /* !defined(TOR_CIRCUITMUX_EWMA_H) */
+
diff --git a/src/or/circuitstats.c b/src/core/or/circuitstats.c
index 735b3f0fc2..2cde21fa1f 100644
--- a/src/or/circuitstats.c
+++ b/src/core/or/circuitstats.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,17 +25,29 @@
 
 #define CIRCUITSTATS_PRIVATE
 
-#include "or.h"
-#include "circuitbuild.h"
-#include "circuitstats.h"
-#include "config.h"
-#include "confparse.h"
-#include "control.h"
-#include "main.h"
-#include "networkstatus.h"
-#include "rendclient.h"
-#include "rendservice.h"
-#include "statefile.h"
+#include "core/or/or.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitstats.h"
+#include "app/config/config.h"
+#include "app/config/confparse.h"
+#include "feature/control/control.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "core/mainloop/mainloop.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/rend/rendclient.h"
+#include "feature/rend/rendservice.h"
+#include "feature/relay/router.h"
+#include "app/config/statefile.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "lib/math/fp.h"
+#include "lib/time/tvdiff.h"
+#include "lib/encoding/confline.h"
+#include "feature/dirauth/authmode.h"
+
+#include "core/or/crypt_path_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "app/config/or_state_st.h"
 
 #undef log
 #include <math.h>
@@ -43,6 +55,7 @@
 static void cbt_control_event_buildtimeout_set(
                                   const circuit_build_times_t *cbt,
                                   buildtimeout_set_event_t type);
+static void circuit_build_times_scale_circ_counts(circuit_build_times_t *cbt);
 
 #define CBT_BIN_TO_MS(bin) ((bin)*CBT_BIN_WIDTH + (CBT_BIN_WIDTH/2))
 
@@ -60,7 +73,7 @@ static circuit_build_times_t circ_times;
 static int unit_tests = 0;
 #else
 #define unit_tests 0
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /** Return a pointer to the data structure describing our current circuit
  * build time history and computations. */
@@ -101,22 +114,29 @@ get_circuit_build_timeout_ms(void)
  *  2. If the torrc option LearnCircuitBuildTimeout is false.
  *  3. If we are a directory authority
  *  4. If we fail to write circuit build time history to our state file.
- *  5. If we are compiled or configured in Tor2web mode
- *  6. If we are configured in Single Onion mode
+ *  5. If we are configured in Single Onion mode
  */
 int
-circuit_build_times_disabled(void)
+circuit_build_times_disabled(const or_options_t *options)
+{
+  return circuit_build_times_disabled_(options, 0);
+}
+
+/** As circuit_build_times_disabled, but take options as an argument. */
+int
+circuit_build_times_disabled_(const or_options_t *options,
+                              int ignore_consensus)
 {
   if (unit_tests) {
     return 0;
   } else {
-    const or_options_t *options = get_options();
-    int consensus_disabled = networkstatus_get_param(NULL, "cbtdisabled",
+    int consensus_disabled =
+      ignore_consensus ? 0 : networkstatus_get_param(NULL, "cbtdisabled",
                                                      0, 0, 1);
     int config_disabled = !options->LearnCircuitBuildTimeout;
-    int dirauth_disabled = options->AuthoritativeDir;
+    int dirauth_disabled = authdir_mode(options);
     int state_disabled = did_last_state_file_write_fail() ? 1 : 0;
-    /* LearnCircuitBuildTimeout and Tor2web/Single Onion Services are
+    /* LearnCircuitBuildTimeout and Single Onion Services are
      * incompatible in two ways:
      *
      * - LearnCircuitBuildTimeout results in a low CBT, which
@@ -128,19 +148,18 @@ circuit_build_times_disabled(void)
      *
      * If we fix both of these issues someday, we should test
      * these modes with LearnCircuitBuildTimeout on again. */
-    int tor2web_disabled = rend_client_allow_non_anonymous_connection(options);
     int single_onion_disabled = rend_service_allow_non_anonymous_connection(
                                                                       options);
 
     if (consensus_disabled || config_disabled || dirauth_disabled ||
-        state_disabled || tor2web_disabled || single_onion_disabled) {
+        state_disabled || single_onion_disabled) {
 #if 0
       log_debug(LD_CIRC,
                "CircuitBuildTime learning is disabled. "
                "Consensus=%d, Config=%d, AuthDir=%d, StateFile=%d",
                consensus_disabled, config_disabled, dirauth_disabled,
                state_disabled);
-#endif
+#endif /* 0 */
       return 1;
     } else {
 #if 0
@@ -149,14 +168,14 @@ circuit_build_times_disabled(void)
                 "Consensus=%d, Config=%d, AuthDir=%d, StateFile=%d",
                 consensus_disabled, config_disabled, dirauth_disabled,
                 state_disabled);
-#endif
+#endif /* 0 */
       return 0;
     }
   }
 }
 
 /**
- * Retrieve and bounds-check the cbtmaxtimeouts consensus paramter.
+ * Retrieve and bounds-check the cbtmaxtimeouts consensus parameter.
  *
  * Effect: When this many timeouts happen in the last 'cbtrecentcount'
  * circuit attempts, the client should discard all of its history and
@@ -183,7 +202,7 @@ circuit_build_times_max_timeouts(void)
 }
 
 /**
- * Retrieve and bounds-check the cbtnummodes consensus paramter.
+ * Retrieve and bounds-check the cbtnummodes consensus parameter.
  *
  * Effect: This value governs how many modes to use in the weighted
  * average calculation of Pareto parameter Xm. A value of 3 introduces
@@ -210,7 +229,7 @@ circuit_build_times_default_num_xm_modes(void)
 }
 
 /**
- * Retrieve and bounds-check the cbtmincircs consensus paramter.
+ * Retrieve and bounds-check the cbtmincircs consensus parameter.
  *
  * Effect: This is the minimum number of circuits to build before
  * computing a timeout.
@@ -242,7 +261,7 @@ circuit_build_times_enough_to_compute(const circuit_build_times_t *cbt)
 }
 
 /**
- * Retrieve and bounds-check the cbtquantile consensus paramter.
+ * Retrieve and bounds-check the cbtquantile consensus parameter.
  *
  * Effect: This is the position on the quantile curve to use to set the
  * timeout value. It is a percent (10-99).
@@ -266,7 +285,7 @@ circuit_build_times_quantile_cutoff(void)
 }
 
 /**
- * Retrieve and bounds-check the cbtclosequantile consensus paramter.
+ * Retrieve and bounds-check the cbtclosequantile consensus parameter.
  *
  * Effect: This is the position on the quantile curve to use to set the
  * timeout value to use to actually close circuits. It is a percent
@@ -298,7 +317,7 @@ circuit_build_times_close_quantile(void)
 }
 
 /**
- * Retrieve and bounds-check the cbttestfreq consensus paramter.
+ * Retrieve and bounds-check the cbttestfreq consensus parameter.
  *
  * Effect: Describes how often in seconds to build a test circuit to
  * gather timeout values. Only applies if less than 'cbtmincircs'
@@ -345,7 +364,7 @@ circuit_build_times_min_timeout(void)
 }
 
 /**
- * Retrieve and bounds-check the cbtinitialtimeout consensus paramter.
+ * Retrieve and bounds-check the cbtinitialtimeout consensus parameter.
  *
  * Effect: This is the timeout value to use before computing a timeout,
  * in milliseconds.
@@ -375,7 +394,7 @@ circuit_build_times_initial_timeout(void)
 }
 
 /**
- * Retrieve and bounds-check the cbtrecentcount consensus paramter.
+ * Retrieve and bounds-check the cbtrecentcount consensus parameter.
  *
  * Effect: This is the number of circuit build times to keep track of
  * for deciding if we hit cbtmaxtimeouts and need to reset our state
@@ -417,15 +436,20 @@ circuit_build_times_new_consensus_params(circuit_build_times_t *cbt,
    * update if we aren't.
    */
 
-  if (!circuit_build_times_disabled()) {
+  if (!circuit_build_times_disabled(get_options())) {
     num = circuit_build_times_recent_circuit_count(ns);
 
     if (num > 0) {
       if (num != cbt->liveness.num_recent_circs) {
         int8_t *recent_circs;
-        log_notice(LD_CIRC, "The Tor Directory Consensus has changed how many "
-                   "circuits we must track to detect network failures from %d "
-                   "to %d.", cbt->liveness.num_recent_circs, num);
+        if (cbt->liveness.num_recent_circs > 0) {
+          log_notice(LD_CIRC, "The Tor Directory Consensus has changed how "
+                     "many circuits we must track to detect network failures "
+                     "from %d to %d.", cbt->liveness.num_recent_circs, num);
+        } else {
+          log_notice(LD_CIRC, "Upon receiving a consensus directory, "
+                     "re-enabling circuit-based network failure detection.");
+        }
 
         tor_assert(cbt->liveness.timeouts_after_firsthop ||
                    cbt->liveness.num_recent_circs == 0);
@@ -493,14 +517,15 @@ static double
 circuit_build_times_get_initial_timeout(void)
 {
   double timeout;
+  const or_options_t *options = get_options();
 
   /*
    * Check if we have LearnCircuitBuildTimeout, and if we don't,
    * always use CircuitBuildTimeout, no questions asked.
    */
-  if (!unit_tests && get_options()->CircuitBuildTimeout) {
-    timeout = get_options()->CircuitBuildTimeout*1000;
-    if (!circuit_build_times_disabled() &&
+  if (!unit_tests && options->CircuitBuildTimeout) {
+    timeout = options->CircuitBuildTimeout*1000;
+    if (!circuit_build_times_disabled(options) &&
         timeout < circuit_build_times_min_timeout()) {
       log_warn(LD_CIRC, "Config CircuitBuildTimeout too low. Setting to %ds",
                circuit_build_times_min_timeout()/1000);
@@ -526,6 +551,11 @@ circuit_build_times_reset(circuit_build_times_t *cbt)
   cbt->total_build_times = 0;
   cbt->build_times_idx = 0;
   cbt->have_computed_timeout = 0;
+
+  // Reset timeout and close counts
+  cbt->num_circ_succeeded = 0;
+  cbt->num_circ_closed = 0;
+  cbt->num_circ_timeouts = 0;
 }
 
 /**
@@ -542,7 +572,7 @@ circuit_build_times_init(circuit_build_times_t *cbt)
    * Check if we really are using adaptive timeouts, and don't keep
    * track of this stuff if not.
    */
-  if (!circuit_build_times_disabled()) {
+  if (!circuit_build_times_disabled(get_options())) {
     cbt->liveness.num_recent_circs =
       circuit_build_times_recent_circuit_count(NULL);
     cbt->liveness.timeouts_after_firsthop =
@@ -599,7 +629,124 @@ circuit_build_times_rewind_history(circuit_build_times_t *cbt, int n)
           "Rewound history by %d places. Current index: %d. "
           "Total: %d", n, cbt->build_times_idx, cbt->total_build_times);
 }
-#endif
+#endif /* 0 */
+
+/**
+ * Mark this circuit as timed out, but change its purpose
+ * so that it continues to build, allowing us to measure
+ * its full build time.
+ */
+void
+circuit_build_times_mark_circ_as_measurement_only(origin_circuit_t *circ)
+{
+  control_event_circuit_status(circ,
+                               CIRC_EVENT_FAILED,
+                               END_CIRC_REASON_TIMEOUT);
+  circuit_change_purpose(TO_CIRCUIT(circ),
+                         CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT);
+  /* Record this event to check for too many timeouts
+   * in a row. This function does not record a time value yet
+   * (we do that later); it only counts the fact that we did
+   * have a timeout. We also want to avoid double-counting
+   * already "relaxed" circuits, which are counted in
+   * circuit_expire_building(). */
+  if (!circ->relaxed_timeout) {
+    int first_hop_succeeded = circ->cpath &&
+          circ->cpath->state == CPATH_STATE_OPEN;
+
+    circuit_build_times_count_timeout(
+                                 get_circuit_build_times_mutable(),
+                                 first_hop_succeeded);
+  }
+}
+
+/**
+ * Perform the build time work that needs to be done when a circuit
+ * completes a hop.
+ *
+ * This function decides if we should record a circuit's build time
+ * in our histogram data and other statistics, and if so, records it.
+ * It also will mark circuits that have already timed out as
+ * measurement-only circuits, so they can continue to build but
+ * not get used.
+ *
+ * For this, we want to consider circuits that will eventually make
+ * it to the third hop. For circuits longer than 3 hops, we want to
+ * record their build time when they reach the third hop, but let
+ * them continue (and not count them later). For circuits that are
+ * exactly 3 hops, this will count them when they are completed. We
+ * do this so that CBT is always gathering statistics on circuits
+ * of the same length, regardless of their type.
+ */
+void
+circuit_build_times_handle_completed_hop(origin_circuit_t *circ)
+{
+  struct timeval end;
+  long timediff;
+
+  /* If circuit build times are disabled, let circuit_expire_building()
+   * handle it.. */
+  if (circuit_build_times_disabled(get_options())) {
+    return;
+  }
+
+  /* Is this a circuit for which the timeout applies in a straight-forward
+   * way? If so, handle it below. If not, just return (and let
+   * circuit_expire_building() eventually take care of it).
+   */
+  if (!circuit_timeout_want_to_count_circ(circ)) {
+    return;
+  }
+
+  tor_gettimeofday(&end);
+  timediff = tv_mdiff(&circ->base_.timestamp_began, &end);
+
+  /* Check if we would have timed out already. If so, change the
+   * purpose here. But don't do any timeout handling here if there
+   * are no circuits opened yet. Save it for circuit_expire_building()
+   * (to allow it to handle timeout "relaxing" over there). */
+  if (timediff > get_circuit_build_timeout_ms() &&
+      circuit_any_opened_circuits_cached()) {
+
+    /* Circuits are allowed to last longer for measurement.
+     * Switch their purpose and wait. */
+    if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
+      log_info(LD_CIRC,
+               "Deciding to timeout circuit %"PRIu32"\n",
+               (circ->global_identifier));
+      circuit_build_times_mark_circ_as_measurement_only(circ);
+    }
+  }
+
+  /* If the circuit is built to exactly the DEFAULT_ROUTE_LEN,
+   * add it to our buildtimes. */
+  if (circuit_get_cpath_opened_len(circ) == DEFAULT_ROUTE_LEN) {
+    /* If the circuit build time is much greater than we would have cut
+     * it off at, we probably had a suspend event along this codepath,
+     * and we should discard the value.
+     */
+    if (timediff < 0 ||
+        timediff > 2*get_circuit_build_close_time_ms()+1000) {
+      log_notice(LD_CIRC, "Strange value for circuit build time: %ldmsec. "
+                 "Assuming clock jump. Purpose %d (%s)", timediff,
+                 circ->base_.purpose,
+                 circuit_purpose_to_string(circ->base_.purpose));
+    } else {
+      /* Only count circuit times if the network is live */
+      if (circuit_build_times_network_check_live(
+                            get_circuit_build_times())) {
+        circuit_build_times_add_time(get_circuit_build_times_mutable(),
+                                     (build_time_t)timediff);
+        circuit_build_times_set_timeout(get_circuit_build_times_mutable());
+      }
+
+      if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
+        circuit_build_times_network_circ_success(
+                                      get_circuit_build_times_mutable());
+      }
+    }
+  }
+}
 
 /**
  * Add a new build time value <b>time</b> to the set of build times. Time
@@ -667,7 +814,7 @@ circuit_build_times_min(circuit_build_times_t *cbt)
   }
   return min_build_time;
 }
-#endif
+#endif /* 0 */
 
 /**
  * Calculate and return a histogram for the set of build times.
@@ -913,12 +1060,12 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt,
   int tot_values = 0;
   uint32_t loaded_cnt = 0, N = 0;
   config_line_t *line;
-  unsigned int i;
+  int i;
   build_time_t *loaded_times;
   int err = 0;
   circuit_build_times_init(cbt);
 
-  if (circuit_build_times_disabled()) {
+  if (circuit_build_times_disabled(get_options())) {
     return 0;
   }
 
@@ -942,7 +1089,7 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt,
       uint32_t count, k;
       build_time_t ms;
       int ok;
-      ms = (build_time_t)tor_parse_ulong(ms_str, 0, 0,
+      ms = (build_time_t)tor_parse_ulong(ms_str, 10, 0,
                                          CBT_BUILD_TIME_MAX, &ok, NULL);
       if (!ok) {
         log_warn(LD_GENERAL, "Unable to parse circuit build times: "
@@ -952,7 +1099,7 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt,
         smartlist_free(args);
         break;
       }
-      count = (uint32_t)tor_parse_ulong(count_str, 0, 0,
+      count = (uint32_t)tor_parse_ulong(count_str, 10, 0,
                                         UINT32_MAX, &ok, NULL);
       if (!ok) {
         log_warn(LD_GENERAL, "Unable to parse circuit build times: "
@@ -963,8 +1110,8 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt,
         break;
       }
 
-      if (loaded_cnt+count+state->CircuitBuildAbandonedCount
-            > state->TotalBuildTimes) {
+      if (loaded_cnt+count+ (unsigned)state->CircuitBuildAbandonedCount
+          > (unsigned) state->TotalBuildTimes) {
         log_warn(LD_CIRC,
                  "Too many build times in state file. "
                  "Stopping short before %d",
@@ -989,7 +1136,7 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt,
     loaded_times[loaded_cnt++] = CBT_BUILD_ABANDONED;
   }
 
-  if (loaded_cnt != state->TotalBuildTimes) {
+  if (loaded_cnt != (unsigned)state->TotalBuildTimes) {
     log_warn(LD_CIRC,
             "Corrupt state file? Build times count mismatch. "
             "Read %d times, but file says %d", loaded_cnt,
@@ -1172,7 +1319,7 @@ circuit_build_times_cdf(circuit_build_times_t *cbt, double x)
   tor_assert(0 <= ret && ret <= 1.0);
   return ret;
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 #ifdef TOR_UNIT_TESTS
 /**
@@ -1207,7 +1354,7 @@ circuit_build_times_generate_sample(circuit_build_times_t *cbt,
   tor_assert(ret > 0);
   return ret;
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 #ifdef TOR_UNIT_TESTS
 /**
@@ -1230,7 +1377,7 @@ circuit_build_times_initial_alpha(circuit_build_times_t *cbt,
     (tor_mathlog(cbt->Xm)-tor_mathlog(timeout_ms));
   tor_assert(cbt->alpha > 0);
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /**
  * Returns true if we need circuits to be built
@@ -1292,9 +1439,32 @@ circuit_build_times_network_is_live(circuit_build_times_t *cbt)
 }
 
 /**
- * Called to indicate that we completed a circuit. Because this circuit
+ * Non-destructively scale all of our circuit success, timeout, and close
+ * counts down by a factor of two. Scaling in this way preserves the
+ * ratios between succeeded vs timed out vs closed circuits, so that
+ * our statistics don't change when we scale.
+ *
+ * This is used only in the rare event that we build more than
+ * INT32_MAX circuits.  Since the num_circ_* variables are
+ * uint32_t, we won't even be close to overflowing them.
+ */
+void
+circuit_build_times_scale_circ_counts(circuit_build_times_t *cbt)
+{
+  cbt->num_circ_succeeded /= 2;
+  cbt->num_circ_timeouts /= 2;
+  cbt->num_circ_closed /= 2;
+}
+
+/**
+ * Called to indicate that we "completed" a circuit. Because this circuit
  * succeeded, it doesn't count as a timeout-after-the-first-hop.
  *
+ * (For the purposes of the cbt code, we consider a circuit "completed" if
+ * it has 3 hops, regardless of its final hop count. We do this because
+ * we're trying to answer the question, "how long should a circuit take to
+ * reach the 3-hop count".)
+ *
  * This is used by circuit_build_times_network_check_changed() to determine
  * if we had too many recent timeouts and need to reset our learned timeout
  * to something higher.
@@ -1302,6 +1472,14 @@ circuit_build_times_network_is_live(circuit_build_times_t *cbt)
 void
 circuit_build_times_network_circ_success(circuit_build_times_t *cbt)
 {
+  // Count circuit success
+  cbt->num_circ_succeeded++;
+
+  // If we're going to wrap int32, scale everything
+  if (cbt->num_circ_succeeded >= INT32_MAX) {
+    circuit_build_times_scale_circ_counts(cbt);
+  }
+
   /* Check for NULLness because we might not be using adaptive timeouts */
   if (cbt->liveness.timeouts_after_firsthop &&
       cbt->liveness.num_recent_circs > 0) {
@@ -1324,6 +1502,14 @@ static void
 circuit_build_times_network_timeout(circuit_build_times_t *cbt,
                                     int did_onehop)
 {
+  // Count circuit timeout
+  cbt->num_circ_timeouts++;
+
+  // If we're going to wrap int32, scale everything
+  if (cbt->num_circ_timeouts >= INT32_MAX) {
+    circuit_build_times_scale_circ_counts(cbt);
+  }
+
   /* Check for NULLness because we might not be using adaptive timeouts */
   if (cbt->liveness.timeouts_after_firsthop &&
       cbt->liveness.num_recent_circs > 0) {
@@ -1349,6 +1535,15 @@ circuit_build_times_network_close(circuit_build_times_t *cbt,
                                     int did_onehop, time_t start_time)
 {
   time_t now = time(NULL);
+
+  // Count circuit close
+  cbt->num_circ_closed++;
+
+  // If we're going to wrap int32, scale everything
+  if (cbt->num_circ_closed >= INT32_MAX) {
+    circuit_build_times_scale_circ_counts(cbt);
+  }
+
   /*
    * Check if this is a timeout that was for a circuit that spent its
    * entire existence during a time where we have had no network activity.
@@ -1447,7 +1642,7 @@ circuit_build_times_network_check_changed(circuit_build_times_t *cbt)
 
 #define MAX_TIMEOUT ((int32_t) (INT32_MAX/2))
   /* Check to see if this has happened before. If so, double the timeout
-   * to give people on abysmally bad network connections a shot at access */
+   * to give clients on abysmally bad network connections a shot at access */
   if (cbt->timeout_ms >= circuit_build_times_get_initial_timeout()) {
     if (cbt->timeout_ms > MAX_TIMEOUT || cbt->close_ms > MAX_TIMEOUT) {
       log_warn(LD_CIRC, "Insanely large circuit build timeout value. "
@@ -1523,7 +1718,7 @@ circuit_build_times_count_close(circuit_build_times_t *cbt,
                                 int did_onehop,
                                 time_t start_time)
 {
-  if (circuit_build_times_disabled()) {
+  if (circuit_build_times_disabled(get_options())) {
     cbt->close_ms = cbt->timeout_ms
                   = circuit_build_times_get_initial_timeout();
     return 0;
@@ -1554,7 +1749,7 @@ void
 circuit_build_times_count_timeout(circuit_build_times_t *cbt,
                                   int did_onehop)
 {
-  if (circuit_build_times_disabled()) {
+  if (circuit_build_times_disabled(get_options())) {
     cbt->close_ms = cbt->timeout_ms
                   = circuit_build_times_get_initial_timeout();
     return;
@@ -1628,7 +1823,7 @@ circuit_build_times_set_timeout(circuit_build_times_t *cbt)
   /*
    * Just return if we aren't using adaptive timeouts
    */
-  if (circuit_build_times_disabled())
+  if (circuit_build_times_disabled(get_options()))
     return;
 
   if (!circuit_build_times_set_timeout_worker(cbt))
@@ -1689,7 +1884,7 @@ circuitbuild_running_unit_tests(void)
 {
   unit_tests = 1;
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 void
 circuit_build_times_update_last_circ(circuit_build_times_t *cbt)
@@ -1703,6 +1898,8 @@ cbt_control_event_buildtimeout_set(const circuit_build_times_t *cbt,
 {
   char *args = NULL;
   double qnt;
+  double timeout_rate = 0.0;
+  double close_rate = 0.0;
 
   switch (type) {
     case BUILDTIMEOUT_SET_EVENT_RESET:
@@ -1717,18 +1914,38 @@ cbt_control_event_buildtimeout_set(const circuit_build_times_t *cbt,
       break;
   }
 
+  /* The timeout rate is the ratio of the timeout count over
+   * the total number of circuits attempted. The total number of
+   * circuits is (timeouts+succeeded), since every circuit
+   * either succeeds, or times out. "Closed" circuits are
+   * MEASURE_TIMEOUT circuits whose measurement period expired.
+   * All MEASURE_TIMEOUT circuits are counted in the timeouts stat
+   * before transitioning to MEASURE_TIMEOUT (in
+   * circuit_build_times_mark_circ_as_measurement_only()).
+   * MEASURE_TIMEOUT circuits that succeed are *not* counted as
+   * "succeeded". See circuit_build_times_handle_completed_hop().
+   *
+   * We cast the denominator
+   * to promote it to double before the addition, to avoid int32
+   * overflow. */
+  const double total_circuits =
+    ((double)cbt->num_circ_timeouts) + cbt->num_circ_succeeded;
+  if (total_circuits >= 1.0) {
+    timeout_rate = cbt->num_circ_timeouts / total_circuits;
+    close_rate = cbt->num_circ_closed / total_circuits;
+  }
+
   tor_asprintf(&args, "TOTAL_TIMES=%lu "
                "TIMEOUT_MS=%lu XM=%lu ALPHA=%f CUTOFF_QUANTILE=%f "
                "TIMEOUT_RATE=%f CLOSE_MS=%lu CLOSE_RATE=%f",
                (unsigned long)cbt->total_build_times,
                (unsigned long)cbt->timeout_ms,
                (unsigned long)cbt->Xm, cbt->alpha, qnt,
-               circuit_build_times_timeout_rate(cbt),
+               timeout_rate,
                (unsigned long)cbt->close_ms,
-               circuit_build_times_close_rate(cbt));
+               close_rate);
 
   control_event_buildtimeout_set(type, args);
 
   tor_free(args);
 }
-
diff --git a/src/core/or/circuitstats.h b/src/core/or/circuitstats.h
new file mode 100644
index 0000000000..845d7b6722
--- /dev/null
+++ b/src/core/or/circuitstats.h
@@ -0,0 +1,213 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitstats.h
+ * \brief Header file for circuitstats.c
+ **/
+
+#ifndef TOR_CIRCUITSTATS_H
+#define TOR_CIRCUITSTATS_H
+
+const circuit_build_times_t *get_circuit_build_times(void);
+circuit_build_times_t *get_circuit_build_times_mutable(void);
+double get_circuit_build_close_time_ms(void);
+double get_circuit_build_timeout_ms(void);
+
+int circuit_build_times_disabled(const or_options_t *options);
+int circuit_build_times_disabled_(const or_options_t *options,
+                                  int ignore_consensus);
+
+/** A build_time_t is milliseconds */
+typedef uint32_t build_time_t;
+
+int circuit_build_times_enough_to_compute(const circuit_build_times_t *cbt);
+void circuit_build_times_update_state(const circuit_build_times_t *cbt,
+                                      or_state_t *state);
+int circuit_build_times_parse_state(circuit_build_times_t *cbt,
+                                    or_state_t *state);
+void circuit_build_times_count_timeout(circuit_build_times_t *cbt,
+                                       int did_onehop);
+int circuit_build_times_count_close(circuit_build_times_t *cbt,
+                                    int did_onehop, time_t start_time);
+void circuit_build_times_set_timeout(circuit_build_times_t *cbt);
+int circuit_build_times_add_time(circuit_build_times_t *cbt,
+                                 build_time_t time);
+int circuit_build_times_needs_circuits(const circuit_build_times_t *cbt);
+void circuit_build_times_handle_completed_hop(origin_circuit_t *circ);
+
+int circuit_build_times_needs_circuits_now(const circuit_build_times_t *cbt);
+void circuit_build_times_init(circuit_build_times_t *cbt);
+void circuit_build_times_free_timeouts(circuit_build_times_t *cbt);
+void circuit_build_times_new_consensus_params(circuit_build_times_t *cbt,
+                                              networkstatus_t *ns);
+double circuit_build_times_timeout_rate(const circuit_build_times_t *cbt);
+double circuit_build_times_close_rate(const circuit_build_times_t *cbt);
+
+void circuit_build_times_update_last_circ(circuit_build_times_t *cbt);
+void circuit_build_times_mark_circ_as_measurement_only(origin_circuit_t *circ);
+
+/** Total size of the circuit timeout history to accumulate.
+ * 1000 is approx 2.5 days worth of continual-use circuits. */
+#define CBT_NCIRCUITS_TO_OBSERVE 1000
+
+/** Width of the histogram bins in milliseconds */
+#define CBT_BIN_WIDTH ((build_time_t)50)
+
+/** Number of modes to use in the weighted-avg computation of Xm */
+#define CBT_DEFAULT_NUM_XM_MODES 3
+#define CBT_MIN_NUM_XM_MODES 1
+#define CBT_MAX_NUM_XM_MODES 20
+
+/**
+ * CBT_BUILD_ABANDONED is our flag value to represent a force-closed
+ * circuit (Aka a 'right-censored' pareto value).
+ */
+#define CBT_BUILD_ABANDONED ((build_time_t)(INT32_MAX-1))
+#define CBT_BUILD_TIME_MAX ((build_time_t)(INT32_MAX))
+
+/** Save state every 10 circuits */
+#define CBT_SAVE_STATE_EVERY 10
+
+/* Circuit build times consensus parameters */
+
+/**
+ * How long to wait before actually closing circuits that take too long to
+ * build in terms of CDF quantile.
+ */
+#define CBT_DEFAULT_CLOSE_QUANTILE 95
+#define CBT_MIN_CLOSE_QUANTILE CBT_MIN_QUANTILE_CUTOFF
+#define CBT_MAX_CLOSE_QUANTILE CBT_MAX_QUANTILE_CUTOFF
+
+/**
+ * How many circuits count as recent when considering if the
+ * connection has gone gimpy or changed.
+ */
+#define CBT_DEFAULT_RECENT_CIRCUITS 20
+#define CBT_MIN_RECENT_CIRCUITS 3
+#define CBT_MAX_RECENT_CIRCUITS 1000
+
+/**
+ * Maximum count of timeouts that finish the first hop in the past
+ * RECENT_CIRCUITS before calculating a new timeout.
+ *
+ * This tells us whether to abandon timeout history and set
+ * the timeout back to whatever circuit_build_times_get_initial_timeout()
+ * gives us.
+ */
+#define CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT (CBT_DEFAULT_RECENT_CIRCUITS*9/10)
+#define CBT_MIN_MAX_RECENT_TIMEOUT_COUNT 3
+#define CBT_MAX_MAX_RECENT_TIMEOUT_COUNT 10000
+
+/** Minimum circuits before estimating a timeout */
+#define CBT_DEFAULT_MIN_CIRCUITS_TO_OBSERVE 100
+#define CBT_MIN_MIN_CIRCUITS_TO_OBSERVE 1
+#define CBT_MAX_MIN_CIRCUITS_TO_OBSERVE 10000
+
+/** Cutoff percentile on the CDF for our timeout estimation. */
+#define CBT_DEFAULT_QUANTILE_CUTOFF 80
+#define CBT_MIN_QUANTILE_CUTOFF 10
+#define CBT_MAX_QUANTILE_CUTOFF 99
+double circuit_build_times_quantile_cutoff(void);
+
+/** How often in seconds should we build a test circuit */
+#define CBT_DEFAULT_TEST_FREQUENCY 10
+#define CBT_MIN_TEST_FREQUENCY 1
+#define CBT_MAX_TEST_FREQUENCY INT32_MAX
+
+/** Lowest allowable value for CircuitBuildTimeout in milliseconds */
+#define CBT_DEFAULT_TIMEOUT_MIN_VALUE (1500)
+#define CBT_MIN_TIMEOUT_MIN_VALUE 500
+#define CBT_MAX_TIMEOUT_MIN_VALUE INT32_MAX
+
+/** Initial circuit build timeout in milliseconds */
+#define CBT_DEFAULT_TIMEOUT_INITIAL_VALUE (60*1000)
+#define CBT_MIN_TIMEOUT_INITIAL_VALUE CBT_MIN_TIMEOUT_MIN_VALUE
+#define CBT_MAX_TIMEOUT_INITIAL_VALUE INT32_MAX
+int32_t circuit_build_times_initial_timeout(void);
+
+#if CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT < CBT_MIN_MAX_RECENT_TIMEOUT_COUNT
+#error "RECENT_CIRCUITS is set too low."
+#endif
+
+#ifdef CIRCUITSTATS_PRIVATE
+STATIC double circuit_build_times_calculate_timeout(circuit_build_times_t *cbt,
+                                             double quantile);
+STATIC int circuit_build_times_update_alpha(circuit_build_times_t *cbt);
+STATIC void circuit_build_times_reset(circuit_build_times_t *cbt);
+
+/* Network liveness functions */
+STATIC int circuit_build_times_network_check_changed(
+                                             circuit_build_times_t *cbt);
+#endif /* defined(CIRCUITSTATS_PRIVATE) */
+
+#ifdef TOR_UNIT_TESTS
+build_time_t circuit_build_times_generate_sample(circuit_build_times_t *cbt,
+                                                 double q_lo, double q_hi);
+double circuit_build_times_cdf(circuit_build_times_t *cbt, double x);
+void circuit_build_times_initial_alpha(circuit_build_times_t *cbt,
+                                       double quantile, double time_ms);
+void circuitbuild_running_unit_tests(void);
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/* Network liveness functions */
+void circuit_build_times_network_is_live(circuit_build_times_t *cbt);
+int circuit_build_times_network_check_live(const circuit_build_times_t *cbt);
+void circuit_build_times_network_circ_success(circuit_build_times_t *cbt);
+
+#ifdef CIRCUITSTATS_PRIVATE
+/** Information about the state of our local network connection */
+typedef struct {
+  /** The timestamp we last completed a TLS handshake or received a cell */
+  time_t network_last_live;
+  /** If the network is not live, how many timeouts has this caused? */
+  int nonlive_timeouts;
+  /** Circular array of circuits that have made it to the first hop. Slot is
+   * 1 if circuit timed out, 0 if circuit succeeded */
+  int8_t *timeouts_after_firsthop;
+  /** Number of elements allocated for the above array */
+  int num_recent_circs;
+  /** Index into circular array. */
+  int after_firsthop_idx;
+} network_liveness_t;
+
+/** Structure for circuit build times history */
+struct circuit_build_times_s {
+  /** The circular array of recorded build times in milliseconds */
+  build_time_t circuit_build_times[CBT_NCIRCUITS_TO_OBSERVE];
+  /** Current index in the circuit_build_times circular array */
+  int build_times_idx;
+  /** Total number of build times accumulated. Max CBT_NCIRCUITS_TO_OBSERVE */
+  int total_build_times;
+  /** Information about the state of our local network connection */
+  network_liveness_t liveness;
+  /** Last time we built a circuit. Used to decide to build new test circs */
+  time_t last_circ_at;
+  /** "Minimum" value of our pareto distribution (actually mode) */
+  build_time_t Xm;
+  /** alpha exponent for pareto dist. */
+  double alpha;
+  /** Have we computed a timeout? */
+  int have_computed_timeout;
+  /** The exact value for that timeout in milliseconds. Stored as a double
+   * to maintain precision from calculations to and from quantile value. */
+  double timeout_ms;
+  /** How long we wait before actually closing the circuit. */
+  double close_ms;
+  /** Total succeeded counts. Old measurements may be scaled downward if
+   * we've seen a lot of circuits. */
+  uint32_t num_circ_succeeded;
+  /** Total timeout counts.  Old measurements may be scaled downward if
+   * we've seen a lot of circuits. */
+  uint32_t num_circ_timeouts;
+  /** Total closed counts.  Old measurements may be scaled downward if
+   * we've seen a lot of circuits.*/
+  uint32_t num_circ_closed;
+
+};
+#endif /* defined(CIRCUITSTATS_PRIVATE) */
+
+#endif /* !defined(TOR_CIRCUITSTATS_H) */
diff --git a/src/or/circuituse.c b/src/core/or/circuituse.c
index 03b54fc3d5..000a7c36da 100644
--- a/src/or/circuituse.c
+++ b/src/core/or/circuituse.c
@@ -1,40 +1,107 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file circuituse.c
- * \brief Launch the right sort of circuits and attach streams to them.
+ * \brief Launch the right sort of circuits and attach the right streams to
+ * them.
+ *
+ * As distinct from circuitlist.c, which manages lookups to find circuits, and
+ * circuitbuild.c, which handles the logistics of circuit construction, this
+ * module keeps track of which streams can be attached to which circuits (in
+ * circuit_get_best()), and attaches streams to circuits (with
+ * circuit_try_attaching_streams(), connection_ap_handshake_attach_circuit(),
+ * and connection_ap_handshake_attach_chosen_circuit() ).
+ *
+ * This module also makes sure that we are building circuits for all of the
+ * predicted ports, using circuit_remove_handled_ports(),
+ * circuit_stream_is_being_handled(), and circuit_build_needed_cirs().  It
+ * handles launching circuits for specific targets using
+ * circuit_launch_by_extend_info().
+ *
+ * This is also where we handle expiring circuits that have been around for
+ * too long without actually completing, along with the circuit_build_timeout
+ * logic in circuitstats.c.
  **/
 
-#include "or.h"
-#include "addressmap.h"
-#include "channel.h"
-#include "circpathbias.h"
-#include "circuitbuild.h"
-#include "circuitlist.h"
-#include "circuitstats.h"
-#include "circuituse.h"
-#include "config.h"
-#include "connection.h"
-#include "connection_edge.h"
-#include "control.h"
-#include "entrynodes.h"
-#include "nodelist.h"
-#include "networkstatus.h"
-#include "policies.h"
-#include "rendclient.h"
-#include "rendcommon.h"
-#include "rendservice.h"
-#include "rephist.h"
-#include "router.h"
-#include "routerlist.h"
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/channel.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitstats.h"
+#include "core/or/circuituse.h"
+#include "core/or/connection_edge.h"
+#include "core/or/policies.h"
+#include "feature/client/addressmap.h"
+#include "feature/client/bridges.h"
+#include "feature/client/circpathbias.h"
+#include "feature/client/entrynodes.h"
+#include "feature/control/control.h"
+#include "feature/dircommon/directory.h"
+#include "feature/hs/hs_circuit.h"
+#include "feature/hs/hs_client.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_ident.h"
+#include "feature/hs/hs_stats.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/routermode.h"
+#include "feature/relay/selftest.h"
+#include "feature/rend/rendclient.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/rend/rendservice.h"
+#include "feature/stats/predict_ports.h"
+#include "lib/math/fp.h"
+#include "lib/time/tvdiff.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/extend_info_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
 
 static void circuit_expire_old_circuits_clientside(void);
 static void circuit_increment_failure_count(void);
 
+/** Check whether the hidden service destination of the stream at
+ *  <b>edge_conn</b> is the same as the destination of the circuit at
+ *  <b>origin_circ</b>. */
+static int
+circuit_matches_with_rend_stream(const edge_connection_t *edge_conn,
+                                 const origin_circuit_t *origin_circ)
+{
+  /* Check if this is a v2 rendezvous circ/stream */
+  if ((edge_conn->rend_data && !origin_circ->rend_data) ||
+      (!edge_conn->rend_data && origin_circ->rend_data) ||
+      (edge_conn->rend_data && origin_circ->rend_data &&
+       rend_cmp_service_ids(rend_data_get_address(edge_conn->rend_data),
+                            rend_data_get_address(origin_circ->rend_data)))) {
+    /* this circ is not for this conn */
+    return 0;
+  }
+
+  /* Check if this is a v3 rendezvous circ/stream */
+  if ((edge_conn->hs_ident && !origin_circ->hs_ident) ||
+      (!edge_conn->hs_ident && origin_circ->hs_ident) ||
+      (edge_conn->hs_ident && origin_circ->hs_ident &&
+       !ed25519_pubkey_eq(&edge_conn->hs_ident->identity_pk,
+                          &origin_circ->hs_ident->identity_pk))) {
+    /* this circ is not for this conn */
+    return 0;
+  }
+
+  return 1;
+}
+
 /** Return 1 if <b>circ</b> could be returned by circuit_get_best().
  * Else return 0.
  */
@@ -80,6 +147,9 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ,
   }
 
   if (purpose == CIRCUIT_PURPOSE_C_GENERAL ||
+      purpose == CIRCUIT_PURPOSE_C_HSDIR_GET ||
+      purpose == CIRCUIT_PURPOSE_S_HSDIR_POST ||
+      purpose == CIRCUIT_PURPOSE_HS_VANGUARDS ||
       purpose == CIRCUIT_PURPOSE_C_REND_JOINED) {
     if (circ->timestamp_dirty &&
        circ->timestamp_dirty+get_options()->MaxCircuitDirtiness <= now)
@@ -103,7 +173,9 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ,
   if (need_internal != build_state->is_internal)
     return 0;
 
-  if (purpose == CIRCUIT_PURPOSE_C_GENERAL) {
+  if (purpose == CIRCUIT_PURPOSE_C_GENERAL ||
+      purpose == CIRCUIT_PURPOSE_S_HSDIR_POST ||
+      purpose == CIRCUIT_PURPOSE_C_HSDIR_GET) {
     tor_addr_t addr;
     const int family = tor_addr_parse(&addr, conn->socks_request->address);
     if (!exitnode && !build_state->onehop_tunnel) {
@@ -149,14 +221,9 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ,
       /* can't exit from this router */
       return 0;
     }
-  } else { /* not general */
+  } else { /* not general: this might be a rend circuit */
     const edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(conn);
-    if ((edge_conn->rend_data && !origin_circ->rend_data) ||
-        (!edge_conn->rend_data && origin_circ->rend_data) ||
-        (edge_conn->rend_data && origin_circ->rend_data &&
-         rend_cmp_service_ids(edge_conn->rend_data->onion_address,
-                              origin_circ->rend_data->onion_address))) {
-      /* this circ is not for this conn */
+    if (!circuit_matches_with_rend_stream(edge_conn, origin_circ)) {
       return 0;
     }
   }
@@ -190,6 +257,8 @@ circuit_is_better(const origin_circuit_t *oa, const origin_circuit_t *ob,
     return 1; /* oa is better. It's not relaxed. */
 
   switch (purpose) {
+    case CIRCUIT_PURPOSE_S_HSDIR_POST:
+    case CIRCUIT_PURPOSE_C_HSDIR_GET:
     case CIRCUIT_PURPOSE_C_GENERAL:
       /* if it's used but less dirty it's best;
        * else if it's more recently created it's best
@@ -275,6 +344,9 @@ circuit_get_best(const entry_connection_t *conn,
   tor_assert(conn);
 
   tor_assert(purpose == CIRCUIT_PURPOSE_C_GENERAL ||
+             purpose == CIRCUIT_PURPOSE_HS_VANGUARDS ||
+             purpose == CIRCUIT_PURPOSE_C_HSDIR_GET ||
+             purpose == CIRCUIT_PURPOSE_S_HSDIR_POST ||
              purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT ||
              purpose == CIRCUIT_PURPOSE_C_REND_JOINED);
 
@@ -289,7 +361,8 @@ circuit_get_best(const entry_connection_t *conn,
     /* Log an info message if we're going to launch a new intro circ in
      * parallel */
     if (purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT &&
-        !must_be_open && origin_circ->hs_circ_has_timed_out) {
+        !must_be_open && origin_circ->hs_circ_has_timed_out &&
+        !circ->marked_for_close) {
         intro_going_on_but_too_old = 1;
         continue;
     }
@@ -323,7 +396,7 @@ count_pending_general_client_circuits(void)
   SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
     if (circ->marked_for_close ||
         circ->state == CIRCUIT_STATE_OPEN ||
-        circ->purpose != CIRCUIT_PURPOSE_C_GENERAL ||
+        !CIRCUIT_PURPOSE_COUNTS_TOWARDS_MAXPENDING(circ->purpose) ||
         !CIRCUIT_IS_ORIGIN(circ))
       continue;
 
@@ -361,10 +434,19 @@ circuit_conforms_to_options(const origin_circuit_t *circ,
 
   return 1;
 }
-#endif
+#endif /* 0 */
 
-/** Close all circuits that start at us, aren't open, and were born
+/**
+ * Close all circuits that start at us, aren't open, and were born
  * at least CircuitBuildTimeout seconds ago.
+ *
+ * TODO: This function is now partially redundant to
+ * circuit_build_times_handle_completed_hop(), but that function only
+ * covers circuits up to and including 3 hops that are still actually
+ * completing hops. However, circuit_expire_building() also handles longer
+ * circuits, as well as circuits that are completely stalled.
+ * In the future (after prop247/other path selection revamping), we probably
+ * want to eliminate this rats nest in favor of a simpler approach.
  */
 void
 circuit_expire_building(void)
@@ -386,21 +468,7 @@ circuit_expire_building(void)
    * we want to be more lenient with timeouts, in case the
    * user has relocated and/or changed network connections.
    * See bug #3443. */
-  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, next_circ) {
-    if (!CIRCUIT_IS_ORIGIN(next_circ) || /* didn't originate here */
-        next_circ->marked_for_close) { /* don't mess with marked circs */
-      continue;
-    }
-
-    if (TO_ORIGIN_CIRCUIT(next_circ)->has_opened &&
-        next_circ->state == CIRCUIT_STATE_OPEN &&
-        TO_ORIGIN_CIRCUIT(next_circ)->build_state &&
-        TO_ORIGIN_CIRCUIT(next_circ)->build_state->desired_path_len
-          == DEFAULT_ROUTE_LEN) {
-      any_opened_circs = 1;
-      break;
-    }
-  } SMARTLIST_FOREACH_END(next_circ);
+  any_opened_circs = circuit_any_opened_circuits();
 
 #define SET_CUTOFF(target, msec) do {                       \
     long ms = tor_lround(msec);                             \
@@ -444,6 +512,10 @@ circuit_expire_building(void)
   SET_CUTOFF(general_cutoff, get_circuit_build_timeout_ms());
   SET_CUTOFF(begindir_cutoff, get_circuit_build_timeout_ms());
 
+  // TODO: We should probably use route_len_for_purpose() here instead,
+  // except that does not count the extra round trip for things like server
+  // intros and rends.
+
   /* > 3hop circs seem to have a 1.0 second delay on their cannibalized
    * 4th hop. */
   SET_CUTOFF(fourhop_cutoff, get_circuit_build_timeout_ms() * (10/6.0) + 1000);
@@ -473,6 +545,8 @@ circuit_expire_building(void)
 
   SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *,victim) {
     struct timeval cutoff;
+    bool fixed_time = circuit_build_times_disabled(get_options());
+
     if (!CIRCUIT_IS_ORIGIN(victim) || /* didn't originate here */
         victim->marked_for_close)     /* don't mess with marked circs */
       continue;
@@ -527,18 +601,19 @@ circuit_expire_building(void)
         if (!TO_ORIGIN_CIRCUIT(victim)->relaxed_timeout) {
           int first_hop_succeeded = TO_ORIGIN_CIRCUIT(victim)->cpath->state
                                       == CPATH_STATE_OPEN;
-          log_info(LD_CIRC,
-                 "No circuits are opened. Relaxing timeout for circuit %d "
-                 "(a %s %d-hop circuit in state %s with channel state %s). "
-                 "%d guards are live.",
-                 TO_ORIGIN_CIRCUIT(victim)->global_identifier,
-                 circuit_purpose_to_string(victim->purpose),
-                 TO_ORIGIN_CIRCUIT(victim)->build_state ?
-                   TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len :
-                   -1,
-                 circuit_state_to_string(victim->state),
-                 channel_state_to_string(victim->n_chan->state),
-                 num_live_entry_guards(0));
+          if (!fixed_time) {
+            log_info(LD_CIRC,
+                "No circuits are opened. Relaxing timeout for circuit %d "
+                "(a %s %d-hop circuit in state %s with channel state %s).",
+                TO_ORIGIN_CIRCUIT(victim)->global_identifier,
+                circuit_purpose_to_string(victim->purpose),
+                TO_ORIGIN_CIRCUIT(victim)->build_state ?
+                  TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len :
+                  -1,
+                circuit_state_to_string(victim->state),
+                victim->n_chan ?
+                   channel_state_to_string(victim->n_chan->state) : "none");
+          }
 
           /* We count the timeout here for CBT, because technically this
            * was a timeout, and the timeout value needs to reset if we
@@ -552,20 +627,22 @@ circuit_expire_building(void)
       } else {
         static ratelim_t relax_timeout_limit = RATELIM_INIT(3600);
         const double build_close_ms = get_circuit_build_close_time_ms();
-        log_fn_ratelim(&relax_timeout_limit, LOG_NOTICE, LD_CIRC,
+        if (!fixed_time) {
+          log_fn_ratelim(&relax_timeout_limit, LOG_NOTICE, LD_CIRC,
                  "No circuits are opened. Relaxed timeout for circuit %d "
                  "(a %s %d-hop circuit in state %s with channel state %s) to "
                  "%ldms. However, it appears the circuit has timed out "
-                 "anyway. %d guards are live.",
+                 "anyway.",
                  TO_ORIGIN_CIRCUIT(victim)->global_identifier,
                  circuit_purpose_to_string(victim->purpose),
                  TO_ORIGIN_CIRCUIT(victim)->build_state ?
                    TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len :
                    -1,
                  circuit_state_to_string(victim->state),
-                 channel_state_to_string(victim->n_chan->state),
-                 (long)build_close_ms,
-                 num_live_entry_guards(0));
+                 victim->n_chan ?
+                    channel_state_to_string(victim->n_chan->state) : "none",
+                 (long)build_close_ms);
+        }
       }
     }
 
@@ -587,7 +664,7 @@ circuit_expire_building(void)
                victim->n_circ_id,
                (int)(now - victim->timestamp_dirty));
     }
-#endif
+#endif /* 0 */
 
     /* if circ is !open, or if it's open but purpose is a non-finished
      * intro or rend, then mark it for close */
@@ -604,6 +681,7 @@ circuit_expire_building(void)
            * because that's set when they switch purposes
            */
           if (TO_ORIGIN_CIRCUIT(victim)->rend_data ||
+              TO_ORIGIN_CIRCUIT(victim)->hs_ident ||
               victim->timestamp_dirty > cutoff.tv_sec)
             continue;
           break;
@@ -614,7 +692,7 @@ circuit_expire_building(void)
           break;
         case CIRCUIT_PURPOSE_C_INTRODUCING:
           /* That purpose means that the intro point circuit has been opened
-           * succesfully but the INTRODUCE1 cell hasn't been sent yet because
+           * successfully but the INTRODUCE1 cell hasn't been sent yet because
            * the client is waiting for the rendezvous point circuit to open.
            * Keep this circuit open while waiting for the rendezvous circuit.
            * We let the circuit idle timeout take care of cleaning this
@@ -647,23 +725,16 @@ circuit_expire_building(void)
 
       if (circuit_timeout_want_to_count_circ(TO_ORIGIN_CIRCUIT(victim)) &&
           circuit_build_times_enough_to_compute(get_circuit_build_times())) {
+
+        log_info(LD_CIRC,
+                 "Deciding to count the timeout for circuit %"PRIu32"\n",
+                 TO_ORIGIN_CIRCUIT(victim)->global_identifier);
+
         /* Circuits are allowed to last longer for measurement.
          * Switch their purpose and wait. */
         if (victim->purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
-          control_event_circuit_status(TO_ORIGIN_CIRCUIT(victim),
-                                       CIRC_EVENT_FAILED,
-                                       END_CIRC_REASON_TIMEOUT);
-          circuit_change_purpose(victim, CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT);
-          /* Record this failure to check for too many timeouts
-           * in a row. This function does not record a time value yet
-           * (we do that later); it only counts the fact that we did
-           * have a timeout. We also want to avoid double-counting
-           * already "relaxed" circuits, which are counted above. */
-          if (!TO_ORIGIN_CIRCUIT(victim)->relaxed_timeout) {
-            circuit_build_times_count_timeout(
-                                         get_circuit_build_times_mutable(),
-                                         first_hop_succeeded);
-          }
+          circuit_build_times_mark_circ_as_measurement_only(TO_ORIGIN_CIRCUIT(
+                                                            victim));
           continue;
         }
 
@@ -688,18 +759,15 @@ circuit_expire_building(void)
       }
     }
 
-    /* If this is a hidden service client circuit which is far enough
-     * along in connecting to its destination, and we haven't already
-     * flagged it as 'timed out', and the user has not told us to
-     * close such circs immediately on timeout, flag it as 'timed out'
-     * so we'll launch another intro or rend circ, but don't mark it
-     * for close yet.
+    /* If this is a hidden service client circuit which is far enough along in
+     * connecting to its destination, and we haven't already flagged it as
+     * 'timed out', flag it so we'll launch another intro or rend circ, but
+     * don't mark it for close yet.
      *
      * (Circs flagged as 'timed out' are given a much longer timeout
      * period above, so we won't close them in the next call to
      * circuit_expire_building.) */
-    if (!(options->CloseHSClientCircuitsImmediatelyOnTimeout) &&
-        !(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out)) {
+    if (!(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out)) {
       switch (victim->purpose) {
       case CIRCUIT_PURPOSE_C_REND_READY:
         /* We only want to spare a rend circ if it has been specified in
@@ -731,8 +799,7 @@ circuit_expire_building(void)
     /* If this is a service-side rendezvous circuit which is far
      * enough along in connecting to its destination, consider sparing
      * it. */
-    if (!(options->CloseHSServiceRendCircuitsImmediatelyOnTimeout) &&
-        !(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out) &&
+    if (!(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out) &&
         victim->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND) {
       log_info(LD_CIRC,"Marking circ %u (state %d:%s, purpose %d) "
                "as timed-out HS circ; relaunching rendezvous attempt.",
@@ -740,7 +807,7 @@ circuit_expire_building(void)
                victim->state, circuit_state_to_string(victim->state),
                victim->purpose);
       TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out = 1;
-      rend_service_relaunch_rendezvous(TO_ORIGIN_CIRCUIT(victim));
+      hs_circ_retry_service_rendezvous_point(TO_ORIGIN_CIRCUIT(victim));
       continue;
     }
 
@@ -778,6 +845,25 @@ circuit_expire_building(void)
   } SMARTLIST_FOREACH_END(victim);
 }
 
+/**
+ * Mark for close all circuits that start here, that were built through a
+ * guard we weren't sure if we wanted to use, and that have been waiting
+ * around for way too long.
+ */
+void
+circuit_expire_waiting_for_better_guard(void)
+{
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_origin_circuit_list(),
+                          origin_circuit_t *, circ) {
+    if (TO_CIRCUIT(circ)->marked_for_close)
+      continue;
+    if (circ->guard_state == NULL)
+      continue;
+    if (entry_guard_state_should_expire(circ->guard_state))
+      circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_NONE);
+  } SMARTLIST_FOREACH_END(circ);
+}
+
 /** For debugging #8387: track when we last called
  * circuit_expire_old_circuits_clientside. */
 static time_t last_expired_clientside_circuits = 0;
@@ -785,8 +871,7 @@ static time_t last_expired_clientside_circuits = 0;
 /**
  * As a diagnostic for bug 8387, log information about how many one-hop
  * circuits we have around that have been there for at least <b>age</b>
- * seconds. Log a few of them.
- * Ignores Single Onion Service intro and Tor2web redezvous circuits, they are
+ * seconds. Log a few of them. Ignores Single Onion Service intro, it is
  * expected to be long-term one-hop circuits.
  */
 void
@@ -811,13 +896,6 @@ circuit_log_ancient_one_hop_circuits(int age)
         (circ->purpose == CIRCUIT_PURPOSE_S_INTRO ||
          circ->purpose == CIRCUIT_PURPOSE_S_REND_JOINED))
       continue;
-    /* Tor2web deliberately makes long term one-hop rend connections,
-     * particularly when Tor2webRendezvousPoints is used. We only ignore
-     * active rend point connections, if we take a long time to rendezvous,
-     * that's worth logging. */
-    if (rend_client_allow_non_anonymous_connection(options) &&
-        circ->purpose == CIRCUIT_PURPOSE_C_REND_JOINED)
-      continue;
     ocirc = CONST_TO_ORIGIN_CIRCUIT(circ);
 
     if (ocirc->build_state && ocirc->build_state->onehop_tunnel) {
@@ -939,7 +1017,7 @@ circuit_remove_handled_ports(smartlist_t *needed_ports)
     tor_assert(*port);
     if (circuit_stream_is_being_handled(NULL, *port,
                                         MIN_CIRCUITS_HANDLING_STREAM)) {
-//      log_debug(LD_CIRC,"Port %d is already being handled; removing.", port);
+      log_debug(LD_CIRC,"Port %d is already being handled; removing.", *port);
       smartlist_del(needed_ports, i--);
       tor_free(port);
     } else {
@@ -976,6 +1054,10 @@ circuit_stream_is_being_handled(entry_connection_t *conn,
         continue;
       if (origin_circ->unusable_for_new_conns)
         continue;
+      if (origin_circ->isolation_values_set &&
+          (conn == NULL ||
+           !connection_edge_compatible_with_circuit(conn, origin_circ)))
+        continue;
 
       exitnode = build_state_get_exit_node(build_state);
       if (exitnode && (!need_uptime || build_state->need_uptime)) {
@@ -1001,8 +1083,157 @@ circuit_stream_is_being_handled(entry_connection_t *conn,
 /** Don't keep more than this many unused open circuits around. */
 #define MAX_UNUSED_OPEN_CIRCUITS 14
 
-/** Figure out how many circuits we have open that are clean. Make
- * sure it's enough for all the upcoming behaviors we predict we'll have.
+/* Return true if a circuit is available for use, meaning that it is open,
+ * clean, usable for new multi-hop connections, and a general purpose origin
+ * circuit.
+ * Accept any kind of circuit, return false if the above conditions are not
+ * met. */
+STATIC int
+circuit_is_available_for_use(const circuit_t *circ)
+{
+  const origin_circuit_t *origin_circ;
+  cpath_build_state_t *build_state;
+
+  if (!CIRCUIT_IS_ORIGIN(circ))
+    return 0; /* We first filter out only origin circuits before doing the
+                 following checks. */
+  if (circ->marked_for_close)
+    return 0; /* Don't mess with marked circs */
+  if (circ->timestamp_dirty)
+    return 0; /* Only count clean circs */
+  if (circ->purpose != CIRCUIT_PURPOSE_C_GENERAL &&
+      circ->purpose != CIRCUIT_PURPOSE_HS_VANGUARDS)
+    return 0; /* We only pay attention to general purpose circuits.
+                 General purpose circuits are always origin circuits. */
+
+  origin_circ = CONST_TO_ORIGIN_CIRCUIT(circ);
+  if (origin_circ->unusable_for_new_conns)
+    return 0;
+
+  build_state = origin_circ->build_state;
+  if (build_state->onehop_tunnel)
+    return 0;
+
+  return 1;
+}
+
+/* Return true if we need any more exit circuits.
+ * needs_uptime and needs_capacity are set only if we need more exit circuits.
+ * Check if we know of a port that's been requested recently and no circuit
+ * is currently available that can handle it. */
+STATIC int
+needs_exit_circuits(time_t now, int *needs_uptime, int *needs_capacity)
+{
+  return (!circuit_all_predicted_ports_handled(now, needs_uptime,
+                                               needs_capacity) &&
+          router_have_consensus_path() == CONSENSUS_PATH_EXIT);
+}
+
+/* Hidden services need at least this many internal circuits */
+#define SUFFICIENT_UPTIME_INTERNAL_HS_SERVERS 3
+
+/* Return true if we need any more hidden service server circuits.
+ * HS servers only need an internal circuit. */
+STATIC int
+needs_hs_server_circuits(time_t now, int num_uptime_internal)
+{
+  if (!rend_num_services() && !hs_service_get_num_services()) {
+    /* No services, we don't need anything. */
+    goto no_need;
+  }
+
+  if (num_uptime_internal >= SUFFICIENT_UPTIME_INTERNAL_HS_SERVERS) {
+    /* We have sufficient amount of internal circuit. */
+    goto no_need;
+  }
+
+  if (router_have_consensus_path() == CONSENSUS_PATH_UNKNOWN) {
+    /* Consensus hasn't been checked or might be invalid so requesting
+     * internal circuits is not wise. */
+    goto no_need;
+  }
+
+  /* At this point, we need a certain amount of circuits and we will most
+   * likely use them for rendezvous so we note down the use of internal
+   * circuit for our prediction for circuit needing uptime and capacity. */
+  rep_hist_note_used_internal(now, 1, 1);
+
+  return 1;
+ no_need:
+  return 0;
+}
+
+/* We need at least this many internal circuits for hidden service clients */
+#define SUFFICIENT_INTERNAL_HS_CLIENTS 3
+
+/* We need at least this much uptime for internal circuits for hidden service
+ * clients */
+#define SUFFICIENT_UPTIME_INTERNAL_HS_CLIENTS 2
+
+/* Return true if we need any more hidden service client circuits.
+ * HS clients only need an internal circuit. */
+STATIC int
+needs_hs_client_circuits(time_t now, int *needs_uptime, int *needs_capacity,
+    int num_internal, int num_uptime_internal)
+{
+  int used_internal_recently = rep_hist_get_predicted_internal(now,
+                                                               needs_uptime,
+                                                               needs_capacity);
+  int requires_uptime = num_uptime_internal <
+                        SUFFICIENT_UPTIME_INTERNAL_HS_CLIENTS &&
+                        needs_uptime;
+
+  return (used_internal_recently &&
+         (requires_uptime || num_internal < SUFFICIENT_INTERNAL_HS_CLIENTS) &&
+          router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN);
+}
+
+/* This is how many circuits can be opened concurrently during the cbt learning
+ * phase. This number cannot exceed the tor-wide MAX_UNUSED_OPEN_CIRCUITS. */
+#define DFLT_CBT_UNUSED_OPEN_CIRCS (10)
+#define MIN_CBT_UNUSED_OPEN_CIRCS 0
+#define MAX_CBT_UNUSED_OPEN_CIRCS MAX_UNUSED_OPEN_CIRCUITS
+
+/* Return true if we need more circuits for a good build timeout.
+ * XXXX make the assumption that build timeout streams should be
+ * created whenever we can build internal circuits. */
+STATIC int
+needs_circuits_for_build(int num)
+{
+  if (router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) {
+    if (num < networkstatus_get_param(NULL, "cbtmaxopencircs",
+                              DFLT_CBT_UNUSED_OPEN_CIRCS,
+                              MIN_CBT_UNUSED_OPEN_CIRCS,
+                              MAX_CBT_UNUSED_OPEN_CIRCS) &&
+        !circuit_build_times_disabled(get_options()) &&
+        circuit_build_times_needs_circuits_now(get_circuit_build_times())) {
+      return 1;
+    }
+  }
+  return 0;
+}
+
+/**
+ * Launch the appropriate type of predicted circuit for hidden
+ * services, depending on our options.
+ */
+static void
+circuit_launch_predicted_hs_circ(int flags)
+{
+  /* K.I.S.S. implementation of bug #23101: If we are using
+   * vanguards or pinned middles, pre-build a specific purpose
+   * for HS circs. */
+  if (circuit_should_use_vanguards(CIRCUIT_PURPOSE_HS_VANGUARDS)) {
+    circuit_launch(CIRCUIT_PURPOSE_HS_VANGUARDS, flags);
+  } else {
+    /* If no vanguards, then no HS-specific prebuilt circuits are needed.
+     * Normal GENERAL circs are fine */
+    circuit_launch(CIRCUIT_PURPOSE_C_GENERAL, flags);
+  }
+}
+
+/** Determine how many circuits we have open that are clean,
+ * Make sure it's enough for all the upcoming behaviors we predict we'll have.
  * But put an upper bound on the total number of circuits.
  */
 static void
@@ -1014,25 +1245,14 @@ circuit_predict_and_launch_new(void)
   time_t now = time(NULL);
   int flags = 0;
 
-  /* First, count how many of each type of circuit we have already. */
+  /* Count how many of each type of circuit we currently have. */
   SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
-    cpath_build_state_t *build_state;
-    origin_circuit_t *origin_circ;
-    if (!CIRCUIT_IS_ORIGIN(circ))
-      continue;
-    if (circ->marked_for_close)
-      continue; /* don't mess with marked circs */
-    if (circ->timestamp_dirty)
-      continue; /* only count clean circs */
-    if (circ->purpose != CIRCUIT_PURPOSE_C_GENERAL)
-      continue; /* only pay attention to general-purpose circs */
-    origin_circ = TO_ORIGIN_CIRCUIT(circ);
-    if (origin_circ->unusable_for_new_conns)
-      continue;
-    build_state = origin_circ->build_state;
-    if (build_state->onehop_tunnel)
+    if (!circuit_is_available_for_use(circ))
       continue;
+
     num++;
+
+    cpath_build_state_t *build_state = TO_ORIGIN_CIRCUIT(circ)->build_state;
     if (build_state->is_internal)
       num_internal++;
     if (build_state->need_uptime && build_state->is_internal)
@@ -1042,19 +1262,14 @@ circuit_predict_and_launch_new(void)
 
   /* If that's enough, then stop now. */
   if (num >= MAX_UNUSED_OPEN_CIRCUITS)
-    return; /* we already have many, making more probably will hurt */
-
-  /* Second, see if we need any more exit circuits. */
-  /* check if we know of a port that's been requested recently
-   * and no circuit is currently available that can handle it.
-   * Exits (obviously) require an exit circuit. */
-  if (!circuit_all_predicted_ports_handled(now, &port_needs_uptime,
-                                           &port_needs_capacity)
-      && router_have_consensus_path() == CONSENSUS_PATH_EXIT) {
+    return;
+
+  if (needs_exit_circuits(now, &port_needs_uptime, &port_needs_capacity)) {
     if (port_needs_uptime)
       flags |= CIRCLAUNCH_NEED_UPTIME;
     if (port_needs_capacity)
       flags |= CIRCLAUNCH_NEED_CAPACITY;
+
     log_info(LD_CIRC,
              "Have %d clean circs (%d internal), need another exit circ.",
              num, num_internal);
@@ -1062,68 +1277,56 @@ circuit_predict_and_launch_new(void)
     return;
   }
 
-  /* Third, see if we need any more hidden service (server) circuits.
-   * HS servers only need an internal circuit. */
-  if (num_rend_services() && num_uptime_internal < 3
-      && router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) {
+  if (needs_hs_server_circuits(now, num_uptime_internal)) {
     flags = (CIRCLAUNCH_NEED_CAPACITY | CIRCLAUNCH_NEED_UPTIME |
              CIRCLAUNCH_IS_INTERNAL);
+
     log_info(LD_CIRC,
              "Have %d clean circs (%d internal), need another internal "
              "circ for my hidden service.",
              num, num_internal);
-    circuit_launch(CIRCUIT_PURPOSE_C_GENERAL, flags);
+    circuit_launch_predicted_hs_circ(flags);
     return;
   }
 
-  /* Fourth, see if we need any more hidden service (client) circuits.
-   * HS clients only need an internal circuit. */
-  if (rep_hist_get_predicted_internal(now, &hidserv_needs_uptime,
-                                      &hidserv_needs_capacity) &&
-      ((num_uptime_internal<2 && hidserv_needs_uptime) ||
-        num_internal<3)
-        && router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) {
+  if (needs_hs_client_circuits(now, &hidserv_needs_uptime,
+                               &hidserv_needs_capacity,
+                               num_internal, num_uptime_internal))
+  {
     if (hidserv_needs_uptime)
       flags |= CIRCLAUNCH_NEED_UPTIME;
     if (hidserv_needs_capacity)
       flags |= CIRCLAUNCH_NEED_CAPACITY;
     flags |= CIRCLAUNCH_IS_INTERNAL;
+
     log_info(LD_CIRC,
              "Have %d clean circs (%d uptime-internal, %d internal), need"
              " another hidden service circ.",
              num, num_uptime_internal, num_internal);
-    circuit_launch(CIRCUIT_PURPOSE_C_GENERAL, flags);
+
+    circuit_launch_predicted_hs_circ(flags);
     return;
   }
 
-  /* Finally, check to see if we still need more circuits to learn
-   * a good build timeout. But if we're close to our max number we
-   * want, don't do another -- we want to leave a few slots open so
-   * we can still build circuits preemptively as needed.
-   * XXXX make the assumption that build timeout streams should be
-   * created whenever we can build internal circuits. */
-  if (router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) {
-    if (num < MAX_UNUSED_OPEN_CIRCUITS-2 &&
-        ! circuit_build_times_disabled() &&
-        circuit_build_times_needs_circuits_now(get_circuit_build_times())) {
-      flags = CIRCLAUNCH_NEED_CAPACITY;
-      /* if there are no exits in the consensus, make timeout
-       * circuits internal */
-      if (router_have_consensus_path() == CONSENSUS_PATH_INTERNAL)
-        flags |= CIRCLAUNCH_IS_INTERNAL;
+  if (needs_circuits_for_build(num)) {
+    flags = CIRCLAUNCH_NEED_CAPACITY;
+    /* if there are no exits in the consensus, make timeout
+     * circuits internal */
+    if (router_have_consensus_path() == CONSENSUS_PATH_INTERNAL)
+      flags |= CIRCLAUNCH_IS_INTERNAL;
+
       log_info(LD_CIRC,
                "Have %d clean circs need another buildtime test circ.", num);
       circuit_launch(CIRCUIT_PURPOSE_C_GENERAL, flags);
       return;
-    }
   }
 }
 
 /** Build a new test circuit every 5 minutes */
 #define TESTING_CIRCUIT_INTERVAL 300
 
-/** This function is called once a second, if router_have_min_dir_info() is
- * true. Its job is to make sure all services we offer have enough circuits
+/** This function is called once a second, if router_have_minimum_dir_info()
+ * is true. Its job is to make sure all services we offer have enough circuits
  * available. Some services just want enough circuits for current tasks,
  * whereas others want a minimum set of idle circuits hanging around.
  */
@@ -1139,11 +1342,6 @@ circuit_build_needed_circs(time_t now)
   if (router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN)
     connection_ap_rescan_and_attach_pending();
 
-  /* make sure any hidden services have enough intro points
-   * HS intro point streams only require an internal circuit */
-  if (router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN)
-    rend_consider_services_intro_points();
-
   circuit_expire_old_circs_as_needed(now);
 
   if (!options->DisablePredictedCircuits)
@@ -1179,7 +1377,7 @@ circuit_expire_old_circs_as_needed(time_t now)
       log_fn(LOG_INFO,"Creating a new testing circuit.");
       circuit_launch(CIRCUIT_PURPOSE_C_GENERAL, 0);
     }
-#endif
+#endif /* 0 */
   }
 }
 
@@ -1225,8 +1423,7 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn)
        * number of streams on the circuit associated with the rend service.
        */
       if (circ->purpose == CIRCUIT_PURPOSE_S_REND_JOINED) {
-        tor_assert(origin_circ->rend_data);
-        origin_circ->rend_data->nr_streams--;
+        hs_dec_rdv_stream_counter(origin_circ);
       }
       return;
     }
@@ -1265,11 +1462,6 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn)
   tor_fragile_assert();
 }
 
-/** If we haven't yet decided on a good timeout value for circuit
- * building, we close idles circuits aggressively so we can get more
- * data points. */
-#define IDLE_TIMEOUT_WHILE_LEARNING (10*60)
-
 /** Find each circuit that has been unused for too long, or dirty
  * for too long and has no streams on it: mark it for close.
  */
@@ -1279,21 +1471,15 @@ circuit_expire_old_circuits_clientside(void)
   struct timeval cutoff, now;
 
   tor_gettimeofday(&now);
-  cutoff = now;
   last_expired_clientside_circuits = now.tv_sec;
 
-  if (! circuit_build_times_disabled() &&
-      circuit_build_times_needs_circuits(get_circuit_build_times())) {
-    /* Circuits should be shorter lived if we need more of them
-     * for learning a good build timeout */
-    cutoff.tv_sec -= IDLE_TIMEOUT_WHILE_LEARNING;
-  } else {
-    cutoff.tv_sec -= get_options()->CircuitIdleTimeout;
-  }
-
   SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
     if (circ->marked_for_close || !CIRCUIT_IS_ORIGIN(circ))
       continue;
+
+    cutoff = now;
+    cutoff.tv_sec -= TO_ORIGIN_CIRCUIT(circ)->circuit_idle_timeout;
+
     /* If the circuit has been dirty for too long, and there are no streams
      * on it, mark it for close.
      */
@@ -1313,15 +1499,20 @@ circuit_expire_old_circuits_clientside(void)
     } else if (!circ->timestamp_dirty && circ->state == CIRCUIT_STATE_OPEN) {
       if (timercmp(&circ->timestamp_began, &cutoff, OP_LT)) {
         if (circ->purpose == CIRCUIT_PURPOSE_C_GENERAL ||
+                circ->purpose == CIRCUIT_PURPOSE_C_HSDIR_GET ||
+                circ->purpose == CIRCUIT_PURPOSE_S_HSDIR_POST ||
+                circ->purpose == CIRCUIT_PURPOSE_HS_VANGUARDS ||
                 circ->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT ||
                 circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
                 circ->purpose == CIRCUIT_PURPOSE_TESTING ||
                 (circ->purpose >= CIRCUIT_PURPOSE_C_INTRODUCING &&
                 circ->purpose <= CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED) ||
                 circ->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND) {
-          log_debug(LD_CIRC,
-                    "Closing circuit that has been unused for %ld msec.",
-                    tv_mdiff(&circ->timestamp_began, &now));
+          log_info(LD_CIRC,
+                    "Closing circuit %"PRIu32
+                    " that has been unused for %ld msec.",
+                   TO_ORIGIN_CIRCUIT(circ)->global_identifier,
+                   tv_mdiff(&circ->timestamp_began, &now));
           circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED);
         } else if (!TO_ORIGIN_CIRCUIT(circ)->is_ancient) {
           /* Server-side rend joined circuits can end up really old, because
@@ -1364,7 +1555,7 @@ circuit_expire_old_circuits_clientside(void)
 #define IDLE_ONE_HOP_CIRC_TIMEOUT 60
 
 /** Find each non-origin circuit that has been unused for too long,
- * has no streams on it, used a create_fast, and ends here: mark it
+ * has no streams on it, came from a client, and ends here: mark it
  * for close.
  */
 void
@@ -1383,9 +1574,10 @@ circuit_expire_old_circuits_serverside(time_t now)
      * Also if there is a rend_splice on it, it's a single onion service
      * circuit and we should not close it.
      */
-    if (or_circ->is_first_hop && !circ->n_chan &&
+    if (or_circ->p_chan && channel_is_client(or_circ->p_chan) &&
+        !circ->n_chan &&
         !or_circ->n_streams && !or_circ->resolving_streams &&
-        or_circ->p_chan && !or_circ->rend_splice &&
+        !or_circ->rend_splice &&
         channel_when_last_xmit(or_circ->p_chan) <= cutoff) {
       log_info(LD_CIRC, "Closing circ_id %u (empty %d secs ago)",
                (unsigned)or_circ->p_circ_id,
@@ -1453,7 +1645,7 @@ circuit_testing_opened(origin_circuit_t *circ)
     router_perform_bandwidth_test(NUM_PARALLEL_TESTING_CIRCS, time(NULL));
     have_performed_bandwidth_test = 1;
   } else
-    consider_testing_reachability(1, 0);
+    router_do_reachability_checks(1, 0);
 }
 
 /** A testing circuit has failed to build. Take whatever stats we want. */
@@ -1491,7 +1683,7 @@ circuit_has_opened(origin_circuit_t *circ)
 
   switch (TO_CIRCUIT(circ)->purpose) {
     case CIRCUIT_PURPOSE_C_ESTABLISH_REND:
-      rend_client_rendcirc_has_opened(circ);
+      hs_client_circuit_has_opened(circ);
       /* Start building an intro circ if we don't have one yet. */
       connection_ap_attach_pending(1);
       /* This isn't a call to circuit_try_attaching_streams because a
@@ -1503,20 +1695,22 @@ circuit_has_opened(origin_circuit_t *circ)
        * state. */
       break;
     case CIRCUIT_PURPOSE_C_INTRODUCING:
-      rend_client_introcirc_has_opened(circ);
+      hs_client_circuit_has_opened(circ);
       break;
     case CIRCUIT_PURPOSE_C_GENERAL:
+    case CIRCUIT_PURPOSE_C_HSDIR_GET:
+    case CIRCUIT_PURPOSE_S_HSDIR_POST:
       /* Tell any AP connections that have been waiting for a new
        * circuit that one is ready. */
       circuit_try_attaching_streams(circ);
       break;
     case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO:
       /* at the service, waiting for introductions */
-      rend_service_intro_has_opened(circ);
+      hs_service_circuit_has_opened(circ);
       break;
     case CIRCUIT_PURPOSE_S_CONNECT_REND:
       /* at the service, connecting to rend point */
-      rend_service_rendezvous_has_opened(circ);
+      hs_service_circuit_has_opened(circ);
       break;
     case CIRCUIT_PURPOSE_TESTING:
       circuit_testing_opened(circ);
@@ -1576,6 +1770,39 @@ circuit_build_failed(origin_circuit_t *circ)
    * the last hop or an earlier hop. then use this info below.
    */
   int failed_at_last_hop = 0;
+
+  /* First, check to see if this was a path failure, rather than build
+   * failure.
+   *
+   * Note that we deliberately use circuit_get_cpath_len() (and not
+   * circuit_get_cpath_opened_len()) because we only want to ensure
+   * that a full path is *chosen*. This is different than a full path
+   * being *built*. We only want to count *build* failures below.
+   *
+   * Path selection failures can happen spuriously for a number
+   * of reasons (such as aggressive/invalid user-specified path
+   * restrictions in the torrc, insufficient microdescriptors, and
+   * non-user reasons like exitpolicy issues), and so should not be
+   * counted as failures below.
+   */
+  if (circuit_get_cpath_len(circ) < circ->build_state->desired_path_len) {
+    static ratelim_t pathfail_limit = RATELIM_INIT(3600);
+    log_fn_ratelim(&pathfail_limit, LOG_NOTICE, LD_CIRC,
+             "Our circuit %u (id: %" PRIu32 ") died due to an invalid "
+             "selected path, purpose %s. This may be a torrc "
+             "configuration issue, or a bug.",
+              TO_CIRCUIT(circ)->n_circ_id, circ->global_identifier,
+              circuit_purpose_to_string(TO_CIRCUIT(circ)->purpose));
+
+    /* If the path failed on an RP, retry it. */
+    if (TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND)
+      hs_circ_retry_service_rendezvous_point(circ);
+
+    /* In all other cases, just bail. The rest is just failure accounting
+     * that we don't want to do */
+    return;
+  }
+
   /* If the last hop isn't open, and the second-to-last is, we failed
    * at the last hop. */
   if (circ->cpath &&
@@ -1583,13 +1810,16 @@ circuit_build_failed(origin_circuit_t *circ)
       circ->cpath->prev->prev->state == CPATH_STATE_OPEN) {
     failed_at_last_hop = 1;
   }
+
+  /* Check if we failed at first hop */
   if (circ->cpath &&
       circ->cpath->state != CPATH_STATE_OPEN &&
       ! circ->base_.received_destroy) {
     /* We failed at the first hop for some reason other than a DESTROY cell.
      * If there's an OR connection to blame, blame it. Also, avoid this relay
      * for a while, and fail any one-hop directory fetches destined for it. */
-    const char *n_chan_id = circ->cpath->extend_info->identity_digest;
+    const char *n_chan_ident = circ->cpath->extend_info->identity_digest;
+    tor_assert(n_chan_ident);
     int already_marked = 0;
     if (circ->base_.n_chan) {
       n_chan = circ->base_.n_chan;
@@ -1605,23 +1835,35 @@ circuit_build_failed(origin_circuit_t *circ)
         already_marked = 1;
       }
       log_info(LD_OR,
-               "Our circuit failed to get a response from the first hop "
-               "(%s). I'm going to try to rotate to a better connection.",
+               "Our circuit %u (id: %" PRIu32 ") failed to get a response "
+               "from the first hop (%s). I'm going to try to rotate to a "
+               "better connection.",
+               TO_CIRCUIT(circ)->n_circ_id, circ->global_identifier,
                channel_get_canonical_remote_descr(n_chan));
       n_chan->is_bad_for_new_circs = 1;
     } else {
       log_info(LD_OR,
-               "Our circuit died before the first hop with no connection");
+               "Our circuit %u (id: %" PRIu32 ") died before the first hop "
+               "with no connection",
+               TO_CIRCUIT(circ)->n_circ_id, circ->global_identifier);
     }
-    if (n_chan_id && !already_marked) {
-      entry_guard_register_connect_status(n_chan_id, 0, 1, time(NULL));
+    if (!already_marked) {
+      /*
+       * If we have guard state (new guard API) and our path selection
+       * code actually chose a full path, then blame the failure of this
+       * circuit on the guard.
+       */
+      if (circ->guard_state)
+        entry_guard_failed(&circ->guard_state);
       /* if there are any one-hop streams waiting on this circuit, fail
        * them now so they can retry elsewhere. */
-      connection_ap_fail_onehop(n_chan_id, circ->build_state);
+      connection_ap_fail_onehop(n_chan_ident, circ->build_state);
     }
   }
 
   switch (circ->base_.purpose) {
+    case CIRCUIT_PURPOSE_C_HSDIR_GET:
+    case CIRCUIT_PURPOSE_S_HSDIR_POST:
     case CIRCUIT_PURPOSE_C_GENERAL:
       /* If we never built the circuit, note it as a failure. */
       circuit_increment_failure_count();
@@ -1664,7 +1906,7 @@ circuit_build_failed(origin_circuit_t *circ)
                "(%s hop failed).",
                escaped(build_state_get_exit_nickname(circ->build_state)),
                failed_at_last_hop?"last":"non-last");
-      rend_service_relaunch_rendezvous(circ);
+      hs_circ_retry_service_rendezvous_point(circ);
       break;
     /* default:
      * This won't happen in normal operation, but might happen if the
@@ -1706,6 +1948,104 @@ have_enough_path_info(int need_exit)
     return router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN;
 }
 
+/**
+ * Tell us if a circuit is a hidden service circuit.
+ */
+int
+circuit_purpose_is_hidden_service(uint8_t purpose)
+{
+   if (purpose == CIRCUIT_PURPOSE_HS_VANGUARDS) {
+     return 1;
+   }
+
+   /* Client-side purpose */
+   if (purpose >= CIRCUIT_PURPOSE_C_HS_MIN_ &&
+       purpose <= CIRCUIT_PURPOSE_C_HS_MAX_) {
+     return 1;
+   }
+
+   /* Service-side purpose */
+   if (purpose >= CIRCUIT_PURPOSE_S_HS_MIN_ &&
+       purpose <= CIRCUIT_PURPOSE_S_HS_MAX_) {
+     return 1;
+   }
+
+   return 0;
+}
+
+/**
+ * Return true if this circuit purpose should use vanguards
+ * or pinned Layer2 or Layer3 guards.
+ *
+ * This function takes both the circuit purpose and the
+ * torrc options for pinned middles/vanguards into account
+ * (ie: the circuit must be a hidden service circuit and
+ * vanguards/pinned middles must be enabled for it to return
+ * true).
+ */
+int
+circuit_should_use_vanguards(uint8_t purpose)
+{
+  const or_options_t *options = get_options();
+
+  /* Only hidden service circuits use vanguards */
+  if (!circuit_purpose_is_hidden_service(purpose))
+    return 0;
+
+  /* Pinned middles are effectively vanguards */
+  if (options->HSLayer2Nodes || options->HSLayer3Nodes)
+    return 1;
+
+  return 0;
+}
+
+/**
+ * Return true for the set of conditions for which it is OK to use
+ * a cannibalized circuit.
+ *
+ * Don't cannibalize for onehops, or certain purposes.
+ */
+static int
+circuit_should_cannibalize_to_build(uint8_t purpose_to_build,
+                                    int has_extend_info,
+                                    int onehop_tunnel)
+{
+
+  /* Do not try to cannibalize if this is a one hop circuit. */
+  if (onehop_tunnel) {
+    return 0;
+  }
+
+  /* Don't try to cannibalize for general purpose circuits that do not
+   * specify a custom exit. */
+  if (purpose_to_build == CIRCUIT_PURPOSE_C_GENERAL && !has_extend_info) {
+    return 0;
+  }
+
+  /* Don't cannibalize for testing circuits. We want to see if they
+   * complete normally. Also don't cannibalize for vanguard-purpose
+   * circuits, since those are specially pre-built for later
+   * cannibalization by the actual specific circuit types that need
+   * vanguards.
+   */
+  if (purpose_to_build == CIRCUIT_PURPOSE_TESTING ||
+      purpose_to_build == CIRCUIT_PURPOSE_HS_VANGUARDS) {
+    return 0;
+  }
+
+  /* For vanguards, the server-side intro circ is not cannibalized
+   * because we pre-build 4 hop HS circuits, and it only needs a 3 hop
+   * circuit. It is also long-lived, so it is more important that
+   * it have lower latency than get built fast.
+   */
+  if (circuit_should_use_vanguards(purpose_to_build) &&
+      purpose_to_build == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) {
+    return 0;
+  }
+
+  return 1;
+}
+
 /** Launch a new circuit with purpose <b>purpose</b> and exit node
  * <b>extend_info</b> (or NULL to select a random exit node).  If flags
  * contains CIRCLAUNCH_NEED_UPTIME, choose among routers with high uptime.  If
@@ -1721,7 +2061,11 @@ circuit_launch_by_extend_info(uint8_t purpose,
   origin_circuit_t *circ;
   int onehop_tunnel = (flags & CIRCLAUNCH_ONEHOP_TUNNEL) != 0;
   int have_path = have_enough_path_info(! (flags & CIRCLAUNCH_IS_INTERNAL) );
-  int need_specific_rp = 0;
+
+  /* Keep some stats about our attempts to launch HS rendezvous circuits */
+  if (purpose == CIRCUIT_PURPOSE_S_CONNECT_REND) {
+    hs_stats_note_service_rendezvous_launch();
+  }
 
   if (!onehop_tunnel && (!router_have_minimum_dir_info() || !have_path)) {
     log_debug(LD_CIRC,"Haven't %s yet; canceling "
@@ -1732,17 +2076,11 @@ circuit_launch_by_extend_info(uint8_t purpose,
     return NULL;
   }
 
-  /* If Tor2webRendezvousPoints is enabled and we are dealing with an
-     RP circuit, we want a specific RP node so we shouldn't canibalize
-     an already existing circuit. */
-  if (get_options()->Tor2webRendezvousPoints &&
-      purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND) {
-    need_specific_rp = 1;
-  }
-
-  if ((extend_info || purpose != CIRCUIT_PURPOSE_C_GENERAL) &&
-      purpose != CIRCUIT_PURPOSE_TESTING &&
-      !onehop_tunnel && !need_specific_rp) {
+  /* If we can/should cannibalize another circuit to build this one,
+   * then do so. */
+  if (circuit_should_cannibalize_to_build(purpose,
+                                          extend_info != NULL,
+                                          onehop_tunnel)) {
     /* see if there are appropriate circs available to cannibalize. */
     /* XXX if we're planning to add a hop, perhaps we want to look for
      * internal circs rather than exit circs? -RD */
@@ -1751,8 +2089,9 @@ circuit_launch_by_extend_info(uint8_t purpose,
       uint8_t old_purpose = circ->base_.purpose;
       struct timeval old_timestamp_began = circ->base_.timestamp_began;
 
-      log_info(LD_CIRC,"Cannibalizing circ '%s' for purpose %d (%s)",
-               build_state_get_exit_nickname(circ->build_state), purpose,
+      log_info(LD_CIRC, "Cannibalizing circ %u (id: %" PRIu32 ") for "
+                        "purpose %d (%s)",
+               TO_CIRCUIT(circ)->n_circ_id, circ->global_identifier, purpose,
                circuit_purpose_to_string(purpose));
 
       if ((purpose == CIRCUIT_PURPOSE_S_CONNECT_REND ||
@@ -1796,6 +2135,8 @@ circuit_launch_by_extend_info(uint8_t purpose,
         case CIRCUIT_PURPOSE_C_INTRODUCING:
         case CIRCUIT_PURPOSE_S_CONNECT_REND:
         case CIRCUIT_PURPOSE_C_GENERAL:
+        case CIRCUIT_PURPOSE_S_HSDIR_POST:
+        case CIRCUIT_PURPOSE_C_HSDIR_GET:
         case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO:
           /* need to add a new hop */
           tor_assert(extend_info);
@@ -1875,16 +2216,22 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
             c->state, conn_state_to_string(c->type, c->state));
   }
   tor_assert(ENTRY_TO_CONN(conn)->state == AP_CONN_STATE_CIRCUIT_WAIT);
+
+  /* Will the exit policy of the exit node apply to this stream? */
   check_exit_policy =
       conn->socks_request->command == SOCKS_COMMAND_CONNECT &&
       !conn->use_begindir &&
       !connection_edge_is_rendezvous_stream(ENTRY_TO_EDGE_CONN(conn));
+
+  /* Does this connection want a one-hop circuit? */
   want_onehop = conn->want_onehop;
 
+  /* Do we need a high-uptime circuit? */
   need_uptime = !conn->want_onehop && !conn->use_begindir &&
                 smartlist_contains_int_as_string(options->LongLivedPorts,
                                           conn->socks_request->port);
 
+  /* Do we need an "internal" circuit? */
   if (desired_circuit_purpose != CIRCUIT_PURPOSE_C_GENERAL)
     need_internal = 1;
   else if (conn->use_begindir || conn->want_onehop)
@@ -1892,23 +2239,37 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
   else
     need_internal = 0;
 
-  circ = circuit_get_best(conn, 1, desired_circuit_purpose,
+  /* We now know what kind of circuit we need.  See if there is an
+   * open circuit that we can use for this stream */
+  circ = circuit_get_best(conn, 1 /* Insist on open circuits */,
+                          desired_circuit_purpose,
                           need_uptime, need_internal);
 
   if (circ) {
+    /* We got a circuit that will work for this stream!  We can return it. */
     *circp = circ;
     return 1; /* we're happy */
   }
 
+  /* Okay, there's no circuit open that will work for this stream. Let's
+   * see if there's an in-progress circuit or if we have to launch one */
+
+  /* Do we know enough directory info to build circuits at all? */
   int have_path = have_enough_path_info(!need_internal);
 
   if (!want_onehop && (!router_have_minimum_dir_info() || !have_path)) {
+    /* If we don't have enough directory information, we can't build
+     * multihop circuits.
+     */
     if (!connection_get_by_type(CONN_TYPE_DIR)) {
       int severity = LOG_NOTICE;
-      /* FFFF if this is a tunneled directory fetch, don't yell
-       * as loudly. the user doesn't even know it's happening. */
-      if (entry_list_is_constrained(options) &&
-          entries_known_but_down(options)) {
+      /* Retry some stuff that might help the connection work. */
+      /* If we are configured with EntryNodes or UseBridges */
+      if (entry_list_is_constrained(options)) {
+        /* Retry all our guards / bridges.
+         * guards_retry_optimistic() always returns true here. */
+        int rv = guards_retry_optimistic(options);
+        tor_assert_nonfatal_once(rv);
         log_fn(severity, LD_APP|LD_DIR,
                "Application request when we haven't %s. "
                "Optimistically trying known %s again.",
@@ -1916,8 +2277,12 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
                "used client functionality lately" :
                "received a consensus with exits",
                options->UseBridges ? "bridges" : "entrynodes");
-        entries_retry_all(options);
-      } else if (!options->UseBridges || any_bridge_descriptors_known()) {
+      } else {
+        /* Getting directory documents doesn't help much if we have a limited
+         * number of guards */
+        tor_assert_nonfatal(!options->UseBridges);
+        tor_assert_nonfatal(!options->EntryNodes);
+        /* Retry our directory fetches, so we have a fresh set of guard info */
         log_fn(severity, LD_APP|LD_DIR,
                "Application request when we haven't %s. "
                "Optimistically trying directory fetches again.",
@@ -1927,14 +2292,16 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
         routerlist_retry_directory_downloads(time(NULL));
       }
     }
-    /* the stream will be dealt with when router_have_minimum_dir_info becomes
-     * 1, or when all directory attempts fail and directory_all_unreachable()
+    /* Since we didn't have enough directory info, we can't attach now.  The
+     * stream will be dealt with when router_have_minimum_dir_info becomes 1,
+     * or when all directory attempts fail and directory_all_unreachable()
      * kills it.
      */
     return 0;
   }
 
-  /* Do we need to check exit policy? */
+  /* Check whether the exit policy of the chosen exit, or the exit policies
+   * of _all_ nodes, would forbid this node. */
   if (check_exit_policy) {
     if (!conn->chosen_exit_name) {
       struct in_addr in;
@@ -1955,7 +2322,7 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
     } else {
       /* XXXX Duplicates checks in connection_ap_handshake_attach_circuit:
        * refactor into a single function. */
-      const node_t *node = node_get_by_nickname(conn->chosen_exit_name, 1);
+      const node_t *node = node_get_by_nickname(conn->chosen_exit_name, 0);
       int opt = conn->chosen_exit_optional;
       if (node && !connection_ap_can_use_exit(conn, node)) {
         log_fn(opt ? LOG_INFO : LOG_WARN, LD_APP,
@@ -1975,16 +2342,25 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
     }
   }
 
-  /* is one already on the way? */
-  circ = circuit_get_best(conn, 0, desired_circuit_purpose,
+  /* Now, check whether there already a circuit on the way that could handle
+   * this stream. This check matches the one above, but this time we
+   * do not require that the circuit will work. */
+  circ = circuit_get_best(conn, 0 /* don't insist on open circuits */,
+                          desired_circuit_purpose,
                           need_uptime, need_internal);
   if (circ)
     log_debug(LD_CIRC, "one on the way!");
+
   if (!circ) {
+    /* No open or in-progress circuit could handle this stream!  We
+     * will have to launch one!
+     */
+
+    /* The chosen exit node, if there is one. */
     extend_info_t *extend_info=NULL;
-    uint8_t new_circ_purpose;
     const int n_pending = count_pending_general_client_circuits();
 
+    /* Do we have too many pending circuits? */
     if (n_pending >= options->MaxClientCircuitsPending) {
       static ratelim_t delay_limit = RATELIM_INIT(10*60);
       char *m;
@@ -1998,34 +2374,40 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
       return 0;
     }
 
+    /* If this is a hidden service trying to start an introduction point,
+     * handle that case. */
     if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT) {
+      const edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(conn);
       /* need to pick an intro point */
-      rend_data_t *rend_data = ENTRY_TO_EDGE_CONN(conn)->rend_data;
-      tor_assert(rend_data);
-      extend_info = rend_client_get_random_intro(rend_data);
+      extend_info = hs_client_get_random_intro_from_edge(edge_conn);
       if (!extend_info) {
-        log_info(LD_REND,
-                 "No intro points for '%s': re-fetching service descriptor.",
-                 safe_str_client(rend_data->onion_address));
-        rend_client_refetch_v2_renddesc(rend_data);
-        connection_ap_mark_as_non_pending_circuit(conn);
-        ENTRY_TO_CONN(conn)->state = AP_CONN_STATE_RENDDESC_WAIT;
+        log_info(LD_REND, "No intro points: re-fetching service descriptor.");
+        if (edge_conn->rend_data) {
+          rend_client_refetch_v2_renddesc(edge_conn->rend_data);
+        } else {
+          hs_client_refetch_hsdesc(&edge_conn->hs_ident->identity_pk);
+        }
+        connection_ap_mark_as_waiting_for_renddesc(conn);
         return 0;
       }
       log_info(LD_REND,"Chose %s as intro point for '%s'.",
                extend_info_describe(extend_info),
-               safe_str_client(rend_data->onion_address));
+               (edge_conn->rend_data) ?
+               safe_str_client(rend_data_get_address(edge_conn->rend_data)) :
+               "service");
     }
 
     /* If we have specified a particular exit node for our
      * connection, then be sure to open a circuit to that exit node.
      */
-    if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_GENERAL) {
+    if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_GENERAL ||
+        desired_circuit_purpose == CIRCUIT_PURPOSE_S_HSDIR_POST ||
+        desired_circuit_purpose == CIRCUIT_PURPOSE_C_HSDIR_GET) {
       if (conn->chosen_exit_name) {
         const node_t *r;
         int opt = conn->chosen_exit_optional;
-        r = node_get_by_nickname(conn->chosen_exit_name, 1);
-        if (r && node_has_descriptor(r)) {
+        r = node_get_by_nickname(conn->chosen_exit_name, 0);
+        if (r && node_has_preferred_descriptor(r, conn->want_onehop ? 1 : 0)) {
           /* We might want to connect to an IPv6 bridge for loading
              descriptors so we use the preferred address rather than
              the primary. */
@@ -2035,12 +2417,16 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
                      "Discarding this circuit.", conn->chosen_exit_name);
             return -1;
           }
-        } else {
+        } else  { /* ! (r && node_has_preferred_descriptor(...)) */
           log_debug(LD_DIR, "considering %d, %s",
                     want_onehop, conn->chosen_exit_name);
           if (want_onehop && conn->chosen_exit_name[0] == '$') {
             /* We're asking for a one-hop circuit to a router that
              * we don't have a routerinfo about. Make up an extend_info. */
+            /* XXX prop220: we need to make chosen_exit_name able to
+             * encode both key formats. This is not absolutely critical
+             * since this is just for one-hop circuits, but we should
+             * still get it done */
             char digest[DIGEST_LEN];
             char *hexdigest = conn->chosen_exit_name+1;
             tor_addr_t addr;
@@ -2055,10 +2441,13 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
                        escaped_safe_str_client(conn->socks_request->address));
               return -1;
             }
+            /* XXXX prop220 add a workaround for ed25519 ID below*/
             extend_info = extend_info_new(conn->chosen_exit_name+1,
-                                          digest, NULL, NULL, &addr,
-                                          conn->socks_request->port);
-          } else {
+                                          digest,
+                                          NULL, /* Ed25519 ID */
+                                          NULL, NULL, /* onion keys */
+                                          &addr, conn->socks_request->port);
+          } else { /* ! (want_onehop && conn->chosen_exit_name[0] == '$') */
             /* We will need an onion key for the router, and we
              * don't have one. Refuse or relax requirements. */
             log_fn(opt ? LOG_INFO : LOG_WARN, LD_APP,
@@ -2076,8 +2465,10 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
           }
         }
       }
-    }
+    } /* Done checking for general circutis with chosen exits. */
 
+    /* What purpose do we need to launch this circuit with? */
+    uint8_t new_circ_purpose;
     if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_REND_JOINED)
       new_circ_purpose = CIRCUIT_PURPOSE_C_ESTABLISH_REND;
     else if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT)
@@ -2085,26 +2476,33 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
     else
       new_circ_purpose = desired_circuit_purpose;
 
-#ifdef ENABLE_TOR2WEB_MODE
-    if (options->Tor2webMode &&
-        (new_circ_purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND ||
-         new_circ_purpose == CIRCUIT_PURPOSE_C_INTRODUCING)) {
-      want_onehop = 1;
-    }
-#endif
-
+    /* Determine what kind of a circuit to launch, and actually launch it. */
     {
       int flags = CIRCLAUNCH_NEED_CAPACITY;
       if (want_onehop) flags |= CIRCLAUNCH_ONEHOP_TUNNEL;
       if (need_uptime) flags |= CIRCLAUNCH_NEED_UPTIME;
       if (need_internal) flags |= CIRCLAUNCH_IS_INTERNAL;
+
+      /* If we are about to pick a v3 RP right now, make sure we pick a
+       * rendezvous point that supports the v3 protocol! */
+      if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_REND_JOINED &&
+          new_circ_purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND &&
+          ENTRY_TO_EDGE_CONN(conn)->hs_ident) {
+        flags |= CIRCLAUNCH_IS_V3_RP;
+        log_info(LD_GENERAL, "Getting rendezvous circuit to v3 service!");
+      }
+
       circ = circuit_launch_by_extend_info(new_circ_purpose, extend_info,
                                            flags);
     }
 
     extend_info_free(extend_info);
 
-    if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_GENERAL) {
+    /* Now trigger things that need to happen when we launch circuits */
+
+    if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_GENERAL ||
+        desired_circuit_purpose == CIRCUIT_PURPOSE_C_HSDIR_GET ||
+        desired_circuit_purpose == CIRCUIT_PURPOSE_S_HSDIR_POST) {
       /* We just caused a circuit to get built because of this stream.
        * If this stream has caused a _lot_ of circuits to be built, that's
        * a bad sign: we should tell the user. */
@@ -2119,14 +2517,25 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
       /* help predict this next time */
       rep_hist_note_used_internal(time(NULL), need_uptime, 1);
       if (circ) {
-        /* write the service_id into circ */
-        circ->rend_data = rend_data_dup(ENTRY_TO_EDGE_CONN(conn)->rend_data);
+        const edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(conn);
+        if (edge_conn->rend_data) {
+          /* write the service_id into circ */
+          circ->rend_data = rend_data_dup(edge_conn->rend_data);
+        } else if (edge_conn->hs_ident) {
+          circ->hs_ident =
+            hs_ident_circuit_new(&edge_conn->hs_ident->identity_pk,
+                                 HS_IDENT_CIRCUIT_INTRO);
+        }
         if (circ->base_.purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND &&
             circ->base_.state == CIRCUIT_STATE_OPEN)
           circuit_has_opened(circ);
       }
     }
   } /* endif (!circ) */
+
+  /* We either found a good circuit, or launched a new circuit, or failed to
+   * do so. Report success, and delay. */
+
   if (circ) {
     /* Mark the circuit with the isolation fields for this connection.
      * When the circuit arrives, we'll clear these flags: this is
@@ -2188,7 +2597,7 @@ link_apconn_to_circ(entry_connection_t *apconn, origin_circuit_t *circ,
   log_debug(LD_APP|LD_CIRC, "attaching new conn to circ. n_circ_id %u.",
             (unsigned)circ->base_.n_circ_id);
   /* reset it, so we can measure circ timeouts */
-  ENTRY_TO_CONN(apconn)->timestamp_lastread = time(NULL);
+  ENTRY_TO_CONN(apconn)->timestamp_last_read_allowed = time(NULL);
   ENTRY_TO_EDGE_CONN(apconn)->next_stream = circ->p_streams;
   ENTRY_TO_EDGE_CONN(apconn)->on_circuit = TO_CIRCUIT(circ);
   /* assert_connection_ok(conn, time(NULL)); */
@@ -2198,8 +2607,7 @@ link_apconn_to_circ(entry_connection_t *apconn, origin_circuit_t *circ,
     /* We are attaching a stream to a rendezvous circuit.  That means
      * that an attempt to connect to a hidden service just
      * succeeded.  Tell rendclient.c. */
-    rend_client_note_connection_attempt_ended(
-                    ENTRY_TO_EDGE_CONN(apconn)->rend_data);
+    hs_client_note_connection_attempt_succeeded(ENTRY_TO_EDGE_CONN(apconn));
   }
 
   if (cpath) { /* we were given one; use it */
@@ -2223,6 +2631,8 @@ link_apconn_to_circ(entry_connection_t *apconn, origin_circuit_t *circ,
   /* See if we can use optimistic data on this circuit */
   if (optimistic_data_enabled() &&
       (circ->base_.purpose == CIRCUIT_PURPOSE_C_GENERAL ||
+       circ->base_.purpose == CIRCUIT_PURPOSE_C_HSDIR_GET ||
+       circ->base_.purpose == CIRCUIT_PURPOSE_S_HSDIR_POST ||
        circ->base_.purpose == CIRCUIT_PURPOSE_C_REND_JOINED))
     apconn->may_use_optimistic_data = 1;
   else
@@ -2326,7 +2736,9 @@ connection_ap_handshake_attach_chosen_circuit(entry_connection_t *conn,
 
   pathbias_count_use_attempt(circ);
 
+  /* Now, actually link the connection. */
   link_apconn_to_circ(conn, circ, cpath);
+
   tor_assert(conn->socks_request);
   if (conn->socks_request->command == SOCKS_COMMAND_CONNECT) {
     if (!conn->use_begindir)
@@ -2341,12 +2753,44 @@ connection_ap_handshake_attach_chosen_circuit(entry_connection_t *conn,
   return 1;
 }
 
-/** Try to find a safe live circuit for CONN_TYPE_AP connection conn. If
- * we don't find one: if conn cannot be handled by any known nodes,
- * warn and return -1 (conn needs to die, and is maybe already marked);
- * else launch new circuit (if necessary) and return 0.
- * Otherwise, associate conn with a safe live circuit, do the
- * right next step, and return 1.
+/**
+ * Return an appropriate circuit purpose for non-rend streams.
+ * We don't handle rends here because a rend stream triggers two
+ * circuit builds with different purposes, so it is handled elsewhere.
+ *
+ * This function just figures out what type of hsdir activity this is,
+ * and tells us. Everything else is general.
+ */
+static int
+connection_ap_get_nonrend_circ_purpose(const entry_connection_t *conn)
+{
+  const connection_t *base_conn = ENTRY_TO_CONN(conn);
+  tor_assert_nonfatal(!connection_edge_is_rendezvous_stream(
+                      ENTRY_TO_EDGE_CONN(conn)));
+
+  if (base_conn->linked_conn &&
+      base_conn->linked_conn->type == CONN_TYPE_DIR) {
+    /* Set a custom purpose for hsdir activity */
+    if (base_conn->linked_conn->purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2 ||
+       base_conn->linked_conn->purpose == DIR_PURPOSE_UPLOAD_HSDESC) {
+      return CIRCUIT_PURPOSE_S_HSDIR_POST;
+    } else if (base_conn->linked_conn->purpose
+                 == DIR_PURPOSE_FETCH_RENDDESC_V2 ||
+               base_conn->linked_conn->purpose
+                 == DIR_PURPOSE_FETCH_HSDESC) {
+      return CIRCUIT_PURPOSE_C_HSDIR_GET;
+    }
+  }
+
+  /* All other purposes are general for now */
+  return CIRCUIT_PURPOSE_C_GENERAL;
+}
+
+/** Try to find a safe live circuit for stream <b>conn</b>.  If we find one,
+ * attach the stream, send appropriate cells, and return 1.  Otherwise,
+ * try to launch new circuit(s) for the stream.  If we can launch
+ * circuits, return 0.  Otherwise, if we simply can't proceed with
+ * this stream, return -1. (conn needs to die, and is maybe already marked).
  */
 /* XXXX this function should mark for close whenever it returns -1;
  * its callers shouldn't have to worry about that. */
@@ -2365,6 +2809,7 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
 
   conn_age = (int)(time(NULL) - base_conn->timestamp_created);
 
+  /* Is this connection so old that we should give up on it? */
   if (conn_age >= get_options()->SocksTimeout) {
     int severity = (tor_addr_is_null(&base_conn->addr) && !base_conn->port) ?
       LOG_INFO : LOG_NOTICE;
@@ -2375,12 +2820,14 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
     return -1;
   }
 
+  /* We handle "general" (non-onion) connections much more straightforwardly.
+   */
   if (!connection_edge_is_rendezvous_stream(ENTRY_TO_EDGE_CONN(conn))) {
     /* we're a general conn */
     origin_circuit_t *circ=NULL;
 
     /* Are we linked to a dir conn that aims to fetch a consensus?
-     * We check here because this conn might no longer be needed. */
+     * We check here because the conn might no longer be needed. */
     if (base_conn->linked_conn &&
         base_conn->linked_conn->type == CONN_TYPE_DIR &&
         base_conn->linked_conn->purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
@@ -2398,8 +2845,11 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
       }
     }
 
+    /* If we have a chosen exit, we need to use a circuit that's
+     * open to that exit. See what exit we meant, and whether we can use it.
+     */
     if (conn->chosen_exit_name) {
-      const node_t *node = node_get_by_nickname(conn->chosen_exit_name, 1);
+      const node_t *node = node_get_by_nickname(conn->chosen_exit_name, 0);
       int opt = conn->chosen_exit_optional;
       if (!node && !want_onehop) {
         /* We ran into this warning when trying to extend a circuit to a
@@ -2411,6 +2861,7 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
                "Requested exit point '%s' is not known. %s.",
                conn->chosen_exit_name, opt ? "Trying others" : "Closing");
         if (opt) {
+          /* If we are allowed to ignore the .exit request, do so */
           conn->chosen_exit_optional = 0;
           tor_free(conn->chosen_exit_name);
           return 0;
@@ -2423,6 +2874,7 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
                "would refuse request. %s.",
                conn->chosen_exit_name, opt ? "Trying others" : "Closing");
         if (opt) {
+          /* If we are allowed to ignore the .exit request, do so */
           conn->chosen_exit_optional = 0;
           tor_free(conn->chosen_exit_name);
           return 0;
@@ -2431,20 +2883,28 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
       }
     }
 
-    /* find the circuit that we should use, if there is one. */
-    retval = circuit_get_open_circ_or_launch(
-        conn, CIRCUIT_PURPOSE_C_GENERAL, &circ);
-    if (retval < 1) // XXXX++ if we totally fail, this still returns 0 -RD
+    /* Find the circuit that we should use, if there is one. Otherwise
+     * launch it
+     */
+    retval = circuit_get_open_circ_or_launch(conn,
+            connection_ap_get_nonrend_circ_purpose(conn),
+            &circ);
+
+    if (retval < 1) {
+      /* We were either told "-1" (complete failure) or 0 (circuit in
+       * progress); we can't attach this stream yet. */
       return retval;
+    }
 
     log_debug(LD_APP|LD_CIRC,
               "Attaching apconn to circ %u (stream %d sec old).",
               (unsigned)circ->base_.n_circ_id, conn_age);
-    /* print the circ's path, so people can figure out which circs are
+    /* print the circ's path, so clients can figure out which circs are
      * sucking. */
     circuit_log_path(LOG_INFO,LD_APP|LD_CIRC,circ);
 
-    /* We have found a suitable circuit for our conn. Hurray. */
+    /* We have found a suitable circuit for our conn. Hurray.  Do
+     * the attachment. */
     return connection_ap_handshake_attach_chosen_circuit(conn, circ, NULL);
 
   } else { /* we're a rendezvous conn */
@@ -2462,9 +2922,10 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
       tor_assert(rendcirc);
       /* one is already established, attach */
       log_info(LD_REND,
-               "rend joined circ %u already here. attaching. "
-               "(stream %d sec old)",
-               (unsigned)rendcirc->base_.n_circ_id, conn_age);
+               "rend joined circ %u (id: %" PRIu32 ") already here. "
+               "Attaching. (stream %d sec old)",
+               (unsigned) TO_CIRCUIT(rendcirc)->n_circ_id,
+               rendcirc->global_identifier, conn_age);
       /* Mark rendezvous circuits as 'newly dirty' every time you use
        * them, since the process of rebuilding a rendezvous circ is so
        * expensive. There is a tradeoff between linkability and
@@ -2497,9 +2958,10 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
     if (rendcirc && (rendcirc->base_.purpose ==
                      CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED)) {
       log_info(LD_REND,
-               "pending-join circ %u already here, with intro ack. "
-               "Stalling. (stream %d sec old)",
-               (unsigned)rendcirc->base_.n_circ_id, conn_age);
+               "pending-join circ %u (id: %" PRIu32 ") already here, with "
+               "intro ack. Stalling. (stream %d sec old)",
+               (unsigned) TO_CIRCUIT(rendcirc)->n_circ_id,
+               rendcirc->global_identifier, conn_age);
       return 0;
     }
 
@@ -2511,10 +2973,13 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
     if (retval > 0) {
       /* one has already sent the intro. keep waiting. */
       tor_assert(introcirc);
-      log_info(LD_REND, "Intro circ %u present and awaiting ack (rend %u). "
-               "Stalling. (stream %d sec old)",
-               (unsigned)introcirc->base_.n_circ_id,
-               rendcirc ? (unsigned)rendcirc->base_.n_circ_id : 0,
+      log_info(LD_REND, "Intro circ %u (id: %" PRIu32 ") present and "
+                        "awaiting ACK. Rend circuit %u (id: %" PRIu32 "). "
+                        "Stalling. (stream %d sec old)",
+               (unsigned) TO_CIRCUIT(introcirc)->n_circ_id,
+               introcirc->global_identifier,
+               rendcirc ? (unsigned) TO_CIRCUIT(rendcirc)->n_circ_id : 0,
+               rendcirc ? rendcirc->global_identifier : 0,
                conn_age);
       return 0;
     }
@@ -2524,19 +2989,26 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
     if (rendcirc && introcirc &&
         rendcirc->base_.purpose == CIRCUIT_PURPOSE_C_REND_READY) {
       log_info(LD_REND,
-               "ready rend circ %u already here (no intro-ack yet on "
-               "intro %u). (stream %d sec old)",
-               (unsigned)rendcirc->base_.n_circ_id,
-               (unsigned)introcirc->base_.n_circ_id, conn_age);
+               "ready rend circ %u (id: %" PRIu32 ") already here. No"
+               "intro-ack yet on intro %u (id: %" PRIu32 "). "
+               "(stream %d sec old)",
+               (unsigned) TO_CIRCUIT(rendcirc)->n_circ_id,
+               rendcirc->global_identifier,
+               (unsigned) TO_CIRCUIT(introcirc)->n_circ_id,
+               introcirc->global_identifier, conn_age);
 
       tor_assert(introcirc->base_.purpose == CIRCUIT_PURPOSE_C_INTRODUCING);
       if (introcirc->base_.state == CIRCUIT_STATE_OPEN) {
-        log_info(LD_REND,"found open intro circ %u (rend %u); sending "
-                 "introduction. (stream %d sec old)",
-                 (unsigned)introcirc->base_.n_circ_id,
-                 (unsigned)rendcirc->base_.n_circ_id,
-                 conn_age);
-        switch (rend_client_send_introduction(introcirc, rendcirc)) {
+        int ret;
+        log_info(LD_REND, "Found open intro circ %u (id: %" PRIu32 "). "
+                          "Rend circuit %u (id: %" PRIu32 "); Sending "
+                          "introduction. (stream %d sec old)",
+                 (unsigned) TO_CIRCUIT(introcirc)->n_circ_id,
+                 introcirc->global_identifier,
+                 (unsigned) TO_CIRCUIT(rendcirc)->n_circ_id,
+                 rendcirc->global_identifier, conn_age);
+        ret = hs_client_send_introduce1(introcirc, rendcirc);
+        switch (ret) {
         case 0: /* success */
           rendcirc->base_.timestamp_dirty = time(NULL);
           introcirc->base_.timestamp_dirty = time(NULL);
@@ -2558,10 +3030,13 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
       }
     }
 
-    log_info(LD_REND, "Intro (%u) and rend (%u) circs are not both ready. "
-             "Stalling conn. (%d sec old)",
-             introcirc ? (unsigned)introcirc->base_.n_circ_id : 0,
-             rendcirc ? (unsigned)rendcirc->base_.n_circ_id : 0, conn_age);
+    log_info(LD_REND, "Intro %u (id: %" PRIu32 ") and rend circuit %u "
+                      "(id: %" PRIu32 ") circuits are not both ready. "
+                      "Stalling conn. (%d sec old)",
+             introcirc ? (unsigned) TO_CIRCUIT(introcirc)->n_circ_id : 0,
+             introcirc ? introcirc->global_identifier : 0,
+             rendcirc ? (unsigned) TO_CIRCUIT(rendcirc)->n_circ_id : 0,
+             rendcirc ? rendcirc->global_identifier : 0, conn_age);
     return 0;
   }
 }
@@ -2591,6 +3066,12 @@ circuit_change_purpose(circuit_t *circ, uint8_t new_purpose)
               circ->purpose,
               circuit_purpose_to_string(new_purpose),
               new_purpose);
+
+    /* Take specific actions if we are repurposing a hidden service circuit. */
+    if (circuit_purpose_is_hidden_service(circ->purpose) &&
+        !circuit_purpose_is_hidden_service(new_purpose)) {
+      hs_circ_cleanup(circ);
+    }
   }
 
   old_purpose = circ->purpose;
@@ -2622,3 +3103,40 @@ mark_circuit_unusable_for_new_conns(origin_circuit_t *circ)
   circ->unusable_for_new_conns = 1;
 }
 
+/**
+ * Add relay_body_len and RELAY_PAYLOAD_SIZE-relay_body_len to
+ * the valid delivered written fields and the overhead field,
+ * respectively.
+ */
+void
+circuit_sent_valid_data(origin_circuit_t *circ, uint16_t relay_body_len)
+{
+  if (!circ) return;
+
+  tor_assert_nonfatal(relay_body_len <= RELAY_PAYLOAD_SIZE);
+
+  circ->n_delivered_written_circ_bw =
+      tor_add_u32_nowrap(circ->n_delivered_written_circ_bw, relay_body_len);
+  circ->n_overhead_written_circ_bw =
+      tor_add_u32_nowrap(circ->n_overhead_written_circ_bw,
+                         RELAY_PAYLOAD_SIZE-relay_body_len);
+}
+
+/**
+ * Add relay_body_len and RELAY_PAYLOAD_SIZE-relay_body_len to
+ * the valid delivered read field and the overhead field,
+ * respectively.
+ */
+void
+circuit_read_valid_data(origin_circuit_t *circ, uint16_t relay_body_len)
+{
+  if (!circ) return;
+
+  tor_assert_nonfatal(relay_body_len <= RELAY_PAYLOAD_SIZE);
+
+  circ->n_delivered_read_circ_bw =
+      tor_add_u32_nowrap(circ->n_delivered_read_circ_bw, relay_body_len);
+  circ->n_overhead_read_circ_bw =
+      tor_add_u32_nowrap(circ->n_overhead_read_circ_bw,
+                         RELAY_PAYLOAD_SIZE-relay_body_len);
+}
diff --git a/src/or/circuituse.h b/src/core/or/circuituse.h
index 5973978c45..25588dbb11 100644
--- a/src/or/circuituse.h
+++ b/src/core/or/circuituse.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,6 +13,7 @@
 #define TOR_CIRCUITUSE_H
 
 void circuit_expire_building(void);
+void circuit_expire_waiting_for_better_guard(void);
 void circuit_remove_handled_ports(smartlist_t *needed_ports);
 int circuit_stream_is_being_handled(entry_connection_t *conn, uint16_t port,
                                     int min);
@@ -43,6 +44,9 @@ void circuit_build_failed(origin_circuit_t *circ);
 /** Flag to set when the last hop of a circuit doesn't need to be an
  * exit node. */
 #define CIRCLAUNCH_IS_INTERNAL    (1<<3)
+/** Flag to set when we are trying to launch a v3 rendezvous circuit. We need
+ *  to apply some additional filters on the node picked. */
+#define CIRCLAUNCH_IS_V3_RP (1<<4)
 origin_circuit_t *circuit_launch_by_extend_info(uint8_t purpose,
                                                 extend_info_t *info,
                                                 int flags);
@@ -59,5 +63,31 @@ int hostname_in_track_host_exits(const or_options_t *options,
                                  const char *address);
 void mark_circuit_unusable_for_new_conns(origin_circuit_t *circ);
 
-#endif
+int circuit_purpose_is_hidden_service(uint8_t);
+int circuit_should_use_vanguards(uint8_t);
+void circuit_sent_valid_data(origin_circuit_t *circ, uint16_t relay_body_len);
+void circuit_read_valid_data(origin_circuit_t *circ, uint16_t relay_body_len);
+
+#ifdef TOR_UNIT_TESTS
+/* Used only by circuituse.c and test_circuituse.c */
+
+STATIC int circuit_is_available_for_use(const circuit_t *circ);
+
+STATIC int needs_exit_circuits(time_t now,
+                               int *port_needs_uptime,
+                               int *port_needs_capacity);
+STATIC int needs_hs_server_circuits(time_t now,
+                                    int num_uptime_internal);
+
+STATIC int needs_hs_client_circuits(time_t now,
+                                    int *needs_uptime,
+                                    int *needs_capacity,
+                                    int num_internal,
+                                    int num_uptime_internal);
+
+STATIC int needs_circuits_for_build(int num);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* !defined(TOR_CIRCUITUSE_H) */
 
diff --git a/src/or/command.c b/src/core/or/command.c
index 3f5386c5a3..5fb6640c22 100644
--- a/src/or/command.c
+++ b/src/core/or/command.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -36,24 +36,32 @@
  *   callbacks registered in command_setup_channel(),
  *   called when channels are created in circuitbuild.c
  */
-#include "or.h"
-#include "channel.h"
-#include "circuitbuild.h"
-#include "circuitlist.h"
-#include "command.h"
-#include "connection.h"
-#include "connection_or.h"
-#include "config.h"
-#include "control.h"
-#include "cpuworker.h"
-#include "dos.h"
-#include "hibernate.h"
-#include "nodelist.h"
-#include "onion.h"
-#include "rephist.h"
-#include "relay.h"
-#include "router.h"
-#include "routerlist.h"
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/crypto/onion_crypto.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/cpuworker.h"
+#include "core/or/channel.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/command.h"
+#include "core/or/connection_or.h"
+#include "core/or/dos.h"
+#include "core/or/onion.h"
+#include "core/or/relay.h"
+#include "feature/control/control.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/routermode.h"
+#include "feature/stats/rephist.h"
+#include "lib/crypt_ops/crypto_util.h"
+
+#include "core/or/cell_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/var_cell_st.h"
 
 /** How many CELL_CREATE cells have we received, ever? */
 uint64_t stats_n_create_cells_processed = 0;
@@ -129,7 +137,7 @@ command_time_process_cell(cell_t *cell, channel_t *chan, int *time,
   }
   *time += time_passed;
 }
-#endif
+#endif /* defined(KEEP_TIMING_STATS) */
 
 /** Process a <b>cell</b> that was just received on <b>chan</b>. Keep internal
  * statistics about how many of each cell we've processed so far
@@ -166,7 +174,7 @@ command_process_cell(channel_t *chan, cell_t *cell)
     /* remember which second it is, for next time */
     current_second = now;
   }
-#endif
+#endif /* defined(KEEP_TIMING_STATS) */
 
 #ifdef KEEP_TIMING_STATS
 #define PROCESS_CELL(tp, cl, cn) STMT_BEGIN {                   \
@@ -174,9 +182,9 @@ command_process_cell(channel_t *chan, cell_t *cell)
     command_time_process_cell(cl, cn, & tp ## time ,            \
                               command_process_ ## tp ## _cell);  \
   } STMT_END
-#else
+#else /* !(defined(KEEP_TIMING_STATS)) */
 #define PROCESS_CELL(tp, cl, cn) command_process_ ## tp ## _cell(cl, cn)
-#endif
+#endif /* defined(KEEP_TIMING_STATS) */
 
   switch (cell->command) {
     case CELL_CREATE:
@@ -243,10 +251,10 @@ command_process_create_cell(cell_t *cell, channel_t *chan)
   tor_assert(chan);
 
   log_debug(LD_OR,
-            "Got a CREATE cell for circ_id %u on channel " U64_FORMAT
+            "Got a CREATE cell for circ_id %u on channel %"PRIu64
             " (%p)",
             (unsigned)cell->circ_id,
-            U64_PRINTF_ARG(chan->global_identifier), chan);
+            (chan->global_identifier), chan);
 
   /* First thing we do, even though the cell might be invalid, is inform the
    * DoS mitigation subsystem layer of this event. Validation is done by this
@@ -286,7 +294,7 @@ command_process_create_cell(cell_t *cell, channel_t *chan)
              "Received create cell but we're shutting down. Sending back "
              "destroy.");
     channel_send_destroy(cell->circ_id, chan,
-                               END_CIRC_REASON_HIBERNATING);
+                         END_CIRC_REASON_HIBERNATING);
     return;
   }
 
@@ -339,10 +347,15 @@ command_process_create_cell(cell_t *cell, channel_t *chan)
     return;
   }
 
+  if (!channel_is_client(chan)) {
+    /* remember create types we've seen, but don't remember them from
+     * clients, to be extra conservative about client statistics. */
+    rep_hist_note_circuit_handshake_requested(create_cell->handshake_type);
+  }
+
   if (create_cell->handshake_type != ONION_HANDSHAKE_TYPE_FAST) {
     /* hand it off to the cpuworkers, and then return. */
-    if (connection_or_digest_is_known_relay(chan->identity_digest))
-      rep_hist_note_circuit_handshake_requested(create_cell->handshake_type);
+
     if (assign_onionskin_to_cpuworker(circ, create_cell) < 0) {
       log_debug(LD_GENERAL,"Failed to hand off onionskin. Closing.");
       circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_RESOURCELIMIT);
@@ -375,7 +388,8 @@ command_process_create_cell(cell_t *cell, channel_t *chan)
     created_cell.handshake_len = len;
 
     if (onionskin_answer(circ, &created_cell,
-                         (const char *)keys, rend_circ_nonce)<0) {
+                         (const char *)keys, sizeof(keys),
+                         rend_circ_nonce)<0) {
       log_warn(LD_OR,"Failed to reply to CREATE_FAST cell. Closing.");
       circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
       return;
@@ -468,6 +482,8 @@ command_process_relay_cell(cell_t *cell, channel_t *chan)
   const or_options_t *options = get_options();
   circuit_t *circ;
   int reason, direction;
+  uint32_t orig_delivered_bw = 0;
+  uint32_t orig_overhead_bw = 0;
 
   circ = circuit_get_by_circid_channel(cell->circ_id, chan);
 
@@ -489,6 +505,26 @@ command_process_relay_cell(cell_t *cell, channel_t *chan)
     /* if we're a relay and treating connections with recent local
      * traffic better, then this is one of them. */
     channel_timestamp_client(chan);
+
+    /* Count all circuit bytes here for control port accuracy. We want
+     * to count even invalid/dropped relay cells, hence counting
+     * before the recognized check and the connection_edge_process_relay
+     * cell checks.
+     */
+    origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+
+    /* Count the payload bytes only. We don't care about cell headers */
+    ocirc->n_read_circ_bw = tor_add_u32_nowrap(ocirc->n_read_circ_bw,
+                                               CELL_PAYLOAD_SIZE);
+
+    /* Stash the original delivered and overhead values. These values are
+     * updated by circuit_read_valid_data() during cell processing by
+     * connection_edge_process_relay_cell(), called from
+     * circuit_receive_relay_cell() below. If they do not change, we inform
+     * the control port about dropped cells immediately after the call
+     * to circuit_receive_relay_cell() below. */
+    orig_delivered_bw = ocirc->n_delivered_read_circ_bw;
+    orig_overhead_bw = ocirc->n_overhead_read_circ_bw;
   }
 
   if (!CIRCUIT_IS_ORIGIN(circ) &&
@@ -512,6 +548,8 @@ command_process_relay_cell(cell_t *cell, channel_t *chan)
                (unsigned)cell->circ_id);
       if (CIRCUIT_IS_ORIGIN(circ)) {
         circuit_log_path(LOG_WARN, LD_OR, TO_ORIGIN_CIRCUIT(circ));
+        /* Always emit a bandwidth event for closed circs */
+        control_event_circ_bandwidth_used_for_circ(TO_ORIGIN_CIRCUIT(circ));
       } else if (circ->n_chan) {
         log_warn(LD_OR, " upstream=%s",
                  channel_get_actual_remote_descr(circ->n_chan));
@@ -537,9 +575,33 @@ command_process_relay_cell(cell_t *cell, channel_t *chan)
     log_fn(LOG_PROTOCOL_WARN,LD_PROTOCOL,"circuit_receive_relay_cell "
            "(%s) failed. Closing.",
            direction==CELL_DIRECTION_OUT?"forward":"backward");
+    /* Always emit a bandwidth event for closed circs */
+    if (CIRCUIT_IS_ORIGIN(circ)) {
+      control_event_circ_bandwidth_used_for_circ(TO_ORIGIN_CIRCUIT(circ));
+    }
     circuit_mark_for_close(circ, -reason);
   }
 
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+
+    /* If neither the delivered nor overhead values changed, this cell
+     * was dropped due to being invalid by one of the error codepaths in
+     * connection_edge_process_relay_cell(), called by
+     * circuit_receive_relay_cell().
+     *
+     * Valid cells, on the other hand, call circuit_read_valid_data()
+     * to update these values upon processing them.
+     *
+     * So, if the values are the same as those stored above,
+     * emit a control port event for CIRC_BW, so the controller can
+     * react quickly to invalid cells. */
+    if (orig_delivered_bw == ocirc->n_delivered_read_circ_bw &&
+        orig_overhead_bw == ocirc->n_overhead_read_circ_bw) {
+      control_event_circ_bandwidth_used_for_circ(ocirc);
+    }
+  }
+
   /* If this is a cell in an RP circuit, count it as part of the
      hidden service stats */
   if (options->HiddenServiceStatistics &&
@@ -639,4 +701,3 @@ command_setup_listener(channel_listener_t *listener)
 
   channel_listener_set_listener_fn(listener, command_handle_incoming_channel);
 }
-
diff --git a/src/or/command.h b/src/core/or/command.h
index 12cda6a463..8c90e1de6f 100644
--- a/src/or/command.h
+++ b/src/core/or/command.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,7 +12,7 @@
 #ifndef TOR_COMMAND_H
 #define TOR_COMMAND_H
 
-#include "channel.h"
+#include "core/or/channel.h"
 
 void command_process_cell(channel_t *chan, cell_t *cell);
 void command_process_var_cell(channel_t *chan, var_cell_t *cell);
@@ -27,5 +27,5 @@ extern uint64_t stats_n_created_cells_processed;
 extern uint64_t stats_n_relay_cells_processed;
 extern uint64_t stats_n_destroy_cells_processed;
 
-#endif
+#endif /* !defined(TOR_COMMAND_H) */
 
diff --git a/src/or/connection_edge.c b/src/core/or/connection_edge.c
index 5638d9a1be..90991107dc 100644
--- a/src/or/connection_edge.c
+++ b/src/core/or/connection_edge.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,7 +29,7 @@
  *   <li>DNS lookup streams, created on the exit side in response to
  *     a RELAY_RESOLVE cell from a client.
  *   <li>Tunneled directory streams, created on the directory cache side
- *     in response to a RELAY_BEGINDIR cell.  These streams attach directly
+ *     in response to a RELAY_BEGIN_DIR cell.  These streams attach directly
  *     to a dir_connection_t object without ever using TCP.
  *   </ul>
  *
@@ -55,39 +55,62 @@
  **/
 #define CONNECTION_EDGE_PRIVATE
 
-#include "or.h"
-
-#include "backtrace.h"
-
-#include "addressmap.h"
-#include "buffers.h"
-#include "channel.h"
-#include "circpathbias.h"
-#include "circuitlist.h"
-#include "circuituse.h"
-#include "config.h"
-#include "connection.h"
-#include "connection_edge.h"
-#include "connection_or.h"
-#include "control.h"
-#include "dns.h"
-#include "dnsserv.h"
-#include "directory.h"
-#include "dirserv.h"
-#include "hibernate.h"
-#include "main.h"
-#include "nodelist.h"
-#include "policies.h"
-#include "reasons.h"
-#include "relay.h"
-#include "rendclient.h"
-#include "rendcommon.h"
-#include "rendservice.h"
-#include "rephist.h"
-#include "router.h"
-#include "routerlist.h"
-#include "routerset.h"
-#include "circuitbuild.h"
+#include "core/or/or.h"
+
+#include "lib/err/backtrace.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/channel.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "core/or/policies.h"
+#include "core/or/reasons.h"
+#include "core/or/relay.h"
+#include "core/proto/proto_http.h"
+#include "core/proto/proto_socks.h"
+#include "feature/client/addressmap.h"
+#include "feature/client/circpathbias.h"
+#include "feature/client/dnsserv.h"
+#include "feature/control/control.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dircommon/directory.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/hs/hs_cache.h"
+#include "feature/hs/hs_circuit.h"
+#include "feature/hs/hs_client.h"
+#include "feature/hs/hs_common.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/relay/dns.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+#include "feature/rend/rendclient.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/rend/rendservice.h"
+#include "feature/stats/predict_ports.h"
+#include "feature/stats/rephist.h"
+#include "lib/container/buffers.h"
+#include "lib/crypt_ops/crypto_util.h"
+
+#include "core/or/cell_st.h"
+#include "core/or/cpath_build_state_st.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/extend_info_st.h"
+#include "feature/nodelist/node_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/half_edge_st.h"
+#include "core/or/socks_request_st.h"
+#include "lib/evloop/compat_libevent.h"
 
 #ifdef HAVE_LINUX_TYPES_H
 #include <linux/types.h>
@@ -108,6 +131,16 @@
 #define TRANS_NETFILTER
 #define TRANS_NETFILTER_IPV6
 #endif
+#endif /* defined(HAVE_LINUX_NETFILTER_IPV6_IP6_TABLES_H) */
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_SYS_IOCTL_H
+#include <sys/ioctl.h>
+#endif
+#ifdef HAVE_SYS_PARAM_H
+#include <sys/param.h>
 #endif
 
 #if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H)
@@ -129,6 +162,30 @@ static int connection_exit_connect_dir(edge_connection_t *exitconn);
 static int consider_plaintext_ports(entry_connection_t *conn, uint16_t port);
 static int connection_ap_supports_optimistic_data(const entry_connection_t *);
 
+/** Convert a connection_t* to an edge_connection_t*; assert if the cast is
+ * invalid. */
+edge_connection_t *
+TO_EDGE_CONN(connection_t *c)
+{
+  tor_assert(c->magic == EDGE_CONNECTION_MAGIC ||
+             c->magic == ENTRY_CONNECTION_MAGIC);
+  return DOWNCAST(edge_connection_t, c);
+}
+
+entry_connection_t *
+TO_ENTRY_CONN(connection_t *c)
+{
+  tor_assert(c->magic == ENTRY_CONNECTION_MAGIC);
+  return (entry_connection_t*) SUBTYPE_P(c, entry_connection_t, edge_.base_);
+}
+
+entry_connection_t *
+EDGE_TO_ENTRY_CONN(edge_connection_t *c)
+{
+  tor_assert(c->base_.magic == ENTRY_CONNECTION_MAGIC);
+  return (entry_connection_t*) SUBTYPE_P(c, entry_connection_t, edge_);
+}
+
 /** An AP stream has failed/finished. If it hasn't already sent back
  * a socks reply, send one now (based on endreason). Also set
  * has_sent_end to 1, and mark the conn.
@@ -151,7 +208,9 @@ connection_mark_unattached_ap_,(entry_connection_t *conn, int endreason,
    * but we should fix it someday anyway. */
   if ((edge_conn->on_circuit != NULL || edge_conn->edge_has_sent_end) &&
       connection_edge_is_rendezvous_stream(edge_conn)) {
-    rend_client_note_connection_attempt_ended(edge_conn->rend_data);
+    if (edge_conn->rend_data) {
+      rend_client_note_connection_attempt_ended(edge_conn->rend_data);
+    }
   }
 
   if (base_conn->marked_for_close) {
@@ -235,6 +294,11 @@ connection_edge_process_inbuf(edge_connection_t *conn, int package_partial)
         return -1;
       }
       return 0;
+    case AP_CONN_STATE_HTTP_CONNECT_WAIT:
+      if (connection_ap_process_http_connect(EDGE_TO_ENTRY_CONN(conn)) < 0) {
+        return -1;
+      }
+      return 0;
     case AP_CONN_STATE_OPEN:
     case EXIT_CONN_STATE_OPEN:
       if (connection_edge_package_raw_inbuf(conn, package_partial, NULL) < 0) {
@@ -325,10 +389,41 @@ relay_send_end_cell_from_edge(streamid_t stream_id, circuit_t *circ,
 
   payload[0] = (char) reason;
 
+  /* Note: we have to use relay_send_command_from_edge here, not
+   * connection_edge_end or connection_edge_send_command, since those require
+   * that we have a stream connected to a circuit, and we don't connect to a
+   * circuit until we have a pending/successful resolve. */
   return relay_send_command_from_edge(stream_id, circ, RELAY_COMMAND_END,
                                       payload, 1, cpath_layer);
 }
 
+/* If the connection <b>conn</b> is attempting to connect to an external
+ * destination that is an hidden service and the reason is a connection
+ * refused or timeout, log it so the operator can take appropriate actions.
+ * The log statement is a rate limited warning. */
+static void
+warn_if_hs_unreachable(const edge_connection_t *conn, uint8_t reason)
+{
+  tor_assert(conn);
+
+  if (conn->base_.type == CONN_TYPE_EXIT &&
+      connection_edge_is_rendezvous_stream(conn) &&
+      (reason == END_STREAM_REASON_CONNECTREFUSED ||
+       reason == END_STREAM_REASON_TIMEOUT)) {
+#define WARN_FAILED_HS_CONNECTION 300
+    static ratelim_t warn_limit = RATELIM_INIT(WARN_FAILED_HS_CONNECTION);
+    char *m;
+    if ((m = rate_limit_log(&warn_limit, approx_time()))) {
+      log_warn(LD_EDGE, "Onion service connection to %s failed (%s)",
+               (conn->base_.socket_family == AF_UNIX) ?
+               safe_str(conn->base_.address) :
+               safe_str(fmt_addrport(&conn->base_.addr, conn->base_.port)),
+               stream_end_reason_to_string(reason));
+      tor_free(m);
+    }
+  }
+}
+
 /** Send a relay end cell from stream <b>conn</b> down conn's circuit, and
  * remember that we've done so.  If this is not a client connection, set the
  * relay end cell's reason for closing as <b>reason</b>.
@@ -384,8 +479,17 @@ connection_edge_end(edge_connection_t *conn, uint8_t reason)
   if (circ && !circ->marked_for_close) {
     log_debug(LD_EDGE,"Sending end on conn (fd "TOR_SOCKET_T_FORMAT").",
               conn->base_.s);
+
+    if (CIRCUIT_IS_ORIGIN(circ)) {
+      origin_circuit_t *origin_circ = TO_ORIGIN_CIRCUIT(circ);
+      connection_half_edge_add(conn, origin_circ);
+    }
+
     connection_edge_send_command(conn, RELAY_COMMAND_END,
                                  payload, payload_len);
+    /* We'll log warn if the connection was an hidden service and couldn't be
+     * made because the service wasn't available. */
+    warn_if_hs_unreachable(conn, control_reason);
   } else {
     log_debug(LD_EDGE,"No circ to send end on conn "
               "(fd "TOR_SOCKET_T_FORMAT").",
@@ -397,6 +501,236 @@ connection_edge_end(edge_connection_t *conn, uint8_t reason)
   return 0;
 }
 
+/**
+ * Helper function for bsearch.
+ *
+ * As per smartlist_bsearch, return < 0 if key preceeds member,
+ * > 0 if member preceeds key, and 0 if they are equal.
+ *
+ * This is equivalent to subtraction of the values of key - member
+ * (why does no one ever say that explicitly?).
+ */
+static int
+connection_half_edge_compare_bsearch(const void *key, const void **member)
+{
+  const half_edge_t *e2;
+  tor_assert(key);
+  tor_assert(member && *(half_edge_t**)member);
+  e2 = *(const half_edge_t **)member;
+
+  return *(const streamid_t*)key - e2->stream_id;
+}
+
+/** Total number of half_edge_t objects allocated */
+static size_t n_half_conns_allocated = 0;
+
+/**
+ * Add a half-closed connection to the list, to watch for activity.
+ *
+ * These connections are removed from the list upon receiving an end
+ * cell.
+ */
+STATIC void
+connection_half_edge_add(const edge_connection_t *conn,
+                         origin_circuit_t *circ)
+{
+  half_edge_t *half_conn = NULL;
+  int insert_at = 0;
+  int ignored;
+
+  /* Double-check for re-insertion. This should not happen,
+   * but this check is cheap compared to the sort anyway */
+  if (connection_half_edge_find_stream_id(circ->half_streams,
+                                          conn->stream_id)) {
+    log_warn(LD_BUG, "Duplicate stream close for stream %d on circuit %d",
+             conn->stream_id, circ->global_identifier);
+    return;
+  }
+
+  half_conn = tor_malloc_zero(sizeof(half_edge_t));
+  ++n_half_conns_allocated;
+
+  if (!circ->half_streams) {
+    circ->half_streams = smartlist_new();
+  }
+
+  half_conn->stream_id = conn->stream_id;
+
+  // How many sendme's should I expect?
+  half_conn->sendmes_pending =
+   (STREAMWINDOW_START-conn->package_window)/STREAMWINDOW_INCREMENT;
+
+   // Is there a connected cell pending?
+  half_conn->connected_pending = conn->base_.state ==
+      AP_CONN_STATE_CONNECT_WAIT;
+
+  /* Data should only arrive if we're not waiting on a resolved cell.
+   * It can arrive after waiting on connected, because of optimistic
+   * data. */
+  if (conn->base_.state != AP_CONN_STATE_RESOLVE_WAIT) {
+    // How many more data cells can arrive on this id?
+    half_conn->data_pending = conn->deliver_window;
+  }
+
+  insert_at = smartlist_bsearch_idx(circ->half_streams, &half_conn->stream_id,
+                                    connection_half_edge_compare_bsearch,
+                                    &ignored);
+  smartlist_insert(circ->half_streams, insert_at, half_conn);
+}
+
+/** Release space held by <b>he</b> */
+void
+half_edge_free_(half_edge_t *he)
+{
+  if (!he)
+    return;
+  --n_half_conns_allocated;
+  tor_free(he);
+}
+
+/** Return the number of bytes devoted to storing info on half-open streams. */
+size_t
+half_streams_get_total_allocation(void)
+{
+  return n_half_conns_allocated * sizeof(half_edge_t);
+}
+
+/**
+ * Find a stream_id_t in the list in O(lg(n)).
+ *
+ * Returns NULL if the list is empty or element is not found.
+ * Returns a pointer to the element if found.
+ */
+STATIC half_edge_t *
+connection_half_edge_find_stream_id(const smartlist_t *half_conns,
+                                    streamid_t stream_id)
+{
+  if (!half_conns)
+    return NULL;
+
+  return smartlist_bsearch(half_conns, &stream_id,
+                           connection_half_edge_compare_bsearch);
+}
+
+/**
+ * Check if this stream_id is in a half-closed state. If so,
+ * check if it still has data cells pending, and decrement that
+ * window if so.
+ *
+ * Return 1 if the data window was not empty.
+ * Return 0 otherwise.
+ */
+int
+connection_half_edge_is_valid_data(const smartlist_t *half_conns,
+                                   streamid_t stream_id)
+{
+  half_edge_t *half = connection_half_edge_find_stream_id(half_conns,
+                                                          stream_id);
+
+  if (!half)
+    return 0;
+
+  if (half->data_pending > 0) {
+    half->data_pending--;
+    return 1;
+  }
+
+  return 0;
+}
+
+/**
+ * Check if this stream_id is in a half-closed state. If so,
+ * check if it still has a connected cell pending, and decrement
+ * that window if so.
+ *
+ * Return 1 if the connected window was not empty.
+ * Return 0 otherwise.
+ */
+int
+connection_half_edge_is_valid_connected(const smartlist_t *half_conns,
+                                        streamid_t stream_id)
+{
+  half_edge_t *half = connection_half_edge_find_stream_id(half_conns,
+                                                          stream_id);
+
+  if (!half)
+    return 0;
+
+  if (half->connected_pending) {
+    half->connected_pending = 0;
+    return 1;
+  }
+
+  return 0;
+}
+
+/**
+ * Check if this stream_id is in a half-closed state. If so,
+ * check if it still has sendme cells pending, and decrement that
+ * window if so.
+ *
+ * Return 1 if the sendme window was not empty.
+ * Return 0 otherwise.
+ */
+int
+connection_half_edge_is_valid_sendme(const smartlist_t *half_conns,
+                                     streamid_t stream_id)
+{
+  half_edge_t *half = connection_half_edge_find_stream_id(half_conns,
+                                                          stream_id);
+
+  if (!half)
+    return 0;
+
+  if (half->sendmes_pending > 0) {
+    half->sendmes_pending--;
+    return 1;
+  }
+
+  return 0;
+}
+
+/**
+ * Check if this stream_id is in a half-closed state. If so, remove
+ * it from the list. No other data should come after the END cell.
+ *
+ * Return 1 if stream_id was in half-closed state.
+ * Return 0 otherwise.
+ */
+int
+connection_half_edge_is_valid_end(smartlist_t *half_conns,
+                                  streamid_t stream_id)
+{
+  half_edge_t *half;
+  int found, remove_idx;
+
+  if (!half_conns)
+    return 0;
+
+  remove_idx = smartlist_bsearch_idx(half_conns, &stream_id,
+                                    connection_half_edge_compare_bsearch,
+                                    &found);
+  if (!found)
+    return 0;
+
+  half = smartlist_get(half_conns, remove_idx);
+  smartlist_del_keeporder(half_conns, remove_idx);
+  half_edge_free(half);
+  return 1;
+}
+
+/**
+ * Streams that were used to send a RESOLVE cell are closed
+ * when they get the RESOLVED, without an end. So treat
+ * a RESOLVED just like an end, and remove from the list.
+ */
+int
+connection_half_edge_is_valid_resolved(smartlist_t *half_conns,
+                                       streamid_t stream_id)
+{
+  return connection_half_edge_is_valid_end(half_conns, stream_id);
+}
+
 /** An error has just occurred on an operation on an edge connection
  * <b>conn</b>.  Extract the errno; convert it to an end reason, and send an
  * appropriate relay end cell to the other end of the connection's circuit.
@@ -454,6 +788,7 @@ connection_edge_finished_flushing(edge_connection_t *conn)
     case AP_CONN_STATE_CONNECT_WAIT:
     case AP_CONN_STATE_CONTROLLER_WAIT:
     case AP_CONN_STATE_RESOLVE_WAIT:
+    case AP_CONN_STATE_HTTP_CONNECT_WAIT:
       return 0;
     default:
       log_warn(LD_BUG, "Called in unexpected state %d.",conn->base_.state);
@@ -505,6 +840,46 @@ connected_cell_format_payload(uint8_t *payload_out,
   return connected_payload_len;
 }
 
+/* This is an onion service client connection: Export the client circuit ID
+ * according to the HAProxy proxy protocol. */
+STATIC void
+export_hs_client_circuit_id(edge_connection_t *edge_conn,
+                            hs_circuit_id_protocol_t protocol)
+{
+  /* We only support HAProxy right now. */
+  if (protocol != HS_CIRCUIT_ID_PROTOCOL_HAPROXY)
+    return;
+
+  char *buf = NULL;
+  const char dst_ipv6[] = "::1";
+  /* See RFC4193 regarding fc00::/7 */
+  const char src_ipv6_prefix[] = "fc00:dead:beef:4dad:";
+  uint16_t dst_port = 0;
+  uint16_t src_port = 1; /* default value */
+  uint32_t gid = 0; /* default value */
+
+  /* Generate a GID and source port for this client */
+  if (edge_conn->on_circuit != NULL) {
+    gid = TO_ORIGIN_CIRCUIT(edge_conn->on_circuit)->global_identifier;
+    src_port = gid & 0x0000ffff;
+  }
+
+  /* Grab the original dest port from the hs ident */
+  if (edge_conn->hs_ident) {
+    dst_port = edge_conn->hs_ident->orig_virtual_port;
+  }
+
+  /* Build the string */
+  tor_asprintf(&buf, "PROXY TCP6 %s:%x:%x %s %d %d\r\n",
+               src_ipv6_prefix,
+               gid >> 16, gid & 0x0000ffff,
+               dst_ipv6, src_port, dst_port);
+
+  connection_buf_add(buf, strlen(buf), TO_CONN(edge_conn));
+
+  tor_free(buf);
+}
+
 /** Connected handler for exit connections: start writing pending
  * data, deliver 'CONNECTED' relay cells as appropriate, and check
  * any pending data that may have been received. */
@@ -525,6 +900,7 @@ connection_edge_finished_connecting(edge_connection_t *edge_conn)
   rep_hist_note_exit_stream_opened(conn->port);
 
   conn->state = EXIT_CONN_STATE_OPEN;
+
   connection_watch_events(conn, READ_EVENT); /* stop writing, keep reading */
   if (connection_get_outbuf_len(conn)) /* in case there are any queued relay
                                         * cells */
@@ -562,6 +938,12 @@ static smartlist_t *pending_entry_connections = NULL;
 
 static int untried_pending_connections = 0;
 
+/**
+ * Mainloop event to tell us to scan for pending connections that can
+ * be attached.
+ */
+static mainloop_event_t *attach_pending_entry_connections_ev = NULL;
+
 /** Common code to connection_(ap|exit)_about_to_close. */
 static void
 connection_edge_about_to_close(edge_connection_t *edge_conn)
@@ -616,7 +998,7 @@ connection_ap_about_to_close(entry_connection_t *entry_conn)
     connection_ap_warn_and_unmark_if_pending_circ(entry_conn,
                                                   "about_to_close");
   }
-#endif
+#endif /* 1 */
 
   control_event_stream_bandwidth(edge_conn);
   control_event_stream_status(entry_conn, STREAM_EVENT_CLOSED,
@@ -690,7 +1072,7 @@ connection_ap_expire_beginning(void)
     /* if it's an internal linked connection, don't yell its status. */
     severity = (tor_addr_is_null(&base_conn->addr) && !base_conn->port)
       ? LOG_INFO : LOG_NOTICE;
-    seconds_idle = (int)( now - base_conn->timestamp_lastread );
+    seconds_idle = (int)( now - base_conn->timestamp_last_read_allowed );
     seconds_since_born = (int)( now - base_conn->timestamp_created );
 
     if (base_conn->state == AP_CONN_STATE_OPEN)
@@ -743,7 +1125,10 @@ connection_ap_expire_beginning(void)
       }
       continue;
     }
+
     if (circ->purpose != CIRCUIT_PURPOSE_C_GENERAL &&
+        circ->purpose != CIRCUIT_PURPOSE_C_HSDIR_GET &&
+        circ->purpose != CIRCUIT_PURPOSE_S_HSDIR_POST &&
         circ->purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT &&
         circ->purpose != CIRCUIT_PURPOSE_PATH_BIAS_TESTING) {
       log_warn(LD_BUG, "circuit->purpose == CIRCUIT_PURPOSE_C_GENERAL failed. "
@@ -773,7 +1158,7 @@ connection_ap_expire_beginning(void)
     mark_circuit_unusable_for_new_conns(TO_ORIGIN_CIRCUIT(circ));
 
     /* give our stream another 'cutoff' seconds to try */
-    conn->base_.timestamp_lastread += cutoff;
+    conn->base_.timestamp_last_read_allowed += cutoff;
     if (entry_conn->num_socks_retries < 250) /* avoid overflow */
       entry_conn->num_socks_retries++;
     /* move it back into 'pending' state, and try to attach. */
@@ -826,12 +1211,13 @@ connection_ap_rescan_and_attach_pending(void)
     entry_conn->marked_pending_circ_line = 0;   \
     entry_conn->marked_pending_circ_file = 0;   \
   } while (0)
-#else
+#else /* !(defined(DEBUGGING_17659)) */
 #define UNMARK() do { } while (0)
-#endif
+#endif /* defined(DEBUGGING_17659) */
 
 /** Tell any AP streams that are listed as waiting for a new circuit to try
- * again, either attaching to an available circ or launching a new one.
+ * again.  If there is an available circuit for a stream, attach it. Otherwise,
+ * launch a new circuit.
  *
  * If <b>retry</b> is false, only check the list if it contains at least one
  * streams that we have not yet tried to attach to a circuit.
@@ -846,8 +1232,9 @@ connection_ap_attach_pending(int retry)
   if (untried_pending_connections == 0 && !retry)
     return;
 
-  /* Don't allow modifications to pending_entry_connections while we are
-   * iterating over it. */
+  /* Don't allow any modifications to list while we are iterating over
+   * it.  We'll put streams back on this list if we can't attach them
+   * immediately. */
   smartlist_t *pending = pending_entry_connections;
   pending_entry_connections = smartlist_new();
 
@@ -866,9 +1253,7 @@ connection_ap_attach_pending(int retry)
       continue;
     }
     if (conn->state != AP_CONN_STATE_CIRCUIT_WAIT) {
-      // XXXX 030 -- this is downgraded in 0.2.9, since we apparently
-      // XXXX are running into it in practice.  It's harmless.
-      log_info(LD_BUG, "%p is no longer in circuit_wait. Its current state "
+      log_warn(LD_BUG, "%p is no longer in circuit_wait. Its current state "
                "is %s. Why is it on pending_entry_connections?",
                entry_conn,
                conn_state_to_string(conn->type, conn->state));
@@ -876,6 +1261,7 @@ connection_ap_attach_pending(int retry)
       continue;
     }
 
+    /* Okay, we're through the sanity checks. Try to handle this stream. */
     if (connection_ap_handshake_attach_circuit(entry_conn) < 0) {
       if (!conn->marked_for_close)
         connection_mark_unattached_ap(entry_conn,
@@ -885,12 +1271,17 @@ connection_ap_attach_pending(int retry)
     if (! conn->marked_for_close &&
         conn->type == CONN_TYPE_AP &&
         conn->state == AP_CONN_STATE_CIRCUIT_WAIT) {
+      /* Is it still waiting for a circuit? If so, we didn't attach it,
+       * so it's still pending.  Put it back on the list.
+       */
       if (!smartlist_contains(pending_entry_connections, entry_conn)) {
         smartlist_add(pending_entry_connections, entry_conn);
         continue;
       }
     }
 
+    /* If we got here, then we either closed the connection, or
+     * we attached it. */
     UNMARK();
   } SMARTLIST_FOREACH_END(entry_conn);
 
@@ -898,6 +1289,14 @@ connection_ap_attach_pending(int retry)
   untried_pending_connections = 0;
 }
 
+static void
+attach_pending_entry_connections_cb(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  (void)arg;
+  connection_ap_attach_pending(0);
+}
+
 /** Mark <b>entry_conn</b> as needing to get attached to a circuit.
  *
  * And <b>entry_conn</b> must be in AP_CONN_STATE_CIRCUIT_WAIT,
@@ -915,9 +1314,13 @@ connection_ap_mark_as_pending_circuit_(entry_connection_t *entry_conn,
   if (conn->marked_for_close)
     return;
 
-  if (PREDICT_UNLIKELY(NULL == pending_entry_connections))
+  if (PREDICT_UNLIKELY(NULL == pending_entry_connections)) {
     pending_entry_connections = smartlist_new();
-
+  }
+  if (PREDICT_UNLIKELY(NULL == attach_pending_entry_connections_ev)) {
+    attach_pending_entry_connections_ev = mainloop_event_postloop_new(
+                                  attach_pending_entry_connections_cb, NULL);
+  }
   if (PREDICT_UNLIKELY(smartlist_contains(pending_entry_connections,
                                           entry_conn))) {
     log_warn(LD_BUG, "What?? pending_entry_connections already contains %p! "
@@ -928,7 +1331,7 @@ connection_ap_mark_as_pending_circuit_(entry_connection_t *entry_conn,
     log_warn(LD_BUG, "(Previously called from %s:%d.)\n",
              f2 ? f2 : "<NULL>",
              entry_conn->marked_pending_circ_line);
-#endif
+#endif /* defined(DEBUGGING_17659) */
     log_backtrace(LOG_WARN, LD_BUG, "To debug, this may help");
     return;
   }
@@ -941,14 +1344,7 @@ connection_ap_mark_as_pending_circuit_(entry_connection_t *entry_conn,
   untried_pending_connections = 1;
   smartlist_add(pending_entry_connections, entry_conn);
 
-  /* Work-around for bug 19969: we handle pending_entry_connections at
-   * the end of run_main_loop_once(), but in many cases that function will
-   * take a very long time, if ever, to finish its call to event_base_loop().
-   *
-   * So the fix is to tell it right now that it ought to finish its loop at
-   * its next available opportunity.
-   */
-  tell_event_loop_to_finish();
+  mainloop_event_activate(attach_pending_entry_connections_ev);
 }
 
 /** Mark <b>entry_conn</b> as no longer waiting for a circuit. */
@@ -961,6 +1357,21 @@ connection_ap_mark_as_non_pending_circuit(entry_connection_t *entry_conn)
   smartlist_remove(pending_entry_connections, entry_conn);
 }
 
+/** Mark <b>entry_conn</b> as waiting for a rendezvous descriptor. This
+ * function will remove the entry connection from the waiting for a circuit
+ * list (pending_entry_connections).
+ *
+ * This pattern is used across the code base because a connection in state
+ * AP_CONN_STATE_RENDDESC_WAIT must not be in the pending list. */
+void
+connection_ap_mark_as_waiting_for_renddesc(entry_connection_t *entry_conn)
+{
+  tor_assert(entry_conn);
+
+  connection_ap_mark_as_non_pending_circuit(entry_conn);
+  ENTRY_TO_CONN(entry_conn)->state = AP_CONN_STATE_RENDDESC_WAIT;
+}
+
 /* DOCDOC */
 void
 connection_ap_warn_and_unmark_if_pending_circ(entry_connection_t *entry_conn,
@@ -1036,7 +1447,8 @@ circuit_discard_optional_exit_enclaves(extend_info_t *info)
     if (!entry_conn->chosen_exit_optional &&
         !entry_conn->chosen_exit_retries)
       continue;
-    r1 = node_get_by_nickname(entry_conn->chosen_exit_name, 0);
+    r1 = node_get_by_nickname(entry_conn->chosen_exit_name,
+                              NNF_NO_WARN_UNNAMED);
     r2 = node_get_by_id(info->identity_digest);
     if (!r1 || !r2 || r1 != r2)
       continue;
@@ -1076,7 +1488,7 @@ connection_ap_detach_retriable(entry_connection_t *conn,
                                int reason)
 {
   control_event_stream_status(conn, STREAM_EVENT_FAILED_RETRIABLE, reason);
-  ENTRY_TO_CONN(conn)->timestamp_lastread = time(NULL);
+  ENTRY_TO_CONN(conn)->timestamp_last_read_allowed = time(NULL);
 
   /* Roll back path bias use state so that we probe the circuit
    * if nothing else succeeds on it */
@@ -1139,10 +1551,10 @@ consider_plaintext_ports(entry_connection_t *conn, uint16_t port)
  *  See connection_ap_handshake_rewrite_and_attach()'s
  *  documentation for arguments and return value.
  */
-int
-connection_ap_rewrite_and_attach_if_allowed(entry_connection_t *conn,
-                                            origin_circuit_t *circ,
-                                            crypt_path_t *cpath)
+MOCK_IMPL(int,
+connection_ap_rewrite_and_attach_if_allowed,(entry_connection_t *conn,
+                                             origin_circuit_t *circ,
+                                             crypt_path_t *cpath))
 {
   const or_options_t *options = get_options();
 
@@ -1185,12 +1597,11 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
   /* Check for whether this is a .exit address.  By default, those are
    * disallowed when they're coming straight from the client, but you're
    * allowed to have them in MapAddress commands and so forth. */
-  if (!strcmpend(socks->address, ".exit") && !options->AllowDotExit) {
+  if (!strcmpend(socks->address, ".exit")) {
     static ratelim_t exit_warning_limit = RATELIM_INIT(60*15);
     log_fn_ratelim(&exit_warning_limit, LOG_WARN, LD_APP,
                    "The  \".exit\" notation is disabled in Tor due to "
-                   "security risks.  Set AllowDotExit in your torrc to enable "
-                   "it (at your own risk).");
+                   "security risks.");
     control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s",
                                 escaped(socks->address));
     out->end_reason = END_STREAM_REASON_TORPROTOCOL;
@@ -1200,6 +1611,8 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
 
   /* Remember the original address so we can tell the user about what
    * they actually said, not just what it turned into. */
+  /* XXX yes, this is the same as out->orig_address above. One is
+   * in the output, and one is in the connection. */
   if (! conn->original_dest_address) {
     /* Is the 'if' necessary here? XXXX */
     conn->original_dest_address = tor_strdup(conn->socks_request->address);
@@ -1207,7 +1620,7 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
 
   /* First, apply MapAddress and MAPADDRESS mappings. We need to do
    * these only for non-reverse lookups, since they don't exist for those.
-   * We need to do this before we consider automapping, since we might
+   * We also need to do this before we consider automapping, since we might
    * e.g. resolve irc.oftc.net into irconionaddress.onion, at which point
    * we'd need to automap it. */
   if (socks->command != SOCKS_COMMAND_RESOLVE_PTR) {
@@ -1219,9 +1632,12 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
     }
   }
 
-  /* Now, handle automapping.  Automapping happens when we're asked to
-   * resolve a hostname, and AutomapHostsOnResolve is set, and
-   * the hostname has a suffix listed in AutomapHostsSuffixes.
+  /* Now see if we need to create or return an existing Hostname->IP
+   * automapping.  Automapping happens when we're asked to resolve a
+   * hostname, and AutomapHostsOnResolve is set, and the hostname has a
+   * suffix listed in AutomapHostsSuffixes.  It's a handy feature
+   * that lets you have Tor assign e.g. IPv6 addresses for .onion
+   * names, and return them safely from DNSPort.
    */
   if (socks->command == SOCKS_COMMAND_RESOLVE &&
       tor_addr_parse(&addr_tmp, socks->address)<0 &&
@@ -1261,7 +1677,8 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
   }
 
   /* Now handle reverse lookups, if they're in the cache.  This doesn't
-   * happen too often, since client-side DNS caching is off by default. */
+   * happen too often, since client-side DNS caching is off by default,
+   * and very deprecated. */
   if (socks->command == SOCKS_COMMAND_RESOLVE_PTR) {
     unsigned rewrite_flags = 0;
     if (conn->entry_cfg.use_cached_ipv4_answers)
@@ -1290,7 +1707,7 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
      * an internal address?  If so, we should reject it if we're configured to
      * do so. */
     if (options->ClientDNSRejectInternalAddresses) {
-      /* Don't let people try to do a reverse lookup on 10.0.0.1. */
+      /* Don't let clients try to do a reverse lookup on 10.0.0.1. */
       tor_addr_t addr;
       int ok;
       ok = tor_addr_parse_PTR_name(
@@ -1306,11 +1723,12 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
     }
   }
 
-  /* If we didn't automap it before, then this is still the address
-   * that came straight from the user, mapped according to any
-   * MapAddress/MAPADDRESS commands.  Now other mappings, including
-   * previously registered Automap entries, TrackHostExits entries,
-   * and client-side DNS cache entries (not recommended).
+  /* If we didn't automap it before, then this is still the address that
+   * came straight from the user, mapped according to any
+   * MapAddress/MAPADDRESS commands.  Now apply other mappings,
+   * including previously registered Automap entries (IP back to
+   * hostname), TrackHostExits entries, and client-side DNS cache
+   * entries (if they're turned on).
    */
   if (socks->command != SOCKS_COMMAND_RESOLVE_PTR &&
       !out->automap) {
@@ -1349,6 +1767,199 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
   }
 }
 
+/** We just received a SOCKS request in <b>conn</b> to an onion address of type
+ *  <b>addresstype</b>. Start connecting to the onion service. */
+static int
+connection_ap_handle_onion(entry_connection_t *conn,
+                           socks_request_t *socks,
+                           origin_circuit_t *circ,
+                           hostname_type_t addresstype)
+{
+  time_t now = approx_time();
+  connection_t *base_conn = ENTRY_TO_CONN(conn);
+
+  /* If .onion address requests are disabled, refuse the request */
+  if (!conn->entry_cfg.onion_traffic) {
+    log_warn(LD_APP, "Onion address %s requested from a port with .onion "
+             "disabled", safe_str_client(socks->address));
+    connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
+    return -1;
+  }
+
+  /* Check whether it's RESOLVE or RESOLVE_PTR.  We don't handle those
+   * for hidden service addresses. */
+  if (SOCKS_COMMAND_IS_RESOLVE(socks->command)) {
+    /* if it's a resolve request, fail it right now, rather than
+     * building all the circuits and then realizing it won't work. */
+    log_warn(LD_APP,
+             "Resolve requests to hidden services not allowed. Failing.");
+    connection_ap_handshake_socks_resolved(conn,RESOLVED_TYPE_ERROR,
+                                           0,NULL,-1,TIME_MAX);
+    connection_mark_unattached_ap(conn,
+                               END_STREAM_REASON_SOCKSPROTOCOL |
+                               END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED);
+    return -1;
+  }
+
+  /* If we were passed a circuit, then we need to fail.  .onion addresses
+   * only work when we launch our own circuits for now. */
+  if (circ) {
+    log_warn(LD_CONTROL, "Attachstream to a circuit is not "
+             "supported for .onion addresses currently. Failing.");
+    connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL);
+    return -1;
+  }
+
+  /* Interface: Regardless of HS version after the block below we should have
+     set onion_address, rend_cache_lookup_result, and descriptor_is_usable. */
+  const char *onion_address = NULL;
+  int rend_cache_lookup_result = -ENOENT;
+  int descriptor_is_usable = 0;
+
+  if (addresstype == ONION_V2_HOSTNAME) { /* it's a v2 hidden service */
+    rend_cache_entry_t *entry = NULL;
+    /* Look up if we have client authorization configured for this hidden
+     * service.  If we do, associate it with the rend_data. */
+    rend_service_authorization_t *client_auth =
+      rend_client_lookup_service_authorization(socks->address);
+
+    const uint8_t *cookie = NULL;
+    rend_auth_type_t auth_type = REND_NO_AUTH;
+    if (client_auth) {
+      log_info(LD_REND, "Using previously configured client authorization "
+               "for hidden service request.");
+      auth_type = client_auth->auth_type;
+      cookie = client_auth->descriptor_cookie;
+    }
+
+    /* Fill in the rend_data field so we can start doing a connection to
+     * a hidden service. */
+    rend_data_t *rend_data = ENTRY_TO_EDGE_CONN(conn)->rend_data =
+      rend_data_client_create(socks->address, NULL, (char *) cookie,
+                              auth_type);
+    if (rend_data == NULL) {
+      return -1;
+    }
+    onion_address = rend_data_get_address(rend_data);
+    log_info(LD_REND,"Got a hidden service request for ID '%s'",
+             safe_str_client(onion_address));
+
+    rend_cache_lookup_result = rend_cache_lookup_entry(onion_address,-1,
+                                                       &entry);
+    if (!rend_cache_lookup_result && entry) {
+      descriptor_is_usable = rend_client_any_intro_points_usable(entry);
+    }
+  } else { /* it's a v3 hidden service */
+    tor_assert(addresstype == ONION_V3_HOSTNAME);
+    const hs_descriptor_t *cached_desc = NULL;
+    int retval;
+    /* Create HS conn identifier with HS pubkey */
+    hs_ident_edge_conn_t *hs_conn_ident =
+      tor_malloc_zero(sizeof(hs_ident_edge_conn_t));
+
+    retval = hs_parse_address(socks->address, &hs_conn_ident->identity_pk,
+                              NULL, NULL);
+    if (retval < 0) {
+      log_warn(LD_GENERAL, "failed to parse hs address");
+      tor_free(hs_conn_ident);
+      return -1;
+    }
+    ENTRY_TO_EDGE_CONN(conn)->hs_ident = hs_conn_ident;
+
+    onion_address = socks->address;
+
+    /* Check the v3 desc cache */
+    cached_desc = hs_cache_lookup_as_client(&hs_conn_ident->identity_pk);
+    if (cached_desc) {
+      rend_cache_lookup_result = 0;
+      descriptor_is_usable =
+        hs_client_any_intro_points_usable(&hs_conn_ident->identity_pk,
+                                          cached_desc);
+      log_info(LD_GENERAL, "Found %s descriptor in cache for %s. %s.",
+               (descriptor_is_usable) ? "usable" : "unusable",
+               safe_str_client(onion_address),
+               (descriptor_is_usable) ? "Not fetching." : "Refecting.");
+    } else {
+      rend_cache_lookup_result = -ENOENT;
+    }
+  }
+
+  /* Lookup the given onion address. If invalid, stop right now.
+   * Otherwise, we might have it in the cache or not. */
+  unsigned int refetch_desc = 0;
+  if (rend_cache_lookup_result < 0) {
+    switch (-rend_cache_lookup_result) {
+    case EINVAL:
+      /* We should already have rejected this address! */
+      log_warn(LD_BUG,"Invalid service name '%s'",
+               safe_str_client(onion_address));
+      connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL);
+      return -1;
+    case ENOENT:
+      /* We didn't have this; we should look it up. */
+      log_info(LD_REND, "No descriptor found in our cache for %s. Fetching.",
+               safe_str_client(onion_address));
+      refetch_desc = 1;
+      break;
+    default:
+      log_warn(LD_BUG, "Unknown cache lookup error %d",
+               rend_cache_lookup_result);
+      return -1;
+    }
+  }
+
+  /* Help predict that we'll want to do hidden service circuits in the
+   * future. We're not sure if it will need a stable circuit yet, but
+   * we know we'll need *something*. */
+  rep_hist_note_used_internal(now, 0, 1);
+
+  /* Now we have a descriptor but is it usable or not? If not, refetch.
+   * Also, a fetch could have been requested if the onion address was not
+   * found in the cache previously. */
+  if (refetch_desc || !descriptor_is_usable) {
+    edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(conn);
+    connection_ap_mark_as_non_pending_circuit(conn);
+    base_conn->state = AP_CONN_STATE_RENDDESC_WAIT;
+    if (addresstype == ONION_V2_HOSTNAME) {
+      tor_assert(edge_conn->rend_data);
+      rend_client_refetch_v2_renddesc(edge_conn->rend_data);
+      /* Whatever the result of the refetch, we don't go further. */
+      return 0;
+    } else {
+      tor_assert(addresstype == ONION_V3_HOSTNAME);
+      tor_assert(edge_conn->hs_ident);
+      /* Attempt to fetch the hsv3 descriptor. Check the retval to see how it
+       * went and act accordingly. */
+      int ret = hs_client_refetch_hsdesc(&edge_conn->hs_ident->identity_pk);
+      switch (ret) {
+      case HS_CLIENT_FETCH_MISSING_INFO:
+        /* Keeping the connection in descriptor wait state is fine because
+         * once we get enough dirinfo or a new live consensus, the HS client
+         * subsystem is notified and every connection in that state will
+         * trigger a fetch for the service key. */
+      case HS_CLIENT_FETCH_LAUNCHED:
+      case HS_CLIENT_FETCH_PENDING:
+      case HS_CLIENT_FETCH_HAVE_DESC:
+        return 0;
+      case HS_CLIENT_FETCH_ERROR:
+      case HS_CLIENT_FETCH_NO_HSDIRS:
+      case HS_CLIENT_FETCH_NOT_ALLOWED:
+        /* Can't proceed further and better close the SOCKS request. */
+        return -1;
+      }
+    }
+  }
+
+  /* We have the descriptor!  So launch a connection to the HS. */
+  log_info(LD_REND, "Descriptor is here. Great.");
+
+  base_conn->state = AP_CONN_STATE_CIRCUIT_WAIT;
+  /* We'll try to attach it at the next event loop, or whenever
+   * we call connection_ap_attach_pending() */
+  connection_ap_mark_as_pending_circuit(conn);
+  return 0;
+}
+
 /** Connection <b>conn</b> just finished its socks handshake, or the
  * controller asked us to take care of it. If <b>circ</b> is defined,
  * then that's where we'll want to attach it. Otherwise we have to
@@ -1375,11 +1986,14 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
   time_t now = time(NULL);
   rewrite_result_t rr;
 
+  /* First we'll do the rewrite part.  Let's see if we get a reasonable
+   * answer.
+   */
   memset(&rr, 0, sizeof(rr));
   connection_ap_handshake_rewrite(conn,&rr);
 
   if (rr.should_close) {
-    /* connection_ap_handshake_rewrite told us to close the connection,
+    /* connection_ap_handshake_rewrite told us to close the connection:
      * either because it sent back an answer, or because it sent back an
      * error */
     connection_mark_unattached_ap(conn, rr.end_reason);
@@ -1393,8 +2007,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
   const int automap = rr.automap;
   const addressmap_entry_source_t exit_source = rr.exit_source;
 
-  /* Parse the address provided by SOCKS.  Modify it in-place if it
-   * specifies a hidden-service (.onion) or particular exit node (.exit).
+  /* Now, we parse the address to see if it's an .onion or .exit or
+   * other special address.
    */
   const hostname_type_t addresstype = parse_extended_hostname(socks->address);
 
@@ -1408,8 +2022,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
   }
 
   /* If this is a .exit hostname, strip off the .name.exit part, and
-   * see whether we're going to connect there, and otherwise handle it.
-   * (The ".exit" part got stripped off by "parse_extended_hostname").
+   * see whether we're willing to connect there, and and otherwise handle the
+   * .exit address.
    *
    * We'll set chosen_exit_name and/or close the connection as appropriate.
    */
@@ -1421,23 +2035,23 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
     const node_t *node = NULL;
 
     /* If this .exit was added by an AUTOMAP, then it came straight from
-     * a user.  Make sure that options->AllowDotExit permits that. */
-    if (exit_source == ADDRMAPSRC_AUTOMAP && !options->AllowDotExit) {
-      /* Whoops; this one is stale.  It must have gotten added earlier,
-       * when AllowDotExit was on. */
-      log_warn(LD_APP,"Stale automapped address for '%s.exit', with "
-               "AllowDotExit disabled. Refusing.",
+     * a user.  That's not safe. */
+    if (exit_source == ADDRMAPSRC_AUTOMAP) {
+      /* Whoops; this one is stale.  It must have gotten added earlier?
+       * (Probably this is not possible, since AllowDotExit no longer
+       * exists.) */
+      log_warn(LD_APP,"Stale automapped address for '%s.exit'. Refusing.",
                safe_str_client(socks->address));
       control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s",
                                   escaped(socks->address));
       connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL);
+      tor_assert_nonfatal_unreached();
       return -1;
     }
 
     /* Double-check to make sure there are no .exits coming from
      * impossible/weird sources. */
-    if (exit_source == ADDRMAPSRC_DNS ||
-        (exit_source == ADDRMAPSRC_NONE && !options->AllowDotExit)) {
+    if (exit_source == ADDRMAPSRC_DNS || exit_source == ADDRMAPSRC_NONE) {
       /* It shouldn't be possible to get a .exit address from any of these
        * sources. */
       log_warn(LD_BUG,"Address '%s.exit', with impossible source for the "
@@ -1450,14 +2064,19 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
     }
 
     tor_assert(!automap);
-    /* Now, find the character before the .(name) part. */
+
+    /* Now, find the character before the .(name) part.
+     * (The ".exit" part got stripped off by "parse_extended_hostname").
+     *
+     * We're going to put the exit name into conn->chosen_exit_name, and
+     * look up a node correspondingly. */
     char *s = strrchr(socks->address,'.');
     if (s) {
       /* The address was of the form "(stuff).(name).exit */
       if (s[1] != '\0') {
         /* Looks like a real .exit one. */
         conn->chosen_exit_name = tor_strdup(s+1);
-        node = node_get_by_nickname(conn->chosen_exit_name, 1);
+        node = node_get_by_nickname(conn->chosen_exit_name, 0);
 
         if (exit_source == ADDRMAPSRC_TRACKEXIT) {
           /* We 5 tries before it expires the addressmap */
@@ -1478,7 +2097,7 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
        * form that means (foo's address).foo.exit. */
 
       conn->chosen_exit_name = tor_strdup(socks->address);
-      node = node_get_by_nickname(conn->chosen_exit_name, 1);
+      node = node_get_by_nickname(conn->chosen_exit_name, 0);
       if (node) {
         *socks->address = 0;
         node_get_address_string(node, socks->address, sizeof(socks->address));
@@ -1506,10 +2125,12 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
        implies no. */
   }
 
-  /* Now, handle everything that isn't a .onion address. */
-  if (addresstype != ONION_HOSTNAME) {
+  /* Now, we handle everything that isn't a .onion address. */
+  if (addresstype != ONION_V2_HOSTNAME && addresstype != ONION_V3_HOSTNAME) {
     /* Not a hidden-service request.  It's either a hostname or an IP,
-     * possibly with a .exit that we stripped off. */
+     * possibly with a .exit that we stripped off.  We're going to check
+     * if we're allowed to connect/resolve there, and then launch the
+     * appropriate request. */
 
     /* Check for funny characters in the address. */
     if (address_is_invalid_destination(socks->address, 1)) {
@@ -1522,18 +2143,6 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
       return -1;
     }
 
-#ifdef ENABLE_TOR2WEB_MODE
-    /* If we're running in Tor2webMode, we don't allow anything BUT .onion
-     * addresses. */
-    if (options->Tor2webMode) {
-      log_warn(LD_APP, "Refusing to connect to non-hidden-service hostname "
-               "or IP address %s because tor2web mode is enabled.",
-               safe_str_client(socks->address));
-      connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
-      return -1;
-    }
-#endif
-
     /* socks->address is a non-onion hostname or IP address.
      * If we can't do any non-onion requests, refuse the connection.
      * If we have a hostname but can't do DNS, refuse the connection.
@@ -1556,30 +2165,37 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
     }
 
     /* Then check if we have a hostname or IP address, and whether DNS or
-     * the IP address family are permitted */
+     * the IP address family are permitted.  Reject if not. */
     tor_addr_t dummy_addr;
     int socks_family = tor_addr_parse(&dummy_addr, socks->address);
     /* family will be -1 for a non-onion hostname that's not an IP */
-    if (socks_family == -1 && !conn->entry_cfg.dns_request) {
-      log_warn(LD_APP, "Refusing to connect to hostname %s "
-               "because Port has NoDNSRequest set.",
-               safe_str_client(socks->address));
-      connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
-      return -1;
-    } else if (socks_family == AF_INET && !conn->entry_cfg.ipv4_traffic) {
-      log_warn(LD_APP, "Refusing to connect to IPv4 address %s because "
-               "Port has NoIPv4Traffic set.",
-               safe_str_client(socks->address));
-      connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
-      return -1;
-    } else if (socks_family == AF_INET6 && !conn->entry_cfg.ipv6_traffic) {
-      log_warn(LD_APP, "Refusing to connect to IPv6 address %s because "
-               "Port has NoIPv6Traffic set.",
-               safe_str_client(socks->address));
-      connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
-      return -1;
+    if (socks_family == -1) {
+      if (!conn->entry_cfg.dns_request) {
+        log_warn(LD_APP, "Refusing to connect to hostname %s "
+                 "because Port has NoDNSRequest set.",
+                 safe_str_client(socks->address));
+        connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
+        return -1;
+      }
+    } else if (socks_family == AF_INET) {
+      if (!conn->entry_cfg.ipv4_traffic) {
+        log_warn(LD_APP, "Refusing to connect to IPv4 address %s because "
+                 "Port has NoIPv4Traffic set.",
+                 safe_str_client(socks->address));
+        connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
+        return -1;
+      }
+    } else if (socks_family == AF_INET6) {
+      if (!conn->entry_cfg.ipv6_traffic) {
+        log_warn(LD_APP, "Refusing to connect to IPv6 address %s because "
+                 "Port has NoIPv6Traffic set.",
+                 safe_str_client(socks->address));
+        connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
+        return -1;
+      }
+    } else {
+      tor_assert_nonfatal_unreached_once();
     }
-    /* No else, we've covered all possible returned value. */
 
     /* See if this is a hostname lookup that we can answer immediately.
      * (For example, an attempt to look up the IP address for an IP address.)
@@ -1600,7 +2216,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
       tor_assert(!automap);
       rep_hist_note_used_resolve(now); /* help predict this next time */
     } else if (socks->command == SOCKS_COMMAND_CONNECT) {
-      /* Special handling for attempts to connect */
+      /* Now see if this is a connect request that we can reject immediately */
+
       tor_assert(!automap);
       /* Don't allow connections to port 0. */
       if (socks->port == 0) {
@@ -1609,7 +2226,7 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
         return -1;
       }
       /* You can't make connections to internal addresses, by default.
-       * Exceptions are begindir requests (where the address is meaningless,
+       * Exceptions are begindir requests (where the address is meaningless),
        * or cases where you've hand-configured a particular exit, thereby
        * making the local address meaningful. */
       if (options->ClientRejectInternalAddresses &&
@@ -1653,7 +2270,9 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
       } /* end "if we should check for internal addresses" */
 
       /* Okay.  We're still doing a CONNECT, and it wasn't a private
-       * address.  Do special handling for literal IP addresses */
+       * address.  Here we do special handling for literal IP addresses,
+       * to see if we should reject this preemptively, and to set up
+       * fields in conn->entry_cfg to tell the exit what AF we want. */
       {
         tor_addr_t addr;
         /* XXX Duplicate call to tor_addr_parse. */
@@ -1696,11 +2315,15 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
         }
       }
 
+      /* we never allow IPv6 answers on socks4. (TODO: Is this smart?) */
       if (socks->socks_version == 4)
         conn->entry_cfg.ipv6_traffic = 0;
 
       /* Still handling CONNECT. Now, check for exit enclaves.  (Which we
-       * don't do on BEGINDIR, or there is a chosen exit.)
+       * don't do on BEGIN_DIR, or when there is a chosen exit.)
+       *
+       * TODO: Should we remove this?  Exit enclaves are nutty and don't
+       * work very well
        */
       if (!conn->use_begindir && !conn->chosen_exit_name && !circ) {
         /* see if we can find a suitable enclave exit */
@@ -1724,7 +2347,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
         if (consider_plaintext_ports(conn, socks->port) < 0)
           return -1;
 
-      /* Remember the port so that we do predicted requests there. */
+      /* Remember the port so that we will predict that more requests
+         there will happen in the future. */
       if (!conn->use_begindir) {
         /* help predict this next time */
         rep_hist_note_used_port(now, socks->port);
@@ -1733,7 +2357,7 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
       rep_hist_note_used_resolve(now); /* help predict this next time */
       /* no extra processing needed */
     } else {
-      /* We should only be doing CONNECT or RESOLVE! */
+      /* We should only be doing CONNECT, RESOLVE, or RESOLVE_PTR! */
       tor_fragile_assert();
     }
 
@@ -1749,6 +2373,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
     if (circ) {
       rv = connection_ap_handshake_attach_chosen_circuit(conn, circ, cpath);
     } else {
+      /* We'll try to attach it at the next event loop, or whenever
+       * we call connection_ap_attach_pending() */
       connection_ap_mark_as_pending_circuit(conn);
       rv = 0;
     }
@@ -1766,110 +2392,10 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
     return 0;
   } else {
     /* If we get here, it's a request for a .onion address! */
+    tor_assert(addresstype == ONION_V2_HOSTNAME ||
+               addresstype == ONION_V3_HOSTNAME);
     tor_assert(!automap);
-
-    /* If .onion address requests are disabled, refuse the request */
-    if (!conn->entry_cfg.onion_traffic) {
-      log_warn(LD_APP, "Onion address %s requested from a port with .onion "
-                       "disabled", safe_str_client(socks->address));
-      connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
-      return -1;
-    }
-
-    /* Check whether it's RESOLVE or RESOLVE_PTR.  We don't handle those
-     * for hidden service addresses. */
-    if (SOCKS_COMMAND_IS_RESOLVE(socks->command)) {
-      /* if it's a resolve request, fail it right now, rather than
-       * building all the circuits and then realizing it won't work. */
-      log_warn(LD_APP,
-               "Resolve requests to hidden services not allowed. Failing.");
-      connection_ap_handshake_socks_resolved(conn,RESOLVED_TYPE_ERROR,
-                                             0,NULL,-1,TIME_MAX);
-      connection_mark_unattached_ap(conn,
-                                END_STREAM_REASON_SOCKSPROTOCOL |
-                                END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED);
-      return -1;
-    }
-
-    /* If we were passed a circuit, then we need to fail.  .onion addresses
-     * only work when we launch our own circuits for now. */
-    if (circ) {
-      log_warn(LD_CONTROL, "Attachstream to a circuit is not "
-               "supported for .onion addresses currently. Failing.");
-      connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL);
-      return -1;
-    }
-
-    /* Look up if we have client authorization configured for this hidden
-     * service.  If we do, associate it with the rend_data. */
-    rend_service_authorization_t *client_auth =
-      rend_client_lookup_service_authorization(socks->address);
-
-    const uint8_t *cookie = NULL;
-    rend_auth_type_t auth_type = REND_NO_AUTH;
-    if (client_auth) {
-      log_info(LD_REND, "Using previously configured client authorization "
-               "for hidden service request.");
-      auth_type = client_auth->auth_type;
-      cookie = client_auth->descriptor_cookie;
-    }
-
-    /* Fill in the rend_data field so we can start doing a connection to
-     * a hidden service. */
-    rend_data_t *rend_data = ENTRY_TO_EDGE_CONN(conn)->rend_data =
-      rend_data_client_create(socks->address, NULL, (char *) cookie,
-                              auth_type);
-    if (rend_data == NULL) {
-      return -1;
-    }
-    log_info(LD_REND,"Got a hidden service request for ID '%s'",
-             safe_str_client(rend_data->onion_address));
-
-    /* Lookup the given onion address. If invalid, stop right now else we
-     * might have it in the cache or not, it will be tested later on. */
-    unsigned int refetch_desc = 0;
-    rend_cache_entry_t *entry = NULL;
-    const int rend_cache_lookup_result =
-      rend_cache_lookup_entry(rend_data->onion_address, -1, &entry);
-    if (rend_cache_lookup_result < 0) {
-      switch (-rend_cache_lookup_result) {
-      case EINVAL:
-        /* We should already have rejected this address! */
-        log_warn(LD_BUG,"Invalid service name '%s'",
-            safe_str_client(rend_data->onion_address));
-        connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL);
-        return -1;
-      case ENOENT:
-        refetch_desc = 1;
-        break;
-      default:
-        log_warn(LD_BUG, "Unknown cache lookup error %d",
-            rend_cache_lookup_result);
-        return -1;
-      }
-    }
-
-    /* Help predict this next time. We're not sure if it will need
-     * a stable circuit yet, but we know we'll need *something*. */
-    rep_hist_note_used_internal(now, 0, 1);
-
-    /* Now we have a descriptor but is it usable or not? If not, refetch.
-     * Also, a fetch could have been requested if the onion address was not
-     * found in the cache previously. */
-    if (refetch_desc || !rend_client_any_intro_points_usable(entry)) {
-      connection_ap_mark_as_non_pending_circuit(conn);
-      base_conn->state = AP_CONN_STATE_RENDDESC_WAIT;
-      log_info(LD_REND, "Unknown descriptor %s. Fetching.",
-          safe_str_client(rend_data->onion_address));
-      rend_client_refetch_v2_renddesc(rend_data);
-      return 0;
-    }
-
-    /* We have the descriptor so launch a connection to the HS. */
-    base_conn->state = AP_CONN_STATE_CIRCUIT_WAIT;
-    log_info(LD_REND, "Descriptor is here. Great.");
-    connection_ap_mark_as_pending_circuit(conn);
-    return 0;
+    return connection_ap_handle_onion(conn, socks, circ, addresstype);
   }
 
   return 0; /* unreached but keeps the compiler happy */
@@ -1885,13 +2411,13 @@ get_pf_socket(void)
   if (pf_socket >= 0)
     return pf_socket;
 
-#ifdef OPENBSD
+#if defined(OpenBSD)
   /* only works on OpenBSD */
   pf = tor_open_cloexec("/dev/pf", O_RDONLY, 0);
 #else
   /* works on NetBSD and FreeBSD */
   pf = tor_open_cloexec("/dev/pf", O_RDWR, 0);
-#endif
+#endif /* defined(OpenBSD) */
 
   if (pf < 0) {
     log_warn(LD_NET, "open(\"/dev/pf\") failed: %s", strerror(errno));
@@ -1901,9 +2427,10 @@ get_pf_socket(void)
   pf_socket = pf;
   return pf_socket;
 }
-#endif
+#endif /* defined(TRANS_PF) */
 
-#if defined(TRANS_NETFILTER) || defined(TRANS_PF) || defined(TRANS_TPROXY)
+#if defined(TRANS_NETFILTER) || defined(TRANS_PF) || \
+  defined(TRANS_TPROXY)
 /** Try fill in the address of <b>req</b> from the socket configured
  * with <b>conn</b>. */
 static int
@@ -1923,7 +2450,7 @@ destination_from_socket(entry_connection_t *conn, socks_request_t *req)
     }
     goto done;
   }
-#endif
+#endif /* defined(TRANS_TPROXY) */
 
 #ifdef TRANS_NETFILTER
   int rv = -1;
@@ -1933,13 +2460,13 @@ destination_from_socket(entry_connection_t *conn, socks_request_t *req)
       rv = getsockopt(ENTRY_TO_CONN(conn)->s, SOL_IP, SO_ORIGINAL_DST,
                   (struct sockaddr*)&orig_dst, &orig_dst_len);
       break;
-#endif
+#endif /* defined(TRANS_NETFILTER_IPV4) */
 #ifdef TRANS_NETFILTER_IPV6
     case AF_INET6:
       rv = getsockopt(ENTRY_TO_CONN(conn)->s, SOL_IPV6, IP6T_SO_ORIGINAL_DST,
                   (struct sockaddr*)&orig_dst, &orig_dst_len);
       break;
-#endif
+#endif /* defined(TRANS_NETFILTER_IPV6) */
     default:
       log_warn(LD_BUG,
                "Received transparent data from an unsuported socket family %d",
@@ -1965,7 +2492,7 @@ destination_from_socket(entry_connection_t *conn, socks_request_t *req)
   (void)req;
   log_warn(LD_BUG, "Unable to determine destination from socket.");
   return -1;
-#endif
+#endif /* defined(TRANS_NETFILTER) || ... */
 
  done:
   tor_addr_from_sockaddr(&addr, (struct sockaddr*)&orig_dst, &req->port);
@@ -1973,7 +2500,7 @@ destination_from_socket(entry_connection_t *conn, socks_request_t *req)
 
   return 0;
 }
-#endif
+#endif /* defined(TRANS_NETFILTER) || defined(TRANS_PF) || ... */
 
 #ifdef TRANS_PF
 static int
@@ -2007,7 +2534,7 @@ destination_from_pf(entry_connection_t *conn, socks_request_t *req)
 
     return 0;
   }
-#endif
+#endif /* defined(__FreeBSD__) */
 
   memset(&pnl, 0, sizeof(pnl));
   pnl.proto           = IPPROTO_TCP;
@@ -2022,8 +2549,11 @@ destination_from_pf(entry_connection_t *conn, socks_request_t *req)
   } else if (proxy_sa->sa_family == AF_INET6) {
     struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)proxy_sa;
     pnl.af = AF_INET6;
-    memcpy(&pnl.saddr.v6, tor_addr_to_in6(&ENTRY_TO_CONN(conn)->addr),
-           sizeof(struct in6_addr));
+    const struct in6_addr *dest_in6 =
+      tor_addr_to_in6(&ENTRY_TO_CONN(conn)->addr);
+    if (BUG(!dest_in6))
+      return -1;
+    memcpy(&pnl.saddr.v6, dest_in6, sizeof(struct in6_addr));
     pnl.sport = htons(ENTRY_TO_CONN(conn)->port);
     memcpy(&pnl.daddr.v6, &sin6->sin6_addr, sizeof(struct in6_addr));
     pnl.dport = sin6->sin6_port;
@@ -2056,7 +2586,7 @@ destination_from_pf(entry_connection_t *conn, socks_request_t *req)
 
   return 0;
 }
-#endif
+#endif /* defined(TRANS_PF) */
 
 /** Fetch the original destination address and port from a
  * system-specific interface and put them into a
@@ -2092,7 +2622,7 @@ connection_ap_get_original_destination(entry_connection_t *conn,
   log_warn(LD_BUG, "Called connection_ap_get_original_destination, but no "
            "transparent proxy method was configured.");
   return -1;
-#endif
+#endif /* defined(TRANS_NETFILTER) || ... */
 }
 
 /** connection_edge_process_inbuf() found a conn in state
@@ -2127,7 +2657,7 @@ connection_ap_handshake_process_socks(entry_connection_t *conn)
 
   if (socks->replylen) {
     had_reply = 1;
-    connection_write_to_buf((const char*)socks->reply, socks->replylen,
+    connection_buf_add((const char*)socks->reply, socks->replylen,
                             base_conn);
     socks->replylen = 0;
     if (sockshere == -1) {
@@ -2224,7 +2754,7 @@ connection_ap_process_natd(entry_connection_t *conn)
 
   /* look for LF-terminated "[DEST ip_addr port]"
    * where ip_addr is a dotted-quad and port is in string form */
-  err = connection_fetch_from_buf_line(ENTRY_TO_CONN(conn), tmp_buf, &tlen);
+  err = connection_buf_get_line(ENTRY_TO_CONN(conn), tmp_buf, &tlen);
   if (err == 0)
     return 0;
   if (err < 0) {
@@ -2273,6 +2803,111 @@ connection_ap_process_natd(entry_connection_t *conn)
   return connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL);
 }
 
+/** Called on an HTTP CONNECT entry connection when some bytes have arrived,
+ * but we have not yet received a full HTTP CONNECT request.  Try to parse an
+ * HTTP CONNECT request from the connection's inbuf.  On success, set up the
+ * connection's socks_request field and try to attach the connection.  On
+ * failure, send an HTTP reply, and mark the connection.
+ */
+STATIC int
+connection_ap_process_http_connect(entry_connection_t *conn)
+{
+  if (BUG(ENTRY_TO_CONN(conn)->state != AP_CONN_STATE_HTTP_CONNECT_WAIT))
+    return -1;
+
+  char *headers = NULL, *body = NULL;
+  char *command = NULL, *addrport = NULL;
+  char *addr = NULL;
+  size_t bodylen = 0;
+
+  const char *errmsg = NULL;
+  int rv = 0;
+
+  const int http_status =
+    fetch_from_buf_http(ENTRY_TO_CONN(conn)->inbuf, &headers, 8192,
+                        &body, &bodylen, 1024, 0);
+  if (http_status < 0) {
+    /* Bad http status */
+    errmsg = "HTTP/1.0 400 Bad Request\r\n\r\n";
+    goto err;
+  } else if (http_status == 0) {
+    /* no HTTP request yet. */
+    goto done;
+  }
+
+  const int cmd_status = parse_http_command(headers, &command, &addrport);
+  if (cmd_status < 0) {
+    errmsg = "HTTP/1.0 400 Bad Request\r\n\r\n";
+    goto err;
+  }
+  tor_assert(command);
+  tor_assert(addrport);
+  if (strcasecmp(command, "connect")) {
+    errmsg = "HTTP/1.0 405 Method Not Allowed\r\n\r\n";
+    goto err;
+  }
+
+  tor_assert(conn->socks_request);
+  socks_request_t *socks = conn->socks_request;
+  uint16_t port;
+  if (tor_addr_port_split(LOG_WARN, addrport, &addr, &port) < 0) {
+    errmsg = "HTTP/1.0 400 Bad Request\r\n\r\n";
+    goto err;
+  }
+  if (strlen(addr) >= MAX_SOCKS_ADDR_LEN) {
+    errmsg = "HTTP/1.0 414 Request-URI Too Long\r\n\r\n";
+    goto err;
+  }
+
+  /* Abuse the 'username' and 'password' fields here. They are already an
+  * abuse. */
+  {
+    char *authorization = http_get_header(headers, "Proxy-Authorization: ");
+    if (authorization) {
+      socks->username = authorization; // steal reference
+      socks->usernamelen = strlen(authorization);
+    }
+    char *isolation = http_get_header(headers, "X-Tor-Stream-Isolation: ");
+    if (isolation) {
+      socks->password = isolation; // steal reference
+      socks->passwordlen = strlen(isolation);
+    }
+  }
+
+  socks->command = SOCKS_COMMAND_CONNECT;
+  socks->listener_type = CONN_TYPE_AP_HTTP_CONNECT_LISTENER;
+  strlcpy(socks->address, addr, sizeof(socks->address));
+  socks->port = port;
+
+  control_event_stream_status(conn, STREAM_EVENT_NEW, 0);
+
+  rv = connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL);
+
+  // XXXX send a "100 Continue" message?
+
+  goto done;
+
+ err:
+  if (BUG(errmsg == NULL))
+    errmsg = "HTTP/1.0 400 Bad Request\r\n\r\n";
+  log_info(LD_EDGE, "HTTP tunnel error: saying %s", escaped(errmsg));
+  connection_buf_add(errmsg, strlen(errmsg), ENTRY_TO_CONN(conn));
+  /* Mark it as "has_finished" so that we don't try to send an extra socks
+   * reply. */
+  conn->socks_request->has_finished = 1;
+  connection_mark_unattached_ap(conn,
+                                END_STREAM_REASON_HTTPPROTOCOL|
+                                END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED);
+
+ done:
+  tor_free(headers);
+  tor_free(body);
+  tor_free(command);
+  tor_free(addrport);
+  tor_free(addr);
+  return rv;
+}
+
 /** Iterate over the two bytes of stream_id until we get one that is not
  * already in use; return it. Return 0 if can't get a unique stream_id.
  */
@@ -2295,6 +2930,11 @@ get_unique_stream_id_by_circ(origin_circuit_t *circ)
   for (tmpconn = circ->p_streams; tmpconn; tmpconn=tmpconn->next_stream)
     if (tmpconn->stream_id == test_stream_id)
       goto again;
+
+  if (connection_half_edge_find_stream_id(circ->half_streams,
+                                           test_stream_id))
+    goto again;
+
   return test_stream_id;
 }
 
@@ -2309,6 +2949,8 @@ connection_ap_supports_optimistic_data(const entry_connection_t *conn)
   if (edge_conn->on_circuit == NULL ||
       edge_conn->on_circuit->state != CIRCUIT_STATE_OPEN ||
       (edge_conn->on_circuit->purpose != CIRCUIT_PURPOSE_C_GENERAL &&
+       edge_conn->on_circuit->purpose != CIRCUIT_PURPOSE_C_HSDIR_GET &&
+       edge_conn->on_circuit->purpose != CIRCUIT_PURPOSE_S_HSDIR_POST &&
        edge_conn->on_circuit->purpose != CIRCUIT_PURPOSE_C_REND_JOINED))
     return 0;
 
@@ -2380,8 +3022,8 @@ connection_ap_get_begincell_flags(entry_connection_t *ap_conn)
  *
  * If ap_conn is broken, mark it for close and return -1. Else return 0.
  */
-int
-connection_ap_handshake_send_begin(entry_connection_t *ap_conn)
+MOCK_IMPL(int,
+connection_ap_handshake_send_begin,(entry_connection_t *ap_conn))
 {
   char payload[CELL_PAYLOAD_SIZE];
   int payload_len;
@@ -2453,8 +3095,10 @@ connection_ap_handshake_send_begin(entry_connection_t *ap_conn)
     /* Sensitive directory connections must have an anonymous path length.
      * Otherwise, directory connections are typically one-hop.
      * This matches the earlier check for directory connection path anonymity
-     * in directory_initiate_command_rend(). */
-    if (is_sensitive_dir_purpose(linked_dir_conn_base->purpose)) {
+     * in directory_initiate_request(). */
+    if (purpose_needs_anonymity(linked_dir_conn_base->purpose,
+                    TO_DIR_CONN(linked_dir_conn_base)->router_purpose,
+                    TO_DIR_CONN(linked_dir_conn_base)->requested_resource)) {
       assert_circ_anonymity_ok(circ, options);
     }
   } else {
@@ -2863,9 +3507,9 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply,
        !CIRCUIT_IS_ORIGIN(conn->edge_.on_circuit)) {
       if (endreason != END_STREAM_REASON_RESOLVEFAILED) {
         log_info(LD_BUG,
-                 "No origin circuit for successful SOCKS stream "U64_FORMAT
+                 "No origin circuit for successful SOCKS stream %"PRIu64
                  ". Reason: %d",
-                 U64_PRINTF_ARG(ENTRY_TO_CONN(conn)->global_identifier),
+                 (ENTRY_TO_CONN(conn)->global_identifier),
                  endreason);
       }
       /*
@@ -2890,15 +3534,22 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply,
     return;
   }
   if (replylen) { /* we already have a reply in mind */
-    connection_write_to_buf(reply, replylen, ENTRY_TO_CONN(conn));
+    connection_buf_add(reply, replylen, ENTRY_TO_CONN(conn));
     conn->socks_request->has_finished = 1;
     return;
   }
-  if (conn->socks_request->socks_version == 4) {
+  if (conn->socks_request->listener_type ==
+       CONN_TYPE_AP_HTTP_CONNECT_LISTENER) {
+    const char *response = end_reason_to_http_connect_response_line(endreason);
+    if (!response) {
+      response = "HTTP/1.0 400 Bad Request\r\n\r\n";
+    }
+    connection_buf_add(response, strlen(response), ENTRY_TO_CONN(conn));
+  } else if (conn->socks_request->socks_version == 4) {
     memset(buf,0,SOCKS4_NETWORK_LEN);
     buf[1] = (status==SOCKS5_SUCCEEDED ? SOCKS4_GRANTED : SOCKS4_REJECT);
     /* leave version, destport, destip zero */
-    connection_write_to_buf(buf, SOCKS4_NETWORK_LEN, ENTRY_TO_CONN(conn));
+    connection_buf_add(buf, SOCKS4_NETWORK_LEN, ENTRY_TO_CONN(conn));
   } else if (conn->socks_request->socks_version == 5) {
     size_t buf_len;
     memset(buf,0,sizeof(buf));
@@ -2917,7 +3568,7 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply,
       /* 4 bytes for the header, 2 bytes for the port, 16 for the address. */
       buf_len = 22;
     }
-    connection_write_to_buf(buf,buf_len,ENTRY_TO_CONN(conn));
+    connection_buf_add(buf,buf_len,ENTRY_TO_CONN(conn));
   }
   /* If socks_version isn't 4 or 5, don't send anything.
    * This can happen in the case of AP bridges. */
@@ -2925,12 +3576,12 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply,
   return;
 }
 
-/** Read a RELAY_BEGIN or RELAY_BEGINDIR cell from <b>cell</b>, decode it, and
+/** Read a RELAY_BEGIN or RELAY_BEGIN_DIR cell from <b>cell</b>, decode it, and
  * place the result in <b>bcell</b>.  On success return 0; on failure return
  * <0 and set *<b>end_reason_out</b> to the end reason we should send back to
  * the client.
  *
- * Return -1 in the case where want to send a RELAY_END cell, and < -1 when
+ * Return -1 in the case where we want to send a RELAY_END cell, and < -1 when
  * we don't.
  **/
 STATIC int
@@ -2989,6 +3640,96 @@ begin_cell_parse(const cell_t *cell, begin_cell_t *bcell,
   return 0;
 }
 
+/** For the given <b>circ</b> and the edge connection <b>conn</b>, setup the
+ * connection, attach it to the circ and connect it. Return 0 on success
+ * or END_CIRC_AT_ORIGIN if we can't find the requested hidden service port
+ * where the caller should close the circuit. */
+static int
+handle_hs_exit_conn(circuit_t *circ, edge_connection_t *conn)
+{
+  int ret;
+  origin_circuit_t *origin_circ;
+
+  assert_circuit_ok(circ);
+  tor_assert(circ->purpose == CIRCUIT_PURPOSE_S_REND_JOINED);
+  tor_assert(conn);
+
+  log_debug(LD_REND, "Connecting the hidden service rendezvous circuit "
+                     "to the service destination.");
+
+  origin_circ = TO_ORIGIN_CIRCUIT(circ);
+  conn->base_.address = tor_strdup("(rendezvous)");
+  conn->base_.state = EXIT_CONN_STATE_CONNECTING;
+
+  /* The circuit either has an hs identifier for v3+ or a rend_data for legacy
+   * service. */
+  if (origin_circ->rend_data) {
+    conn->rend_data = rend_data_dup(origin_circ->rend_data);
+    tor_assert(connection_edge_is_rendezvous_stream(conn));
+    ret = rend_service_set_connection_addr_port(conn, origin_circ);
+  } else if (origin_circ->hs_ident) {
+    /* Setup the identifier to be the one for the circuit service. */
+    conn->hs_ident =
+      hs_ident_edge_conn_new(&origin_circ->hs_ident->identity_pk);
+    tor_assert(connection_edge_is_rendezvous_stream(conn));
+    ret = hs_service_set_conn_addr_port(origin_circ, conn);
+  } else {
+    /* We should never get here if the circuit's purpose is rendezvous. */
+    tor_assert_nonfatal_unreached();
+    return -1;
+  }
+  if (ret < 0) {
+    log_info(LD_REND, "Didn't find rendezvous service (addr%s, port %d)",
+             fmt_addr(&TO_CONN(conn)->addr), TO_CONN(conn)->port);
+    /* Send back reason DONE because we want to make hidden service port
+     * scanning harder thus instead of returning that the exit policy
+     * didn't match, which makes it obvious that the port is closed,
+     * return DONE and kill the circuit. That way, a user (malicious or
+     * not) needs one circuit per bad port unless it matches the policy of
+     * the hidden service. */
+    relay_send_end_cell_from_edge(conn->stream_id, circ,
+                                  END_STREAM_REASON_DONE,
+                                  origin_circ->cpath->prev);
+    connection_free_(TO_CONN(conn));
+
+    /* Drop the circuit here since it might be someone deliberately
+     * scanning the hidden service ports. Note that this mitigates port
+     * scanning by adding more work on the attacker side to successfully
+     * scan but does not fully solve it. */
+    if (ret < -1) {
+      return END_CIRC_AT_ORIGIN;
+    } else {
+      return 0;
+    }
+  }
+
+  /* Link the circuit and the connection crypt path. */
+  conn->cpath_layer = origin_circ->cpath->prev;
+
+  /* Add it into the linked list of p_streams on this circuit */
+  conn->next_stream = origin_circ->p_streams;
+  origin_circ->p_streams = conn;
+  conn->on_circuit = circ;
+  assert_circuit_ok(circ);
+
+  hs_inc_rdv_stream_counter(origin_circ);
+
+  /* If it's an onion service connection, we might want to include the proxy
+   * protocol header: */
+  if (conn->hs_ident) {
+    hs_circuit_id_protocol_t circuit_id_protocol =
+      hs_service_exports_circuit_id(&conn->hs_ident->identity_pk);
+    export_hs_client_circuit_id(conn, circuit_id_protocol);
+  }
+
+  /* Connect tor to the hidden service destination. */
+  connection_exit_connect(conn);
+
+  /* For path bias: This circuit was used successfully */
+  pathbias_mark_use_success(origin_circ);
+  return 0;
+}
+
 /** A relay 'begin' or 'begin_dir' cell has arrived, and either we are
  * an exit hop for the circuit, or we are the origin and it is a
  * rendezvous begin.
@@ -3015,24 +3756,26 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
   char *address = NULL;
   uint16_t port = 0;
   or_circuit_t *or_circ = NULL;
+  origin_circuit_t *origin_circ = NULL;
+  crypt_path_t *layer_hint = NULL;
   const or_options_t *options = get_options();
   begin_cell_t bcell;
   int rv;
   uint8_t end_reason=0;
 
   assert_circuit_ok(circ);
-  if (!CIRCUIT_IS_ORIGIN(circ))
+  if (!CIRCUIT_IS_ORIGIN(circ)) {
     or_circ = TO_OR_CIRCUIT(circ);
+  } else {
+    tor_assert(circ->purpose == CIRCUIT_PURPOSE_S_REND_JOINED);
+    origin_circ = TO_ORIGIN_CIRCUIT(circ);
+    layer_hint = origin_circ->cpath->prev;
+  }
 
   relay_header_unpack(&rh, cell->payload);
   if (rh.length > RELAY_PAYLOAD_SIZE)
     return -END_CIRC_REASON_TORPROTOCOL;
 
-  /* Note: we have to use relay_send_command_from_edge here, not
-   * connection_edge_end or connection_edge_send_command, since those require
-   * that we have a stream connected to a circuit, and we don't connect to a
-   * circuit until we have a pending/successful resolve. */
-
   if (!server_mode(options) &&
       circ->purpose != CIRCUIT_PURPOSE_S_REND_JOINED) {
     log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
@@ -3047,7 +3790,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
     return -END_CIRC_REASON_TORPROTOCOL;
   } else if (rv == -1) {
     tor_free(bcell.address);
-    relay_send_end_cell_from_edge(rh.stream_id, circ, end_reason, NULL);
+    relay_send_end_cell_from_edge(rh.stream_id, circ, end_reason, layer_hint);
     return 0;
   }
 
@@ -3057,22 +3800,21 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
     port = bcell.port;
 
     if (or_circ && or_circ->p_chan) {
-      if (!options->AllowSingleHopExits &&
-           (or_circ->is_first_hop ||
-            (!connection_or_digest_is_known_relay(
+      const int client_chan = channel_is_client(or_circ->p_chan);
+      if ((client_chan ||
+           (!connection_or_digest_is_known_relay(
                 or_circ->p_chan->identity_digest) &&
           should_refuse_unknown_exits(options)))) {
-        /* Don't let clients use us as a single-hop proxy, unless the user
-         * has explicitly allowed that in the config. It attracts attackers
-         * and users who'd be better off with, well, single-hop proxies.
-         */
+        /* Don't let clients use us as a single-hop proxy. It attracts
+         * attackers and users who'd be better off with, well, single-hop
+         * proxies. */
         log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
                "Attempt by %s to open a stream %s. Closing.",
                safe_str(channel_get_canonical_remote_descr(or_circ->p_chan)),
-               or_circ->is_first_hop ? "on first hop of circuit" :
-                                       "from unknown relay");
+               client_chan ? "on first hop of circuit" :
+                             "from unknown relay");
         relay_send_end_cell_from_edge(rh.stream_id, circ,
-                                      or_circ->is_first_hop ?
+                                      client_chan ?
                                         END_STREAM_REASON_TORPROTOCOL :
                                         END_STREAM_REASON_MISC,
                                       NULL);
@@ -3084,7 +3826,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
     if (!directory_permits_begindir_requests(options) ||
         circ->purpose != CIRCUIT_PURPOSE_OR) {
       relay_send_end_cell_from_edge(rh.stream_id, circ,
-                                    END_STREAM_REASON_NOTDIRECTORY, NULL);
+                                  END_STREAM_REASON_NOTDIRECTORY, layer_hint);
       return 0;
     }
     /* Make sure to get the 'real' address of the previous hop: the
@@ -3101,7 +3843,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
   } else {
     log_warn(LD_BUG, "Got an unexpected command %d", (int)rh.command);
     relay_send_end_cell_from_edge(rh.stream_id, circ,
-                                  END_STREAM_REASON_INTERNAL, NULL);
+                                  END_STREAM_REASON_INTERNAL, layer_hint);
     return 0;
   }
 
@@ -3112,7 +3854,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
     if (bcell.flags & BEGIN_FLAG_IPV4_NOT_OK) {
       tor_free(address);
       relay_send_end_cell_from_edge(rh.stream_id, circ,
-                                    END_STREAM_REASON_EXITPOLICY, NULL);
+                                    END_STREAM_REASON_EXITPOLICY, layer_hint);
       return 0;
     }
   }
@@ -3135,68 +3877,28 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
   n_stream->deliver_window = STREAMWINDOW_START;
 
   if (circ->purpose == CIRCUIT_PURPOSE_S_REND_JOINED) {
-    origin_circuit_t *origin_circ = TO_ORIGIN_CIRCUIT(circ);
-    log_info(LD_REND,"begin is for rendezvous. configuring stream.");
-    n_stream->base_.address = tor_strdup("(rendezvous)");
-    n_stream->base_.state = EXIT_CONN_STATE_CONNECTING;
-    n_stream->rend_data = rend_data_dup(origin_circ->rend_data);
-    tor_assert(connection_edge_is_rendezvous_stream(n_stream));
-    assert_circuit_ok(circ);
-
-    const int r = rend_service_set_connection_addr_port(n_stream, origin_circ);
-    if (r < 0) {
-      log_info(LD_REND,"Didn't find rendezvous service (port %d)",
-               n_stream->base_.port);
-      /* Send back reason DONE because we want to make hidden service port
-       * scanning harder thus instead of returning that the exit policy
-       * didn't match, which makes it obvious that the port is closed,
-       * return DONE and kill the circuit. That way, a user (malicious or
-       * not) needs one circuit per bad port unless it matches the policy of
-       * the hidden service. */
-      relay_send_end_cell_from_edge(rh.stream_id, circ,
-                                    END_STREAM_REASON_DONE,
-                                    origin_circ->cpath->prev);
-      connection_free(TO_CONN(n_stream));
-      tor_free(address);
+    int ret;
+    tor_free(address);
+    /* We handle this circuit and stream in this function for all supported
+     * hidden service version. */
+    ret = handle_hs_exit_conn(circ, n_stream);
 
-      /* Drop the circuit here since it might be someone deliberately
-       * scanning the hidden service ports. Note that this mitigates port
-       * scanning by adding more work on the attacker side to successfully
-       * scan but does not fully solve it. */
-      if (r < -1)
-        return END_CIRC_AT_ORIGIN;
-      else
-        return 0;
+    if (ret == 0) {
+      /* This was a valid cell. Count it as delivered + overhead. */
+      circuit_read_valid_data(origin_circ, rh.length);
     }
-    assert_circuit_ok(circ);
-    log_debug(LD_REND,"Finished assigning addr/port");
-    n_stream->cpath_layer = origin_circ->cpath->prev; /* link it */
-
-    /* add it into the linked list of p_streams on this circuit */
-    n_stream->next_stream = origin_circ->p_streams;
-    n_stream->on_circuit = circ;
-    origin_circ->p_streams = n_stream;
-    assert_circuit_ok(circ);
-
-    origin_circ->rend_data->nr_streams++;
-
-    connection_exit_connect(n_stream);
-
-    /* For path bias: This circuit was used successfully */
-    pathbias_mark_use_success(origin_circ);
-
-    tor_free(address);
-    return 0;
+    return ret;
   }
   tor_strlower(address);
   n_stream->base_.address = address;
   n_stream->base_.state = EXIT_CONN_STATE_RESOLVEFAILED;
   /* default to failed, change in dns_resolve if it turns out not to fail */
 
+  /* If we're hibernating or shutting down, we refuse to open new streams. */
   if (we_are_hibernating()) {
     relay_send_end_cell_from_edge(rh.stream_id, circ,
                                   END_STREAM_REASON_HIBERNATING, NULL);
-    connection_free(TO_CONN(n_stream));
+    connection_free_(TO_CONN(n_stream));
     return 0;
   }
 
@@ -3274,7 +3976,7 @@ connection_exit_begin_resolve(cell_t *cell, or_circuit_t *circ)
       return 0;
     case 1: /* The result was cached; a resolved cell was sent. */
       if (!dummy_conn->base_.marked_for_close)
-        connection_free(TO_CONN(dummy_conn));
+        connection_free_(TO_CONN(dummy_conn));
       return 0;
     case 0: /* resolve added to pending list */
       assert_circuit_ok(TO_CIRCUIT(circ));
@@ -3447,8 +4149,8 @@ connection_exit_connect_dir(edge_connection_t *exitconn)
 
   if (connection_add(TO_CONN(exitconn))<0) {
     connection_edge_end(exitconn, END_STREAM_REASON_RESOURCELIMIT);
-    connection_free(TO_CONN(exitconn));
-    connection_free(TO_CONN(dirconn));
+    connection_free_(TO_CONN(exitconn));
+    connection_free_(TO_CONN(dirconn));
     return 0;
   }
 
@@ -3460,7 +4162,7 @@ connection_exit_connect_dir(edge_connection_t *exitconn)
     connection_edge_end(exitconn, END_STREAM_REASON_RESOURCELIMIT);
     connection_close_immediate(TO_CONN(exitconn));
     connection_mark_for_close(TO_CONN(exitconn));
-    connection_free(TO_CONN(dirconn));
+    connection_free_(TO_CONN(dirconn));
     return 0;
   }
 
@@ -3481,11 +4183,15 @@ connection_exit_connect_dir(edge_connection_t *exitconn)
  * it is a general stream.
  */
 int
-connection_edge_is_rendezvous_stream(edge_connection_t *conn)
+connection_edge_is_rendezvous_stream(const edge_connection_t *conn)
 {
   tor_assert(conn);
-  if (conn->rend_data)
+  /* It should not be possible to set both of these structs */
+  tor_assert_nonfatal(!(conn->rend_data && conn->hs_ident));
+
+  if (conn->rend_data || conn->hs_ident) {
     return 1;
+  }
   return 0;
 }
 
@@ -3509,7 +4215,7 @@ connection_ap_can_use_exit(const entry_connection_t *conn,
    */
   if (conn->chosen_exit_name) {
     const node_t *chosen_exit =
-      node_get_by_nickname(conn->chosen_exit_name, 1);
+      node_get_by_nickname(conn->chosen_exit_name, 0);
     if (!chosen_exit || tor_memneq(chosen_exit->identity,
                                exit_node->identity, DIGEST_LEN)) {
       /* doesn't match */
@@ -3558,10 +4264,12 @@ connection_ap_can_use_exit(const entry_connection_t *conn,
 }
 
 /** If address is of the form "y.onion" with a well-formed handle y:
- *     Put a NUL after y, lower-case it, and return ONION_HOSTNAME.
+ *     Put a NUL after y, lower-case it, and return ONION_V2_HOSTNAME or
+ *     ONION_V3_HOSTNAME depending on the HS version.
  *
  *  If address is of the form "x.y.onion" with a well-formed handle x:
- *     Drop "x.", put a NUL after y, lower-case it, and return ONION_HOSTNAME.
+ *     Drop "x.", put a NUL after y, lower-case it, and return
+ *     ONION_V2_HOSTNAME or ONION_V3_HOSTNAME depending on the HS version.
  *
  * If address is of the form "y.onion" with a badly-formed handle y:
  *     Return BAD_HOSTNAME and log a message.
@@ -3577,7 +4285,7 @@ parse_extended_hostname(char *address)
 {
     char *s;
     char *q;
-    char query[REND_SERVICE_ID_LEN_BASE32+1];
+    char query[HS_SERVICE_ADDR_LEN_BASE32+1];
 
     s = strrchr(address,'.');
     if (!s)
@@ -3597,14 +4305,17 @@ parse_extended_hostname(char *address)
       goto failed; /* reject sub-domain, as DNS does */
     }
     q = (NULL == q) ? address : q + 1;
-    if (strlcpy(query, q, REND_SERVICE_ID_LEN_BASE32+1) >=
-        REND_SERVICE_ID_LEN_BASE32+1)
+    if (strlcpy(query, q, HS_SERVICE_ADDR_LEN_BASE32+1) >=
+        HS_SERVICE_ADDR_LEN_BASE32+1)
       goto failed;
     if (q != address) {
       memmove(address, q, strlen(q) + 1 /* also get \0 */);
     }
-    if (rend_valid_service_id(query)) {
-      return ONION_HOSTNAME; /* success */
+    if (rend_valid_v2_service_id(query)) {
+      return ONION_V2_HOSTNAME; /* success */
+    }
+    if (hs_address_is_valid(query)) {
+      return ONION_V3_HOSTNAME;
     }
  failed:
     /* otherwise, return to previous state and return 0 */
@@ -3824,4 +4535,5 @@ connection_edge_free_all(void)
   untried_pending_connections = 0;
   smartlist_free(pending_entry_connections);
   pending_entry_connections = NULL;
+  mainloop_event_free(attach_pending_entry_connections_ev);
 }
diff --git a/src/or/connection_edge.h b/src/core/or/connection_edge.h
index 5dfc8af901..68d8b19a11 100644
--- a/src/or/connection_edge.h
+++ b/src/core/or/connection_edge.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,9 +12,63 @@
 #ifndef TOR_CONNECTION_EDGE_H
 #define TOR_CONNECTION_EDGE_H
 
-#include "testsupport.h"
+#include "lib/testsupport/testsupport.h"
 
-#define connection_mark_unattached_ap(conn, endreason) \
+#include "feature/hs/hs_service.h"
+
+edge_connection_t *TO_EDGE_CONN(connection_t *);
+entry_connection_t *TO_ENTRY_CONN(connection_t *);
+entry_connection_t *EDGE_TO_ENTRY_CONN(edge_connection_t *);
+
+#define EXIT_CONN_STATE_MIN_ 1
+/** State for an exit connection: waiting for response from DNS farm. */
+#define EXIT_CONN_STATE_RESOLVING 1
+/** State for an exit connection: waiting for connect() to finish. */
+#define EXIT_CONN_STATE_CONNECTING 2
+/** State for an exit connection: open and ready to transmit data. */
+#define EXIT_CONN_STATE_OPEN 3
+/** State for an exit connection: waiting to be removed. */
+#define EXIT_CONN_STATE_RESOLVEFAILED 4
+#define EXIT_CONN_STATE_MAX_ 4
+
+/* The AP state values must be disjoint from the EXIT state values. */
+#define AP_CONN_STATE_MIN_ 5
+/** State for a SOCKS connection: waiting for SOCKS request. */
+#define AP_CONN_STATE_SOCKS_WAIT 5
+/** State for a SOCKS connection: got a y.onion URL; waiting to receive
+ * rendezvous descriptor. */
+#define AP_CONN_STATE_RENDDESC_WAIT 6
+/** The controller will attach this connection to a circuit; it isn't our
+ * job to do so. */
+#define AP_CONN_STATE_CONTROLLER_WAIT 7
+/** State for a SOCKS connection: waiting for a completed circuit. */
+#define AP_CONN_STATE_CIRCUIT_WAIT 8
+/** State for a SOCKS connection: sent BEGIN, waiting for CONNECTED. */
+#define AP_CONN_STATE_CONNECT_WAIT 9
+/** State for a SOCKS connection: sent RESOLVE, waiting for RESOLVED. */
+#define AP_CONN_STATE_RESOLVE_WAIT 10
+/** State for a SOCKS connection: ready to send and receive. */
+#define AP_CONN_STATE_OPEN 11
+/** State for a transparent natd connection: waiting for original
+ * destination. */
+#define AP_CONN_STATE_NATD_WAIT 12
+/** State for an HTTP tunnel: waiting for an HTTP CONNECT command. */
+#define AP_CONN_STATE_HTTP_CONNECT_WAIT 13
+#define AP_CONN_STATE_MAX_ 13
+
+#define EXIT_PURPOSE_MIN_ 1
+/** This exit stream wants to do an ordinary connect. */
+#define EXIT_PURPOSE_CONNECT 1
+/** This exit stream wants to do a resolve (either normal or reverse). */
+#define EXIT_PURPOSE_RESOLVE 2
+#define EXIT_PURPOSE_MAX_ 2
+
+/** True iff the AP_CONN_STATE_* value <b>s</b> means that the corresponding
+ * edge connection is not attached to any circuit. */
+#define AP_CONN_STATE_IS_UNATTACHED(s) \
+  ((s) <= AP_CONN_STATE_CIRCUIT_WAIT || (s) == AP_CONN_STATE_NATD_WAIT)
+
+#define connection_mark_unattached_ap(conn, endreason)                  \
   connection_mark_unattached_ap_((conn), (endreason), __LINE__, SHORT_FILE__)
 
 MOCK_DECL(void,connection_mark_unattached_ap_,
@@ -33,7 +87,8 @@ int connection_edge_finished_connecting(edge_connection_t *conn);
 void connection_ap_about_to_close(entry_connection_t *edge_conn);
 void connection_exit_about_to_close(edge_connection_t *edge_conn);
 
-int connection_ap_handshake_send_begin(entry_connection_t *ap_conn);
+MOCK_DECL(int,
+          connection_ap_handshake_send_begin,(entry_connection_t *ap_conn));
 int connection_ap_handshake_send_resolve(entry_connection_t *ap_conn);
 
 entry_connection_t  *connection_ap_make_link(connection_t *partner,
@@ -60,7 +115,7 @@ void connection_ap_handshake_socks_resolved_addr(entry_connection_t *conn,
 int connection_exit_begin_conn(cell_t *cell, circuit_t *circ);
 int connection_exit_begin_resolve(cell_t *cell, or_circuit_t *circ);
 void connection_exit_connect(edge_connection_t *conn);
-int connection_edge_is_rendezvous_stream(edge_connection_t *conn);
+int connection_edge_is_rendezvous_stream(const edge_connection_t *conn);
 int connection_ap_can_use_exit(const entry_connection_t *conn,
                                const node_t *exit);
 void connection_ap_expire_beginning(void);
@@ -71,6 +126,9 @@ void connection_ap_mark_as_pending_circuit_(entry_connection_t *entry_conn,
 #define connection_ap_mark_as_pending_circuit(c) \
   connection_ap_mark_as_pending_circuit_((c), __FILE__, __LINE__)
 void connection_ap_mark_as_non_pending_circuit(entry_connection_t *entry_conn);
+void connection_ap_mark_as_waiting_for_renddesc(
+                                       entry_connection_t *entry_conn);
+
 #define CONNECTION_AP_EXPECT_NONPENDING(c) do {                         \
     if (ENTRY_TO_CONN(c)->state == AP_CONN_STATE_CIRCUIT_WAIT) {        \
       log_warn(LD_BUG, "At %s:%d: %p was unexpectedly in circuit_wait.", \
@@ -88,16 +146,18 @@ int connection_ap_process_transparent(entry_connection_t *conn);
 
 int address_is_invalid_destination(const char *address, int client);
 
-int connection_ap_rewrite_and_attach_if_allowed(entry_connection_t *conn,
-                                                origin_circuit_t *circ,
-                                                crypt_path_t *cpath);
+MOCK_DECL(int, connection_ap_rewrite_and_attach_if_allowed,
+                                                (entry_connection_t *conn,
+                                                 origin_circuit_t *circ,
+                                                 crypt_path_t *cpath));
 int connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
                                                origin_circuit_t *circ,
                                                crypt_path_t *cpath);
 
 /** Possible return values for parse_extended_hostname. */
 typedef enum hostname_type_t {
-  NORMAL_HOSTNAME, ONION_HOSTNAME, EXIT_HOSTNAME, BAD_HOSTNAME
+  NORMAL_HOSTNAME, ONION_V2_HOSTNAME, ONION_V3_HOSTNAME,
+  EXIT_HOSTNAME, BAD_HOSTNAME
 } hostname_type_t;
 hostname_type_t parse_extended_hostname(char *address);
 
@@ -119,6 +179,23 @@ void connection_ap_warn_and_unmark_if_pending_circ(
                                              entry_connection_t *entry_conn,
                                              const char *where);
 
+int connection_half_edge_is_valid_data(const smartlist_t *half_conns,
+                                       streamid_t stream_id);
+int connection_half_edge_is_valid_sendme(const smartlist_t *half_conns,
+                                         streamid_t stream_id);
+int connection_half_edge_is_valid_connected(const smartlist_t *half_conns,
+                                            streamid_t stream_id);
+int connection_half_edge_is_valid_end(smartlist_t *half_conns,
+                                      streamid_t stream_id);
+int connection_half_edge_is_valid_resolved(smartlist_t *half_conns,
+                                           streamid_t stream_id);
+
+size_t half_streams_get_total_allocation(void);
+struct half_edge_t;
+void half_edge_free_(struct half_edge_t *he);
+#define half_edge_free(he) \
+  FREE_AND_NULL(half_edge_t, half_edge_free_, (he))
+
 /** @name Begin-cell flags
  *
  * These flags are used in RELAY_BEGIN cells to change the default behavior
@@ -186,7 +263,17 @@ typedef struct {
 
 STATIC void connection_ap_handshake_rewrite(entry_connection_t *conn,
                                             rewrite_result_t *out);
-#endif
 
-#endif
+STATIC int connection_ap_process_http_connect(entry_connection_t *conn);
+STATIC void export_hs_client_circuit_id(edge_connection_t *edge_conn,
+                            hs_circuit_id_protocol_t protocol);
+
+struct half_edge_t;
+STATIC void connection_half_edge_add(const edge_connection_t *conn,
+                                     origin_circuit_t *circ);
+STATIC struct half_edge_t *connection_half_edge_find_stream_id(
+                                     const smartlist_t *half_conns,
+                                     streamid_t stream_id);
+#endif /* defined(CONNECTION_EDGE_PRIVATE) */
 
+#endif /* !defined(TOR_CONNECTION_EDGE_H) */
diff --git a/src/or/connection_or.c b/src/core/or/connection_or.c
index a01d086279..67157d8d0b 100644
--- a/src/or/connection_or.c
+++ b/src/core/or/connection_or.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,39 +20,63 @@
  *
  * This module also implements the client side of the v3 Tor link handshake,
  **/
-#include "or.h"
-#include "buffers.h"
+#include "core/or/or.h"
+#include "feature/client/bridges.h"
+#include "lib/container/buffers.h"
 /*
  * Define this so we get channel internal functions, since we're implementing
  * part of a subclass (channel_tls_t).
  */
 #define TOR_CHANNEL_INTERNAL_
-#include "channel.h"
-#include "channeltls.h"
-#include "circuitbuild.h"
-#include "circuitlist.h"
-#include "circuitstats.h"
-#include "command.h"
-#include "config.h"
-#include "connection.h"
-#include "connection_or.h"
-#include "control.h"
-#include "dirserv.h"
-#include "entrynodes.h"
-#include "geoip.h"
-#include "main.h"
-#include "link_handshake.h"
-#include "microdesc.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "reasons.h"
-#include "relay.h"
-#include "rendcommon.h"
-#include "rephist.h"
-#include "router.h"
-#include "routerlist.h"
-#include "ext_orport.h"
-#include "scheduler.h"
+#define CONNECTION_OR_PRIVATE
+#include "core/or/channel.h"
+#include "core/or/channeltls.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitstats.h"
+#include "core/or/command.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/connection_or.h"
+#include "feature/control/control.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "feature/dirauth/reachability.h"
+#include "feature/client/entrynodes.h"
+#include "lib/geoip/geoip.h"
+#include "core/mainloop/mainloop.h"
+#include "trunnel/link_handshake.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "core/proto/proto_cell.h"
+#include "core/or/reasons.h"
+#include "core/or/relay.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/stats/rephist.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routerkeys.h"
+#include "feature/relay/routermode.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/ext_orport.h"
+#include "core/or/scheduler.h"
+#include "feature/nodelist/torcert.h"
+#include "core/or/channelpadding.h"
+#include "feature/dirauth/authmode.h"
+
+#include "core/or/cell_st.h"
+#include "core/or/cell_queue_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/or_handshake_certs_st.h"
+#include "core/or/or_handshake_state_st.h"
+#include "app/config/or_state_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "core/or/var_cell_st.h"
+#include "lib/crypt_ops/crypto_format.h"
+
+#include "lib/tls/tortls.h"
+#include "lib/tls/x509.h"
 
 static int connection_tls_finish_handshake(or_connection_t *conn);
 static int connection_or_launch_v3_or_handshake(or_connection_t *conn);
@@ -74,56 +98,34 @@ static void connection_or_mark_bad_for_new_circs(or_connection_t *or_conn);
 
 static void connection_or_change_state(or_connection_t *conn, uint8_t state);
 
+static void connection_or_check_canonicity(or_connection_t *conn,
+                                           int started_here);
+
 /**************************************************************/
 
-/** Map from identity digest of connected OR or desired OR to a connection_t
- * with that identity digest.  If there is more than one such connection_t,
- * they form a linked list, with next_with_same_id as the next pointer. */
-static digestmap_t *orconn_identity_map = NULL;
+/** Convert a connection_t* to an or_connection_t*; assert if the cast is
+ * invalid. */
+or_connection_t *
+TO_OR_CONN(connection_t *c)
+{
+  tor_assert(c->magic == OR_CONNECTION_MAGIC);
+  return DOWNCAST(or_connection_t, c);
+}
 
 /** Global map between Extended ORPort identifiers and OR
  *  connections. */
 static digestmap_t *orconn_ext_or_id_map = NULL;
 
-/** If conn is listed in orconn_identity_map, remove it, and clear
- * conn->identity_digest.  Otherwise do nothing. */
+/** Clear clear conn->identity_digest and update other data
+ * structures as appropriate.*/
 void
-connection_or_remove_from_identity_map(or_connection_t *conn)
+connection_or_clear_identity(or_connection_t *conn)
 {
-  or_connection_t *tmp;
   tor_assert(conn);
-  if (!orconn_identity_map)
-    return;
-  tmp = digestmap_get(orconn_identity_map, conn->identity_digest);
-  if (!tmp) {
-    if (!tor_digest_is_zero(conn->identity_digest)) {
-      log_warn(LD_BUG, "Didn't find connection '%s' on identity map when "
-               "trying to remove it.",
-               conn->nickname ? conn->nickname : "NULL");
-    }
-    return;
-  }
-  if (conn == tmp) {
-    if (conn->next_with_same_id)
-      digestmap_set(orconn_identity_map, conn->identity_digest,
-                    conn->next_with_same_id);
-    else
-      digestmap_remove(orconn_identity_map, conn->identity_digest);
-  } else {
-    while (tmp->next_with_same_id) {
-      if (tmp->next_with_same_id == conn) {
-        tmp->next_with_same_id = conn->next_with_same_id;
-        break;
-      }
-      tmp = tmp->next_with_same_id;
-    }
-  }
   memset(conn->identity_digest, 0, DIGEST_LEN);
-  conn->next_with_same_id = NULL;
 }
 
-/** Remove all entries from the identity-to-orconn map, and clear
- * all identities in OR conns.*/
+/** Clear all identities in OR conns.*/
 void
 connection_or_clear_identity_map(void)
 {
@@ -131,57 +133,72 @@ connection_or_clear_identity_map(void)
   SMARTLIST_FOREACH(conns, connection_t *, conn,
   {
     if (conn->type == CONN_TYPE_OR) {
-      or_connection_t *or_conn = TO_OR_CONN(conn);
-      memset(or_conn->identity_digest, 0, DIGEST_LEN);
-      or_conn->next_with_same_id = NULL;
+      connection_or_clear_identity(TO_OR_CONN(conn));
     }
   });
-
-  digestmap_free(orconn_identity_map, NULL);
-  orconn_identity_map = NULL;
 }
 
 /** Change conn->identity_digest to digest, and add conn into
- * orconn_digest_map. */
+ * the appropriate digest maps.
+ *
+ * NOTE that this function only allows two kinds of transitions: from
+ * unset identity to set identity, and from idempotent re-settings
+ * of the same identity.  It's not allowed to clear an identity or to
+ * change an identity.  Return 0 on success, and -1 if the transition
+ * is not allowed.
+ **/
 static void
-connection_or_set_identity_digest(or_connection_t *conn, const char *digest)
+connection_or_set_identity_digest(or_connection_t *conn,
+                                  const char *rsa_digest,
+                                  const ed25519_public_key_t *ed_id)
 {
-  or_connection_t *tmp;
+  channel_t *chan = NULL;
   tor_assert(conn);
-  tor_assert(digest);
+  tor_assert(rsa_digest);
+
+  if (conn->chan)
+    chan = TLS_CHAN_TO_BASE(conn->chan);
 
-  if (!orconn_identity_map)
-    orconn_identity_map = digestmap_new();
-  if (tor_memeq(conn->identity_digest, digest, DIGEST_LEN))
+  log_info(LD_HANDSHAKE, "Set identity digest for %p (%s): %s %s.",
+           conn,
+           escaped_safe_str(conn->base_.address),
+           hex_str(rsa_digest, DIGEST_LEN),
+           ed25519_fmt(ed_id));
+  log_info(LD_HANDSHAKE, "   (Previously: %s %s)",
+           hex_str(conn->identity_digest, DIGEST_LEN),
+           chan ? ed25519_fmt(&chan->ed25519_identity) : "<null>");
+
+  const int rsa_id_was_set = ! tor_digest_is_zero(conn->identity_digest);
+  const int ed_id_was_set =
+    chan && !ed25519_public_key_is_zero(&chan->ed25519_identity);
+  const int rsa_changed =
+    tor_memneq(conn->identity_digest, rsa_digest, DIGEST_LEN);
+  const int ed_changed = ed_id_was_set &&
+    (!ed_id || !ed25519_pubkey_eq(ed_id, &chan->ed25519_identity));
+
+  tor_assert(!rsa_changed || !rsa_id_was_set);
+  tor_assert(!ed_changed || !ed_id_was_set);
+
+  if (!rsa_changed && !ed_changed)
     return;
 
   /* If the identity was set previously, remove the old mapping. */
-  if (! tor_digest_is_zero(conn->identity_digest)) {
-    connection_or_remove_from_identity_map(conn);
-    if (conn->chan)
-      channel_clear_identity_digest(TLS_CHAN_TO_BASE(conn->chan));
+  if (rsa_id_was_set) {
+    connection_or_clear_identity(conn);
+    if (chan)
+      channel_clear_identity_digest(chan);
   }
 
-  memcpy(conn->identity_digest, digest, DIGEST_LEN);
+  memcpy(conn->identity_digest, rsa_digest, DIGEST_LEN);
 
-  /* If we're setting the ID to zero, don't add a mapping. */
-  if (tor_digest_is_zero(digest))
+  /* If we're initializing the IDs to zero, don't add a mapping yet. */
+  if (tor_digest_is_zero(rsa_digest) &&
+      (!ed_id || ed25519_public_key_is_zero(ed_id)))
     return;
 
-  tmp = digestmap_set(orconn_identity_map, digest, conn);
-  conn->next_with_same_id = tmp;
-
   /* Deal with channels */
-  if (conn->chan)
-    channel_set_identity_digest(TLS_CHAN_TO_BASE(conn->chan), digest);
-
-#if 1
-  /* Testing code to check for bugs in representation. */
-  for (; tmp; tmp = tmp->next_with_same_id) {
-    tor_assert(tor_memeq(tmp->identity_digest, digest, DIGEST_LEN));
-    tor_assert(tmp != conn);
-  }
-#endif
+  if (chan)
+    channel_set_identity_digest(chan, rsa_digest, ed_id);
 }
 
 /** Remove the Extended ORPort identifier of <b>conn</b> from the
@@ -485,7 +502,7 @@ var_cell_pack_header(const var_cell_t *cell, char *hdr_out, int wide_circ_ids)
 var_cell_t *
 var_cell_new(uint16_t payload_len)
 {
-  size_t size = STRUCT_OFFSET(var_cell_t, payload) + payload_len;
+  size_t size = offsetof(var_cell_t, payload) + payload_len;
   var_cell_t *cell = tor_malloc_zero(size);
   cell->payload_len = payload_len;
   cell->command = 0;
@@ -504,7 +521,7 @@ var_cell_copy(const var_cell_t *src)
   size_t size = 0;
 
   if (src != NULL) {
-    size = STRUCT_OFFSET(var_cell_t, payload) + src->payload_len;
+    size = offsetof(var_cell_t, payload) + src->payload_len;
     copy = tor_malloc_zero(size);
     copy->payload_len = src->payload_len;
     copy->command = src->command;
@@ -517,7 +534,7 @@ var_cell_copy(const var_cell_t *src)
 
 /** Release all space held by <b>cell</b>. */
 void
-var_cell_free(var_cell_t *cell)
+var_cell_free_(var_cell_t *cell)
 {
   tor_free(cell);
 }
@@ -604,8 +621,9 @@ connection_or_flushed_some(or_connection_t *conn)
 {
   size_t datalen;
 
-  /* The channel will want to update its estimated queue size */
-  channel_update_xmit_queue_size(TLS_CHAN_TO_BASE(conn->chan));
+  /* Update the channel's active timestamp if there is one */
+  if (conn->chan)
+    channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
 
   /* If we're under the low water mark, add cells until we're just over the
    * high water mark. */
@@ -667,6 +685,11 @@ connection_or_finished_flushing(or_connection_t *conn)
       tor_fragile_assert();
       return -1;
   }
+
+  /* Update the channel's active timestamp if there is one */
+  if (conn->chan)
+    channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
+
   return 0;
 }
 
@@ -685,6 +708,7 @@ connection_or_finished_connecting(or_connection_t *or_conn)
   log_debug(LD_HANDSHAKE,"OR connect() to router at %s:%u finished.",
             conn->address,conn->port);
   control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0);
+  control_event_boot_first_orconn();
 
   if (proxy_type != PROXY_NONE) {
     /* start proxy handshake */
@@ -711,7 +735,6 @@ connection_or_finished_connecting(or_connection_t *or_conn)
 void
 connection_or_about_to_close(or_connection_t *or_conn)
 {
-  time_t now = time(NULL);
   connection_t *conn = TO_CONN(or_conn);
 
   /* Tell the controlling channel we're closed */
@@ -731,15 +754,14 @@ connection_or_about_to_close(or_connection_t *or_conn)
     if (connection_or_nonopen_was_started_here(or_conn)) {
       const or_options_t *options = get_options();
       connection_or_note_state_when_broken(or_conn);
-      rep_hist_note_connect_failed(or_conn->identity_digest, now);
-      entry_guard_register_connect_status(or_conn->identity_digest,0,
-                                          !options->HTTPSProxy, now);
+      /* Tell the new guard API about the channel failure */
+      entry_guard_chan_failed(TLS_CHAN_TO_BASE(or_conn->chan));
       if (conn->state >= OR_CONN_STATE_TLS_HANDSHAKING) {
         int reason = tls_error_to_orconn_end_reason(or_conn->tls_error);
         control_event_or_conn_status(or_conn, OR_CONN_EVENT_FAILED,
                                      reason);
         if (!authdir_mode_tests_reachability(options))
-          control_event_bootstrap_problem(
+          control_event_bootstrap_prob_or(
                 orconn_end_reason_to_control_string(reason),
                 reason, or_conn);
       }
@@ -747,11 +769,9 @@ connection_or_about_to_close(or_connection_t *or_conn)
   } else if (conn->hold_open_until_flushed) {
     /* We only set hold_open_until_flushed when we're intentionally
      * closing a connection. */
-    rep_hist_note_disconnect(or_conn->identity_digest, now);
     control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED,
                 tls_error_to_orconn_end_reason(or_conn->tls_error));
   } else if (!tor_digest_is_zero(or_conn->identity_digest)) {
-    rep_hist_note_connection_died(or_conn->identity_digest, now);
     control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED,
                 tls_error_to_orconn_end_reason(or_conn->tls_error));
   }
@@ -802,18 +822,10 @@ connection_or_update_token_buckets_helper(or_connection_t *conn, int reset,
                                 (int)options->BandwidthBurst, 1, INT32_MAX);
   }
 
-  conn->bandwidthrate = rate;
-  conn->bandwidthburst = burst;
-  if (reset) { /* set up the token buckets to be full */
-    conn->read_bucket = conn->write_bucket = burst;
-    return;
+  token_bucket_rw_adjust(&conn->bucket, rate, burst);
+  if (reset) {
+    token_bucket_rw_reset(&conn->bucket, monotime_coarse_get_stamp());
   }
-  /* If the new token bucket is smaller, take out the extra tokens.
-   * (If it's larger, don't -- the buckets can grow to reach the cap.) */
-  if (conn->read_bucket > burst)
-    conn->read_bucket = burst;
-  if (conn->write_bucket > burst)
-    conn->write_bucket = burst;
 }
 
 /** Either our set of relays or our per-conn rate limits have changed.
@@ -830,24 +842,6 @@ connection_or_update_token_buckets(smartlist_t *conns,
   });
 }
 
-/** How long do we wait before killing non-canonical OR connections with no
- * circuits?  In Tor versions up to 0.2.1.25 and 0.2.2.12-alpha, we waited 15
- * minutes before cancelling these connections, which caused fast relays to
- * accrue many many idle connections. Hopefully 3-4.5 minutes is low enough
- * that it kills most idle connections, without being so low that we cause
- * clients to bounce on and off.
- *
- * For canonical connections, the limit is higher, at 15-22.5 minutes.
- *
- * For each OR connection, we randomly add up to 50% extra to its idle_timeout
- * field, to avoid exposing when exactly the last circuit closed.  Since we're
- * storing idle_timeout in a uint16_t, don't let these values get higher than
- * 12 hours or so without revising connection_or_set_canonical and/or expanding
- * idle_timeout.
- */
-#define IDLE_OR_CONN_TIMEOUT_NONCANONICAL 180
-#define IDLE_OR_CONN_TIMEOUT_CANONICAL 900
-
 /* Mark <b>or_conn</b> as canonical if <b>is_canonical</b> is set, and
  * non-canonical otherwise. Adjust idle_timeout accordingly.
  */
@@ -855,9 +849,6 @@ void
 connection_or_set_canonical(or_connection_t *or_conn,
                             int is_canonical)
 {
-  const unsigned int timeout_base = is_canonical ?
-    IDLE_OR_CONN_TIMEOUT_CANONICAL : IDLE_OR_CONN_TIMEOUT_NONCANONICAL;
-
   if (bool_eq(is_canonical, or_conn->is_canonical) &&
       or_conn->idle_timeout != 0) {
     /* Don't recalculate an existing idle_timeout unless the canonical
@@ -866,7 +857,14 @@ connection_or_set_canonical(or_connection_t *or_conn,
   }
 
   or_conn->is_canonical = !! is_canonical; /* force to a 1-bit boolean */
-  or_conn->idle_timeout = timeout_base + crypto_rand_int(timeout_base / 2);
+  or_conn->idle_timeout = channelpadding_get_channel_idle_timeout(
+          TLS_CHAN_TO_BASE(or_conn->chan), is_canonical);
+
+  log_info(LD_CIRC,
+          "Channel %"PRIu64 " chose an idle timeout of %d.",
+          or_conn->chan ?
+          (TLS_CHAN_TO_BASE(or_conn->chan)->global_identifier):0,
+          or_conn->idle_timeout);
 }
 
 /** If we don't necessarily know the router we're connecting to, but we
@@ -878,15 +876,47 @@ void
 connection_or_init_conn_from_address(or_connection_t *conn,
                                      const tor_addr_t *addr, uint16_t port,
                                      const char *id_digest,
+                                     const ed25519_public_key_t *ed_id,
                                      int started_here)
 {
-  const node_t *r = node_get_by_id(id_digest);
-  connection_or_set_identity_digest(conn, id_digest);
+  log_debug(LD_HANDSHAKE, "init conn from address %s: %s, %s (%d)",
+            fmt_addr(addr),
+            hex_str((const char*)id_digest, DIGEST_LEN),
+            ed25519_fmt(ed_id),
+            started_here);
+
+  connection_or_set_identity_digest(conn, id_digest, ed_id);
   connection_or_update_token_buckets_helper(conn, 1, get_options());
 
   conn->base_.port = port;
   tor_addr_copy(&conn->base_.addr, addr);
   tor_addr_copy(&conn->real_addr, addr);
+
+  connection_or_check_canonicity(conn, started_here);
+}
+
+/** Check whether the identity of <b>conn</b> matches a known node.  If it
+ * does, check whether the address of conn matches the expected address, and
+ * update the connection's is_canonical flag, nickname, and address fields as
+ * appropriate. */
+static void
+connection_or_check_canonicity(or_connection_t *conn, int started_here)
+{
+  const char *id_digest = conn->identity_digest;
+  const ed25519_public_key_t *ed_id = NULL;
+  const tor_addr_t *addr = &conn->real_addr;
+  if (conn->chan)
+    ed_id = & TLS_CHAN_TO_BASE(conn->chan)->ed25519_identity;
+
+  const node_t *r = node_get_by_id(id_digest);
+  if (r &&
+      node_supports_ed25519_link_authentication(r, 1) &&
+      ! node_ed25519_id_matches(r, ed_id)) {
+    /* If this node is capable of proving an ed25519 ID,
+     * we can't call this a canonical connection unless both IDs match. */
+     r = NULL;
+  }
+
   if (r) {
     tor_addr_port_t node_ap;
     node_get_pref_orport(r, &node_ap);
@@ -908,10 +938,12 @@ connection_or_init_conn_from_address(or_connection_t *conn,
       tor_addr_copy(&conn->base_.addr, &node_ap.addr);
       conn->base_.port = node_ap.port;
     }
+    tor_free(conn->nickname);
     conn->nickname = tor_strdup(node_get_nickname(r));
     tor_free(conn->base_.address);
     conn->base_.address = tor_addr_to_str_dup(&node_ap.addr);
   } else {
+    tor_free(conn->nickname);
     conn->nickname = tor_malloc(HEX_DIGEST_LEN+2);
     conn->nickname[0] = '$';
     base16_encode(conn->nickname+1, HEX_DIGEST_LEN+1,
@@ -957,7 +989,37 @@ connection_or_mark_bad_for_new_circs(or_connection_t *or_conn)
  * too old for new circuits? */
 #define TIME_BEFORE_OR_CONN_IS_TOO_OLD (60*60*24*7)
 
-/** Given the head of the linked list for all the or_connections with a given
+/** Expire an or_connection if it is too old. Helper for
+ * connection_or_group_set_badness_ and fast path for
+ * channel_rsa_id_group_set_badness.
+ *
+ * Returns 1 if the connection was already expired, else 0.
+ */
+int
+connection_or_single_set_badness_(time_t now,
+                                  or_connection_t *or_conn,
+                                  int force)
+{
+  /* XXXX this function should also be about channels? */
+  if (or_conn->base_.marked_for_close ||
+      connection_or_is_bad_for_new_circs(or_conn))
+    return 1;
+
+  if (force ||
+      or_conn->base_.timestamp_created + TIME_BEFORE_OR_CONN_IS_TOO_OLD
+        < now) {
+    log_info(LD_OR,
+             "Marking OR conn to %s:%d as too old for new circuits "
+             "(fd "TOR_SOCKET_T_FORMAT", %d secs old).",
+             or_conn->base_.address, or_conn->base_.port, or_conn->base_.s,
+             (int)(now - or_conn->base_.timestamp_created));
+    connection_or_mark_bad_for_new_circs(or_conn);
+  }
+
+  return 0;
+}
+
+/** Given a list of all the or_connections with a given
  * identity, set elements of that list as is_bad_for_new_circs as
  * appropriate. Helper for connection_or_set_bad_connections().
  *
@@ -974,29 +1036,21 @@ connection_or_mark_bad_for_new_circs(or_connection_t *or_conn)
  * See channel_is_better() in channel.c for our idea of what makes one OR
  * connection better than another.
  */
-static void
-connection_or_group_set_badness(or_connection_t *head, int force)
+void
+connection_or_group_set_badness_(smartlist_t *group, int force)
 {
-  or_connection_t *or_conn = NULL, *best = NULL;
+  /* XXXX this function should be entirely about channels, not OR
+   * XXXX connections. */
+
+  or_connection_t *best = NULL;
   int n_old = 0, n_inprogress = 0, n_canonical = 0, n_other = 0;
   time_t now = time(NULL);
 
   /* Pass 1: expire everything that's old, and see what the status of
    * everything else is. */
-  for (or_conn = head; or_conn; or_conn = or_conn->next_with_same_id) {
-    if (or_conn->base_.marked_for_close ||
-        connection_or_is_bad_for_new_circs(or_conn))
+  SMARTLIST_FOREACH_BEGIN(group, or_connection_t *, or_conn) {
+    if (connection_or_single_set_badness_(now, or_conn, force))
       continue;
-    if (force ||
-        or_conn->base_.timestamp_created + TIME_BEFORE_OR_CONN_IS_TOO_OLD
-          < now) {
-      log_info(LD_OR,
-               "Marking OR conn to %s:%d as too old for new circuits "
-               "(fd "TOR_SOCKET_T_FORMAT", %d secs old).",
-               or_conn->base_.address, or_conn->base_.port, or_conn->base_.s,
-               (int)(now - or_conn->base_.timestamp_created));
-      connection_or_mark_bad_for_new_circs(or_conn);
-    }
 
     if (connection_or_is_bad_for_new_circs(or_conn)) {
       ++n_old;
@@ -1007,11 +1061,11 @@ connection_or_group_set_badness(or_connection_t *head, int force)
     } else {
       ++n_other;
     }
-  }
+  } SMARTLIST_FOREACH_END(or_conn);
 
   /* Pass 2: We know how about how good the best connection is.
    * expire everything that's worse, and find the very best if we can. */
-  for (or_conn = head; or_conn; or_conn = or_conn->next_with_same_id) {
+  SMARTLIST_FOREACH_BEGIN(group, or_connection_t *, or_conn) {
     if (or_conn->base_.marked_for_close ||
         connection_or_is_bad_for_new_circs(or_conn))
       continue; /* This one doesn't need to be marked bad. */
@@ -1032,13 +1086,11 @@ connection_or_group_set_badness(or_connection_t *head, int force)
     }
 
     if (!best ||
-        channel_is_better(now,
-                          TLS_CHAN_TO_BASE(or_conn->chan),
-                          TLS_CHAN_TO_BASE(best->chan),
-                          0)) {
+        channel_is_better(TLS_CHAN_TO_BASE(or_conn->chan),
+                          TLS_CHAN_TO_BASE(best->chan))) {
       best = or_conn;
     }
-  }
+  } SMARTLIST_FOREACH_END(or_conn);
 
   if (!best)
     return;
@@ -1057,17 +1109,15 @@ connection_or_group_set_badness(or_connection_t *head, int force)
    *   0.1.2.x dies out, the first case will go away, and the second one is
    *   "mostly harmless", so a fix can wait until somebody is bored.
    */
-  for (or_conn = head; or_conn; or_conn = or_conn->next_with_same_id) {
+  SMARTLIST_FOREACH_BEGIN(group, or_connection_t *, or_conn) {
     if (or_conn->base_.marked_for_close ||
         connection_or_is_bad_for_new_circs(or_conn) ||
         or_conn->base_.state != OR_CONN_STATE_OPEN)
       continue;
     if (or_conn != best &&
-        channel_is_better(now,
-                          TLS_CHAN_TO_BASE(best->chan),
-                          TLS_CHAN_TO_BASE(or_conn->chan), 1)) {
-      /* This isn't the best conn, _and_ the best conn is better than it,
-         even when we're being forgiving. */
+        channel_is_better(TLS_CHAN_TO_BASE(best->chan),
+                          TLS_CHAN_TO_BASE(or_conn->chan))) {
+      /* This isn't the best conn, _and_ the best conn is better than it */
       if (best->is_canonical) {
         log_info(LD_OR,
                  "Marking OR conn to %s:%d as unsuitable for new circuits: "
@@ -1091,24 +1141,217 @@ connection_or_group_set_badness(or_connection_t *head, int force)
         connection_or_mark_bad_for_new_circs(or_conn);
       }
     }
+  } SMARTLIST_FOREACH_END(or_conn);
+}
+
+/* Lifetime of a connection failure. After that, we'll retry. This is in
+ * seconds. */
+#define OR_CONNECT_FAILURE_LIFETIME 60
+/* The interval to use with when to clean up the failure cache. */
+#define OR_CONNECT_FAILURE_CLEANUP_INTERVAL 60
+
+/* When is the next time we have to cleanup the failure map. We keep this
+ * because we clean it opportunistically. */
+static time_t or_connect_failure_map_next_cleanup_ts = 0;
+
+/* OR connection failure entry data structure. It is kept in the connection
+ * failure map defined below and indexed by OR identity digest, address and
+ * port.
+ *
+ * We need to identify a connection failure with these three values because we
+ * want to avoid to wrongfully blacklist a relay if someone is trying to
+ * extend to a known identity digest but with the wrong IP/port. For instance,
+ * it can happen if a relay changed its port but the client still has an old
+ * descriptor with the old port. We want to stop connecting to that
+ * IP/port/identity all together, not only the relay identity. */
+typedef struct or_connect_failure_entry_t {
+  HT_ENTRY(or_connect_failure_entry_t) node;
+  /* Identity digest of the connection where it is connecting to. */
+  uint8_t identity_digest[DIGEST_LEN];
+  /* This is the connection address from the base connection_t. After the
+   * connection is checked for canonicity, the base address should represent
+   * what we know instead of where we are connecting to. This is what we need
+   * so we can correlate known relays within the consensus. */
+  tor_addr_t addr;
+  uint16_t port;
+  /* Last time we were unable to connect. */
+  time_t last_failed_connect_ts;
+} or_connect_failure_entry_t;
+
+/* Map where we keep connection failure entries. They are indexed by addr,
+ * port and identity digest. */
+static HT_HEAD(or_connect_failure_ht, or_connect_failure_entry_t)
+       or_connect_failures_map = HT_INITIALIZER();
+
+/* Helper: Hashtable equal function. Return 1 if equal else 0. */
+static int
+or_connect_failure_ht_eq(const or_connect_failure_entry_t *a,
+                         const or_connect_failure_entry_t *b)
+{
+  return fast_memeq(a->identity_digest, b->identity_digest, DIGEST_LEN) &&
+         tor_addr_eq(&a->addr, &b->addr) &&
+         a->port == b->port;
+}
+
+/* Helper: Return the hash for the hashtable of the given entry. For this
+ * table, it is a combination of address, port and identity digest. */
+static unsigned int
+or_connect_failure_ht_hash(const or_connect_failure_entry_t *entry)
+{
+  size_t offset = 0, addr_size;
+  const void *addr_ptr;
+  /* Largest size is IPv6 and IPv4 is smaller so it is fine. */
+  uint8_t data[16 + sizeof(uint16_t) + DIGEST_LEN];
+
+  /* Get the right address bytes depending on the family. */
+  switch (tor_addr_family(&entry->addr)) {
+  case AF_INET:
+    addr_size = 4;
+    addr_ptr = &entry->addr.addr.in_addr.s_addr;
+    break;
+  case AF_INET6:
+    addr_size = 16;
+    addr_ptr = &entry->addr.addr.in6_addr.s6_addr;
+    break;
+  default:
+    tor_assert_nonfatal_unreached();
+    return 0;
+  }
+
+  memcpy(data, addr_ptr, addr_size);
+  offset += addr_size;
+  memcpy(data + offset, entry->identity_digest, DIGEST_LEN);
+  offset += DIGEST_LEN;
+  set_uint16(data + offset, entry->port);
+  offset += sizeof(uint16_t);
+
+  return (unsigned int) siphash24g(data, offset);
+}
+
+HT_PROTOTYPE(or_connect_failure_ht, or_connect_failure_entry_t, node,
+             or_connect_failure_ht_hash, or_connect_failure_ht_eq)
+
+HT_GENERATE2(or_connect_failure_ht, or_connect_failure_entry_t, node,
+             or_connect_failure_ht_hash, or_connect_failure_ht_eq,
+             0.6, tor_reallocarray_, tor_free_)
+
+/* Initialize a given connect failure entry with the given identity_digest,
+ * addr and port. All field are optional except ocf. */
+static void
+or_connect_failure_init(const char *identity_digest, const tor_addr_t *addr,
+                        uint16_t port, or_connect_failure_entry_t *ocf)
+{
+  tor_assert(ocf);
+  if (identity_digest) {
+    memcpy(ocf->identity_digest, identity_digest,
+           sizeof(ocf->identity_digest));
+  }
+  if (addr) {
+    tor_addr_copy(&ocf->addr, addr);
   }
+  ocf->port = port;
 }
 
-/** Go through all the OR connections (or if <b>digest</b> is non-NULL, just
- * the OR connections with that digest), and set the is_bad_for_new_circs
- * flag based on the rules in connection_or_group_set_badness() (or just
- * always set it if <b>force</b> is true).
- */
-void
-connection_or_set_bad_connections(const char *digest, int force)
+/* Return a newly allocated connection failure entry. It is initialized with
+ * the given or_conn data. This can't fail. */
+static or_connect_failure_entry_t *
+or_connect_failure_new(const or_connection_t *or_conn)
 {
-  if (!orconn_identity_map)
-    return;
+  or_connect_failure_entry_t *ocf = tor_malloc_zero(sizeof(*ocf));
+  or_connect_failure_init(or_conn->identity_digest, &or_conn->real_addr,
+                          TO_CONN(or_conn)->port, ocf);
+  return ocf;
+}
+
+/* Return a connection failure entry matching the given or_conn. NULL is
+ * returned if not found. */
+static or_connect_failure_entry_t *
+or_connect_failure_find(const or_connection_t *or_conn)
+{
+  or_connect_failure_entry_t lookup;
+  tor_assert(or_conn);
+  or_connect_failure_init(or_conn->identity_digest, &TO_CONN(or_conn)->addr,
+                          TO_CONN(or_conn)->port, &lookup);
+  return HT_FIND(or_connect_failure_ht, &or_connect_failures_map, &lookup);
+}
+
+/* Note down in the connection failure cache that a failure occurred on the
+ * given or_conn. */
+STATIC void
+note_or_connect_failed(const or_connection_t *or_conn)
+{
+  or_connect_failure_entry_t *ocf = NULL;
+
+  tor_assert(or_conn);
+
+  ocf = or_connect_failure_find(or_conn);
+  if (ocf == NULL) {
+    ocf = or_connect_failure_new(or_conn);
+    HT_INSERT(or_connect_failure_ht, &or_connect_failures_map, ocf);
+  }
+  ocf->last_failed_connect_ts = approx_time();
+}
+
+/* Cleanup the connection failure cache and remove all entries below the
+ * given cutoff. */
+static void
+or_connect_failure_map_cleanup(time_t cutoff)
+{
+  or_connect_failure_entry_t **ptr, **next, *entry;
+
+  for (ptr = HT_START(or_connect_failure_ht, &or_connect_failures_map);
+       ptr != NULL; ptr = next) {
+    entry = *ptr;
+    if (entry->last_failed_connect_ts <= cutoff) {
+      next = HT_NEXT_RMV(or_connect_failure_ht, &or_connect_failures_map, ptr);
+      tor_free(entry);
+    } else {
+      next = HT_NEXT(or_connect_failure_ht, &or_connect_failures_map, ptr);
+    }
+  }
+}
 
-  DIGESTMAP_FOREACH(orconn_identity_map, identity, or_connection_t *, conn) {
-    if (!digest || tor_memeq(digest, conn->identity_digest, DIGEST_LEN))
-      connection_or_group_set_badness(conn, force);
-  } DIGESTMAP_FOREACH_END;
+/* Return true iff the given OR connection can connect to its destination that
+ * is the triplet identity_digest, address and port.
+ *
+ * The or_conn MUST have gone through connection_or_check_canonicity() so the
+ * base address is properly set to what we know or doesn't know. */
+STATIC int
+should_connect_to_relay(const or_connection_t *or_conn)
+{
+  time_t now, cutoff;
+  time_t connect_failed_since_ts = 0;
+  or_connect_failure_entry_t *ocf;
+
+  tor_assert(or_conn);
+
+  now = approx_time();
+  cutoff = now - OR_CONNECT_FAILURE_LIFETIME;
+
+  /* Opportunistically try to cleanup the failure cache. We do that at regular
+   * interval so it doesn't grow too big. */
+  if (or_connect_failure_map_next_cleanup_ts <= now) {
+    or_connect_failure_map_cleanup(cutoff);
+    or_connect_failure_map_next_cleanup_ts =
+      now + OR_CONNECT_FAILURE_CLEANUP_INTERVAL;
+  }
+
+  /* Look if we have failed previously to the same destination as this
+   * OR connection. */
+  ocf = or_connect_failure_find(or_conn);
+  if (ocf) {
+    connect_failed_since_ts = ocf->last_failed_connect_ts;
+  }
+  /* If we do have an unable to connect timestamp and it is below cutoff, we
+   * can connect. Or we have never failed before so let it connect. */
+  if (connect_failed_since_ts > cutoff) {
+    goto no_connect;
+  }
+
+  /* Ok we can connect! */
+  return 1;
+ no_connect:
+  return 0;
 }
 
 /** <b>conn</b> is in the 'connecting' state, and it failed to complete
@@ -1123,7 +1366,8 @@ connection_or_connect_failed(or_connection_t *conn,
 {
   control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED, reason);
   if (!authdir_mode_tests_reachability(get_options()))
-    control_event_bootstrap_problem(msg, reason, conn);
+    control_event_bootstrap_prob_or(msg, reason, conn);
+  note_or_connect_failed(conn);
 }
 
 /** <b>conn</b> got an error in connection_handle_read_impl() or
@@ -1174,7 +1418,9 @@ connection_or_notify_error(or_connection_t *conn,
 
 MOCK_IMPL(or_connection_t *,
 connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
-                        const char *id_digest, channel_tls_t *chan))
+                        const char *id_digest,
+                        const ed25519_public_key_t *ed_id,
+                        channel_tls_t *chan))
 {
   or_connection_t *conn;
   const or_options_t *options = get_options();
@@ -1194,6 +1440,11 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
     log_info(LD_PROTOCOL,"Client asked me to connect to myself. Refusing.");
     return NULL;
   }
+  if (server_mode(options) && router_ed25519_id_is_me(ed_id)) {
+    log_info(LD_PROTOCOL,"Client asked me to connect to myself by Ed25519 "
+             "identity. Refusing.");
+    return NULL;
+  }
 
   conn = or_connection_new(CONN_TYPE_OR, tor_addr_family(&addr));
 
@@ -1206,7 +1457,20 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
    */
   conn->chan = chan;
   chan->conn = conn;
-  connection_or_init_conn_from_address(conn, &addr, port, id_digest, 1);
+  connection_or_init_conn_from_address(conn, &addr, port, id_digest, ed_id, 1);
+
+  /* We have a proper OR connection setup, now check if we can connect to it
+   * that is we haven't had a failure earlier. This is to avoid to try to
+   * constantly connect to relays that we think are not reachable. */
+  if (!should_connect_to_relay(conn)) {
+    log_info(LD_GENERAL, "Can't connect to identity %s at %s:%u because we "
+                         "failed earlier. Refusing.",
+             hex_str(id_digest, DIGEST_LEN), fmt_addr(&TO_CONN(conn)->addr),
+             TO_CONN(conn)->port);
+    connection_free_(TO_CONN(conn));
+    return NULL;
+  }
+
   connection_or_change_state(conn, OR_CONN_STATE_CONNECTING);
   control_event_or_conn_status(conn, OR_CONN_EVENT_LAUNCHED, 0);
 
@@ -1239,7 +1503,7 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
                fmt_addrport(&TO_CONN(conn)->addr, TO_CONN(conn)->port),
                transport_name, transport_name);
 
-      control_event_bootstrap_problem(
+      control_event_bootstrap_prob_or(
                                 "Can't connect to bridge",
                                 END_OR_CONN_REASON_PT_MISSING,
                                 conn);
@@ -1250,7 +1514,7 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
                fmt_addrport(&TO_CONN(conn)->addr, TO_CONN(conn)->port));
     }
 
-    connection_free(TO_CONN(conn));
+    connection_free_(TO_CONN(conn));
     return NULL;
   }
 
@@ -1263,7 +1527,7 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
       connection_or_connect_failed(conn,
                                    errno_to_orconn_end_reason(socket_error),
                                    tor_socket_strerror(socket_error));
-      connection_free(TO_CONN(conn));
+      connection_free_(TO_CONN(conn));
       return NULL;
     case 0:
       connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT);
@@ -1373,7 +1637,6 @@ connection_tls_start_handshake,(or_connection_t *conn, int receiving))
   connection_start_reading(TO_CONN(conn));
   log_debug(LD_HANDSHAKE,"starting TLS handshake on fd "TOR_SOCKET_T_FORMAT,
             conn->base_.s);
-  note_crypto_pk_op(receiving ? TLS_HANDSHAKE_S : TLS_HANDSHAKE_C);
 
   if (connection_tls_continue_handshake(conn) < 0)
     return -1;
@@ -1553,7 +1816,10 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn,
   }
 
   if (identity_rcvd) {
-    crypto_pk_get_digest(identity_rcvd, digest_rcvd_out);
+    if (crypto_pk_get_digest(identity_rcvd, digest_rcvd_out) < 0) {
+      crypto_pk_free(identity_rcvd);
+      return -1;
+    }
   } else {
     memset(digest_rcvd_out, 0, DIGEST_LEN);
   }
@@ -1563,18 +1829,25 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn,
 
   crypto_pk_free(identity_rcvd);
 
-  if (started_here)
+  if (started_here) {
+    /* A TLS handshake can't teach us an Ed25519 ID, so we set it to NULL
+     * here. */
+    log_debug(LD_HANDSHAKE, "Calling client_learned_peer_id from "
+              "check_valid_tls_handshake");
     return connection_or_client_learned_peer_id(conn,
-                                     (const uint8_t*)digest_rcvd_out);
+                                        (const uint8_t*)digest_rcvd_out,
+                                        NULL);
+  }
 
   return 0;
 }
 
 /** Called when we (as a connection initiator) have definitively,
  * authenticatedly, learned that ID of the Tor instance on the other
- * side of <b>conn</b> is <b>peer_id</b>.  For v1 and v2 handshakes,
+ * side of <b>conn</b> is <b>rsa_peer_id</b> and optionally <b>ed_peer_id</b>.
+ * For v1 and v2 handshakes,
  * this is right after we get a certificate chain in a TLS handshake
- * or renegotiation.  For v3 handshakes, this is right after we get a
+ * or renegotiation.  For v3+ handshakes, this is right after we get a
  * certificate chain in a CERTS cell.
  *
  * If we did not know the ID before, record the one we got.
@@ -1595,12 +1868,31 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn,
  */
 int
 connection_or_client_learned_peer_id(or_connection_t *conn,
-                                     const uint8_t *peer_id)
+                                     const uint8_t *rsa_peer_id,
+                                     const ed25519_public_key_t *ed_peer_id)
 {
   const or_options_t *options = get_options();
-
-  if (tor_digest_is_zero(conn->identity_digest)) {
-    connection_or_set_identity_digest(conn, (const char*)peer_id);
+  channel_tls_t *chan_tls = conn->chan;
+  channel_t *chan = channel_tls_to_base(chan_tls);
+  int changed_identity = 0;
+  tor_assert(chan);
+
+  const int expected_rsa_key =
+    ! tor_digest_is_zero(conn->identity_digest);
+  const int expected_ed_key =
+    ! ed25519_public_key_is_zero(&chan->ed25519_identity);
+
+  log_info(LD_HANDSHAKE, "learned peer id for %p (%s): %s, %s",
+           conn,
+           safe_str_client(conn->base_.address),
+           hex_str((const char*)rsa_peer_id, DIGEST_LEN),
+           ed25519_fmt(ed_peer_id));
+
+  if (! expected_rsa_key && ! expected_ed_key) {
+    log_info(LD_HANDSHAKE, "(we had no ID in mind when we made this "
+             "connection.");
+    connection_or_set_identity_digest(conn,
+                                      (const char*)rsa_peer_id, ed_peer_id);
     tor_free(conn->nickname);
     conn->nickname = tor_malloc(HEX_DIGEST_LEN+2);
     conn->nickname[0] = '$';
@@ -1612,16 +1904,39 @@ connection_or_client_learned_peer_id(or_connection_t *conn,
     /* if it's a bridge and we didn't know its identity fingerprint, now
      * we do -- remember it for future attempts. */
     learned_router_identity(&conn->base_.addr, conn->base_.port,
-                            (const char*)peer_id);
+                            (const char*)rsa_peer_id, ed_peer_id);
+    changed_identity = 1;
   }
 
-  if (tor_memneq(peer_id, conn->identity_digest, DIGEST_LEN)) {
+  const int rsa_mismatch = expected_rsa_key &&
+    tor_memneq(rsa_peer_id, conn->identity_digest, DIGEST_LEN);
+  /* It only counts as an ed25519 mismatch if we wanted an ed25519 identity
+   * and didn't get it. It's okay if we get one that we didn't ask for. */
+  const int ed25519_mismatch =
+    expected_ed_key &&
+    (ed_peer_id == NULL ||
+     ! ed25519_pubkey_eq(&chan->ed25519_identity, ed_peer_id));
+
+  if (rsa_mismatch || ed25519_mismatch) {
     /* I was aiming for a particular digest. I didn't get it! */
-    char seen[HEX_DIGEST_LEN+1];
-    char expected[HEX_DIGEST_LEN+1];
-    base16_encode(seen, sizeof(seen), (const char*)peer_id, DIGEST_LEN);
-    base16_encode(expected, sizeof(expected), conn->identity_digest,
+    char seen_rsa[HEX_DIGEST_LEN+1];
+    char expected_rsa[HEX_DIGEST_LEN+1];
+    char seen_ed[ED25519_BASE64_LEN+1];
+    char expected_ed[ED25519_BASE64_LEN+1];
+    base16_encode(seen_rsa, sizeof(seen_rsa),
+                  (const char*)rsa_peer_id, DIGEST_LEN);
+    base16_encode(expected_rsa, sizeof(expected_rsa), conn->identity_digest,
                   DIGEST_LEN);
+    if (ed_peer_id) {
+      ed25519_public_to_base64(seen_ed, ed_peer_id);
+    } else {
+      strlcpy(seen_ed, "no ed25519 key", sizeof(seen_ed));
+    }
+    if (! ed25519_public_key_is_zero(&chan->ed25519_identity)) {
+      ed25519_public_to_base64(expected_ed, &chan->ed25519_identity);
+    } else {
+      strlcpy(expected_ed, "no ed25519 key", sizeof(expected_ed));
+    }
     const int using_hardcoded_fingerprints =
       !networkstatus_get_reasonably_live_consensus(time(NULL),
                                                    usable_consensus_flavor());
@@ -1633,7 +1948,7 @@ connection_or_client_learned_peer_id(or_connection_t *conn,
     int severity;
     const char *extra_log = "";
 
-    /* Relays, Single Onion Services, and Tor2web make direct connections using
+    /* Relays and Single Onion Services make direct connections using
      * untrusted authentication keys. */
     if (server_mode(options) || non_anonymous_mode) {
       severity = LOG_PROTOCOL_WARN;
@@ -1659,30 +1974,48 @@ connection_or_client_learned_peer_id(or_connection_t *conn,
     }
 
     log_fn(severity, LD_HANDSHAKE,
-           "Tried connecting to router at %s:%d, but identity key was not "
-           "as expected: wanted %s but got %s.%s",
-           conn->base_.address, conn->base_.port, expected, seen, extra_log);
-    entry_guard_register_connect_status(conn->identity_digest, 0, 1,
-                                        time(NULL));
+           "Tried connecting to router at %s:%d, but RSA + ed25519 identity "
+           "keys were not as expected: wanted %s + %s but got %s + %s.%s",
+           conn->base_.address, conn->base_.port,
+           expected_rsa, expected_ed, seen_rsa, seen_ed, extra_log);
+
+    /* Tell the new guard API about the channel failure */
+    entry_guard_chan_failed(TLS_CHAN_TO_BASE(conn->chan));
     control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED,
                                  END_OR_CONN_REASON_OR_IDENTITY);
     if (!authdir_mode_tests_reachability(options))
-      control_event_bootstrap_problem(
+      control_event_bootstrap_prob_or(
                                 "Unexpected identity in router certificate",
                                 END_OR_CONN_REASON_OR_IDENTITY,
                                 conn);
     return -1;
   }
+
+  if (!expected_ed_key && ed_peer_id) {
+    log_info(LD_HANDSHAKE, "(We had no Ed25519 ID in mind when we made this "
+             "connection.)");
+    connection_or_set_identity_digest(conn,
+                                      (const char*)rsa_peer_id, ed_peer_id);
+    changed_identity = 1;
+  }
+
+  if (changed_identity) {
+    /* If we learned an identity for this connection, then we might have
+     * just discovered it to be canonical. */
+    connection_or_check_canonicity(conn, conn->handshake_state->started_here);
+  }
+
   if (authdir_mode_tests_reachability(options)) {
     dirserv_orconn_tls_done(&conn->base_.addr, conn->base_.port,
-                            (const char*)peer_id);
+                            (const char*)rsa_peer_id, ed_peer_id);
   }
 
   return 0;
 }
 
-/** Return when a client used this, for connection.c, since client_used
- * is now one of the timestamps of channel_t */
+/** Return when we last used this channel for client activity (origin
+ * circuits). This is called from connection.c, since client_used is now one
+ * of the timestamps in channel_t */
 
 time_t
 connection_or_client_used(or_connection_t *conn)
@@ -1696,7 +2029,7 @@ connection_or_client_used(or_connection_t *conn)
 
 /** The v1/v2 TLS handshake is finished.
  *
- * Make sure we are happy with the person we just handshaked with.
+ * Make sure we are happy with the peer we just handshaked with.
  *
  * If they initiated the connection, make sure they're not already connected,
  * then initialize conn from the information in router.
@@ -1731,7 +2064,8 @@ connection_tls_finish_handshake(or_connection_t *conn)
   if (tor_tls_used_v1_handshake(conn->tls)) {
     conn->link_proto = 1;
     connection_or_init_conn_from_address(conn, &conn->base_.addr,
-                                         conn->base_.port, digest_rcvd, 0);
+                                         conn->base_.port, digest_rcvd,
+                                         NULL, 0);
     tor_tls_block_renegotiation(conn->tls);
     rep_hist_note_negotiated_link_proto(1, started_here);
     return connection_or_set_state_open(conn);
@@ -1740,7 +2074,8 @@ connection_tls_finish_handshake(or_connection_t *conn)
     if (connection_init_or_handshake_state(conn, started_here) < 0)
       return -1;
     connection_or_init_conn_from_address(conn, &conn->base_.addr,
-                                         conn->base_.port, digest_rcvd, 0);
+                                         conn->base_.port, digest_rcvd,
+                                         NULL, 0);
     return connection_or_send_versions(conn, 0);
   }
 }
@@ -1779,19 +2114,24 @@ connection_init_or_handshake_state(or_connection_t *conn, int started_here)
   s->started_here = started_here ? 1 : 0;
   s->digest_sent_data = 1;
   s->digest_received_data = 1;
+  if (! started_here && get_current_link_cert_cert()) {
+    s->own_link_cert = tor_cert_dup(get_current_link_cert_cert());
+  }
+  s->certs = or_handshake_certs_new();
+  s->certs->started_here = s->started_here;
   return 0;
 }
 
 /** Free all storage held by <b>state</b>. */
 void
-or_handshake_state_free(or_handshake_state_t *state)
+or_handshake_state_free_(or_handshake_state_t *state)
 {
   if (!state)
     return;
   crypto_digest_free(state->digest_sent);
   crypto_digest_free(state->digest_received);
-  tor_x509_cert_free(state->auth_cert);
-  tor_x509_cert_free(state->id_cert);
+  or_handshake_certs_free(state->certs);
+  tor_cert_free(state->own_link_cert);
   memwipe(state, 0xBE, sizeof(or_handshake_state_t));
   tor_free(state);
 }
@@ -1912,12 +2252,23 @@ connection_or_write_cell_to_buf(const cell_t *cell, or_connection_t *conn)
 
   cell_pack(&networkcell, cell, conn->wide_circ_ids);
 
-  connection_write_to_buf(networkcell.body, cell_network_size, TO_CONN(conn));
+  rep_hist_padding_count_write(PADDING_TYPE_TOTAL);
+  if (cell->command == CELL_PADDING)
+    rep_hist_padding_count_write(PADDING_TYPE_CELL);
+
+  connection_buf_add(networkcell.body, cell_network_size, TO_CONN(conn));
 
   /* Touch the channel's active timestamp if there is one */
-  if (conn->chan)
+  if (conn->chan) {
     channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
 
+    if (TLS_CHAN_TO_BASE(conn->chan)->currently_padding) {
+      rep_hist_padding_count_write(PADDING_TYPE_ENABLED_TOTAL);
+      if (cell->command == CELL_PADDING)
+        rep_hist_padding_count_write(PADDING_TYPE_ENABLED_CELL);
+    }
+  }
+
   if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3)
     or_handshake_state_record_cell(conn, conn->handshake_state, cell, 0);
 }
@@ -1935,8 +2286,8 @@ connection_or_write_var_cell_to_buf,(const var_cell_t *cell,
   tor_assert(cell);
   tor_assert(conn);
   n = var_cell_pack_header(cell, hdr, conn->wide_circ_ids);
-  connection_write_to_buf(hdr, n, TO_CONN(conn));
-  connection_write_to_buf((char*)cell->payload,
+  connection_buf_add(hdr, n, TO_CONN(conn));
+  connection_buf_add((char*)cell->payload,
                           cell->payload_len, TO_CONN(conn));
   if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3)
     or_handshake_state_record_var_cell(conn, conn->handshake_state, cell, 0);
@@ -2011,7 +2362,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn)
         channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
 
       circuit_build_times_network_is_live(get_circuit_build_times_mutable());
-      connection_fetch_from_buf(buf, cell_network_size, TO_CONN(conn));
+      connection_buf_get_bytes(buf, cell_network_size, TO_CONN(conn));
 
       /* retrieve cell info from buf (create the host-order struct from the
        * network-order string) */
@@ -2023,7 +2374,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn)
 }
 
 /** Array of recognized link protocol versions. */
-static const uint16_t or_protocol_versions[] = { 1, 2, 3, 4 };
+static const uint16_t or_protocol_versions[] = { 1, 2, 3, 4, 5 };
 /** Number of versions in <b>or_protocol_versions</b>. */
 static const int n_or_protocol_versions =
   (int)( sizeof(or_protocol_versions)/sizeof(uint16_t) );
@@ -2144,66 +2495,198 @@ connection_or_send_netinfo,(or_connection_t *conn))
   return 0;
 }
 
+/** Helper used to add an encoded certs to a cert cell */
+static void
+add_certs_cell_cert_helper(certs_cell_t *certs_cell,
+                           uint8_t cert_type,
+                           const uint8_t *cert_encoded,
+                           size_t cert_len)
+{
+  tor_assert(cert_len <= UINT16_MAX);
+  certs_cell_cert_t *ccc = certs_cell_cert_new();
+  ccc->cert_type = cert_type;
+  ccc->cert_len = cert_len;
+  certs_cell_cert_setlen_body(ccc, cert_len);
+  memcpy(certs_cell_cert_getarray_body(ccc), cert_encoded, cert_len);
+
+  certs_cell_add_certs(certs_cell, ccc);
+}
+
+/** Add an encoded X509 cert (stored as <b>cert_len</b> bytes at
+ * <b>cert_encoded</b>) to the trunnel certs_cell_t object that we are
+ * building in <b>certs_cell</b>.  Set its type field to <b>cert_type</b>.
+ * (If <b>cert</b> is NULL, take no action.) */
+static void
+add_x509_cert(certs_cell_t *certs_cell,
+              uint8_t cert_type,
+              const tor_x509_cert_t *cert)
+{
+  if (NULL == cert)
+    return;
+
+  const uint8_t *cert_encoded = NULL;
+  size_t cert_len;
+  tor_x509_cert_get_der(cert, &cert_encoded, &cert_len);
+
+  add_certs_cell_cert_helper(certs_cell, cert_type, cert_encoded, cert_len);
+}
+
+/** Add an Ed25519 cert from <b>cert</b> to the trunnel certs_cell_t object
+ * that we are building in <b>certs_cell</b>.  Set its type field to
+ * <b>cert_type</b>. (If <b>cert</b> is NULL, take no action.) */
+static void
+add_ed25519_cert(certs_cell_t *certs_cell,
+                 uint8_t cert_type,
+                 const tor_cert_t *cert)
+{
+  if (NULL == cert)
+    return;
+
+  add_certs_cell_cert_helper(certs_cell, cert_type,
+                             cert->encoded, cert->encoded_len);
+}
+
+#ifdef TOR_UNIT_TESTS
+int certs_cell_ed25519_disabled_for_testing = 0;
+#else
+#define certs_cell_ed25519_disabled_for_testing 0
+#endif
+
 /** Send a CERTS cell on the connection <b>conn</b>.  Return 0 on success, -1
  * on failure. */
 int
 connection_or_send_certs_cell(or_connection_t *conn)
 {
-  const tor_x509_cert_t *global_link_cert = NULL, *id_cert = NULL,
-    *using_link_cert = NULL;
+  const tor_x509_cert_t *global_link_cert = NULL, *id_cert = NULL;
   tor_x509_cert_t *own_link_cert = NULL;
-  const uint8_t *link_encoded = NULL, *id_encoded = NULL;
-  size_t link_len, id_len;
   var_cell_t *cell;
-  size_t cell_len;
-  ssize_t pos;
+
+  certs_cell_t *certs_cell = NULL;
 
   tor_assert(conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3);
 
   if (! conn->handshake_state)
     return -1;
+
   const int conn_in_server_mode = ! conn->handshake_state->started_here;
+
+  /* Get the encoded values of the X509 certificates */
   if (tor_tls_get_my_certs(conn_in_server_mode,
                            &global_link_cert, &id_cert) < 0)
     return -1;
+
+  if (conn_in_server_mode) {
+    own_link_cert = tor_tls_get_own_cert(conn->tls);
+  }
+  tor_assert(id_cert);
+
+  certs_cell = certs_cell_new();
+
+  /* Start adding certs.  First the link cert or auth1024 cert. */
   if (conn_in_server_mode) {
-    using_link_cert = own_link_cert = tor_tls_get_own_cert(conn->tls);
+    tor_assert_nonfatal(own_link_cert);
+    add_x509_cert(certs_cell,
+                  OR_CERT_TYPE_TLS_LINK, own_link_cert);
   } else {
-    using_link_cert = global_link_cert;
+    tor_assert(global_link_cert);
+    add_x509_cert(certs_cell,
+                  OR_CERT_TYPE_AUTH_1024, global_link_cert);
   }
-  tor_x509_cert_get_der(using_link_cert, &link_encoded, &link_len);
-  tor_x509_cert_get_der(id_cert, &id_encoded, &id_len);
 
-  cell_len = 1 /* 1 byte: num certs in cell */ +
-             2 * ( 1 + 2 ) /* For each cert: 1 byte for type, 2 for length */ +
-             link_len + id_len;
-  cell = var_cell_new(cell_len);
-  cell->command = CELL_CERTS;
-  cell->payload[0] = 2;
-  pos = 1;
+  /* Next the RSA->RSA ID cert */
+  add_x509_cert(certs_cell,
+                OR_CERT_TYPE_ID_1024, id_cert);
 
-  if (conn_in_server_mode)
-    cell->payload[pos] = OR_CERT_TYPE_TLS_LINK; /* Link cert  */
-  else
-    cell->payload[pos] = OR_CERT_TYPE_AUTH_1024; /* client authentication */
-  set_uint16(&cell->payload[pos+1], htons(link_len));
-  memcpy(&cell->payload[pos+3], link_encoded, link_len);
-  pos += 3 + link_len;
+  /* Next the Ed25519 certs */
+  add_ed25519_cert(certs_cell,
+                   CERTTYPE_ED_ID_SIGN,
+                   get_master_signing_key_cert());
+  if (conn_in_server_mode) {
+    tor_assert_nonfatal(conn->handshake_state->own_link_cert ||
+                        certs_cell_ed25519_disabled_for_testing);
+    add_ed25519_cert(certs_cell,
+                     CERTTYPE_ED_SIGN_LINK,
+                     conn->handshake_state->own_link_cert);
+  } else {
+    add_ed25519_cert(certs_cell,
+                     CERTTYPE_ED_SIGN_AUTH,
+                     get_current_auth_key_cert());
+  }
+
+  /* And finally the crosscert. */
+  {
+    const uint8_t *crosscert=NULL;
+    size_t crosscert_len;
+    get_master_rsa_crosscert(&crosscert, &crosscert_len);
+    if (crosscert) {
+      add_certs_cell_cert_helper(certs_cell,
+                               CERTTYPE_RSA1024_ID_EDID,
+                               crosscert, crosscert_len);
+    }
+  }
 
-  cell->payload[pos] = OR_CERT_TYPE_ID_1024; /* ID cert */
-  set_uint16(&cell->payload[pos+1], htons(id_len));
-  memcpy(&cell->payload[pos+3], id_encoded, id_len);
-  pos += 3 + id_len;
+  /* We've added all the certs; make the cell. */
+  certs_cell->n_certs = certs_cell_getlen_certs(certs_cell);
 
-  tor_assert(pos == (int)cell_len); /* Otherwise we just smashed the heap */
+  ssize_t alloc_len = certs_cell_encoded_len(certs_cell);
+  tor_assert(alloc_len >= 0 && alloc_len <= UINT16_MAX);
+  cell = var_cell_new(alloc_len);
+  cell->command = CELL_CERTS;
+  ssize_t enc_len = certs_cell_encode(cell->payload, alloc_len, certs_cell);
+  tor_assert(enc_len > 0 && enc_len <= alloc_len);
+  cell->payload_len = enc_len;
 
   connection_or_write_var_cell_to_buf(cell, conn);
   var_cell_free(cell);
+  certs_cell_free(certs_cell);
   tor_x509_cert_free(own_link_cert);
 
   return 0;
 }
 
+#ifdef TOR_UNIT_TESTS
+int testing__connection_or_pretend_TLSSECRET_is_supported = 0;
+#else
+#define testing__connection_or_pretend_TLSSECRET_is_supported 0
+#endif
+
+/** Return true iff <b>challenge_type</b> is an AUTHCHALLENGE type that
+ * we can send and receive. */
+int
+authchallenge_type_is_supported(uint16_t challenge_type)
+{
+  switch (challenge_type) {
+     case AUTHTYPE_RSA_SHA256_TLSSECRET:
+#ifdef HAVE_WORKING_TOR_TLS_GET_TLSSECRETS
+       return 1;
+#else
+       return testing__connection_or_pretend_TLSSECRET_is_supported;
+#endif
+     case AUTHTYPE_ED25519_SHA256_RFC5705:
+       return 1;
+     case AUTHTYPE_RSA_SHA256_RFC5705:
+     default:
+       return 0;
+  }
+}
+
+/** Return true iff <b>challenge_type_a</b> is one that we would rather
+ * use than <b>challenge_type_b</b>. */
+int
+authchallenge_type_is_better(uint16_t challenge_type_a,
+                             uint16_t challenge_type_b)
+{
+  /* Any supported type is better than an unsupported one;
+   * all unsupported types are equally bad. */
+  if (!authchallenge_type_is_supported(challenge_type_a))
+    return 0;
+  if (!authchallenge_type_is_supported(challenge_type_b))
+    return 1;
+  /* It happens that types are superior in numerically ascending order.
+   * If that ever changes, this must change too. */
+  return (challenge_type_a > challenge_type_b);
+}
+
 /** Send an AUTH_CHALLENGE cell on the connection <b>conn</b>. Return 0
  * on success, -1 on failure. */
 int
@@ -2218,17 +2701,28 @@ connection_or_send_auth_challenge_cell(or_connection_t *conn)
 
   auth_challenge_cell_t *ac = auth_challenge_cell_new();
 
+  tor_assert(sizeof(ac->challenge) == 32);
   crypto_rand((char*)ac->challenge, sizeof(ac->challenge));
 
-  auth_challenge_cell_add_methods(ac, AUTHTYPE_RSA_SHA256_TLSSECRET);
+  if (authchallenge_type_is_supported(AUTHTYPE_RSA_SHA256_TLSSECRET))
+    auth_challenge_cell_add_methods(ac, AUTHTYPE_RSA_SHA256_TLSSECRET);
+  /* Disabled, because everything that supports this method also supports
+   * the much-superior ED25519_SHA256_RFC5705 */
+  /* auth_challenge_cell_add_methods(ac, AUTHTYPE_RSA_SHA256_RFC5705); */
+  if (authchallenge_type_is_supported(AUTHTYPE_ED25519_SHA256_RFC5705))
+    auth_challenge_cell_add_methods(ac, AUTHTYPE_ED25519_SHA256_RFC5705);
   auth_challenge_cell_set_n_methods(ac,
                                     auth_challenge_cell_getlen_methods(ac));
 
   cell = var_cell_new(auth_challenge_cell_encoded_len(ac));
   ssize_t len = auth_challenge_cell_encode(cell->payload, cell->payload_len,
                                            ac);
-  if (len != cell->payload_len)
+  if (len != cell->payload_len) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Encoded auth challenge cell length not as expected");
     goto done;
+    /* LCOV_EXCL_STOP */
+  }
   cell->command = CELL_AUTH_CHALLENGE;
 
   connection_or_write_var_cell_to_buf(cell, conn);
@@ -2242,8 +2736,8 @@ connection_or_send_auth_challenge_cell(or_connection_t *conn)
 }
 
 /** Compute the main body of an AUTHENTICATE cell that a client can use
- * to authenticate itself on a v3 handshake for <b>conn</b>.  Write it to the
- * <b>outlen</b>-byte buffer at <b>out</b>.
+ * to authenticate itself on a v3 handshake for <b>conn</b>.  Return it
+ * in a var_cell_t.
  *
  * If <b>server</b> is true, only calculate the first
  * V3_AUTH_FIXED_PART_LEN bytes -- the part of the authenticator that's
@@ -2259,24 +2753,44 @@ connection_or_send_auth_challenge_cell(or_connection_t *conn)
  *
  * Return the length of the cell body on success, and -1 on failure.
  */
-int
+var_cell_t *
 connection_or_compute_authenticate_cell_body(or_connection_t *conn,
-                                             uint8_t *out, size_t outlen,
+                                             const int authtype,
                                              crypto_pk_t *signing_key,
-                                             int server)
+                                      const ed25519_keypair_t *ed_signing_key,
+                                      int server)
 {
   auth1_t *auth = NULL;
   auth_ctx_t *ctx = auth_ctx_new();
-  int result;
+  var_cell_t *result = NULL;
+  int old_tlssecrets_algorithm = 0;
+  const char *authtype_str = NULL;
 
-  /* assert state is reasonable XXXX */
+  int is_ed = 0;
 
-  ctx->is_ed = 0;
+  /* assert state is reasonable XXXX */
+  switch (authtype) {
+  case AUTHTYPE_RSA_SHA256_TLSSECRET:
+    authtype_str = "AUTH0001";
+    old_tlssecrets_algorithm = 1;
+    break;
+  case AUTHTYPE_RSA_SHA256_RFC5705:
+    authtype_str = "AUTH0002";
+    break;
+  case AUTHTYPE_ED25519_SHA256_RFC5705:
+    authtype_str = "AUTH0003";
+    is_ed = 1;
+    break;
+  default:
+    tor_assert(0);
+    break;
+  }
 
   auth = auth1_new();
+  ctx->is_ed = is_ed;
 
   /* Type: 8 bytes. */
-  memcpy(auth1_getarray_type(auth), "AUTH0001", 8);
+  memcpy(auth1_getarray_type(auth), authtype_str, 8);
 
   {
     const tor_x509_cert_t *id_cert=NULL;
@@ -2286,7 +2800,7 @@ connection_or_compute_authenticate_cell_body(or_connection_t *conn,
       goto err;
     my_digests = tor_x509_cert_get_id_digests(id_cert);
     their_digests =
-      tor_x509_cert_get_id_digests(conn->handshake_state->id_cert);
+      tor_x509_cert_get_id_digests(conn->handshake_state->certs->id_cert);
     tor_assert(my_digests);
     tor_assert(their_digests);
     my_id = (uint8_t*)my_digests->d[DIGEST_SHA256];
@@ -2302,6 +2816,22 @@ connection_or_compute_authenticate_cell_body(or_connection_t *conn,
     memcpy(auth->sid, server_id, 32);
   }
 
+  if (is_ed) {
+    const ed25519_public_key_t *my_ed_id, *their_ed_id;
+    if (!conn->handshake_state->certs->ed_id_sign) {
+      log_warn(LD_OR, "Ed authenticate without Ed ID cert from peer.");
+      goto err;
+    }
+    my_ed_id = get_master_identity_key();
+    their_ed_id = &conn->handshake_state->certs->ed_id_sign->signing_key;
+
+    const uint8_t *cid_ed = (server ? their_ed_id : my_ed_id)->pubkey;
+    const uint8_t *sid_ed = (server ? my_ed_id : their_ed_id)->pubkey;
+
+    memcpy(auth->u1_cid_ed, cid_ed, ED25519_PUBKEY_LEN);
+    memcpy(auth->u1_sid_ed, sid_ed, ED25519_PUBKEY_LEN);
+  }
+
   {
     crypto_digest_t *server_d, *client_d;
     if (server) {
@@ -2328,7 +2858,8 @@ connection_or_compute_authenticate_cell_body(or_connection_t *conn,
       cert = tor_tls_get_peer_cert(conn->tls);
     }
     if (!cert) {
-      log_warn(LD_OR, "Unable to find cert when making AUTH1 data.");
+      log_warn(LD_OR, "Unable to find cert when making %s data.",
+               authtype_str);
       goto err;
     }
 
@@ -2339,36 +2870,89 @@ connection_or_compute_authenticate_cell_body(or_connection_t *conn,
   }
 
   /* HMAC of clientrandom and serverrandom using master key : 32 octets */
-  tor_tls_get_tlssecrets(conn->tls, auth->tlssecrets);
+  if (old_tlssecrets_algorithm) {
+    if (tor_tls_get_tlssecrets(conn->tls, auth->tlssecrets) < 0) {
+      log_fn(LOG_PROTOCOL_WARN, LD_OR, "Somebody asked us for an older TLS "
+         "authentication method (AUTHTYPE_RSA_SHA256_TLSSECRET) "
+         "which we don't support.");
+    }
+  } else {
+    char label[128];
+    tor_snprintf(label, sizeof(label),
+                 "EXPORTER FOR TOR TLS CLIENT BINDING %s", authtype_str);
+    int r = tor_tls_export_key_material(conn->tls, auth->tlssecrets,
+                                        auth->cid, sizeof(auth->cid),
+                                        label);
+    if (r < 0) {
+      if (r != -2)
+        log_warn(LD_BUG, "TLS key export failed for unknown reason.");
+      // If r == -2, this was openssl bug 7712.
+      goto err;
+    }
+  }
 
   /* 8 octets were reserved for the current time, but we're trying to get out
    * of the habit of sending time around willynilly.  Fortunately, nothing
    * checks it.  That's followed by 16 bytes of nonce. */
   crypto_rand((char*)auth->rand, 24);
 
+  ssize_t maxlen = auth1_encoded_len(auth, ctx);
+  if (ed_signing_key && is_ed) {
+    maxlen += ED25519_SIG_LEN;
+  } else if (signing_key && !is_ed) {
+    maxlen += crypto_pk_keysize(signing_key);
+  }
+
+  const int AUTH_CELL_HEADER_LEN = 4; /* 2 bytes of type, 2 bytes of length */
+  result = var_cell_new(AUTH_CELL_HEADER_LEN + maxlen);
+  uint8_t *const out = result->payload + AUTH_CELL_HEADER_LEN;
+  const size_t outlen = maxlen;
   ssize_t len;
+
+  result->command = CELL_AUTHENTICATE;
+  set_uint16(result->payload, htons(authtype));
+
   if ((len = auth1_encode(out, outlen, auth, ctx)) < 0) {
-    log_warn(LD_OR, "Unable to encode signed part of AUTH1 data.");
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Unable to encode signed part of AUTH1 data.");
     goto err;
+    /* LCOV_EXCL_STOP */
   }
 
   if (server) {
     auth1_t *tmp = NULL;
     ssize_t len2 = auth1_parse(&tmp, out, len, ctx);
     if (!tmp) {
-      log_warn(LD_OR, "Unable to parse signed part of AUTH1 data.");
+      /* LCOV_EXCL_START */
+      log_warn(LD_BUG, "Unable to parse signed part of AUTH1 data that "
+               "we just encoded");
       goto err;
+      /* LCOV_EXCL_STOP */
     }
-    result = (int) (tmp->end_of_fixed_part - out);
+    result->payload_len = (tmp->end_of_signed - result->payload);
+
     auth1_free(tmp);
     if (len2 != len) {
-      log_warn(LD_OR, "Mismatched length when re-parsing AUTH1 data.");
+      /* LCOV_EXCL_START */
+      log_warn(LD_BUG, "Mismatched length when re-parsing AUTH1 data.");
       goto err;
+      /* LCOV_EXCL_STOP */
     }
     goto done;
   }
 
-  if (signing_key) {
+  if (ed_signing_key && is_ed) {
+    ed25519_signature_t sig;
+    if (ed25519_sign(&sig, out, len, ed_signing_key) < 0) {
+      /* LCOV_EXCL_START */
+      log_warn(LD_BUG, "Unable to sign ed25519 authentication data");
+      goto err;
+      /* LCOV_EXCL_STOP */
+    }
+    auth1_setlen_sig(auth, ED25519_SIG_LEN);
+    memcpy(auth1_getarray_sig(auth), sig.sig, ED25519_SIG_LEN);
+
+  } else if (signing_key && !is_ed) {
     auth1_setlen_sig(auth, crypto_pk_keysize(signing_key));
 
     char d[32];
@@ -2383,18 +2967,24 @@ connection_or_compute_authenticate_cell_body(or_connection_t *conn,
     }
 
     auth1_setlen_sig(auth, siglen);
+  }
 
-    len = auth1_encode(out, outlen, auth, ctx);
-    if (len < 0) {
-      log_warn(LD_OR, "Unable to encode signed AUTH1 data.");
-      goto err;
-    }
+  len = auth1_encode(out, outlen, auth, ctx);
+  if (len < 0) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Unable to encode signed AUTH1 data.");
+    goto err;
+    /* LCOV_EXCL_STOP */
   }
-  result = (int) len;
+  tor_assert(len + AUTH_CELL_HEADER_LEN <= result->payload_len);
+  result->payload_len = len + AUTH_CELL_HEADER_LEN;
+  set_uint16(result->payload+2, htons(len));
+
   goto done;
 
  err:
-  result = -1;
+  var_cell_free(result);
+  result = NULL;
  done:
   auth1_free(auth);
   auth_ctx_free(ctx);
@@ -2408,47 +2998,29 @@ connection_or_send_authenticate_cell,(or_connection_t *conn, int authtype))
 {
   var_cell_t *cell;
   crypto_pk_t *pk = tor_tls_get_my_client_auth_key();
-  int authlen;
-  size_t cell_maxlen;
   /* XXXX make sure we're actually supposed to send this! */
 
   if (!pk) {
     log_warn(LD_BUG, "Can't compute authenticate cell: no client auth key");
     return -1;
   }
-  if (authtype != AUTHTYPE_RSA_SHA256_TLSSECRET) {
+  if (! authchallenge_type_is_supported(authtype)) {
     log_warn(LD_BUG, "Tried to send authenticate cell with unknown "
              "authentication type %d", authtype);
     return -1;
   }
 
-  cell_maxlen = 4 + /* overhead */
-    V3_AUTH_BODY_LEN + /* Authentication body */
-    crypto_pk_keysize(pk) + /* Max signature length */
-    16 /* add a few extra bytes just in case. */;
-
-  cell = var_cell_new(cell_maxlen);
-  cell->command = CELL_AUTHENTICATE;
-  set_uint16(cell->payload, htons(AUTHTYPE_RSA_SHA256_TLSSECRET));
-  /* skip over length ; we don't know that yet. */
-
-  authlen = connection_or_compute_authenticate_cell_body(conn,
-                                                         cell->payload+4,
-                                                         cell_maxlen-4,
-                                                         pk,
-                                                         0 /* not server */);
-  if (authlen < 0) {
-    log_warn(LD_BUG, "Unable to compute authenticate cell!");
-    var_cell_free(cell);
+  cell = connection_or_compute_authenticate_cell_body(conn,
+                                                 authtype,
+                                                 pk,
+                                                 get_current_auth_keypair(),
+                                                 0 /* not server */);
+  if (! cell) {
+    log_fn(LOG_PROTOCOL_WARN, LD_NET, "Unable to compute authenticate cell!");
     return -1;
   }
-  tor_assert(authlen + 4 <= cell->payload_len);
-  set_uint16(cell->payload+2, htons(authlen));
-  cell->payload_len = authlen + 4;
-
   connection_or_write_var_cell_to_buf(cell, conn);
   var_cell_free(cell);
 
   return 0;
 }
-
diff --git a/src/or/connection_or.h b/src/core/or/connection_or.h
index 2e8c6066cc..817bcdd317 100644
--- a/src/or/connection_or.h
+++ b/src/core/or/connection_or.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,14 +12,45 @@
 #ifndef TOR_CONNECTION_OR_H
 #define TOR_CONNECTION_OR_H
 
-void connection_or_remove_from_identity_map(or_connection_t *conn);
+struct ed25519_public_key_t;
+struct ed25519_keypair_t;
+
+or_connection_t *TO_OR_CONN(connection_t *);
+
+#define OR_CONN_STATE_MIN_ 1
+/** State for a connection to an OR: waiting for connect() to finish. */
+#define OR_CONN_STATE_CONNECTING 1
+/** State for a connection to an OR: waiting for proxy handshake to complete */
+#define OR_CONN_STATE_PROXY_HANDSHAKING 2
+/** State for an OR connection client: SSL is handshaking, not done
+ * yet. */
+#define OR_CONN_STATE_TLS_HANDSHAKING 3
+/** State for a connection to an OR: We're doing a second SSL handshake for
+ * renegotiation purposes. (V2 handshake only.) */
+#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4
+/** State for a connection at an OR: We're waiting for the client to
+ * renegotiate (to indicate a v2 handshake) or send a versions cell (to
+ * indicate a v3 handshake) */
+#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5
+/** State for an OR connection: We're done with our SSL handshake, we've done
+ * renegotiation, but we haven't yet negotiated link protocol versions and
+ * sent a netinfo cell. */
+#define OR_CONN_STATE_OR_HANDSHAKING_V2 6
+/** State for an OR connection: We're done with our SSL handshake, but we
+ * haven't yet negotiated link protocol versions, done a V3 handshake, and
+ * sent a netinfo cell. */
+#define OR_CONN_STATE_OR_HANDSHAKING_V3 7
+/** State for an OR connection: Ready to send/receive cells. */
+#define OR_CONN_STATE_OPEN 8
+#define OR_CONN_STATE_MAX_ 8
+
+void connection_or_clear_identity(or_connection_t *conn);
 void connection_or_clear_identity_map(void);
 void clear_broken_connection_map(int disable);
 or_connection_t *connection_or_get_for_extend(const char *digest,
                                               const tor_addr_t *target_addr,
                                               const char **msg_out,
                                               int *launch_out);
-void connection_or_set_bad_connections(const char *digest, int force);
 
 void connection_or_block_renegotiation(or_connection_t *conn);
 int connection_or_reached_eof(or_connection_t *conn);
@@ -40,7 +71,9 @@ void connection_or_notify_error(or_connection_t *conn,
 MOCK_DECL(or_connection_t *,
           connection_or_connect,
           (const tor_addr_t *addr, uint16_t port,
-           const char *id_digest, channel_tls_t *chan));
+           const char *id_digest,
+           const struct ed25519_public_key_t *ed_id,
+           channel_tls_t *chan));
 
 void connection_or_close_normally(or_connection_t *orconn, int flush);
 MOCK_DECL(void,connection_or_close_for_error,
@@ -57,15 +90,19 @@ void connection_or_set_canonical(or_connection_t *or_conn,
 int connection_init_or_handshake_state(or_connection_t *conn,
                                        int started_here);
 void connection_or_init_conn_from_address(or_connection_t *conn,
-                                          const tor_addr_t *addr,
-                                          uint16_t port,
-                                          const char *id_digest,
-                                          int started_here);
+                                    const tor_addr_t *addr,
+                                    uint16_t port,
+                                    const char *rsa_id_digest,
+                                    const struct ed25519_public_key_t *ed_id,
+                                    int started_here);
 int connection_or_client_learned_peer_id(or_connection_t *conn,
-                                         const uint8_t *peer_id);
+                              const uint8_t *rsa_peer_id,
+                              const struct ed25519_public_key_t *ed_peer_id);
 time_t connection_or_client_used(or_connection_t *conn);
 MOCK_DECL(int, connection_or_get_num_circuits, (or_connection_t *conn));
-void or_handshake_state_free(or_handshake_state_t *state);
+void or_handshake_state_free_(or_handshake_state_t *state);
+#define or_handshake_state_free(state) \
+  FREE_AND_NULL(or_handshake_state_t, or_handshake_state_free_, (state))
 void or_handshake_state_record_cell(or_connection_t *conn,
                                     or_handshake_state_t *state,
                                     const cell_t *cell,
@@ -84,10 +121,15 @@ int connection_or_send_versions(or_connection_t *conn, int v3_plus);
 MOCK_DECL(int,connection_or_send_netinfo,(or_connection_t *conn));
 int connection_or_send_certs_cell(or_connection_t *conn);
 int connection_or_send_auth_challenge_cell(or_connection_t *conn);
-int connection_or_compute_authenticate_cell_body(or_connection_t *conn,
-                                                 uint8_t *out, size_t outlen,
-                                                 crypto_pk_t *signing_key,
-                                                 int server);
+int authchallenge_type_is_supported(uint16_t challenge_type);
+int authchallenge_type_is_better(uint16_t challenge_type_a,
+                                 uint16_t challenge_type_b);
+var_cell_t *connection_or_compute_authenticate_cell_body(
+                              or_connection_t *conn,
+                              const int authtype,
+                              crypto_pk_t *signing_key,
+                              const struct ed25519_keypair_t *ed_signing_key,
+                              int server);
 MOCK_DECL(int,connection_or_send_authenticate_cell,
           (or_connection_t *conn, int type));
 
@@ -98,10 +140,27 @@ int var_cell_pack_header(const var_cell_t *cell, char *hdr_out,
                          int wide_circ_ids);
 var_cell_t *var_cell_new(uint16_t payload_len);
 var_cell_t *var_cell_copy(const var_cell_t *src);
-void var_cell_free(var_cell_t *cell);
+void var_cell_free_(var_cell_t *cell);
+#define var_cell_free(cell) FREE_AND_NULL(var_cell_t, var_cell_free_, (cell))
 
 /* DOCDOC */
 #define MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS 4
+#define MIN_LINK_PROTO_FOR_CHANNEL_PADDING 5
+#define MAX_LINK_PROTO MIN_LINK_PROTO_FOR_CHANNEL_PADDING
+
+int connection_or_single_set_badness_(time_t now,
+                                      or_connection_t *or_conn,
+                                      int force);
+void connection_or_group_set_badness_(smartlist_t *group, int force);
+
+#ifdef CONNECTION_OR_PRIVATE
+STATIC int should_connect_to_relay(const or_connection_t *or_conn);
+STATIC void note_or_connect_failed(const or_connection_t *or_conn);
+#endif
 
+#ifdef TOR_UNIT_TESTS
+extern int certs_cell_ed25519_disabled_for_testing;
+extern int testing__connection_or_pretend_TLSSECRET_is_supported;
 #endif
 
+#endif /* !defined(TOR_CONNECTION_OR_H) */
diff --git a/src/core/or/connection_st.h b/src/core/or/connection_st.h
new file mode 100644
index 0000000000..d1430eda14
--- /dev/null
+++ b/src/core/or/connection_st.h
@@ -0,0 +1,149 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef CONNECTION_ST_H
+#define CONNECTION_ST_H
+
+struct buf_t;
+
+/* Values for connection_t.magic: used to make sure that downcasts (casts from
+* connection_t to foo_connection_t) are safe. */
+#define BASE_CONNECTION_MAGIC 0x7C3C304Eu
+#define OR_CONNECTION_MAGIC 0x7D31FF03u
+#define EDGE_CONNECTION_MAGIC 0xF0374013u
+#define ENTRY_CONNECTION_MAGIC 0xbb4a5703
+#define DIR_CONNECTION_MAGIC 0x9988ffeeu
+#define CONTROL_CONNECTION_MAGIC 0x8abc765du
+#define LISTENER_CONNECTION_MAGIC 0x1a1ac741u
+
+/** Description of a connection to another host or process, and associated
+ * data.
+ *
+ * A connection is named based on what it's connected to -- an "OR
+ * connection" has a Tor node on the other end, an "exit
+ * connection" has a website or other server on the other end, and an
+ * "AP connection" has an application proxy (and thus a user) on the
+ * other end.
+ *
+ * Every connection has a type and a state.  Connections never change
+ * their type, but can go through many state changes in their lifetime.
+ *
+ * Every connection has two associated input and output buffers.
+ * Listeners don't use them.  For non-listener connections, incoming
+ * data is appended to conn->inbuf, and outgoing data is taken from
+ * conn->outbuf.  Connections differ primarily in the functions called
+ * to fill and drain these buffers.
+ */
+struct connection_t {
+  uint32_t magic; /**< For memory debugging: must equal one of
+                   * *_CONNECTION_MAGIC. */
+
+  uint8_t state; /**< Current state of this connection. */
+  unsigned int type:5; /**< What kind of connection is this? */
+  unsigned int purpose:5; /**< Only used for DIR and EXIT types currently. */
+
+  /* The next fields are all one-bit booleans. Some are only applicable to
+   * connection subtypes, but we hold them here anyway, to save space.
+   */
+  unsigned int read_blocked_on_bw:1; /**< Boolean: should we start reading
+                            * again once the bandwidth throttler allows it? */
+  unsigned int write_blocked_on_bw:1; /**< Boolean: should we start writing
+                             * again once the bandwidth throttler allows
+                             * writes? */
+  unsigned int hold_open_until_flushed:1; /**< Despite this connection's being
+                                      * marked for close, do we flush it
+                                      * before closing it? */
+  unsigned int inbuf_reached_eof:1; /**< Boolean: did read() return 0 on this
+                                     * conn? */
+  /** Set to 1 when we're inside connection_flushed_some to keep us from
+   * calling connection_handle_write() recursively. */
+  unsigned int in_flushed_some:1;
+  /** True if connection_handle_write is currently running on this connection.
+   */
+  unsigned int in_connection_handle_write:1;
+
+  /* For linked connections:
+   */
+  unsigned int linked:1; /**< True if there is, or has been, a linked_conn. */
+  /** True iff we'd like to be notified about read events from the
+   * linked conn. */
+  unsigned int reading_from_linked_conn:1;
+  /** True iff we're willing to write to the linked conn. */
+  unsigned int writing_to_linked_conn:1;
+  /** True iff we're currently able to read on the linked conn, and our
+   * read_event should be made active with libevent. */
+  unsigned int active_on_link:1;
+  /** True iff we've called connection_close_immediate() on this linked
+   * connection. */
+  unsigned int linked_conn_is_closed:1;
+
+  /** CONNECT/SOCKS proxy client handshake state (for outgoing connections). */
+  unsigned int proxy_state:4;
+
+  /** Our socket; set to TOR_INVALID_SOCKET if this connection is closed,
+   * or has no socket. */
+  tor_socket_t s;
+  int conn_array_index; /**< Index into the global connection array. */
+
+  struct event *read_event; /**< Libevent event structure. */
+  struct event *write_event; /**< Libevent event structure. */
+  struct buf_t *inbuf; /**< Buffer holding data read over this connection. */
+  struct buf_t *outbuf; /**< Buffer holding data to write over this
+                         * connection. */
+  size_t outbuf_flushlen; /**< How much data should we try to flush from the
+                           * outbuf? */
+  time_t timestamp_last_read_allowed; /**< When was the last time libevent said
+                                       * we could read? */
+  time_t timestamp_last_write_allowed; /**< When was the last time libevent
+                                        * said we could write? */
+
+  time_t timestamp_created; /**< When was this connection_t created? */
+
+  int socket_family; /**< Address family of this connection's socket.  Usually
+                      * AF_INET, but it can also be AF_UNIX, or AF_INET6 */
+  tor_addr_t addr; /**< IP that socket "s" is directly connected to;
+                    * may be the IP address for a proxy or pluggable transport,
+                    * see "address" for the address of the final destination.
+                    */
+  uint16_t port; /**< If non-zero, port that socket "s" is directly connected
+                  * to; may be the port for a proxy or pluggable transport,
+                  * see "address" for the port at the final destination. */
+  uint16_t marked_for_close; /**< Should we close this conn on the next
+                              * iteration of the main loop? (If true, holds
+                              * the line number where this connection was
+                              * marked.) */
+  const char *marked_for_close_file; /**< For debugging: in which file were
+                                      * we marked for close? */
+  char *address; /**< FQDN (or IP) and port of the final destination for this
+                  * connection; this is always the remote address, it is
+                  * passed to a proxy or pluggable transport if one in use.
+                  * See "addr" and "port" for the address that socket "s" is
+                  * directly connected to.
+                  * strdup into this, because free_connection() frees it. */
+  /** Another connection that's connected to this one in lieu of a socket. */
+  struct connection_t *linked_conn;
+
+  /** Unique identifier for this connection on this Tor instance. */
+  uint64_t global_identifier;
+
+  /** Bytes read since last call to control_event_conn_bandwidth_used().
+   * Only used if we're configured to emit CONN_BW events. */
+  uint32_t n_read_conn_bw;
+
+  /** Bytes written since last call to control_event_conn_bandwidth_used().
+   * Only used if we're configured to emit CONN_BW events. */
+  uint32_t n_written_conn_bw;
+};
+
+/** True iff <b>x</b> is an edge connection. */
+#define CONN_IS_EDGE(x) \
+  ((x)->type == CONN_TYPE_EXIT || (x)->type == CONN_TYPE_AP)
+
+/** True iff the purpose of <b>conn</b> means that it's a server-side
+ * directory connection. */
+#define DIR_CONN_IS_SERVER(conn) ((conn)->purpose == DIR_PURPOSE_SERVER)
+
+#endif
diff --git a/src/core/or/cpath_build_state_st.h b/src/core/or/cpath_build_state_st.h
new file mode 100644
index 0000000000..dbe596d851
--- /dev/null
+++ b/src/core/or/cpath_build_state_st.h
@@ -0,0 +1,38 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef CIRCUIT_BUILD_STATE_ST_ST_H
+#define CIRCUIT_BUILD_STATE_ST_ST_H
+
+/** Information used to build a circuit. */
+struct cpath_build_state_t {
+  /** Intended length of the final circuit. */
+  int desired_path_len;
+  /** How to extend to the planned exit node. */
+  extend_info_t *chosen_exit;
+  /** Whether every node in the circ must have adequate uptime. */
+  unsigned int need_uptime : 1;
+  /** Whether every node in the circ must have adequate capacity. */
+  unsigned int need_capacity : 1;
+  /** Whether the last hop was picked with exiting in mind. */
+  unsigned int is_internal : 1;
+  /** Did we pick this as a one-hop tunnel (not safe for other streams)?
+   * These are for encrypted dir conns that exit to this router, not
+   * for arbitrary exits from the circuit. */
+  unsigned int onehop_tunnel : 1;
+  /** The crypt_path_t to append after rendezvous: used for rendezvous. */
+  crypt_path_t *pending_final_cpath;
+  /** A ref-counted reference to the crypt_path_t to append after
+   * rendezvous; used on the service side. */
+  crypt_path_reference_t *service_pending_final_cpath_ref;
+  /** How many times has building a circuit for this task failed? */
+  int failure_count;
+  /** At what time should we give up on this task? */
+  time_t expiry_time;
+};
+
+#endif
+
diff --git a/src/core/or/crypt_path_reference_st.h b/src/core/or/crypt_path_reference_st.h
new file mode 100644
index 0000000000..3d79f26c1c
--- /dev/null
+++ b/src/core/or/crypt_path_reference_st.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef CRYPT_PATH_REFERENCE_ST_H
+#define CRYPT_PATH_REFERENCE_ST_H
+
+/** A reference-counted pointer to a crypt_path_t, used only to share
+ * the final rendezvous cpath to be used on a service-side rendezvous
+ * circuit among multiple circuits built in parallel to the same
+ * destination rendezvous point. */
+struct crypt_path_reference_t {
+  /** The reference count. */
+  unsigned int refcount;
+  /** The pointer.  Set to NULL when the crypt_path_t is put into use
+   * on an opened rendezvous circuit. */
+  crypt_path_t *cpath;
+};
+
+#endif
+
diff --git a/src/core/or/crypt_path_st.h b/src/core/or/crypt_path_st.h
new file mode 100644
index 0000000000..429480f8ab
--- /dev/null
+++ b/src/core/or/crypt_path_st.h
@@ -0,0 +1,70 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef CRYPT_PATH_ST_H
+#define CRYPT_PATH_ST_H
+
+#include "core/or/relay_crypto_st.h"
+struct crypto_dh_t;
+
+#define CRYPT_PATH_MAGIC 0x70127012u
+
+struct fast_handshake_state_t;
+struct ntor_handshake_state_t;
+struct crypto_dh_t;
+struct onion_handshake_state_t {
+  uint16_t tag;
+  union {
+    struct fast_handshake_state_t *fast;
+    struct crypto_dh_t *tap;
+    struct ntor_handshake_state_t *ntor;
+  } u;
+};
+
+/** Holds accounting information for a single step in the layered encryption
+ * performed by a circuit.  Used only at the client edge of a circuit. */
+struct crypt_path_t {
+  uint32_t magic;
+
+  /** Cryptographic state used for encrypting and authenticating relay
+   * cells to and from this hop. */
+  relay_crypto_t crypto;
+
+  /** Current state of the handshake as performed with the OR at this
+   * step. */
+  onion_handshake_state_t handshake_state;
+  /** Diffie-hellman handshake state for performing an introduction
+   * operations */
+  struct crypto_dh_t *rend_dh_handshake_state;
+
+  /** Negotiated key material shared with the OR at this step. */
+  char rend_circ_nonce[DIGEST_LEN];/* KH in tor-spec.txt */
+
+  /** Information to extend to the OR at this step. */
+  extend_info_t *extend_info;
+
+  /** Is the circuit built to this step?  Must be one of:
+   *    - CPATH_STATE_CLOSED (The circuit has not been extended to this step)
+   *    - CPATH_STATE_AWAITING_KEYS (We have sent an EXTEND/CREATE to this step
+   *      and not received an EXTENDED/CREATED)
+   *    - CPATH_STATE_OPEN (The circuit has been extended to this step) */
+  uint8_t state;
+#define CPATH_STATE_CLOSED 0
+#define CPATH_STATE_AWAITING_KEYS 1
+#define CPATH_STATE_OPEN 2
+  struct crypt_path_t *next; /**< Link to next crypt_path_t in the circuit.
+                              * (The list is circular, so the last node
+                              * links to the first.) */
+  struct crypt_path_t *prev; /**< Link to previous crypt_path_t in the
+                              * circuit. */
+
+  int package_window; /**< How many cells are we allowed to originate ending
+                       * at this step? */
+  int deliver_window; /**< How many cells are we willing to deliver originating
+                       * at this step? */
+};
+
+#endif
diff --git a/src/core/or/destroy_cell_queue_st.h b/src/core/or/destroy_cell_queue_st.h
new file mode 100644
index 0000000000..56630670ba
--- /dev/null
+++ b/src/core/or/destroy_cell_queue_st.h
@@ -0,0 +1,27 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef DESTROY_CELL_QUEUE_ST_H
+#define DESTROY_CELL_QUEUE_ST_H
+
+/** A single queued destroy cell. */
+struct destroy_cell_t {
+  TOR_SIMPLEQ_ENTRY(destroy_cell_t) next;
+  circid_t circid;
+  uint32_t inserted_timestamp; /**< Time (in timestamp units) when this cell
+                                * was inserted */
+  uint8_t reason;
+};
+
+/** A queue of destroy cells on a channel. */
+struct destroy_cell_queue_t {
+  /** Linked list of packed_cell_t */
+  TOR_SIMPLEQ_HEAD(dcell_simpleq, destroy_cell_t) head;
+  int n; /**< The number of cells in the queue. */
+};
+
+#endif
+
diff --git a/src/or/dos.c b/src/core/or/dos.c
index 4d1797eece..5f9bbf90ab 100644
--- a/src/or/dos.c
+++ b/src/core/or/dos.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018, The Tor Project, Inc. */
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /*
@@ -8,16 +8,22 @@
 
 #define DOS_PRIVATE
 
-#include "or.h"
-#include "channel.h"
-#include "config.h"
-#include "geoip.h"
-#include "main.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "router.h"
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/channel.h"
+#include "core/or/connection_or.h"
+#include "core/or/relay.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/relay/routermode.h"
+#include "feature/stats/geoip_stats.h"
+#include "lib/crypt_ops/crypto_rand.h"
 
-#include "dos.h"
+#include "core/or/dos.h"
+
+#include "core/or/or_connection_st.h"
 
 /*
  * Circuit creation denial of service mitigation.
@@ -622,10 +628,12 @@ dos_log_heartbeat(void)
   char *conn_msg = NULL;
   char *cc_msg = NULL;
   char *single_hop_client_msg = NULL;
+  char *circ_stats_msg = NULL;
 
-  if (!dos_is_enabled()) {
-    goto end;
-  }
+  /* Stats number coming from relay.c append_cell_to_circuit_queue(). */
+  tor_asprintf(&circ_stats_msg,
+               " %" PRIu64 " circuits killed with too many cells.",
+               stats_n_circ_max_cell_reached);
 
   if (dos_cc_enabled) {
     tor_asprintf(&cc_msg,
@@ -647,7 +655,8 @@ dos_log_heartbeat(void)
   }
 
   log_notice(LD_HEARTBEAT,
-             "DoS mitigation since startup:%s%s%s",
+             "DoS mitigation since startup:%s%s%s%s",
+             circ_stats_msg,
              (cc_msg != NULL) ? cc_msg : " [cc not enabled]",
              (conn_msg != NULL) ? conn_msg : " [conn not enabled]",
              (single_hop_client_msg != NULL) ? single_hop_client_msg : "");
@@ -655,8 +664,7 @@ dos_log_heartbeat(void)
   tor_free(conn_msg);
   tor_free(cc_msg);
   tor_free(single_hop_client_msg);
-
- end:
+  tor_free(circ_stats_msg);
   return;
 }
 
@@ -791,4 +799,3 @@ dos_init(void)
   /* To initialize, we only need to get the parameters. */
   set_dos_parameters(NULL);
 }
-
diff --git a/src/or/dos.h b/src/core/or/dos.h
index 5d35a2b12e..95448d0530 100644
--- a/src/or/dos.h
+++ b/src/core/or/dos.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018, The Tor Project, Inc. */
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /*
diff --git a/src/core/or/edge_connection_st.h b/src/core/or/edge_connection_st.h
new file mode 100644
index 0000000000..1665b8589f
--- /dev/null
+++ b/src/core/or/edge_connection_st.h
@@ -0,0 +1,77 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef EDGE_CONNECTION_ST_H
+#define EDGE_CONNECTION_ST_H
+
+#include "core/or/or.h"
+
+#include "core/or/connection_st.h"
+
+/** Subtype of connection_t for an "edge connection" -- that is, an entry (ap)
+ * connection, or an exit. */
+struct edge_connection_t {
+  connection_t base_;
+
+  struct edge_connection_t *next_stream; /**< Points to the next stream at this
+                                          * edge, if any */
+  int package_window; /**< How many more relay cells can I send into the
+                       * circuit? */
+  int deliver_window; /**< How many more relay cells can end at me? */
+
+  struct circuit_t *on_circuit; /**< The circuit (if any) that this edge
+                                 * connection is using. */
+
+  /** A pointer to which node in the circ this conn exits at.  Set for AP
+   * connections and for hidden service exit connections. */
+  struct crypt_path_t *cpath_layer;
+  /** What rendezvous service are we querying for (if an AP) or providing (if
+   * an exit)? */
+  rend_data_t *rend_data;
+
+  /* Hidden service connection identifier for edge connections. Used by the HS
+   * client-side code to identify client SOCKS connections and by the
+   * service-side code to match HS circuits with their streams. */
+  struct hs_ident_edge_conn_t *hs_ident;
+
+  uint32_t address_ttl; /**< TTL for address-to-addr mapping on exit
+                         * connection.  Exit connections only. */
+  uint32_t begincell_flags; /** Flags sent or received in the BEGIN cell
+                             * for this connection */
+
+  streamid_t stream_id; /**< The stream ID used for this edge connection on its
+                         * circuit */
+
+  /** The reason why this connection is closing; passed to the controller. */
+  uint16_t end_reason;
+
+  /** Bytes read since last call to control_event_stream_bandwidth_used() */
+  uint32_t n_read;
+
+  /** Bytes written since last call to control_event_stream_bandwidth_used() */
+  uint32_t n_written;
+
+  /** True iff this connection is for a DNS request only. */
+  unsigned int is_dns_request:1;
+  /** True iff this connection is for a PTR DNS request. (exit only) */
+  unsigned int is_reverse_dns_lookup:1;
+
+  unsigned int edge_has_sent_end:1; /**< For debugging; only used on edge
+                         * connections.  Set once we've set the stream end,
+                         * and check in connection_about_to_close_connection().
+                         */
+  /** True iff we've blocked reading until the circuit has fewer queued
+   * cells. */
+  unsigned int edge_blocked_on_circ:1;
+
+  /** Unique ID for directory requests; this used to be in connection_t, but
+   * that's going away and being used on channels instead.  We still tag
+   * edge connections with dirreq_id from circuits, so it's copied here. */
+  uint64_t dirreq_id;
+};
+
+#endif
+
diff --git a/src/core/or/entry_connection_st.h b/src/core/or/entry_connection_st.h
new file mode 100644
index 0000000000..45621fadbf
--- /dev/null
+++ b/src/core/or/entry_connection_st.h
@@ -0,0 +1,100 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef ENTRY_CONNECTION_ST_H
+#define ENTRY_CONNECTION_ST_H
+
+#include "core/or/edge_connection_st.h"
+
+/** Subtype of edge_connection_t for an "entry connection" -- that is, a SOCKS
+ * connection, a DNS request, a TransPort connection or a NATD connection */
+struct entry_connection_t {
+  struct edge_connection_t edge_;
+
+  /** Nickname of planned exit node -- used with .exit support. */
+  /* XXX prop220: we need to make chosen_exit_name able to encode Ed IDs too.
+   * That's logically part of the UI parts for prop220 though. */
+  char *chosen_exit_name;
+
+  socks_request_t *socks_request; /**< SOCKS structure describing request (AP
+                                   * only.) */
+
+  /* === Isolation related, AP only. === */
+  entry_port_cfg_t entry_cfg;
+  /** AP only: The newnym epoch in which we created this connection. */
+  unsigned nym_epoch;
+
+  /** AP only: The original requested address before we rewrote it. */
+  char *original_dest_address;
+  /* Other fields to isolate on already exist.  The ClientAddr is addr.  The
+     ClientProtocol is a combination of type and socks_request->
+     socks_version.  SocksAuth is socks_request->username/password.
+     DestAddr is in socks_request->address. */
+
+  /** Number of times we've reassigned this application connection to
+   * a new circuit. We keep track because the timeout is longer if we've
+   * already retried several times. */
+  uint8_t num_socks_retries;
+
+  /** For AP connections only: buffer for data that we have sent
+   * optimistically, which we might need to re-send if we have to
+   * retry this connection. */
+  struct buf_t *pending_optimistic_data;
+  /* For AP connections only: buffer for data that we previously sent
+  * optimistically which we are currently re-sending as we retry this
+  * connection. */
+  struct buf_t *sending_optimistic_data;
+
+  /** If this is a DNSPort connection, this field holds the pending DNS
+   * request that we're going to try to answer.  */
+  struct evdns_server_request *dns_server_request;
+
+#define DEBUGGING_17659
+
+#ifdef DEBUGGING_17659
+  uint16_t marked_pending_circ_line;
+  const char *marked_pending_circ_file;
+#endif
+
+#define NUM_CIRCUITS_LAUNCHED_THRESHOLD 10
+  /** Number of times we've launched a circuit to handle this stream. If
+    * it gets too high, that could indicate an inconsistency between our
+    * "launch a circuit to handle this stream" logic and our "attach our
+    * stream to one of the available circuits" logic. */
+  unsigned int num_circuits_launched:4;
+
+  /** True iff this stream must attach to a one-hop circuit (e.g. for
+   * begin_dir). */
+  unsigned int want_onehop:1;
+  /** True iff this stream should use a BEGIN_DIR relay command to establish
+   * itself rather than BEGIN (either via onehop or via a whole circuit). */
+  unsigned int use_begindir:1;
+
+  /** For AP connections only. If 1, and we fail to reach the chosen exit,
+   * stop requiring it. */
+  unsigned int chosen_exit_optional:1;
+  /** For AP connections only. If non-zero, this exit node was picked as
+   * a result of the TrackHostExit, and the value decrements every time
+   * we fail to complete a circuit to our chosen exit -- if it reaches
+   * zero, abandon the associated mapaddress. */
+  unsigned int chosen_exit_retries:3;
+
+  /** True iff this is an AP connection that came from a transparent or
+   * NATd connection */
+  unsigned int is_transparent_ap:1;
+
+  /** For AP connections only: Set if this connection's target exit node
+   * allows optimistic data (that is, data sent on this stream before
+   * the exit has sent a CONNECTED cell) and we have chosen to use it.
+   */
+  unsigned int may_use_optimistic_data : 1;
+};
+
+/** Cast a entry_connection_t subtype pointer to a edge_connection_t **/
+#define ENTRY_TO_EDGE_CONN(c) (&(((c))->edge_))
+
+#endif
+
diff --git a/src/core/or/entry_port_cfg_st.h b/src/core/or/entry_port_cfg_st.h
new file mode 100644
index 0000000000..87dfb331e5
--- /dev/null
+++ b/src/core/or/entry_port_cfg_st.h
@@ -0,0 +1,54 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef ENTRY_PORT_CFG_ST_H
+#define ENTRY_PORT_CFG_ST_H
+
+#include "lib/cc/torint.h"
+#include "core/or/or.h"
+
+struct entry_port_cfg_t {
+  /* Client port types (socks, dns, trans, natd) only: */
+  uint8_t isolation_flags; /**< Zero or more isolation flags */
+  int session_group; /**< A session group, or -1 if this port is not in a
+                      * session group. */
+
+  /* Socks only: */
+  /** When both no-auth and user/pass are advertised by a SOCKS client, select
+   * no-auth. */
+  unsigned int socks_prefer_no_auth : 1;
+  /** When ISO_SOCKSAUTH is in use, Keep-Alive circuits indefinitely. */
+  unsigned int socks_iso_keep_alive : 1;
+
+  /* Client port types only: */
+  unsigned int ipv4_traffic : 1;
+  unsigned int ipv6_traffic : 1;
+  unsigned int prefer_ipv6 : 1;
+  unsigned int dns_request : 1;
+  unsigned int onion_traffic : 1;
+
+  /** For a socks listener: should we cache IPv4/IPv6 DNS information that
+   * exit nodes tell us?
+   *
+   * @{ */
+  unsigned int cache_ipv4_answers : 1;
+  unsigned int cache_ipv6_answers : 1;
+  /** @} */
+  /** For a socks listeners: if we find an answer in our client-side DNS cache,
+   * should we use it?
+   *
+   * @{ */
+  unsigned int use_cached_ipv4_answers : 1;
+  unsigned int use_cached_ipv6_answers : 1;
+  /** @} */
+  /** For socks listeners: When we can automap an address to IPv4 or IPv6,
+   * do we prefer IPv6? */
+  unsigned int prefer_ipv6_virtaddr : 1;
+
+};
+
+#endif
+
diff --git a/src/core/or/extend_info_st.h b/src/core/or/extend_info_st.h
new file mode 100644
index 0000000000..bc7a77b1b2
--- /dev/null
+++ b/src/core/or/extend_info_st.h
@@ -0,0 +1,30 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef EXTEND_INFO_ST_H
+#define EXTEND_INFO_ST_H
+
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+/** Information on router used when extending a circuit. We don't need a
+ * full routerinfo_t to extend: we only need addr:port:keyid to build an OR
+ * connection, and onion_key to create the onionskin. Note that for onehop
+ * general-purpose tunnels, the onion_key is NULL. */
+struct extend_info_t {
+  char nickname[MAX_HEX_NICKNAME_LEN+1]; /**< This router's nickname for
+                                          * display. */
+  /** Hash of this router's RSA identity key. */
+  char identity_digest[DIGEST_LEN];
+  /** Ed25519 identity for this router, if any. */
+  ed25519_public_key_t ed_identity;
+  uint16_t port; /**< OR port. */
+  tor_addr_t addr; /**< IP address. */
+  crypto_pk_t *onion_key; /**< Current onionskin key. */
+  curve25519_public_key_t curve25519_onion_key;
+};
+
+#endif
diff --git a/src/core/or/half_edge_st.h b/src/core/or/half_edge_st.h
new file mode 100644
index 0000000000..d4617be108
--- /dev/null
+++ b/src/core/or/half_edge_st.h
@@ -0,0 +1,34 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef HALF_EDGE_ST_H
+#define HALF_EDGE_ST_H
+
+#include "core/or/or.h"
+
+/**
+ * Struct to track a connection that we closed that the other end
+ * still thinks is open. Exists in origin_circuit_t.half_streams until
+ * we get an end cell or a resolved cell for this stream id.
+ */
+typedef struct half_edge_t {
+  /** stream_id for the half-closed connection */
+  streamid_t stream_id;
+
+  /** How many sendme's can the other end still send, based on how
+   * much data we had sent at the time of close */
+  int sendmes_pending;
+
+  /** How much more data can the other end still send, based on
+   * our deliver window */
+  int data_pending;
+
+  /** Is there a connected cell pending? */
+  int connected_pending : 1;
+} half_edge_t;
+
+#endif
+
diff --git a/src/core/or/listener_connection_st.h b/src/core/or/listener_connection_st.h
new file mode 100644
index 0000000000..8989a39dc8
--- /dev/null
+++ b/src/core/or/listener_connection_st.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef LISTENER_CONNECTION_ST_H
+#define LISTENER_CONNECTION_ST_H
+
+#include "core/or/connection_st.h"
+
+/** Subtype of connection_t; used for a listener socket. */
+struct listener_connection_t {
+  connection_t base_;
+
+  /** If the connection is a CONN_TYPE_AP_DNS_LISTENER, this field points
+   * to the evdns_server_port it uses to listen to and answer connections. */
+  struct evdns_server_port *dns_server_port;
+
+  entry_port_cfg_t entry_cfg;
+
+};
+
+#endif
+
diff --git a/src/core/or/onion.c b/src/core/or/onion.c
new file mode 100644
index 0000000000..aa77465b96
--- /dev/null
+++ b/src/core/or/onion.c
@@ -0,0 +1,720 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file onion.c
+ * \brief Functions to queue create cells,
+ * and parse and create the CREATE cell and its allies.
+ *
+ * This module has a few functions, all related to the CREATE/CREATED
+ * handshake that we use on links in order to create a circuit, and the
+ * related EXTEND/EXTENDED handshake that we use over circuits in order to
+ * extend them an additional hop.
+ *
+ * Clients invoke these functions when creating or extending a circuit,
+ * from circuitbuild.c.
+ *
+ * Relays invoke these functions when they receive a CREATE or EXTEND
+ * cell in command.c or relay.c, in order to queue the pending request.
+ * They also invoke them from cpuworker.c, which handles dispatching
+ * onionskin requests to different worker threads.
+ *
+ * <br>
+ *
+ * This module also handles:
+ *  <ul>
+ *  <li> Queueing incoming onionskins on the relay side before passing
+ *      them to worker threads.
+ *   <li>Expiring onionskins on the relay side if they have waited for
+ *     too long.
+ *   <li>Packaging private keys on the server side in order to pass
+ *     them to worker threads.
+ *   <li>Encoding and decoding CREATE, CREATED, CREATE2, and CREATED2 cells.
+ *   <li>Encoding and decodign EXTEND, EXTENDED, EXTEND2, and EXTENDED2
+ *    relay cells.
+ * </ul>
+ **/
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/crypto/onion_crypto.h"
+#include "core/crypto/onion_fast.h"
+#include "core/crypto/onion_ntor.h"
+#include "core/crypto/onion_tap.h"
+#include "core/or/onion.h"
+#include "feature/nodelist/networkstatus.h"
+
+#include "core/or/cell_st.h"
+
+// trunnel
+#include "trunnel/ed25519_cert.h"
+
+/** Helper: return 0 if <b>cell</b> appears valid, -1 otherwise. If
+ * <b>unknown_ok</b> is true, allow cells with handshake types we don't
+ * recognize. */
+static int
+check_create_cell(const create_cell_t *cell, int unknown_ok)
+{
+  switch (cell->cell_type) {
+  case CELL_CREATE:
+    if (cell->handshake_type != ONION_HANDSHAKE_TYPE_TAP &&
+        cell->handshake_type != ONION_HANDSHAKE_TYPE_NTOR)
+      return -1;
+    break;
+  case CELL_CREATE_FAST:
+    if (cell->handshake_type != ONION_HANDSHAKE_TYPE_FAST)
+      return -1;
+    break;
+  case CELL_CREATE2:
+    break;
+  default:
+    return -1;
+  }
+
+  switch (cell->handshake_type) {
+  case ONION_HANDSHAKE_TYPE_TAP:
+    if (cell->handshake_len != TAP_ONIONSKIN_CHALLENGE_LEN)
+      return -1;
+    break;
+  case ONION_HANDSHAKE_TYPE_FAST:
+    if (cell->handshake_len != CREATE_FAST_LEN)
+      return -1;
+    break;
+  case ONION_HANDSHAKE_TYPE_NTOR:
+    if (cell->handshake_len != NTOR_ONIONSKIN_LEN)
+      return -1;
+    break;
+  default:
+    if (! unknown_ok)
+      return -1;
+  }
+
+  return 0;
+}
+
+/** Write the various parameters into the create cell. Separate from
+ * create_cell_parse() to make unit testing easier.
+ */
+void
+create_cell_init(create_cell_t *cell_out, uint8_t cell_type,
+                 uint16_t handshake_type, uint16_t handshake_len,
+                 const uint8_t *onionskin)
+{
+  memset(cell_out, 0, sizeof(*cell_out));
+
+  cell_out->cell_type = cell_type;
+  cell_out->handshake_type = handshake_type;
+  cell_out->handshake_len = handshake_len;
+  memcpy(cell_out->onionskin, onionskin, handshake_len);
+}
+
+/** Helper: parse the CREATE2 payload at <b>p</b>, which could be up to
+ * <b>p_len</b> bytes long, and use it to fill the fields of
+ * <b>cell_out</b>. Return 0 on success and -1 on failure.
+ *
+ * Note that part of the body of an EXTEND2 cell is a CREATE2 payload, so
+ * this function is also used for parsing those.
+ */
+static int
+parse_create2_payload(create_cell_t *cell_out, const uint8_t *p, size_t p_len)
+{
+  uint16_t handshake_type, handshake_len;
+
+  if (p_len < 4)
+    return -1;
+
+  handshake_type = ntohs(get_uint16(p));
+  handshake_len = ntohs(get_uint16(p+2));
+
+  if (handshake_len > CELL_PAYLOAD_SIZE - 4 || handshake_len > p_len - 4)
+    return -1;
+  if (handshake_type == ONION_HANDSHAKE_TYPE_FAST)
+    return -1;
+
+  create_cell_init(cell_out, CELL_CREATE2, handshake_type, handshake_len,
+                   p+4);
+  return 0;
+}
+
+/** Magic string which, in a CREATE or EXTEND cell, indicates that a seeming
+ * TAP payload is really an ntor payload.  We'd do away with this if every
+ * relay supported EXTEND2, but we want to be able to extend from A to B with
+ * ntor even when A doesn't understand EXTEND2 and so can't generate a
+ * CREATE2 cell.
+ **/
+#define NTOR_CREATE_MAGIC "ntorNTORntorNTOR"
+
+/** Parse a CREATE, CREATE_FAST, or CREATE2 cell from <b>cell_in</b> into
+ * <b>cell_out</b>. Return 0 on success, -1 on failure. (We reject some
+ * syntactically valid CREATE2 cells that we can't generate or react to.) */
+int
+create_cell_parse(create_cell_t *cell_out, const cell_t *cell_in)
+{
+  switch (cell_in->command) {
+  case CELL_CREATE:
+    if (tor_memeq(cell_in->payload, NTOR_CREATE_MAGIC, 16)) {
+      create_cell_init(cell_out, CELL_CREATE, ONION_HANDSHAKE_TYPE_NTOR,
+                       NTOR_ONIONSKIN_LEN, cell_in->payload+16);
+    } else {
+      create_cell_init(cell_out, CELL_CREATE, ONION_HANDSHAKE_TYPE_TAP,
+                       TAP_ONIONSKIN_CHALLENGE_LEN, cell_in->payload);
+    }
+    break;
+  case CELL_CREATE_FAST:
+    create_cell_init(cell_out, CELL_CREATE_FAST, ONION_HANDSHAKE_TYPE_FAST,
+                     CREATE_FAST_LEN, cell_in->payload);
+    break;
+  case CELL_CREATE2:
+    if (parse_create2_payload(cell_out, cell_in->payload,
+                              CELL_PAYLOAD_SIZE) < 0)
+      return -1;
+    break;
+  default:
+    return -1;
+  }
+
+  return check_create_cell(cell_out, 0);
+}
+
+/** Helper: return 0 if <b>cell</b> appears valid, -1 otherwise. */
+static int
+check_created_cell(const created_cell_t *cell)
+{
+  switch (cell->cell_type) {
+  case CELL_CREATED:
+    if (cell->handshake_len != TAP_ONIONSKIN_REPLY_LEN &&
+        cell->handshake_len != NTOR_REPLY_LEN)
+      return -1;
+    break;
+  case CELL_CREATED_FAST:
+    if (cell->handshake_len != CREATED_FAST_LEN)
+      return -1;
+    break;
+  case CELL_CREATED2:
+    if (cell->handshake_len > RELAY_PAYLOAD_SIZE-2)
+      return -1;
+    break;
+  }
+
+  return 0;
+}
+
+/** Parse a CREATED, CREATED_FAST, or CREATED2 cell from <b>cell_in</b> into
+ * <b>cell_out</b>. Return 0 on success, -1 on failure. */
+int
+created_cell_parse(created_cell_t *cell_out, const cell_t *cell_in)
+{
+  memset(cell_out, 0, sizeof(*cell_out));
+
+  switch (cell_in->command) {
+  case CELL_CREATED:
+    cell_out->cell_type = CELL_CREATED;
+    cell_out->handshake_len = TAP_ONIONSKIN_REPLY_LEN;
+    memcpy(cell_out->reply, cell_in->payload, TAP_ONIONSKIN_REPLY_LEN);
+    break;
+  case CELL_CREATED_FAST:
+    cell_out->cell_type = CELL_CREATED_FAST;
+    cell_out->handshake_len = CREATED_FAST_LEN;
+    memcpy(cell_out->reply, cell_in->payload, CREATED_FAST_LEN);
+    break;
+  case CELL_CREATED2:
+    {
+      const uint8_t *p = cell_in->payload;
+      cell_out->cell_type = CELL_CREATED2;
+      cell_out->handshake_len = ntohs(get_uint16(p));
+      if (cell_out->handshake_len > CELL_PAYLOAD_SIZE - 2)
+        return -1;
+      memcpy(cell_out->reply, p+2, cell_out->handshake_len);
+      break;
+    }
+  }
+
+  return check_created_cell(cell_out);
+}
+
+/** Helper: return 0 if <b>cell</b> appears valid, -1 otherwise. */
+static int
+check_extend_cell(const extend_cell_t *cell)
+{
+  if (tor_digest_is_zero((const char*)cell->node_id))
+    return -1;
+  /* We don't currently allow EXTEND2 cells without an IPv4 address */
+  if (tor_addr_family(&cell->orport_ipv4.addr) == AF_UNSPEC)
+    return -1;
+  if (cell->create_cell.cell_type == CELL_CREATE) {
+    if (cell->cell_type != RELAY_COMMAND_EXTEND)
+      return -1;
+  } else if (cell->create_cell.cell_type == CELL_CREATE2) {
+    if (cell->cell_type != RELAY_COMMAND_EXTEND2 &&
+        cell->cell_type != RELAY_COMMAND_EXTEND)
+      return -1;
+  } else {
+    /* In particular, no CREATE_FAST cells are allowed */
+    return -1;
+  }
+  if (cell->create_cell.handshake_type == ONION_HANDSHAKE_TYPE_FAST)
+    return -1;
+
+  return check_create_cell(&cell->create_cell, 1);
+}
+
+static int
+extend_cell_from_extend1_cell_body(extend_cell_t *cell_out,
+                                   const extend1_cell_body_t *cell)
+{
+  tor_assert(cell_out);
+  tor_assert(cell);
+  memset(cell_out, 0, sizeof(*cell_out));
+  tor_addr_make_unspec(&cell_out->orport_ipv4.addr);
+  tor_addr_make_unspec(&cell_out->orport_ipv6.addr);
+
+  cell_out->cell_type = RELAY_COMMAND_EXTEND;
+  tor_addr_from_ipv4h(&cell_out->orport_ipv4.addr, cell->ipv4addr);
+  cell_out->orport_ipv4.port = cell->port;
+  if (tor_memeq(cell->onionskin, NTOR_CREATE_MAGIC, 16)) {
+    cell_out->create_cell.cell_type = CELL_CREATE2;
+    cell_out->create_cell.handshake_type = ONION_HANDSHAKE_TYPE_NTOR;
+    cell_out->create_cell.handshake_len = NTOR_ONIONSKIN_LEN;
+    memcpy(cell_out->create_cell.onionskin, cell->onionskin + 16,
+           NTOR_ONIONSKIN_LEN);
+  } else {
+    cell_out->create_cell.cell_type = CELL_CREATE;
+    cell_out->create_cell.handshake_type = ONION_HANDSHAKE_TYPE_TAP;
+    cell_out->create_cell.handshake_len = TAP_ONIONSKIN_CHALLENGE_LEN;
+    memcpy(cell_out->create_cell.onionskin, cell->onionskin,
+           TAP_ONIONSKIN_CHALLENGE_LEN);
+  }
+  memcpy(cell_out->node_id, cell->identity, DIGEST_LEN);
+  return 0;
+}
+
+static int
+create_cell_from_create2_cell_body(create_cell_t *cell_out,
+                                   const create2_cell_body_t *cell)
+{
+  tor_assert(cell_out);
+  tor_assert(cell);
+  memset(cell_out, 0, sizeof(create_cell_t));
+  if (BUG(cell->handshake_len > sizeof(cell_out->onionskin))) {
+    /* This should be impossible because there just isn't enough room in the
+     * input cell to make the handshake_len this large and provide a
+     * handshake_data to match. */
+    return -1;
+  }
+
+  cell_out->cell_type = CELL_CREATE2;
+  cell_out->handshake_type = cell->handshake_type;
+  cell_out->handshake_len = cell->handshake_len;
+  memcpy(cell_out->onionskin,
+       create2_cell_body_getconstarray_handshake_data(cell),
+       cell->handshake_len);
+  return 0;
+}
+
+static int
+extend_cell_from_extend2_cell_body(extend_cell_t *cell_out,
+                                   const extend2_cell_body_t *cell)
+{
+  tor_assert(cell_out);
+  tor_assert(cell);
+  int found_ipv4 = 0, found_ipv6 = 0, found_rsa_id = 0, found_ed_id = 0;
+  memset(cell_out, 0, sizeof(*cell_out));
+  tor_addr_make_unspec(&cell_out->orport_ipv4.addr);
+  tor_addr_make_unspec(&cell_out->orport_ipv6.addr);
+  cell_out->cell_type = RELAY_COMMAND_EXTEND2;
+
+  unsigned i;
+  for (i = 0; i < cell->n_spec; ++i) {
+    const link_specifier_t *ls = extend2_cell_body_getconst_ls(cell, i);
+    switch (ls->ls_type) {
+      case LS_IPV4:
+        if (found_ipv4)
+          continue;
+        found_ipv4 = 1;
+        tor_addr_from_ipv4h(&cell_out->orport_ipv4.addr, ls->un_ipv4_addr);
+        cell_out->orport_ipv4.port = ls->un_ipv4_port;
+        break;
+      case LS_IPV6:
+        if (found_ipv6)
+          continue;
+        found_ipv6 = 1;
+        tor_addr_from_ipv6_bytes(&cell_out->orport_ipv6.addr,
+                                 (const char *)ls->un_ipv6_addr);
+        cell_out->orport_ipv6.port = ls->un_ipv6_port;
+        break;
+      case LS_LEGACY_ID:
+        if (found_rsa_id)
+          return -1;
+        found_rsa_id = 1;
+        memcpy(cell_out->node_id, ls->un_legacy_id, 20);
+        break;
+      case LS_ED25519_ID:
+        if (found_ed_id)
+          return -1;
+        found_ed_id = 1;
+        memcpy(cell_out->ed_pubkey.pubkey, ls->un_ed25519_id, 32);
+        break;
+      default:
+        /* Ignore this, whatever it is. */
+        break;
+    }
+  }
+
+  if (!found_rsa_id || !found_ipv4) /* These are mandatory */
+    return -1;
+
+  return create_cell_from_create2_cell_body(&cell_out->create_cell,
+                                            cell->create2);
+}
+
+/** Parse an EXTEND or EXTEND2 cell (according to <b>command</b>) from the
+ * <b>payload_length</b> bytes of <b>payload</b> into <b>cell_out</b>. Return
+ * 0 on success, -1 on failure. */
+int
+extend_cell_parse(extend_cell_t *cell_out, const uint8_t command,
+                  const uint8_t *payload, size_t payload_length)
+{
+
+  tor_assert(cell_out);
+  tor_assert(payload);
+
+  if (payload_length > RELAY_PAYLOAD_SIZE)
+    return -1;
+
+  switch (command) {
+  case RELAY_COMMAND_EXTEND:
+    {
+      extend1_cell_body_t *cell = NULL;
+      if (extend1_cell_body_parse(&cell, payload, payload_length)<0 ||
+          cell == NULL) {
+        if (cell)
+          extend1_cell_body_free(cell);
+        return -1;
+      }
+      int r = extend_cell_from_extend1_cell_body(cell_out, cell);
+      extend1_cell_body_free(cell);
+      if (r < 0)
+        return r;
+    }
+    break;
+  case RELAY_COMMAND_EXTEND2:
+    {
+      extend2_cell_body_t *cell = NULL;
+      if (extend2_cell_body_parse(&cell, payload, payload_length) < 0 ||
+          cell == NULL) {
+        if (cell)
+          extend2_cell_body_free(cell);
+        return -1;
+      }
+      int r = extend_cell_from_extend2_cell_body(cell_out, cell);
+      extend2_cell_body_free(cell);
+      if (r < 0)
+        return r;
+    }
+    break;
+  default:
+    return -1;
+  }
+
+  return check_extend_cell(cell_out);
+}
+
+/** Helper: return 0 if <b>cell</b> appears valid, -1 otherwise. */
+static int
+check_extended_cell(const extended_cell_t *cell)
+{
+  tor_assert(cell);
+  if (cell->created_cell.cell_type == CELL_CREATED) {
+    if (cell->cell_type != RELAY_COMMAND_EXTENDED)
+      return -1;
+  } else if (cell->created_cell.cell_type == CELL_CREATED2) {
+    if (cell->cell_type != RELAY_COMMAND_EXTENDED2)
+      return -1;
+  } else {
+    return -1;
+  }
+
+  return check_created_cell(&cell->created_cell);
+}
+
+/** Parse an EXTENDED or EXTENDED2 cell (according to <b>command</b>) from the
+ * <b>payload_length</b> bytes of <b>payload</b> into <b>cell_out</b>. Return
+ * 0 on success, -1 on failure. */
+int
+extended_cell_parse(extended_cell_t *cell_out,
+                    const uint8_t command, const uint8_t *payload,
+                    size_t payload_len)
+{
+  tor_assert(cell_out);
+  tor_assert(payload);
+
+  memset(cell_out, 0, sizeof(*cell_out));
+  if (payload_len > RELAY_PAYLOAD_SIZE)
+    return -1;
+
+  switch (command) {
+  case RELAY_COMMAND_EXTENDED:
+    if (payload_len != TAP_ONIONSKIN_REPLY_LEN)
+      return -1;
+    cell_out->cell_type = RELAY_COMMAND_EXTENDED;
+    cell_out->created_cell.cell_type = CELL_CREATED;
+    cell_out->created_cell.handshake_len = TAP_ONIONSKIN_REPLY_LEN;
+    memcpy(cell_out->created_cell.reply, payload, TAP_ONIONSKIN_REPLY_LEN);
+    break;
+  case RELAY_COMMAND_EXTENDED2:
+    {
+      cell_out->cell_type = RELAY_COMMAND_EXTENDED2;
+      cell_out->created_cell.cell_type = CELL_CREATED2;
+      cell_out->created_cell.handshake_len = ntohs(get_uint16(payload));
+      if (cell_out->created_cell.handshake_len > RELAY_PAYLOAD_SIZE - 2 ||
+          cell_out->created_cell.handshake_len > payload_len - 2)
+        return -1;
+      memcpy(cell_out->created_cell.reply, payload+2,
+             cell_out->created_cell.handshake_len);
+    }
+    break;
+  default:
+    return -1;
+  }
+
+  return check_extended_cell(cell_out);
+}
+
+/** Fill <b>cell_out</b> with a correctly formatted version of the
+ * CREATE{,_FAST,2} cell in <b>cell_in</b>. Return 0 on success, -1 on
+ * failure.  This is a cell we didn't originate if <b>relayed</b> is true. */
+static int
+create_cell_format_impl(cell_t *cell_out, const create_cell_t *cell_in,
+                        int relayed)
+{
+  uint8_t *p;
+  size_t space;
+  if (check_create_cell(cell_in, relayed) < 0)
+    return -1;
+
+  memset(cell_out->payload, 0, sizeof(cell_out->payload));
+  cell_out->command = cell_in->cell_type;
+
+  p = cell_out->payload;
+  space = sizeof(cell_out->payload);
+
+  switch (cell_in->cell_type) {
+  case CELL_CREATE:
+    if (cell_in->handshake_type == ONION_HANDSHAKE_TYPE_NTOR) {
+      memcpy(p, NTOR_CREATE_MAGIC, 16);
+      p += 16;
+      space -= 16;
+    }
+    /* Fall through */
+  case CELL_CREATE_FAST:
+    tor_assert(cell_in->handshake_len <= space);
+    memcpy(p, cell_in->onionskin, cell_in->handshake_len);
+    break;
+  case CELL_CREATE2:
+    tor_assert(cell_in->handshake_len <= sizeof(cell_out->payload)-4);
+    set_uint16(cell_out->payload, htons(cell_in->handshake_type));
+    set_uint16(cell_out->payload+2, htons(cell_in->handshake_len));
+    memcpy(cell_out->payload + 4, cell_in->onionskin, cell_in->handshake_len);
+    break;
+  default:
+    return -1;
+  }
+
+  return 0;
+}
+
+int
+create_cell_format(cell_t *cell_out, const create_cell_t *cell_in)
+{
+  return create_cell_format_impl(cell_out, cell_in, 0);
+}
+
+int
+create_cell_format_relayed(cell_t *cell_out, const create_cell_t *cell_in)
+{
+  return create_cell_format_impl(cell_out, cell_in, 1);
+}
+
+/** Fill <b>cell_out</b> with a correctly formatted version of the
+ * CREATED{,_FAST,2} cell in <b>cell_in</b>. Return 0 on success, -1 on
+ * failure. */
+int
+created_cell_format(cell_t *cell_out, const created_cell_t *cell_in)
+{
+  if (check_created_cell(cell_in) < 0)
+    return -1;
+
+  memset(cell_out->payload, 0, sizeof(cell_out->payload));
+  cell_out->command = cell_in->cell_type;
+
+  switch (cell_in->cell_type) {
+  case CELL_CREATED:
+  case CELL_CREATED_FAST:
+    tor_assert(cell_in->handshake_len <= sizeof(cell_out->payload));
+    memcpy(cell_out->payload, cell_in->reply, cell_in->handshake_len);
+    break;
+  case CELL_CREATED2:
+    tor_assert(cell_in->handshake_len <= sizeof(cell_out->payload)-2);
+    set_uint16(cell_out->payload, htons(cell_in->handshake_len));
+    memcpy(cell_out->payload + 2, cell_in->reply, cell_in->handshake_len);
+    break;
+  default:
+    return -1;
+  }
+  return 0;
+}
+
+/** Return true iff we are configured (by torrc or by the networkstatus
+ * parameters) to use Ed25519 identities in our Extend2 cells. */
+static int
+should_include_ed25519_id_extend_cells(const networkstatus_t *ns,
+                                       const or_options_t *options)
+{
+  if (options->ExtendByEd25519ID != -1)
+    return options->ExtendByEd25519ID; /* The user has an opinion. */
+
+  return (int) networkstatus_get_param(ns, "ExtendByEd25519ID",
+                                       0 /* default */,
+                                       0 /* min */,
+                                       1 /*max*/);
+}
+
+/** Format the EXTEND{,2} cell in <b>cell_in</b>, storing its relay payload in
+ * <b>payload_out</b>, the number of bytes used in *<b>len_out</b>, and the
+ * relay command in *<b>command_out</b>. The <b>payload_out</b> must have
+ * RELAY_PAYLOAD_SIZE bytes available.  Return 0 on success, -1 on failure. */
+int
+extend_cell_format(uint8_t *command_out, uint16_t *len_out,
+                   uint8_t *payload_out, const extend_cell_t *cell_in)
+{
+  uint8_t *p;
+  if (check_extend_cell(cell_in) < 0)
+    return -1;
+
+  p = payload_out;
+
+  memset(p, 0, RELAY_PAYLOAD_SIZE);
+
+  switch (cell_in->cell_type) {
+  case RELAY_COMMAND_EXTEND:
+    {
+      *command_out = RELAY_COMMAND_EXTEND;
+      *len_out = 6 + TAP_ONIONSKIN_CHALLENGE_LEN + DIGEST_LEN;
+      set_uint32(p, tor_addr_to_ipv4n(&cell_in->orport_ipv4.addr));
+      set_uint16(p+4, htons(cell_in->orport_ipv4.port));
+      if (cell_in->create_cell.handshake_type == ONION_HANDSHAKE_TYPE_NTOR) {
+        memcpy(p+6, NTOR_CREATE_MAGIC, 16);
+        memcpy(p+22, cell_in->create_cell.onionskin, NTOR_ONIONSKIN_LEN);
+      } else {
+        memcpy(p+6, cell_in->create_cell.onionskin,
+               TAP_ONIONSKIN_CHALLENGE_LEN);
+      }
+      memcpy(p+6+TAP_ONIONSKIN_CHALLENGE_LEN, cell_in->node_id, DIGEST_LEN);
+    }
+    break;
+  case RELAY_COMMAND_EXTEND2:
+    {
+      uint8_t n_specifiers = 2;
+      *command_out = RELAY_COMMAND_EXTEND2;
+      extend2_cell_body_t *cell = extend2_cell_body_new();
+      link_specifier_t *ls;
+      {
+        /* IPv4 specifier first. */
+        ls = link_specifier_new();
+        extend2_cell_body_add_ls(cell, ls);
+        ls->ls_type = LS_IPV4;
+        ls->ls_len = 6;
+        ls->un_ipv4_addr = tor_addr_to_ipv4h(&cell_in->orport_ipv4.addr);
+        ls->un_ipv4_port = cell_in->orport_ipv4.port;
+      }
+      {
+        /* Then RSA id */
+        ls = link_specifier_new();
+        extend2_cell_body_add_ls(cell, ls);
+        ls->ls_type = LS_LEGACY_ID;
+        ls->ls_len = DIGEST_LEN;
+        memcpy(ls->un_legacy_id, cell_in->node_id, DIGEST_LEN);
+      }
+      if (should_include_ed25519_id_extend_cells(NULL, get_options()) &&
+          !ed25519_public_key_is_zero(&cell_in->ed_pubkey)) {
+        /* Then, maybe, the ed25519 id! */
+        ++n_specifiers;
+        ls = link_specifier_new();
+        extend2_cell_body_add_ls(cell, ls);
+        ls->ls_type = LS_ED25519_ID;
+        ls->ls_len = 32;
+        memcpy(ls->un_ed25519_id, cell_in->ed_pubkey.pubkey, 32);
+      }
+      cell->n_spec = n_specifiers;
+
+      /* Now, the handshake */
+      cell->create2 = create2_cell_body_new();
+      cell->create2->handshake_type = cell_in->create_cell.handshake_type;
+      cell->create2->handshake_len = cell_in->create_cell.handshake_len;
+      create2_cell_body_setlen_handshake_data(cell->create2,
+                                         cell_in->create_cell.handshake_len);
+      memcpy(create2_cell_body_getarray_handshake_data(cell->create2),
+             cell_in->create_cell.onionskin,
+             cell_in->create_cell.handshake_len);
+
+      ssize_t len_encoded = extend2_cell_body_encode(
+                             payload_out, RELAY_PAYLOAD_SIZE,
+                             cell);
+      extend2_cell_body_free(cell);
+      if (len_encoded < 0 || len_encoded > UINT16_MAX)
+        return -1;
+      *len_out = (uint16_t) len_encoded;
+    }
+    break;
+  default:
+    return -1;
+  }
+
+  return 0;
+}
+
+/** Format the EXTENDED{,2} cell in <b>cell_in</b>, storing its relay payload
+ * in <b>payload_out</b>, the number of bytes used in *<b>len_out</b>, and the
+ * relay command in *<b>command_out</b>. The <b>payload_out</b> must have
+ * RELAY_PAYLOAD_SIZE bytes available.  Return 0 on success, -1 on failure. */
+int
+extended_cell_format(uint8_t *command_out, uint16_t *len_out,
+                     uint8_t *payload_out, const extended_cell_t *cell_in)
+{
+  uint8_t *p;
+  if (check_extended_cell(cell_in) < 0)
+    return -1;
+
+  p = payload_out;
+  memset(p, 0, RELAY_PAYLOAD_SIZE);
+
+  switch (cell_in->cell_type) {
+  case RELAY_COMMAND_EXTENDED:
+    {
+      *command_out = RELAY_COMMAND_EXTENDED;
+      *len_out = TAP_ONIONSKIN_REPLY_LEN;
+      memcpy(payload_out, cell_in->created_cell.reply,
+             TAP_ONIONSKIN_REPLY_LEN);
+    }
+    break;
+  case RELAY_COMMAND_EXTENDED2:
+    {
+      *command_out = RELAY_COMMAND_EXTENDED2;
+      *len_out = 2 + cell_in->created_cell.handshake_len;
+      set_uint16(payload_out, htons(cell_in->created_cell.handshake_len));
+      if (2+cell_in->created_cell.handshake_len > RELAY_PAYLOAD_SIZE)
+        return -1;
+      memcpy(payload_out+2, cell_in->created_cell.reply,
+             cell_in->created_cell.handshake_len);
+    }
+    break;
+  default:
+    return -1;
+  }
+
+  return 0;
+}
diff --git a/src/or/onion.h b/src/core/or/onion.h
index 0275fa00d2..bb0b5b8dfd 100644
--- a/src/or/onion.h
+++ b/src/core/or/onion.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,45 +13,13 @@
 #define TOR_ONION_H
 
 struct create_cell_t;
-int onion_pending_add(or_circuit_t *circ, struct create_cell_t *onionskin);
-or_circuit_t *onion_next_task(struct create_cell_t **onionskin_out);
-int onion_num_pending(uint16_t handshake_type);
-void onion_pending_remove(or_circuit_t *circ);
-void clear_pending_onions(void);
-
-typedef struct server_onion_keys_t {
-  uint8_t my_identity[DIGEST_LEN];
-  crypto_pk_t *onion_key;
-  crypto_pk_t *last_onion_key;
-  di_digest256_map_t *curve25519_key_map;
-  curve25519_keypair_t *junk_keypair;
-} server_onion_keys_t;
+struct curve25519_keypair_t;
+struct curve25519_public_key_t;
+#include "lib/crypt_ops/crypto_ed25519.h"
 
 #define MAX_ONIONSKIN_CHALLENGE_LEN 255
 #define MAX_ONIONSKIN_REPLY_LEN 255
 
-server_onion_keys_t *server_onion_keys_new(void);
-void server_onion_keys_free(server_onion_keys_t *keys);
-
-void onion_handshake_state_release(onion_handshake_state_t *state);
-
-int onion_skin_create(int type,
-                      const extend_info_t *node,
-                      onion_handshake_state_t *state_out,
-                      uint8_t *onion_skin_out);
-int onion_skin_server_handshake(int type,
-                      const uint8_t *onion_skin, size_t onionskin_len,
-                      const server_onion_keys_t *keys,
-                      uint8_t *reply_out,
-                      uint8_t *keys_out, size_t key_out_len,
-                      uint8_t *rend_nonce_out);
-int onion_skin_client_handshake(int type,
-                      const onion_handshake_state_t *handshake_state,
-                      const uint8_t *reply, size_t reply_len,
-                      uint8_t *keys_out, size_t key_out_len,
-                      uint8_t *rend_authenticator_out,
-                      const char **msg_out);
-
 /** A parsed CREATE, CREATE_FAST, or CREATE2 cell. */
 typedef struct create_cell_t {
   /** The cell command. One of CREATE{,_FAST,2} */
@@ -85,6 +53,8 @@ typedef struct extend_cell_t {
   tor_addr_port_t orport_ipv6;
   /** Identity fingerprint of the node we're conecting to.*/
   uint8_t node_id[DIGEST_LEN];
+  /** Ed25519 public identity key. Zero if not set. */
+  struct ed25519_public_key_t ed_pubkey;
   /** The "create cell" embedded in this extend cell. Note that unlike the
    * create cells we generate ourself, this once can have a handshake type we
    * don't recognize. */
@@ -117,5 +87,4 @@ int extend_cell_format(uint8_t *command_out, uint16_t *len_out,
 int extended_cell_format(uint8_t *command_out, uint16_t *len_out,
                          uint8_t *payload_out, const extended_cell_t *cell_in);
 
-#endif
-
+#endif /* !defined(TOR_ONION_H) */
diff --git a/src/core/or/or.h b/src/core/or/or.h
new file mode 100644
index 0000000000..7c601e49b3
--- /dev/null
+++ b/src/core/or/or.h
@@ -0,0 +1,1094 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file or.h
+ * \brief Master header file for Tor-specific functionality.
+ **/
+
+#ifndef TOR_OR_H
+#define TOR_OR_H
+
+#include "orconfig.h"
+#include "lib/cc/torint.h"
+
+#ifdef HAVE_SIGNAL_H
+#include <signal.h>
+#endif
+#ifdef HAVE_TIME_H
+#include <time.h>
+#endif
+
+#include "lib/arch/bytes.h"
+#include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+#include "lib/container/map.h"
+#include "lib/container/buffers.h"
+#include "lib/container/smartlist.h"
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "lib/crypt_ops/crypto_rsa.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/defs/dh_sizes.h"
+#include "lib/encoding/binascii.h"
+#include "lib/encoding/cstring.h"
+#include "lib/encoding/time_fmt.h"
+#include "lib/err/torerr.h"
+#include "lib/fs/dir.h"
+#include "lib/fs/files.h"
+#include "lib/fs/mmap.h"
+#include "lib/fs/path.h"
+#include "lib/fs/userdb.h"
+#include "lib/geoip/country.h"
+#include "lib/intmath/addsub.h"
+#include "lib/intmath/bits.h"
+#include "lib/intmath/cmp.h"
+#include "lib/intmath/logic.h"
+#include "lib/intmath/muldiv.h"
+#include "lib/log/escape.h"
+#include "lib/log/ratelim.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/net/address.h"
+#include "lib/net/inaddr.h"
+#include "lib/net/socket.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/string/compat_string.h"
+#include "lib/string/parse_int.h"
+#include "lib/string/printf.h"
+#include "lib/string/scanf.h"
+#include "lib/string/util_string.h"
+#include "lib/testsupport/testsupport.h"
+#include "lib/thread/threads.h"
+#include "lib/time/compat_time.h"
+#include "lib/wallclock/approx_time.h"
+#include "lib/wallclock/timeval.h"
+
+#include "ht.h"
+
+// These, more than other includes, are for keeping the other struct
+// definitions working. We should remove them when we minimize our includes.
+#include "core/or/entry_port_cfg_st.h"
+
+struct ed25519_public_key_t;
+struct curve25519_public_key_t;
+
+/* These signals are defined to help handle_control_signal work.
+ */
+#ifndef SIGHUP
+#define SIGHUP 1
+#endif
+#ifndef SIGINT
+#define SIGINT 2
+#endif
+#ifndef SIGUSR1
+#define SIGUSR1 10
+#endif
+#ifndef SIGUSR2
+#define SIGUSR2 12
+#endif
+#ifndef SIGTERM
+#define SIGTERM 15
+#endif
+/* Controller signals start at a high number so we don't
+ * conflict with system-defined signals. */
+#define SIGNEWNYM 129
+#define SIGCLEARDNSCACHE 130
+#define SIGHEARTBEAT 131
+
+#if (SIZEOF_CELL_T != 0)
+/* On Irix, stdlib.h defines a cell_t type, so we need to make sure
+ * that our stuff always calls cell_t something different. */
+#define cell_t tor_cell_t
+#endif
+
+/** Helper macro: Given a pointer to to.base_, of type from*, return &to. */
+#define DOWNCAST(to, ptr) ((to*)SUBTYPE_P(ptr, to, base_))
+
+/** Length of longest allowable configured nickname. */
+#define MAX_NICKNAME_LEN 19
+/** Length of a router identity encoded as a hexadecimal digest, plus
+ * possible dollar sign. */
+#define MAX_HEX_NICKNAME_LEN (HEX_DIGEST_LEN+1)
+/** Maximum length of verbose router identifier: dollar sign, hex ID digest,
+ * equal sign or tilde, nickname. */
+#define MAX_VERBOSE_NICKNAME_LEN (1+HEX_DIGEST_LEN+1+MAX_NICKNAME_LEN)
+
+/** For HTTP parsing: Maximum number of bytes we'll accept in the headers
+ * of an HTTP request or response. */
+#define MAX_HEADERS_SIZE 50000
+
+/** Maximum size, in bytes, of a single router descriptor uploaded to us
+ * as a directory authority. Caches and clients fetch whatever descriptors
+ * the authorities tell them to fetch, and don't care about size. */
+#define MAX_DESCRIPTOR_UPLOAD_SIZE 20000
+
+/** Maximum size of a single extrainfo document, as above. */
+#define MAX_EXTRAINFO_UPLOAD_SIZE 50000
+
+/** Minimum lifetime for an onion key in days. */
+#define MIN_ONION_KEY_LIFETIME_DAYS (1)
+
+/** Maximum lifetime for an onion key in days. */
+#define MAX_ONION_KEY_LIFETIME_DAYS (90)
+
+/** Default lifetime for an onion key in days. */
+#define DEFAULT_ONION_KEY_LIFETIME_DAYS (28)
+
+/** Minimum grace period for acceptance of an onion key in days.
+ * The maximum value is defined in proposal #274 as being the current network
+ * consensus parameter for "onion-key-rotation-days". */
+#define MIN_ONION_KEY_GRACE_PERIOD_DAYS (1)
+
+/** Default grace period for acceptance of an onion key in days. */
+#define DEFAULT_ONION_KEY_GRACE_PERIOD_DAYS (7)
+
+/** How often we should check the network consensus if it is time to rotate or
+ * expire onion keys. */
+#define ONION_KEY_CONSENSUS_CHECK_INTERVAL (60*60)
+
+/** How often do we rotate TLS contexts? */
+#define MAX_SSL_KEY_LIFETIME_INTERNAL (2*60*60)
+
+/** How old do we allow a router to get before removing it
+ * from the router list? In seconds. */
+#define ROUTER_MAX_AGE (60*60*48)
+/** How old can a router get before we (as a server) will no longer
+ * consider it live? In seconds. */
+#define ROUTER_MAX_AGE_TO_PUBLISH (60*60*24)
+/** How old do we let a saved descriptor get before force-removing it? */
+#define OLD_ROUTER_DESC_MAX_AGE (60*60*24*5)
+
+/* Proxy client types */
+#define PROXY_NONE 0
+#define PROXY_CONNECT 1
+#define PROXY_SOCKS4 2
+#define PROXY_SOCKS5 3
+/* !!!! If there is ever a PROXY_* type over 3, we must grow the proxy_type
+ * field in or_connection_t */
+
+/* Pluggable transport proxy type. Don't use this in or_connection_t,
+ * instead use the actual underlying proxy type (see above).  */
+#define PROXY_PLUGGABLE 4
+
+/** How many circuits do we want simultaneously in-progress to handle
+ * a given stream? */
+#define MIN_CIRCUITS_HANDLING_STREAM 2
+
+/* These RELAY_COMMAND constants define values for relay cell commands, and
+* must match those defined in tor-spec.txt. */
+#define RELAY_COMMAND_BEGIN 1
+#define RELAY_COMMAND_DATA 2
+#define RELAY_COMMAND_END 3
+#define RELAY_COMMAND_CONNECTED 4
+#define RELAY_COMMAND_SENDME 5
+#define RELAY_COMMAND_EXTEND 6
+#define RELAY_COMMAND_EXTENDED 7
+#define RELAY_COMMAND_TRUNCATE 8
+#define RELAY_COMMAND_TRUNCATED 9
+#define RELAY_COMMAND_DROP 10
+#define RELAY_COMMAND_RESOLVE 11
+#define RELAY_COMMAND_RESOLVED 12
+#define RELAY_COMMAND_BEGIN_DIR 13
+#define RELAY_COMMAND_EXTEND2 14
+#define RELAY_COMMAND_EXTENDED2 15
+
+#define RELAY_COMMAND_ESTABLISH_INTRO 32
+#define RELAY_COMMAND_ESTABLISH_RENDEZVOUS 33
+#define RELAY_COMMAND_INTRODUCE1 34
+#define RELAY_COMMAND_INTRODUCE2 35
+#define RELAY_COMMAND_RENDEZVOUS1 36
+#define RELAY_COMMAND_RENDEZVOUS2 37
+#define RELAY_COMMAND_INTRO_ESTABLISHED 38
+#define RELAY_COMMAND_RENDEZVOUS_ESTABLISHED 39
+#define RELAY_COMMAND_INTRODUCE_ACK 40
+
+/* Reasons why an OR connection is closed. */
+#define END_OR_CONN_REASON_DONE           1
+#define END_OR_CONN_REASON_REFUSED        2 /* connection refused */
+#define END_OR_CONN_REASON_OR_IDENTITY    3
+#define END_OR_CONN_REASON_CONNRESET      4 /* connection reset by peer */
+#define END_OR_CONN_REASON_TIMEOUT        5
+#define END_OR_CONN_REASON_NO_ROUTE       6 /* no route to host/net */
+#define END_OR_CONN_REASON_IO_ERROR       7 /* read/write error */
+#define END_OR_CONN_REASON_RESOURCE_LIMIT 8 /* sockets, buffers, etc */
+#define END_OR_CONN_REASON_PT_MISSING     9 /* PT failed or not available */
+#define END_OR_CONN_REASON_MISC           10
+
+/* Reasons why we (or a remote OR) might close a stream. See tor-spec.txt for
+ * documentation of these.  The values must match. */
+#define END_STREAM_REASON_MISC 1
+#define END_STREAM_REASON_RESOLVEFAILED 2
+#define END_STREAM_REASON_CONNECTREFUSED 3
+#define END_STREAM_REASON_EXITPOLICY 4
+#define END_STREAM_REASON_DESTROY 5
+#define END_STREAM_REASON_DONE 6
+#define END_STREAM_REASON_TIMEOUT 7
+#define END_STREAM_REASON_NOROUTE 8
+#define END_STREAM_REASON_HIBERNATING 9
+#define END_STREAM_REASON_INTERNAL 10
+#define END_STREAM_REASON_RESOURCELIMIT 11
+#define END_STREAM_REASON_CONNRESET 12
+#define END_STREAM_REASON_TORPROTOCOL 13
+#define END_STREAM_REASON_NOTDIRECTORY 14
+#define END_STREAM_REASON_ENTRYPOLICY 15
+
+/* These high-numbered end reasons are not part of the official spec,
+ * and are not intended to be put in relay end cells. They are here
+ * to be more informative when sending back socks replies to the
+ * application. */
+/* XXXX 256 is no longer used; feel free to reuse it. */
+/** We were unable to attach the connection to any circuit at all. */
+/* XXXX the ways we use this one don't make a lot of sense. */
+#define END_STREAM_REASON_CANT_ATTACH 257
+/** We can't connect to any directories at all, so we killed our streams
+ * before they can time out. */
+#define END_STREAM_REASON_NET_UNREACHABLE 258
+/** This is a SOCKS connection, and the client used (or misused) the SOCKS
+ * protocol in a way we couldn't handle. */
+#define END_STREAM_REASON_SOCKSPROTOCOL 259
+/** This is a transparent proxy connection, but we can't extract the original
+ * target address:port. */
+#define END_STREAM_REASON_CANT_FETCH_ORIG_DEST 260
+/** This is a connection on the NATD port, and the destination IP:Port was
+ * either ill-formed or out-of-range. */
+#define END_STREAM_REASON_INVALID_NATD_DEST 261
+/** The target address is in a private network (like 127.0.0.1 or 10.0.0.1);
+ * you don't want to do that over a randomly chosen exit */
+#define END_STREAM_REASON_PRIVATE_ADDR 262
+/** This is an HTTP tunnel connection and the client used or misused HTTP in a
+ * way we can't handle.
+ */
+#define END_STREAM_REASON_HTTPPROTOCOL 263
+
+/** Bitwise-and this value with endreason to mask out all flags. */
+#define END_STREAM_REASON_MASK 511
+
+/** Bitwise-or this with the argument to control_event_stream_status
+ * to indicate that the reason came from an END cell. */
+#define END_STREAM_REASON_FLAG_REMOTE 512
+/** Bitwise-or this with the argument to control_event_stream_status
+ * to indicate that we already sent a CLOSED stream event. */
+#define END_STREAM_REASON_FLAG_ALREADY_SENT_CLOSED 1024
+/** Bitwise-or this with endreason to indicate that we already sent
+ * a socks reply, and no further reply needs to be sent from
+ * connection_mark_unattached_ap(). */
+#define END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED 2048
+
+/* 'type' values to use in RESOLVED cells.  Specified in tor-spec.txt. */
+#define RESOLVED_TYPE_HOSTNAME 0
+#define RESOLVED_TYPE_IPV4 4
+#define RESOLVED_TYPE_IPV6 6
+#define RESOLVED_TYPE_ERROR_TRANSIENT 0xF0
+#define RESOLVED_TYPE_ERROR 0xF1
+
+/* Negative reasons are internal: we never send them in a DESTROY or TRUNCATE
+ * call; they only go to the controller for tracking  */
+
+/* Closing introduction point that were opened in parallel. */
+#define END_CIRC_REASON_IP_NOW_REDUNDANT -4
+
+/** Our post-timeout circuit time measurement period expired.
+ * We must give up now */
+#define END_CIRC_REASON_MEASUREMENT_EXPIRED -3
+
+/** We couldn't build a path for this circuit. */
+#define END_CIRC_REASON_NOPATH          -2
+/** Catch-all "other" reason for closing origin circuits. */
+#define END_CIRC_AT_ORIGIN              -1
+
+/* Reasons why we (or a remote OR) might close a circuit. See tor-spec.txt
+ * section 5.4 for documentation of these. */
+#define END_CIRC_REASON_MIN_            0
+#define END_CIRC_REASON_NONE            0
+#define END_CIRC_REASON_TORPROTOCOL     1
+#define END_CIRC_REASON_INTERNAL        2
+#define END_CIRC_REASON_REQUESTED       3
+#define END_CIRC_REASON_HIBERNATING     4
+#define END_CIRC_REASON_RESOURCELIMIT   5
+#define END_CIRC_REASON_CONNECTFAILED   6
+#define END_CIRC_REASON_OR_IDENTITY     7
+#define END_CIRC_REASON_CHANNEL_CLOSED  8
+#define END_CIRC_REASON_FINISHED        9
+#define END_CIRC_REASON_TIMEOUT         10
+#define END_CIRC_REASON_DESTROYED       11
+#define END_CIRC_REASON_NOSUCHSERVICE   12
+#define END_CIRC_REASON_MAX_            12
+
+/** Bitwise-OR this with the argument to circuit_mark_for_close() or
+ * control_event_circuit_status() to indicate that the reason was
+ * passed through from a destroy or truncate cell. */
+#define END_CIRC_REASON_FLAG_REMOTE     512
+
+/** Length of 'y' portion of 'y.onion' URL. */
+#define REND_SERVICE_ID_LEN_BASE32 16
+
+/** Length of 'y.onion' including '.onion' URL. */
+#define REND_SERVICE_ADDRESS_LEN (16+1+5)
+
+/** Length of a binary-encoded rendezvous service ID. */
+#define REND_SERVICE_ID_LEN 10
+
+/** Time period for which a v2 descriptor will be valid. */
+#define REND_TIME_PERIOD_V2_DESC_VALIDITY (24*60*60)
+
+/** Time period within which two sets of v2 descriptors will be uploaded in
+ * parallel. */
+#define REND_TIME_PERIOD_OVERLAPPING_V2_DESCS (60*60)
+
+/** Number of non-consecutive replicas (i.e. distributed somewhere
+ * in the ring) for a descriptor. */
+#define REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS 2
+
+/** Number of consecutive replicas for a descriptor. */
+#define REND_NUMBER_OF_CONSECUTIVE_REPLICAS 3
+
+/** Length of v2 descriptor ID (32 base32 chars = 160 bits). */
+#define REND_DESC_ID_V2_LEN_BASE32 BASE32_DIGEST_LEN
+
+/** Length of the base32-encoded secret ID part of versioned hidden service
+ * descriptors. */
+#define REND_SECRET_ID_PART_LEN_BASE32 BASE32_DIGEST_LEN
+
+/** Length of the base32-encoded hash of an introduction point's
+ * identity key. */
+#define REND_INTRO_POINT_ID_LEN_BASE32 BASE32_DIGEST_LEN
+
+/** Length of the descriptor cookie that is used for client authorization
+ * to hidden services. */
+#define REND_DESC_COOKIE_LEN 16
+
+/** Length of the base64-encoded descriptor cookie that is used for
+ * exchanging client authorization between hidden service and client. */
+#define REND_DESC_COOKIE_LEN_BASE64 22
+
+/** Length of client identifier in encrypted introduction points for hidden
+ * service authorization type 'basic'. */
+#define REND_BASIC_AUTH_CLIENT_ID_LEN 4
+
+/** Multiple of the number of clients to which the real number of clients
+ * is padded with fake clients for hidden service authorization type
+ * 'basic'. */
+#define REND_BASIC_AUTH_CLIENT_MULTIPLE 16
+
+/** Length of client entry consisting of client identifier and encrypted
+ * session key for hidden service authorization type 'basic'. */
+#define REND_BASIC_AUTH_CLIENT_ENTRY_LEN (REND_BASIC_AUTH_CLIENT_ID_LEN \
+                                          + CIPHER_KEY_LEN)
+
+/** Maximum size of v2 hidden service descriptors. */
+#define REND_DESC_MAX_SIZE (20 * 1024)
+
+/** Legal characters for use in authorized client names for a hidden
+ * service. */
+#define REND_LEGAL_CLIENTNAME_CHARACTERS \
+  "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+-_"
+
+/** Maximum length of authorized client names for a hidden service. */
+#define REND_CLIENTNAME_MAX_LEN 16
+
+/** Length of the rendezvous cookie that is used to connect circuits at the
+ * rendezvous point. */
+#define REND_COOKIE_LEN DIGEST_LEN
+
+/** Client authorization type that a hidden service performs. */
+typedef enum rend_auth_type_t {
+  REND_NO_AUTH      = 0,
+  REND_BASIC_AUTH   = 1,
+  REND_STEALTH_AUTH = 2,
+} rend_auth_type_t;
+
+/** Client-side configuration of authorization for a hidden service. */
+typedef struct rend_service_authorization_t {
+  uint8_t descriptor_cookie[REND_DESC_COOKIE_LEN];
+  char onion_address[REND_SERVICE_ADDRESS_LEN+1];
+  rend_auth_type_t auth_type;
+} rend_service_authorization_t;
+
+/** Client- and server-side data that is used for hidden service connection
+ * establishment. Not all fields contain data depending on where this struct
+ * is used. */
+typedef struct rend_data_t {
+  /* Hidden service protocol version of this base object. */
+  uint32_t version;
+
+  /** List of HSDir fingerprints on which this request has been sent to. This
+   * contains binary identity digest of the directory of size DIGEST_LEN. */
+  smartlist_t *hsdirs_fp;
+
+  /** Rendezvous cookie used by both, client and service. */
+  char rend_cookie[REND_COOKIE_LEN];
+
+  /** Number of streams associated with this rendezvous circuit. */
+  int nr_streams;
+} rend_data_t;
+
+typedef struct rend_data_v2_t {
+  /* Rendezvous base data. */
+  rend_data_t base_;
+
+  /** Onion address (without the .onion part) that a client requests. */
+  char onion_address[REND_SERVICE_ID_LEN_BASE32+1];
+
+  /** Descriptor ID for each replicas computed from the onion address. If
+   * the onion address is empty, this array MUST be empty. We keep them so
+   * we know when to purge our entry in the last hsdir request table. */
+  char descriptor_id[REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS][DIGEST_LEN];
+
+  /** (Optional) descriptor cookie that is used by a client. */
+  char descriptor_cookie[REND_DESC_COOKIE_LEN];
+
+  /** Authorization type for accessing a service used by a client. */
+  rend_auth_type_t auth_type;
+
+  /** Descriptor ID for a client request. The control port command HSFETCH
+   * uses this. It's set if the descriptor query should only use this
+   * descriptor ID. */
+  char desc_id_fetch[DIGEST_LEN];
+
+  /** Hash of the hidden service's PK used by a service. */
+  char rend_pk_digest[DIGEST_LEN];
+} rend_data_v2_t;
+
+/* From a base rend_data_t object <b>d</d>, return the v2 object. */
+static inline
+rend_data_v2_t *TO_REND_DATA_V2(const rend_data_t *d)
+{
+  tor_assert(d);
+  tor_assert(d->version == 2);
+  return DOWNCAST(rend_data_v2_t, d);
+}
+
+/* Stub because we can't include hs_ident.h. */
+struct hs_ident_edge_conn_t;
+struct hs_ident_dir_conn_t;
+struct hs_ident_circuit_t;
+
+typedef struct hsdir_index_t hsdir_index_t;
+
+/** Time interval for tracking replays of DH public keys received in
+ * INTRODUCE2 cells.  Used only to avoid launching multiple
+ * simultaneous attempts to connect to the same rendezvous point. */
+#define REND_REPLAY_TIME_INTERVAL (5 * 60)
+
+/** Used to indicate which way a cell is going on a circuit. */
+typedef enum {
+  CELL_DIRECTION_IN=1, /**< The cell is moving towards the origin. */
+  CELL_DIRECTION_OUT=2, /**< The cell is moving away from the origin. */
+} cell_direction_t;
+
+/**
+ * An enum to allow us to specify which channel in a circuit
+ * we're interested in.
+ *
+ * This is needed because our data structures and other fields
+ * for channel delivery are disassociated from the channel.
+ */
+typedef enum {
+  CIRCUIT_N_CHAN = 0,
+  CIRCUIT_P_CHAN = 1
+} circuit_channel_direction_t;
+
+/** Initial value for both sides of a circuit transmission window when the
+ * circuit is initialized.  Measured in cells. */
+#define CIRCWINDOW_START 1000
+#define CIRCWINDOW_START_MIN 100
+#define CIRCWINDOW_START_MAX 1000
+/** Amount to increment a circuit window when we get a circuit SENDME. */
+#define CIRCWINDOW_INCREMENT 100
+/** Initial value on both sides of a stream transmission window when the
+ * stream is initialized.  Measured in cells. */
+#define STREAMWINDOW_START 500
+#define STREAMWINDOW_START_MAX 500
+/** Amount to increment a stream window when we get a stream SENDME. */
+#define STREAMWINDOW_INCREMENT 50
+
+/** Maximum number of queued cells on a circuit for which we are the
+ * midpoint before we give up and kill it.  This must be >= circwindow
+ * to avoid killing innocent circuits, and >= circwindow*2 to give
+ * leaky-pipe a chance of working someday. The ORCIRC_MAX_MIDDLE_KILL_THRESH
+ * ratio controls the margin of error between emitting a warning and
+ * killing the circuit.
+ */
+#define ORCIRC_MAX_MIDDLE_CELLS (CIRCWINDOW_START_MAX*2)
+/** Ratio of hard (circuit kill) to soft (warning) thresholds for the
+ * ORCIRC_MAX_MIDDLE_CELLS tests.
+ */
+#define ORCIRC_MAX_MIDDLE_KILL_THRESH (1.1f)
+
+/* Cell commands.  These values are defined in tor-spec.txt. */
+#define CELL_PADDING 0
+#define CELL_CREATE 1
+#define CELL_CREATED 2
+#define CELL_RELAY 3
+#define CELL_DESTROY 4
+#define CELL_CREATE_FAST 5
+#define CELL_CREATED_FAST 6
+#define CELL_VERSIONS 7
+#define CELL_NETINFO 8
+#define CELL_RELAY_EARLY 9
+#define CELL_CREATE2 10
+#define CELL_CREATED2 11
+#define CELL_PADDING_NEGOTIATE 12
+
+#define CELL_VPADDING 128
+#define CELL_CERTS 129
+#define CELL_AUTH_CHALLENGE 130
+#define CELL_AUTHENTICATE 131
+#define CELL_AUTHORIZE 132
+#define CELL_COMMAND_MAX_ 132
+
+/** How long to test reachability before complaining to the user. */
+#define TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT (20*60)
+
+/** Legal characters in a nickname. */
+#define LEGAL_NICKNAME_CHARACTERS \
+  "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+
+/** Name to use in client TLS certificates if no nickname is given. Once
+ * Tor 0.1.2.x is obsolete, we can remove this. */
+#define DEFAULT_CLIENT_NICKNAME "client"
+
+/** Name chosen by routers that don't configure nicknames */
+#define UNNAMED_ROUTER_NICKNAME "Unnamed"
+
+/** Number of bytes in a SOCKS4 header. */
+#define SOCKS4_NETWORK_LEN 8
+
+/*
+ * Relay payload:
+ *         Relay command           [1 byte]
+ *         Recognized              [2 bytes]
+ *         Stream ID               [2 bytes]
+ *         Partial SHA-1           [4 bytes]
+ *         Length                  [2 bytes]
+ *         Relay payload           [498 bytes]
+ */
+
+/** Number of bytes in a cell, minus cell header. */
+#define CELL_PAYLOAD_SIZE 509
+/** Number of bytes in a cell transmitted over the network, in the longest
+ * form */
+#define CELL_MAX_NETWORK_SIZE 514
+
+/** Maximum length of a header on a variable-length cell. */
+#define VAR_CELL_MAX_HEADER_SIZE 7
+
+static int get_cell_network_size(int wide_circ_ids);
+static inline int get_cell_network_size(int wide_circ_ids)
+{
+  return wide_circ_ids ? CELL_MAX_NETWORK_SIZE : CELL_MAX_NETWORK_SIZE - 2;
+}
+static int get_var_cell_header_size(int wide_circ_ids);
+static inline int get_var_cell_header_size(int wide_circ_ids)
+{
+  return wide_circ_ids ? VAR_CELL_MAX_HEADER_SIZE :
+    VAR_CELL_MAX_HEADER_SIZE - 2;
+}
+static int get_circ_id_size(int wide_circ_ids);
+static inline int get_circ_id_size(int wide_circ_ids)
+{
+  return wide_circ_ids ? 4 : 2;
+}
+
+/** Number of bytes in a relay cell's header (not including general cell
+ * header). */
+#define RELAY_HEADER_SIZE (1+2+2+4+2)
+/** Largest number of bytes that can fit in a relay cell payload. */
+#define RELAY_PAYLOAD_SIZE (CELL_PAYLOAD_SIZE-RELAY_HEADER_SIZE)
+
+/** Identifies a circuit on an or_connection */
+typedef uint32_t circid_t;
+/** Identifies a stream on a circuit */
+typedef uint16_t streamid_t;
+
+/* channel_t typedef; struct channel_s is in channel.h */
+
+typedef struct channel_s channel_t;
+
+/* channel_listener_t typedef; struct channel_listener_s is in channel.h */
+
+typedef struct channel_listener_s channel_listener_t;
+
+/* TLS channel stuff */
+
+typedef struct channel_tls_s channel_tls_t;
+
+/* circuitmux_t typedef; struct circuitmux_s is in circuitmux.h */
+
+typedef struct circuitmux_s circuitmux_t;
+
+typedef struct cell_t cell_t;
+typedef struct var_cell_t var_cell_t;
+typedef struct packed_cell_t packed_cell_t;
+typedef struct cell_queue_t cell_queue_t;
+typedef struct destroy_cell_t destroy_cell_t;
+typedef struct destroy_cell_queue_t destroy_cell_queue_t;
+typedef struct ext_or_cmd_t ext_or_cmd_t;
+
+/** Beginning of a RELAY cell payload. */
+typedef struct {
+  uint8_t command; /**< The end-to-end relay command. */
+  uint16_t recognized; /**< Used to tell whether cell is for us. */
+  streamid_t stream_id; /**< Which stream is this cell associated with? */
+  char integrity[4]; /**< Used to tell whether cell is corrupted. */
+  uint16_t length; /**< How long is the payload body? */
+} relay_header_t;
+
+typedef struct socks_request_t socks_request_t;
+typedef struct entry_port_cfg_t entry_port_cfg_t;
+typedef struct server_port_cfg_t server_port_cfg_t;
+
+/** Minimum length of the random part of an AUTH_CHALLENGE cell. */
+#define OR_AUTH_CHALLENGE_LEN 32
+
+/**
+ * @name Certificate types for CERTS cells.
+ *
+ * These values are defined by the protocol, and affect how an X509
+ * certificate in a CERTS cell is interpreted and used.
+ *
+ * @{ */
+/** A certificate that authenticates a TLS link key.  The subject key
+ * must match the key used in the TLS handshake; it must be signed by
+ * the identity key. */
+#define OR_CERT_TYPE_TLS_LINK 1
+/** A self-signed identity certificate. The subject key must be a
+ * 1024-bit RSA key. */
+#define OR_CERT_TYPE_ID_1024 2
+/** A certificate that authenticates a key used in an AUTHENTICATE cell
+ * in the v3 handshake.  The subject key must be a 1024-bit RSA key; it
+ * must be signed by the identity key */
+#define OR_CERT_TYPE_AUTH_1024 3
+/* DOCDOC */
+#define OR_CERT_TYPE_RSA_ED_CROSSCERT 7
+/**@}*/
+
+/** The first supported type of AUTHENTICATE cell.  It contains
+ * a bunch of structures signed with an RSA1024 key.  The signed
+ * structures include a HMAC using negotiated TLS secrets, and a digest
+ * of all cells sent or received before the AUTHENTICATE cell (including
+ * the random server-generated AUTH_CHALLENGE cell).
+ */
+#define AUTHTYPE_RSA_SHA256_TLSSECRET 1
+/** As AUTHTYPE_RSA_SHA256_TLSSECRET, but instead of using the
+ * negotiated TLS secrets, uses exported keying material from the TLS
+ * session as described in RFC 5705.
+ *
+ * Not used by today's tors, since everything that supports this
+ * also supports ED25519_SHA256_5705, which is better.
+ **/
+#define AUTHTYPE_RSA_SHA256_RFC5705 2
+/** As AUTHTYPE_RSA_SHA256_RFC5705, but uses an Ed25519 identity key to
+ * authenticate.  */
+#define AUTHTYPE_ED25519_SHA256_RFC5705 3
+/*
+ * NOTE: authchallenge_type_is_better() relies on these AUTHTYPE codes
+ * being sorted in order of preference.  If we someday add one with
+ * a higher numerical value that we don't like as much, we should revise
+ * authchallenge_type_is_better().
+ */
+
+/** The length of the part of the AUTHENTICATE cell body that the client and
+ * server can generate independently (when using RSA_SHA256_TLSSECRET). It
+ * contains everything except the client's timestamp, the client's randomly
+ * generated nonce, and the signature. */
+#define V3_AUTH_FIXED_PART_LEN (8+(32*6))
+/** The length of the part of the AUTHENTICATE cell body that the client
+ * signs. */
+#define V3_AUTH_BODY_LEN (V3_AUTH_FIXED_PART_LEN + 8 + 16)
+
+typedef struct or_handshake_certs_t or_handshake_certs_t;
+typedef struct or_handshake_state_t or_handshake_state_t;
+
+/** Length of Extended ORPort connection identifier. */
+#define EXT_OR_CONN_ID_LEN DIGEST_LEN /* 20 */
+/*
+ * OR_CONN_HIGHWATER and OR_CONN_LOWWATER moved from connection_or.c so
+ * channeltls.c can see them too.
+ */
+
+/** When adding cells to an OR connection's outbuf, keep adding until the
+ * outbuf is at least this long, or we run out of cells. */
+#define OR_CONN_HIGHWATER (32*1024)
+
+/** Add cells to an OR connection's outbuf whenever the outbuf's data length
+ * drops below this size. */
+#define OR_CONN_LOWWATER (16*1024)
+
+typedef struct connection_t connection_t;
+typedef struct control_connection_t control_connection_t;
+typedef struct dir_connection_t dir_connection_t;
+typedef struct edge_connection_t edge_connection_t;
+typedef struct entry_connection_t entry_connection_t;
+typedef struct listener_connection_t listener_connection_t;
+typedef struct or_connection_t or_connection_t;
+
+/** Cast a connection_t subtype pointer to a connection_t **/
+#define TO_CONN(c) (&(((c)->base_)))
+
+/** Cast a entry_connection_t subtype pointer to a connection_t **/
+#define ENTRY_TO_CONN(c) (TO_CONN(ENTRY_TO_EDGE_CONN(c)))
+
+typedef struct addr_policy_t addr_policy_t;
+
+typedef struct cached_dir_t cached_dir_t;
+
+/** Enum used to remember where a signed_descriptor_t is stored and how to
+ * manage the memory for signed_descriptor_body.  */
+typedef enum {
+  /** The descriptor isn't stored on disk at all: the copy in memory is
+   * canonical; the saved_offset field is meaningless. */
+  SAVED_NOWHERE=0,
+  /** The descriptor is stored in the cached_routers file: the
+   * signed_descriptor_body is meaningless; the signed_descriptor_len and
+   * saved_offset are used to index into the mmaped cache file. */
+  SAVED_IN_CACHE,
+  /** The descriptor is stored in the cached_routers.new file: the
+   * signed_descriptor_body and saved_offset fields are both set. */
+  /* FFFF (We could also mmap the file and grow the mmap as needed, or
+   * lazy-load the descriptor text by using seek and read.  We don't, for
+   * now.)
+   */
+  SAVED_IN_JOURNAL
+} saved_location_t;
+#define saved_location_bitfield_t ENUM_BF(saved_location_t)
+
+/** Enumeration: what directory object is being downloaded?
+ * This determines which schedule is selected to perform the download. */
+typedef enum {
+  DL_SCHED_GENERIC = 0,
+  DL_SCHED_CONSENSUS = 1,
+  DL_SCHED_BRIDGE = 2,
+} download_schedule_t;
+#define download_schedule_bitfield_t ENUM_BF(download_schedule_t)
+
+/** Enumeration: is the download schedule for downloading from an authority,
+ * or from any available directory mirror?
+ * During bootstrap, "any" means a fallback (or an authority, if there
+ * are no fallbacks).
+ * When we have a valid consensus, "any" means any directory server. */
+typedef enum {
+  DL_WANT_ANY_DIRSERVER = 0,
+  DL_WANT_AUTHORITY = 1,
+} download_want_authority_t;
+#define download_want_authority_bitfield_t \
+                                        ENUM_BF(download_want_authority_t)
+
+/** Enumeration: do we want to increment the schedule position each time a
+ * connection is attempted (these attempts can be concurrent), or do we want
+ * to increment the schedule position after a connection fails? */
+typedef enum {
+  DL_SCHED_INCREMENT_FAILURE = 0,
+  DL_SCHED_INCREMENT_ATTEMPT = 1,
+} download_schedule_increment_t;
+#define download_schedule_increment_bitfield_t \
+                                        ENUM_BF(download_schedule_increment_t)
+
+typedef struct download_status_t download_status_t;
+
+/** If n_download_failures is this high, the download can never happen. */
+#define IMPOSSIBLE_TO_DOWNLOAD 255
+
+/** The max size we expect router descriptor annotations we create to
+ * be. We'll accept larger ones if we see them on disk, but we won't
+ * create any that are larger than this. */
+#define ROUTER_ANNOTATION_BUF_LEN 256
+
+typedef struct signed_descriptor_t signed_descriptor_t;
+
+/** Flags used to summarize the declared protocol versions of a relay,
+ * so we don't need to parse them again and again. */
+typedef struct protover_summary_flags_t {
+  /** True iff we have a proto line for this router, or a versions line
+   * from which we could infer the protocols. */
+  unsigned int protocols_known:1;
+
+  /** True iff this router has a version or protocol list that allows it to
+   * accept EXTEND2 cells. This requires Relay=2. */
+  unsigned int supports_extend2_cells:1;
+
+  /** True iff this router has a protocol list that allows it to negotiate
+   * ed25519 identity keys on a link handshake with us. This
+   * requires LinkAuth=3. */
+  unsigned int supports_ed25519_link_handshake_compat:1;
+
+  /** True iff this router has a protocol list that allows it to negotiate
+   * ed25519 identity keys on a link handshake, at all. This requires some
+   * LinkAuth=X for X >= 3. */
+  unsigned int supports_ed25519_link_handshake_any:1;
+
+  /** True iff this router has a protocol list that allows it to be an
+   * introduction point supporting ed25519 authentication key which is part of
+   * the v3 protocol detailed in proposal 224. This requires HSIntro=4. */
+  unsigned int supports_ed25519_hs_intro : 1;
+
+  /** True iff this router has a protocol list that allows it to be an hidden
+   * service directory supporting version 3 as seen in proposal 224. This
+   * requires HSDir=2. */
+  unsigned int supports_v3_hsdir : 1;
+
+  /** True iff this router has a protocol list that allows it to be an hidden
+   * service rendezvous point supporting version 3 as seen in proposal 224.
+   * This requires HSRend=2. */
+  unsigned int supports_v3_rendezvous_point: 1;
+} protover_summary_flags_t;
+
+typedef struct routerinfo_t routerinfo_t;
+typedef struct extrainfo_t extrainfo_t;
+typedef struct routerstatus_t routerstatus_t;
+
+typedef struct microdesc_t microdesc_t;
+typedef struct node_t node_t;
+typedef struct vote_microdesc_hash_t vote_microdesc_hash_t;
+typedef struct vote_routerstatus_t vote_routerstatus_t;
+typedef struct document_signature_t document_signature_t;
+typedef struct networkstatus_voter_info_t networkstatus_voter_info_t;
+typedef struct networkstatus_sr_info_t networkstatus_sr_info_t;
+
+/** Enumerates recognized flavors of a consensus networkstatus document.  All
+ * flavors of a consensus are generated from the same set of votes, but they
+ * present different types information to different versions of Tor. */
+typedef enum {
+  FLAV_NS = 0,
+  FLAV_MICRODESC = 1,
+} consensus_flavor_t;
+
+/** How many different consensus flavors are there? */
+#define N_CONSENSUS_FLAVORS ((int)(FLAV_MICRODESC)+1)
+
+typedef struct networkstatus_t networkstatus_t;
+typedef struct ns_detached_signatures_t ns_detached_signatures_t;
+typedef struct desc_store_t desc_store_t;
+typedef struct routerlist_t routerlist_t;
+typedef struct extend_info_t extend_info_t;
+typedef struct authority_cert_t authority_cert_t;
+
+/** Bitfield enum type listing types of information that directory authorities
+ * can be authoritative about, and that directory caches may or may not cache.
+ *
+ * Note that the granularity here is based on authority granularity and on
+ * cache capabilities.  Thus, one particular bit may correspond in practice to
+ * a few types of directory info, so long as every authority that pronounces
+ * officially about one of the types prounounces officially about all of them,
+ * and so long as every cache that caches one of them caches all of them.
+ */
+typedef enum {
+  NO_DIRINFO      = 0,
+  /** Serves/signs v3 directory information: votes, consensuses, certs */
+  V3_DIRINFO      = 1 << 2,
+  /** Serves bridge descriptors. */
+  BRIDGE_DIRINFO  = 1 << 4,
+  /** Serves extrainfo documents. */
+  EXTRAINFO_DIRINFO=1 << 5,
+  /** Serves microdescriptors. */
+  MICRODESC_DIRINFO=1 << 6,
+} dirinfo_type_t;
+
+#define ALL_DIRINFO ((dirinfo_type_t)((1<<7)-1))
+
+#define ONION_HANDSHAKE_TYPE_TAP  0x0000
+#define ONION_HANDSHAKE_TYPE_FAST 0x0001
+#define ONION_HANDSHAKE_TYPE_NTOR 0x0002
+#define MAX_ONION_HANDSHAKE_TYPE 0x0002
+
+typedef struct onion_handshake_state_t onion_handshake_state_t;
+typedef struct relay_crypto_t relay_crypto_t;
+typedef struct crypt_path_t crypt_path_t;
+typedef struct crypt_path_reference_t crypt_path_reference_t;
+
+#define CPATH_KEY_MATERIAL_LEN (20*2+16*2)
+
+typedef struct cpath_build_state_t cpath_build_state_t;
+
+struct create_cell_t;
+
+/** Entry in the cell stats list of a circuit; used only if CELL_STATS
+ * events are enabled. */
+typedef struct testing_cell_stats_entry_t {
+  uint8_t command; /**< cell command number. */
+  /** Waiting time in centiseconds if this event is for a removed cell,
+   * or 0 if this event is for adding a cell to the queue.  22 bits can
+   * store more than 11 hours, enough to assume that a circuit with this
+   * delay would long have been closed. */
+  unsigned int waiting_time:22;
+  unsigned int removed:1; /**< 0 for added to, 1 for removed from queue. */
+  unsigned int exitward:1; /**< 0 for app-ward, 1 for exit-ward. */
+} testing_cell_stats_entry_t;
+
+typedef struct circuit_t circuit_t;
+typedef struct origin_circuit_t origin_circuit_t;
+typedef struct or_circuit_t or_circuit_t;
+
+/** Largest number of relay_early cells that we can send on a given
+ * circuit. */
+#define MAX_RELAY_EARLY_CELLS_PER_CIRCUIT 8
+
+typedef enum path_state_t path_state_t;
+#define path_state_bitfield_t ENUM_BF(path_state_t)
+
+#if REND_COOKIE_LEN != DIGEST_LEN
+#error "The REND_TOKEN_LEN macro assumes REND_COOKIE_LEN == DIGEST_LEN"
+#endif
+#define REND_TOKEN_LEN DIGEST_LEN
+
+/** Convert a circuit subtype to a circuit_t. */
+#define TO_CIRCUIT(x)  (&((x)->base_))
+
+/** @name Isolation flags
+
+    Ways to isolate client streams
+
+    @{
+*/
+/** Isolate based on destination port */
+#define ISO_DESTPORT    (1u<<0)
+/** Isolate based on destination address */
+#define ISO_DESTADDR    (1u<<1)
+/** Isolate based on SOCKS authentication */
+#define ISO_SOCKSAUTH   (1u<<2)
+/** Isolate based on client protocol choice */
+#define ISO_CLIENTPROTO (1u<<3)
+/** Isolate based on client address */
+#define ISO_CLIENTADDR  (1u<<4)
+/** Isolate based on session group (always on). */
+#define ISO_SESSIONGRP  (1u<<5)
+/** Isolate based on newnym epoch (always on). */
+#define ISO_NYM_EPOCH   (1u<<6)
+/** Isolate all streams (Internal only). */
+#define ISO_STREAM      (1u<<7)
+/**@}*/
+
+/** Default isolation level for ports. */
+#define ISO_DEFAULT (ISO_CLIENTADDR|ISO_SOCKSAUTH|ISO_SESSIONGRP|ISO_NYM_EPOCH)
+
+/** Indicates that we haven't yet set a session group on a port_cfg_t. */
+#define SESSION_GROUP_UNSET -1
+/** Session group reserved for directory connections */
+#define SESSION_GROUP_DIRCONN -2
+/** Session group reserved for resolve requests launched by a controller */
+#define SESSION_GROUP_CONTROL_RESOLVE -3
+/** First automatically allocated session group number */
+#define SESSION_GROUP_FIRST_AUTO -4
+
+typedef struct port_cfg_t port_cfg_t;
+typedef struct routerset_t routerset_t;
+
+/** A magic value for the (Socks|OR|...)Port options below, telling Tor
+ * to pick its own port. */
+#define CFG_AUTO_PORT 0xc4005e
+
+typedef struct or_options_t or_options_t;
+
+#define LOG_PROTOCOL_WARN (get_protocol_warning_severity_level())
+
+typedef struct or_state_t or_state_t;
+
+#define MAX_SOCKS_ADDR_LEN 256
+
+/********************************* circuitbuild.c **********************/
+
+/** How many hops does a general-purpose circuit have by default? */
+#define DEFAULT_ROUTE_LEN 3
+
+/* Circuit Build Timeout "public" structures. */
+
+/** Precision multiplier for the Bw weights */
+#define BW_WEIGHT_SCALE   10000
+#define BW_MIN_WEIGHT_SCALE 1
+#define BW_MAX_WEIGHT_SCALE INT32_MAX
+
+typedef struct circuit_build_times_s circuit_build_times_t;
+
+/********************************* config.c ***************************/
+
+/********************************* connection_edge.c *************************/
+
+/** Enumerates possible origins of a client-side address mapping. */
+typedef enum {
+  /** We're remapping this address because the controller told us to. */
+  ADDRMAPSRC_CONTROLLER,
+  /** We're remapping this address because of an AutomapHostsOnResolve
+   * configuration. */
+  ADDRMAPSRC_AUTOMAP,
+  /** We're remapping this address because our configuration (via torrc, the
+   * command line, or a SETCONF command) told us to. */
+  ADDRMAPSRC_TORRC,
+  /** We're remapping this address because we have TrackHostExit configured,
+   * and we want to remember to use the same exit next time. */
+  ADDRMAPSRC_TRACKEXIT,
+  /** We're remapping this address because we got a DNS resolution from a
+   * Tor server that told us what its value was. */
+  ADDRMAPSRC_DNS,
+
+  /** No remapping has occurred.  This isn't a possible value for an
+   * addrmap_entry_t; it's used as a null value when we need to answer "Why
+   * did this remapping happen." */
+  ADDRMAPSRC_NONE
+} addressmap_entry_source_t;
+#define addressmap_entry_source_bitfield_t ENUM_BF(addressmap_entry_source_t)
+
+#define WRITE_STATS_INTERVAL (24*60*60)
+
+/********************************* dirvote.c ************************/
+
+typedef struct vote_timing_t vote_timing_t;
+
+/********************************* microdesc.c *************************/
+
+typedef struct microdesc_cache_t microdesc_cache_t;
+
+/********************************* rendcommon.c ***************************/
+
+typedef struct rend_authorized_client_t rend_authorized_client_t;
+typedef struct rend_encoded_v2_service_descriptor_t
+               rend_encoded_v2_service_descriptor_t;
+
+/** The maximum number of non-circuit-build-timeout failures a hidden
+ * service client will tolerate while trying to build a circuit to an
+ * introduction point.  See also rend_intro_point_t.unreachable_count. */
+#define MAX_INTRO_POINT_REACHABILITY_FAILURES 5
+
+/** The minimum and maximum number of distinct INTRODUCE2 cells which a
+ * hidden service's introduction point will receive before it begins to
+ * expire. */
+#define INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS 16384
+/* Double the minimum value so the interval is [min, min * 2]. */
+#define INTRO_POINT_MAX_LIFETIME_INTRODUCTIONS \
+  (INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS * 2)
+
+/** The minimum number of seconds that an introduction point will last
+ * before expiring due to old age.  (If it receives
+ * INTRO_POINT_LIFETIME_INTRODUCTIONS INTRODUCE2 cells, it may expire
+ * sooner.)
+ *
+ * XXX Should this be configurable? */
+#define INTRO_POINT_LIFETIME_MIN_SECONDS (18*60*60)
+/** The maximum number of seconds that an introduction point will last
+ * before expiring due to old age.
+ *
+ * XXX Should this be configurable? */
+#define INTRO_POINT_LIFETIME_MAX_SECONDS (24*60*60)
+
+/** The maximum number of circuit creation retry we do to an intro point
+ * before giving up. We try to reuse intro point that fails during their
+ * lifetime so this is a hard limit on the amount of time we do that. */
+#define MAX_INTRO_POINT_CIRCUIT_RETRIES 3
+
+typedef struct rend_intro_point_t rend_intro_point_t;
+typedef struct rend_service_descriptor_t rend_service_descriptor_t;
+
+/********************************* routerlist.c ***************************/
+
+typedef struct dir_server_t dir_server_t;
+
+#define RELAY_REQUIRED_MIN_BANDWIDTH (75*1024)
+#define BRIDGE_REQUIRED_MIN_BANDWIDTH (50*1024)
+
+#define ROUTER_MAX_DECLARED_BANDWIDTH INT32_MAX
+
+typedef struct tor_version_t tor_version_t;
+
+#endif /* !defined(TOR_OR_H) */
diff --git a/src/core/or/or_circuit_st.h b/src/core/or/or_circuit_st.h
new file mode 100644
index 0000000000..6b6feb9d89
--- /dev/null
+++ b/src/core/or/or_circuit_st.h
@@ -0,0 +1,80 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef OR_CIRCUIT_ST_H
+#define OR_CIRCUIT_ST_H
+
+#include "core/or/or.h"
+
+#include "core/or/circuit_st.h"
+#include "core/or/crypt_path_st.h"
+
+struct onion_queue_t;
+
+/** An or_circuit_t holds information needed to implement a circuit at an
+ * OR. */
+struct or_circuit_t {
+  circuit_t base_;
+
+  /** Pointer to an entry on the onion queue, if this circuit is waiting for a
+   * chance to give an onionskin to a cpuworker. Used only in onion.c */
+  struct onion_queue_t *onionqueue_entry;
+  /** Pointer to a workqueue entry, if this circuit has given an onionskin to
+   * a cpuworker and is waiting for a response. Used to decide whether it is
+   * safe to free a circuit or if it is still in use by a cpuworker. */
+  struct workqueue_entry_s *workqueue_entry;
+
+  /** The circuit_id used in the previous (backward) hop of this circuit. */
+  circid_t p_circ_id;
+  /** Queue of cells waiting to be transmitted on p_conn. */
+  cell_queue_t p_chan_cells;
+  /** The channel that is previous in this circuit. */
+  channel_t *p_chan;
+  /**
+   * Circuit mux associated with p_chan to which this circuit is attached;
+   * NULL if we have no p_chan.
+   */
+  circuitmux_t *p_mux;
+  /** Linked list of Exit streams associated with this circuit. */
+  edge_connection_t *n_streams;
+  /** Linked list of Exit streams associated with this circuit that are
+   * still being resolved. */
+  edge_connection_t *resolving_streams;
+
+  /** Cryptographic state used for encrypting and authenticating relay
+   * cells to and from this hop. */
+  relay_crypto_t crypto;
+
+  /** Points to spliced circuit if purpose is REND_ESTABLISHED, and circuit
+   * is not marked for close. */
+  struct or_circuit_t *rend_splice;
+
+  /** Stores KH for the handshake. */
+  char rend_circ_nonce[DIGEST_LEN];/* KH in tor-spec.txt */
+
+  /** How many more relay_early cells can we send on this circuit, according
+   * to the specification? */
+  unsigned int remaining_relay_early_cells : 4;
+
+  /* We have already received an INTRODUCE1 cell on this circuit. */
+  unsigned int already_received_introduce1 : 1;
+
+  /** If set, this circuit carries HS traffic. Consider it in any HS
+   *  statistics. */
+  unsigned int circuit_carries_hs_traffic_stats : 1;
+
+  /** Number of cells that were removed from circuit queue; reset every
+   * time when writing buffer stats to disk. */
+  uint32_t processed_cells;
+
+  /** Total time in milliseconds that cells spent in both app-ward and
+   * exit-ward queues of this circuit; reset every time when writing
+   * buffer stats to disk. */
+  uint64_t total_cell_waiting_time;
+};
+
+#endif
+
diff --git a/src/core/or/or_connection_st.h b/src/core/or/or_connection_st.h
new file mode 100644
index 0000000000..d5db5e8694
--- /dev/null
+++ b/src/core/or/or_connection_st.h
@@ -0,0 +1,92 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef OR_CONNECTION_ST_H
+#define OR_CONNECTION_ST_H
+
+#include "core/or/connection_st.h"
+#include "lib/evloop/token_bucket.h"
+
+struct tor_tls_t;
+
+/** Subtype of connection_t for an "OR connection" -- that is, one that speaks
+ * cells over TLS. */
+struct or_connection_t {
+  connection_t base_;
+
+  /** Hash of the public RSA key for the other side's identity key, or zeroes
+   * if the other side hasn't shown us a valid identity key. */
+  char identity_digest[DIGEST_LEN];
+
+  /** Extended ORPort connection identifier. */
+  char *ext_or_conn_id;
+  /** This is the ClientHash value we expect to receive from the
+   *  client during the Extended ORPort authentication protocol. We
+   *  compute it upon receiving the ClientNoce from the client, and we
+   *  compare it with the acual ClientHash value sent by the
+   *  client. */
+  char *ext_or_auth_correct_client_hash;
+  /** String carrying the name of the pluggable transport
+   *  (e.g. "obfs2") that is obfuscating this connection. If no
+   *  pluggable transports are used, it's NULL. */
+  char *ext_or_transport;
+
+  char *nickname; /**< Nickname of OR on other side (if any). */
+
+  struct tor_tls_t *tls; /**< TLS connection state. */
+  int tls_error; /**< Last tor_tls error code. */
+  /** When we last used this conn for any client traffic. If not
+   * recent, we can rate limit it further. */
+
+  /* Channel using this connection */
+  channel_tls_t *chan;
+
+  tor_addr_t real_addr; /**< The actual address that this connection came from
+                       * or went to.  The <b>addr</b> field is prone to
+                       * getting overridden by the address from the router
+                       * descriptor matching <b>identity_digest</b>. */
+
+  /** Should this connection be used for extending circuits to the server
+   * matching the <b>identity_digest</b> field?  Set to true if we're pretty
+   * sure we aren't getting MITMed, either because we're connected to an
+   * address listed in a server descriptor, or because an authenticated
+   * NETINFO cell listed the address we're connected to as recognized. */
+  unsigned int is_canonical:1;
+
+  /** True iff this is an outgoing connection. */
+  unsigned int is_outgoing:1;
+  unsigned int proxy_type:2; /**< One of PROXY_NONE...PROXY_SOCKS5 */
+  unsigned int wide_circ_ids:1;
+  /** True iff this connection has had its bootstrap failure logged with
+   * control_event_bootstrap_problem. */
+  unsigned int have_noted_bootstrap_problem:1;
+  /** True iff this is a client connection and its address has been put in the
+   * geoip cache and handled by the DoS mitigation subsystem. We use this to
+   * insure we have a coherent count of concurrent connection. */
+  unsigned int tracked_for_dos_mitigation : 1;
+
+  uint16_t link_proto; /**< What protocol version are we using? 0 for
+                        * "none negotiated yet." */
+  uint16_t idle_timeout; /**< How long can this connection sit with no
+                          * circuits on it before we close it? Based on
+                          * IDLE_CIRCUIT_TIMEOUT_{NON,}CANONICAL and
+                          * on is_canonical, randomized. */
+  or_handshake_state_t *handshake_state; /**< If we are setting this connection
+                                          * up, state information to do so. */
+
+  time_t timestamp_lastempty; /**< When was the outbuf last completely empty?*/
+
+  token_bucket_rw_t bucket; /**< Used for rate limiting when the connection is
+                          * in state CONN_OPEN. */
+
+  /*
+   * Count the number of bytes flushed out on this orconn, and the number of
+   * bytes TLS actually sent - used for overhead estimation for scheduling.
+   */
+  uint64_t bytes_xmitted, bytes_xmitted_by_tls;
+};
+
+#endif
diff --git a/src/core/or/or_handshake_certs_st.h b/src/core/or/or_handshake_certs_st.h
new file mode 100644
index 0000000000..a93b7104aa
--- /dev/null
+++ b/src/core/or/or_handshake_certs_st.h
@@ -0,0 +1,40 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef OR_HANDSHAKE_CERTS_ST
+#define OR_HANDSHAKE_CERTS_ST
+
+struct tor_x509_cert_t;
+
+/** Structure to hold all the certificates we've received on an OR connection
+ */
+struct or_handshake_certs_t {
+  /** True iff we originated this connection. */
+  int started_here;
+  /** The cert for the 'auth' RSA key that's supposed to sign the AUTHENTICATE
+   * cell. Signed with the RSA identity key. */
+  struct tor_x509_cert_t *auth_cert;
+  /** The cert for the 'link' RSA key that was used to negotiate the TLS
+   *  connection.  Signed with the RSA identity key. */
+  struct tor_x509_cert_t *link_cert;
+  /** A self-signed identity certificate: the RSA identity key signed
+   * with itself.  */
+  struct tor_x509_cert_t *id_cert;
+  /** The Ed25519 signing key, signed with the Ed25519 identity key. */
+  struct tor_cert_st *ed_id_sign;
+  /** A digest of the X509 link certificate for the TLS connection, signed
+   * with the Ed25519 siging key. */
+  struct tor_cert_st *ed_sign_link;
+  /** The Ed25519 authentication key (that's supposed to sign an AUTHENTICATE
+   * cell) , signed with the Ed25519 siging key. */
+  struct tor_cert_st *ed_sign_auth;
+  /** The Ed25519 identity key, crosssigned with the RSA identity key. */
+  uint8_t *ed_rsa_crosscert;
+  /** The length of <b>ed_rsa_crosscert</b> in bytes */
+  size_t ed_rsa_crosscert_len;
+};
+
+#endif
diff --git a/src/core/or/or_handshake_state_st.h b/src/core/or/or_handshake_state_st.h
new file mode 100644
index 0000000000..09a8a34179
--- /dev/null
+++ b/src/core/or/or_handshake_state_st.h
@@ -0,0 +1,78 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef OR_HANDSHAKE_STATE_ST
+#define OR_HANDSHAKE_STATE_ST
+
+/** Stores flags and information related to the portion of a v2/v3 Tor OR
+ * connection handshake that happens after the TLS handshake is finished.
+ */
+struct or_handshake_state_t {
+  /** When was the VERSIONS cell sent on this connection?  Used to get
+   * an estimate of the skew in the returning NETINFO reply. */
+  time_t sent_versions_at;
+  /** True iff we originated this connection */
+  unsigned int started_here : 1;
+  /** True iff we have received and processed a VERSIONS cell. */
+  unsigned int received_versions : 1;
+  /** True iff we have received and processed an AUTH_CHALLENGE cell */
+  unsigned int received_auth_challenge : 1;
+  /** True iff we have received and processed a CERTS cell. */
+  unsigned int received_certs_cell : 1;
+  /** True iff we have received and processed an AUTHENTICATE cell */
+  unsigned int received_authenticate : 1;
+
+  /* True iff we've received valid authentication to some identity. */
+  unsigned int authenticated : 1;
+  unsigned int authenticated_rsa : 1;
+  unsigned int authenticated_ed25519 : 1;
+
+  /* True iff we have sent a netinfo cell */
+  unsigned int sent_netinfo : 1;
+
+  /** The signing->ed25519 link certificate corresponding to the x509
+   * certificate we used on the TLS connection (if this is a server-side
+   * connection). We make a copy of this here to prevent a race condition
+   * caused by TLS context rotation. */
+  struct tor_cert_st *own_link_cert;
+
+  /** True iff we should feed outgoing cells into digest_sent and
+   * digest_received respectively.
+   *
+   * From the server's side of the v3 handshake, we want to capture everything
+   * from the VERSIONS cell through and including the AUTH_CHALLENGE cell.
+   * From the client's, we want to capture everything from the VERSIONS cell
+   * through but *not* including the AUTHENTICATE cell.
+   *
+   * @{ */
+  unsigned int digest_sent_data : 1;
+  unsigned int digest_received_data : 1;
+  /**@}*/
+
+  /** Identity RSA digest that we have received and authenticated for our peer
+   * on this connection. */
+  uint8_t authenticated_rsa_peer_id[DIGEST_LEN];
+  /** Identity Ed25519 public key that we have received and authenticated for
+   * our peer on this connection. */
+  ed25519_public_key_t authenticated_ed25519_peer_id;
+
+  /** Digests of the cells that we have sent or received as part of a V3
+   * handshake.  Used for making and checking AUTHENTICATE cells.
+   *
+   * @{
+   */
+  crypto_digest_t *digest_sent;
+  crypto_digest_t *digest_received;
+  /** @} */
+
+  /** Certificates that a connection initiator sent us in a CERTS cell; we're
+   * holding on to them until we get an AUTHENTICATE cell.
+   */
+  or_handshake_certs_t *certs;
+};
+
+#endif
+
diff --git a/src/core/or/origin_circuit_st.h b/src/core/or/origin_circuit_st.h
new file mode 100644
index 0000000000..f55416db14
--- /dev/null
+++ b/src/core/or/origin_circuit_st.h
@@ -0,0 +1,294 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef ORIGIN_CIRCUIT_ST_H
+#define ORIGIN_CIRCUIT_ST_H
+
+#include "core/or/or.h"
+
+#include "core/or/circuit_st.h"
+
+struct onion_queue_t;
+
+/**
+ * Describes the circuit building process in simplified terms based
+ * on the path bias accounting state for a circuit.
+ *
+ * NOTE: These state values are enumerated in the order for which we
+ * expect circuits to transition through them. If you add states,
+ * you need to preserve this overall ordering. The various pathbias
+ * state transition and accounting functions (pathbias_mark_* and
+ * pathbias_count_*) contain ordinal comparisons to enforce proper
+ * state transitions for corrections.
+ *
+ * This state machine and the associated logic was created to prevent
+ * miscounting due to unknown cases of circuit reuse. See also tickets
+ * #6475 and #7802.
+ */
+enum path_state_t {
+    /** This circuit is "new". It has not yet completed a first hop
+     * or been counted by the path bias code. */
+    PATH_STATE_NEW_CIRC = 0,
+    /** This circuit has completed one/two hops, and has been counted by
+     * the path bias logic. */
+    PATH_STATE_BUILD_ATTEMPTED = 1,
+    /** This circuit has been completely built */
+    PATH_STATE_BUILD_SUCCEEDED = 2,
+    /** Did we try to attach any SOCKS streams or hidserv introductions to
+      * this circuit?
+      *
+      * Note: If we ever implement end-to-end stream timing through test
+      * stream probes (#5707), we must *not* set this for those probes
+      * (or any other automatic streams) because the adversary could
+      * just tag at a later point.
+      */
+    PATH_STATE_USE_ATTEMPTED = 3,
+    /** Did any SOCKS streams or hidserv introductions actually succeed on
+      * this circuit?
+      *
+      * If any streams detatch/fail from this circuit, the code transitions
+      * the circuit back to PATH_STATE_USE_ATTEMPTED to ensure we probe. See
+      * pathbias_mark_use_rollback() for that.
+      */
+    PATH_STATE_USE_SUCCEEDED = 4,
+
+    /**
+     * This is a special state to indicate that we got a corrupted
+     * relay cell on a circuit and we don't intend to probe it.
+     */
+    PATH_STATE_USE_FAILED = 5,
+
+    /**
+     * This is a special state to indicate that we already counted
+     * the circuit. Used to guard against potential state machine
+     * violations.
+     */
+    PATH_STATE_ALREADY_COUNTED = 6,
+};
+
+/** An origin_circuit_t holds data necessary to build and use a circuit.
+ */
+struct origin_circuit_t {
+  circuit_t base_;
+
+  /** Linked list of AP streams (or EXIT streams if hidden service)
+   * associated with this circuit. */
+  edge_connection_t *p_streams;
+
+  /** Smartlist of half-closed streams (half_edge_t*) that still
+   * have pending activity */
+  smartlist_t *half_streams;
+
+  /** Bytes read on this circuit since last call to
+   * control_event_circ_bandwidth_used().  Only used if we're configured
+   * to emit CIRC_BW events. */
+  uint32_t n_read_circ_bw;
+
+  /** Bytes written to on this circuit since last call to
+   * control_event_circ_bandwidth_used().  Only used if we're configured
+   * to emit CIRC_BW events. */
+  uint32_t n_written_circ_bw;
+
+  /** Total known-valid relay cell bytes since last call to
+   * control_event_circ_bandwidth_used().  Only used if we're configured
+   * to emit CIRC_BW events. */
+  uint32_t n_delivered_read_circ_bw;
+
+  /** Total written relay cell bytes since last call to
+   * control_event_circ_bandwidth_used().  Only used if we're configured
+   * to emit CIRC_BW events. */
+  uint32_t n_delivered_written_circ_bw;
+
+  /** Total overhead data in all known-valid relay data cells since last
+   * call to control_event_circ_bandwidth_used().  Only used if we're
+   * configured to emit CIRC_BW events. */
+  uint32_t n_overhead_read_circ_bw;
+
+  /** Total written overhead data in all relay data cells since last call to
+   * control_event_circ_bandwidth_used().  Only used if we're configured
+   * to emit CIRC_BW events. */
+  uint32_t n_overhead_written_circ_bw;
+
+  /** Build state for this circuit. It includes the intended path
+   * length, the chosen exit router, rendezvous information, etc.
+   */
+  cpath_build_state_t *build_state;
+  /** The doubly-linked list of crypt_path_t entries, one per hop,
+   * for this circuit. This includes ciphers for each hop,
+   * integrity-checking digests for each hop, and package/delivery
+   * windows for each hop.
+   */
+  crypt_path_t *cpath;
+
+  /** Holds all rendezvous data on either client or service side. */
+  rend_data_t *rend_data;
+
+  /** Holds hidden service identifier on either client or service side. This
+   * is for both introduction and rendezvous circuit. */
+  struct hs_ident_circuit_t *hs_ident;
+
+  /** Holds the data that the entry guard system uses to track the
+   * status of the guard this circuit is using, and thereby to determine
+   * whether this circuit can be used. */
+  struct circuit_guard_state_t *guard_state;
+
+  /** Index into global_origin_circuit_list for this circuit. -1 if not
+   * present. */
+  int global_origin_circuit_list_idx;
+
+  /** How many more relay_early cells can we send on this circuit, according
+   * to the specification? */
+  unsigned int remaining_relay_early_cells : 4;
+
+  /** Set if this circuit is insanely old and we already informed the user */
+  unsigned int is_ancient : 1;
+
+  /** Set if this circuit has already been opened. Used to detect
+   * cannibalized circuits. */
+  unsigned int has_opened : 1;
+
+  /**
+   * Path bias state machine. Used to ensure integrity of our
+   * circuit building and usage accounting. See path_state_t
+   * for more details.
+   */
+  path_state_bitfield_t path_state : 3;
+
+  /* If this flag is set, we should not consider attaching any more
+   * connections to this circuit. */
+  unsigned int unusable_for_new_conns : 1;
+
+  /**
+   * Tristate variable to guard against pathbias miscounting
+   * due to circuit purpose transitions changing the decision
+   * of pathbias_should_count(). This variable is informational
+   * only. The current results of pathbias_should_count() are
+   * the official decision for pathbias accounting.
+   */
+  uint8_t pathbias_shouldcount;
+#define PATHBIAS_SHOULDCOUNT_UNDECIDED 0
+#define PATHBIAS_SHOULDCOUNT_IGNORED   1
+#define PATHBIAS_SHOULDCOUNT_COUNTED   2
+
+  /** For path probing. Store the temporary probe stream ID
+   * for response comparison */
+  streamid_t pathbias_probe_id;
+
+  /** For path probing. Store the temporary probe address nonce
+   * (in host byte order) for response comparison. */
+  uint32_t pathbias_probe_nonce;
+
+  /** Set iff this is a hidden-service circuit which has timed out
+   * according to our current circuit-build timeout, but which has
+   * been kept around because it might still succeed in connecting to
+   * its destination, and which is not a fully-connected rendezvous
+   * circuit.
+   *
+   * (We clear this flag for client-side rendezvous circuits when they
+   * are 'joined' to the other side's rendezvous circuit, so that
+   * connection_ap_handshake_attach_circuit can put client streams on
+   * the circuit.  We also clear this flag for service-side rendezvous
+   * circuits when they are 'joined' to a client's rend circ, but only
+   * for symmetry with the client case.  Client-side introduction
+   * circuits are closed when we get a joined rend circ, and
+   * service-side introduction circuits never have this flag set.) */
+  unsigned int hs_circ_has_timed_out : 1;
+
+  /** Set iff this circuit has been given a relaxed timeout because
+   * no circuits have opened. Used to prevent spamming logs. */
+  unsigned int relaxed_timeout : 1;
+
+  /** Set iff this is a service-side rendezvous circuit for which a
+   * new connection attempt has been launched.  We consider launching
+   * a new service-side rend circ to a client when the previous one
+   * fails; now that we don't necessarily close a service-side rend
+   * circ when we launch a new one to the same client, this flag keeps
+   * us from launching two retries for the same failed rend circ. */
+  unsigned int hs_service_side_rend_circ_has_been_relaunched : 1;
+
+  /** What commands were sent over this circuit that decremented the
+   * RELAY_EARLY counter? This is for debugging task 878. */
+  uint8_t relay_early_commands[MAX_RELAY_EARLY_CELLS_PER_CIRCUIT];
+
+  /** How many RELAY_EARLY cells have been sent over this circuit? This is
+   * for debugging task 878, too. */
+  int relay_early_cells_sent;
+
+  /** The next stream_id that will be tried when we're attempting to
+   * construct a new AP stream originating at this circuit. */
+  streamid_t next_stream_id;
+
+  /* The intro key replaces the hidden service's public key if purpose is
+   * S_ESTABLISH_INTRO or S_INTRO, provided that no unversioned rendezvous
+   * descriptor is used. */
+  crypto_pk_t *intro_key;
+
+  /** Quasi-global identifier for this circuit; used for control.c */
+  /* XXXX NM This can get re-used after 2**32 circuits. */
+  uint32_t global_identifier;
+
+  /** True if we have associated one stream to this circuit, thereby setting
+   * the isolation parameters for this circuit.  Note that this doesn't
+   * necessarily mean that we've <em>attached</em> any streams to the circuit:
+   * we may only have marked up this circuit during the launch process.
+   */
+  unsigned int isolation_values_set : 1;
+  /** True iff any stream has <em>ever</em> been attached to this circuit.
+   *
+   * In a better world we could use timestamp_dirty for this, but
+   * timestamp_dirty is far too overloaded at the moment.
+   */
+  unsigned int isolation_any_streams_attached : 1;
+
+  /** A bitfield of ISO_* flags for every isolation field such that this
+   * circuit has had streams with more than one value for that field
+   * attached to it. */
+  uint8_t isolation_flags_mixed;
+
+  /** @name Isolation parameters
+   *
+   * If any streams have been associated with this circ (isolation_values_set
+   * == 1), and all streams associated with the circuit have had the same
+   * value for some field ((isolation_flags_mixed & ISO_FOO) == 0), then these
+   * elements hold the value for that field.
+   *
+   * Note again that "associated" is not the same as "attached": we
+   * preliminarily associate streams with a circuit while the circuit is being
+   * launched, so that we can tell whether we need to launch more circuits.
+   *
+   * @{
+   */
+  uint8_t client_proto_type;
+  uint8_t client_proto_socksver;
+  uint16_t dest_port;
+  tor_addr_t client_addr;
+  char *dest_address;
+  int session_group;
+  unsigned nym_epoch;
+  size_t socks_username_len;
+  uint8_t socks_password_len;
+  /* Note that the next two values are NOT NUL-terminated; see
+     socks_username_len and socks_password_len for their lengths. */
+  char *socks_username;
+  char *socks_password;
+  /** Global identifier for the first stream attached here; used by
+   * ISO_STREAM. */
+  uint64_t associated_isolated_stream_global_id;
+  /**@}*/
+  /** A list of addr_policy_t for this circuit in particular. Used by
+   * adjust_exit_policy_from_exitpolicy_failure.
+   */
+  smartlist_t *prepend_policy;
+
+  /** How long do we wait before closing this circuit if it remains
+   * completely idle after it was built, in seconds? This value
+   * is randomized on a per-circuit basis from CircuitsAvailableTimoeut
+   * to 2*CircuitsAvailableTimoeut. */
+  int circuit_idle_timeout;
+
+};
+
+#endif
diff --git a/src/or/policies.c b/src/core/or/policies.c
index f58bf329ad..3ed282d785 100644
--- a/src/or/policies.c
+++ b/src/core/or/policies.c
@@ -1,25 +1,46 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file policies.c
  * \brief Code to parse and use address policies and exit policies.
+ *
+ * We have two key kinds of address policy: full and compressed.  A full
+ * policy is an array of accept/reject patterns, to be applied in order.
+ * A short policy is simply a list of ports.  This module handles both
+ * kinds, including generic functions to apply them to addresses, and
+ * also including code to manage the global policies that we apply to
+ * incoming and outgoing connections.
  **/
 
 #define POLICIES_PRIVATE
 
-#include "or.h"
-#include "config.h"
-#include "dirserv.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "policies.h"
-#include "router.h"
-#include "routerparse.h"
-#include "geoip.h"
+#include "core/or/or.h"
+#include "feature/client/bridges.h"
+#include "app/config/config.h"
+#include "core/or/policies.h"
+#include "feature/dirparse/policy_parse.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+#include "lib/geoip/geoip.h"
 #include "ht.h"
+#include "lib/encoding/confline.h"
+
+#include "core/or/addr_policy_st.h"
+#include "feature/dirclient/dir_server_st.h"
+#include "feature/nodelist/microdesc_st.h"
+#include "feature/nodelist/node_st.h"
+#include "core/or/port_cfg_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+/** Maximum length of an exit policy summary. */
+#define MAX_EXITPOLICY_SUMMARY_LEN 1000
 
 /** Policy that addresses for incoming SOCKS connections must match. */
 static smartlist_t *socks_policy = NULL;
@@ -73,7 +94,8 @@ static int policies_parse_exit_policy_internal(
                                       const smartlist_t *configured_addresses,
                                       int reject_interface_addresses,
                                       int reject_configured_port_addresses,
-                                      int add_default_policy);
+                                      int add_default_policy,
+                                      int add_reduced_policy);
 
 /** Replace all "private" entries in *<b>policy</b> with their expanded
  * equivalents. */
@@ -290,14 +312,22 @@ parse_reachable_addresses(void)
     } else if (fascist_firewall_use_ipv6(options)
        && (policy_is_reject_star(reachable_or_addr_policy, AF_INET6, 0)
          || policy_is_reject_star(reachable_dir_addr_policy, AF_INET6, 0))) {
-          log_warn(LD_CONFIG, "You have configured tor to use IPv6 "
-                   "(ClientUseIPv6 1 or UseBridges 1), but "
+          log_warn(LD_CONFIG, "You have configured tor to use or prefer IPv6 "
+                   "(or UseBridges 1), but "
                    "ReachableAddresses, ReachableORAddresses, or "
                    "ReachableDirAddresses reject all IPv6 addresses. "
                    "Tor will not connect using IPv6.");
     }
   }
 
+  /* Append a reject *:* to reachable_(or|dir)_addr_policy */
+  if (!ret && (options->ReachableDirAddresses ||
+               options->ReachableORAddresses ||
+               options->ReachableAddresses)) {
+    append_exit_policy_string(&reachable_or_addr_policy, "reject *:*");
+    append_exit_policy_string(&reachable_dir_addr_policy, "reject *:*");
+  }
+
   return ret;
 }
 
@@ -309,10 +339,8 @@ firewall_is_fascist_impl(void)
   const or_options_t *options = get_options();
   /* Assume every non-bridge relay has an IPv4 address.
    * Clients which use bridges may only know the IPv6 address of their
-   * bridge. */
-  return (options->ClientUseIPv4 == 0
-          || (!fascist_firewall_use_ipv6(options)
-              && options->UseBridges == 1));
+   * bridge, but they will connect regardless of the ClientUseIPv6 setting. */
+  return options->ClientUseIPv4 == 0;
 }
 
 /** Return true iff the firewall options, including ClientUseIPv4 0 and
@@ -419,6 +447,9 @@ fascist_firewall_allows_address(const tor_addr_t *addr,
 }
 
 /** Is this client configured to use IPv6?
+ * Returns true if the client might use IPv6 for some of its connections
+ * (including dual-stack and IPv6-only clients), and false if it will never
+ * use IPv6 for any connections.
  * Use node_ipv6_or/dir_preferred() when checking a specific node and OR/Dir
  * port: it supports bridge client per-node IPv6 preferences.
  */
@@ -426,9 +457,11 @@ int
 fascist_firewall_use_ipv6(const or_options_t *options)
 {
   /* Clients use IPv6 if it's set, or they use bridges, or they don't use
-   * IPv4 */
-  return (options->ClientUseIPv6 == 1 || options->UseBridges == 1
-          || options->ClientUseIPv4 == 0);
+   * IPv4, or they prefer it.
+   * ClientPreferIPv6DirPort is deprecated, but check it anyway. */
+  return (options->ClientUseIPv6 == 1 || options->ClientUseIPv4 == 0 ||
+          options->ClientPreferIPv6ORPort == 1 ||
+          options->ClientPreferIPv6DirPort == 1 || options->UseBridges == 1);
 }
 
 /** Do we prefer to connect to IPv6, ignoring ClientPreferIPv6ORPort and
@@ -812,9 +845,8 @@ fascist_firewall_choose_address(const tor_addr_port_t *a,
  * If pref_only, only choose preferred addresses. In either case, choose
  * a preferred address before an address that's not preferred.
  * If both addresses could be chosen (they are both preferred or both allowed)
- * choose IPv6 if pref_ipv6 is true, otherwise choose IPv4.
- * If neither address is chosen, return 0, else return 1. */
-static int
+ * choose IPv6 if pref_ipv6 is true, otherwise choose IPv4. */
+static void
 fascist_firewall_choose_address_base(const tor_addr_t *ipv4_addr,
                                      uint16_t ipv4_orport,
                                      uint16_t ipv4_dirport,
@@ -832,6 +864,9 @@ fascist_firewall_choose_address_base(const tor_addr_t *ipv4_addr,
   tor_assert(ipv6_addr);
   tor_assert(ap);
 
+  tor_addr_make_null(&ap->addr, AF_UNSPEC);
+  ap->port = 0;
+
   tor_addr_port_t ipv4_ap;
   tor_addr_copy(&ipv4_ap.addr, ipv4_addr);
   ipv4_ap.port = (fw_connection == FIREWALL_OR_CONNECTION
@@ -852,17 +887,12 @@ fascist_firewall_choose_address_base(const tor_addr_t *ipv4_addr,
   if (result) {
     tor_addr_copy(&ap->addr, &result->addr);
     ap->port = result->port;
-    return 1;
-  } else {
-    tor_addr_make_null(&ap->addr, AF_UNSPEC);
-    ap->port = 0;
-    return 0;
   }
 }
 
 /** Like fascist_firewall_choose_address_base(), but takes a host-order IPv4
  * address as the first parameter. */
-static int
+static void
 fascist_firewall_choose_address_ipv4h(uint32_t ipv4h_addr,
                                       uint16_t ipv4_orport,
                                       uint16_t ipv4_dirport,
@@ -876,11 +906,59 @@ fascist_firewall_choose_address_ipv4h(uint32_t ipv4h_addr,
 {
   tor_addr_t ipv4_addr;
   tor_addr_from_ipv4h(&ipv4_addr, ipv4h_addr);
-  return fascist_firewall_choose_address_base(&ipv4_addr, ipv4_orport,
-                                              ipv4_dirport, ipv6_addr,
-                                              ipv6_orport, ipv6_dirport,
-                                              fw_connection, pref_only,
-                                              pref_ipv6, ap);
+  tor_assert(ap);
+
+  tor_addr_make_null(&ap->addr, AF_UNSPEC);
+  ap->port = 0;
+
+  fascist_firewall_choose_address_base(&ipv4_addr, ipv4_orport,
+                                       ipv4_dirport, ipv6_addr,
+                                       ipv6_orport, ipv6_dirport,
+                                       fw_connection, pref_only,
+                                       pref_ipv6, ap);
+}
+
+/* Some microdescriptor consensus methods have no IPv6 addresses in rs: they
+ * are in the microdescriptors. For these consensus methods, we can't rely on
+ * the node's IPv6 address until its microdescriptor is available (when using
+ * microdescs).
+ * But for bridges, rewrite_node_address_for_bridge() updates node->ri with
+ * the configured address, so we can trust bridge addresses.
+ * (Bridges could gain an IPv6 address if their microdescriptor arrives, but
+ * this will never be their preferred address: that is in the config.)
+ * Returns true if the node needs a microdescriptor for its IPv6 address, and
+ * false if the addresses in the node are already up-to-date.
+ */
+static int
+node_awaiting_ipv6(const or_options_t* options, const node_t *node)
+{
+  tor_assert(node);
+
+  /* There's no point waiting for an IPv6 address if we'd never use it */
+  if (!fascist_firewall_use_ipv6(options)) {
+    return 0;
+  }
+
+  /* If the node has an IPv6 address, we're not waiting */
+  if (node_has_ipv6_addr(node)) {
+    return 0;
+  }
+
+  /* If the current consensus method and flavour has IPv6 addresses, we're not
+   * waiting */
+  if (networkstatus_consensus_has_ipv6(options)) {
+    return 0;
+  }
+
+  /* Bridge clients never use the address from a bridge's md, so there's no
+   * need to wait for it. */
+  if (node_is_a_configured_bridge(node)) {
+    return 0;
+  }
+
+  /* We are waiting if we_use_microdescriptors_for_circuits() and we have no
+   * md. */
+  return (!node->md && we_use_microdescriptors_for_circuits(options));
 }
 
 /** Like fascist_firewall_choose_address_base(), but takes <b>rs</b>.
@@ -888,58 +966,70 @@ fascist_firewall_choose_address_ipv4h(uint32_t ipv4h_addr,
  * This should only happen when there's no valid consensus, and rs doesn't
  * correspond to a bridge client's bridge.
  */
-int
+void
 fascist_firewall_choose_address_rs(const routerstatus_t *rs,
                                    firewall_connection_t fw_connection,
                                    int pref_only, tor_addr_port_t* ap)
 {
+  tor_assert(ap);
+
+  tor_addr_make_null(&ap->addr, AF_UNSPEC);
+  ap->port = 0;
+
   if (!rs) {
-    return 0;
+    return;
   }
 
-  tor_assert(ap);
-
+  const or_options_t *options = get_options();
   const node_t *node = node_get_by_id(rs->identity_digest);
 
-  if (node) {
-    return fascist_firewall_choose_address_node(node, fw_connection, pref_only,
-                                                ap);
+  if (node && !node_awaiting_ipv6(options, node)) {
+    fascist_firewall_choose_address_node(node, fw_connection, pref_only, ap);
   } else {
     /* There's no node-specific IPv6 preference, so use the generic IPv6
      * preference instead. */
-    const or_options_t *options = get_options();
     int pref_ipv6 = (fw_connection == FIREWALL_OR_CONNECTION
                      ? fascist_firewall_prefer_ipv6_orport(options)
                      : fascist_firewall_prefer_ipv6_dirport(options));
 
     /* Assume IPv4 and IPv6 DirPorts are the same.
      * Assume the IPv6 OR and Dir addresses are the same. */
-    return fascist_firewall_choose_address_ipv4h(rs->addr,
-                                                 rs->or_port,
-                                                 rs->dir_port,
-                                                 &rs->ipv6_addr,
-                                                 rs->ipv6_orport,
-                                                 rs->dir_port,
-                                                 fw_connection,
-                                                 pref_only,
-                                                 pref_ipv6,
-                                                 ap);
+    fascist_firewall_choose_address_ipv4h(rs->addr, rs->or_port, rs->dir_port,
+                                          &rs->ipv6_addr, rs->ipv6_orport,
+                                          rs->dir_port, fw_connection,
+                                          pref_only, pref_ipv6, ap);
   }
 }
 
 /** Like fascist_firewall_choose_address_base(), but takes <b>node</b>, and
  * looks up the node's IPv6 preference rather than taking an argument
  * for pref_ipv6. */
-int
+void
 fascist_firewall_choose_address_node(const node_t *node,
                                      firewall_connection_t fw_connection,
                                      int pref_only, tor_addr_port_t *ap)
 {
+  tor_assert(ap);
+
+  tor_addr_make_null(&ap->addr, AF_UNSPEC);
+  ap->port = 0;
+
   if (!node) {
-    return 0;
+    return;
   }
 
   node_assert_ok(node);
+  /* Calling fascist_firewall_choose_address_node() when the node is missing
+   * IPv6 information breaks IPv6-only clients.
+   * If the node is a hard-coded fallback directory or authority, call
+   * fascist_firewall_choose_address_rs() on the fake (hard-coded) routerstatus
+   * for the node.
+   * If it is not hard-coded, check that the node has a microdescriptor, full
+   * descriptor (routerinfo), or is one of our configured bridges before
+   * calling this function. */
+  if (BUG(node_awaiting_ipv6(get_options(), node))) {
+    return;
+  }
 
   const int pref_ipv6_node = (fw_connection == FIREWALL_OR_CONNECTION
                               ? node_ipv6_or_preferred(node)
@@ -956,27 +1046,27 @@ fascist_firewall_choose_address_node(const node_t *node,
   node_get_pref_ipv6_dirport(node, &ipv6_dir_ap);
 
   /* Assume the IPv6 OR and Dir addresses are the same. */
-  return fascist_firewall_choose_address_base(&ipv4_or_ap.addr,
-                                              ipv4_or_ap.port,
-                                              ipv4_dir_ap.port,
-                                              &ipv6_or_ap.addr,
-                                              ipv6_or_ap.port,
-                                              ipv6_dir_ap.port,
-                                              fw_connection,
-                                              pref_only,
-                                              pref_ipv6_node,
-                                              ap);
+  fascist_firewall_choose_address_base(&ipv4_or_ap.addr, ipv4_or_ap.port,
+                                       ipv4_dir_ap.port, &ipv6_or_ap.addr,
+                                       ipv6_or_ap.port, ipv6_dir_ap.port,
+                                       fw_connection, pref_only,
+                                       pref_ipv6_node, ap);
 }
 
 /** Like fascist_firewall_choose_address_rs(), but takes <b>ds</b>. */
-int
+void
 fascist_firewall_choose_address_dir_server(const dir_server_t *ds,
                                            firewall_connection_t fw_connection,
                                            int pref_only,
                                            tor_addr_port_t *ap)
 {
+  tor_assert(ap);
+
+  tor_addr_make_null(&ap->addr, AF_UNSPEC);
+  ap->port = 0;
+
   if (!ds) {
-    return 0;
+    return;
   }
 
   /* A dir_server_t always has a fake_status. As long as it has the same
@@ -984,8 +1074,8 @@ fascist_firewall_choose_address_dir_server(const dir_server_t *ds,
    * (See #17867.)
    * This function relies on fascist_firewall_choose_address_rs looking up the
    * node if it can, because that will get the latest info for the relay. */
-  return fascist_firewall_choose_address_rs(&ds->fake_status, fw_connection,
-                                            pref_only, ap);
+  fascist_firewall_choose_address_rs(&ds->fake_status, fw_connection,
+                                     pref_only, ap);
 }
 
 /** Return 1 if <b>addr</b> is permitted to connect to our dir port,
@@ -1076,31 +1166,16 @@ validate_addr_policies(const or_options_t *options, char **msg)
     REJECT("Error in ExitPolicy entry.");
   }
 
-  static int warned_about_exitrelay = 0;
-
-  const int exitrelay_setting_is_auto = options->ExitRelay == -1;
-  const int policy_accepts_something =
-    ! (policy_is_reject_star(addr_policy, AF_INET, 1) &&
-       policy_is_reject_star(addr_policy, AF_INET6, 1));
+  static int warned_about_nonexit = 0;
 
-  if (server_mode(options) &&
-      ! warned_about_exitrelay &&
-      exitrelay_setting_is_auto &&
-      policy_accepts_something) {
-      /* Policy accepts something */
-    warned_about_exitrelay = 1;
-    log_warn(LD_CONFIG,
-             "Tor is running as an exit relay%s. If you did not want this "
-             "behavior, please set the ExitRelay option to 0. If you do "
-             "want to run an exit Relay, please set the ExitRelay option "
-             "to 1 to disable this warning, and for forward compatibility.",
-             options->ExitPolicy == NULL ?
-                 " with the default exit policy" : "");
-    if (options->ExitPolicy == NULL) {
-      log_warn(LD_CONFIG,
-               "In a future version of Tor, ExitRelay 0 may become the "
-               "default when no ExitPolicy is given.");
-    }
+  if (public_server_mode(options) &&
+      !warned_about_nonexit && options->ExitPolicy == NULL &&
+      options->ExitRelay == -1 && options->ReducedExitPolicy == 0) {
+    warned_about_nonexit = 1;
+    log_notice(LD_CONFIG, "By default, Tor does not run as an exit relay. "
+               "If you want to be an exit relay, "
+               "set ExitRelay to 1. To suppress this message in the future, "
+               "set ExitRelay to 0.");
   }
 
   /* The rest of these calls *append* to addr_policy. So don't actually
@@ -1743,14 +1818,14 @@ policies_parse_exit_policy_reject_private(
     /* Reject public IPv4 addresses on any interface */
     public_addresses = get_interface_address6_list(LOG_INFO, AF_INET, 0);
     addr_policy_append_reject_addr_list_filter(dest, public_addresses, 1, 0);
-    free_interface_address6_list(public_addresses);
+    interface_address6_list_free(public_addresses);
 
     /* Don't look for IPv6 addresses if we're configured as IPv4-only */
     if (ipv6_exit) {
       /* Reject public IPv6 addresses on any interface */
       public_addresses = get_interface_address6_list(LOG_INFO, AF_INET6, 0);
       addr_policy_append_reject_addr_list_filter(dest, public_addresses, 0, 1);
-      free_interface_address6_list(public_addresses);
+      interface_address6_list_free(public_addresses);
     }
   }
 
@@ -1829,6 +1904,24 @@ policies_log_first_redundant_entry(const smartlist_t *policy)
   "reject *:563,reject *:1214,reject *:4661-4666,"                  \
   "reject *:6346-6429,reject *:6699,reject *:6881-6999,accept *:*"
 
+#define REDUCED_EXIT_POLICY                                                   \
+  "accept *:20-23,accept *:43,accept *:53,accept *:79-81,accept *:88,"        \
+  "accept *:110,accept *:143,accept *:194,accept *:220,accept *:389,"         \
+  "accept *:443,accept *:464,accept *:465,accept *:531,accept *:543-544,"     \
+  "accept *:554,accept *:563,accept *:587,accept *:636,accept *:706,"         \
+  "accept *:749,accept *:873,accept *:902-904,accept *:981,accept *:989-995," \
+  "accept *:1194,accept *:1220,accept *:1293,accept *:1500,accept *:1533,"    \
+  "accept *:1677,accept *:1723,accept *:1755,accept *:1863,"                  \
+  "accept *:2082-2083,accept *:2086-2087,accept *:2095-2096,"                 \
+  "accept *:2102-2104,accept *:3128,accept *:3389,accept *:3690,"             \
+  "accept *:4321,accept *:4643,accept *:5050,accept *:5190,"                  \
+  "accept *:5222-5223,accept *:5228,accept *:5900,accept *:6660-6669,"        \
+  "accept *:6679,accept *:6697,accept *:8000,accept *:8008,accept *:8074,"    \
+  "accept *:8080,accept *:8082,accept *:8087-8088,accept *:8232-8233,"        \
+  "accept *:8332-8333,accept *:8443,accept *:8888,accept *:9418,"             \
+  "accept *:9999,accept *:10000,accept *:11371,accept *:19294,"               \
+  "accept *:19638,accept *:50002,accept *:64738,reject *:*"
+
 /** Parse the exit policy <b>cfg</b> into the linked list *<b>dest</b>.
  *
  * If <b>ipv6_exit</b> is false, prepend "reject *6:*" to the policy.
@@ -1864,7 +1957,8 @@ policies_parse_exit_policy_internal(config_line_t *cfg,
                                     const smartlist_t *configured_addresses,
                                     int reject_interface_addresses,
                                     int reject_configured_port_addresses,
-                                    int add_default_policy)
+                                    int add_default_policy,
+                                    int add_reduced_policy)
 {
   if (!ipv6_exit) {
     append_exit_policy_string(dest, "reject *6:*");
@@ -1890,7 +1984,9 @@ policies_parse_exit_policy_internal(config_line_t *cfg,
    * effect, and are most likely an error. */
   policies_log_first_redundant_entry(*dest);
 
-  if (add_default_policy) {
+  if (add_reduced_policy) {
+    append_exit_policy_string(dest, REDUCED_EXIT_POLICY);
+  } else if (add_default_policy) {
     append_exit_policy_string(dest, DEFAULT_EXIT_POLICY);
   } else {
     append_exit_policy_string(dest, "reject *4:*");
@@ -1931,13 +2027,15 @@ policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest,
   int add_default = (options & EXIT_POLICY_ADD_DEFAULT) ? 1 : 0;
   int reject_local_interfaces = (options &
                                  EXIT_POLICY_REJECT_LOCAL_INTERFACES) ? 1 : 0;
+  int add_reduced = (options & EXIT_POLICY_ADD_REDUCED) ? 1 : 0;
 
   return policies_parse_exit_policy_internal(cfg,dest,ipv6_enabled,
                                              reject_private,
                                              configured_addresses,
                                              reject_local_interfaces,
                                              reject_local_interfaces,
-                                             add_default);
+                                             add_default,
+                                             add_reduced);
 }
 
 /** Helper function that adds a copy of addr to a smartlist as long as it is
@@ -1971,10 +2069,10 @@ policies_copy_ipv4h_to_smartlist(smartlist_t *addr_list, uint32_t ipv4h_addr)
   }
 }
 
-/** Helper function that adds copies of
- * or_options->OutboundBindAddressIPv[4|6]_ to a smartlist as tor_addr_t *, as
- * long as or_options is non-NULL, and the addresses are not
- * tor_addr_is_null(), by passing them to policies_add_addr_to_smartlist.
+/** Helper function that adds copies of or_options->OutboundBindAddresses
+ * to a smartlist as tor_addr_t *, as long as or_options is non-NULL, and
+ * the addresses are not tor_addr_is_null(), by passing them to
+ * policies_add_addr_to_smartlist.
  *
  * The caller is responsible for freeing all the tor_addr_t* in the smartlist.
  */
@@ -1983,10 +2081,14 @@ policies_copy_outbound_addresses_to_smartlist(smartlist_t *addr_list,
                                               const or_options_t *or_options)
 {
   if (or_options) {
-    policies_copy_addr_to_smartlist(addr_list,
-                                    &or_options->OutboundBindAddressIPv4_);
-    policies_copy_addr_to_smartlist(addr_list,
-                                    &or_options->OutboundBindAddressIPv6_);
+    for (int i=0;i<OUTBOUND_ADDR_MAX;i++) {
+      for (int j=0;j<2;j++) {
+        if (!tor_addr_is_null(&or_options->OutboundBindAddresses[i][j])) {
+          policies_copy_addr_to_smartlist(addr_list,
+                          &or_options->OutboundBindAddresses[i][j]);
+        }
+      }
+    }
   }
 }
 
@@ -2003,16 +2105,16 @@ policies_copy_outbound_addresses_to_smartlist(smartlist_t *addr_list,
  *  - if ipv6_local_address is non-NULL, and not the null tor_addr_t, add it
  *    to the list of configured addresses.
  * If <b>or_options->ExitPolicyRejectLocalInterfaces</b> is true:
- *  - if or_options->OutboundBindAddressIPv4_ is not the null tor_addr_t, add
- *    it to the list of configured addresses.
- *  - if or_options->OutboundBindAddressIPv6_ is not the null tor_addr_t, add
- *    it to the list of configured addresses.
+ *  - if or_options->OutboundBindAddresses[][0] (=IPv4) is not the null
+ *    tor_addr_t, add it to the list of configured addresses.
+ *  - if or_options->OutboundBindAddresses[][1] (=IPv6) is not the null
+ *    tor_addr_t, add it to the list of configured addresses.
  *
  * If <b>or_options->BridgeRelay</b> is false, append entries of default
  * Tor exit policy into <b>result</b> smartlist.
  *
- * If or_options->ExitRelay is false, then make our exit policy into
- * "reject *:*" regardless.
+ * If or_options->ExitRelay is false, or is auto without specifying an exit
+ * policy, then make our exit policy into "reject *:*" regardless.
  */
 int
 policies_parse_exit_policy_from_options(const or_options_t *or_options,
@@ -2024,8 +2126,10 @@ policies_parse_exit_policy_from_options(const or_options_t *or_options,
   smartlist_t *configured_addresses = NULL;
   int rv = 0;
 
-  /* Short-circuit for non-exit relays */
-  if (or_options->ExitRelay == 0) {
+  /* Short-circuit for non-exit relays, or for relays where we didn't specify
+   * ExitPolicy or ReducedExitPolicy and ExitRelay is auto. */
+  if (or_options->ExitRelay == 0 || (or_options->ExitPolicy == NULL &&
+      or_options->ExitRelay == -1 && or_options->ReducedExitPolicy == 0)) {
     append_exit_policy_string(result, "reject *4:*");
     append_exit_policy_string(result, "reject *6:*");
     return 0;
@@ -2043,7 +2147,10 @@ policies_parse_exit_policy_from_options(const or_options_t *or_options,
   }
 
   if (!or_options->BridgeRelay) {
-    parser_cfg |= EXIT_POLICY_ADD_DEFAULT;
+    if (or_options->ReducedExitPolicy)
+      parser_cfg |= EXIT_POLICY_ADD_REDUCED;
+    else
+      parser_cfg |= EXIT_POLICY_ADD_DEFAULT;
   }
 
   if (or_options->ExitPolicyRejectLocalInterfaces) {
@@ -2134,21 +2241,16 @@ exit_policy_is_general_exit_helper(smartlist_t *policy, int port)
 }
 
 /** Return true iff <b>ri</b> is "useful as an exit node", meaning
- * it allows exit to at least one /8 address space for at least
- * two of ports 80, 443, and 6667. */
+ * it allows exit to at least one /8 address space for each of ports 80
+ * and 443. */
 int
 exit_policy_is_general_exit(smartlist_t *policy)
 {
-  static const int ports[] = { 80, 443, 6667 };
-  int n_allowed = 0;
-  int i;
   if (!policy) /*XXXX disallow NULL policies? */
     return 0;
 
-  for (i = 0; i < 3; ++i) {
-    n_allowed += exit_policy_is_general_exit_helper(policy, ports[i]);
-  }
-  return n_allowed >= 2;
+  return (exit_policy_is_general_exit_helper(policy, 80) &&
+          exit_policy_is_general_exit_helper(policy, 443));
 }
 
 /** Return false if <b>policy</b> might permit access to some addr:port;
@@ -2306,7 +2408,7 @@ policy_summary_item_split(policy_summary_item_t* old, uint16_t new_starts)
 #define REJECT_CUTOFF_SCALE_IPV4 (0)
 /* Ports are rejected in an IPv4 summary if they are rejected in more than two
  * IPv4 /8 address blocks */
-#define REJECT_CUTOFF_COUNT_IPV4 (U64_LITERAL(1) << \
+#define REJECT_CUTOFF_COUNT_IPV4 (UINT64_C(1) << \
                                   (IPV4_BITS - REJECT_CUTOFF_SCALE_IPV4 - 7))
 
 #define IPV6_BITS                (128)
@@ -2314,11 +2416,11 @@ policy_summary_item_split(policy_summary_item_t* old, uint16_t new_starts)
 #define REJECT_CUTOFF_SCALE_IPV6 (64)
 /* Ports are rejected in an IPv6 summary if they are rejected in more than one
  * IPv6 /16 address block.
- * This is rougly equivalent to the IPv4 cutoff, as only five IPv6 /12s (and
+ * This is roughly equivalent to the IPv4 cutoff, as only five IPv6 /12s (and
  * some scattered smaller blocks) have been allocated to the RIRs.
  * Network providers are typically allocated one or more IPv6 /32s.
  */
-#define REJECT_CUTOFF_COUNT_IPV6 (U64_LITERAL(1) << \
+#define REJECT_CUTOFF_COUNT_IPV6 (UINT64_C(1) << \
                                   (IPV6_BITS - REJECT_CUTOFF_SCALE_IPV6 - 16))
 
 /** Split an exit policy summary so that prt_min and prt_max
@@ -2413,7 +2515,7 @@ policy_summary_reject(smartlist_t *summary,
      * in the range. */
     count = UINT64_MAX;
   } else {
-    count = (U64_LITERAL(1) << (addrbits - scale - maskbits));
+    count = (UINT64_C(1) << (addrbits - scale - maskbits));
   }
   tor_assert_nonfatal_once(count > 0);
   while (i < smartlist_len(summary) &&
@@ -2528,9 +2630,9 @@ policy_summarize(smartlist_t *policy, sa_family_t family)
         tor_snprintf(buf, sizeof(buf), "%d-%d", start_prt, AT(i)->prt_max);
 
       if (AT(i)->accepted)
-        smartlist_add(accepts, tor_strdup(buf));
+        smartlist_add_strdup(accepts, buf);
       else
-        smartlist_add(rejects, tor_strdup(buf));
+        smartlist_add_strdup(rejects, buf);
 
       if (last)
         break;
@@ -2679,7 +2781,7 @@ parse_short_policy(const char *summary)
   }
 
   {
-    size_t size = STRUCT_OFFSET(short_policy_t, entries) +
+    size_t size = offsetof(short_policy_t, entries) +
       sizeof(short_policy_entry_t)*(n_entries);
     result = tor_malloc_zero(size);
 
@@ -2710,7 +2812,7 @@ write_short_policy(const short_policy_t *policy)
       smartlist_add_asprintf(sl, "%d-%d", e->min_port, e->max_port);
     }
     if (i < policy->n_entries-1)
-      smartlist_add(sl, tor_strdup(","));
+      smartlist_add_strdup(sl, ",");
   }
   answer = smartlist_join_strings(sl, "", 0, NULL);
   SMARTLIST_FOREACH(sl, char *, a, tor_free(a));
@@ -2720,7 +2822,7 @@ write_short_policy(const short_policy_t *policy)
 
 /** Release all storage held in <b>policy</b>. */
 void
-short_policy_free(short_policy_t *policy)
+short_policy_free_(short_policy_t *policy)
 {
   tor_free(policy);
 }
@@ -2904,11 +3006,12 @@ getinfo_helper_policies(control_connection_t *conn,
     smartlist_free(private_policy_strings);
   } else if (!strcmp(question, "exit-policy/reject-private/relay")) {
     const or_options_t *options = get_options();
-    const routerinfo_t *me = router_get_my_routerinfo();
+    int err = 0;
+    const routerinfo_t *me = router_get_my_routerinfo_with_err(&err);
 
     if (!me) {
-      *errmsg = "router_get_my_routerinfo returned NULL";
-      return -1;
+      *errmsg = routerinfo_err_to_string(err);
+      return routerinfo_err_is_transient(err) ? -1 : 0;
     }
 
     if (!options->ExitPolicyRejectPrivate &&
@@ -2943,11 +3046,17 @@ getinfo_helper_policies(control_connection_t *conn,
     SMARTLIST_FOREACH(configured_addresses, tor_addr_t *, a, tor_free(a));
     smartlist_free(configured_addresses);
   } else if (!strcmpstart(question, "exit-policy/")) {
-    const routerinfo_t *me = router_get_my_routerinfo();
-
     int include_ipv4 = 0;
     int include_ipv6 = 0;
 
+    int err = 0;
+    const routerinfo_t *me = router_get_my_routerinfo_with_err(&err);
+
+    if (!me) {
+      *errmsg = routerinfo_err_to_string(err);
+      return routerinfo_err_is_transient(err) ? -1 : 0;
+    }
+
     if (!strcmp(question, "exit-policy/ipv4")) {
       include_ipv4 = 1;
     } else if (!strcmp(question, "exit-policy/ipv6")) {
@@ -2958,19 +3067,16 @@ getinfo_helper_policies(control_connection_t *conn,
       return 0; /* No such key. */
     }
 
-    if (!me) {
-      *errmsg = "router_get_my_routerinfo returned NULL";
-      return -1;
-    }
-
-    *answer = router_dump_exit_policy_to_string(me,include_ipv4,include_ipv6);
+    *answer = router_dump_exit_policy_to_string(me,include_ipv4,
+                                                include_ipv6);
   }
+
   return 0;
 }
 
 /** Release all storage held by <b>p</b>. */
 void
-addr_policy_list_free(smartlist_t *lst)
+addr_policy_list_free_(smartlist_t *lst)
 {
   if (!lst)
     return;
@@ -2980,7 +3086,7 @@ addr_policy_list_free(smartlist_t *lst)
 
 /** Release all storage held by <b>p</b>. */
 void
-addr_policy_free(addr_policy_t *p)
+addr_policy_free_(addr_policy_t *p)
 {
   if (!p)
     return;
@@ -3037,4 +3143,3 @@ policies_free_all(void)
   }
   HT_CLEAR(policy_map, &policy_root);
 }
-
diff --git a/src/or/policies.h b/src/core/or/policies.h
index f73f850c21..2c38de362f 100644
--- a/src/or/policies.h
+++ b/src/core/or/policies.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,7 +22,8 @@
 #define EXIT_POLICY_REJECT_PRIVATE           (1 << 1)
 #define EXIT_POLICY_ADD_DEFAULT              (1 << 2)
 #define EXIT_POLICY_REJECT_LOCAL_INTERFACES  (1 << 3)
-#define EXIT_POLICY_OPTION_MAX             EXIT_POLICY_REJECT_LOCAL_INTERFACES
+#define EXIT_POLICY_ADD_REDUCED              (1 << 4)
+#define EXIT_POLICY_OPTION_MAX             EXIT_POLICY_ADD_REDUCED
 /* All options set: used for unit testing */
 #define EXIT_POLICY_OPTION_ALL             ((EXIT_POLICY_OPTION_MAX << 1) - 1)
 
@@ -33,6 +34,39 @@ typedef enum firewall_connection_t {
 
 typedef int exit_policy_parser_cfg_t;
 
+/** Outcome of applying an address policy to an address. */
+typedef enum {
+  /** The address was accepted */
+  ADDR_POLICY_ACCEPTED=0,
+  /** The address was rejected */
+  ADDR_POLICY_REJECTED=-1,
+  /** Part of the address was unknown, but as far as we can tell, it was
+   * accepted. */
+  ADDR_POLICY_PROBABLY_ACCEPTED=1,
+  /** Part of the address was unknown, but as far as we can tell, it was
+   * rejected. */
+  ADDR_POLICY_PROBABLY_REJECTED=2,
+} addr_policy_result_t;
+
+/** A single entry in a parsed policy summary, describing a range of ports. */
+typedef struct short_policy_entry_t {
+  uint16_t min_port, max_port;
+} short_policy_entry_t;
+
+/** A short_poliy_t is the parsed version of a policy summary. */
+typedef struct short_policy_t {
+  /** True if the members of 'entries' are port ranges to accept; false if
+   * they are port ranges to reject */
+  unsigned int is_accept : 1;
+  /** The actual number of values in 'entries'. */
+  unsigned int n_entries : 31;
+  /** An array of 0 or more short_policy_entry_t values, each describing a
+   * range of ports that this policy accepts or rejects (depending on the
+   * value of is_accept).
+   */
+  short_policy_entry_t entries[FLEXIBLE_ARRAY_MEMBER];
+} short_policy_t;
+
 int firewall_is_fascist_or(void);
 int firewall_is_fascist_dir(void);
 int fascist_firewall_use_ipv6(const or_options_t *options);
@@ -54,13 +88,13 @@ int fascist_firewall_allows_dir_server(const dir_server_t *ds,
                                        firewall_connection_t fw_connection,
                                        int pref_only);
 
-int fascist_firewall_choose_address_rs(const routerstatus_t *rs,
-                                       firewall_connection_t fw_connection,
-                                       int pref_only, tor_addr_port_t* ap);
-int fascist_firewall_choose_address_node(const node_t *node,
-                                         firewall_connection_t fw_connection,
-                                         int pref_only, tor_addr_port_t* ap);
-int fascist_firewall_choose_address_dir_server(const dir_server_t *ds,
+void fascist_firewall_choose_address_rs(const routerstatus_t *rs,
+                                        firewall_connection_t fw_connection,
+                                        int pref_only, tor_addr_port_t* ap);
+void fascist_firewall_choose_address_node(const node_t *node,
+                                          firewall_connection_t fw_connection,
+                                          int pref_only, tor_addr_port_t* ap);
+void fascist_firewall_choose_address_dir_server(const dir_server_t *ds,
                                           firewall_connection_t fw_connection,
                                           int pref_only, tor_addr_port_t* ap);
 
@@ -87,7 +121,8 @@ int policies_parse_exit_policy_from_options(
                                           uint32_t local_address,
                                           const tor_addr_t *ipv6_local_address,
                                           smartlist_t **result);
-int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest,
+struct config_line_t;
+int policies_parse_exit_policy(struct config_line_t *cfg, smartlist_t **dest,
                                exit_policy_parser_cfg_t options,
                                const smartlist_t *configured_addresses);
 void policies_parse_exit_policy_reject_private(
@@ -114,15 +149,21 @@ int getinfo_helper_policies(control_connection_t *conn,
 int policy_write_item(char *buf, size_t buflen, const addr_policy_t *item,
                       int format_for_desc);
 
-void addr_policy_list_free(smartlist_t *p);
-void addr_policy_free(addr_policy_t *p);
+void addr_policy_list_free_(smartlist_t *p);
+#define addr_policy_list_free(lst) \
+  FREE_AND_NULL(smartlist_t, addr_policy_list_free_, (lst))
+void addr_policy_free_(addr_policy_t *p);
+#define addr_policy_free(p) \
+  FREE_AND_NULL(addr_policy_t, addr_policy_free_, (p))
 void policies_free_all(void);
 
 char *policy_summarize(smartlist_t *policy, sa_family_t family);
 
 short_policy_t *parse_short_policy(const char *summary);
 char *write_short_policy(const short_policy_t *policy);
-void short_policy_free(short_policy_t *policy);
+void short_policy_free_(short_policy_t *policy);
+#define short_policy_free(p) \
+  FREE_AND_NULL(short_policy_t, short_policy_free_, (p))
 int short_policy_is_reject_star(const short_policy_t *policy);
 addr_policy_result_t compare_tor_addr_to_short_policy(
                           const tor_addr_t *addr, uint16_t port,
@@ -141,7 +182,6 @@ STATIC const tor_addr_port_t * fascist_firewall_choose_address(
                                           firewall_connection_t fw_connection,
                                           int pref_only, int pref_ipv6);
 
-#endif
-
-#endif
+#endif /* defined(POLICIES_PRIVATE) */
 
+#endif /* !defined(TOR_POLICIES_H) */
diff --git a/src/core/or/port_cfg_st.h b/src/core/or/port_cfg_st.h
new file mode 100644
index 0000000000..b67091ce32
--- /dev/null
+++ b/src/core/or/port_cfg_st.h
@@ -0,0 +1,35 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef PORT_CFG_ST_H
+#define PORT_CFG_ST_H
+
+#include "core/or/entry_port_cfg_st.h"
+#include "core/or/server_port_cfg_st.h"
+
+/** Configuration for a single port that we're listening on. */
+struct port_cfg_t {
+  tor_addr_t addr; /**< The actual IP to listen on, if !is_unix_addr. */
+  int port; /**< The configured port, or CFG_AUTO_PORT to tell Tor to pick its
+             * own port. */
+  uint8_t type; /**< One of CONN_TYPE_*_LISTENER */
+  unsigned is_unix_addr : 1; /**< True iff this is an AF_UNIX address. */
+
+  unsigned is_group_writable : 1;
+  unsigned is_world_writable : 1;
+  unsigned relax_dirmode_check : 1;
+
+  entry_port_cfg_t entry_cfg;
+
+  server_port_cfg_t server_cfg;
+
+  /* Unix sockets only: */
+  /** Path for an AF_UNIX address */
+  char unix_addr[FLEXIBLE_ARRAY_MEMBER];
+};
+
+#endif
+
diff --git a/src/or/protover.c b/src/core/or/protover.c
index 2c5d5ab1fc..17979d04ea 100644
--- a/src/or/protover.c
+++ b/src/core/or/protover.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -23,16 +23,19 @@
 
 #define PROTOVER_PRIVATE
 
-#include "compat.h"
-#include "or.h"
-#include "protover.h"
-#include "routerparse.h"
+#include "core/or/or.h"
+#include "core/or/protover.h"
+#include "core/or/versions.h"
+#include "lib/tls/tortls.h"
+
+#ifndef HAVE_RUST
 
 static const smartlist_t *get_supported_protocol_list(void);
 static int protocol_list_contains(const smartlist_t *protos,
                                   protocol_type_t pr, uint32_t ver);
 
 /** Mapping between protocol type string and protocol type. */
+/// C_RUST_COUPLED: src/rust/protover/protover.rs `PROTOCOL_NAMES`
 static const struct {
   protocol_type_t protover_type;
   const char *name;
@@ -51,6 +54,11 @@ static const struct {
 
 #define N_PROTOCOL_NAMES ARRAY_LENGTH(PROTOCOL_NAMES)
 
+/* Maximum allowed length of any single subprotocol name. */
+// C_RUST_COUPLED: src/rust/protover/protover.rs
+//                 `MAX_PROTOCOL_NAME_LENGTH`
+static const unsigned MAX_PROTOCOL_NAME_LENGTH = 100;
+
 /**
  * Given a protocol_type_t, return the corresponding string used in
  * descriptors.
@@ -94,7 +102,7 @@ str_to_protocol_type(const char *s, protocol_type_t *pr_out)
  * Release all space held by a single proto_entry_t structure
  */
 STATIC void
-proto_entry_free(proto_entry_t *entry)
+proto_entry_free_(proto_entry_t *entry)
 {
   if (!entry)
     return;
@@ -206,6 +214,16 @@ parse_single_entry(const char *s, const char *end_of_entry)
   if (equals == s)
     goto error;
 
+  /* The name must not be longer than MAX_PROTOCOL_NAME_LENGTH. */
+  if (equals - s > (int)MAX_PROTOCOL_NAME_LENGTH) {
+    log_warn(LD_NET, "When parsing a protocol entry, I got a very large "
+             "protocol name. This is possibly an attack or a bug, unless "
+             "the Tor network truly supports protocol names larger than "
+             "%ud characters. The offending string was: %s",
+             MAX_PROTOCOL_NAME_LENGTH, escaped(out->name));
+    goto error;
+  }
+
   /* The name must contain only alphanumeric characters and hyphens. */
   if (!is_valid_keyword(s, equals-s))
     goto error;
@@ -275,6 +293,21 @@ parse_protocol_list(const char *s)
 }
 
 /**
+ * Return true if the unparsed protover in <b>s</b> would contain a protocol
+ * name longer than MAX_PROTOCOL_NAME_LENGTH, and false otherwise.
+ */
+bool
+protover_contains_long_protocol_names(const char *s)
+{
+  smartlist_t *list = parse_protocol_list(s);
+  if (!list)
+    return true; /* yes, has a dangerous name */
+  SMARTLIST_FOREACH(list, proto_entry_t *, ent, proto_entry_free(ent));
+  smartlist_free(list);
+  return false; /* no, looks fine */
+}
+
+/**
  * Given a protocol type and version number, return true iff we know
  * how to speak that protocol.
  */
@@ -307,20 +340,61 @@ protocol_list_supports_protocol(const char *list, protocol_type_t tp,
   return contains;
 }
 
+/**
+ * Return true iff "list" encodes a protocol list that includes support for
+ * the indicated protocol and version, or some later version.
+ */
+int
+protocol_list_supports_protocol_or_later(const char *list,
+                                         protocol_type_t tp,
+                                         uint32_t version)
+{
+  /* NOTE: This is a pretty inefficient implementation. If it ever shows
+   * up in profiles, we should memoize it.
+   */
+  smartlist_t *protocols = parse_protocol_list(list);
+  if (!protocols) {
+    return 0;
+  }
+  const char *pr_name = protocol_type_to_str(tp);
+
+  int contains = 0;
+  SMARTLIST_FOREACH_BEGIN(protocols, proto_entry_t *, proto) {
+    if (strcasecmp(proto->name, pr_name))
+      continue;
+    SMARTLIST_FOREACH_BEGIN(proto->ranges, const proto_range_t *, range) {
+      if (range->high >= version) {
+        contains = 1;
+        goto found;
+      }
+    } SMARTLIST_FOREACH_END(range);
+  } SMARTLIST_FOREACH_END(proto);
+
+ found:
+  SMARTLIST_FOREACH(protocols, proto_entry_t *, ent, proto_entry_free(ent));
+  smartlist_free(protocols);
+  return contains;
+}
+
 /** Return the canonical string containing the list of protocols
  * that we support. */
+/// C_RUST_COUPLED: src/rust/protover/protover.rs `SUPPORTED_PROTOCOLS`
 const char *
 protover_get_supported_protocols(void)
 {
   return
     "Cons=1-2 "
     "Desc=1-2 "
-    "DirCache=1 "
-    "HSDir=1 "
-    "HSIntro=3 "
+    "DirCache=1-2 "
+    "HSDir=1-2 "
+    "HSIntro=3-4 "
     "HSRend=1-2 "
-    "Link=1-4 "
-    "LinkAuth=1 "
+    "Link=1-5 "
+#ifdef HAVE_WORKING_TOR_TLS_GET_TLSSECRETS
+    "LinkAuth=1,3 "
+#else
+    "LinkAuth=3 "
+#endif
     "Microdesc=1-2 "
     "Relay=1-2";
 }
@@ -375,7 +449,7 @@ encode_protocol_list(const smartlist_t *sl)
   const char *separator = "";
   smartlist_t *chunks = smartlist_new();
   SMARTLIST_FOREACH_BEGIN(sl, const proto_entry_t *, ent) {
-    smartlist_add(chunks, tor_strdup(separator));
+    smartlist_add_strdup(chunks, separator);
 
     proto_entry_encode_into(chunks, ent);
 
@@ -392,6 +466,8 @@ encode_protocol_list(const smartlist_t *sl)
 
 /* We treat any protocol list with more than this many subprotocols in it
  * as a DoS attempt. */
+/// C_RUST_COUPLED: src/rust/protover/protover.rs
+///                 `MAX_PROTOCOLS_TO_EXPAND`
 static const int MAX_PROTOCOLS_TO_EXPAND = (1<<16);
 
 /** Voting helper: Given a list of proto_entry_t, return a newly allocated
@@ -412,6 +488,14 @@ expand_protocol_list(const smartlist_t *protos)
 
   SMARTLIST_FOREACH_BEGIN(protos, const proto_entry_t *, ent) {
     const char *name = ent->name;
+    if (strlen(name) > MAX_PROTOCOL_NAME_LENGTH) {
+      log_warn(LD_NET, "When expanding a protocol entry, I got a very large "
+               "protocol name. This is possibly an attack or a bug, unless "
+               "the Tor network truly supports protocol names larger than "
+               "%ud characters. The offending string was: %s",
+               MAX_PROTOCOL_NAME_LENGTH, escaped(name));
+      continue;
+    }
     SMARTLIST_FOREACH_BEGIN(ent->ranges, const proto_range_t *, range) {
       uint32_t u;
       for (u = range->low; u <= range->high; ++u) {
@@ -508,7 +592,7 @@ contract_protocol_list(const smartlist_t *proto_strings)
     smartlist_sort(lst, cmp_single_ent_by_version);
 
     if (! first_entry)
-      smartlist_add(chunks, tor_strdup(" "));
+      smartlist_add_strdup(chunks, " ");
 
     /* We're going to construct this entry from the ranges. */
     proto_entry_t *entry = tor_malloc_zero(sizeof(proto_entry_t));
@@ -657,7 +741,9 @@ int
 protover_all_supported(const char *s, char **missing_out)
 {
   int all_supported = 1;
-  smartlist_t *missing;
+  smartlist_t *missing_some;
+  smartlist_t *missing_completely;
+  smartlist_t *missing_all;
 
   if (!s) {
     return 1;
@@ -670,7 +756,8 @@ protover_all_supported(const char *s, char **missing_out)
     return 1;
   }
 
-  missing = smartlist_new();
+  missing_some = smartlist_new();
+  missing_completely = smartlist_new();
 
   SMARTLIST_FOREACH_BEGIN(entries, const proto_entry_t *, ent) {
     protocol_type_t tp;
@@ -682,26 +769,86 @@ protover_all_supported(const char *s, char **missing_out)
     }
 
     SMARTLIST_FOREACH_BEGIN(ent->ranges, const proto_range_t *, range) {
+      proto_entry_t *unsupported = tor_malloc_zero(sizeof(proto_entry_t));
+      proto_range_t *versions = tor_malloc_zero(sizeof(proto_range_t));
       uint32_t i;
+
+      unsupported->name = tor_strdup(ent->name);
+      unsupported->ranges = smartlist_new();
+
       for (i = range->low; i <= range->high; ++i) {
         if (!protover_is_supported_here(tp, i)) {
-          goto unsupported;
+          if (versions->low == 0 && versions->high == 0) {
+            versions->low = i;
+            /* Pre-emptively add the high now, just in case we're in a single
+             * version range (e.g. "Link=999"). */
+            versions->high = i;
+          }
+          /* If the last one to be unsupported is one less than the current
+           * one, we're in a continuous range, so set the high field. */
+          if ((versions->high && versions->high == i - 1) ||
+              /* Similarly, if the last high wasn't set and we're currently
+               * one higher than the low, add current index as the highest
+               * known high. */
+              (!versions->high && versions->low == i - 1)) {
+            versions->high = i;
+            continue;
+          }
+        } else {
+          /* If we hit a supported version, and we previously had a range,
+           * we've hit a non-continuity. Copy the previous range and add it to
+           * the unsupported->ranges list and zero-out the previous range for
+           * the next iteration. */
+          if (versions->low != 0 && versions->high != 0) {
+            proto_range_t *versions_to_add = tor_malloc(sizeof(proto_range_t));
+
+            versions_to_add->low = versions->low;
+            versions_to_add->high = versions->high;
+            smartlist_add(unsupported->ranges, versions_to_add);
+
+            versions->low = 0;
+            versions->high = 0;
+          }
         }
       }
+      /* Once we've run out of versions to check, see if we had any unsupported
+       * ones and, if so, add them to unsupported->ranges. */
+      if (versions->low != 0 && versions->high != 0) {
+        smartlist_add(unsupported->ranges, versions);
+      }
+      /* Finally, if we had something unsupported, add it to the list of
+       * missing_some things and mark that there was something missing. */
+      if (smartlist_len(unsupported->ranges) != 0) {
+        smartlist_add(missing_some, (void*) unsupported);
+        all_supported = 0;
+      } else {
+        proto_entry_free(unsupported);
+        tor_free(versions);
+      }
     } SMARTLIST_FOREACH_END(range);
 
     continue;
 
   unsupported:
     all_supported = 0;
-    smartlist_add(missing, (void*) ent);
+    smartlist_add(missing_completely, (void*) ent);
   } SMARTLIST_FOREACH_END(ent);
 
+  /* We keep the two smartlists separate so that we can free the proto_entry_t
+   * we created and put in missing_some, so here we add them together to build
+   * the string. */
+  missing_all = smartlist_new();
+  smartlist_add_all(missing_all, missing_some);
+  smartlist_add_all(missing_all, missing_completely);
+
   if (missing_out && !all_supported) {
-    tor_assert(0 != smartlist_len(missing));
-    *missing_out = encode_protocol_list(missing);
+    tor_assert(smartlist_len(missing_all) != 0);
+    *missing_out = encode_protocol_list(missing_all);
   }
-  smartlist_free(missing);
+  SMARTLIST_FOREACH(missing_some, proto_entry_t *, ent, proto_entry_free(ent));
+  smartlist_free(missing_some);
+  smartlist_free(missing_completely);
+  smartlist_free(missing_all);
 
   SMARTLIST_FOREACH(entries, proto_entry_t *, ent, proto_entry_free(ent));
   smartlist_free(entries);
@@ -742,6 +889,7 @@ protocol_list_contains(const smartlist_t *protos,
  * Note that this is only used to infer protocols for Tor versions that
  * can't declare their own.
  **/
+/// C_RUST_COUPLED: src/rust/protover/protover.rs `compute_for_old_tor`
 const char *
 protover_compute_for_old_tor(const char *version)
 {
@@ -791,3 +939,4 @@ protover_free_all(void)
   }
 }
 
+#endif /* !defined(HAVE_RUST) */
diff --git a/src/or/protover.h b/src/core/or/protover.h
index 5c658931ea..7e181ba97a 100644
--- a/src/or/protover.h
+++ b/src/core/or/protover.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,15 +9,29 @@
 #ifndef TOR_PROTOVER_H
 #define TOR_PROTOVER_H
 
-#include "container.h"
+#include <stdbool.h>
+#include "lib/cc/torint.h"
+#include "lib/testsupport/testsupport.h"
+struct smartlist_t;
 
 /** The first version of Tor that included "proto" entries in its
  * descriptors.  Authorities should use this to decide whether to
  * guess proto lines. */
 /* This is a guess. */
+/// C_RUST_COUPLED: src/rust/protover/protover.rs
+///                 `FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS`
 #define FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS "0.2.9.3-alpha"
 
+/** The protover version number that signifies HSDir support for HSv3 */
+#define PROTOVER_HSDIR_V3 2
+/** The protover version number that signifies HSv3 intro point support */
+#define PROTOVER_HS_INTRO_V3 4
+/** The protover version number that signifies HSv3 rendezvous point support */
+#define PROTOVER_HS_RENDEZVOUS_POINT_V3 2
+
 /** List of recognized subprotocols. */
+/// C_RUST_COUPLED: src/rust/protover/ffi.rs `translate_to_rust`
+/// C_RUST_COUPLED: src/rust/protover/protover.rs `Proto`
 typedef enum protocol_type_t {
   PRT_LINK,
   PRT_LINKAUTH,
@@ -31,15 +45,19 @@ typedef enum protocol_type_t {
   PRT_CONS,
 } protocol_type_t;
 
+bool protover_contains_long_protocol_names(const char *s);
 int protover_all_supported(const char *s, char **missing);
 int protover_is_supported_here(protocol_type_t pr, uint32_t ver);
 const char *protover_get_supported_protocols(void);
 
-char *protover_compute_vote(const smartlist_t *list_of_proto_strings,
+char *protover_compute_vote(const struct smartlist_t *list_of_proto_strings,
                             int threshold);
 const char *protover_compute_for_old_tor(const char *version);
 int protocol_list_supports_protocol(const char *list, protocol_type_t tp,
                                     uint32_t version);
+int protocol_list_supports_protocol_or_later(const char *list,
+                                             protocol_type_t tp,
+                                             uint32_t version);
 
 void protover_free_all(void);
 
@@ -60,15 +78,20 @@ typedef struct proto_entry_t {
    */
   char *name;
   /** Smartlist of proto_range_t */
-  smartlist_t *ranges;
+  struct smartlist_t *ranges;
 } proto_entry_t;
 
-STATIC smartlist_t *parse_protocol_list(const char *s);
-STATIC void proto_entry_free(proto_entry_t *entry);
-STATIC char *encode_protocol_list(const smartlist_t *sl);
+#if !defined(HAVE_RUST) && defined(TOR_UNIT_TESTS)
+STATIC struct smartlist_t *parse_protocol_list(const char *s);
+STATIC char *encode_protocol_list(const struct smartlist_t *sl);
 STATIC const char *protocol_type_to_str(protocol_type_t pr);
 STATIC int str_to_protocol_type(const char *s, protocol_type_t *pr_out);
-#endif
+STATIC void proto_entry_free_(proto_entry_t *entry);
+#endif /* !defined(HAVE_RUST) && defined(TOR_UNIT_TESTS) */
+
+#define proto_entry_free(entry) \
+  FREE_AND_NULL(proto_entry_t, proto_entry_free_, (entry))
 
-#endif
+#endif /* defined(PROTOVER_PRIVATE) */
 
+#endif /* !defined(TOR_PROTOVER_H) */
diff --git a/src/core/or/protover_rust.c b/src/core/or/protover_rust.c
new file mode 100644
index 0000000000..bc56ea11d0
--- /dev/null
+++ b/src/core/or/protover_rust.c
@@ -0,0 +1,34 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/*
+ * \file protover_rust.c
+ * \brief Provide a C wrapper for functions exposed in /src/rust/protover,
+ * and safe translation/handling between the Rust/C boundary.
+ */
+
+#include "core/or/or.h"
+#include "core/or/protover.h"
+
+#ifdef HAVE_RUST
+
+/* Define for compatibility, used in main.c */
+void
+protover_free_all(void)
+{
+}
+
+int protover_contains_long_protocol_names_(const char *s);
+
+/**
+ * Return true if the unparsed protover in <b>s</b> would contain a protocol
+ * name longer than MAX_PROTOCOL_NAME_LENGTH, and false otherwise.
+ */
+bool
+protover_contains_long_protocol_names(const char *s)
+{
+  return protover_contains_long_protocol_names_(s) != 0;
+}
+
+#endif /* defined(HAVE_RUST) */
+
diff --git a/src/or/reasons.c b/src/core/or/reasons.c
index a1566e2299..a7952279ba 100644
--- a/src/or/reasons.c
+++ b/src/core/or/reasons.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,9 +14,11 @@
  * to another.
  **/
 
-#include "or.h"
-#include "config.h"
-#include "reasons.h"
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/or/reasons.h"
+#include "feature/nodelist/node_select.h"
+#include "lib/tls/tortls.h"
 
 /***************************** Edge (stream) reasons **********************/
 
@@ -45,6 +47,8 @@ stream_end_reason_to_control_string(int reason)
     case END_STREAM_REASON_CANT_ATTACH: return "CANT_ATTACH";
     case END_STREAM_REASON_NET_UNREACHABLE: return "NET_UNREACHABLE";
     case END_STREAM_REASON_SOCKSPROTOCOL: return "SOCKS_PROTOCOL";
+    // XXXX Controlspec
+    case END_STREAM_REASON_HTTPPROTOCOL: return "HTTP_PROTOCOL";
 
     case END_STREAM_REASON_PRIVATE_ADDR: return "PRIVATE_ADDR";
 
@@ -138,6 +142,11 @@ stream_end_reason_to_socks5_response(int reason)
       return SOCKS5_NET_UNREACHABLE;
     case END_STREAM_REASON_SOCKSPROTOCOL:
       return SOCKS5_GENERAL_ERROR;
+    case END_STREAM_REASON_HTTPPROTOCOL:
+      // LCOV_EXCL_START
+      tor_assert_nonfatal_unreached();
+      return SOCKS5_GENERAL_ERROR;
+      // LCOV_EXCL_STOP
     case END_STREAM_REASON_PRIVATE_ADDR:
       return SOCKS5_GENERAL_ERROR;
 
@@ -160,7 +169,7 @@ stream_end_reason_to_socks5_response(int reason)
 #else
 #define E_CASE(s) case s
 #define S_CASE(s) case s
-#endif
+#endif /* defined(_WIN32) */
 
 /** Given an errno from a failed exit connection, return a reason code
  * appropriate for use in a RELAY END cell. */
@@ -421,7 +430,7 @@ socks5_response_code_to_string(uint8_t code)
   }
 }
 
-/** Return a string corresponding to a bandwidht_weight_rule_t */
+/** Return a string corresponding to a bandwidth_weight_rule_t */
 const char *
 bandwidth_weight_rule_to_string(bandwidth_weight_rule_t rule)
 {
@@ -442,3 +451,47 @@ bandwidth_weight_rule_to_string(bandwidth_weight_rule_t rule)
   }
 }
 
+/** Given a RELAY_END reason value, convert it to an HTTP response to be
+ * send over an HTTP tunnel connection. */
+const char *
+end_reason_to_http_connect_response_line(int endreason)
+{
+  endreason &= END_STREAM_REASON_MASK;
+  /* XXXX these are probably all wrong. Should they all be 502? */
+  switch (endreason) {
+    case 0:
+      return "HTTP/1.0 200 OK\r\n\r\n";
+    case END_STREAM_REASON_MISC:
+      return "HTTP/1.0 500 Internal Server Error\r\n\r\n";
+    case END_STREAM_REASON_RESOLVEFAILED:
+      return "HTTP/1.0 404 Not Found (resolve failed)\r\n\r\n";
+    case END_STREAM_REASON_NOROUTE:
+      return "HTTP/1.0 404 Not Found (no route)\r\n\r\n";
+    case END_STREAM_REASON_CONNECTREFUSED:
+      return "HTTP/1.0 403 Forbidden (connection refused)\r\n\r\n";
+    case END_STREAM_REASON_EXITPOLICY:
+      return "HTTP/1.0 403 Forbidden (exit policy)\r\n\r\n";
+    case END_STREAM_REASON_DESTROY:
+      return "HTTP/1.0 502 Bad Gateway (destroy cell received)\r\n\r\n";
+    case END_STREAM_REASON_DONE:
+      return "HTTP/1.0 502 Bad Gateway (unexpected close)\r\n\r\n";
+    case END_STREAM_REASON_TIMEOUT:
+      return "HTTP/1.0 504 Gateway Timeout\r\n\r\n";
+    case END_STREAM_REASON_HIBERNATING:
+      return "HTTP/1.0 502 Bad Gateway (hibernating server)\r\n\r\n";
+    case END_STREAM_REASON_INTERNAL:
+      return "HTTP/1.0 502 Bad Gateway (internal error)\r\n\r\n";
+    case END_STREAM_REASON_RESOURCELIMIT:
+      return "HTTP/1.0 502 Bad Gateway (resource limit)\r\n\r\n";
+    case END_STREAM_REASON_CONNRESET:
+      return "HTTP/1.0 403 Forbidden (connection reset)\r\n\r\n";
+    case END_STREAM_REASON_TORPROTOCOL:
+      return "HTTP/1.0 502 Bad Gateway (tor protocol violation)\r\n\r\n";
+    case END_STREAM_REASON_ENTRYPOLICY:
+      return "HTTP/1.0 403 Forbidden (entry policy violation)\r\n\r\n";
+    case END_STREAM_REASON_NOTDIRECTORY: /* Fall Through */
+    default:
+      tor_assert_nonfatal_unreached();
+      return "HTTP/1.0 500 Internal Server Error (weird end reason)\r\n\r\n";
+  }
+}
diff --git a/src/or/reasons.h b/src/core/or/reasons.h
index 2e12c93728..c45a8bc38d 100644
--- a/src/or/reasons.h
+++ b/src/core/or/reasons.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,6 +12,9 @@
 #ifndef TOR_REASONS_H
 #define TOR_REASONS_H
 
+#include "lib/net/socks5_status.h"
+enum bandwidth_weight_rule_t;
+
 const char *stream_end_reason_to_control_string(int reason);
 const char *stream_end_reason_to_string(int reason);
 socks5_reply_status_t stream_end_reason_to_socks5_response(int reason);
@@ -26,6 +29,6 @@ const char *socks4_response_code_to_string(uint8_t code);
 const char *socks5_response_code_to_string(uint8_t code);
 
 const char *bandwidth_weight_rule_to_string(enum bandwidth_weight_rule_t rule);
+const char *end_reason_to_http_connect_response_line(int endreason);
 
-#endif
-
+#endif /* !defined(TOR_REASONS_H) */
diff --git a/src/or/relay.c b/src/core/or/relay.c
index d1c7820c7c..dc88a6f649 100644
--- a/src/or/relay.c
+++ b/src/core/or/relay.c
@@ -1,51 +1,105 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file relay.c
  * \brief Handle relay cell encryption/decryption, plus packaging and
  *    receiving from circuits, plus queuing on circuits.
+ *
+ * This is a core modules that makes Tor work. It's responsible for
+ * dealing with RELAY cells (the ones that travel more than one hop along a
+ * circuit), by:
+ *  <ul>
+ *   <li>constructing relays cells,
+ *   <li>encrypting relay cells,
+ *   <li>decrypting relay cells,
+ *   <li>demultiplexing relay cells as they arrive on a connection,
+ *   <li>queueing relay cells for retransmission,
+ *   <li>or handling relay cells that are for us to receive (as an exit or a
+ *   client).
+ *  </ul>
+ *
+ * RELAY cells are generated throughout the code at the client or relay side,
+ * using relay_send_command_from_edge() or one of the functions like
+ * connection_edge_send_command() that calls it.  Of particular interest is
+ * connection_edge_package_raw_inbuf(), which takes information that has
+ * arrived on an edge connection socket, and packages it as a RELAY_DATA cell
+ * -- this is how information is actually sent across the Tor network.  The
+ * cryptography for these functions is handled deep in
+ * circuit_package_relay_cell(), which either adds a single layer of
+ * encryption (if we're an exit), or multiple layers (if we're the origin of
+ * the circuit).  After construction and encryption, the RELAY cells are
+ * passed to append_cell_to_circuit_queue(), which queues them for
+ * transmission and tells the circuitmux (see circuitmux.c) that the circuit
+ * is waiting to send something.
+ *
+ * Incoming RELAY cells arrive at circuit_receive_relay_cell(), called from
+ * command.c.  There they are decrypted and, if they are for us, are passed to
+ * connection_edge_process_relay_cell(). If they're not for us, they're
+ * re-queued for retransmission again with append_cell_to_circuit_queue().
+ *
+ * The connection_edge_process_relay_cell() function handles all the different
+ * types of relay cells, launching requests or transmitting data as needed.
  **/
 
 #define RELAY_PRIVATE
-#include "or.h"
-#include "addressmap.h"
-#include "buffers.h"
-#include "channel.h"
-#include "circpathbias.h"
-#include "circuitbuild.h"
-#include "circuitlist.h"
-#include "circuituse.h"
-#include "config.h"
-#include "connection.h"
-#include "connection_edge.h"
-#include "connection_or.h"
-#include "control.h"
-#include "geoip.h"
-#include "main.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "onion.h"
-#include "policies.h"
-#include "reasons.h"
-#include "relay.h"
-#include "rendcache.h"
-#include "rendcommon.h"
-#include "router.h"
-#include "routerlist.h"
-#include "routerparse.h"
-#include "scheduler.h"
+#include "core/or/or.h"
+#include "feature/client/addressmap.h"
+#include "lib/err/backtrace.h"
+#include "lib/container/buffers.h"
+#include "core/or/channel.h"
+#include "feature/client/circpathbias.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "lib/compress/compress.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "feature/control/control.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "feature/dircommon/directory.h"
+#include "feature/relay/dns.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/hs/hs_cache.h"
+#include "core/mainloop/mainloop.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "core/or/onion.h"
+#include "core/or/policies.h"
+#include "core/or/reasons.h"
+#include "core/or/relay.h"
+#include "core/crypto/relay_crypto.h"
+#include "feature/rend/rendcache.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/routerlist.h"
+#include "core/or/scheduler.h"
+#include "feature/stats/rephist.h"
+
+#include "core/or/cell_st.h"
+#include "core/or/cell_queue_st.h"
+#include "core/or/cpath_build_state_st.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "core/or/destroy_cell_queue_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/extend_info_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "core/or/socks_request_st.h"
+
+#include "lib/intmath/weakrng.h"
 
 static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell,
                                             cell_direction_t cell_direction,
                                             crypt_path_t *layer_hint);
 
-static int connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
-                                              edge_connection_t *conn,
-                                              crypt_path_t *layer_hint);
 static void circuit_consider_sending_sendme(circuit_t *circ,
                                             crypt_path_t *layer_hint);
 static void circuit_resume_edge_reading(circuit_t *circ,
@@ -60,9 +114,6 @@ static void adjust_exit_policy_from_exitpolicy_failure(origin_circuit_t *circ,
                                                   entry_connection_t *conn,
                                                   node_t *node,
                                                   const tor_addr_t *addr);
-#if 0
-static int get_max_middle_cells(void);
-#endif
 
 /** Stop reading on edge connections when we have this many cells
  * waiting on the appropriate queue. */
@@ -79,85 +130,87 @@ uint64_t stats_n_relay_cells_relayed = 0;
  * hop?
  */
 uint64_t stats_n_relay_cells_delivered = 0;
+/** Stats: how many circuits have we closed due to the cell queue limit being
+ * reached (see append_cell_to_circuit_queue()) */
+uint64_t stats_n_circ_max_cell_reached = 0;
 
 /** Used to tell which stream to read from first on a circuit. */
 static tor_weak_rng_t stream_choice_rng = TOR_WEAK_RNG_INIT;
 
-/** Update digest from the payload of cell. Assign integrity part to
- * cell.
+/**
+ * Update channel usage state based on the type of relay cell and
+ * circuit properties.
+ *
+ * This is needed to determine if a client channel is being
+ * used for application traffic, and if a relay channel is being
+ * used for multihop circuits and application traffic. The decision
+ * to pad in channelpadding.c depends upon this info (as well as
+ * consensus parameters) to decide what channels to pad.
  */
 static void
-relay_set_digest(crypto_digest_t *digest, cell_t *cell)
+circuit_update_channel_usage(circuit_t *circ, cell_t *cell)
 {
-  char integrity[4];
-  relay_header_t rh;
-
-  crypto_digest_add_bytes(digest, (char*)cell->payload, CELL_PAYLOAD_SIZE);
-  crypto_digest_get_digest(digest, integrity, 4);
-//  log_fn(LOG_DEBUG,"Putting digest of %u %u %u %u into relay cell.",
-//    integrity[0], integrity[1], integrity[2], integrity[3]);
-  relay_header_unpack(&rh, cell->payload);
-  memcpy(rh.integrity, integrity, 4);
-  relay_header_pack(cell->payload, &rh);
-}
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    /*
+     * The client state was first set much earlier in
+     * circuit_send_next_onion_skin(), so we can start padding as early as
+     * possible.
+     *
+     * However, if padding turns out to be expensive, we may want to not do
+     * it until actual application traffic starts flowing (which is controlled
+     * via consensus param nf_pad_before_usage).
+     *
+     * So: If we're an origin circuit and we've created a full length circuit,
+     * then any CELL_RELAY cell means application data. Increase the usage
+     * state of the channel to indicate this.
+     *
+     * We want to wait for CELL_RELAY specifically here, so we know that
+     * the channel was definitely being used for data and not for extends.
+     * By default, we pad as soon as a channel has been used for *any*
+     * circuits, so this state is irrelevant to the padding decision in
+     * the default case. However, if padding turns out to be expensive,
+     * we would like the ability to avoid padding until we're absolutely
+     * sure that a channel is used for enough application data to be worth
+     * padding.
+     *
+     * (So it does not matter that CELL_RELAY_EARLY can actually contain
+     * application data. This is only a load reducing option and that edge
+     * case does not matter if we're desperately trying to reduce overhead
+     * anyway. See also consensus parameter nf_pad_before_usage).
+     */
+    if (BUG(!circ->n_chan))
+      return;
 
-/** Does the digest for this circuit indicate that this cell is for us?
- *
- * Update digest from the payload of cell (with the integrity part set
- * to 0). If the integrity part is valid, return 1, else restore digest
- * and cell to their original state and return 0.
- */
-static int
-relay_digest_matches(crypto_digest_t *digest, cell_t *cell)
-{
-  uint32_t received_integrity, calculated_integrity;
-  relay_header_t rh;
-  crypto_digest_t *backup_digest=NULL;
+    if (circ->n_chan->channel_usage == CHANNEL_USED_FOR_FULL_CIRCS &&
+        cell->command == CELL_RELAY) {
+      circ->n_chan->channel_usage = CHANNEL_USED_FOR_USER_TRAFFIC;
+    }
+  } else {
+    /* If we're a relay circuit, the question is more complicated. Basically:
+     * we only want to pad connections that carry multihop (anonymous)
+     * circuits.
+     *
+     * We assume we're more than one hop if either the previous hop
+     * is not a client, or if the previous hop is a client and there's
+     * a next hop. Then, circuit traffic starts at RELAY_EARLY, and
+     * user application traffic starts when we see RELAY cells.
+     */
+    or_circuit_t *or_circ = TO_OR_CIRCUIT(circ);
 
-  backup_digest = crypto_digest_dup(digest);
+    if (BUG(!or_circ->p_chan))
+      return;
 
-  relay_header_unpack(&rh, cell->payload);
-  memcpy(&received_integrity, rh.integrity, 4);
-  memset(rh.integrity, 0, 4);
-  relay_header_pack(cell->payload, &rh);
-
-//  log_fn(LOG_DEBUG,"Reading digest of %u %u %u %u from relay cell.",
-//    received_integrity[0], received_integrity[1],
-//    received_integrity[2], received_integrity[3]);
-
-  crypto_digest_add_bytes(digest, (char*) cell->payload, CELL_PAYLOAD_SIZE);
-  crypto_digest_get_digest(digest, (char*) &calculated_integrity, 4);
-
-  if (calculated_integrity != received_integrity) {
-//    log_fn(LOG_INFO,"Recognized=0 but bad digest. Not recognizing.");
-// (%d vs %d).", received_integrity, calculated_integrity);
-    /* restore digest to its old form */
-    crypto_digest_assign(digest, backup_digest);
-    /* restore the relay header */
-    memcpy(rh.integrity, &received_integrity, 4);
-    relay_header_pack(cell->payload, &rh);
-    crypto_digest_free(backup_digest);
-    return 0;
+    if (!channel_is_client(or_circ->p_chan) ||
+        (channel_is_client(or_circ->p_chan) && circ->n_chan)) {
+      if (cell->command == CELL_RELAY_EARLY) {
+        if (or_circ->p_chan->channel_usage < CHANNEL_USED_FOR_FULL_CIRCS) {
+          or_circ->p_chan->channel_usage = CHANNEL_USED_FOR_FULL_CIRCS;
+        }
+      } else if (cell->command == CELL_RELAY) {
+        or_circ->p_chan->channel_usage = CHANNEL_USED_FOR_USER_TRAFFIC;
+      }
+    }
   }
-  crypto_digest_free(backup_digest);
-  return 1;
-}
-
-/** Apply <b>cipher</b> to CELL_PAYLOAD_SIZE bytes of <b>in</b>
- * (in place).
- *
- * If <b>encrypt_mode</b> is 1 then encrypt, else decrypt.
- *
- * Returns 0.
- */
-static int
-relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in,
-                        int encrypt_mode)
-{
-  (void)encrypt_mode;
-  crypto_cipher_crypt_inplace(cipher, (char*) in, CELL_PAYLOAD_SIZE);
-
-  return 0;
 }
 
 /** Receive a relay cell:
@@ -188,16 +241,22 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
   if (circ->marked_for_close)
     return 0;
 
-  if (relay_crypt(circ, cell, cell_direction, &layer_hint, &recognized) < 0) {
-    log_warn(LD_BUG,"relay crypt failed. Dropping connection.");
+  if (relay_decrypt_cell(circ, cell, cell_direction, &layer_hint, &recognized)
+      < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "relay crypt failed. Dropping connection.");
     return -END_CIRC_REASON_INTERNAL;
   }
 
+  circuit_update_channel_usage(circ, cell);
+
   if (recognized) {
     edge_connection_t *conn = NULL;
 
     if (circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) {
-      pathbias_check_probe_response(circ, cell);
+      if (pathbias_check_probe_response(circ, cell) == -1) {
+        pathbias_count_valid_cells(circ, cell);
+      }
 
       /* We need to drop this cell no matter what to avoid code that expects
        * a certain purpose (such as the hidserv code). */
@@ -221,8 +280,13 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
       log_debug(LD_OR,"Sending to origin.");
       if ((reason = connection_edge_process_relay_cell(cell, circ, conn,
                                                        layer_hint)) < 0) {
-        log_warn(LD_OR,
-                 "connection_edge_process_relay_cell (at origin) failed.");
+        /* If a client is trying to connect to unknown hidden service port,
+         * END_CIRC_AT_ORIGIN is sent back so we can then close the circuit.
+         * Do not log warn as this is an expected behavior for a service. */
+        if (reason != END_CIRC_AT_ORIGIN) {
+          log_warn(LD_OR,
+                   "connection_edge_process_relay_cell (at origin) failed.");
+        }
         return reason;
       }
     }
@@ -285,92 +349,6 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
   return 0;
 }
 
-/** Do the appropriate en/decryptions for <b>cell</b> arriving on
- * <b>circ</b> in direction <b>cell_direction</b>.
- *
- * If cell_direction == CELL_DIRECTION_IN:
- *   - If we're at the origin (we're the OP), for hops 1..N,
- *     decrypt cell. If recognized, stop.
- *   - Else (we're not the OP), encrypt one hop. Cell is not recognized.
- *
- * If cell_direction == CELL_DIRECTION_OUT:
- *   - decrypt one hop. Check if recognized.
- *
- * If cell is recognized, set *recognized to 1, and set
- * *layer_hint to the hop that recognized it.
- *
- * Return -1 to indicate that we should mark the circuit for close,
- * else return 0.
- */
-int
-relay_crypt(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction,
-            crypt_path_t **layer_hint, char *recognized)
-{
-  relay_header_t rh;
-
-  tor_assert(circ);
-  tor_assert(cell);
-  tor_assert(recognized);
-  tor_assert(cell_direction == CELL_DIRECTION_IN ||
-             cell_direction == CELL_DIRECTION_OUT);
-
-  if (cell_direction == CELL_DIRECTION_IN) {
-    if (CIRCUIT_IS_ORIGIN(circ)) { /* We're at the beginning of the circuit.
-                                    * We'll want to do layered decrypts. */
-      crypt_path_t *thishop, *cpath = TO_ORIGIN_CIRCUIT(circ)->cpath;
-      thishop = cpath;
-      if (thishop->state != CPATH_STATE_OPEN) {
-        log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-               "Relay cell before first created cell? Closing.");
-        return -1;
-      }
-      do { /* Remember: cpath is in forward order, that is, first hop first. */
-        tor_assert(thishop);
-
-        if (relay_crypt_one_payload(thishop->b_crypto, cell->payload, 0) < 0)
-          return -1;
-
-        relay_header_unpack(&rh, cell->payload);
-        if (rh.recognized == 0) {
-          /* it's possibly recognized. have to check digest to be sure. */
-          if (relay_digest_matches(thishop->b_digest, cell)) {
-            *recognized = 1;
-            *layer_hint = thishop;
-            return 0;
-          }
-        }
-
-        thishop = thishop->next;
-      } while (thishop != cpath && thishop->state == CPATH_STATE_OPEN);
-      log_fn(LOG_PROTOCOL_WARN, LD_OR,
-             "Incoming cell at client not recognized. Closing.");
-      return -1;
-    } else { /* we're in the middle. Just one crypt. */
-      if (relay_crypt_one_payload(TO_OR_CIRCUIT(circ)->p_crypto,
-                                  cell->payload, 1) < 0)
-        return -1;
-//      log_fn(LOG_DEBUG,"Skipping recognized check, because we're not "
-//             "the client.");
-    }
-  } else /* cell_direction == CELL_DIRECTION_OUT */ {
-    /* we're in the middle. Just one crypt. */
-
-    if (relay_crypt_one_payload(TO_OR_CIRCUIT(circ)->n_crypto,
-                                cell->payload, 0) < 0)
-      return -1;
-
-    relay_header_unpack(&rh, cell->payload);
-    if (rh.recognized == 0) {
-      /* it's possibly recognized. have to check digest to be sure. */
-      if (relay_digest_matches(TO_OR_CIRCUIT(circ)->n_digest, cell)) {
-        *recognized = 1;
-        return 0;
-      }
-    }
-  }
-  return 0;
-}
-
 /** Package a relay cell from an edge:
  *  - Encrypt it to the right layer
  *  - Append it to the appropriate cell_queue on <b>circ</b>.
@@ -389,36 +367,36 @@ circuit_package_relay_cell(cell_t *cell, circuit_t *circ,
   }
 
   if (cell_direction == CELL_DIRECTION_OUT) {
-    crypt_path_t *thishop; /* counter for repeated crypts */
     chan = circ->n_chan;
     if (!chan) {
       log_warn(LD_BUG,"outgoing relay cell sent from %s:%d has n_chan==NULL."
-               " Dropping.", filename, lineno);
+               " Dropping. Circuit is in state %s (%d), and is "
+               "%smarked for close. (%s:%d, %d)", filename, lineno,
+               circuit_state_to_string(circ->state), circ->state,
+               circ->marked_for_close ? "" : "not ",
+               circ->marked_for_close_file?circ->marked_for_close_file:"",
+               circ->marked_for_close, circ->marked_for_close_reason);
+      if (CIRCUIT_IS_ORIGIN(circ)) {
+        circuit_log_path(LOG_WARN, LD_BUG, TO_ORIGIN_CIRCUIT(circ));
+      }
+      log_backtrace(LOG_WARN,LD_BUG,"");
       return 0; /* just drop it */
     }
     if (!CIRCUIT_IS_ORIGIN(circ)) {
       log_warn(LD_BUG,"outgoing relay cell sent from %s:%d on non-origin "
                "circ. Dropping.", filename, lineno);
+      log_backtrace(LOG_WARN,LD_BUG,"");
       return 0; /* just drop it */
     }
 
-    relay_set_digest(layer_hint->f_digest, cell);
+    relay_encrypt_cell_outbound(cell, TO_ORIGIN_CIRCUIT(circ), layer_hint);
 
-    thishop = layer_hint;
-    /* moving from farthest to nearest hop */
-    do {
-      tor_assert(thishop);
-      /* XXXX RD This is a bug, right? */
-      log_debug(LD_OR,"crypting a layer of the relay cell.");
-      if (relay_crypt_one_payload(thishop->f_crypto, cell->payload, 1) < 0) {
-        return -1;
-      }
-
-      thishop = thishop->prev;
-    } while (thishop != TO_ORIGIN_CIRCUIT(circ)->cpath->prev);
+    /* Update circ written totals for control port */
+    origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+    ocirc->n_written_circ_bw = tor_add_u32_nowrap(ocirc->n_written_circ_bw,
+                                                  CELL_PAYLOAD_SIZE);
 
   } else { /* incoming cell */
-    or_circuit_t *or_circ;
     if (CIRCUIT_IS_ORIGIN(circ)) {
       /* We should never package an _incoming_ cell from the circuit
        * origin; that means we messed up somewhere. */
@@ -426,11 +404,9 @@ circuit_package_relay_cell(cell_t *cell, circuit_t *circ,
       assert_circuit_ok(circ);
       return 0; /* just drop it */
     }
-    or_circ = TO_OR_CIRCUIT(circ);
+    or_circuit_t *or_circ = TO_OR_CIRCUIT(circ);
+    relay_encrypt_cell_inbound(cell, or_circ);
     chan = or_circ->p_chan;
-    relay_set_digest(or_circ->p_digest, cell);
-    if (relay_crypt_one_payload(or_circ->p_crypto, cell->payload, 1) < 0)
-      return -1;
   }
   ++stats_n_relay_cells_relayed;
 
@@ -564,11 +540,11 @@ relay_command_to_string(uint8_t command)
  * If you can't send the cell, mark the circuit for close and return -1. Else
  * return 0.
  */
-int
-relay_send_command_from_edge_(streamid_t stream_id, circuit_t *circ,
-                              uint8_t relay_command, const char *payload,
-                              size_t payload_len, crypt_path_t *cpath_layer,
-                              const char *filename, int lineno)
+MOCK_IMPL(int,
+relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ,
+                               uint8_t relay_command, const char *payload,
+                               size_t payload_len, crypt_path_t *cpath_layer,
+                               const char *filename, int lineno))
 {
   cell_t cell;
   relay_header_t rh;
@@ -580,14 +556,14 @@ relay_send_command_from_edge_(streamid_t stream_id, circuit_t *circ,
 
   memset(&cell, 0, sizeof(cell_t));
   cell.command = CELL_RELAY;
-  if (cpath_layer) {
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    tor_assert(cpath_layer);
     cell.circ_id = circ->n_circ_id;
     cell_direction = CELL_DIRECTION_OUT;
-  } else if (! CIRCUIT_IS_ORIGIN(circ)) {
+  } else {
+    tor_assert(! cpath_layer);
     cell.circ_id = TO_OR_CIRCUIT(circ)->p_circ_id;
     cell_direction = CELL_DIRECTION_IN;
-  } else {
-    return -1;
   }
 
   memset(&rh, 0, sizeof(rh));
@@ -601,6 +577,9 @@ relay_send_command_from_edge_(streamid_t stream_id, circuit_t *circ,
   log_debug(LD_OR,"delivering %d cell %s.", relay_command,
             cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward");
 
+  if (relay_command == RELAY_COMMAND_DROP)
+    rep_hist_padding_count_write(PADDING_TYPE_DROP);
+
   /* If we are sending an END cell and this circuit is used for a tunneled
    * directory request, advance its state. */
   if (relay_command == RELAY_COMMAND_END && circ->dirreq_id)
@@ -648,6 +627,10 @@ relay_send_command_from_edge_(streamid_t stream_id, circuit_t *circ,
       tor_free(commands);
       smartlist_free(commands_list);
     }
+
+    /* Let's assume we're well-behaved: Anything that we decide to send is
+     * valid, delivered data. */
+    circuit_sent_valid_data(origin_circ, rh.length);
   }
 
   if (circuit_package_relay_cell(&cell, circ, cell_direction, cpath_layer,
@@ -707,6 +690,16 @@ connection_edge_send_command(edge_connection_t *fromconn,
     return -1;
   }
 
+#ifdef MEASUREMENTS_21206
+  /* Keep track of the number of RELAY_DATA cells sent for directory
+   * connections. */
+  connection_t *linked_conn = TO_CONN(fromconn)->linked_conn;
+
+  if (linked_conn && linked_conn->type == CONN_TYPE_DIR) {
+    ++(TO_DIR_CONN(linked_conn)->data_cells_sent);
+  }
+#endif /* defined(MEASUREMENTS_21206) */
+
   return relay_send_command_from_edge(fromconn->stream_id, circ,
                                       relay_command, payload,
                                       payload_len, cpath_layer);
@@ -767,6 +760,9 @@ connection_ap_process_end_not_open(
     }
   }
 
+  /* This end cell is now valid. */
+  circuit_read_valid_data(circ, rh->length);
+
   if (rh->length == 0) {
     reason = END_STREAM_REASON_MISC;
   }
@@ -986,7 +982,12 @@ remap_event_helper(entry_connection_t *conn, const tor_addr_t *new_addr)
  * header has already been parsed into <b>rh</b>. On success, set
  * <b>addr_out</b> to the address we're connected to, and <b>ttl_out</b> to
  * the ttl of that address, in seconds, and return 0.  On failure, return
- * -1. */
+ * -1.
+ *
+ * Note that the resulting address can be UNSPEC if the connected cell had no
+ * address (as for a stream to an union service or a tunneled directory
+ * connection), and that the ttl can be absent (in which case <b>ttl_out</b>
+ * is set to -1). */
 STATIC int
 connected_cell_parse(const relay_header_t *rh, const cell_t *cell,
                      tor_addr_t *addr_out, int *ttl_out)
@@ -1026,7 +1027,7 @@ connected_cell_parse(const relay_header_t *rh, const cell_t *cell,
 
 /** Drop all storage held by <b>addr</b>. */
 STATIC void
-address_ttl_free(address_ttl_t *addr)
+address_ttl_free_(address_ttl_t *addr)
 {
   if (!addr)
     return;
@@ -1257,6 +1258,12 @@ connection_edge_process_resolved_cell(edge_connection_t *conn,
     }
   }
 
+  /* This is valid data at this point. Count it */
+  if (conn->on_circuit && CIRCUIT_IS_ORIGIN(conn->on_circuit)) {
+    circuit_read_valid_data(TO_ORIGIN_CIRCUIT(conn->on_circuit),
+                            rh->length);
+  }
+
   connection_ap_handshake_socks_got_resolved_cell(entry_conn,
                                                   errcode,
                                                   resolved_addresses);
@@ -1317,7 +1324,7 @@ connection_edge_process_relay_cell_not_open(
              "after %d seconds.",
              (unsigned)circ->n_circ_id,
              rh->stream_id,
-             (int)(time(NULL) - conn->base_.timestamp_lastread));
+             (int)(time(NULL) - conn->base_.timestamp_last_read_allowed));
     if (connected_cell_parse(rh, cell, &addr, &ttl) < 0) {
       log_fn(LOG_PROTOCOL_WARN, LD_APP,
              "Got a badly formatted connected cell. Closing.");
@@ -1326,6 +1333,9 @@ connection_edge_process_relay_cell_not_open(
       return 0;
     }
     if (tor_addr_family(&addr) != AF_UNSPEC) {
+      /* The family is not UNSPEC: so we were given an address in the
+       * connected cell. (This is normal, except for BEGINDIR and onion
+       * service streams.) */
       const sa_family_t family = tor_addr_family(&addr);
       if (tor_addr_is_null(&addr) ||
           (get_options()->ClientDNSRejectInternalAddresses &&
@@ -1358,8 +1368,9 @@ connection_edge_process_relay_cell_not_open(
     circuit_log_path(LOG_INFO,LD_APP,TO_ORIGIN_CIRCUIT(circ));
     /* don't send a socks reply to transparent conns */
     tor_assert(entry_conn->socks_request != NULL);
-    if (!entry_conn->socks_request->has_finished)
+    if (!entry_conn->socks_request->has_finished) {
       connection_ap_handshake_socks_reply(entry_conn, NULL, 0, 0);
+    }
 
     /* Was it a linked dir conn? If so, a dir request just started to
      * fetch something; this could be a bootstrap status milestone. */
@@ -1379,8 +1390,8 @@ connection_edge_process_relay_cell_not_open(
         case DIR_PURPOSE_FETCH_SERVERDESC:
         case DIR_PURPOSE_FETCH_MICRODESC:
           if (TO_DIR_CONN(dirconn)->router_purpose == ROUTER_PURPOSE_GENERAL)
-            control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS,
-                                    count_loading_descriptors_progress());
+            control_event_boot_dir(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS,
+                                   count_loading_descriptors_progress());
           break;
       }
     }
@@ -1391,6 +1402,9 @@ connection_edge_process_relay_cell_not_open(
       entry_conn->pending_optimistic_data = NULL;
     }
 
+    /* This is valid data at this point. Count it */
+    circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length);
+
     /* handle anything that might have queued */
     if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) {
       /* (We already sent an end cell if possible) */
@@ -1423,7 +1437,7 @@ connection_edge_process_relay_cell_not_open(
  *
  * Return -reason if you want to warn and tear down the circuit, else 0.
  */
-static int
+STATIC int
 connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
                                    edge_connection_t *conn,
                                    crypt_path_t *layer_hint)
@@ -1490,6 +1504,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
 
   switch (rh.command) {
     case RELAY_COMMAND_DROP:
+      rep_hist_padding_count_read(PADDING_TYPE_DROP);
 //      log_info(domain,"Got a relay-level padding cell. Dropping.");
       return 0;
     case RELAY_COMMAND_BEGIN:
@@ -1522,7 +1537,6 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
         circ->dirreq_id = ++next_id;
         TO_OR_CIRCUIT(circ)->p_chan->dirreq_id = circ->dirreq_id;
       }
-
       return connection_exit_begin_conn(cell, circ);
     case RELAY_COMMAND_DATA:
       ++stats_n_data_cells_received;
@@ -1548,6 +1562,17 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
                "stream_id. Dropping.");
         return 0;
       } else if (!conn) {
+        if (CIRCUIT_IS_ORIGIN(circ)) {
+          origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+          if (connection_half_edge_is_valid_data(ocirc->half_streams,
+                                                 rh.stream_id)) {
+            circuit_read_valid_data(ocirc, rh.length);
+            log_info(domain,
+                     "data cell on circ %u valid on half-closed "
+                     "stream id %d", ocirc->global_identifier, rh.stream_id);
+          }
+        }
+
         log_info(domain,"data cell dropped, unknown stream (streamid %d).",
                  rh.stream_id);
         return 0;
@@ -1558,11 +1583,25 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
                "(relay data) conn deliver_window below 0. Killing.");
         return -END_CIRC_REASON_TORPROTOCOL;
       }
+      /* Total all valid application bytes delivered */
+      if (CIRCUIT_IS_ORIGIN(circ) && rh.length > 0) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+      }
 
       stats_n_data_bytes_received += rh.length;
-      connection_write_to_buf((char*)(cell->payload + RELAY_HEADER_SIZE),
+      connection_buf_add((char*)(cell->payload + RELAY_HEADER_SIZE),
                               rh.length, TO_CONN(conn));
 
+#ifdef MEASUREMENTS_21206
+      /* Count number of RELAY_DATA cells received on a linked directory
+       * connection. */
+      connection_t *linked_conn = TO_CONN(conn)->linked_conn;
+
+      if (linked_conn && linked_conn->type == CONN_TYPE_DIR) {
+        ++(TO_DIR_CONN(linked_conn)->data_cells_received);
+      }
+#endif /* defined(MEASUREMENTS_21206) */
+
       if (!optimistic_data) {
         /* Only send a SENDME if we're not getting optimistic data; otherwise
          * a SENDME could arrive before the CONNECTED.
@@ -1575,6 +1614,20 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       reason = rh.length > 0 ?
         get_uint8(cell->payload+RELAY_HEADER_SIZE) : END_STREAM_REASON_MISC;
       if (!conn) {
+        if (CIRCUIT_IS_ORIGIN(circ)) {
+          origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+          if (connection_half_edge_is_valid_end(ocirc->half_streams,
+                                                rh.stream_id)) {
+
+            circuit_read_valid_data(ocirc, rh.length);
+            log_info(domain,
+                     "end cell (%s) on circ %u valid on half-closed "
+                     "stream id %d",
+                     stream_end_reason_to_string(reason),
+                     ocirc->global_identifier, rh.stream_id);
+            return 0;
+          }
+        }
         log_info(domain,"end cell (%s) dropped, unknown stream.",
                  stream_end_reason_to_string(reason));
         return 0;
@@ -1600,6 +1653,11 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
         /* only mark it if not already marked. it's possible to
          * get the 'end' right around when the client hangs up on us. */
         connection_mark_and_flush(TO_CONN(conn));
+
+        /* Total all valid application bytes delivered */
+        if (CIRCUIT_IS_ORIGIN(circ)) {
+          circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+        }
       }
       return 0;
     case RELAY_COMMAND_EXTEND:
@@ -1665,6 +1723,10 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
         log_info(domain,"circuit_send_next_onion_skin() failed.");
         return reason;
       }
+      /* Total all valid bytes delivered. */
+      if (CIRCUIT_IS_ORIGIN(circ)) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+      }
       return 0;
     case RELAY_COMMAND_TRUNCATE:
       if (layer_hint) {
@@ -1702,7 +1764,14 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
                "'truncated' unsupported at non-origin. Dropping.");
         return 0;
       }
-      circuit_truncated(TO_ORIGIN_CIRCUIT(circ), layer_hint,
+
+      /* Count the truncated as valid, for completeness. The
+       * circuit is being torn down anyway, though.  */
+      if (CIRCUIT_IS_ORIGIN(circ)) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ),
+                                rh.length);
+      }
+      circuit_truncated(TO_ORIGIN_CIRCUIT(circ),
                         get_uint8(cell->payload + RELAY_HEADER_SIZE));
       return 0;
     case RELAY_COMMAND_CONNECTED:
@@ -1711,6 +1780,19 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
                "'connected' unsupported while open. Closing circ.");
         return -END_CIRC_REASON_TORPROTOCOL;
       }
+
+      if (CIRCUIT_IS_ORIGIN(circ)) {
+        origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+        if (connection_half_edge_is_valid_connected(ocirc->half_streams,
+                                                    rh.stream_id)) {
+          circuit_read_valid_data(ocirc, rh.length);
+          log_info(domain,
+                   "connected cell on circ %u valid on half-closed "
+                   "stream id %d", ocirc->global_identifier, rh.stream_id);
+          return 0;
+        }
+      }
+
       log_info(domain,
                "'connected' received on circid %u for streamid %d, "
                "no conn attached anymore. Ignoring.",
@@ -1731,6 +1813,15 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
           log_debug(LD_APP,"circ-level sendme at origin, packagewindow %d.",
                     layer_hint->package_window);
           circuit_resume_edge_reading(circ, layer_hint);
+
+          /* We count circuit-level sendme's as valid delivered data because
+           * they are rate limited.
+           */
+          if (CIRCUIT_IS_ORIGIN(circ)) {
+            circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ),
+                                    rh.length);
+          }
+
         } else {
           if (circ->package_window + CIRCWINDOW_INCREMENT >
                 CIRCWINDOW_START_MAX) {
@@ -1750,10 +1841,42 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
         return 0;
       }
       if (!conn) {
+        if (CIRCUIT_IS_ORIGIN(circ)) {
+          origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+          if (connection_half_edge_is_valid_sendme(ocirc->half_streams,
+                                                   rh.stream_id)) {
+            circuit_read_valid_data(ocirc, rh.length);
+            log_info(domain,
+                    "sendme cell on circ %u valid on half-closed "
+                    "stream id %d", ocirc->global_identifier, rh.stream_id);
+          }
+        }
+
         log_info(domain,"sendme cell dropped, unknown stream (streamid %d).",
                  rh.stream_id);
         return 0;
       }
+
+      /* Don't allow the other endpoint to request more than our maximum
+       * (i.e. initial) stream SENDME window worth of data. Well-behaved
+       * stock clients will not request more than this max (as per the check
+       * in the while loop of connection_edge_consider_sending_sendme()).
+       */
+      if (conn->package_window + STREAMWINDOW_INCREMENT >
+          STREAMWINDOW_START_MAX) {
+        static struct ratelim_t stream_warn_ratelim = RATELIM_INIT(600);
+        log_fn_ratelim(&stream_warn_ratelim, LOG_PROTOCOL_WARN, LD_PROTOCOL,
+               "Unexpected stream sendme cell. Closing circ (window %d).",
+               conn->package_window);
+        return -END_CIRC_REASON_TORPROTOCOL;
+      }
+
+      /* At this point, the stream sendme is valid */
+      if (CIRCUIT_IS_ORIGIN(circ)) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ),
+                                rh.length);
+      }
+
       conn->package_window += STREAMWINDOW_INCREMENT;
       log_debug(domain,"stream-level sendme, packagewindow now %d.",
                 conn->package_window);
@@ -1792,6 +1915,19 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
                "'resolved' unsupported while open. Closing circ.");
         return -END_CIRC_REASON_TORPROTOCOL;
       }
+
+      if (CIRCUIT_IS_ORIGIN(circ)) {
+        origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+        if (connection_half_edge_is_valid_resolved(ocirc->half_streams,
+                                                    rh.stream_id)) {
+          circuit_read_valid_data(ocirc, rh.length);
+          log_info(domain,
+                   "resolved cell on circ %u valid on half-closed "
+                   "stream id %d", ocirc->global_identifier, rh.stream_id);
+          return 0;
+        }
+      }
+
       log_info(domain,
                "'resolved' received, no conn attached anymore. Ignoring.");
       return 0;
@@ -1919,13 +2055,13 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial,
     /* XXXX We could be more efficient here by sometimes packing
      * previously-sent optimistic data in the same cell with data
      * from the inbuf. */
-    fetch_from_buf(payload, length, entry_conn->sending_optimistic_data);
+    buf_get_bytes(entry_conn->sending_optimistic_data, payload, length);
     if (!buf_datalen(entry_conn->sending_optimistic_data)) {
         buf_free(entry_conn->sending_optimistic_data);
         entry_conn->sending_optimistic_data = NULL;
     }
   } else {
-    connection_fetch_from_buf(payload, length, TO_CONN(conn));
+    connection_buf_get_bytes(payload, length, TO_CONN(conn));
   }
 
   log_debug(domain,TOR_SOCKET_T_FORMAT": Packaging %d bytes (%d waiting).",
@@ -1937,13 +2073,14 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial,
        retry */
     if (!entry_conn->pending_optimistic_data)
       entry_conn->pending_optimistic_data = buf_new();
-    write_to_buf(payload, length, entry_conn->pending_optimistic_data);
+    buf_add(entry_conn->pending_optimistic_data, payload, length);
   }
 
   if (connection_edge_send_command(conn, RELAY_COMMAND_DATA,
-                                   payload, length) < 0 )
+                                   payload, length) < 0 ) {
     /* circuit got marked for close, don't continue, don't need to mark conn */
     return 0;
+  }
 
   if (!cpath_layer) { /* non-rendezvous exit */
     tor_assert(circ->package_window > 0);
@@ -2255,13 +2392,6 @@ circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint)
   }
 }
 
-#ifdef ACTIVE_CIRCUITS_PARANOIA
-#define assert_cmux_ok_paranoid(chan) \
-     assert_circuit_mux_okay(chan)
-#else
-#define assert_cmux_ok_paranoid(chan)
-#endif
-
 /** The total number of cells we have allocated. */
 static size_t total_cells_allocated = 0;
 
@@ -2283,7 +2413,7 @@ packed_cell_new(void)
 
 /** Return a packed cell used outside by channel_t lower layer */
 void
-packed_cell_free(packed_cell_t *cell)
+packed_cell_free_(packed_cell_t *cell)
 {
   if (!cell)
     return;
@@ -2340,7 +2470,7 @@ cell_queue_append_packed_copy(circuit_t *circ, cell_queue_t *queue,
   (void)exitward;
   (void)use_stats;
 
-  copy->inserted_time = (uint32_t) monotime_coarse_absolute_msec();
+  copy->inserted_timestamp = monotime_coarse_get_stamp();
 
   cell_queue_append(queue, copy);
 }
@@ -2423,7 +2553,7 @@ destroy_cell_queue_append(destroy_cell_queue_t *queue,
   cell->circid = circid;
   cell->reason = reason;
   /* Not yet used, but will be required for OOM handling. */
-  cell->inserted_time = (uint32_t) monotime_coarse_absolute_msec();
+  cell->inserted_timestamp = monotime_coarse_get_stamp();
 
   TOR_SIMPLEQ_INSERT_TAIL(&queue->head, cell, next);
   ++queue->n;
@@ -2454,7 +2584,7 @@ packed_cell_mem_cost(void)
 }
 
 /* DOCDOC */
-STATIC size_t
+size_t
 cell_queues_get_total_allocation(void)
 {
   return total_cells_allocated * packed_cell_mem_cost();
@@ -2473,13 +2603,16 @@ cell_queues_check_size(void)
 {
   time_t now = time(NULL);
   size_t alloc = cell_queues_get_total_allocation();
+  alloc += half_streams_get_total_allocation();
   alloc += buf_get_total_allocation();
-  alloc += tor_zlib_get_total_allocation();
+  alloc += tor_compress_get_total_allocation();
   const size_t rend_cache_total = rend_cache_get_total_allocation();
   alloc += rend_cache_total;
   const size_t geoip_client_cache_total =
     geoip_client_cache_total_allocation();
   alloc += geoip_client_cache_total;
+  const size_t dns_cache_total = dns_cache_total_allocation();
+  alloc += dns_cache_total;
   if (alloc >= get_options()->MaxMemInQueues_low_threshold) {
     last_time_under_memory_pressure = approx_time();
     if (alloc >= get_options()->MaxMemInQueues) {
@@ -2489,9 +2622,7 @@ cell_queues_check_size(void)
       if (rend_cache_total > get_options()->MaxMemInQueues / 5) {
         const size_t bytes_to_remove =
           rend_cache_total - (size_t)(get_options()->MaxMemInQueues / 10);
-        rend_cache_clean_v2_descs_as_dir(now, bytes_to_remove);
-        alloc -= rend_cache_total;
-        alloc += rend_cache_get_total_allocation();
+        alloc -= hs_cache_handle_oom(now, bytes_to_remove);
       }
       if (geoip_client_cache_total > get_options()->MaxMemInQueues / 5) {
         const size_t bytes_to_remove =
@@ -2499,6 +2630,11 @@ cell_queues_check_size(void)
           (size_t)(get_options()->MaxMemInQueues / 10);
         alloc -= geoip_client_cache_handle_oom(now, bytes_to_remove);
       }
+      if (dns_cache_total > get_options()->MaxMemInQueues / 5) {
+        const size_t bytes_to_remove =
+          dns_cache_total - (size_t)(get_options()->MaxMemInQueues / 10);
+        alloc -= dns_cache_handle_oom(now, bytes_to_remove);
+      }
       circuits_handle_oom(alloc);
       return 1;
     }
@@ -2551,16 +2687,12 @@ update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction,
   }
   tor_assert(circuitmux_attached_circuit_direction(cmux, circ) == direction);
 
-  assert_cmux_ok_paranoid(chan);
-
   /* Update the number of cells we have for the circuit mux */
   if (direction == CELL_DIRECTION_OUT) {
     circuitmux_set_num_cells(cmux, circ, circ->n_chan_cells.n);
   } else {
     circuitmux_set_num_cells(cmux, circ, or_circ->p_chan_cells.n);
   }
-
-  assert_cmux_ok_paranoid(chan);
 }
 
 /** Remove all circuits from the cmux on <b>chan</b>.
@@ -2690,8 +2822,13 @@ channel_flush_from_first_active_circuit, (channel_t *chan, int max))
       tor_assert(dcell);
       /* frees dcell */
       cell = destroy_cell_to_packed_cell(dcell, chan->wide_circ_ids);
-      /* frees cell */
-      channel_write_packed_cell(chan, cell);
+      /* Send the DESTROY cell. It is very unlikely that this fails but just
+       * in case, get rid of the channel. */
+      if (channel_write_packed_cell(chan, cell) < 0) {
+        /* The cell has been freed. */
+        channel_mark_for_close(chan);
+        continue;
+      }
       /* Update the cmux destroy counter */
       circuitmux_notify_xmit_destroy(cmux);
       cell = NULL;
@@ -2700,7 +2837,6 @@ channel_flush_from_first_active_circuit, (channel_t *chan, int max))
     }
     /* If it returns NULL, no cells left to send */
     if (!circ) break;
-    assert_cmux_ok_paranoid(chan);
 
     if (circ->n_chan == chan) {
       queue = &circ->n_chan_cells;
@@ -2734,9 +2870,10 @@ channel_flush_from_first_active_circuit, (channel_t *chan, int max))
     /* Calculate the exact time that this cell has spent in the queue. */
     if (get_options()->CellStatistics ||
         get_options()->TestingEnableCellStatsEvent) {
-      uint32_t msec_waiting;
-      uint32_t msec_now = (uint32_t)monotime_coarse_absolute_msec();
-      msec_waiting = msec_now - cell->inserted_time;
+      uint32_t timestamp_now = monotime_coarse_get_stamp();
+      uint32_t msec_waiting =
+        (uint32_t) monotime_coarse_stamp_units_to_approx_msec(
+                         timestamp_now - cell->inserted_timestamp);
 
       if (get_options()->CellStatistics && !CIRCUIT_IS_ORIGIN(circ)) {
         or_circ = TO_OR_CIRCUIT(circ);
@@ -2767,8 +2904,13 @@ channel_flush_from_first_active_circuit, (channel_t *chan, int max))
                                 DIRREQ_TUNNELED,
                                 DIRREQ_CIRC_QUEUE_FLUSHED);
 
-    /* Now send the cell */
-    channel_write_packed_cell(chan, cell);
+    /* Now send the cell. It is very unlikely that this fails but just in
+     * case, get rid of the channel. */
+    if (channel_write_packed_cell(chan, cell) < 0) {
+      /* The cell has been freed at this point. */
+      channel_mark_for_close(chan);
+      continue;
+    }
     cell = NULL;
 
     /*
@@ -2798,27 +2940,70 @@ channel_flush_from_first_active_circuit, (channel_t *chan, int max))
   }
 
   /* Okay, we're done sending now */
-  assert_cmux_ok_paranoid(chan);
-
   return n_flushed;
 }
 
-#if 0
-/** Indicate the current preferred cap for middle circuits; zero disables
- * the cap.  Right now it's just a constant, ORCIRC_MAX_MIDDLE_CELLS, but
- * the logic in append_cell_to_circuit_queue() is written to be correct
- * if we want to base it on a consensus param or something that might change
- * in the future.
- */
-static int
-get_max_middle_cells(void)
+/* Minimum value is the maximum circuit window size.
+ *
+ * SENDME cells makes it that we can control how many cells can be inflight on
+ * a circuit from end to end. This logic makes it that on any circuit cell
+ * queue, we have a maximum of cells possible.
+ *
+ * Because the Tor protocol allows for a client to exit at any hop in a
+ * circuit and a circuit can be of a maximum of 8 hops, so in theory the
+ * normal worst case will be the circuit window start value times the maximum
+ * number of hops (8). Having more cells then that means something is wrong.
+ *
+ * However, because padding cells aren't counted in the package window, we set
+ * the maximum size to a reasonably large size for which we expect that we'll
+ * never reach in theory. And if we ever do because of future changes, we'll
+ * be able to control it with a consensus parameter.
+ *
+ * XXX: Unfortunately, END cells aren't accounted for in the circuit window
+ * which means that for instance if a client opens 8001 streams, the 8001
+ * following END cells will queue up in the circuit which will get closed if
+ * the max limit is 8000. Which is sad because it is allowed by the Tor
+ * protocol. But, we need an upper bound on circuit queue in order to avoid
+ * DoS memory pressure so the default size is a middle ground between not
+ * having any limit and having a very restricted one. This is why we can also
+ * control it through a consensus parameter. */
+#define RELAY_CIRC_CELL_QUEUE_SIZE_MIN CIRCWINDOW_START_MAX
+/* We can't have a consensus parameter above this value. */
+#define RELAY_CIRC_CELL_QUEUE_SIZE_MAX INT32_MAX
+/* Default value is set to a large value so we can handle padding cells
+ * properly which aren't accounted for in the SENDME window. Default is 50000
+ * allowed cells in the queue resulting in ~25MB. */
+#define RELAY_CIRC_CELL_QUEUE_SIZE_DEFAULT \
+  (50 * RELAY_CIRC_CELL_QUEUE_SIZE_MIN)
+
+/* The maximum number of cell a circuit queue can contain. This is updated at
+ * every new consensus and controlled by a parameter. */
+static int32_t max_circuit_cell_queue_size =
+  RELAY_CIRC_CELL_QUEUE_SIZE_DEFAULT;
+
+/* Called when the consensus has changed. At this stage, the global consensus
+ * object has NOT been updated. It is called from
+ * notify_before_networkstatus_changes(). */
+void
+relay_consensus_has_changed(const networkstatus_t *ns)
 {
-  return ORCIRC_MAX_MIDDLE_CELLS;
+  tor_assert(ns);
+
+  /* Update the circuit max cell queue size from the consensus. */
+  max_circuit_cell_queue_size =
+    networkstatus_get_param(ns, "circ_max_cell_queue_size",
+                            RELAY_CIRC_CELL_QUEUE_SIZE_DEFAULT,
+                            RELAY_CIRC_CELL_QUEUE_SIZE_MIN,
+                            RELAY_CIRC_CELL_QUEUE_SIZE_MAX);
 }
-#endif
 
 /** Add <b>cell</b> to the queue of <b>circ</b> writing to <b>chan</b>
- * transmitting in <b>direction</b>. */
+ * transmitting in <b>direction</b>.
+ *
+ * The given <b>cell</b> is copied onto the circuit queue so the caller must
+ * cleanup the memory.
+ *
+ * This function is part of the fast path. */
 void
 append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan,
                              cell_t *cell, cell_direction_t direction,
@@ -2827,10 +3012,6 @@ append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan,
   or_circuit_t *orcirc = NULL;
   cell_queue_t *queue;
   int streams_blocked;
-#if 0
-  uint32_t tgt_max_middle_cells, p_len, n_len, tmp, hard_max_middle_cells;
-#endif
-
   int exitward;
   if (circ->marked_for_close)
     return;
@@ -2845,93 +3026,25 @@ append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan,
     streams_blocked = circ->streams_blocked_on_p_chan;
   }
 
-  /*
-   * Disabling this for now because of a possible guard discovery attack
-   */
-#if 0
-  /* Are we a middle circuit about to exceed ORCIRC_MAX_MIDDLE_CELLS? */
-  if ((circ->n_chan != NULL) && CIRCUIT_IS_ORCIRC(circ)) {
-    orcirc = TO_OR_CIRCUIT(circ);
-    if (orcirc->p_chan) {
-      /* We are a middle circuit if we have both n_chan and p_chan */
-      /* We'll need to know the current preferred maximum */
-      tgt_max_middle_cells = get_max_middle_cells();
-      if (tgt_max_middle_cells > 0) {
-        /* Do we need to initialize middle_max_cells? */
-        if (orcirc->max_middle_cells == 0) {
-          orcirc->max_middle_cells = tgt_max_middle_cells;
-        } else {
-          if (tgt_max_middle_cells > orcirc->max_middle_cells) {
-            /* If we want to increase the cap, we can do so right away */
-            orcirc->max_middle_cells = tgt_max_middle_cells;
-          } else if (tgt_max_middle_cells < orcirc->max_middle_cells) {
-            /*
-             * If we're shrinking the cap, we can't shrink past either queue;
-             * compare tgt_max_middle_cells rather than tgt_max_middle_cells *
-             * ORCIRC_MAX_MIDDLE_KILL_THRESH so the queues don't shrink enough
-             * to generate spurious warnings, either.
-             */
-            n_len = circ->n_chan_cells.n;
-            p_len = orcirc->p_chan_cells.n;
-            tmp = tgt_max_middle_cells;
-            if (tmp < n_len) tmp = n_len;
-            if (tmp < p_len) tmp = p_len;
-            orcirc->max_middle_cells = tmp;
-          }
-          /* else no change */
-        }
-      } else {
-        /* tgt_max_middle_cells == 0 indicates we should disable the cap */
-        orcirc->max_middle_cells = 0;
-      }
-
-      /* Now we know orcirc->max_middle_cells is set correctly */
-      if (orcirc->max_middle_cells > 0) {
-        hard_max_middle_cells =
-          (uint32_t)(((double)orcirc->max_middle_cells) *
-                     ORCIRC_MAX_MIDDLE_KILL_THRESH);
-
-        if ((unsigned)queue->n + 1 >= hard_max_middle_cells) {
-          /* Queueing this cell would put queue over the kill theshold */
-          log_warn(LD_CIRC,
-                   "Got a cell exceeding the hard cap of %u in the "
-                   "%s direction on middle circ ID %u on chan ID "
-                   U64_FORMAT "; killing the circuit.",
-                   hard_max_middle_cells,
-                   (direction == CELL_DIRECTION_OUT) ? "n" : "p",
-                   (direction == CELL_DIRECTION_OUT) ?
-                     circ->n_circ_id : orcirc->p_circ_id,
-                   U64_PRINTF_ARG(
-                     (direction == CELL_DIRECTION_OUT) ?
-                        circ->n_chan->global_identifier :
-                        orcirc->p_chan->global_identifier));
-          circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT);
-          return;
-        } else if ((unsigned)queue->n + 1 == orcirc->max_middle_cells) {
-          /* Only use ==, not >= for this test so we don't spam the log */
-          log_warn(LD_CIRC,
-                   "While trying to queue a cell, reached the soft cap of %u "
-                   "in the %s direction on middle circ ID %u "
-                   "on chan ID " U64_FORMAT ".",
-                   orcirc->max_middle_cells,
-                   (direction == CELL_DIRECTION_OUT) ? "n" : "p",
-                   (direction == CELL_DIRECTION_OUT) ?
-                     circ->n_circ_id : orcirc->p_circ_id,
-                   U64_PRINTF_ARG(
-                     (direction == CELL_DIRECTION_OUT) ?
-                        circ->n_chan->global_identifier :
-                        orcirc->p_chan->global_identifier));
-        }
-      }
-    }
+  if (PREDICT_UNLIKELY(queue->n >= max_circuit_cell_queue_size)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "%s circuit has %d cells in its queue, maximum allowed is %d. "
+           "Closing circuit for safety reasons.",
+           (exitward) ? "Outbound" : "Inbound", queue->n,
+           max_circuit_cell_queue_size);
+    circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT);
+    stats_n_circ_max_cell_reached++;
+    return;
   }
-#endif
 
+  /* Very important that we copy to the circuit queue because all calls to
+   * this function use the stack for the cell memory. */
   cell_queue_append_packed_copy(circ, queue, exitward, cell,
                                 chan->wide_circ_ids, 1);
 
+  /* Check and run the OOM if needed. */
   if (PREDICT_UNLIKELY(cell_queues_check_size())) {
-    /* We ran the OOM handler */
+    /* We ran the OOM handler which might have closed this circuit. */
     if (circ->marked_for_close)
       return;
   }
@@ -3042,17 +3155,6 @@ circuit_clear_cell_queue(circuit_t *circ, channel_t *chan)
     update_circuit_on_cmux(circ, direction);
 }
 
-/** Fail with an assert if the circuit mux on chan is corrupt
- */
-void
-assert_circuit_mux_okay(channel_t *chan)
-{
-  tor_assert(chan);
-  tor_assert(chan->cmux);
-
-  circuitmux_assert_okay(chan->cmux);
-}
-
 /** Return 1 if we shouldn't restart reading on this circuit, even if
  * we get a SENDME.  Else return 0.
 */
@@ -3065,4 +3167,3 @@ circuit_queue_streams_are_blocked(circuit_t *circ)
     return circ->streams_blocked_on_p_chan;
   }
 }
-
diff --git a/src/or/relay.h b/src/core/or/relay.h
index c4f98d92ff..7cc3c43e43 100644
--- a/src/or/relay.h
+++ b/src/core/or/relay.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,16 +14,22 @@
 
 extern uint64_t stats_n_relay_cells_relayed;
 extern uint64_t stats_n_relay_cells_delivered;
+extern uint64_t stats_n_circ_max_cell_reached;
 
+void relay_consensus_has_changed(const networkstatus_t *ns);
 int circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
                                cell_direction_t cell_direction);
+size_t cell_queues_get_total_allocation(void);
 
 void relay_header_pack(uint8_t *dest, const relay_header_t *src);
 void relay_header_unpack(relay_header_t *dest, const uint8_t *src);
-int relay_send_command_from_edge_(streamid_t stream_id, circuit_t *circ,
+MOCK_DECL(int,
+relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ,
                                uint8_t relay_command, const char *payload,
                                size_t payload_len, crypt_path_t *cpath_layer,
-                               const char *filename, int lineno);
+                               const char *filename, int lineno));
+/* Indicates to relay_send_command_from_edge() that it is a control cell. */
+#define CONTROL_CELL_ID 0
 #define relay_send_command_from_edge(stream_id, circ, relay_command, payload, \
                                      payload_len, cpath_layer)          \
   relay_send_command_from_edge_((stream_id), (circ), (relay_command),   \
@@ -48,7 +54,9 @@ size_t packed_cell_mem_cost(void);
 int have_been_under_memory_pressure(void);
 
 /* For channeltls.c */
-void packed_cell_free(packed_cell_t *cell);
+void packed_cell_free_(packed_cell_t *cell);
+#define packed_cell_free(cell) \
+  FREE_AND_NULL(packed_cell_t, packed_cell_free_, (cell))
 
 void cell_queue_init(cell_queue_t *queue);
 void cell_queue_clear(cell_queue_t *queue);
@@ -70,7 +78,6 @@ void destroy_cell_queue_append(destroy_cell_queue_t *queue,
 void channel_unlink_all_circuits(channel_t *chan, smartlist_t *detached_out);
 MOCK_DECL(int, channel_flush_from_first_active_circuit,
           (channel_t *chan, int max));
-void assert_circuit_mux_okay(channel_t *chan);
 void update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction,
                              const char *file, int lineno);
 #define update_circuit_on_cmux(circ, direction) \
@@ -84,9 +91,6 @@ void circuit_clear_cell_queue(circuit_t *circ, channel_t *chan);
 
 void stream_choice_seed_weak_rng(void);
 
-int relay_crypt(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction,
-                crypt_path_t **layer_hint, char *recognized);
-
 circid_t packed_cell_get_circid(const packed_cell_t *cell, int wide_circ_ids);
 
 #ifdef RELAY_PRIVATE
@@ -98,7 +102,9 @@ typedef struct address_ttl_s {
   char *hostname;
   int ttl;
 } address_ttl_t;
-STATIC void address_ttl_free(address_ttl_t *addr);
+STATIC void address_ttl_free_(address_ttl_t *addr);
+#define address_ttl_free(addr) \
+  FREE_AND_NULL(address_ttl_t, address_ttl_free_, (addr))
 STATIC int resolved_cell_parse(const cell_t *cell, const relay_header_t *rh,
                                smartlist_t *addresses_out, int *errcode_out);
 STATIC int connection_edge_process_resolved_cell(edge_connection_t *conn,
@@ -107,9 +113,12 @@ STATIC int connection_edge_process_resolved_cell(edge_connection_t *conn,
 STATIC packed_cell_t *packed_cell_new(void);
 STATIC packed_cell_t *cell_queue_pop(cell_queue_t *queue);
 STATIC destroy_cell_t *destroy_cell_queue_pop(destroy_cell_queue_t *queue);
-STATIC size_t cell_queues_get_total_allocation(void);
 STATIC int cell_queues_check_size(void);
-#endif
+STATIC int connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
+                                   edge_connection_t *conn,
+                                   crypt_path_t *layer_hint);
+
+#endif /* defined(RELAY_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_RELAY_H) */
 
diff --git a/src/core/or/relay_crypto_st.h b/src/core/or/relay_crypto_st.h
new file mode 100644
index 0000000000..dafce257c7
--- /dev/null
+++ b/src/core/or/relay_crypto_st.h
@@ -0,0 +1,31 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef RELAY_CRYPTO_ST_H
+#define RELAY_CRYPTO_ST_H
+
+#define crypto_cipher_t aes_cnt_cipher
+struct crypto_cipher_t;
+struct crypto_digest_t;
+
+struct relay_crypto_t {
+  /* crypto environments */
+  /** Encryption key and counter for cells heading towards the OR at this
+   * step. */
+  struct crypto_cipher_t *f_crypto;
+  /** Encryption key and counter for cells heading back from the OR at this
+   * step. */
+  struct crypto_cipher_t *b_crypto;
+
+  /** Digest state for cells heading towards the OR at this step. */
+  struct crypto_digest_t *f_digest; /* for integrity checking */
+  /** Digest state for cells heading away from the OR at this step. */
+  struct crypto_digest_t *b_digest;
+
+};
+#undef crypto_cipher_t
+
+#endif
diff --git a/src/core/or/scheduler.c b/src/core/or/scheduler.c
new file mode 100644
index 0000000000..f85201b7d5
--- /dev/null
+++ b/src/core/or/scheduler.c
@@ -0,0 +1,768 @@
+/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+
+#include "lib/evloop/compat_libevent.h"
+#define SCHEDULER_PRIVATE_
+#define SCHEDULER_KIST_PRIVATE
+#include "core/or/scheduler.h"
+#include "core/mainloop/mainloop.h"
+#include "lib/container/buffers.h"
+#define TOR_CHANNEL_INTERNAL_
+#include "core/or/channeltls.h"
+#include "lib/evloop/compat_libevent.h"
+
+#include "core/or/or_connection_st.h"
+
+/**
+ * \file scheduler.c
+ * \brief Channel scheduling system: decides which channels should send and
+ * receive when.
+ *
+ * This module is the global/common parts of the scheduling system. This system
+ * is what decides what channels get to send cells on their circuits and when.
+ *
+ * Terms:
+ * - "Scheduling system": the collection of scheduler*.{h,c} files and their
+ *   aggregate behavior.
+ * - "Scheduler implementation": a scheduler_t. The scheduling system has one
+ *   active scheduling implementation at a time.
+ *
+ * In this file you will find state that any scheduler implementation can have
+ * access to as well as the functions the rest of Tor uses to interact with the
+ * scheduling system.
+ *
+ * The earliest versions of Tor approximated a kind of round-robin system
+ * among active connections, but only approximated it. It would only consider
+ * one connection (roughly equal to a channel in today's terms) at a time, and
+ * thus could only prioritize circuits against others on the same connection.
+ *
+ * Then in response to the KIST paper[0], Tor implemented a global
+ * circuit scheduler. It was supposed to prioritize circuits across many
+ * channels, but wasn't effective. It is preserved in scheduler_vanilla.c.
+ *
+ * [0]: http://www.robgjansen.com/publications/kist-sec2014.pdf
+ *
+ * Then we actually got around to implementing KIST for real. We decided to
+ * modularize the scheduler so new ones can be implemented. You can find KIST
+ * in scheduler_kist.c.
+ *
+ * Channels have one of four scheduling states based on whether or not they
+ * have cells to send and whether or not they are able to send.
+ *
+ * <ol>
+ * <li>
+ *   Not open for writes, no cells to send.
+ *     <ul><li> Not much to do here, and the channel will have scheduler_state
+ *       == SCHED_CHAN_IDLE
+ *     <li> Transitions from:
+ *       <ul>
+ *       <li>Open for writes/has cells by simultaneously draining all circuit
+ *         queues and filling the output buffer.
+ *       </ul>
+ *     <li> Transitions to:
+ *      <ul>
+ *       <li> Not open for writes/has cells by arrival of cells on an attached
+ *         circuit (this would be driven from append_cell_to_circuit_queue())
+ *       <li> Open for writes/no cells by a channel type specific path;
+ *         driven from connection_or_flushed_some() for channel_tls_t.
+ *      </ul>
+ *    </ul>
+ *
+ * <li> Open for writes, no cells to send
+ *   <ul>
+ *     <li>Not much here either; this will be the state an idle but open
+ *       channel can be expected to settle in.  It will have scheduler_state
+ *       == SCHED_CHAN_WAITING_FOR_CELLS
+ *     <li> Transitions from:
+ *       <ul>
+ *       <li>Not open for writes/no cells by flushing some of the output
+ *         buffer.
+ *       <li>Open for writes/has cells by the scheduler moving cells from
+ *         circuit queues to channel output queue, but not having enough
+ *         to fill the output queue.
+ *       </ul>
+ *     <li> Transitions to:
+ *       <ul>
+ *        <li>Open for writes/has cells by arrival of new cells on an attached
+ *         circuit, in append_cell_to_circuit_queue()
+ *       </ul>
+ *     </ul>
+ *
+ * <li>Not open for writes, cells to send
+ *     <ul>
+ *     <li>This is the state of a busy circuit limited by output bandwidth;
+ *       cells have piled up in the circuit queues waiting to be relayed.
+ *       The channel will have scheduler_state == SCHED_CHAN_WAITING_TO_WRITE.
+ *     <li> Transitions from:
+ *       <ul>
+ *       <li>Not open for writes/no cells by arrival of cells on an attached
+ *         circuit
+ *       <li>Open for writes/has cells by filling an output buffer without
+ *         draining all cells from attached circuits
+ *       </ul>
+ *    <li> Transitions to:
+ *       <ul>
+ *       <li>Opens for writes/has cells by draining some of the output buffer
+ *         via the connection_or_flushed_some() path (for channel_tls_t).
+ *       </ul>
+ *    </ul>
+ *
+ * <li>Open for writes, cells to send
+ *     <ul>
+ *     <li>This connection is ready to relay some cells and waiting for
+ *       the scheduler to choose it.  The channel will have scheduler_state ==
+ *       SCHED_CHAN_PENDING.
+ *     <li>Transitions from:
+ *       <ul>
+ *       <li>Not open for writes/has cells by the connection_or_flushed_some()
+ *         path
+ *       <li>Open for writes/no cells by the append_cell_to_circuit_queue()
+ *         path
+ *       </ul>
+ *     <li> Transitions to:
+ *       <ul>
+ *        <li>Not open for writes/no cells by draining all circuit queues and
+ *          simultaneously filling the output buffer.
+ *        <li>Not open for writes/has cells by writing enough cells to fill the
+ *         output buffer
+ *        <li>Open for writes/no cells by draining all attached circuit queues
+ *         without also filling the output buffer
+ *       </ul>
+ *    </ul>
+ * </ol>
+ *
+ * Other event-driven parts of the code move channels between these scheduling
+ * states by calling scheduler functions. The scheduling system builds up a
+ * list of channels in the SCHED_CHAN_PENDING state that the scheduler
+ * implementation should then use when it runs. Scheduling implementations need
+ * to properly update channel states during their scheduler_t->run() function
+ * as that is the only opportunity for channels to move from SCHED_CHAN_PENDING
+ * to any other state.
+ *
+ * The remainder of this file is a small amount of state that any scheduler
+ * implementation should have access to, and the functions the rest of Tor uses
+ * to interact with the scheduling system.
+ */
+
+/*****************************************************************************
+ * Scheduling system state
+ *
+ * State that can be accessed from any scheduler implementation (but not
+ * outside the scheduling system)
+ *****************************************************************************/
+
+/** DOCDOC */
+STATIC const scheduler_t *the_scheduler;
+
+/**
+ * We keep a list of channels that are pending - i.e, have cells to write
+ * and can accept them to send. The enum scheduler_state in channel_t
+ * is reserved for our use.
+ *
+ * Priority queue of channels that can write and have cells (pending work)
+ */
+STATIC smartlist_t *channels_pending = NULL;
+
+/**
+ * This event runs the scheduler from its callback, and is manually
+ * activated whenever a channel enters open for writes/cells to send.
+ */
+STATIC struct mainloop_event_t *run_sched_ev = NULL;
+
+static int have_logged_kist_suddenly_disabled = 0;
+
+/*****************************************************************************
+ * Scheduling system static function definitions
+ *
+ * Functions that can only be accessed from this file.
+ *****************************************************************************/
+
+/** Return a human readable string for the given scheduler type. */
+static const char *
+get_scheduler_type_string(scheduler_types_t type)
+{
+  switch (type) {
+  case SCHEDULER_VANILLA:
+    return "Vanilla";
+  case SCHEDULER_KIST:
+    return "KIST";
+  case SCHEDULER_KIST_LITE:
+    return "KISTLite";
+  case SCHEDULER_NONE:
+    /* fallthrough */
+  default:
+    tor_assert_unreached();
+    return "(N/A)";
+  }
+}
+
+/**
+ * Scheduler event callback; this should get triggered once per event loop
+ * if any scheduling work was created during the event loop.
+ */
+static void
+scheduler_evt_callback(mainloop_event_t *event, void *arg)
+{
+  (void) event;
+  (void) arg;
+
+  log_debug(LD_SCHED, "Scheduler event callback called");
+
+  /* Run the scheduler. This is a mandatory function. */
+
+  /* We might as well assert on this. If this function doesn't exist, no cells
+   * are getting scheduled. Things are very broken. scheduler_t says the run()
+   * function is mandatory. */
+  tor_assert(the_scheduler->run);
+  the_scheduler->run();
+
+  /* Schedule itself back in if it has more work. */
+
+  /* Again, might as well assert on this mandatory scheduler_t function. If it
+   * doesn't exist, there's no way to tell libevent to run the scheduler again
+   * in the future. */
+  tor_assert(the_scheduler->schedule);
+  the_scheduler->schedule();
+}
+
+/** Using the global options, select the scheduler we should be using. */
+static void
+select_scheduler(void)
+{
+  scheduler_t *new_scheduler = NULL;
+
+#ifdef TOR_UNIT_TESTS
+  /* This is hella annoying to set in the options for every test that passes
+   * through the scheduler and there are many so if we don't explicitly have
+   * a list of types set, just put the vanilla one. */
+  if (get_options()->SchedulerTypes_ == NULL) {
+    the_scheduler = get_vanilla_scheduler();
+    return;
+  }
+#endif /* defined(TOR_UNIT_TESTS) */
+
+  /* This list is ordered that is first entry has the first priority. Thus, as
+   * soon as we find a scheduler type that we can use, we use it and stop. */
+  SMARTLIST_FOREACH_BEGIN(get_options()->SchedulerTypes_, int *, type) {
+    switch (*type) {
+    case SCHEDULER_VANILLA:
+      new_scheduler = get_vanilla_scheduler();
+      goto end;
+    case SCHEDULER_KIST:
+      if (!scheduler_can_use_kist()) {
+#ifdef HAVE_KIST_SUPPORT
+        if (!have_logged_kist_suddenly_disabled) {
+          /* We should only log this once in most cases. If it was the kernel
+           * losing support for kist that caused scheduler_can_use_kist() to
+           * return false, then this flag makes sure we only log this message
+           * once. If it was the consensus that switched from "yes use kist"
+           * to "no don't use kist", then we still set the flag so we log
+           * once, but we unset the flag elsewhere if we ever can_use_kist()
+           * again.
+           */
+          have_logged_kist_suddenly_disabled = 1;
+          log_notice(LD_SCHED, "Scheduler type KIST has been disabled by "
+                               "the consensus or no kernel support.");
+        }
+#else /* !(defined(HAVE_KIST_SUPPORT)) */
+        log_info(LD_SCHED, "Scheduler type KIST not built in");
+#endif /* defined(HAVE_KIST_SUPPORT) */
+        continue;
+      }
+      /* This flag will only get set in one of two cases:
+       * 1 - the kernel lost support for kist. In that case, we don't expect to
+       *     ever end up here
+       * 2 - the consensus went from "yes use kist" to "no don't use kist".
+       * We might end up here if the consensus changes back to "yes", in which
+       * case we might want to warn the user again if it goes back to "no"
+       * yet again. Thus we unset the flag */
+      have_logged_kist_suddenly_disabled = 0;
+      new_scheduler = get_kist_scheduler();
+      scheduler_kist_set_full_mode();
+      goto end;
+    case SCHEDULER_KIST_LITE:
+      new_scheduler = get_kist_scheduler();
+      scheduler_kist_set_lite_mode();
+      goto end;
+    case SCHEDULER_NONE:
+      /* fallthrough */
+    default:
+      /* Our option validation should have caught this. */
+      tor_assert_unreached();
+    }
+  } SMARTLIST_FOREACH_END(type);
+
+ end:
+  if (new_scheduler == NULL) {
+    log_err(LD_SCHED, "Tor was unable to select a scheduler type. Please "
+                      "make sure Schedulers is correctly configured with "
+                      "what Tor does support.");
+    /* We weren't able to choose a scheduler which means that none of the ones
+     * set in Schedulers are supported or usable. We will respect the user
+     * wishes of using what it has been configured and don't do a sneaky
+     * fallback. Because this can be changed at runtime, we have to stop tor
+     * right now. */
+    exit(1); // XXXX bad exit
+  }
+
+  /* Set the chosen scheduler. */
+  the_scheduler = new_scheduler;
+}
+
+/**
+ * Helper function called from a few different places. It changes the
+ * scheduler implementation, if necessary. And if it did, it then tells the
+ * old one to free its state and the new one to initialize.
+ */
+static void
+set_scheduler(void)
+{
+  const scheduler_t *old_scheduler = the_scheduler;
+  scheduler_types_t old_scheduler_type = SCHEDULER_NONE;
+
+  /* We keep track of the type in order to log only if the type switched. We
+   * can't just use the scheduler pointers because KIST and KISTLite share the
+   * same object. */
+  if (the_scheduler) {
+    old_scheduler_type = the_scheduler->type;
+  }
+
+  /* From the options, select the scheduler type to set. */
+  select_scheduler();
+  tor_assert(the_scheduler);
+
+  /* We look at the pointer difference in case the old sched and new sched
+   * share the same scheduler object, as is the case with KIST and KISTLite. */
+  if (old_scheduler != the_scheduler) {
+    /* Allow the old scheduler to clean up, if needed. */
+    if (old_scheduler && old_scheduler->free_all) {
+      old_scheduler->free_all();
+    }
+
+    /* Initialize the new scheduler. */
+    if (the_scheduler->init) {
+      the_scheduler->init();
+    }
+  }
+
+  /* Finally we notice log if we switched schedulers. We use the type in case
+   * two schedulers share a scheduler object. */
+  if (old_scheduler_type != the_scheduler->type) {
+    log_info(LD_CONFIG, "Scheduler type %s has been enabled.",
+             get_scheduler_type_string(the_scheduler->type));
+  }
+}
+
+/*****************************************************************************
+ * Scheduling system private function definitions
+ *
+ * Functions that can only be accessed from scheduler*.c
+ *****************************************************************************/
+
+/** Returns human readable string for the given channel scheduler state. */
+const char *
+get_scheduler_state_string(int scheduler_state)
+{
+  switch (scheduler_state) {
+  case SCHED_CHAN_IDLE:
+    return "IDLE";
+  case SCHED_CHAN_WAITING_FOR_CELLS:
+    return "WAITING_FOR_CELLS";
+  case SCHED_CHAN_WAITING_TO_WRITE:
+    return "WAITING_TO_WRITE";
+  case SCHED_CHAN_PENDING:
+    return "PENDING";
+  default:
+    return "(invalid)";
+  }
+}
+
+/** Helper that logs channel scheduler_state changes. Use this instead of
+ * setting scheduler_state directly. */
+void
+scheduler_set_channel_state(channel_t *chan, int new_state)
+{
+  log_debug(LD_SCHED, "chan %" PRIu64 " changed from scheduler state %s to %s",
+      chan->global_identifier,
+      get_scheduler_state_string(chan->scheduler_state),
+      get_scheduler_state_string(new_state));
+  chan->scheduler_state = new_state;
+}
+
+/** Return the pending channel list. */
+smartlist_t *
+get_channels_pending(void)
+{
+  return channels_pending;
+}
+
+/** Comparison function to use when sorting pending channels. */
+MOCK_IMPL(int,
+scheduler_compare_channels, (const void *c1_v, const void *c2_v))
+{
+  const channel_t *c1 = NULL, *c2 = NULL;
+  /* These are a workaround for -Wbad-function-cast throwing a fit */
+  const circuitmux_policy_t *p1, *p2;
+  uintptr_t p1_i, p2_i;
+
+  tor_assert(c1_v);
+  tor_assert(c2_v);
+
+  c1 = (const channel_t *)(c1_v);
+  c2 = (const channel_t *)(c2_v);
+
+  if (c1 != c2) {
+    if (circuitmux_get_policy(c1->cmux) ==
+        circuitmux_get_policy(c2->cmux)) {
+      /* Same cmux policy, so use the mux comparison */
+      return circuitmux_compare_muxes(c1->cmux, c2->cmux);
+    } else {
+      /*
+       * Different policies; not important to get this edge case perfect
+       * because the current code never actually gives different channels
+       * different cmux policies anyway.  Just use this arbitrary but
+       * definite choice.
+       */
+      p1 = circuitmux_get_policy(c1->cmux);
+      p2 = circuitmux_get_policy(c2->cmux);
+      p1_i = (uintptr_t)p1;
+      p2_i = (uintptr_t)p2;
+
+      return (p1_i < p2_i) ? -1 : 1;
+    }
+  } else {
+    /* c1 == c2, so always equal */
+    return 0;
+  }
+}
+
+/*****************************************************************************
+ * Scheduling system global functions
+ *
+ * Functions that can be accessed from anywhere in Tor.
+ *****************************************************************************/
+
+/**
+ * This is how the scheduling system is notified of Tor's configuration
+ * changing. For example: a SIGHUP was issued.
+ */
+void
+scheduler_conf_changed(void)
+{
+  /* Let the scheduler decide what it should do. */
+  set_scheduler();
+
+  /* Then tell the (possibly new) scheduler that we have new options. */
+  if (the_scheduler->on_new_options) {
+    the_scheduler->on_new_options();
+  }
+}
+
+/**
+ * Whenever we get a new consensus, this function is called.
+ */
+void
+scheduler_notify_networkstatus_changed(void)
+{
+  /* Maybe the consensus param made us change the scheduler. */
+  set_scheduler();
+
+  /* Then tell the (possibly new) scheduler that we have a new consensus */
+  if (the_scheduler->on_new_consensus) {
+    the_scheduler->on_new_consensus();
+  }
+}
+
+/**
+ * Free everything scheduling-related from main.c. Note this is only called
+ * when Tor is shutting down, while scheduler_t->free_all() is called both when
+ * Tor is shutting down and when we are switching schedulers.
+ */
+void
+scheduler_free_all(void)
+{
+  log_debug(LD_SCHED, "Shutting down scheduler");
+
+  if (run_sched_ev) {
+    mainloop_event_free(run_sched_ev);
+    run_sched_ev = NULL;
+  }
+
+  if (channels_pending) {
+    /* We don't have ownership of the objects in this list. */
+    smartlist_free(channels_pending);
+    channels_pending = NULL;
+  }
+
+  if (the_scheduler && the_scheduler->free_all) {
+    the_scheduler->free_all();
+  }
+  the_scheduler = NULL;
+}
+
+/** Mark a channel as no longer ready to accept writes. */
+MOCK_IMPL(void,
+scheduler_channel_doesnt_want_writes,(channel_t *chan))
+{
+  IF_BUG_ONCE(!chan) {
+    return;
+  }
+  IF_BUG_ONCE(!channels_pending) {
+    return;
+  }
+
+  /* If it's already in pending, we can put it in waiting_to_write */
+  if (chan->scheduler_state == SCHED_CHAN_PENDING) {
+    /*
+     * It's in channels_pending, so it shouldn't be in any of
+     * the other lists.  It can't write any more, so it goes to
+     * channels_waiting_to_write.
+     */
+    smartlist_pqueue_remove(channels_pending,
+                            scheduler_compare_channels,
+                            offsetof(channel_t, sched_heap_idx),
+                            chan);
+    scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_TO_WRITE);
+  } else {
+    /*
+     * It's not in pending, so it can't become waiting_to_write; it's
+     * either not in any of the lists (nothing to do) or it's already in
+     * waiting_for_cells (remove it, can't write any more).
+     */
+    if (chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS) {
+      scheduler_set_channel_state(chan, SCHED_CHAN_IDLE);
+    }
+  }
+}
+
+/** Mark a channel as having waiting cells. */
+MOCK_IMPL(void,
+scheduler_channel_has_waiting_cells,(channel_t *chan))
+{
+  IF_BUG_ONCE(!chan) {
+    return;
+  }
+  IF_BUG_ONCE(!channels_pending) {
+    return;
+  }
+
+  /* First, check if it's also writeable */
+  if (chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS) {
+    /*
+     * It's in channels_waiting_for_cells, so it shouldn't be in any of
+     * the other lists.  It has waiting cells now, so it goes to
+     * channels_pending.
+     */
+    scheduler_set_channel_state(chan, SCHED_CHAN_PENDING);
+    if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) {
+      smartlist_pqueue_add(channels_pending,
+                           scheduler_compare_channels,
+                           offsetof(channel_t, sched_heap_idx),
+                           chan);
+    }
+    /* If we made a channel pending, we potentially have scheduling work to
+     * do. */
+    the_scheduler->schedule();
+  } else {
+    /*
+     * It's not in waiting_for_cells, so it can't become pending; it's
+     * either not in any of the lists (we add it to waiting_to_write)
+     * or it's already in waiting_to_write or pending (we do nothing)
+     */
+    if (!(chan->scheduler_state == SCHED_CHAN_WAITING_TO_WRITE ||
+          chan->scheduler_state == SCHED_CHAN_PENDING)) {
+      scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_TO_WRITE);
+    }
+  }
+}
+
+/** Add the scheduler event to the set of pending events with next_run being
+ * the longest time libevent should wait before triggering the event. */
+void
+scheduler_ev_add(const struct timeval *next_run)
+{
+  tor_assert(run_sched_ev);
+  tor_assert(next_run);
+  if (BUG(mainloop_event_schedule(run_sched_ev, next_run) < 0)) {
+    log_warn(LD_SCHED, "Adding to libevent failed. Next run time was set to: "
+                       "%ld.%06ld", next_run->tv_sec, (long)next_run->tv_usec);
+    return;
+  }
+}
+
+/** Make the scheduler event active with the given flags. */
+void
+scheduler_ev_active(void)
+{
+  tor_assert(run_sched_ev);
+  mainloop_event_activate(run_sched_ev);
+}
+
+/*
+ * Initialize everything scheduling-related from config.c. Note this is only
+ * called when Tor is starting up, while scheduler_t->init() is called both
+ * when Tor is starting up and when we are switching schedulers.
+ */
+void
+scheduler_init(void)
+{
+  log_debug(LD_SCHED, "Initting scheduler");
+
+  // Two '!' because we really do want to check if the pointer is non-NULL
+  IF_BUG_ONCE(!!run_sched_ev) {
+    log_warn(LD_SCHED, "We should not already have a libevent scheduler event."
+             "I'll clean the old one up, but this is odd.");
+    mainloop_event_free(run_sched_ev);
+    run_sched_ev = NULL;
+  }
+  run_sched_ev = mainloop_event_new(scheduler_evt_callback, NULL);
+  channels_pending = smartlist_new();
+
+  set_scheduler();
+}
+
+/*
+ * If a channel is going away, this is how the scheduling system is informed
+ * so it can do any freeing necessary. This ultimately calls
+ * scheduler_t->on_channel_free() so the current scheduler can release any
+ * state specific to this channel.
+ */
+MOCK_IMPL(void,
+scheduler_release_channel,(channel_t *chan))
+{
+  IF_BUG_ONCE(!chan) {
+    return;
+  }
+  IF_BUG_ONCE(!channels_pending) {
+    return;
+  }
+
+  /* Try to remove the channel from the pending list regardless of its
+   * scheduler state. We can release a channel in many places in the tor code
+   * so we can't rely on the channel state (PENDING) to remove it from the
+   * list.
+   *
+   * For instance, the channel can change state from OPEN to CLOSING while
+   * being handled in the scheduler loop leading to the channel being in
+   * PENDING state but not in the pending list. Furthermore, we release the
+   * channel when it changes state to close and a second time when we free it.
+   * Not ideal at all but for now that is the way it is. */
+  if (chan->sched_heap_idx != -1) {
+    smartlist_pqueue_remove(channels_pending,
+                            scheduler_compare_channels,
+                            offsetof(channel_t, sched_heap_idx),
+                            chan);
+  }
+
+  if (the_scheduler->on_channel_free) {
+    the_scheduler->on_channel_free(chan);
+  }
+  scheduler_set_channel_state(chan, SCHED_CHAN_IDLE);
+}
+
+/** Mark a channel as ready to accept writes */
+
+void
+scheduler_channel_wants_writes(channel_t *chan)
+{
+  IF_BUG_ONCE(!chan) {
+    return;
+  }
+  IF_BUG_ONCE(!channels_pending) {
+    return;
+  }
+
+  /* If it's already in waiting_to_write, we can put it in pending */
+  if (chan->scheduler_state == SCHED_CHAN_WAITING_TO_WRITE) {
+    /*
+     * It can write now, so it goes to channels_pending.
+     */
+    scheduler_set_channel_state(chan, SCHED_CHAN_PENDING);
+    if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) {
+      smartlist_pqueue_add(channels_pending,
+                           scheduler_compare_channels,
+                           offsetof(channel_t, sched_heap_idx),
+                           chan);
+    }
+    /* We just made a channel pending, we have scheduling work to do. */
+    the_scheduler->schedule();
+  } else {
+    /*
+     * It's not in SCHED_CHAN_WAITING_TO_WRITE, so it can't become pending;
+     * it's either idle and goes to WAITING_FOR_CELLS, or it's a no-op.
+     */
+    if (!(chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS ||
+          chan->scheduler_state == SCHED_CHAN_PENDING)) {
+      scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_FOR_CELLS);
+    }
+  }
+}
+
+/* Log warn the given channel and extra scheduler context as well. This is
+ * used by SCHED_BUG() in order to be able to extract as much information as
+ * we can when we hit a bug. Channel chan can be NULL. */
+void
+scheduler_bug_occurred(const channel_t *chan)
+{
+  char buf[128];
+
+  if (chan != NULL) {
+    const size_t outbuf_len =
+      buf_datalen(TO_CONN(BASE_CHAN_TO_TLS((channel_t *) chan)->conn)->outbuf);
+    tor_snprintf(buf, sizeof(buf),
+                 "Channel %" PRIu64 " in state %s and scheduler state %s."
+                 " Num cells on cmux: %d. Connection outbuf len: %lu.",
+                 chan->global_identifier,
+                 channel_state_to_string(chan->state),
+                 get_scheduler_state_string(chan->scheduler_state),
+                 circuitmux_num_cells(chan->cmux),
+                 (unsigned long)outbuf_len);
+  }
+
+  {
+    char *msg;
+    /* Rate limit every 60 seconds. If we start seeing this every 60 sec, we
+     * know something is stuck/wrong. It *should* be loud but not too much. */
+    static ratelim_t rlimit = RATELIM_INIT(60);
+    if ((msg = rate_limit_log(&rlimit, approx_time()))) {
+      log_warn(LD_BUG, "%s Num pending channels: %d. "
+                       "Channel in pending list: %s.%s",
+               (chan != NULL) ? buf : "No channel in bug context.",
+               smartlist_len(channels_pending),
+               (smartlist_pos(channels_pending, chan) == -1) ? "no" : "yes",
+               msg);
+      tor_free(msg);
+    }
+  }
+}
+
+#ifdef TOR_UNIT_TESTS
+
+/*
+ * Notify scheduler that a channel's queue position may have changed.
+ */
+void
+scheduler_touch_channel(channel_t *chan)
+{
+  IF_BUG_ONCE(!chan) {
+    return;
+  }
+
+  if (chan->scheduler_state == SCHED_CHAN_PENDING) {
+    /* Remove and re-add it */
+    smartlist_pqueue_remove(channels_pending,
+                            scheduler_compare_channels,
+                            offsetof(channel_t, sched_heap_idx),
+                            chan);
+    smartlist_pqueue_add(channels_pending,
+                         scheduler_compare_channels,
+                         offsetof(channel_t, sched_heap_idx),
+                         chan);
+  }
+  /* else no-op, since it isn't in the queue */
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/core/or/scheduler.h b/src/core/or/scheduler.h
new file mode 100644
index 0000000000..843be2603c
--- /dev/null
+++ b/src/core/or/scheduler.h
@@ -0,0 +1,218 @@
+/* * Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file scheduler.h
+ * \brief Header file for scheduler*.c
+ **/
+
+#ifndef TOR_SCHEDULER_H
+#define TOR_SCHEDULER_H
+
+#include "core/or/or.h"
+#include "core/or/channel.h"
+#include "lib/testsupport/testsupport.h"
+
+/** Scheduler type, we build an ordered list with those values from the
+ * parsed strings in Schedulers. The reason to do such a thing is so we can
+ * quickly and without parsing strings select the scheduler at anytime. */
+typedef enum {
+  SCHEDULER_NONE =     -1,
+  SCHEDULER_VANILLA =   1,
+  SCHEDULER_KIST =      2,
+  SCHEDULER_KIST_LITE = 3,
+} scheduler_types_t;
+
+/**
+ * A scheduler implementation is a collection of function pointers. If you
+ * would like to add a new scheduler called foo, create scheduler_foo.c,
+ * implement at least the mandatory ones, and implement get_foo_scheduler()
+ * that returns a complete scheduler_t for your foo scheduler. See
+ * scheduler_kist.c for an example.
+ *
+ * These function pointers SHOULD NOT be used anywhere outside of the
+ * scheduling source files. The rest of Tor should communicate with the
+ * scheduling system through the functions near the bottom of this file, and
+ * those functions will call into the current scheduler implementation as
+ * necessary.
+ *
+ * If your scheduler doesn't need to implement something (for example: it
+ * doesn't create any state for itself, thus it has nothing to free when Tor
+ * is shutting down), then set that function pointer to NULL.
+ */
+typedef struct scheduler_s {
+  /* Scheduler type. This is used for logging when the scheduler is switched
+   * during runtime. */
+  scheduler_types_t type;
+
+  /* (Optional) To be called when we want to prepare a scheduler for use.
+   * Perhaps Tor just started and we are the lucky chosen scheduler, or
+   * perhaps Tor is switching to this scheduler. No matter the case, this is
+   * where we would prepare any state and initialize parameters. You might
+   * think of this as the opposite of free_all(). */
+  void (*init)(void);
+
+  /* (Optional) To be called when we want to tell the scheduler to delete all
+   * of its state (if any). Perhaps Tor is shutting down or perhaps we are
+   * switching schedulers. */
+  void (*free_all)(void);
+
+  /* (Mandatory) Libevent controls the main event loop in Tor, and this is
+   * where we register with libevent the next execution of run_sched_ev [which
+   * ultimately calls run()]. */
+  void (*schedule)(void);
+
+  /* (Mandatory) This is the heart of a scheduler! This is where the
+   * excitement happens! Here libevent has given us the chance to execute, and
+   * we should do whatever we need to do in order to move some cells from
+   * their circuit queues to output buffers in an intelligent manner. We
+   * should do this quickly. When we are done, we'll try to schedule() ourself
+   * if more work needs to be done to setup the next scheduling run. */
+  void (*run)(void);
+
+  /*
+   * External event not related to the scheduler but that can influence it.
+   */
+
+  /* (Optional) To be called whenever Tor finds out about a new consensus.
+   * First the scheduling system as a whole will react to the new consensus
+   * and change the scheduler if needed. After that, the current scheduler
+   * (which might be new) will call this so it has the chance to react to the
+   * new consensus too. If there's a consensus parameter that your scheduler
+   * wants to keep an eye on, this is where you should check for it.  */
+  void (*on_new_consensus)(void);
+
+  /* (Optional) To be called when a channel is being freed. Sometimes channels
+   * go away (for example: the relay on the other end is shutting down). If
+   * the scheduler keeps any channel-specific state and has memory to free
+   * when channels go away, implement this and free it here. */
+  void (*on_channel_free)(const channel_t *);
+
+  /* (Optional) To be called whenever Tor is reloading configuration options.
+   * For example: SIGHUP was issued and Tor is rereading its torrc. A
+   * scheduler should use this as an opportunity to parse and cache torrc
+   * options so that it doesn't have to call get_options() all the time. */
+  void (*on_new_options)(void);
+} scheduler_t;
+
+/*****************************************************************************
+ * Globally visible scheduler variables/values
+ *
+ * These are variables/constants that all of Tor should be able to see.
+ *****************************************************************************/
+
+/* Default interval that KIST runs (in ms). */
+#define KIST_SCHED_RUN_INTERVAL_DEFAULT 10
+/* Minimum interval that KIST runs. This value disables KIST. */
+#define KIST_SCHED_RUN_INTERVAL_MIN 0
+/* Maximum interval that KIST runs (in ms). */
+#define KIST_SCHED_RUN_INTERVAL_MAX 100
+
+/*****************************************************************************
+ * Globally visible scheduler functions
+ *
+ * These functions are how the rest of Tor communicates with the scheduling
+ * system.
+ *****************************************************************************/
+
+void scheduler_init(void);
+void scheduler_free_all(void);
+void scheduler_conf_changed(void);
+void scheduler_notify_networkstatus_changed(void);
+MOCK_DECL(void, scheduler_release_channel, (channel_t *chan));
+
+/*
+ * Ways for a channel to interact with the scheduling system. A channel only
+ * really knows (i) whether or not it has cells it wants to send, and
+ * (ii) whether or not it would like to write.
+ */
+void scheduler_channel_wants_writes(channel_t *chan);
+MOCK_DECL(void, scheduler_channel_doesnt_want_writes, (channel_t *chan));
+MOCK_DECL(void, scheduler_channel_has_waiting_cells, (channel_t *chan));
+
+/*****************************************************************************
+ * Private scheduler functions
+ *
+ * These functions are only visible to the scheduling system, the current
+ * scheduler implementation, and tests.
+ *****************************************************************************/
+#ifdef SCHEDULER_PRIVATE_
+
+/*********************************
+ * Defined in scheduler.c
+ *********************************/
+
+void scheduler_set_channel_state(channel_t *chan, int new_state);
+const char *get_scheduler_state_string(int scheduler_state);
+
+/* Triggers a BUG() and extra information with chan if available. */
+#define SCHED_BUG(cond, chan) \
+  (PREDICT_UNLIKELY(cond) ? \
+   ((BUG(cond)) ? (scheduler_bug_occurred(chan), 1) : 0) : 0)
+
+void scheduler_bug_occurred(const channel_t *chan);
+
+smartlist_t *get_channels_pending(void);
+MOCK_DECL(int, scheduler_compare_channels,
+          (const void *c1_v, const void *c2_v));
+void scheduler_ev_active(void);
+void scheduler_ev_add(const struct timeval *next_run);
+
+#ifdef TOR_UNIT_TESTS
+extern smartlist_t *channels_pending;
+extern struct mainloop_event_t *run_sched_ev;
+extern const scheduler_t *the_scheduler;
+void scheduler_touch_channel(channel_t *chan);
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/*********************************
+ * Defined in scheduler_kist.c
+ *********************************/
+
+#ifdef SCHEDULER_KIST_PRIVATE
+
+/* Socket table entry which holds information of a channel's socket and kernel
+ * TCP information. Only used by KIST. */
+typedef struct socket_table_ent_s {
+  HT_ENTRY(socket_table_ent_s) node;
+  const channel_t *chan;
+  /* Amount written this scheduling run */
+  uint64_t written;
+  /* Amount that can be written this scheduling run */
+  uint64_t limit;
+  /* TCP info from the kernel */
+  uint32_t cwnd;
+  uint32_t unacked;
+  uint32_t mss;
+  uint32_t notsent;
+} socket_table_ent_t;
+
+typedef HT_HEAD(outbuf_table_s, outbuf_table_ent_s) outbuf_table_t;
+
+MOCK_DECL(int, channel_should_write_to_kernel,
+          (outbuf_table_t *table, channel_t *chan));
+MOCK_DECL(void, channel_write_to_kernel, (channel_t *chan));
+MOCK_DECL(void, update_socket_info_impl, (socket_table_ent_t *ent));
+
+int scheduler_can_use_kist(void);
+void scheduler_kist_set_full_mode(void);
+void scheduler_kist_set_lite_mode(void);
+scheduler_t *get_kist_scheduler(void);
+int kist_scheduler_run_interval(void);
+
+#ifdef TOR_UNIT_TESTS
+extern int32_t sched_run_interval;
+#endif /* TOR_UNIT_TESTS */
+
+#endif /* defined(SCHEDULER_KIST_PRIVATE) */
+
+/*********************************
+ * Defined in scheduler_vanilla.c
+ *********************************/
+
+scheduler_t *get_vanilla_scheduler(void);
+
+#endif /* defined(SCHEDULER_PRIVATE_) */
+
+#endif /* !defined(TOR_SCHEDULER_H) */
+
diff --git a/src/core/or/scheduler_kist.c b/src/core/or/scheduler_kist.c
new file mode 100644
index 0000000000..79ecb0bc7e
--- /dev/null
+++ b/src/core/or/scheduler_kist.c
@@ -0,0 +1,844 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define SCHEDULER_KIST_PRIVATE
+
+#include "core/or/or.h"
+#include "lib/container/buffers.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "feature/nodelist/networkstatus.h"
+#define TOR_CHANNEL_INTERNAL_
+#include "core/or/channel.h"
+#include "core/or/channeltls.h"
+#define SCHEDULER_PRIVATE_
+#include "core/or/scheduler.h"
+#include "lib/math/fp.h"
+
+#include "core/or/or_connection_st.h"
+
+#ifdef HAVE_SYS_IOCTL_H
+#include <sys/ioctl.h>
+#endif
+
+#ifdef HAVE_KIST_SUPPORT
+/* Kernel interface needed for KIST. */
+#include <netinet/tcp.h>
+#include <linux/sockios.h>
+#endif /* HAVE_KIST_SUPPORT */
+
+/*****************************************************************************
+ * Data structures and supporting functions
+ *****************************************************************************/
+
+/* Socket_table hash table stuff. The socket_table keeps track of per-socket
+ * limit information imposed by kist and used by kist. */
+
+static uint32_t
+socket_table_ent_hash(const socket_table_ent_t *ent)
+{
+  return (uint32_t)ent->chan->global_identifier;
+}
+
+static unsigned
+socket_table_ent_eq(const socket_table_ent_t *a, const socket_table_ent_t *b)
+{
+  return a->chan == b->chan;
+}
+
+typedef HT_HEAD(socket_table_s, socket_table_ent_s) socket_table_t;
+
+static socket_table_t socket_table = HT_INITIALIZER();
+
+HT_PROTOTYPE(socket_table_s, socket_table_ent_s, node, socket_table_ent_hash,
+             socket_table_ent_eq)
+HT_GENERATE2(socket_table_s, socket_table_ent_s, node, socket_table_ent_hash,
+             socket_table_ent_eq, 0.6, tor_reallocarray, tor_free_)
+
+/* outbuf_table hash table stuff. The outbuf_table keeps track of which
+ * channels have data sitting in their outbuf so the kist scheduler can force
+ * a write from outbuf to kernel periodically during a run and at the end of a
+ * run. */
+
+typedef struct outbuf_table_ent_s {
+  HT_ENTRY(outbuf_table_ent_s) node;
+  channel_t *chan;
+} outbuf_table_ent_t;
+
+static uint32_t
+outbuf_table_ent_hash(const outbuf_table_ent_t *ent)
+{
+  return (uint32_t)ent->chan->global_identifier;
+}
+
+static unsigned
+outbuf_table_ent_eq(const outbuf_table_ent_t *a, const outbuf_table_ent_t *b)
+{
+  return a->chan->global_identifier == b->chan->global_identifier;
+}
+
+HT_PROTOTYPE(outbuf_table_s, outbuf_table_ent_s, node, outbuf_table_ent_hash,
+             outbuf_table_ent_eq)
+HT_GENERATE2(outbuf_table_s, outbuf_table_ent_s, node, outbuf_table_ent_hash,
+             outbuf_table_ent_eq, 0.6, tor_reallocarray, tor_free_)
+
+/*****************************************************************************
+ * Other internal data
+ *****************************************************************************/
+
+/* Store the last time the scheduler was run so we can decide when to next run
+ * the scheduler based on it. */
+static monotime_t scheduler_last_run;
+/* This is a factor for the extra_space calculation in kist per-socket limits.
+ * It is the number of extra congestion windows we want to write to the kernel.
+ */
+static double sock_buf_size_factor = 1.0;
+/* How often the scheduler runs. */
+STATIC int sched_run_interval = KIST_SCHED_RUN_INTERVAL_DEFAULT;
+
+#ifdef HAVE_KIST_SUPPORT
+/* Indicate if KIST lite mode is on or off. We can disable it at runtime.
+ * Important to have because of the KISTLite -> KIST possible transition. */
+static unsigned int kist_lite_mode = 0;
+/* Indicate if we don't have the kernel support. This can happen if the kernel
+ * changed and it doesn't recognized the values passed to the syscalls needed
+ * by KIST. In that case, fallback to the naive approach. */
+static unsigned int kist_no_kernel_support = 0;
+#else /* !(defined(HAVE_KIST_SUPPORT)) */
+static unsigned int kist_lite_mode = 1;
+#endif /* defined(HAVE_KIST_SUPPORT) */
+
+/*****************************************************************************
+ * Internally called function implementations
+ *****************************************************************************/
+
+/* Little helper function to get the length of a channel's output buffer */
+static inline size_t
+channel_outbuf_length(channel_t *chan)
+{
+  tor_assert(chan);
+  /* In theory, this can not happen because we can not scheduler a channel
+   * without a connection that has its outbuf initialized. Just in case, bug
+   * on this so we can understand a bit more why it happened. */
+  if (SCHED_BUG(BASE_CHAN_TO_TLS(chan)->conn == NULL, chan)) {
+    return 0;
+  }
+  return buf_datalen(TO_CONN(BASE_CHAN_TO_TLS(chan)->conn)->outbuf);
+}
+
+/* Little helper function for HT_FOREACH_FN. */
+static int
+each_channel_write_to_kernel(outbuf_table_ent_t *ent, void *data)
+{
+  (void) data; /* Make compiler happy. */
+  channel_write_to_kernel(ent->chan);
+  return 0; /* Returning non-zero removes the element from the table. */
+}
+
+/* Free the given outbuf table entry ent. */
+static int
+free_outbuf_info_by_ent(outbuf_table_ent_t *ent, void *data)
+{
+  (void) data; /* Make compiler happy. */
+  log_debug(LD_SCHED, "Freeing outbuf table entry from chan=%" PRIu64,
+            ent->chan->global_identifier);
+  tor_free(ent);
+  return 1; /* So HT_FOREACH_FN will remove the element */
+}
+
+/* Free the given socket table entry ent. */
+static int
+free_socket_info_by_ent(socket_table_ent_t *ent, void *data)
+{
+  (void) data; /* Make compiler happy. */
+  log_debug(LD_SCHED, "Freeing socket table entry from chan=%" PRIu64,
+            ent->chan->global_identifier);
+  tor_free(ent);
+  return 1; /* So HT_FOREACH_FN will remove the element */
+}
+
+/* Clean up socket_table. Probably because the KIST sched impl is going away */
+static void
+free_all_socket_info(void)
+{
+  HT_FOREACH_FN(socket_table_s, &socket_table, free_socket_info_by_ent, NULL);
+  HT_CLEAR(socket_table_s, &socket_table);
+}
+
+static socket_table_ent_t *
+socket_table_search(socket_table_t *table, const channel_t *chan)
+{
+  socket_table_ent_t search, *ent = NULL;
+  search.chan = chan;
+  ent = HT_FIND(socket_table_s, table, &search);
+  return ent;
+}
+
+/* Free a socket entry in table for the given chan. */
+static void
+free_socket_info_by_chan(socket_table_t *table, const channel_t *chan)
+{
+  socket_table_ent_t *ent = NULL;
+  ent = socket_table_search(table, chan);
+  if (!ent)
+    return;
+  log_debug(LD_SCHED, "scheduler free socket info for chan=%" PRIu64,
+            chan->global_identifier);
+  HT_REMOVE(socket_table_s, table, ent);
+  free_socket_info_by_ent(ent, NULL);
+}
+
+/* Perform system calls for the given socket in order to calculate kist's
+ * per-socket limit as documented in the function body. */
+MOCK_IMPL(void,
+update_socket_info_impl, (socket_table_ent_t *ent))
+{
+#ifdef HAVE_KIST_SUPPORT
+  int64_t tcp_space, extra_space;
+  tor_assert(ent);
+  tor_assert(ent->chan);
+  const tor_socket_t sock =
+    TO_CONN(BASE_CHAN_TO_TLS((channel_t *) ent->chan)->conn)->s;
+  struct tcp_info tcp;
+  socklen_t tcp_info_len = sizeof(tcp);
+
+  if (kist_no_kernel_support || kist_lite_mode) {
+    goto fallback;
+  }
+
+  /* Gather information */
+  if (getsockopt(sock, SOL_TCP, TCP_INFO, (void *)&(tcp), &tcp_info_len) < 0) {
+    if (errno == EINVAL) {
+      /* Oops, this option is not provided by the kernel, we'll have to
+       * disable KIST entirely. This can happen if tor was built on a machine
+       * with the support previously or if the kernel was updated and lost the
+       * support. */
+      log_notice(LD_SCHED, "Looks like our kernel doesn't have the support "
+                           "for KIST anymore. We will fallback to the naive "
+                           "approach. Remove KIST from the Schedulers list "
+                           "to disable.");
+      kist_no_kernel_support = 1;
+    }
+    goto fallback;
+  }
+  if (ioctl(sock, SIOCOUTQNSD, &(ent->notsent)) < 0) {
+    if (errno == EINVAL) {
+      log_notice(LD_SCHED, "Looks like our kernel doesn't have the support "
+                           "for KIST anymore. We will fallback to the naive "
+                           "approach. Remove KIST from the Schedulers list "
+                           "to disable.");
+      /* Same reason as the above. */
+      kist_no_kernel_support = 1;
+    }
+    goto fallback;
+  }
+  ent->cwnd = tcp.tcpi_snd_cwnd;
+  ent->unacked = tcp.tcpi_unacked;
+  ent->mss = tcp.tcpi_snd_mss;
+
+  /* In order to reduce outbound kernel queuing delays and thus improve Tor's
+   * ability to prioritize circuits, KIST wants to set a socket write limit
+   * that is near the amount that the socket would be able to immediately send
+   * into the Internet.
+   *
+   * We first calculate how much the socket could send immediately (assuming
+   * completely full packets) according to the congestion window and the number
+   * of unacked packets.
+   *
+   * Then we add a little extra space in a controlled way. We do this so any
+   * when the kernel gets ACKs back for data currently sitting in the "TCP
+   * space", it will already have some more data to send immediately. It will
+   * not have to wait for the scheduler to run again. The amount of extra space
+   * is a factor of the current congestion window. With the suggested
+   * sock_buf_size_factor value of 1.0, we allow at most 2*cwnd bytes to sit in
+   * the kernel: 1 cwnd on the wire waiting for ACKs and 1 cwnd ready and
+   * waiting to be sent when those ACKs finally come.
+   *
+   * In the below diagram, we see some bytes in the TCP-space (denoted by '*')
+   * that have be sent onto the wire and are waiting for ACKs. We have a little
+   * more room in "TCP space" that we can fill with data that will be
+   * immediately sent. We also see the "extra space" KIST calculates. The sum
+   * of the empty "TCP space" and the "extra space" is the kist-imposed write
+   * limit for this socket.
+   *
+   * <----------------kernel-outbound-socket-queue----------------|
+   * <*********---------------------------------------------------|
+   * |----TCP-space-----|----extra-space-----|
+   * |------------------|
+   *                    ^ ((cwnd - unacked) * mss) bytes
+   *                    |--------------------|
+   *                                         ^ ((cwnd * mss) * factor) bytes
+   */
+
+  /* These values from the kernel are uint32_t, they will always fit into a
+   * int64_t tcp_space variable but if the congestion window cwnd is smaller
+   * than the unacked packets, the remaining TCP space is set to 0. */
+  if (ent->cwnd >= ent->unacked) {
+    tcp_space = (ent->cwnd - ent->unacked) * (int64_t)(ent->mss);
+  } else {
+    tcp_space = 0;
+  }
+
+  /* The clamp_double_to_int64 makes sure the first part fits into an int64_t.
+   * In fact, if sock_buf_size_factor is still forced to be >= 0 in config.c,
+   * then it will be positive for sure. Then we subtract a uint32_t. Getting a
+   * negative value is OK, see after how it is being handled. */
+  extra_space =
+    clamp_double_to_int64(
+                 (ent->cwnd * (int64_t)ent->mss) * sock_buf_size_factor) -
+    ent->notsent - (int64_t)channel_outbuf_length((channel_t *) ent->chan);
+  if ((tcp_space + extra_space) < 0) {
+    /* This means that the "notsent" queue is just too big so we shouldn't put
+     * more in the kernel for now. */
+    ent->limit = 0;
+  } else {
+    /* The positive sum of two int64_t will always fit into an uint64_t.
+     * And we know this will always be positive, since we checked above. */
+    ent->limit = (uint64_t)tcp_space + (uint64_t)extra_space;
+  }
+  return;
+
+#else /* !(defined(HAVE_KIST_SUPPORT)) */
+  goto fallback;
+#endif /* defined(HAVE_KIST_SUPPORT) */
+
+ fallback:
+  /* If all of a sudden we don't have kist support, we just zero out all the
+   * variables for this socket since we don't know what they should be. We
+   * also allow the socket to write as much as it can from the estimated
+   * number of cells the lower layer can accept, effectively returning it to
+   * Vanilla scheduler behavior. */
+  ent->cwnd = ent->unacked = ent->mss = ent->notsent = 0;
+  /* This function calls the specialized channel object (currently channeltls)
+   * and ask how many cells it can write on the outbuf which we then multiply
+   * by the size of the cells for this channel. The cast is because this
+   * function requires a non-const channel object, meh. */
+  ent->limit = channel_num_cells_writeable((channel_t *) ent->chan) *
+               (get_cell_network_size(ent->chan->wide_circ_ids) +
+                TLS_PER_CELL_OVERHEAD);
+}
+
+/* Given a socket that isn't in the table, add it.
+ * Given a socket that is in the table, re-init values that need init-ing
+ * every scheduling run
+ */
+static void
+init_socket_info(socket_table_t *table, const channel_t *chan)
+{
+  socket_table_ent_t *ent = NULL;
+  ent = socket_table_search(table, chan);
+  if (!ent) {
+    log_debug(LD_SCHED, "scheduler init socket info for chan=%" PRIu64,
+              chan->global_identifier);
+    ent = tor_malloc_zero(sizeof(*ent));
+    ent->chan = chan;
+    HT_INSERT(socket_table_s, table, ent);
+  }
+  ent->written = 0;
+}
+
+/* Add chan to the outbuf table if it isn't already in it. If it is, then don't
+ * do anything */
+static void
+outbuf_table_add(outbuf_table_t *table, channel_t *chan)
+{
+  outbuf_table_ent_t search, *ent;
+  search.chan = chan;
+  ent = HT_FIND(outbuf_table_s, table, &search);
+  if (!ent) {
+    log_debug(LD_SCHED, "scheduler init outbuf info for chan=%" PRIu64,
+              chan->global_identifier);
+    ent = tor_malloc_zero(sizeof(*ent));
+    ent->chan = chan;
+    HT_INSERT(outbuf_table_s, table, ent);
+  }
+}
+
+static void
+outbuf_table_remove(outbuf_table_t *table, channel_t *chan)
+{
+  outbuf_table_ent_t search, *ent;
+  search.chan = chan;
+  ent = HT_FIND(outbuf_table_s, table, &search);
+  if (ent) {
+    HT_REMOVE(outbuf_table_s, table, ent);
+    free_outbuf_info_by_ent(ent, NULL);
+  }
+}
+
+/* Set the scheduler running interval. */
+static void
+set_scheduler_run_interval(void)
+{
+  int old_sched_run_interval = sched_run_interval;
+  sched_run_interval = kist_scheduler_run_interval();
+  if (old_sched_run_interval != sched_run_interval) {
+    log_info(LD_SCHED, "Scheduler KIST changing its running interval "
+                       "from %" PRId32 " to %" PRId32,
+             old_sched_run_interval, sched_run_interval);
+  }
+}
+
+/* Return true iff the channel hasn't hit its kist-imposed write limit yet */
+static int
+socket_can_write(socket_table_t *table, const channel_t *chan)
+{
+  socket_table_ent_t *ent = NULL;
+  ent = socket_table_search(table, chan);
+  if (SCHED_BUG(!ent, chan)) {
+    return 1; // Just return true, saying that kist wouldn't limit the socket
+  }
+
+  /* We previously calculated a write limit for this socket. In the below
+   * calculation, first determine how much room is left in bytes. Then divide
+   * that by the amount of space a cell takes. If there's room for at least 1
+   * cell, then KIST will allow the socket to write. */
+  int64_t kist_limit_space =
+    (int64_t) (ent->limit - ent->written) /
+    (CELL_MAX_NETWORK_SIZE + TLS_PER_CELL_OVERHEAD);
+  return kist_limit_space > 0;
+}
+
+/* Update the channel's socket kernel information. */
+static void
+update_socket_info(socket_table_t *table, const channel_t *chan)
+{
+  socket_table_ent_t *ent = NULL;
+  ent = socket_table_search(table, chan);
+  if (SCHED_BUG(!ent, chan)) {
+    return; // Whelp. Entry didn't exist for some reason so nothing to do.
+  }
+  update_socket_info_impl(ent);
+  log_debug(LD_SCHED, "chan=%" PRIu64 " updated socket info, limit: %" PRIu64
+                      ", cwnd: %" PRIu32 ", unacked: %" PRIu32
+                      ", notsent: %" PRIu32 ", mss: %" PRIu32,
+            ent->chan->global_identifier, ent->limit, ent->cwnd, ent->unacked,
+            ent->notsent, ent->mss);
+}
+
+/* Increment the channel's socket written value by the number of bytes. */
+static void
+update_socket_written(socket_table_t *table, channel_t *chan, size_t bytes)
+{
+  socket_table_ent_t *ent = NULL;
+  ent = socket_table_search(table, chan);
+  if (SCHED_BUG(!ent, chan)) {
+    return; // Whelp. Entry didn't exist so nothing to do.
+  }
+
+  log_debug(LD_SCHED, "chan=%" PRIu64 " wrote %lu bytes, old was %" PRIi64,
+            chan->global_identifier, (unsigned long) bytes, ent->written);
+
+  ent->written += bytes;
+}
+
+/*
+ * A naive KIST impl would write every single cell all the way to the kernel.
+ * That would take a lot of system calls. A less bad KIST impl would write a
+ * channel's outbuf to the kernel only when we are switching to a different
+ * channel. But if we have two channels with equal priority, we end up writing
+ * one cell for each and bouncing back and forth. This KIST impl avoids that
+ * by only writing a channel's outbuf to the kernel if it has 8 cells or more
+ * in it.
+ */
+MOCK_IMPL(int, channel_should_write_to_kernel,
+          (outbuf_table_t *table, channel_t *chan))
+{
+  outbuf_table_add(table, chan);
+  /* CELL_MAX_NETWORK_SIZE * 8 because we only want to write the outbuf to the
+   * kernel if there's 8 or more cells waiting */
+  return channel_outbuf_length(chan) > (CELL_MAX_NETWORK_SIZE * 8);
+}
+
+/* Little helper function to write a channel's outbuf all the way to the
+ * kernel */
+MOCK_IMPL(void, channel_write_to_kernel, (channel_t *chan))
+{
+  tor_assert(chan);
+  log_debug(LD_SCHED, "Writing %lu bytes to kernel for chan %" PRIu64,
+            (unsigned long)channel_outbuf_length(chan),
+            chan->global_identifier);
+  connection_handle_write(TO_CONN(BASE_CHAN_TO_TLS(chan)->conn), 0);
+}
+
+/* Return true iff the scheduler has work to perform. */
+static int
+have_work(void)
+{
+  smartlist_t *cp = get_channels_pending();
+  IF_BUG_ONCE(!cp) {
+    return 0; // channels_pending doesn't exist so... no work?
+  }
+  return smartlist_len(cp) > 0;
+}
+
+/* Function of the scheduler interface: free_all() */
+static void
+kist_free_all(void)
+{
+  free_all_socket_info();
+}
+
+/* Function of the scheduler interface: on_channel_free() */
+static void
+kist_on_channel_free_fn(const channel_t *chan)
+{
+  free_socket_info_by_chan(&socket_table, chan);
+}
+
+/* Function of the scheduler interface: on_new_consensus() */
+static void
+kist_scheduler_on_new_consensus(void)
+{
+  set_scheduler_run_interval();
+}
+
+/* Function of the scheduler interface: on_new_options() */
+static void
+kist_scheduler_on_new_options(void)
+{
+  sock_buf_size_factor = get_options()->KISTSockBufSizeFactor;
+
+  /* Calls kist_scheduler_run_interval which calls get_options(). */
+  set_scheduler_run_interval();
+}
+
+/* Function of the scheduler interface: init() */
+static void
+kist_scheduler_init(void)
+{
+  /* When initializing the scheduler, the last run could be 0 because it is
+   * declared static or a value in the past that was set when it was last
+   * used. In both cases, we want to initialize it to now so we don't risk
+   * using the value 0 which doesn't play well with our monotonic time
+   * interface.
+   *
+   * One side effect is that the first scheduler run will be at the next tick
+   * that is in now + 10 msec (KIST_SCHED_RUN_INTERVAL_DEFAULT) by default. */
+  monotime_get(&scheduler_last_run);
+
+  kist_scheduler_on_new_options();
+  IF_BUG_ONCE(sched_run_interval == 0) {
+    log_warn(LD_SCHED, "We are initing the KIST scheduler and noticed the "
+             "KISTSchedRunInterval is telling us to not use KIST. That's "
+             "weird! We'll continue using KIST, but at %" PRId32 "ms.",
+             KIST_SCHED_RUN_INTERVAL_DEFAULT);
+    sched_run_interval = KIST_SCHED_RUN_INTERVAL_DEFAULT;
+  }
+}
+
+/* Function of the scheduler interface: schedule() */
+static void
+kist_scheduler_schedule(void)
+{
+  struct monotime_t now;
+  struct timeval next_run;
+  int64_t diff;
+
+  if (!have_work()) {
+    return;
+  }
+  monotime_get(&now);
+
+  /* If time is really monotonic, we can never have now being smaller than the
+   * last scheduler run. The scheduler_last_run at first is set to 0.
+   * Unfortunately, not all platforms guarantee monotonic time so we log at
+   * info level but don't make it more noisy. */
+  diff = monotime_diff_msec(&scheduler_last_run, &now);
+  if (diff < 0) {
+    log_info(LD_SCHED, "Monotonic time between now and last run of scheduler "
+                       "is negative: %" PRId64 ". Setting diff to 0.", diff);
+    diff = 0;
+  }
+  if (diff < sched_run_interval) {
+    next_run.tv_sec = 0;
+    /* Takes 1000 ms -> us. This will always be valid because diff can NOT be
+     * negative and can NOT be bigger than sched_run_interval so values can
+     * only go from 1000 usec (diff set to interval - 1) to 100000 usec (diff
+     * set to 0) for the maximum allowed run interval (100ms). */
+    next_run.tv_usec = (int) ((sched_run_interval - diff) * 1000);
+    /* Re-adding an event reschedules it. It does not duplicate it. */
+    scheduler_ev_add(&next_run);
+  } else {
+    scheduler_ev_active();
+  }
+}
+
+/* Function of the scheduler interface: run() */
+static void
+kist_scheduler_run(void)
+{
+  /* Define variables */
+  channel_t *chan = NULL; // current working channel
+  /* The last distinct chan served in a sched loop. */
+  channel_t *prev_chan = NULL;
+  int flush_result; // temporarily store results from flush calls
+  /* Channels to be re-adding to pending at the end */
+  smartlist_t *to_readd = NULL;
+  smartlist_t *cp = get_channels_pending();
+
+  outbuf_table_t outbuf_table = HT_INITIALIZER();
+
+  /* For each pending channel, collect new kernel information */
+  SMARTLIST_FOREACH_BEGIN(cp, const channel_t *, pchan) {
+      init_socket_info(&socket_table, pchan);
+      update_socket_info(&socket_table, pchan);
+  } SMARTLIST_FOREACH_END(pchan);
+
+  log_debug(LD_SCHED, "Running the scheduler. %d channels pending",
+            smartlist_len(cp));
+
+  /* The main scheduling loop. Loop until there are no more pending channels */
+  while (smartlist_len(cp) > 0) {
+    /* get best channel */
+    chan = smartlist_pqueue_pop(cp, scheduler_compare_channels,
+                                offsetof(channel_t, sched_heap_idx));
+    if (SCHED_BUG(!chan, NULL)) {
+      /* Some-freaking-how a NULL got into the channels_pending. That should
+       * never happen, but it should be harmless to ignore it and keep looping.
+       */
+      continue;
+    }
+    outbuf_table_add(&outbuf_table, chan);
+
+    /* if we have switched to a new channel, consider writing the previous
+     * channel's outbuf to the kernel. */
+    if (!prev_chan) {
+      prev_chan = chan;
+    }
+    if (prev_chan != chan) {
+      if (channel_should_write_to_kernel(&outbuf_table, prev_chan)) {
+        channel_write_to_kernel(prev_chan);
+        outbuf_table_remove(&outbuf_table, prev_chan);
+      }
+      prev_chan = chan;
+    }
+
+    /* Only flush and write if the per-socket limit hasn't been hit */
+    if (socket_can_write(&socket_table, chan)) {
+      /* flush to channel queue/outbuf */
+      flush_result = (int)channel_flush_some_cells(chan, 1); // 1 for num cells
+      /* XXX: While flushing cells, it is possible that the connection write
+       * fails leading to the channel to be closed which triggers a release
+       * and free its entry in the socket table. And because of a engineering
+       * design issue, the error is not propagated back so we don't get an
+       * error at this point. So before we continue, make sure the channel is
+       * open and if not just ignore it. See #23751. */
+      if (!CHANNEL_IS_OPEN(chan)) {
+        /* Channel isn't open so we put it back in IDLE mode. It is either
+         * renegotiating its TLS session or about to be released. */
+        scheduler_set_channel_state(chan, SCHED_CHAN_IDLE);
+        continue;
+      }
+      /* flush_result has the # cells flushed */
+      if (flush_result > 0) {
+        update_socket_written(&socket_table, chan, flush_result *
+                              (CELL_MAX_NETWORK_SIZE + TLS_PER_CELL_OVERHEAD));
+      } else {
+        /* XXX: This can happen because tor sometimes does flush in an
+         * opportunistic way cells from the circuit to the outbuf so the
+         * channel can end up here without having anything to flush nor needed
+         * to write to the kernel. Hopefully we'll fix that soon but for now
+         * we have to handle this case which happens kind of often. */
+        log_debug(LD_SCHED,
+                 "We didn't flush anything on a chan that we think "
+                 "can write and wants to write. The channel's state is '%s' "
+                 "and in scheduler state '%s'. We're going to mark it as "
+                 "waiting_for_cells (as that's most likely the issue) and "
+                 "stop scheduling it this round.",
+                 channel_state_to_string(chan->state),
+                 get_scheduler_state_string(chan->scheduler_state));
+        scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_FOR_CELLS);
+        continue;
+      }
+    }
+
+    /* Decide what to do with the channel now */
+
+    if (!channel_more_to_flush(chan) &&
+        !socket_can_write(&socket_table, chan)) {
+
+      /* Case 1: no more cells to send, and cannot write */
+
+      /*
+       * You might think we should put the channel in SCHED_CHAN_IDLE. And
+       * you're probably correct. While implementing KIST, we found that the
+       * scheduling system would sometimes lose track of channels when we did
+       * that. We suspect it has to do with the difference between "can't
+       * write because socket/outbuf is full" and KIST's "can't write because
+       * we've arbitrarily decided that that's enough for now." Sometimes
+       * channels run out of cells at the same time they hit their
+       * kist-imposed write limit and maybe the rest of Tor doesn't put the
+       * channel back in pending when it is supposed to.
+       *
+       * This should be investigated again. It is as simple as changing
+       * SCHED_CHAN_WAITING_FOR_CELLS to SCHED_CHAN_IDLE and seeing if Tor
+       * starts having serious throughput issues. Best done in shadow/chutney.
+       */
+      scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_FOR_CELLS);
+    } else if (!channel_more_to_flush(chan)) {
+
+      /* Case 2: no more cells to send, but still open for writes */
+
+      scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_FOR_CELLS);
+    } else if (!socket_can_write(&socket_table, chan)) {
+
+      /* Case 3: cells to send, but cannot write */
+
+      /*
+       * We want to write, but can't. If we left the channel in
+       * channels_pending, we would never exit the scheduling loop. We need to
+       * add it to a temporary list of channels to be added to channels_pending
+       * after the scheduling loop is over. They can hopefully be taken care of
+       * in the next scheduling round.
+       */
+      if (!to_readd) {
+        to_readd = smartlist_new();
+      }
+      smartlist_add(to_readd, chan);
+    } else {
+
+      /* Case 4: cells to send, and still open for writes */
+
+      scheduler_set_channel_state(chan, SCHED_CHAN_PENDING);
+      if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) {
+        smartlist_pqueue_add(cp, scheduler_compare_channels,
+                             offsetof(channel_t, sched_heap_idx), chan);
+      }
+    }
+  } /* End of main scheduling loop */
+
+  /* Write the outbuf of any channels that still have data */
+  HT_FOREACH_FN(outbuf_table_s, &outbuf_table, each_channel_write_to_kernel,
+                NULL);
+  /* We are done with it. */
+  HT_FOREACH_FN(outbuf_table_s, &outbuf_table, free_outbuf_info_by_ent, NULL);
+  HT_CLEAR(outbuf_table_s, &outbuf_table);
+
+  log_debug(LD_SCHED, "len pending=%d, len to_readd=%d",
+            smartlist_len(cp),
+            (to_readd ? smartlist_len(to_readd) : -1));
+
+  /* Re-add any channels we need to */
+  if (to_readd) {
+    SMARTLIST_FOREACH_BEGIN(to_readd, channel_t *, readd_chan) {
+      scheduler_set_channel_state(readd_chan, SCHED_CHAN_PENDING);
+      if (!smartlist_contains(cp, readd_chan)) {
+        if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) {
+          /* XXXX Note that the check above is in theory redundant with
+           * the smartlist_contains check.  But let's make sure we're
+           * not messing anything up, and leave them both for now. */
+          smartlist_pqueue_add(cp, scheduler_compare_channels,
+                             offsetof(channel_t, sched_heap_idx), readd_chan);
+        }
+      }
+    } SMARTLIST_FOREACH_END(readd_chan);
+    smartlist_free(to_readd);
+  }
+
+  monotime_get(&scheduler_last_run);
+}
+
+/*****************************************************************************
+ * Externally called function implementations not called through scheduler_t
+ *****************************************************************************/
+
+/* Stores the kist scheduler function pointers. */
+static scheduler_t kist_scheduler = {
+  .type = SCHEDULER_KIST,
+  .free_all = kist_free_all,
+  .on_channel_free = kist_on_channel_free_fn,
+  .init = kist_scheduler_init,
+  .on_new_consensus = kist_scheduler_on_new_consensus,
+  .schedule = kist_scheduler_schedule,
+  .run = kist_scheduler_run,
+  .on_new_options = kist_scheduler_on_new_options,
+};
+
+/* Return the KIST scheduler object. If it didn't exists, return a newly
+ * allocated one but init() is not called. */
+scheduler_t *
+get_kist_scheduler(void)
+{
+  return &kist_scheduler;
+}
+
+/* Check the torrc (and maybe consensus) for the configured KIST scheduler run
+ * interval.
+ * - If torrc > 0, then return the positive torrc value (should use KIST, and
+ *   should use the set value)
+ * - If torrc == 0, then look in the consensus for what the value should be.
+ *   - If == 0, then return 0 (don't use KIST)
+ *   - If > 0, then return the positive consensus value
+ *   - If consensus doesn't say anything, return 10 milliseconds, default.
+ */
+int
+kist_scheduler_run_interval(void)
+{
+  int run_interval = get_options()->KISTSchedRunInterval;
+
+  if (run_interval != 0) {
+    log_debug(LD_SCHED, "Found KISTSchedRunInterval=%" PRId32 " in torrc. "
+                        "Using that.", run_interval);
+    return run_interval;
+  }
+
+  log_debug(LD_SCHED, "KISTSchedRunInterval=0, turning to the consensus.");
+
+  /* Will either be the consensus value or the default. Note that 0 can be
+   * returned which means the consensus wants us to NOT use KIST. */
+  return networkstatus_get_param(NULL, "KISTSchedRunInterval",
+                                 KIST_SCHED_RUN_INTERVAL_DEFAULT,
+                                 KIST_SCHED_RUN_INTERVAL_MIN,
+                                 KIST_SCHED_RUN_INTERVAL_MAX);
+}
+
+/* Set KISTLite mode that is KIST without kernel support. */
+void
+scheduler_kist_set_lite_mode(void)
+{
+  kist_lite_mode = 1;
+  kist_scheduler.type = SCHEDULER_KIST_LITE;
+  log_info(LD_SCHED,
+           "Setting KIST scheduler without kernel support (KISTLite mode)");
+}
+
+/* Set KIST mode that is KIST with kernel support. */
+void
+scheduler_kist_set_full_mode(void)
+{
+  kist_lite_mode = 0;
+  kist_scheduler.type = SCHEDULER_KIST;
+  log_info(LD_SCHED,
+           "Setting KIST scheduler with kernel support (KIST mode)");
+}
+
+#ifdef HAVE_KIST_SUPPORT
+
+/* Return true iff the scheduler subsystem should use KIST. */
+int
+scheduler_can_use_kist(void)
+{
+  if (kist_no_kernel_support) {
+    /* We have no kernel support so we can't use KIST. */
+    return 0;
+  }
+
+  /* We do have the support, time to check if we can get the interval that the
+   * consensus can be disabling. */
+  int run_interval = kist_scheduler_run_interval();
+  log_debug(LD_SCHED, "Determined KIST sched_run_interval should be "
+                      "%" PRId32 ". Can%s use KIST.",
+           run_interval, (run_interval > 0 ? "" : " not"));
+  return run_interval > 0;
+}
+
+#else /* !(defined(HAVE_KIST_SUPPORT)) */
+
+int
+scheduler_can_use_kist(void)
+{
+  return 0;
+}
+
+#endif /* defined(HAVE_KIST_SUPPORT) */
diff --git a/src/core/or/scheduler_vanilla.c b/src/core/or/scheduler_vanilla.c
new file mode 100644
index 0000000000..33536ae04b
--- /dev/null
+++ b/src/core/or/scheduler_vanilla.c
@@ -0,0 +1,175 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#define TOR_CHANNEL_INTERNAL_
+#include "core/or/channel.h"
+#define SCHEDULER_PRIVATE_
+#include "core/or/scheduler.h"
+
+/*****************************************************************************
+ * Other internal data
+ *****************************************************************************/
+
+/* Maximum cells to flush in a single call to channel_flush_some_cells(); */
+#define MAX_FLUSH_CELLS 1000
+
+/*****************************************************************************
+ * Externally called function implementations
+ *****************************************************************************/
+
+/* Return true iff the scheduler has work to perform. */
+static int
+have_work(void)
+{
+  smartlist_t *cp = get_channels_pending();
+  IF_BUG_ONCE(!cp) {
+    return 0; // channels_pending doesn't exist so... no work?
+  }
+  return smartlist_len(cp) > 0;
+}
+
+/** Re-trigger the scheduler in a way safe to use from the callback */
+
+static void
+vanilla_scheduler_schedule(void)
+{
+  if (!have_work()) {
+    return;
+  }
+
+  /* Activate our event so it can process channels. */
+  scheduler_ev_active();
+}
+
+static void
+vanilla_scheduler_run(void)
+{
+  int n_cells, n_chans_before, n_chans_after;
+  ssize_t flushed, flushed_this_time;
+  smartlist_t *cp = get_channels_pending();
+  smartlist_t *to_readd = NULL;
+  channel_t *chan = NULL;
+
+  log_debug(LD_SCHED, "We have a chance to run the scheduler");
+
+  n_chans_before = smartlist_len(cp);
+
+  while (smartlist_len(cp) > 0) {
+    /* Pop off a channel */
+    chan = smartlist_pqueue_pop(cp,
+                                scheduler_compare_channels,
+                                offsetof(channel_t, sched_heap_idx));
+    IF_BUG_ONCE(!chan) {
+      /* Some-freaking-how a NULL got into the channels_pending. That should
+       * never happen, but it should be harmless to ignore it and keep looping.
+       */
+      continue;
+    }
+
+    /* Figure out how many cells we can write */
+    n_cells = channel_num_cells_writeable(chan);
+    if (n_cells > 0) {
+      log_debug(LD_SCHED,
+                "Scheduler saw pending channel %"PRIu64 " at %p with "
+                "%d cells writeable",
+                (chan->global_identifier), chan, n_cells);
+
+      flushed = 0;
+      while (flushed < n_cells) {
+        flushed_this_time =
+          channel_flush_some_cells(chan,
+                        MIN(MAX_FLUSH_CELLS, (size_t) n_cells - flushed));
+        if (flushed_this_time <= 0) break;
+        flushed += flushed_this_time;
+      }
+
+      if (flushed < n_cells) {
+        /* We ran out of cells to flush */
+        scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_FOR_CELLS);
+      } else {
+        /* The channel may still have some cells */
+        if (channel_more_to_flush(chan)) {
+        /* The channel goes to either pending or waiting_to_write */
+          if (channel_num_cells_writeable(chan) > 0) {
+            /* Add it back to pending later */
+            if (!to_readd) to_readd = smartlist_new();
+            smartlist_add(to_readd, chan);
+            log_debug(LD_SCHED,
+                      "Channel %"PRIu64 " at %p "
+                      "is still pending",
+                      (chan->global_identifier),
+                      chan);
+          } else {
+            /* It's waiting to be able to write more */
+            scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_TO_WRITE);
+          }
+        } else {
+          /* No cells left; it can go to idle or waiting_for_cells */
+          if (channel_num_cells_writeable(chan) > 0) {
+            /*
+             * It can still accept writes, so it goes to
+             * waiting_for_cells
+             */
+            scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_FOR_CELLS);
+          } else {
+            /*
+             * We exactly filled up the output queue with all available
+             * cells; go to idle.
+             */
+            scheduler_set_channel_state(chan, SCHED_CHAN_IDLE);
+          }
+        }
+      }
+
+      log_debug(LD_SCHED,
+                "Scheduler flushed %d cells onto pending channel "
+                "%"PRIu64 " at %p",
+                (int)flushed, (chan->global_identifier),
+                chan);
+    } else {
+      log_info(LD_SCHED,
+               "Scheduler saw pending channel %"PRIu64 " at %p with "
+               "no cells writeable",
+               (chan->global_identifier), chan);
+      /* Put it back to WAITING_TO_WRITE */
+      scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_TO_WRITE);
+    }
+  }
+
+  /* Readd any channels we need to */
+  if (to_readd) {
+    SMARTLIST_FOREACH_BEGIN(to_readd, channel_t *, readd_chan) {
+      scheduler_set_channel_state(readd_chan, SCHED_CHAN_PENDING);
+      smartlist_pqueue_add(cp,
+                           scheduler_compare_channels,
+                           offsetof(channel_t, sched_heap_idx),
+                           readd_chan);
+    } SMARTLIST_FOREACH_END(readd_chan);
+    smartlist_free(to_readd);
+  }
+
+  n_chans_after = smartlist_len(cp);
+  log_debug(LD_SCHED, "Scheduler handled %d of %d pending channels",
+            n_chans_before - n_chans_after, n_chans_before);
+}
+
+/* Stores the vanilla scheduler function pointers. */
+static scheduler_t vanilla_scheduler = {
+  .type = SCHEDULER_VANILLA,
+  .free_all = NULL,
+  .on_channel_free = NULL,
+  .init = NULL,
+  .on_new_consensus = NULL,
+  .schedule = vanilla_scheduler_schedule,
+  .run = vanilla_scheduler_run,
+  .on_new_options = NULL,
+};
+
+scheduler_t *
+get_vanilla_scheduler(void)
+{
+  return &vanilla_scheduler;
+}
+
diff --git a/src/core/or/server_port_cfg_st.h b/src/core/or/server_port_cfg_st.h
new file mode 100644
index 0000000000..bd026af7ee
--- /dev/null
+++ b/src/core/or/server_port_cfg_st.h
@@ -0,0 +1,20 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef SERVER_PORT_CFG_ST_H
+#define SERVER_PORT_CFG_ST_H
+
+struct server_port_cfg_t {
+  /* Server port types (or, dir) only: */
+  unsigned int no_advertise : 1;
+  unsigned int no_listen : 1;
+  unsigned int all_addrs : 1;
+  unsigned int bind_ipv4_only : 1;
+  unsigned int bind_ipv6_only : 1;
+};
+
+#endif
+
diff --git a/src/core/or/socks_request_st.h b/src/core/or/socks_request_st.h
new file mode 100644
index 0000000000..5922870c61
--- /dev/null
+++ b/src/core/or/socks_request_st.h
@@ -0,0 +1,77 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef SOCKS_REQUEST_ST_H
+#define SOCKS_REQUEST_ST_H
+
+#define MAX_SOCKS_REPLY_LEN 1024
+
+#define SOCKS_NO_AUTH 0x00
+#define SOCKS_USER_PASS 0x02
+
+/** Please open a TCP connection to this addr:port. */
+#define SOCKS_COMMAND_CONNECT       0x01
+/** Please turn this FQDN into an IP address, privately. */
+#define SOCKS_COMMAND_RESOLVE       0xF0
+/** Please turn this IP address into an FQDN, privately. */
+#define SOCKS_COMMAND_RESOLVE_PTR   0xF1
+
+/* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */
+#define SOCKS_COMMAND_IS_CONNECT(c) (((c)==SOCKS_COMMAND_CONNECT) || 0)
+#define SOCKS_COMMAND_IS_RESOLVE(c) ((c)==SOCKS_COMMAND_RESOLVE || \
+                                     (c)==SOCKS_COMMAND_RESOLVE_PTR)
+
+/** State of a SOCKS request from a user to an OP.  Also used to encode other
+ * information for non-socks user request (such as those on TransPort and
+ * DNSPort) */
+struct socks_request_t {
+  /** Which version of SOCKS did the client use? One of "0, 4, 5" -- where
+   * 0 means that no socks handshake ever took place, and this is just a
+   * stub connection (e.g. see connection_ap_make_link()). */
+  uint8_t socks_version;
+  /** If using socks5 authentication, which authentication type did we
+   * negotiate?  currently we support 0 (no authentication) and 2
+   * (username/password). */
+  uint8_t auth_type;
+  /** What is this stream's goal? One of the SOCKS_COMMAND_* values */
+  uint8_t command;
+  /** Which kind of listener created this stream? */
+  uint8_t listener_type;
+  size_t replylen; /**< Length of <b>reply</b>. */
+  uint8_t reply[MAX_SOCKS_REPLY_LEN]; /**< Write an entry into this string if
+                                    * we want to specify our own socks reply,
+                                    * rather than using the default socks4 or
+                                    * socks5 socks reply. We use this for the
+                                    * two-stage socks5 handshake.
+                                    */
+  char address[MAX_SOCKS_ADDR_LEN]; /**< What address did the client ask to
+                                       connect to/resolve? */
+  uint16_t port; /**< What port did the client ask to connect to? */
+  unsigned int has_finished : 1; /**< Has the SOCKS handshake finished? Used to
+                              * make sure we send back a socks reply for
+                              * every connection. */
+  unsigned int got_auth : 1; /**< Have we received any authentication data? */
+  /** If this is set, we will choose "no authentication" instead of
+   * "username/password" authentication if both are offered. Used as input to
+   * parse_socks. */
+  unsigned int socks_prefer_no_auth : 1;
+
+  /** Number of bytes in username; 0 if username is NULL */
+  size_t usernamelen;
+  /** Number of bytes in password; 0 if password is NULL */
+  uint8_t passwordlen;
+  /** The negotiated username value if any (for socks5), or the entire
+   * authentication string (for socks4).  This value is NOT nul-terminated;
+   * see usernamelen for its length. */
+  char *username;
+  /** The negotiated password value if any (for socks5). This value is NOT
+   * nul-terminated; see passwordlen for its length. */
+  char *password;
+
+  uint8_t socks5_atyp; /* SOCKS5 address type */
+};
+
+#endif
diff --git a/src/or/status.c b/src/core/or/status.c
index fa2238b9f9..46494ca76c 100644
--- a/src/or/status.c
+++ b/src/core/or/status.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,23 +14,29 @@
 
 #define STATUS_PRIVATE
 
-#include "or.h"
-#include "circuituse.h"
-#include "config.h"
-#include "status.h"
-#include "nodelist.h"
-#include "relay.h"
-#include "router.h"
-#include "circuitlist.h"
-#include "main.h"
-#include "rephist.h"
-#include "hibernate.h"
-#include "rephist.h"
-#include "statefile.h"
-#include "dos.h"
+#include "core/or/or.h"
+#include "core/or/circuituse.h"
+#include "app/config/config.h"
+#include "core/or/status.h"
+#include "feature/nodelist/nodelist.h"
+#include "core/or/relay.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+#include "core/or/circuitlist.h"
+#include "core/mainloop/mainloop.h"
+#include "feature/stats/rephist.h"
+#include "feature/hibernate/hibernate.h"
+#include "app/config/statefile.h"
+#include "feature/hs/hs_stats.h"
+#include "feature/hs/hs_service.h"
+#include "core/or/dos.h"
+#include "feature/stats/geoip_stats.h"
+
+#include "app/config/or_state_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "lib/tls/tortls.h"
 
 static void log_accounting(const time_t now, const or_options_t *options);
-#include "geoip.h"
 
 /** Return the total number of circuits. */
 STATIC int
@@ -40,7 +46,7 @@ count_circuits(void)
 }
 
 /** Take seconds <b>secs</b> and return a newly allocated human-readable
- * uptime string */
+ * uptime string. */
 STATIC char *
 secs_to_uptime(long secs)
 {
@@ -74,18 +80,38 @@ bytes_to_usage(uint64_t bytes)
   char *bw_string = NULL;
 
   if (bytes < (1<<20)) { /* Less than a megabyte. */
-    tor_asprintf(&bw_string, U64_FORMAT" kB", U64_PRINTF_ARG(bytes>>10));
+    tor_asprintf(&bw_string, "%"PRIu64" kB", (bytes>>10));
   } else if (bytes < (1<<30)) { /* Megabytes. Let's add some precision. */
-    double bw = U64_TO_DBL(bytes);
+    double bw = ((double)bytes);
     tor_asprintf(&bw_string, "%.2f MB", bw/(1<<20));
   } else { /* Gigabytes. */
-    double bw = U64_TO_DBL(bytes);
+    double bw = ((double)bytes);
     tor_asprintf(&bw_string, "%.2f GB", bw/(1<<30));
   }
 
   return bw_string;
 }
 
+/** Log some usage info about our onion service(s). */
+static void
+log_onion_service_stats(void)
+{
+  unsigned int num_services = hs_service_get_num_services();
+
+  /* If there are no active onion services, no need to print logs */
+  if (num_services == 0) {
+    return;
+  }
+
+  log_notice(LD_HEARTBEAT,
+             "Our onion service%s received %u v2 and %u v3 INTRODUCE2 cells "
+             "and attempted to launch %d rendezvous circuits.",
+             num_services == 1 ? "" : "s",
+             hs_stats_get_n_introduce2_v2_cells(),
+             hs_stats_get_n_introduce2_v3_cells(),
+             hs_stats_get_n_rendezvous_launches());
+}
+
 /** Log a "heartbeat" message describing Tor's status and history so that the
  * user can know that there is indeed a running Tor.  Return 0 on success and
  * -1 on failure. */
@@ -127,8 +153,8 @@ log_heartbeat(time_t now)
   double fullness_pct = 100;
   if (stats_n_data_cells_packaged && !hibernating) {
     fullness_pct =
-      100*(U64_TO_DBL(stats_n_data_bytes_packaged) /
-           U64_TO_DBL(stats_n_data_cells_packaged*RELAY_PAYLOAD_SIZE));
+      100*(((double)stats_n_data_bytes_packaged) /
+           ((double)stats_n_data_cells_packaged*RELAY_PAYLOAD_SIZE));
   }
   const double overhead_pct = ( r - 1.0 ) * 100.0;
 
@@ -159,6 +185,23 @@ log_heartbeat(time_t now)
     tor_free(msg);
   }
 
+  if (options->MainloopStats) {
+    const uint64_t main_loop_success_count = get_main_loop_success_count();
+    const uint64_t main_loop_error_count = get_main_loop_error_count();
+    const uint64_t main_loop_idle_count = get_main_loop_idle_count();
+
+    log_fn(LOG_NOTICE, LD_HEARTBEAT, "Main event loop statistics: "
+         "%"PRIu64 " successful returns, "
+         "%"PRIu64 " erroneous returns, and "
+         "%"PRIu64 " idle returns.",
+         (main_loop_success_count),
+         (main_loop_error_count),
+         (main_loop_idle_count));
+  }
+
+  /** Now, if we are an HS service, log some stats about our usage */
+  log_onion_service_stats();
+
   tor_free(uptime);
   tor_free(bw_sent);
   tor_free(bw_rcvd);
@@ -207,4 +250,3 @@ log_accounting(const time_t now, const or_options_t *options)
   tor_free(acc_max);
   tor_free(remaining);
 }
-
diff --git a/src/or/status.h b/src/core/or/status.h
index b97e835037..3467501ebb 100644
--- a/src/or/status.h
+++ b/src/core/or/status.h
@@ -1,10 +1,10 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_STATUS_H
 #define TOR_STATUS_H
 
-#include "testsupport.h"
+#include "lib/testsupport/testsupport.h"
 
 int log_heartbeat(time_t now);
 
@@ -14,5 +14,5 @@ STATIC char *secs_to_uptime(long secs);
 STATIC char *bytes_to_usage(uint64_t bytes);
 #endif
 
-#endif
+#endif /* !defined(TOR_STATUS_H) */
 
diff --git a/src/core/or/tor_version_st.h b/src/core/or/tor_version_st.h
new file mode 100644
index 0000000000..716429bd32
--- /dev/null
+++ b/src/core/or/tor_version_st.h
@@ -0,0 +1,32 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_VERSION_ST_H
+#define TOR_VERSION_ST_H
+
+#define MAX_STATUS_TAG_LEN 32
+/** Structure to hold parsed Tor versions.  This is a little messier
+ * than we would like it to be, because we changed version schemes with 0.1.0.
+ *
+ * See version-spec.txt for the whole business.
+ */
+struct tor_version_t {
+  int major;
+  int minor;
+  int micro;
+  /** Release status.  For version in the post-0.1 format, this is always
+   * VER_RELEASE. */
+  enum { VER_PRE=0, VER_RC=1, VER_RELEASE=2, } status;
+  int patchlevel;
+  char status_tag[MAX_STATUS_TAG_LEN];
+  int svn_revision;
+
+  int git_tag_len;
+  char git_tag[DIGEST_LEN];
+};
+
+#endif
+
diff --git a/src/core/or/var_cell_st.h b/src/core/or/var_cell_st.h
new file mode 100644
index 0000000000..4287c83f6d
--- /dev/null
+++ b/src/core/or/var_cell_st.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef VAR_CELL_ST_H
+#define VAR_CELL_ST_H
+
+/** Parsed variable-length onion routing cell. */
+struct var_cell_t {
+  /** Type of the cell: CELL_VERSIONS, etc. */
+  uint8_t command;
+  /** Circuit thich received the cell */
+  circid_t circ_id;
+  /** Number of bytes actually stored in <b>payload</b> */
+  uint16_t payload_len;
+  /** Payload of this cell */
+  uint8_t payload[FLEXIBLE_ARRAY_MEMBER];
+};
+
+#endif
+
diff --git a/src/core/or/versions.c b/src/core/or/versions.c
new file mode 100644
index 0000000000..33273a5294
--- /dev/null
+++ b/src/core/or/versions.c
@@ -0,0 +1,422 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file versions.c
+ * \brief Code to manipulate, parse, and compare Tor versions.
+ */
+#include "core/or/or.h"
+
+#include "core/or/protover.h"
+#include "core/or/versions.h"
+#include "lib/crypt_ops/crypto_util.h"
+
+#include "core/or/tor_version_st.h"
+
+/** Return VS_RECOMMENDED if <b>myversion</b> is contained in
+ * <b>versionlist</b>.  Else, return VS_EMPTY if versionlist has no
+ * entries. Else, return VS_OLD if every member of
+ * <b>versionlist</b> is newer than <b>myversion</b>.  Else, return
+ * VS_NEW_IN_SERIES if there is at least one member of <b>versionlist</b> in
+ * the same series (major.minor.micro) as <b>myversion</b>, but no such member
+ * is newer than <b>myversion.</b>.  Else, return VS_NEW if every member of
+ * <b>versionlist</b> is older than <b>myversion</b>.  Else, return
+ * VS_UNRECOMMENDED.
+ *
+ * (versionlist is a comma-separated list of version strings,
+ * optionally prefixed with "Tor".  Versions that can't be parsed are
+ * ignored.)
+ */
+version_status_t
+tor_version_is_obsolete(const char *myversion, const char *versionlist)
+{
+  tor_version_t mine, other;
+  int found_newer = 0, found_older = 0, found_newer_in_series = 0,
+    found_any_in_series = 0, r, same;
+  version_status_t ret = VS_UNRECOMMENDED;
+  smartlist_t *version_sl;
+
+  log_debug(LD_CONFIG,"Checking whether version '%s' is in '%s'",
+            myversion, versionlist);
+
+  if (tor_version_parse(myversion, &mine)) {
+    log_err(LD_BUG,"I couldn't parse my own version (%s)", myversion);
+    tor_assert(0);
+  }
+  version_sl = smartlist_new();
+  smartlist_split_string(version_sl, versionlist, ",", SPLIT_SKIP_SPACE, 0);
+
+  if (!strlen(versionlist)) { /* no authorities cared or agreed */
+    ret = VS_EMPTY;
+    goto done;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(version_sl, const char *, cp) {
+    if (!strcmpstart(cp, "Tor "))
+      cp += 4;
+
+    if (tor_version_parse(cp, &other)) {
+      /* Couldn't parse other; it can't be a match. */
+    } else {
+      same = tor_version_same_series(&mine, &other);
+      if (same)
+        found_any_in_series = 1;
+      r = tor_version_compare(&mine, &other);
+      if (r==0) {
+        ret = VS_RECOMMENDED;
+        goto done;
+      } else if (r<0) {
+        found_newer = 1;
+        if (same)
+          found_newer_in_series = 1;
+      } else if (r>0) {
+        found_older = 1;
+      }
+    }
+  } SMARTLIST_FOREACH_END(cp);
+
+  /* We didn't find the listed version. Is it new or old? */
+  if (found_any_in_series && !found_newer_in_series && found_newer) {
+    ret = VS_NEW_IN_SERIES;
+  } else if (found_newer && !found_older) {
+    ret = VS_OLD;
+  } else if (found_older && !found_newer) {
+    ret = VS_NEW;
+  } else {
+    ret = VS_UNRECOMMENDED;
+  }
+
+ done:
+  SMARTLIST_FOREACH(version_sl, char *, version, tor_free(version));
+  smartlist_free(version_sl);
+  return ret;
+}
+
+/** Extract a Tor version from a <b>platform</b> line from a router
+ * descriptor, and place the result in <b>router_version</b>.
+ *
+ * Return 1 on success, -1 on parsing failure, and 0 if the
+ * platform line does not indicate some version of Tor.
+ *
+ * If <b>strict</b> is non-zero, finding any weird version components
+ * (like negative numbers) counts as a parsing failure.
+ */
+int
+tor_version_parse_platform(const char *platform,
+                           tor_version_t *router_version,
+                           int strict)
+{
+  char tmp[128];
+  char *s, *s2, *start;
+
+  if (strcmpstart(platform,"Tor ")) /* nonstandard Tor; say 0. */
+    return 0;
+
+  start = (char *)eat_whitespace(platform+3);
+  if (!*start) return -1;
+  s = (char *)find_whitespace(start); /* also finds '\0', which is fine */
+  s2 = (char*)eat_whitespace(s);
+  if (!strcmpstart(s2, "(r") || !strcmpstart(s2, "(git-"))
+    s = (char*)find_whitespace(s2);
+
+  if ((size_t)(s-start+1) >= sizeof(tmp)) /* too big, no */
+    return -1;
+  strlcpy(tmp, start, s-start+1);
+
+  if (tor_version_parse(tmp, router_version)<0) {
+    log_info(LD_DIR,"Router version '%s' unparseable.",tmp);
+    return -1;
+  }
+
+  if (strict) {
+    if (router_version->major < 0 ||
+        router_version->minor < 0 ||
+        router_version->micro < 0 ||
+        router_version->patchlevel < 0 ||
+        router_version->svn_revision < 0) {
+      return -1;
+    }
+  }
+
+  return 1;
+}
+
+/** Parse the Tor version of the platform string <b>platform</b>,
+ * and compare it to the version in <b>cutoff</b>. Return 1 if
+ * the router is at least as new as the cutoff, else return 0.
+ */
+int
+tor_version_as_new_as(const char *platform, const char *cutoff)
+{
+  tor_version_t cutoff_version, router_version;
+  int r;
+  tor_assert(platform);
+
+  if (tor_version_parse(cutoff, &cutoff_version)<0) {
+    log_warn(LD_BUG,"cutoff version '%s' unparseable.",cutoff);
+    return 0;
+  }
+
+  r = tor_version_parse_platform(platform, &router_version, 0);
+  if (r == 0) {
+    /* nonstandard Tor; be safe and say yes */
+    return 1;
+  } else if (r < 0) {
+    /* unparseable version; be safe and say yes. */
+    return 1;
+  }
+
+  /* Here's why we don't need to do any special handling for svn revisions:
+   * - If neither has an svn revision, we're fine.
+   * - If the router doesn't have an svn revision, we can't assume that it
+   *   is "at least" any svn revision, so we need to return 0.
+   * - If the target version doesn't have an svn revision, any svn revision
+   *   (or none at all) is good enough, so return 1.
+   * - If both target and router have an svn revision, we compare them.
+   */
+
+  return tor_version_compare(&router_version, &cutoff_version) >= 0;
+}
+
+/** Parse a tor version from <b>s</b>, and store the result in <b>out</b>.
+ * Return 0 on success, -1 on failure. */
+int
+tor_version_parse(const char *s, tor_version_t *out)
+{
+  char *eos=NULL;
+  const char *cp=NULL;
+  int ok = 1;
+  /* Format is:
+   *   "Tor " ? NUM dot NUM [ dot NUM [ ( pre | rc | dot ) NUM ] ] [ - tag ]
+   */
+  tor_assert(s);
+  tor_assert(out);
+
+  memset(out, 0, sizeof(tor_version_t));
+  out->status = VER_RELEASE;
+  if (!strcasecmpstart(s, "Tor "))
+    s += 4;
+
+  cp = s;
+
+#define NUMBER(m)                               \
+  do {                                          \
+    if (!cp || *cp < '0' || *cp > '9')          \
+      return -1;                                \
+    out->m = (int)tor_parse_uint64(cp, 10, 0, INT32_MAX, &ok, &eos);    \
+    if (!ok)                                    \
+      return -1;                                \
+    if (!eos || eos == cp)                      \
+      return -1;                                \
+    cp = eos;                                   \
+  } while (0)
+
+#define DOT()                                   \
+  do {                                          \
+    if (*cp != '.')                             \
+      return -1;                                \
+    ++cp;                                       \
+  } while (0)
+
+  NUMBER(major);
+  DOT();
+  NUMBER(minor);
+  if (*cp == 0)
+    return 0;
+  else if (*cp == '-')
+    goto status_tag;
+  DOT();
+  NUMBER(micro);
+
+  /* Get status */
+  if (*cp == 0) {
+    return 0;
+  } else if (*cp == '.') {
+    ++cp;
+  } else if (*cp == '-') {
+    goto status_tag;
+  } else if (0==strncmp(cp, "pre", 3)) {
+    out->status = VER_PRE;
+    cp += 3;
+  } else if (0==strncmp(cp, "rc", 2)) {
+    out->status = VER_RC;
+    cp += 2;
+  } else {
+    return -1;
+  }
+
+  NUMBER(patchlevel);
+
+ status_tag:
+  /* Get status tag. */
+  if (*cp == '-' || *cp == '.')
+    ++cp;
+  eos = (char*) find_whitespace(cp);
+  if (eos-cp >= (int)sizeof(out->status_tag))
+    strlcpy(out->status_tag, cp, sizeof(out->status_tag));
+  else {
+    memcpy(out->status_tag, cp, eos-cp);
+    out->status_tag[eos-cp] = 0;
+  }
+  cp = eat_whitespace(eos);
+
+  if (!strcmpstart(cp, "(r")) {
+    cp += 2;
+    out->svn_revision = (int) strtol(cp,&eos,10);
+  } else if (!strcmpstart(cp, "(git-")) {
+    char *close_paren = strchr(cp, ')');
+    int hexlen;
+    char digest[DIGEST_LEN];
+    if (! close_paren)
+      return -1;
+    cp += 5;
+    if (close_paren-cp > HEX_DIGEST_LEN)
+      return -1;
+    hexlen = (int)(close_paren-cp);
+    memwipe(digest, 0, sizeof(digest));
+    if ( hexlen == 0 || (hexlen % 2) == 1)
+      return -1;
+    if (base16_decode(digest, hexlen/2, cp, hexlen) != hexlen/2)
+      return -1;
+    memcpy(out->git_tag, digest, hexlen/2);
+    out->git_tag_len = hexlen/2;
+  }
+
+  return 0;
+#undef NUMBER
+#undef DOT
+}
+
+/** Compare two tor versions; Return <0 if a < b; 0 if a ==b, >0 if a >
+ * b. */
+int
+tor_version_compare(tor_version_t *a, tor_version_t *b)
+{
+  int i;
+  tor_assert(a);
+  tor_assert(b);
+
+  /* We take this approach to comparison to ensure the same (bogus!) behavior
+   * on all inputs as we would have seen before bug #21278 was fixed. The
+   * only important difference here is that this method doesn't cause
+   * a signed integer underflow.
+   */
+#define CMP(field) do {                               \
+    unsigned aval = (unsigned) a->field;              \
+    unsigned bval = (unsigned) b->field;              \
+    int result = (int) (aval - bval);                 \
+    if (result < 0)                                   \
+      return -1;                                      \
+    else if (result > 0)                              \
+      return 1;                                       \
+  } while (0)
+
+  CMP(major);
+  CMP(minor);
+  CMP(micro);
+  CMP(status);
+  CMP(patchlevel);
+  if ((i = strcmp(a->status_tag, b->status_tag)))
+     return i;
+  CMP(svn_revision);
+  CMP(git_tag_len);
+  if (a->git_tag_len)
+     return fast_memcmp(a->git_tag, b->git_tag, a->git_tag_len);
+  else
+     return 0;
+
+#undef CMP
+}
+
+/** Return true iff versions <b>a</b> and <b>b</b> belong to the same series.
+ */
+int
+tor_version_same_series(tor_version_t *a, tor_version_t *b)
+{
+  tor_assert(a);
+  tor_assert(b);
+  return ((a->major == b->major) &&
+          (a->minor == b->minor) &&
+          (a->micro == b->micro));
+}
+
+/** Helper: Given pointers to two strings describing tor versions, return -1
+ * if _a precedes _b, 1 if _b precedes _a, and 0 if they are equivalent.
+ * Used to sort a list of versions. */
+static int
+compare_tor_version_str_ptr_(const void **_a, const void **_b)
+{
+  const char *a = *_a, *b = *_b;
+  int ca, cb;
+  tor_version_t va, vb;
+  ca = tor_version_parse(a, &va);
+  cb = tor_version_parse(b, &vb);
+  /* If they both parse, compare them. */
+  if (!ca && !cb)
+    return tor_version_compare(&va,&vb);
+  /* If one parses, it comes first. */
+  if (!ca && cb)
+    return -1;
+  if (ca && !cb)
+    return 1;
+  /* If neither parses, compare strings.  Also, the directory server admin
+  ** needs to be smacked upside the head.  But Tor is tolerant and gentle. */
+  return strcmp(a,b);
+}
+
+/** Sort a list of string-representations of versions in ascending order. */
+void
+sort_version_list(smartlist_t *versions, int remove_duplicates)
+{
+  smartlist_sort(versions, compare_tor_version_str_ptr_);
+
+  if (remove_duplicates)
+    smartlist_uniq(versions, compare_tor_version_str_ptr_, tor_free_);
+}
+
+/** Summarize the protocols listed in <b>protocols</b> into <b>out</b>,
+ * falling back or correcting them based on <b>version</b> as appropriate.
+ */
+void
+summarize_protover_flags(protover_summary_flags_t *out,
+                         const char *protocols,
+                         const char *version)
+{
+  tor_assert(out);
+  memset(out, 0, sizeof(*out));
+  if (protocols) {
+    out->protocols_known = 1;
+    out->supports_extend2_cells =
+      protocol_list_supports_protocol(protocols, PRT_RELAY, 2);
+    out->supports_ed25519_link_handshake_compat =
+      protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3);
+    out->supports_ed25519_link_handshake_any =
+      protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3);
+    out->supports_ed25519_hs_intro =
+      protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4);
+    out->supports_v3_hsdir =
+      protocol_list_supports_protocol(protocols, PRT_HSDIR,
+                                      PROTOVER_HSDIR_V3);
+    out->supports_v3_rendezvous_point =
+      protocol_list_supports_protocol(protocols, PRT_HSREND,
+                                      PROTOVER_HS_RENDEZVOUS_POINT_V3);
+  }
+  if (version && !strcmpstart(version, "Tor ")) {
+    if (!out->protocols_known) {
+      /* The version is a "Tor" version, and where there is no
+       * list of protocol versions that we should be looking at instead. */
+
+      out->supports_extend2_cells =
+        tor_version_as_new_as(version, "0.2.4.8-alpha");
+      out->protocols_known = 1;
+    } else {
+      /* Bug #22447 forces us to filter on this version. */
+      if (!tor_version_as_new_as(version, "0.3.0.8")) {
+        out->supports_v3_hsdir = 0;
+      }
+    }
+  }
+}
diff --git a/src/core/or/versions.h b/src/core/or/versions.h
new file mode 100644
index 0000000000..22f3be176f
--- /dev/null
+++ b/src/core/or/versions.h
@@ -0,0 +1,44 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file versions.h
+ * \brief Header file for versions.c.
+ **/
+
+#ifndef TOR_VERSIONS_H
+#define TOR_VERSIONS_H
+
+/** Possible statuses of a version of Tor, given opinions from the directory
+ * servers. */
+typedef enum version_status_t {
+  VS_RECOMMENDED=0, /**< This version is listed as recommended. */
+  VS_OLD=1, /**< This version is older than any recommended version. */
+  VS_NEW=2, /**< This version is newer than any recommended version. */
+  VS_NEW_IN_SERIES=3, /**< This version is newer than any recommended version
+                       * in its series, but later recommended versions exist.
+                       */
+  VS_UNRECOMMENDED=4, /**< This version is not recommended (general case). */
+  VS_EMPTY=5, /**< The version list was empty; no agreed-on versions. */
+  VS_UNKNOWN, /**< We have no idea. */
+} version_status_t;
+
+version_status_t tor_version_is_obsolete(const char *myversion,
+                                         const char *versionlist);
+int tor_version_parse_platform(const char *platform,
+                               tor_version_t *version_out,
+                               int strict);
+int tor_version_as_new_as(const char *platform, const char *cutoff);
+int tor_version_parse(const char *s, tor_version_t *out);
+int tor_version_compare(tor_version_t *a, tor_version_t *b);
+int tor_version_same_series(tor_version_t *a, tor_version_t *b);
+void sort_version_list(smartlist_t *lst, int remove_duplicates);
+
+void summarize_protover_flags(protover_summary_flags_t *out,
+                              const char *protocols,
+                              const char *version);
+
+#endif /* !defined(TOR_VERSIONS_H) */
diff --git a/src/core/proto/proto_cell.c b/src/core/proto/proto_cell.c
new file mode 100644
index 0000000000..0442e2c6ee
--- /dev/null
+++ b/src/core/proto/proto_cell.c
@@ -0,0 +1,86 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "lib/container/buffers.h"
+#include "core/proto/proto_cell.h"
+
+#include "core/or/connection_or.h"
+
+#include "core/or/var_cell_st.h"
+
+/** True iff the cell command <b>command</b> is one that implies a
+ * variable-length cell in Tor link protocol <b>linkproto</b>. */
+static inline int
+cell_command_is_var_length(uint8_t command, int linkproto)
+{
+  /* If linkproto is v2 (2), CELL_VERSIONS is the only variable-length cells
+   * work as implemented here. If it's 1, there are no variable-length cells.
+   * Tor does not support other versions right now, and so can't negotiate
+   * them.
+   */
+  switch (linkproto) {
+  case 1:
+    /* Link protocol version 1 has no variable-length cells. */
+    return 0;
+  case 2:
+    /* In link protocol version 2, VERSIONS is the only variable-length cell */
+    return command == CELL_VERSIONS;
+  case 0:
+  case 3:
+  default:
+    /* In link protocol version 3 and later, and in version "unknown",
+     * commands 128 and higher indicate variable-length. VERSIONS is
+     * grandfathered in. */
+    return command == CELL_VERSIONS || command >= 128;
+  }
+}
+
+/** Check <b>buf</b> for a variable-length cell according to the rules of link
+ * protocol version <b>linkproto</b>.  If one is found, pull it off the buffer
+ * and assign a newly allocated var_cell_t to *<b>out</b>, and return 1.
+ * Return 0 if whatever is on the start of buf_t is not a variable-length
+ * cell.  Return 1 and set *<b>out</b> to NULL if there seems to be the start
+ * of a variable-length cell on <b>buf</b>, but the whole thing isn't there
+ * yet. */
+int
+fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto)
+{
+  char hdr[VAR_CELL_MAX_HEADER_SIZE];
+  var_cell_t *result;
+  uint8_t command;
+  uint16_t length;
+  const int wide_circ_ids = linkproto >= MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS;
+  const int circ_id_len = get_circ_id_size(wide_circ_ids);
+  const unsigned header_len = get_var_cell_header_size(wide_circ_ids);
+  *out = NULL;
+  if (buf_datalen(buf) < header_len)
+    return 0;
+  buf_peek(buf, hdr, header_len);
+
+  command = get_uint8(hdr + circ_id_len);
+  if (!(cell_command_is_var_length(command, linkproto)))
+    return 0;
+
+  length = ntohs(get_uint16(hdr + circ_id_len + 1));
+  if (buf_datalen(buf) < (size_t)(header_len+length))
+    return 1;
+
+  result = var_cell_new(length);
+  result->command = command;
+  if (wide_circ_ids)
+    result->circ_id = ntohl(get_uint32(hdr));
+  else
+    result->circ_id = ntohs(get_uint16(hdr));
+
+  buf_drain(buf, header_len);
+  buf_peek(buf, (char*) result->payload, length);
+  buf_drain(buf, length);
+
+  *out = result;
+  return 1;
+}
+
diff --git a/src/core/proto/proto_cell.h b/src/core/proto/proto_cell.h
new file mode 100644
index 0000000000..4f3982ea43
--- /dev/null
+++ b/src/core/proto/proto_cell.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PROTO_CELL_H
+#define TOR_PROTO_CELL_H
+
+struct buf_t;
+struct var_cell_t;
+
+int fetch_var_cell_from_buf(struct buf_t *buf, struct var_cell_t **out,
+                            int linkproto);
+
+#endif /* !defined(TOR_PROTO_CELL_H) */
+
diff --git a/src/core/proto/proto_control0.c b/src/core/proto/proto_control0.c
new file mode 100644
index 0000000000..21fa328f02
--- /dev/null
+++ b/src/core/proto/proto_control0.c
@@ -0,0 +1,26 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "lib/container/buffers.h"
+#include "core/proto/proto_control0.h"
+
+/** Return 1 iff buf looks more like it has an (obsolete) v0 controller
+ * command on it than any valid v1 controller command. */
+int
+peek_buf_has_control0_command(buf_t *buf)
+{
+  if (buf_datalen(buf) >= 4) {
+    char header[4];
+    uint16_t cmd;
+    buf_peek(buf, header, sizeof(header));
+    cmd = ntohs(get_uint16(header+2));
+    if (cmd <= 0x14)
+      return 1; /* This is definitely not a v1 control command. */
+  }
+  return 0;
+}
+
diff --git a/src/core/proto/proto_control0.h b/src/core/proto/proto_control0.h
new file mode 100644
index 0000000000..162e513a1b
--- /dev/null
+++ b/src/core/proto/proto_control0.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PROTO_CONTROL0_H
+#define TOR_PROTO_CONTROL0_H
+
+struct buf_t;
+int peek_buf_has_control0_command(struct buf_t *buf);
+
+#endif /* !defined(TOR_PROTO_CONTROL0_H) */
+
diff --git a/src/core/proto/proto_ext_or.c b/src/core/proto/proto_ext_or.c
new file mode 100644
index 0000000000..edbc51b10c
--- /dev/null
+++ b/src/core/proto/proto_ext_or.c
@@ -0,0 +1,40 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "lib/container/buffers.h"
+#include "feature/relay/ext_orport.h"
+#include "core/proto/proto_ext_or.h"
+
+/** The size of the header of an Extended ORPort message: 2 bytes for
+ *  COMMAND, 2 bytes for BODYLEN */
+#define EXT_OR_CMD_HEADER_SIZE 4
+
+/** Read <b>buf</b>, which should contain an Extended ORPort message
+ *  from a transport proxy. If well-formed, create and populate
+ *  <b>out</b> with the Extended ORport message. Return 0 if the
+ *  buffer was incomplete, 1 if it was well-formed and -1 if we
+ *  encountered an error while parsing it.  */
+int
+fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out)
+{
+  char hdr[EXT_OR_CMD_HEADER_SIZE];
+  uint16_t len;
+
+  if (buf_datalen(buf) < EXT_OR_CMD_HEADER_SIZE)
+    return 0;
+  buf_peek(buf, hdr, sizeof(hdr));
+  len = ntohs(get_uint16(hdr+2));
+  if (buf_datalen(buf) < (unsigned)len + EXT_OR_CMD_HEADER_SIZE)
+    return 0;
+  *out = ext_or_cmd_new(len);
+  (*out)->cmd = ntohs(get_uint16(hdr));
+  (*out)->len = len;
+  buf_drain(buf, EXT_OR_CMD_HEADER_SIZE);
+  buf_get_bytes(buf, (*out)->body, len);
+  return 1;
+}
+
diff --git a/src/core/proto/proto_ext_or.h b/src/core/proto/proto_ext_or.h
new file mode 100644
index 0000000000..b2bc64af85
--- /dev/null
+++ b/src/core/proto/proto_ext_or.h
@@ -0,0 +1,22 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PROTO_EXT_OR_H
+#define TOR_PROTO_EXT_OR_H
+
+struct buf_t;
+
+/** A parsed Extended ORPort message. */
+struct ext_or_cmd_t {
+  uint16_t cmd; /** Command type */
+  uint16_t len; /** Body length */
+  char body[FLEXIBLE_ARRAY_MEMBER]; /** Message body */
+};
+
+int fetch_ext_or_command_from_buf(struct buf_t *buf,
+                                  struct ext_or_cmd_t **out);
+
+#endif /* !defined(TOR_PROTO_EXT_OR_H) */
diff --git a/src/core/proto/proto_http.c b/src/core/proto/proto_http.c
new file mode 100644
index 0000000000..5c86fc4979
--- /dev/null
+++ b/src/core/proto/proto_http.c
@@ -0,0 +1,171 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define PROTO_HTTP_PRIVATE
+#include "core/or/or.h"
+#include "lib/container/buffers.h"
+#include "core/proto/proto_http.h"
+
+/** Return true if <b>cmd</b> looks like a HTTP (proxy) request. */
+int
+peek_buf_has_http_command(const buf_t *buf)
+{
+  if (buf_peek_startswith(buf, "CONNECT ") ||
+      buf_peek_startswith(buf, "DELETE ") ||
+      buf_peek_startswith(buf, "GET ") ||
+      buf_peek_startswith(buf, "POST ") ||
+      buf_peek_startswith(buf, "PUT " ))
+    return 1;
+  return 0;
+}
+
+/** There is a (possibly incomplete) http statement on <b>buf</b>, of the
+ * form "\%s\\r\\n\\r\\n\%s", headers, body. (body may contain NULs.)
+ * If a) the headers include a Content-Length field and all bytes in
+ * the body are present, or b) there's no Content-Length field and
+ * all headers are present, then:
+ *
+ *  - strdup headers into <b>*headers_out</b>, and NUL-terminate it.
+ *  - memdup body into <b>*body_out</b>, and NUL-terminate it.
+ *  - Then remove them from <b>buf</b>, and return 1.
+ *
+ *  - If headers or body is NULL, discard that part of the buf.
+ *  - If a headers or body doesn't fit in the arg, return -1.
+ *  (We ensure that the headers or body don't exceed max len,
+ *   _even if_ we're planning to discard them.)
+ *  - If force_complete is true, then succeed even if not all of the
+ *    content has arrived.
+ *
+ * Else, change nothing and return 0.
+ */
+int
+fetch_from_buf_http(buf_t *buf,
+                    char **headers_out, size_t max_headerlen,
+                    char **body_out, size_t *body_used, size_t max_bodylen,
+                    int force_complete)
+{
+  const char *headers;
+  size_t headerlen, bodylen, contentlen=0;
+  int crlf_offset;
+  int r;
+
+  if (buf_datalen(buf) == 0)
+    return 0;
+
+  crlf_offset = buf_find_string_offset(buf, "\r\n\r\n", 4);
+  if (crlf_offset > (int)max_headerlen ||
+      (crlf_offset < 0 && buf_datalen(buf) > max_headerlen)) {
+    log_debug(LD_HTTP,"headers too long.");
+    return -1;
+  } else if (crlf_offset < 0) {
+    log_debug(LD_HTTP,"headers not all here yet.");
+    return 0;
+  }
+  /* Okay, we have a full header.  Make sure it all appears in the first
+   * chunk. */
+  headerlen = crlf_offset + 4;
+  size_t headers_in_chunk = 0;
+  buf_pullup(buf, headerlen, &headers, &headers_in_chunk);
+
+  bodylen = buf_datalen(buf) - headerlen;
+  log_debug(LD_HTTP,"headerlen %d, bodylen %d.", (int)headerlen, (int)bodylen);
+
+  if (max_headerlen <= headerlen) {
+    log_warn(LD_HTTP,"headerlen %d larger than %d. Failing.",
+             (int)headerlen, (int)max_headerlen-1);
+    return -1;
+  }
+  if (max_bodylen <= bodylen) {
+    log_warn(LD_HTTP,"bodylen %d larger than %d. Failing.",
+             (int)bodylen, (int)max_bodylen-1);
+    return -1;
+  }
+
+  r = buf_http_find_content_length(headers, headerlen, &contentlen);
+  if (r == -1) {
+    log_warn(LD_PROTOCOL, "Content-Length is bogus; maybe "
+             "someone is trying to crash us.");
+    return -1;
+  } else if (r == 1) {
+    /* if content-length is malformed, then our body length is 0. fine. */
+    log_debug(LD_HTTP,"Got a contentlen of %d.",(int)contentlen);
+    if (bodylen < contentlen) {
+      if (!force_complete) {
+        log_debug(LD_HTTP,"body not all here yet.");
+        return 0; /* not all there yet */
+      }
+    }
+    if (bodylen > contentlen) {
+      bodylen = contentlen;
+      log_debug(LD_HTTP,"bodylen reduced to %d.",(int)bodylen);
+    }
+  } else {
+    tor_assert(r == 0);
+    /* Leave bodylen alone */
+  }
+
+  /* all happy. copy into the appropriate places, and return 1 */
+  if (headers_out) {
+    *headers_out = tor_malloc(headerlen+1);
+    buf_get_bytes(buf, *headers_out, headerlen);
+    (*headers_out)[headerlen] = 0; /* NUL terminate it */
+  }
+  if (body_out) {
+    tor_assert(body_used);
+    *body_used = bodylen;
+    *body_out = tor_malloc(bodylen+1);
+    buf_get_bytes(buf, *body_out, bodylen);
+    (*body_out)[bodylen] = 0; /* NUL terminate it */
+  }
+  return 1;
+}
+
+/**
+ * Scan the HTTP headers in the <b>headerlen</b>-byte memory range at
+ * <b>headers</b>, looking for a "Content-Length" header.  Try to set
+ * *<b>result_out</b> to the numeric value of that header if possible.
+ * Return -1 if the header was malformed, 0 if it was missing, and 1 if
+ * it was present and well-formed.
+ */
+STATIC int
+buf_http_find_content_length(const char *headers, size_t headerlen,
+                             size_t *result_out)
+{
+  const char *p, *newline;
+  char *len_str, *eos=NULL;
+  size_t remaining, result;
+  int ok;
+  *result_out = 0; /* The caller shouldn't look at this unless the
+                    * return value is 1, but let's prevent confusion */
+
+#define CONTENT_LENGTH "\r\nContent-Length: "
+  p = (char*) tor_memstr(headers, headerlen, CONTENT_LENGTH);
+  if (p == NULL)
+    return 0;
+
+  tor_assert(p >= headers && p < headers+headerlen);
+  remaining = (headers+headerlen)-p;
+  p += strlen(CONTENT_LENGTH);
+  remaining -= strlen(CONTENT_LENGTH);
+
+  newline = memchr(p, '\n', remaining);
+  if (newline == NULL)
+    return -1;
+
+  len_str = tor_memdup_nulterm(p, newline-p);
+  /* We limit the size to INT_MAX because other parts of the buffer.c
+   * code don't like buffers to be any bigger than that. */
+  result = (size_t) tor_parse_uint64(len_str, 10, 0, INT_MAX, &ok, &eos);
+  if (eos && !tor_strisspace(eos)) {
+    ok = 0;
+  } else {
+    *result_out = result;
+  }
+  tor_free(len_str);
+
+  return ok ? 1 : -1;
+}
+
diff --git a/src/core/proto/proto_http.h b/src/core/proto/proto_http.h
new file mode 100644
index 0000000000..cd70050205
--- /dev/null
+++ b/src/core/proto/proto_http.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PROTO_HTTP_H
+#define TOR_PROTO_HTTP_H
+
+struct buf_t;
+
+int fetch_from_buf_http(struct buf_t *buf,
+                        char **headers_out, size_t max_headerlen,
+                        char **body_out, size_t *body_used, size_t max_bodylen,
+                        int force_complete);
+int peek_buf_has_http_command(const struct buf_t *buf);
+
+#ifdef PROTO_HTTP_PRIVATE
+STATIC int buf_http_find_content_length(const char *headers, size_t headerlen,
+                                        size_t *result_out);
+#endif
+
+#endif /* !defined(TOR_PROTO_HTTP_H) */
+
diff --git a/src/core/proto/proto_socks.c b/src/core/proto/proto_socks.c
new file mode 100644
index 0000000000..8b78ed44c2
--- /dev/null
+++ b/src/core/proto/proto_socks.c
@@ -0,0 +1,1133 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "feature/client/addressmap.h"
+#include "lib/container/buffers.h"
+#include "core/mainloop/connection.h"
+#include "feature/control/control.h"
+#include "app/config/config.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "feature/relay/ext_orport.h"
+#include "core/proto/proto_socks.h"
+#include "core/or/reasons.h"
+
+#include "core/or/socks_request_st.h"
+
+#include "trunnel/socks5.h"
+
+#define SOCKS_VER_5 0x05 /* First octet of non-auth SOCKS5 messages */
+#define SOCKS_VER_4 0x04 /*                SOCKS4 messages */
+#define SOCKS_AUTH  0x01 /*                SOCKS5 auth messages */
+
+typedef enum {
+  SOCKS_RESULT_INVALID       = -1, /* Message invalid. */
+  SOCKS_RESULT_TRUNCATED     =  0, /* Message incomplete/truncated. */
+  SOCKS_RESULT_DONE          =  1, /* OK, we're done. */
+  SOCKS_RESULT_MORE_EXPECTED =  2, /* OK, more messages expected. */
+} socks_result_t;
+
+static void socks_request_set_socks5_error(socks_request_t *req,
+                              socks5_reply_status_t reason);
+
+static socks_result_t parse_socks(const char *data,
+                                  size_t datalen,
+                                  socks_request_t *req,
+                                  int log_sockstype,
+                                  int safe_socks,
+                                  size_t *drain_out);
+static int parse_socks_client(const uint8_t *data, size_t datalen,
+                              int state, char **reason,
+                              ssize_t *drain_out);
+/**
+ * Wait this many seconds before warning the user about using SOCKS unsafely
+ * again. */
+#define SOCKS_WARN_INTERVAL 5
+
+/** Warn that the user application has made an unsafe socks request using
+ * protocol <b>socks_protocol</b> on port <b>port</b>.  Don't warn more than
+ * once per SOCKS_WARN_INTERVAL, unless <b>safe_socks</b> is set. */
+static void
+log_unsafe_socks_warning(int socks_protocol, const char *address,
+                         uint16_t port, int safe_socks)
+{
+  static ratelim_t socks_ratelim = RATELIM_INIT(SOCKS_WARN_INTERVAL);
+
+  if (safe_socks) {
+    log_fn_ratelim(&socks_ratelim, LOG_WARN, LD_APP,
+             "Your application (using socks%d to port %d) is giving "
+             "Tor only an IP address. Applications that do DNS resolves "
+             "themselves may leak information. Consider using Socks4A "
+             "(e.g. via privoxy or socat) instead. For more information, "
+             "please see https://wiki.torproject.org/TheOnionRouter/"
+             "TorFAQ#SOCKSAndDNS.%s",
+             socks_protocol,
+             (int)port,
+             safe_socks ? " Rejecting." : "");
+  }
+  control_event_client_status(LOG_WARN,
+                              "DANGEROUS_SOCKS PROTOCOL=SOCKS%d ADDRESS=%s:%d",
+                              socks_protocol, address, (int)port);
+}
+
+/** Do not attempt to parse socks messages longer than this.  This value is
+ * actually significantly higher than the longest possible socks message. */
+#define MAX_SOCKS_MESSAGE_LEN 512
+
+/** Return a new socks_request_t. */
+socks_request_t *
+socks_request_new(void)
+{
+  return tor_malloc_zero(sizeof(socks_request_t));
+}
+
+/** Free all storage held in the socks_request_t <b>req</b>. */
+void
+socks_request_free_(socks_request_t *req)
+{
+  if (!req)
+    return;
+  if (req->username) {
+    memwipe(req->username, 0x10, req->usernamelen);
+    tor_free(req->username);
+  }
+  if (req->password) {
+    memwipe(req->password, 0x04, req->passwordlen);
+    tor_free(req->password);
+  }
+  memwipe(req, 0xCC, sizeof(socks_request_t));
+  tor_free(req);
+}
+
+/**
+ * Parse a single SOCKS4 request from buffer <b>raw_data</b> of length
+ * <b>datalen</b> and update relevant fields of <b>req</b>. If SOCKS4a
+ * request is detected, set <b>*is_socks4a<b> to true. Set <b>*drain_out</b>
+ * to number of bytes we parsed so far.
+ *
+ * Return SOCKS_RESULT_DONE if parsing succeeded, SOCKS_RESULT_INVALID if
+ * parsing failed because of invalid input or SOCKS_RESULT_TRUNCATED if it
+ * failed due to incomplete (truncated) input.
+ */
+static socks_result_t
+parse_socks4_request(const uint8_t *raw_data, socks_request_t *req,
+                     size_t datalen, int *is_socks4a, size_t *drain_out)
+{
+  // http://ss5.sourceforge.net/socks4.protocol.txt
+  // http://ss5.sourceforge.net/socks4A.protocol.txt
+  socks_result_t res = SOCKS_RESULT_DONE;
+  tor_addr_t destaddr;
+
+  tor_assert(is_socks4a);
+  tor_assert(drain_out);
+
+  *is_socks4a = 0;
+  *drain_out = 0;
+
+  req->socks_version = SOCKS_VER_4;
+
+  socks4_client_request_t *trunnel_req;
+
+  ssize_t parsed =
+  socks4_client_request_parse(&trunnel_req, raw_data, datalen);
+
+  if (parsed == -1) {
+    log_warn(LD_APP, "socks4: parsing failed - invalid request.");
+    res = SOCKS_RESULT_INVALID;
+    goto end;
+  } else if (parsed == -2) {
+    res = SOCKS_RESULT_TRUNCATED;
+    if (datalen >= MAX_SOCKS_MESSAGE_LEN) {
+      log_warn(LD_APP, "socks4: parsing failed - invalid request.");
+      res = SOCKS_RESULT_INVALID;
+    }
+    goto end;
+  }
+
+  tor_assert(parsed >= 0);
+  *drain_out = (size_t)parsed;
+
+  uint8_t command = socks4_client_request_get_command(trunnel_req);
+  req->command = command;
+
+  req->port = socks4_client_request_get_port(trunnel_req);
+  uint32_t dest_ip = socks4_client_request_get_addr(trunnel_req);
+
+  if ((!req->port && req->command != SOCKS_COMMAND_RESOLVE) ||
+      dest_ip == 0) {
+    log_warn(LD_APP, "socks4: Port or DestIP is zero. Rejecting.");
+    res = SOCKS_RESULT_INVALID;
+    goto end;
+  }
+
+  *is_socks4a = (dest_ip >> 8) == 0;
+
+  const char *username = socks4_client_request_get_username(trunnel_req);
+  const size_t usernamelen = username ? strlen(username) : 0;
+  if (username && usernamelen) {
+    if (usernamelen > MAX_SOCKS_MESSAGE_LEN) {
+      log_warn(LD_APP, "Socks4 user name too long; rejecting.");
+      res = SOCKS_RESULT_INVALID;
+      goto end;
+    }
+
+    tor_free(req->username);
+    req->got_auth = 1;
+    req->username = tor_strdup(username);
+    req->usernamelen = usernamelen;
+  }
+
+  if (*is_socks4a) {
+    // We cannot rely on trunnel here, as we want to detect if
+    // we have abnormally long hostname field.
+    const char *hostname = (char *)raw_data + SOCKS4_NETWORK_LEN +
+     usernamelen + 1;
+    size_t hostname_len = (char *)raw_data + datalen - hostname;
+
+    if (hostname_len <= sizeof(req->address)) {
+      const char *trunnel_hostname =
+      socks4_client_request_get_socks4a_addr_hostname(trunnel_req);
+
+      if (trunnel_hostname)
+        strlcpy(req->address, trunnel_hostname, sizeof(req->address));
+    } else {
+      log_warn(LD_APP, "socks4: Destaddr too long. Rejecting.");
+      res = SOCKS_RESULT_INVALID;
+      goto end;
+    }
+  } else {
+    tor_addr_from_ipv4h(&destaddr, dest_ip);
+
+    if (!tor_addr_to_str(req->address, &destaddr,
+                         MAX_SOCKS_ADDR_LEN, 0)) {
+      res = SOCKS_RESULT_INVALID;
+      goto end;
+    }
+  }
+
+  end:
+  socks4_client_request_free(trunnel_req);
+
+  return res;
+}
+
+/**
+ * Validate SOCKS4/4a related fields in <b>req</b>. Expect SOCKS4a
+ * if <b>is_socks4a</b> is true. If <b>log_sockstype</b> is true,
+ * log a notice about possible DNS leaks on local system. If
+ * <b>safe_socks</b> is true, reject insecure usage of SOCKS
+ * protocol.
+ *
+ * Return SOCKS_RESULT_DONE if validation passed or
+ * SOCKS_RESULT_INVALID if it failed.
+ */
+static socks_result_t
+process_socks4_request(const socks_request_t *req, int is_socks4a,
+                       int log_sockstype, int safe_socks)
+{
+  if (is_socks4a && !addressmap_have_mapping(req->address, 0)) {
+    log_unsafe_socks_warning(4, req->address, req->port, safe_socks);
+
+    if (safe_socks)
+      return SOCKS_RESULT_INVALID;
+  }
+
+  if (req->command != SOCKS_COMMAND_CONNECT &&
+      req->command != SOCKS_COMMAND_RESOLVE) {
+    /* not a connect or resolve? we don't support it. (No resolve_ptr with
+     * socks4.) */
+    log_warn(LD_APP, "socks4: command %d not recognized. Rejecting.",
+             req->command);
+    return SOCKS_RESULT_INVALID;
+  }
+
+  if (is_socks4a) {
+    if (log_sockstype)
+      log_notice(LD_APP,
+                 "Your application (using socks4a to port %d) instructed "
+                 "Tor to take care of the DNS resolution itself if "
+                 "necessary. This is good.", req->port);
+  }
+
+  if (!string_is_valid_dest(req->address)) {
+    log_warn(LD_PROTOCOL,
+             "Your application (using socks4 to port %d) gave Tor "
+             "a malformed hostname: %s. Rejecting the connection.",
+             req->port, escaped_safe_str_client(req->address));
+     return SOCKS_RESULT_INVALID;
+  }
+
+  return SOCKS_RESULT_DONE;
+}
+
+/** Parse a single SOCKS5 version identifier/method selection message
+ * from buffer <b>raw_data</b> (of length <b>datalen</b>). Update
+ * relevant fields of <b>req</b> (if any). Set <b>*have_user_pass</b> to
+ * true if username/password method is found. Set <b>*have_no_auth</b>
+ * if no-auth method is found. Set <b>*drain_out</b> to number of bytes
+ * we parsed so far.
+ *
+ * Return SOCKS_RESULT_DONE if parsing succeeded, SOCKS_RESULT_INVALID if
+ * parsing failed because of invalid input or SOCKS_RESULT_TRUNCATED if it
+ * failed due to incomplete (truncated) input.
+ */
+static socks_result_t
+parse_socks5_methods_request(const uint8_t *raw_data, socks_request_t *req,
+                             size_t datalen, int *have_user_pass,
+                             int *have_no_auth, size_t *drain_out)
+{
+  socks_result_t res = SOCKS_RESULT_DONE;
+  socks5_client_version_t *trunnel_req;
+
+  ssize_t parsed = socks5_client_version_parse(&trunnel_req, raw_data,
+                                               datalen);
+
+  (void)req;
+
+  tor_assert(have_no_auth);
+  tor_assert(have_user_pass);
+  tor_assert(drain_out);
+
+  *drain_out = 0;
+
+  if (parsed == -1) {
+    log_warn(LD_APP, "socks5: parsing failed - invalid version "
+                     "id/method selection message.");
+    res = SOCKS_RESULT_INVALID;
+    goto end;
+  } else if (parsed == -2) {
+    res = SOCKS_RESULT_TRUNCATED;
+    if (datalen > MAX_SOCKS_MESSAGE_LEN) {
+      log_warn(LD_APP, "socks5: parsing failed - invalid version "
+                       "id/method selection message.");
+      res = SOCKS_RESULT_INVALID;
+    }
+    goto end;
+  }
+
+  tor_assert(parsed >= 0);
+  *drain_out = (size_t)parsed;
+
+  size_t n_methods = (size_t)socks5_client_version_get_n_methods(trunnel_req);
+  if (n_methods == 0) {
+    res = SOCKS_RESULT_INVALID;
+    goto end;
+  }
+
+  *have_no_auth = 0;
+  *have_user_pass = 0;
+
+  for (size_t i = 0; i < n_methods; i++) {
+    uint8_t method = socks5_client_version_get_methods(trunnel_req,
+                                                       i);
+
+    if (method == SOCKS_USER_PASS) {
+      *have_user_pass = 1;
+    } else if (method == SOCKS_NO_AUTH) {
+      *have_no_auth = 1;
+    }
+  }
+
+  end:
+  socks5_client_version_free(trunnel_req);
+
+  return res;
+}
+
+/**
+ * Validate and respond to version identifier/method selection message
+ * we parsed in parse_socks5_methods_request (corresponding to <b>req</b>
+ * and having user/pass method if <b>have_user_pass</b> is true, no-auth
+ * method if <b>have_no_auth</b> is true). Set <b>req->reply</b> to
+ * an appropriate response (in SOCKS5 wire format).
+ *
+ * On success, return SOCKS_RESULT_DONE. On failure, return
+ * SOCKS_RESULT_INVALID.
+ */
+static socks_result_t
+process_socks5_methods_request(socks_request_t *req, int have_user_pass,
+                               int have_no_auth)
+{
+  socks_result_t res = SOCKS_RESULT_DONE;
+  socks5_server_method_t *trunnel_resp = socks5_server_method_new();
+  tor_assert(trunnel_resp);
+
+  socks5_server_method_set_version(trunnel_resp, SOCKS_VER_5);
+
+  if (have_user_pass && !(have_no_auth && req->socks_prefer_no_auth)) {
+    req->auth_type = SOCKS_USER_PASS;
+    socks5_server_method_set_method(trunnel_resp, SOCKS_USER_PASS);
+
+    req->socks_version = SOCKS_VER_5;
+    // FIXME: come up with better way to remember
+    // that we negotiated auth
+
+    log_debug(LD_APP,"socks5: accepted method 2 (username/password)");
+  } else if (have_no_auth) {
+    req->auth_type = SOCKS_NO_AUTH;
+    socks5_server_method_set_method(trunnel_resp, SOCKS_NO_AUTH);
+
+    req->socks_version = SOCKS_VER_5;
+
+    log_debug(LD_APP,"socks5: accepted method 0 (no authentication)");
+  } else {
+    log_warn(LD_APP,
+             "socks5: offered methods don't include 'no auth' or "
+             "username/password. Rejecting.");
+    socks5_server_method_set_method(trunnel_resp, 0xFF); // reject all
+    res = SOCKS_RESULT_INVALID;
+  }
+
+  const char *errmsg = socks5_server_method_check(trunnel_resp);
+  if (errmsg) {
+    log_warn(LD_APP, "socks5: method selection validation failed: %s",
+             errmsg);
+    res = SOCKS_RESULT_INVALID;
+  } else {
+    ssize_t encoded =
+    socks5_server_method_encode(req->reply, sizeof(req->reply),
+                                trunnel_resp);
+
+    if (encoded < 0) {
+      log_warn(LD_APP, "socks5: method selection encoding failed");
+      res = SOCKS_RESULT_INVALID;
+    } else {
+      req->replylen = (size_t)encoded;
+    }
+  }
+
+  socks5_server_method_free(trunnel_resp);
+  return res;
+}
+
+/**
+ * Parse SOCKS5/RFC1929 username/password request from buffer
+ * <b>raw_data</b> of length <b>datalen</b> and update relevant
+ * fields of <b>req</b>. Set <b>*drain_out</b> to number of bytes
+ * we parsed so far.
+ *
+ * Return SOCKS_RESULT_DONE if parsing succeeded, SOCKS_RESULT_INVALID if
+ * parsing failed because of invalid input or SOCKS_RESULT_TRUNCATED if it
+ * failed due to incomplete (truncated) input.
+ */
+static socks_result_t
+parse_socks5_userpass_auth(const uint8_t *raw_data, socks_request_t *req,
+                           size_t datalen, size_t *drain_out)
+{
+  socks_result_t res = SOCKS_RESULT_DONE;
+  socks5_client_userpass_auth_t *trunnel_req = NULL;
+  ssize_t parsed = socks5_client_userpass_auth_parse(&trunnel_req, raw_data,
+                                                     datalen);
+  tor_assert(drain_out);
+  *drain_out = 0;
+
+  if (parsed == -1) {
+    log_warn(LD_APP, "socks5: parsing failed - invalid user/pass "
+                     "authentication message.");
+    res = SOCKS_RESULT_INVALID;
+    goto end;
+  } else if (parsed == -2) {
+    res = SOCKS_RESULT_TRUNCATED;
+    goto end;
+  }
+
+  tor_assert(parsed >= 0);
+  *drain_out = (size_t)parsed;
+
+  uint8_t usernamelen =
+   socks5_client_userpass_auth_get_username_len(trunnel_req);
+  uint8_t passwordlen =
+   socks5_client_userpass_auth_get_passwd_len(trunnel_req);
+  const char *username =
+   socks5_client_userpass_auth_getconstarray_username(trunnel_req);
+  const char *password =
+   socks5_client_userpass_auth_getconstarray_passwd(trunnel_req);
+
+  if (usernamelen && username) {
+    tor_free(req->username);
+    req->username = tor_memdup_nulterm(username, usernamelen);
+    req->usernamelen = usernamelen;
+  }
+
+  if (passwordlen && password) {
+    tor_free(req->password);
+    req->password = tor_memdup_nulterm(password, passwordlen);
+    req->passwordlen = passwordlen;
+  }
+
+  /**
+   * Yes, we allow username and/or password to be empty. Yes, that does
+   * violate RFC 1929. However, some client software can send a username/
+   * password message with these fields being empty and we want to allow them
+   * to be used with Tor.
+   */
+  req->got_auth = 1;
+
+  end:
+  socks5_client_userpass_auth_free(trunnel_req);
+  return res;
+}
+
+/**
+ * Validate and respond to SOCKS5 username/password request we
+ * parsed in parse_socks5_userpass_auth (corresponding to <b>req</b>.
+ * Set <b>req->reply</b> to appropriate responsed. Return
+ * SOCKS_RESULT_DONE on success or SOCKS_RESULT_INVALID on failure.
+ */
+static socks_result_t
+process_socks5_userpass_auth(socks_request_t *req)
+{
+  socks_result_t res = SOCKS_RESULT_DONE;
+  socks5_server_userpass_auth_t *trunnel_resp =
+    socks5_server_userpass_auth_new();
+  tor_assert(trunnel_resp);
+
+  if (req->socks_version != SOCKS_VER_5) {
+    res = SOCKS_RESULT_INVALID;
+    goto end;
+  }
+
+  if (req->auth_type != SOCKS_USER_PASS &&
+      req->auth_type != SOCKS_NO_AUTH) {
+    res = SOCKS_RESULT_INVALID;
+    goto end;
+  }
+
+  socks5_server_userpass_auth_set_version(trunnel_resp, SOCKS_AUTH);
+  socks5_server_userpass_auth_set_status(trunnel_resp, 0); // auth OK
+
+  const char *errmsg = socks5_server_userpass_auth_check(trunnel_resp);
+  if (errmsg) {
+    log_warn(LD_APP, "socks5: server userpass auth validation failed: %s",
+             errmsg);
+    res = SOCKS_RESULT_INVALID;
+    goto end;
+  }
+
+  ssize_t encoded = socks5_server_userpass_auth_encode(req->reply,
+                                                       sizeof(req->reply),
+                                                       trunnel_resp);
+
+  if (encoded < 0) {
+    log_warn(LD_APP, "socks5: server userpass auth encoding failed");
+    res = SOCKS_RESULT_INVALID;
+    goto end;
+  }
+
+  req->replylen = (size_t)encoded;
+
+  end:
+  socks5_server_userpass_auth_free(trunnel_resp);
+  return res;
+}
+
+/**
+ * Parse a single SOCKS5 client request (RFC 1928 section 4) from buffer
+ * <b>raw_data</b> of length <b>datalen</b> and update relevant field of
+ * <b>req</b>. Set <b>*drain_out</b> to number of bytes we parsed so far.
+ *
+ * Return SOCKS_RESULT_DONE if parsing succeeded, SOCKS_RESULT_INVALID if
+ * parsing failed because of invalid input or SOCKS_RESULT_TRUNCATED if it
+ * failed due to incomplete (truncated) input.
+ */
+static socks_result_t
+parse_socks5_client_request(const uint8_t *raw_data, socks_request_t *req,
+                            size_t datalen, size_t *drain_out)
+{
+  socks_result_t res = SOCKS_RESULT_DONE;
+  tor_addr_t destaddr;
+  socks5_client_request_t *trunnel_req = NULL;
+  ssize_t parsed =
+   socks5_client_request_parse(&trunnel_req, raw_data, datalen);
+  if (parsed == -1) {
+    log_warn(LD_APP, "socks5: parsing failed - invalid client request");
+    res = SOCKS_RESULT_INVALID;
+    goto end;
+  } else if (parsed == -2) {
+    res = SOCKS_RESULT_TRUNCATED;
+    goto end;
+  }
+
+  tor_assert(parsed >= 0);
+  *drain_out = (size_t)parsed;
+
+  if (socks5_client_request_get_version(trunnel_req) != 5) {
+    res = SOCKS_RESULT_INVALID;
+    goto end;
+  }
+
+  req->command = socks5_client_request_get_command(trunnel_req);
+
+  req->port = socks5_client_request_get_dest_port(trunnel_req);
+
+  uint8_t atype = socks5_client_request_get_atype(trunnel_req);
+  req->socks5_atyp = atype;
+
+  switch (atype) {
+    case 1: {
+      uint32_t ipv4 = socks5_client_request_get_dest_addr_ipv4(trunnel_req);
+      tor_addr_from_ipv4h(&destaddr, ipv4);
+
+      tor_addr_to_str(req->address, &destaddr, sizeof(req->address), 1);
+    } break;
+    case 3: {
+      const struct domainname_st *dns_name =
+        socks5_client_request_getconst_dest_addr_domainname(trunnel_req);
+
+      const char *hostname = domainname_getconstarray_name(dns_name);
+
+      strlcpy(req->address, hostname, sizeof(req->address));
+    } break;
+    case 4: {
+      const char *ipv6 =
+        (const char *)socks5_client_request_getarray_dest_addr_ipv6(
+          trunnel_req);
+      tor_addr_from_ipv6_bytes(&destaddr, ipv6);
+
+      tor_addr_to_str(req->address, &destaddr, sizeof(req->address), 1);
+    } break;
+    default: {
+      res = -1;
+    } break;
+  }
+
+  end:
+  socks5_client_request_free(trunnel_req);
+  return res;
+}
+
+/**
+ * Validate and respond to SOCKS5 request we parsed in
+ * parse_socks5_client_request (corresponding to <b>req</b>.
+ * Write appropriate response to <b>req->reply</b> (in
+ * SOCKS5 wire format). If <b>log_sockstype</b> is true, log a
+ * notice about possible DNS leaks on local system. If
+ * <b>safe_socks</b> is true, disallow insecure usage of SOCKS
+ * protocol. Return SOCKS_RESULT_DONE on success or
+ * SOCKS_RESULT_INVALID on failure.
+ */
+static socks_result_t
+process_socks5_client_request(socks_request_t *req,
+                              int log_sockstype,
+                              int safe_socks)
+{
+  socks_result_t res = SOCKS_RESULT_DONE;
+
+  if (req->command != SOCKS_COMMAND_CONNECT &&
+      req->command != SOCKS_COMMAND_RESOLVE &&
+      req->command != SOCKS_COMMAND_RESOLVE_PTR) {
+    socks_request_set_socks5_error(req,SOCKS5_COMMAND_NOT_SUPPORTED);
+    res = SOCKS_RESULT_INVALID;
+    goto end;
+  }
+
+  if (req->command == SOCKS_COMMAND_RESOLVE_PTR &&
+      !string_is_valid_ipv4_address(req->address) &&
+      !string_is_valid_ipv6_address(req->address)) {
+    socks_request_set_socks5_error(req, SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED);
+    log_warn(LD_APP, "socks5 received RESOLVE_PTR command with "
+                     "hostname type. Rejecting.");
+
+    res = SOCKS_RESULT_INVALID;
+    goto end;
+  }
+
+  if (!string_is_valid_dest(req->address)) {
+    socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR);
+
+    log_warn(LD_PROTOCOL,
+             "Your application (using socks5 to port %d) gave Tor "
+             "a malformed hostname: %s. Rejecting the connection.",
+             req->port, escaped_safe_str_client(req->address));
+
+    res = SOCKS_RESULT_INVALID;
+    goto end;
+  }
+
+  if (req->socks5_atyp == 1 || req->socks5_atyp == 4) {
+    if (req->command != SOCKS_COMMAND_RESOLVE_PTR &&
+        !addressmap_have_mapping(req->address,0)) {
+      log_unsafe_socks_warning(5, req->address, req->port, safe_socks);
+      if (safe_socks) {
+        socks_request_set_socks5_error(req, SOCKS5_NOT_ALLOWED);
+        res = SOCKS_RESULT_INVALID;
+        goto end;
+      }
+    }
+  }
+
+  if (log_sockstype)
+    log_notice(LD_APP,
+              "Your application (using socks5 to port %d) instructed "
+              "Tor to take care of the DNS resolution itself if "
+              "necessary. This is good.", req->port);
+
+  end:
+  return res;
+}
+
+/**
+ * Handle (parse, validate, process, respond) a single SOCKS
+ * message in buffer <b>raw_data</b> of length <b>datalen</b>.
+ * Update relevant fields of <b>req</b>. If <b>log_sockstype</b>
+ * is true, log a warning about possible DNS leaks on local
+ * system. If <b>safe_socks</b> is true, disallow insecure
+ * usage of SOCKS protocol. Set <b>*drain_out</b> to number
+ * of bytes in <b>raw_data</b> that we processed so far and
+ * that can be safely drained from buffer.
+ *
+ * Return:
+ *  - SOCKS_RESULT_DONE if succeeded and not expecting further
+ *    messages from client.
+ *  - SOCKS_RESULT_INVALID if any of the steps failed due to
+ *    request being invalid or unexpected given current state.
+ *  - SOCKS_RESULT_TRUNCATED if we do not found an expected
+ *    SOCKS message in its entirety (more stuff has to arrive
+ *    from client).
+ *  - SOCKS_RESULT_MORE_EXPECTED if we handled current message
+ *    successfully, but we expect more messages from the
+ *    client.
+ */
+static socks_result_t
+handle_socks_message(const uint8_t *raw_data, size_t datalen,
+                     socks_request_t *req, int log_sockstype,
+                     int safe_socks, size_t *drain_out)
+{
+  socks_result_t res = SOCKS_RESULT_DONE;
+
+  uint8_t socks_version = raw_data[0];
+
+  if (socks_version == SOCKS_AUTH)
+    socks_version = SOCKS_VER_5; // SOCKS5 username/pass subnegotiation
+
+  if (socks_version == SOCKS_VER_4) {
+    if (datalen < SOCKS4_NETWORK_LEN) {
+      res = 0;
+      goto end;
+    }
+
+    int is_socks4a = 0;
+    res = parse_socks4_request((const uint8_t *)raw_data, req, datalen,
+                               &is_socks4a, drain_out);
+
+    if (res != SOCKS_RESULT_DONE) {
+      goto end;
+    }
+
+    res = process_socks4_request(req, is_socks4a,log_sockstype,
+                                 safe_socks);
+
+    if (res != SOCKS_RESULT_DONE) {
+      goto end;
+    }
+
+    goto end;
+  } else if (socks_version == SOCKS_VER_5) {
+    if (datalen < 2) { /* version and another byte */
+      res = 0;
+      goto end;
+    }
+    /* RFC1929 SOCKS5 username/password subnegotiation. */
+    if (!req->got_auth && (raw_data[0] == 1 ||
+        req->auth_type == SOCKS_USER_PASS)) {
+      res = parse_socks5_userpass_auth(raw_data, req, datalen,
+                                       drain_out);
+
+      if (res != SOCKS_RESULT_DONE) {
+        goto end;
+      }
+
+      res = process_socks5_userpass_auth(req);
+      if (res != SOCKS_RESULT_DONE) {
+        goto end;
+      }
+
+      res = SOCKS_RESULT_MORE_EXPECTED;
+      goto end;
+    } else if (req->socks_version != SOCKS_VER_5) {
+      int have_user_pass=0, have_no_auth=0;
+      res = parse_socks5_methods_request(raw_data, req, datalen,
+                                         &have_user_pass,
+                                         &have_no_auth,
+                                         drain_out);
+
+      if (res != SOCKS_RESULT_DONE) {
+        goto end;
+      }
+
+      res = process_socks5_methods_request(req, have_user_pass,
+                                           have_no_auth);
+
+      if (res != SOCKS_RESULT_DONE) {
+        goto end;
+      }
+
+      res = SOCKS_RESULT_MORE_EXPECTED;
+      goto end;
+    } else {
+      res = parse_socks5_client_request(raw_data, req,
+                                        datalen, drain_out);
+      if (res != SOCKS_RESULT_DONE) {
+        socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR);
+        goto end;
+      }
+
+      res = process_socks5_client_request(req, log_sockstype,
+                                          safe_socks);
+
+      if (res != SOCKS_RESULT_DONE) {
+        goto end;
+      }
+    }
+  } else {
+    *drain_out = datalen;
+    res = SOCKS_RESULT_INVALID;
+  }
+
+  end:
+  return res;
+}
+
+/** There is a (possibly incomplete) socks handshake on <b>buf</b>, of one
+ * of the forms
+ *  - socks4: "socksheader username\\0"
+ *  - socks4a: "socksheader username\\0 destaddr\\0"
+ *  - socks5 phase one: "version #methods methods"
+ *  - socks5 phase two: "version command 0 addresstype..."
+ * If it's a complete and valid handshake, and destaddr fits in
+ *   MAX_SOCKS_ADDR_LEN bytes, then pull the handshake off the buf,
+ *   assign to <b>req</b>, and return 1.
+ *
+ * If it's invalid or too big, return -1.
+ *
+ * Else it's not all there yet, leave buf alone and return 0.
+ *
+ * If you want to specify the socks reply, write it into <b>req->reply</b>
+ *   and set <b>req->replylen</b>, else leave <b>req->replylen</b> alone.
+ *
+ * If <b>log_sockstype</b> is non-zero, then do a notice-level log of whether
+ * the connection is possibly leaking DNS requests locally or not.
+ *
+ * If <b>safe_socks</b> is true, then reject unsafe socks protocols.
+ *
+ * If returning 0 or -1, <b>req->address</b> and <b>req->port</b> are
+ * undefined.
+ */
+int
+fetch_from_buf_socks(buf_t *buf, socks_request_t *req,
+                     int log_sockstype, int safe_socks)
+{
+  int res = 0;
+  size_t datalen = buf_datalen(buf);
+  size_t n_drain;
+  const char *head = NULL;
+  socks_result_t socks_res;
+  size_t n_pullup;
+
+  if (buf_datalen(buf) < 2) { /* version and another byte */
+    res = 0;
+    goto end;
+  }
+
+  do {
+    n_drain = 0;
+    n_pullup = MIN(MAX_SOCKS_MESSAGE_LEN, buf_datalen(buf));
+    buf_pullup(buf, n_pullup, &head, &datalen);
+    tor_assert(head && datalen >= 2);
+
+    socks_res = parse_socks(head, datalen, req, log_sockstype,
+                            safe_socks, &n_drain);
+
+    if (socks_res == SOCKS_RESULT_INVALID)
+      buf_clear(buf);
+    else if (socks_res != SOCKS_RESULT_TRUNCATED && n_drain > 0)
+      buf_drain(buf, n_drain);
+
+    switch (socks_res) {
+      case SOCKS_RESULT_INVALID:
+        res = -1;
+        break;
+      case SOCKS_RESULT_DONE:
+        res = 1;
+        break;
+      case SOCKS_RESULT_TRUNCATED:
+        if (datalen == n_pullup)
+          return 0;
+        /* FALLTHRU */
+      case SOCKS_RESULT_MORE_EXPECTED:
+        res = 0;
+        break;
+    }
+  } while (res == 0 && head && buf_datalen(buf) >= 2);
+
+  end:
+  return res;
+}
+
+/** Create a SOCKS5 reply message with <b>reason</b> in its REP field and
+ * have Tor send it as error response to <b>req</b>.
+ */
+static void
+socks_request_set_socks5_error(socks_request_t *req,
+                  socks5_reply_status_t reason)
+{
+  socks5_server_reply_t *trunnel_resp = socks5_server_reply_new();
+  tor_assert(trunnel_resp);
+
+  socks5_server_reply_set_version(trunnel_resp, SOCKS_VER_5);
+  socks5_server_reply_set_reply(trunnel_resp, reason);
+  socks5_server_reply_set_atype(trunnel_resp, 0x01);
+
+  const char *errmsg = socks5_server_reply_check(trunnel_resp);
+  if (errmsg) {
+    log_warn(LD_APP, "socks5: reply validation failed: %s",
+             errmsg);
+    goto end;
+  }
+
+  ssize_t encoded = socks5_server_reply_encode(req->reply,
+                                               sizeof(req->reply),
+                                               trunnel_resp);
+  if (encoded < 0) {
+    log_warn(LD_APP, "socks5: reply encoding failed: %d",
+             (int)encoded);
+  } else {
+    req->replylen = (size_t)encoded;
+  }
+
+  end:
+  socks5_server_reply_free(trunnel_resp);
+}
+
+static const char SOCKS_PROXY_IS_NOT_AN_HTTP_PROXY_MSG[] =
+  "HTTP/1.0 501 Tor is not an HTTP Proxy\r\n"
+  "Content-Type: text/html; charset=iso-8859-1\r\n\r\n"
+  "<html>\n"
+  "<head>\n"
+  "<title>This is a SOCKS Proxy, Not An HTTP Proxy</title>\n"
+  "</head>\n"
+  "<body>\n"
+  "<h1>This is a SOCKs proxy, not an HTTP proxy.</h1>\n"
+  "<p>\n"
+  "It appears you have configured your web browser to use this Tor port as\n"
+  "an HTTP proxy.\n"
+  "</p><p>\n"
+  "This is not correct: This port is configured as a SOCKS proxy, not\n"
+  "an HTTP proxy. If you need an HTTP proxy tunnel, use the HTTPTunnelPort\n"
+  "configuration option in place of, or in addition to, SOCKSPort.\n"
+  "Please configure your client accordingly.\n"
+  "</p>\n"
+  "<p>\n"
+  "See <a href=\"https://www.torproject.org/documentation.html\">"
+  "https://www.torproject.org/documentation.html</a> for more "
+  "information.\n"
+  "</p>\n"
+  "</body>\n"
+  "</html>\n";
+
+/** Implementation helper to implement fetch_from_*_socks.  Instead of looking
+ * at a buffer's contents, we look at the <b>datalen</b> bytes of data in
+ * <b>data</b>. Instead of removing data from the buffer, we set
+ * <b>drain_out</b> to the amount of data that should be removed (or -1 if the
+ * buffer should be cleared).  Instead of pulling more data into the first
+ * chunk of the buffer, we set *<b>want_length_out</b> to the number of bytes
+ * we'd like to see in the input buffer, if they're available. */
+static int
+parse_socks(const char *data, size_t datalen, socks_request_t *req,
+            int log_sockstype, int safe_socks, size_t *drain_out)
+{
+  uint8_t first_octet;
+
+  if (datalen < 2) {
+    /* We always need at least 2 bytes. */
+    return 0;
+  }
+
+  first_octet = get_uint8(data);
+
+  if (first_octet == SOCKS_VER_5 || first_octet == SOCKS_VER_4 ||
+      first_octet == SOCKS_AUTH) { // XXX: RFC 1929
+    return handle_socks_message((const uint8_t *)data, datalen, req,
+                                log_sockstype, safe_socks, drain_out);
+  }
+
+  switch (first_octet) { /* which version of socks? */
+    case 'G': /* get */
+    case 'H': /* head */
+    case 'P': /* put/post */
+    case 'C': /* connect */
+      strlcpy((char*)req->reply, SOCKS_PROXY_IS_NOT_AN_HTTP_PROXY_MSG,
+              MAX_SOCKS_REPLY_LEN);
+      req->replylen = strlen((char*)req->reply)+1;
+      /* fall through */
+    default: /* version is not socks4 or socks5 */
+      log_warn(LD_APP,
+               "Socks version %d not recognized. (This port is not an "
+               "HTTP proxy; did you want to use HTTPTunnelPort?)",
+               *(data));
+      {
+        /* Tell the controller the first 8 bytes. */
+        char *tmp = tor_strndup(data, datalen < 8 ? datalen : 8);
+        control_event_client_status(LOG_WARN,
+                                    "SOCKS_UNKNOWN_PROTOCOL DATA=\"%s\"",
+                                    escaped(tmp));
+        tor_free(tmp);
+      }
+      return -1;
+  }
+
+  tor_assert_unreached();
+  return -1;
+}
+
+/** Inspect a reply from SOCKS server stored in <b>buf</b> according
+ * to <b>state</b>, removing the protocol data upon success. Return 0 on
+ * incomplete response, 1 on success and -1 on error, in which case
+ * <b>reason</b> is set to a descriptive message (free() when finished
+ * with it).
+ *
+ * As a special case, 2 is returned when user/pass is required
+ * during SOCKS5 handshake and user/pass is configured.
+ */
+int
+fetch_from_buf_socks_client(buf_t *buf, int state, char **reason)
+{
+  ssize_t drain = 0;
+  int r;
+  const char *head = NULL;
+  size_t datalen = 0;
+
+  if (buf_datalen(buf) < 2)
+    return 0;
+
+  buf_pullup(buf, MAX_SOCKS_MESSAGE_LEN, &head, &datalen);
+  tor_assert(head && datalen >= 2);
+
+  r = parse_socks_client((uint8_t*)head, datalen,
+                         state, reason, &drain);
+  if (drain > 0)
+    buf_drain(buf, drain);
+  else if (drain < 0)
+    buf_clear(buf);
+
+  return r;
+}
+
+/** Implementation logic for fetch_from_*_socks_client. */
+static int
+parse_socks_client(const uint8_t *data, size_t datalen,
+                   int state, char **reason,
+                   ssize_t *drain_out)
+{
+  unsigned int addrlen;
+  *drain_out = 0;
+  if (datalen < 2)
+    return 0;
+
+  switch (state) {
+    case PROXY_SOCKS4_WANT_CONNECT_OK:
+      /* Wait for the complete response */
+      if (datalen < 8)
+        return 0;
+
+      if (data[1] != 0x5a) {
+        *reason = tor_strdup(socks4_response_code_to_string(data[1]));
+        return -1;
+      }
+
+      /* Success */
+      *drain_out = 8;
+      return 1;
+
+    case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE:
+      /* we don't have any credentials */
+      if (data[1] != 0x00) {
+        *reason = tor_strdup("server doesn't support any of our "
+                             "available authentication methods");
+        return -1;
+      }
+
+      log_info(LD_NET, "SOCKS 5 client: continuing without authentication");
+      *drain_out = -1;
+      return 1;
+
+    case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929:
+      /* we have a username and password. return 1 if we can proceed without
+       * providing authentication, or 2 otherwise. */
+      switch (data[1]) {
+        case 0x00:
+          log_info(LD_NET, "SOCKS 5 client: we have auth details but server "
+                            "doesn't require authentication.");
+          *drain_out = -1;
+          return 1;
+        case 0x02:
+          log_info(LD_NET, "SOCKS 5 client: need authentication.");
+          *drain_out = -1;
+          return 2;
+        /* fall through */
+      }
+
+      *reason = tor_strdup("server doesn't support any of our available "
+                           "authentication methods");
+      return -1;
+
+    case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK:
+      /* handle server reply to rfc1929 authentication */
+      if (data[1] != 0x00) {
+        *reason = tor_strdup("authentication failed");
+        return -1;
+      }
+
+      log_info(LD_NET, "SOCKS 5 client: authentication successful.");
+      *drain_out = -1;
+      return 1;
+
+    case PROXY_SOCKS5_WANT_CONNECT_OK:
+      /* response is variable length. BND.ADDR, etc, isn't needed
+       * (don't bother with buf_pullup()), but make sure to eat all
+       * the data used */
+
+      /* wait for address type field to arrive */
+      if (datalen < 4)
+        return 0;
+
+      switch (data[3]) {
+        case 0x01: /* ip4 */
+          addrlen = 4;
+          break;
+        case 0x04: /* ip6 */
+          addrlen = 16;
+          break;
+        case 0x03: /* fqdn (can this happen here?) */
+          if (datalen < 5)
+            return 0;
+          addrlen = 1 + data[4];
+          break;
+        default:
+          *reason = tor_strdup("invalid response to connect request");
+          return -1;
+      }
+
+      /* wait for address and port */
+      if (datalen < 6 + addrlen)
+        return 0;
+
+      if (data[1] != 0x00) {
+        *reason = tor_strdup(socks5_response_code_to_string(data[1]));
+        return -1;
+      }
+
+      *drain_out = 6 + addrlen;
+      return 1;
+  }
+
+  /* LCOV_EXCL_START */
+  /* shouldn't get here if the input state is one we know about... */
+  tor_assert(0);
+
+  return -1;
+  /* LCOV_EXCL_STOP */
+}
diff --git a/src/core/proto/proto_socks.h b/src/core/proto/proto_socks.h
new file mode 100644
index 0000000000..2a387bf848
--- /dev/null
+++ b/src/core/proto/proto_socks.h
@@ -0,0 +1,21 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PROTO_SOCKS_H
+#define TOR_PROTO_SOCKS_H
+
+struct socks_request_t;
+struct buf_t;
+
+struct socks_request_t *socks_request_new(void);
+void socks_request_free_(struct socks_request_t *req);
+#define socks_request_free(req) \
+  FREE_AND_NULL(socks_request_t, socks_request_free_, (req))
+int fetch_from_buf_socks(struct buf_t *buf, socks_request_t *req,
+                         int log_sockstype, int safe_socks);
+int fetch_from_buf_socks_client(buf_t *buf, int state, char **reason);
+
+#endif /* !defined(TOR_PROTO_SOCKS_H) */
diff --git a/src/ext/OpenBSD_malloc_Linux.c b/src/ext/OpenBSD_malloc_Linux.c
index 855c912310..9c30570c41 100644
--- a/src/ext/OpenBSD_malloc_Linux.c
+++ b/src/ext/OpenBSD_malloc_Linux.c
@@ -59,7 +59,7 @@
 #include <errno.h>
 #include <err.h>
 /* For SIZE_MAX */
-#include "torint.h"
+#include "lib/cc/torint.h"
 
 //#include "thread_private.h"
 
diff --git a/src/ext/byteorder.h b/src/ext/byteorder.h
new file mode 100644
index 0000000000..95e080b14d
--- /dev/null
+++ b/src/ext/byteorder.h
@@ -0,0 +1,71 @@
+/* <MIT License>
+ Copyright (c) 2013-2014  Marek Majkowski <marek@popcount.org>
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+ </MIT License>
+
+ Original location:
+    https://github.com/majek/csiphash/
+
+ Solution inspired by code from:
+    Samuel Neves (supercop/crypto_auth/siphash24/little)
+    djb (supercop/crypto_auth/siphash24/little2)
+    Jean-Philippe Aumasson (https://131002.net/siphash/siphash24.c)
+*/
+
+#ifdef HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
+
+/* This code is extracted from csiphash.h */
+
+#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) &&      \
+	__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#  define _le64toh(x) ((uint64_t)(x))
+#elif defined(_WIN32)
+/* Windows is always little endian, unless you're on xbox360
+   http://msdn.microsoft.com/en-us/library/b0084kay(v=vs.80).aspx */
+#  define _le64toh(x) ((uint64_t)(x))
+#elif defined(__APPLE__)
+#  include <libkern/OSByteOrder.h>
+#  define _le64toh(x) OSSwapLittleToHostInt64(x)
+#elif defined(sun) || defined(__sun)
+#  include <sys/byteorder.h>
+#  define _le64toh(x) LE_64(x)
+
+#else
+
+/* See: http://sourceforge.net/p/predef/wiki/Endianness/ */
+#  if defined(__FreeBSD__) || defined(__NetBSD__) || defined(OpenBSD)
+#    include <sys/endian.h>
+#  else
+#    include <endian.h>
+#  endif
+#  if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && \
+	__BYTE_ORDER == __LITTLE_ENDIAN
+#    define _le64toh(x) ((uint64_t)(x))
+#  else
+#    if defined(OpenBSD)
+#      define _le64toh(x) letoh64(x)
+#    else
+#      define _le64toh(x) le64toh(x)
+#    endif
+#  endif
+
+#endif
diff --git a/src/ext/csiphash.c b/src/ext/csiphash.c
index 49a6dc4778..a6a9846db4 100644
--- a/src/ext/csiphash.c
+++ b/src/ext/csiphash.c
@@ -29,47 +29,13 @@
     Jean-Philippe Aumasson (https://131002.net/siphash/siphash24.c)
 */
 
-#include "torint.h"
+#include "lib/cc/torint.h"
+#include "lib/log/util_bug.h"
+
 #include "siphash.h"
-/* for tor_assert */
-#include "util.h"
-/* for memcpy */
 #include <string.h>
-
-#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \
-	__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-#  define _le64toh(x) ((uint64_t)(x))
-#elif defined(_WIN32)
-/* Windows is always little endian, unless you're on xbox360
-   http://msdn.microsoft.com/en-us/library/b0084kay(v=vs.80).aspx */
-#  define _le64toh(x) ((uint64_t)(x))
-#elif defined(__APPLE__)
-#  include <libkern/OSByteOrder.h>
-#  define _le64toh(x) OSSwapLittleToHostInt64(x)
-#elif defined(sun) || defined(__sun)
-#  include <sys/byteorder.h>
-#  define _le64toh(x) LE_64(x)
-
-#else
-
-/* See: http://sourceforge.net/p/predef/wiki/Endianness/ */
-#  if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)
-#    include <sys/endian.h>
-#  else
-#    include <endian.h>
-#  endif
-#  if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && \
-	__BYTE_ORDER == __LITTLE_ENDIAN
-#    define _le64toh(x) ((uint64_t)(x))
-#  else
-#    if defined(__OpenBSD__)
-#      define _le64toh(x) letoh64(x)
-#    else
-#      define _le64toh(x) le64toh(x)
-#    endif
-#  endif
-
-#endif
+#include <stdlib.h>
+#include "byteorder.h"
 
 #define ROTATE(x, b) (uint64_t)( ((x) << (b)) | ( (x) >> (64 - (b))) )
 
@@ -157,3 +123,9 @@ void siphash_set_global_key(const struct sipkey *key)
 	the_siphash_key.k1 = key->k1;
 	the_siphash_key_is_set = 1;
 }
+
+void siphash_unset_global_key(void)
+{
+	the_siphash_key_is_set = 0;
+	memset(&the_siphash_key, 0, sizeof(the_siphash_key));
+}
diff --git a/src/ext/curve25519_donna/curve25519-donna-c64.c b/src/ext/curve25519_donna/curve25519-donna-c64.c
index b68ff3695a..45da7bf1e6 100644
--- a/src/ext/curve25519_donna/curve25519-donna-c64.c
+++ b/src/ext/curve25519_donna/curve25519-donna-c64.c
@@ -25,7 +25,7 @@
 #include "orconfig.h"
 
 #include <string.h>
-#include "torint.h"
+#include "lib/cc/torint.h"
 
 typedef uint8_t u8;
 typedef uint64_t limb;
diff --git a/src/ext/curve25519_donna/curve25519-donna.c b/src/ext/curve25519_donna/curve25519-donna.c
index 1c5a27ab8a..d64b95c113 100644
--- a/src/ext/curve25519_donna/curve25519-donna.c
+++ b/src/ext/curve25519_donna/curve25519-donna.c
@@ -48,7 +48,7 @@
 #include "orconfig.h"
 
 #include <string.h>
-#include "torint.h"
+#include "lib/cc/torint.h"
 
 typedef uint8_t u8;
 typedef int32_t s32;
diff --git a/src/ext/ed25519/donna/ed25519-donna-impl-base.h b/src/ext/ed25519/donna/ed25519-donna-impl-base.h
index 48913edcb4..67b3b49b96 100644
--- a/src/ext/ed25519/donna/ed25519-donna-impl-base.h
+++ b/src/ext/ed25519/donna/ed25519-donna-impl-base.h
@@ -6,15 +6,15 @@ DONNA_INLINE static void
 ge25519_p1p1_to_partial(ge25519 *r, const ge25519_p1p1 *p) {
 	curve25519_mul(r->x, p->x, p->t);
 	curve25519_mul(r->y, p->y, p->z);
-	curve25519_mul(r->z, p->z, p->t); 
+	curve25519_mul(r->z, p->z, p->t);
 }
 
 DONNA_INLINE static void
 ge25519_p1p1_to_full(ge25519 *r, const ge25519_p1p1 *p) {
 	curve25519_mul(r->x, p->x, p->t);
 	curve25519_mul(r->y, p->y, p->z);
-	curve25519_mul(r->z, p->z, p->t); 
-	curve25519_mul(r->t, p->x, p->y); 
+	curve25519_mul(r->z, p->z, p->t);
+	curve25519_mul(r->t, p->x, p->y);
 }
 
 static void
@@ -249,7 +249,7 @@ ge25519_unpack_negative_vartime(ge25519 *r, const unsigned char p[32]) {
 #define S2_TABLE_SIZE (1<<(S2_SWINDOWSIZE-2))
 
 /* computes [s1]p1 + [s2]basepoint */
-static void 
+static void
 ge25519_double_scalarmult_vartime(ge25519 *r, const ge25519 *p1, const bignum256modm s1, const bignum256modm s2) {
 	signed char slide1[256], slide2[256];
 	ge25519_pniels pre1[S1_TABLE_SIZE];
@@ -336,6 +336,7 @@ ge25519_scalarmult_base_niels(ge25519 *r, const uint8_t basepoint_table[256][96]
 	signed char b[64];
 	uint32_t i;
 	ge25519_niels t;
+        memset(&t, 0, sizeof(t));
 
 	contract256_window4_modm(b, s);
 
@@ -344,7 +345,7 @@ ge25519_scalarmult_base_niels(ge25519 *r, const uint8_t basepoint_table[256][96]
 	curve25519_add_reduce(r->y, t.xaddy, t.ysubx);
 	memset(r->z, 0, sizeof(bignum25519));
 	curve25519_copy(r->t, t.t2d);
-	r->z[0] = 2;	
+	r->z[0] = 2;
 	for (i = 3; i < 64; i += 2) {
 		ge25519_scalarmult_base_choose_niels(&t, basepoint_table, i / 2, b[i]);
 		ge25519_nielsadd2(r, &t);
@@ -361,4 +362,3 @@ ge25519_scalarmult_base_niels(ge25519 *r, const uint8_t basepoint_table[256][96]
 		ge25519_nielsadd2(r, &t);
 	}
 }
-
diff --git a/src/ext/ed25519/donna/ed25519-donna-portable-identify.h b/src/ext/ed25519/donna/ed25519-donna-portable-identify.h
index 26a264cf9e..3e0f9cfc50 100644
--- a/src/ext/ed25519/donna/ed25519-donna-portable-identify.h
+++ b/src/ext/ed25519/donna/ed25519-donna-portable-identify.h
@@ -14,7 +14,7 @@
 			#define OS_OSX
 		#elif defined(macintosh) || defined(Macintosh)
 			#define OS_MAC
-		#elif defined(__OpenBSD__)
+		#elif defined(OpenBSD)
 			#define OS_OPENBSD
 		#endif
 	#endif
diff --git a/src/ext/ed25519/donna/ed25519-hash-custom.h b/src/ext/ed25519/donna/ed25519-hash-custom.h
index 7dc249129d..ff8bbde3da 100644
--- a/src/ext/ed25519/donna/ed25519-hash-custom.h
+++ b/src/ext/ed25519/donna/ed25519-hash-custom.h
@@ -9,3 +9,34 @@
 	void ed25519_hash(uint8_t *hash, const uint8_t *in, size_t inlen);
 */
 
+#include "lib/crypt_ops/crypto_digest.h"
+
+typedef struct ed25519_hash_context {
+  crypto_digest_t *ctx;
+} ed25519_hash_context;
+
+
+static void
+ed25519_hash_init(ed25519_hash_context *ctx)
+{
+  ctx->ctx = crypto_digest512_new(DIGEST_SHA512);
+}
+static void
+ed25519_hash_update(ed25519_hash_context *ctx, const uint8_t *in, size_t inlen)
+{
+  crypto_digest_add_bytes(ctx->ctx, (const char *)in, inlen);
+}
+static void
+ed25519_hash_final(ed25519_hash_context *ctx, uint8_t *hash)
+{
+  crypto_digest_get_digest(ctx->ctx, (char *)hash, DIGEST512_LEN);
+  crypto_digest_free(ctx->ctx);
+  ctx->ctx = NULL;
+}
+static void
+ed25519_hash(uint8_t *hash, const uint8_t *in, size_t inlen)
+{
+  crypto_digest512((char *)hash, (const char *)in, inlen,
+		   DIGEST_SHA512);
+}
+
diff --git a/src/ext/ed25519/donna/ed25519-randombytes-custom.h b/src/ext/ed25519/donna/ed25519-randombytes-custom.h
index 3fb0959fc4..d92a51d1d3 100644
--- a/src/ext/ed25519/donna/ed25519-randombytes-custom.h
+++ b/src/ext/ed25519/donna/ed25519-randombytes-custom.h
@@ -8,7 +8,7 @@
 */
 
 /* Tor: Instead of calling OpenSSL's CSPRNG directly, call the wrapper. */
-#include "crypto.h"
+#include "lib/crypt_ops/crypto_rand.h"
 
 static void
 ED25519_FN(ed25519_randombytes_unsafe) (void *p, size_t len)
diff --git a/src/ext/ed25519/donna/ed25519_donna_tor.h b/src/ext/ed25519/donna/ed25519_donna_tor.h
index d225407b1c..20e9b5e99c 100644
--- a/src/ext/ed25519/donna/ed25519_donna_tor.h
+++ b/src/ext/ed25519/donna/ed25519_donna_tor.h
@@ -1,7 +1,7 @@
 /* Added for Tor. */
 #ifndef SRC_EXT_ED25519_DONNA_H_INCLUDED_
 #define SRC_EXT_ED25519_DONNA_H_INCLUDED_
-#include <torint.h>
+#include "lib/cc/torint.h"
 
 typedef unsigned char curved25519_key[32];
 
@@ -30,4 +30,9 @@ int ed25519_donna_blind_public_key(unsigned char *out, const unsigned char *inp,
 int ed25519_donna_pubkey_from_curve25519_pubkey(unsigned char *out,
   const unsigned char *inp, int signbit);
 
+
+int
+ed25519_donna_scalarmult_with_group_order(unsigned char *out,
+                                          const unsigned char *pubkey);
+
 #endif
diff --git a/src/ext/ed25519/donna/ed25519_tor.c b/src/ext/ed25519/donna/ed25519_tor.c
index 9537ae66a1..7f5ab398d8 100644
--- a/src/ext/ed25519/donna/ed25519_tor.c
+++ b/src/ext/ed25519/donna/ed25519_tor.c
@@ -40,6 +40,8 @@
 #include "ed25519-randombytes.h"
 #include "ed25519-hash.h"
 
+#include "lib/crypt_ops/crypto_util.h"
+
 typedef unsigned char ed25519_signature[64];
 typedef unsigned char ed25519_public_key[32];
 typedef unsigned char ed25519_secret_key[32];
@@ -132,7 +134,7 @@ ED25519_FN(curved25519_scalarmult_basepoint) (curved25519_key pk, const curved25
 }
 
 /*
-   Tor has a specific idea of how an Ed25519 implementaion should behave.
+   Tor has a specific idea of how an Ed25519 implementation should behave.
    Implement such a beast using the ed25519-donna primitives/internals.
 
     * Private key generation using Tor's CSPRNG.
@@ -245,13 +247,7 @@ ed25519_donna_sign(unsigned char *sig, const unsigned char *m, size_t mlen,
 static void
 ed25519_donna_gettweak(unsigned char *out, const unsigned char *param)
 {
-  static const char str[] = "Derive temporary signing key";
-  ed25519_hash_context ctx;
-
-  ed25519_hash_init(&ctx);
-  ed25519_hash_update(&ctx, (const unsigned char*)str, strlen(str));
-  ed25519_hash_update(&ctx, param, 32);
-  ed25519_hash_final(&ctx, out);
+  memcpy(out, param, 32);
 
   out[0] &= 248;  /* Is this necessary ? */
   out[31] &= 63;
@@ -304,7 +300,9 @@ ed25519_donna_blind_public_key(unsigned char *out, const unsigned char *inp,
   /* No "ge25519_unpack", negate the public key. */
   memcpy(pkcopy, inp, 32);
   pkcopy[31] ^= (1<<7);
-  ge25519_unpack_negative_vartime(&A, pkcopy);
+  if (!ge25519_unpack_negative_vartime(&A, pkcopy)) {
+    return -1;
+  }
 
   /* A' = [tweak] * A + [0] * basepoint. */
   ge25519_double_scalarmult_vartime(&Aprime, &A, t, zero);
@@ -340,5 +338,32 @@ ed25519_donna_pubkey_from_curve25519_pubkey(unsigned char *out,
   return 0;
 }
 
+/* Do the scalar multiplication of <b>pubkey</b> with the group order
+ * <b>modm_m</b>.  Place the result in <b>out</b> which must be at least 32
+ * bytes long. */
+int
+ed25519_donna_scalarmult_with_group_order(unsigned char *out,
+                                          const unsigned char *pubkey)
+{
+  static const bignum256modm ALIGN(16) zero = { 0 };
+  unsigned char pkcopy[32];
+  ge25519 ALIGN(16) Point, Result;
+
+  /* No "ge25519_unpack", negate the public key and unpack it back.
+   * See ed25519_donna_blind_public_key() */
+  memcpy(pkcopy, pubkey, 32);
+  pkcopy[31] ^= (1<<7);
+  if (!ge25519_unpack_negative_vartime(&Point, pkcopy)) {
+    return -1; /* error: bail out */
+  }
+
+  /* There is no regular scalarmult function so we have to do:
+   * Result = l*P + 0*B */
+  ge25519_double_scalarmult_vartime(&Result, &Point, modm_m, zero);
+  ge25519_pack(out, &Result);
+
+  return 0;
+}
+
 #include "test-internals.c"
 
diff --git a/src/ext/ed25519/ref10/blinding.c b/src/ext/ed25519/ref10/blinding.c
index ee3e8666fa..8485524e5d 100644
--- a/src/ext/ed25519/ref10/blinding.c
+++ b/src/ext/ed25519/ref10/blinding.c
@@ -7,13 +7,13 @@
 #include "ed25519_ref10.h"
 
 #include <string.h>
-#include "crypto.h"
+#include "lib/crypt_ops/crypto_util.h"
 
 static void
 ed25519_ref10_gettweak(unsigned char *out, const unsigned char *param)
 {
-  const char str[] = "Derive temporary signing key";
-  crypto_hash_sha512_2(out, (const unsigned char*)str, strlen(str), param, 32);
+  memcpy(out, param, 32);
+
   out[0] &= 248;  /* Is this necessary necessary ? */
   out[31] &= 63;
   out[31] |= 64;
@@ -49,6 +49,7 @@ int ed25519_ref10_blind_public_key(unsigned char *out,
   unsigned char pkcopy[32];
   ge_p3 A;
   ge_p2 Aprime;
+  int retval = -1;
 
   ed25519_ref10_gettweak(tweak, param);
 
@@ -62,15 +63,57 @@ int ed25519_ref10_blind_public_key(unsigned char *out,
    * "ge_frombytes", we'd use that, but there isn't. */
   memcpy(pkcopy, inp, 32);
   pkcopy[31] ^= (1<<7);
-  ge_frombytes_negate_vartime(&A, pkcopy);
+  if (ge_frombytes_negate_vartime(&A, pkcopy) != 0) {
+    goto done;
+  }
   /* There isn't a regular ge_scalarmult -- we have to do tweak*A + zero*B. */
   ge_double_scalarmult_vartime(&Aprime, tweak, &A, zero);
   ge_tobytes(out, &Aprime);
 
+  retval = 0;
+
+ done:
   memwipe(tweak, 0, sizeof(tweak));
   memwipe(&A, 0, sizeof(A));
   memwipe(&Aprime, 0, sizeof(Aprime));
   memwipe(pkcopy, 0, sizeof(pkcopy));
 
+  return retval;
+}
+
+/* This is the group order encoded in a format that
+ * ge_double_scalarmult_vartime() understands. The group order m is:
+ * m = 	 2^252 +  27742317777372353535851937790883648493 =
+ *       0x1000000000000000000000000000000014def9dea2f79cd65812631a5cf5d3ed
+ */
+static const uint8_t modm_m[32] = {0xed,0xd3,0xf5,0x5c,0x1a,0x63,0x12,0x58,
+                                   0xd6,0x9c,0xf7,0xa2,0xde,0xf9,0xde,0x14,
+                                   0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+                                   0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x10};
+
+/* Do the scalar multiplication of <b>pubkey</b> with the group order
+ * <b>modm_m</b>.  Place the result in <b>out</b> which must be at least 32
+ * bytes long. */
+int
+ed25519_ref10_scalarmult_with_group_order(unsigned char *out,
+                                          const unsigned char *pubkey)
+{
+  unsigned char pkcopy[32];
+  unsigned char zero[32] = {0};
+  ge_p3 Point;
+  ge_p2 Result;
+
+  /* All this is done to fit 'pubkey' in 'Point' so that it can be used by
+   * ed25519 ref code. Same thing as in blinding function */
+  memcpy(pkcopy, pubkey, 32);
+  pkcopy[31] ^= (1<<7);
+  if (ge_frombytes_negate_vartime(&Point, pkcopy) != 0) {
+    return -1; /* error: bail out */
+  }
+
+  /* There isn't a regular scalarmult -- we have to do r = l*P + 0*B */
+  ge_double_scalarmult_vartime(&Result, modm_m, &Point, zero);
+  ge_tobytes(out, &Result);
+
   return 0;
 }
diff --git a/src/ext/ed25519/ref10/crypto_hash_sha512.h b/src/ext/ed25519/ref10/crypto_hash_sha512.h
index 0278571522..25e6a90cec 100644
--- a/src/ext/ed25519/ref10/crypto_hash_sha512.h
+++ b/src/ext/ed25519/ref10/crypto_hash_sha512.h
@@ -1,30 +1,32 @@
 /* Added for Tor. */
-#include <openssl/sha.h>
+#include "lib/crypt_ops/crypto_digest.h"
 
 /* Set 'out' to the 512-bit SHA512 hash of the 'len'-byte string in 'inp' */
 #define crypto_hash_sha512(out, inp, len) \
-  SHA512((inp), (len), (out))
+  crypto_digest512((char *)(out), (const char *)(inp), (len), DIGEST_SHA512)
 
 /* Set 'out' to the 512-bit SHA512 hash of the 'len1'-byte string in 'inp1',
  * concatenated with the 'len2'-byte string in 'inp2'. */
 #define crypto_hash_sha512_2(out, inp1, len1, inp2, len2)               \
   do {                                                                  \
-      SHA512_CTX sha_ctx_;                                              \
-      SHA512_Init(&sha_ctx_);                                           \
-      SHA512_Update(&sha_ctx_, (inp1), (len1));                         \
-      SHA512_Update(&sha_ctx_, (inp2), (len2));                         \
-      SHA512_Final((out), &sha_ctx_);                                   \
- } while(0)
+    crypto_digest_t *sha_ctx_;						\
+    sha_ctx_ = crypto_digest512_new(DIGEST_SHA512);			\
+    crypto_digest_add_bytes(sha_ctx_, (const char *)(inp1), (len1));	\
+    crypto_digest_add_bytes(sha_ctx_, (const char *)(inp2), (len2));	\
+    crypto_digest_get_digest(sha_ctx_, (char *)out, DIGEST512_LEN);	\
+    crypto_digest_free(sha_ctx_);					\
+  } while (0)
 
 /* Set 'out' to the 512-bit SHA512 hash of the 'len1'-byte string in 'inp1',
  * concatenated with the 'len2'-byte string in 'inp2', concatenated with
  * the 'len3'-byte string in 'len3'. */
 #define crypto_hash_sha512_3(out, inp1, len1, inp2, len2, inp3, len3)   \
   do {                                                                  \
-      SHA512_CTX sha_ctx_;                                              \
-      SHA512_Init(&sha_ctx_);                                           \
-      SHA512_Update(&sha_ctx_, (inp1), (len1));                         \
-      SHA512_Update(&sha_ctx_, (inp2), (len2));                         \
-      SHA512_Update(&sha_ctx_, (inp3), (len3));                         \
-      SHA512_Final((out), &sha_ctx_);                                   \
+    crypto_digest_t *sha_ctx_;						\
+    sha_ctx_ = crypto_digest512_new(DIGEST_SHA512);			\
+    crypto_digest_add_bytes(sha_ctx_, (const char *)(inp1), (len1));	\
+    crypto_digest_add_bytes(sha_ctx_, (const char *)(inp2), (len2));	\
+    crypto_digest_add_bytes(sha_ctx_, (const char *)(inp3), (len3));	\
+    crypto_digest_get_digest(sha_ctx_, (char *)out, DIGEST512_LEN);	\
+    crypto_digest_free(sha_ctx_);					\
  } while(0)
diff --git a/src/ext/ed25519/ref10/crypto_int32.h b/src/ext/ed25519/ref10/crypto_int32.h
index dd13c91bd0..26271e917b 100644
--- a/src/ext/ed25519/ref10/crypto_int32.h
+++ b/src/ext/ed25519/ref10/crypto_int32.h
@@ -3,7 +3,7 @@
 #ifndef CRYPTO_INT32_H
 #define CRYPTO_INT32_H
 
-#include "torint.h"
+#include "lib/cc/torint.h"
 #define crypto_int32 int32_t
 #define crypto_uint32 uint32_t
 
diff --git a/src/ext/ed25519/ref10/crypto_int64.h b/src/ext/ed25519/ref10/crypto_int64.h
index 46e8852ed0..3b066a9c0c 100644
--- a/src/ext/ed25519/ref10/crypto_int64.h
+++ b/src/ext/ed25519/ref10/crypto_int64.h
@@ -3,7 +3,7 @@
 #ifndef CRYPTO_INT64_H
 #define CRYPTO_INT64_H
 
-#include "torint.h"
+#include "lib/cc/torint.h"
 #define crypto_int64 int64_t
 #define crypto_uint64 uint64_t
 
diff --git a/src/ext/ed25519/ref10/crypto_uint32.h b/src/ext/ed25519/ref10/crypto_uint32.h
index 62655a5b66..a7a77723bd 100644
--- a/src/ext/ed25519/ref10/crypto_uint32.h
+++ b/src/ext/ed25519/ref10/crypto_uint32.h
@@ -1,3 +1,3 @@
 /* Added for Tor. */
-#include "torint.h"
+#include "lib/cc/torint.h"
 #define crypto_uint32 uint32_t
diff --git a/src/ext/ed25519/ref10/crypto_uint64.h b/src/ext/ed25519/ref10/crypto_uint64.h
index cbda882a6a..adaaa08042 100644
--- a/src/ext/ed25519/ref10/crypto_uint64.h
+++ b/src/ext/ed25519/ref10/crypto_uint64.h
@@ -1,3 +1,3 @@
 /* Added for Tor. */
-#include "torint.h"
+#include "lib/cc/torint.h"
 #define crypto_uint64 uint64_t
diff --git a/src/ext/ed25519/ref10/crypto_verify_32.h b/src/ext/ed25519/ref10/crypto_verify_32.h
index 0f63efc7a3..5299928754 100644
--- a/src/ext/ed25519/ref10/crypto_verify_32.h
+++ b/src/ext/ed25519/ref10/crypto_verify_32.h
@@ -1,5 +1,4 @@
 /* Added for Tor. */
-#include "di_ops.h"
+#include "lib/ctime/di_ops.h"
 #define crypto_verify_32(a,b) \
   (! tor_memeq((a), (b), 32))
-
diff --git a/src/ext/ed25519/ref10/ed25519_ref10.h b/src/ext/ed25519/ref10/ed25519_ref10.h
index af7e21a2ad..bb72af6c0b 100644
--- a/src/ext/ed25519/ref10/ed25519_ref10.h
+++ b/src/ext/ed25519/ref10/ed25519_ref10.h
@@ -1,7 +1,7 @@
 /* Added for Tor */
 #ifndef SRC_EXT_ED25519_REF10_H_INCLUDED_
 #define SRC_EXT_ED25519_REF10_H_INCLUDED_
-#include <torint.h>
+#include "lib/cc/torint.h"
 
 int ed25519_ref10_seckey(unsigned char *sk);
 int ed25519_ref10_seckey_expand(unsigned char *sk, const unsigned char *sk_seed);
@@ -27,4 +27,8 @@ int ed25519_ref10_blind_public_key(unsigned char *out,
                               const unsigned char *inp,
                               const unsigned char *param);
 
+int
+ed25519_ref10_scalarmult_with_group_order(unsigned char *out,
+                                          const unsigned char *pubkey);
+
 #endif
diff --git a/src/ext/ed25519/ref10/keypair.c b/src/ext/ed25519/ref10/keypair.c
index 68a88f9adc..a6e2d4c781 100644
--- a/src/ext/ed25519/ref10/keypair.c
+++ b/src/ext/ed25519/ref10/keypair.c
@@ -6,6 +6,9 @@
 #include "crypto_hash_sha512.h"
 #include "ge.h"
 
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+
 int
 crypto_sign_seckey(unsigned char *sk)
 {
@@ -49,4 +52,3 @@ int crypto_sign_keypair(unsigned char *pk,unsigned char *sk)
 
   return 0;
 }
-
diff --git a/src/ext/ed25519/ref10/randombytes.h b/src/ext/ed25519/ref10/randombytes.h
index 8bf31631f0..c2cef10ceb 100644
--- a/src/ext/ed25519/ref10/randombytes.h
+++ b/src/ext/ed25519/ref10/randombytes.h
@@ -1,4 +1,4 @@
 /* Added for Tor. */
-#include "crypto.h"
+#include "lib/crypt_ops/crypto_rand.h"
 #define randombytes(b, n) \
   (crypto_strongest_rand((b), (n)), 0)
diff --git a/src/ext/getdelim.c b/src/ext/getdelim.c
new file mode 100644
index 0000000000..1c29baffd9
--- /dev/null
+++ b/src/ext/getdelim.c
@@ -0,0 +1,79 @@
+/*	$NetBSD: getdelim.c,v 1.2 2015/12/25 20:12:46 joerg Exp $	*/
+/*	NetBSD-src: getline.c,v 1.2 2014/09/16 17:23:50 christos Exp 	*/
+
+/*-
+ * Copyright (c) 2011 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Christos Zoulas.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef BUFSIZ
+#define BUFSIZ 512
+#endif
+
+ssize_t
+compat_getdelim_(char **buf, size_t *bufsiz, int delimiter, FILE *fp)
+{
+	char *ptr, *eptr;
+
+
+	if (*buf == NULL || *bufsiz == 0) {
+		*bufsiz = BUFSIZ;
+		if ((*buf = raw_malloc(*bufsiz)) == NULL)
+			return -1;
+	}
+
+	for (ptr = *buf, eptr = *buf + *bufsiz;;) {
+		int c = fgetc(fp);
+		if (c == -1) {
+			if (feof(fp)) {
+				ssize_t diff = (ssize_t)(ptr - *buf);
+				if (diff != 0) {
+					*ptr = '\0';
+					return diff;
+				}
+			}
+			return -1;
+		}
+		*ptr++ = c;
+		if (c == delimiter) {
+			*ptr = '\0';
+			return ptr - *buf;
+		}
+		if (ptr + 2 >= eptr) {
+			char *nbuf;
+			size_t nbufsiz = *bufsiz * 2;
+			ssize_t d = ptr - *buf;
+			if (nbufsiz < *bufsiz ||
+			    (nbuf = raw_realloc(*buf, nbufsiz)) == NULL)
+				return -1;
+			*buf = nbuf;
+			*bufsiz = nbufsiz;
+			eptr = nbuf + nbufsiz;
+			ptr = nbuf + d;
+		}
+	}
+}
diff --git a/src/ext/ht.h b/src/ext/ht.h
index a441d0b685..54e5eb7cba 100644
--- a/src/ext/ht.h
+++ b/src/ext/ht.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2002, Christopher Clark.
  * Copyright (c) 2005-2006, Nick Mathewson.
- * Copyright (c) 2007-2015, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See license at end. */
 
 /* Based on ideas by Christopher Clark and interfaces from Niels Provos. */
@@ -150,6 +150,8 @@
 #define HT_CLEAR(name, head)         name##_HT_CLEAR(head)
 #define HT_INIT(name, head)          name##_HT_INIT(head)
 #define HT_REP_IS_BAD_(name, head)    name##_HT_REP_IS_BAD_(head)
+#define HT_FOREACH_FN(name, head, fn, data) \
+   name##_HT_FOREACH_FN((head), (fn), (data))
 /* Helper: */
 static inline unsigned
 ht_improve_hash(unsigned h)
diff --git a/src/ext/include.am b/src/ext/include.am
index f00f3e031e..6bdce2d79e 100644
--- a/src/ext/include.am
+++ b/src/ext/include.am
@@ -5,10 +5,12 @@ EXTRA_DIST += src/ext/README
 
 EXTHEADERS = \
   src/ext/ht.h		\
+  src/ext/byteorder.h   \
   src/ext/tinytest.h	\
   src/ext/tor_readpassphrase.h \
   src/ext/strlcat.c	\
   src/ext/strlcpy.c	\
+  src/ext/getdelim.c    \
   src/ext/tinytest_macros.h \
   src/ext/tor_queue.h	\
   src/ext/siphash.h \
@@ -100,6 +102,7 @@ noinst_LIBRARIES += $(LIBED25519_REF10)
 src_ext_ed25519_donna_libed25519_donna_a_CFLAGS=\
   @CFLAGS_CONSTTIME@ \
   -DED25519_CUSTOMRANDOM \
+  -DED25519_CUSTOMHASH \
   -DED25519_SUFFIX=_donna
 
 src_ext_ed25519_donna_libed25519_donna_a_SOURCES= \
@@ -171,4 +174,3 @@ EXTRA_DIST += \
 	src/ext/timeouts/Makefile			\
 	src/ext/timeouts/Rules.shrc			\
 	src/ext/timeouts/test-timeout.c
-
diff --git a/src/ext/keccak-tiny/keccak-tiny-unrolled.c b/src/ext/keccak-tiny/keccak-tiny-unrolled.c
index d1342c3601..05cf0ec3f0 100644
--- a/src/ext/keccak-tiny/keccak-tiny-unrolled.c
+++ b/src/ext/keccak-tiny/keccak-tiny-unrolled.c
@@ -9,29 +9,22 @@
 #include "keccak-tiny.h"
 
 #include <string.h>
-#include "crypto.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "byteorder.h"
 
 /******** Endianness conversion helpers ********/
 
 static inline uint64_t
 loadu64le(const unsigned char *x) {
-  uint64_t r = 0;
-  size_t i;
-
-  for (i = 0; i < 8; ++i) {
-    r |= (uint64_t)x[i] << 8 * i;
-  }
-  return r;
+  uint64_t r;
+  memcpy(&r, x, sizeof(r));
+  return _le64toh(r);
 }
 
 static inline void
 storeu64le(uint8_t *x, uint64_t u) {
-  size_t i;
-
-  for(i=0; i<8; ++i) {
-    x[i] = u;
-    u >>= 8;
-  }
+  uint64_t val = _le64toh(u);
+  memcpy(x, &val, sizeof(u));
 }
 
 /******** The Keccak-f[1600] permutation ********/
diff --git a/src/ext/keccak-tiny/keccak-tiny.h b/src/ext/keccak-tiny/keccak-tiny.h
index 7efea2319e..a9c8ed6420 100644
--- a/src/ext/keccak-tiny/keccak-tiny.h
+++ b/src/ext/keccak-tiny/keccak-tiny.h
@@ -2,7 +2,7 @@
 #define KECCAK_FIPS202_H
 
 #include <stddef.h>
-#include "torint.h"
+#include "lib/cc/torint.h"
 
 #define KECCAK_MAX_RATE 200
 
diff --git a/src/ext/mulodi/mulodi4.c b/src/ext/mulodi/mulodi4.c
index 9891bbf1af..accce1ce01 100644
--- a/src/ext/mulodi/mulodi4.c
+++ b/src/ext/mulodi/mulodi4.c
@@ -18,7 +18,7 @@
 #define COMPILER_RT_ABI
 #define di_int int64_t
 #define di_uint uint64_t
-#include "torint.h"
+#include "lib/cc/torint.h"
 
 di_int __mulodi4(di_int a, di_int b, int* overflow);
 #endif
diff --git a/src/ext/rust b/src/ext/rust
new file mode 160000
+Subproject aa37fb84fb829902e83ca11a7244bbc6b86b809
diff --git a/src/ext/siphash.h b/src/ext/siphash.h
index d9b34b8980..730e49937d 100644
--- a/src/ext/siphash.h
+++ b/src/ext/siphash.h
@@ -9,5 +9,6 @@ uint64_t siphash24(const void *src, unsigned long src_sz, const struct sipkey *k
 
 void siphash_set_global_key(const struct sipkey *key);
 uint64_t siphash24g(const void *src, unsigned long src_sz);
+void siphash_unset_global_key(void);
 
 #endif
diff --git a/src/ext/timeouts/timeout-bitops.c b/src/ext/timeouts/timeout-bitops.c
index 45466f6cb3..68db817933 100644
--- a/src/ext/timeouts/timeout-bitops.c
+++ b/src/ext/timeouts/timeout-bitops.c
@@ -231,7 +231,8 @@ main(int c, char **v)
 	int result = 0;
 
 	for (i = 0; i <= 63; ++i) {
-		uint64_t x = 1 << i;
+		uint64_t x = 1;
+		x <<= i;
 		if (!check(x))
 			result = 1;
 		--x;
diff --git a/src/ext/timeouts/timeout.c b/src/ext/timeouts/timeout.c
index 713ec219ce..d4b514d2c5 100644
--- a/src/ext/timeouts/timeout.c
+++ b/src/ext/timeouts/timeout.c
@@ -150,7 +150,7 @@
 #else
 #define ctz(n) ctz32(n)
 #define clz(n) clz32(n)
-#define fls(n) ((int)(32 - clz32(n)))
+#define fls(n) ((int)(32 - clz32((uint32_t)n)))
 #endif
 
 #if WHEEL_BIT == 6
@@ -432,7 +432,7 @@ TIMEOUT_PUBLIC void timeouts_update(struct timeouts *T, abstime_t curtime) {
 			 * or can be replaced with a simpler operation.
 			 */
 			oslot = WHEEL_MASK & (T->curtime >> (wheel * WHEEL_BIT));
-			pending = rotl(((UINT64_C(1) << _elapsed) - 1), oslot);
+			pending = rotl(((WHEEL_C(1) << _elapsed) - 1), oslot);
 
 			nslot = WHEEL_MASK & (curtime >> (wheel * WHEEL_BIT));
 			pending |= rotr(rotl(((WHEEL_C(1) << _elapsed) - 1), nslot), (int)_elapsed);
diff --git a/src/ext/tinytest.c b/src/ext/tinytest.c
index a51cd6011a..16f11e4639 100644
--- a/src/ext/tinytest.c
+++ b/src/ext/tinytest.c
@@ -25,6 +25,7 @@
 #ifdef TINYTEST_LOCAL
 #include "tinytest_local.h"
 #endif
+#define TINYTEST_POSTFORK
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -118,6 +119,14 @@ testcase_run_bare_(const struct testcase_t *testcase)
 
 #ifndef NO_FORKING
 
+#ifdef TINYTEST_POSTFORK
+void tinytest_prefork(void);
+void tinytest_postfork(void);
+#else
+static void tinytest_prefork(void) { }
+static void tinytest_postfork(void) { }
+#endif
+
 static enum outcome
 testcase_run_forked_(const struct testgroup_t *group,
 		     const struct testcase_t *testcase)
@@ -145,7 +154,7 @@ testcase_run_forked_(const struct testgroup_t *group,
 	if (opt_verbosity>0)
 		printf("[forking] ");
 
-	snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s",
+	snprintf(buffer, sizeof(buffer), "\"%s\" --RUNNING-FORKED %s %s%s",
 		 commandname, verbosity_flag, group->prefix, testcase->name);
 
 	memset(&si, 0, sizeof(si));
@@ -178,10 +187,12 @@ testcase_run_forked_(const struct testgroup_t *group,
 
 	if (opt_verbosity>0)
 		printf("[forking] ");
+	tinytest_prefork();
 	pid = fork();
 #ifdef FORK_BREAKS_GCOV
 	vproc_transaction_begin(0);
 #endif
+	tinytest_postfork();
 	if (!pid) {
 		/* child. */
 		int test_r, write_r;
diff --git a/src/ext/trunnel/trunnel-impl.h b/src/ext/trunnel/trunnel-impl.h
index 3ffde6e09b..15d1c8633e 100644
--- a/src/ext/trunnel/trunnel-impl.h
+++ b/src/ext/trunnel/trunnel-impl.h
@@ -1,11 +1,11 @@
-/* trunnel-impl.h -- copied from Trunnel v1.4.6
+/* trunnel-impl.h -- copied from Trunnel v1.5.2
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
 /* trunnel-impl.h -- Implementation helpers for trunnel, included by
  * generated trunnel files
  *
- * Copyright 2014-2015, The Tor Project, Inc.
+ * Copyright 2014-2019, The Tor Project, Inc.
  * See license at the end of this file for copying information.
  */
 
@@ -17,6 +17,7 @@
 #include "trunnel.h"
 #include <assert.h>
 #include <string.h>
+#include <stdlib.h>
 
 #if defined(_MSC_VER) && (_MSC_VER < 1600)
 #define uint8_t unsigned char
diff --git a/src/ext/trunnel/trunnel.c b/src/ext/trunnel/trunnel.c
index 3994422643..3ae3fe02c8 100644
--- a/src/ext/trunnel/trunnel.c
+++ b/src/ext/trunnel/trunnel.c
@@ -1,10 +1,10 @@
-/* trunnel.c -- copied from Trunnel v1.4.6
+/* trunnel.c -- copied from Trunnel v1.5.2
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
 /* trunnel.c -- Helper functions to implement trunnel.
  *
- * Copyright 2014-2015, The Tor Project, Inc.
+ * Copyright 2014-2019, The Tor Project, Inc.
  * See license at the end of this file for copying information.
  *
  * See trunnel-impl.h for documentation of these functions.
@@ -14,6 +14,10 @@
 #include <stdlib.h>
 #include <string.h>
 
+#ifdef HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
+
 #if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \
 	__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 #  define IS_LITTLE_ENDIAN 1
@@ -31,7 +35,7 @@
 #    define IS_LITTLE_ENDIAN
 #  endif
 #else
-# if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)
+# if defined(__FreeBSD__) || defined(__NetBSD__) || defined(OpenBSD)
 #  include <sys/endian.h>
 # else
 #  include <endian.h>
diff --git a/src/ext/trunnel/trunnel.h b/src/ext/trunnel/trunnel.h
index 41068b8fb3..9b708437b8 100644
--- a/src/ext/trunnel/trunnel.h
+++ b/src/ext/trunnel/trunnel.h
@@ -1,11 +1,11 @@
-/* trunnel.h -- copied from Trunnel v1.4.6
+/* trunnel.h -- copied from Trunnel v1.5.2
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
 /* trunnel.h -- Public declarations for trunnel, to be included
  * in trunnel header files.
 
- * Copyright 2014-2015, The Tor Project, Inc.
+ * Copyright 2014-2019, The Tor Project, Inc.
  * See license at the end of this file for copying information.
  */
 
diff --git a/src/feature/api/tor_api.c b/src/feature/api/tor_api.c
new file mode 100644
index 0000000000..697397d46b
--- /dev/null
+++ b/src/feature/api/tor_api.c
@@ -0,0 +1,167 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file tor_api.c
+ **/
+
+#ifdef _WIN32
+#include <winsock2.h>
+#include <windows.h>
+#endif
+
+#include "feature/api/tor_api.h"
+
+// Include this after the above headers, to insure that they don't
+// depend on anything else.
+#include "orconfig.h"
+#include "lib/cc/torint.h"
+#include "feature/api/tor_api_internal.h"
+#include "lib/cc/compat_compiler.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// We don't want to use tor_malloc and tor_free here, since this needs
+// to run before anything is initialized at all, and ought to run when
+// we're not linked to anything at all.
+
+#define raw_malloc malloc
+#define raw_free free
+#define raw_realloc realloc
+#define raw_strdup strdup
+
+#ifdef _WIN32
+#include "lib/net/socketpair.h"
+#define raw_socketpair tor_ersatz_socketpair
+#define raw_closesocket closesocket
+#define snprintf _snprintf
+#else
+#define raw_socketpair socketpair
+#define raw_closesocket close
+#endif
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/**
+ * Helper: Add a copy of <b>arg</b> to the owned arguments of <b>cfg</b>.
+ * Return 0 on success, -1 on failure.
+ */
+static int
+cfg_add_owned_arg(tor_main_configuration_t *cfg, const char *arg)
+{
+  /* We aren't using amortized realloc here, because libc should do it for us,
+   * and because this function is not critical-path. */
+  char **new_argv = raw_realloc(cfg->argv_owned,
+                                sizeof(char*) * (cfg->argc_owned+1));
+  if (new_argv == NULL)
+    return -1;
+  cfg->argv_owned = new_argv;
+  if (NULL == (cfg->argv_owned[cfg->argc_owned] = raw_strdup(arg)))
+    return -1;
+  ++cfg->argc_owned;
+  return 0;
+}
+
+tor_main_configuration_t *
+tor_main_configuration_new(void)
+{
+  static const char *fake_argv[] = { "tor" };
+  tor_main_configuration_t *cfg = raw_malloc(sizeof(*cfg));
+  if (cfg == NULL)
+    return NULL;
+
+  memset(cfg, 0, sizeof(*cfg));
+
+  cfg->argc = 1;
+  cfg->argv = (char **) fake_argv;
+
+  cfg->owning_controller_socket = TOR_INVALID_SOCKET;
+
+  return cfg;
+}
+
+int
+tor_main_configuration_set_command_line(tor_main_configuration_t *cfg,
+                                        int argc, char *argv[])
+{
+  if (cfg == NULL)
+    return -1;
+  cfg->argc = argc;
+  cfg->argv = argv;
+  return 0;
+}
+
+tor_control_socket_t
+tor_main_configuration_setup_control_socket(tor_main_configuration_t *cfg)
+{
+  if (SOCKET_OK(cfg->owning_controller_socket))
+    return INVALID_TOR_CONTROL_SOCKET;
+
+  tor_socket_t fds[2];
+  if (raw_socketpair(AF_UNIX, SOCK_STREAM, 0, fds) < 0) {
+    return INVALID_TOR_CONTROL_SOCKET;
+  }
+  char buf[32];
+  snprintf(buf, sizeof(buf), "%"PRIu64, (uint64_t)fds[1]);
+
+  cfg_add_owned_arg(cfg, "__OwningControllerFD");
+  cfg_add_owned_arg(cfg, buf);
+
+  cfg->owning_controller_socket = fds[1];
+  return fds[0];
+}
+
+void
+tor_main_configuration_free(tor_main_configuration_t *cfg)
+{
+  if (cfg == NULL)
+    return;
+  if (cfg->argv_owned) {
+    for (int i = 0; i < cfg->argc_owned; ++i) {
+      raw_free(cfg->argv_owned[i]);
+    }
+    raw_free(cfg->argv_owned);
+  }
+  if (SOCKET_OK(cfg->owning_controller_socket)) {
+    raw_closesocket(cfg->owning_controller_socket);
+  }
+  raw_free(cfg);
+}
+
+const char *
+tor_api_get_provider_version(void)
+{
+  return "tor " VERSION;
+}
+
+/* Main entry point for the Tor process.  Called from main().
+ *
+ * This function is distinct from main() only so we can link main.c into
+ * the unittest binary without conflicting with the unittests' main.
+ *
+ * Some embedders have historically called this function; but that usage is
+ * deprecated: they should use tor_run_main() instead.
+ */
+int
+tor_main(int argc, char *argv[])
+{
+  tor_main_configuration_t *cfg = tor_main_configuration_new();
+  if (!cfg) {
+    puts("INTERNAL ERROR: Allocation failure. Cannot proceed");
+    return 1;
+  }
+  if (tor_main_configuration_set_command_line(cfg, argc, argv) < 0) {
+    puts("INTERNAL ERROR: Can't set command line. Cannot proceed.");
+    return 1;
+  }
+  int rv = tor_run_main(cfg);
+  tor_main_configuration_free(cfg);
+  return rv;
+}
diff --git a/src/feature/api/tor_api.h b/src/feature/api/tor_api.h
new file mode 100644
index 0000000000..2bf130c376
--- /dev/null
+++ b/src/feature/api/tor_api.h
@@ -0,0 +1,129 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file tor_api.h
+ * \brief Public C API for the Tor network service.
+ *
+ * This interface is intended for use by programs that need to link Tor as
+ * a library, and launch it in a separate thread.  If you have the ability
+ * to run Tor as a separate executable, you should probably do that instead
+ * of embedding it as a library.
+ *
+ * To use this API, first construct a tor_main_configuration_t object using
+ * tor_main_configuration_new().  Then, you use one or more other function
+ * calls (such as tor_main_configuration_set_command_line() to configure how
+ * Tor should be run.  Finally, you pass the configuration object to
+ * tor_run_main().
+ *
+ * At this point, tor_run_main() will block its thread to run a Tor daemon;
+ * when the Tor daemon exits, it will return.  See notes on bugs and
+ * limitations below.
+ *
+ * There is no other public C API to Tor: calling any C Tor function not
+ * documented in this file is not guaranteed to be stable.
+ **/
+
+#ifndef TOR_API_H
+#define TOR_API_H
+
+typedef struct tor_main_configuration_t tor_main_configuration_t;
+
+/**
+ * Create and return a new tor_main_configuration().
+ */
+tor_main_configuration_t *tor_main_configuration_new(void);
+
+/**
+ * Set the command-line arguments in <b>cfg</b>.
+ *
+ * The <b>argc</b> and <b>argv</b> values here are as for main().  The
+ * contents of the argv pointer must remain unchanged until tor_run_main() has
+ * finished and you call tor_main_configuration_free().
+ *
+ * Return 0 on success, -1 on failure.
+ */
+int tor_main_configuration_set_command_line(tor_main_configuration_t *cfg,
+                                            int argc, char *argv[]);
+
+#ifdef _WIN32
+typedef SOCKET tor_control_socket_t;
+#define INVALID_TOR_CONTROL_SOCKET INVALID_SOCKET
+#else
+typedef int tor_control_socket_t;
+#define INVALID_TOR_CONTROL_SOCKET (-1)
+#endif
+
+/** DOCDOC */
+tor_control_socket_t tor_main_configuration_setup_control_socket(
+                                          tor_main_configuration_t *cfg);
+
+/**
+ * Release all storage held in <b>cfg</b>.
+ *
+ * Once you have passed a tor_main_configuration_t to tor_run_main(), you
+ * must not free it until tor_run_main() has finished.
+ */
+void tor_main_configuration_free(tor_main_configuration_t *cfg);
+
+/**
+ * Return the name and version of the software implementing the tor_api
+ * functionality.  Current implementors are "tor" and "libtorrunner".
+ *
+ * Note that if you're using libtorrunner, you'll see the version of
+ * libtorrunner, not the version of Tor that it's invoking for you.
+ *
+ * Added in Tor 0.3.5.1-alpha.
+ *
+ * Example return values include "tor 0.3.5.1-alpha" when linked directly
+ * against tor, and "libtorrunner 0.3.5.1-alpha" when linked against
+ * libtorrunner while it is invoking an arbitrary version of Tor.  HOWEVER,
+ * the user MUST NOT depend on any particular format or contents of this
+ * string: there may be other things that implement Tor in the future.
+ **/
+const char *tor_api_get_provider_version(void);
+
+/**
+ * Run the tor process, as if from the command line.
+ *
+ * The command line arguments from tor_main_configuration_set_command_line()
+ * are taken as if they had been passed to main().
+ *
+ * This function will not return until Tor is done running.  It returns zero
+ * on success, and nonzero on failure.
+ *
+ * If you want to control when Tor exits, make sure to configure a control
+ * socket. The OwningControllerFD option may be helpful there.
+ *
+ * BUG 23847: Sometimes, if you call tor_main a second time (after it has
+ * returned), Tor may crash or behave strangely.  We have fixed all issues of
+ * this type that we could find, but more may remain.
+ *
+ * LIMITATION: You cannot run more than one instance of Tor in the same
+ * process at the same time. Concurrent calls will cause undefined behavior.
+ * We do not currently have plans to change this.
+ *
+ * LIMITATION: While we will try to fix any problems found here, you
+ * should be aware that Tor was originally written to run as its own
+ * process, and that the functionality of this file was added later.  If
+ * you find any bugs or strange behavior, please report them, and we'll
+ * try to straighten them out.
+ */
+int tor_run_main(const tor_main_configuration_t *);
+
+/**
+ * Run the tor process, as if from the command line.
+ *
+ * @deprecated Using this function from outside Tor is deprecated; you should
+ * use tor_run_main() instead.
+ *
+ * BUGS: This function has all the same bugs as tor_run_main().
+ *
+ * LIMITATIONS: This function has all the limitations of tor_run_main().
+ */
+int tor_main(int argc, char **argv);
+
+#endif /* !defined(TOR_API_H) */
diff --git a/src/feature/api/tor_api_internal.h b/src/feature/api/tor_api_internal.h
new file mode 100644
index 0000000000..60e0f3aa59
--- /dev/null
+++ b/src/feature/api/tor_api_internal.h
@@ -0,0 +1,29 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_API_INTERNAL_H
+#define TOR_API_INTERNAL_H
+
+#include "lib/net/nettypes.h"
+
+/* The contents of this type are private; don't mess with them from outside
+ * Tor. */
+struct tor_main_configuration_t {
+  /** As in main() */
+  int argc;
+  /** As in main(). This pointer is owned by the caller */
+  char **argv;
+
+  /** As argc, but describes the number of elements in argv_owned */
+  int argc_owned;
+  /** As argv, but is owned by the tor_main_configuration_t object. */
+  char **argv_owned;
+
+  /** Socket that Tor will use as an owning control socket. Owned. */
+  tor_socket_t owning_controller_socket;
+};
+
+#endif /* !defined(TOR_API_INTERNAL_H) */
diff --git a/src/or/addressmap.c b/src/feature/client/addressmap.c
index 33fd7e0f4a..bbe786a6a2 100644
--- a/src/or/addressmap.c
+++ b/src/feature/client/addressmap.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,15 +15,19 @@
 
 #define ADDRESSMAP_PRIVATE
 
-#include "or.h"
-#include "addressmap.h"
-#include "circuituse.h"
-#include "config.h"
-#include "connection_edge.h"
-#include "control.h"
-#include "dns.h"
-#include "routerset.h"
-#include "nodelist.h"
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "core/or/or.h"
+#include "feature/client/addressmap.h"
+#include "core/or/circuituse.h"
+#include "app/config/config.h"
+#include "core/or/connection_edge.h"
+#include "feature/control/control.h"
+#include "feature/relay/dns.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerset.h"
+
+#include "core/or/entry_connection_st.h"
 
 /** A client-side struct to remember requests to rewrite addresses
  * to new addresses. These structs are stored in the hash table
@@ -90,34 +94,47 @@ addressmap_init(void)
   virtaddress_reversemap = strmap_new();
 }
 
+#define addressmap_ent_free(ent)                                        \
+  FREE_AND_NULL(addressmap_entry_t, addressmap_ent_free_, (ent))
+
 /** Free the memory associated with the addressmap entry <b>_ent</b>. */
 static void
-addressmap_ent_free(void *_ent)
+addressmap_ent_free_(addressmap_entry_t *ent)
 {
-  addressmap_entry_t *ent;
-  if (!_ent)
+  if (!ent)
     return;
 
-  ent = _ent;
   tor_free(ent->new_address);
   tor_free(ent);
 }
 
+static void
+addressmap_ent_free_void(void *ent)
+{
+  addressmap_ent_free_(ent);
+}
+
+#define addressmap_virtaddress_ent_free(ent)                            \
+  FREE_AND_NULL(virtaddress_entry_t, addressmap_virtaddress_ent_free_, (ent))
+
 /** Free storage held by a virtaddress_entry_t* entry in <b>_ent</b>. */
 static void
-addressmap_virtaddress_ent_free(void *_ent)
+addressmap_virtaddress_ent_free_(virtaddress_entry_t *ent)
 {
-  virtaddress_entry_t *ent;
-  if (!_ent)
+  if (!ent)
     return;
-
-  ent = _ent;
   tor_free(ent->ipv4_address);
   tor_free(ent->ipv6_address);
   tor_free(ent->hostname_address);
   tor_free(ent);
 }
 
+static void
+addressmap_virtaddress_ent_free_void(void *ent)
+{
+  addressmap_virtaddress_ent_free_(ent);
+}
+
 /** Remove <b>address</b> (which must map to <b>ent</b>) from the
  * virtual address map. */
 static void
@@ -213,8 +230,8 @@ addressmap_clear_excluded_trackexithosts(const or_options_t *options)
     while (dot > target && *dot != '.')
       dot--;
     if (*dot == '.') dot++;
-    nodename = tor_strndup(dot, len-5-(dot-target));;
-    node = node_get_by_nickname(nodename, 0);
+    nodename = tor_strndup(dot, len-5-(dot-target));
+    node = node_get_by_nickname(nodename, NNF_NO_WARN_UNNAMED);
     tor_free(nodename);
     if (!node ||
         (allow_nodes && !routerset_contains_node(allow_nodes, node)) ||
@@ -311,10 +328,10 @@ addressmap_clean(time_t now)
 void
 addressmap_free_all(void)
 {
-  strmap_free(addressmap, addressmap_ent_free);
+  strmap_free(addressmap, addressmap_ent_free_void);
   addressmap = NULL;
 
-  strmap_free(virtaddress_reversemap, addressmap_virtaddress_ent_free);
+  strmap_free(virtaddress_reversemap, addressmap_virtaddress_ent_free_void);
   virtaddress_reversemap = NULL;
 }
 
@@ -376,29 +393,38 @@ addressmap_rewrite(char *address, size_t maxlen,
   char *addr_orig = tor_strdup(address);
   char *log_addr_orig = NULL;
 
+  /* We use a loop here to limit the total number of rewrites we do,
+   * so that we can't hit an infinite loop. */
   for (rewrites = 0; rewrites < 16; rewrites++) {
     int exact_match = 0;
     log_addr_orig = tor_strdup(escaped_safe_str_client(address));
 
+    /* First check to see if there's an exact match for this address */
     ent = strmap_get(addressmap, address);
 
     if (!ent || !ent->new_address) {
+      /* And if we don't have an exact match, try to check whether
+       * we have a pattern-based match.
+       */
       ent = addressmap_match_superdomains(address);
     } else {
       if (ent->src_wildcard && !ent->dst_wildcard &&
           !strcasecmp(address, ent->new_address)) {
-        /* This is a rule like *.example.com example.com, and we just got
-         * "example.com" */
+        /* This is a rule like "rewrite *.example.com to example.com", and we
+         * just got "example.com". Instead of calling it an infinite loop,
+         * call it complete. */
         goto done;
       }
-
       exact_match = 1;
     }
 
     if (!ent || !ent->new_address) {
+      /* We still have no match at all.  We're done! */
       goto done;
     }
 
+    /* Check wither the flags we were passed tell us not to use this
+     * mapping. */
     switch (ent->source) {
       case ADDRMAPSRC_DNS:
         {
@@ -431,6 +457,8 @@ addressmap_rewrite(char *address, size_t maxlen,
         goto done;
     }
 
+    /* Now fill in the address with the new address. That might be via
+     * appending some new stuff to the end, or via just replacing it. */
     if (ent->dst_wildcard && !exact_match) {
       strlcat(address, ".", maxlen);
       strlcat(address, ent->new_address, maxlen);
@@ -438,6 +466,7 @@ addressmap_rewrite(char *address, size_t maxlen,
       strlcpy(address, ent->new_address, maxlen);
     }
 
+    /* Is this now a .exit address?  If so, remember where we got it.*/
     if (!strcmpend(address, ".exit") &&
         strcmpend(addr_orig, ".exit") &&
         exit_source == ADDRMAPSRC_NONE) {
@@ -529,7 +558,7 @@ addressmap_have_mapping(const char *address, int update_expiry)
  * (virtual address mapping) from the controller.)
  *
  * <b>new_address</b> should be a newly dup'ed string, which we'll use or
- * free as appropriate. We will leave address alone.
+ * free as appropriate. We will leave <b>address</b> alone.
  *
  * If <b>wildcard_addr</b> is true, then the mapping will match any address
  * equal to <b>address</b>, or any address ending with a period followed by
@@ -542,7 +571,6 @@ addressmap_have_mapping(const char *address, int update_expiry)
  * <b>wildcard_new_addr</b>, remove any mappings that exist from
  * <b>address</b>.
  *
- *
  * It is an error to set <b>wildcard_new_addr</b> if <b>wildcard_addr</b> is
  * not set. */
 void
@@ -615,7 +643,7 @@ client_dns_incr_failures(const char *address)
     ent->expires = time(NULL) + MAX_DNS_ENTRY_AGE;
     strmap_set(addressmap,address,ent);
   }
-  if (ent->num_resolve_failures < SHORT_MAX)
+  if (ent->num_resolve_failures < SHRT_MAX)
     ++ent->num_resolve_failures; /* don't overflow */
   log_info(LD_APP, "Address %s now has %d resolve failures.",
            safe_str_client(address),
@@ -802,7 +830,7 @@ parse_virtual_addr_network(const char *val, sa_family_t family,
                    ipv6?"IPv6":"");
     return -1;
   }
-#endif
+#endif /* 0 */
 
   if (bits > max_prefix_bits) {
     if (msg)
@@ -935,9 +963,11 @@ addressmap_get_virtual_address(int type)
         char tmp[TOR_ADDR_BUF_LEN];
         tor_addr_to_str(tmp, &addr, sizeof(tmp), 0);
         if (strmap_get(addressmap, tmp)) {
+          // LCOV_EXCL_START
           log_warn(LD_BUG, "%s wasn't in the addressmap, but %s was.",
                    buf, tmp);
           continue;
+          // LCOV_EXCL_STOP
         }
 
         return tor_strdup(buf);
@@ -946,8 +976,10 @@ addressmap_get_virtual_address(int type)
     log_warn(LD_CONFIG, "Ran out of virtual addresses!");
     return NULL;
   } else {
+    // LCOV_EXCL_START
     log_warn(LD_BUG, "Called with unsupported address type (%d)", type);
     return NULL;
+    // LCOV_EXCL_STOP
   }
 }
 
@@ -1032,7 +1064,7 @@ addressmap_register_virtual_address(int type, char *new_address)
              safe_str_client(*addrp),
              safe_str_client(new_address));
   }
-#endif
+#endif /* 0 */
 
   return *addrp;
 }
@@ -1122,4 +1154,3 @@ addressmap_get_mappings(smartlist_t *sl, time_t min_expires,
      iter = strmap_iter_next(addressmap,iter);
    }
 }
-
diff --git a/src/or/addressmap.h b/src/feature/client/addressmap.h
index 67648d0518..9179aef1d0 100644
--- a/src/or/addressmap.h
+++ b/src/feature/client/addressmap.h
@@ -1,13 +1,13 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_ADDRESSMAP_H
 #define TOR_ADDRESSMAP_H
 
-#include "testsupport.h"
+#include "lib/testsupport/testsupport.h"
 
 void addressmap_init(void);
 void addressmap_clear_excluded_trackexithosts(const or_options_t *options);
@@ -59,7 +59,7 @@ typedef struct virtual_addr_conf_t {
 
 STATIC void get_random_virtual_addr(const virtual_addr_conf_t *conf,
                                     tor_addr_t *addr_out);
-#endif
+#endif /* defined(ADDRESSMAP_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_ADDRESSMAP_H) */
 
diff --git a/src/feature/client/bridges.c b/src/feature/client/bridges.c
new file mode 100644
index 0000000000..626c5efcae
--- /dev/null
+++ b/src/feature/client/bridges.c
@@ -0,0 +1,1029 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file bridges.c
+ * \brief Code to manage bridges and bridge selection.
+ *
+ * Bridges are fixed entry nodes, used for censorship circumvention.
+ **/
+
+#define TOR_BRIDGES_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/policies.h"
+#include "feature/client/bridges.h"
+#include "feature/client/entrynodes.h"
+#include "feature/client/transports.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
+#include "feature/dircommon/directory.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerinfo.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerset.h"
+
+#include "core/or/extend_info_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+#include "feature/nodelist/microdesc_st.h"
+
+/** Information about a configured bridge. Currently this just matches the
+ * ones in the torrc file, but one day we may be able to learn about new
+ * bridges on our own, and remember them in the state file. */
+struct bridge_info_t {
+  /** Address and port of the bridge, as configured by the user.*/
+  tor_addr_port_t addrport_configured;
+  /** Address of the bridge. */
+  tor_addr_t addr;
+  /** TLS port for the bridge. */
+  uint16_t port;
+  /** Boolean: We are re-parsing our bridge list, and we are going to remove
+   * this one if we don't find it in the list of configured bridges. */
+  unsigned marked_for_removal : 1;
+  /** Expected identity digest, or all zero bytes if we don't know what the
+   * digest should be. */
+  char identity[DIGEST_LEN];
+
+  /** Name of pluggable transport protocol taken from its config line. */
+  char *transport_name;
+
+  /** When should we next try to fetch a descriptor for this bridge? */
+  download_status_t fetch_status;
+
+  /** A smartlist of k=v values to be passed to the SOCKS proxy, if
+      transports are used for this bridge. */
+  smartlist_t *socks_args;
+};
+
+#define bridge_free(bridge) \
+  FREE_AND_NULL(bridge_info_t, bridge_free_, (bridge))
+
+static void bridge_free_(bridge_info_t *bridge);
+static void rewrite_node_address_for_bridge(const bridge_info_t *bridge,
+                                            node_t *node);
+
+/** A list of configured bridges. Whenever we actually get a descriptor
+ * for one, we add it as an entry guard.  Note that the order of bridges
+ * in this list does not necessarily correspond to the order of bridges
+ * in the torrc. */
+static smartlist_t *bridge_list = NULL;
+
+/** Mark every entry of the bridge list to be removed on our next call to
+ * sweep_bridge_list unless it has first been un-marked. */
+void
+mark_bridge_list(void)
+{
+  if (!bridge_list)
+    bridge_list = smartlist_new();
+  SMARTLIST_FOREACH(bridge_list, bridge_info_t *, b,
+                    b->marked_for_removal = 1);
+}
+
+/** Remove every entry of the bridge list that was marked with
+ * mark_bridge_list if it has not subsequently been un-marked. */
+void
+sweep_bridge_list(void)
+{
+  if (!bridge_list)
+    bridge_list = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) {
+    if (b->marked_for_removal) {
+      SMARTLIST_DEL_CURRENT(bridge_list, b);
+      bridge_free(b);
+    }
+  } SMARTLIST_FOREACH_END(b);
+}
+
+/** Initialize the bridge list to empty, creating it if needed. */
+STATIC void
+clear_bridge_list(void)
+{
+  if (!bridge_list)
+    bridge_list = smartlist_new();
+  SMARTLIST_FOREACH(bridge_list, bridge_info_t *, b, bridge_free(b));
+  smartlist_clear(bridge_list);
+}
+
+/** Free the bridge <b>bridge</b>. */
+static void
+bridge_free_(bridge_info_t *bridge)
+{
+  if (!bridge)
+    return;
+
+  tor_free(bridge->transport_name);
+  if (bridge->socks_args) {
+    SMARTLIST_FOREACH(bridge->socks_args, char*, s, tor_free(s));
+    smartlist_free(bridge->socks_args);
+  }
+
+  tor_free(bridge);
+}
+
+/** Return a list of all the configured bridges, as bridge_info_t pointers. */
+const smartlist_t *
+bridge_list_get(void)
+{
+  if (!bridge_list)
+    bridge_list = smartlist_new();
+  return bridge_list;
+}
+
+/**
+ * Given a <b>bridge</b>, return a pointer to its RSA identity digest, or
+ * NULL if we don't know one for it.
+ */
+const uint8_t *
+bridge_get_rsa_id_digest(const bridge_info_t *bridge)
+{
+  tor_assert(bridge);
+  if (tor_digest_is_zero(bridge->identity))
+    return NULL;
+  else
+    return (const uint8_t *) bridge->identity;
+}
+
+/**
+ * Given a <b>bridge</b>, return a pointer to its configured addr:port
+ * combination.
+ */
+const tor_addr_port_t *
+bridge_get_addr_port(const bridge_info_t *bridge)
+{
+  tor_assert(bridge);
+  return &bridge->addrport_configured;
+}
+
+/** If we have a bridge configured whose digest matches <b>digest</b>, or a
+ * bridge with no known digest whose address matches any of the
+ * tor_addr_port_t's in <b>orports</b>, return that bridge.  Else return
+ * NULL. */
+STATIC bridge_info_t *
+get_configured_bridge_by_orports_digest(const char *digest,
+                                        const smartlist_t *orports)
+{
+  if (!bridge_list)
+    return NULL;
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge)
+    {
+      if (tor_digest_is_zero(bridge->identity)) {
+        SMARTLIST_FOREACH_BEGIN(orports, tor_addr_port_t *, ap)
+          {
+            if (tor_addr_compare(&bridge->addr, &ap->addr, CMP_EXACT) == 0 &&
+                bridge->port == ap->port)
+              return bridge;
+          }
+        SMARTLIST_FOREACH_END(ap);
+      }
+      if (digest && tor_memeq(bridge->identity, digest, DIGEST_LEN))
+        return bridge;
+    }
+  SMARTLIST_FOREACH_END(bridge);
+  return NULL;
+}
+
+/** If we have a bridge configured whose digest matches <b>digest</b>, or a
+ * bridge with no known digest whose address matches <b>addr</b>:<b>port</b>,
+ * return that bridge.  Else return NULL. If <b>digest</b> is NULL, check for
+ * address/port matches only. */
+bridge_info_t *
+get_configured_bridge_by_addr_port_digest(const tor_addr_t *addr,
+                                          uint16_t port,
+                                          const char *digest)
+{
+  if (!bridge_list)
+    return NULL;
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge)
+    {
+      if ((tor_digest_is_zero(bridge->identity) || digest == NULL) &&
+          !tor_addr_compare(&bridge->addr, addr, CMP_EXACT) &&
+          bridge->port == port)
+        return bridge;
+      if (digest && tor_memeq(bridge->identity, digest, DIGEST_LEN))
+        return bridge;
+    }
+  SMARTLIST_FOREACH_END(bridge);
+  return NULL;
+}
+
+/**
+ * As get_configured_bridge_by_addr_port, but require that the
+ * address match <b>addr</b>:<b>port</b>, and that the ID digest match
+ * <b>digest</b>.  (The other function will ignore the address if the
+ * digest matches.)
+ */
+bridge_info_t *
+get_configured_bridge_by_exact_addr_port_digest(const tor_addr_t *addr,
+                                                uint16_t port,
+                                                const char *digest)
+{
+  if (!bridge_list)
+    return NULL;
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge) {
+    if (!tor_addr_compare(&bridge->addr, addr, CMP_EXACT) &&
+        bridge->port == port) {
+
+      if (digest && tor_memeq(bridge->identity, digest, DIGEST_LEN))
+        return bridge;
+      else if (!digest || tor_digest_is_zero(bridge->identity))
+        return bridge;
+    }
+
+  } SMARTLIST_FOREACH_END(bridge);
+  return NULL;
+}
+
+/** If we have a bridge configured whose digest matches <b>digest</b>, or a
+ * bridge with no known digest whose address matches <b>addr</b>:<b>port</b>,
+ * return 1.  Else return 0. If <b>digest</b> is NULL, check for
+ * address/port matches only. */
+int
+addr_is_a_configured_bridge(const tor_addr_t *addr,
+                                uint16_t port,
+                                const char *digest)
+{
+  tor_assert(addr);
+  return get_configured_bridge_by_addr_port_digest(addr, port, digest) ? 1 : 0;
+}
+
+/** If we have a bridge configured whose digest matches
+ * <b>ei->identity_digest</b>, or a bridge with no known digest whose address
+ * matches <b>ei->addr</b>:<b>ei->port</b>, return 1.  Else return 0.
+ * If <b>ei->onion_key</b> is NULL, check for address/port matches only. */
+int
+extend_info_is_a_configured_bridge(const extend_info_t *ei)
+{
+  const char *digest = ei->onion_key ? ei->identity_digest : NULL;
+  return addr_is_a_configured_bridge(&ei->addr, ei->port, digest);
+}
+
+/** Wrapper around get_configured_bridge_by_addr_port_digest() to look
+ * it up via router descriptor <b>ri</b>. */
+static bridge_info_t *
+get_configured_bridge_by_routerinfo(const routerinfo_t *ri)
+{
+  bridge_info_t *bi = NULL;
+  smartlist_t *orports = router_get_all_orports(ri);
+  bi = get_configured_bridge_by_orports_digest(ri->cache_info.identity_digest,
+                                               orports);
+  SMARTLIST_FOREACH(orports, tor_addr_port_t *, p, tor_free(p));
+  smartlist_free(orports);
+  return bi;
+}
+
+/** Return 1 if <b>ri</b> is one of our known bridges, else 0. */
+int
+routerinfo_is_a_configured_bridge(const routerinfo_t *ri)
+{
+  return get_configured_bridge_by_routerinfo(ri) ? 1 : 0;
+}
+
+/**
+ * Return 1 iff <b>bridge_list</b> contains entry matching
+ * given; IPv4 address in host byte order (<b>ipv4_addr</b>
+ * and <b>port</b> (and no identity digest) OR it contains an
+ * entry whose identity matches <b>digest</b>. Otherwise,
+ * return 0.
+ */
+static int
+bridge_exists_with_ipv4h_addr_and_port(const uint32_t ipv4_addr,
+                                       const uint16_t port,
+                                       const char *digest)
+{
+  tor_addr_t node_ipv4;
+
+  if (tor_addr_port_is_valid_ipv4h(ipv4_addr, port, 0)) {
+    tor_addr_from_ipv4h(&node_ipv4, ipv4_addr);
+
+   bridge_info_t *bridge =
+    get_configured_bridge_by_addr_port_digest(&node_ipv4,
+                                              port,
+                                              digest);
+
+   return (bridge != NULL);
+  }
+
+  return 0;
+}
+
+/**
+ * Return 1 iff <b>bridge_list</b> contains entry matching given
+ * <b>ipv6_addr</b> and <b>port</b> (and no identity digest) OR
+ * it contains an  entry whose identity matches <b>digest</b>.
+ * Otherwise, return 0.
+ */
+static int
+bridge_exists_with_ipv6_addr_and_port(const tor_addr_t *ipv6_addr,
+                                      const uint16_t port,
+                                      const char *digest)
+{
+  if (!tor_addr_port_is_valid(ipv6_addr, port, 0))
+    return 0;
+
+  bridge_info_t *bridge =
+   get_configured_bridge_by_addr_port_digest(ipv6_addr,
+                                             port,
+                                             digest);
+
+  return (bridge != NULL);
+}
+
+/** Return 1 if <b>node</b> is one of our configured bridges, else 0.
+ * More specifically, return 1 iff: a bridge_info_t object exists in
+ * <b>bridge_list</b> such that: 1) It's identity is equal to node
+ * identity OR 2) It's identity digest is zero, but it matches
+ * address and port of any ORPort in the node.
+ */
+int
+node_is_a_configured_bridge(const node_t *node)
+{
+  /* First, let's try searching for a bridge with matching identity. */
+  if (BUG(tor_digest_is_zero(node->identity)))
+    return 0;
+
+  if (find_bridge_by_digest(node->identity) != NULL)
+    return 1;
+
+  /* At this point, we have established that no bridge exists with
+   * matching identity digest. However, we still pass it into
+   * bridge_exists_* functions because we want further code to
+   * check for absence of identity digest in a bridge.
+   */
+  if (node->ri) {
+    if (bridge_exists_with_ipv4h_addr_and_port(node->ri->addr,
+                                               node->ri->or_port,
+                                               node->identity))
+      return 1;
+
+    if (bridge_exists_with_ipv6_addr_and_port(&node->ri->ipv6_addr,
+                                              node->ri->ipv6_orport,
+                                              node->identity))
+      return 1;
+  } else if (node->rs) {
+    if (bridge_exists_with_ipv4h_addr_and_port(node->rs->addr,
+                                               node->rs->or_port,
+                                               node->identity))
+      return 1;
+
+    if (bridge_exists_with_ipv6_addr_and_port(&node->rs->ipv6_addr,
+                                              node->rs->ipv6_orport,
+                                              node->identity))
+      return 1;
+  }  else if (node->md) {
+    if (bridge_exists_with_ipv6_addr_and_port(&node->md->ipv6_addr,
+                                              node->md->ipv6_orport,
+                                              node->identity))
+      return 1;
+  }
+
+  return 0;
+}
+
+/** We made a connection to a router at <b>addr</b>:<b>port</b>
+ * without knowing its digest. Its digest turned out to be <b>digest</b>.
+ * If it was a bridge, and we still don't know its digest, record it.
+ */
+void
+learned_router_identity(const tor_addr_t *addr, uint16_t port,
+                        const char *digest,
+                        const ed25519_public_key_t *ed_id)
+{
+  // XXXX prop220 use ed_id here, once there is some way to specify
+  (void)ed_id;
+  int learned = 0;
+  bridge_info_t *bridge =
+    get_configured_bridge_by_exact_addr_port_digest(addr, port, digest);
+  if (bridge && tor_digest_is_zero(bridge->identity)) {
+    memcpy(bridge->identity, digest, DIGEST_LEN);
+    learned = 1;
+  }
+  /* XXXX prop220 remember bridge ed25519 identities -- add a field */
+#if 0
+  if (bridge && ed_id &&
+      ed25519_public_key_is_zero(&bridge->ed25519_identity) &&
+      !ed25519_public_key_is_zero(ed_id)) {
+    memcpy(&bridge->ed25519_identity, ed_id, sizeof(*ed_id));
+    learned = 1;
+  }
+#endif /* 0 */
+  if (learned) {
+    char *transport_info = NULL;
+    const char *transport_name =
+      find_transport_name_by_bridge_addrport(addr, port);
+    if (transport_name)
+      tor_asprintf(&transport_info, " (with transport '%s')", transport_name);
+
+    // XXXX prop220 log both fingerprints.
+    log_notice(LD_DIR, "Learned fingerprint %s for bridge %s%s.",
+               hex_str(digest, DIGEST_LEN), fmt_addrport(addr, port),
+               transport_info ? transport_info : "");
+    tor_free(transport_info);
+    entry_guard_learned_bridge_identity(&bridge->addrport_configured,
+                                        (const uint8_t *)digest);
+  }
+}
+
+/** Return true if <b>bridge</b> has the same identity digest as
+ *  <b>digest</b>. If <b>digest</b> is NULL, it matches
+ *  bridges with unspecified identity digests. */
+static int
+bridge_has_digest(const bridge_info_t *bridge, const char *digest)
+{
+  if (digest)
+    return tor_memeq(digest, bridge->identity, DIGEST_LEN);
+  else
+    return tor_digest_is_zero(bridge->identity);
+}
+
+/** We are about to add a new bridge at <b>addr</b>:<b>port</b>, with optional
+ * <b>digest</b> and <b>transport_name</b>. Mark for removal any previously
+ * existing bridge with the same address and port, and warn the user as
+ * appropriate.
+ */
+STATIC void
+bridge_resolve_conflicts(const tor_addr_t *addr, uint16_t port,
+                         const char *digest, const char *transport_name)
+{
+  /* Iterate the already-registered bridge list:
+
+     If you find a bridge with the same address and port, mark it for
+     removal. It doesn't make sense to have two active bridges with
+     the same IP:PORT. If the bridge in question has a different
+     digest or transport than <b>digest</b>/<b>transport_name</b>,
+     it's probably a misconfiguration and we should warn the user.
+  */
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge) {
+    if (bridge->marked_for_removal)
+      continue;
+
+    if (tor_addr_eq(&bridge->addr, addr) && (bridge->port == port)) {
+
+      bridge->marked_for_removal = 1;
+
+      if (!bridge_has_digest(bridge, digest) ||
+          strcmp_opt(bridge->transport_name, transport_name)) {
+        /* warn the user */
+        char *bridge_description_new, *bridge_description_old;
+        tor_asprintf(&bridge_description_new, "%s:%s:%s",
+                     fmt_addrport(addr, port),
+                     digest ? hex_str(digest, DIGEST_LEN) : "",
+                     transport_name ? transport_name : "");
+        tor_asprintf(&bridge_description_old, "%s:%s:%s",
+                     fmt_addrport(&bridge->addr, bridge->port),
+                     tor_digest_is_zero(bridge->identity) ?
+                     "" : hex_str(bridge->identity,DIGEST_LEN),
+                     bridge->transport_name ? bridge->transport_name : "");
+
+        log_warn(LD_GENERAL,"Tried to add bridge '%s', but we found a conflict"
+                 " with the already registered bridge '%s'. We will discard"
+                 " the old bridge and keep '%s'. If this is not what you"
+                 " wanted, please change your configuration file accordingly.",
+                 bridge_description_new, bridge_description_old,
+                 bridge_description_new);
+
+        tor_free(bridge_description_new);
+        tor_free(bridge_description_old);
+      }
+    }
+  } SMARTLIST_FOREACH_END(bridge);
+}
+
+/** Return True if we have a bridge that uses a transport with name
+ *  <b>transport_name</b>. */
+MOCK_IMPL(int,
+transport_is_needed, (const char *transport_name))
+{
+  if (!bridge_list)
+    return 0;
+
+  SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) {
+    if (bridge->transport_name &&
+        !strcmp(bridge->transport_name, transport_name))
+      return 1;
+  } SMARTLIST_FOREACH_END(bridge);
+
+  return 0;
+}
+
+/** Register the bridge information in <b>bridge_line</b> to the
+ *  bridge subsystem. Steals reference of <b>bridge_line</b>. */
+void
+bridge_add_from_config(bridge_line_t *bridge_line)
+{
+  bridge_info_t *b;
+
+  // XXXX prop220 add a way to specify ed25519 ID to bridge_line_t.
+
+  { /* Log the bridge we are about to register: */
+    log_debug(LD_GENERAL, "Registering bridge at %s (transport: %s) (%s)",
+              fmt_addrport(&bridge_line->addr, bridge_line->port),
+              bridge_line->transport_name ?
+              bridge_line->transport_name : "no transport",
+              tor_digest_is_zero(bridge_line->digest) ?
+              "no key listed" : hex_str(bridge_line->digest, DIGEST_LEN));
+
+    if (bridge_line->socks_args) { /* print socks arguments */
+      int i = 0;
+
+      tor_assert(smartlist_len(bridge_line->socks_args) > 0);
+
+      log_debug(LD_GENERAL, "Bridge uses %d SOCKS arguments:",
+                smartlist_len(bridge_line->socks_args));
+      SMARTLIST_FOREACH(bridge_line->socks_args, const char *, arg,
+                        log_debug(LD_CONFIG, "%d: %s", ++i, arg));
+    }
+  }
+
+  bridge_resolve_conflicts(&bridge_line->addr,
+                           bridge_line->port,
+                           bridge_line->digest,
+                           bridge_line->transport_name);
+
+  b = tor_malloc_zero(sizeof(bridge_info_t));
+  tor_addr_copy(&b->addrport_configured.addr, &bridge_line->addr);
+  b->addrport_configured.port = bridge_line->port;
+  tor_addr_copy(&b->addr, &bridge_line->addr);
+  b->port = bridge_line->port;
+  memcpy(b->identity, bridge_line->digest, DIGEST_LEN);
+  if (bridge_line->transport_name)
+    b->transport_name = bridge_line->transport_name;
+  b->fetch_status.schedule = DL_SCHED_BRIDGE;
+  b->fetch_status.increment_on = DL_SCHED_INCREMENT_ATTEMPT;
+  /* We can't reset the bridge's download status here, because UseBridges
+   * might be 0 now, and it might be changed to 1 much later. */
+  b->socks_args = bridge_line->socks_args;
+  if (!bridge_list)
+    bridge_list = smartlist_new();
+
+  tor_free(bridge_line); /* Deallocate bridge_line now. */
+
+  smartlist_add(bridge_list, b);
+}
+
+/** If <b>digest</b> is one of our known bridges, return it. */
+STATIC bridge_info_t *
+find_bridge_by_digest(const char *digest)
+{
+  if (! bridge_list)
+    return NULL;
+  SMARTLIST_FOREACH(bridge_list, bridge_info_t *, bridge,
+    {
+      if (tor_memeq(bridge->identity, digest, DIGEST_LEN))
+        return bridge;
+    });
+  return NULL;
+}
+
+/** Given the <b>addr</b> and <b>port</b> of a bridge, if that bridge
+ *  supports a pluggable transport, return its name. Otherwise, return
+ *  NULL. */
+const char *
+find_transport_name_by_bridge_addrport(const tor_addr_t *addr, uint16_t port)
+{
+  if (!bridge_list)
+    return NULL;
+
+  SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) {
+    if (tor_addr_eq(&bridge->addr, addr) &&
+        (bridge->port == port))
+      return bridge->transport_name;
+  } SMARTLIST_FOREACH_END(bridge);
+
+  return NULL;
+}
+
+/** If <b>addr</b> and <b>port</b> match the address and port of a
+ * bridge of ours that uses pluggable transports, place its transport
+ * in <b>transport</b>.
+ *
+ * Return 0 on success (found a transport, or found a bridge with no
+ * transport, or found no bridge); return -1 if we should be using a
+ * transport, but the transport could not be found.
+ */
+int
+get_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port,
+                                  const transport_t **transport)
+{
+  *transport = NULL;
+  if (!bridge_list)
+    return 0;
+
+  SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) {
+    if (tor_addr_eq(&bridge->addr, addr) &&
+        (bridge->port == port)) { /* bridge matched */
+      if (bridge->transport_name) { /* it also uses pluggable transports */
+        *transport = transport_get_by_name(bridge->transport_name);
+        if (*transport == NULL) { /* it uses pluggable transports, but
+                                     the transport could not be found! */
+          return -1;
+        }
+        return 0;
+      } else { /* bridge matched, but it doesn't use transports. */
+        break;
+      }
+    }
+  } SMARTLIST_FOREACH_END(bridge);
+
+  *transport = NULL;
+  return 0;
+}
+
+/** Return a smartlist containing all the SOCKS arguments that we
+ *  should pass to the SOCKS proxy. */
+const smartlist_t *
+get_socks_args_by_bridge_addrport(const tor_addr_t *addr, uint16_t port)
+{
+  bridge_info_t *bridge = get_configured_bridge_by_addr_port_digest(addr,
+                                                                    port,
+                                                                    NULL);
+  return bridge ? bridge->socks_args : NULL;
+}
+
+/** We need to ask <b>bridge</b> for its server descriptor. */
+static void
+launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge)
+{
+  const or_options_t *options = get_options();
+  circuit_guard_state_t *guard_state = NULL;
+
+  if (connection_get_by_type_addr_port_purpose(
+      CONN_TYPE_DIR, &bridge->addr, bridge->port,
+      DIR_PURPOSE_FETCH_SERVERDESC))
+    return; /* it's already on the way */
+
+  if (routerset_contains_bridge(options->ExcludeNodes, bridge)) {
+    download_status_mark_impossible(&bridge->fetch_status);
+    log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.",
+             safe_str_client(fmt_and_decorate_addr(&bridge->addr)));
+    return;
+  }
+
+  /* Until we get a descriptor for the bridge, we only know one address for
+   * it. */
+  if (!fascist_firewall_allows_address_addr(&bridge->addr, bridge->port,
+                                            FIREWALL_OR_CONNECTION, 0, 0)) {
+    log_notice(LD_CONFIG, "Tried to fetch a descriptor directly from a "
+               "bridge, but that bridge is not reachable through our "
+               "firewall.");
+    return;
+  }
+
+  /* If we already have a node_t for this bridge, rewrite its address now. */
+  node_t *node = node_get_mutable_by_id(bridge->identity);
+  if (node) {
+    rewrite_node_address_for_bridge(bridge, node);
+  }
+
+  tor_addr_port_t bridge_addrport;
+  memcpy(&bridge_addrport.addr, &bridge->addr, sizeof(tor_addr_t));
+  bridge_addrport.port = bridge->port;
+
+  guard_state = get_guard_state_for_bridge_desc_fetch(bridge->identity);
+
+  directory_request_t *req =
+    directory_request_new(DIR_PURPOSE_FETCH_SERVERDESC);
+  directory_request_set_or_addr_port(req, &bridge_addrport);
+  directory_request_set_directory_id_digest(req, bridge->identity);
+  directory_request_set_router_purpose(req, ROUTER_PURPOSE_BRIDGE);
+  directory_request_set_resource(req, "authority.z");
+  if (guard_state) {
+    directory_request_set_guard_state(req, guard_state);
+  }
+  directory_initiate_request(req);
+  directory_request_free(req);
+}
+
+/** Fetching the bridge descriptor from the bridge authority returned a
+ * "not found". Fall back to trying a direct fetch. */
+void
+retry_bridge_descriptor_fetch_directly(const char *digest)
+{
+  bridge_info_t *bridge = find_bridge_by_digest(digest);
+  if (!bridge)
+    return; /* not found? oh well. */
+
+  launch_direct_bridge_descriptor_fetch(bridge);
+}
+
+/** For each bridge in our list for which we don't currently have a
+ * descriptor, fetch a new copy of its descriptor -- either directly
+ * from the bridge or via a bridge authority. */
+void
+fetch_bridge_descriptors(const or_options_t *options, time_t now)
+{
+  int num_bridge_auths = get_n_authorities(BRIDGE_DIRINFO);
+  int ask_bridge_directly;
+  int can_use_bridge_authority;
+
+  if (!bridge_list)
+    return;
+
+  /* If we still have unconfigured managed proxies, don't go and
+     connect to a bridge. */
+  if (pt_proxies_configuration_pending())
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge)
+    {
+      /* This resets the download status on first use */
+      if (!download_status_is_ready(&bridge->fetch_status, now))
+        continue; /* don't bother, no need to retry yet */
+      if (routerset_contains_bridge(options->ExcludeNodes, bridge)) {
+        download_status_mark_impossible(&bridge->fetch_status);
+        log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.",
+                 safe_str_client(fmt_and_decorate_addr(&bridge->addr)));
+        continue;
+      }
+
+      /* schedule the next attempt
+       * we can't increment after a failure, because sometimes we use the
+       * bridge authority, and sometimes we use the bridge direct */
+      download_status_increment_attempt(
+                        &bridge->fetch_status,
+                        safe_str_client(fmt_and_decorate_addr(&bridge->addr)),
+                        now);
+
+      can_use_bridge_authority = !tor_digest_is_zero(bridge->identity) &&
+                                 num_bridge_auths;
+      ask_bridge_directly = !can_use_bridge_authority ||
+                            !options->UpdateBridgesFromAuthority;
+      log_debug(LD_DIR, "ask_bridge_directly=%d (%d, %d, %d)",
+                ask_bridge_directly, tor_digest_is_zero(bridge->identity),
+                !options->UpdateBridgesFromAuthority, !num_bridge_auths);
+
+      if (ask_bridge_directly &&
+          !fascist_firewall_allows_address_addr(&bridge->addr, bridge->port,
+                                                FIREWALL_OR_CONNECTION, 0,
+                                                0)) {
+        log_notice(LD_DIR, "Bridge at '%s' isn't reachable by our "
+                   "firewall policy. %s.",
+                   fmt_addrport(&bridge->addr, bridge->port),
+                   can_use_bridge_authority ?
+                     "Asking bridge authority instead" : "Skipping");
+        if (can_use_bridge_authority)
+          ask_bridge_directly = 0;
+        else
+          continue;
+      }
+
+      if (ask_bridge_directly) {
+        /* we need to ask the bridge itself for its descriptor. */
+        launch_direct_bridge_descriptor_fetch(bridge);
+      } else {
+        /* We have a digest and we want to ask an authority. We could
+         * combine all the requests into one, but that may give more
+         * hints to the bridge authority than we want to give. */
+        char resource[10 + HEX_DIGEST_LEN];
+        memcpy(resource, "fp/", 3);
+        base16_encode(resource+3, HEX_DIGEST_LEN+1,
+                      bridge->identity, DIGEST_LEN);
+        memcpy(resource+3+HEX_DIGEST_LEN, ".z", 3);
+        log_info(LD_DIR, "Fetching bridge info '%s' from bridge authority.",
+                 resource);
+        directory_get_from_dirserver(DIR_PURPOSE_FETCH_SERVERDESC,
+                ROUTER_PURPOSE_BRIDGE, resource, 0, DL_WANT_AUTHORITY);
+      }
+    }
+  SMARTLIST_FOREACH_END(bridge);
+}
+
+/** If our <b>bridge</b> is configured to be a different address than
+ * the bridge gives in <b>node</b>, rewrite the routerinfo
+ * we received to use the address we meant to use. Now we handle
+ * multihomed bridges better.
+ */
+static void
+rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node)
+{
+  /* XXXX move this function. */
+  /* XXXX overridden addresses should really live in the node_t, so that the
+   *   routerinfo_t and the microdesc_t can be immutable.  But we can only
+   *   do that safely if we know that no function that connects to an OR
+   *   does so through an address from any source other than node_get_addr().
+   */
+  tor_addr_t addr;
+  const or_options_t *options = get_options();
+
+  if (node->ri) {
+    routerinfo_t *ri = node->ri;
+    tor_addr_from_ipv4h(&addr, ri->addr);
+    if ((!tor_addr_compare(&bridge->addr, &addr, CMP_EXACT) &&
+         bridge->port == ri->or_port) ||
+        (!tor_addr_compare(&bridge->addr, &ri->ipv6_addr, CMP_EXACT) &&
+         bridge->port == ri->ipv6_orport)) {
+      /* they match, so no need to do anything */
+    } else {
+      if (tor_addr_family(&bridge->addr) == AF_INET) {
+        ri->addr = tor_addr_to_ipv4h(&bridge->addr);
+        ri->or_port = bridge->port;
+        log_info(LD_DIR,
+                 "Adjusted bridge routerinfo for '%s' to match configured "
+                 "address %s:%d.",
+                 ri->nickname, fmt_addr32(ri->addr), ri->or_port);
+      } else if (tor_addr_family(&bridge->addr) == AF_INET6) {
+        tor_addr_copy(&ri->ipv6_addr, &bridge->addr);
+        ri->ipv6_orport = bridge->port;
+        log_info(LD_DIR,
+                 "Adjusted bridge routerinfo for '%s' to match configured "
+                 "address %s.",
+                 ri->nickname, fmt_addrport(&ri->ipv6_addr, ri->ipv6_orport));
+      } else {
+        log_err(LD_BUG, "Address family not supported: %d.",
+                tor_addr_family(&bridge->addr));
+        return;
+      }
+    }
+
+    if (options->ClientPreferIPv6ORPort == -1) {
+      /* Mark which address to use based on which bridge_t we got. */
+      node->ipv6_preferred = (tor_addr_family(&bridge->addr) == AF_INET6 &&
+                              !tor_addr_is_null(&node->ri->ipv6_addr));
+    } else {
+      /* Mark which address to use based on user preference */
+      node->ipv6_preferred = (fascist_firewall_prefer_ipv6_orport(options) &&
+                              !tor_addr_is_null(&node->ri->ipv6_addr));
+    }
+
+    /* XXXipv6 we lack support for falling back to another address for
+       the same relay, warn the user */
+    if (!tor_addr_is_null(&ri->ipv6_addr)) {
+      tor_addr_port_t ap;
+      node_get_pref_orport(node, &ap);
+      log_notice(LD_CONFIG,
+                 "Bridge '%s' has both an IPv4 and an IPv6 address.  "
+                 "Will prefer using its %s address (%s) based on %s.",
+                 ri->nickname,
+                 node->ipv6_preferred ? "IPv6" : "IPv4",
+                 fmt_addrport(&ap.addr, ap.port),
+                 options->ClientPreferIPv6ORPort == -1 ?
+                 "the configured Bridge address" :
+                 "ClientPreferIPv6ORPort");
+    }
+  }
+  if (node->rs) {
+    routerstatus_t *rs = node->rs;
+    tor_addr_from_ipv4h(&addr, rs->addr);
+
+    if ((!tor_addr_compare(&bridge->addr, &addr, CMP_EXACT) &&
+        bridge->port == rs->or_port) ||
+       (!tor_addr_compare(&bridge->addr, &rs->ipv6_addr, CMP_EXACT) &&
+        bridge->port == rs->ipv6_orport)) {
+      /* they match, so no need to do anything */
+    } else {
+      if (tor_addr_family(&bridge->addr) == AF_INET) {
+        rs->addr = tor_addr_to_ipv4h(&bridge->addr);
+        rs->or_port = bridge->port;
+        log_info(LD_DIR,
+                 "Adjusted bridge routerstatus for '%s' to match "
+                 "configured address %s.",
+                 rs->nickname, fmt_addrport(&bridge->addr, rs->or_port));
+      /* set IPv6 preferences even if there is no ri */
+      } else if (tor_addr_family(&bridge->addr) == AF_INET6) {
+        tor_addr_copy(&rs->ipv6_addr, &bridge->addr);
+        rs->ipv6_orport = bridge->port;
+        log_info(LD_DIR,
+                 "Adjusted bridge routerstatus for '%s' to match configured"
+                 " address %s.",
+                 rs->nickname, fmt_addrport(&rs->ipv6_addr, rs->ipv6_orport));
+      } else {
+        log_err(LD_BUG, "Address family not supported: %d.",
+                tor_addr_family(&bridge->addr));
+        return;
+      }
+    }
+
+    if (options->ClientPreferIPv6ORPort == -1) {
+      /* Mark which address to use based on which bridge_t we got. */
+      node->ipv6_preferred = (tor_addr_family(&bridge->addr) == AF_INET6 &&
+                              !tor_addr_is_null(&node->rs->ipv6_addr));
+    } else {
+      /* Mark which address to use based on user preference */
+      node->ipv6_preferred = (fascist_firewall_prefer_ipv6_orport(options) &&
+                              !tor_addr_is_null(&node->rs->ipv6_addr));
+    }
+
+    /* XXXipv6 we lack support for falling back to another address for
+    the same relay, warn the user */
+    if (!tor_addr_is_null(&rs->ipv6_addr)) {
+      tor_addr_port_t ap;
+      node_get_pref_orport(node, &ap);
+      log_notice(LD_CONFIG,
+                 "Bridge '%s' has both an IPv4 and an IPv6 address.  "
+                 "Will prefer using its %s address (%s) based on %s.",
+                 rs->nickname,
+                 node->ipv6_preferred ? "IPv6" : "IPv4",
+                 fmt_addrport(&ap.addr, ap.port),
+                 options->ClientPreferIPv6ORPort == -1 ?
+                 "the configured Bridge address" :
+                 "ClientPreferIPv6ORPort");
+    }
+  }
+}
+
+/** We just learned a descriptor for a bridge. See if that
+ * digest is in our entry guard list, and add it if not. */
+void
+learned_bridge_descriptor(routerinfo_t *ri, int from_cache)
+{
+  tor_assert(ri);
+  tor_assert(ri->purpose == ROUTER_PURPOSE_BRIDGE);
+  if (get_options()->UseBridges) {
+    /* Retry directory downloads whenever we get a bridge descriptor:
+     * - when bootstrapping, and
+     * - when we aren't sure if any of our bridges are reachable.
+     * Keep on retrying until we have at least one reachable bridge. */
+    int first = num_bridges_usable(0) < 1;
+    bridge_info_t *bridge = get_configured_bridge_by_routerinfo(ri);
+    time_t now = time(NULL);
+    router_set_status(ri->cache_info.identity_digest, 1);
+
+    if (bridge) { /* if we actually want to use this one */
+      node_t *node;
+      /* it's here; schedule its re-fetch for a long time from now. */
+      if (!from_cache) {
+        /* This schedules the re-fetch at a constant interval, which produces
+         * a pattern of bridge traffic. But it's better than trying all
+         * configured briges several times in the first few minutes. */
+        download_status_reset(&bridge->fetch_status);
+      }
+
+      node = node_get_mutable_by_id(ri->cache_info.identity_digest);
+      tor_assert(node);
+      rewrite_node_address_for_bridge(bridge, node);
+      if (tor_digest_is_zero(bridge->identity)) {
+        memcpy(bridge->identity,ri->cache_info.identity_digest, DIGEST_LEN);
+        log_notice(LD_DIR, "Learned identity %s for bridge at %s:%d",
+                   hex_str(bridge->identity, DIGEST_LEN),
+                   fmt_and_decorate_addr(&bridge->addr),
+                   (int) bridge->port);
+      }
+      entry_guard_learned_bridge_identity(&bridge->addrport_configured,
+                              (const uint8_t*)ri->cache_info.identity_digest);
+
+      log_notice(LD_DIR, "new bridge descriptor '%s' (%s): %s", ri->nickname,
+                 from_cache ? "cached" : "fresh", router_describe(ri));
+      /* If we didn't have a reachable bridge before this one, try directory
+       * documents again. */
+      if (first) {
+        routerlist_retry_directory_downloads(now);
+      }
+    }
+  }
+}
+
+/** Return a smartlist containing all bridge identity digests */
+MOCK_IMPL(smartlist_t *,
+list_bridge_identities, (void))
+{
+  smartlist_t *result = NULL;
+  char *digest_tmp;
+
+  if (get_options()->UseBridges && bridge_list) {
+    result = smartlist_new();
+
+    SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) {
+      digest_tmp = tor_malloc(DIGEST_LEN);
+      memcpy(digest_tmp, b->identity, DIGEST_LEN);
+      smartlist_add(result, digest_tmp);
+    } SMARTLIST_FOREACH_END(b);
+  }
+
+  return result;
+}
+
+/** Get the download status for a bridge descriptor given its identity */
+MOCK_IMPL(download_status_t *,
+get_bridge_dl_status_by_id, (const char *digest))
+{
+  download_status_t *dl = NULL;
+
+  if (digest && get_options()->UseBridges && bridge_list) {
+    SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) {
+      if (tor_memeq(digest, b->identity, DIGEST_LEN)) {
+        dl = &(b->fetch_status);
+        break;
+      }
+    } SMARTLIST_FOREACH_END(b);
+  }
+
+  return dl;
+}
+
+/** Release all storage held in bridges.c */
+void
+bridges_free_all(void)
+{
+  clear_bridge_list();
+  smartlist_free(bridge_list);
+  bridge_list = NULL;
+}
diff --git a/src/feature/client/bridges.h b/src/feature/client/bridges.h
new file mode 100644
index 0000000000..27b2750a45
--- /dev/null
+++ b/src/feature/client/bridges.h
@@ -0,0 +1,80 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file bridges.h
+ * \brief Header file for circuitbuild.c.
+ **/
+
+#ifndef TOR_BRIDGES_H
+#define TOR_BRIDGES_H
+
+struct bridge_line_t;
+struct ed25519_public_key_t;
+
+/* Opaque handle to a configured bridge */
+typedef struct bridge_info_t bridge_info_t;
+
+void mark_bridge_list(void);
+void sweep_bridge_list(void);
+const smartlist_t *bridge_list_get(void);
+const uint8_t *bridge_get_rsa_id_digest(const bridge_info_t *bridge);
+const tor_addr_port_t * bridge_get_addr_port(const bridge_info_t *bridge);
+bridge_info_t *get_configured_bridge_by_addr_port_digest(
+                                          const tor_addr_t *addr,
+                                          uint16_t port,
+                                          const char *digest);
+bridge_info_t *get_configured_bridge_by_exact_addr_port_digest(
+                                          const tor_addr_t *addr,
+                                          uint16_t port,
+                                          const char *digest);
+
+int addr_is_a_configured_bridge(const tor_addr_t *addr, uint16_t port,
+                                const char *digest);
+int extend_info_is_a_configured_bridge(const extend_info_t *ei);
+int routerinfo_is_a_configured_bridge(const routerinfo_t *ri);
+int node_is_a_configured_bridge(const node_t *node);
+void learned_router_identity(const tor_addr_t *addr, uint16_t port,
+                             const char *digest,
+                             const struct ed25519_public_key_t *ed_id);
+
+void bridge_add_from_config(struct bridge_line_t *bridge_line);
+void retry_bridge_descriptor_fetch_directly(const char *digest);
+void fetch_bridge_descriptors(const or_options_t *options, time_t now);
+void learned_bridge_descriptor(routerinfo_t *ri, int from_cache);
+const smartlist_t *get_socks_args_by_bridge_addrport(const tor_addr_t *addr,
+                                                     uint16_t port);
+
+int any_bridges_dont_support_microdescriptors(void);
+
+const char *find_transport_name_by_bridge_addrport(const tor_addr_t *addr,
+                                                   uint16_t port);
+struct transport_t;
+int get_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port,
+                                      const struct transport_t **transport);
+
+MOCK_DECL(int, transport_is_needed, (const char *transport_name));
+int validate_pluggable_transports_config(void);
+
+MOCK_DECL(smartlist_t *, list_bridge_identities, (void));
+MOCK_DECL(download_status_t *, get_bridge_dl_status_by_id,
+          (const char *digest));
+
+void bridges_free_all(void);
+
+#ifdef TOR_BRIDGES_PRIVATE
+STATIC void clear_bridge_list(void);
+STATIC bridge_info_t *find_bridge_by_digest(const char *digest);
+STATIC bridge_info_t *get_configured_bridge_by_orports_digest(
+                                                  const char *digest,
+                                                  const smartlist_t *orports);
+STATIC void bridge_resolve_conflicts(const tor_addr_t *addr,
+                                     uint16_t port,
+                                     const char *digest,
+                                     const char *transport_name);
+#endif /* defined(TOR_BRIDGES_PRIVATE) */
+
+#endif /* !defined(TOR_BRIDGES_H) */
diff --git a/src/or/circpathbias.c b/src/feature/client/circpathbias.c
index 9f93e737f7..1743ab5a81 100644
--- a/src/or/circpathbias.c
+++ b/src/feature/client/circpathbias.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,20 +11,37 @@
  * different tor nodes, in an attempt to detect attacks where
  * an attacker deliberately causes circuits to fail until the client
  * choses a path they like.
+ *
+ * This code is currently configured in a warning-only mode, though false
+ * positives appear to be rare in practice.  There is also support for
+ * disabling really bad guards, but it's quite experimental and may have bad
+ * anonymity effects.
+ *
+ * The information here is associated with the entry_guard_t object for
+ * each guard, and stored persistently in the state file.
  */
 
-#include "or.h"
-#include "channel.h"
-#include "circpathbias.h"
-#include "circuitbuild.h"
-#include "circuitlist.h"
-#include "circuituse.h"
-#include "circuitstats.h"
-#include "connection_edge.h"
-#include "config.h"
-#include "entrynodes.h"
-#include "networkstatus.h"
-#include "relay.h"
+#include "core/or/or.h"
+#include "core/or/channel.h"
+#include "feature/client/circpathbias.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/circuitstats.h"
+#include "core/or/connection_edge.h"
+#include "app/config/config.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "feature/client/entrynodes.h"
+#include "feature/nodelist/networkstatus.h"
+#include "core/or/relay.h"
+#include "lib/math/fp.h"
+#include "lib/math/laplace.h"
+
+#include "core/or/cell_st.h"
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/crypt_path_st.h"
+#include "core/or/extend_info_st.h"
+#include "core/or/origin_circuit_st.h"
 
 static void pathbias_count_successful_close(origin_circuit_t *circ);
 static void pathbias_count_collapse(origin_circuit_t *circ);
@@ -43,19 +60,21 @@ static int entry_guard_inc_circ_attempt_count(entry_guard_t *guard);
 static int
 entry_guard_inc_circ_attempt_count(entry_guard_t *guard)
 {
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
   entry_guards_changed();
 
   pathbias_measure_close_rate(guard);
 
-  if (guard->path_bias_disabled)
+  if (pb->path_bias_disabled)
     return -1;
 
   pathbias_scale_close_rates(guard);
-  guard->circ_attempts++;
+  pb->circ_attempts++;
 
-  log_info(LD_CIRC, "Got success count %f/%f for guard %s ($%s)",
-           guard->circ_successes, guard->circ_attempts, guard->nickname,
-           hex_str(guard->identity, DIGEST_LEN));
+  log_info(LD_CIRC, "Got success count %f/%f for guard %s",
+           pb->circ_successes, pb->circ_attempts,
+           entry_guard_describe(guard));
   return 0;
 }
 
@@ -282,7 +301,7 @@ pathbias_is_new_circ_attempt(origin_circuit_t *circ)
   return circ->cpath &&
          circ->cpath->next != circ->cpath &&
          circ->cpath->next->state == CPATH_STATE_AWAITING_KEYS;
-#else
+#else /* !(defined(N2N_TAGGING_IS_POSSIBLE)) */
   /* If tagging attacks are no longer possible, we probably want to
    * count bias from the first hop. However, one could argue that
    * timing-based tagging is still more useful than per-hop failure.
@@ -290,7 +309,7 @@ pathbias_is_new_circ_attempt(origin_circuit_t *circ)
    */
   return circ->cpath &&
          circ->cpath->state == CPATH_STATE_AWAITING_KEYS;
-#endif
+#endif /* defined(N2N_TAGGING_IS_POSSIBLE) */
 }
 
 /**
@@ -505,14 +524,16 @@ pathbias_count_build_success(origin_circuit_t *circ)
     }
 
     if (guard) {
+      guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
       if (circ->path_state == PATH_STATE_BUILD_ATTEMPTED) {
         circ->path_state = PATH_STATE_BUILD_SUCCEEDED;
-        guard->circ_successes++;
+        pb->circ_successes++;
         entry_guards_changed();
 
-        log_info(LD_CIRC, "Got success count %f/%f for guard %s ($%s)",
-                 guard->circ_successes, guard->circ_attempts,
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN));
+        log_info(LD_CIRC, "Got success count %f/%f for guard %s",
+                 pb->circ_successes, pb->circ_attempts,
+                 entry_guard_describe(guard));
       } else {
         if ((rate_msg = rate_limit_log(&success_notice_limit,
                 approx_time()))) {
@@ -527,11 +548,11 @@ pathbias_count_build_success(origin_circuit_t *circ)
         }
       }
 
-      if (guard->circ_attempts < guard->circ_successes) {
+      if (pb->circ_attempts < pb->circ_successes) {
         log_notice(LD_BUG, "Unexpectedly high successes counts (%f/%f) "
-                 "for guard %s ($%s)",
-                 guard->circ_successes, guard->circ_attempts,
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN));
+                 "for guard %s",
+                 pb->circ_successes, pb->circ_attempts,
+                 entry_guard_describe(guard));
       }
     /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to
      * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here.
@@ -574,8 +595,6 @@ pathbias_count_build_success(origin_circuit_t *circ)
 void
 pathbias_count_use_attempt(origin_circuit_t *circ)
 {
-  entry_guard_t *guard;
-
   if (!pathbias_should_count(circ)) {
     return;
   }
@@ -588,19 +607,21 @@ pathbias_count_use_attempt(origin_circuit_t *circ)
         circuit_purpose_to_string(circ->base_.purpose),
         circuit_state_to_string(circ->base_.state));
   } else if (circ->path_state < PATH_STATE_USE_ATTEMPTED) {
-    guard = entry_guard_get_by_id_digest(
+    entry_guard_t *guard = entry_guard_get_by_id_digest(
                 circ->cpath->extend_info->identity_digest);
     if (guard) {
+      guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
       pathbias_measure_use_rate(guard);
       pathbias_scale_use_rates(guard);
-      guard->use_attempts++;
+      pb->use_attempts++;
       entry_guards_changed();
 
       log_debug(LD_CIRC,
-               "Marked circuit %d (%f/%f) as used for guard %s ($%s).",
+               "Marked circuit %d (%f/%f) as used for guard %s.",
                circ->global_identifier,
-               guard->use_successes, guard->use_attempts,
-               guard->nickname, hex_str(guard->identity, DIGEST_LEN));
+               pb->use_successes, pb->use_attempts,
+               entry_guard_describe(guard));
     }
 
     circ->path_state = PATH_STATE_USE_ATTEMPTED;
@@ -702,22 +723,23 @@ pathbias_count_use_success(origin_circuit_t *circ)
     guard = entry_guard_get_by_id_digest(
                 circ->cpath->extend_info->identity_digest);
     if (guard) {
-      guard->use_successes++;
+      guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+      pb->use_successes++;
       entry_guards_changed();
 
-      if (guard->use_attempts < guard->use_successes) {
+      if (pb->use_attempts < pb->use_successes) {
         log_notice(LD_BUG, "Unexpectedly high use successes counts (%f/%f) "
-                 "for guard %s=%s",
-                 guard->use_successes, guard->use_attempts,
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN));
+                 "for guard %s",
+                 pb->use_successes, pb->use_attempts,
+                 entry_guard_describe(guard));
       }
 
       log_debug(LD_CIRC,
-                "Marked circuit %d (%f/%f) as used successfully for guard "
-                "%s ($%s).",
-                circ->global_identifier, guard->use_successes,
-                guard->use_attempts, guard->nickname,
-                hex_str(guard->identity, DIGEST_LEN));
+                "Marked circuit %d (%f/%f) as used successfully for guard %s",
+                circ->global_identifier, pb->use_successes,
+                pb->use_attempts,
+                entry_guard_describe(guard));
     }
   }
 
@@ -879,6 +901,7 @@ pathbias_check_probe_response(circuit_t *circ, const cell_t *cell)
     /* Check nonce */
     if (ipv4_host == ocirc->pathbias_probe_nonce) {
       pathbias_mark_use_success(ocirc);
+      circuit_read_valid_data(ocirc, rh.length);
       circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED);
       log_info(LD_CIRC,
                "Got valid path bias probe back for circ %d, stream %d.",
@@ -900,6 +923,68 @@ pathbias_check_probe_response(circuit_t *circ, const cell_t *cell)
 }
 
 /**
+ * Check if a cell is counts as valid data for a circuit,
+ * and if so, count it as valid.
+ */
+void
+pathbias_count_valid_cells(circuit_t *circ, const cell_t *cell)
+{
+  origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+  relay_header_t rh;
+
+  relay_header_unpack(&rh, cell->payload);
+
+  /* Check to see if this is a cell from a previous connection,
+   * or is a request to close the circuit. */
+  switch (rh.command) {
+    case RELAY_COMMAND_TRUNCATED:
+      /* Truncated cells can arrive on path bias circs. When they do,
+       * just process them. This closes the circ, but it was junk anyway.
+       * No reason to wait for the probe. */
+      circuit_read_valid_data(ocirc, rh.length);
+      circuit_truncated(TO_ORIGIN_CIRCUIT(circ),
+                        get_uint8(cell->payload + RELAY_HEADER_SIZE));
+
+      break;
+
+    case RELAY_COMMAND_END:
+      if (connection_half_edge_is_valid_end(ocirc->half_streams,
+                                             rh.stream_id)) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+      }
+      break;
+
+    case RELAY_COMMAND_DATA:
+      if (connection_half_edge_is_valid_data(ocirc->half_streams,
+                                             rh.stream_id)) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+      }
+      break;
+
+    case RELAY_COMMAND_SENDME:
+      if (connection_half_edge_is_valid_sendme(ocirc->half_streams,
+                                             rh.stream_id)) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+      }
+      break;
+
+    case RELAY_COMMAND_CONNECTED:
+      if (connection_half_edge_is_valid_connected(ocirc->half_streams,
+                                                  rh.stream_id)) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+      }
+      break;
+
+    case RELAY_COMMAND_RESOLVED:
+      if (connection_half_edge_is_valid_resolved(ocirc->half_streams,
+                                                 rh.stream_id)) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+      }
+      break;
+  }
+}
+
+/**
  * Check if a circuit was used and/or closed successfully.
  *
  * If we attempted to use the circuit to carry a stream but failed
@@ -1018,9 +1103,11 @@ pathbias_count_successful_close(origin_circuit_t *circ)
   }
 
   if (guard) {
+    guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
     /* In the long run: circuit_success ~= successful_circuit_close +
      *                                     circ_failure + stream_failure */
-    guard->successful_circuits_closed++;
+    pb->successful_circuits_closed++;
     entry_guards_changed();
   } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
    /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to
@@ -1057,7 +1144,9 @@ pathbias_count_collapse(origin_circuit_t *circ)
   }
 
   if (guard) {
-    guard->collapsed_circuits++;
+    guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+    pb->collapsed_circuits++;
     entry_guards_changed();
   } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
    /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to
@@ -1090,7 +1179,9 @@ pathbias_count_use_failed(origin_circuit_t *circ)
   }
 
   if (guard) {
-    guard->unusable_circuits++;
+    guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+    pb->unusable_circuits++;
     entry_guards_changed();
   } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
    /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to
@@ -1133,7 +1224,9 @@ pathbias_count_timeout(origin_circuit_t *circ)
   }
 
   if (guard) {
-    guard->timeouts++;
+    guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+    pb->timeouts++;
     entry_guards_changed();
   }
 }
@@ -1165,7 +1258,7 @@ pathbias_count_circs_in_states(entry_guard_t *guard,
     if (ocirc->path_state >= from &&
         ocirc->path_state <= to &&
         pathbias_should_count(ocirc) &&
-        fast_memeq(guard->identity,
+        fast_memeq(entry_guard_get_rsa_id_digest(guard),
                    ocirc->cpath->extend_info->identity_digest,
                    DIGEST_LEN)) {
       log_debug(LD_CIRC, "Found opened circuit %d in path_state %s",
@@ -1189,7 +1282,9 @@ pathbias_count_circs_in_states(entry_guard_t *guard,
 double
 pathbias_get_close_success_count(entry_guard_t *guard)
 {
-  return guard->successful_circuits_closed +
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+  return pb->successful_circuits_closed +
          pathbias_count_circs_in_states(guard,
                        PATH_STATE_BUILD_SUCCEEDED,
                        PATH_STATE_USE_SUCCEEDED);
@@ -1205,7 +1300,9 @@ pathbias_get_close_success_count(entry_guard_t *guard)
 double
 pathbias_get_use_success_count(entry_guard_t *guard)
 {
-  return guard->use_successes +
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+  return pb->use_successes +
          pathbias_count_circs_in_states(guard,
                        PATH_STATE_USE_ATTEMPTED,
                        PATH_STATE_USE_SUCCEEDED);
@@ -1223,18 +1320,19 @@ static void
 pathbias_measure_use_rate(entry_guard_t *guard)
 {
   const or_options_t *options = get_options();
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
 
-  if (guard->use_attempts > pathbias_get_min_use(options)) {
+  if (pb->use_attempts > pathbias_get_min_use(options)) {
     /* Note: We rely on the < comparison here to allow us to set a 0
      * rate and disable the feature entirely. If refactoring, don't
      * change to <= */
-    if (pathbias_get_use_success_count(guard)/guard->use_attempts
+    if (pathbias_get_use_success_count(guard)/pb->use_attempts
         < pathbias_get_extreme_use_rate(options)) {
       /* Dropping is currently disabled by default. */
       if (pathbias_get_dropguards(options)) {
-        if (!guard->path_bias_disabled) {
+        if (!pb->path_bias_disabled) {
           log_warn(LD_CIRC,
-                 "Your Guard %s ($%s) is failing to carry an extremely large "
+                 "Guard %s is failing to carry an extremely large "
                  "amount of stream on its circuits. "
                  "To avoid potential route manipulation attacks, Tor has "
                  "disabled use of this guard. "
@@ -1242,25 +1340,23 @@ pathbias_measure_use_rate(entry_guard_t *guard)
                  "%ld circuits completed, %ld were unusable, %ld collapsed, "
                  "and %ld timed out. "
                  "For reference, your timeout cutoff is %ld seconds.",
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN),
+                 entry_guard_describe(guard),
                  tor_lround(pathbias_get_use_success_count(guard)),
-                 tor_lround(guard->use_attempts),
+                 tor_lround(pb->use_attempts),
                  tor_lround(pathbias_get_close_success_count(guard)),
-                 tor_lround(guard->circ_attempts),
-                 tor_lround(guard->circ_successes),
-                 tor_lround(guard->unusable_circuits),
-                 tor_lround(guard->collapsed_circuits),
-                 tor_lround(guard->timeouts),
+                 tor_lround(pb->circ_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
                  tor_lround(get_circuit_build_close_time_ms()/1000));
-          guard->path_bias_disabled = 1;
-          guard->bad_since = approx_time();
-          entry_guards_changed();
+          pb->path_bias_disabled = 1;
           return;
         }
-      } else if (!guard->path_bias_use_extreme) {
-        guard->path_bias_use_extreme = 1;
+      } else if (!pb->path_bias_use_extreme) {
+        pb->path_bias_use_extreme = 1;
         log_warn(LD_CIRC,
-                 "Your Guard %s ($%s) is failing to carry an extremely large "
+                 "Guard %s is failing to carry an extremely large "
                  "amount of streams on its circuits. "
                  "This could indicate a route manipulation attack, network "
                  "overload, bad local network connectivity, or a bug. "
@@ -1268,23 +1364,23 @@ pathbias_measure_use_rate(entry_guard_t *guard)
                  "%ld circuits completed, %ld were unusable, %ld collapsed, "
                  "and %ld timed out. "
                  "For reference, your timeout cutoff is %ld seconds.",
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN),
+                 entry_guard_describe(guard),
                  tor_lround(pathbias_get_use_success_count(guard)),
-                 tor_lround(guard->use_attempts),
+                 tor_lround(pb->use_attempts),
                  tor_lround(pathbias_get_close_success_count(guard)),
-                 tor_lround(guard->circ_attempts),
-                 tor_lround(guard->circ_successes),
-                 tor_lround(guard->unusable_circuits),
-                 tor_lround(guard->collapsed_circuits),
-                 tor_lround(guard->timeouts),
+                 tor_lround(pb->circ_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
                  tor_lround(get_circuit_build_close_time_ms()/1000));
       }
-    } else if (pathbias_get_use_success_count(guard)/guard->use_attempts
+    } else if (pathbias_get_use_success_count(guard)/pb->use_attempts
                < pathbias_get_notice_use_rate(options)) {
-      if (!guard->path_bias_use_noticed) {
-        guard->path_bias_use_noticed = 1;
+      if (!pb->path_bias_use_noticed) {
+        pb->path_bias_use_noticed = 1;
         log_notice(LD_CIRC,
-                 "Your Guard %s ($%s) is failing to carry more streams on its "
+                 "Guard %s is failing to carry more streams on its "
                  "circuits than usual. "
                  "Most likely this means the Tor network is overloaded "
                  "or your network connection is poor. "
@@ -1292,15 +1388,15 @@ pathbias_measure_use_rate(entry_guard_t *guard)
                  "%ld circuits completed, %ld were unusable, %ld collapsed, "
                  "and %ld timed out. "
                  "For reference, your timeout cutoff is %ld seconds.",
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN),
+                 entry_guard_describe(guard),
                  tor_lround(pathbias_get_use_success_count(guard)),
-                 tor_lround(guard->use_attempts),
+                 tor_lround(pb->use_attempts),
                  tor_lround(pathbias_get_close_success_count(guard)),
-                 tor_lround(guard->circ_attempts),
-                 tor_lround(guard->circ_successes),
-                 tor_lround(guard->unusable_circuits),
-                 tor_lround(guard->collapsed_circuits),
-                 tor_lround(guard->timeouts),
+                 tor_lround(pb->circ_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
                  tor_lround(get_circuit_build_close_time_ms()/1000));
       }
     }
@@ -1329,18 +1425,19 @@ static void
 pathbias_measure_close_rate(entry_guard_t *guard)
 {
   const or_options_t *options = get_options();
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
 
-  if (guard->circ_attempts > pathbias_get_min_circs(options)) {
+  if (pb->circ_attempts > pathbias_get_min_circs(options)) {
     /* Note: We rely on the < comparison here to allow us to set a 0
      * rate and disable the feature entirely. If refactoring, don't
      * change to <= */
-    if (pathbias_get_close_success_count(guard)/guard->circ_attempts
+    if (pathbias_get_close_success_count(guard)/pb->circ_attempts
         < pathbias_get_extreme_rate(options)) {
       /* Dropping is currently disabled by default. */
       if (pathbias_get_dropguards(options)) {
-        if (!guard->path_bias_disabled) {
+        if (!pb->path_bias_disabled) {
           log_warn(LD_CIRC,
-                 "Your Guard %s ($%s) is failing an extremely large "
+                 "Guard %s is failing an extremely large "
                  "amount of circuits. "
                  "To avoid potential route manipulation attacks, Tor has "
                  "disabled use of this guard. "
@@ -1348,25 +1445,23 @@ pathbias_measure_close_rate(entry_guard_t *guard)
                  "%ld circuits completed, %ld were unusable, %ld collapsed, "
                  "and %ld timed out. "
                  "For reference, your timeout cutoff is %ld seconds.",
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN),
+                 entry_guard_describe(guard),
                  tor_lround(pathbias_get_close_success_count(guard)),
-                 tor_lround(guard->circ_attempts),
+                 tor_lround(pb->circ_attempts),
                  tor_lround(pathbias_get_use_success_count(guard)),
-                 tor_lround(guard->use_attempts),
-                 tor_lround(guard->circ_successes),
-                 tor_lround(guard->unusable_circuits),
-                 tor_lround(guard->collapsed_circuits),
-                 tor_lround(guard->timeouts),
+                 tor_lround(pb->use_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
                  tor_lround(get_circuit_build_close_time_ms()/1000));
-          guard->path_bias_disabled = 1;
-          guard->bad_since = approx_time();
-          entry_guards_changed();
+          pb->path_bias_disabled = 1;
           return;
         }
-      } else if (!guard->path_bias_extreme) {
-        guard->path_bias_extreme = 1;
+      } else if (!pb->path_bias_extreme) {
+        pb->path_bias_extreme = 1;
         log_warn(LD_CIRC,
-                 "Your Guard %s ($%s) is failing an extremely large "
+                 "Guard %s is failing an extremely large "
                  "amount of circuits. "
                  "This could indicate a route manipulation attack, "
                  "extreme network overload, or a bug. "
@@ -1374,23 +1469,23 @@ pathbias_measure_close_rate(entry_guard_t *guard)
                  "%ld circuits completed, %ld were unusable, %ld collapsed, "
                  "and %ld timed out. "
                  "For reference, your timeout cutoff is %ld seconds.",
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN),
+                 entry_guard_describe(guard),
                  tor_lround(pathbias_get_close_success_count(guard)),
-                 tor_lround(guard->circ_attempts),
+                 tor_lround(pb->circ_attempts),
                  tor_lround(pathbias_get_use_success_count(guard)),
-                 tor_lround(guard->use_attempts),
-                 tor_lround(guard->circ_successes),
-                 tor_lround(guard->unusable_circuits),
-                 tor_lround(guard->collapsed_circuits),
-                 tor_lround(guard->timeouts),
+                 tor_lround(pb->use_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
                  tor_lround(get_circuit_build_close_time_ms()/1000));
       }
-    } else if (pathbias_get_close_success_count(guard)/guard->circ_attempts
+    } else if (pathbias_get_close_success_count(guard)/pb->circ_attempts
                 < pathbias_get_warn_rate(options)) {
-      if (!guard->path_bias_warned) {
-        guard->path_bias_warned = 1;
+      if (!pb->path_bias_warned) {
+        pb->path_bias_warned = 1;
         log_warn(LD_CIRC,
-                 "Your Guard %s ($%s) is failing a very large "
+                 "Guard %s is failing a very large "
                  "amount of circuits. "
                  "Most likely this means the Tor network is "
                  "overloaded, but it could also mean an attack against "
@@ -1399,38 +1494,38 @@ pathbias_measure_close_rate(entry_guard_t *guard)
                  "%ld circuits completed, %ld were unusable, %ld collapsed, "
                  "and %ld timed out. "
                  "For reference, your timeout cutoff is %ld seconds.",
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN),
+                 entry_guard_describe(guard),
                  tor_lround(pathbias_get_close_success_count(guard)),
-                 tor_lround(guard->circ_attempts),
+                 tor_lround(pb->circ_attempts),
                  tor_lround(pathbias_get_use_success_count(guard)),
-                 tor_lround(guard->use_attempts),
-                 tor_lround(guard->circ_successes),
-                 tor_lround(guard->unusable_circuits),
-                 tor_lround(guard->collapsed_circuits),
-                 tor_lround(guard->timeouts),
+                 tor_lround(pb->use_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
                  tor_lround(get_circuit_build_close_time_ms()/1000));
       }
-    } else if (pathbias_get_close_success_count(guard)/guard->circ_attempts
+    } else if (pathbias_get_close_success_count(guard)/pb->circ_attempts
                < pathbias_get_notice_rate(options)) {
-      if (!guard->path_bias_noticed) {
-        guard->path_bias_noticed = 1;
+      if (!pb->path_bias_noticed) {
+        pb->path_bias_noticed = 1;
         log_notice(LD_CIRC,
-                 "Your Guard %s ($%s) is failing more circuits than "
+                 "Guard %s is failing more circuits than "
                  "usual. "
                  "Most likely this means the Tor network is overloaded. "
                  "Success counts are %ld/%ld. Use counts are %ld/%ld. "
                  "%ld circuits completed, %ld were unusable, %ld collapsed, "
                  "and %ld timed out. "
                  "For reference, your timeout cutoff is %ld seconds.",
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN),
+                 entry_guard_describe(guard),
                  tor_lround(pathbias_get_close_success_count(guard)),
-                 tor_lround(guard->circ_attempts),
+                 tor_lround(pb->circ_attempts),
                  tor_lround(pathbias_get_use_success_count(guard)),
-                 tor_lround(guard->use_attempts),
-                 tor_lround(guard->circ_successes),
-                 tor_lround(guard->unusable_circuits),
-                 tor_lround(guard->collapsed_circuits),
-                 tor_lround(guard->timeouts),
+                 tor_lround(pb->use_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
                  tor_lround(get_circuit_build_close_time_ms()/1000));
       }
     }
@@ -1444,15 +1539,16 @@ pathbias_measure_close_rate(entry_guard_t *guard)
  *
  * XXX: The attempt count transfer stuff here might be done
  * better by keeping separate pending counters that get
- * transfered at circuit close. See ticket #8160.
+ * transferred at circuit close. See ticket #8160.
  */
 static void
 pathbias_scale_close_rates(entry_guard_t *guard)
 {
   const or_options_t *options = get_options();
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
 
   /* If we get a ton of circuits, just scale everything down */
-  if (guard->circ_attempts > pathbias_get_scale_threshold(options)) {
+  if (pb->circ_attempts > pathbias_get_scale_threshold(options)) {
     double scale_ratio = pathbias_get_scale_ratio(options);
     int opened_attempts = pathbias_count_circs_in_states(guard,
             PATH_STATE_BUILD_ATTEMPTED, PATH_STATE_BUILD_ATTEMPTED);
@@ -1460,38 +1556,38 @@ pathbias_scale_close_rates(entry_guard_t *guard)
                         PATH_STATE_BUILD_SUCCEEDED,
                         PATH_STATE_USE_FAILED);
     /* Verify that the counts are sane before and after scaling */
-    int counts_are_sane = (guard->circ_attempts >= guard->circ_successes);
+    int counts_are_sane = (pb->circ_attempts >= pb->circ_successes);
 
-    guard->circ_attempts -= (opened_attempts+opened_built);
-    guard->circ_successes -= opened_built;
+    pb->circ_attempts -= (opened_attempts+opened_built);
+    pb->circ_successes -= opened_built;
 
-    guard->circ_attempts *= scale_ratio;
-    guard->circ_successes *= scale_ratio;
-    guard->timeouts *= scale_ratio;
-    guard->successful_circuits_closed *= scale_ratio;
-    guard->collapsed_circuits *= scale_ratio;
-    guard->unusable_circuits *= scale_ratio;
+    pb->circ_attempts *= scale_ratio;
+    pb->circ_successes *= scale_ratio;
+    pb->timeouts *= scale_ratio;
+    pb->successful_circuits_closed *= scale_ratio;
+    pb->collapsed_circuits *= scale_ratio;
+    pb->unusable_circuits *= scale_ratio;
 
-    guard->circ_attempts += (opened_attempts+opened_built);
-    guard->circ_successes += opened_built;
+    pb->circ_attempts += (opened_attempts+opened_built);
+    pb->circ_successes += opened_built;
 
     entry_guards_changed();
 
     log_info(LD_CIRC,
              "Scaled pathbias counts to (%f,%f)/%f (%d/%d open) for guard "
-             "%s ($%s)",
-             guard->circ_successes, guard->successful_circuits_closed,
-             guard->circ_attempts, opened_built, opened_attempts,
-             guard->nickname, hex_str(guard->identity, DIGEST_LEN));
+             "%s",
+             pb->circ_successes, pb->successful_circuits_closed,
+             pb->circ_attempts, opened_built, opened_attempts,
+             entry_guard_describe(guard));
 
     /* Have the counts just become invalid by this scaling attempt? */
-    if (counts_are_sane && guard->circ_attempts < guard->circ_successes) {
+    if (counts_are_sane && pb->circ_attempts < pb->circ_successes) {
       log_notice(LD_BUG,
                "Scaling has mangled pathbias counts to %f/%f (%d/%d open) "
-               "for guard %s ($%s)",
-               guard->circ_successes, guard->circ_attempts, opened_built,
-               opened_attempts, guard->nickname,
-               hex_str(guard->identity, DIGEST_LEN));
+               "for guard %s",
+               pb->circ_successes, pb->circ_attempts, opened_built,
+               opened_attempts,
+               entry_guard_describe(guard));
     }
   }
 }
@@ -1503,44 +1599,43 @@ pathbias_scale_close_rates(entry_guard_t *guard)
  *
  * XXX: The attempt count transfer stuff here might be done
  * better by keeping separate pending counters that get
- * transfered at circuit close. See ticket #8160.
+ * transferred at circuit close. See ticket #8160.
  */
 void
 pathbias_scale_use_rates(entry_guard_t *guard)
 {
   const or_options_t *options = get_options();
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
 
   /* If we get a ton of circuits, just scale everything down */
-  if (guard->use_attempts > pathbias_get_scale_use_threshold(options)) {
+  if (pb->use_attempts > pathbias_get_scale_use_threshold(options)) {
     double scale_ratio = pathbias_get_scale_ratio(options);
     int opened_attempts = pathbias_count_circs_in_states(guard,
             PATH_STATE_USE_ATTEMPTED, PATH_STATE_USE_SUCCEEDED);
     /* Verify that the counts are sane before and after scaling */
-    int counts_are_sane = (guard->use_attempts >= guard->use_successes);
+    int counts_are_sane = (pb->use_attempts >= pb->use_successes);
 
-    guard->use_attempts -= opened_attempts;
+    pb->use_attempts -= opened_attempts;
 
-    guard->use_attempts *= scale_ratio;
-    guard->use_successes *= scale_ratio;
+    pb->use_attempts *= scale_ratio;
+    pb->use_successes *= scale_ratio;
 
-    guard->use_attempts += opened_attempts;
+    pb->use_attempts += opened_attempts;
 
     log_info(LD_CIRC,
-           "Scaled pathbias use counts to %f/%f (%d open) for guard %s ($%s)",
-           guard->use_successes, guard->use_attempts, opened_attempts,
-           guard->nickname, hex_str(guard->identity, DIGEST_LEN));
+           "Scaled pathbias use counts to %f/%f (%d open) for guard %s",
+           pb->use_successes, pb->use_attempts, opened_attempts,
+           entry_guard_describe(guard));
 
     /* Have the counts just become invalid by this scaling attempt? */
-    if (counts_are_sane && guard->use_attempts < guard->use_successes) {
+    if (counts_are_sane && pb->use_attempts < pb->use_successes) {
       log_notice(LD_BUG,
                "Scaling has mangled pathbias usage counts to %f/%f "
-               "(%d open) for guard %s ($%s)",
-               guard->circ_successes, guard->circ_attempts,
-               opened_attempts, guard->nickname,
-               hex_str(guard->identity, DIGEST_LEN));
+               "(%d open) for guard %s",
+               pb->circ_successes, pb->circ_attempts,
+               opened_attempts, entry_guard_describe(guard));
     }
 
     entry_guards_changed();
   }
 }
-
diff --git a/src/or/circpathbias.h b/src/feature/client/circpathbias.h
index ce76689d5f..a9a8d18df2 100644
--- a/src/or/circpathbias.h
+++ b/src/feature/client/circpathbias.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,10 +20,10 @@ void pathbias_count_build_success(origin_circuit_t *circ);
 int pathbias_count_build_attempt(origin_circuit_t *circ);
 int pathbias_check_close(origin_circuit_t *circ, int reason);
 int pathbias_check_probe_response(circuit_t *circ, const cell_t *cell);
+void pathbias_count_valid_cells(circuit_t *circ, const cell_t *cell);
 void pathbias_count_use_attempt(origin_circuit_t *circ);
 void pathbias_mark_use_success(origin_circuit_t *circ);
 void pathbias_mark_use_rollback(origin_circuit_t *circ);
-const char *pathbias_state_to_string(path_state_t state);
-
-#endif
+const char *pathbias_state_to_string(enum path_state_t state);
 
+#endif /* !defined(TOR_CIRCPATHBIAS_H) */
diff --git a/src/or/dnsserv.c b/src/feature/client/dnsserv.c
index f5a4f2ac0f..cf593cfb68 100644
--- a/src/or/dnsserv.c
+++ b/src/feature/client/dnsserv.c
@@ -1,22 +1,41 @@
-/* Copyright (c) 2007-2016, The Tor Project, Inc. */
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file dnsserv.c
- * \brief Implements client-side DNS proxy server code.  Note:
- * this is the DNS Server code, not the Server DNS code.  Confused?  This code
- * runs on client-side, and acts as a DNS server.  The code in dns.c, on the
- * other hand, runs on Tor servers, and acts as a DNS client.
+ * \brief Implements client-side DNS proxy server code.
+ *
+ * When a user enables the DNSPort configuration option to have their local
+ * Tor client handle DNS requests, this module handles it.  It functions as a
+ * "DNS Server" on the client side, which client applications use.
+ *
+ * Inbound DNS requests are represented as entry_connection_t here (since
+ * that's how Tor represents client-side streams), which are kept associated
+ * with an evdns_server_request structure as exposed by Libevent's
+ * evdns code.
+ *
+ * Upon receiving a DNS request, libevent calls our evdns_server_callback()
+ * function here, which causes this module to create an entry_connection_t
+ * request as appropriate.  Later, when that request is answered,
+ * connection_edge.c calls dnsserv_resolved() so we can finish up and tell the
+ * DNS client.
  **/
 
-#include "or.h"
-#include "dnsserv.h"
-#include "config.h"
-#include "connection.h"
-#include "connection_edge.h"
-#include "control.h"
-#include "main.h"
-#include "policies.h"
+#include "core/or/or.h"
+#include "feature/client/dnsserv.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/connection_edge.h"
+#include "feature/control/control.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/policies.h"
+
+#include "feature/control/control_connection_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/listener_connection_st.h"
+#include "core/or/socks_request_st.h"
+#include "lib/evloop/compat_libevent.h"
+
 #include <event2/dns.h>
 #include <event2/dns_compat.h>
 /* XXXX this implies we want an improved evdns  */
@@ -160,7 +179,7 @@ evdns_server_callback(struct evdns_server_request *req, void *data_)
   if (connection_add(ENTRY_TO_CONN(entry_conn)) < 0) {
     log_warn(LD_APP, "Couldn't register dummy connection for DNS request");
     evdns_server_request_respond(req, DNS_ERR_SERVERFAILED);
-    connection_free(ENTRY_TO_CONN(entry_conn));
+    connection_free_(ENTRY_TO_CONN(entry_conn));
     return;
   }
 
@@ -196,6 +215,7 @@ dnsserv_launch_request(const char *name, int reverse,
 
   /* Make a new dummy AP connection, and attach the request to it. */
   entry_conn = entry_connection_new(CONN_TYPE_AP, AF_INET);
+  entry_conn->entry_cfg.dns_request = 1;
   conn = ENTRY_TO_EDGE_CONN(entry_conn);
   CONNECTION_AP_EXPECT_NONPENDING(entry_conn);
   conn->base_.state = AP_CONN_STATE_RESOLVE_WAIT;
@@ -214,10 +234,10 @@ dnsserv_launch_request(const char *name, int reverse,
     TO_CONN(conn)->port = control_conn->base_.port;
     TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr);
   }
-#else
+#else /* !(defined(AF_UNIX)) */
   TO_CONN(conn)->port = control_conn->base_.port;
   TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr);
-#endif
+#endif /* defined(AF_UNIX) */
 
   if (reverse)
     entry_conn->socks_request->command = SOCKS_COMMAND_RESOLVE_PTR;
@@ -237,7 +257,7 @@ dnsserv_launch_request(const char *name, int reverse,
 
   if (connection_add(TO_CONN(conn))<0) {
     log_warn(LD_APP, "Couldn't register dummy connection for RESOLVE request");
-    connection_free(TO_CONN(conn));
+    connection_free_(TO_CONN(conn));
     return -1;
   }
 
@@ -272,7 +292,7 @@ dnsserv_reject_request(entry_connection_t *conn)
 }
 
 /** Look up the original name that corresponds to 'addr' in req.  We use this
- * to preserve case in order to facilitate people using 0x20-hacks to avoid
+ * to preserve case in order to facilitate clients using 0x20-hacks to avoid
  * DNS poisoning. */
 static const char *
 evdns_get_orig_address(const struct evdns_server_request *req,
@@ -393,4 +413,3 @@ dnsserv_close_listener(connection_t *conn)
     listener_conn->dns_server_port = NULL;
   }
 }
-
diff --git a/src/or/dnsserv.h b/src/feature/client/dnsserv.h
index ad0e248c83..fff1ed2adb 100644
--- a/src/or/dnsserv.h
+++ b/src/feature/client/dnsserv.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -23,5 +23,5 @@ void dnsserv_reject_request(entry_connection_t *conn);
 int dnsserv_launch_request(const char *name, int is_reverse,
                            control_connection_t *control_conn);
 
-#endif
+#endif /* !defined(TOR_DNSSERV_H) */
 
diff --git a/src/feature/client/entrynodes.c b/src/feature/client/entrynodes.c
new file mode 100644
index 0000000000..1bfb62538e
--- /dev/null
+++ b/src/feature/client/entrynodes.c
@@ -0,0 +1,3825 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file entrynodes.c
+ * \brief Code to manage our fixed first nodes for various functions.
+ *
+ * Entry nodes can be guards (for general use) or bridges (for censorship
+ * circumvention).
+ *
+ * In general, we use entry guards to prevent traffic-sampling attacks:
+ * if we chose every circuit independently, an adversary controlling
+ * some fraction of paths on the network would observe a sample of every
+ * user's traffic. Using guards gives users a chance of not being
+ * profiled.
+ *
+ * The current entry guard selection code is designed to try to avoid
+ * _ever_ trying every guard on the network, to try to stick to guards
+ * that we've used before, to handle hostile/broken networks, and
+ * to behave sanely when the network goes up and down.
+ *
+ * Our algorithm works as follows: First, we maintain a SAMPLE of guards
+ * we've seen in the networkstatus consensus.  We maintain this sample
+ * over time, and store it persistently; it is chosen without reference
+ * to our configuration or firewall rules.  Guards remain in the sample
+ * as they enter and leave the consensus.  We expand this sample as
+ * needed, up to a maximum size.
+ *
+ * As a subset of the sample, we maintain a FILTERED SET of the guards
+ * that we would be willing to use if we could connect to them.  The
+ * filter removes all the guards that we're excluding because they're
+ * bridges (or not bridges), because we have restrictive firewall rules,
+ * because of ExcludeNodes, because we of path bias restrictions,
+ * because they're absent from the network at present, and so on.
+ *
+ * As a subset of the filtered set, we keep a REACHABLE FILTERED SET
+ * (also called a "usable filtered set") of those guards that we call
+ * "reachable" or "maybe reachable".  A guard is reachable if we've
+ * connected to it more recently than we've failed.  A guard is "maybe
+ * reachable" if we have never tried to connect to it, or if we
+ * failed to connect to it so long ago that we no longer think our
+ * failure means it's down.
+ *
+ * As a persistent ordered list whose elements are taken from the
+ * sampled set, we track a CONFIRMED GUARDS LIST.  A guard becomes
+ * confirmed when we successfully build a circuit through it, and decide
+ * to use that circuit.  We order the guards on this list by the order
+ * in which they became confirmed.
+ *
+ * And as a final group, we have an ordered list of PRIMARY GUARDS,
+ * whose elements are taken from the filtered set. We prefer
+ * confirmed guards to non-confirmed guards for this list, and place
+ * other restrictions on it.  The primary guards are the ones that we
+ * connect to "when nothing is wrong" -- circuits through them can be used
+ * immediately.
+ *
+ * To build circuits, we take a primary guard if possible -- or a
+ * reachable filtered confirmed guard if no primary guard is possible --
+ * or a random reachable filtered guard otherwise.  If the guard is
+ * primary, we can use the circuit immediately on success.  Otherwise,
+ * the guard is now "pending" -- we won't use its circuit unless all
+ * of the circuits we're trying to build through better guards have
+ * definitely failed.
+ *
+ * While we're building circuits, we track a little "guard state" for
+ * each circuit. We use this to keep track of whether the circuit is
+ * one that we can use as soon as it's done, or whether it's one that
+ * we should keep around to see if we can do better.  In the latter case,
+ * a periodic call to entry_guards_upgrade_waiting_circuits() will
+ * eventually upgrade it.
+ **/
+/* DOCDOC -- expand this.
+ *
+ * Information invariants:
+ *
+ * [x] whenever a guard becomes unreachable, clear its usable_filtered flag.
+ *
+ * [x] Whenever a guard becomes reachable or maybe-reachable, if its filtered
+ * flag is set, set its usable_filtered flag.
+ *
+ * [x] Whenever we get a new consensus, call update_from_consensus(). (LATER.)
+ *
+ * [x] Whenever the configuration changes in a relevant way, update the
+ * filtered/usable flags. (LATER.)
+ *
+ * [x] Whenever we add a guard to the sample, make sure its filtered/usable
+ * flags are set as possible.
+ *
+ * [x] Whenever we remove a guard from the sample, remove it from the primary
+ * and confirmed lists.
+ *
+ * [x] When we make a guard confirmed, update the primary list.
+ *
+ * [x] When we make a guard filtered or unfiltered, update the primary list.
+ *
+ * [x] When we are about to pick a guard, make sure that the primary list is
+ * full.
+ *
+ * [x] Before calling sample_reachable_filtered_entry_guards(), make sure
+ * that the filtered, primary, and confirmed flags are up-to-date.
+ *
+ * [x] Call entry_guard_consider_retry every time we are about to check
+ * is_usable_filtered or is_reachable, and every time we set
+ * is_filtered to 1.
+ *
+ * [x] Call entry_guards_changed_for_guard_selection() whenever we update
+ * a persistent field.
+ */
+
+#define ENTRYNODES_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "app/config/confparse.h"
+#include "app/config/statefile.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/channel.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitstats.h"
+#include "core/or/circuituse.h"
+#include "core/or/policies.h"
+#include "feature/client/bridges.h"
+#include "feature/client/circpathbias.h"
+#include "feature/client/entrynodes.h"
+#include "feature/client/transports.h"
+#include "feature/control/control.h"
+#include "feature/dircommon/directory.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nickname.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/node_select.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/relay/router.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/digestset.h"
+#include "lib/encoding/confline.h"
+#include "lib/math/fp.h"
+
+#include "feature/nodelist/node_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "app/config/or_state_st.h"
+
+/** A list of existing guard selection contexts. */
+static smartlist_t *guard_contexts = NULL;
+/** The currently enabled guard selection context. */
+static guard_selection_t *curr_guard_context = NULL;
+
+/** A value of 1 means that at least one context has changed,
+ * and those changes need to be flushed to disk. */
+static int entry_guards_dirty = 0;
+
+static void entry_guard_set_filtered_flags(const or_options_t *options,
+                                           guard_selection_t *gs,
+                                           entry_guard_t *guard);
+static void pathbias_check_use_success_count(entry_guard_t *guard);
+static void pathbias_check_close_success_count(entry_guard_t *guard);
+static int node_is_possible_guard(const node_t *node);
+static int node_passes_guard_filter(const or_options_t *options,
+                                    const node_t *node);
+static entry_guard_t *entry_guard_add_to_sample_impl(guard_selection_t *gs,
+                               const uint8_t *rsa_id_digest,
+                               const char *nickname,
+                               const tor_addr_port_t *bridge_addrport);
+static entry_guard_t *get_sampled_guard_by_bridge_addr(guard_selection_t *gs,
+                                              const tor_addr_port_t *addrport);
+static int entry_guard_obeys_restriction(const entry_guard_t *guard,
+                                         const entry_guard_restriction_t *rst);
+
+/** Return 0 if we should apply guardfraction information found in the
+ *  consensus. A specific consensus can be specified with the
+ *  <b>ns</b> argument, if NULL the most recent one will be picked.*/
+int
+should_apply_guardfraction(const networkstatus_t *ns)
+{
+  /* We need to check the corresponding torrc option and the consensus
+   * parameter if we need to. */
+  const or_options_t *options = get_options();
+
+  /* If UseGuardFraction is 'auto' then check the same-named consensus
+   * parameter. If the consensus parameter is not present, default to
+   * "off". */
+  if (options->UseGuardFraction == -1) {
+    return networkstatus_get_param(ns, "UseGuardFraction",
+                                   0, /* default to "off" */
+                                   0, 1);
+  }
+
+  return options->UseGuardFraction;
+}
+
+/** Return true iff we know a preferred descriptor for <b>guard</b> */
+static int
+guard_has_descriptor(const entry_guard_t *guard)
+{
+  const node_t *node = node_get_by_id(guard->identity);
+  if (!node)
+    return 0;
+  return node_has_preferred_descriptor(node, 1);
+}
+
+/**
+ * Try to determine the correct type for a selection named "name",
+ * if <b>type</b> is GS_TYPE_INFER.
+ */
+STATIC guard_selection_type_t
+guard_selection_infer_type(guard_selection_type_t type,
+                           const char *name)
+{
+  if (type == GS_TYPE_INFER) {
+    if (!strcmp(name, "bridges"))
+      type = GS_TYPE_BRIDGE;
+    else if (!strcmp(name, "restricted"))
+      type = GS_TYPE_RESTRICTED;
+    else
+      type = GS_TYPE_NORMAL;
+  }
+  return type;
+}
+
+/**
+ * Allocate and return a new guard_selection_t, with the name <b>name</b>.
+ */
+STATIC guard_selection_t *
+guard_selection_new(const char *name,
+                    guard_selection_type_t type)
+{
+  guard_selection_t *gs;
+
+  type = guard_selection_infer_type(type, name);
+
+  gs = tor_malloc_zero(sizeof(*gs));
+  gs->name = tor_strdup(name);
+  gs->type = type;
+  gs->sampled_entry_guards = smartlist_new();
+  gs->confirmed_entry_guards = smartlist_new();
+  gs->primary_entry_guards = smartlist_new();
+
+  return gs;
+}
+
+/**
+ * Return the guard selection called <b>name</b>. If there is none, and
+ * <b>create_if_absent</b> is true, then create and return it.  If there
+ * is none, and <b>create_if_absent</b> is false, then return NULL.
+ */
+STATIC guard_selection_t *
+get_guard_selection_by_name(const char *name,
+                            guard_selection_type_t type,
+                            int create_if_absent)
+{
+  if (!guard_contexts) {
+    guard_contexts = smartlist_new();
+  }
+  SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+    if (!strcmp(gs->name, name))
+      return gs;
+  } SMARTLIST_FOREACH_END(gs);
+
+  if (! create_if_absent)
+    return NULL;
+
+  log_debug(LD_GUARD, "Creating a guard selection called %s", name);
+  guard_selection_t *new_selection = guard_selection_new(name, type);
+  smartlist_add(guard_contexts, new_selection);
+
+  return new_selection;
+}
+
+/**
+ * Allocate the first guard context that we're planning to use,
+ * and make it the current context.
+ */
+static void
+create_initial_guard_context(void)
+{
+  tor_assert(! curr_guard_context);
+  if (!guard_contexts) {
+    guard_contexts = smartlist_new();
+  }
+  guard_selection_type_t type = GS_TYPE_INFER;
+  const char *name = choose_guard_selection(
+                             get_options(),
+                             networkstatus_get_reasonably_live_consensus(
+                                                    approx_time(),
+                                                    usable_consensus_flavor()),
+                             NULL,
+                             &type);
+  tor_assert(name); // "name" can only be NULL if we had an old name.
+  tor_assert(type != GS_TYPE_INFER);
+  log_notice(LD_GUARD, "Starting with guard context \"%s\"", name);
+  curr_guard_context = get_guard_selection_by_name(name, type, 1);
+}
+
+/** Get current default guard_selection_t, creating it if necessary */
+guard_selection_t *
+get_guard_selection_info(void)
+{
+  if (!curr_guard_context) {
+    create_initial_guard_context();
+  }
+
+  return curr_guard_context;
+}
+
+/** Return a statically allocated human-readable description of <b>guard</b>
+ */
+const char *
+entry_guard_describe(const entry_guard_t *guard)
+{
+  static char buf[256];
+  tor_snprintf(buf, sizeof(buf),
+               "%s ($%s)",
+               strlen(guard->nickname) ? guard->nickname : "[bridge]",
+               hex_str(guard->identity, DIGEST_LEN));
+  return buf;
+}
+
+/** Return <b>guard</b>'s 20-byte RSA identity digest */
+const char *
+entry_guard_get_rsa_id_digest(const entry_guard_t *guard)
+{
+  return guard->identity;
+}
+
+/** Return the pathbias state associated with <b>guard</b>. */
+guard_pathbias_t *
+entry_guard_get_pathbias_state(entry_guard_t *guard)
+{
+  return &guard->pb;
+}
+
+HANDLE_IMPL(entry_guard, entry_guard_t, ATTR_UNUSED STATIC)
+
+/** Return an interval betweeen 'now' and 'max_backdate' seconds in the past,
+ * chosen uniformly at random.  We use this before recording persistent
+ * dates, so that we aren't leaking exactly when we recorded it.
+ */
+MOCK_IMPL(STATIC time_t,
+randomize_time,(time_t now, time_t max_backdate))
+{
+  tor_assert(max_backdate > 0);
+
+  time_t earliest = now - max_backdate;
+  time_t latest = now;
+  if (earliest <= 0)
+    earliest = 1;
+  if (latest <= earliest)
+    latest = earliest + 1;
+
+  return crypto_rand_time_range(earliest, latest);
+}
+
+/**
+ * @name parameters for networkstatus algorithm
+ *
+ * These parameters are taken from the consensus; some are overrideable in
+ * the torrc.
+ */
+/**@{*/
+/**
+ * We never let our sampled guard set grow larger than this fraction
+ * of the guards on the network.
+ */
+STATIC double
+get_max_sample_threshold(void)
+{
+  int32_t pct =
+    networkstatus_get_param(NULL, "guard-max-sample-threshold-percent",
+                            DFLT_MAX_SAMPLE_THRESHOLD_PERCENT,
+                            1, 100);
+  return pct / 100.0;
+}
+/**
+ * We never let our sampled guard set grow larger than this number.
+ */
+STATIC int
+get_max_sample_size_absolute(void)
+{
+  return (int) networkstatus_get_param(NULL, "guard-max-sample-size",
+                                       DFLT_MAX_SAMPLE_SIZE,
+                                       1, INT32_MAX);
+}
+/**
+ * We always try to make our sample contain at least this many guards.
+ */
+STATIC int
+get_min_filtered_sample_size(void)
+{
+  return networkstatus_get_param(NULL, "guard-min-filtered-sample-size",
+                                 DFLT_MIN_FILTERED_SAMPLE_SIZE,
+                                 1, INT32_MAX);
+}
+/**
+ * If a guard is unlisted for this many days in a row, we remove it.
+ */
+STATIC int
+get_remove_unlisted_guards_after_days(void)
+{
+  return networkstatus_get_param(NULL,
+                                 "guard-remove-unlisted-guards-after-days",
+                                 DFLT_REMOVE_UNLISTED_GUARDS_AFTER_DAYS,
+                                 1, 365*10);
+}
+
+/**
+ * Return number of seconds that will make a guard no longer eligible
+ * for selection if unlisted for this long.
+ */
+static time_t
+get_remove_unlisted_guards_after_seconds(void)
+{
+  return get_remove_unlisted_guards_after_days() * 24 * 60 * 60;
+}
+
+/**
+ * We remove unconfirmed guards from the sample after this many days,
+ * regardless of whether they are listed or unlisted.
+ */
+STATIC int
+get_guard_lifetime(void)
+{
+  if (get_options()->GuardLifetime >= 86400)
+    return get_options()->GuardLifetime;
+  int32_t days;
+  days = networkstatus_get_param(NULL,
+                                 "guard-lifetime-days",
+                                 DFLT_GUARD_LIFETIME_DAYS, 1, 365*10);
+  return days * 86400;
+}
+/**
+ * We remove confirmed guards from the sample if they were sampled
+ * GUARD_LIFETIME_DAYS ago and confirmed this many days ago.
+ */
+STATIC int
+get_guard_confirmed_min_lifetime(void)
+{
+  if (get_options()->GuardLifetime >= 86400)
+    return get_options()->GuardLifetime;
+  int32_t days;
+  days = networkstatus_get_param(NULL, "guard-confirmed-min-lifetime-days",
+                                 DFLT_GUARD_CONFIRMED_MIN_LIFETIME_DAYS,
+                                 1, 365*10);
+  return days * 86400;
+}
+/**
+ * How many guards do we try to keep on our primary guard list?
+ */
+STATIC int
+get_n_primary_guards(void)
+{
+  /* If the user has explicitly configured the number of primary guards, do
+   * what the user wishes to do */
+  const int configured_primaries = get_options()->NumPrimaryGuards;
+  if (configured_primaries) {
+    return configured_primaries;
+  }
+
+  /* otherwise check for consensus parameter and if that's not set either, just
+   * use the default value. */
+  return networkstatus_get_param(NULL,
+                                 "guard-n-primary-guards",
+                                 DFLT_N_PRIMARY_GUARDS, 1, INT32_MAX);
+}
+/**
+ * Return the number of the live primary guards we should look at when
+ * making a circuit.
+ */
+STATIC int
+get_n_primary_guards_to_use(guard_usage_t usage)
+{
+  int configured;
+  const char *param_name;
+  int param_default;
+
+  /* If the user has explicitly configured the amount of guards, use
+     that. Otherwise, fall back to the default value. */
+  if (usage == GUARD_USAGE_DIRGUARD) {
+    configured = get_options()->NumDirectoryGuards;
+    param_name = "guard-n-primary-dir-guards-to-use";
+    param_default = DFLT_N_PRIMARY_DIR_GUARDS_TO_USE;
+  } else {
+    configured = get_options()->NumEntryGuards;
+    param_name = "guard-n-primary-guards-to-use";
+    param_default = DFLT_N_PRIMARY_GUARDS_TO_USE;
+  }
+  if (configured >= 1) {
+    return configured;
+  }
+  return networkstatus_get_param(NULL,
+                                 param_name, param_default, 1, INT32_MAX);
+}
+/**
+ * If we haven't successfully built or used a circuit in this long, then
+ * consider that the internet is probably down.
+ */
+STATIC int
+get_internet_likely_down_interval(void)
+{
+  return networkstatus_get_param(NULL, "guard-internet-likely-down-interval",
+                                 DFLT_INTERNET_LIKELY_DOWN_INTERVAL,
+                                 1, INT32_MAX);
+}
+/**
+ * If we're trying to connect to a nonprimary guard for at least this
+ * many seconds, and we haven't gotten the connection to work, we will treat
+ * lower-priority guards as usable.
+ */
+STATIC int
+get_nonprimary_guard_connect_timeout(void)
+{
+  return networkstatus_get_param(NULL,
+                                 "guard-nonprimary-guard-connect-timeout",
+                                 DFLT_NONPRIMARY_GUARD_CONNECT_TIMEOUT,
+                                 1, INT32_MAX);
+}
+/**
+ * If a circuit has been sitting around in 'waiting for better guard' state
+ * for at least this long, we'll expire it.
+ */
+STATIC int
+get_nonprimary_guard_idle_timeout(void)
+{
+  return networkstatus_get_param(NULL,
+                                 "guard-nonprimary-guard-idle-timeout",
+                                 DFLT_NONPRIMARY_GUARD_IDLE_TIMEOUT,
+                                 1, INT32_MAX);
+}
+/**
+ * If our configuration retains fewer than this fraction of guards from the
+ * torrc, we are in a restricted setting.
+ */
+STATIC double
+get_meaningful_restriction_threshold(void)
+{
+  int32_t pct = networkstatus_get_param(NULL,
+                                        "guard-meaningful-restriction-percent",
+                                        DFLT_MEANINGFUL_RESTRICTION_PERCENT,
+                                        1, INT32_MAX);
+  return pct / 100.0;
+}
+/**
+ * If our configuration retains fewer than this fraction of guards from the
+ * torrc, we are in an extremely restricted setting, and should warn.
+ */
+STATIC double
+get_extreme_restriction_threshold(void)
+{
+  int32_t pct = networkstatus_get_param(NULL,
+                                        "guard-extreme-restriction-percent",
+                                        DFLT_EXTREME_RESTRICTION_PERCENT,
+                                        1, INT32_MAX);
+  return pct / 100.0;
+}
+
+/* Mark <b>guard</b> as maybe reachable again. */
+static void
+mark_guard_maybe_reachable(entry_guard_t *guard)
+{
+  if (guard->is_reachable != GUARD_REACHABLE_NO) {
+    return;
+  }
+
+  /* Note that we do not clear failing_since: this guard is now only
+   * _maybe-reachable_. */
+  guard->is_reachable = GUARD_REACHABLE_MAYBE;
+  if (guard->is_filtered_guard)
+    guard->is_usable_filtered_guard = 1;
+}
+
+/**
+ * Called when the network comes up after having seemed to be down for
+ * a while: Mark the primary guards as maybe-reachable so that we'll
+ * try them again.
+ */
+STATIC void
+mark_primary_guards_maybe_reachable(guard_selection_t *gs)
+{
+  tor_assert(gs);
+
+  if (!gs->primary_guards_up_to_date)
+    entry_guards_update_primary(gs);
+
+  SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards, entry_guard_t *, guard) {
+    mark_guard_maybe_reachable(guard);
+  } SMARTLIST_FOREACH_END(guard);
+}
+
+/* Called when we exhaust all guards in our sampled set: Marks all guards as
+   maybe-reachable so that we 'll try them again. */
+static void
+mark_all_guards_maybe_reachable(guard_selection_t *gs)
+{
+  tor_assert(gs);
+
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    mark_guard_maybe_reachable(guard);
+  } SMARTLIST_FOREACH_END(guard);
+}
+
+/**@}*/
+
+/**
+ * Given our options and our list of nodes, return the name of the
+ * guard selection that we should use.  Return NULL for "use the
+ * same selection you were using before.
+ */
+STATIC const char *
+choose_guard_selection(const or_options_t *options,
+                       const networkstatus_t *live_ns,
+                       const guard_selection_t *old_selection,
+                       guard_selection_type_t *type_out)
+{
+  tor_assert(options);
+  tor_assert(type_out);
+
+  if (options->UseBridges) {
+    *type_out = GS_TYPE_BRIDGE;
+    return "bridges";
+  }
+
+  if (! live_ns) {
+    /* without a networkstatus, we can't tell any more than that. */
+    *type_out = GS_TYPE_NORMAL;
+    return "default";
+  }
+
+  const smartlist_t *nodes = nodelist_get_list();
+  int n_guards = 0, n_passing_filter = 0;
+  SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) {
+    if (node_is_possible_guard(node)) {
+      ++n_guards;
+      if (node_passes_guard_filter(options, node)) {
+        ++n_passing_filter;
+      }
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  /* We use separate 'high' and 'low' thresholds here to prevent flapping
+   * back and forth */
+  const int meaningful_threshold_high =
+    (int)(n_guards * get_meaningful_restriction_threshold() * 1.05);
+  const int meaningful_threshold_mid =
+    (int)(n_guards * get_meaningful_restriction_threshold());
+  const int meaningful_threshold_low =
+    (int)(n_guards * get_meaningful_restriction_threshold() * .95);
+  const int extreme_threshold =
+    (int)(n_guards * get_extreme_restriction_threshold());
+
+  /*
+    If we have no previous selection, then we're "restricted" iff we are
+    below the meaningful restriction threshold.  That's easy enough.
+
+    But if we _do_ have a previous selection, we make it a little
+    "sticky": we only move from "restricted" to "default" when we find
+    that we're above the threshold plus 5%, and we only move from
+    "default" to "restricted" when we're below the threshold minus 5%.
+    That should prevent us from flapping back and forth if we happen to
+    be hovering very close to the default.
+
+    The extreme threshold is for warning only.
+  */
+
+  static int have_warned_extreme_threshold = 0;
+  if (n_guards &&
+      n_passing_filter < extreme_threshold &&
+      ! have_warned_extreme_threshold) {
+    have_warned_extreme_threshold = 1;
+    const double exclude_frac =
+      (n_guards - n_passing_filter) / (double)n_guards;
+    log_warn(LD_GUARD, "Your configuration excludes %d%% of all possible "
+             "guards. That's likely to make you stand out from the "
+             "rest of the world.", (int)(exclude_frac * 100));
+  }
+
+  /* Easy case: no previous selection. Just check if we are in restricted or
+     normal guard selection. */
+  if (old_selection == NULL) {
+    if (n_passing_filter >= meaningful_threshold_mid) {
+      *type_out = GS_TYPE_NORMAL;
+      return "default";
+    } else {
+      *type_out = GS_TYPE_RESTRICTED;
+      return "restricted";
+    }
+  }
+
+  /* Trickier case: we do have a previous guard selection context. */
+  tor_assert(old_selection);
+
+  /* Use high and low thresholds to decide guard selection, and if we fall in
+     the middle then keep the current guard selection context. */
+  if (n_passing_filter >= meaningful_threshold_high) {
+    *type_out = GS_TYPE_NORMAL;
+    return "default";
+  } else if (n_passing_filter < meaningful_threshold_low) {
+    *type_out = GS_TYPE_RESTRICTED;
+    return "restricted";
+  } else {
+    /* we are in the middle: maintain previous guard selection */
+    *type_out = old_selection->type;
+    return old_selection->name;
+  }
+}
+
+/**
+ * Check whether we should switch from our current guard selection to a
+ * different one.  If so, switch and return 1.  Return 0 otherwise.
+ *
+ * On a 1 return, the caller should mark all currently live circuits unusable
+ * for new streams, by calling circuit_mark_all_unused_circs() and
+ * circuit_mark_all_dirty_circs_as_unusable().
+ */
+int
+update_guard_selection_choice(const or_options_t *options)
+{
+  if (!curr_guard_context) {
+    create_initial_guard_context();
+    return 1;
+  }
+
+  guard_selection_type_t type = GS_TYPE_INFER;
+  const char *new_name = choose_guard_selection(
+                             options,
+                             networkstatus_get_reasonably_live_consensus(
+                                                    approx_time(),
+                                                    usable_consensus_flavor()),
+                             curr_guard_context,
+                             &type);
+  tor_assert(new_name);
+  tor_assert(type != GS_TYPE_INFER);
+
+  const char *cur_name = curr_guard_context->name;
+  if (! strcmp(cur_name, new_name)) {
+    log_debug(LD_GUARD,
+              "Staying with guard context \"%s\" (no change)", new_name);
+    return 0; // No change
+  }
+
+  log_notice(LD_GUARD, "Switching to guard context \"%s\" (was using \"%s\")",
+             new_name, cur_name);
+  guard_selection_t *new_guard_context;
+  new_guard_context = get_guard_selection_by_name(new_name, type, 1);
+  tor_assert(new_guard_context);
+  tor_assert(new_guard_context != curr_guard_context);
+  curr_guard_context = new_guard_context;
+
+  return 1;
+}
+
+/**
+ * Return true iff <b>node</b> has all the flags needed for us to consider it
+ * a possible guard when sampling guards.
+ */
+static int
+node_is_possible_guard(const node_t *node)
+{
+  /* The "GUARDS" set is all nodes in the nodelist for which this predicate
+   * holds. */
+
+  tor_assert(node);
+  return (node->is_possible_guard &&
+          node->is_stable &&
+          node->is_fast &&
+          node->is_valid &&
+          node_is_dir(node) &&
+          !router_digest_is_me(node->identity));
+}
+
+/**
+ * Return the sampled guard with the RSA identity digest <b>rsa_id</b>, or
+ * NULL if we don't have one. */
+STATIC entry_guard_t *
+get_sampled_guard_with_id(guard_selection_t *gs,
+                          const uint8_t *rsa_id)
+{
+  tor_assert(gs);
+  tor_assert(rsa_id);
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    if (tor_memeq(guard->identity, rsa_id, DIGEST_LEN))
+      return guard;
+  } SMARTLIST_FOREACH_END(guard);
+  return NULL;
+}
+
+/** If <b>gs</b> contains a sampled entry guard matching <b>bridge</b>,
+ * return that guard. Otherwise return NULL. */
+static entry_guard_t *
+get_sampled_guard_for_bridge(guard_selection_t *gs,
+                             const bridge_info_t *bridge)
+{
+  const uint8_t *id = bridge_get_rsa_id_digest(bridge);
+  const tor_addr_port_t *addrport = bridge_get_addr_port(bridge);
+  entry_guard_t *guard;
+  if (BUG(!addrport))
+    return NULL; // LCOV_EXCL_LINE
+  guard = get_sampled_guard_by_bridge_addr(gs, addrport);
+  if (! guard || (id && tor_memneq(id, guard->identity, DIGEST_LEN)))
+    return NULL;
+  else
+    return guard;
+}
+
+/** If we know a bridge_info_t matching <b>guard</b>, return that
+ * bridge.  Otherwise return NULL. */
+static bridge_info_t *
+get_bridge_info_for_guard(const entry_guard_t *guard)
+{
+  const uint8_t *identity = NULL;
+  if (! tor_digest_is_zero(guard->identity)) {
+    identity = (const uint8_t *)guard->identity;
+  }
+  if (BUG(guard->bridge_addr == NULL))
+    return NULL;
+
+  return get_configured_bridge_by_exact_addr_port_digest(
+                                                 &guard->bridge_addr->addr,
+                                                 guard->bridge_addr->port,
+                                                 (const char*)identity);
+}
+
+/**
+ * Return true iff we have a sampled guard with the RSA identity digest
+ * <b>rsa_id</b>. */
+static inline int
+have_sampled_guard_with_id(guard_selection_t *gs, const uint8_t *rsa_id)
+{
+  return get_sampled_guard_with_id(gs, rsa_id) != NULL;
+}
+
+/**
+ * Allocate a new entry_guard_t object for <b>node</b>, add it to the
+ * sampled entry guards in <b>gs</b>, and return it. <b>node</b> must
+ * not currently be a sampled guard in <b>gs</b>.
+ */
+STATIC entry_guard_t *
+entry_guard_add_to_sample(guard_selection_t *gs,
+                          const node_t *node)
+{
+  log_info(LD_GUARD, "Adding %s to the entry guard sample set.",
+           node_describe(node));
+
+  /* make sure that the guard is not already sampled. */
+  if (BUG(have_sampled_guard_with_id(gs, (const uint8_t*)node->identity)))
+    return NULL; // LCOV_EXCL_LINE
+
+  return entry_guard_add_to_sample_impl(gs,
+                                        (const uint8_t*)node->identity,
+                                        node_get_nickname(node),
+                                        NULL);
+}
+
+/**
+ * Backend: adds a new sampled guard to <b>gs</b>, with given identity,
+ * nickname, and ORPort.  rsa_id_digest and bridge_addrport are optional, but
+ * we need one of them. nickname is optional. The caller is responsible for
+ * maintaining the size limit of the SAMPLED_GUARDS set.
+ */
+static entry_guard_t *
+entry_guard_add_to_sample_impl(guard_selection_t *gs,
+                               const uint8_t *rsa_id_digest,
+                               const char *nickname,
+                               const tor_addr_port_t *bridge_addrport)
+{
+  const int GUARD_LIFETIME = get_guard_lifetime();
+  tor_assert(gs);
+
+  // XXXX #20827 take ed25519 identity here too.
+
+  /* Make sure we can actually identify the guard. */
+  if (BUG(!rsa_id_digest && !bridge_addrport))
+    return NULL; // LCOV_EXCL_LINE
+
+  entry_guard_t *guard = tor_malloc_zero(sizeof(entry_guard_t));
+
+  /* persistent fields */
+  guard->is_persistent = (rsa_id_digest != NULL);
+  guard->selection_name = tor_strdup(gs->name);
+  if (rsa_id_digest)
+    memcpy(guard->identity, rsa_id_digest, DIGEST_LEN);
+  if (nickname)
+    strlcpy(guard->nickname, nickname, sizeof(guard->nickname));
+  guard->sampled_on_date = randomize_time(approx_time(), GUARD_LIFETIME/10);
+  tor_free(guard->sampled_by_version);
+  guard->sampled_by_version = tor_strdup(VERSION);
+  guard->currently_listed = 1;
+  guard->confirmed_idx = -1;
+
+  /* non-persistent fields */
+  guard->is_reachable = GUARD_REACHABLE_MAYBE;
+  if (bridge_addrport)
+    guard->bridge_addr = tor_memdup(bridge_addrport, sizeof(*bridge_addrport));
+
+  smartlist_add(gs->sampled_entry_guards, guard);
+  guard->in_selection = gs;
+  entry_guard_set_filtered_flags(get_options(), gs, guard);
+  entry_guards_changed_for_guard_selection(gs);
+  return guard;
+}
+
+/**
+ * Add an entry guard to the "bridges" guard selection sample, with
+ * information taken from <b>bridge</b>. Return that entry guard.
+ */
+static entry_guard_t *
+entry_guard_add_bridge_to_sample(guard_selection_t *gs,
+                                 const bridge_info_t *bridge)
+{
+  const uint8_t *id_digest = bridge_get_rsa_id_digest(bridge);
+  const tor_addr_port_t *addrport = bridge_get_addr_port(bridge);
+
+  tor_assert(addrport);
+
+  /* make sure that the guard is not already sampled. */
+  if (BUG(get_sampled_guard_for_bridge(gs, bridge)))
+    return NULL; // LCOV_EXCL_LINE
+
+  return entry_guard_add_to_sample_impl(gs, id_digest, NULL, addrport);
+}
+
+/**
+ * Return the entry_guard_t in <b>gs</b> whose address is <b>addrport</b>,
+ * or NULL if none exists.
+*/
+static entry_guard_t *
+get_sampled_guard_by_bridge_addr(guard_selection_t *gs,
+                                 const tor_addr_port_t *addrport)
+{
+  if (! gs)
+    return NULL;
+  if (BUG(!addrport))
+    return NULL;
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, g) {
+    if (g->bridge_addr && tor_addr_port_eq(addrport, g->bridge_addr))
+      return g;
+  } SMARTLIST_FOREACH_END(g);
+  return NULL;
+}
+
+/** Update the guard subsystem's knowledge of the identity of the bridge
+ * at <b>addrport</b>.  Idempotent.
+ */
+void
+entry_guard_learned_bridge_identity(const tor_addr_port_t *addrport,
+                                    const uint8_t *rsa_id_digest)
+{
+  guard_selection_t *gs = get_guard_selection_by_name("bridges",
+                                                      GS_TYPE_BRIDGE,
+                                                      0);
+  if (!gs)
+    return;
+
+  entry_guard_t *g = get_sampled_guard_by_bridge_addr(gs, addrport);
+  if (!g)
+    return;
+
+  int make_persistent = 0;
+
+  if (tor_digest_is_zero(g->identity)) {
+    memcpy(g->identity, rsa_id_digest, DIGEST_LEN);
+    make_persistent = 1;
+  } else if (tor_memeq(g->identity, rsa_id_digest, DIGEST_LEN)) {
+    /* Nothing to see here; we learned something we already knew. */
+    if (BUG(! g->is_persistent))
+      make_persistent = 1;
+  } else {
+    char old_id[HEX_DIGEST_LEN+1];
+    base16_encode(old_id, sizeof(old_id), g->identity, sizeof(g->identity));
+    log_warn(LD_BUG, "We 'learned' an identity %s for a bridge at %s:%d, but "
+             "we already knew a different one (%s). Ignoring the new info as "
+             "possibly bogus.",
+             hex_str((const char *)rsa_id_digest, DIGEST_LEN),
+             fmt_and_decorate_addr(&addrport->addr), addrport->port,
+             old_id);
+    return; // redundant, but let's be clear: we're not making this persistent.
+  }
+
+  if (make_persistent) {
+    g->is_persistent = 1;
+    entry_guards_changed_for_guard_selection(gs);
+  }
+}
+
+/**
+ * Return the number of sampled guards in <b>gs</b> that are "filtered"
+ * (that is, we're willing to connect to them) and that are "usable"
+ * (that is, either "reachable" or "maybe reachable").
+ *
+ * If a restriction is provided in <b>rst</b>, do not count any guards that
+ * violate it.
+ */
+STATIC int
+num_reachable_filtered_guards(const guard_selection_t *gs,
+                              const entry_guard_restriction_t *rst)
+{
+  int n_reachable_filtered_guards = 0;
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);
+    if (! entry_guard_obeys_restriction(guard, rst))
+      continue;
+    if (guard->is_usable_filtered_guard)
+      ++n_reachable_filtered_guards;
+  } SMARTLIST_FOREACH_END(guard);
+  return n_reachable_filtered_guards;
+}
+
+/** Return the actual maximum size for the sample in <b>gs</b>,
+ * given that we know about <b>n_guards</b> total. */
+static int
+get_max_sample_size(guard_selection_t *gs,
+                    int n_guards)
+{
+  const int using_bridges = (gs->type == GS_TYPE_BRIDGE);
+  const int min_sample = get_min_filtered_sample_size();
+
+  /* If we are in bridge mode, expand our sample set as needed without worrying
+   * about max size. We should respect the user's wishes to use many bridges if
+   * that's what they have specified in their configuration file. */
+  if (using_bridges)
+    return INT_MAX;
+
+  const int max_sample_by_pct = (int)(n_guards * get_max_sample_threshold());
+  const int max_sample_absolute = get_max_sample_size_absolute();
+  const int max_sample = MIN(max_sample_by_pct, max_sample_absolute);
+  if (max_sample < min_sample)
+    return min_sample;
+  else
+    return max_sample;
+}
+
+/**
+ * Return a smartlist of the all the guards that are not currently
+ * members of the sample (GUARDS - SAMPLED_GUARDS).  The elements of
+ * this list are node_t pointers in the non-bridge case, and
+ * bridge_info_t pointers in the bridge case.  Set *<b>n_guards_out/b>
+ * to the number of guards that we found in GUARDS, including those
+ * that were already sampled.
+ */
+static smartlist_t *
+get_eligible_guards(const or_options_t *options,
+                    guard_selection_t *gs,
+                    int *n_guards_out)
+{
+  /* Construct eligible_guards as GUARDS - SAMPLED_GUARDS */
+  smartlist_t *eligible_guards = smartlist_new();
+  int n_guards = 0; // total size of "GUARDS"
+
+  if (gs->type == GS_TYPE_BRIDGE) {
+    const smartlist_t *bridges = bridge_list_get();
+    SMARTLIST_FOREACH_BEGIN(bridges, bridge_info_t *, bridge) {
+      ++n_guards;
+      if (NULL != get_sampled_guard_for_bridge(gs, bridge)) {
+        continue;
+      }
+      smartlist_add(eligible_guards, bridge);
+    } SMARTLIST_FOREACH_END(bridge);
+  } else {
+    const smartlist_t *nodes = nodelist_get_list();
+    const int n_sampled = smartlist_len(gs->sampled_entry_guards);
+
+    /* Build a bloom filter of our current guards: let's keep this O(N). */
+    digestset_t *sampled_guard_ids = digestset_new(n_sampled);
+    SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, const entry_guard_t *,
+                            guard) {
+      digestset_add(sampled_guard_ids, guard->identity);
+    } SMARTLIST_FOREACH_END(guard);
+
+    SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) {
+      if (! node_is_possible_guard(node))
+        continue;
+      if (gs->type == GS_TYPE_RESTRICTED) {
+        /* In restricted mode, we apply the filter BEFORE sampling, so
+         * that we are sampling from the nodes that we might actually
+         * select. If we sampled first, we might wind up with a sample
+         * that didn't include any EntryNodes at all. */
+        if (! node_passes_guard_filter(options, node))
+          continue;
+      }
+      ++n_guards;
+      if (digestset_probably_contains(sampled_guard_ids, node->identity))
+        continue;
+      smartlist_add(eligible_guards, (node_t*)node);
+    } SMARTLIST_FOREACH_END(node);
+
+    /* Now we can free that bloom filter. */
+    digestset_free(sampled_guard_ids);
+  }
+
+  *n_guards_out = n_guards;
+  return eligible_guards;
+}
+
+/** Helper: given a smartlist of either bridge_info_t (if gs->type is
+ * GS_TYPE_BRIDGE) or node_t (otherwise), pick one that can be a guard,
+ * add it as a guard, remove it from the list, and return a new
+ * entry_guard_t.  Return NULL on failure. */
+static entry_guard_t *
+select_and_add_guard_item_for_sample(guard_selection_t *gs,
+                                     smartlist_t *eligible_guards)
+{
+  entry_guard_t *added_guard;
+  if (gs->type == GS_TYPE_BRIDGE) {
+    const bridge_info_t *bridge = smartlist_choose(eligible_guards);
+    if (BUG(!bridge))
+      return NULL; // LCOV_EXCL_LINE
+    smartlist_remove(eligible_guards, bridge);
+    added_guard = entry_guard_add_bridge_to_sample(gs, bridge);
+  } else {
+    const node_t *node =
+      node_sl_choose_by_bandwidth(eligible_guards, WEIGHT_FOR_GUARD);
+    if (BUG(!node))
+      return NULL; // LCOV_EXCL_LINE
+    smartlist_remove(eligible_guards, node);
+    added_guard = entry_guard_add_to_sample(gs, node);
+  }
+
+  return added_guard;
+}
+
+/**
+ * Return true iff we need a consensus to update our guards, but we don't
+ * have one. (We can return 0 here either if the consensus is _not_ missing,
+ * or if we don't need a consensus because we're using bridges.)
+ */
+static int
+reasonably_live_consensus_is_missing(const guard_selection_t *gs)
+{
+  tor_assert(gs);
+  if (gs->type == GS_TYPE_BRIDGE) {
+    /* We don't update bridges from the consensus; they aren't there. */
+    return 0;
+  }
+  return networkstatus_get_reasonably_live_consensus(
+                                            approx_time(),
+                                            usable_consensus_flavor()) == NULL;
+}
+
+/**
+ * Add new guards to the sampled guards in <b>gs</b> until there are
+ * enough usable filtered guards, but never grow the sample beyond its
+ * maximum size.  Return the last guard added, or NULL if none were
+ * added.
+ */
+STATIC entry_guard_t *
+entry_guards_expand_sample(guard_selection_t *gs)
+{
+  tor_assert(gs);
+  const or_options_t *options = get_options();
+
+  if (reasonably_live_consensus_is_missing(gs)) {
+    log_info(LD_GUARD, "Not expanding the sample guard set; we have "
+             "no reasonably live consensus.");
+    return NULL;
+  }
+
+  int n_sampled = smartlist_len(gs->sampled_entry_guards);
+  entry_guard_t *added_guard = NULL;
+  int n_usable_filtered_guards = num_reachable_filtered_guards(gs, NULL);
+  int n_guards = 0;
+  smartlist_t *eligible_guards = get_eligible_guards(options, gs, &n_guards);
+
+  const int max_sample = get_max_sample_size(gs, n_guards);
+  const int min_filtered_sample = get_min_filtered_sample_size();
+
+  log_info(LD_GUARD, "Expanding the sample guard set. We have %d guards "
+           "in the sample, and %d eligible guards to extend it with.",
+           n_sampled, smartlist_len(eligible_guards));
+
+  while (n_usable_filtered_guards < min_filtered_sample) {
+    /* Has our sample grown too large to expand? */
+    if (n_sampled >= max_sample) {
+      log_info(LD_GUARD, "Not expanding the guard sample any further; "
+               "just hit the maximum sample threshold of %d",
+               max_sample);
+      goto done;
+    }
+
+    /* Did we run out of guards? */
+    if (smartlist_len(eligible_guards) == 0) {
+      /* LCOV_EXCL_START
+         As long as MAX_SAMPLE_THRESHOLD makes can't be adjusted to
+         allow all guards to be sampled, this can't be reached.
+       */
+      log_info(LD_GUARD, "Not expanding the guard sample any further; "
+               "just ran out of eligible guards");
+      goto done;
+      /* LCOV_EXCL_STOP */
+    }
+
+    /* Otherwise we can add at least one new guard. */
+    added_guard = select_and_add_guard_item_for_sample(gs, eligible_guards);
+    if (!added_guard)
+      goto done; // LCOV_EXCL_LINE -- only fails on BUG.
+
+    ++n_sampled;
+
+    if (added_guard->is_usable_filtered_guard)
+      ++n_usable_filtered_guards;
+  }
+
+ done:
+  smartlist_free(eligible_guards);
+  return added_guard;
+}
+
+/**
+ * Helper: <b>guard</b> has just been removed from the sampled guards:
+ * also remove it from primary and confirmed. */
+static void
+remove_guard_from_confirmed_and_primary_lists(guard_selection_t *gs,
+                                              entry_guard_t *guard)
+{
+  if (guard->is_primary) {
+    guard->is_primary = 0;
+    smartlist_remove_keeporder(gs->primary_entry_guards, guard);
+  } else {
+    if (BUG(smartlist_contains(gs->primary_entry_guards, guard))) {
+      smartlist_remove_keeporder(gs->primary_entry_guards, guard);
+    }
+  }
+
+  if (guard->confirmed_idx >= 0) {
+    smartlist_remove_keeporder(gs->confirmed_entry_guards, guard);
+    guard->confirmed_idx = -1;
+    guard->confirmed_on_date = 0;
+  } else {
+    if (BUG(smartlist_contains(gs->confirmed_entry_guards, guard))) {
+      // LCOV_EXCL_START
+      smartlist_remove_keeporder(gs->confirmed_entry_guards, guard);
+      // LCOV_EXCL_STOP
+    }
+  }
+}
+
+/** Return true iff <b>guard</b> is currently "listed" -- that is, it
+ * appears in the consensus, or as a configured bridge (as
+ * appropriate) */
+MOCK_IMPL(STATIC int,
+entry_guard_is_listed,(guard_selection_t *gs, const entry_guard_t *guard))
+{
+  if (gs->type == GS_TYPE_BRIDGE) {
+    return NULL != get_bridge_info_for_guard(guard);
+  } else {
+    const node_t *node = node_get_by_id(guard->identity);
+
+    return node && node_is_possible_guard(node);
+  }
+}
+
+/**
+ * Enumerate <b>sampled_entry_guards</b> smartlist in <b>gs</b>.
+ * For each <b>entry_guard_t</b> object in smartlist, do the following:
+ *  * Update <b>currently_listed</b> field to reflect if guard is listed
+ *    in guard selection <b>gs</b>.
+ *  * Set <b>unlisted_since_date</b> to approximate UNIX time of
+ *    unlisting if guard is unlisted (randomize within 20% of
+ *    get_remove_unlisted_guards_after_seconds()). Otherwise,
+ *    set it to 0.
+ *
+ * Require <b>gs</b> to be non-null pointer.
+ * Return a number of entries updated.
+ */
+static size_t
+sampled_guards_update_consensus_presence(guard_selection_t *gs)
+{
+  size_t n_changes = 0;
+
+  tor_assert(gs);
+
+  const time_t unlisted_since_slop =
+    get_remove_unlisted_guards_after_seconds() /  5;
+
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    /* XXXX #20827 check ed ID too */
+    const int is_listed = entry_guard_is_listed(gs, guard);
+
+    if (is_listed && ! guard->currently_listed) {
+      ++n_changes;
+      guard->currently_listed = 1;
+      guard->unlisted_since_date = 0;
+      log_info(LD_GUARD, "Sampled guard %s is now listed again.",
+               entry_guard_describe(guard));
+    } else if (!is_listed && guard->currently_listed) {
+      ++n_changes;
+      guard->currently_listed = 0;
+      guard->unlisted_since_date = randomize_time(approx_time(),
+                                                  unlisted_since_slop);
+      log_info(LD_GUARD, "Sampled guard %s is now unlisted.",
+               entry_guard_describe(guard));
+    } else if (is_listed && guard->currently_listed) {
+      log_debug(LD_GUARD, "Sampled guard %s is still listed.",
+               entry_guard_describe(guard));
+    } else {
+      tor_assert(! is_listed && ! guard->currently_listed);
+      log_debug(LD_GUARD, "Sampled guard %s is still unlisted.",
+                entry_guard_describe(guard));
+    }
+
+    /* Clean up unlisted_since_date, just in case. */
+    if (guard->currently_listed && guard->unlisted_since_date) {
+      ++n_changes;
+      guard->unlisted_since_date = 0;
+      log_warn(LD_BUG, "Sampled guard %s was listed, but with "
+               "unlisted_since_date set. Fixing.",
+               entry_guard_describe(guard));
+    } else if (!guard->currently_listed && ! guard->unlisted_since_date) {
+      ++n_changes;
+      guard->unlisted_since_date = randomize_time(approx_time(),
+                                                  unlisted_since_slop);
+      log_warn(LD_BUG, "Sampled guard %s was unlisted, but with "
+               "unlisted_since_date unset. Fixing.",
+               entry_guard_describe(guard));
+    }
+  } SMARTLIST_FOREACH_END(guard);
+
+  return n_changes;
+}
+
+/**
+ * Enumerate <b>sampled_entry_guards</b> smartlist in <b>gs</b>.
+ * For each <b>entry_guard_t</b> object in smartlist, do the following:
+ * * If <b>currently_listed</b> is false and <b>unlisted_since_date</b>
+ *   is earlier than <b>remove_if_unlisted_since</b> - remove it.
+ * * Otherwise, check if <b>sampled_on_date</b> is earlier than
+ *   <b>maybe_remove_if_sampled_before</b>.
+ *   * When above condition is correct, remove the guard if:
+ *     * It was never confirmed.
+ *     * It was confirmed before <b>remove_if_confirmed_before</b>.
+ *
+ * Require <b>gs</b> to be non-null pointer.
+ * Return number of entries deleted.
+ */
+static size_t
+sampled_guards_prune_obsolete_entries(guard_selection_t *gs,
+                                  const time_t remove_if_unlisted_since,
+                                  const time_t maybe_remove_if_sampled_before,
+                                  const time_t remove_if_confirmed_before)
+{
+  size_t n_changes = 0;
+
+  tor_assert(gs);
+
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    int rmv = 0;
+
+    if (guard->currently_listed == 0 &&
+        guard->unlisted_since_date < remove_if_unlisted_since) {
+      /*
+        "We have a live consensus, and {IS_LISTED} is false, and
+         {FIRST_UNLISTED_AT} is over get_remove_unlisted_guards_after_days()
+         days in the past."
+      */
+      log_info(LD_GUARD, "Removing sampled guard %s: it has been unlisted "
+               "for over %d days", entry_guard_describe(guard),
+               get_remove_unlisted_guards_after_days());
+      rmv = 1;
+    } else if (guard->sampled_on_date < maybe_remove_if_sampled_before) {
+      /* We have a live consensus, and {ADDED_ON_DATE} is over
+        {GUARD_LIFETIME} ago, *and* {CONFIRMED_ON_DATE} is either
+        "never", or over {GUARD_CONFIRMED_MIN_LIFETIME} ago.
+      */
+      if (guard->confirmed_on_date == 0) {
+        rmv = 1;
+        log_info(LD_GUARD, "Removing sampled guard %s: it was sampled "
+                 "over %d days ago, but never confirmed.",
+                 entry_guard_describe(guard),
+                 get_guard_lifetime() / 86400);
+      } else if (guard->confirmed_on_date < remove_if_confirmed_before) {
+        rmv = 1;
+        log_info(LD_GUARD, "Removing sampled guard %s: it was sampled "
+                 "over %d days ago, and confirmed over %d days ago.",
+                 entry_guard_describe(guard),
+                 get_guard_lifetime() / 86400,
+                 get_guard_confirmed_min_lifetime() / 86400);
+      }
+    }
+
+    if (rmv) {
+      ++n_changes;
+      SMARTLIST_DEL_CURRENT(gs->sampled_entry_guards, guard);
+      remove_guard_from_confirmed_and_primary_lists(gs, guard);
+      entry_guard_free(guard);
+    }
+  } SMARTLIST_FOREACH_END(guard);
+
+  return n_changes;
+}
+
+/**
+ * Update the status of all sampled guards based on the arrival of a
+ * new consensus networkstatus document.  This will include marking
+ * some guards as listed or unlisted, and removing expired guards. */
+STATIC void
+sampled_guards_update_from_consensus(guard_selection_t *gs)
+{
+  tor_assert(gs);
+
+  // It's important to use a reasonably live consensus here; we want clients
+  // to bootstrap even if their clock is skewed by more than 2-3 hours.
+  // But we don't want to make changes based on anything that's really old.
+  if (reasonably_live_consensus_is_missing(gs)) {
+    log_info(LD_GUARD, "Not updating the sample guard set; we have "
+             "no reasonably live consensus.");
+    return;
+  }
+  log_info(LD_GUARD, "Updating sampled guard status based on received "
+           "consensus.");
+
+  /* First: Update listed/unlisted. */
+  size_t n_changes = sampled_guards_update_consensus_presence(gs);
+
+  const time_t remove_if_unlisted_since =
+    approx_time() - get_remove_unlisted_guards_after_seconds();
+  const time_t maybe_remove_if_sampled_before =
+    approx_time() - get_guard_lifetime();
+  const time_t remove_if_confirmed_before =
+    approx_time() - get_guard_confirmed_min_lifetime();
+
+  /* Then: remove the ones that have been junk for too long */
+  n_changes +=
+    sampled_guards_prune_obsolete_entries(gs,
+                                          remove_if_unlisted_since,
+                                          maybe_remove_if_sampled_before,
+                                          remove_if_confirmed_before);
+
+  if (n_changes) {
+    gs->primary_guards_up_to_date = 0;
+    entry_guards_update_filtered_sets(gs);
+    /* We don't need to rebuild the confirmed list right here -- we may have
+     * removed confirmed guards above, but we can't have added any new
+     * confirmed guards.
+     */
+    entry_guards_changed_for_guard_selection(gs);
+  }
+}
+
+/**
+ * Return true iff <b>node</b> is a Tor relay that we are configured to
+ * be able to connect to. */
+static int
+node_passes_guard_filter(const or_options_t *options,
+                         const node_t *node)
+{
+  /* NOTE: Make sure that this function stays in sync with
+   * options_transition_affects_entry_guards */
+  if (routerset_contains_node(options->ExcludeNodes, node))
+    return 0;
+
+  if (options->EntryNodes &&
+      !routerset_contains_node(options->EntryNodes, node))
+    return 0;
+
+  if (!fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION, 0))
+    return 0;
+
+  if (node_is_a_configured_bridge(node))
+    return 0;
+
+  return 1;
+}
+
+/** Helper: Return true iff <b>bridge</b> passes our configuration
+ * filter-- if it is a relay that we are configured to be able to
+ * connect to. */
+static int
+bridge_passes_guard_filter(const or_options_t *options,
+                           const bridge_info_t *bridge)
+{
+  tor_assert(bridge);
+  if (!bridge)
+    return 0;
+
+  if (routerset_contains_bridge(options->ExcludeNodes, bridge))
+    return 0;
+
+  /* Ignore entrynodes */
+  const tor_addr_port_t *addrport = bridge_get_addr_port(bridge);
+
+  if (!fascist_firewall_allows_address_addr(&addrport->addr,
+                                            addrport->port,
+                                            FIREWALL_OR_CONNECTION,
+                                            0, 0))
+    return 0;
+
+  return 1;
+}
+
+/**
+ * Return true iff <b>guard</b> is a Tor relay that we are configured to
+ * be able to connect to, and we haven't disabled it for omission from
+ * the consensus or path bias issues. */
+static int
+entry_guard_passes_filter(const or_options_t *options, guard_selection_t *gs,
+                          entry_guard_t *guard)
+{
+  if (guard->currently_listed == 0)
+    return 0;
+  if (guard->pb.path_bias_disabled)
+    return 0;
+
+  if (gs->type == GS_TYPE_BRIDGE) {
+    const bridge_info_t *bridge = get_bridge_info_for_guard(guard);
+    if (bridge == NULL)
+      return 0;
+    return bridge_passes_guard_filter(options, bridge);
+  } else {
+    const node_t *node = node_get_by_id(guard->identity);
+    if (node == NULL) {
+      // This can happen when currently_listed is true, and we're not updating
+      // it because we don't have a live consensus.
+      return 0;
+    }
+
+    return node_passes_guard_filter(options, node);
+  }
+}
+
+/** Return true iff <b>guard</b> is in the same family as <b>node</b>.
+ */
+static int
+guard_in_node_family(const entry_guard_t *guard, const node_t *node)
+{
+  const node_t *guard_node = node_get_by_id(guard->identity);
+  if (guard_node) {
+    return nodes_in_same_family(guard_node, node);
+  } else {
+    /* If we don't have a node_t for the guard node, we might have
+     * a bridge_info_t for it. So let's check to see whether the bridge
+     * address matches has any family issues.
+     *
+     * (Strictly speaking, I believe this check is unnecessary, since we only
+     * use it to avoid the exit's family when building circuits, and we don't
+     * build multihop circuits until we have a routerinfo_t for the
+     * bridge... at which point, we'll also have a node_t for the
+     * bridge. Nonetheless, it seems wise to include it, in case our
+     * assumptions change down the road.  -nickm.)
+     */
+    if (get_options()->EnforceDistinctSubnets && guard->bridge_addr) {
+      tor_addr_t node_addr;
+      node_get_addr(node, &node_addr);
+      if (addrs_in_same_network_family(&node_addr,
+                                       &guard->bridge_addr->addr)) {
+        return 1;
+      }
+    }
+    return 0;
+  }
+}
+
+/* Allocate and return a new exit guard restriction (where <b>exit_id</b> is of
+ * size DIGEST_LEN) */
+STATIC entry_guard_restriction_t *
+guard_create_exit_restriction(const uint8_t *exit_id)
+{
+  entry_guard_restriction_t *rst = NULL;
+  rst = tor_malloc_zero(sizeof(entry_guard_restriction_t));
+  rst->type = RST_EXIT_NODE;
+  memcpy(rst->exclude_id, exit_id, DIGEST_LEN);
+  return rst;
+}
+
+/** If we have fewer than this many possible usable guards, don't set
+ * MD-availability-based restrictions: we might blacklist all of them. */
+#define MIN_GUARDS_FOR_MD_RESTRICTION 10
+
+/** Return true if we should set md dirserver restrictions. We might not want
+ *  to set those if our guard options are too restricted, since we don't want
+ *  to blacklist all of them. */
+static int
+should_set_md_dirserver_restriction(void)
+{
+  const guard_selection_t *gs = get_guard_selection_info();
+  int num_usable_guards = num_reachable_filtered_guards(gs, NULL);
+
+  /* Don't set restriction if too few reachable filtered guards. */
+  if (num_usable_guards < MIN_GUARDS_FOR_MD_RESTRICTION) {
+    log_info(LD_GUARD, "Not setting md restriction: only %d"
+             " usable guards.", num_usable_guards);
+    return 0;
+  }
+
+  /* We have enough usable guards: set MD restriction */
+  return 1;
+}
+
+/** Allocate and return an outdated md guard restriction. Return NULL if no
+ *  such restriction is needed. */
+STATIC entry_guard_restriction_t *
+guard_create_dirserver_md_restriction(void)
+{
+  entry_guard_restriction_t *rst = NULL;
+
+  if (!should_set_md_dirserver_restriction()) {
+    log_debug(LD_GUARD, "Not setting md restriction: too few "
+              "filtered guards.");
+    return NULL;
+  }
+
+  rst = tor_malloc_zero(sizeof(entry_guard_restriction_t));
+  rst->type = RST_OUTDATED_MD_DIRSERVER;
+
+  return rst;
+}
+
+/* Return True if <b>guard</b> obeys the exit restriction <b>rst</b>. */
+static int
+guard_obeys_exit_restriction(const entry_guard_t *guard,
+                             const entry_guard_restriction_t *rst)
+{
+  tor_assert(rst->type == RST_EXIT_NODE);
+
+  // Exclude the exit ID and all of its family.
+  const node_t *node = node_get_by_id((const char*)rst->exclude_id);
+  if (node && guard_in_node_family(guard, node))
+    return 0;
+
+  return tor_memneq(guard->identity, rst->exclude_id, DIGEST_LEN);
+}
+
+/** Return True if <b>guard</b> should be used as a dirserver for fetching
+ *  microdescriptors. */
+static int
+guard_obeys_md_dirserver_restriction(const entry_guard_t *guard)
+{
+  /* If this guard is an outdated dirserver, don't use it. */
+  if (microdesc_relay_is_outdated_dirserver(guard->identity)) {
+    log_info(LD_GENERAL, "Skipping %s dirserver: outdated",
+             hex_str(guard->identity, DIGEST_LEN));
+    return 0;
+  }
+
+  log_debug(LD_GENERAL, "%s dirserver obeys md restrictions",
+            hex_str(guard->identity, DIGEST_LEN));
+
+  return 1;
+}
+
+/**
+ * Return true iff <b>guard</b> obeys the restrictions defined in <b>rst</b>.
+ * (If <b>rst</b> is NULL, there are no restrictions.)
+ */
+static int
+entry_guard_obeys_restriction(const entry_guard_t *guard,
+                              const entry_guard_restriction_t *rst)
+{
+  tor_assert(guard);
+  if (! rst)
+    return 1; // No restriction?  No problem.
+
+  if (rst->type == RST_EXIT_NODE) {
+    return guard_obeys_exit_restriction(guard, rst);
+  } else if (rst->type == RST_OUTDATED_MD_DIRSERVER) {
+    return guard_obeys_md_dirserver_restriction(guard);
+  }
+
+  tor_assert_nonfatal_unreached();
+  return 0;
+}
+
+/**
+ * Update the <b>is_filtered_guard</b> and <b>is_usable_filtered_guard</b>
+ * flags on <b>guard</b>. */
+void
+entry_guard_set_filtered_flags(const or_options_t *options,
+                               guard_selection_t *gs,
+                               entry_guard_t *guard)
+{
+  unsigned was_filtered = guard->is_filtered_guard;
+  guard->is_filtered_guard = 0;
+  guard->is_usable_filtered_guard = 0;
+
+  if (entry_guard_passes_filter(options, gs, guard)) {
+    guard->is_filtered_guard = 1;
+
+    if (guard->is_reachable != GUARD_REACHABLE_NO)
+      guard->is_usable_filtered_guard = 1;
+
+    entry_guard_consider_retry(guard);
+  }
+  log_debug(LD_GUARD, "Updated sampled guard %s: filtered=%d; "
+            "reachable_filtered=%d.", entry_guard_describe(guard),
+            guard->is_filtered_guard, guard->is_usable_filtered_guard);
+
+  if (!bool_eq(was_filtered, guard->is_filtered_guard)) {
+    /* This guard might now be primary or nonprimary. */
+    gs->primary_guards_up_to_date = 0;
+  }
+}
+
+/**
+ * Update the <b>is_filtered_guard</b> and <b>is_usable_filtered_guard</b>
+ * flag on every guard in <b>gs</b>. */
+STATIC void
+entry_guards_update_filtered_sets(guard_selection_t *gs)
+{
+  const or_options_t *options = get_options();
+
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    entry_guard_set_filtered_flags(options, gs, guard);
+  } SMARTLIST_FOREACH_END(guard);
+}
+
+/**
+ * Return a random guard from the reachable filtered sample guards
+ * in <b>gs</b>, subject to the exclusion rules listed in <b>flags</b>.
+ * Return NULL if no such guard can be found.
+ *
+ * Make sure that the sample is big enough, and that all the filter flags
+ * are set correctly, before calling this function.
+ *
+ * If a restriction is provided in <b>rst</b>, do not return any guards that
+ * violate it.
+ **/
+STATIC entry_guard_t *
+sample_reachable_filtered_entry_guards(guard_selection_t *gs,
+                                       const entry_guard_restriction_t *rst,
+                                       unsigned flags)
+{
+  tor_assert(gs);
+  entry_guard_t *result = NULL;
+  const unsigned exclude_confirmed = flags & SAMPLE_EXCLUDE_CONFIRMED;
+  const unsigned exclude_primary = flags & SAMPLE_EXCLUDE_PRIMARY;
+  const unsigned exclude_pending = flags & SAMPLE_EXCLUDE_PENDING;
+  const unsigned no_update_primary = flags & SAMPLE_NO_UPDATE_PRIMARY;
+  const unsigned need_descriptor = flags & SAMPLE_EXCLUDE_NO_DESCRIPTOR;
+
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);
+  } SMARTLIST_FOREACH_END(guard);
+
+  const int n_reachable_filtered = num_reachable_filtered_guards(gs, rst);
+
+  log_info(LD_GUARD, "Trying to sample a reachable guard: We know of %d "
+           "in the USABLE_FILTERED set.", n_reachable_filtered);
+
+  const int min_filtered_sample = get_min_filtered_sample_size();
+  if (n_reachable_filtered < min_filtered_sample) {
+    log_info(LD_GUARD, "  (That isn't enough. Trying to expand the sample.)");
+    entry_guards_expand_sample(gs);
+  }
+
+  if (exclude_primary && !gs->primary_guards_up_to_date && !no_update_primary)
+    entry_guards_update_primary(gs);
+
+  /* Build the set of reachable filtered guards. */
+  smartlist_t *reachable_filtered_sample = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);// redundant, but cheap.
+    if (! entry_guard_obeys_restriction(guard, rst))
+      continue;
+    if (! guard->is_usable_filtered_guard)
+      continue;
+    if (exclude_confirmed && guard->confirmed_idx >= 0)
+      continue;
+    if (exclude_primary && guard->is_primary)
+      continue;
+    if (exclude_pending && guard->is_pending)
+      continue;
+    if (need_descriptor && !guard_has_descriptor(guard))
+      continue;
+    smartlist_add(reachable_filtered_sample, guard);
+  } SMARTLIST_FOREACH_END(guard);
+
+  log_info(LD_GUARD, "  (After filters [%x], we have %d guards to consider.)",
+           flags, smartlist_len(reachable_filtered_sample));
+
+  if (smartlist_len(reachable_filtered_sample)) {
+    result = smartlist_choose(reachable_filtered_sample);
+    log_info(LD_GUARD, "  (Selected %s.)",
+             result ? entry_guard_describe(result) : "<null>");
+  }
+  smartlist_free(reachable_filtered_sample);
+
+  return result;
+}
+
+/**
+ * Helper: compare two entry_guard_t by their confirmed_idx values.
+ * Used to sort the confirmed list.
+ */
+static int
+compare_guards_by_confirmed_idx(const void **a_, const void **b_)
+{
+  const entry_guard_t *a = *a_, *b = *b_;
+  if (a->confirmed_idx < b->confirmed_idx)
+    return -1;
+  else if (a->confirmed_idx > b->confirmed_idx)
+    return 1;
+  else
+    return 0;
+}
+
+/**
+ * Find the confirmed guards from among the sampled guards in <b>gs</b>,
+ * and put them in confirmed_entry_guards in the correct
+ * order. Recalculate their indices.
+ */
+STATIC void
+entry_guards_update_confirmed(guard_selection_t *gs)
+{
+  smartlist_clear(gs->confirmed_entry_guards);
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    if (guard->confirmed_idx >= 0)
+      smartlist_add(gs->confirmed_entry_guards, guard);
+  } SMARTLIST_FOREACH_END(guard);
+
+  smartlist_sort(gs->confirmed_entry_guards, compare_guards_by_confirmed_idx);
+
+  int any_changed = 0;
+  SMARTLIST_FOREACH_BEGIN(gs->confirmed_entry_guards, entry_guard_t *, guard) {
+    if (guard->confirmed_idx != guard_sl_idx) {
+      any_changed = 1;
+      guard->confirmed_idx = guard_sl_idx;
+    }
+  } SMARTLIST_FOREACH_END(guard);
+
+  gs->next_confirmed_idx = smartlist_len(gs->confirmed_entry_guards);
+
+  if (any_changed) {
+    entry_guards_changed_for_guard_selection(gs);
+  }
+}
+
+/**
+ * Mark <b>guard</b> as a confirmed guard -- that is, one that we have
+ * connected to, and intend to use again.
+ */
+STATIC void
+make_guard_confirmed(guard_selection_t *gs, entry_guard_t *guard)
+{
+  if (BUG(guard->confirmed_on_date && guard->confirmed_idx >= 0))
+    return; // LCOV_EXCL_LINE
+
+  if (BUG(smartlist_contains(gs->confirmed_entry_guards, guard)))
+    return; // LCOV_EXCL_LINE
+
+  const int GUARD_LIFETIME = get_guard_lifetime();
+  guard->confirmed_on_date = randomize_time(approx_time(), GUARD_LIFETIME/10);
+
+  log_info(LD_GUARD, "Marking %s as a confirmed guard (index %d)",
+           entry_guard_describe(guard),
+           gs->next_confirmed_idx);
+
+  guard->confirmed_idx = gs->next_confirmed_idx++;
+  smartlist_add(gs->confirmed_entry_guards, guard);
+
+  // This confirmed guard might kick something else out of the primary
+  // guards.
+  gs->primary_guards_up_to_date = 0;
+
+  entry_guards_changed_for_guard_selection(gs);
+}
+
+/**
+ * Recalculate the list of primary guards (the ones we'd prefer to use) from
+ * the filtered sample and the confirmed list.
+ */
+STATIC void
+entry_guards_update_primary(guard_selection_t *gs)
+{
+  tor_assert(gs);
+
+  // prevent recursion. Recursion is potentially very bad here.
+  static int running = 0;
+  tor_assert(!running);
+  running = 1;
+
+  const int N_PRIMARY_GUARDS = get_n_primary_guards();
+
+  smartlist_t *new_primary_guards = smartlist_new();
+  smartlist_t *old_primary_guards = smartlist_new();
+  smartlist_add_all(old_primary_guards, gs->primary_entry_guards);
+
+  /* Set this flag now, to prevent the calls below from recursing. */
+  gs->primary_guards_up_to_date = 1;
+
+  /* First, can we fill it up with confirmed guards? */
+  SMARTLIST_FOREACH_BEGIN(gs->confirmed_entry_guards, entry_guard_t *, guard) {
+    if (smartlist_len(new_primary_guards) >= N_PRIMARY_GUARDS)
+      break;
+    if (! guard->is_filtered_guard)
+      continue;
+    guard->is_primary = 1;
+    smartlist_add(new_primary_guards, guard);
+  } SMARTLIST_FOREACH_END(guard);
+
+  SMARTLIST_FOREACH_BEGIN(old_primary_guards, entry_guard_t *, guard) {
+    /* Can we keep any older primary guards? First remove all the ones
+     * that we already kept. */
+    if (smartlist_contains(new_primary_guards, guard)) {
+      SMARTLIST_DEL_CURRENT_KEEPORDER(old_primary_guards, guard);
+      continue;
+    }
+
+    /* Now add any that are still good. */
+    if (smartlist_len(new_primary_guards) < N_PRIMARY_GUARDS &&
+        guard->is_filtered_guard) {
+      guard->is_primary = 1;
+      smartlist_add(new_primary_guards, guard);
+      SMARTLIST_DEL_CURRENT_KEEPORDER(old_primary_guards, guard);
+    } else {
+      /* Mark the remaining previous primary guards as non-primary */
+      guard->is_primary = 0;
+    }
+  } SMARTLIST_FOREACH_END(guard);
+
+  /* Finally, fill out the list with sampled guards. */
+  while (smartlist_len(new_primary_guards) < N_PRIMARY_GUARDS) {
+    entry_guard_t *guard = sample_reachable_filtered_entry_guards(gs, NULL,
+                                            SAMPLE_EXCLUDE_CONFIRMED|
+                                            SAMPLE_EXCLUDE_PRIMARY|
+                                            SAMPLE_NO_UPDATE_PRIMARY);
+    if (!guard)
+      break;
+    guard->is_primary = 1;
+    smartlist_add(new_primary_guards, guard);
+  }
+
+#if 1
+  /* Debugging. */
+  SMARTLIST_FOREACH(gs->sampled_entry_guards, entry_guard_t *, guard, {
+    tor_assert_nonfatal(
+                   bool_eq(guard->is_primary,
+                           smartlist_contains(new_primary_guards, guard)));
+  });
+#endif /* 1 */
+
+  const int any_change = !smartlist_ptrs_eq(gs->primary_entry_guards,
+                                            new_primary_guards);
+  if (any_change) {
+    log_info(LD_GUARD, "Primary entry guards have changed. "
+             "New primary guard list is: ");
+    int n = smartlist_len(new_primary_guards);
+    SMARTLIST_FOREACH_BEGIN(new_primary_guards, entry_guard_t *, g) {
+      log_info(LD_GUARD, "  %d/%d: %s%s%s",
+               g_sl_idx+1, n, entry_guard_describe(g),
+               g->confirmed_idx >= 0 ? " (confirmed)" : "",
+               g->is_filtered_guard ? "" : " (excluded by filter)");
+    } SMARTLIST_FOREACH_END(g);
+  }
+
+  smartlist_free(old_primary_guards);
+  smartlist_free(gs->primary_entry_guards);
+  gs->primary_entry_guards = new_primary_guards;
+  gs->primary_guards_up_to_date = 1;
+  running = 0;
+}
+
+/**
+ * Return the number of seconds after the last attempt at which we should
+ * retry a guard that has been failing since <b>failing_since</b>.
+ */
+static int
+get_retry_schedule(time_t failing_since, time_t now,
+                   int is_primary)
+{
+  const unsigned SIX_HOURS = 6 * 3600;
+  const unsigned FOUR_DAYS = 4 * 86400;
+  const unsigned SEVEN_DAYS = 7 * 86400;
+
+  time_t tdiff;
+  if (now > failing_since) {
+    tdiff = now - failing_since;
+  } else {
+    tdiff = 0;
+  }
+
+  const struct {
+    time_t maximum; int primary_delay; int nonprimary_delay;
+  } delays[] = {
+    { SIX_HOURS,    10*60,  1*60*60 },
+    { FOUR_DAYS,    90*60,  4*60*60 },
+    { SEVEN_DAYS, 4*60*60, 18*60*60 },
+    { TIME_MAX,   9*60*60, 36*60*60 }
+  };
+
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(delays); ++i) {
+    if (tdiff <= delays[i].maximum) {
+      return is_primary ? delays[i].primary_delay : delays[i].nonprimary_delay;
+    }
+  }
+  /* LCOV_EXCL_START -- can't reach, since delays ends with TIME_MAX. */
+  tor_assert_nonfatal_unreached();
+  return 36*60*60;
+  /* LCOV_EXCL_STOP */
+}
+
+/**
+ * If <b>guard</b> is unreachable, consider whether enough time has passed
+ * to consider it maybe-reachable again.
+ */
+STATIC void
+entry_guard_consider_retry(entry_guard_t *guard)
+{
+  if (guard->is_reachable != GUARD_REACHABLE_NO)
+    return; /* No retry needed. */
+
+  const time_t now = approx_time();
+  const int delay =
+    get_retry_schedule(guard->failing_since, now, guard->is_primary);
+  const time_t last_attempt = guard->last_tried_to_connect;
+
+  if (BUG(last_attempt == 0) ||
+      now >= last_attempt + delay) {
+    /* We should mark this retriable. */
+    char tbuf[ISO_TIME_LEN+1];
+    format_local_iso_time(tbuf, last_attempt);
+    log_info(LD_GUARD, "Marked %s%sguard %s for possible retry, since we "
+             "haven't tried to use it since %s.",
+             guard->is_primary?"primary ":"",
+             guard->confirmed_idx>=0?"confirmed ":"",
+             entry_guard_describe(guard),
+             tbuf);
+
+    guard->is_reachable = GUARD_REACHABLE_MAYBE;
+    if (guard->is_filtered_guard)
+      guard->is_usable_filtered_guard = 1;
+  }
+}
+
+/** Tell the entry guards subsystem that we have confirmed that as of
+ * just now, we're on the internet. */
+void
+entry_guards_note_internet_connectivity(guard_selection_t *gs)
+{
+  gs->last_time_on_internet = approx_time();
+}
+
+/**
+ * Pick a primary guard for use with a circuit, if available. Update the
+ * <b>last_tried_to_connect</b> time and the <b>is_pending</b> fields of the
+ * guard as appropriate.  Set <b>state_out</b> to the new guard-state
+ * of the circuit.
+ */
+static entry_guard_t *
+select_primary_guard_for_circuit(guard_selection_t *gs,
+                                 guard_usage_t usage,
+                                 const entry_guard_restriction_t *rst,
+                                 unsigned *state_out)
+{
+  const int need_descriptor = (usage == GUARD_USAGE_TRAFFIC);
+  entry_guard_t *chosen_guard = NULL;
+
+  int num_entry_guards = get_n_primary_guards_to_use(usage);
+  smartlist_t *usable_primary_guards = smartlist_new();
+
+  SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);
+    if (! entry_guard_obeys_restriction(guard, rst))
+      continue;
+    if (guard->is_reachable != GUARD_REACHABLE_NO) {
+      if (need_descriptor && !guard_has_descriptor(guard)) {
+        continue;
+      }
+      *state_out = GUARD_CIRC_STATE_USABLE_ON_COMPLETION;
+      guard->last_tried_to_connect = approx_time();
+      smartlist_add(usable_primary_guards, guard);
+      if (smartlist_len(usable_primary_guards) >= num_entry_guards)
+        break;
+    }
+  } SMARTLIST_FOREACH_END(guard);
+
+  if (smartlist_len(usable_primary_guards)) {
+    chosen_guard = smartlist_choose(usable_primary_guards);
+    smartlist_free(usable_primary_guards);
+    log_info(LD_GUARD, "Selected primary guard %s for circuit.",
+             entry_guard_describe(chosen_guard));
+  }
+
+  smartlist_free(usable_primary_guards);
+  return chosen_guard;
+}
+
+/**
+ * For use with a circuit, pick a non-pending running filtered confirmed guard,
+ * if one is available. Update the <b>last_tried_to_connect</b> time and the
+ * <b>is_pending</b> fields of the guard as appropriate. Set <b>state_out</b>
+ * to the new guard-state of the circuit.
+ */
+static entry_guard_t *
+select_confirmed_guard_for_circuit(guard_selection_t *gs,
+                                  guard_usage_t usage,
+                                  const entry_guard_restriction_t *rst,
+                                  unsigned *state_out)
+{
+  const int need_descriptor = (usage == GUARD_USAGE_TRAFFIC);
+
+  SMARTLIST_FOREACH_BEGIN(gs->confirmed_entry_guards, entry_guard_t *, guard) {
+    if (guard->is_primary)
+      continue; /* we already considered this one. */
+    if (! entry_guard_obeys_restriction(guard, rst))
+      continue;
+    entry_guard_consider_retry(guard);
+    if (guard->is_usable_filtered_guard && ! guard->is_pending) {
+      if (need_descriptor && !guard_has_descriptor(guard))
+        continue; /* not a bug */
+      guard->is_pending = 1;
+      guard->last_tried_to_connect = approx_time();
+      *state_out = GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD;
+      log_info(LD_GUARD, "No primary guards available. Selected confirmed "
+               "guard %s for circuit. Will try other guards before using "
+               "this circuit.",
+               entry_guard_describe(guard));
+      return guard;
+    }
+  } SMARTLIST_FOREACH_END(guard);
+
+  return NULL;
+}
+
+/**
+ * For use with a circuit, pick a confirmed usable filtered guard
+ * at random. Update the <b>last_tried_to_connect</b> time and the
+ * <b>is_pending</b> fields of the guard as appropriate. Set <b>state_out</b>
+ * to the new guard-state of the circuit.
+ */
+static entry_guard_t *
+select_filtered_guard_for_circuit(guard_selection_t *gs,
+                                  guard_usage_t usage,
+                                  const entry_guard_restriction_t *rst,
+                                  unsigned *state_out)
+{
+  const int need_descriptor = (usage == GUARD_USAGE_TRAFFIC);
+  entry_guard_t *chosen_guard = NULL;
+  unsigned flags = 0;
+  if (need_descriptor)
+    flags |= SAMPLE_EXCLUDE_NO_DESCRIPTOR;
+  chosen_guard = sample_reachable_filtered_entry_guards(gs,
+                                                 rst,
+                                                 SAMPLE_EXCLUDE_CONFIRMED |
+                                                 SAMPLE_EXCLUDE_PRIMARY |
+                                                 SAMPLE_EXCLUDE_PENDING |
+                                                 flags);
+  if (!chosen_guard) {
+    return NULL;
+  }
+
+  chosen_guard->is_pending = 1;
+  chosen_guard->last_tried_to_connect = approx_time();
+  *state_out = GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD;
+  log_info(LD_GUARD, "No primary or confirmed guards available. Selected "
+           "random guard %s for circuit. Will try other guards before "
+           "using this circuit.",
+           entry_guard_describe(chosen_guard));
+  return chosen_guard;
+}
+
+/**
+ * Get a guard for use with a circuit.  Prefer to pick a running primary
+ * guard; then a non-pending running filtered confirmed guard; then a
+ * non-pending runnable filtered guard.  Update the
+ * <b>last_tried_to_connect</b> time and the <b>is_pending</b> fields of the
+ * guard as appropriate.  Set <b>state_out</b> to the new guard-state
+ * of the circuit.
+ */
+STATIC entry_guard_t *
+select_entry_guard_for_circuit(guard_selection_t *gs,
+                               guard_usage_t usage,
+                               const entry_guard_restriction_t *rst,
+                               unsigned *state_out)
+{
+  entry_guard_t *chosen_guard = NULL;
+  tor_assert(gs);
+  tor_assert(state_out);
+
+  if (!gs->primary_guards_up_to_date)
+    entry_guards_update_primary(gs);
+
+  /* "If any entry in PRIMARY_GUARDS has {is_reachable} status of
+      <maybe> or <yes>, return the first such guard." */
+  chosen_guard = select_primary_guard_for_circuit(gs, usage, rst, state_out);
+  if (chosen_guard)
+    return chosen_guard;
+
+  /* "Otherwise, if the ordered intersection of {CONFIRMED_GUARDS}
+      and {USABLE_FILTERED_GUARDS} is nonempty, return the first
+      entry in that intersection that has {is_pending} set to
+      false." */
+  chosen_guard = select_confirmed_guard_for_circuit(gs, usage, rst, state_out);
+  if (chosen_guard)
+    return chosen_guard;
+
+  /* "Otherwise, if there is no such entry, select a member at
+      random from {USABLE_FILTERED_GUARDS}." */
+  chosen_guard = select_filtered_guard_for_circuit(gs, usage, rst, state_out);
+
+  if (chosen_guard == NULL) {
+    log_info(LD_GUARD, "Absolutely no sampled guards were available. "
+             "Marking all guards for retry and starting from top again.");
+    mark_all_guards_maybe_reachable(gs);
+    return NULL;
+  }
+
+  return chosen_guard;
+}
+
+/**
+ * Note that we failed to connect to or build circuits through <b>guard</b>.
+ * Use with a guard returned by select_entry_guard_for_circuit().
+ */
+STATIC void
+entry_guards_note_guard_failure(guard_selection_t *gs,
+                                entry_guard_t *guard)
+{
+  tor_assert(gs);
+
+  guard->is_reachable = GUARD_REACHABLE_NO;
+  guard->is_usable_filtered_guard = 0;
+
+  guard->is_pending = 0;
+  if (guard->failing_since == 0)
+    guard->failing_since = approx_time();
+
+  log_info(LD_GUARD, "Recorded failure for %s%sguard %s",
+           guard->is_primary?"primary ":"",
+           guard->confirmed_idx>=0?"confirmed ":"",
+           entry_guard_describe(guard));
+}
+
+/**
+ * Note that we successfully connected to, and built a circuit through
+ * <b>guard</b>. Given the old guard-state of the circuit in <b>old_state</b>,
+ * return the new guard-state of the circuit.
+ *
+ * Be aware: the circuit is only usable when its guard-state becomes
+ * GUARD_CIRC_STATE_COMPLETE.
+ **/
+STATIC unsigned
+entry_guards_note_guard_success(guard_selection_t *gs,
+                                entry_guard_t *guard,
+                                unsigned old_state)
+{
+  tor_assert(gs);
+
+  /* Save this, since we're about to overwrite it. */
+  const time_t last_time_on_internet = gs->last_time_on_internet;
+  gs->last_time_on_internet = approx_time();
+
+  guard->is_reachable = GUARD_REACHABLE_YES;
+  guard->failing_since = 0;
+  guard->is_pending = 0;
+  if (guard->is_filtered_guard)
+    guard->is_usable_filtered_guard = 1;
+
+  if (guard->confirmed_idx < 0) {
+    make_guard_confirmed(gs, guard);
+    if (!gs->primary_guards_up_to_date)
+      entry_guards_update_primary(gs);
+  }
+
+  unsigned new_state;
+  switch (old_state) {
+    case GUARD_CIRC_STATE_COMPLETE:
+    case GUARD_CIRC_STATE_USABLE_ON_COMPLETION:
+      new_state = GUARD_CIRC_STATE_COMPLETE;
+      break;
+    default:
+      tor_assert_nonfatal_unreached();
+      /* Fall through. */
+    case GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD:
+      if (guard->is_primary) {
+        /* XXXX #20832 -- I don't actually like this logic. It seems to make
+         * us a little more susceptible to evil-ISP attacks.  The mitigations
+         * I'm thinking of, however, aren't local to this point, so I'll leave
+         * it alone. */
+        /* This guard may have become primary by virtue of being confirmed.
+         * If so, the circuit for it is now complete.
+         */
+        new_state = GUARD_CIRC_STATE_COMPLETE;
+      } else {
+        new_state = GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD;
+      }
+      break;
+  }
+
+  if (! guard->is_primary) {
+    if (last_time_on_internet + get_internet_likely_down_interval()
+        < approx_time()) {
+      mark_primary_guards_maybe_reachable(gs);
+    }
+  }
+
+  log_info(LD_GUARD, "Recorded success for %s%sguard %s",
+           guard->is_primary?"primary ":"",
+           guard->confirmed_idx>=0?"confirmed ":"",
+           entry_guard_describe(guard));
+
+  return new_state;
+}
+
+/**
+ * Helper: Return true iff <b>a</b> has higher priority than <b>b</b>.
+ */
+STATIC int
+entry_guard_has_higher_priority(entry_guard_t *a, entry_guard_t *b)
+{
+  tor_assert(a && b);
+  if (a == b)
+    return 0;
+
+  /* Confirmed is always better than unconfirmed; lower index better
+     than higher */
+  if (a->confirmed_idx < 0) {
+    if (b->confirmed_idx >= 0)
+      return 0;
+  } else {
+    if (b->confirmed_idx < 0)
+      return 1;
+
+    /* Lower confirmed_idx is better than higher. */
+    return (a->confirmed_idx < b->confirmed_idx);
+  }
+
+  /* If we reach this point, both are unconfirmed. If one is pending, it
+   * has higher priority. */
+  if (a->is_pending) {
+    if (! b->is_pending)
+      return 1;
+
+    /* Both are pending: earlier last_tried_connect wins. */
+    return a->last_tried_to_connect < b->last_tried_to_connect;
+  } else {
+    if (b->is_pending)
+      return 0;
+
+    /* Neither is pending: priorities are equal. */
+    return 0;
+  }
+}
+
+/** Release all storage held in <b>restriction</b> */
+STATIC void
+entry_guard_restriction_free_(entry_guard_restriction_t *rst)
+{
+  tor_free(rst);
+}
+
+/**
+ * Release all storage held in <b>state</b>.
+ */
+void
+circuit_guard_state_free_(circuit_guard_state_t *state)
+{
+  if (!state)
+    return;
+  entry_guard_restriction_free(state->restrictions);
+  entry_guard_handle_free(state->guard);
+  tor_free(state);
+}
+
+/** Allocate and return a new circuit_guard_state_t to track the result
+ * of using <b>guard</b> for a given operation. */
+MOCK_IMPL(STATIC circuit_guard_state_t *,
+circuit_guard_state_new,(entry_guard_t *guard, unsigned state,
+                         entry_guard_restriction_t *rst))
+{
+  circuit_guard_state_t *result;
+
+  result = tor_malloc_zero(sizeof(circuit_guard_state_t));
+  result->guard = entry_guard_handle_new(guard);
+  result->state = state;
+  result->state_set_at = approx_time();
+  result->restrictions = rst;
+
+  return result;
+}
+
+/**
+ * Pick a suitable entry guard for a circuit in, and place that guard
+ * in *<b>chosen_node_out</b>. Set *<b>guard_state_out</b> to an opaque
+ * state object that will record whether the circuit is ready to be used
+ * or not. Return 0 on success; on failure, return -1.
+ *
+ * If a restriction is provided in <b>rst</b>, do not return any guards that
+ * violate it, and remember that restriction in <b>guard_state_out</b> for
+ * later use. (Takes ownership of the <b>rst</b> object.)
+ */
+int
+entry_guard_pick_for_circuit(guard_selection_t *gs,
+                             guard_usage_t usage,
+                             entry_guard_restriction_t *rst,
+                             const node_t **chosen_node_out,
+                             circuit_guard_state_t **guard_state_out)
+{
+  tor_assert(gs);
+  tor_assert(chosen_node_out);
+  tor_assert(guard_state_out);
+  *chosen_node_out = NULL;
+  *guard_state_out = NULL;
+
+  unsigned state = 0;
+  entry_guard_t *guard =
+    select_entry_guard_for_circuit(gs, usage, rst, &state);
+  if (! guard)
+    goto fail;
+  if (BUG(state == 0))
+    goto fail;
+  const node_t *node = node_get_by_id(guard->identity);
+  // XXXX #20827 check Ed ID.
+  if (! node)
+    goto fail;
+  if (BUG(usage != GUARD_USAGE_DIRGUARD &&
+          !node_has_preferred_descriptor(node, 1)))
+    goto fail;
+
+  *chosen_node_out = node;
+  *guard_state_out = circuit_guard_state_new(guard, state, rst);
+
+  return 0;
+ fail:
+  entry_guard_restriction_free(rst);
+  return -1;
+}
+
+/**
+ * Called by the circuit building module when a circuit has succeeded: informs
+ * the guards code that the guard in *<b>guard_state_p</b> is working, and
+ * advances the state of the guard module.  On a GUARD_USABLE_NEVER return
+ * value, the circuit is broken and should not be used.  On a GUARD_USABLE_NOW
+ * return value, the circuit is ready to use.  On a GUARD_MAYBE_USABLE_LATER
+ * return value, the circuit should not be used until we find out whether
+ * preferred guards will work for us.
+ */
+guard_usable_t
+entry_guard_succeeded(circuit_guard_state_t **guard_state_p)
+{
+  if (BUG(*guard_state_p == NULL))
+    return GUARD_USABLE_NEVER;
+
+  entry_guard_t *guard = entry_guard_handle_get((*guard_state_p)->guard);
+  if (! guard || BUG(guard->in_selection == NULL))
+    return GUARD_USABLE_NEVER;
+
+  unsigned newstate =
+    entry_guards_note_guard_success(guard->in_selection, guard,
+                                    (*guard_state_p)->state);
+
+  (*guard_state_p)->state = newstate;
+  (*guard_state_p)->state_set_at = approx_time();
+
+  if (newstate == GUARD_CIRC_STATE_COMPLETE) {
+    return GUARD_USABLE_NOW;
+  } else {
+    return GUARD_MAYBE_USABLE_LATER;
+  }
+}
+
+/** Cancel the selection of *<b>guard_state_p</b> without declaring
+ * success or failure. It is safe to call this function if success or
+ * failure _has_ already been declared. */
+void
+entry_guard_cancel(circuit_guard_state_t **guard_state_p)
+{
+  if (BUG(*guard_state_p == NULL))
+    return;
+  entry_guard_t *guard = entry_guard_handle_get((*guard_state_p)->guard);
+  if (! guard)
+    return;
+
+  /* XXXX prop271 -- last_tried_to_connect_at will be erroneous here, but this
+   * function will only get called in "bug" cases anyway. */
+  guard->is_pending = 0;
+  circuit_guard_state_free(*guard_state_p);
+  *guard_state_p = NULL;
+}
+
+/**
+ * Called by the circuit building module when a circuit has failed:
+ * informs the guards code that the guard in *<b>guard_state_p</b> is
+ * not working, and advances the state of the guard module.
+ */
+void
+entry_guard_failed(circuit_guard_state_t **guard_state_p)
+{
+  if (BUG(*guard_state_p == NULL))
+    return;
+
+  entry_guard_t *guard = entry_guard_handle_get((*guard_state_p)->guard);
+  if (! guard || BUG(guard->in_selection == NULL))
+    return;
+
+  entry_guards_note_guard_failure(guard->in_selection, guard);
+
+  (*guard_state_p)->state = GUARD_CIRC_STATE_DEAD;
+  (*guard_state_p)->state_set_at = approx_time();
+}
+
+/**
+ * Run the entry_guard_failed() function on every circuit that is
+ * pending on <b>chan</b>.
+ */
+void
+entry_guard_chan_failed(channel_t *chan)
+{
+  if (!chan)
+    return;
+
+  smartlist_t *pending = smartlist_new();
+  circuit_get_all_pending_on_channel(pending, chan);
+  SMARTLIST_FOREACH_BEGIN(pending, circuit_t *, circ) {
+    if (!CIRCUIT_IS_ORIGIN(circ))
+      continue;
+
+    origin_circuit_t *origin_circ = TO_ORIGIN_CIRCUIT(circ);
+    if (origin_circ->guard_state) {
+      /* We might have no guard state if we didn't use a guard on this
+       * circuit (eg it's for a fallback directory). */
+      entry_guard_failed(&origin_circ->guard_state);
+    }
+  } SMARTLIST_FOREACH_END(circ);
+  smartlist_free(pending);
+}
+
+/**
+ * Return true iff every primary guard in <b>gs</b> is believed to
+ * be unreachable.
+ */
+STATIC int
+entry_guards_all_primary_guards_are_down(guard_selection_t *gs)
+{
+  tor_assert(gs);
+  if (!gs->primary_guards_up_to_date)
+    entry_guards_update_primary(gs);
+  SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);
+    if (guard->is_reachable != GUARD_REACHABLE_NO)
+      return 0;
+  } SMARTLIST_FOREACH_END(guard);
+  return 1;
+}
+
+/** Wrapper for entry_guard_has_higher_priority that compares the
+ * guard-priorities of a pair of circuits. Return 1 if <b>a</b> has higher
+ * priority than <b>b</b>.
+ *
+ * If a restriction is provided in <b>rst</b>, then do not consider
+ * <b>a</b> to have higher priority if it violates the restriction.
+ */
+static int
+circ_state_has_higher_priority(origin_circuit_t *a,
+                               const entry_guard_restriction_t *rst,
+                               origin_circuit_t *b)
+{
+  circuit_guard_state_t *state_a = origin_circuit_get_guard_state(a);
+  circuit_guard_state_t *state_b = origin_circuit_get_guard_state(b);
+
+  tor_assert(state_a);
+  tor_assert(state_b);
+
+  entry_guard_t *guard_a = entry_guard_handle_get(state_a->guard);
+  entry_guard_t *guard_b = entry_guard_handle_get(state_b->guard);
+
+  if (! guard_a) {
+    /* Unknown guard -- never higher priority. */
+    return 0;
+  } else if (! guard_b) {
+    /* Known guard -- higher priority than any unknown guard. */
+    return 1;
+  } else  if (! entry_guard_obeys_restriction(guard_a, rst)) {
+    /* Restriction violated; guard_a cannot have higher priority. */
+    return 0;
+  } else {
+    /* Both known -- compare.*/
+    return entry_guard_has_higher_priority(guard_a, guard_b);
+  }
+}
+
+/**
+ * Look at all of the origin_circuit_t * objects in <b>all_circuits_in</b>,
+ * and see if any of them that were previously not ready to use for
+ * guard-related reasons are now ready to use. Place those circuits
+ * in <b>newly_complete_out</b>, and mark them COMPLETE.
+ *
+ * Return 1 if we upgraded any circuits, and 0 otherwise.
+ */
+int
+entry_guards_upgrade_waiting_circuits(guard_selection_t *gs,
+                                      const smartlist_t *all_circuits_in,
+                                      smartlist_t *newly_complete_out)
+{
+  tor_assert(gs);
+  tor_assert(all_circuits_in);
+  tor_assert(newly_complete_out);
+
+  if (! entry_guards_all_primary_guards_are_down(gs)) {
+    /* We only upgrade a waiting circuit if the primary guards are all
+     * down. */
+    log_debug(LD_GUARD, "Considered upgrading guard-stalled circuits, "
+              "but not all primary guards were definitely down.");
+    return 0;
+  }
+
+  int n_waiting = 0;
+  int n_complete = 0;
+  int n_complete_blocking = 0;
+  origin_circuit_t *best_waiting_circuit = NULL;
+  smartlist_t *all_circuits = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(all_circuits_in, origin_circuit_t *, circ) {
+    // We filter out circuits that aren't ours, or which we can't
+    // reason about.
+    circuit_guard_state_t *state = origin_circuit_get_guard_state(circ);
+    if (state == NULL)
+      continue;
+    entry_guard_t *guard = entry_guard_handle_get(state->guard);
+    if (!guard || guard->in_selection != gs)
+      continue;
+    if (TO_CIRCUIT(circ)->marked_for_close) {
+      /* Don't consider any marked for close circuits. */
+      continue;
+    }
+
+    smartlist_add(all_circuits, circ);
+  } SMARTLIST_FOREACH_END(circ);
+
+  SMARTLIST_FOREACH_BEGIN(all_circuits, origin_circuit_t *, circ) {
+    circuit_guard_state_t *state = origin_circuit_get_guard_state(circ);
+    if (BUG(state == NULL))
+      continue;
+
+    if (state->state == GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD) {
+      ++n_waiting;
+      if (! best_waiting_circuit ||
+          circ_state_has_higher_priority(circ, NULL, best_waiting_circuit)) {
+        best_waiting_circuit = circ;
+      }
+    }
+  } SMARTLIST_FOREACH_END(circ);
+
+  if (! best_waiting_circuit) {
+    log_debug(LD_GUARD, "Considered upgrading guard-stalled circuits, "
+              "but didn't find any.");
+    goto no_change;
+  }
+
+  /* We'll need to keep track of what restrictions were used when picking this
+   * circuit, so that we don't allow any circuit without those restrictions to
+   * block it. */
+  const entry_guard_restriction_t *rst_on_best_waiting =
+    origin_circuit_get_guard_state(best_waiting_circuit)->restrictions;
+
+  /* First look at the complete circuits: Do any block this circuit? */
+  SMARTLIST_FOREACH_BEGIN(all_circuits, origin_circuit_t *, circ) {
+    /* "C2 "blocks" C1 if:
+        * C2 obeys all the restrictions that C1 had to obey, AND
+        * C2 has higher priority than C1, AND
+        * Either C2 is <complete>, or C2 is <waiting_for_better_guard>,
+          or C2 has been <usable_if_no_better_guard> for no more than
+          {NONPRIMARY_GUARD_CONNECT_TIMEOUT} seconds."
+    */
+    circuit_guard_state_t *state = origin_circuit_get_guard_state(circ);
+    if BUG((state == NULL))
+      continue;
+    if (state->state != GUARD_CIRC_STATE_COMPLETE)
+      continue;
+    ++n_complete;
+    if (circ_state_has_higher_priority(circ, rst_on_best_waiting,
+                                       best_waiting_circuit))
+      ++n_complete_blocking;
+  } SMARTLIST_FOREACH_END(circ);
+
+  if (n_complete_blocking) {
+    log_debug(LD_GUARD, "Considered upgrading guard-stalled circuits: found "
+              "%d complete and %d guard-stalled. At least one complete "
+              "circuit had higher priority, so not upgrading.",
+              n_complete, n_waiting);
+    goto no_change;
+  }
+
+  /* " * If any circuit C1 is <waiting_for_better_guard>, AND:
+          * All primary guards have reachable status of <no>.
+          * There is no circuit C2 that "blocks" C1.
+         Then, upgrade C1 to <complete>.""
+  */
+  int n_blockers_found = 0;
+  const time_t state_set_at_cutoff =
+    approx_time() - get_nonprimary_guard_connect_timeout();
+  SMARTLIST_FOREACH_BEGIN(all_circuits, origin_circuit_t *, circ) {
+    circuit_guard_state_t *state = origin_circuit_get_guard_state(circ);
+    if (BUG(state == NULL))
+      continue;
+    if (state->state != GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD)
+      continue;
+    if (state->state_set_at <= state_set_at_cutoff)
+      continue;
+    if (circ_state_has_higher_priority(circ, rst_on_best_waiting,
+                                       best_waiting_circuit))
+      ++n_blockers_found;
+  } SMARTLIST_FOREACH_END(circ);
+
+  if (n_blockers_found) {
+    log_debug(LD_GUARD, "Considered upgrading guard-stalled circuits: found "
+              "%d guard-stalled, but %d pending circuit(s) had higher "
+              "guard priority, so not upgrading.",
+              n_waiting, n_blockers_found);
+    goto no_change;
+  }
+
+  /* Okay. We have a best waiting circuit, and we aren't waiting for
+     anything better.  Add all circuits with that priority to the
+     list, and call them COMPLETE. */
+  int n_succeeded = 0;
+  SMARTLIST_FOREACH_BEGIN(all_circuits, origin_circuit_t *, circ) {
+    circuit_guard_state_t *state = origin_circuit_get_guard_state(circ);
+    if (BUG(state == NULL))
+      continue;
+    if (circ != best_waiting_circuit && rst_on_best_waiting) {
+      /* Can't upgrade other circ with same priority as best; might
+         be blocked. */
+      continue;
+    }
+    if (state->state != GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD)
+      continue;
+    if (circ_state_has_higher_priority(best_waiting_circuit, NULL, circ))
+      continue;
+
+    state->state = GUARD_CIRC_STATE_COMPLETE;
+    state->state_set_at = approx_time();
+    smartlist_add(newly_complete_out, circ);
+    ++n_succeeded;
+  } SMARTLIST_FOREACH_END(circ);
+
+  log_info(LD_GUARD, "Considered upgrading guard-stalled circuits: found "
+           "%d guard-stalled, %d complete. %d of the guard-stalled "
+           "circuit(s) had high enough priority to upgrade.",
+           n_waiting, n_complete, n_succeeded);
+
+  tor_assert_nonfatal(n_succeeded >= 1);
+  smartlist_free(all_circuits);
+  return 1;
+
+ no_change:
+  smartlist_free(all_circuits);
+  return 0;
+}
+
+/**
+ * Return true iff the circuit whose state is <b>guard_state</b> should
+ * expire.
+ */
+int
+entry_guard_state_should_expire(circuit_guard_state_t *guard_state)
+{
+  if (guard_state == NULL)
+    return 0;
+  const time_t expire_if_waiting_since =
+    approx_time() - get_nonprimary_guard_idle_timeout();
+  return (guard_state->state == GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD
+          && guard_state->state_set_at < expire_if_waiting_since);
+}
+
+/**
+ * Update all derived pieces of the guard selection state in <b>gs</b>.
+ * Return true iff we should stop using all previously generated circuits.
+ */
+int
+entry_guards_update_all(guard_selection_t *gs)
+{
+  sampled_guards_update_from_consensus(gs);
+  entry_guards_update_filtered_sets(gs);
+  entry_guards_update_confirmed(gs);
+  entry_guards_update_primary(gs);
+  return 0;
+}
+
+/**
+ * Return a newly allocated string for encoding the persistent parts of
+ * <b>guard</b> to the state file.
+ */
+STATIC char *
+entry_guard_encode_for_state(entry_guard_t *guard)
+{
+  /*
+   * The meta-format we use is K=V K=V K=V... where K can be any
+   * characters excepts space and =, and V can be any characters except
+   * space.  The order of entries is not allowed to matter.
+   * Unrecognized K=V entries are persisted; recognized but erroneous
+   * entries are corrected.
+   */
+
+  smartlist_t *result = smartlist_new();
+  char tbuf[ISO_TIME_LEN+1];
+
+  tor_assert(guard);
+
+  smartlist_add_asprintf(result, "in=%s", guard->selection_name);
+  smartlist_add_asprintf(result, "rsa_id=%s",
+                         hex_str(guard->identity, DIGEST_LEN));
+  if (guard->bridge_addr) {
+    smartlist_add_asprintf(result, "bridge_addr=%s:%d",
+                           fmt_and_decorate_addr(&guard->bridge_addr->addr),
+                           guard->bridge_addr->port);
+  }
+  if (strlen(guard->nickname) && is_legal_nickname(guard->nickname)) {
+    smartlist_add_asprintf(result, "nickname=%s", guard->nickname);
+  }
+
+  format_iso_time_nospace(tbuf, guard->sampled_on_date);
+  smartlist_add_asprintf(result, "sampled_on=%s", tbuf);
+
+  if (guard->sampled_by_version) {
+    smartlist_add_asprintf(result, "sampled_by=%s",
+                           guard->sampled_by_version);
+  }
+
+  if (guard->unlisted_since_date > 0) {
+    format_iso_time_nospace(tbuf, guard->unlisted_since_date);
+    smartlist_add_asprintf(result, "unlisted_since=%s", tbuf);
+  }
+
+  smartlist_add_asprintf(result, "listed=%d",
+                         (int)guard->currently_listed);
+
+  if (guard->confirmed_idx >= 0) {
+    format_iso_time_nospace(tbuf, guard->confirmed_on_date);
+    smartlist_add_asprintf(result, "confirmed_on=%s", tbuf);
+
+    smartlist_add_asprintf(result, "confirmed_idx=%d", guard->confirmed_idx);
+  }
+
+  const double EPSILON = 1.0e-6;
+
+  /* Make a copy of the pathbias object, since we will want to update
+     some of them */
+  guard_pathbias_t *pb = tor_memdup(&guard->pb, sizeof(*pb));
+  pb->use_successes = pathbias_get_use_success_count(guard);
+  pb->successful_circuits_closed = pathbias_get_close_success_count(guard);
+
+  #define PB_FIELD(field) do {                                          \
+      if (pb->field >= EPSILON) {                                       \
+        smartlist_add_asprintf(result, "pb_" #field "=%f", pb->field);  \
+      }                                                                 \
+    } while (0)
+  PB_FIELD(use_attempts);
+  PB_FIELD(use_successes);
+  PB_FIELD(circ_attempts);
+  PB_FIELD(circ_successes);
+  PB_FIELD(successful_circuits_closed);
+  PB_FIELD(collapsed_circuits);
+  PB_FIELD(unusable_circuits);
+  PB_FIELD(timeouts);
+  tor_free(pb);
+#undef PB_FIELD
+
+  if (guard->extra_state_fields)
+    smartlist_add_strdup(result, guard->extra_state_fields);
+
+  char *joined = smartlist_join_strings(result, " ", 0, NULL);
+  SMARTLIST_FOREACH(result, char *, cp, tor_free(cp));
+  smartlist_free(result);
+
+  return joined;
+}
+
+/**
+ * Given a string generated by entry_guard_encode_for_state(), parse it
+ * (if possible) and return an entry_guard_t object for it.  Return NULL
+ * on complete failure.
+ */
+STATIC entry_guard_t *
+entry_guard_parse_from_state(const char *s)
+{
+  /* Unrecognized entries get put in here. */
+  smartlist_t *extra = smartlist_new();
+
+  /* These fields get parsed from the string. */
+  char *in = NULL;
+  char *rsa_id = NULL;
+  char *nickname = NULL;
+  char *sampled_on = NULL;
+  char *sampled_by = NULL;
+  char *unlisted_since = NULL;
+  char *listed  = NULL;
+  char *confirmed_on = NULL;
+  char *confirmed_idx = NULL;
+  char *bridge_addr = NULL;
+
+  // pathbias
+  char *pb_use_attempts = NULL;
+  char *pb_use_successes = NULL;
+  char *pb_circ_attempts = NULL;
+  char *pb_circ_successes = NULL;
+  char *pb_successful_circuits_closed = NULL;
+  char *pb_collapsed_circuits = NULL;
+  char *pb_unusable_circuits = NULL;
+  char *pb_timeouts = NULL;
+
+  /* Split up the entries.  Put the ones we know about in strings and the
+   * rest in "extra". */
+  {
+    smartlist_t *entries = smartlist_new();
+
+    strmap_t *vals = strmap_new(); // Maps keyword to location
+#define FIELD(f) \
+    strmap_set(vals, #f, &f);
+    FIELD(in);
+    FIELD(rsa_id);
+    FIELD(nickname);
+    FIELD(sampled_on);
+    FIELD(sampled_by);
+    FIELD(unlisted_since);
+    FIELD(listed);
+    FIELD(confirmed_on);
+    FIELD(confirmed_idx);
+    FIELD(bridge_addr);
+    FIELD(pb_use_attempts);
+    FIELD(pb_use_successes);
+    FIELD(pb_circ_attempts);
+    FIELD(pb_circ_successes);
+    FIELD(pb_successful_circuits_closed);
+    FIELD(pb_collapsed_circuits);
+    FIELD(pb_unusable_circuits);
+    FIELD(pb_timeouts);
+#undef FIELD
+
+    smartlist_split_string(entries, s, " ",
+                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+
+    SMARTLIST_FOREACH_BEGIN(entries, char *, entry) {
+      const char *eq = strchr(entry, '=');
+      if (!eq) {
+        smartlist_add(extra, entry);
+        continue;
+      }
+      char *key = tor_strndup(entry, eq-entry);
+      char **target = strmap_get(vals, key);
+      if (target == NULL || *target != NULL) {
+        /* unrecognized or already set */
+        smartlist_add(extra, entry);
+        tor_free(key);
+        continue;
+      }
+
+      *target = tor_strdup(eq+1);
+      tor_free(key);
+      tor_free(entry);
+    } SMARTLIST_FOREACH_END(entry);
+
+    smartlist_free(entries);
+    strmap_free(vals, NULL);
+  }
+
+  entry_guard_t *guard = tor_malloc_zero(sizeof(entry_guard_t));
+  guard->is_persistent = 1;
+
+  if (in == NULL) {
+    log_warn(LD_CIRC, "Guard missing 'in' field");
+    goto err;
+  }
+
+  guard->selection_name = in;
+  in = NULL;
+
+  if (rsa_id == NULL) {
+    log_warn(LD_CIRC, "Guard missing RSA ID field");
+    goto err;
+  }
+
+  /* Process the identity and nickname. */
+  if (base16_decode(guard->identity, sizeof(guard->identity),
+                    rsa_id, strlen(rsa_id)) != DIGEST_LEN) {
+    log_warn(LD_CIRC, "Unable to decode guard identity %s", escaped(rsa_id));
+    goto err;
+  }
+
+  if (nickname) {
+    strlcpy(guard->nickname, nickname, sizeof(guard->nickname));
+  } else {
+    guard->nickname[0]='$';
+    base16_encode(guard->nickname+1, sizeof(guard->nickname)-1,
+                  guard->identity, DIGEST_LEN);
+  }
+
+  if (bridge_addr) {
+    tor_addr_port_t res;
+    memset(&res, 0, sizeof(res));
+    int r = tor_addr_port_parse(LOG_WARN, bridge_addr,
+                                &res.addr, &res.port, -1);
+    if (r == 0)
+      guard->bridge_addr = tor_memdup(&res, sizeof(res));
+    /* On error, we already warned. */
+  }
+
+  /* Process the various time fields. */
+
+#define HANDLE_TIME(field) do {                                 \
+    if (field) {                                                \
+      int r = parse_iso_time_nospace(field, &field ## _time);   \
+      if (r < 0) {                                              \
+        log_warn(LD_CIRC, "Unable to parse %s %s from guard",   \
+                 #field, escaped(field));                       \
+        field##_time = -1;                                      \
+      }                                                         \
+    }                                                           \
+  } while (0)
+
+  time_t sampled_on_time = 0;
+  time_t unlisted_since_time = 0;
+  time_t confirmed_on_time = 0;
+
+  HANDLE_TIME(sampled_on);
+  HANDLE_TIME(unlisted_since);
+  HANDLE_TIME(confirmed_on);
+
+  if (sampled_on_time <= 0)
+    sampled_on_time = approx_time();
+  if (unlisted_since_time < 0)
+    unlisted_since_time = 0;
+  if (confirmed_on_time < 0)
+    confirmed_on_time = 0;
+
+  #undef HANDLE_TIME
+
+  guard->sampled_on_date = sampled_on_time;
+  guard->unlisted_since_date = unlisted_since_time;
+  guard->confirmed_on_date = confirmed_on_time;
+
+  /* Take sampled_by_version verbatim. */
+  guard->sampled_by_version = sampled_by;
+  sampled_by = NULL; /* prevent free */
+
+  /* Listed is a boolean */
+  if (listed && strcmp(listed, "0"))
+    guard->currently_listed = 1;
+
+  /* The index is a nonnegative integer. */
+  guard->confirmed_idx = -1;
+  if (confirmed_idx) {
+    int ok=1;
+    long idx = tor_parse_long(confirmed_idx, 10, 0, INT_MAX, &ok, NULL);
+    if (! ok) {
+      log_warn(LD_GUARD, "Guard has invalid confirmed_idx %s",
+               escaped(confirmed_idx));
+    } else {
+      guard->confirmed_idx = (int)idx;
+    }
+  }
+
+  /* Anything we didn't recognize gets crammed together */
+  if (smartlist_len(extra) > 0) {
+    guard->extra_state_fields = smartlist_join_strings(extra, " ", 0, NULL);
+  }
+
+  /* initialize non-persistent fields */
+  guard->is_reachable = GUARD_REACHABLE_MAYBE;
+
+#define PB_FIELD(field)                                                 \
+  do {                                                                  \
+    if (pb_ ## field) {                                                 \
+      int ok = 1;                                                       \
+      double r = tor_parse_double(pb_ ## field, 0.0, 1e9, &ok, NULL);   \
+      if (! ok) {                                                       \
+        log_warn(LD_CIRC, "Guard has invalid pb_%s %s",                 \
+                 #field, pb_ ## field);                                 \
+      } else {                                                          \
+        guard->pb.field = r;                                            \
+      }                                                                 \
+    }                                                                   \
+  } while (0)
+  PB_FIELD(use_attempts);
+  PB_FIELD(use_successes);
+  PB_FIELD(circ_attempts);
+  PB_FIELD(circ_successes);
+  PB_FIELD(successful_circuits_closed);
+  PB_FIELD(collapsed_circuits);
+  PB_FIELD(unusable_circuits);
+  PB_FIELD(timeouts);
+#undef PB_FIELD
+
+  pathbias_check_use_success_count(guard);
+  pathbias_check_close_success_count(guard);
+
+  /* We update everything on this guard later, after we've parsed
+   * everything.  */
+
+  goto done;
+
+ err:
+  // only consider it an error if the guard state was totally unparseable.
+  entry_guard_free(guard);
+  guard = NULL;
+
+ done:
+  tor_free(in);
+  tor_free(rsa_id);
+  tor_free(nickname);
+  tor_free(sampled_on);
+  tor_free(sampled_by);
+  tor_free(unlisted_since);
+  tor_free(listed);
+  tor_free(confirmed_on);
+  tor_free(confirmed_idx);
+  tor_free(bridge_addr);
+  tor_free(pb_use_attempts);
+  tor_free(pb_use_successes);
+  tor_free(pb_circ_attempts);
+  tor_free(pb_circ_successes);
+  tor_free(pb_successful_circuits_closed);
+  tor_free(pb_collapsed_circuits);
+  tor_free(pb_unusable_circuits);
+  tor_free(pb_timeouts);
+
+  SMARTLIST_FOREACH(extra, char *, cp, tor_free(cp));
+  smartlist_free(extra);
+
+  return guard;
+}
+
+/**
+ * Replace the Guards entries in <b>state</b> with a list of all our sampled
+ * guards.
+ */
+static void
+entry_guards_update_guards_in_state(or_state_t *state)
+{
+  if (!guard_contexts)
+    return;
+  config_line_t *lines = NULL;
+  config_line_t **nextline = &lines;
+
+  SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+    SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+      if (guard->is_persistent == 0)
+        continue;
+      *nextline = tor_malloc_zero(sizeof(config_line_t));
+      (*nextline)->key = tor_strdup("Guard");
+      (*nextline)->value = entry_guard_encode_for_state(guard);
+      nextline = &(*nextline)->next;
+    } SMARTLIST_FOREACH_END(guard);
+  } SMARTLIST_FOREACH_END(gs);
+
+  config_free_lines(state->Guard);
+  state->Guard = lines;
+}
+
+/**
+ * Replace our sampled guards from the Guards entries in <b>state</b>. Return 0
+ * on success, -1 on failure. (If <b>set</b> is true, replace nothing -- only
+ * check whether replacing would work.)
+ */
+static int
+entry_guards_load_guards_from_state(or_state_t *state, int set)
+{
+  const config_line_t *line = state->Guard;
+  int n_errors = 0;
+
+  if (!guard_contexts)
+    guard_contexts = smartlist_new();
+
+  /* Wipe all our existing guard info. (we shouldn't have any, but
+   * let's be safe.) */
+  if (set) {
+    SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+      guard_selection_free(gs);
+      if (curr_guard_context == gs)
+        curr_guard_context = NULL;
+      SMARTLIST_DEL_CURRENT(guard_contexts, gs);
+    } SMARTLIST_FOREACH_END(gs);
+  }
+
+  for ( ; line != NULL; line = line->next) {
+    entry_guard_t *guard = entry_guard_parse_from_state(line->value);
+    if (guard == NULL) {
+      ++n_errors;
+      continue;
+    }
+    tor_assert(guard->selection_name);
+    if (!strcmp(guard->selection_name, "legacy")) {
+      ++n_errors;
+      entry_guard_free(guard);
+      continue;
+    }
+
+    if (set) {
+      guard_selection_t *gs;
+      gs = get_guard_selection_by_name(guard->selection_name,
+                                       GS_TYPE_INFER, 1);
+      tor_assert(gs);
+      smartlist_add(gs->sampled_entry_guards, guard);
+      guard->in_selection = gs;
+    } else {
+      entry_guard_free(guard);
+    }
+  }
+
+  if (set) {
+    SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+      entry_guards_update_all(gs);
+    } SMARTLIST_FOREACH_END(gs);
+  }
+  return n_errors ? -1 : 0;
+}
+
+/** If <b>digest</b> matches the identity of any node in the
+ * entry_guards list for the provided guard selection state,
+ return that node. Else return NULL. */
+entry_guard_t *
+entry_guard_get_by_id_digest_for_guard_selection(guard_selection_t *gs,
+                                                 const char *digest)
+{
+  return get_sampled_guard_with_id(gs, (const uint8_t*)digest);
+}
+
+/** Return the node_t associated with a single entry_guard_t. May
+ * return NULL if the guard is not currently in the consensus. */
+const node_t *
+entry_guard_find_node(const entry_guard_t *guard)
+{
+  tor_assert(guard);
+  return node_get_by_id(guard->identity);
+}
+
+/** If <b>digest</b> matches the identity of any node in the
+ * entry_guards list for the default guard selection state,
+ return that node. Else return NULL. */
+entry_guard_t *
+entry_guard_get_by_id_digest(const char *digest)
+{
+  return entry_guard_get_by_id_digest_for_guard_selection(
+      get_guard_selection_info(), digest);
+}
+
+/** We are about to connect to bridge with identity <b>digest</b> to fetch its
+ *  descriptor. Create a new guard state for this connection and return it. */
+circuit_guard_state_t *
+get_guard_state_for_bridge_desc_fetch(const char *digest)
+{
+  circuit_guard_state_t *guard_state = NULL;
+  entry_guard_t *guard = NULL;
+
+  guard = entry_guard_get_by_id_digest_for_guard_selection(
+                                    get_guard_selection_info(), digest);
+  if (!guard) {
+    return NULL;
+  }
+
+  /* Update the guard last_tried_to_connect time since it's checked by the
+   * guard susbsystem. */
+  guard->last_tried_to_connect = approx_time();
+
+  /* Create the guard state */
+  guard_state = circuit_guard_state_new(guard,
+                                        GUARD_CIRC_STATE_USABLE_ON_COMPLETION,
+                                        NULL);
+
+  return guard_state;
+}
+
+/** Release all storage held by <b>e</b>. */
+STATIC void
+entry_guard_free_(entry_guard_t *e)
+{
+  if (!e)
+    return;
+  entry_guard_handles_clear(e);
+  tor_free(e->sampled_by_version);
+  tor_free(e->extra_state_fields);
+  tor_free(e->selection_name);
+  tor_free(e->bridge_addr);
+  tor_free(e);
+}
+
+/** Return 0 if we're fine adding arbitrary routers out of the
+ * directory to our entry guard list, or return 1 if we have a
+ * list already and we must stick to it.
+ */
+int
+entry_list_is_constrained(const or_options_t *options)
+{
+  // XXXX #21425 look at the current selection.
+  if (options->EntryNodes)
+    return 1;
+  if (options->UseBridges)
+    return 1;
+  return 0;
+}
+
+/** Return the number of bridges that have descriptors that are marked with
+ * purpose 'bridge' and are running. If use_maybe_reachable is
+ * true, include bridges that might be reachable in the count.
+ * Otherwise, if it is false, only include bridges that have recently been
+ * found running in the count.
+ *
+ * We use this function to decide if we're ready to start building
+ * circuits through our bridges, or if we need to wait until the
+ * directory "server/authority" requests finish. */
+MOCK_IMPL(int,
+num_bridges_usable,(int use_maybe_reachable))
+{
+  int n_options = 0;
+
+  if (BUG(!get_options()->UseBridges)) {
+    return 0;
+  }
+  guard_selection_t *gs  = get_guard_selection_info();
+  if (BUG(gs->type != GS_TYPE_BRIDGE)) {
+    return 0;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    /* Not a bridge, or not one we are configured to be able to use. */
+    if (! guard->is_filtered_guard)
+      continue;
+    /* Definitely not usable */
+    if (guard->is_reachable == GUARD_REACHABLE_NO)
+      continue;
+    /* If we want to be really sure the bridges will work, skip maybes */
+    if (!use_maybe_reachable && guard->is_reachable == GUARD_REACHABLE_MAYBE)
+      continue;
+    if (tor_digest_is_zero(guard->identity))
+      continue;
+    const node_t *node = node_get_by_id(guard->identity);
+    if (node && node->ri)
+      ++n_options;
+  } SMARTLIST_FOREACH_END(guard);
+
+  return n_options;
+}
+
+/** Check the pathbias use success count of <b>node</b> and disable it if it
+ *  goes over our thresholds. */
+static void
+pathbias_check_use_success_count(entry_guard_t *node)
+{
+  const or_options_t *options = get_options();
+  const double EPSILON = 1.0e-9;
+
+  /* Note: We rely on the < comparison here to allow us to set a 0
+   * rate and disable the feature entirely. If refactoring, don't
+   * change to <= */
+  if (node->pb.use_attempts > EPSILON &&
+      pathbias_get_use_success_count(node)/node->pb.use_attempts
+      < pathbias_get_extreme_use_rate(options) &&
+      pathbias_get_dropguards(options)) {
+    node->pb.path_bias_disabled = 1;
+    log_info(LD_GENERAL,
+             "Path use bias is too high (%f/%f); disabling node %s",
+             node->pb.circ_successes, node->pb.circ_attempts,
+             node->nickname);
+  }
+}
+
+/** Check the pathbias close count of <b>node</b> and disable it if it goes
+ *  over our thresholds. */
+static void
+pathbias_check_close_success_count(entry_guard_t *node)
+{
+  const or_options_t *options = get_options();
+  const double EPSILON = 1.0e-9;
+
+  /* Note: We rely on the < comparison here to allow us to set a 0
+   * rate and disable the feature entirely. If refactoring, don't
+   * change to <= */
+  if (node->pb.circ_attempts > EPSILON &&
+      pathbias_get_close_success_count(node)/node->pb.circ_attempts
+      < pathbias_get_extreme_rate(options) &&
+      pathbias_get_dropguards(options)) {
+    node->pb.path_bias_disabled = 1;
+    log_info(LD_GENERAL,
+             "Path bias is too high (%f/%f); disabling node %s",
+             node->pb.circ_successes, node->pb.circ_attempts,
+             node->nickname);
+  }
+}
+
+/** Parse <b>state</b> and learn about the entry guards it describes.
+ * If <b>set</b> is true, and there are no errors, replace the guard
+ * list in the default guard selection context with what we find.
+ * On success, return 0. On failure, alloc into *<b>msg</b> a string
+ * describing the error, and return -1.
+ */
+int
+entry_guards_parse_state(or_state_t *state, int set, char **msg)
+{
+  entry_guards_dirty = 0;
+  int r1 = entry_guards_load_guards_from_state(state, set);
+  entry_guards_dirty = 0;
+
+  if (r1 < 0) {
+    if (msg && *msg == NULL) {
+      *msg = tor_strdup("parsing error");
+    }
+    return -1;
+  }
+  return 0;
+}
+
+/** How long will we let a change in our guard nodes stay un-saved
+ * when we are trying to avoid disk writes? */
+#define SLOW_GUARD_STATE_FLUSH_TIME 600
+/** How long will we let a change in our guard nodes stay un-saved
+ * when we are not trying to avoid disk writes? */
+#define FAST_GUARD_STATE_FLUSH_TIME 30
+
+/** Our list of entry guards has changed for a particular guard selection
+ * context, or some element of one of our entry guards has changed for one.
+ * Write the changes to disk within the next few minutes.
+ */
+void
+entry_guards_changed_for_guard_selection(guard_selection_t *gs)
+{
+  time_t when;
+
+  tor_assert(gs != NULL);
+
+  entry_guards_dirty = 1;
+
+  if (get_options()->AvoidDiskWrites)
+    when = time(NULL) + SLOW_GUARD_STATE_FLUSH_TIME;
+  else
+    when = time(NULL) + FAST_GUARD_STATE_FLUSH_TIME;
+
+  /* or_state_save() will call entry_guards_update_state() and
+     entry_guards_update_guards_in_state()
+  */
+  or_state_mark_dirty(get_or_state(), when);
+}
+
+/** Our list of entry guards has changed for the default guard selection
+ * context, or some element of one of our entry guards has changed. Write
+ * the changes to disk within the next few minutes.
+ */
+void
+entry_guards_changed(void)
+{
+  entry_guards_changed_for_guard_selection(get_guard_selection_info());
+}
+
+/** If the entry guard info has not changed, do nothing and return.
+ * Otherwise, free the EntryGuards piece of <b>state</b> and create
+ * a new one out of the global entry_guards list, and then mark
+ * <b>state</b> dirty so it will get saved to disk.
+ */
+void
+entry_guards_update_state(or_state_t *state)
+{
+  entry_guards_dirty = 0;
+
+  // Handles all guard info.
+  entry_guards_update_guards_in_state(state);
+
+  entry_guards_dirty = 0;
+
+  if (!get_options()->AvoidDiskWrites)
+    or_state_mark_dirty(get_or_state(), 0);
+  entry_guards_dirty = 0;
+}
+
+/** Return true iff the circuit's guard can succeed that is can be used. */
+int
+entry_guard_could_succeed(const circuit_guard_state_t *guard_state)
+{
+  if (!guard_state) {
+    return 0;
+  }
+
+  entry_guard_t *guard = entry_guard_handle_get(guard_state->guard);
+  if (!guard || BUG(guard->in_selection == NULL)) {
+    return 0;
+  }
+
+  return 1;
+}
+
+/**
+ * Format a single entry guard in the format expected by the controller.
+ * Return a newly allocated string.
+ */
+STATIC char *
+getinfo_helper_format_single_entry_guard(const entry_guard_t *e)
+{
+  const char *status = NULL;
+  time_t when = 0;
+  const node_t *node;
+  char tbuf[ISO_TIME_LEN+1];
+  char nbuf[MAX_VERBOSE_NICKNAME_LEN+1];
+
+  /* This is going to be a bit tricky, since the status
+   * codes weren't really intended for prop271 guards.
+   *
+   * XXXX use a more appropriate format for exporting this information
+   */
+  if (e->confirmed_idx < 0) {
+    status = "never-connected";
+  } else if (! e->currently_listed) {
+    when = e->unlisted_since_date;
+    status = "unusable";
+  } else if (! e->is_filtered_guard) {
+    status = "unusable";
+  } else if (e->is_reachable == GUARD_REACHABLE_NO) {
+    when = e->failing_since;
+    status = "down";
+  } else {
+    status = "up";
+  }
+
+  node = entry_guard_find_node(e);
+  if (node) {
+    node_get_verbose_nickname(node, nbuf);
+  } else {
+    nbuf[0] = '$';
+    base16_encode(nbuf+1, sizeof(nbuf)-1, e->identity, DIGEST_LEN);
+    /* e->nickname field is not very reliable if we don't know about
+     * this router any longer; don't include it. */
+  }
+
+  char *result = NULL;
+  if (when) {
+    format_iso_time(tbuf, when);
+    tor_asprintf(&result, "%s %s %s\n", nbuf, status, tbuf);
+  } else {
+    tor_asprintf(&result, "%s %s\n", nbuf, status);
+  }
+  return result;
+}
+
+/** If <b>question</b> is the string "entry-guards", then dump
+ * to *<b>answer</b> a newly allocated string describing all of
+ * the nodes in the global entry_guards list. See control-spec.txt
+ * for details.
+ * For backward compatibility, we also handle the string "helper-nodes".
+ *
+ * XXX this should be totally redesigned after prop 271 too, and that's
+ * going to take some control spec work.
+ * */
+int
+getinfo_helper_entry_guards(control_connection_t *conn,
+                            const char *question, char **answer,
+                            const char **errmsg)
+{
+  guard_selection_t *gs = get_guard_selection_info();
+
+  tor_assert(gs != NULL);
+
+  (void) conn;
+  (void) errmsg;
+
+  if (!strcmp(question,"entry-guards") ||
+      !strcmp(question,"helper-nodes")) {
+    const smartlist_t *guards;
+    guards = gs->sampled_entry_guards;
+
+    smartlist_t *sl = smartlist_new();
+
+    SMARTLIST_FOREACH_BEGIN(guards, const entry_guard_t *, e) {
+      char *cp = getinfo_helper_format_single_entry_guard(e);
+      smartlist_add(sl, cp);
+    } SMARTLIST_FOREACH_END(e);
+    *answer = smartlist_join_strings(sl, "", 0, NULL);
+    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
+    smartlist_free(sl);
+  }
+  return 0;
+}
+
+/* Given the original bandwidth of a guard and its guardfraction,
+ * calculate how much bandwidth the guard should have as a guard and
+ * as a non-guard.
+ *
+ * Quoting from proposal236:
+ *
+ *   Let Wpf denote the weight from the 'bandwidth-weights' line a
+ *   client would apply to N for position p if it had the guard
+ *   flag, Wpn the weight if it did not have the guard flag, and B the
+ *   measured bandwidth of N in the consensus.  Then instead of choosing
+ *   N for position p proportionally to Wpf*B or Wpn*B, clients should
+ *   choose N proportionally to F*Wpf*B + (1-F)*Wpn*B.
+ *
+ * This function fills the <b>guardfraction_bw</b> structure. It sets
+ * <b>guard_bw</b> to F*B and <b>non_guard_bw</b> to (1-F)*B.
+ */
+void
+guard_get_guardfraction_bandwidth(guardfraction_bandwidth_t *guardfraction_bw,
+                                  int orig_bandwidth,
+                                  uint32_t guardfraction_percentage)
+{
+  double guardfraction_fraction;
+
+  /* Turn the percentage into a fraction. */
+  tor_assert(guardfraction_percentage <= 100);
+  guardfraction_fraction = guardfraction_percentage / 100.0;
+
+  long guard_bw = tor_lround(guardfraction_fraction * orig_bandwidth);
+  tor_assert(guard_bw <= INT_MAX);
+
+  guardfraction_bw->guard_bw = (int) guard_bw;
+
+  guardfraction_bw->non_guard_bw = orig_bandwidth - (int) guard_bw;
+}
+
+/** Helper: Update the status of all entry guards, in whatever algorithm
+ * is used. Return true if we should stop using all previously generated
+ * circuits, by calling circuit_mark_all_unused_circs() and
+ * circuit_mark_all_dirty_circs_as_unusable().
+ */
+int
+guards_update_all(void)
+{
+  int mark_circuits = 0;
+  if (update_guard_selection_choice(get_options()))
+    mark_circuits = 1;
+
+  tor_assert(curr_guard_context);
+
+  if (entry_guards_update_all(curr_guard_context))
+    mark_circuits = 1;
+
+  return mark_circuits;
+}
+
+/** Helper: pick a guard for a circuit, with whatever algorithm is
+    used. */
+const node_t *
+guards_choose_guard(cpath_build_state_t *state,
+                    uint8_t purpose,
+                    circuit_guard_state_t **guard_state_out)
+{
+  const node_t *r = NULL;
+  const uint8_t *exit_id = NULL;
+  entry_guard_restriction_t *rst = NULL;
+
+  /* Only apply restrictions if we have a specific exit node in mind, and only
+   * if we are not doing vanguard circuits: we don't want to apply guard
+   * restrictions to vanguard circuits. */
+  if (state && !circuit_should_use_vanguards(purpose) &&
+      (exit_id = build_state_get_exit_rsa_id(state))) {
+    /* We're building to a targeted exit node, so that node can't be
+     * chosen as our guard for this circuit.  Remember that fact in a
+     * restriction. */
+    rst = guard_create_exit_restriction(exit_id);
+    tor_assert(rst);
+  }
+  if (entry_guard_pick_for_circuit(get_guard_selection_info(),
+                                   GUARD_USAGE_TRAFFIC,
+                                   rst,
+                                   &r,
+                                   guard_state_out) < 0) {
+    tor_assert(r == NULL);
+  }
+  return r;
+}
+
+/** Remove all currently listed entry guards for a given guard selection
+ * context.  This frees and replaces <b>gs</b>, so don't use <b>gs</b>
+ * after calling this function. */
+void
+remove_all_entry_guards_for_guard_selection(guard_selection_t *gs)
+{
+  // This function shouldn't exist. XXXX
+  tor_assert(gs != NULL);
+  char *old_name = tor_strdup(gs->name);
+  guard_selection_type_t old_type = gs->type;
+
+  SMARTLIST_FOREACH(gs->sampled_entry_guards, entry_guard_t *, entry, {
+    control_event_guard(entry->nickname, entry->identity, "DROPPED");
+  });
+
+  if (gs == curr_guard_context) {
+    curr_guard_context = NULL;
+  }
+
+  smartlist_remove(guard_contexts, gs);
+  guard_selection_free(gs);
+
+  gs = get_guard_selection_by_name(old_name, old_type, 1);
+  entry_guards_changed_for_guard_selection(gs);
+  tor_free(old_name);
+}
+
+/** Remove all currently listed entry guards, so new ones will be chosen.
+ *
+ * XXXX This function shouldn't exist -- it's meant to support the DROPGUARDS
+ * command, which is deprecated.
+ */
+void
+remove_all_entry_guards(void)
+{
+  remove_all_entry_guards_for_guard_selection(get_guard_selection_info());
+}
+
+/** Helper: pick a directory guard, with whatever algorithm is used. */
+const node_t *
+guards_choose_dirguard(uint8_t dir_purpose,
+                       circuit_guard_state_t **guard_state_out)
+{
+  const node_t *r = NULL;
+  entry_guard_restriction_t *rst = NULL;
+
+  /* If we are fetching microdescs, don't query outdated dirservers. */
+  if (dir_purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+    rst = guard_create_dirserver_md_restriction();
+  }
+
+  if (entry_guard_pick_for_circuit(get_guard_selection_info(),
+                                   GUARD_USAGE_DIRGUARD,
+                                   rst,
+                                   &r,
+                                   guard_state_out) < 0) {
+    tor_assert(r == NULL);
+  }
+  return r;
+}
+
+/**
+ * If we're running with a constrained guard set, then maybe mark our guards
+ * usable.  Return 1 if we do; 0 if we don't.
+ */
+int
+guards_retry_optimistic(const or_options_t *options)
+{
+  if (! entry_list_is_constrained(options))
+    return 0;
+
+  mark_primary_guards_maybe_reachable(get_guard_selection_info());
+
+  return 1;
+}
+
+/**
+ * Check if we are missing any crucial dirinfo for the guard subsystem to
+ * work. Return NULL if everything went well, otherwise return a newly
+ * allocated string with an informative error message. In the latter case, use
+ * the genreal descriptor information <b>using_mds</b>, <b>num_present</b> and
+ * <b>num_usable</b> to improve the error message. */
+char *
+guard_selection_get_err_str_if_dir_info_missing(guard_selection_t *gs,
+                                        int using_mds,
+                                        int num_present, int num_usable)
+{
+  if (!gs->primary_guards_up_to_date)
+    entry_guards_update_primary(gs);
+
+  char *ret_str = NULL;
+  int n_missing_descriptors = 0;
+  int n_considered = 0;
+  int num_primary_to_check;
+
+  /* We want to check for the descriptor of at least the first two primary
+   * guards in our list, since these are the guards that we typically use for
+   * circuits. */
+  num_primary_to_check = get_n_primary_guards_to_use(GUARD_USAGE_TRAFFIC);
+  num_primary_to_check++;
+
+  SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);
+    if (guard->is_reachable == GUARD_REACHABLE_NO)
+      continue;
+    n_considered++;
+    if (!guard_has_descriptor(guard))
+      n_missing_descriptors++;
+    if (n_considered >= num_primary_to_check)
+      break;
+  } SMARTLIST_FOREACH_END(guard);
+
+  /* If we are not missing any descriptors, return NULL. */
+  if (!n_missing_descriptors) {
+    return NULL;
+  }
+
+  /* otherwise return a helpful error string */
+  tor_asprintf(&ret_str, "We're missing descriptors for %d/%d of our "
+               "primary entry guards (total %sdescriptors: %d/%d). "
+               "That's ok. We will try to fetch missing descriptors soon.",
+               n_missing_descriptors, num_primary_to_check,
+               using_mds?"micro":"", num_present, num_usable);
+
+  return ret_str;
+}
+
+/** As guard_selection_have_enough_dir_info_to_build_circuits, but uses
+ * the default guard selection. */
+char *
+entry_guards_get_err_str_if_dir_info_missing(int using_mds,
+                                     int num_present, int num_usable)
+{
+  return guard_selection_get_err_str_if_dir_info_missing(
+                                                 get_guard_selection_info(),
+                                                 using_mds,
+                                                 num_present, num_usable);
+}
+
+/** Free one guard selection context */
+STATIC void
+guard_selection_free_(guard_selection_t *gs)
+{
+  if (!gs) return;
+
+  tor_free(gs->name);
+
+  if (gs->sampled_entry_guards) {
+    SMARTLIST_FOREACH(gs->sampled_entry_guards, entry_guard_t *, e,
+                      entry_guard_free(e));
+    smartlist_free(gs->sampled_entry_guards);
+    gs->sampled_entry_guards = NULL;
+  }
+
+  smartlist_free(gs->confirmed_entry_guards);
+  smartlist_free(gs->primary_entry_guards);
+
+  tor_free(gs);
+}
+
+/** Release all storage held by the list of entry guards and related
+ * memory structs. */
+void
+entry_guards_free_all(void)
+{
+  /* Null out the default */
+  curr_guard_context = NULL;
+  /* Free all the guard contexts */
+  if (guard_contexts != NULL) {
+    SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+      guard_selection_free(gs);
+    } SMARTLIST_FOREACH_END(gs);
+    smartlist_free(guard_contexts);
+    guard_contexts = NULL;
+  }
+  circuit_build_times_free_timeouts(get_circuit_build_times_mutable());
+}
diff --git a/src/feature/client/entrynodes.h b/src/feature/client/entrynodes.h
new file mode 100644
index 0000000000..4e5eb4e960
--- /dev/null
+++ b/src/feature/client/entrynodes.h
@@ -0,0 +1,639 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file entrynodes.h
+ * \brief Header file for circuitbuild.c.
+ **/
+
+#ifndef TOR_ENTRYNODES_H
+#define TOR_ENTRYNODES_H
+
+#include "lib/container/handles.h"
+
+/* Forward declare for guard_selection_t; entrynodes.c has the real struct */
+typedef struct guard_selection_s guard_selection_t;
+
+/* Forward declare for entry_guard_t; the real declaration is private. */
+typedef struct entry_guard_t entry_guard_t;
+
+/* Forward declaration for circuit_guard_state_t; the real declaration is
+   private. */
+typedef struct circuit_guard_state_t circuit_guard_state_t;
+
+/* Forward declaration for entry_guard_restriction_t; the real declaration is
+   private. */
+typedef struct entry_guard_restriction_t entry_guard_restriction_t;
+
+/* Information about a guard's pathbias status.
+ * These fields are used in circpathbias.c to try to detect entry
+ * nodes that are failing circuits at a suspicious frequency.
+ */
+typedef struct guard_pathbias_t {
+  unsigned int path_bias_noticed : 1; /**< Did we alert the user about path
+                                       * bias for this node already? */
+  unsigned int path_bias_warned : 1; /**< Did we alert the user about path bias
+                                      * for this node already? */
+  unsigned int path_bias_extreme : 1; /**< Did we alert the user about path
+                                       * bias for this node already? */
+  unsigned int path_bias_disabled : 1; /**< Have we disabled this node because
+                                        * of path bias issues? */
+  unsigned int path_bias_use_noticed : 1; /**< Did we alert the user about path
+                                       * use bias for this node already? */
+  unsigned int path_bias_use_extreme : 1; /**< Did we alert the user about path
+                                       * use bias for this node already? */
+
+  double circ_attempts; /**< Number of circuits this guard has "attempted" */
+  double circ_successes; /**< Number of successfully built circuits using
+                               * this guard as first hop. */
+  double successful_circuits_closed; /**< Number of circuits that carried
+                                        * streams successfully. */
+  double collapsed_circuits; /**< Number of fully built circuits that were
+                                 * remotely closed before any streams were
+                                 * attempted. */
+  double unusable_circuits; /**< Number of circuits for which streams were
+                                *  attempted, but none succeeded. */
+  double timeouts; /**< Number of 'right-censored' circuit timeouts for this
+                       * guard. */
+  double use_attempts; /**< Number of circuits we tried to use with streams */
+  double use_successes; /**< Number of successfully used circuits using
+                               * this guard as first hop. */
+} guard_pathbias_t;
+
+#if defined(ENTRYNODES_PRIVATE)
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+/**
+ * @name values for entry_guard_t.is_reachable.
+ *
+ * See entry_guard_t.is_reachable for more information.
+ */
+/**@{*/
+#define GUARD_REACHABLE_NO    0
+#define GUARD_REACHABLE_YES   1
+#define GUARD_REACHABLE_MAYBE 2
+/**@}*/
+
+/** An entry_guard_t represents our information about a chosen long-term
+ * first hop, known as a "helper" node in the literature. We can't just
+ * use a node_t, since we want to remember these even when we
+ * don't have any directory info. */
+struct entry_guard_t {
+  HANDLE_ENTRY(entry_guard, entry_guard_t);
+
+  char nickname[MAX_HEX_NICKNAME_LEN+1];
+  char identity[DIGEST_LEN];
+  ed25519_public_key_t ed_id;
+
+  /**
+   * @name new guard selection algorithm fields.
+   *
+   * Only the new (prop271) algorithm uses these.  For a more full
+   * description of the algorithm, see the module documentation for
+   * entrynodes.c
+   */
+  /**@{*/
+
+  /* == Persistent fields, present for all sampled guards. */
+  /** When was this guard added to the sample? */
+  time_t sampled_on_date;
+  /** Since what date has this guard been "unlisted"?  A guard counts as
+   * unlisted if we have a live consensus that does not include it, or
+   * if we have a live consensus that does not include it as a usable
+   * guard.  This field is zero when the guard is listed. */
+  time_t unlisted_since_date; // can be zero
+  /** What version of Tor added this guard to the sample? */
+  char *sampled_by_version;
+  /** Is this guard listed right now? If this is set, then
+   * unlisted_since_date should be set too. */
+  unsigned currently_listed : 1;
+
+  /* == Persistent fields, for confirmed guards only */
+  /** When was this guard confirmed? (That is, when did we first use it
+   * successfully and decide to keep it?) This field is zero if this is not a
+   * confirmed guard. */
+  time_t confirmed_on_date; /* 0 if not confirmed */
+  /**
+   * In what order was this guard confirmed? Guards with lower indices
+   * appear earlier on the confirmed list.  If the confirmed list is compacted,
+   * this field corresponds to the index of this guard on the confirmed list.
+   *
+   * This field is set to -1 if this guard is not confirmed.
+   */
+  int confirmed_idx; /* -1 if not confirmed; otherwise the order that this
+                      * item should occur in the CONFIRMED_GUARDS ordered
+                      * list */
+
+  /**
+   * Which selection does this guard belong to?
+   */
+  char *selection_name;
+
+  /** Bridges only: address of the bridge. */
+  tor_addr_port_t *bridge_addr;
+
+  /* ==== Non-persistent fields. */
+  /* == These are used by sampled guards */
+  /** When did we last decide to try using this guard for a circuit? 0 for
+   * "not since we started up." */
+  time_t last_tried_to_connect;
+  /** How reachable do we consider this guard to be? One of
+   * GUARD_REACHABLE_NO, GUARD_REACHABLE_YES, or GUARD_REACHABLE_MAYBE. */
+  unsigned is_reachable : 2;
+  /** Boolean: true iff this guard is pending. A pending guard is one
+   * that we have an in-progress circuit through, and which we do not plan
+   * to try again until it either succeeds or fails. Primary guards can
+   * never be pending. */
+  unsigned is_pending : 1;
+  /** If true, don't write this guard to disk. (Used for bridges with unknown
+   * identities) */
+  unsigned is_persistent : 1;
+  /** When did we get the earliest connection failure for this guard?
+   * We clear this field on a successful connect.  We do _not_ clear it
+   * when we mark the guard as "MAYBE" reachable.
+   */
+  time_t failing_since;
+
+  /* == Set inclusion flags. */
+  /** If true, this guard is in the filtered set.  The filtered set includes
+   * all sampled guards that our configuration allows us to use. */
+  unsigned is_filtered_guard : 1;
+  /** If true, this guard is in the usable filtered set. The usable filtered
+   * set includes all filtered guards that are not believed to be
+   * unreachable. (That is, those for which is_reachable is not
+   * GUARD_REACHABLE_NO) */
+  unsigned is_usable_filtered_guard : 1;
+  unsigned is_primary:1;
+
+  /** This string holds any fields that we are maintaining because
+   * we saw them in the state, even if we don't understand them. */
+  char *extra_state_fields;
+
+  /** Backpointer to the guard selection that this guard belongs to.
+   * The entry_guard_t must never outlive its guard_selection. */
+  guard_selection_t *in_selection;
+  /**@}*/
+
+  /** Path bias information for this guard. */
+  guard_pathbias_t pb;
+};
+
+/**
+ * Possible rules for a guard selection to follow
+ */
+typedef enum guard_selection_type_t {
+  /** Infer the type of this selection from its name. */
+  GS_TYPE_INFER=0,
+  /** Use the normal guard selection algorithm, taking our sample from the
+   * complete list of guards in the consensus. */
+  GS_TYPE_NORMAL=1,
+  /** Use the normal guard selection algorithm, taking our sample from the
+   * configured bridges, and allowing it to grow as large as all the configured
+   * bridges */
+  GS_TYPE_BRIDGE,
+  /** Use the normal guard selection algorithm, taking our sample from the
+   * set of filtered nodes. */
+  GS_TYPE_RESTRICTED,
+} guard_selection_type_t;
+
+/**
+ * All of the the context for guard selection on a particular client.
+ *
+ * We maintain multiple guard selection contexts for a client, depending
+ * aspects on its current configuration -- whether an extremely
+ * restrictive EntryNodes is used, whether UseBridges is enabled, and so
+ * on.)
+ *
+ * See the module documentation for entrynodes.c for more information
+ * about guard selection algorithms.
+ */
+struct guard_selection_s {
+  /**
+   * The name for this guard-selection object. (Must not contain spaces).
+   */
+  char *name;
+
+  /**
+   * What rules does this guard-selection object follow?
+   */
+  guard_selection_type_t type;
+
+  /**
+   * A value of 1 means that primary_entry_guards is up-to-date with respect to
+   * the consensus and status info that we currently have; 0 means we need to
+   * recalculate it before using primary_entry_guards or the is_primary flag on
+   * any guard.
+   */
+  int primary_guards_up_to_date;
+
+  /**
+   * A list of the sampled entry guards, as entry_guard_t structures.
+   * Not in any particular order.  When we 'sample' a guard, we are
+   * noting it as a possible guard to pick in the future. The use of
+   * sampling here prevents us from being forced by an attacker to try
+   * every guard on the network. This list is persistent.
+   */
+  smartlist_t *sampled_entry_guards;
+
+  /**
+   * Ordered list (from highest to lowest priority) of guards that we
+   * have successfully contacted and decided to use. Every member of
+   * this list is a member of sampled_entry_guards. Every member should
+   * have confirmed_on_date set, and have confirmed_idx greater than
+   * any earlier member of the list.
+   *
+   * This list is persistent. It is a subset of the elements in
+   * sampled_entry_guards, and its pointers point to elements of
+   * sampled_entry_guards.
+   */
+  smartlist_t *confirmed_entry_guards;
+
+  /**
+   * Ordered list (from highest to lowest priority) of guards that we
+   * are willing to use the most happily.  These guards may or may not
+   * yet be confirmed yet.  If we can use one of these guards, we are
+   * probably not on a network that is trying to restrict our guard
+   * choices.
+   *
+   * This list is a subset of the elements in
+   * sampled_entry_guards, and its pointers point to elements of
+   * sampled_entry_guards.
+   */
+  smartlist_t *primary_entry_guards;
+
+  /** When did we last successfully build a circuit or use a circuit? */
+  time_t last_time_on_internet;
+
+  /** What confirmed_idx value should the next-added member of
+   * confirmed_entry_guards receive? */
+  int next_confirmed_idx;
+
+};
+
+struct entry_guard_handle_t;
+
+/** Types of restrictions we impose when picking guard nodes */
+typedef enum guard_restriction_type_t {
+  /* Don't pick the same guard node as our exit node (or its family) */
+  RST_EXIT_NODE = 0,
+  /* Don't pick dirguards that have previously shown to be outdated */
+  RST_OUTDATED_MD_DIRSERVER = 1
+} guard_restriction_type_t;
+
+/**
+ * A restriction to remember which entry guards are off-limits for a given
+ * circuit.
+ *
+ * Note: This mechanism is NOT for recording which guards are never to be
+ * used: only which guards cannot be used on <em>one particular circuit</em>.
+ */
+struct entry_guard_restriction_t {
+  /* What type of restriction are we imposing? */
+  guard_restriction_type_t type;
+
+  /* In case of restriction type RST_EXIT_NODE, the guard's RSA identity
+   * digest must not equal this; and it must not be in the same family as any
+   * node with this digest. */
+  uint8_t exclude_id[DIGEST_LEN];
+};
+
+/**
+ * Per-circuit state to track whether we'll be able to use the circuit.
+ */
+struct circuit_guard_state_t {
+  /** Handle to the entry guard object for this circuit. */
+  struct entry_guard_handle_t *guard;
+  /** The time at which <b>state</b> last changed. */
+  time_t state_set_at;
+  /** One of GUARD_CIRC_STATE_* */
+  uint8_t state;
+
+  /**
+   * A set of restrictions that were placed on this guard when we selected it
+   * for this particular circuit.  We need to remember the restrictions here,
+   * since any guard that breaks these restrictions will not block this
+   * circuit from becoming COMPLETE.
+   */
+  entry_guard_restriction_t *restrictions;
+};
+#endif /* defined(ENTRYNODES_PRIVATE) */
+
+/* Common entry points for old and new guard code */
+int guards_update_all(void);
+const node_t *guards_choose_guard(cpath_build_state_t *state,
+                                  uint8_t purpose,
+                                  circuit_guard_state_t **guard_state_out);
+const node_t *guards_choose_dirguard(uint8_t dir_purpose,
+                                     circuit_guard_state_t **guard_state_out);
+
+#if 1
+/* XXXX NM I would prefer that all of this stuff be private to
+ * entrynodes.c. */
+entry_guard_t *entry_guard_get_by_id_digest_for_guard_selection(
+    guard_selection_t *gs, const char *digest);
+entry_guard_t *entry_guard_get_by_id_digest(const char *digest);
+
+circuit_guard_state_t *
+get_guard_state_for_bridge_desc_fetch(const char *digest);
+
+void entry_guards_changed_for_guard_selection(guard_selection_t *gs);
+void entry_guards_changed(void);
+guard_selection_t * get_guard_selection_info(void);
+int num_live_entry_guards_for_guard_selection(
+    guard_selection_t *gs,
+    int for_directory);
+int num_live_entry_guards(int for_directory);
+#endif /* 1 */
+
+const node_t *entry_guard_find_node(const entry_guard_t *guard);
+const char *entry_guard_get_rsa_id_digest(const entry_guard_t *guard);
+const char *entry_guard_describe(const entry_guard_t *guard);
+guard_pathbias_t *entry_guard_get_pathbias_state(entry_guard_t *guard);
+
+/** Enum to specify how we're going to use a given guard, when we're picking
+ * one for immediate use. */
+typedef enum {
+  GUARD_USAGE_TRAFFIC = 0,
+  GUARD_USAGE_DIRGUARD = 1
+} guard_usage_t;
+
+#define circuit_guard_state_free(val) \
+  FREE_AND_NULL(circuit_guard_state_t, circuit_guard_state_free_, (val))
+
+void circuit_guard_state_free_(circuit_guard_state_t *state);
+int entry_guard_pick_for_circuit(guard_selection_t *gs,
+                                 guard_usage_t usage,
+                                 entry_guard_restriction_t *rst,
+                                 const node_t **chosen_node_out,
+                                 circuit_guard_state_t **guard_state_out);
+
+/* We just connected to an entry guard. What should we do with the circuit? */
+typedef enum {
+  GUARD_USABLE_NEVER = -1, /* Never use the circuit */
+  GUARD_MAYBE_USABLE_LATER = 0, /* Keep it. We might use it in the future */
+  GUARD_USABLE_NOW = 1, /* Use it right now */
+} guard_usable_t;
+
+guard_usable_t entry_guard_succeeded(circuit_guard_state_t **guard_state_p);
+void entry_guard_failed(circuit_guard_state_t **guard_state_p);
+void entry_guard_cancel(circuit_guard_state_t **guard_state_p);
+void entry_guard_chan_failed(channel_t *chan);
+int entry_guards_update_all(guard_selection_t *gs);
+int entry_guards_upgrade_waiting_circuits(guard_selection_t *gs,
+                                          const smartlist_t *all_circuits,
+                                          smartlist_t *newly_complete_out);
+int entry_guard_state_should_expire(circuit_guard_state_t *guard_state);
+void entry_guards_note_internet_connectivity(guard_selection_t *gs);
+
+int update_guard_selection_choice(const or_options_t *options);
+
+int entry_guard_could_succeed(const circuit_guard_state_t *guard_state);
+
+MOCK_DECL(int,num_bridges_usable,(int use_maybe_reachable));
+
+#ifdef ENTRYNODES_PRIVATE
+/**
+ * @name Default values for the parameters for the new (prop271) entry guard
+ * algorithm.
+ */
+/**@{*/
+/**
+ * We never let our sampled guard set grow larger than this percentage
+ * of the guards on the network.
+ */
+#define DFLT_MAX_SAMPLE_THRESHOLD_PERCENT 20
+/**
+ * We never let our sampled guard set grow larger than this number of
+ * guards.
+ */
+#define DFLT_MAX_SAMPLE_SIZE 60
+/**
+ * We always try to make our sample contain at least this many guards.
+ */
+#define DFLT_MIN_FILTERED_SAMPLE_SIZE 20
+/**
+ * If a guard is unlisted for this many days in a row, we remove it.
+ */
+#define DFLT_REMOVE_UNLISTED_GUARDS_AFTER_DAYS 20
+/**
+ * We remove unconfirmed guards from the sample after this many days,
+ * regardless of whether they are listed or unlisted.
+ */
+#define DFLT_GUARD_LIFETIME_DAYS 120
+/**
+ * We remove confirmed guards from the sample if they were sampled
+ * GUARD_LIFETIME_DAYS ago and confirmed this many days ago.
+ */
+#define DFLT_GUARD_CONFIRMED_MIN_LIFETIME_DAYS 60
+/**
+ * How many guards do we try to keep on our primary guard list?
+ */
+#define DFLT_N_PRIMARY_GUARDS 3
+/**
+ * Of the live guards on the primary guard list, how many do we consider when
+ * choosing a guard to use?
+ */
+#define DFLT_N_PRIMARY_GUARDS_TO_USE 1
+/**
+ * As DFLT_N_PRIMARY_GUARDS, but for choosing which directory guard to use.
+ */
+#define DFLT_N_PRIMARY_DIR_GUARDS_TO_USE 3
+/**
+ * If we haven't successfully built or used a circuit in this long, then
+ * consider that the internet is probably down.
+ */
+#define DFLT_INTERNET_LIKELY_DOWN_INTERVAL (10*60)
+/**
+ * If we're trying to connect to a nonprimary guard for at least this
+ * many seconds, and we haven't gotten the connection to work, we will treat
+ * lower-priority guards as usable.
+ */
+#define DFLT_NONPRIMARY_GUARD_CONNECT_TIMEOUT 15
+/**
+ * If a circuit has been sitting around in 'waiting for better guard' state
+ * for at least this long, we'll expire it.
+ */
+#define DFLT_NONPRIMARY_GUARD_IDLE_TIMEOUT (10*60)
+/**
+ * If our configuration retains fewer than this fraction of guards from the
+ * torrc, we are in a restricted setting.
+ */
+#define DFLT_MEANINGFUL_RESTRICTION_PERCENT 20
+/**
+ * If our configuration retains fewer than this fraction of guards from the
+ * torrc, we are in an extremely restricted setting, and should warn.
+ */
+#define DFLT_EXTREME_RESTRICTION_PERCENT 1
+/**@}*/
+
+STATIC double get_max_sample_threshold(void);
+STATIC int get_max_sample_size_absolute(void);
+STATIC int get_min_filtered_sample_size(void);
+STATIC int get_remove_unlisted_guards_after_days(void);
+STATIC int get_guard_lifetime(void);
+STATIC int get_guard_confirmed_min_lifetime(void);
+STATIC int get_n_primary_guards(void);
+STATIC int get_n_primary_guards_to_use(guard_usage_t usage);
+STATIC int get_internet_likely_down_interval(void);
+STATIC int get_nonprimary_guard_connect_timeout(void);
+STATIC int get_nonprimary_guard_idle_timeout(void);
+STATIC double get_meaningful_restriction_threshold(void);
+STATIC double get_extreme_restriction_threshold(void);
+
+HANDLE_DECL(entry_guard, entry_guard_t, STATIC)
+#define entry_guard_handle_free(h)    \
+  FREE_AND_NULL(entry_guard_handle_t, entry_guard_handle_free_, (h))
+
+STATIC guard_selection_type_t guard_selection_infer_type(
+                           guard_selection_type_t type_in,
+                           const char *name);
+STATIC guard_selection_t *guard_selection_new(const char *name,
+                                              guard_selection_type_t type);
+STATIC guard_selection_t *get_guard_selection_by_name(
+          const char *name, guard_selection_type_t type, int create_if_absent);
+STATIC void guard_selection_free_(guard_selection_t *gs);
+#define guard_selection_free(gs) \
+  FREE_AND_NULL(guard_selection_t, guard_selection_free_, (gs))
+MOCK_DECL(STATIC int, entry_guard_is_listed,
+          (guard_selection_t *gs, const entry_guard_t *guard));
+STATIC const char *choose_guard_selection(const or_options_t *options,
+                                        const networkstatus_t *ns,
+                                        const guard_selection_t *old_selection,
+                                        guard_selection_type_t *type_out);
+STATIC entry_guard_t *get_sampled_guard_with_id(guard_selection_t *gs,
+                                                const uint8_t *rsa_id);
+
+MOCK_DECL(STATIC time_t, randomize_time, (time_t now, time_t max_backdate));
+
+MOCK_DECL(STATIC circuit_guard_state_t *,
+          circuit_guard_state_new,(entry_guard_t *guard, unsigned state,
+                                   entry_guard_restriction_t *rst));
+
+STATIC entry_guard_t *entry_guard_add_to_sample(guard_selection_t *gs,
+                                                const node_t *node);
+STATIC entry_guard_t *entry_guards_expand_sample(guard_selection_t *gs);
+STATIC char *entry_guard_encode_for_state(entry_guard_t *guard);
+STATIC entry_guard_t *entry_guard_parse_from_state(const char *s);
+#define entry_guard_free(e) \
+  FREE_AND_NULL(entry_guard_t, entry_guard_free_, (e))
+STATIC void entry_guard_free_(entry_guard_t *e);
+STATIC void entry_guards_update_filtered_sets(guard_selection_t *gs);
+STATIC int entry_guards_all_primary_guards_are_down(guard_selection_t *gs);
+/**
+ * @name Flags for sample_reachable_filtered_entry_guards()
+ */
+/**@{*/
+#define SAMPLE_EXCLUDE_CONFIRMED   (1u<<0)
+#define SAMPLE_EXCLUDE_PRIMARY     (1u<<1)
+#define SAMPLE_EXCLUDE_PENDING     (1u<<2)
+#define SAMPLE_NO_UPDATE_PRIMARY   (1u<<3)
+#define SAMPLE_EXCLUDE_NO_DESCRIPTOR (1u<<4)
+/**@}*/
+STATIC entry_guard_t *sample_reachable_filtered_entry_guards(
+                                    guard_selection_t *gs,
+                                    const entry_guard_restriction_t *rst,
+                                    unsigned flags);
+STATIC void entry_guard_consider_retry(entry_guard_t *guard);
+STATIC void make_guard_confirmed(guard_selection_t *gs, entry_guard_t *guard);
+STATIC void entry_guards_update_confirmed(guard_selection_t *gs);
+STATIC void entry_guards_update_primary(guard_selection_t *gs);
+STATIC int num_reachable_filtered_guards(const guard_selection_t *gs,
+                                         const entry_guard_restriction_t *rst);
+STATIC void sampled_guards_update_from_consensus(guard_selection_t *gs);
+/**
+ * @name Possible guard-states for a circuit.
+ */
+/**@{*/
+/** State for a circuit that can (so far as the guard subsystem is
+ * concerned) be used for actual traffic as soon as it is successfully
+ * opened. */
+#define GUARD_CIRC_STATE_USABLE_ON_COMPLETION 1
+/** State for an non-open circuit that we shouldn't use for actual
+ * traffic, when it completes, unless other circuits to preferable
+ * guards fail. */
+#define GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD 2
+/** State for an open circuit that we shouldn't use for actual traffic
+ * unless other circuits to preferable guards fail. */
+#define GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD 3
+/** State for a circuit that can (so far as the guard subsystem is
+ * concerned) be used for actual traffic. */
+#define GUARD_CIRC_STATE_COMPLETE 4
+/** State for a circuit that is unusable, and will not become usable. */
+#define GUARD_CIRC_STATE_DEAD 5
+/**@}*/
+STATIC void entry_guards_note_guard_failure(guard_selection_t *gs,
+                                            entry_guard_t *guard);
+STATIC entry_guard_t *select_entry_guard_for_circuit(guard_selection_t *gs,
+                                          guard_usage_t usage,
+                                          const entry_guard_restriction_t *rst,
+                                          unsigned *state_out);
+STATIC void mark_primary_guards_maybe_reachable(guard_selection_t *gs);
+STATIC unsigned entry_guards_note_guard_success(guard_selection_t *gs,
+                                                entry_guard_t *guard,
+                                                unsigned old_state);
+STATIC int entry_guard_has_higher_priority(entry_guard_t *a, entry_guard_t *b);
+STATIC char *getinfo_helper_format_single_entry_guard(const entry_guard_t *e);
+
+STATIC entry_guard_restriction_t *guard_create_exit_restriction(
+                                                      const uint8_t *exit_id);
+
+STATIC entry_guard_restriction_t *guard_create_dirserver_md_restriction(void);
+
+STATIC void entry_guard_restriction_free_(entry_guard_restriction_t *rst);
+#define entry_guard_restriction_free(rst)  \
+  FREE_AND_NULL(entry_guard_restriction_t, \
+                entry_guard_restriction_free_, (rst))
+
+#endif /* defined(ENTRYNODES_PRIVATE) */
+
+void remove_all_entry_guards_for_guard_selection(guard_selection_t *gs);
+void remove_all_entry_guards(void);
+
+struct bridge_info_t;
+void entry_guard_learned_bridge_identity(const tor_addr_port_t *addrport,
+                                         const uint8_t *rsa_id_digest);
+
+int entry_list_is_constrained(const or_options_t *options);
+int guards_retry_optimistic(const or_options_t *options);
+int entry_guards_parse_state_for_guard_selection(
+    guard_selection_t *gs, or_state_t *state, int set, char **msg);
+int entry_guards_parse_state(or_state_t *state, int set, char **msg);
+void entry_guards_update_state(or_state_t *state);
+int getinfo_helper_entry_guards(control_connection_t *conn,
+                                const char *question, char **answer,
+                                const char **errmsg);
+
+int entries_known_but_down(const or_options_t *options);
+void entries_retry_all(const or_options_t *options);
+
+char *entry_guards_get_err_str_if_dir_info_missing(int using_mds,
+                                           int num_present, int num_usable);
+char *guard_selection_get_err_str_if_dir_info_missing(guard_selection_t *gs,
+                                              int using_mds,
+                                              int num_present, int num_usable);
+
+void entry_guards_free_all(void);
+
+double pathbias_get_close_success_count(entry_guard_t *guard);
+double pathbias_get_use_success_count(entry_guard_t *guard);
+
+/** Contains the bandwidth of a relay as a guard and as a non-guard
+ *  after the guardfraction has been considered. */
+typedef struct guardfraction_bandwidth_t {
+  /** Bandwidth as a guard after guardfraction has been considered. */
+  int guard_bw;
+  /** Bandwidth as a non-guard after guardfraction has been considered. */
+  int non_guard_bw;
+} guardfraction_bandwidth_t;
+
+int should_apply_guardfraction(const networkstatus_t *ns);
+
+void
+guard_get_guardfraction_bandwidth(guardfraction_bandwidth_t *guardfraction_bw,
+                                  int orig_bandwidth,
+                                  uint32_t guardfraction_percentage);
+
+#endif /* !defined(TOR_ENTRYNODES_H) */
diff --git a/src/or/transports.c b/src/feature/client/transports.c
index 7a52b737e4..0d1342c87b 100644
--- a/src/or/transports.c
+++ b/src/feature/client/transports.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2016, The Tor Project, Inc. */
+/* Copyright (c) 2011-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -90,17 +90,20 @@
  **/
 
 #define PT_PRIVATE
-#include "or.h"
-#include "config.h"
-#include "circuitbuild.h"
-#include "transports.h"
-#include "util.h"
-#include "router.h"
-#include "statefile.h"
-#include "entrynodes.h"
-#include "connection_or.h"
-#include "ext_orport.h"
-#include "control.h"
+#include "core/or/or.h"
+#include "feature/client/bridges.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/circuitbuild.h"
+#include "feature/client/transports.h"
+#include "feature/relay/router.h"
+#include "app/config/statefile.h"
+#include "core/or/connection_or.h"
+#include "feature/relay/ext_orport.h"
+#include "feature/control/control.h"
+
+#include "lib/process/env.h"
+#include "lib/process/subprocess.h"
 
 static process_environment_t *
 create_managed_proxy_environment(const managed_proxy_t *mp);
@@ -135,7 +138,7 @@ static smartlist_t *transport_list = NULL;
 /** Returns a transport_t struct for a transport proxy supporting the
     protocol <b>name</b> listening at <b>addr</b>:<b>port</b> using
     SOCKS version <b>socks_ver</b>. */
-static transport_t *
+STATIC transport_t *
 transport_new(const tor_addr_t *addr, uint16_t port,
               const char *name, int socks_ver,
               const char *extra_info_args)
@@ -154,7 +157,7 @@ transport_new(const tor_addr_t *addr, uint16_t port,
 
 /** Free the pluggable transport struct <b>transport</b>. */
 void
-transport_free(transport_t *transport)
+transport_free_(transport_t *transport)
 {
   if (!transport)
     return;
@@ -222,8 +225,8 @@ transport_copy(const transport_t *transport)
 
 /** Returns the transport in our transport list that has the name <b>name</b>.
  *  Else returns NULL. */
-transport_t *
-transport_get_by_name(const char *name)
+MOCK_IMPL(transport_t *,
+transport_get_by_name,(const char *name))
 {
   tor_assert(name);
 
@@ -430,7 +433,7 @@ add_transport_to_proxy(const char *transport, managed_proxy_t *mp)
 {
   tor_assert(mp->transports_to_launch);
   if (!smartlist_contains_string(mp->transports_to_launch, transport))
-    smartlist_add(mp->transports_to_launch, tor_strdup(transport));
+    smartlist_add_strdup(mp->transports_to_launch, transport);
 }
 
 /** Called when a SIGHUP occurs. Returns true if managed proxy
@@ -480,7 +483,6 @@ proxy_needs_restart(const managed_proxy_t *mp)
  *  preparations and then flag its state so that it will be relaunched
  *  in the next tick. */
 static void
-
 proxy_prepare_for_restart(managed_proxy_t *mp)
 {
   transport_t *t_tmp = NULL;
@@ -528,12 +530,12 @@ launch_managed_proxy(managed_proxy_t *mp)
                                 (const char **)mp->argv,
                                 env,
                                 &mp->process_handle);
-#else
+#else /* !(defined(_WIN32)) */
   retval = tor_spawn_background(mp->argv[0],
                                 (const char **)mp->argv,
                                 env,
                                 &mp->process_handle);
-#endif
+#endif /* defined(_WIN32) */
 
   process_environment_free(env);
 
@@ -591,7 +593,7 @@ pt_configure_remaining_proxies(void)
     }
 
     /* If the proxy is not fully configured, try to configure it
-       futher. */
+       further. */
     if (!proxy_configuration_finished(mp))
       if (configure_proxy(mp) == 1)
         at_least_a_proxy_config_finished = 1;
@@ -1026,48 +1028,71 @@ parse_method_error(const char *line, int is_server)
            line+strlen(error)+1);
 }
 
-/** Parses an SMETHOD <b>line</b> and if well-formed it registers the
- *  new transport in <b>mp</b>. */
-STATIC int
-parse_smethod_line(const char *line, managed_proxy_t *mp)
+/** A helper for parse_{c,s}method_line(), bootstraps its
+ *  functionalities. If <b>is_smethod</b> is true then the
+ *  the line to parse is a SMETHOD line otherwise it is a
+ *  CMETHOD line*/
+static int
+parse_method_line_helper(const char *line,
+                         managed_proxy_t *mp,
+                         int is_smethod)
 {
+  int item_index = 0;
   int r;
-  smartlist_t *items = NULL;
 
-  char *method_name=NULL;
+  char *transport_name=NULL;
   char *args_string=NULL;
   char *addrport=NULL;
-  tor_addr_t tor_addr;
+  int socks_ver=PROXY_NONE;
   char *address=NULL;
   uint16_t port = 0;
 
+  const char *method_str = is_smethod ? PROTO_SMETHOD : PROTO_CMETHOD;
+  const int min_args_count = is_smethod ? 3 : 4;
+
+  tor_addr_t tor_addr;
   transport_t *transport=NULL;
+  smartlist_t *items= smartlist_new();
 
-  items = smartlist_new();
   smartlist_split_string(items, line, NULL,
                          SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
-  if (smartlist_len(items) < 3) {
-    log_warn(LD_CONFIG, "Server managed proxy sent us a SMETHOD line "
-             "with too few arguments.");
+  if (smartlist_len(items) < min_args_count) {
+    log_warn(LD_CONFIG, "Managed proxy sent us a %s line "
+             "with too few arguments.", method_str);
     goto err;
   }
 
-  /* Example of legit SMETHOD line:
-     SMETHOD obfs2 0.0.0.0:25612 ARGS:secret=supersekrit,key=superkey */
-
-  tor_assert(!strcmp(smartlist_get(items,0),PROTO_SMETHOD));
+  tor_assert(!strcmp(smartlist_get(items, item_index),method_str));
+  ++item_index;
 
-  method_name = smartlist_get(items,1);
-  if (!string_is_C_identifier(method_name)) {
+  transport_name = smartlist_get(items,item_index);
+  ++item_index;
+  if (!string_is_C_identifier(transport_name)) {
     log_warn(LD_CONFIG, "Transport name is not a C identifier (%s).",
-             method_name);
+             transport_name);
     goto err;
   }
 
-  addrport = smartlist_get(items, 2);
+  /** Check for the proxy method sent to us in CMETHOD line. */
+  if (!is_smethod) {
+    const char *socks_ver_str = smartlist_get(items,item_index);
+    ++item_index;
+
+    if (!strcmp(socks_ver_str,"socks4")) {
+      socks_ver = PROXY_SOCKS4;
+    } else if (!strcmp(socks_ver_str,"socks5")) {
+      socks_ver = PROXY_SOCKS5;
+    } else {
+      log_warn(LD_CONFIG, "Client managed proxy sent us a proxy protocol "
+               "we don't recognize. (%s)", socks_ver_str);
+      goto err;
+    }
+  }
+
+  addrport = smartlist_get(items, item_index);
+  ++item_index;
   if (tor_addr_port_split(LOG_WARN, addrport, &address, &port)<0) {
-    log_warn(LD_CONFIG, "Error parsing transport "
-             "address '%s'", addrport);
+    log_warn(LD_CONFIG, "Error parsing transport address '%s'", addrport);
     goto err;
   }
 
@@ -1082,10 +1107,11 @@ parse_smethod_line(const char *line, managed_proxy_t *mp)
     goto err;
   }
 
-  if (smartlist_len(items) > 3) {
+  /** Check for options in the SMETHOD line. */
+  if (is_smethod && smartlist_len(items) > min_args_count) {
     /* Seems like there are also some [options] in the SMETHOD line.
        Let's see if we can parse them. */
-    char *options_string = smartlist_get(items, 3);
+    char *options_string = smartlist_get(items, item_index);
     log_debug(LD_CONFIG, "Got options_string: %s", options_string);
     if (!strcmpstart(options_string, "ARGS:")) {
       args_string = options_string+strlen("ARGS:");
@@ -1093,17 +1119,20 @@ parse_smethod_line(const char *line, managed_proxy_t *mp)
     }
   }
 
-  transport = transport_new(&tor_addr, port, method_name,
-                            PROXY_NONE, args_string);
-  if (!transport)
-    goto err;
+  transport = transport_new(&tor_addr, port, transport_name,
+                            socks_ver, args_string);
 
   smartlist_add(mp->transports, transport);
 
-  /* For now, notify the user so that they know where the server
-     transport is listening. */
-  log_info(LD_CONFIG, "Server transport %s at %s:%d.",
-           method_name, address, (int)port);
+  /** Logs info about line parsing success for client or server */
+  if (is_smethod) {
+    log_info(LD_CONFIG, "Server transport %s at %s:%d.",
+             transport_name, address, (int)port);
+  } else {
+    log_info(LD_CONFIG, "Transport %s at %s:%d with SOCKS %d. "
+             "Attached to managed proxy.",
+             transport_name, address, (int)port, socks_ver);
+  }
 
   r=0;
   goto done;
@@ -1118,95 +1147,24 @@ parse_smethod_line(const char *line, managed_proxy_t *mp)
   return r;
 }
 
+/** Parses an SMETHOD <b>line</b> and if well-formed it registers the
+ *  new transport in <b>mp</b>. */
+STATIC int
+parse_smethod_line(const char *line, managed_proxy_t *mp)
+{
+  /* Example of legit SMETHOD line:
+     SMETHOD obfs2 0.0.0.0:25612 ARGS:secret=supersekrit,key=superkey */
+  return parse_method_line_helper(line, mp, 1);
+}
+
 /** Parses a CMETHOD <b>line</b>, and if well-formed it registers
  *  the new transport in <b>mp</b>. */
 STATIC int
 parse_cmethod_line(const char *line, managed_proxy_t *mp)
 {
-  int r;
-  smartlist_t *items = NULL;
-
-  char *method_name=NULL;
-
-  char *socks_ver_str=NULL;
-  int socks_ver=PROXY_NONE;
-
-  char *addrport=NULL;
-  tor_addr_t tor_addr;
-  char *address=NULL;
-  uint16_t port = 0;
-
-  transport_t *transport=NULL;
-
-  items = smartlist_new();
-  smartlist_split_string(items, line, NULL,
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
-  if (smartlist_len(items) < 4) {
-    log_warn(LD_CONFIG, "Client managed proxy sent us a CMETHOD line "
-             "with too few arguments.");
-    goto err;
-  }
-
-  tor_assert(!strcmp(smartlist_get(items,0),PROTO_CMETHOD));
-
-  method_name = smartlist_get(items,1);
-  if (!string_is_C_identifier(method_name)) {
-    log_warn(LD_CONFIG, "Transport name is not a C identifier (%s).",
-             method_name);
-    goto err;
-  }
-
-  socks_ver_str = smartlist_get(items,2);
-
-  if (!strcmp(socks_ver_str,"socks4")) {
-    socks_ver = PROXY_SOCKS4;
-  } else if (!strcmp(socks_ver_str,"socks5")) {
-    socks_ver = PROXY_SOCKS5;
-  } else {
-    log_warn(LD_CONFIG, "Client managed proxy sent us a proxy protocol "
-             "we don't recognize. (%s)", socks_ver_str);
-    goto err;
-  }
-
-  addrport = smartlist_get(items, 3);
-  if (tor_addr_port_split(LOG_WARN, addrport, &address, &port)<0) {
-    log_warn(LD_CONFIG, "Error parsing transport "
-             "address '%s'", addrport);
-    goto err;
-  }
-
-  if (!port) {
-    log_warn(LD_CONFIG,
-             "Transport address '%s' has no port.", addrport);
-    goto err;
-  }
-
-  if (tor_addr_parse(&tor_addr, address) < 0) {
-    log_warn(LD_CONFIG, "Error parsing transport address '%s'", address);
-    goto err;
-  }
-
-  transport = transport_new(&tor_addr, port, method_name, socks_ver, NULL);
-  if (!transport)
-    goto err;
-
-  smartlist_add(mp->transports, transport);
-
-  log_info(LD_CONFIG, "Transport %s at %s:%d with SOCKS %d. "
-           "Attached to managed proxy.",
-           method_name, address, (int)port, socks_ver);
-
-  r=0;
-  goto done;
-
- err:
-  r = -1;
-
- done:
-  SMARTLIST_FOREACH(items, char*, s, tor_free(s));
-  smartlist_free(items);
-  tor_free(address);
-  return r;
+  /* Example of legit CMETHOD line:
+     CMETHOD obfs2 socks5 127.0.0.1:35713 */
+  return parse_method_line_helper(line, mp, 0);
 }
 
 /** Parses an PROXY-ERROR <b>line</b> and warns the user accordingly. */
@@ -1322,7 +1280,7 @@ create_managed_proxy_environment(const managed_proxy_t *mp)
     tor_free(state_tmp);
   }
 
-  smartlist_add(envs, tor_strdup("TOR_PT_MANAGED_TRANSPORT_VER=1"));
+  smartlist_add_strdup(envs, "TOR_PT_MANAGED_TRANSPORT_VER=1");
 
   {
     char *transports_to_launch =
@@ -1742,3 +1700,39 @@ pt_free_all(void)
   }
 }
 
+/** Return a newly allocated string equal to <b>string</b>, except that every
+ * character in <b>chars_to_escape</b> is preceded by a backslash. */
+char *
+tor_escape_str_for_pt_args(const char *string, const char *chars_to_escape)
+{
+  char *new_string = NULL;
+  char *new_cp = NULL;
+  size_t length, new_length;
+
+  tor_assert(string);
+
+  length = strlen(string);
+
+  if (!length) /* If we were given the empty string, return the same. */
+    return tor_strdup("");
+  /* (new_length > SIZE_MAX) => ((length * 2) + 1 > SIZE_MAX) =>
+     (length*2 > SIZE_MAX - 1) => (length > (SIZE_MAX - 1)/2) */
+  if (length > (SIZE_MAX - 1)/2) /* check for overflow */
+    return NULL;
+
+  /* this should be enough even if all characters must be escaped */
+  new_length = (length * 2) + 1;
+
+  new_string = new_cp = tor_malloc(new_length);
+
+  while (*string) {
+    if (strchr(chars_to_escape, *string))
+      *new_cp++ = '\\';
+
+    *new_cp++ = *string++;
+  }
+
+  *new_cp = '\0'; /* NUL-terminate the new string */
+
+  return new_string;
+}
diff --git a/src/or/transports.h b/src/feature/client/transports.h
index 7de90dcbec..e2fa45828f 100644
--- a/src/or/transports.h
+++ b/src/feature/client/transports.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -35,9 +35,10 @@ void sweep_transport_list(void);
 MOCK_DECL(int, transport_add_from_config,
           (const tor_addr_t *addr, uint16_t port,
            const char *name, int socks_ver));
-void transport_free(transport_t *transport);
+void transport_free_(transport_t *transport);
+#define transport_free(tr) FREE_AND_NULL(transport_t, transport_free_, (tr))
 
-transport_t *transport_get_by_name(const char *name);
+MOCK_DECL(transport_t*, transport_get_by_name, (const char *name));
 
 MOCK_DECL(void, pt_kickstart_proxy,
           (const smartlist_t *transport_list, char **proxy_argv,
@@ -65,6 +66,9 @@ char *pt_stringify_socks_args(const smartlist_t *socks_args);
 char *pt_get_socks_args_for_proxy_addrport(const tor_addr_t *addr,
                                             uint16_t port);
 
+char *tor_escape_str_for_pt_args(const char *string,
+                                 const char *chars_to_escape);
+
 #ifdef PT_PRIVATE
 /** State of the managed proxy configuration protocol. */
 enum pt_proto_state {
@@ -77,6 +81,8 @@ enum pt_proto_state {
   PT_PROTO_FAILED_LAUNCH /* failed while launching */
 };
 
+struct process_handle_t;
+
 /** Structure containing information of a managed proxy. */
 typedef struct {
   enum pt_proto_state conf_state; /* the current configuration state */
@@ -89,7 +95,7 @@ typedef struct {
   int is_server; /* is it a server proxy? */
 
   /* A pointer to the process handle of this managed proxy. */
-  process_handle_t *process_handle;
+  struct process_handle_t *process_handle;
 
   int pid; /* The Process ID this managed proxy is using. */
 
@@ -112,6 +118,9 @@ typedef struct {
   smartlist_t *transports;
 } managed_proxy_t;
 
+STATIC transport_t *transport_new(const tor_addr_t *addr, uint16_t port,
+                                  const char *name, int socks_ver,
+                                  const char *extra_info_args);
 STATIC int parse_cmethod_line(const char *line, managed_proxy_t *mp);
 STATIC int parse_smethod_line(const char *line, managed_proxy_t *mp);
 
@@ -133,7 +142,6 @@ STATIC char* get_pt_proxy_uri(void);
 
 STATIC void free_execve_args(char **arg);
 
-#endif
-
-#endif
+#endif /* defined(PT_PRIVATE) */
 
+#endif /* !defined(TOR_TRANSPORTS_H) */
diff --git a/src/or/control.c b/src/feature/control/control.c
index ff7f2e8b85..cc7ecff2ff 100644
--- a/src/or/control.c
+++ b/src/feature/control/control.c
@@ -1,5 +1,6 @@
+
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -34,51 +35,97 @@
 
 #define CONTROL_PRIVATE
 
-#include "or.h"
-#include "addressmap.h"
-#include "buffers.h"
-#include "channel.h"
-#include "channeltls.h"
-#include "circuitbuild.h"
-#include "circuitlist.h"
-#include "circuitstats.h"
-#include "circuituse.h"
-#include "command.h"
-#include "compat_libevent.h"
-#include "config.h"
-#include "confparse.h"
-#include "connection.h"
-#include "connection_edge.h"
-#include "connection_or.h"
-#include "control.h"
-#include "directory.h"
-#include "dirserv.h"
-#include "dnsserv.h"
-#include "entrynodes.h"
-#include "geoip.h"
-#include "hibernate.h"
-#include "main.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "policies.h"
-#include "reasons.h"
-#include "rendclient.h"
-#include "rendcommon.h"
-#include "rendservice.h"
-#include "rephist.h"
-#include "router.h"
-#include "routerlist.h"
-#include "routerparse.h"
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "app/config/confparse.h"
+#include "app/main/main.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/channel.h"
+#include "core/or/channeltls.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitstats.h"
+#include "core/or/circuituse.h"
+#include "core/or/command.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "core/or/policies.h"
+#include "core/or/reasons.h"
+#include "core/or/versions.h"
+#include "core/proto/proto_control0.h"
+#include "core/proto/proto_http.h"
+#include "feature/client/addressmap.h"
+#include "feature/client/bridges.h"
+#include "feature/client/dnsserv.h"
+#include "feature/client/entrynodes.h"
+#include "feature/control/control.h"
+#include "feature/control/fmt_serverstatus.h"
+#include "feature/control/getinfo_geoip.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
+#include "feature/dircommon/directory.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/hs/hs_cache.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_control.h"
+#include "feature/hs_common/shared_random_client.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerinfo.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+#include "feature/relay/selftest.h"
+#include "feature/rend/rendclient.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/rend/rendparse.h"
+#include "feature/rend/rendservice.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/stats/predict_ports.h"
+#include "lib/container/buffers.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/encoding/confline.h"
+#include "lib/evloop/compat_libevent.h"
+
+#include "feature/dircache/cached_dir_st.h"
+#include "feature/control/control_connection_st.h"
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/entry_connection_st.h"
+#include "feature/nodelist/extrainfo_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "feature/nodelist/microdesc_st.h"
+#include "feature/rend/rend_authorized_client_st.h"
+#include "feature/rend/rend_encoded_v2_service_descriptor_st.h"
+#include "feature/rend/rend_service_descriptor_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerlist_st.h"
+#include "core/or/socks_request_st.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
 
 #ifndef _WIN32
 #include <pwd.h>
 #include <sys/resource.h>
 #endif
 
-#include <event2/event.h>
-
-#include "crypto_s2k.h"
-#include "procmon.h"
+#include "lib/crypt_ops/crypto_s2k.h"
+#include "lib/evloop/procmon.h"
+#include "lib/evloop/compat_libevent.h"
 
 /** Yield true iff <b>s</b> is the state of a control_connection_t that has
  * finished authentication and is accepting commands. */
@@ -104,6 +151,10 @@ static int disable_log_messages = 0;
 #define EVENT_IS_INTERESTING(e) \
   (!! (global_event_mask & EVENT_MASK_(e)))
 
+/** Macro: true if any event from the bitfield 'e' is interesting. */
+#define ANY_EVENT_IS_INTERESTING(e) \
+  (!! (global_event_mask & (e)))
+
 /** If we're using cookie-type authentication, how long should our cookies be?
  */
 #define AUTHENTICATION_COOKIE_LEN 32
@@ -208,9 +259,19 @@ static void orconn_target_get_name(char *buf, size_t len,
 static int get_cached_network_liveness(void);
 static void set_cached_network_liveness(int liveness);
 
-static void flush_queued_events_cb(evutil_socket_t fd, short what, void *arg);
+static void flush_queued_events_cb(mainloop_event_t *event, void *arg);
 
 static char * download_status_to_string(const download_status_t *dl);
+static void control_get_bytes_rw_last_sec(uint64_t *r, uint64_t *w);
+
+/** Convert a connection_t* to an control_connection_t*; assert if the cast is
+ * invalid. */
+control_connection_t *
+TO_CONTROL_CONN(connection_t *c)
+{
+  tor_assert(c->magic == CONTROL_CONNECTION_MAGIC);
+  return DOWNCAST(control_connection_t, c);
+}
 
 /** Given a control event code for a message event, return the corresponding
  * log severity. */
@@ -251,6 +312,8 @@ clear_circ_bw_fields(void)
       continue;
     ocirc = TO_ORIGIN_CIRCUIT(circ);
     ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
+    ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
+    ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
   }
   SMARTLIST_FOREACH_END(circ);
 }
@@ -263,6 +326,7 @@ control_update_global_event_mask(void)
   smartlist_t *conns = get_connection_array();
   event_mask_t old_mask, new_mask;
   old_mask = global_event_mask;
+  int any_old_per_sec_events = control_any_per_second_event_enabled();
 
   global_event_mask = 0;
   SMARTLIST_FOREACH(conns, connection_t *, _conn,
@@ -280,10 +344,13 @@ control_update_global_event_mask(void)
    * we want to hear...*/
   control_adjust_event_log_severity();
 
+  /* Macro: true if ev was false before and is true now. */
+#define NEWLY_ENABLED(ev) \
+  (! (old_mask & (ev)) && (new_mask & (ev)))
+
   /* ...then, if we've started logging stream or circ bw, clear the
    * appropriate fields. */
-  if (! (old_mask & EVENT_STREAM_BANDWIDTH_USED) &&
-      (new_mask & EVENT_STREAM_BANDWIDTH_USED)) {
+  if (NEWLY_ENABLED(EVENT_STREAM_BANDWIDTH_USED)) {
     SMARTLIST_FOREACH(conns, connection_t *, conn,
     {
       if (conn->type == CONN_TYPE_AP) {
@@ -292,10 +359,18 @@ control_update_global_event_mask(void)
       }
     });
   }
-  if (! (old_mask & EVENT_CIRC_BANDWIDTH_USED) &&
-      (new_mask & EVENT_CIRC_BANDWIDTH_USED)) {
+  if (NEWLY_ENABLED(EVENT_CIRC_BANDWIDTH_USED)) {
     clear_circ_bw_fields();
   }
+  if (NEWLY_ENABLED(EVENT_BANDWIDTH_USED)) {
+    uint64_t r, w;
+    control_get_bytes_rw_last_sec(&r, &w);
+  }
+  if (any_old_per_sec_events != control_any_per_second_event_enabled()) {
+    reschedule_per_second_timer();
+  }
+
+#undef NEWLY_ENABLED
 }
 
 /** Adjust the log severities that result in control_event_logmsg being called
@@ -344,6 +419,65 @@ control_event_is_interesting(int event)
   return EVENT_IS_INTERESTING(event);
 }
 
+/** Return true if any event that needs to fire once a second is enabled. */
+int
+control_any_per_second_event_enabled(void)
+{
+  return ANY_EVENT_IS_INTERESTING(
+      EVENT_MASK_(EVENT_BANDWIDTH_USED) |
+      EVENT_MASK_(EVENT_CELL_STATS) |
+      EVENT_MASK_(EVENT_CIRC_BANDWIDTH_USED) |
+      EVENT_MASK_(EVENT_CONN_BW) |
+      EVENT_MASK_(EVENT_STREAM_BANDWIDTH_USED)
+  );
+}
+
+/* The value of 'get_bytes_read()' the previous time that
+ * control_get_bytes_rw_last_sec() as called. */
+static uint64_t stats_prev_n_read = 0;
+/* The value of 'get_bytes_written()' the previous time that
+ * control_get_bytes_rw_last_sec() as called. */
+static uint64_t stats_prev_n_written = 0;
+
+/**
+ * Set <b>n_read</b> and <b>n_written</b> to the total number of bytes read
+ * and written by Tor since the last call to this function.
+ *
+ * Call this only from the main thread.
+ */
+static void
+control_get_bytes_rw_last_sec(uint64_t *n_read,
+                              uint64_t *n_written)
+{
+  const uint64_t stats_n_bytes_read = get_bytes_read();
+  const uint64_t stats_n_bytes_written = get_bytes_written();
+
+  *n_read = stats_n_bytes_read - stats_prev_n_read;
+  *n_written = stats_n_bytes_written - stats_prev_n_written;
+  stats_prev_n_read = stats_n_bytes_read;
+  stats_prev_n_written = stats_n_bytes_written;
+}
+
+/**
+ * Run all the controller events (if any) that are scheduled to trigger once
+ * per second.
+ */
+void
+control_per_second_events(void)
+{
+  if (!control_any_per_second_event_enabled())
+    return;
+
+  uint64_t bytes_read, bytes_written;
+  control_get_bytes_rw_last_sec(&bytes_read, &bytes_written);
+  control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written);
+
+  control_event_stream_bandwidth_used();
+  control_event_conn_bandwidth_used();
+  control_event_circ_bandwidth_used();
+  control_event_circuit_cell_stats();
+}
+
 /** Append a NUL-terminated string <b>s</b> to the end of
  * <b>conn</b>-\>outbuf.
  */
@@ -351,7 +485,7 @@ static inline void
 connection_write_str_to_buf(const char *s, control_connection_t *conn)
 {
   size_t len = strlen(s);
-  connection_write_to_buf(s, len, TO_CONN(conn));
+  connection_buf_add(s, len, TO_CONN(conn));
 }
 
 /** Given a <b>len</b>-character string in <b>data</b>, made of lines
@@ -364,16 +498,23 @@ connection_write_str_to_buf(const char *s, control_connection_t *conn)
 STATIC size_t
 write_escaped_data(const char *data, size_t len, char **out)
 {
-  size_t sz_out = len+8;
+  tor_assert(len < SIZE_MAX - 9);
+  size_t sz_out = len+8+1;
   char *outp;
   const char *start = data, *end;
-  int i;
+  size_t i;
   int start_of_line;
-  for (i=0; i<(int)len; ++i) {
-    if (data[i]== '\n')
+  for (i=0; i < len; ++i) {
+    if (data[i] == '\n') {
       sz_out += 2; /* Maybe add a CR; maybe add a dot. */
+      if (sz_out >= SIZE_T_CEILING) {
+        log_warn(LD_BUG, "Input to write_escaped_data was too long");
+        *out = tor_strdup(".\r\n");
+        return 3;
+      }
+    }
   }
-  *out = outp = tor_malloc(sz_out+1);
+  *out = outp = tor_malloc(sz_out);
   end = data+len;
   start_of_line = 1;
   while (data < end) {
@@ -399,7 +540,8 @@ write_escaped_data(const char *data, size_t len, char **out)
   *outp++ = '\r';
   *outp++ = '\n';
   *outp = '\0'; /* NUL-terminate just in case. */
-  tor_assert((outp - *out) <= (int)sz_out);
+  tor_assert(outp >= *out);
+  tor_assert((size_t)(outp - *out) <= sz_out);
   return outp - *out;
 }
 
@@ -535,6 +677,49 @@ decode_escaped_string(const char *start, size_t in_len_max,
   return end+1;
 }
 
+/** Create and add a new controller connection on <b>sock</b>.  If
+ * <b>CC_LOCAL_FD_IS_OWNER</b> is set in <b>flags</b>, this Tor process should
+ * exit when the connection closes.  If <b>CC_LOCAL_FD_IS_AUTHENTICATED</b>
+ * is set, then the connection does not need to authenticate.
+ */
+int
+control_connection_add_local_fd(tor_socket_t sock, unsigned flags)
+{
+  if (BUG(! SOCKET_OK(sock)))
+    return -1;
+  const int is_owner = !!(flags & CC_LOCAL_FD_IS_OWNER);
+  const int is_authenticated = !!(flags & CC_LOCAL_FD_IS_AUTHENTICATED);
+  control_connection_t *control_conn = control_connection_new(AF_UNSPEC);
+  connection_t *conn = TO_CONN(control_conn);
+  conn->s = sock;
+  tor_addr_make_unspec(&conn->addr);
+  conn->port = 1;
+  conn->address = tor_strdup("<local socket>");
+
+  /* We take ownership of this socket so that later, when we close it,
+   * we don't freak out. */
+  tor_take_socket_ownership(sock);
+
+  if (set_socket_nonblocking(sock) < 0 ||
+      connection_add(conn) < 0) {
+    connection_free(conn);
+    return -1;
+  }
+
+  control_conn->is_owning_control_connection = is_owner;
+
+  if (connection_init_accepted_conn(conn, NULL) < 0) {
+    connection_mark_for_close(conn);
+    return -1;
+  }
+
+  if (is_authenticated) {
+    conn->state = CONTROL_CONN_STATE_OPEN;
+  }
+
+  return 0;
+}
+
 /** Acts like sprintf, but writes its formatted string to the end of
  * <b>conn</b>-\>outbuf. */
 static void
@@ -553,7 +738,7 @@ connection_printf_to_buf(control_connection_t *conn, const char *format, ...)
     tor_assert(0);
   }
 
-  connection_write_to_buf(buf, (size_t)len, TO_CONN(conn));
+  connection_buf_add(buf, (size_t)len, TO_CONN(conn));
 
   tor_free(buf);
 }
@@ -579,7 +764,7 @@ control_ports_write_to_file(void)
       smartlist_add_asprintf(lines, "UNIX_PORT=%s\n", conn->address);
       continue;
     }
-#endif
+#endif /* defined(AF_UNIX) */
     smartlist_add_asprintf(lines, "PORT=%s:%d\n", conn->address, conn->port);
   } SMARTLIST_FOREACH_END(conn);
 
@@ -596,7 +781,7 @@ control_ports_write_to_file(void)
                options->ControlPortWriteToFile);
     }
   }
-#endif
+#endif /* !defined(_WIN32) */
   tor_free(joined);
   SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
   smartlist_free(lines);
@@ -632,7 +817,7 @@ static tor_mutex_t *queued_control_events_lock = NULL;
 
 /** An event that should fire in order to flush the contents of
  * queued_control_events. */
-static struct event *flush_queued_events_event = NULL;
+static mainloop_event_t *flush_queued_events_event = NULL;
 
 void
 control_initialize_event_queue(void)
@@ -644,9 +829,8 @@ control_initialize_event_queue(void)
   if (flush_queued_events_event == NULL) {
     struct event_base *b = tor_libevent_get_base();
     if (b) {
-      flush_queued_events_event = tor_event_new(b,
-                                              -1, 0, flush_queued_events_cb,
-                                              NULL);
+      flush_queued_events_event =
+        mainloop_event_new(flush_queued_events_cb, NULL);
       tor_assert(flush_queued_events_event);
     }
   }
@@ -722,13 +906,16 @@ queue_control_event_string,(uint16_t event, char *msg))
    */
   if (activate_event) {
     tor_assert(flush_queued_events_event);
-    event_active(flush_queued_events_event, EV_READ, 1);
+    mainloop_event_activate(flush_queued_events_event);
   }
 }
 
+#define queued_event_free(ev) \
+  FREE_AND_NULL(queued_event_t, queued_event_free_, (ev))
+
 /** Release all storage held by <b>ev</b>. */
 static void
-queued_event_free(queued_event_t *ev)
+queued_event_free_(queued_event_t *ev)
 {
   if (ev == NULL)
     return;
@@ -744,6 +931,9 @@ queued_event_free(queued_event_t *ev)
 static void
 queued_events_flush_all(int force)
 {
+  /* Make sure that we get all the pending log events, if there are any. */
+  flush_pending_log_callbacks();
+
   if (PREDICT_UNLIKELY(queued_control_events == NULL)) {
     return;
   }
@@ -778,7 +968,7 @@ queued_events_flush_all(int force)
     SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *,
                             control_conn) {
       if (control_conn->event_mask & bit) {
-        connection_write_to_buf(ev->msg, msg_len, TO_CONN(control_conn));
+        connection_buf_add(ev->msg, msg_len, TO_CONN(control_conn));
       }
     } SMARTLIST_FOREACH_END(control_conn);
 
@@ -799,12 +989,11 @@ queued_events_flush_all(int force)
 }
 
 /** Libevent callback: Flushes pending events to controllers that are
- * interested in them */
+ * interested in them. */
 static void
-flush_queued_events_cb(evutil_socket_t fd, short what, void *arg)
+flush_queued_events_cb(mainloop_event_t *event, void *arg)
 {
-  (void) fd;
-  (void) what;
+  (void) event;
   (void) arg;
   queued_events_flush_all(0);
 }
@@ -942,7 +1131,7 @@ control_setconf_helper(control_connection_t *conn, uint32_t len, char *body,
       ++body;
   }
 
-  smartlist_add(entries, tor_strdup(""));
+  smartlist_add_strdup(entries, "");
   config = smartlist_join_strings(entries, "\n", 0, NULL);
   SMARTLIST_FOREACH(entries, char *, cp, tor_free(cp));
   smartlist_free(entries);
@@ -1060,7 +1249,7 @@ handle_control_getconf(control_connection_t *conn, uint32_t body_len,
     tor_assert(strlen(tmp)>4);
     tmp[3] = ' ';
     msg = smartlist_join_strings(answers, "", 0, &msg_len);
-    connection_write_to_buf(msg, msg_len, TO_CONN(conn));
+    connection_buf_add(msg, msg_len, TO_CONN(conn));
   } else {
     connection_write_str_to_buf("250 OK\r\n", conn);
   }
@@ -1142,7 +1331,6 @@ static const struct control_event_t control_event_table[] = {
   { EVENT_ERR_MSG, "ERR" },
   { EVENT_NEW_DESC, "NEWDESC" },
   { EVENT_ADDRMAP, "ADDRMAP" },
-  { EVENT_AUTHDIR_NEWDESCS, "AUTHDIR_NEWDESCS" },
   { EVENT_DESCCHANGED, "DESCCHANGED" },
   { EVENT_NS, "NS" },
   { EVENT_STATUS_GENERAL, "STATUS_GENERAL" },
@@ -1157,7 +1345,6 @@ static const struct control_event_t control_event_table[] = {
   { EVENT_CONF_CHANGED, "CONF_CHANGED"},
   { EVENT_CONN_BW, "CONN_BW" },
   { EVENT_CELL_STATS, "CELL_STATS" },
-  { EVENT_TB_EMPTY, "TB_EMPTY" },
   { EVENT_CIRC_BANDWIDTH_USED, "CIRC_BW" },
   { EVENT_TRANSPORT_LAUNCHED, "TRANSPORT_LAUNCHED" },
   { EVENT_HS_DESC, "HS_DESC" },
@@ -1182,7 +1369,10 @@ handle_control_setevents(control_connection_t *conn, uint32_t len,
                          SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
   SMARTLIST_FOREACH_BEGIN(events, const char *, ev)
     {
-      if (!strcasecmp(ev, "EXTENDED")) {
+      if (!strcasecmp(ev, "EXTENDED") ||
+          !strcasecmp(ev, "AUTHDIR_NEWDESCS")) {
+        log_warn(LD_CONTROL, "The \"%s\" SETEVENTS argument is no longer "
+                 "supported.", ev);
         continue;
       } else {
         int i;
@@ -1459,8 +1649,10 @@ handle_control_saveconf(control_connection_t *conn, uint32_t len,
                         const char *body)
 {
   (void) len;
-  (void) body;
-  if (options_save_current()<0) {
+
+  int force = !strcmpstart(body, "FORCE");
+  const or_options_t *options = get_options();
+  if ((!force && options->IncludeUsed) || options_save_current() < 0) {
     connection_write_str_to_buf(
       "551 Unable to write configuration to disk.\r\n", conn);
   } else {
@@ -1640,12 +1832,12 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len,
   if (smartlist_len(reply)) {
     ((char*)smartlist_get(reply,smartlist_len(reply)-1))[3] = ' ';
     r = smartlist_join_strings(reply, "\r\n", 1, &sz);
-    connection_write_to_buf(r, sz, TO_CONN(conn));
+    connection_buf_add(r, sz, TO_CONN(conn));
     tor_free(r);
   } else {
     const char *response =
       "512 syntax error: not enough arguments to mapaddress.\r\n";
-    connection_write_to_buf(response, strlen(response), TO_CONN(conn));
+    connection_buf_add(response, strlen(response), TO_CONN(conn));
   }
 
   SMARTLIST_FOREACH(reply, char *, cp, tor_free(cp));
@@ -1674,6 +1866,8 @@ getinfo_helper_misc(control_connection_t *conn, const char *question,
       *answer = tor_strdup(a);
   } else if (!strcmp(question, "config-text")) {
     *answer = options_dump(get_options(), OPTIONS_DUMP_MINIMAL);
+  } else if (!strcmp(question, "config-can-saveconf")) {
+    *answer = tor_strdup(get_options()->IncludeUsed ? "0" : "1");
   } else if (!strcmp(question, "info/names")) {
     *answer = list_getinfo_options();
   } else if (!strcmp(question, "dormant")) {
@@ -1710,30 +1904,33 @@ getinfo_helper_misc(control_connection_t *conn, const char *question,
     }
     *answer = tor_dup_ip(addr);
   } else if (!strcmp(question, "traffic/read")) {
-    tor_asprintf(answer, U64_FORMAT, U64_PRINTF_ARG(get_bytes_read()));
+    tor_asprintf(answer, "%"PRIu64, (get_bytes_read()));
   } else if (!strcmp(question, "traffic/written")) {
-    tor_asprintf(answer, U64_FORMAT, U64_PRINTF_ARG(get_bytes_written()));
+    tor_asprintf(answer, "%"PRIu64, (get_bytes_written()));
+  } else if (!strcmp(question, "uptime")) {
+    long uptime_secs = get_uptime();
+    tor_asprintf(answer, "%ld", uptime_secs);
   } else if (!strcmp(question, "process/pid")) {
     int myPid = -1;
 
-    #ifdef _WIN32
+#ifdef _WIN32
       myPid = _getpid();
-    #else
+#else
       myPid = getpid();
-    #endif
+#endif
 
     tor_asprintf(answer, "%d", myPid);
   } else if (!strcmp(question, "process/uid")) {
-    #ifdef _WIN32
+#ifdef _WIN32
       *answer = tor_strdup("-1");
-    #else
+#else
       int myUid = geteuid();
       tor_asprintf(answer, "%d", myUid);
-    #endif
+#endif /* defined(_WIN32) */
   } else if (!strcmp(question, "process/user")) {
-    #ifdef _WIN32
+#ifdef _WIN32
       *answer = tor_strdup("");
-    #else
+#else
       int myUid = geteuid();
       const struct passwd *myPwEntry = tor_getpwuid(myUid);
 
@@ -1742,13 +1939,13 @@ getinfo_helper_misc(control_connection_t *conn, const char *question,
       } else {
         *answer = tor_strdup("");
       }
-    #endif
+#endif /* defined(_WIN32) */
   } else if (!strcmp(question, "process/descriptor-limit")) {
     int max_fds = get_max_sockets();
     tor_asprintf(answer, "%d", max_fds);
   } else if (!strcmp(question, "limits/max-mem-in-queues")) {
-    tor_asprintf(answer, U64_FORMAT,
-                 U64_PRINTF_ARG(get_options()->MaxMemInQueues));
+    tor_asprintf(answer, "%"PRIu64,
+                 (get_options()->MaxMemInQueues));
   } else if (!strcmp(question, "fingerprint")) {
     crypto_pk_t *server_key;
     if (!server_mode(get_options())) {
@@ -1828,6 +2025,8 @@ getinfo_helper_listeners(control_connection_t *control_conn,
 
   if (!strcmp(question, "net/listeners/or"))
     type = CONN_TYPE_OR_LISTENER;
+  else if (!strcmp(question, "net/listeners/extor"))
+    type = CONN_TYPE_EXT_OR_LISTENER;
   else if (!strcmp(question, "net/listeners/dir"))
     type = CONN_TYPE_DIR_LISTENER;
   else if (!strcmp(question, "net/listeners/socks"))
@@ -1836,6 +2035,8 @@ getinfo_helper_listeners(control_connection_t *control_conn,
     type = CONN_TYPE_AP_TRANS_LISTENER;
   else if (!strcmp(question, "net/listeners/natd"))
     type = CONN_TYPE_AP_NATD_LISTENER;
+  else if (!strcmp(question, "net/listeners/httptunnel"))
+    type = CONN_TYPE_AP_HTTP_CONNECT_LISTENER;
   else if (!strcmp(question, "net/listeners/dns"))
     type = CONN_TYPE_AP_DNS_LISTENER;
   else if (!strcmp(question, "net/listeners/control"))
@@ -1868,9 +2069,34 @@ getinfo_helper_listeners(control_connection_t *control_conn,
   return 0;
 }
 
+/** Implementation helper for GETINFO: answers requests for information about
+ * the current time in both local and UTC forms. */
+STATIC int
+getinfo_helper_current_time(control_connection_t *control_conn,
+                         const char *question,
+                         char **answer, const char **errmsg)
+{
+  (void)control_conn;
+  (void)errmsg;
+
+  struct timeval now;
+  tor_gettimeofday(&now);
+  char timebuf[ISO_TIME_LEN+1];
+
+  if (!strcmp(question, "current-time/local"))
+    format_local_iso_time_nospace(timebuf, (time_t)now.tv_sec);
+  else if (!strcmp(question, "current-time/utc"))
+    format_iso_time_nospace(timebuf, (time_t)now.tv_sec);
+  else
+    return 0;
+
+  *answer = tor_strdup(timebuf);
+  return 0;
+}
+
 /** Implementation helper for GETINFO: knows the answers for questions about
  * directory information. */
-static int
+STATIC int
 getinfo_helper_dir(control_connection_t *control_conn,
                    const char *question, char **answer,
                    const char **errmsg)
@@ -1878,27 +2104,42 @@ getinfo_helper_dir(control_connection_t *control_conn,
   (void) control_conn;
   if (!strcmpstart(question, "desc/id/")) {
     const routerinfo_t *ri = NULL;
-    const node_t *node = node_get_by_hex_id(question+strlen("desc/id/"));
+    const node_t *node = node_get_by_hex_id(question+strlen("desc/id/"), 0);
     if (node)
       ri = node->ri;
     if (ri) {
       const char *body = signed_descriptor_get_body(&ri->cache_info);
       if (body)
         *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
+    } else if (! we_fetch_router_descriptors(get_options())) {
+      /* Descriptors won't be available, provide proper error */
+      *errmsg = "We fetch microdescriptors, not router "
+                "descriptors. You'll need to use md/id/* "
+                "instead of desc/id/*.";
+      return 0;
     }
   } else if (!strcmpstart(question, "desc/name/")) {
     const routerinfo_t *ri = NULL;
     /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
      * warning goes to the user, not to the controller. */
     const node_t *node =
-      node_get_by_nickname(question+strlen("desc/name/"), 1);
+      node_get_by_nickname(question+strlen("desc/name/"), 0);
     if (node)
       ri = node->ri;
     if (ri) {
       const char *body = signed_descriptor_get_body(&ri->cache_info);
       if (body)
         *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
+    } else if (! we_fetch_router_descriptors(get_options())) {
+      /* Descriptors won't be available, provide proper error */
+      *errmsg = "We fetch microdescriptors, not router "
+                "descriptors. You'll need to use md/name/* "
+                "instead of desc/name/*.";
+      return 0;
     }
+  } else if (!strcmp(question, "desc/download-enabled")) {
+    int r = we_fetch_router_descriptors(get_options());
+    tor_asprintf(answer, "%d", !!r);
   } else if (!strcmp(question, "desc/all-recent")) {
     routerlist_t *routerlist = router_get_routerlist();
     smartlist_t *sl = smartlist_new();
@@ -1936,39 +2177,113 @@ getinfo_helper_dir(control_connection_t *control_conn,
     SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
     smartlist_free(sl);
   } else if (!strcmpstart(question, "hs/client/desc/id/")) {
-    rend_cache_entry_t *e = NULL;
+    hostname_type_t addr_type;
 
     question += strlen("hs/client/desc/id/");
-    if (strlen(question) != REND_SERVICE_ID_LEN_BASE32) {
+    if (rend_valid_v2_service_id(question)) {
+      addr_type = ONION_V2_HOSTNAME;
+    } else if (hs_address_is_valid(question)) {
+      addr_type = ONION_V3_HOSTNAME;
+    } else {
       *errmsg = "Invalid address";
       return -1;
     }
 
-    if (!rend_cache_lookup_entry(question, -1, &e)) {
-      /* Descriptor found in cache */
-      *answer = tor_strdup(e->desc);
+    if (addr_type == ONION_V2_HOSTNAME) {
+      rend_cache_entry_t *e = NULL;
+      if (!rend_cache_lookup_entry(question, -1, &e)) {
+        /* Descriptor found in cache */
+        *answer = tor_strdup(e->desc);
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
     } else {
-      *errmsg = "Not found in cache";
-      return -1;
+      ed25519_public_key_t service_pk;
+      const char *desc;
+
+      /* The check before this if/else makes sure of this. */
+      tor_assert(addr_type == ONION_V3_HOSTNAME);
+
+      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
+        *errmsg = "Invalid v3 address";
+        return -1;
+      }
+
+      desc = hs_cache_lookup_encoded_as_client(&service_pk);
+      if (desc) {
+        *answer = tor_strdup(desc);
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
     }
   } else if (!strcmpstart(question, "hs/service/desc/id/")) {
-    rend_cache_entry_t *e = NULL;
+    hostname_type_t addr_type;
 
     question += strlen("hs/service/desc/id/");
-    if (strlen(question) != REND_SERVICE_ID_LEN_BASE32) {
+    if (rend_valid_v2_service_id(question)) {
+      addr_type = ONION_V2_HOSTNAME;
+    } else if (hs_address_is_valid(question)) {
+      addr_type = ONION_V3_HOSTNAME;
+    } else {
       *errmsg = "Invalid address";
       return -1;
     }
+    rend_cache_entry_t *e = NULL;
 
-    if (!rend_cache_lookup_v2_desc_as_service(question, &e)) {
-      /* Descriptor found in cache */
-      *answer = tor_strdup(e->desc);
+    if (addr_type == ONION_V2_HOSTNAME) {
+      if (!rend_cache_lookup_v2_desc_as_service(question, &e)) {
+        /* Descriptor found in cache */
+        *answer = tor_strdup(e->desc);
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
     } else {
-      *errmsg = "Not found in cache";
-      return -1;
+      ed25519_public_key_t service_pk;
+      char *desc;
+
+      /* The check before this if/else makes sure of this. */
+      tor_assert(addr_type == ONION_V3_HOSTNAME);
+
+      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
+        *errmsg = "Invalid v3 address";
+        return -1;
+      }
+
+      desc = hs_service_lookup_current_desc(&service_pk);
+      if (desc) {
+        /* Newly allocated string, we have ownership. */
+        *answer = desc;
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
     }
+  } else if (!strcmp(question, "md/all")) {
+    const smartlist_t *nodes = nodelist_get_list();
+    tor_assert(nodes);
+
+    if (smartlist_len(nodes) == 0) {
+      *answer = tor_strdup("");
+      return 0;
+    }
+
+    smartlist_t *microdescs = smartlist_new();
+
+    SMARTLIST_FOREACH_BEGIN(nodes, node_t *, n) {
+      if (n->md && n->md->body) {
+        char *copy = tor_strndup(n->md->body, n->md->bodylen);
+        smartlist_add(microdescs, copy);
+      }
+    } SMARTLIST_FOREACH_END(n);
+
+    *answer = smartlist_join_strings(microdescs, "", 0, NULL);
+    SMARTLIST_FOREACH(microdescs, char *, md, tor_free(md));
+    smartlist_free(microdescs);
   } else if (!strcmpstart(question, "md/id/")) {
-    const node_t *node = node_get_by_hex_id(question+strlen("md/id/"));
+    const node_t *node = node_get_by_hex_id(question+strlen("md/id/"), 0);
     const microdesc_t *md = NULL;
     if (node) md = node->md;
     if (md && md->body) {
@@ -1977,17 +2292,20 @@ getinfo_helper_dir(control_connection_t *control_conn,
   } else if (!strcmpstart(question, "md/name/")) {
     /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
      * warning goes to the user, not to the controller. */
-    const node_t *node = node_get_by_nickname(question+strlen("md/name/"), 1);
+    const node_t *node = node_get_by_nickname(question+strlen("md/name/"), 0);
     /* XXXX duplicated code */
     const microdesc_t *md = NULL;
     if (node) md = node->md;
     if (md && md->body) {
       *answer = tor_strndup(md->body, md->bodylen);
     }
+  } else if (!strcmp(question, "md/download-enabled")) {
+    int r = we_fetch_microdescriptors(get_options());
+    tor_asprintf(answer, "%d", !!r);
   } else if (!strcmpstart(question, "desc-annotations/id/")) {
     const routerinfo_t *ri = NULL;
     const node_t *node =
-      node_get_by_hex_id(question+strlen("desc-annotations/id/"));
+      node_get_by_hex_id(question+strlen("desc-annotations/id/"), 0);
     if (node)
       ri = node->ri;
     if (ri) {
@@ -2028,15 +2346,13 @@ getinfo_helper_dir(control_connection_t *control_conn,
   } else if (!strcmpstart(question, "dir/status/")) {
     *answer = tor_strdup("");
   } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */
-    if (directory_caches_dir_info(get_options())) {
+    if (we_want_to_fetch_flavor(get_options(), FLAV_NS)) {
       const cached_dir_t *consensus = dirserv_get_consensus("ns");
       if (consensus)
         *answer = tor_strdup(consensus->dir);
     }
     if (!*answer) { /* try loading it from disk */
-      char *filename = get_datadir_fname("cached-consensus");
-      *answer = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
-      tor_free(filename);
+      *answer = networkstatus_read_cached_consensus("ns");
       if (!*answer) { /* generate an error */
         *errmsg = "Could not open cached consensus. "
           "Make sure FetchUselessDescriptors is set to 1.";
@@ -2044,6 +2360,12 @@ getinfo_helper_dir(control_connection_t *control_conn,
       }
     }
   } else if (!strcmp(question, "network-status")) { /* v1 */
+    static int network_status_warned = 0;
+    if (!network_status_warned) {
+      log_warn(LD_CONTROL, "GETINFO network-status is deprecated; it will "
+               "go away in a future version of Tor.");
+      network_status_warned = 1;
+    }
     routerlist_t *routerlist = router_get_routerlist();
     if (!routerlist || !routerlist->routers ||
         list_server_status_v1(routerlist->routers, answer, 1) < 0) {
@@ -2107,14 +2429,14 @@ digest_list_to_string(const smartlist_t *sl)
 static char *
 download_status_to_string(const download_status_t *dl)
 {
-  char *rv = NULL, *tmp;
+  char *rv = NULL;
   char tbuf[ISO_TIME_LEN+1];
   const char *schedule_str, *want_authority_str;
   const char *increment_on_str, *backoff_str;
 
   if (dl) {
     /* Get some substrings of the eventual output ready */
-    format_iso_time(tbuf, dl->next_attempt_at);
+    format_iso_time(tbuf, download_status_get_next_attempt_at(dl));
 
     switch (dl->schedule) {
       case DL_SCHED_GENERIC:
@@ -2155,49 +2477,28 @@ download_status_to_string(const download_status_t *dl)
         break;
     }
 
-    switch (dl->backoff) {
-      case DL_SCHED_DETERMINISTIC:
-        backoff_str = "DL_SCHED_DETERMINISTIC";
-        break;
-      case DL_SCHED_RANDOM_EXPONENTIAL:
-        backoff_str = "DL_SCHED_RANDOM_EXPONENTIAL";
-        break;
-      default:
-        backoff_str = "unknown";
-        break;
-    }
+    backoff_str = "DL_SCHED_RANDOM_EXPONENTIAL";
 
     /* Now assemble them */
-    tor_asprintf(&tmp,
+    tor_asprintf(&rv,
                  "next-attempt-at %s\n"
                  "n-download-failures %u\n"
                  "n-download-attempts %u\n"
                  "schedule %s\n"
                  "want-authority %s\n"
                  "increment-on %s\n"
-                 "backoff %s\n",
+                 "backoff %s\n"
+                 "last-backoff-position %u\n"
+                 "last-delay-used %d\n",
                  tbuf,
                  dl->n_download_failures,
                  dl->n_download_attempts,
                  schedule_str,
                  want_authority_str,
                  increment_on_str,
-                 backoff_str);
-
-    if (dl->backoff == DL_SCHED_RANDOM_EXPONENTIAL) {
-      /* Additional fields become relevant in random-exponential mode */
-      tor_asprintf(&rv,
-                   "%s"
-                   "last-backoff-position %u\n"
-                   "last-delay-used %d\n",
-                   tmp,
-                   dl->last_backoff_position,
-                   dl->last_delay_used);
-      tor_free(tmp);
-    } else {
-      /* That was it */
-      rv = tmp;
-    }
+                 backoff_str,
+                 dl->last_backoff_position,
+                 dl->last_delay_used);
   }
 
   return rv;
@@ -2537,9 +2838,16 @@ circuit_describe_status_for_controller(origin_circuit_t *circ)
     }
   }
 
-  if (circ->rend_data != NULL) {
-    smartlist_add_asprintf(descparts, "REND_QUERY=%s",
-                 circ->rend_data->onion_address);
+  if (circ->rend_data != NULL || circ->hs_ident != NULL) {
+    char addr[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+    const char *onion_address;
+    if (circ->rend_data) {
+      onion_address = rend_data_get_address(circ->rend_data);
+    } else {
+      hs_build_address(&circ->hs_ident->identity_pk, HS_VERSION_THREE, addr);
+      onion_address = addr;
+    }
+    smartlist_add_asprintf(descparts, "REND_QUERY=%s", onion_address);
   }
 
   {
@@ -2594,6 +2902,8 @@ getinfo_helper_events(control_connection_t *control_conn,
 
       if (circ->base_.state == CIRCUIT_STATE_OPEN)
         state = "BUILT";
+      else if (circ->base_.state == CIRCUIT_STATE_GUARD_WAIT)
+        state = "GUARD_WAIT";
       else if (circ->cpath)
         state = "EXTENDED";
       else
@@ -2819,12 +3129,13 @@ getinfo_helper_events(control_connection_t *control_conn,
 
 /** Implementation helper for GETINFO: knows how to enumerate hidden services
  * created via the control port. */
-static int
+STATIC int
 getinfo_helper_onions(control_connection_t *control_conn,
                       const char *question, char **answer,
                       const char **errmsg)
 {
   smartlist_t *onion_list = NULL;
+  (void) errmsg;  /* no errors from this method */
 
   if (control_conn && !strcmp(question, "onions/current")) {
     onion_list = control_conn->ephemeral_onion_services;
@@ -2834,13 +3145,13 @@ getinfo_helper_onions(control_connection_t *control_conn,
     return 0;
   }
   if (!onion_list || smartlist_len(onion_list) == 0) {
-    if (errmsg) {
-        *errmsg = "No onion services of the specified type.";
+    if (answer) {
+      *answer = tor_strdup("");
     }
-    return -1;
-  }
-  if (answer) {
+  } else {
+    if (answer) {
       *answer = smartlist_join_strings(onion_list, "\r\n", 0, NULL);
+    }
   }
 
   return 0;
@@ -2866,12 +3177,33 @@ getinfo_helper_liveness(control_connection_t *control_conn,
   return 0;
 }
 
+/** Implementation helper for GETINFO: answers queries about shared random
+ * value. */
+static int
+getinfo_helper_sr(control_connection_t *control_conn,
+                  const char *question, char **answer,
+                  const char **errmsg)
+{
+  (void) control_conn;
+  (void) errmsg;
+
+  if (!strcmp(question, "sr/current")) {
+    *answer = sr_get_current_for_control();
+  } else if (!strcmp(question, "sr/previous")) {
+    *answer = sr_get_previous_for_control();
+  }
+  /* Else statement here is unrecognized key so do nothing. */
+
+  return 0;
+}
+
 /** Callback function for GETINFO: on a given control connection, try to
  * answer the question <b>q</b> and store the newly-allocated answer in
  * *<b>a</b>. If an internal error occurs, return -1 and optionally set
  * *<b>error_out</b> to point to an error message to be delivered to the
  * controller. On success, _or if the key is not recognized_, return 0. Do not
- * set <b>a</b> if the key is not recognized.
+ * set <b>a</b> if the key is not recognized but you may set <b>error_out</b>
+ * to improve the error message.
  */
 typedef int (*getinfo_helper_t)(control_connection_t *,
                                 const char *q, char **a,
@@ -2899,6 +3231,8 @@ static const getinfo_item_t getinfo_items[] = {
   ITEM("config-defaults-file", misc, "Current location of the defaults file."),
   ITEM("config-text", misc,
        "Return the string that would be written by a saveconf command."),
+  ITEM("config-can-saveconf", misc,
+       "Is it possible to save the configuration to the \"torrc\" file?"),
   ITEM("accounting/bytes", accounting,
        "Number of bytes read/written so far in the accounting interval."),
   ITEM("accounting/bytes-left", accounting,
@@ -2921,6 +3255,9 @@ static const getinfo_item_t getinfo_items[] = {
   DOC("config/defaults",
       "List of default values for configuration options. "
       "See also config/names"),
+  PREFIX("current-time/", current_time, "Current time."),
+  DOC("current-time/local", "Current time on the local system."),
+  DOC("current-time/utc", "Current UTC time."),
   PREFIX("downloads/networkstatus/", downloads,
          "Download statuses for networkstatus objects"),
   DOC("downloads/networkstatus/ns",
@@ -2974,9 +3311,14 @@ static const getinfo_item_t getinfo_items[] = {
   PREFIX("desc/name/", dir, "Router descriptors by nickname."),
   ITEM("desc/all-recent", dir,
        "All non-expired, non-superseded router descriptors."),
+  ITEM("desc/download-enabled", dir,
+       "Do we try to download router descriptors?"),
   ITEM("desc/all-recent-extrainfo-hack", dir, NULL), /* Hack. */
+  ITEM("md/all", dir, "All known microdescriptors."),
   PREFIX("md/id/", dir, "Microdescriptors by ID"),
   PREFIX("md/name/", dir, "Microdescriptors by name"),
+  ITEM("md/download-enabled", dir,
+       "Do we try to download microdescriptors?"),
   PREFIX("extra-info/digest/", dir, "Extra-info documents by digest."),
   PREFIX("hs/client/desc/id", dir,
          "Hidden Service descriptor in client's cache by onion."),
@@ -3030,6 +3372,7 @@ static const getinfo_item_t getinfo_items[] = {
   ITEM("traffic/read", misc,"Bytes read since the process was started."),
   ITEM("traffic/written", misc,
        "Bytes written since the process was started."),
+  ITEM("uptime", misc, "Uptime of the Tor daemon in seconds."),
   ITEM("process/pid", misc, "Process id belonging to the main tor process."),
   ITEM("process/uid", misc, "User id running the tor process."),
   ITEM("process/user", misc,
@@ -3058,6 +3401,8 @@ static const getinfo_item_t getinfo_items[] = {
        "Onion services owned by the current control connection."),
   ITEM("onions/detached", onions,
        "Onion services detached from the control connection."),
+  ITEM("sr/current", sr, "Get current shared random value."),
+  ITEM("sr/previous", sr, "Get previous shared random value."),
   { NULL, NULL, NULL, 0 }
 };
 
@@ -3122,7 +3467,7 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len,
   smartlist_t *questions = smartlist_new();
   smartlist_t *answers = smartlist_new();
   smartlist_t *unrecognized = smartlist_new();
-  char *msg = NULL, *ans = NULL;
+  char *ans = NULL;
   int i;
   (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */
 
@@ -3130,6 +3475,7 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len,
                          SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
   SMARTLIST_FOREACH_BEGIN(questions, const char *, q) {
     const char *errmsg = NULL;
+
     if (handle_getinfo_helper(conn, q, &ans, &errmsg) < 0) {
       if (!errmsg)
         errmsg = "Internal error";
@@ -3137,20 +3483,26 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len,
       goto done;
     }
     if (!ans) {
-      smartlist_add(unrecognized, (char*)q);
+      if (errmsg) /* use provided error message */
+        smartlist_add_strdup(unrecognized, errmsg);
+      else /* use default error message */
+        smartlist_add_asprintf(unrecognized, "Unrecognized key \"%s\"", q);
     } else {
-      smartlist_add(answers, tor_strdup(q));
+      smartlist_add_strdup(answers, q);
       smartlist_add(answers, ans);
     }
   } SMARTLIST_FOREACH_END(q);
+
   if (smartlist_len(unrecognized)) {
+    /* control-spec section 2.3, mid-reply '-' or end of reply ' ' */
     for (i=0; i < smartlist_len(unrecognized)-1; ++i)
       connection_printf_to_buf(conn,
-                               "552-Unrecognized key \"%s\"\r\n",
-                               (char*)smartlist_get(unrecognized, i));
+                               "552-%s\r\n",
+                               (char *)smartlist_get(unrecognized, i));
+
     connection_printf_to_buf(conn,
-                             "552 Unrecognized key \"%s\"\r\n",
-                             (char*)smartlist_get(unrecognized, i));
+                             "552 %s\r\n",
+                             (char *)smartlist_get(unrecognized, i));
     goto done;
   }
 
@@ -3166,7 +3518,7 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len,
       size_t esc_len;
       esc_len = write_escaped_data(v, strlen(v), &esc);
       connection_printf_to_buf(conn, "250+%s=\r\n", k);
-      connection_write_to_buf(esc, esc_len, TO_CONN(conn));
+      connection_buf_add(esc, esc_len, TO_CONN(conn));
       tor_free(esc);
     }
   }
@@ -3177,8 +3529,8 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len,
   smartlist_free(answers);
   SMARTLIST_FOREACH(questions, char *, cp, tor_free(cp));
   smartlist_free(questions);
+  SMARTLIST_FOREACH(unrecognized, char *, cp, tor_free(cp));
   smartlist_free(unrecognized);
-  tor_free(msg);
 
   return 0;
 }
@@ -3323,17 +3675,19 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len,
   smartlist_free(args);
 
   nodes = smartlist_new();
+  int first_node = zero_circ;
   SMARTLIST_FOREACH_BEGIN(router_nicknames, const char *, n) {
-    const node_t *node = node_get_by_nickname(n, 1);
+    const node_t *node = node_get_by_nickname(n, 0);
     if (!node) {
       connection_printf_to_buf(conn, "552 No such router \"%s\"\r\n", n);
       goto done;
     }
-    if (!node_has_descriptor(node)) {
+    if (!node_has_preferred_descriptor(node, first_node)) {
       connection_printf_to_buf(conn, "552 No descriptor for \"%s\"\r\n", n);
       goto done;
     }
     smartlist_add(nodes, (void*)node);
+    first_node = 0;
   } SMARTLIST_FOREACH_END(n);
   if (!smartlist_len(nodes)) {
     connection_write_str_to_buf("512 No router names provided\r\n", conn);
@@ -3346,22 +3700,20 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len,
   }
 
   /* now circ refers to something that is ready to be extended */
-  int first_node = zero_circ;
+  first_node = zero_circ;
   SMARTLIST_FOREACH(nodes, const node_t *, node,
   {
     extend_info_t *info = extend_info_from_node(node, first_node);
-    if (first_node && !info) {
+    if (!info) {
+      tor_assert_nonfatal(first_node);
       log_warn(LD_CONTROL,
-               "controller tried to connect to a node that doesn't have any "
+               "controller tried to connect to a node that lacks a suitable "
+               "descriptor, or which doesn't have any "
                "addresses that are allowed by the firewall configuration; "
                "circuit marked for closing.");
       circuit_mark_for_close(TO_CIRCUIT(circ), -END_CIRC_REASON_CONNECTFAILED);
       connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn);
       goto done;
-    } else {
-      /* True, since node_has_descriptor(node) == true and we are extending
-       * to the node's primary address */
-      tor_assert(info);
     }
     circuit_append_new_exit(circ, info);
     if (circ->build_state->desired_path_len > 1) {
@@ -3380,7 +3732,8 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len,
       goto done;
     }
   } else {
-    if (circ->base_.state == CIRCUIT_STATE_OPEN) {
+    if (circ->base_.state == CIRCUIT_STATE_OPEN ||
+        circ->base_.state == CIRCUIT_STATE_GUARD_WAIT) {
       int err_reason = 0;
       circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_BUILDING);
       if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) {
@@ -3522,24 +3875,9 @@ handle_control_attachstream(control_connection_t *conn, uint32_t len,
   }
   /* Is this a single hop circuit? */
   if (circ && (circuit_get_cpath_len(circ)<2 || hop==1)) {
-    const node_t *node = NULL;
-    char *exit_digest = NULL;
-    if (circ->build_state &&
-        circ->build_state->chosen_exit &&
-        !tor_digest_is_zero(circ->build_state->chosen_exit->identity_digest)) {
-      exit_digest = circ->build_state->chosen_exit->identity_digest;
-      node = node_get_by_id(exit_digest);
-    }
-    /* Do both the client and relay allow one-hop exit circuits? */
-    if (!node ||
-        !node_allows_single_hop_exits(node) ||
-        !get_options()->AllowSingleHopCircuits) {
-      connection_write_str_to_buf(
-      "551 Can't attach stream to this one-hop circuit.\r\n", conn);
-      return 0;
-    }
-    tor_assert(exit_digest);
-    ap_conn->chosen_exit_name = tor_strdup(hex_str(exit_digest, DIGEST_LEN));
+    connection_write_str_to_buf(
+               "551 Can't attach stream to this one-hop circuit.\r\n", conn);
+    return 0;
   }
 
   if (circ && hop>0) {
@@ -4042,6 +4380,14 @@ handle_control_dropguards(control_connection_t *conn,
   smartlist_split_string(args, body, " ",
                          SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
 
+  static int have_warned = 0;
+  if (! have_warned) {
+    log_warn(LD_CONTROL, "DROPGUARDS is dangerous; make sure you understand "
+             "the risks before using it. It may be removed in a future "
+             "version of Tor.");
+    have_warned = 1;
+  }
+
   if (smartlist_len(args)) {
     connection_printf_to_buf(conn, "512 Too many arguments to DROPGUARDS\r\n");
   } else {
@@ -4077,7 +4423,7 @@ handle_control_hsfetch(control_connection_t *conn, uint32_t len,
   /* Extract the first argument (either HSAddress or DescID). */
   arg1 = smartlist_get(args, 0);
   /* Test if it's an HS address without the .onion part. */
-  if (rend_valid_service_id(arg1)) {
+  if (rend_valid_v2_service_id(arg1)) {
     hsaddress = arg1;
   } else if (strcmpstart(arg1, v2_str) == 0 &&
              rend_valid_descriptor_id(arg1 + v2_str_len) &&
@@ -4087,7 +4433,7 @@ handle_control_hsfetch(control_connection_t *conn, uint32_t len,
      * of the id. */
     desc_id = digest;
   } else {
-    connection_printf_to_buf(conn, "513 Unrecognized \"%s\"\r\n",
+    connection_printf_to_buf(conn, "513 Invalid argument \"%s\"\r\n",
                              arg1);
     goto done;
   }
@@ -4103,7 +4449,7 @@ handle_control_hsfetch(control_connection_t *conn, uint32_t len,
       const char *server;
 
       server = arg + strlen(opt_server);
-      node = node_get_by_hex_id(server);
+      node = node_get_by_hex_id(server, 0);
       if (!node) {
         connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n",
                                  server);
@@ -4164,9 +4510,11 @@ handle_control_hspost(control_connection_t *conn,
                       const char *body)
 {
   static const char *opt_server = "SERVER=";
+  static const char *opt_hsaddress = "HSADDRESS=";
   smartlist_t *hs_dirs = NULL;
   const char *encoded_desc = body;
   size_t encoded_desc_len = len;
+  const char *onion_address = NULL;
 
   char *cp = memchr(body, '\n', len);
   if (cp == NULL) {
@@ -4177,8 +4525,10 @@ handle_control_hspost(control_connection_t *conn,
 
   smartlist_t *args = smartlist_new();
 
-  /* If any SERVER= options were specified, try parse the options line */
-  if (!strcasecmpstart(argline, opt_server)) {
+  /* If any SERVER= or HSADDRESS= options were specified, try to parse
+   * the options line. */
+  if (!strcasecmpstart(argline, opt_server) ||
+      !strcasecmpstart(argline, opt_hsaddress)) {
     /* encoded_desc begins after a newline character */
     cp = cp + 1;
     encoded_desc = cp;
@@ -4189,22 +4539,24 @@ handle_control_hspost(control_connection_t *conn,
     SMARTLIST_FOREACH_BEGIN(args, const char *, arg) {
       if (!strcasecmpstart(arg, opt_server)) {
         const char *server = arg + strlen(opt_server);
-        const node_t *node = node_get_by_hex_id(server);
+        const node_t *node = node_get_by_hex_id(server, 0);
 
         if (!node || !node->rs) {
           connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n",
                                    server);
           goto done;
         }
-        if (!node->rs->is_hs_dir) {
-          connection_printf_to_buf(conn, "552 Server \"%s\" is not a HSDir"
-                                         "\r\n", server);
-          goto done;
-        }
         /* Valid server, add it to our local list. */
         if (!hs_dirs)
           hs_dirs = smartlist_new();
         smartlist_add(hs_dirs, node->rs);
+      } else if (!strcasecmpstart(arg, opt_hsaddress)) {
+        const char *address = arg + strlen(opt_hsaddress);
+        if (!hs_address_is_valid(address)) {
+          connection_printf_to_buf(conn, "512 Malformed onion address\r\n");
+          goto done;
+        }
+        onion_address = address;
       } else {
         connection_printf_to_buf(conn, "512 Unexpected argument \"%s\"\r\n",
                                  arg);
@@ -4213,6 +4565,21 @@ handle_control_hspost(control_connection_t *conn,
     } SMARTLIST_FOREACH_END(arg);
   }
 
+  /* Handle the v3 case. */
+  if (onion_address) {
+    char *desc_str = NULL;
+    read_escaped_data(encoded_desc, encoded_desc_len, &desc_str);
+    if (hs_control_hspost_command(desc_str, onion_address, hs_dirs) < 0) {
+      connection_printf_to_buf(conn, "554 Invalid descriptor\r\n");
+    } else {
+      send_control_done(conn);
+    }
+    tor_free(desc_str);
+    goto done;
+  }
+
+  /* From this point on, it is only v2. */
+
   /* Read the dot encoded descriptor, and parse it. */
   rend_encoded_v2_service_descriptor_t *desc =
       tor_malloc_zero(sizeof(rend_encoded_v2_service_descriptor_t));
@@ -4257,6 +4624,52 @@ handle_control_hspost(control_connection_t *conn,
   return 0;
 }
 
+/* Helper function for ADD_ONION that adds an ephemeral service depending on
+ * the given hs_version.
+ *
+ * The secret key in pk depends on the hs_version. The ownership of the key
+ * used in pk is given to the HS subsystem so the caller must stop accessing
+ * it after.
+ *
+ * The port_cfgs is a list of service port. Ownership transferred to service.
+ * The max_streams refers to the MaxStreams= key.
+ * The max_streams_close_circuit refers to the MaxStreamsCloseCircuit key.
+ * The auth_type is the authentication type of the clients in auth_clients.
+ * The ownership of that list is transferred to the service.
+ *
+ * On success (RSAE_OKAY), the address_out points to a newly allocated string
+ * containing the onion address without the .onion part. On error, address_out
+ * is untouched. */
+static hs_service_add_ephemeral_status_t
+add_onion_helper_add_service(int hs_version,
+                             add_onion_secret_key_t *pk,
+                             smartlist_t *port_cfgs, int max_streams,
+                             int max_streams_close_circuit, int auth_type,
+                             smartlist_t *auth_clients, char **address_out)
+{
+  hs_service_add_ephemeral_status_t ret;
+
+  tor_assert(pk);
+  tor_assert(port_cfgs);
+  tor_assert(address_out);
+
+  switch (hs_version) {
+  case HS_VERSION_TWO:
+    ret = rend_service_add_ephemeral(pk->v2, port_cfgs, max_streams,
+                                     max_streams_close_circuit, auth_type,
+                                     auth_clients, address_out);
+    break;
+  case HS_VERSION_THREE:
+    ret = hs_service_add_ephemeral(pk->v3, port_cfgs, max_streams,
+                                   max_streams_close_circuit, address_out);
+    break;
+  default:
+    tor_assert_unreached();
+  }
+
+  return ret;
+}
+
 /** Called when we get a ADD_ONION command; parse the body, and set up
  * the new ephemeral Onion Service. */
 static int
@@ -4265,7 +4678,7 @@ handle_control_add_onion(control_connection_t *conn,
                          const char *body)
 {
   smartlist_t *args;
-  size_t arg_len;
+  int arg_len;
   (void) len; /* body is nul-terminated; it's safe to ignore the length */
   args = getargs_helper("ADD_ONION", conn, body, 2, -1);
   if (!args)
@@ -4286,13 +4699,13 @@ handle_control_add_onion(control_connection_t *conn,
   rend_auth_type_t auth_type = REND_NO_AUTH;
   /* Default to adding an anonymous hidden service if no flag is given */
   int non_anonymous = 0;
-  for (size_t i = 1; i < arg_len; i++) {
+  for (int i = 1; i < arg_len; i++) {
     static const char *port_prefix = "Port=";
     static const char *flags_prefix = "Flags=";
     static const char *max_s_prefix = "MaxStreams=";
     static const char *auth_prefix = "ClientAuth=";
 
-    const char *arg = smartlist_get(args, i);
+    const char *arg = smartlist_get(args, (int)i);
     if (!strcasecmpstart(arg, port_prefix)) {
       /* "Port=VIRTPORT[,TARGET]". */
       const char *port_str = arg + strlen(port_prefix);
@@ -4438,15 +4851,15 @@ handle_control_add_onion(control_connection_t *conn,
   }
 
   /* Parse the "keytype:keyblob" argument. */
-  crypto_pk_t *pk = NULL;
+  int hs_version = 0;
+  add_onion_secret_key_t pk = { NULL };
   const char *key_new_alg = NULL;
   char *key_new_blob = NULL;
   char *err_msg = NULL;
 
-  pk = add_onion_helper_keyarg(smartlist_get(args, 0), discard_pk,
-                               &key_new_alg, &key_new_blob,
-                               &err_msg);
-  if (!pk) {
+  if (add_onion_helper_keyarg(smartlist_get(args, 0), discard_pk,
+                              &key_new_alg, &key_new_blob, &pk, &hs_version,
+                              &err_msg) < 0) {
     if (err_msg) {
       connection_write_str_to_buf(err_msg, conn);
       tor_free(err_msg);
@@ -4455,16 +4868,23 @@ handle_control_add_onion(control_connection_t *conn,
   }
   tor_assert(!err_msg);
 
+  /* Hidden service version 3 don't have client authentication support so if
+   * ClientAuth was given, send back an error. */
+  if (hs_version == HS_VERSION_THREE && auth_clients) {
+    connection_printf_to_buf(conn, "513 ClientAuth not supported\r\n");
+    goto out;
+  }
+
   /* Create the HS, using private key pk, client authentication auth_type,
    * the list of auth_clients, and port config port_cfg.
    * rend_service_add_ephemeral() will take ownership of pk and port_cfg,
    * regardless of success/failure.
    */
   char *service_id = NULL;
-  int ret = rend_service_add_ephemeral(pk, port_cfgs, max_streams,
-                                       max_streams_close_circuit,
-                                       auth_type, auth_clients,
-                                       &service_id);
+  int ret = add_onion_helper_add_service(hs_version, &pk, port_cfgs,
+                                         max_streams,
+                                         max_streams_close_circuit, auth_type,
+                                         auth_clients, &service_id);
   port_cfgs = NULL; /* port_cfgs is now owned by the rendservice code. */
   auth_clients = NULL; /* so is auth_clients */
   switch (ret) {
@@ -4557,9 +4977,10 @@ handle_control_add_onion(control_connection_t *conn,
  * Note: The error messages returned are deliberately vague to avoid echoing
  * key material.
  */
-STATIC crypto_pk_t *
+STATIC int
 add_onion_helper_keyarg(const char *arg, int discard_pk,
                         const char **key_new_alg_out, char **key_new_blob_out,
+                        add_onion_secret_key_t *decoded_key, int *hs_version,
                         char **err_msg_out)
 {
   smartlist_t *key_args = smartlist_new();
@@ -4567,7 +4988,7 @@ add_onion_helper_keyarg(const char *arg, int discard_pk,
   const char *key_new_alg = NULL;
   char *key_new_blob = NULL;
   char *err_msg = NULL;
-  int ok = 0;
+  int ret = -1;
 
   smartlist_split_string(key_args, arg, ":", SPLIT_IGNORE_BLANK, 0);
   if (smartlist_len(key_args) != 2) {
@@ -4579,21 +5000,36 @@ add_onion_helper_keyarg(const char *arg, int discard_pk,
   static const char *key_type_new = "NEW";
   static const char *key_type_best = "BEST";
   static const char *key_type_rsa1024 = "RSA1024";
+  static const char *key_type_ed25519_v3 = "ED25519-V3";
 
   const char *key_type = smartlist_get(key_args, 0);
   const char *key_blob = smartlist_get(key_args, 1);
 
   if (!strcasecmp(key_type_rsa1024, key_type)) {
     /* "RSA:<Base64 Blob>" - Loading a pre-existing RSA1024 key. */
-    pk = crypto_pk_base64_decode(key_blob, strlen(key_blob));
+    pk = crypto_pk_base64_decode_private(key_blob, strlen(key_blob));
     if (!pk) {
       err_msg = tor_strdup("512 Failed to decode RSA key\r\n");
       goto err;
     }
     if (crypto_pk_num_bits(pk) != PK_BYTES*8) {
+      crypto_pk_free(pk);
       err_msg = tor_strdup("512 Invalid RSA key size\r\n");
       goto err;
     }
+    decoded_key->v2 = pk;
+    *hs_version = HS_VERSION_TWO;
+  } else if (!strcasecmp(key_type_ed25519_v3, key_type)) {
+    /* "ED25519-V3:<Base64 Blob>" - Loading a pre-existing ed25519 key. */
+    ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
+    if (base64_decode((char *) sk->seckey, sizeof(sk->seckey), key_blob,
+                      strlen(key_blob)) != sizeof(sk->seckey)) {
+      tor_free(sk);
+      err_msg = tor_strdup("512 Failed to decode ED25519-V3 key\r\n");
+      goto err;
+    }
+    decoded_key->v3 = sk;
+    *hs_version = HS_VERSION_THREE;
   } else if (!strcasecmp(key_type_new, key_type)) {
     /* "NEW:<Algorithm>" - Generating a new key, blob as algorithm. */
     if (!strcasecmp(key_type_rsa1024, key_blob) ||
@@ -4606,13 +5042,39 @@ add_onion_helper_keyarg(const char *arg, int discard_pk,
         goto err;
       }
       if (!discard_pk) {
-        if (crypto_pk_base64_encode(pk, &key_new_blob)) {
+        if (crypto_pk_base64_encode_private(pk, &key_new_blob)) {
+          crypto_pk_free(pk);
           tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n",
                        key_type_rsa1024);
           goto err;
         }
         key_new_alg = key_type_rsa1024;
       }
+      decoded_key->v2 = pk;
+      *hs_version = HS_VERSION_TWO;
+    } else if (!strcasecmp(key_type_ed25519_v3, key_blob)) {
+      ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
+      if (ed25519_secret_key_generate(sk, 1) < 0) {
+        tor_free(sk);
+        tor_asprintf(&err_msg, "551 Failed to generate %s key\r\n",
+                     key_type_ed25519_v3);
+        goto err;
+      }
+      if (!discard_pk) {
+        ssize_t len = base64_encode_size(sizeof(sk->seckey), 0) + 1;
+        key_new_blob = tor_malloc_zero(len);
+        if (base64_encode(key_new_blob, len, (const char *) sk->seckey,
+                          sizeof(sk->seckey), 0) != (len - 1)) {
+          tor_free(sk);
+          tor_free(key_new_blob);
+          tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n",
+                       key_type_ed25519_v3);
+          goto err;
+        }
+        key_new_alg = key_type_ed25519_v3;
+      }
+      decoded_key->v3 = sk;
+      *hs_version = HS_VERSION_THREE;
     } else {
       err_msg = tor_strdup("513 Invalid key type\r\n");
       goto err;
@@ -4622,9 +5084,8 @@ add_onion_helper_keyarg(const char *arg, int discard_pk,
     goto err;
   }
 
-  /* Succeded in loading or generating a private key. */
-  tor_assert(pk);
-  ok = 1;
+  /* Succeeded in loading or generating a private key. */
+  ret = 0;
 
  err:
   SMARTLIST_FOREACH(key_args, char *, cp, {
@@ -4633,10 +5094,6 @@ add_onion_helper_keyarg(const char *arg, int discard_pk,
   });
   smartlist_free(key_args);
 
-  if (!ok) {
-    crypto_pk_free(pk);
-    pk = NULL;
-  }
   if (err_msg_out) {
     *err_msg_out = err_msg;
   } else {
@@ -4645,7 +5102,7 @@ add_onion_helper_keyarg(const char *arg, int discard_pk,
   *key_new_alg_out = key_new_alg;
   *key_new_blob_out = key_new_blob;
 
-  return pk;
+  return ret;
 }
 
 /** Helper function to handle parsing a ClientAuth argument to the
@@ -4714,6 +5171,7 @@ handle_control_del_onion(control_connection_t *conn,
                           uint32_t len,
                           const char *body)
 {
+  int hs_version = 0;
   smartlist_t *args;
   (void) len; /* body is nul-terminated; it's safe to ignore the length */
   args = getargs_helper("DEL_ONION", conn, body, 1, 1);
@@ -4721,7 +5179,11 @@ handle_control_del_onion(control_connection_t *conn,
     return 0;
 
   const char *service_id = smartlist_get(args, 0);
-  if (!rend_valid_service_id(service_id)) {
+  if (rend_valid_v2_service_id(service_id)) {
+    hs_version = HS_VERSION_TWO;
+  } else if (hs_address_is_valid(service_id)) {
+    hs_version = HS_VERSION_THREE;
+  } else {
     connection_printf_to_buf(conn, "512 Malformed Onion Service id\r\n");
     goto out;
   }
@@ -4748,8 +5210,20 @@ handle_control_del_onion(control_connection_t *conn,
   if (onion_services == NULL) {
     connection_printf_to_buf(conn, "552 Unknown Onion Service id\r\n");
   } else {
-    int ret = rend_service_del_ephemeral(service_id);
-    if (ret) {
+    int ret = -1;
+    switch (hs_version) {
+    case HS_VERSION_TWO:
+      ret = rend_service_del_ephemeral(service_id);
+      break;
+    case HS_VERSION_THREE:
+      ret = hs_service_del_ephemeral(service_id);
+      break;
+    default:
+      /* The ret value will be -1 thus hitting the warning below. This should
+       * never happen because of the check at the start of the function. */
+      break;
+    }
+    if (ret < 0) {
       /* This should *NEVER* fail, since the service is on either the
        * per-control connection list, or the global one.
        */
@@ -4819,9 +5293,16 @@ connection_control_closed(control_connection_t *conn)
    * The list and it's contents are scrubbed/freed in connection_free_.
    */
   if (conn->ephemeral_onion_services) {
-    SMARTLIST_FOREACH(conn->ephemeral_onion_services, char *, cp, {
-      rend_service_del_ephemeral(cp);
-    });
+    SMARTLIST_FOREACH_BEGIN(conn->ephemeral_onion_services, char *, cp) {
+      if (rend_valid_v2_service_id(cp)) {
+        rend_service_del_ephemeral(cp);
+      } else if (hs_address_is_valid(cp)) {
+        hs_service_del_ephemeral(cp);
+      } else {
+        /* An invalid .onion in our list should NEVER happen */
+        tor_fragile_assert();
+      }
+    } SMARTLIST_FOREACH_END(cp);
   }
 
   if (conn->is_owning_control_connection) {
@@ -4862,6 +5343,38 @@ peek_connection_has_control0_command(connection_t *conn)
   return peek_buf_has_control0_command(conn->inbuf);
 }
 
+static int
+peek_connection_has_http_command(connection_t *conn)
+{
+  return peek_buf_has_http_command(conn->inbuf);
+}
+
+static const char CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG[] =
+  "HTTP/1.0 501 Tor ControlPort is not an HTTP proxy"
+  "\r\nContent-Type: text/html; charset=iso-8859-1\r\n\r\n"
+  "<html>\n"
+  "<head>\n"
+  "<title>Tor's ControlPort is not an HTTP proxy</title>\n"
+  "</head>\n"
+  "<body>\n"
+  "<h1>Tor's ControlPort is not an HTTP proxy</h1>\n"
+  "<p>\n"
+  "It appears you have configured your web browser to use Tor's control port"
+  " as an HTTP proxy.\n"
+  "This is not correct: Tor's default SOCKS proxy port is 9050.\n"
+  "Please configure your client accordingly.\n"
+  "</p>\n"
+  "<p>\n"
+  "See <a href=\"https://www.torproject.org/documentation.html\">"
+  "https://www.torproject.org/documentation.html</a> for more "
+  "information.\n"
+  "<!-- Plus this comment, to make the body response more than 512 bytes, so "
+  "     IE will be willing to display it. Comment comment comment comment "
+  "     comment comment comment comment comment comment comment comment.-->\n"
+  "</p>\n"
+  "</body>\n"
+  "</html>\n";
+
 /** Called when data has arrived on a v1 control connection: Try to fetch
  * commands from conn->inbuf, and execute them.
  */
@@ -4895,12 +5408,21 @@ connection_control_process_inbuf(control_connection_t *conn)
             sizeof(buf)-6);
     body_len = 2+strlen(buf+6)+2; /* code, msg, nul. */
     set_uint16(buf+0, htons(body_len));
-    connection_write_to_buf(buf, 4+body_len, TO_CONN(conn));
+    connection_buf_add(buf, 4+body_len, TO_CONN(conn));
 
     connection_mark_and_flush(TO_CONN(conn));
     return 0;
   }
 
+  /* If the user has the HTTP proxy port and the control port confused. */
+  if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
+      peek_connection_has_http_command(TO_CONN(conn))) {
+    connection_write_str_to_buf(CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG, conn);
+    log_notice(LD_CONTROL, "Received HTTP request on ControlPort");
+    connection_mark_and_flush(TO_CONN(conn));
+    return 0;
+  }
+
  again:
   while (1) {
     size_t last_idx;
@@ -4908,7 +5430,7 @@ connection_control_process_inbuf(control_connection_t *conn)
     /* First, fetch a line. */
     do {
       data_len = conn->incoming_cmd_len - conn->incoming_cmd_cur_len;
-      r = connection_fetch_from_buf_line(TO_CONN(conn),
+      r = connection_buf_get_line(TO_CONN(conn),
                               conn->incoming_cmd+conn->incoming_cmd_cur_len,
                               &data_len);
       if (r == 0)
@@ -5386,8 +5908,8 @@ control_event_stream_status(entry_connection_t *conn, stream_status_event_t tp,
   if (circ && CIRCUIT_IS_ORIGIN(circ))
     origin_circ = TO_ORIGIN_CIRCUIT(circ);
   send_control_event(EVENT_STREAM_STATUS,
-                        "650 STREAM "U64_FORMAT" %s %lu %s%s%s%s\r\n",
-                     U64_PRINTF_ARG(ENTRY_TO_CONN(conn)->global_identifier),
+                        "650 STREAM %"PRIu64" %s %lu %s%s%s%s\r\n",
+                     (ENTRY_TO_CONN(conn)->global_identifier),
                      status,
                         origin_circ?
                            (unsigned long)origin_circ->global_identifier : 0ul,
@@ -5458,12 +5980,12 @@ control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp,
 
   orconn_target_get_name(name, sizeof(name), conn);
   send_control_event(EVENT_OR_CONN_STATUS,
-                              "650 ORCONN %s %s%s%s%s ID="U64_FORMAT"\r\n",
+                              "650 ORCONN %s %s%s%s%s ID=%"PRIu64"\r\n",
                               name, status,
                               reason ? " REASON=" : "",
                               orconn_end_reason_to_control_string(reason),
                               ncircs_buf,
-                              U64_PRINTF_ARG(conn->base_.global_identifier));
+                              (conn->base_.global_identifier));
 
   return 0;
 }
@@ -5474,24 +5996,21 @@ control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp,
 int
 control_event_stream_bandwidth(edge_connection_t *edge_conn)
 {
-  circuit_t *circ;
-  origin_circuit_t *ocirc;
+  struct timeval now;
+  char tbuf[ISO_TIME_USEC_LEN+1];
   if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
     if (!edge_conn->n_read && !edge_conn->n_written)
       return 0;
 
+    tor_gettimeofday(&now);
+    format_iso_time_nospace_usec(tbuf, &now);
     send_control_event(EVENT_STREAM_BANDWIDTH_USED,
-                       "650 STREAM_BW "U64_FORMAT" %lu %lu\r\n",
-                       U64_PRINTF_ARG(edge_conn->base_.global_identifier),
+                       "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n",
+                       (edge_conn->base_.global_identifier),
                        (unsigned long)edge_conn->n_read,
-                       (unsigned long)edge_conn->n_written);
+                       (unsigned long)edge_conn->n_written,
+                       tbuf);
 
-    circ = circuit_get_by_edge_conn(edge_conn);
-    if (circ && CIRCUIT_IS_ORIGIN(circ)) {
-      ocirc = TO_ORIGIN_CIRCUIT(circ);
-      ocirc->n_read_circ_bw += edge_conn->n_read;
-      ocirc->n_written_circ_bw += edge_conn->n_written;
-    }
     edge_conn->n_written = edge_conn->n_read = 0;
   }
 
@@ -5506,6 +6025,8 @@ control_event_stream_bandwidth_used(void)
   if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
     smartlist_t *conns = get_connection_array();
     edge_connection_t *edge_conn;
+    struct timeval now;
+    char tbuf[ISO_TIME_USEC_LEN+1];
 
     SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn)
     {
@@ -5515,11 +6036,14 @@ control_event_stream_bandwidth_used(void)
         if (!edge_conn->n_read && !edge_conn->n_written)
           continue;
 
+        tor_gettimeofday(&now);
+        format_iso_time_nospace_usec(tbuf, &now);
         send_control_event(EVENT_STREAM_BANDWIDTH_USED,
-                           "650 STREAM_BW "U64_FORMAT" %lu %lu\r\n",
-                           U64_PRINTF_ARG(edge_conn->base_.global_identifier),
+                           "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n",
+                           (edge_conn->base_.global_identifier),
                            (unsigned long)edge_conn->n_read,
-                           (unsigned long)edge_conn->n_written);
+                           (unsigned long)edge_conn->n_written,
+                           tbuf);
 
         edge_conn->n_written = edge_conn->n_read = 0;
     }
@@ -5534,28 +6058,69 @@ control_event_stream_bandwidth_used(void)
 int
 control_event_circ_bandwidth_used(void)
 {
-  origin_circuit_t *ocirc;
   if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED))
     return 0;
 
   SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
     if (!CIRCUIT_IS_ORIGIN(circ))
       continue;
-    ocirc = TO_ORIGIN_CIRCUIT(circ);
-    if (!ocirc->n_read_circ_bw && !ocirc->n_written_circ_bw)
-      continue;
-    send_control_event(EVENT_CIRC_BANDWIDTH_USED,
-                       "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu\r\n",
-                       ocirc->global_identifier,
-                       (unsigned long)ocirc->n_read_circ_bw,
-                       (unsigned long)ocirc->n_written_circ_bw);
-    ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
+
+    control_event_circ_bandwidth_used_for_circ(TO_ORIGIN_CIRCUIT(circ));
   }
   SMARTLIST_FOREACH_END(circ);
 
   return 0;
 }
 
+/**
+ * Emit a CIRC_BW event line for a specific circuit.
+ *
+ * This function sets the values it emits to 0, and does not emit
+ * an event if there is no new data to report since the last call.
+ *
+ * Therefore, it may be called at any frequency.
+ */
+int
+control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc)
+{
+  struct timeval now;
+  char tbuf[ISO_TIME_USEC_LEN+1];
+
+  tor_assert(ocirc);
+
+  if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED))
+    return 0;
+
+  /* n_read_circ_bw and n_written_circ_bw are always updated
+   * when there is any new cell on a circuit, and set to 0 after
+   * the event, below.
+   *
+   * Therefore, checking them is sufficient to determine if there
+   * is new data to report. */
+  if (!ocirc->n_read_circ_bw && !ocirc->n_written_circ_bw)
+    return 0;
+
+  tor_gettimeofday(&now);
+  format_iso_time_nospace_usec(tbuf, &now);
+  send_control_event(EVENT_CIRC_BANDWIDTH_USED,
+                     "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu TIME=%s "
+                     "DELIVERED_READ=%lu OVERHEAD_READ=%lu "
+                     "DELIVERED_WRITTEN=%lu OVERHEAD_WRITTEN=%lu\r\n",
+                     ocirc->global_identifier,
+                     (unsigned long)ocirc->n_read_circ_bw,
+                     (unsigned long)ocirc->n_written_circ_bw,
+                     tbuf,
+                     (unsigned long)ocirc->n_delivered_read_circ_bw,
+                     (unsigned long)ocirc->n_overhead_read_circ_bw,
+                     (unsigned long)ocirc->n_delivered_written_circ_bw,
+                     (unsigned long)ocirc->n_overhead_written_circ_bw);
+  ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
+  ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
+  ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
+
+  return 0;
+}
+
 /** Print out CONN_BW event for a single OR/DIR/EXIT <b>conn</b> and reset
   * bandwidth counters. */
 int
@@ -5581,9 +6146,9 @@ control_event_conn_bandwidth(connection_t *conn)
       return 0;
   }
   send_control_event(EVENT_CONN_BW,
-                     "650 CONN_BW ID="U64_FORMAT" TYPE=%s "
+                     "650 CONN_BW ID=%"PRIu64" TYPE=%s "
                      "READ=%lu WRITTEN=%lu\r\n",
-                     U64_PRINTF_ARG(conn->global_identifier),
+                     (conn->global_identifier),
                      conn_type_str,
                      (unsigned long)conn->n_read_conn_bw,
                      (unsigned long)conn->n_written_conn_bw);
@@ -5648,9 +6213,9 @@ append_cell_stats_by_command(smartlist_t *event_parts, const char *key,
   int i;
   for (i = 0; i <= CELL_COMMAND_MAX_; i++) {
     if (include_if_non_zero[i] > 0) {
-      smartlist_add_asprintf(key_value_strings, "%s:"U64_FORMAT,
+      smartlist_add_asprintf(key_value_strings, "%s:%"PRIu64,
                              cell_command_to_string(i),
-                             U64_PRINTF_ARG(number_to_include[i]));
+                             (number_to_include[i]));
     }
   }
   if (smartlist_len(key_value_strings) > 0) {
@@ -5677,8 +6242,8 @@ format_cell_stats(char **event_string, circuit_t *circ,
     or_circuit_t *or_circ = TO_OR_CIRCUIT(circ);
     smartlist_add_asprintf(event_parts, "InboundQueue=%lu",
                  (unsigned long)or_circ->p_circ_id);
-    smartlist_add_asprintf(event_parts, "InboundConn="U64_FORMAT,
-                 U64_PRINTF_ARG(or_circ->p_chan->global_identifier));
+    smartlist_add_asprintf(event_parts, "InboundConn=%"PRIu64,
+                 (or_circ->p_chan->global_identifier));
     append_cell_stats_by_command(event_parts, "InboundAdded",
                                  cell_stats->added_cells_appward,
                                  cell_stats->added_cells_appward);
@@ -5692,8 +6257,8 @@ format_cell_stats(char **event_string, circuit_t *circ,
   if (circ->n_chan) {
     smartlist_add_asprintf(event_parts, "OutboundQueue=%lu",
                      (unsigned long)circ->n_circ_id);
-    smartlist_add_asprintf(event_parts, "OutboundConn="U64_FORMAT,
-                 U64_PRINTF_ARG(circ->n_chan->global_identifier));
+    smartlist_add_asprintf(event_parts, "OutboundConn=%"PRIu64,
+                 (circ->n_chan->global_identifier));
     append_cell_stats_by_command(event_parts, "OutboundAdded",
                                  cell_stats->added_cells_exitward,
                                  cell_stats->added_cells_exitward);
@@ -5719,7 +6284,7 @@ control_event_circuit_cell_stats(void)
   if (!get_options()->TestingEnableCellStatsEvent ||
       !EVENT_IS_INTERESTING(EVENT_CELL_STATS))
     return 0;
-  cell_stats = tor_malloc(sizeof(cell_stats_t));;
+  cell_stats = tor_malloc(sizeof(cell_stats_t));
   SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
     if (!circ->testing_cell_stats)
       continue;
@@ -5734,28 +6299,6 @@ control_event_circuit_cell_stats(void)
   return 0;
 }
 
-/** Tokens in <b>bucket</b> have been refilled: the read bucket was empty
- * for <b>read_empty_time</b> millis, the write bucket was empty for
- * <b>write_empty_time</b> millis, and buckets were last refilled
- * <b>milliseconds_elapsed</b> millis ago.  Only emit TB_EMPTY event if
- * either read or write bucket have been empty before. */
-int
-control_event_tb_empty(const char *bucket, uint32_t read_empty_time,
-                       uint32_t write_empty_time,
-                       int milliseconds_elapsed)
-{
-  if (get_options()->TestingEnableTbEmptyEvent &&
-      EVENT_IS_INTERESTING(EVENT_TB_EMPTY) &&
-      (read_empty_time > 0 || write_empty_time > 0)) {
-    send_control_event(EVENT_TB_EMPTY,
-                       "650 TB_EMPTY %s READ=%d WRITTEN=%d "
-                       "LAST=%d\r\n",
-                       bucket, read_empty_time, write_empty_time,
-                       milliseconds_elapsed);
-  }
-  return 0;
-}
-
 /* about 5 minutes worth. */
 #define N_BW_EVENTS_TO_CACHE 300
 /* Index into cached_bw_events to next write. */
@@ -5843,7 +6386,7 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg)
   int event;
 
   /* Don't even think of trying to add stuff to a buffer from a cpuworker
-   * thread. */
+   * thread. (See #25987 for plan to fix.) */
   if (! in_main_thread())
     return;
 
@@ -5889,6 +6432,23 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg)
   }
 }
 
+/**
+ * Logging callback: called when there is a queued pending log callback.
+ */
+void
+control_event_logmsg_pending(void)
+{
+  if (! in_main_thread()) {
+    /* We can't handle this case yet, since we're using a
+     * mainloop_event_t to invoke queued_events_flush_all.  We ought to
+     * use a different mechanism instead: see #25987.
+     **/
+    return;
+  }
+  tor_assert(flush_queued_events_event);
+  mainloop_event_activate(flush_queued_events_event);
+}
+
 /** Called whenever we receive new router descriptors: tell any
  * interested control connections.  <b>routers</b> is a list of
  * routerinfo_t's.
@@ -5954,47 +6514,6 @@ control_event_address_mapped(const char *from, const char *to, time_t expires,
   return 0;
 }
 
-/** The authoritative dirserver has received a new descriptor that
- * has passed basic syntax checks and is properly self-signed.
- *
- * Notify any interested party of the new descriptor and what has
- * been done with it, and also optionally give an explanation/reason. */
-int
-control_event_or_authdir_new_descriptor(const char *action,
-                                        const char *desc, size_t desclen,
-                                        const char *msg)
-{
-  char firstline[1024];
-  char *buf;
-  size_t totallen;
-  char *esc = NULL;
-  size_t esclen;
-
-  if (!EVENT_IS_INTERESTING(EVENT_AUTHDIR_NEWDESCS))
-    return 0;
-
-  tor_snprintf(firstline, sizeof(firstline),
-               "650+AUTHDIR_NEWDESC=\r\n%s\r\n%s\r\n",
-               action,
-               msg ? msg : "");
-
-  /* Escape the server descriptor properly */
-  esclen = write_escaped_data(desc, desclen, &esc);
-
-  totallen = strlen(firstline) + esclen + 1;
-  buf = tor_malloc(totallen);
-  strlcpy(buf, firstline, totallen);
-  strlcpy(buf+strlen(firstline), esc, totallen);
-  send_control_event_string(EVENT_AUTHDIR_NEWDESCS,
-                            buf);
-  send_control_event_string(EVENT_AUTHDIR_NEWDESCS,
-                            "650 OK\r\n");
-  tor_free(esc);
-  tor_free(buf);
-
-  return 0;
-}
-
 /** Cached liveness for network liveness events and GETINFO
  */
 
@@ -6056,9 +6575,9 @@ control_event_networkstatus_changed_helper(smartlist_t *statuses,
     return 0;
 
   strs = smartlist_new();
-  smartlist_add(strs, tor_strdup("650+"));
-  smartlist_add(strs, tor_strdup(event_string));
-  smartlist_add(strs, tor_strdup("\r\n"));
+  smartlist_add_strdup(strs, "650+");
+  smartlist_add_strdup(strs, event_string);
+  smartlist_add_strdup(strs, "\r\n");
   SMARTLIST_FOREACH(statuses, const routerstatus_t *, rs,
     {
       s = networkstatus_getinfo_helper_single(rs);
@@ -6485,8 +7004,7 @@ monitor_owning_controller_process(const char *process_spec)
             "owning controller: %s.  Exiting.",
             msg);
     owning_controller_process_spec = NULL;
-    tor_cleanup();
-    exit(0);
+    tor_shutdown_event_loop_and_exit(1);
   }
 }
 
@@ -6592,6 +7110,11 @@ bootstrap_status_to_string(bootstrap_status_t s, const char **tag,
  * Tor initializes. */
 static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF;
 
+/** Like bootstrap_percent, but only takes on the enumerated values in
+ * bootstrap_status_t.
+ */
+static int bootstrap_phase = BOOTSTRAP_STATUS_UNDEF;
+
 /** As bootstrap_percent, but holds the bootstrapping level at which we last
  * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT,
  * to avoid flooding the log with a new message every time we get a few more
@@ -6614,23 +7137,46 @@ static int bootstrap_problems = 0;
  */
 #define BOOTSTRAP_PCT_INCREMENT 5
 
+/** Do the actual logging and notifications for
+ * control_event_bootstrap().  Doesn't change any state beyond that.
+ */
+static void
+control_event_bootstrap_core(int loglevel, bootstrap_status_t status,
+                             int progress)
+{
+  char buf[BOOTSTRAP_MSG_LEN];
+  const char *tag, *summary;
+
+  bootstrap_status_to_string(status, &tag, &summary);
+  /* Locally reset status if there's incremental progress */
+  if (progress)
+    status = progress;
+
+  tor_log(loglevel, LD_CONTROL,
+          "Bootstrapped %d%%: %s", status, summary);
+  tor_snprintf(buf, sizeof(buf),
+               "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"",
+               status, tag, summary);
+  tor_snprintf(last_sent_bootstrap_message,
+               sizeof(last_sent_bootstrap_message),
+               "NOTICE %s", buf);
+  control_event_client_status(LOG_NOTICE, "%s", buf);
+}
+
 /** Called when Tor has made progress at bootstrapping its directory
  * information and initial circuits.
  *
  * <b>status</b> is the new status, that is, what task we will be doing
  * next. <b>progress</b> is zero if we just started this task, else it
  * represents progress on the task.
- *
- * Return true if we logged a message at level NOTICE, and false otherwise.
  */
-int
+void
 control_event_bootstrap(bootstrap_status_t status, int progress)
 {
-  const char *tag, *summary;
-  char buf[BOOTSTRAP_MSG_LEN];
+  int loglevel = LOG_NOTICE;
 
   if (bootstrap_percent == BOOTSTRAP_STATUS_DONE)
-    return 0; /* already bootstrapped; nothing to be done here. */
+    return; /* already bootstrapped; nothing to be done here. */
 
   /* special case for handshaking status, since our TLS handshaking code
    * can't distinguish what the connection is going to be for. */
@@ -6642,122 +7188,179 @@ control_event_bootstrap(bootstrap_status_t status, int progress)
     }
   }
 
-  if (status > bootstrap_percent ||
-      (progress && progress > bootstrap_percent)) {
-    int loglevel = LOG_NOTICE;
-    bootstrap_status_to_string(status, &tag, &summary);
-
-    if (status <= bootstrap_percent &&
-        (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT)) {
-      /* We log the message at info if the status hasn't advanced, and if less
-       * than BOOTSTRAP_PCT_INCREMENT progress has been made.
-       */
+  if (status <= bootstrap_percent) {
+    /* If there's no new progress, return early. */
+    if (!progress || progress <= bootstrap_percent)
+      return;
+    /* Log at INFO if not enough progress happened. */
+    if (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT)
       loglevel = LOG_INFO;
-    }
+  }
 
-    tor_log(loglevel, LD_CONTROL,
-            "Bootstrapped %d%%: %s", progress ? progress : status, summary);
-    tor_snprintf(buf, sizeof(buf),
-        "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"",
-        progress ? progress : status, tag, summary);
-    tor_snprintf(last_sent_bootstrap_message,
-                 sizeof(last_sent_bootstrap_message),
-                 "NOTICE %s", buf);
-    control_event_client_status(LOG_NOTICE, "%s", buf);
-    if (status > bootstrap_percent) {
-      bootstrap_percent = status; /* new milestone reached */
-    }
-    if (progress > bootstrap_percent) {
-      /* incremental progress within a milestone */
-      bootstrap_percent = progress;
-      bootstrap_problems = 0; /* Progress! Reset our problem counter. */
-    }
-    if (loglevel == LOG_NOTICE &&
-        bootstrap_percent > notice_bootstrap_percent) {
-      /* Remember that we gave a notice at this level. */
-      notice_bootstrap_percent = bootstrap_percent;
-    }
-    return loglevel == LOG_NOTICE;
+  control_event_bootstrap_core(loglevel, status, progress);
+
+  if (status > bootstrap_percent) {
+    bootstrap_phase = status; /* new milestone reached */
+    bootstrap_percent = status;
+  }
+  if (progress > bootstrap_percent) {
+    /* incremental progress within a milestone */
+    bootstrap_percent = progress;
+    bootstrap_problems = 0; /* Progress! Reset our problem counter. */
+  }
+  if (loglevel == LOG_NOTICE &&
+      bootstrap_percent > notice_bootstrap_percent) {
+    /* Remember that we gave a notice at this level. */
+    notice_bootstrap_percent = bootstrap_percent;
   }
+}
 
-  return 0;
+/** Flag whether we've opened an OR_CONN yet  */
+static int bootstrap_first_orconn = 0;
+
+/** Like bootstrap_phase, but for (possibly deferred) directory progress */
+static int bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF;
+
+/** Like bootstrap_problems, but for (possibly deferred) directory progress  */
+static int bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF;
+
+/** Defer directory info bootstrap events until we have successfully
+ * completed our first connection to a router.  */
+void
+control_event_boot_dir(bootstrap_status_t status, int progress)
+{
+  if (status > bootstrap_dir_progress) {
+    bootstrap_dir_progress = status;
+    bootstrap_dir_phase = status;
+  }
+  if (progress && progress >= bootstrap_dir_progress) {
+    bootstrap_dir_progress = progress;
+  }
+
+  /* Don't report unless we have successfully opened at least one OR_CONN */
+  if (!bootstrap_first_orconn)
+    return;
+
+  control_event_bootstrap(status, progress);
+}
+
+/** Set a flag to allow reporting of directory bootstrap progress.
+ * (Code that reports completion of an OR_CONN calls this.)  Also,
+ * report directory progress so far. */
+void
+control_event_boot_first_orconn(void)
+{
+  bootstrap_first_orconn = 1;
+  control_event_bootstrap(bootstrap_dir_phase, bootstrap_dir_progress);
 }
 
 /** Called when Tor has failed to make bootstrapping progress in a way
- * that indicates a problem. <b>warn</b> gives a hint as to why, and
- * <b>reason</b> provides an "or_conn_end_reason" tag.  <b>or_conn</b>
- * is the connection that caused this problem.
+ * that indicates a problem. <b>warn</b> gives a human-readable hint
+ * as to why, and <b>reason</b> provides a controller-facing short
+ * tag.  <b>conn</b> is the connection that caused this problem and
+ * can be NULL if a connection cannot be easily identified.
  */
-MOCK_IMPL(void,
-          control_event_bootstrap_problem, (const char *warn, int reason,
-                                            or_connection_t *or_conn))
+void
+control_event_bootstrap_problem(const char *warn, const char *reason,
+                                const connection_t *conn, int dowarn)
 {
   int status = bootstrap_percent;
   const char *tag = "", *summary = "";
   char buf[BOOTSTRAP_MSG_LEN];
   const char *recommendation = "ignore";
   int severity;
+  char *or_id = NULL, *hostaddr = NULL;
+  or_connection_t *or_conn = NULL;
 
   /* bootstrap_percent must not be in "undefined" state here. */
   tor_assert(status >= 0);
 
-  if (or_conn->have_noted_bootstrap_problem)
-    return;
-
-  or_conn->have_noted_bootstrap_problem = 1;
-
   if (bootstrap_percent == 100)
     return; /* already bootstrapped; nothing to be done here. */
 
   bootstrap_problems++;
 
   if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD)
-    recommendation = "warn";
+    dowarn = 1;
 
-  if (reason == END_OR_CONN_REASON_NO_ROUTE)
-    recommendation = "warn";
+  /* Don't warn about our bootstrapping status if we are hibernating or
+   * shutting down. */
+  if (we_are_hibernating())
+    dowarn = 0;
 
-  /* If we are using bridges and all our OR connections are now
-     closed, it means that we totally failed to connect to our
-     bridges. Throw a warning. */
-  if (get_options()->UseBridges && !any_other_active_or_conns(or_conn))
-    recommendation = "warn";
+  tor_assert(bootstrap_status_to_string(bootstrap_phase, &tag, &summary) == 0);
 
-  if (we_are_hibernating())
-    recommendation = "ignore";
+  severity = dowarn ? LOG_WARN : LOG_INFO;
 
-  while (status>=0 && bootstrap_status_to_string(status, &tag, &summary) < 0)
-    status--; /* find a recognized status string based on current progress */
-  status = bootstrap_percent; /* set status back to the actual number */
+  if (dowarn)
+    recommendation = "warn";
 
-  severity = !strcmp(recommendation, "warn") ? LOG_WARN : LOG_INFO;
+  if (conn && conn->type == CONN_TYPE_OR) {
+    /* XXX TO_OR_CONN can't deal with const */
+    or_conn = TO_OR_CONN((connection_t *)conn);
+    or_id = tor_strdup(hex_str(or_conn->identity_digest, DIGEST_LEN));
+  } else {
+    or_id = tor_strdup("?");
+  }
+
+  if (conn)
+    tor_asprintf(&hostaddr, "%s:%d", conn->address, (int)conn->port);
+  else
+    hostaddr = tor_strdup("?");
 
   log_fn(severity,
          LD_CONTROL, "Problem bootstrapping. Stuck at %d%%: %s. (%s; %s; "
-         "count %d; recommendation %s; host %s at %s:%d)",
-         status, summary, warn,
-         orconn_end_reason_to_control_string(reason),
+         "count %d; recommendation %s; host %s at %s)",
+         status, summary, warn, reason,
          bootstrap_problems, recommendation,
-         hex_str(or_conn->identity_digest, DIGEST_LEN),
-         or_conn->base_.address,
-         or_conn->base_.port);
+         or_id, hostaddr);
 
   connection_or_report_broken_states(severity, LD_HANDSHAKE);
 
   tor_snprintf(buf, sizeof(buf),
       "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s "
-      "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s:%d\"",
-      bootstrap_percent, tag, summary, warn,
-      orconn_end_reason_to_control_string(reason), bootstrap_problems,
+      "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s\"",
+      bootstrap_percent, tag, summary, warn, reason, bootstrap_problems,
       recommendation,
-      hex_str(or_conn->identity_digest, DIGEST_LEN),
-      or_conn->base_.address,
-      (int)or_conn->base_.port);
+      or_id, hostaddr);
 
   tor_snprintf(last_sent_bootstrap_message,
                sizeof(last_sent_bootstrap_message),
                "WARN %s", buf);
   control_event_client_status(LOG_WARN, "%s", buf);
+
+  tor_free(hostaddr);
+  tor_free(or_id);
+}
+
+/** Called when Tor has failed to make bootstrapping progress in a way
+ * that indicates a problem. <b>warn</b> gives a hint as to why, and
+ * <b>reason</b> provides an "or_conn_end_reason" tag.  <b>or_conn</b>
+ * is the connection that caused this problem.
+ */
+MOCK_IMPL(void,
+control_event_bootstrap_prob_or, (const char *warn, int reason,
+                                  or_connection_t *or_conn))
+{
+  int dowarn = 0;
+
+  if (or_conn->have_noted_bootstrap_problem)
+    return;
+
+  or_conn->have_noted_bootstrap_problem = 1;
+
+  if (reason == END_OR_CONN_REASON_NO_ROUTE)
+    dowarn = 1;
+
+  /* If we are using bridges and all our OR connections are now
+     closed, it means that we totally failed to connect to our
+     bridges. Throw a warning. */
+  if (get_options()->UseBridges && !any_other_active_or_conns(or_conn))
+    dowarn = 1;
+
+  control_event_bootstrap_problem(warn,
+                                  orconn_end_reason_to_control_string(reason),
+                                  TO_CONN(or_conn), dowarn);
 }
 
 /** We just generated a new summary of which countries we've seen clients
@@ -6852,25 +7455,33 @@ rend_hsaddress_str_or_unknown(const char *onion_address)
  * <b>rend_query</b> is used to fetch requested onion address and auth type.
  * <b>hs_dir</b> is the description of contacting hs directory.
  * <b>desc_id_base32</b> is the ID of requested hs descriptor.
+ * <b>hsdir_index</b> is the HSDir fetch index value for v3, an hex string.
  */
 void
-control_event_hs_descriptor_requested(const rend_data_t *rend_query,
+control_event_hs_descriptor_requested(const char *onion_address,
+                                      rend_auth_type_t auth_type,
                                       const char *id_digest,
-                                      const char *desc_id_base32)
+                                      const char *desc_id,
+                                      const char *hsdir_index)
 {
-  if (!id_digest || !rend_query || !desc_id_base32) {
-    log_warn(LD_BUG, "Called with rend_query==%p, "
-             "id_digest==%p, desc_id_base32==%p",
-             rend_query, id_digest, desc_id_base32);
+  char *hsdir_index_field = NULL;
+
+  if (BUG(!id_digest || !desc_id)) {
     return;
   }
 
+  if (hsdir_index) {
+    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
+  }
+
   send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC REQUESTED %s %s %s %s\r\n",
-                     rend_hsaddress_str_or_unknown(rend_query->onion_address),
-                     rend_auth_type_to_string(rend_query->auth_type),
+                     "650 HS_DESC REQUESTED %s %s %s %s%s\r\n",
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     rend_auth_type_to_string(auth_type),
                      node_describe_longname_by_id(id_digest),
-                     desc_id_base32);
+                     desc_id,
+                     hsdir_index_field ? hsdir_index_field : "");
+  tor_free(hsdir_index_field);
 }
 
 /** For an HS descriptor query <b>rend_data</b>, using the
@@ -6884,19 +7495,25 @@ get_desc_id_from_query(const rend_data_t *rend_data, const char *hsdir_fp)
 {
   int replica;
   const char *desc_id = NULL;
+  const rend_data_v2_t *rend_data_v2 = TO_REND_DATA_V2(rend_data);
 
   /* Possible if the fetch was done using a descriptor ID. This means that
    * the HSFETCH command was used. */
-  if (!tor_digest_is_zero(rend_data->desc_id_fetch)) {
-    desc_id = rend_data->desc_id_fetch;
+  if (!tor_digest_is_zero(rend_data_v2->desc_id_fetch)) {
+    desc_id = rend_data_v2->desc_id_fetch;
     goto end;
   }
 
+  /* Without a directory fingerprint at this stage, we can't do much. */
+  if (hsdir_fp == NULL) {
+     goto end;
+  }
+
   /* OK, we have an onion address so now let's find which descriptor ID
    * is the one associated with the HSDir fingerprint. */
   for (replica = 0; replica < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS;
        replica++) {
-    const char *digest = rend_data->descriptor_id[replica];
+    const char *digest = rend_data_get_desc_id(rend_data, replica, NULL);
 
     SMARTLIST_FOREACH_BEGIN(rend_data->hsdirs_fp, char *, fingerprint) {
       if (tor_memcmp(fingerprint, hsdir_fp, DIGEST_LEN) == 0) {
@@ -6913,90 +7530,87 @@ get_desc_id_from_query(const rend_data_t *rend_data, const char *hsdir_fp)
 
 /** send HS_DESC CREATED event when a local service generates a descriptor.
  *
- * <b>service_id</b> is the descriptor onion address.
- * <b>desc_id_base32</b> is the descriptor ID.
- * <b>replica</b> is the the descriptor replica number.
+ * <b>onion_address</b> is service address.
+ * <b>desc_id</b> is the descriptor ID.
+ * <b>replica</b> is the the descriptor replica number. If it is negative, it
+ * is ignored.
  */
 void
-control_event_hs_descriptor_created(const char *service_id,
-                                    const char *desc_id_base32,
+control_event_hs_descriptor_created(const char *onion_address,
+                                    const char *desc_id,
                                     int replica)
 {
-  if (!service_id || !desc_id_base32) {
-    log_warn(LD_BUG, "Called with service_digest==%p, "
-             "desc_id_base32==%p", service_id, desc_id_base32);
+  char *replica_field = NULL;
+
+  if (BUG(!onion_address || !desc_id)) {
     return;
   }
 
+  if (replica >= 0) {
+    tor_asprintf(&replica_field, " REPLICA=%d", replica);
+  }
+
   send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC CREATED %s UNKNOWN UNKNOWN %s "
-                     "REPLICA=%d\r\n",
-                     service_id,
-                     desc_id_base32,
-                     replica);
+                     "650 HS_DESC CREATED %s UNKNOWN UNKNOWN %s%s\r\n",
+                     onion_address, desc_id,
+                     replica_field ? replica_field : "");
+  tor_free(replica_field);
 }
 
 /** send HS_DESC upload event.
  *
- * <b>service_id</b> is the descriptor onion address.
+ * <b>onion_address</b> is service address.
  * <b>hs_dir</b> is the description of contacting hs directory.
- * <b>desc_id_base32</b> is the ID of requested hs descriptor.
+ * <b>desc_id</b> is the ID of requested hs descriptor.
  */
 void
-control_event_hs_descriptor_upload(const char *service_id,
+control_event_hs_descriptor_upload(const char *onion_address,
                                    const char *id_digest,
-                                   const char *desc_id_base32)
+                                   const char *desc_id,
+                                   const char *hsdir_index)
 {
-  if (!service_id || !id_digest || !desc_id_base32) {
-    log_warn(LD_BUG, "Called with service_digest==%p, "
-             "desc_id_base32==%p, id_digest==%p", service_id,
-             desc_id_base32, id_digest);
+  char *hsdir_index_field = NULL;
+
+  if (BUG(!onion_address || !id_digest || !desc_id)) {
     return;
   }
 
+  if (hsdir_index) {
+    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
+  }
+
   send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC UPLOAD %s UNKNOWN %s %s\r\n",
-                     service_id,
+                     "650 HS_DESC UPLOAD %s UNKNOWN %s %s%s\r\n",
+                     onion_address,
                      node_describe_longname_by_id(id_digest),
-                     desc_id_base32);
+                     desc_id,
+                     hsdir_index_field ? hsdir_index_field : "");
+  tor_free(hsdir_index_field);
 }
 
 /** send HS_DESC event after got response from hs directory.
  *
  * NOTE: this is an internal function used by following functions:
- * control_event_hs_descriptor_received
- * control_event_hs_descriptor_failed
+ * control_event_hsv2_descriptor_received
+ * control_event_hsv2_descriptor_failed
+ * control_event_hsv3_descriptor_failed
  *
  * So do not call this function directly.
  */
-void
-control_event_hs_descriptor_receive_end(const char *action,
-                                        const char *onion_address,
-                                        const rend_data_t *rend_data,
-                                        const char *id_digest,
-                                        const char *reason)
+static void
+event_hs_descriptor_receive_end(const char *action,
+                                const char *onion_address,
+                                const char *desc_id,
+                                rend_auth_type_t auth_type,
+                                const char *hsdir_id_digest,
+                                const char *reason)
 {
-  char *desc_id_field = NULL;
   char *reason_field = NULL;
-  char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
-  const char *desc_id = NULL;
 
-  if (!action || !id_digest || !rend_data || !onion_address) {
-    log_warn(LD_BUG, "Called with action==%p, id_digest==%p, "
-             "rend_data==%p, onion_address==%p", action, id_digest,
-             rend_data, onion_address);
+  if (BUG(!action || !onion_address)) {
     return;
   }
 
-  desc_id = get_desc_id_from_query(rend_data, id_digest);
-  if (desc_id != NULL) {
-    /* Set the descriptor ID digest to base32 so we can send it. */
-    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
-                  DIGEST_LEN);
-    /* Extra whitespace is needed before the value. */
-    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
-  }
-
   if (reason) {
     tor_asprintf(&reason_field, " REASON=%s", reason);
   }
@@ -7005,12 +7619,13 @@ control_event_hs_descriptor_receive_end(const char *action,
                      "650 HS_DESC %s %s %s %s%s%s\r\n",
                      action,
                      rend_hsaddress_str_or_unknown(onion_address),
-                     rend_auth_type_to_string(rend_data->auth_type),
-                     node_describe_longname_by_id(id_digest),
-                     desc_id_field ? desc_id_field : "",
+                     rend_auth_type_to_string(auth_type),
+                     hsdir_id_digest ?
+                        node_describe_longname_by_id(hsdir_id_digest) :
+                        "UNKNOWN",
+                     desc_id ? desc_id : "",
                      reason_field ? reason_field : "");
 
-  tor_free(desc_id_field);
   tor_free(reason_field);
 }
 
@@ -7030,9 +7645,7 @@ control_event_hs_descriptor_upload_end(const char *action,
 {
   char *reason_field = NULL;
 
-  if (!action || !id_digest) {
-    log_warn(LD_BUG, "Called with action==%p, id_digest==%p", action,
-             id_digest);
+  if (BUG(!action || !id_digest)) {
     return;
   }
 
@@ -7055,17 +7668,54 @@ control_event_hs_descriptor_upload_end(const char *action,
  * called when we successfully received a hidden service descriptor.
  */
 void
-control_event_hs_descriptor_received(const char *onion_address,
-                                     const rend_data_t *rend_data,
-                                     const char *id_digest)
+control_event_hsv2_descriptor_received(const char *onion_address,
+                                       const rend_data_t *rend_data,
+                                       const char *hsdir_id_digest)
 {
-  if (!rend_data || !id_digest || !onion_address) {
-    log_warn(LD_BUG, "Called with rend_data==%p, id_digest==%p, "
-             "onion_address==%p", rend_data, id_digest, onion_address);
+  char *desc_id_field = NULL;
+  const char *desc_id;
+
+  if (BUG(!rend_data || !hsdir_id_digest || !onion_address)) {
+    return;
+  }
+
+  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
+  if (desc_id != NULL) {
+    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+    /* Set the descriptor ID digest to base32 so we can send it. */
+    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
+                  DIGEST_LEN);
+    /* Extra whitespace is needed before the value. */
+    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
+  }
+
+  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
+                                  TO_REND_DATA_V2(rend_data)->auth_type,
+                                  hsdir_id_digest, NULL);
+  tor_free(desc_id_field);
+}
+
+/* Send HS_DESC RECEIVED event
+ *
+ * Called when we successfully received a hidden service descriptor. */
+void
+control_event_hsv3_descriptor_received(const char *onion_address,
+                                       const char *desc_id,
+                                       const char *hsdir_id_digest)
+{
+  char *desc_id_field = NULL;
+
+  if (BUG(!onion_address || !desc_id || !hsdir_id_digest)) {
     return;
   }
-  control_event_hs_descriptor_receive_end("RECEIVED", onion_address,
-                                          rend_data, id_digest, NULL);
+
+  /* Because DescriptorID is an optional positional value, we need to add a
+   * whitespace before in order to not be next to the HsDir value. */
+  tor_asprintf(&desc_id_field, " %s", desc_id);
+
+  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
+                                  REND_NO_AUTH, hsdir_id_digest, NULL);
+  tor_free(desc_id_field);
 }
 
 /** send HS_DESC UPLOADED event
@@ -7076,9 +7726,7 @@ void
 control_event_hs_descriptor_uploaded(const char *id_digest,
                                      const char *onion_address)
 {
-  if (!id_digest) {
-    log_warn(LD_BUG, "Called with id_digest==%p",
-             id_digest);
+  if (BUG(!id_digest)) {
     return;
   }
 
@@ -7086,28 +7734,71 @@ control_event_hs_descriptor_uploaded(const char *id_digest,
                                          id_digest, NULL);
 }
 
-/** Send HS_DESC event to inform controller that query <b>rend_query</b>
- * failed to retrieve hidden service descriptor identified by
- * <b>id_digest</b>. If <b>reason</b> is not NULL, add it to REASON=
- * field.
+/** Send HS_DESC event to inform controller that query <b>rend_data</b>
+ * failed to retrieve hidden service descriptor from directory identified by
+ * <b>id_digest</b>. If NULL, "UNKNOWN" is used. If <b>reason</b> is not NULL,
+ * add it to REASON= field.
  */
 void
-control_event_hs_descriptor_failed(const rend_data_t *rend_data,
-                                   const char *id_digest,
-                                   const char *reason)
+control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
+                                     const char *hsdir_id_digest,
+                                     const char *reason)
+{
+  char *desc_id_field = NULL;
+  const char *desc_id;
+
+  if (BUG(!rend_data)) {
+    return;
+  }
+
+  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
+  if (desc_id != NULL) {
+    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+    /* Set the descriptor ID digest to base32 so we can send it. */
+    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
+                  DIGEST_LEN);
+    /* Extra whitespace is needed before the value. */
+    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
+  }
+
+  event_hs_descriptor_receive_end("FAILED", rend_data_get_address(rend_data),
+                                  desc_id_field,
+                                  TO_REND_DATA_V2(rend_data)->auth_type,
+                                  hsdir_id_digest, reason);
+  tor_free(desc_id_field);
+}
+
+/** Send HS_DESC event to inform controller that the query to
+ * <b>onion_address</b> failed to retrieve hidden service descriptor
+ * <b>desc_id</b> from directory identified by <b>hsdir_id_digest</b>. If
+ * NULL, "UNKNOWN" is used.  If <b>reason</b> is not NULL, add it to REASON=
+ * field. */
+void
+control_event_hsv3_descriptor_failed(const char *onion_address,
+                                     const char *desc_id,
+                                     const char *hsdir_id_digest,
+                                     const char *reason)
 {
-  if (!rend_data || !id_digest) {
-    log_warn(LD_BUG, "Called with rend_data==%p, id_digest==%p",
-             rend_data, id_digest);
+  char *desc_id_field = NULL;
+
+  if (BUG(!onion_address || !desc_id || !reason)) {
     return;
   }
-  control_event_hs_descriptor_receive_end("FAILED",
-                                          rend_data->onion_address,
-                                          rend_data, id_digest, reason);
+
+  /* Because DescriptorID is an optional positional value, we need to add a
+   * whitespace before in order to not be next to the HsDir value. */
+  tor_asprintf(&desc_id_field, " %s", desc_id);
+
+  event_hs_descriptor_receive_end("FAILED", onion_address, desc_id_field,
+                                  REND_NO_AUTH, hsdir_id_digest, reason);
+  tor_free(desc_id_field);
 }
 
-/** send HS_DESC_CONTENT event after completion of a successful fetch from
- * hs directory. */
+/** Send HS_DESC_CONTENT event after completion of a successful fetch
+ * from hs directory. If <b>hsdir_id_digest</b> is NULL, it is replaced
+ * by "UNKNOWN". If <b>content</b> is NULL, it is replaced by an empty
+ * string. The  <b>onion_address</b> or <b>desc_id</b> set to NULL will
+ * not trigger the control event. */
 void
 control_event_hs_descriptor_content(const char *onion_address,
                                     const char *desc_id,
@@ -7117,9 +7808,9 @@ control_event_hs_descriptor_content(const char *onion_address,
   static const char *event_name = "HS_DESC_CONTENT";
   char *esc_content = NULL;
 
-  if (!onion_address || !desc_id || !hsdir_id_digest) {
-    log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, "
-             "hsdir_id_digest==%p", onion_address, desc_id, hsdir_id_digest);
+  if (!onion_address || !desc_id) {
+    log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, ",
+             onion_address, desc_id);
     return;
   }
 
@@ -7134,7 +7825,9 @@ control_event_hs_descriptor_content(const char *onion_address,
                      event_name,
                      rend_hsaddress_str_or_unknown(onion_address),
                      desc_id,
-                     node_describe_longname_by_id(hsdir_id_digest),
+                     hsdir_id_digest ?
+                        node_describe_longname_by_id(hsdir_id_digest) :
+                        "UNKNOWN",
                      esc_content);
   tor_free(esc_content);
 }
@@ -7148,12 +7841,10 @@ control_event_hs_descriptor_upload_failed(const char *id_digest,
                                           const char *onion_address,
                                           const char *reason)
 {
-  if (!id_digest) {
-    log_warn(LD_BUG, "Called with id_digest==%p",
-             id_digest);
+  if (BUG(!id_digest)) {
     return;
   }
-  control_event_hs_descriptor_upload_end("UPLOAD_FAILED", onion_address,
+  control_event_hs_descriptor_upload_end("FAILED", onion_address,
                                          id_digest, reason);
 }
 
@@ -7161,22 +7852,44 @@ control_event_hs_descriptor_upload_failed(const char *id_digest,
 void
 control_free_all(void)
 {
+  smartlist_t *queued_events = NULL;
+
+  stats_prev_n_read = stats_prev_n_written = 0;
+
   if (authentication_cookie) /* Free the auth cookie */
     tor_free(authentication_cookie);
   if (detached_onion_services) { /* Free the detached onion services */
     SMARTLIST_FOREACH(detached_onion_services, char *, cp, tor_free(cp));
     smartlist_free(detached_onion_services);
   }
-  if (queued_control_events) {
-    SMARTLIST_FOREACH(queued_control_events, queued_event_t *, ev,
-                      queued_event_free(ev));
-    smartlist_free(queued_control_events);
+
+  if (queued_control_events_lock) {
+    tor_mutex_acquire(queued_control_events_lock);
+    flush_queued_event_pending = 0;
+    queued_events = queued_control_events;
     queued_control_events = NULL;
+    tor_mutex_release(queued_control_events_lock);
+  }
+  if (queued_events) {
+    SMARTLIST_FOREACH(queued_events, queued_event_t *, ev,
+                      queued_event_free(ev));
+    smartlist_free(queued_events);
   }
   if (flush_queued_events_event) {
-    tor_event_free(flush_queued_events_event);
+    mainloop_event_free(flush_queued_events_event);
     flush_queued_events_event = NULL;
   }
+  bootstrap_percent = BOOTSTRAP_STATUS_UNDEF;
+  bootstrap_phase = BOOTSTRAP_STATUS_UNDEF;
+  notice_bootstrap_percent = 0;
+  bootstrap_problems = 0;
+  bootstrap_first_orconn = 0;
+  bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF;
+  bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF;
+  authentication_cookie_is_set = 0;
+  global_event_mask = 0;
+  disable_log_messages = 0;
+  memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message));
 }
 
 #ifdef TOR_UNIT_TESTS
@@ -7186,5 +7899,4 @@ control_testing_set_global_event_mask(uint64_t mask)
 {
   global_event_mask = mask;
 }
-#endif
-
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/or/control.h b/src/feature/control/control.h
index 6330c85571..a09c1cd11b 100644
--- a/src/or/control.h
+++ b/src/feature/control/control.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,6 +12,93 @@
 #ifndef TOR_CONTROL_H
 #define TOR_CONTROL_H
 
+/** Used to indicate the type of a circuit event passed to the controller.
+ * The various types are defined in control-spec.txt */
+typedef enum circuit_status_event_t {
+  CIRC_EVENT_LAUNCHED = 0,
+  CIRC_EVENT_BUILT    = 1,
+  CIRC_EVENT_EXTENDED = 2,
+  CIRC_EVENT_FAILED   = 3,
+  CIRC_EVENT_CLOSED   = 4,
+} circuit_status_event_t;
+
+/** Used to indicate the type of a CIRC_MINOR event passed to the controller.
+ * The various types are defined in control-spec.txt . */
+typedef enum circuit_status_minor_event_t {
+  CIRC_MINOR_EVENT_PURPOSE_CHANGED,
+  CIRC_MINOR_EVENT_CANNIBALIZED,
+} circuit_status_minor_event_t;
+
+/** Used to indicate the type of a stream event passed to the controller.
+ * The various types are defined in control-spec.txt */
+typedef enum stream_status_event_t {
+  STREAM_EVENT_SENT_CONNECT = 0,
+  STREAM_EVENT_SENT_RESOLVE = 1,
+  STREAM_EVENT_SUCCEEDED    = 2,
+  STREAM_EVENT_FAILED       = 3,
+  STREAM_EVENT_CLOSED       = 4,
+  STREAM_EVENT_NEW          = 5,
+  STREAM_EVENT_NEW_RESOLVE  = 6,
+  STREAM_EVENT_FAILED_RETRIABLE = 7,
+  STREAM_EVENT_REMAP        = 8
+} stream_status_event_t;
+
+/** Used to indicate the type of an OR connection event passed to the
+ * controller.  The various types are defined in control-spec.txt */
+typedef enum or_conn_status_event_t {
+  OR_CONN_EVENT_LAUNCHED     = 0,
+  OR_CONN_EVENT_CONNECTED    = 1,
+  OR_CONN_EVENT_FAILED       = 2,
+  OR_CONN_EVENT_CLOSED       = 3,
+  OR_CONN_EVENT_NEW          = 4,
+} or_conn_status_event_t;
+
+/** Used to indicate the type of a buildtime event */
+typedef enum buildtimeout_set_event_t {
+  BUILDTIMEOUT_SET_EVENT_COMPUTED  = 0,
+  BUILDTIMEOUT_SET_EVENT_RESET     = 1,
+  BUILDTIMEOUT_SET_EVENT_SUSPENDED = 2,
+  BUILDTIMEOUT_SET_EVENT_DISCARD = 3,
+  BUILDTIMEOUT_SET_EVENT_RESUME = 4
+} buildtimeout_set_event_t;
+
+/** Enum describing various stages of bootstrapping, for use with controller
+ * bootstrap status events. The values range from 0 to 100. */
+typedef enum {
+  BOOTSTRAP_STATUS_UNDEF=-1,
+  BOOTSTRAP_STATUS_STARTING=0,
+  BOOTSTRAP_STATUS_CONN_DIR=5,
+  BOOTSTRAP_STATUS_HANDSHAKE=-2,
+  BOOTSTRAP_STATUS_HANDSHAKE_DIR=10,
+  BOOTSTRAP_STATUS_ONEHOP_CREATE=15,
+  BOOTSTRAP_STATUS_REQUESTING_STATUS=20,
+  BOOTSTRAP_STATUS_LOADING_STATUS=25,
+  BOOTSTRAP_STATUS_LOADING_KEYS=40,
+  BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS=45,
+  BOOTSTRAP_STATUS_LOADING_DESCRIPTORS=50,
+  BOOTSTRAP_STATUS_CONN_OR=80,
+  BOOTSTRAP_STATUS_HANDSHAKE_OR=85,
+  BOOTSTRAP_STATUS_CIRCUIT_CREATE=90,
+  BOOTSTRAP_STATUS_DONE=100
+} bootstrap_status_t;
+
+control_connection_t *TO_CONTROL_CONN(connection_t *);
+
+#define CONTROL_CONN_STATE_MIN_ 1
+/** State for a control connection: Authenticated and accepting v1 commands. */
+#define CONTROL_CONN_STATE_OPEN 1
+/** State for a control connection: Waiting for authentication; speaking
+ * protocol v1. */
+#define CONTROL_CONN_STATE_NEEDAUTH 2
+#define CONTROL_CONN_STATE_MAX_ 2
+
+/** Reason for remapping an AP connection's address: we have a cached
+ * answer. */
+#define REMAP_STREAM_SOURCE_CACHE 1
+/** Reason for remapping an AP connection's address: the exit node told us an
+ * answer. */
+#define REMAP_STREAM_SOURCE_EXIT 2
+
 void control_initialize_event_queue(void);
 
 void control_update_global_event_mask(void);
@@ -27,16 +114,22 @@ void control_ports_write_to_file(void);
 #define LOG_FN_CONN(conn, args)                 \
   CONN_LOG_PROTECT(conn, log_fn args)
 
+#define CC_LOCAL_FD_IS_OWNER (1u<<0)
+#define CC_LOCAL_FD_IS_AUTHENTICATED (1u<<1)
+int control_connection_add_local_fd(tor_socket_t sock, unsigned flags);
+
 int connection_control_finished_flushing(control_connection_t *conn);
 int connection_control_reached_eof(control_connection_t *conn);
 void connection_control_closed(control_connection_t *conn);
 
 int connection_control_process_inbuf(control_connection_t *conn);
 
-#define EVENT_AUTHDIR_NEWDESCS 0x000D
 #define EVENT_NS 0x000F
 int control_event_is_interesting(int event);
 
+void control_per_second_events(void);
+int control_any_per_second_event_enabled(void);
+
 int control_event_circuit_status(origin_circuit_t *circ,
                                  circuit_status_event_t e, int reason);
 int control_event_circuit_purpose_changed(origin_circuit_t *circ,
@@ -53,21 +146,16 @@ int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written);
 int control_event_stream_bandwidth(edge_connection_t *edge_conn);
 int control_event_stream_bandwidth_used(void);
 int control_event_circ_bandwidth_used(void);
+int control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc);
 int control_event_conn_bandwidth(connection_t *conn);
 int control_event_conn_bandwidth_used(void);
 int control_event_circuit_cell_stats(void);
-int control_event_tb_empty(const char *bucket, uint32_t read_empty_time,
-                           uint32_t write_empty_time,
-                           int milliseconds_elapsed);
 void control_event_logmsg(int severity, uint32_t domain, const char *msg);
+void control_event_logmsg_pending(void);
 int control_event_descriptors_changed(smartlist_t *routers);
 int control_event_address_mapped(const char *from, const char *to,
                                  time_t expires, const char *error,
                                  const int cached);
-int control_event_or_authdir_new_descriptor(const char *action,
-                                            const char *desc,
-                                            size_t desclen,
-                                            const char *msg);
 int control_event_my_descriptor_changed(void);
 int control_event_network_liveness_update(int liveness);
 int control_event_networkstatus_changed(smartlist_t *statuses);
@@ -97,16 +185,21 @@ int control_event_signal(uintptr_t signal);
 
 int init_control_cookie_authentication(int enabled);
 char *get_controller_cookie_file_name(void);
-smartlist_t *decode_hashed_passwords(config_line_t *passwords);
+struct config_line_t;
+smartlist_t *decode_hashed_passwords(struct config_line_t *passwords);
 void disable_control_logging(void);
 void enable_control_logging(void);
 
 void monitor_owning_controller_process(const char *process_spec);
 
-int control_event_bootstrap(bootstrap_status_t status, int progress);
-MOCK_DECL(void, control_event_bootstrap_problem,(const char *warn,
+void control_event_bootstrap(bootstrap_status_t status, int progress);
+MOCK_DECL(void, control_event_bootstrap_prob_or,(const char *warn,
                                                  int reason,
                                                  or_connection_t *or_conn));
+void control_event_boot_dir(bootstrap_status_t status, int progress);
+void control_event_boot_first_orconn(void);
+void control_event_bootstrap_problem(const char *warn, const char *reason,
+                                     const connection_t *conn, int dowarn);
 
 void control_event_clients_seen(const char *controller_str);
 void control_event_transport_launched(const char *mode,
@@ -114,32 +207,39 @@ void control_event_transport_launched(const char *mode,
                                       tor_addr_t *addr, uint16_t port);
 const char *rend_auth_type_to_string(rend_auth_type_t auth_type);
 MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest));
-void control_event_hs_descriptor_requested(const rend_data_t *rend_query,
-                                           const char *desc_id_base32,
-                                           const char *hs_dir);
-void control_event_hs_descriptor_created(const char *service_id,
-                                         const char *desc_id_base32,
+void control_event_hs_descriptor_requested(const char *onion_address,
+                                           rend_auth_type_t auth_type,
+                                           const char *id_digest,
+                                           const char *desc_id,
+                                           const char *hsdir_index);
+void control_event_hs_descriptor_created(const char *onion_address,
+                                         const char *desc_id,
                                          int replica);
-void control_event_hs_descriptor_upload(const char *service_id,
-                                        const char *desc_id_base32,
-                                        const char *hs_dir);
-void control_event_hs_descriptor_receive_end(const char *action,
-                                             const char *onion_address,
-                                             const rend_data_t *rend_data,
-                                             const char *id_digest,
-                                             const char *reason);
+void control_event_hs_descriptor_upload(const char *onion_address,
+                                        const char *desc_id,
+                                        const char *hs_dir,
+                                        const char *hsdir_index);
 void control_event_hs_descriptor_upload_end(const char *action,
                                             const char *onion_address,
                                             const char *hs_dir,
                                             const char *reason);
-void control_event_hs_descriptor_received(const char *onion_address,
-                                          const rend_data_t *rend_data,
-                                          const char *id_digest);
 void control_event_hs_descriptor_uploaded(const char *hs_dir,
                                           const char *onion_address);
-void control_event_hs_descriptor_failed(const rend_data_t *rend_data,
-                                        const char *id_digest,
-                                        const char *reason);
+/* Hidden service v2 HS_DESC specific. */
+void control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
+                                          const char *id_digest,
+                                          const char *reason);
+void control_event_hsv2_descriptor_received(const char *onion_address,
+                                            const rend_data_t *rend_data,
+                                            const char *id_digest);
+/* Hidden service v3 HS_DESC specific. */
+void control_event_hsv3_descriptor_failed(const char *onion_address,
+                                          const char *desc_id,
+                                          const char *hsdir_id_digest,
+                                          const char *reason);
+void control_event_hsv3_descriptor_received(const char *onion_address,
+                                            const char *desc_id,
+                                            const char *hsdir_id_digest);
 void control_event_hs_descriptor_upload_failed(const char *hs_dir,
                                                const char *onion_address,
                                                const char *reason);
@@ -147,10 +247,11 @@ void control_event_hs_descriptor_content(const char *onion_address,
                                          const char *desc_id,
                                          const char *hsdir_fp,
                                          const char *content);
-
 void control_free_all(void);
 
 #ifdef CONTROL_PRIVATE
+#include "lib/crypt_ops/crypto_ed25519.h"
+
 /* Recognized asynchronous event types.  It's okay to expand this list
  * because it is used both as a list of v0 event types, and as indices
  * into the bitfield to determine which controllers want which events.
@@ -169,8 +270,8 @@ void control_free_all(void);
 #define EVENT_WARN_MSG                0x000A
 #define EVENT_ERR_MSG                 0x000B
 #define EVENT_ADDRMAP                 0x000C
-/* Exposed above */
-// #define EVENT_AUTHDIR_NEWDESCS     0x000D
+/* There was an AUTHDIR_NEWDESCS event, but it no longer exists.  We
+   can reclaim 0x000D. */
 #define EVENT_DESCCHANGED             0x000E
 /* Exposed above */
 // #define EVENT_NS                   0x000F
@@ -186,7 +287,7 @@ void control_free_all(void);
 #define EVENT_CONF_CHANGED            0x0019
 #define EVENT_CONN_BW                 0x001A
 #define EVENT_CELL_STATS              0x001B
-#define EVENT_TB_EMPTY                0x001C
+/* UNUSED :                           0x001C */
 #define EVENT_CIRC_BANDWIDTH_USED     0x001D
 #define EVENT_TRANSPORT_LAUNCHED      0x0020
 #define EVENT_HS_DESC                 0x0021
@@ -228,7 +329,7 @@ MOCK_DECL(STATIC void,
           queue_control_event_string,(uint16_t event, char *msg));
 
 void control_testing_set_global_event_mask(uint64_t mask);
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /** Helper structure: temporarily stores cell statistics for a circuit. */
 typedef struct cell_stats_t {
@@ -255,13 +356,30 @@ void format_cell_stats(char **event_string, circuit_t *circ,
                        cell_stats_t *cell_stats);
 STATIC char *get_bw_samples(void);
 
-STATIC crypto_pk_t *add_onion_helper_keyarg(const char *arg, int discard_pk,
-                                            const char **key_new_alg_out,
-                                            char **key_new_blob_out,
-                                            char **err_msg_out);
+/* ADD_ONION secret key to create an ephemeral service. The command supports
+ * multiple versions so this union stores the key and passes it to the HS
+ * subsystem depending on the requested version. */
+typedef union add_onion_secret_key_t {
+  /* Hidden service v2 secret key. */
+  crypto_pk_t *v2;
+  /* Hidden service v3 secret key. */
+  ed25519_secret_key_t *v3;
+} add_onion_secret_key_t;
+
+STATIC int add_onion_helper_keyarg(const char *arg, int discard_pk,
+                                   const char **key_new_alg_out,
+                                   char **key_new_blob_out,
+                                   add_onion_secret_key_t *decoded_key,
+                                   int *hs_version, char **err_msg_out);
+
 STATIC rend_authorized_client_t *
 add_onion_helper_clientauth(const char *arg, int *created, char **err_msg_out);
 
+STATIC int getinfo_helper_onions(
+    control_connection_t *control_conn,
+    const char *question,
+    char **answer,
+    const char **errmsg);
 STATIC void getinfo_helper_downloads_networkstatus(
     const char *flavor,
     download_status_t **dl_to_emit,
@@ -285,8 +403,15 @@ STATIC int getinfo_helper_downloads(
     control_connection_t *control_conn,
     const char *question, char **answer,
     const char **errmsg);
+STATIC int getinfo_helper_dir(
+    control_connection_t *control_conn,
+    const char *question, char **answer,
+    const char **errmsg);
+STATIC int getinfo_helper_current_time(
+    control_connection_t *control_conn,
+    const char *question, char **answer,
+    const char **errmsg);
 
-#endif
-
-#endif
+#endif /* defined(CONTROL_PRIVATE) */
 
+#endif /* !defined(TOR_CONTROL_H) */
diff --git a/src/feature/control/control_connection_st.h b/src/feature/control/control_connection_st.h
new file mode 100644
index 0000000000..177a916257
--- /dev/null
+++ b/src/feature/control/control_connection_st.h
@@ -0,0 +1,46 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef CONTROL_CONNECTION_ST_H
+#define CONTROL_CONNECTION_ST_H
+
+#include "core/or/or.h"
+#include "core/or/connection_st.h"
+
+/** Subtype of connection_t for an connection to a controller. */
+struct control_connection_t {
+  connection_t base_;
+
+  uint64_t event_mask; /**< Bitfield: which events does this controller
+                        * care about?
+                        * EVENT_MAX_ is >31, so we need a 64 bit mask */
+
+  /** True if we have sent a protocolinfo reply on this connection. */
+  unsigned int have_sent_protocolinfo:1;
+  /** True if we have received a takeownership command on this
+   * connection. */
+  unsigned int is_owning_control_connection:1;
+
+  /** List of ephemeral onion services belonging to this connection. */
+  smartlist_t *ephemeral_onion_services;
+
+  /** If we have sent an AUTHCHALLENGE reply on this connection and
+   * have not received a successful AUTHENTICATE command, points to
+   * the value which the client must send to authenticate itself;
+   * otherwise, NULL. */
+  char *safecookie_client_hash;
+
+  /** Amount of space allocated in incoming_cmd. */
+  uint32_t incoming_cmd_len;
+  /** Number of bytes currently stored in incoming_cmd. */
+  uint32_t incoming_cmd_cur_len;
+  /** A control command that we're reading from the inbuf, but which has not
+   * yet arrived completely. */
+  char *incoming_cmd;
+};
+
+#endif
+
diff --git a/src/feature/control/fmt_serverstatus.c b/src/feature/control/fmt_serverstatus.c
new file mode 100644
index 0000000000..a1ddd2119a
--- /dev/null
+++ b/src/feature/control/fmt_serverstatus.c
@@ -0,0 +1,104 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "feature/control/fmt_serverstatus.h"
+
+#include "app/config/config.h"
+#include "feature/dirauth/authmode.h"
+#include "feature/dirauth/voteflags.h"// XXXX remove
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerinfo.h"
+
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+
+/**
+ * Allocate and return a description of the status of the server <b>desc</b>,
+ * for use in a v1-style router-status line.  The server is listed
+ * as running iff <b>is_live</b> is true.
+ *
+ * This is deprecated: it's only used for controllers that want outputs in
+ * the old format.
+ */
+static char *
+list_single_server_status(const routerinfo_t *desc, int is_live)
+{
+  char buf[MAX_NICKNAME_LEN+HEX_DIGEST_LEN+4]; /* !nickname=$hexdigest\0 */
+  char *cp;
+  const node_t *node;
+
+  tor_assert(desc);
+
+  cp = buf;
+  if (!is_live) {
+    *cp++ = '!';
+  }
+  node = node_get_by_id(desc->cache_info.identity_digest);
+  if (node && node->is_valid) {
+    strlcpy(cp, desc->nickname, sizeof(buf)-(cp-buf));
+    cp += strlen(cp);
+    *cp++ = '=';
+  }
+  *cp++ = '$';
+  base16_encode(cp, HEX_DIGEST_LEN+1, desc->cache_info.identity_digest,
+                DIGEST_LEN);
+  return tor_strdup(buf);
+}
+
+/** Based on the routerinfo_ts in <b>routers</b>, allocate the
+ * contents of a v1-style router-status line, and store it in
+ * *<b>router_status_out</b>.  Return 0 on success, -1 on failure.
+ *
+ * If for_controller is true, include the routers with very old descriptors.
+ *
+ * This is deprecated: it's only used for controllers that want outputs in
+ * the old format.
+ */
+int
+list_server_status_v1(smartlist_t *routers, char **router_status_out,
+                      int for_controller)
+{
+  /* List of entries in a router-status style: An optional !, then an optional
+   * equals-suffixed nickname, then a dollar-prefixed hexdigest. */
+  smartlist_t *rs_entries;
+  time_t now = time(NULL);
+  time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
+  const or_options_t *options = get_options();
+  /* We include v2 dir auths here too, because they need to answer
+   * controllers. Eventually we'll deprecate this whole function;
+   * see also networkstatus_getinfo_by_purpose(). */
+  int authdir = authdir_mode_publishes_statuses(options);
+  tor_assert(router_status_out);
+
+  rs_entries = smartlist_new();
+
+  SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
+    const node_t *node = node_get_by_id(ri->cache_info.identity_digest);
+    tor_assert(node);
+    if (authdir) {
+      /* Update router status in routerinfo_t. */
+      dirserv_set_router_is_running(ri, now);
+    }
+    if (for_controller) {
+      char name_buf[MAX_VERBOSE_NICKNAME_LEN+2];
+      char *cp = name_buf;
+      if (!node->is_running)
+        *cp++ = '!';
+      router_get_verbose_nickname(cp, ri);
+      smartlist_add_strdup(rs_entries, name_buf);
+    } else if (ri->cache_info.published_on >= cutoff) {
+      smartlist_add(rs_entries, list_single_server_status(ri,
+                                                          node->is_running));
+    }
+  } SMARTLIST_FOREACH_END(ri);
+
+  *router_status_out = smartlist_join_strings(rs_entries, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(rs_entries, char *, cp, tor_free(cp));
+  smartlist_free(rs_entries);
+
+  return 0;
+}
diff --git a/src/feature/control/fmt_serverstatus.h b/src/feature/control/fmt_serverstatus.h
new file mode 100644
index 0000000000..4b95e5b59f
--- /dev/null
+++ b/src/feature/control/fmt_serverstatus.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file fmt_serverstatus.h
+ * \brief Header file for fmt_serverstatus.c.
+ **/
+
+#ifndef TOR_FMT_SERVERSTATUS_H
+#define TOR_FMT_SERVERSTATUS_H
+
+int list_server_status_v1(smartlist_t *routers, char **router_status_out,
+                          int for_controller);
+
+#endif
diff --git a/src/feature/control/getinfo_geoip.c b/src/feature/control/getinfo_geoip.c
new file mode 100644
index 0000000000..d188725fa3
--- /dev/null
+++ b/src/feature/control/getinfo_geoip.c
@@ -0,0 +1,45 @@
+
+#include "core/or/or.h"
+#include "core/mainloop/connection.h"
+#include "feature/control/control.h"
+#include "feature/control/getinfo_geoip.h"
+#include "lib/geoip/geoip.h"
+
+/** Helper used to implement GETINFO ip-to-country/... controller command. */
+int
+getinfo_helper_geoip(control_connection_t *control_conn,
+                     const char *question, char **answer,
+                     const char **errmsg)
+{
+  (void)control_conn;
+  if (!strcmpstart(question, "ip-to-country/")) {
+    int c;
+    sa_family_t family;
+    tor_addr_t addr;
+    question += strlen("ip-to-country/");
+
+    if (!strcmp(question, "ipv4-available") ||
+        !strcmp(question, "ipv6-available")) {
+      family = !strcmp(question, "ipv4-available") ? AF_INET : AF_INET6;
+      const int available = geoip_is_loaded(family);
+      tor_asprintf(answer, "%d", !! available);
+      return 0;
+    }
+
+    family = tor_addr_parse(&addr, question);
+    if (family != AF_INET && family != AF_INET6) {
+      *errmsg = "Invalid address family";
+      return -1;
+    }
+    if (!geoip_is_loaded(family)) {
+      *errmsg = "GeoIP data not loaded";
+      return -1;
+    }
+    if (family == AF_INET)
+      c = geoip_get_country_by_ipv4(tor_addr_to_ipv4h(&addr));
+    else                      /* AF_INET6 */
+      c = geoip_get_country_by_ipv6(tor_addr_to_in6(&addr));
+    *answer = tor_strdup(geoip_get_country_name(c));
+  }
+  return 0;
+}
diff --git a/src/feature/control/getinfo_geoip.h b/src/feature/control/getinfo_geoip.h
new file mode 100644
index 0000000000..fe22137859
--- /dev/null
+++ b/src/feature/control/getinfo_geoip.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_GETINFO_GEOIP_H
+#define TOR_GETINFO_GEOIP_H
+
+int getinfo_helper_geoip(control_connection_t *control_conn,
+                     const char *question, char **answer,
+                     const char **errmsg);
+
+#endif
diff --git a/src/feature/dirauth/authmode.c b/src/feature/dirauth/authmode.c
new file mode 100644
index 0000000000..29fcc6d1a9
--- /dev/null
+++ b/src/feature/dirauth/authmode.c
@@ -0,0 +1,70 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file authmode.c
+ * \brief What kind of directory authority are we?
+ *
+ * If we're not an authority, these functions are all replaced with 0 in
+ * authmode.h.
+ **/
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "feature/dirauth/authmode.h"
+
+#include "feature/nodelist/routerinfo_st.h"
+
+/** Return true iff we believe ourselves to be an authoritative
+ * directory server.
+ */
+int
+authdir_mode(const or_options_t *options)
+{
+  return options->AuthoritativeDir != 0;
+}
+/** Return true iff we are an authoritative directory server that is
+ * authoritative about receiving and serving descriptors of type
+ * <b>purpose</b> on its dirport.
+ */
+int
+authdir_mode_handles_descs(const or_options_t *options, int purpose)
+{
+  if (BUG(purpose < 0)) /* Deprecated. */
+    return authdir_mode(options);
+  else if (purpose == ROUTER_PURPOSE_GENERAL)
+    return authdir_mode_v3(options);
+  else if (purpose == ROUTER_PURPOSE_BRIDGE)
+    return authdir_mode_bridge(options);
+  else
+    return 0;
+}
+/** Return true iff we are an authoritative directory server that
+ * publishes its own network statuses.
+ */
+int
+authdir_mode_publishes_statuses(const or_options_t *options)
+{
+  if (authdir_mode_bridge(options))
+    return 0;
+  return authdir_mode(options);
+}
+/** Return true iff we are an authoritative directory server that
+ * tests reachability of the descriptors it learns about.
+ */
+int
+authdir_mode_tests_reachability(const or_options_t *options)
+{
+  return authdir_mode(options);
+}
+/** Return true iff we believe ourselves to be a bridge authoritative
+ * directory server.
+ */
+int
+authdir_mode_bridge(const or_options_t *options)
+{
+  return authdir_mode(options) && options->BridgeAuthoritativeDir != 0;
+}
diff --git a/src/feature/dirauth/authmode.h b/src/feature/dirauth/authmode.h
new file mode 100644
index 0000000000..876a1f947b
--- /dev/null
+++ b/src/feature/dirauth/authmode.h
@@ -0,0 +1,46 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file authmode.h
+ * \brief Header file for directory authority mode.
+ **/
+
+#ifndef TOR_DIRAUTH_MODE_H
+#define TOR_DIRAUTH_MODE_H
+
+#include "feature/relay/router.h"
+
+#ifdef HAVE_MODULE_DIRAUTH
+
+int authdir_mode(const or_options_t *options);
+int authdir_mode_handles_descs(const or_options_t *options, int purpose);
+int authdir_mode_publishes_statuses(const or_options_t *options);
+int authdir_mode_tests_reachability(const or_options_t *options);
+int authdir_mode_bridge(const or_options_t *options);
+
+/* Return true iff we believe ourselves to be a v3 authoritative directory
+ * server. */
+static inline int
+authdir_mode_v3(const or_options_t *options)
+{
+  return authdir_mode(options) && options->V3AuthoritativeDir != 0;
+}
+
+#define have_module_dirauth() (1)
+
+#else /* HAVE_MODULE_DIRAUTH */
+
+#define authdir_mode(options) (((void)(options)),0)
+#define authdir_mode_handles_descs(options,purpose) \
+  (((void)(options)),((void)(purpose)),0)
+#define authdir_mode_publishes_statuses(options) (((void)(options)),0)
+#define authdir_mode_tests_reachability(options) (((void)(options)),0)
+#define authdir_mode_bridge(options) (((void)(options)),0)
+#define authdir_mode_v3(options) (((void)(options)),0)
+
+#define have_module_dirauth() (0)
+
+#endif /* HAVE_MODULE_DIRAUTH */
+
+#endif /* TOR_MODE_H */
diff --git a/src/feature/dirauth/bwauth.c b/src/feature/dirauth/bwauth.c
new file mode 100644
index 0000000000..12f9399e9f
--- /dev/null
+++ b/src/feature/dirauth/bwauth.c
@@ -0,0 +1,459 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file bwauth.c
+ * \brief Code to read and apply bandwidth authority data.
+ **/
+
+#define BWAUTH_PRIVATE
+#include "core/or/or.h"
+#include "feature/dirauth/bwauth.h"
+
+#include "app/config/config.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/dirparse/ns_parse.h"
+
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/vote_routerstatus_st.h"
+
+#include "lib/encoding/keyval.h"
+
+/** Total number of routers with measured bandwidth; this is set by
+ * dirserv_count_measured_bs() before the loop in
+ * dirserv_generate_networkstatus_vote_obj() and checked by
+ * dirserv_get_credible_bandwidth() and
+ * dirserv_compute_performance_thresholds() */
+static int routers_with_measured_bw = 0;
+
+/** Look through the routerlist, and using the measured bandwidth cache count
+ * how many measured bandwidths we know.  This is used to decide whether we
+ * ever trust advertised bandwidths for purposes of assigning flags. */
+void
+dirserv_count_measured_bws(const smartlist_t *routers)
+{
+  /* Initialize this first */
+  routers_with_measured_bw = 0;
+
+  /* Iterate over the routerlist and count measured bandwidths */
+  SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) {
+    /* Check if we know a measured bandwidth for this one */
+    if (dirserv_has_measured_bw(ri->cache_info.identity_digest)) {
+      ++routers_with_measured_bw;
+    }
+  } SMARTLIST_FOREACH_END(ri);
+}
+
+/** Return the last-computed result from dirserv_count_mesured_bws(). */
+int
+dirserv_get_last_n_measured_bws(void)
+{
+  return routers_with_measured_bw;
+}
+
+/** Measured bandwidth cache entry */
+typedef struct mbw_cache_entry_s {
+  long mbw_kb;
+  time_t as_of;
+} mbw_cache_entry_t;
+
+/** Measured bandwidth cache - keys are identity_digests, values are
+ * mbw_cache_entry_t *. */
+static digestmap_t *mbw_cache = NULL;
+
+/** Store a measured bandwidth cache entry when reading the measured
+ * bandwidths file. */
+STATIC void
+dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line,
+                          time_t as_of)
+{
+  mbw_cache_entry_t *e = NULL;
+
+  tor_assert(parsed_line);
+
+  /* Allocate a cache if we need */
+  if (!mbw_cache) mbw_cache = digestmap_new();
+
+  /* Check if we have an existing entry */
+  e = digestmap_get(mbw_cache, parsed_line->node_id);
+  /* If we do, we can re-use it */
+  if (e) {
+    /* Check that we really are newer, and update */
+    if (as_of > e->as_of) {
+      e->mbw_kb = parsed_line->bw_kb;
+      e->as_of = as_of;
+    }
+  } else {
+    /* We'll have to insert a new entry */
+    e = tor_malloc(sizeof(*e));
+    e->mbw_kb = parsed_line->bw_kb;
+    e->as_of = as_of;
+    digestmap_set(mbw_cache, parsed_line->node_id, e);
+  }
+}
+
+/** Clear and free the measured bandwidth cache */
+void
+dirserv_clear_measured_bw_cache(void)
+{
+  if (mbw_cache) {
+    /* Free the map and all entries */
+    digestmap_free(mbw_cache, tor_free_);
+    mbw_cache = NULL;
+  }
+}
+
+/** Scan the measured bandwidth cache and remove expired entries */
+STATIC void
+dirserv_expire_measured_bw_cache(time_t now)
+{
+
+  if (mbw_cache) {
+    /* Iterate through the cache and check each entry */
+    DIGESTMAP_FOREACH_MODIFY(mbw_cache, k, mbw_cache_entry_t *, e) {
+      if (now > e->as_of + MAX_MEASUREMENT_AGE) {
+        tor_free(e);
+        MAP_DEL_CURRENT(k);
+      }
+    } DIGESTMAP_FOREACH_END;
+
+    /* Check if we cleared the whole thing and free if so */
+    if (digestmap_size(mbw_cache) == 0) {
+      digestmap_free(mbw_cache, tor_free_);
+      mbw_cache = 0;
+    }
+  }
+}
+
+/** Query the cache by identity digest, return value indicates whether
+ * we found it. The bw_out and as_of_out pointers receive the cached
+ * bandwidth value and the time it was cached if not NULL. */
+int
+dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_kb_out,
+                                   time_t *as_of_out)
+{
+  mbw_cache_entry_t *v = NULL;
+  int rv = 0;
+
+  if (mbw_cache && node_id) {
+    v = digestmap_get(mbw_cache, node_id);
+    if (v) {
+      /* Found something */
+      rv = 1;
+      if (bw_kb_out) *bw_kb_out = v->mbw_kb;
+      if (as_of_out) *as_of_out = v->as_of;
+    }
+  }
+
+  return rv;
+}
+
+/** Predicate wrapper for dirserv_query_measured_bw_cache() */
+int
+dirserv_has_measured_bw(const char *node_id)
+{
+  return dirserv_query_measured_bw_cache_kb(node_id, NULL, NULL);
+}
+
+/** Get the current size of the measured bandwidth cache */
+int
+dirserv_get_measured_bw_cache_size(void)
+{
+  if (mbw_cache) return digestmap_size(mbw_cache);
+  else return 0;
+}
+
+/** Return the bandwidth we believe for assigning flags; prefer measured
+ * over advertised, and if we have above a threshold quantity of measured
+ * bandwidths, we don't want to ever give flags to unmeasured routers, so
+ * return 0. */
+uint32_t
+dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri)
+{
+  int threshold;
+  uint32_t bw_kb = 0;
+  long mbw_kb;
+
+  tor_assert(ri);
+  /* Check if we have a measured bandwidth, and check the threshold if not */
+  if (!(dirserv_query_measured_bw_cache_kb(ri->cache_info.identity_digest,
+                                       &mbw_kb, NULL))) {
+    threshold = get_options()->MinMeasuredBWsForAuthToIgnoreAdvertised;
+    if (routers_with_measured_bw > threshold) {
+      /* Return zero for unmeasured bandwidth if we are above threshold */
+      bw_kb = 0;
+    } else {
+      /* Return an advertised bandwidth otherwise */
+      bw_kb = router_get_advertised_bandwidth_capped(ri) / 1000;
+    }
+  } else {
+    /* We have the measured bandwidth in mbw */
+    bw_kb = (uint32_t)mbw_kb;
+  }
+
+  return bw_kb;
+}
+
+/**
+ * Read the measured bandwidth list file, apply it to the list of
+ * vote_routerstatus_t and store all the headers in <b>bw_file_headers</b>.
+ * Returns -1 on error, 0 otherwise.
+ */
+int
+dirserv_read_measured_bandwidths(const char *from_file,
+                                 smartlist_t *routerstatuses,
+                                 smartlist_t *bw_file_headers)
+{
+  FILE *fp = tor_fopen_cloexec(from_file, "r");
+  int applied_lines = 0;
+  time_t file_time, now;
+  int ok;
+   /* This flag will be 1 only when the first successful bw measurement line
+   * has been encountered, so that measured_bw_line_parse don't give warnings
+   * if there are additional header lines, as introduced in Bandwidth List spec
+   * version 1.1.0 */
+  int line_is_after_headers = 0;
+  int rv = -1;
+  char *line = NULL;
+  size_t n = 0;
+
+  /* Initialise line, so that we can't possibly run off the end. */
+
+  if (fp == NULL) {
+    log_warn(LD_CONFIG, "Can't open bandwidth file at configured location: %s",
+             from_file);
+    goto err;
+  }
+
+  /* If fgets fails, line is either unmodified, or indeterminate. */
+  if (tor_getline(&line,&n,fp) <= 0) {
+    log_warn(LD_DIRSERV, "Empty bandwidth file");
+    goto err;
+  }
+
+  if (!strlen(line) || line[strlen(line)-1] != '\n') {
+    log_warn(LD_DIRSERV, "Long or truncated time in bandwidth file: %s",
+             escaped(line));
+    goto err;
+  }
+
+  line[strlen(line)-1] = '\0';
+  file_time = (time_t)tor_parse_ulong(line, 10, 0, ULONG_MAX, &ok, NULL);
+  if (!ok) {
+    log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s",
+             escaped(line));
+    goto err;
+  }
+
+  now = time(NULL);
+  if ((now - file_time) > MAX_MEASUREMENT_AGE) {
+    log_warn(LD_DIRSERV, "Bandwidth measurement file stale. Age: %u",
+             (unsigned)(time(NULL) - file_time));
+    goto err;
+  }
+
+  /* If timestamp was correct and bw_file_headers is not NULL,
+   * add timestamp to bw_file_headers */
+  if (bw_file_headers)
+    smartlist_add_asprintf(bw_file_headers, "timestamp=%lu",
+                           (unsigned long)file_time);
+
+  if (routerstatuses)
+    smartlist_sort(routerstatuses, compare_vote_routerstatus_entries);
+
+  while (!feof(fp)) {
+    measured_bw_line_t parsed_line;
+    if (tor_getline(&line, &n, fp) >= 0) {
+      if (measured_bw_line_parse(&parsed_line, line,
+                                 line_is_after_headers) != -1) {
+        /* This condition will be true when the first complete valid bw line
+         * has been encountered, which means the end of the header lines. */
+        line_is_after_headers = 1;
+        /* Also cache the line for dirserv_get_bandwidth_for_router() */
+        dirserv_cache_measured_bw(&parsed_line, file_time);
+        if (measured_bw_line_apply(&parsed_line, routerstatuses) > 0)
+          applied_lines++;
+      /* if the terminator is found, it is the end of header lines, set the
+       * flag but do not store anything */
+      } else if (strcmp(line, BW_FILE_HEADERS_TERMINATOR) == 0) {
+        line_is_after_headers = 1;
+      /* if the line was not a correct relay line nor the terminator and
+       * the end of the header lines has not been detected yet
+       * and it is key_value and bw_file_headers did not reach the maximum
+       * number of headers,
+       * then assume this line is a header and add it to bw_file_headers */
+      } else if (bw_file_headers &&
+              (line_is_after_headers == 0) &&
+              string_is_key_value(LOG_DEBUG, line) &&
+              !strchr(line, ' ') &&
+              (smartlist_len(bw_file_headers)
+               < MAX_BW_FILE_HEADER_COUNT_IN_VOTE)) {
+        line[strlen(line)-1] = '\0';
+        smartlist_add_strdup(bw_file_headers, line);
+      };
+    }
+  }
+
+  /* Now would be a nice time to clean the cache, too */
+  dirserv_expire_measured_bw_cache(now);
+
+  log_info(LD_DIRSERV,
+           "Bandwidth measurement file successfully read. "
+           "Applied %d measurements.", applied_lines);
+  rv = 0;
+
+ err:
+  if (line) {
+    // we need to raw_free this buffer because we got it from tor_getdelim()
+    raw_free(line);
+  }
+  if (fp)
+    fclose(fp);
+  return rv;
+}
+
+/**
+ * Helper function to parse out a line in the measured bandwidth file
+ * into a measured_bw_line_t output structure.
+ *
+ * If <b>line_is_after_headers</b> is true, then if we encounter an incomplete
+ * bw line, return -1 and warn, since we are after the headers and we should
+ * only parse bw lines. Return 0 otherwise.
+ *
+ * If <b>line_is_after_headers</b> is false then it means that we are not past
+ * the header block yet. If we encounter an incomplete bw line, return -1 but
+ * don't warn since there could be additional header lines coming. If we
+ * encounter a proper bw line, return 0 (and we got past the headers).
+ */
+STATIC int
+measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line,
+                       int line_is_after_headers)
+{
+  char *line = tor_strdup(orig_line);
+  char *cp = line;
+  int got_bw = 0;
+  int got_node_id = 0;
+  char *strtok_state; /* lame sauce d'jour */
+
+  if (strlen(line) == 0) {
+    log_warn(LD_DIRSERV, "Empty line in bandwidth file");
+    tor_free(line);
+    return -1;
+  }
+
+  /* Remove end of line character, so that is not part of the token */
+  if (line[strlen(line) - 1] == '\n') {
+    line[strlen(line) - 1] = '\0';
+  }
+
+  cp = tor_strtok_r(cp, " \t", &strtok_state);
+
+  if (!cp) {
+    log_warn(LD_DIRSERV, "Invalid line in bandwidth file: %s",
+             escaped(orig_line));
+    tor_free(line);
+    return -1;
+  }
+
+  if (orig_line[strlen(orig_line)-1] != '\n') {
+    log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s",
+             escaped(orig_line));
+    tor_free(line);
+    return -1;
+  }
+
+  do {
+    // If the line contains vote=0, ignore it.
+    if (strcmpstart(cp, "vote=0") == 0) {
+      log_debug(LD_DIRSERV, "Ignoring bandwidth file line that contains "
+                "vote=0: %s",escaped(orig_line));
+      tor_free(line);
+      return -1;
+    } else if (strcmpstart(cp, "bw=") == 0) {
+      int parse_ok = 0;
+      char *endptr;
+      if (got_bw) {
+        log_warn(LD_DIRSERV, "Double bw= in bandwidth file line: %s",
+                 escaped(orig_line));
+        tor_free(line);
+        return -1;
+      }
+      cp+=strlen("bw=");
+
+      out->bw_kb = tor_parse_long(cp, 10, 0, LONG_MAX, &parse_ok, &endptr);
+      if (!parse_ok || (*endptr && !TOR_ISSPACE(*endptr))) {
+        log_warn(LD_DIRSERV, "Invalid bandwidth in bandwidth file line: %s",
+                 escaped(orig_line));
+        tor_free(line);
+        return -1;
+      }
+      got_bw=1;
+    } else if (strcmpstart(cp, "node_id=$") == 0) {
+      if (got_node_id) {
+        log_warn(LD_DIRSERV, "Double node_id= in bandwidth file line: %s",
+                 escaped(orig_line));
+        tor_free(line);
+        return -1;
+      }
+      cp+=strlen("node_id=$");
+
+      if (strlen(cp) != HEX_DIGEST_LEN ||
+          base16_decode(out->node_id, DIGEST_LEN,
+                        cp, HEX_DIGEST_LEN) != DIGEST_LEN) {
+        log_warn(LD_DIRSERV, "Invalid node_id in bandwidth file line: %s",
+                 escaped(orig_line));
+        tor_free(line);
+        return -1;
+      }
+      strlcpy(out->node_hex, cp, sizeof(out->node_hex));
+      got_node_id=1;
+    }
+  } while ((cp = tor_strtok_r(NULL, " \t", &strtok_state)));
+
+  if (got_bw && got_node_id) {
+    tor_free(line);
+    return 0;
+  } else if (line_is_after_headers == 0) {
+    /* There could be additional header lines, therefore do not give warnings
+     * but returns -1 since it's not a complete bw line. */
+    log_debug(LD_DIRSERV, "Missing bw or node_id in bandwidth file line: %s",
+             escaped(orig_line));
+    tor_free(line);
+    return -1;
+  } else {
+    log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s",
+             escaped(orig_line));
+    tor_free(line);
+    return -1;
+  }
+}
+
+/**
+ * Helper function to apply a parsed measurement line to a list
+ * of bandwidth statuses. Returns true if a line is found,
+ * false otherwise.
+ */
+STATIC int
+measured_bw_line_apply(measured_bw_line_t *parsed_line,
+                       smartlist_t *routerstatuses)
+{
+  vote_routerstatus_t *rs = NULL;
+  if (!routerstatuses)
+    return 0;
+
+  rs = smartlist_bsearch(routerstatuses, parsed_line->node_id,
+                         compare_digest_to_vote_routerstatus_entry);
+
+  if (rs) {
+    rs->has_measured_bw = 1;
+    rs->measured_bw_kb = (uint32_t)parsed_line->bw_kb;
+  } else {
+    log_info(LD_DIRSERV, "Node ID %s not found in routerstatus list",
+             parsed_line->node_hex);
+  }
+
+  return rs != NULL;
+}
diff --git a/src/feature/dirauth/bwauth.h b/src/feature/dirauth/bwauth.h
new file mode 100644
index 0000000000..4507728458
--- /dev/null
+++ b/src/feature/dirauth/bwauth.h
@@ -0,0 +1,58 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file bwauth.h
+ * \brief Header file for bwauth.c
+ **/
+
+#ifndef TOR_BWAUTH_H
+#define TOR_BWAUTH_H
+
+/** Maximum allowable length of bandwidth headers in a bandwidth file */
+#define MAX_BW_FILE_HEADER_COUNT_IN_VOTE 50
+
+/** Terminatore that separates bandwidth file headers from bandwidth file
+ * relay lines */
+#define BW_FILE_HEADERS_TERMINATOR "=====\n"
+
+int dirserv_read_measured_bandwidths(const char *from_file,
+                                     smartlist_t *routerstatuses,
+                                     smartlist_t *bw_file_headers);
+
+int dirserv_query_measured_bw_cache_kb(const char *node_id,
+                                       long *bw_out,
+                                       time_t *as_of_out);
+void dirserv_clear_measured_bw_cache(void);
+int dirserv_has_measured_bw(const char *node_id);
+int dirserv_get_measured_bw_cache_size(void);
+void dirserv_count_measured_bws(const smartlist_t *routers);
+int dirserv_get_last_n_measured_bws(void);
+
+uint32_t dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri);
+
+#ifdef BWAUTH_PRIVATE
+typedef struct measured_bw_line_t {
+  char node_id[DIGEST_LEN];
+  char node_hex[MAX_HEX_NICKNAME_LEN+1];
+  long int bw_kb;
+} measured_bw_line_t;
+
+/* Put the MAX_MEASUREMENT_AGE #define here so unit tests can see it */
+#define MAX_MEASUREMENT_AGE (3*24*60*60) /* 3 days */
+
+STATIC int measured_bw_line_parse(measured_bw_line_t *out, const char *line,
+                                  int line_is_after_headers);
+
+STATIC int measured_bw_line_apply(measured_bw_line_t *parsed_line,
+                           smartlist_t *routerstatuses);
+
+STATIC void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line,
+                               time_t as_of);
+STATIC void dirserv_expire_measured_bw_cache(time_t now);
+#endif /* defined(BWAUTH_PRIVATE) */
+
+#endif
diff --git a/src/or/dircollate.c b/src/feature/dirauth/dircollate.c
index 033a7afe0f..7992e3a85f 100644
--- a/src/or/dircollate.c
+++ b/src/feature/dirauth/dircollate.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,10 +22,12 @@
  */
 
 #define DIRCOLLATE_PRIVATE
-#include "dircollate.h"
-#include "dirvote.h"
+#include "feature/dirauth/dircollate.h"
+#include "feature/dirauth/dirvote.h"
+
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/vote_routerstatus_st.h"
 
-static void dircollator_collate_by_rsa(dircollator_t *dc);
 static void dircollator_collate_by_ed25519(dircollator_t *dc);
 
 /** Hashtable entry mapping a pair of digests (actually an ed25519 key and an
@@ -41,9 +43,12 @@ typedef struct ddmap_entry_s {
   vote_routerstatus_t *vrs_lst[FLEXIBLE_ARRAY_MEMBER];
 } ddmap_entry_t;
 
+#define ddmap_entry_free(e) \
+  FREE_AND_NULL(ddmap_entry_t, ddmap_entry_free_, (e))
+
 /** Release all storage held by e. */
 static void
-ddmap_entry_free(ddmap_entry_t *e)
+ddmap_entry_free_(ddmap_entry_t *e)
 {
   tor_free(e);
 }
@@ -53,7 +58,7 @@ ddmap_entry_free(ddmap_entry_t *e)
 static ddmap_entry_t *
 ddmap_entry_new(int n_votes)
 {
-  return tor_malloc_zero(STRUCT_OFFSET(ddmap_entry_t, vrs_lst) +
+  return tor_malloc_zero(offsetof(ddmap_entry_t, vrs_lst) +
                          sizeof(vote_routerstatus_t *) * n_votes);
 }
 
@@ -158,7 +163,7 @@ dircollator_new(int n_votes, int n_authorities)
 
 /** Release all storage held by <b>dc</b>. */
 void
-dircollator_free(dircollator_t *dc)
+dircollator_free_(dircollator_t *dc)
 {
   if (!dc)
     return;
@@ -205,49 +210,18 @@ dircollator_add_vote(dircollator_t *dc, networkstatus_t *v)
 void
 dircollator_collate(dircollator_t *dc, int consensus_method)
 {
+  (void) consensus_method;
+
   tor_assert(!dc->is_collated);
   dc->all_rsa_sha1_lst = smartlist_new();
 
-  if (consensus_method < MIN_METHOD_FOR_ED25519_ID_VOTING)
-    dircollator_collate_by_rsa(dc);
-  else
-    dircollator_collate_by_ed25519(dc);
+  dircollator_collate_by_ed25519(dc);
 
   smartlist_sort_digests(dc->all_rsa_sha1_lst);
   dc->is_collated = 1;
 }
 
 /**
- * Collation function for RSA-only consensuses: collate the votes for each
- * entry in <b>dc</b> by their RSA keys.
- *
- * The rule is:
- *    If an RSA identity key is listed by more than half of the authorities,
- *    include that identity, and treat all descriptors with that RSA identity
- *    as describing the same router.
- */
-static void
-dircollator_collate_by_rsa(dircollator_t *dc)
-{
-  const int total_authorities = dc->n_authorities;
-
-  DIGESTMAP_FOREACH(dc->by_rsa_sha1, k, vote_routerstatus_t **, vrs_lst) {
-    int n = 0, i;
-    for (i = 0; i < dc->n_votes; ++i) {
-      if (vrs_lst[i] != NULL)
-        ++n;
-    }
-
-    if (n <= total_authorities / 2)
-      continue;
-
-    smartlist_add(dc->all_rsa_sha1_lst, (char *)k);
-  } DIGESTMAP_FOREACH_END;
-
-  dc->by_collated_rsa_sha1 = dc->by_rsa_sha1;
-}
-
-/**
  * Collation function for ed25519 consensuses: collate the votes for each
  * entry in <b>dc</b> by ed25519 key and by RSA key.
  *
diff --git a/src/or/dircollate.h b/src/feature/dirauth/dircollate.h
index 358c730cbb..754a094817 100644
--- a/src/or/dircollate.h
+++ b/src/feature/dirauth/dircollate.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,13 +12,15 @@
 #ifndef TOR_DIRCOLLATE_H
 #define TOR_DIRCOLLATE_H
 
-#include "testsupport.h"
-#include "or.h"
+#include "lib/testsupport/testsupport.h"
+#include "core/or/or.h"
 
 typedef struct dircollator_s dircollator_t;
 
 dircollator_t *dircollator_new(int n_votes, int n_authorities);
-void dircollator_free(dircollator_t *obj);
+void dircollator_free_(dircollator_t *obj);
+#define dircollator_free(c) \
+  FREE_AND_NULL(dircollator_t, dircollator_free_, (c))
 void dircollator_add_vote(dircollator_t *dc, networkstatus_t *v);
 
 void dircollator_collate(dircollator_t *dc, int consensus_method);
@@ -62,7 +64,7 @@ struct dircollator_s {
    * identity digests .*/
   smartlist_t *all_rsa_sha1_lst;
 };
-#endif
+#endif /* defined(DIRCOLLATE_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_DIRCOLLATE_H) */
 
diff --git a/src/or/dirvote.c b/src/feature/dirauth/dirvote.c
index e6a745612e..af8b3dc207 100644
--- a/src/or/dirvote.c
+++ b/src/feature/dirauth/dirvote.c
@@ -1,31 +1,101 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define DIRVOTE_PRIVATE
-#include "or.h"
-#include "config.h"
-#include "dircollate.h"
-#include "directory.h"
-#include "dirserv.h"
-#include "dirvote.h"
-#include "microdesc.h"
-#include "networkstatus.h"
-#include "policies.h"
-#include "protover.h"
-#include "rephist.h"
-#include "router.h"
-#include "routerkeys.h"
-#include "routerlist.h"
-#include "routerparse.h"
-#include "entrynodes.h" /* needed for guardfraction methods */
-#include "torcert.h"
-#include "shared_random_state.h"
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/or/policies.h"
+#include "core/or/protover.h"
+#include "core/or/tor_version_st.h"
+#include "core/or/versions.h"
+#include "feature/dirauth/bwauth.h"
+#include "feature/dirauth/dircollate.h"
+#include "feature/dirauth/dsigs_parse.h"
+#include "feature/dirauth/guardfraction.h"
+#include "feature/dirauth/recommend_pkg.h"
+#include "feature/dirauth/voteflags.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirparse/microdesc_parse.h"
+#include "feature/dirparse/ns_parse.h"
+#include "feature/dirparse/parsecommon.h"
+#include "feature/dirparse/signing.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/fmt_routerstatus.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routerkeys.h"
+#include "feature/stats/rephist.h"
+#include "feature/client/entrynodes.h" /* needed for guardfraction methods */
+#include "feature/nodelist/torcert.h"
+#include "feature/dircommon/voting_schedule.h"
+
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/authmode.h"
+#include "feature/dirauth/shared_random_state.h"
+
+#include "feature/nodelist/authority_cert_st.h"
+#include "feature/dircache/cached_dir_st.h"
+#include "feature/dirclient/dir_server_st.h"
+#include "feature/nodelist/document_signature_st.h"
+#include "feature/nodelist/microdesc_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/networkstatus_voter_info_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/dirauth/ns_detached_signatures_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerlist_st.h"
+#include "feature/dirauth/vote_microdesc_hash_st.h"
+#include "feature/nodelist/vote_routerstatus_st.h"
+#include "feature/dircommon/vote_timing_st.h"
+
+#include "lib/container/order.h"
+#include "lib/encoding/confline.h"
+#include "lib/crypt_ops/crypto_format.h"
 
 /**
  * \file dirvote.c
  * \brief Functions to compute directory consensus, and schedule voting.
+ *
+ * This module is the center of the consensus-voting based directory
+ * authority system.  With this system, a set of authorities first
+ * publish vote based on their opinions of the network, and then compute
+ * a consensus from those votes.  Each authority signs the consensus,
+ * and clients trust the consensus if enough known authorities have
+ * signed it.
+ *
+ * The code in this module is only invoked on directory authorities.  It's
+ * responsible for:
+ *
+ * <ul>
+ *   <li>Generating this authority's vote networkstatus, based on the
+ *       authority's view of the network as represented in dirserv.c
+ *   <li>Formatting the vote networkstatus objects.
+ *   <li>Generating the microdescriptors that correspond to our own
+ *       vote.
+ *   <li>Sending votes to all the other authorities.
+ *   <li>Trying to fetch missing votes from other authorities.
+ *   <li>Computing the consensus from a set of votes, as well as
+ *       a "detached signature" object for other authorities to fetch.
+ *   <li>Collecting other authorities' signatures on the same consensus,
+ *       until there are enough.
+ *   <li>Publishing the consensus to the reset of the directory system.
+ *   <li>Scheduling all of the above operations.
+ * </ul>
+ *
+ * The main entry points are in dirvote_act(), which handles scheduled
+ * actions; and dirvote_add_vote() and dirvote_add_signatures(), which
+ * handle uploaded and downloaded votes and signatures.
+ *
+ * (See dir-spec.txt from torspec.git for a complete specification of
+ * the directory protocol and voting algorithms.)
  **/
 
 /** A consensus that we have built and are appending signatures to.  Once it's
@@ -59,6 +129,30 @@ static int dirvote_compute_consensuses(void);
 static int dirvote_publish_consensus(void);
 
 /* =====
+ * Certificate functions
+ * ===== */
+
+/** Allocate and return a new authority_cert_t with the same contents as
+ * <b>cert</b>. */
+STATIC authority_cert_t *
+authority_cert_dup(authority_cert_t *cert)
+{
+  authority_cert_t *out = tor_malloc(sizeof(authority_cert_t));
+  tor_assert(cert);
+
+  memcpy(out, cert, sizeof(authority_cert_t));
+  /* Now copy pointed-to things. */
+  out->cache_info.signed_descriptor_body =
+    tor_strndup(cert->cache_info.signed_descriptor_body,
+                cert->cache_info.signed_descriptor_len);
+  out->cache_info.saved_location = SAVED_NOWHERE;
+  out->identity_key = crypto_pk_dup_key(cert->identity_key);
+  out->signing_key = crypto_pk_dup_key(cert->signing_key);
+
+  return out;
+}
+
+/* =====
  * Voting
  * =====*/
 
@@ -173,6 +267,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
     /* XXXX Abstraction violation: should be pulling a field out of v3_ns.*/
     char *flag_thresholds = dirserv_get_flag_thresholds_line();
     char *params;
+    char *bw_headers_line = NULL;
     authority_cert_t *cert = v3_ns->cert;
     char *methods =
       make_consensus_method_list(MIN_SUPPORTED_CONSENSUS_METHOD,
@@ -186,8 +281,32 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
       params = smartlist_join_strings(v3_ns->net_params, " ", 0, NULL);
     else
       params = tor_strdup("");
-
     tor_assert(cert);
+
+    /* v3_ns->bw_file_headers is only set when V3BandwidthsFile is
+     * configured */
+    if (v3_ns->bw_file_headers) {
+      char *bw_file_headers = NULL;
+      /* If there are too many headers, leave the header string NULL */
+      if (! BUG(smartlist_len(v3_ns->bw_file_headers)
+                > MAX_BW_FILE_HEADER_COUNT_IN_VOTE)) {
+        bw_file_headers = smartlist_join_strings(v3_ns->bw_file_headers, " ",
+                                                 0, NULL);
+        if (BUG(strlen(bw_file_headers) > MAX_BW_FILE_HEADERS_LINE_LEN)) {
+          /* Free and set to NULL, because the line was too long */
+          tor_free(bw_file_headers);
+        }
+      }
+      if (!bw_file_headers) {
+          /* If parsing failed, add a bandwidth header line with no entries */
+          bw_file_headers = tor_strdup("");
+      }
+      /* At this point, the line will always be present */
+      bw_headers_line = format_line_if_present("bandwidth-file-headers",
+                                               bw_file_headers);
+      tor_free(bw_file_headers);
+    }
+
     smartlist_add_asprintf(chunks,
                  "network-status-version 3\n"
                  "vote-status %s\n"
@@ -203,9 +322,11 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
                  "known-flags %s\n"
                  "flag-thresholds %s\n"
                  "params %s\n"
+                 "%s" /* bandwidth file headers */
                  "dir-source %s %s %s %s %d %d\n"
                  "contact %s\n"
-                 "%s", /* shared randomness information */
+                 "%s" /* shared randomness information */
+                 ,
                  v3_ns->type == NS_TYPE_VOTE ? "vote" : "opinion",
                  methods,
                  published, va, fu, vu,
@@ -217,6 +338,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
                  flags,
                  flag_thresholds,
                  params,
+                 bw_headers_line ? bw_headers_line : "",
                  voter->nickname, fingerprint, voter->address,
                  fmt_addr32(addr), voter->dir_port, voter->or_port,
                  voter->contact,
@@ -228,6 +350,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
     tor_free(flag_thresholds);
     tor_free(methods);
     tor_free(shared_random_vote_str);
+    tor_free(bw_headers_line);
 
     if (!tor_digest_is_zero(voter->legacy_id_digest)) {
       char fpbuf[HEX_DIGEST_LEN+1];
@@ -245,16 +368,18 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
     vote_microdesc_hash_t *h;
     rsf = routerstatus_format_entry(&vrs->status,
                                     vrs->version, vrs->protocols,
-                                    NS_V3_VOTE, vrs);
+                                    NS_V3_VOTE,
+                                    ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD,
+                                    vrs);
     if (rsf)
       smartlist_add(chunks, rsf);
 
     for (h = vrs->microdesc; h; h = h->next) {
-      smartlist_add(chunks, tor_strdup(h->microdesc_hash_line));
+      smartlist_add_strdup(chunks, h->microdesc_hash_line);
     }
   } SMARTLIST_FOREACH_END(vrs);
 
-  smartlist_add(chunks, tor_strdup("directory-footer\n"));
+  smartlist_add_strdup(chunks, "directory-footer\n");
 
   /* The digest includes everything up through the space after
    * directory-signature.  (Yuck.) */
@@ -273,7 +398,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
                            signing_key_fingerprint);
   }
 
-  note_crypto_pk_op(SIGN_DIR);
   {
     char *sig = router_get_dirobj_signature(digest, DIGEST_LEN,
                                             private_signing_key);
@@ -313,10 +437,73 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
   return status;
 }
 
+/** Set *<b>timing_out</b> to the intervals at which we would like to vote.
+ * Note that these aren't the intervals we'll use to vote; they're the ones
+ * that we'll vote to use. */
+static void
+dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out)
+{
+  const or_options_t *options = get_options();
+
+  tor_assert(timing_out);
+
+  timing_out->vote_interval = options->V3AuthVotingInterval;
+  timing_out->n_intervals_valid = options->V3AuthNIntervalsValid;
+  timing_out->vote_delay = options->V3AuthVoteDelay;
+  timing_out->dist_delay = options->V3AuthDistDelay;
+}
+
 /* =====
  * Consensus generation
  * ===== */
 
+/** If <b>vrs</b> has a hash made for the consensus method <b>method</b> with
+ * the digest algorithm <b>alg</b>, decode it and copy it into
+ * <b>digest256_out</b> and return 0.  Otherwise return -1. */
+static int
+vote_routerstatus_find_microdesc_hash(char *digest256_out,
+                                      const vote_routerstatus_t *vrs,
+                                      int method,
+                                      digest_algorithm_t alg)
+{
+  /* XXXX only returns the sha256 method. */
+  const vote_microdesc_hash_t *h;
+  char mstr[64];
+  size_t mlen;
+  char dstr[64];
+
+  tor_snprintf(mstr, sizeof(mstr), "%d", method);
+  mlen = strlen(mstr);
+  tor_snprintf(dstr, sizeof(dstr), " %s=",
+               crypto_digest_algorithm_get_name(alg));
+
+  for (h = vrs->microdesc; h; h = h->next) {
+    const char *cp = h->microdesc_hash_line;
+    size_t num_len;
+    /* cp looks like \d+(,\d+)* (digesttype=val )+ .  Let's hunt for mstr in
+     * the first part. */
+    while (1) {
+      num_len = strspn(cp, "1234567890");
+      if (num_len == mlen && fast_memeq(mstr, cp, mlen)) {
+        /* This is the line. */
+        char buf[BASE64_DIGEST256_LEN+1];
+        /* XXXX ignores extraneous stuff if the digest is too long.  This
+         * seems harmless enough, right? */
+        cp = strstr(cp, dstr);
+        if (!cp)
+          return -1;
+        cp += strlen(dstr);
+        strlcpy(buf, cp, sizeof(buf));
+        return digest256_from_base64(digest256_out, buf);
+      }
+      if (num_len == 0 || cp[num_len] != ',')
+        break;
+      cp += num_len + 1;
+    }
+  }
+  return -1;
+}
+
 /** Given a vote <b>vote</b> (not a consensus!), return its associated
  * networkstatus_voter_info_t. */
 static networkstatus_voter_info_t *
@@ -329,20 +516,6 @@ get_voter(const networkstatus_t *vote)
   return smartlist_get(vote->voters, 0);
 }
 
-/** Return the signature made by <b>voter</b> using the algorithm
- * <b>alg</b>, or NULL if none is found. */
-document_signature_t *
-voter_get_sig_by_algorithm(const networkstatus_voter_info_t *voter,
-                           digest_algorithm_t alg)
-{
-  if (!voter->sigs)
-    return NULL;
-  SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig,
-    if (sig->alg == alg)
-      return sig);
-  return NULL;
-}
-
 /** Temporary structure used in constructing a list of dir-source entries
  * for a consensus.  One of these is generated for every vote, and one more
  * for every legacy key in each vote. */
@@ -487,7 +660,7 @@ compute_routerstatus_consensus(smartlist_t *votes, int consensus_method,
 
   /* compare_vote_rs_() sorts the items by identity digest (all the same),
    * then by SD digest.  That way, if we have a tie that the published_on
-   * date cannot tie, we use the descriptor with the smaller digest.
+   * date cannot break, we use the descriptor with the smaller digest.
    */
   smartlist_sort(votes, compare_vote_rs_);
   SMARTLIST_FOREACH_BEGIN(votes, vote_routerstatus_t *, rs) {
@@ -509,18 +682,18 @@ compute_routerstatus_consensus(smartlist_t *votes, int consensus_method,
   if (cur_n > most_n ||
       (cur && cur_n == most_n && cur->status.published_on > most_published)) {
     most = cur;
-    most_n = cur_n;
-    most_published = cur->status.published_on;
+    // most_n = cur_n; // unused after this point.
+    // most_published = cur->status.published_on; // unused after this point.
   }
 
   tor_assert(most);
 
-  /* If we're producing "a" lines, vote on potential alternative (sets
-   * of) OR port(s) in the winning routerstatuses.
+  /* Vote on potential alternative (sets of) OR port(s) in the winning
+   * routerstatuses.
    *
    * XXX prop186 There's at most one alternative OR port (_the_ IPv6
    * port) for now. */
-  if (consensus_method >= MIN_METHOD_FOR_A_LINES && best_alt_orport_out) {
+  if (best_alt_orport_out) {
     smartlist_t *alt_orports = smartlist_new();
     const tor_addr_port_t *most_alt_orport = NULL;
 
@@ -630,13 +803,6 @@ compute_consensus_method(smartlist_t *votes)
 static int
 consensus_method_is_supported(int method)
 {
-  if (method == MIN_METHOD_FOR_ED25519_ID_IN_MD) {
-    /* This method was broken due to buggy code accidently left in
-     * dircollate.c; do not actually use it.
-     */
-    return 0;
-  }
-
   return (method >= MIN_SUPPORTED_CONSENSUS_METHOD) &&
     (method <= MAX_SUPPORTED_CONSENSUS_METHOD);
 }
@@ -712,12 +878,12 @@ dirvote_get_intermediate_param_value(const smartlist_t *param_list,
     }
   } SMARTLIST_FOREACH_END(k_v_pair);
 
-  if (n_found == 1)
+  if (n_found == 1) {
     return value;
-  else if (BUG(n_found > 1))
-    return default_val;
-  else
+  } else {
+    tor_assert_nonfatal(n_found == 0);
     return default_val;
+  }
 }
 
 /** Minimum number of directory authorities voting for a parameter to
@@ -771,6 +937,9 @@ dirvote_compute_params(smartlist_t *votes, int method, int total_authorities)
   output = smartlist_new();
 
   SMARTLIST_FOREACH_BEGIN(param_list, const char *, param) {
+    /* resolve spurious clang shallow analysis null pointer errors */
+    tor_assert(param);
+
     const char *next_param;
     int ok=0;
     eq = strchr(param, '=');
@@ -783,8 +952,7 @@ dirvote_compute_params(smartlist_t *votes, int method, int total_authorities)
       next_param = NULL;
     else
       next_param = smartlist_get(param_list, param_sl_idx+1);
-    /* resolve spurious clang shallow analysis null pointer errors */
-    tor_assert(param);
+
     if (!next_param || strncmp(next_param, param, cur_param_len)) {
       /* We've reached the end of a series. */
       /* Make sure enough authorities voted on this param, unless the
@@ -883,13 +1051,13 @@ networkstatus_check_weights(int64_t Wgg, int64_t Wgd, int64_t Wmg,
  out:
   if (berr) {
     log_info(LD_DIR,
-             "Bw weight mismatch %d. G="I64_FORMAT" M="I64_FORMAT
-             " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT
+             "Bw weight mismatch %d. G=%"PRId64" M=%"PRId64
+             " E=%"PRId64" D=%"PRId64" T=%"PRId64
              " Wmd=%d Wme=%d Wmg=%d Wed=%d Wee=%d"
              " Wgd=%d Wgg=%d Wme=%d Wmg=%d",
              berr,
-             I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-             I64_PRINTF_ARG(D), I64_PRINTF_ARG(T),
+             (G), (M), (E),
+             (D), (T),
              (int)Wmd, (int)Wme, (int)Wmg, (int)Wed, (int)Wee,
              (int)Wgd, (int)Wgg, (int)Wme, (int)Wmg);
   }
@@ -902,7 +1070,7 @@ networkstatus_check_weights(int64_t Wgg, int64_t Wgd, int64_t Wmg,
  *
  * It returns true if weights could be computed, false otherwise.
  */
-static int
+int
 networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
                                      int64_t M, int64_t E, int64_t D,
                                      int64_t T, int64_t weight_scale)
@@ -915,10 +1083,10 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
 
   if (G <= 0 || M <= 0 || E <= 0 || D <= 0) {
     log_warn(LD_DIR, "Consensus with empty bandwidth: "
-                     "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT
-                     " D="I64_FORMAT" T="I64_FORMAT,
-             I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-             I64_PRINTF_ARG(D), I64_PRINTF_ARG(T));
+                     "G=%"PRId64" M=%"PRId64" E=%"PRId64
+                     " D=%"PRId64" T=%"PRId64,
+             (G), (M), (E),
+             (D), (T));
     return 0;
   }
 
@@ -949,13 +1117,13 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
 
     if (berr) {
       log_warn(LD_DIR,
-             "Bw Weights error %d for %s v10. G="I64_FORMAT" M="I64_FORMAT
-             " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT
+             "Bw Weights error %d for %s v10. G=%"PRId64" M=%"PRId64
+             " E=%"PRId64" D=%"PRId64" T=%"PRId64
              " Wmd=%d Wme=%d Wmg=%d Wed=%d Wee=%d"
              " Wgd=%d Wgg=%d Wme=%d Wmg=%d weight_scale=%d",
              berr, casename,
-             I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-             I64_PRINTF_ARG(D), I64_PRINTF_ARG(T),
+             (G), (M), (E),
+             (D), (T),
              (int)Wmd, (int)Wme, (int)Wmg, (int)Wed, (int)Wee,
              (int)Wgd, (int)Wgg, (int)Wme, (int)Wmg, (int)weight_scale);
       return 0;
@@ -984,7 +1152,7 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
         Wgd = weight_scale;
       }
     } else { // Subcase b: R+D >= S
-      casename = "Case 2b1 (Wgg=1, Wmd=Wgd)";
+      casename = "Case 2b1 (Wgg=weight_scale, Wmd=Wgd)";
       Wee = (weight_scale*(E - G + M))/E;
       Wed = (weight_scale*(D - 2*E + 4*G - 2*M))/(3*D);
       Wme = (weight_scale*(G-M))/E;
@@ -997,7 +1165,7 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
                                        weight_scale, G, M, E, D, T, 10, 1);
 
       if (berr) {
-        casename = "Case 2b2 (Wgg=1, Wee=1)";
+        casename = "Case 2b2 (Wgg=weight_scale, Wee=weight_scale)";
         Wgg = weight_scale;
         Wee = weight_scale;
         Wed = (weight_scale*(D - 2*E + G + M))/(3*D);
@@ -1020,13 +1188,13 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
       if (berr != BW_WEIGHTS_NO_ERROR &&
               berr != BW_WEIGHTS_BALANCE_MID_ERROR) {
         log_warn(LD_DIR,
-             "Bw Weights error %d for %s v10. G="I64_FORMAT" M="I64_FORMAT
-             " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT
+             "Bw Weights error %d for %s v10. G=%"PRId64" M=%"PRId64
+             " E=%"PRId64" D=%"PRId64" T=%"PRId64
              " Wmd=%d Wme=%d Wmg=%d Wed=%d Wee=%d"
              " Wgd=%d Wgg=%d Wme=%d Wmg=%d weight_scale=%d",
              berr, casename,
-             I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-             I64_PRINTF_ARG(D), I64_PRINTF_ARG(T),
+             (G), (M), (E),
+             (D), (T),
              (int)Wmd, (int)Wme, (int)Wmg, (int)Wed, (int)Wee,
              (int)Wgd, (int)Wgg, (int)Wme, (int)Wmg, (int)weight_scale);
         return 0;
@@ -1037,10 +1205,10 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
     // Case 3: Exactly one of Guard or Exit is scarce
     if (!(3*E < T || 3*G < T) || !(3*G >= T || 3*E >= T)) {
       log_warn(LD_BUG,
-           "Bw-Weights Case 3 v10 but with G="I64_FORMAT" M="
-           I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT,
-               I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-               I64_PRINTF_ARG(D), I64_PRINTF_ARG(T));
+           "Bw-Weights Case 3 v10 but with G=%"PRId64" M="
+           "%"PRId64" E=%"PRId64" D=%"PRId64" T=%"PRId64,
+               (G), (M), (E),
+               (D), (T));
     }
 
     if (3*(S+D) < T) { // Subcase a: S+D < T/3
@@ -1066,7 +1234,7 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
     } else { // Subcase b: S+D >= T/3
       // D != 0 because S+D >= T/3
       if (G < E) {
-        casename = "Case 3bg (G scarce, Wgg=1, Wmd == Wed)";
+        casename = "Case 3bg (G scarce, Wgg=weight_scale, Wmd == Wed)";
         Wgg = weight_scale;
         Wgd = (weight_scale*(D - 2*G + E + M))/(3*D);
         Wmg = 0;
@@ -1078,7 +1246,7 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
         berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee,
                     Wed, weight_scale, G, M, E, D, T, 10, 1);
       } else { // G >= E
-        casename = "Case 3be (E scarce, Wee=1, Wmd == Wgd)";
+        casename = "Case 3be (E scarce, Wee=weight_scale, Wmd == Wgd)";
         Wee = weight_scale;
         Wed = (weight_scale*(D - 2*E + G + M))/(3*D);
         Wme = 0;
@@ -1092,13 +1260,13 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
       }
       if (berr) {
         log_warn(LD_DIR,
-             "Bw Weights error %d for %s v10. G="I64_FORMAT" M="I64_FORMAT
-             " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT
+             "Bw Weights error %d for %s v10. G=%"PRId64" M=%"PRId64
+             " E=%"PRId64" D=%"PRId64" T=%"PRId64
              " Wmd=%d Wme=%d Wmg=%d Wed=%d Wee=%d"
              " Wgd=%d Wgg=%d Wme=%d Wmg=%d weight_scale=%d",
              berr, casename,
-             I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-             I64_PRINTF_ARG(D), I64_PRINTF_ARG(T),
+             (G), (M), (E),
+             (D), (T),
              (int)Wmd, (int)Wme, (int)Wmg, (int)Wed, (int)Wee,
              (int)Wgd, (int)Wgg, (int)Wme, (int)Wmg, (int)weight_scale);
         return 0;
@@ -1112,7 +1280,7 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
   tor_assert(0 < weight_scale && weight_scale <= INT32_MAX);
 
   /*
-   * Provide Wgm=Wgg, Wmm=1, Wem=Wee, Weg=Wed. May later determine
+   * Provide Wgm=Wgg, Wmm=weight_scale, Wem=Wee, Weg=Wed. May later determine
    * that middle nodes need different bandwidth weights for dirport traffic,
    * or that weird exit policies need special weight, or that bridges
    * need special weight.
@@ -1132,11 +1300,11 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
      (int)weight_scale, (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale);
 
   log_notice(LD_CIRC, "Computed bandwidth weights for %s with v10: "
-             "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT
-             " T="I64_FORMAT,
+             "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+             " T=%"PRId64,
              casename,
-             I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-             I64_PRINTF_ARG(D), I64_PRINTF_ARG(T));
+             (G), (M), (E),
+             (D), (T));
   return 1;
 }
 
@@ -1291,12 +1459,23 @@ compute_nth_protocol_set(int n, int n_voters, const smartlist_t *votes)
 /** Given a list of vote networkstatus_t in <b>votes</b>, our public
  * authority <b>identity_key</b>, our private authority <b>signing_key</b>,
  * and the number of <b>total_authorities</b> that we believe exist in our
- * voting quorum, generate the text of a new v3 consensus vote, and return the
- * value in a newly allocated string.
+ * voting quorum, generate the text of a new v3 consensus or microdescriptor
+ * consensus (depending on <b>flavor</b>), and return the value in a newly
+ * allocated string.
  *
  * Note: this function DOES NOT check whether the votes are from
- * recognized authorities.   (dirvote_add_vote does that.) */
-char *
+ * recognized authorities.   (dirvote_add_vote does that.)
+ *
+ * <strong>WATCH OUT</strong>: You need to think before you change the
+ * behavior of this function, or of the functions it calls! If some
+ * authorities compute the consensus with a different algorithm than
+ * others, they will not reach the same result, and they will not all
+ * sign the same thing!  If you really need to change the algorithm
+ * here, you should allocate a new "consensus_method" for the new
+ * behavior, and make the new behavior conditional on a new-enough
+ * consensus_method.
+ **/
+STATIC char *
 networkstatus_compute_consensus(smartlist_t *votes,
                                 int total_authorities,
                                 crypto_pk_t *identity_key,
@@ -1314,7 +1493,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
   smartlist_t *flags;
   const char *flavor_name;
   uint32_t max_unmeasured_bw_kb = DEFAULT_MAX_UNMEASURED_BW_KB;
-  int64_t G=0, M=0, E=0, D=0, T=0; /* For bandwidth weights */
+  int64_t G, M, E, D, T; /* For bandwidth weights */
   const routerstatus_format_type_t rs_format =
     flavor == FLAV_NS ? NS_V3_CONSENSUS : NS_V3_CONSENSUS_MICRODESC;
   char *params = NULL;
@@ -1346,6 +1525,16 @@ networkstatus_compute_consensus(smartlist_t *votes,
     consensus_method = MAX_SUPPORTED_CONSENSUS_METHOD;
   }
 
+  if (consensus_method >= MIN_METHOD_FOR_INIT_BW_WEIGHTS_ONE) {
+    /* It's smarter to initialize these weights to 1, so that later on,
+     * we can't accidentally divide by zero. */
+    G = M = E = D = 1;
+    T = 4;
+  } else {
+    /* ...but originally, they were set to zero. */
+    G = M = E = D = T = 0;
+  }
+
   /* Compute medians of time-related things, and figure out how many
    * routers we might need to talk about. */
   {
@@ -1385,7 +1574,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
         smartlist_free(sv); /* elements get freed later. */
       }
       SMARTLIST_FOREACH(v->known_flags, const char *, cp,
-                        smartlist_add(flags, tor_strdup(cp)));
+                        smartlist_add_strdup(flags, cp));
     } SMARTLIST_FOREACH_END(v);
     valid_after = median_time(va_times, n_votes);
     fresh_until = median_time(fu_times, n_votes);
@@ -1406,19 +1595,14 @@ networkstatus_compute_consensus(smartlist_t *votes,
                                                       n_versioning_servers);
     client_versions = compute_consensus_versions_list(combined_client_versions,
                                                       n_versioning_clients);
-    if (consensus_method >= MIN_METHOD_FOR_PACKAGE_LINES) {
-      packages = compute_consensus_package_lines(votes);
-    } else {
-      packages = tor_strdup("");
-    }
+    packages = compute_consensus_package_lines(votes);
 
     SMARTLIST_FOREACH(combined_server_versions, char *, cp, tor_free(cp));
     SMARTLIST_FOREACH(combined_client_versions, char *, cp, tor_free(cp));
     smartlist_free(combined_server_versions);
     smartlist_free(combined_client_versions);
 
-    if (consensus_method >= MIN_METHOD_FOR_ED25519_ID_VOTING)
-      smartlist_add(flags, tor_strdup("NoEdConsensus"));
+    smartlist_add_strdup(flags, "NoEdConsensus");
 
     smartlist_sort_strings(flags);
     smartlist_uniq_strings(flags);
@@ -1467,7 +1651,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
     tor_free(flaglist);
   }
 
-  if (consensus_method >= MIN_METHOD_FOR_RECOMMENDED_PROTOCOLS) {
+  {
     int num_dirauth = get_n_authorities(V3_DIRINFO);
     int idx;
     for (idx = 0; idx < 4; ++idx) {
@@ -1482,12 +1666,12 @@ networkstatus_compute_consensus(smartlist_t *votes,
                                       total_authorities);
   if (smartlist_len(param_list)) {
     params = smartlist_join_strings(param_list, " ", 0, NULL);
-    smartlist_add(chunks, tor_strdup("params "));
+    smartlist_add_strdup(chunks, "params ");
     smartlist_add(chunks, params);
-    smartlist_add(chunks, tor_strdup("\n"));
+    smartlist_add_strdup(chunks, "\n");
   }
 
-  if (consensus_method >= MIN_METHOD_FOR_SHARED_RANDOM) {
+  {
     int num_dirauth = get_n_authorities(V3_DIRINFO);
     /* Default value of this is 2/3 of the total number of authorities. For
      * instance, if we have 9 dirauth, the default value is 6. The following
@@ -1552,7 +1736,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
     smartlist_free(dir_sources);
   }
 
-  if (consensus_method >= MIN_METHOD_TO_CLIP_UNMEASURED_BW) {
+  {
     char *max_unmeasured_param = NULL;
     /* XXXX Extract this code into a common function.  Or don't!  see #19011 */
     if (params) {
@@ -1624,9 +1808,9 @@ networkstatus_compute_consensus(smartlist_t *votes,
 
     /* Build the flag indexes. Note that no vote can have more than 64 members
      * for known_flags, so no value will be greater than 63, so it's safe to
-     * do U64_LITERAL(1) << index on these values.  But note also that
+     * do UINT64_C(1) << index on these values.  But note also that
      * named_flag and unnamed_flag are initialized to -1, so we need to check
-     * that they're actually set before doing U64_LITERAL(1) << index with
+     * that they're actually set before doing UINT64_C(1) << index with
      * them.*/
     SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) {
       flag_map[v_sl_idx] = tor_calloc(smartlist_len(v->known_flags),
@@ -1655,7 +1839,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
         uint64_t nf;
         if (named_flag[v_sl_idx]<0)
           continue;
-        nf = U64_LITERAL(1) << named_flag[v_sl_idx];
+        nf = UINT64_C(1) << named_flag[v_sl_idx];
         SMARTLIST_FOREACH_BEGIN(v->routerstatus_list,
                                 vote_routerstatus_t *, rs) {
 
@@ -1680,7 +1864,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
         uint64_t uf;
         if (unnamed_flag[v_sl_idx]<0)
           continue;
-        uf = U64_LITERAL(1) << unnamed_flag[v_sl_idx];
+        uf = UINT64_C(1) << unnamed_flag[v_sl_idx];
         SMARTLIST_FOREACH_BEGIN(v->routerstatus_list,
                                 vote_routerstatus_t *, rs) {
           if ((rs->flags & uf) != 0) {
@@ -1770,11 +1954,11 @@ networkstatus_compute_consensus(smartlist_t *votes,
 
         /* Tally up all the flags. */
         for (int flag = 0; flag < n_voter_flags[voter_idx]; ++flag) {
-          if (rs->flags & (U64_LITERAL(1) << flag))
+          if (rs->flags & (UINT64_C(1) << flag))
             ++flag_counts[flag_map[voter_idx][flag]];
         }
         if (named_flag[voter_idx] >= 0 &&
-            (rs->flags & (U64_LITERAL(1) << named_flag[voter_idx]))) {
+            (rs->flags & (UINT64_C(1) << named_flag[voter_idx]))) {
           if (chosen_name && strcmp(chosen_name, rs->status.nickname)) {
             log_notice(LD_DIR, "Conflict on naming for router: %s vs %s",
                        chosen_name, rs->status.nickname);
@@ -1814,7 +1998,6 @@ networkstatus_compute_consensus(smartlist_t *votes,
         continue;
 
       if (ed_consensus > 0) {
-        tor_assert(consensus_method >= MIN_METHOD_FOR_ED25519_ID_VOTING);
         if (ed_consensus <= total_authorities / 2) {
           log_warn(LD_BUG, "Not enough entries had ed_consensus set; how "
                    "can we have a consensus of %d?", ed_consensus);
@@ -1841,10 +2024,8 @@ networkstatus_compute_consensus(smartlist_t *votes,
       rs_out.published_on = rs->status.published_on;
       rs_out.dir_port = rs->status.dir_port;
       rs_out.or_port = rs->status.or_port;
-      if (consensus_method >= MIN_METHOD_FOR_A_LINES) {
-        tor_addr_copy(&rs_out.ipv6_addr, &alt_orport.addr);
-        rs_out.ipv6_orport = alt_orport.port;
-      }
+      tor_addr_copy(&rs_out.ipv6_addr, &alt_orport.addr);
+      rs_out.ipv6_orport = alt_orport.port;
       rs_out.has_bandwidth = 0;
       rs_out.has_exitsummary = 0;
 
@@ -1874,8 +2055,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
         } else if (!strcmp(fl, "Unnamed")) {
           if (is_unnamed)
             smartlist_add(chosen_flags, (char*)fl);
-        } else if (!strcmp(fl, "NoEdConsensus") &&
-                   consensus_method >= MIN_METHOD_FOR_ED25519_ID_VOTING) {
+        } else if (!strcmp(fl, "NoEdConsensus")) {
           if (ed_consensus <= total_authorities/2)
             smartlist_add(chosen_flags, (char*)fl);
         } else {
@@ -1902,8 +2082,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
 
       /* Starting with consensus method 24, we don't list servers
        * that are not valid in a consensus.  See Proposal 272 */
-      if (!is_valid &&
-          consensus_method >= MIN_METHOD_FOR_EXCLUDING_INVALID_NODES)
+      if (!is_valid)
         continue;
 
       /* Pick the version. */
@@ -1924,8 +2103,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
 
       /* If it's a guard and we have enough guardfraction votes,
          calculate its consensus guardfraction value. */
-      if (is_guard && num_guardfraction_inputs > 2 &&
-          consensus_method >= MIN_METHOD_FOR_GUARDFRACTION) {
+      if (is_guard && num_guardfraction_inputs > 2) {
         rs_out.has_guardfraction = 1;
         rs_out.guardfraction_percentage = median_uint32(measured_guardfraction,
                                                      num_guardfraction_inputs);
@@ -1942,8 +2120,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
         rs_out.has_bandwidth = 1;
         rs_out.bw_is_unmeasured = 1;
         rs_out.bandwidth_kb = median_uint32(bandwidths_kb, num_bandwidths);
-        if (consensus_method >= MIN_METHOD_TO_CLIP_UNMEASURED_BW &&
-            n_authorities_measuring_bandwidth > 2) {
+        if (n_authorities_measuring_bandwidth > 2) {
           /* Cap non-measured bandwidths. */
           if (rs_out.bandwidth_kb > max_unmeasured_bw_kb) {
             rs_out.bandwidth_kb = max_unmeasured_bw_kb;
@@ -2055,7 +2232,8 @@ networkstatus_compute_consensus(smartlist_t *votes,
         char *buf;
         /* Okay!! Now we can write the descriptor... */
         /*     First line goes into "buf". */
-        buf = routerstatus_format_entry(&rs_out, NULL, NULL, rs_format, NULL);
+        buf = routerstatus_format_entry(&rs_out, NULL, NULL,
+                                        rs_format, consensus_method, NULL);
         if (buf)
           smartlist_add(chunks, buf);
       }
@@ -2071,10 +2249,10 @@ networkstatus_compute_consensus(smartlist_t *votes,
                     smartlist_join_strings(chosen_flags, " ", 0, NULL));
       /*     Now the version line. */
       if (chosen_version) {
-        smartlist_add(chunks, tor_strdup("\nv "));
-        smartlist_add(chunks, tor_strdup(chosen_version));
+        smartlist_add_strdup(chunks, "\nv ");
+        smartlist_add_strdup(chunks, chosen_version);
       }
-      smartlist_add(chunks, tor_strdup("\n"));
+      smartlist_add_strdup(chunks, "\n");
       if (chosen_protocol_list &&
           consensus_method >= MIN_METHOD_FOR_RS_PROTOCOLS) {
         smartlist_add_asprintf(chunks, "pr %s\n", chosen_protocol_list);
@@ -2082,8 +2260,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
       /*     Now the weight line. */
       if (rs_out.has_bandwidth) {
         char *guardfraction_str = NULL;
-        int unmeasured = rs_out.bw_is_unmeasured &&
-          consensus_method >= MIN_METHOD_TO_CLIP_UNMEASURED_BW;
+        int unmeasured = rs_out.bw_is_unmeasured;
 
         /* If we have guardfraction info, include it in the 'w' line. */
         if (rs_out.has_guardfraction) {
@@ -2127,7 +2304,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
   }
 
   /* Mark the directory footer region */
-  smartlist_add(chunks, tor_strdup("directory-footer\n"));
+  smartlist_add_strdup(chunks, "directory-footer\n");
 
   {
     int64_t weight_scale = BW_WEIGHT_SCALE;
@@ -2178,7 +2355,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
     const char *algname = crypto_digest_algorithm_get_name(digest_alg);
     char *signature;
 
-    smartlist_add(chunks, tor_strdup("directory-signature "));
+    smartlist_add_strdup(chunks, "directory-signature ");
 
     /* Compute the hash of the chunks. */
     crypto_digest_smartlist(digest, digest_len, chunks, "", digest_alg);
@@ -2205,7 +2382,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
     smartlist_add(chunks, signature);
 
     if (legacy_id_key_digest && legacy_signing_key) {
-      smartlist_add(chunks, tor_strdup("directory-signature "));
+      smartlist_add_strdup(chunks, "directory-signature ");
       base16_encode(fingerprint, sizeof(fingerprint),
                     legacy_id_key_digest, DIGEST_LEN);
       crypto_pk_get_fingerprint(legacy_signing_key,
@@ -2342,7 +2519,7 @@ compute_consensus_package_lines(smartlist_t *votes)
  * new signature is verifiable.)  Return the number of signatures added or
  * changed, or -1 if the document signed by <b>sigs</b> isn't the same
  * document as <b>target</b>. */
-int
+STATIC int
 networkstatus_add_detached_signatures(networkstatus_t *target,
                                       ns_detached_signatures_t *sigs,
                                       const char *source,
@@ -2426,7 +2603,7 @@ networkstatus_add_detached_signatures(networkstatus_t *target,
       continue;
     }
 
-    old_sig = voter_get_sig_by_algorithm(target_voter, sig->alg);
+    old_sig = networkstatus_get_voter_sig_by_alg(target_voter, sig->alg);
 
     /* If the target already has a good signature from this voter, then skip
      * this one. */
@@ -2518,7 +2695,7 @@ networkstatus_format_signatures(networkstatus_t *consensus,
       base64_encode(buf, sizeof(buf), sig->signature, sig->signature_len,
                     BASE64_ENCODE_MULTILINE);
       strlcat(buf, "-----END SIGNATURE-----\n", sizeof(buf));
-      smartlist_add(elements, tor_strdup(buf));
+      smartlist_add_strdup(elements, buf);
     } SMARTLIST_FOREACH_END(sig);
   } SMARTLIST_FOREACH_END(v);
 
@@ -2534,7 +2711,7 @@ networkstatus_format_signatures(networkstatus_t *consensus,
  * corresponding to the signatures on <b>consensuses</b>, which must contain
  * exactly one FLAV_NS consensus, and no more than one consensus for each
  * other flavor. */
-char *
+STATIC char *
 networkstatus_get_detached_signatures(smartlist_t *consensuses)
 {
   smartlist_t *elements;
@@ -2639,256 +2816,66 @@ get_detached_signatures_from_pending_consensuses(pending_consensus_t *pending,
   return signatures;
 }
 
-/** Release all storage held in <b>s</b>. */
-void
-ns_detached_signatures_free(ns_detached_signatures_t *s)
-{
-  if (!s)
-    return;
-  if (s->signatures) {
-    STRMAP_FOREACH(s->signatures, flavor, smartlist_t *, sigs) {
-      SMARTLIST_FOREACH(sigs, document_signature_t *, sig,
-                        document_signature_free(sig));
-      smartlist_free(sigs);
-    } STRMAP_FOREACH_END;
-    strmap_free(s->signatures, NULL);
-    strmap_free(s->digests, tor_free_);
-  }
-
-  tor_free(s);
-}
-
-/* =====
- * Certificate functions
- * ===== */
-
-/** Allocate and return a new authority_cert_t with the same contents as
- * <b>cert</b>. */
-authority_cert_t *
-authority_cert_dup(authority_cert_t *cert)
-{
-  authority_cert_t *out = tor_malloc(sizeof(authority_cert_t));
-  tor_assert(cert);
-
-  memcpy(out, cert, sizeof(authority_cert_t));
-  /* Now copy pointed-to things. */
-  out->cache_info.signed_descriptor_body =
-    tor_strndup(cert->cache_info.signed_descriptor_body,
-                cert->cache_info.signed_descriptor_len);
-  out->cache_info.saved_location = SAVED_NOWHERE;
-  out->identity_key = crypto_pk_dup_key(cert->identity_key);
-  out->signing_key = crypto_pk_dup_key(cert->signing_key);
-
-  return out;
-}
-
-/* =====
- * Vote scheduling
- * ===== */
-
-/** Set *<b>timing_out</b> to the intervals at which we would like to vote.
- * Note that these aren't the intervals we'll use to vote; they're the ones
- * that we'll vote to use. */
-void
-dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out)
-{
-  const or_options_t *options = get_options();
-
-  tor_assert(timing_out);
-
-  timing_out->vote_interval = options->V3AuthVotingInterval;
-  timing_out->n_intervals_valid = options->V3AuthNIntervalsValid;
-  timing_out->vote_delay = options->V3AuthVoteDelay;
-  timing_out->dist_delay = options->V3AuthDistDelay;
-}
-
-/** Return the start of the next interval of size <b>interval</b> (in
- * seconds) after <b>now</b>, plus <b>offset</b>. Midnight always
- * starts a fresh interval, and if the last interval of a day would be
- * truncated to less than half its size, it is rolled into the
- * previous interval. */
-time_t
-dirvote_get_start_of_next_interval(time_t now, int interval, int offset)
-{
-  struct tm tm;
-  time_t midnight_today=0;
-  time_t midnight_tomorrow;
-  time_t next;
-
-  tor_gmtime_r(&now, &tm);
-  tm.tm_hour = 0;
-  tm.tm_min = 0;
-  tm.tm_sec = 0;
-
-  if (tor_timegm(&tm, &midnight_today) < 0) {
-    log_warn(LD_BUG, "Ran into an invalid time when trying to find midnight.");
-  }
-  midnight_tomorrow = midnight_today + (24*60*60);
-
-  next = midnight_today + ((now-midnight_today)/interval + 1)*interval;
-
-  /* Intervals never cross midnight. */
-  if (next > midnight_tomorrow)
-    next = midnight_tomorrow;
-
-  /* If the interval would only last half as long as it's supposed to, then
-   * skip over to the next day. */
-  if (next + interval/2 > midnight_tomorrow)
-    next = midnight_tomorrow;
-
-  next += offset;
-  if (next - interval > now)
-    next -= interval;
-
-  return next;
-}
-
-/* Using the time <b>now</b>, return the next voting valid-after time. */
+/**
+ * Entry point: Take whatever voting actions are pending as of <b>now</b>.
+ *
+ * Return the time at which the next action should be taken.
+ */
 time_t
-get_next_valid_after_time(time_t now)
-{
-  time_t next_valid_after_time;
-  const or_options_t *options = get_options();
-  voting_schedule_t *new_voting_schedule =
-    get_voting_schedule(options, now, LOG_INFO);
-  tor_assert(new_voting_schedule);
-
-  next_valid_after_time = new_voting_schedule->interval_starts;
-  voting_schedule_free(new_voting_schedule);
-
-  return next_valid_after_time;
-}
-
-static voting_schedule_t voting_schedule;
-
-/** Set voting_schedule to hold the timing for the next vote we should be
- * doing. */
-void
-dirvote_recalculate_timing(const or_options_t *options, time_t now)
-{
-  voting_schedule_t *new_voting_schedule;
-
-  if (!authdir_mode_v3(options)) {
-    return;
-  }
-
-  /* get the new voting schedule */
-  new_voting_schedule = get_voting_schedule(options, now, LOG_NOTICE);
-  tor_assert(new_voting_schedule);
-
-  /* Fill in the global static struct now */
-  memcpy(&voting_schedule, new_voting_schedule, sizeof(voting_schedule));
-  voting_schedule_free(new_voting_schedule);
-}
-
-/* Populate and return a new voting_schedule_t that can be used to schedule
- * voting. The object is allocated on the heap and it's the responsibility of
- * the caller to free it. Can't fail. */
-voting_schedule_t *
-get_voting_schedule(const or_options_t *options, time_t now, int severity)
-{
-  int interval, vote_delay, dist_delay;
-  time_t start;
-  time_t end;
-  networkstatus_t *consensus;
-  voting_schedule_t *new_voting_schedule;
-
-  new_voting_schedule = tor_malloc_zero(sizeof(voting_schedule_t));
-
-  consensus = networkstatus_get_live_consensus(now);
-
-  if (consensus) {
-    interval = (int)( consensus->fresh_until - consensus->valid_after );
-    vote_delay = consensus->vote_seconds;
-    dist_delay = consensus->dist_seconds;
-  } else {
-    interval = options->TestingV3AuthInitialVotingInterval;
-    vote_delay = options->TestingV3AuthInitialVoteDelay;
-    dist_delay = options->TestingV3AuthInitialDistDelay;
-  }
-
-  tor_assert(interval > 0);
-
-  if (vote_delay + dist_delay > interval/2)
-    vote_delay = dist_delay = interval / 4;
-
-  start = new_voting_schedule->interval_starts =
-    dirvote_get_start_of_next_interval(now,interval,
-                                      options->TestingV3AuthVotingStartOffset);
-  end = dirvote_get_start_of_next_interval(start+1, interval,
-                                      options->TestingV3AuthVotingStartOffset);
-
-  tor_assert(end > start);
-
-  new_voting_schedule->fetch_missing_signatures = start - (dist_delay/2);
-  new_voting_schedule->voting_ends = start - dist_delay;
-  new_voting_schedule->fetch_missing_votes =
-    start - dist_delay - (vote_delay/2);
-  new_voting_schedule->voting_starts = start - dist_delay - vote_delay;
-
-  {
-    char tbuf[ISO_TIME_LEN+1];
-    format_iso_time(tbuf, new_voting_schedule->interval_starts);
-    tor_log(severity, LD_DIR,"Choosing expected valid-after time as %s: "
-            "consensus_set=%d, interval=%d",
-            tbuf, consensus?1:0, interval);
-  }
-
-  return new_voting_schedule;
-}
-
-/** Frees a voting_schedule_t. This should be used instead of the generic
- * tor_free. */
-void
-voting_schedule_free(voting_schedule_t *voting_schedule_to_free)
-{
-  if (!voting_schedule_to_free)
-    return;
-  tor_free(voting_schedule_to_free);
-}
-
-/** Entry point: Take whatever voting actions are pending as of <b>now</b>. */
-void
 dirvote_act(const or_options_t *options, time_t now)
 {
   if (!authdir_mode_v3(options))
-    return;
-  if (!voting_schedule.voting_starts) {
+    return TIME_MAX;
+  tor_assert_nonfatal(voting_schedule.voting_starts);
+  /* If we haven't initialized this object through this codeflow, we need to
+   * recalculate the timings to match our vote. The reason to do that is if we
+   * have a voting schedule initialized 1 minute ago, the voting timings might
+   * not be aligned to what we should expect with "now". This is especially
+   * true for TestingTorNetwork using smaller timings.  */
+  if (voting_schedule.created_on_demand) {
     char *keys = list_v3_auth_ids();
     authority_cert_t *c = get_my_v3_authority_cert();
     log_notice(LD_DIR, "Scheduling voting.  Known authority IDs are %s. "
                "Mine is %s.",
                keys, hex_str(c->cache_info.identity_digest, DIGEST_LEN));
     tor_free(keys);
-    dirvote_recalculate_timing(options, now);
+    voting_schedule_recalculate_timing(options, now);
+  }
+
+#define IF_TIME_FOR_NEXT_ACTION(when_field, done_field) \
+  if (! voting_schedule.done_field) {                   \
+    if (voting_schedule.when_field > now) {             \
+      return voting_schedule.when_field;                \
+    } else {
+#define ENDIF \
+    }           \
   }
-  if (voting_schedule.voting_starts < now && !voting_schedule.have_voted) {
+
+  IF_TIME_FOR_NEXT_ACTION(voting_starts, have_voted) {
     log_notice(LD_DIR, "Time to vote.");
     dirvote_perform_vote();
     voting_schedule.have_voted = 1;
-  }
-  if (voting_schedule.fetch_missing_votes < now &&
-      !voting_schedule.have_fetched_missing_votes) {
+  } ENDIF
+  IF_TIME_FOR_NEXT_ACTION(fetch_missing_votes, have_fetched_missing_votes) {
     log_notice(LD_DIR, "Time to fetch any votes that we're missing.");
     dirvote_fetch_missing_votes();
     voting_schedule.have_fetched_missing_votes = 1;
-  }
-  if (voting_schedule.voting_ends < now &&
-      !voting_schedule.have_built_consensus) {
+  } ENDIF
+  IF_TIME_FOR_NEXT_ACTION(voting_ends, have_built_consensus) {
     log_notice(LD_DIR, "Time to compute a consensus.");
     dirvote_compute_consensuses();
     /* XXXX We will want to try again later if we haven't got enough
      * votes yet.  Implement this if it turns out to ever happen. */
     voting_schedule.have_built_consensus = 1;
-  }
-  if (voting_schedule.fetch_missing_signatures < now &&
-      !voting_schedule.have_fetched_missing_signatures) {
+  } ENDIF
+  IF_TIME_FOR_NEXT_ACTION(fetch_missing_signatures,
+                          have_fetched_missing_signatures) {
     log_notice(LD_DIR, "Time to fetch any signatures that we're missing.");
     dirvote_fetch_missing_signatures();
     voting_schedule.have_fetched_missing_signatures = 1;
-  }
-  if (voting_schedule.interval_starts < now &&
-      !voting_schedule.have_published_consensus) {
+  } ENDIF
+  IF_TIME_FOR_NEXT_ACTION(interval_starts,
+                          have_published_consensus) {
     log_notice(LD_DIR, "Time to publish the consensus and discard old votes");
     dirvote_publish_consensus();
     dirvote_clear_votes(0);
@@ -2898,8 +2885,15 @@ dirvote_act(const or_options_t *options, time_t now)
                 networkstatus_get_latest_consensus_by_flavor(FLAV_NS));
     /* XXXX We will want to try again later if we haven't got enough
      * signatures yet.  Implement this if it turns out to ever happen. */
-    dirvote_recalculate_timing(options, now);
-  }
+    voting_schedule_recalculate_timing(options, now);
+    return voting_schedule.voting_starts;
+  } ENDIF
+
+  tor_assert_nonfatal_unreached();
+  return now + 1;
+
+#undef ENDIF
+#undef IF_TIME_FOR_NEXT_ACTION
 }
 
 /** A vote networkstatus_t and its unparsed body: held around so we can
@@ -3505,7 +3499,13 @@ dirvote_add_signatures_to_pending_consensus(
   }
   r = networkstatus_add_detached_signatures(pc->consensus, sigs,
                                             source, severity, msg_out);
-  log_info(LD_DIR,"Added %d signatures to consensus.", r);
+  if (r >= 0) {
+    log_info(LD_DIR,"Added %d signatures to consensus.", r);
+  } else {
+    log_fn(LOG_PROTOCOL_WARN, LD_DIR,
+           "Unable to add signatures to consensus: %s",
+           *msg_out ? *msg_out : "(unknown)");
+  }
 
   if (r >= 1) {
     char *new_signatures =
@@ -3628,8 +3628,8 @@ dirvote_add_signatures(const char *detached_signatures_body,
                        "Queuing it for the next consensus.", source);
     if (!pending_consensus_signature_list)
       pending_consensus_signature_list = smartlist_new();
-    smartlist_add(pending_consensus_signature_list,
-                  tor_strdup(detached_signatures_body));
+    smartlist_add_strdup(pending_consensus_signature_list,
+                  detached_signatures_body);
     *msg = "Signature queued";
     return 0;
   }
@@ -3757,7 +3757,7 @@ dirvote_get_vote(const char *fp, int flags)
 /** Construct and return a new microdescriptor from a routerinfo <b>ri</b>
  * according to <b>consensus_method</b>.
  **/
-microdesc_t *
+STATIC microdesc_t *
 dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method)
 {
   microdesc_t *result = NULL;
@@ -3765,8 +3765,10 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method)
   size_t keylen;
   smartlist_t *chunks = smartlist_new();
   char *output = NULL;
+  crypto_pk_t *rsa_pubkey = router_get_rsa_onion_pkey(ri->onion_pkey,
+                                                      ri->onion_pkey_len);
 
-  if (crypto_pk_write_public_key_to_string(ri->onion_pkey, &key, &keylen)<0)
+  if (crypto_pk_write_public_key_to_string(rsa_pubkey, &key, &keylen)<0)
     goto done;
   summary = policy_summarize(ri->exit_policy, AF_INET);
   if (ri->declared_family)
@@ -3774,8 +3776,7 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method)
 
   smartlist_add_asprintf(chunks, "onion-key\n%s", key);
 
-  if (consensus_method >= MIN_METHOD_FOR_NTOR_KEY &&
-      ri->onion_curve25519_pkey) {
+  if (ri->onion_curve25519_pkey) {
     char kbuf[128];
     base64_encode(kbuf, sizeof(kbuf),
                   (const char*)ri->onion_curve25519_pkey->public_key,
@@ -3783,7 +3784,9 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method)
     smartlist_add_asprintf(chunks, "ntor-onion-key %s", kbuf);
   }
 
-  if (consensus_method >= MIN_METHOD_FOR_A_LINES &&
+  /* We originally put a lines in the micrdescriptors, but then we worked out
+   * that we needed them in the microdesc consensus. See #20916. */
+  if (consensus_method < MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC &&
       !tor_addr_is_null(&ri->ipv6_addr) && ri->ipv6_orport)
     smartlist_add_asprintf(chunks, "a %s\n",
                            fmt_addrport(&ri->ipv6_addr, ri->ipv6_orport));
@@ -3794,8 +3797,7 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method)
   if (summary && strcmp(summary, "reject 1-65535"))
     smartlist_add_asprintf(chunks, "p %s\n", summary);
 
-  if (consensus_method >= MIN_METHOD_FOR_P6_LINES &&
-      ri->ipv6_exit_policy) {
+  if (ri->ipv6_exit_policy) {
     /* XXXX+++ This doesn't match proposal 208, which says these should
      * be taken unchanged from the routerinfo.  That's bogosity, IMO:
      * the proposal should have said to do this instead.*/
@@ -3805,11 +3807,10 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method)
     tor_free(p6);
   }
 
-  if (consensus_method >= MIN_METHOD_FOR_ID_HASH_IN_MD) {
+  {
     char idbuf[ED25519_BASE64_LEN+1];
     const char *keytype;
-    if (consensus_method >= MIN_METHOD_FOR_ED25519_ID_IN_MD &&
-        ri->cache_info.signing_key_cert &&
+    if (ri->cache_info.signing_key_cert &&
         ri->cache_info.signing_key_cert->signing_key_included) {
       keytype = "ed25519";
       ed25519_public_to_base64(idbuf,
@@ -3838,6 +3839,7 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method)
   }
 
  done:
+  crypto_pk_free(rsa_pubkey);
   tor_free(output);
   tor_free(key);
   tor_free(summary);
@@ -3853,7 +3855,7 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method)
  * in a consensus vote document.  Write it into the <b>out_len</b>-byte buffer
  * in <b>out</b>.  Return -1 on failure and the number of characters written
  * on success. */
-ssize_t
+static ssize_t
 dirvote_format_microdesc_vote_line(char *out_buf, size_t out_buf_len,
                                    const microdesc_t *md,
                                    int consensus_method_low,
@@ -3887,12 +3889,8 @@ static const struct consensus_method_range_t {
   int low;
   int high;
 } microdesc_consensus_methods[] = {
-  {MIN_SUPPORTED_CONSENSUS_METHOD, MIN_METHOD_FOR_A_LINES - 1},
-  {MIN_METHOD_FOR_A_LINES, MIN_METHOD_FOR_P6_LINES - 1},
-  {MIN_METHOD_FOR_P6_LINES, MIN_METHOD_FOR_NTOR_KEY - 1},
-  {MIN_METHOD_FOR_NTOR_KEY, MIN_METHOD_FOR_ID_HASH_IN_MD - 1},
-  {MIN_METHOD_FOR_ID_HASH_IN_MD, MIN_METHOD_FOR_ED25519_ID_IN_MD - 1},
-  {MIN_METHOD_FOR_ED25519_ID_IN_MD, MAX_SUPPORTED_CONSENSUS_METHOD},
+  {MIN_SUPPORTED_CONSENSUS_METHOD, MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC - 1},
+  {MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC, MAX_SUPPORTED_CONSENSUS_METHOD},
   {-1, -1}
 };
 
@@ -3949,14 +3947,15 @@ dirvote_format_all_microdesc_vote_lines(const routerinfo_t *ri, time_t now,
   while ((ep = entries)) {
     char buf[128];
     vote_microdesc_hash_t *h;
-    dirvote_format_microdesc_vote_line(buf, sizeof(buf), ep->md,
-                                       ep->low, ep->high);
-    h = tor_malloc_zero(sizeof(vote_microdesc_hash_t));
-    h->microdesc_hash_line = tor_strdup(buf);
-    h->next = result;
-    result = h;
-    ep->md->last_listed = now;
-    smartlist_add(microdescriptors_out, ep->md);
+    if (dirvote_format_microdesc_vote_line(buf, sizeof(buf), ep->md,
+                                           ep->low, ep->high) >= 0) {
+      h = tor_malloc_zero(sizeof(vote_microdesc_hash_t));
+      h->microdesc_hash_line = tor_strdup(buf);
+      h->next = result;
+      result = h;
+      ep->md->last_listed = now;
+      smartlist_add(microdescriptors_out, ep->md);
+    }
     entries = ep->next;
     tor_free(ep);
   }
@@ -3964,49 +3963,696 @@ dirvote_format_all_microdesc_vote_lines(const routerinfo_t *ri, time_t now,
   return result;
 }
 
-/** If <b>vrs</b> has a hash made for the consensus method <b>method</b> with
- * the digest algorithm <b>alg</b>, decode it and copy it into
- * <b>digest256_out</b> and return 0.  Otherwise return -1. */
-int
-vote_routerstatus_find_microdesc_hash(char *digest256_out,
-                                      const vote_routerstatus_t *vrs,
-                                      int method,
-                                      digest_algorithm_t alg)
+/** Parse and extract all SR commits from <b>tokens</b> and place them in
+ *  <b>ns</b>. */
+static void
+extract_shared_random_commits(networkstatus_t *ns, const smartlist_t *tokens)
 {
-  /* XXXX only returns the sha256 method. */
-  const vote_microdesc_hash_t *h;
-  char mstr[64];
-  size_t mlen;
-  char dstr[64];
+  smartlist_t *chunks = NULL;
 
-  tor_snprintf(mstr, sizeof(mstr), "%d", method);
-  mlen = strlen(mstr);
-  tor_snprintf(dstr, sizeof(dstr), " %s=",
-               crypto_digest_algorithm_get_name(alg));
+  tor_assert(ns);
+  tor_assert(tokens);
+  /* Commits are only present in a vote. */
+  tor_assert(ns->type == NS_TYPE_VOTE);
 
-  for (h = vrs->microdesc; h; h = h->next) {
-    const char *cp = h->microdesc_hash_line;
-    size_t num_len;
-    /* cp looks like \d+(,\d+)* (digesttype=val )+ .  Let's hunt for mstr in
-     * the first part. */
-    while (1) {
-      num_len = strspn(cp, "1234567890");
-      if (num_len == mlen && fast_memeq(mstr, cp, mlen)) {
-        /* This is the line. */
-        char buf[BASE64_DIGEST256_LEN+1];
-        /* XXXX ignores extraneous stuff if the digest is too long.  This
-         * seems harmless enough, right? */
-        cp = strstr(cp, dstr);
-        if (!cp)
-          return -1;
-        cp += strlen(dstr);
-        strlcpy(buf, cp, sizeof(buf));
-        return digest256_from_base64(digest256_out, buf);
+  ns->sr_info.commits = smartlist_new();
+
+  smartlist_t *commits = find_all_by_keyword(tokens, K_COMMIT);
+  /* It's normal that a vote might contain no commits even if it participates
+   * in the SR protocol. Don't treat it as an error. */
+  if (commits == NULL) {
+    goto end;
+  }
+
+  /* Parse the commit. We do NO validation of number of arguments or ordering
+   * for forward compatibility, it's the parse commit job to inform us if it's
+   * supported or not. */
+  chunks = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(commits, directory_token_t *, tok) {
+    /* Extract all arguments and put them in the chunks list. */
+    for (int i = 0; i < tok->n_args; i++) {
+      smartlist_add(chunks, tok->args[i]);
+    }
+    sr_commit_t *commit = sr_parse_commit(chunks);
+    smartlist_clear(chunks);
+    if (commit == NULL) {
+      /* Get voter identity so we can warn that this dirauth vote contains
+       * commit we can't parse. */
+      networkstatus_voter_info_t *voter = smartlist_get(ns->voters, 0);
+      tor_assert(voter);
+      log_warn(LD_DIR, "SR: Unable to parse commit %s from vote of voter %s.",
+               escaped(tok->object_body),
+               hex_str(voter->identity_digest,
+                       sizeof(voter->identity_digest)));
+      /* Commitment couldn't be parsed. Continue onto the next commit because
+       * this one could be unsupported for instance. */
+      continue;
+    }
+    /* Add newly created commit object to the vote. */
+    smartlist_add(ns->sr_info.commits, commit);
+  } SMARTLIST_FOREACH_END(tok);
+
+ end:
+  smartlist_free(chunks);
+  smartlist_free(commits);
+}
+
+/* Using the given directory tokens in tokens, parse the shared random commits
+ * and put them in the given vote document ns.
+ *
+ * This also sets the SR participation flag if present in the vote. */
+void
+dirvote_parse_sr_commits(networkstatus_t *ns, const smartlist_t *tokens)
+{
+  /* Does this authority participates in the SR protocol? */
+  directory_token_t *tok = find_opt_by_keyword(tokens, K_SR_FLAG);
+  if (tok) {
+    ns->sr_info.participate = 1;
+    /* Get the SR commitments and reveals from the vote. */
+    extract_shared_random_commits(ns, tokens);
+  }
+}
+
+/* For the given vote, free the shared random commits if any. */
+void
+dirvote_clear_commits(networkstatus_t *ns)
+{
+  tor_assert(ns->type == NS_TYPE_VOTE);
+
+  if (ns->sr_info.commits) {
+    SMARTLIST_FOREACH(ns->sr_info.commits, sr_commit_t *, c,
+                      sr_commit_free(c));
+    smartlist_free(ns->sr_info.commits);
+  }
+}
+
+/* The given url is the /tor/status-vote GET directory request. Populates the
+ * items list with strings that we can compress on the fly and dir_items with
+ * cached_dir_t objects that have a precompressed deflated version. */
+void
+dirvote_dirreq_get_status_vote(const char *url, smartlist_t *items,
+                               smartlist_t *dir_items)
+{
+  int current;
+
+  url += strlen("/tor/status-vote/");
+  current = !strcmpstart(url, "current/");
+  url = strchr(url, '/');
+  tor_assert(url);
+  ++url;
+  if (!strcmp(url, "consensus")) {
+    const char *item;
+    tor_assert(!current); /* we handle current consensus specially above,
+                           * since it wants to be spooled. */
+    if ((item = dirvote_get_pending_consensus(FLAV_NS)))
+      smartlist_add(items, (char*)item);
+  } else if (!current && !strcmp(url, "consensus-signatures")) {
+    /* XXXX the spec says that we should implement
+     * current/consensus-signatures too.  It doesn't seem to be needed,
+     * though. */
+    const char *item;
+    if ((item=dirvote_get_pending_detached_signatures()))
+      smartlist_add(items, (char*)item);
+  } else if (!strcmp(url, "authority")) {
+    const cached_dir_t *d;
+    int flags = DGV_BY_ID |
+      (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING);
+    if ((d=dirvote_get_vote(NULL, flags)))
+      smartlist_add(dir_items, (cached_dir_t*)d);
+  } else {
+    const cached_dir_t *d;
+    smartlist_t *fps = smartlist_new();
+    int flags;
+    if (!strcmpstart(url, "d/")) {
+      url += 2;
+      flags = DGV_INCLUDE_PENDING | DGV_INCLUDE_PREVIOUS;
+    } else {
+      flags = DGV_BY_ID |
+        (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING);
+    }
+    dir_split_resource_into_fingerprints(url, fps, NULL,
+                                         DSR_HEX|DSR_SORT_UNIQ);
+    SMARTLIST_FOREACH(fps, char *, fp, {
+                      if ((d = dirvote_get_vote(fp, flags)))
+                      smartlist_add(dir_items, (cached_dir_t*)d);
+                      tor_free(fp);
+                      });
+    smartlist_free(fps);
+  }
+}
+
+/** Get the best estimate of a router's bandwidth for dirauth purposes,
+ * preferring measured to advertised values if available. */
+static uint32_t
+dirserv_get_bandwidth_for_router_kb(const routerinfo_t *ri)
+{
+  uint32_t bw_kb = 0;
+  /*
+   * Yeah, measured bandwidths in measured_bw_line_t are (implicitly
+   * signed) longs and the ones router_get_advertised_bandwidth() returns
+   * are uint32_t.
+   */
+  long mbw_kb = 0;
+
+  if (ri) {
+    /*
+     * * First try to see if we have a measured bandwidth; don't bother with
+     * as_of_out here, on the theory that a stale measured bandwidth is still
+     * better to trust than an advertised one.
+     */
+    if (dirserv_query_measured_bw_cache_kb(ri->cache_info.identity_digest,
+                                           &mbw_kb, NULL)) {
+      /* Got one! */
+      bw_kb = (uint32_t)mbw_kb;
+    } else {
+      /* If not, fall back to advertised */
+      bw_kb = router_get_advertised_bandwidth(ri) / 1000;
+    }
+  }
+
+  return bw_kb;
+}
+
+/** Helper for sorting: compares two routerinfos first by address, and then by
+ * descending order of "usefulness".  (An authority is more useful than a
+ * non-authority; a running router is more useful than a non-running router;
+ * and a router with more bandwidth is more useful than one with less.)
+ **/
+static int
+compare_routerinfo_by_ip_and_bw_(const void **a, const void **b)
+{
+  routerinfo_t *first = *(routerinfo_t **)a, *second = *(routerinfo_t **)b;
+  int first_is_auth, second_is_auth;
+  uint32_t bw_kb_first, bw_kb_second;
+  const node_t *node_first, *node_second;
+  int first_is_running, second_is_running;
+
+  /* we return -1 if first should appear before second... that is,
+   * if first is a better router. */
+  if (first->addr < second->addr)
+    return -1;
+  else if (first->addr > second->addr)
+    return 1;
+
+  /* Potentially, this next bit could cause k n lg n memeq calls.  But in
+   * reality, we will almost never get here, since addresses will usually be
+   * different. */
+
+  first_is_auth =
+    router_digest_is_trusted_dir(first->cache_info.identity_digest);
+  second_is_auth =
+    router_digest_is_trusted_dir(second->cache_info.identity_digest);
+
+  if (first_is_auth && !second_is_auth)
+    return -1;
+  else if (!first_is_auth && second_is_auth)
+    return 1;
+
+  node_first = node_get_by_id(first->cache_info.identity_digest);
+  node_second = node_get_by_id(second->cache_info.identity_digest);
+  first_is_running = node_first && node_first->is_running;
+  second_is_running = node_second && node_second->is_running;
+
+  if (first_is_running && !second_is_running)
+    return -1;
+  else if (!first_is_running && second_is_running)
+    return 1;
+
+  bw_kb_first = dirserv_get_bandwidth_for_router_kb(first);
+  bw_kb_second = dirserv_get_bandwidth_for_router_kb(second);
+
+  if (bw_kb_first > bw_kb_second)
+    return -1;
+  else if (bw_kb_first < bw_kb_second)
+    return 1;
+
+  /* They're equal! Compare by identity digest, so there's a
+   * deterministic order and we avoid flapping. */
+  return fast_memcmp(first->cache_info.identity_digest,
+                     second->cache_info.identity_digest,
+                     DIGEST_LEN);
+}
+
+/** Given a list of routerinfo_t in <b>routers</b>, return a new digestmap_t
+ * whose keys are the identity digests of those routers that we're going to
+ * exclude for Sybil-like appearance. */
+static digestmap_t *
+get_possible_sybil_list(const smartlist_t *routers)
+{
+  const or_options_t *options = get_options();
+  digestmap_t *omit_as_sybil;
+  smartlist_t *routers_by_ip = smartlist_new();
+  uint32_t last_addr;
+  int addr_count;
+  /* Allow at most this number of Tor servers on a single IP address, ... */
+  int max_with_same_addr = options->AuthDirMaxServersPerAddr;
+  if (max_with_same_addr <= 0)
+    max_with_same_addr = INT_MAX;
+
+  smartlist_add_all(routers_by_ip, routers);
+  smartlist_sort(routers_by_ip, compare_routerinfo_by_ip_and_bw_);
+  omit_as_sybil = digestmap_new();
+
+  last_addr = 0;
+  addr_count = 0;
+  SMARTLIST_FOREACH_BEGIN(routers_by_ip, routerinfo_t *, ri) {
+    if (last_addr != ri->addr) {
+      last_addr = ri->addr;
+      addr_count = 1;
+    } else if (++addr_count > max_with_same_addr) {
+      digestmap_set(omit_as_sybil, ri->cache_info.identity_digest, ri);
+    }
+  } SMARTLIST_FOREACH_END(ri);
+
+  smartlist_free(routers_by_ip);
+  return omit_as_sybil;
+}
+
+/** Given a platform string as in a routerinfo_t (possibly null), return a
+ * newly allocated version string for a networkstatus document, or NULL if the
+ * platform doesn't give a Tor version. */
+static char *
+version_from_platform(const char *platform)
+{
+  if (platform && !strcmpstart(platform, "Tor ")) {
+    const char *eos = find_whitespace(platform+4);
+    if (eos && !strcmpstart(eos, " (r")) {
+      /* XXXX Unify this logic with the other version extraction
+       * logic in routerparse.c. */
+      eos = find_whitespace(eos+1);
+    }
+    if (eos) {
+      return tor_strndup(platform, eos-platform);
+    }
+  }
+  return NULL;
+}
+
+/** Given a (possibly empty) list of config_line_t, each line of which contains
+ * a list of comma-separated version numbers surrounded by optional space,
+ * allocate and return a new string containing the version numbers, in order,
+ * separated by commas.  Used to generate Recommended(Client|Server)?Versions
+ */
+char *
+format_recommended_version_list(const config_line_t *ln, int warn)
+{
+  smartlist_t *versions;
+  char *result;
+  versions = smartlist_new();
+  for ( ; ln; ln = ln->next) {
+    smartlist_split_string(versions, ln->value, ",",
+                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  }
+
+  /* Handle the case where a dirauth operator has accidentally made some
+   * versions space-separated instead of comma-separated. */
+  smartlist_t *more_versions = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(versions, char *, v) {
+    if (strchr(v, ' ')) {
+      if (warn)
+        log_warn(LD_DIRSERV, "Unexpected space in versions list member %s. "
+                 "(These are supposed to be comma-separated; I'll pretend you "
+                 "used commas instead.)", escaped(v));
+      SMARTLIST_DEL_CURRENT(versions, v);
+      smartlist_split_string(more_versions, v, NULL,
+                             SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+      tor_free(v);
+    }
+  } SMARTLIST_FOREACH_END(v);
+  smartlist_add_all(versions, more_versions);
+  smartlist_free(more_versions);
+
+  /* Check to make sure everything looks like a version. */
+  if (warn) {
+    SMARTLIST_FOREACH_BEGIN(versions, const char *, v) {
+      tor_version_t ver;
+      if (tor_version_parse(v, &ver) < 0) {
+        log_warn(LD_DIRSERV, "Recommended version %s does not look valid. "
+                 " (I'll include it anyway, since you told me to.)",
+                 escaped(v));
       }
-      if (num_len == 0 || cp[num_len] != ',')
-        break;
-      cp += num_len + 1;
+    } SMARTLIST_FOREACH_END(v);
+  }
+
+  sort_version_list(versions, 1);
+  result = smartlist_join_strings(versions,",",0,NULL);
+  SMARTLIST_FOREACH(versions,char *,s,tor_free(s));
+  smartlist_free(versions);
+  return result;
+}
+
+/** If there are entries in <b>routers</b> with exactly the same ed25519 keys,
+ * remove the older one.  If they are exactly the same age, remove the one
+ * with the greater descriptor digest. May alter the order of the list. */
+static void
+routers_make_ed_keys_unique(smartlist_t *routers)
+{
+  routerinfo_t *ri2;
+  digest256map_t *by_ed_key = digest256map_new();
+
+  SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
+    ri->omit_from_vote = 0;
+    if (ri->cache_info.signing_key_cert == NULL)
+      continue; /* No ed key */
+    const uint8_t *pk = ri->cache_info.signing_key_cert->signing_key.pubkey;
+    if ((ri2 = digest256map_get(by_ed_key, pk))) {
+      /* Duplicate; must omit one.  Set the omit_from_vote flag in whichever
+       * one has the earlier published_on. */
+      const time_t ri_pub = ri->cache_info.published_on;
+      const time_t ri2_pub = ri2->cache_info.published_on;
+      if (ri2_pub < ri_pub ||
+          (ri2_pub == ri_pub &&
+           fast_memcmp(ri->cache_info.signed_descriptor_digest,
+                     ri2->cache_info.signed_descriptor_digest,DIGEST_LEN)<0)) {
+        digest256map_set(by_ed_key, pk, ri);
+        ri2->omit_from_vote = 1;
+      } else {
+        ri->omit_from_vote = 1;
+      }
+    } else {
+      /* Add to map */
+      digest256map_set(by_ed_key, pk, ri);
     }
+  } SMARTLIST_FOREACH_END(ri);
+
+  digest256map_free(by_ed_key, NULL);
+
+  /* Now remove every router where the omit_from_vote flag got set. */
+  SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) {
+    if (ri->omit_from_vote) {
+      SMARTLIST_DEL_CURRENT(routers, ri);
+    }
+  } SMARTLIST_FOREACH_END(ri);
+}
+
+/** Routerstatus <b>rs</b> is part of a group of routers that are on
+ * too narrow an IP-space. Clear out its flags since we don't want it be used
+ * because of its Sybil-like appearance.
+ *
+ * Leave its BadExit flag alone though, since if we think it's a bad exit,
+ * we want to vote that way in case all the other authorities are voting
+ * Running and Exit.
+ */
+static void
+clear_status_flags_on_sybil(routerstatus_t *rs)
+{
+  rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast =
+    rs->is_flagged_running = rs->is_named = rs->is_valid =
+    rs->is_hs_dir = rs->is_v2_dir = rs->is_possible_guard = 0;
+  /* FFFF we might want some mechanism to check later on if we
+   * missed zeroing any flags: it's easy to add a new flag but
+   * forget to add it to this clause. */
+}
+
+/** Return a new networkstatus_t* containing our current opinion. (For v3
+ * authorities) */
+networkstatus_t *
+dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
+                                        authority_cert_t *cert)
+{
+  const or_options_t *options = get_options();
+  networkstatus_t *v3_out = NULL;
+  uint32_t addr;
+  char *hostname = NULL, *client_versions = NULL, *server_versions = NULL;
+  const char *contact;
+  smartlist_t *routers, *routerstatuses;
+  char identity_digest[DIGEST_LEN];
+  char signing_key_digest[DIGEST_LEN];
+  int listbadexits = options->AuthDirListBadExits;
+  routerlist_t *rl = router_get_routerlist();
+  time_t now = time(NULL);
+  time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
+  networkstatus_voter_info_t *voter = NULL;
+  vote_timing_t timing;
+  digestmap_t *omit_as_sybil = NULL;
+  const int vote_on_reachability = running_long_enough_to_decide_unreachable();
+  smartlist_t *microdescriptors = NULL;
+  smartlist_t *bw_file_headers = NULL;
+
+  tor_assert(private_key);
+  tor_assert(cert);
+
+  if (crypto_pk_get_digest(private_key, signing_key_digest)<0) {
+    log_err(LD_BUG, "Error computing signing key digest");
+    return NULL;
   }
-  return -1;
+  if (crypto_pk_get_digest(cert->identity_key, identity_digest)<0) {
+    log_err(LD_BUG, "Error computing identity key digest");
+    return NULL;
+  }
+  if (resolve_my_address(LOG_WARN, options, &addr, NULL, &hostname)<0) {
+    log_warn(LD_NET, "Couldn't resolve my hostname");
+    return NULL;
+  }
+  if (!hostname || !strchr(hostname, '.')) {
+    tor_free(hostname);
+    hostname = tor_dup_ip(addr);
+  }
+
+  if (options->VersioningAuthoritativeDir) {
+    client_versions =
+      format_recommended_version_list(options->RecommendedClientVersions, 0);
+    server_versions =
+      format_recommended_version_list(options->RecommendedServerVersions, 0);
+  }
+
+  contact = get_options()->ContactInfo;
+  if (!contact)
+    contact = "(none)";
+
+  /*
+   * Do this so dirserv_compute_performance_thresholds() and
+   * set_routerstatus_from_routerinfo() see up-to-date bandwidth info.
+   */
+  if (options->V3BandwidthsFile) {
+    dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL);
+  } else {
+    /*
+     * No bandwidths file; clear the measured bandwidth cache in case we had
+     * one last time around.
+     */
+    if (dirserv_get_measured_bw_cache_size() > 0) {
+      dirserv_clear_measured_bw_cache();
+    }
+  }
+
+  /* precompute this part, since we need it to decide what "stable"
+   * means. */
+  SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, {
+                    dirserv_set_router_is_running(ri, now);
+                    });
+
+  routers = smartlist_new();
+  smartlist_add_all(routers, rl->routers);
+  routers_make_ed_keys_unique(routers);
+  /* After this point, don't use rl->routers; use 'routers' instead. */
+  routers_sort_by_identity(routers);
+  omit_as_sybil = get_possible_sybil_list(routers);
+
+  DIGESTMAP_FOREACH(omit_as_sybil, sybil_id, void *, ignore) {
+    (void) ignore;
+    rep_hist_make_router_pessimal(sybil_id, now);
+  } DIGESTMAP_FOREACH_END;
+
+  /* Count how many have measured bandwidths so we know how to assign flags;
+   * this must come before dirserv_compute_performance_thresholds() */
+  dirserv_count_measured_bws(routers);
+
+  dirserv_compute_performance_thresholds(omit_as_sybil);
+
+  routerstatuses = smartlist_new();
+  microdescriptors = smartlist_new();
+
+  SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
+    /* If it has a protover list and contains a protocol name greater than
+     * MAX_PROTOCOL_NAME_LENGTH, skip it. */
+    if (ri->protocol_list &&
+        protover_contains_long_protocol_names(ri->protocol_list)) {
+      continue;
+    }
+    if (ri->cache_info.published_on >= cutoff) {
+      routerstatus_t *rs;
+      vote_routerstatus_t *vrs;
+      node_t *node = node_get_mutable_by_id(ri->cache_info.identity_digest);
+      if (!node)
+        continue;
+
+      vrs = tor_malloc_zero(sizeof(vote_routerstatus_t));
+      rs = &vrs->status;
+      set_routerstatus_from_routerinfo(rs, node, ri, now,
+                                       listbadexits);
+
+      if (ri->cache_info.signing_key_cert) {
+        memcpy(vrs->ed25519_id,
+               ri->cache_info.signing_key_cert->signing_key.pubkey,
+               ED25519_PUBKEY_LEN);
+      }
+
+      if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest))
+        clear_status_flags_on_sybil(rs);
+
+      if (!vote_on_reachability)
+        rs->is_flagged_running = 0;
+
+      vrs->version = version_from_platform(ri->platform);
+      if (ri->protocol_list) {
+        vrs->protocols = tor_strdup(ri->protocol_list);
+      } else {
+        vrs->protocols = tor_strdup(
+                                protover_compute_for_old_tor(vrs->version));
+      }
+      vrs->microdesc = dirvote_format_all_microdesc_vote_lines(ri, now,
+                                                            microdescriptors);
+
+      smartlist_add(routerstatuses, vrs);
+    }
+  } SMARTLIST_FOREACH_END(ri);
+
+  {
+    smartlist_t *added =
+      microdescs_add_list_to_cache(get_microdesc_cache(),
+                                   microdescriptors, SAVED_NOWHERE, 0);
+    smartlist_free(added);
+    smartlist_free(microdescriptors);
+  }
+
+  smartlist_free(routers);
+  digestmap_free(omit_as_sybil, NULL);
+
+  /* Apply guardfraction information to routerstatuses. */
+  if (options->GuardfractionFile) {
+    dirserv_read_guardfraction_file(options->GuardfractionFile,
+                                    routerstatuses);
+  }
+
+  /* This pass through applies the measured bw lines to the routerstatuses */
+  if (options->V3BandwidthsFile) {
+    /* Only set bw_file_headers when V3BandwidthsFile is configured */
+    bw_file_headers = smartlist_new();
+    dirserv_read_measured_bandwidths(options->V3BandwidthsFile,
+                                     routerstatuses, bw_file_headers);
+  } else {
+    /*
+     * No bandwidths file; clear the measured bandwidth cache in case we had
+     * one last time around.
+     */
+    if (dirserv_get_measured_bw_cache_size() > 0) {
+      dirserv_clear_measured_bw_cache();
+    }
+  }
+
+  v3_out = tor_malloc_zero(sizeof(networkstatus_t));
+
+  v3_out->type = NS_TYPE_VOTE;
+  dirvote_get_preferred_voting_intervals(&timing);
+  v3_out->published = now;
+  {
+    char tbuf[ISO_TIME_LEN+1];
+    networkstatus_t *current_consensus =
+      networkstatus_get_live_consensus(now);
+    long last_consensus_interval; /* only used to pick a valid_after */
+    if (current_consensus)
+      last_consensus_interval = current_consensus->fresh_until -
+        current_consensus->valid_after;
+    else
+      last_consensus_interval = options->TestingV3AuthInitialVotingInterval;
+    v3_out->valid_after =
+      voting_schedule_get_start_of_next_interval(now,
+                                   (int)last_consensus_interval,
+                                   options->TestingV3AuthVotingStartOffset);
+    format_iso_time(tbuf, v3_out->valid_after);
+    log_notice(LD_DIR,"Choosing valid-after time in vote as %s: "
+               "consensus_set=%d, last_interval=%d",
+               tbuf, current_consensus?1:0, (int)last_consensus_interval);
+  }
+  v3_out->fresh_until = v3_out->valid_after + timing.vote_interval;
+  v3_out->valid_until = v3_out->valid_after +
+    (timing.vote_interval * timing.n_intervals_valid);
+  v3_out->vote_seconds = timing.vote_delay;
+  v3_out->dist_seconds = timing.dist_delay;
+  tor_assert(v3_out->vote_seconds > 0);
+  tor_assert(v3_out->dist_seconds > 0);
+  tor_assert(timing.n_intervals_valid > 0);
+
+  v3_out->client_versions = client_versions;
+  v3_out->server_versions = server_versions;
+
+  /* These are hardwired, to avoid disaster. */
+  v3_out->recommended_relay_protocols =
+    tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
+               "Link=4 Microdesc=1-2 Relay=2");
+  v3_out->recommended_client_protocols =
+    tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
+               "Link=4 Microdesc=1-2 Relay=2");
+  v3_out->required_client_protocols =
+    tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
+               "Link=4 Microdesc=1-2 Relay=2");
+  v3_out->required_relay_protocols =
+    tor_strdup("Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
+               "Link=3-4 Microdesc=1 Relay=1-2");
+
+  /* We are not allowed to vote to require anything we don't have. */
+  tor_assert(protover_all_supported(v3_out->required_relay_protocols, NULL));
+  tor_assert(protover_all_supported(v3_out->required_client_protocols, NULL));
+
+  /* We should not recommend anything we don't have. */
+  tor_assert_nonfatal(protover_all_supported(
+                               v3_out->recommended_relay_protocols, NULL));
+  tor_assert_nonfatal(protover_all_supported(
+                               v3_out->recommended_client_protocols, NULL));
+
+  v3_out->package_lines = smartlist_new();
+  {
+    config_line_t *cl;
+    for (cl = get_options()->RecommendedPackages; cl; cl = cl->next) {
+      if (validate_recommended_package_line(cl->value))
+        smartlist_add_strdup(v3_out->package_lines, cl->value);
+    }
+  }
+
+  v3_out->known_flags = smartlist_new();
+  smartlist_split_string(v3_out->known_flags,
+                         "Authority Exit Fast Guard Stable V2Dir Valid HSDir",
+                         0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  if (vote_on_reachability)
+    smartlist_add_strdup(v3_out->known_flags, "Running");
+  if (listbadexits)
+    smartlist_add_strdup(v3_out->known_flags, "BadExit");
+  smartlist_sort_strings(v3_out->known_flags);
+
+  if (options->ConsensusParams) {
+    v3_out->net_params = smartlist_new();
+    smartlist_split_string(v3_out->net_params,
+                           options->ConsensusParams, NULL, 0, 0);
+    smartlist_sort_strings(v3_out->net_params);
+  }
+  v3_out->bw_file_headers = bw_file_headers;
+
+  voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t));
+  voter->nickname = tor_strdup(options->Nickname);
+  memcpy(voter->identity_digest, identity_digest, DIGEST_LEN);
+  voter->sigs = smartlist_new();
+  voter->address = hostname;
+  voter->addr = addr;
+  voter->dir_port = router_get_advertised_dir_port(options, 0);
+  voter->or_port = router_get_advertised_or_port(options);
+  voter->contact = tor_strdup(contact);
+  if (options->V3AuthUseLegacyKey) {
+    authority_cert_t *c = get_my_v3_legacy_cert();
+    if (c) {
+      if (crypto_pk_get_digest(c->identity_key, voter->legacy_id_digest)) {
+        log_warn(LD_BUG, "Unable to compute digest of legacy v3 identity key");
+        memset(voter->legacy_id_digest, 0, DIGEST_LEN);
+      }
+    }
+  }
+
+  v3_out->voters = smartlist_new();
+  smartlist_add(v3_out->voters, voter);
+  v3_out->cert = authority_cert_dup(cert);
+  v3_out->routerstatus_list = routerstatuses;
+  /* Note: networkstatus_digest is unset; it won't get set until we actually
+   * format the vote. */
+
+  return v3_out;
 }
diff --git a/src/or/dirvote.h b/src/feature/dirauth/dirvote.h
index efd233ef5f..02d88d19d1 100644
--- a/src/or/dirvote.h
+++ b/src/feature/dirauth/dirvote.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,8 +12,6 @@
 #ifndef TOR_DIRVOTE_H
 #define TOR_DIRVOTE_H
 
-#include "testsupport.h"
-
 /*
  * Ideally, assuming synced clocks, we should only need 1 second for each of:
  *  - Vote
@@ -51,57 +49,15 @@
 #define MIN_VOTE_INTERVAL_TESTING_INITIAL \
                 ((MIN_VOTE_SECONDS_TESTING)+(MIN_DIST_SECONDS_TESTING)+1)
 
+/* A placeholder for routerstatus_format_entry() when the consensus method
+ * argument is not applicable. */
+#define ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD 0
+
 /** The lowest consensus method that we currently support. */
-#define MIN_SUPPORTED_CONSENSUS_METHOD 13
+#define MIN_SUPPORTED_CONSENSUS_METHOD 25
 
 /** The highest consensus method that we currently support. */
-#define MAX_SUPPORTED_CONSENSUS_METHOD 25
-
-/** Lowest consensus method where microdesc consensuses omit any entry
- * with no microdesc. */
-#define MIN_METHOD_FOR_MANDATORY_MICRODESC 13
-
-/** Lowest consensus method that contains "a" lines. */
-#define MIN_METHOD_FOR_A_LINES 14
-
-/** Lowest consensus method where microdescs may include a "p6" line. */
-#define MIN_METHOD_FOR_P6_LINES 15
-
-/** Lowest consensus method where microdescs may include an onion-key-ntor
- * line */
-#define MIN_METHOD_FOR_NTOR_KEY 16
-
-/** Lowest consensus method that ensures that authorities output an
- * Unmeasured=1 flag for unmeasured bandwidths */
-#define MIN_METHOD_TO_CLIP_UNMEASURED_BW 17
-
-/** Lowest consensus method where authorities may include an "id" line in
- * microdescriptors. */
-#define MIN_METHOD_FOR_ID_HASH_IN_MD 18
-
-/** Lowest consensus method where we include "package" lines*/
-#define MIN_METHOD_FOR_PACKAGE_LINES 19
-
-/** Lowest consensus method where authorities may include
- * GuardFraction information in microdescriptors. */
-#define MIN_METHOD_FOR_GUARDFRACTION 20
-
-/** Lowest consensus method where authorities may include an "id" line for
- * ed25519 identities in microdescriptors. (Broken; see
- * consensus_method_is_supported() for more info.) */
-#define MIN_METHOD_FOR_ED25519_ID_IN_MD 21
-
-/** Lowest consensus method where authorities vote on ed25519 ids and ensure
- * ed25519 id consistency. */
-#define MIN_METHOD_FOR_ED25519_ID_VOTING 22
-
-/** Lowest consensus method where authorities may include a shared random
- * value(s). */
-#define MIN_METHOD_FOR_SHARED_RANDOM 23
-
-/** Lowest consensus method where authorities drop all nodes that don't get
- * the Valid flag. */
-#define MIN_METHOD_FOR_EXCLUDING_INVALID_NODES 24
+#define MAX_SUPPORTED_CONSENSUS_METHOD 28
 
 /** Lowest consensus method where authorities vote on required/recommended
  * protocols. */
@@ -111,75 +67,47 @@
  * entries. */
 #define MIN_METHOD_FOR_RS_PROTOCOLS 25
 
+/** Lowest consensus method where authorities initialize bandwidth weights to 1
+ * instead of 0. See #14881 */
+#define MIN_METHOD_FOR_INIT_BW_WEIGHTS_ONE 26
+
+/** Lowest consensus method where the microdesc consensus contains relay IPv6
+ * addresses. See #23826 and #20916. */
+#define MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS 27
+
+/** Lowest consensus method where microdescriptors do not contain relay IPv6
+ * addresses. See #23828 and #20916. */
+#define MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC 28
+
 /** Default bandwidth to clip unmeasured bandwidths to using method >=
  * MIN_METHOD_TO_CLIP_UNMEASURED_BW.  (This is not a consensus method; do not
  * get confused with the above macros.) */
 #define DEFAULT_MAX_UNMEASURED_BW_KB 20
 
+/* Directory Get Vote (DGV) flags for dirvote_get_vote(). */
+#define DGV_BY_ID 1
+#define DGV_INCLUDE_PENDING 2
+#define DGV_INCLUDE_PREVIOUS 4
+
+/** Maximum size of a line in a vote. */
+#define MAX_BW_FILE_HEADERS_LINE_LEN 1024
+
+/*
+ * Public API. Used outside of the dirauth subsystem.
+ *
+ * We need to nullify them if the module is disabled.
+ */
+#ifdef HAVE_MODULE_DIRAUTH
+
+time_t dirvote_act(const or_options_t *options, time_t now);
 void dirvote_free_all(void);
 
-/* vote manipulation */
-char *networkstatus_compute_consensus(smartlist_t *votes,
-                                      int total_authorities,
-                                      crypto_pk_t *identity_key,
-                                      crypto_pk_t *signing_key,
-                                      const char *legacy_identity_key_digest,
-                                      crypto_pk_t *legacy_signing_key,
-                                      consensus_flavor_t flavor);
-int networkstatus_add_detached_signatures(networkstatus_t *target,
-                                          ns_detached_signatures_t *sigs,
-                                          const char *source,
-                                          int severity,
-                                          const char **msg_out);
-char *networkstatus_get_detached_signatures(smartlist_t *consensuses);
-void ns_detached_signatures_free(ns_detached_signatures_t *s);
-
-/* cert manipulation */
-authority_cert_t *authority_cert_dup(authority_cert_t *cert);
-
-/* vote scheduling */
-
-/** Scheduling information for a voting interval. */
-typedef struct {
-  /** When do we generate and distribute our vote for this interval? */
-  time_t voting_starts;
-  /** When do we send an HTTP request for any votes that we haven't
-   * been posted yet?*/
-  time_t fetch_missing_votes;
-  /** When do we give up on getting more votes and generate a consensus? */
-  time_t voting_ends;
-  /** When do we send an HTTP request for any signatures we're expecting to
-   * see on the consensus? */
-  time_t fetch_missing_signatures;
-  /** When do we publish the consensus? */
-  time_t interval_starts;
-
-  /* True iff we have generated and distributed our vote. */
-  int have_voted;
-  /* True iff we've requested missing votes. */
-  int have_fetched_missing_votes;
-  /* True iff we have built a consensus and sent the signatures around. */
-  int have_built_consensus;
-  /* True iff we've fetched missing signatures. */
-  int have_fetched_missing_signatures;
-  /* True iff we have published our consensus. */
-  int have_published_consensus;
-} voting_schedule_t;
-
-voting_schedule_t *get_voting_schedule(const or_options_t *options,
-                                       time_t now, int severity);
-
-void voting_schedule_free(voting_schedule_t *voting_schedule_to_free);
-
-void dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out);
-time_t dirvote_get_start_of_next_interval(time_t now,
-                                          int interval,
-                                          int offset);
-void dirvote_recalculate_timing(const or_options_t *options, time_t now);
-void dirvote_act(const or_options_t *options, time_t now);
-time_t get_next_valid_after_time(time_t now);
-
-/* invoked on timers and by outside triggers. */
+void dirvote_parse_sr_commits(networkstatus_t *ns, const smartlist_t *tokens);
+void dirvote_clear_commits(networkstatus_t *ns);
+void dirvote_dirreq_get_status_vote(const char *url, smartlist_t *items,
+                                    smartlist_t *dir_items);
+
+/* Storing signatures and votes functions */
 struct pending_vote_t * dirvote_add_vote(const char *vote_body,
                                          const char **msg_out,
                                          int *status_out);
@@ -187,43 +115,103 @@ int dirvote_add_signatures(const char *detached_signatures_body,
                            const char *source,
                            const char **msg_out);
 
+struct config_line_t;
+char *format_recommended_version_list(const struct config_line_t *line,
+                                      int warn);
+
+#else /* HAVE_MODULE_DIRAUTH */
+
+static inline time_t
+dirvote_act(const or_options_t *options, time_t now)
+{
+  (void) options;
+  (void) now;
+  return TIME_MAX;
+}
+
+static inline void
+dirvote_free_all(void)
+{
+}
+
+static inline void
+dirvote_parse_sr_commits(networkstatus_t *ns, const smartlist_t *tokens)
+{
+  (void) ns;
+  (void) tokens;
+}
+
+static inline void
+dirvote_clear_commits(networkstatus_t *ns)
+{
+  (void) ns;
+}
+
+static inline void
+dirvote_dirreq_get_status_vote(const char *url, smartlist_t *items,
+                               smartlist_t *dir_items)
+{
+  (void) url;
+  (void) items;
+  (void) dir_items;
+}
+
+static inline struct pending_vote_t *
+dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out)
+{
+  (void) vote_body;
+  /* If the dirauth module is disabled, this should NEVER be called else we
+   * failed to safeguard the dirauth module. */
+  tor_assert_nonfatal_unreached();
+
+  /* We need to send out an error code. */
+  *status_out = 400;
+  *msg_out = "No directory authority support";
+  return NULL;
+}
+
+static inline int
+dirvote_add_signatures(const char *detached_signatures_body,
+                       const char *source,
+                       const char **msg_out)
+{
+  (void) detached_signatures_body;
+  (void) source;
+  (void) msg_out;
+  /* If the dirauth module is disabled, this should NEVER be called else we
+   * failed to safeguard the dirauth module. */
+  tor_assert_nonfatal_unreached();
+  return 0;
+}
+
+#endif /* HAVE_MODULE_DIRAUTH */
+
 /* Item access */
 MOCK_DECL(const char*, dirvote_get_pending_consensus,
           (consensus_flavor_t flav));
 MOCK_DECL(const char*, dirvote_get_pending_detached_signatures, (void));
-
-#define DGV_BY_ID 1
-#define DGV_INCLUDE_PENDING 2
-#define DGV_INCLUDE_PREVIOUS 4
 const cached_dir_t *dirvote_get_vote(const char *fp, int flags);
-void set_routerstatus_from_routerinfo(routerstatus_t *rs,
-                                      node_t *node,
-                                      routerinfo_t *ri, time_t now,
-                                      int listbadexits);
+
+/*
+ * API used _only_ by the dirauth subsystem.
+ */
+
 networkstatus_t *
 dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
                                         authority_cert_t *cert);
 
-microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri,
-                                            int consensus_method);
-ssize_t dirvote_format_microdesc_vote_line(char *out, size_t out_len,
-                                           const microdesc_t *md,
-                                           int consensus_method_low,
-                                           int consensus_method_high);
 vote_microdesc_hash_t *dirvote_format_all_microdesc_vote_lines(
                                         const routerinfo_t *ri,
                                         time_t now,
                                         smartlist_t *microdescriptors_out);
 
-int vote_routerstatus_find_microdesc_hash(char *digest256_out,
-                                          const vote_routerstatus_t *vrs,
-                                          int method,
-                                          digest_algorithm_t alg);
-document_signature_t *voter_get_sig_by_algorithm(
-                           const networkstatus_voter_info_t *voter,
-                           digest_algorithm_t alg);
-
+/*
+ * Exposed functions for unit tests.
+ */
 #ifdef DIRVOTE_PRIVATE
+
+/* Cert manipulation */
+STATIC authority_cert_t *authority_cert_dup(authority_cert_t *cert);
 STATIC int32_t dirvote_get_intermediate_param_value(
                                    const smartlist_t *param_list,
                                    const char *keyword,
@@ -234,7 +222,29 @@ STATIC smartlist_t *dirvote_compute_params(smartlist_t *votes, int method,
                              int total_authorities);
 STATIC char *compute_consensus_package_lines(smartlist_t *votes);
 STATIC char *make_consensus_method_list(int low, int high, const char *sep);
-#endif
+STATIC int
+networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
+                                     int64_t M, int64_t E, int64_t D,
+                                     int64_t T, int64_t weight_scale);
+STATIC
+char *networkstatus_compute_consensus(smartlist_t *votes,
+                                      int total_authorities,
+                                      crypto_pk_t *identity_key,
+                                      crypto_pk_t *signing_key,
+                                      const char *legacy_identity_key_digest,
+                                      crypto_pk_t *legacy_signing_key,
+                                      consensus_flavor_t flavor);
+STATIC
+int networkstatus_add_detached_signatures(networkstatus_t *target,
+                                          ns_detached_signatures_t *sigs,
+                                          const char *source,
+                                          int severity,
+                                          const char **msg_out);
+STATIC
+char *networkstatus_get_detached_signatures(smartlist_t *consensuses);
+STATIC microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri,
+                                                   int consensus_method);
 
-#endif
+#endif /* defined(DIRVOTE_PRIVATE) */
 
+#endif /* !defined(TOR_DIRVOTE_H) */
diff --git a/src/feature/dirauth/dsigs_parse.c b/src/feature/dirauth/dsigs_parse.c
new file mode 100644
index 0000000000..d88176fee9
--- /dev/null
+++ b/src/feature/dirauth/dsigs_parse.c
@@ -0,0 +1,282 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dsigs_parse.h
+ * \brief Code to parse and validate detached-signature objects
+ **/
+
+#include "core/or/or.h"
+#include "feature/dirparse/parsecommon.h"
+#include "feature/dirparse/unparseable.h"
+#include "feature/nodelist/networkstatus.h"
+#include "lib/memarea/memarea.h"
+
+#include "feature/dirauth/dsigs_parse.h"
+#include "feature/dirauth/ns_detached_signatures_st.h"
+#include "feature/nodelist/document_signature_st.h"
+
+/** List of tokens recognized in detached networkstatus signature documents. */
+static token_rule_t networkstatus_detached_signature_token_table[] = {
+  T1_START("consensus-digest", K_CONSENSUS_DIGEST, GE(1),       NO_OBJ ),
+  T("additional-digest",       K_ADDITIONAL_DIGEST,GE(3),       NO_OBJ ),
+  T1("valid-after",            K_VALID_AFTER,      CONCAT_ARGS, NO_OBJ ),
+  T1("fresh-until",            K_FRESH_UNTIL,      CONCAT_ARGS, NO_OBJ ),
+  T1("valid-until",            K_VALID_UNTIL,      CONCAT_ARGS, NO_OBJ ),
+  T("additional-signature",  K_ADDITIONAL_SIGNATURE, GE(4),   NEED_OBJ ),
+  T1N("directory-signature", K_DIRECTORY_SIGNATURE,  GE(2),   NEED_OBJ ),
+  END_OF_TABLE
+};
+
+/** Return the common_digests_t that holds the digests of the
+ * <b>flavor_name</b>-flavored networkstatus according to the detached
+ * signatures document <b>sigs</b>, allocating a new common_digests_t as
+ * needed. */
+static common_digests_t *
+detached_get_digests(ns_detached_signatures_t *sigs, const char *flavor_name)
+{
+  common_digests_t *d = strmap_get(sigs->digests, flavor_name);
+  if (!d) {
+    d = tor_malloc_zero(sizeof(common_digests_t));
+    strmap_set(sigs->digests, flavor_name, d);
+  }
+  return d;
+}
+
+/** Return the list of signatures of the <b>flavor_name</b>-flavored
+ * networkstatus according to the detached signatures document <b>sigs</b>,
+ * allocating a new common_digests_t as needed. */
+static smartlist_t *
+detached_get_signatures(ns_detached_signatures_t *sigs,
+                        const char *flavor_name)
+{
+  smartlist_t *sl = strmap_get(sigs->signatures, flavor_name);
+  if (!sl) {
+    sl = smartlist_new();
+    strmap_set(sigs->signatures, flavor_name, sl);
+  }
+  return sl;
+}
+
+/** Parse a detached v3 networkstatus signature document between <b>s</b> and
+ * <b>eos</b> and return the result.  Return -1 on failure. */
+ns_detached_signatures_t *
+networkstatus_parse_detached_signatures(const char *s, const char *eos)
+{
+  /* XXXX there is too much duplicate shared between this function and
+   * networkstatus_parse_vote_from_string(). */
+  directory_token_t *tok;
+  memarea_t *area = NULL;
+  common_digests_t *digests;
+
+  smartlist_t *tokens = smartlist_new();
+  ns_detached_signatures_t *sigs =
+    tor_malloc_zero(sizeof(ns_detached_signatures_t));
+  sigs->digests = strmap_new();
+  sigs->signatures = strmap_new();
+
+  if (!eos)
+    eos = s + strlen(s);
+
+  area = memarea_new();
+  if (tokenize_string(area,s, eos, tokens,
+                      networkstatus_detached_signature_token_table, 0)) {
+    log_warn(LD_DIR, "Error tokenizing detached networkstatus signatures");
+    goto err;
+  }
+
+  /* Grab all the digest-like tokens. */
+  SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) {
+    const char *algname;
+    digest_algorithm_t alg;
+    const char *flavor;
+    const char *hexdigest;
+    size_t expected_length, digest_length;
+
+    tok = _tok;
+
+    if (tok->tp == K_CONSENSUS_DIGEST) {
+      algname = "sha1";
+      alg = DIGEST_SHA1;
+      flavor = "ns";
+      hexdigest = tok->args[0];
+    } else if (tok->tp == K_ADDITIONAL_DIGEST) {
+      int a = crypto_digest_algorithm_parse_name(tok->args[1]);
+      if (a<0) {
+        log_warn(LD_DIR, "Unrecognized algorithm name %s", tok->args[0]);
+        continue;
+      }
+      alg = (digest_algorithm_t) a;
+      flavor = tok->args[0];
+      algname = tok->args[1];
+      hexdigest = tok->args[2];
+    } else {
+      continue;
+    }
+
+    digest_length = crypto_digest_algorithm_get_length(alg);
+    expected_length = digest_length * 2; /* hex encoding */
+
+    if (strlen(hexdigest) != expected_length) {
+      log_warn(LD_DIR, "Wrong length on consensus-digest in detached "
+               "networkstatus signatures");
+      goto err;
+    }
+    digests = detached_get_digests(sigs, flavor);
+    tor_assert(digests);
+    if (!tor_mem_is_zero(digests->d[alg], digest_length)) {
+      log_warn(LD_DIR, "Multiple digests for %s with %s on detached "
+               "signatures document", flavor, algname);
+      continue;
+    }
+    if (base16_decode(digests->d[alg], digest_length,
+                      hexdigest, strlen(hexdigest)) != (int) digest_length) {
+      log_warn(LD_DIR, "Bad encoding on consensus-digest in detached "
+               "networkstatus signatures");
+      goto err;
+    }
+  } SMARTLIST_FOREACH_END(_tok);
+
+  tok = find_by_keyword(tokens, K_VALID_AFTER);
+  if (parse_iso_time(tok->args[0], &sigs->valid_after)) {
+    log_warn(LD_DIR, "Bad valid-after in detached networkstatus signatures");
+    goto err;
+  }
+
+  tok = find_by_keyword(tokens, K_FRESH_UNTIL);
+  if (parse_iso_time(tok->args[0], &sigs->fresh_until)) {
+    log_warn(LD_DIR, "Bad fresh-until in detached networkstatus signatures");
+    goto err;
+  }
+
+  tok = find_by_keyword(tokens, K_VALID_UNTIL);
+  if (parse_iso_time(tok->args[0], &sigs->valid_until)) {
+    log_warn(LD_DIR, "Bad valid-until in detached networkstatus signatures");
+    goto err;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) {
+    const char *id_hexdigest;
+    const char *sk_hexdigest;
+    const char *algname;
+    const char *flavor;
+    digest_algorithm_t alg;
+
+    char id_digest[DIGEST_LEN];
+    char sk_digest[DIGEST_LEN];
+    smartlist_t *siglist;
+    document_signature_t *sig;
+    int is_duplicate;
+
+    tok = _tok;
+    if (tok->tp == K_DIRECTORY_SIGNATURE) {
+      tor_assert(tok->n_args >= 2);
+      flavor = "ns";
+      algname = "sha1";
+      id_hexdigest = tok->args[0];
+      sk_hexdigest = tok->args[1];
+    } else if (tok->tp == K_ADDITIONAL_SIGNATURE) {
+      tor_assert(tok->n_args >= 4);
+      flavor = tok->args[0];
+      algname = tok->args[1];
+      id_hexdigest = tok->args[2];
+      sk_hexdigest = tok->args[3];
+    } else {
+      continue;
+    }
+
+    {
+      int a = crypto_digest_algorithm_parse_name(algname);
+      if (a<0) {
+        log_warn(LD_DIR, "Unrecognized algorithm name %s", algname);
+        continue;
+      }
+      alg = (digest_algorithm_t) a;
+    }
+
+    if (!tok->object_type ||
+        strcmp(tok->object_type, "SIGNATURE") ||
+        tok->object_size < 128 || tok->object_size > 512) {
+      log_warn(LD_DIR, "Bad object type or length on directory-signature");
+      goto err;
+    }
+
+    if (strlen(id_hexdigest) != HEX_DIGEST_LEN ||
+        base16_decode(id_digest, sizeof(id_digest),
+                      id_hexdigest, HEX_DIGEST_LEN) != sizeof(id_digest)) {
+      log_warn(LD_DIR, "Error decoding declared identity %s in "
+               "network-status vote.", escaped(id_hexdigest));
+      goto err;
+    }
+    if (strlen(sk_hexdigest) != HEX_DIGEST_LEN ||
+        base16_decode(sk_digest, sizeof(sk_digest),
+                      sk_hexdigest, HEX_DIGEST_LEN) != sizeof(sk_digest)) {
+      log_warn(LD_DIR, "Error decoding declared signing key digest %s in "
+               "network-status vote.", escaped(sk_hexdigest));
+      goto err;
+    }
+
+    siglist = detached_get_signatures(sigs, flavor);
+    is_duplicate = 0;
+    SMARTLIST_FOREACH(siglist, document_signature_t *, dsig, {
+      if (dsig->alg == alg &&
+          tor_memeq(id_digest, dsig->identity_digest, DIGEST_LEN) &&
+          tor_memeq(sk_digest, dsig->signing_key_digest, DIGEST_LEN)) {
+        is_duplicate = 1;
+      }
+    });
+    if (is_duplicate) {
+      log_warn(LD_DIR, "Two signatures with identical keys and algorithm "
+               "found.");
+      continue;
+    }
+
+    sig = tor_malloc_zero(sizeof(document_signature_t));
+    sig->alg = alg;
+    memcpy(sig->identity_digest, id_digest, DIGEST_LEN);
+    memcpy(sig->signing_key_digest, sk_digest, DIGEST_LEN);
+    if (tok->object_size >= INT_MAX || tok->object_size >= SIZE_T_CEILING) {
+      tor_free(sig);
+      goto err;
+    }
+    sig->signature = tor_memdup(tok->object_body, tok->object_size);
+    sig->signature_len = (int) tok->object_size;
+
+    smartlist_add(siglist, sig);
+  } SMARTLIST_FOREACH_END(_tok);
+
+  goto done;
+ err:
+  ns_detached_signatures_free(sigs);
+  sigs = NULL;
+ done:
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_free(tokens);
+  if (area) {
+    DUMP_AREA(area, "detached signatures");
+    memarea_drop_all(area);
+  }
+  return sigs;
+}
+
+/** Release all storage held in <b>s</b>. */
+void
+ns_detached_signatures_free_(ns_detached_signatures_t *s)
+{
+  if (!s)
+    return;
+  if (s->signatures) {
+    STRMAP_FOREACH(s->signatures, flavor, smartlist_t *, sigs) {
+      SMARTLIST_FOREACH(sigs, document_signature_t *, sig,
+                        document_signature_free(sig));
+      smartlist_free(sigs);
+    } STRMAP_FOREACH_END;
+    strmap_free(s->signatures, NULL);
+    strmap_free(s->digests, tor_free_);
+  }
+
+  tor_free(s);
+}
diff --git a/src/feature/dirauth/dsigs_parse.h b/src/feature/dirauth/dsigs_parse.h
new file mode 100644
index 0000000000..fec51ba488
--- /dev/null
+++ b/src/feature/dirauth/dsigs_parse.h
@@ -0,0 +1,22 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dsigs_parse.h
+ * \brief Header file for dsigs_parse.c.
+ **/
+
+#ifndef TOR_DSIGS_PARSE_H
+#define TOR_DSIGS_PARSE_H
+
+ns_detached_signatures_t *networkstatus_parse_detached_signatures(
+                                      const char *s, const char *eos);
+
+void ns_detached_signatures_free_(ns_detached_signatures_t *s);
+#define ns_detached_signatures_free(s) \
+  FREE_AND_NULL(ns_detached_signatures_t, ns_detached_signatures_free_, (s))
+
+#endif
diff --git a/src/feature/dirauth/guardfraction.c b/src/feature/dirauth/guardfraction.c
new file mode 100644
index 0000000000..d1a7f194d4
--- /dev/null
+++ b/src/feature/dirauth/guardfraction.c
@@ -0,0 +1,333 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file bwauth.c
+ * \brief Code to read and apply guard fraction data.
+ **/
+
+#define GUARDFRACTION_PRIVATE
+#include "core/or/or.h"
+#include "feature/dirauth/guardfraction.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/dirparse/ns_parse.h"
+
+#include "feature/nodelist/vote_routerstatus_st.h"
+
+#include "lib/encoding/confline.h"
+
+/** The guardfraction of the guard with identity fingerprint <b>guard_id</b>
+ *  is <b>guardfraction_percentage</b>. See if we have a vote routerstatus for
+ *  this guard in <b>vote_routerstatuses</b>, and if we do, register the
+ *  information to it.
+ *
+ *  Return 1 if we applied the information and 0 if we couldn't find a
+ *  matching guard.
+ *
+ * Requires that <b>vote_routerstatuses</b> be sorted.
+ */
+static int
+guardfraction_line_apply(const char *guard_id,
+                      uint32_t guardfraction_percentage,
+                      smartlist_t *vote_routerstatuses)
+{
+  vote_routerstatus_t *vrs = NULL;
+
+  tor_assert(vote_routerstatuses);
+
+  vrs = smartlist_bsearch(vote_routerstatuses, guard_id,
+                         compare_digest_to_vote_routerstatus_entry);
+
+  if (!vrs) {
+    return 0;
+  }
+
+  vrs->status.has_guardfraction = 1;
+  vrs->status.guardfraction_percentage = guardfraction_percentage;
+
+  return 1;
+}
+
+/* Given a guard line from a guardfraction file, parse it and register
+ * its information to <b>vote_routerstatuses</b>.
+ *
+ * Return:
+ * * 1 if the line was proper and its information got registered.
+ * * 0 if the line was proper but no currently active guard was found
+ *     to register the guardfraction information to.
+ * * -1 if the line could not be parsed and set <b>err_msg</b> to a
+      newly allocated string containing the error message.
+ */
+static int
+guardfraction_file_parse_guard_line(const char *guard_line,
+                                    smartlist_t *vote_routerstatuses,
+                                    char **err_msg)
+{
+  char guard_id[DIGEST_LEN];
+  uint32_t guardfraction;
+  char *inputs_tmp = NULL;
+  int num_ok = 1;
+
+  smartlist_t *sl = smartlist_new();
+  int retval = -1;
+
+  tor_assert(err_msg);
+
+  /* guard_line should contain something like this:
+     <hex digest> <guardfraction> <appearances> */
+  smartlist_split_string(sl, guard_line, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3);
+  if (smartlist_len(sl) < 3) {
+    tor_asprintf(err_msg, "bad line '%s'", guard_line);
+    goto done;
+  }
+
+  inputs_tmp = smartlist_get(sl, 0);
+  if (strlen(inputs_tmp) != HEX_DIGEST_LEN ||
+      base16_decode(guard_id, DIGEST_LEN,
+                    inputs_tmp, HEX_DIGEST_LEN) != DIGEST_LEN) {
+    tor_asprintf(err_msg, "bad digest '%s'", inputs_tmp);
+    goto done;
+  }
+
+  inputs_tmp = smartlist_get(sl, 1);
+  /* Guardfraction is an integer in [0, 100]. */
+  guardfraction =
+    (uint32_t) tor_parse_long(inputs_tmp, 10, 0, 100, &num_ok, NULL);
+  if (!num_ok) {
+    tor_asprintf(err_msg, "wrong percentage '%s'", inputs_tmp);
+    goto done;
+  }
+
+  /* If routerstatuses were provided, apply this info to actual routers. */
+  if (vote_routerstatuses) {
+    retval = guardfraction_line_apply(guard_id, guardfraction,
+                                      vote_routerstatuses);
+  } else {
+    retval = 0; /* If we got this far, line was correctly formatted. */
+  }
+
+ done:
+
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_free(sl);
+
+  return retval;
+}
+
+/** Given an inputs line from a guardfraction file, parse it and
+ *  register its information to <b>total_consensuses</b> and
+ *  <b>total_days</b>.
+ *
+ *  Return 0 if it parsed well. Return -1 if there was an error, and
+ *  set <b>err_msg</b> to a newly allocated string containing the
+ *  error message.
+ */
+static int
+guardfraction_file_parse_inputs_line(const char *inputs_line,
+                                     int *total_consensuses,
+                                     int *total_days,
+                                     char **err_msg)
+{
+  int retval = -1;
+  char *inputs_tmp = NULL;
+  int num_ok = 1;
+  smartlist_t *sl = smartlist_new();
+
+  tor_assert(err_msg);
+
+  /* Second line is inputs information:
+   *   n-inputs <total_consensuses> <total_days>. */
+  smartlist_split_string(sl, inputs_line, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3);
+  if (smartlist_len(sl) < 2) {
+    tor_asprintf(err_msg, "incomplete line '%s'", inputs_line);
+    goto done;
+  }
+
+  inputs_tmp = smartlist_get(sl, 0);
+  *total_consensuses =
+    (int) tor_parse_long(inputs_tmp, 10, 0, INT_MAX, &num_ok, NULL);
+  if (!num_ok) {
+    tor_asprintf(err_msg, "unparseable consensus '%s'", inputs_tmp);
+    goto done;
+  }
+
+  inputs_tmp = smartlist_get(sl, 1);
+  *total_days =
+    (int) tor_parse_long(inputs_tmp, 10, 0, INT_MAX, &num_ok, NULL);
+  if (!num_ok) {
+    tor_asprintf(err_msg, "unparseable days '%s'", inputs_tmp);
+    goto done;
+  }
+
+  retval = 0;
+
+ done:
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_free(sl);
+
+  return retval;
+}
+
+/* Maximum age of a guardfraction file that we are willing to accept. */
+#define MAX_GUARDFRACTION_FILE_AGE (7*24*60*60) /* approx a week */
+
+/** Static strings of guardfraction files. */
+#define GUARDFRACTION_DATE_STR "written-at"
+#define GUARDFRACTION_INPUTS "n-inputs"
+#define GUARDFRACTION_GUARD "guard-seen"
+#define GUARDFRACTION_VERSION "guardfraction-file-version"
+
+/** Given a guardfraction file in a string, parse it and register the
+ *  guardfraction information to the provided vote routerstatuses.
+ *
+ *  This is the rough format of the guardfraction file:
+ *
+ *      guardfraction-file-version 1
+ *      written-at <date and time>
+ *      n-inputs <number of consesuses parsed> <number of days considered>
+ *
+ *      guard-seen <fpr 1> <guardfraction percentage> <consensus appearances>
+ *      guard-seen <fpr 2> <guardfraction percentage> <consensus appearances>
+ *      guard-seen <fpr 3> <guardfraction percentage> <consensus appearances>
+ *      guard-seen <fpr 4> <guardfraction percentage> <consensus appearances>
+ *      guard-seen <fpr 5> <guardfraction percentage> <consensus appearances>
+ *      ...
+ *
+ *  Return -1 if the parsing failed and 0 if it went smoothly. Parsing
+ *  should tolerate errors in all lines but the written-at header.
+ */
+STATIC int
+dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str,
+                                      smartlist_t *vote_routerstatuses)
+{
+  config_line_t *front=NULL, *line;
+  int ret_tmp;
+  int retval = -1;
+  int current_line_n = 0; /* line counter for better log messages */
+
+  /* Guardfraction info to be parsed */
+  int total_consensuses = 0;
+  int total_days = 0;
+
+  /* Stats */
+  int guards_read_n = 0;
+  int guards_applied_n = 0;
+
+  /* Parse file and split it in lines */
+  ret_tmp = config_get_lines(guardfraction_file_str, &front, 0);
+  if (ret_tmp < 0) {
+    log_warn(LD_CONFIG, "Error reading from guardfraction file");
+    goto done;
+  }
+
+  /* Sort routerstatuses (needed later when applying guardfraction info) */
+  if (vote_routerstatuses)
+    smartlist_sort(vote_routerstatuses, compare_vote_routerstatus_entries);
+
+  for (line = front; line; line=line->next) {
+    current_line_n++;
+
+    if (!strcmp(line->key, GUARDFRACTION_VERSION)) {
+      int num_ok = 1;
+      unsigned int version;
+
+      version =
+        (unsigned int) tor_parse_long(line->value,
+                                      10, 0, INT_MAX, &num_ok, NULL);
+
+      if (!num_ok || version != 1) {
+        log_warn(LD_GENERAL, "Got unknown guardfraction version %d.", version);
+        goto done;
+      }
+    } else if (!strcmp(line->key, GUARDFRACTION_DATE_STR)) {
+      time_t file_written_at;
+      time_t now = time(NULL);
+
+      /* First line is 'written-at <date>' */
+      if (parse_iso_time(line->value, &file_written_at) < 0) {
+        log_warn(LD_CONFIG, "Guardfraction:%d: Bad date '%s'. Ignoring",
+                 current_line_n, line->value);
+        goto done; /* don't tolerate failure here. */
+      }
+      if (file_written_at < now - MAX_GUARDFRACTION_FILE_AGE) {
+        log_warn(LD_CONFIG, "Guardfraction:%d: was written very long ago '%s'",
+                 current_line_n, line->value);
+        goto done; /* don't tolerate failure here. */
+      }
+    } else if (!strcmp(line->key, GUARDFRACTION_INPUTS)) {
+      char *err_msg = NULL;
+
+      if (guardfraction_file_parse_inputs_line(line->value,
+                                               &total_consensuses,
+                                               &total_days,
+                                               &err_msg) < 0) {
+        log_warn(LD_CONFIG, "Guardfraction:%d: %s",
+                 current_line_n, err_msg);
+        tor_free(err_msg);
+        continue;
+      }
+
+    } else if (!strcmp(line->key, GUARDFRACTION_GUARD)) {
+      char *err_msg = NULL;
+
+      ret_tmp = guardfraction_file_parse_guard_line(line->value,
+                                                    vote_routerstatuses,
+                                                    &err_msg);
+      if (ret_tmp < 0) { /* failed while parsing the guard line */
+        log_warn(LD_CONFIG, "Guardfraction:%d: %s",
+                 current_line_n, err_msg);
+        tor_free(err_msg);
+        continue;
+      }
+
+      /* Successfully parsed guard line. Check if it was applied properly. */
+      guards_read_n++;
+      if (ret_tmp > 0) {
+        guards_applied_n++;
+      }
+    } else {
+      log_warn(LD_CONFIG, "Unknown guardfraction line %d (%s %s)",
+               current_line_n, line->key, line->value);
+    }
+  }
+
+  retval = 0;
+
+  log_info(LD_CONFIG,
+           "Successfully parsed guardfraction file with %d consensuses over "
+           "%d days. Parsed %d nodes and applied %d of them%s.",
+           total_consensuses, total_days, guards_read_n, guards_applied_n,
+           vote_routerstatuses ? "" : " (no routerstatus provided)" );
+
+ done:
+  config_free_lines(front);
+
+  if (retval < 0) {
+    return retval;
+  } else {
+    return guards_read_n;
+  }
+}
+
+/** Read a guardfraction file at <b>fname</b> and load all its
+ *  information to <b>vote_routerstatuses</b>. */
+int
+dirserv_read_guardfraction_file(const char *fname,
+                             smartlist_t *vote_routerstatuses)
+{
+  char *guardfraction_file_str;
+
+  /* Read file to a string */
+  guardfraction_file_str = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
+  if (!guardfraction_file_str) {
+      log_warn(LD_FS, "Cannot open guardfraction file '%s'. Failing.", fname);
+      return -1;
+  }
+
+  return dirserv_read_guardfraction_file_from_str(guardfraction_file_str,
+                                               vote_routerstatuses);
+}
diff --git a/src/feature/dirauth/guardfraction.h b/src/feature/dirauth/guardfraction.h
new file mode 100644
index 0000000000..72404907a4
--- /dev/null
+++ b/src/feature/dirauth/guardfraction.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file guardfraction.h
+ * \brief Header file for guardfraction.c
+ **/
+
+#ifndef TOR_GUARDFRACTION_H
+#define TOR_GUARDFRACTION_H
+
+#ifdef GUARDFRACTION_PRIVATE
+STATIC int
+dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str,
+                                      smartlist_t *vote_routerstatuses);
+#endif /* defined(DIRSERV_PRIVATE) */
+
+int dirserv_read_guardfraction_file(const char *fname,
+                                 smartlist_t *vote_routerstatuses);
+
+#endif
diff --git a/src/or/keypin.c b/src/feature/dirauth/keypin.c
index 2d4c4e92d2..667feb2c03 100644
--- a/src/or/keypin.c
+++ b/src/feature/dirauth/keypin.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,17 +11,28 @@
 #define KEYPIN_PRIVATE
 
 #include "orconfig.h"
-#include "compat.h"
-#include "crypto.h"
-#include "crypto_format.h"
-#include "di_ops.h"
+
+#include "lib/cc/torint.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/encoding/binascii.h"
+#include "lib/encoding/time_fmt.h"
+#include "lib/fdio/fdio.h"
+#include "lib/fs/files.h"
+#include "lib/fs/mmap.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/string/printf.h"
+#include "lib/wallclock/approx_time.h"
+
 #include "ht.h"
-#include "keypin.h"
+#include "feature/dirauth/keypin.h"
+
 #include "siphash.h"
-#include "torint.h"
-#include "torlog.h"
-#include "util.h"
-#include "util_format.h"
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
@@ -34,6 +45,10 @@
 #include <io.h>
 #endif
 
+#include <errno.h>
+#include <string.h>
+#include <stdlib.h>
+
 /**
  * @file keypin.c
  * @brief Key-pinning for RSA and Ed25519 identity keys at directory
@@ -289,8 +304,10 @@ static int keypin_journal_fd = -1;
 int
 keypin_open_journal(const char *fname)
 {
-  /* O_SYNC ??*/
-  int fd = tor_open_cloexec(fname, O_WRONLY|O_CREAT|O_BINARY, 0600);
+#ifndef O_SYNC
+#define O_SYNC 0
+#endif
+  int fd = tor_open_cloexec(fname, O_WRONLY|O_CREAT|O_BINARY|O_SYNC, 0600);
   if (fd < 0)
     goto err;
 
@@ -306,7 +323,7 @@ keypin_open_journal(const char *fname)
   char tbuf[ISO_TIME_LEN+1];
   format_iso_time(tbuf, approx_time());
   tor_snprintf(buf, sizeof(buf), "@opened-at %s\n", tbuf);
-  if (write_all(fd, buf, strlen(buf), 0) < 0)
+  if (write_all_to_fd(fd, buf, strlen(buf)) < 0)
     goto err;
 
   keypin_journal_fd = fd;
@@ -345,7 +362,7 @@ keypin_journal_append_entry(const uint8_t *rsa_id_digest,
                       (const char*)ed25519_id_key);
   line[BASE64_DIGEST_LEN+1+BASE64_DIGEST256_LEN] = '\n';
 
-  if (write_all(keypin_journal_fd, line, JOURNAL_LINE_LEN, 0)<0) {
+  if (write_all_to_fd(keypin_journal_fd, line, JOURNAL_LINE_LEN)<0) {
     log_warn(LD_DIRSERV, "Error while adding a line to the key-pinning "
              "journal: %s", strerror(errno));
     keypin_close_journal();
@@ -417,10 +434,11 @@ keypin_load_journal_impl(const char *data, size_t size)
     ++n_entries;
   }
 
-  int severity = (n_corrupt_lines || n_duplicates) ? LOG_WARN : LOG_INFO;
+  int severity = (n_corrupt_lines || n_duplicates) ? LOG_NOTICE : LOG_INFO;
   tor_log(severity, LD_DIRSERV,
           "Loaded %d entries from keypin journal. "
-          "Found %d corrupt lines, %d duplicates, and %d conflicts.",
+          "Found %d corrupt lines (ignored), %d duplicates (harmless), "
+          "and %d conflicts (resolved in favor or more recent entry).",
           n_entries, n_corrupt_lines, n_duplicates, n_conflicts);
 
   return 0;
@@ -495,4 +513,3 @@ keypin_clear(void)
              bad_entries);
   }
 }
-
diff --git a/src/or/keypin.h b/src/feature/dirauth/keypin.h
index 673f24d9e3..722b6ca5fc 100644
--- a/src/or/keypin.h
+++ b/src/feature/dirauth/keypin.h
@@ -1,10 +1,10 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_KEYPIN_H
 #define TOR_KEYPIN_H
 
-#include "testsupport.h"
+#include "lib/testsupport/testsupport.h"
 
 int keypin_check_and_add(const uint8_t *rsa_id_digest,
                          const uint8_t *ed25519_id_key,
@@ -41,7 +41,7 @@ STATIC keypin_ent_t * keypin_parse_journal_line(const char *cp);
 STATIC int keypin_load_journal_impl(const char *data, size_t size);
 
 MOCK_DECL(STATIC void, keypin_add_entry_to_map, (keypin_ent_t *ent));
-#endif
+#endif /* defined(KEYPIN_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_KEYPIN_H) */
 
diff --git a/src/feature/dirauth/ns_detached_signatures_st.h b/src/feature/dirauth/ns_detached_signatures_st.h
new file mode 100644
index 0000000000..0f92be2f0d
--- /dev/null
+++ b/src/feature/dirauth/ns_detached_signatures_st.h
@@ -0,0 +1,22 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef NS_DETACHED_SIGNATURES_ST_H
+#define NS_DETACHED_SIGNATURES_ST_H
+
+/** A set of signatures for a networkstatus consensus.  Unless otherwise
+ * noted, all fields are as for networkstatus_t. */
+struct ns_detached_signatures_t {
+  time_t valid_after;
+  time_t fresh_until;
+  time_t valid_until;
+  strmap_t *digests; /**< Map from flavor name to digestset_t */
+  strmap_t *signatures; /**< Map from flavor name to list of
+                         * document_signature_t */
+};
+
+#endif
+
diff --git a/src/feature/dirauth/process_descs.c b/src/feature/dirauth/process_descs.c
new file mode 100644
index 0000000000..21b8e239ec
--- /dev/null
+++ b/src/feature/dirauth/process_descs.c
@@ -0,0 +1,839 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_descs.c
+ * \brief Make decisions about uploaded descriptors
+ *
+ * Authorities use the code in this module to decide what to do with just-
+ * uploaded descriptors, and to manage the fingerprint file that helps
+ * them make those decisions.
+ **/
+
+#include "core/or/or.h"
+#include "feature/dirauth/process_descs.h"
+
+#include "app/config/config.h"
+#include "core/or/policies.h"
+#include "core/or/versions.h"
+#include "feature/dirauth/keypin.h"
+#include "feature/dirauth/reachability.h"
+#include "feature/dirclient/dlstatus.h"
+#include "feature/dircommon/directory.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerinfo.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/dirparse/routerparse.h"
+#include "feature/nodelist/torcert.h"
+#include "feature/relay/router.h"
+
+#include "core/or/tor_version_st.h"
+#include "feature/nodelist/extrainfo_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+#include "lib/encoding/confline.h"
+
+/** How far in the future do we allow a router to get? (seconds) */
+#define ROUTER_ALLOW_SKEW (60*60*12)
+
+static void directory_remove_invalid(void);
+struct authdir_config_t;
+static was_router_added_t dirserv_add_extrainfo(extrainfo_t *ei,
+                                                const char **msg);
+static uint32_t
+dirserv_get_status_impl(const char *fp, const char *nickname,
+                        uint32_t addr, uint16_t or_port,
+                        const char *platform, const char **msg,
+                        int severity);
+
+/*                 1  Historically used to indicate Named */
+#define FP_INVALID 2  /**< Believed invalid. */
+#define FP_REJECT  4  /**< We will not publish this router. */
+/*                 8  Historically used to avoid using this as a dir. */
+#define FP_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */
+/*                 32 Historically used to indicade Unnamed */
+
+/** Target of status_by_digest map. */
+typedef uint32_t router_status_t;
+
+static void add_fingerprint_to_dir(const char *fp,
+                                   struct authdir_config_t *list,
+                                   router_status_t add_status);
+
+/** List of nickname-\>identity fingerprint mappings for all the routers
+ * that we name.  Used to prevent router impersonation. */
+typedef struct authdir_config_t {
+  strmap_t *fp_by_name; /**< Map from lc nickname to fingerprint. */
+  digestmap_t *status_by_digest; /**< Map from digest to router_status_t. */
+} authdir_config_t;
+
+/** Should be static; exposed for testing. */
+static authdir_config_t *fingerprint_list = NULL;
+
+/** Allocate and return a new, empty, authdir_config_t. */
+static authdir_config_t *
+authdir_config_new(void)
+{
+  authdir_config_t *list = tor_malloc_zero(sizeof(authdir_config_t));
+  list->fp_by_name = strmap_new();
+  list->status_by_digest = digestmap_new();
+  return list;
+}
+
+/** Add the fingerprint <b>fp</b> to the smartlist of fingerprint_entry_t's
+ * <b>list</b>, or-ing the currently set status flags with
+ * <b>add_status</b>.
+ */
+/* static */ void
+add_fingerprint_to_dir(const char *fp, authdir_config_t *list,
+                       router_status_t add_status)
+{
+  char *fingerprint;
+  char d[DIGEST_LEN];
+  router_status_t *status;
+  tor_assert(fp);
+  tor_assert(list);
+
+  fingerprint = tor_strdup(fp);
+  tor_strstrip(fingerprint, " ");
+  if (base16_decode(d, DIGEST_LEN,
+                    fingerprint, strlen(fingerprint)) != DIGEST_LEN) {
+    log_warn(LD_DIRSERV, "Couldn't decode fingerprint \"%s\"",
+             escaped(fp));
+    tor_free(fingerprint);
+    return;
+  }
+
+  status = digestmap_get(list->status_by_digest, d);
+  if (!status) {
+    status = tor_malloc_zero(sizeof(router_status_t));
+    digestmap_set(list->status_by_digest, d, status);
+  }
+
+  tor_free(fingerprint);
+  *status |= add_status;
+  return;
+}
+
+/** Add the fingerprint for this OR to the global list of recognized
+ * identity key fingerprints. */
+int
+dirserv_add_own_fingerprint(crypto_pk_t *pk)
+{
+  char fp[FINGERPRINT_LEN+1];
+  if (crypto_pk_get_fingerprint(pk, fp, 0)<0) {
+    log_err(LD_BUG, "Error computing fingerprint");
+    return -1;
+  }
+  if (!fingerprint_list)
+    fingerprint_list = authdir_config_new();
+  add_fingerprint_to_dir(fp, fingerprint_list, 0);
+  return 0;
+}
+
+/** Load the nickname-\>fingerprint mappings stored in the approved-routers
+ * file.  The file format is line-based, with each non-blank holding one
+ * nickname, some space, and a fingerprint for that nickname.  On success,
+ * replace the current fingerprint list with the new list and return 0.  On
+ * failure, leave the current fingerprint list untouched, and return -1. */
+int
+dirserv_load_fingerprint_file(void)
+{
+  char *fname;
+  char *cf;
+  char *nickname, *fingerprint;
+  authdir_config_t *fingerprint_list_new;
+  int result;
+  config_line_t *front=NULL, *list;
+
+  fname = get_datadir_fname("approved-routers");
+  log_info(LD_GENERAL,
+           "Reloading approved fingerprints from \"%s\"...", fname);
+
+  cf = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
+  if (!cf) {
+    log_warn(LD_FS, "Cannot open fingerprint file '%s'. That's ok.", fname);
+    tor_free(fname);
+    return 0;
+  }
+  tor_free(fname);
+
+  result = config_get_lines(cf, &front, 0);
+  tor_free(cf);
+  if (result < 0) {
+    log_warn(LD_CONFIG, "Error reading from fingerprint file");
+    return -1;
+  }
+
+  fingerprint_list_new = authdir_config_new();
+
+  for (list=front; list; list=list->next) {
+    char digest_tmp[DIGEST_LEN];
+    router_status_t add_status = 0;
+    nickname = list->key; fingerprint = list->value;
+    tor_strstrip(fingerprint, " "); /* remove spaces */
+    if (strlen(fingerprint) != HEX_DIGEST_LEN ||
+        base16_decode(digest_tmp, sizeof(digest_tmp),
+                      fingerprint, HEX_DIGEST_LEN) != sizeof(digest_tmp)) {
+      log_notice(LD_CONFIG,
+                 "Invalid fingerprint (nickname '%s', "
+                 "fingerprint %s). Skipping.",
+                 nickname, fingerprint);
+      continue;
+    }
+    if (!strcasecmp(nickname, "!reject")) {
+        add_status = FP_REJECT;
+    } else if (!strcasecmp(nickname, "!badexit")) {
+        add_status = FP_BADEXIT;
+    } else if (!strcasecmp(nickname, "!invalid")) {
+        add_status = FP_INVALID;
+    }
+    add_fingerprint_to_dir(fingerprint, fingerprint_list_new, add_status);
+  }
+
+  config_free_lines(front);
+  dirserv_free_fingerprint_list();
+  fingerprint_list = fingerprint_list_new;
+  /* Delete any routers whose fingerprints we no longer recognize */
+  directory_remove_invalid();
+  return 0;
+}
+
+/* If this is set, then we don't allow routers that have advertised an Ed25519
+ * identity to stop doing so.  This is going to be essential for good identity
+ * security: otherwise anybody who can attack RSA-1024 but not Ed25519 could
+ * just sign fake descriptors missing the Ed25519 key.  But we won't actually
+ * be able to prevent that kind of thing until we're confident that there isn't
+ * actually a legit reason to downgrade to 0.2.5.  Now we are not recommending
+ * 0.2.5 anymore so there is no reason to keep the #undef.
+ */
+
+#define DISABLE_DISABLING_ED25519
+
+/** Check whether <b>router</b> has a nickname/identity key combination that
+ * we recognize from the fingerprint list, or an IP we automatically act on
+ * according to our configuration.  Return the appropriate router status.
+ *
+ * If the status is 'FP_REJECT' and <b>msg</b> is provided, set
+ * *<b>msg</b> to an explanation of why. */
+uint32_t
+dirserv_router_get_status(const routerinfo_t *router, const char **msg,
+                          int severity)
+{
+  char d[DIGEST_LEN];
+  const int key_pinning = get_options()->AuthDirPinKeys;
+
+  if (crypto_pk_get_digest(router->identity_pkey, d)) {
+    log_warn(LD_BUG,"Error computing fingerprint");
+    if (msg)
+      *msg = "Bug: Error computing fingerprint";
+    return FP_REJECT;
+  }
+
+  /* Check for the more usual versions to reject a router first. */
+  const uint32_t r = dirserv_get_status_impl(d, router->nickname,
+                                             router->addr, router->or_port,
+                                             router->platform, msg, severity);
+  if (r)
+    return r;
+
+  /* dirserv_get_status_impl already rejects versions older than 0.2.4.18-rc,
+   * and onion_curve25519_pkey was introduced in 0.2.4.8-alpha.
+   * But just in case a relay doesn't provide or lies about its version, or
+   * doesn't include an ntor key in its descriptor, check that it exists,
+   * and is non-zero (clients check that it's non-zero before using it). */
+  if (!routerinfo_has_curve25519_onion_key(router)) {
+    log_fn(severity, LD_DIR,
+           "Descriptor from router %s is missing an ntor curve25519 onion "
+           "key.", router_describe(router));
+    if (msg)
+      *msg = "Missing ntor curve25519 onion key. Please upgrade!";
+    return FP_REJECT;
+  }
+
+  if (router->cache_info.signing_key_cert) {
+    /* This has an ed25519 identity key. */
+    if (KEYPIN_MISMATCH ==
+        keypin_check((const uint8_t*)router->cache_info.identity_digest,
+                   router->cache_info.signing_key_cert->signing_key.pubkey)) {
+      log_fn(severity, LD_DIR,
+             "Descriptor from router %s has an Ed25519 key, "
+               "but the <rsa,ed25519> keys don't match what they were before.",
+               router_describe(router));
+      if (key_pinning) {
+        if (msg) {
+          *msg = "Ed25519 identity key or RSA identity key has changed.";
+        }
+        return FP_REJECT;
+      }
+    }
+  } else {
+    /* No ed25519 key */
+    if (KEYPIN_MISMATCH == keypin_check_lone_rsa(
+                        (const uint8_t*)router->cache_info.identity_digest)) {
+      log_fn(severity, LD_DIR,
+               "Descriptor from router %s has no Ed25519 key, "
+               "when we previously knew an Ed25519 for it. Ignoring for now, "
+               "since Ed25519 keys are fairly new.",
+               router_describe(router));
+#ifdef DISABLE_DISABLING_ED25519
+      if (key_pinning) {
+        if (msg) {
+          *msg = "Ed25519 identity key has disappeared.";
+        }
+        return FP_REJECT;
+      }
+#endif /* defined(DISABLE_DISABLING_ED25519) */
+    }
+  }
+
+  return 0;
+}
+
+/** Return true if there is no point in downloading the router described by
+ * <b>rs</b> because this directory would reject it. */
+int
+dirserv_would_reject_router(const routerstatus_t *rs)
+{
+  uint32_t res;
+
+  res = dirserv_get_status_impl(rs->identity_digest, rs->nickname,
+                                rs->addr, rs->or_port,
+                                NULL, NULL, LOG_DEBUG);
+
+  return (res & FP_REJECT) != 0;
+}
+
+/** Helper: As dirserv_router_get_status, but takes the router fingerprint
+ * (hex, no spaces), nickname, address (used for logging only), IP address, OR
+ * port and platform (logging only) as arguments.
+ *
+ * Log messages at 'severity'. (There's not much point in
+ * logging that we're rejecting servers we'll not download.)
+ */
+static uint32_t
+dirserv_get_status_impl(const char *id_digest, const char *nickname,
+                        uint32_t addr, uint16_t or_port,
+                        const char *platform, const char **msg, int severity)
+{
+  uint32_t result = 0;
+  router_status_t *status_by_digest;
+
+  if (!fingerprint_list)
+    fingerprint_list = authdir_config_new();
+
+  log_debug(LD_DIRSERV, "%d fingerprints, %d digests known.",
+            strmap_size(fingerprint_list->fp_by_name),
+            digestmap_size(fingerprint_list->status_by_digest));
+
+  if (platform) {
+    tor_version_t ver_tmp;
+    if (tor_version_parse_platform(platform, &ver_tmp, 1) < 0) {
+      if (msg) {
+        *msg = "Malformed platform string.";
+      }
+      return FP_REJECT;
+    }
+  }
+
+  /* Versions before Tor 0.2.4.18-rc are too old to support, and are
+   * missing some important security fixes too. Disable them. */
+  if (platform && !tor_version_as_new_as(platform,"0.2.4.18-rc")) {
+    if (msg)
+      *msg = "Tor version is insecure or unsupported. Please upgrade!";
+    return FP_REJECT;
+  }
+
+  /* Tor 0.2.9.x where x<5 suffers from bug #20499, where relays don't
+   * keep their consensus up to date so they make bad guards.
+   * The simple fix is to just drop them from the network. */
+  if (platform &&
+      tor_version_as_new_as(platform,"0.2.9.0-alpha") &&
+      !tor_version_as_new_as(platform,"0.2.9.5-alpha")) {
+    if (msg)
+      *msg = "Tor version contains bug 20499. Please upgrade!";
+    return FP_REJECT;
+  }
+
+  status_by_digest = digestmap_get(fingerprint_list->status_by_digest,
+                                   id_digest);
+  if (status_by_digest)
+    result |= *status_by_digest;
+
+  if (result & FP_REJECT) {
+    if (msg)
+      *msg = "Fingerprint is marked rejected -- if you think this is a "
+             "mistake please set a valid email address in ContactInfo and "
+             "send an email to bad-relays@lists.torproject.org mentioning "
+             "your fingerprint(s)?";
+    return FP_REJECT;
+  } else if (result & FP_INVALID) {
+    if (msg)
+      *msg = "Fingerprint is marked invalid";
+  }
+
+  if (authdir_policy_badexit_address(addr, or_port)) {
+    log_fn(severity, LD_DIRSERV,
+           "Marking '%s' as bad exit because of address '%s'",
+               nickname, fmt_addr32(addr));
+    result |= FP_BADEXIT;
+  }
+
+  if (!authdir_policy_permits_address(addr, or_port)) {
+    log_fn(severity, LD_DIRSERV, "Rejecting '%s' because of address '%s'",
+               nickname, fmt_addr32(addr));
+    if (msg)
+      *msg = "Suspicious relay address range -- if you think this is a "
+             "mistake please set a valid email address in ContactInfo and "
+             "send an email to bad-relays@lists.torproject.org mentioning "
+             "your address(es) and fingerprint(s)?";
+    return FP_REJECT;
+  }
+  if (!authdir_policy_valid_address(addr, or_port)) {
+    log_fn(severity, LD_DIRSERV,
+           "Not marking '%s' valid because of address '%s'",
+               nickname, fmt_addr32(addr));
+    result |= FP_INVALID;
+  }
+
+  return result;
+}
+
+/** Clear the current fingerprint list. */
+void
+dirserv_free_fingerprint_list(void)
+{
+  if (!fingerprint_list)
+    return;
+
+  strmap_free(fingerprint_list->fp_by_name, tor_free_);
+  digestmap_free(fingerprint_list->status_by_digest, tor_free_);
+  tor_free(fingerprint_list);
+}
+
+/*
+ *    Descriptor list
+ */
+
+/** Return -1 if <b>ri</b> has a private or otherwise bad address,
+ * unless we're configured to not care. Return 0 if all ok. */
+static int
+dirserv_router_has_valid_address(routerinfo_t *ri)
+{
+  tor_addr_t addr;
+  if (get_options()->DirAllowPrivateAddresses)
+    return 0; /* whatever it is, we're fine with it */
+  tor_addr_from_ipv4h(&addr, ri->addr);
+
+  if (tor_addr_is_internal(&addr, 0)) {
+    log_info(LD_DIRSERV,
+             "Router %s published internal IP address. Refusing.",
+             router_describe(ri));
+    return -1; /* it's a private IP, we should reject it */
+  }
+  return 0;
+}
+
+/** Check whether we, as a directory server, want to accept <b>ri</b>.  If so,
+ * set its is_valid,running fields and return 0.  Otherwise, return -1.
+ *
+ * If the router is rejected, set *<b>msg</b> to an explanation of why.
+ *
+ * If <b>complain</b> then explain at log-level 'notice' why we refused
+ * a descriptor; else explain at log-level 'info'.
+ */
+int
+authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
+                               int complain, int *valid_out)
+{
+  /* Okay.  Now check whether the fingerprint is recognized. */
+  time_t now;
+  int severity = (complain && ri->contact_info) ? LOG_NOTICE : LOG_INFO;
+  uint32_t status = dirserv_router_get_status(ri, msg, severity);
+  tor_assert(msg);
+  if (status & FP_REJECT)
+    return -1; /* msg is already set. */
+
+  /* Is there too much clock skew? */
+  now = time(NULL);
+  if (ri->cache_info.published_on > now+ROUTER_ALLOW_SKEW) {
+    log_fn(severity, LD_DIRSERV, "Publication time for %s is too "
+           "far (%d minutes) in the future; possible clock skew. Not adding "
+           "(%s)",
+           router_describe(ri),
+           (int)((ri->cache_info.published_on-now)/60),
+           esc_router_info(ri));
+    *msg = "Rejected: Your clock is set too far in the future, or your "
+      "timezone is not correct.";
+    return -1;
+  }
+  if (ri->cache_info.published_on < now-ROUTER_MAX_AGE_TO_PUBLISH) {
+    log_fn(severity, LD_DIRSERV,
+           "Publication time for %s is too far "
+           "(%d minutes) in the past. Not adding (%s)",
+           router_describe(ri),
+           (int)((now-ri->cache_info.published_on)/60),
+           esc_router_info(ri));
+    *msg = "Rejected: Server is expired, or your clock is too far in the past,"
+      " or your timezone is not correct.";
+    return -1;
+  }
+  if (dirserv_router_has_valid_address(ri) < 0) {
+    log_fn(severity, LD_DIRSERV,
+           "Router %s has invalid address. Not adding (%s).",
+           router_describe(ri),
+           esc_router_info(ri));
+    *msg = "Rejected: Address is a private address.";
+    return -1;
+  }
+
+  *valid_out = ! (status & FP_INVALID);
+
+  return 0;
+}
+
+/** Update the relevant flags of <b>node</b> based on our opinion as a
+ * directory authority in <b>authstatus</b>, as returned by
+ * dirserv_router_get_status or equivalent.  */
+void
+dirserv_set_node_flags_from_authoritative_status(node_t *node,
+                                                 uint32_t authstatus)
+{
+  node->is_valid = (authstatus & FP_INVALID) ? 0 : 1;
+  node->is_bad_exit = (authstatus & FP_BADEXIT) ? 1 : 0;
+}
+
+/** True iff <b>a</b> is more severe than <b>b</b>. */
+static int
+WRA_MORE_SEVERE(was_router_added_t a, was_router_added_t b)
+{
+  return a < b;
+}
+
+/** As for dirserv_add_descriptor(), but accepts multiple documents, and
+ * returns the most severe error that occurred for any one of them. */
+was_router_added_t
+dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
+                                 const char *source,
+                                 const char **msg)
+{
+  was_router_added_t r, r_tmp;
+  const char *msg_out;
+  smartlist_t *list;
+  const char *s;
+  int n_parsed = 0;
+  time_t now = time(NULL);
+  char annotation_buf[ROUTER_ANNOTATION_BUF_LEN];
+  char time_buf[ISO_TIME_LEN+1];
+  int general = purpose == ROUTER_PURPOSE_GENERAL;
+  tor_assert(msg);
+
+  r=ROUTER_ADDED_SUCCESSFULLY; /*Least severe return value. */
+
+  format_iso_time(time_buf, now);
+  if (tor_snprintf(annotation_buf, sizeof(annotation_buf),
+                   "@uploaded-at %s\n"
+                   "@source %s\n"
+                   "%s%s%s", time_buf, escaped(source),
+                   !general ? "@purpose " : "",
+                   !general ? router_purpose_to_string(purpose) : "",
+                   !general ? "\n" : "")<0) {
+    *msg = "Couldn't format annotations";
+    /* XXX Not cool: we return -1 below, but (was_router_added_t)-1 is
+     * ROUTER_BAD_EI, which isn't what's gone wrong here. :( */
+    return -1;
+  }
+
+  s = desc;
+  list = smartlist_new();
+  if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 0, 0,
+                                     annotation_buf, NULL)) {
+    SMARTLIST_FOREACH(list, routerinfo_t *, ri, {
+        msg_out = NULL;
+        tor_assert(ri->purpose == purpose);
+        r_tmp = dirserv_add_descriptor(ri, &msg_out, source);
+        if (WRA_MORE_SEVERE(r_tmp, r)) {
+          r = r_tmp;
+          *msg = msg_out;
+        }
+      });
+  }
+  n_parsed += smartlist_len(list);
+  smartlist_clear(list);
+
+  s = desc;
+  if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 1, 0,
+                                     NULL, NULL)) {
+    SMARTLIST_FOREACH(list, extrainfo_t *, ei, {
+        msg_out = NULL;
+
+        r_tmp = dirserv_add_extrainfo(ei, &msg_out);
+        if (WRA_MORE_SEVERE(r_tmp, r)) {
+          r = r_tmp;
+          *msg = msg_out;
+        }
+      });
+  }
+  n_parsed += smartlist_len(list);
+  smartlist_free(list);
+
+  if (! *msg) {
+    if (!n_parsed) {
+      *msg = "No descriptors found in your POST.";
+      if (WRA_WAS_ADDED(r))
+        r = ROUTER_IS_ALREADY_KNOWN;
+    } else {
+      *msg = "(no message)";
+    }
+  }
+
+  return r;
+}
+
+/** Examine the parsed server descriptor in <b>ri</b> and maybe insert it into
+ * the list of server descriptors. Set *<b>msg</b> to a message that should be
+ * passed back to the origin of this descriptor, or NULL if there is no such
+ * message. Use <b>source</b> to produce better log messages.
+ *
+ * If <b>ri</b> is not added to the list of server descriptors, free it.
+ * That means the caller must not access <b>ri</b> after this function
+ * returns, since it might have been freed.
+ *
+ * Return the status of the operation.
+ *
+ * This function is only called when fresh descriptors are posted, not when
+ * we re-load the cache.
+ */
+was_router_added_t
+dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source)
+{
+  was_router_added_t r;
+  routerinfo_t *ri_old;
+  char *desc, *nickname;
+  const size_t desclen = ri->cache_info.signed_descriptor_len +
+      ri->cache_info.annotations_len;
+  const int key_pinning = get_options()->AuthDirPinKeys;
+  *msg = NULL;
+
+  /* If it's too big, refuse it now. Otherwise we'll cache it all over the
+   * network and it'll clog everything up. */
+  if (ri->cache_info.signed_descriptor_len > MAX_DESCRIPTOR_UPLOAD_SIZE) {
+    log_notice(LD_DIR, "Somebody attempted to publish a router descriptor '%s'"
+               " (source: %s) with size %d. Either this is an attack, or the "
+               "MAX_DESCRIPTOR_UPLOAD_SIZE (%d) constant is too low.",
+               ri->nickname, source, (int)ri->cache_info.signed_descriptor_len,
+               MAX_DESCRIPTOR_UPLOAD_SIZE);
+    *msg = "Router descriptor was too large.";
+    r = ROUTER_AUTHDIR_REJECTS;
+    goto fail;
+  }
+
+  /* Check whether this descriptor is semantically identical to the last one
+   * from this server.  (We do this here and not in router_add_to_routerlist
+   * because we want to be able to accept the newest router descriptor that
+   * another authority has, so we all converge on the same one.) */
+  ri_old = router_get_mutable_by_digest(ri->cache_info.identity_digest);
+  if (ri_old && ri_old->cache_info.published_on < ri->cache_info.published_on
+      && router_differences_are_cosmetic(ri_old, ri)
+      && !router_is_me(ri)) {
+    log_info(LD_DIRSERV,
+             "Not replacing descriptor from %s (source: %s); "
+             "differences are cosmetic.",
+             router_describe(ri), source);
+    *msg = "Not replacing router descriptor; no information has changed since "
+      "the last one with this identity.";
+    r = ROUTER_IS_ALREADY_KNOWN;
+    goto fail;
+  }
+
+  /* Do keypinning again ... this time, to add the pin if appropriate */
+  int keypin_status;
+  if (ri->cache_info.signing_key_cert) {
+    ed25519_public_key_t *pkey = &ri->cache_info.signing_key_cert->signing_key;
+    /* First let's validate this pubkey before pinning it */
+    if (ed25519_validate_pubkey(pkey) < 0) {
+      log_warn(LD_DIRSERV, "Received bad key from %s (source %s)",
+               router_describe(ri), source);
+      routerinfo_free(ri);
+      return ROUTER_AUTHDIR_REJECTS;
+    }
+
+    /* Now pin it! */
+    keypin_status = keypin_check_and_add(
+      (const uint8_t*)ri->cache_info.identity_digest,
+      pkey->pubkey, ! key_pinning);
+  } else {
+    keypin_status = keypin_check_lone_rsa(
+      (const uint8_t*)ri->cache_info.identity_digest);
+#ifndef DISABLE_DISABLING_ED25519
+    if (keypin_status == KEYPIN_MISMATCH)
+      keypin_status = KEYPIN_NOT_FOUND;
+#endif
+  }
+  if (keypin_status == KEYPIN_MISMATCH && key_pinning) {
+    log_info(LD_DIRSERV, "Dropping descriptor from %s (source: %s) because "
+             "its key did not match an older RSA/Ed25519 keypair",
+             router_describe(ri), source);
+    *msg = "Looks like your keypair has changed? This authority previously "
+      "recorded a different RSA identity for this Ed25519 identity (or vice "
+      "versa.) Did you replace or copy some of your key files, but not "
+      "the others? You should either restore the expected keypair, or "
+      "delete your keys and restart Tor to start your relay with a new "
+      "identity.";
+    r = ROUTER_AUTHDIR_REJECTS;
+    goto fail;
+  }
+
+  /* Make a copy of desc, since router_add_to_routerlist might free
+   * ri and its associated signed_descriptor_t. */
+  desc = tor_strndup(ri->cache_info.signed_descriptor_body, desclen);
+  nickname = tor_strdup(ri->nickname);
+
+  /* Tell if we're about to need to launch a test if we add this. */
+  ri->needs_retest_if_added =
+    dirserv_should_launch_reachability_test(ri, ri_old);
+
+  r = router_add_to_routerlist(ri, msg, 0, 0);
+  if (!WRA_WAS_ADDED(r)) {
+    /* unless the routerinfo was fine, just out-of-date */
+    log_info(LD_DIRSERV,
+             "Did not add descriptor from '%s' (source: %s): %s.",
+             nickname, source, *msg ? *msg : "(no message)");
+  } else {
+    smartlist_t *changed;
+
+    changed = smartlist_new();
+    smartlist_add(changed, ri);
+    routerlist_descriptors_added(changed, 0);
+    smartlist_free(changed);
+    if (!*msg) {
+      *msg =  "Descriptor accepted";
+    }
+    log_info(LD_DIRSERV,
+             "Added descriptor from '%s' (source: %s): %s.",
+             nickname, source, *msg);
+  }
+  tor_free(desc);
+  tor_free(nickname);
+  return r;
+ fail:
+  {
+    const char *desc_digest = ri->cache_info.signed_descriptor_digest;
+    download_status_t *dls =
+      router_get_dl_status_by_descriptor_digest(desc_digest);
+    if (dls) {
+      log_info(LD_GENERAL, "Marking router with descriptor %s as rejected, "
+               "and therefore undownloadable",
+               hex_str(desc_digest, DIGEST_LEN));
+      download_status_mark_impossible(dls);
+    }
+    routerinfo_free(ri);
+  }
+  return r;
+}
+
+/** As dirserv_add_descriptor, but for an extrainfo_t <b>ei</b>. */
+static was_router_added_t
+dirserv_add_extrainfo(extrainfo_t *ei, const char **msg)
+{
+  routerinfo_t *ri;
+  int r;
+  was_router_added_t rv;
+  tor_assert(msg);
+  *msg = NULL;
+
+  /* Needs to be mutable so routerinfo_incompatible_with_extrainfo
+   * can mess with some of the flags in ri->cache_info. */
+  ri = router_get_mutable_by_digest(ei->cache_info.identity_digest);
+  if (!ri) {
+    *msg = "No corresponding router descriptor for extra-info descriptor";
+    rv = ROUTER_BAD_EI;
+    goto fail;
+  }
+
+  /* If it's too big, refuse it now. Otherwise we'll cache it all over the
+   * network and it'll clog everything up. */
+  if (ei->cache_info.signed_descriptor_len > MAX_EXTRAINFO_UPLOAD_SIZE) {
+    log_notice(LD_DIR, "Somebody attempted to publish an extrainfo "
+               "with size %d. Either this is an attack, or the "
+               "MAX_EXTRAINFO_UPLOAD_SIZE (%d) constant is too low.",
+               (int)ei->cache_info.signed_descriptor_len,
+               MAX_EXTRAINFO_UPLOAD_SIZE);
+    *msg = "Extrainfo document was too large";
+    rv = ROUTER_BAD_EI;
+    goto fail;
+  }
+
+  if ((r = routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei,
+                                                  &ri->cache_info, msg))) {
+    if (r<0) {
+      extrainfo_free(ei);
+      return ROUTER_IS_ALREADY_KNOWN;
+    }
+    rv = ROUTER_BAD_EI;
+    goto fail;
+  }
+  router_add_extrainfo_to_routerlist(ei, msg, 0, 0);
+  return ROUTER_ADDED_SUCCESSFULLY;
+ fail:
+  {
+    const char *d = ei->cache_info.signed_descriptor_digest;
+    signed_descriptor_t *sd = router_get_by_extrainfo_digest((char*)d);
+    if (sd) {
+      log_info(LD_GENERAL, "Marking extrainfo with descriptor %s as "
+               "rejected, and therefore undownloadable",
+               hex_str((char*)d,DIGEST_LEN));
+      download_status_mark_impossible(&sd->ei_dl_status);
+    }
+    extrainfo_free(ei);
+  }
+  return rv;
+}
+
+/** Remove all descriptors whose nicknames or fingerprints no longer
+ * are allowed by our fingerprint list. (Descriptors that used to be
+ * good can become bad when we reload the fingerprint list.)
+ */
+static void
+directory_remove_invalid(void)
+{
+  routerlist_t *rl = router_get_routerlist();
+  smartlist_t *nodes = smartlist_new();
+  smartlist_add_all(nodes, nodelist_get_list());
+
+  SMARTLIST_FOREACH_BEGIN(nodes, node_t *, node) {
+    const char *msg = NULL;
+    const char *description;
+    routerinfo_t *ent = node->ri;
+    uint32_t r;
+    if (!ent)
+      continue;
+    r = dirserv_router_get_status(ent, &msg, LOG_INFO);
+    description = router_describe(ent);
+    if (r & FP_REJECT) {
+      log_info(LD_DIRSERV, "Router %s is now rejected: %s",
+               description, msg?msg:"");
+      routerlist_remove(rl, ent, 0, time(NULL));
+      continue;
+    }
+    if (bool_neq((r & FP_INVALID), !node->is_valid)) {
+      log_info(LD_DIRSERV, "Router '%s' is now %svalid.", description,
+               (r&FP_INVALID) ? "in" : "");
+      node->is_valid = (r&FP_INVALID)?0:1;
+    }
+    if (bool_neq((r & FP_BADEXIT), node->is_bad_exit)) {
+      log_info(LD_DIRSERV, "Router '%s' is now a %s exit", description,
+               (r & FP_BADEXIT) ? "bad" : "good");
+      node->is_bad_exit = (r&FP_BADEXIT) ? 1: 0;
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  routerlist_assert_ok(rl);
+  smartlist_free(nodes);
+}
diff --git a/src/feature/dirauth/process_descs.h b/src/feature/dirauth/process_descs.h
new file mode 100644
index 0000000000..ae2d6ad25d
--- /dev/null
+++ b/src/feature/dirauth/process_descs.h
@@ -0,0 +1,38 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_descs.h
+ * \brief Header file for process_descs.c.
+ **/
+
+#ifndef TOR_RECV_UPLOADS_H
+#define TOR_RECV_UPLOADS_H
+
+int dirserv_load_fingerprint_file(void);
+void dirserv_free_fingerprint_list(void);
+int dirserv_add_own_fingerprint(crypto_pk_t *pk);
+
+enum was_router_added_t dirserv_add_multiple_descriptors(
+                                     const char *desc, uint8_t purpose,
+                                     const char *source,
+                                     const char **msg);
+enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri,
+                                               const char **msg,
+                                               const char *source);
+
+int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
+                                   int complain,
+                                   int *valid_out);
+uint32_t dirserv_router_get_status(const routerinfo_t *router,
+                                   const char **msg,
+                                   int severity);
+void dirserv_set_node_flags_from_authoritative_status(node_t *node,
+                                                      uint32_t authstatus);
+
+int dirserv_would_reject_router(const routerstatus_t *rs);
+
+#endif
diff --git a/src/feature/dirauth/reachability.c b/src/feature/dirauth/reachability.c
new file mode 100644
index 0000000000..883b692cbb
--- /dev/null
+++ b/src/feature/dirauth/reachability.c
@@ -0,0 +1,207 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file reachability.c
+ * \brief Router reachability testing; run by authorities to tell who is
+ * running.
+ */
+
+#include "core/or/or.h"
+#include "feature/dirauth/reachability.h"
+
+#include "app/config/config.h"
+#include "core/or/channel.h"
+#include "core/or/channeltls.h"
+#include "core/or/command.h"
+#include "feature/dirauth/authmode.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerinfo.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/torcert.h"
+#include "feature/stats/rephist.h"
+
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerlist_st.h"
+
+/** Called when a TLS handshake has completed successfully with a
+ * router listening at <b>address</b>:<b>or_port</b>, and has yielded
+ * a certificate with digest <b>digest_rcvd</b>.
+ *
+ * Inform the reachability checker that we could get to this relay.
+ */
+void
+dirserv_orconn_tls_done(const tor_addr_t *addr,
+                        uint16_t or_port,
+                        const char *digest_rcvd,
+                        const ed25519_public_key_t *ed_id_rcvd)
+{
+  node_t *node = NULL;
+  tor_addr_port_t orport;
+  routerinfo_t *ri = NULL;
+  time_t now = time(NULL);
+  tor_assert(addr);
+  tor_assert(digest_rcvd);
+
+  node = node_get_mutable_by_id(digest_rcvd);
+  if (node == NULL || node->ri == NULL)
+    return;
+
+  ri = node->ri;
+
+  if (get_options()->AuthDirTestEd25519LinkKeys &&
+      node_supports_ed25519_link_authentication(node, 1) &&
+      ri->cache_info.signing_key_cert) {
+    /* We allow the node to have an ed25519 key if we haven't been told one in
+     * the routerinfo, but if we *HAVE* been told one in the routerinfo, it
+     * needs to match. */
+    const ed25519_public_key_t *expected_id =
+      &ri->cache_info.signing_key_cert->signing_key;
+    tor_assert(!ed25519_public_key_is_zero(expected_id));
+    if (! ed_id_rcvd || ! ed25519_pubkey_eq(ed_id_rcvd, expected_id)) {
+      log_info(LD_DIRSERV, "Router at %s:%d with RSA ID %s "
+               "did not present expected Ed25519 ID.",
+               fmt_addr(addr), or_port, hex_str(digest_rcvd, DIGEST_LEN));
+      return; /* Don't mark it as reachable. */
+    }
+  }
+
+  tor_addr_copy(&orport.addr, addr);
+  orport.port = or_port;
+  if (router_has_orport(ri, &orport)) {
+    /* Found the right router.  */
+    if (!authdir_mode_bridge(get_options()) ||
+        ri->purpose == ROUTER_PURPOSE_BRIDGE) {
+      char addrstr[TOR_ADDR_BUF_LEN];
+      /* This is a bridge or we're not a bridge authority --
+         mark it as reachable.  */
+      log_info(LD_DIRSERV, "Found router %s to be reachable at %s:%d. Yay.",
+               router_describe(ri),
+               tor_addr_to_str(addrstr, addr, sizeof(addrstr), 1),
+               ri->or_port);
+      if (tor_addr_family(addr) == AF_INET) {
+        rep_hist_note_router_reachable(digest_rcvd, addr, or_port, now);
+        node->last_reachable = now;
+      } else if (tor_addr_family(addr) == AF_INET6) {
+        /* No rephist for IPv6.  */
+        node->last_reachable6 = now;
+      }
+    }
+  }
+}
+
+/** Called when we, as an authority, receive a new router descriptor either as
+ * an upload or a download.  Used to decide whether to relaunch reachability
+ * testing for the server. */
+int
+dirserv_should_launch_reachability_test(const routerinfo_t *ri,
+                                        const routerinfo_t *ri_old)
+{
+  if (!authdir_mode_handles_descs(get_options(), ri->purpose))
+    return 0;
+  if (!ri_old) {
+    /* New router: Launch an immediate reachability test, so we will have an
+     * opinion soon in case we're generating a consensus soon */
+    return 1;
+  }
+  if (ri_old->is_hibernating && !ri->is_hibernating) {
+    /* It just came out of hibernation; launch a reachability test */
+    return 1;
+  }
+  if (! routers_have_same_or_addrs(ri, ri_old)) {
+    /* Address or port changed; launch a reachability test */
+    return 1;
+  }
+  return 0;
+}
+
+/** Helper function for dirserv_test_reachability(). Start a TLS
+ * connection to <b>router</b>, and annotate it with when we started
+ * the test. */
+void
+dirserv_single_reachability_test(time_t now, routerinfo_t *router)
+{
+  const or_options_t *options = get_options();
+  channel_t *chan = NULL;
+  const node_t *node = NULL;
+  tor_addr_t router_addr;
+  const ed25519_public_key_t *ed_id_key;
+  (void) now;
+
+  tor_assert(router);
+  node = node_get_by_id(router->cache_info.identity_digest);
+  tor_assert(node);
+
+  if (options->AuthDirTestEd25519LinkKeys &&
+      node_supports_ed25519_link_authentication(node, 1) &&
+      router->cache_info.signing_key_cert) {
+    ed_id_key = &router->cache_info.signing_key_cert->signing_key;
+  } else {
+    ed_id_key = NULL;
+  }
+
+  /* IPv4. */
+  log_debug(LD_OR,"Testing reachability of %s at %s:%u.",
+            router->nickname, fmt_addr32(router->addr), router->or_port);
+  tor_addr_from_ipv4h(&router_addr, router->addr);
+  chan = channel_tls_connect(&router_addr, router->or_port,
+                             router->cache_info.identity_digest,
+                             ed_id_key);
+  if (chan) command_setup_channel(chan);
+
+  /* Possible IPv6. */
+  if (get_options()->AuthDirHasIPv6Connectivity == 1 &&
+      !tor_addr_is_null(&router->ipv6_addr)) {
+    char addrstr[TOR_ADDR_BUF_LEN];
+    log_debug(LD_OR, "Testing reachability of %s at %s:%u.",
+              router->nickname,
+              tor_addr_to_str(addrstr, &router->ipv6_addr, sizeof(addrstr), 1),
+              router->ipv6_orport);
+    chan = channel_tls_connect(&router->ipv6_addr, router->ipv6_orport,
+                               router->cache_info.identity_digest,
+                               ed_id_key);
+    if (chan) command_setup_channel(chan);
+  }
+}
+
+/** Auth dir server only: load balance such that we only
+ * try a few connections per call.
+ *
+ * The load balancing is such that if we get called once every ten
+ * seconds, we will cycle through all the tests in
+ * REACHABILITY_TEST_CYCLE_PERIOD seconds (a bit over 20 minutes).
+ */
+void
+dirserv_test_reachability(time_t now)
+{
+  /* XXX decide what to do here; see or-talk thread "purging old router
+   * information, revocation." -NM
+   * We can't afford to mess with this in 0.1.2.x. The reason is that
+   * if we stop doing reachability tests on some of routerlist, then
+   * we'll for-sure think they're down, which may have unexpected
+   * effects in other parts of the code. It doesn't hurt much to do
+   * the testing, and directory authorities are easy to upgrade. Let's
+   * wait til 0.2.0. -RD */
+//  time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
+  routerlist_t *rl = router_get_routerlist();
+  static char ctr = 0;
+  int bridge_auth = authdir_mode_bridge(get_options());
+
+  SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, router) {
+    const char *id_digest = router->cache_info.identity_digest;
+    if (router_is_me(router))
+      continue;
+    if (bridge_auth && router->purpose != ROUTER_PURPOSE_BRIDGE)
+      continue; /* bridge authorities only test reachability on bridges */
+//    if (router->cache_info.published_on > cutoff)
+//      continue;
+    if ((((uint8_t)id_digest[0]) % REACHABILITY_MODULO_PER_TEST) == ctr) {
+      dirserv_single_reachability_test(now, router);
+    }
+  } SMARTLIST_FOREACH_END(router);
+  ctr = (ctr + 1) % REACHABILITY_MODULO_PER_TEST; /* increment ctr */
+}
diff --git a/src/feature/dirauth/reachability.h b/src/feature/dirauth/reachability.h
new file mode 100644
index 0000000000..5a938673ff
--- /dev/null
+++ b/src/feature/dirauth/reachability.h
@@ -0,0 +1,36 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file reachability.h
+ * \brief Header file for reachability.c.
+ **/
+
+#ifndef TOR_REACHABILITY_H
+#define TOR_REACHABILITY_H
+
+/** What fraction (1 over this number) of the relay ID space do we
+ * (as a directory authority) launch connections to at each reachability
+ * test? */
+#define REACHABILITY_MODULO_PER_TEST 128
+
+/** How often (in seconds) do we launch reachability tests? */
+#define REACHABILITY_TEST_INTERVAL 10
+
+/** How many seconds apart are the reachability tests for a given relay? */
+#define REACHABILITY_TEST_CYCLE_PERIOD \
+  (REACHABILITY_TEST_INTERVAL*REACHABILITY_MODULO_PER_TEST)
+
+void dirserv_orconn_tls_done(const tor_addr_t *addr,
+                             uint16_t or_port,
+                             const char *digest_rcvd,
+                             const struct ed25519_public_key_t *ed_id_rcvd);
+int dirserv_should_launch_reachability_test(const routerinfo_t *ri,
+                                            const routerinfo_t *ri_old);
+void dirserv_single_reachability_test(time_t now, routerinfo_t *router);
+void dirserv_test_reachability(time_t now);
+
+#endif
diff --git a/src/feature/dirauth/recommend_pkg.c b/src/feature/dirauth/recommend_pkg.c
new file mode 100644
index 0000000000..0456ff8463
--- /dev/null
+++ b/src/feature/dirauth/recommend_pkg.c
@@ -0,0 +1,90 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file recommend_pkg.c
+ * \brief Code related to the recommended-packages subsystem.
+ *
+ * Currently unused.
+ **/
+
+#include "core/or/or.h"
+#include "feature/dirauth/recommend_pkg.h"
+
+/** Return true iff <b>line</b> is a valid RecommendedPackages line.
+ */
+/*
+  The grammar is:
+
+    "package" SP PACKAGENAME SP VERSION SP URL SP DIGESTS NL
+
+      PACKAGENAME = NONSPACE
+      VERSION = NONSPACE
+      URL = NONSPACE
+      DIGESTS = DIGEST | DIGESTS SP DIGEST
+      DIGEST = DIGESTTYPE "=" DIGESTVAL
+
+      NONSPACE = one or more non-space printing characters
+
+      DIGESTVAL = DIGESTTYPE = one or more non-=, non-" " characters.
+
+      SP = " "
+      NL = a newline
+
+ */
+int
+validate_recommended_package_line(const char *line)
+{
+  const char *cp = line;
+
+#define WORD()                                  \
+  do {                                          \
+    if (*cp == ' ')                             \
+      return 0;                                 \
+    cp = strchr(cp, ' ');                       \
+    if (!cp)                                    \
+      return 0;                                 \
+  } while (0)
+
+  WORD(); /* skip packagename */
+  ++cp;
+  WORD(); /* skip version */
+  ++cp;
+  WORD(); /* Skip URL */
+  ++cp;
+
+  /* Skip digesttype=digestval + */
+  int n_entries = 0;
+  while (1) {
+    const char *start_of_word = cp;
+    const char *end_of_word = strchr(cp, ' ');
+    if (! end_of_word)
+      end_of_word = cp + strlen(cp);
+
+    if (start_of_word == end_of_word)
+      return 0;
+
+    const char *eq = memchr(start_of_word, '=', end_of_word - start_of_word);
+
+    if (!eq)
+      return 0;
+    if (eq == start_of_word)
+      return 0;
+    if (eq == end_of_word - 1)
+      return 0;
+    if (memchr(eq+1, '=', end_of_word - (eq+1)))
+      return 0;
+
+    ++n_entries;
+    if (0 == *end_of_word)
+      break;
+
+    cp = end_of_word + 1;
+  }
+
+  /* If we reach this point, we have at least 1 entry. */
+  tor_assert(n_entries > 0);
+  return 1;
+}
diff --git a/src/feature/dirauth/recommend_pkg.h b/src/feature/dirauth/recommend_pkg.h
new file mode 100644
index 0000000000..8200d78f72
--- /dev/null
+++ b/src/feature/dirauth/recommend_pkg.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file recommend_pkg.h
+ * \brief Header file for recommend_pkg.c
+ **/
+
+#ifndef TOR_RECOMMEND_PKG_H
+#define TOR_RECOMMEND_PKG_H
+
+int validate_recommended_package_line(const char *line);
+
+#endif
diff --git a/src/or/shared_random.c b/src/feature/dirauth/shared_random.c
index 5f6b03f1ba..34b2283250 100644
--- a/src/or/shared_random.c
+++ b/src/feature/dirauth/shared_random.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -87,17 +87,25 @@
 
 #define SHARED_RANDOM_PRIVATE
 
-#include "or.h"
-#include "shared_random.h"
-#include "config.h"
-#include "confparse.h"
-#include "dirvote.h"
-#include "networkstatus.h"
-#include "routerkeys.h"
-#include "router.h"
-#include "routerlist.h"
-#include "shared_random_state.h"
-#include "util.h"
+#include "core/or/or.h"
+#include "feature/dirauth/shared_random.h"
+#include "app/config/config.h"
+#include "app/config/confparse.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routerkeys.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/hs_common/shared_random_client.h"
+#include "feature/dirauth/shared_random_state.h"
+#include "feature/dircommon/voting_schedule.h"
+
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/authmode.h"
+
+#include "feature/nodelist/authority_cert_st.h"
+#include "feature/nodelist/networkstatus_st.h"
 
 /* String prefix of shared random values in votes/consensuses. */
 static const char previous_srv_str[] = "shared-rand-previous-value";
@@ -192,7 +200,7 @@ verify_commit_and_reveal(const sr_commit_t *commit)
     /* Use the invariant length since the encoded reveal variable has an
      * extra byte for the NUL terminated byte. */
     if (crypto_digest256(received_hashed_reveal, commit->encoded_reveal,
-                         SR_REVEAL_BASE64_LEN, commit->alg)) {
+                         SR_REVEAL_BASE64_LEN, commit->alg) < 0) {
       /* Unable to digest the reveal blob, this is unlikely. */
       goto invalid;
     }
@@ -230,9 +238,7 @@ commit_decode(const char *encoded, sr_commit_t *commit)
 {
   int decoded_len = 0;
   size_t offset = 0;
-  /* XXX: Needs two extra bytes for the base64 decode calculation matches
-   * the binary length once decoded. #17868. */
-  char b64_decoded[SR_COMMIT_LEN + 2];
+  char b64_decoded[SR_COMMIT_LEN];
 
   tor_assert(encoded);
   tor_assert(commit);
@@ -284,9 +290,7 @@ STATIC int
 reveal_decode(const char *encoded, sr_commit_t *commit)
 {
   int decoded_len = 0;
-  /* XXX: Needs two extra bytes for the base64 decode calculation matches
-   * the binary length once decoded. #17868. */
-  char b64_decoded[SR_REVEAL_LEN + 2];
+  char b64_decoded[SR_REVEAL_LEN];
 
   tor_assert(encoded);
   tor_assert(commit);
@@ -388,7 +392,7 @@ commit_encode(const sr_commit_t *commit, char *dst, size_t len)
 static void
 sr_cleanup(void)
 {
-  sr_state_free();
+  sr_state_free_all();
 }
 
 /* Using <b>commit</b>, return a newly allocated string containing the commit
@@ -864,30 +868,9 @@ get_majority_srv_from_votes(const smartlist_t *votes, int current)
   return the_srv;
 }
 
-/* Encode the given shared random value and put it in dst. Destination
- * buffer must be at least SR_SRV_VALUE_BASE64_LEN plus the NULL byte. */
-void
-sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv)
-{
-  int ret;
-  /* Extra byte for the NULL terminated char. */
-  char buf[SR_SRV_VALUE_BASE64_LEN + 1];
-
-  tor_assert(dst);
-  tor_assert(srv);
-  tor_assert(dst_len >= sizeof(buf));
-
-  ret = base64_encode(buf, sizeof(buf), (const char *) srv->value,
-                      sizeof(srv->value), 0);
-  /* Always expect the full length without the NULL byte. */
-  tor_assert(ret == (sizeof(buf) - 1));
-  tor_assert(ret <= (int) dst_len);
-  strlcpy(dst, buf, dst_len);
-}
-
 /* Free a commit object. */
 void
-sr_commit_free(sr_commit_t *commit)
+sr_commit_free_(sr_commit_t *commit)
 {
   if (commit == NULL) {
     return;
@@ -932,7 +915,7 @@ sr_generate_our_commit(time_t timestamp, const authority_cert_t *my_rsa_cert)
   /* The invariant length is used here since the encoded reveal variable
    * has an extra byte added for the NULL terminated byte. */
   if (crypto_digest256(commit->hashed_reveal, commit->encoded_reveal,
-                       SR_REVEAL_BASE64_LEN, commit->alg)) {
+                       SR_REVEAL_BASE64_LEN, commit->alg) < 0) {
     goto error;
   }
 
@@ -966,7 +949,8 @@ sr_compute_srv(void)
   /* Computing a shared random value in the commit phase is very wrong. This
    * should only happen at the very end of the reveal phase when a new
    * protocol run is about to start. */
-  tor_assert(sr_state_get_phase() == SR_PHASE_REVEAL);
+  if (BUG(sr_state_get_phase() != SR_PHASE_REVEAL))
+    return;
   state_commits = sr_state_get_commits();
 
   commits = smartlist_new();
@@ -1012,7 +996,7 @@ sr_compute_srv(void)
     SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
     smartlist_free(chunks);
     if (crypto_digest256(hashed_reveals, reveals, strlen(reveals),
-                         SR_DIGEST_ALG)) {
+                         SR_DIGEST_ALG) < 0) {
       goto end;
     }
     current_srv = generate_srv(hashed_reveals, reveal_num,
@@ -1026,55 +1010,6 @@ sr_compute_srv(void)
   tor_free(reveals);
 }
 
-/* Parse a list of arguments from a SRV value either from a vote, consensus
- * or from our disk state and return a newly allocated srv object. NULL is
- * returned on error.
- *
- * The arguments' order:
- *    num_reveals, value
- */
-sr_srv_t *
-sr_parse_srv(const smartlist_t *args)
-{
-  char *value;
-  int ok, ret;
-  uint64_t num_reveals;
-  sr_srv_t *srv = NULL;
-
-  tor_assert(args);
-
-  if (smartlist_len(args) < 2) {
-    goto end;
-  }
-
-  /* First argument is the number of reveal values */
-  num_reveals = tor_parse_uint64(smartlist_get(args, 0),
-                                 10, 0, UINT64_MAX, &ok, NULL);
-  if (!ok) {
-    goto end;
-  }
-  /* Second and last argument is the shared random value it self. */
-  value = smartlist_get(args, 1);
-  if (strlen(value) != SR_SRV_VALUE_BASE64_LEN) {
-    goto end;
-  }
-
-  srv = tor_malloc_zero(sizeof(*srv));
-  srv->num_reveals = num_reveals;
-  /* We substract one byte from the srclen because the function ignores the
-   * '=' character in the given buffer. This is broken but it's a documented
-   * behavior of the implementation. */
-  ret = base64_decode((char *) srv->value, sizeof(srv->value), value,
-                      SR_SRV_VALUE_BASE64_LEN - 1);
-  if (ret != sizeof(srv->value)) {
-    tor_free(srv);
-    srv = NULL;
-    goto end;
-  }
- end:
-  return srv;
-}
-
 /* Parse a commit from a vote or from our disk state and return a newly
  * allocated commit object. NULL is returned on error.
  *
@@ -1323,13 +1258,7 @@ sr_act_post_consensus(const networkstatus_t *consensus)
   }
 
   /* Prepare our state so that it's ready for the next voting period. */
-  {
-    voting_schedule_t *voting_schedule =
-      get_voting_schedule(options,time(NULL), LOG_NOTICE);
-    time_t interval_starts = voting_schedule->interval_starts;
-    sr_state_update(interval_starts);
-    voting_schedule_free(voting_schedule);
-  }
+  sr_state_update(voting_schedule_get_next_valid_after_time());
 }
 
 /* Initialize shared random subsystem. This MUST be called early in the boot
@@ -1359,5 +1288,4 @@ set_num_srv_agreements(int32_t value)
   num_srv_agreements_from_vote = value;
 }
 
-#endif /* TOR_UNIT_TESTS */
-
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/or/shared_random.h b/src/feature/dirauth/shared_random.h
index 9885934cc7..25d95ebbc7 100644
--- a/src/or/shared_random.h
+++ b/src/feature/dirauth/shared_random.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_SHARED_RANDOM_H
@@ -10,7 +10,7 @@
  * with "sr_" which stands for shared random.
  */
 
-#include "or.h"
+#include "core/or/or.h"
 
 /* Protocol version */
 #define SR_PROTO_VERSION  1
@@ -36,17 +36,14 @@
 
 /* Length of base64 encoded commit NOT including the NUL terminated byte.
  * Formula is taken from base64_encode_size. This adds up to 56 bytes. */
-#define SR_COMMIT_BASE64_LEN \
-  (((SR_COMMIT_LEN - 1) / 3) * 4 + 4)
+#define SR_COMMIT_BASE64_LEN (BASE64_LEN(SR_COMMIT_LEN))
 /* Length of base64 encoded reveal NOT including the NUL terminated byte.
  * Formula is taken from base64_encode_size. This adds up to 56 bytes. */
-#define SR_REVEAL_BASE64_LEN \
-  (((SR_REVEAL_LEN - 1) / 3) * 4 + 4)
+#define SR_REVEAL_BASE64_LEN (BASE64_LEN(SR_REVEAL_LEN))
 /* Length of base64 encoded shared random value. It's 32 bytes long so 44
  * bytes from the base64_encode_size formula. That includes the '='
  * character at the end. */
-#define SR_SRV_VALUE_BASE64_LEN \
-  (((DIGEST256_LEN - 1) / 3) * 4 + 4)
+#define SR_SRV_VALUE_BASE64_LEN (BASE64_LEN(DIGEST256_LEN))
 
 /* Assert if commit valid flag is not set. */
 #define ASSERT_COMMIT_VALID(c) tor_assert((c)->valid)
@@ -104,20 +101,48 @@ typedef struct sr_commit_t {
 
 /* API */
 
-/* Public methods: */
+/* Public methods used _outside_ of the module.
+ *
+ * We need to nullify them if the module is disabled. */
+#ifdef HAVE_MODULE_DIRAUTH
 
 int sr_init(int save_to_disk);
 void sr_save_and_cleanup(void);
 void sr_act_post_consensus(const networkstatus_t *consensus);
+
+#else /* HAVE_MODULE_DIRAUTH */
+
+static inline int
+sr_init(int save_to_disk)
+{
+  (void) save_to_disk;
+  /* Always return success. */
+  return 0;
+}
+
+static inline void
+sr_save_and_cleanup(void)
+{
+}
+
+static inline void
+sr_act_post_consensus(const networkstatus_t *consensus)
+{
+  (void) consensus;
+}
+
+#endif /* HAVE_MODULE_DIRAUTH */
+
+/* Public methods used only by dirauth code. */
+
 void sr_handle_received_commits(smartlist_t *commits,
                                 crypto_pk_t *voter_key);
 sr_commit_t *sr_parse_commit(const smartlist_t *args);
-sr_srv_t *sr_parse_srv(const smartlist_t *args);
 char *sr_get_string_for_vote(void);
 char *sr_get_string_for_consensus(const smartlist_t *votes,
                                   int32_t num_srv_agreements);
-void sr_commit_free(sr_commit_t *commit);
-void sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv);
+void sr_commit_free_(sr_commit_t *commit);
+#define sr_commit_free(sr) FREE_AND_NULL(sr_commit_t, sr_commit_free_, (sr))
 
 /* Private methods (only used by shared_random_state.c): */
 static inline
@@ -129,6 +154,7 @@ const char *sr_commit_get_rsa_fpr(const sr_commit_t *commit)
 void sr_compute_srv(void);
 sr_commit_t *sr_generate_our_commit(time_t timestamp,
                                     const authority_cert_t *my_rsa_cert);
+
 #ifdef SHARED_RANDOM_PRIVATE
 
 /* Encode */
@@ -156,7 +182,7 @@ STATIC int should_keep_commit(const sr_commit_t *commit,
                               sr_phase_t phase);
 STATIC void save_commit_during_reveal_phase(const sr_commit_t *commit);
 
-#endif /* SHARED_RANDOM_PRIVATE */
+#endif /* defined(SHARED_RANDOM_PRIVATE) */
 
 #ifdef TOR_UNIT_TESTS
 
@@ -164,5 +190,5 @@ void set_num_srv_agreements(int32_t value);
 
 #endif /* TOR_UNIT_TESTS */
 
-#endif /* TOR_SHARED_RANDOM_H */
+#endif /* !defined(TOR_SHARED_RANDOM_H) */
 
diff --git a/src/or/shared_random_state.c b/src/feature/dirauth/shared_random_state.c
index f27eccafc7..b3e4a4ef92 100644
--- a/src/or/shared_random_state.c
+++ b/src/feature/dirauth/shared_random_state.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -10,14 +10,20 @@
 
 #define SHARED_RANDOM_STATE_PRIVATE
 
-#include "or.h"
-#include "shared_random.h"
-#include "config.h"
-#include "confparse.h"
-#include "dirvote.h"
-#include "networkstatus.h"
-#include "router.h"
-#include "shared_random_state.h"
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "app/config/confparse.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/relay/router.h"
+#include "feature/dirauth/shared_random.h"
+#include "feature/hs_common/shared_random_client.h"
+#include "feature/dirauth/shared_random_state.h"
+#include "feature/dircommon/voting_schedule.h"
+#include "lib/encoding/confline.h"
+
+#include "app/config/or_state_st.h"
 
 /* Default filename of the shared random state on disk. */
 static const char default_fname[] = "sr-state";
@@ -40,23 +46,24 @@ static const char dstate_commit_key[] = "Commit";
 static const char dstate_prev_srv_key[] = "SharedRandPreviousValue";
 static const char dstate_cur_srv_key[] = "SharedRandCurrentValue";
 
+/** dummy instance of sr_disk_state_t, used for type-checking its
+ * members with CONF_CHECK_VAR_TYPE. */
+DUMMY_TYPECHECK_INSTANCE(sr_disk_state_t);
+
 /* These next two are duplicates or near-duplicates from config.c */
 #define VAR(name, conftype, member, initvalue)                              \
-  { name, CONFIG_TYPE_ ## conftype, STRUCT_OFFSET(sr_disk_state_t, member), \
-    initvalue }
+  { name, CONFIG_TYPE_ ## conftype, offsetof(sr_disk_state_t, member),      \
+      initvalue CONF_TEST_MEMBERS(sr_disk_state_t, conftype, member) }
 /* As VAR, but the option name and member name are the same. */
 #define V(member, conftype, initvalue) \
   VAR(#member, conftype, member, initvalue)
 /* Our persistent state magic number. */
 #define SR_DISK_STATE_MAGIC 0x98AB1254
-/* Each protocol phase has 12 rounds  */
-#define SHARED_RANDOM_N_ROUNDS 12
-/* Number of phase we have in a protocol. */
-#define SHARED_RANDOM_N_PHASES 2
 
 static int
 disk_state_validate_cb(void *old_state, void *state, void *default_state,
                        int from_setconf, char **msg);
+static void disk_state_free_cb(void *);
 
 /* Array of variables that are saved to disk as a persistent state. */
 static config_var_t state_vars[] = {
@@ -70,25 +77,27 @@ static config_var_t state_vars[] = {
   V(SharedRandValues,           LINELIST_V, NULL),
   VAR("SharedRandPreviousValue",LINELIST_S, SharedRandValues, NULL),
   VAR("SharedRandCurrentValue", LINELIST_S, SharedRandValues, NULL),
-  { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL }
+  END_OF_CONFIG_VARS
 };
 
 /* "Extra" variable in the state that receives lines we can't parse. This
  * lets us preserve options from versions of Tor newer than us. */
 static config_var_t state_extra_var = {
   "__extra", CONFIG_TYPE_LINELIST,
-  STRUCT_OFFSET(sr_disk_state_t, ExtraLines), NULL
+  offsetof(sr_disk_state_t, ExtraLines), NULL
+  CONF_TEST_MEMBERS(sr_disk_state_t, LINELIST, ExtraLines)
 };
 
 /* Configuration format of sr_disk_state_t. */
 static const config_format_t state_format = {
   sizeof(sr_disk_state_t),
   SR_DISK_STATE_MAGIC,
-  STRUCT_OFFSET(sr_disk_state_t, magic_),
+  offsetof(sr_disk_state_t, magic_),
   NULL,
   NULL,
   state_vars,
   disk_state_validate_cb,
+  disk_state_free_cb,
   &state_extra_var,
 };
 
@@ -110,52 +119,6 @@ get_phase_str(sr_phase_t phase)
 
   return the_string;
 }
-
-/* Return the voting interval of the tor vote subsystem. */
-static int
-get_voting_interval(void)
-{
-  int interval;
-  networkstatus_t *consensus = networkstatus_get_live_consensus(time(NULL));
-
-  if (consensus) {
-    interval = (int)(consensus->fresh_until - consensus->valid_after);
-  } else {
-    /* Same for both a testing and real network. We voluntarily ignore the
-     * InitialVotingInterval since it complexifies things and it doesn't
-     * affect the SR protocol. */
-    interval = get_options()->V3AuthVotingInterval;
-  }
-  tor_assert(interval > 0);
-  return interval;
-}
-
-/* Given the time <b>now</b>, return the start time of the current round of
- * the SR protocol. For example, if it's 23:47:08, the current round thus
- * started at 23:47:00 for a voting interval of 10 seconds. */
-static time_t
-get_start_time_of_current_round(time_t now)
-{
-  const or_options_t *options = get_options();
-  int voting_interval = get_voting_interval();
-  voting_schedule_t *new_voting_schedule =
-    get_voting_schedule(options, now, LOG_INFO);
-  tor_assert(new_voting_schedule);
-
-  /* First, get the start time of the next round */
-  time_t next_start = new_voting_schedule->interval_starts;
-  /* Now roll back next_start by a voting interval to find the start time of
-     the current round. */
-  time_t curr_start = dirvote_get_start_of_next_interval(
-                                     next_start - voting_interval - 1,
-                                     voting_interval,
-                                     options->TestingV3AuthVotingStartOffset);
-
-  voting_schedule_free(new_voting_schedule);
-
-  return curr_start;
-}
-
 /* Return the time we should expire the state file created at <b>now</b>.
  * We expire the state file in the beginning of the next protocol run. */
 STATIC time_t
@@ -167,7 +130,7 @@ get_state_valid_until_time(time_t now)
 
   voting_interval = get_voting_interval();
   /* Find the time the current round started. */
-  beginning_of_current_round = get_start_time_of_current_round(now);
+  beginning_of_current_round = get_start_time_of_current_round();
 
   /* Find how many rounds are left till the end of the protocol run */
   current_round = (now / voting_interval) % total_rounds;
@@ -237,12 +200,15 @@ commit_add_to_state(sr_commit_t *commit, sr_state_t *state)
 static void
 commit_free_(void *p)
 {
-  sr_commit_free(p);
+  sr_commit_free_(p);
 }
 
+#define state_free(val) \
+  FREE_AND_NULL(sr_state_t, state_free_, (val))
+
 /* Free a state that was allocated with state_new(). */
 static void
-state_free(sr_state_t *state)
+state_free_(sr_state_t *state)
 {
   if (state == NULL) {
     return;
@@ -284,9 +250,12 @@ state_set(sr_state_t *state)
   sr_state = state;
 }
 
+#define disk_state_free(val) \
+  FREE_AND_NULL(sr_disk_state_t, disk_state_free_, (val))
+
 /* Free an allocated disk state. */
 static void
-disk_state_free(sr_disk_state_t *state)
+disk_state_free_(sr_disk_state_t *state)
 {
   if (state == NULL) {
     return;
@@ -375,6 +344,12 @@ disk_state_validate_cb(void *old_state, void *state, void *default_state,
   return 0;
 }
 
+static void
+disk_state_free_cb(void *state)
+{
+  disk_state_free_(state);
+}
+
 /* Parse the Commit line(s) in the disk state and translate them to the
  * the memory state. Return 0 on success else -1 on error. */
 static int
@@ -619,8 +594,10 @@ disk_state_update(void)
 {
   config_line_t **next, *line;
 
-  tor_assert(sr_disk_state);
-  tor_assert(sr_state);
+  if (BUG(!sr_disk_state))
+    return;
+  if (BUG(!sr_state))
+    return;
 
   /* Reset current disk state. */
   disk_state_reset();
@@ -784,7 +761,8 @@ disk_state_save_to_disk(void)
 STATIC void
 reset_state_for_new_protocol_run(time_t valid_after)
 {
-  tor_assert(sr_state);
+  if (BUG(!sr_state))
+    return;
 
   /* Keep counters in track */
   sr_state->n_reveal_rounds = 0;
@@ -1062,7 +1040,7 @@ sr_state_get_previous_srv(void)
 }
 
 /* Set the current SRV value from our state. Value CAN be NULL. The srv
- * object ownership is transfered to the state object. */
+ * object ownership is transferred to the state object. */
 void
 sr_state_set_previous_srv(const sr_srv_t *srv)
 {
@@ -1081,7 +1059,7 @@ sr_state_get_current_srv(void)
 }
 
 /* Set the current SRV value from our state. Value CAN be NULL. The srv
- * object ownership is transfered to the state object. */
+ * object ownership is transferred to the state object. */
 void
 sr_state_set_current_srv(const sr_srv_t *srv)
 {
@@ -1116,7 +1094,8 @@ sr_state_update(time_t valid_after)
 {
   sr_phase_t next_phase;
 
-  tor_assert(sr_state);
+  if (BUG(!sr_state))
+    return;
 
   /* Don't call this function twice in the same voting period. */
   if (valid_after <= sr_state->valid_after) {
@@ -1155,7 +1134,8 @@ sr_state_update(time_t valid_after)
   /* Count the current round */
   if (sr_state->phase == SR_PHASE_COMMIT) {
     /* invariant check: we've not entered reveal phase yet */
-    tor_assert(sr_state->n_reveal_rounds == 0);
+    if (BUG(sr_state->n_reveal_rounds != 0))
+      return;
     sr_state->n_commit_rounds++;
   } else {
     sr_state->n_reveal_rounds++;
@@ -1186,7 +1166,7 @@ sr_state_get_commit(const char *rsa_identity)
 }
 
 /* Add <b>commit</b> to the permanent state. The commit object ownership is
- * transfered to the state so the caller MUST not free it. */
+ * transferred to the state so the caller MUST not free it. */
 void
 sr_state_add_commit(sr_commit_t *commit)
 {
@@ -1253,7 +1233,7 @@ sr_state_srv_is_fresh(void)
 
 /* Cleanup and free our disk and memory state. */
 void
-sr_state_free(void)
+sr_state_free_all(void)
 {
   state_free(sr_state);
   disk_state_free(sr_disk_state);
@@ -1330,7 +1310,7 @@ sr_state_init(int save_to_disk, int read_from_disk)
   /* We have a state in memory, let's make sure it's updated for the current
    * and next voting round. */
   {
-    time_t valid_after = get_next_valid_after_time(now);
+    time_t valid_after = voting_schedule_get_next_valid_after_time();
     sr_state_update(valid_after);
   }
   return 0;
@@ -1345,7 +1325,8 @@ sr_state_init(int save_to_disk, int read_from_disk)
 void
 set_sr_phase(sr_phase_t phase)
 {
-  tor_assert(sr_state);
+  if (BUG(!sr_state))
+    return;
   sr_state->phase = phase;
 }
 
@@ -1356,5 +1337,4 @@ get_sr_state(void)
   return sr_state;
 }
 
-#endif /* TOR_UNIT_TESTS */
-
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/or/shared_random_state.h b/src/feature/dirauth/shared_random_state.h
index cf027f2d35..08f999f9d4 100644
--- a/src/or/shared_random_state.h
+++ b/src/feature/dirauth/shared_random_state.h
@@ -1,10 +1,10 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_SHARED_RANDOM_STATE_H
 #define TOR_SHARED_RANDOM_STATE_H
 
-#include "shared_random.h"
+#include "feature/dirauth/shared_random.h"
 
 /* Action that can be performed on the state for any objects. */
 typedef enum {
@@ -77,7 +77,7 @@ typedef struct sr_state_t {
 typedef struct sr_disk_state_t {
   uint32_t magic_;
   /* Version of the protocol. */
-  uint32_t Version;
+  int Version;
   /* Version of our running tor. */
   char *TorVersion;
   /* Creation time of this state */
@@ -85,11 +85,11 @@ typedef struct sr_disk_state_t {
   /* State valid until? */
   time_t ValidUntil;
   /* All commits seen that are valid. */
-  config_line_t *Commit;
+  struct config_line_t *Commit;
   /* Previous and current shared random value. */
-  config_line_t *SharedRandValues;
+  struct config_line_t *SharedRandValues;
   /* Extra Lines for configuration we might not know. */
-  config_line_t *ExtraLines;
+  struct config_line_t *ExtraLines;
 } sr_disk_state_t;
 
 /* API */
@@ -119,7 +119,7 @@ void sr_state_unset_fresh_srv(void);
 int sr_state_init(int save_to_disk, int read_from_disk);
 int sr_state_is_initialized(void);
 void sr_state_save(void);
-void sr_state_free(void);
+void sr_state_free_all(void);
 
 #ifdef SHARED_RANDOM_STATE_PRIVATE
 
@@ -134,7 +134,7 @@ STATIC void new_protocol_run(time_t valid_after);
 STATIC void state_rotate_srv(void);
 STATIC int is_phase_transition(sr_phase_t next_phase);
 
-#endif /* SHARED_RANDOM_STATE_PRIVATE */
+#endif /* defined(SHARED_RANDOM_STATE_PRIVATE) */
 
 #ifdef TOR_UNIT_TESTS
 
@@ -143,7 +143,6 @@ STATIC sr_state_t *get_sr_state(void);
 STATIC void state_del_previous_srv(void);
 STATIC void state_del_current_srv(void);
 
-#endif /* TOR_UNIT_TESTS */
-
-#endif /* TOR_SHARED_RANDOM_STATE_H */
+#endif /* defined(TOR_UNIT_TESTS) */
 
+#endif /* !defined(TOR_SHARED_RANDOM_STATE_H) */
diff --git a/src/feature/dirauth/vote_microdesc_hash_st.h b/src/feature/dirauth/vote_microdesc_hash_st.h
new file mode 100644
index 0000000000..92acdf1157
--- /dev/null
+++ b/src/feature/dirauth/vote_microdesc_hash_st.h
@@ -0,0 +1,22 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef VOTE_MICRODESC_HASH_ST_H
+#define VOTE_MICRODESC_HASH_ST_H
+
+/** Linked list of microdesc hash lines for a single router in a directory
+ * vote.
+ */
+struct vote_microdesc_hash_t {
+  /** Next element in the list, or NULL. */
+  struct vote_microdesc_hash_t *next;
+  /** The raw contents of the microdesc hash line, from the "m" through the
+   * newline. */
+  char *microdesc_hash_line;
+};
+
+#endif
+
diff --git a/src/feature/dirauth/voteflags.c b/src/feature/dirauth/voteflags.c
new file mode 100644
index 0000000000..54c70b989a
--- /dev/null
+++ b/src/feature/dirauth/voteflags.c
@@ -0,0 +1,644 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file voteflags.c
+ * \brief Authority code for deciding the performance thresholds for flags,
+ *   and assigning flags to routers.
+ **/
+
+#define VOTEFLAGS_PRIVATE
+#include "core/or/or.h"
+#include "feature/dirauth/voteflags.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/policies.h"
+#include "feature/dirauth/bwauth.h"
+#include "feature/dirauth/reachability.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/relay/router.h"
+#include "feature/stats/rephist.h"
+
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/vote_routerstatus_st.h"
+
+#include "lib/container/order.h"
+
+/** If a router's uptime is at least this value, then it is always
+ * considered stable, regardless of the rest of the network. This
+ * way we resist attacks where an attacker doubles the size of the
+ * network using allegedly high-uptime nodes, displacing all the
+ * current guards. */
+#define UPTIME_TO_GUARANTEE_STABLE (3600*24*30)
+/** If a router's MTBF is at least this value, then it is always stable.
+ * See above.  (Corresponds to about 7 days for current decay rates.) */
+#define MTBF_TO_GUARANTEE_STABLE (60*60*24*5)
+/** Similarly, every node with at least this much weighted time known can be
+ * considered familiar enough to be a guard.  Corresponds to about 20 days for
+ * current decay rates.
+ */
+#define TIME_KNOWN_TO_GUARANTEE_FAMILIAR (8*24*60*60)
+/** Similarly, every node with sufficient WFU is around enough to be a guard.
+ */
+#define WFU_TO_GUARANTEE_GUARD (0.98)
+
+/* Thresholds for server performance: set by
+ * dirserv_compute_performance_thresholds, and used by
+ * generate_v2_networkstatus */
+
+/** Any router with an uptime of at least this value is stable. */
+static uint32_t stable_uptime = 0; /* start at a safe value */
+/** Any router with an mtbf of at least this value is stable. */
+static double stable_mtbf = 0.0;
+/** If true, we have measured enough mtbf info to look at stable_mtbf rather
+ * than stable_uptime. */
+static int enough_mtbf_info = 0;
+/** Any router with a weighted fractional uptime of at least this much might
+ * be good as a guard. */
+static double guard_wfu = 0.0;
+/** Don't call a router a guard unless we've known about it for at least this
+ * many seconds. */
+static long guard_tk = 0;
+/** Any router with a bandwidth at least this high is "Fast" */
+static uint32_t fast_bandwidth_kb = 0;
+/** If exits can be guards, then all guards must have a bandwidth this
+ * high. */
+static uint32_t guard_bandwidth_including_exits_kb = 0;
+/** If exits can't be guards, then all guards must have a bandwidth this
+ * high. */
+static uint32_t guard_bandwidth_excluding_exits_kb = 0;
+
+/** Helper: estimate the uptime of a router given its stated uptime and the
+ * amount of time since it last stated its stated uptime. */
+static inline long
+real_uptime(const routerinfo_t *router, time_t now)
+{
+  if (now < router->cache_info.published_on)
+    return router->uptime;
+  else
+    return router->uptime + (now - router->cache_info.published_on);
+}
+
+/** Return 1 if <b>router</b> is not suitable for these parameters, else 0.
+ * If <b>need_uptime</b> is non-zero, we require a minimum uptime.
+ * If <b>need_capacity</b> is non-zero, we require a minimum advertised
+ * bandwidth.
+ */
+static int
+dirserv_thinks_router_is_unreliable(time_t now,
+                                    routerinfo_t *router,
+                                    int need_uptime, int need_capacity)
+{
+  if (need_uptime) {
+    if (!enough_mtbf_info) {
+      /* XXXX We should change the rule from
+       * "use uptime if we don't have mtbf data" to "don't advertise Stable on
+       * v3 if we don't have enough mtbf data."  Or maybe not, since if we ever
+       * hit a point where we need to reset a lot of authorities at once,
+       * none of them would be in a position to declare Stable.
+       */
+      long uptime = real_uptime(router, now);
+      if ((unsigned)uptime < stable_uptime &&
+          (unsigned)uptime < UPTIME_TO_GUARANTEE_STABLE)
+        return 1;
+    } else {
+      double mtbf =
+        rep_hist_get_stability(router->cache_info.identity_digest, now);
+      if (mtbf < stable_mtbf &&
+          mtbf < MTBF_TO_GUARANTEE_STABLE)
+        return 1;
+    }
+  }
+  if (need_capacity) {
+    uint32_t bw_kb = dirserv_get_credible_bandwidth_kb(router);
+    if (bw_kb < fast_bandwidth_kb)
+      return 1;
+  }
+  return 0;
+}
+
+/** Return 1 if <b>ri</b>'s descriptor is "active" -- running, valid,
+ * not hibernating, having observed bw greater 0, and not too old. Else
+ * return 0.
+ */
+static int
+router_is_active(const routerinfo_t *ri, const node_t *node, time_t now)
+{
+  time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
+  if (ri->cache_info.published_on < cutoff) {
+    return 0;
+  }
+  if (!node->is_running || !node->is_valid || ri->is_hibernating) {
+    return 0;
+  }
+  /* Only require bandwidth capacity in non-test networks, or
+   * if TestingTorNetwork, and TestingMinExitFlagThreshold is non-zero */
+  if (!ri->bandwidthcapacity) {
+    if (get_options()->TestingTorNetwork) {
+      if (get_options()->TestingMinExitFlagThreshold > 0) {
+        /* If we're in a TestingTorNetwork, and TestingMinExitFlagThreshold is,
+         * then require bandwidthcapacity */
+        return 0;
+      }
+    } else {
+      /* If we're not in a TestingTorNetwork, then require bandwidthcapacity */
+      return 0;
+    }
+  }
+  return 1;
+}
+
+/** Return true iff <b>router</b> should be assigned the "HSDir" flag.
+ *
+ * Right now this means it advertises support for it, it has a high uptime,
+ * it's a directory cache, it has the Stable and Fast flags, and it's currently
+ * considered Running.
+ *
+ * This function needs to be called after router-\>is_running has
+ * been set.
+ */
+static int
+dirserv_thinks_router_is_hs_dir(const routerinfo_t *router,
+                                const node_t *node, time_t now)
+{
+
+  long uptime;
+
+  /* If we haven't been running for at least
+   * get_options()->MinUptimeHidServDirectoryV2 seconds, we can't
+   * have accurate data telling us a relay has been up for at least
+   * that long. We also want to allow a bit of slack: Reachability
+   * tests aren't instant. If we haven't been running long enough,
+   * trust the relay. */
+
+  if (get_uptime() >
+      get_options()->MinUptimeHidServDirectoryV2 * 1.1)
+    uptime = MIN(rep_hist_get_uptime(router->cache_info.identity_digest, now),
+                 real_uptime(router, now));
+  else
+    uptime = real_uptime(router, now);
+
+  return (router->wants_to_be_hs_dir &&
+          router->supports_tunnelled_dir_requests &&
+          node->is_stable && node->is_fast &&
+          uptime >= get_options()->MinUptimeHidServDirectoryV2 &&
+          router_is_active(router, node, now));
+}
+
+/** Don't consider routers with less bandwidth than this when computing
+ * thresholds. */
+#define ABSOLUTE_MIN_BW_VALUE_TO_CONSIDER_KB 4
+
+/** Helper for dirserv_compute_performance_thresholds(): Decide whether to
+ * include a router in our calculations, and return true iff we should; the
+ * require_mbw parameter is passed in by
+ * dirserv_compute_performance_thresholds() and controls whether we ever
+ * count routers with only advertised bandwidths */
+static int
+router_counts_toward_thresholds(const node_t *node, time_t now,
+                                const digestmap_t *omit_as_sybil,
+                                int require_mbw)
+{
+  /* Have measured bw? */
+  int have_mbw =
+    dirserv_has_measured_bw(node->identity);
+  uint64_t min_bw_kb = ABSOLUTE_MIN_BW_VALUE_TO_CONSIDER_KB;
+  const or_options_t *options = get_options();
+
+  if (options->TestingTorNetwork) {
+    min_bw_kb = (int64_t)options->TestingMinExitFlagThreshold / 1000;
+  }
+
+  return node->ri && router_is_active(node->ri, node, now) &&
+    !digestmap_get(omit_as_sybil, node->identity) &&
+    (dirserv_get_credible_bandwidth_kb(node->ri) >= min_bw_kb) &&
+    (have_mbw || !require_mbw);
+}
+
+/** Look through the routerlist, the Mean Time Between Failure history, and
+ * the Weighted Fractional Uptime history, and use them to set thresholds for
+ * the Stable, Fast, and Guard flags.  Update the fields stable_uptime,
+ * stable_mtbf, enough_mtbf_info, guard_wfu, guard_tk, fast_bandwidth,
+ * guard_bandwidth_including_exits, and guard_bandwidth_excluding_exits.
+ *
+ * Also, set the is_exit flag of each router appropriately. */
+void
+dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil)
+{
+  int n_active, n_active_nonexit, n_familiar;
+  uint32_t *uptimes, *bandwidths_kb, *bandwidths_excluding_exits_kb;
+  long *tks;
+  double *mtbfs, *wfus;
+  smartlist_t *nodelist;
+  time_t now = time(NULL);
+  const or_options_t *options = get_options();
+
+  /* Require mbw? */
+  int require_mbw =
+    (dirserv_get_last_n_measured_bws() >
+     options->MinMeasuredBWsForAuthToIgnoreAdvertised) ? 1 : 0;
+
+  /* initialize these all here, in case there are no routers */
+  stable_uptime = 0;
+  stable_mtbf = 0;
+  fast_bandwidth_kb = 0;
+  guard_bandwidth_including_exits_kb = 0;
+  guard_bandwidth_excluding_exits_kb = 0;
+  guard_tk = 0;
+  guard_wfu = 0;
+
+  nodelist_assert_ok();
+  nodelist = nodelist_get_list();
+
+  /* Initialize arrays that will hold values for each router.  We'll
+   * sort them and use that to compute thresholds. */
+  n_active = n_active_nonexit = 0;
+  /* Uptime for every active router. */
+  uptimes = tor_calloc(smartlist_len(nodelist), sizeof(uint32_t));
+  /* Bandwidth for every active router. */
+  bandwidths_kb = tor_calloc(smartlist_len(nodelist), sizeof(uint32_t));
+  /* Bandwidth for every active non-exit router. */
+  bandwidths_excluding_exits_kb =
+    tor_calloc(smartlist_len(nodelist), sizeof(uint32_t));
+  /* Weighted mean time between failure for each active router. */
+  mtbfs = tor_calloc(smartlist_len(nodelist), sizeof(double));
+  /* Time-known for each active router. */
+  tks = tor_calloc(smartlist_len(nodelist), sizeof(long));
+  /* Weighted fractional uptime for each active router. */
+  wfus = tor_calloc(smartlist_len(nodelist), sizeof(double));
+
+  /* Now, fill in the arrays. */
+  SMARTLIST_FOREACH_BEGIN(nodelist, node_t *, node) {
+    if (options->BridgeAuthoritativeDir &&
+        node->ri &&
+        node->ri->purpose != ROUTER_PURPOSE_BRIDGE)
+      continue;
+
+    routerinfo_t *ri = node->ri;
+    if (ri) {
+      node->is_exit = (!router_exit_policy_rejects_all(ri) &&
+                       exit_policy_is_general_exit(ri->exit_policy));
+    }
+
+    if (router_counts_toward_thresholds(node, now, omit_as_sybil,
+                                        require_mbw)) {
+      const char *id = node->identity;
+      uint32_t bw_kb;
+
+      /* resolve spurious clang shallow analysis null pointer errors */
+      tor_assert(ri);
+
+      uptimes[n_active] = (uint32_t)real_uptime(ri, now);
+      mtbfs[n_active] = rep_hist_get_stability(id, now);
+      tks  [n_active] = rep_hist_get_weighted_time_known(id, now);
+      bandwidths_kb[n_active] = bw_kb = dirserv_get_credible_bandwidth_kb(ri);
+      if (!node->is_exit || node->is_bad_exit) {
+        bandwidths_excluding_exits_kb[n_active_nonexit] = bw_kb;
+        ++n_active_nonexit;
+      }
+      ++n_active;
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  /* Now, compute thresholds. */
+  if (n_active) {
+    /* The median uptime is stable. */
+    stable_uptime = median_uint32(uptimes, n_active);
+    /* The median mtbf is stable, if we have enough mtbf info */
+    stable_mtbf = median_double(mtbfs, n_active);
+    /* The 12.5th percentile bandwidth is fast. */
+    fast_bandwidth_kb = find_nth_uint32(bandwidths_kb, n_active, n_active/8);
+    /* (Now bandwidths is sorted.) */
+    if (fast_bandwidth_kb < RELAY_REQUIRED_MIN_BANDWIDTH/(2 * 1000))
+      fast_bandwidth_kb = bandwidths_kb[n_active/4];
+    guard_bandwidth_including_exits_kb =
+      third_quartile_uint32(bandwidths_kb, n_active);
+    guard_tk = find_nth_long(tks, n_active, n_active/8);
+  }
+
+  if (guard_tk > TIME_KNOWN_TO_GUARANTEE_FAMILIAR)
+    guard_tk = TIME_KNOWN_TO_GUARANTEE_FAMILIAR;
+
+  {
+    /* We can vote on a parameter for the minimum and maximum. */
+#define ABSOLUTE_MIN_VALUE_FOR_FAST_FLAG 4
+    int32_t min_fast_kb, max_fast_kb, min_fast, max_fast;
+    min_fast = networkstatus_get_param(NULL, "FastFlagMinThreshold",
+      ABSOLUTE_MIN_VALUE_FOR_FAST_FLAG,
+      ABSOLUTE_MIN_VALUE_FOR_FAST_FLAG,
+      INT32_MAX);
+    if (options->TestingTorNetwork) {
+      min_fast = (int32_t)options->TestingMinFastFlagThreshold;
+    }
+    max_fast = networkstatus_get_param(NULL, "FastFlagMaxThreshold",
+                                       INT32_MAX, min_fast, INT32_MAX);
+    min_fast_kb = min_fast / 1000;
+    max_fast_kb = max_fast / 1000;
+
+    if (fast_bandwidth_kb < (uint32_t)min_fast_kb)
+      fast_bandwidth_kb = min_fast_kb;
+    if (fast_bandwidth_kb > (uint32_t)max_fast_kb)
+      fast_bandwidth_kb = max_fast_kb;
+  }
+  /* Protect sufficiently fast nodes from being pushed out of the set
+   * of Fast nodes. */
+  if (options->AuthDirFastGuarantee &&
+      fast_bandwidth_kb > options->AuthDirFastGuarantee/1000)
+    fast_bandwidth_kb = (uint32_t)options->AuthDirFastGuarantee/1000;
+
+  /* Now that we have a time-known that 7/8 routers are known longer than,
+   * fill wfus with the wfu of every such "familiar" router. */
+  n_familiar = 0;
+
+  SMARTLIST_FOREACH_BEGIN(nodelist, node_t *, node) {
+      if (router_counts_toward_thresholds(node, now,
+                                          omit_as_sybil, require_mbw)) {
+        routerinfo_t *ri = node->ri;
+        const char *id = ri->cache_info.identity_digest;
+        long tk = rep_hist_get_weighted_time_known(id, now);
+        if (tk < guard_tk)
+          continue;
+        wfus[n_familiar++] = rep_hist_get_weighted_fractional_uptime(id, now);
+      }
+  } SMARTLIST_FOREACH_END(node);
+  if (n_familiar)
+    guard_wfu = median_double(wfus, n_familiar);
+  if (guard_wfu > WFU_TO_GUARANTEE_GUARD)
+    guard_wfu = WFU_TO_GUARANTEE_GUARD;
+
+  enough_mtbf_info = rep_hist_have_measured_enough_stability();
+
+  if (n_active_nonexit) {
+    guard_bandwidth_excluding_exits_kb =
+      find_nth_uint32(bandwidths_excluding_exits_kb,
+                      n_active_nonexit, n_active_nonexit*3/4);
+  }
+
+  log_info(LD_DIRSERV,
+      "Cutoffs: For Stable, %lu sec uptime, %lu sec MTBF. "
+      "For Fast: %lu kilobytes/sec. "
+      "For Guard: WFU %.03f%%, time-known %lu sec, "
+      "and bandwidth %lu or %lu kilobytes/sec. "
+      "We%s have enough stability data.",
+      (unsigned long)stable_uptime,
+      (unsigned long)stable_mtbf,
+      (unsigned long)fast_bandwidth_kb,
+      guard_wfu*100,
+      (unsigned long)guard_tk,
+      (unsigned long)guard_bandwidth_including_exits_kb,
+      (unsigned long)guard_bandwidth_excluding_exits_kb,
+      enough_mtbf_info ? "" : " don't");
+
+  tor_free(uptimes);
+  tor_free(mtbfs);
+  tor_free(bandwidths_kb);
+  tor_free(bandwidths_excluding_exits_kb);
+  tor_free(tks);
+  tor_free(wfus);
+}
+
+/* Use dirserv_compute_performance_thresholds() to compute the thresholds
+ * for the status flags, specifically for bridges.
+ *
+ * This is only called by a Bridge Authority from
+ * networkstatus_getinfo_by_purpose().
+ */
+void
+dirserv_compute_bridge_flag_thresholds(void)
+{
+  digestmap_t *omit_as_sybil = digestmap_new();
+  dirserv_compute_performance_thresholds(omit_as_sybil);
+  digestmap_free(omit_as_sybil, NULL);
+}
+
+/** Give a statement of our current performance thresholds for inclusion
+ * in a vote document. */
+char *
+dirserv_get_flag_thresholds_line(void)
+{
+  char *result=NULL;
+  const int measured_threshold =
+    get_options()->MinMeasuredBWsForAuthToIgnoreAdvertised;
+  const int enough_measured_bw =
+    dirserv_get_last_n_measured_bws() > measured_threshold;
+
+  tor_asprintf(&result,
+      "stable-uptime=%lu stable-mtbf=%lu "
+      "fast-speed=%lu "
+      "guard-wfu=%.03f%% guard-tk=%lu "
+      "guard-bw-inc-exits=%lu guard-bw-exc-exits=%lu "
+      "enough-mtbf=%d ignoring-advertised-bws=%d",
+      (unsigned long)stable_uptime,
+      (unsigned long)stable_mtbf,
+      (unsigned long)fast_bandwidth_kb*1000,
+      guard_wfu*100,
+      (unsigned long)guard_tk,
+      (unsigned long)guard_bandwidth_including_exits_kb*1000,
+      (unsigned long)guard_bandwidth_excluding_exits_kb*1000,
+      enough_mtbf_info ? 1 : 0,
+      enough_measured_bw ? 1 : 0);
+
+  return result;
+}
+
+/* DOCDOC running_long_enough_to_decide_unreachable */
+int
+running_long_enough_to_decide_unreachable(void)
+{
+  return time_of_process_start
+    + get_options()->TestingAuthDirTimeToLearnReachability < approx_time();
+}
+
+/** Each server needs to have passed a reachability test no more
+ * than this number of seconds ago, or it is listed as down in
+ * the directory. */
+#define REACHABLE_TIMEOUT (45*60)
+
+/** If we tested a router and found it reachable _at least this long_ after it
+ * declared itself hibernating, it is probably done hibernating and we just
+ * missed a descriptor from it. */
+#define HIBERNATION_PUBLICATION_SKEW (60*60)
+
+/** Treat a router as alive if
+ *    - It's me, and I'm not hibernating.
+ * or - We've found it reachable recently. */
+void
+dirserv_set_router_is_running(routerinfo_t *router, time_t now)
+{
+  /*XXXX This function is a mess.  Separate out the part that calculates
+    whether it's reachable and the part that tells rephist that the router was
+    unreachable.
+   */
+  int answer;
+  const or_options_t *options = get_options();
+  node_t *node = node_get_mutable_by_id(router->cache_info.identity_digest);
+  tor_assert(node);
+
+  if (router_is_me(router)) {
+    /* We always know if we are shutting down or hibernating ourselves. */
+    answer = ! we_are_hibernating();
+  } else if (router->is_hibernating &&
+             (router->cache_info.published_on +
+              HIBERNATION_PUBLICATION_SKEW) > node->last_reachable) {
+    /* A hibernating router is down unless we (somehow) had contact with it
+     * since it declared itself to be hibernating. */
+    answer = 0;
+  } else if (options->AssumeReachable) {
+    /* If AssumeReachable, everybody is up unless they say they are down! */
+    answer = 1;
+  } else {
+    /* Otherwise, a router counts as up if we found all announced OR
+       ports reachable in the last REACHABLE_TIMEOUT seconds.
+
+       XXX prop186 For now there's always one IPv4 and at most one
+       IPv6 OR port.
+
+       If we're not on IPv6, don't consider reachability of potential
+       IPv6 OR port since that'd kill all dual stack relays until a
+       majority of the dir auths have IPv6 connectivity. */
+    answer = (now < node->last_reachable + REACHABLE_TIMEOUT &&
+              (options->AuthDirHasIPv6Connectivity != 1 ||
+               tor_addr_is_null(&router->ipv6_addr) ||
+               now < node->last_reachable6 + REACHABLE_TIMEOUT));
+  }
+
+  if (!answer && running_long_enough_to_decide_unreachable()) {
+    /* Not considered reachable. tell rephist about that.
+
+       Because we launch a reachability test for each router every
+       REACHABILITY_TEST_CYCLE_PERIOD seconds, then the router has probably
+       been down since at least that time after we last successfully reached
+       it.
+
+       XXX ipv6
+     */
+    time_t when = now;
+    if (node->last_reachable &&
+        node->last_reachable + REACHABILITY_TEST_CYCLE_PERIOD < now)
+      when = node->last_reachable + REACHABILITY_TEST_CYCLE_PERIOD;
+    rep_hist_note_router_unreachable(router->cache_info.identity_digest, when);
+  }
+
+  node->is_running = answer;
+}
+
+/** Extract status information from <b>ri</b> and from other authority
+ * functions and store it in <b>rs</b>. <b>rs</b> is zeroed out before it is
+ * set.
+ *
+ * We assume that ri-\>is_running has already been set, e.g. by
+ *   dirserv_set_router_is_running(ri, now);
+ */
+void
+set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                 node_t *node,
+                                 routerinfo_t *ri,
+                                 time_t now,
+                                 int listbadexits)
+{
+  const or_options_t *options = get_options();
+  uint32_t routerbw_kb = dirserv_get_credible_bandwidth_kb(ri);
+
+  memset(rs, 0, sizeof(routerstatus_t));
+
+  rs->is_authority =
+    router_digest_is_trusted_dir(ri->cache_info.identity_digest);
+
+  /* Already set by compute_performance_thresholds. */
+  rs->is_exit = node->is_exit;
+  rs->is_stable = node->is_stable =
+    !dirserv_thinks_router_is_unreliable(now, ri, 1, 0);
+  rs->is_fast = node->is_fast =
+    !dirserv_thinks_router_is_unreliable(now, ri, 0, 1);
+  rs->is_flagged_running = node->is_running; /* computed above */
+
+  rs->is_valid = node->is_valid;
+
+  if (node->is_fast && node->is_stable &&
+      ri->supports_tunnelled_dir_requests &&
+      ((options->AuthDirGuardBWGuarantee &&
+        routerbw_kb >= options->AuthDirGuardBWGuarantee/1000) ||
+       routerbw_kb >= MIN(guard_bandwidth_including_exits_kb,
+                          guard_bandwidth_excluding_exits_kb))) {
+    long tk = rep_hist_get_weighted_time_known(
+                                      node->identity, now);
+    double wfu = rep_hist_get_weighted_fractional_uptime(
+                                      node->identity, now);
+    rs->is_possible_guard = (wfu >= guard_wfu && tk >= guard_tk) ? 1 : 0;
+  } else {
+    rs->is_possible_guard = 0;
+  }
+
+  rs->is_bad_exit = listbadexits && node->is_bad_exit;
+  rs->is_hs_dir = node->is_hs_dir =
+    dirserv_thinks_router_is_hs_dir(ri, node, now);
+
+  rs->is_named = rs->is_unnamed = 0;
+
+  rs->published_on = ri->cache_info.published_on;
+  memcpy(rs->identity_digest, node->identity, DIGEST_LEN);
+  memcpy(rs->descriptor_digest, ri->cache_info.signed_descriptor_digest,
+         DIGEST_LEN);
+  rs->addr = ri->addr;
+  strlcpy(rs->nickname, ri->nickname, sizeof(rs->nickname));
+  rs->or_port = ri->or_port;
+  rs->dir_port = ri->dir_port;
+  rs->is_v2_dir = ri->supports_tunnelled_dir_requests;
+  if (options->AuthDirHasIPv6Connectivity == 1 &&
+      !tor_addr_is_null(&ri->ipv6_addr) &&
+      node->last_reachable6 >= now - REACHABLE_TIMEOUT) {
+    /* We're configured as having IPv6 connectivity. There's an IPv6
+       OR port and it's reachable so copy it to the routerstatus.  */
+    tor_addr_copy(&rs->ipv6_addr, &ri->ipv6_addr);
+    rs->ipv6_orport = ri->ipv6_orport;
+  } else {
+    tor_addr_make_null(&rs->ipv6_addr, AF_INET6);
+    rs->ipv6_orport = 0;
+  }
+
+  if (options->TestingTorNetwork) {
+    dirserv_set_routerstatus_testing(rs);
+  }
+}
+
+/** Use TestingDirAuthVoteExit, TestingDirAuthVoteGuard, and
+ * TestingDirAuthVoteHSDir to give out the Exit, Guard, and HSDir flags,
+ * respectively. But don't set the corresponding node flags.
+ * Should only be called if TestingTorNetwork is set. */
+STATIC void
+dirserv_set_routerstatus_testing(routerstatus_t *rs)
+{
+  const or_options_t *options = get_options();
+
+  tor_assert(options->TestingTorNetwork);
+
+  if (routerset_contains_routerstatus(options->TestingDirAuthVoteExit,
+                                      rs, 0)) {
+    rs->is_exit = 1;
+  } else if (options->TestingDirAuthVoteExitIsStrict) {
+    rs->is_exit = 0;
+  }
+
+  if (routerset_contains_routerstatus(options->TestingDirAuthVoteGuard,
+                                      rs, 0)) {
+    rs->is_possible_guard = 1;
+  } else if (options->TestingDirAuthVoteGuardIsStrict) {
+    rs->is_possible_guard = 0;
+  }
+
+  if (routerset_contains_routerstatus(options->TestingDirAuthVoteHSDir,
+                                      rs, 0)) {
+    rs->is_hs_dir = 1;
+  } else if (options->TestingDirAuthVoteHSDirIsStrict) {
+    rs->is_hs_dir = 0;
+  }
+}
diff --git a/src/feature/dirauth/voteflags.h b/src/feature/dirauth/voteflags.h
new file mode 100644
index 0000000000..aa7b6ed082
--- /dev/null
+++ b/src/feature/dirauth/voteflags.h
@@ -0,0 +1,31 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file voteflags.h
+ * \brief Header file for voteflags.c
+ **/
+
+#ifndef TOR_VOTEFLAGS_H
+#define TOR_VOTEFLAGS_H
+
+void dirserv_set_router_is_running(routerinfo_t *router, time_t now);
+char *dirserv_get_flag_thresholds_line(void);
+void dirserv_compute_bridge_flag_thresholds(void);
+int running_long_enough_to_decide_unreachable(void);
+
+void set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                      node_t *node,
+                                      routerinfo_t *ri, time_t now,
+                                      int listbadexits);
+
+void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil);
+
+#ifdef VOTEFLAGS_PRIVATE
+STATIC void dirserv_set_routerstatus_testing(routerstatus_t *rs);
+#endif
+
+#endif
diff --git a/src/feature/dircache/cached_dir_st.h b/src/feature/dircache/cached_dir_st.h
new file mode 100644
index 0000000000..71dca8c3a2
--- /dev/null
+++ b/src/feature/dircache/cached_dir_st.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef CACHED_DIR_ST_H
+#define CACHED_DIR_ST_H
+
+/** A cached_dir_t represents a cacheable directory object, along with its
+ * compressed form. */
+struct cached_dir_t {
+  char *dir; /**< Contents of this object, NUL-terminated. */
+  char *dir_compressed; /**< Compressed contents of this object. */
+  size_t dir_len; /**< Length of <b>dir</b> (not counting its NUL). */
+  size_t dir_compressed_len; /**< Length of <b>dir_compressed</b>. */
+  time_t published; /**< When was this object published. */
+  common_digests_t digests; /**< Digests of this object (networkstatus only) */
+  /** Sha3 digest (also ns only) */
+  uint8_t digest_sha3_as_signed[DIGEST256_LEN];
+  int refcnt; /**< Reference count for this cached_dir_t. */
+};
+
+#endif
+
diff --git a/src/feature/dircache/conscache.c b/src/feature/dircache/conscache.c
new file mode 100644
index 0000000000..cf4fe8701d
--- /dev/null
+++ b/src/feature/dircache/conscache.c
@@ -0,0 +1,627 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "feature/dircache/conscache.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/fs/storagedir.h"
+#include "lib/encoding/confline.h"
+
+#define CCE_MAGIC 0x17162253
+
+#ifdef _WIN32
+/* On Windows, unlink won't work on a file if the file is actively mmap()ed.
+ * That forces us to be less aggressive about unlinking files, and causes other
+ * changes throughout our logic.
+ */
+#define MUST_UNMAP_TO_UNLINK
+#endif /* defined(_WIN32) */
+
+/**
+ * A consensus_cache_entry_t is a reference-counted handle to an
+ * item in a consensus_cache_t.  It can be mmapped into RAM, or not,
+ * depending whether it's currently in use.
+ */
+struct consensus_cache_entry_t {
+  uint32_t magic; /**< Must be set to CCE_MAGIC */
+  HANDLE_ENTRY(consensus_cache_entry, consensus_cache_entry_t);
+  int32_t refcnt; /**< Reference count. */
+  unsigned can_remove : 1; /**< If true, we want to delete this file. */
+  /** If true, we intend to unmap this file as soon as we're done with it. */
+  unsigned release_aggressively : 1;
+
+  /** Filename for this object within the storage_dir_t */
+  char *fname;
+  /** Labels associated with this object. Immutable once the object
+   * is created. */
+  config_line_t *labels;
+  /** Pointer to the cache that includes this entry (if any). */
+  consensus_cache_t *in_cache;
+
+  /** Since what time has this object been mapped into RAM, but with the cache
+   * being the only having a reference to it? */
+  time_t unused_since;
+  /** mmaped contents of the underlying file.  May be NULL */
+  tor_mmap_t *map;
+  /** Length of the body within <b>map</b>. */
+  size_t bodylen;
+  /** Pointer to the body within <b>map</b>. */
+  const uint8_t *body;
+};
+
+/**
+ * A consensus_cache_t holds a directory full of labeled items.
+ */
+struct consensus_cache_t {
+  /** Underling storage_dir_t to handle persistence */
+  storage_dir_t *dir;
+  /** List of all the entries in the directory. */
+  smartlist_t *entries;
+
+  /** The maximum number of entries that we'd like to allow in this cache.
+   * This is the same as the storagedir limit when MUST_UNMAP_TO_UNLINK is
+   * not defined. */
+  unsigned max_entries;
+};
+
+static void consensus_cache_clear(consensus_cache_t *cache);
+static void consensus_cache_rescan(consensus_cache_t *);
+static void consensus_cache_entry_map(consensus_cache_t *,
+                                      consensus_cache_entry_t *);
+static void consensus_cache_entry_unmap(consensus_cache_entry_t *ent);
+
+/**
+ * Helper: Open a consensus cache in subdirectory <b>subdir</b> of the
+ * data directory, to hold up to <b>max_entries</b> of data.
+ */
+consensus_cache_t *
+consensus_cache_open(const char *subdir, int max_entries)
+{
+  int storagedir_max_entries;
+  consensus_cache_t *cache = tor_malloc_zero(sizeof(consensus_cache_t));
+  char *directory = get_cachedir_fname(subdir);
+  cache->max_entries = max_entries;
+
+#ifdef MUST_UNMAP_TO_UNLINK
+  /* If we can't unlink the files that we're still using, then we need to
+   * tell the storagedir backend to allow far more files than this consensus
+   * cache actually wants, so that it can hold files which, from this cache's
+   * perspective, have become useless.
+   */
+#define VERY_LARGE_STORAGEDIR_LIMIT (1000*1000)
+  storagedir_max_entries = VERY_LARGE_STORAGEDIR_LIMIT;
+#else /* !(defined(MUST_UNMAP_TO_UNLINK)) */
+  /* Otherwise, we can just tell the storagedir to use the same limits
+   * as this cache. */
+  storagedir_max_entries = max_entries;
+#endif /* defined(MUST_UNMAP_TO_UNLINK) */
+
+  cache->dir = storage_dir_new(directory, storagedir_max_entries);
+  tor_free(directory);
+  if (!cache->dir) {
+    tor_free(cache);
+    return NULL;
+  }
+
+  consensus_cache_rescan(cache);
+  return cache;
+}
+
+/** Return true if it's okay to put more entries in this cache than
+ * its official file limit.
+ *
+ * (We need this method on Windows, where we can't unlink files that are still
+ * in use, and therefore might need to temporarily exceed the file limit until
+ * the no-longer-wanted files are deletable.)
+ */
+int
+consensus_cache_may_overallocate(consensus_cache_t *cache)
+{
+  (void) cache;
+#ifdef MUST_UNMAP_TO_UNLINK
+  return 1;
+#else
+  return 0;
+#endif
+}
+
+/**
+ * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
+ * operations that <b>cache</b> will need.
+ */
+int
+consensus_cache_register_with_sandbox(consensus_cache_t *cache,
+                                      struct sandbox_cfg_elem **cfg)
+{
+#ifdef MUST_UNMAP_TO_UNLINK
+  /* Our Linux sandbox doesn't support huge file lists like the one that would
+   * be generated by using VERY_LARGE_STORAGEDIR_LIMIT above in
+   * consensus_cache_open().  Since the Linux sandbox is the only one we have
+   * right now, we just assert that we never reach this point when we've had
+   * to use VERY_LARGE_STORAGEDIR_LIMIT.
+   *
+   * If at some point in the future we have a different sandbox mechanism that
+   * can handle huge file lists, we can remove this assertion or make it
+   * conditional.
+   */
+  tor_assert_nonfatal_unreached();
+#endif /* defined(MUST_UNMAP_TO_UNLINK) */
+  return storage_dir_register_with_sandbox(cache->dir, cfg);
+}
+
+/**
+ * Helper: clear all entries from <b>cache</b> (but do not delete
+ * any that aren't marked for removal
+ */
+static void
+consensus_cache_clear(consensus_cache_t *cache)
+{
+  consensus_cache_delete_pending(cache, 0);
+
+  SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+    ent->in_cache = NULL;
+    consensus_cache_entry_decref(ent);
+  } SMARTLIST_FOREACH_END(ent);
+  smartlist_free(cache->entries);
+  cache->entries = NULL;
+}
+
+/**
+ * Drop all storage held by <b>cache</b>.
+ */
+void
+consensus_cache_free_(consensus_cache_t *cache)
+{
+  if (! cache)
+    return;
+
+  if (cache->entries) {
+    consensus_cache_clear(cache);
+  }
+  storage_dir_free(cache->dir);
+  tor_free(cache);
+}
+
+/**
+ * Write <b>datalen</b> bytes of data at <b>data</b> into the <b>cache</b>,
+ * labeling that data with <b>labels</b>.  On failure, return NULL. On
+ * success, return a newly created consensus_cache_entry_t.
+ *
+ * The returned value will be owned by the cache, and you will have a
+ * reference to it.  Call consensus_cache_entry_decref() when you are
+ * done with it.
+ *
+ * The provided <b>labels</b> MUST have distinct keys: if they don't,
+ * this API does not specify which values (if any) for the duplicate keys
+ * will be considered.
+ */
+consensus_cache_entry_t *
+consensus_cache_add(consensus_cache_t *cache,
+                    const config_line_t *labels,
+                    const uint8_t *data,
+                    size_t datalen)
+{
+  char *fname = NULL;
+  int r = storage_dir_save_labeled_to_file(cache->dir,
+                                            labels, data, datalen, &fname);
+  if (r < 0 || fname == NULL) {
+    return NULL;
+  }
+  consensus_cache_entry_t *ent =
+    tor_malloc_zero(sizeof(consensus_cache_entry_t));
+  ent->magic = CCE_MAGIC;
+  ent->fname = fname;
+  ent->labels = config_lines_dup(labels);
+  ent->in_cache = cache;
+  ent->unused_since = TIME_MAX;
+  smartlist_add(cache->entries, ent);
+  /* Start the reference count at 2: the caller owns one copy, and the
+   * cache owns another.
+   */
+  ent->refcnt = 2;
+
+  return ent;
+}
+
+/**
+ * Given a <b>cache</b>, return some entry for which <b>key</b>=<b>value</b>.
+ * Return NULL if no such entry exists.
+ *
+ * Does not adjust reference counts.
+ */
+consensus_cache_entry_t *
+consensus_cache_find_first(consensus_cache_t *cache,
+                           const char *key,
+                           const char *value)
+{
+  smartlist_t *tmp = smartlist_new();
+  consensus_cache_find_all(tmp, cache, key, value);
+  consensus_cache_entry_t *ent = NULL;
+  if (smartlist_len(tmp))
+    ent = smartlist_get(tmp, 0);
+  smartlist_free(tmp);
+  return ent;
+}
+
+/**
+ * Given a <b>cache</b>, add every entry to <b>out<b> for which
+ * <b>key</b>=<b>value</b>.  If <b>key</b> is NULL, add every entry.
+ *
+ * Do not add any entry that has been marked for removal.
+ *
+ * Does not adjust reference counts.
+ */
+void
+consensus_cache_find_all(smartlist_t *out,
+                         consensus_cache_t *cache,
+                         const char *key,
+                         const char *value)
+{
+  SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+    if (ent->can_remove == 1) {
+      /* We want to delete this; pretend it isn't there. */
+      continue;
+    }
+    if (! key) {
+      smartlist_add(out, ent);
+      continue;
+    }
+    const char *found_val = consensus_cache_entry_get_value(ent, key);
+    if (found_val && !strcmp(value, found_val)) {
+      smartlist_add(out, ent);
+    }
+  } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * Given a list of consensus_cache_entry_t, remove all those entries
+ * that do not have <b>key</b>=<b>value</b> in their labels.
+ *
+ * Does not adjust reference counts.
+ */
+void
+consensus_cache_filter_list(smartlist_t *lst,
+                            const char *key,
+                            const char *value)
+{
+  if (BUG(lst == NULL))
+    return; // LCOV_EXCL_LINE
+  if (key == NULL)
+    return;
+  SMARTLIST_FOREACH_BEGIN(lst, consensus_cache_entry_t *, ent) {
+    const char *found_val = consensus_cache_entry_get_value(ent, key);
+    if (! found_val || strcmp(value, found_val)) {
+      SMARTLIST_DEL_CURRENT(lst, ent);
+    }
+  } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * If <b>ent</b> has a label with the given <b>key</b>, return its
+ * value.  Otherwise return NULL.
+ *
+ * The return value is only guaranteed to be valid for as long as you
+ * hold a reference to <b>ent</b>.
+ */
+const char *
+consensus_cache_entry_get_value(const consensus_cache_entry_t *ent,
+                                const char *key)
+{
+  const config_line_t *match = config_line_find(ent->labels, key);
+  if (match)
+    return match->value;
+  else
+    return NULL;
+}
+
+/**
+ * Return a pointer to the labels in <b>ent</b>.
+ *
+ * This pointer is only guaranteed to be valid for as long as you
+ * hold a reference to <b>ent</b>.
+ */
+const config_line_t *
+consensus_cache_entry_get_labels(const consensus_cache_entry_t *ent)
+{
+  return ent->labels;
+}
+
+/**
+ * Increase the reference count of <b>ent</b>.
+ */
+void
+consensus_cache_entry_incref(consensus_cache_entry_t *ent)
+{
+  if (BUG(ent->magic != CCE_MAGIC))
+    return; // LCOV_EXCL_LINE
+  ++ent->refcnt;
+  ent->unused_since = TIME_MAX;
+}
+
+/**
+ * Release a reference held to <b>ent</b>.
+ *
+ * If it was the last reference, ent will be freed. Therefore, you must not
+ * use <b>ent</b> after calling this function.
+ */
+void
+consensus_cache_entry_decref(consensus_cache_entry_t *ent)
+{
+  if (! ent)
+    return;
+  if (BUG(ent->refcnt <= 0))
+    return; // LCOV_EXCL_LINE
+  if (BUG(ent->magic != CCE_MAGIC))
+    return; // LCOV_EXCL_LINE
+
+  --ent->refcnt;
+
+  if (ent->refcnt == 1 && ent->in_cache) {
+    /* Only the cache has a reference: we don't need to keep the file
+     * mapped */
+    if (ent->map) {
+      if (ent->release_aggressively) {
+        consensus_cache_entry_unmap(ent);
+      } else {
+        ent->unused_since = approx_time();
+      }
+    }
+    return;
+  }
+
+  if (ent->refcnt > 0)
+    return;
+
+  /* Refcount is zero; we can free it. */
+  if (ent->map) {
+    consensus_cache_entry_unmap(ent);
+  }
+  tor_free(ent->fname);
+  config_free_lines(ent->labels);
+  consensus_cache_entry_handles_clear(ent);
+  memwipe(ent, 0, sizeof(consensus_cache_entry_t));
+  tor_free(ent);
+}
+
+/**
+ * Mark <b>ent</b> for deletion from the cache.  Deletion will not occur
+ * until the cache is the only place that holds a reference to <b>ent</b>.
+ */
+void
+consensus_cache_entry_mark_for_removal(consensus_cache_entry_t *ent)
+{
+  ent->can_remove = 1;
+}
+
+/**
+ * Mark <b>ent</b> as the kind of entry that we don't need to keep mmap'd for
+ * any longer than we're actually using it.
+ */
+void
+consensus_cache_entry_mark_for_aggressive_release(consensus_cache_entry_t *ent)
+{
+  ent->release_aggressively = 1;
+}
+
+/**
+ * Try to read the body of <b>ent</b> into memory if it isn't already
+ * loaded.  On success, set *<b>body_out</b> to the body, *<b>sz_out</b>
+ * to its size, and return 0.  On failure return -1.
+ *
+ * The resulting body pointer will only be valid for as long as you
+ * hold a reference to <b>ent</b>.
+ */
+int
+consensus_cache_entry_get_body(const consensus_cache_entry_t *ent,
+                               const uint8_t **body_out,
+                               size_t *sz_out)
+{
+  if (BUG(ent->magic != CCE_MAGIC))
+    return -1; // LCOV_EXCL_LINE
+
+  if (! ent->map) {
+    if (! ent->in_cache)
+      return -1;
+
+    consensus_cache_entry_map((consensus_cache_t *)ent->in_cache,
+                              (consensus_cache_entry_t *)ent);
+    if (! ent->map) {
+      return -1;
+    }
+  }
+
+  *body_out = ent->body;
+  *sz_out = ent->bodylen;
+  return 0;
+}
+
+/**
+ * Unmap every mmap'd element of <b>cache</b> that has been unused
+ * since <b>cutoff</b>.
+ */
+void
+consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff)
+{
+  SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+    tor_assert_nonfatal(ent->in_cache == cache);
+    if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) {
+      /* Somebody is using this entry right now */
+      continue;
+    }
+    if (ent->unused_since > cutoff) {
+      /* Has been unused only for a little while */
+      continue;
+    }
+    if (ent->map == NULL) {
+      /* Not actually mapped. */
+      continue;
+    }
+    consensus_cache_entry_unmap(ent);
+  } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * Return the number of currently unused filenames available in this cache.
+ */
+int
+consensus_cache_get_n_filenames_available(consensus_cache_t *cache)
+{
+  tor_assert(cache);
+  int max = cache->max_entries;
+  int used = smartlist_len(storage_dir_list(cache->dir));
+#ifdef MUST_UNMAP_TO_UNLINK
+  if (used > max)
+    return 0;
+#else
+  tor_assert_nonfatal(max >= used);
+#endif /* defined(MUST_UNMAP_TO_UNLINK) */
+  return max - used;
+}
+
+/**
+ * Delete every element of <b>cache</b> has been marked with
+ * consensus_cache_entry_mark_for_removal. If <b>force</b> is false,
+ * retain those entries which are in use by something other than the cache.
+ */
+void
+consensus_cache_delete_pending(consensus_cache_t *cache, int force)
+{
+  SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+    tor_assert_nonfatal(ent->in_cache == cache);
+    int force_ent = force;
+#ifdef MUST_UNMAP_TO_UNLINK
+    /* We cannot delete anything with an active mmap on win32, so no
+     * force-deletion. */
+    if (ent->map) {
+      force_ent = 0;
+    }
+#endif /* defined(MUST_UNMAP_TO_UNLINK) */
+    if (! force_ent) {
+      if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) {
+        /* Somebody is using this entry right now */
+        continue;
+      }
+    }
+    if (ent->can_remove == 0) {
+      /* Don't want to delete this. */
+      continue;
+    }
+    if (BUG(ent->refcnt <= 0)) {
+      continue; // LCOV_EXCL_LINE
+    }
+
+    SMARTLIST_DEL_CURRENT(cache->entries, ent);
+    ent->in_cache = NULL;
+    char *fname = tor_strdup(ent->fname); /* save a copy */
+    consensus_cache_entry_decref(ent);
+    storage_dir_remove_file(cache->dir, fname);
+    tor_free(fname);
+  } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * Internal helper: rescan <b>cache</b> and rebuild its list of entries.
+ */
+static void
+consensus_cache_rescan(consensus_cache_t *cache)
+{
+  if (cache->entries) {
+    consensus_cache_clear(cache);
+  }
+
+  cache->entries = smartlist_new();
+  const smartlist_t *fnames = storage_dir_list(cache->dir);
+  SMARTLIST_FOREACH_BEGIN(fnames, const char *, fname) {
+    tor_mmap_t *map = NULL;
+    config_line_t *labels = NULL;
+    const uint8_t *body;
+    size_t bodylen;
+    map = storage_dir_map_labeled(cache->dir, fname,
+                                  &labels, &body, &bodylen);
+    if (! map) {
+      /* The ERANGE error might come from tor_mmap_file() -- it means the file
+       * was empty. EINVAL might come from ..map_labeled() -- it means the
+       * file was misformatted. In both cases, we should just delete it.
+       */
+      if (errno == ERANGE || errno == EINVAL) {
+        log_warn(LD_FS, "Found %s file %s in consensus cache; removing it.",
+                 errno == ERANGE ? "empty" : "misformatted",
+                 escaped(fname));
+        storage_dir_remove_file(cache->dir, fname);
+      } else {
+        /* Can't load this; continue */
+        log_warn(LD_FS, "Unable to map file %s from consensus cache: %s",
+                 escaped(fname), strerror(errno));
+      }
+      continue;
+    }
+    consensus_cache_entry_t *ent =
+      tor_malloc_zero(sizeof(consensus_cache_entry_t));
+    ent->magic = CCE_MAGIC;
+    ent->fname = tor_strdup(fname);
+    ent->labels = labels;
+    ent->refcnt = 1;
+    ent->in_cache = cache;
+    ent->unused_since = TIME_MAX;
+    smartlist_add(cache->entries, ent);
+    tor_munmap_file(map); /* don't actually need to keep this around */
+  } SMARTLIST_FOREACH_END(fname);
+}
+
+/**
+ * Make sure that <b>ent</b> is mapped into RAM.
+ */
+static void
+consensus_cache_entry_map(consensus_cache_t *cache,
+                          consensus_cache_entry_t *ent)
+{
+  if (ent->map)
+    return;
+
+  ent->map = storage_dir_map_labeled(cache->dir, ent->fname,
+                                     NULL, &ent->body, &ent->bodylen);
+  ent->unused_since = TIME_MAX;
+}
+
+/**
+ * Unmap <b>ent</b> from RAM.
+ *
+ * Do not call this if something other than the cache is holding a reference
+ * to <b>ent</b>
+ */
+static void
+consensus_cache_entry_unmap(consensus_cache_entry_t *ent)
+{
+  ent->unused_since = TIME_MAX;
+  if (!ent->map)
+    return;
+
+  tor_munmap_file(ent->map);
+  ent->map = NULL;
+  ent->body = NULL;
+  ent->bodylen = 0;
+  ent->unused_since = TIME_MAX;
+}
+
+HANDLE_IMPL(consensus_cache_entry, consensus_cache_entry_t, )
+
+#ifdef TOR_UNIT_TESTS
+/**
+ * Testing only: Return true iff <b>ent</b> is mapped into memory.
+ *
+ * (In normal operation, this information is not exposed.)
+ */
+int
+consensus_cache_entry_is_mapped(consensus_cache_entry_t *ent)
+{
+  if (ent->map) {
+    tor_assert(ent->body);
+    return 1;
+  } else {
+    tor_assert(!ent->body);
+    return 0;
+  }
+}
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/feature/dircache/conscache.h b/src/feature/dircache/conscache.h
new file mode 100644
index 0000000000..d848e57617
--- /dev/null
+++ b/src/feature/dircache/conscache.h
@@ -0,0 +1,66 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_CONSCACHE_H
+#define TOR_CONSCACHE_H
+
+#include "lib/container/handles.h"
+
+typedef struct consensus_cache_entry_t consensus_cache_entry_t;
+typedef struct consensus_cache_t consensus_cache_t;
+
+HANDLE_DECL(consensus_cache_entry, consensus_cache_entry_t, )
+#define consensus_cache_entry_handle_free(h)    \
+  FREE_AND_NULL(consensus_cache_entry_handle_t, \
+                consensus_cache_entry_handle_free_, (h))
+
+consensus_cache_t *consensus_cache_open(const char *subdir, int max_entries);
+void consensus_cache_free_(consensus_cache_t *cache);
+#define consensus_cache_free(cache) \
+  FREE_AND_NULL(consensus_cache_t, consensus_cache_free_, (cache))
+struct sandbox_cfg_elem;
+int consensus_cache_may_overallocate(consensus_cache_t *cache);
+int consensus_cache_register_with_sandbox(consensus_cache_t *cache,
+                                          struct sandbox_cfg_elem **cfg);
+void consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff);
+void consensus_cache_delete_pending(consensus_cache_t *cache,
+                                    int force);
+int consensus_cache_get_n_filenames_available(consensus_cache_t *cache);
+consensus_cache_entry_t *consensus_cache_add(consensus_cache_t *cache,
+                                           const struct config_line_t *labels,
+                                           const uint8_t *data,
+                                           size_t datalen);
+
+consensus_cache_entry_t *consensus_cache_find_first(
+                                             consensus_cache_t *cache,
+                                             const char *key,
+                                             const char *value);
+
+void consensus_cache_find_all(smartlist_t *out,
+                              consensus_cache_t *cache,
+                              const char *key,
+                              const char *value);
+void consensus_cache_filter_list(smartlist_t *lst,
+                                 const char *key,
+                                 const char *value);
+
+const char *consensus_cache_entry_get_value(const consensus_cache_entry_t *ent,
+                                            const char *key);
+const struct config_line_t *consensus_cache_entry_get_labels(
+                                          const consensus_cache_entry_t *ent);
+
+void consensus_cache_entry_incref(consensus_cache_entry_t *ent);
+void consensus_cache_entry_decref(consensus_cache_entry_t *ent);
+
+void consensus_cache_entry_mark_for_removal(consensus_cache_entry_t *ent);
+void consensus_cache_entry_mark_for_aggressive_release(
+                                            consensus_cache_entry_t *ent);
+int consensus_cache_entry_get_body(const consensus_cache_entry_t *ent,
+                                   const uint8_t **body_out,
+                                   size_t *sz_out);
+
+#ifdef TOR_UNIT_TESTS
+int consensus_cache_entry_is_mapped(consensus_cache_entry_t *ent);
+#endif
+
+#endif /* !defined(TOR_CONSCACHE_H) */
diff --git a/src/feature/dircache/consdiffmgr.c b/src/feature/dircache/consdiffmgr.c
new file mode 100644
index 0000000000..025361fa60
--- /dev/null
+++ b/src/feature/dircache/consdiffmgr.c
@@ -0,0 +1,1945 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file consdiffmsr.c
+ *
+ * \brief consensus diff manager functions
+ *
+ * This module is run by directory authorities and caches in order
+ * to remember a number of past consensus documents, and to generate
+ * and serve the diffs from those documents to the latest consensus.
+ */
+
+#define CONSDIFFMGR_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "feature/dircache/conscache.h"
+#include "feature/dircommon/consdiff.h"
+#include "feature/dircache/consdiffmgr.h"
+#include "core/mainloop/cpuworker.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/dirparse/ns_parse.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/evloop/workqueue.h"
+#include "lib/compress/compress.h"
+#include "lib/encoding/confline.h"
+
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/networkstatus_voter_info_st.h"
+
+/**
+ * Labels to apply to items in the conscache object.
+ *
+ * @{
+ */
+/* One of DOCTYPE_CONSENSUS or DOCTYPE_CONSENSUS_DIFF */
+#define LABEL_DOCTYPE "document-type"
+/* The valid-after time for a consensus (or for the target consensus of a
+ * diff), encoded as ISO UTC. */
+#define LABEL_VALID_AFTER "consensus-valid-after"
+/* The fresh-until time for a consensus (or for the target consensus of a
+ * diff), encoded as ISO UTC. */
+#define LABEL_FRESH_UNTIL "consensus-fresh-until"
+/* The valid-until time for a consensus (or for the target consensus of a
+ * diff), encoded as ISO UTC. */
+#define LABEL_VALID_UNTIL "consensus-valid-until"
+/* Comma-separated list of hex-encoded identity digests for the voting
+ * authorities. */
+#define LABEL_SIGNATORIES "consensus-signatories"
+/* A hex encoded SHA3 digest of the object, as compressed (if any) */
+#define LABEL_SHA3_DIGEST "sha3-digest"
+/* A hex encoded SHA3 digest of the object before compression. */
+#define LABEL_SHA3_DIGEST_UNCOMPRESSED "sha3-digest-uncompressed"
+/* A hex encoded SHA3 digest-as-signed of a consensus */
+#define LABEL_SHA3_DIGEST_AS_SIGNED "sha3-digest-as-signed"
+/* The flavor of the consensus or consensuses diff */
+#define LABEL_FLAVOR "consensus-flavor"
+/* Diff only: the SHA3 digest-as-signed of the source consensus. */
+#define LABEL_FROM_SHA3_DIGEST "from-sha3-digest"
+/* Diff only: the SHA3 digest-in-full of the target consensus. */
+#define LABEL_TARGET_SHA3_DIGEST "target-sha3-digest"
+/* Diff only: the valid-after date of the source consensus. */
+#define LABEL_FROM_VALID_AFTER "from-valid-after"
+/* What kind of compression was used? */
+#define LABEL_COMPRESSION_TYPE "compression"
+/** @} */
+
+#define DOCTYPE_CONSENSUS "consensus"
+#define DOCTYPE_CONSENSUS_DIFF "consensus-diff"
+
+/**
+ * Underlying directory that stores consensuses and consensus diffs.  Don't
+ * use this directly: use cdm_cache_get() instead.
+ */
+static consensus_cache_t *cons_diff_cache = NULL;
+/**
+ * If true, we have learned at least one new consensus since the
+ * consensus cache was last up-to-date.
+ */
+static int cdm_cache_dirty = 0;
+/**
+ * If true, we have scanned the cache to update our hashtable of diffs.
+ */
+static int cdm_cache_loaded = 0;
+
+/**
+ * Possible status values for cdm_diff_t.cdm_diff_status
+ **/
+typedef enum cdm_diff_status_t {
+  CDM_DIFF_PRESENT=1,
+  CDM_DIFF_IN_PROGRESS=2,
+  CDM_DIFF_ERROR=3,
+} cdm_diff_status_t;
+
+/** Which methods do we use for precompressing diffs? */
+static const compress_method_t compress_diffs_with[] = {
+  NO_METHOD,
+  GZIP_METHOD,
+#ifdef HAVE_LZMA
+  LZMA_METHOD,
+#endif
+#ifdef HAVE_ZSTD
+  ZSTD_METHOD,
+#endif
+};
+
+/**
+ * Event for rescanning the cache.
+ */
+static mainloop_event_t *consdiffmgr_rescan_ev = NULL;
+
+static void consdiffmgr_rescan_cb(mainloop_event_t *ev, void *arg);
+static void mark_cdm_cache_dirty(void);
+
+/** How many different methods will we try to use for diff compression? */
+STATIC unsigned
+n_diff_compression_methods(void)
+{
+  return ARRAY_LENGTH(compress_diffs_with);
+}
+
+/** Which methods do we use for precompressing consensuses? */
+static const compress_method_t compress_consensus_with[] = {
+  ZLIB_METHOD,
+#ifdef HAVE_LZMA
+  LZMA_METHOD,
+#endif
+#ifdef HAVE_ZSTD
+  ZSTD_METHOD,
+#endif
+};
+
+/** How many different methods will we try to use for diff compression? */
+STATIC unsigned
+n_consensus_compression_methods(void)
+{
+  return ARRAY_LENGTH(compress_consensus_with);
+}
+
+/** For which compression method do we retain old consensuses?  There's no
+ * need to keep all of them, since we won't be serving them.  We'll
+ * go with ZLIB_METHOD because it's pretty fast and everyone has it.
+ */
+#define RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD
+
+/** Handles pointing to the latest consensus entries as compressed and
+ * stored. */
+static consensus_cache_entry_handle_t *
+                  latest_consensus[N_CONSENSUS_FLAVORS]
+                                  [ARRAY_LENGTH(compress_consensus_with)];
+
+/** Hashtable node used to remember the current status of the diff
+ * from a given sha3 digest to the current consensus.  */
+typedef struct cdm_diff_t {
+  HT_ENTRY(cdm_diff_t) node;
+
+  /** Consensus flavor for this diff (part of ht key) */
+  consensus_flavor_t flavor;
+  /** SHA3-256 digest of the consensus that this diff is _from_. (part of the
+   * ht key) */
+  uint8_t from_sha3[DIGEST256_LEN];
+  /** Method by which the diff is compressed. (part of the ht key */
+  compress_method_t compress_method;
+
+  /** One of the CDM_DIFF_* values, depending on whether this diff
+   * is available, in progress, or impossible to compute. */
+  cdm_diff_status_t cdm_diff_status;
+  /** SHA3-256 digest of the consensus that this diff is _to. */
+  uint8_t target_sha3[DIGEST256_LEN];
+
+  /** Handle to the cache entry for this diff, if any.  We use a handle here
+   * to avoid thinking too hard about cache entry lifetime issues. */
+  consensus_cache_entry_handle_t *entry;
+} cdm_diff_t;
+
+/** Hashtable mapping flavor and source consensus digest to status. */
+static HT_HEAD(cdm_diff_ht, cdm_diff_t) cdm_diff_ht = HT_INITIALIZER();
+
+/**
+ * Configuration for this module
+ */
+static consdiff_cfg_t consdiff_cfg = {
+  // XXXX I'd like to make this number bigger, but it interferes with the
+  // XXXX seccomp2 syscall filter, which tops out at BPF_MAXINS (4096)
+  // XXXX rules.
+  /* .cache_max_num = */ 128
+};
+
+static int consdiffmgr_ensure_space_for_files(int n);
+static int consensus_queue_compression_work(const char *consensus,
+                                            const networkstatus_t *as_parsed);
+static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
+                                          consensus_cache_entry_t *diff_to);
+static void consdiffmgr_set_cache_flags(void);
+
+/* =====
+ * Hashtable setup
+ * ===== */
+
+/** Helper: hash the key of a cdm_diff_t. */
+static unsigned
+cdm_diff_hash(const cdm_diff_t *diff)
+{
+  uint8_t tmp[DIGEST256_LEN + 2];
+  memcpy(tmp, diff->from_sha3, DIGEST256_LEN);
+  tmp[DIGEST256_LEN] = (uint8_t) diff->flavor;
+  tmp[DIGEST256_LEN+1] = (uint8_t) diff->compress_method;
+  return (unsigned) siphash24g(tmp, sizeof(tmp));
+}
+/** Helper: compare two cdm_diff_t objects for key equality */
+static int
+cdm_diff_eq(const cdm_diff_t *diff1, const cdm_diff_t *diff2)
+{
+  return fast_memeq(diff1->from_sha3, diff2->from_sha3, DIGEST256_LEN) &&
+    diff1->flavor == diff2->flavor &&
+    diff1->compress_method == diff2->compress_method;
+}
+
+HT_PROTOTYPE(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq)
+HT_GENERATE2(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq,
+             0.6, tor_reallocarray, tor_free_)
+
+#define cdm_diff_free(diff) \
+  FREE_AND_NULL(cdm_diff_t, cdm_diff_free_, (diff))
+
+/** Release all storage held in <b>diff</b>. */
+static void
+cdm_diff_free_(cdm_diff_t *diff)
+{
+  if (!diff)
+    return;
+  consensus_cache_entry_handle_free(diff->entry);
+  tor_free(diff);
+}
+
+/** Create and return a new cdm_diff_t with the given values.  Does not
+ * add it to the hashtable. */
+static cdm_diff_t *
+cdm_diff_new(consensus_flavor_t flav,
+             const uint8_t *from_sha3,
+             const uint8_t *target_sha3,
+             compress_method_t method)
+{
+  cdm_diff_t *ent;
+  ent = tor_malloc_zero(sizeof(cdm_diff_t));
+  ent->flavor = flav;
+  memcpy(ent->from_sha3, from_sha3, DIGEST256_LEN);
+  memcpy(ent->target_sha3, target_sha3, DIGEST256_LEN);
+  ent->compress_method = method;
+  return ent;
+}
+
+/**
+ * Examine the diff hashtable to see whether we know anything about computing
+ * a diff of type <b>flav</b> between consensuses with the two provided
+ * SHA3-256 digests.  If a computation is in progress, or if the computation
+ * has already been tried and failed, return 1.  Otherwise, note the
+ * computation as "in progress" so that we don't reattempt it later, and
+ * return 0.
+ */
+static int
+cdm_diff_ht_check_and_note_pending(consensus_flavor_t flav,
+                                   const uint8_t *from_sha3,
+                                   const uint8_t *target_sha3)
+{
+  struct cdm_diff_t search, *ent;
+  unsigned u;
+  int result = 0;
+  for (u = 0; u < n_diff_compression_methods(); ++u) {
+    compress_method_t method = compress_diffs_with[u];
+    memset(&search, 0, sizeof(cdm_diff_t));
+    search.flavor = flav;
+    search.compress_method = method;
+    memcpy(search.from_sha3, from_sha3, DIGEST256_LEN);
+    ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
+    if (ent) {
+      tor_assert_nonfatal(ent->cdm_diff_status != CDM_DIFF_PRESENT);
+      result = 1;
+      continue;
+    }
+    ent = cdm_diff_new(flav, from_sha3, target_sha3, method);
+    ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
+    HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
+  }
+  return result;
+}
+
+/**
+ * Update the status of the diff of type <b>flav</b> between consensuses with
+ * the two provided SHA3-256 digests, so that its status becomes
+ * <b>status</b>, and its value becomes the <b>handle</b>.  If <b>handle</b>
+ * is NULL, then the old handle (if any) is freed, and replaced with NULL.
+ */
+static void
+cdm_diff_ht_set_status(consensus_flavor_t flav,
+                       const uint8_t *from_sha3,
+                       const uint8_t *to_sha3,
+                       compress_method_t method,
+                       int status,
+                       consensus_cache_entry_handle_t *handle)
+{
+  if (handle == NULL) {
+    tor_assert_nonfatal(status != CDM_DIFF_PRESENT);
+  }
+
+  struct cdm_diff_t search, *ent;
+  memset(&search, 0, sizeof(cdm_diff_t));
+  search.flavor = flav;
+  search.compress_method = method,
+  memcpy(search.from_sha3, from_sha3, DIGEST256_LEN);
+  ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
+  if (!ent) {
+    ent = cdm_diff_new(flav, from_sha3, to_sha3, method);
+    ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
+    HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
+  } else if (fast_memneq(ent->target_sha3, to_sha3, DIGEST256_LEN)) {
+    // This can happen under certain really pathological conditions
+    // if we decide we don't care about a diff before it is actually
+    // done computing.
+    return;
+  }
+
+  tor_assert_nonfatal(ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS);
+
+  ent->cdm_diff_status = status;
+  consensus_cache_entry_handle_free(ent->entry);
+  ent->entry = handle;
+}
+
+/**
+ * Helper: Remove from the hash table every present (actually computed) diff
+ * of type <b>flav</b> whose target digest does not match
+ * <b>unless_target_sha3_matches</b>.
+ *
+ * This function is used for the hash table to throw away references to diffs
+ * that do not lead to the most given consensus of a given flavor.
+ */
+static void
+cdm_diff_ht_purge(consensus_flavor_t flav,
+                  const uint8_t *unless_target_sha3_matches)
+{
+  cdm_diff_t **diff, **next;
+  for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
+    cdm_diff_t *this = *diff;
+
+    if ((*diff)->cdm_diff_status == CDM_DIFF_PRESENT &&
+        flav == (*diff)->flavor) {
+
+      if (BUG((*diff)->entry == NULL) ||
+          consensus_cache_entry_handle_get((*diff)->entry) == NULL) {
+        /* the underlying entry has gone away; drop this. */
+        next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
+        cdm_diff_free(this);
+        continue;
+      }
+
+      if (unless_target_sha3_matches &&
+          fast_memneq(unless_target_sha3_matches, (*diff)->target_sha3,
+                      DIGEST256_LEN)) {
+        /* target hash doesn't match; drop this. */
+        next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
+        cdm_diff_free(this);
+        continue;
+      }
+    }
+    next = HT_NEXT(cdm_diff_ht, &cdm_diff_ht, diff);
+  }
+}
+
+/**
+ * Helper: initialize <b>cons_diff_cache</b>.
+ */
+static void
+cdm_cache_init(void)
+{
+  unsigned n_entries = consdiff_cfg.cache_max_num * 2;
+
+  tor_assert(cons_diff_cache == NULL);
+  cons_diff_cache = consensus_cache_open("diff-cache", n_entries);
+  if (cons_diff_cache == NULL) {
+    // LCOV_EXCL_START
+    log_err(LD_FS, "Error: Couldn't open storage for consensus diffs.");
+    tor_assert_unreached();
+    // LCOV_EXCL_STOP
+  } else {
+    consdiffmgr_set_cache_flags();
+  }
+  consdiffmgr_rescan_ev =
+    mainloop_event_postloop_new(consdiffmgr_rescan_cb, NULL);
+  mark_cdm_cache_dirty();
+  cdm_cache_loaded = 0;
+}
+
+/**
+ * Helper: return the consensus_cache_t * that backs this manager,
+ * initializing it if needed.
+ */
+STATIC consensus_cache_t *
+cdm_cache_get(void)
+{
+  if (PREDICT_UNLIKELY(cons_diff_cache == NULL)) {
+    cdm_cache_init();
+  }
+  return cons_diff_cache;
+}
+
+/**
+ * Helper: given a list of labels, prepend the hex-encoded SHA3 digest
+ * of the <b>bodylen</b>-byte object at <b>body</b> to those labels,
+ * with <b>label</b> as its label.
+ */
+static void
+cdm_labels_prepend_sha3(config_line_t **labels,
+                        const char *label,
+                        const uint8_t *body,
+                        size_t bodylen)
+{
+  uint8_t sha3_digest[DIGEST256_LEN];
+  char hexdigest[HEX_DIGEST256_LEN+1];
+  crypto_digest256((char *)sha3_digest,
+                   (const char *)body, bodylen, DIGEST_SHA3_256);
+  base16_encode(hexdigest, sizeof(hexdigest),
+                (const char *)sha3_digest, sizeof(sha3_digest));
+
+  config_line_prepend(labels, label, hexdigest);
+}
+
+/** Helper: if there is a sha3-256 hex-encoded digest in <b>ent</b> with the
+ * given label, set <b>digest_out</b> to that value (decoded), and return 0.
+ *
+ * Return -1 if there is no such label, and -2 if it is badly formatted. */
+STATIC int
+cdm_entry_get_sha3_value(uint8_t *digest_out,
+                         consensus_cache_entry_t *ent,
+                         const char *label)
+{
+  if (ent == NULL)
+    return -1;
+
+  const char *hex = consensus_cache_entry_get_value(ent, label);
+  if (hex == NULL)
+    return -1;
+
+  int n = base16_decode((char*)digest_out, DIGEST256_LEN, hex, strlen(hex));
+  if (n != DIGEST256_LEN)
+    return -2;
+  else
+    return 0;
+}
+
+/**
+ * Helper: look for a consensus with the given <b>flavor</b> and
+ * <b>valid_after</b> time in the cache. Return that consensus if it's
+ * present, or NULL if it's missing.
+ */
+STATIC consensus_cache_entry_t *
+cdm_cache_lookup_consensus(consensus_flavor_t flavor, time_t valid_after)
+{
+  char formatted_time[ISO_TIME_LEN+1];
+  format_iso_time_nospace(formatted_time, valid_after);
+  const char *flavname = networkstatus_get_flavor_name(flavor);
+
+  /* We'll filter by valid-after time first, since that should
+   * match the fewest documents. */
+  /* We could add an extra hashtable here, but since we only do this scan
+   * when adding a new consensus, it probably doesn't matter much. */
+  smartlist_t *matches = smartlist_new();
+  consensus_cache_find_all(matches, cdm_cache_get(),
+                           LABEL_VALID_AFTER, formatted_time);
+  consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname);
+  consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+
+  consensus_cache_entry_t *result = NULL;
+  if (smartlist_len(matches)) {
+    result = smartlist_get(matches, 0);
+  }
+  smartlist_free(matches);
+
+  return result;
+}
+
+/** Return the maximum age (in seconds) of consensuses that we should consider
+ * storing. The available space in the directory may impose additional limits
+ * on how much we store. */
+static int32_t
+get_max_age_to_cache(void)
+{
+  const int32_t DEFAULT_MAX_AGE_TO_CACHE = 8192;
+  const int32_t MIN_MAX_AGE_TO_CACHE = 0;
+  const int32_t MAX_MAX_AGE_TO_CACHE = 8192;
+  const char MAX_AGE_TO_CACHE_NAME[] = "max-consensus-age-to-cache-for-diff";
+
+  const or_options_t *options = get_options();
+
+  if (options->MaxConsensusAgeForDiffs) {
+    const int v = options->MaxConsensusAgeForDiffs;
+    if (v >= MAX_MAX_AGE_TO_CACHE * 3600)
+      return MAX_MAX_AGE_TO_CACHE;
+    else
+      return v;
+  }
+
+  /* The parameter is in hours, so we multiply */
+  return 3600 * networkstatus_get_param(NULL,
+                                        MAX_AGE_TO_CACHE_NAME,
+                                        DEFAULT_MAX_AGE_TO_CACHE,
+                                        MIN_MAX_AGE_TO_CACHE,
+                                        MAX_MAX_AGE_TO_CACHE);
+}
+
+/**
+ * Given a string containing a networkstatus consensus, and the results of
+ * having parsed that consensus, add that consensus to the cache if it is not
+ * already present and not too old.  Create new consensus diffs from or to
+ * that consensus as appropriate.
+ *
+ * Return 0 on success and -1 on failure.
+ */
+int
+consdiffmgr_add_consensus(const char *consensus,
+                          const networkstatus_t *as_parsed)
+{
+  if (BUG(consensus == NULL) || BUG(as_parsed == NULL))
+    return -1; // LCOV_EXCL_LINE
+  if (BUG(as_parsed->type != NS_TYPE_CONSENSUS))
+    return -1; // LCOV_EXCL_LINE
+
+  const consensus_flavor_t flavor = as_parsed->flavor;
+  const time_t valid_after = as_parsed->valid_after;
+
+  if (valid_after < approx_time() - get_max_age_to_cache()) {
+    log_info(LD_DIRSERV, "We don't care about this consensus document; it's "
+             "too old.");
+    return -1;
+  }
+
+  /* Do we already have this one? */
+  consensus_cache_entry_t *entry =
+    cdm_cache_lookup_consensus(flavor, valid_after);
+  if (entry) {
+    log_info(LD_DIRSERV, "We already have a copy of that consensus");
+    return -1;
+  }
+
+  /* We don't have it. Add it to the cache. */
+  return consensus_queue_compression_work(consensus, as_parsed);
+}
+
+/**
+ * Helper: used to sort two smartlists of consensus_cache_entry_t by their
+ * LABEL_VALID_AFTER labels.
+ */
+static int
+compare_by_valid_after_(const void **a, const void **b)
+{
+  const consensus_cache_entry_t *e1 = *a;
+  const consensus_cache_entry_t *e2 = *b;
+  /* We're in luck here: sorting UTC iso-encoded values lexically will work
+   * fine (until 9999). */
+  return strcmp_opt(consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER),
+                    consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER));
+}
+
+/**
+ * Helper: Sort <b>lst</b> by LABEL_VALID_AFTER and return the most recent
+ * entry.
+ */
+static consensus_cache_entry_t *
+sort_and_find_most_recent(smartlist_t *lst)
+{
+  smartlist_sort(lst, compare_by_valid_after_);
+  if (smartlist_len(lst)) {
+    return smartlist_get(lst, smartlist_len(lst) - 1);
+  } else {
+    return NULL;
+  }
+}
+
+/** Return i such that compress_consensus_with[i] == method. Return
+ * -1 if no such i exists. */
+static int
+consensus_compression_method_pos(compress_method_t method)
+{
+  unsigned i;
+  for (i = 0; i < n_consensus_compression_methods(); ++i) {
+    if (compress_consensus_with[i] == method) {
+      return i;
+    }
+  }
+  return -1;
+}
+
+/**
+ * If we know a consensus with the flavor <b>flavor</b> compressed with
+ * <b>method</b>, set *<b>entry_out</b> to that value.  Return values are as
+ * for consdiffmgr_find_diff_from().
+ */
+consdiff_status_t
+consdiffmgr_find_consensus(struct consensus_cache_entry_t **entry_out,
+                           consensus_flavor_t flavor,
+                           compress_method_t method)
+{
+  tor_assert(entry_out);
+  tor_assert((int)flavor < N_CONSENSUS_FLAVORS);
+
+  int pos = consensus_compression_method_pos(method);
+  if (pos < 0) {
+     // We don't compress consensuses with this method.
+    return CONSDIFF_NOT_FOUND;
+  }
+  consensus_cache_entry_handle_t *handle = latest_consensus[flavor][pos];
+  if (!handle)
+    return CONSDIFF_NOT_FOUND;
+  *entry_out = consensus_cache_entry_handle_get(handle);
+  if (*entry_out)
+    return CONSDIFF_AVAILABLE;
+  else
+    return CONSDIFF_NOT_FOUND;
+}
+
+/**
+ * Look up consensus_cache_entry_t for the consensus of type <b>flavor</b>,
+ * from the source consensus with the specified digest (which must be SHA3).
+ *
+ * If the diff is present, store it into *<b>entry_out</b> and return
+ * CONSDIFF_AVAILABLE. Otherwise return CONSDIFF_NOT_FOUND or
+ * CONSDIFF_IN_PROGRESS.
+ */
+consdiff_status_t
+consdiffmgr_find_diff_from(consensus_cache_entry_t **entry_out,
+                           consensus_flavor_t flavor,
+                           int digest_type,
+                           const uint8_t *digest,
+                           size_t digestlen,
+                           compress_method_t method)
+{
+  if (BUG(digest_type != DIGEST_SHA3_256) ||
+      BUG(digestlen != DIGEST256_LEN)) {
+    return CONSDIFF_NOT_FOUND; // LCOV_EXCL_LINE
+  }
+
+  // Try to look up the entry in the hashtable.
+  cdm_diff_t search, *ent;
+  memset(&search, 0, sizeof(search));
+  search.flavor = flavor;
+  search.compress_method = method;
+  memcpy(search.from_sha3, digest, DIGEST256_LEN);
+  ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
+
+  if (ent == NULL ||
+      ent->cdm_diff_status == CDM_DIFF_ERROR) {
+    return CONSDIFF_NOT_FOUND;
+  } else if (ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS) {
+    return CONSDIFF_IN_PROGRESS;
+  } else if (BUG(ent->cdm_diff_status != CDM_DIFF_PRESENT)) {
+    return CONSDIFF_IN_PROGRESS;
+  }
+
+  if (BUG(ent->entry == NULL)) {
+    return CONSDIFF_NOT_FOUND;
+  }
+  *entry_out = consensus_cache_entry_handle_get(ent->entry);
+  return (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
+
+#if 0
+  // XXXX Remove this.  I'm keeping it around for now in case we need to
+  // XXXX debug issues in the hashtable.
+  char hex[HEX_DIGEST256_LEN+1];
+  base16_encode(hex, sizeof(hex), (const char *)digest, digestlen);
+  const char *flavname = networkstatus_get_flavor_name(flavor);
+
+  smartlist_t *matches = smartlist_new();
+  consensus_cache_find_all(matches, cdm_cache_get(),
+                           LABEL_FROM_SHA3_DIGEST, hex);
+  consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname);
+  consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
+
+  *entry_out = sort_and_find_most_recent(matches);
+  consdiff_status_t result =
+    (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
+  smartlist_free(matches);
+
+  return result;
+#endif /* 0 */
+}
+
+/**
+ * Perform periodic cleanup tasks on the consensus diff cache.  Return
+ * the number of objects marked for deletion.
+ */
+int
+consdiffmgr_cleanup(void)
+{
+  smartlist_t *objects = smartlist_new();
+  smartlist_t *consensuses = smartlist_new();
+  smartlist_t *diffs = smartlist_new();
+  int n_to_delete = 0;
+
+  log_debug(LD_DIRSERV, "Looking for consdiffmgr entries to remove");
+
+  // 1. Delete any consensus or diff or anything whose valid_after is too old.
+  const time_t valid_after_cutoff = approx_time() - get_max_age_to_cache();
+
+  consensus_cache_find_all(objects, cdm_cache_get(),
+                           NULL, NULL);
+  SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
+    const char *lv_valid_after =
+      consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
+    if (! lv_valid_after) {
+      log_debug(LD_DIRSERV, "Ignoring entry because it had no %s label",
+                LABEL_VALID_AFTER);
+      continue;
+    }
+    time_t valid_after = 0;
+    if (parse_iso_time_nospace(lv_valid_after, &valid_after) < 0) {
+      log_debug(LD_DIRSERV, "Ignoring entry because its %s value (%s) was "
+                "unparseable", LABEL_VALID_AFTER, escaped(lv_valid_after));
+      continue;
+    }
+    if (valid_after < valid_after_cutoff) {
+      log_debug(LD_DIRSERV, "Deleting entry because its %s value (%s) was "
+                "too old", LABEL_VALID_AFTER, lv_valid_after);
+      consensus_cache_entry_mark_for_removal(ent);
+      ++n_to_delete;
+    }
+  } SMARTLIST_FOREACH_END(ent);
+
+  // 2. Delete all diffs that lead to a consensus whose valid-after is not the
+  // latest.
+  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
+    const char *flavname = networkstatus_get_flavor_name(flav);
+    /* Determine the most recent consensus of this flavor */
+    consensus_cache_find_all(consensuses, cdm_cache_get(),
+                             LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+    consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname);
+    consensus_cache_entry_t *most_recent =
+      sort_and_find_most_recent(consensuses);
+    if (most_recent == NULL)
+      continue;
+    const char *most_recent_sha3 =
+      consensus_cache_entry_get_value(most_recent,
+                                      LABEL_SHA3_DIGEST_UNCOMPRESSED);
+    if (BUG(most_recent_sha3 == NULL))
+      continue; // LCOV_EXCL_LINE
+
+    /* consider all such-flavored diffs, and look to see if they match. */
+    consensus_cache_find_all(diffs, cdm_cache_get(),
+                             LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
+    consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname);
+    SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
+      const char *this_diff_target_sha3 =
+        consensus_cache_entry_get_value(diff, LABEL_TARGET_SHA3_DIGEST);
+      if (!this_diff_target_sha3)
+        continue;
+      if (strcmp(this_diff_target_sha3, most_recent_sha3)) {
+        consensus_cache_entry_mark_for_removal(diff);
+        ++n_to_delete;
+      }
+    } SMARTLIST_FOREACH_END(diff);
+    smartlist_clear(consensuses);
+    smartlist_clear(diffs);
+  }
+
+  // 3. Delete all consensuses except the most recent that are compressed with
+  // an un-preferred method.
+  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
+    const char *flavname = networkstatus_get_flavor_name(flav);
+    /* Determine the most recent consensus of this flavor */
+    consensus_cache_find_all(consensuses, cdm_cache_get(),
+                             LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+    consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname);
+    consensus_cache_entry_t *most_recent =
+      sort_and_find_most_recent(consensuses);
+    if (most_recent == NULL)
+      continue;
+    const char *most_recent_sha3_uncompressed =
+      consensus_cache_entry_get_value(most_recent,
+                                      LABEL_SHA3_DIGEST_UNCOMPRESSED);
+    const char *retain_methodname = compression_method_get_name(
+                               RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD);
+
+    if (BUG(most_recent_sha3_uncompressed == NULL))
+      continue;
+    SMARTLIST_FOREACH_BEGIN(consensuses, consensus_cache_entry_t *, ent) {
+      const char *lv_sha3_uncompressed =
+        consensus_cache_entry_get_value(ent, LABEL_SHA3_DIGEST_UNCOMPRESSED);
+      if (BUG(! lv_sha3_uncompressed))
+        continue;
+      if (!strcmp(lv_sha3_uncompressed, most_recent_sha3_uncompressed))
+        continue; // This _is_ the most recent.
+      const char *lv_methodname =
+        consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
+      if (! lv_methodname || strcmp(lv_methodname, retain_methodname)) {
+        consensus_cache_entry_mark_for_removal(ent);
+        ++n_to_delete;
+      }
+    } SMARTLIST_FOREACH_END(ent);
+  }
+
+  smartlist_free(objects);
+  smartlist_free(consensuses);
+  smartlist_free(diffs);
+
+  // Actually remove files, if they're not used.
+  consensus_cache_delete_pending(cdm_cache_get(), 0);
+  return n_to_delete;
+}
+
+/**
+ * Initialize the consensus diff manager and its cache, and configure
+ * its parameters based on the latest torrc and networkstatus parameters.
+ */
+void
+consdiffmgr_configure(const consdiff_cfg_t *cfg)
+{
+  if (cfg)
+    memcpy(&consdiff_cfg, cfg, sizeof(consdiff_cfg));
+
+  (void) cdm_cache_get();
+}
+
+/**
+ * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
+ * operations that the consensus diff manager will need.
+ */
+int
+consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg)
+{
+  return consensus_cache_register_with_sandbox(cdm_cache_get(), cfg);
+}
+
+/**
+ * Scan the consensus diff manager's cache for any grossly malformed entries,
+ * and mark them as deletable.  Return 0 if no problems were found; 1
+ * if problems were found and fixed.
+ */
+int
+consdiffmgr_validate(void)
+{
+  /* Right now, we only check for entries that have bad sha3 values */
+  int problems = 0;
+
+  smartlist_t *objects = smartlist_new();
+  consensus_cache_find_all(objects, cdm_cache_get(),
+                           NULL, NULL);
+  SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, obj) {
+    uint8_t sha3_expected[DIGEST256_LEN];
+    uint8_t sha3_received[DIGEST256_LEN];
+    int r = cdm_entry_get_sha3_value(sha3_expected, obj, LABEL_SHA3_DIGEST);
+    if (r == -1) {
+      /* digest isn't there; that's allowed */
+      continue;
+    } else if (r == -2) {
+      /* digest is malformed; that's not allowed */
+      problems = 1;
+      consensus_cache_entry_mark_for_removal(obj);
+      continue;
+    }
+    const uint8_t *body;
+    size_t bodylen;
+    consensus_cache_entry_incref(obj);
+    r = consensus_cache_entry_get_body(obj, &body, &bodylen);
+    if (r == 0) {
+      crypto_digest256((char *)sha3_received, (const char *)body, bodylen,
+                       DIGEST_SHA3_256);
+    }
+    consensus_cache_entry_decref(obj);
+    if (r < 0)
+      continue;
+
+    // Deconfuse coverity about the possibility of sha3_received being
+    // uninitialized
+    tor_assert(r <= 0);
+
+    if (fast_memneq(sha3_received, sha3_expected, DIGEST256_LEN)) {
+      problems = 1;
+      consensus_cache_entry_mark_for_removal(obj);
+      continue;
+    }
+
+  } SMARTLIST_FOREACH_END(obj);
+  smartlist_free(objects);
+  return problems;
+}
+
+/**
+ * Helper: build new diffs of <b>flavor</b> as needed
+ */
+static void
+consdiffmgr_rescan_flavor_(consensus_flavor_t flavor)
+{
+  smartlist_t *matches = NULL;
+  smartlist_t *diffs = NULL;
+  smartlist_t *compute_diffs_from = NULL;
+  strmap_t *have_diff_from = NULL;
+
+  // look for the most recent consensus, and for all previous in-range
+  // consensuses.  Do they all have diffs to it?
+  const char *flavname = networkstatus_get_flavor_name(flavor);
+
+  // 1. find the most recent consensus, and the ones that we might want
+  //    to diff to it.
+  const char *methodname = compression_method_get_name(
+                             RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD);
+
+  matches = smartlist_new();
+  consensus_cache_find_all(matches, cdm_cache_get(),
+                           LABEL_FLAVOR, flavname);
+  consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+  consensus_cache_filter_list(matches, LABEL_COMPRESSION_TYPE, methodname);
+  consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches);
+  if (!most_recent) {
+    log_info(LD_DIRSERV, "No 'most recent' %s consensus found; "
+             "not making diffs", flavname);
+    goto done;
+  }
+  tor_assert(smartlist_len(matches));
+  smartlist_del(matches, smartlist_len(matches) - 1);
+
+  const char *most_recent_valid_after =
+    consensus_cache_entry_get_value(most_recent, LABEL_VALID_AFTER);
+  if (BUG(most_recent_valid_after == NULL))
+    goto done; //LCOV_EXCL_LINE
+  uint8_t most_recent_sha3[DIGEST256_LEN];
+  if (BUG(cdm_entry_get_sha3_value(most_recent_sha3, most_recent,
+                                   LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
+    goto done; //LCOV_EXCL_LINE
+
+  // 2. Find all the relevant diffs _to_ this consensus. These are ones
+  //    that we don't need to compute.
+  diffs = smartlist_new();
+  consensus_cache_find_all(diffs, cdm_cache_get(),
+                           LABEL_VALID_AFTER, most_recent_valid_after);
+  consensus_cache_filter_list(diffs, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
+  consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname);
+  have_diff_from = strmap_new();
+  SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
+    const char *va = consensus_cache_entry_get_value(diff,
+                                                     LABEL_FROM_VALID_AFTER);
+    if (BUG(va == NULL))
+      continue; // LCOV_EXCL_LINE
+    strmap_set(have_diff_from, va, diff);
+  } SMARTLIST_FOREACH_END(diff);
+
+  // 3. See which consensuses in 'matches' don't have diffs yet.
+  smartlist_reverse(matches); // from newest to oldest.
+  compute_diffs_from = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) {
+    const char *va = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
+    if (BUG(va == NULL))
+      continue; // LCOV_EXCL_LINE
+    if (strmap_get(have_diff_from, va) != NULL)
+      continue; /* we already have this one. */
+    smartlist_add(compute_diffs_from, ent);
+    /* Since we are not going to serve this as the most recent consensus
+     * any more, we should stop keeping it mmap'd when it's not in use.
+     */
+    consensus_cache_entry_mark_for_aggressive_release(ent);
+  } SMARTLIST_FOREACH_END(ent);
+
+  log_info(LD_DIRSERV,
+           "The most recent %s consensus is valid-after %s. We have diffs to "
+           "this consensus for %d/%d older %s consensuses. Generating diffs "
+           "for the other %d.",
+           flavname,
+           most_recent_valid_after,
+           smartlist_len(matches) - smartlist_len(compute_diffs_from),
+           smartlist_len(matches),
+           flavname,
+           smartlist_len(compute_diffs_from));
+
+  // 4. Update the hashtable; remove entries in this flavor to other
+  //    target consensuses.
+  cdm_diff_ht_purge(flavor, most_recent_sha3);
+
+  // 5. Actually launch the requests.
+  SMARTLIST_FOREACH_BEGIN(compute_diffs_from, consensus_cache_entry_t *, c) {
+    if (BUG(c == most_recent))
+      continue; // LCOV_EXCL_LINE
+
+    uint8_t this_sha3[DIGEST256_LEN];
+    if (cdm_entry_get_sha3_value(this_sha3, c,
+                                 LABEL_SHA3_DIGEST_AS_SIGNED)<0) {
+      // Not actually a bug, since we might be running with a directory
+      // with stale files from before the #22143 fixes.
+      continue;
+    }
+    if (cdm_diff_ht_check_and_note_pending(flavor,
+                                           this_sha3, most_recent_sha3)) {
+      // This is already pending, or we encountered an error.
+      continue;
+    }
+    consensus_diff_queue_diff_work(c, most_recent);
+  } SMARTLIST_FOREACH_END(c);
+
+ done:
+  smartlist_free(matches);
+  smartlist_free(diffs);
+  smartlist_free(compute_diffs_from);
+  strmap_free(have_diff_from, NULL);
+}
+
+/**
+ * Scan the cache for the latest consensuses and add their handles to
+ * latest_consensus
+ */
+static void
+consdiffmgr_consensus_load(void)
+{
+  smartlist_t *matches = smartlist_new();
+  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
+    const char *flavname = networkstatus_get_flavor_name(flav);
+    smartlist_clear(matches);
+    consensus_cache_find_all(matches, cdm_cache_get(),
+                             LABEL_FLAVOR, flavname);
+    consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+    consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches);
+    if (! most_recent)
+      continue; // no consensuses.
+    const char *most_recent_sha3 =
+      consensus_cache_entry_get_value(most_recent,
+                                      LABEL_SHA3_DIGEST_UNCOMPRESSED);
+    if (BUG(most_recent_sha3 == NULL))
+      continue; // LCOV_EXCL_LINE
+    consensus_cache_filter_list(matches, LABEL_SHA3_DIGEST_UNCOMPRESSED,
+                                most_recent_sha3);
+
+    // Everything that remains matches the most recent consensus of this
+    // flavor.
+    SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) {
+      const char *lv_compression =
+        consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
+      compress_method_t method =
+        compression_method_get_by_name(lv_compression);
+      int pos = consensus_compression_method_pos(method);
+      if (pos < 0)
+        continue;
+      consensus_cache_entry_handle_free(latest_consensus[flav][pos]);
+      latest_consensus[flav][pos] = consensus_cache_entry_handle_new(ent);
+    } SMARTLIST_FOREACH_END(ent);
+  }
+  smartlist_free(matches);
+}
+
+/**
+ * Scan the cache for diffs, and add them to the hashtable.
+ */
+static void
+consdiffmgr_diffs_load(void)
+{
+  smartlist_t *diffs = smartlist_new();
+  consensus_cache_find_all(diffs, cdm_cache_get(),
+                           LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
+  SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
+    const char *lv_flavor =
+      consensus_cache_entry_get_value(diff, LABEL_FLAVOR);
+    if (!lv_flavor)
+      continue;
+    int flavor = networkstatus_parse_flavor_name(lv_flavor);
+    if (flavor < 0)
+      continue;
+    const char *lv_compression =
+      consensus_cache_entry_get_value(diff, LABEL_COMPRESSION_TYPE);
+    compress_method_t method = NO_METHOD;
+    if (lv_compression) {
+      method = compression_method_get_by_name(lv_compression);
+      if (method == UNKNOWN_METHOD) {
+        continue;
+      }
+    }
+
+    uint8_t from_sha3[DIGEST256_LEN];
+    uint8_t to_sha3[DIGEST256_LEN];
+    if (cdm_entry_get_sha3_value(from_sha3, diff, LABEL_FROM_SHA3_DIGEST)<0)
+      continue;
+    if (cdm_entry_get_sha3_value(to_sha3, diff, LABEL_TARGET_SHA3_DIGEST)<0)
+      continue;
+
+    cdm_diff_ht_set_status(flavor, from_sha3, to_sha3,
+                           method,
+                           CDM_DIFF_PRESENT,
+                           consensus_cache_entry_handle_new(diff));
+  } SMARTLIST_FOREACH_END(diff);
+  smartlist_free(diffs);
+}
+
+/**
+ * Build new diffs as needed.
+ */
+void
+consdiffmgr_rescan(void)
+{
+  if (cdm_cache_dirty == 0)
+    return;
+
+  // Clean up here to make room for new diffs, and to ensure that older
+  // consensuses do not have any entries.
+  consdiffmgr_cleanup();
+
+  if (cdm_cache_loaded == 0) {
+    consdiffmgr_diffs_load();
+    consdiffmgr_consensus_load();
+    cdm_cache_loaded = 1;
+  }
+
+  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
+    consdiffmgr_rescan_flavor_((consensus_flavor_t) flav);
+  }
+
+  cdm_cache_dirty = 0;
+}
+
+/** Callback wrapper for consdiffmgr_rescan */
+static void
+consdiffmgr_rescan_cb(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  (void)arg;
+  consdiffmgr_rescan();
+}
+
+/** Mark the cache as dirty, and schedule a rescan event. */
+static void
+mark_cdm_cache_dirty(void)
+{
+  cdm_cache_dirty = 1;
+  tor_assert(consdiffmgr_rescan_ev);
+  mainloop_event_activate(consdiffmgr_rescan_ev);
+}
+
+/**
+ * Helper: compare two files by their from-valid-after and valid-after labels,
+ * trying to sort in ascending order by from-valid-after (when present) and
+ * valid-after (when not).  Place everything that has neither label first in
+ * the list.
+ */
+static int
+compare_by_staleness_(const void **a, const void **b)
+{
+  const consensus_cache_entry_t *e1 = *a;
+  const consensus_cache_entry_t *e2 = *b;
+  const char *va1, *fva1, *va2, *fva2;
+  va1 = consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER);
+  va2 = consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER);
+  fva1 = consensus_cache_entry_get_value(e1, LABEL_FROM_VALID_AFTER);
+  fva2 = consensus_cache_entry_get_value(e2, LABEL_FROM_VALID_AFTER);
+
+  if (fva1)
+    va1 = fva1;
+  if (fva2)
+    va2 = fva2;
+
+  /* See note about iso-encoded values in compare_by_valid_after_.  Also note
+   * that missing dates will get placed first. */
+  return strcmp_opt(va1, va2);
+}
+
+/** If there are not enough unused filenames to store <b>n</b> files, then
+ * delete old consensuses until there are.  (We have to keep track of the
+ * number of filenames because of the way that the seccomp2 cache works.)
+ *
+ * Return 0 on success, -1 on failure.
+ **/
+static int
+consdiffmgr_ensure_space_for_files(int n)
+{
+  consensus_cache_t *cache = cdm_cache_get();
+  if (consensus_cache_get_n_filenames_available(cache) >= n) {
+    // there are already enough unused filenames.
+    return 0;
+  }
+  // Try a cheap deletion of stuff that's waiting to get deleted.
+  consensus_cache_delete_pending(cache, 0);
+  if (consensus_cache_get_n_filenames_available(cache) >= n) {
+    // okay, _that_ made enough filenames available.
+    return 0;
+  }
+  // Let's get more assertive: clean out unused stuff, and force-remove
+  // the files that we can.
+  consdiffmgr_cleanup();
+  consensus_cache_delete_pending(cache, 1);
+  const int n_to_remove = n - consensus_cache_get_n_filenames_available(cache);
+  if (n_to_remove <= 0) {
+    // okay, finally!
+    return 0;
+  }
+
+  // At this point, we're going to have to throw out objects that will be
+  // missed.  Too bad!
+  smartlist_t *objects = smartlist_new();
+  consensus_cache_find_all(objects, cache, NULL, NULL);
+  smartlist_sort(objects, compare_by_staleness_);
+  int n_marked = 0;
+  SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
+    consensus_cache_entry_mark_for_removal(ent);
+    if (++n_marked >= n_to_remove)
+      break;
+  } SMARTLIST_FOREACH_END(ent);
+  smartlist_free(objects);
+
+  consensus_cache_delete_pending(cache, 1);
+
+  if (consensus_cache_may_overallocate(cache)) {
+    /* If we're allowed to throw extra files into the cache, let's do so
+     * rather getting upset.
+     */
+    return 0;
+  }
+
+  if (BUG(n_marked < n_to_remove))
+    return -1;
+  else
+    return 0;
+}
+
+/**
+ * Set consensus cache flags on the objects in this consdiffmgr.
+ */
+static void
+consdiffmgr_set_cache_flags(void)
+{
+  /* Right now, we just mark the consensus objects for aggressive release,
+   * so that they get mmapped for as little time as possible. */
+  smartlist_t *objects = smartlist_new();
+  consensus_cache_find_all(objects, cdm_cache_get(), LABEL_DOCTYPE,
+                           DOCTYPE_CONSENSUS);
+  SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
+    consensus_cache_entry_mark_for_aggressive_release(ent);
+  } SMARTLIST_FOREACH_END(ent);
+  smartlist_free(objects);
+}
+
+/**
+ * Called before shutdown: drop all storage held by the consdiffmgr.c module.
+ */
+void
+consdiffmgr_free_all(void)
+{
+  cdm_diff_t **diff, **next;
+  for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
+    cdm_diff_t *this = *diff;
+    next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
+    cdm_diff_free(this);
+  }
+  int i;
+  unsigned j;
+  for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) {
+    for (j = 0; j < n_consensus_compression_methods(); ++j) {
+      consensus_cache_entry_handle_free(latest_consensus[i][j]);
+    }
+  }
+  memset(latest_consensus, 0, sizeof(latest_consensus));
+  consensus_cache_free(cons_diff_cache);
+  cons_diff_cache = NULL;
+  mainloop_event_free(consdiffmgr_rescan_ev);
+}
+
+/* =====
+   Thread workers
+   =====*/
+
+typedef struct compressed_result_t {
+  config_line_t *labels;
+  /**
+   * Output: Body of the diff, as compressed.
+   */
+  uint8_t *body;
+  /**
+   * Output: length of body_out
+   */
+  size_t bodylen;
+} compressed_result_t;
+
+/**
+ * Compress the bytestring <b>input</b> of length <b>len</b> using the
+ * <n>n_methods</b> compression methods listed in the array <b>methods</b>.
+ *
+ * For each successful compression, set the fields in the <b>results_out</b>
+ * array in the position corresponding to the compression method. Use
+ * <b>labels_in</b> as a basis for the labels of the result.
+ *
+ * Return 0 if all compression succeeded; -1 if any failed.
+ */
+static int
+compress_multiple(compressed_result_t *results_out, int n_methods,
+                  const compress_method_t *methods,
+                  const uint8_t *input, size_t len,
+                  const config_line_t *labels_in)
+{
+  int rv = 0;
+  int i;
+  for (i = 0; i < n_methods; ++i) {
+    compress_method_t method = methods[i];
+    const char *methodname = compression_method_get_name(method);
+    char *result;
+    size_t sz;
+    if (0 == tor_compress(&result, &sz, (const char*)input, len, method)) {
+      results_out[i].body = (uint8_t*)result;
+      results_out[i].bodylen = sz;
+      results_out[i].labels = config_lines_dup(labels_in);
+      cdm_labels_prepend_sha3(&results_out[i].labels, LABEL_SHA3_DIGEST,
+                              results_out[i].body,
+                              results_out[i].bodylen);
+      config_line_prepend(&results_out[i].labels,
+                          LABEL_COMPRESSION_TYPE,
+                          methodname);
+    } else {
+      rv = -1;
+    }
+  }
+  return rv;
+}
+
+/**
+ * Given an array of <b>n</b> compressed_result_t in <b>results</b>,
+ * as produced by compress_multiple, store them all into the
+ * consdiffmgr, and store handles to them in the <b>handles_out</b>
+ * array.
+ *
+ * Return CDM_DIFF_PRESENT if any was stored, and CDM_DIFF_ERROR if none
+ * was stored.
+ */
+static cdm_diff_status_t
+store_multiple(consensus_cache_entry_handle_t **handles_out,
+               int n,
+               const compress_method_t *methods,
+               const compressed_result_t *results,
+               const char *description)
+{
+  cdm_diff_status_t status = CDM_DIFF_ERROR;
+  consdiffmgr_ensure_space_for_files(n);
+
+  int i;
+  for (i = 0; i < n; ++i) {
+    compress_method_t method = methods[i];
+    uint8_t *body_out = results[i].body;
+    size_t bodylen_out = results[i].bodylen;
+    config_line_t *labels = results[i].labels;
+    const char *methodname = compression_method_get_name(method);
+    if (body_out && bodylen_out && labels) {
+      /* Success! Store the results */
+      log_info(LD_DIRSERV, "Adding %s, compressed with %s",
+               description, methodname);
+
+      consensus_cache_entry_t *ent =
+        consensus_cache_add(cdm_cache_get(),
+                            labels,
+                            body_out,
+                            bodylen_out);
+      if (ent == NULL) {
+        static ratelim_t cant_store_ratelim = RATELIM_INIT(5*60);
+        log_fn_ratelim(&cant_store_ratelim, LOG_WARN, LD_FS,
+                       "Unable to store object %s compressed with %s.",
+                       description, methodname);
+        continue;
+      }
+
+      status = CDM_DIFF_PRESENT;
+      handles_out[i] = consensus_cache_entry_handle_new(ent);
+      consensus_cache_entry_decref(ent);
+    }
+  }
+  return status;
+}
+
+/**
+ * An object passed to a worker thread that will try to produce a consensus
+ * diff.
+ */
+typedef struct consensus_diff_worker_job_t {
+  /**
+   * Input: The consensus to compute the diff from.  Holds a reference to the
+   * cache entry, which must not be released until the job is passed back to
+   * the main thread. The body must be mapped into memory in the main thread.
+   */
+  consensus_cache_entry_t *diff_from;
+  /**
+   * Input: The consensus to compute the diff to.  Holds a reference to the
+   * cache entry, which must not be released until the job is passed back to
+   * the main thread. The body must be mapped into memory in the main thread.
+   */
+  consensus_cache_entry_t *diff_to;
+
+  /** Output: labels and bodies */
+  compressed_result_t out[ARRAY_LENGTH(compress_diffs_with)];
+} consensus_diff_worker_job_t;
+
+/** Given a consensus_cache_entry_t, check whether it has a label claiming
+ * that it was compressed.  If so, uncompress its contents into <b>out</b> and
+ * set <b>outlen</b> to hold their size.  If not, just copy the body into
+ * <b>out</b> and set <b>outlen</b> to its length.  Return 0 on success,
+ * -1 on failure.
+ *
+ * In all cases, the output is nul-terminated. */
+STATIC int
+uncompress_or_copy(char **out, size_t *outlen,
+                   consensus_cache_entry_t *ent)
+{
+  const uint8_t *body;
+  size_t bodylen;
+
+  if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0)
+    return -1;
+
+  const char *lv_compression =
+    consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
+  compress_method_t method = NO_METHOD;
+
+  if (lv_compression)
+    method = compression_method_get_by_name(lv_compression);
+
+  return tor_uncompress(out, outlen, (const char *)body, bodylen,
+                        method, 1, LOG_WARN);
+}
+
+/**
+ * Worker function. This function runs inside a worker thread and receives
+ * a consensus_diff_worker_job_t as its input.
+ */
+static workqueue_reply_t
+consensus_diff_worker_threadfn(void *state_, void *work_)
+{
+  (void)state_;
+  consensus_diff_worker_job_t *job = work_;
+  const uint8_t *diff_from, *diff_to;
+  size_t len_from, len_to;
+  int r;
+  /* We need to have the body already mapped into RAM here.
+   */
+  r = consensus_cache_entry_get_body(job->diff_from, &diff_from, &len_from);
+  if (BUG(r < 0))
+    return WQ_RPL_REPLY; // LCOV_EXCL_LINE
+  r = consensus_cache_entry_get_body(job->diff_to, &diff_to, &len_to);
+  if (BUG(r < 0))
+    return WQ_RPL_REPLY; // LCOV_EXCL_LINE
+
+  const char *lv_to_valid_after =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_AFTER);
+  const char *lv_to_fresh_until =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_FRESH_UNTIL);
+  const char *lv_to_valid_until =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_UNTIL);
+  const char *lv_to_signatories =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_SIGNATORIES);
+  const char *lv_from_valid_after =
+    consensus_cache_entry_get_value(job->diff_from, LABEL_VALID_AFTER);
+  const char *lv_from_digest =
+    consensus_cache_entry_get_value(job->diff_from,
+                                    LABEL_SHA3_DIGEST_AS_SIGNED);
+  const char *lv_from_flavor =
+    consensus_cache_entry_get_value(job->diff_from, LABEL_FLAVOR);
+  const char *lv_to_flavor =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR);
+  const char *lv_to_digest =
+    consensus_cache_entry_get_value(job->diff_to,
+                                    LABEL_SHA3_DIGEST_UNCOMPRESSED);
+
+  if (! lv_from_digest) {
+    /* This isn't a bug right now, since it can happen if you're migrating
+     * from an older version of master to a newer one.  The older ones didn't
+     * annotate their stored consensus objects with sha3-digest-as-signed.
+    */
+    return WQ_RPL_REPLY; // LCOV_EXCL_LINE
+  }
+
+  /* All these values are mandatory on the input */
+  if (BUG(!lv_to_valid_after) ||
+      BUG(!lv_from_valid_after) ||
+      BUG(!lv_from_flavor) ||
+      BUG(!lv_to_flavor)) {
+    return WQ_RPL_REPLY; // LCOV_EXCL_LINE
+  }
+  /* The flavors need to match */
+  if (BUG(strcmp(lv_from_flavor, lv_to_flavor))) {
+    return WQ_RPL_REPLY; // LCOV_EXCL_LINE
+  }
+
+  char *consensus_diff;
+  {
+    char *diff_from_nt = NULL, *diff_to_nt = NULL;
+    size_t diff_from_nt_len, diff_to_nt_len;
+
+    if (uncompress_or_copy(&diff_from_nt, &diff_from_nt_len,
+                           job->diff_from) < 0) {
+      return WQ_RPL_REPLY;
+    }
+    if (uncompress_or_copy(&diff_to_nt, &diff_to_nt_len,
+                           job->diff_to) < 0) {
+      tor_free(diff_from_nt);
+      return WQ_RPL_REPLY;
+    }
+    tor_assert(diff_from_nt);
+    tor_assert(diff_to_nt);
+
+    // XXXX ugh; this is going to calculate the SHA3 of both its
+    // XXXX inputs again, even though we already have that. Maybe it's time
+    // XXXX to change the API here?
+    consensus_diff = consensus_diff_generate(diff_from_nt, diff_to_nt);
+    tor_free(diff_from_nt);
+    tor_free(diff_to_nt);
+  }
+  if (!consensus_diff) {
+    /* Couldn't generate consensus; we'll leave the reply blank. */
+    return WQ_RPL_REPLY;
+  }
+
+  /* Compress the results and send the reply */
+  tor_assert(compress_diffs_with[0] == NO_METHOD);
+  size_t difflen = strlen(consensus_diff);
+  job->out[0].body = (uint8_t *) consensus_diff;
+  job->out[0].bodylen = difflen;
+
+  config_line_t *common_labels = NULL;
+  if (lv_to_valid_until)
+    config_line_prepend(&common_labels, LABEL_VALID_UNTIL, lv_to_valid_until);
+  if (lv_to_fresh_until)
+    config_line_prepend(&common_labels, LABEL_FRESH_UNTIL, lv_to_fresh_until);
+  if (lv_to_signatories)
+    config_line_prepend(&common_labels, LABEL_SIGNATORIES, lv_to_signatories);
+  cdm_labels_prepend_sha3(&common_labels,
+                          LABEL_SHA3_DIGEST_UNCOMPRESSED,
+                          job->out[0].body,
+                          job->out[0].bodylen);
+  config_line_prepend(&common_labels, LABEL_FROM_VALID_AFTER,
+                      lv_from_valid_after);
+  config_line_prepend(&common_labels, LABEL_VALID_AFTER,
+                      lv_to_valid_after);
+  config_line_prepend(&common_labels, LABEL_FLAVOR, lv_from_flavor);
+  config_line_prepend(&common_labels, LABEL_FROM_SHA3_DIGEST,
+                      lv_from_digest);
+  config_line_prepend(&common_labels, LABEL_TARGET_SHA3_DIGEST,
+                      lv_to_digest);
+  config_line_prepend(&common_labels, LABEL_DOCTYPE,
+                      DOCTYPE_CONSENSUS_DIFF);
+
+  job->out[0].labels = config_lines_dup(common_labels);
+  cdm_labels_prepend_sha3(&job->out[0].labels,
+                          LABEL_SHA3_DIGEST,
+                          job->out[0].body,
+                          job->out[0].bodylen);
+
+  compress_multiple(job->out+1,
+                    n_diff_compression_methods()-1,
+                    compress_diffs_with+1,
+                    (const uint8_t*)consensus_diff, difflen, common_labels);
+
+  config_free_lines(common_labels);
+  return WQ_RPL_REPLY;
+}
+
+#define consensus_diff_worker_job_free(job)             \
+  FREE_AND_NULL(consensus_diff_worker_job_t,            \
+                consensus_diff_worker_job_free_, (job))
+
+/**
+ * Helper: release all storage held in <b>job</b>.
+ */
+static void
+consensus_diff_worker_job_free_(consensus_diff_worker_job_t *job)
+{
+  if (!job)
+    return;
+  unsigned u;
+  for (u = 0; u < n_diff_compression_methods(); ++u) {
+    config_free_lines(job->out[u].labels);
+    tor_free(job->out[u].body);
+  }
+  consensus_cache_entry_decref(job->diff_from);
+  consensus_cache_entry_decref(job->diff_to);
+  tor_free(job);
+}
+
+/**
+ * Worker function: This function runs in the main thread, and receives
+ * a consensus_diff_worker_job_t that the worker thread has already
+ * processed.
+ */
+static void
+consensus_diff_worker_replyfn(void *work_)
+{
+  tor_assert(in_main_thread());
+  tor_assert(work_);
+
+  consensus_diff_worker_job_t *job = work_;
+
+  const char *lv_from_digest =
+    consensus_cache_entry_get_value(job->diff_from,
+                                    LABEL_SHA3_DIGEST_AS_SIGNED);
+  const char *lv_to_digest =
+    consensus_cache_entry_get_value(job->diff_to,
+                                    LABEL_SHA3_DIGEST_UNCOMPRESSED);
+  const char *lv_flavor =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR);
+  if (BUG(lv_from_digest == NULL))
+    lv_from_digest = "???"; // LCOV_EXCL_LINE
+  if (BUG(lv_to_digest == NULL))
+    lv_to_digest = "???"; // LCOV_EXCL_LINE
+
+  uint8_t from_sha3[DIGEST256_LEN];
+  uint8_t to_sha3[DIGEST256_LEN];
+  int flav = -1;
+  int cache = 1;
+  if (BUG(cdm_entry_get_sha3_value(from_sha3, job->diff_from,
+                                   LABEL_SHA3_DIGEST_AS_SIGNED) < 0))
+    cache = 0;
+  if (BUG(cdm_entry_get_sha3_value(to_sha3, job->diff_to,
+                                   LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
+    cache = 0;
+  if (BUG(lv_flavor == NULL)) {
+    cache = 0;
+  } else if ((flav = networkstatus_parse_flavor_name(lv_flavor)) < 0) {
+    cache = 0;
+  }
+
+  consensus_cache_entry_handle_t *handles[ARRAY_LENGTH(compress_diffs_with)];
+  memset(handles, 0, sizeof(handles));
+
+  char description[128];
+  tor_snprintf(description, sizeof(description),
+               "consensus diff from %s to %s",
+               lv_from_digest, lv_to_digest);
+
+  int status = store_multiple(handles,
+                              n_diff_compression_methods(),
+                              compress_diffs_with,
+                              job->out,
+                              description);
+
+  if (status != CDM_DIFF_PRESENT) {
+    /* Failure! Nothing to do but complain */
+    log_warn(LD_DIRSERV,
+             "Worker was unable to compute consensus diff "
+             "from %s to %s", lv_from_digest, lv_to_digest);
+    /* Cache this error so we don't try to compute this one again. */
+    status = CDM_DIFF_ERROR;
+  }
+
+  unsigned u;
+  for (u = 0; u < ARRAY_LENGTH(handles); ++u) {
+    compress_method_t method = compress_diffs_with[u];
+    if (cache) {
+      consensus_cache_entry_handle_t *h = handles[u];
+      int this_status = status;
+      if (h == NULL) {
+        this_status = CDM_DIFF_ERROR;
+      }
+      tor_assert_nonfatal(h != NULL || this_status == CDM_DIFF_ERROR);
+      cdm_diff_ht_set_status(flav, from_sha3, to_sha3, method, this_status, h);
+    } else {
+      consensus_cache_entry_handle_free(handles[u]);
+    }
+  }
+
+  consensus_diff_worker_job_free(job);
+}
+
+/**
+ * Queue the job of computing the diff from <b>diff_from</b> to <b>diff_to</b>
+ * in a worker thread.
+ */
+static int
+consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
+                               consensus_cache_entry_t *diff_to)
+{
+  tor_assert(in_main_thread());
+
+  consensus_cache_entry_incref(diff_from);
+  consensus_cache_entry_incref(diff_to);
+
+  consensus_diff_worker_job_t *job = tor_malloc_zero(sizeof(*job));
+  job->diff_from = diff_from;
+  job->diff_to = diff_to;
+
+  /* Make sure body is mapped. */
+  const uint8_t *body;
+  size_t bodylen;
+  int r1 = consensus_cache_entry_get_body(diff_from, &body, &bodylen);
+  int r2 = consensus_cache_entry_get_body(diff_to, &body, &bodylen);
+  if (r1 < 0 || r2 < 0)
+    goto err;
+
+  workqueue_entry_t *work;
+  work = cpuworker_queue_work(WQ_PRI_LOW,
+                              consensus_diff_worker_threadfn,
+                              consensus_diff_worker_replyfn,
+                              job);
+  if (!work)
+    goto err;
+
+  return 0;
+ err:
+  consensus_diff_worker_job_free(job); // includes decrefs.
+  return -1;
+}
+
+/**
+ * Holds requests and replies for consensus_compress_workers.
+ */
+typedef struct consensus_compress_worker_job_t {
+  char *consensus;
+  size_t consensus_len;
+  consensus_flavor_t flavor;
+  config_line_t *labels_in;
+  compressed_result_t out[ARRAY_LENGTH(compress_consensus_with)];
+} consensus_compress_worker_job_t;
+
+#define consensus_compress_worker_job_free(job) \
+  FREE_AND_NULL(consensus_compress_worker_job_t, \
+                consensus_compress_worker_job_free_, (job))
+
+/**
+ * Free all resources held in <b>job</b>
+ */
+static void
+consensus_compress_worker_job_free_(consensus_compress_worker_job_t *job)
+{
+  if (!job)
+    return;
+  tor_free(job->consensus);
+  config_free_lines(job->labels_in);
+  unsigned u;
+  for (u = 0; u < n_consensus_compression_methods(); ++u) {
+    config_free_lines(job->out[u].labels);
+    tor_free(job->out[u].body);
+  }
+  tor_free(job);
+}
+/**
+ * Worker function. This function runs inside a worker thread and receives
+ * a consensus_compress_worker_job_t as its input.
+ */
+static workqueue_reply_t
+consensus_compress_worker_threadfn(void *state_, void *work_)
+{
+  (void)state_;
+  consensus_compress_worker_job_t *job = work_;
+  consensus_flavor_t flavor = job->flavor;
+  const char *consensus = job->consensus;
+  size_t bodylen = job->consensus_len;
+
+  config_line_t *labels = config_lines_dup(job->labels_in);
+  const char *flavname = networkstatus_get_flavor_name(flavor);
+
+  cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED,
+                          (const uint8_t *)consensus, bodylen);
+  {
+    const char *start, *end;
+    if (router_get_networkstatus_v3_signed_boundaries(consensus,
+                                                        &start, &end) < 0) {
+      start = consensus;
+      end = consensus+bodylen;
+    }
+    cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_AS_SIGNED,
+                            (const uint8_t *)start,
+                            end - start);
+  }
+  config_line_prepend(&labels, LABEL_FLAVOR, flavname);
+  config_line_prepend(&labels, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+
+  compress_multiple(job->out,
+                    n_consensus_compression_methods(),
+                    compress_consensus_with,
+                    (const uint8_t*)consensus, bodylen, labels);
+  config_free_lines(labels);
+  return WQ_RPL_REPLY;
+}
+
+/**
+ * Worker function: This function runs in the main thread, and receives
+ * a consensus_diff_compress_job_t that the worker thread has already
+ * processed.
+ */
+static void
+consensus_compress_worker_replyfn(void *work_)
+{
+  consensus_compress_worker_job_t *job = work_;
+
+  consensus_cache_entry_handle_t *handles[
+                               ARRAY_LENGTH(compress_consensus_with)];
+  memset(handles, 0, sizeof(handles));
+
+  store_multiple(handles,
+                 n_consensus_compression_methods(),
+                 compress_consensus_with,
+                 job->out,
+                 "consensus");
+  mark_cdm_cache_dirty();
+
+  unsigned u;
+  consensus_flavor_t f = job->flavor;
+  tor_assert((int)f < N_CONSENSUS_FLAVORS);
+  for (u = 0; u < ARRAY_LENGTH(handles); ++u) {
+    if (handles[u] == NULL)
+      continue;
+    consensus_cache_entry_handle_free(latest_consensus[f][u]);
+    latest_consensus[f][u] = handles[u];
+  }
+
+  consensus_compress_worker_job_free(job);
+}
+
+/**
+ * If true, we compress in worker threads.
+ */
+static int background_compression = 0;
+
+/**
+ * Queue a job to compress <b>consensus</b> and store its compressed
+ * text in the cache.
+ */
+static int
+consensus_queue_compression_work(const char *consensus,
+                                 const networkstatus_t *as_parsed)
+{
+  tor_assert(consensus);
+  tor_assert(as_parsed);
+
+  consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job));
+  job->consensus = tor_strdup(consensus);
+  job->consensus_len = strlen(consensus);
+  job->flavor = as_parsed->flavor;
+
+  char va_str[ISO_TIME_LEN+1];
+  char vu_str[ISO_TIME_LEN+1];
+  char fu_str[ISO_TIME_LEN+1];
+  format_iso_time_nospace(va_str, as_parsed->valid_after);
+  format_iso_time_nospace(fu_str, as_parsed->fresh_until);
+  format_iso_time_nospace(vu_str, as_parsed->valid_until);
+  config_line_append(&job->labels_in, LABEL_VALID_AFTER, va_str);
+  config_line_append(&job->labels_in, LABEL_FRESH_UNTIL, fu_str);
+  config_line_append(&job->labels_in, LABEL_VALID_UNTIL, vu_str);
+  if (as_parsed->voters) {
+    smartlist_t *hexvoters = smartlist_new();
+    SMARTLIST_FOREACH_BEGIN(as_parsed->voters,
+                            networkstatus_voter_info_t *, vi) {
+      if (smartlist_len(vi->sigs) == 0)
+        continue; // didn't sign.
+      char d[HEX_DIGEST_LEN+1];
+      base16_encode(d, sizeof(d), vi->identity_digest, DIGEST_LEN);
+      smartlist_add_strdup(hexvoters, d);
+    } SMARTLIST_FOREACH_END(vi);
+    char *signers = smartlist_join_strings(hexvoters, ",", 0, NULL);
+    config_line_prepend(&job->labels_in, LABEL_SIGNATORIES, signers);
+    tor_free(signers);
+    SMARTLIST_FOREACH(hexvoters, char *, cp, tor_free(cp));
+    smartlist_free(hexvoters);
+  }
+
+  if (background_compression) {
+    workqueue_entry_t *work;
+    work = cpuworker_queue_work(WQ_PRI_LOW,
+                                consensus_compress_worker_threadfn,
+                                consensus_compress_worker_replyfn,
+                                job);
+    if (!work) {
+      consensus_compress_worker_job_free(job);
+      return -1;
+    }
+
+    return 0;
+  } else {
+    consensus_compress_worker_threadfn(NULL, job);
+    consensus_compress_worker_replyfn(job);
+    return 0;
+  }
+}
+
+/**
+ * Tell the consdiffmgr backend to compress consensuses in worker threads.
+ */
+void
+consdiffmgr_enable_background_compression(void)
+{
+  // This isn't the default behavior because it would break unit tests.
+  background_compression = 1;
+}
+
+/** Read the set of voters from the cached object <b>ent</b> into
+ * <b>out</b>, as a list of hex-encoded digests. Return 0 on success,
+ * -1 if no signatories were recorded. */
+int
+consensus_cache_entry_get_voter_id_digests(const consensus_cache_entry_t *ent,
+                                           smartlist_t *out)
+{
+  tor_assert(ent);
+  tor_assert(out);
+  const char *s;
+  s = consensus_cache_entry_get_value(ent, LABEL_SIGNATORIES);
+  if (s == NULL)
+    return -1;
+  smartlist_split_string(out, s, ",", SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE, 0);
+  return 0;
+}
+
+/** Read the fresh-until time of cached object <b>ent</b> into *<b>out</b>
+ * and return 0, or return -1 if no such time was recorded. */
+int
+consensus_cache_entry_get_fresh_until(const consensus_cache_entry_t *ent,
+                                      time_t *out)
+{
+  tor_assert(ent);
+  tor_assert(out);
+  const char *s;
+  s = consensus_cache_entry_get_value(ent, LABEL_FRESH_UNTIL);
+  if (s == NULL || parse_iso_time_nospace(s, out) < 0)
+    return -1;
+  else
+    return 0;
+}
+
+/** Read the valid until timestamp from the cached object <b>ent</b> into
+ * *<b>out</b> and return 0, or return -1 if no such time was recorded. */
+int
+consensus_cache_entry_get_valid_until(const consensus_cache_entry_t *ent,
+                                      time_t *out)
+{
+  tor_assert(ent);
+  tor_assert(out);
+
+  const char *s;
+  s = consensus_cache_entry_get_value(ent, LABEL_VALID_UNTIL);
+  if (s == NULL || parse_iso_time_nospace(s, out) < 0)
+    return -1;
+  else
+    return 0;
+}
+
+/** Read the valid after timestamp from the cached object <b>ent</b> into
+ * *<b>out</b> and return 0, or return -1 if no such time was recorded. */
+int
+consensus_cache_entry_get_valid_after(const consensus_cache_entry_t *ent,
+                                      time_t *out)
+{
+  tor_assert(ent);
+  tor_assert(out);
+
+  const char *s;
+  s = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
+
+  if (s == NULL || parse_iso_time_nospace(s, out) < 0)
+    return -1;
+  else
+    return 0;
+}
diff --git a/src/feature/dircache/consdiffmgr.h b/src/feature/dircache/consdiffmgr.h
new file mode 100644
index 0000000000..39e8fa31cb
--- /dev/null
+++ b/src/feature/dircache/consdiffmgr.h
@@ -0,0 +1,75 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_CONSDIFFMGR_H
+#define TOR_CONSDIFFMGR_H
+
+enum compress_method_t;
+
+/**
+ * Possible outcomes from trying to look up a given consensus diff.
+ */
+typedef enum consdiff_status_t {
+  CONSDIFF_AVAILABLE,
+  CONSDIFF_NOT_FOUND,
+  CONSDIFF_IN_PROGRESS,
+} consdiff_status_t;
+
+typedef struct consdiff_cfg_t {
+  int32_t cache_max_num;
+} consdiff_cfg_t;
+
+struct consensus_cache_entry_t; // from conscache.h
+
+int consdiffmgr_add_consensus(const char *consensus,
+                              const networkstatus_t *as_parsed);
+
+consdiff_status_t consdiffmgr_find_consensus(
+                           struct consensus_cache_entry_t **entry_out,
+                           consensus_flavor_t flavor,
+                           enum compress_method_t method);
+
+consdiff_status_t consdiffmgr_find_diff_from(
+                           struct consensus_cache_entry_t **entry_out,
+                           consensus_flavor_t flavor,
+                           int digest_type,
+                           const uint8_t *digest,
+                           size_t digestlen,
+                           enum compress_method_t method);
+
+int consensus_cache_entry_get_voter_id_digests(
+                                  const struct consensus_cache_entry_t *ent,
+                                  smartlist_t *out);
+int consensus_cache_entry_get_fresh_until(
+                                  const struct consensus_cache_entry_t *ent,
+                                  time_t *out);
+int consensus_cache_entry_get_valid_until(
+                                  const struct consensus_cache_entry_t *ent,
+                                  time_t *out);
+int consensus_cache_entry_get_valid_after(
+                                  const struct consensus_cache_entry_t *ent,
+                                  time_t *out);
+
+void consdiffmgr_rescan(void);
+int consdiffmgr_cleanup(void);
+void consdiffmgr_enable_background_compression(void);
+void consdiffmgr_configure(const consdiff_cfg_t *cfg);
+struct sandbox_cfg_elem;
+int consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg);
+void consdiffmgr_free_all(void);
+int consdiffmgr_validate(void);
+
+#ifdef CONSDIFFMGR_PRIVATE
+STATIC unsigned n_diff_compression_methods(void);
+STATIC unsigned n_consensus_compression_methods(void);
+STATIC consensus_cache_t *cdm_cache_get(void);
+STATIC consensus_cache_entry_t *cdm_cache_lookup_consensus(
+                          consensus_flavor_t flavor, time_t valid_after);
+STATIC int cdm_entry_get_sha3_value(uint8_t *digest_out,
+                                    consensus_cache_entry_t *ent,
+                                    const char *label);
+STATIC int uncompress_or_copy(char **out, size_t *outlen,
+                              consensus_cache_entry_t *ent);
+#endif /* defined(CONSDIFFMGR_PRIVATE) */
+
+#endif /* !defined(TOR_CONSDIFFMGR_H) */
diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c
new file mode 100644
index 0000000000..e8cb284165
--- /dev/null
+++ b/src/feature/dircache/dircache.c
@@ -0,0 +1,1740 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DIRCACHE_PRIVATE
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/relay.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/authmode.h"
+#include "feature/dirauth/process_descs.h"
+#include "feature/dircache/conscache.h"
+#include "feature/dircache/consdiffmgr.h"
+#include "feature/dircache/dircache.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dircommon/fp_pair.h"
+#include "feature/hs/hs_cache.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/routermode.h"
+#include "feature/rend/rendcache.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/stats/rephist.h"
+#include "lib/compress/compress.h"
+
+#include "feature/dircache/cached_dir_st.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/nodelist/authority_cert_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+
+/** Maximum size, in bytes, for any directory object that we're accepting
+ * as an upload. */
+#define MAX_DIR_UL_SIZE ((1<<24)-1) /* 16MB-1 */
+
+/** HTTP cache control: how long do we tell proxies they can cache each
+ * kind of document we serve? */
+#define FULL_DIR_CACHE_LIFETIME (60*60)
+#define RUNNINGROUTERS_CACHE_LIFETIME (20*60)
+#define DIRPORTFRONTPAGE_CACHE_LIFETIME (20*60)
+#define NETWORKSTATUS_CACHE_LIFETIME (5*60)
+#define ROUTERDESC_CACHE_LIFETIME (30*60)
+#define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
+#define ROBOTS_CACHE_LIFETIME (24*60*60)
+#define MICRODESC_CACHE_LIFETIME (48*60*60)
+
+/** Parse an HTTP request string <b>headers</b> of the form
+ * \verbatim
+ * "\%s [http[s]://]\%s HTTP/1..."
+ * \endverbatim
+ * If it's well-formed, strdup the second \%s into *<b>url</b>, and
+ * nul-terminate it. If the url doesn't start with "/tor/", rewrite it
+ * so it does. Return 0.
+ * Otherwise, return -1.
+ */
+STATIC int
+parse_http_url(const char *headers, char **url)
+{
+  char *command = NULL;
+  if (parse_http_command(headers, &command, url) < 0) {
+    return -1;
+  }
+  if (strcmpstart(*url, "/tor/")) {
+    char *new_url = NULL;
+    tor_asprintf(&new_url, "/tor%s%s",
+                 *url[0] == '/' ? "" : "/",
+                 *url);
+    tor_free(*url);
+    *url = new_url;
+  }
+  tor_free(command);
+  return 0;
+}
+
+/** Create an http response for the client <b>conn</b> out of
+ * <b>status</b> and <b>reason_phrase</b>. Write it to <b>conn</b>.
+ */
+static void
+write_short_http_response(dir_connection_t *conn, int status,
+                       const char *reason_phrase)
+{
+  char *buf = NULL;
+  char *datestring = NULL;
+
+  IF_BUG_ONCE(!reason_phrase) { /* bullet-proofing */
+    reason_phrase = "unspecified";
+  }
+
+  if (server_mode(get_options())) {
+    /* include the Date: header, but only if we're a relay or bridge */
+    char datebuf[RFC1123_TIME_LEN+1];
+    format_rfc1123_time(datebuf, time(NULL));
+    tor_asprintf(&datestring, "Date: %s\r\n", datebuf);
+  }
+
+  tor_asprintf(&buf, "HTTP/1.0 %d %s\r\n%s\r\n",
+               status, reason_phrase, datestring?datestring:"");
+
+  log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase);
+  connection_buf_add(buf, strlen(buf), TO_CONN(conn));
+
+  tor_free(datestring);
+  tor_free(buf);
+}
+
+/** Write the header for an HTTP/1.0 response onto <b>conn</b>-\>outbuf,
+ * with <b>type</b> as the Content-Type.
+ *
+ * If <b>length</b> is nonnegative, it is the Content-Length.
+ * If <b>encoding</b> is provided, it is the Content-Encoding.
+ * If <b>cache_lifetime</b> is greater than 0, the content may be cached for
+ * up to cache_lifetime seconds.  Otherwise, the content may not be cached. */
+static void
+write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
+                           const char *type, const char *encoding,
+                           const char *extra_headers,
+                           long cache_lifetime)
+{
+  char date[RFC1123_TIME_LEN+1];
+  time_t now = time(NULL);
+  buf_t *buf = buf_new_with_capacity(1024);
+
+  tor_assert(conn);
+
+  format_rfc1123_time(date, now);
+
+  buf_add_printf(buf, "HTTP/1.0 200 OK\r\nDate: %s\r\n", date);
+  if (type) {
+    buf_add_printf(buf, "Content-Type: %s\r\n", type);
+  }
+  if (!is_local_addr(&conn->base_.addr)) {
+    /* Don't report the source address for a nearby/private connection.
+     * Otherwise we tend to mis-report in cases where incoming ports are
+     * being forwarded to a Tor server running behind the firewall. */
+    buf_add_printf(buf, X_ADDRESS_HEADER "%s\r\n", conn->base_.address);
+  }
+  if (encoding) {
+    buf_add_printf(buf, "Content-Encoding: %s\r\n", encoding);
+  }
+  if (length >= 0) {
+    buf_add_printf(buf, "Content-Length: %ld\r\n", (long)length);
+  }
+  if (cache_lifetime > 0) {
+    char expbuf[RFC1123_TIME_LEN+1];
+    format_rfc1123_time(expbuf, (time_t)(now + cache_lifetime));
+    /* We could say 'Cache-control: max-age=%d' here if we start doing
+     * http/1.1 */
+    buf_add_printf(buf, "Expires: %s\r\n", expbuf);
+  } else if (cache_lifetime == 0) {
+    /* We could say 'Cache-control: no-cache' here if we start doing
+     * http/1.1 */
+    buf_add_string(buf, "Pragma: no-cache\r\n");
+  }
+  if (extra_headers) {
+    buf_add_string(buf, extra_headers);
+  }
+  buf_add_string(buf, "\r\n");
+
+  connection_buf_add_buf(TO_CONN(conn), buf);
+  buf_free(buf);
+}
+
+/** As write_http_response_header_impl, but sets encoding and content-typed
+ * based on whether the response will be <b>compressed</b> or not. */
+static void
+write_http_response_headers(dir_connection_t *conn, ssize_t length,
+                            compress_method_t method,
+                            const char *extra_headers, long cache_lifetime)
+{
+  const char *methodname = compression_method_get_name(method);
+  const char *doctype;
+  if (method == NO_METHOD)
+    doctype = "text/plain";
+  else
+    doctype = "application/octet-stream";
+  write_http_response_header_impl(conn, length,
+                                  doctype,
+                                  methodname,
+                                  extra_headers,
+                                  cache_lifetime);
+}
+
+/** As write_http_response_headers, but assumes extra_headers is NULL */
+static void
+write_http_response_header(dir_connection_t *conn, ssize_t length,
+                           compress_method_t method,
+                           long cache_lifetime)
+{
+  write_http_response_headers(conn, length, method, NULL, cache_lifetime);
+}
+
+/** Array of compression methods to use (if supported) for serving
+ * precompressed data, ordered from best to worst. */
+static compress_method_t srv_meth_pref_precompressed[] = {
+  LZMA_METHOD,
+  ZSTD_METHOD,
+  ZLIB_METHOD,
+  GZIP_METHOD,
+  NO_METHOD
+};
+
+/** Array of compression methods to use (if supported) for serving
+ * streamed data, ordered from best to worst. */
+static compress_method_t srv_meth_pref_streaming_compression[] = {
+  ZSTD_METHOD,
+  ZLIB_METHOD,
+  GZIP_METHOD,
+  NO_METHOD
+};
+
+/** Parse the compression methods listed in an Accept-Encoding header <b>h</b>,
+ * and convert them to a bitfield where compression method x is supported if
+ * and only if 1 &lt;&lt; x is set in the bitfield. */
+STATIC unsigned
+parse_accept_encoding_header(const char *h)
+{
+  unsigned result = (1u << NO_METHOD);
+  smartlist_t *methods = smartlist_new();
+  smartlist_split_string(methods, h, ",",
+             SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+
+  SMARTLIST_FOREACH_BEGIN(methods, const char *, m) {
+    compress_method_t method = compression_method_get_by_name(m);
+    if (method != UNKNOWN_METHOD) {
+      tor_assert(((unsigned)method) < 8*sizeof(unsigned));
+      result |= (1u << method);
+    }
+  } SMARTLIST_FOREACH_END(m);
+  SMARTLIST_FOREACH_BEGIN(methods, char *, m) {
+    tor_free(m);
+  } SMARTLIST_FOREACH_END(m);
+  smartlist_free(methods);
+  return result;
+}
+
+/** Decide whether a client would accept the consensus we have.
+ *
+ * Clients can say they only want a consensus if it's signed by more
+ * than half the authorities in a list.  They pass this list in
+ * the url as "...consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>".
+ *
+ * <b>fpr</b> may be an abbreviated fingerprint, i.e. only a left substring
+ * of the full authority identity digest. (Only strings of even length,
+ * i.e. encodings of full bytes, are handled correctly.  In the case
+ * of an odd number of hex digits the last one is silently ignored.)
+ *
+ * Returns 1 if more than half of the requested authorities signed the
+ * consensus, 0 otherwise.
+ */
+static int
+client_likes_consensus(const struct consensus_cache_entry_t *ent,
+                       const char *want_url)
+{
+  smartlist_t *voters = smartlist_new();
+  int need_at_least;
+  int have = 0;
+
+  if (consensus_cache_entry_get_voter_id_digests(ent, voters) != 0) {
+    smartlist_free(voters);
+    return 1; // We don't know the voters; assume the client won't mind. */
+  }
+
+  smartlist_t *want_authorities = smartlist_new();
+  dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0);
+  need_at_least = smartlist_len(want_authorities)/2+1;
+
+  SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, want_digest) {
+
+    SMARTLIST_FOREACH_BEGIN(voters, const char *, digest) {
+      if (!strcasecmpstart(digest, want_digest)) {
+        have++;
+        break;
+      };
+    } SMARTLIST_FOREACH_END(digest);
+
+    /* early exit, if we already have enough */
+    if (have >= need_at_least)
+      break;
+  } SMARTLIST_FOREACH_END(want_digest);
+
+  SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d));
+  smartlist_free(want_authorities);
+  SMARTLIST_FOREACH(voters, char *, cp, tor_free(cp));
+  smartlist_free(voters);
+  return (have >= need_at_least);
+}
+
+/** Return the compression level we should use for sending a compressed
+ * response of size <b>n_bytes</b>. */
+STATIC compression_level_t
+choose_compression_level(ssize_t n_bytes)
+{
+  if (! have_been_under_memory_pressure()) {
+    return HIGH_COMPRESSION; /* we have plenty of RAM. */
+  } else if (n_bytes < 0) {
+    return HIGH_COMPRESSION; /* unknown; might be big. */
+  } else if (n_bytes < 1024) {
+    return LOW_COMPRESSION;
+  } else if (n_bytes < 2048) {
+    return MEDIUM_COMPRESSION;
+  } else {
+    return HIGH_COMPRESSION;
+  }
+}
+
+/** Information passed to handle a GET request. */
+typedef struct get_handler_args_t {
+  /** Bitmask of compression methods that the client said (or implied) it
+   * supported. */
+  unsigned compression_supported;
+  /** If nonzero, the time included an if-modified-since header with this
+   * value. */
+  time_t if_modified_since;
+  /** String containing the requested URL or resource. */
+  const char *url;
+  /** String containing the HTTP headers */
+  const char *headers;
+} get_handler_args_t;
+
+/** Entry for handling an HTTP GET request.
+ *
+ * This entry matches a request if "string" is equal to the requested
+ * resource, or if "is_prefix" is true and "string" is a prefix of the
+ * requested resource.
+ *
+ * The 'handler' function is called to handle the request.  It receives
+ * an arguments structure, and must return 0 on success or -1 if we should
+ * close the connection.
+ **/
+typedef struct url_table_ent_s {
+  const char *string;
+  int is_prefix;
+  int (*handler)(dir_connection_t *conn, const get_handler_args_t *args);
+} url_table_ent_t;
+
+static int handle_get_frontpage(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+static int handle_get_current_consensus(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+static int handle_get_status_vote(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+static int handle_get_microdesc(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+static int handle_get_descriptor(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+static int handle_get_keys(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+static int handle_get_hs_descriptor_v2(dir_connection_t *conn,
+                                       const get_handler_args_t *args);
+static int handle_get_robots(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+static int handle_get_networkstatus_bridges(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+
+/** Table for handling GET requests. */
+static const url_table_ent_t url_table[] = {
+  { "/tor/", 0, handle_get_frontpage },
+  { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus },
+  { "/tor/status-vote/current/", 1, handle_get_status_vote },
+  { "/tor/status-vote/next/", 1, handle_get_status_vote },
+  { "/tor/micro/d/", 1, handle_get_microdesc },
+  { "/tor/server/", 1, handle_get_descriptor },
+  { "/tor/extra/", 1, handle_get_descriptor },
+  { "/tor/keys/", 1, handle_get_keys },
+  { "/tor/rendezvous2/", 1, handle_get_hs_descriptor_v2 },
+  { "/tor/hs/3/", 1, handle_get_hs_descriptor_v3 },
+  { "/tor/robots.txt", 0, handle_get_robots },
+  { "/tor/networkstatus-bridges", 0, handle_get_networkstatus_bridges },
+  { NULL, 0, NULL },
+};
+
+/** Helper function: called when a dirserver gets a complete HTTP GET
+ * request.  Look for a request for a directory or for a rendezvous
+ * service descriptor.  On finding one, write a response into
+ * conn-\>outbuf.  If the request is unrecognized, send a 404.
+ * Return 0 if we handled this successfully, or -1 if we need to close
+ * the connection. */
+MOCK_IMPL(STATIC int,
+directory_handle_command_get,(dir_connection_t *conn, const char *headers,
+                              const char *req_body, size_t req_body_len))
+{
+  char *url, *url_mem, *header;
+  time_t if_modified_since = 0;
+  int zlib_compressed_in_url;
+  unsigned compression_methods_supported;
+
+  /* We ignore the body of a GET request. */
+  (void)req_body;
+  (void)req_body_len;
+
+  log_debug(LD_DIRSERV,"Received GET command.");
+
+  conn->base_.state = DIR_CONN_STATE_SERVER_WRITING;
+
+  if (parse_http_url(headers, &url) < 0) {
+    write_short_http_response(conn, 400, "Bad request");
+    return 0;
+  }
+  if ((header = http_get_header(headers, "If-Modified-Since: "))) {
+    struct tm tm;
+    if (parse_http_time(header, &tm) == 0) {
+      if (tor_timegm(&tm, &if_modified_since)<0) {
+        if_modified_since = 0;
+      } else {
+        log_debug(LD_DIRSERV, "If-Modified-Since is '%s'.", escaped(header));
+      }
+    }
+    /* The correct behavior on a malformed If-Modified-Since header is to
+     * act as if no If-Modified-Since header had been given. */
+    tor_free(header);
+  }
+  log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url));
+
+  url_mem = url;
+  {
+    size_t url_len = strlen(url);
+
+    zlib_compressed_in_url = url_len > 2 && !strcmp(url+url_len-2, ".z");
+    if (zlib_compressed_in_url) {
+      url[url_len-2] = '\0';
+    }
+  }
+
+  if ((header = http_get_header(headers, "Accept-Encoding: "))) {
+    compression_methods_supported = parse_accept_encoding_header(header);
+    tor_free(header);
+  } else {
+    compression_methods_supported = (1u << NO_METHOD);
+  }
+  if (zlib_compressed_in_url) {
+    compression_methods_supported |= (1u << ZLIB_METHOD);
+  }
+
+  /* Remove all methods that we don't both support. */
+  compression_methods_supported &= tor_compress_get_supported_method_bitmask();
+
+  get_handler_args_t args;
+  args.url = url;
+  args.headers = headers;
+  args.if_modified_since = if_modified_since;
+  args.compression_supported = compression_methods_supported;
+
+  int i, result = -1;
+  for (i = 0; url_table[i].string; ++i) {
+    int match;
+    if (url_table[i].is_prefix) {
+      match = !strcmpstart(url, url_table[i].string);
+    } else {
+      match = !strcmp(url, url_table[i].string);
+    }
+    if (match) {
+      result = url_table[i].handler(conn, &args);
+      goto done;
+    }
+  }
+
+  /* we didn't recognize the url */
+  write_short_http_response(conn, 404, "Not found");
+  result = 0;
+
+ done:
+  tor_free(url_mem);
+  return result;
+}
+
+/** Helper function for GET / or GET /tor/
+ */
+static int
+handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args)
+{
+  (void) args; /* unused */
+  const char *frontpage = get_dirportfrontpage();
+
+  if (frontpage) {
+    size_t dlen;
+    dlen = strlen(frontpage);
+    /* Let's return a disclaimer page (users shouldn't use V1 anymore,
+       and caches don't fetch '/', so this is safe). */
+
+    /* [We don't check for write_bucket_low here, since we want to serve
+     *  this page no matter what.] */
+    write_http_response_header_impl(conn, dlen, "text/html", "identity",
+                                    NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME);
+    connection_buf_add(frontpage, dlen, TO_CONN(conn));
+  } else {
+    write_short_http_response(conn, 404, "Not found");
+  }
+  return 0;
+}
+
+/** Warn that the cached consensus <b>consensus</b> of type
+ * <b>flavor</b> is too old and will not be served to clients. Rate-limit the
+ * warning to avoid logging an entry on every request.
+ */
+static void
+warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus,
+                          const char *flavor, time_t now)
+{
+#define TOO_OLD_WARNING_INTERVAL (60*60)
+  static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL);
+  char timestamp[ISO_TIME_LEN+1];
+  time_t valid_until;
+  char *dupes;
+
+  if (consensus_cache_entry_get_valid_until(consensus, &valid_until))
+    return;
+
+  if ((dupes = rate_limit_log(&warned, now))) {
+    format_local_iso_time(timestamp, valid_until);
+    log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not "
+             "serve it to clients. It was valid until %s local time and we "
+             "continued to serve it for up to 24 hours after it expired.%s",
+             flavor ? flavor : "", flavor ? " " : "", timestamp, dupes);
+    tor_free(dupes);
+  }
+}
+
+/**
+ * Parse a single hex-encoded sha3-256 digest from <b>hex</b> into
+ * <b>digest</b>. Return 0 on success.  On failure, report that the hash came
+ * from <b>location</b>, report that we are taking <b>action</b> with it, and
+ * return -1.
+ */
+static int
+parse_one_diff_hash(uint8_t *digest, const char *hex, const char *location,
+                    const char *action)
+{
+  if (base16_decode((char*)digest, DIGEST256_LEN, hex, strlen(hex)) ==
+      DIGEST256_LEN) {
+    return 0;
+  } else {
+    log_fn(LOG_PROTOCOL_WARN, LD_DIR,
+           "%s contained bogus digest %s; %s.",
+           location, escaped(hex), action);
+    return -1;
+  }
+}
+
+/** If there is an X-Or-Diff-From-Consensus header included in <b>headers</b>,
+ * set <b>digest_out<b> to a new smartlist containing every 256-bit
+ * hex-encoded digest listed in that header and return 0.  Otherwise return
+ * -1.  */
+static int
+parse_or_diff_from_header(smartlist_t **digests_out, const char *headers)
+{
+  char *hdr = http_get_header(headers, X_OR_DIFF_FROM_CONSENSUS_HEADER);
+  if (hdr == NULL) {
+    return -1;
+  }
+  smartlist_t *hex_digests = smartlist_new();
+  *digests_out = smartlist_new();
+  smartlist_split_string(hex_digests, hdr, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
+  SMARTLIST_FOREACH_BEGIN(hex_digests, const char *, hex) {
+    uint8_t digest[DIGEST256_LEN];
+    if (!parse_one_diff_hash(digest, hex, "X-Or-Diff-From-Consensus header",
+                             "ignoring")) {
+      smartlist_add(*digests_out, tor_memdup(digest, sizeof(digest)));
+    }
+  } SMARTLIST_FOREACH_END(hex);
+  SMARTLIST_FOREACH(hex_digests, char *, cp, tor_free(cp));
+  smartlist_free(hex_digests);
+  tor_free(hdr);
+  return 0;
+}
+
+/** Fallback compression method.  The fallback compression method is used in
+ * case a client requests a non-compressed document. We only store compressed
+ * documents, so we use this compression method to fetch the document and let
+ * the spooling system do the streaming decompression.
+ */
+#define FALLBACK_COMPRESS_METHOD ZLIB_METHOD
+
+/**
+ * Try to find the best consensus diff possible in order to serve a client
+ * request for a diff from one of the consensuses in <b>digests</b> to the
+ * current consensus of flavor <b>flav</b>.  The client supports the
+ * compression methods listed in the <b>compression_methods</b> bitfield:
+ * place the method chosen (if any) into <b>compression_used_out</b>.
+ */
+static struct consensus_cache_entry_t *
+find_best_diff(const smartlist_t *digests, int flav,
+               unsigned compression_methods,
+               compress_method_t *compression_used_out)
+{
+  struct consensus_cache_entry_t *result = NULL;
+
+  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
+    unsigned u;
+    for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
+      compress_method_t method = srv_meth_pref_precompressed[u];
+      if (0 == (compression_methods & (1u<<method)))
+        continue; // client doesn't like this one, or we don't have it.
+      if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256,
+                                     diff_from, DIGEST256_LEN,
+                                     method) == CONSDIFF_AVAILABLE) {
+        tor_assert_nonfatal(result);
+        *compression_used_out = method;
+        return result;
+      }
+    }
+  } SMARTLIST_FOREACH_END(diff_from);
+
+  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
+    if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, diff_from,
+          DIGEST256_LEN, FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
+      tor_assert_nonfatal(result);
+      *compression_used_out = FALLBACK_COMPRESS_METHOD;
+      return result;
+    }
+  } SMARTLIST_FOREACH_END(diff_from);
+
+  return NULL;
+}
+
+/** Lookup the cached consensus document by the flavor found in <b>flav</b>.
+ * The preferred set of compression methods should be listed in the
+ * <b>compression_methods</b> bitfield. The compression method chosen (if any)
+ * is stored in <b>compression_used_out</b>. */
+static struct consensus_cache_entry_t *
+find_best_consensus(int flav,
+                    unsigned compression_methods,
+                    compress_method_t *compression_used_out)
+{
+  struct consensus_cache_entry_t *result = NULL;
+  unsigned u;
+
+  for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
+    compress_method_t method = srv_meth_pref_precompressed[u];
+
+    if (0 == (compression_methods & (1u<<method)))
+      continue;
+
+    if (consdiffmgr_find_consensus(&result, flav,
+                                   method) == CONSDIFF_AVAILABLE) {
+      tor_assert_nonfatal(result);
+      *compression_used_out = method;
+      return result;
+    }
+  }
+
+  if (consdiffmgr_find_consensus(&result, flav,
+        FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
+    tor_assert_nonfatal(result);
+    *compression_used_out = FALLBACK_COMPRESS_METHOD;
+    return result;
+  }
+
+  return NULL;
+}
+
+/** Try to find the best supported compression method possible from a given
+ * <b>compression_methods</b>. Return NO_METHOD if no mutually supported
+ * compression method could be found. */
+static compress_method_t
+find_best_compression_method(unsigned compression_methods, int stream)
+{
+  unsigned u;
+  compress_method_t *methods;
+  size_t length;
+
+  if (stream) {
+    methods = srv_meth_pref_streaming_compression;
+    length = ARRAY_LENGTH(srv_meth_pref_streaming_compression);
+  } else {
+    methods = srv_meth_pref_precompressed;
+    length = ARRAY_LENGTH(srv_meth_pref_precompressed);
+  }
+
+  for (u = 0; u < length; ++u) {
+    compress_method_t method = methods[u];
+    if (compression_methods & (1u<<method))
+      return method;
+  }
+
+  return NO_METHOD;
+}
+
+/** Check if any of the digests in <b>digests</b> matches the latest consensus
+ *  flavor (given in <b>flavor</b>) that we have available. */
+static int
+digest_list_contains_best_consensus(consensus_flavor_t flavor,
+                                    const smartlist_t *digests)
+{
+  const networkstatus_t *ns = NULL;
+
+  if (digests == NULL)
+    return 0;
+
+  ns = networkstatus_get_latest_consensus_by_flavor(flavor);
+
+  if (ns == NULL)
+    return 0;
+
+  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, digest) {
+    if (tor_memeq(ns->digest_sha3_as_signed, digest, DIGEST256_LEN))
+      return 1;
+  } SMARTLIST_FOREACH_END(digest);
+
+  return 0;
+}
+
+/** Encodes the results of parsing a consensus request to figure out what
+ * consensus, and possibly what diffs, the user asked for. */
+typedef struct {
+  /** name of the flavor to retrieve. */
+  char *flavor;
+  /** flavor to retrive, as enum. */
+  consensus_flavor_t flav;
+  /** plus-separated list of authority fingerprints; see
+   * client_likes_consensus(). Aliases the URL in the request passed to
+   * parse_consensus_request(). */
+  const char *want_fps;
+  /** Optionally, a smartlist of sha3 digests-as-signed of the consensuses
+   * to return a diff from. */
+  smartlist_t *diff_from_digests;
+  /** If true, never send a full consensus. If there is no diff, send
+   * a 404 instead. */
+  int diff_only;
+} parsed_consensus_request_t;
+
+/** Remove all data held in <b>req</b>. Do not free <b>req</b> itself, since
+ * it is stack-allocated. */
+static void
+parsed_consensus_request_clear(parsed_consensus_request_t *req)
+{
+  if (!req)
+    return;
+  tor_free(req->flavor);
+  if (req->diff_from_digests) {
+    SMARTLIST_FOREACH(req->diff_from_digests, uint8_t *, d, tor_free(d));
+    smartlist_free(req->diff_from_digests);
+  }
+  memset(req, 0, sizeof(parsed_consensus_request_t));
+}
+
+/**
+ * Parse the URL and relevant headers of <b>args</b> for a current-consensus
+ * request to learn what flavor of consensus we want, what keys it must be
+ * signed with, and what diffs we would accept (or demand) instead. Return 0
+ * on success and -1 on failure.
+ */
+static int
+parse_consensus_request(parsed_consensus_request_t *out,
+                        const get_handler_args_t *args)
+{
+  const char *url = args->url;
+  memset(out, 0, sizeof(parsed_consensus_request_t));
+  out->flav = FLAV_NS;
+
+  const char CONSENSUS_URL_PREFIX[] = "/tor/status-vote/current/consensus/";
+  const char CONSENSUS_FLAVORED_PREFIX[] =
+    "/tor/status-vote/current/consensus-";
+
+  /* figure out the flavor if any, and who we wanted to sign the thing */
+  const char *after_flavor = NULL;
+
+  if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) {
+    const char *f, *cp;
+    f = url + strlen(CONSENSUS_FLAVORED_PREFIX);
+    cp = strchr(f, '/');
+    if (cp) {
+      after_flavor = cp+1;
+      out->flavor = tor_strndup(f, cp-f);
+    } else {
+      out->flavor = tor_strdup(f);
+    }
+    int flav = networkstatus_parse_flavor_name(out->flavor);
+    if (flav < 0)
+      flav = FLAV_NS;
+    out->flav = flav;
+  } else {
+    if (!strcmpstart(url, CONSENSUS_URL_PREFIX))
+      after_flavor = url+strlen(CONSENSUS_URL_PREFIX);
+  }
+
+  /* see whether we've been asked explicitly for a diff from an older
+   * consensus. (The user might also have said that a diff would be okay,
+   * via X-Or-Diff-From-Consensus */
+  const char DIFF_COMPONENT[] = "diff/";
+  char *diff_hash_in_url = NULL;
+  if (after_flavor && !strcmpstart(after_flavor, DIFF_COMPONENT)) {
+    after_flavor += strlen(DIFF_COMPONENT);
+    const char *cp = strchr(after_flavor, '/');
+    if (cp) {
+      diff_hash_in_url = tor_strndup(after_flavor, cp-after_flavor);
+      out->want_fps = cp+1;
+    } else {
+      diff_hash_in_url = tor_strdup(after_flavor);
+      out->want_fps = NULL;
+    }
+  } else {
+    out->want_fps = after_flavor;
+  }
+
+  if (diff_hash_in_url) {
+    uint8_t diff_from[DIGEST256_LEN];
+    out->diff_from_digests = smartlist_new();
+    out->diff_only = 1;
+    int ok = !parse_one_diff_hash(diff_from, diff_hash_in_url, "URL",
+                                  "rejecting");
+    tor_free(diff_hash_in_url);
+    if (ok) {
+      smartlist_add(out->diff_from_digests,
+                    tor_memdup(diff_from, DIGEST256_LEN));
+    } else {
+      return -1;
+    }
+  } else {
+    parse_or_diff_from_header(&out->diff_from_digests, args->headers);
+  }
+
+  return 0;
+}
+
+/** Helper function for GET /tor/status-vote/current/consensus
+ */
+static int
+handle_get_current_consensus(dir_connection_t *conn,
+                             const get_handler_args_t *args)
+{
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 0);
+  const time_t if_modified_since = args->if_modified_since;
+  int clear_spool = 0;
+
+  /* v3 network status fetch. */
+  long lifetime = NETWORKSTATUS_CACHE_LIFETIME;
+
+  time_t now = time(NULL);
+  parsed_consensus_request_t req;
+
+  if (parse_consensus_request(&req, args) < 0) {
+    write_short_http_response(conn, 404, "Couldn't parse request");
+    goto done;
+  }
+
+  if (digest_list_contains_best_consensus(req.flav,
+                                          req.diff_from_digests)) {
+    write_short_http_response(conn, 304, "Not modified");
+    geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED);
+    goto done;
+  }
+
+  struct consensus_cache_entry_t *cached_consensus = NULL;
+
+  compress_method_t compression_used = NO_METHOD;
+  if (req.diff_from_digests) {
+    cached_consensus = find_best_diff(req.diff_from_digests, req.flav,
+                                      args->compression_supported,
+                                      &compression_used);
+  }
+
+  if (req.diff_only && !cached_consensus) {
+    write_short_http_response(conn, 404, "No such diff available");
+    // XXXX warn_consensus_is_too_old(v, req.flavor, now);
+    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+    goto done;
+  }
+
+  if (! cached_consensus) {
+    cached_consensus = find_best_consensus(req.flav,
+                                           args->compression_supported,
+                                           &compression_used);
+  }
+
+  time_t fresh_until, valid_until;
+  int have_fresh_until = 0, have_valid_until = 0;
+  if (cached_consensus) {
+    have_fresh_until =
+      !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until);
+    have_valid_until =
+      !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until);
+  }
+
+  if (cached_consensus && have_valid_until &&
+      !networkstatus_valid_until_is_reasonably_live(valid_until, now)) {
+    write_short_http_response(conn, 404, "Consensus is too old");
+    warn_consensus_is_too_old(cached_consensus, req.flavor, now);
+    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+    goto done;
+  }
+
+  if (cached_consensus && req.want_fps &&
+      !client_likes_consensus(cached_consensus, req.want_fps)) {
+    write_short_http_response(conn, 404, "Consensus not signed by sufficient "
+                           "number of requested authorities");
+    geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS);
+    goto done;
+  }
+
+  conn->spool = smartlist_new();
+  clear_spool = 1;
+  {
+    spooled_resource_t *spooled;
+    if (cached_consensus) {
+      spooled = spooled_resource_new_from_cache_entry(cached_consensus);
+      smartlist_add(conn->spool, spooled);
+    }
+  }
+
+  lifetime = (have_fresh_until && fresh_until > now) ? fresh_until - now : 0;
+
+  size_t size_guess = 0;
+  int n_expired = 0;
+  dirserv_spool_remove_missing_and_guess_size(conn, if_modified_since,
+                                              compress_method != NO_METHOD,
+                                              &size_guess,
+                                              &n_expired);
+
+  if (!smartlist_len(conn->spool) && !n_expired) {
+    write_short_http_response(conn, 404, "Not found");
+    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+    goto done;
+  } else if (!smartlist_len(conn->spool)) {
+    write_short_http_response(conn, 304, "Not modified");
+    geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED);
+    goto done;
+  }
+
+  if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+    log_debug(LD_DIRSERV,
+              "Client asked for network status lists, but we've been "
+              "writing too many bytes lately. Sending 503 Dir busy.");
+    write_short_http_response(conn, 503, "Directory busy, try again later");
+    geoip_note_ns_response(GEOIP_REJECT_BUSY);
+    goto done;
+  }
+
+  tor_addr_t addr;
+  if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) {
+    geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS,
+                           &addr, NULL,
+                           time(NULL));
+    geoip_note_ns_response(GEOIP_SUCCESS);
+    /* Note that a request for a network status has started, so that we
+     * can measure the download time later on. */
+    if (conn->dirreq_id)
+      geoip_start_dirreq(conn->dirreq_id, size_guess, DIRREQ_TUNNELED);
+    else
+      geoip_start_dirreq(TO_CONN(conn)->global_identifier, size_guess,
+                         DIRREQ_DIRECT);
+  }
+
+  /* Use this header to tell caches that the response depends on the
+   * X-Or-Diff-From-Consensus header (or lack thereof). */
+  const char vary_header[] = "Vary: X-Or-Diff-From-Consensus\r\n";
+
+  clear_spool = 0;
+
+  // The compress_method might have been NO_METHOD, but we store the data
+  // compressed. Decompress them using `compression_used`. See fallback code in
+  // find_best_consensus() and find_best_diff().
+  write_http_response_headers(conn, -1,
+                             compress_method == NO_METHOD ?
+                               NO_METHOD : compression_used,
+                             vary_header,
+                             smartlist_len(conn->spool) == 1 ? lifetime : 0);
+
+  if (compress_method == NO_METHOD && smartlist_len(conn->spool))
+    conn->compress_state = tor_compress_new(0, compression_used,
+                                            HIGH_COMPRESSION);
+
+  /* Prime the connection with some data. */
+  const int initial_flush_result = connection_dirserv_flushed_some(conn);
+  tor_assert_nonfatal(initial_flush_result == 0);
+  goto done;
+
+ done:
+  parsed_consensus_request_clear(&req);
+  if (clear_spool) {
+    dir_conn_clear_spool(conn);
+  }
+  return 0;
+}
+
+/** Helper function for GET /tor/status-vote/{current,next}/...
+ */
+static int
+handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
+{
+  const char *url = args->url;
+  {
+    ssize_t body_len = 0;
+    ssize_t estimated_len = 0;
+    int lifetime = 60; /* XXXX?? should actually use vote intervals. */
+    /* This smartlist holds strings that we can compress on the fly. */
+    smartlist_t *items = smartlist_new();
+    /* This smartlist holds cached_dir_t objects that have a precompressed
+     * deflated version. */
+    smartlist_t *dir_items = smartlist_new();
+    dirvote_dirreq_get_status_vote(url, items, dir_items);
+    if (!smartlist_len(dir_items) && !smartlist_len(items)) {
+      write_short_http_response(conn, 404, "Not found");
+      goto vote_done;
+    }
+
+    /* We're sending items from at most one kind of source */
+    tor_assert_nonfatal(smartlist_len(items) == 0 ||
+                        smartlist_len(dir_items) == 0);
+
+    int streaming;
+    unsigned mask;
+    if (smartlist_len(items)) {
+      /* We're taking strings and compressing them on the fly. */
+      streaming = 1;
+      mask = ~0u;
+    } else {
+      /* We're taking cached_dir_t objects. We only have them uncompressed
+       * or deflated. */
+      streaming = 0;
+      mask = (1u<<NO_METHOD) | (1u<<ZLIB_METHOD);
+    }
+    const compress_method_t compress_method = find_best_compression_method(
+                              args->compression_supported&mask, streaming);
+
+    SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
+                      body_len += compress_method != NO_METHOD ?
+                        d->dir_compressed_len : d->dir_len);
+    estimated_len += body_len;
+    SMARTLIST_FOREACH(items, const char *, item, {
+        size_t ln = strlen(item);
+        if (compress_method != NO_METHOD) {
+          estimated_len += ln/2;
+        } else {
+          body_len += ln; estimated_len += ln;
+        }
+      });
+
+    if (global_write_bucket_low(TO_CONN(conn), estimated_len, 2)) {
+      write_short_http_response(conn, 503, "Directory busy, try again later");
+      goto vote_done;
+    }
+    write_http_response_header(conn, body_len ? body_len : -1,
+                 compress_method,
+                 lifetime);
+
+    if (smartlist_len(items)) {
+      if (compress_method != NO_METHOD) {
+        conn->compress_state = tor_compress_new(1, compress_method,
+                           choose_compression_level(estimated_len));
+        SMARTLIST_FOREACH(items, const char *, c,
+                 connection_buf_add_compress(c, strlen(c), conn, 0));
+        connection_buf_add_compress("", 0, conn, 1);
+      } else {
+        SMARTLIST_FOREACH(items, const char *, c,
+                         connection_buf_add(c, strlen(c), TO_CONN(conn)));
+      }
+    } else {
+      SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
+          connection_buf_add(compress_method != NO_METHOD ?
+                                    d->dir_compressed : d->dir,
+                                  compress_method != NO_METHOD ?
+                                    d->dir_compressed_len : d->dir_len,
+                                  TO_CONN(conn)));
+    }
+  vote_done:
+    smartlist_free(items);
+    smartlist_free(dir_items);
+    goto done;
+  }
+ done:
+  return 0;
+}
+
+/** Helper function for GET /tor/micro/d/...
+ */
+static int
+handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args)
+{
+  const char *url = args->url;
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 1);
+  int clear_spool = 1;
+  {
+    conn->spool = smartlist_new();
+
+    dir_split_resource_into_spoolable(url+strlen("/tor/micro/d/"),
+                                      DIR_SPOOL_MICRODESC,
+                                      conn->spool, NULL,
+                                      DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ);
+
+    size_t size_guess = 0;
+    dirserv_spool_remove_missing_and_guess_size(conn, 0,
+                                                compress_method != NO_METHOD,
+                                                &size_guess, NULL);
+    if (smartlist_len(conn->spool) == 0) {
+      write_short_http_response(conn, 404, "Not found");
+      goto done;
+    }
+    if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+      log_info(LD_DIRSERV,
+               "Client asked for server descriptors, but we've been "
+               "writing too many bytes lately. Sending 503 Dir busy.");
+      write_short_http_response(conn, 503, "Directory busy, try again later");
+      goto done;
+    }
+
+    clear_spool = 0;
+    write_http_response_header(conn, -1,
+                               compress_method,
+                               MICRODESC_CACHE_LIFETIME);
+
+    if (compress_method != NO_METHOD)
+      conn->compress_state = tor_compress_new(1, compress_method,
+                                      choose_compression_level(size_guess));
+
+    const int initial_flush_result = connection_dirserv_flushed_some(conn);
+    tor_assert_nonfatal(initial_flush_result == 0);
+    goto done;
+  }
+
+ done:
+  if (clear_spool) {
+    dir_conn_clear_spool(conn);
+  }
+  return 0;
+}
+
+/** Helper function for GET /tor/{server,extra}/...
+ */
+static int
+handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args)
+{
+  const char *url = args->url;
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 1);
+  const or_options_t *options = get_options();
+  int clear_spool = 1;
+  if (!strcmpstart(url,"/tor/server/") ||
+      (!options->BridgeAuthoritativeDir &&
+       !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) {
+    int res;
+    const char *msg = NULL;
+    int cache_lifetime = 0;
+    int is_extra = !strcmpstart(url,"/tor/extra/");
+    url += is_extra ? strlen("/tor/extra/") : strlen("/tor/server/");
+    dir_spool_source_t source;
+    time_t publish_cutoff = 0;
+    if (!strcmpstart(url, "d/")) {
+      source =
+        is_extra ? DIR_SPOOL_EXTRA_BY_DIGEST : DIR_SPOOL_SERVER_BY_DIGEST;
+    } else {
+      source =
+        is_extra ? DIR_SPOOL_EXTRA_BY_FP : DIR_SPOOL_SERVER_BY_FP;
+      /* We only want to apply a publish cutoff when we're requesting
+       * resources by fingerprint. */
+      publish_cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH;
+    }
+
+    conn->spool = smartlist_new();
+    res = dirserv_get_routerdesc_spool(conn->spool, url,
+                                       source,
+                                       connection_dir_is_encrypted(conn),
+                                       &msg);
+
+    if (!strcmpstart(url, "all")) {
+      cache_lifetime = FULL_DIR_CACHE_LIFETIME;
+    } else if (smartlist_len(conn->spool) == 1) {
+      cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME;
+    }
+
+    size_t size_guess = 0;
+    int n_expired = 0;
+    dirserv_spool_remove_missing_and_guess_size(conn, publish_cutoff,
+                                                compress_method != NO_METHOD,
+                                                &size_guess, &n_expired);
+
+    /* If we are the bridge authority and the descriptor is a bridge
+     * descriptor, remember that we served this descriptor for desc stats. */
+    /* XXXX it's a bit of a kludge to have this here. */
+    if (get_options()->BridgeAuthoritativeDir &&
+        source == DIR_SPOOL_SERVER_BY_FP) {
+      SMARTLIST_FOREACH_BEGIN(conn->spool, spooled_resource_t *, spooled) {
+        const routerinfo_t *router =
+          router_get_by_id_digest((const char *)spooled->digest);
+        /* router can be NULL here when the bridge auth is asked for its own
+         * descriptor. */
+        if (router && router->purpose == ROUTER_PURPOSE_BRIDGE)
+          rep_hist_note_desc_served(router->cache_info.identity_digest);
+      } SMARTLIST_FOREACH_END(spooled);
+    }
+
+    if (res < 0 || size_guess == 0 || smartlist_len(conn->spool) == 0) {
+      if (msg == NULL)
+        msg = "Not found";
+      write_short_http_response(conn, 404, msg);
+    } else {
+      if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+        log_info(LD_DIRSERV,
+                 "Client asked for server descriptors, but we've been "
+                 "writing too many bytes lately. Sending 503 Dir busy.");
+        write_short_http_response(conn, 503,
+                                  "Directory busy, try again later");
+        dir_conn_clear_spool(conn);
+        goto done;
+      }
+      write_http_response_header(conn, -1, compress_method, cache_lifetime);
+      if (compress_method != NO_METHOD)
+        conn->compress_state = tor_compress_new(1, compress_method,
+                                        choose_compression_level(size_guess));
+      clear_spool = 0;
+      /* Prime the connection with some data. */
+      int initial_flush_result = connection_dirserv_flushed_some(conn);
+      tor_assert_nonfatal(initial_flush_result == 0);
+    }
+    goto done;
+  }
+ done:
+  if (clear_spool)
+    dir_conn_clear_spool(conn);
+  return 0;
+}
+
+/** Helper function for GET /tor/keys/...
+ */
+static int
+handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
+{
+  const char *url = args->url;
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 1);
+  const time_t if_modified_since = args->if_modified_since;
+  {
+    smartlist_t *certs = smartlist_new();
+    ssize_t len = -1;
+    if (!strcmp(url, "/tor/keys/all")) {
+      authority_cert_get_all(certs);
+    } else if (!strcmp(url, "/tor/keys/authority")) {
+      authority_cert_t *cert = get_my_v3_authority_cert();
+      if (cert)
+        smartlist_add(certs, cert);
+    } else if (!strcmpstart(url, "/tor/keys/fp/")) {
+      smartlist_t *fps = smartlist_new();
+      dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"),
+                                           fps, NULL,
+                                           DSR_HEX|DSR_SORT_UNIQ);
+      SMARTLIST_FOREACH(fps, char *, d, {
+          authority_cert_t *c = authority_cert_get_newest_by_id(d);
+          if (c) smartlist_add(certs, c);
+          tor_free(d);
+      });
+      smartlist_free(fps);
+    } else if (!strcmpstart(url, "/tor/keys/sk/")) {
+      smartlist_t *fps = smartlist_new();
+      dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"),
+                                           fps, NULL,
+                                           DSR_HEX|DSR_SORT_UNIQ);
+      SMARTLIST_FOREACH(fps, char *, d, {
+          authority_cert_t *c = authority_cert_get_by_sk_digest(d);
+          if (c) smartlist_add(certs, c);
+          tor_free(d);
+      });
+      smartlist_free(fps);
+    } else if (!strcmpstart(url, "/tor/keys/fp-sk/")) {
+      smartlist_t *fp_sks = smartlist_new();
+      dir_split_resource_into_fingerprint_pairs(url+strlen("/tor/keys/fp-sk/"),
+                                                fp_sks);
+      SMARTLIST_FOREACH(fp_sks, fp_pair_t *, pair, {
+          authority_cert_t *c = authority_cert_get_by_digests(pair->first,
+                                                              pair->second);
+          if (c) smartlist_add(certs, c);
+          tor_free(pair);
+      });
+      smartlist_free(fp_sks);
+    } else {
+      write_short_http_response(conn, 400, "Bad request");
+      goto keys_done;
+    }
+    if (!smartlist_len(certs)) {
+      write_short_http_response(conn, 404, "Not found");
+      goto keys_done;
+    }
+    SMARTLIST_FOREACH(certs, authority_cert_t *, c,
+      if (c->cache_info.published_on < if_modified_since)
+        SMARTLIST_DEL_CURRENT(certs, c));
+    if (!smartlist_len(certs)) {
+      write_short_http_response(conn, 304, "Not modified");
+      goto keys_done;
+    }
+    len = 0;
+    SMARTLIST_FOREACH(certs, authority_cert_t *, c,
+                      len += c->cache_info.signed_descriptor_len);
+
+    if (global_write_bucket_low(TO_CONN(conn),
+                                compress_method != NO_METHOD ? len/2 : len,
+                                2)) {
+      write_short_http_response(conn, 503, "Directory busy, try again later");
+      goto keys_done;
+    }
+
+    write_http_response_header(conn,
+                               compress_method != NO_METHOD ? -1 : len,
+                               compress_method,
+                               60*60);
+    if (compress_method != NO_METHOD) {
+      conn->compress_state = tor_compress_new(1, compress_method,
+                                              choose_compression_level(len));
+      SMARTLIST_FOREACH(certs, authority_cert_t *, c,
+            connection_buf_add_compress(
+                c->cache_info.signed_descriptor_body,
+                c->cache_info.signed_descriptor_len,
+                conn, 0));
+      connection_buf_add_compress("", 0, conn, 1);
+    } else {
+      SMARTLIST_FOREACH(certs, authority_cert_t *, c,
+            connection_buf_add(c->cache_info.signed_descriptor_body,
+                                    c->cache_info.signed_descriptor_len,
+                                    TO_CONN(conn)));
+    }
+  keys_done:
+    smartlist_free(certs);
+    goto done;
+  }
+ done:
+  return 0;
+}
+
+/** Helper function for GET /tor/rendezvous2/
+ */
+static int
+handle_get_hs_descriptor_v2(dir_connection_t *conn,
+                            const get_handler_args_t *args)
+{
+  const char *url = args->url;
+  if (connection_dir_is_encrypted(conn)) {
+    /* Handle v2 rendezvous descriptor fetch request. */
+    const char *descp;
+    const char *query = url + strlen("/tor/rendezvous2/");
+    if (rend_valid_descriptor_id(query)) {
+      log_info(LD_REND, "Got a v2 rendezvous descriptor request for ID '%s'",
+               safe_str(escaped(query)));
+      switch (rend_cache_lookup_v2_desc_as_dir(query, &descp)) {
+        case 1: /* valid */
+          write_http_response_header(conn, strlen(descp), NO_METHOD, 0);
+          connection_buf_add(descp, strlen(descp), TO_CONN(conn));
+          break;
+        case 0: /* well-formed but not present */
+          write_short_http_response(conn, 404, "Not found");
+          break;
+        case -1: /* not well-formed */
+          write_short_http_response(conn, 400, "Bad request");
+          break;
+      }
+    } else { /* not well-formed */
+      write_short_http_response(conn, 400, "Bad request");
+    }
+    goto done;
+  } else {
+    /* Not encrypted! */
+    write_short_http_response(conn, 404, "Not found");
+  }
+ done:
+  return 0;
+}
+
+/** Helper function for GET /tor/hs/3/<z>. Only for version 3.
+ */
+STATIC int
+handle_get_hs_descriptor_v3(dir_connection_t *conn,
+                            const get_handler_args_t *args)
+{
+  int retval;
+  const char *desc_str = NULL;
+  const char *pubkey_str = NULL;
+  const char *url = args->url;
+
+  /* Reject unencrypted dir connections */
+  if (!connection_dir_is_encrypted(conn)) {
+    write_short_http_response(conn, 404, "Not found");
+    goto done;
+  }
+
+  /* After the path prefix follows the base64 encoded blinded pubkey which we
+   * use to get the descriptor from the cache. Skip the prefix and get the
+   * pubkey. */
+  tor_assert(!strcmpstart(url, "/tor/hs/3/"));
+  pubkey_str = url + strlen("/tor/hs/3/");
+  retval = hs_cache_lookup_as_dir(HS_VERSION_THREE,
+                                  pubkey_str, &desc_str);
+  if (retval <= 0 || desc_str == NULL) {
+    write_short_http_response(conn, 404, "Not found");
+    goto done;
+  }
+
+  /* Found requested descriptor! Pass it to this nice client. */
+  write_http_response_header(conn, strlen(desc_str), NO_METHOD, 0);
+  connection_buf_add(desc_str, strlen(desc_str), TO_CONN(conn));
+
+ done:
+  return 0;
+}
+
+/** Helper function for GET /tor/networkstatus-bridges
+ */
+static int
+handle_get_networkstatus_bridges(dir_connection_t *conn,
+                                 const get_handler_args_t *args)
+{
+  const char *headers = args->headers;
+
+  const or_options_t *options = get_options();
+  if (options->BridgeAuthoritativeDir &&
+      options->BridgePassword_AuthDigest_ &&
+      connection_dir_is_encrypted(conn)) {
+    char *status;
+    char digest[DIGEST256_LEN];
+
+    char *header = http_get_header(headers, "Authorization: Basic ");
+    if (header)
+      crypto_digest256(digest, header, strlen(header), DIGEST_SHA256);
+
+    /* now make sure the password is there and right */
+    if (!header ||
+        tor_memneq(digest,
+                   options->BridgePassword_AuthDigest_, DIGEST256_LEN)) {
+      write_short_http_response(conn, 404, "Not found");
+      tor_free(header);
+      goto done;
+    }
+    tor_free(header);
+
+    /* all happy now. send an answer. */
+    status = networkstatus_getinfo_by_purpose("bridge", time(NULL));
+    size_t dlen = strlen(status);
+    write_http_response_header(conn, dlen, NO_METHOD, 0);
+    connection_buf_add(status, dlen, TO_CONN(conn));
+    tor_free(status);
+    goto done;
+  }
+ done:
+  return 0;
+}
+
+/** Helper function for GET robots.txt or /tor/robots.txt */
+static int
+handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args)
+{
+  (void)args;
+  {
+    const char robots[] = "User-agent: *\r\nDisallow: /\r\n";
+    size_t len = strlen(robots);
+    write_http_response_header(conn, len, NO_METHOD, ROBOTS_CACHE_LIFETIME);
+    connection_buf_add(robots, len, TO_CONN(conn));
+  }
+  return 0;
+}
+
+/* Given the <b>url</b> from a POST request, try to extract the version number
+ * using the provided <b>prefix</b>. The version should be after the prefix and
+ * ending with the separator "/". For instance:
+ *      /tor/hs/3/publish
+ *
+ * On success, <b>end_pos</b> points to the position right after the version
+ * was found. On error, it is set to NULL.
+ *
+ * Return version on success else negative value. */
+STATIC int
+parse_hs_version_from_post(const char *url, const char *prefix,
+                           const char **end_pos)
+{
+  int ok;
+  unsigned long version;
+  const char *start;
+  char *end = NULL;
+
+  tor_assert(url);
+  tor_assert(prefix);
+  tor_assert(end_pos);
+
+  /* Check if the prefix does start the url. */
+  if (strcmpstart(url, prefix)) {
+    goto err;
+  }
+  /* Move pointer to the end of the prefix string. */
+  start = url + strlen(prefix);
+  /* Try this to be the HS version and if we are still at the separator, next
+   * will be move to the right value. */
+  version = tor_parse_long(start, 10, 0, INT_MAX, &ok, &end);
+  if (!ok) {
+    goto err;
+  }
+
+  *end_pos = end;
+  return (int) version;
+ err:
+  *end_pos = NULL;
+  return -1;
+}
+
+/* Handle the POST request for a hidden service descripror. The request is in
+ * <b>url</b>, the body of the request is in <b>body</b>. Return 200 on success
+ * else return 400 indicating a bad request. */
+STATIC int
+handle_post_hs_descriptor(const char *url, const char *body)
+{
+  int version;
+  const char *end_pos;
+
+  tor_assert(url);
+  tor_assert(body);
+
+  version = parse_hs_version_from_post(url, "/tor/hs/", &end_pos);
+  if (version < 0) {
+    goto err;
+  }
+
+  /* We have a valid version number, now make sure it's a publish request. Use
+   * the end position just after the version and check for the command. */
+  if (strcmpstart(end_pos, "/publish")) {
+    goto err;
+  }
+
+  switch (version) {
+  case HS_VERSION_THREE:
+    if (hs_cache_store_as_dir(body) < 0) {
+      goto err;
+    }
+    log_info(LD_REND, "Publish request for HS descriptor handled "
+                      "successfully.");
+    break;
+  default:
+    /* Unsupported version, return a bad request. */
+    goto err;
+  }
+
+  return 200;
+ err:
+  /* Bad request. */
+  return 400;
+}
+
+/** Helper function: called when a dirserver gets a complete HTTP POST
+ * request.  Look for an uploaded server descriptor or rendezvous
+ * service descriptor.  On finding one, process it and write a
+ * response into conn-\>outbuf.  If the request is unrecognized, send a
+ * 400.  Always return 0. */
+MOCK_IMPL(STATIC int,
+directory_handle_command_post,(dir_connection_t *conn, const char *headers,
+                               const char *body, size_t body_len))
+{
+  char *url = NULL;
+  const or_options_t *options = get_options();
+
+  log_debug(LD_DIRSERV,"Received POST command.");
+
+  conn->base_.state = DIR_CONN_STATE_SERVER_WRITING;
+
+  if (!public_server_mode(options)) {
+    log_info(LD_DIR, "Rejected dir post request from %s "
+             "since we're not a public relay.", conn->base_.address);
+    write_short_http_response(conn, 503, "Not acting as a public relay");
+    goto done;
+  }
+
+  if (parse_http_url(headers, &url) < 0) {
+    write_short_http_response(conn, 400, "Bad request");
+    return 0;
+  }
+  log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url));
+
+  /* Handle v2 rendezvous service publish request. */
+  if (connection_dir_is_encrypted(conn) &&
+      !strcmpstart(url,"/tor/rendezvous2/publish")) {
+    if (rend_cache_store_v2_desc_as_dir(body) < 0) {
+      log_warn(LD_REND, "Rejected v2 rend descriptor (body size %d) from %s.",
+               (int)body_len, conn->base_.address);
+      write_short_http_response(conn, 400,
+                             "Invalid v2 service descriptor rejected");
+    } else {
+      write_short_http_response(conn, 200, "Service descriptor (v2) stored");
+      log_info(LD_REND, "Handled v2 rendezvous descriptor post: accepted");
+    }
+    goto done;
+  }
+
+  /* Handle HS descriptor publish request. */
+  /* XXX: This should be disabled with a consensus param until we want to
+   * the prop224 be deployed and thus use. */
+  if (connection_dir_is_encrypted(conn) && !strcmpstart(url, "/tor/hs/")) {
+    const char *msg = "HS descriptor stored successfully.";
+
+    /* We most probably have a publish request for an HS descriptor. */
+    int code = handle_post_hs_descriptor(url, body);
+    if (code != 200) {
+      msg = "Invalid HS descriptor. Rejected.";
+    }
+    write_short_http_response(conn, code, msg);
+    goto done;
+  }
+
+  if (!authdir_mode(options)) {
+    /* we just provide cached directories; we don't want to
+     * receive anything. */
+    write_short_http_response(conn, 400, "Nonauthoritative directory does not "
+                           "accept posted server descriptors");
+    goto done;
+  }
+
+  if (authdir_mode(options) &&
+      !strcmp(url,"/tor/")) { /* server descriptor post */
+    const char *msg = "[None]";
+    uint8_t purpose = authdir_mode_bridge(options) ?
+                      ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL;
+    was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose,
+                                             conn->base_.address, &msg);
+    tor_assert(msg);
+
+    if (r == ROUTER_ADDED_SUCCESSFULLY) {
+      write_short_http_response(conn, 200, msg);
+    } else if (WRA_WAS_OUTDATED(r)) {
+      write_http_response_header_impl(conn, -1, NULL, NULL,
+                                      "X-Descriptor-Not-New: Yes\r\n", -1);
+    } else {
+      log_info(LD_DIRSERV,
+               "Rejected router descriptor or extra-info from %s "
+               "(\"%s\").",
+               conn->base_.address, msg);
+      write_short_http_response(conn, 400, msg);
+    }
+    goto done;
+  }
+
+  if (authdir_mode_v3(options) &&
+      !strcmp(url,"/tor/post/vote")) { /* v3 networkstatus vote */
+    const char *msg = "OK";
+    int status;
+    if (dirvote_add_vote(body, &msg, &status)) {
+      write_short_http_response(conn, status, "Vote stored");
+    } else {
+      tor_assert(msg);
+      log_warn(LD_DIRSERV, "Rejected vote from %s (\"%s\").",
+               conn->base_.address, msg);
+      write_short_http_response(conn, status, msg);
+    }
+    goto done;
+  }
+
+  if (authdir_mode_v3(options) &&
+      !strcmp(url,"/tor/post/consensus-signature")) { /* sigs on consensus. */
+    const char *msg = NULL;
+    if (dirvote_add_signatures(body, conn->base_.address, &msg)>=0) {
+      write_short_http_response(conn, 200, msg?msg:"Signatures stored");
+    } else {
+      log_warn(LD_DIR, "Unable to store signatures posted by %s: %s",
+               conn->base_.address, msg?msg:"???");
+      write_short_http_response(conn, 400,
+                                msg?msg:"Unable to store signatures");
+    }
+    goto done;
+  }
+
+  /* we didn't recognize the url */
+  write_short_http_response(conn, 404, "Not found");
+
+ done:
+  tor_free(url);
+  return 0;
+}
+
+/** If <b>headers</b> indicates that a proxy was involved, then rewrite
+ * <b>conn</b>-\>address to describe our best guess of the address that
+ * originated this HTTP request. */
+static void
+http_set_address_origin(const char *headers, connection_t *conn)
+{
+  char *fwd;
+
+  fwd = http_get_header(headers, "Forwarded-For: ");
+  if (!fwd)
+    fwd = http_get_header(headers, "X-Forwarded-For: ");
+  if (fwd) {
+    tor_addr_t toraddr;
+    if (tor_addr_parse(&toraddr,fwd) == -1 ||
+        tor_addr_is_internal(&toraddr,0)) {
+      log_debug(LD_DIR, "Ignoring local/internal IP %s", escaped(fwd));
+      tor_free(fwd);
+      return;
+    }
+
+    tor_free(conn->address);
+    conn->address = tor_strdup(fwd);
+    tor_free(fwd);
+  }
+}
+
+/** Called when a dirserver receives data on a directory connection;
+ * looks for an HTTP request.  If the request is complete, remove it
+ * from the inbuf, try to process it; otherwise, leave it on the
+ * buffer.  Return a 0 on success, or -1 on error.
+ */
+int
+directory_handle_command(dir_connection_t *conn)
+{
+  char *headers=NULL, *body=NULL;
+  size_t body_len=0;
+  int r;
+
+  tor_assert(conn);
+  tor_assert(conn->base_.type == CONN_TYPE_DIR);
+
+  switch (connection_fetch_from_buf_http(TO_CONN(conn),
+                              &headers, MAX_HEADERS_SIZE,
+                              &body, &body_len, MAX_DIR_UL_SIZE, 0)) {
+    case -1: /* overflow */
+      log_warn(LD_DIRSERV,
+               "Request too large from address '%s' to DirPort. Closing.",
+               safe_str(conn->base_.address));
+      return -1;
+    case 0:
+      log_debug(LD_DIRSERV,"command not all here yet.");
+      return 0;
+    /* case 1, fall through */
+  }
+
+  http_set_address_origin(headers, TO_CONN(conn));
+  // we should escape headers here as well,
+  // but we can't call escaped() twice, as it uses the same buffer
+  //log_debug(LD_DIRSERV,"headers %s, body %s.", headers, escaped(body));
+
+  if (!strncasecmp(headers,"GET",3))
+    r = directory_handle_command_get(conn, headers, body, body_len);
+  else if (!strncasecmp(headers,"POST",4))
+    r = directory_handle_command_post(conn, headers, body, body_len);
+  else {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Got headers %s with unknown command. Closing.",
+           escaped(headers));
+    r = -1;
+  }
+
+  tor_free(headers); tor_free(body);
+  return r;
+}
diff --git a/src/feature/dircache/dircache.h b/src/feature/dircache/dircache.h
new file mode 100644
index 0000000000..236ea649ef
--- /dev/null
+++ b/src/feature/dircache/dircache.h
@@ -0,0 +1,43 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dircache.h
+ * \brief Header file for dircache.c.
+ **/
+
+#ifndef TOR_DIRCACHE_H
+#define TOR_DIRCACHE_H
+
+int directory_handle_command(dir_connection_t *conn);
+
+#ifdef DIRCACHE_PRIVATE
+MOCK_DECL(STATIC int, directory_handle_command_get,(dir_connection_t *conn,
+                                                    const char *headers,
+                                                    const char *req_body,
+                                                    size_t req_body_len));
+MOCK_DECL(STATIC int, directory_handle_command_post,(dir_connection_t *conn,
+                                                     const char *headers,
+                                                     const char *body,
+                                                     size_t body_len));
+
+STATIC int handle_post_hs_descriptor(const char *url, const char *body);
+enum compression_level_t;
+STATIC enum compression_level_t choose_compression_level(ssize_t n_bytes);
+
+struct get_handler_args_t;
+STATIC int handle_get_hs_descriptor_v3(dir_connection_t *conn,
+                                       const struct get_handler_args_t *args);
+
+STATIC int parse_http_url(const char *headers, char **url);
+
+STATIC int parse_hs_version_from_post(const char *url, const char *prefix,
+                                      const char **end_pos);
+
+STATIC unsigned parse_accept_encoding_header(const char *h);
+#endif
+
+#endif /* !defined(TOR_DIRCACHE_H) */
diff --git a/src/feature/dircache/dirserv.c b/src/feature/dircache/dirserv.c
new file mode 100644
index 0000000000..213c490314
--- /dev/null
+++ b/src/feature/dircache/dirserv.c
@@ -0,0 +1,918 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DIRSERV_PRIVATE
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "feature/dircache/conscache.h"
+#include "feature/dircache/consdiffmgr.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+#include "feature/stats/predict_ports.h"
+
+#include "feature/dircache/cached_dir_st.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/nodelist/extrainfo_st.h"
+#include "feature/nodelist/microdesc_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerlist_st.h"
+
+#include "lib/compress/compress.h"
+
+/**
+ * \file dirserv.c
+ * \brief Directory server core implementation. Manages directory
+ * contents and generates directory documents.
+ *
+ * This module implements most of directory cache functionality, and some of
+ * the directory authority functionality.  The directory.c module delegates
+ * here in order to handle incoming requests from clients, via
+ * connection_dirserv_flushed_some() and its kin.  In order to save RAM, this
+ * module is responsible for spooling directory objects (in whole or in part)
+ * onto buf_t instances, and then closing the dir_connection_t once the
+ * objects are totally flushed.
+ *
+ * The directory.c module also delegates here for handling descriptor uploads
+ * via dirserv_add_multiple_descriptors().
+ *
+ * Additionally, this module handles some aspects of voting, including:
+ * deciding how to vote on individual flags (based on decisions reached in
+ * rephist.c), of formatting routerstatus lines, and deciding what relays to
+ * include in an authority's vote.  (TODO: Those functions could profitably be
+ * split off.  They only live in this file because historically they were
+ * shared among the v1, v2, and v3 directory code.)
+ */
+
+static void clear_cached_dir(cached_dir_t *d);
+static const signed_descriptor_t *get_signed_descriptor_by_fp(
+                                                        const uint8_t *fp,
+                                                        int extrainfo);
+
+static int spooled_resource_lookup_body(const spooled_resource_t *spooled,
+                                        int conn_is_encrypted,
+                                        const uint8_t **body_out,
+                                        size_t *size_out,
+                                        time_t *published_out);
+static cached_dir_t *spooled_resource_lookup_cached_dir(
+                                   const spooled_resource_t *spooled,
+                                   time_t *published_out);
+static cached_dir_t *lookup_cached_dir_by_fp(const uint8_t *fp);
+
+/********************************************************************/
+
+/* A set of functions to answer questions about how we'd like to behave
+ * as a directory mirror/client. */
+
+/** Return 1 if we fetch our directory material directly from the
+ * authorities, rather than from a mirror. */
+int
+directory_fetches_from_authorities(const or_options_t *options)
+{
+  const routerinfo_t *me;
+  uint32_t addr;
+  int refuseunknown;
+  if (options->FetchDirInfoEarly)
+    return 1;
+  if (options->BridgeRelay == 1)
+    return 0;
+  if (server_mode(options) &&
+      router_pick_published_address(options, &addr, 1) < 0)
+    return 1; /* we don't know our IP address; ask an authority. */
+  refuseunknown = ! router_my_exit_policy_is_reject_star() &&
+    should_refuse_unknown_exits(options);
+  if (!dir_server_mode(options) && !refuseunknown)
+    return 0;
+  if (!server_mode(options) || !advertised_server_mode())
+    return 0;
+  me = router_get_my_routerinfo();
+  if (!me || (!me->supports_tunnelled_dir_requests && !refuseunknown))
+    return 0; /* if we don't service directory requests, return 0 too */
+  return 1;
+}
+
+/** Return 1 if we should fetch new networkstatuses, descriptors, etc
+ * on the "mirror" schedule rather than the "client" schedule.
+ */
+int
+directory_fetches_dir_info_early(const or_options_t *options)
+{
+  return directory_fetches_from_authorities(options);
+}
+
+/** Return 1 if we should fetch new networkstatuses, descriptors, etc
+ * on a very passive schedule -- waiting long enough for ordinary clients
+ * to probably have the info we want. These would include bridge users,
+ * and maybe others in the future e.g. if a Tor client uses another Tor
+ * client as a directory guard.
+ */
+int
+directory_fetches_dir_info_later(const or_options_t *options)
+{
+  return options->UseBridges != 0;
+}
+
+/** Return true iff we want to serve certificates for authorities
+ * that we don't acknowledge as authorities ourself.
+ * Use we_want_to_fetch_unknown_auth_certs to check if we want to fetch
+ * and keep these certificates.
+ */
+int
+directory_caches_unknown_auth_certs(const or_options_t *options)
+{
+  return dir_server_mode(options) || options->BridgeRelay;
+}
+
+/** Return 1 if we want to fetch and serve descriptors, networkstatuses, etc
+ * Else return 0.
+ * Check options->DirPort_set and directory_permits_begindir_requests()
+ * to see if we are willing to serve these directory documents to others via
+ * the DirPort and begindir-over-ORPort, respectively.
+ *
+ * To check if we should fetch documents, use we_want_to_fetch_flavor and
+ * we_want_to_fetch_unknown_auth_certs instead of this function.
+ */
+int
+directory_caches_dir_info(const or_options_t *options)
+{
+  if (options->BridgeRelay || dir_server_mode(options))
+    return 1;
+  if (!server_mode(options) || !advertised_server_mode())
+    return 0;
+  /* We need an up-to-date view of network info if we're going to try to
+   * block exit attempts from unknown relays. */
+  return ! router_my_exit_policy_is_reject_star() &&
+    should_refuse_unknown_exits(options);
+}
+
+/** Return 1 if we want to allow remote clients to ask us directory
+ * requests via the "begin_dir" interface, which doesn't require
+ * having any separate port open. */
+int
+directory_permits_begindir_requests(const or_options_t *options)
+{
+  return options->BridgeRelay != 0 || dir_server_mode(options);
+}
+
+/** Return 1 if we have no need to fetch new descriptors. This generally
+ * happens when we're not a dir cache and we haven't built any circuits
+ * lately.
+ */
+int
+directory_too_idle_to_fetch_descriptors(const or_options_t *options,
+                                        time_t now)
+{
+  return !directory_caches_dir_info(options) &&
+         !options->FetchUselessDescriptors &&
+         rep_hist_circbuilding_dormant(now);
+}
+
+/********************************************************************/
+
+/** Map from flavor name to the cached_dir_t for the v3 consensuses that we're
+ * currently serving. */
+static strmap_t *cached_consensuses = NULL;
+
+/** Decrement the reference count on <b>d</b>, and free it if it no longer has
+ * any references. */
+void
+cached_dir_decref(cached_dir_t *d)
+{
+  if (!d || --d->refcnt > 0)
+    return;
+  clear_cached_dir(d);
+  tor_free(d);
+}
+
+/** Allocate and return a new cached_dir_t containing the string <b>s</b>,
+ * published at <b>published</b>. */
+cached_dir_t *
+new_cached_dir(char *s, time_t published)
+{
+  cached_dir_t *d = tor_malloc_zero(sizeof(cached_dir_t));
+  d->refcnt = 1;
+  d->dir = s;
+  d->dir_len = strlen(s);
+  d->published = published;
+  if (tor_compress(&(d->dir_compressed), &(d->dir_compressed_len),
+                   d->dir, d->dir_len, ZLIB_METHOD)) {
+    log_warn(LD_BUG, "Error compressing directory");
+  }
+  return d;
+}
+
+/** Remove all storage held in <b>d</b>, but do not free <b>d</b> itself. */
+static void
+clear_cached_dir(cached_dir_t *d)
+{
+  tor_free(d->dir);
+  tor_free(d->dir_compressed);
+  memset(d, 0, sizeof(cached_dir_t));
+}
+
+/** Free all storage held by the cached_dir_t in <b>d</b>. */
+static void
+free_cached_dir_(void *_d)
+{
+  cached_dir_t *d;
+  if (!_d)
+    return;
+
+  d = (cached_dir_t *)_d;
+  cached_dir_decref(d);
+}
+
+/** Replace the v3 consensus networkstatus of type <b>flavor_name</b> that
+ * we're serving with <b>networkstatus</b>, published at <b>published</b>.  No
+ * validation is performed. */
+void
+dirserv_set_cached_consensus_networkstatus(const char *networkstatus,
+                                           const char *flavor_name,
+                                           const common_digests_t *digests,
+                                           const uint8_t *sha3_as_signed,
+                                           time_t published)
+{
+  cached_dir_t *new_networkstatus;
+  cached_dir_t *old_networkstatus;
+  if (!cached_consensuses)
+    cached_consensuses = strmap_new();
+
+  new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published);
+  memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t));
+  memcpy(&new_networkstatus->digest_sha3_as_signed, sha3_as_signed,
+         DIGEST256_LEN);
+  old_networkstatus = strmap_set(cached_consensuses, flavor_name,
+                                 new_networkstatus);
+  if (old_networkstatus)
+    cached_dir_decref(old_networkstatus);
+}
+
+/** Return the latest downloaded consensus networkstatus in encoded, signed,
+ * optionally compressed format, suitable for sending to clients. */
+cached_dir_t *
+dirserv_get_consensus(const char *flavor_name)
+{
+  if (!cached_consensuses)
+    return NULL;
+  return strmap_get(cached_consensuses, flavor_name);
+}
+
+/** As dirserv_get_routerdescs(), but instead of getting signed_descriptor_t
+ * pointers, adds copies of digests to fps_out, and doesn't use the
+ * /tor/server/ prefix.  For a /d/ request, adds descriptor digests; for other
+ * requests, adds identity digests.
+ */
+int
+dirserv_get_routerdesc_spool(smartlist_t *spool_out,
+                             const char *key,
+                             dir_spool_source_t source,
+                             int conn_is_encrypted,
+                             const char **msg_out)
+{
+  *msg_out = NULL;
+
+  if (!strcmp(key, "all")) {
+    const routerlist_t *rl = router_get_routerlist();
+    SMARTLIST_FOREACH_BEGIN(rl->routers, const routerinfo_t *, r) {
+      spooled_resource_t *spooled;
+      spooled = spooled_resource_new(source,
+                              (const uint8_t *)r->cache_info.identity_digest,
+                              DIGEST_LEN);
+      /* Treat "all" requests as if they were unencrypted */
+      conn_is_encrypted = 0;
+      smartlist_add(spool_out, spooled);
+    } SMARTLIST_FOREACH_END(r);
+  } else if (!strcmp(key, "authority")) {
+    const routerinfo_t *ri = router_get_my_routerinfo();
+    if (ri)
+      smartlist_add(spool_out,
+                    spooled_resource_new(source,
+                             (const uint8_t *)ri->cache_info.identity_digest,
+                             DIGEST_LEN));
+  } else if (!strcmpstart(key, "d/")) {
+    key += strlen("d/");
+    dir_split_resource_into_spoolable(key, source, spool_out, NULL,
+                                  DSR_HEX|DSR_SORT_UNIQ);
+  } else if (!strcmpstart(key, "fp/")) {
+    key += strlen("fp/");
+    dir_split_resource_into_spoolable(key, source, spool_out, NULL,
+                                  DSR_HEX|DSR_SORT_UNIQ);
+  } else {
+    *msg_out = "Not found";
+    return -1;
+  }
+
+  if (! conn_is_encrypted) {
+    /* Remove anything that insists it not be sent unencrypted. */
+    SMARTLIST_FOREACH_BEGIN(spool_out, spooled_resource_t *, spooled) {
+      const uint8_t *body = NULL;
+      size_t bodylen = 0;
+      int r = spooled_resource_lookup_body(spooled, conn_is_encrypted,
+                                           &body, &bodylen, NULL);
+      if (r < 0 || body == NULL || bodylen == 0) {
+        SMARTLIST_DEL_CURRENT(spool_out, spooled);
+        spooled_resource_free(spooled);
+      }
+    } SMARTLIST_FOREACH_END(spooled);
+  }
+
+  if (!smartlist_len(spool_out)) {
+    *msg_out = "Servers unavailable";
+    return -1;
+  }
+  return 0;
+}
+
+/** Add a signed_descriptor_t to <b>descs_out</b> for each router matching
+ * <b>key</b>.  The key should be either
+ *   - "/tor/server/authority" for our own routerinfo;
+ *   - "/tor/server/all" for all the routerinfos we have, concatenated;
+ *   - "/tor/server/fp/FP" where FP is a plus-separated sequence of
+ *     hex identity digests; or
+ *   - "/tor/server/d/D" where D is a plus-separated sequence
+ *     of server descriptor digests, in hex.
+ *
+ * Return 0 if we found some matching descriptors, or -1 if we do not
+ * have any descriptors, no matching descriptors, or if we did not
+ * recognize the key (URL).
+ * If -1 is returned *<b>msg</b> will be set to an appropriate error
+ * message.
+ *
+ * XXXX rename this function.  It's only called from the controller.
+ * XXXX in fact, refactor this function, merging as much as possible.
+ */
+int
+dirserv_get_routerdescs(smartlist_t *descs_out, const char *key,
+                        const char **msg)
+{
+  *msg = NULL;
+
+  if (!strcmp(key, "/tor/server/all")) {
+    routerlist_t *rl = router_get_routerlist();
+    SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r,
+                      smartlist_add(descs_out, &(r->cache_info)));
+  } else if (!strcmp(key, "/tor/server/authority")) {
+    const routerinfo_t *ri = router_get_my_routerinfo();
+    if (ri)
+      smartlist_add(descs_out, (void*) &(ri->cache_info));
+  } else if (!strcmpstart(key, "/tor/server/d/")) {
+    smartlist_t *digests = smartlist_new();
+    key += strlen("/tor/server/d/");
+    dir_split_resource_into_fingerprints(key, digests, NULL,
+                                         DSR_HEX|DSR_SORT_UNIQ);
+    SMARTLIST_FOREACH(digests, const char *, d,
+       {
+         signed_descriptor_t *sd = router_get_by_descriptor_digest(d);
+         if (sd)
+           smartlist_add(descs_out,sd);
+       });
+    SMARTLIST_FOREACH(digests, char *, d, tor_free(d));
+    smartlist_free(digests);
+  } else if (!strcmpstart(key, "/tor/server/fp/")) {
+    smartlist_t *digests = smartlist_new();
+    time_t cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH;
+    key += strlen("/tor/server/fp/");
+    dir_split_resource_into_fingerprints(key, digests, NULL,
+                                         DSR_HEX|DSR_SORT_UNIQ);
+    SMARTLIST_FOREACH_BEGIN(digests, const char *, d) {
+         if (router_digest_is_me(d)) {
+           /* calling router_get_my_routerinfo() to make sure it exists */
+           const routerinfo_t *ri = router_get_my_routerinfo();
+           if (ri)
+             smartlist_add(descs_out, (void*) &(ri->cache_info));
+         } else {
+           const routerinfo_t *ri = router_get_by_id_digest(d);
+           /* Don't actually serve a descriptor that everyone will think is
+            * expired.  This is an (ugly) workaround to keep buggy 0.1.1.10
+            * Tors from downloading descriptors that they will throw away.
+            */
+           if (ri && ri->cache_info.published_on > cutoff)
+             smartlist_add(descs_out, (void*) &(ri->cache_info));
+         }
+    } SMARTLIST_FOREACH_END(d);
+    SMARTLIST_FOREACH(digests, char *, d, tor_free(d));
+    smartlist_free(digests);
+  } else {
+    *msg = "Key not recognized";
+    return -1;
+  }
+
+  if (!smartlist_len(descs_out)) {
+    *msg = "Servers unavailable";
+    return -1;
+  }
+  return 0;
+}
+
+/* ==========
+ * Spooling code.
+ * ========== */
+
+spooled_resource_t *
+spooled_resource_new(dir_spool_source_t source,
+                     const uint8_t *digest, size_t digestlen)
+{
+  spooled_resource_t *spooled = tor_malloc_zero(sizeof(spooled_resource_t));
+  spooled->spool_source = source;
+  switch (source) {
+    case DIR_SPOOL_NETWORKSTATUS:
+      spooled->spool_eagerly = 0;
+      break;
+    case DIR_SPOOL_SERVER_BY_DIGEST:
+    case DIR_SPOOL_SERVER_BY_FP:
+    case DIR_SPOOL_EXTRA_BY_DIGEST:
+    case DIR_SPOOL_EXTRA_BY_FP:
+    case DIR_SPOOL_MICRODESC:
+    default:
+      spooled->spool_eagerly = 1;
+      break;
+    case DIR_SPOOL_CONSENSUS_CACHE_ENTRY:
+      tor_assert_unreached();
+      break;
+  }
+  tor_assert(digestlen <= sizeof(spooled->digest));
+  if (digest)
+    memcpy(spooled->digest, digest, digestlen);
+  return spooled;
+}
+
+/**
+ * Create a new spooled_resource_t to spool the contents of <b>entry</b> to
+ * the user.  Return the spooled object on success, or NULL on failure (which
+ * is probably caused by a failure to map the body of the item from disk).
+ *
+ * Adds a reference to entry's reference counter.
+ */
+spooled_resource_t *
+spooled_resource_new_from_cache_entry(consensus_cache_entry_t *entry)
+{
+  spooled_resource_t *spooled = tor_malloc_zero(sizeof(spooled_resource_t));
+  spooled->spool_source = DIR_SPOOL_CONSENSUS_CACHE_ENTRY;
+  spooled->spool_eagerly = 0;
+  consensus_cache_entry_incref(entry);
+  spooled->consensus_cache_entry = entry;
+
+  int r = consensus_cache_entry_get_body(entry,
+                                         &spooled->cce_body,
+                                         &spooled->cce_len);
+  if (r == 0) {
+    return spooled;
+  } else {
+    spooled_resource_free(spooled);
+    return NULL;
+  }
+}
+
+/** Release all storage held by <b>spooled</b>. */
+void
+spooled_resource_free_(spooled_resource_t *spooled)
+{
+  if (spooled == NULL)
+    return;
+
+  if (spooled->cached_dir_ref) {
+    cached_dir_decref(spooled->cached_dir_ref);
+  }
+
+  if (spooled->consensus_cache_entry) {
+    consensus_cache_entry_decref(spooled->consensus_cache_entry);
+  }
+
+  tor_free(spooled);
+}
+
+/** When spooling data from a cached_dir_t object, we always add
+ * at least this much. */
+#define DIRSERV_CACHED_DIR_CHUNK_SIZE 8192
+
+/** Return an compression ratio for compressing objects from <b>source</b>.
+ */
+static double
+estimate_compression_ratio(dir_spool_source_t source)
+{
+  /* We should put in better estimates here, depending on the number of
+     objects and their type */
+  (void) source;
+  return 0.5;
+}
+
+/** Return an estimated number of bytes needed for transmitting the
+ * resource in <b>spooled</b> on <b>conn</b>
+ *
+ * As a convenient side-effect, set *<b>published_out</b> to the resource's
+ * publication time.
+ */
+static size_t
+spooled_resource_estimate_size(const spooled_resource_t *spooled,
+                               dir_connection_t *conn,
+                               int compressed,
+                               time_t *published_out)
+{
+  if (spooled->spool_eagerly) {
+    const uint8_t *body = NULL;
+    size_t bodylen = 0;
+    int r = spooled_resource_lookup_body(spooled,
+                                         connection_dir_is_encrypted(conn),
+                                         &body, &bodylen,
+                                         published_out);
+    if (r == -1 || body == NULL || bodylen == 0)
+      return 0;
+    if (compressed) {
+      double ratio = estimate_compression_ratio(spooled->spool_source);
+      bodylen = (size_t)(bodylen * ratio);
+    }
+    return bodylen;
+  } else {
+    cached_dir_t *cached;
+    if (spooled->consensus_cache_entry) {
+      if (published_out) {
+        consensus_cache_entry_get_valid_after(
+            spooled->consensus_cache_entry, published_out);
+      }
+
+      return spooled->cce_len;
+    }
+    if (spooled->cached_dir_ref) {
+      cached = spooled->cached_dir_ref;
+    } else {
+      cached = spooled_resource_lookup_cached_dir(spooled,
+                                                  published_out);
+    }
+    if (cached == NULL) {
+      return 0;
+    }
+    size_t result = compressed ? cached->dir_compressed_len : cached->dir_len;
+    return result;
+  }
+}
+
+/** Return code for spooled_resource_flush_some */
+typedef enum {
+  SRFS_ERR = -1,
+  SRFS_MORE = 0,
+  SRFS_DONE
+} spooled_resource_flush_status_t;
+
+/** Flush some or all of the bytes from <b>spooled</b> onto <b>conn</b>.
+ * Return SRFS_ERR on error, SRFS_MORE if there are more bytes to flush from
+ * this spooled resource, or SRFS_DONE if we are done flushing this spooled
+ * resource.
+ */
+static spooled_resource_flush_status_t
+spooled_resource_flush_some(spooled_resource_t *spooled,
+                            dir_connection_t *conn)
+{
+  if (spooled->spool_eagerly) {
+    /* Spool_eagerly resources are sent all-at-once. */
+    const uint8_t *body = NULL;
+    size_t bodylen = 0;
+    int r = spooled_resource_lookup_body(spooled,
+                                         connection_dir_is_encrypted(conn),
+                                         &body, &bodylen, NULL);
+    if (r == -1 || body == NULL || bodylen == 0) {
+      /* Absent objects count as "done". */
+      return SRFS_DONE;
+    }
+    if (conn->compress_state) {
+      connection_buf_add_compress((const char*)body, bodylen, conn, 0);
+    } else {
+      connection_buf_add((const char*)body, bodylen, TO_CONN(conn));
+    }
+    return SRFS_DONE;
+  } else {
+    cached_dir_t *cached = spooled->cached_dir_ref;
+    consensus_cache_entry_t *cce = spooled->consensus_cache_entry;
+    if (cached == NULL && cce == NULL) {
+      /* The cached_dir_t hasn't been materialized yet. So let's look it up. */
+      cached = spooled->cached_dir_ref =
+        spooled_resource_lookup_cached_dir(spooled, NULL);
+      if (!cached) {
+        /* Absent objects count as done. */
+        return SRFS_DONE;
+      }
+      ++cached->refcnt;
+      tor_assert_nonfatal(spooled->cached_dir_offset == 0);
+    }
+
+    if (BUG(!cached && !cce))
+      return SRFS_DONE;
+
+    int64_t total_len;
+    const char *ptr;
+    if (cached) {
+      total_len = cached->dir_compressed_len;
+      ptr = cached->dir_compressed;
+    } else {
+      total_len = spooled->cce_len;
+      ptr = (const char *)spooled->cce_body;
+    }
+    /* How many bytes left to flush? */
+    int64_t remaining;
+    remaining = total_len - spooled->cached_dir_offset;
+    if (BUG(remaining < 0))
+      return SRFS_ERR;
+    ssize_t bytes = (ssize_t) MIN(DIRSERV_CACHED_DIR_CHUNK_SIZE, remaining);
+    if (conn->compress_state) {
+      connection_buf_add_compress(
+              ptr + spooled->cached_dir_offset,
+              bytes, conn, 0);
+    } else {
+      connection_buf_add(ptr + spooled->cached_dir_offset,
+                              bytes, TO_CONN(conn));
+    }
+    spooled->cached_dir_offset += bytes;
+    if (spooled->cached_dir_offset >= (off_t)total_len) {
+      return SRFS_DONE;
+    } else {
+      return SRFS_MORE;
+    }
+  }
+}
+
+/** Helper: find the cached_dir_t for a spooled_resource_t, for
+ * sending it to <b>conn</b>. Set *<b>published_out</b>, if provided,
+ * to the published time of the cached_dir_t.
+ *
+ * DOES NOT increase the reference count on the result.  Callers must do that
+ * themselves if they mean to hang on to it.
+ */
+static cached_dir_t *
+spooled_resource_lookup_cached_dir(const spooled_resource_t *spooled,
+                                   time_t *published_out)
+{
+  tor_assert(spooled->spool_eagerly == 0);
+  cached_dir_t *d = lookup_cached_dir_by_fp(spooled->digest);
+  if (d != NULL) {
+    if (published_out)
+      *published_out = d->published;
+  }
+  return d;
+}
+
+/** Helper: Look up the body for an eagerly-served spooled_resource.  If
+ * <b>conn_is_encrypted</b> is false, don't look up any resource that
+ * shouldn't be sent over an unencrypted connection.  On success, set
+ * <b>body_out</b>, <b>size_out</b>, and <b>published_out</b> to refer
+ * to the resource's body, size, and publication date, and return 0.
+ * On failure return -1. */
+static int
+spooled_resource_lookup_body(const spooled_resource_t *spooled,
+                             int conn_is_encrypted,
+                             const uint8_t **body_out,
+                             size_t *size_out,
+                             time_t *published_out)
+{
+  tor_assert(spooled->spool_eagerly == 1);
+
+  const signed_descriptor_t *sd = NULL;
+
+  switch (spooled->spool_source) {
+    case DIR_SPOOL_EXTRA_BY_FP: {
+      sd = get_signed_descriptor_by_fp(spooled->digest, 1);
+      break;
+    }
+    case DIR_SPOOL_SERVER_BY_FP: {
+      sd = get_signed_descriptor_by_fp(spooled->digest, 0);
+      break;
+    }
+    case DIR_SPOOL_SERVER_BY_DIGEST: {
+      sd = router_get_by_descriptor_digest((const char *)spooled->digest);
+      break;
+    }
+    case DIR_SPOOL_EXTRA_BY_DIGEST: {
+      sd = extrainfo_get_by_descriptor_digest((const char *)spooled->digest);
+      break;
+    }
+    case DIR_SPOOL_MICRODESC: {
+      microdesc_t *md = microdesc_cache_lookup_by_digest256(
+                                  get_microdesc_cache(),
+                                  (const char *)spooled->digest);
+      if (! md || ! md->body) {
+        return -1;
+      }
+      *body_out = (const uint8_t *)md->body;
+      *size_out = md->bodylen;
+      if (published_out)
+        *published_out = TIME_MAX;
+      return 0;
+    }
+    case DIR_SPOOL_NETWORKSTATUS:
+    case DIR_SPOOL_CONSENSUS_CACHE_ENTRY:
+    default:
+      /* LCOV_EXCL_START */
+      tor_assert_nonfatal_unreached();
+      return -1;
+      /* LCOV_EXCL_STOP */
+  }
+
+  /* If we get here, then we tried to set "sd" to a signed_descriptor_t. */
+
+  if (sd == NULL) {
+    return -1;
+  }
+  if (sd->send_unencrypted == 0 && ! conn_is_encrypted) {
+    /* we did this check once before (so we could have an accurate size
+     * estimate and maybe send a 404 if somebody asked for only bridges on
+     * a connection), but we need to do it again in case a previously
+     * unknown bridge descriptor has shown up between then and now. */
+    return -1;
+  }
+  *body_out = (const uint8_t *) signed_descriptor_get_body(sd);
+  *size_out = sd->signed_descriptor_len;
+  if (published_out)
+    *published_out = sd->published_on;
+  return 0;
+}
+
+/** Given a fingerprint <b>fp</b> which is either set if we're looking for a
+ * v2 status, or zeroes if we're looking for a v3 status, or a NUL-padded
+ * flavor name if we want a flavored v3 status, return a pointer to the
+ * appropriate cached dir object, or NULL if there isn't one available. */
+static cached_dir_t *
+lookup_cached_dir_by_fp(const uint8_t *fp)
+{
+  cached_dir_t *d = NULL;
+  if (tor_digest_is_zero((const char *)fp) && cached_consensuses) {
+    d = strmap_get(cached_consensuses, "ns");
+  } else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses) {
+    /* this here interface is a nasty hack: we're shoving a flavor into
+     * a digest field. */
+    d = strmap_get(cached_consensuses, (const char *)fp);
+  }
+  return d;
+}
+
+/** Try to guess the number of bytes that will be needed to send the
+ * spooled objects for <b>conn</b>'s outgoing spool.  In the process,
+ * remove every element of the spool that refers to an absent object, or
+ * which was published earlier than <b>cutoff</b>.  Set *<b>size_out</b>
+ * to the number of bytes, and *<b>n_expired_out</b> to the number of
+ * objects removed for being too old. */
+void
+dirserv_spool_remove_missing_and_guess_size(dir_connection_t *conn,
+                                            time_t cutoff,
+                                            int compression,
+                                            size_t *size_out,
+                                            int *n_expired_out)
+{
+  if (BUG(!conn))
+    return;
+
+  smartlist_t *spool = conn->spool;
+  if (!spool) {
+    if (size_out)
+      *size_out = 0;
+    if (n_expired_out)
+      *n_expired_out = 0;
+    return;
+  }
+  int n_expired = 0;
+  uint64_t total = 0;
+  SMARTLIST_FOREACH_BEGIN(spool, spooled_resource_t *, spooled) {
+    time_t published = TIME_MAX;
+    size_t sz = spooled_resource_estimate_size(spooled, conn,
+                                               compression, &published);
+    if (published < cutoff) {
+      ++n_expired;
+      SMARTLIST_DEL_CURRENT(spool, spooled);
+      spooled_resource_free(spooled);
+    } else if (sz == 0) {
+      SMARTLIST_DEL_CURRENT(spool, spooled);
+      spooled_resource_free(spooled);
+    } else {
+      total += sz;
+    }
+  } SMARTLIST_FOREACH_END(spooled);
+
+  if (size_out) {
+    *size_out = (total > SIZE_MAX) ? SIZE_MAX : (size_t)total;
+  }
+  if (n_expired_out)
+    *n_expired_out = n_expired;
+}
+
+/** Helper: used to sort a connection's spool. */
+static int
+dirserv_spool_sort_comparison_(const void **a_, const void **b_)
+{
+  const spooled_resource_t *a = *a_;
+  const spooled_resource_t *b = *b_;
+  return fast_memcmp(a->digest, b->digest, sizeof(a->digest));
+}
+
+/** Sort all the entries in <b>conn</b> by digest. */
+void
+dirserv_spool_sort(dir_connection_t *conn)
+{
+  if (conn->spool == NULL)
+    return;
+  smartlist_sort(conn->spool, dirserv_spool_sort_comparison_);
+}
+
+/** Return the cache-info for identity fingerprint <b>fp</b>, or
+ * its extra-info document if <b>extrainfo</b> is true. Return
+ * NULL if not found or if the descriptor is older than
+ * <b>publish_cutoff</b>. */
+static const signed_descriptor_t *
+get_signed_descriptor_by_fp(const uint8_t *fp, int extrainfo)
+{
+  if (router_digest_is_me((const char *)fp)) {
+    if (extrainfo)
+      return &(router_get_my_extrainfo()->cache_info);
+    else
+      return &(router_get_my_routerinfo()->cache_info);
+  } else {
+    const routerinfo_t *ri = router_get_by_id_digest((const char *)fp);
+    if (ri) {
+      if (extrainfo)
+        return extrainfo_get_by_descriptor_digest(
+                                     ri->cache_info.extra_info_digest);
+      else
+        return &ri->cache_info;
+    }
+  }
+  return NULL;
+}
+
+/** When we're spooling data onto our outbuf, add more whenever we dip
+ * below this threshold. */
+#define DIRSERV_BUFFER_MIN 16384
+
+/**
+ * Called whenever we have flushed some directory data in state
+ * SERVER_WRITING, or whenever we want to fill the buffer with initial
+ * directory data (so that subsequent writes will occur, and trigger this
+ * function again.)
+ *
+ * Return 0 on success, and -1 on failure.
+ */
+int
+connection_dirserv_flushed_some(dir_connection_t *conn)
+{
+  tor_assert(conn->base_.state == DIR_CONN_STATE_SERVER_WRITING);
+  if (conn->spool == NULL)
+    return 0;
+
+  while (connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN &&
+         smartlist_len(conn->spool)) {
+    spooled_resource_t *spooled =
+      smartlist_get(conn->spool, smartlist_len(conn->spool)-1);
+    spooled_resource_flush_status_t status;
+    status = spooled_resource_flush_some(spooled, conn);
+    if (status == SRFS_ERR) {
+      return -1;
+    } else if (status == SRFS_MORE) {
+      return 0;
+    }
+    tor_assert(status == SRFS_DONE);
+
+    /* If we're here, we're done flushing this resource. */
+    tor_assert(smartlist_pop_last(conn->spool) == spooled);
+    spooled_resource_free(spooled);
+  }
+
+  if (smartlist_len(conn->spool) > 0) {
+    /* We're still spooling something. */
+    return 0;
+  }
+
+  /* If we get here, we're done. */
+  smartlist_free(conn->spool);
+  conn->spool = NULL;
+  if (conn->compress_state) {
+    /* Flush the compression state: there could be more bytes pending in there,
+     * and we don't want to omit bytes. */
+    connection_buf_add_compress("", 0, conn, 1);
+    tor_compress_free(conn->compress_state);
+    conn->compress_state = NULL;
+  }
+  return 0;
+}
+
+/** Remove every element from <b>conn</b>'s outgoing spool, and delete
+ * the spool. */
+void
+dir_conn_clear_spool(dir_connection_t *conn)
+{
+  if (!conn || ! conn->spool)
+    return;
+  SMARTLIST_FOREACH(conn->spool, spooled_resource_t *, s,
+                    spooled_resource_free(s));
+  smartlist_free(conn->spool);
+  conn->spool = NULL;
+}
+
+/** Release all storage used by the directory server. */
+void
+dirserv_free_all(void)
+{
+  strmap_free(cached_consensuses, free_cached_dir_);
+  cached_consensuses = NULL;
+}
diff --git a/src/feature/dircache/dirserv.h b/src/feature/dircache/dirserv.h
new file mode 100644
index 0000000000..890b10fd80
--- /dev/null
+++ b/src/feature/dircache/dirserv.h
@@ -0,0 +1,119 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dirserv.h
+ * \brief Header file for dirserv.c.
+ **/
+
+#ifndef TOR_DIRSERV_H
+#define TOR_DIRSERV_H
+
+struct ed25519_public_key_t;
+
+#include "lib/testsupport/testsupport.h"
+
+/** Ways to convert a spoolable_resource_t to a bunch of bytes. */
+typedef enum dir_spool_source_t {
+    DIR_SPOOL_SERVER_BY_DIGEST=1, DIR_SPOOL_SERVER_BY_FP,
+    DIR_SPOOL_EXTRA_BY_DIGEST, DIR_SPOOL_EXTRA_BY_FP,
+    DIR_SPOOL_MICRODESC,
+    DIR_SPOOL_NETWORKSTATUS,
+    DIR_SPOOL_CONSENSUS_CACHE_ENTRY,
+} dir_spool_source_t;
+#define dir_spool_source_bitfield_t ENUM_BF(dir_spool_source_t)
+
+/** Object to remember the identity of an object that we are spooling,
+ * or about to spool, in response to a directory request.
+ *
+ * (Why do we spool?  Because some directory responses are very large,
+ * and we don't want to just shove the complete answer into the output
+ * buffer: that would take a ridiculous amount of RAM.)
+ *
+ * If the spooled resource is relatively small (like microdescriptors,
+ * descriptors, etc), we look them up by ID as needed, and add the whole
+ * thing onto the output buffer at once.  If the spooled reseource is
+ * big (like networkstatus documents), we reference-count it, and add it
+ * a few K at a time.
+ */
+typedef struct spooled_resource_t {
+  /**
+   * If true, we add the entire object to the outbuf.  If false,
+   * we spool the object a few K at a time.
+   */
+  unsigned spool_eagerly : 1;
+  /**
+   * Tells us what kind of object to get, and how to look it up.
+   */
+  dir_spool_source_bitfield_t spool_source : 7;
+  /**
+   * Tells us the specific object to spool.
+   */
+  uint8_t digest[DIGEST256_LEN];
+  /**
+   * A large object that we're spooling. Holds a reference count.  Only
+   * used when spool_eagerly is false.
+   */
+  struct cached_dir_t *cached_dir_ref;
+  /**
+   * A different kind of large object that we might be spooling. Also
+   * reference-counted.  Also only used when spool_eagerly is false.
+   */
+  struct consensus_cache_entry_t *consensus_cache_entry;
+  const uint8_t *cce_body;
+  size_t cce_len;
+  /**
+   * The current offset into cached_dir or cce_body. Only used when
+   * spool_eagerly is false */
+  off_t cached_dir_offset;
+} spooled_resource_t;
+
+int connection_dirserv_flushed_some(dir_connection_t *conn);
+
+int directory_fetches_from_authorities(const or_options_t *options);
+int directory_fetches_dir_info_early(const or_options_t *options);
+int directory_fetches_dir_info_later(const or_options_t *options);
+int directory_caches_unknown_auth_certs(const or_options_t *options);
+int directory_caches_dir_info(const or_options_t *options);
+int directory_permits_begindir_requests(const or_options_t *options);
+int directory_too_idle_to_fetch_descriptors(const or_options_t *options,
+                                            time_t now);
+
+cached_dir_t *dirserv_get_consensus(const char *flavor_name);
+void dirserv_set_cached_consensus_networkstatus(const char *consensus,
+                                              const char *flavor_name,
+                                              const common_digests_t *digests,
+                                              const uint8_t *sha3_as_signed,
+                                              time_t published);
+void dirserv_clear_old_networkstatuses(time_t cutoff);
+int dirserv_get_routerdesc_spool(smartlist_t *spools_out, const char *key,
+                                 dir_spool_source_t source,
+                                 int conn_is_encrypted,
+                                 const char **msg_out);
+int dirserv_get_routerdescs(smartlist_t *descs_out, const char *key,
+                            const char **msg);
+
+void dirserv_free_all(void);
+void cached_dir_decref(cached_dir_t *d);
+cached_dir_t *new_cached_dir(char *s, time_t published);
+
+spooled_resource_t *spooled_resource_new(dir_spool_source_t source,
+                                         const uint8_t *digest,
+                                         size_t digestlen);
+spooled_resource_t *spooled_resource_new_from_cache_entry(
+                                      struct consensus_cache_entry_t *entry);
+void spooled_resource_free_(spooled_resource_t *spooled);
+#define spooled_resource_free(sp) \
+  FREE_AND_NULL(spooled_resource_t, spooled_resource_free_, (sp))
+void dirserv_spool_remove_missing_and_guess_size(dir_connection_t *conn,
+                                                 time_t cutoff,
+                                                 int compression,
+                                                 size_t *size_out,
+                                                 int *n_expired_out);
+void dirserv_spool_sort(dir_connection_t *conn);
+void dir_conn_clear_spool(dir_connection_t *conn);
+
+#endif /* !defined(TOR_DIRSERV_H) */
diff --git a/src/feature/dirclient/dir_server_st.h b/src/feature/dirclient/dir_server_st.h
new file mode 100644
index 0000000000..2f5706cdd9
--- /dev/null
+++ b/src/feature/dirclient/dir_server_st.h
@@ -0,0 +1,54 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef DIR_SERVER_ST_H
+#define DIR_SERVER_ST_H
+
+#include "lib/cc/torint.h"
+#include "core/or/or.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+/** Represents information about a single trusted or fallback directory
+ * server. */
+struct dir_server_t {
+  char *description;
+  char *nickname;
+  char *address; /**< Hostname. */
+  /* XX/teor - why do we duplicate the address and port fields here and in
+   *           fake_status? Surely we could just use fake_status (#17867). */
+  tor_addr_t ipv6_addr; /**< IPv6 address if present; AF_UNSPEC if not */
+  uint32_t addr; /**< IPv4 address. */
+  uint16_t dir_port; /**< Directory port. */
+  uint16_t or_port; /**< OR port: Used for tunneling connections. */
+  uint16_t ipv6_orport; /**< OR port corresponding to ipv6_addr. */
+  double weight; /** Weight used when selecting this node at random */
+  char digest[DIGEST_LEN]; /**< Digest of identity key. */
+  char v3_identity_digest[DIGEST_LEN]; /**< Digest of v3 (authority only,
+                                        * high-security) identity key. */
+
+  unsigned int is_running:1; /**< True iff we think this server is running. */
+  unsigned int is_authority:1; /**< True iff this is a directory authority
+                                * of some kind. */
+
+  /** True iff this server has accepted the most recent server descriptor
+   * we tried to upload to it. */
+  unsigned int has_accepted_serverdesc:1;
+
+  /** What kind of authority is this? (Bitfield.) */
+  dirinfo_type_t type;
+
+  time_t addr_current_at; /**< When was the document that we derived the
+                           * address information from published? */
+
+  routerstatus_t fake_status; /**< Used when we need to pass this trusted
+                               * dir_server_t to
+                               * directory_request_set_routerstatus.
+                               * as a routerstatus_t.  Not updated by the
+                               * router-status management code!
+                               **/
+};
+
+#endif
diff --git a/src/feature/dirclient/dirclient.c b/src/feature/dirclient/dirclient.c
new file mode 100644
index 0000000000..0fd1a47017
--- /dev/null
+++ b/src/feature/dirclient/dirclient.c
@@ -0,0 +1,3206 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DIRCLIENT_PRIVATE
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/connection_edge.h"
+#include "core/or/policies.h"
+#include "feature/client/bridges.h"
+#include "feature/client/entrynodes.h"
+#include "feature/control/control.h"
+#include "feature/dirauth/authmode.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/shared_random.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
+#include "feature/dircommon/consdiff.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dircommon/fp_pair.h"
+#include "feature/hs/hs_cache.h"
+#include "feature/hs/hs_client.h"
+#include "feature/hs/hs_control.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/node_select.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerinfo.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/relay/routermode.h"
+#include "feature/relay/selftest.h"
+#include "feature/rend/rendcache.h"
+#include "feature/rend/rendclient.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/rend/rendservice.h"
+#include "feature/stats/predict_ports.h"
+
+#include "lib/compress/compress.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/encoding/confline.h"
+#include "lib/err/backtrace.h"
+
+#include "core/or/entry_connection_st.h"
+#include "feature/dircache/cached_dir_st.h"
+#include "feature/dirclient/dir_server_st.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/rend/rend_service_descriptor_st.h"
+
+/** Maximum size, in bytes, for any directory object that we've downloaded. */
+#define MAX_DIR_DL_SIZE ((1<<24)-1) /* 16 MB - 1 */
+
+/** How far in the future do we allow a directory server to tell us it is
+ * before deciding that one of us has the wrong time? */
+#define ALLOW_DIRECTORY_TIME_SKEW (30*60)
+
+static int body_is_plausible(const char *body, size_t body_len, int purpose);
+static void connection_dir_download_routerdesc_failed(dir_connection_t *conn);
+static void connection_dir_bridge_routerdesc_failed(dir_connection_t *conn);
+static void connection_dir_download_cert_failed(
+                               dir_connection_t *conn, int status_code);
+static void connection_dir_retry_bridges(smartlist_t *descs);
+static void dir_routerdesc_download_failed(smartlist_t *failed,
+                                           int status_code,
+                                           int router_purpose,
+                                           int was_extrainfo,
+                                           int was_descriptor_digests);
+static void dir_microdesc_download_failed(smartlist_t *failed,
+                                          int status_code,
+                                          const char *dir_id);
+static void directory_send_command(dir_connection_t *conn,
+                                   const int direct,
+                                   const directory_request_t *req);
+static void connection_dir_close_consensus_fetches(
+                   dir_connection_t *except_this_one, const char *resource);
+
+/** Return a string describing a given directory connection purpose. */
+STATIC const char *
+dir_conn_purpose_to_string(int purpose)
+{
+  switch (purpose)
+    {
+    case DIR_PURPOSE_UPLOAD_DIR:
+      return "server descriptor upload";
+    case DIR_PURPOSE_UPLOAD_VOTE:
+      return "server vote upload";
+    case DIR_PURPOSE_UPLOAD_SIGNATURES:
+      return "consensus signature upload";
+    case DIR_PURPOSE_FETCH_SERVERDESC:
+      return "server descriptor fetch";
+    case DIR_PURPOSE_FETCH_EXTRAINFO:
+      return "extra-info fetch";
+    case DIR_PURPOSE_FETCH_CONSENSUS:
+      return "consensus network-status fetch";
+    case DIR_PURPOSE_FETCH_CERTIFICATE:
+      return "authority cert fetch";
+    case DIR_PURPOSE_FETCH_STATUS_VOTE:
+      return "status vote fetch";
+    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
+      return "consensus signature fetch";
+    case DIR_PURPOSE_FETCH_RENDDESC_V2:
+      return "hidden-service v2 descriptor fetch";
+    case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
+      return "hidden-service v2 descriptor upload";
+    case DIR_PURPOSE_FETCH_HSDESC:
+      return "hidden-service descriptor fetch";
+    case DIR_PURPOSE_UPLOAD_HSDESC:
+      return "hidden-service descriptor upload";
+    case DIR_PURPOSE_FETCH_MICRODESC:
+      return "microdescriptor fetch";
+    }
+
+  log_warn(LD_BUG, "Called with unknown purpose %d", purpose);
+  return "(unknown)";
+}
+
+/** Return the requisite directory information types. */
+STATIC dirinfo_type_t
+dir_fetch_type(int dir_purpose, int router_purpose, const char *resource)
+{
+  dirinfo_type_t type;
+  switch (dir_purpose) {
+    case DIR_PURPOSE_FETCH_EXTRAINFO:
+      type = EXTRAINFO_DIRINFO;
+      if (router_purpose == ROUTER_PURPOSE_BRIDGE)
+        type |= BRIDGE_DIRINFO;
+      else
+        type |= V3_DIRINFO;
+      break;
+    case DIR_PURPOSE_FETCH_SERVERDESC:
+      if (router_purpose == ROUTER_PURPOSE_BRIDGE)
+        type = BRIDGE_DIRINFO;
+      else
+        type = V3_DIRINFO;
+      break;
+    case DIR_PURPOSE_FETCH_STATUS_VOTE:
+    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
+    case DIR_PURPOSE_FETCH_CERTIFICATE:
+      type = V3_DIRINFO;
+      break;
+    case DIR_PURPOSE_FETCH_CONSENSUS:
+      type = V3_DIRINFO;
+      if (resource && !strcmp(resource, "microdesc"))
+        type |= MICRODESC_DIRINFO;
+      break;
+    case DIR_PURPOSE_FETCH_MICRODESC:
+      type = MICRODESC_DIRINFO;
+      break;
+    default:
+      log_warn(LD_BUG, "Unexpected purpose %d", (int)dir_purpose);
+      type = NO_DIRINFO;
+      break;
+  }
+  return type;
+}
+
+/** Return true iff <b>identity_digest</b> is the digest of a router which
+ * says that it caches extrainfos.  (If <b>is_authority</b> we always
+ * believe that to be true.) */
+int
+router_supports_extrainfo(const char *identity_digest, int is_authority)
+{
+  const node_t *node = node_get_by_id(identity_digest);
+
+  if (node && node->ri) {
+    if (node->ri->caches_extra_info)
+      return 1;
+  }
+  if (is_authority) {
+    return 1;
+  }
+  return 0;
+}
+
+/** Return true iff any trusted directory authority has accepted our
+ * server descriptor.
+ *
+ * We consider any authority sufficient because waiting for all of
+ * them means it never happens while any authority is down; we don't
+ * go for something more complex in the middle (like \>1/3 or \>1/2 or
+ * \>=1/2) because that doesn't seem necessary yet.
+ */
+int
+directories_have_accepted_server_descriptor(void)
+{
+  const smartlist_t *servers = router_get_trusted_dir_servers();
+  const or_options_t *options = get_options();
+  SMARTLIST_FOREACH(servers, dir_server_t *, d, {
+    if ((d->type & options->PublishServerDescriptor_) &&
+        d->has_accepted_serverdesc) {
+      return 1;
+    }
+  });
+  return 0;
+}
+
+/** Start a connection to every suitable directory authority, using
+ * connection purpose <b>dir_purpose</b> and uploading <b>payload</b>
+ * (of length <b>payload_len</b>). The dir_purpose should be one of
+ * 'DIR_PURPOSE_UPLOAD_{DIR|VOTE|SIGNATURES}'.
+ *
+ * <b>router_purpose</b> describes the type of descriptor we're
+ * publishing, if we're publishing a descriptor -- e.g. general or bridge.
+ *
+ * <b>type</b> specifies what sort of dir authorities (V3,
+ * BRIDGE, etc) we should upload to.
+ *
+ * If <b>extrainfo_len</b> is nonzero, the first <b>payload_len</b> bytes of
+ * <b>payload</b> hold a router descriptor, and the next <b>extrainfo_len</b>
+ * bytes of <b>payload</b> hold an extra-info document.  Upload the descriptor
+ * to all authorities, and the extra-info document to all authorities that
+ * support it.
+ */
+void
+directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
+                             dirinfo_type_t type,
+                             const char *payload,
+                             size_t payload_len, size_t extrainfo_len)
+{
+  const or_options_t *options = get_options();
+  dir_indirection_t indirection;
+  const smartlist_t *dirservers = router_get_trusted_dir_servers();
+  int found = 0;
+  const int exclude_self = (dir_purpose == DIR_PURPOSE_UPLOAD_VOTE ||
+                            dir_purpose == DIR_PURPOSE_UPLOAD_SIGNATURES);
+  tor_assert(dirservers);
+  /* This tries dirservers which we believe to be down, but ultimately, that's
+   * harmless, and we may as well err on the side of getting things uploaded.
+   */
+  SMARTLIST_FOREACH_BEGIN(dirservers, dir_server_t *, ds) {
+      routerstatus_t *rs = &(ds->fake_status);
+      size_t upload_len = payload_len;
+
+      if ((type & ds->type) == 0)
+        continue;
+
+      if (exclude_self && router_digest_is_me(ds->digest)) {
+        /* we don't upload to ourselves, but at least there's now at least
+         * one authority of this type that has what we wanted to upload. */
+        found = 1;
+        continue;
+      }
+
+      if (options->StrictNodes &&
+          routerset_contains_routerstatus(options->ExcludeNodes, rs, -1)) {
+        log_warn(LD_DIR, "Wanted to contact authority '%s' for %s, but "
+                 "it's in our ExcludedNodes list and StrictNodes is set. "
+                 "Skipping.",
+                 ds->nickname,
+                 dir_conn_purpose_to_string(dir_purpose));
+        continue;
+      }
+
+      found = 1; /* at least one authority of this type was listed */
+      if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR)
+        ds->has_accepted_serverdesc = 0;
+
+      if (extrainfo_len && router_supports_extrainfo(ds->digest, 1)) {
+        upload_len += extrainfo_len;
+        log_info(LD_DIR, "Uploading an extrainfo too (length %d)",
+                 (int) extrainfo_len);
+      }
+      if (purpose_needs_anonymity(dir_purpose, router_purpose, NULL)) {
+        indirection = DIRIND_ANONYMOUS;
+      } else if (!fascist_firewall_allows_dir_server(ds,
+                                                     FIREWALL_DIR_CONNECTION,
+                                                     0)) {
+        if (fascist_firewall_allows_dir_server(ds, FIREWALL_OR_CONNECTION, 0))
+          indirection = DIRIND_ONEHOP;
+        else
+          indirection = DIRIND_ANONYMOUS;
+      } else {
+        indirection = DIRIND_DIRECT_CONN;
+      }
+
+      directory_request_t *req = directory_request_new(dir_purpose);
+      directory_request_set_routerstatus(req, rs);
+      directory_request_set_router_purpose(req, router_purpose);
+      directory_request_set_indirection(req, indirection);
+      directory_request_set_payload(req, payload, upload_len);
+      directory_initiate_request(req);
+      directory_request_free(req);
+  } SMARTLIST_FOREACH_END(ds);
+  if (!found) {
+    char *s = authdir_type_to_string(type);
+    log_warn(LD_DIR, "Publishing server descriptor to directory authorities "
+             "of type '%s', but no authorities of that type listed!", s);
+    tor_free(s);
+  }
+}
+
+/** Return true iff, according to the values in <b>options</b>, we should be
+ * using directory guards for direct downloads of directory information. */
+STATIC int
+should_use_directory_guards(const or_options_t *options)
+{
+  /* Public (non-bridge) servers never use directory guards. */
+  if (public_server_mode(options))
+    return 0;
+  /* If guards are disabled, we can't use directory guards.
+   */
+  if (!options->UseEntryGuards)
+    return 0;
+  /* If we're configured to fetch directory info aggressively or of a
+   * nonstandard type, don't use directory guards. */
+  if (options->DownloadExtraInfo || options->FetchDirInfoEarly ||
+      options->FetchDirInfoExtraEarly || options->FetchUselessDescriptors)
+    return 0;
+  return 1;
+}
+
+/** Pick an unconstrained directory server from among our guards, the latest
+ * networkstatus, or the fallback dirservers, for use in downloading
+ * information of type <b>type</b>, and return its routerstatus. */
+static const routerstatus_t *
+directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags,
+                                 uint8_t dir_purpose,
+                                 circuit_guard_state_t **guard_state_out)
+{
+  const routerstatus_t *rs = NULL;
+  const or_options_t *options = get_options();
+
+  if (options->UseBridges)
+    log_warn(LD_BUG, "Called when we have UseBridges set.");
+
+  if (should_use_directory_guards(options)) {
+    const node_t *node = guards_choose_dirguard(dir_purpose, guard_state_out);
+    if (node)
+      rs = node->rs;
+  } else {
+    /* anybody with a non-zero dirport will do */
+    rs = router_pick_directory_server(type, pds_flags);
+  }
+  if (!rs) {
+    log_info(LD_DIR, "No router found for %s; falling back to "
+             "dirserver list.", dir_conn_purpose_to_string(dir_purpose));
+    rs = router_pick_fallback_dirserver(type, pds_flags);
+  }
+
+  return rs;
+}
+
+/**
+ * Set the extra fields in <b>req</b> that are used when requesting a
+ * consensus of type <b>resource</b>.
+ *
+ * Right now, these fields are if-modified-since and x-or-diff-from-consensus.
+ */
+static void
+dir_consensus_request_set_additional_headers(directory_request_t *req,
+                                             const char *resource)
+{
+  time_t if_modified_since = 0;
+  uint8_t or_diff_from[DIGEST256_LEN];
+  int or_diff_from_is_set = 0;
+
+  /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus
+   * period of 1 hour.
+   */
+  const int DEFAULT_IF_MODIFIED_SINCE_DELAY = 180;
+  const int32_t DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER = 72;
+  const int32_t MIN_TRY_DIFF_FOR_CONSENSUS_NEWER = 0;
+  const int32_t MAX_TRY_DIFF_FOR_CONSENSUS_NEWER = 8192;
+  const char TRY_DIFF_FOR_CONSENSUS_NEWER_NAME[] =
+    "try-diff-for-consensus-newer-than";
+
+  int flav = FLAV_NS;
+  if (resource)
+    flav = networkstatus_parse_flavor_name(resource);
+
+  int32_t max_age_for_diff = 3600 *
+    networkstatus_get_param(NULL,
+                            TRY_DIFF_FOR_CONSENSUS_NEWER_NAME,
+                            DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER,
+                            MIN_TRY_DIFF_FOR_CONSENSUS_NEWER,
+                            MAX_TRY_DIFF_FOR_CONSENSUS_NEWER);
+
+  if (flav != -1) {
+    /* IF we have a parsed consensus of this type, we can do an
+     * if-modified-time based on it. */
+    networkstatus_t *v;
+    v = networkstatus_get_latest_consensus_by_flavor(flav);
+    if (v) {
+      /* In networks with particularly short V3AuthVotingIntervals,
+       * ask for the consensus if it's been modified since half the
+       * V3AuthVotingInterval of the most recent consensus. */
+      time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY;
+      if (v->fresh_until > v->valid_after
+          && ims_delay > (v->fresh_until - v->valid_after)/2) {
+        ims_delay = (v->fresh_until - v->valid_after)/2;
+      }
+      if_modified_since = v->valid_after + ims_delay;
+      if (v->valid_after >= approx_time() - max_age_for_diff) {
+        memcpy(or_diff_from, v->digest_sha3_as_signed, DIGEST256_LEN);
+        or_diff_from_is_set = 1;
+      }
+    }
+  } else {
+    /* Otherwise it might be a consensus we don't parse, but which we
+     * do cache.  Look at the cached copy, perhaps. */
+    cached_dir_t *cd = dirserv_get_consensus(resource);
+    /* We have no method of determining the voting interval from an
+     * unparsed consensus, so we use the default. */
+    if (cd) {
+      if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY;
+      if (cd->published >= approx_time() - max_age_for_diff) {
+        memcpy(or_diff_from, cd->digest_sha3_as_signed, DIGEST256_LEN);
+        or_diff_from_is_set = 1;
+      }
+    }
+  }
+
+  if (if_modified_since > 0)
+    directory_request_set_if_modified_since(req, if_modified_since);
+  if (or_diff_from_is_set) {
+    char hex[HEX_DIGEST256_LEN + 1];
+    base16_encode(hex, sizeof(hex),
+                  (const char*)or_diff_from, sizeof(or_diff_from));
+    directory_request_add_header(req, X_OR_DIFF_FROM_CONSENSUS_HEADER, hex);
+  }
+}
+/** Start a connection to a random running directory server, using
+ * connection purpose <b>dir_purpose</b>, intending to fetch descriptors
+ * of purpose <b>router_purpose</b>, and requesting <b>resource</b>.
+ * Use <b>pds_flags</b> as arguments to router_pick_directory_server()
+ * or router_pick_trusteddirserver().
+ */
+MOCK_IMPL(void,
+directory_get_from_dirserver,(
+                            uint8_t dir_purpose,
+                            uint8_t router_purpose,
+                            const char *resource,
+                            int pds_flags,
+                            download_want_authority_t want_authority))
+{
+  const routerstatus_t *rs = NULL;
+  const or_options_t *options = get_options();
+  int prefer_authority = (directory_fetches_from_authorities(options)
+                          || want_authority == DL_WANT_AUTHORITY);
+  int require_authority = 0;
+  int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose,
+                                            resource);
+  dirinfo_type_t type = dir_fetch_type(dir_purpose, router_purpose, resource);
+
+  if (type == NO_DIRINFO)
+    return;
+
+  if (!options->FetchServerDescriptors)
+    return;
+
+  circuit_guard_state_t *guard_state = NULL;
+  if (!get_via_tor) {
+    if (options->UseBridges && !(type & BRIDGE_DIRINFO)) {
+      /* We want to ask a running bridge for which we have a descriptor.
+       *
+       * When we ask choose_random_entry() for a bridge, we specify what
+       * sort of dir fetch we'll be doing, so it won't return a bridge
+       * that can't answer our question.
+       */
+      const node_t *node = guards_choose_dirguard(dir_purpose, &guard_state);
+      if (node && node->ri) {
+        /* every bridge has a routerinfo. */
+        routerinfo_t *ri = node->ri;
+        /* clients always make OR connections to bridges */
+        tor_addr_port_t or_ap;
+        directory_request_t *req = directory_request_new(dir_purpose);
+        /* we are willing to use a non-preferred address if we need to */
+        fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0,
+                                             &or_ap);
+        directory_request_set_or_addr_port(req, &or_ap);
+        directory_request_set_directory_id_digest(req,
+                                            ri->cache_info.identity_digest);
+        directory_request_set_router_purpose(req, router_purpose);
+        directory_request_set_resource(req, resource);
+        if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS)
+          dir_consensus_request_set_additional_headers(req, resource);
+        directory_request_set_guard_state(req, guard_state);
+        directory_initiate_request(req);
+        directory_request_free(req);
+      } else {
+        if (guard_state) {
+          entry_guard_cancel(&guard_state);
+        }
+        log_notice(LD_DIR, "Ignoring directory request, since no bridge "
+                           "nodes are available yet.");
+      }
+
+      return;
+    } else {
+      if (prefer_authority || (type & BRIDGE_DIRINFO)) {
+        /* only ask authdirservers, and don't ask myself */
+        rs = router_pick_trusteddirserver(type, pds_flags);
+        if (rs == NULL && (pds_flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
+                                        PDS_NO_EXISTING_MICRODESC_FETCH))) {
+          /* We don't want to fetch from any authorities that we're currently
+           * fetching server descriptors from, and we got no match.  Did we
+           * get no match because all the authorities have connections
+           * fetching server descriptors (in which case we should just
+           * return,) or because all the authorities are down or on fire or
+           * unreachable or something (in which case we should go on with
+           * our fallback code)? */
+          pds_flags &= ~(PDS_NO_EXISTING_SERVERDESC_FETCH|
+                         PDS_NO_EXISTING_MICRODESC_FETCH);
+          rs = router_pick_trusteddirserver(type, pds_flags);
+          if (rs) {
+            log_debug(LD_DIR, "Deferring serverdesc fetch: all authorities "
+                      "are in use.");
+            return;
+          }
+        }
+        if (rs == NULL && require_authority) {
+          log_info(LD_DIR, "No authorities were available for %s: will try "
+                   "later.", dir_conn_purpose_to_string(dir_purpose));
+          return;
+        }
+      }
+      if (!rs && !(type & BRIDGE_DIRINFO)) {
+        rs = directory_pick_generic_dirserver(type, pds_flags,
+                                              dir_purpose,
+                                              &guard_state);
+        if (!rs)
+          get_via_tor = 1; /* last resort: try routing it via Tor */
+      }
+    }
+  }
+
+  if (get_via_tor) {
+    /* Never use fascistfirewall; we're going via Tor. */
+    pds_flags |= PDS_IGNORE_FASCISTFIREWALL;
+    rs = router_pick_directory_server(type, pds_flags);
+  }
+
+  /* If we have any hope of building an indirect conn, we know some router
+   * descriptors.  If (rs==NULL), we can't build circuits anyway, so
+   * there's no point in falling back to the authorities in this case. */
+  if (rs) {
+    const dir_indirection_t indirection =
+      get_via_tor ? DIRIND_ANONYMOUS : DIRIND_ONEHOP;
+    directory_request_t *req = directory_request_new(dir_purpose);
+    directory_request_set_routerstatus(req, rs);
+    directory_request_set_router_purpose(req, router_purpose);
+    directory_request_set_indirection(req, indirection);
+    directory_request_set_resource(req, resource);
+    if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS)
+      dir_consensus_request_set_additional_headers(req, resource);
+    if (guard_state)
+      directory_request_set_guard_state(req, guard_state);
+    directory_initiate_request(req);
+    directory_request_free(req);
+  } else {
+    log_notice(LD_DIR,
+               "While fetching directory info, "
+               "no running dirservers known. Will try again later. "
+               "(purpose %d)", dir_purpose);
+    if (!purpose_needs_anonymity(dir_purpose, router_purpose, resource)) {
+      /* remember we tried them all and failed. */
+      directory_all_unreachable(time(NULL));
+    }
+  }
+}
+
+/** As directory_get_from_dirserver, but initiates a request to <i>every</i>
+ * directory authority other than ourself.  Only for use by authorities when
+ * searching for missing information while voting. */
+void
+directory_get_from_all_authorities(uint8_t dir_purpose,
+                                   uint8_t router_purpose,
+                                   const char *resource)
+{
+  tor_assert(dir_purpose == DIR_PURPOSE_FETCH_STATUS_VOTE ||
+             dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES);
+
+  SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
+                          dir_server_t *, ds) {
+      if (router_digest_is_me(ds->digest))
+        continue;
+      if (!(ds->type & V3_DIRINFO))
+        continue;
+      const routerstatus_t *rs = &ds->fake_status;
+      directory_request_t *req = directory_request_new(dir_purpose);
+      directory_request_set_routerstatus(req, rs);
+      directory_request_set_router_purpose(req, router_purpose);
+      directory_request_set_resource(req, resource);
+      directory_initiate_request(req);
+      directory_request_free(req);
+  } SMARTLIST_FOREACH_END(ds);
+}
+
+/** Return true iff <b>ind</b> requires a multihop circuit. */
+static int
+dirind_is_anon(dir_indirection_t ind)
+{
+  return ind == DIRIND_ANON_DIRPORT || ind == DIRIND_ANONYMOUS;
+}
+
+/* Choose reachable OR and Dir addresses and ports from status, copying them
+ * into use_or_ap and use_dir_ap. If indirection is anonymous, then we're
+ * connecting via another relay, so choose the primary IPv4 address and ports.
+ *
+ * status should have at least one reachable address, if we can't choose a
+ * reachable address, warn and return -1. Otherwise, return 0.
+ */
+static int
+directory_choose_address_routerstatus(const routerstatus_t *status,
+                                      dir_indirection_t indirection,
+                                      tor_addr_port_t *use_or_ap,
+                                      tor_addr_port_t *use_dir_ap)
+{
+  tor_assert(status != NULL);
+  tor_assert(use_or_ap != NULL);
+  tor_assert(use_dir_ap != NULL);
+
+  const or_options_t *options = get_options();
+  int have_or = 0, have_dir = 0;
+
+  /* We expect status to have at least one reachable address if we're
+   * connecting to it directly.
+   *
+   * Therefore, we can simply use the other address if the one we want isn't
+   * allowed by the firewall.
+   *
+   * (When Tor uploads and downloads a hidden service descriptor, it uses
+   * DIRIND_ANONYMOUS. Even Single Onion Servers (NYI) use DIRIND_ANONYMOUS,
+   * to avoid HSDirs denying service by rejecting descriptors.)
+   */
+
+  /* Initialise the OR / Dir addresses */
+  tor_addr_make_null(&use_or_ap->addr, AF_UNSPEC);
+  use_or_ap->port = 0;
+  tor_addr_make_null(&use_dir_ap->addr, AF_UNSPEC);
+  use_dir_ap->port = 0;
+
+  /* ORPort connections */
+  if (indirection == DIRIND_ANONYMOUS) {
+    if (status->addr) {
+      /* Since we're going to build a 3-hop circuit and ask the 2nd relay
+       * to extend to this address, always use the primary (IPv4) OR address */
+      tor_addr_from_ipv4h(&use_or_ap->addr, status->addr);
+      use_or_ap->port = status->or_port;
+      have_or = 1;
+    }
+  } else if (indirection == DIRIND_ONEHOP) {
+    /* We use an IPv6 address if we have one and we prefer it.
+     * Use the preferred address and port if they are reachable, otherwise,
+     * use the alternate address and port (if any).
+     */
+    fascist_firewall_choose_address_rs(status, FIREWALL_OR_CONNECTION, 0,
+                                       use_or_ap);
+    have_or = tor_addr_port_is_valid_ap(use_or_ap, 0);
+  }
+
+  /* DirPort connections
+   * DIRIND_ONEHOP uses ORPort, but may fall back to the DirPort on relays */
+  if (indirection == DIRIND_DIRECT_CONN ||
+      indirection == DIRIND_ANON_DIRPORT ||
+      (indirection == DIRIND_ONEHOP
+       && !directory_must_use_begindir(options))) {
+    fascist_firewall_choose_address_rs(status, FIREWALL_DIR_CONNECTION, 0,
+                                       use_dir_ap);
+    have_dir = tor_addr_port_is_valid_ap(use_dir_ap, 0);
+  }
+
+  /* We rejected all addresses in the relay's status. This means we can't
+   * connect to it. */
+  if (!have_or && !have_dir) {
+    static int logged_backtrace = 0;
+    log_info(LD_BUG, "Rejected all OR and Dir addresses from %s when "
+             "launching an outgoing directory connection to: IPv4 %s OR %d "
+             "Dir %d IPv6 %s OR %d Dir %d", routerstatus_describe(status),
+             fmt_addr32(status->addr), status->or_port,
+             status->dir_port, fmt_addr(&status->ipv6_addr),
+             status->ipv6_orport, status->dir_port);
+    if (!logged_backtrace) {
+      log_backtrace(LOG_INFO, LD_BUG, "Addresses came from");
+      logged_backtrace = 1;
+    }
+    return -1;
+  }
+
+  return 0;
+}
+
+/** Return true iff <b>conn</b> is the client side of a directory connection
+ * we launched to ourself in order to determine the reachability of our
+ * dir_port. */
+static int
+directory_conn_is_self_reachability_test(dir_connection_t *conn)
+{
+  if (conn->requested_resource &&
+      !strcmpstart(conn->requested_resource,"authority")) {
+    const routerinfo_t *me = router_get_my_routerinfo();
+    if (me &&
+        router_digest_is_me(conn->identity_digest) &&
+        tor_addr_eq_ipv4h(&conn->base_.addr, me->addr) && /*XXXX prop 118*/
+        me->dir_port == conn->base_.port)
+      return 1;
+  }
+  return 0;
+}
+
+/** Called when we are unable to complete the client's request to a directory
+ * server due to a network error: Mark the router as down and try again if
+ * possible.
+ */
+void
+connection_dir_client_request_failed(dir_connection_t *conn)
+{
+  if (conn->guard_state) {
+    /* We haven't seen a success on this guard state, so consider it to have
+     * failed. */
+    entry_guard_failed(&conn->guard_state);
+  }
+  if (directory_conn_is_self_reachability_test(conn)) {
+    return; /* this was a test fetch. don't retry. */
+  }
+  if (!entry_list_is_constrained(get_options()))
+    router_set_status(conn->identity_digest, 0); /* don't try this one again */
+  if (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
+             conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) {
+    log_info(LD_DIR, "Giving up on serverdesc/extrainfo fetch from "
+             "directory server at '%s'; retrying",
+             conn->base_.address);
+    if (conn->router_purpose == ROUTER_PURPOSE_BRIDGE)
+      connection_dir_bridge_routerdesc_failed(conn);
+    connection_dir_download_routerdesc_failed(conn);
+  } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
+    if (conn->requested_resource)
+      networkstatus_consensus_download_failed(0, conn->requested_resource);
+  } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) {
+    log_info(LD_DIR, "Giving up on certificate fetch from directory server "
+             "at '%s'; retrying",
+             conn->base_.address);
+    connection_dir_download_cert_failed(conn, 0);
+  } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) {
+    log_info(LD_DIR, "Giving up downloading detached signatures from '%s'",
+             conn->base_.address);
+  } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) {
+    log_info(LD_DIR, "Giving up downloading votes from '%s'",
+             conn->base_.address);
+  } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+    log_info(LD_DIR, "Giving up on downloading microdescriptors from "
+             "directory server at '%s'; will retry", conn->base_.address);
+    connection_dir_download_routerdesc_failed(conn);
+  }
+}
+
+/** Helper: Attempt to fetch directly the descriptors of each bridge
+ * listed in <b>failed</b>.
+ */
+static void
+connection_dir_retry_bridges(smartlist_t *descs)
+{
+  char digest[DIGEST_LEN];
+  SMARTLIST_FOREACH(descs, const char *, cp,
+  {
+    if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp)) != DIGEST_LEN) {
+      log_warn(LD_BUG, "Malformed fingerprint in list: %s",
+              escaped(cp));
+      continue;
+    }
+    retry_bridge_descriptor_fetch_directly(digest);
+  });
+}
+
+/** Called when an attempt to download one or more router descriptors
+ * or extra-info documents on connection <b>conn</b> failed.
+ */
+static void
+connection_dir_download_routerdesc_failed(dir_connection_t *conn)
+{
+  /* No need to increment the failure count for routerdescs, since
+   * it's not their fault. */
+
+  /* No need to relaunch descriptor downloads here: we already do it
+   * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
+             conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
+             conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
+
+  (void) conn;
+}
+
+/** Called when an attempt to download a bridge's routerdesc from
+ * one of the authorities failed due to a network error. If
+ * possible attempt to download descriptors from the bridge directly.
+ */
+static void
+connection_dir_bridge_routerdesc_failed(dir_connection_t *conn)
+{
+  smartlist_t *which = NULL;
+
+  /* Requests for bridge descriptors are in the form 'fp/', so ignore
+     anything else. */
+  if (!conn->requested_resource || strcmpstart(conn->requested_resource,"fp/"))
+    return;
+
+  which = smartlist_new();
+  dir_split_resource_into_fingerprints(conn->requested_resource
+                                        + strlen("fp/"),
+                                       which, NULL, 0);
+
+  tor_assert(conn->base_.purpose != DIR_PURPOSE_FETCH_EXTRAINFO);
+  if (smartlist_len(which)) {
+    connection_dir_retry_bridges(which);
+    SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
+  }
+  smartlist_free(which);
+}
+
+/** Called when an attempt to fetch a certificate fails. */
+static void
+connection_dir_download_cert_failed(dir_connection_t *conn, int status)
+{
+  const char *fp_pfx = "fp/";
+  const char *fpsk_pfx = "fp-sk/";
+  smartlist_t *failed;
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE);
+
+  if (!conn->requested_resource)
+    return;
+  failed = smartlist_new();
+  /*
+   * We have two cases download by fingerprint (resource starts
+   * with "fp/") or download by fingerprint/signing key pair
+   * (resource starts with "fp-sk/").
+   */
+  if (!strcmpstart(conn->requested_resource, fp_pfx)) {
+    /* Download by fingerprint case */
+    dir_split_resource_into_fingerprints(conn->requested_resource +
+                                         strlen(fp_pfx),
+                                         failed, NULL, DSR_HEX);
+    SMARTLIST_FOREACH_BEGIN(failed, char *, cp) {
+      /* Null signing key digest indicates download by fp only */
+      authority_cert_dl_failed(cp, NULL, status);
+      tor_free(cp);
+    } SMARTLIST_FOREACH_END(cp);
+  } else if (!strcmpstart(conn->requested_resource, fpsk_pfx)) {
+    /* Download by (fp,sk) pairs */
+    dir_split_resource_into_fingerprint_pairs(conn->requested_resource +
+                                              strlen(fpsk_pfx), failed);
+    SMARTLIST_FOREACH_BEGIN(failed, fp_pair_t *, cp) {
+      authority_cert_dl_failed(cp->first, cp->second, status);
+      tor_free(cp);
+    } SMARTLIST_FOREACH_END(cp);
+  } else {
+    log_warn(LD_DIR,
+             "Don't know what to do with failure for cert fetch %s",
+             conn->requested_resource);
+  }
+
+  smartlist_free(failed);
+
+  update_certificate_downloads(time(NULL));
+}
+
+/* Should this tor instance only use begindir for all its directory requests?
+ */
+int
+directory_must_use_begindir(const or_options_t *options)
+{
+  /* Clients, onion services, and bridges must use begindir,
+   * relays and authorities do not have to */
+  return !public_server_mode(options);
+}
+
+/** Evaluate the situation and decide if we should use an encrypted
+ * "begindir-style" connection for this directory request.
+ * 0) If there is no DirPort, yes.
+ * 1) If or_port is 0, or it's a direct conn and or_port is firewalled
+ *    or we're a dir mirror, no.
+ * 2) If we prefer to avoid begindir conns, and we're not fetching or
+ *    publishing a bridge relay descriptor, no.
+ * 3) Else yes.
+ * If returning 0, return in *reason why we can't use begindir.
+ * reason must not be NULL.
+ */
+static int
+directory_command_should_use_begindir(const or_options_t *options,
+                                      const directory_request_t *req,
+                                      const char **reason)
+{
+  const tor_addr_t *or_addr = &req->or_addr_port.addr;
+  //const tor_addr_t *dir_addr = &req->dir_addr_port.addr;
+  const int or_port = req->or_addr_port.port;
+  const int dir_port = req->dir_addr_port.port;
+
+  const dir_indirection_t indirection = req->indirection;
+
+  tor_assert(reason);
+  *reason = NULL;
+
+  /* Reasons why we must use begindir */
+  if (!dir_port) {
+    *reason = "(using begindir - directory with no DirPort)";
+    return 1; /* We don't know a DirPort -- must begindir. */
+  }
+  /* Reasons why we can't possibly use begindir */
+  if (!or_port) {
+    *reason = "directory with unknown ORPort";
+    return 0; /* We don't know an ORPort -- no chance. */
+  }
+  if (indirection == DIRIND_DIRECT_CONN ||
+      indirection == DIRIND_ANON_DIRPORT) {
+    *reason = "DirPort connection";
+    return 0;
+  }
+  if (indirection == DIRIND_ONEHOP) {
+    /* We're firewalled and want a direct OR connection */
+    if (!fascist_firewall_allows_address_addr(or_addr, or_port,
+                                              FIREWALL_OR_CONNECTION, 0, 0)) {
+      *reason = "ORPort not reachable";
+      return 0;
+    }
+  }
+  /* Reasons why we want to avoid using begindir */
+  if (indirection == DIRIND_ONEHOP) {
+    if (!directory_must_use_begindir(options)) {
+      *reason = "in relay mode";
+      return 0;
+    }
+  }
+  /* DIRIND_ONEHOP on a client, or DIRIND_ANONYMOUS
+   */
+  *reason = "(using begindir)";
+  return 1;
+}
+
+/**
+ * Create and return a new directory_request_t with purpose
+ * <b>dir_purpose</b>.
+ */
+directory_request_t *
+directory_request_new(uint8_t dir_purpose)
+{
+  tor_assert(dir_purpose >= DIR_PURPOSE_MIN_);
+  tor_assert(dir_purpose <= DIR_PURPOSE_MAX_);
+  tor_assert(dir_purpose != DIR_PURPOSE_SERVER);
+  tor_assert(dir_purpose != DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2);
+  tor_assert(dir_purpose != DIR_PURPOSE_HAS_FETCHED_HSDESC);
+
+  directory_request_t *result = tor_malloc_zero(sizeof(*result));
+  tor_addr_make_null(&result->or_addr_port.addr, AF_INET);
+  result->or_addr_port.port = 0;
+  tor_addr_make_null(&result->dir_addr_port.addr, AF_INET);
+  result->dir_addr_port.port = 0;
+  result->dir_purpose = dir_purpose;
+  result->router_purpose = ROUTER_PURPOSE_GENERAL;
+  result->indirection = DIRIND_ONEHOP;
+  return result;
+}
+/**
+ * Release all resources held by <b>req</b>.
+ */
+void
+directory_request_free_(directory_request_t *req)
+{
+  if (req == NULL)
+    return;
+  config_free_lines(req->additional_headers);
+  tor_free(req);
+}
+/**
+ * Set the address and OR port to use for this directory request.  If there is
+ * no OR port, we'll have to connect over the dirport.  (If there are both,
+ * the indirection setting determines which to use.)
+ */
+void
+directory_request_set_or_addr_port(directory_request_t *req,
+                                   const tor_addr_port_t *p)
+{
+  memcpy(&req->or_addr_port, p, sizeof(*p));
+}
+/**
+ * Set the address and dirport to use for this directory request.  If there
+ * is no dirport, we'll have to connect over the OR port.  (If there are both,
+ * the indirection setting determines which to use.)
+ */
+void
+directory_request_set_dir_addr_port(directory_request_t *req,
+                                    const tor_addr_port_t *p)
+{
+  memcpy(&req->dir_addr_port, p, sizeof(*p));
+}
+/**
+ * Set the RSA identity digest of the directory to use for this directory
+ * request.
+ */
+void
+directory_request_set_directory_id_digest(directory_request_t *req,
+                                          const char *digest)
+{
+  memcpy(req->digest, digest, DIGEST_LEN);
+}
+/**
+ * Set the router purpose associated with uploaded and downloaded router
+ * descriptors and extrainfo documents in this directory request.  The purpose
+ * must be one of ROUTER_PURPOSE_GENERAL (the default) or
+ * ROUTER_PURPOSE_BRIDGE.
+ */
+void
+directory_request_set_router_purpose(directory_request_t *req,
+                                     uint8_t router_purpose)
+{
+  tor_assert(router_purpose == ROUTER_PURPOSE_GENERAL ||
+             router_purpose == ROUTER_PURPOSE_BRIDGE);
+  // assert that it actually makes sense to set this purpose, given
+  // the dir_purpose.
+  req->router_purpose = router_purpose;
+}
+/**
+ * Set the indirection to be used for the directory request.  The indirection
+ * parameter configures whether to connect to a DirPort or ORPort, and whether
+ * to anonymize the connection.  DIRIND_ONEHOP (use ORPort, don't anonymize)
+ * is the default.  See dir_indirection_t for more information.
+ */
+void
+directory_request_set_indirection(directory_request_t *req,
+                                  dir_indirection_t indirection)
+{
+  req->indirection = indirection;
+}
+
+/**
+ * Set a pointer to the resource to request from a directory.  Different
+ * request types use resources to indicate different components of their URL.
+ * Note that only an alias to <b>resource</b> is stored, so the
+ * <b>resource</b> must outlive the request.
+ */
+void
+directory_request_set_resource(directory_request_t *req,
+                               const char *resource)
+{
+  req->resource = resource;
+}
+/**
+ * Set a pointer to the payload to include with this directory request, along
+ * with its length.  Note that only an alias to <b>payload</b> is stored, so
+ * the <b>payload</b> must outlive the request.
+ */
+void
+directory_request_set_payload(directory_request_t *req,
+                              const char *payload,
+                              size_t payload_len)
+{
+  tor_assert(DIR_PURPOSE_IS_UPLOAD(req->dir_purpose));
+
+  req->payload = payload;
+  req->payload_len = payload_len;
+}
+/**
+ * Set an if-modified-since date to send along with the request.  The
+ * default is 0 (meaning, send no if-modified-since header).
+ */
+void
+directory_request_set_if_modified_since(directory_request_t *req,
+                                        time_t if_modified_since)
+{
+  req->if_modified_since = if_modified_since;
+}
+
+/** Include a header of name <b>key</b> with content <b>val</b> in the
+ * request. Neither may include newlines or other odd characters. Their
+ * ordering is not currently guaranteed.
+ *
+ * Note that, as elsewhere in this module, header keys include a trailing
+ * colon and space.
+ */
+void
+directory_request_add_header(directory_request_t *req,
+                             const char *key,
+                             const char *val)
+{
+  config_line_prepend(&req->additional_headers, key, val);
+}
+/**
+ * Set an object containing HS data to be associated with this request.  Note
+ * that only an alias to <b>query</b> is stored, so the <b>query</b> object
+ * must outlive the request.
+ */
+void
+directory_request_set_rend_query(directory_request_t *req,
+                                 const rend_data_t *query)
+{
+  if (query) {
+    tor_assert(req->dir_purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 ||
+               req->dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2);
+  }
+  req->rend_query = query;
+}
+/**
+ * Set an object containing HS connection identifier to be associated with
+ * this request. Note that only an alias to <b>ident</b> is stored, so the
+ * <b>ident</b> object must outlive the request.
+ */
+void
+directory_request_upload_set_hs_ident(directory_request_t *req,
+                                      const hs_ident_dir_conn_t *ident)
+{
+  if (ident) {
+    tor_assert(req->dir_purpose == DIR_PURPOSE_UPLOAD_HSDESC);
+  }
+  req->hs_ident = ident;
+}
+/**
+ * Set an object containing HS connection identifier to be associated with
+ * this fetch request. Note that only an alias to <b>ident</b> is stored, so
+ * the <b>ident</b> object must outlive the request.
+ */
+void
+directory_request_fetch_set_hs_ident(directory_request_t *req,
+                                     const hs_ident_dir_conn_t *ident)
+{
+  if (ident) {
+    tor_assert(req->dir_purpose == DIR_PURPOSE_FETCH_HSDESC);
+  }
+  req->hs_ident = ident;
+}
+/** Set a static circuit_guard_state_t object to affliate with the request in
+ * <b>req</b>.  This object will receive notification when the attempt to
+ * connect to the guard either succeeds or fails. */
+void
+directory_request_set_guard_state(directory_request_t *req,
+                                  circuit_guard_state_t *state)
+{
+  req->guard_state = state;
+}
+
+/**
+ * Internal: Return true if any information for contacting the directory in
+ * <b>req</b> has been set, other than by the routerstatus. */
+static int
+directory_request_dir_contact_info_specified(const directory_request_t *req)
+{
+  /* We only check for ports here, since we don't use an addr unless the port
+   * is set */
+  return (req->or_addr_port.port ||
+          req->dir_addr_port.port ||
+          ! tor_digest_is_zero(req->digest));
+}
+
+/**
+ * Set the routerstatus to use for the directory associated with this
+ * request.  If this option is set, then no other function to set the
+ * directory's address or identity should be called.
+ */
+void
+directory_request_set_routerstatus(directory_request_t *req,
+                                   const routerstatus_t *status)
+{
+  req->routerstatus = status;
+}
+/**
+ * Helper: update the addresses, ports, and identities in <b>req</b>
+ * from the routerstatus object in <b>req</b>.  Return 0 on success.
+ * On failure, warn and return -1.
+ */
+static int
+directory_request_set_dir_from_routerstatus(directory_request_t *req)
+
+{
+  const routerstatus_t *status = req->routerstatus;
+  if (BUG(status == NULL))
+    return -1;
+  const or_options_t *options = get_options();
+  const node_t *node;
+  tor_addr_port_t use_or_ap, use_dir_ap;
+  const int anonymized_connection = dirind_is_anon(req->indirection);
+
+  tor_assert(status != NULL);
+
+  node = node_get_by_id(status->identity_digest);
+
+  /* XXX The below check is wrong: !node means it's not in the consensus,
+   * but we haven't checked if we have a descriptor for it -- and also,
+   * we only care about the descriptor if it's a begindir-style anonymized
+   * connection. */
+  if (!node && anonymized_connection) {
+    log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we "
+             "don't have its router descriptor.",
+             routerstatus_describe(status));
+    return -1;
+  }
+
+  if (options->ExcludeNodes && options->StrictNodes &&
+      routerset_contains_routerstatus(options->ExcludeNodes, status, -1)) {
+    log_warn(LD_DIR, "Wanted to contact directory mirror %s for %s, but "
+             "it's in our ExcludedNodes list and StrictNodes is set. "
+             "Skipping. This choice might make your Tor not work.",
+             routerstatus_describe(status),
+             dir_conn_purpose_to_string(req->dir_purpose));
+    return -1;
+  }
+
+    /* At this point, if we are a client making a direct connection to a
+   * directory server, we have selected a server that has at least one address
+   * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This
+   * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if
+   * possible. (If UseBridges is set, clients always use IPv6, and prefer it
+   * by default.)
+   *
+   * Now choose an address that we can use to connect to the directory server.
+   */
+  if (directory_choose_address_routerstatus(status,
+                                            req->indirection, &use_or_ap,
+                                            &use_dir_ap) < 0) {
+    return -1;
+  }
+
+  directory_request_set_or_addr_port(req, &use_or_ap);
+  directory_request_set_dir_addr_port(req, &use_dir_ap);
+  directory_request_set_directory_id_digest(req, status->identity_digest);
+  return 0;
+}
+
+/**
+ * Launch the provided directory request, configured in <b>request</b>.
+ * After this function is called, you can free <b>request</b>.
+ */
+MOCK_IMPL(void,
+directory_initiate_request,(directory_request_t *request))
+{
+  tor_assert(request);
+  if (request->routerstatus) {
+    tor_assert_nonfatal(
+               ! directory_request_dir_contact_info_specified(request));
+    if (directory_request_set_dir_from_routerstatus(request) < 0) {
+      return;
+    }
+  }
+
+  const tor_addr_port_t *or_addr_port = &request->or_addr_port;
+  const tor_addr_port_t *dir_addr_port = &request->dir_addr_port;
+  const char *digest = request->digest;
+  const uint8_t dir_purpose = request->dir_purpose;
+  const uint8_t router_purpose = request->router_purpose;
+  const dir_indirection_t indirection = request->indirection;
+  const char *resource = request->resource;
+  const rend_data_t *rend_query = request->rend_query;
+  const hs_ident_dir_conn_t *hs_ident = request->hs_ident;
+  circuit_guard_state_t *guard_state = request->guard_state;
+
+  tor_assert(or_addr_port->port || dir_addr_port->port);
+  tor_assert(digest);
+
+  dir_connection_t *conn;
+  const or_options_t *options = get_options();
+  int socket_error = 0;
+  const char *begindir_reason = NULL;
+  /* Should the connection be to a relay's OR port (and inside that we will
+   * send our directory request)? */
+  const int use_begindir =
+    directory_command_should_use_begindir(options, request, &begindir_reason);
+
+  /* Will the connection go via a three-hop Tor circuit? Note that this
+   * is separate from whether it will use_begindir. */
+  const int anonymized_connection = dirind_is_anon(indirection);
+
+  /* What is the address we want to make the directory request to? If
+   * we're making a begindir request this is the ORPort of the relay
+   * we're contacting; if not a begindir request, this is its DirPort.
+   * Note that if anonymized_connection is true, we won't be initiating
+   * a connection directly to this address. */
+  tor_addr_t addr;
+  tor_addr_copy(&addr, &(use_begindir ? or_addr_port : dir_addr_port)->addr);
+  uint16_t port = (use_begindir ? or_addr_port : dir_addr_port)->port;
+
+  log_debug(LD_DIR, "anonymized %d, use_begindir %d.",
+            anonymized_connection, use_begindir);
+
+  log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose));
+
+  if (purpose_needs_anonymity(dir_purpose, router_purpose, resource)) {
+    tor_assert(anonymized_connection ||
+               rend_non_anonymous_mode_enabled(options));
+  }
+
+  /* use encrypted begindir connections for everything except relays
+   * this provides better protection for directory fetches */
+  if (!use_begindir && directory_must_use_begindir(options)) {
+    log_warn(LD_BUG, "Client could not use begindir connection: %s",
+             begindir_reason ? begindir_reason : "(NULL)");
+    return;
+  }
+
+  /* ensure that we don't make direct connections when a SOCKS server is
+   * configured. */
+  if (!anonymized_connection && !use_begindir && !options->HTTPProxy &&
+      (options->Socks4Proxy || options->Socks5Proxy)) {
+    log_warn(LD_DIR, "Cannot connect to a directory server through a "
+                     "SOCKS proxy!");
+    return;
+  }
+
+  /* Make sure that the destination addr and port we picked is viable. */
+  if (!port || tor_addr_is_null(&addr)) {
+    static int logged_backtrace = 0;
+    log_warn(LD_DIR,
+             "Cannot make an outgoing %sconnection without a remote %sPort.",
+             use_begindir ? "begindir " : "",
+             use_begindir ? "OR" : "Dir");
+    if (!logged_backtrace) {
+      log_backtrace(LOG_INFO, LD_BUG, "Address came from");
+      logged_backtrace = 1;
+    }
+    return;
+  }
+
+  conn = dir_connection_new(tor_addr_family(&addr));
+
+  /* set up conn so it's got all the data we need to remember */
+  tor_addr_copy(&conn->base_.addr, &addr);
+  conn->base_.port = port;
+  conn->base_.address = tor_addr_to_str_dup(&addr);
+  memcpy(conn->identity_digest, digest, DIGEST_LEN);
+
+  conn->base_.purpose = dir_purpose;
+  conn->router_purpose = router_purpose;
+
+  /* give it an initial state */
+  conn->base_.state = DIR_CONN_STATE_CONNECTING;
+
+  /* decide whether we can learn our IP address from this conn */
+  /* XXXX This is a bad name for this field now. */
+  conn->dirconn_direct = !anonymized_connection;
+
+  /* copy rendezvous data, if any */
+  if (rend_query) {
+    /* We can't have both v2 and v3+ identifier. */
+    tor_assert_nonfatal(!hs_ident);
+    conn->rend_data = rend_data_dup(rend_query);
+  }
+  if (hs_ident) {
+    /* We can't have both v2 and v3+ identifier. */
+    tor_assert_nonfatal(!rend_query);
+    conn->hs_ident = hs_ident_dir_conn_dup(hs_ident);
+  }
+
+  if (!anonymized_connection && !use_begindir) {
+    /* then we want to connect to dirport directly */
+
+    if (options->HTTPProxy) {
+      tor_addr_copy(&addr, &options->HTTPProxyAddr);
+      port = options->HTTPProxyPort;
+    }
+
+    // In this case we should not have picked a directory guard.
+    if (BUG(guard_state)) {
+      entry_guard_cancel(&guard_state);
+    }
+
+    switch (connection_connect(TO_CONN(conn), conn->base_.address, &addr,
+                               port, &socket_error)) {
+      case -1:
+        connection_mark_for_close(TO_CONN(conn));
+        return;
+      case 1:
+        /* start flushing conn */
+        conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING;
+        /* fall through */
+      case 0:
+        /* queue the command on the outbuf */
+        directory_send_command(conn, 1, request);
+        connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT);
+        /* writable indicates finish, readable indicates broken link,
+           error indicates broken link in windowsland. */
+    }
+  } else {
+    /* We will use a Tor circuit (maybe 1-hop, maybe 3-hop, maybe with
+     * begindir, maybe not with begindir) */
+
+    entry_connection_t *linked_conn;
+
+    /* Anonymized tunneled connections can never share a circuit.
+     * One-hop directory connections can share circuits with each other
+     * but nothing else. */
+    int iso_flags = anonymized_connection ? ISO_STREAM : ISO_SESSIONGRP;
+
+    /* If it's an anonymized connection, remember the fact that we
+     * wanted it for later: maybe we'll want it again soon. */
+    if (anonymized_connection && use_begindir)
+      rep_hist_note_used_internal(time(NULL), 0, 1);
+    else if (anonymized_connection && !use_begindir)
+      rep_hist_note_used_port(time(NULL), conn->base_.port);
+
+    // In this case we should not have a directory guard; we'll
+    // get a regular guard later when we build the circuit.
+    if (BUG(anonymized_connection && guard_state)) {
+      entry_guard_cancel(&guard_state);
+    }
+
+    conn->guard_state = guard_state;
+
+    /* make an AP connection
+     * populate it and add it at the right state
+     * hook up both sides
+     */
+    linked_conn =
+      connection_ap_make_link(TO_CONN(conn),
+                              conn->base_.address, conn->base_.port,
+                              digest,
+                              SESSION_GROUP_DIRCONN, iso_flags,
+                              use_begindir, !anonymized_connection);
+    if (!linked_conn) {
+      log_warn(LD_NET,"Making tunnel to dirserver failed.");
+      connection_mark_for_close(TO_CONN(conn));
+      return;
+    }
+
+    if (connection_add(TO_CONN(conn)) < 0) {
+      log_warn(LD_NET,"Unable to add connection for link to dirserver.");
+      connection_mark_for_close(TO_CONN(conn));
+      return;
+    }
+    conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING;
+    /* queue the command on the outbuf */
+    directory_send_command(conn, 0, request);
+
+    connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT);
+    connection_start_reading(ENTRY_TO_CONN(linked_conn));
+  }
+}
+
+/** Helper for sorting
+ *
+ * sort strings alphabetically
+ *
+ * XXXX we have a smartlist_sort_strings() function, right?
+ */
+static int
+compare_strs_(const void **a, const void **b)
+{
+  const char *s1 = *a, *s2 = *b;
+  return strcmp(s1, s2);
+}
+
+#define CONDITIONAL_CONSENSUS_FPR_LEN 3
+#if (CONDITIONAL_CONSENSUS_FPR_LEN > DIGEST_LEN)
+#error "conditional consensus fingerprint length is larger than digest length"
+#endif
+
+/** Return the URL we should use for a consensus download.
+ *
+ * Use the "conditional consensus downloading" feature described in
+ * dir-spec.txt, i.e.
+ * GET .../consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>
+ *
+ * If 'resource' is provided, it is the name of a consensus flavor to request.
+ */
+static char *
+directory_get_consensus_url(const char *resource)
+{
+  char *url = NULL;
+  const char *hyphen, *flavor;
+  if (resource==NULL || strcmp(resource, "ns")==0) {
+    flavor = ""; /* Request ns consensuses as "", so older servers will work*/
+    hyphen = "";
+  } else {
+    flavor = resource;
+    hyphen = "-";
+  }
+
+  {
+    char *authority_id_list;
+    smartlist_t *authority_digests = smartlist_new();
+
+    SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
+                            dir_server_t *, ds) {
+        char *hex;
+        if (!(ds->type & V3_DIRINFO))
+          continue;
+
+        hex = tor_malloc(2*CONDITIONAL_CONSENSUS_FPR_LEN+1);
+        base16_encode(hex, 2*CONDITIONAL_CONSENSUS_FPR_LEN+1,
+                      ds->v3_identity_digest, CONDITIONAL_CONSENSUS_FPR_LEN);
+        smartlist_add(authority_digests, hex);
+    } SMARTLIST_FOREACH_END(ds);
+    smartlist_sort(authority_digests, compare_strs_);
+    authority_id_list = smartlist_join_strings(authority_digests,
+                                               "+", 0, NULL);
+
+    tor_asprintf(&url, "/tor/status-vote/current/consensus%s%s/%s.z",
+                 hyphen, flavor, authority_id_list);
+
+    SMARTLIST_FOREACH(authority_digests, char *, cp, tor_free(cp));
+    smartlist_free(authority_digests);
+    tor_free(authority_id_list);
+  }
+  return url;
+}
+
+/**
+ * Copies the ipv6 from source to destination, subject to buffer size limit
+ * size. If decorate is true, makes sure the copied address is decorated.
+ */
+static void
+copy_ipv6_address(char* destination, const char* source, size_t len,
+                  int decorate) {
+  tor_assert(destination);
+  tor_assert(source);
+
+  if (decorate && source[0] != '[') {
+    tor_snprintf(destination, len, "[%s]", source);
+  } else {
+    strlcpy(destination, source, len);
+  }
+}
+
+/** Queue an appropriate HTTP command for <b>request</b> on
+ * <b>conn</b>-\>outbuf.  If <b>direct</b> is true, we're making a
+ * non-anonymized connection to the dirport.
+ */
+static void
+directory_send_command(dir_connection_t *conn,
+                       const int direct,
+                       const directory_request_t *req)
+{
+  tor_assert(req);
+  const int purpose = req->dir_purpose;
+  const char *resource = req->resource;
+  const char *payload = req->payload;
+  const size_t payload_len = req->payload_len;
+  const time_t if_modified_since = req->if_modified_since;
+  const int anonymized_connection = dirind_is_anon(req->indirection);
+
+  char proxystring[256];
+  char hoststring[128];
+  /* NEEDS to be the same size hoststring.
+   Will be decorated with brackets around it if it is ipv6. */
+  char decorated_address[128];
+  smartlist_t *headers = smartlist_new();
+  char *url;
+  char *accept_encoding;
+  size_t url_len;
+  char request[8192];
+  size_t request_len, total_request_len = 0;
+  const char *httpcommand = NULL;
+
+  tor_assert(conn);
+  tor_assert(conn->base_.type == CONN_TYPE_DIR);
+
+  tor_free(conn->requested_resource);
+  if (resource)
+    conn->requested_resource = tor_strdup(resource);
+
+  /* decorate the ip address if it is ipv6 */
+  if (strchr(conn->base_.address, ':')) {
+    copy_ipv6_address(decorated_address, conn->base_.address,
+                      sizeof(decorated_address), 1);
+  } else {
+    strlcpy(decorated_address, conn->base_.address, sizeof(decorated_address));
+  }
+
+  /* come up with a string for which Host: we want */
+  if (conn->base_.port == 80) {
+    strlcpy(hoststring, decorated_address, sizeof(hoststring));
+  } else {
+    tor_snprintf(hoststring, sizeof(hoststring), "%s:%d",
+                 decorated_address, conn->base_.port);
+  }
+
+  /* Format if-modified-since */
+  if (if_modified_since) {
+    char b[RFC1123_TIME_LEN+1];
+    format_rfc1123_time(b, if_modified_since);
+    smartlist_add_asprintf(headers, "If-Modified-Since: %s\r\n", b);
+  }
+
+  /* come up with some proxy lines, if we're using one. */
+  if (direct && get_options()->HTTPProxy) {
+    char *base64_authenticator=NULL;
+    const char *authenticator = get_options()->HTTPProxyAuthenticator;
+
+    tor_snprintf(proxystring, sizeof(proxystring),"http://%s", hoststring);
+    if (authenticator) {
+      base64_authenticator = alloc_http_authenticator(authenticator);
+      if (!base64_authenticator)
+        log_warn(LD_BUG, "Encoding http authenticator failed");
+    }
+    if (base64_authenticator) {
+      smartlist_add_asprintf(headers,
+                   "Proxy-Authorization: Basic %s\r\n",
+                   base64_authenticator);
+      tor_free(base64_authenticator);
+    }
+  } else {
+    proxystring[0] = 0;
+  }
+
+  if (! anonymized_connection) {
+    /* Add Accept-Encoding. */
+    accept_encoding = accept_encoding_header();
+    smartlist_add_asprintf(headers, "Accept-Encoding: %s\r\n",
+                           accept_encoding);
+    tor_free(accept_encoding);
+  }
+
+  /* Add additional headers, if any */
+  {
+    config_line_t *h;
+    for (h = req->additional_headers; h; h = h->next) {
+      smartlist_add_asprintf(headers, "%s%s\r\n", h->key, h->value);
+    }
+  }
+
+  switch (purpose) {
+    case DIR_PURPOSE_FETCH_CONSENSUS:
+      /* resource is optional.  If present, it's a flavor name */
+      tor_assert(!payload);
+      httpcommand = "GET";
+      url = directory_get_consensus_url(resource);
+      log_info(LD_DIR, "Downloading consensus from %s using %s",
+               hoststring, url);
+      break;
+    case DIR_PURPOSE_FETCH_CERTIFICATE:
+      tor_assert(resource);
+      tor_assert(!payload);
+      httpcommand = "GET";
+      tor_asprintf(&url, "/tor/keys/%s", resource);
+      break;
+    case DIR_PURPOSE_FETCH_STATUS_VOTE:
+      tor_assert(resource);
+      tor_assert(!payload);
+      httpcommand = "GET";
+      tor_asprintf(&url, "/tor/status-vote/next/%s.z", resource);
+      break;
+    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
+      tor_assert(!resource);
+      tor_assert(!payload);
+      httpcommand = "GET";
+      url = tor_strdup("/tor/status-vote/next/consensus-signatures.z");
+      break;
+    case DIR_PURPOSE_FETCH_SERVERDESC:
+      tor_assert(resource);
+      httpcommand = "GET";
+      tor_asprintf(&url, "/tor/server/%s", resource);
+      break;
+    case DIR_PURPOSE_FETCH_EXTRAINFO:
+      tor_assert(resource);
+      httpcommand = "GET";
+      tor_asprintf(&url, "/tor/extra/%s", resource);
+      break;
+    case DIR_PURPOSE_FETCH_MICRODESC:
+      tor_assert(resource);
+      httpcommand = "GET";
+      tor_asprintf(&url, "/tor/micro/%s", resource);
+      break;
+    case DIR_PURPOSE_UPLOAD_DIR: {
+      const char *why = router_get_descriptor_gen_reason();
+      tor_assert(!resource);
+      tor_assert(payload);
+      httpcommand = "POST";
+      url = tor_strdup("/tor/");
+      if (!why) {
+        why = "for no reason at all";
+      }
+      smartlist_add_asprintf(headers, "X-Desc-Gen-Reason: %s\r\n", why);
+      break;
+    }
+    case DIR_PURPOSE_UPLOAD_VOTE:
+      tor_assert(!resource);
+      tor_assert(payload);
+      httpcommand = "POST";
+      url = tor_strdup("/tor/post/vote");
+      break;
+    case DIR_PURPOSE_UPLOAD_SIGNATURES:
+      tor_assert(!resource);
+      tor_assert(payload);
+      httpcommand = "POST";
+      url = tor_strdup("/tor/post/consensus-signature");
+      break;
+    case DIR_PURPOSE_FETCH_RENDDESC_V2:
+      tor_assert(resource);
+      tor_assert(strlen(resource) <= REND_DESC_ID_V2_LEN_BASE32);
+      tor_assert(!payload);
+      httpcommand = "GET";
+      tor_asprintf(&url, "/tor/rendezvous2/%s", resource);
+      break;
+    case DIR_PURPOSE_FETCH_HSDESC:
+      tor_assert(resource);
+      tor_assert(strlen(resource) <= ED25519_BASE64_LEN);
+      tor_assert(!payload);
+      httpcommand = "GET";
+      tor_asprintf(&url, "/tor/hs/3/%s", resource);
+      break;
+    case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
+      tor_assert(!resource);
+      tor_assert(payload);
+      httpcommand = "POST";
+      url = tor_strdup("/tor/rendezvous2/publish");
+      break;
+    case DIR_PURPOSE_UPLOAD_HSDESC:
+      tor_assert(resource);
+      tor_assert(payload);
+      httpcommand = "POST";
+      tor_asprintf(&url, "/tor/hs/%s/publish", resource);
+      break;
+    default:
+      tor_assert(0);
+      return;
+  }
+
+  /* warn in the non-tunneled case */
+  if (direct && (strlen(proxystring) + strlen(url) >= 4096)) {
+    log_warn(LD_BUG,
+             "Squid does not like URLs longer than 4095 bytes, and this "
+             "one is %d bytes long: %s%s",
+             (int)(strlen(proxystring) + strlen(url)), proxystring, url);
+  }
+
+  tor_snprintf(request, sizeof(request), "%s %s", httpcommand, proxystring);
+
+  request_len = strlen(request);
+  total_request_len += request_len;
+  connection_buf_add(request, request_len, TO_CONN(conn));
+
+  url_len = strlen(url);
+  total_request_len += url_len;
+  connection_buf_add(url, url_len, TO_CONN(conn));
+  tor_free(url);
+
+  if (!strcmp(httpcommand, "POST") || payload) {
+    smartlist_add_asprintf(headers, "Content-Length: %lu\r\n",
+                 payload ? (unsigned long)payload_len : 0);
+  }
+
+  {
+    char *header = smartlist_join_strings(headers, "", 0, NULL);
+    tor_snprintf(request, sizeof(request), " HTTP/1.0\r\nHost: %s\r\n%s\r\n",
+                 hoststring, header);
+    tor_free(header);
+  }
+
+  request_len = strlen(request);
+  total_request_len += request_len;
+  connection_buf_add(request, request_len, TO_CONN(conn));
+
+  if (payload) {
+    /* then send the payload afterwards too */
+    connection_buf_add(payload, payload_len, TO_CONN(conn));
+    total_request_len += payload_len;
+  }
+
+  SMARTLIST_FOREACH(headers, char *, h, tor_free(h));
+  smartlist_free(headers);
+
+  log_debug(LD_DIR,
+            "Sent request to directory server '%s:%d': "
+            "(purpose: %d, request size: %"TOR_PRIuSZ", "
+            "payload size: %"TOR_PRIuSZ")",
+            conn->base_.address, conn->base_.port,
+            conn->base_.purpose,
+            (total_request_len),
+            (payload ? payload_len : 0));
+}
+
+/** Return true iff <b>body</b> doesn't start with a plausible router or
+ * network-status or microdescriptor opening.  This is a sign of possible
+ * compression. */
+static int
+body_is_plausible(const char *body, size_t len, int purpose)
+{
+  int i;
+  if (len == 0)
+    return 1; /* empty bodies don't need decompression */
+  if (len < 32)
+    return 0;
+  if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+    return (!strcmpstart(body,"onion-key"));
+  }
+
+  if (!strcmpstart(body,"router") ||
+      !strcmpstart(body,"network-status"))
+    return 1;
+  for (i=0;i<32;++i) {
+    if (!TOR_ISPRINT(body[i]) && !TOR_ISSPACE(body[i]))
+      return 0;
+  }
+
+  return 1;
+}
+
+/** Called when we've just fetched a bunch of router descriptors in
+ * <b>body</b>.  The list <b>which</b>, if present, holds digests for
+ * descriptors we requested: descriptor digests if <b>descriptor_digests</b>
+ * is true, or identity digests otherwise.  Parse the descriptors, validate
+ * them, and annotate them as having purpose <b>purpose</b> and as having been
+ * downloaded from <b>source</b>.
+ *
+ * Return the number of routers actually added. */
+static int
+load_downloaded_routers(const char *body, smartlist_t *which,
+                        int descriptor_digests,
+                        int router_purpose,
+                        const char *source)
+{
+  char buf[256];
+  char time_buf[ISO_TIME_LEN+1];
+  int added = 0;
+  int general = router_purpose == ROUTER_PURPOSE_GENERAL;
+  format_iso_time(time_buf, time(NULL));
+  tor_assert(source);
+
+  if (tor_snprintf(buf, sizeof(buf),
+                   "@downloaded-at %s\n"
+                   "@source %s\n"
+                   "%s%s%s", time_buf, escaped(source),
+                   !general ? "@purpose " : "",
+                   !general ? router_purpose_to_string(router_purpose) : "",
+                   !general ? "\n" : "")<0)
+    return added;
+
+  added = router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which,
+                                  descriptor_digests, buf);
+  if (added && general)
+    control_event_boot_dir(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS,
+                           count_loading_descriptors_progress());
+  return added;
+}
+
+static int handle_response_fetch_certificate(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_fetch_status_vote(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_fetch_detached_signatures(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_fetch_desc(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_upload_dir(dir_connection_t *,
+                                      const response_handler_args_t *);
+static int handle_response_upload_vote(dir_connection_t *,
+                                       const response_handler_args_t *);
+static int handle_response_upload_signatures(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_fetch_renddesc_v2(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_upload_renddesc_v2(dir_connection_t *,
+                                              const response_handler_args_t *);
+static int handle_response_upload_hsdesc(dir_connection_t *,
+                                         const response_handler_args_t *);
+
+static int
+dir_client_decompress_response_body(char **bodyp, size_t *bodylenp,
+                                    dir_connection_t *conn,
+                                    compress_method_t compression,
+                                    int anonymized_connection)
+{
+  int rv = 0;
+  const char *body = *bodyp;
+  size_t body_len = *bodylenp;
+  int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
+                       conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
+                       conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
+
+  int plausible = body_is_plausible(body, body_len, conn->base_.purpose);
+
+  if (plausible && compression == NO_METHOD) {
+    return 0;
+  }
+
+  int severity = LOG_DEBUG;
+  char *new_body = NULL;
+  size_t new_len = 0;
+  const char *description1, *description2;
+  int want_to_try_both = 0;
+  int tried_both = 0;
+  compress_method_t guessed = detect_compression_method(body, body_len);
+
+  description1 = compression_method_get_human_name(compression);
+
+  if (BUG(description1 == NULL))
+    description1 = compression_method_get_human_name(UNKNOWN_METHOD);
+
+  if (guessed == UNKNOWN_METHOD && !plausible)
+    description2 = "confusing binary junk";
+  else
+    description2 = compression_method_get_human_name(guessed);
+
+  /* Tell the user if we don't believe what we're told about compression.*/
+  want_to_try_both = (compression == UNKNOWN_METHOD ||
+                      guessed != compression);
+  if (want_to_try_both) {
+    severity = LOG_PROTOCOL_WARN;
+  }
+
+  tor_log(severity, LD_HTTP,
+          "HTTP body from server '%s:%d' was labeled as %s, "
+          "%s it seems to be %s.%s",
+          conn->base_.address, conn->base_.port, description1,
+          guessed != compression?"but":"and",
+          description2,
+          (compression>0 && guessed>0 && want_to_try_both)?
+          "  Trying both.":"");
+
+  /* Try declared compression first if we can.
+   * tor_compress_supports_method() also returns true for NO_METHOD.
+   * Ensure that the server is not sending us data compressed using a
+   * compression method that is not allowed for anonymous connections. */
+  if (anonymized_connection &&
+      ! allowed_anonymous_connection_compression_method(compression)) {
+    warn_disallowed_anonymous_compression_method(compression);
+    rv = -1;
+    goto done;
+  }
+
+  if (tor_compress_supports_method(compression)) {
+    tor_uncompress(&new_body, &new_len, body, body_len, compression,
+                   !allow_partial, LOG_PROTOCOL_WARN);
+    if (new_body) {
+      /* We succeeded with the declared compression method. Great! */
+      rv = 0;
+      goto done;
+    }
+  }
+
+  /* Okay, if that didn't work, and we think that it was compressed
+   * differently, try that. */
+  if (anonymized_connection &&
+      ! allowed_anonymous_connection_compression_method(guessed)) {
+    warn_disallowed_anonymous_compression_method(guessed);
+    rv = -1;
+    goto done;
+  }
+
+  if (tor_compress_supports_method(guessed) &&
+      compression != guessed) {
+    tor_uncompress(&new_body, &new_len, body, body_len, guessed,
+                   !allow_partial, LOG_INFO);
+    tried_both = 1;
+  }
+  /* If we're pretty sure that we have a compressed directory, and
+   * we didn't manage to uncompress it, then warn and bail. */
+  if (!plausible && !new_body) {
+    log_fn(LOG_PROTOCOL_WARN, LD_HTTP,
+           "Unable to decompress HTTP body (tried %s%s%s, server '%s:%d').",
+           description1,
+           tried_both?" and ":"",
+           tried_both?description2:"",
+           conn->base_.address, conn->base_.port);
+    rv = -1;
+    goto done;
+  }
+
+ done:
+  if (new_body) {
+    if (rv == 0) {
+      /* success! */
+      tor_free(*bodyp);
+      *bodyp = new_body;
+      *bodylenp = new_len;
+    } else {
+      tor_free(new_body);
+    }
+  }
+
+  return rv;
+}
+
+/** We are a client, and we've finished reading the server's
+ * response. Parse it and act appropriately.
+ *
+ * If we're still happy with using this directory server in the future, return
+ * 0. Otherwise return -1; and the caller should consider trying the request
+ * again.
+ *
+ * The caller will take care of marking the connection for close.
+ */
+static int
+connection_dir_client_reached_eof(dir_connection_t *conn)
+{
+  char *body = NULL;
+  char *headers = NULL;
+  char *reason = NULL;
+  size_t body_len = 0;
+  int status_code;
+  time_t date_header = 0;
+  long apparent_skew;
+  compress_method_t compression;
+  int skewed = 0;
+  int rv;
+  int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
+                       conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
+                       conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
+  size_t received_bytes;
+  const int anonymized_connection =
+    purpose_needs_anonymity(conn->base_.purpose,
+                            conn->router_purpose,
+                            conn->requested_resource);
+
+  received_bytes = connection_get_inbuf_len(TO_CONN(conn));
+
+  switch (connection_fetch_from_buf_http(TO_CONN(conn),
+                              &headers, MAX_HEADERS_SIZE,
+                              &body, &body_len, MAX_DIR_DL_SIZE,
+                              allow_partial)) {
+    case -1: /* overflow */
+      log_warn(LD_PROTOCOL,
+               "'fetch' response too large (server '%s:%d'). Closing.",
+               conn->base_.address, conn->base_.port);
+      return -1;
+    case 0:
+      log_info(LD_HTTP,
+               "'fetch' response not all here, but we're at eof. Closing.");
+      return -1;
+    /* case 1, fall through */
+  }
+
+  if (parse_http_response(headers, &status_code, &date_header,
+                          &compression, &reason) < 0) {
+    log_warn(LD_HTTP,"Unparseable headers (server '%s:%d'). Closing.",
+             conn->base_.address, conn->base_.port);
+
+    rv = -1;
+    goto done;
+  }
+  if (!reason) reason = tor_strdup("[no reason given]");
+
+  tor_log(LOG_DEBUG, LD_DIR,
+            "Received response from directory server '%s:%d': %d %s "
+            "(purpose: %d, response size: %"TOR_PRIuSZ
+#ifdef MEASUREMENTS_21206
+            ", data cells received: %d, data cells sent: %d"
+#endif
+            ", compression: %d)",
+            conn->base_.address, conn->base_.port, status_code,
+            escaped(reason), conn->base_.purpose,
+            (received_bytes),
+#ifdef MEASUREMENTS_21206
+            conn->data_cells_received, conn->data_cells_sent,
+#endif
+            compression);
+
+  if (conn->guard_state) {
+    /* we count the connection as successful once we can read from it.  We do
+     * not, however, delay use of the circuit here, since it's just for a
+     * one-hop directory request. */
+    /* XXXXprop271 note that this will not do the right thing for other
+     * waiting circuits that would be triggered by this circuit becoming
+     * complete/usable. But that's ok, I think.
+     */
+    entry_guard_succeeded(&conn->guard_state);
+    circuit_guard_state_free(conn->guard_state);
+    conn->guard_state = NULL;
+  }
+
+  /* now check if it's got any hints for us about our IP address. */
+  if (conn->dirconn_direct) {
+    char *guess = http_get_header(headers, X_ADDRESS_HEADER);
+    if (guess) {
+      router_new_address_suggestion(guess, conn);
+      tor_free(guess);
+    }
+  }
+
+  if (date_header > 0) {
+    /* The date header was written very soon after we sent our request,
+     * so compute the skew as the difference between sending the request
+     * and the date header.  (We used to check now-date_header, but that's
+     * inaccurate if we spend a lot of time downloading.)
+     */
+    apparent_skew = conn->base_.timestamp_last_write_allowed - date_header;
+    if (labs(apparent_skew)>ALLOW_DIRECTORY_TIME_SKEW) {
+      int trusted = router_digest_is_trusted_dir(conn->identity_digest);
+      clock_skew_warning(TO_CONN(conn), apparent_skew, trusted, LD_HTTP,
+                         "directory", "DIRSERV");
+      skewed = 1; /* don't check the recommended-versions line */
+    } else {
+      log_debug(LD_HTTP, "Time on received directory is within tolerance; "
+                "we are %ld seconds skewed.  (That's okay.)", apparent_skew);
+    }
+  }
+  (void) skewed; /* skewed isn't used yet. */
+
+  if (status_code == 503) {
+    routerstatus_t *rs;
+    dir_server_t *ds;
+    const char *id_digest = conn->identity_digest;
+    log_info(LD_DIR,"Received http status code %d (%s) from server "
+             "'%s:%d'. I'll try again soon.",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port);
+    time_t now = approx_time();
+    if ((rs = router_get_mutable_consensus_status_by_id(id_digest)))
+      rs->last_dir_503_at = now;
+    if ((ds = router_get_fallback_dirserver_by_digest(id_digest)))
+      ds->fake_status.last_dir_503_at = now;
+
+    rv = -1;
+    goto done;
+  }
+
+  if (dir_client_decompress_response_body(&body, &body_len,
+                             conn, compression, anonymized_connection) < 0) {
+    rv = -1;
+    goto done;
+  }
+
+  response_handler_args_t args;
+  memset(&args, 0, sizeof(args));
+  args.status_code = status_code;
+  args.reason = reason;
+  args.body = body;
+  args.body_len = body_len;
+  args.headers = headers;
+
+  switch (conn->base_.purpose) {
+    case DIR_PURPOSE_FETCH_CONSENSUS:
+      rv = handle_response_fetch_consensus(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_CERTIFICATE:
+      rv = handle_response_fetch_certificate(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_STATUS_VOTE:
+      rv = handle_response_fetch_status_vote(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
+      rv = handle_response_fetch_detached_signatures(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_SERVERDESC:
+    case DIR_PURPOSE_FETCH_EXTRAINFO:
+      rv = handle_response_fetch_desc(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_MICRODESC:
+      rv = handle_response_fetch_microdesc(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_RENDDESC_V2:
+      rv = handle_response_fetch_renddesc_v2(conn, &args);
+      break;
+    case DIR_PURPOSE_UPLOAD_DIR:
+      rv = handle_response_upload_dir(conn, &args);
+      break;
+    case DIR_PURPOSE_UPLOAD_SIGNATURES:
+      rv = handle_response_upload_signatures(conn, &args);
+      break;
+    case DIR_PURPOSE_UPLOAD_VOTE:
+      rv = handle_response_upload_vote(conn, &args);
+      break;
+    case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
+      rv = handle_response_upload_renddesc_v2(conn, &args);
+      break;
+    case DIR_PURPOSE_UPLOAD_HSDESC:
+      rv = handle_response_upload_hsdesc(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_HSDESC:
+      rv = handle_response_fetch_hsdesc_v3(conn, &args);
+      break;
+    default:
+      tor_assert_nonfatal_unreached();
+      rv = -1;
+      break;
+  }
+
+ done:
+  tor_free(body);
+  tor_free(headers);
+  tor_free(reason);
+  return rv;
+}
+
+/**
+ * Handler function: processes a response to a request for a networkstatus
+ * consensus document by checking the consensus, storing it, and marking
+ * router requests as reachable.
+ **/
+STATIC int
+handle_response_fetch_consensus(dir_connection_t *conn,
+                                const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS);
+  const int status_code = args->status_code;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+  const char *reason = args->reason;
+  const time_t now = approx_time();
+
+  const char *consensus;
+  char *new_consensus = NULL;
+  const char *sourcename;
+
+  int r;
+  const char *flavname = conn->requested_resource;
+  if (status_code != 200) {
+    int severity = (status_code == 304) ? LOG_INFO : LOG_WARN;
+    tor_log(severity, LD_DIR,
+            "Received http status code %d (%s) from server "
+            "'%s:%d' while fetching consensus directory.",
+            status_code, escaped(reason), conn->base_.address,
+            conn->base_.port);
+    networkstatus_consensus_download_failed(status_code, flavname);
+    return -1;
+  }
+
+  if (looks_like_a_consensus_diff(body, body_len)) {
+    /* First find our previous consensus. Maybe it's in ram, maybe not. */
+    cached_dir_t *cd = dirserv_get_consensus(flavname);
+    const char *consensus_body;
+    char *owned_consensus = NULL;
+    if (cd) {
+      consensus_body = cd->dir;
+    } else {
+      owned_consensus = networkstatus_read_cached_consensus(flavname);
+      consensus_body = owned_consensus;
+    }
+    if (!consensus_body) {
+      log_warn(LD_DIR, "Received a consensus diff, but we can't find "
+               "any %s-flavored consensus in our current cache.",flavname);
+      networkstatus_consensus_download_failed(0, flavname);
+      // XXXX if this happens too much, see below
+      return -1;
+    }
+
+    new_consensus = consensus_diff_apply(consensus_body, body);
+    tor_free(owned_consensus);
+    if (new_consensus == NULL) {
+      log_warn(LD_DIR, "Could not apply consensus diff received from server "
+               "'%s:%d'", conn->base_.address, conn->base_.port);
+      // XXXX If this happens too many times, we should maybe not use
+      // XXXX this directory for diffs any more?
+      networkstatus_consensus_download_failed(0, flavname);
+      return -1;
+    }
+    log_info(LD_DIR, "Applied consensus diff (size %d) from server "
+             "'%s:%d', resulting in a new consensus document (size %d).",
+             (int)body_len, conn->base_.address, conn->base_.port,
+             (int)strlen(new_consensus));
+    consensus = new_consensus;
+    sourcename = "generated based on a diff";
+  } else {
+    log_info(LD_DIR,"Received consensus directory (body size %d) from server "
+             "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port);
+    consensus = body;
+    sourcename = "downloaded";
+  }
+
+  if ((r=networkstatus_set_current_consensus(consensus, flavname, 0,
+                                             conn->identity_digest))<0) {
+    log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
+           "Unable to load %s consensus directory %s from "
+           "server '%s:%d'. I'll try again soon.",
+           flavname, sourcename, conn->base_.address, conn->base_.port);
+    networkstatus_consensus_download_failed(0, flavname);
+    tor_free(new_consensus);
+    return -1;
+  }
+
+  /* If we launched other fetches for this consensus, cancel them. */
+  connection_dir_close_consensus_fetches(conn, flavname);
+
+  /* update the list of routers and directory guards */
+  routers_update_all_from_networkstatus(now, 3);
+  update_microdescs_from_networkstatus(now);
+  directory_info_has_arrived(now, 0, 0);
+
+  if (authdir_mode_v3(get_options())) {
+    sr_act_post_consensus(
+                     networkstatus_get_latest_consensus_by_flavor(FLAV_NS));
+  }
+  log_info(LD_DIR, "Successfully loaded consensus.");
+
+  tor_free(new_consensus);
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for one or more
+ * authority certificates
+ **/
+static int
+handle_response_fetch_certificate(dir_connection_t *conn,
+                                  const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  if (status_code != 200) {
+    log_warn(LD_DIR,
+             "Received http status code %d (%s) from server "
+             "'%s:%d' while fetching \"/tor/keys/%s\".",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port, conn->requested_resource);
+    connection_dir_download_cert_failed(conn, status_code);
+    return -1;
+  }
+  log_info(LD_DIR,"Received authority certificates (body size %d) from "
+           "server '%s:%d'",
+           (int)body_len, conn->base_.address, conn->base_.port);
+
+  /*
+   * Tell trusted_dirs_load_certs_from_string() whether it was by fp
+   * or fp-sk pair.
+   */
+  int src_code = -1;
+  if (!strcmpstart(conn->requested_resource, "fp/")) {
+    src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST;
+  } else if (!strcmpstart(conn->requested_resource, "fp-sk/")) {
+    src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST;
+  }
+
+  if (src_code != -1) {
+    if (trusted_dirs_load_certs_from_string(body, src_code, 1,
+                                            conn->identity_digest)<0) {
+      log_warn(LD_DIR, "Unable to parse fetched certificates");
+      /* if we fetched more than one and only some failed, the successful
+       * ones got flushed to disk so it's safe to call this on them */
+      connection_dir_download_cert_failed(conn, status_code);
+    } else {
+      time_t now = approx_time();
+      directory_info_has_arrived(now, 0, 0);
+      log_info(LD_DIR, "Successfully loaded certificates from fetch.");
+    }
+  } else {
+    log_warn(LD_DIR,
+             "Couldn't figure out what to do with fetched certificates for "
+             "unknown resource %s",
+             conn->requested_resource);
+    connection_dir_download_cert_failed(conn, status_code);
+  }
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for an authority's
+ * current networkstatus vote.
+ **/
+static int
+handle_response_fetch_status_vote(dir_connection_t *conn,
+                                  const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  const char *msg;
+  int st;
+  log_info(LD_DIR,"Got votes (body size %d) from server %s:%d",
+           (int)body_len, conn->base_.address, conn->base_.port);
+  if (status_code != 200) {
+    log_warn(LD_DIR,
+             "Received http status code %d (%s) from server "
+             "'%s:%d' while fetching \"/tor/status-vote/next/%s.z\".",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port, conn->requested_resource);
+    return -1;
+  }
+  dirvote_add_vote(body, &msg, &st);
+  if (st > 299) {
+    log_warn(LD_DIR, "Error adding retrieved vote: %s", msg);
+  } else {
+    log_info(LD_DIR, "Added vote(s) successfully [msg: %s]", msg);
+  }
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for the signatures
+ * that an authority knows about on a given consensus.
+ **/
+static int
+handle_response_fetch_detached_signatures(dir_connection_t *conn,
+                                          const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  const char *msg = NULL;
+  log_info(LD_DIR,"Got detached signatures (body size %d) from server %s:%d",
+           (int)body_len, conn->base_.address, conn->base_.port);
+  if (status_code != 200) {
+    log_warn(LD_DIR,
+        "Received http status code %d (%s) from server '%s:%d' while fetching "
+        "\"/tor/status-vote/next/consensus-signatures.z\".",
+        status_code, escaped(reason), conn->base_.address,
+        conn->base_.port);
+    return -1;
+  }
+  if (dirvote_add_signatures(body, conn->base_.address, &msg)<0) {
+    log_warn(LD_DIR, "Problem adding detached signatures from %s:%d: %s",
+             conn->base_.address, conn->base_.port, msg?msg:"???");
+  }
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for a group of server
+ * descriptors or an extrainfo documents.
+ **/
+static int
+handle_response_fetch_desc(dir_connection_t *conn,
+                           const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
+             conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  int was_ei = conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO;
+  smartlist_t *which = NULL;
+  int n_asked_for = 0;
+  int descriptor_digests = conn->requested_resource &&
+    !strcmpstart(conn->requested_resource,"d/");
+  log_info(LD_DIR,"Received %s (body size %d) from server '%s:%d'",
+           was_ei ? "extra server info" : "server info",
+           (int)body_len, conn->base_.address, conn->base_.port);
+  if (conn->requested_resource &&
+      (!strcmpstart(conn->requested_resource,"d/") ||
+       !strcmpstart(conn->requested_resource,"fp/"))) {
+    which = smartlist_new();
+    dir_split_resource_into_fingerprints(conn->requested_resource +
+                                         (descriptor_digests ? 2 : 3),
+                                         which, NULL, 0);
+    n_asked_for = smartlist_len(which);
+  }
+  if (status_code != 200) {
+    int dir_okay = status_code == 404 ||
+      (status_code == 400 && !strcmp(reason, "Servers unavailable."));
+    /* 404 means that it didn't have them; no big deal.
+     * Older (pre-0.1.1.8) servers said 400 Servers unavailable instead. */
+    log_fn(dir_okay ? LOG_INFO : LOG_WARN, LD_DIR,
+           "Received http status code %d (%s) from server '%s:%d' "
+           "while fetching \"/tor/server/%s\". I'll try again soon.",
+           status_code, escaped(reason), conn->base_.address,
+           conn->base_.port, conn->requested_resource);
+    if (!which) {
+      connection_dir_download_routerdesc_failed(conn);
+    } else {
+      dir_routerdesc_download_failed(which, status_code,
+                                     conn->router_purpose,
+                                     was_ei, descriptor_digests);
+      SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
+      smartlist_free(which);
+    }
+    return dir_okay ? 0 : -1;
+  }
+  /* Learn the routers, assuming we requested by fingerprint or "all"
+   * or "authority".
+   *
+   * We use "authority" to fetch our own descriptor for
+   * testing, and to fetch bridge descriptors for bootstrapping. Ignore
+   * the output of "authority" requests unless we are using bridges,
+   * since otherwise they'll be the response from reachability tests,
+   * and we don't really want to add that to our routerlist. */
+  if (which || (conn->requested_resource &&
+                (!strcmpstart(conn->requested_resource, "all") ||
+                 (!strcmpstart(conn->requested_resource, "authority") &&
+                  get_options()->UseBridges)))) {
+    /* as we learn from them, we remove them from 'which' */
+    if (was_ei) {
+      router_load_extrainfo_from_string(body, NULL, SAVED_NOWHERE, which,
+                                        descriptor_digests);
+    } else {
+      //router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which,
+      //                       descriptor_digests, conn->router_purpose);
+      if (load_downloaded_routers(body, which, descriptor_digests,
+                                  conn->router_purpose,
+                                  conn->base_.address)) {
+        time_t now = approx_time();
+        directory_info_has_arrived(now, 0, 1);
+      }
+    }
+  }
+  if (which) { /* mark remaining ones as failed */
+    log_info(LD_DIR, "Received %d/%d %s requested from %s:%d",
+             n_asked_for-smartlist_len(which), n_asked_for,
+             was_ei ? "extra-info documents" : "router descriptors",
+             conn->base_.address, (int)conn->base_.port);
+    if (smartlist_len(which)) {
+      dir_routerdesc_download_failed(which, status_code,
+                                     conn->router_purpose,
+                                     was_ei, descriptor_digests);
+    }
+    SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
+    smartlist_free(which);
+  }
+  if (directory_conn_is_self_reachability_test(conn))
+    router_dirport_found_reachable();
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for a group of
+ * microdescriptors
+ **/
+STATIC int
+handle_response_fetch_microdesc(dir_connection_t *conn,
+                                const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  smartlist_t *which = NULL;
+  log_info(LD_DIR,"Received answer to microdescriptor request (status %d, "
+           "body size %d) from server '%s:%d'",
+           status_code, (int)body_len, conn->base_.address,
+           conn->base_.port);
+  tor_assert(conn->requested_resource &&
+             !strcmpstart(conn->requested_resource, "d/"));
+  tor_assert_nonfatal(!tor_mem_is_zero(conn->identity_digest, DIGEST_LEN));
+  which = smartlist_new();
+  dir_split_resource_into_fingerprints(conn->requested_resource+2,
+                                       which, NULL,
+                                       DSR_DIGEST256|DSR_BASE64);
+  if (status_code != 200) {
+    log_info(LD_DIR, "Received status code %d (%s) from server "
+             "'%s:%d' while fetching \"/tor/micro/%s\".  I'll try again "
+             "soon.",
+             status_code, escaped(reason), conn->base_.address,
+             (int)conn->base_.port, conn->requested_resource);
+    dir_microdesc_download_failed(which, status_code, conn->identity_digest);
+    SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
+    smartlist_free(which);
+    return 0;
+  } else {
+    smartlist_t *mds;
+    time_t now = approx_time();
+    mds = microdescs_add_to_cache(get_microdesc_cache(),
+                                  body, body+body_len, SAVED_NOWHERE, 0,
+                                  now, which);
+    if (smartlist_len(which)) {
+      /* Mark remaining ones as failed. */
+      dir_microdesc_download_failed(which, status_code, conn->identity_digest);
+    }
+    if (mds && smartlist_len(mds)) {
+      control_event_boot_dir(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS,
+                             count_loading_descriptors_progress());
+      directory_info_has_arrived(now, 0, 1);
+    }
+    SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
+    smartlist_free(which);
+    smartlist_free(mds);
+  }
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a POST request to upload our
+ * router descriptor.
+ **/
+static int
+handle_response_upload_dir(dir_connection_t *conn,
+                           const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_DIR);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *headers = args->headers;
+
+  switch (status_code) {
+  case 200: {
+    dir_server_t *ds =
+      router_get_trusteddirserver_by_digest(conn->identity_digest);
+    char *rejected_hdr = http_get_header(headers,
+                                         "X-Descriptor-Not-New: ");
+    if (rejected_hdr) {
+      if (!strcmp(rejected_hdr, "Yes")) {
+        log_info(LD_GENERAL,
+                 "Authority '%s' declined our descriptor (not new)",
+                 ds->nickname);
+        /* XXXX use this information; be sure to upload next one
+         * sooner. -NM */
+        /* XXXX++ On further thought, the task above implies that we're
+         * basing our regenerate-descriptor time on when we uploaded the
+         * last descriptor, not on the published time of the last
+         * descriptor.  If those are different, that's a bad thing to
+         * do. -NM */
+      }
+      tor_free(rejected_hdr);
+    }
+    log_info(LD_GENERAL,"eof (status 200) after uploading server "
+             "descriptor: finished.");
+    control_event_server_status(
+                   LOG_NOTICE, "ACCEPTED_SERVER_DESCRIPTOR DIRAUTH=%s:%d",
+                   conn->base_.address, conn->base_.port);
+
+    ds->has_accepted_serverdesc = 1;
+    if (directories_have_accepted_server_descriptor())
+      control_event_server_status(LOG_NOTICE, "GOOD_SERVER_DESCRIPTOR");
+  }
+    break;
+  case 400:
+    log_warn(LD_GENERAL,"http status 400 (%s) response from "
+             "dirserver '%s:%d'. Please correct.",
+             escaped(reason), conn->base_.address, conn->base_.port);
+    control_event_server_status(LOG_WARN,
+                    "BAD_SERVER_DESCRIPTOR DIRAUTH=%s:%d REASON=\"%s\"",
+                    conn->base_.address, conn->base_.port, escaped(reason));
+    break;
+  default:
+    log_warn(LD_GENERAL,
+             "HTTP status %d (%s) was unexpected while uploading "
+             "descriptor to server '%s:%d'. Possibly the server is "
+             "misconfigured?",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port);
+    break;
+  }
+  /* return 0 in all cases, since we don't want to mark any
+   * dirservers down just because they don't like us. */
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to POST request to upload our
+ * own networkstatus vote.
+ **/
+static int
+handle_response_upload_vote(dir_connection_t *conn,
+                            const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_VOTE);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+
+  switch (status_code) {
+  case 200: {
+    log_notice(LD_DIR,"Uploaded a vote to dirserver %s:%d",
+               conn->base_.address, conn->base_.port);
+  }
+    break;
+  case 400:
+    log_warn(LD_DIR,"http status 400 (%s) response after uploading "
+             "vote to dirserver '%s:%d'. Please correct.",
+             escaped(reason), conn->base_.address, conn->base_.port);
+    break;
+  default:
+    log_warn(LD_GENERAL,
+             "HTTP status %d (%s) was unexpected while uploading "
+             "vote to server '%s:%d'.",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port);
+    break;
+  }
+  /* return 0 in all cases, since we don't want to mark any
+   * dirservers down just because they don't like us. */
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to POST request to upload our
+ * view of the signatures on the current consensus.
+ **/
+static int
+handle_response_upload_signatures(dir_connection_t *conn,
+                                  const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_SIGNATURES);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+
+  switch (status_code) {
+  case 200: {
+    log_notice(LD_DIR,"Uploaded signature(s) to dirserver %s:%d",
+               conn->base_.address, conn->base_.port);
+  }
+    break;
+  case 400:
+    log_warn(LD_DIR,"http status 400 (%s) response after uploading "
+             "signatures to dirserver '%s:%d'. Please correct.",
+             escaped(reason), conn->base_.address, conn->base_.port);
+    break;
+  default:
+    log_warn(LD_GENERAL,
+             "HTTP status %d (%s) was unexpected while uploading "
+             "signatures to server '%s:%d'.",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port);
+    break;
+  }
+  /* return 0 in all cases, since we don't want to mark any
+   * dirservers down just because they don't like us. */
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for a v3 hidden service
+ * descriptor.
+ **/
+STATIC int
+handle_response_fetch_hsdesc_v3(dir_connection_t *conn,
+                                const response_handler_args_t *args)
+{
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  tor_assert(conn->hs_ident);
+
+  log_info(LD_REND,"Received v3 hsdesc (body size %d, status %d (%s))",
+           (int)body_len, status_code, escaped(reason));
+
+  switch (status_code) {
+  case 200:
+    /* We got something: Try storing it in the cache. */
+    if (hs_cache_store_as_client(body, &conn->hs_ident->identity_pk) < 0) {
+      log_info(LD_REND, "Failed to store hidden service descriptor");
+      /* Fire control port FAILED event. */
+      hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
+                                   "BAD_DESC");
+      hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
+                                    NULL);
+    } else {
+      log_info(LD_REND, "Stored hidden service descriptor successfully.");
+      TO_CONN(conn)->purpose = DIR_PURPOSE_HAS_FETCHED_HSDESC;
+      hs_client_desc_has_arrived(conn->hs_ident);
+      /* Fire control port RECEIVED event. */
+      hs_control_desc_event_received(conn->hs_ident, conn->identity_digest);
+      hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
+                                    body);
+    }
+    break;
+  case 404:
+    /* Not there. We'll retry when connection_about_to_close_connection()
+     * tries to clean this conn up. */
+    log_info(LD_REND, "Fetching hidden service v3 descriptor not found: "
+                      "Retrying at another directory.");
+    /* Fire control port FAILED event. */
+    hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
+                                 "NOT_FOUND");
+    hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
+                                  NULL);
+    break;
+  case 400:
+    log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: "
+                      "http status 400 (%s). Dirserver didn't like our "
+                      "query? Retrying at another directory.",
+             escaped(reason));
+    /* Fire control port FAILED event. */
+    hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
+                                 "QUERY_REJECTED");
+    hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
+                                  NULL);
+    break;
+  default:
+    log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: "
+             "http status %d (%s) response unexpected from HSDir server "
+             "'%s:%d'. Retrying at another directory.",
+             status_code, escaped(reason), TO_CONN(conn)->address,
+             TO_CONN(conn)->port);
+    /* Fire control port FAILED event. */
+    hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
+                                 "UNEXPECTED");
+    hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
+                                  NULL);
+    break;
+  }
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for a v2 hidden service
+ * descriptor.
+ **/
+static int
+handle_response_fetch_renddesc_v2(dir_connection_t *conn,
+                                  const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+#define SEND_HS_DESC_FAILED_EVENT(reason)                               \
+  (control_event_hsv2_descriptor_failed(conn->rend_data,                \
+                                        conn->identity_digest,          \
+                                        reason))
+#define SEND_HS_DESC_FAILED_CONTENT()                                   \
+  (control_event_hs_descriptor_content(                                 \
+                                rend_data_get_address(conn->rend_data), \
+                                conn->requested_resource,               \
+                                conn->identity_digest,                  \
+                                NULL))
+
+  tor_assert(conn->rend_data);
+  log_info(LD_REND,"Received rendezvous descriptor (body size %d, status %d "
+           "(%s))",
+           (int)body_len, status_code, escaped(reason));
+  switch (status_code) {
+  case 200:
+    {
+      rend_cache_entry_t *entry = NULL;
+
+      if (rend_cache_store_v2_desc_as_client(body,
+                                             conn->requested_resource,
+                                             conn->rend_data, &entry) < 0) {
+        log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. "
+                 "Retrying at another directory.");
+        /* We'll retry when connection_about_to_close_connection()
+         * cleans this dir conn up. */
+        SEND_HS_DESC_FAILED_EVENT("BAD_DESC");
+        SEND_HS_DESC_FAILED_CONTENT();
+      } else {
+        char service_id[REND_SERVICE_ID_LEN_BASE32 + 1];
+        /* Should never be NULL here if we found the descriptor. */
+        tor_assert(entry);
+        rend_get_service_id(entry->parsed->pk, service_id);
+
+        /* success. notify pending connections about this. */
+        log_info(LD_REND, "Successfully fetched v2 rendezvous "
+                 "descriptor.");
+        control_event_hsv2_descriptor_received(service_id,
+                                               conn->rend_data,
+                                               conn->identity_digest);
+        control_event_hs_descriptor_content(service_id,
+                                            conn->requested_resource,
+                                            conn->identity_digest,
+                                            body);
+        conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2;
+        rend_client_desc_trynow(service_id);
+        memwipe(service_id, 0, sizeof(service_id));
+      }
+      break;
+    }
+  case 404:
+    /* Not there. We'll retry when
+     * connection_about_to_close_connection() cleans this conn up. */
+    log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: "
+             "Retrying at another directory.");
+    SEND_HS_DESC_FAILED_EVENT("NOT_FOUND");
+    SEND_HS_DESC_FAILED_CONTENT();
+    break;
+  case 400:
+    log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: "
+             "http status 400 (%s). Dirserver didn't like our "
+             "v2 rendezvous query? Retrying at another directory.",
+             escaped(reason));
+    SEND_HS_DESC_FAILED_EVENT("QUERY_REJECTED");
+    SEND_HS_DESC_FAILED_CONTENT();
+    break;
+  default:
+    log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: "
+             "http status %d (%s) response unexpected while "
+             "fetching v2 hidden service descriptor (server '%s:%d'). "
+             "Retrying at another directory.",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port);
+    SEND_HS_DESC_FAILED_EVENT("UNEXPECTED");
+    SEND_HS_DESC_FAILED_CONTENT();
+    break;
+  }
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a POST request to upload a v2
+ * hidden service descriptor.
+ **/
+static int
+handle_response_upload_renddesc_v2(dir_connection_t *conn,
+                                   const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+
+#define SEND_HS_DESC_UPLOAD_FAILED_EVENT(reason)                        \
+    (control_event_hs_descriptor_upload_failed(                         \
+                                conn->identity_digest,                  \
+                                rend_data_get_address(conn->rend_data), \
+                                reason))
+
+  log_info(LD_REND,"Uploaded rendezvous descriptor (status %d "
+           "(%s))",
+           status_code, escaped(reason));
+  /* Without the rend data, we'll have a problem identifying what has been
+   * uploaded for which service. */
+  tor_assert(conn->rend_data);
+  switch (status_code) {
+  case 200:
+    log_info(LD_REND,
+             "Uploading rendezvous descriptor: finished with status "
+             "200 (%s)", escaped(reason));
+    control_event_hs_descriptor_uploaded(conn->identity_digest,
+                                   rend_data_get_address(conn->rend_data));
+    rend_service_desc_has_uploaded(conn->rend_data);
+    break;
+  case 400:
+    log_warn(LD_REND,"http status 400 (%s) response from dirserver "
+             "'%s:%d'. Malformed rendezvous descriptor?",
+             escaped(reason), conn->base_.address, conn->base_.port);
+    SEND_HS_DESC_UPLOAD_FAILED_EVENT("UPLOAD_REJECTED");
+    break;
+  default:
+    log_warn(LD_REND,"http status %d (%s) response unexpected (server "
+             "'%s:%d').",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port);
+    SEND_HS_DESC_UPLOAD_FAILED_EVENT("UNEXPECTED");
+    break;
+  }
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a POST request to upload an
+ * hidden service descriptor.
+ **/
+static int
+handle_response_upload_hsdesc(dir_connection_t *conn,
+                              const response_handler_args_t *args)
+{
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+
+  tor_assert(conn);
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_HSDESC);
+
+  log_info(LD_REND, "Uploaded hidden service descriptor (status %d "
+                    "(%s))",
+           status_code, escaped(reason));
+  /* For this directory response, it MUST have an hidden service identifier on
+   * this connection. */
+  tor_assert(conn->hs_ident);
+  switch (status_code) {
+  case 200:
+    log_info(LD_REND, "Uploading hidden service descriptor: "
+                      "finished with status 200 (%s)", escaped(reason));
+    hs_control_desc_event_uploaded(conn->hs_ident, conn->identity_digest);
+    break;
+  case 400:
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Uploading hidden service descriptor: http "
+           "status 400 (%s) response from dirserver "
+           "'%s:%d'. Malformed hidden service descriptor?",
+           escaped(reason), conn->base_.address, conn->base_.port);
+    hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
+                                 "UPLOAD_REJECTED");
+    break;
+  default:
+    log_warn(LD_REND, "Uploading hidden service descriptor: http "
+                      "status %d (%s) response unexpected (server "
+                      "'%s:%d').",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port);
+    hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
+                                 "UNEXPECTED");
+    break;
+  }
+
+  return 0;
+}
+
+/** Called when a directory connection reaches EOF. */
+int
+connection_dir_reached_eof(dir_connection_t *conn)
+{
+  int retval;
+  if (conn->base_.state != DIR_CONN_STATE_CLIENT_READING) {
+    log_info(LD_HTTP,"conn reached eof, not reading. [state=%d] Closing.",
+             conn->base_.state);
+    connection_close_immediate(TO_CONN(conn)); /* error: give up on flushing */
+    connection_mark_for_close(TO_CONN(conn));
+    return -1;
+  }
+
+  retval = connection_dir_client_reached_eof(conn);
+  if (retval == 0) /* success */
+    conn->base_.state = DIR_CONN_STATE_CLIENT_FINISHED;
+  connection_mark_for_close(TO_CONN(conn));
+  return retval;
+}
+/** We are closing a dir connection: If <b>dir_conn</b> is a dir connection
+ *  that tried to fetch an HS descriptor, check if it successfully fetched it,
+ *  or if we need to try again. */
+void
+connection_dir_client_refetch_hsdesc_if_needed(dir_connection_t *dir_conn)
+{
+  connection_t *conn = TO_CONN(dir_conn);
+
+  /* If we were trying to fetch a v2 rend desc and did not succeed, retry as
+   * needed. (If a fetch is successful, the connection state is changed to
+   * DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 or DIR_PURPOSE_HAS_FETCHED_HSDESC to
+   * mark that refetching is unnecessary.) */
+  if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 &&
+      dir_conn->rend_data &&
+      rend_valid_v2_service_id(
+           rend_data_get_address(dir_conn->rend_data))) {
+    rend_client_refetch_v2_renddesc(dir_conn->rend_data);
+  }
+
+  /* Check for v3 rend desc fetch */
+  if (conn->purpose == DIR_PURPOSE_FETCH_HSDESC &&
+      dir_conn->hs_ident &&
+      !ed25519_public_key_is_zero(&dir_conn->hs_ident->identity_pk)) {
+    hs_client_refetch_hsdesc(&dir_conn->hs_ident->identity_pk);
+  }
+}
+
+/** Array of compression methods to use (if supported) for requesting
+ * compressed data, ordered from best to worst. */
+static compress_method_t client_meth_pref[] = {
+  LZMA_METHOD,
+  ZSTD_METHOD,
+  ZLIB_METHOD,
+  GZIP_METHOD,
+  NO_METHOD
+};
+
+/** Array of allowed compression methods to use (if supported) when receiving a
+ * response from a request that was required to be anonymous. */
+static compress_method_t client_meth_allowed_anonymous_compression[] = {
+  ZLIB_METHOD,
+  GZIP_METHOD,
+  NO_METHOD
+};
+
+/** Return a newly allocated string containing a comma separated list of
+ * supported encodings. */
+STATIC char *
+accept_encoding_header(void)
+{
+  smartlist_t *methods = smartlist_new();
+  char *header = NULL;
+  compress_method_t method;
+  unsigned i;
+
+  for (i = 0; i < ARRAY_LENGTH(client_meth_pref); ++i) {
+    method = client_meth_pref[i];
+    if (tor_compress_supports_method(method))
+      smartlist_add(methods, (char *)compression_method_get_name(method));
+  }
+
+  header = smartlist_join_strings(methods, ", ", 0, NULL);
+  smartlist_free(methods);
+
+  return header;
+}
+
+/** Check if the given compression method is allowed for a connection that is
+ * supposed to be anonymous. Returns 1 if the compression method is allowed,
+ * otherwise 0. */
+STATIC int
+allowed_anonymous_connection_compression_method(compress_method_t method)
+{
+  unsigned u;
+
+  for (u = 0; u < ARRAY_LENGTH(client_meth_allowed_anonymous_compression);
+       ++u) {
+    compress_method_t allowed_method =
+      client_meth_allowed_anonymous_compression[u];
+
+    if (! tor_compress_supports_method(allowed_method))
+      continue;
+
+    if (method == allowed_method)
+      return 1;
+  }
+
+  return 0;
+}
+
+/** Log a warning when a remote server has sent us a document using a
+ * compression method that is not allowed for anonymous directory requests. */
+STATIC void
+warn_disallowed_anonymous_compression_method(compress_method_t method)
+{
+  log_fn(LOG_PROTOCOL_WARN, LD_HTTP,
+         "Received a %s HTTP response, which is not "
+         "allowed for anonymous directory requests.",
+         compression_method_get_human_name(method));
+}
+
+/* We just got a new consensus! If there are other in-progress requests
+ * for this consensus flavor (for example because we launched several in
+ * parallel), cancel them.
+ *
+ * We do this check here (not just in
+ * connection_ap_handshake_attach_circuit()) to handle the edge case where
+ * a consensus fetch begins and ends before some other one tries to attach to
+ * a circuit, in which case the other one won't know that we're all happy now.
+ *
+ * Don't mark the conn that just gave us the consensus -- otherwise we
+ * would end up double-marking it when it cleans itself up.
+ */
+static void
+connection_dir_close_consensus_fetches(dir_connection_t *except_this_one,
+                                       const char *resource)
+{
+  smartlist_t *conns_to_close =
+    connection_dir_list_by_purpose_and_resource(DIR_PURPOSE_FETCH_CONSENSUS,
+                                                resource);
+  SMARTLIST_FOREACH_BEGIN(conns_to_close, dir_connection_t *, d) {
+    if (d == except_this_one)
+      continue;
+    log_info(LD_DIR, "Closing consensus fetch (to %s) since one "
+             "has just arrived.", TO_CONN(d)->address);
+    connection_mark_for_close(TO_CONN(d));
+  } SMARTLIST_FOREACH_END(d);
+  smartlist_free(conns_to_close);
+}
+/** Called when one or more routerdesc (or extrainfo, if <b>was_extrainfo</b>)
+ * fetches have failed (with uppercase fingerprints listed in <b>failed</b>,
+ * either as descriptor digests or as identity digests based on
+ * <b>was_descriptor_digests</b>).
+ */
+static void
+dir_routerdesc_download_failed(smartlist_t *failed, int status_code,
+                               int router_purpose,
+                               int was_extrainfo, int was_descriptor_digests)
+{
+  char digest[DIGEST_LEN];
+  time_t now = time(NULL);
+  int server = directory_fetches_from_authorities(get_options());
+  if (!was_descriptor_digests) {
+    if (router_purpose == ROUTER_PURPOSE_BRIDGE) {
+      tor_assert(!was_extrainfo);
+      connection_dir_retry_bridges(failed);
+    }
+    return; /* FFFF should implement for other-than-router-purpose someday */
+  }
+  SMARTLIST_FOREACH_BEGIN(failed, const char *, cp) {
+    download_status_t *dls = NULL;
+    if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp)) != DIGEST_LEN) {
+      log_warn(LD_BUG, "Malformed fingerprint in list: %s", escaped(cp));
+      continue;
+    }
+    if (was_extrainfo) {
+      signed_descriptor_t *sd =
+        router_get_by_extrainfo_digest(digest);
+      if (sd)
+        dls = &sd->ei_dl_status;
+    } else {
+      dls = router_get_dl_status_by_descriptor_digest(digest);
+    }
+    if (!dls)
+      continue;
+    download_status_increment_failure(dls, status_code, cp, server, now);
+  } SMARTLIST_FOREACH_END(cp);
+
+  /* No need to relaunch descriptor downloads here: we already do it
+   * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
+}
+
+/** Called when a connection to download microdescriptors from relay with
+ * <b>dir_id</b> has failed in whole or in part. <b>failed</b> is a list
+ * of every microdesc digest we didn't get. <b>status_code</b> is the http
+ * status code we received. Reschedule the microdesc downloads as
+ * appropriate. */
+static void
+dir_microdesc_download_failed(smartlist_t *failed,
+                              int status_code, const char *dir_id)
+{
+  networkstatus_t *consensus
+    = networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
+  routerstatus_t *rs;
+  download_status_t *dls;
+  time_t now = time(NULL);
+  int server = directory_fetches_from_authorities(get_options());
+
+  if (! consensus)
+    return;
+
+  /* We failed to fetch a microdescriptor from 'dir_id', note it down
+   * so that we don't try the same relay next time... */
+  microdesc_note_outdated_dirserver(dir_id);
+
+  SMARTLIST_FOREACH_BEGIN(failed, const char *, d) {
+    rs = router_get_mutable_consensus_status_by_descriptor_digest(consensus,d);
+    if (!rs)
+      continue;
+    dls = &rs->dl_status;
+
+    { /* Increment the failure count for this md fetch */
+      char buf[BASE64_DIGEST256_LEN+1];
+      digest256_to_base64(buf, d);
+      log_info(LD_DIR, "Failed to download md %s from %s",
+               buf, hex_str(dir_id, DIGEST_LEN));
+      download_status_increment_failure(dls, status_code, buf,
+                                        server, now);
+    }
+  } SMARTLIST_FOREACH_END(d);
+}
diff --git a/src/feature/dirclient/dirclient.h b/src/feature/dirclient/dirclient.h
new file mode 100644
index 0000000000..1a93265dc3
--- /dev/null
+++ b/src/feature/dirclient/dirclient.h
@@ -0,0 +1,172 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dirclient.h
+ * \brief Header file for dirclient.c.
+ **/
+
+#ifndef TOR_DIRCLIENT_H
+#define TOR_DIRCLIENT_H
+
+#include "feature/hs/hs_ident.h"
+
+int directories_have_accepted_server_descriptor(void);
+void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
+                                  dirinfo_type_t type, const char *payload,
+                                  size_t payload_len, size_t extrainfo_len);
+MOCK_DECL(void, directory_get_from_dirserver, (
+                          uint8_t dir_purpose,
+                          uint8_t router_purpose,
+                          const char *resource,
+                          int pds_flags,
+                          download_want_authority_t want_authority));
+void directory_get_from_all_authorities(uint8_t dir_purpose,
+                                        uint8_t router_purpose,
+                                        const char *resource);
+
+/** Enumeration of ways to connect to a directory server */
+typedef enum {
+  /** Default: connect over a one-hop Tor circuit. Relays fall back to direct
+   * DirPort connections, clients, onion services, and bridges do not */
+  DIRIND_ONEHOP=0,
+  /** Connect over a multi-hop anonymizing Tor circuit */
+  DIRIND_ANONYMOUS=1,
+  /** Connect to the DirPort directly */
+  DIRIND_DIRECT_CONN,
+  /** Connect over a multi-hop anonymizing Tor circuit to our dirport */
+  DIRIND_ANON_DIRPORT,
+} dir_indirection_t;
+
+int directory_must_use_begindir(const or_options_t *options);
+
+/**
+ * A directory_request_t describes the information about a directory request
+ * at the client side.  It describes what we're going to ask for, which
+ * directory we're going to ask for it, how we're going to contact that
+ * directory, and (in some cases) what to do with it when we're done.
+ */
+typedef struct directory_request_t directory_request_t;
+directory_request_t *directory_request_new(uint8_t dir_purpose);
+void directory_request_free_(directory_request_t *req);
+#define directory_request_free(req) \
+  FREE_AND_NULL(directory_request_t, directory_request_free_, (req))
+void directory_request_set_or_addr_port(directory_request_t *req,
+                                        const tor_addr_port_t *p);
+void directory_request_set_dir_addr_port(directory_request_t *req,
+                                         const tor_addr_port_t *p);
+void directory_request_set_directory_id_digest(directory_request_t *req,
+                                               const char *digest);
+struct circuit_guard_state_t;
+void directory_request_set_guard_state(directory_request_t *req,
+                                       struct circuit_guard_state_t *state);
+void directory_request_set_router_purpose(directory_request_t *req,
+                                          uint8_t router_purpose);
+void directory_request_set_indirection(directory_request_t *req,
+                                       dir_indirection_t indirection);
+void directory_request_set_resource(directory_request_t *req,
+                                    const char *resource);
+void directory_request_set_payload(directory_request_t *req,
+                                   const char *payload,
+                                   size_t payload_len);
+void directory_request_set_if_modified_since(directory_request_t *req,
+                                             time_t if_modified_since);
+void directory_request_set_rend_query(directory_request_t *req,
+                                      const rend_data_t *query);
+void directory_request_upload_set_hs_ident(directory_request_t *req,
+                                           const hs_ident_dir_conn_t *ident);
+void directory_request_fetch_set_hs_ident(directory_request_t *req,
+                                          const hs_ident_dir_conn_t *ident);
+
+void directory_request_set_routerstatus(directory_request_t *req,
+                                        const routerstatus_t *rs);
+void directory_request_add_header(directory_request_t *req,
+                                  const char *key,
+                                  const char *val);
+MOCK_DECL(void, directory_initiate_request, (directory_request_t *request));
+
+int router_supports_extrainfo(const char *identity_digest, int is_authority);
+
+void connection_dir_client_request_failed(dir_connection_t *conn);
+void connection_dir_client_refetch_hsdesc_if_needed(
+                                          dir_connection_t *dir_conn);
+
+#ifdef DIRCLIENT_PRIVATE
+struct directory_request_t {
+  /**
+   * These fields specify which directory we're contacting.  Routerstatus,
+   * if present, overrides the other fields.
+   *
+   * @{ */
+  tor_addr_port_t or_addr_port;
+  tor_addr_port_t dir_addr_port;
+  char digest[DIGEST_LEN];
+
+  const routerstatus_t *routerstatus;
+  /** @} */
+  /** One of DIR_PURPOSE_* other than DIR_PURPOSE_SERVER. Describes what
+   * kind of operation we'll be doing (upload/download), and of what kind
+   * of document. */
+  uint8_t dir_purpose;
+  /** One of ROUTER_PURPOSE_*; used for uploads and downloads of routerinfo
+   * and extrainfo docs.  */
+  uint8_t router_purpose;
+  /** Enum: determines whether to anonymize, and whether to use dirport or
+   * orport. */
+  dir_indirection_t indirection;
+  /** Alias to the variable part of the URL for this request */
+  const char *resource;
+  /** Alias to the payload to upload (if any) */
+  const char *payload;
+  /** Number of bytes to upload from payload</b> */
+  size_t payload_len;
+  /** Value to send in an if-modified-since header, or 0 for none. */
+  time_t if_modified_since;
+  /** Hidden-service-specific information v2. */
+  const rend_data_t *rend_query;
+  /** Extra headers to append to the request */
+  struct config_line_t *additional_headers;
+  /** Hidden-service-specific information for v3+. */
+  const hs_ident_dir_conn_t *hs_ident;
+  /** Used internally to directory.c: gets informed when the attempt to
+   * connect to the directory succeeds or fails, if that attempt bears on the
+   * directory's usability as a directory guard. */
+  struct circuit_guard_state_t *guard_state;
+};
+
+/** A structure to hold arguments passed into each directory response
+ * handler */
+typedef struct response_handler_args_t {
+  int status_code;
+  const char *reason;
+  const char *body;
+  size_t body_len;
+  const char *headers;
+} response_handler_args_t;
+
+enum compress_method_t;
+STATIC int allowed_anonymous_connection_compression_method(
+                                               enum compress_method_t);
+STATIC void warn_disallowed_anonymous_compression_method(
+                                               enum compress_method_t);
+
+STATIC int should_use_directory_guards(const or_options_t *options);
+STATIC char *accept_encoding_header(void);
+STATIC const char *dir_conn_purpose_to_string(int purpose);
+
+STATIC int handle_response_fetch_hsdesc_v3(dir_connection_t *conn,
+                                          const response_handler_args_t *args);
+STATIC int handle_response_fetch_microdesc(dir_connection_t *conn,
+                                 const response_handler_args_t *args);
+
+STATIC int handle_response_fetch_consensus(dir_connection_t *conn,
+                                         const response_handler_args_t *args);
+
+STATIC dirinfo_type_t dir_fetch_type(int dir_purpose, int router_purpose,
+                                     const char *resource);
+#endif
+
+#endif /* !defined(TOR_DIRCLIENT_H) */
diff --git a/src/feature/dirclient/dlstatus.c b/src/feature/dirclient/dlstatus.c
new file mode 100644
index 0000000000..0842a2c676
--- /dev/null
+++ b/src/feature/dirclient/dlstatus.c
@@ -0,0 +1,422 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DLSTATUS_PRIVATE
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "feature/client/entrynodes.h"
+#include "feature/dirclient/dlstatus.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/relay/routermode.h"
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "feature/dirclient/download_status_st.h"
+
+/** Decide which download schedule we want to use based on descriptor type
+ * in <b>dls</b> and <b>options</b>.
+ *
+ * Then, return the initial delay for that download schedule, in seconds.
+ *
+ * Helper function for download_status_increment_failure(),
+ * download_status_reset(), and download_status_increment_attempt(). */
+STATIC int
+find_dl_min_delay(const download_status_t *dls, const or_options_t *options)
+{
+  tor_assert(dls);
+  tor_assert(options);
+
+  switch (dls->schedule) {
+    case DL_SCHED_GENERIC:
+      /* Any other directory document */
+      if (dir_server_mode(options)) {
+        /* A directory authority or directory mirror */
+        return options->TestingServerDownloadInitialDelay;
+      } else {
+        return options->TestingClientDownloadInitialDelay;
+      }
+    case DL_SCHED_CONSENSUS:
+      if (!networkstatus_consensus_can_use_multiple_directories(options)) {
+        /* A public relay */
+        return options->TestingServerConsensusDownloadInitialDelay;
+      } else {
+        /* A client or bridge */
+        if (networkstatus_consensus_is_bootstrapping(time(NULL))) {
+          /* During bootstrapping */
+          if (!networkstatus_consensus_can_use_extra_fallbacks(options)) {
+            /* A bootstrapping client without extra fallback directories */
+            return options->
+              ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay;
+          } else if (dls->want_authority) {
+            /* A bootstrapping client with extra fallback directories, but
+             * connecting to an authority */
+            return
+             options->ClientBootstrapConsensusAuthorityDownloadInitialDelay;
+          } else {
+            /* A bootstrapping client connecting to extra fallback directories
+             */
+            return
+              options->ClientBootstrapConsensusFallbackDownloadInitialDelay;
+          }
+        } else {
+          /* A client with a reasonably live consensus, with or without
+           * certificates */
+          return options->TestingClientConsensusDownloadInitialDelay;
+        }
+      }
+    case DL_SCHED_BRIDGE:
+      if (options->UseBridges && num_bridges_usable(0) > 0) {
+        /* A bridge client that is sure that one or more of its bridges are
+         * running can afford to wait longer to update bridge descriptors. */
+        return options->TestingBridgeDownloadInitialDelay;
+      } else {
+        /* A bridge client which might have no running bridges, must try to
+         * get bridge descriptors straight away. */
+        return options->TestingBridgeBootstrapDownloadInitialDelay;
+      }
+    default:
+      tor_assert(0);
+  }
+
+  /* Impossible, but gcc will fail with -Werror without a `return`. */
+  return 0;
+}
+
+/** As next_random_exponential_delay() below, but does not compute a random
+ * value. Instead, compute the range of values that
+ * next_random_exponential_delay() should use when computing its random value.
+ * Store the low bound into *<b>low_bound_out</b>, and the high bound into
+ * *<b>high_bound_out</b>.  Guarantees that the low bound is strictly less
+ * than the high bound. */
+STATIC void
+next_random_exponential_delay_range(int *low_bound_out,
+                                    int *high_bound_out,
+                                    int delay,
+                                    int base_delay)
+{
+  // This is the "decorrelated jitter" approach, from
+  //    https://www.awsarchitectureblog.com/2015/03/backoff.html
+  // The formula is
+  //    sleep = min(cap, random_between(base, sleep * 3))
+
+  const int delay_times_3 = delay < INT_MAX/3 ? delay * 3 : INT_MAX;
+  *low_bound_out = base_delay;
+  if (delay_times_3 > base_delay) {
+    *high_bound_out = delay_times_3;
+  } else {
+    *high_bound_out = base_delay+1;
+  }
+}
+
+/** Advance one delay step.  The algorithm will generate a random delay,
+ * such that each failure is possibly (random) longer than the ones before.
+ *
+ * We then clamp that value to be no larger than max_delay, and return it.
+ *
+ * The <b>base_delay</b> parameter is lowest possible delay time (can't be
+ * zero); the <b>backoff_position</b> parameter is the number of times we've
+ * generated a delay; and the <b>delay</b> argument is the most recently used
+ * delay.
+ */
+STATIC int
+next_random_exponential_delay(int delay,
+                              int base_delay)
+{
+  /* Check preconditions */
+  if (BUG(delay < 0))
+    delay = 0;
+
+  if (base_delay < 1)
+    base_delay = 1;
+
+  int low_bound=0, high_bound=INT_MAX;
+
+  next_random_exponential_delay_range(&low_bound, &high_bound,
+                                      delay, base_delay);
+
+  return crypto_rand_int_range(low_bound, high_bound);
+}
+
+/** Find the current delay for dls based on min_delay.
+ *
+ * This function sets dls->next_attempt_at based on now, and returns the delay.
+ * Helper for download_status_increment_failure and
+ * download_status_increment_attempt. */
+STATIC int
+download_status_schedule_get_delay(download_status_t *dls,
+                                   int min_delay,
+                                   time_t now)
+{
+  tor_assert(dls);
+  /* If we're using random exponential backoff, we do need min/max delay */
+  tor_assert(min_delay >= 0);
+
+  int delay = INT_MAX;
+  uint8_t dls_schedule_position = (dls->increment_on
+                                   == DL_SCHED_INCREMENT_ATTEMPT
+                                   ? dls->n_download_attempts
+                                   : dls->n_download_failures);
+
+  /* Check if we missed a reset somehow */
+  IF_BUG_ONCE(dls->last_backoff_position > dls_schedule_position) {
+    dls->last_backoff_position = 0;
+    dls->last_delay_used = 0;
+  }
+
+  if (dls_schedule_position > 0) {
+    delay = dls->last_delay_used;
+
+    while (dls->last_backoff_position < dls_schedule_position) {
+      /* Do one increment step */
+      delay = next_random_exponential_delay(delay, min_delay);
+      /* Update our position */
+      ++(dls->last_backoff_position);
+    }
+  } else {
+    /* If we're just starting out, use the minimum delay */
+    delay = min_delay;
+  }
+
+  /* Clamp it within min/max if we have them */
+  if (min_delay >= 0 && delay < min_delay) delay = min_delay;
+
+  /* Store it for next time */
+  dls->last_backoff_position = dls_schedule_position;
+  dls->last_delay_used = delay;
+
+  /* A negative delay makes no sense. Knowing that delay is
+   * non-negative allows us to safely do the wrapping check below. */
+  tor_assert(delay >= 0);
+
+  /* Avoid now+delay overflowing TIME_MAX, by comparing with a subtraction
+   * that won't overflow (since delay is non-negative). */
+  if (delay < INT_MAX && now <= TIME_MAX - delay) {
+    dls->next_attempt_at = now+delay;
+  } else {
+    dls->next_attempt_at = TIME_MAX;
+  }
+
+  return delay;
+}
+
+/* Log a debug message about item, which increments on increment_action, has
+ * incremented dls_n_download_increments times. The message varies based on
+ * was_schedule_incremented (if not, not_incremented_response is logged), and
+ * the values of increment, dls_next_attempt_at, and now.
+ * Helper for download_status_increment_failure and
+ * download_status_increment_attempt. */
+static void
+download_status_log_helper(const char *item, int was_schedule_incremented,
+                           const char *increment_action,
+                           const char *not_incremented_response,
+                           uint8_t dls_n_download_increments, int increment,
+                           time_t dls_next_attempt_at, time_t now)
+{
+  if (item) {
+    if (!was_schedule_incremented)
+      log_debug(LD_DIR, "%s %s %d time(s); I'll try again %s.",
+                item, increment_action, (int)dls_n_download_increments,
+                not_incremented_response);
+    else if (increment == 0)
+      log_debug(LD_DIR, "%s %s %d time(s); I'll try again immediately.",
+                item, increment_action, (int)dls_n_download_increments);
+    else if (dls_next_attempt_at < TIME_MAX)
+      log_debug(LD_DIR, "%s %s %d time(s); I'll try again in %d seconds.",
+                item, increment_action, (int)dls_n_download_increments,
+                (int)(dls_next_attempt_at-now));
+    else
+      log_debug(LD_DIR, "%s %s %d time(s); Giving up for a while.",
+                item, increment_action, (int)dls_n_download_increments);
+  }
+}
+
+/** Determine when a failed download attempt should be retried.
+ * Called when an attempt to download <b>dls</b> has failed with HTTP status
+ * <b>status_code</b>.  Increment the failure count (if the code indicates a
+ * real failure, or if we're a server) and set <b>dls</b>-\>next_attempt_at to
+ * an appropriate time in the future and return it.
+ * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_ATTEMPT, increment the
+ * failure count, and return a time in the far future for the next attempt (to
+ * avoid an immediate retry). */
+time_t
+download_status_increment_failure(download_status_t *dls, int status_code,
+                                  const char *item, int server, time_t now)
+{
+  (void) status_code; // XXXX no longer used.
+  (void) server; // XXXX no longer used.
+  int increment = -1;
+  int min_delay = 0;
+
+  tor_assert(dls);
+
+  /* dls wasn't reset before it was used */
+  if (dls->next_attempt_at == 0) {
+    download_status_reset(dls);
+  }
+
+  /* count the failure */
+  if (dls->n_download_failures < IMPOSSIBLE_TO_DOWNLOAD-1) {
+    ++dls->n_download_failures;
+  }
+
+  if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) {
+    /* We don't find out that a failure-based schedule has attempted a
+     * connection until that connection fails.
+     * We'll never find out about successful connections, but this doesn't
+     * matter, because schedules are reset after a successful download.
+     */
+    if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1)
+      ++dls->n_download_attempts;
+
+    /* only return a failure retry time if this schedule increments on failures
+     */
+    min_delay = find_dl_min_delay(dls, get_options());
+    increment = download_status_schedule_get_delay(dls, min_delay, now);
+  }
+
+  download_status_log_helper(item, !dls->increment_on, "failed",
+                             "concurrently", dls->n_download_failures,
+                             increment,
+                             download_status_get_next_attempt_at(dls),
+                             now);
+
+  if (dls->increment_on == DL_SCHED_INCREMENT_ATTEMPT) {
+    /* stop this schedule retrying on failure, it will launch concurrent
+     * connections instead */
+    return TIME_MAX;
+  } else {
+    return download_status_get_next_attempt_at(dls);
+  }
+}
+
+/** Determine when the next download attempt should be made when using an
+ * attempt-based (potentially concurrent) download schedule.
+ * Called when an attempt to download <b>dls</b> is being initiated.
+ * Increment the attempt count and set <b>dls</b>-\>next_attempt_at to an
+ * appropriate time in the future and return it.
+ * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_FAILURE, don't increment
+ * the attempts, and return a time in the far future (to avoid launching a
+ * concurrent attempt). */
+time_t
+download_status_increment_attempt(download_status_t *dls, const char *item,
+                                  time_t now)
+{
+  int delay = -1;
+  int min_delay = 0;
+
+  tor_assert(dls);
+
+  /* dls wasn't reset before it was used */
+  if (dls->next_attempt_at == 0) {
+    download_status_reset(dls);
+  }
+
+  if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) {
+    /* this schedule should retry on failure, and not launch any concurrent
+     attempts */
+    log_warn(LD_BUG, "Tried to launch an attempt-based connection on a "
+             "failure-based schedule.");
+    return TIME_MAX;
+  }
+
+  if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1)
+    ++dls->n_download_attempts;
+
+  min_delay = find_dl_min_delay(dls, get_options());
+  delay = download_status_schedule_get_delay(dls, min_delay, now);
+
+  download_status_log_helper(item, dls->increment_on, "attempted",
+                             "on failure", dls->n_download_attempts,
+                             delay, download_status_get_next_attempt_at(dls),
+                             now);
+
+  return download_status_get_next_attempt_at(dls);
+}
+
+static time_t
+download_status_get_initial_delay_from_now(const download_status_t *dls)
+{
+  /* We use constant initial delays, even in exponential backoff
+   * schedules. */
+  return time(NULL) + find_dl_min_delay(dls, get_options());
+}
+
+/** Reset <b>dls</b> so that it will be considered downloadable
+ * immediately, and/or to show that we don't need it anymore.
+ *
+ * Must be called to initialise a download schedule, otherwise the zeroth item
+ * in the schedule will never be used.
+ *
+ * (We find the zeroth element of the download schedule, and set
+ * next_attempt_at to be the appropriate offset from 'now'. In most
+ * cases this means setting it to 'now', so the item will be immediately
+ * downloadable; when using authorities with fallbacks, there is a few seconds'
+ * delay.) */
+void
+download_status_reset(download_status_t *dls)
+{
+  if (dls->n_download_failures == IMPOSSIBLE_TO_DOWNLOAD
+      || dls->n_download_attempts == IMPOSSIBLE_TO_DOWNLOAD)
+    return; /* Don't reset this. */
+
+  dls->n_download_failures = 0;
+  dls->n_download_attempts = 0;
+  dls->next_attempt_at = download_status_get_initial_delay_from_now(dls);
+  dls->last_backoff_position = 0;
+  dls->last_delay_used = 0;
+  /* Don't reset dls->want_authority or dls->increment_on */
+}
+
+/** Return true iff, as of <b>now</b>, the resource tracked by <b>dls</b> is
+ * ready to get its download reattempted. */
+int
+download_status_is_ready(download_status_t *dls, time_t now)
+{
+  /* dls wasn't reset before it was used */
+  if (dls->next_attempt_at == 0) {
+    download_status_reset(dls);
+  }
+
+  return download_status_get_next_attempt_at(dls) <= now;
+}
+
+/** Mark <b>dl</b> as never downloadable. */
+void
+download_status_mark_impossible(download_status_t *dl)
+{
+  dl->n_download_failures = IMPOSSIBLE_TO_DOWNLOAD;
+  dl->n_download_attempts = IMPOSSIBLE_TO_DOWNLOAD;
+}
+
+/** Return the number of failures on <b>dls</b> since the last success (if
+ * any). */
+int
+download_status_get_n_failures(const download_status_t *dls)
+{
+  return dls->n_download_failures;
+}
+
+/** Return the number of attempts to download <b>dls</b> since the last success
+ * (if any). This can differ from download_status_get_n_failures() due to
+ * outstanding concurrent attempts. */
+int
+download_status_get_n_attempts(const download_status_t *dls)
+{
+  return dls->n_download_attempts;
+}
+
+/** Return the next time to attempt to download <b>dls</b>. */
+time_t
+download_status_get_next_attempt_at(const download_status_t *dls)
+{
+  /* dls wasn't reset before it was used */
+  if (dls->next_attempt_at == 0) {
+    /* so give the answer we would have given if it had been */
+    return download_status_get_initial_delay_from_now(dls);
+  }
+
+  return dls->next_attempt_at;
+}
diff --git a/src/feature/dirclient/dlstatus.h b/src/feature/dirclient/dlstatus.h
new file mode 100644
index 0000000000..99e0d0225b
--- /dev/null
+++ b/src/feature/dirclient/dlstatus.h
@@ -0,0 +1,58 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dlstatus.h
+ * \brief Header file for dlstatus.c.
+ **/
+
+#ifndef TOR_DLSTATUS_H
+#define TOR_DLSTATUS_H
+
+time_t download_status_increment_failure(download_status_t *dls,
+                                         int status_code, const char *item,
+                                         int server, time_t now);
+time_t download_status_increment_attempt(download_status_t *dls,
+                                         const char *item,  time_t now);
+/** Increment the failure count of the download_status_t <b>dls</b>, with
+ * the optional status code <b>sc</b>. */
+#define download_status_failed(dls, sc)                                 \
+  download_status_increment_failure((dls), (sc), NULL,                  \
+                                    dir_server_mode(get_options()), \
+                                    time(NULL))
+
+void download_status_reset(download_status_t *dls);
+int download_status_is_ready(download_status_t *dls, time_t now);
+time_t download_status_get_next_attempt_at(const download_status_t *dls);
+void download_status_mark_impossible(download_status_t *dl);
+
+int download_status_get_n_failures(const download_status_t *dls);
+int download_status_get_n_attempts(const download_status_t *dls);
+
+#ifdef DLSTATUS_PRIVATE
+STATIC int download_status_schedule_get_delay(download_status_t *dls,
+                                              int min_delay,
+                                              time_t now);
+
+STATIC int find_dl_min_delay(const download_status_t *dls,
+                             const or_options_t *options);
+
+STATIC int next_random_exponential_delay(int delay,
+                                         int base_delay);
+
+STATIC void next_random_exponential_delay_range(int *low_bound_out,
+                                                int *high_bound_out,
+                                                int delay,
+                                                int base_delay);
+
+/* no more than quadruple the previous delay (multiplier + 1) */
+#define DIR_DEFAULT_RANDOM_MULTIPLIER (3)
+/* no more than triple the previous delay */
+#define DIR_TEST_NET_RANDOM_MULTIPLIER (2)
+
+#endif
+
+#endif /* !defined(TOR_DLSTATUS_H) */
diff --git a/src/feature/dirclient/download_status_st.h b/src/feature/dirclient/download_status_st.h
new file mode 100644
index 0000000000..11555a1dcc
--- /dev/null
+++ b/src/feature/dirclient/download_status_st.h
@@ -0,0 +1,65 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef DOWNLOAD_STATUS_ST_H
+#define DOWNLOAD_STATUS_ST_H
+
+/** Information about our plans for retrying downloads for a downloadable
+ * directory object.
+ * Each type of downloadable directory object has a corresponding retry
+ * <b>schedule</b>, which can be different depending on whether the object is
+ * being downloaded from an authority or a mirror (<b>want_authority</b>).
+ * <b>next_attempt_at</b> contains the next time we will attempt to download
+ * the object.
+ * For schedules that <b>increment_on</b> failure, <b>n_download_failures</b>
+ * is used to determine the position in the schedule. (Each schedule is a
+ * smartlist of integer delays, parsed from a CSV option.) Every time a
+ * connection attempt fails, <b>n_download_failures</b> is incremented,
+ * the new delay value is looked up from the schedule, and
+ * <b>next_attempt_at</b> is set delay seconds from the time the previous
+ * connection failed. Therefore, at most one failure-based connection can be
+ * in progress for each download_status_t.
+ * For schedules that <b>increment_on</b> attempt, <b>n_download_attempts</b>
+ * is used to determine the position in the schedule. Every time a
+ * connection attempt is made, <b>n_download_attempts</b> is incremented,
+ * the new delay value is looked up from the schedule, and
+ * <b>next_attempt_at</b> is set delay seconds from the time the previous
+ * connection was attempted. Therefore, multiple concurrent attempted-based
+ * connections can be in progress for each download_status_t.
+ * After an object is successfully downloaded, any other concurrent connections
+ * are terminated. A new schedule which starts at position 0 is used for
+ * subsequent downloads of the same object.
+ */
+struct download_status_t {
+  time_t next_attempt_at; /**< When should we try downloading this object
+                           * again? */
+  uint8_t n_download_failures; /**< Number of failed downloads of the most
+                                * recent object, since the last success. */
+  uint8_t n_download_attempts; /**< Number of (potentially concurrent) attempts
+                                * to download the most recent object, since
+                                * the last success. */
+  download_schedule_bitfield_t schedule : 8; /**< What kind of object is being
+                                              * downloaded? This determines the
+                                              * schedule used for the download.
+                                              */
+  download_want_authority_bitfield_t want_authority : 1; /**< Is the download
+                                              * happening from an authority
+                                              * or a mirror? This determines
+                                              * the schedule used for the
+                                              * download. */
+  download_schedule_increment_bitfield_t increment_on : 1; /**< does this
+                                        * schedule increment on each attempt,
+                                        * or after each failure? */
+  uint8_t last_backoff_position; /**< number of attempts/failures, depending
+                                  * on increment_on, when we last recalculated
+                                  * the delay.  Only updated if backoff
+                                  * == 1. */
+  int last_delay_used; /**< last delay used for random exponential backoff;
+                        * only updated if backoff == 1 */
+};
+
+#endif
+
diff --git a/src/feature/dircommon/consdiff.c b/src/feature/dircommon/consdiff.c
new file mode 100644
index 0000000000..d0f7594ce3
--- /dev/null
+++ b/src/feature/dircommon/consdiff.c
@@ -0,0 +1,1414 @@
+/* Copyright (c) 2014, Daniel Martí
+ * Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file consdiff.c
+ * \brief Consensus diff implementation, including both the generation and the
+ * application of diffs in a minimal ed format.
+ *
+ * The consensus diff application is done in consdiff_apply_diff, which relies
+ * on apply_ed_diff for the main ed diff part and on some digest helper
+ * functions to check the digest hashes found in the consensus diff header.
+ *
+ * The consensus diff generation is more complex. consdiff_gen_diff generates
+ * it, relying on gen_ed_diff to generate the ed diff and some digest helper
+ * functions to generate the digest hashes.
+ *
+ * gen_ed_diff is the tricky bit. In it simplest form, it will take quadratic
+ * time and linear space to generate an ed diff given two smartlists. As shown
+ * in its comment section, calling calc_changes on the entire two consensuses
+ * will calculate what is to be added and what is to be deleted in the diff.
+ * Its comment section briefly explains how it works.
+ *
+ * In our case specific to consensuses, we take advantage of the fact that
+ * consensuses list routers sorted by their identities. We use that
+ * information to avoid running calc_changes on the whole smartlists.
+ * gen_ed_diff will navigate through the two consensuses identity by identity
+ * and will send small couples of slices to calc_changes, keeping the running
+ * time near-linear. This is explained in more detail in the gen_ed_diff
+ * comments.
+ *
+ * The allocation strategy tries to save time and memory by avoiding needless
+ * copies.  Instead of actually splitting the inputs into separate strings, we
+ * allocate cdline_t objects, each of which represents a line in the original
+ * object or in the output.  We use memarea_t allocators to manage the
+ * temporary memory we use when generating or applying diffs.
+ **/
+
+#define CONSDIFF_PRIVATE
+
+#include "core/or/or.h"
+#include "feature/dircommon/consdiff.h"
+#include "lib/memarea/memarea.h"
+#include "feature/dirparse/ns_parse.h"
+
+static const char* ns_diff_version = "network-status-diff-version 1";
+static const char* hash_token = "hash";
+
+static char *consensus_join_lines(const smartlist_t *inp);
+
+/** Return true iff a and b have the same contents. */
+STATIC int
+lines_eq(const cdline_t *a, const cdline_t *b)
+{
+  return a->len == b->len && fast_memeq(a->s, b->s, a->len);
+}
+
+/** Return true iff a has the same contents as the nul-terminated string b. */
+STATIC int
+line_str_eq(const cdline_t *a, const char *b)
+{
+  const size_t len = strlen(b);
+  tor_assert(len <= UINT32_MAX);
+  cdline_t bline = { b, (uint32_t)len };
+  return lines_eq(a, &bline);
+}
+
+/** Return true iff a begins with the same contents as the nul-terminated
+ * string b. */
+static int
+line_starts_with_str(const cdline_t *a, const char *b)
+{
+  const size_t len = strlen(b);
+  tor_assert(len <= UINT32_MAX);
+  return a->len >= len && fast_memeq(a->s, b, len);
+}
+
+/** Return a new cdline_t holding as its contents the nul-terminated
+ * string s. Use the provided memory area for storage. */
+static cdline_t *
+cdline_linecpy(memarea_t *area, const char *s)
+{
+  size_t len = strlen(s);
+  const char *ss = memarea_memdup(area, s, len);
+  cdline_t *line = memarea_alloc(area, sizeof(cdline_t));
+  line->s = ss;
+  line->len = (uint32_t)len;
+  return line;
+}
+
+/** Add a cdline_t to <b>lst</b> holding as its contents the nul-terminated
+ * string s.  Use the provided memory area for storage. */
+STATIC void
+smartlist_add_linecpy(smartlist_t *lst, memarea_t *area, const char *s)
+{
+  smartlist_add(lst, cdline_linecpy(area, s));
+}
+
+/** Compute the digest of <b>cons</b>, and store the result in
+ * <b>digest_out</b>. Return 0 on success, -1 on failure. */
+/* This is a separate, mockable function so that we can override it when
+ * fuzzing. */
+MOCK_IMPL(STATIC int,
+consensus_compute_digest,(const char *cons,
+                          consensus_digest_t *digest_out))
+{
+  int r = crypto_digest256((char*)digest_out->sha3_256,
+                           cons, strlen(cons), DIGEST_SHA3_256);
+  return r;
+}
+
+/** Compute the digest-as-signed of <b>cons</b>, and store the result in
+ * <b>digest_out</b>. Return 0 on success, -1 on failure. */
+/* This is a separate, mockable function so that we can override it when
+ * fuzzing. */
+MOCK_IMPL(STATIC int,
+consensus_compute_digest_as_signed,(const char *cons,
+                                    consensus_digest_t *digest_out))
+{
+  return router_get_networkstatus_v3_sha3_as_signed(digest_out->sha3_256,
+                                                    cons);
+}
+
+/** Return true iff <b>d1</b> and <b>d2</b> contain the same digest */
+/* This is a separate, mockable function so that we can override it when
+ * fuzzing. */
+MOCK_IMPL(STATIC int,
+consensus_digest_eq,(const uint8_t *d1,
+                     const uint8_t *d2))
+{
+  return fast_memeq(d1, d2, DIGEST256_LEN);
+}
+
+/** Create (allocate) a new slice from a smartlist. Assumes that the start
+ * and the end indexes are within the bounds of the initial smartlist. The end
+ * element is not part of the resulting slice. If end is -1, the slice is to
+ * reach the end of the smartlist.
+ */
+STATIC smartlist_slice_t *
+smartlist_slice(const smartlist_t *list, int start, int end)
+{
+  int list_len = smartlist_len(list);
+  tor_assert(start >= 0);
+  tor_assert(start <= list_len);
+  if (end == -1) {
+    end = list_len;
+  }
+  tor_assert(start <= end);
+
+  smartlist_slice_t *slice = tor_malloc(sizeof(smartlist_slice_t));
+  slice->list = list;
+  slice->offset = start;
+  slice->len = end - start;
+  return slice;
+}
+
+/** Helper: Compute the longest common subsequence lengths for the two slices.
+ * Used as part of the diff generation to find the column at which to split
+ * slice2 while still having the optimal solution.
+ * If direction is -1, the navigation is reversed. Otherwise it must be 1.
+ * The length of the resulting integer array is that of the second slice plus
+ * one.
+ */
+STATIC int *
+lcs_lengths(const smartlist_slice_t *slice1, const smartlist_slice_t *slice2,
+            int direction)
+{
+  size_t a_size = sizeof(int) * (slice2->len+1);
+
+  /* Resulting lcs lengths. */
+  int *result = tor_malloc_zero(a_size);
+  /* Copy of the lcs lengths from the last iteration. */
+  int *prev = tor_malloc(a_size);
+
+  tor_assert(direction == 1 || direction == -1);
+
+  int si = slice1->offset;
+  if (direction == -1) {
+    si += (slice1->len-1);
+  }
+
+  for (int i = 0; i < slice1->len; ++i, si+=direction) {
+
+    const cdline_t *line1 = smartlist_get(slice1->list, si);
+    /* Store the last results. */
+    memcpy(prev, result, a_size);
+
+    int sj = slice2->offset;
+    if (direction == -1) {
+      sj += (slice2->len-1);
+    }
+
+    for (int j = 0; j < slice2->len; ++j, sj+=direction) {
+
+      const cdline_t *line2 = smartlist_get(slice2->list, sj);
+      if (lines_eq(line1, line2)) {
+        /* If the lines are equal, the lcs is one line longer. */
+        result[j + 1] = prev[j] + 1;
+      } else {
+        /* If not, see what lcs parent path is longer. */
+        result[j + 1] = MAX(result[j], prev[j + 1]);
+      }
+    }
+  }
+  tor_free(prev);
+  return result;
+}
+
+/** Helper: Trim any number of lines that are equally at the start or the end
+ * of both slices.
+ */
+STATIC void
+trim_slices(smartlist_slice_t *slice1, smartlist_slice_t *slice2)
+{
+  while (slice1->len>0 && slice2->len>0) {
+    const cdline_t *line1 = smartlist_get(slice1->list, slice1->offset);
+    const cdline_t *line2 = smartlist_get(slice2->list, slice2->offset);
+    if (!lines_eq(line1, line2)) {
+      break;
+    }
+    slice1->offset++; slice1->len--;
+    slice2->offset++; slice2->len--;
+  }
+
+  int i1 = (slice1->offset+slice1->len)-1;
+  int i2 = (slice2->offset+slice2->len)-1;
+
+  while (slice1->len>0 && slice2->len>0) {
+    const cdline_t *line1 = smartlist_get(slice1->list, i1);
+    const cdline_t *line2 = smartlist_get(slice2->list, i2);
+    if (!lines_eq(line1, line2)) {
+      break;
+    }
+    i1--;
+    slice1->len--;
+    i2--;
+    slice2->len--;
+  }
+}
+
+/** Like smartlist_string_pos, but uses a cdline_t, and is restricted to the
+ * bounds of the slice.
+ */
+STATIC int
+smartlist_slice_string_pos(const smartlist_slice_t *slice,
+                           const cdline_t *string)
+{
+  int end = slice->offset + slice->len;
+  for (int i = slice->offset; i < end; ++i) {
+    const cdline_t *el = smartlist_get(slice->list, i);
+    if (lines_eq(el, string)) {
+      return i;
+    }
+  }
+  return -1;
+}
+
+/** Helper: Set all the appropriate changed booleans to true. The first slice
+ * must be of length 0 or 1. All the lines of slice1 and slice2 which are not
+ * present in the other slice will be set to changed in their bool array.
+ * The two changed bool arrays are passed in the same order as the slices.
+ */
+STATIC void
+set_changed(bitarray_t *changed1, bitarray_t *changed2,
+            const smartlist_slice_t *slice1, const smartlist_slice_t *slice2)
+{
+  int toskip = -1;
+  tor_assert(slice1->len == 0 || slice1->len == 1);
+
+  if (slice1->len == 1) {
+    const cdline_t *line_common = smartlist_get(slice1->list, slice1->offset);
+    toskip = smartlist_slice_string_pos(slice2, line_common);
+    if (toskip == -1) {
+      bitarray_set(changed1, slice1->offset);
+    }
+  }
+  int end = slice2->offset + slice2->len;
+  for (int i = slice2->offset; i < end; ++i) {
+    if (i != toskip) {
+      bitarray_set(changed2, i);
+    }
+  }
+}
+
+/*
+ * Helper: Given that slice1 has been split by half into top and bot, we want
+ * to fetch the column at which to split slice2 so that we are still on track
+ * to the optimal diff solution, i.e. the shortest one. We use lcs_lengths
+ * since the shortest diff is just another way to say the longest common
+ * subsequence.
+ */
+static int
+optimal_column_to_split(const smartlist_slice_t *top,
+                        const smartlist_slice_t *bot,
+                        const smartlist_slice_t *slice2)
+{
+  int *lens_top = lcs_lengths(top, slice2, 1);
+  int *lens_bot = lcs_lengths(bot, slice2, -1);
+  int column=0, max_sum=-1;
+
+  for (int i = 0; i < slice2->len+1; ++i) {
+    int sum = lens_top[i] + lens_bot[slice2->len-i];
+    if (sum > max_sum) {
+      column = i;
+      max_sum = sum;
+    }
+  }
+  tor_free(lens_top);
+  tor_free(lens_bot);
+
+  return column;
+}
+
+/**
+ * Helper: Figure out what elements are new or gone on the second smartlist
+ * relative to the first smartlist, and store the booleans in the bitarrays.
+ * True on the first bitarray means the element is gone, true on the second
+ * bitarray means it's new.
+ *
+ * In its base case, either of the smartlists is of length <= 1 and we can
+ * quickly see what elements are new or are gone. In the other case, we will
+ * split one smartlist by half and we'll use optimal_column_to_split to find
+ * the optimal column at which to split the second smartlist so that we are
+ * finding the smallest diff possible.
+ */
+STATIC void
+calc_changes(smartlist_slice_t *slice1,
+             smartlist_slice_t *slice2,
+             bitarray_t *changed1, bitarray_t *changed2)
+{
+  trim_slices(slice1, slice2);
+
+  if (slice1->len <= 1) {
+    set_changed(changed1, changed2, slice1, slice2);
+
+  } else if (slice2->len <= 1) {
+    set_changed(changed2, changed1, slice2, slice1);
+
+  /* Keep on splitting the slices in two. */
+  } else {
+    smartlist_slice_t *top, *bot, *left, *right;
+
+    /* Split the first slice in half. */
+    int mid = slice1->len/2;
+    top = smartlist_slice(slice1->list, slice1->offset, slice1->offset+mid);
+    bot = smartlist_slice(slice1->list, slice1->offset+mid,
+        slice1->offset+slice1->len);
+
+    /* Split the second slice by the optimal column. */
+    int mid2 = optimal_column_to_split(top, bot, slice2);
+    left = smartlist_slice(slice2->list, slice2->offset, slice2->offset+mid2);
+    right = smartlist_slice(slice2->list, slice2->offset+mid2,
+        slice2->offset+slice2->len);
+
+    calc_changes(top, left, changed1, changed2);
+    calc_changes(bot, right, changed1, changed2);
+    tor_free(top);
+    tor_free(bot);
+    tor_free(left);
+    tor_free(right);
+  }
+}
+
+/* This table is from crypto.c. The SP and PAD defines are different. */
+#define NOT_VALID_BASE64 255
+#define X NOT_VALID_BASE64
+#define SP NOT_VALID_BASE64
+#define PAD NOT_VALID_BASE64
+static const uint8_t base64_compare_table[256] = {
+  X, X, X, X, X, X, X, X, X, SP, SP, SP, X, SP, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  SP, X, X, X, X, X, X, X, X, X, X, 62, X, X, X, 63,
+  52, 53, 54, 55, 56, 57, 58, 59, 60, 61, X, X, X, PAD, X, X,
+  X, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
+  15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, X, X, X, X, X,
+  X, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
+  41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+};
+
+/** Helper: Get the identity hash from a router line, assuming that the line
+ * at least appears to be a router line and thus starts with "r ".
+ *
+ * If an identity hash is found, store it (without decoding it) in
+ * <b>hash_out</b>, and return 0.  On failure, return -1.
+ */
+STATIC int
+get_id_hash(const cdline_t *line, cdline_t *hash_out)
+{
+  if (line->len < 2)
+    return -1;
+
+  /* Skip the router name. */
+  const char *hash = memchr(line->s + 2, ' ', line->len - 2);
+  if (!hash) {
+    return -1;
+  }
+
+  hash++;
+  const char *hash_end = hash;
+  /* Stop when the first non-base64 character is found. Use unsigned chars to
+   * avoid negative indexes causing crashes.
+   */
+  while (base64_compare_table[*((unsigned char*)hash_end)]
+           != NOT_VALID_BASE64 &&
+         hash_end < line->s + line->len) {
+    hash_end++;
+  }
+
+  /* Empty hash. */
+  if (hash_end == hash) {
+    return -1;
+  }
+
+  hash_out->s = hash;
+  /* Always true because lines are limited to this length */
+  tor_assert(hash_end >= hash);
+  tor_assert((size_t)(hash_end - hash) <= UINT32_MAX);
+  hash_out->len = (uint32_t)(hash_end - hash);
+
+  return 0;
+}
+
+/** Helper: Check that a line is a valid router entry. We must at least be
+ * able to fetch a proper identity hash from it for it to be valid.
+ */
+STATIC int
+is_valid_router_entry(const cdline_t *line)
+{
+  if (line->len < 2 || fast_memneq(line->s, "r ", 2))
+    return 0;
+  cdline_t tmp;
+  return (get_id_hash(line, &tmp) == 0);
+}
+
+/** Helper: Find the next router line starting at the current position.
+ * Assumes that cur is lower than the length of the smartlist, i.e. it is a
+ * line within the bounds of the consensus. The only exception is when we
+ * don't want to skip the first line, in which case cur will be -1.
+ */
+STATIC int
+next_router(const smartlist_t *cons, int cur)
+{
+  int len = smartlist_len(cons);
+  tor_assert(cur >= -1 && cur < len);
+
+  if (++cur >= len) {
+    return len;
+  }
+
+  const cdline_t *line = smartlist_get(cons, cur);
+  while (!is_valid_router_entry(line)) {
+    if (++cur >= len) {
+      return len;
+    }
+    line = smartlist_get(cons, cur);
+  }
+  return cur;
+}
+
+/** Helper: compare two base64-encoded identity hashes, which may be of
+ * different lengths. Comparison ends when the first non-base64 char is found.
+ */
+STATIC int
+base64cmp(const cdline_t *hash1, const cdline_t *hash2)
+{
+  /* NULL is always lower, useful for last_hash which starts at NULL. */
+  if (!hash1->s && !hash2->s) {
+    return 0;
+  }
+  if (!hash1->s) {
+    return -1;
+  }
+  if (!hash2->s) {
+    return 1;
+  }
+
+  /* Don't index with a char; char may be signed. */
+  const unsigned char *a = (unsigned char*)hash1->s;
+  const unsigned char *b = (unsigned char*)hash2->s;
+  const unsigned char *a_end = a + hash1->len;
+  const unsigned char *b_end = b + hash2->len;
+  while (1) {
+    uint8_t av = base64_compare_table[*a];
+    uint8_t bv = base64_compare_table[*b];
+    if (av == NOT_VALID_BASE64) {
+      if (bv == NOT_VALID_BASE64) {
+        /* Both ended with exactly the same characters. */
+        return 0;
+      } else {
+        /* hash2 goes on longer than hash1 and thus hash1 is lower. */
+        return -1;
+      }
+    } else if (bv == NOT_VALID_BASE64) {
+      /* hash1 goes on longer than hash2 and thus hash1 is greater. */
+      return 1;
+    } else if (av < bv) {
+      /* The first difference shows that hash1 is lower. */
+      return -1;
+    } else if (av > bv) {
+      /* The first difference shows that hash1 is greater. */
+      return 1;
+    } else {
+      a++;
+      b++;
+      if (a == a_end) {
+        if (b == b_end) {
+          return 0;
+        } else {
+          return -1;
+        }
+      } else if (b == b_end) {
+        return 1;
+      }
+    }
+  }
+}
+
+/** Structure used to remember the previous and current identity hash of
+ * the "r " lines in a consensus, to enforce well-ordering. */
+typedef struct router_id_iterator_t {
+  cdline_t last_hash;
+  cdline_t hash;
+} router_id_iterator_t;
+
+/**
+ * Initializer for a router_id_iterator_t.
+ */
+#define ROUTER_ID_ITERATOR_INIT { { NULL, 0 }, { NULL, 0 } }
+
+/** Given an index *<b>idxp</b> into the consensus at <b>cons</b>, advance
+ * the index to the next router line ("r ...") in the consensus, or to
+ * an index one after the end of the list if there is no such line.
+ *
+ * Use <b>iter</b> to record the hash of the found router line, if any,
+ * and to enforce ordering on the hashes.  If the hashes are mis-ordered,
+ * return -1.  Else, return 0.
+ **/
+static int
+find_next_router_line(const smartlist_t *cons,
+                      const char *consname,
+                      int *idxp,
+                      router_id_iterator_t *iter)
+{
+  *idxp = next_router(cons, *idxp);
+  if (*idxp < smartlist_len(cons)) {
+    memcpy(&iter->last_hash, &iter->hash, sizeof(cdline_t));
+    if (get_id_hash(smartlist_get(cons, *idxp), &iter->hash) < 0 ||
+        base64cmp(&iter->hash, &iter->last_hash) <= 0) {
+      log_warn(LD_CONSDIFF, "Refusing to generate consensus diff because "
+               "the %s consensus doesn't have its router entries sorted "
+               "properly.", consname);
+      return -1;
+    }
+  }
+  return 0;
+}
+
+/** Line-prefix indicating the beginning of the signatures section that we
+ * intend to delete. */
+#define START_OF_SIGNATURES_SECTION "directory-signature "
+
+/** Pre-process a consensus in <b>cons</b> (represented as a list of cdline_t)
+ * to remove the signatures from it.  If the footer is removed, return a
+ * cdline_t containing a delete command to delete the footer, allocated in
+ * <b>area</>.  If no footer is removed, return NULL.
+ *
+ * We remove the signatures here because they are not themselves signed, and
+ * as such there might be different encodings for them.
+ */
+static cdline_t *
+preprocess_consensus(memarea_t *area,
+                     smartlist_t *cons)
+{
+  int idx;
+  int dirsig_idx = -1;
+  for (idx = 0; idx < smartlist_len(cons); ++idx) {
+    cdline_t *line = smartlist_get(cons, idx);
+    if (line_starts_with_str(line, START_OF_SIGNATURES_SECTION)) {
+      dirsig_idx = idx;
+      break;
+    }
+  }
+  if (dirsig_idx >= 0) {
+    char buf[64];
+    while (smartlist_len(cons) > dirsig_idx)
+      smartlist_del(cons, dirsig_idx);
+    tor_snprintf(buf, sizeof(buf), "%d,$d", dirsig_idx+1);
+    return cdline_linecpy(area, buf);
+  } else {
+    return NULL;
+  }
+}
+
+/** Generate an ed diff as a smartlist from two consensuses, also given as
+ * smartlists. Will return NULL if the diff could not be generated, which can
+ * happen if any lines the script had to add matched "." or if the routers
+ * were not properly ordered.
+ *
+ * All cdline_t objects in the resulting object are either references to lines
+ * in one of the inputs, or are newly allocated lines in the provided memarea.
+ *
+ * This implementation is consensus-specific. To generate an ed diff for any
+ * given input in quadratic time, you can replace all the code until the
+ * navigation in reverse order with the following:
+ *
+ *   int len1 = smartlist_len(cons1);
+ *   int len2 = smartlist_len(cons2);
+ *   bitarray_t *changed1 = bitarray_init_zero(len1);
+ *   bitarray_t *changed2 = bitarray_init_zero(len2);
+ *   cons1_sl = smartlist_slice(cons1, 0, -1);
+ *   cons2_sl = smartlist_slice(cons2, 0, -1);
+ *   calc_changes(cons1_sl, cons2_sl, changed1, changed2);
+ */
+STATIC smartlist_t *
+gen_ed_diff(const smartlist_t *cons1_orig, const smartlist_t *cons2,
+            memarea_t *area)
+{
+  smartlist_t *cons1 = smartlist_new();
+  smartlist_add_all(cons1, cons1_orig);
+  cdline_t *remove_trailer = preprocess_consensus(area, cons1);
+
+  int len1 = smartlist_len(cons1);
+  int len2 = smartlist_len(cons2);
+  smartlist_t *result = smartlist_new();
+
+  if (remove_trailer) {
+    /* There's a delete-the-trailer line at the end, so add it here. */
+    smartlist_add(result, remove_trailer);
+  }
+
+  /* Initialize the changed bitarrays to zero, so that calc_changes only needs
+   * to set the ones that matter and leave the rest untouched.
+   */
+  bitarray_t *changed1 = bitarray_init_zero(len1);
+  bitarray_t *changed2 = bitarray_init_zero(len2);
+  int i1=-1, i2=-1;
+  int start1=0, start2=0;
+
+  /* To check that hashes are ordered properly */
+  router_id_iterator_t iter1 = ROUTER_ID_ITERATOR_INIT;
+  router_id_iterator_t iter2 = ROUTER_ID_ITERATOR_INIT;
+
+  /* i1 and i2 are initialized at the first line of each consensus. They never
+   * reach past len1 and len2 respectively, since next_router doesn't let that
+   * happen. i1 and i2 are advanced by at least one line at each iteration as
+   * long as they have not yet reached len1 and len2, so the loop is
+   * guaranteed to end, and each pair of (i1,i2) will be inspected at most
+   * once.
+   */
+  while (i1 < len1 || i2 < len2) {
+
+    /* Advance each of the two navigation positions by one router entry if not
+     * yet at the end.
+     */
+    if (i1 < len1) {
+      if (find_next_router_line(cons1, "base", &i1, &iter1) < 0) {
+        goto error_cleanup;
+      }
+    }
+
+    if (i2 < len2) {
+      if (find_next_router_line(cons2, "target", &i2, &iter2) < 0) {
+        goto error_cleanup;
+      }
+    }
+
+    /* If we have reached the end of both consensuses, there is no need to
+     * compare hashes anymore, since this is the last iteration.
+     */
+    if (i1 < len1 || i2 < len2) {
+
+      /* Keep on advancing the lower (by identity hash sorting) position until
+       * we have two matching positions. The only other possible outcome is
+       * that a lower position reaches the end of the consensus before it can
+       * reach a hash that is no longer the lower one. Since there will always
+       * be a lower hash for as long as the loop runs, one of the two indexes
+       * will always be incremented, thus assuring that the loop must end
+       * after a finite number of iterations. If that cannot be because said
+       * consensus has already reached the end, both are extended to their
+       * respecting ends since we are done.
+       */
+      int cmp = base64cmp(&iter1.hash, &iter2.hash);
+      while (cmp != 0) {
+        if (i1 < len1 && cmp < 0) {
+          if (find_next_router_line(cons1, "base", &i1, &iter1) < 0) {
+            goto error_cleanup;
+          }
+          if (i1 == len1) {
+            /* We finished the first consensus, so grab all the remaining
+             * lines of the second consensus and finish up.
+             */
+            i2 = len2;
+            break;
+          }
+        } else if (i2 < len2 && cmp > 0) {
+          if (find_next_router_line(cons2, "target", &i2, &iter2) < 0) {
+            goto error_cleanup;
+          }
+          if (i2 == len2) {
+            /* We finished the second consensus, so grab all the remaining
+             * lines of the first consensus and finish up.
+             */
+            i1 = len1;
+            break;
+          }
+        } else {
+          i1 = len1;
+          i2 = len2;
+          break;
+        }
+        cmp = base64cmp(&iter1.hash, &iter2.hash);
+      }
+    }
+
+    /* Make slices out of these chunks (up to the common router entry) and
+     * calculate the changes for them.
+     * Error if any of the two slices are longer than 10K lines. That should
+     * never happen with any pair of real consensuses. Feeding more than 10K
+     * lines to calc_changes would be very slow anyway.
+     */
+#define MAX_LINE_COUNT (10000)
+    if (i1-start1 > MAX_LINE_COUNT || i2-start2 > MAX_LINE_COUNT) {
+      log_warn(LD_CONSDIFF, "Refusing to generate consensus diff because "
+          "we found too few common router ids.");
+      goto error_cleanup;
+    }
+
+    smartlist_slice_t *cons1_sl = smartlist_slice(cons1, start1, i1);
+    smartlist_slice_t *cons2_sl = smartlist_slice(cons2, start2, i2);
+    calc_changes(cons1_sl, cons2_sl, changed1, changed2);
+    tor_free(cons1_sl);
+    tor_free(cons2_sl);
+    start1 = i1, start2 = i2;
+  }
+
+  /* Navigate the changes in reverse order and generate one ed command for
+   * each chunk of changes.
+   */
+  i1=len1-1, i2=len2-1;
+  char buf[128];
+  while (i1 >= 0 || i2 >= 0) {
+
+    int start1x, start2x, end1, end2, added, deleted;
+
+    /* We are at a point were no changed bools are true, so just keep going. */
+    if (!(i1 >= 0 && bitarray_is_set(changed1, i1)) &&
+        !(i2 >= 0 && bitarray_is_set(changed2, i2))) {
+      if (i1 >= 0) {
+        i1--;
+      }
+      if (i2 >= 0) {
+        i2--;
+      }
+      continue;
+    }
+
+    end1 = i1, end2 = i2;
+
+    /* Grab all contiguous changed lines */
+    while (i1 >= 0 && bitarray_is_set(changed1, i1)) {
+      i1--;
+    }
+    while (i2 >= 0 && bitarray_is_set(changed2, i2)) {
+      i2--;
+    }
+
+    start1x = i1+1, start2x = i2+1;
+    added = end2-i2, deleted = end1-i1;
+
+    if (added == 0) {
+      if (deleted == 1) {
+        tor_snprintf(buf, sizeof(buf), "%id", start1x+1);
+        smartlist_add_linecpy(result, area, buf);
+      } else {
+        tor_snprintf(buf, sizeof(buf), "%i,%id", start1x+1, start1x+deleted);
+        smartlist_add_linecpy(result, area, buf);
+      }
+    } else {
+      int i;
+      if (deleted == 0) {
+        tor_snprintf(buf, sizeof(buf), "%ia", start1x);
+        smartlist_add_linecpy(result, area, buf);
+      } else if (deleted == 1) {
+        tor_snprintf(buf, sizeof(buf), "%ic", start1x+1);
+        smartlist_add_linecpy(result, area, buf);
+      } else {
+        tor_snprintf(buf, sizeof(buf), "%i,%ic", start1x+1, start1x+deleted);
+        smartlist_add_linecpy(result, area, buf);
+      }
+
+      for (i = start2x; i <= end2; ++i) {
+        cdline_t *line = smartlist_get(cons2, i);
+        if (line_str_eq(line, ".")) {
+          log_warn(LD_CONSDIFF, "Cannot generate consensus diff because "
+              "one of the lines to be added is \".\".");
+          goto error_cleanup;
+        }
+        smartlist_add(result, line);
+      }
+      smartlist_add_linecpy(result, area, ".");
+    }
+  }
+
+  smartlist_free(cons1);
+  bitarray_free(changed1);
+  bitarray_free(changed2);
+
+  return result;
+
+ error_cleanup:
+
+  smartlist_free(cons1);
+  bitarray_free(changed1);
+  bitarray_free(changed2);
+
+  smartlist_free(result);
+
+  return NULL;
+}
+
+/* Helper: Read a base-10 number between 0 and INT32_MAX from <b>s</b> and
+ * store it in <b>num_out</b>.  Advance <b>s</b> to the characer immediately
+ * after the number.  Return 0 on success, -1 on failure. */
+static int
+get_linenum(const char **s, int *num_out)
+{
+  int ok;
+  char *next;
+  if (!TOR_ISDIGIT(**s)) {
+    return -1;
+  }
+  *num_out = (int) tor_parse_long(*s, 10, 0, INT32_MAX, &ok, &next);
+  if (ok && next) {
+    *s = next;
+    return 0;
+  } else {
+    return -1;
+  }
+}
+
+/** Apply the ed diff, starting at <b>diff_starting_line</b>, to the consensus
+ * and return a new consensus, also as a line-based smartlist. Will return
+ * NULL if the ed diff is not properly formatted.
+ *
+ * All cdline_t objects in the resulting object are references to lines
+ * in one of the inputs; nothing is copied.
+ */
+STATIC smartlist_t *
+apply_ed_diff(const smartlist_t *cons1, const smartlist_t *diff,
+              int diff_starting_line)
+{
+  int diff_len = smartlist_len(diff);
+  int j = smartlist_len(cons1);
+  smartlist_t *cons2 = smartlist_new();
+
+  for (int i=diff_starting_line; i<diff_len; ++i) {
+    const cdline_t *diff_cdline = smartlist_get(diff, i);
+    char diff_line[128];
+
+    if (diff_cdline->len > sizeof(diff_line) - 1) {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+               "an ed command was far too long");
+      goto error_cleanup;
+    }
+    /* Copy the line to make it nul-terminated. */
+    memcpy(diff_line, diff_cdline->s, diff_cdline->len);
+    diff_line[diff_cdline->len] = 0;
+    const char *ptr = diff_line;
+    int start = 0, end = 0;
+    int had_range = 0;
+    int end_was_eof = 0;
+    if (get_linenum(&ptr, &start) < 0) {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+               "an ed command was missing a line number.");
+      goto error_cleanup;
+    }
+    if (*ptr == ',') {
+      /* Two-item range */
+      had_range = 1;
+      ++ptr;
+      if (*ptr == '$') {
+        end_was_eof = 1;
+        end = smartlist_len(cons1);
+        ++ptr;
+      } else if (get_linenum(&ptr, &end) < 0) {
+        log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+                 "an ed command was missing a range end line number.");
+        goto error_cleanup;
+      }
+      /* Incoherent range. */
+      if (end <= start) {
+        log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+                 "an invalid range was found in an ed command.");
+        goto error_cleanup;
+      }
+    } else {
+      /* We'll take <n1> as <n1>,<n1> for simplicity. */
+      end = start;
+    }
+
+    if (end > j) {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+          "its commands are not properly sorted in reverse order.");
+      goto error_cleanup;
+    }
+
+    if (*ptr == '\0') {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+               "a line with no ed command was found");
+      goto error_cleanup;
+    }
+
+    if (*(ptr+1) != '\0') {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+          "an ed command longer than one char was found.");
+      goto error_cleanup;
+    }
+
+    char action = *ptr;
+
+    switch (action) {
+      case 'a':
+      case 'c':
+      case 'd':
+        break;
+      default:
+        log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+            "an unrecognised ed command was found.");
+        goto error_cleanup;
+    }
+
+    /** $ is not allowed with non-d actions. */
+    if (end_was_eof && action != 'd') {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+               "it wanted to use $ with a command other than delete");
+      goto error_cleanup;
+    }
+
+    /* 'a' commands are not allowed to have ranges. */
+    if (had_range && action == 'a') {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+          "it wanted to add lines after a range.");
+      goto error_cleanup;
+    }
+
+    /* Add unchanged lines. */
+    for (; j && j > end; --j) {
+      cdline_t *cons_line = smartlist_get(cons1, j-1);
+      smartlist_add(cons2, cons_line);
+    }
+
+    /* Ignore removed lines. */
+    if (action == 'c' || action == 'd') {
+      while (--j >= start) {
+        /* Skip line */
+      }
+    }
+
+    /* Add new lines in reverse order, since it will all be reversed at the
+     * end.
+     */
+    if (action == 'a' || action == 'c') {
+      int added_end = i;
+
+      i++; /* Skip the line with the range and command. */
+      while (i < diff_len) {
+        if (line_str_eq(smartlist_get(diff, i), ".")) {
+          break;
+        }
+        if (++i == diff_len) {
+          log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+              "it has lines to be inserted that don't end with a \".\".");
+          goto error_cleanup;
+        }
+      }
+
+      int added_i = i-1;
+
+      /* It would make no sense to add zero new lines. */
+      if (added_i == added_end) {
+        log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+            "it has an ed command that tries to insert zero lines.");
+        goto error_cleanup;
+      }
+
+      while (added_i > added_end) {
+        cdline_t *added_line = smartlist_get(diff, added_i--);
+        smartlist_add(cons2, added_line);
+      }
+    }
+  }
+
+  /* Add remaining unchanged lines. */
+  for (; j > 0; --j) {
+    cdline_t *cons_line = smartlist_get(cons1, j-1);
+    smartlist_add(cons2, cons_line);
+  }
+
+  /* Reverse the whole thing since we did it from the end. */
+  smartlist_reverse(cons2);
+  return cons2;
+
+ error_cleanup:
+
+  smartlist_free(cons2);
+
+  return NULL;
+}
+
+/** Generate a consensus diff as a smartlist from two given consensuses, also
+ * as smartlists. Will return NULL if the consensus diff could not be
+ * generated. Neither of the two consensuses are modified in any way, so it's
+ * up to the caller to free their resources.
+ */
+smartlist_t *
+consdiff_gen_diff(const smartlist_t *cons1,
+                  const smartlist_t *cons2,
+                  const consensus_digest_t *digests1,
+                  const consensus_digest_t *digests2,
+                  memarea_t *area)
+{
+  smartlist_t *ed_diff = gen_ed_diff(cons1, cons2, area);
+  /* ed diff could not be generated - reason already logged by gen_ed_diff. */
+  if (!ed_diff) {
+    goto error_cleanup;
+  }
+
+  /* See that the script actually produces what we want. */
+  smartlist_t *ed_cons2 = apply_ed_diff(cons1, ed_diff, 0);
+  if (!ed_cons2) {
+    /* LCOV_EXCL_START -- impossible if diff generation is correct */
+    log_warn(LD_BUG|LD_CONSDIFF, "Refusing to generate consensus diff because "
+        "the generated ed diff could not be tested to successfully generate "
+        "the target consensus.");
+    goto error_cleanup;
+    /* LCOV_EXCL_STOP */
+  }
+
+  int cons2_eq = 1;
+  if (smartlist_len(cons2) == smartlist_len(ed_cons2)) {
+    SMARTLIST_FOREACH_BEGIN(cons2, const cdline_t *, line1) {
+      const cdline_t *line2 = smartlist_get(ed_cons2, line1_sl_idx);
+      if (! lines_eq(line1, line2) ) {
+        cons2_eq = 0;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(line1);
+  } else {
+    cons2_eq = 0;
+  }
+  smartlist_free(ed_cons2);
+  if (!cons2_eq) {
+    /* LCOV_EXCL_START -- impossible if diff generation is correct. */
+    log_warn(LD_BUG|LD_CONSDIFF, "Refusing to generate consensus diff because "
+        "the generated ed diff did not generate the target consensus "
+        "successfully when tested.");
+    goto error_cleanup;
+    /* LCOV_EXCL_STOP */
+  }
+
+  char cons1_hash_hex[HEX_DIGEST256_LEN+1];
+  char cons2_hash_hex[HEX_DIGEST256_LEN+1];
+  base16_encode(cons1_hash_hex, HEX_DIGEST256_LEN+1,
+                (const char*)digests1->sha3_256, DIGEST256_LEN);
+  base16_encode(cons2_hash_hex, HEX_DIGEST256_LEN+1,
+                (const char*)digests2->sha3_256, DIGEST256_LEN);
+
+  /* Create the resulting consensus diff. */
+  char buf[160];
+  smartlist_t *result = smartlist_new();
+  tor_snprintf(buf, sizeof(buf), "%s", ns_diff_version);
+  smartlist_add_linecpy(result, area, buf);
+  tor_snprintf(buf, sizeof(buf), "%s %s %s", hash_token,
+      cons1_hash_hex, cons2_hash_hex);
+  smartlist_add_linecpy(result, area, buf);
+  smartlist_add_all(result, ed_diff);
+  smartlist_free(ed_diff);
+  return result;
+
+ error_cleanup:
+
+  if (ed_diff) {
+    /* LCOV_EXCL_START -- ed_diff is NULL except in unreachable cases above */
+    smartlist_free(ed_diff);
+    /* LCOV_EXCL_STOP */
+  }
+
+  return NULL;
+}
+
+/** Fetch the digest of the base consensus in the consensus diff, encoded in
+ * base16 as found in the diff itself. digest1_out and digest2_out must be of
+ * length DIGEST256_LEN or larger if not NULL.
+ */
+int
+consdiff_get_digests(const smartlist_t *diff,
+                     char *digest1_out,
+                     char *digest2_out)
+{
+  smartlist_t *hash_words = NULL;
+  const cdline_t *format;
+  char cons1_hash[DIGEST256_LEN], cons2_hash[DIGEST256_LEN];
+  char *cons1_hash_hex, *cons2_hash_hex;
+  if (smartlist_len(diff) < 2) {
+    log_info(LD_CONSDIFF, "The provided consensus diff is too short.");
+    goto error_cleanup;
+  }
+
+  /* Check that it's the format and version we know. */
+  format = smartlist_get(diff, 0);
+  if (!line_str_eq(format, ns_diff_version)) {
+    log_warn(LD_CONSDIFF, "The provided consensus diff format is not known.");
+    goto error_cleanup;
+  }
+
+  /* Grab the base16 digests. */
+  hash_words = smartlist_new();
+  {
+    const cdline_t *line2 = smartlist_get(diff, 1);
+    char *h = tor_memdup_nulterm(line2->s, line2->len);
+    smartlist_split_string(hash_words, h, " ", 0, 0);
+    tor_free(h);
+  }
+
+  /* There have to be three words, the first of which must be hash_token. */
+  if (smartlist_len(hash_words) != 3 ||
+      strcmp(smartlist_get(hash_words, 0), hash_token)) {
+    log_info(LD_CONSDIFF, "The provided consensus diff does not include "
+        "the necessary digests.");
+    goto error_cleanup;
+  }
+
+  /* Expected hashes as found in the consensus diff header. They must be of
+   * length HEX_DIGEST256_LEN, normally 64 hexadecimal characters.
+   * If any of the decodings fail, error to make sure that the hashes are
+   * proper base16-encoded digests.
+   */
+  cons1_hash_hex = smartlist_get(hash_words, 1);
+  cons2_hash_hex = smartlist_get(hash_words, 2);
+  if (strlen(cons1_hash_hex) != HEX_DIGEST256_LEN ||
+      strlen(cons2_hash_hex) != HEX_DIGEST256_LEN) {
+    log_info(LD_CONSDIFF, "The provided consensus diff includes "
+        "base16-encoded digests of incorrect size.");
+    goto error_cleanup;
+  }
+
+  if (base16_decode(cons1_hash, DIGEST256_LEN,
+          cons1_hash_hex, HEX_DIGEST256_LEN) != DIGEST256_LEN ||
+      base16_decode(cons2_hash, DIGEST256_LEN,
+          cons2_hash_hex, HEX_DIGEST256_LEN) != DIGEST256_LEN) {
+    log_info(LD_CONSDIFF, "The provided consensus diff includes "
+        "malformed digests.");
+    goto error_cleanup;
+  }
+
+  if (digest1_out) {
+    memcpy(digest1_out, cons1_hash, DIGEST256_LEN);
+  }
+  if (digest2_out) {
+    memcpy(digest2_out, cons2_hash, DIGEST256_LEN);
+  }
+
+  SMARTLIST_FOREACH(hash_words, char *, cp, tor_free(cp));
+  smartlist_free(hash_words);
+  return 0;
+
+ error_cleanup:
+
+  if (hash_words) {
+    SMARTLIST_FOREACH(hash_words, char *, cp, tor_free(cp));
+    smartlist_free(hash_words);
+  }
+  return 1;
+}
+
+/** Apply the consensus diff to the given consensus and return a new
+ * consensus, also as a line-based smartlist. Will return NULL if the diff
+ * could not be applied. Neither the consensus nor the diff are modified in
+ * any way, so it's up to the caller to free their resources.
+ */
+char *
+consdiff_apply_diff(const smartlist_t *cons1,
+                    const smartlist_t *diff,
+                    const consensus_digest_t *digests1)
+{
+  smartlist_t *cons2 = NULL;
+  char *cons2_str = NULL;
+  char e_cons1_hash[DIGEST256_LEN];
+  char e_cons2_hash[DIGEST256_LEN];
+
+  if (consdiff_get_digests(diff, e_cons1_hash, e_cons2_hash) != 0) {
+    goto error_cleanup;
+  }
+
+  /* See that the consensus that was given to us matches its hash. */
+  if (!consensus_digest_eq(digests1->sha3_256,
+                           (const uint8_t*)e_cons1_hash)) {
+    char hex_digest1[HEX_DIGEST256_LEN+1];
+    char e_hex_digest1[HEX_DIGEST256_LEN+1];
+    log_warn(LD_CONSDIFF, "Refusing to apply consensus diff because "
+        "the base consensus doesn't match the digest as found in "
+        "the consensus diff header.");
+    base16_encode(hex_digest1, HEX_DIGEST256_LEN+1,
+                  (const char *)digests1->sha3_256, DIGEST256_LEN);
+    base16_encode(e_hex_digest1, HEX_DIGEST256_LEN+1,
+                  e_cons1_hash, DIGEST256_LEN);
+    log_warn(LD_CONSDIFF, "Expected: %s; found: %s",
+             hex_digest1, e_hex_digest1);
+    goto error_cleanup;
+  }
+
+  /* Grab the ed diff and calculate the resulting consensus. */
+  /* Skip the first two lines. */
+  cons2 = apply_ed_diff(cons1, diff, 2);
+
+  /* ed diff could not be applied - reason already logged by apply_ed_diff. */
+  if (!cons2) {
+    goto error_cleanup;
+  }
+
+  cons2_str = consensus_join_lines(cons2);
+
+  consensus_digest_t cons2_digests;
+  if (consensus_compute_digest(cons2_str, &cons2_digests) < 0) {
+    /* LCOV_EXCL_START -- digest can't fail */
+    log_warn(LD_CONSDIFF, "Could not compute digests of the consensus "
+        "resulting from applying a consensus diff.");
+    goto error_cleanup;
+    /* LCOV_EXCL_STOP */
+  }
+
+  /* See that the resulting consensus matches its hash. */
+  if (!consensus_digest_eq(cons2_digests.sha3_256,
+                           (const uint8_t*)e_cons2_hash)) {
+    log_warn(LD_CONSDIFF, "Refusing to apply consensus diff because "
+        "the resulting consensus doesn't match the digest as found in "
+        "the consensus diff header.");
+    char hex_digest2[HEX_DIGEST256_LEN+1];
+    char e_hex_digest2[HEX_DIGEST256_LEN+1];
+    base16_encode(hex_digest2, HEX_DIGEST256_LEN+1,
+        (const char *)cons2_digests.sha3_256, DIGEST256_LEN);
+    base16_encode(e_hex_digest2, HEX_DIGEST256_LEN+1,
+        e_cons2_hash, DIGEST256_LEN);
+    log_warn(LD_CONSDIFF, "Expected: %s; found: %s",
+             hex_digest2, e_hex_digest2);
+    goto error_cleanup;
+  }
+
+  goto done;
+
+ error_cleanup:
+  tor_free(cons2_str); /* Sets it to NULL */
+
+ done:
+  if (cons2) {
+    smartlist_free(cons2);
+  }
+
+  return cons2_str;
+}
+
+/** Any consensus line longer than this means that the input is invalid. */
+#define CONSENSUS_LINE_MAX_LEN (1<<20)
+
+/**
+ * Helper: For every NL-terminated line in <b>s</b>, add a cdline referring to
+ * that line (without trailing newline) to <b>out</b>.  Return -1 if there are
+ * any non-NL terminated lines; 0 otherwise.
+ *
+ * Unlike tor_split_lines, this function avoids ambiguity on its
+ * handling of a final line that isn't NL-terminated.
+ *
+ * All cdline_t objects are allocated in the provided memarea.  Strings
+ * are not copied: if <b>s</b> changes or becomes invalid, then all
+ * generated cdlines will become invalid.
+ */
+STATIC int
+consensus_split_lines(smartlist_t *out, const char *s, memarea_t *area)
+{
+  const char *end_of_str = s + strlen(s);
+  tor_assert(*end_of_str == '\0');
+
+  while (*s) {
+    const char *eol = memchr(s, '\n', end_of_str - s);
+    if (!eol) {
+      /* File doesn't end with newline. */
+      return -1;
+    }
+    if (eol - s > CONSENSUS_LINE_MAX_LEN) {
+      /* Line is far too long. */
+      return -1;
+    }
+    cdline_t *line = memarea_alloc(area, sizeof(cdline_t));
+    line->s = s;
+    line->len = (uint32_t)(eol - s);
+    smartlist_add(out, line);
+    s = eol+1;
+  }
+  return 0;
+}
+
+/** Given a list of cdline_t, return a newly allocated string containing
+ * all of the lines, terminated with NL, concatenated.
+ *
+ * Unlike smartlist_join_strings(), avoids lossy operations on empty
+ * lists.  */
+static char *
+consensus_join_lines(const smartlist_t *inp)
+{
+  size_t n = 0;
+  SMARTLIST_FOREACH(inp, const cdline_t *, cdline, n += cdline->len + 1);
+  n += 1;
+  char *result = tor_malloc(n);
+  char *out = result;
+  SMARTLIST_FOREACH_BEGIN(inp, const cdline_t *, cdline) {
+    memcpy(out, cdline->s, cdline->len);
+    out += cdline->len;
+    *out++ = '\n';
+  } SMARTLIST_FOREACH_END(cdline);
+  *out++ = '\0';
+  tor_assert(out == result+n);
+  return result;
+}
+
+/** Given two consensus documents, try to compute a diff between them.  On
+ * success, retun a newly allocated string containing that diff.  On failure,
+ * return NULL. */
+char *
+consensus_diff_generate(const char *cons1,
+                        const char *cons2)
+{
+  consensus_digest_t d1, d2;
+  smartlist_t *lines1 = NULL, *lines2 = NULL, *result_lines = NULL;
+  int r1, r2;
+  char *result = NULL;
+
+  r1 = consensus_compute_digest_as_signed(cons1, &d1);
+  r2 = consensus_compute_digest(cons2, &d2);
+  if (BUG(r1 < 0 || r2 < 0))
+    return NULL; // LCOV_EXCL_LINE
+
+  memarea_t *area = memarea_new();
+  lines1 = smartlist_new();
+  lines2 = smartlist_new();
+  if (consensus_split_lines(lines1, cons1, area) < 0)
+    goto done;
+  if (consensus_split_lines(lines2, cons2, area) < 0)
+    goto done;
+
+  result_lines = consdiff_gen_diff(lines1, lines2, &d1, &d2, area);
+
+ done:
+  if (result_lines) {
+    result = consensus_join_lines(result_lines);
+    smartlist_free(result_lines);
+  }
+
+  memarea_drop_all(area);
+  smartlist_free(lines1);
+  smartlist_free(lines2);
+
+  return result;
+}
+
+/** Given a consensus document and a diff, try to apply the diff to the
+ * consensus.  On success return a newly allocated string containing the new
+ * consensus.  On failure, return NULL. */
+char *
+consensus_diff_apply(const char *consensus,
+                     const char *diff)
+{
+  consensus_digest_t d1;
+  smartlist_t *lines1 = NULL, *lines2 = NULL;
+  int r1;
+  char *result = NULL;
+  memarea_t *area = memarea_new();
+
+  r1 = consensus_compute_digest_as_signed(consensus, &d1);
+  if (BUG(r1 < 0))
+    goto done;
+
+  lines1 = smartlist_new();
+  lines2 = smartlist_new();
+  if (consensus_split_lines(lines1, consensus, area) < 0)
+    goto done;
+  if (consensus_split_lines(lines2, diff, area) < 0)
+    goto done;
+
+  result = consdiff_apply_diff(lines1, lines2, &d1);
+
+ done:
+  smartlist_free(lines1);
+  smartlist_free(lines2);
+  memarea_drop_all(area);
+
+  return result;
+}
+
+/** Return true iff, based on its header, <b>document</b> is likely
+ * to be a consensus diff. */
+int
+looks_like_a_consensus_diff(const char *document, size_t len)
+{
+  return (len >= strlen(ns_diff_version) &&
+          fast_memeq(document, ns_diff_version, strlen(ns_diff_version)));
+}
diff --git a/src/feature/dircommon/consdiff.h b/src/feature/dircommon/consdiff.h
new file mode 100644
index 0000000000..98217e6d46
--- /dev/null
+++ b/src/feature/dircommon/consdiff.h
@@ -0,0 +1,99 @@
+/* Copyright (c) 2014, Daniel Martí
+ * Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_CONSDIFF_H
+#define TOR_CONSDIFF_H
+
+#include "core/or/or.h"
+
+char *consensus_diff_generate(const char *cons1,
+                              const char *cons2);
+char *consensus_diff_apply(const char *consensus,
+                           const char *diff);
+
+int looks_like_a_consensus_diff(const char *document, size_t len);
+
+#ifdef CONSDIFF_PRIVATE
+#include "lib/container/bitarray.h"
+
+struct memarea_t;
+
+/** Line type used for constructing consensus diffs.  Each of these lines
+ * refers to a chunk of memory allocated elsewhere, and is not necessarily
+ * NUL-terminated: this helps us avoid copies and save memory. */
+typedef struct cdline_t {
+  const char *s;
+  uint32_t len;
+} cdline_t;
+
+typedef struct consensus_digest_t {
+  uint8_t sha3_256[DIGEST256_LEN];
+} consensus_digest_t;
+
+STATIC smartlist_t *consdiff_gen_diff(const smartlist_t *cons1,
+                                      const smartlist_t *cons2,
+                                      const consensus_digest_t *digests1,
+                                      const consensus_digest_t *digests2,
+                                      struct memarea_t *area);
+STATIC char *consdiff_apply_diff(const smartlist_t *cons1,
+                                 const smartlist_t *diff,
+                                 const consensus_digest_t *digests1);
+STATIC int consdiff_get_digests(const smartlist_t *diff,
+                                char *digest1_out,
+                                char *digest2_out);
+
+/** Data structure to define a slice of a smarltist. */
+typedef struct smartlist_slice_t {
+  /**
+   * Smartlist that this slice is made from.
+   * References the whole original smartlist that the slice was made out of.
+   * */
+  const smartlist_t *list;
+  /** Starting position of the slice in the smartlist. */
+  int offset;
+  /** Length of the slice, i.e. the number of elements it holds. */
+  int len;
+} smartlist_slice_t;
+STATIC smartlist_t *gen_ed_diff(const smartlist_t *cons1,
+                                const smartlist_t *cons2,
+                                struct memarea_t *area);
+STATIC smartlist_t *apply_ed_diff(const smartlist_t *cons1,
+                                  const smartlist_t *diff,
+                                  int start_line);
+STATIC void calc_changes(smartlist_slice_t *slice1, smartlist_slice_t *slice2,
+                         bitarray_t *changed1, bitarray_t *changed2);
+STATIC smartlist_slice_t *smartlist_slice(const smartlist_t *list,
+                                          int start, int end);
+STATIC int next_router(const smartlist_t *cons, int cur);
+STATIC int *lcs_lengths(const smartlist_slice_t *slice1,
+                        const smartlist_slice_t *slice2,
+                        int direction);
+STATIC void trim_slices(smartlist_slice_t *slice1, smartlist_slice_t *slice2);
+STATIC int base64cmp(const cdline_t *hash1, const cdline_t *hash2);
+STATIC int get_id_hash(const cdline_t *line, cdline_t *hash_out);
+STATIC int is_valid_router_entry(const cdline_t *line);
+STATIC int smartlist_slice_string_pos(const smartlist_slice_t *slice,
+                                      const cdline_t *string);
+STATIC void set_changed(bitarray_t *changed1, bitarray_t *changed2,
+                        const smartlist_slice_t *slice1,
+                        const smartlist_slice_t *slice2);
+STATIC int consensus_split_lines(smartlist_t *out, const char *s,
+                                 struct memarea_t *area);
+STATIC void smartlist_add_linecpy(smartlist_t *lst, struct memarea_t *area,
+                                  const char *s);
+STATIC int lines_eq(const cdline_t *a, const cdline_t *b);
+STATIC int line_str_eq(const cdline_t *a, const char *b);
+
+MOCK_DECL(STATIC int,
+          consensus_compute_digest,(const char *cons,
+                                    consensus_digest_t *digest_out));
+MOCK_DECL(STATIC int,
+          consensus_compute_digest_as_signed,(const char *cons,
+                                              consensus_digest_t *digest_out));
+MOCK_DECL(STATIC int,
+          consensus_digest_eq,(const uint8_t *d1,
+                               const uint8_t *d2));
+#endif /* defined(CONSDIFF_PRIVATE) */
+
+#endif /* !defined(TOR_CONSDIFF_H) */
diff --git a/src/feature/dircommon/dir_connection_st.h b/src/feature/dircommon/dir_connection_st.h
new file mode 100644
index 0000000000..8c59cc7a46
--- /dev/null
+++ b/src/feature/dircommon/dir_connection_st.h
@@ -0,0 +1,67 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef DIR_CONNECTION_ST_H
+#define DIR_CONNECTION_ST_H
+
+#include "core/or/connection_st.h"
+
+struct tor_compress_state_t;
+
+/** Subtype of connection_t for an "directory connection" -- that is, an HTTP
+ * connection to retrieve or serve directory material. */
+struct dir_connection_t {
+  connection_t base_;
+
+ /** Which 'resource' did we ask the directory for? This is typically the part
+  * of the URL string that defines, relative to the directory conn purpose,
+  * what thing we want.  For example, in router descriptor downloads by
+  * descriptor digest, it contains "d/", then one or more +-separated
+  * fingerprints.
+  **/
+  char *requested_resource;
+  unsigned int dirconn_direct:1; /**< Is this dirconn direct, or via Tor? */
+
+  /** If we're fetching descriptors, what router purpose shall we assign
+   * to them? */
+  uint8_t router_purpose;
+
+  /** List of spooled_resource_t for objects that we're spooling. We use
+   * it from back to front. */
+  smartlist_t *spool;
+  /** The compression object doing on-the-fly compression for spooled data. */
+  struct tor_compress_state_t *compress_state;
+
+  /** What rendezvous service are we querying for? */
+  rend_data_t *rend_data;
+
+  /* Hidden service connection identifier for dir connections: Used by HS
+     client-side code to fetch HS descriptors, and by the service-side code to
+     upload descriptors. */
+  struct hs_ident_dir_conn_t *hs_ident;
+
+  /** If this is a one-hop connection, tracks the state of the directory guard
+   * for this connection (if any). */
+  struct circuit_guard_state_t *guard_state;
+
+  char identity_digest[DIGEST_LEN]; /**< Hash of the public RSA key for
+                                     * the directory server's signing key. */
+
+  /** Unique ID for directory requests; this used to be in connection_t, but
+   * that's going away and being used on channels instead.  The dirserver still
+   * needs this for the incoming side, so it's moved here. */
+  uint64_t dirreq_id;
+
+#ifdef MEASUREMENTS_21206
+  /** Number of RELAY_DATA cells received. */
+  uint32_t data_cells_received;
+
+  /** Number of RELAY_DATA cells sent. */
+  uint32_t data_cells_sent;
+#endif /* defined(MEASUREMENTS_21206) */
+};
+
+#endif
diff --git a/src/feature/dircommon/directory.c b/src/feature/dircommon/directory.c
new file mode 100644
index 0000000000..9e6f72e9ac
--- /dev/null
+++ b/src/feature/dircommon/directory.c
@@ -0,0 +1,651 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "feature/dircache/dircache.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dircommon/fp_pair.h"
+#include "feature/stats/geoip_stats.h"
+#include "lib/compress/compress.h"
+
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+
+/**
+ * \file directory.c
+ * \brief Code to send and fetch information from directory authorities and
+ * caches via HTTP.
+ *
+ * Directory caches and authorities use dirserv.c to generate the results of a
+ * query and stream them to the connection; clients use routerparse.c to parse
+ * them.
+ *
+ * Every directory request has a dir_connection_t on the client side and on
+ * the server side.  In most cases, the dir_connection_t object is a linked
+ * connection, tunneled through an edge_connection_t so that it can be a
+ * stream on the Tor network.  The only non-tunneled connections are those
+ * that are used to upload material (descriptors and votes) to authorities.
+ * Among tunneled connections, some use one-hop circuits, and others use
+ * multi-hop circuits for anonymity.
+ *
+ * Directory requests are launched by calling
+ * directory_initiate_request(). This
+ * launch the connection, will construct an HTTP request with
+ * directory_send_command(), send the and wait for a response.  The client
+ * later handles the response with connection_dir_client_reached_eof(),
+ * which passes the information received to another part of Tor.
+ *
+ * On the server side, requests are read in directory_handle_command(),
+ * which dispatches first on the request type (GET or POST), and then on
+ * the URL requested. GET requests are processed with a table-based
+ * dispatcher in url_table[].  The process of handling larger GET requests
+ * is complicated because we need to avoid allocating a copy of all the
+ * data to be sent to the client in one huge buffer.  Instead, we spool the
+ * data into the buffer using logic in connection_dirserv_flushed_some() in
+ * dirserv.c.  (TODO: If we extended buf.c to have a zero-copy
+ * reference-based buffer type, we could remove most of that code, at the
+ * cost of a bit more reference counting.)
+ **/
+
+/* In-points to directory.c:
+ *
+ * - directory_post_to_dirservers(), called from
+ *   router_upload_dir_desc_to_dirservers() in router.c
+ *   upload_service_descriptor() in rendservice.c
+ * - directory_get_from_dirserver(), called from
+ *   rend_client_refetch_renddesc() in rendclient.c
+ *   run_scheduled_events() in main.c
+ *   do_hup() in main.c
+ * - connection_dir_process_inbuf(), called from
+ *   connection_process_inbuf() in connection.c
+ * - connection_dir_finished_flushing(), called from
+ *   connection_finished_flushing() in connection.c
+ * - connection_dir_finished_connecting(), called from
+ *   connection_finished_connecting() in connection.c
+ */
+
+/** Convert a connection_t* to a dir_connection_t*; assert if the cast is
+ * invalid. */
+dir_connection_t *
+TO_DIR_CONN(connection_t *c)
+{
+  tor_assert(c->magic == DIR_CONNECTION_MAGIC);
+  return DOWNCAST(dir_connection_t, c);
+}
+
+/** Return false if the directory purpose <b>dir_purpose</b>
+ * does not require an anonymous (three-hop) connection.
+ *
+ * Return true 1) by default, 2) if all directory actions have
+ * specifically been configured to be over an anonymous connection,
+ * or 3) if the router is a bridge */
+int
+purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose,
+                        const char *resource)
+{
+  if (get_options()->AllDirActionsPrivate)
+    return 1;
+
+  if (router_purpose == ROUTER_PURPOSE_BRIDGE) {
+    if (dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC
+        && resource && !strcmp(resource, "authority.z")) {
+      /* We are asking a bridge for its own descriptor. That doesn't need
+         anonymity. */
+      return 0;
+    }
+    /* Assume all other bridge stuff needs anonymity. */
+    return 1; /* if no circuits yet, this might break bootstrapping, but it's
+               * needed to be safe. */
+  }
+
+  switch (dir_purpose)
+  {
+    case DIR_PURPOSE_UPLOAD_DIR:
+    case DIR_PURPOSE_UPLOAD_VOTE:
+    case DIR_PURPOSE_UPLOAD_SIGNATURES:
+    case DIR_PURPOSE_FETCH_STATUS_VOTE:
+    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
+    case DIR_PURPOSE_FETCH_CONSENSUS:
+    case DIR_PURPOSE_FETCH_CERTIFICATE:
+    case DIR_PURPOSE_FETCH_SERVERDESC:
+    case DIR_PURPOSE_FETCH_EXTRAINFO:
+    case DIR_PURPOSE_FETCH_MICRODESC:
+      return 0;
+    case DIR_PURPOSE_HAS_FETCHED_HSDESC:
+    case DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2:
+    case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
+    case DIR_PURPOSE_FETCH_RENDDESC_V2:
+    case DIR_PURPOSE_FETCH_HSDESC:
+    case DIR_PURPOSE_UPLOAD_HSDESC:
+      return 1;
+    case DIR_PURPOSE_SERVER:
+    default:
+      log_warn(LD_BUG, "Called with dir_purpose=%d, router_purpose=%d",
+               dir_purpose, router_purpose);
+      tor_assert_nonfatal_unreached();
+      return 1; /* Assume it needs anonymity; better safe than sorry. */
+  }
+}
+
+/** Return a newly allocated string describing <b>auth</b>. Only describes
+ * authority features. */
+char *
+authdir_type_to_string(dirinfo_type_t auth)
+{
+  char *result;
+  smartlist_t *lst = smartlist_new();
+  if (auth & V3_DIRINFO)
+    smartlist_add(lst, (void*)"V3");
+  if (auth & BRIDGE_DIRINFO)
+    smartlist_add(lst, (void*)"Bridge");
+  if (smartlist_len(lst)) {
+    result = smartlist_join_strings(lst, ", ", 0, NULL);
+  } else {
+    result = tor_strdup("[Not an authority]");
+  }
+  smartlist_free(lst);
+  return result;
+}
+
+/** Return true iff anything we say on <b>conn</b> is being encrypted before
+ * we send it to the client/server. */
+int
+connection_dir_is_encrypted(const dir_connection_t *conn)
+{
+  /* Right now it's sufficient to see if conn is or has been linked, since
+   * the only thing it could be linked to is an edge connection on a
+   * circuit, and the only way it could have been unlinked is at the edge
+   * connection getting closed.
+   */
+  return TO_CONN(conn)->linked;
+}
+
+/** Parse an HTTP request line at the start of a headers string.  On failure,
+ * return -1.  On success, set *<b>command_out</b> to a copy of the HTTP
+ * command ("get", "post", etc), set *<b>url_out</b> to a copy of the URL, and
+ * return 0. */
+int
+parse_http_command(const char *headers, char **command_out, char **url_out)
+{
+  const char *command, *end_of_command;
+  char *s, *start, *tmp;
+
+  s = (char *)eat_whitespace_no_nl(headers);
+  if (!*s) return -1;
+  command = s;
+  s = (char *)find_whitespace(s); /* get past GET/POST */
+  if (!*s) return -1;
+  end_of_command = s;
+  s = (char *)eat_whitespace_no_nl(s);
+  if (!*s) return -1;
+  start = s; /* this is the URL, assuming it's valid */
+  s = (char *)find_whitespace(start);
+  if (!*s) return -1;
+
+  /* tolerate the http[s] proxy style of putting the hostname in the url */
+  if (s-start >= 4 && !strcmpstart(start,"http")) {
+    tmp = start + 4;
+    if (*tmp == 's')
+      tmp++;
+    if (s-tmp >= 3 && !strcmpstart(tmp,"://")) {
+      tmp = strchr(tmp+3, '/');
+      if (tmp && tmp < s) {
+        log_debug(LD_DIR,"Skipping over 'http[s]://hostname/' string");
+        start = tmp;
+      }
+    }
+  }
+
+  /* Check if the header is well formed (next sequence
+   * should be HTTP/1.X\r\n). Assumes we're supporting 1.0? */
+  {
+    unsigned minor_ver;
+    char ch;
+    char *e = (char *)eat_whitespace_no_nl(s);
+    if (2 != tor_sscanf(e, "HTTP/1.%u%c", &minor_ver, &ch)) {
+      return -1;
+    }
+    if (ch != '\r')
+      return -1;
+  }
+
+  *url_out = tor_memdup_nulterm(start, s-start);
+  *command_out = tor_memdup_nulterm(command, end_of_command - command);
+  return 0;
+}
+
+/** Return a copy of the first HTTP header in <b>headers</b> whose key is
+ * <b>which</b>.  The key should be given with a terminating colon and space;
+ * this function copies everything after, up to but not including the
+ * following \\r\\n. */
+char *
+http_get_header(const char *headers, const char *which)
+{
+  const char *cp = headers;
+  while (cp) {
+    if (!strcasecmpstart(cp, which)) {
+      char *eos;
+      cp += strlen(which);
+      if ((eos = strchr(cp,'\r')))
+        return tor_strndup(cp, eos-cp);
+      else
+        return tor_strdup(cp);
+    }
+    cp = strchr(cp, '\n');
+    if (cp)
+      ++cp;
+  }
+  return NULL;
+}
+/** Parse an HTTP response string <b>headers</b> of the form
+ * \verbatim
+ * "HTTP/1.\%d \%d\%s\r\n...".
+ * \endverbatim
+ *
+ * If it's well-formed, assign the status code to *<b>code</b> and
+ * return 0.  Otherwise, return -1.
+ *
+ * On success: If <b>date</b> is provided, set *date to the Date
+ * header in the http headers, or 0 if no such header is found.  If
+ * <b>compression</b> is provided, set *<b>compression</b> to the
+ * compression method given in the Content-Encoding header, or 0 if no
+ * such header is found, or -1 if the value of the header is not
+ * recognized.  If <b>reason</b> is provided, strdup the reason string
+ * into it.
+ */
+int
+parse_http_response(const char *headers, int *code, time_t *date,
+                    compress_method_t *compression, char **reason)
+{
+  unsigned n1, n2;
+  char datestr[RFC1123_TIME_LEN+1];
+  smartlist_t *parsed_headers;
+  tor_assert(headers);
+  tor_assert(code);
+
+  while (TOR_ISSPACE(*headers)) headers++; /* tolerate leading whitespace */
+
+  if (tor_sscanf(headers, "HTTP/1.%u %u", &n1, &n2) < 2 ||
+      (n1 != 0 && n1 != 1) ||
+      (n2 < 100 || n2 >= 600)) {
+    log_warn(LD_HTTP,"Failed to parse header %s",escaped(headers));
+    return -1;
+  }
+  *code = n2;
+
+  parsed_headers = smartlist_new();
+  smartlist_split_string(parsed_headers, headers, "\n",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
+  if (reason) {
+    smartlist_t *status_line_elements = smartlist_new();
+    tor_assert(smartlist_len(parsed_headers));
+    smartlist_split_string(status_line_elements,
+                           smartlist_get(parsed_headers, 0),
+                           " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3);
+    tor_assert(smartlist_len(status_line_elements) <= 3);
+    if (smartlist_len(status_line_elements) == 3) {
+      *reason = smartlist_get(status_line_elements, 2);
+      smartlist_set(status_line_elements, 2, NULL); /* Prevent free */
+    }
+    SMARTLIST_FOREACH(status_line_elements, char *, cp, tor_free(cp));
+    smartlist_free(status_line_elements);
+  }
+  if (date) {
+    *date = 0;
+    SMARTLIST_FOREACH(parsed_headers, const char *, s,
+      if (!strcmpstart(s, "Date: ")) {
+        strlcpy(datestr, s+6, sizeof(datestr));
+        /* This will do nothing on failure, so we don't need to check
+           the result.   We shouldn't warn, since there are many other valid
+           date formats besides the one we use. */
+        parse_rfc1123_time(datestr, date);
+        break;
+      });
+  }
+  if (compression) {
+    const char *enc = NULL;
+    SMARTLIST_FOREACH(parsed_headers, const char *, s,
+      if (!strcmpstart(s, "Content-Encoding: ")) {
+        enc = s+18; break;
+      });
+
+    if (enc == NULL)
+      *compression = NO_METHOD;
+    else {
+      *compression = compression_method_get_by_name(enc);
+
+      if (*compression == UNKNOWN_METHOD)
+        log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.",
+                 escaped(enc));
+    }
+  }
+  SMARTLIST_FOREACH(parsed_headers, char *, s, tor_free(s));
+  smartlist_free(parsed_headers);
+
+  return 0;
+}
+
+/** If any directory object is arriving, and it's over 10MB large, we're
+ * getting DoS'd.  (As of 0.1.2.x, raw directories are about 1MB, and we never
+ * ask for more than 96 router descriptors at a time.)
+ */
+#define MAX_DIRECTORY_OBJECT_SIZE (10*(1<<20))
+
+#define MAX_VOTE_DL_SIZE (MAX_DIRECTORY_OBJECT_SIZE * 5)
+
+/** Read handler for directory connections.  (That's connections <em>to</em>
+ * directory servers and connections <em>at</em> directory servers.)
+ */
+int
+connection_dir_process_inbuf(dir_connection_t *conn)
+{
+  size_t max_size;
+  tor_assert(conn);
+  tor_assert(conn->base_.type == CONN_TYPE_DIR);
+
+  /* Directory clients write, then read data until they receive EOF;
+   * directory servers read data until they get an HTTP command, then
+   * write their response (when it's finished flushing, they mark for
+   * close).
+   */
+
+  /* If we're on the dirserver side, look for a command. */
+  if (conn->base_.state == DIR_CONN_STATE_SERVER_COMMAND_WAIT) {
+    if (directory_handle_command(conn) < 0) {
+      connection_mark_for_close(TO_CONN(conn));
+      return -1;
+    }
+    return 0;
+  }
+
+  max_size =
+    (TO_CONN(conn)->purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) ?
+    MAX_VOTE_DL_SIZE : MAX_DIRECTORY_OBJECT_SIZE;
+
+  if (connection_get_inbuf_len(TO_CONN(conn)) > max_size) {
+    log_warn(LD_HTTP,
+             "Too much data received from directory connection (%s): "
+             "denial of service attempt, or you need to upgrade?",
+             conn->base_.address);
+    connection_mark_for_close(TO_CONN(conn));
+    return -1;
+  }
+
+  if (!conn->base_.inbuf_reached_eof)
+    log_debug(LD_HTTP,"Got data, not eof. Leaving on inbuf.");
+  return 0;
+}
+
+/** Called when we're about to finally unlink and free a directory connection:
+ * perform necessary accounting and cleanup */
+void
+connection_dir_about_to_close(dir_connection_t *dir_conn)
+{
+  connection_t *conn = TO_CONN(dir_conn);
+
+  if (conn->state < DIR_CONN_STATE_CLIENT_FINISHED) {
+    /* It's a directory connection and connecting or fetching
+     * failed: forget about this router, and maybe try again. */
+    connection_dir_client_request_failed(dir_conn);
+  }
+
+  connection_dir_client_refetch_hsdesc_if_needed(dir_conn);
+}
+
+/** Write handler for directory connections; called when all data has
+ * been flushed.  Close the connection or wait for a response as
+ * appropriate.
+ */
+int
+connection_dir_finished_flushing(dir_connection_t *conn)
+{
+  tor_assert(conn);
+  tor_assert(conn->base_.type == CONN_TYPE_DIR);
+
+  if (conn->base_.marked_for_close)
+    return 0;
+
+  /* Note that we have finished writing the directory response. For direct
+   * connections this means we're done; for tunneled connections it's only
+   * an intermediate step. */
+  if (conn->dirreq_id)
+    geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED,
+                              DIRREQ_FLUSHING_DIR_CONN_FINISHED);
+  else
+    geoip_change_dirreq_state(TO_CONN(conn)->global_identifier,
+                              DIRREQ_DIRECT,
+                              DIRREQ_FLUSHING_DIR_CONN_FINISHED);
+  switch (conn->base_.state) {
+    case DIR_CONN_STATE_CONNECTING:
+    case DIR_CONN_STATE_CLIENT_SENDING:
+      log_debug(LD_DIR,"client finished sending command.");
+      conn->base_.state = DIR_CONN_STATE_CLIENT_READING;
+      return 0;
+    case DIR_CONN_STATE_SERVER_WRITING:
+      if (conn->spool) {
+        log_warn(LD_BUG, "Emptied a dirserv buffer, but it's still spooling!");
+        connection_mark_for_close(TO_CONN(conn));
+      } else {
+        log_debug(LD_DIRSERV, "Finished writing server response. Closing.");
+        connection_mark_for_close(TO_CONN(conn));
+      }
+      return 0;
+    default:
+      log_warn(LD_BUG,"called in unexpected state %d.",
+               conn->base_.state);
+      tor_fragile_assert();
+      return -1;
+  }
+  return 0;
+}
+
+/** Connected handler for directory connections: begin sending data to the
+ * server, and return 0.
+ * Only used when connections don't immediately connect. */
+int
+connection_dir_finished_connecting(dir_connection_t *conn)
+{
+  tor_assert(conn);
+  tor_assert(conn->base_.type == CONN_TYPE_DIR);
+  tor_assert(conn->base_.state == DIR_CONN_STATE_CONNECTING);
+
+  log_debug(LD_HTTP,"Dir connection to router %s:%u established.",
+            conn->base_.address,conn->base_.port);
+
+  /* start flushing conn */
+  conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING;
+  return 0;
+}
+
+/** Helper.  Compare two fp_pair_t objects, and return negative, 0, or
+ * positive as appropriate. */
+static int
+compare_pairs_(const void **a, const void **b)
+{
+  const fp_pair_t *fp1 = *a, *fp2 = *b;
+  int r;
+  if ((r = fast_memcmp(fp1->first, fp2->first, DIGEST_LEN)))
+    return r;
+  else
+    return fast_memcmp(fp1->second, fp2->second, DIGEST_LEN);
+}
+
+/** Divide a string <b>res</b> of the form FP1-FP2+FP3-FP4...[.z], where each
+ * FP is a hex-encoded fingerprint, into a sequence of distinct sorted
+ * fp_pair_t. Skip malformed pairs. On success, return 0 and add those
+ * fp_pair_t into <b>pairs_out</b>.  On failure, return -1. */
+int
+dir_split_resource_into_fingerprint_pairs(const char *res,
+                                          smartlist_t *pairs_out)
+{
+  smartlist_t *pairs_tmp = smartlist_new();
+  smartlist_t *pairs_result = smartlist_new();
+
+  smartlist_split_string(pairs_tmp, res, "+", 0, 0);
+  if (smartlist_len(pairs_tmp)) {
+    char *last = smartlist_get(pairs_tmp,smartlist_len(pairs_tmp)-1);
+    size_t last_len = strlen(last);
+    if (last_len > 2 && !strcmp(last+last_len-2, ".z")) {
+      last[last_len-2] = '\0';
+    }
+  }
+  SMARTLIST_FOREACH_BEGIN(pairs_tmp, char *, cp) {
+    if (strlen(cp) != HEX_DIGEST_LEN*2+1) {
+      log_info(LD_DIR,
+             "Skipping digest pair %s with non-standard length.", escaped(cp));
+    } else if (cp[HEX_DIGEST_LEN] != '-') {
+      log_info(LD_DIR,
+             "Skipping digest pair %s with missing dash.", escaped(cp));
+    } else {
+      fp_pair_t pair;
+      if (base16_decode(pair.first, DIGEST_LEN,
+                        cp, HEX_DIGEST_LEN) != DIGEST_LEN ||
+          base16_decode(pair.second,DIGEST_LEN,
+                        cp+HEX_DIGEST_LEN+1, HEX_DIGEST_LEN) != DIGEST_LEN) {
+        log_info(LD_DIR, "Skipping non-decodable digest pair %s", escaped(cp));
+      } else {
+        smartlist_add(pairs_result, tor_memdup(&pair, sizeof(pair)));
+      }
+    }
+    tor_free(cp);
+  } SMARTLIST_FOREACH_END(cp);
+  smartlist_free(pairs_tmp);
+
+  /* Uniq-and-sort */
+  smartlist_sort(pairs_result, compare_pairs_);
+  smartlist_uniq(pairs_result, compare_pairs_, tor_free_);
+
+  smartlist_add_all(pairs_out, pairs_result);
+  smartlist_free(pairs_result);
+  return 0;
+}
+
+/** Given a directory <b>resource</b> request, containing zero
+ * or more strings separated by plus signs, followed optionally by ".z", store
+ * the strings, in order, into <b>fp_out</b>.  If <b>compressed_out</b> is
+ * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0.
+ *
+ * If (flags & DSR_HEX), then delete all elements that aren't hex digests, and
+ * decode the rest.  If (flags & DSR_BASE64), then use "-" rather than "+" as
+ * a separator, delete all the elements that aren't base64-encoded digests,
+ * and decode the rest.  If (flags & DSR_DIGEST256), these digests should be
+ * 256 bits long; else they should be 160.
+ *
+ * If (flags & DSR_SORT_UNIQ), then sort the list and remove all duplicates.
+ */
+int
+dir_split_resource_into_fingerprints(const char *resource,
+                                     smartlist_t *fp_out, int *compressed_out,
+                                     int flags)
+{
+  const int decode_hex = flags & DSR_HEX;
+  const int decode_base64 = flags & DSR_BASE64;
+  const int digests_are_256 = flags & DSR_DIGEST256;
+  const int sort_uniq = flags & DSR_SORT_UNIQ;
+
+  const int digest_len = digests_are_256 ? DIGEST256_LEN : DIGEST_LEN;
+  const int hex_digest_len = digests_are_256 ?
+    HEX_DIGEST256_LEN : HEX_DIGEST_LEN;
+  const int base64_digest_len = digests_are_256 ?
+    BASE64_DIGEST256_LEN : BASE64_DIGEST_LEN;
+  smartlist_t *fp_tmp = smartlist_new();
+
+  tor_assert(!(decode_hex && decode_base64));
+  tor_assert(fp_out);
+
+  smartlist_split_string(fp_tmp, resource, decode_base64?"-":"+", 0, 0);
+  if (compressed_out)
+    *compressed_out = 0;
+  if (smartlist_len(fp_tmp)) {
+    char *last = smartlist_get(fp_tmp,smartlist_len(fp_tmp)-1);
+    size_t last_len = strlen(last);
+    if (last_len > 2 && !strcmp(last+last_len-2, ".z")) {
+      last[last_len-2] = '\0';
+      if (compressed_out)
+        *compressed_out = 1;
+    }
+  }
+  if (decode_hex || decode_base64) {
+    const size_t encoded_len = decode_hex ? hex_digest_len : base64_digest_len;
+    int i;
+    char *cp, *d = NULL;
+    for (i = 0; i < smartlist_len(fp_tmp); ++i) {
+      cp = smartlist_get(fp_tmp, i);
+      if (strlen(cp) != encoded_len) {
+        log_info(LD_DIR,
+                 "Skipping digest %s with non-standard length.", escaped(cp));
+        smartlist_del_keeporder(fp_tmp, i--);
+        goto again;
+      }
+      d = tor_malloc_zero(digest_len);
+      if (decode_hex ?
+          (base16_decode(d, digest_len, cp, hex_digest_len) != digest_len) :
+          (base64_decode(d, digest_len, cp, base64_digest_len)
+                         != digest_len)) {
+          log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp));
+          smartlist_del_keeporder(fp_tmp, i--);
+          goto again;
+      }
+      smartlist_set(fp_tmp, i, d);
+      d = NULL;
+    again:
+      tor_free(cp);
+      tor_free(d);
+    }
+  }
+  if (sort_uniq) {
+    if (decode_hex || decode_base64) {
+      if (digests_are_256) {
+        smartlist_sort_digests256(fp_tmp);
+        smartlist_uniq_digests256(fp_tmp);
+      } else {
+        smartlist_sort_digests(fp_tmp);
+        smartlist_uniq_digests(fp_tmp);
+      }
+    } else {
+      smartlist_sort_strings(fp_tmp);
+      smartlist_uniq_strings(fp_tmp);
+    }
+  }
+  smartlist_add_all(fp_out, fp_tmp);
+  smartlist_free(fp_tmp);
+  return 0;
+}
+
+/** As dir_split_resource_into_fingerprints, but instead fills
+ * <b>spool_out</b> with a list of spoolable_resource_t for the resource
+ * identified through <b>source</b>. */
+int
+dir_split_resource_into_spoolable(const char *resource,
+                                  dir_spool_source_t source,
+                                  smartlist_t *spool_out,
+                                  int *compressed_out,
+                                  int flags)
+{
+  smartlist_t *fingerprints = smartlist_new();
+
+  tor_assert(flags & (DSR_HEX|DSR_BASE64));
+  const size_t digest_len =
+    (flags & DSR_DIGEST256) ? DIGEST256_LEN : DIGEST_LEN;
+
+  int r = dir_split_resource_into_fingerprints(resource, fingerprints,
+                                               compressed_out, flags);
+  /* This is not a very efficient implementation XXXX */
+  SMARTLIST_FOREACH_BEGIN(fingerprints, uint8_t *, digest) {
+    spooled_resource_t *spooled =
+      spooled_resource_new(source, digest, digest_len);
+    if (spooled)
+      smartlist_add(spool_out, spooled);
+    tor_free(digest);
+  } SMARTLIST_FOREACH_END(digest);
+
+  smartlist_free(fingerprints);
+  return r;
+}
diff --git a/src/feature/dircommon/directory.h b/src/feature/dircommon/directory.h
new file mode 100644
index 0000000000..ba3f8c1b0e
--- /dev/null
+++ b/src/feature/dircommon/directory.h
@@ -0,0 +1,129 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file directory.h
+ * \brief Header file for directory.c.
+ **/
+
+#ifndef TOR_DIRECTORY_H
+#define TOR_DIRECTORY_H
+
+dir_connection_t *TO_DIR_CONN(connection_t *c);
+
+#define DIR_CONN_STATE_MIN_ 1
+/** State for connection to directory server: waiting for connect(). */
+#define DIR_CONN_STATE_CONNECTING 1
+/** State for connection to directory server: sending HTTP request. */
+#define DIR_CONN_STATE_CLIENT_SENDING 2
+/** State for connection to directory server: reading HTTP response. */
+#define DIR_CONN_STATE_CLIENT_READING 3
+/** State for connection to directory server: happy and finished. */
+#define DIR_CONN_STATE_CLIENT_FINISHED 4
+/** State for connection at directory server: waiting for HTTP request. */
+#define DIR_CONN_STATE_SERVER_COMMAND_WAIT 5
+/** State for connection at directory server: sending HTTP response. */
+#define DIR_CONN_STATE_SERVER_WRITING 6
+#define DIR_CONN_STATE_MAX_ 6
+
+#define DIR_PURPOSE_MIN_ 4
+/** A connection to a directory server: set after a v2 rendezvous
+ * descriptor is downloaded. */
+#define DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 4
+/** A connection to a directory server: download one or more server
+ * descriptors. */
+#define DIR_PURPOSE_FETCH_SERVERDESC 6
+/** A connection to a directory server: download one or more extra-info
+ * documents. */
+#define DIR_PURPOSE_FETCH_EXTRAINFO 7
+/** A connection to a directory server: upload a server descriptor. */
+#define DIR_PURPOSE_UPLOAD_DIR 8
+/** A connection to a directory server: upload a v3 networkstatus vote. */
+#define DIR_PURPOSE_UPLOAD_VOTE 10
+/** A connection to a directory server: upload a v3 consensus signature */
+#define DIR_PURPOSE_UPLOAD_SIGNATURES 11
+/** A connection to a directory server: download one or more v3 networkstatus
+ * votes. */
+#define DIR_PURPOSE_FETCH_STATUS_VOTE 12
+/** A connection to a directory server: download a v3 detached signatures
+ * object for a consensus. */
+#define DIR_PURPOSE_FETCH_DETACHED_SIGNATURES 13
+/** A connection to a directory server: download a v3 networkstatus
+ * consensus. */
+#define DIR_PURPOSE_FETCH_CONSENSUS 14
+/** A connection to a directory server: download one or more directory
+ * authority certificates. */
+#define DIR_PURPOSE_FETCH_CERTIFICATE 15
+
+/** Purpose for connection at a directory server. */
+#define DIR_PURPOSE_SERVER 16
+/** A connection to a hidden service directory server: upload a v2 rendezvous
+ * descriptor. */
+#define DIR_PURPOSE_UPLOAD_RENDDESC_V2 17
+/** A connection to a hidden service directory server: download a v2 rendezvous
+ * descriptor. */
+#define DIR_PURPOSE_FETCH_RENDDESC_V2 18
+/** A connection to a directory server: download a microdescriptor. */
+#define DIR_PURPOSE_FETCH_MICRODESC 19
+/** A connection to a hidden service directory: upload a v3 descriptor. */
+#define DIR_PURPOSE_UPLOAD_HSDESC 20
+/** A connection to a hidden service directory: fetch a v3 descriptor. */
+#define DIR_PURPOSE_FETCH_HSDESC 21
+/** A connection to a directory server: set after a hidden service descriptor
+ * is downloaded. */
+#define DIR_PURPOSE_HAS_FETCHED_HSDESC 22
+#define DIR_PURPOSE_MAX_ 22
+
+/** True iff <b>p</b> is a purpose corresponding to uploading
+ * data to a directory server. */
+#define DIR_PURPOSE_IS_UPLOAD(p)                \
+  ((p)==DIR_PURPOSE_UPLOAD_DIR ||               \
+   (p)==DIR_PURPOSE_UPLOAD_VOTE ||              \
+   (p)==DIR_PURPOSE_UPLOAD_SIGNATURES ||        \
+   (p)==DIR_PURPOSE_UPLOAD_RENDDESC_V2 ||       \
+   (p)==DIR_PURPOSE_UPLOAD_HSDESC)
+
+enum compress_method_t;
+int parse_http_response(const char *headers, int *code, time_t *date,
+                        enum compress_method_t *compression, char **response);
+int parse_http_command(const char *headers,
+                       char **command_out, char **url_out);
+char *http_get_header(const char *headers, const char *which);
+
+int connection_dir_is_encrypted(const dir_connection_t *conn);
+int connection_dir_reached_eof(dir_connection_t *conn);
+int connection_dir_process_inbuf(dir_connection_t *conn);
+int connection_dir_finished_flushing(dir_connection_t *conn);
+int connection_dir_finished_connecting(dir_connection_t *conn);
+void connection_dir_about_to_close(dir_connection_t *dir_conn);
+
+#define DSR_HEX       (1<<0)
+#define DSR_BASE64    (1<<1)
+#define DSR_DIGEST256 (1<<2)
+#define DSR_SORT_UNIQ (1<<3)
+int dir_split_resource_into_fingerprints(const char *resource,
+                                     smartlist_t *fp_out, int *compressed_out,
+                                     int flags);
+enum dir_spool_source_t;
+int dir_split_resource_into_spoolable(const char *resource,
+                                      enum dir_spool_source_t source,
+                                      smartlist_t *spool_out,
+                                      int *compressed_out,
+                                      int flags);
+int dir_split_resource_into_fingerprint_pairs(const char *res,
+                                              smartlist_t *pairs_out);
+char *directory_dump_request_log(void);
+void note_request(const char *key, size_t bytes);
+
+int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose,
+                            const char *resource);
+
+char *authdir_type_to_string(dirinfo_type_t auth);
+
+#define X_ADDRESS_HEADER "X-Your-Address-Is: "
+#define X_OR_DIFF_FROM_CONSENSUS_HEADER "X-Or-Diff-From-Consensus: "
+
+#endif /* !defined(TOR_DIRECTORY_H) */
diff --git a/src/or/fp_pair.c b/src/feature/dircommon/fp_pair.c
index eeeb0f1de3..284600df77 100644
--- a/src/or/fp_pair.c
+++ b/src/feature/dircommon/fp_pair.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,8 +17,8 @@
  * certificate for any (ID key, signing key) pair.
  **/
 
-#include "or.h"
-#include "fp_pair.h"
+#include "core/or/or.h"
+#include "feature/dircommon/fp_pair.h"
 
 /* Define fp_pair_map_t structures */
 
@@ -196,7 +196,7 @@ fp_pair_map_remove(fp_pair_map_t *map, const fp_pair_t *key)
  */
 
 void
-fp_pair_map_free(fp_pair_map_t *map, void (*free_val)(void*))
+fp_pair_map_free_(fp_pair_map_t *map, void (*free_val)(void*))
 {
   fp_pair_map_entry_t **ent, **next, *this;
 
diff --git a/src/or/fp_pair.h b/src/feature/dircommon/fp_pair.h
index b1466581d2..5041583e88 100644
--- a/src/or/fp_pair.h
+++ b/src/feature/dircommon/fp_pair.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,6 +9,12 @@
 #ifndef _TOR_FP_PAIR_H
 #define _TOR_FP_PAIR_H
 
+/** A pair of digests created by dir_split_resource_info_fingerprint_pairs() */
+typedef struct {
+  char first[DIGEST_LEN];
+  char second[DIGEST_LEN];
+} fp_pair_t;
+
 /*
  * Declare fp_pair_map_t functions and structs
  */
@@ -26,7 +32,12 @@ void * fp_pair_map_get(const fp_pair_map_t *map, const fp_pair_t *key);
 void * fp_pair_map_get_by_digests(const fp_pair_map_t *map,
                                   const char *first, const char *second);
 void * fp_pair_map_remove(fp_pair_map_t *map, const fp_pair_t *key);
-void fp_pair_map_free(fp_pair_map_t *map, void (*free_val)(void*));
+void fp_pair_map_free_(fp_pair_map_t *map, void (*free_val)(void*));
+#define fp_pair_map_free(map, free_val) do {            \
+    fp_pair_map_free_((map), (free_val));               \
+    (map) = NULL;                                       \
+  } while (0)
+
 int fp_pair_map_isempty(const fp_pair_map_t *map);
 int fp_pair_map_size(const fp_pair_map_t *map);
 fp_pair_map_iter_t * fp_pair_map_iter_init(fp_pair_map_t *map);
@@ -41,5 +52,5 @@ void fp_pair_map_assert_ok(const fp_pair_map_t *map);
 
 #undef DECLARE_MAP_FNS
 
-#endif
+#endif /* !defined(_TOR_FP_PAIR_H) */
 
diff --git a/src/feature/dircommon/vote_timing_st.h b/src/feature/dircommon/vote_timing_st.h
new file mode 100644
index 0000000000..47b90ab009
--- /dev/null
+++ b/src/feature/dircommon/vote_timing_st.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef VOTE_TIMING_ST_H
+#define VOTE_TIMING_ST_H
+
+/** Describes the schedule by which votes should be generated. */
+struct vote_timing_t {
+  /** Length in seconds between one consensus becoming valid and the next
+   * becoming valid. */
+  int vote_interval;
+  /** For how many intervals is a consensus valid? */
+  int n_intervals_valid;
+  /** Time in seconds allowed to propagate votes */
+  int vote_delay;
+  /** Time in seconds allowed to propagate signatures */
+  int dist_delay;
+};
+
+#endif
+
diff --git a/src/feature/dircommon/voting_schedule.c b/src/feature/dircommon/voting_schedule.c
new file mode 100644
index 0000000000..0a7476eda7
--- /dev/null
+++ b/src/feature/dircommon/voting_schedule.c
@@ -0,0 +1,194 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file voting_schedule.c
+ * \brief This file contains functions that are from the directory authority
+ *        subsystem related to voting specifically but used by many part of
+ *        tor. The full feature is built as part of the dirauth module.
+ **/
+
+#define VOTING_SCHEDULE_PRIVATE
+#include "feature/dircommon/voting_schedule.h"
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "feature/nodelist/networkstatus.h"
+
+#include "feature/nodelist/networkstatus_st.h"
+
+/* =====
+ * Vote scheduling
+ * ===== */
+
+/** Return the start of the next interval of size <b>interval</b> (in
+ * seconds) after <b>now</b>, plus <b>offset</b>. Midnight always
+ * starts a fresh interval, and if the last interval of a day would be
+ * truncated to less than half its size, it is rolled into the
+ * previous interval. */
+time_t
+voting_schedule_get_start_of_next_interval(time_t now, int interval,
+                                           int offset)
+{
+  struct tm tm;
+  time_t midnight_today=0;
+  time_t midnight_tomorrow;
+  time_t next;
+
+  tor_gmtime_r(&now, &tm);
+  tm.tm_hour = 0;
+  tm.tm_min = 0;
+  tm.tm_sec = 0;
+
+  if (tor_timegm(&tm, &midnight_today) < 0) {
+    // LCOV_EXCL_START
+    log_warn(LD_BUG, "Ran into an invalid time when trying to find midnight.");
+    // LCOV_EXCL_STOP
+  }
+  midnight_tomorrow = midnight_today + (24*60*60);
+
+  next = midnight_today + ((now-midnight_today)/interval + 1)*interval;
+
+  /* Intervals never cross midnight. */
+  if (next > midnight_tomorrow)
+    next = midnight_tomorrow;
+
+  /* If the interval would only last half as long as it's supposed to, then
+   * skip over to the next day. */
+  if (next + interval/2 > midnight_tomorrow)
+    next = midnight_tomorrow;
+
+  next += offset;
+  if (next - interval > now)
+    next -= interval;
+
+  return next;
+}
+
+/* Populate and return a new voting_schedule_t that can be used to schedule
+ * voting. The object is allocated on the heap and it's the responsibility of
+ * the caller to free it. Can't fail. */
+static voting_schedule_t *
+get_voting_schedule(const or_options_t *options, time_t now, int severity)
+{
+  int interval, vote_delay, dist_delay;
+  time_t start;
+  time_t end;
+  networkstatus_t *consensus;
+  voting_schedule_t *new_voting_schedule;
+
+  new_voting_schedule = tor_malloc_zero(sizeof(voting_schedule_t));
+
+  consensus = networkstatus_get_live_consensus(now);
+
+  if (consensus) {
+    interval = (int)( consensus->fresh_until - consensus->valid_after );
+    vote_delay = consensus->vote_seconds;
+    dist_delay = consensus->dist_seconds;
+
+    /* Note down the consensus valid after, so that we detect outdated voting
+     * schedules in case of skewed clocks etc. */
+    new_voting_schedule->live_consensus_valid_after = consensus->valid_after;
+  } else {
+    interval = options->TestingV3AuthInitialVotingInterval;
+    vote_delay = options->TestingV3AuthInitialVoteDelay;
+    dist_delay = options->TestingV3AuthInitialDistDelay;
+  }
+
+  tor_assert(interval > 0);
+
+  if (vote_delay + dist_delay > interval/2)
+    vote_delay = dist_delay = interval / 4;
+
+  start = new_voting_schedule->interval_starts =
+    voting_schedule_get_start_of_next_interval(now,interval,
+                                      options->TestingV3AuthVotingStartOffset);
+  end = voting_schedule_get_start_of_next_interval(start+1, interval,
+                                      options->TestingV3AuthVotingStartOffset);
+
+  tor_assert(end > start);
+
+  new_voting_schedule->fetch_missing_signatures = start - (dist_delay/2);
+  new_voting_schedule->voting_ends = start - dist_delay;
+  new_voting_schedule->fetch_missing_votes =
+    start - dist_delay - (vote_delay/2);
+  new_voting_schedule->voting_starts = start - dist_delay - vote_delay;
+
+  {
+    char tbuf[ISO_TIME_LEN+1];
+    format_iso_time(tbuf, new_voting_schedule->interval_starts);
+    tor_log(severity, LD_DIR,"Choosing expected valid-after time as %s: "
+            "consensus_set=%d, interval=%d",
+            tbuf, consensus?1:0, interval);
+  }
+
+  return new_voting_schedule;
+}
+
+#define voting_schedule_free(s) \
+  FREE_AND_NULL(voting_schedule_t, voting_schedule_free_, (s))
+
+/** Frees a voting_schedule_t. This should be used instead of the generic
+ * tor_free. */
+static void
+voting_schedule_free_(voting_schedule_t *voting_schedule_to_free)
+{
+  if (!voting_schedule_to_free)
+    return;
+  tor_free(voting_schedule_to_free);
+}
+
+voting_schedule_t voting_schedule;
+
+/* Using the time <b>now</b>, return the next voting valid-after time. */
+time_t
+voting_schedule_get_next_valid_after_time(void)
+{
+  time_t now = approx_time();
+  bool need_to_recalculate_voting_schedule = false;
+
+  /* This is a safe guard in order to make sure that the voting schedule
+   * static object is at least initialized. Using this function with a zeroed
+   * voting schedule can lead to bugs. */
+  if (tor_mem_is_zero((const char *) &voting_schedule,
+                      sizeof(voting_schedule))) {
+    need_to_recalculate_voting_schedule = true;
+    goto done; /* no need for next check if we have to recalculate anyway */
+  }
+
+  /* Also make sure we are not using an outdated voting schedule. If we have a
+   * newer consensus, make sure we recalculate the voting schedule. */
+  const networkstatus_t *ns = networkstatus_get_live_consensus(now);
+  if (ns && ns->valid_after != voting_schedule.live_consensus_valid_after) {
+    log_info(LD_DIR, "Voting schedule is outdated: recalculating (%d/%d)",
+             (int) ns->valid_after,
+             (int) voting_schedule.live_consensus_valid_after);
+    need_to_recalculate_voting_schedule = true;
+  }
+
+ done:
+  if (need_to_recalculate_voting_schedule) {
+    voting_schedule_recalculate_timing(get_options(), approx_time());
+    voting_schedule.created_on_demand = 1;
+  }
+
+  return voting_schedule.interval_starts;
+}
+
+/** Set voting_schedule to hold the timing for the next vote we should be
+ * doing. All type of tor do that because HS subsystem needs the timing as
+ * well to function properly. */
+void
+voting_schedule_recalculate_timing(const or_options_t *options, time_t now)
+{
+  voting_schedule_t *new_voting_schedule;
+
+  /* get the new voting schedule */
+  new_voting_schedule = get_voting_schedule(options, now, LOG_INFO);
+  tor_assert(new_voting_schedule);
+
+  /* Fill in the global static struct now */
+  memcpy(&voting_schedule, new_voting_schedule, sizeof(voting_schedule));
+  voting_schedule_free(new_voting_schedule);
+}
+
diff --git a/src/feature/dircommon/voting_schedule.h b/src/feature/dircommon/voting_schedule.h
new file mode 100644
index 0000000000..bafd81184e
--- /dev/null
+++ b/src/feature/dircommon/voting_schedule.h
@@ -0,0 +1,65 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file voting_schedule.h
+ * \brief Header file for voting_schedule.c.
+ **/
+
+#ifndef TOR_VOTING_SCHEDULE_H
+#define TOR_VOTING_SCHEDULE_H
+
+#include "core/or/or.h"
+
+/** Scheduling information for a voting interval. */
+typedef struct {
+  /** When do we generate and distribute our vote for this interval? */
+  time_t voting_starts;
+  /** When do we send an HTTP request for any votes that we haven't
+   * been posted yet?*/
+  time_t fetch_missing_votes;
+  /** When do we give up on getting more votes and generate a consensus? */
+  time_t voting_ends;
+  /** When do we send an HTTP request for any signatures we're expecting to
+   * see on the consensus? */
+  time_t fetch_missing_signatures;
+  /** When do we publish the consensus? */
+  time_t interval_starts;
+
+  /* True iff we have generated and distributed our vote. */
+  int have_voted;
+  /* True iff we've requested missing votes. */
+  int have_fetched_missing_votes;
+  /* True iff we have built a consensus and sent the signatures around. */
+  int have_built_consensus;
+  /* True iff we've fetched missing signatures. */
+  int have_fetched_missing_signatures;
+  /* True iff we have published our consensus. */
+  int have_published_consensus;
+
+  /* True iff this voting schedule was set on demand meaning not through the
+   * normal vote operation of a dirauth or when a consensus is set. This only
+   * applies to a directory authority that needs to recalculate the voting
+   * timings only for the first vote even though this object was initilized
+   * prior to voting. */
+  int created_on_demand;
+
+  /** The valid-after time of the last live consensus that filled this voting
+   *  schedule.  It's used to detect outdated voting schedules. */
+  time_t live_consensus_valid_after;
+} voting_schedule_t;
+
+/* Public API. */
+
+extern voting_schedule_t voting_schedule;
+
+void voting_schedule_recalculate_timing(const or_options_t *options,
+                                        time_t now);
+
+time_t voting_schedule_get_start_of_next_interval(time_t now,
+                                                  int interval,
+                                                  int offset);
+time_t voting_schedule_get_next_valid_after_time(void);
+
+#endif /* TOR_VOTING_SCHEDULE_H */
+
diff --git a/src/feature/dirparse/authcert_members.i b/src/feature/dirparse/authcert_members.i
new file mode 100644
index 0000000000..08cffca97a
--- /dev/null
+++ b/src/feature/dirparse/authcert_members.i
@@ -0,0 +1,13 @@
+/*
+ * List of tokens common to V3 authority certificates and V3 consensuses.
+ */
+  T1("dir-key-certificate-version", K_DIR_KEY_CERTIFICATE_VERSION,
+                                                     GE(1),       NO_OBJ ),
+  T1("dir-identity-key", K_DIR_IDENTITY_KEY,         NO_ARGS,     NEED_KEY ),
+  T1("dir-key-published",K_DIR_KEY_PUBLISHED,        CONCAT_ARGS, NO_OBJ),
+  T1("dir-key-expires",  K_DIR_KEY_EXPIRES,          CONCAT_ARGS, NO_OBJ),
+  T1("dir-signing-key",  K_DIR_SIGNING_KEY,          NO_ARGS,     NEED_KEY ),
+  T1("dir-key-crosscert", K_DIR_KEY_CROSSCERT,       NO_ARGS,     NEED_OBJ ),
+  T1("dir-key-certification", K_DIR_KEY_CERTIFICATION,
+                                                     NO_ARGS,     NEED_OBJ),
+  T01("dir-address",     K_DIR_ADDRESS,              GE(1),       NO_OBJ),
diff --git a/src/feature/dirparse/authcert_parse.c b/src/feature/dirparse/authcert_parse.c
new file mode 100644
index 0000000000..1680bdbf30
--- /dev/null
+++ b/src/feature/dirparse/authcert_parse.c
@@ -0,0 +1,207 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "feature/dirparse/authcert_parse.h"
+#include "feature/dirparse/parsecommon.h"
+#include "feature/dirparse/sigcommon.h"
+#include "feature/dirparse/unparseable.h"
+#include "feature/nodelist/authcert.h"
+#include "lib/memarea/memarea.h"
+
+#include "feature/nodelist/authority_cert_st.h"
+
+/** List of tokens recognized in V3 authority certificates. */
+static token_rule_t dir_key_certificate_table[] = {
+#include "feature/dirparse/authcert_members.i"
+  T1("fingerprint",      K_FINGERPRINT,              CONCAT_ARGS, NO_OBJ ),
+  END_OF_TABLE
+};
+
+/** Parse a key certificate from <b>s</b>; point <b>end-of-string</b> to
+ * the first character after the certificate. */
+authority_cert_t *
+authority_cert_parse_from_string(const char *s, const char **end_of_string)
+{
+  /** Reject any certificate at least this big; it is probably an overflow, an
+   * attack, a bug, or some other nonsense. */
+#define MAX_CERT_SIZE (128*1024)
+
+  authority_cert_t *cert = NULL, *old_cert;
+  smartlist_t *tokens = NULL;
+  char digest[DIGEST_LEN];
+  directory_token_t *tok;
+  char fp_declared[DIGEST_LEN];
+  char *eos;
+  size_t len;
+  int found;
+  memarea_t *area = NULL;
+  const char *s_dup = s;
+
+  s = eat_whitespace(s);
+  eos = strstr(s, "\ndir-key-certification");
+  if (! eos) {
+    log_warn(LD_DIR, "No signature found on key certificate");
+    return NULL;
+  }
+  eos = strstr(eos, "\n-----END SIGNATURE-----\n");
+  if (! eos) {
+    log_warn(LD_DIR, "No end-of-signature found on key certificate");
+    return NULL;
+  }
+  eos = strchr(eos+2, '\n');
+  tor_assert(eos);
+  ++eos;
+  len = eos - s;
+
+  if (len > MAX_CERT_SIZE) {
+    log_warn(LD_DIR, "Certificate is far too big (at %lu bytes long); "
+             "rejecting", (unsigned long)len);
+    return NULL;
+  }
+
+  tokens = smartlist_new();
+  area = memarea_new();
+  if (tokenize_string(area,s, eos, tokens, dir_key_certificate_table, 0) < 0) {
+    log_warn(LD_DIR, "Error tokenizing key certificate");
+    goto err;
+  }
+  if (router_get_hash_impl(s, strlen(s), digest, "dir-key-certificate-version",
+                           "\ndir-key-certification", '\n', DIGEST_SHA1) < 0)
+    goto err;
+  tok = smartlist_get(tokens, 0);
+  if (tok->tp != K_DIR_KEY_CERTIFICATE_VERSION || strcmp(tok->args[0], "3")) {
+    log_warn(LD_DIR,
+             "Key certificate does not begin with a recognized version (3).");
+    goto err;
+  }
+
+  cert = tor_malloc_zero(sizeof(authority_cert_t));
+  memcpy(cert->cache_info.signed_descriptor_digest, digest, DIGEST_LEN);
+
+  tok = find_by_keyword(tokens, K_DIR_SIGNING_KEY);
+  tor_assert(tok->key);
+  cert->signing_key = tok->key;
+  tok->key = NULL;
+  if (crypto_pk_get_digest(cert->signing_key, cert->signing_key_digest))
+    goto err;
+
+  tok = find_by_keyword(tokens, K_DIR_IDENTITY_KEY);
+  tor_assert(tok->key);
+  cert->identity_key = tok->key;
+  tok->key = NULL;
+
+  tok = find_by_keyword(tokens, K_FINGERPRINT);
+  tor_assert(tok->n_args);
+  if (base16_decode(fp_declared, DIGEST_LEN, tok->args[0],
+                    strlen(tok->args[0])) != DIGEST_LEN) {
+    log_warn(LD_DIR, "Couldn't decode key certificate fingerprint %s",
+             escaped(tok->args[0]));
+    goto err;
+  }
+
+  if (crypto_pk_get_digest(cert->identity_key,
+                           cert->cache_info.identity_digest))
+    goto err;
+
+  if (tor_memneq(cert->cache_info.identity_digest, fp_declared, DIGEST_LEN)) {
+    log_warn(LD_DIR, "Digest of certificate key didn't match declared "
+             "fingerprint");
+    goto err;
+  }
+
+  tok = find_opt_by_keyword(tokens, K_DIR_ADDRESS);
+  if (tok) {
+    struct in_addr in;
+    char *address = NULL;
+    tor_assert(tok->n_args);
+    /* XXX++ use some tor_addr parse function below instead. -RD */
+    if (tor_addr_port_split(LOG_WARN, tok->args[0], &address,
+                            &cert->dir_port) < 0 ||
+        tor_inet_aton(address, &in) == 0) {
+      log_warn(LD_DIR, "Couldn't parse dir-address in certificate");
+      tor_free(address);
+      goto err;
+    }
+    cert->addr = ntohl(in.s_addr);
+    tor_free(address);
+  }
+
+  tok = find_by_keyword(tokens, K_DIR_KEY_PUBLISHED);
+  if (parse_iso_time(tok->args[0], &cert->cache_info.published_on) < 0) {
+     goto err;
+  }
+  tok = find_by_keyword(tokens, K_DIR_KEY_EXPIRES);
+  if (parse_iso_time(tok->args[0], &cert->expires) < 0) {
+     goto err;
+  }
+
+  tok = smartlist_get(tokens, smartlist_len(tokens)-1);
+  if (tok->tp != K_DIR_KEY_CERTIFICATION) {
+    log_warn(LD_DIR, "Certificate didn't end with dir-key-certification.");
+    goto err;
+  }
+
+  /* If we already have this cert, don't bother checking the signature. */
+  old_cert = authority_cert_get_by_digests(
+                                     cert->cache_info.identity_digest,
+                                     cert->signing_key_digest);
+  found = 0;
+  if (old_cert) {
+    /* XXXX We could just compare signed_descriptor_digest, but that wouldn't
+     * buy us much. */
+    if (old_cert->cache_info.signed_descriptor_len == len &&
+        old_cert->cache_info.signed_descriptor_body &&
+        tor_memeq(s, old_cert->cache_info.signed_descriptor_body, len)) {
+      log_debug(LD_DIR, "We already checked the signature on this "
+                "certificate; no need to do so again.");
+      found = 1;
+    }
+  }
+  if (!found) {
+    if (check_signature_token(digest, DIGEST_LEN, tok, cert->identity_key, 0,
+                              "key certificate")) {
+      goto err;
+    }
+
+    tok = find_by_keyword(tokens, K_DIR_KEY_CROSSCERT);
+    if (check_signature_token(cert->cache_info.identity_digest,
+                              DIGEST_LEN,
+                              tok,
+                              cert->signing_key,
+                              CST_NO_CHECK_OBJTYPE,
+                              "key cross-certification")) {
+      goto err;
+    }
+  }
+
+  cert->cache_info.signed_descriptor_len = len;
+  cert->cache_info.signed_descriptor_body = tor_malloc(len+1);
+  memcpy(cert->cache_info.signed_descriptor_body, s, len);
+  cert->cache_info.signed_descriptor_body[len] = 0;
+  cert->cache_info.saved_location = SAVED_NOWHERE;
+
+  if (end_of_string) {
+    *end_of_string = eat_whitespace(eos);
+  }
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_free(tokens);
+  if (area) {
+    DUMP_AREA(area, "authority cert");
+    memarea_drop_all(area);
+  }
+  return cert;
+ err:
+  dump_desc(s_dup, "authority cert");
+  authority_cert_free(cert);
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_free(tokens);
+  if (area) {
+    DUMP_AREA(area, "authority cert");
+    memarea_drop_all(area);
+  }
+  return NULL;
+}
diff --git a/src/feature/dirparse/authcert_parse.h b/src/feature/dirparse/authcert_parse.h
new file mode 100644
index 0000000000..ca475ad0e3
--- /dev/null
+++ b/src/feature/dirparse/authcert_parse.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file authcert_parse.h
+ * \brief Header file for authcert_parse.c.
+ **/
+
+#ifndef TOR_AUTHCERT_PARSE_H
+#define TOR_AUTHCERT_PARSE_H
+
+authority_cert_t *authority_cert_parse_from_string(const char *s,
+                                                   const char **end_of_string);
+
+#endif /* !defined(TOR_AUTHCERT_PARSE_H) */
diff --git a/src/feature/dirparse/microdesc_parse.c b/src/feature/dirparse/microdesc_parse.c
new file mode 100644
index 0000000000..5a75af3994
--- /dev/null
+++ b/src/feature/dirparse/microdesc_parse.c
@@ -0,0 +1,267 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file microdesc_parse.c
+ * \brief Code to parse and validate microdescriptors.
+ **/
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/or/policies.h"
+#include "feature/dirparse/microdesc_parse.h"
+#include "feature/dirparse/parsecommon.h"
+#include "feature/dirparse/routerparse.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/nickname.h"
+#include "feature/relay/router.h"
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/memarea/memarea.h"
+
+#include "feature/nodelist/microdesc_st.h"
+
+/** List of tokens recognized in microdescriptors */
+static token_rule_t microdesc_token_table[] = {
+  T1_START("onion-key",        K_ONION_KEY,        NO_ARGS,     NEED_KEY_1024),
+  T01("ntor-onion-key",        K_ONION_KEY_NTOR,   GE(1),       NO_OBJ ),
+  T0N("id",                    K_ID,               GE(2),       NO_OBJ ),
+  T0N("a",                     K_A,                GE(1),       NO_OBJ ),
+  T01("family",                K_FAMILY,           ARGS,        NO_OBJ ),
+  T01("p",                     K_P,                CONCAT_ARGS, NO_OBJ ),
+  T01("p6",                    K_P6,               CONCAT_ARGS, NO_OBJ ),
+  A01("@last-listed",          A_LAST_LISTED,      CONCAT_ARGS, NO_OBJ ),
+  END_OF_TABLE
+};
+
+/** Assuming that s starts with a microdesc, return the start of the
+ * *NEXT* one.  Return NULL on "not found." */
+static const char *
+find_start_of_next_microdesc(const char *s, const char *eos)
+{
+  int started_with_annotations;
+  s = eat_whitespace_eos(s, eos);
+  if (!s)
+    return NULL;
+
+#define CHECK_LENGTH() STMT_BEGIN \
+    if (eos - s < 32)             \
+      return NULL;                \
+  STMT_END
+
+#define NEXT_LINE() STMT_BEGIN            \
+    s = memchr(s, '\n', eos-s);           \
+    if (!s || eos - s <= 1)               \
+      return NULL;                        \
+    s++;                                  \
+  STMT_END
+
+  CHECK_LENGTH();
+
+  started_with_annotations = (*s == '@');
+
+  if (started_with_annotations) {
+    /* Start by advancing to the first non-annotation line. */
+    while (*s == '@')
+      NEXT_LINE();
+  }
+  CHECK_LENGTH();
+
+  /* Now we should be pointed at an onion-key line.  If we are, then skip
+   * it. */
+  if (!strcmpstart(s, "onion-key"))
+    NEXT_LINE();
+
+  /* Okay, now we're pointed at the first line of the microdescriptor which is
+     not an annotation or onion-key.  The next line that _is_ an annotation or
+     onion-key is the start of the next microdescriptor. */
+  while (eos - s > 32) {
+    if (*s == '@' || !strcmpstart(s, "onion-key"))
+      return s;
+    NEXT_LINE();
+  }
+  return NULL;
+
+#undef CHECK_LENGTH
+#undef NEXT_LINE
+}
+
+/** Parse as many microdescriptors as are found from the string starting at
+ * <b>s</b> and ending at <b>eos</b>.  If allow_annotations is set, read any
+ * annotations we recognize and ignore ones we don't.
+ *
+ * If <b>saved_location</b> isn't SAVED_IN_CACHE, make a local copy of each
+ * descriptor in the body field of each microdesc_t.
+ *
+ * Return all newly parsed microdescriptors in a newly allocated
+ * smartlist_t. If <b>invalid_disgests_out</b> is provided, add a SHA256
+ * microdesc digest to it for every microdesc that we found to be badly
+ * formed. (This may cause duplicates) */
+smartlist_t *
+microdescs_parse_from_string(const char *s, const char *eos,
+                             int allow_annotations,
+                             saved_location_t where,
+                             smartlist_t *invalid_digests_out)
+{
+  smartlist_t *tokens;
+  smartlist_t *result;
+  microdesc_t *md = NULL;
+  memarea_t *area;
+  const char *start = s;
+  const char *start_of_next_microdesc;
+  int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0;
+  const int copy_body = (where != SAVED_IN_CACHE);
+
+  directory_token_t *tok;
+
+  if (!eos)
+    eos = s + strlen(s);
+
+  s = eat_whitespace_eos(s, eos);
+  area = memarea_new();
+  result = smartlist_new();
+  tokens = smartlist_new();
+
+  while (s < eos) {
+    int okay = 0;
+
+    start_of_next_microdesc = find_start_of_next_microdesc(s, eos);
+    if (!start_of_next_microdesc)
+      start_of_next_microdesc = eos;
+
+    md = tor_malloc_zero(sizeof(microdesc_t));
+    {
+      const char *cp = tor_memstr(s, start_of_next_microdesc-s,
+                                  "onion-key");
+      const int no_onion_key = (cp == NULL);
+      if (no_onion_key) {
+        cp = s; /* So that we have *some* junk to put in the body */
+      }
+
+      md->bodylen = start_of_next_microdesc - cp;
+      md->saved_location = where;
+      if (copy_body)
+        md->body = tor_memdup_nulterm(cp, md->bodylen);
+      else
+        md->body = (char*)cp;
+      md->off = cp - start;
+      crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256);
+      if (no_onion_key) {
+        log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Malformed or truncated descriptor");
+        goto next;
+      }
+    }
+
+    if (tokenize_string(area, s, start_of_next_microdesc, tokens,
+                        microdesc_token_table, flags)) {
+      log_warn(LD_DIR, "Unparseable microdescriptor");
+      goto next;
+    }
+
+    if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) {
+      if (parse_iso_time(tok->args[0], &md->last_listed)) {
+        log_warn(LD_DIR, "Bad last-listed time in microdescriptor");
+        goto next;
+      }
+    }
+
+    tok = find_by_keyword(tokens, K_ONION_KEY);
+    if (!crypto_pk_public_exponent_ok(tok->key)) {
+      log_warn(LD_DIR,
+               "Relay's onion key had invalid exponent.");
+      goto next;
+    }
+    router_set_rsa_onion_pkey(tok->key, &md->onion_pkey,
+                              &md->onion_pkey_len);
+    crypto_pk_free(tok->key);
+
+    if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) {
+      curve25519_public_key_t k;
+      tor_assert(tok->n_args >= 1);
+      if (curve25519_public_from_base64(&k, tok->args[0]) < 0) {
+        log_warn(LD_DIR, "Bogus ntor-onion-key in microdesc");
+        goto next;
+      }
+      md->onion_curve25519_pkey =
+        tor_memdup(&k, sizeof(curve25519_public_key_t));
+    }
+
+    smartlist_t *id_lines = find_all_by_keyword(tokens, K_ID);
+    if (id_lines) {
+      SMARTLIST_FOREACH_BEGIN(id_lines, directory_token_t *, t) {
+        tor_assert(t->n_args >= 2);
+        if (!strcmp(t->args[0], "ed25519")) {
+          if (md->ed25519_identity_pkey) {
+            log_warn(LD_DIR, "Extra ed25519 key in microdesc");
+            smartlist_free(id_lines);
+            goto next;
+          }
+          ed25519_public_key_t k;
+          if (ed25519_public_from_base64(&k, t->args[1])<0) {
+            log_warn(LD_DIR, "Bogus ed25519 key in microdesc");
+            smartlist_free(id_lines);
+            goto next;
+          }
+          md->ed25519_identity_pkey = tor_memdup(&k, sizeof(k));
+        }
+      } SMARTLIST_FOREACH_END(t);
+      smartlist_free(id_lines);
+    }
+
+    {
+      smartlist_t *a_lines = find_all_by_keyword(tokens, K_A);
+      if (a_lines) {
+        find_single_ipv6_orport(a_lines, &md->ipv6_addr, &md->ipv6_orport);
+        smartlist_free(a_lines);
+      }
+    }
+
+    if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) {
+      int i;
+      md->family = smartlist_new();
+      for (i=0;i<tok->n_args;++i) {
+        if (!is_legal_nickname_or_hexdigest(tok->args[i])) {
+          log_warn(LD_DIR, "Illegal nickname %s in family line",
+                   escaped(tok->args[i]));
+          goto next;
+        }
+        smartlist_add_strdup(md->family, tok->args[i]);
+      }
+    }
+
+    if ((tok = find_opt_by_keyword(tokens, K_P))) {
+      md->exit_policy = parse_short_policy(tok->args[0]);
+    }
+    if ((tok = find_opt_by_keyword(tokens, K_P6))) {
+      md->ipv6_exit_policy = parse_short_policy(tok->args[0]);
+    }
+
+    smartlist_add(result, md);
+    okay = 1;
+
+    md = NULL;
+  next:
+    if (! okay && invalid_digests_out) {
+      smartlist_add(invalid_digests_out,
+                    tor_memdup(md->digest, DIGEST256_LEN));
+    }
+    microdesc_free(md);
+    md = NULL;
+
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    memarea_clear(area);
+    smartlist_clear(tokens);
+    s = start_of_next_microdesc;
+  }
+
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+
+  return result;
+}
diff --git a/src/feature/dirparse/microdesc_parse.h b/src/feature/dirparse/microdesc_parse.h
new file mode 100644
index 0000000000..23a90084b1
--- /dev/null
+++ b/src/feature/dirparse/microdesc_parse.h
@@ -0,0 +1,20 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file microdesc_parse.h
+ * \brief Header file for microdesc_parse.c.
+ **/
+
+#ifndef TOR_MICRODESC_PARSE_H
+#define TOR_MICRODESC_PARSE_H
+
+smartlist_t *microdescs_parse_from_string(const char *s, const char *eos,
+                                          int allow_annotations,
+                                          saved_location_t where,
+                                          smartlist_t *invalid_digests_out);
+
+#endif
diff --git a/src/feature/dirparse/ns_parse.c b/src/feature/dirparse/ns_parse.c
new file mode 100644
index 0000000000..109eebeb66
--- /dev/null
+++ b/src/feature/dirparse/ns_parse.c
@@ -0,0 +1,1685 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file routerparse.c
+ * \brief Code to parse and validate consensus documents and votes.
+ */
+
+#define NS_PARSE_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/or/versions.h"
+#include "feature/client/entrynodes.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirparse/authcert_parse.h"
+#include "feature/dirparse/ns_parse.h"
+#include "feature/dirparse/parsecommon.h"
+#include "feature/dirparse/routerparse.h"
+#include "feature/dirparse/sigcommon.h"
+#include "feature/dirparse/unparseable.h"
+#include "feature/hs_common/shared_random_client.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nickname.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/memarea/memarea.h"
+
+#include "feature/dirauth/vote_microdesc_hash_st.h"
+#include "feature/nodelist/authority_cert_st.h"
+#include "feature/nodelist/document_signature_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/networkstatus_voter_info_st.h"
+#include "feature/nodelist/vote_routerstatus_st.h"
+
+#undef log
+#include <math.h>
+
+/** List of tokens recognized in the body part of v3 networkstatus
+ * documents. */
+static token_rule_t rtrstatus_token_table[] = {
+  T01("p",                   K_P,               CONCAT_ARGS, NO_OBJ ),
+  T1( "r",                   K_R,                   GE(7),   NO_OBJ ),
+  T0N("a",                   K_A,                   GE(1),   NO_OBJ ),
+  T1( "s",                   K_S,                   ARGS,    NO_OBJ ),
+  T01("v",                   K_V,               CONCAT_ARGS, NO_OBJ ),
+  T01("w",                   K_W,                   ARGS,    NO_OBJ ),
+  T0N("m",                   K_M,               CONCAT_ARGS, NO_OBJ ),
+  T0N("id",                  K_ID,                  GE(2),   NO_OBJ ),
+  T01("pr",                  K_PROTO,           CONCAT_ARGS, NO_OBJ ),
+  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in V3 networkstatus votes. */
+static token_rule_t networkstatus_token_table[] = {
+  T1_START("network-status-version", K_NETWORK_STATUS_VERSION,
+                                                   GE(1),       NO_OBJ ),
+  T1("vote-status",            K_VOTE_STATUS,      GE(1),       NO_OBJ ),
+  T1("published",              K_PUBLISHED,        CONCAT_ARGS, NO_OBJ ),
+  T1("valid-after",            K_VALID_AFTER,      CONCAT_ARGS, NO_OBJ ),
+  T1("fresh-until",            K_FRESH_UNTIL,      CONCAT_ARGS, NO_OBJ ),
+  T1("valid-until",            K_VALID_UNTIL,      CONCAT_ARGS, NO_OBJ ),
+  T1("voting-delay",           K_VOTING_DELAY,     GE(2),       NO_OBJ ),
+  T1("known-flags",            K_KNOWN_FLAGS,      ARGS,        NO_OBJ ),
+  T01("params",                K_PARAMS,           ARGS,        NO_OBJ ),
+  T( "fingerprint",            K_FINGERPRINT,      CONCAT_ARGS, NO_OBJ ),
+  T01("signing-ed25519",       K_SIGNING_CERT_ED,  NO_ARGS ,    NEED_OBJ ),
+  T01("shared-rand-participate",K_SR_FLAG,         NO_ARGS,     NO_OBJ ),
+  T0N("shared-rand-commit",    K_COMMIT,           GE(3),       NO_OBJ ),
+  T01("shared-rand-previous-value", K_PREVIOUS_SRV,EQ(2),       NO_OBJ ),
+  T01("shared-rand-current-value",  K_CURRENT_SRV, EQ(2),       NO_OBJ ),
+  T0N("package",               K_PACKAGE,          CONCAT_ARGS, NO_OBJ ),
+  T01("recommended-client-protocols", K_RECOMMENDED_CLIENT_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+  T01("recommended-relay-protocols", K_RECOMMENDED_RELAY_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+  T01("required-client-protocols",    K_REQUIRED_CLIENT_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+  T01("required-relay-protocols",    K_REQUIRED_RELAY_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+
+#include "feature/dirparse/authcert_members.i"
+
+  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
+  T1( "contact",             K_CONTACT,         CONCAT_ARGS, NO_OBJ ),
+  T1( "dir-source",          K_DIR_SOURCE,      GE(6),       NO_OBJ ),
+  T01("legacy-dir-key",      K_LEGACY_DIR_KEY,  GE(1),       NO_OBJ ),
+  T1( "known-flags",         K_KNOWN_FLAGS,     CONCAT_ARGS, NO_OBJ ),
+  T01("client-versions",     K_CLIENT_VERSIONS, CONCAT_ARGS, NO_OBJ ),
+  T01("server-versions",     K_SERVER_VERSIONS, CONCAT_ARGS, NO_OBJ ),
+  T1( "consensus-methods",   K_CONSENSUS_METHODS, GE(1),     NO_OBJ ),
+
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in V3 networkstatus consensuses. */
+static token_rule_t networkstatus_consensus_token_table[] = {
+  T1_START("network-status-version", K_NETWORK_STATUS_VERSION,
+                                                   GE(1),       NO_OBJ ),
+  T1("vote-status",            K_VOTE_STATUS,      GE(1),       NO_OBJ ),
+  T1("valid-after",            K_VALID_AFTER,      CONCAT_ARGS, NO_OBJ ),
+  T1("fresh-until",            K_FRESH_UNTIL,      CONCAT_ARGS, NO_OBJ ),
+  T1("valid-until",            K_VALID_UNTIL,      CONCAT_ARGS, NO_OBJ ),
+  T1("voting-delay",           K_VOTING_DELAY,     GE(2),       NO_OBJ ),
+
+  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
+
+  T1N("dir-source",          K_DIR_SOURCE,          GE(6),   NO_OBJ ),
+  T1N("contact",             K_CONTACT,         CONCAT_ARGS, NO_OBJ ),
+  T1N("vote-digest",         K_VOTE_DIGEST,         GE(1),   NO_OBJ ),
+
+  T1( "known-flags",         K_KNOWN_FLAGS,     CONCAT_ARGS, NO_OBJ ),
+
+  T01("client-versions",     K_CLIENT_VERSIONS, CONCAT_ARGS, NO_OBJ ),
+  T01("server-versions",     K_SERVER_VERSIONS, CONCAT_ARGS, NO_OBJ ),
+  T01("consensus-method",    K_CONSENSUS_METHOD,    EQ(1),   NO_OBJ),
+  T01("params",                K_PARAMS,           ARGS,        NO_OBJ ),
+
+  T01("shared-rand-previous-value", K_PREVIOUS_SRV, EQ(2),   NO_OBJ ),
+  T01("shared-rand-current-value",  K_CURRENT_SRV,  EQ(2),   NO_OBJ ),
+
+  T01("recommended-client-protocols", K_RECOMMENDED_CLIENT_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+  T01("recommended-relay-protocols", K_RECOMMENDED_RELAY_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+  T01("required-client-protocols",    K_REQUIRED_CLIENT_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+  T01("required-relay-protocols",    K_REQUIRED_RELAY_PROTOCOLS,
+      CONCAT_ARGS, NO_OBJ ),
+
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in the footer of v1 directory footers. */
+static token_rule_t networkstatus_vote_footer_token_table[] = {
+  T01("directory-footer",    K_DIRECTORY_FOOTER,    NO_ARGS,   NO_OBJ ),
+  T01("bandwidth-weights",   K_BW_WEIGHTS,          ARGS,      NO_OBJ ),
+  T(  "directory-signature", K_DIRECTORY_SIGNATURE, GE(2),     NEED_OBJ ),
+  END_OF_TABLE
+};
+
+/** Try to find the start and end of the signed portion of a networkstatus
+ * document in <b>s</b>. On success, set <b>start_out</b> to the first
+ * character of the document, and <b>end_out</b> to a position one after the
+ * final character of the signed document, and return 0.  On failure, return
+ * -1. */
+int
+router_get_networkstatus_v3_signed_boundaries(const char *s,
+                                              const char **start_out,
+                                              const char **end_out)
+{
+  return router_get_hash_impl_helper(s, strlen(s),
+                                     "network-status-version",
+                                     "\ndirectory-signature",
+                                     ' ', LOG_INFO,
+                                     start_out, end_out);
+}
+
+/** Set <b>digest_out</b> to the SHA3-256 digest of the signed portion of the
+ * networkstatus vote in <b>s</b> -- or of the entirety of <b>s</b> if no
+ * signed portion can be identified.  Return 0 on success, -1 on failure. */
+int
+router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
+                                           const char *s)
+{
+  const char *start, *end;
+  if (router_get_networkstatus_v3_signed_boundaries(s, &start, &end) < 0) {
+    start = s;
+    end = s + strlen(s);
+  }
+  tor_assert(start);
+  tor_assert(end);
+  return crypto_digest256((char*)digest_out, start, end-start,
+                          DIGEST_SHA3_256);
+}
+
+/** Set <b>digests</b> to all the digests of the consensus document in
+ * <b>s</b> */
+int
+router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests)
+{
+  return router_get_hashes_impl(s,strlen(s),digests,
+                                "network-status-version",
+                                "\ndirectory-signature",
+                                ' ');
+}
+
+/** Helper: given a string <b>s</b>, return the start of the next router-status
+ * object (starting with "r " at the start of a line).  If none is found,
+ * return the start of the directory footer, or the next directory signature.
+ * If none is found, return the end of the string. */
+static inline const char *
+find_start_of_next_routerstatus(const char *s)
+{
+  const char *eos, *footer, *sig;
+  if ((eos = strstr(s, "\nr ")))
+    ++eos;
+  else
+    eos = s + strlen(s);
+
+  footer = tor_memstr(s, eos-s, "\ndirectory-footer");
+  sig = tor_memstr(s, eos-s, "\ndirectory-signature");
+
+  if (footer && sig)
+    return MIN(footer, sig) + 1;
+  else if (footer)
+    return footer+1;
+  else if (sig)
+    return sig+1;
+  else
+    return eos;
+}
+
+/** Parse the GuardFraction string from a consensus or vote.
+ *
+ *  If <b>vote</b> or <b>vote_rs</b> are set the document getting
+ *  parsed is a vote routerstatus. Otherwise it's a consensus. This is
+ *  the same semantic as in routerstatus_parse_entry_from_string(). */
+STATIC int
+routerstatus_parse_guardfraction(const char *guardfraction_str,
+                                 networkstatus_t *vote,
+                                 vote_routerstatus_t *vote_rs,
+                                 routerstatus_t *rs)
+{
+  int ok;
+  const char *end_of_header = NULL;
+  int is_consensus = !vote_rs;
+  uint32_t guardfraction;
+
+  tor_assert(bool_eq(vote, vote_rs));
+
+  /* If this info comes from a consensus, but we should't apply
+     guardfraction, just exit. */
+  if (is_consensus && !should_apply_guardfraction(NULL)) {
+    return 0;
+  }
+
+  end_of_header = strchr(guardfraction_str, '=');
+  if (!end_of_header) {
+    return -1;
+  }
+
+  guardfraction = (uint32_t)tor_parse_ulong(end_of_header+1,
+                                            10, 0, 100, &ok, NULL);
+  if (!ok) {
+    log_warn(LD_DIR, "Invalid GuardFraction %s", escaped(guardfraction_str));
+    return -1;
+  }
+
+  log_debug(LD_GENERAL, "[*] Parsed %s guardfraction '%s' for '%s'.",
+            is_consensus ? "consensus" : "vote",
+            guardfraction_str, rs->nickname);
+
+  if (!is_consensus) { /* We are parsing a vote */
+    vote_rs->status.guardfraction_percentage = guardfraction;
+    vote_rs->status.has_guardfraction = 1;
+  } else {
+    /* We are parsing a consensus. Only apply guardfraction to guards. */
+    if (rs->is_possible_guard) {
+      rs->guardfraction_percentage = guardfraction;
+      rs->has_guardfraction = 1;
+    } else {
+      log_warn(LD_BUG, "Got GuardFraction for non-guard %s. "
+               "This is not supposed to happen. Not applying. ", rs->nickname);
+    }
+  }
+
+  return 0;
+}
+
+/** Given a string at *<b>s</b>, containing a routerstatus object, and an
+ * empty smartlist at <b>tokens</b>, parse and return the first router status
+ * object in the string, and advance *<b>s</b> to just after the end of the
+ * router status.  Return NULL and advance *<b>s</b> on error.
+ *
+ * If <b>vote</b> and <b>vote_rs</b> are provided, don't allocate a fresh
+ * routerstatus but use <b>vote_rs</b> instead.
+ *
+ * If <b>consensus_method</b> is nonzero, this routerstatus is part of a
+ * consensus, and we should parse it according to the method used to
+ * make that consensus.
+ *
+ * Parse according to the syntax used by the consensus flavor <b>flav</b>.
+ **/
+STATIC routerstatus_t *
+routerstatus_parse_entry_from_string(memarea_t *area,
+                                     const char **s, smartlist_t *tokens,
+                                     networkstatus_t *vote,
+                                     vote_routerstatus_t *vote_rs,
+                                     int consensus_method,
+                                     consensus_flavor_t flav)
+{
+  const char *eos, *s_dup = *s;
+  routerstatus_t *rs = NULL;
+  directory_token_t *tok;
+  char timebuf[ISO_TIME_LEN+1];
+  struct in_addr in;
+  int offset = 0;
+  tor_assert(tokens);
+  tor_assert(bool_eq(vote, vote_rs));
+
+  if (!consensus_method)
+    flav = FLAV_NS;
+  tor_assert(flav == FLAV_NS || flav == FLAV_MICRODESC);
+
+  eos = find_start_of_next_routerstatus(*s);
+
+  if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) {
+    log_warn(LD_DIR, "Error tokenizing router status");
+    goto err;
+  }
+  if (smartlist_len(tokens) < 1) {
+    log_warn(LD_DIR, "Impossibly short router status");
+    goto err;
+  }
+  tok = find_by_keyword(tokens, K_R);
+  tor_assert(tok->n_args >= 7); /* guaranteed by GE(7) in K_R setup */
+  if (flav == FLAV_NS) {
+    if (tok->n_args < 8) {
+      log_warn(LD_DIR, "Too few arguments to r");
+      goto err;
+    }
+  } else if (flav == FLAV_MICRODESC) {
+    offset = -1; /* There is no descriptor digest in an md consensus r line */
+  }
+
+  if (vote_rs) {
+    rs = &vote_rs->status;
+  } else {
+    rs = tor_malloc_zero(sizeof(routerstatus_t));
+  }
+
+  if (!is_legal_nickname(tok->args[0])) {
+    log_warn(LD_DIR,
+             "Invalid nickname %s in router status; skipping.",
+             escaped(tok->args[0]));
+    goto err;
+  }
+  strlcpy(rs->nickname, tok->args[0], sizeof(rs->nickname));
+
+  if (digest_from_base64(rs->identity_digest, tok->args[1])) {
+    log_warn(LD_DIR, "Error decoding identity digest %s",
+             escaped(tok->args[1]));
+    goto err;
+  }
+
+  if (flav == FLAV_NS) {
+    if (digest_from_base64(rs->descriptor_digest, tok->args[2])) {
+      log_warn(LD_DIR, "Error decoding descriptor digest %s",
+               escaped(tok->args[2]));
+      goto err;
+    }
+  }
+
+  if (tor_snprintf(timebuf, sizeof(timebuf), "%s %s",
+                   tok->args[3+offset], tok->args[4+offset]) < 0 ||
+      parse_iso_time(timebuf, &rs->published_on)<0) {
+    log_warn(LD_DIR, "Error parsing time '%s %s' [%d %d]",
+             tok->args[3+offset], tok->args[4+offset],
+             offset, (int)flav);
+    goto err;
+  }
+
+  if (tor_inet_aton(tok->args[5+offset], &in) == 0) {
+    log_warn(LD_DIR, "Error parsing router address in network-status %s",
+             escaped(tok->args[5+offset]));
+    goto err;
+  }
+  rs->addr = ntohl(in.s_addr);
+
+  rs->or_port = (uint16_t) tor_parse_long(tok->args[6+offset],
+                                         10,0,65535,NULL,NULL);
+  rs->dir_port = (uint16_t) tor_parse_long(tok->args[7+offset],
+                                           10,0,65535,NULL,NULL);
+
+  {
+    smartlist_t *a_lines = find_all_by_keyword(tokens, K_A);
+    if (a_lines) {
+      find_single_ipv6_orport(a_lines, &rs->ipv6_addr, &rs->ipv6_orport);
+      smartlist_free(a_lines);
+    }
+  }
+
+  tok = find_opt_by_keyword(tokens, K_S);
+  if (tok && vote) {
+    int i;
+    vote_rs->flags = 0;
+    for (i=0; i < tok->n_args; ++i) {
+      int p = smartlist_string_pos(vote->known_flags, tok->args[i]);
+      if (p >= 0) {
+        vote_rs->flags |= (UINT64_C(1)<<p);
+      } else {
+        log_warn(LD_DIR, "Flags line had a flag %s not listed in known_flags.",
+                 escaped(tok->args[i]));
+        goto err;
+      }
+    }
+  } else if (tok) {
+    /* This is a consensus, not a vote. */
+    int i;
+    for (i=0; i < tok->n_args; ++i) {
+      if (!strcmp(tok->args[i], "Exit"))
+        rs->is_exit = 1;
+      else if (!strcmp(tok->args[i], "Stable"))
+        rs->is_stable = 1;
+      else if (!strcmp(tok->args[i], "Fast"))
+        rs->is_fast = 1;
+      else if (!strcmp(tok->args[i], "Running"))
+        rs->is_flagged_running = 1;
+      else if (!strcmp(tok->args[i], "Named"))
+        rs->is_named = 1;
+      else if (!strcmp(tok->args[i], "Valid"))
+        rs->is_valid = 1;
+      else if (!strcmp(tok->args[i], "Guard"))
+        rs->is_possible_guard = 1;
+      else if (!strcmp(tok->args[i], "BadExit"))
+        rs->is_bad_exit = 1;
+      else if (!strcmp(tok->args[i], "Authority"))
+        rs->is_authority = 1;
+      else if (!strcmp(tok->args[i], "Unnamed") &&
+               consensus_method >= 2) {
+        /* Unnamed is computed right by consensus method 2 and later. */
+        rs->is_unnamed = 1;
+      } else if (!strcmp(tok->args[i], "HSDir")) {
+        rs->is_hs_dir = 1;
+      } else if (!strcmp(tok->args[i], "V2Dir")) {
+        rs->is_v2_dir = 1;
+      }
+    }
+    /* These are implied true by having been included in a consensus made
+     * with a given method */
+    rs->is_flagged_running = 1; /* Starting with consensus method 4. */
+    rs->is_valid = 1; /* Starting with consensus method 24. */
+  }
+  {
+    const char *protocols = NULL, *version = NULL;
+    if ((tok = find_opt_by_keyword(tokens, K_PROTO))) {
+      tor_assert(tok->n_args == 1);
+      protocols = tok->args[0];
+    }
+    if ((tok = find_opt_by_keyword(tokens, K_V))) {
+      tor_assert(tok->n_args == 1);
+      version = tok->args[0];
+      if (vote_rs) {
+        vote_rs->version = tor_strdup(tok->args[0]);
+      }
+    }
+
+    summarize_protover_flags(&rs->pv, protocols, version);
+  }
+
+  /* handle weighting/bandwidth info */
+  if ((tok = find_opt_by_keyword(tokens, K_W))) {
+    int i;
+    for (i=0; i < tok->n_args; ++i) {
+      if (!strcmpstart(tok->args[i], "Bandwidth=")) {
+        int ok;
+        rs->bandwidth_kb =
+          (uint32_t)tor_parse_ulong(strchr(tok->args[i], '=')+1,
+                                    10, 0, UINT32_MAX,
+                                    &ok, NULL);
+        if (!ok) {
+          log_warn(LD_DIR, "Invalid Bandwidth %s", escaped(tok->args[i]));
+          goto err;
+        }
+        rs->has_bandwidth = 1;
+      } else if (!strcmpstart(tok->args[i], "Measured=") && vote_rs) {
+        int ok;
+        vote_rs->measured_bw_kb =
+            (uint32_t)tor_parse_ulong(strchr(tok->args[i], '=')+1,
+                                      10, 0, UINT32_MAX, &ok, NULL);
+        if (!ok) {
+          log_warn(LD_DIR, "Invalid Measured Bandwidth %s",
+                   escaped(tok->args[i]));
+          goto err;
+        }
+        vote_rs->has_measured_bw = 1;
+        vote->has_measured_bws = 1;
+      } else if (!strcmpstart(tok->args[i], "Unmeasured=1")) {
+        rs->bw_is_unmeasured = 1;
+      } else if (!strcmpstart(tok->args[i], "GuardFraction=")) {
+        if (routerstatus_parse_guardfraction(tok->args[i],
+                                             vote, vote_rs, rs) < 0) {
+          goto err;
+        }
+      }
+    }
+  }
+
+  /* parse exit policy summaries */
+  if ((tok = find_opt_by_keyword(tokens, K_P))) {
+    tor_assert(tok->n_args == 1);
+    if (strcmpstart(tok->args[0], "accept ") &&
+        strcmpstart(tok->args[0], "reject ")) {
+      log_warn(LD_DIR, "Unknown exit policy summary type %s.",
+               escaped(tok->args[0]));
+      goto err;
+    }
+    /* XXX weasel: parse this into ports and represent them somehow smart,
+     * maybe not here but somewhere on if we need it for the client.
+     * we should still parse it here to check it's valid tho.
+     */
+    rs->exitsummary = tor_strdup(tok->args[0]);
+    rs->has_exitsummary = 1;
+  }
+
+  if (vote_rs) {
+    SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, t) {
+      if (t->tp == K_M && t->n_args) {
+        vote_microdesc_hash_t *line =
+          tor_malloc(sizeof(vote_microdesc_hash_t));
+        line->next = vote_rs->microdesc;
+        line->microdesc_hash_line = tor_strdup(t->args[0]);
+        vote_rs->microdesc = line;
+      }
+      if (t->tp == K_ID) {
+        tor_assert(t->n_args >= 2);
+        if (!strcmp(t->args[0], "ed25519")) {
+          vote_rs->has_ed25519_listing = 1;
+          if (strcmp(t->args[1], "none") &&
+              digest256_from_base64((char*)vote_rs->ed25519_id,
+                                    t->args[1])<0) {
+            log_warn(LD_DIR, "Bogus ed25519 key in networkstatus vote");
+            goto err;
+          }
+        }
+      }
+      if (t->tp == K_PROTO) {
+        tor_assert(t->n_args == 1);
+        vote_rs->protocols = tor_strdup(t->args[0]);
+      }
+    } SMARTLIST_FOREACH_END(t);
+  } else if (flav == FLAV_MICRODESC) {
+    tok = find_opt_by_keyword(tokens, K_M);
+    if (tok) {
+      tor_assert(tok->n_args);
+      if (digest256_from_base64(rs->descriptor_digest, tok->args[0])) {
+        log_warn(LD_DIR, "Error decoding microdescriptor digest %s",
+                 escaped(tok->args[0]));
+        goto err;
+      }
+    } else {
+      log_info(LD_BUG, "Found an entry in networkstatus with no "
+               "microdescriptor digest. (Router %s ($%s) at %s:%d.)",
+               rs->nickname, hex_str(rs->identity_digest, DIGEST_LEN),
+               fmt_addr32(rs->addr), rs->or_port);
+    }
+  }
+
+  if (!strcasecmp(rs->nickname, UNNAMED_ROUTER_NICKNAME))
+    rs->is_named = 0;
+
+  goto done;
+ err:
+  dump_desc(s_dup, "routerstatus entry");
+  if (rs && !vote_rs)
+    routerstatus_free(rs);
+  rs = NULL;
+ done:
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_clear(tokens);
+  if (area) {
+    DUMP_AREA(area, "routerstatus entry");
+    memarea_clear(area);
+  }
+  *s = eos;
+
+  return rs;
+}
+
+int
+compare_vote_routerstatus_entries(const void **_a, const void **_b)
+{
+  const vote_routerstatus_t *a = *_a, *b = *_b;
+  return fast_memcmp(a->status.identity_digest, b->status.identity_digest,
+                     DIGEST_LEN);
+}
+
+/** Verify the bandwidth weights of a network status document */
+int
+networkstatus_verify_bw_weights(networkstatus_t *ns, int consensus_method)
+{
+  int64_t G=0, M=0, E=0, D=0, T=0;
+  double Wgg, Wgm, Wgd, Wmg, Wmm, Wme, Wmd, Weg, Wem, Wee, Wed;
+  double Gtotal=0, Mtotal=0, Etotal=0;
+  const char *casename = NULL;
+  int valid = 1;
+  (void) consensus_method;
+
+  const int64_t weight_scale = networkstatus_get_weight_scale_param(ns);
+  tor_assert(weight_scale >= 1);
+  Wgg = networkstatus_get_bw_weight(ns, "Wgg", -1);
+  Wgm = networkstatus_get_bw_weight(ns, "Wgm", -1);
+  Wgd = networkstatus_get_bw_weight(ns, "Wgd", -1);
+  Wmg = networkstatus_get_bw_weight(ns, "Wmg", -1);
+  Wmm = networkstatus_get_bw_weight(ns, "Wmm", -1);
+  Wme = networkstatus_get_bw_weight(ns, "Wme", -1);
+  Wmd = networkstatus_get_bw_weight(ns, "Wmd", -1);
+  Weg = networkstatus_get_bw_weight(ns, "Weg", -1);
+  Wem = networkstatus_get_bw_weight(ns, "Wem", -1);
+  Wee = networkstatus_get_bw_weight(ns, "Wee", -1);
+  Wed = networkstatus_get_bw_weight(ns, "Wed", -1);
+
+  if (Wgg<0 || Wgm<0 || Wgd<0 || Wmg<0 || Wmm<0 || Wme<0 || Wmd<0 || Weg<0
+          || Wem<0 || Wee<0 || Wed<0) {
+    log_warn(LD_BUG, "No bandwidth weights produced in consensus!");
+    return 0;
+  }
+
+  // First, sanity check basic summing properties that hold for all cases
+  // We use > 1 as the check for these because they are computed as integers.
+  // Sometimes there are rounding errors.
+  if (fabs(Wmm - weight_scale) > 1) {
+    log_warn(LD_BUG, "Wmm=%f != %"PRId64,
+             Wmm, (weight_scale));
+    valid = 0;
+  }
+
+  if (fabs(Wem - Wee) > 1) {
+    log_warn(LD_BUG, "Wem=%f != Wee=%f", Wem, Wee);
+    valid = 0;
+  }
+
+  if (fabs(Wgm - Wgg) > 1) {
+    log_warn(LD_BUG, "Wgm=%f != Wgg=%f", Wgm, Wgg);
+    valid = 0;
+  }
+
+  if (fabs(Weg - Wed) > 1) {
+    log_warn(LD_BUG, "Wed=%f != Weg=%f", Wed, Weg);
+    valid = 0;
+  }
+
+  if (fabs(Wgg + Wmg - weight_scale) > 0.001*weight_scale) {
+    log_warn(LD_BUG, "Wgg=%f != %"PRId64" - Wmg=%f", Wgg,
+             (weight_scale), Wmg);
+    valid = 0;
+  }
+
+  if (fabs(Wee + Wme - weight_scale) > 0.001*weight_scale) {
+    log_warn(LD_BUG, "Wee=%f != %"PRId64" - Wme=%f", Wee,
+             (weight_scale), Wme);
+    valid = 0;
+  }
+
+  if (fabs(Wgd + Wmd + Wed - weight_scale) > 0.001*weight_scale) {
+    log_warn(LD_BUG, "Wgd=%f + Wmd=%f + Wed=%f != %"PRId64,
+             Wgd, Wmd, Wed, (weight_scale));
+    valid = 0;
+  }
+
+  Wgg /= weight_scale;
+  Wgm /= weight_scale; (void) Wgm; // unused from here on.
+  Wgd /= weight_scale;
+
+  Wmg /= weight_scale;
+  Wmm /= weight_scale;
+  Wme /= weight_scale;
+  Wmd /= weight_scale;
+
+  Weg /= weight_scale; (void) Weg; // unused from here on.
+  Wem /= weight_scale; (void) Wem; // unused from here on.
+  Wee /= weight_scale;
+  Wed /= weight_scale;
+
+  // Then, gather G, M, E, D, T to determine case
+  SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
+    int is_exit = 0;
+    /* Bug #2203: Don't count bad exits as exits for balancing */
+    is_exit = rs->is_exit && !rs->is_bad_exit;
+    if (rs->has_bandwidth) {
+      T += rs->bandwidth_kb;
+      if (is_exit && rs->is_possible_guard) {
+        D += rs->bandwidth_kb;
+        Gtotal += Wgd*rs->bandwidth_kb;
+        Mtotal += Wmd*rs->bandwidth_kb;
+        Etotal += Wed*rs->bandwidth_kb;
+      } else if (is_exit) {
+        E += rs->bandwidth_kb;
+        Mtotal += Wme*rs->bandwidth_kb;
+        Etotal += Wee*rs->bandwidth_kb;
+      } else if (rs->is_possible_guard) {
+        G += rs->bandwidth_kb;
+        Gtotal += Wgg*rs->bandwidth_kb;
+        Mtotal += Wmg*rs->bandwidth_kb;
+      } else {
+        M += rs->bandwidth_kb;
+        Mtotal += Wmm*rs->bandwidth_kb;
+      }
+    } else {
+      log_warn(LD_BUG, "Missing consensus bandwidth for router %s",
+               routerstatus_describe(rs));
+    }
+  } SMARTLIST_FOREACH_END(rs);
+
+  // Finally, check equality conditions depending upon case 1, 2 or 3
+  // Full equality cases: 1, 3b
+  // Partial equality cases: 2b (E=G), 3a (M=E)
+  // Fully unknown: 2a
+  if (3*E >= T && 3*G >= T) {
+    // Case 1: Neither are scarce
+    casename = "Case 1";
+    if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) {
+      log_warn(LD_DIR,
+               "Bw Weight Failure for %s: Etotal %f != Mtotal %f. "
+               "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+               " T=%"PRId64". "
+               "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+               casename, Etotal, Mtotal,
+               (G), (M), (E),
+               (D), (T),
+               Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+      valid = 0;
+    }
+    if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) {
+      log_warn(LD_DIR,
+               "Bw Weight Failure for %s: Etotal %f != Gtotal %f. "
+               "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+               " T=%"PRId64". "
+               "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+               casename, Etotal, Gtotal,
+               (G), (M), (E),
+               (D), (T),
+               Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+      valid = 0;
+    }
+    if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) {
+      log_warn(LD_DIR,
+               "Bw Weight Failure for %s: Mtotal %f != Gtotal %f. "
+               "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+               " T=%"PRId64". "
+               "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+               casename, Mtotal, Gtotal,
+               (G), (M), (E),
+               (D), (T),
+               Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+      valid = 0;
+    }
+  } else if (3*E < T && 3*G < T) {
+    int64_t R = MIN(E, G);
+    int64_t S = MAX(E, G);
+    /*
+     * Case 2: Both Guards and Exits are scarce
+     * Balance D between E and G, depending upon
+     * D capacity and scarcity. Devote no extra
+     * bandwidth to middle nodes.
+     */
+    if (R+D < S) { // Subcase a
+      double Rtotal, Stotal;
+      if (E < G) {
+        Rtotal = Etotal;
+        Stotal = Gtotal;
+      } else {
+        Rtotal = Gtotal;
+        Stotal = Etotal;
+      }
+      casename = "Case 2a";
+      // Rtotal < Stotal
+      if (Rtotal > Stotal) {
+        log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: Rtotal %f > Stotal %f. "
+                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                   " T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Rtotal, Stotal,
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+      // Rtotal < T/3
+      if (3*Rtotal > T) {
+        log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: 3*Rtotal %f > T "
+                   "%"PRId64". G=%"PRId64" M=%"PRId64" E=%"PRId64
+                   " D=%"PRId64" T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Rtotal*3, (T),
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+      // Stotal < T/3
+      if (3*Stotal > T) {
+        log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: 3*Stotal %f > T "
+                   "%"PRId64". G=%"PRId64" M=%"PRId64" E=%"PRId64
+                   " D=%"PRId64" T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Stotal*3, (T),
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+      // Mtotal > T/3
+      if (3*Mtotal < T) {
+        log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: 3*Mtotal %f < T "
+                   "%"PRId64". "
+                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                   " T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Mtotal*3, (T),
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+    } else { // Subcase b: R+D > S
+      casename = "Case 2b";
+
+      /* Check the rare-M redirect case. */
+      if (D != 0 && 3*M < T) {
+        casename = "Case 2b (balanced)";
+        if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) {
+          log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: Etotal %f != Mtotal %f. "
+                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                   " T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Etotal, Mtotal,
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+          valid = 0;
+        }
+        if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) {
+          log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: Etotal %f != Gtotal %f. "
+                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                   " T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Etotal, Gtotal,
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+          valid = 0;
+        }
+        if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) {
+          log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: Mtotal %f != Gtotal %f. "
+                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                   " T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Mtotal, Gtotal,
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+          valid = 0;
+        }
+      } else {
+        if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) {
+          log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: Etotal %f != Gtotal %f. "
+                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                   " T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Etotal, Gtotal,
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+          valid = 0;
+        }
+      }
+    }
+  } else { // if (E < T/3 || G < T/3) {
+    int64_t S = MIN(E, G);
+    int64_t NS = MAX(E, G);
+    if (3*(S+D) < T) { // Subcase a:
+      double Stotal;
+      double NStotal;
+      if (G < E) {
+        casename = "Case 3a (G scarce)";
+        Stotal = Gtotal;
+        NStotal = Etotal;
+      } else { // if (G >= E) {
+        casename = "Case 3a (E scarce)";
+        NStotal = Gtotal;
+        Stotal = Etotal;
+      }
+      // Stotal < T/3
+      if (3*Stotal > T) {
+        log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: 3*Stotal %f > T "
+                   "%"PRId64". G=%"PRId64" M=%"PRId64" E=%"PRId64
+                   " D=%"PRId64" T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, Stotal*3, (T),
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+      if (NS >= M) {
+        if (fabs(NStotal-Mtotal) > 0.01*MAX(NStotal,Mtotal)) {
+          log_warn(LD_DIR,
+                   "Bw Weight Failure for %s: NStotal %f != Mtotal %f. "
+                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                   " T=%"PRId64". "
+                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                   casename, NStotal, Mtotal,
+                   (G), (M), (E),
+                   (D), (T),
+                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+          valid = 0;
+        }
+      } else {
+        // if NS < M, NStotal > T/3 because only one of G or E is scarce
+        if (3*NStotal < T) {
+          log_warn(LD_DIR,
+                     "Bw Weight Failure for %s: 3*NStotal %f < T "
+                     "%"PRId64". G=%"PRId64" M=%"PRId64
+                     " E=%"PRId64" D=%"PRId64" T=%"PRId64". "
+                     "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                     casename, NStotal*3, (T),
+                     (G), (M), (E),
+                     (D), (T),
+                     Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+          valid = 0;
+        }
+      }
+    } else { // Subcase b: S+D >= T/3
+      casename = "Case 3b";
+      if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) {
+        log_warn(LD_DIR,
+                 "Bw Weight Failure for %s: Etotal %f != Mtotal %f. "
+                 "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                 " T=%"PRId64". "
+                 "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                 casename, Etotal, Mtotal,
+                 (G), (M), (E),
+                 (D), (T),
+                 Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+      if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) {
+        log_warn(LD_DIR,
+                 "Bw Weight Failure for %s: Etotal %f != Gtotal %f. "
+                 "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                 " T=%"PRId64". "
+                 "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                 casename, Etotal, Gtotal,
+                 (G), (M), (E),
+                 (D), (T),
+                 Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+      if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) {
+        log_warn(LD_DIR,
+                 "Bw Weight Failure for %s: Mtotal %f != Gtotal %f. "
+                 "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
+                 " T=%"PRId64". "
+                 "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
+                 casename, Mtotal, Gtotal,
+                 (G), (M), (E),
+                 (D), (T),
+                 Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
+        valid = 0;
+      }
+    }
+  }
+
+  if (valid)
+    log_notice(LD_DIR, "Bandwidth-weight %s is verified and valid.",
+               casename);
+
+  return valid;
+}
+
+/** Check if a shared random value of type <b>srv_type</b> is in
+ *  <b>tokens</b>. If there is, parse it and set it to <b>srv_out</b>. Return
+ *  -1 on failure, 0 on success. The resulting srv is allocated on the heap and
+ *  it's the responsibility of the caller to free it. */
+static int
+extract_one_srv(smartlist_t *tokens, directory_keyword srv_type,
+                sr_srv_t **srv_out)
+{
+  int ret = -1;
+  directory_token_t *tok;
+  sr_srv_t *srv = NULL;
+  smartlist_t *chunks;
+
+  tor_assert(tokens);
+
+  chunks = smartlist_new();
+  tok = find_opt_by_keyword(tokens, srv_type);
+  if (!tok) {
+    /* That's fine, no SRV is allowed. */
+    ret = 0;
+    goto end;
+  }
+  for (int i = 0; i < tok->n_args; i++) {
+    smartlist_add(chunks, tok->args[i]);
+  }
+  srv = sr_parse_srv(chunks);
+  if (srv == NULL) {
+    log_warn(LD_DIR, "SR: Unparseable SRV %s", escaped(tok->object_body));
+    goto end;
+  }
+  /* All is good. */
+  *srv_out = srv;
+  ret = 0;
+ end:
+  smartlist_free(chunks);
+  return ret;
+}
+
+/** Extract any shared random values found in <b>tokens</b> and place them in
+ *  the networkstatus <b>ns</b>. */
+static void
+extract_shared_random_srvs(networkstatus_t *ns, smartlist_t *tokens)
+{
+  const char *voter_identity;
+  networkstatus_voter_info_t *voter;
+
+  tor_assert(ns);
+  tor_assert(tokens);
+  /* Can be only one of them else code flow. */
+  tor_assert(ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_CONSENSUS);
+
+  if (ns->type == NS_TYPE_VOTE) {
+    voter = smartlist_get(ns->voters, 0);
+    tor_assert(voter);
+    voter_identity = hex_str(voter->identity_digest,
+                             sizeof(voter->identity_digest));
+  } else {
+    /* Consensus has multiple voters so no specific voter. */
+    voter_identity = "consensus";
+  }
+
+  /* We extract both, and on error everything is stopped because it means
+   * the vote is malformed for the shared random value(s). */
+  if (extract_one_srv(tokens, K_PREVIOUS_SRV, &ns->sr_info.previous_srv) < 0) {
+    log_warn(LD_DIR, "SR: Unable to parse previous SRV from %s",
+             voter_identity);
+    /* Maybe we have a chance with the current SRV so let's try it anyway. */
+  }
+  if (extract_one_srv(tokens, K_CURRENT_SRV, &ns->sr_info.current_srv) < 0) {
+    log_warn(LD_DIR, "SR: Unable to parse current SRV from %s",
+             voter_identity);
+  }
+}
+
+/** Parse a v3 networkstatus vote, opinion, or consensus (depending on
+ * ns_type), from <b>s</b>, and return the result.  Return NULL on failure. */
+networkstatus_t *
+networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
+                                     networkstatus_type_t ns_type)
+{
+  smartlist_t *tokens = smartlist_new();
+  smartlist_t *rs_tokens = NULL, *footer_tokens = NULL;
+  networkstatus_voter_info_t *voter = NULL;
+  networkstatus_t *ns = NULL;
+  common_digests_t ns_digests;
+  uint8_t sha3_as_signed[DIGEST256_LEN];
+  const char *cert, *end_of_header, *end_of_footer, *s_dup = s;
+  directory_token_t *tok;
+  struct in_addr in;
+  int i, inorder, n_signatures = 0;
+  memarea_t *area = NULL, *rs_area = NULL;
+  consensus_flavor_t flav = FLAV_NS;
+  char *last_kwd=NULL;
+
+  tor_assert(s);
+
+  if (eos_out)
+    *eos_out = NULL;
+
+  if (router_get_networkstatus_v3_hashes(s, &ns_digests) ||
+      router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, s)<0) {
+    log_warn(LD_DIR, "Unable to compute digest of network-status");
+    goto err;
+  }
+
+  area = memarea_new();
+  end_of_header = find_start_of_next_routerstatus(s);
+  if (tokenize_string(area, s, end_of_header, tokens,
+                      (ns_type == NS_TYPE_CONSENSUS) ?
+                      networkstatus_consensus_token_table :
+                      networkstatus_token_table, 0)) {
+    log_warn(LD_DIR, "Error tokenizing network-status header");
+    goto err;
+  }
+
+  ns = tor_malloc_zero(sizeof(networkstatus_t));
+  memcpy(&ns->digests, &ns_digests, sizeof(ns_digests));
+  memcpy(&ns->digest_sha3_as_signed, sha3_as_signed, sizeof(sha3_as_signed));
+
+  tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION);
+  tor_assert(tok);
+  if (tok->n_args > 1) {
+    int flavor = networkstatus_parse_flavor_name(tok->args[1]);
+    if (flavor < 0) {
+      log_warn(LD_DIR, "Can't parse document with unknown flavor %s",
+               escaped(tok->args[1]));
+      goto err;
+    }
+    ns->flavor = flav = flavor;
+  }
+  if (flav != FLAV_NS && ns_type != NS_TYPE_CONSENSUS) {
+    log_warn(LD_DIR, "Flavor found on non-consensus networkstatus.");
+    goto err;
+  }
+
+  if (ns_type != NS_TYPE_CONSENSUS) {
+    const char *end_of_cert = NULL;
+    if (!(cert = strstr(s, "\ndir-key-certificate-version")))
+      goto err;
+    ++cert;
+    ns->cert = authority_cert_parse_from_string(cert, &end_of_cert);
+    if (!ns->cert || !end_of_cert || end_of_cert > end_of_header)
+      goto err;
+  }
+
+  tok = find_by_keyword(tokens, K_VOTE_STATUS);
+  tor_assert(tok->n_args);
+  if (!strcmp(tok->args[0], "vote")) {
+    ns->type = NS_TYPE_VOTE;
+  } else if (!strcmp(tok->args[0], "consensus")) {
+    ns->type = NS_TYPE_CONSENSUS;
+  } else if (!strcmp(tok->args[0], "opinion")) {
+    ns->type = NS_TYPE_OPINION;
+  } else {
+    log_warn(LD_DIR, "Unrecognized vote status %s in network-status",
+             escaped(tok->args[0]));
+    goto err;
+  }
+  if (ns_type != ns->type) {
+    log_warn(LD_DIR, "Got the wrong kind of v3 networkstatus.");
+    goto err;
+  }
+
+  if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_OPINION) {
+    tok = find_by_keyword(tokens, K_PUBLISHED);
+    if (parse_iso_time(tok->args[0], &ns->published))
+      goto err;
+
+    ns->supported_methods = smartlist_new();
+    tok = find_opt_by_keyword(tokens, K_CONSENSUS_METHODS);
+    if (tok) {
+      for (i=0; i < tok->n_args; ++i)
+        smartlist_add_strdup(ns->supported_methods, tok->args[i]);
+    } else {
+      smartlist_add_strdup(ns->supported_methods, "1");
+    }
+  } else {
+    tok = find_opt_by_keyword(tokens, K_CONSENSUS_METHOD);
+    if (tok) {
+      int num_ok;
+      ns->consensus_method = (int)tor_parse_long(tok->args[0], 10, 1, INT_MAX,
+                                                 &num_ok, NULL);
+      if (!num_ok)
+        goto err;
+    } else {
+      ns->consensus_method = 1;
+    }
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_RECOMMENDED_CLIENT_PROTOCOLS)))
+    ns->recommended_client_protocols = tor_strdup(tok->args[0]);
+  if ((tok = find_opt_by_keyword(tokens, K_RECOMMENDED_RELAY_PROTOCOLS)))
+    ns->recommended_relay_protocols = tor_strdup(tok->args[0]);
+  if ((tok = find_opt_by_keyword(tokens, K_REQUIRED_CLIENT_PROTOCOLS)))
+    ns->required_client_protocols = tor_strdup(tok->args[0]);
+  if ((tok = find_opt_by_keyword(tokens, K_REQUIRED_RELAY_PROTOCOLS)))
+    ns->required_relay_protocols = tor_strdup(tok->args[0]);
+
+  tok = find_by_keyword(tokens, K_VALID_AFTER);
+  if (parse_iso_time(tok->args[0], &ns->valid_after))
+    goto err;
+
+  tok = find_by_keyword(tokens, K_FRESH_UNTIL);
+  if (parse_iso_time(tok->args[0], &ns->fresh_until))
+    goto err;
+
+  tok = find_by_keyword(tokens, K_VALID_UNTIL);
+  if (parse_iso_time(tok->args[0], &ns->valid_until))
+    goto err;
+
+  tok = find_by_keyword(tokens, K_VOTING_DELAY);
+  tor_assert(tok->n_args >= 2);
+  {
+    int ok;
+    ns->vote_seconds =
+      (int) tor_parse_long(tok->args[0], 10, 0, INT_MAX, &ok, NULL);
+    if (!ok)
+      goto err;
+    ns->dist_seconds =
+      (int) tor_parse_long(tok->args[1], 10, 0, INT_MAX, &ok, NULL);
+    if (!ok)
+      goto err;
+  }
+  if (ns->valid_after +
+      (get_options()->TestingTorNetwork ?
+       MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL) > ns->fresh_until) {
+    log_warn(LD_DIR, "Vote/consensus freshness interval is too short");
+    goto err;
+  }
+  if (ns->valid_after +
+      (get_options()->TestingTorNetwork ?
+       MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL)*2 > ns->valid_until) {
+    log_warn(LD_DIR, "Vote/consensus liveness interval is too short");
+    goto err;
+  }
+  if (ns->vote_seconds < MIN_VOTE_SECONDS) {
+    log_warn(LD_DIR, "Vote seconds is too short");
+    goto err;
+  }
+  if (ns->dist_seconds < MIN_DIST_SECONDS) {
+    log_warn(LD_DIR, "Dist seconds is too short");
+    goto err;
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_CLIENT_VERSIONS))) {
+    ns->client_versions = tor_strdup(tok->args[0]);
+  }
+  if ((tok = find_opt_by_keyword(tokens, K_SERVER_VERSIONS))) {
+    ns->server_versions = tor_strdup(tok->args[0]);
+  }
+
+  {
+    smartlist_t *package_lst = find_all_by_keyword(tokens, K_PACKAGE);
+    ns->package_lines = smartlist_new();
+    if (package_lst) {
+      SMARTLIST_FOREACH(package_lst, directory_token_t *, t,
+                    smartlist_add_strdup(ns->package_lines, t->args[0]));
+    }
+    smartlist_free(package_lst);
+  }
+
+  tok = find_by_keyword(tokens, K_KNOWN_FLAGS);
+  ns->known_flags = smartlist_new();
+  inorder = 1;
+  for (i = 0; i < tok->n_args; ++i) {
+    smartlist_add_strdup(ns->known_flags, tok->args[i]);
+    if (i>0 && strcmp(tok->args[i-1], tok->args[i])>= 0) {
+      log_warn(LD_DIR, "%s >= %s", tok->args[i-1], tok->args[i]);
+      inorder = 0;
+    }
+  }
+  if (!inorder) {
+    log_warn(LD_DIR, "known-flags not in order");
+    goto err;
+  }
+  if (ns->type != NS_TYPE_CONSENSUS &&
+      smartlist_len(ns->known_flags) > MAX_KNOWN_FLAGS_IN_VOTE) {
+    /* If we allowed more than 64 flags in votes, then parsing them would make
+     * us invoke undefined behavior whenever we used 1<<flagnum to do a
+     * bit-shift. This is only for votes and opinions: consensus users don't
+     * care about flags they don't recognize, and so don't build a bitfield
+     * for them. */
+    log_warn(LD_DIR, "Too many known-flags in consensus vote or opinion");
+    goto err;
+  }
+
+  tok = find_opt_by_keyword(tokens, K_PARAMS);
+  if (tok) {
+    int any_dups = 0;
+    inorder = 1;
+    ns->net_params = smartlist_new();
+    for (i = 0; i < tok->n_args; ++i) {
+      int ok=0;
+      char *eq = strchr(tok->args[i], '=');
+      size_t eq_pos;
+      if (!eq) {
+        log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i]));
+        goto err;
+      }
+      eq_pos = eq-tok->args[i];
+      tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL);
+      if (!ok) {
+        log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i]));
+        goto err;
+      }
+      if (i > 0 && strcmp(tok->args[i-1], tok->args[i]) >= 0) {
+        log_warn(LD_DIR, "%s >= %s", tok->args[i-1], tok->args[i]);
+        inorder = 0;
+      }
+      if (last_kwd && eq_pos == strlen(last_kwd) &&
+          fast_memeq(last_kwd, tok->args[i], eq_pos)) {
+        log_warn(LD_DIR, "Duplicate value for %s parameter",
+                 escaped(tok->args[i]));
+        any_dups = 1;
+      }
+      tor_free(last_kwd);
+      last_kwd = tor_strndup(tok->args[i], eq_pos);
+      smartlist_add_strdup(ns->net_params, tok->args[i]);
+    }
+    if (!inorder) {
+      log_warn(LD_DIR, "params not in order");
+      goto err;
+    }
+    if (any_dups) {
+      log_warn(LD_DIR, "Duplicate in parameters");
+      goto err;
+    }
+  }
+
+  ns->voters = smartlist_new();
+
+  SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) {
+    tok = _tok;
+    if (tok->tp == K_DIR_SOURCE) {
+      tor_assert(tok->n_args >= 6);
+
+      if (voter)
+        smartlist_add(ns->voters, voter);
+      voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t));
+      voter->sigs = smartlist_new();
+      if (ns->type != NS_TYPE_CONSENSUS)
+        memcpy(voter->vote_digest, ns_digests.d[DIGEST_SHA1], DIGEST_LEN);
+
+      voter->nickname = tor_strdup(tok->args[0]);
+      if (strlen(tok->args[1]) != HEX_DIGEST_LEN ||
+          base16_decode(voter->identity_digest, sizeof(voter->identity_digest),
+                        tok->args[1], HEX_DIGEST_LEN)
+                        != sizeof(voter->identity_digest)) {
+        log_warn(LD_DIR, "Error decoding identity digest %s in "
+                 "network-status document.", escaped(tok->args[1]));
+        goto err;
+      }
+      if (ns->type != NS_TYPE_CONSENSUS &&
+          tor_memneq(ns->cert->cache_info.identity_digest,
+                 voter->identity_digest, DIGEST_LEN)) {
+        log_warn(LD_DIR,"Mismatch between identities in certificate and vote");
+        goto err;
+      }
+      if (ns->type != NS_TYPE_CONSENSUS) {
+        if (authority_cert_is_blacklisted(ns->cert)) {
+          log_warn(LD_DIR, "Rejecting vote signature made with blacklisted "
+                   "signing key %s",
+                   hex_str(ns->cert->signing_key_digest, DIGEST_LEN));
+          goto err;
+        }
+      }
+      voter->address = tor_strdup(tok->args[2]);
+      if (!tor_inet_aton(tok->args[3], &in)) {
+        log_warn(LD_DIR, "Error decoding IP address %s in network-status.",
+                 escaped(tok->args[3]));
+        goto err;
+      }
+      voter->addr = ntohl(in.s_addr);
+      int ok;
+      voter->dir_port = (uint16_t)
+        tor_parse_long(tok->args[4], 10, 0, 65535, &ok, NULL);
+      if (!ok)
+        goto err;
+      voter->or_port = (uint16_t)
+        tor_parse_long(tok->args[5], 10, 0, 65535, &ok, NULL);
+      if (!ok)
+        goto err;
+    } else if (tok->tp == K_CONTACT) {
+      if (!voter || voter->contact) {
+        log_warn(LD_DIR, "contact element is out of place.");
+        goto err;
+      }
+      voter->contact = tor_strdup(tok->args[0]);
+    } else if (tok->tp == K_VOTE_DIGEST) {
+      tor_assert(ns->type == NS_TYPE_CONSENSUS);
+      tor_assert(tok->n_args >= 1);
+      if (!voter || ! tor_digest_is_zero(voter->vote_digest)) {
+        log_warn(LD_DIR, "vote-digest element is out of place.");
+        goto err;
+      }
+      if (strlen(tok->args[0]) != HEX_DIGEST_LEN ||
+        base16_decode(voter->vote_digest, sizeof(voter->vote_digest),
+                      tok->args[0], HEX_DIGEST_LEN)
+                      != sizeof(voter->vote_digest)) {
+        log_warn(LD_DIR, "Error decoding vote digest %s in "
+                 "network-status consensus.", escaped(tok->args[0]));
+        goto err;
+      }
+    }
+  } SMARTLIST_FOREACH_END(_tok);
+  if (voter) {
+    smartlist_add(ns->voters, voter);
+    voter = NULL;
+  }
+  if (smartlist_len(ns->voters) == 0) {
+    log_warn(LD_DIR, "Missing dir-source elements in a networkstatus.");
+    goto err;
+  } else if (ns->type != NS_TYPE_CONSENSUS && smartlist_len(ns->voters) != 1) {
+    log_warn(LD_DIR, "Too many dir-source elements in a vote networkstatus.");
+    goto err;
+  }
+
+  if (ns->type != NS_TYPE_CONSENSUS &&
+      (tok = find_opt_by_keyword(tokens, K_LEGACY_DIR_KEY))) {
+    int bad = 1;
+    if (strlen(tok->args[0]) == HEX_DIGEST_LEN) {
+      networkstatus_voter_info_t *voter_0 = smartlist_get(ns->voters, 0);
+      if (base16_decode(voter_0->legacy_id_digest, DIGEST_LEN,
+                        tok->args[0], HEX_DIGEST_LEN) != DIGEST_LEN)
+        bad = 1;
+      else
+        bad = 0;
+    }
+    if (bad) {
+      log_warn(LD_DIR, "Invalid legacy key digest %s on vote.",
+               escaped(tok->args[0]));
+    }
+  }
+
+  /* If this is a vote document, check if information about the shared
+     randomness protocol is included, and extract it. */
+  if (ns->type == NS_TYPE_VOTE) {
+    dirvote_parse_sr_commits(ns, tokens);
+  }
+  /* For both a vote and consensus, extract the shared random values. */
+  if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_CONSENSUS) {
+    extract_shared_random_srvs(ns, tokens);
+  }
+
+  /* Parse routerstatus lines. */
+  rs_tokens = smartlist_new();
+  rs_area = memarea_new();
+  s = end_of_header;
+  ns->routerstatus_list = smartlist_new();
+
+  while (!strcmpstart(s, "r ")) {
+    if (ns->type != NS_TYPE_CONSENSUS) {
+      vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t));
+      if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns,
+                                               rs, 0, 0)) {
+        smartlist_add(ns->routerstatus_list, rs);
+      } else {
+        vote_routerstatus_free(rs);
+      }
+    } else {
+      routerstatus_t *rs;
+      if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens,
+                                                     NULL, NULL,
+                                                     ns->consensus_method,
+                                                     flav))) {
+        /* Use exponential-backoff scheduling when downloading microdescs */
+        smartlist_add(ns->routerstatus_list, rs);
+      }
+    }
+  }
+  for (i = 1; i < smartlist_len(ns->routerstatus_list); ++i) {
+    routerstatus_t *rs1, *rs2;
+    if (ns->type != NS_TYPE_CONSENSUS) {
+      vote_routerstatus_t *a = smartlist_get(ns->routerstatus_list, i-1);
+      vote_routerstatus_t *b = smartlist_get(ns->routerstatus_list, i);
+      rs1 = &a->status; rs2 = &b->status;
+    } else {
+      rs1 = smartlist_get(ns->routerstatus_list, i-1);
+      rs2 = smartlist_get(ns->routerstatus_list, i);
+    }
+    if (fast_memcmp(rs1->identity_digest, rs2->identity_digest, DIGEST_LEN)
+        >= 0) {
+      log_warn(LD_DIR, "Networkstatus entries not sorted by identity digest");
+      goto err;
+    }
+  }
+  if (ns_type != NS_TYPE_CONSENSUS) {
+    digest256map_t *ed_id_map = digest256map_new();
+    SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, vote_routerstatus_t *,
+                            vrs) {
+      if (! vrs->has_ed25519_listing ||
+          tor_mem_is_zero((const char *)vrs->ed25519_id, DIGEST256_LEN))
+        continue;
+      if (digest256map_get(ed_id_map, vrs->ed25519_id) != NULL) {
+        log_warn(LD_DIR, "Vote networkstatus ed25519 identities were not "
+                 "unique");
+        digest256map_free(ed_id_map, NULL);
+        goto err;
+      }
+      digest256map_set(ed_id_map, vrs->ed25519_id, (void*)1);
+    } SMARTLIST_FOREACH_END(vrs);
+    digest256map_free(ed_id_map, NULL);
+  }
+
+  /* Parse footer; check signature. */
+  footer_tokens = smartlist_new();
+  if ((end_of_footer = strstr(s, "\nnetwork-status-version ")))
+    ++end_of_footer;
+  else
+    end_of_footer = s + strlen(s);
+  if (tokenize_string(area,s, end_of_footer, footer_tokens,
+                      networkstatus_vote_footer_token_table, 0)) {
+    log_warn(LD_DIR, "Error tokenizing network-status vote footer.");
+    goto err;
+  }
+
+  {
+    int found_sig = 0;
+    SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) {
+      tok = _tok;
+      if (tok->tp == K_DIRECTORY_SIGNATURE)
+        found_sig = 1;
+      else if (found_sig) {
+        log_warn(LD_DIR, "Extraneous token after first directory-signature");
+        goto err;
+      }
+    } SMARTLIST_FOREACH_END(_tok);
+  }
+
+  if ((tok = find_opt_by_keyword(footer_tokens, K_DIRECTORY_FOOTER))) {
+    if (tok != smartlist_get(footer_tokens, 0)) {
+      log_warn(LD_DIR, "Misplaced directory-footer token");
+      goto err;
+    }
+  }
+
+  tok = find_opt_by_keyword(footer_tokens, K_BW_WEIGHTS);
+  if (tok) {
+    ns->weight_params = smartlist_new();
+    for (i = 0; i < tok->n_args; ++i) {
+      int ok=0;
+      char *eq = strchr(tok->args[i], '=');
+      if (!eq) {
+        log_warn(LD_DIR, "Bad element '%s' in weight params",
+                 escaped(tok->args[i]));
+        goto err;
+      }
+      tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL);
+      if (!ok) {
+        log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i]));
+        goto err;
+      }
+      smartlist_add_strdup(ns->weight_params, tok->args[i]);
+    }
+  }
+
+  SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) {
+    char declared_identity[DIGEST_LEN];
+    networkstatus_voter_info_t *v;
+    document_signature_t *sig;
+    const char *id_hexdigest = NULL;
+    const char *sk_hexdigest = NULL;
+    digest_algorithm_t alg = DIGEST_SHA1;
+    tok = _tok;
+    if (tok->tp != K_DIRECTORY_SIGNATURE)
+      continue;
+    tor_assert(tok->n_args >= 2);
+    if (tok->n_args == 2) {
+      id_hexdigest = tok->args[0];
+      sk_hexdigest = tok->args[1];
+    } else {
+      const char *algname = tok->args[0];
+      int a;
+      id_hexdigest = tok->args[1];
+      sk_hexdigest = tok->args[2];
+      a = crypto_digest_algorithm_parse_name(algname);
+      if (a<0) {
+        log_warn(LD_DIR, "Unknown digest algorithm %s; skipping",
+                 escaped(algname));
+        continue;
+      }
+      alg = a;
+    }
+
+    if (!tok->object_type ||
+        strcmp(tok->object_type, "SIGNATURE") ||
+        tok->object_size < 128 || tok->object_size > 512) {
+      log_warn(LD_DIR, "Bad object type or length on directory-signature");
+      goto err;
+    }
+
+    if (strlen(id_hexdigest) != HEX_DIGEST_LEN ||
+        base16_decode(declared_identity, sizeof(declared_identity),
+                      id_hexdigest, HEX_DIGEST_LEN)
+                      != sizeof(declared_identity)) {
+      log_warn(LD_DIR, "Error decoding declared identity %s in "
+               "network-status document.", escaped(id_hexdigest));
+      goto err;
+    }
+    if (!(v = networkstatus_get_voter_by_id(ns, declared_identity))) {
+      log_warn(LD_DIR, "ID on signature on network-status document does "
+               "not match any declared directory source.");
+      goto err;
+    }
+    sig = tor_malloc_zero(sizeof(document_signature_t));
+    memcpy(sig->identity_digest, v->identity_digest, DIGEST_LEN);
+    sig->alg = alg;
+    if (strlen(sk_hexdigest) != HEX_DIGEST_LEN ||
+        base16_decode(sig->signing_key_digest, sizeof(sig->signing_key_digest),
+                      sk_hexdigest, HEX_DIGEST_LEN)
+                      != sizeof(sig->signing_key_digest)) {
+      log_warn(LD_DIR, "Error decoding declared signing key digest %s in "
+               "network-status document.", escaped(sk_hexdigest));
+      tor_free(sig);
+      goto err;
+    }
+
+    if (ns->type != NS_TYPE_CONSENSUS) {
+      if (tor_memneq(declared_identity, ns->cert->cache_info.identity_digest,
+                 DIGEST_LEN)) {
+        log_warn(LD_DIR, "Digest mismatch between declared and actual on "
+                 "network-status vote.");
+        tor_free(sig);
+        goto err;
+      }
+    }
+
+    if (networkstatus_get_voter_sig_by_alg(v, sig->alg)) {
+      /* We already parsed a vote with this algorithm from this voter. Use the
+         first one. */
+      log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus "
+             "that contains two signatures from the same voter with the same "
+             "algorithm. Ignoring the second signature.");
+      tor_free(sig);
+      continue;
+    }
+
+    if (ns->type != NS_TYPE_CONSENSUS) {
+      if (check_signature_token(ns_digests.d[DIGEST_SHA1], DIGEST_LEN,
+                                tok, ns->cert->signing_key, 0,
+                                "network-status document")) {
+        tor_free(sig);
+        goto err;
+      }
+      sig->good_signature = 1;
+    } else {
+      if (tok->object_size >= INT_MAX || tok->object_size >= SIZE_T_CEILING) {
+        tor_free(sig);
+        goto err;
+      }
+      sig->signature = tor_memdup(tok->object_body, tok->object_size);
+      sig->signature_len = (int) tok->object_size;
+    }
+    smartlist_add(v->sigs, sig);
+
+    ++n_signatures;
+  } SMARTLIST_FOREACH_END(_tok);
+
+  if (! n_signatures) {
+    log_warn(LD_DIR, "No signatures on networkstatus document.");
+    goto err;
+  } else if (ns->type == NS_TYPE_VOTE && n_signatures != 1) {
+    log_warn(LD_DIR, "Received more than one signature on a "
+             "network-status vote.");
+    goto err;
+  }
+
+  if (eos_out)
+    *eos_out = end_of_footer;
+
+  goto done;
+ err:
+  dump_desc(s_dup, "v3 networkstatus");
+  networkstatus_vote_free(ns);
+  ns = NULL;
+ done:
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  if (voter) {
+    if (voter->sigs) {
+      SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig,
+                        document_signature_free(sig));
+      smartlist_free(voter->sigs);
+    }
+    tor_free(voter->nickname);
+    tor_free(voter->address);
+    tor_free(voter->contact);
+    tor_free(voter);
+  }
+  if (rs_tokens) {
+    SMARTLIST_FOREACH(rs_tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(rs_tokens);
+  }
+  if (footer_tokens) {
+    SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(footer_tokens);
+  }
+  if (area) {
+    DUMP_AREA(area, "v3 networkstatus");
+    memarea_drop_all(area);
+  }
+  if (rs_area)
+    memarea_drop_all(rs_area);
+  tor_free(last_kwd);
+
+  return ns;
+}
diff --git a/src/feature/dirparse/ns_parse.h b/src/feature/dirparse/ns_parse.h
new file mode 100644
index 0000000000..10a6f9cefc
--- /dev/null
+++ b/src/feature/dirparse/ns_parse.h
@@ -0,0 +1,45 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file ns_parse.h
+ * \brief Header file for ns_parse.c.
+ **/
+
+#ifndef TOR_NS_PARSE_H
+#define TOR_NS_PARSE_H
+
+int router_get_networkstatus_v3_hashes(const char *s,
+                                       common_digests_t *digests);
+int router_get_networkstatus_v3_signed_boundaries(const char *s,
+                                                  const char **start_out,
+                                                  const char **end_out);
+int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
+                                               const char *s);
+int compare_vote_routerstatus_entries(const void **_a, const void **_b);
+
+int networkstatus_verify_bw_weights(networkstatus_t *ns, int);
+enum networkstatus_type_t;
+networkstatus_t *networkstatus_parse_vote_from_string(const char *s,
+                                           const char **eos_out,
+                                           enum networkstatus_type_t ns_type);
+
+#ifdef NS_PARSE_PRIVATE
+STATIC int routerstatus_parse_guardfraction(const char *guardfraction_str,
+                                            networkstatus_t *vote,
+                                            vote_routerstatus_t *vote_rs,
+                                            routerstatus_t *rs);
+struct memarea_t;
+STATIC routerstatus_t *routerstatus_parse_entry_from_string(
+                                     struct memarea_t *area,
+                                     const char **s, smartlist_t *tokens,
+                                     networkstatus_t *vote,
+                                     vote_routerstatus_t *vote_rs,
+                                     int consensus_method,
+                                     consensus_flavor_t flav);
+#endif
+
+#endif
diff --git a/src/feature/dirparse/parsecommon.c b/src/feature/dirparse/parsecommon.c
new file mode 100644
index 0000000000..632f5adff0
--- /dev/null
+++ b/src/feature/dirparse/parsecommon.c
@@ -0,0 +1,458 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file parsecommon.c
+ * \brief Common code to parse and validate various type of descriptors.
+ **/
+
+#include "feature/dirparse/parsecommon.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/encoding/binascii.h"
+#include "lib/container/smartlist.h"
+#include "lib/string/util_string.h"
+#include "lib/string/printf.h"
+#include "lib/memarea/memarea.h"
+#include "lib/crypt_ops/crypto_rsa.h"
+
+#include <string.h>
+
+#define MIN_ANNOTATION A_PURPOSE
+#define MAX_ANNOTATION A_UNKNOWN_
+
+#define ALLOC_ZERO(sz) memarea_alloc_zero(area,sz)
+#define ALLOC(sz) memarea_alloc(area,sz)
+#define STRDUP(str) memarea_strdup(area,str)
+#define STRNDUP(str,n) memarea_strndup(area,(str),(n))
+
+#define RET_ERR(msg)                                               \
+  STMT_BEGIN                                                       \
+    if (tok) token_clear(tok);                                      \
+    tok = ALLOC_ZERO(sizeof(directory_token_t));                   \
+    tok->tp = ERR_;                                                \
+    tok->error = STRDUP(msg);                                      \
+    goto done_tokenizing;                                          \
+  STMT_END
+
+/** Free all resources allocated for <b>tok</b> */
+void
+token_clear(directory_token_t *tok)
+{
+  if (tok->key)
+    crypto_pk_free(tok->key);
+}
+
+/** Read all tokens from a string between <b>start</b> and <b>end</b>, and add
+ * them to <b>out</b>.  Parse according to the token rules in <b>table</b>.
+ * Caller must free tokens in <b>out</b>.  If <b>end</b> is NULL, use the
+ * entire string.
+ */
+int
+tokenize_string(memarea_t *area,
+                const char *start, const char *end, smartlist_t *out,
+                const token_rule_t *table, int flags)
+{
+  const char **s;
+  directory_token_t *tok = NULL;
+  int counts[NIL_];
+  int i;
+  int first_nonannotation;
+  int prev_len = smartlist_len(out);
+  tor_assert(area);
+
+  s = &start;
+  if (!end) {
+    end = start+strlen(start);
+  } else {
+    /* it's only meaningful to check for nuls if we got an end-of-string ptr */
+    if (memchr(start, '\0', end-start)) {
+      log_warn(LD_DIR, "parse error: internal NUL character.");
+      return -1;
+    }
+  }
+  for (i = 0; i < NIL_; ++i)
+    counts[i] = 0;
+
+  SMARTLIST_FOREACH(out, const directory_token_t *, t, ++counts[t->tp]);
+
+  while (*s < end && (!tok || tok->tp != EOF_)) {
+    tok = get_next_token(area, s, end, table);
+    if (tok->tp == ERR_) {
+      log_warn(LD_DIR, "parse error: %s", tok->error);
+      token_clear(tok);
+      return -1;
+    }
+    ++counts[tok->tp];
+    smartlist_add(out, tok);
+    *s = eat_whitespace_eos(*s, end);
+  }
+
+  if (flags & TS_NOCHECK)
+    return 0;
+
+  if ((flags & TS_ANNOTATIONS_OK)) {
+    first_nonannotation = -1;
+    for (i = 0; i < smartlist_len(out); ++i) {
+      tok = smartlist_get(out, i);
+      if (tok->tp < MIN_ANNOTATION || tok->tp > MAX_ANNOTATION) {
+        first_nonannotation = i;
+        break;
+      }
+    }
+    if (first_nonannotation < 0) {
+      log_warn(LD_DIR, "parse error: item contains only annotations");
+      return -1;
+    }
+    for (i=first_nonannotation;  i < smartlist_len(out); ++i) {
+      tok = smartlist_get(out, i);
+      if (tok->tp >= MIN_ANNOTATION && tok->tp <= MAX_ANNOTATION) {
+        log_warn(LD_DIR, "parse error: Annotations mixed with keywords");
+        return -1;
+      }
+    }
+    if ((flags & TS_NO_NEW_ANNOTATIONS)) {
+      if (first_nonannotation != prev_len) {
+        log_warn(LD_DIR, "parse error: Unexpected annotations.");
+        return -1;
+      }
+    }
+  } else {
+    for (i=0;  i < smartlist_len(out); ++i) {
+      tok = smartlist_get(out, i);
+      if (tok->tp >= MIN_ANNOTATION && tok->tp <= MAX_ANNOTATION) {
+        log_warn(LD_DIR, "parse error: no annotations allowed.");
+        return -1;
+      }
+    }
+    first_nonannotation = 0;
+  }
+  for (i = 0; table[i].t; ++i) {
+    if (counts[table[i].v] < table[i].min_cnt) {
+      log_warn(LD_DIR, "Parse error: missing %s element.", table[i].t);
+      return -1;
+    }
+    if (counts[table[i].v] > table[i].max_cnt) {
+      log_warn(LD_DIR, "Parse error: too many %s elements.", table[i].t);
+      return -1;
+    }
+    if (table[i].pos & AT_START) {
+      if (smartlist_len(out) < 1 ||
+          (tok = smartlist_get(out, first_nonannotation))->tp != table[i].v) {
+        log_warn(LD_DIR, "Parse error: first item is not %s.", table[i].t);
+        return -1;
+      }
+    }
+    if (table[i].pos & AT_END) {
+      if (smartlist_len(out) < 1 ||
+          (tok = smartlist_get(out, smartlist_len(out)-1))->tp != table[i].v) {
+        log_warn(LD_DIR, "Parse error: last item is not %s.", table[i].t);
+        return -1;
+      }
+    }
+  }
+  return 0;
+}
+
+/** Helper: parse space-separated arguments from the string <b>s</b> ending at
+ * <b>eol</b>, and store them in the args field of <b>tok</b>.  Store the
+ * number of parsed elements into the n_args field of <b>tok</b>.  Allocate
+ * all storage in <b>area</b>.  Return the number of arguments parsed, or
+ * return -1 if there was an insanely high number of arguments. */
+static inline int
+get_token_arguments(memarea_t *area, directory_token_t *tok,
+                    const char *s, const char *eol)
+{
+/** Largest number of arguments we'll accept to any token, ever. */
+#define MAX_ARGS 512
+  char *mem = memarea_strndup(area, s, eol-s);
+  char *cp = mem;
+  int j = 0;
+  char *args[MAX_ARGS];
+  memset(args, 0, sizeof(args));
+  while (*cp) {
+    if (j == MAX_ARGS)
+      return -1;
+    args[j++] = cp;
+    cp = (char*)find_whitespace(cp);
+    if (!cp || !*cp)
+      break; /* End of the line. */
+    *cp++ = '\0';
+    cp = (char*)eat_whitespace(cp);
+  }
+  tok->n_args = j;
+  tok->args = memarea_memdup(area, args, j*sizeof(char*));
+  return j;
+#undef MAX_ARGS
+}
+
+/** Helper: make sure that the token <b>tok</b> with keyword <b>kwd</b> obeys
+ * the object syntax of <b>o_syn</b>.  Allocate all storage in <b>area</b>.
+ * Return <b>tok</b> on success, or a new ERR_ token if the token didn't
+ * conform to the syntax we wanted.
+ **/
+static inline directory_token_t *
+token_check_object(memarea_t *area, const char *kwd,
+                   directory_token_t *tok, obj_syntax o_syn)
+{
+  char ebuf[128];
+  switch (o_syn) {
+    case NO_OBJ:
+      /* No object is allowed for this token. */
+      if (tok->object_body) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Unexpected object for %s", kwd);
+        RET_ERR(ebuf);
+      }
+      if (tok->key) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Unexpected public key for %s", kwd);
+        RET_ERR(ebuf);
+      }
+      break;
+    case NEED_OBJ:
+      /* There must be a (non-key) object. */
+      if (!tok->object_body) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Missing object for %s", kwd);
+        RET_ERR(ebuf);
+      }
+      break;
+    case NEED_KEY_1024: /* There must be a 1024-bit public key. */
+    case NEED_SKEY_1024: /* There must be a 1024-bit private key. */
+      if (tok->key && crypto_pk_num_bits(tok->key) != PK_BYTES*8) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Wrong size on key for %s: %d bits",
+                     kwd, crypto_pk_num_bits(tok->key));
+        RET_ERR(ebuf);
+      }
+      /* fall through */
+    case NEED_KEY: /* There must be some kind of key. */
+      if (!tok->key) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Missing public key for %s", kwd);
+        RET_ERR(ebuf);
+      }
+      if (o_syn != NEED_SKEY_1024) {
+        if (crypto_pk_key_is_private(tok->key)) {
+          tor_snprintf(ebuf, sizeof(ebuf),
+               "Private key given for %s, which wants a public key", kwd);
+          RET_ERR(ebuf);
+        }
+      } else { /* o_syn == NEED_SKEY_1024 */
+        if (!crypto_pk_key_is_private(tok->key)) {
+          tor_snprintf(ebuf, sizeof(ebuf),
+               "Public key given for %s, which wants a private key", kwd);
+          RET_ERR(ebuf);
+        }
+      }
+      break;
+    case OBJ_OK:
+      /* Anything goes with this token. */
+      break;
+  }
+
+ done_tokenizing:
+  return tok;
+}
+
+/** Helper function: read the next token from *s, advance *s to the end of the
+ * token, and return the parsed token.  Parse *<b>s</b> according to the list
+ * of tokens in <b>table</b>.
+ */
+directory_token_t *
+get_next_token(memarea_t *area,
+               const char **s, const char *eos, const token_rule_t *table)
+{
+  /** Reject any object at least this big; it is probably an overflow, an
+   * attack, a bug, or some other nonsense. */
+#define MAX_UNPARSED_OBJECT_SIZE (128*1024)
+  /** Reject any line at least this big; it is probably an overflow, an
+   * attack, a bug, or some other nonsense. */
+#define MAX_LINE_LENGTH (128*1024)
+
+  const char *next, *eol, *obstart;
+  size_t obname_len;
+  int i;
+  directory_token_t *tok;
+  obj_syntax o_syn = NO_OBJ;
+  char ebuf[128];
+  const char *kwd = "";
+
+  tor_assert(area);
+  tok = ALLOC_ZERO(sizeof(directory_token_t));
+  tok->tp = ERR_;
+
+  /* Set *s to first token, eol to end-of-line, next to after first token */
+  *s = eat_whitespace_eos(*s, eos); /* eat multi-line whitespace */
+  tor_assert(eos >= *s);
+  eol = memchr(*s, '\n', eos-*s);
+  if (!eol)
+    eol = eos;
+  if (eol - *s > MAX_LINE_LENGTH) {
+    RET_ERR("Line far too long");
+  }
+
+  next = find_whitespace_eos(*s, eol);
+
+  if (!strcmp_len(*s, "opt", next-*s)) {
+    /* Skip past an "opt" at the start of the line. */
+    *s = eat_whitespace_eos_no_nl(next, eol);
+    next = find_whitespace_eos(*s, eol);
+  } else if (*s == eos) {  /* If no "opt", and end-of-line, line is invalid */
+    RET_ERR("Unexpected EOF");
+  }
+
+  /* Search the table for the appropriate entry.  (I tried a binary search
+   * instead, but it wasn't any faster.) */
+  for (i = 0; table[i].t ; ++i) {
+    if (!strcmp_len(*s, table[i].t, next-*s)) {
+      /* We've found the keyword. */
+      kwd = table[i].t;
+      tok->tp = table[i].v;
+      o_syn = table[i].os;
+      *s = eat_whitespace_eos_no_nl(next, eol);
+      /* We go ahead whether there are arguments or not, so that tok->args is
+       * always set if we want arguments. */
+      if (table[i].concat_args) {
+        /* The keyword takes the line as a single argument */
+        tok->args = ALLOC(sizeof(char*));
+        tok->args[0] = STRNDUP(*s,eol-*s); /* Grab everything on line */
+        tok->n_args = 1;
+      } else {
+        /* This keyword takes multiple arguments. */
+        if (get_token_arguments(area, tok, *s, eol)<0) {
+          tor_snprintf(ebuf, sizeof(ebuf),"Far too many arguments to %s", kwd);
+          RET_ERR(ebuf);
+        }
+        *s = eol;
+      }
+      if (tok->n_args < table[i].min_args) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Too few arguments to %s", kwd);
+        RET_ERR(ebuf);
+      } else if (tok->n_args > table[i].max_args) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Too many arguments to %s", kwd);
+        RET_ERR(ebuf);
+      }
+      break;
+    }
+  }
+
+  if (tok->tp == ERR_) {
+    /* No keyword matched; call it an "K_opt" or "A_unrecognized" */
+    if (*s < eol && **s == '@')
+      tok->tp = A_UNKNOWN_;
+    else
+      tok->tp = K_OPT;
+    tok->args = ALLOC(sizeof(char*));
+    tok->args[0] = STRNDUP(*s, eol-*s);
+    tok->n_args = 1;
+    o_syn = OBJ_OK;
+  }
+
+  /* Check whether there's an object present */
+  *s = eat_whitespace_eos(eol, eos);  /* Scan from end of first line */
+  tor_assert(eos >= *s);
+  eol = memchr(*s, '\n', eos-*s);
+  if (!eol || eol-*s<11 || strcmpstart(*s, "-----BEGIN ")) /* No object. */
+    goto check_object;
+
+  obstart = *s; /* Set obstart to start of object spec */
+  if (eol - *s <= 16 || memchr(*s+11,'\0',eol-*s-16) || /* no short lines, */
+      strcmp_len(eol-5, "-----", 5) ||           /* nuls or invalid endings */
+      (eol-*s) > MAX_UNPARSED_OBJECT_SIZE) {     /* name too long */
+    RET_ERR("Malformed object: bad begin line");
+  }
+  tok->object_type = STRNDUP(*s+11, eol-*s-16);
+  obname_len = eol-*s-16; /* store objname length here to avoid a strlen() */
+  *s = eol+1;    /* Set *s to possible start of object data (could be eos) */
+
+  /* Go to the end of the object */
+  next = tor_memstr(*s, eos-*s, "-----END ");
+  if (!next) {
+    RET_ERR("Malformed object: missing object end line");
+  }
+  tor_assert(eos >= next);
+  eol = memchr(next, '\n', eos-next);
+  if (!eol)  /* end-of-line marker, or eos if there's no '\n' */
+    eol = eos;
+  /* Validate the ending tag, which should be 9 + NAME + 5 + eol */
+  if ((size_t)(eol-next) != 9+obname_len+5 ||
+      strcmp_len(next+9, tok->object_type, obname_len) ||
+      strcmp_len(eol-5, "-----", 5)) {
+    tor_snprintf(ebuf, sizeof(ebuf), "Malformed object: mismatched end tag %s",
+             tok->object_type);
+    ebuf[sizeof(ebuf)-1] = '\0';
+    RET_ERR(ebuf);
+  }
+  if (next - *s > MAX_UNPARSED_OBJECT_SIZE)
+    RET_ERR("Couldn't parse object: missing footer or object much too big.");
+
+  if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) { /* If it's a public key */
+    tok->key = crypto_pk_new();
+    if (crypto_pk_read_public_key_from_string(tok->key, obstart, eol-obstart))
+      RET_ERR("Couldn't parse public key.");
+  } else if (!strcmp(tok->object_type, "RSA PRIVATE KEY")) { /* private key */
+    tok->key = crypto_pk_new();
+    if (crypto_pk_read_private_key_from_string(tok->key, obstart, eol-obstart))
+      RET_ERR("Couldn't parse private key.");
+  } else { /* If it's something else, try to base64-decode it */
+    int r;
+    tok->object_body = ALLOC(next-*s); /* really, this is too much RAM. */
+    r = base64_decode(tok->object_body, next-*s, *s, next-*s);
+    if (r<0)
+      RET_ERR("Malformed object: bad base64-encoded data");
+    tok->object_size = r;
+  }
+  *s = eol;
+
+ check_object:
+  tok = token_check_object(area, kwd, tok, o_syn);
+
+ done_tokenizing:
+  return tok;
+
+#undef RET_ERR
+#undef ALLOC
+#undef ALLOC_ZERO
+#undef STRDUP
+#undef STRNDUP
+}
+
+/** Find the first token in <b>s</b> whose keyword is <b>keyword</b>; fail
+ * with an assert if no such keyword is found.
+ */
+directory_token_t *
+find_by_keyword_(smartlist_t *s, directory_keyword keyword,
+                 const char *keyword_as_string)
+{
+  directory_token_t *tok = find_opt_by_keyword(s, keyword);
+  if (PREDICT_UNLIKELY(!tok)) {
+    log_err(LD_BUG, "Missing %s [%d] in directory object that should have "
+         "been validated. Internal error.", keyword_as_string, (int)keyword);
+    tor_assert(tok);
+  }
+  return tok;
+}
+
+/** Find the first token in <b>s</b> whose keyword is <b>keyword</b>; return
+ * NULL if no such keyword is found.
+ */
+directory_token_t *
+find_opt_by_keyword(const smartlist_t *s, directory_keyword keyword)
+{
+  SMARTLIST_FOREACH(s, directory_token_t *, t, if (t->tp == keyword) return t);
+  return NULL;
+}
+
+/** If there are any directory_token_t entries in <b>s</b> whose keyword is
+ * <b>k</b>, return a newly allocated smartlist_t containing all such entries,
+ * in the same order in which they occur in <b>s</b>.  Otherwise return
+ * NULL. */
+smartlist_t *
+find_all_by_keyword(const smartlist_t *s, directory_keyword k)
+{
+  smartlist_t *out = NULL;
+  SMARTLIST_FOREACH(s, directory_token_t *, t,
+                    if (t->tp == k) {
+                    if (!out)
+                    out = smartlist_new();
+                    smartlist_add(out, t);
+                    });
+  return out;
+}
diff --git a/src/feature/dirparse/parsecommon.h b/src/feature/dirparse/parsecommon.h
new file mode 100644
index 0000000000..ef74925b26
--- /dev/null
+++ b/src/feature/dirparse/parsecommon.h
@@ -0,0 +1,324 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file parsecommon.h
+ * \brief Header file for parsecommon.c
+ **/
+
+#ifndef TOR_PARSECOMMON_H
+#define TOR_PARSECOMMON_H
+
+#include <stddef.h>
+
+struct smartlist_t;
+struct crypto_pk_t;
+struct memarea_t;
+
+/** Enumeration of possible token types.  The ones starting with K_ correspond
+* to directory 'keywords'. A_ is for an annotation, R or C is related to
+* hidden services, ERR_ is an error in the tokenizing process, EOF_ is an
+* end-of-file marker, and NIL_ is used to encode not-a-token.
+*/
+typedef enum {
+  K_ACCEPT = 0,
+  K_ACCEPT6,
+  K_DIRECTORY_SIGNATURE,
+  K_RECOMMENDED_SOFTWARE,
+  K_REJECT,
+  K_REJECT6,
+  K_ROUTER,
+  K_SIGNED_DIRECTORY,
+  K_SIGNING_KEY,
+  K_ONION_KEY,
+  K_ONION_KEY_NTOR,
+  K_ROUTER_SIGNATURE,
+  K_PUBLISHED,
+  K_RUNNING_ROUTERS,
+  K_ROUTER_STATUS,
+  K_PLATFORM,
+  K_PROTO,
+  K_OPT,
+  K_BANDWIDTH,
+  K_CONTACT,
+  K_NETWORK_STATUS,
+  K_UPTIME,
+  K_DIR_SIGNING_KEY,
+  K_FAMILY,
+  K_FINGERPRINT,
+  K_HIBERNATING,
+  K_READ_HISTORY,
+  K_WRITE_HISTORY,
+  K_NETWORK_STATUS_VERSION,
+  K_DIR_SOURCE,
+  K_DIR_OPTIONS,
+  K_CLIENT_VERSIONS,
+  K_SERVER_VERSIONS,
+  K_RECOMMENDED_CLIENT_PROTOCOLS,
+  K_RECOMMENDED_RELAY_PROTOCOLS,
+  K_REQUIRED_CLIENT_PROTOCOLS,
+  K_REQUIRED_RELAY_PROTOCOLS,
+  K_OR_ADDRESS,
+  K_ID,
+  K_P,
+  K_P6,
+  K_R,
+  K_A,
+  K_S,
+  K_V,
+  K_W,
+  K_M,
+  K_EXTRA_INFO,
+  K_EXTRA_INFO_DIGEST,
+  K_CACHES_EXTRA_INFO,
+  K_HIDDEN_SERVICE_DIR,
+  K_ALLOW_SINGLE_HOP_EXITS,
+  K_IPV6_POLICY,
+  K_ROUTER_SIG_ED25519,
+  K_IDENTITY_ED25519,
+  K_MASTER_KEY_ED25519,
+  K_ONION_KEY_CROSSCERT,
+  K_NTOR_ONION_KEY_CROSSCERT,
+
+  K_DIRREQ_END,
+  K_DIRREQ_V2_IPS,
+  K_DIRREQ_V3_IPS,
+  K_DIRREQ_V2_REQS,
+  K_DIRREQ_V3_REQS,
+  K_DIRREQ_V2_SHARE,
+  K_DIRREQ_V3_SHARE,
+  K_DIRREQ_V2_RESP,
+  K_DIRREQ_V3_RESP,
+  K_DIRREQ_V2_DIR,
+  K_DIRREQ_V3_DIR,
+  K_DIRREQ_V2_TUN,
+  K_DIRREQ_V3_TUN,
+  K_ENTRY_END,
+  K_ENTRY_IPS,
+  K_CELL_END,
+  K_CELL_PROCESSED,
+  K_CELL_QUEUED,
+  K_CELL_TIME,
+  K_CELL_CIRCS,
+  K_EXIT_END,
+  K_EXIT_WRITTEN,
+  K_EXIT_READ,
+  K_EXIT_OPENED,
+
+  K_DIR_KEY_CERTIFICATE_VERSION,
+  K_DIR_IDENTITY_KEY,
+  K_DIR_KEY_PUBLISHED,
+  K_DIR_KEY_EXPIRES,
+  K_DIR_KEY_CERTIFICATION,
+  K_DIR_KEY_CROSSCERT,
+  K_DIR_ADDRESS,
+  K_DIR_TUNNELLED,
+
+  K_VOTE_STATUS,
+  K_VALID_AFTER,
+  K_FRESH_UNTIL,
+  K_VALID_UNTIL,
+  K_VOTING_DELAY,
+
+  K_KNOWN_FLAGS,
+  K_PARAMS,
+  K_BW_WEIGHTS,
+  K_VOTE_DIGEST,
+  K_CONSENSUS_DIGEST,
+  K_ADDITIONAL_DIGEST,
+  K_ADDITIONAL_SIGNATURE,
+  K_CONSENSUS_METHODS,
+  K_CONSENSUS_METHOD,
+  K_LEGACY_DIR_KEY,
+  K_DIRECTORY_FOOTER,
+  K_SIGNING_CERT_ED,
+  K_SR_FLAG,
+  K_COMMIT,
+  K_PREVIOUS_SRV,
+  K_CURRENT_SRV,
+  K_PACKAGE,
+
+  A_PURPOSE,
+  A_LAST_LISTED,
+  A_UNKNOWN_,
+
+  R_RENDEZVOUS_SERVICE_DESCRIPTOR,
+  R_VERSION,
+  R_PERMANENT_KEY,
+  R_SECRET_ID_PART,
+  R_PUBLICATION_TIME,
+  R_PROTOCOL_VERSIONS,
+  R_INTRODUCTION_POINTS,
+  R_SIGNATURE,
+
+  R_HS_DESCRIPTOR, /* From version 3, this MUST be generic to all future
+                      descriptor versions thus making it R_. */
+  R3_DESC_LIFETIME,
+  R3_DESC_SIGNING_CERT,
+  R3_REVISION_COUNTER,
+  R3_SUPERENCRYPTED,
+  R3_SIGNATURE,
+  R3_CREATE2_FORMATS,
+  R3_INTRO_AUTH_REQUIRED,
+  R3_SINGLE_ONION_SERVICE,
+  R3_INTRODUCTION_POINT,
+  R3_INTRO_ONION_KEY,
+  R3_INTRO_AUTH_KEY,
+  R3_INTRO_ENC_KEY,
+  R3_INTRO_ENC_KEY_CERT,
+  R3_INTRO_LEGACY_KEY,
+  R3_INTRO_LEGACY_KEY_CERT,
+  R3_DESC_AUTH_TYPE,
+  R3_DESC_AUTH_KEY,
+  R3_DESC_AUTH_CLIENT,
+  R3_ENCRYPTED,
+
+  R_IPO_IDENTIFIER,
+  R_IPO_IP_ADDRESS,
+  R_IPO_ONION_PORT,
+  R_IPO_ONION_KEY,
+  R_IPO_SERVICE_KEY,
+
+  C_CLIENT_NAME,
+  C_DESCRIPTOR_COOKIE,
+  C_CLIENT_KEY,
+
+  ERR_,
+  EOF_,
+  NIL_
+} directory_keyword;
+
+/** Structure to hold a single directory token.
+ *
+ * We parse a directory by breaking it into "tokens", each consisting
+ * of a keyword, a line full of arguments, and a binary object.  The
+ * arguments and object are both optional, depending on the keyword
+ * type.
+ *
+ * This structure is only allocated in memareas; do not allocate it on
+ * the heap, or token_clear() won't work.
+ */
+typedef struct directory_token_t {
+  directory_keyword tp;        /**< Type of the token. */
+  int n_args:30;               /**< Number of elements in args */
+  char **args;                 /**< Array of arguments from keyword line. */
+
+  char *object_type;           /**< -----BEGIN [object_type]-----*/
+  size_t object_size;          /**< Bytes in object_body */
+  char *object_body;           /**< Contents of object, base64-decoded. */
+
+  struct crypto_pk_t *key;     /**< For public keys only.  Heap-allocated. */
+
+  char *error;                 /**< For ERR_ tokens only. */
+} directory_token_t;
+
+/** We use a table of rules to decide how to parse each token type. */
+
+/** Rules for whether the keyword needs an object. */
+typedef enum {
+  NO_OBJ,        /**< No object, ever. */
+  NEED_OBJ,      /**< Object is required. */
+  NEED_SKEY_1024,/**< Object is required, and must be a 1024 bit private key */
+  NEED_KEY_1024, /**< Object is required, and must be a 1024 bit public key */
+  NEED_KEY,      /**< Object is required, and must be a public key. */
+  OBJ_OK,        /**< Object is optional. */
+} obj_syntax;
+
+#define AT_START 1
+#define AT_END 2
+
+#define TS_ANNOTATIONS_OK 1
+#define TS_NOCHECK 2
+#define TS_NO_NEW_ANNOTATIONS 4
+
+/**
+ * @name macros for defining token rules
+ *
+ * Helper macros to define token tables.  's' is a string, 't' is a
+ * directory_keyword, 'a' is a trio of argument multiplicities, and 'o' is an
+ * object syntax.
+ */
+/**@{*/
+
+/** Appears to indicate the end of a table. */
+#define END_OF_TABLE { NULL, NIL_, 0,0,0, NO_OBJ, 0, INT_MAX, 0, 0 }
+/** An item with no restrictions: used for obsolete document types */
+#define T(s,t,a,o)    { s, t, a, o, 0, INT_MAX, 0, 0 }
+/** An item with no restrictions on multiplicity or location. */
+#define T0N(s,t,a,o)  { s, t, a, o, 0, INT_MAX, 0, 0 }
+/** An item that must appear exactly once */
+#define T1(s,t,a,o)   { s, t, a, o, 1, 1, 0, 0 }
+/** An item that must appear exactly once, at the start of the document */
+#define T1_START(s,t,a,o)   { s, t, a, o, 1, 1, AT_START, 0 }
+/** An item that must appear exactly once, at the end of the document */
+#define T1_END(s,t,a,o)   { s, t, a, o, 1, 1, AT_END, 0 }
+/** An item that must appear one or more times */
+#define T1N(s,t,a,o)  { s, t, a, o, 1, INT_MAX, 0, 0 }
+/** An item that must appear no more than once */
+#define T01(s,t,a,o)  { s, t, a, o, 0, 1, 0, 0 }
+/** An annotation that must appear no more than once */
+#define A01(s,t,a,o)  { s, t, a, o, 0, 1, 0, 1 }
+
+/** Argument multiplicity: any number of arguments. */
+#define ARGS        0,INT_MAX,0
+/** Argument multiplicity: no arguments. */
+#define NO_ARGS     0,0,0
+/** Argument multiplicity: concatenate all arguments. */
+#define CONCAT_ARGS 1,1,1
+/** Argument multiplicity: at least <b>n</b> arguments. */
+#define GE(n)       n,INT_MAX,0
+/** Argument multiplicity: exactly <b>n</b> arguments. */
+#define EQ(n)       n,n,0
+/**@}*/
+
+/** Determines the parsing rules for a single token type. */
+typedef struct token_rule_t {
+  /** The string value of the keyword identifying the type of item. */
+  const char *t;
+  /** The corresponding directory_keyword enum. */
+  directory_keyword v;
+  /** Minimum number of arguments for this item */
+  int min_args;
+  /** Maximum number of arguments for this item */
+  int max_args;
+  /** If true, we concatenate all arguments for this item into a single
+   * string. */
+  int concat_args;
+  /** Requirements on object syntax for this item. */
+  obj_syntax os;
+  /** Lowest number of times this item may appear in a document. */
+  int min_cnt;
+  /** Highest number of times this item may appear in a document. */
+  int max_cnt;
+  /** One or more of AT_START/AT_END to limit where the item may appear in a
+   * document. */
+  int pos;
+  /** True iff this token is an annotation. */
+  int is_annotation;
+} token_rule_t;
+
+void token_clear(directory_token_t *tok);
+
+int tokenize_string(struct memarea_t *area,
+                    const char *start, const char *end,
+                    struct smartlist_t *out,
+                    const token_rule_t *table,
+                    int flags);
+directory_token_t *get_next_token(struct memarea_t *area,
+                                  const char **s,
+                                  const char *eos,
+                                  const token_rule_t *table);
+
+directory_token_t *find_by_keyword_(struct smartlist_t *s,
+                                    directory_keyword keyword,
+                                    const char *keyword_str);
+
+#define find_by_keyword(s, keyword) \
+  find_by_keyword_((s), (keyword), #keyword)
+
+directory_token_t *find_opt_by_keyword(const struct smartlist_t *s,
+                                       directory_keyword keyword);
+struct smartlist_t * find_all_by_keyword(const struct smartlist_t *s,
+                                         directory_keyword k);
+
+#endif /* !defined(TOR_PARSECOMMON_H) */
diff --git a/src/feature/dirparse/policy_parse.c b/src/feature/dirparse/policy_parse.c
new file mode 100644
index 0000000000..7562ae409b
--- /dev/null
+++ b/src/feature/dirparse/policy_parse.c
@@ -0,0 +1,224 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file policy_parse.c
+ * \brief Code to parse address policies.
+ **/
+
+#define  EXPOSE_ROUTERDESC_TOKEN_TABLE
+
+#include "core/or/or.h"
+
+#include "core/or/policies.h"
+#include "feature/dirparse/parsecommon.h"
+#include "feature/dirparse/policy_parse.h"
+#include "feature/dirparse/routerparse.h"
+#include "feature/dirparse/unparseable.h"
+#include "lib/memarea/memarea.h"
+
+#include "core/or/addr_policy_st.h"
+
+static addr_policy_t *router_parse_addr_policy_private(directory_token_t *tok);
+
+/** Parse the addr policy in the string <b>s</b> and return it.  If
+ * assume_action is nonnegative, then insert its action (ADDR_POLICY_ACCEPT or
+ * ADDR_POLICY_REJECT) for items that specify no action.
+ *
+ * Returns NULL on policy errors.
+ *
+ * Set *<b>malformed_list</b> to true if the entire policy list should be
+ * discarded. Otherwise, set it to false, and only this item should be ignored
+ * on error - the rest of the policy list can continue to be processed and
+ * used.
+ *
+ * The addr_policy_t returned by this function can have its address set to
+ * AF_UNSPEC for '*'.  Use policy_expand_unspec() to turn this into a pair
+ * of AF_INET and AF_INET6 items.
+ */
+MOCK_IMPL(addr_policy_t *,
+router_parse_addr_policy_item_from_string,(const char *s, int assume_action,
+                                           int *malformed_list))
+{
+  directory_token_t *tok = NULL;
+  const char *cp, *eos;
+  /* Longest possible policy is
+   * "accept6 [ffff:ffff:..255]/128:10000-65535",
+   * which contains a max-length IPv6 address, plus 26 characters.
+   * But note that there can be an arbitrary amount of space between the
+   * accept and the address:mask/port element.
+   * We don't need to multiply TOR_ADDR_BUF_LEN by 2, as there is only one
+   * IPv6 address. But making the buffer shorter might cause valid long lines,
+   * which parsed in previous versions, to fail to parse in new versions.
+   * (These lines would have to have excessive amounts of whitespace.) */
+  char line[TOR_ADDR_BUF_LEN*2 + 32];
+  addr_policy_t *r;
+  memarea_t *area = NULL;
+
+  tor_assert(malformed_list);
+  *malformed_list = 0;
+
+  s = eat_whitespace(s);
+  /* We can only do assume_action on []-quoted IPv6, as "a" (accept)
+   * and ":" (port separator) are ambiguous */
+  if ((*s == '*' || *s == '[' || TOR_ISDIGIT(*s)) && assume_action >= 0) {
+    if (tor_snprintf(line, sizeof(line), "%s %s",
+               assume_action == ADDR_POLICY_ACCEPT?"accept":"reject", s)<0) {
+      log_warn(LD_DIR, "Policy %s is too long.", escaped(s));
+      return NULL;
+    }
+    cp = line;
+    tor_strlower(line);
+  } else { /* assume an already well-formed address policy line */
+    cp = s;
+  }
+
+  eos = cp + strlen(cp);
+  area = memarea_new();
+  tok = get_next_token(area, &cp, eos, routerdesc_token_table);
+  if (tok->tp == ERR_) {
+    log_warn(LD_DIR, "Error reading address policy: %s", tok->error);
+    goto err;
+  }
+  if (tok->tp != K_ACCEPT && tok->tp != K_ACCEPT6 &&
+      tok->tp != K_REJECT && tok->tp != K_REJECT6) {
+    log_warn(LD_DIR, "Expected 'accept' or 'reject'.");
+    goto err;
+  }
+
+  /* Use the extended interpretation of accept/reject *,
+   * expanding it into an IPv4 wildcard and an IPv6 wildcard.
+   * Also permit *4 and *6 for IPv4 and IPv6 only wildcards. */
+  r = router_parse_addr_policy(tok, TAPMP_EXTENDED_STAR);
+  if (!r) {
+    goto err;
+  }
+
+  /* Ensure that accept6/reject6 fields are followed by IPv6 addresses.
+   * AF_UNSPEC addresses are only permitted on the accept/reject field type.
+   * Unlike descriptors, torrcs exit policy accept/reject can be followed by
+   * either an IPv4 or IPv6 address. */
+  if ((tok->tp == K_ACCEPT6 || tok->tp == K_REJECT6) &&
+       tor_addr_family(&r->addr) != AF_INET6) {
+    /* This is a non-fatal error, just ignore this one entry. */
+    *malformed_list = 0;
+    log_warn(LD_DIR, "IPv4 address '%s' with accept6/reject6 field type in "
+             "exit policy. Ignoring, but continuing to parse rules. (Use "
+             "accept/reject with IPv4 addresses.)",
+             tok->n_args == 1 ? tok->args[0] : "");
+    addr_policy_free(r);
+    r = NULL;
+    goto done;
+  }
+
+  goto done;
+ err:
+  *malformed_list = 1;
+  r = NULL;
+ done:
+  token_clear(tok);
+  if (area) {
+    DUMP_AREA(area, "policy item");
+    memarea_drop_all(area);
+  }
+  return r;
+}
+
+/** Given a K_ACCEPT[6] or K_REJECT[6] token and a router, create and return
+ * a new exit_policy_t corresponding to the token. If TAPMP_EXTENDED_STAR
+ * is set in fmt_flags, K_ACCEPT6 and K_REJECT6 tokens followed by *
+ * expand to IPv6-only policies, otherwise they expand to IPv4 and IPv6
+ * policies */
+addr_policy_t *
+router_parse_addr_policy(directory_token_t *tok, unsigned fmt_flags)
+{
+  addr_policy_t newe;
+  char *arg;
+
+  tor_assert(tok->tp == K_REJECT || tok->tp == K_REJECT6 ||
+             tok->tp == K_ACCEPT || tok->tp == K_ACCEPT6);
+
+  if (tok->n_args != 1)
+    return NULL;
+  arg = tok->args[0];
+
+  if (!strcmpstart(arg,"private"))
+    return router_parse_addr_policy_private(tok);
+
+  memset(&newe, 0, sizeof(newe));
+
+  if (tok->tp == K_REJECT || tok->tp == K_REJECT6)
+    newe.policy_type = ADDR_POLICY_REJECT;
+  else
+    newe.policy_type = ADDR_POLICY_ACCEPT;
+
+  /* accept6/reject6 * produces an IPv6 wildcard address only.
+   * (accept/reject * produces rules for IPv4 and IPv6 wildcard addresses.) */
+  if ((fmt_flags & TAPMP_EXTENDED_STAR)
+      && (tok->tp == K_ACCEPT6 || tok->tp == K_REJECT6)) {
+    fmt_flags |= TAPMP_STAR_IPV6_ONLY;
+  }
+
+  if (tor_addr_parse_mask_ports(arg, fmt_flags, &newe.addr, &newe.maskbits,
+                                &newe.prt_min, &newe.prt_max) < 0) {
+    log_warn(LD_DIR,"Couldn't parse line %s. Dropping", escaped(arg));
+    return NULL;
+  }
+
+  addr_policy_t *result = addr_policy_get_canonical_entry(&newe);
+  /* It would be a nasty error to return 'newe', and sometimes
+   * addr_policy_get_canonical_entry() can return its argument.  But in this
+   * case, it won't, since newe is *not* canonical.  We assert here to make
+   * sure that the compiler knows it too.
+   */
+  tor_assert(result != &newe);
+  return result;
+}
+
+/** Parse an exit policy line of the format "accept[6]/reject[6] private:...".
+ * This didn't exist until Tor 0.1.1.15, so nobody should generate it in
+ * router descriptors until earlier versions are obsolete.
+ *
+ * accept/reject and accept6/reject6 private all produce rules for both
+ * IPv4 and IPv6 addresses.
+ */
+static addr_policy_t *
+router_parse_addr_policy_private(directory_token_t *tok)
+{
+  const char *arg;
+  uint16_t port_min, port_max;
+  addr_policy_t result;
+
+  arg = tok->args[0];
+  if (strcmpstart(arg, "private"))
+    return NULL;
+
+  arg += strlen("private");
+  arg = (char*) eat_whitespace(arg);
+  if (!arg || *arg != ':')
+    return NULL;
+
+  if (parse_port_range(arg+1, &port_min, &port_max)<0)
+    return NULL;
+
+  memset(&result, 0, sizeof(result));
+  if (tok->tp == K_REJECT || tok->tp == K_REJECT6)
+    result.policy_type = ADDR_POLICY_REJECT;
+  else
+    result.policy_type = ADDR_POLICY_ACCEPT;
+  result.is_private = 1;
+  result.prt_min = port_min;
+  result.prt_max = port_max;
+
+  if (tok->tp == K_ACCEPT6 || tok->tp == K_REJECT6) {
+    log_warn(LD_GENERAL,
+             "'%s' expands into rules which apply to all private IPv4 and "
+             "IPv6 addresses. (Use accept/reject private:* for IPv4 and "
+             "IPv6.)", tok->n_args == 1 ? tok->args[0] : "");
+  }
+
+  return addr_policy_get_canonical_entry(&result);
+}
diff --git a/src/feature/dirparse/policy_parse.h b/src/feature/dirparse/policy_parse.h
new file mode 100644
index 0000000000..e09ee5559f
--- /dev/null
+++ b/src/feature/dirparse/policy_parse.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file policy_parse.h
+ * \brief Header file for policy_parse.c.
+ **/
+
+#ifndef TOR_POLICY_PARSE_H
+#define TOR_POLICY_PARSE_H
+
+#include "lib/testsupport/testsupport.h"
+
+struct directory_token_t;
+
+MOCK_DECL(addr_policy_t *, router_parse_addr_policy_item_from_string,
+         (const char *s, int assume_action, int *malformed_list));
+
+addr_policy_t *router_parse_addr_policy(struct directory_token_t *tok,
+                                        unsigned fmt_flags);
+
+#endif /* !defined(TOR_POLICY_PARSE_H) */
diff --git a/src/feature/dirparse/routerparse.c b/src/feature/dirparse/routerparse.c
new file mode 100644
index 0000000000..e44fbf77f9
--- /dev/null
+++ b/src/feature/dirparse/routerparse.c
@@ -0,0 +1,1245 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file routerparse.c
+ * \brief Code to parse and validate router descriptors, consenus directories,
+ *   and similar objects.
+ *
+ * The objects parsed by this module use a common text-based metaformat,
+ * documented in dir-spec.txt in torspec.git.  This module is itself divided
+ * into two major kinds of function: code to handle the metaformat, and code
+ * to convert from particular instances of the metaformat into the
+ * objects that Tor uses.
+ *
+ * The generic parsing code works by calling a table-based tokenizer on the
+ * input string.  Each token corresponds to a single line with a token, plus
+ * optional arguments on that line, plus an optional base-64 encoded object
+ * after that line.  Each token has a definition in a table of token_rule_t
+ * entries that describes how many arguments it can take, whether it takes an
+ * object, how many times it may appear, whether it must appear first, and so
+ * on.
+ *
+ * The tokenizer function tokenize_string() converts its string input into a
+ * smartlist full of instances of directory_token_t, according to a provided
+ * table of token_rule_t.
+ *
+ * The generic parts of this module additionally include functions for
+ * finding the start and end of signed information inside a signed object, and
+ * computing the digest that will be signed.
+ *
+ * There are also functions for saving objects to disk that have caused
+ * parsing to fail.
+ *
+ * The specific parts of this module describe conversions between
+ * particular lists of directory_token_t and particular objects.  The
+ * kinds of objects that can be parsed here are:
+ *  <ul>
+ *  <li>router descriptors (managed from routerlist.c)
+ *  <li>extra-info documents (managed from routerlist.c)
+ *  <li>microdescriptors (managed from microdesc.c)
+ *  <li>vote and consensus networkstatus documents, and the routerstatus_t
+ *    objects that they comprise (managed from networkstatus.c)
+ *  <li>detached-signature objects used by authorities for gathering
+ *    signatures on the networkstatus consensus (managed from dirvote.c)
+ *  <li>authority key certificates (managed from routerlist.c)
+ *  <li>hidden service descriptors (managed from rendcommon.c and rendcache.c)
+ * </ul>
+ **/
+
+#define EXPOSE_ROUTERDESC_TOKEN_TABLE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/or/policies.h"
+#include "core/or/versions.h"
+#include "feature/dirparse/parsecommon.h"
+#include "feature/dirparse/policy_parse.h"
+#include "feature/dirparse/routerparse.h"
+#include "feature/dirparse/sigcommon.h"
+#include "feature/dirparse/unparseable.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/nickname.h"
+#include "feature/nodelist/routerinfo.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/torcert.h"
+#include "feature/relay/router.h"
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/memarea/memarea.h"
+#include "lib/sandbox/sandbox.h"
+
+#include "core/or/addr_policy_st.h"
+#include "feature/nodelist/extrainfo_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerlist_st.h"
+
+/****************************************************************************/
+
+/** List of tokens recognized in router descriptors */
+const token_rule_t routerdesc_token_table[] = {
+  T0N("reject",              K_REJECT,              ARGS,    NO_OBJ ),
+  T0N("accept",              K_ACCEPT,              ARGS,    NO_OBJ ),
+  T0N("reject6",             K_REJECT6,             ARGS,    NO_OBJ ),
+  T0N("accept6",             K_ACCEPT6,             ARGS,    NO_OBJ ),
+  T1_START( "router",        K_ROUTER,              GE(5),   NO_OBJ ),
+  T01("ipv6-policy",         K_IPV6_POLICY,         CONCAT_ARGS, NO_OBJ),
+  T1( "signing-key",         K_SIGNING_KEY,         NO_ARGS, NEED_KEY_1024 ),
+  T1( "onion-key",           K_ONION_KEY,           NO_ARGS, NEED_KEY_1024 ),
+  T01("ntor-onion-key",      K_ONION_KEY_NTOR,      GE(1), NO_OBJ ),
+  T1_END( "router-signature",    K_ROUTER_SIGNATURE,    NO_ARGS, NEED_OBJ ),
+  T1( "published",           K_PUBLISHED,       CONCAT_ARGS, NO_OBJ ),
+  T01("uptime",              K_UPTIME,              GE(1),   NO_OBJ ),
+  T01("fingerprint",         K_FINGERPRINT,     CONCAT_ARGS, NO_OBJ ),
+  T01("hibernating",         K_HIBERNATING,         GE(1),   NO_OBJ ),
+  T01("platform",            K_PLATFORM,        CONCAT_ARGS, NO_OBJ ),
+  T01("proto",               K_PROTO,           CONCAT_ARGS, NO_OBJ ),
+  T01("contact",             K_CONTACT,         CONCAT_ARGS, NO_OBJ ),
+  T01("read-history",        K_READ_HISTORY,        ARGS,    NO_OBJ ),
+  T01("write-history",       K_WRITE_HISTORY,       ARGS,    NO_OBJ ),
+  T01("extra-info-digest",   K_EXTRA_INFO_DIGEST,   GE(1),   NO_OBJ ),
+  T01("hidden-service-dir",  K_HIDDEN_SERVICE_DIR,  NO_ARGS, NO_OBJ ),
+  T01("identity-ed25519",    K_IDENTITY_ED25519,    NO_ARGS, NEED_OBJ ),
+  T01("master-key-ed25519",  K_MASTER_KEY_ED25519,  GE(1),   NO_OBJ ),
+  T01("router-sig-ed25519",  K_ROUTER_SIG_ED25519,  GE(1),   NO_OBJ ),
+  T01("onion-key-crosscert", K_ONION_KEY_CROSSCERT, NO_ARGS, NEED_OBJ ),
+  T01("ntor-onion-key-crosscert", K_NTOR_ONION_KEY_CROSSCERT,
+                                                    EQ(1),   NEED_OBJ ),
+
+  T01("allow-single-hop-exits",K_ALLOW_SINGLE_HOP_EXITS,    NO_ARGS, NO_OBJ ),
+
+  T01("family",              K_FAMILY,              ARGS,    NO_OBJ ),
+  T01("caches-extra-info",   K_CACHES_EXTRA_INFO,   NO_ARGS, NO_OBJ ),
+  T0N("or-address",          K_OR_ADDRESS,          GE(1),   NO_OBJ ),
+
+  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
+  T1( "bandwidth",           K_BANDWIDTH,           GE(3),   NO_OBJ ),
+  A01("@purpose",            A_PURPOSE,             GE(1),   NO_OBJ ),
+  T01("tunnelled-dir-server",K_DIR_TUNNELLED,       NO_ARGS, NO_OBJ ),
+
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in extra-info documents. */
+static token_rule_t extrainfo_token_table[] = {
+  T1_END( "router-signature",    K_ROUTER_SIGNATURE,    NO_ARGS, NEED_OBJ ),
+  T1( "published",           K_PUBLISHED,       CONCAT_ARGS, NO_OBJ ),
+  T01("identity-ed25519",    K_IDENTITY_ED25519,    NO_ARGS, NEED_OBJ ),
+  T01("router-sig-ed25519",  K_ROUTER_SIG_ED25519,  GE(1),   NO_OBJ ),
+  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
+  T01("read-history",        K_READ_HISTORY,        ARGS,    NO_OBJ ),
+  T01("write-history",       K_WRITE_HISTORY,       ARGS,    NO_OBJ ),
+  T01("dirreq-stats-end",    K_DIRREQ_END,          ARGS,    NO_OBJ ),
+  T01("dirreq-v2-ips",       K_DIRREQ_V2_IPS,       ARGS,    NO_OBJ ),
+  T01("dirreq-v3-ips",       K_DIRREQ_V3_IPS,       ARGS,    NO_OBJ ),
+  T01("dirreq-v2-reqs",      K_DIRREQ_V2_REQS,      ARGS,    NO_OBJ ),
+  T01("dirreq-v3-reqs",      K_DIRREQ_V3_REQS,      ARGS,    NO_OBJ ),
+  T01("dirreq-v2-share",     K_DIRREQ_V2_SHARE,     ARGS,    NO_OBJ ),
+  T01("dirreq-v3-share",     K_DIRREQ_V3_SHARE,     ARGS,    NO_OBJ ),
+  T01("dirreq-v2-resp",      K_DIRREQ_V2_RESP,      ARGS,    NO_OBJ ),
+  T01("dirreq-v3-resp",      K_DIRREQ_V3_RESP,      ARGS,    NO_OBJ ),
+  T01("dirreq-v2-direct-dl", K_DIRREQ_V2_DIR,       ARGS,    NO_OBJ ),
+  T01("dirreq-v3-direct-dl", K_DIRREQ_V3_DIR,       ARGS,    NO_OBJ ),
+  T01("dirreq-v2-tunneled-dl", K_DIRREQ_V2_TUN,     ARGS,    NO_OBJ ),
+  T01("dirreq-v3-tunneled-dl", K_DIRREQ_V3_TUN,     ARGS,    NO_OBJ ),
+  T01("entry-stats-end",     K_ENTRY_END,           ARGS,    NO_OBJ ),
+  T01("entry-ips",           K_ENTRY_IPS,           ARGS,    NO_OBJ ),
+  T01("cell-stats-end",      K_CELL_END,            ARGS,    NO_OBJ ),
+  T01("cell-processed-cells", K_CELL_PROCESSED,     ARGS,    NO_OBJ ),
+  T01("cell-queued-cells",   K_CELL_QUEUED,         ARGS,    NO_OBJ ),
+  T01("cell-time-in-queue",  K_CELL_TIME,           ARGS,    NO_OBJ ),
+  T01("cell-circuits-per-decile", K_CELL_CIRCS,     ARGS,    NO_OBJ ),
+  T01("exit-stats-end",      K_EXIT_END,            ARGS,    NO_OBJ ),
+  T01("exit-kibibytes-written", K_EXIT_WRITTEN,     ARGS,    NO_OBJ ),
+  T01("exit-kibibytes-read", K_EXIT_READ,           ARGS,    NO_OBJ ),
+  T01("exit-streams-opened", K_EXIT_OPENED,         ARGS,    NO_OBJ ),
+
+  T1_START( "extra-info",          K_EXTRA_INFO,          GE(2),   NO_OBJ ),
+
+  END_OF_TABLE
+};
+
+#undef T
+
+/* static function prototypes */
+static int router_add_exit_policy(routerinfo_t *router,directory_token_t *tok);
+static smartlist_t *find_all_exitpolicy(smartlist_t *s);
+
+/** Set <b>digest</b> to the SHA-1 digest of the hash of the first router in
+ * <b>s</b>. Return 0 on success, -1 on failure.
+ */
+int
+router_get_router_hash(const char *s, size_t s_len, char *digest)
+{
+  return router_get_hash_impl(s, s_len, digest,
+                              "router ","\nrouter-signature", '\n',
+                              DIGEST_SHA1);
+}
+
+/** Set <b>digest</b> to the SHA-1 digest of the hash of the <b>s_len</b>-byte
+ * extrainfo string at <b>s</b>.  Return 0 on success, -1 on failure. */
+int
+router_get_extrainfo_hash(const char *s, size_t s_len, char *digest)
+{
+  return router_get_hash_impl(s, s_len, digest, "extra-info",
+                              "\nrouter-signature",'\n', DIGEST_SHA1);
+}
+
+/** Helper: move *<b>s_ptr</b> ahead to the next router, the next extra-info,
+ * or to the first of the annotations proceeding the next router or
+ * extra-info---whichever comes first.  Set <b>is_extrainfo_out</b> to true if
+ * we found an extrainfo, or false if found a router. Do not scan beyond
+ * <b>eos</b>.  Return -1 if we found nothing; 0 if we found something. */
+static int
+find_start_of_next_router_or_extrainfo(const char **s_ptr,
+                                       const char *eos,
+                                       int *is_extrainfo_out)
+{
+  const char *annotations = NULL;
+  const char *s = *s_ptr;
+
+  s = eat_whitespace_eos(s, eos);
+
+  while (s < eos-32) {  /* 32 gives enough room for a the first keyword. */
+    /* We're at the start of a line. */
+    tor_assert(*s != '\n');
+
+    if (*s == '@' && !annotations) {
+      annotations = s;
+    } else if (*s == 'r' && !strcmpstart(s, "router ")) {
+      *s_ptr = annotations ? annotations : s;
+      *is_extrainfo_out = 0;
+      return 0;
+    } else if (*s == 'e' && !strcmpstart(s, "extra-info ")) {
+      *s_ptr = annotations ? annotations : s;
+      *is_extrainfo_out = 1;
+      return 0;
+    }
+
+    if (!(s = memchr(s+1, '\n', eos-(s+1))))
+      break;
+    s = eat_whitespace_eos(s, eos);
+  }
+  return -1;
+}
+
+/** Given a string *<b>s</b> containing a concatenated sequence of router
+ * descriptors (or extra-info documents if <b>want_extrainfo</b> is set),
+ * parses them and stores the result in <b>dest</b>. All routers are marked
+ * running and valid. Advances *s to a point immediately following the last
+ * router entry. Ignore any trailing router entries that are not complete.
+ *
+ * If <b>saved_location</b> isn't SAVED_IN_CACHE, make a local copy of each
+ * descriptor in the signed_descriptor_body field of each routerinfo_t.  If it
+ * isn't SAVED_NOWHERE, remember the offset of each descriptor.
+ *
+ * Returns 0 on success and -1 on failure.  Adds a digest to
+ * <b>invalid_digests_out</b> for every entry that was unparseable or
+ * invalid. (This may cause duplicate entries.)
+ */
+int
+router_parse_list_from_string(const char **s, const char *eos,
+                              smartlist_t *dest,
+                              saved_location_t saved_location,
+                              int want_extrainfo,
+                              int allow_annotations,
+                              const char *prepend_annotations,
+                              smartlist_t *invalid_digests_out)
+{
+  routerinfo_t *router;
+  extrainfo_t *extrainfo;
+  signed_descriptor_t *signed_desc = NULL;
+  void *elt;
+  const char *end, *start;
+  int have_extrainfo;
+
+  tor_assert(s);
+  tor_assert(*s);
+  tor_assert(dest);
+
+  start = *s;
+  if (!eos)
+    eos = *s + strlen(*s);
+
+  tor_assert(eos >= *s);
+
+  while (1) {
+    char raw_digest[DIGEST_LEN];
+    int have_raw_digest = 0;
+    int dl_again = 0;
+    if (find_start_of_next_router_or_extrainfo(s, eos, &have_extrainfo) < 0)
+      break;
+
+    end = tor_memstr(*s, eos-*s, "\nrouter-signature");
+    if (end)
+      end = tor_memstr(end, eos-end, "\n-----END SIGNATURE-----\n");
+    if (end)
+      end += strlen("\n-----END SIGNATURE-----\n");
+
+    if (!end)
+      break;
+
+    elt = NULL;
+
+    if (have_extrainfo && want_extrainfo) {
+      routerlist_t *rl = router_get_routerlist();
+      have_raw_digest = router_get_extrainfo_hash(*s, end-*s, raw_digest) == 0;
+      extrainfo = extrainfo_parse_entry_from_string(*s, end,
+                                       saved_location != SAVED_IN_CACHE,
+                                       rl->identity_map, &dl_again);
+      if (extrainfo) {
+        signed_desc = &extrainfo->cache_info;
+        elt = extrainfo;
+      }
+    } else if (!have_extrainfo && !want_extrainfo) {
+      have_raw_digest = router_get_router_hash(*s, end-*s, raw_digest) == 0;
+      router = router_parse_entry_from_string(*s, end,
+                                              saved_location != SAVED_IN_CACHE,
+                                              allow_annotations,
+                                              prepend_annotations, &dl_again);
+      if (router) {
+        log_debug(LD_DIR, "Read router '%s', purpose '%s'",
+                  router_describe(router),
+                  router_purpose_to_string(router->purpose));
+        signed_desc = &router->cache_info;
+        elt = router;
+      }
+    }
+    if (! elt && ! dl_again && have_raw_digest && invalid_digests_out) {
+      smartlist_add(invalid_digests_out, tor_memdup(raw_digest, DIGEST_LEN));
+    }
+    if (!elt) {
+      *s = end;
+      continue;
+    }
+    if (saved_location != SAVED_NOWHERE) {
+      tor_assert(signed_desc);
+      signed_desc->saved_location = saved_location;
+      signed_desc->saved_offset = *s - start;
+    }
+    *s = end;
+    smartlist_add(dest, elt);
+  }
+
+  return 0;
+}
+
+/** Try to find an IPv6 OR port in <b>list</b> of directory_token_t's
+ * with at least one argument (use GE(1) in setup). If found, store
+ * address and port number to <b>addr_out</b> and
+ * <b>port_out</b>. Return number of OR ports found. */
+int
+find_single_ipv6_orport(const smartlist_t *list,
+                        tor_addr_t *addr_out,
+                        uint16_t *port_out)
+{
+  int ret = 0;
+  tor_assert(list != NULL);
+  tor_assert(addr_out != NULL);
+  tor_assert(port_out != NULL);
+
+  SMARTLIST_FOREACH_BEGIN(list, directory_token_t *, t) {
+    tor_addr_t a;
+    maskbits_t bits;
+    uint16_t port_min, port_max;
+    tor_assert(t->n_args >= 1);
+    /* XXXX Prop186 the full spec allows much more than this. */
+    if (tor_addr_parse_mask_ports(t->args[0], 0,
+                                  &a, &bits, &port_min,
+                                  &port_max) == AF_INET6 &&
+        bits == 128 &&
+        port_min == port_max) {
+      /* Okay, this is one we can understand. Use it and ignore
+         any potential more addresses in list. */
+      tor_addr_copy(addr_out, &a);
+      *port_out = port_min;
+      ret = 1;
+      break;
+    }
+  } SMARTLIST_FOREACH_END(t);
+
+  return ret;
+}
+
+/** Helper function: reads a single router entry from *<b>s</b> ...
+ * *<b>end</b>.  Mallocs a new router and returns it if all goes well, else
+ * returns NULL.  If <b>cache_copy</b> is true, duplicate the contents of
+ * s through end into the signed_descriptor_body of the resulting
+ * routerinfo_t.
+ *
+ * If <b>end</b> is NULL, <b>s</b> must be properly NUL-terminated.
+ *
+ * If <b>allow_annotations</b>, it's okay to encounter annotations in <b>s</b>
+ * before the router; if it's false, reject the router if it's annotated.  If
+ * <b>prepend_annotations</b> is set, it should contain some annotations:
+ * append them to the front of the router before parsing it, and keep them
+ * around when caching the router.
+ *
+ * Only one of allow_annotations and prepend_annotations may be set.
+ *
+ * If <b>can_dl_again_out</b> is provided, set *<b>can_dl_again_out</b> to 1
+ * if it's okay to try to download a descriptor with this same digest again,
+ * and 0 if it isn't.  (It might not be okay to download it again if part of
+ * the part covered by the digest is invalid.)
+ */
+routerinfo_t *
+router_parse_entry_from_string(const char *s, const char *end,
+                               int cache_copy, int allow_annotations,
+                               const char *prepend_annotations,
+                               int *can_dl_again_out)
+{
+  routerinfo_t *router = NULL;
+  char digest[128];
+  smartlist_t *tokens = NULL, *exit_policy_tokens = NULL;
+  directory_token_t *tok;
+  struct in_addr in;
+  const char *start_of_annotations, *cp, *s_dup = s;
+  size_t prepend_len = prepend_annotations ? strlen(prepend_annotations) : 0;
+  int ok = 1;
+  memarea_t *area = NULL;
+  tor_cert_t *ntor_cc_cert = NULL;
+  /* Do not set this to '1' until we have parsed everything that we intend to
+   * parse that's covered by the hash. */
+  int can_dl_again = 0;
+  crypto_pk_t *rsa_pubkey = NULL;
+
+  tor_assert(!allow_annotations || !prepend_annotations);
+
+  if (!end) {
+    end = s + strlen(s);
+  }
+
+  /* point 'end' to a point immediately after the final newline. */
+  while (end > s+2 && *(end-1) == '\n' && *(end-2) == '\n')
+    --end;
+
+  area = memarea_new();
+  tokens = smartlist_new();
+  if (prepend_annotations) {
+    if (tokenize_string(area,prepend_annotations,NULL,tokens,
+                        routerdesc_token_table,TS_NOCHECK)) {
+      log_warn(LD_DIR, "Error tokenizing router descriptor (annotations).");
+      goto err;
+    }
+  }
+
+  start_of_annotations = s;
+  cp = tor_memstr(s, end-s, "\nrouter ");
+  if (!cp) {
+    if (end-s < 7 || strcmpstart(s, "router ")) {
+      log_warn(LD_DIR, "No router keyword found.");
+      goto err;
+    }
+  } else {
+    s = cp+1;
+  }
+
+  if (start_of_annotations != s) { /* We have annotations */
+    if (allow_annotations) {
+      if (tokenize_string(area,start_of_annotations,s,tokens,
+                          routerdesc_token_table,TS_NOCHECK)) {
+        log_warn(LD_DIR, "Error tokenizing router descriptor (annotations).");
+        goto err;
+      }
+    } else {
+      log_warn(LD_DIR, "Found unexpected annotations on router descriptor not "
+               "loaded from disk.  Dropping it.");
+      goto err;
+    }
+  }
+
+  if (router_get_router_hash(s, end - s, digest) < 0) {
+    log_warn(LD_DIR, "Couldn't compute router hash.");
+    goto err;
+  }
+  {
+    int flags = 0;
+    if (allow_annotations)
+      flags |= TS_ANNOTATIONS_OK;
+    if (prepend_annotations)
+      flags |= TS_ANNOTATIONS_OK|TS_NO_NEW_ANNOTATIONS;
+
+    if (tokenize_string(area,s,end,tokens,routerdesc_token_table, flags)) {
+      log_warn(LD_DIR, "Error tokenizing router descriptor.");
+      goto err;
+    }
+  }
+
+  if (smartlist_len(tokens) < 2) {
+    log_warn(LD_DIR, "Impossibly short router descriptor.");
+    goto err;
+  }
+
+  tok = find_by_keyword(tokens, K_ROUTER);
+  const int router_token_pos = smartlist_pos(tokens, tok);
+  tor_assert(tok->n_args >= 5);
+
+  router = tor_malloc_zero(sizeof(routerinfo_t));
+  router->cert_expiration_time = TIME_MAX;
+  router->cache_info.routerlist_index = -1;
+  router->cache_info.annotations_len = s-start_of_annotations + prepend_len;
+  router->cache_info.signed_descriptor_len = end-s;
+  if (cache_copy) {
+    size_t len = router->cache_info.signed_descriptor_len +
+                 router->cache_info.annotations_len;
+    char *signed_body =
+      router->cache_info.signed_descriptor_body = tor_malloc(len+1);
+    if (prepend_annotations) {
+      memcpy(signed_body, prepend_annotations, prepend_len);
+      signed_body += prepend_len;
+    }
+    /* This assertion will always succeed.
+     * len == signed_desc_len + annotations_len
+     *     == end-s + s-start_of_annotations + prepend_len
+     *     == end-start_of_annotations + prepend_len
+     * We already wrote prepend_len bytes into the buffer; now we're
+     * writing end-start_of_annotations -NM. */
+    tor_assert(signed_body+(end-start_of_annotations) ==
+               router->cache_info.signed_descriptor_body+len);
+    memcpy(signed_body, start_of_annotations, end-start_of_annotations);
+    router->cache_info.signed_descriptor_body[len] = '\0';
+    tor_assert(strlen(router->cache_info.signed_descriptor_body) == len);
+  }
+  memcpy(router->cache_info.signed_descriptor_digest, digest, DIGEST_LEN);
+
+  router->nickname = tor_strdup(tok->args[0]);
+  if (!is_legal_nickname(router->nickname)) {
+    log_warn(LD_DIR,"Router nickname is invalid");
+    goto err;
+  }
+  if (!tor_inet_aton(tok->args[1], &in)) {
+    log_warn(LD_DIR,"Router address is not an IP address.");
+    goto err;
+  }
+  router->addr = ntohl(in.s_addr);
+
+  router->or_port =
+    (uint16_t) tor_parse_long(tok->args[2],10,0,65535,&ok,NULL);
+  if (!ok) {
+    log_warn(LD_DIR,"Invalid OR port %s", escaped(tok->args[2]));
+    goto err;
+  }
+  router->dir_port =
+    (uint16_t) tor_parse_long(tok->args[4],10,0,65535,&ok,NULL);
+  if (!ok) {
+    log_warn(LD_DIR,"Invalid dir port %s", escaped(tok->args[4]));
+    goto err;
+  }
+
+  tok = find_by_keyword(tokens, K_BANDWIDTH);
+  tor_assert(tok->n_args >= 3);
+  router->bandwidthrate = (int)
+    tor_parse_long(tok->args[0],10,1,INT_MAX,&ok,NULL);
+
+  if (!ok) {
+    log_warn(LD_DIR, "bandwidthrate %s unreadable or 0. Failing.",
+             escaped(tok->args[0]));
+    goto err;
+  }
+  router->bandwidthburst =
+    (int) tor_parse_long(tok->args[1],10,0,INT_MAX,&ok,NULL);
+  if (!ok) {
+    log_warn(LD_DIR, "Invalid bandwidthburst %s", escaped(tok->args[1]));
+    goto err;
+  }
+  router->bandwidthcapacity = (int)
+    tor_parse_long(tok->args[2],10,0,INT_MAX,&ok,NULL);
+  if (!ok) {
+    log_warn(LD_DIR, "Invalid bandwidthcapacity %s", escaped(tok->args[1]));
+    goto err;
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, A_PURPOSE))) {
+    tor_assert(tok->n_args);
+    router->purpose = router_purpose_from_string(tok->args[0]);
+    if (router->purpose == ROUTER_PURPOSE_UNKNOWN) {
+      goto err;
+    }
+  } else {
+    router->purpose = ROUTER_PURPOSE_GENERAL;
+  }
+  router->cache_info.send_unencrypted =
+    (router->purpose == ROUTER_PURPOSE_GENERAL) ? 1 : 0;
+
+  if ((tok = find_opt_by_keyword(tokens, K_UPTIME))) {
+    tor_assert(tok->n_args >= 1);
+    router->uptime = tor_parse_long(tok->args[0],10,0,LONG_MAX,&ok,NULL);
+    if (!ok) {
+      log_warn(LD_DIR, "Invalid uptime %s", escaped(tok->args[0]));
+      goto err;
+    }
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_HIBERNATING))) {
+    tor_assert(tok->n_args >= 1);
+    router->is_hibernating
+      = (tor_parse_long(tok->args[0],10,0,LONG_MAX,NULL,NULL) != 0);
+  }
+
+  tok = find_by_keyword(tokens, K_PUBLISHED);
+  tor_assert(tok->n_args == 1);
+  if (parse_iso_time(tok->args[0], &router->cache_info.published_on) < 0)
+    goto err;
+
+  tok = find_by_keyword(tokens, K_ONION_KEY);
+  if (!crypto_pk_public_exponent_ok(tok->key)) {
+    log_warn(LD_DIR,
+             "Relay's onion key had invalid exponent.");
+    goto err;
+  }
+  router_set_rsa_onion_pkey(tok->key, &router->onion_pkey,
+                            &router->onion_pkey_len);
+  crypto_pk_free(tok->key);
+
+  if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) {
+    curve25519_public_key_t k;
+    tor_assert(tok->n_args >= 1);
+    if (curve25519_public_from_base64(&k, tok->args[0]) < 0) {
+      log_warn(LD_DIR, "Bogus ntor-onion-key in routerinfo");
+      goto err;
+    }
+    router->onion_curve25519_pkey =
+      tor_memdup(&k, sizeof(curve25519_public_key_t));
+  }
+
+  tok = find_by_keyword(tokens, K_SIGNING_KEY);
+  router->identity_pkey = tok->key;
+  tok->key = NULL; /* Prevent free */
+  if (crypto_pk_get_digest(router->identity_pkey,
+                           router->cache_info.identity_digest)) {
+    log_warn(LD_DIR, "Couldn't calculate key digest"); goto err;
+  }
+
+  {
+    directory_token_t *ed_sig_tok, *ed_cert_tok, *cc_tap_tok, *cc_ntor_tok,
+      *master_key_tok;
+    ed_sig_tok = find_opt_by_keyword(tokens, K_ROUTER_SIG_ED25519);
+    ed_cert_tok = find_opt_by_keyword(tokens, K_IDENTITY_ED25519);
+    master_key_tok = find_opt_by_keyword(tokens, K_MASTER_KEY_ED25519);
+    cc_tap_tok = find_opt_by_keyword(tokens, K_ONION_KEY_CROSSCERT);
+    cc_ntor_tok = find_opt_by_keyword(tokens, K_NTOR_ONION_KEY_CROSSCERT);
+    int n_ed_toks = !!ed_sig_tok + !!ed_cert_tok +
+      !!cc_tap_tok + !!cc_ntor_tok;
+    if ((n_ed_toks != 0 && n_ed_toks != 4) ||
+        (n_ed_toks == 4 && !router->onion_curve25519_pkey)) {
+      log_warn(LD_DIR, "Router descriptor with only partial ed25519/"
+               "cross-certification support");
+      goto err;
+    }
+    if (master_key_tok && !ed_sig_tok) {
+      log_warn(LD_DIR, "Router descriptor has ed25519 master key but no "
+               "certificate");
+      goto err;
+    }
+    if (ed_sig_tok) {
+      tor_assert(ed_cert_tok && cc_tap_tok && cc_ntor_tok);
+      const int ed_cert_token_pos = smartlist_pos(tokens, ed_cert_tok);
+      if (ed_cert_token_pos == -1 || router_token_pos == -1 ||
+          (ed_cert_token_pos != router_token_pos + 1 &&
+           ed_cert_token_pos != router_token_pos - 1)) {
+        log_warn(LD_DIR, "Ed25519 certificate in wrong position");
+        goto err;
+      }
+      if (ed_sig_tok != smartlist_get(tokens, smartlist_len(tokens)-2)) {
+        log_warn(LD_DIR, "Ed25519 signature in wrong position");
+        goto err;
+      }
+      if (strcmp(ed_cert_tok->object_type, "ED25519 CERT")) {
+        log_warn(LD_DIR, "Wrong object type on identity-ed25519 in decriptor");
+        goto err;
+      }
+      if (strcmp(cc_ntor_tok->object_type, "ED25519 CERT")) {
+        log_warn(LD_DIR, "Wrong object type on ntor-onion-key-crosscert "
+                 "in decriptor");
+        goto err;
+      }
+      if (strcmp(cc_tap_tok->object_type, "CROSSCERT")) {
+        log_warn(LD_DIR, "Wrong object type on onion-key-crosscert "
+                 "in decriptor");
+        goto err;
+      }
+      if (strcmp(cc_ntor_tok->args[0], "0") &&
+          strcmp(cc_ntor_tok->args[0], "1")) {
+        log_warn(LD_DIR, "Bad sign bit on ntor-onion-key-crosscert");
+        goto err;
+      }
+      int ntor_cc_sign_bit = !strcmp(cc_ntor_tok->args[0], "1");
+
+      uint8_t d256[DIGEST256_LEN];
+      const char *signed_start, *signed_end;
+      tor_cert_t *cert = tor_cert_parse(
+                       (const uint8_t*)ed_cert_tok->object_body,
+                       ed_cert_tok->object_size);
+      if (! cert) {
+        log_warn(LD_DIR, "Couldn't parse ed25519 cert");
+        goto err;
+      }
+      /* makes sure it gets freed. */
+      router->cache_info.signing_key_cert = cert;
+
+      if (cert->cert_type != CERT_TYPE_ID_SIGNING ||
+          ! cert->signing_key_included) {
+        log_warn(LD_DIR, "Invalid form for ed25519 cert");
+        goto err;
+      }
+
+      if (master_key_tok) {
+        /* This token is optional, but if it's present, it must match
+         * the signature in the signing cert, or supplant it. */
+        tor_assert(master_key_tok->n_args >= 1);
+        ed25519_public_key_t pkey;
+        if (ed25519_public_from_base64(&pkey, master_key_tok->args[0])<0) {
+          log_warn(LD_DIR, "Can't parse ed25519 master key");
+          goto err;
+        }
+
+        if (fast_memneq(&cert->signing_key.pubkey,
+                        pkey.pubkey, ED25519_PUBKEY_LEN)) {
+          log_warn(LD_DIR, "Ed25519 master key does not match "
+                   "key in certificate");
+          goto err;
+        }
+      }
+      ntor_cc_cert = tor_cert_parse((const uint8_t*)cc_ntor_tok->object_body,
+                                    cc_ntor_tok->object_size);
+      if (!ntor_cc_cert) {
+        log_warn(LD_DIR, "Couldn't parse ntor-onion-key-crosscert cert");
+        goto err;
+      }
+      if (ntor_cc_cert->cert_type != CERT_TYPE_ONION_ID ||
+          ! ed25519_pubkey_eq(&ntor_cc_cert->signed_key, &cert->signing_key)) {
+        log_warn(LD_DIR, "Invalid contents for ntor-onion-key-crosscert cert");
+        goto err;
+      }
+
+      ed25519_public_key_t ntor_cc_pk;
+      if (ed25519_public_key_from_curve25519_public_key(&ntor_cc_pk,
+                                            router->onion_curve25519_pkey,
+                                            ntor_cc_sign_bit)<0) {
+        log_warn(LD_DIR, "Error converting onion key to ed25519");
+        goto err;
+      }
+
+      if (router_get_hash_impl_helper(s, end-s, "router ",
+                                      "\nrouter-sig-ed25519",
+                                      ' ', LOG_WARN,
+                                      &signed_start, &signed_end) < 0) {
+        log_warn(LD_DIR, "Can't find ed25519-signed portion of descriptor");
+        goto err;
+      }
+      crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA256);
+      crypto_digest_add_bytes(d, ED_DESC_SIGNATURE_PREFIX,
+        strlen(ED_DESC_SIGNATURE_PREFIX));
+      crypto_digest_add_bytes(d, signed_start, signed_end-signed_start);
+      crypto_digest_get_digest(d, (char*)d256, sizeof(d256));
+      crypto_digest_free(d);
+
+      ed25519_checkable_t check[3];
+      int check_ok[3];
+      time_t expires = TIME_MAX;
+      if (tor_cert_get_checkable_sig(&check[0], cert, NULL, &expires) < 0) {
+        log_err(LD_BUG, "Couldn't create 'checkable' for cert.");
+        goto err;
+      }
+      if (tor_cert_get_checkable_sig(&check[1],
+                               ntor_cc_cert, &ntor_cc_pk, &expires) < 0) {
+        log_err(LD_BUG, "Couldn't create 'checkable' for ntor_cc_cert.");
+        goto err;
+      }
+
+      if (ed25519_signature_from_base64(&check[2].signature,
+                                        ed_sig_tok->args[0])<0) {
+        log_warn(LD_DIR, "Couldn't decode ed25519 signature");
+        goto err;
+      }
+      check[2].pubkey = &cert->signed_key;
+      check[2].msg = d256;
+      check[2].len = DIGEST256_LEN;
+
+      if (ed25519_checksig_batch(check_ok, check, 3) < 0) {
+        log_warn(LD_DIR, "Incorrect ed25519 signature(s)");
+        goto err;
+      }
+
+      rsa_pubkey = router_get_rsa_onion_pkey(router->onion_pkey,
+                                             router->onion_pkey_len);
+      if (check_tap_onion_key_crosscert(
+                      (const uint8_t*)cc_tap_tok->object_body,
+                      (int)cc_tap_tok->object_size,
+                      rsa_pubkey,
+                      &cert->signing_key,
+                      (const uint8_t*)router->cache_info.identity_digest)<0) {
+        log_warn(LD_DIR, "Incorrect TAP cross-verification");
+        goto err;
+      }
+
+      /* We check this before adding it to the routerlist. */
+      router->cert_expiration_time = expires;
+    }
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_FINGERPRINT))) {
+    /* If there's a fingerprint line, it must match the identity digest. */
+    char d[DIGEST_LEN];
+    tor_assert(tok->n_args == 1);
+    tor_strstrip(tok->args[0], " ");
+    if (base16_decode(d, DIGEST_LEN,
+                      tok->args[0], strlen(tok->args[0])) != DIGEST_LEN) {
+      log_warn(LD_DIR, "Couldn't decode router fingerprint %s",
+               escaped(tok->args[0]));
+      goto err;
+    }
+    if (tor_memneq(d,router->cache_info.identity_digest, DIGEST_LEN)) {
+      log_warn(LD_DIR, "Fingerprint '%s' does not match identity digest.",
+               tok->args[0]);
+      goto err;
+    }
+  }
+
+  {
+    const char *version = NULL, *protocols = NULL;
+    if ((tok = find_opt_by_keyword(tokens, K_PLATFORM))) {
+      router->platform = tor_strdup(tok->args[0]);
+      version = tok->args[0];
+    }
+
+    if ((tok = find_opt_by_keyword(tokens, K_PROTO))) {
+      router->protocol_list = tor_strdup(tok->args[0]);
+      protocols = tok->args[0];
+    }
+
+    summarize_protover_flags(&router->pv, protocols, version);
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_CONTACT))) {
+    router->contact_info = tor_strdup(tok->args[0]);
+  }
+
+  if (find_opt_by_keyword(tokens, K_REJECT6) ||
+      find_opt_by_keyword(tokens, K_ACCEPT6)) {
+    log_warn(LD_DIR, "Rejecting router with reject6/accept6 line: they crash "
+             "older Tors.");
+    goto err;
+  }
+  {
+    smartlist_t *or_addresses = find_all_by_keyword(tokens, K_OR_ADDRESS);
+    if (or_addresses) {
+      find_single_ipv6_orport(or_addresses, &router->ipv6_addr,
+                              &router->ipv6_orport);
+      smartlist_free(or_addresses);
+    }
+  }
+  exit_policy_tokens = find_all_exitpolicy(tokens);
+  if (!smartlist_len(exit_policy_tokens)) {
+    log_warn(LD_DIR, "No exit policy tokens in descriptor.");
+    goto err;
+  }
+  SMARTLIST_FOREACH(exit_policy_tokens, directory_token_t *, t,
+                    if (router_add_exit_policy(router,t)<0) {
+                      log_warn(LD_DIR,"Error in exit policy");
+                      goto err;
+                    });
+  policy_expand_private(&router->exit_policy);
+
+  if ((tok = find_opt_by_keyword(tokens, K_IPV6_POLICY)) && tok->n_args) {
+    router->ipv6_exit_policy = parse_short_policy(tok->args[0]);
+    if (! router->ipv6_exit_policy) {
+      log_warn(LD_DIR , "Error in ipv6-policy %s", escaped(tok->args[0]));
+      goto err;
+    }
+  }
+
+  if (policy_is_reject_star(router->exit_policy, AF_INET, 1) &&
+      (!router->ipv6_exit_policy ||
+       short_policy_is_reject_star(router->ipv6_exit_policy)))
+    router->policy_is_reject_star = 1;
+
+  if ((tok = find_opt_by_keyword(tokens, K_FAMILY)) && tok->n_args) {
+    int i;
+    router->declared_family = smartlist_new();
+    for (i=0;i<tok->n_args;++i) {
+      if (!is_legal_nickname_or_hexdigest(tok->args[i])) {
+        log_warn(LD_DIR, "Illegal nickname %s in family line",
+                 escaped(tok->args[i]));
+        goto err;
+      }
+      smartlist_add_strdup(router->declared_family, tok->args[i]);
+    }
+  }
+
+  if (find_opt_by_keyword(tokens, K_CACHES_EXTRA_INFO))
+    router->caches_extra_info = 1;
+
+  if (find_opt_by_keyword(tokens, K_ALLOW_SINGLE_HOP_EXITS))
+    router->allow_single_hop_exits = 1;
+
+  if ((tok = find_opt_by_keyword(tokens, K_EXTRA_INFO_DIGEST))) {
+    tor_assert(tok->n_args >= 1);
+    if (strlen(tok->args[0]) == HEX_DIGEST_LEN) {
+      if (base16_decode(router->cache_info.extra_info_digest, DIGEST_LEN,
+                        tok->args[0], HEX_DIGEST_LEN) != DIGEST_LEN) {
+          log_warn(LD_DIR,"Invalid extra info digest");
+      }
+    } else {
+      log_warn(LD_DIR, "Invalid extra info digest %s", escaped(tok->args[0]));
+    }
+
+    if (tok->n_args >= 2) {
+      if (digest256_from_base64(router->cache_info.extra_info_digest256,
+                                tok->args[1]) < 0) {
+        log_warn(LD_DIR, "Invalid extra info digest256 %s",
+                 escaped(tok->args[1]));
+      }
+    }
+  }
+
+  if (find_opt_by_keyword(tokens, K_HIDDEN_SERVICE_DIR)) {
+    router->wants_to_be_hs_dir = 1;
+  }
+
+  /* This router accepts tunnelled directory requests via begindir if it has
+   * an open dirport or it included "tunnelled-dir-server". */
+  if (find_opt_by_keyword(tokens, K_DIR_TUNNELLED) || router->dir_port > 0) {
+    router->supports_tunnelled_dir_requests = 1;
+  }
+
+  tok = find_by_keyword(tokens, K_ROUTER_SIGNATURE);
+
+  if (!router->or_port) {
+    log_warn(LD_DIR,"or_port unreadable or 0. Failing.");
+    goto err;
+  }
+
+  /* We've checked everything that's covered by the hash. */
+  can_dl_again = 1;
+  if (check_signature_token(digest, DIGEST_LEN, tok, router->identity_pkey, 0,
+                            "router descriptor") < 0)
+    goto err;
+
+  if (!router->platform) {
+    router->platform = tor_strdup("<unknown>");
+  }
+  goto done;
+
+ err:
+  dump_desc(s_dup, "router descriptor");
+  routerinfo_free(router);
+  router = NULL;
+ done:
+  crypto_pk_free(rsa_pubkey);
+  tor_cert_free(ntor_cc_cert);
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  smartlist_free(exit_policy_tokens);
+  if (area) {
+    DUMP_AREA(area, "routerinfo");
+    memarea_drop_all(area);
+  }
+  if (can_dl_again_out)
+    *can_dl_again_out = can_dl_again;
+  return router;
+}
+
+/** Parse a single extrainfo entry from the string <b>s</b>, ending at
+ * <b>end</b>.  (If <b>end</b> is NULL, parse up to the end of <b>s</b>.)  If
+ * <b>cache_copy</b> is true, make a copy of the extra-info document in the
+ * cache_info fields of the result.  If <b>routermap</b> is provided, use it
+ * as a map from router identity to routerinfo_t when looking up signing keys.
+ *
+ * If <b>can_dl_again_out</b> is provided, set *<b>can_dl_again_out</b> to 1
+ * if it's okay to try to download an extrainfo with this same digest again,
+ * and 0 if it isn't.  (It might not be okay to download it again if part of
+ * the part covered by the digest is invalid.)
+ */
+extrainfo_t *
+extrainfo_parse_entry_from_string(const char *s, const char *end,
+                            int cache_copy, struct digest_ri_map_t *routermap,
+                            int *can_dl_again_out)
+{
+  extrainfo_t *extrainfo = NULL;
+  char digest[128];
+  smartlist_t *tokens = NULL;
+  directory_token_t *tok;
+  crypto_pk_t *key = NULL;
+  routerinfo_t *router = NULL;
+  memarea_t *area = NULL;
+  const char *s_dup = s;
+  /* Do not set this to '1' until we have parsed everything that we intend to
+   * parse that's covered by the hash. */
+  int can_dl_again = 0;
+
+  if (BUG(s == NULL))
+    return NULL;
+
+  if (!end) {
+    end = s + strlen(s);
+  }
+
+  /* point 'end' to a point immediately after the final newline. */
+  while (end > s+2 && *(end-1) == '\n' && *(end-2) == '\n')
+    --end;
+
+  if (router_get_extrainfo_hash(s, end-s, digest) < 0) {
+    log_warn(LD_DIR, "Couldn't compute router hash.");
+    goto err;
+  }
+  tokens = smartlist_new();
+  area = memarea_new();
+  if (tokenize_string(area,s,end,tokens,extrainfo_token_table,0)) {
+    log_warn(LD_DIR, "Error tokenizing extra-info document.");
+    goto err;
+  }
+
+  if (smartlist_len(tokens) < 2) {
+    log_warn(LD_DIR, "Impossibly short extra-info document.");
+    goto err;
+  }
+
+  /* XXXX Accept this in position 1 too, and ed identity in position 0. */
+  tok = smartlist_get(tokens,0);
+  if (tok->tp != K_EXTRA_INFO) {
+    log_warn(LD_DIR,"Entry does not start with \"extra-info\"");
+    goto err;
+  }
+
+  extrainfo = tor_malloc_zero(sizeof(extrainfo_t));
+  extrainfo->cache_info.is_extrainfo = 1;
+  if (cache_copy)
+    extrainfo->cache_info.signed_descriptor_body = tor_memdup_nulterm(s,end-s);
+  extrainfo->cache_info.signed_descriptor_len = end-s;
+  memcpy(extrainfo->cache_info.signed_descriptor_digest, digest, DIGEST_LEN);
+  crypto_digest256((char*)extrainfo->digest256, s, end-s, DIGEST_SHA256);
+
+  tor_assert(tok->n_args >= 2);
+  if (!is_legal_nickname(tok->args[0])) {
+    log_warn(LD_DIR,"Bad nickname %s on \"extra-info\"",escaped(tok->args[0]));
+    goto err;
+  }
+  strlcpy(extrainfo->nickname, tok->args[0], sizeof(extrainfo->nickname));
+  if (strlen(tok->args[1]) != HEX_DIGEST_LEN ||
+      base16_decode(extrainfo->cache_info.identity_digest, DIGEST_LEN,
+                    tok->args[1], HEX_DIGEST_LEN) != DIGEST_LEN) {
+    log_warn(LD_DIR,"Invalid fingerprint %s on \"extra-info\"",
+             escaped(tok->args[1]));
+    goto err;
+  }
+
+  tok = find_by_keyword(tokens, K_PUBLISHED);
+  if (parse_iso_time(tok->args[0], &extrainfo->cache_info.published_on)) {
+    log_warn(LD_DIR,"Invalid published time %s on \"extra-info\"",
+             escaped(tok->args[0]));
+    goto err;
+  }
+
+  {
+    directory_token_t *ed_sig_tok, *ed_cert_tok;
+    ed_sig_tok = find_opt_by_keyword(tokens, K_ROUTER_SIG_ED25519);
+    ed_cert_tok = find_opt_by_keyword(tokens, K_IDENTITY_ED25519);
+    int n_ed_toks = !!ed_sig_tok + !!ed_cert_tok;
+    if (n_ed_toks != 0 && n_ed_toks != 2) {
+      log_warn(LD_DIR, "Router descriptor with only partial ed25519/"
+               "cross-certification support");
+      goto err;
+    }
+    if (ed_sig_tok) {
+      tor_assert(ed_cert_tok);
+      const int ed_cert_token_pos = smartlist_pos(tokens, ed_cert_tok);
+      if (ed_cert_token_pos != 1) {
+        /* Accept this in position 0 XXXX */
+        log_warn(LD_DIR, "Ed25519 certificate in wrong position");
+        goto err;
+      }
+      if (ed_sig_tok != smartlist_get(tokens, smartlist_len(tokens)-2)) {
+        log_warn(LD_DIR, "Ed25519 signature in wrong position");
+        goto err;
+      }
+      if (strcmp(ed_cert_tok->object_type, "ED25519 CERT")) {
+        log_warn(LD_DIR, "Wrong object type on identity-ed25519 in decriptor");
+        goto err;
+      }
+
+      uint8_t d256[DIGEST256_LEN];
+      const char *signed_start, *signed_end;
+      tor_cert_t *cert = tor_cert_parse(
+                       (const uint8_t*)ed_cert_tok->object_body,
+                       ed_cert_tok->object_size);
+      if (! cert) {
+        log_warn(LD_DIR, "Couldn't parse ed25519 cert");
+        goto err;
+      }
+      /* makes sure it gets freed. */
+      extrainfo->cache_info.signing_key_cert = cert;
+
+      if (cert->cert_type != CERT_TYPE_ID_SIGNING ||
+          ! cert->signing_key_included) {
+        log_warn(LD_DIR, "Invalid form for ed25519 cert");
+        goto err;
+      }
+
+      if (router_get_hash_impl_helper(s, end-s, "extra-info ",
+                                      "\nrouter-sig-ed25519",
+                                      ' ', LOG_WARN,
+                                      &signed_start, &signed_end) < 0) {
+        log_warn(LD_DIR, "Can't find ed25519-signed portion of extrainfo");
+        goto err;
+      }
+      crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA256);
+      crypto_digest_add_bytes(d, ED_DESC_SIGNATURE_PREFIX,
+        strlen(ED_DESC_SIGNATURE_PREFIX));
+      crypto_digest_add_bytes(d, signed_start, signed_end-signed_start);
+      crypto_digest_get_digest(d, (char*)d256, sizeof(d256));
+      crypto_digest_free(d);
+
+      ed25519_checkable_t check[2];
+      int check_ok[2];
+      if (tor_cert_get_checkable_sig(&check[0], cert, NULL, NULL) < 0) {
+        log_err(LD_BUG, "Couldn't create 'checkable' for cert.");
+        goto err;
+      }
+
+      if (ed25519_signature_from_base64(&check[1].signature,
+                                        ed_sig_tok->args[0])<0) {
+        log_warn(LD_DIR, "Couldn't decode ed25519 signature");
+        goto err;
+      }
+      check[1].pubkey = &cert->signed_key;
+      check[1].msg = d256;
+      check[1].len = DIGEST256_LEN;
+
+      if (ed25519_checksig_batch(check_ok, check, 2) < 0) {
+        log_warn(LD_DIR, "Incorrect ed25519 signature(s)");
+        goto err;
+      }
+      /* We don't check the certificate expiration time: checking that it
+       * matches the cert in the router descriptor is adequate. */
+    }
+  }
+
+  /* We've checked everything that's covered by the hash. */
+  can_dl_again = 1;
+
+  if (routermap &&
+      (router = digestmap_get((digestmap_t*)routermap,
+                              extrainfo->cache_info.identity_digest))) {
+    key = router->identity_pkey;
+  }
+
+  tok = find_by_keyword(tokens, K_ROUTER_SIGNATURE);
+  if (strcmp(tok->object_type, "SIGNATURE") ||
+      tok->object_size < 128 || tok->object_size > 512) {
+    log_warn(LD_DIR, "Bad object type or length on extra-info signature");
+    goto err;
+  }
+
+  if (key) {
+    if (check_signature_token(digest, DIGEST_LEN, tok, key, 0,
+                              "extra-info") < 0)
+      goto err;
+
+    if (router)
+      extrainfo->cache_info.send_unencrypted =
+        router->cache_info.send_unencrypted;
+  } else {
+    extrainfo->pending_sig = tor_memdup(tok->object_body,
+                                        tok->object_size);
+    extrainfo->pending_sig_len = tok->object_size;
+  }
+
+  goto done;
+ err:
+  dump_desc(s_dup, "extra-info descriptor");
+  extrainfo_free(extrainfo);
+  extrainfo = NULL;
+ done:
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  if (area) {
+    DUMP_AREA(area, "extrainfo");
+    memarea_drop_all(area);
+  }
+  if (can_dl_again_out)
+    *can_dl_again_out = can_dl_again;
+  return extrainfo;
+}
+
+/** Add an exit policy stored in the token <b>tok</b> to the router info in
+ * <b>router</b>.  Return 0 on success, -1 on failure. */
+static int
+router_add_exit_policy(routerinfo_t *router, directory_token_t *tok)
+{
+  addr_policy_t *newe;
+  /* Use the standard interpretation of accept/reject *, an IPv4 wildcard. */
+  newe = router_parse_addr_policy(tok, 0);
+  if (!newe)
+    return -1;
+  if (! router->exit_policy)
+    router->exit_policy = smartlist_new();
+
+  /* Ensure that in descriptors, accept/reject fields are followed by
+   * IPv4 addresses, and accept6/reject6 fields are followed by
+   * IPv6 addresses. Unlike torrcs, descriptor exit policies do not permit
+   * accept/reject followed by IPv6. */
+  if (((tok->tp == K_ACCEPT6 || tok->tp == K_REJECT6) &&
+       tor_addr_family(&newe->addr) == AF_INET)
+      ||
+      ((tok->tp == K_ACCEPT || tok->tp == K_REJECT) &&
+       tor_addr_family(&newe->addr) == AF_INET6)) {
+    /* There's nothing the user can do about other relays' descriptors,
+     * so we don't provide usage advice here. */
+    log_warn(LD_DIR, "Mismatch between field type and address type in exit "
+             "policy '%s'. Discarding entire router descriptor.",
+             tok->n_args == 1 ? tok->args[0] : "");
+    addr_policy_free(newe);
+    return -1;
+  }
+
+  smartlist_add(router->exit_policy, newe);
+
+  return 0;
+}
+
+/** Return a newly allocated smartlist of all accept or reject tokens in
+ * <b>s</b>.
+ */
+static smartlist_t *
+find_all_exitpolicy(smartlist_t *s)
+{
+  smartlist_t *out = smartlist_new();
+  SMARTLIST_FOREACH(s, directory_token_t *, t,
+      if (t->tp == K_ACCEPT || t->tp == K_ACCEPT6 ||
+          t->tp == K_REJECT || t->tp == K_REJECT6)
+        smartlist_add(out,t));
+  return out;
+}
+
+/** Called on startup; right now we just handle scanning the unparseable
+ * descriptor dumps, but hang anything else we might need to do in the
+ * future here as well.
+ */
+void
+routerparse_init(void)
+{
+  /*
+   * Check both if the sandbox is active and whether it's configured; no
+   * point in loading all that if we won't be able to use it after the
+   * sandbox becomes active.
+   */
+  if (!(sandbox_is_active() || get_options()->Sandbox)) {
+    dump_desc_init();
+  }
+}
+
+/** Clean up all data structures used by routerparse.c at exit */
+void
+routerparse_free_all(void)
+{
+  dump_desc_fifo_cleanup();
+}
diff --git a/src/feature/dirparse/routerparse.h b/src/feature/dirparse/routerparse.h
new file mode 100644
index 0000000000..f9a13f2168
--- /dev/null
+++ b/src/feature/dirparse/routerparse.h
@@ -0,0 +1,49 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file routerparse.h
+ * \brief Header file for routerparse.c.
+ **/
+
+#ifndef TOR_ROUTERPARSE_H
+#define TOR_ROUTERPARSE_H
+
+int router_get_router_hash(const char *s, size_t s_len, char *digest);
+int router_get_extrainfo_hash(const char *s, size_t s_len, char *digest);
+
+int router_parse_list_from_string(const char **s, const char *eos,
+                                  smartlist_t *dest,
+                                  saved_location_t saved_location,
+                                  int is_extrainfo,
+                                  int allow_annotations,
+                                  const char *prepend_annotations,
+                                  smartlist_t *invalid_digests_out);
+
+routerinfo_t *router_parse_entry_from_string(const char *s, const char *end,
+                                             int cache_copy,
+                                             int allow_annotations,
+                                             const char *prepend_annotations,
+                                             int *can_dl_again_out);
+struct digest_ri_map_t;
+extrainfo_t *extrainfo_parse_entry_from_string(const char *s, const char *end,
+                             int cache_copy, struct digest_ri_map_t *routermap,
+                             int *can_dl_again_out);
+
+int find_single_ipv6_orport(const smartlist_t *list,
+                            tor_addr_t *addr_out,
+                            uint16_t *port_out);
+
+void routerparse_init(void);
+void routerparse_free_all(void);
+
+#ifdef EXPOSE_ROUTERDESC_TOKEN_TABLE
+extern const struct token_rule_t routerdesc_token_table[];
+#endif
+
+#define ED_DESC_SIGNATURE_PREFIX "Tor router descriptor signature v1"
+
+#endif /* !defined(TOR_ROUTERPARSE_H) */
diff --git a/src/feature/dirparse/sigcommon.c b/src/feature/dirparse/sigcommon.c
new file mode 100644
index 0000000000..2019e09918
--- /dev/null
+++ b/src/feature/dirparse/sigcommon.c
@@ -0,0 +1,185 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file sigcommon.c
+ * \brief Shared hashing, signing, and signature-checking code for directory
+ *    objects.
+ **/
+
+#define SIGCOMMON_PRIVATE
+
+#include "core/or/or.h"
+#include "feature/dirparse/parsecommon.h"
+#include "feature/dirparse/sigcommon.h"
+
+/** Helper function for <b>router_get_hash_impl</b>: given <b>s</b>,
+ * <b>s_len</b>, <b>start_str</b>, <b>end_str</b>, and <b>end_c</b> with the
+ * same semantics as in that function, set *<b>start_out</b> (inclusive) and
+ * *<b>end_out</b> (exclusive) to the boundaries of the string to be hashed.
+ *
+ * Return 0 on success and -1 on failure.
+ */
+int
+router_get_hash_impl_helper(const char *s, size_t s_len,
+                            const char *start_str,
+                            const char *end_str, char end_c,
+                            int log_severity,
+                            const char **start_out, const char **end_out)
+{
+  const char *start, *end;
+  start = tor_memstr(s, s_len, start_str);
+  if (!start) {
+    log_fn(log_severity,LD_DIR,
+           "couldn't find start of hashed material \"%s\"",start_str);
+    return -1;
+  }
+  if (start != s && *(start-1) != '\n') {
+    log_fn(log_severity,LD_DIR,
+             "first occurrence of \"%s\" is not at the start of a line",
+             start_str);
+    return -1;
+  }
+  end = tor_memstr(start+strlen(start_str),
+                   s_len - (start-s) - strlen(start_str), end_str);
+  if (!end) {
+    log_fn(log_severity,LD_DIR,
+           "couldn't find end of hashed material \"%s\"",end_str);
+    return -1;
+  }
+  end = memchr(end+strlen(end_str), end_c, s_len - (end-s) - strlen(end_str));
+  if (!end) {
+    log_fn(log_severity,LD_DIR,
+           "couldn't find EOL");
+    return -1;
+  }
+  ++end;
+
+  *start_out = start;
+  *end_out = end;
+  return 0;
+}
+
+/** Compute the digest of the substring of <b>s</b> taken from the first
+ * occurrence of <b>start_str</b> through the first instance of c after the
+ * first subsequent occurrence of <b>end_str</b>; store the 20-byte or 32-byte
+ * result in <b>digest</b>; return 0 on success.
+ *
+ * If no such substring exists, return -1.
+ */
+int
+router_get_hash_impl(const char *s, size_t s_len, char *digest,
+                     const char *start_str,
+                     const char *end_str, char end_c,
+                     digest_algorithm_t alg)
+{
+  const char *start=NULL, *end=NULL;
+  if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c,LOG_WARN,
+                                  &start,&end)<0)
+    return -1;
+
+  return router_compute_hash_final(digest, start, end-start, alg);
+}
+
+/** Compute the digest of the <b>len</b>-byte directory object at
+ * <b>start</b>, using <b>alg</b>. Store the result in <b>digest</b>, which
+ * must be long enough to hold it. */
+MOCK_IMPL(STATIC int,
+router_compute_hash_final,(char *digest,
+                           const char *start, size_t len,
+                           digest_algorithm_t alg))
+{
+  if (alg == DIGEST_SHA1) {
+    if (crypto_digest(digest, start, len) < 0) {
+      log_warn(LD_BUG,"couldn't compute digest");
+      return -1;
+    }
+  } else {
+    if (crypto_digest256(digest, start, len, alg) < 0) {
+      log_warn(LD_BUG,"couldn't compute digest");
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+/** As router_get_hash_impl, but compute all hashes. */
+int
+router_get_hashes_impl(const char *s, size_t s_len, common_digests_t *digests,
+                       const char *start_str,
+                       const char *end_str, char end_c)
+{
+  const char *start=NULL, *end=NULL;
+  if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c,LOG_WARN,
+                                  &start,&end)<0)
+    return -1;
+
+  if (crypto_common_digests(digests, start, end-start)) {
+    log_warn(LD_BUG,"couldn't compute digests");
+    return -1;
+  }
+
+  return 0;
+}
+
+MOCK_IMPL(STATIC int,
+signed_digest_equals, (const uint8_t *d1, const uint8_t *d2, size_t len))
+{
+  return tor_memeq(d1, d2, len);
+}
+
+/** Check whether the object body of the token in <b>tok</b> has a good
+ * signature for <b>digest</b> using key <b>pkey</b>.
+ * If <b>CST_NO_CHECK_OBJTYPE</b> is set, do not check
+ * the object type of the signature object. Use <b>doctype</b> as the type of
+ * the document when generating log messages.  Return 0 on success, negative
+ * on failure.
+ */
+int
+check_signature_token(const char *digest,
+                      ssize_t digest_len,
+                      directory_token_t *tok,
+                      crypto_pk_t *pkey,
+                      int flags,
+                      const char *doctype)
+{
+  char *signed_digest;
+  size_t keysize;
+  const int check_objtype = ! (flags & CST_NO_CHECK_OBJTYPE);
+
+  tor_assert(pkey);
+  tor_assert(tok);
+  tor_assert(digest);
+  tor_assert(doctype);
+
+  if (check_objtype) {
+    if (strcmp(tok->object_type, "SIGNATURE")) {
+      log_warn(LD_DIR, "Bad object type on %s signature", doctype);
+      return -1;
+    }
+  }
+
+  keysize = crypto_pk_keysize(pkey);
+  signed_digest = tor_malloc(keysize);
+  if (crypto_pk_public_checksig(pkey, signed_digest, keysize,
+                                tok->object_body, tok->object_size)
+      < digest_len) {
+    log_warn(LD_DIR, "Error reading %s: invalid signature.", doctype);
+    tor_free(signed_digest);
+    return -1;
+  }
+  //  log_debug(LD_DIR,"Signed %s hash starts %s", doctype,
+  //            hex_str(signed_digest,4));
+  if (! signed_digest_equals((const uint8_t *)digest,
+                             (const uint8_t *)signed_digest, digest_len)) {
+    log_warn(LD_DIR, "Error reading %s: signature does not match.", doctype);
+    tor_free(signed_digest);
+    return -1;
+  }
+  tor_free(signed_digest);
+  return 0;
+}
diff --git a/src/feature/dirparse/sigcommon.h b/src/feature/dirparse/sigcommon.h
new file mode 100644
index 0000000000..fdd8e839a9
--- /dev/null
+++ b/src/feature/dirparse/sigcommon.h
@@ -0,0 +1,48 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file sigcommon.h
+ * \brief Header file for sigcommon.c.
+ **/
+
+#ifndef TOR_SIGCOMMON_H
+#define TOR_SIGCOMMON_H
+
+/* TODO: Rename all of these functions */
+int router_get_hash_impl(const char *s, size_t s_len, char *digest,
+                         const char *start_str, const char *end_str,
+                         char end_char,
+                         digest_algorithm_t alg);
+
+#define CST_NO_CHECK_OBJTYPE  (1<<0)
+struct directory_token_t;
+int check_signature_token(const char *digest,
+                          ssize_t digest_len,
+                          struct directory_token_t *tok,
+                          crypto_pk_t *pkey,
+                          int flags,
+                          const char *doctype);
+
+int router_get_hash_impl_helper(const char *s, size_t s_len,
+                            const char *start_str,
+                            const char *end_str, char end_c,
+                            int log_severity,
+                            const char **start_out, const char **end_out);
+int router_get_hashes_impl(const char *s, size_t s_len,
+                           common_digests_t *digests,
+                           const char *start_str, const char *end_str,
+                           char end_char);
+
+#ifdef SIGCOMMON_PRIVATE
+MOCK_DECL(STATIC int, signed_digest_equals,
+          (const uint8_t *d1, const uint8_t *d2, size_t len));
+MOCK_DECL(STATIC int, router_compute_hash_final,(char *digest,
+                           const char *start, size_t len,
+                           digest_algorithm_t alg));
+#endif
+
+#endif /* !defined(TOR_SIGCOMMON_H) */
diff --git a/src/feature/dirparse/signing.c b/src/feature/dirparse/signing.c
new file mode 100644
index 0000000000..3ab40c3807
--- /dev/null
+++ b/src/feature/dirparse/signing.c
@@ -0,0 +1,98 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file signing.c
+ * \brief Code to sign directory objects.
+ **/
+
+#include "core/or/or.h"
+#include "feature/dirparse/signing.h"
+
+/** Helper: used to generate signatures for routers, directories and
+ * network-status objects.  Given a <b>digest_len</b>-byte digest in
+ * <b>digest</b> and a secret <b>private_key</b>, generate an PKCS1-padded
+ * signature, BASE64-encode it, surround it with -----BEGIN/END----- pairs,
+ * and return the new signature on success or NULL on failure.
+ */
+char *
+router_get_dirobj_signature(const char *digest,
+                            size_t digest_len,
+                            const crypto_pk_t *private_key)
+{
+  char *signature;
+  size_t i, keysize;
+  int siglen;
+  char *buf = NULL;
+  size_t buf_len;
+  /* overestimate of BEGIN/END lines total len. */
+#define BEGIN_END_OVERHEAD_LEN 64
+
+  keysize = crypto_pk_keysize(private_key);
+  signature = tor_malloc(keysize);
+  siglen = crypto_pk_private_sign(private_key, signature, keysize,
+                                  digest, digest_len);
+  if (siglen < 0) {
+    log_warn(LD_BUG,"Couldn't sign digest.");
+    goto err;
+  }
+
+  /* The *2 here is a ridiculous overestimate of base-64 overhead. */
+  buf_len = (siglen * 2) + BEGIN_END_OVERHEAD_LEN;
+  buf = tor_malloc(buf_len);
+
+  if (strlcpy(buf, "-----BEGIN SIGNATURE-----\n", buf_len) >= buf_len)
+    goto truncated;
+
+  i = strlen(buf);
+  if (base64_encode(buf+i, buf_len-i, signature, siglen,
+                    BASE64_ENCODE_MULTILINE) < 0) {
+    log_warn(LD_BUG,"couldn't base64-encode signature");
+    goto err;
+  }
+
+  if (strlcat(buf, "-----END SIGNATURE-----\n", buf_len) >= buf_len)
+    goto truncated;
+
+  tor_free(signature);
+  return buf;
+
+ truncated:
+  log_warn(LD_BUG,"tried to exceed string length.");
+ err:
+  tor_free(signature);
+  tor_free(buf);
+  return NULL;
+}
+
+/** Helper: used to generate signatures for routers, directories and
+ * network-status objects.  Given a digest in <b>digest</b> and a secret
+ * <b>private_key</b>, generate a PKCS1-padded signature, BASE64-encode it,
+ * surround it with -----BEGIN/END----- pairs, and write it to the
+ * <b>buf_len</b>-byte buffer at <b>buf</b>.  Return 0 on success, -1 on
+ * failure.
+ */
+int
+router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest,
+                               size_t digest_len, crypto_pk_t *private_key)
+{
+  size_t sig_len, s_len;
+  char *sig = router_get_dirobj_signature(digest, digest_len, private_key);
+  if (!sig) {
+    log_warn(LD_BUG, "No signature generated");
+    return -1;
+  }
+  sig_len = strlen(sig);
+  s_len = strlen(buf);
+  if (sig_len + s_len + 1 > buf_len) {
+    log_warn(LD_BUG, "Not enough room for signature");
+    tor_free(sig);
+    return -1;
+  }
+  memcpy(buf+s_len, sig, sig_len+1);
+  tor_free(sig);
+  return 0;
+}
diff --git a/src/feature/dirparse/signing.h b/src/feature/dirparse/signing.h
new file mode 100644
index 0000000000..2e3699baf8
--- /dev/null
+++ b/src/feature/dirparse/signing.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file signing.h
+ * \brief Header file for signing.c.
+ **/
+
+#ifndef TOR_SIGNING_H
+#define TOR_SIGNING_H
+
+#define DIROBJ_MAX_SIG_LEN 256
+char *router_get_dirobj_signature(const char *digest,
+                                  size_t digest_len,
+                                  const crypto_pk_t *private_key);
+int router_append_dirobj_signature(char *buf, size_t buf_len,
+                                   const char *digest,
+                                   size_t digest_len,
+                                   crypto_pk_t *private_key);
+#endif
diff --git a/src/feature/dirparse/unparseable.c b/src/feature/dirparse/unparseable.c
new file mode 100644
index 0000000000..941b5a1f6d
--- /dev/null
+++ b/src/feature/dirparse/unparseable.c
@@ -0,0 +1,591 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define UNPARSEABLE_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "feature/dirparse/unparseable.h"
+#include "lib/sandbox/sandbox.h"
+
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
+/* Dump mechanism for unparseable descriptors */
+
+/** List of dumped descriptors for FIFO cleanup purposes */
+STATIC smartlist_t *descs_dumped = NULL;
+/** Total size of dumped descriptors for FIFO cleanup */
+STATIC uint64_t len_descs_dumped = 0;
+/** Directory to stash dumps in */
+static int have_dump_desc_dir = 0;
+static int problem_with_dump_desc_dir = 0;
+
+#define DESC_DUMP_DATADIR_SUBDIR "unparseable-descs"
+#define DESC_DUMP_BASE_FILENAME "unparseable-desc"
+
+/** Find the dump directory and check if we'll be able to create it */
+void
+dump_desc_init(void)
+{
+  char *dump_desc_dir;
+
+  dump_desc_dir = get_datadir_fname(DESC_DUMP_DATADIR_SUBDIR);
+
+  /*
+   * We just check for it, don't create it at this point; we'll
+   * create it when we need it if it isn't already there.
+   */
+  if (check_private_dir(dump_desc_dir, CPD_CHECK, get_options()->User) < 0) {
+    /* Error, log and flag it as having a problem */
+    log_notice(LD_DIR,
+               "Doesn't look like we'll be able to create descriptor dump "
+               "directory %s; dumps will be disabled.",
+               dump_desc_dir);
+    problem_with_dump_desc_dir = 1;
+    tor_free(dump_desc_dir);
+    return;
+  }
+
+  /* Check if it exists */
+  switch (file_status(dump_desc_dir)) {
+    case FN_DIR:
+      /* We already have a directory */
+      have_dump_desc_dir = 1;
+      break;
+    case FN_NOENT:
+      /* Nothing, we'll need to create it later */
+      have_dump_desc_dir = 0;
+      break;
+    case FN_ERROR:
+      /* Log and flag having a problem */
+      log_notice(LD_DIR,
+                 "Couldn't check whether descriptor dump directory %s already"
+                 " exists: %s",
+                 dump_desc_dir, strerror(errno));
+      problem_with_dump_desc_dir = 1;
+      break;
+    case FN_FILE:
+    case FN_EMPTY:
+    default:
+      /* Something else was here! */
+      log_notice(LD_DIR,
+                 "Descriptor dump directory %s already exists and isn't a "
+                 "directory",
+                 dump_desc_dir);
+      problem_with_dump_desc_dir = 1;
+  }
+
+  if (have_dump_desc_dir && !problem_with_dump_desc_dir) {
+    dump_desc_populate_fifo_from_directory(dump_desc_dir);
+  }
+
+  tor_free(dump_desc_dir);
+}
+
+/** Create the dump directory if needed and possible */
+static void
+dump_desc_create_dir(void)
+{
+  char *dump_desc_dir;
+
+  /* If the problem flag is set, skip it */
+  if (problem_with_dump_desc_dir) return;
+
+  /* Do we need it? */
+  if (!have_dump_desc_dir) {
+    dump_desc_dir = get_datadir_fname(DESC_DUMP_DATADIR_SUBDIR);
+
+    if (check_private_dir(dump_desc_dir, CPD_CREATE,
+                          get_options()->User) < 0) {
+      log_notice(LD_DIR,
+                 "Failed to create descriptor dump directory %s",
+                 dump_desc_dir);
+      problem_with_dump_desc_dir = 1;
+    }
+
+    /* Okay, we created it */
+    have_dump_desc_dir = 1;
+
+    tor_free(dump_desc_dir);
+  }
+}
+
+/** Dump desc FIFO/cleanup; take ownership of the given filename, add it to
+ * the FIFO, and clean up the oldest entries to the extent they exceed the
+ * configured cap.  If any old entries with a matching hash existed, they
+ * just got overwritten right before this was called and we should adjust
+ * the total size counter without deleting them.
+ */
+static void
+dump_desc_fifo_add_and_clean(char *filename, const uint8_t *digest_sha256,
+                             size_t len)
+{
+  dumped_desc_t *ent = NULL, *tmp;
+  uint64_t max_len;
+
+  tor_assert(filename != NULL);
+  tor_assert(digest_sha256 != NULL);
+
+  if (descs_dumped == NULL) {
+    /* We better have no length, then */
+    tor_assert(len_descs_dumped == 0);
+    /* Make a smartlist */
+    descs_dumped = smartlist_new();
+  }
+
+  /* Make a new entry to put this one in */
+  ent = tor_malloc_zero(sizeof(*ent));
+  ent->filename = filename;
+  ent->len = len;
+  ent->when = time(NULL);
+  memcpy(ent->digest_sha256, digest_sha256, DIGEST256_LEN);
+
+  /* Do we need to do some cleanup? */
+  max_len = get_options()->MaxUnparseableDescSizeToLog;
+  /* Iterate over the list until we've freed enough space */
+  while (len > max_len - len_descs_dumped &&
+         smartlist_len(descs_dumped) > 0) {
+    /* Get the oldest thing on the list */
+    tmp = (dumped_desc_t *)(smartlist_get(descs_dumped, 0));
+
+    /*
+     * Check if it matches the filename we just added, so we don't delete
+     * something we just emitted if we get repeated identical descriptors.
+     */
+    if (strcmp(tmp->filename, filename) != 0) {
+      /* Delete it and adjust the length counter */
+      tor_unlink(tmp->filename);
+      tor_assert(len_descs_dumped >= tmp->len);
+      len_descs_dumped -= tmp->len;
+      log_info(LD_DIR,
+               "Deleting old unparseable descriptor dump %s due to "
+               "space limits",
+               tmp->filename);
+    } else {
+      /*
+       * Don't delete, but do adjust the counter since we will bump it
+       * later
+       */
+      tor_assert(len_descs_dumped >= tmp->len);
+      len_descs_dumped -= tmp->len;
+      log_info(LD_DIR,
+               "Replacing old descriptor dump %s with new identical one",
+               tmp->filename);
+    }
+
+    /* Free it and remove it from the list */
+    smartlist_del_keeporder(descs_dumped, 0);
+    tor_free(tmp->filename);
+    tor_free(tmp);
+  }
+
+  /* Append our entry to the end of the list and bump the counter */
+  smartlist_add(descs_dumped, ent);
+  len_descs_dumped += len;
+}
+
+/** Check if we already have a descriptor for this hash and move it to the
+ * head of the queue if so.  Return 1 if one existed and 0 otherwise.
+ */
+static int
+dump_desc_fifo_bump_hash(const uint8_t *digest_sha256)
+{
+  dumped_desc_t *match = NULL;
+
+  tor_assert(digest_sha256);
+
+  if (descs_dumped) {
+    /* Find a match if one exists */
+    SMARTLIST_FOREACH_BEGIN(descs_dumped, dumped_desc_t *, ent) {
+      if (ent &&
+          tor_memeq(ent->digest_sha256, digest_sha256, DIGEST256_LEN)) {
+        /*
+         * Save a pointer to the match and remove it from its current
+         * position.
+         */
+        match = ent;
+        SMARTLIST_DEL_CURRENT_KEEPORDER(descs_dumped, ent);
+        break;
+      }
+    } SMARTLIST_FOREACH_END(ent);
+
+    if (match) {
+      /* Update the timestamp */
+      match->when = time(NULL);
+      /* Add it back at the end of the list */
+      smartlist_add(descs_dumped, match);
+
+      /* Indicate we found one */
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
+/** Clean up on exit; just memory, leave the dumps behind
+ */
+void
+dump_desc_fifo_cleanup(void)
+{
+  if (descs_dumped) {
+    /* Free each descriptor */
+    SMARTLIST_FOREACH_BEGIN(descs_dumped, dumped_desc_t *, ent) {
+      tor_assert(ent);
+      tor_free(ent->filename);
+      tor_free(ent);
+    } SMARTLIST_FOREACH_END(ent);
+    /* Free the list */
+    smartlist_free(descs_dumped);
+    descs_dumped = NULL;
+    len_descs_dumped = 0;
+  }
+}
+
+/** Handle one file for dump_desc_populate_fifo_from_directory(); make sure
+ * the filename is sensibly formed and matches the file content, and either
+ * return a dumped_desc_t for it or remove the file and return NULL.
+ */
+MOCK_IMPL(STATIC dumped_desc_t *,
+dump_desc_populate_one_file, (const char *dirname, const char *f))
+{
+  dumped_desc_t *ent = NULL;
+  char *path = NULL, *desc = NULL;
+  const char *digest_str;
+  char digest[DIGEST256_LEN], content_digest[DIGEST256_LEN];
+  /* Expected prefix before digest in filenames */
+  const char *f_pfx = DESC_DUMP_BASE_FILENAME ".";
+  /*
+   * Stat while reading; this is important in case the file
+   * contains a NUL character.
+   */
+  struct stat st;
+
+  /* Sanity-check args */
+  tor_assert(dirname != NULL);
+  tor_assert(f != NULL);
+
+  /* Form the full path */
+  tor_asprintf(&path, "%s" PATH_SEPARATOR "%s", dirname, f);
+
+  /* Check that f has the form DESC_DUMP_BASE_FILENAME.<digest256> */
+
+  if (!strcmpstart(f, f_pfx)) {
+    /* It matches the form, but is the digest parseable as such? */
+    digest_str = f + strlen(f_pfx);
+    if (base16_decode(digest, DIGEST256_LEN,
+                      digest_str, strlen(digest_str)) != DIGEST256_LEN) {
+      /* We failed to decode it */
+      digest_str = NULL;
+    }
+  } else {
+    /* No match */
+    digest_str = NULL;
+  }
+
+  if (!digest_str) {
+    /* We couldn't get a sensible digest */
+    log_notice(LD_DIR,
+               "Removing unrecognized filename %s from unparseable "
+               "descriptors directory", f);
+    tor_unlink(path);
+    /* We're done */
+    goto done;
+  }
+
+  /*
+   * The filename has the form DESC_DUMP_BASE_FILENAME "." <digest256> and
+   * we've decoded the digest.  Next, check that we can read it and the
+   * content matches this digest.  We are relying on the fact that if the
+   * file contains a '\0', read_file_to_str() will allocate space for and
+   * read the entire file and return the correct size in st.
+   */
+  desc = read_file_to_str(path, RFTS_IGNORE_MISSING|RFTS_BIN, &st);
+  if (!desc) {
+    /* We couldn't read it */
+    log_notice(LD_DIR,
+               "Failed to read %s from unparseable descriptors directory; "
+               "attempting to remove it.", f);
+    tor_unlink(path);
+    /* We're done */
+    goto done;
+  }
+
+#if SIZE_MAX > UINT64_MAX
+  if (BUG((uint64_t)st.st_size > (uint64_t)SIZE_MAX)) {
+    /* LCOV_EXCL_START
+     * Should be impossible since RFTS above should have failed to read the
+     * huge file into RAM. */
+    goto done;
+    /* LCOV_EXCL_STOP */
+  }
+#endif /* SIZE_MAX > UINT64_MAX */
+  if (BUG(st.st_size < 0)) {
+    /* LCOV_EXCL_START
+     * Should be impossible, since the OS isn't supposed to be b0rken. */
+    goto done;
+    /* LCOV_EXCL_STOP */
+  }
+  /* (Now we can be sure that st.st_size is safe to cast to a size_t.) */
+
+  /*
+   * We got one; now compute its digest and check that it matches the
+   * filename.
+   */
+  if (crypto_digest256((char *)content_digest, desc, (size_t) st.st_size,
+                       DIGEST_SHA256) < 0) {
+    /* Weird, but okay */
+    log_info(LD_DIR,
+             "Unable to hash content of %s from unparseable descriptors "
+             "directory", f);
+    tor_unlink(path);
+    /* We're done */
+    goto done;
+  }
+
+  /* Compare the digests */
+  if (tor_memneq(digest, content_digest, DIGEST256_LEN)) {
+    /* No match */
+    log_info(LD_DIR,
+             "Hash of %s from unparseable descriptors directory didn't "
+             "match its filename; removing it", f);
+    tor_unlink(path);
+    /* We're done */
+    goto done;
+  }
+
+  /* Okay, it's a match, we should prepare ent */
+  ent = tor_malloc_zero(sizeof(dumped_desc_t));
+  ent->filename = path;
+  memcpy(ent->digest_sha256, digest, DIGEST256_LEN);
+  ent->len = (size_t) st.st_size;
+  ent->when = st.st_mtime;
+  /* Null out path so we don't free it out from under ent */
+  path = NULL;
+
+ done:
+  /* Free allocations if we had them */
+  tor_free(desc);
+  tor_free(path);
+
+  return ent;
+}
+
+/** Sort helper for dump_desc_populate_fifo_from_directory(); compares
+ * the when field of dumped_desc_ts in a smartlist to put the FIFO in
+ * the correct order after reconstructing it from the directory.
+ */
+static int
+dump_desc_compare_fifo_entries(const void **a_v, const void **b_v)
+{
+  const dumped_desc_t **a = (const dumped_desc_t **)a_v;
+  const dumped_desc_t **b = (const dumped_desc_t **)b_v;
+
+  if ((a != NULL) && (*a != NULL)) {
+    if ((b != NULL) && (*b != NULL)) {
+      /* We have sensible dumped_desc_ts to compare */
+      if ((*a)->when < (*b)->when) {
+        return -1;
+      } else if ((*a)->when == (*b)->when) {
+        return 0;
+      } else {
+        return 1;
+      }
+    } else {
+      /*
+       * We shouldn't see this, but what the hell, NULLs precede everythin
+       * else
+       */
+      return 1;
+    }
+  } else {
+    return -1;
+  }
+}
+
+/** Scan the contents of the directory, and update FIFO/counters; this will
+ * consistency-check descriptor dump filenames against hashes of descriptor
+ * dump file content, and remove any inconsistent/unreadable dumps, and then
+ * reconstruct the dump FIFO as closely as possible for the last time the
+ * tor process shut down.  If a previous dump was repeated more than once and
+ * moved ahead in the FIFO, the mtime will not have been updated and the
+ * reconstructed order will be wrong, but will always be a permutation of
+ * the original.
+ */
+STATIC void
+dump_desc_populate_fifo_from_directory(const char *dirname)
+{
+  smartlist_t *files = NULL;
+  dumped_desc_t *ent = NULL;
+
+  tor_assert(dirname != NULL);
+
+  /* Get a list of files */
+  files = tor_listdir(dirname);
+  if (!files) {
+    log_notice(LD_DIR,
+               "Unable to get contents of unparseable descriptor dump "
+               "directory %s",
+               dirname);
+    return;
+  }
+
+  /*
+   * Iterate through the list and decide which files should go in the
+   * FIFO and which should be purged.
+   */
+
+  SMARTLIST_FOREACH_BEGIN(files, char *, f) {
+    /* Try to get a FIFO entry */
+    ent = dump_desc_populate_one_file(dirname, f);
+    if (ent) {
+      /*
+       * We got one; add it to the FIFO.  No need for duplicate checking
+       * here since we just verified the name and digest match.
+       */
+
+      /* Make sure we have a list to add it to */
+      if (!descs_dumped) {
+        descs_dumped = smartlist_new();
+        len_descs_dumped = 0;
+      }
+
+      /* Add it and adjust the counter */
+      smartlist_add(descs_dumped, ent);
+      len_descs_dumped += ent->len;
+    }
+    /*
+     * If we didn't, we will have unlinked the file if necessary and
+     * possible, and emitted a log message about it, so just go on to
+     * the next.
+     */
+  } SMARTLIST_FOREACH_END(f);
+
+  /* Did we get anything? */
+  if (descs_dumped != NULL) {
+    /* Sort the FIFO in order of increasing timestamp */
+    smartlist_sort(descs_dumped, dump_desc_compare_fifo_entries);
+
+    /* Log some stats */
+    log_info(LD_DIR,
+             "Reloaded unparseable descriptor dump FIFO with %d dump(s) "
+             "totaling %"PRIu64 " bytes",
+             smartlist_len(descs_dumped), (len_descs_dumped));
+  }
+
+  /* Free the original list */
+  SMARTLIST_FOREACH(files, char *, f, tor_free(f));
+  smartlist_free(files);
+}
+
+/** For debugging purposes, dump unparseable descriptor *<b>desc</b> of
+ * type *<b>type</b> to file $DATADIR/unparseable-desc. Do not write more
+ * than one descriptor to disk per minute. If there is already such a
+ * file in the data directory, overwrite it. */
+MOCK_IMPL(void,
+dump_desc,(const char *desc, const char *type))
+{
+  tor_assert(desc);
+  tor_assert(type);
+  size_t len;
+  /* The SHA256 of the string */
+  uint8_t digest_sha256[DIGEST256_LEN];
+  char digest_sha256_hex[HEX_DIGEST256_LEN+1];
+  /* Filename to log it to */
+  char *debugfile, *debugfile_base;
+
+  /* Get the hash for logging purposes anyway */
+  len = strlen(desc);
+  if (crypto_digest256((char *)digest_sha256, desc, len,
+                       DIGEST_SHA256) < 0) {
+    log_info(LD_DIR,
+             "Unable to parse descriptor of type %s, and unable to even hash"
+             " it!", type);
+    goto err;
+  }
+
+  base16_encode(digest_sha256_hex, sizeof(digest_sha256_hex),
+                (const char *)digest_sha256, sizeof(digest_sha256));
+
+  /*
+   * We mention type and hash in the main log; don't clutter up the files
+   * with anything but the exact dump.
+   */
+  tor_asprintf(&debugfile_base,
+               DESC_DUMP_BASE_FILENAME ".%s", digest_sha256_hex);
+  debugfile = get_datadir_fname2(DESC_DUMP_DATADIR_SUBDIR, debugfile_base);
+
+  /*
+   * Check if the sandbox is active or will become active; see comment
+   * below at the log message for why.
+   */
+  if (!(sandbox_is_active() || get_options()->Sandbox)) {
+    if (len <= get_options()->MaxUnparseableDescSizeToLog) {
+      if (!dump_desc_fifo_bump_hash(digest_sha256)) {
+        /* Create the directory if needed */
+        dump_desc_create_dir();
+        /* Make sure we've got it */
+        if (have_dump_desc_dir && !problem_with_dump_desc_dir) {
+          /* Write it, and tell the main log about it */
+          write_str_to_file(debugfile, desc, 1);
+          log_info(LD_DIR,
+                   "Unable to parse descriptor of type %s with hash %s and "
+                   "length %lu. See file %s in data directory for details.",
+                   type, digest_sha256_hex, (unsigned long)len,
+                   debugfile_base);
+          dump_desc_fifo_add_and_clean(debugfile, digest_sha256, len);
+          /* Since we handed ownership over, don't free debugfile later */
+          debugfile = NULL;
+        } else {
+          /* Problem with the subdirectory */
+          log_info(LD_DIR,
+                   "Unable to parse descriptor of type %s with hash %s and "
+                   "length %lu. Descriptor not dumped because we had a "
+                   "problem creating the " DESC_DUMP_DATADIR_SUBDIR
+                   " subdirectory",
+                   type, digest_sha256_hex, (unsigned long)len);
+          /* We do have to free debugfile in this case */
+        }
+      } else {
+        /* We already had one with this hash dumped */
+        log_info(LD_DIR,
+                 "Unable to parse descriptor of type %s with hash %s and "
+                 "length %lu. Descriptor not dumped because one with that "
+                 "hash has already been dumped.",
+                 type, digest_sha256_hex, (unsigned long)len);
+        /* We do have to free debugfile in this case */
+      }
+    } else {
+      /* Just log that it happened without dumping */
+      log_info(LD_DIR,
+               "Unable to parse descriptor of type %s with hash %s and "
+               "length %lu. Descriptor not dumped because it exceeds maximum"
+               " log size all by itself.",
+               type, digest_sha256_hex, (unsigned long)len);
+      /* We do have to free debugfile in this case */
+    }
+  } else {
+    /*
+     * Not logging because the sandbox is active and seccomp2 apparently
+     * doesn't have a sensible way to allow filenames according to a pattern
+     * match.  (If we ever figure out how to say "allow writes to /regex/",
+     * remove this checK).
+     */
+    log_info(LD_DIR,
+             "Unable to parse descriptor of type %s with hash %s and "
+             "length %lu. Descriptor not dumped because the sandbox is "
+             "configured",
+             type, digest_sha256_hex, (unsigned long)len);
+  }
+
+  tor_free(debugfile_base);
+  tor_free(debugfile);
+
+ err:
+  return;
+}
diff --git a/src/feature/dirparse/unparseable.h b/src/feature/dirparse/unparseable.h
new file mode 100644
index 0000000000..853fe8cb0f
--- /dev/null
+++ b/src/feature/dirparse/unparseable.h
@@ -0,0 +1,56 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file unparseable.h
+ * \brief Header file for unparseable.c.
+ **/
+
+#ifndef TOR_UNPARSEABLE_H
+#define TOR_UNPARSEABLE_H
+
+#include "lib/cc/torint.h"
+
+MOCK_DECL(void,dump_desc,(const char *desc, const char *type));
+void dump_desc_fifo_cleanup(void);
+void dump_desc_init(void);
+
+#undef DEBUG_AREA_ALLOC
+#ifdef DEBUG_AREA_ALLOC
+#define DUMP_AREA(a,name) STMT_BEGIN                              \
+  size_t alloc=0, used=0;                                         \
+  memarea_get_stats((a),&alloc,&used);                            \
+  log_debug(LD_MM, "Area for %s has %lu allocated; using %lu.",   \
+            name, (unsigned long)alloc, (unsigned long)used);     \
+  STMT_END
+#else /* !(defined(DEBUG_AREA_ALLOC)) */
+#define DUMP_AREA(a,name) STMT_NIL
+#endif /* defined(DEBUG_AREA_ALLOC) */
+
+#ifdef UNPARSEABLE_PRIVATE
+
+/*
+ * One entry in the list of dumped descriptors; filename dumped to, length,
+ * SHA-256 and timestamp.
+ */
+
+typedef struct {
+  char *filename;
+  size_t len;
+  uint8_t digest_sha256[DIGEST256_LEN];
+  time_t when;
+} dumped_desc_t;
+struct smartlist_t;
+
+EXTERN(uint64_t, len_descs_dumped)
+EXTERN(struct smartlist_t *, descs_dumped)
+
+MOCK_DECL(STATIC dumped_desc_t *, dump_desc_populate_one_file,
+    (const char *dirname, const char *f));
+STATIC void dump_desc_populate_fifo_from_directory(const char *dirname);
+#endif
+
+#endif /* !defined(TOR_UNPARSEABLE_H) */
diff --git a/src/or/hibernate.c b/src/feature/hibernate/hibernate.c
index e3c80b5f14..09932c97ac 100644
--- a/src/or/hibernate.c
+++ b/src/feature/hibernate/hibernate.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -28,17 +28,37 @@ hibernating, phase 2:
 */
 
 #define HIBERNATE_PRIVATE
-#include "or.h"
-#include "channel.h"
-#include "channeltls.h"
-#include "config.h"
-#include "connection.h"
-#include "connection_edge.h"
-#include "control.h"
-#include "hibernate.h"
-#include "main.h"
-#include "router.h"
-#include "statefile.h"
+#include "core/or/or.h"
+#include "core/or/channel.h"
+#include "core/or/channeltls.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "feature/control/control.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "feature/hibernate/hibernate.h"
+#include "core/mainloop/mainloop.h"
+#include "feature/relay/router.h"
+#include "app/config/statefile.h"
+#include "lib/evloop/compat_libevent.h"
+
+#include "core/or/or_connection_st.h"
+#include "app/config/or_state_st.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifdef HAVE_SYSTEMD
+#  if defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__)
+/* Systemd's use of gcc's __INCLUDE_LEVEL__ extension macro appears to confuse
+ * Coverity. Here's a kludge to unconfuse it.
+ */
+#   define __INCLUDE_LEVEL__ 2
+# endif /* defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) */
+#include <systemd/sd-daemon.h>
+#endif /* defined(HAVE_SYSTEMD) */
 
 /** Are we currently awake, asleep, running out of bandwidth, or shutting
  * down? */
@@ -50,6 +70,10 @@ static time_t hibernate_end_time = 0;
  * we aren't shutting down. */
 static time_t shutdown_time = 0;
 
+/** A timed event that we'll use when it's time to wake up from
+ * hibernation. */
+static mainloop_event_t *wakeup_event = NULL;
+
 /** Possible accounting periods. */
 typedef enum {
   UNIT_MONTH=1, UNIT_WEEK=2, UNIT_DAY=3,
@@ -129,6 +153,8 @@ static time_t start_of_accounting_period_after(time_t now);
 static time_t start_of_accounting_period_containing(time_t now);
 static void accounting_set_wakeup_time(void);
 static void on_hibernate_state_change(hibernate_state_t prev_state);
+static void hibernate_schedule_wakeup_event(time_t now, time_t end_time);
+static void wakeup_event_callback(mainloop_event_t *ev, void *data);
 
 /**
  * Return the human-readable name for the hibernation state <b>state</b>
@@ -296,7 +322,7 @@ accounting_get_end_time,(void))
   return interval_end_time;
 }
 
-/** Called from main.c to tell us that <b>seconds</b> seconds have
+/** Called from connection.c to tell us that <b>seconds</b> seconds have
  * passed, <b>n_read</b> bytes have been read, and <b>n_written</b>
  * bytes have been written. */
 void
@@ -424,8 +450,8 @@ configure_accounting(time_t now)
     if (-0.50 <= delta && delta <= 0.50) {
       /* The start of the period is now a little later or earlier than we
        * remembered.  That's fine; we might lose some bytes we could otherwise
-       * have written, but better to err on the side of obeying people's
-       * accounting settings. */
+       * have written, but better to err on the side of obeying accounting
+       * settings. */
       log_info(LD_ACCT, "Accounting interval moved by %.02f%%; "
                "that's fine.", delta*100);
       interval_end_time = start_of_accounting_period_after(now);
@@ -587,7 +613,10 @@ accounting_set_wakeup_time(void)
     char buf[ISO_TIME_LEN+1];
     format_iso_time(buf, interval_start_time);
 
-    crypto_pk_get_digest(get_server_identity_key(), digest);
+    if (crypto_pk_get_digest(get_server_identity_key(), digest) < 0) {
+      log_err(LD_BUG, "Error getting our key's digest.");
+      tor_assert(0);
+    }
 
     d_env = crypto_digest_new();
     crypto_digest_add_bytes(d_env, buf, ISO_TIME_LEN);
@@ -748,13 +777,13 @@ read_bandwidth_usage(void)
        "Successfully read bandwidth accounting info from state written at %s "
        "for interval starting at %s.  We have been active for %lu seconds in "
        "this interval.  At the start of the interval, we expected to use "
-       "about %lu KB per second. ("U64_FORMAT" bytes read so far, "
-       U64_FORMAT" bytes written so far)",
+       "about %lu KB per second. (%"PRIu64" bytes read so far, "
+       "%"PRIu64" bytes written so far)",
        tbuf1, tbuf2,
        (unsigned long)n_seconds_active_in_interval,
        (unsigned long)(expected_bandwidth_usage*1024/60),
-       U64_PRINTF_ARG(n_bytes_read_in_interval),
-       U64_PRINTF_ARG(n_bytes_written_in_interval));
+       (n_bytes_read_in_interval),
+       (n_bytes_written_in_interval));
   }
 
   return 0;
@@ -786,7 +815,7 @@ hibernate_soft_limit_reached(void)
    *   - We have used up 95% of our bytes.
    *   - We have less than 500MB of bytes left.
    */
-  uint64_t soft_limit = DBL_TO_U64(U64_TO_DBL(acct_max) * SOFT_LIM_PCT);
+  uint64_t soft_limit = (uint64_t) (acct_max * SOFT_LIM_PCT);
   if (acct_max > SOFT_LIM_BYTES && acct_max - SOFT_LIM_BYTES > soft_limit) {
     soft_limit = acct_max - SOFT_LIM_BYTES;
   }
@@ -802,6 +831,8 @@ hibernate_soft_limit_reached(void)
   return get_accounting_bytes() >= soft_limit;
 }
 
+#define TOR_USEC_PER_SEC (1000000)
+
 /** Called when we get a SIGINT, or when bandwidth soft limit is
  * reached. Puts us into "loose hibernation": we don't accept new
  * connections, but we continue handling old ones. */
@@ -815,8 +846,8 @@ hibernate_begin(hibernate_state_t new_state, time_t now)
     log_notice(LD_GENERAL,"SIGINT received %s; exiting now.",
                hibernate_state == HIBERNATE_STATE_EXITING ?
                "a second time" : "while hibernating");
-    tor_cleanup();
-    exit(0);
+    tor_shutdown_event_loop_and_exit(0);
+    return;
   }
 
   if (new_state == HIBERNATE_STATE_LOWBANDWIDTH &&
@@ -837,6 +868,26 @@ hibernate_begin(hibernate_state_t new_state, time_t now)
                "connections, and will shut down in %d seconds. Interrupt "
                "again to exit now.", options->ShutdownWaitLength);
     shutdown_time = time(NULL) + options->ShutdownWaitLength;
+#ifdef HAVE_SYSTEMD
+    /* tell systemd that we may need more than the default 90 seconds to shut
+     * down so they don't kill us. add some extra time to actually finish
+     * shutting down, otherwise systemd will kill us immediately after the
+     * EXTEND_TIMEOUT_USEC expires. this is an *upper* limit; tor will probably
+     * only take one or two more seconds, but assume that maybe we got swapped
+     * out and it takes a little while longer.
+     *
+     * as of writing, this is a no-op with all-defaults: ShutdownWaitLength is
+     * 30 seconds, so this will extend the timeout to 60 seconds.
+     * default systemd DefaultTimeoutStopSec is 90 seconds, so systemd will
+     * wait (up to) 90 seconds anyways.
+     *
+     * 2^31 usec = ~2147 sec = ~35 min. probably nobody will actually set
+     * ShutdownWaitLength to more than that, but use a longer type so we don't
+     * need to think about UB on overflow
+     */
+    sd_notifyf(0, "EXTEND_TIMEOUT_USEC=%" PRIu64,
+            ((uint64_t)(options->ShutdownWaitLength) + 30) * TOR_USEC_PER_SEC);
+#endif
   } else { /* soft limit reached */
     hibernate_end_time = interval_end_time;
   }
@@ -862,7 +913,7 @@ hibernate_end(hibernate_state_t new_state)
 
   hibernate_state = new_state;
   hibernate_end_time = 0; /* no longer hibernating */
-  stats_n_seconds_working = 0; /* reset published uptime */
+  reset_uptime(); /* reset published uptime */
 }
 
 /** A wrapper around hibernate_begin, for when we get SIGINT. */
@@ -872,13 +923,26 @@ hibernate_begin_shutdown(void)
   hibernate_begin(HIBERNATE_STATE_EXITING, time(NULL));
 }
 
-/** Return true iff we are currently hibernating. */
+/**
+ * Return true iff we are currently hibernating -- that is, if we are in
+ * any non-live state.
+ */
 MOCK_IMPL(int,
 we_are_hibernating,(void))
 {
   return hibernate_state != HIBERNATE_STATE_LIVE;
 }
 
+/**
+ * Return true iff we are currently _fully_ hibernating -- that is, if we are
+ * in a state where we expect to handle no network activity at all.
+ */
+MOCK_IMPL(int,
+we_are_fully_hibernating,(void))
+{
+  return hibernate_state == HIBERNATE_STATE_DORMANT;
+}
+
 /** If we aren't currently dormant, close all connections and become
  * dormant. */
 static void
@@ -896,27 +960,30 @@ hibernate_go_dormant(time_t now)
   log_notice(LD_ACCT,"Going dormant. Blowing away remaining connections.");
 
   /* Close all OR/AP/exit conns. Leave dir conns because we still want
-   * to be able to upload server descriptors so people know we're still
+   * to be able to upload server descriptors so clients know we're still
    * running, and download directories so we can detect if we're obsolete.
    * Leave control conns because we still want to be controllable.
    */
   while ((conn = connection_get_by_type(CONN_TYPE_OR)) ||
          (conn = connection_get_by_type(CONN_TYPE_AP)) ||
          (conn = connection_get_by_type(CONN_TYPE_EXIT))) {
-    if (CONN_IS_EDGE(conn))
+    if (CONN_IS_EDGE(conn)) {
       connection_edge_end(TO_EDGE_CONN(conn), END_STREAM_REASON_HIBERNATING);
+    }
     log_info(LD_NET,"Closing conn type %d", conn->type);
-    if (conn->type == CONN_TYPE_AP) /* send socks failure if needed */
+    if (conn->type == CONN_TYPE_AP) {
+      /* send socks failure if needed */
       connection_mark_unattached_ap(TO_ENTRY_CONN(conn),
                                     END_STREAM_REASON_HIBERNATING);
-    else if (conn->type == CONN_TYPE_OR) {
+    } else if (conn->type == CONN_TYPE_OR) {
       if (TO_OR_CONN(conn)->chan) {
-        channel_mark_for_close(TLS_CHAN_TO_BASE(TO_OR_CONN(conn)->chan));
+        connection_or_close_normally(TO_OR_CONN(conn), 0);
       } else {
          connection_mark_for_close(conn);
       }
-    } else
+    } else {
       connection_mark_for_close(conn);
+    }
   }
 
   if (now < interval_wakeup_time)
@@ -928,6 +995,63 @@ hibernate_go_dormant(time_t now)
 
   or_state_mark_dirty(get_or_state(),
                       get_options()->AvoidDiskWrites ? now+600 : 0);
+
+  hibernate_schedule_wakeup_event(now, hibernate_end_time);
+}
+
+/**
+ * Schedule a mainloop event at <b>end_time</b> to wake up from a dormant
+ * state.  We can't rely on this happening from second_elapsed_callback,
+ * since second_elapsed_callback will be shut down when we're dormant.
+ *
+ * (Note that We might immediately go back to sleep after we set the next
+ * wakeup time.)
+ */
+static void
+hibernate_schedule_wakeup_event(time_t now, time_t end_time)
+{
+  struct timeval delay = { 0, 0 };
+
+  if (now >= end_time) {
+    // In these cases we always wait at least a second, to avoid running
+    // the callback in a tight loop.
+    delay.tv_sec = 1;
+  } else {
+    delay.tv_sec = (end_time - now);
+  }
+
+  if (!wakeup_event) {
+    wakeup_event = mainloop_event_postloop_new(wakeup_event_callback, NULL);
+  }
+
+  mainloop_event_schedule(wakeup_event, &delay);
+}
+
+/**
+ * Called at the end of the interval, or at the wakeup time of the current
+ * interval, to exit the dormant state.
+ **/
+static void
+wakeup_event_callback(mainloop_event_t *ev, void *data)
+{
+  (void) ev;
+  (void) data;
+
+  const time_t now = time(NULL);
+  accounting_run_housekeeping(now);
+  consider_hibernation(now);
+  if (hibernate_state != HIBERNATE_STATE_DORMANT) {
+    /* We woke up, so everything's great here */
+    return;
+  }
+
+  /* We're still dormant. */
+  if (now < interval_wakeup_time)
+    hibernate_end_time = interval_wakeup_time;
+  else
+    hibernate_end_time = interval_end_time;
+
+  hibernate_schedule_wakeup_event(now, hibernate_end_time);
 }
 
 /** Called when hibernate_end_time has arrived. */
@@ -977,8 +1101,7 @@ consider_hibernation(time_t now)
     tor_assert(shutdown_time);
     if (shutdown_time <= now) {
       log_notice(LD_GENERAL, "Clean shutdown finished. Exiting.");
-      tor_cleanup();
-      exit(0);
+      tor_shutdown_event_loop_and_exit(0);
     }
     return; /* if exiting soon, don't worry about bandwidth limits */
   }
@@ -1049,9 +1172,9 @@ getinfo_helper_accounting(control_connection_t *conn,
     *answer = tor_strdup(hibernate_state_to_string(hibernate_state));
     tor_strlower(*answer);
   } else if (!strcmp(question, "accounting/bytes")) {
-      tor_asprintf(answer, U64_FORMAT" "U64_FORMAT,
-                 U64_PRINTF_ARG(n_bytes_read_in_interval),
-                 U64_PRINTF_ARG(n_bytes_written_in_interval));
+      tor_asprintf(answer, "%"PRIu64" %"PRIu64,
+                 (n_bytes_read_in_interval),
+                 (n_bytes_written_in_interval));
   } else if (!strcmp(question, "accounting/bytes-left")) {
     uint64_t limit = get_options()->AccountingMax;
     if (get_options()->AccountingRule == ACCT_SUM) {
@@ -1059,28 +1182,28 @@ getinfo_helper_accounting(control_connection_t *conn,
       uint64_t total_bytes = get_accounting_bytes();
       if (total_bytes < limit)
         total_left = limit - total_bytes;
-      tor_asprintf(answer, U64_FORMAT" "U64_FORMAT,
-                   U64_PRINTF_ARG(total_left), U64_PRINTF_ARG(total_left));
+      tor_asprintf(answer, "%"PRIu64" %"PRIu64,
+                   (total_left), (total_left));
     } else if (get_options()->AccountingRule == ACCT_IN) {
       uint64_t read_left = 0;
       if (n_bytes_read_in_interval < limit)
         read_left = limit - n_bytes_read_in_interval;
-      tor_asprintf(answer, U64_FORMAT" "U64_FORMAT,
-                   U64_PRINTF_ARG(read_left), U64_PRINTF_ARG(limit));
+      tor_asprintf(answer, "%"PRIu64" %"PRIu64,
+                   (read_left), (limit));
     } else if (get_options()->AccountingRule == ACCT_OUT) {
       uint64_t write_left = 0;
       if (n_bytes_written_in_interval < limit)
         write_left = limit - n_bytes_written_in_interval;
-      tor_asprintf(answer, U64_FORMAT" "U64_FORMAT,
-                   U64_PRINTF_ARG(limit), U64_PRINTF_ARG(write_left));
+      tor_asprintf(answer, "%"PRIu64" %"PRIu64,
+                   (limit), (write_left));
     } else {
       uint64_t read_left = 0, write_left = 0;
       if (n_bytes_read_in_interval < limit)
         read_left = limit - n_bytes_read_in_interval;
       if (n_bytes_written_in_interval < limit)
         write_left = limit - n_bytes_written_in_interval;
-      tor_asprintf(answer, U64_FORMAT" "U64_FORMAT,
-                   U64_PRINTF_ARG(read_left), U64_PRINTF_ARG(write_left));
+      tor_asprintf(answer, "%"PRIu64" %"PRIu64,
+                   (read_left), (write_left));
     }
   } else if (!strcmp(question, "accounting/interval-start")) {
     *answer = tor_malloc(ISO_TIME_LEN+1);
@@ -1105,10 +1228,30 @@ getinfo_helper_accounting(control_connection_t *conn,
 static void
 on_hibernate_state_change(hibernate_state_t prev_state)
 {
-  (void)prev_state; /* Should we do something with this? */
   control_event_server_status(LOG_NOTICE,
                               "HIBERNATION_STATUS STATUS=%s",
                               hibernate_state_to_string(hibernate_state));
+
+  /* We are changing hibernation state, this can affect the main loop event
+   * list. Rescan it to update the events state. We do this whatever the new
+   * hibernation state because they can each possibly affect an event. The
+   * initial state means we are booting up so we shouldn't scan here because
+   * at this point the events in the list haven't been initialized. */
+  if (prev_state != HIBERNATE_STATE_INITIAL) {
+    rescan_periodic_events(get_options());
+  }
+
+  reschedule_per_second_timer();
+}
+
+/** Free all resources held by the accounting module */
+void
+accounting_free_all(void)
+{
+  mainloop_event_free(wakeup_event);
+  hibernate_state = HIBERNATE_STATE_INITIAL;
+  hibernate_end_time = 0;
+  shutdown_time = 0;
 }
 
 #ifdef TOR_UNIT_TESTS
@@ -1121,5 +1264,4 @@ hibernate_set_state_for_testing_(hibernate_state_t newstate)
 {
   hibernate_state = newstate;
 }
-#endif
-
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/or/hibernate.h b/src/feature/hibernate/hibernate.h
index fa9da6de39..3309ef0ce3 100644
--- a/src/or/hibernate.h
+++ b/src/feature/hibernate/hibernate.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,7 +12,7 @@
 #ifndef TOR_HIBERNATE_H
 #define TOR_HIBERNATE_H
 
-#include "testsupport.h"
+#include "lib/testsupport/testsupport.h"
 
 int accounting_parse_options(const or_options_t *options, int validate_only);
 MOCK_DECL(int, accounting_is_enabled, (const or_options_t *options));
@@ -25,11 +25,13 @@ void accounting_add_bytes(size_t n_read, size_t n_written, int seconds);
 int accounting_record_bandwidth_usage(time_t now, or_state_t *state);
 void hibernate_begin_shutdown(void);
 MOCK_DECL(int, we_are_hibernating, (void));
+MOCK_DECL(int, we_are_fully_hibernating,(void));
 void consider_hibernation(time_t now);
 int getinfo_helper_accounting(control_connection_t *conn,
                               const char *question, char **answer,
                               const char **errmsg);
 uint64_t get_accounting_max_total(void);
+void accounting_free_all(void);
 
 #ifdef HIBERNATE_PRIVATE
 /** Possible values of hibernate_state */
@@ -53,7 +55,7 @@ typedef enum {
 #ifdef TOR_UNIT_TESTS
 void hibernate_set_state_for_testing_(hibernate_state_t newstate);
 #endif
-#endif
+#endif /* defined(HIBERNATE_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_HIBERNATE_H) */
 
diff --git a/src/feature/hs/hs_cache.c b/src/feature/hs/hs_cache.c
new file mode 100644
index 0000000000..05f9940ae6
--- /dev/null
+++ b/src/feature/hs/hs_cache.c
@@ -0,0 +1,986 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_cache.c
+ * \brief Handle hidden service descriptor caches.
+ **/
+
+/* For unit tests.*/
+#define HS_CACHE_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "feature/hs/hs_ident.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_client.h"
+#include "feature/hs/hs_descriptor.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/rend/rendcache.h"
+
+#include "feature/hs/hs_cache.h"
+
+#include "feature/nodelist/networkstatus_st.h"
+
+static int cached_client_descriptor_has_expired(time_t now,
+           const hs_cache_client_descriptor_t *cached_desc);
+
+/********************** Directory HS cache ******************/
+
+/* Directory descriptor cache. Map indexed by blinded key. */
+static digest256map_t *hs_cache_v3_dir;
+
+/* Remove a given descriptor from our cache. */
+static void
+remove_v3_desc_as_dir(const hs_cache_dir_descriptor_t *desc)
+{
+  tor_assert(desc);
+  digest256map_remove(hs_cache_v3_dir, desc->key);
+}
+
+/* Store a given descriptor in our cache. */
+static void
+store_v3_desc_as_dir(hs_cache_dir_descriptor_t *desc)
+{
+  tor_assert(desc);
+  digest256map_set(hs_cache_v3_dir, desc->key, desc);
+}
+
+/* Query our cache and return the entry or NULL if not found. */
+static hs_cache_dir_descriptor_t *
+lookup_v3_desc_as_dir(const uint8_t *key)
+{
+  tor_assert(key);
+  return digest256map_get(hs_cache_v3_dir, key);
+}
+
+#define cache_dir_desc_free(val) \
+  FREE_AND_NULL(hs_cache_dir_descriptor_t, cache_dir_desc_free_, (val))
+
+/* Free a directory descriptor object. */
+static void
+cache_dir_desc_free_(hs_cache_dir_descriptor_t *desc)
+{
+  if (desc == NULL) {
+    return;
+  }
+  hs_desc_plaintext_data_free(desc->plaintext_data);
+  tor_free(desc->encoded_desc);
+  tor_free(desc);
+}
+
+/* Helper function: Use by the free all function using the digest256map
+ * interface to cache entries. */
+static void
+cache_dir_desc_free_void(void *ptr)
+{
+  cache_dir_desc_free_(ptr);
+}
+
+/* Create a new directory cache descriptor object from a encoded descriptor.
+ * On success, return the heap-allocated cache object, otherwise return NULL if
+ * we can't decode the descriptor. */
+static hs_cache_dir_descriptor_t *
+cache_dir_desc_new(const char *desc)
+{
+  hs_cache_dir_descriptor_t *dir_desc;
+
+  tor_assert(desc);
+
+  dir_desc = tor_malloc_zero(sizeof(hs_cache_dir_descriptor_t));
+  dir_desc->plaintext_data =
+    tor_malloc_zero(sizeof(hs_desc_plaintext_data_t));
+  dir_desc->encoded_desc = tor_strdup(desc);
+
+  if (hs_desc_decode_plaintext(desc, dir_desc->plaintext_data) < 0) {
+    log_debug(LD_DIR, "Unable to decode descriptor. Rejecting.");
+    goto err;
+  }
+
+  /* The blinded pubkey is the indexed key. */
+  dir_desc->key = dir_desc->plaintext_data->blinded_pubkey.pubkey;
+  dir_desc->created_ts = time(NULL);
+  return dir_desc;
+
+ err:
+  cache_dir_desc_free(dir_desc);
+  return NULL;
+}
+
+/* Return the size of a cache entry in bytes. */
+static size_t
+cache_get_dir_entry_size(const hs_cache_dir_descriptor_t *entry)
+{
+  return (sizeof(*entry) + hs_desc_plaintext_obj_size(entry->plaintext_data)
+          + strlen(entry->encoded_desc));
+}
+
+/* Try to store a valid version 3 descriptor in the directory cache. Return 0
+ * on success else a negative value is returned indicating that we have a
+ * newer version in our cache. On error, caller is responsible to free the
+ * given descriptor desc. */
+static int
+cache_store_v3_as_dir(hs_cache_dir_descriptor_t *desc)
+{
+  hs_cache_dir_descriptor_t *cache_entry;
+
+  tor_assert(desc);
+
+  /* Verify if we have an entry in the cache for that key and if yes, check
+   * if we should replace it? */
+  cache_entry = lookup_v3_desc_as_dir(desc->key);
+  if (cache_entry != NULL) {
+    /* Only replace descriptor if revision-counter is greater than the one
+     * in our cache */
+    if (cache_entry->plaintext_data->revision_counter >=
+        desc->plaintext_data->revision_counter) {
+      log_info(LD_REND, "Descriptor revision counter in our cache is "
+               "greater or equal than the one we received (%d/%d). "
+               "Rejecting!",
+               (int)cache_entry->plaintext_data->revision_counter,
+               (int)desc->plaintext_data->revision_counter);
+      goto err;
+    }
+    /* We now know that the descriptor we just received is a new one so
+     * remove the entry we currently have from our cache so we can then
+     * store the new one. */
+    remove_v3_desc_as_dir(cache_entry);
+    rend_cache_decrement_allocation(cache_get_dir_entry_size(cache_entry));
+    cache_dir_desc_free(cache_entry);
+  }
+  /* Store the descriptor we just got. We are sure here that either we
+   * don't have the entry or we have a newer descriptor and the old one
+   * has been removed from the cache. */
+  store_v3_desc_as_dir(desc);
+
+  /* Update our total cache size with this entry for the OOM. This uses the
+   * old HS protocol cache subsystem for which we are tied with. */
+  rend_cache_increment_allocation(cache_get_dir_entry_size(desc));
+
+  /* XXX: Update HS statistics. We should have specific stats for v3. */
+
+  return 0;
+
+ err:
+  return -1;
+}
+
+/* Using the query which is the base64 encoded blinded key of a version 3
+ * descriptor, lookup in our directory cache the entry. If found, 1 is
+ * returned and desc_out is populated with a newly allocated string being the
+ * encoded descriptor. If not found, 0 is returned and desc_out is untouched.
+ * On error, a negative value is returned and desc_out is untouched. */
+static int
+cache_lookup_v3_as_dir(const char *query, const char **desc_out)
+{
+  int found = 0;
+  ed25519_public_key_t blinded_key;
+  const hs_cache_dir_descriptor_t *entry;
+
+  tor_assert(query);
+
+  /* Decode blinded key using the given query value. */
+  if (ed25519_public_from_base64(&blinded_key, query) < 0) {
+    log_info(LD_REND, "Unable to decode the v3 HSDir query %s.",
+             safe_str_client(query));
+    goto err;
+  }
+
+  entry = lookup_v3_desc_as_dir(blinded_key.pubkey);
+  if (entry != NULL) {
+    found = 1;
+    if (desc_out) {
+      *desc_out = entry->encoded_desc;
+    }
+  }
+
+  return found;
+
+ err:
+  return -1;
+}
+
+/* Clean the v3 cache by removing any entry that has expired using the
+ * <b>global_cutoff</b> value. If <b>global_cutoff</b> is 0, the cleaning
+ * process will use the lifetime found in the plaintext data section. Return
+ * the number of bytes cleaned. */
+STATIC size_t
+cache_clean_v3_as_dir(time_t now, time_t global_cutoff)
+{
+  size_t bytes_removed = 0;
+
+  /* Code flow error if this ever happens. */
+  tor_assert(global_cutoff >= 0);
+
+  if (!hs_cache_v3_dir) { /* No cache to clean. Just return. */
+    return 0;
+  }
+
+  DIGEST256MAP_FOREACH_MODIFY(hs_cache_v3_dir, key,
+                              hs_cache_dir_descriptor_t *, entry) {
+    size_t entry_size;
+    time_t cutoff = global_cutoff;
+    if (!cutoff) {
+      /* Cutoff is the lifetime of the entry found in the descriptor. */
+      cutoff = now - entry->plaintext_data->lifetime_sec;
+    }
+
+    /* If the entry has been created _after_ the cutoff, not expired so
+     * continue to the next entry in our v3 cache. */
+    if (entry->created_ts > cutoff) {
+      continue;
+    }
+    /* Here, our entry has expired, remove and free. */
+    MAP_DEL_CURRENT(key);
+    entry_size = cache_get_dir_entry_size(entry);
+    bytes_removed += entry_size;
+    /* Entry is not in the cache anymore, destroy it. */
+    cache_dir_desc_free(entry);
+    /* Update our cache entry allocation size for the OOM. */
+    rend_cache_decrement_allocation(entry_size);
+    /* Logging. */
+    {
+      char key_b64[BASE64_DIGEST256_LEN + 1];
+      digest256_to_base64(key_b64, (const char *) key);
+      log_info(LD_REND, "Removing v3 descriptor '%s' from HSDir cache",
+               safe_str_client(key_b64));
+    }
+  } DIGEST256MAP_FOREACH_END;
+
+  return bytes_removed;
+}
+
+/* Given an encoded descriptor, store it in the directory cache depending on
+ * which version it is. Return a negative value on error. On success, 0 is
+ * returned. */
+int
+hs_cache_store_as_dir(const char *desc)
+{
+  hs_cache_dir_descriptor_t *dir_desc = NULL;
+
+  tor_assert(desc);
+
+  /* Create a new cache object. This can fail if the descriptor plaintext data
+   * is unparseable which in this case a log message will be triggered. */
+  dir_desc = cache_dir_desc_new(desc);
+  if (dir_desc == NULL) {
+    goto err;
+  }
+
+  /* Call the right function against the descriptor version. At this point,
+   * we are sure that the descriptor's version is supported else the
+   * decoding would have failed. */
+  switch (dir_desc->plaintext_data->version) {
+  case HS_VERSION_THREE:
+  default:
+    if (cache_store_v3_as_dir(dir_desc) < 0) {
+      goto err;
+    }
+    break;
+  }
+  return 0;
+
+ err:
+  cache_dir_desc_free(dir_desc);
+  return -1;
+}
+
+/* Using the query, lookup in our directory cache the entry. If found, 1 is
+ * returned and desc_out is populated with a newly allocated string being
+ * the encoded descriptor. If not found, 0 is returned and desc_out is
+ * untouched. On error, a negative value is returned and desc_out is
+ * untouched. */
+int
+hs_cache_lookup_as_dir(uint32_t version, const char *query,
+                       const char **desc_out)
+{
+  int found;
+
+  tor_assert(query);
+  /* This should never be called with an unsupported version. */
+  tor_assert(hs_desc_is_supported_version(version));
+
+  switch (version) {
+  case HS_VERSION_THREE:
+  default:
+    found = cache_lookup_v3_as_dir(query, desc_out);
+    break;
+  }
+
+  return found;
+}
+
+/* Clean all directory caches using the current time now. */
+void
+hs_cache_clean_as_dir(time_t now)
+{
+  time_t cutoff;
+
+  /* Start with v2 cache cleaning. */
+  cutoff = now - rend_cache_max_entry_lifetime();
+  rend_cache_clean_v2_descs_as_dir(cutoff);
+
+  /* Now, clean the v3 cache. Set the cutoff to 0 telling the cleanup function
+   * to compute the cutoff by itself using the lifetime value. */
+  cache_clean_v3_as_dir(now, 0);
+}
+
+/********************** Client-side HS cache ******************/
+
+/* Client-side HS descriptor cache. Map indexed by service identity key. */
+static digest256map_t *hs_cache_v3_client;
+
+/* Client-side introduction point state cache. Map indexed by service public
+ * identity key (onion address). It contains hs_cache_client_intro_state_t
+ * objects all related to a specific service. */
+static digest256map_t *hs_cache_client_intro_state;
+
+/* Return the size of a client cache entry in bytes. */
+static size_t
+cache_get_client_entry_size(const hs_cache_client_descriptor_t *entry)
+{
+  return sizeof(*entry) +
+         strlen(entry->encoded_desc) + hs_desc_obj_size(entry->desc);
+}
+
+/* Remove a given descriptor from our cache. */
+static void
+remove_v3_desc_as_client(const hs_cache_client_descriptor_t *desc)
+{
+  tor_assert(desc);
+  digest256map_remove(hs_cache_v3_client, desc->key.pubkey);
+  /* Update cache size with this entry for the OOM handler. */
+  rend_cache_decrement_allocation(cache_get_client_entry_size(desc));
+}
+
+/* Store a given descriptor in our cache. */
+static void
+store_v3_desc_as_client(hs_cache_client_descriptor_t *desc)
+{
+  tor_assert(desc);
+  digest256map_set(hs_cache_v3_client, desc->key.pubkey, desc);
+  /* Update cache size with this entry for the OOM handler. */
+  rend_cache_increment_allocation(cache_get_client_entry_size(desc));
+}
+
+/* Query our cache and return the entry or NULL if not found or if expired. */
+STATIC hs_cache_client_descriptor_t *
+lookup_v3_desc_as_client(const uint8_t *key)
+{
+  time_t now = approx_time();
+  hs_cache_client_descriptor_t *cached_desc;
+
+  tor_assert(key);
+
+  /* Do the lookup */
+  cached_desc = digest256map_get(hs_cache_v3_client, key);
+  if (!cached_desc) {
+    return NULL;
+  }
+
+  /* Don't return expired entries */
+  if (cached_client_descriptor_has_expired(now, cached_desc)) {
+    return NULL;
+  }
+
+  return cached_desc;
+}
+
+/* Parse the encoded descriptor in <b>desc_str</b> using
+ * <b>service_identity_pk<b> to decrypt it first.
+ *
+ * If everything goes well, allocate and return a new
+ * hs_cache_client_descriptor_t object. In case of error, return NULL. */
+static hs_cache_client_descriptor_t *
+cache_client_desc_new(const char *desc_str,
+                      const ed25519_public_key_t *service_identity_pk)
+{
+  hs_descriptor_t *desc = NULL;
+  hs_cache_client_descriptor_t *client_desc = NULL;
+
+  tor_assert(desc_str);
+  tor_assert(service_identity_pk);
+
+  /* Decode the descriptor we just fetched. */
+  if (hs_client_decode_descriptor(desc_str, service_identity_pk, &desc) < 0) {
+    goto end;
+  }
+  tor_assert(desc);
+
+  /* All is good: make a cache object for this descriptor */
+  client_desc = tor_malloc_zero(sizeof(hs_cache_client_descriptor_t));
+  ed25519_pubkey_copy(&client_desc->key, service_identity_pk);
+  /* Set expiration time for this cached descriptor to be the start of the next
+   * time period since that's when clients need to start using the next blinded
+   * pk of the service (and hence will need its next descriptor). */
+  client_desc->expiration_ts = hs_get_start_time_of_next_time_period(0);
+  client_desc->desc = desc;
+  client_desc->encoded_desc = tor_strdup(desc_str);
+
+ end:
+  return client_desc;
+}
+
+#define cache_client_desc_free(val) \
+  FREE_AND_NULL(hs_cache_client_descriptor_t, cache_client_desc_free_, (val))
+
+/** Free memory allocated by <b>desc</b>. */
+static void
+cache_client_desc_free_(hs_cache_client_descriptor_t *desc)
+{
+  if (desc == NULL) {
+    return;
+  }
+  hs_descriptor_free(desc->desc);
+  memwipe(&desc->key, 0, sizeof(desc->key));
+  memwipe(desc->encoded_desc, 0, strlen(desc->encoded_desc));
+  tor_free(desc->encoded_desc);
+  tor_free(desc);
+}
+
+/** Helper function: Use by the free all function to clear the client cache */
+static void
+cache_client_desc_free_void(void *ptr)
+{
+  hs_cache_client_descriptor_t *desc = ptr;
+  cache_client_desc_free(desc);
+}
+
+/* Return a newly allocated and initialized hs_cache_intro_state_t object. */
+static hs_cache_intro_state_t *
+cache_intro_state_new(void)
+{
+  hs_cache_intro_state_t *state = tor_malloc_zero(sizeof(*state));
+  state->created_ts = approx_time();
+  return state;
+}
+
+#define cache_intro_state_free(val) \
+  FREE_AND_NULL(hs_cache_intro_state_t, cache_intro_state_free_, (val))
+
+/* Free an hs_cache_intro_state_t object. */
+static void
+cache_intro_state_free_(hs_cache_intro_state_t *state)
+{
+  tor_free(state);
+}
+
+/* Helper function: used by the free all function. */
+static void
+cache_intro_state_free_void(void *state)
+{
+  cache_intro_state_free_(state);
+}
+
+/* Return a newly allocated and initialized hs_cache_client_intro_state_t
+ * object. */
+static hs_cache_client_intro_state_t *
+cache_client_intro_state_new(void)
+{
+  hs_cache_client_intro_state_t *cache = tor_malloc_zero(sizeof(*cache));
+  cache->intro_points = digest256map_new();
+  return cache;
+}
+
+#define cache_client_intro_state_free(val)              \
+  FREE_AND_NULL(hs_cache_client_intro_state_t,          \
+                cache_client_intro_state_free_, (val))
+
+/* Free a cache_client_intro_state object. */
+static void
+cache_client_intro_state_free_(hs_cache_client_intro_state_t *cache)
+{
+  if (cache == NULL) {
+    return;
+  }
+  digest256map_free(cache->intro_points, cache_intro_state_free_void);
+  tor_free(cache);
+}
+
+/* Helper function: used by the free all function. */
+static void
+cache_client_intro_state_free_void(void *entry)
+{
+  cache_client_intro_state_free_(entry);
+}
+
+/* For the given service identity key service_pk and an introduction
+ * authentication key auth_key, lookup the intro state object. Return 1 if
+ * found and put it in entry if not NULL. Return 0 if not found and entry is
+ * untouched. */
+static int
+cache_client_intro_state_lookup(const ed25519_public_key_t *service_pk,
+                                const ed25519_public_key_t *auth_key,
+                                hs_cache_intro_state_t **entry)
+{
+  hs_cache_intro_state_t *state;
+  hs_cache_client_intro_state_t *cache;
+
+  tor_assert(service_pk);
+  tor_assert(auth_key);
+
+  /* Lookup the intro state cache for this service key. */
+  cache = digest256map_get(hs_cache_client_intro_state, service_pk->pubkey);
+  if (cache == NULL) {
+    goto not_found;
+  }
+
+  /* From the cache we just found for the service, lookup in the introduction
+   * points map for the given authentication key. */
+  state = digest256map_get(cache->intro_points, auth_key->pubkey);
+  if (state == NULL) {
+    goto not_found;
+  }
+  if (entry) {
+    *entry = state;
+  }
+  return 1;
+ not_found:
+  return 0;
+}
+
+/* Note the given failure in state. */
+static void
+cache_client_intro_state_note(hs_cache_intro_state_t *state,
+                              rend_intro_point_failure_t failure)
+{
+  tor_assert(state);
+  switch (failure) {
+  case INTRO_POINT_FAILURE_GENERIC:
+    state->error = 1;
+    break;
+  case INTRO_POINT_FAILURE_TIMEOUT:
+    state->timed_out = 1;
+    break;
+  case INTRO_POINT_FAILURE_UNREACHABLE:
+    state->unreachable_count++;
+    break;
+  default:
+    tor_assert_nonfatal_unreached();
+    return;
+  }
+}
+
+/* For the given service identity key service_pk and an introduction
+ * authentication key auth_key, add an entry in the client intro state cache
+ * If no entry exists for the service, it will create one. If state is non
+ * NULL, it will point to the new intro state entry. */
+static void
+cache_client_intro_state_add(const ed25519_public_key_t *service_pk,
+                             const ed25519_public_key_t *auth_key,
+                             hs_cache_intro_state_t **state)
+{
+  hs_cache_intro_state_t *entry, *old_entry;
+  hs_cache_client_intro_state_t *cache;
+
+  tor_assert(service_pk);
+  tor_assert(auth_key);
+
+  /* Lookup the state cache for this service key. */
+  cache = digest256map_get(hs_cache_client_intro_state, service_pk->pubkey);
+  if (cache == NULL) {
+    cache = cache_client_intro_state_new();
+    digest256map_set(hs_cache_client_intro_state, service_pk->pubkey, cache);
+  }
+
+  entry = cache_intro_state_new();
+  old_entry = digest256map_set(cache->intro_points, auth_key->pubkey, entry);
+  /* This should never happened because the code flow is to lookup the entry
+   * before adding it. But, just in case, non fatal assert and free it. */
+  tor_assert_nonfatal(old_entry == NULL);
+  tor_free(old_entry);
+
+  if (state) {
+    *state = entry;
+  }
+}
+
+/* Remove every intro point state entry from cache that has been created
+ * before or at the cutoff. */
+static void
+cache_client_intro_state_clean(time_t cutoff,
+                               hs_cache_client_intro_state_t *cache)
+{
+  tor_assert(cache);
+
+  DIGEST256MAP_FOREACH_MODIFY(cache->intro_points, key,
+                              hs_cache_intro_state_t *, entry) {
+    if (entry->created_ts <= cutoff) {
+      cache_intro_state_free(entry);
+      MAP_DEL_CURRENT(key);
+    }
+  } DIGEST256MAP_FOREACH_END;
+}
+
+/* Return true iff no intro points are in this cache. */
+static int
+cache_client_intro_state_is_empty(const hs_cache_client_intro_state_t *cache)
+{
+  return digest256map_isempty(cache->intro_points);
+}
+
+/** Check whether <b>client_desc</b> is useful for us, and store it in the
+ *  client-side HS cache if so. The client_desc is freed if we already have a
+ *  fresher (higher revision counter count) in the cache. */
+static int
+cache_store_as_client(hs_cache_client_descriptor_t *client_desc)
+{
+  hs_cache_client_descriptor_t *cache_entry;
+
+  /* TODO: Heavy code duplication with cache_store_as_dir(). Consider
+   * refactoring and uniting! */
+
+  tor_assert(client_desc);
+
+  /* Check if we already have a descriptor from this HS in cache. If we do,
+   * check if this descriptor is newer than the cached one */
+  cache_entry = lookup_v3_desc_as_client(client_desc->key.pubkey);
+  if (cache_entry != NULL) {
+    /* If we have an entry in our cache that has a revision counter greater
+     * than the one we just fetched, discard the one we fetched. */
+    if (cache_entry->desc->plaintext_data.revision_counter >
+        client_desc->desc->plaintext_data.revision_counter) {
+      cache_client_desc_free(client_desc);
+      goto done;
+    }
+    /* Remove old entry. Make space for the new one! */
+    remove_v3_desc_as_client(cache_entry);
+
+    /* We just removed an old descriptor and will replace it. We'll close all
+     * intro circuits related to this old one so we don't have leftovers. We
+     * leave the rendezvous circuits opened because they could be in use. */
+    hs_client_close_intro_circuits_from_desc(cache_entry->desc);
+
+    /* Free it. */
+    cache_client_desc_free(cache_entry);
+  }
+
+  /* Store descriptor in cache */
+  store_v3_desc_as_client(client_desc);
+
+ done:
+  return 0;
+}
+
+/* Return true iff the cached client descriptor at <b>cached_desc</b has
+ * expired. */
+static int
+cached_client_descriptor_has_expired(time_t now,
+                               const hs_cache_client_descriptor_t *cached_desc)
+{
+  /* We use the current consensus time to see if we should expire this
+   * descriptor since we use consensus time for all other parts of the protocol
+   * as well (e.g. to build the blinded key and compute time periods). */
+  const networkstatus_t *ns = networkstatus_get_live_consensus(now);
+  /* If we don't have a recent consensus, consider this entry expired since we
+   * will want to fetch a new HS desc when we get a live consensus. */
+  if (!ns) {
+    return 1;
+  }
+
+  if (cached_desc->expiration_ts <= ns->valid_after) {
+    return 1;
+  }
+
+  return 0;
+}
+
+/* clean the client cache using now as the current time. Return the total size
+ * of removed bytes from the cache. */
+static size_t
+cache_clean_v3_as_client(time_t now)
+{
+  size_t bytes_removed = 0;
+
+  if (!hs_cache_v3_client) { /* No cache to clean. Just return. */
+    return 0;
+  }
+
+  DIGEST256MAP_FOREACH_MODIFY(hs_cache_v3_client, key,
+                              hs_cache_client_descriptor_t *, entry) {
+    size_t entry_size;
+
+    /* If the entry has not expired, continue to the next cached entry */
+    if (!cached_client_descriptor_has_expired(now, entry)) {
+      continue;
+    }
+    /* Here, our entry has expired, remove and free. */
+    MAP_DEL_CURRENT(key);
+    entry_size = cache_get_client_entry_size(entry);
+    bytes_removed += entry_size;
+    /* Entry is not in the cache anymore, destroy it. */
+    cache_client_desc_free(entry);
+    /* Update our OOM. We didn't use the remove() function because we are in
+     * a loop so we have to explicitly decrement. */
+    rend_cache_decrement_allocation(entry_size);
+    /* Logging. */
+    {
+      char key_b64[BASE64_DIGEST256_LEN + 1];
+      digest256_to_base64(key_b64, (const char *) key);
+      log_info(LD_REND, "Removing hidden service v3 descriptor '%s' "
+                        "from client cache",
+               safe_str_client(key_b64));
+    }
+  } DIGEST256MAP_FOREACH_END;
+
+  return bytes_removed;
+}
+
+/** Public API: Given the HS ed25519 identity public key in <b>key</b>, return
+ *  its HS encoded descriptor if it's stored in our cache, or NULL if not. */
+const char *
+hs_cache_lookup_encoded_as_client(const ed25519_public_key_t *key)
+{
+  hs_cache_client_descriptor_t *cached_desc = NULL;
+
+  tor_assert(key);
+
+  cached_desc = lookup_v3_desc_as_client(key->pubkey);
+  if (cached_desc) {
+    tor_assert(cached_desc->encoded_desc);
+    return cached_desc->encoded_desc;
+  }
+
+  return NULL;
+}
+
+/** Public API: Given the HS ed25519 identity public key in <b>key</b>, return
+ *  its HS descriptor if it's stored in our cache, or NULL if not. */
+const hs_descriptor_t *
+hs_cache_lookup_as_client(const ed25519_public_key_t *key)
+{
+  hs_cache_client_descriptor_t *cached_desc = NULL;
+
+  tor_assert(key);
+
+  cached_desc = lookup_v3_desc_as_client(key->pubkey);
+  if (cached_desc) {
+    tor_assert(cached_desc->desc);
+    return cached_desc->desc;
+  }
+
+  return NULL;
+}
+
+/** Public API: Given an encoded descriptor, store it in the client HS
+ *  cache. Return -1 on error, 0 on success .*/
+int
+hs_cache_store_as_client(const char *desc_str,
+                         const ed25519_public_key_t *identity_pk)
+{
+  hs_cache_client_descriptor_t *client_desc = NULL;
+
+  tor_assert(desc_str);
+  tor_assert(identity_pk);
+
+  /* Create client cache descriptor object */
+  client_desc = cache_client_desc_new(desc_str, identity_pk);
+  if (!client_desc) {
+    log_warn(LD_GENERAL, "HSDesc parsing failed!");
+    log_debug(LD_GENERAL, "Failed to parse HSDesc: %s.", escaped(desc_str));
+    goto err;
+  }
+
+  /* Push it to the cache */
+  if (cache_store_as_client(client_desc) < 0) {
+    goto err;
+  }
+
+  return 0;
+
+ err:
+  cache_client_desc_free(client_desc);
+  return -1;
+}
+
+/* Clean all client caches using the current time now. */
+void
+hs_cache_clean_as_client(time_t now)
+{
+  /* Start with v2 cache cleaning. */
+  rend_cache_clean(now, REND_CACHE_TYPE_CLIENT);
+  /* Now, clean the v3 cache. Set the cutoff to 0 telling the cleanup function
+   * to compute the cutoff by itself using the lifetime value. */
+  cache_clean_v3_as_client(now);
+}
+
+/* Purge the client descriptor cache. */
+void
+hs_cache_purge_as_client(void)
+{
+  DIGEST256MAP_FOREACH_MODIFY(hs_cache_v3_client, key,
+                              hs_cache_client_descriptor_t *, entry) {
+    size_t entry_size = cache_get_client_entry_size(entry);
+    MAP_DEL_CURRENT(key);
+    cache_client_desc_free(entry);
+    /* Update our OOM. We didn't use the remove() function because we are in
+     * a loop so we have to explicitly decrement. */
+    rend_cache_decrement_allocation(entry_size);
+  } DIGEST256MAP_FOREACH_END;
+
+  log_info(LD_REND, "Hidden service client descriptor cache purged.");
+}
+
+/* For a given service identity public key and an introduction authentication
+ * key, note the given failure in the client intro state cache. */
+void
+hs_cache_client_intro_state_note(const ed25519_public_key_t *service_pk,
+                                 const ed25519_public_key_t *auth_key,
+                                 rend_intro_point_failure_t failure)
+{
+  int found;
+  hs_cache_intro_state_t *entry;
+
+  tor_assert(service_pk);
+  tor_assert(auth_key);
+
+  found = cache_client_intro_state_lookup(service_pk, auth_key, &entry);
+  if (!found) {
+    /* Create a new entry and add it to the cache. */
+    cache_client_intro_state_add(service_pk, auth_key, &entry);
+  }
+  /* Note down the entry. */
+  cache_client_intro_state_note(entry, failure);
+}
+
+/* For a given service identity public key and an introduction authentication
+ * key, return true iff it is present in the failure cache. */
+const hs_cache_intro_state_t *
+hs_cache_client_intro_state_find(const ed25519_public_key_t *service_pk,
+                                 const ed25519_public_key_t *auth_key)
+{
+  hs_cache_intro_state_t *state = NULL;
+  cache_client_intro_state_lookup(service_pk, auth_key, &state);
+  return state;
+}
+
+/* Cleanup the client introduction state cache. */
+void
+hs_cache_client_intro_state_clean(time_t now)
+{
+  time_t cutoff = now - HS_CACHE_CLIENT_INTRO_STATE_MAX_AGE;
+
+  DIGEST256MAP_FOREACH_MODIFY(hs_cache_client_intro_state, key,
+                              hs_cache_client_intro_state_t *, cache) {
+    /* Cleanup intro points failure. */
+    cache_client_intro_state_clean(cutoff, cache);
+
+    /* Is this cache empty for this service key? If yes, remove it from the
+     * cache. Else keep it. */
+    if (cache_client_intro_state_is_empty(cache)) {
+      cache_client_intro_state_free(cache);
+      MAP_DEL_CURRENT(key);
+    }
+  } DIGEST256MAP_FOREACH_END;
+}
+
+/* Purge the client introduction state cache. */
+void
+hs_cache_client_intro_state_purge(void)
+{
+  DIGEST256MAP_FOREACH_MODIFY(hs_cache_client_intro_state, key,
+                              hs_cache_client_intro_state_t *, cache) {
+    MAP_DEL_CURRENT(key);
+    cache_client_intro_state_free(cache);
+  } DIGEST256MAP_FOREACH_END;
+
+  log_info(LD_REND, "Hidden service client introduction point state "
+                    "cache purged.");
+}
+
+/**************** Generics *********************************/
+
+/* Do a round of OOM cleanup on all directory caches. Return the amount of
+ * removed bytes. It is possible that the returned value is lower than
+ * min_remove_bytes if the caches get emptied out so the caller should be
+ * aware of this. */
+size_t
+hs_cache_handle_oom(time_t now, size_t min_remove_bytes)
+{
+  time_t k;
+  size_t bytes_removed = 0;
+
+  /* Our OOM handler called with 0 bytes to remove is a code flow error. */
+  tor_assert(min_remove_bytes != 0);
+
+  /* The algorithm is as follow. K is the oldest expected descriptor age.
+   *
+   *   1) Deallocate all entries from v2 cache that are older than K hours.
+   *      1.1) If the amount of remove bytes has been reached, stop.
+   *   2) Deallocate all entries from v3 cache that are older than K hours
+   *      2.1) If the amount of remove bytes has been reached, stop.
+   *   3) Set K = K - RendPostPeriod and repeat process until K is < 0.
+   *
+   * This ends up being O(Kn).
+   */
+
+  /* Set K to the oldest expected age in seconds which is the maximum
+   * lifetime of a cache entry. We'll use the v2 lifetime because it's much
+   * bigger than the v3 thus leading to cleaning older descriptors. */
+  k = rend_cache_max_entry_lifetime();
+
+  do {
+    time_t cutoff;
+
+    /* If K becomes negative, it means we've empty the caches so stop and
+     * return what we were able to cleanup. */
+    if (k < 0) {
+      break;
+    }
+    /* Compute a cutoff value with K and the current time. */
+    cutoff = now - k;
+
+    /* Start by cleaning the v2 cache with that cutoff. */
+    bytes_removed += rend_cache_clean_v2_descs_as_dir(cutoff);
+
+    if (bytes_removed < min_remove_bytes) {
+      /* We haven't remove enough bytes so clean v3 cache. */
+      bytes_removed += cache_clean_v3_as_dir(now, cutoff);
+      /* Decrement K by a post period to shorten the cutoff. */
+      k -= get_options()->RendPostPeriod;
+    }
+  } while (bytes_removed < min_remove_bytes);
+
+  return bytes_removed;
+}
+
+/* Return the maximum size of a v3 HS descriptor. */
+unsigned int
+hs_cache_get_max_descriptor_size(void)
+{
+  return (unsigned) networkstatus_get_param(NULL,
+                                            "HSV3MaxDescriptorSize",
+                                            HS_DESC_MAX_LEN, 1, INT32_MAX);
+}
+
+/* Initialize the hidden service cache subsystem. */
+void
+hs_cache_init(void)
+{
+  /* Calling this twice is very wrong code flow. */
+  tor_assert(!hs_cache_v3_dir);
+  hs_cache_v3_dir = digest256map_new();
+
+  tor_assert(!hs_cache_v3_client);
+  hs_cache_v3_client = digest256map_new();
+
+  tor_assert(!hs_cache_client_intro_state);
+  hs_cache_client_intro_state = digest256map_new();
+}
+
+/* Cleanup the hidden service cache subsystem. */
+void
+hs_cache_free_all(void)
+{
+  digest256map_free(hs_cache_v3_dir, cache_dir_desc_free_void);
+  hs_cache_v3_dir = NULL;
+
+  digest256map_free(hs_cache_v3_client, cache_client_desc_free_void);
+  hs_cache_v3_client = NULL;
+
+  digest256map_free(hs_cache_client_intro_state,
+                    cache_client_intro_state_free_void);
+  hs_cache_client_intro_state = NULL;
+}
diff --git a/src/feature/hs/hs_cache.h b/src/feature/hs/hs_cache.h
new file mode 100644
index 0000000000..079d31d437
--- /dev/null
+++ b/src/feature/hs/hs_cache.h
@@ -0,0 +1,130 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_cache.h
+ * \brief Header file for hs_cache.c
+ **/
+
+#ifndef TOR_HS_CACHE_H
+#define TOR_HS_CACHE_H
+
+#include <stdint.h>
+
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_descriptor.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/nodelist/torcert.h"
+
+struct ed25519_public_key_t;
+
+/* This is the maximum time an introduction point state object can stay in the
+ * client cache in seconds (2 mins or 120 seconds). */
+#define HS_CACHE_CLIENT_INTRO_STATE_MAX_AGE (2 * 60)
+
+/* Introduction point state. */
+typedef struct hs_cache_intro_state_t {
+  /* When this entry was created and put in the cache. */
+  time_t created_ts;
+
+  /* Did it suffered a generic error? */
+  unsigned int error : 1;
+
+  /* Did it timed out? */
+  unsigned int timed_out : 1;
+
+  /* How many times we tried to reached it and it was unreachable. */
+  uint32_t unreachable_count;
+} hs_cache_intro_state_t;
+
+typedef struct hs_cache_client_intro_state_t {
+  /* Contains hs_cache_intro_state_t object indexed by introduction point
+   * authentication key. */
+  digest256map_t *intro_points;
+} hs_cache_client_intro_state_t;
+
+/* Descriptor representation on the directory side which is a subset of
+ * information that the HSDir can decode and serve it. */
+typedef struct hs_cache_dir_descriptor_t {
+  /* This object is indexed using the blinded pubkey located in the plaintext
+   * data which is populated only once the descriptor has been successfully
+   * decoded and validated. This simply points to that pubkey. */
+  const uint8_t *key;
+
+  /* When does this entry has been created. Used to expire entries. */
+  time_t created_ts;
+
+  /* Descriptor plaintext information. Obviously, we can't decrypt the
+   * encrypted part of the descriptor. */
+  hs_desc_plaintext_data_t *plaintext_data;
+
+  /* Encoded descriptor which is basically in text form. It's a NUL terminated
+   * string thus safe to strlen(). */
+  char *encoded_desc;
+} hs_cache_dir_descriptor_t;
+
+/* Public API */
+
+void hs_cache_init(void);
+void hs_cache_free_all(void);
+void hs_cache_clean_as_dir(time_t now);
+size_t hs_cache_handle_oom(time_t now, size_t min_remove_bytes);
+
+unsigned int hs_cache_get_max_descriptor_size(void);
+
+/* Store and Lookup function. They are version agnostic that is depending on
+ * the requested version of the descriptor, it will be re-routed to the
+ * right function. */
+int hs_cache_store_as_dir(const char *desc);
+int hs_cache_lookup_as_dir(uint32_t version, const char *query,
+                           const char **desc_out);
+
+const hs_descriptor_t *
+hs_cache_lookup_as_client(const struct ed25519_public_key_t *key);
+const char *
+hs_cache_lookup_encoded_as_client(const struct ed25519_public_key_t *key);
+int hs_cache_store_as_client(const char *desc_str,
+                             const struct ed25519_public_key_t *identity_pk);
+void hs_cache_clean_as_client(time_t now);
+void hs_cache_purge_as_client(void);
+
+/* Client failure cache. */
+void hs_cache_client_intro_state_note(
+                              const struct ed25519_public_key_t *service_pk,
+                              const struct ed25519_public_key_t *auth_key,
+                              rend_intro_point_failure_t failure);
+const hs_cache_intro_state_t *hs_cache_client_intro_state_find(
+                              const struct ed25519_public_key_t *service_pk,
+                              const struct ed25519_public_key_t *auth_key);
+void hs_cache_client_intro_state_clean(time_t now);
+void hs_cache_client_intro_state_purge(void);
+
+#ifdef HS_CACHE_PRIVATE
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+/** Represents a locally cached HS descriptor on a hidden service client. */
+typedef struct hs_cache_client_descriptor_t {
+  /* This object is indexed using the service identity public key */
+  struct ed25519_public_key_t key;
+
+  /* When will this entry expire? We expire cached client descriptors in the
+   * start of the next time period, since that's when clients need to start
+   * using the next blinded key of the service. */
+  time_t expiration_ts;
+
+  /* The cached descriptor, this object is the owner. It can't be NULL. A
+   * cache object without a valid descriptor is not possible. */
+  hs_descriptor_t *desc;
+
+  /* Encoded descriptor in string form. Can't be NULL. */
+  char *encoded_desc;
+} hs_cache_client_descriptor_t;
+
+STATIC size_t cache_clean_v3_as_dir(time_t now, time_t global_cutoff);
+
+STATIC hs_cache_client_descriptor_t *
+lookup_v3_desc_as_client(const uint8_t *key);
+
+#endif /* defined(HS_CACHE_PRIVATE) */
+
+#endif /* !defined(TOR_HS_CACHE_H) */
diff --git a/src/feature/hs/hs_cell.c b/src/feature/hs/hs_cell.c
new file mode 100644
index 0000000000..613ffe7260
--- /dev/null
+++ b/src/feature/hs/hs_cell.c
@@ -0,0 +1,952 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_cell.c
+ * \brief Hidden service API for cell creation and handling.
+ **/
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "feature/rend/rendservice.h"
+#include "feature/hs_common/replaycache.h"
+
+#include "feature/hs/hs_cell.h"
+#include "core/crypto/hs_ntor.h"
+
+#include "core/or/origin_circuit_st.h"
+
+/* Trunnel. */
+#include "trunnel/ed25519_cert.h"
+#include "trunnel/hs/cell_common.h"
+#include "trunnel/hs/cell_establish_intro.h"
+#include "trunnel/hs/cell_introduce1.h"
+#include "trunnel/hs/cell_rendezvous.h"
+
+/* Compute the MAC of an INTRODUCE cell in mac_out. The encoded_cell param is
+ * the cell content up to the ENCRYPTED section of length encoded_cell_len.
+ * The encrypted param is the start of the ENCRYPTED section of length
+ * encrypted_len. The mac_key is the key needed for the computation of the MAC
+ * derived from the ntor handshake of length mac_key_len.
+ *
+ * The length mac_out_len must be at least DIGEST256_LEN. */
+static void
+compute_introduce_mac(const uint8_t *encoded_cell, size_t encoded_cell_len,
+                      const uint8_t *encrypted, size_t encrypted_len,
+                      const uint8_t *mac_key, size_t mac_key_len,
+                      uint8_t *mac_out, size_t mac_out_len)
+{
+  size_t offset = 0;
+  size_t mac_msg_len;
+  uint8_t mac_msg[RELAY_PAYLOAD_SIZE] = {0};
+
+  tor_assert(encoded_cell);
+  tor_assert(encrypted);
+  tor_assert(mac_key);
+  tor_assert(mac_out);
+  tor_assert(mac_out_len >= DIGEST256_LEN);
+
+  /* Compute the size of the message which is basically the entire cell until
+   * the MAC field of course. */
+  mac_msg_len = encoded_cell_len + (encrypted_len - DIGEST256_LEN);
+  tor_assert(mac_msg_len <= sizeof(mac_msg));
+
+  /* First, put the encoded cell in the msg. */
+  memcpy(mac_msg, encoded_cell, encoded_cell_len);
+  offset += encoded_cell_len;
+  /* Second, put the CLIENT_PK + ENCRYPTED_DATA but ommit the MAC field (which
+   * is junk at this point). */
+  memcpy(mac_msg + offset, encrypted, (encrypted_len - DIGEST256_LEN));
+  offset += (encrypted_len - DIGEST256_LEN);
+  tor_assert(offset == mac_msg_len);
+
+  crypto_mac_sha3_256(mac_out, mac_out_len,
+                      mac_key, mac_key_len,
+                      mac_msg, mac_msg_len);
+  memwipe(mac_msg, 0, sizeof(mac_msg));
+}
+
+/* From a set of keys, subcredential and the ENCRYPTED section of an
+ * INTRODUCE2 cell, return a newly allocated intro cell keys structure.
+ * Finally, the client public key is copied in client_pk. On error, return
+ * NULL. */
+static hs_ntor_intro_cell_keys_t *
+get_introduce2_key_material(const ed25519_public_key_t *auth_key,
+                            const curve25519_keypair_t *enc_key,
+                            const uint8_t *subcredential,
+                            const uint8_t *encrypted_section,
+                            curve25519_public_key_t *client_pk)
+{
+  hs_ntor_intro_cell_keys_t *keys;
+
+  tor_assert(auth_key);
+  tor_assert(enc_key);
+  tor_assert(subcredential);
+  tor_assert(encrypted_section);
+  tor_assert(client_pk);
+
+  keys = tor_malloc_zero(sizeof(*keys));
+
+  /* First bytes of the ENCRYPTED section are the client public key. */
+  memcpy(client_pk->public_key, encrypted_section, CURVE25519_PUBKEY_LEN);
+
+  if (hs_ntor_service_get_introduce1_keys(auth_key, enc_key, client_pk,
+                                          subcredential, keys) < 0) {
+    /* Don't rely on the caller to wipe this on error. */
+    memwipe(client_pk, 0, sizeof(curve25519_public_key_t));
+    tor_free(keys);
+    keys = NULL;
+  }
+  return keys;
+}
+
+/* Using the given encryption key, decrypt the encrypted_section of length
+ * encrypted_section_len of an INTRODUCE2 cell and return a newly allocated
+ * buffer containing the decrypted data. On decryption failure, NULL is
+ * returned. */
+static uint8_t *
+decrypt_introduce2(const uint8_t *enc_key, const uint8_t *encrypted_section,
+                   size_t encrypted_section_len)
+{
+  uint8_t *decrypted = NULL;
+  crypto_cipher_t *cipher = NULL;
+
+  tor_assert(enc_key);
+  tor_assert(encrypted_section);
+
+  /* Decrypt ENCRYPTED section. */
+  cipher = crypto_cipher_new_with_bits((char *) enc_key,
+                                       CURVE25519_PUBKEY_LEN * 8);
+  tor_assert(cipher);
+
+  /* This is symmetric encryption so can't be bigger than the encrypted
+   * section length. */
+  decrypted = tor_malloc_zero(encrypted_section_len);
+  if (crypto_cipher_decrypt(cipher, (char *) decrypted,
+                            (const char *) encrypted_section,
+                            encrypted_section_len) < 0) {
+    tor_free(decrypted);
+    decrypted = NULL;
+    goto done;
+  }
+
+ done:
+  crypto_cipher_free(cipher);
+  return decrypted;
+}
+
+/* Given a pointer to the decrypted data of the ENCRYPTED section of an
+ * INTRODUCE2 cell of length decrypted_len, parse and validate the cell
+ * content. Return a newly allocated cell structure or NULL on error. The
+ * circuit and service object are only used for logging purposes. */
+static trn_cell_introduce_encrypted_t *
+parse_introduce2_encrypted(const uint8_t *decrypted_data,
+                           size_t decrypted_len, const origin_circuit_t *circ,
+                           const hs_service_t *service)
+{
+  trn_cell_introduce_encrypted_t *enc_cell = NULL;
+
+  tor_assert(decrypted_data);
+  tor_assert(circ);
+  tor_assert(service);
+
+  if (trn_cell_introduce_encrypted_parse(&enc_cell, decrypted_data,
+                                         decrypted_len) < 0) {
+    log_info(LD_REND, "Unable to parse the decrypted ENCRYPTED section of "
+                      "the INTRODUCE2 cell on circuit %u for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto err;
+  }
+
+  if (trn_cell_introduce_encrypted_get_onion_key_type(enc_cell) !=
+      TRUNNEL_HS_INTRO_ONION_KEY_TYPE_NTOR) {
+    log_info(LD_REND, "INTRODUCE2 onion key type is invalid. Got %u but "
+                      "expected %u on circuit %u for service %s",
+             trn_cell_introduce_encrypted_get_onion_key_type(enc_cell),
+             TRUNNEL_HS_INTRO_ONION_KEY_TYPE_NTOR,
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto err;
+  }
+
+  if (trn_cell_introduce_encrypted_getlen_onion_key(enc_cell) !=
+      CURVE25519_PUBKEY_LEN) {
+    log_info(LD_REND, "INTRODUCE2 onion key length is invalid. Got %u but "
+                      "expected %d on circuit %u for service %s",
+             (unsigned)trn_cell_introduce_encrypted_getlen_onion_key(enc_cell),
+             CURVE25519_PUBKEY_LEN, TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto err;
+  }
+  /* XXX: Validate NSPEC field as well. */
+
+  return enc_cell;
+ err:
+  trn_cell_introduce_encrypted_free(enc_cell);
+  return NULL;
+}
+
+/* Build a legacy ESTABLISH_INTRO cell with the given circuit nonce and RSA
+ * encryption key. The encoded cell is put in cell_out that MUST at least be
+ * of the size of RELAY_PAYLOAD_SIZE. Return the encoded cell length on
+ * success else a negative value and cell_out is untouched. */
+static ssize_t
+build_legacy_establish_intro(const char *circ_nonce, crypto_pk_t *enc_key,
+                             uint8_t *cell_out)
+{
+  ssize_t cell_len;
+
+  tor_assert(circ_nonce);
+  tor_assert(enc_key);
+  tor_assert(cell_out);
+
+  memwipe(cell_out, 0, RELAY_PAYLOAD_SIZE);
+
+  cell_len = rend_service_encode_establish_intro_cell((char*)cell_out,
+                                                      RELAY_PAYLOAD_SIZE,
+                                                      enc_key, circ_nonce);
+  return cell_len;
+}
+
+/* Parse an INTRODUCE2 cell from payload of size payload_len for the given
+ * service and circuit which are used only for logging purposes. The resulting
+ * parsed cell is put in cell_ptr_out.
+ *
+ * This function only parses prop224 INTRODUCE2 cells even when the intro point
+ * is a legacy intro point. That's because intro points don't actually care
+ * about the contents of the introduce cell. Legacy INTRODUCE cells are only
+ * used by the legacy system now.
+ *
+ * Return 0 on success else a negative value and cell_ptr_out is untouched. */
+static int
+parse_introduce2_cell(const hs_service_t *service,
+                      const origin_circuit_t *circ, const uint8_t *payload,
+                      size_t payload_len,
+                      trn_cell_introduce1_t **cell_ptr_out)
+{
+  trn_cell_introduce1_t *cell = NULL;
+
+  tor_assert(service);
+  tor_assert(circ);
+  tor_assert(payload);
+  tor_assert(cell_ptr_out);
+
+  /* Parse the cell so we can start cell validation. */
+  if (trn_cell_introduce1_parse(&cell, payload, payload_len) < 0) {
+    log_info(LD_PROTOCOL, "Unable to parse INTRODUCE2 cell on circuit %u "
+                          "for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto err;
+  }
+
+  /* Success. */
+  *cell_ptr_out = cell;
+  return 0;
+ err:
+  return -1;
+}
+
+/* Set the onion public key onion_pk in cell, the encrypted section of an
+ * INTRODUCE1 cell. */
+static void
+introduce1_set_encrypted_onion_key(trn_cell_introduce_encrypted_t *cell,
+                                   const uint8_t *onion_pk)
+{
+  tor_assert(cell);
+  tor_assert(onion_pk);
+  /* There is only one possible key type for a non legacy cell. */
+  trn_cell_introduce_encrypted_set_onion_key_type(cell,
+                                   TRUNNEL_HS_INTRO_ONION_KEY_TYPE_NTOR);
+  trn_cell_introduce_encrypted_set_onion_key_len(cell, CURVE25519_PUBKEY_LEN);
+  trn_cell_introduce_encrypted_setlen_onion_key(cell, CURVE25519_PUBKEY_LEN);
+  memcpy(trn_cell_introduce_encrypted_getarray_onion_key(cell), onion_pk,
+         trn_cell_introduce_encrypted_getlen_onion_key(cell));
+}
+
+/* Set the link specifiers in lspecs in cell, the encrypted section of an
+ * INTRODUCE1 cell. */
+static void
+introduce1_set_encrypted_link_spec(trn_cell_introduce_encrypted_t *cell,
+                                   const smartlist_t *lspecs)
+{
+  tor_assert(cell);
+  tor_assert(lspecs);
+  tor_assert(smartlist_len(lspecs) > 0);
+  tor_assert(smartlist_len(lspecs) <= UINT8_MAX);
+
+  uint8_t lspecs_num = (uint8_t) smartlist_len(lspecs);
+  trn_cell_introduce_encrypted_set_nspec(cell, lspecs_num);
+  /* We aren't duplicating the link specifiers object here which means that
+   * the ownership goes to the trn_cell_introduce_encrypted_t cell and those
+   * object will be freed when the cell is. */
+  SMARTLIST_FOREACH(lspecs, link_specifier_t *, ls,
+                    trn_cell_introduce_encrypted_add_nspecs(cell, ls));
+}
+
+/* Set padding in the enc_cell only if needed that is the total length of both
+ * sections are below the mininum required for an INTRODUCE1 cell. */
+static void
+introduce1_set_encrypted_padding(const trn_cell_introduce1_t *cell,
+                                 trn_cell_introduce_encrypted_t *enc_cell)
+{
+  tor_assert(cell);
+  tor_assert(enc_cell);
+  /* This is the length we expect to have once encoded of the whole cell. */
+  ssize_t full_len = trn_cell_introduce1_encoded_len(cell) +
+                     trn_cell_introduce_encrypted_encoded_len(enc_cell);
+  tor_assert(full_len > 0);
+  if (full_len < HS_CELL_INTRODUCE1_MIN_SIZE) {
+    size_t padding = HS_CELL_INTRODUCE1_MIN_SIZE - full_len;
+    trn_cell_introduce_encrypted_setlen_pad(enc_cell, padding);
+    memset(trn_cell_introduce_encrypted_getarray_pad(enc_cell), 0,
+           trn_cell_introduce_encrypted_getlen_pad(enc_cell));
+  }
+}
+
+/* Encrypt the ENCRYPTED payload and encode it in the cell using the enc_cell
+ * and the INTRODUCE1 data.
+ *
+ * This can't fail but it is very important that the caller sets every field
+ * in data so the computation of the INTRODUCE1 keys doesn't fail. */
+static void
+introduce1_encrypt_and_encode(trn_cell_introduce1_t *cell,
+                              const trn_cell_introduce_encrypted_t *enc_cell,
+                              const hs_cell_introduce1_data_t *data)
+{
+  size_t offset = 0;
+  ssize_t encrypted_len;
+  ssize_t encoded_cell_len, encoded_enc_cell_len;
+  uint8_t encoded_cell[RELAY_PAYLOAD_SIZE] = {0};
+  uint8_t encoded_enc_cell[RELAY_PAYLOAD_SIZE] = {0};
+  uint8_t *encrypted = NULL;
+  uint8_t mac[DIGEST256_LEN];
+  crypto_cipher_t *cipher = NULL;
+  hs_ntor_intro_cell_keys_t keys;
+
+  tor_assert(cell);
+  tor_assert(enc_cell);
+  tor_assert(data);
+
+  /* Encode the cells up to now of what we have to we can perform the MAC
+   * computation on it. */
+  encoded_cell_len = trn_cell_introduce1_encode(encoded_cell,
+                                                sizeof(encoded_cell), cell);
+  /* We have a much more serious issue if this isn't true. */
+  tor_assert(encoded_cell_len > 0);
+
+  encoded_enc_cell_len =
+    trn_cell_introduce_encrypted_encode(encoded_enc_cell,
+                                        sizeof(encoded_enc_cell), enc_cell);
+  /* We have a much more serious issue if this isn't true. */
+  tor_assert(encoded_enc_cell_len > 0);
+
+  /* Get the key material for the encryption. */
+  if (hs_ntor_client_get_introduce1_keys(data->auth_pk, data->enc_pk,
+                                         data->client_kp,
+                                         data->subcredential, &keys) < 0) {
+    tor_assert_unreached();
+  }
+
+  /* Prepare cipher with the encryption key just computed. */
+  cipher = crypto_cipher_new_with_bits((const char *) keys.enc_key,
+                                       sizeof(keys.enc_key) * 8);
+  tor_assert(cipher);
+
+  /* Compute the length of the ENCRYPTED section which is the CLIENT_PK,
+   * ENCRYPTED_DATA and MAC length. */
+  encrypted_len = sizeof(data->client_kp->pubkey) + encoded_enc_cell_len +
+                  sizeof(mac);
+  tor_assert(encrypted_len < RELAY_PAYLOAD_SIZE);
+  encrypted = tor_malloc_zero(encrypted_len);
+
+  /* Put the CLIENT_PK first. */
+  memcpy(encrypted, data->client_kp->pubkey.public_key,
+         sizeof(data->client_kp->pubkey.public_key));
+  offset += sizeof(data->client_kp->pubkey.public_key);
+  /* Then encrypt and set the ENCRYPTED_DATA. This can't fail. */
+  crypto_cipher_encrypt(cipher, (char *) encrypted + offset,
+                        (const char *) encoded_enc_cell, encoded_enc_cell_len);
+  crypto_cipher_free(cipher);
+  offset += encoded_enc_cell_len;
+  /* Compute MAC from the above and put it in the buffer. This function will
+   * make the adjustment to the encrypted_len to omit the MAC length. */
+  compute_introduce_mac(encoded_cell, encoded_cell_len,
+                        encrypted, encrypted_len,
+                        keys.mac_key, sizeof(keys.mac_key),
+                        mac, sizeof(mac));
+  memcpy(encrypted + offset, mac, sizeof(mac));
+  offset += sizeof(mac);
+  tor_assert(offset == (size_t) encrypted_len);
+
+  /* Set the ENCRYPTED section in the cell. */
+  trn_cell_introduce1_setlen_encrypted(cell, encrypted_len);
+  memcpy(trn_cell_introduce1_getarray_encrypted(cell),
+         encrypted, encrypted_len);
+
+  /* Cleanup. */
+  memwipe(&keys, 0, sizeof(keys));
+  memwipe(mac, 0, sizeof(mac));
+  memwipe(encrypted, 0, sizeof(encrypted_len));
+  memwipe(encoded_enc_cell, 0, sizeof(encoded_enc_cell));
+  tor_free(encrypted);
+}
+
+/* Using the INTRODUCE1 data, setup the ENCRYPTED section in cell. This means
+ * set it, encrypt it and encode it. */
+static void
+introduce1_set_encrypted(trn_cell_introduce1_t *cell,
+                         const hs_cell_introduce1_data_t *data)
+{
+  trn_cell_introduce_encrypted_t *enc_cell;
+  trn_cell_extension_t *ext;
+
+  tor_assert(cell);
+  tor_assert(data);
+
+  enc_cell = trn_cell_introduce_encrypted_new();
+  tor_assert(enc_cell);
+
+  /* Set extension data. None are used. */
+  ext = trn_cell_extension_new();
+  tor_assert(ext);
+  trn_cell_extension_set_num(ext, 0);
+  trn_cell_introduce_encrypted_set_extensions(enc_cell, ext);
+
+  /* Set the rendezvous cookie. */
+  memcpy(trn_cell_introduce_encrypted_getarray_rend_cookie(enc_cell),
+         data->rendezvous_cookie, REND_COOKIE_LEN);
+
+  /* Set the onion public key. */
+  introduce1_set_encrypted_onion_key(enc_cell, data->onion_pk->public_key);
+
+  /* Set the link specifiers. */
+  introduce1_set_encrypted_link_spec(enc_cell, data->link_specifiers);
+
+  /* Set padding. */
+  introduce1_set_encrypted_padding(cell, enc_cell);
+
+  /* Encrypt and encode it in the cell. */
+  introduce1_encrypt_and_encode(cell, enc_cell, data);
+
+  /* Cleanup. */
+  trn_cell_introduce_encrypted_free(enc_cell);
+}
+
+/* Set the authentication key in the INTRODUCE1 cell from the given data. */
+static void
+introduce1_set_auth_key(trn_cell_introduce1_t *cell,
+                        const hs_cell_introduce1_data_t *data)
+{
+  tor_assert(cell);
+  tor_assert(data);
+  /* There is only one possible type for a non legacy cell. */
+  trn_cell_introduce1_set_auth_key_type(cell,
+                                   TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_ED25519);
+  trn_cell_introduce1_set_auth_key_len(cell, ED25519_PUBKEY_LEN);
+  trn_cell_introduce1_setlen_auth_key(cell, ED25519_PUBKEY_LEN);
+  memcpy(trn_cell_introduce1_getarray_auth_key(cell),
+         data->auth_pk->pubkey, trn_cell_introduce1_getlen_auth_key(cell));
+}
+
+/* Set the legacy ID field in the INTRODUCE1 cell from the given data. */
+static void
+introduce1_set_legacy_id(trn_cell_introduce1_t *cell,
+                         const hs_cell_introduce1_data_t *data)
+{
+  tor_assert(cell);
+  tor_assert(data);
+
+  if (data->is_legacy) {
+    uint8_t digest[DIGEST_LEN];
+    if (BUG(crypto_pk_get_digest(data->legacy_key, (char *) digest) < 0)) {
+      return;
+    }
+    memcpy(trn_cell_introduce1_getarray_legacy_key_id(cell),
+           digest, trn_cell_introduce1_getlen_legacy_key_id(cell));
+  } else {
+    /* We have to zeroed the LEGACY_KEY_ID field. */
+    memset(trn_cell_introduce1_getarray_legacy_key_id(cell), 0,
+           trn_cell_introduce1_getlen_legacy_key_id(cell));
+  }
+}
+
+/* ========== */
+/* Public API */
+/* ========== */
+
+/* Build an ESTABLISH_INTRO cell with the given circuit nonce and intro point
+ * object. The encoded cell is put in cell_out that MUST at least be of the
+ * size of RELAY_PAYLOAD_SIZE. Return the encoded cell length on success else
+ * a negative value and cell_out is untouched. This function also supports
+ * legacy cell creation. */
+ssize_t
+hs_cell_build_establish_intro(const char *circ_nonce,
+                              const hs_service_intro_point_t *ip,
+                              uint8_t *cell_out)
+{
+  ssize_t cell_len = -1;
+  uint16_t sig_len = ED25519_SIG_LEN;
+  trn_cell_extension_t *ext;
+  trn_cell_establish_intro_t *cell = NULL;
+
+  tor_assert(circ_nonce);
+  tor_assert(ip);
+
+  /* Quickly handle the legacy IP. */
+  if (ip->base.is_only_legacy) {
+    tor_assert(ip->legacy_key);
+    cell_len = build_legacy_establish_intro(circ_nonce, ip->legacy_key,
+                                            cell_out);
+    tor_assert(cell_len <= RELAY_PAYLOAD_SIZE);
+    /* Success or not we are done here. */
+    goto done;
+  }
+
+  /* Set extension data. None used here. */
+  ext = trn_cell_extension_new();
+  trn_cell_extension_set_num(ext, 0);
+  cell = trn_cell_establish_intro_new();
+  trn_cell_establish_intro_set_extensions(cell, ext);
+  /* Set signature size. Array is then allocated in the cell. We need to do
+   * this early so we can use trunnel API to get the signature length. */
+  trn_cell_establish_intro_set_sig_len(cell, sig_len);
+  trn_cell_establish_intro_setlen_sig(cell, sig_len);
+
+  /* Set AUTH_KEY_TYPE: 2 means ed25519 */
+  trn_cell_establish_intro_set_auth_key_type(cell,
+                                    TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_ED25519);
+
+  /* Set AUTH_KEY and AUTH_KEY_LEN field. Must also set byte-length of
+   * AUTH_KEY to match */
+  {
+    uint16_t auth_key_len = ED25519_PUBKEY_LEN;
+    trn_cell_establish_intro_set_auth_key_len(cell, auth_key_len);
+    trn_cell_establish_intro_setlen_auth_key(cell, auth_key_len);
+    /* We do this call _after_ setting the length because it's reallocated at
+     * that point only. */
+    uint8_t *auth_key_ptr = trn_cell_establish_intro_getarray_auth_key(cell);
+    memcpy(auth_key_ptr, ip->auth_key_kp.pubkey.pubkey, auth_key_len);
+  }
+
+  /* Calculate HANDSHAKE_AUTH field (MAC). */
+  {
+    ssize_t tmp_cell_enc_len = 0;
+    ssize_t tmp_cell_mac_offset =
+      sig_len + sizeof(cell->sig_len) +
+      trn_cell_establish_intro_getlen_handshake_mac(cell);
+    uint8_t tmp_cell_enc[RELAY_PAYLOAD_SIZE] = {0};
+    uint8_t mac[TRUNNEL_SHA3_256_LEN], *handshake_ptr;
+
+    /* We first encode the current fields we have in the cell so we can
+     * compute the MAC using the raw bytes. */
+    tmp_cell_enc_len = trn_cell_establish_intro_encode(tmp_cell_enc,
+                                                       sizeof(tmp_cell_enc),
+                                                       cell);
+    if (BUG(tmp_cell_enc_len < 0)) {
+      goto done;
+    }
+    /* Sanity check. */
+    tor_assert(tmp_cell_enc_len > tmp_cell_mac_offset);
+
+    /* Circuit nonce is always DIGEST_LEN according to tor-spec.txt. */
+    crypto_mac_sha3_256(mac, sizeof(mac),
+                        (uint8_t *) circ_nonce, DIGEST_LEN,
+                        tmp_cell_enc, tmp_cell_enc_len - tmp_cell_mac_offset);
+    handshake_ptr = trn_cell_establish_intro_getarray_handshake_mac(cell);
+    memcpy(handshake_ptr, mac, sizeof(mac));
+
+    memwipe(mac, 0, sizeof(mac));
+    memwipe(tmp_cell_enc, 0, sizeof(tmp_cell_enc));
+  }
+
+  /* Calculate the cell signature SIG. */
+  {
+    ssize_t tmp_cell_enc_len = 0;
+    ssize_t tmp_cell_sig_offset = (sig_len + sizeof(cell->sig_len));
+    uint8_t tmp_cell_enc[RELAY_PAYLOAD_SIZE] = {0}, *sig_ptr;
+    ed25519_signature_t sig;
+
+    /* We first encode the current fields we have in the cell so we can
+     * compute the signature from the raw bytes of the cell. */
+    tmp_cell_enc_len = trn_cell_establish_intro_encode(tmp_cell_enc,
+                                                       sizeof(tmp_cell_enc),
+                                                       cell);
+    if (BUG(tmp_cell_enc_len < 0)) {
+      goto done;
+    }
+
+    if (ed25519_sign_prefixed(&sig, tmp_cell_enc,
+                              tmp_cell_enc_len - tmp_cell_sig_offset,
+                              ESTABLISH_INTRO_SIG_PREFIX, &ip->auth_key_kp)) {
+      log_warn(LD_BUG, "Unable to make signature for ESTABLISH_INTRO cell.");
+      goto done;
+    }
+    /* Copy the signature into the cell. */
+    sig_ptr = trn_cell_establish_intro_getarray_sig(cell);
+    memcpy(sig_ptr, sig.sig, sig_len);
+
+    memwipe(tmp_cell_enc, 0, sizeof(tmp_cell_enc));
+  }
+
+  /* Encode the cell. Can't be bigger than a standard cell. */
+  cell_len = trn_cell_establish_intro_encode(cell_out, RELAY_PAYLOAD_SIZE,
+                                             cell);
+
+ done:
+  trn_cell_establish_intro_free(cell);
+  return cell_len;
+}
+
+/* Parse the INTRO_ESTABLISHED cell in the payload of size payload_len. If we
+ * are successful at parsing it, return the length of the parsed cell else a
+ * negative value on error. */
+ssize_t
+hs_cell_parse_intro_established(const uint8_t *payload, size_t payload_len)
+{
+  ssize_t ret;
+  trn_cell_intro_established_t *cell = NULL;
+
+  tor_assert(payload);
+
+  /* Try to parse the payload into a cell making sure we do actually have a
+   * valid cell. */
+  ret = trn_cell_intro_established_parse(&cell, payload, payload_len);
+  if (ret >= 0) {
+    /* On success, we do not keep the cell, we just notify the caller that it
+     * was successfully parsed. */
+    trn_cell_intro_established_free(cell);
+  }
+  return ret;
+}
+
+/* Parse the INTRODUCE2 cell using data which contains everything we need to
+ * do so and contains the destination buffers of information we extract and
+ * compute from the cell. Return 0 on success else a negative value. The
+ * service and circ are only used for logging purposes. */
+ssize_t
+hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data,
+                         const origin_circuit_t *circ,
+                         const hs_service_t *service)
+{
+  int ret = -1;
+  time_t elapsed;
+  uint8_t *decrypted = NULL;
+  size_t encrypted_section_len;
+  const uint8_t *encrypted_section;
+  trn_cell_introduce1_t *cell = NULL;
+  trn_cell_introduce_encrypted_t *enc_cell = NULL;
+  hs_ntor_intro_cell_keys_t *intro_keys = NULL;
+
+  tor_assert(data);
+  tor_assert(circ);
+  tor_assert(service);
+
+  /* Parse the cell into a decoded data structure pointed by cell_ptr. */
+  if (parse_introduce2_cell(service, circ, data->payload, data->payload_len,
+                            &cell) < 0) {
+    goto done;
+  }
+
+  log_info(LD_REND, "Received a decodable INTRODUCE2 cell on circuit %u "
+                    "for service %s. Decoding encrypted section...",
+           TO_CIRCUIT(circ)->n_circ_id,
+           safe_str_client(service->onion_address));
+
+  encrypted_section = trn_cell_introduce1_getconstarray_encrypted(cell);
+  encrypted_section_len = trn_cell_introduce1_getlen_encrypted(cell);
+
+  /* Encrypted section must at least contain the CLIENT_PK and MAC which is
+   * defined in section 3.3.2 of the specification. */
+  if (encrypted_section_len < (CURVE25519_PUBKEY_LEN + DIGEST256_LEN)) {
+    log_info(LD_REND, "Invalid INTRODUCE2 encrypted section length "
+                      "for service %s. Dropping cell.",
+             safe_str_client(service->onion_address));
+    goto done;
+  }
+
+  /* Check our replay cache for this introduction point. */
+  if (replaycache_add_test_and_elapsed(data->replay_cache, encrypted_section,
+                                       encrypted_section_len, &elapsed)) {
+    log_warn(LD_REND, "Possible replay detected! An INTRODUCE2 cell with the"
+                      "same ENCRYPTED section was seen %ld seconds ago. "
+                      "Dropping cell.", (long int) elapsed);
+    goto done;
+  }
+
+  /* Build the key material out of the key material found in the cell. */
+  intro_keys = get_introduce2_key_material(data->auth_pk, data->enc_kp,
+                                           data->subcredential,
+                                           encrypted_section,
+                                           &data->client_pk);
+  if (intro_keys == NULL) {
+    log_info(LD_REND, "Invalid INTRODUCE2 encrypted data. Unable to "
+                      "compute key material on circuit %u for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto done;
+  }
+
+  /* Validate MAC from the cell and our computed key material. The MAC field
+   * in the cell is at the end of the encrypted section. */
+  {
+    uint8_t mac[DIGEST256_LEN];
+    /* The MAC field is at the very end of the ENCRYPTED section. */
+    size_t mac_offset = encrypted_section_len - sizeof(mac);
+    /* Compute the MAC. Use the entire encoded payload with a length up to the
+     * ENCRYPTED section. */
+    compute_introduce_mac(data->payload,
+                          data->payload_len - encrypted_section_len,
+                          encrypted_section, encrypted_section_len,
+                          intro_keys->mac_key, sizeof(intro_keys->mac_key),
+                          mac, sizeof(mac));
+    if (tor_memcmp(mac, encrypted_section + mac_offset, sizeof(mac))) {
+      log_info(LD_REND, "Invalid MAC validation for INTRODUCE2 cell on "
+                        "circuit %u for service %s",
+               TO_CIRCUIT(circ)->n_circ_id,
+               safe_str_client(service->onion_address));
+      goto done;
+    }
+  }
+
+  {
+    /* The ENCRYPTED_DATA section starts just after the CLIENT_PK. */
+    const uint8_t *encrypted_data =
+      encrypted_section + sizeof(data->client_pk);
+    /* It's symmetric encryption so it's correct to use the ENCRYPTED length
+     * for decryption. Computes the length of ENCRYPTED_DATA meaning removing
+     * the CLIENT_PK and MAC length. */
+    size_t encrypted_data_len =
+      encrypted_section_len - (sizeof(data->client_pk) + DIGEST256_LEN);
+
+    /* This decrypts the ENCRYPTED_DATA section of the cell. */
+    decrypted = decrypt_introduce2(intro_keys->enc_key,
+                                   encrypted_data, encrypted_data_len);
+    if (decrypted == NULL) {
+      log_info(LD_REND, "Unable to decrypt the ENCRYPTED section of an "
+                        "INTRODUCE2 cell on circuit %u for service %s",
+               TO_CIRCUIT(circ)->n_circ_id,
+               safe_str_client(service->onion_address));
+      goto done;
+    }
+
+    /* Parse this blob into an encrypted cell structure so we can then extract
+     * the data we need out of it. */
+    enc_cell = parse_introduce2_encrypted(decrypted, encrypted_data_len,
+                                          circ, service);
+    memwipe(decrypted, 0, encrypted_data_len);
+    if (enc_cell == NULL) {
+      goto done;
+    }
+  }
+
+  /* XXX: Implement client authorization checks. */
+
+  /* Extract onion key and rendezvous cookie from the cell used for the
+   * rendezvous point circuit e2e encryption. */
+  memcpy(data->onion_pk.public_key,
+         trn_cell_introduce_encrypted_getconstarray_onion_key(enc_cell),
+         CURVE25519_PUBKEY_LEN);
+  memcpy(data->rendezvous_cookie,
+         trn_cell_introduce_encrypted_getconstarray_rend_cookie(enc_cell),
+         sizeof(data->rendezvous_cookie));
+
+  /* Extract rendezvous link specifiers. */
+  for (size_t idx = 0;
+       idx < trn_cell_introduce_encrypted_get_nspec(enc_cell); idx++) {
+    link_specifier_t *lspec =
+      trn_cell_introduce_encrypted_get_nspecs(enc_cell, idx);
+    smartlist_add(data->link_specifiers, hs_link_specifier_dup(lspec));
+  }
+
+  /* Success. */
+  ret = 0;
+  log_info(LD_REND, "Valid INTRODUCE2 cell. Launching rendezvous circuit.");
+
+ done:
+  if (intro_keys) {
+    memwipe(intro_keys, 0, sizeof(hs_ntor_intro_cell_keys_t));
+    tor_free(intro_keys);
+  }
+  tor_free(decrypted);
+  trn_cell_introduce_encrypted_free(enc_cell);
+  trn_cell_introduce1_free(cell);
+  return ret;
+}
+
+/* Build a RENDEZVOUS1 cell with the given rendezvous cookie and handshake
+ * info. The encoded cell is put in cell_out and the length of the data is
+ * returned. This can't fail. */
+ssize_t
+hs_cell_build_rendezvous1(const uint8_t *rendezvous_cookie,
+                          size_t rendezvous_cookie_len,
+                          const uint8_t *rendezvous_handshake_info,
+                          size_t rendezvous_handshake_info_len,
+                          uint8_t *cell_out)
+{
+  ssize_t cell_len;
+  trn_cell_rendezvous1_t *cell;
+
+  tor_assert(rendezvous_cookie);
+  tor_assert(rendezvous_handshake_info);
+  tor_assert(cell_out);
+
+  cell = trn_cell_rendezvous1_new();
+  /* Set the RENDEZVOUS_COOKIE. */
+  memcpy(trn_cell_rendezvous1_getarray_rendezvous_cookie(cell),
+         rendezvous_cookie, rendezvous_cookie_len);
+  /* Set the HANDSHAKE_INFO. */
+  trn_cell_rendezvous1_setlen_handshake_info(cell,
+                                            rendezvous_handshake_info_len);
+  memcpy(trn_cell_rendezvous1_getarray_handshake_info(cell),
+         rendezvous_handshake_info, rendezvous_handshake_info_len);
+  /* Encoding. */
+  cell_len = trn_cell_rendezvous1_encode(cell_out, RELAY_PAYLOAD_SIZE, cell);
+  tor_assert(cell_len > 0);
+
+  trn_cell_rendezvous1_free(cell);
+  return cell_len;
+}
+
+/* Build an INTRODUCE1 cell from the given data. The encoded cell is put in
+ * cell_out which must be of at least size RELAY_PAYLOAD_SIZE. On success, the
+ * encoded length is returned else a negative value and the content of
+ * cell_out should be ignored. */
+ssize_t
+hs_cell_build_introduce1(const hs_cell_introduce1_data_t *data,
+                         uint8_t *cell_out)
+{
+  ssize_t cell_len;
+  trn_cell_introduce1_t *cell;
+  trn_cell_extension_t *ext;
+
+  tor_assert(data);
+  tor_assert(cell_out);
+
+  cell = trn_cell_introduce1_new();
+  tor_assert(cell);
+
+  /* Set extension data. None are used. */
+  ext = trn_cell_extension_new();
+  tor_assert(ext);
+  trn_cell_extension_set_num(ext, 0);
+  trn_cell_introduce1_set_extensions(cell, ext);
+
+  /* Set the legacy ID field. */
+  introduce1_set_legacy_id(cell, data);
+
+  /* Set the authentication key. */
+  introduce1_set_auth_key(cell, data);
+
+  /* Set the encrypted section. This will set, encrypt and encode the
+   * ENCRYPTED section in the cell. After this, we'll be ready to encode. */
+  introduce1_set_encrypted(cell, data);
+
+  /* Final encoding. */
+  cell_len = trn_cell_introduce1_encode(cell_out, RELAY_PAYLOAD_SIZE, cell);
+
+  trn_cell_introduce1_free(cell);
+  return cell_len;
+}
+
+/* Build an ESTABLISH_RENDEZVOUS cell from the given rendezvous_cookie. The
+ * encoded cell is put in cell_out which must be of at least
+ * RELAY_PAYLOAD_SIZE. On success, the encoded length is returned and the
+ * caller should clear up the content of the cell.
+ *
+ * This function can't fail. */
+ssize_t
+hs_cell_build_establish_rendezvous(const uint8_t *rendezvous_cookie,
+                                   uint8_t *cell_out)
+{
+  tor_assert(rendezvous_cookie);
+  tor_assert(cell_out);
+
+  memcpy(cell_out, rendezvous_cookie, HS_REND_COOKIE_LEN);
+  return HS_REND_COOKIE_LEN;
+}
+
+/* Handle an INTRODUCE_ACK cell encoded in payload of length payload_len.
+ * Return the status code on success else a negative value if the cell as not
+ * decodable. */
+int
+hs_cell_parse_introduce_ack(const uint8_t *payload, size_t payload_len)
+{
+  int ret = -1;
+  trn_cell_introduce_ack_t *cell = NULL;
+
+  tor_assert(payload);
+
+  /* If it is a legacy IP, rend-spec.txt specifies that a ACK is 0 byte and a
+   * NACK is 1 byte. We can't use the legacy function for this so we have to
+   * do a special case. */
+  if (payload_len <= 1) {
+    if (payload_len == 0) {
+      ret = TRUNNEL_HS_INTRO_ACK_STATUS_SUCCESS;
+    } else {
+      ret = TRUNNEL_HS_INTRO_ACK_STATUS_UNKNOWN_ID;
+    }
+    goto end;
+  }
+
+  if (trn_cell_introduce_ack_parse(&cell, payload, payload_len) < 0) {
+    log_info(LD_REND, "Invalid INTRODUCE_ACK cell. Unable to parse it.");
+    goto end;
+  }
+
+  ret = trn_cell_introduce_ack_get_status(cell);
+
+ end:
+  trn_cell_introduce_ack_free(cell);
+  return ret;
+}
+
+/* Handle a RENDEZVOUS2 cell encoded in payload of length payload_len. On
+ * success, handshake_info contains the data in the HANDSHAKE_INFO field, and
+ * 0 is returned. On error, a negative value is returned. */
+int
+hs_cell_parse_rendezvous2(const uint8_t *payload, size_t payload_len,
+                          uint8_t *handshake_info, size_t handshake_info_len)
+{
+  int ret = -1;
+  trn_cell_rendezvous2_t *cell = NULL;
+
+  tor_assert(payload);
+  tor_assert(handshake_info);
+
+  if (trn_cell_rendezvous2_parse(&cell, payload, payload_len) < 0) {
+    log_info(LD_REND, "Invalid RENDEZVOUS2 cell. Unable to parse it.");
+    goto end;
+  }
+
+  /* Static size, we should never have an issue with this else we messed up
+   * our code flow. */
+  tor_assert(trn_cell_rendezvous2_getlen_handshake_info(cell) ==
+             handshake_info_len);
+  memcpy(handshake_info,
+         trn_cell_rendezvous2_getconstarray_handshake_info(cell),
+         handshake_info_len);
+  ret = 0;
+
+ end:
+  trn_cell_rendezvous2_free(cell);
+  return ret;
+}
+
+/* Clear the given INTRODUCE1 data structure data. */
+void
+hs_cell_introduce1_data_clear(hs_cell_introduce1_data_t *data)
+{
+  if (data == NULL) {
+    return;
+  }
+  /* Object in this list have been moved to the cell object when building it
+   * so they've been freed earlier. We do that in order to avoid duplicating
+   * them leading to more memory and CPU time being used for nothing. */
+  smartlist_free(data->link_specifiers);
+  /* The data object has no ownership of any members. */
+  memwipe(data, 0, sizeof(hs_cell_introduce1_data_t));
+}
+
diff --git a/src/feature/hs/hs_cell.h b/src/feature/hs/hs_cell.h
new file mode 100644
index 0000000000..9569de535e
--- /dev/null
+++ b/src/feature/hs/hs_cell.h
@@ -0,0 +1,109 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_cell.h
+ * \brief Header file containing cell data for the whole HS subsytem.
+ **/
+
+#ifndef TOR_HS_CELL_H
+#define TOR_HS_CELL_H
+
+#include "core/or/or.h"
+#include "feature/hs/hs_service.h"
+
+/* An INTRODUCE1 cell requires at least this amount of bytes (see section
+ * 3.2.2 of the specification). Below this value, the cell must be padded. */
+#define HS_CELL_INTRODUCE1_MIN_SIZE 246
+
+/* This data structure contains data that we need to build an INTRODUCE1 cell
+ * used by the INTRODUCE1 build function. */
+typedef struct hs_cell_introduce1_data_t {
+  /* Is this a legacy introduction point? */
+  unsigned int is_legacy : 1;
+  /* (Legacy only) The encryption key for a legacy intro point. Only set if
+   * is_legacy is true. */
+  const crypto_pk_t *legacy_key;
+  /* Introduction point authentication public key. */
+  const ed25519_public_key_t *auth_pk;
+  /* Introduction point encryption public key. */
+  const curve25519_public_key_t *enc_pk;
+  /* Subcredentials of the service. */
+  const uint8_t *subcredential;
+  /* Onion public key for the ntor handshake. */
+  const curve25519_public_key_t *onion_pk;
+  /* Rendezvous cookie. */
+  const uint8_t *rendezvous_cookie;
+  /* Public key put before the encrypted data (CLIENT_PK). */
+  const curve25519_keypair_t *client_kp;
+  /* Rendezvous point link specifiers. */
+  smartlist_t *link_specifiers;
+} hs_cell_introduce1_data_t;
+
+/* This data structure contains data that we need to parse an INTRODUCE2 cell
+ * which is used by the INTRODUCE2 cell parsing function. On a successful
+ * parsing, the onion_pk and rendezvous_cookie will be populated with the
+ * computed key material from the cell data. This structure is only used during
+ * INTRO2 parsing and discarded after that. */
+typedef struct hs_cell_introduce2_data_t {
+  /*** Immutable Section: Set on structure init. ***/
+
+  /* Introduction point authentication public key. Pointer owned by the
+     introduction point object through which we received the INTRO2 cell. */
+  const ed25519_public_key_t *auth_pk;
+  /* Introduction point encryption keypair for the ntor handshake. Pointer
+     owned by the introduction point object through which we received the
+     INTRO2 cell*/
+  const curve25519_keypair_t *enc_kp;
+  /* Subcredentials of the service. Pointer owned by the descriptor that owns
+     the introduction point through which we received the INTRO2 cell. */
+  const uint8_t *subcredential;
+  /* Payload of the received encoded cell. */
+  const uint8_t *payload;
+  /* Size of the payload of the received encoded cell. */
+  size_t payload_len;
+
+  /*** Mutable Section: Set upon parsing INTRODUCE2 cell. ***/
+
+  /* Onion public key computed using the INTRODUCE2 encrypted section. */
+  curve25519_public_key_t onion_pk;
+  /* Rendezvous cookie taken from the INTRODUCE2 encrypted section. */
+  uint8_t rendezvous_cookie[REND_COOKIE_LEN];
+  /* Client public key from the INTRODUCE2 encrypted section. */
+  curve25519_public_key_t client_pk;
+  /* Link specifiers of the rendezvous point. Contains link_specifier_t. */
+  smartlist_t *link_specifiers;
+  /* Replay cache of the introduction point. */
+  replaycache_t *replay_cache;
+} hs_cell_introduce2_data_t;
+
+/* Build cell API. */
+ssize_t hs_cell_build_establish_intro(const char *circ_nonce,
+                                      const hs_service_intro_point_t *ip,
+                                      uint8_t *cell_out);
+ssize_t hs_cell_build_rendezvous1(const uint8_t *rendezvous_cookie,
+                                  size_t rendezvous_cookie_len,
+                                  const uint8_t *rendezvous_handshake_info,
+                                  size_t rendezvous_handshake_info_len,
+                                  uint8_t *cell_out);
+ssize_t hs_cell_build_introduce1(const hs_cell_introduce1_data_t *data,
+                                 uint8_t *cell_out);
+ssize_t hs_cell_build_establish_rendezvous(const uint8_t *rendezvous_cookie,
+                                           uint8_t *cell_out);
+
+/* Parse cell API. */
+ssize_t hs_cell_parse_intro_established(const uint8_t *payload,
+                                        size_t payload_len);
+ssize_t hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data,
+                                 const origin_circuit_t *circ,
+                                 const hs_service_t *service);
+int hs_cell_parse_introduce_ack(const uint8_t *payload, size_t payload_len);
+int hs_cell_parse_rendezvous2(const uint8_t *payload, size_t payload_len,
+                              uint8_t *handshake_info,
+                              size_t handshake_info_len);
+
+/* Util API. */
+void hs_cell_introduce1_data_clear(hs_cell_introduce1_data_t *data);
+
+#endif /* !defined(TOR_HS_CELL_H) */
+
diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c
new file mode 100644
index 0000000000..8acfcbd65b
--- /dev/null
+++ b/src/feature/hs/hs_circuit.c
@@ -0,0 +1,1287 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_circuit.c
+ **/
+
+#define HS_CIRCUIT_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/crypto/hs_ntor.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/policies.h"
+#include "core/or/relay.h"
+#include "feature/client/circpathbias.h"
+#include "feature/hs/hs_cell.h"
+#include "feature/hs/hs_circuit.h"
+#include "feature/hs/hs_circuitmap.h"
+#include "feature/hs/hs_ident.h"
+#include "feature/hs/hs_service.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/rend/rendservice.h"
+#include "feature/stats/rephist.h"
+#include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+
+/* Trunnel. */
+#include "trunnel/ed25519_cert.h"
+#include "trunnel/hs/cell_common.h"
+#include "trunnel/hs/cell_establish_intro.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/crypt_path_st.h"
+#include "feature/nodelist/node_st.h"
+#include "core/or/origin_circuit_st.h"
+
+/* A circuit is about to become an e2e rendezvous circuit. Check
+ * <b>circ_purpose</b> and ensure that it's properly set. Return true iff
+ * circuit purpose is properly set, otherwise return false. */
+static int
+circuit_purpose_is_correct_for_rend(unsigned int circ_purpose,
+                                    int is_service_side)
+{
+  if (is_service_side) {
+    if (circ_purpose != CIRCUIT_PURPOSE_S_CONNECT_REND) {
+      log_warn(LD_BUG,
+            "HS e2e circuit setup with wrong purpose (%d)", circ_purpose);
+      return 0;
+    }
+  }
+
+  if (!is_service_side) {
+    if (circ_purpose != CIRCUIT_PURPOSE_C_REND_READY &&
+        circ_purpose != CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED) {
+      log_warn(LD_BUG,
+            "Client e2e circuit setup with wrong purpose (%d)", circ_purpose);
+      return 0;
+    }
+  }
+
+  return 1;
+}
+
+/* Create and return a crypt path for the final hop of a v3 prop224 rendezvous
+ * circuit. Initialize the crypt path crypto using the output material from the
+ * ntor key exchange at <b>ntor_key_seed</b>.
+ *
+ * If <b>is_service_side</b> is set, we are the hidden service and the final
+ * hop of the rendezvous circuit is the client on the other side. */
+static crypt_path_t *
+create_rend_cpath(const uint8_t *ntor_key_seed, size_t seed_len,
+                  int is_service_side)
+{
+  uint8_t keys[HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN];
+  crypt_path_t *cpath = NULL;
+
+  /* Do the key expansion */
+  if (hs_ntor_circuit_key_expansion(ntor_key_seed, seed_len,
+                                    keys, sizeof(keys)) < 0) {
+    goto err;
+  }
+
+  /* Setup the cpath */
+  cpath = tor_malloc_zero(sizeof(crypt_path_t));
+  cpath->magic = CRYPT_PATH_MAGIC;
+
+  if (circuit_init_cpath_crypto(cpath, (char*)keys, sizeof(keys),
+                                is_service_side, 1) < 0) {
+    tor_free(cpath);
+    goto err;
+  }
+
+ err:
+  memwipe(keys, 0, sizeof(keys));
+  return cpath;
+}
+
+/* We are a v2 legacy HS client: Create and return a crypt path for the hidden
+ * service on the other side of the rendezvous circuit <b>circ</b>. Initialize
+ * the crypt path crypto using the body of the RENDEZVOUS1 cell at
+ * <b>rend_cell_body</b> (which must be at least DH1024_KEY_LEN+DIGEST_LEN
+ * bytes).
+ */
+static crypt_path_t *
+create_rend_cpath_legacy(origin_circuit_t *circ, const uint8_t *rend_cell_body)
+{
+  crypt_path_t *hop = NULL;
+  char keys[DIGEST_LEN+CPATH_KEY_MATERIAL_LEN];
+
+  /* first DH1024_KEY_LEN bytes are g^y from the service. Finish the dh
+   * handshake...*/
+  tor_assert(circ->build_state);
+  tor_assert(circ->build_state->pending_final_cpath);
+  hop = circ->build_state->pending_final_cpath;
+
+  tor_assert(hop->rend_dh_handshake_state);
+  if (crypto_dh_compute_secret(LOG_PROTOCOL_WARN, hop->rend_dh_handshake_state,
+                               (char*)rend_cell_body, DH1024_KEY_LEN,
+                               keys, DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) {
+    log_warn(LD_GENERAL, "Couldn't complete DH handshake.");
+    goto err;
+  }
+  /* ... and set up cpath. */
+  if (circuit_init_cpath_crypto(hop,
+                                keys+DIGEST_LEN, sizeof(keys)-DIGEST_LEN,
+                                0, 0) < 0)
+    goto err;
+
+  /* Check whether the digest is right... */
+  if (tor_memneq(keys, rend_cell_body+DH1024_KEY_LEN, DIGEST_LEN)) {
+    log_warn(LD_PROTOCOL, "Incorrect digest of key material.");
+    goto err;
+  }
+
+  /* clean up the crypto stuff we just made */
+  crypto_dh_free(hop->rend_dh_handshake_state);
+  hop->rend_dh_handshake_state = NULL;
+
+  goto done;
+
+ err:
+  hop = NULL;
+
+ done:
+  memwipe(keys, 0, sizeof(keys));
+  return hop;
+}
+
+/* Append the final <b>hop</b> to the cpath of the rend <b>circ</b>, and mark
+ * <b>circ</b> ready for use to transfer HS relay cells. */
+static void
+finalize_rend_circuit(origin_circuit_t *circ, crypt_path_t *hop,
+                      int is_service_side)
+{
+  tor_assert(circ);
+  tor_assert(hop);
+
+  /* Notify the circuit state machine that we are splicing this circuit */
+  int new_circ_purpose = is_service_side ?
+    CIRCUIT_PURPOSE_S_REND_JOINED : CIRCUIT_PURPOSE_C_REND_JOINED;
+  circuit_change_purpose(TO_CIRCUIT(circ), new_circ_purpose);
+
+  /* All is well. Extend the circuit. */
+  hop->state = CPATH_STATE_OPEN;
+  /* Set the windows to default. */
+  hop->package_window = circuit_initial_package_window();
+  hop->deliver_window = CIRCWINDOW_START;
+
+  /* Now that this circuit has finished connecting to its destination,
+   * make sure circuit_get_open_circ_or_launch is willing to return it
+   * so we can actually use it. */
+  circ->hs_circ_has_timed_out = 0;
+
+  /* Append the hop to the cpath of this circuit */
+  onion_append_to_cpath(&circ->cpath, hop);
+
+  /* In legacy code, 'pending_final_cpath' points to the final hop we just
+   * appended to the cpath. We set the original pointer to NULL so that we
+   * don't double free it. */
+  if (circ->build_state) {
+    circ->build_state->pending_final_cpath = NULL;
+  }
+
+  /* Finally, mark circuit as ready to be used for client streams */
+  if (!is_service_side) {
+    circuit_try_attaching_streams(circ);
+  }
+}
+
+/* For a given circuit and a service introduction point object, register the
+ * intro circuit to the circuitmap. This supports legacy intro point. */
+static void
+register_intro_circ(const hs_service_intro_point_t *ip,
+                    origin_circuit_t *circ)
+{
+  tor_assert(ip);
+  tor_assert(circ);
+
+  if (ip->base.is_only_legacy) {
+    hs_circuitmap_register_intro_circ_v2_service_side(circ,
+                                                      ip->legacy_key_digest);
+  } else {
+    hs_circuitmap_register_intro_circ_v3_service_side(circ,
+                                         &ip->auth_key_kp.pubkey);
+  }
+}
+
+/* Return the number of opened introduction circuit for the given circuit that
+ * is matching its identity key. */
+static unsigned int
+count_opened_desc_intro_point_circuits(const hs_service_t *service,
+                                       const hs_service_descriptor_t *desc)
+{
+  unsigned int count = 0;
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  DIGEST256MAP_FOREACH(desc->intro_points.map, key,
+                       const hs_service_intro_point_t *, ip) {
+    const circuit_t *circ;
+    const origin_circuit_t *ocirc = hs_circ_service_get_intro_circ(ip);
+    if (ocirc == NULL) {
+      continue;
+    }
+    circ = TO_CIRCUIT(ocirc);
+    tor_assert(circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
+               circ->purpose == CIRCUIT_PURPOSE_S_INTRO);
+    /* Having a circuit not for the requested service is really bad. */
+    tor_assert(ed25519_pubkey_eq(&service->keys.identity_pk,
+                                 &ocirc->hs_ident->identity_pk));
+    /* Only count opened circuit and skip circuit that will be closed. */
+    if (!circ->marked_for_close && circ->state == CIRCUIT_STATE_OPEN) {
+      count++;
+    }
+  } DIGEST256MAP_FOREACH_END;
+  return count;
+}
+
+/* From a given service, rendezvous cookie and handshake info, create a
+ * rendezvous point circuit identifier. This can't fail. */
+STATIC hs_ident_circuit_t *
+create_rp_circuit_identifier(const hs_service_t *service,
+                             const uint8_t *rendezvous_cookie,
+                             const curve25519_public_key_t *server_pk,
+                             const hs_ntor_rend_cell_keys_t *keys)
+{
+  hs_ident_circuit_t *ident;
+  uint8_t handshake_info[CURVE25519_PUBKEY_LEN + DIGEST256_LEN];
+
+  tor_assert(service);
+  tor_assert(rendezvous_cookie);
+  tor_assert(server_pk);
+  tor_assert(keys);
+
+  ident = hs_ident_circuit_new(&service->keys.identity_pk,
+                               HS_IDENT_CIRCUIT_RENDEZVOUS);
+  /* Copy the RENDEZVOUS_COOKIE which is the unique identifier. */
+  memcpy(ident->rendezvous_cookie, rendezvous_cookie,
+         sizeof(ident->rendezvous_cookie));
+  /* Build the HANDSHAKE_INFO which looks like this:
+   *    SERVER_PK        [32 bytes]
+   *    AUTH_INPUT_MAC   [32 bytes]
+   */
+  memcpy(handshake_info, server_pk->public_key, CURVE25519_PUBKEY_LEN);
+  memcpy(handshake_info + CURVE25519_PUBKEY_LEN, keys->rend_cell_auth_mac,
+         DIGEST256_LEN);
+  tor_assert(sizeof(ident->rendezvous_handshake_info) ==
+             sizeof(handshake_info));
+  memcpy(ident->rendezvous_handshake_info, handshake_info,
+         sizeof(ident->rendezvous_handshake_info));
+  /* Finally copy the NTOR_KEY_SEED for e2e encryption on the circuit. */
+  tor_assert(sizeof(ident->rendezvous_ntor_key_seed) ==
+             sizeof(keys->ntor_key_seed));
+  memcpy(ident->rendezvous_ntor_key_seed, keys->ntor_key_seed,
+         sizeof(ident->rendezvous_ntor_key_seed));
+  return ident;
+}
+
+/* From a given service and service intro point, create an introduction point
+ * circuit identifier. This can't fail. */
+static hs_ident_circuit_t *
+create_intro_circuit_identifier(const hs_service_t *service,
+                                const hs_service_intro_point_t *ip)
+{
+  hs_ident_circuit_t *ident;
+
+  tor_assert(service);
+  tor_assert(ip);
+
+  ident = hs_ident_circuit_new(&service->keys.identity_pk,
+                               HS_IDENT_CIRCUIT_INTRO);
+  ed25519_pubkey_copy(&ident->intro_auth_pk, &ip->auth_key_kp.pubkey);
+
+  return ident;
+}
+
+/* For a given introduction point and an introduction circuit, send the
+ * ESTABLISH_INTRO cell. The service object is used for logging. This can fail
+ * and if so, the circuit is closed and the intro point object is flagged
+ * that the circuit is not established anymore which is important for the
+ * retry mechanism. */
+static void
+send_establish_intro(const hs_service_t *service,
+                     hs_service_intro_point_t *ip, origin_circuit_t *circ)
+{
+  ssize_t cell_len;
+  uint8_t payload[RELAY_PAYLOAD_SIZE];
+
+  tor_assert(service);
+  tor_assert(ip);
+  tor_assert(circ);
+
+  /* Encode establish intro cell. */
+  cell_len = hs_cell_build_establish_intro(circ->cpath->prev->rend_circ_nonce,
+                                           ip, payload);
+  if (cell_len < 0) {
+    log_warn(LD_REND, "Unable to encode ESTABLISH_INTRO cell for service %s "
+                      "on circuit %u. Closing circuit.",
+             safe_str_client(service->onion_address),
+             TO_CIRCUIT(circ)->n_circ_id);
+    goto err;
+  }
+
+  /* Send the cell on the circuit. */
+  if (relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(circ),
+                                   RELAY_COMMAND_ESTABLISH_INTRO,
+                                   (char *) payload, cell_len,
+                                   circ->cpath->prev) < 0) {
+    log_info(LD_REND, "Unable to send ESTABLISH_INTRO cell for service %s "
+                      "on circuit %u.",
+             safe_str_client(service->onion_address),
+             TO_CIRCUIT(circ)->n_circ_id);
+    /* On error, the circuit has been closed. */
+    goto done;
+  }
+
+  /* Record the attempt to use this circuit. */
+  pathbias_count_use_attempt(circ);
+  goto done;
+
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
+ done:
+  memwipe(payload, 0, sizeof(payload));
+}
+
+/* Return a string constant describing the anonymity of service. */
+static const char *
+get_service_anonymity_string(const hs_service_t *service)
+{
+  if (service->config.is_single_onion) {
+    return "single onion";
+  } else {
+    return "hidden";
+  }
+}
+
+/* For a given service, the ntor onion key and a rendezvous cookie, launch a
+ * circuit to the rendezvous point specified by the link specifiers. On
+ * success, a circuit identifier is attached to the circuit with the needed
+ * data. This function will try to open a circuit for a maximum value of
+ * MAX_REND_FAILURES then it will give up. */
+static void
+launch_rendezvous_point_circuit(const hs_service_t *service,
+                                const hs_service_intro_point_t *ip,
+                                const hs_cell_introduce2_data_t *data)
+{
+  int circ_needs_uptime;
+  time_t now = time(NULL);
+  extend_info_t *info = NULL;
+  origin_circuit_t *circ;
+
+  tor_assert(service);
+  tor_assert(ip);
+  tor_assert(data);
+
+  circ_needs_uptime = hs_service_requires_uptime_circ(service->config.ports);
+
+  /* Get the extend info data structure for the chosen rendezvous point
+   * specified by the given link specifiers. */
+  info = hs_get_extend_info_from_lspecs(data->link_specifiers,
+                                        &data->onion_pk,
+                                        service->config.is_single_onion);
+  if (info == NULL) {
+    /* We are done here, we can't extend to the rendezvous point.
+     * If you're running an IPv6-only v3 single onion service on 0.3.2 or with
+     * 0.3.2 clients, and somehow disable the option check, it will fail here.
+     */
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Not enough info to open a circuit to a rendezvous point for "
+           "%s service %s.",
+           get_service_anonymity_string(service),
+           safe_str_client(service->onion_address));
+    goto end;
+  }
+
+  for (int i = 0; i < MAX_REND_FAILURES; i++) {
+    int circ_flags = CIRCLAUNCH_NEED_CAPACITY | CIRCLAUNCH_IS_INTERNAL;
+    if (circ_needs_uptime) {
+      circ_flags |= CIRCLAUNCH_NEED_UPTIME;
+    }
+    /* Firewall and policies are checked when getting the extend info.
+     *
+     * We only use a one-hop path on the first attempt. If the first attempt
+     * fails, we use a 3-hop path for reachability / reliability.
+     * See the comment in retry_service_rendezvous_point() for details. */
+    if (service->config.is_single_onion && i == 0) {
+      circ_flags |= CIRCLAUNCH_ONEHOP_TUNNEL;
+    }
+
+    circ = circuit_launch_by_extend_info(CIRCUIT_PURPOSE_S_CONNECT_REND, info,
+                                         circ_flags);
+    if (circ != NULL) {
+      /* Stop retrying, we have a circuit! */
+      break;
+    }
+  }
+  if (circ == NULL) {
+    log_warn(LD_REND, "Giving up on launching a rendezvous circuit to %s "
+                      "for %s service %s",
+             safe_str_client(extend_info_describe(info)),
+             get_service_anonymity_string(service),
+             safe_str_client(service->onion_address));
+    goto end;
+  }
+  log_info(LD_REND, "Rendezvous circuit launched to %s with cookie %s "
+                    "for %s service %s",
+           safe_str_client(extend_info_describe(info)),
+           safe_str_client(hex_str((const char *) data->rendezvous_cookie,
+                                   REND_COOKIE_LEN)),
+           get_service_anonymity_string(service),
+           safe_str_client(service->onion_address));
+  tor_assert(circ->build_state);
+  /* Rendezvous circuit have a specific timeout for the time spent on trying
+   * to connect to the rendezvous point. */
+  circ->build_state->expiry_time = now + MAX_REND_TIMEOUT;
+
+  /* Create circuit identifier and key material. */
+  {
+    hs_ntor_rend_cell_keys_t keys;
+    curve25519_keypair_t ephemeral_kp;
+    /* No need for extra strong, this is only for this circuit life time. This
+     * key will be used for the RENDEZVOUS1 cell that will be sent on the
+     * circuit once opened. */
+    curve25519_keypair_generate(&ephemeral_kp, 0);
+    if (hs_ntor_service_get_rendezvous1_keys(&ip->auth_key_kp.pubkey,
+                                             &ip->enc_key_kp,
+                                             &ephemeral_kp, &data->client_pk,
+                                             &keys) < 0) {
+      /* This should not really happened but just in case, don't make tor
+       * freak out, close the circuit and move on. */
+      log_info(LD_REND, "Unable to get RENDEZVOUS1 key material for "
+                        "service %s",
+               safe_str_client(service->onion_address));
+      circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
+      goto end;
+    }
+    circ->hs_ident = create_rp_circuit_identifier(service,
+                                                  data->rendezvous_cookie,
+                                                  &ephemeral_kp.pubkey, &keys);
+    memwipe(&ephemeral_kp, 0, sizeof(ephemeral_kp));
+    memwipe(&keys, 0, sizeof(keys));
+    tor_assert(circ->hs_ident);
+  }
+
+ end:
+  extend_info_free(info);
+}
+
+/* Return true iff the given service rendezvous circuit circ is allowed for a
+ * relaunch to the rendezvous point. */
+static int
+can_relaunch_service_rendezvous_point(const origin_circuit_t *circ)
+{
+  tor_assert(circ);
+  /* This is initialized when allocating an origin circuit. */
+  tor_assert(circ->build_state);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
+
+  /* XXX: Retrying under certain condition. This is related to #22455. */
+
+  /* Avoid to relaunch twice a circuit to the same rendezvous point at the
+   * same time. */
+  if (circ->hs_service_side_rend_circ_has_been_relaunched) {
+    log_info(LD_REND, "Rendezvous circuit to %s has already been retried. "
+                      "Skipping retry.",
+             safe_str_client(
+                  extend_info_describe(circ->build_state->chosen_exit)));
+    goto disallow;
+  }
+
+  /* We check failure_count >= hs_get_service_max_rend_failures()-1 below, and
+   * the -1 is because we increment the failure count for our current failure
+   * *after* this clause. */
+  int max_rend_failures = hs_get_service_max_rend_failures() - 1;
+
+  /* A failure count that has reached maximum allowed or circuit that expired,
+   * we skip relaunching. */
+  if (circ->build_state->failure_count > max_rend_failures ||
+      circ->build_state->expiry_time <= time(NULL)) {
+    log_info(LD_REND, "Attempt to build a rendezvous circuit to %s has "
+                      "failed with %d attempts and expiry time %ld. "
+                      "Giving up building.",
+             safe_str_client(
+                  extend_info_describe(circ->build_state->chosen_exit)),
+             circ->build_state->failure_count,
+             (long int) circ->build_state->expiry_time);
+    goto disallow;
+  }
+
+  /* Allowed to relaunch. */
+  return 1;
+ disallow:
+  return 0;
+}
+
+/* Retry the rendezvous point of circ by launching a new circuit to it. */
+static void
+retry_service_rendezvous_point(const origin_circuit_t *circ)
+{
+  int flags = 0;
+  origin_circuit_t *new_circ;
+  cpath_build_state_t *bstate;
+
+  tor_assert(circ);
+  /* This is initialized when allocating an origin circuit. */
+  tor_assert(circ->build_state);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
+
+  /* Ease our life. */
+  bstate = circ->build_state;
+
+  log_info(LD_REND, "Retrying rendezvous point circuit to %s",
+           safe_str_client(extend_info_describe(bstate->chosen_exit)));
+
+  /* Get the current build state flags for the next circuit. */
+  flags |= (bstate->need_uptime) ? CIRCLAUNCH_NEED_UPTIME : 0;
+  flags |= (bstate->need_capacity) ? CIRCLAUNCH_NEED_CAPACITY : 0;
+  flags |= (bstate->is_internal) ? CIRCLAUNCH_IS_INTERNAL : 0;
+
+  /* We do NOT add the onehop tunnel flag even though it might be a single
+   * onion service. The reason is that if we failed once to connect to the RP
+   * with a direct connection, we consider that chances are that we will fail
+   * again so try a 3-hop circuit and hope for the best. Because the service
+   * has no anonymity (single onion), this change of behavior won't affect
+   * security directly. */
+
+  new_circ = circuit_launch_by_extend_info(CIRCUIT_PURPOSE_S_CONNECT_REND,
+                                           bstate->chosen_exit, flags);
+  if (new_circ == NULL) {
+    log_warn(LD_REND, "Failed to launch rendezvous circuit to %s",
+             safe_str_client(extend_info_describe(bstate->chosen_exit)));
+    goto done;
+  }
+
+  /* Transfer build state information to the new circuit state in part to
+   * catch any other failures. */
+  new_circ->build_state->failure_count = bstate->failure_count+1;
+  new_circ->build_state->expiry_time = bstate->expiry_time;
+  new_circ->hs_ident = hs_ident_circuit_dup(circ->hs_ident);
+
+ done:
+  return;
+}
+
+/* Add all possible link specifiers in node to lspecs:
+ *  - legacy ID is mandatory thus MUST be present in node;
+ *  - include ed25519 link specifier if present in the node, and the node
+ *    supports ed25519 link authentication, even if its link versions are not
+ *    compatible with us;
+ *  - include IPv4 link specifier, if the primary address is not IPv4, log a
+ *    BUG() warning, and return an empty smartlist;
+ *  - include IPv6 link specifier if present in the node. */
+static void
+get_lspecs_from_node(const node_t *node, smartlist_t *lspecs)
+{
+  link_specifier_t *ls;
+  tor_addr_port_t ap;
+
+  tor_assert(node);
+  tor_assert(lspecs);
+
+  /* Get the relay's IPv4 address. */
+  node_get_prim_orport(node, &ap);
+
+  /* We expect the node's primary address to be a valid IPv4 address.
+   * This conforms to the protocol, which requires either an IPv4 or IPv6
+   * address (or both). */
+  if (BUG(!tor_addr_is_v4(&ap.addr)) ||
+      BUG(!tor_addr_port_is_valid_ap(&ap, 0))) {
+    return;
+  }
+
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_IPV4);
+  link_specifier_set_un_ipv4_addr(ls, tor_addr_to_ipv4h(&ap.addr));
+  link_specifier_set_un_ipv4_port(ls, ap.port);
+  /* Four bytes IPv4 and two bytes port. */
+  link_specifier_set_ls_len(ls, sizeof(ap.addr.addr.in_addr) +
+                            sizeof(ap.port));
+  smartlist_add(lspecs, ls);
+
+  /* Legacy ID is mandatory and will always be present in node. */
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_LEGACY_ID);
+  memcpy(link_specifier_getarray_un_legacy_id(ls), node->identity,
+         link_specifier_getlen_un_legacy_id(ls));
+  link_specifier_set_ls_len(ls, link_specifier_getlen_un_legacy_id(ls));
+  smartlist_add(lspecs, ls);
+
+  /* ed25519 ID is only included if the node has it, and the node declares a
+     protocol version that supports ed25519 link authentication, even if that
+     link version is not compatible with us. (We are sending the ed25519 key
+     to another tor, which may support different link versions.) */
+  if (!ed25519_public_key_is_zero(&node->ed25519_id) &&
+      node_supports_ed25519_link_authentication(node, 0)) {
+    ls = link_specifier_new();
+    link_specifier_set_ls_type(ls, LS_ED25519_ID);
+    memcpy(link_specifier_getarray_un_ed25519_id(ls), &node->ed25519_id,
+           link_specifier_getlen_un_ed25519_id(ls));
+    link_specifier_set_ls_len(ls, link_specifier_getlen_un_ed25519_id(ls));
+    smartlist_add(lspecs, ls);
+  }
+
+  /* Check for IPv6. If so, include it as well. */
+  if (node_has_ipv6_orport(node)) {
+    ls = link_specifier_new();
+    node_get_pref_ipv6_orport(node, &ap);
+    link_specifier_set_ls_type(ls, LS_IPV6);
+    size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls);
+    const uint8_t *in6_addr = tor_addr_to_in6_addr8(&ap.addr);
+    uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls);
+    memcpy(ipv6_array, in6_addr, addr_len);
+    link_specifier_set_un_ipv6_port(ls, ap.port);
+    /* Sixteen bytes IPv6 and two bytes port. */
+    link_specifier_set_ls_len(ls, addr_len + sizeof(ap.port));
+    smartlist_add(lspecs, ls);
+  }
+}
+
+/* Using the given descriptor intro point ip, the node of the
+ * rendezvous point rp_node and the service's subcredential, populate the
+ * already allocated intro1_data object with the needed key material and link
+ * specifiers.
+ *
+ * Return 0 on success or a negative value if we couldn't properly filled the
+ * introduce1 data from the RP node. In other word, it means the RP node is
+ * unusable to use in the introduction. */
+static int
+setup_introduce1_data(const hs_desc_intro_point_t *ip,
+                      const node_t *rp_node,
+                      const uint8_t *subcredential,
+                      hs_cell_introduce1_data_t *intro1_data)
+{
+  int ret = -1;
+  smartlist_t *rp_lspecs;
+
+  tor_assert(ip);
+  tor_assert(rp_node);
+  tor_assert(subcredential);
+  tor_assert(intro1_data);
+
+  /* Build the link specifiers from the extend information of the rendezvous
+   * circuit that we've picked previously. */
+  rp_lspecs = smartlist_new();
+  get_lspecs_from_node(rp_node, rp_lspecs);
+  if (smartlist_len(rp_lspecs) == 0) {
+    /* We can't rendezvous without link specifiers. */
+    smartlist_free(rp_lspecs);
+    goto end;
+  }
+
+  /* Populate the introduce1 data object. */
+  memset(intro1_data, 0, sizeof(hs_cell_introduce1_data_t));
+  if (ip->legacy.key != NULL) {
+    intro1_data->is_legacy = 1;
+    intro1_data->legacy_key = ip->legacy.key;
+  }
+  intro1_data->auth_pk = &ip->auth_key_cert->signed_key;
+  intro1_data->enc_pk = &ip->enc_key;
+  intro1_data->subcredential = subcredential;
+  intro1_data->link_specifiers = rp_lspecs;
+  intro1_data->onion_pk = node_get_curve25519_onion_key(rp_node);
+  if (intro1_data->onion_pk == NULL) {
+    /* We can't rendezvous without the curve25519 onion key. */
+    goto end;
+  }
+  /* Success, we have valid introduce data. */
+  ret = 0;
+
+ end:
+  return ret;
+}
+
+/* ========== */
+/* Public API */
+/* ========== */
+
+/* Return an introduction point circuit matching the given intro point object.
+ * NULL is returned is no such circuit can be found. */
+origin_circuit_t *
+hs_circ_service_get_intro_circ(const hs_service_intro_point_t *ip)
+{
+  tor_assert(ip);
+
+  if (ip->base.is_only_legacy) {
+    return hs_circuitmap_get_intro_circ_v2_service_side(ip->legacy_key_digest);
+  } else {
+    return hs_circuitmap_get_intro_circ_v3_service_side(
+                                        &ip->auth_key_kp.pubkey);
+  }
+}
+
+/* Called when we fail building a rendezvous circuit at some point other than
+ * the last hop: launches a new circuit to the same rendezvous point. This
+ * supports legacy service.
+ *
+ * We currently relaunch connections to rendezvous points if:
+ * - A rendezvous circuit timed out before connecting to RP.
+ * - The rendezvous circuit failed to connect to the RP.
+ *
+ * We avoid relaunching a connection to this rendezvous point if:
+ * - We have already tried MAX_REND_FAILURES times to connect to this RP,
+ * - We've been trying to connect to this RP for more than MAX_REND_TIMEOUT
+ *   seconds, or
+ * - We've already retried this specific rendezvous circuit.
+ */
+void
+hs_circ_retry_service_rendezvous_point(origin_circuit_t *circ)
+{
+  tor_assert(circ);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
+
+  /* Check if we are allowed to relaunch to the rendezvous point of circ. */
+  if (!can_relaunch_service_rendezvous_point(circ)) {
+    goto done;
+  }
+
+  /* Flag the circuit that we are relaunching, to avoid to relaunch twice a
+   * circuit to the same rendezvous point at the same time. */
+  circ->hs_service_side_rend_circ_has_been_relaunched = 1;
+
+  /* Legacy services don't have a hidden service ident. */
+  if (circ->hs_ident) {
+    retry_service_rendezvous_point(circ);
+  } else {
+    rend_service_relaunch_rendezvous(circ);
+  }
+
+ done:
+  return;
+}
+
+/* For a given service and a service intro point, launch a circuit to the
+ * extend info ei. If the service is a single onion, and direct_conn is true,
+ * a one-hop circuit will be requested.
+ *
+ * Return 0 if the circuit was successfully launched and tagged
+ * with the correct identifier. On error, a negative value is returned. */
+int
+hs_circ_launch_intro_point(hs_service_t *service,
+                           const hs_service_intro_point_t *ip,
+                           extend_info_t *ei,
+                           bool direct_conn)
+{
+  /* Standard flags for introduction circuit. */
+  int ret = -1, circ_flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
+  origin_circuit_t *circ;
+
+  tor_assert(service);
+  tor_assert(ip);
+  tor_assert(ei);
+
+  /* Update circuit flags in case of a single onion service that requires a
+   * direct connection. */
+  tor_assert_nonfatal(ip->circuit_retries > 0);
+  /* Only single onion services can make direct conns */
+  if (BUG(!service->config.is_single_onion && direct_conn)) {
+    goto end;
+  }
+  /* We only use a one-hop path on the first attempt. If the first attempt
+   * fails, we use a 3-hop path for reachability / reliability.
+   * (Unlike v2, retries is incremented by the caller before it calls this
+   * function.) */
+  if (direct_conn && ip->circuit_retries == 1) {
+    circ_flags |= CIRCLAUNCH_ONEHOP_TUNNEL;
+  }
+
+  log_info(LD_REND, "Launching a circuit to intro point %s for service %s.",
+           safe_str_client(extend_info_describe(ei)),
+           safe_str_client(service->onion_address));
+
+  /* Note down the launch for the retry period. Even if the circuit fails to
+   * be launched, we still want to respect the retry period to avoid stress on
+   * the circuit subsystem. */
+  service->state.num_intro_circ_launched++;
+  circ = circuit_launch_by_extend_info(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO,
+                                       ei, circ_flags);
+  if (circ == NULL) {
+    goto end;
+  }
+
+  /* Setup the circuit identifier and attach it to it. */
+  circ->hs_ident = create_intro_circuit_identifier(service, ip);
+  tor_assert(circ->hs_ident);
+  /* Register circuit in the global circuitmap. */
+  register_intro_circ(ip, circ);
+
+  /* Success. */
+  ret = 0;
+ end:
+  return ret;
+}
+
+/* Called when a service introduction point circuit is done building. Given
+ * the service and intro point object, this function will send the
+ * ESTABLISH_INTRO cell on the circuit. Return 0 on success. Return 1 if the
+ * circuit has been repurposed to General because we already have too many
+ * opened. */
+int
+hs_circ_service_intro_has_opened(hs_service_t *service,
+                                 hs_service_intro_point_t *ip,
+                                 const hs_service_descriptor_t *desc,
+                                 origin_circuit_t *circ)
+{
+  int ret = 0;
+  unsigned int num_intro_circ, num_needed_circ;
+
+  tor_assert(service);
+  tor_assert(ip);
+  tor_assert(desc);
+  tor_assert(circ);
+
+  /* Cound opened circuits that have sent ESTABLISH_INTRO cells or are already
+   * established introduction circuits */
+  num_intro_circ = count_opened_desc_intro_point_circuits(service, desc);
+  num_needed_circ = service->config.num_intro_points;
+  if (num_intro_circ > num_needed_circ) {
+    /* There are too many opened valid intro circuit for what the service
+     * needs so repurpose this one. */
+
+    /* XXX: Legacy code checks options->ExcludeNodes and if not NULL it just
+     * closes the circuit. I have NO idea why it does that so it hasn't been
+     * added here. I can only assume in case our ExcludeNodes list changes but
+     * in that case, all circuit are flagged unusable (config.c). --dgoulet */
+
+    log_info(LD_CIRC | LD_REND, "Introduction circuit just opened but we "
+                                "have enough for service %s. Repurposing "
+                                "it to general and leaving internal.",
+             safe_str_client(service->onion_address));
+    tor_assert(circ->build_state->is_internal);
+    /* Remove it from the circuitmap. */
+    hs_circuitmap_remove_circuit(TO_CIRCUIT(circ));
+    /* Cleaning up the hidden service identifier and repurpose. */
+    hs_ident_circuit_free(circ->hs_ident);
+    circ->hs_ident = NULL;
+    if (circuit_should_use_vanguards(TO_CIRCUIT(circ)->purpose))
+      circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_HS_VANGUARDS);
+    else
+      circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_GENERAL);
+
+    /* Inform that this circuit just opened for this new purpose. */
+    circuit_has_opened(circ);
+    /* This return value indicate to the caller that the IP object should be
+     * removed from the service because it's corresponding circuit has just
+     * been repurposed. */
+    ret = 1;
+    goto done;
+  }
+
+  log_info(LD_REND, "Introduction circuit %u established for service %s.",
+           TO_CIRCUIT(circ)->n_circ_id,
+           safe_str_client(service->onion_address));
+  circuit_log_path(LOG_INFO, LD_REND, circ);
+
+  /* Time to send an ESTABLISH_INTRO cell on this circuit. On error, this call
+   * makes sure the circuit gets closed. */
+  send_establish_intro(service, ip, circ);
+
+ done:
+  return ret;
+}
+
+/* Called when a service rendezvous point circuit is done building. Given the
+ * service and the circuit, this function will send a RENDEZVOUS1 cell on the
+ * circuit using the information in the circuit identifier. If the cell can't
+ * be sent, the circuit is closed. */
+void
+hs_circ_service_rp_has_opened(const hs_service_t *service,
+                              origin_circuit_t *circ)
+{
+  size_t payload_len;
+  uint8_t payload[RELAY_PAYLOAD_SIZE] = {0};
+
+  tor_assert(service);
+  tor_assert(circ);
+  tor_assert(circ->hs_ident);
+
+  /* Some useful logging. */
+  log_info(LD_REND, "Rendezvous circuit %u has opened with cookie %s "
+                    "for service %s",
+           TO_CIRCUIT(circ)->n_circ_id,
+           hex_str((const char *) circ->hs_ident->rendezvous_cookie,
+                   REND_COOKIE_LEN),
+           safe_str_client(service->onion_address));
+  circuit_log_path(LOG_INFO, LD_REND, circ);
+
+  /* This can't fail. */
+  payload_len = hs_cell_build_rendezvous1(
+                        circ->hs_ident->rendezvous_cookie,
+                        sizeof(circ->hs_ident->rendezvous_cookie),
+                        circ->hs_ident->rendezvous_handshake_info,
+                        sizeof(circ->hs_ident->rendezvous_handshake_info),
+                        payload);
+
+  /* Pad the payload with random bytes so it matches the size of a legacy cell
+   * which is normally always bigger. Also, the size of a legacy cell is
+   * always smaller than the RELAY_PAYLOAD_SIZE so this is safe. */
+  if (payload_len < HS_LEGACY_RENDEZVOUS_CELL_SIZE) {
+    crypto_rand((char *) payload + payload_len,
+                HS_LEGACY_RENDEZVOUS_CELL_SIZE - payload_len);
+    payload_len = HS_LEGACY_RENDEZVOUS_CELL_SIZE;
+  }
+
+  if (relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(circ),
+                                   RELAY_COMMAND_RENDEZVOUS1,
+                                   (const char *) payload, payload_len,
+                                   circ->cpath->prev) < 0) {
+    /* On error, circuit is closed. */
+    log_warn(LD_REND, "Unable to send RENDEZVOUS1 cell on circuit %u "
+                      "for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto done;
+  }
+
+  /* Setup end-to-end rendezvous circuit between the client and us. */
+  if (hs_circuit_setup_e2e_rend_circ(circ,
+                       circ->hs_ident->rendezvous_ntor_key_seed,
+                       sizeof(circ->hs_ident->rendezvous_ntor_key_seed),
+                       1) < 0) {
+    log_warn(LD_GENERAL, "Failed to setup circ");
+    goto done;
+  }
+
+ done:
+  memwipe(payload, 0, sizeof(payload));
+}
+
+/* Circ has been expecting an INTRO_ESTABLISHED cell that just arrived. Handle
+ * the INTRO_ESTABLISHED cell payload of length payload_len arriving on the
+ * given introduction circuit circ. The service is only used for logging
+ * purposes. Return 0 on success else a negative value. */
+int
+hs_circ_handle_intro_established(const hs_service_t *service,
+                                 const hs_service_intro_point_t *ip,
+                                 origin_circuit_t *circ,
+                                 const uint8_t *payload, size_t payload_len)
+{
+  int ret = -1;
+
+  tor_assert(service);
+  tor_assert(ip);
+  tor_assert(circ);
+  tor_assert(payload);
+
+  if (BUG(TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO)) {
+    goto done;
+  }
+
+  /* Try to parse the payload into a cell making sure we do actually have a
+   * valid cell. For a legacy node, it's an empty payload so as long as we
+   * have the cell, we are good. */
+  if (!ip->base.is_only_legacy &&
+      hs_cell_parse_intro_established(payload, payload_len) < 0) {
+    log_warn(LD_REND, "Unable to parse the INTRO_ESTABLISHED cell on "
+                      "circuit %u for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto done;
+  }
+
+  /* Switch the purpose to a fully working intro point. */
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_S_INTRO);
+  /* Getting a valid INTRODUCE_ESTABLISHED means we've successfully used the
+   * circuit so update our pathbias subsystem. */
+  pathbias_mark_use_success(circ);
+  /* Success. */
+  ret = 0;
+
+ done:
+  return ret;
+}
+
+/* We just received an INTRODUCE2 cell on the established introduction circuit
+ * circ.  Handle the INTRODUCE2 payload of size payload_len for the given
+ * circuit and service. This cell is associated with the intro point object ip
+ * and the subcredential. Return 0 on success else a negative value. */
+int
+hs_circ_handle_introduce2(const hs_service_t *service,
+                          const origin_circuit_t *circ,
+                          hs_service_intro_point_t *ip,
+                          const uint8_t *subcredential,
+                          const uint8_t *payload, size_t payload_len)
+{
+  int ret = -1;
+  time_t elapsed;
+  hs_cell_introduce2_data_t data;
+
+  tor_assert(service);
+  tor_assert(circ);
+  tor_assert(ip);
+  tor_assert(subcredential);
+  tor_assert(payload);
+
+  /* Populate the data structure with everything we need for the cell to be
+   * parsed, decrypted and key material computed correctly. */
+  data.auth_pk = &ip->auth_key_kp.pubkey;
+  data.enc_kp = &ip->enc_key_kp;
+  data.subcredential = subcredential;
+  data.payload = payload;
+  data.payload_len = payload_len;
+  data.link_specifiers = smartlist_new();
+  data.replay_cache = ip->replay_cache;
+
+  if (hs_cell_parse_introduce2(&data, circ, service) < 0) {
+    goto done;
+  }
+
+  /* Check whether we've seen this REND_COOKIE before to detect repeats. */
+  if (replaycache_add_test_and_elapsed(
+           service->state.replay_cache_rend_cookie,
+           data.rendezvous_cookie, sizeof(data.rendezvous_cookie),
+           &elapsed)) {
+    /* A Tor client will send a new INTRODUCE1 cell with the same REND_COOKIE
+     * as its previous one if its intro circ times out while in state
+     * CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT. If we received the first
+     * INTRODUCE1 cell (the intro-point relay converts it into an INTRODUCE2
+     * cell), we are already trying to connect to that rend point (and may
+     * have already succeeded); drop this cell. */
+    log_info(LD_REND, "We received an INTRODUCE2 cell with same REND_COOKIE "
+                      "field %ld seconds ago. Dropping cell.",
+             (long int) elapsed);
+    goto done;
+  }
+
+  /* At this point, we just confirmed that the full INTRODUCE2 cell is valid
+   * so increment our counter that we've seen one on this intro point. */
+  ip->introduce2_count++;
+
+  /* Launch rendezvous circuit with the onion key and rend cookie. */
+  launch_rendezvous_point_circuit(service, ip, &data);
+  /* Success. */
+  ret = 0;
+
+ done:
+  SMARTLIST_FOREACH(data.link_specifiers, link_specifier_t *, lspec,
+                    link_specifier_free(lspec));
+  smartlist_free(data.link_specifiers);
+  memwipe(&data, 0, sizeof(data));
+  return ret;
+}
+
+/* Circuit <b>circ</b> just finished the rend ntor key exchange. Use the key
+ * exchange output material at <b>ntor_key_seed</b> and setup <b>circ</b> to
+ * serve as a rendezvous end-to-end circuit between the client and the
+ * service. If <b>is_service_side</b> is set, then we are the hidden service
+ * and the other side is the client.
+ *
+ * Return 0 if the operation went well; in case of error return -1. */
+int
+hs_circuit_setup_e2e_rend_circ(origin_circuit_t *circ,
+                               const uint8_t *ntor_key_seed, size_t seed_len,
+                               int is_service_side)
+{
+  if (BUG(!circuit_purpose_is_correct_for_rend(TO_CIRCUIT(circ)->purpose,
+                                        is_service_side))) {
+    return -1;
+  }
+
+  crypt_path_t *hop = create_rend_cpath(ntor_key_seed, seed_len,
+                                        is_service_side);
+  if (!hop) {
+    log_warn(LD_REND, "Couldn't get v3 %s cpath!",
+             is_service_side ? "service-side" : "client-side");
+    return -1;
+  }
+
+  finalize_rend_circuit(circ, hop, is_service_side);
+
+  return 0;
+}
+
+/* We are a v2 legacy HS client and we just received a RENDEZVOUS1 cell
+ * <b>rend_cell_body</b> on <b>circ</b>. Finish up the DH key exchange and then
+ * extend the crypt path of <b>circ</b> so that the hidden service is on the
+ * other side. */
+int
+hs_circuit_setup_e2e_rend_circ_legacy_client(origin_circuit_t *circ,
+                                             const uint8_t *rend_cell_body)
+{
+
+  if (BUG(!circuit_purpose_is_correct_for_rend(
+                                      TO_CIRCUIT(circ)->purpose, 0))) {
+    return -1;
+  }
+
+  crypt_path_t *hop = create_rend_cpath_legacy(circ, rend_cell_body);
+  if (!hop) {
+    log_warn(LD_GENERAL, "Couldn't get v2 cpath.");
+    return -1;
+  }
+
+  finalize_rend_circuit(circ, hop, 0);
+
+  return 0;
+}
+
+/* Given the introduction circuit intro_circ, the rendezvous circuit
+ * rend_circ, a descriptor intro point object ip and the service's
+ * subcredential, send an INTRODUCE1 cell on intro_circ.
+ *
+ * This will also setup the circuit identifier on rend_circ containing the key
+ * material for the handshake and e2e encryption. Return 0 on success else
+ * negative value. Because relay_send_command_from_edge() closes the circuit
+ * on error, it is possible that intro_circ is closed on error. */
+int
+hs_circ_send_introduce1(origin_circuit_t *intro_circ,
+                        origin_circuit_t *rend_circ,
+                        const hs_desc_intro_point_t *ip,
+                        const uint8_t *subcredential)
+{
+  int ret = -1;
+  ssize_t payload_len;
+  uint8_t payload[RELAY_PAYLOAD_SIZE] = {0};
+  hs_cell_introduce1_data_t intro1_data;
+
+  tor_assert(intro_circ);
+  tor_assert(rend_circ);
+  tor_assert(ip);
+  tor_assert(subcredential);
+
+  /* It is undefined behavior in hs_cell_introduce1_data_clear() if intro1_data
+   * has been declared on the stack but not initialized. Here, we set it to 0.
+   */
+  memset(&intro1_data, 0, sizeof(hs_cell_introduce1_data_t));
+
+  /* This takes various objects in order to populate the introduce1 data
+   * object which is used to build the content of the cell. */
+  const node_t *exit_node = build_state_get_exit_node(rend_circ->build_state);
+  if (exit_node == NULL) {
+    log_info(LD_REND, "Unable to get rendezvous point for circuit %u. "
+             "Failing.", TO_CIRCUIT(intro_circ)->n_circ_id);
+    goto done;
+  }
+
+  /* We should never select an invalid rendezvous point in theory but if we
+   * do, this function will fail to populate the introduce data. */
+  if (setup_introduce1_data(ip, exit_node, subcredential, &intro1_data) < 0) {
+    log_warn(LD_REND, "Unable to setup INTRODUCE1 data. The chosen rendezvous "
+                      "point is unusable. Closing circuit.");
+    goto close;
+  }
+
+  /* Final step before we encode a cell, we setup the circuit identifier which
+   * will generate both the rendezvous cookie and client keypair for this
+   * connection. Those are put in the ident. */
+  intro1_data.rendezvous_cookie = rend_circ->hs_ident->rendezvous_cookie;
+  intro1_data.client_kp = &rend_circ->hs_ident->rendezvous_client_kp;
+
+  memcpy(intro_circ->hs_ident->rendezvous_cookie,
+         rend_circ->hs_ident->rendezvous_cookie,
+         sizeof(intro_circ->hs_ident->rendezvous_cookie));
+
+  /* From the introduce1 data object, this will encode the INTRODUCE1 cell
+   * into payload which is then ready to be sent as is. */
+  payload_len = hs_cell_build_introduce1(&intro1_data, payload);
+  if (BUG(payload_len < 0)) {
+    goto close;
+  }
+
+  if (relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(intro_circ),
+                                   RELAY_COMMAND_INTRODUCE1,
+                                   (const char *) payload, payload_len,
+                                   intro_circ->cpath->prev) < 0) {
+    /* On error, circuit is closed. */
+    log_warn(LD_REND, "Unable to send INTRODUCE1 cell on circuit %u.",
+             TO_CIRCUIT(intro_circ)->n_circ_id);
+    goto done;
+  }
+
+  /* Success. */
+  ret = 0;
+  goto done;
+
+ close:
+  circuit_mark_for_close(TO_CIRCUIT(rend_circ), END_CIRC_REASON_INTERNAL);
+ done:
+  hs_cell_introduce1_data_clear(&intro1_data);
+  memwipe(payload, 0, sizeof(payload));
+  return ret;
+}
+
+/* Send an ESTABLISH_RENDEZVOUS cell along the rendezvous circuit circ. On
+ * success, 0 is returned else -1 and the circuit is marked for close. */
+int
+hs_circ_send_establish_rendezvous(origin_circuit_t *circ)
+{
+  ssize_t cell_len = 0;
+  uint8_t cell[RELAY_PAYLOAD_SIZE] = {0};
+
+  tor_assert(circ);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND);
+
+  log_info(LD_REND, "Send an ESTABLISH_RENDEZVOUS cell on circuit %u",
+           TO_CIRCUIT(circ)->n_circ_id);
+
+  /* Set timestamp_dirty, because circuit_expire_building expects it,
+   * and the rend cookie also means we've used the circ. */
+  TO_CIRCUIT(circ)->timestamp_dirty = time(NULL);
+
+  /* We've attempted to use this circuit. Probe it if we fail */
+  pathbias_count_use_attempt(circ);
+
+  /* Generate the RENDEZVOUS_COOKIE and place it in the identifier so we can
+   * complete the handshake when receiving the acknowledgement. */
+  crypto_rand((char *) circ->hs_ident->rendezvous_cookie, HS_REND_COOKIE_LEN);
+  /* Generate the client keypair. No need to be extra strong, not long term */
+  curve25519_keypair_generate(&circ->hs_ident->rendezvous_client_kp, 0);
+
+  cell_len =
+    hs_cell_build_establish_rendezvous(circ->hs_ident->rendezvous_cookie,
+                                       cell);
+  if (BUG(cell_len < 0)) {
+    goto err;
+  }
+
+  if (relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(circ),
+                                   RELAY_COMMAND_ESTABLISH_RENDEZVOUS,
+                                   (const char *) cell, cell_len,
+                                   circ->cpath->prev) < 0) {
+    /* Circuit has been marked for close */
+    log_warn(LD_REND, "Unable to send ESTABLISH_RENDEZVOUS cell on "
+                      "circuit %u", TO_CIRCUIT(circ)->n_circ_id);
+    memwipe(cell, 0, cell_len);
+    goto err;
+  }
+
+  memwipe(cell, 0, cell_len);
+  return 0;
+ err:
+  return -1;
+}
+
+/* We are about to close or free this <b>circ</b>. Clean it up from any
+ * related HS data structures. This function can be called multiple times
+ * safely for the same circuit. */
+void
+hs_circ_cleanup(circuit_t *circ)
+{
+  tor_assert(circ);
+
+  /* If it's a service-side intro circ, notify the HS subsystem for the intro
+   * point circuit closing so it can be dealt with cleanly. */
+  if (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
+      circ->purpose == CIRCUIT_PURPOSE_S_INTRO) {
+    hs_service_intro_circ_has_closed(TO_ORIGIN_CIRCUIT(circ));
+  }
+
+  /* Clear HS circuitmap token for this circ (if any). Very important to be
+   * done after the HS subsystem has been notified of the close else the
+   * circuit will not be found.
+   *
+   * We do this at the close if possible because from that point on, the
+   * circuit is good as dead. We can't rely on removing it in the circuit
+   * free() function because we open a race window between the close and free
+   * where we can't register a new circuit for the same intro point. */
+  if (circ->hs_token) {
+    hs_circuitmap_remove_circuit(circ);
+  }
+}
diff --git a/src/feature/hs/hs_circuit.h b/src/feature/hs/hs_circuit.h
new file mode 100644
index 0000000000..e168b301f1
--- /dev/null
+++ b/src/feature/hs/hs_circuit.h
@@ -0,0 +1,76 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_circuit.h
+ * \brief Header file containing circuit data for the whole HS subsytem.
+ **/
+
+#ifndef TOR_HS_CIRCUIT_H
+#define TOR_HS_CIRCUIT_H
+
+#include "core/or/or.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+#include "feature/hs/hs_service.h"
+
+/* Cleanup function when the circuit is closed or/and freed. */
+void hs_circ_cleanup(circuit_t *circ);
+
+/* Circuit API. */
+int hs_circ_service_intro_has_opened(hs_service_t *service,
+                                     hs_service_intro_point_t *ip,
+                                     const hs_service_descriptor_t *desc,
+                                     origin_circuit_t *circ);
+void hs_circ_service_rp_has_opened(const hs_service_t *service,
+                                   origin_circuit_t *circ);
+int hs_circ_launch_intro_point(hs_service_t *service,
+                               const hs_service_intro_point_t *ip,
+                               extend_info_t *ei,
+                               bool direct_conn);
+int hs_circ_launch_rendezvous_point(const hs_service_t *service,
+                                    const curve25519_public_key_t *onion_key,
+                                    const uint8_t *rendezvous_cookie);
+void hs_circ_retry_service_rendezvous_point(origin_circuit_t *circ);
+
+origin_circuit_t *hs_circ_service_get_intro_circ(
+                                      const hs_service_intro_point_t *ip);
+
+/* Cell API. */
+int hs_circ_handle_intro_established(const hs_service_t *service,
+                                     const hs_service_intro_point_t *ip,
+                                     origin_circuit_t *circ,
+                                     const uint8_t *payload,
+                                     size_t payload_len);
+int hs_circ_handle_introduce2(const hs_service_t *service,
+                              const origin_circuit_t *circ,
+                              hs_service_intro_point_t *ip,
+                              const uint8_t *subcredential,
+                              const uint8_t *payload, size_t payload_len);
+int hs_circ_send_introduce1(origin_circuit_t *intro_circ,
+                            origin_circuit_t *rend_circ,
+                            const hs_desc_intro_point_t *ip,
+                            const uint8_t *subcredential);
+int hs_circ_send_establish_rendezvous(origin_circuit_t *circ);
+
+/* e2e circuit API. */
+
+int hs_circuit_setup_e2e_rend_circ(origin_circuit_t *circ,
+                                   const uint8_t *ntor_key_seed,
+                                   size_t seed_len,
+                                   int is_service_side);
+int hs_circuit_setup_e2e_rend_circ_legacy_client(origin_circuit_t *circ,
+                                          const uint8_t *rend_cell_body);
+
+#ifdef HS_CIRCUIT_PRIVATE
+
+STATIC hs_ident_circuit_t *
+create_rp_circuit_identifier(const hs_service_t *service,
+                             const uint8_t *rendezvous_cookie,
+                             const curve25519_public_key_t *server_pk,
+                             const hs_ntor_rend_cell_keys_t *keys);
+
+#endif /* defined(HS_CIRCUIT_PRIVATE) */
+
+#endif /* !defined(TOR_HS_CIRCUIT_H) */
+
diff --git a/src/feature/hs/hs_circuitmap.c b/src/feature/hs/hs_circuitmap.c
new file mode 100644
index 0000000000..5480d5eb84
--- /dev/null
+++ b/src/feature/hs/hs_circuitmap.c
@@ -0,0 +1,585 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_circuitmap.c
+ *
+ * \brief Hidden service circuitmap: A hash table that maps binary tokens to
+ *  introduction and rendezvous circuits; it's used:
+ *  (a) by relays acting as intro points and rendezvous points
+ *  (b) by hidden services to find intro and rend circuits and
+ *  (c) by HS clients to find rendezvous circuits.
+ **/
+
+#define HS_CIRCUITMAP_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/or/circuitlist.h"
+#include "feature/hs/hs_circuitmap.h"
+
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+
+/************************** HS circuitmap code *******************************/
+
+/* This is the hidden service circuitmap. It's a hash table that maps
+   introduction and rendezvous tokens to specific circuits such that given a
+   token it's easy to find the corresponding circuit. */
+static struct hs_circuitmap_ht *the_hs_circuitmap = NULL;
+
+/* This is a helper function used by the hash table code (HT_). It returns 1 if
+ * two circuits have the same HS token. */
+static int
+hs_circuits_have_same_token(const circuit_t *first_circuit,
+                            const circuit_t *second_circuit)
+{
+  const hs_token_t *first_token;
+  const hs_token_t *second_token;
+
+  tor_assert(first_circuit);
+  tor_assert(second_circuit);
+
+  first_token = first_circuit->hs_token;
+  second_token = second_circuit->hs_token;
+
+  /* Both circs must have a token */
+  if (BUG(!first_token) || BUG(!second_token)) {
+    return 0;
+  }
+
+  if (first_token->type != second_token->type) {
+    return 0;
+  }
+
+  if (first_token->token_len != second_token->token_len)
+    return 0;
+
+  return tor_memeq(first_token->token,
+                   second_token->token,
+                   first_token->token_len);
+}
+
+/* This is a helper function for the hash table code (HT_). It hashes a circuit
+ * HS token into an unsigned int for use as a key by the hash table routines.*/
+static inline unsigned int
+hs_circuit_hash_token(const circuit_t *circuit)
+{
+  tor_assert(circuit->hs_token);
+
+  return (unsigned) siphash24g(circuit->hs_token->token,
+                               circuit->hs_token->token_len);
+}
+
+/* Register the circuitmap hash table */
+HT_PROTOTYPE(hs_circuitmap_ht, // The name of the hashtable struct
+             circuit_t,    // The name of the element struct,
+             hs_circuitmap_node,        // The name of HT_ENTRY member
+             hs_circuit_hash_token, hs_circuits_have_same_token)
+
+HT_GENERATE2(hs_circuitmap_ht, circuit_t, hs_circuitmap_node,
+             hs_circuit_hash_token, hs_circuits_have_same_token,
+             0.6, tor_reallocarray, tor_free_)
+
+#ifdef TOR_UNIT_TESTS
+
+/* Return the global HS circuitmap. Used by unittests. */
+hs_circuitmap_ht *
+get_hs_circuitmap(void)
+{
+  return the_hs_circuitmap;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/****************** HS circuitmap utility functions **************************/
+
+/** Return a new HS token of type <b>type</b> containing <b>token</b>. */
+static hs_token_t *
+hs_token_new(hs_token_type_t type, size_t token_len,
+             const uint8_t *token)
+{
+  tor_assert(token);
+
+  hs_token_t *hs_token = tor_malloc_zero(sizeof(hs_token_t));
+  hs_token->type = type;
+  hs_token->token_len = token_len;
+  hs_token->token = tor_memdup(token, token_len);
+
+  return hs_token;
+}
+
+#define hs_token_free(val) \
+  FREE_AND_NULL(hs_token_t, hs_token_free_, (val))
+
+/** Free memory allocated by this <b>hs_token</b>. */
+static void
+hs_token_free_(hs_token_t *hs_token)
+{
+  if (!hs_token) {
+    return;
+  }
+
+  tor_free(hs_token->token);
+  tor_free(hs_token);
+}
+
+/** Return the circuit from the circuitmap with token <b>search_token</b>. */
+static circuit_t *
+get_circuit_with_token(hs_token_t *search_token)
+{
+  tor_assert(the_hs_circuitmap);
+
+  /* We use a dummy circuit object for the hash table search routine. */
+  circuit_t search_circ;
+  search_circ.hs_token = search_token;
+  return HT_FIND(hs_circuitmap_ht, the_hs_circuitmap, &search_circ);
+}
+
+/* Helper function that registers <b>circ</b> with <b>token</b> on the HS
+   circuitmap. This function steals reference of <b>token</b>. */
+static void
+hs_circuitmap_register_impl(circuit_t *circ, hs_token_t *token)
+{
+  tor_assert(circ);
+  tor_assert(token);
+  tor_assert(the_hs_circuitmap);
+
+  /* If this circuit already has a token, clear it. */
+  if (circ->hs_token) {
+    hs_circuitmap_remove_circuit(circ);
+  }
+
+  /* Kill old circuits with the same token. We want new intro/rend circuits to
+     take precedence over old ones, so that HSes and clients and reestablish
+     killed circuits without changing the HS token. */
+  {
+    circuit_t *found_circ;
+    found_circ = get_circuit_with_token(token);
+    if (found_circ) {
+      hs_circuitmap_remove_circuit(found_circ);
+      if (!found_circ->marked_for_close) {
+        circuit_mark_for_close(found_circ, END_CIRC_REASON_FINISHED);
+      }
+    }
+  }
+
+  /* Register circuit and token to circuitmap. */
+  circ->hs_token = token;
+  HT_INSERT(hs_circuitmap_ht, the_hs_circuitmap, circ);
+}
+
+/** Helper function: Register <b>circ</b> of <b>type</b> on the HS
+ *  circuitmap. Use the HS <b>token</b> as the key to the hash table.  If
+ *  <b>token</b> is not set, clear the circuit of any HS tokens. */
+static void
+hs_circuitmap_register_circuit(circuit_t *circ,
+                               hs_token_type_t type, size_t token_len,
+                               const uint8_t *token)
+{
+  hs_token_t *hs_token = NULL;
+
+  /* Create a new token and register it to the circuitmap */
+  tor_assert(token);
+  hs_token = hs_token_new(type, token_len, token);
+  tor_assert(hs_token);
+  hs_circuitmap_register_impl(circ, hs_token);
+}
+
+/* Helper function for hs_circuitmap_get_origin_circuit() and
+ * hs_circuitmap_get_or_circuit(). Because only circuit_t are indexed in the
+ * circuitmap, this function returns object type so the specialized functions
+ * using this helper can upcast it to the right type.
+ *
+ * Return NULL if not such circuit is found. */
+static circuit_t *
+hs_circuitmap_get_circuit_impl(hs_token_type_t type,
+                               size_t token_len,
+                               const uint8_t *token,
+                               uint8_t wanted_circ_purpose)
+{
+  circuit_t *found_circ = NULL;
+
+  tor_assert(the_hs_circuitmap);
+
+  /* Check the circuitmap if we have a circuit with this token */
+  {
+    hs_token_t *search_hs_token = hs_token_new(type, token_len, token);
+    tor_assert(search_hs_token);
+    found_circ = get_circuit_with_token(search_hs_token);
+    hs_token_free(search_hs_token);
+  }
+
+  /* Check that the circuit is useful to us */
+  if (!found_circ ||
+      found_circ->purpose != wanted_circ_purpose ||
+      found_circ->marked_for_close) {
+    return NULL;
+  }
+
+  return found_circ;
+}
+
+/* Helper function: Query circuitmap for origin circuit with <b>token</b> of
+ * size <b>token_len</b> and <b>type</b>.  Only returns a circuit with purpose
+ * equal to the <b>wanted_circ_purpose</b> parameter and if it is NOT marked
+ * for close. Return NULL if no such circuit is found. */
+static origin_circuit_t *
+hs_circuitmap_get_origin_circuit(hs_token_type_t type,
+                                 size_t token_len,
+                                 const uint8_t *token,
+                                 uint8_t wanted_circ_purpose)
+{
+  circuit_t *circ;
+  tor_assert(token);
+  tor_assert(CIRCUIT_PURPOSE_IS_ORIGIN(wanted_circ_purpose));
+
+  circ = hs_circuitmap_get_circuit_impl(type, token_len, token,
+                                        wanted_circ_purpose);
+  if (!circ) {
+    return NULL;
+  }
+
+  tor_assert(CIRCUIT_IS_ORIGIN(circ));
+  return TO_ORIGIN_CIRCUIT(circ);
+}
+
+/* Helper function: Query circuitmap for OR circuit with <b>token</b> of size
+ * <b>token_len</b> and <b>type</b>.  Only returns a circuit with purpose equal
+ * to the <b>wanted_circ_purpose</b> parameter and if it is NOT marked for
+ * close. Return NULL if no such circuit is found. */
+static or_circuit_t *
+hs_circuitmap_get_or_circuit(hs_token_type_t type,
+                             size_t token_len,
+                             const uint8_t *token,
+                             uint8_t wanted_circ_purpose)
+{
+  circuit_t *circ;
+  tor_assert(token);
+  tor_assert(!CIRCUIT_PURPOSE_IS_ORIGIN(wanted_circ_purpose));
+
+  circ = hs_circuitmap_get_circuit_impl(type, token_len, token,
+                                        wanted_circ_purpose);
+  if (!circ) {
+    return NULL;
+  }
+
+  tor_assert(CIRCUIT_IS_ORCIRC(circ));
+  return TO_OR_CIRCUIT(circ);
+}
+
+/************** Public circuitmap API ****************************************/
+
+/**** Public relay-side getters: */
+
+/* Public function: Return a v3 introduction circuit to this relay with
+ * <b>auth_key</b>. Return NULL if no such circuit is found in the
+ * circuitmap. */
+or_circuit_t *
+hs_circuitmap_get_intro_circ_v3_relay_side(
+                                          const ed25519_public_key_t *auth_key)
+{
+  return hs_circuitmap_get_or_circuit(HS_TOKEN_INTRO_V3_RELAY_SIDE,
+                                      ED25519_PUBKEY_LEN, auth_key->pubkey,
+                                      CIRCUIT_PURPOSE_INTRO_POINT);
+}
+
+/* Public function: Return v2 introduction circuit to this relay with
+ * <b>digest</b>. Return NULL if no such circuit is found in the circuitmap. */
+or_circuit_t *
+hs_circuitmap_get_intro_circ_v2_relay_side(const uint8_t *digest)
+{
+  return hs_circuitmap_get_or_circuit(HS_TOKEN_INTRO_V2_RELAY_SIDE,
+                                      REND_TOKEN_LEN, digest,
+                                      CIRCUIT_PURPOSE_INTRO_POINT);
+}
+
+/* Public function: Return rendezvous circuit to this relay with rendezvous
+ * <b>cookie</b>. Return NULL if no such circuit is found in the circuitmap. */
+or_circuit_t *
+hs_circuitmap_get_rend_circ_relay_side(const uint8_t *cookie)
+{
+  return hs_circuitmap_get_or_circuit(HS_TOKEN_REND_RELAY_SIDE,
+                                      REND_TOKEN_LEN, cookie,
+                                      CIRCUIT_PURPOSE_REND_POINT_WAITING);
+}
+
+/** Public relay-side setters: */
+
+/* Public function: Register rendezvous circuit with key <b>cookie</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_rend_circ_relay_side(or_circuit_t *circ,
+                                            const uint8_t *cookie)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_REND_RELAY_SIDE,
+                                 REND_TOKEN_LEN, cookie);
+}
+/* Public function: Register v2 intro circuit with key <b>digest</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_intro_circ_v2_relay_side(or_circuit_t *circ,
+                                                const uint8_t *digest)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_INTRO_V2_RELAY_SIDE,
+                                 REND_TOKEN_LEN, digest);
+}
+
+/* Public function: Register v3 intro circuit with key <b>auth_key</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_intro_circ_v3_relay_side(or_circuit_t *circ,
+                                          const ed25519_public_key_t *auth_key)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_INTRO_V3_RELAY_SIDE,
+                                 ED25519_PUBKEY_LEN, auth_key->pubkey);
+}
+
+/**** Public servide-side getters: */
+
+/* Public function: Return v3 introduction circuit with <b>auth_key</b>
+ * originating from this hidden service. Return NULL if no such circuit is
+ * found in the circuitmap. */
+origin_circuit_t *
+hs_circuitmap_get_intro_circ_v3_service_side(const
+                                             ed25519_public_key_t *auth_key)
+{
+  origin_circuit_t *circ = NULL;
+
+  /* Check first for established intro circuits */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_INTRO_V3_SERVICE_SIDE,
+                                          ED25519_PUBKEY_LEN, auth_key->pubkey,
+                                          CIRCUIT_PURPOSE_S_INTRO);
+  if (circ) {
+    return circ;
+  }
+
+  /* ...if nothing found, check for pending intro circs */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_INTRO_V3_SERVICE_SIDE,
+                                          ED25519_PUBKEY_LEN, auth_key->pubkey,
+                                          CIRCUIT_PURPOSE_S_ESTABLISH_INTRO);
+
+  return circ;
+}
+
+/* Public function: Return v2 introduction circuit originating from this hidden
+ * service with <b>digest</b>. Return NULL if no such circuit is found in the
+ * circuitmap. */
+origin_circuit_t *
+hs_circuitmap_get_intro_circ_v2_service_side(const uint8_t *digest)
+{
+  origin_circuit_t *circ = NULL;
+
+  /* Check first for established intro circuits */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_INTRO_V2_SERVICE_SIDE,
+                                          REND_TOKEN_LEN, digest,
+                                          CIRCUIT_PURPOSE_S_INTRO);
+  if (circ) {
+    return circ;
+  }
+
+  /* ...if nothing found, check for pending intro circs */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_INTRO_V2_SERVICE_SIDE,
+                                          REND_TOKEN_LEN, digest,
+                                          CIRCUIT_PURPOSE_S_ESTABLISH_INTRO);
+
+  return circ;
+}
+
+/* Public function: Return rendezvous circuit originating from this hidden
+ * service with rendezvous <b>cookie</b>. Return NULL if no such circuit is
+ * found in the circuitmap. */
+origin_circuit_t *
+hs_circuitmap_get_rend_circ_service_side(const uint8_t *cookie)
+{
+  origin_circuit_t *circ = NULL;
+
+  /* Try to check if we have a connecting circuit. */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_SERVICE_SIDE,
+                                          REND_TOKEN_LEN, cookie,
+                                          CIRCUIT_PURPOSE_S_CONNECT_REND);
+  if (circ) {
+    return circ;
+  }
+
+  /* Then try for connected circuit. */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_SERVICE_SIDE,
+                                          REND_TOKEN_LEN, cookie,
+                                          CIRCUIT_PURPOSE_S_REND_JOINED);
+  return circ;
+}
+
+/* Public function: Return client-side rendezvous circuit with rendezvous
+ * <b>cookie</b>. It will look for circuits with the following purposes:
+
+ * a) CIRCUIT_PURPOSE_C_REND_READY: Established rend circuit (received
+ *    RENDEZVOUS_ESTABLISHED). Waiting for RENDEZVOUS2 from service, and for
+ *    INTRODUCE_ACK from intro point.
+ *
+ * b) CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED: Established rend circuit and
+ *    introduce circuit acked. Waiting for RENDEZVOUS2 from service.
+ *
+ * c) CIRCUIT_PURPOSE_C_REND_JOINED: Established rend circuit and received
+ *    RENDEZVOUS2 from service.
+ *
+ * d) CIRCUIT_PURPOSE_C_ESTABLISH_REND: Rend circuit open but not yet
+ *    established.
+ *
+ * Return NULL if no such circuit is found in the circuitmap. */
+origin_circuit_t *
+hs_circuitmap_get_rend_circ_client_side(const uint8_t *cookie)
+{
+  origin_circuit_t *circ = NULL;
+
+  circ = hs_circuitmap_get_established_rend_circ_client_side(cookie);
+  if (circ) {
+    return circ;
+  }
+
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_CLIENT_SIDE,
+                                          REND_TOKEN_LEN, cookie,
+                                          CIRCUIT_PURPOSE_C_ESTABLISH_REND);
+  return circ;
+}
+
+/*  Public function: Return client-side established rendezvous circuit with
+ *  rendezvous <b>cookie</b>. It will look for circuits with the following
+ *  purposes:
+ *
+ * a) CIRCUIT_PURPOSE_C_REND_READY: Established rend circuit (received
+ *    RENDEZVOUS_ESTABLISHED). Waiting for RENDEZVOUS2 from service, and for
+ *    INTRODUCE_ACK from intro point.
+ *
+ * b) CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED: Established rend circuit and
+ *    introduce circuit acked. Waiting for RENDEZVOUS2 from service.
+ *
+ * c) CIRCUIT_PURPOSE_C_REND_JOINED: Established rend circuit and received
+ *    RENDEZVOUS2 from service.
+ *
+ * Return NULL if no such circuit is found in the circuitmap. */
+origin_circuit_t *
+hs_circuitmap_get_established_rend_circ_client_side(const uint8_t *cookie)
+{
+  origin_circuit_t *circ = NULL;
+
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_CLIENT_SIDE,
+                                          REND_TOKEN_LEN, cookie,
+                                          CIRCUIT_PURPOSE_C_REND_READY);
+  if (circ) {
+    return circ;
+  }
+
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_CLIENT_SIDE,
+                                          REND_TOKEN_LEN, cookie,
+                                 CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED);
+  if (circ) {
+    return circ;
+  }
+
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_CLIENT_SIDE,
+                                          REND_TOKEN_LEN, cookie,
+                                          CIRCUIT_PURPOSE_C_REND_JOINED);
+  return circ;
+}
+
+/**** Public servide-side setters: */
+
+/* Public function: Register v2 intro circuit with key <b>digest</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_intro_circ_v2_service_side(origin_circuit_t *circ,
+                                                  const uint8_t *digest)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_INTRO_V2_SERVICE_SIDE,
+                                 REND_TOKEN_LEN, digest);
+}
+
+/* Public function: Register v3 intro circuit with key <b>auth_key</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_intro_circ_v3_service_side(origin_circuit_t *circ,
+                                          const ed25519_public_key_t *auth_key)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_INTRO_V3_SERVICE_SIDE,
+                                 ED25519_PUBKEY_LEN, auth_key->pubkey);
+}
+
+/* Public function: Register rendezvous circuit with key <b>cookie</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_rend_circ_service_side(origin_circuit_t *circ,
+                                              const uint8_t *cookie)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_REND_SERVICE_SIDE,
+                                 REND_TOKEN_LEN, cookie);
+}
+
+/* Public function: Register rendezvous circuit with key <b>cookie</b> to the
+ * client-side circuitmap. */
+void
+hs_circuitmap_register_rend_circ_client_side(origin_circuit_t *or_circ,
+                                             const uint8_t *cookie)
+{
+  circuit_t *circ = TO_CIRCUIT(or_circ);
+  { /* Basic circ purpose sanity checking */
+    tor_assert_nonfatal(circ->purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND);
+  }
+
+  hs_circuitmap_register_circuit(circ, HS_TOKEN_REND_CLIENT_SIDE,
+                                 REND_TOKEN_LEN, cookie);
+}
+
+/**** Misc public functions: */
+
+/** Public function: Remove this circuit from the HS circuitmap. Clear its HS
+ *  token, and remove it from the hashtable. */
+void
+hs_circuitmap_remove_circuit(circuit_t *circ)
+{
+  tor_assert(the_hs_circuitmap);
+
+  if (!circ || !circ->hs_token) {
+    return;
+  }
+
+  /* Remove circ from circuitmap */
+  circuit_t *tmp;
+  tmp = HT_REMOVE(hs_circuitmap_ht, the_hs_circuitmap, circ);
+  /* ... and ensure the removal was successful. */
+  if (tmp) {
+    tor_assert(tmp == circ);
+  } else {
+    log_warn(LD_BUG, "Could not find circuit (%u) in circuitmap.",
+             circ->n_circ_id);
+  }
+
+  /* Clear token from circ */
+  hs_token_free(circ->hs_token);
+  circ->hs_token = NULL;
+}
+
+/* Public function: Initialize the global HS circuitmap. */
+void
+hs_circuitmap_init(void)
+{
+  tor_assert(!the_hs_circuitmap);
+
+  the_hs_circuitmap = tor_malloc_zero(sizeof(struct hs_circuitmap_ht));
+  HT_INIT(hs_circuitmap_ht, the_hs_circuitmap);
+}
+
+/* Public function: Free all memory allocated by the global HS circuitmap. */
+void
+hs_circuitmap_free_all(void)
+{
+  if (the_hs_circuitmap) {
+    HT_CLEAR(hs_circuitmap_ht, the_hs_circuitmap);
+    tor_free(the_hs_circuitmap);
+  }
+}
diff --git a/src/feature/hs/hs_circuitmap.h b/src/feature/hs/hs_circuitmap.h
new file mode 100644
index 0000000000..c1bbb1ff1c
--- /dev/null
+++ b/src/feature/hs/hs_circuitmap.h
@@ -0,0 +1,112 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_circuitmap.h
+ * \brief Header file for hs_circuitmap.c.
+ **/
+
+#ifndef TOR_HS_CIRCUITMAP_H
+#define TOR_HS_CIRCUITMAP_H
+
+typedef HT_HEAD(hs_circuitmap_ht, circuit_t) hs_circuitmap_ht;
+
+typedef struct hs_token_t hs_token_t;
+struct or_circuit_t;
+struct origin_circuit_t;
+
+/** Public HS circuitmap API: */
+
+/** Public relay-side API: */
+
+struct or_circuit_t *
+hs_circuitmap_get_intro_circ_v3_relay_side(const
+                                           ed25519_public_key_t *auth_key);
+struct or_circuit_t *
+hs_circuitmap_get_intro_circ_v2_relay_side(const uint8_t *digest);
+struct or_circuit_t *
+hs_circuitmap_get_rend_circ_relay_side(const uint8_t *cookie);
+
+void hs_circuitmap_register_rend_circ_relay_side(struct or_circuit_t *circ,
+                                                 const uint8_t *cookie);
+void hs_circuitmap_register_intro_circ_v2_relay_side(struct or_circuit_t *circ,
+                                                     const uint8_t *digest);
+void hs_circuitmap_register_intro_circ_v3_relay_side(struct or_circuit_t *circ,
+                                         const ed25519_public_key_t *auth_key);
+
+/** Public service-side API: */
+
+struct origin_circuit_t *
+hs_circuitmap_get_intro_circ_v3_service_side(const
+                                             ed25519_public_key_t *auth_key);
+struct origin_circuit_t *
+hs_circuitmap_get_intro_circ_v2_service_side(const uint8_t *digest);
+struct origin_circuit_t *
+hs_circuitmap_get_rend_circ_service_side(const uint8_t *cookie);
+struct origin_circuit_t *
+hs_circuitmap_get_rend_circ_client_side(const uint8_t *cookie);
+struct origin_circuit_t *
+hs_circuitmap_get_established_rend_circ_client_side(const uint8_t *cookie);
+
+void hs_circuitmap_register_intro_circ_v2_service_side(
+                                        struct origin_circuit_t *circ,
+                                        const uint8_t *digest);
+void hs_circuitmap_register_intro_circ_v3_service_side(
+                                        struct origin_circuit_t *circ,
+                                        const ed25519_public_key_t *auth_key);
+void hs_circuitmap_register_rend_circ_service_side(
+                                        struct origin_circuit_t *circ,
+                                        const uint8_t *cookie);
+void hs_circuitmap_register_rend_circ_client_side(
+                                      struct origin_circuit_t *circ,
+                                      const uint8_t *cookie);
+
+void hs_circuitmap_remove_circuit(struct circuit_t *circ);
+
+void hs_circuitmap_init(void);
+void hs_circuitmap_free_all(void);
+
+#ifdef HS_CIRCUITMAP_PRIVATE
+
+/** Represents the type of HS token. */
+typedef enum {
+  /** A rendezvous cookie on a relay (128bit)*/
+  HS_TOKEN_REND_RELAY_SIDE,
+  /** A v2 introduction point pubkey on a relay (160bit) */
+  HS_TOKEN_INTRO_V2_RELAY_SIDE,
+  /** A v3 introduction point pubkey on a relay (256bit) */
+  HS_TOKEN_INTRO_V3_RELAY_SIDE,
+
+  /** A rendezvous cookie on a hidden service (128bit)*/
+  HS_TOKEN_REND_SERVICE_SIDE,
+  /** A v2 introduction point pubkey on a hidden service (160bit) */
+  HS_TOKEN_INTRO_V2_SERVICE_SIDE,
+  /** A v3 introduction point pubkey on a hidden service (256bit) */
+  HS_TOKEN_INTRO_V3_SERVICE_SIDE,
+
+  /** A rendezvous cookie on the client side (128bit) */
+  HS_TOKEN_REND_CLIENT_SIDE,
+} hs_token_type_t;
+
+/** Represents a token used in the HS protocol. Each such token maps to a
+ *  specific introduction or rendezvous circuit. */
+struct hs_token_t {
+  /* Type of HS token. */
+  hs_token_type_t type;
+
+  /* The size of the token (depends on the type). */
+  size_t token_len;
+
+  /* The token itself. Memory allocated at runtime. */
+  uint8_t *token;
+};
+
+#endif /* defined(HS_CIRCUITMAP_PRIVATE) */
+
+#ifdef TOR_UNIT_TESTS
+
+hs_circuitmap_ht *get_hs_circuitmap(void);
+
+#endif /* TOR_UNIT_TESTS */
+
+#endif /* !defined(TOR_HS_CIRCUITMAP_H) */
diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c
new file mode 100644
index 0000000000..fd2d266453
--- /dev/null
+++ b/src/feature/hs/hs_client.c
@@ -0,0 +1,1949 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_client.c
+ * \brief Implement next generation hidden service client functionality
+ **/
+
+#define HS_CLIENT_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/crypto/hs_ntor.h"
+#include "core/mainloop/connection.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/connection_edge.h"
+#include "core/or/reasons.h"
+#include "feature/client/circpathbias.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dircommon/directory.h"
+#include "feature/hs/hs_cache.h"
+#include "feature/hs/hs_cell.h"
+#include "feature/hs/hs_circuit.h"
+#include "feature/hs/hs_circuitmap.h"
+#include "feature/hs/hs_client.h"
+#include "feature/hs/hs_control.h"
+#include "feature/hs/hs_descriptor.h"
+#include "feature/hs/hs_ident.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/rend/rendclient.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/extend_info_st.h"
+#include "core/or/origin_circuit_st.h"
+
+/* Client-side authorizations for hidden services; map of service identity
+ * public key to hs_client_service_authorization_t *. */
+static digest256map_t *client_auths = NULL;
+
+#include "trunnel/hs/cell_introduce1.h"
+
+/* Return a human-readable string for the client fetch status code. */
+static const char *
+fetch_status_to_string(hs_client_fetch_status_t status)
+{
+  switch (status) {
+  case HS_CLIENT_FETCH_ERROR:
+    return "Internal error";
+  case HS_CLIENT_FETCH_LAUNCHED:
+    return "Descriptor fetch launched";
+  case HS_CLIENT_FETCH_HAVE_DESC:
+    return "Already have descriptor";
+  case HS_CLIENT_FETCH_NO_HSDIRS:
+    return "No more HSDir available to query";
+  case HS_CLIENT_FETCH_NOT_ALLOWED:
+    return "Fetching descriptors is not allowed";
+  case HS_CLIENT_FETCH_MISSING_INFO:
+    return "Missing directory information";
+  case HS_CLIENT_FETCH_PENDING:
+    return "Pending descriptor fetch";
+  default:
+    return "(Unknown client fetch status code)";
+  }
+}
+
+/* Return true iff tor should close the SOCKS request(s) for the descriptor
+ * fetch that ended up with this given status code. */
+static int
+fetch_status_should_close_socks(hs_client_fetch_status_t status)
+{
+  switch (status) {
+  case HS_CLIENT_FETCH_NO_HSDIRS:
+    /* No more HSDir to query, we can't complete the SOCKS request(s). */
+  case HS_CLIENT_FETCH_ERROR:
+    /* The fetch triggered an internal error. */
+  case HS_CLIENT_FETCH_NOT_ALLOWED:
+    /* Client is not allowed to fetch (FetchHidServDescriptors 0). */
+    goto close;
+  case HS_CLIENT_FETCH_MISSING_INFO:
+  case HS_CLIENT_FETCH_HAVE_DESC:
+  case HS_CLIENT_FETCH_PENDING:
+  case HS_CLIENT_FETCH_LAUNCHED:
+    /* The rest doesn't require tor to close the SOCKS request(s). */
+    goto no_close;
+  }
+
+ no_close:
+  return 0;
+ close:
+  return 1;
+}
+
+/* Cancel all descriptor fetches currently in progress. */
+static void
+cancel_descriptor_fetches(void)
+{
+  smartlist_t *conns =
+    connection_list_by_type_state(CONN_TYPE_DIR, DIR_PURPOSE_FETCH_HSDESC);
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
+    const hs_ident_dir_conn_t *ident = TO_DIR_CONN(conn)->hs_ident;
+    if (BUG(ident == NULL)) {
+      /* A directory connection fetching a service descriptor can't have an
+       * empty hidden service identifier. */
+      continue;
+    }
+    log_debug(LD_REND, "Marking for close a directory connection fetching "
+                       "a hidden service descriptor for service %s.",
+              safe_str_client(ed25519_fmt(&ident->identity_pk)));
+    connection_mark_for_close(conn);
+  } SMARTLIST_FOREACH_END(conn);
+
+  /* No ownership of the objects in this list. */
+  smartlist_free(conns);
+  log_info(LD_REND, "Hidden service client descriptor fetches cancelled.");
+}
+
+/* Get all connections that are waiting on a circuit and flag them back to
+ * waiting for a hidden service descriptor for the given service key
+ * service_identity_pk. */
+static void
+flag_all_conn_wait_desc(const ed25519_public_key_t *service_identity_pk)
+{
+  tor_assert(service_identity_pk);
+
+  smartlist_t *conns =
+    connection_list_by_type_state(CONN_TYPE_AP, AP_CONN_STATE_CIRCUIT_WAIT);
+
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
+    edge_connection_t *edge_conn;
+    if (BUG(!CONN_IS_EDGE(conn))) {
+      continue;
+    }
+    edge_conn = TO_EDGE_CONN(conn);
+    if (edge_conn->hs_ident &&
+        ed25519_pubkey_eq(&edge_conn->hs_ident->identity_pk,
+                          service_identity_pk)) {
+      connection_ap_mark_as_waiting_for_renddesc(TO_ENTRY_CONN(conn));
+    }
+  } SMARTLIST_FOREACH_END(conn);
+
+  smartlist_free(conns);
+}
+
+/* Remove tracked HSDir requests from our history for this hidden service
+ * identity public key. */
+static void
+purge_hid_serv_request(const ed25519_public_key_t *identity_pk)
+{
+  char base64_blinded_pk[ED25519_BASE64_LEN + 1];
+  ed25519_public_key_t blinded_pk;
+
+  tor_assert(identity_pk);
+
+  /* Get blinded pubkey of hidden service. It is possible that we just moved
+   * to a new time period meaning that we won't be able to purge the request
+   * from the previous time period. That is fine because they will expire at
+   * some point and we don't care about those anymore. */
+  hs_build_blinded_pubkey(identity_pk, NULL, 0,
+                          hs_get_time_period_num(0), &blinded_pk);
+  if (BUG(ed25519_public_to_base64(base64_blinded_pk, &blinded_pk) < 0)) {
+    return;
+  }
+  /* Purge last hidden service request from cache for this blinded key. */
+  hs_purge_hid_serv_from_last_hid_serv_requests(base64_blinded_pk);
+}
+
+/* Return true iff there is at least one pending directory descriptor request
+ * for the service identity_pk. */
+static int
+directory_request_is_pending(const ed25519_public_key_t *identity_pk)
+{
+  int ret = 0;
+  smartlist_t *conns =
+    connection_list_by_type_purpose(CONN_TYPE_DIR, DIR_PURPOSE_FETCH_HSDESC);
+
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
+    const hs_ident_dir_conn_t *ident = TO_DIR_CONN(conn)->hs_ident;
+    if (BUG(ident == NULL)) {
+      /* A directory connection fetching a service descriptor can't have an
+       * empty hidden service identifier. */
+      continue;
+    }
+    if (!ed25519_pubkey_eq(identity_pk, &ident->identity_pk)) {
+      continue;
+    }
+    ret = 1;
+    break;
+  } SMARTLIST_FOREACH_END(conn);
+
+  /* No ownership of the objects in this list. */
+  smartlist_free(conns);
+  return ret;
+}
+
+/* Helper function that changes the state of an entry connection to waiting
+ * for a circuit. For this to work properly, the connection timestamps are set
+ * to now and the connection is then marked as pending for a circuit. */
+static void
+mark_conn_as_waiting_for_circuit(connection_t *conn, time_t now)
+{
+  tor_assert(conn);
+
+  /* Because the connection can now proceed to opening circuit and ultimately
+   * connect to the service, reset those timestamp so the connection is
+   * considered "fresh" and can continue without being closed too early. */
+  conn->timestamp_created = now;
+  conn->timestamp_last_read_allowed = now;
+  conn->timestamp_last_write_allowed = now;
+  /* Change connection's state into waiting for a circuit. */
+  conn->state = AP_CONN_STATE_CIRCUIT_WAIT;
+
+  connection_ap_mark_as_pending_circuit(TO_ENTRY_CONN(conn));
+}
+
+/* We failed to fetch a descriptor for the service with <b>identity_pk</b>
+ * because of <b>status</b>. Find all pending SOCKS connections for this
+ * service that are waiting on the descriptor and close them with
+ * <b>reason</b>. */
+static void
+close_all_socks_conns_waiting_for_desc(const ed25519_public_key_t *identity_pk,
+                                       hs_client_fetch_status_t status,
+                                       int reason)
+{
+  unsigned int count = 0;
+  time_t now = approx_time();
+  smartlist_t *conns =
+    connection_list_by_type_state(CONN_TYPE_AP, AP_CONN_STATE_RENDDESC_WAIT);
+
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+    entry_connection_t *entry_conn = TO_ENTRY_CONN(base_conn);
+    const edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(entry_conn);
+
+    /* Only consider the entry connections that matches the service for which
+     * we tried to get the descriptor */
+    if (!edge_conn->hs_ident ||
+        !ed25519_pubkey_eq(identity_pk,
+                           &edge_conn->hs_ident->identity_pk)) {
+      continue;
+    }
+    assert_connection_ok(base_conn, now);
+    /* Unattach the entry connection which will close for the reason. */
+    connection_mark_unattached_ap(entry_conn, reason);
+    count++;
+  } SMARTLIST_FOREACH_END(base_conn);
+
+  if (count > 0) {
+    char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+    hs_build_address(identity_pk, HS_VERSION_THREE, onion_address);
+    log_notice(LD_REND, "Closed %u streams for service %s.onion "
+                        "for reason %s. Fetch status: %s.",
+               count, safe_str_client(onion_address),
+               stream_end_reason_to_string(reason),
+               fetch_status_to_string(status));
+  }
+
+  /* No ownership of the object(s) in this list. */
+  smartlist_free(conns);
+}
+
+/* Find all pending SOCKS connection waiting for a descriptor and retry them
+ * all. This is called when the directory information changed. */
+STATIC void
+retry_all_socks_conn_waiting_for_desc(void)
+{
+  smartlist_t *conns =
+    connection_list_by_type_state(CONN_TYPE_AP, AP_CONN_STATE_RENDDESC_WAIT);
+
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+    hs_client_fetch_status_t status;
+    const edge_connection_t *edge_conn =
+      ENTRY_TO_EDGE_CONN(TO_ENTRY_CONN(base_conn));
+
+    /* Ignore non HS or non v3 connection. */
+    if (edge_conn->hs_ident == NULL) {
+      continue;
+    }
+    /* In this loop, we will possibly try to fetch a descriptor for the
+     * pending connections because we just got more directory information.
+     * However, the refetch process can cleanup all SOCKS request to the same
+     * service if an internal error happens. Thus, we can end up with closed
+     * connections in our list. */
+    if (base_conn->marked_for_close) {
+      continue;
+    }
+
+    /* XXX: There is an optimization we could do which is that for a service
+     * key, we could check if we can fetch and remember that decision. */
+
+    /* Order a refetch in case it works this time. */
+    status = hs_client_refetch_hsdesc(&edge_conn->hs_ident->identity_pk);
+    if (status == HS_CLIENT_FETCH_HAVE_DESC) {
+      /* This is a rare case where a SOCKS connection is in state waiting for
+       * a descriptor but we do have it in the cache.
+       *
+       * This can happen is tor comes back from suspend where it previously
+       * had the descriptor but the intro points were not usuable. Once it
+       * came back to life, the intro point failure cache was cleaned up and
+       * thus the descriptor became usable again leaving us in this code path.
+       *
+       * We'll mark the connection as waiting for a circuit so the descriptor
+       * can be retried. This is safe because a connection in state waiting
+       * for a descriptor can not be in the entry connection pending list. */
+      mark_conn_as_waiting_for_circuit(base_conn, approx_time());
+      continue;
+    }
+    /* In the case of an error, either all SOCKS connections have been
+     * closed or we are still missing directory information. Leave the
+     * connection in renddesc wait state so when we get more info, we'll be
+     * able to try it again. */
+  } SMARTLIST_FOREACH_END(base_conn);
+
+  /* We don't have ownership of those objects. */
+  smartlist_free(conns);
+}
+
+/* A v3 HS circuit successfully connected to the hidden service. Update the
+ * stream state at <b>hs_conn_ident</b> appropriately. */
+static void
+note_connection_attempt_succeeded(const hs_ident_edge_conn_t *hs_conn_ident)
+{
+  tor_assert(hs_conn_ident);
+
+  /* Remove from the hid serv cache all requests for that service so we can
+   * query the HSDir again later on for various reasons. */
+  purge_hid_serv_request(&hs_conn_ident->identity_pk);
+
+  /* The v2 subsystem cleans up the intro point time out flag at this stage.
+   * We don't try to do it here because we still need to keep intact the intro
+   * point state for future connections. Even though we are able to connect to
+   * the service, doesn't mean we should reset the timed out intro points.
+   *
+   * It is not possible to have successfully connected to an intro point
+   * present in our cache that was on error or timed out. Every entry in that
+   * cache have a 2 minutes lifetime so ultimately the intro point(s) state
+   * will be reset and thus possible to be retried. */
+}
+
+/* Given the pubkey of a hidden service in <b>onion_identity_pk</b>, fetch its
+ * descriptor by launching a dir connection to <b>hsdir</b>. Return a
+ * hs_client_fetch_status_t status code depending on how it went. */
+static hs_client_fetch_status_t
+directory_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk,
+                               const routerstatus_t *hsdir)
+{
+  uint64_t current_time_period = hs_get_time_period_num(0);
+  ed25519_public_key_t blinded_pubkey;
+  char base64_blinded_pubkey[ED25519_BASE64_LEN + 1];
+  hs_ident_dir_conn_t hs_conn_dir_ident;
+  int retval;
+
+  tor_assert(hsdir);
+  tor_assert(onion_identity_pk);
+
+  /* Get blinded pubkey */
+  hs_build_blinded_pubkey(onion_identity_pk, NULL, 0,
+                          current_time_period, &blinded_pubkey);
+  /* ...and base64 it. */
+  retval = ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
+  if (BUG(retval < 0)) {
+    return HS_CLIENT_FETCH_ERROR;
+  }
+
+  /* Copy onion pk to a dir_ident so that we attach it to the dir conn */
+  hs_ident_dir_conn_init(onion_identity_pk, &blinded_pubkey,
+                         &hs_conn_dir_ident);
+
+  /* Setup directory request */
+  directory_request_t *req =
+    directory_request_new(DIR_PURPOSE_FETCH_HSDESC);
+  directory_request_set_routerstatus(req, hsdir);
+  directory_request_set_indirection(req, DIRIND_ANONYMOUS);
+  directory_request_set_resource(req, base64_blinded_pubkey);
+  directory_request_fetch_set_hs_ident(req, &hs_conn_dir_ident);
+  directory_initiate_request(req);
+  directory_request_free(req);
+
+  log_info(LD_REND, "Descriptor fetch request for service %s with blinded "
+                    "key %s to directory %s",
+           safe_str_client(ed25519_fmt(onion_identity_pk)),
+           safe_str_client(base64_blinded_pubkey),
+           safe_str_client(routerstatus_describe(hsdir)));
+
+  /* Fire a REQUESTED event on the control port. */
+  hs_control_desc_event_requested(onion_identity_pk, base64_blinded_pubkey,
+                                  hsdir);
+
+  /* Cleanup memory. */
+  memwipe(&blinded_pubkey, 0, sizeof(blinded_pubkey));
+  memwipe(base64_blinded_pubkey, 0, sizeof(base64_blinded_pubkey));
+  memwipe(&hs_conn_dir_ident, 0, sizeof(hs_conn_dir_ident));
+
+  return HS_CLIENT_FETCH_LAUNCHED;
+}
+
+/** Return the HSDir we should use to fetch the descriptor of the hidden
+ *  service with identity key <b>onion_identity_pk</b>. */
+STATIC routerstatus_t *
+pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk)
+{
+  int retval;
+  char base64_blinded_pubkey[ED25519_BASE64_LEN + 1];
+  uint64_t current_time_period = hs_get_time_period_num(0);
+  smartlist_t *responsible_hsdirs = NULL;
+  ed25519_public_key_t blinded_pubkey;
+  routerstatus_t *hsdir_rs = NULL;
+
+  tor_assert(onion_identity_pk);
+
+  /* Get blinded pubkey of hidden service */
+  hs_build_blinded_pubkey(onion_identity_pk, NULL, 0,
+                          current_time_period, &blinded_pubkey);
+  /* ...and base64 it. */
+  retval = ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
+  if (BUG(retval < 0)) {
+    return NULL;
+  }
+
+  /* Get responsible hsdirs of service for this time period */
+  responsible_hsdirs = smartlist_new();
+
+  hs_get_responsible_hsdirs(&blinded_pubkey, current_time_period,
+                            0, 1, responsible_hsdirs);
+
+  log_debug(LD_REND, "Found %d responsible HSDirs and about to pick one.",
+           smartlist_len(responsible_hsdirs));
+
+  /* Pick an HSDir from the responsible ones. The ownership of
+   * responsible_hsdirs is given to this function so no need to free it. */
+  hsdir_rs = hs_pick_hsdir(responsible_hsdirs, base64_blinded_pubkey);
+
+  return hsdir_rs;
+}
+
+/** Fetch a v3 descriptor using the given <b>onion_identity_pk</b>.
+ *
+ * On success, HS_CLIENT_FETCH_LAUNCHED is returned. Otherwise, an error from
+ * hs_client_fetch_status_t is returned. */
+MOCK_IMPL(STATIC hs_client_fetch_status_t,
+fetch_v3_desc, (const ed25519_public_key_t *onion_identity_pk))
+{
+  routerstatus_t *hsdir_rs =NULL;
+
+  tor_assert(onion_identity_pk);
+
+  hsdir_rs = pick_hsdir_v3(onion_identity_pk);
+  if (!hsdir_rs) {
+    log_info(LD_REND, "Couldn't pick a v3 hsdir.");
+    return HS_CLIENT_FETCH_NO_HSDIRS;
+  }
+
+  return directory_launch_v3_desc_fetch(onion_identity_pk, hsdir_rs);
+}
+
+/* Make sure that the given v3 origin circuit circ is a valid correct
+ * introduction circuit. This will BUG() on any problems and hard assert if
+ * the anonymity of the circuit is not ok. Return 0 on success else -1 where
+ * the circuit should be mark for closed immediately. */
+static int
+intro_circ_is_ok(const origin_circuit_t *circ)
+{
+  int ret = 0;
+
+  tor_assert(circ);
+
+  if (BUG(TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_C_INTRODUCING &&
+          TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT &&
+          TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_C_INTRODUCE_ACKED)) {
+    ret = -1;
+  }
+  if (BUG(circ->hs_ident == NULL)) {
+    ret = -1;
+  }
+  if (BUG(!hs_ident_intro_circ_is_valid(circ->hs_ident))) {
+    ret = -1;
+  }
+
+  /* This can stop the tor daemon but we want that since if we don't have
+   * anonymity on this circuit, something went really wrong. */
+  assert_circ_anonymity_ok(circ, get_options());
+  return ret;
+}
+
+/* Find a descriptor intro point object that matches the given ident in the
+ * given descriptor desc. Return NULL if not found. */
+static const hs_desc_intro_point_t *
+find_desc_intro_point_by_ident(const hs_ident_circuit_t *ident,
+                               const hs_descriptor_t *desc)
+{
+  const hs_desc_intro_point_t *intro_point = NULL;
+
+  tor_assert(ident);
+  tor_assert(desc);
+
+  SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                          const hs_desc_intro_point_t *, ip) {
+    if (ed25519_pubkey_eq(&ident->intro_auth_pk,
+                          &ip->auth_key_cert->signed_key)) {
+      intro_point = ip;
+      break;
+    }
+  } SMARTLIST_FOREACH_END(ip);
+
+  return intro_point;
+}
+
+/* Find a descriptor intro point object from the descriptor object desc that
+ * matches the given legacy identity digest in legacy_id. Return NULL if not
+ * found. */
+static hs_desc_intro_point_t *
+find_desc_intro_point_by_legacy_id(const char *legacy_id,
+                                   const hs_descriptor_t *desc)
+{
+  hs_desc_intro_point_t *ret_ip = NULL;
+
+  tor_assert(legacy_id);
+  tor_assert(desc);
+
+  /* We will go over every intro point and try to find which one is linked to
+   * that circuit. Those lists are small so it's not that expensive. */
+  SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                          hs_desc_intro_point_t *, ip) {
+    SMARTLIST_FOREACH_BEGIN(ip->link_specifiers,
+                            const hs_desc_link_specifier_t *, lspec) {
+      /* Not all tor node have an ed25519 identity key so we still rely on the
+       * legacy identity digest. */
+      if (lspec->type != LS_LEGACY_ID) {
+        continue;
+      }
+      if (fast_memneq(legacy_id, lspec->u.legacy_id, DIGEST_LEN)) {
+        break;
+      }
+      /* Found it. */
+      ret_ip = ip;
+      goto end;
+    } SMARTLIST_FOREACH_END(lspec);
+  } SMARTLIST_FOREACH_END(ip);
+
+ end:
+  return ret_ip;
+}
+
+/* Send an INTRODUCE1 cell along the intro circuit and populate the rend
+ * circuit identifier with the needed key material for the e2e encryption.
+ * Return 0 on success, -1 if there is a transient error such that an action
+ * has been taken to recover and -2 if there is a permanent error indicating
+ * that both circuits were closed. */
+static int
+send_introduce1(origin_circuit_t *intro_circ,
+                origin_circuit_t *rend_circ)
+{
+  int status;
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+  const ed25519_public_key_t *service_identity_pk = NULL;
+  const hs_desc_intro_point_t *ip;
+
+  tor_assert(rend_circ);
+  if (intro_circ_is_ok(intro_circ) < 0) {
+    goto perm_err;
+  }
+
+  service_identity_pk = &intro_circ->hs_ident->identity_pk;
+  /* For logging purposes. There will be a time where the hs_ident will have a
+   * version number but for now there is none because it's all v3. */
+  hs_build_address(service_identity_pk, HS_VERSION_THREE, onion_address);
+
+  log_info(LD_REND, "Sending INTRODUCE1 cell to service %s on circuit %u",
+           safe_str_client(onion_address), TO_CIRCUIT(intro_circ)->n_circ_id);
+
+  /* 1) Get descriptor from our cache. */
+  const hs_descriptor_t *desc =
+    hs_cache_lookup_as_client(service_identity_pk);
+  if (desc == NULL || !hs_client_any_intro_points_usable(service_identity_pk,
+                                                         desc)) {
+    log_info(LD_REND, "Request to %s %s. Trying to fetch a new descriptor.",
+             safe_str_client(onion_address),
+             (desc) ? "didn't have usable intro points" :
+             "didn't have a descriptor");
+    hs_client_refetch_hsdesc(service_identity_pk);
+    /* We just triggered a refetch, make sure every connections are back
+     * waiting for that descriptor. */
+    flag_all_conn_wait_desc(service_identity_pk);
+    /* We just asked for a refetch so this is a transient error. */
+    goto tran_err;
+  }
+
+  /* We need to find which intro point in the descriptor we are connected to
+   * on intro_circ. */
+  ip = find_desc_intro_point_by_ident(intro_circ->hs_ident, desc);
+  if (BUG(ip == NULL)) {
+    /* If we can find a descriptor from this introduction circuit ident, we
+     * must have a valid intro point object. Permanent error. */
+    goto perm_err;
+  }
+
+  /* Send the INTRODUCE1 cell. */
+  if (hs_circ_send_introduce1(intro_circ, rend_circ, ip,
+                              desc->subcredential) < 0) {
+    if (TO_CIRCUIT(intro_circ)->marked_for_close) {
+      /* If the introduction circuit was closed, we were unable to send the
+       * cell for some reasons. In any case, the intro circuit has to be
+       * closed by the above function. We'll return a transient error so tor
+       * can recover and pick a new intro point. To avoid picking that same
+       * intro point, we'll note down the intro point failure so it doesn't
+       * get reused. */
+      hs_cache_client_intro_state_note(service_identity_pk,
+                                       &intro_circ->hs_ident->intro_auth_pk,
+                                       INTRO_POINT_FAILURE_GENERIC);
+    }
+    /* It is also possible that the rendezvous circuit was closed due to being
+     * unable to use the rendezvous point node_t so in that case, we also want
+     * to recover and let tor pick a new one. */
+    goto tran_err;
+  }
+
+  /* Cell has been sent successfully. Copy the introduction point
+   * authentication and encryption key in the rendezvous circuit identifier so
+   * we can compute the ntor keys when we receive the RENDEZVOUS2 cell. */
+  memcpy(&rend_circ->hs_ident->intro_enc_pk, &ip->enc_key,
+         sizeof(rend_circ->hs_ident->intro_enc_pk));
+  ed25519_pubkey_copy(&rend_circ->hs_ident->intro_auth_pk,
+                      &intro_circ->hs_ident->intro_auth_pk);
+
+  /* Now, we wait for an ACK or NAK on this circuit. */
+  circuit_change_purpose(TO_CIRCUIT(intro_circ),
+                         CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT);
+  /* Set timestamp_dirty, because circuit_expire_building expects it to
+   * specify when a circuit entered the _C_INTRODUCE_ACK_WAIT state. */
+  TO_CIRCUIT(intro_circ)->timestamp_dirty = time(NULL);
+  pathbias_count_use_attempt(intro_circ);
+
+  /* Success. */
+  status = 0;
+  goto end;
+
+ perm_err:
+  /* Permanent error: it is possible that the intro circuit was closed prior
+   * because we weren't able to send the cell. Make sure we don't double close
+   * it which would result in a warning. */
+  if (!TO_CIRCUIT(intro_circ)->marked_for_close) {
+    circuit_mark_for_close(TO_CIRCUIT(intro_circ), END_CIRC_REASON_INTERNAL);
+  }
+  circuit_mark_for_close(TO_CIRCUIT(rend_circ), END_CIRC_REASON_INTERNAL);
+  status = -2;
+  goto end;
+
+ tran_err:
+  status = -1;
+
+ end:
+  memwipe(onion_address, 0, sizeof(onion_address));
+  return status;
+}
+
+/* Using the introduction circuit circ, setup the authentication key of the
+ * intro point this circuit has extended to. */
+static void
+setup_intro_circ_auth_key(origin_circuit_t *circ)
+{
+  const hs_descriptor_t *desc;
+  const hs_desc_intro_point_t *ip;
+
+  tor_assert(circ);
+
+  desc = hs_cache_lookup_as_client(&circ->hs_ident->identity_pk);
+  if (desc == NULL) {
+    /* There is a very small race window between the opening of this circuit
+     * and the client descriptor cache that gets purged (NEWNYM) or the
+     * cleaned up because it expired. Mark the circuit for close so a new
+     * descriptor fetch can occur. */
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
+    goto end;
+  }
+
+  /* We will go over every intro point and try to find which one is linked to
+   * that circuit. Those lists are small so it's not that expensive. */
+  ip = find_desc_intro_point_by_legacy_id(
+                       circ->build_state->chosen_exit->identity_digest, desc);
+  if (ip) {
+    /* We got it, copy its authentication key to the identifier. */
+    ed25519_pubkey_copy(&circ->hs_ident->intro_auth_pk,
+                        &ip->auth_key_cert->signed_key);
+    goto end;
+  }
+
+  /* Reaching this point means we didn't find any intro point for this circuit
+   * which is not suppose to happen. */
+  tor_assert_nonfatal_unreached();
+
+ end:
+  return;
+}
+
+/* Called when an introduction circuit has opened. */
+static void
+client_intro_circ_has_opened(origin_circuit_t *circ)
+{
+  tor_assert(circ);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_C_INTRODUCING);
+  log_info(LD_REND, "Introduction circuit %u has opened. Attaching streams.",
+           (unsigned int) TO_CIRCUIT(circ)->n_circ_id);
+
+  /* This is an introduction circuit so we'll attach the correct
+   * authentication key to the circuit identifier so it can be identified
+   * properly later on. */
+  setup_intro_circ_auth_key(circ);
+
+  connection_ap_attach_pending(1);
+}
+
+/* Called when a rendezvous circuit has opened. */
+static void
+client_rendezvous_circ_has_opened(origin_circuit_t *circ)
+{
+  tor_assert(circ);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND);
+
+  const extend_info_t *rp_ei = circ->build_state->chosen_exit;
+
+  /* Check that we didn't accidentally choose a node that does not understand
+   * the v3 rendezvous protocol */
+  if (rp_ei) {
+    const node_t *rp_node = node_get_by_id(rp_ei->identity_digest);
+    if (rp_node) {
+      if (BUG(!node_supports_v3_rendezvous_point(rp_node))) {
+        return;
+      }
+    }
+  }
+
+  log_info(LD_REND, "Rendezvous circuit has opened to %s.",
+           safe_str_client(extend_info_describe(rp_ei)));
+
+  /* Ignore returned value, nothing we can really do. On failure, the circuit
+   * will be marked for close. */
+  hs_circ_send_establish_rendezvous(circ);
+
+  /* Register rend circuit in circuitmap if it's still alive. */
+  if (!TO_CIRCUIT(circ)->marked_for_close) {
+    hs_circuitmap_register_rend_circ_client_side(circ,
+                                     circ->hs_ident->rendezvous_cookie);
+  }
+}
+
+/* This is an helper function that convert a descriptor intro point object ip
+ * to a newly allocated extend_info_t object fully initialized. Return NULL if
+ * we can't convert it for which chances are that we are missing or malformed
+ * link specifiers. */
+STATIC extend_info_t *
+desc_intro_point_to_extend_info(const hs_desc_intro_point_t *ip)
+{
+  extend_info_t *ei;
+  smartlist_t *lspecs = smartlist_new();
+
+  tor_assert(ip);
+
+  /* We first encode the descriptor link specifiers into the binary
+   * representation which is a trunnel object. */
+  SMARTLIST_FOREACH_BEGIN(ip->link_specifiers,
+                          const hs_desc_link_specifier_t *, desc_lspec) {
+    link_specifier_t *lspec = hs_desc_lspec_to_trunnel(desc_lspec);
+    smartlist_add(lspecs, lspec);
+  } SMARTLIST_FOREACH_END(desc_lspec);
+
+  /* Explicitly put the direct connection option to 0 because this is client
+   * side and there is no such thing as a non anonymous client. */
+  ei = hs_get_extend_info_from_lspecs(lspecs, &ip->onion_key, 0);
+
+  SMARTLIST_FOREACH(lspecs, link_specifier_t *, ls, link_specifier_free(ls));
+  smartlist_free(lspecs);
+  return ei;
+}
+
+/* Return true iff the intro point ip for the service service_pk is usable.
+ * This function checks if the intro point is in the client intro state cache
+ * and checks at the failures. It is considered usable if:
+ *   - No error happened (INTRO_POINT_FAILURE_GENERIC)
+ *   - It is not flagged as timed out (INTRO_POINT_FAILURE_TIMEOUT)
+ *   - The unreachable count is lower than
+ *     MAX_INTRO_POINT_REACHABILITY_FAILURES (INTRO_POINT_FAILURE_UNREACHABLE)
+ */
+static int
+intro_point_is_usable(const ed25519_public_key_t *service_pk,
+                      const hs_desc_intro_point_t *ip)
+{
+  const hs_cache_intro_state_t *state;
+
+  tor_assert(service_pk);
+  tor_assert(ip);
+
+  state = hs_cache_client_intro_state_find(service_pk,
+                                           &ip->auth_key_cert->signed_key);
+  if (state == NULL) {
+    /* This means we've never encountered any problem thus usable. */
+    goto usable;
+  }
+  if (state->error) {
+    log_info(LD_REND, "Intro point with auth key %s had an error. Not usable",
+             safe_str_client(ed25519_fmt(&ip->auth_key_cert->signed_key)));
+    goto not_usable;
+  }
+  if (state->timed_out) {
+    log_info(LD_REND, "Intro point with auth key %s timed out. Not usable",
+             safe_str_client(ed25519_fmt(&ip->auth_key_cert->signed_key)));
+    goto not_usable;
+  }
+  if (state->unreachable_count >= MAX_INTRO_POINT_REACHABILITY_FAILURES) {
+    log_info(LD_REND, "Intro point with auth key %s unreachable. Not usable",
+             safe_str_client(ed25519_fmt(&ip->auth_key_cert->signed_key)));
+    goto not_usable;
+  }
+
+ usable:
+  return 1;
+ not_usable:
+  return 0;
+}
+
+/* Using a descriptor desc, return a newly allocated extend_info_t object of a
+ * randomly picked introduction point from its list. Return NULL if none are
+ * usable. */
+STATIC extend_info_t *
+client_get_random_intro(const ed25519_public_key_t *service_pk)
+{
+  extend_info_t *ei = NULL, *ei_excluded = NULL;
+  smartlist_t *usable_ips = NULL;
+  const hs_descriptor_t *desc;
+  const hs_desc_encrypted_data_t *enc_data;
+  const or_options_t *options = get_options();
+  /* Calculate the onion address for logging purposes */
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+
+  tor_assert(service_pk);
+
+  desc = hs_cache_lookup_as_client(service_pk);
+  /* Assume the service is v3 if the descriptor is missing. This is ok,
+   * because we only use the address in log messages */
+  hs_build_address(service_pk,
+                   desc ? desc->plaintext_data.version : HS_VERSION_THREE,
+                   onion_address);
+  if (desc == NULL || !hs_client_any_intro_points_usable(service_pk,
+                                                         desc)) {
+    log_info(LD_REND, "Unable to randomly select an introduction point "
+             "for service %s because descriptor %s. We can't connect.",
+             safe_str_client(onion_address),
+             (desc) ? "doesn't have any usable intro points"
+                    : "is missing (assuming v3 onion address)");
+    goto end;
+  }
+
+  enc_data = &desc->encrypted_data;
+  usable_ips = smartlist_new();
+  smartlist_add_all(usable_ips, enc_data->intro_points);
+  while (smartlist_len(usable_ips) != 0) {
+    int idx;
+    const hs_desc_intro_point_t *ip;
+
+    /* Pick a random intro point and immediately remove it from the usable
+     * list so we don't pick it again if we have to iterate more. */
+    idx = crypto_rand_int(smartlist_len(usable_ips));
+    ip = smartlist_get(usable_ips, idx);
+    smartlist_del(usable_ips, idx);
+
+    /* We need to make sure we have a usable intro points which is in a good
+     * state in our cache. */
+    if (!intro_point_is_usable(service_pk, ip)) {
+      continue;
+    }
+
+    /* Generate an extend info object from the intro point object. */
+    ei = desc_intro_point_to_extend_info(ip);
+    if (ei == NULL) {
+      /* We can get here for instance if the intro point is a private address
+       * and we aren't allowed to extend to those. */
+      log_info(LD_REND, "Unable to select introduction point with auth key %s "
+               "for service %s, because we could not extend to it.",
+               safe_str_client(ed25519_fmt(&ip->auth_key_cert->signed_key)),
+               safe_str_client(onion_address));
+      continue;
+    }
+
+    /* Test the pick against ExcludeNodes. */
+    if (routerset_contains_extendinfo(options->ExcludeNodes, ei)) {
+      /* If this pick is in the ExcludeNodes list, we keep its reference so if
+       * we ever end up not being able to pick anything else and StrictNodes is
+       * unset, we'll use it. */
+      if (ei_excluded) {
+        /* If something was already here free it. After the loop is gone we
+         * will examine the last excluded intro point, and that's fine since
+         * that's random anyway */
+        extend_info_free(ei_excluded);
+      }
+      ei_excluded = ei;
+      continue;
+    }
+
+    /* Good pick! Let's go with this. */
+    goto end;
+  }
+
+  /* Reaching this point means a couple of things. Either we can't use any of
+   * the intro point listed because the IP address can't be extended to or it
+   * is listed in the ExcludeNodes list. In the later case, if StrictNodes is
+   * set, we are forced to not use anything. */
+  ei = ei_excluded;
+  if (options->StrictNodes) {
+    log_warn(LD_REND, "Every introduction point for service %s is in the "
+             "ExcludeNodes set and StrictNodes is set. We can't connect.",
+             safe_str_client(onion_address));
+    extend_info_free(ei);
+    ei = NULL;
+  } else {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND, "Every introduction point for service "
+           "%s is unusable or we can't extend to it. We can't connect.",
+           safe_str_client(onion_address));
+  }
+
+ end:
+  smartlist_free(usable_ips);
+  memwipe(onion_address, 0, sizeof(onion_address));
+  return ei;
+}
+
+/* For this introduction circuit, we'll look at if we have any usable
+ * introduction point left for this service. If so, we'll use the circuit to
+ * re-extend to a new intro point. Else, we'll close the circuit and its
+ * corresponding rendezvous circuit. Return 0 if we are re-extending else -1
+ * if we are closing the circuits.
+ *
+ * This is called when getting an INTRODUCE_ACK cell with a NACK. */
+static int
+close_or_reextend_intro_circ(origin_circuit_t *intro_circ)
+{
+  int ret = -1;
+  const hs_descriptor_t *desc;
+  origin_circuit_t *rend_circ;
+
+  tor_assert(intro_circ);
+
+  desc = hs_cache_lookup_as_client(&intro_circ->hs_ident->identity_pk);
+  if (BUG(desc == NULL)) {
+    /* We can't continue without a descriptor. */
+    goto close;
+  }
+  /* We still have the descriptor, great! Let's try to see if we can
+   * re-extend by looking up if there are any usable intro points. */
+  if (!hs_client_any_intro_points_usable(&intro_circ->hs_ident->identity_pk,
+                                         desc)) {
+    goto close;
+  }
+  /* Try to re-extend now. */
+  if (hs_client_reextend_intro_circuit(intro_circ) < 0) {
+    goto close;
+  }
+  /* Success on re-extending. Don't return an error. */
+  ret = 0;
+  goto end;
+
+ close:
+  /* Change the intro circuit purpose before so we don't report an intro point
+   * failure again triggering an extra descriptor fetch. The circuit can
+   * already be closed on failure to re-extend. */
+  if (!TO_CIRCUIT(intro_circ)->marked_for_close) {
+    circuit_change_purpose(TO_CIRCUIT(intro_circ),
+                           CIRCUIT_PURPOSE_C_INTRODUCE_ACKED);
+    circuit_mark_for_close(TO_CIRCUIT(intro_circ), END_CIRC_REASON_FINISHED);
+  }
+  /* Close the related rendezvous circuit. */
+  rend_circ = hs_circuitmap_get_rend_circ_client_side(
+                                     intro_circ->hs_ident->rendezvous_cookie);
+  /* The rendezvous circuit might have collapsed while the INTRODUCE_ACK was
+   * inflight so we can't expect one every time. */
+  if (rend_circ) {
+    circuit_mark_for_close(TO_CIRCUIT(rend_circ), END_CIRC_REASON_FINISHED);
+  }
+
+ end:
+  return ret;
+}
+
+/* Called when we get an INTRODUCE_ACK success status code. Do the appropriate
+ * actions for the rendezvous point and finally close intro_circ. */
+static void
+handle_introduce_ack_success(origin_circuit_t *intro_circ)
+{
+  origin_circuit_t *rend_circ = NULL;
+
+  tor_assert(intro_circ);
+
+  log_info(LD_REND, "Received INTRODUCE_ACK ack! Informing rendezvous");
+
+  /* Get the rendezvous circuit for this rendezvous cookie. */
+  uint8_t *rendezvous_cookie = intro_circ->hs_ident->rendezvous_cookie;
+  rend_circ =
+  hs_circuitmap_get_established_rend_circ_client_side(rendezvous_cookie);
+  if (rend_circ == NULL) {
+    log_warn(LD_REND, "Can't find any rendezvous circuit. Stopping");
+    goto end;
+  }
+
+  assert_circ_anonymity_ok(rend_circ, get_options());
+
+  /* It is possible to get a RENDEZVOUS2 cell before the INTRODUCE_ACK which
+   * means that the circuit will be joined and already transmitting data. In
+   * that case, simply skip the purpose change and close the intro circuit
+   * like it should be. */
+  if (TO_CIRCUIT(rend_circ)->purpose == CIRCUIT_PURPOSE_C_REND_JOINED) {
+    goto end;
+  }
+  circuit_change_purpose(TO_CIRCUIT(rend_circ),
+                         CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED);
+  /* Set timestamp_dirty, because circuit_expire_building expects it to
+   * specify when a circuit entered the
+   * CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED state. */
+  TO_CIRCUIT(rend_circ)->timestamp_dirty = time(NULL);
+
+ end:
+  /* We don't need the intro circuit anymore. It did what it had to do! */
+  circuit_change_purpose(TO_CIRCUIT(intro_circ),
+                         CIRCUIT_PURPOSE_C_INTRODUCE_ACKED);
+  circuit_mark_for_close(TO_CIRCUIT(intro_circ), END_CIRC_REASON_FINISHED);
+
+  /* XXX: Close pending intro circuits we might have in parallel. */
+  return;
+}
+
+/* Called when we get an INTRODUCE_ACK failure status code. Depending on our
+ * failure cache status, either close the circuit or re-extend to a new
+ * introduction point. */
+static void
+handle_introduce_ack_bad(origin_circuit_t *circ, int status)
+{
+  tor_assert(circ);
+
+  log_info(LD_REND, "Received INTRODUCE_ACK nack by %s. Reason: %u",
+      safe_str_client(extend_info_describe(circ->build_state->chosen_exit)),
+      status);
+
+  /* It's a NAK. The introduction point didn't relay our request. */
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_INTRODUCING);
+
+  /* Note down this failure in the intro point failure cache. Depending on how
+   * many times we've tried this intro point, close it or reextend. */
+  hs_cache_client_intro_state_note(&circ->hs_ident->identity_pk,
+                                   &circ->hs_ident->intro_auth_pk,
+                                   INTRO_POINT_FAILURE_GENERIC);
+}
+
+/* Called when we get an INTRODUCE_ACK on the intro circuit circ. The encoded
+ * cell is in payload of length payload_len. Return 0 on success else a
+ * negative value. The circuit is either close or reuse to re-extend to a new
+ * introduction point. */
+static int
+handle_introduce_ack(origin_circuit_t *circ, const uint8_t *payload,
+                     size_t payload_len)
+{
+  int status, ret = -1;
+
+  tor_assert(circ);
+  tor_assert(circ->build_state);
+  tor_assert(circ->build_state->chosen_exit);
+  assert_circ_anonymity_ok(circ, get_options());
+  tor_assert(payload);
+
+  status = hs_cell_parse_introduce_ack(payload, payload_len);
+  switch (status) {
+  case TRUNNEL_HS_INTRO_ACK_STATUS_SUCCESS:
+    ret = 0;
+    handle_introduce_ack_success(circ);
+    goto end;
+  case TRUNNEL_HS_INTRO_ACK_STATUS_UNKNOWN_ID:
+  case TRUNNEL_HS_INTRO_ACK_STATUS_BAD_FORMAT:
+  /* It is possible that the intro point can send us an unknown status code
+   * for the NACK that we do not know about like a new code for instance.
+   * Just fallthrough so we can note down the NACK and re-extend. */
+  default:
+    handle_introduce_ack_bad(circ, status);
+    /* We are going to see if we have to close the circuits (IP and RP) or we
+     * can re-extend to a new intro point. */
+    ret = close_or_reextend_intro_circ(circ);
+    break;
+  }
+
+ end:
+  return ret;
+}
+
+/* Called when we get a RENDEZVOUS2 cell on the rendezvous circuit circ. The
+ * encoded cell is in payload of length payload_len. Return 0 on success or a
+ * negative value on error. On error, the circuit is marked for close. */
+STATIC int
+handle_rendezvous2(origin_circuit_t *circ, const uint8_t *payload,
+                   size_t payload_len)
+{
+  int ret = -1;
+  curve25519_public_key_t server_pk;
+  uint8_t auth_mac[DIGEST256_LEN] = {0};
+  uint8_t handshake_info[CURVE25519_PUBKEY_LEN + sizeof(auth_mac)] = {0};
+  hs_ntor_rend_cell_keys_t keys;
+  const hs_ident_circuit_t *ident;
+
+  tor_assert(circ);
+  tor_assert(payload);
+
+  /* Make things easier. */
+  ident = circ->hs_ident;
+  tor_assert(ident);
+
+  if (hs_cell_parse_rendezvous2(payload, payload_len, handshake_info,
+                                sizeof(handshake_info)) < 0) {
+    goto err;
+  }
+  /* Get from the handshake info the SERVER_PK and AUTH_MAC. */
+  memcpy(&server_pk, handshake_info, CURVE25519_PUBKEY_LEN);
+  memcpy(auth_mac, handshake_info + CURVE25519_PUBKEY_LEN, sizeof(auth_mac));
+
+  /* Generate the handshake info. */
+  if (hs_ntor_client_get_rendezvous1_keys(&ident->intro_auth_pk,
+                                          &ident->rendezvous_client_kp,
+                                          &ident->intro_enc_pk, &server_pk,
+                                          &keys) < 0) {
+    log_info(LD_REND, "Unable to compute the rendezvous keys.");
+    goto err;
+  }
+
+  /* Critical check, make sure that the MAC matches what we got with what we
+   * computed just above. */
+  if (!hs_ntor_client_rendezvous2_mac_is_good(&keys, auth_mac)) {
+    log_info(LD_REND, "Invalid MAC in RENDEZVOUS2. Rejecting cell.");
+    goto err;
+  }
+
+  /* Setup the e2e encryption on the circuit and finalize its state. */
+  if (hs_circuit_setup_e2e_rend_circ(circ, keys.ntor_key_seed,
+                                     sizeof(keys.ntor_key_seed), 0) < 0) {
+    log_info(LD_REND, "Unable to setup the e2e encryption.");
+    goto err;
+  }
+  /* Success. Hidden service connection finalized! */
+  ret = 0;
+  goto end;
+
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+ end:
+  memwipe(&keys, 0, sizeof(keys));
+  return ret;
+}
+
+/* Return true iff the client can fetch a descriptor for this service public
+ * identity key and status_out if not NULL is untouched. If the client can
+ * _not_ fetch the descriptor and if status_out is not NULL, it is set with
+ * the fetch status code. */
+static unsigned int
+can_client_refetch_desc(const ed25519_public_key_t *identity_pk,
+                        hs_client_fetch_status_t *status_out)
+{
+  hs_client_fetch_status_t status;
+
+  tor_assert(identity_pk);
+
+  /* Are we configured to fetch descriptors? */
+  if (!get_options()->FetchHidServDescriptors) {
+    log_warn(LD_REND, "We received an onion address for a hidden service "
+                      "descriptor but we are configured to not fetch.");
+    status = HS_CLIENT_FETCH_NOT_ALLOWED;
+    goto cannot;
+  }
+
+  /* Without a live consensus we can't do any client actions. It is needed to
+   * compute the hashring for a service. */
+  if (!networkstatus_get_live_consensus(approx_time())) {
+    log_info(LD_REND, "Can't fetch descriptor for service %s because we "
+                      "are missing a live consensus. Stalling connection.",
+             safe_str_client(ed25519_fmt(identity_pk)));
+    status = HS_CLIENT_FETCH_MISSING_INFO;
+    goto cannot;
+  }
+
+  if (!router_have_minimum_dir_info()) {
+    log_info(LD_REND, "Can't fetch descriptor for service %s because we "
+                      "dont have enough descriptors. Stalling connection.",
+             safe_str_client(ed25519_fmt(identity_pk)));
+    status = HS_CLIENT_FETCH_MISSING_INFO;
+    goto cannot;
+  }
+
+  /* Check if fetching a desc for this HS is useful to us right now */
+  {
+    const hs_descriptor_t *cached_desc = NULL;
+    cached_desc = hs_cache_lookup_as_client(identity_pk);
+    if (cached_desc && hs_client_any_intro_points_usable(identity_pk,
+                                                         cached_desc)) {
+      log_info(LD_GENERAL, "We would fetch a v3 hidden service descriptor "
+                           "but we already have a usable descriptor.");
+      status = HS_CLIENT_FETCH_HAVE_DESC;
+      goto cannot;
+    }
+  }
+
+  /* Don't try to refetch while we have a pending request for it. */
+  if (directory_request_is_pending(identity_pk)) {
+    log_info(LD_REND, "Already a pending directory request. Waiting on it.");
+    status = HS_CLIENT_FETCH_PENDING;
+    goto cannot;
+  }
+
+  /* Yes, client can fetch! */
+  return 1;
+ cannot:
+  if (status_out) {
+    *status_out = status;
+  }
+  return 0;
+}
+
+/* Return the client auth in the map using the service identity public key.
+ * Return NULL if it does not exist in the map. */
+static hs_client_service_authorization_t *
+find_client_auth(const ed25519_public_key_t *service_identity_pk)
+{
+  /* If the map is not allocated, we can assume that we do not have any client
+   * auth information. */
+  if (!client_auths) {
+    return NULL;
+  }
+  return digest256map_get(client_auths, service_identity_pk->pubkey);
+}
+
+/* ========== */
+/* Public API */
+/* ========== */
+
+/** A circuit just finished connecting to a hidden service that the stream
+ *  <b>conn</b> has been waiting for. Let the HS subsystem know about this. */
+void
+hs_client_note_connection_attempt_succeeded(const edge_connection_t *conn)
+{
+  tor_assert(connection_edge_is_rendezvous_stream(conn));
+
+  if (BUG(conn->rend_data && conn->hs_ident)) {
+    log_warn(LD_BUG, "Stream had both rend_data and hs_ident..."
+             "Prioritizing hs_ident");
+  }
+
+  if (conn->hs_ident) { /* It's v3: pass it to the prop224 handler */
+    note_connection_attempt_succeeded(conn->hs_ident);
+    return;
+  } else if (conn->rend_data) { /* It's v2: pass it to the legacy handler */
+    rend_client_note_connection_attempt_ended(conn->rend_data);
+    return;
+  }
+}
+
+/* With the given encoded descriptor in desc_str and the service key in
+ * service_identity_pk, decode the descriptor and set the desc pointer with a
+ * newly allocated descriptor object.
+ *
+ * Return 0 on success else a negative value and desc is set to NULL. */
+int
+hs_client_decode_descriptor(const char *desc_str,
+                            const ed25519_public_key_t *service_identity_pk,
+                            hs_descriptor_t **desc)
+{
+  int ret;
+  uint8_t subcredential[DIGEST256_LEN];
+  ed25519_public_key_t blinded_pubkey;
+  hs_client_service_authorization_t *client_auth = NULL;
+  curve25519_secret_key_t *client_auht_sk = NULL;
+
+  tor_assert(desc_str);
+  tor_assert(service_identity_pk);
+  tor_assert(desc);
+
+  /* Check if we have a client authorization for this service in the map. */
+  client_auth = find_client_auth(service_identity_pk);
+  if (client_auth) {
+    client_auht_sk = &client_auth->enc_seckey;
+  }
+
+  /* Create subcredential for this HS so that we can decrypt */
+  {
+    uint64_t current_time_period = hs_get_time_period_num(0);
+    hs_build_blinded_pubkey(service_identity_pk, NULL, 0, current_time_period,
+                            &blinded_pubkey);
+    hs_get_subcredential(service_identity_pk, &blinded_pubkey, subcredential);
+  }
+
+  /* Parse descriptor */
+  ret = hs_desc_decode_descriptor(desc_str, subcredential,
+                                  client_auht_sk, desc);
+  memwipe(subcredential, 0, sizeof(subcredential));
+  if (ret < 0) {
+    goto err;
+  }
+
+  /* Make sure the descriptor signing key cross certifies with the computed
+   * blinded key. Without this validation, anyone knowing the subcredential
+   * and onion address can forge a descriptor. */
+  tor_cert_t *cert = (*desc)->plaintext_data.signing_key_cert;
+  if (tor_cert_checksig(cert,
+                        &blinded_pubkey, approx_time()) < 0) {
+    log_warn(LD_GENERAL, "Descriptor signing key certificate signature "
+             "doesn't validate with computed blinded key: %s",
+             tor_cert_describe_signature_status(cert));
+    goto err;
+  }
+
+  return 0;
+ err:
+  return -1;
+}
+
+/* Return true iff there are at least one usable intro point in the service
+ * descriptor desc. */
+int
+hs_client_any_intro_points_usable(const ed25519_public_key_t *service_pk,
+                                  const hs_descriptor_t *desc)
+{
+  tor_assert(service_pk);
+  tor_assert(desc);
+
+  SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                          const hs_desc_intro_point_t *, ip) {
+    if (intro_point_is_usable(service_pk, ip)) {
+      goto usable;
+    }
+  } SMARTLIST_FOREACH_END(ip);
+
+  return 0;
+ usable:
+  return 1;
+}
+
+/** Launch a connection to a hidden service directory to fetch a hidden
+ * service descriptor using <b>identity_pk</b> to get the necessary keys.
+ *
+ * A hs_client_fetch_status_t code is returned. */
+int
+hs_client_refetch_hsdesc(const ed25519_public_key_t *identity_pk)
+{
+  hs_client_fetch_status_t status;
+
+  tor_assert(identity_pk);
+
+  if (!can_client_refetch_desc(identity_pk, &status)) {
+    return status;
+  }
+
+  /* Try to fetch the desc and if we encounter an unrecoverable error, mark
+   * the desc as unavailable for now. */
+  status = fetch_v3_desc(identity_pk);
+  if (fetch_status_should_close_socks(status)) {
+    close_all_socks_conns_waiting_for_desc(identity_pk, status,
+                                           END_STREAM_REASON_RESOLVEFAILED);
+    /* Remove HSDir fetch attempts so that we can retry later if the user
+     * wants us to regardless of if we closed any connections. */
+    purge_hid_serv_request(identity_pk);
+  }
+  return status;
+}
+
+/* This is called when we are trying to attach an AP connection to these
+ * hidden service circuits from connection_ap_handshake_attach_circuit().
+ * Return 0 on success, -1 for a transient error that is actions were
+ * triggered to recover or -2 for a permenent error where both circuits will
+ * marked for close.
+ *
+ * The following supports every hidden service version. */
+int
+hs_client_send_introduce1(origin_circuit_t *intro_circ,
+                          origin_circuit_t *rend_circ)
+{
+  return (intro_circ->hs_ident) ? send_introduce1(intro_circ, rend_circ) :
+                                  rend_client_send_introduction(intro_circ,
+                                                                rend_circ);
+}
+
+/* Called when the client circuit circ has been established. It can be either
+ * an introduction or rendezvous circuit. This function handles all hidden
+ * service versions. */
+void
+hs_client_circuit_has_opened(origin_circuit_t *circ)
+{
+  tor_assert(circ);
+
+  /* Handle both version. v2 uses rend_data and v3 uses the hs circuit
+   * identifier hs_ident. Can't be both. */
+  switch (TO_CIRCUIT(circ)->purpose) {
+  case CIRCUIT_PURPOSE_C_INTRODUCING:
+    if (circ->hs_ident) {
+      client_intro_circ_has_opened(circ);
+    } else {
+      rend_client_introcirc_has_opened(circ);
+    }
+    break;
+  case CIRCUIT_PURPOSE_C_ESTABLISH_REND:
+    if (circ->hs_ident) {
+      client_rendezvous_circ_has_opened(circ);
+    } else {
+      rend_client_rendcirc_has_opened(circ);
+    }
+    break;
+  default:
+    tor_assert_nonfatal_unreached();
+  }
+}
+
+/* Called when we receive a RENDEZVOUS_ESTABLISHED cell. Change the state of
+ * the circuit to CIRCUIT_PURPOSE_C_REND_READY. Return 0 on success else a
+ * negative value and the circuit marked for close. */
+int
+hs_client_receive_rendezvous_acked(origin_circuit_t *circ,
+                                   const uint8_t *payload, size_t payload_len)
+{
+  tor_assert(circ);
+  tor_assert(payload);
+
+  (void) payload_len;
+
+  if (TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_C_ESTABLISH_REND) {
+    log_warn(LD_PROTOCOL, "Got a RENDEZVOUS_ESTABLISHED but we were not "
+                          "expecting one. Closing circuit.");
+    goto err;
+  }
+
+  log_info(LD_REND, "Received an RENDEZVOUS_ESTABLISHED. This circuit is "
+                    "now ready for rendezvous.");
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_REND_READY);
+
+  /* Set timestamp_dirty, because circuit_expire_building expects it to
+   * specify when a circuit entered the _C_REND_READY state. */
+  TO_CIRCUIT(circ)->timestamp_dirty = time(NULL);
+
+  /* From a path bias point of view, this circuit is now successfully used.
+   * Waiting any longer opens us up to attacks from malicious hidden services.
+   * They could induce the client to attempt to connect to their hidden
+   * service and never reply to the client's rend requests */
+  pathbias_mark_use_success(circ);
+
+  /* If we already have the introduction circuit built, make sure we send
+   * the INTRODUCE cell _now_ */
+  connection_ap_attach_pending(1);
+
+  return 0;
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+  return -1;
+}
+
+#define client_service_authorization_free(auth)                      \
+  FREE_AND_NULL(hs_client_service_authorization_t,                   \
+                client_service_authorization_free_, (auth))
+
+static void
+client_service_authorization_free_(hs_client_service_authorization_t *auth)
+{
+  if (auth) {
+    memwipe(auth, 0, sizeof(*auth));
+  }
+  tor_free(auth);
+}
+
+/** Helper for digest256map_free. */
+static void
+client_service_authorization_free_void(void *auth)
+{
+  client_service_authorization_free_(auth);
+}
+
+static void
+client_service_authorization_free_all(void)
+{
+  if (!client_auths) {
+    return;
+  }
+  digest256map_free(client_auths, client_service_authorization_free_void);
+}
+
+/* Check if the auth key file name is valid or not. Return 1 if valid,
+ * otherwise return 0. */
+STATIC int
+auth_key_filename_is_valid(const char *filename)
+{
+  int ret = 1;
+  const char *valid_extension = ".auth_private";
+
+  tor_assert(filename);
+
+  /* The length of the filename must be greater than the length of the
+   * extension and the valid extension must be at the end of filename. */
+  if (!strcmpend(filename, valid_extension) &&
+      strlen(filename) != strlen(valid_extension)) {
+    ret = 1;
+  } else {
+    ret = 0;
+  }
+
+  return ret;
+}
+
+STATIC hs_client_service_authorization_t *
+parse_auth_file_content(const char *client_key_str)
+{
+  char *onion_address = NULL;
+  char *auth_type = NULL;
+  char *key_type = NULL;
+  char *seckey_b32 = NULL;
+  hs_client_service_authorization_t *auth = NULL;
+  smartlist_t *fields = smartlist_new();
+
+  tor_assert(client_key_str);
+
+  smartlist_split_string(fields, client_key_str, ":",
+                         SPLIT_SKIP_SPACE, 0);
+  /* Wrong number of fields. */
+  if (smartlist_len(fields) != 4) {
+    goto err;
+  }
+
+  onion_address = smartlist_get(fields, 0);
+  auth_type = smartlist_get(fields, 1);
+  key_type = smartlist_get(fields, 2);
+  seckey_b32 = smartlist_get(fields, 3);
+
+  /* Currently, the only supported auth type is "descriptor" and the only
+   * supported key type is "x25519". */
+  if (strcmp(auth_type, "descriptor") || strcmp(key_type, "x25519")) {
+    goto err;
+  }
+
+  if (strlen(seckey_b32) != BASE32_NOPAD_LEN(CURVE25519_PUBKEY_LEN)) {
+    log_warn(LD_REND, "Client authorization encoded base32 private key "
+                      "length is invalid: %s", seckey_b32);
+    goto err;
+  }
+
+  auth = tor_malloc_zero(sizeof(hs_client_service_authorization_t));
+  if (base32_decode((char *) auth->enc_seckey.secret_key,
+                    sizeof(auth->enc_seckey.secret_key),
+                    seckey_b32, strlen(seckey_b32)) < 0) {
+    goto err;
+  }
+  strncpy(auth->onion_address, onion_address, HS_SERVICE_ADDR_LEN_BASE32);
+
+  /* Success. */
+  goto done;
+
+ err:
+  client_service_authorization_free(auth);
+ done:
+  /* It is also a good idea to wipe the private key. */
+  if (seckey_b32) {
+    memwipe(seckey_b32, 0, strlen(seckey_b32));
+  }
+  tor_assert(fields);
+  SMARTLIST_FOREACH(fields, char *, s, tor_free(s));
+  smartlist_free(fields);
+  return auth;
+}
+
+/* From a set of <b>options</b>, setup every client authorization detail
+ * found. Return 0 on success or -1 on failure. If <b>validate_only</b>
+ * is set, parse, warn and return as normal, but don't actually change
+ * the configuration. */
+int
+hs_config_client_authorization(const or_options_t *options,
+                               int validate_only)
+{
+  int ret = -1;
+  digest256map_t *auths = digest256map_new();
+  char *key_dir = NULL;
+  smartlist_t *file_list = NULL;
+  char *client_key_str = NULL;
+  char *client_key_file_path = NULL;
+
+  tor_assert(options);
+
+  /* There is no client auth configured. We can just silently ignore this
+   * function. */
+  if (!options->ClientOnionAuthDir) {
+    ret = 0;
+    goto end;
+  }
+
+  key_dir = tor_strdup(options->ClientOnionAuthDir);
+
+  /* Make sure the directory exists and is private enough. */
+  if (check_private_dir(key_dir, 0, options->User) < 0) {
+    goto end;
+  }
+
+  file_list = tor_listdir(key_dir);
+  if (file_list == NULL) {
+    log_warn(LD_REND, "Client authorization key directory %s can't be listed.",
+             key_dir);
+    goto end;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(file_list, char *, filename) {
+
+    hs_client_service_authorization_t *auth = NULL;
+    ed25519_public_key_t identity_pk;
+    log_info(LD_REND, "Loading a client authorization key file %s...",
+             filename);
+
+    if (!auth_key_filename_is_valid(filename)) {
+      log_notice(LD_REND, "Client authorization unrecognized filename %s. "
+                          "File must end in .auth_private. Ignoring.",
+                 filename);
+      continue;
+    }
+
+    /* Create a full path for a file. */
+    client_key_file_path = hs_path_from_filename(key_dir, filename);
+    client_key_str = read_file_to_str(client_key_file_path, 0, NULL);
+    /* Free the file path immediately after using it. */
+    tor_free(client_key_file_path);
+
+    /* If we cannot read the file, continue with the next file. */
+    if (!client_key_str) {
+      log_warn(LD_REND, "The file %s cannot be read.", filename);
+      continue;
+    }
+
+    auth = parse_auth_file_content(client_key_str);
+    /* Free immediately after using it. */
+    tor_free(client_key_str);
+
+    if (auth) {
+      /* Parse the onion address to get an identity public key and use it
+       * as a key of global map in the future. */
+      if (hs_parse_address(auth->onion_address, &identity_pk,
+                           NULL, NULL) < 0) {
+        log_warn(LD_REND, "The onion address \"%s\" is invalid in "
+                          "file %s", filename, auth->onion_address);
+        client_service_authorization_free(auth);
+        continue;
+      }
+
+      if (digest256map_get(auths, identity_pk.pubkey)) {
+        log_warn(LD_REND, "Duplicate authorization for the same hidden "
+                          "service address %s.",
+                 safe_str_client_opts(options, auth->onion_address));
+        client_service_authorization_free(auth);
+        goto end;
+      }
+
+      digest256map_set(auths, identity_pk.pubkey, auth);
+      log_info(LD_REND, "Loaded a client authorization key file %s.",
+               filename);
+    }
+  } SMARTLIST_FOREACH_END(filename);
+
+  /* Success. */
+  ret = 0;
+
+ end:
+  tor_free(key_dir);
+  tor_free(client_key_str);
+  tor_free(client_key_file_path);
+  if (file_list) {
+    SMARTLIST_FOREACH(file_list, char *, s, tor_free(s));
+    smartlist_free(file_list);
+  }
+
+  if (!validate_only && ret == 0) {
+    client_service_authorization_free_all();
+    client_auths = auths;
+  } else {
+    digest256map_free(auths, client_service_authorization_free_void);
+  }
+
+  return ret;
+}
+
+/* This is called when a descriptor has arrived following a fetch request and
+ * has been stored in the client cache. Every entry connection that matches
+ * the service identity key in the ident will get attached to the hidden
+ * service circuit. */
+void
+hs_client_desc_has_arrived(const hs_ident_dir_conn_t *ident)
+{
+  time_t now = time(NULL);
+  smartlist_t *conns = NULL;
+
+  tor_assert(ident);
+
+  conns = connection_list_by_type_state(CONN_TYPE_AP,
+                                        AP_CONN_STATE_RENDDESC_WAIT);
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+    const hs_descriptor_t *desc;
+    entry_connection_t *entry_conn = TO_ENTRY_CONN(base_conn);
+    const edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(entry_conn);
+
+    /* Only consider the entry connections that matches the service for which
+     * we just fetched its descriptor. */
+    if (!edge_conn->hs_ident ||
+        !ed25519_pubkey_eq(&ident->identity_pk,
+                           &edge_conn->hs_ident->identity_pk)) {
+      continue;
+    }
+    assert_connection_ok(base_conn, now);
+
+    /* We were just called because we stored the descriptor for this service
+     * so not finding a descriptor means we have a bigger problem. */
+    desc = hs_cache_lookup_as_client(&ident->identity_pk);
+    if (BUG(desc == NULL)) {
+      goto end;
+    }
+
+    if (!hs_client_any_intro_points_usable(&ident->identity_pk, desc)) {
+      log_info(LD_REND, "Hidden service descriptor is unusable. "
+                        "Closing streams.");
+      connection_mark_unattached_ap(entry_conn,
+                                    END_STREAM_REASON_RESOLVEFAILED);
+      /* We are unable to use the descriptor so remove the directory request
+       * from the cache so the next connection can try again. */
+      note_connection_attempt_succeeded(edge_conn->hs_ident);
+      continue;
+    }
+
+    log_info(LD_REND, "Descriptor has arrived. Launching circuits.");
+
+    /* Mark connection as waiting for a circuit since we do have a usable
+     * descriptor now. */
+    mark_conn_as_waiting_for_circuit(base_conn, now);
+  } SMARTLIST_FOREACH_END(base_conn);
+
+ end:
+  /* We don't have ownership of the objects in this list. */
+  smartlist_free(conns);
+}
+
+/* Return a newly allocated extend_info_t for a randomly chosen introduction
+ * point for the given edge connection identifier ident. Return NULL if we
+ * can't pick any usable introduction points. */
+extend_info_t *
+hs_client_get_random_intro_from_edge(const edge_connection_t *edge_conn)
+{
+  tor_assert(edge_conn);
+
+  return (edge_conn->hs_ident) ?
+    client_get_random_intro(&edge_conn->hs_ident->identity_pk) :
+    rend_client_get_random_intro(edge_conn->rend_data);
+}
+
+/* Called when get an INTRODUCE_ACK cell on the introduction circuit circ.
+ * Return 0 on success else a negative value is returned. The circuit will be
+ * closed or reuse to extend again to another intro point. */
+int
+hs_client_receive_introduce_ack(origin_circuit_t *circ,
+                                const uint8_t *payload, size_t payload_len)
+{
+  int ret = -1;
+
+  tor_assert(circ);
+  tor_assert(payload);
+
+  if (TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT) {
+    log_warn(LD_PROTOCOL, "Unexpected INTRODUCE_ACK on circuit %u.",
+             (unsigned int) TO_CIRCUIT(circ)->n_circ_id);
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+    goto end;
+  }
+
+  ret = (circ->hs_ident) ? handle_introduce_ack(circ, payload, payload_len) :
+                           rend_client_introduction_acked(circ, payload,
+                                                          payload_len);
+  /* For path bias: This circuit was used successfully. NACK or ACK counts. */
+  pathbias_mark_use_success(circ);
+
+ end:
+  return ret;
+}
+
+/* Called when get a RENDEZVOUS2 cell on the rendezvous circuit circ.  Return
+ * 0 on success else a negative value is returned. The circuit will be closed
+ * on error. */
+int
+hs_client_receive_rendezvous2(origin_circuit_t *circ,
+                              const uint8_t *payload, size_t payload_len)
+{
+  int ret = -1;
+
+  tor_assert(circ);
+  tor_assert(payload);
+
+  /* Circuit can possibly be in both state because we could receive a
+   * RENDEZVOUS2 cell before the INTRODUCE_ACK has been received. */
+  if (TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_C_REND_READY &&
+      TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED) {
+    log_warn(LD_PROTOCOL, "Unexpected RENDEZVOUS2 cell on circuit %u. "
+                          "Closing circuit.",
+             (unsigned int) TO_CIRCUIT(circ)->n_circ_id);
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+    goto end;
+  }
+
+  log_info(LD_REND, "Got RENDEZVOUS2 cell from hidden service on circuit %u.",
+           TO_CIRCUIT(circ)->n_circ_id);
+
+  ret = (circ->hs_ident) ? handle_rendezvous2(circ, payload, payload_len) :
+                           rend_client_receive_rendezvous(circ, payload,
+                                                          payload_len);
+ end:
+  return ret;
+}
+
+/* Extend the introduction circuit circ to another valid introduction point
+ * for the hidden service it is trying to connect to, or mark it and launch a
+ * new circuit if we can't extend it.  Return 0 on success or possible
+ * success. Return -1 and mark the introduction circuit for close on permanent
+ * failure.
+ *
+ * On failure, the caller is responsible for marking the associated rendezvous
+ * circuit for close. */
+int
+hs_client_reextend_intro_circuit(origin_circuit_t *circ)
+{
+  int ret = -1;
+  extend_info_t *ei;
+
+  tor_assert(circ);
+
+  ei = (circ->hs_ident) ?
+    client_get_random_intro(&circ->hs_ident->identity_pk) :
+    rend_client_get_random_intro(circ->rend_data);
+  if (ei == NULL) {
+    log_warn(LD_REND, "No usable introduction points left. Closing.");
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
+    goto end;
+  }
+
+  if (circ->remaining_relay_early_cells) {
+    log_info(LD_REND, "Re-extending circ %u, this time to %s.",
+             (unsigned int) TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(extend_info_describe(ei)));
+    ret = circuit_extend_to_new_exit(circ, ei);
+    if (ret == 0) {
+      /* We were able to extend so update the timestamp so we avoid expiring
+       * this circuit too early. The intro circuit is short live so the
+       * linkability issue is minimized, we just need the circuit to hold a
+       * bit longer so we can introduce. */
+      TO_CIRCUIT(circ)->timestamp_dirty = time(NULL);
+    }
+  } else {
+    log_info(LD_REND, "Closing intro circ %u (out of RELAY_EARLY cells).",
+             (unsigned int) TO_CIRCUIT(circ)->n_circ_id);
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED);
+    /* connection_ap_handshake_attach_circuit will launch a new intro circ. */
+    ret = 0;
+  }
+
+ end:
+  extend_info_free(ei);
+  return ret;
+}
+
+/* Close all client introduction circuits related to the given descriptor.
+ * This is called with a descriptor that is about to get replaced in the
+ * client cache.
+ *
+ * Even though the introduction point might be exactly the same, we'll rebuild
+ * them if needed but the odds are very low that an existing matching
+ * introduction circuit exists at that stage. */
+void
+hs_client_close_intro_circuits_from_desc(const hs_descriptor_t *desc)
+{
+  origin_circuit_t *ocirc = NULL;
+
+  tor_assert(desc);
+
+  /* We iterate over all client intro circuits because they aren't kept in the
+   * HS circuitmap. That is probably something we want to do one day. */
+  while ((ocirc = circuit_get_next_intro_circ(ocirc, true))) {
+    if (ocirc->hs_ident == NULL) {
+      /* Not a v3 circuit, ignore it. */
+      continue;
+    }
+
+    /* Does it match any IP in the given descriptor? If not, ignore. */
+    if (find_desc_intro_point_by_ident(ocirc->hs_ident, desc) == NULL) {
+      continue;
+    }
+
+    /* We have a match. Close the circuit as consider it expired. */
+    circuit_mark_for_close(TO_CIRCUIT(ocirc), END_CIRC_REASON_FINISHED);
+  }
+}
+
+/* Release all the storage held by the client subsystem. */
+void
+hs_client_free_all(void)
+{
+  /* Purge the hidden service request cache. */
+  hs_purge_last_hid_serv_requests();
+  client_service_authorization_free_all();
+}
+
+/* Purge all potentially remotely-detectable state held in the hidden
+ * service client code. Called on SIGNAL NEWNYM. */
+void
+hs_client_purge_state(void)
+{
+  /* v2 subsystem. */
+  rend_client_purge_state();
+
+  /* Cancel all descriptor fetches. Do this first so once done we are sure
+   * that our descriptor cache won't modified. */
+  cancel_descriptor_fetches();
+  /* Purge the introduction point state cache. */
+  hs_cache_client_intro_state_purge();
+  /* Purge the descriptor cache. */
+  hs_cache_purge_as_client();
+  /* Purge the last hidden service request cache. */
+  hs_purge_last_hid_serv_requests();
+
+  log_info(LD_REND, "Hidden service client state has been purged.");
+}
+
+/* Called when our directory information has changed. */
+void
+hs_client_dir_info_changed(void)
+{
+  /* We have possibly reached the minimum directory information or new
+   * consensus so retry all pending SOCKS connection in
+   * AP_CONN_STATE_RENDDESC_WAIT state in order to fetch the descriptor. */
+  retry_all_socks_conn_waiting_for_desc();
+}
+
+#ifdef TOR_UNIT_TESTS
+
+STATIC digest256map_t *
+get_hs_client_auths_map(void)
+{
+  return client_auths;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/feature/hs/hs_client.h b/src/feature/hs/hs_client.h
new file mode 100644
index 0000000000..dadfa024b8
--- /dev/null
+++ b/src/feature/hs/hs_client.h
@@ -0,0 +1,119 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_client.h
+ * \brief Header file containing client data for the HS subsytem.
+ **/
+
+#ifndef TOR_HS_CLIENT_H
+#define TOR_HS_CLIENT_H
+
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "feature/hs/hs_descriptor.h"
+#include "feature/hs/hs_ident.h"
+
+/* Status code of a descriptor fetch request. */
+typedef enum {
+  /* Something internally went wrong. */
+  HS_CLIENT_FETCH_ERROR        = -1,
+  /* The fetch request has been launched successfully. */
+  HS_CLIENT_FETCH_LAUNCHED     = 0,
+  /* We already have a usable descriptor. No fetch. */
+  HS_CLIENT_FETCH_HAVE_DESC    = 1,
+  /* No more HSDir available to query. */
+  HS_CLIENT_FETCH_NO_HSDIRS    = 2,
+  /* The fetch request is not allowed. */
+  HS_CLIENT_FETCH_NOT_ALLOWED  = 3,
+  /* We are missing information to be able to launch a request. */
+  HS_CLIENT_FETCH_MISSING_INFO = 4,
+  /* There is a pending fetch for the requested service. */
+  HS_CLIENT_FETCH_PENDING      = 5,
+} hs_client_fetch_status_t;
+
+/** Client-side configuration of authorization for a service. */
+typedef struct hs_client_service_authorization_t {
+  /* An curve25519 secret key used to compute decryption keys that
+   * allow the client to decrypt the hidden service descriptor. */
+  curve25519_secret_key_t enc_seckey;
+
+  /* An onion address that is used to connect to the onion service. */
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32+1];
+} hs_client_service_authorization_t;
+
+void hs_client_note_connection_attempt_succeeded(
+                                       const edge_connection_t *conn);
+
+int hs_client_decode_descriptor(
+                     const char *desc_str,
+                     const ed25519_public_key_t *service_identity_pk,
+                     hs_descriptor_t **desc);
+int hs_client_any_intro_points_usable(const ed25519_public_key_t *service_pk,
+                                      const hs_descriptor_t *desc);
+int hs_client_refetch_hsdesc(const ed25519_public_key_t *identity_pk);
+void hs_client_dir_info_changed(void);
+
+int hs_client_send_introduce1(origin_circuit_t *intro_circ,
+                              origin_circuit_t *rend_circ);
+
+void hs_client_circuit_has_opened(origin_circuit_t *circ);
+
+int hs_client_receive_rendezvous_acked(origin_circuit_t *circ,
+                                       const uint8_t *payload,
+                                       size_t payload_len);
+int hs_client_receive_introduce_ack(origin_circuit_t *circ,
+                                    const uint8_t *payload,
+                                    size_t payload_len);
+int hs_client_receive_rendezvous2(origin_circuit_t *circ,
+                                  const uint8_t *payload,
+                                  size_t payload_len);
+
+void hs_client_desc_has_arrived(const hs_ident_dir_conn_t *ident);
+
+extend_info_t *hs_client_get_random_intro_from_edge(
+                                          const edge_connection_t *edge_conn);
+
+int hs_config_client_authorization(const or_options_t *options,
+                                   int validate_only);
+
+int hs_client_reextend_intro_circuit(origin_circuit_t *circ);
+void hs_client_close_intro_circuits_from_desc(const hs_descriptor_t *desc);
+
+void hs_client_purge_state(void);
+
+void hs_client_free_all(void);
+
+#ifdef HS_CLIENT_PRIVATE
+
+STATIC int auth_key_filename_is_valid(const char *filename);
+
+STATIC hs_client_service_authorization_t *
+parse_auth_file_content(const char *client_key_str);
+
+STATIC routerstatus_t *
+pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk);
+
+STATIC extend_info_t *
+client_get_random_intro(const ed25519_public_key_t *service_pk);
+
+STATIC extend_info_t *
+desc_intro_point_to_extend_info(const hs_desc_intro_point_t *ip);
+
+STATIC int handle_rendezvous2(origin_circuit_t *circ, const uint8_t *payload,
+                              size_t payload_len);
+
+MOCK_DECL(STATIC hs_client_fetch_status_t,
+          fetch_v3_desc, (const ed25519_public_key_t *onion_identity_pk));
+
+STATIC void retry_all_socks_conn_waiting_for_desc(void);
+
+#ifdef TOR_UNIT_TESTS
+
+STATIC digest256map_t *get_hs_client_auths_map(void);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(HS_CLIENT_PRIVATE) */
+
+#endif /* !defined(TOR_HS_CLIENT_H) */
+
diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c
new file mode 100644
index 0000000000..ebe49f09a5
--- /dev/null
+++ b/src/feature/hs/hs_common.c
@@ -0,0 +1,1829 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_common.c
+ * \brief Contains code shared between different HS protocol version as well
+ *        as useful data structures and accessors used by other subsystems.
+ *        The rendcommon.c should only contains code relating to the v2
+ *        protocol.
+ **/
+
+#define HS_COMMON_PRIVATE
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/policies.h"
+#include "feature/dirauth/shared_random_state.h"
+#include "feature/hs/hs_cache.h"
+#include "feature/hs/hs_circuitmap.h"
+#include "feature/hs/hs_client.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_ident.h"
+#include "feature/hs/hs_service.h"
+#include "feature/hs_common/shared_random_client.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/rend/rendservice.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+
+#include "core/or/edge_connection_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+/* Trunnel */
+#include "trunnel/ed25519_cert.h"
+
+/* Ed25519 Basepoint value. Taken from section 5 of
+ * https://tools.ietf.org/html/draft-josefsson-eddsa-ed25519-03 */
+static const char *str_ed25519_basepoint =
+  "(15112221349535400772501151409588531511"
+  "454012693041857206046113283949847762202, "
+  "463168356949264781694283940034751631413"
+  "07993866256225615783033603165251855960)";
+
+#ifdef HAVE_SYS_UN_H
+
+/** Given <b>ports</b>, a smarlist containing rend_service_port_config_t,
+ * add the given <b>p</b>, a AF_UNIX port to the list. Return 0 on success
+ * else return -ENOSYS if AF_UNIX is not supported (see function in the
+ * #else statement below). */
+static int
+add_unix_port(smartlist_t *ports, rend_service_port_config_t *p)
+{
+  tor_assert(ports);
+  tor_assert(p);
+  tor_assert(p->is_unix_addr);
+
+  smartlist_add(ports, p);
+  return 0;
+}
+
+/** Given <b>conn</b> set it to use the given port <b>p</b> values. Return 0
+ * on success else return -ENOSYS if AF_UNIX is not supported (see function
+ * in the #else statement below). */
+static int
+set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p)
+{
+  tor_assert(conn);
+  tor_assert(p);
+  tor_assert(p->is_unix_addr);
+
+  conn->base_.socket_family = AF_UNIX;
+  tor_addr_make_unspec(&conn->base_.addr);
+  conn->base_.port = 1;
+  conn->base_.address = tor_strdup(p->unix_addr);
+  return 0;
+}
+
+#else /* !(defined(HAVE_SYS_UN_H)) */
+
+static int
+set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p)
+{
+  (void) conn;
+  (void) p;
+  return -ENOSYS;
+}
+
+static int
+add_unix_port(smartlist_t *ports, rend_service_port_config_t *p)
+{
+  (void) ports;
+  (void) p;
+  return -ENOSYS;
+}
+
+#endif /* defined(HAVE_SYS_UN_H) */
+
+/* Helper function: The key is a digest that we compare to a node_t object
+ * current hsdir_index. */
+static int
+compare_digest_to_fetch_hsdir_index(const void *_key, const void **_member)
+{
+  const char *key = _key;
+  const node_t *node = *_member;
+  return tor_memcmp(key, node->hsdir_index.fetch, DIGEST256_LEN);
+}
+
+/* Helper function: The key is a digest that we compare to a node_t object
+ * next hsdir_index. */
+static int
+compare_digest_to_store_first_hsdir_index(const void *_key,
+                                          const void **_member)
+{
+  const char *key = _key;
+  const node_t *node = *_member;
+  return tor_memcmp(key, node->hsdir_index.store_first, DIGEST256_LEN);
+}
+
+/* Helper function: The key is a digest that we compare to a node_t object
+ * next hsdir_index. */
+static int
+compare_digest_to_store_second_hsdir_index(const void *_key,
+                                          const void **_member)
+{
+  const char *key = _key;
+  const node_t *node = *_member;
+  return tor_memcmp(key, node->hsdir_index.store_second, DIGEST256_LEN);
+}
+
+/* Helper function: Compare two node_t objects current hsdir_index. */
+static int
+compare_node_fetch_hsdir_index(const void **a, const void **b)
+{
+  const node_t *node1= *a;
+  const node_t *node2 = *b;
+  return tor_memcmp(node1->hsdir_index.fetch,
+                    node2->hsdir_index.fetch,
+                    DIGEST256_LEN);
+}
+
+/* Helper function: Compare two node_t objects next hsdir_index. */
+static int
+compare_node_store_first_hsdir_index(const void **a, const void **b)
+{
+  const node_t *node1= *a;
+  const node_t *node2 = *b;
+  return tor_memcmp(node1->hsdir_index.store_first,
+                    node2->hsdir_index.store_first,
+                    DIGEST256_LEN);
+}
+
+/* Helper function: Compare two node_t objects next hsdir_index. */
+static int
+compare_node_store_second_hsdir_index(const void **a, const void **b)
+{
+  const node_t *node1= *a;
+  const node_t *node2 = *b;
+  return tor_memcmp(node1->hsdir_index.store_second,
+                    node2->hsdir_index.store_second,
+                    DIGEST256_LEN);
+}
+
+/* Allocate and return a string containing the path to filename in directory.
+ * This function will never return NULL. The caller must free this path. */
+char *
+hs_path_from_filename(const char *directory, const char *filename)
+{
+  char *file_path = NULL;
+
+  tor_assert(directory);
+  tor_assert(filename);
+
+  tor_asprintf(&file_path, "%s%s%s", directory, PATH_SEPARATOR, filename);
+  return file_path;
+}
+
+/* Make sure that the directory for <b>service</b> is private, using the config
+ * <b>username</b>.
+ * If <b>create</b> is true:
+ *  - if the directory exists, change permissions if needed,
+ *  - if the directory does not exist, create it with the correct permissions.
+ * If <b>create</b> is false:
+ *  - if the directory exists, check permissions,
+ *  - if the directory does not exist, check if we think we can create it.
+ * Return 0 on success, -1 on failure. */
+int
+hs_check_service_private_dir(const char *username, const char *path,
+                             unsigned int dir_group_readable,
+                             unsigned int create)
+{
+  cpd_check_t check_opts = CPD_NONE;
+
+  tor_assert(path);
+
+  if (create) {
+    check_opts |= CPD_CREATE;
+  } else {
+    check_opts |= CPD_CHECK_MODE_ONLY;
+    check_opts |= CPD_CHECK;
+  }
+  if (dir_group_readable) {
+    check_opts |= CPD_GROUP_READ;
+  }
+  /* Check/create directory */
+  if (check_private_dir(path, check_opts, username) < 0) {
+    return -1;
+  }
+  return 0;
+}
+
+/* Default, minimum, and maximum values for the maximum rendezvous failures
+ * consensus parameter. */
+#define MAX_REND_FAILURES_DEFAULT 2
+#define MAX_REND_FAILURES_MIN 1
+#define MAX_REND_FAILURES_MAX 10
+
+/** How many times will a hidden service operator attempt to connect to
+ * a requested rendezvous point before giving up? */
+int
+hs_get_service_max_rend_failures(void)
+{
+  return networkstatus_get_param(NULL, "hs_service_max_rdv_failures",
+                                 MAX_REND_FAILURES_DEFAULT,
+                                 MAX_REND_FAILURES_MIN,
+                                 MAX_REND_FAILURES_MAX);
+}
+
+/** Get the default HS time period length in minutes from the consensus. */
+STATIC uint64_t
+get_time_period_length(void)
+{
+  /* If we are on a test network, make the time period smaller than normal so
+     that we actually see it rotate. Specifically, make it the same length as
+     an SRV protocol run. */
+  if (get_options()->TestingTorNetwork) {
+    unsigned run_duration = sr_state_get_protocol_run_duration();
+    /* An SRV run should take more than a minute (it's 24 rounds) */
+    tor_assert_nonfatal(run_duration > 60);
+    /* Turn it from seconds to minutes before returning: */
+    return sr_state_get_protocol_run_duration() / 60;
+  }
+
+  int32_t time_period_length = networkstatus_get_param(NULL, "hsdir_interval",
+                                             HS_TIME_PERIOD_LENGTH_DEFAULT,
+                                             HS_TIME_PERIOD_LENGTH_MIN,
+                                             HS_TIME_PERIOD_LENGTH_MAX);
+  /* Make sure it's a positive value. */
+  tor_assert(time_period_length > 0);
+  /* uint64_t will always be able to contain a positive int32_t */
+  return (uint64_t) time_period_length;
+}
+
+/** Get the HS time period number at time <b>now</b>. If <b>now</b> is not set,
+ *  we try to get the time ourselves from a live consensus. */
+uint64_t
+hs_get_time_period_num(time_t now)
+{
+  uint64_t time_period_num;
+  time_t current_time;
+
+  /* If no time is specified, set current time based on consensus time, and
+   * only fall back to system time if that fails. */
+  if (now != 0) {
+    current_time = now;
+  } else {
+    networkstatus_t *ns = networkstatus_get_live_consensus(approx_time());
+    current_time = ns ? ns->valid_after : approx_time();
+  }
+
+  /* Start by calculating minutes since the epoch */
+  uint64_t time_period_length = get_time_period_length();
+  uint64_t minutes_since_epoch = current_time / 60;
+
+  /* Apply the rotation offset as specified by prop224 (section
+   * [TIME-PERIODS]), so that new time periods synchronize nicely with SRV
+   * publication */
+  unsigned int time_period_rotation_offset = sr_state_get_phase_duration();
+  time_period_rotation_offset /= 60; /* go from seconds to minutes */
+  tor_assert(minutes_since_epoch > time_period_rotation_offset);
+  minutes_since_epoch -= time_period_rotation_offset;
+
+  /* Calculate the time period */
+  time_period_num = minutes_since_epoch / time_period_length;
+  return time_period_num;
+}
+
+/** Get the number of the _upcoming_ HS time period, given that the current
+ *  time is <b>now</b>. If <b>now</b> is not set, we try to get the time from a
+ *  live consensus. */
+uint64_t
+hs_get_next_time_period_num(time_t now)
+{
+  return hs_get_time_period_num(now) + 1;
+}
+
+/* Get the number of the _previous_ HS time period, given that the current time
+ * is <b>now</b>. If <b>now</b> is not set, we try to get the time from a live
+ * consensus. */
+uint64_t
+hs_get_previous_time_period_num(time_t now)
+{
+  return hs_get_time_period_num(now) - 1;
+}
+
+/* Return the start time of the upcoming time period based on <b>now</b>. If
+   <b>now</b> is not set, we try to get the time ourselves from a live
+   consensus. */
+time_t
+hs_get_start_time_of_next_time_period(time_t now)
+{
+  uint64_t time_period_length = get_time_period_length();
+
+  /* Get start time of next time period */
+  uint64_t next_time_period_num = hs_get_next_time_period_num(now);
+  uint64_t start_of_next_tp_in_mins = next_time_period_num *time_period_length;
+
+  /* Apply rotation offset as specified by prop224 section [TIME-PERIODS] */
+  unsigned int time_period_rotation_offset = sr_state_get_phase_duration();
+  return (time_t)(start_of_next_tp_in_mins * 60 + time_period_rotation_offset);
+}
+
+/* Create a new rend_data_t for a specific given <b>version</b>.
+ * Return a pointer to the newly allocated data structure. */
+static rend_data_t *
+rend_data_alloc(uint32_t version)
+{
+  rend_data_t *rend_data = NULL;
+
+  switch (version) {
+  case HS_VERSION_TWO:
+  {
+    rend_data_v2_t *v2 = tor_malloc_zero(sizeof(*v2));
+    v2->base_.version = HS_VERSION_TWO;
+    v2->base_.hsdirs_fp = smartlist_new();
+    rend_data = &v2->base_;
+    break;
+  }
+  default:
+    tor_assert(0);
+    break;
+  }
+
+  return rend_data;
+}
+
+/** Free all storage associated with <b>data</b> */
+void
+rend_data_free_(rend_data_t *data)
+{
+  if (!data) {
+    return;
+  }
+  /* By using our allocation function, this should always be set. */
+  tor_assert(data->hsdirs_fp);
+  /* Cleanup the HSDir identity digest. */
+  SMARTLIST_FOREACH(data->hsdirs_fp, char *, d, tor_free(d));
+  smartlist_free(data->hsdirs_fp);
+  /* Depending on the version, cleanup. */
+  switch (data->version) {
+  case HS_VERSION_TWO:
+  {
+    rend_data_v2_t *v2_data = TO_REND_DATA_V2(data);
+    tor_free(v2_data);
+    break;
+  }
+  default:
+    tor_assert(0);
+  }
+}
+
+/* Allocate and return a deep copy of <b>data</b>. */
+rend_data_t *
+rend_data_dup(const rend_data_t *data)
+{
+  rend_data_t *data_dup = NULL;
+  smartlist_t *hsdirs_fp = smartlist_new();
+
+  tor_assert(data);
+  tor_assert(data->hsdirs_fp);
+
+  SMARTLIST_FOREACH(data->hsdirs_fp, char *, fp,
+                    smartlist_add(hsdirs_fp, tor_memdup(fp, DIGEST_LEN)));
+
+  switch (data->version) {
+  case HS_VERSION_TWO:
+  {
+    rend_data_v2_t *v2_data = tor_memdup(TO_REND_DATA_V2(data),
+                                         sizeof(*v2_data));
+    data_dup = &v2_data->base_;
+    data_dup->hsdirs_fp = hsdirs_fp;
+    break;
+  }
+  default:
+    tor_assert(0);
+    break;
+  }
+
+  return data_dup;
+}
+
+/* Compute the descriptor ID for each HS descriptor replica and save them. A
+ * valid onion address must be present in the <b>rend_data</b>.
+ *
+ * Return 0 on success else -1. */
+static int
+compute_desc_id(rend_data_t *rend_data)
+{
+  int ret = 0;
+  unsigned replica;
+  time_t now = time(NULL);
+
+  tor_assert(rend_data);
+
+  switch (rend_data->version) {
+  case HS_VERSION_TWO:
+  {
+    rend_data_v2_t *v2_data = TO_REND_DATA_V2(rend_data);
+    /* Compute descriptor ID for each replicas. */
+    for (replica = 0; replica < ARRAY_LENGTH(v2_data->descriptor_id);
+         replica++) {
+      ret = rend_compute_v2_desc_id(v2_data->descriptor_id[replica],
+                                    v2_data->onion_address,
+                                    v2_data->descriptor_cookie,
+                                    now, replica);
+      if (ret < 0) {
+        goto end;
+      }
+    }
+    break;
+  }
+  default:
+    tor_assert(0);
+  }
+
+ end:
+  return ret;
+}
+
+/* Allocate and initialize a rend_data_t object for a service using the
+ * provided arguments. All arguments are optional (can be NULL), except from
+ * <b>onion_address</b> which MUST be set. The <b>pk_digest</b> is the hash of
+ * the service private key. The <b>cookie</b> is the rendezvous cookie and
+ * <b>auth_type</b> is which authentiation this service is configured with.
+ *
+ * Return a valid rend_data_t pointer. This only returns a version 2 object of
+ * rend_data_t. */
+rend_data_t *
+rend_data_service_create(const char *onion_address, const char *pk_digest,
+                         const uint8_t *cookie, rend_auth_type_t auth_type)
+{
+  /* Create a rend_data_t object for version 2. */
+  rend_data_t *rend_data = rend_data_alloc(HS_VERSION_TWO);
+  rend_data_v2_t *v2= TO_REND_DATA_V2(rend_data);
+
+  /* We need at least one else the call is wrong. */
+  tor_assert(onion_address != NULL);
+
+  if (pk_digest) {
+    memcpy(v2->rend_pk_digest, pk_digest, sizeof(v2->rend_pk_digest));
+  }
+  if (cookie) {
+    memcpy(rend_data->rend_cookie, cookie, sizeof(rend_data->rend_cookie));
+  }
+
+  strlcpy(v2->onion_address, onion_address, sizeof(v2->onion_address));
+  v2->auth_type = auth_type;
+
+  return rend_data;
+}
+
+/* Allocate and initialize a rend_data_t object for a client request using the
+ * given arguments. Either an onion address or a descriptor ID is needed. Both
+ * can be given but in this case only the onion address will be used to make
+ * the descriptor fetch. The <b>cookie</b> is the rendezvous cookie and
+ * <b>auth_type</b> is which authentiation the service is configured with.
+ *
+ * Return a valid rend_data_t pointer or NULL on error meaning the
+ * descriptor IDs couldn't be computed from the given data. */
+rend_data_t *
+rend_data_client_create(const char *onion_address, const char *desc_id,
+                        const char *cookie, rend_auth_type_t auth_type)
+{
+  /* Create a rend_data_t object for version 2. */
+  rend_data_t *rend_data = rend_data_alloc(HS_VERSION_TWO);
+  rend_data_v2_t *v2= TO_REND_DATA_V2(rend_data);
+
+  /* We need at least one else the call is wrong. */
+  tor_assert(onion_address != NULL || desc_id != NULL);
+
+  if (cookie) {
+    memcpy(v2->descriptor_cookie, cookie, sizeof(v2->descriptor_cookie));
+  }
+  if (desc_id) {
+    memcpy(v2->desc_id_fetch, desc_id, sizeof(v2->desc_id_fetch));
+  }
+  if (onion_address) {
+    strlcpy(v2->onion_address, onion_address, sizeof(v2->onion_address));
+    if (compute_desc_id(rend_data) < 0) {
+      goto error;
+    }
+  }
+
+  v2->auth_type = auth_type;
+
+  return rend_data;
+
+ error:
+  rend_data_free(rend_data);
+  return NULL;
+}
+
+/* Return the onion address from the rend data. Depending on the version,
+ * the size of the address can vary but it's always NUL terminated. */
+const char *
+rend_data_get_address(const rend_data_t *rend_data)
+{
+  tor_assert(rend_data);
+
+  switch (rend_data->version) {
+  case HS_VERSION_TWO:
+    return TO_REND_DATA_V2(rend_data)->onion_address;
+  default:
+    /* We should always have a supported version. */
+    tor_assert_unreached();
+  }
+}
+
+/* Return the descriptor ID for a specific replica number from the rend
+ * data. The returned data is a binary digest and depending on the version its
+ * size can vary. The size of the descriptor ID is put in <b>len_out</b> if
+ * non NULL. */
+const char *
+rend_data_get_desc_id(const rend_data_t *rend_data, uint8_t replica,
+                      size_t *len_out)
+{
+  tor_assert(rend_data);
+
+  switch (rend_data->version) {
+  case HS_VERSION_TWO:
+    tor_assert(replica < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS);
+    if (len_out) {
+      *len_out = DIGEST_LEN;
+    }
+    return TO_REND_DATA_V2(rend_data)->descriptor_id[replica];
+  default:
+    /* We should always have a supported version. */
+    tor_assert_unreached();
+  }
+}
+
+/* Return the public key digest using the given <b>rend_data</b>. The size of
+ * the digest is put in <b>len_out</b> (if set) which can differ depending on
+ * the version. */
+const uint8_t *
+rend_data_get_pk_digest(const rend_data_t *rend_data, size_t *len_out)
+{
+  tor_assert(rend_data);
+
+  switch (rend_data->version) {
+  case HS_VERSION_TWO:
+  {
+    const rend_data_v2_t *v2_data = TO_REND_DATA_V2(rend_data);
+    if (len_out) {
+      *len_out = sizeof(v2_data->rend_pk_digest);
+    }
+    return (const uint8_t *) v2_data->rend_pk_digest;
+  }
+  default:
+    /* We should always have a supported version. */
+    tor_assert_unreached();
+  }
+}
+
+/* Using the given time period number, compute the disaster shared random
+ * value and put it in srv_out. It MUST be at least DIGEST256_LEN bytes. */
+static void
+compute_disaster_srv(uint64_t time_period_num, uint8_t *srv_out)
+{
+  crypto_digest_t *digest;
+
+  tor_assert(srv_out);
+
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+
+  /* Start setting up payload:
+   *  H("shared-random-disaster" | INT_8(period_length) | INT_8(period_num)) */
+  crypto_digest_add_bytes(digest, HS_SRV_DISASTER_PREFIX,
+                          HS_SRV_DISASTER_PREFIX_LEN);
+
+  /* Setup INT_8(period_length) | INT_8(period_num) */
+  {
+    uint64_t time_period_length = get_time_period_length();
+    char period_stuff[sizeof(uint64_t)*2];
+    size_t offset = 0;
+    set_uint64(period_stuff, tor_htonll(time_period_length));
+    offset += sizeof(uint64_t);
+    set_uint64(period_stuff+offset, tor_htonll(time_period_num));
+    offset += sizeof(uint64_t);
+    tor_assert(offset == sizeof(period_stuff));
+
+    crypto_digest_add_bytes(digest, period_stuff,  sizeof(period_stuff));
+  }
+
+  crypto_digest_get_digest(digest, (char *) srv_out, DIGEST256_LEN);
+  crypto_digest_free(digest);
+}
+
+/** Due to the high cost of computing the disaster SRV and that potentially we
+ *  would have to do it thousands of times in a row, we always cache the
+ *  computer disaster SRV (and its corresponding time period num) in case we
+ *  want to reuse it soon after. We need to cache two SRVs, one for each active
+ *  time period.
+ */
+static uint8_t cached_disaster_srv[2][DIGEST256_LEN];
+static uint64_t cached_time_period_nums[2] = {0};
+
+/** Compute the disaster SRV value for this <b>time_period_num</b> and put it
+ *  in <b>srv_out</b> (of size at least DIGEST256_LEN). First check our caches
+ *  to see if we have already computed it. */
+STATIC void
+get_disaster_srv(uint64_t time_period_num, uint8_t *srv_out)
+{
+  if (time_period_num == cached_time_period_nums[0]) {
+    memcpy(srv_out, cached_disaster_srv[0], DIGEST256_LEN);
+    return;
+  } else if (time_period_num == cached_time_period_nums[1]) {
+    memcpy(srv_out, cached_disaster_srv[1], DIGEST256_LEN);
+    return;
+  } else {
+    int replace_idx;
+    // Replace the lower period number.
+    if (cached_time_period_nums[0] <= cached_time_period_nums[1]) {
+      replace_idx = 0;
+    } else {
+      replace_idx = 1;
+    }
+    cached_time_period_nums[replace_idx] = time_period_num;
+    compute_disaster_srv(time_period_num, cached_disaster_srv[replace_idx]);
+    memcpy(srv_out, cached_disaster_srv[replace_idx], DIGEST256_LEN);
+    return;
+  }
+}
+
+#ifdef TOR_UNIT_TESTS
+
+/** Get the first cached disaster SRV. Only used by unittests. */
+STATIC uint8_t *
+get_first_cached_disaster_srv(void)
+{
+  return cached_disaster_srv[0];
+}
+
+/** Get the second cached disaster SRV. Only used by unittests. */
+STATIC uint8_t *
+get_second_cached_disaster_srv(void)
+{
+  return cached_disaster_srv[1];
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/* When creating a blinded key, we need a parameter which construction is as
+ * follow: H(pubkey | [secret] | ed25519-basepoint | nonce).
+ *
+ * The nonce has a pre-defined format which uses the time period number
+ * period_num and the start of the period in second start_time_period.
+ *
+ * The secret of size secret_len is optional meaning that it can be NULL and
+ * thus will be ignored for the param construction.
+ *
+ * The result is put in param_out. */
+static void
+build_blinded_key_param(const ed25519_public_key_t *pubkey,
+                        const uint8_t *secret, size_t secret_len,
+                        uint64_t period_num, uint64_t period_length,
+                        uint8_t *param_out)
+{
+  size_t offset = 0;
+  const char blind_str[] = "Derive temporary signing key";
+  uint8_t nonce[HS_KEYBLIND_NONCE_LEN];
+  crypto_digest_t *digest;
+
+  tor_assert(pubkey);
+  tor_assert(param_out);
+
+  /* Create the nonce N. The construction is as follow:
+   *    N = "key-blind" || INT_8(period_num) || INT_8(period_length) */
+  memcpy(nonce, HS_KEYBLIND_NONCE_PREFIX, HS_KEYBLIND_NONCE_PREFIX_LEN);
+  offset += HS_KEYBLIND_NONCE_PREFIX_LEN;
+  set_uint64(nonce + offset, tor_htonll(period_num));
+  offset += sizeof(uint64_t);
+  set_uint64(nonce + offset, tor_htonll(period_length));
+  offset += sizeof(uint64_t);
+  tor_assert(offset == HS_KEYBLIND_NONCE_LEN);
+
+  /* Generate the parameter h and the construction is as follow:
+   *    h = H(BLIND_STRING | pubkey | [secret] | ed25519-basepoint | N) */
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+  crypto_digest_add_bytes(digest, blind_str, sizeof(blind_str));
+  crypto_digest_add_bytes(digest, (char *) pubkey, ED25519_PUBKEY_LEN);
+  /* Optional secret. */
+  if (secret) {
+    crypto_digest_add_bytes(digest, (char *) secret, secret_len);
+  }
+  crypto_digest_add_bytes(digest, str_ed25519_basepoint,
+                          strlen(str_ed25519_basepoint));
+  crypto_digest_add_bytes(digest, (char *) nonce, sizeof(nonce));
+
+  /* Extract digest and put it in the param. */
+  crypto_digest_get_digest(digest, (char *) param_out, DIGEST256_LEN);
+  crypto_digest_free(digest);
+
+  memwipe(nonce, 0, sizeof(nonce));
+}
+
+/* Using an ed25519 public key and version to build the checksum of an
+ * address. Put in checksum_out. Format is:
+ *    SHA3-256(".onion checksum" || PUBKEY || VERSION)
+ *
+ * checksum_out must be large enough to receive 32 bytes (DIGEST256_LEN). */
+static void
+build_hs_checksum(const ed25519_public_key_t *key, uint8_t version,
+                  uint8_t *checksum_out)
+{
+  size_t offset = 0;
+  char data[HS_SERVICE_ADDR_CHECKSUM_INPUT_LEN];
+
+  /* Build checksum data. */
+  memcpy(data, HS_SERVICE_ADDR_CHECKSUM_PREFIX,
+         HS_SERVICE_ADDR_CHECKSUM_PREFIX_LEN);
+  offset += HS_SERVICE_ADDR_CHECKSUM_PREFIX_LEN;
+  memcpy(data + offset, key->pubkey, ED25519_PUBKEY_LEN);
+  offset += ED25519_PUBKEY_LEN;
+  set_uint8(data + offset, version);
+  offset += sizeof(version);
+  tor_assert(offset == HS_SERVICE_ADDR_CHECKSUM_INPUT_LEN);
+
+  /* Hash the data payload to create the checksum. */
+  crypto_digest256((char *) checksum_out, data, sizeof(data),
+                   DIGEST_SHA3_256);
+}
+
+/* Using an ed25519 public key, checksum and version to build the binary
+ * representation of a service address. Put in addr_out. Format is:
+ *    addr_out = PUBKEY || CHECKSUM || VERSION
+ *
+ * addr_out must be large enough to receive HS_SERVICE_ADDR_LEN bytes. */
+static void
+build_hs_address(const ed25519_public_key_t *key, const uint8_t *checksum,
+                 uint8_t version, char *addr_out)
+{
+  size_t offset = 0;
+
+  tor_assert(key);
+  tor_assert(checksum);
+
+  memcpy(addr_out, key->pubkey, ED25519_PUBKEY_LEN);
+  offset += ED25519_PUBKEY_LEN;
+  memcpy(addr_out + offset, checksum, HS_SERVICE_ADDR_CHECKSUM_LEN_USED);
+  offset += HS_SERVICE_ADDR_CHECKSUM_LEN_USED;
+  set_uint8(addr_out + offset, version);
+  offset += sizeof(uint8_t);
+  tor_assert(offset == HS_SERVICE_ADDR_LEN);
+}
+
+/* Helper for hs_parse_address(): Using a binary representation of a service
+ * address, parse its content into the key_out, checksum_out and version_out.
+ * Any out variable can be NULL in case the caller would want only one field.
+ * checksum_out MUST at least be 2 bytes long. address must be at least
+ * HS_SERVICE_ADDR_LEN bytes but doesn't need to be NUL terminated. */
+static void
+hs_parse_address_impl(const char *address, ed25519_public_key_t *key_out,
+                      uint8_t *checksum_out, uint8_t *version_out)
+{
+  size_t offset = 0;
+
+  tor_assert(address);
+
+  if (key_out) {
+    /* First is the key. */
+    memcpy(key_out->pubkey, address, ED25519_PUBKEY_LEN);
+  }
+  offset += ED25519_PUBKEY_LEN;
+  if (checksum_out) {
+    /* Followed by a 2 bytes checksum. */
+    memcpy(checksum_out, address + offset, HS_SERVICE_ADDR_CHECKSUM_LEN_USED);
+  }
+  offset += HS_SERVICE_ADDR_CHECKSUM_LEN_USED;
+  if (version_out) {
+    /* Finally, version value is 1 byte. */
+    *version_out = get_uint8(address + offset);
+  }
+  offset += sizeof(uint8_t);
+  /* Extra safety. */
+  tor_assert(offset == HS_SERVICE_ADDR_LEN);
+}
+
+/* Using the given identity public key and a blinded public key, compute the
+ * subcredential and put it in subcred_out (must be of size DIGEST256_LEN).
+ * This can't fail. */
+void
+hs_get_subcredential(const ed25519_public_key_t *identity_pk,
+                     const ed25519_public_key_t *blinded_pk,
+                     uint8_t *subcred_out)
+{
+  uint8_t credential[DIGEST256_LEN];
+  crypto_digest_t *digest;
+
+  tor_assert(identity_pk);
+  tor_assert(blinded_pk);
+  tor_assert(subcred_out);
+
+  /* First, build the credential. Construction is as follow:
+   *  credential = H("credential" | public-identity-key) */
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+  crypto_digest_add_bytes(digest, HS_CREDENTIAL_PREFIX,
+                          HS_CREDENTIAL_PREFIX_LEN);
+  crypto_digest_add_bytes(digest, (const char *) identity_pk->pubkey,
+                          ED25519_PUBKEY_LEN);
+  crypto_digest_get_digest(digest, (char *) credential, DIGEST256_LEN);
+  crypto_digest_free(digest);
+
+  /* Now, compute the subcredential. Construction is as follow:
+   *  subcredential = H("subcredential" | credential | blinded-public-key). */
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+  crypto_digest_add_bytes(digest, HS_SUBCREDENTIAL_PREFIX,
+                          HS_SUBCREDENTIAL_PREFIX_LEN);
+  crypto_digest_add_bytes(digest, (const char *) credential,
+                          sizeof(credential));
+  crypto_digest_add_bytes(digest, (const char *) blinded_pk->pubkey,
+                          ED25519_PUBKEY_LEN);
+  crypto_digest_get_digest(digest, (char *) subcred_out, DIGEST256_LEN);
+  crypto_digest_free(digest);
+
+  memwipe(credential, 0, sizeof(credential));
+}
+
+/* From the given list of hidden service ports, find the ones that match the
+ * given edge connection conn, pick one at random and use it to set the
+ * connection address. Return 0 on success or -1 if none. */
+int
+hs_set_conn_addr_port(const smartlist_t *ports, edge_connection_t *conn)
+{
+  rend_service_port_config_t *chosen_port;
+  unsigned int warn_once = 0;
+  smartlist_t *matching_ports;
+
+  tor_assert(ports);
+  tor_assert(conn);
+
+  matching_ports = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(ports, rend_service_port_config_t *, p) {
+    if (TO_CONN(conn)->port != p->virtual_port) {
+      continue;
+    }
+    if (!(p->is_unix_addr)) {
+      smartlist_add(matching_ports, p);
+    } else {
+      if (add_unix_port(matching_ports, p)) {
+        if (!warn_once) {
+          /* Unix port not supported so warn only once. */
+          log_warn(LD_REND, "Saw AF_UNIX virtual port mapping for port %d "
+                            "which is unsupported on this platform. "
+                            "Ignoring it.",
+                   TO_CONN(conn)->port);
+        }
+        warn_once++;
+      }
+    }
+  } SMARTLIST_FOREACH_END(p);
+
+  chosen_port = smartlist_choose(matching_ports);
+  smartlist_free(matching_ports);
+  if (chosen_port) {
+    if (!(chosen_port->is_unix_addr)) {
+      /* save the original destination before we overwrite it */
+      if (conn->hs_ident) {
+        conn->hs_ident->orig_virtual_port = TO_CONN(conn)->port;
+      }
+
+      /* Get a non-AF_UNIX connection ready for connection_exit_connect() */
+      tor_addr_copy(&TO_CONN(conn)->addr, &chosen_port->real_addr);
+      TO_CONN(conn)->port = chosen_port->real_port;
+    } else {
+      if (set_unix_port(conn, chosen_port)) {
+        /* Simply impossible to end up here else we were able to add a Unix
+         * port without AF_UNIX support... ? */
+        tor_assert(0);
+      }
+    }
+  }
+  return (chosen_port) ? 0 : -1;
+}
+
+/* Using a base32 representation of a service address, parse its content into
+ * the key_out, checksum_out and version_out. Any out variable can be NULL in
+ * case the caller would want only one field. checksum_out MUST at least be 2
+ * bytes long.
+ *
+ * Return 0 if parsing went well; return -1 in case of error. */
+int
+hs_parse_address(const char *address, ed25519_public_key_t *key_out,
+                 uint8_t *checksum_out, uint8_t *version_out)
+{
+  char decoded[HS_SERVICE_ADDR_LEN];
+
+  tor_assert(address);
+
+  /* Obvious length check. */
+  if (strlen(address) != HS_SERVICE_ADDR_LEN_BASE32) {
+    log_warn(LD_REND, "Service address %s has an invalid length. "
+                      "Expected %lu but got %lu.",
+             escaped_safe_str(address),
+             (unsigned long) HS_SERVICE_ADDR_LEN_BASE32,
+             (unsigned long) strlen(address));
+    goto invalid;
+  }
+
+  /* Decode address so we can extract needed fields. */
+  if (base32_decode(decoded, sizeof(decoded), address, strlen(address)) < 0) {
+    log_warn(LD_REND, "Service address %s can't be decoded.",
+             escaped_safe_str(address));
+    goto invalid;
+  }
+
+  /* Parse the decoded address into the fields we need. */
+  hs_parse_address_impl(decoded, key_out, checksum_out, version_out);
+
+  return 0;
+ invalid:
+  return -1;
+}
+
+/* Validate a given onion address. The length, the base32 decoding and
+ * checksum are validated. Return 1 if valid else 0. */
+int
+hs_address_is_valid(const char *address)
+{
+  uint8_t version;
+  uint8_t checksum[HS_SERVICE_ADDR_CHECKSUM_LEN_USED];
+  uint8_t target_checksum[DIGEST256_LEN];
+  ed25519_public_key_t service_pubkey;
+
+  /* Parse the decoded address into the fields we need. */
+  if (hs_parse_address(address, &service_pubkey, checksum, &version) < 0) {
+    goto invalid;
+  }
+
+  /* Get the checksum it's suppose to be and compare it with what we have
+   * encoded in the address. */
+  build_hs_checksum(&service_pubkey, version, target_checksum);
+  if (tor_memcmp(checksum, target_checksum, sizeof(checksum))) {
+    log_warn(LD_REND, "Service address %s invalid checksum.",
+             escaped_safe_str(address));
+    goto invalid;
+  }
+
+  /* Validate that this pubkey does not have a torsion component. We need to do
+   * this on the prop224 client-side so that attackers can't give equivalent
+   * forms of an onion address to users. */
+  if (ed25519_validate_pubkey(&service_pubkey) < 0) {
+    log_warn(LD_REND, "Service address %s has bad pubkey .",
+             escaped_safe_str(address));
+    goto invalid;
+  }
+
+  /* Valid address. */
+  return 1;
+ invalid:
+  return 0;
+}
+
+/* Build a service address using an ed25519 public key and a given version.
+ * The returned address is base32 encoded and put in addr_out. The caller MUST
+ * make sure the addr_out is at least HS_SERVICE_ADDR_LEN_BASE32 + 1 long.
+ *
+ * Format is as follow:
+ *     base32(PUBKEY || CHECKSUM || VERSION)
+ *     CHECKSUM = H(".onion checksum" || PUBKEY || VERSION)
+ * */
+void
+hs_build_address(const ed25519_public_key_t *key, uint8_t version,
+                 char *addr_out)
+{
+  uint8_t checksum[DIGEST256_LEN];
+  char address[HS_SERVICE_ADDR_LEN];
+
+  tor_assert(key);
+  tor_assert(addr_out);
+
+  /* Get the checksum of the address. */
+  build_hs_checksum(key, version, checksum);
+  /* Get the binary address representation. */
+  build_hs_address(key, checksum, version, address);
+
+  /* Encode the address. addr_out will be NUL terminated after this. */
+  base32_encode(addr_out, HS_SERVICE_ADDR_LEN_BASE32 + 1, address,
+                sizeof(address));
+  /* Validate what we just built. */
+  tor_assert(hs_address_is_valid(addr_out));
+}
+
+/* Return a newly allocated copy of lspec. */
+link_specifier_t *
+hs_link_specifier_dup(const link_specifier_t *lspec)
+{
+  link_specifier_t *result = link_specifier_new();
+  memcpy(result, lspec, sizeof(*result));
+  /* The unrecognized field is a dynamic array so make sure to copy its
+   * content and not the pointer. */
+  link_specifier_setlen_un_unrecognized(
+                  result, link_specifier_getlen_un_unrecognized(lspec));
+  if (link_specifier_getlen_un_unrecognized(result)) {
+    memcpy(link_specifier_getarray_un_unrecognized(result),
+           link_specifier_getconstarray_un_unrecognized(lspec),
+           link_specifier_getlen_un_unrecognized(result));
+  }
+  return result;
+}
+
+/* From a given ed25519 public key pk and an optional secret, compute a
+ * blinded public key and put it in blinded_pk_out. This is only useful to
+ * the client side because the client only has access to the identity public
+ * key of the service. */
+void
+hs_build_blinded_pubkey(const ed25519_public_key_t *pk,
+                        const uint8_t *secret, size_t secret_len,
+                        uint64_t time_period_num,
+                        ed25519_public_key_t *blinded_pk_out)
+{
+  /* Our blinding key API requires a 32 bytes parameter. */
+  uint8_t param[DIGEST256_LEN];
+
+  tor_assert(pk);
+  tor_assert(blinded_pk_out);
+  tor_assert(!tor_mem_is_zero((char *) pk, ED25519_PUBKEY_LEN));
+
+  build_blinded_key_param(pk, secret, secret_len,
+                          time_period_num, get_time_period_length(), param);
+  ed25519_public_blind(blinded_pk_out, pk, param);
+
+  memwipe(param, 0, sizeof(param));
+}
+
+/* From a given ed25519 keypair kp and an optional secret, compute a blinded
+ * keypair for the current time period and put it in blinded_kp_out. This is
+ * only useful by the service side because the client doesn't have access to
+ * the identity secret key. */
+void
+hs_build_blinded_keypair(const ed25519_keypair_t *kp,
+                         const uint8_t *secret, size_t secret_len,
+                         uint64_t time_period_num,
+                         ed25519_keypair_t *blinded_kp_out)
+{
+  /* Our blinding key API requires a 32 bytes parameter. */
+  uint8_t param[DIGEST256_LEN];
+
+  tor_assert(kp);
+  tor_assert(blinded_kp_out);
+  /* Extra safety. A zeroed key is bad. */
+  tor_assert(!tor_mem_is_zero((char *) &kp->pubkey, ED25519_PUBKEY_LEN));
+  tor_assert(!tor_mem_is_zero((char *) &kp->seckey, ED25519_SECKEY_LEN));
+
+  build_blinded_key_param(&kp->pubkey, secret, secret_len,
+                          time_period_num, get_time_period_length(), param);
+  ed25519_keypair_blind(blinded_kp_out, kp, param);
+
+  memwipe(param, 0, sizeof(param));
+}
+
+/* Return true if we are currently in the time segment between a new time
+ * period and a new SRV (in the real network that happens between 12:00 and
+ * 00:00 UTC). Here is a diagram showing exactly when this returns true:
+ *
+ *    +------------------------------------------------------------------+
+ *    |                                                                  |
+ *    | 00:00      12:00       00:00       12:00       00:00       12:00 |
+ *    | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+ *    |                                                                  |
+ *    |  $==========|-----------$===========|-----------$===========|    |
+ *    |             ^^^^^^^^^^^^            ^^^^^^^^^^^^                 |
+ *    |                                                                  |
+ *    +------------------------------------------------------------------+
+ */
+MOCK_IMPL(int,
+hs_in_period_between_tp_and_srv,(const networkstatus_t *consensus, time_t now))
+{
+  time_t valid_after;
+  time_t srv_start_time, tp_start_time;
+
+  if (!consensus) {
+    consensus = networkstatus_get_live_consensus(now);
+    if (!consensus) {
+      return 0;
+    }
+  }
+
+  /* Get start time of next TP and of current SRV protocol run, and check if we
+   * are between them. */
+  valid_after = consensus->valid_after;
+  srv_start_time = sr_state_get_start_time_of_current_protocol_run();
+  tp_start_time = hs_get_start_time_of_next_time_period(srv_start_time);
+
+  if (valid_after >= srv_start_time && valid_after < tp_start_time) {
+    return 0;
+  }
+
+  return 1;
+}
+
+/* Return 1 if any virtual port in ports needs a circuit with good uptime.
+ * Else return 0. */
+int
+hs_service_requires_uptime_circ(const smartlist_t *ports)
+{
+  tor_assert(ports);
+
+  SMARTLIST_FOREACH_BEGIN(ports, rend_service_port_config_t *, p) {
+    if (smartlist_contains_int_as_string(get_options()->LongLivedPorts,
+                                         p->virtual_port)) {
+      return 1;
+    }
+  } SMARTLIST_FOREACH_END(p);
+  return 0;
+}
+
+/* Build hs_index which is used to find the responsible hsdirs. This index
+ * value is used to select the responsible HSDir where their hsdir_index is
+ * closest to this value.
+ *    SHA3-256("store-at-idx" | blinded_public_key |
+ *             INT_8(replicanum) | INT_8(period_length) | INT_8(period_num) )
+ *
+ * hs_index_out must be large enough to receive DIGEST256_LEN bytes. */
+void
+hs_build_hs_index(uint64_t replica, const ed25519_public_key_t *blinded_pk,
+                  uint64_t period_num, uint8_t *hs_index_out)
+{
+  crypto_digest_t *digest;
+
+  tor_assert(blinded_pk);
+  tor_assert(hs_index_out);
+
+  /* Build hs_index. See construction at top of function comment. */
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+  crypto_digest_add_bytes(digest, HS_INDEX_PREFIX, HS_INDEX_PREFIX_LEN);
+  crypto_digest_add_bytes(digest, (const char *) blinded_pk->pubkey,
+                          ED25519_PUBKEY_LEN);
+
+  /* Now setup INT_8(replicanum) | INT_8(period_length) | INT_8(period_num) */
+  {
+    uint64_t period_length = get_time_period_length();
+    char buf[sizeof(uint64_t)*3];
+    size_t offset = 0;
+    set_uint64(buf, tor_htonll(replica));
+    offset += sizeof(uint64_t);
+    set_uint64(buf+offset, tor_htonll(period_length));
+    offset += sizeof(uint64_t);
+    set_uint64(buf+offset, tor_htonll(period_num));
+    offset += sizeof(uint64_t);
+    tor_assert(offset == sizeof(buf));
+
+    crypto_digest_add_bytes(digest, buf, sizeof(buf));
+  }
+
+  crypto_digest_get_digest(digest, (char *) hs_index_out, DIGEST256_LEN);
+  crypto_digest_free(digest);
+}
+
+/* Build hsdir_index which is used to find the responsible hsdirs. This is the
+ * index value that is compare to the hs_index when selecting an HSDir.
+ *    SHA3-256("node-idx" | node_identity |
+ *             shared_random_value | INT_8(period_length) | INT_8(period_num) )
+ *
+ * hsdir_index_out must be large enough to receive DIGEST256_LEN bytes. */
+void
+hs_build_hsdir_index(const ed25519_public_key_t *identity_pk,
+                     const uint8_t *srv_value, uint64_t period_num,
+                     uint8_t *hsdir_index_out)
+{
+  crypto_digest_t *digest;
+
+  tor_assert(identity_pk);
+  tor_assert(srv_value);
+  tor_assert(hsdir_index_out);
+
+  /* Build hsdir_index. See construction at top of function comment. */
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+  crypto_digest_add_bytes(digest, HSDIR_INDEX_PREFIX, HSDIR_INDEX_PREFIX_LEN);
+  crypto_digest_add_bytes(digest, (const char *) identity_pk->pubkey,
+                          ED25519_PUBKEY_LEN);
+  crypto_digest_add_bytes(digest, (const char *) srv_value, DIGEST256_LEN);
+
+  {
+    uint64_t time_period_length = get_time_period_length();
+    char period_stuff[sizeof(uint64_t)*2];
+    size_t offset = 0;
+    set_uint64(period_stuff, tor_htonll(period_num));
+    offset += sizeof(uint64_t);
+    set_uint64(period_stuff+offset, tor_htonll(time_period_length));
+    offset += sizeof(uint64_t);
+    tor_assert(offset == sizeof(period_stuff));
+
+    crypto_digest_add_bytes(digest, period_stuff,  sizeof(period_stuff));
+  }
+
+  crypto_digest_get_digest(digest, (char *) hsdir_index_out, DIGEST256_LEN);
+  crypto_digest_free(digest);
+}
+
+/* Return a newly allocated buffer containing the current shared random value
+ * or if not present, a disaster value is computed using the given time period
+ * number. If a consensus is provided in <b>ns</b>, use it to get the SRV
+ * value. This function can't fail. */
+uint8_t *
+hs_get_current_srv(uint64_t time_period_num, const networkstatus_t *ns)
+{
+  uint8_t *sr_value = tor_malloc_zero(DIGEST256_LEN);
+  const sr_srv_t *current_srv = sr_get_current(ns);
+
+  if (current_srv) {
+    memcpy(sr_value, current_srv->value, sizeof(current_srv->value));
+  } else {
+    /* Disaster mode. */
+    get_disaster_srv(time_period_num, sr_value);
+  }
+  return sr_value;
+}
+
+/* Return a newly allocated buffer containing the previous shared random
+ * value or if not present, a disaster value is computed using the given time
+ * period number. This function can't fail. */
+uint8_t *
+hs_get_previous_srv(uint64_t time_period_num, const networkstatus_t *ns)
+{
+  uint8_t *sr_value = tor_malloc_zero(DIGEST256_LEN);
+  const sr_srv_t *previous_srv = sr_get_previous(ns);
+
+  if (previous_srv) {
+    memcpy(sr_value, previous_srv->value, sizeof(previous_srv->value));
+  } else {
+    /* Disaster mode. */
+    get_disaster_srv(time_period_num, sr_value);
+  }
+  return sr_value;
+}
+
+/* Return the number of replicas defined by a consensus parameter or the
+ * default value. */
+int32_t
+hs_get_hsdir_n_replicas(void)
+{
+  /* The [1,16] range is a specification requirement. */
+  return networkstatus_get_param(NULL, "hsdir_n_replicas",
+                                 HS_DEFAULT_HSDIR_N_REPLICAS, 1, 16);
+}
+
+/* Return the spread fetch value defined by a consensus parameter or the
+ * default value. */
+int32_t
+hs_get_hsdir_spread_fetch(void)
+{
+  /* The [1,128] range is a specification requirement. */
+  return networkstatus_get_param(NULL, "hsdir_spread_fetch",
+                                 HS_DEFAULT_HSDIR_SPREAD_FETCH, 1, 128);
+}
+
+/* Return the spread store value defined by a consensus parameter or the
+ * default value. */
+int32_t
+hs_get_hsdir_spread_store(void)
+{
+  /* The [1,128] range is a specification requirement. */
+  return networkstatus_get_param(NULL, "hsdir_spread_store",
+                                 HS_DEFAULT_HSDIR_SPREAD_STORE, 1, 128);
+}
+
+/** <b>node</b> is an HSDir so make sure that we have assigned an hsdir index.
+ *  Return 0 if everything is as expected, else return -1. */
+static int
+node_has_hsdir_index(const node_t *node)
+{
+  tor_assert(node_supports_v3_hsdir(node));
+
+  /* A node can't have an HSDir index without a descriptor since we need desc
+   * to get its ed25519 key.  for_direct_connect should be zero, since we
+   * always use the consensus-indexed node's keys to build the hash ring, even
+   * if some of the consensus-indexed nodes are also bridges. */
+  if (!node_has_preferred_descriptor(node, 0)) {
+    return 0;
+  }
+
+  /* At this point, since the node has a desc, this node must also have an
+   * hsdir index. If not, something went wrong, so BUG out. */
+  if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.fetch,
+                          DIGEST256_LEN))) {
+    return 0;
+  }
+  if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.store_first,
+                          DIGEST256_LEN))) {
+    return 0;
+  }
+  if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.store_second,
+                          DIGEST256_LEN))) {
+    return 0;
+  }
+
+  return 1;
+}
+
+/* For a given blinded key and time period number, get the responsible HSDir
+ * and put their routerstatus_t object in the responsible_dirs list. If
+ * 'use_second_hsdir_index' is true, use the second hsdir_index of the node_t
+ * is used. If 'for_fetching' is true, the spread fetch consensus parameter is
+ * used else the spread store is used which is only for upload. This function
+ * can't fail but it is possible that the responsible_dirs list contains fewer
+ * nodes than expected.
+ *
+ * This function goes over the latest consensus routerstatus list and sorts it
+ * by their node_t hsdir_index then does a binary search to find the closest
+ * node. All of this makes it a bit CPU intensive so use it wisely. */
+void
+hs_get_responsible_hsdirs(const ed25519_public_key_t *blinded_pk,
+                          uint64_t time_period_num, int use_second_hsdir_index,
+                          int for_fetching, smartlist_t *responsible_dirs)
+{
+  smartlist_t *sorted_nodes;
+  /* The compare function used for the smartlist bsearch. We have two
+   * different depending on is_next_period. */
+  int (*cmp_fct)(const void *, const void **);
+
+  tor_assert(blinded_pk);
+  tor_assert(responsible_dirs);
+
+  sorted_nodes = smartlist_new();
+
+  /* Make sure we actually have a live consensus */
+  networkstatus_t *c = networkstatus_get_live_consensus(approx_time());
+  if (!c || smartlist_len(c->routerstatus_list) == 0) {
+      log_warn(LD_REND, "No live consensus so we can't get the responsible "
+               "hidden service directories.");
+      goto done;
+  }
+
+  /* Ensure the nodelist is fresh, since it contains the HSDir indices. */
+  nodelist_ensure_freshness(c);
+
+  /* Add every node_t that support HSDir v3 for which we do have a valid
+   * hsdir_index already computed for them for this consensus. */
+  {
+    SMARTLIST_FOREACH_BEGIN(c->routerstatus_list, const routerstatus_t *, rs) {
+      /* Even though this node_t object won't be modified and should be const,
+       * we can't add const object in a smartlist_t. */
+      node_t *n = node_get_mutable_by_id(rs->identity_digest);
+      tor_assert(n);
+      if (node_supports_v3_hsdir(n) && rs->is_hs_dir) {
+        if (!node_has_hsdir_index(n)) {
+          log_info(LD_GENERAL, "Node %s was found without hsdir index.",
+                   node_describe(n));
+          continue;
+        }
+        smartlist_add(sorted_nodes, n);
+      }
+    } SMARTLIST_FOREACH_END(rs);
+  }
+  if (smartlist_len(sorted_nodes) == 0) {
+    log_warn(LD_REND, "No nodes found to be HSDir or supporting v3.");
+    goto done;
+  }
+
+  /* First thing we have to do is sort all node_t by hsdir_index. The
+   * is_next_period tells us if we want the current or the next one. Set the
+   * bsearch compare function also while we are at it. */
+  if (for_fetching) {
+    smartlist_sort(sorted_nodes, compare_node_fetch_hsdir_index);
+    cmp_fct = compare_digest_to_fetch_hsdir_index;
+  } else if (use_second_hsdir_index) {
+    smartlist_sort(sorted_nodes, compare_node_store_second_hsdir_index);
+    cmp_fct = compare_digest_to_store_second_hsdir_index;
+  } else {
+    smartlist_sort(sorted_nodes, compare_node_store_first_hsdir_index);
+    cmp_fct = compare_digest_to_store_first_hsdir_index;
+  }
+
+  /* For all replicas, we'll select a set of HSDirs using the consensus
+   * parameters and the sorted list. The replica starting at value 1 is
+   * defined by the specification. */
+  for (int replica = 1; replica <= hs_get_hsdir_n_replicas(); replica++) {
+    int idx, start, found, n_added = 0;
+    uint8_t hs_index[DIGEST256_LEN] = {0};
+    /* Number of node to add to the responsible dirs list depends on if we are
+     * trying to fetch or store. A client always fetches. */
+    int n_to_add = (for_fetching) ? hs_get_hsdir_spread_fetch() :
+                                    hs_get_hsdir_spread_store();
+
+    /* Get the index that we should use to select the node. */
+    hs_build_hs_index(replica, blinded_pk, time_period_num, hs_index);
+    /* The compare function pointer has been set correctly earlier. */
+    start = idx = smartlist_bsearch_idx(sorted_nodes, hs_index, cmp_fct,
+                                        &found);
+    /* Getting the length of the list if no member is greater than the key we
+     * are looking for so start at the first element. */
+    if (idx == smartlist_len(sorted_nodes)) {
+      start = idx = 0;
+    }
+    while (n_added < n_to_add) {
+      const node_t *node = smartlist_get(sorted_nodes, idx);
+      /* If the node has already been selected which is possible between
+       * replicas, the specification says to skip over. */
+      if (!smartlist_contains(responsible_dirs, node->rs)) {
+        smartlist_add(responsible_dirs, node->rs);
+        ++n_added;
+      }
+      if (++idx == smartlist_len(sorted_nodes)) {
+        /* Wrap if we've reached the end of the list. */
+        idx = 0;
+      }
+      if (idx == start) {
+        /* We've gone over the whole list, stop and avoid infinite loop. */
+        break;
+      }
+    }
+  }
+
+ done:
+  smartlist_free(sorted_nodes);
+}
+
+/*********************** HSDir request tracking ***************************/
+
+/** Return the period for which a hidden service directory cannot be queried
+ * for the same descriptor ID again, taking TestingTorNetwork into account. */
+time_t
+hs_hsdir_requery_period(const or_options_t *options)
+{
+  tor_assert(options);
+
+  if (options->TestingTorNetwork) {
+    return REND_HID_SERV_DIR_REQUERY_PERIOD_TESTING;
+  } else {
+    return REND_HID_SERV_DIR_REQUERY_PERIOD;
+  }
+}
+
+/** Tracks requests for fetching hidden service descriptors. It's used by
+ *  hidden service clients, to avoid querying HSDirs that have already failed
+ *  giving back a descriptor. The same data structure is used to track both v2
+ *  and v3 HS descriptor requests.
+ *
+ * The string map is a key/value store that contains the last request times to
+ * hidden service directories for certain queries. Specifically:
+ *
+ *   key = base32(hsdir_identity) + base32(hs_identity)
+ *   value = time_t of last request for that hs_identity to that HSDir
+ *
+ * where 'hsdir_identity' is the identity digest of the HSDir node, and
+ * 'hs_identity' is the descriptor ID of the HS in the v2 case, or the ed25519
+ * blinded public key of the HS in the v3 case. */
+static strmap_t *last_hid_serv_requests_ = NULL;
+
+/** Returns last_hid_serv_requests_, initializing it to a new strmap if
+ * necessary. */
+STATIC strmap_t *
+get_last_hid_serv_requests(void)
+{
+  if (!last_hid_serv_requests_)
+    last_hid_serv_requests_ = strmap_new();
+  return last_hid_serv_requests_;
+}
+
+/** Look up the last request time to hidden service directory <b>hs_dir</b>
+ * for descriptor request key <b>req_key_str</b> which is the descriptor ID
+ * for a v2 service or the blinded key for v3. If <b>set</b> is non-zero,
+ * assign the current time <b>now</b> and return that.  Otherwise, return the
+ * most recent request time, or 0 if no such request has been sent before. */
+time_t
+hs_lookup_last_hid_serv_request(routerstatus_t *hs_dir,
+                                const char *req_key_str,
+                                time_t now, int set)
+{
+  char hsdir_id_base32[BASE32_DIGEST_LEN + 1];
+  char *hsdir_desc_comb_id = NULL;
+  time_t *last_request_ptr;
+  strmap_t *last_hid_serv_requests = get_last_hid_serv_requests();
+
+  /* Create the key */
+  base32_encode(hsdir_id_base32, sizeof(hsdir_id_base32),
+                hs_dir->identity_digest, DIGEST_LEN);
+  tor_asprintf(&hsdir_desc_comb_id, "%s%s", hsdir_id_base32, req_key_str);
+
+  if (set) {
+    time_t *oldptr;
+    last_request_ptr = tor_malloc_zero(sizeof(time_t));
+    *last_request_ptr = now;
+    oldptr = strmap_set(last_hid_serv_requests, hsdir_desc_comb_id,
+                        last_request_ptr);
+    tor_free(oldptr);
+  } else {
+    last_request_ptr = strmap_get(last_hid_serv_requests,
+                                  hsdir_desc_comb_id);
+  }
+
+  tor_free(hsdir_desc_comb_id);
+  return (last_request_ptr) ? *last_request_ptr : 0;
+}
+
+/** Clean the history of request times to hidden service directories, so that
+ * it does not contain requests older than REND_HID_SERV_DIR_REQUERY_PERIOD
+ * seconds any more. */
+void
+hs_clean_last_hid_serv_requests(time_t now)
+{
+  strmap_iter_t *iter;
+  time_t cutoff = now - hs_hsdir_requery_period(get_options());
+  strmap_t *last_hid_serv_requests = get_last_hid_serv_requests();
+  for (iter = strmap_iter_init(last_hid_serv_requests);
+       !strmap_iter_done(iter); ) {
+    const char *key;
+    void *val;
+    time_t *ent;
+    strmap_iter_get(iter, &key, &val);
+    ent = (time_t *) val;
+    if (*ent < cutoff) {
+      iter = strmap_iter_next_rmv(last_hid_serv_requests, iter);
+      tor_free(ent);
+    } else {
+      iter = strmap_iter_next(last_hid_serv_requests, iter);
+    }
+  }
+}
+
+/** Remove all requests related to the descriptor request key string
+ * <b>req_key_str</b> from the history of times of requests to hidden service
+ * directories.
+ *
+ * This is called from rend_client_note_connection_attempt_ended(), which
+ * must be idempotent, so any future changes to this function must leave it
+ * idempotent too. */
+void
+hs_purge_hid_serv_from_last_hid_serv_requests(const char *req_key_str)
+{
+  strmap_iter_t *iter;
+  strmap_t *last_hid_serv_requests = get_last_hid_serv_requests();
+
+  for (iter = strmap_iter_init(last_hid_serv_requests);
+       !strmap_iter_done(iter); ) {
+    const char *key;
+    void *val;
+    strmap_iter_get(iter, &key, &val);
+
+    /* XXX: The use of REND_DESC_ID_V2_LEN_BASE32 is very wrong in terms of
+     * semantic, see #23305. */
+
+    /* This strmap contains variable-sized elements so this is a basic length
+     * check on the strings we are about to compare. The key is variable sized
+     * since it's composed as follows:
+     *   key = base32(hsdir_identity) + base32(req_key_str)
+     * where 'req_key_str' is the descriptor ID of the HS in the v2 case, or
+     * the ed25519 blinded public key of the HS in the v3 case. */
+    if (strlen(key) < REND_DESC_ID_V2_LEN_BASE32 + strlen(req_key_str)) {
+      iter = strmap_iter_next(last_hid_serv_requests, iter);
+      continue;
+    }
+
+    /* Check if the tracked request matches our request key */
+    if (tor_memeq(key + REND_DESC_ID_V2_LEN_BASE32, req_key_str,
+                  strlen(req_key_str))) {
+      iter = strmap_iter_next_rmv(last_hid_serv_requests, iter);
+      tor_free(val);
+    } else {
+      iter = strmap_iter_next(last_hid_serv_requests, iter);
+    }
+  }
+}
+
+/** Purge the history of request times to hidden service directories,
+ * so that future lookups of an HS descriptor will not fail because we
+ * accessed all of the HSDir relays responsible for the descriptor
+ * recently. */
+void
+hs_purge_last_hid_serv_requests(void)
+{
+ /* Don't create the table if it doesn't exist yet (and it may very
+   * well not exist if the user hasn't accessed any HSes)... */
+  strmap_t *old_last_hid_serv_requests = last_hid_serv_requests_;
+  /* ... and let get_last_hid_serv_requests re-create it for us if
+   * necessary. */
+  last_hid_serv_requests_ = NULL;
+
+  if (old_last_hid_serv_requests != NULL) {
+    log_info(LD_REND, "Purging client last-HS-desc-request-time table");
+    strmap_free(old_last_hid_serv_requests, tor_free_);
+  }
+}
+
+/***********************************************************************/
+
+/** Given the list of responsible HSDirs in <b>responsible_dirs</b>, pick the
+ *  one that we should use to fetch a descriptor right now. Take into account
+ *  previous failed attempts at fetching this descriptor from HSDirs using the
+ *  string identifier <b>req_key_str</b>.
+ *
+ *  Steals ownership of <b>responsible_dirs</b>.
+ *
+ *  Return the routerstatus of the chosen HSDir if successful, otherwise return
+ *  NULL if no HSDirs are worth trying right now. */
+routerstatus_t *
+hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
+{
+  smartlist_t *usable_responsible_dirs = smartlist_new();
+  const or_options_t *options = get_options();
+  routerstatus_t *hs_dir;
+  time_t now = time(NULL);
+  int excluded_some;
+
+  tor_assert(req_key_str);
+
+  /* Clean outdated request history first. */
+  hs_clean_last_hid_serv_requests(now);
+
+  /* Only select those hidden service directories to which we did not send a
+   * request recently and for which we have a router descriptor here.
+   *
+   * Use for_direct_connect==0 even if we will be connecting to the node
+   * directly, since we always use the key information in the
+   * consensus-indexed node descriptors for building the index.
+   **/
+  SMARTLIST_FOREACH_BEGIN(responsible_dirs, routerstatus_t *, dir) {
+    time_t last = hs_lookup_last_hid_serv_request(dir, req_key_str, 0, 0);
+    const node_t *node = node_get_by_id(dir->identity_digest);
+    if (last + hs_hsdir_requery_period(options) >= now ||
+        !node || !node_has_preferred_descriptor(node, 0)) {
+      SMARTLIST_DEL_CURRENT(responsible_dirs, dir);
+      continue;
+    }
+    if (!routerset_contains_node(options->ExcludeNodes, node)) {
+      smartlist_add(usable_responsible_dirs, dir);
+    }
+  } SMARTLIST_FOREACH_END(dir);
+
+  excluded_some =
+    smartlist_len(usable_responsible_dirs) < smartlist_len(responsible_dirs);
+
+  hs_dir = smartlist_choose(usable_responsible_dirs);
+  if (!hs_dir && !options->StrictNodes) {
+    hs_dir = smartlist_choose(responsible_dirs);
+  }
+
+  smartlist_free(responsible_dirs);
+  smartlist_free(usable_responsible_dirs);
+  if (!hs_dir) {
+    log_info(LD_REND, "Could not pick one of the responsible hidden "
+                      "service directories, because we requested them all "
+                      "recently without success.");
+    if (options->StrictNodes && excluded_some) {
+      log_warn(LD_REND, "Could not pick a hidden service directory for the "
+               "requested hidden service: they are all either down or "
+               "excluded, and StrictNodes is set.");
+    }
+  } else {
+    /* Remember that we are requesting a descriptor from this hidden service
+     * directory now. */
+    hs_lookup_last_hid_serv_request(hs_dir, req_key_str, now, 1);
+  }
+
+  return hs_dir;
+}
+
+/* From a list of link specifier, an onion key and if we are requesting a
+ * direct connection (ex: single onion service), return a newly allocated
+ * extend_info_t object. This function always returns an extend info with
+ * an IPv4 address, or NULL.
+ *
+ * It performs the following checks:
+ *  if either IPv4 or legacy ID is missing, return NULL.
+ *  if direct_conn, and we can't reach the IPv4 address, return NULL.
+ */
+extend_info_t *
+hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
+                               const curve25519_public_key_t *onion_key,
+                               int direct_conn)
+{
+  int have_v4 = 0, have_legacy_id = 0, have_ed25519_id = 0;
+  char legacy_id[DIGEST_LEN] = {0};
+  uint16_t port_v4 = 0;
+  tor_addr_t addr_v4;
+  ed25519_public_key_t ed25519_pk;
+  extend_info_t *info = NULL;
+
+  tor_assert(lspecs);
+
+  SMARTLIST_FOREACH_BEGIN(lspecs, const link_specifier_t *, ls) {
+    switch (link_specifier_get_ls_type(ls)) {
+    case LS_IPV4:
+      /* Skip if we already seen a v4. */
+      if (have_v4) continue;
+      tor_addr_from_ipv4h(&addr_v4,
+                          link_specifier_get_un_ipv4_addr(ls));
+      port_v4 = link_specifier_get_un_ipv4_port(ls);
+      have_v4 = 1;
+      break;
+    case LS_LEGACY_ID:
+      /* Make sure we do have enough bytes for the legacy ID. */
+      if (link_specifier_getlen_un_legacy_id(ls) < sizeof(legacy_id)) {
+        break;
+      }
+      memcpy(legacy_id, link_specifier_getconstarray_un_legacy_id(ls),
+             sizeof(legacy_id));
+      have_legacy_id = 1;
+      break;
+    case LS_ED25519_ID:
+      memcpy(ed25519_pk.pubkey,
+             link_specifier_getconstarray_un_ed25519_id(ls),
+             ED25519_PUBKEY_LEN);
+      have_ed25519_id = 1;
+      break;
+    default:
+      /* Ignore unknown. */
+      break;
+    }
+  } SMARTLIST_FOREACH_END(ls);
+
+  /* Legacy ID is mandatory, and we require IPv4. */
+  if (!have_v4 || !have_legacy_id) {
+    goto done;
+  }
+
+  /* We know we have IPv4, because we just checked. */
+  if (!direct_conn) {
+    /* All clients can extend to any IPv4 via a 3-hop path. */
+    goto validate;
+  } else if (direct_conn &&
+             fascist_firewall_allows_address_addr(&addr_v4, port_v4,
+                                                  FIREWALL_OR_CONNECTION,
+                                                  0, 0)) {
+    /* Direct connection and we can reach it in IPv4 so go for it. */
+    goto validate;
+
+    /* We will add support for falling back to a 3-hop path in a later
+     * release. */
+  } else {
+    /* If we can't reach IPv4, return NULL. */
+    goto done;
+  }
+
+  /* We will add support for IPv6 in a later release. */
+
+ validate:
+  /* We'll validate now that the address we've picked isn't a private one. If
+   * it is, are we allowing to extend to private address? */
+  if (!extend_info_addr_is_allowed(&addr_v4)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Requested address is private and we are not allowed to extend to "
+           "it: %s:%u", fmt_addr(&addr_v4), port_v4);
+    goto done;
+  }
+
+  /* We do have everything for which we think we can connect successfully. */
+  info = extend_info_new(NULL, legacy_id,
+                         (have_ed25519_id) ? &ed25519_pk : NULL, NULL,
+                         onion_key, &addr_v4, port_v4);
+ done:
+  return info;
+}
+
+/***********************************************************************/
+
+/* Initialize the entire HS subsytem. This is called in tor_init() before any
+ * torrc options are loaded. Only for >= v3. */
+void
+hs_init(void)
+{
+  hs_circuitmap_init();
+  hs_service_init();
+  hs_cache_init();
+}
+
+/* Release and cleanup all memory of the HS subsystem (all version). This is
+ * called by tor_free_all(). */
+void
+hs_free_all(void)
+{
+  hs_circuitmap_free_all();
+  hs_service_free_all();
+  hs_cache_free_all();
+  hs_client_free_all();
+}
+
+/* For the given origin circuit circ, decrement the number of rendezvous
+ * stream counter. This handles every hidden service version. */
+void
+hs_dec_rdv_stream_counter(origin_circuit_t *circ)
+{
+  tor_assert(circ);
+
+  if (circ->rend_data) {
+    circ->rend_data->nr_streams--;
+  } else if (circ->hs_ident) {
+    circ->hs_ident->num_rdv_streams--;
+  } else {
+    /* Should not be called if this circuit is not for hidden service. */
+    tor_assert_nonfatal_unreached();
+  }
+}
+
+/* For the given origin circuit circ, increment the number of rendezvous
+ * stream counter. This handles every hidden service version. */
+void
+hs_inc_rdv_stream_counter(origin_circuit_t *circ)
+{
+  tor_assert(circ);
+
+  if (circ->rend_data) {
+    circ->rend_data->nr_streams++;
+  } else if (circ->hs_ident) {
+    circ->hs_ident->num_rdv_streams++;
+  } else {
+    /* Should not be called if this circuit is not for hidden service. */
+    tor_assert_nonfatal_unreached();
+  }
+}
diff --git a/src/feature/hs/hs_common.h b/src/feature/hs/hs_common.h
new file mode 100644
index 0000000000..a44505930a
--- /dev/null
+++ b/src/feature/hs/hs_common.h
@@ -0,0 +1,288 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_common.h
+ * \brief Header file containing common data for the whole HS subsytem.
+ **/
+
+#ifndef TOR_HS_COMMON_H
+#define TOR_HS_COMMON_H
+
+#include "core/or/or.h"
+#include "lib/defs/x25519_sizes.h"
+
+struct curve25519_public_key_t;
+struct ed25519_public_key_t;
+struct ed25519_keypair_t;
+
+/* Trunnel */
+#include "trunnel/ed25519_cert.h"
+
+/* Protocol version 2. Use this instead of hardcoding "2" in the code base,
+ * this adds a clearer semantic to the value when used. */
+#define HS_VERSION_TWO 2
+/* Version 3 of the protocol (prop224). */
+#define HS_VERSION_THREE 3
+/* Earliest and latest version we support. */
+#define HS_VERSION_MIN HS_VERSION_TWO
+#define HS_VERSION_MAX HS_VERSION_THREE
+
+/** Try to maintain this many intro points per service by default. */
+#define NUM_INTRO_POINTS_DEFAULT 3
+/** Maximum number of intro points per generic and version 2 service. */
+#define NUM_INTRO_POINTS_MAX 10
+/** Number of extra intro points we launch if our set of intro nodes is empty.
+ * See proposal 155, section 4. */
+#define NUM_INTRO_POINTS_EXTRA 2
+
+/** If we can't build our intro circuits, don't retry for this long. */
+#define INTRO_CIRC_RETRY_PERIOD (60*5)
+/** Don't try to build more than this many circuits before giving up for a
+ * while.*/
+#define MAX_INTRO_CIRCS_PER_PERIOD 10
+/** How many times will a hidden service operator attempt to connect to a
+ * requested rendezvous point before giving up? */
+#define MAX_REND_FAILURES 1
+/** How many seconds should we spend trying to connect to a requested
+ * rendezvous point before giving up? */
+#define MAX_REND_TIMEOUT 30
+
+/* String prefix for the signature of ESTABLISH_INTRO */
+#define ESTABLISH_INTRO_SIG_PREFIX "Tor establish-intro cell v1"
+
+/* The default HS time period length */
+#define HS_TIME_PERIOD_LENGTH_DEFAULT 1440 /* 1440 minutes == one day */
+/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+#define HS_TIME_PERIOD_LENGTH_MIN 30 /* minutes */
+/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+#define HS_TIME_PERIOD_LENGTH_MAX (60 * 24 * 10) /* 10 days or 14400 minutes */
+
+/* Prefix of the onion address checksum. */
+#define HS_SERVICE_ADDR_CHECKSUM_PREFIX ".onion checksum"
+/* Length of the checksum prefix minus the NUL terminated byte. */
+#define HS_SERVICE_ADDR_CHECKSUM_PREFIX_LEN \
+  (sizeof(HS_SERVICE_ADDR_CHECKSUM_PREFIX) - 1)
+/* Length of the resulting checksum of the address. The construction of this
+ * checksum looks like:
+ *   CHECKSUM = ".onion checksum" || PUBKEY || VERSION
+ * where VERSION is 1 byte. This is pre-hashing. */
+#define HS_SERVICE_ADDR_CHECKSUM_INPUT_LEN \
+  (HS_SERVICE_ADDR_CHECKSUM_PREFIX_LEN + ED25519_PUBKEY_LEN + sizeof(uint8_t))
+/* The amount of bytes we use from the address checksum. */
+#define HS_SERVICE_ADDR_CHECKSUM_LEN_USED 2
+/* Length of the binary encoded service address which is of course before the
+ * base32 encoding. Construction is:
+ *    PUBKEY || CHECKSUM || VERSION
+ * with 1 byte VERSION and 2 bytes CHECKSUM. The following is 35 bytes. */
+#define HS_SERVICE_ADDR_LEN \
+  (ED25519_PUBKEY_LEN + HS_SERVICE_ADDR_CHECKSUM_LEN_USED + sizeof(uint8_t))
+/* Length of 'y' portion of 'y.onion' URL. This is base32 encoded and the
+ * length ends up to 56 bytes (not counting the terminated NUL byte.) */
+#define HS_SERVICE_ADDR_LEN_BASE32 \
+  (CEIL_DIV(HS_SERVICE_ADDR_LEN * 8, 5))
+
+/* The default HS time period length */
+#define HS_TIME_PERIOD_LENGTH_DEFAULT 1440 /* 1440 minutes == one day */
+/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+#define HS_TIME_PERIOD_LENGTH_MIN 30 /* minutes */
+/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+#define HS_TIME_PERIOD_LENGTH_MAX (60 * 24 * 10) /* 10 days or 14400 minutes */
+/* The time period rotation offset as seen in prop224 section [TIME-PERIODS] */
+#define HS_TIME_PERIOD_ROTATION_OFFSET (12 * 60) /* minutes */
+
+/* Keyblinding parameter construction is as follow:
+ *    "key-blind" || INT_8(period_num) || INT_8(start_period_sec) */
+#define HS_KEYBLIND_NONCE_PREFIX "key-blind"
+#define HS_KEYBLIND_NONCE_PREFIX_LEN (sizeof(HS_KEYBLIND_NONCE_PREFIX) - 1)
+#define HS_KEYBLIND_NONCE_LEN \
+  (HS_KEYBLIND_NONCE_PREFIX_LEN + sizeof(uint64_t) + sizeof(uint64_t))
+
+/* Credential and subcredential prefix value. */
+#define HS_CREDENTIAL_PREFIX "credential"
+#define HS_CREDENTIAL_PREFIX_LEN (sizeof(HS_CREDENTIAL_PREFIX) - 1)
+#define HS_SUBCREDENTIAL_PREFIX "subcredential"
+#define HS_SUBCREDENTIAL_PREFIX_LEN (sizeof(HS_SUBCREDENTIAL_PREFIX) - 1)
+
+/* Node hidden service stored at index prefix value. */
+#define HS_INDEX_PREFIX "store-at-idx"
+#define HS_INDEX_PREFIX_LEN (sizeof(HS_INDEX_PREFIX) - 1)
+
+/* Node hidden service directory index prefix value. */
+#define HSDIR_INDEX_PREFIX "node-idx"
+#define HSDIR_INDEX_PREFIX_LEN (sizeof(HSDIR_INDEX_PREFIX) - 1)
+
+/* Prefix of the shared random value disaster mode. */
+#define HS_SRV_DISASTER_PREFIX "shared-random-disaster"
+#define HS_SRV_DISASTER_PREFIX_LEN (sizeof(HS_SRV_DISASTER_PREFIX) - 1)
+
+/* Default value of number of hsdir replicas (hsdir_n_replicas). */
+#define HS_DEFAULT_HSDIR_N_REPLICAS 2
+/* Default value of hsdir spread store (hsdir_spread_store). */
+#define HS_DEFAULT_HSDIR_SPREAD_STORE 4
+/* Default value of hsdir spread fetch (hsdir_spread_fetch). */
+#define HS_DEFAULT_HSDIR_SPREAD_FETCH 3
+
+/* The size of a legacy RENDEZVOUS1 cell which adds up to 168 bytes. It is
+ * bigger than the 84 bytes needed for version 3 so we need to pad up to that
+ * length so it is indistinguishable between versions. */
+#define HS_LEGACY_RENDEZVOUS_CELL_SIZE \
+  (REND_COOKIE_LEN + DH1024_KEY_LEN + DIGEST_LEN)
+
+/* Type of authentication key used by an introduction point. */
+typedef enum {
+  HS_AUTH_KEY_TYPE_LEGACY  = 1,
+  HS_AUTH_KEY_TYPE_ED25519 = 2,
+} hs_auth_key_type_t;
+
+/* Return value when adding an ephemeral service through the ADD_ONION
+ * control port command. Both v2 and v3 share these. */
+typedef enum {
+  RSAE_BADAUTH     = -5, /**< Invalid auth_type/auth_clients */
+  RSAE_BADVIRTPORT = -4, /**< Invalid VIRTPORT/TARGET(s) */
+  RSAE_ADDREXISTS  = -3, /**< Onion address collision */
+  RSAE_BADPRIVKEY  = -2, /**< Invalid public key */
+  RSAE_INTERNAL    = -1, /**< Internal error */
+  RSAE_OKAY        = 0   /**< Service added as expected */
+} hs_service_add_ephemeral_status_t;
+
+/* Represents the mapping from a virtual port of a rendezvous service to a
+ * real port on some IP. */
+typedef struct rend_service_port_config_t {
+  /* The incoming HS virtual port we're mapping */
+  uint16_t virtual_port;
+  /* Is this an AF_UNIX port? */
+  unsigned int is_unix_addr:1;
+  /* The outgoing TCP port to use, if !is_unix_addr */
+  uint16_t real_port;
+  /* The outgoing IPv4 or IPv6 address to use, if !is_unix_addr */
+  tor_addr_t real_addr;
+  /* The socket path to connect to, if is_unix_addr */
+  char unix_addr[FLEXIBLE_ARRAY_MEMBER];
+} rend_service_port_config_t;
+
+void hs_init(void);
+void hs_free_all(void);
+
+void hs_cleanup_circ(circuit_t *circ);
+
+int hs_check_service_private_dir(const char *username, const char *path,
+                                 unsigned int dir_group_readable,
+                                 unsigned int create);
+int hs_get_service_max_rend_failures(void);
+
+char *hs_path_from_filename(const char *directory, const char *filename);
+void hs_build_address(const struct ed25519_public_key_t *key, uint8_t version,
+                      char *addr_out);
+int hs_address_is_valid(const char *address);
+int hs_parse_address(const char *address, struct ed25519_public_key_t *key_out,
+                     uint8_t *checksum_out, uint8_t *version_out);
+
+void hs_build_blinded_pubkey(const struct ed25519_public_key_t *pubkey,
+                             const uint8_t *secret, size_t secret_len,
+                             uint64_t time_period_num,
+                             struct ed25519_public_key_t *pubkey_out);
+void hs_build_blinded_keypair(const struct ed25519_keypair_t *kp,
+                              const uint8_t *secret, size_t secret_len,
+                              uint64_t time_period_num,
+                              struct ed25519_keypair_t *kp_out);
+int hs_service_requires_uptime_circ(const smartlist_t *ports);
+
+void rend_data_free_(rend_data_t *data);
+#define rend_data_free(data) \
+  FREE_AND_NULL(rend_data_t, rend_data_free_, (data))
+rend_data_t *rend_data_dup(const rend_data_t *data);
+rend_data_t *rend_data_client_create(const char *onion_address,
+                                     const char *desc_id,
+                                     const char *cookie,
+                                     rend_auth_type_t auth_type);
+rend_data_t *rend_data_service_create(const char *onion_address,
+                                      const char *pk_digest,
+                                      const uint8_t *cookie,
+                                      rend_auth_type_t auth_type);
+const char *rend_data_get_address(const rend_data_t *rend_data);
+const char *rend_data_get_desc_id(const rend_data_t *rend_data,
+                                  uint8_t replica, size_t *len_out);
+const uint8_t *rend_data_get_pk_digest(const rend_data_t *rend_data,
+                                       size_t *len_out);
+
+routerstatus_t *pick_hsdir(const char *desc_id, const char *desc_id_base32);
+
+void hs_get_subcredential(const struct ed25519_public_key_t *identity_pk,
+                          const struct ed25519_public_key_t *blinded_pk,
+                          uint8_t *subcred_out);
+
+uint64_t hs_get_previous_time_period_num(time_t now);
+uint64_t hs_get_time_period_num(time_t now);
+uint64_t hs_get_next_time_period_num(time_t now);
+time_t hs_get_start_time_of_next_time_period(time_t now);
+
+link_specifier_t *hs_link_specifier_dup(const link_specifier_t *lspec);
+
+MOCK_DECL(int, hs_in_period_between_tp_and_srv,
+          (const networkstatus_t *consensus, time_t now));
+
+uint8_t *hs_get_current_srv(uint64_t time_period_num,
+                            const networkstatus_t *ns);
+uint8_t *hs_get_previous_srv(uint64_t time_period_num,
+                             const networkstatus_t *ns);
+
+void hs_build_hsdir_index(const struct ed25519_public_key_t *identity_pk,
+                          const uint8_t *srv, uint64_t period_num,
+                          uint8_t *hsdir_index_out);
+void hs_build_hs_index(uint64_t replica,
+                       const struct ed25519_public_key_t *blinded_pk,
+                       uint64_t period_num, uint8_t *hs_index_out);
+
+int32_t hs_get_hsdir_n_replicas(void);
+int32_t hs_get_hsdir_spread_fetch(void);
+int32_t hs_get_hsdir_spread_store(void);
+
+void hs_get_responsible_hsdirs(const struct ed25519_public_key_t *blinded_pk,
+                              uint64_t time_period_num,
+                              int use_second_hsdir_index,
+                              int for_fetching, smartlist_t *responsible_dirs);
+routerstatus_t *hs_pick_hsdir(smartlist_t *responsible_dirs,
+                              const char *req_key_str);
+
+time_t hs_hsdir_requery_period(const or_options_t *options);
+time_t hs_lookup_last_hid_serv_request(routerstatus_t *hs_dir,
+                                       const char *desc_id_base32,
+                                       time_t now, int set);
+void hs_clean_last_hid_serv_requests(time_t now);
+void hs_purge_hid_serv_from_last_hid_serv_requests(const char *desc_id);
+void hs_purge_last_hid_serv_requests(void);
+
+int hs_set_conn_addr_port(const smartlist_t *ports, edge_connection_t *conn);
+
+void hs_inc_rdv_stream_counter(origin_circuit_t *circ);
+void hs_dec_rdv_stream_counter(origin_circuit_t *circ);
+
+extend_info_t *hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
+                          const struct curve25519_public_key_t *onion_key,
+                          int direct_conn);
+
+#ifdef HS_COMMON_PRIVATE
+
+STATIC void get_disaster_srv(uint64_t time_period_num, uint8_t *srv_out);
+
+/** The period for which a hidden service directory cannot be queried for
+ * the same descriptor ID again. */
+#define REND_HID_SERV_DIR_REQUERY_PERIOD (15 * 60)
+/** Test networks generate a new consensus every 5 or 10 seconds.
+ * So allow them to requery HSDirs much faster. */
+#define REND_HID_SERV_DIR_REQUERY_PERIOD_TESTING (5)
+
+#ifdef TOR_UNIT_TESTS
+
+STATIC strmap_t *get_last_hid_serv_requests(void);
+STATIC uint64_t get_time_period_length(void);
+
+STATIC uint8_t *get_first_cached_disaster_srv(void);
+STATIC uint8_t *get_second_cached_disaster_srv(void);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(HS_COMMON_PRIVATE) */
+
+#endif /* !defined(TOR_HS_COMMON_H) */
diff --git a/src/feature/hs/hs_config.c b/src/feature/hs/hs_config.c
new file mode 100644
index 0000000000..ee4499ef5b
--- /dev/null
+++ b/src/feature/hs/hs_config.c
@@ -0,0 +1,696 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_config.c
+ * \brief Implement hidden service configuration subsystem.
+ *
+ * \details
+ *
+ * This file has basically one main entry point: hs_config_service_all(). It
+ * takes the torrc options and configure hidden service from it. In validate
+ * mode, nothing is added to the global service list or keys are not generated
+ * nor loaded.
+ *
+ * A service is configured in two steps. It is first created using the tor
+ * options and then put in a staging list. It will stay there until
+ * hs_service_load_all_keys() is called. That function is responsible to
+ * load/generate the keys for the service in the staging list and if
+ * successful, transfert the service to the main global service list where
+ * at that point it is ready to be used.
+ *
+ * Configuration functions are per-version and there is a main generic one for
+ * every option that is common to all version (config_generic_service).
+ **/
+
+#define HS_CONFIG_PRIVATE
+
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_config.h"
+#include "feature/hs/hs_client.h"
+#include "feature/hs/hs_service.h"
+#include "feature/rend/rendclient.h"
+#include "feature/rend/rendservice.h"
+#include "lib/encoding/confline.h"
+#include "app/config/or_options_st.h"
+
+/* Using the given list of services, stage them into our global state. Every
+ * service version are handled. This function can remove entries in the given
+ * service_list.
+ *
+ * Staging a service means that we take all services in service_list and we
+ * put them in the staging list (global) which acts as a temporary list that
+ * is used by the service loading key process. In other words, staging a
+ * service puts it in a list to be considered when loading the keys and then
+ * moved to the main global list. */
+static void
+stage_services(smartlist_t *service_list)
+{
+  tor_assert(service_list);
+
+  /* This is v2 specific. Trigger service pruning which will make sure the
+   * just configured services end up in the main global list. It should only
+   * be done in non validation mode because v2 subsystem handles service
+   * object differently. */
+  rend_service_prune_list();
+
+  /* Cleanup v2 service from the list, we don't need those object anymore
+   * because we validated them all against the others and we want to stage
+   * only >= v3 service. And remember, v2 has a different object type which is
+   * shadow copied from an hs_service_t type. */
+  SMARTLIST_FOREACH_BEGIN(service_list, hs_service_t *, s) {
+    if (s->config.version == HS_VERSION_TWO) {
+      SMARTLIST_DEL_CURRENT(service_list, s);
+      hs_service_free(s);
+    }
+  } SMARTLIST_FOREACH_END(s);
+
+  /* This is >= v3 specific. Using the newly configured service list, stage
+   * them into our global state. Every object ownership is lost after. */
+  hs_service_stage_services(service_list);
+}
+
+/* Validate the given service against all service in the given list. If the
+ * service is ephemeral, this function ignores it. Services with the same
+ * directory path aren't allowed and will return an error. If a duplicate is
+ * found, 1 is returned else 0 if none found. */
+static int
+service_is_duplicate_in_list(const smartlist_t *service_list,
+                             const hs_service_t *service)
+{
+  int ret = 0;
+
+  tor_assert(service_list);
+  tor_assert(service);
+
+  /* Ephemeral service don't have a directory configured so no need to check
+   * for a service in the list having the same path. */
+  if (service->config.is_ephemeral) {
+    goto end;
+  }
+
+  /* XXX: Validate if we have any service that has the given service dir path.
+   * This has two problems:
+   *
+   * a) It's O(n^2), but the same comment from the bottom of
+   *    rend_config_services() should apply.
+   *
+   * b) We only compare directory paths as strings, so we can't
+   *    detect two distinct paths that specify the same directory
+   *    (which can arise from symlinks, case-insensitivity, bind
+   *    mounts, etc.).
+   *
+   * It also can't detect that two separate Tor instances are trying
+   * to use the same HiddenServiceDir; for that, we would need a
+   * lock file.  But this is enough to detect a simple mistake that
+   * at least one person has actually made. */
+  SMARTLIST_FOREACH_BEGIN(service_list, const hs_service_t *, s) {
+    if (!strcmp(s->config.directory_path, service->config.directory_path)) {
+      log_warn(LD_REND, "Another hidden service is already configured "
+                        "for directory %s",
+               escaped(service->config.directory_path));
+      ret = 1;
+      goto end;
+    }
+  } SMARTLIST_FOREACH_END(s);
+
+ end:
+  return ret;
+}
+
+/* Helper function: Given an configuration option name, its value, a minimum
+ * min and a maxium max, parse the value as a uint64_t. On success, ok is set
+ * to 1 and ret is the parsed value. On error, ok is set to 0 and ret must be
+ * ignored. This function logs both on error and success. */
+static uint64_t
+helper_parse_uint64(const char *opt, const char *value, uint64_t min,
+                    uint64_t max, int *ok)
+{
+  uint64_t ret = 0;
+
+  tor_assert(opt);
+  tor_assert(value);
+  tor_assert(ok);
+
+  *ok = 0;
+  ret = tor_parse_uint64(value, 10, min, max, ok, NULL);
+  if (!*ok) {
+    log_warn(LD_CONFIG, "%s must be between %" PRIu64 " and %"PRIu64
+                        ", not %s.",
+             opt, min, max, value);
+    goto err;
+  }
+  log_info(LD_CONFIG, "%s was parsed to %" PRIu64, opt, ret);
+ err:
+  return ret;
+}
+
+/** Helper function: Given a configuration option and its value, parse the
+ * value as a hs_circuit_id_protocol_t. On success, ok is set to 1 and ret is
+ * the parse value. On error, ok is set to 0 and the "none"
+ * hs_circuit_id_protocol_t is returned. This function logs on error. */
+static hs_circuit_id_protocol_t
+helper_parse_circuit_id_protocol(const char *key, const char *value, int *ok)
+{
+  tor_assert(value);
+  tor_assert(ok);
+
+  hs_circuit_id_protocol_t ret = HS_CIRCUIT_ID_PROTOCOL_NONE;
+  *ok = 0;
+
+  if (! strcasecmp(value, "haproxy")) {
+    *ok = 1;
+    ret = HS_CIRCUIT_ID_PROTOCOL_HAPROXY;
+  } else if (! strcasecmp(value, "none")) {
+    *ok = 1;
+    ret = HS_CIRCUIT_ID_PROTOCOL_NONE;
+  } else {
+    log_warn(LD_CONFIG, "%s must be 'haproxy' or 'none'.", key);
+    goto err;
+  }
+
+ err:
+  return ret;
+}
+
+/* Return the service version by trying to learn it from the key on disk if
+ * any. If nothing is found, the current service configured version is
+ * returned. */
+static int
+config_learn_service_version(hs_service_t *service)
+{
+  int version;
+
+  tor_assert(service);
+
+  version = hs_service_get_version_from_key(service);
+  if (version < 0) {
+    version = service->config.version;
+  }
+
+  return version;
+}
+
+/* Return true iff the given options starting at line_ for a hidden service
+ * contains at least one invalid option. Each hidden service option don't
+ * apply to all versions so this function can find out. The line_ MUST start
+ * right after the HiddenServiceDir line of this service.
+ *
+ * This is mainly for usability so we can inform the user of any invalid
+ * option for the hidden service version instead of silently ignoring. */
+static int
+config_has_invalid_options(const config_line_t *line_,
+                           const hs_service_t *service)
+{
+  int ret = 0;
+  const char **optlist;
+  const config_line_t *line;
+
+  tor_assert(service);
+  tor_assert(service->config.version <= HS_VERSION_MAX);
+
+  /* List of options that a v3 service doesn't support thus must exclude from
+   * its configuration. */
+  const char *opts_exclude_v3[] = {
+    "HiddenServiceAuthorizeClient",
+    NULL /* End marker. */
+  };
+
+  const char *opts_exclude_v2[] = {
+    "HiddenServiceExportCircuitID",
+    NULL /* End marker. */
+  };
+
+  /* Defining the size explicitly allows us to take advantage of the compiler
+   * which warns us if we ever bump the max version but forget to grow this
+   * array. The plus one is because we have a version 0 :). */
+  struct {
+    const char **list;
+  } exclude_lists[HS_VERSION_MAX + 1] = {
+    { NULL }, /* v0. */
+    { NULL }, /* v1. */
+    { opts_exclude_v2 }, /* v2 */
+    { opts_exclude_v3 }, /* v3. */
+  };
+
+  optlist = exclude_lists[service->config.version].list;
+  if (optlist == NULL) {
+    /* No exclude options to look at for this version. */
+    goto end;
+  }
+  for (int i = 0; optlist[i]; i++) {
+    const char *opt = optlist[i];
+    for (line = line_; line; line = line->next) {
+      if (!strcasecmp(line->key, "HiddenServiceDir")) {
+        /* We just hit the next hidden service, stop right now. */
+        goto end;
+      }
+      if (!strcasecmp(line->key, opt)) {
+        log_warn(LD_CONFIG, "Hidden service option %s is incompatible with "
+                            "version %" PRIu32 " of service in %s",
+                 opt, service->config.version,
+                 service->config.directory_path);
+        ret = 1;
+        /* Continue the loop so we can find all possible options. */
+        continue;
+      }
+    }
+  }
+ end:
+  return ret;
+}
+
+/* Validate service configuration. This is used when loading the configuration
+ * and once we've setup a service object, it's config object is passed to this
+ * function for further validation. This does not validate service key
+ * material. Return 0 if valid else -1 if invalid. */
+static int
+config_validate_service(const hs_service_config_t *config)
+{
+  tor_assert(config);
+
+  /* Amount of ports validation. */
+  if (!config->ports || smartlist_len(config->ports) == 0) {
+    log_warn(LD_CONFIG, "Hidden service (%s) with no ports configured.",
+             escaped(config->directory_path));
+    goto invalid;
+  }
+
+  /* Valid. */
+  return 0;
+ invalid:
+  return -1;
+}
+
+/* Configuration funcion for a version 3 service. The line_ must be pointing
+ * to the directive directly after a HiddenServiceDir. That way, when hitting
+ * the next HiddenServiceDir line or reaching the end of the list of lines, we
+ * know that we have to stop looking for more options. The given service
+ * object must be already allocated and passed through
+ * config_generic_service() prior to calling this function.
+ *
+ * Return 0 on success else a negative value. */
+static int
+config_service_v3(const config_line_t *line_,
+                  hs_service_config_t *config)
+{
+  int have_num_ip = 0;
+  bool export_circuit_id = false; /* just to detect duplicate options */
+  const char *dup_opt_seen = NULL;
+  const config_line_t *line;
+
+  tor_assert(config);
+
+  for (line = line_; line; line = line->next) {
+    int ok = 0;
+    if (!strcasecmp(line->key, "HiddenServiceDir")) {
+      /* We just hit the next hidden service, stop right now. */
+      break;
+    }
+    /* Number of introduction points. */
+    if (!strcasecmp(line->key, "HiddenServiceNumIntroductionPoints")) {
+      config->num_intro_points =
+        (unsigned int) helper_parse_uint64(line->key, line->value,
+                                           NUM_INTRO_POINTS_DEFAULT,
+                                           HS_CONFIG_V3_MAX_INTRO_POINTS,
+                                           &ok);
+      if (!ok || have_num_ip) {
+        if (have_num_ip)
+          dup_opt_seen = line->key;
+        goto err;
+      }
+      have_num_ip = 1;
+      continue;
+    }
+    if (!strcasecmp(line->key, "HiddenServiceExportCircuitID")) {
+      config->circuit_id_protocol =
+        helper_parse_circuit_id_protocol(line->key, line->value, &ok);
+      if (!ok || export_circuit_id) {
+        if (export_circuit_id) {
+          dup_opt_seen = line->key;
+        }
+        goto err;
+      }
+      export_circuit_id = true;
+      continue;
+    }
+  }
+
+  /* We do not load the key material for the service at this stage. This is
+   * done later once tor can confirm that it is in a running state. */
+
+  /* We are about to return a fully configured service so do one last pass of
+   * validation at it. */
+  if (config_validate_service(config) < 0) {
+    goto err;
+  }
+
+  return 0;
+ err:
+  if (dup_opt_seen) {
+    log_warn(LD_CONFIG, "Duplicate directive %s.", dup_opt_seen);
+  }
+  return -1;
+}
+
+/* Configure a service using the given options in line_ and options. This is
+ * called for any service regardless of its version which means that all
+ * directives in this function are generic to any service version. This
+ * function will also check the validity of the service directory path.
+ *
+ * The line_ must be pointing to the directive directly after a
+ * HiddenServiceDir. That way, when hitting the next HiddenServiceDir line or
+ * reaching the end of the list of lines, we know that we have to stop looking
+ * for more options.
+ *
+ * Return 0 on success else -1. */
+static int
+config_generic_service(const config_line_t *line_,
+                       const or_options_t *options,
+                       hs_service_t *service)
+{
+  int dir_seen = 0;
+  const config_line_t *line;
+  hs_service_config_t *config;
+  /* If this is set, we've seen a duplicate of this option. Keep the string
+   * so we can log the directive. */
+  const char *dup_opt_seen = NULL;
+  /* These variables will tell us if we ever have duplicate. */
+  int have_version = 0, have_allow_unknown_ports = 0;
+  int have_dir_group_read = 0, have_max_streams = 0;
+  int have_max_streams_close = 0;
+
+  tor_assert(line_);
+  tor_assert(options);
+  tor_assert(service);
+
+  /* Makes thing easier. */
+  config = &service->config;
+
+  /* The first line starts with HiddenServiceDir so we consider what's next is
+   * the configuration of the service. */
+  for (line = line_; line ; line = line->next) {
+    int ok = 0;
+
+    /* This indicate that we have a new service to configure. */
+    if (!strcasecmp(line->key, "HiddenServiceDir")) {
+      /* This function only configures one service at a time so if we've
+       * already seen one, stop right now. */
+      if (dir_seen) {
+        break;
+      }
+      /* Ok, we've seen one and we are about to configure it. */
+      dir_seen = 1;
+      config->directory_path = tor_strdup(line->value);
+      log_info(LD_CONFIG, "HiddenServiceDir=%s. Configuring...",
+               escaped(config->directory_path));
+      continue;
+    }
+    if (BUG(!dir_seen)) {
+      goto err;
+    }
+    /* Version of the service. */
+    if (!strcasecmp(line->key, "HiddenServiceVersion")) {
+      service->config.version =
+        (uint32_t) helper_parse_uint64(line->key, line->value, HS_VERSION_MIN,
+                                       HS_VERSION_MAX, &ok);
+      if (!ok || have_version) {
+        if (have_version)
+          dup_opt_seen = line->key;
+        goto err;
+      }
+      have_version = service->config.hs_version_explicitly_set = 1;
+      continue;
+    }
+    /* Virtual port. */
+    if (!strcasecmp(line->key, "HiddenServicePort")) {
+      char *err_msg = NULL;
+      /* XXX: Can we rename this? */
+      rend_service_port_config_t *portcfg =
+        rend_service_parse_port_config(line->value, " ", &err_msg);
+      if (!portcfg) {
+        if (err_msg) {
+          log_warn(LD_CONFIG, "%s", err_msg);
+        }
+        tor_free(err_msg);
+        goto err;
+      }
+      tor_assert(!err_msg);
+      smartlist_add(config->ports, portcfg);
+      log_info(LD_CONFIG, "HiddenServicePort=%s for %s",
+               line->value, escaped(config->directory_path));
+      continue;
+    }
+    /* Do we allow unknown ports. */
+    if (!strcasecmp(line->key, "HiddenServiceAllowUnknownPorts")) {
+      config->allow_unknown_ports =
+        (unsigned int) helper_parse_uint64(line->key, line->value, 0, 1, &ok);
+      if (!ok || have_allow_unknown_ports) {
+        if (have_allow_unknown_ports)
+          dup_opt_seen = line->key;
+        goto err;
+      }
+      have_allow_unknown_ports = 1;
+      continue;
+    }
+    /* Directory group readable. */
+    if (!strcasecmp(line->key, "HiddenServiceDirGroupReadable")) {
+      config->dir_group_readable =
+        (unsigned int) helper_parse_uint64(line->key, line->value, 0, 1, &ok);
+      if (!ok || have_dir_group_read) {
+        if (have_dir_group_read)
+          dup_opt_seen = line->key;
+        goto err;
+      }
+      have_dir_group_read = 1;
+      continue;
+    }
+    /* Maximum streams per circuit. */
+    if (!strcasecmp(line->key, "HiddenServiceMaxStreams")) {
+      config->max_streams_per_rdv_circuit =
+        helper_parse_uint64(line->key, line->value, 0,
+                            HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT, &ok);
+      if (!ok || have_max_streams) {
+        if (have_max_streams)
+          dup_opt_seen = line->key;
+        goto err;
+      }
+      have_max_streams = 1;
+      continue;
+    }
+    /* Maximum amount of streams before we close the circuit. */
+    if (!strcasecmp(line->key, "HiddenServiceMaxStreamsCloseCircuit")) {
+      config->max_streams_close_circuit =
+        (unsigned int) helper_parse_uint64(line->key, line->value, 0, 1, &ok);
+      if (!ok || have_max_streams_close) {
+        if (have_max_streams_close)
+          dup_opt_seen = line->key;
+        goto err;
+      }
+      have_max_streams_close = 1;
+      continue;
+    }
+  }
+
+  /* Check if we are configured in non anonymous mode meaning every service
+   * becomes a single onion service. */
+  if (rend_service_non_anonymous_mode_enabled(options)) {
+    config->is_single_onion = 1;
+    /* We will add support for IPv6-only v3 single onion services in a future
+     * Tor version. This won't catch "ReachableAddresses reject *4", but that
+     * option doesn't work anyway. */
+    if (options->ClientUseIPv4 == 0 && config->version == HS_VERSION_THREE) {
+      log_warn(LD_CONFIG, "IPv6-only v3 single onion services are not "
+               "supported. Set HiddenServiceSingleHopMode 0 and "
+               "HiddenServiceNonAnonymousMode 0, or set ClientUseIPv4 1.");
+      goto err;
+    }
+  }
+
+  /* Success */
+  return 0;
+ err:
+  if (dup_opt_seen) {
+    log_warn(LD_CONFIG, "Duplicate directive %s.", dup_opt_seen);
+  }
+  return -1;
+}
+
+/* Configure a service using the given line and options. This function will
+ * call the corresponding configuration function for a specific service
+ * version and validate the service against the other ones. On success, add
+ * the service to the given list and return 0. On error, nothing is added to
+ * the list and a negative value is returned. */
+static int
+config_service(const config_line_t *line, const or_options_t *options,
+               smartlist_t *service_list)
+{
+  int ret;
+  hs_service_t *service = NULL;
+
+  tor_assert(line);
+  tor_assert(options);
+  tor_assert(service_list);
+
+  /* We have a new hidden service. */
+  service = hs_service_new(options);
+
+  /* We'll configure that service as a generic one and then pass it to a
+   * specific function according to the configured version number. */
+  if (config_generic_service(line, options, service) < 0) {
+    goto err;
+  }
+
+  tor_assert(service->config.version <= HS_VERSION_MAX);
+
+  /* Check permission on service directory that was just parsed. And this must
+   * be done regardless of the service version. Do not ask for the directory
+   * to be created, this is done when the keys are loaded because we could be
+   * in validation mode right now. */
+  if (hs_check_service_private_dir(options->User,
+                                   service->config.directory_path,
+                                   service->config.dir_group_readable,
+                                   0) < 0) {
+    goto err;
+  }
+
+  /* We'll try to learn the service version here by loading the key(s) if
+   * present and we did not set HiddenServiceVersion. Depending on the key
+   * format, we can figure out the service version. */
+  if (!service->config.hs_version_explicitly_set) {
+    service->config.version = config_learn_service_version(service);
+  }
+
+  /* We make sure that this set of options for a service are valid that is for
+   * instance an option only for v2 is not used for v3. */
+  if (config_has_invalid_options(line->next, service)) {
+    goto err;
+  }
+
+  /* Different functions are in charge of specific options for a version. We
+   * start just after the service directory line so once we hit another
+   * directory line, the function knows that it has to stop parsing. */
+  switch (service->config.version) {
+  case HS_VERSION_TWO:
+    ret = rend_config_service(line->next, options, &service->config);
+    break;
+  case HS_VERSION_THREE:
+    ret = config_service_v3(line->next, &service->config);
+    break;
+  default:
+    /* We do validate before if we support the parsed version. */
+    tor_assert_nonfatal_unreached();
+    goto err;
+  }
+  if (ret < 0) {
+    goto err;
+  }
+
+  /* We'll check if this service can be kept depending on the others
+   * configured previously. */
+  if (service_is_duplicate_in_list(service_list, service)) {
+    goto err;
+  }
+
+  /* Passes, add it to the given list. */
+  smartlist_add(service_list, service);
+
+  return 0;
+
+ err:
+  hs_service_free(service);
+  return -1;
+}
+
+/* From a set of <b>options</b>, setup every hidden service found. Return 0 on
+ * success or -1 on failure. If <b>validate_only</b> is set, parse, warn and
+ * return as normal, but don't actually change the configured services. */
+int
+hs_config_service_all(const or_options_t *options, int validate_only)
+{
+  int dir_option_seen = 0, ret = -1;
+  const config_line_t *line;
+  smartlist_t *new_service_list = NULL;
+
+  tor_assert(options);
+
+  /* Newly configured service are put in that list which is then used for
+   * validation and staging for >= v3. */
+  new_service_list = smartlist_new();
+
+  for (line = options->RendConfigLines; line; line = line->next) {
+    /* Ignore all directives that aren't the start of a service. */
+    if (strcasecmp(line->key, "HiddenServiceDir")) {
+      if (!dir_option_seen) {
+        log_warn(LD_CONFIG, "%s with no preceding HiddenServiceDir directive",
+                 line->key);
+        goto err;
+      }
+      continue;
+    }
+    /* Flag that we've seen a directory directive and we'll use it to make
+     * sure that the torrc options ordering is actually valid. */
+    dir_option_seen = 1;
+
+    /* Try to configure this service now. On success, it will be added to the
+     * list and validated against the service in that same list. */
+    if (config_service(line, options, new_service_list) < 0) {
+      goto err;
+    }
+  }
+
+  /* In non validation mode, we'll stage those services we just successfully
+   * configured. Service ownership is transferred from the list to the global
+   * state. If any service is invalid, it will be removed from the list and
+   * freed. All versions are handled in that function. */
+  if (!validate_only) {
+    stage_services(new_service_list);
+  } else {
+    /* We've just validated that we were able to build a clean working list of
+     * services. We don't need those objects anymore. */
+    SMARTLIST_FOREACH(new_service_list, hs_service_t *, s,
+                      hs_service_free(s));
+    /* For the v2 subsystem, the configuration function adds the service
+     * object to the staging list and it is transferred in the main list
+     * through the prunning process. In validation mode, we thus have to purge
+     * the staging list so it's not kept in memory as valid service. */
+    rend_service_free_staging_list();
+  }
+
+  /* Success. Note that the service list has no ownership of its content. */
+  ret = 0;
+  goto end;
+
+ err:
+  SMARTLIST_FOREACH(new_service_list, hs_service_t *, s, hs_service_free(s));
+
+ end:
+  smartlist_free(new_service_list);
+  /* Tor main should call the free all function on error. */
+  return ret;
+}
+
+/* From a set of <b>options</b>, setup every client authorization found.
+ * Return 0 on success or -1 on failure. If <b>validate_only</b> is set,
+ * parse, warn and return as normal, but don't actually change the
+ * configured state. */
+int
+hs_config_client_auth_all(const or_options_t *options, int validate_only)
+{
+  int ret = -1;
+
+  /* Configure v2 authorization. */
+  if (rend_parse_service_authorization(options, validate_only) < 0) {
+    goto done;
+  }
+
+  /* Configure v3 authorization. */
+  if (hs_config_client_authorization(options, validate_only) < 0) {
+    goto done;
+  }
+
+  /* Success. */
+  ret = 0;
+ done:
+  return ret;
+}
diff --git a/src/feature/hs/hs_config.h b/src/feature/hs/hs_config.h
new file mode 100644
index 0000000000..040e451f13
--- /dev/null
+++ b/src/feature/hs/hs_config.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_config.h
+ * \brief Header file containing configuration ABI/API for the HS subsytem.
+ **/
+
+#ifndef TOR_HS_CONFIG_H
+#define TOR_HS_CONFIG_H
+
+#include "core/or/or.h"
+
+/* Max value for HiddenServiceMaxStreams */
+#define HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT 65535
+/* Maximum number of intro points per version 3 services. */
+#define HS_CONFIG_V3_MAX_INTRO_POINTS 20
+
+/* API */
+
+int hs_config_service_all(const or_options_t *options, int validate_only);
+int hs_config_client_auth_all(const or_options_t *options, int validate_only);
+
+#endif /* !defined(TOR_HS_CONFIG_H) */
+
diff --git a/src/feature/hs/hs_control.c b/src/feature/hs/hs_control.c
new file mode 100644
index 0000000000..9970fdd123
--- /dev/null
+++ b/src/feature/hs/hs_control.c
@@ -0,0 +1,261 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_control.c
+ * \brief Contains control port event related code.
+ **/
+
+#include "core/or/or.h"
+#include "feature/control/control.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_control.h"
+#include "feature/hs/hs_descriptor.h"
+#include "feature/hs/hs_service.h"
+#include "feature/nodelist/nodelist.h"
+
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+/* Send on the control port the "HS_DESC REQUESTED [...]" event.
+ *
+ * The onion_pk is the onion service public key, base64_blinded_pk is the
+ * base64 encoded blinded key for the service and hsdir_rs is the routerstatus
+ * object of the HSDir that this request is for. */
+void
+hs_control_desc_event_requested(const ed25519_public_key_t *onion_pk,
+                                const char *base64_blinded_pk,
+                                const routerstatus_t *hsdir_rs)
+{
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+  const uint8_t *hsdir_index;
+  const node_t *hsdir_node;
+
+  tor_assert(onion_pk);
+  tor_assert(base64_blinded_pk);
+  tor_assert(hsdir_rs);
+
+  hs_build_address(onion_pk, HS_VERSION_THREE, onion_address);
+
+  /* Get the node from the routerstatus object to get the HSDir index used for
+   * this request. We can't have a routerstatus entry without a node and we
+   * can't pick a node without an hsdir_index. */
+  hsdir_node = node_get_by_id(hsdir_rs->identity_digest);
+  tor_assert(hsdir_node);
+  /* This is a fetch event. */
+  hsdir_index = hsdir_node->hsdir_index.fetch;
+
+  /* Trigger the event. */
+  control_event_hs_descriptor_requested(onion_address, REND_NO_AUTH,
+                                        hsdir_rs->identity_digest,
+                                        base64_blinded_pk,
+                                        hex_str((const char *) hsdir_index,
+                                                DIGEST256_LEN));
+  memwipe(onion_address, 0, sizeof(onion_address));
+}
+
+/* Send on the control port the "HS_DESC FAILED [...]" event.
+ *
+ * Using a directory connection identifier, the HSDir identity digest and a
+ * reason for the failure. None can be NULL. */
+void
+hs_control_desc_event_failed(const hs_ident_dir_conn_t *ident,
+                             const char *hsdir_id_digest,
+                             const char *reason)
+{
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+  char base64_blinded_pk[ED25519_BASE64_LEN + 1];
+
+  tor_assert(ident);
+  tor_assert(hsdir_id_digest);
+  tor_assert(reason);
+
+  /* Build onion address and encoded blinded key. */
+  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk,
+                                       &ident->blinded_pk) < 0) {
+    return;
+  }
+  hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
+
+  control_event_hsv3_descriptor_failed(onion_address, base64_blinded_pk,
+                                       hsdir_id_digest, reason);
+}
+
+/* Send on the control port the "HS_DESC RECEIVED [...]" event.
+ *
+ * Using a directory connection identifier and the HSDir identity digest.
+ * None can be NULL. */
+void
+hs_control_desc_event_received(const hs_ident_dir_conn_t *ident,
+                               const char *hsdir_id_digest)
+{
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+  char base64_blinded_pk[ED25519_BASE64_LEN + 1];
+
+  tor_assert(ident);
+  tor_assert(hsdir_id_digest);
+
+  /* Build onion address and encoded blinded key. */
+  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk,
+                                       &ident->blinded_pk) < 0) {
+    return;
+  }
+  hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
+
+  control_event_hsv3_descriptor_received(onion_address, base64_blinded_pk,
+                                         hsdir_id_digest);
+}
+
+/* Send on the control port the "HS_DESC CREATED [...]" event.
+ *
+ * Using the onion address of the descriptor's service and the blinded public
+ * key of the descriptor as a descriptor ID. None can be NULL. */
+void
+hs_control_desc_event_created(const char *onion_address,
+                              const ed25519_public_key_t *blinded_pk)
+{
+  char base64_blinded_pk[ED25519_BASE64_LEN + 1];
+
+  tor_assert(onion_address);
+  tor_assert(blinded_pk);
+
+  /* Build base64 encoded blinded key. */
+  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk, blinded_pk) < 0) {
+    return;
+  }
+
+  /* Version 3 doesn't use the replica number in its descriptor ID computation
+   * so we pass negative value so the control port subsystem can ignore it. */
+  control_event_hs_descriptor_created(onion_address, base64_blinded_pk, -1);
+}
+
+/* Send on the control port the "HS_DESC UPLOAD [...]" event.
+ *
+ * Using the onion address of the descriptor's service, the HSDir identity
+ * digest, the blinded public key of the descriptor as a descriptor ID and the
+ * HSDir index for this particular request. None can be NULL. */
+void
+hs_control_desc_event_upload(const char *onion_address,
+                             const char *hsdir_id_digest,
+                             const ed25519_public_key_t *blinded_pk,
+                             const uint8_t *hsdir_index)
+{
+  char base64_blinded_pk[ED25519_BASE64_LEN + 1];
+
+  tor_assert(onion_address);
+  tor_assert(hsdir_id_digest);
+  tor_assert(blinded_pk);
+  tor_assert(hsdir_index);
+
+  /* Build base64 encoded blinded key. */
+  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk, blinded_pk) < 0) {
+    return;
+  }
+
+  control_event_hs_descriptor_upload(onion_address, hsdir_id_digest,
+                                     base64_blinded_pk,
+                                     hex_str((const char *) hsdir_index,
+                                             DIGEST256_LEN));
+}
+
+/* Send on the control port the "HS_DESC UPLOADED [...]" event.
+ *
+ * Using the directory connection identifier and the HSDir identity digest.
+ * None can be NULL. */
+void
+hs_control_desc_event_uploaded(const hs_ident_dir_conn_t *ident,
+                               const char *hsdir_id_digest)
+{
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+
+  tor_assert(ident);
+  tor_assert(hsdir_id_digest);
+
+  hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
+
+  control_event_hs_descriptor_uploaded(hsdir_id_digest, onion_address);
+}
+
+/* Send on the control port the "HS_DESC_CONTENT [...]" event.
+ *
+ * Using the directory connection identifier, the HSDir identity digest and
+ * the body of the descriptor (as it was received from the directory). None
+ * can be NULL. */
+void
+hs_control_desc_event_content(const hs_ident_dir_conn_t *ident,
+                              const char *hsdir_id_digest,
+                              const char *body)
+{
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+  char base64_blinded_pk[ED25519_BASE64_LEN + 1];
+
+  tor_assert(ident);
+  tor_assert(hsdir_id_digest);
+
+  /* Build onion address and encoded blinded key. */
+  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk,
+                                       &ident->blinded_pk) < 0) {
+    return;
+  }
+  hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
+
+  control_event_hs_descriptor_content(onion_address, base64_blinded_pk,
+                                      hsdir_id_digest, body);
+}
+
+/* Handle the "HSPOST [...]" command. The body is an encoded descriptor for
+ * the given onion_address. The descriptor will be uploaded to each directory
+ * in hsdirs_rs. If NULL, the responsible directories for the current time
+ * period will be selected.
+ *
+ * Return -1 on if the descriptor plaintext section is not decodable. Else, 0
+ * on success. */
+int
+hs_control_hspost_command(const char *body, const char *onion_address,
+                          const smartlist_t *hsdirs_rs)
+{
+  int ret = -1;
+  ed25519_public_key_t identity_pk;
+  hs_desc_plaintext_data_t plaintext;
+  smartlist_t *hsdirs = NULL;
+
+  tor_assert(body);
+  tor_assert(onion_address);
+
+  /* This can't fail because we require the caller to pass us a valid onion
+   * address that has passed hs_address_is_valid(). */
+  if (BUG(hs_parse_address(onion_address, &identity_pk, NULL, NULL) < 0)) {
+    goto done; // LCOV_EXCL_LINE
+  }
+
+  /* Only decode the plaintext part which is what the directory will do to
+   * validate before caching. */
+  if (hs_desc_decode_plaintext(body, &plaintext) < 0) {
+    goto done;
+  }
+
+  /* No HSDir(s) given, we'll compute what the current ones should be. */
+  if (hsdirs_rs == NULL) {
+    hsdirs = smartlist_new();
+    hs_get_responsible_hsdirs(&plaintext.blinded_pubkey,
+                              hs_get_time_period_num(0),
+                              0, /* Always the current descriptor which uses
+                                  * the first hsdir index. */
+                              0, /* It is for storing on a directory. */
+                              hsdirs);
+    hsdirs_rs = hsdirs;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(hsdirs_rs, const routerstatus_t *, rs) {
+    hs_service_upload_desc_to_dir(body, plaintext.version, &identity_pk,
+                                  &plaintext.blinded_pubkey, rs);
+  } SMARTLIST_FOREACH_END(rs);
+  ret = 0;
+
+ done:
+  /* We don't have ownership of the objects in this list. */
+  smartlist_free(hsdirs);
+  return ret;
+}
diff --git a/src/feature/hs/hs_control.h b/src/feature/hs/hs_control.h
new file mode 100644
index 0000000000..f7ab642652
--- /dev/null
+++ b/src/feature/hs/hs_control.h
@@ -0,0 +1,52 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_control.h
+ * \brief Header file containing control port event related code.
+ **/
+
+#ifndef TOR_HS_CONTROL_H
+#define TOR_HS_CONTROL_H
+
+#include "feature/hs/hs_ident.h"
+
+/* Event "HS_DESC REQUESTED [...]" */
+void hs_control_desc_event_requested(const ed25519_public_key_t *onion_pk,
+                                     const char *base64_blinded_pk,
+                                     const routerstatus_t *hsdir_rs);
+
+/* Event "HS_DESC FAILED [...]" */
+void hs_control_desc_event_failed(const hs_ident_dir_conn_t *ident,
+                                  const char *hsdir_id_digest,
+                                  const char *reason);
+
+/* Event "HS_DESC RECEIVED [...]" */
+void hs_control_desc_event_received(const hs_ident_dir_conn_t *ident,
+                                    const char *hsdir_id_digest);
+
+/* Event "HS_DESC CREATED [...]" */
+void hs_control_desc_event_created(const char *onion_address,
+                                   const ed25519_public_key_t *blinded_pk);
+
+/* Event "HS_DESC UPLOAD [...]" */
+void hs_control_desc_event_upload(const char *onion_address,
+                                  const char *hsdir_id_digest,
+                                  const ed25519_public_key_t *blinded_pk,
+                                  const uint8_t *hsdir_index);
+
+/* Event "HS_DESC UPLOADED [...]" */
+void hs_control_desc_event_uploaded(const hs_ident_dir_conn_t *ident,
+                                    const char *hsdir_id_digest);
+
+/* Event "HS_DESC_CONTENT [...]" */
+void hs_control_desc_event_content(const hs_ident_dir_conn_t *ident,
+                                   const char *hsdir_id_digest,
+                                   const char *body);
+
+/* Command "HSPOST [...]" */
+int hs_control_hspost_command(const char *body, const char *onion_address,
+                              const smartlist_t *hsdirs_rs);
+
+#endif /* !defined(TOR_HS_CONTROL_H) */
+
diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c
new file mode 100644
index 0000000000..b6abf14a11
--- /dev/null
+++ b/src/feature/hs/hs_descriptor.c
@@ -0,0 +1,3073 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_descriptor.c
+ * \brief Handle hidden service descriptor encoding/decoding.
+ *
+ * \details
+ * Here is a graphical depiction of an HS descriptor and its layers:
+ *
+ *      +------------------------------------------------------+
+ *      |DESCRIPTOR HEADER:                                    |
+ *      |  hs-descriptor 3                                     |
+ *      |  descriptor-lifetime 180                             |
+ *      |  ...                                                 |
+ *      |  superencrypted                                      |
+ *      |+---------------------------------------------------+ |
+ *      ||SUPERENCRYPTED LAYER (aka OUTER ENCRYPTED LAYER):  | |
+ *      ||  desc-auth-type x25519                            | |
+ *      ||  desc-auth-ephemeral-key                          | |
+ *      ||  auth-client                                      | |
+ *      ||  auth-client                                      | |
+ *      ||  ...                                              | |
+ *      ||  encrypted                                        | |
+ *      ||+-------------------------------------------------+| |
+ *      |||ENCRYPTED LAYER (aka INNER ENCRYPTED LAYER):     || |
+ *      |||  create2-formats                                || |
+ *      |||  intro-auth-required                            || |
+ *      |||  introduction-point                             || |
+ *      |||  introduction-point                             || |
+ *      |||  ...                                            || |
+ *      ||+-------------------------------------------------+| |
+ *      |+---------------------------------------------------+ |
+ *      +------------------------------------------------------+
+ *
+ * The DESCRIPTOR HEADER section is completely unencrypted and contains generic
+ * descriptor metadata.
+ *
+ * The SUPERENCRYPTED LAYER section is the first layer of encryption, and it's
+ * encrypted using the blinded public key of the hidden service to protect
+ * against entities who don't know its onion address. The clients of the hidden
+ * service know its onion address and blinded public key, whereas third-parties
+ * (like HSDirs) don't know it (except if it's a public hidden service).
+ *
+ * The ENCRYPTED LAYER section is the second layer of encryption, and it's
+ * encrypted using the client authorization key material (if those exist). When
+ * client authorization is enabled, this second layer of encryption protects
+ * the descriptor content from unauthorized entities. If client authorization
+ * is disabled, this second layer of encryption does not provide any extra
+ * security but is still present. The plaintext of this layer contains all the
+ * information required to connect to the hidden service like its list of
+ * introduction points.
+ **/
+
+/* For unit tests.*/
+#define HS_DESCRIPTOR_PRIVATE
+
+#include "core/or/or.h"
+#include "trunnel/ed25519_cert.h" /* Trunnel interface. */
+#include "feature/hs/hs_descriptor.h"
+#include "core/or/circuitbuild.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "feature/dirparse/parsecommon.h"
+#include "feature/rend/rendcache.h"
+#include "feature/hs/hs_cache.h"
+#include "feature/hs/hs_config.h"
+#include "feature/nodelist/torcert.h" /* tor_cert_encode_ed22519() */
+#include "lib/memarea/memarea.h"
+#include "lib/crypt_ops/crypto_format.h"
+
+#include "core/or/extend_info_st.h"
+
+/* Constant string value used for the descriptor format. */
+#define str_hs_desc "hs-descriptor"
+#define str_desc_cert "descriptor-signing-key-cert"
+#define str_rev_counter "revision-counter"
+#define str_superencrypted "superencrypted"
+#define str_encrypted "encrypted"
+#define str_signature "signature"
+#define str_lifetime "descriptor-lifetime"
+/* Constant string value for the encrypted part of the descriptor. */
+#define str_create2_formats "create2-formats"
+#define str_intro_auth_required "intro-auth-required"
+#define str_single_onion "single-onion-service"
+#define str_intro_point "introduction-point"
+#define str_ip_onion_key "onion-key"
+#define str_ip_auth_key "auth-key"
+#define str_ip_enc_key "enc-key"
+#define str_ip_enc_key_cert "enc-key-cert"
+#define str_ip_legacy_key "legacy-key"
+#define str_ip_legacy_key_cert "legacy-key-cert"
+#define str_intro_point_start "\n" str_intro_point " "
+/* Constant string value for the construction to encrypt the encrypted data
+ * section. */
+#define str_enc_const_superencryption "hsdir-superencrypted-data"
+#define str_enc_const_encryption "hsdir-encrypted-data"
+/* Prefix required to compute/verify HS desc signatures */
+#define str_desc_sig_prefix "Tor onion service descriptor sig v3"
+#define str_desc_auth_type "desc-auth-type"
+#define str_desc_auth_key "desc-auth-ephemeral-key"
+#define str_desc_auth_client "auth-client"
+#define str_encrypted "encrypted"
+
+/* Authentication supported types. */
+static const struct {
+  hs_desc_auth_type_t type;
+  const char *identifier;
+} intro_auth_types[] = {
+  { HS_DESC_AUTH_ED25519, "ed25519" },
+  /* Indicate end of array. */
+  { 0, NULL }
+};
+
+/* Descriptor ruleset. */
+static token_rule_t hs_desc_v3_token_table[] = {
+  T1_START(str_hs_desc, R_HS_DESCRIPTOR, EQ(1), NO_OBJ),
+  T1(str_lifetime, R3_DESC_LIFETIME, EQ(1), NO_OBJ),
+  T1(str_desc_cert, R3_DESC_SIGNING_CERT, NO_ARGS, NEED_OBJ),
+  T1(str_rev_counter, R3_REVISION_COUNTER, EQ(1), NO_OBJ),
+  T1(str_superencrypted, R3_SUPERENCRYPTED, NO_ARGS, NEED_OBJ),
+  T1_END(str_signature, R3_SIGNATURE, EQ(1), NO_OBJ),
+  END_OF_TABLE
+};
+
+/* Descriptor ruleset for the superencrypted section. */
+static token_rule_t hs_desc_superencrypted_v3_token_table[] = {
+  T1_START(str_desc_auth_type, R3_DESC_AUTH_TYPE, GE(1), NO_OBJ),
+  T1(str_desc_auth_key, R3_DESC_AUTH_KEY, GE(1), NO_OBJ),
+  T1N(str_desc_auth_client, R3_DESC_AUTH_CLIENT, GE(3), NO_OBJ),
+  T1(str_encrypted, R3_ENCRYPTED, NO_ARGS, NEED_OBJ),
+  END_OF_TABLE
+};
+
+/* Descriptor ruleset for the encrypted section. */
+static token_rule_t hs_desc_encrypted_v3_token_table[] = {
+  T1_START(str_create2_formats, R3_CREATE2_FORMATS, CONCAT_ARGS, NO_OBJ),
+  T01(str_intro_auth_required, R3_INTRO_AUTH_REQUIRED, ARGS, NO_OBJ),
+  T01(str_single_onion, R3_SINGLE_ONION_SERVICE, ARGS, NO_OBJ),
+  END_OF_TABLE
+};
+
+/* Descriptor ruleset for the introduction points section. */
+static token_rule_t hs_desc_intro_point_v3_token_table[] = {
+  T1_START(str_intro_point, R3_INTRODUCTION_POINT, EQ(1), NO_OBJ),
+  T1N(str_ip_onion_key, R3_INTRO_ONION_KEY, GE(2), OBJ_OK),
+  T1(str_ip_auth_key, R3_INTRO_AUTH_KEY, NO_ARGS, NEED_OBJ),
+  T1(str_ip_enc_key, R3_INTRO_ENC_KEY, GE(2), OBJ_OK),
+  T1(str_ip_enc_key_cert, R3_INTRO_ENC_KEY_CERT, ARGS, OBJ_OK),
+  T01(str_ip_legacy_key, R3_INTRO_LEGACY_KEY, ARGS, NEED_KEY_1024),
+  T01(str_ip_legacy_key_cert, R3_INTRO_LEGACY_KEY_CERT, ARGS, OBJ_OK),
+  END_OF_TABLE
+};
+
+/* Using a key, salt and encrypted payload, build a MAC and put it in mac_out.
+ * We use SHA3-256 for the MAC computation.
+ * This function can't fail. */
+static void
+build_mac(const uint8_t *mac_key, size_t mac_key_len,
+          const uint8_t *salt, size_t salt_len,
+          const uint8_t *encrypted, size_t encrypted_len,
+          uint8_t *mac_out, size_t mac_len)
+{
+  crypto_digest_t *digest;
+
+  const uint64_t mac_len_netorder = tor_htonll(mac_key_len);
+  const uint64_t salt_len_netorder = tor_htonll(salt_len);
+
+  tor_assert(mac_key);
+  tor_assert(salt);
+  tor_assert(encrypted);
+  tor_assert(mac_out);
+
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+  /* As specified in section 2.5 of proposal 224, first add the mac key
+   * then add the salt first and then the encrypted section. */
+
+  crypto_digest_add_bytes(digest, (const char *) &mac_len_netorder, 8);
+  crypto_digest_add_bytes(digest, (const char *) mac_key, mac_key_len);
+  crypto_digest_add_bytes(digest, (const char *) &salt_len_netorder, 8);
+  crypto_digest_add_bytes(digest, (const char *) salt, salt_len);
+  crypto_digest_add_bytes(digest, (const char *) encrypted, encrypted_len);
+  crypto_digest_get_digest(digest, (char *) mac_out, mac_len);
+  crypto_digest_free(digest);
+}
+
+/* Using a secret data and a given decriptor object, build the secret
+ * input needed for the KDF.
+ *
+ * secret_input = SECRET_DATA | subcredential | INT_8(revision_counter)
+ *
+ * Then, set the newly allocated buffer in secret_input_out and return the
+ * length of the buffer. */
+static size_t
+build_secret_input(const hs_descriptor_t *desc,
+                   const uint8_t *secret_data,
+                   size_t secret_data_len,
+                   uint8_t **secret_input_out)
+{
+  size_t offset = 0;
+  size_t secret_input_len = secret_data_len + DIGEST256_LEN + sizeof(uint64_t);
+  uint8_t *secret_input = NULL;
+
+  tor_assert(desc);
+  tor_assert(secret_data);
+  tor_assert(secret_input_out);
+
+  secret_input = tor_malloc_zero(secret_input_len);
+
+  /* Copy the secret data. */
+  memcpy(secret_input, secret_data, secret_data_len);
+  offset += secret_data_len;
+  /* Copy subcredential. */
+  memcpy(secret_input + offset, desc->subcredential, DIGEST256_LEN);
+  offset += DIGEST256_LEN;
+  /* Copy revision counter value. */
+  set_uint64(secret_input + offset,
+             tor_htonll(desc->plaintext_data.revision_counter));
+  offset += sizeof(uint64_t);
+  tor_assert(secret_input_len == offset);
+
+  *secret_input_out = secret_input;
+
+  return secret_input_len;
+}
+
+/* Do the KDF construction and put the resulting data in key_out which is of
+ * key_out_len length. It uses SHAKE-256 as specified in the spec. */
+static void
+build_kdf_key(const hs_descriptor_t *desc,
+              const uint8_t *secret_data,
+              size_t secret_data_len,
+              const uint8_t *salt, size_t salt_len,
+              uint8_t *key_out, size_t key_out_len,
+              int is_superencrypted_layer)
+{
+  uint8_t *secret_input = NULL;
+  size_t secret_input_len;
+  crypto_xof_t *xof;
+
+  tor_assert(desc);
+  tor_assert(secret_data);
+  tor_assert(salt);
+  tor_assert(key_out);
+
+  /* Build the secret input for the KDF computation. */
+  secret_input_len = build_secret_input(desc, secret_data,
+                                        secret_data_len, &secret_input);
+
+  xof = crypto_xof_new();
+  /* Feed our KDF. [SHAKE it like a polaroid picture --Yawning]. */
+  crypto_xof_add_bytes(xof, secret_input, secret_input_len);
+  crypto_xof_add_bytes(xof, salt, salt_len);
+
+  /* Feed in the right string constant based on the desc layer */
+  if (is_superencrypted_layer) {
+    crypto_xof_add_bytes(xof, (const uint8_t *) str_enc_const_superencryption,
+                         strlen(str_enc_const_superencryption));
+  } else {
+    crypto_xof_add_bytes(xof, (const uint8_t *) str_enc_const_encryption,
+                         strlen(str_enc_const_encryption));
+  }
+
+  /* Eat from our KDF. */
+  crypto_xof_squeeze_bytes(xof, key_out, key_out_len);
+  crypto_xof_free(xof);
+  memwipe(secret_input,  0, secret_input_len);
+
+  tor_free(secret_input);
+}
+
+/* Using the given descriptor, secret data, and salt, run it through our
+ * KDF function and then extract a secret key in key_out, the IV in iv_out
+ * and MAC in mac_out. This function can't fail. */
+static void
+build_secret_key_iv_mac(const hs_descriptor_t *desc,
+                        const uint8_t *secret_data,
+                        size_t secret_data_len,
+                        const uint8_t *salt, size_t salt_len,
+                        uint8_t *key_out, size_t key_len,
+                        uint8_t *iv_out, size_t iv_len,
+                        uint8_t *mac_out, size_t mac_len,
+                        int is_superencrypted_layer)
+{
+  size_t offset = 0;
+  uint8_t kdf_key[HS_DESC_ENCRYPTED_KDF_OUTPUT_LEN];
+
+  tor_assert(desc);
+  tor_assert(secret_data);
+  tor_assert(salt);
+  tor_assert(key_out);
+  tor_assert(iv_out);
+  tor_assert(mac_out);
+
+  build_kdf_key(desc, secret_data, secret_data_len,
+                salt, salt_len, kdf_key, sizeof(kdf_key),
+                is_superencrypted_layer);
+  /* Copy the bytes we need for both the secret key and IV. */
+  memcpy(key_out, kdf_key, key_len);
+  offset += key_len;
+  memcpy(iv_out, kdf_key + offset, iv_len);
+  offset += iv_len;
+  memcpy(mac_out, kdf_key + offset, mac_len);
+  /* Extra precaution to make sure we are not out of bound. */
+  tor_assert((offset + mac_len) == sizeof(kdf_key));
+  memwipe(kdf_key, 0, sizeof(kdf_key));
+}
+
+/* === ENCODING === */
+
+/* Encode the given link specifier objects into a newly allocated string.
+ * This can't fail so caller can always assume a valid string being
+ * returned. */
+STATIC char *
+encode_link_specifiers(const smartlist_t *specs)
+{
+  char *encoded_b64 = NULL;
+  link_specifier_list_t *lslist = link_specifier_list_new();
+
+  tor_assert(specs);
+  /* No link specifiers is a code flow error, can't happen. */
+  tor_assert(smartlist_len(specs) > 0);
+  tor_assert(smartlist_len(specs) <= UINT8_MAX);
+
+  link_specifier_list_set_n_spec(lslist, smartlist_len(specs));
+
+  SMARTLIST_FOREACH_BEGIN(specs, const hs_desc_link_specifier_t *,
+                          spec) {
+    link_specifier_t *ls = hs_desc_lspec_to_trunnel(spec);
+    if (ls) {
+      link_specifier_list_add_spec(lslist, ls);
+    }
+  } SMARTLIST_FOREACH_END(spec);
+
+  {
+    uint8_t *encoded;
+    ssize_t encoded_len, encoded_b64_len, ret;
+
+    encoded_len = link_specifier_list_encoded_len(lslist);
+    tor_assert(encoded_len > 0);
+    encoded = tor_malloc_zero(encoded_len);
+    ret = link_specifier_list_encode(encoded, encoded_len, lslist);
+    tor_assert(ret == encoded_len);
+
+    /* Base64 encode our binary format. Add extra NUL byte for the base64
+     * encoded value. */
+    encoded_b64_len = base64_encode_size(encoded_len, 0) + 1;
+    encoded_b64 = tor_malloc_zero(encoded_b64_len);
+    ret = base64_encode(encoded_b64, encoded_b64_len, (const char *) encoded,
+                        encoded_len, 0);
+    tor_assert(ret == (encoded_b64_len - 1));
+    tor_free(encoded);
+  }
+
+  link_specifier_list_free(lslist);
+  return encoded_b64;
+}
+
+/* Encode an introduction point legacy key and certificate. Return a newly
+ * allocated string with it. On failure, return NULL. */
+static char *
+encode_legacy_key(const hs_desc_intro_point_t *ip)
+{
+  char *key_str, b64_cert[256], *encoded = NULL;
+  size_t key_str_len;
+
+  tor_assert(ip);
+
+  /* Encode cross cert. */
+  if (base64_encode(b64_cert, sizeof(b64_cert),
+                    (const char *) ip->legacy.cert.encoded,
+                    ip->legacy.cert.len, BASE64_ENCODE_MULTILINE) < 0) {
+    log_warn(LD_REND, "Unable to encode legacy crosscert.");
+    goto done;
+  }
+  /* Convert the encryption key to PEM format NUL terminated. */
+  if (crypto_pk_write_public_key_to_string(ip->legacy.key, &key_str,
+                                           &key_str_len) < 0) {
+    log_warn(LD_REND, "Unable to encode legacy encryption key.");
+    goto done;
+  }
+  tor_asprintf(&encoded,
+               "%s \n%s"  /* Newline is added by the call above. */
+               "%s\n"
+               "-----BEGIN CROSSCERT-----\n"
+               "%s"
+               "-----END CROSSCERT-----",
+               str_ip_legacy_key, key_str,
+               str_ip_legacy_key_cert, b64_cert);
+  tor_free(key_str);
+
+ done:
+  return encoded;
+}
+
+/* Encode an introduction point encryption key and certificate. Return a newly
+ * allocated string with it. On failure, return NULL. */
+static char *
+encode_enc_key(const hs_desc_intro_point_t *ip)
+{
+  char *encoded = NULL, *encoded_cert;
+  char key_b64[CURVE25519_BASE64_PADDED_LEN + 1];
+
+  tor_assert(ip);
+
+  /* Base64 encode the encryption key for the "enc-key" field. */
+  if (curve25519_public_to_base64(key_b64, &ip->enc_key) < 0) {
+    goto done;
+  }
+  if (tor_cert_encode_ed22519(ip->enc_key_cert, &encoded_cert) < 0) {
+    goto done;
+  }
+  tor_asprintf(&encoded,
+               "%s ntor %s\n"
+               "%s\n%s",
+               str_ip_enc_key, key_b64,
+               str_ip_enc_key_cert, encoded_cert);
+  tor_free(encoded_cert);
+
+ done:
+  return encoded;
+}
+
+/* Encode an introduction point onion key. Return a newly allocated string
+ * with it. On failure, return NULL. */
+static char *
+encode_onion_key(const hs_desc_intro_point_t *ip)
+{
+  char *encoded = NULL;
+  char key_b64[CURVE25519_BASE64_PADDED_LEN + 1];
+
+  tor_assert(ip);
+
+  /* Base64 encode the encryption key for the "onion-key" field. */
+  if (curve25519_public_to_base64(key_b64, &ip->onion_key) < 0) {
+    goto done;
+  }
+  tor_asprintf(&encoded, "%s ntor %s", str_ip_onion_key, key_b64);
+
+ done:
+  return encoded;
+}
+
+/* Encode an introduction point object and return a newly allocated string
+ * with it. On failure, return NULL. */
+static char *
+encode_intro_point(const ed25519_public_key_t *sig_key,
+                   const hs_desc_intro_point_t *ip)
+{
+  char *encoded_ip = NULL;
+  smartlist_t *lines = smartlist_new();
+
+  tor_assert(ip);
+  tor_assert(sig_key);
+
+  /* Encode link specifier. */
+  {
+    char *ls_str = encode_link_specifiers(ip->link_specifiers);
+    smartlist_add_asprintf(lines, "%s %s", str_intro_point, ls_str);
+    tor_free(ls_str);
+  }
+
+  /* Onion key encoding. */
+  {
+    char *encoded_onion_key = encode_onion_key(ip);
+    if (encoded_onion_key == NULL) {
+      goto err;
+    }
+    smartlist_add_asprintf(lines, "%s", encoded_onion_key);
+    tor_free(encoded_onion_key);
+  }
+
+  /* Authentication key encoding. */
+  {
+    char *encoded_cert;
+    if (tor_cert_encode_ed22519(ip->auth_key_cert, &encoded_cert) < 0) {
+      goto err;
+    }
+    smartlist_add_asprintf(lines, "%s\n%s", str_ip_auth_key, encoded_cert);
+    tor_free(encoded_cert);
+  }
+
+  /* Encryption key encoding. */
+  {
+    char *encoded_enc_key = encode_enc_key(ip);
+    if (encoded_enc_key == NULL) {
+      goto err;
+    }
+    smartlist_add_asprintf(lines, "%s", encoded_enc_key);
+    tor_free(encoded_enc_key);
+  }
+
+  /* Legacy key if any. */
+  if (ip->legacy.key != NULL) {
+    /* Strong requirement else the IP creation was badly done. */
+    tor_assert(ip->legacy.cert.encoded);
+    char *encoded_legacy_key = encode_legacy_key(ip);
+    if (encoded_legacy_key == NULL) {
+      goto err;
+    }
+    smartlist_add_asprintf(lines, "%s", encoded_legacy_key);
+    tor_free(encoded_legacy_key);
+  }
+
+  /* Join them all in one blob of text. */
+  encoded_ip = smartlist_join_strings(lines, "\n", 1, NULL);
+
+ err:
+  SMARTLIST_FOREACH(lines, char *, l, tor_free(l));
+  smartlist_free(lines);
+  return encoded_ip;
+}
+
+/* Given a source length, return the new size including padding for the
+ * plaintext encryption. */
+static size_t
+compute_padded_plaintext_length(size_t plaintext_len)
+{
+  size_t plaintext_padded_len;
+  const int padding_block_length = HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE;
+
+  /* Make sure we won't overflow. */
+  tor_assert(plaintext_len <= (SIZE_T_CEILING - padding_block_length));
+
+  /* Get the extra length we need to add. For example, if srclen is 10200
+   * bytes, this will expand to (2 * 10k) == 20k thus an extra 9800 bytes. */
+  plaintext_padded_len = CEIL_DIV(plaintext_len, padding_block_length) *
+                         padding_block_length;
+  /* Can never be extra careful. Make sure we are _really_ padded. */
+  tor_assert(!(plaintext_padded_len % padding_block_length));
+  return plaintext_padded_len;
+}
+
+/* Given a buffer, pad it up to the encrypted section padding requirement. Set
+ * the newly allocated string in padded_out and return the length of the
+ * padded buffer. */
+STATIC size_t
+build_plaintext_padding(const char *plaintext, size_t plaintext_len,
+                        uint8_t **padded_out)
+{
+  size_t padded_len;
+  uint8_t *padded;
+
+  tor_assert(plaintext);
+  tor_assert(padded_out);
+
+  /* Allocate the final length including padding. */
+  padded_len = compute_padded_plaintext_length(plaintext_len);
+  tor_assert(padded_len >= plaintext_len);
+  padded = tor_malloc_zero(padded_len);
+
+  memcpy(padded, plaintext, plaintext_len);
+  *padded_out = padded;
+  return padded_len;
+}
+
+/* Using a key, IV and plaintext data of length plaintext_len, create the
+ * encrypted section by encrypting it and setting encrypted_out with the
+ * data. Return size of the encrypted data buffer. */
+static size_t
+build_encrypted(const uint8_t *key, const uint8_t *iv, const char *plaintext,
+                size_t plaintext_len, uint8_t **encrypted_out,
+                int is_superencrypted_layer)
+{
+  size_t encrypted_len;
+  uint8_t *padded_plaintext, *encrypted;
+  crypto_cipher_t *cipher;
+
+  tor_assert(key);
+  tor_assert(iv);
+  tor_assert(plaintext);
+  tor_assert(encrypted_out);
+
+  /* If we are encrypting the middle layer of the descriptor, we need to first
+     pad the plaintext */
+  if (is_superencrypted_layer) {
+    encrypted_len = build_plaintext_padding(plaintext, plaintext_len,
+                                            &padded_plaintext);
+    /* Extra precautions that we have a valid padding length. */
+    tor_assert(!(encrypted_len % HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE));
+  } else { /* No padding required for inner layers */
+    padded_plaintext = tor_memdup(plaintext, plaintext_len);
+    encrypted_len = plaintext_len;
+  }
+
+  /* This creates a cipher for AES. It can't fail. */
+  cipher = crypto_cipher_new_with_iv_and_bits(key, iv,
+                                              HS_DESC_ENCRYPTED_BIT_SIZE);
+  /* We use a stream cipher so the encrypted length will be the same as the
+   * plaintext padded length. */
+  encrypted = tor_malloc_zero(encrypted_len);
+  /* This can't fail. */
+  crypto_cipher_encrypt(cipher, (char *) encrypted,
+                        (const char *) padded_plaintext, encrypted_len);
+  *encrypted_out = encrypted;
+  /* Cleanup. */
+  crypto_cipher_free(cipher);
+  tor_free(padded_plaintext);
+  return encrypted_len;
+}
+
+/* Encrypt the given <b>plaintext</b> buffer using <b>desc</b> and
+ * <b>secret_data</b> to get the keys. Set encrypted_out with the encrypted
+ * data and return the length of it. <b>is_superencrypted_layer</b> is set
+ * if this is the outer encrypted layer of the descriptor. */
+static size_t
+encrypt_descriptor_data(const hs_descriptor_t *desc,
+                        const uint8_t *secret_data,
+                        size_t secret_data_len,
+                        const char *plaintext,
+                        char **encrypted_out, int is_superencrypted_layer)
+{
+  char *final_blob;
+  size_t encrypted_len, final_blob_len, offset = 0;
+  uint8_t *encrypted;
+  uint8_t salt[HS_DESC_ENCRYPTED_SALT_LEN];
+  uint8_t secret_key[HS_DESC_ENCRYPTED_KEY_LEN], secret_iv[CIPHER_IV_LEN];
+  uint8_t mac_key[DIGEST256_LEN], mac[DIGEST256_LEN];
+
+  tor_assert(desc);
+  tor_assert(secret_data);
+  tor_assert(plaintext);
+  tor_assert(encrypted_out);
+
+  /* Get our salt. The returned bytes are already hashed. */
+  crypto_strongest_rand(salt, sizeof(salt));
+
+  /* KDF construction resulting in a key from which the secret key, IV and MAC
+   * key are extracted which is what we need for the encryption. */
+  build_secret_key_iv_mac(desc, secret_data, secret_data_len,
+                          salt, sizeof(salt),
+                          secret_key, sizeof(secret_key),
+                          secret_iv, sizeof(secret_iv),
+                          mac_key, sizeof(mac_key),
+                          is_superencrypted_layer);
+
+  /* Build the encrypted part that is do the actual encryption. */
+  encrypted_len = build_encrypted(secret_key, secret_iv, plaintext,
+                                  strlen(plaintext), &encrypted,
+                                  is_superencrypted_layer);
+  memwipe(secret_key, 0, sizeof(secret_key));
+  memwipe(secret_iv, 0, sizeof(secret_iv));
+  /* This construction is specified in section 2.5 of proposal 224. */
+  final_blob_len = sizeof(salt) + encrypted_len + DIGEST256_LEN;
+  final_blob = tor_malloc_zero(final_blob_len);
+
+  /* Build the MAC. */
+  build_mac(mac_key, sizeof(mac_key), salt, sizeof(salt),
+            encrypted, encrypted_len, mac, sizeof(mac));
+  memwipe(mac_key, 0, sizeof(mac_key));
+
+  /* The salt is the first value. */
+  memcpy(final_blob, salt, sizeof(salt));
+  offset = sizeof(salt);
+  /* Second value is the encrypted data. */
+  memcpy(final_blob + offset, encrypted, encrypted_len);
+  offset += encrypted_len;
+  /* Third value is the MAC. */
+  memcpy(final_blob + offset, mac, sizeof(mac));
+  offset += sizeof(mac);
+  /* Cleanup the buffers. */
+  memwipe(salt, 0, sizeof(salt));
+  memwipe(encrypted, 0, encrypted_len);
+  tor_free(encrypted);
+  /* Extra precaution. */
+  tor_assert(offset == final_blob_len);
+
+  *encrypted_out = final_blob;
+  return final_blob_len;
+}
+
+/* Create and return a string containing a client-auth entry. It's the
+ * responsibility of the caller to free the returned string. This function
+ * will never fail. */
+static char *
+get_auth_client_str(const hs_desc_authorized_client_t *client)
+{
+  int ret;
+  char *auth_client_str = NULL;
+  /* We are gonna fill these arrays with base64 data. They are all double
+   * the size of their binary representation to fit the base64 overhead. */
+  char client_id_b64[HS_DESC_CLIENT_ID_LEN * 2];
+  char iv_b64[CIPHER_IV_LEN * 2];
+  char encrypted_cookie_b64[HS_DESC_ENCRYPED_COOKIE_LEN * 2];
+
+#define ASSERT_AND_BASE64(field) STMT_BEGIN                        \
+  tor_assert(!tor_mem_is_zero((char *) client->field,              \
+                              sizeof(client->field)));             \
+  ret = base64_encode_nopad(field##_b64, sizeof(field##_b64),      \
+                            client->field, sizeof(client->field)); \
+  tor_assert(ret > 0);                                             \
+  STMT_END
+
+  ASSERT_AND_BASE64(client_id);
+  ASSERT_AND_BASE64(iv);
+  ASSERT_AND_BASE64(encrypted_cookie);
+
+  /* Build the final string */
+  tor_asprintf(&auth_client_str, "%s %s %s %s", str_desc_auth_client,
+               client_id_b64, iv_b64, encrypted_cookie_b64);
+
+#undef ASSERT_AND_BASE64
+
+  return auth_client_str;
+}
+
+/** Create the "client-auth" part of the descriptor and return a
+ *  newly-allocated string with it. It's the responsibility of the caller to
+ *  free the returned string. */
+static char *
+get_all_auth_client_lines(const hs_descriptor_t *desc)
+{
+  smartlist_t *auth_client_lines = smartlist_new();
+  char *auth_client_lines_str = NULL;
+
+  tor_assert(desc);
+  tor_assert(desc->superencrypted_data.clients);
+  tor_assert(smartlist_len(desc->superencrypted_data.clients) != 0);
+  tor_assert(smartlist_len(desc->superencrypted_data.clients)
+                                 % HS_DESC_AUTH_CLIENT_MULTIPLE == 0);
+
+  /* Make a line for each client */
+  SMARTLIST_FOREACH_BEGIN(desc->superencrypted_data.clients,
+                          const hs_desc_authorized_client_t *, client) {
+    char *auth_client_str = NULL;
+
+    auth_client_str = get_auth_client_str(client);
+
+    smartlist_add(auth_client_lines, auth_client_str);
+  } SMARTLIST_FOREACH_END(client);
+
+  /* Join all lines together to form final string */
+  auth_client_lines_str = smartlist_join_strings(auth_client_lines,
+                                                 "\n", 1, NULL);
+  /* Cleanup the mess */
+  SMARTLIST_FOREACH(auth_client_lines, char *, a, tor_free(a));
+  smartlist_free(auth_client_lines);
+
+  return auth_client_lines_str;
+}
+
+/* Create the inner layer of the descriptor (which includes the intro points,
+ * etc.). Return a newly-allocated string with the layer plaintext, or NULL if
+ * an error occurred. It's the responsibility of the caller to free the
+ * returned string. */
+static char *
+get_inner_encrypted_layer_plaintext(const hs_descriptor_t *desc)
+{
+  char *encoded_str = NULL;
+  smartlist_t *lines = smartlist_new();
+
+  /* Build the start of the section prior to the introduction points. */
+  {
+    if (!desc->encrypted_data.create2_ntor) {
+      log_err(LD_BUG, "HS desc doesn't have recognized handshake type.");
+      goto err;
+    }
+    smartlist_add_asprintf(lines, "%s %d\n", str_create2_formats,
+                           ONION_HANDSHAKE_TYPE_NTOR);
+
+    if (desc->encrypted_data.intro_auth_types &&
+        smartlist_len(desc->encrypted_data.intro_auth_types)) {
+      /* Put the authentication-required line. */
+      char *buf = smartlist_join_strings(desc->encrypted_data.intro_auth_types,
+                                         " ", 0, NULL);
+      smartlist_add_asprintf(lines, "%s %s\n", str_intro_auth_required, buf);
+      tor_free(buf);
+    }
+
+    if (desc->encrypted_data.single_onion_service) {
+      smartlist_add_asprintf(lines, "%s\n", str_single_onion);
+    }
+  }
+
+  /* Build the introduction point(s) section. */
+  SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                          const hs_desc_intro_point_t *, ip) {
+    char *encoded_ip = encode_intro_point(&desc->plaintext_data.signing_pubkey,
+                                          ip);
+    if (encoded_ip == NULL) {
+      log_err(LD_BUG, "HS desc intro point is malformed.");
+      goto err;
+    }
+    smartlist_add(lines, encoded_ip);
+  } SMARTLIST_FOREACH_END(ip);
+
+  /* Build the entire encrypted data section into one encoded plaintext and
+   * then encrypt it. */
+  encoded_str = smartlist_join_strings(lines, "", 0, NULL);
+
+ err:
+  SMARTLIST_FOREACH(lines, char *, l, tor_free(l));
+  smartlist_free(lines);
+
+  return encoded_str;
+}
+
+/* Create the middle layer of the descriptor, which includes the client auth
+ * data and the encrypted inner layer (provided as a base64 string at
+ * <b>layer2_b64_ciphertext</b>). Return a newly-allocated string with the
+ * layer plaintext, or NULL if an error occurred. It's the responsibility of
+ * the caller to free the returned string. */
+static char *
+get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc,
+                                    const char *layer2_b64_ciphertext)
+{
+  char *layer1_str = NULL;
+  smartlist_t *lines = smartlist_new();
+
+  /* Specify auth type */
+  smartlist_add_asprintf(lines, "%s %s\n", str_desc_auth_type, "x25519");
+
+  {  /* Print ephemeral x25519 key */
+    char ephemeral_key_base64[CURVE25519_BASE64_PADDED_LEN + 1];
+    const curve25519_public_key_t *ephemeral_pubkey;
+
+    ephemeral_pubkey = &desc->superencrypted_data.auth_ephemeral_pubkey;
+    tor_assert(!tor_mem_is_zero((char *) ephemeral_pubkey->public_key,
+                                CURVE25519_PUBKEY_LEN));
+
+    if (curve25519_public_to_base64(ephemeral_key_base64,
+                                    ephemeral_pubkey) < 0) {
+      goto done;
+    }
+    smartlist_add_asprintf(lines, "%s %s\n",
+                           str_desc_auth_key, ephemeral_key_base64);
+
+    memwipe(ephemeral_key_base64, 0, sizeof(ephemeral_key_base64));
+  }
+
+  {  /* Create auth-client lines. */
+    char *auth_client_lines = get_all_auth_client_lines(desc);
+    tor_assert(auth_client_lines);
+    smartlist_add(lines, auth_client_lines);
+  }
+
+  /* create encrypted section */
+  {
+    smartlist_add_asprintf(lines,
+                           "%s\n"
+                           "-----BEGIN MESSAGE-----\n"
+                           "%s"
+                           "-----END MESSAGE-----",
+                           str_encrypted, layer2_b64_ciphertext);
+  }
+
+  layer1_str = smartlist_join_strings(lines, "", 0, NULL);
+
+ done:
+  /* We need to memwipe all lines because it contains the ephemeral key */
+  SMARTLIST_FOREACH(lines, char *, a, memwipe(a, 0, strlen(a)));
+  SMARTLIST_FOREACH(lines, char *, a, tor_free(a));
+  smartlist_free(lines);
+
+  return layer1_str;
+}
+
+/* Encrypt <b>encoded_str</b> into an encrypted blob and then base64 it before
+ * returning it. <b>desc</b> is provided to derive the encryption
+ * keys. <b>secret_data</b> is also proved to derive the encryption keys.
+ * <b>is_superencrypted_layer</b> is set if <b>encoded_str</b> is the
+ * middle (superencrypted) layer of the descriptor. It's the responsibility of
+ * the caller to free the returned string. */
+static char *
+encrypt_desc_data_and_base64(const hs_descriptor_t *desc,
+                             const uint8_t *secret_data,
+                             size_t secret_data_len,
+                             const char *encoded_str,
+                             int is_superencrypted_layer)
+{
+  char *enc_b64;
+  ssize_t enc_b64_len, ret_len, enc_len;
+  char *encrypted_blob = NULL;
+
+  enc_len = encrypt_descriptor_data(desc, secret_data, secret_data_len,
+                                    encoded_str, &encrypted_blob,
+                                    is_superencrypted_layer);
+  /* Get the encoded size plus a NUL terminating byte. */
+  enc_b64_len = base64_encode_size(enc_len, BASE64_ENCODE_MULTILINE) + 1;
+  enc_b64 = tor_malloc_zero(enc_b64_len);
+  /* Base64 the encrypted blob before returning it. */
+  ret_len = base64_encode(enc_b64, enc_b64_len, encrypted_blob, enc_len,
+                          BASE64_ENCODE_MULTILINE);
+  /* Return length doesn't count the NUL byte. */
+  tor_assert(ret_len == (enc_b64_len - 1));
+  tor_free(encrypted_blob);
+
+  return enc_b64;
+}
+
+/* Generate the secret data which is used to encrypt/decrypt the descriptor.
+ *
+ * SECRET_DATA = blinded-public-key
+ * SECRET_DATA = blinded-public-key | descriptor_cookie
+ *
+ * The descriptor_cookie is optional but if it exists, it must be at least
+ * HS_DESC_DESCRIPTOR_COOKIE_LEN bytes long.
+ *
+ * A newly allocated secret data is put in secret_data_out. Return the
+ * length of the secret data. This function cannot fail. */
+static size_t
+build_secret_data(const ed25519_public_key_t *blinded_pubkey,
+                  const uint8_t *descriptor_cookie,
+                  uint8_t **secret_data_out)
+{
+  size_t secret_data_len;
+  uint8_t *secret_data;
+
+  tor_assert(blinded_pubkey);
+  tor_assert(secret_data_out);
+
+  if (descriptor_cookie) {
+    /* If the descriptor cookie is present, we need both the blinded
+     * pubkey and the descriptor cookie as a secret data. */
+    secret_data_len = ED25519_PUBKEY_LEN + HS_DESC_DESCRIPTOR_COOKIE_LEN;
+    secret_data = tor_malloc(secret_data_len);
+
+    memcpy(secret_data,
+           blinded_pubkey->pubkey,
+           ED25519_PUBKEY_LEN);
+    memcpy(secret_data + ED25519_PUBKEY_LEN,
+           descriptor_cookie,
+           HS_DESC_DESCRIPTOR_COOKIE_LEN);
+  } else {
+    /* If the descriptor cookie is not present, we need only the blinded
+     * pubkey as a secret data. */
+    secret_data_len = ED25519_PUBKEY_LEN;
+    secret_data = tor_malloc(secret_data_len);
+    memcpy(secret_data,
+           blinded_pubkey->pubkey,
+           ED25519_PUBKEY_LEN);
+  }
+
+  *secret_data_out = secret_data;
+  return secret_data_len;
+}
+
+/* Generate and encode the superencrypted portion of <b>desc</b>. This also
+ * involves generating the encrypted portion of the descriptor, and performing
+ * the superencryption. A newly allocated NUL-terminated string pointer
+ * containing the encrypted encoded blob is put in encrypted_blob_out. Return 0
+ * on success else a negative value. */
+static int
+encode_superencrypted_data(const hs_descriptor_t *desc,
+                           const uint8_t *descriptor_cookie,
+                           char **encrypted_blob_out)
+{
+  int ret = -1;
+  uint8_t *secret_data = NULL;
+  size_t secret_data_len = 0;
+  char *layer2_str = NULL;
+  char *layer2_b64_ciphertext = NULL;
+  char *layer1_str = NULL;
+  char *layer1_b64_ciphertext = NULL;
+
+  tor_assert(desc);
+  tor_assert(encrypted_blob_out);
+
+  /* Func logic: We first create the inner layer of the descriptor (layer2).
+   * We then encrypt it and use it to create the middle layer of the descriptor
+   * (layer1).  Finally we superencrypt the middle layer and return it to our
+   * caller. */
+
+  /* Create inner descriptor layer */
+  layer2_str = get_inner_encrypted_layer_plaintext(desc);
+  if (!layer2_str) {
+    goto err;
+  }
+
+  secret_data_len = build_secret_data(&desc->plaintext_data.blinded_pubkey,
+                                      descriptor_cookie,
+                                      &secret_data);
+
+  /* Encrypt and b64 the inner layer */
+  layer2_b64_ciphertext =
+    encrypt_desc_data_and_base64(desc, secret_data, secret_data_len,
+                                 layer2_str, 0);
+  if (!layer2_b64_ciphertext) {
+    goto err;
+  }
+
+  /* Now create middle descriptor layer given the inner layer */
+  layer1_str = get_outer_encrypted_layer_plaintext(desc,layer2_b64_ciphertext);
+  if (!layer1_str) {
+    goto err;
+  }
+
+  /* Encrypt and base64 the middle layer */
+  layer1_b64_ciphertext =
+    encrypt_desc_data_and_base64(desc,
+                                 desc->plaintext_data.blinded_pubkey.pubkey,
+                                 ED25519_PUBKEY_LEN,
+                                 layer1_str, 1);
+  if (!layer1_b64_ciphertext) {
+    goto err;
+  }
+
+  /* Success! */
+  ret = 0;
+
+ err:
+  memwipe(secret_data, 0, secret_data_len);
+  tor_free(secret_data);
+  tor_free(layer1_str);
+  tor_free(layer2_str);
+  tor_free(layer2_b64_ciphertext);
+
+  *encrypted_blob_out = layer1_b64_ciphertext;
+  return ret;
+}
+
+/* Encode a v3 HS descriptor. Return 0 on success and set encoded_out to the
+ * newly allocated string of the encoded descriptor. On error, -1 is returned
+ * and encoded_out is untouched. */
+static int
+desc_encode_v3(const hs_descriptor_t *desc,
+               const ed25519_keypair_t *signing_kp,
+               const uint8_t *descriptor_cookie,
+               char **encoded_out)
+{
+  int ret = -1;
+  char *encoded_str = NULL;
+  size_t encoded_len;
+  smartlist_t *lines = smartlist_new();
+
+  tor_assert(desc);
+  tor_assert(signing_kp);
+  tor_assert(encoded_out);
+  tor_assert(desc->plaintext_data.version == 3);
+
+  if (BUG(desc->subcredential == NULL)) {
+    goto err;
+  }
+
+  /* Build the non-encrypted values. */
+  {
+    char *encoded_cert;
+    /* Encode certificate then create the first line of the descriptor. */
+    if (desc->plaintext_data.signing_key_cert->cert_type
+        != CERT_TYPE_SIGNING_HS_DESC) {
+      log_err(LD_BUG, "HS descriptor signing key has an unexpected cert type "
+              "(%d)", (int) desc->plaintext_data.signing_key_cert->cert_type);
+      goto err;
+    }
+    if (tor_cert_encode_ed22519(desc->plaintext_data.signing_key_cert,
+                                &encoded_cert) < 0) {
+      /* The function will print error logs. */
+      goto err;
+    }
+    /* Create the hs descriptor line. */
+    smartlist_add_asprintf(lines, "%s %" PRIu32, str_hs_desc,
+                           desc->plaintext_data.version);
+    /* Add the descriptor lifetime line (in minutes). */
+    smartlist_add_asprintf(lines, "%s %" PRIu32, str_lifetime,
+                           desc->plaintext_data.lifetime_sec / 60);
+    /* Create the descriptor certificate line. */
+    smartlist_add_asprintf(lines, "%s\n%s", str_desc_cert, encoded_cert);
+    tor_free(encoded_cert);
+    /* Create the revision counter line. */
+    smartlist_add_asprintf(lines, "%s %" PRIu64, str_rev_counter,
+                           desc->plaintext_data.revision_counter);
+  }
+
+  /* Build the superencrypted data section. */
+  {
+    char *enc_b64_blob=NULL;
+    if (encode_superencrypted_data(desc, descriptor_cookie,
+                                   &enc_b64_blob) < 0) {
+      goto err;
+    }
+    smartlist_add_asprintf(lines,
+                           "%s\n"
+                           "-----BEGIN MESSAGE-----\n"
+                           "%s"
+                           "-----END MESSAGE-----",
+                           str_superencrypted, enc_b64_blob);
+    tor_free(enc_b64_blob);
+  }
+
+  /* Join all lines in one string so we can generate a signature and append
+   * it to the descriptor. */
+  encoded_str = smartlist_join_strings(lines, "\n", 1, &encoded_len);
+
+  /* Sign all fields of the descriptor with our short term signing key. */
+  {
+    ed25519_signature_t sig;
+    char ed_sig_b64[ED25519_SIG_BASE64_LEN + 1];
+    if (ed25519_sign_prefixed(&sig,
+                              (const uint8_t *) encoded_str, encoded_len,
+                              str_desc_sig_prefix, signing_kp) < 0) {
+      log_warn(LD_BUG, "Can't sign encoded HS descriptor!");
+      tor_free(encoded_str);
+      goto err;
+    }
+    if (ed25519_signature_to_base64(ed_sig_b64, &sig) < 0) {
+      log_warn(LD_BUG, "Can't base64 encode descriptor signature!");
+      tor_free(encoded_str);
+      goto err;
+    }
+    /* Create the signature line. */
+    smartlist_add_asprintf(lines, "%s %s", str_signature, ed_sig_b64);
+  }
+  /* Free previous string that we used so compute the signature. */
+  tor_free(encoded_str);
+  encoded_str = smartlist_join_strings(lines, "\n", 1, NULL);
+  *encoded_out = encoded_str;
+
+  if (strlen(encoded_str) >= hs_cache_get_max_descriptor_size()) {
+    log_warn(LD_GENERAL, "We just made an HS descriptor that's too big (%d)."
+             "Failing.", (int)strlen(encoded_str));
+    tor_free(encoded_str);
+    goto err;
+  }
+
+  /* XXX: Trigger a control port event. */
+
+  /* Success! */
+  ret = 0;
+
+ err:
+  SMARTLIST_FOREACH(lines, char *, l, tor_free(l));
+  smartlist_free(lines);
+  return ret;
+}
+
+/* === DECODING === */
+
+/* Given the token tok for an auth client, decode it as
+ * hs_desc_authorized_client_t. tok->args MUST contain at least 3 elements
+ * Return 0 on success else -1 on failure. */
+static int
+decode_auth_client(const directory_token_t *tok,
+                   hs_desc_authorized_client_t *client)
+{
+  int ret = -1;
+
+  tor_assert(tok);
+  tor_assert(tok->n_args >= 3);
+  tor_assert(client);
+
+  if (base64_decode((char *) client->client_id, sizeof(client->client_id),
+                    tok->args[0], strlen(tok->args[0])) !=
+      sizeof(client->client_id)) {
+    goto done;
+  }
+  if (base64_decode((char *) client->iv, sizeof(client->iv),
+                    tok->args[1], strlen(tok->args[1])) !=
+      sizeof(client->iv)) {
+    goto done;
+  }
+  if (base64_decode((char *) client->encrypted_cookie,
+                    sizeof(client->encrypted_cookie),
+                    tok->args[2], strlen(tok->args[2])) !=
+      sizeof(client->encrypted_cookie)) {
+    goto done;
+  }
+
+  /* Success. */
+  ret = 0;
+ done:
+  return ret;
+}
+
+/* Given an encoded string of the link specifiers, return a newly allocated
+ * list of decoded link specifiers. Return NULL on error. */
+STATIC smartlist_t *
+decode_link_specifiers(const char *encoded)
+{
+  int decoded_len;
+  size_t encoded_len, i;
+  uint8_t *decoded;
+  smartlist_t *results = NULL;
+  link_specifier_list_t *specs = NULL;
+
+  tor_assert(encoded);
+
+  encoded_len = strlen(encoded);
+  decoded = tor_malloc(encoded_len);
+  decoded_len = base64_decode((char *) decoded, encoded_len, encoded,
+                              encoded_len);
+  if (decoded_len < 0) {
+    goto err;
+  }
+
+  if (link_specifier_list_parse(&specs, decoded,
+                                (size_t) decoded_len) < decoded_len) {
+    goto err;
+  }
+  tor_assert(specs);
+  results = smartlist_new();
+
+  for (i = 0; i < link_specifier_list_getlen_spec(specs); i++) {
+    hs_desc_link_specifier_t *hs_spec;
+    link_specifier_t *ls = link_specifier_list_get_spec(specs, i);
+    tor_assert(ls);
+
+    hs_spec = tor_malloc_zero(sizeof(*hs_spec));
+    hs_spec->type = link_specifier_get_ls_type(ls);
+    switch (hs_spec->type) {
+    case LS_IPV4:
+      tor_addr_from_ipv4h(&hs_spec->u.ap.addr,
+                          link_specifier_get_un_ipv4_addr(ls));
+      hs_spec->u.ap.port = link_specifier_get_un_ipv4_port(ls);
+      break;
+    case LS_IPV6:
+      tor_addr_from_ipv6_bytes(&hs_spec->u.ap.addr, (const char *)
+                               link_specifier_getarray_un_ipv6_addr(ls));
+      hs_spec->u.ap.port = link_specifier_get_un_ipv6_port(ls);
+      break;
+    case LS_LEGACY_ID:
+      /* Both are known at compile time so let's make sure they are the same
+       * else we can copy memory out of bound. */
+      tor_assert(link_specifier_getlen_un_legacy_id(ls) ==
+                 sizeof(hs_spec->u.legacy_id));
+      memcpy(hs_spec->u.legacy_id, link_specifier_getarray_un_legacy_id(ls),
+             sizeof(hs_spec->u.legacy_id));
+      break;
+    case LS_ED25519_ID:
+      /* Both are known at compile time so let's make sure they are the same
+       * else we can copy memory out of bound. */
+      tor_assert(link_specifier_getlen_un_ed25519_id(ls) ==
+                 sizeof(hs_spec->u.ed25519_id));
+      memcpy(hs_spec->u.ed25519_id,
+             link_specifier_getconstarray_un_ed25519_id(ls),
+             sizeof(hs_spec->u.ed25519_id));
+      break;
+    default:
+      tor_free(hs_spec);
+      goto err;
+    }
+
+    smartlist_add(results, hs_spec);
+  }
+
+  goto done;
+ err:
+  if (results) {
+    SMARTLIST_FOREACH(results, hs_desc_link_specifier_t *, s, tor_free(s));
+    smartlist_free(results);
+    results = NULL;
+  }
+ done:
+  link_specifier_list_free(specs);
+  tor_free(decoded);
+  return results;
+}
+
+/* Given a list of authentication types, decode it and put it in the encrypted
+ * data section. Return 1 if we at least know one of the type or 0 if we know
+ * none of them. */
+static int
+decode_auth_type(hs_desc_encrypted_data_t *desc, const char *list)
+{
+  int match = 0;
+
+  tor_assert(desc);
+  tor_assert(list);
+
+  desc->intro_auth_types = smartlist_new();
+  smartlist_split_string(desc->intro_auth_types, list, " ", 0, 0);
+
+  /* Validate the types that we at least know about one. */
+  SMARTLIST_FOREACH_BEGIN(desc->intro_auth_types, const char *, auth) {
+    for (int idx = 0; intro_auth_types[idx].identifier; idx++) {
+      if (!strncmp(auth, intro_auth_types[idx].identifier,
+                   strlen(intro_auth_types[idx].identifier))) {
+        match = 1;
+        break;
+      }
+    }
+  } SMARTLIST_FOREACH_END(auth);
+
+  return match;
+}
+
+/* Parse a space-delimited list of integers representing CREATE2 formats into
+ * the bitfield in hs_desc_encrypted_data_t. Ignore unrecognized values. */
+static void
+decode_create2_list(hs_desc_encrypted_data_t *desc, const char *list)
+{
+  smartlist_t *tokens;
+
+  tor_assert(desc);
+  tor_assert(list);
+
+  tokens = smartlist_new();
+  smartlist_split_string(tokens, list, " ", 0, 0);
+
+  SMARTLIST_FOREACH_BEGIN(tokens, char *, s) {
+    int ok;
+    unsigned long type = tor_parse_ulong(s, 10, 1, UINT16_MAX, &ok, NULL);
+    if (!ok) {
+      log_warn(LD_REND, "Unparseable value %s in create2 list", escaped(s));
+      continue;
+    }
+    switch (type) {
+    case ONION_HANDSHAKE_TYPE_NTOR:
+      desc->create2_ntor = 1;
+      break;
+    default:
+      /* We deliberately ignore unsupported handshake types */
+      continue;
+    }
+  } SMARTLIST_FOREACH_END(s);
+
+  SMARTLIST_FOREACH(tokens, char *, s, tor_free(s));
+  smartlist_free(tokens);
+}
+
+/* Given a certificate, validate the certificate for certain conditions which
+ * are if the given type matches the cert's one, if the signing key is
+ * included and if the that key was actually used to sign the certificate.
+ *
+ * Return 1 iff if all conditions pass or 0 if one of them fails. */
+STATIC int
+cert_is_valid(tor_cert_t *cert, uint8_t type, const char *log_obj_type)
+{
+  tor_assert(log_obj_type);
+
+  if (cert == NULL) {
+    log_warn(LD_REND, "Certificate for %s couldn't be parsed.", log_obj_type);
+    goto err;
+  }
+  if (cert->cert_type != type) {
+    log_warn(LD_REND, "Invalid cert type %02x for %s.", cert->cert_type,
+             log_obj_type);
+    goto err;
+  }
+  /* All certificate must have its signing key included. */
+  if (!cert->signing_key_included) {
+    log_warn(LD_REND, "Signing key is NOT included for %s.", log_obj_type);
+    goto err;
+  }
+  /* The following will not only check if the signature matches but also the
+   * expiration date and overall validity. */
+  if (tor_cert_checksig(cert, &cert->signing_key, approx_time()) < 0) {
+    log_warn(LD_REND, "Invalid signature for %s: %s", log_obj_type,
+             tor_cert_describe_signature_status(cert));
+    goto err;
+  }
+
+  return 1;
+ err:
+  return 0;
+}
+
+/* Given some binary data, try to parse it to get a certificate object. If we
+ * have a valid cert, validate it using the given wanted type. On error, print
+ * a log using the err_msg has the certificate identifier adding semantic to
+ * the log and cert_out is set to NULL. On success, 0 is returned and cert_out
+ * points to a newly allocated certificate object. */
+static int
+cert_parse_and_validate(tor_cert_t **cert_out, const char *data,
+                        size_t data_len, unsigned int cert_type_wanted,
+                        const char *err_msg)
+{
+  tor_cert_t *cert;
+
+  tor_assert(cert_out);
+  tor_assert(data);
+  tor_assert(err_msg);
+
+  /* Parse certificate. */
+  cert = tor_cert_parse((const uint8_t *) data, data_len);
+  if (!cert) {
+    log_warn(LD_REND, "Certificate for %s couldn't be parsed.", err_msg);
+    goto err;
+  }
+
+  /* Validate certificate. */
+  if (!cert_is_valid(cert, cert_type_wanted, err_msg)) {
+    goto err;
+  }
+
+  *cert_out = cert;
+  return 0;
+
+ err:
+  tor_cert_free(cert);
+  *cert_out = NULL;
+  return -1;
+}
+
+/* Return true iff the given length of the encrypted data of a descriptor
+ * passes validation. */
+STATIC int
+encrypted_data_length_is_valid(size_t len)
+{
+  /* Make sure there is enough data for the salt and the mac. The equality is
+     there to ensure that there is at least one byte of encrypted data. */
+  if (len <= HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN) {
+    log_warn(LD_REND, "Length of descriptor's encrypted data is too small. "
+                      "Got %lu but minimum value is %d",
+             (unsigned long)len, HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN);
+    goto err;
+  }
+
+  return 1;
+ err:
+  return 0;
+}
+
+/* Decrypt the descriptor cookie given the descriptor, the auth client,
+ * and the client secret key. On sucess, return 0 and a newly allocated
+ * descriptor cookie descriptor_cookie_out. On error or if the client id
+ * is invalid, return -1 and descriptor_cookie_out is set to
+ * NULL. */
+static int
+decrypt_descriptor_cookie(const hs_descriptor_t *desc,
+                          const hs_desc_authorized_client_t *client,
+                          const curve25519_secret_key_t *client_auth_sk,
+                          uint8_t **descriptor_cookie_out)
+{
+  int ret = -1;
+  uint8_t secret_seed[CURVE25519_OUTPUT_LEN];
+  uint8_t keystream[HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN];
+  uint8_t *cookie_key = NULL;
+  uint8_t *descriptor_cookie = NULL;
+  crypto_cipher_t *cipher = NULL;
+  crypto_xof_t *xof = NULL;
+
+  tor_assert(desc);
+  tor_assert(client);
+  tor_assert(client_auth_sk);
+  tor_assert(!tor_mem_is_zero(
+        (char *) &desc->superencrypted_data.auth_ephemeral_pubkey,
+        sizeof(desc->superencrypted_data.auth_ephemeral_pubkey)));
+  tor_assert(!tor_mem_is_zero((char *) client_auth_sk,
+                              sizeof(*client_auth_sk)));
+  tor_assert(!tor_mem_is_zero((char *) desc->subcredential, DIGEST256_LEN));
+
+  /* Calculate x25519(client_x, hs_Y) */
+  curve25519_handshake(secret_seed, client_auth_sk,
+                       &desc->superencrypted_data.auth_ephemeral_pubkey);
+
+  /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */
+  xof = crypto_xof_new();
+  crypto_xof_add_bytes(xof, desc->subcredential, DIGEST256_LEN);
+  crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed));
+  crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream));
+  crypto_xof_free(xof);
+
+  /* If the client id of auth client is not the same as the calculcated
+   * client id, it means that this auth client is invaild according to the
+   * client secret key client_auth_sk. */
+  if (tor_memneq(client->client_id, keystream, HS_DESC_CLIENT_ID_LEN)) {
+    goto done;
+  }
+  cookie_key = keystream + HS_DESC_CLIENT_ID_LEN;
+
+  /* This creates a cipher for AES. It can't fail. */
+  cipher = crypto_cipher_new_with_iv_and_bits(cookie_key, client->iv,
+                                              HS_DESC_COOKIE_KEY_BIT_SIZE);
+  descriptor_cookie = tor_malloc_zero(HS_DESC_DESCRIPTOR_COOKIE_LEN);
+  /* This can't fail. */
+  crypto_cipher_decrypt(cipher, (char *) descriptor_cookie,
+                        (const char *) client->encrypted_cookie,
+                        sizeof(client->encrypted_cookie));
+
+  /* Success. */
+  ret = 0;
+ done:
+  *descriptor_cookie_out = descriptor_cookie;
+  if (cipher) {
+    crypto_cipher_free(cipher);
+  }
+  memwipe(secret_seed, 0, sizeof(secret_seed));
+  memwipe(keystream, 0, sizeof(keystream));
+  return ret;
+}
+
+/** Decrypt an encrypted descriptor layer at <b>encrypted_blob</b> of size
+ *  <b>encrypted_blob_size</b>. The descriptor cookie is optional. Use
+ *  the descriptor object <b>desc</b> and <b>descriptor_cookie</b>
+ *  to generate the right decryption keys; set <b>decrypted_out</b> to
+ *  the plaintext. If <b>is_superencrypted_layer</b> is set, this is
+ *  the outter encrypted layer of the descriptor.
+ *
+ * On any error case, including an empty output, return 0 and set
+ * *<b>decrypted_out</b> to NULL.
+ */
+MOCK_IMPL(STATIC size_t,
+decrypt_desc_layer,(const hs_descriptor_t *desc,
+                    const uint8_t *encrypted_blob,
+                    size_t encrypted_blob_size,
+                    const uint8_t *descriptor_cookie,
+                    int is_superencrypted_layer,
+                    char **decrypted_out))
+{
+  uint8_t *decrypted = NULL;
+  uint8_t secret_key[HS_DESC_ENCRYPTED_KEY_LEN], secret_iv[CIPHER_IV_LEN];
+  uint8_t *secret_data = NULL;
+  size_t secret_data_len = 0;
+  uint8_t mac_key[DIGEST256_LEN], our_mac[DIGEST256_LEN];
+  const uint8_t *salt, *encrypted, *desc_mac;
+  size_t encrypted_len, result_len = 0;
+
+  tor_assert(decrypted_out);
+  tor_assert(desc);
+  tor_assert(encrypted_blob);
+
+  /* Construction is as follow: SALT | ENCRYPTED_DATA | MAC .
+   * Make sure we have enough space for all these things. */
+  if (!encrypted_data_length_is_valid(encrypted_blob_size)) {
+    goto err;
+  }
+
+  /* Start of the blob thus the salt. */
+  salt = encrypted_blob;
+
+  /* Next is the encrypted data. */
+  encrypted = encrypted_blob + HS_DESC_ENCRYPTED_SALT_LEN;
+  encrypted_len = encrypted_blob_size -
+    (HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN);
+  tor_assert(encrypted_len > 0); /* guaranteed by the check above */
+
+  /* And last comes the MAC. */
+  desc_mac = encrypted_blob + encrypted_blob_size - DIGEST256_LEN;
+
+  /* Build secret data to be used in the decryption. */
+  secret_data_len = build_secret_data(&desc->plaintext_data.blinded_pubkey,
+                                      descriptor_cookie,
+                                      &secret_data);
+
+  /* KDF construction resulting in a key from which the secret key, IV and MAC
+   * key are extracted which is what we need for the decryption. */
+  build_secret_key_iv_mac(desc, secret_data, secret_data_len,
+                          salt, HS_DESC_ENCRYPTED_SALT_LEN,
+                          secret_key, sizeof(secret_key),
+                          secret_iv, sizeof(secret_iv),
+                          mac_key, sizeof(mac_key),
+                          is_superencrypted_layer);
+
+  /* Build MAC. */
+  build_mac(mac_key, sizeof(mac_key), salt, HS_DESC_ENCRYPTED_SALT_LEN,
+            encrypted, encrypted_len, our_mac, sizeof(our_mac));
+  memwipe(mac_key, 0, sizeof(mac_key));
+  /* Verify MAC; MAC is H(mac_key || salt || encrypted)
+   *
+   * This is a critical check that is making sure the computed MAC matches the
+   * one in the descriptor. */
+  if (!tor_memeq(our_mac, desc_mac, sizeof(our_mac))) {
+    log_info(LD_REND, "Encrypted service descriptor MAC check failed");
+    goto err;
+  }
+
+  {
+    /* Decrypt. Here we are assured that the encrypted length is valid for
+     * decryption. */
+    crypto_cipher_t *cipher;
+
+    cipher = crypto_cipher_new_with_iv_and_bits(secret_key, secret_iv,
+                                                HS_DESC_ENCRYPTED_BIT_SIZE);
+    /* Extra byte for the NUL terminated byte. */
+    decrypted = tor_malloc_zero(encrypted_len + 1);
+    crypto_cipher_decrypt(cipher, (char *) decrypted,
+                          (const char *) encrypted, encrypted_len);
+    crypto_cipher_free(cipher);
+  }
+
+  {
+    /* Adjust length to remove NUL padding bytes */
+    uint8_t *end = memchr(decrypted, 0, encrypted_len);
+    result_len = encrypted_len;
+    if (end) {
+      result_len = end - decrypted;
+    }
+  }
+
+  if (result_len == 0) {
+    /* Treat this as an error, so that somebody will free the output. */
+    goto err;
+  }
+
+  /* Make sure to NUL terminate the string. */
+  decrypted[encrypted_len] = '\0';
+  *decrypted_out = (char *) decrypted;
+  goto done;
+
+ err:
+  if (decrypted) {
+    tor_free(decrypted);
+  }
+  *decrypted_out = NULL;
+  result_len = 0;
+
+ done:
+  memwipe(secret_data, 0, secret_data_len);
+  memwipe(secret_key, 0, sizeof(secret_key));
+  memwipe(secret_iv, 0, sizeof(secret_iv));
+  tor_free(secret_data);
+  return result_len;
+}
+
+/* Decrypt the superencrypted section of the descriptor using the given
+ * descriptor object <b>desc</b>. A newly allocated NUL terminated string is
+ * put in decrypted_out which contains the superencrypted layer of the
+ * descriptor. Return the length of decrypted_out on success else 0 is
+ * returned and decrypted_out is set to NULL. */
+static size_t
+desc_decrypt_superencrypted(const hs_descriptor_t *desc, char **decrypted_out)
+{
+  size_t superencrypted_len = 0;
+  char *superencrypted_plaintext = NULL;
+
+  tor_assert(desc);
+  tor_assert(decrypted_out);
+
+  superencrypted_len = decrypt_desc_layer(desc,
+                                 desc->plaintext_data.superencrypted_blob,
+                                 desc->plaintext_data.superencrypted_blob_size,
+                                 NULL, 1, &superencrypted_plaintext);
+
+  if (!superencrypted_len) {
+    log_warn(LD_REND, "Decrypting superencrypted desc failed.");
+    goto done;
+  }
+  tor_assert(superencrypted_plaintext);
+
+ done:
+  /* In case of error, superencrypted_plaintext is already NULL, so the
+   * following line makes sense. */
+  *decrypted_out = superencrypted_plaintext;
+  /* This makes sense too, because, in case of error, this is zero. */
+  return superencrypted_len;
+}
+
+/* Decrypt the encrypted section of the descriptor using the given descriptor
+ * object <b>desc</b>. A newly allocated NUL terminated string is put in
+ * decrypted_out which contains the encrypted layer of the descriptor.
+ * Return the length of decrypted_out on success else 0 is returned and
+ * decrypted_out is set to NULL. */
+static size_t
+desc_decrypt_encrypted(const hs_descriptor_t *desc,
+                       const curve25519_secret_key_t *client_auth_sk,
+                       char **decrypted_out)
+{
+  size_t encrypted_len = 0;
+  char *encrypted_plaintext = NULL;
+  uint8_t *descriptor_cookie = NULL;
+
+  tor_assert(desc);
+  tor_assert(desc->superencrypted_data.clients);
+  tor_assert(decrypted_out);
+
+  /* If the client secret key is provided, try to find a valid descriptor
+   * cookie. Otherwise, leave it NULL. */
+  if (client_auth_sk) {
+    SMARTLIST_FOREACH_BEGIN(desc->superencrypted_data.clients,
+                            hs_desc_authorized_client_t *, client) {
+      /* If we can decrypt the descriptor cookie successfully, we will use that
+       * descriptor cookie and break from the loop. */
+      if (!decrypt_descriptor_cookie(desc, client, client_auth_sk,
+                                     &descriptor_cookie)) {
+        break;
+      }
+    } SMARTLIST_FOREACH_END(client);
+  }
+
+  encrypted_len = decrypt_desc_layer(desc,
+                                 desc->superencrypted_data.encrypted_blob,
+                                 desc->superencrypted_data.encrypted_blob_size,
+                                 descriptor_cookie, 0, &encrypted_plaintext);
+  if (!encrypted_len) {
+    goto err;
+  }
+  tor_assert(encrypted_plaintext);
+
+ err:
+  /* In case of error, encrypted_plaintext is already NULL, so the
+   * following line makes sense. */
+  *decrypted_out = encrypted_plaintext;
+  if (descriptor_cookie) {
+    memwipe(descriptor_cookie, 0, HS_DESC_DESCRIPTOR_COOKIE_LEN);
+  }
+  tor_free(descriptor_cookie);
+  /* This makes sense too, because, in case of error, this is zero. */
+  return encrypted_len;
+}
+
+/* Given the token tok for an intro point legacy key, the list of tokens, the
+ * introduction point ip being decoded and the descriptor desc from which it
+ * comes from, decode the legacy key and set the intro point object. Return 0
+ * on success else -1 on failure. */
+static int
+decode_intro_legacy_key(const directory_token_t *tok,
+                        smartlist_t *tokens,
+                        hs_desc_intro_point_t *ip,
+                        const hs_descriptor_t *desc)
+{
+  tor_assert(tok);
+  tor_assert(tokens);
+  tor_assert(ip);
+  tor_assert(desc);
+
+  if (!crypto_pk_public_exponent_ok(tok->key)) {
+    log_warn(LD_REND, "Introduction point legacy key is invalid");
+    goto err;
+  }
+  ip->legacy.key = crypto_pk_dup_key(tok->key);
+  /* Extract the legacy cross certification cert which MUST be present if we
+   * have a legacy key. */
+  tok = find_opt_by_keyword(tokens, R3_INTRO_LEGACY_KEY_CERT);
+  if (!tok) {
+    log_warn(LD_REND, "Introduction point legacy key cert is missing");
+    goto err;
+  }
+  tor_assert(tok->object_body);
+  if (strcmp(tok->object_type, "CROSSCERT")) {
+    /* Info level because this might be an unknown field that we should
+     * ignore. */
+    log_info(LD_REND, "Introduction point legacy encryption key "
+                      "cross-certification has an unknown format.");
+    goto err;
+  }
+  /* Keep a copy of the certificate. */
+  ip->legacy.cert.encoded = tor_memdup(tok->object_body, tok->object_size);
+  ip->legacy.cert.len = tok->object_size;
+  /* The check on the expiration date is for the entire lifetime of a
+   * certificate which is 24 hours. However, a descriptor has a maximum
+   * lifetime of 12 hours meaning we have a 12h difference between the two
+   * which ultimately accommodate the clock skewed client. */
+  if (rsa_ed25519_crosscert_check(ip->legacy.cert.encoded,
+                                  ip->legacy.cert.len, ip->legacy.key,
+                                  &desc->plaintext_data.signing_pubkey,
+                                  approx_time() - HS_DESC_CERT_LIFETIME)) {
+    log_warn(LD_REND, "Unable to check cross-certification on the "
+                      "introduction point legacy encryption key.");
+    ip->cross_certified = 0;
+    goto err;
+  }
+
+  /* Success. */
+  return 0;
+ err:
+  return -1;
+}
+
+/* Dig into the descriptor <b>tokens</b> to find the onion key we should use
+ * for this intro point, and set it into <b>onion_key_out</b>. Return 0 if it
+ * was found and well-formed, otherwise return -1 in case of errors. */
+static int
+set_intro_point_onion_key(curve25519_public_key_t *onion_key_out,
+                          const smartlist_t *tokens)
+{
+  int retval = -1;
+  smartlist_t *onion_keys = NULL;
+
+  tor_assert(onion_key_out);
+
+  onion_keys = find_all_by_keyword(tokens, R3_INTRO_ONION_KEY);
+  if (!onion_keys) {
+    log_warn(LD_REND, "Descriptor did not contain intro onion keys");
+    goto err;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(onion_keys, directory_token_t *, tok) {
+    /* This field is using GE(2) so for possible forward compatibility, we
+     * accept more fields but must be at least 2. */
+    tor_assert(tok->n_args >= 2);
+
+    /* Try to find an ntor key, it's the only recognized type right now */
+    if (!strcmp(tok->args[0], "ntor")) {
+      if (curve25519_public_from_base64(onion_key_out, tok->args[1]) < 0) {
+        log_warn(LD_REND, "Introduction point ntor onion-key is invalid");
+        goto err;
+      }
+      /* Got the onion key! Set the appropriate retval */
+      retval = 0;
+    }
+  } SMARTLIST_FOREACH_END(tok);
+
+  /* Log an error if we didn't find it :( */
+  if (retval < 0) {
+    log_warn(LD_REND, "Descriptor did not contain ntor onion keys");
+  }
+
+ err:
+  smartlist_free(onion_keys);
+  return retval;
+}
+
+/* Given the start of a section and the end of it, decode a single
+ * introduction point from that section. Return a newly allocated introduction
+ * point object containing the decoded data. Return NULL if the section can't
+ * be decoded. */
+STATIC hs_desc_intro_point_t *
+decode_introduction_point(const hs_descriptor_t *desc, const char *start)
+{
+  hs_desc_intro_point_t *ip = NULL;
+  memarea_t *area = NULL;
+  smartlist_t *tokens = NULL;
+  const directory_token_t *tok;
+
+  tor_assert(desc);
+  tor_assert(start);
+
+  area = memarea_new();
+  tokens = smartlist_new();
+  if (tokenize_string(area, start, start + strlen(start),
+                      tokens, hs_desc_intro_point_v3_token_table, 0) < 0) {
+    log_warn(LD_REND, "Introduction point is not parseable");
+    goto err;
+  }
+
+  /* Ok we seem to have a well formed section containing enough tokens to
+   * parse. Allocate our IP object and try to populate it. */
+  ip = hs_desc_intro_point_new();
+
+  /* "introduction-point" SP link-specifiers NL */
+  tok = find_by_keyword(tokens, R3_INTRODUCTION_POINT);
+  tor_assert(tok->n_args == 1);
+  /* Our constructor creates this list by default so free it. */
+  smartlist_free(ip->link_specifiers);
+  ip->link_specifiers = decode_link_specifiers(tok->args[0]);
+  if (!ip->link_specifiers) {
+    log_warn(LD_REND, "Introduction point has invalid link specifiers");
+    goto err;
+  }
+
+  /* "onion-key" SP ntor SP key NL */
+  if (set_intro_point_onion_key(&ip->onion_key, tokens) < 0) {
+    goto err;
+  }
+
+  /* "auth-key" NL certificate NL */
+  tok = find_by_keyword(tokens, R3_INTRO_AUTH_KEY);
+  tor_assert(tok->object_body);
+  if (strcmp(tok->object_type, "ED25519 CERT")) {
+    log_warn(LD_REND, "Unexpected object type for introduction auth key");
+    goto err;
+  }
+  /* Parse cert and do some validation. */
+  if (cert_parse_and_validate(&ip->auth_key_cert, tok->object_body,
+                              tok->object_size, CERT_TYPE_AUTH_HS_IP_KEY,
+                              "introduction point auth-key") < 0) {
+    goto err;
+  }
+  /* Validate authentication certificate with descriptor signing key. */
+  if (tor_cert_checksig(ip->auth_key_cert,
+                        &desc->plaintext_data.signing_pubkey, 0) < 0) {
+    log_warn(LD_REND, "Invalid authentication key signature: %s",
+             tor_cert_describe_signature_status(ip->auth_key_cert));
+    goto err;
+  }
+
+  /* Exactly one "enc-key" SP "ntor" SP key NL */
+  tok = find_by_keyword(tokens, R3_INTRO_ENC_KEY);
+  if (!strcmp(tok->args[0], "ntor")) {
+    /* This field is using GE(2) so for possible forward compatibility, we
+     * accept more fields but must be at least 2. */
+    tor_assert(tok->n_args >= 2);
+
+    if (curve25519_public_from_base64(&ip->enc_key, tok->args[1]) < 0) {
+      log_warn(LD_REND, "Introduction point ntor enc-key is invalid");
+      goto err;
+    }
+  } else {
+    /* Unknown key type so we can't use that introduction point. */
+    log_warn(LD_REND, "Introduction point encryption key is unrecognized.");
+    goto err;
+  }
+
+  /* Exactly once "enc-key-cert" NL certificate NL */
+  tok = find_by_keyword(tokens, R3_INTRO_ENC_KEY_CERT);
+  tor_assert(tok->object_body);
+  /* Do the cross certification. */
+  if (strcmp(tok->object_type, "ED25519 CERT")) {
+      log_warn(LD_REND, "Introduction point ntor encryption key "
+                        "cross-certification has an unknown format.");
+      goto err;
+  }
+  if (cert_parse_and_validate(&ip->enc_key_cert, tok->object_body,
+                              tok->object_size, CERT_TYPE_CROSS_HS_IP_KEYS,
+                              "introduction point enc-key-cert") < 0) {
+    goto err;
+  }
+  if (tor_cert_checksig(ip->enc_key_cert,
+                        &desc->plaintext_data.signing_pubkey, 0) < 0) {
+    log_warn(LD_REND, "Invalid encryption key signature: %s",
+             tor_cert_describe_signature_status(ip->enc_key_cert));
+    goto err;
+  }
+  /* It is successfully cross certified. Flag the object. */
+  ip->cross_certified = 1;
+
+  /* Do we have a "legacy-key" SP key NL ?*/
+  tok = find_opt_by_keyword(tokens, R3_INTRO_LEGACY_KEY);
+  if (tok) {
+    if (decode_intro_legacy_key(tok, tokens, ip, desc) < 0) {
+      goto err;
+    }
+  }
+
+  /* Introduction point has been parsed successfully. */
+  goto done;
+
+ err:
+  hs_desc_intro_point_free(ip);
+  ip = NULL;
+
+ done:
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_free(tokens);
+  if (area) {
+    memarea_drop_all(area);
+  }
+
+  return ip;
+}
+
+/* Given a descriptor string at <b>data</b>, decode all possible introduction
+ * points that we can find. Add the introduction point object to desc_enc as we
+ * find them. This function can't fail and it is possible that zero
+ * introduction points can be decoded. */
+static void
+decode_intro_points(const hs_descriptor_t *desc,
+                    hs_desc_encrypted_data_t *desc_enc,
+                    const char *data)
+{
+  smartlist_t *chunked_desc = smartlist_new();
+  smartlist_t *intro_points = smartlist_new();
+
+  tor_assert(desc);
+  tor_assert(desc_enc);
+  tor_assert(data);
+  tor_assert(desc_enc->intro_points);
+
+  /* Take the desc string, and extract the intro point substrings out of it */
+  {
+    /* Split the descriptor string using the intro point header as delimiter */
+    smartlist_split_string(chunked_desc, data, str_intro_point_start, 0, 0);
+
+    /* Check if there are actually any intro points included. The first chunk
+     * should be other descriptor fields (e.g. create2-formats), so it's not an
+     * intro point. */
+    if (smartlist_len(chunked_desc) < 2) {
+      goto done;
+    }
+  }
+
+  /* Take the intro point substrings, and prepare them for parsing */
+  {
+    int i = 0;
+    /* Prepend the introduction-point header to all the chunks, since
+       smartlist_split_string() devoured it. */
+    SMARTLIST_FOREACH_BEGIN(chunked_desc, char *, chunk) {
+      /* Ignore first chunk. It's other descriptor fields. */
+      if (i++ == 0) {
+        continue;
+      }
+
+      smartlist_add_asprintf(intro_points, "%s %s", str_intro_point, chunk);
+    } SMARTLIST_FOREACH_END(chunk);
+  }
+
+  /* Parse the intro points! */
+  SMARTLIST_FOREACH_BEGIN(intro_points, const char *, intro_point) {
+    hs_desc_intro_point_t *ip = decode_introduction_point(desc, intro_point);
+    if (!ip) {
+      /* Malformed introduction point section. We'll ignore this introduction
+       * point and continue parsing. New or unknown fields are possible for
+       * forward compatibility. */
+      continue;
+    }
+    smartlist_add(desc_enc->intro_points, ip);
+  } SMARTLIST_FOREACH_END(intro_point);
+
+ done:
+  SMARTLIST_FOREACH(chunked_desc, char *, a, tor_free(a));
+  smartlist_free(chunked_desc);
+  SMARTLIST_FOREACH(intro_points, char *, a, tor_free(a));
+  smartlist_free(intro_points);
+}
+/* Return 1 iff the given base64 encoded signature in b64_sig from the encoded
+ * descriptor in encoded_desc validates the descriptor content. */
+STATIC int
+desc_sig_is_valid(const char *b64_sig,
+                  const ed25519_public_key_t *signing_pubkey,
+                  const char *encoded_desc, size_t encoded_len)
+{
+  int ret = 0;
+  ed25519_signature_t sig;
+  const char *sig_start;
+
+  tor_assert(b64_sig);
+  tor_assert(signing_pubkey);
+  tor_assert(encoded_desc);
+  /* Verifying nothing won't end well :). */
+  tor_assert(encoded_len > 0);
+
+  /* Signature length check. */
+  if (strlen(b64_sig) != ED25519_SIG_BASE64_LEN) {
+    log_warn(LD_REND, "Service descriptor has an invalid signature length."
+                      "Exptected %d but got %lu",
+             ED25519_SIG_BASE64_LEN, (unsigned long) strlen(b64_sig));
+    goto err;
+  }
+
+  /* First, convert base64 blob to an ed25519 signature. */
+  if (ed25519_signature_from_base64(&sig, b64_sig) != 0) {
+    log_warn(LD_REND, "Service descriptor does not contain a valid "
+                      "signature");
+    goto err;
+  }
+
+  /* Find the start of signature. */
+  sig_start = tor_memstr(encoded_desc, encoded_len, "\n" str_signature " ");
+  /* Getting here means the token parsing worked for the signature so if we
+   * can't find the start of the signature, we have a code flow issue. */
+  if (!sig_start) {
+    log_warn(LD_GENERAL, "Malformed signature line. Rejecting.");
+    goto err;
+  }
+  /* Skip newline, it has to go in the signature check. */
+  sig_start++;
+
+  /* Validate signature with the full body of the descriptor. */
+  if (ed25519_checksig_prefixed(&sig,
+                                (const uint8_t *) encoded_desc,
+                                sig_start - encoded_desc,
+                                str_desc_sig_prefix,
+                                signing_pubkey) != 0) {
+    log_warn(LD_REND, "Invalid signature on service descriptor");
+    goto err;
+  }
+  /* Valid signature! All is good. */
+  ret = 1;
+
+ err:
+  return ret;
+}
+
+/* Decode descriptor plaintext data for version 3. Given a list of tokens, an
+ * allocated plaintext object that will be populated and the encoded
+ * descriptor with its length. The last one is needed for signature
+ * verification. Unknown tokens are simply ignored so this won't error on
+ * unknowns but requires that all v3 token be present and valid.
+ *
+ * Return 0 on success else a negative value. */
+static int
+desc_decode_plaintext_v3(smartlist_t *tokens,
+                         hs_desc_plaintext_data_t *desc,
+                         const char *encoded_desc, size_t encoded_len)
+{
+  int ok;
+  directory_token_t *tok;
+
+  tor_assert(tokens);
+  tor_assert(desc);
+  /* Version higher could still use this function to decode most of the
+   * descriptor and then they decode the extra part. */
+  tor_assert(desc->version >= 3);
+
+  /* Descriptor lifetime parsing. */
+  tok = find_by_keyword(tokens, R3_DESC_LIFETIME);
+  tor_assert(tok->n_args == 1);
+  desc->lifetime_sec = (uint32_t) tor_parse_ulong(tok->args[0], 10, 0,
+                                                  UINT32_MAX, &ok, NULL);
+  if (!ok) {
+    log_warn(LD_REND, "Service descriptor lifetime value is invalid");
+    goto err;
+  }
+  /* Put it from minute to second. */
+  desc->lifetime_sec *= 60;
+  if (desc->lifetime_sec > HS_DESC_MAX_LIFETIME) {
+    log_warn(LD_REND, "Service descriptor lifetime is too big. "
+                      "Got %" PRIu32 " but max is %d",
+             desc->lifetime_sec, HS_DESC_MAX_LIFETIME);
+    goto err;
+  }
+
+  /* Descriptor signing certificate. */
+  tok = find_by_keyword(tokens, R3_DESC_SIGNING_CERT);
+  tor_assert(tok->object_body);
+  /* Expecting a prop220 cert with the signing key extension, which contains
+   * the blinded public key. */
+  if (strcmp(tok->object_type, "ED25519 CERT") != 0) {
+    log_warn(LD_REND, "Service descriptor signing cert wrong type (%s)",
+             escaped(tok->object_type));
+    goto err;
+  }
+  if (cert_parse_and_validate(&desc->signing_key_cert, tok->object_body,
+                              tok->object_size, CERT_TYPE_SIGNING_HS_DESC,
+                              "service descriptor signing key") < 0) {
+    goto err;
+  }
+
+  /* Copy the public keys into signing_pubkey and blinded_pubkey */
+  memcpy(&desc->signing_pubkey, &desc->signing_key_cert->signed_key,
+         sizeof(ed25519_public_key_t));
+  memcpy(&desc->blinded_pubkey, &desc->signing_key_cert->signing_key,
+         sizeof(ed25519_public_key_t));
+
+  /* Extract revision counter value. */
+  tok = find_by_keyword(tokens, R3_REVISION_COUNTER);
+  tor_assert(tok->n_args == 1);
+  desc->revision_counter = tor_parse_uint64(tok->args[0], 10, 0,
+                                            UINT64_MAX, &ok, NULL);
+  if (!ok) {
+    log_warn(LD_REND, "Service descriptor revision-counter is invalid");
+    goto err;
+  }
+
+  /* Extract the superencrypted data section. */
+  tok = find_by_keyword(tokens, R3_SUPERENCRYPTED);
+  tor_assert(tok->object_body);
+  if (strcmp(tok->object_type, "MESSAGE") != 0) {
+    log_warn(LD_REND, "Desc superencrypted data section is invalid");
+    goto err;
+  }
+  /* Make sure the length of the superencrypted blob is valid. */
+  if (!encrypted_data_length_is_valid(tok->object_size)) {
+    goto err;
+  }
+
+  /* Copy the superencrypted blob to the descriptor object so we can handle it
+   * latter if needed. */
+  desc->superencrypted_blob = tor_memdup(tok->object_body, tok->object_size);
+  desc->superencrypted_blob_size = tok->object_size;
+
+  /* Extract signature and verify it. */
+  tok = find_by_keyword(tokens, R3_SIGNATURE);
+  tor_assert(tok->n_args == 1);
+  /* First arg here is the actual encoded signature. */
+  if (!desc_sig_is_valid(tok->args[0], &desc->signing_pubkey,
+                         encoded_desc, encoded_len)) {
+    goto err;
+  }
+
+  return 0;
+
+ err:
+  return -1;
+}
+
+/* Decode the version 3 superencrypted section of the given descriptor desc.
+ * The desc_superencrypted_out will be populated with the decoded data.
+ * Return 0 on success else -1. */
+static int
+desc_decode_superencrypted_v3(const hs_descriptor_t *desc,
+                              hs_desc_superencrypted_data_t *
+                              desc_superencrypted_out)
+{
+  int ret = -1;
+  char *message = NULL;
+  size_t message_len;
+  memarea_t *area = NULL;
+  directory_token_t *tok;
+  smartlist_t *tokens = NULL;
+  /* Rename the parameter because it is too long. */
+  hs_desc_superencrypted_data_t *superencrypted = desc_superencrypted_out;
+
+  tor_assert(desc);
+  tor_assert(desc_superencrypted_out);
+
+  /* Decrypt the superencrypted data that is located in the plaintext section
+   * in the descriptor as a blob of bytes. */
+  message_len = desc_decrypt_superencrypted(desc, &message);
+  if (!message_len) {
+    log_warn(LD_REND, "Service descriptor decryption failed.");
+    goto err;
+  }
+  tor_assert(message);
+
+  area = memarea_new();
+  tokens = smartlist_new();
+  if (tokenize_string(area, message, message + message_len,
+                      tokens, hs_desc_superencrypted_v3_token_table, 0) < 0) {
+    log_warn(LD_REND, "Superencrypted service descriptor is not parseable.");
+    goto err;
+  }
+
+  /* Verify desc auth type */
+  tok = find_by_keyword(tokens, R3_DESC_AUTH_TYPE);
+  tor_assert(tok->n_args >= 1);
+  if (strcmp(tok->args[0], "x25519")) {
+    log_warn(LD_DIR, "Unrecognized desc auth type");
+    goto err;
+  }
+
+  /* Extract desc auth ephemeral key */
+  tok = find_by_keyword(tokens, R3_DESC_AUTH_KEY);
+  tor_assert(tok->n_args >= 1);
+  if (curve25519_public_from_base64(&superencrypted->auth_ephemeral_pubkey,
+                                    tok->args[0]) < 0) {
+    log_warn(LD_DIR, "Bogus desc auth ephemeral key in HS desc");
+    goto err;
+  }
+
+  /* Extract desc auth client items */
+  if (!superencrypted->clients) {
+    superencrypted->clients = smartlist_new();
+  }
+  SMARTLIST_FOREACH_BEGIN(tokens, const directory_token_t *, token) {
+    if (token->tp == R3_DESC_AUTH_CLIENT) {
+      tor_assert(token->n_args >= 3);
+
+      hs_desc_authorized_client_t *client =
+        tor_malloc_zero(sizeof(hs_desc_authorized_client_t));
+
+      if (decode_auth_client(token, client) < 0) {
+        log_warn(LD_REND, "Descriptor client authorization section can't "
+                          "be decoded.");
+        tor_free(client);
+        goto err;
+      }
+      smartlist_add(superencrypted->clients, client);
+    }
+  } SMARTLIST_FOREACH_END(token);
+
+  /* Extract the encrypted data section. */
+  tok = find_by_keyword(tokens, R3_ENCRYPTED);
+  tor_assert(tok->object_body);
+  if (strcmp(tok->object_type, "MESSAGE") != 0) {
+    log_warn(LD_REND, "Desc encrypted data section is invalid");
+    goto err;
+  }
+  /* Make sure the length of the encrypted blob is valid. */
+  if (!encrypted_data_length_is_valid(tok->object_size)) {
+    goto err;
+  }
+
+  /* Copy the encrypted blob to the descriptor object so we can handle it
+   * latter if needed. */
+  tor_assert(tok->object_size <= INT_MAX);
+  superencrypted->encrypted_blob = tor_memdup(tok->object_body,
+                                              tok->object_size);
+  superencrypted->encrypted_blob_size = tok->object_size;
+
+  ret = 0;
+  goto done;
+
+ err:
+  tor_assert(ret < 0);
+  hs_desc_superencrypted_data_free_contents(desc_superencrypted_out);
+
+ done:
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  if (area) {
+    memarea_drop_all(area);
+  }
+  if (message) {
+    tor_free(message);
+  }
+  return ret;
+}
+
+/* Decode the version 3 encrypted section of the given descriptor desc. The
+ * desc_encrypted_out will be populated with the decoded data. Return 0 on
+ * success else -1. */
+static int
+desc_decode_encrypted_v3(const hs_descriptor_t *desc,
+                         const curve25519_secret_key_t *client_auth_sk,
+                         hs_desc_encrypted_data_t *desc_encrypted_out)
+{
+  int ret = -1;
+  char *message = NULL;
+  size_t message_len;
+  memarea_t *area = NULL;
+  directory_token_t *tok;
+  smartlist_t *tokens = NULL;
+
+  tor_assert(desc);
+  tor_assert(desc_encrypted_out);
+
+  /* Decrypt the encrypted data that is located in the superencrypted section
+   * in the descriptor as a blob of bytes. */
+  message_len = desc_decrypt_encrypted(desc, client_auth_sk, &message);
+  if (!message_len) {
+    /* Two possible situation here. Either we have a client authorization
+     * configured that didn't work or we do not have any configured for this
+     * onion address so likely the descriptor is for authorized client only,
+     * we are not. */
+    if (client_auth_sk) {
+      /* At warning level so the client can notice that its client
+       * authorization is failing. */
+      log_warn(LD_REND, "Client authorization for requested onion address "
+                        "is invalid. Can't decrypt the descriptor.");
+    } else {
+      /* Inform at notice level that the onion address requested can't be
+       * reached without client authorization most likely. */
+      log_notice(LD_REND, "Fail to decrypt descriptor for requested onion "
+                        "address. It is likely requiring client "
+                        "authorization.");
+    }
+    goto err;
+  }
+  tor_assert(message);
+
+  area = memarea_new();
+  tokens = smartlist_new();
+  if (tokenize_string(area, message, message + message_len,
+                      tokens, hs_desc_encrypted_v3_token_table, 0) < 0) {
+    log_warn(LD_REND, "Encrypted service descriptor is not parseable.");
+    goto err;
+  }
+
+  /* CREATE2 supported cell format. It's mandatory. */
+  tok = find_by_keyword(tokens, R3_CREATE2_FORMATS);
+  tor_assert(tok);
+  decode_create2_list(desc_encrypted_out, tok->args[0]);
+  /* Must support ntor according to the specification */
+  if (!desc_encrypted_out->create2_ntor) {
+    log_warn(LD_REND, "Service create2-formats does not include ntor.");
+    goto err;
+  }
+
+  /* Authentication type. It's optional but only once. */
+  tok = find_opt_by_keyword(tokens, R3_INTRO_AUTH_REQUIRED);
+  if (tok) {
+    if (!decode_auth_type(desc_encrypted_out, tok->args[0])) {
+      log_warn(LD_REND, "Service descriptor authentication type has "
+                        "invalid entry(ies).");
+      goto err;
+    }
+  }
+
+  /* Is this service a single onion service? */
+  tok = find_opt_by_keyword(tokens, R3_SINGLE_ONION_SERVICE);
+  if (tok) {
+    desc_encrypted_out->single_onion_service = 1;
+  }
+
+  /* Initialize the descriptor's introduction point list before we start
+   * decoding. Having 0 intro point is valid. Then decode them all. */
+  desc_encrypted_out->intro_points = smartlist_new();
+  decode_intro_points(desc, desc_encrypted_out, message);
+
+  /* Validation of maximum introduction points allowed. */
+  if (smartlist_len(desc_encrypted_out->intro_points) >
+      HS_CONFIG_V3_MAX_INTRO_POINTS) {
+    log_warn(LD_REND, "Service descriptor contains too many introduction "
+                      "points. Maximum allowed is %d but we have %d",
+             HS_CONFIG_V3_MAX_INTRO_POINTS,
+             smartlist_len(desc_encrypted_out->intro_points));
+    goto err;
+  }
+
+  /* NOTE: Unknown fields are allowed because this function could be used to
+   * decode other descriptor version. */
+
+  ret = 0;
+  goto done;
+
+ err:
+  tor_assert(ret < 0);
+  hs_desc_encrypted_data_free_contents(desc_encrypted_out);
+
+ done:
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  if (area) {
+    memarea_drop_all(area);
+  }
+  if (message) {
+    tor_free(message);
+  }
+  return ret;
+}
+
+/* Table of encrypted decode function version specific. The function are
+ * indexed by the version number so v3 callback is at index 3 in the array. */
+static int
+  (*decode_encrypted_handlers[])(
+      const hs_descriptor_t *desc,
+      const curve25519_secret_key_t *client_auth_sk,
+      hs_desc_encrypted_data_t *desc_encrypted) =
+{
+  /* v0 */ NULL, /* v1 */ NULL, /* v2 */ NULL,
+  desc_decode_encrypted_v3,
+};
+
+/* Decode the encrypted data section of the given descriptor and store the
+ * data in the given encrypted data object. Return 0 on success else a
+ * negative value on error. */
+int
+hs_desc_decode_encrypted(const hs_descriptor_t *desc,
+                         const curve25519_secret_key_t *client_auth_sk,
+                         hs_desc_encrypted_data_t *desc_encrypted)
+{
+  int ret;
+  uint32_t version;
+
+  tor_assert(desc);
+  /* Ease our life a bit. */
+  version = desc->plaintext_data.version;
+  tor_assert(desc_encrypted);
+  /* Calling this function without an encrypted blob to parse is a code flow
+   * error. The superencrypted parsing should never succeed in the first place
+   * without an encrypted section. */
+  tor_assert(desc->superencrypted_data.encrypted_blob);
+  /* Let's make sure we have a supported version as well. By correctly parsing
+   * the plaintext, this should not fail. */
+  if (BUG(!hs_desc_is_supported_version(version))) {
+    ret = -1;
+    goto err;
+  }
+  /* Extra precaution. Having no handler for the supported version should
+   * never happened else we forgot to add it but we bumped the version. */
+  tor_assert(ARRAY_LENGTH(decode_encrypted_handlers) >= version);
+  tor_assert(decode_encrypted_handlers[version]);
+
+  /* Run the version specific plaintext decoder. */
+  ret = decode_encrypted_handlers[version](desc, client_auth_sk,
+                                           desc_encrypted);
+  if (ret < 0) {
+    goto err;
+  }
+
+ err:
+  return ret;
+}
+
+/* Table of superencrypted decode function version specific. The function are
+ * indexed by the version number so v3 callback is at index 3 in the array. */
+static int
+  (*decode_superencrypted_handlers[])(
+      const hs_descriptor_t *desc,
+      hs_desc_superencrypted_data_t *desc_superencrypted) =
+{
+  /* v0 */ NULL, /* v1 */ NULL, /* v2 */ NULL,
+  desc_decode_superencrypted_v3,
+};
+
+/* Decode the superencrypted data section of the given descriptor and store the
+ * data in the given superencrypted data object. Return 0 on success else a
+ * negative value on error. */
+int
+hs_desc_decode_superencrypted(const hs_descriptor_t *desc,
+                              hs_desc_superencrypted_data_t *
+                              desc_superencrypted)
+{
+  int ret;
+  uint32_t version;
+
+  tor_assert(desc);
+  /* Ease our life a bit. */
+  version = desc->plaintext_data.version;
+  tor_assert(desc_superencrypted);
+  /* Calling this function without an superencrypted blob to parse is
+   * a code flow error. The plaintext parsing should never succeed in
+   * the first place without an superencrypted section. */
+  tor_assert(desc->plaintext_data.superencrypted_blob);
+  /* Let's make sure we have a supported version as well. By correctly parsing
+   * the plaintext, this should not fail. */
+  if (BUG(!hs_desc_is_supported_version(version))) {
+    ret = -1;
+    goto err;
+  }
+  /* Extra precaution. Having no handler for the supported version should
+   * never happened else we forgot to add it but we bumped the version. */
+  tor_assert(ARRAY_LENGTH(decode_superencrypted_handlers) >= version);
+  tor_assert(decode_superencrypted_handlers[version]);
+
+  /* Run the version specific plaintext decoder. */
+  ret = decode_superencrypted_handlers[version](desc, desc_superencrypted);
+  if (ret < 0) {
+    goto err;
+  }
+
+ err:
+  return ret;
+}
+
+/* Table of plaintext decode function version specific. The function are
+ * indexed by the version number so v3 callback is at index 3 in the array. */
+static int
+  (*decode_plaintext_handlers[])(
+      smartlist_t *tokens,
+      hs_desc_plaintext_data_t *desc,
+      const char *encoded_desc,
+      size_t encoded_len) =
+{
+  /* v0 */ NULL, /* v1 */ NULL, /* v2 */ NULL,
+  desc_decode_plaintext_v3,
+};
+
+/* Fully decode the given descriptor plaintext and store the data in the
+ * plaintext data object. Returns 0 on success else a negative value. */
+int
+hs_desc_decode_plaintext(const char *encoded,
+                         hs_desc_plaintext_data_t *plaintext)
+{
+  int ok = 0, ret = -1;
+  memarea_t *area = NULL;
+  smartlist_t *tokens = NULL;
+  size_t encoded_len;
+  directory_token_t *tok;
+
+  tor_assert(encoded);
+  tor_assert(plaintext);
+
+  /* Check that descriptor is within size limits. */
+  encoded_len = strlen(encoded);
+  if (encoded_len >= hs_cache_get_max_descriptor_size()) {
+    log_warn(LD_REND, "Service descriptor is too big (%lu bytes)",
+             (unsigned long) encoded_len);
+    goto err;
+  }
+
+  area = memarea_new();
+  tokens = smartlist_new();
+  /* Tokenize the descriptor so we can start to parse it. */
+  if (tokenize_string(area, encoded, encoded + encoded_len, tokens,
+                      hs_desc_v3_token_table, 0) < 0) {
+    log_warn(LD_REND, "Service descriptor is not parseable");
+    goto err;
+  }
+
+  /* Get the version of the descriptor which is the first mandatory field of
+   * the descriptor. From there, we'll decode the right descriptor version. */
+  tok = find_by_keyword(tokens, R_HS_DESCRIPTOR);
+  tor_assert(tok->n_args == 1);
+  plaintext->version = (uint32_t) tor_parse_ulong(tok->args[0], 10, 0,
+                                                  UINT32_MAX, &ok, NULL);
+  if (!ok) {
+    log_warn(LD_REND, "Service descriptor has unparseable version %s",
+             escaped(tok->args[0]));
+    goto err;
+  }
+  if (!hs_desc_is_supported_version(plaintext->version)) {
+    log_warn(LD_REND, "Service descriptor has unsupported version %" PRIu32,
+             plaintext->version);
+    goto err;
+  }
+  /* Extra precaution. Having no handler for the supported version should
+   * never happened else we forgot to add it but we bumped the version. */
+  tor_assert(ARRAY_LENGTH(decode_plaintext_handlers) >= plaintext->version);
+  tor_assert(decode_plaintext_handlers[plaintext->version]);
+
+  /* Run the version specific plaintext decoder. */
+  ret = decode_plaintext_handlers[plaintext->version](tokens, plaintext,
+                                                      encoded, encoded_len);
+  if (ret < 0) {
+    goto err;
+  }
+  /* Success. Descriptor has been populated with the data. */
+  ret = 0;
+
+ err:
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  if (area) {
+    memarea_drop_all(area);
+  }
+  return ret;
+}
+
+/* Fully decode an encoded descriptor and set a newly allocated descriptor
+ * object in desc_out.  Client secret key is used to decrypt the "encrypted"
+ * section if not NULL else it's ignored.
+ *
+ * Return 0 on success. A negative value is returned on error and desc_out is
+ * set to NULL. */
+int
+hs_desc_decode_descriptor(const char *encoded,
+                          const uint8_t *subcredential,
+                          const curve25519_secret_key_t *client_auth_sk,
+                          hs_descriptor_t **desc_out)
+{
+  int ret = -1;
+  hs_descriptor_t *desc;
+
+  tor_assert(encoded);
+
+  desc = tor_malloc_zero(sizeof(hs_descriptor_t));
+
+  /* Subcredentials are not optional. */
+  if (BUG(!subcredential ||
+          tor_mem_is_zero((char*)subcredential, DIGEST256_LEN))) {
+    log_warn(LD_GENERAL, "Tried to decrypt without subcred. Impossible!");
+    goto err;
+  }
+
+  memcpy(desc->subcredential, subcredential, sizeof(desc->subcredential));
+
+  ret = hs_desc_decode_plaintext(encoded, &desc->plaintext_data);
+  if (ret < 0) {
+    goto err;
+  }
+
+  ret = hs_desc_decode_superencrypted(desc, &desc->superencrypted_data);
+  if (ret < 0) {
+    goto err;
+  }
+
+  ret = hs_desc_decode_encrypted(desc, client_auth_sk, &desc->encrypted_data);
+  if (ret < 0) {
+    goto err;
+  }
+
+  if (desc_out) {
+    *desc_out = desc;
+  } else {
+    hs_descriptor_free(desc);
+  }
+  return ret;
+
+ err:
+  hs_descriptor_free(desc);
+  if (desc_out) {
+    *desc_out = NULL;
+  }
+
+  tor_assert(ret < 0);
+  return ret;
+}
+
+/* Table of encode function version specific. The functions are indexed by the
+ * version number so v3 callback is at index 3 in the array. */
+static int
+  (*encode_handlers[])(
+      const hs_descriptor_t *desc,
+      const ed25519_keypair_t *signing_kp,
+      const uint8_t *descriptor_cookie,
+      char **encoded_out) =
+{
+  /* v0 */ NULL, /* v1 */ NULL, /* v2 */ NULL,
+  desc_encode_v3,
+};
+
+/* Encode the given descriptor desc including signing with the given key pair
+ * signing_kp and encrypting with the given descriptor cookie.
+ *
+ * If the client authorization is enabled, descriptor_cookie must be the same
+ * as the one used to build hs_desc_authorized_client_t in the descriptor.
+ * Otherwise, it must be NULL.  On success, encoded_out points to a newly
+ * allocated NUL terminated string that contains the encoded descriptor as
+ * a string.
+ *
+ * Return 0 on success and encoded_out is a valid pointer. On error, -1 is
+ * returned and encoded_out is set to NULL. */
+MOCK_IMPL(int,
+hs_desc_encode_descriptor,(const hs_descriptor_t *desc,
+                           const ed25519_keypair_t *signing_kp,
+                           const uint8_t *descriptor_cookie,
+                           char **encoded_out))
+{
+  int ret = -1;
+  uint32_t version;
+
+  tor_assert(desc);
+  tor_assert(encoded_out);
+
+  /* Make sure we support the version of the descriptor format. */
+  version = desc->plaintext_data.version;
+  if (!hs_desc_is_supported_version(version)) {
+    goto err;
+  }
+  /* Extra precaution. Having no handler for the supported version should
+   * never happened else we forgot to add it but we bumped the version. */
+  tor_assert(ARRAY_LENGTH(encode_handlers) >= version);
+  tor_assert(encode_handlers[version]);
+
+  ret = encode_handlers[version](desc, signing_kp,
+                                 descriptor_cookie, encoded_out);
+  if (ret < 0) {
+    goto err;
+  }
+
+  /* Try to decode what we just encoded. Symmetry is nice!, but it is
+   * symmetric only if the client auth is disabled. That is, the descriptor
+   * cookie will be NULL. */
+  if (!descriptor_cookie) {
+    ret = hs_desc_decode_descriptor(*encoded_out, desc->subcredential,
+                                    NULL, NULL);
+    if (BUG(ret < 0)) {
+      goto err;
+    }
+  }
+
+  return 0;
+
+ err:
+  *encoded_out = NULL;
+  return ret;
+}
+
+/* Free the content of the plaintext section of a descriptor. */
+void
+hs_desc_plaintext_data_free_contents(hs_desc_plaintext_data_t *desc)
+{
+  if (!desc) {
+    return;
+  }
+
+  if (desc->superencrypted_blob) {
+    tor_free(desc->superencrypted_blob);
+  }
+  tor_cert_free(desc->signing_key_cert);
+
+  memwipe(desc, 0, sizeof(*desc));
+}
+
+/* Free the content of the superencrypted section of a descriptor. */
+void
+hs_desc_superencrypted_data_free_contents(hs_desc_superencrypted_data_t *desc)
+{
+  if (!desc) {
+    return;
+  }
+
+  if (desc->encrypted_blob) {
+    tor_free(desc->encrypted_blob);
+  }
+  if (desc->clients) {
+    SMARTLIST_FOREACH(desc->clients, hs_desc_authorized_client_t *, client,
+                      hs_desc_authorized_client_free(client));
+    smartlist_free(desc->clients);
+  }
+
+  memwipe(desc, 0, sizeof(*desc));
+}
+
+/* Free the content of the encrypted section of a descriptor. */
+void
+hs_desc_encrypted_data_free_contents(hs_desc_encrypted_data_t *desc)
+{
+  if (!desc) {
+    return;
+  }
+
+  if (desc->intro_auth_types) {
+    SMARTLIST_FOREACH(desc->intro_auth_types, char *, a, tor_free(a));
+    smartlist_free(desc->intro_auth_types);
+  }
+  if (desc->intro_points) {
+    SMARTLIST_FOREACH(desc->intro_points, hs_desc_intro_point_t *, ip,
+                      hs_desc_intro_point_free(ip));
+    smartlist_free(desc->intro_points);
+  }
+  memwipe(desc, 0, sizeof(*desc));
+}
+
+/* Free the descriptor plaintext data object. */
+void
+hs_desc_plaintext_data_free_(hs_desc_plaintext_data_t *desc)
+{
+  hs_desc_plaintext_data_free_contents(desc);
+  tor_free(desc);
+}
+
+/* Free the descriptor plaintext data object. */
+void
+hs_desc_superencrypted_data_free_(hs_desc_superencrypted_data_t *desc)
+{
+  hs_desc_superencrypted_data_free_contents(desc);
+  tor_free(desc);
+}
+
+/* Free the descriptor encrypted data object. */
+void
+hs_desc_encrypted_data_free_(hs_desc_encrypted_data_t *desc)
+{
+  hs_desc_encrypted_data_free_contents(desc);
+  tor_free(desc);
+}
+
+/* Free the given descriptor object. */
+void
+hs_descriptor_free_(hs_descriptor_t *desc)
+{
+  if (!desc) {
+    return;
+  }
+
+  hs_desc_plaintext_data_free_contents(&desc->plaintext_data);
+  hs_desc_superencrypted_data_free_contents(&desc->superencrypted_data);
+  hs_desc_encrypted_data_free_contents(&desc->encrypted_data);
+  tor_free(desc);
+}
+
+/* Return the size in bytes of the given plaintext data object. A sizeof() is
+ * not enough because the object contains pointers and the encrypted blob.
+ * This is particularly useful for our OOM subsystem that tracks the HSDir
+ * cache size for instance. */
+size_t
+hs_desc_plaintext_obj_size(const hs_desc_plaintext_data_t *data)
+{
+  tor_assert(data);
+  return (sizeof(*data) + sizeof(*data->signing_key_cert) +
+          data->superencrypted_blob_size);
+}
+
+/* Return the size in bytes of the given encrypted data object. Used by OOM
+ * subsystem. */
+static size_t
+hs_desc_encrypted_obj_size(const hs_desc_encrypted_data_t *data)
+{
+  tor_assert(data);
+  size_t intro_size = 0;
+  if (data->intro_auth_types) {
+    intro_size +=
+      smartlist_len(data->intro_auth_types) * sizeof(intro_auth_types);
+  }
+  if (data->intro_points) {
+    /* XXX could follow pointers here and get more accurate size */
+    intro_size +=
+      smartlist_len(data->intro_points) * sizeof(hs_desc_intro_point_t);
+  }
+
+  return sizeof(*data) + intro_size;
+}
+
+/* Return the size in bytes of the given descriptor object. Used by OOM
+ * subsystem. */
+  size_t
+hs_desc_obj_size(const hs_descriptor_t *data)
+{
+  tor_assert(data);
+  return (hs_desc_plaintext_obj_size(&data->plaintext_data) +
+          hs_desc_encrypted_obj_size(&data->encrypted_data) +
+          sizeof(data->subcredential));
+}
+
+/* Return a newly allocated descriptor intro point. */
+hs_desc_intro_point_t *
+hs_desc_intro_point_new(void)
+{
+  hs_desc_intro_point_t *ip = tor_malloc_zero(sizeof(*ip));
+  ip->link_specifiers = smartlist_new();
+  return ip;
+}
+
+/* Free a descriptor intro point object. */
+void
+hs_desc_intro_point_free_(hs_desc_intro_point_t *ip)
+{
+  if (ip == NULL) {
+    return;
+  }
+  if (ip->link_specifiers) {
+    SMARTLIST_FOREACH(ip->link_specifiers, hs_desc_link_specifier_t *,
+                      ls, hs_desc_link_specifier_free(ls));
+    smartlist_free(ip->link_specifiers);
+  }
+  tor_cert_free(ip->auth_key_cert);
+  tor_cert_free(ip->enc_key_cert);
+  crypto_pk_free(ip->legacy.key);
+  tor_free(ip->legacy.cert.encoded);
+  tor_free(ip);
+}
+
+/* Allocate and build a new fake client info for the descriptor. Return a
+ * newly allocated object. This can't fail. */
+hs_desc_authorized_client_t *
+hs_desc_build_fake_authorized_client(void)
+{
+  hs_desc_authorized_client_t *client_auth =
+    tor_malloc_zero(sizeof(*client_auth));
+
+  crypto_rand((char *) client_auth->client_id,
+              sizeof(client_auth->client_id));
+  crypto_rand((char *) client_auth->iv,
+              sizeof(client_auth->iv));
+  crypto_rand((char *) client_auth->encrypted_cookie,
+              sizeof(client_auth->encrypted_cookie));
+
+  return client_auth;
+}
+
+/* Using the service's subcredential, client public key, auth ephemeral secret
+ * key, and descriptor cookie, build the auth client so we can then encode the
+ * descriptor for publication. client_out must be already allocated. */
+void
+hs_desc_build_authorized_client(const uint8_t *subcredential,
+                                const curve25519_public_key_t *client_auth_pk,
+                                const curve25519_secret_key_t *
+                                auth_ephemeral_sk,
+                                const uint8_t *descriptor_cookie,
+                                hs_desc_authorized_client_t *client_out)
+{
+  uint8_t secret_seed[CURVE25519_OUTPUT_LEN];
+  uint8_t keystream[HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN];
+  uint8_t *cookie_key;
+  crypto_cipher_t *cipher;
+  crypto_xof_t *xof;
+
+  tor_assert(client_auth_pk);
+  tor_assert(auth_ephemeral_sk);
+  tor_assert(descriptor_cookie);
+  tor_assert(client_out);
+  tor_assert(subcredential);
+  tor_assert(!tor_mem_is_zero((char *) auth_ephemeral_sk,
+                              sizeof(*auth_ephemeral_sk)));
+  tor_assert(!tor_mem_is_zero((char *) client_auth_pk,
+                              sizeof(*client_auth_pk)));
+  tor_assert(!tor_mem_is_zero((char *) descriptor_cookie,
+                              HS_DESC_DESCRIPTOR_COOKIE_LEN));
+  tor_assert(!tor_mem_is_zero((char *) subcredential,
+                              DIGEST256_LEN));
+
+  /* Calculate x25519(hs_y, client_X) */
+  curve25519_handshake(secret_seed,
+                       auth_ephemeral_sk,
+                       client_auth_pk);
+
+  /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */
+  xof = crypto_xof_new();
+  crypto_xof_add_bytes(xof, subcredential, DIGEST256_LEN);
+  crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed));
+  crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream));
+  crypto_xof_free(xof);
+
+  memcpy(client_out->client_id, keystream, HS_DESC_CLIENT_ID_LEN);
+  cookie_key = keystream + HS_DESC_CLIENT_ID_LEN;
+
+  /* Random IV */
+  crypto_strongest_rand(client_out->iv, sizeof(client_out->iv));
+
+  /* This creates a cipher for AES. It can't fail. */
+  cipher = crypto_cipher_new_with_iv_and_bits(cookie_key, client_out->iv,
+                                              HS_DESC_COOKIE_KEY_BIT_SIZE);
+  /* This can't fail. */
+  crypto_cipher_encrypt(cipher, (char *) client_out->encrypted_cookie,
+                        (const char *) descriptor_cookie,
+                        HS_DESC_DESCRIPTOR_COOKIE_LEN);
+
+  memwipe(secret_seed, 0, sizeof(secret_seed));
+  memwipe(keystream, 0, sizeof(keystream));
+
+  crypto_cipher_free(cipher);
+}
+
+/* Free an authoriezd client object. */
+void
+hs_desc_authorized_client_free_(hs_desc_authorized_client_t *client)
+{
+  tor_free(client);
+}
+
+/* Free the given descriptor link specifier. */
+void
+hs_desc_link_specifier_free_(hs_desc_link_specifier_t *ls)
+{
+  if (ls == NULL) {
+    return;
+  }
+  tor_free(ls);
+}
+
+/* Return a newly allocated descriptor link specifier using the given extend
+ * info and requested type. Return NULL on error. */
+hs_desc_link_specifier_t *
+hs_desc_link_specifier_new(const extend_info_t *info, uint8_t type)
+{
+  hs_desc_link_specifier_t *ls = NULL;
+
+  tor_assert(info);
+
+  ls = tor_malloc_zero(sizeof(*ls));
+  ls->type = type;
+  switch (ls->type) {
+  case LS_IPV4:
+    if (info->addr.family != AF_INET) {
+      goto err;
+    }
+    tor_addr_copy(&ls->u.ap.addr, &info->addr);
+    ls->u.ap.port = info->port;
+    break;
+  case LS_IPV6:
+    if (info->addr.family != AF_INET6) {
+      goto err;
+    }
+    tor_addr_copy(&ls->u.ap.addr, &info->addr);
+    ls->u.ap.port = info->port;
+    break;
+  case LS_LEGACY_ID:
+    /* Bug out if the identity digest is not set */
+    if (BUG(tor_mem_is_zero(info->identity_digest,
+                            sizeof(info->identity_digest)))) {
+      goto err;
+    }
+    memcpy(ls->u.legacy_id, info->identity_digest, sizeof(ls->u.legacy_id));
+    break;
+  case LS_ED25519_ID:
+    /* ed25519 keys are optional for intro points */
+    if (ed25519_public_key_is_zero(&info->ed_identity)) {
+      goto err;
+    }
+    memcpy(ls->u.ed25519_id, info->ed_identity.pubkey,
+           sizeof(ls->u.ed25519_id));
+    break;
+  default:
+    /* Unknown type is code flow error. */
+    tor_assert(0);
+  }
+
+  return ls;
+ err:
+  tor_free(ls);
+  return NULL;
+}
+
+/* From the given descriptor, remove and free every introduction point. */
+void
+hs_descriptor_clear_intro_points(hs_descriptor_t *desc)
+{
+  smartlist_t *ips;
+
+  tor_assert(desc);
+
+  ips = desc->encrypted_data.intro_points;
+  if (ips) {
+    SMARTLIST_FOREACH(ips, hs_desc_intro_point_t *,
+                      ip, hs_desc_intro_point_free(ip));
+    smartlist_clear(ips);
+  }
+}
+
+/* From a descriptor link specifier object spec, returned a newly allocated
+ * link specifier object that is the encoded representation of spec. Return
+ * NULL on error. */
+link_specifier_t *
+hs_desc_lspec_to_trunnel(const hs_desc_link_specifier_t *spec)
+{
+  tor_assert(spec);
+
+  link_specifier_t *ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, spec->type);
+
+  switch (spec->type) {
+  case LS_IPV4:
+    link_specifier_set_un_ipv4_addr(ls,
+                                    tor_addr_to_ipv4h(&spec->u.ap.addr));
+    link_specifier_set_un_ipv4_port(ls, spec->u.ap.port);
+    /* Four bytes IPv4 and two bytes port. */
+    link_specifier_set_ls_len(ls, sizeof(spec->u.ap.addr.addr.in_addr) +
+                                  sizeof(spec->u.ap.port));
+    break;
+  case LS_IPV6:
+  {
+    size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls);
+    const uint8_t *in6_addr = tor_addr_to_in6_addr8(&spec->u.ap.addr);
+    uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls);
+    memcpy(ipv6_array, in6_addr, addr_len);
+    link_specifier_set_un_ipv6_port(ls, spec->u.ap.port);
+    /* Sixteen bytes IPv6 and two bytes port. */
+    link_specifier_set_ls_len(ls, addr_len + sizeof(spec->u.ap.port));
+    break;
+  }
+  case LS_LEGACY_ID:
+  {
+    size_t legacy_id_len = link_specifier_getlen_un_legacy_id(ls);
+    uint8_t *legacy_id_array = link_specifier_getarray_un_legacy_id(ls);
+    memcpy(legacy_id_array, spec->u.legacy_id, legacy_id_len);
+    link_specifier_set_ls_len(ls, legacy_id_len);
+    break;
+  }
+  case LS_ED25519_ID:
+  {
+    size_t ed25519_id_len = link_specifier_getlen_un_ed25519_id(ls);
+    uint8_t *ed25519_id_array = link_specifier_getarray_un_ed25519_id(ls);
+    memcpy(ed25519_id_array, spec->u.ed25519_id, ed25519_id_len);
+    link_specifier_set_ls_len(ls, ed25519_id_len);
+    break;
+  }
+  default:
+    tor_assert_nonfatal_unreached();
+    link_specifier_free(ls);
+    ls = NULL;
+  }
+
+  return ls;
+}
diff --git a/src/feature/hs/hs_descriptor.h b/src/feature/hs/hs_descriptor.h
new file mode 100644
index 0000000000..04a8e16d63
--- /dev/null
+++ b/src/feature/hs/hs_descriptor.h
@@ -0,0 +1,346 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_descriptor.h
+ * \brief Header file for hs_descriptor.c
+ **/
+
+#ifndef TOR_HS_DESCRIPTOR_H
+#define TOR_HS_DESCRIPTOR_H
+
+#include <stdint.h>
+
+#include "core/or/or.h"
+#include "trunnel/ed25519_cert.h" /* needed for trunnel */
+#include "feature/nodelist/torcert.h"
+
+/* Trunnel */
+struct link_specifier_t;
+
+/* The earliest descriptor format version we support. */
+#define HS_DESC_SUPPORTED_FORMAT_VERSION_MIN 3
+/* The latest descriptor format version we support. */
+#define HS_DESC_SUPPORTED_FORMAT_VERSION_MAX 3
+
+/* Default lifetime of a descriptor in seconds. The valus is set at 3 hours
+ * which is 180 minutes or 10800 seconds. */
+#define HS_DESC_DEFAULT_LIFETIME (3 * 60 * 60)
+/* Maximum lifetime of a descriptor in seconds. The value is set at 12 hours
+ * which is 720 minutes or 43200 seconds. */
+#define HS_DESC_MAX_LIFETIME (12 * 60 * 60)
+/* Lifetime of certificate in the descriptor. This defines the lifetime of the
+ * descriptor signing key and the cross certification cert of that key. It is
+ * set to 54 hours because a descriptor can be around for 48 hours and because
+ * consensuses are used after the hour, add an extra 6 hours to give some time
+ * for the service to stop using it. */
+#define HS_DESC_CERT_LIFETIME (54 * 60 * 60)
+/* Length of the salt needed for the encrypted section of a descriptor. */
+#define HS_DESC_ENCRYPTED_SALT_LEN 16
+/* Length of the KDF output value which is the length of the secret key,
+ * the secret IV and MAC key length which is the length of H() output. */
+#define HS_DESC_ENCRYPTED_KDF_OUTPUT_LEN \
+  CIPHER256_KEY_LEN + CIPHER_IV_LEN + DIGEST256_LEN
+/* Pad plaintext of superencrypted data section before encryption so that its
+ * length is a multiple of this value. */
+#define HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE 10000
+/* Maximum length in bytes of a full hidden service descriptor. */
+#define HS_DESC_MAX_LEN 50000 /* 50kb max size */
+
+/* Key length for the descriptor symmetric encryption. As specified in the
+ * protocol, we use AES-256 for the encrypted section of the descriptor. The
+ * following is the length in bytes and the bit size. */
+#define HS_DESC_ENCRYPTED_KEY_LEN CIPHER256_KEY_LEN
+#define HS_DESC_ENCRYPTED_BIT_SIZE (HS_DESC_ENCRYPTED_KEY_LEN * 8)
+
+/* Length of each components in the auth client section in the descriptor. */
+#define HS_DESC_CLIENT_ID_LEN 8
+#define HS_DESC_DESCRIPTOR_COOKIE_LEN 16
+#define HS_DESC_COOKIE_KEY_LEN 32
+#define HS_DESC_COOKIE_KEY_BIT_SIZE (HS_DESC_COOKIE_KEY_LEN * 8)
+#define HS_DESC_ENCRYPED_COOKIE_LEN HS_DESC_DESCRIPTOR_COOKIE_LEN
+
+/* The number of auth client entries in the descriptor must be the multiple
+ * of this constant. */
+#define HS_DESC_AUTH_CLIENT_MULTIPLE 16
+
+/* Type of authentication in the descriptor. */
+typedef enum {
+  HS_DESC_AUTH_ED25519 = 1
+} hs_desc_auth_type_t;
+
+/* Link specifier object that contains information on how to extend to the
+ * relay that is the address, port and handshake type. */
+typedef struct hs_desc_link_specifier_t {
+  /* Indicate the type of link specifier. See trunnel ed25519_cert
+   * specification. */
+  uint8_t type;
+
+  /* It must be one of these types, can't be more than one. */
+  union {
+    /* IP address and port of the relay use to extend. */
+    tor_addr_port_t ap;
+    /* Legacy identity. A 20-byte SHA1 identity fingerprint. */
+    uint8_t legacy_id[DIGEST_LEN];
+    /* ed25519 identity. A 32-byte key. */
+    uint8_t ed25519_id[ED25519_PUBKEY_LEN];
+  } u;
+} hs_desc_link_specifier_t;
+
+/* Introduction point information located in a descriptor. */
+typedef struct hs_desc_intro_point_t {
+  /* Link specifier(s) which details how to extend to the relay. This list
+   * contains hs_desc_link_specifier_t object. It MUST have at least one. */
+  smartlist_t *link_specifiers;
+
+  /* Onion key of the introduction point used to extend to it for the ntor
+   * handshake. */
+  curve25519_public_key_t onion_key;
+
+  /* Authentication key used to establish the introduction point circuit and
+   * cross-certifies the blinded public key for the replica thus signed by
+   * the blinded key and in turn signs it. */
+  tor_cert_t *auth_key_cert;
+
+  /* Encryption key for the "ntor" type. */
+  curve25519_public_key_t enc_key;
+
+  /* Certificate cross certifying the descriptor signing key by the encryption
+   * curve25519 key. This certificate contains the signing key and is of type
+   * CERT_TYPE_CROSS_HS_IP_KEYS [0B]. */
+  tor_cert_t *enc_key_cert;
+
+  /* (Optional): If this introduction point is a legacy one that is version <=
+   * 0.2.9.x (HSIntro=3), we use this extra key for the intro point to be able
+   * to relay the cells to the service correctly. */
+  struct {
+    /* RSA public key. */
+    crypto_pk_t *key;
+
+    /* Cross certified cert with the descriptor signing key (RSA->Ed). Because
+     * of the cross certification API, we need to keep the certificate binary
+     * blob and its length in order to properly encode it after. */
+    struct {
+      uint8_t *encoded;
+      size_t len;
+    } cert;
+  } legacy;
+
+  /* True iff the introduction point has passed the cross certification. Upon
+   * decoding an intro point, this must be true. */
+  unsigned int cross_certified : 1;
+} hs_desc_intro_point_t;
+
+/* Authorized client information located in a descriptor. */
+typedef struct hs_desc_authorized_client_t {
+  /* An identifier that the client will use to identify which auth client
+   * entry it needs to use. */
+  uint8_t client_id[HS_DESC_CLIENT_ID_LEN];
+
+  /* An IV that is used to decrypt the encrypted descriptor cookie. */
+  uint8_t iv[CIPHER_IV_LEN];
+
+  /* An encrypted descriptor cookie that the client needs to decrypt to use
+   * it to decrypt the descriptor. */
+  uint8_t encrypted_cookie[HS_DESC_ENCRYPED_COOKIE_LEN];
+} hs_desc_authorized_client_t;
+
+/* The encrypted data section of a descriptor. Obviously the data in this is
+ * in plaintext but encrypted once encoded. */
+typedef struct hs_desc_encrypted_data_t {
+  /* Bitfield of CREATE2 cell supported formats. The only currently supported
+   * format is ntor. */
+  unsigned int create2_ntor : 1;
+
+  /* A list of authentication types that a client must at least support one
+   * in order to contact the service. Contains NULL terminated strings. */
+  smartlist_t *intro_auth_types;
+
+  /* Is this descriptor a single onion service? */
+  unsigned int single_onion_service : 1;
+
+  /* A list of intro points. Contains hs_desc_intro_point_t objects. */
+  smartlist_t *intro_points;
+} hs_desc_encrypted_data_t;
+
+/* The superencrypted data section of a descriptor. Obviously the data in
+ * this is in plaintext but encrypted once encoded. */
+typedef struct hs_desc_superencrypted_data_t {
+  /* This field contains ephemeral x25519 public key which is used by
+   * the encryption scheme in the client authorization. */
+  curve25519_public_key_t auth_ephemeral_pubkey;
+
+  /* A list of authorized clients. Contains hs_desc_authorized_client_t
+   * objects. */
+  smartlist_t *clients;
+
+  /* Decoding only: The b64-decoded encrypted blob from the descriptor */
+  uint8_t *encrypted_blob;
+
+  /* Decoding only: Size of the encrypted_blob */
+  size_t encrypted_blob_size;
+} hs_desc_superencrypted_data_t;
+
+/* Plaintext data that is unencrypted information of the descriptor. */
+typedef struct hs_desc_plaintext_data_t {
+  /* Version of the descriptor format. Spec specifies this field as a
+   * positive integer. */
+  uint32_t version;
+
+  /* The lifetime of the descriptor in seconds. */
+  uint32_t lifetime_sec;
+
+  /* Certificate with the short-term ed22519 descriptor signing key for the
+   * replica which is signed by the blinded public key for that replica. */
+  tor_cert_t *signing_key_cert;
+
+  /* Signing public key which is used to sign the descriptor. Same public key
+   * as in the signing key certificate. */
+  ed25519_public_key_t signing_pubkey;
+
+  /* Blinded public key used for this descriptor derived from the master
+   * identity key and generated for a specific replica number. */
+  ed25519_public_key_t blinded_pubkey;
+
+  /* Revision counter is incremented at each upload, regardless of whether
+   * the descriptor has changed. This avoids leaking whether the descriptor
+   * has changed. Spec specifies this as a 8 bytes positive integer. */
+  uint64_t revision_counter;
+
+  /* Decoding only: The b64-decoded superencrypted blob from the descriptor */
+  uint8_t *superencrypted_blob;
+
+  /* Decoding only: Size of the superencrypted_blob */
+  size_t superencrypted_blob_size;
+} hs_desc_plaintext_data_t;
+
+/* Service descriptor in its decoded form. */
+typedef struct hs_descriptor_t {
+  /* Contains the plaintext part of the descriptor. */
+  hs_desc_plaintext_data_t plaintext_data;
+
+  /* The following contains what's in the superencrypted part of the
+   * descriptor. It's only encrypted in the encoded version of the descriptor
+   * thus the data contained in that object is in plaintext. */
+  hs_desc_superencrypted_data_t superencrypted_data;
+
+  /* The following contains what's in the encrypted part of the descriptor.
+   * It's only encrypted in the encoded version of the descriptor thus the
+   * data contained in that object is in plaintext. */
+  hs_desc_encrypted_data_t encrypted_data;
+
+  /* Subcredentials of a service, used by the client and service to decrypt
+   * the encrypted data. */
+  uint8_t subcredential[DIGEST256_LEN];
+} hs_descriptor_t;
+
+/* Return true iff the given descriptor format version is supported. */
+static inline int
+hs_desc_is_supported_version(uint32_t version)
+{
+  if (version < HS_DESC_SUPPORTED_FORMAT_VERSION_MIN ||
+      version > HS_DESC_SUPPORTED_FORMAT_VERSION_MAX) {
+    return 0;
+  }
+  return 1;
+}
+
+/* Public API. */
+
+void hs_descriptor_free_(hs_descriptor_t *desc);
+#define hs_descriptor_free(desc) \
+  FREE_AND_NULL(hs_descriptor_t, hs_descriptor_free_, (desc))
+void hs_desc_plaintext_data_free_(hs_desc_plaintext_data_t *desc);
+#define hs_desc_plaintext_data_free(desc) \
+  FREE_AND_NULL(hs_desc_plaintext_data_t, hs_desc_plaintext_data_free_, (desc))
+void hs_desc_superencrypted_data_free_(hs_desc_superencrypted_data_t *desc);
+#define hs_desc_superencrypted_data_free(desc) \
+  FREE_AND_NULL(hs_desc_superencrypted_data_t, \
+                hs_desc_superencrypted_data_free_, (desc))
+void hs_desc_encrypted_data_free_(hs_desc_encrypted_data_t *desc);
+#define hs_desc_encrypted_data_free(desc) \
+  FREE_AND_NULL(hs_desc_encrypted_data_t, hs_desc_encrypted_data_free_, (desc))
+
+void hs_desc_link_specifier_free_(hs_desc_link_specifier_t *ls);
+#define hs_desc_link_specifier_free(ls) \
+  FREE_AND_NULL(hs_desc_link_specifier_t, hs_desc_link_specifier_free_, (ls))
+
+hs_desc_link_specifier_t *hs_desc_link_specifier_new(
+                                  const extend_info_t *info, uint8_t type);
+void hs_descriptor_clear_intro_points(hs_descriptor_t *desc);
+
+MOCK_DECL(int,
+          hs_desc_encode_descriptor,(const hs_descriptor_t *desc,
+                                     const ed25519_keypair_t *signing_kp,
+                                     const uint8_t *descriptor_cookie,
+                                     char **encoded_out));
+
+int hs_desc_decode_descriptor(const char *encoded,
+                              const uint8_t *subcredential,
+                              const curve25519_secret_key_t *client_auth_sk,
+                              hs_descriptor_t **desc_out);
+int hs_desc_decode_plaintext(const char *encoded,
+                             hs_desc_plaintext_data_t *plaintext);
+int hs_desc_decode_superencrypted(const hs_descriptor_t *desc,
+                                 hs_desc_superencrypted_data_t *desc_out);
+int hs_desc_decode_encrypted(const hs_descriptor_t *desc,
+                             const curve25519_secret_key_t *client_auth_sk,
+                             hs_desc_encrypted_data_t *desc_out);
+
+size_t hs_desc_obj_size(const hs_descriptor_t *data);
+size_t hs_desc_plaintext_obj_size(const hs_desc_plaintext_data_t *data);
+
+hs_desc_intro_point_t *hs_desc_intro_point_new(void);
+void hs_desc_intro_point_free_(hs_desc_intro_point_t *ip);
+#define hs_desc_intro_point_free(ip) \
+  FREE_AND_NULL(hs_desc_intro_point_t, hs_desc_intro_point_free_, (ip))
+void hs_desc_authorized_client_free_(hs_desc_authorized_client_t *client);
+#define hs_desc_authorized_client_free(client) \
+  FREE_AND_NULL(hs_desc_authorized_client_t, \
+                hs_desc_authorized_client_free_, (client))
+
+link_specifier_t *hs_desc_lspec_to_trunnel(
+                                   const hs_desc_link_specifier_t *spec);
+
+hs_desc_authorized_client_t *hs_desc_build_fake_authorized_client(void);
+void hs_desc_build_authorized_client(const uint8_t *subcredential,
+                                     const curve25519_public_key_t *
+                                     client_auth_pk,
+                                     const curve25519_secret_key_t *
+                                     auth_ephemeral_sk,
+                                     const uint8_t *descriptor_cookie,
+                                     hs_desc_authorized_client_t *client_out);
+void hs_desc_plaintext_data_free_contents(hs_desc_plaintext_data_t *desc);
+void hs_desc_superencrypted_data_free_contents(
+                                        hs_desc_superencrypted_data_t *desc);
+void hs_desc_encrypted_data_free_contents(hs_desc_encrypted_data_t *desc);
+
+#ifdef HS_DESCRIPTOR_PRIVATE
+
+/* Encoding. */
+STATIC char *encode_link_specifiers(const smartlist_t *specs);
+STATIC size_t build_plaintext_padding(const char *plaintext,
+                                      size_t plaintext_len,
+                                      uint8_t **padded_out);
+/* Decoding. */
+STATIC smartlist_t *decode_link_specifiers(const char *encoded);
+STATIC hs_desc_intro_point_t *decode_introduction_point(
+                                const hs_descriptor_t *desc,
+                                const char *text);
+STATIC int encrypted_data_length_is_valid(size_t len);
+STATIC int cert_is_valid(tor_cert_t *cert, uint8_t type,
+                         const char *log_obj_type);
+STATIC int desc_sig_is_valid(const char *b64_sig,
+                             const ed25519_public_key_t *signing_pubkey,
+                             const char *encoded_desc, size_t encoded_len);
+
+MOCK_DECL(STATIC size_t, decrypt_desc_layer,(const hs_descriptor_t *desc,
+                                             const uint8_t *encrypted_blob,
+                                             size_t encrypted_blob_size,
+                                             const uint8_t *descriptor_cookie,
+                                             int is_superencrypted_layer,
+                                             char **decrypted_out));
+
+#endif /* defined(HS_DESCRIPTOR_PRIVATE) */
+
+#endif /* !defined(TOR_HS_DESCRIPTOR_H) */
diff --git a/src/feature/hs/hs_ident.c b/src/feature/hs/hs_ident.c
new file mode 100644
index 0000000000..8fd0013941
--- /dev/null
+++ b/src/feature/hs/hs_ident.c
@@ -0,0 +1,127 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_ident.c
+ * \brief Contains circuit and connection identifier code for the whole HS
+ *        subsytem.
+ **/
+
+#include "lib/crypt_ops/crypto_util.h"
+#include "feature/hs/hs_ident.h"
+
+/* Return a newly allocated circuit identifier. The given public key is copied
+ * identity_pk into the identifier. */
+hs_ident_circuit_t *
+hs_ident_circuit_new(const ed25519_public_key_t *identity_pk,
+                     hs_ident_circuit_type_t circuit_type)
+{
+  tor_assert(circuit_type == HS_IDENT_CIRCUIT_INTRO ||
+             circuit_type == HS_IDENT_CIRCUIT_RENDEZVOUS);
+  hs_ident_circuit_t *ident = tor_malloc_zero(sizeof(*ident));
+  ed25519_pubkey_copy(&ident->identity_pk, identity_pk);
+  ident->circuit_type = circuit_type;
+  return ident;
+}
+
+/* Free the given circuit identifier. */
+void
+hs_ident_circuit_free_(hs_ident_circuit_t *ident)
+{
+  if (ident == NULL) {
+    return;
+  }
+  memwipe(ident, 0, sizeof(hs_ident_circuit_t));
+  tor_free(ident);
+}
+
+/* For a given circuit identifier src, return a newly allocated copy of it.
+ * This can't fail. */
+hs_ident_circuit_t *
+hs_ident_circuit_dup(const hs_ident_circuit_t *src)
+{
+  hs_ident_circuit_t *ident = tor_malloc_zero(sizeof(*ident));
+  memcpy(ident, src, sizeof(*ident));
+  return ident;
+}
+
+/* For a given directory connection identifier src, return a newly allocated
+ * copy of it. This can't fail. */
+hs_ident_dir_conn_t *
+hs_ident_dir_conn_dup(const hs_ident_dir_conn_t *src)
+{
+  hs_ident_dir_conn_t *ident = tor_malloc_zero(sizeof(*ident));
+  memcpy(ident, src, sizeof(*ident));
+  return ident;
+}
+
+/* Free the given directory connection identifier. */
+void
+hs_ident_dir_conn_free_(hs_ident_dir_conn_t *ident)
+{
+  if (ident == NULL) {
+    return;
+  }
+  memwipe(ident, 0, sizeof(hs_ident_dir_conn_t));
+  tor_free(ident);
+}
+
+/* Initialized the allocated ident object with identity_pk and blinded_pk.
+ * None of them can be NULL since a valid directory connection identifier must
+ * have all fields set. */
+void
+hs_ident_dir_conn_init(const ed25519_public_key_t *identity_pk,
+                       const ed25519_public_key_t *blinded_pk,
+                       hs_ident_dir_conn_t *ident)
+{
+  tor_assert(identity_pk);
+  tor_assert(blinded_pk);
+  tor_assert(ident);
+
+  ed25519_pubkey_copy(&ident->identity_pk, identity_pk);
+  ed25519_pubkey_copy(&ident->blinded_pk, blinded_pk);
+}
+
+/* Return a newly allocated edge connection identifier. The given public key
+ * identity_pk is copied into the identifier. */
+hs_ident_edge_conn_t *
+hs_ident_edge_conn_new(const ed25519_public_key_t *identity_pk)
+{
+  hs_ident_edge_conn_t *ident = tor_malloc_zero(sizeof(*ident));
+  ed25519_pubkey_copy(&ident->identity_pk, identity_pk);
+  return ident;
+}
+
+/* Free the given edge connection identifier. */
+void
+hs_ident_edge_conn_free_(hs_ident_edge_conn_t *ident)
+{
+  if (ident == NULL) {
+    return;
+  }
+  memwipe(ident, 0, sizeof(hs_ident_edge_conn_t));
+  tor_free(ident);
+}
+
+/* Return true if the given ident is valid for an introduction circuit. */
+int
+hs_ident_intro_circ_is_valid(const hs_ident_circuit_t *ident)
+{
+  if (ident == NULL) {
+    goto invalid;
+  }
+
+  if (ed25519_public_key_is_zero(&ident->identity_pk)) {
+    goto invalid;
+  }
+
+  if (ed25519_public_key_is_zero(&ident->intro_auth_pk)) {
+    goto invalid;
+  }
+
+  /* Valid. */
+  return 1;
+ invalid:
+  return 0;
+}
+
diff --git a/src/feature/hs/hs_ident.h b/src/feature/hs/hs_ident.h
new file mode 100644
index 0000000000..8c46936a1e
--- /dev/null
+++ b/src/feature/hs/hs_ident.h
@@ -0,0 +1,150 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_ident.h
+ * \brief Header file containing circuit and connection identifier data for
+ *        the whole HS subsytem.
+ *
+ * \details
+ * This interface is used to uniquely identify a hidden service on a circuit
+ * or connection using the service identity public key. Once the circuit or
+ * connection subsystem calls in the hidden service one, we use those
+ * identifiers to lookup the corresponding objects like service, intro point
+ * and descriptor.
+ *
+ * Furthermore, the circuit identifier holds cryptographic material needed for
+ * the e2e encryption on the rendezvous circuit which is set once the
+ * rendezvous circuit has opened and ready to be used.
+ **/
+
+#ifndef TOR_HS_IDENT_H
+#define TOR_HS_IDENT_H
+
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+#include "feature/hs/hs_common.h"
+
+/* Length of the rendezvous cookie that is used to connect circuits at the
+ * rendezvous point. */
+#define HS_REND_COOKIE_LEN DIGEST_LEN
+
+/* Type of circuit an hs_ident_t object is associated with. */
+typedef enum {
+  HS_IDENT_CIRCUIT_INTRO      = 1,
+  HS_IDENT_CIRCUIT_RENDEZVOUS = 2,
+} hs_ident_circuit_type_t;
+
+/* Client and service side circuit identifier that is used for hidden service
+ * circuit establishment. Not all fields contain data, it depends on the
+ * circuit purpose. This is attached to an origin_circuit_t. All fields are
+ * used by both client and service. */
+typedef struct hs_ident_circuit_t {
+  /* (All circuit) The public key used to uniquely identify the service. It is
+   * the one found in the onion address. */
+  ed25519_public_key_t identity_pk;
+
+  /* (All circuit) The type of circuit this identifier is attached to.
+   * Accessors of the fields in this object assert non fatal on this circuit
+   * type. In other words, if a rendezvous field is being accessed, the
+   * circuit type MUST BE of type HS_IDENT_CIRCUIT_RENDEZVOUS. This value is
+   * set when an object is initialized in its constructor. */
+  hs_ident_circuit_type_t circuit_type;
+
+  /* (All circuit) Introduction point authentication key. It's also needed on
+   * the rendezvous circuit for the ntor handshake. It's used as the unique key
+   * of the introduction point so it should not be shared between multiple
+   * intro points. */
+  ed25519_public_key_t intro_auth_pk;
+
+  /* (Only client rendezvous circuit) Introduction point encryption public
+   * key. We keep it in the rendezvous identifier for the ntor handshake. */
+  curve25519_public_key_t intro_enc_pk;
+
+  /* (Only rendezvous circuit) Rendezvous cookie sent from the client to the
+   * service with an INTRODUCE1 cell and used by the service in an
+   * RENDEZVOUS1 cell. */
+  uint8_t rendezvous_cookie[HS_REND_COOKIE_LEN];
+
+  /* (Only service rendezvous circuit) The HANDSHAKE_INFO needed in the
+   * RENDEZVOUS1 cell of the service. The construction is as follows:
+   *    SERVER_PK   [32 bytes]
+   *    AUTH_MAC    [32 bytes]
+   */
+  uint8_t rendezvous_handshake_info[CURVE25519_PUBKEY_LEN + DIGEST256_LEN];
+
+  /* (Only client rendezvous circuit) Client ephemeral keypair needed for the
+   * e2e encryption with the service. */
+  curve25519_keypair_t rendezvous_client_kp;
+
+  /* (Only rendezvous circuit) The NTOR_KEY_SEED needed for key derivation for
+   * the e2e encryption with the client on the circuit. */
+  uint8_t rendezvous_ntor_key_seed[DIGEST256_LEN];
+
+  /* (Only rendezvous circuit) Number of streams associated with this
+   * rendezvous circuit. We track this because there is a check on a maximum
+   * value. */
+  uint64_t num_rdv_streams;
+} hs_ident_circuit_t;
+
+/* Client and service side directory connection identifier used for a
+ * directory connection to identify which service is being queried. This is
+ * attached to a dir_connection_t. */
+typedef struct hs_ident_dir_conn_t {
+  /* The public key used to uniquely identify the service. It is the one found
+   * in the onion address. */
+  ed25519_public_key_t identity_pk;
+
+  /* The blinded public key used to uniquely identify the descriptor that this
+   * directory connection identifier is for. Only used by the service-side code
+   * to fine control descriptor uploads. */
+  ed25519_public_key_t blinded_pk;
+
+  /* XXX: Client authorization. */
+} hs_ident_dir_conn_t;
+
+/* Client and service side edge connection identifier used for an edge
+ * connection to identify which service is being queried. This is attached to
+ * a edge_connection_t. */
+typedef struct hs_ident_edge_conn_t {
+  /* The public key used to uniquely identify the service. It is the one found
+   * in the onion address. */
+  ed25519_public_key_t identity_pk;
+
+  /* The original virtual port that was used by the client to access the onion
+   * service, regardless of the internal port forwarding that might have
+   * happened on the service-side. */
+  uint16_t orig_virtual_port;
+  /* XXX: Client authorization. */
+} hs_ident_edge_conn_t;
+
+/* Circuit identifier API. */
+hs_ident_circuit_t *hs_ident_circuit_new(
+                             const ed25519_public_key_t *identity_pk,
+                             hs_ident_circuit_type_t circuit_type);
+void hs_ident_circuit_free_(hs_ident_circuit_t *ident);
+#define hs_ident_circuit_free(id) \
+  FREE_AND_NULL(hs_ident_circuit_t, hs_ident_circuit_free_, (id))
+hs_ident_circuit_t *hs_ident_circuit_dup(const hs_ident_circuit_t *src);
+
+/* Directory connection identifier API. */
+hs_ident_dir_conn_t *hs_ident_dir_conn_dup(const hs_ident_dir_conn_t *src);
+void hs_ident_dir_conn_free_(hs_ident_dir_conn_t *ident);
+#define hs_ident_dir_conn_free(id) \
+  FREE_AND_NULL(hs_ident_dir_conn_t, hs_ident_dir_conn_free_, (id))
+void hs_ident_dir_conn_init(const ed25519_public_key_t *identity_pk,
+                            const ed25519_public_key_t *blinded_pk,
+                            hs_ident_dir_conn_t *ident);
+
+/* Edge connection identifier API. */
+hs_ident_edge_conn_t *hs_ident_edge_conn_new(
+                                    const ed25519_public_key_t *identity_pk);
+void hs_ident_edge_conn_free_(hs_ident_edge_conn_t *ident);
+#define hs_ident_edge_conn_free(id) \
+  FREE_AND_NULL(hs_ident_edge_conn_t, hs_ident_edge_conn_free_, (id))
+
+/* Validators */
+int hs_ident_intro_circ_is_valid(const hs_ident_circuit_t *ident);
+
+#endif /* !defined(TOR_HS_IDENT_H) */
+
diff --git a/src/feature/hs/hs_intropoint.c b/src/feature/hs/hs_intropoint.c
new file mode 100644
index 0000000000..7717ed53d4
--- /dev/null
+++ b/src/feature/hs/hs_intropoint.c
@@ -0,0 +1,609 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_intropoint.c
+ * \brief Implement next generation introductions point functionality
+ **/
+
+#define HS_INTROPOINT_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/relay.h"
+#include "feature/rend/rendmid.h"
+#include "feature/stats/rephist.h"
+#include "lib/crypt_ops/crypto_format.h"
+
+/* Trunnel */
+#include "trunnel/ed25519_cert.h"
+#include "trunnel/hs/cell_common.h"
+#include "trunnel/hs/cell_establish_intro.h"
+#include "trunnel/hs/cell_introduce1.h"
+
+#include "feature/hs/hs_circuitmap.h"
+#include "feature/hs/hs_descriptor.h"
+#include "feature/hs/hs_intropoint.h"
+#include "feature/hs/hs_common.h"
+
+#include "core/or/or_circuit_st.h"
+
+/** Extract the authentication key from an ESTABLISH_INTRO or INTRODUCE1 using
+ * the given <b>cell_type</b> from <b>cell</b> and place it in
+ * <b>auth_key_out</b>. */
+STATIC void
+get_auth_key_from_cell(ed25519_public_key_t *auth_key_out,
+                       unsigned int cell_type, const void *cell)
+{
+  size_t auth_key_len;
+  const uint8_t *key_array;
+
+  tor_assert(auth_key_out);
+  tor_assert(cell);
+
+  switch (cell_type) {
+  case RELAY_COMMAND_ESTABLISH_INTRO:
+  {
+    const trn_cell_establish_intro_t *c_cell = cell;
+    key_array = trn_cell_establish_intro_getconstarray_auth_key(c_cell);
+    auth_key_len = trn_cell_establish_intro_getlen_auth_key(c_cell);
+    break;
+  }
+  case RELAY_COMMAND_INTRODUCE1:
+  {
+    const trn_cell_introduce1_t *c_cell = cell;
+    key_array = trn_cell_introduce1_getconstarray_auth_key(cell);
+    auth_key_len = trn_cell_introduce1_getlen_auth_key(c_cell);
+    break;
+  }
+  default:
+    /* Getting here is really bad as it means we got a unknown cell type from
+     * this file where every call has an hardcoded value. */
+    tor_assert_unreached(); /* LCOV_EXCL_LINE */
+  }
+  tor_assert(key_array);
+  tor_assert(auth_key_len == sizeof(auth_key_out->pubkey));
+  memcpy(auth_key_out->pubkey, key_array, auth_key_len);
+}
+
+/** We received an ESTABLISH_INTRO <b>cell</b>. Verify its signature and MAC,
+ *  given <b>circuit_key_material</b>. Return 0 on success else -1 on error. */
+STATIC int
+verify_establish_intro_cell(const trn_cell_establish_intro_t *cell,
+                            const uint8_t *circuit_key_material,
+                            size_t circuit_key_material_len)
+{
+  /* We only reach this function if the first byte of the cell is 0x02 which
+   * means that auth_key_type is of ed25519 type, hence this check should
+   * always pass. See hs_intro_received_establish_intro().  */
+  if (BUG(cell->auth_key_type != TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_ED25519)) {
+    return -1;
+  }
+
+  /* Make sure the auth key length is of the right size for this type. For
+   * EXTRA safety, we check both the size of the array and the length which
+   * must be the same. Safety first!*/
+  if (trn_cell_establish_intro_getlen_auth_key(cell) != ED25519_PUBKEY_LEN ||
+      trn_cell_establish_intro_get_auth_key_len(cell) != ED25519_PUBKEY_LEN) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "ESTABLISH_INTRO auth key length is invalid");
+    return -1;
+  }
+
+  const uint8_t *msg = cell->start_cell;
+
+  /* Verify the sig */
+  {
+    ed25519_signature_t sig_struct;
+    const uint8_t *sig_array =
+      trn_cell_establish_intro_getconstarray_sig(cell);
+
+    /* Make sure the signature length is of the right size. For EXTRA safety,
+     * we check both the size of the array and the length which must be the
+     * same. Safety first!*/
+    if (trn_cell_establish_intro_getlen_sig(cell) != sizeof(sig_struct.sig) ||
+        trn_cell_establish_intro_get_sig_len(cell) != sizeof(sig_struct.sig)) {
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "ESTABLISH_INTRO sig len is invalid");
+      return -1;
+    }
+    /* We are now sure that sig_len is of the right size. */
+    memcpy(sig_struct.sig, sig_array, cell->sig_len);
+
+    ed25519_public_key_t auth_key;
+    get_auth_key_from_cell(&auth_key, RELAY_COMMAND_ESTABLISH_INTRO, cell);
+
+    const size_t sig_msg_len = cell->end_sig_fields - msg;
+    int sig_mismatch = ed25519_checksig_prefixed(&sig_struct,
+                                                 msg, sig_msg_len,
+                                                 ESTABLISH_INTRO_SIG_PREFIX,
+                                                 &auth_key);
+    if (sig_mismatch) {
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "ESTABLISH_INTRO signature not as expected");
+      return -1;
+    }
+  }
+
+  /* Verify the MAC */
+  {
+    const size_t auth_msg_len = cell->end_mac_fields - msg;
+    uint8_t mac[DIGEST256_LEN];
+    crypto_mac_sha3_256(mac, sizeof(mac),
+                        circuit_key_material, circuit_key_material_len,
+                        msg, auth_msg_len);
+    if (tor_memneq(mac, cell->handshake_mac, sizeof(mac))) {
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "ESTABLISH_INTRO handshake_auth not as expected");
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+/* Send an INTRO_ESTABLISHED cell to <b>circ</b>. */
+MOCK_IMPL(int,
+hs_intro_send_intro_established_cell,(or_circuit_t *circ))
+{
+  int ret;
+  uint8_t *encoded_cell = NULL;
+  ssize_t encoded_len, result_len;
+  trn_cell_intro_established_t *cell;
+  trn_cell_extension_t *ext;
+
+  tor_assert(circ);
+
+  /* Build the cell payload. */
+  cell = trn_cell_intro_established_new();
+  ext = trn_cell_extension_new();
+  trn_cell_extension_set_num(ext, 0);
+  trn_cell_intro_established_set_extensions(cell, ext);
+  /* Encode the cell to binary format. */
+  encoded_len = trn_cell_intro_established_encoded_len(cell);
+  tor_assert(encoded_len > 0);
+  encoded_cell = tor_malloc_zero(encoded_len);
+  result_len = trn_cell_intro_established_encode(encoded_cell, encoded_len,
+                                                cell);
+  tor_assert(encoded_len == result_len);
+
+  ret = relay_send_command_from_edge(0, TO_CIRCUIT(circ),
+                                     RELAY_COMMAND_INTRO_ESTABLISHED,
+                                     (char *) encoded_cell, encoded_len,
+                                     NULL);
+  /* On failure, the above function will close the circuit. */
+  trn_cell_intro_established_free(cell);
+  tor_free(encoded_cell);
+  return ret;
+}
+
+/** We received an ESTABLISH_INTRO <b>parsed_cell</b> on <b>circ</b>. It's
+ *  well-formed and passed our verifications. Perform appropriate actions to
+ *  establish an intro point. */
+static int
+handle_verified_establish_intro_cell(or_circuit_t *circ,
+                               const trn_cell_establish_intro_t *parsed_cell)
+{
+  /* Get the auth key of this intro point */
+  ed25519_public_key_t auth_key;
+  get_auth_key_from_cell(&auth_key, RELAY_COMMAND_ESTABLISH_INTRO,
+                         parsed_cell);
+
+  /* Then notify the hidden service that the intro point is established by
+     sending an INTRO_ESTABLISHED cell */
+  if (hs_intro_send_intro_established_cell(circ)) {
+    log_warn(LD_PROTOCOL, "Couldn't send INTRO_ESTABLISHED cell.");
+    return -1;
+  }
+
+  /* Associate intro point auth key with this circuit. */
+  hs_circuitmap_register_intro_circ_v3_relay_side(circ, &auth_key);
+  /* Repurpose this circuit into an intro circuit. */
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_INTRO_POINT);
+
+  return 0;
+}
+
+/** We just received an ESTABLISH_INTRO cell in <b>circ</b> with payload in
+ *  <b>request</b>. Handle it by making <b>circ</b> an intro circuit. Return 0
+ *  if everything went well, or -1 if there were errors. */
+static int
+handle_establish_intro(or_circuit_t *circ, const uint8_t *request,
+                       size_t request_len)
+{
+  int cell_ok, retval = -1;
+  trn_cell_establish_intro_t *parsed_cell = NULL;
+
+  tor_assert(circ);
+  tor_assert(request);
+
+  log_info(LD_REND, "Received an ESTABLISH_INTRO request on circuit %" PRIu32,
+           circ->p_circ_id);
+
+  /* Check that the circuit is in shape to become an intro point */
+  if (!hs_intro_circuit_is_suitable_for_establish_intro(circ)) {
+    goto err;
+  }
+
+  /* Parse the cell */
+  ssize_t parsing_result = trn_cell_establish_intro_parse(&parsed_cell,
+                                                         request, request_len);
+  if (parsing_result < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting %s ESTABLISH_INTRO cell.",
+           parsing_result == -1 ? "invalid" : "truncated");
+    goto err;
+  }
+
+  cell_ok = verify_establish_intro_cell(parsed_cell,
+                                        (uint8_t *) circ->rend_circ_nonce,
+                                        sizeof(circ->rend_circ_nonce));
+  if (cell_ok < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Failed to verify ESTABLISH_INTRO cell.");
+    goto err;
+  }
+
+  /* This cell is legit. Take the appropriate actions. */
+  cell_ok = handle_verified_establish_intro_cell(circ, parsed_cell);
+  if (cell_ok < 0) {
+    goto err;
+  }
+
+  /* We are done! */
+  retval = 0;
+  goto done;
+
+ err:
+  /* When sending the intro establish ack, on error the circuit can be marked
+   * as closed so avoid a double close. */
+  if (!TO_CIRCUIT(circ)->marked_for_close) {
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+  }
+
+ done:
+  trn_cell_establish_intro_free(parsed_cell);
+  return retval;
+}
+
+/* Return True if circuit is suitable for being an intro circuit. */
+static int
+circuit_is_suitable_intro_point(const or_circuit_t *circ,
+                                const char *log_cell_type_str)
+{
+  tor_assert(circ);
+  tor_assert(log_cell_type_str);
+
+  /* Basic circuit state sanity checks. */
+  if (circ->base_.purpose != CIRCUIT_PURPOSE_OR) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting %s on non-OR circuit.", log_cell_type_str);
+    return 0;
+  }
+
+  if (circ->base_.n_chan) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting %s on non-edge circuit.", log_cell_type_str);
+    return 0;
+  }
+
+  /* Suitable. */
+  return 1;
+}
+
+/* Return True if circuit is suitable for being service-side intro circuit. */
+int
+hs_intro_circuit_is_suitable_for_establish_intro(const or_circuit_t *circ)
+{
+  return circuit_is_suitable_intro_point(circ, "ESTABLISH_INTRO");
+}
+
+/* We just received an ESTABLISH_INTRO cell in <b>circ</b>. Figure out of it's
+ * a legacy or a next gen cell, and pass it to the appropriate handler. */
+int
+hs_intro_received_establish_intro(or_circuit_t *circ, const uint8_t *request,
+                            size_t request_len)
+{
+  tor_assert(circ);
+  tor_assert(request);
+
+  if (request_len == 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Empty ESTABLISH_INTRO cell.");
+    goto err;
+  }
+
+  /* Using the first byte of the cell, figure out the version of
+   * ESTABLISH_INTRO and pass it to the appropriate cell handler */
+  const uint8_t first_byte = request[0];
+  switch (first_byte) {
+    case TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_LEGACY0:
+    case TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_LEGACY1:
+      return rend_mid_establish_intro_legacy(circ, request, request_len);
+    case TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_ED25519:
+      return handle_establish_intro(circ, request, request_len);
+    default:
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "Unrecognized AUTH_KEY_TYPE %u.", first_byte);
+      goto err;
+  }
+
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+  return -1;
+}
+
+/* Send an INTRODUCE_ACK cell onto the circuit <b>circ</b> with the status
+ * value in <b>status</b>. Depending on the status, it can be ACK or a NACK.
+ * Return 0 on success else a negative value on error which will close the
+ * circuit. */
+static int
+send_introduce_ack_cell(or_circuit_t *circ, uint16_t status)
+{
+  int ret = -1;
+  uint8_t *encoded_cell = NULL;
+  ssize_t encoded_len, result_len;
+  trn_cell_introduce_ack_t *cell;
+  trn_cell_extension_t *ext;
+
+  tor_assert(circ);
+
+  /* Setup the INTRODUCE_ACK cell. We have no extensions so the N_EXTENSIONS
+   * field is set to 0 by default with a new object. */
+  cell = trn_cell_introduce_ack_new();
+  ret = trn_cell_introduce_ack_set_status(cell, status);
+  /* We have no cell extensions in an INTRODUCE_ACK cell. */
+  ext = trn_cell_extension_new();
+  trn_cell_extension_set_num(ext, 0);
+  trn_cell_introduce_ack_set_extensions(cell, ext);
+  /* A wrong status is a very bad code flow error as this value is controlled
+   * by the code in this file and not an external input. This means we use a
+   * code that is not known by the trunnel ABI. */
+  tor_assert(ret == 0);
+  /* Encode the payload. We should never fail to get the encoded length. */
+  encoded_len = trn_cell_introduce_ack_encoded_len(cell);
+  tor_assert(encoded_len > 0);
+  encoded_cell = tor_malloc_zero(encoded_len);
+  result_len = trn_cell_introduce_ack_encode(encoded_cell, encoded_len, cell);
+  tor_assert(encoded_len == result_len);
+
+  ret = relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(circ),
+                                     RELAY_COMMAND_INTRODUCE_ACK,
+                                     (char *) encoded_cell, encoded_len,
+                                     NULL);
+  /* On failure, the above function will close the circuit. */
+  trn_cell_introduce_ack_free(cell);
+  tor_free(encoded_cell);
+  return ret;
+}
+
+/* Validate a parsed INTRODUCE1 <b>cell</b>. Return 0 if valid or else a
+ * negative value for an invalid cell that should be NACKed. */
+STATIC int
+validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell)
+{
+  size_t legacy_key_id_len;
+  const uint8_t *legacy_key_id;
+
+  tor_assert(cell);
+
+  /* This code path SHOULD NEVER be reached if the cell is a legacy type so
+   * safety net here. The legacy ID must be zeroes in this case. */
+  legacy_key_id_len = trn_cell_introduce1_getlen_legacy_key_id(cell);
+  legacy_key_id = trn_cell_introduce1_getconstarray_legacy_key_id(cell);
+  if (BUG(!tor_mem_is_zero((char *) legacy_key_id, legacy_key_id_len))) {
+    goto invalid;
+  }
+
+  /* The auth key of an INTRODUCE1 should be of type ed25519 thus leading to a
+   * known fixed length as well. */
+  if (trn_cell_introduce1_get_auth_key_type(cell) !=
+      TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_ED25519) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting invalid INTRODUCE1 cell auth key type. "
+           "Responding with NACK.");
+    goto invalid;
+  }
+  if (trn_cell_introduce1_get_auth_key_len(cell) != ED25519_PUBKEY_LEN ||
+      trn_cell_introduce1_getlen_auth_key(cell) != ED25519_PUBKEY_LEN) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting invalid INTRODUCE1 cell auth key length. "
+           "Responding with NACK.");
+    goto invalid;
+  }
+  if (trn_cell_introduce1_getlen_encrypted(cell) == 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting invalid INTRODUCE1 cell encrypted length. "
+           "Responding with NACK.");
+    goto invalid;
+  }
+
+  return 0;
+ invalid:
+  return -1;
+}
+
+/* We just received a non legacy INTRODUCE1 cell on <b>client_circ</b> with
+ * the payload in <b>request</b> of size <b>request_len</b>. Return 0 if
+ * everything went well, or -1 if an error occurred. This function is in charge
+ * of sending back an INTRODUCE_ACK cell and will close client_circ on error.
+ */
+STATIC int
+handle_introduce1(or_circuit_t *client_circ, const uint8_t *request,
+                  size_t request_len)
+{
+  int ret = -1;
+  or_circuit_t *service_circ;
+  trn_cell_introduce1_t *parsed_cell;
+  uint16_t status = TRUNNEL_HS_INTRO_ACK_STATUS_SUCCESS;
+
+  tor_assert(client_circ);
+  tor_assert(request);
+
+  /* Parse cell. Note that we can only parse the non encrypted section for
+   * which we'll use the authentication key to find the service introduction
+   * circuit and relay the cell on it. */
+  ssize_t cell_size = trn_cell_introduce1_parse(&parsed_cell, request,
+                                               request_len);
+  if (cell_size < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting %s INTRODUCE1 cell. Responding with NACK.",
+           cell_size == -1 ? "invalid" : "truncated");
+    /* Inform client that the INTRODUCE1 has a bad format. */
+    status = TRUNNEL_HS_INTRO_ACK_STATUS_BAD_FORMAT;
+    goto send_ack;
+  }
+
+  /* Once parsed validate the cell format. */
+  if (validate_introduce1_parsed_cell(parsed_cell) < 0) {
+    /* Inform client that the INTRODUCE1 has bad format. */
+    status = TRUNNEL_HS_INTRO_ACK_STATUS_BAD_FORMAT;
+    goto send_ack;
+  }
+
+  /* Find introduction circuit through our circuit map. */
+  {
+    ed25519_public_key_t auth_key;
+    get_auth_key_from_cell(&auth_key, RELAY_COMMAND_INTRODUCE1, parsed_cell);
+    service_circ = hs_circuitmap_get_intro_circ_v3_relay_side(&auth_key);
+    if (service_circ == NULL) {
+      char b64_key[ED25519_BASE64_LEN + 1];
+      ed25519_public_to_base64(b64_key, &auth_key);
+      log_info(LD_REND, "No intro circuit found for INTRODUCE1 cell "
+                        "with auth key %s from circuit %" PRIu32 ". "
+                        "Responding with NACK.",
+               safe_str(b64_key), client_circ->p_circ_id);
+      /* Inform the client that we don't know the requested service ID. */
+      status = TRUNNEL_HS_INTRO_ACK_STATUS_UNKNOWN_ID;
+      goto send_ack;
+    }
+  }
+
+  /* Relay the cell to the service on its intro circuit with an INTRODUCE2
+   * cell which is the same exact payload. */
+  if (relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(service_circ),
+                                   RELAY_COMMAND_INTRODUCE2,
+                                   (char *) request, request_len, NULL)) {
+    log_warn(LD_PROTOCOL, "Unable to send INTRODUCE2 cell to the service.");
+    /* Inform the client that we can't relay the cell. Use the unknown ID
+     * status code since it means that we do not know the service. */
+    status = TRUNNEL_HS_INTRO_ACK_STATUS_UNKNOWN_ID;
+    goto send_ack;
+  }
+
+  /* Success! Send an INTRODUCE_ACK success status onto the client circuit. */
+  status = TRUNNEL_HS_INTRO_ACK_STATUS_SUCCESS;
+  ret = 0;
+
+ send_ack:
+  /* Send INTRODUCE_ACK or INTRODUCE_NACK to client */
+  if (send_introduce_ack_cell(client_circ, status) < 0) {
+    log_warn(LD_PROTOCOL, "Unable to send an INTRODUCE ACK status %d "
+                          "to client.", status);
+    /* Circuit has been closed on failure of transmission. */
+    goto done;
+  }
+ done:
+  trn_cell_introduce1_free(parsed_cell);
+  return ret;
+}
+
+/* Identify if the encoded cell we just received is a legacy one or not. The
+ * <b>request</b> should be at least DIGEST_LEN bytes long. */
+STATIC int
+introduce1_cell_is_legacy(const uint8_t *request)
+{
+  tor_assert(request);
+
+  /* If the first 20 bytes of the cell (DIGEST_LEN) are NOT zeroes, it
+   * indicates a legacy cell (v2). */
+  if (!tor_mem_is_zero((const char *) request, DIGEST_LEN)) {
+    /* Legacy cell. */
+    return 1;
+  }
+  /* Not a legacy cell. */
+  return 0;
+}
+
+/* Return true iff the circuit <b>circ</b> is suitable for receiving an
+ * INTRODUCE1 cell. */
+STATIC int
+circuit_is_suitable_for_introduce1(const or_circuit_t *circ)
+{
+  tor_assert(circ);
+
+  /* Is this circuit an intro point circuit? */
+  if (!circuit_is_suitable_intro_point(circ, "INTRODUCE1")) {
+    return 0;
+  }
+
+  if (circ->already_received_introduce1) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Blocking multiple introductions on the same circuit. "
+           "Someone might be trying to attack a hidden service through "
+           "this relay.");
+    return 0;
+  }
+
+  return 1;
+}
+
+/* We just received an INTRODUCE1 cell on <b>circ</b>. Figure out which type
+ * it is and pass it to the appropriate handler. Return 0 on success else a
+ * negative value and the circuit is closed. */
+int
+hs_intro_received_introduce1(or_circuit_t *circ, const uint8_t *request,
+                             size_t request_len)
+{
+  int ret;
+
+  tor_assert(circ);
+  tor_assert(request);
+
+  /* A cell that can't hold a DIGEST_LEN is invalid as we need to check if
+   * it's a legacy cell or not using the first DIGEST_LEN bytes. */
+  if (request_len < DIGEST_LEN) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Invalid INTRODUCE1 cell length.");
+    goto err;
+  }
+
+  /* Make sure we have a circuit that can have an INTRODUCE1 cell on it. */
+  if (!circuit_is_suitable_for_introduce1(circ)) {
+    /* We do not send a NACK because the circuit is not suitable for any kind
+     * of response or transmission as it's a violation of the protocol. */
+    goto err;
+  }
+  /* Mark the circuit that we got this cell. None are allowed after this as a
+   * DoS mitigation since one circuit with one client can hammer a service. */
+  circ->already_received_introduce1 = 1;
+
+  /* We are sure here to have at least DIGEST_LEN bytes. */
+  if (introduce1_cell_is_legacy(request)) {
+    /* Handle a legacy cell. */
+    ret = rend_mid_introduce_legacy(circ, request, request_len);
+  } else {
+    /* Handle a non legacy cell. */
+    ret = handle_introduce1(circ, request, request_len);
+  }
+  return ret;
+
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+  return -1;
+}
+
+/* Clear memory allocated by the given intropoint object ip (but don't free the
+ * object itself). */
+void
+hs_intropoint_clear(hs_intropoint_t *ip)
+{
+  if (ip == NULL) {
+    return;
+  }
+  tor_cert_free(ip->auth_key_cert);
+  SMARTLIST_FOREACH(ip->link_specifiers, hs_desc_link_specifier_t *, ls,
+                    hs_desc_link_specifier_free(ls));
+  smartlist_free(ip->link_specifiers);
+  memset(ip, 0, sizeof(hs_intropoint_t));
+}
diff --git a/src/feature/hs/hs_intropoint.h b/src/feature/hs/hs_intropoint.h
new file mode 100644
index 0000000000..e82575f052
--- /dev/null
+++ b/src/feature/hs/hs_intropoint.h
@@ -0,0 +1,64 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_intropoint.h
+ * \brief Header file for hs_intropoint.c.
+ **/
+
+#ifndef TOR_HS_INTRO_H
+#define TOR_HS_INTRO_H
+
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "feature/nodelist/torcert.h"
+
+/* Object containing introduction point common data between the service and
+ * the client side. */
+typedef struct hs_intropoint_t {
+  /* Does this intro point only supports legacy ID ?. */
+  unsigned int is_only_legacy : 1;
+
+  /* Authentication key certificate from the descriptor. */
+  tor_cert_t *auth_key_cert;
+  /* A list of link specifier. */
+  smartlist_t *link_specifiers;
+} hs_intropoint_t;
+
+int hs_intro_received_establish_intro(or_circuit_t *circ,
+                                      const uint8_t *request,
+                                      size_t request_len);
+int hs_intro_received_introduce1(or_circuit_t *circ, const uint8_t *request,
+                                 size_t request_len);
+
+MOCK_DECL(int, hs_intro_send_intro_established_cell,(or_circuit_t *circ));
+
+/* also used by rendservice.c */
+int hs_intro_circuit_is_suitable_for_establish_intro(const or_circuit_t *circ);
+
+hs_intropoint_t *hs_intro_new(void);
+void hs_intropoint_clear(hs_intropoint_t *ip);
+
+#ifdef HS_INTROPOINT_PRIVATE
+
+#include "trunnel/hs/cell_establish_intro.h"
+#include "trunnel/hs/cell_introduce1.h"
+
+STATIC int
+verify_establish_intro_cell(const trn_cell_establish_intro_t *out,
+                            const uint8_t *circuit_key_material,
+                            size_t circuit_key_material_len);
+
+STATIC void
+get_auth_key_from_cell(ed25519_public_key_t *auth_key_out,
+                       unsigned int cell_type, const void *cell);
+
+STATIC int introduce1_cell_is_legacy(const uint8_t *request);
+STATIC int handle_introduce1(or_circuit_t *client_circ,
+                             const uint8_t *request, size_t request_len);
+STATIC int validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell);
+STATIC int circuit_is_suitable_for_introduce1(const or_circuit_t *circ);
+
+#endif /* defined(HS_INTROPOINT_PRIVATE) */
+
+#endif /* !defined(TOR_HS_INTRO_H) */
+
diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c
new file mode 100644
index 0000000000..7e150599fc
--- /dev/null
+++ b/src/feature/hs/hs_service.c
@@ -0,0 +1,4195 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_service.c
+ * \brief Implement next generation hidden service functionality
+ **/
+
+#define HS_SERVICE_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "app/config/statefile.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/relay.h"
+#include "feature/client/circpathbias.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dircommon/directory.h"
+#include "feature/hs_common/shared_random_client.h"
+#include "feature/keymgt/loadkey.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nickname.h"
+#include "feature/nodelist/node_select.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/rend/rendservice.h"
+#include "lib/crypt_ops/crypto_ope.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+
+#include "feature/hs/hs_circuit.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_config.h"
+#include "feature/hs/hs_control.h"
+#include "feature/hs/hs_descriptor.h"
+#include "feature/hs/hs_ident.h"
+#include "feature/hs/hs_intropoint.h"
+#include "feature/hs/hs_service.h"
+#include "feature/hs/hs_stats.h"
+
+#include "feature/dircommon/dir_connection_st.h"
+#include "core/or/edge_connection_st.h"
+#include "core/or/extend_info_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "app/config/or_state_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+#include "lib/encoding/confline.h"
+#include "lib/crypt_ops/crypto_format.h"
+
+/* Trunnel */
+#include "trunnel/ed25519_cert.h"
+#include "trunnel/hs/cell_common.h"
+#include "trunnel/hs/cell_establish_intro.h"
+
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/* Helper macro. Iterate over every service in the global map. The var is the
+ * name of the service pointer. */
+#define FOR_EACH_SERVICE_BEGIN(var)                          \
+    STMT_BEGIN                                               \
+    hs_service_t **var##_iter, *var;                         \
+    HT_FOREACH(var##_iter, hs_service_ht, hs_service_map) {  \
+      var = *var##_iter;
+#define FOR_EACH_SERVICE_END } STMT_END ;
+
+/* Helper macro. Iterate over both current and previous descriptor of a
+ * service. The var is the name of the descriptor pointer. This macro skips
+ * any descriptor object of the service that is NULL. */
+#define FOR_EACH_DESCRIPTOR_BEGIN(service, var)                  \
+  STMT_BEGIN                                                     \
+    hs_service_descriptor_t *var;                                \
+    for (int var ## _loop_idx = 0; var ## _loop_idx < 2;         \
+         ++var ## _loop_idx) {                                   \
+      (var ## _loop_idx == 0) ? (var = service->desc_current) :  \
+                                (var = service->desc_next);      \
+      if (var == NULL) continue;
+#define FOR_EACH_DESCRIPTOR_END } STMT_END ;
+
+/* Onion service directory file names. */
+static const char fname_keyfile_prefix[] = "hs_ed25519";
+static const char dname_client_pubkeys[] = "authorized_clients";
+static const char fname_hostname[] = "hostname";
+static const char address_tld[] = "onion";
+
+/* Staging list of service object. When configuring service, we add them to
+ * this list considered a staging area and they will get added to our global
+ * map once the keys have been loaded. These two steps are separated because
+ * loading keys requires that we are an actual running tor process. */
+static smartlist_t *hs_service_staging_list;
+
+/** True if the list of available router descriptors might have changed which
+ *  might result in an altered hash ring. Check if the hash ring changed and
+ *  reupload if needed */
+static int consider_republishing_hs_descriptors = 0;
+
+/* Static declaration. */
+static int load_client_keys(hs_service_t *service);
+static void set_descriptor_revision_counter(hs_service_descriptor_t *hs_desc,
+                                            time_t now, bool is_current);
+static int build_service_desc_superencrypted(const hs_service_t *service,
+                                             hs_service_descriptor_t *desc);
+static void move_descriptors(hs_service_t *src, hs_service_t *dst);
+static int service_encode_descriptor(const hs_service_t *service,
+                                     const hs_service_descriptor_t *desc,
+                                     const ed25519_keypair_t *signing_kp,
+                                     char **encoded_out);
+
+/* Helper: Function to compare two objects in the service map. Return 1 if the
+ * two service have the same master public identity key. */
+static inline int
+hs_service_ht_eq(const hs_service_t *first, const hs_service_t *second)
+{
+  tor_assert(first);
+  tor_assert(second);
+  /* Simple key compare. */
+  return ed25519_pubkey_eq(&first->keys.identity_pk,
+                           &second->keys.identity_pk);
+}
+
+/* Helper: Function for the service hash table code below. The key used is the
+ * master public identity key which is ultimately the onion address. */
+static inline unsigned int
+hs_service_ht_hash(const hs_service_t *service)
+{
+  tor_assert(service);
+  return (unsigned int) siphash24g(service->keys.identity_pk.pubkey,
+                                   sizeof(service->keys.identity_pk.pubkey));
+}
+
+/* This is _the_ global hash map of hidden services which indexed the service
+ * contained in it by master public identity key which is roughly the onion
+ * address of the service. */
+static struct hs_service_ht *hs_service_map;
+
+/* Register the service hash table. */
+HT_PROTOTYPE(hs_service_ht,      /* Name of hashtable. */
+             hs_service_t,       /* Object contained in the map. */
+             hs_service_node,    /* The name of the HT_ENTRY member. */
+             hs_service_ht_hash, /* Hashing function. */
+             hs_service_ht_eq)   /* Compare function for objects. */
+
+HT_GENERATE2(hs_service_ht, hs_service_t, hs_service_node,
+             hs_service_ht_hash, hs_service_ht_eq,
+             0.6, tor_reallocarray, tor_free_)
+
+/* Query the given service map with a public key and return a service object
+ * if found else NULL. It is also possible to set a directory path in the
+ * search query. If pk is NULL, then it will be set to zero indicating the
+ * hash table to compare the directory path instead. */
+STATIC hs_service_t *
+find_service(hs_service_ht *map, const ed25519_public_key_t *pk)
+{
+  hs_service_t dummy_service;
+  tor_assert(map);
+  tor_assert(pk);
+  memset(&dummy_service, 0, sizeof(dummy_service));
+  ed25519_pubkey_copy(&dummy_service.keys.identity_pk, pk);
+  return HT_FIND(hs_service_ht, map, &dummy_service);
+}
+
+/* Register the given service in the given map. If the service already exists
+ * in the map, -1 is returned. On success, 0 is returned and the service
+ * ownership has been transferred to the global map. */
+STATIC int
+register_service(hs_service_ht *map, hs_service_t *service)
+{
+  tor_assert(map);
+  tor_assert(service);
+  tor_assert(!ed25519_public_key_is_zero(&service->keys.identity_pk));
+
+  if (find_service(map, &service->keys.identity_pk)) {
+    /* Existing service with the same key. Do not register it. */
+    return -1;
+  }
+  /* Taking ownership of the object at this point. */
+  HT_INSERT(hs_service_ht, map, service);
+
+  /* If we just modified the global map, we notify. */
+  if (map == hs_service_map) {
+    hs_service_map_has_changed();
+  }
+
+  return 0;
+}
+
+/* Remove a given service from the given map. If service is NULL or the
+ * service key is unset, return gracefully. */
+STATIC void
+remove_service(hs_service_ht *map, hs_service_t *service)
+{
+  hs_service_t *elm;
+
+  tor_assert(map);
+
+  /* Ignore if no service or key is zero. */
+  if (BUG(service == NULL) ||
+      BUG(ed25519_public_key_is_zero(&service->keys.identity_pk))) {
+    return;
+  }
+
+  elm = HT_REMOVE(hs_service_ht, map, service);
+  if (elm) {
+    tor_assert(elm == service);
+  } else {
+    log_warn(LD_BUG, "Could not find service in the global map "
+                     "while removing service %s",
+             escaped(service->config.directory_path));
+  }
+
+  /* If we just modified the global map, we notify. */
+  if (map == hs_service_map) {
+    hs_service_map_has_changed();
+  }
+}
+
+/* Set the default values for a service configuration object <b>c</b>. */
+static void
+set_service_default_config(hs_service_config_t *c,
+                           const or_options_t *options)
+{
+  (void) options;
+  tor_assert(c);
+  c->ports = smartlist_new();
+  c->directory_path = NULL;
+  c->max_streams_per_rdv_circuit = 0;
+  c->max_streams_close_circuit = 0;
+  c->num_intro_points = NUM_INTRO_POINTS_DEFAULT;
+  c->allow_unknown_ports = 0;
+  c->is_single_onion = 0;
+  c->dir_group_readable = 0;
+  c->is_ephemeral = 0;
+}
+
+/* From a service configuration object config, clear everything from it
+ * meaning free allocated pointers and reset the values. */
+STATIC void
+service_clear_config(hs_service_config_t *config)
+{
+  if (config == NULL) {
+    return;
+  }
+  tor_free(config->directory_path);
+  if (config->ports) {
+    SMARTLIST_FOREACH(config->ports, rend_service_port_config_t *, p,
+                      rend_service_port_config_free(p););
+    smartlist_free(config->ports);
+  }
+  if (config->clients) {
+    SMARTLIST_FOREACH(config->clients, hs_service_authorized_client_t *, p,
+                      service_authorized_client_free(p));
+    smartlist_free(config->clients);
+  }
+  memset(config, 0, sizeof(*config));
+}
+
+/* Helper function to return a human readable description of the given intro
+ * point object.
+ *
+ * This function is not thread-safe. Each call to this invalidates the
+ * previous values returned by it. */
+static const char *
+describe_intro_point(const hs_service_intro_point_t *ip)
+{
+  /* Hex identity digest of the IP prefixed by the $ sign and ends with NUL
+   * byte hence the plus two. */
+  static char buf[HEX_DIGEST_LEN + 2];
+  const char *legacy_id = NULL;
+
+  SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
+                          const hs_desc_link_specifier_t *, lspec) {
+    if (lspec->type == LS_LEGACY_ID) {
+      legacy_id = (const char *) lspec->u.legacy_id;
+      break;
+    }
+  } SMARTLIST_FOREACH_END(lspec);
+
+  /* For now, we only print the identity digest but we could improve this with
+   * much more information such as the ed25519 identity has well. */
+  buf[0] = '$';
+  if (legacy_id) {
+    base16_encode(buf + 1, HEX_DIGEST_LEN + 1, legacy_id, DIGEST_LEN);
+  }
+
+  return buf;
+}
+
+/* Return the lower bound of maximum INTRODUCE2 cells per circuit before we
+ * rotate intro point (defined by a consensus parameter or the default
+ * value). */
+static int32_t
+get_intro_point_min_introduce2(void)
+{
+  /* The [0, 2147483647] range is quite large to accommodate anything we decide
+   * in the future. */
+  return networkstatus_get_param(NULL, "hs_intro_min_introduce2",
+                                 INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS,
+                                 0, INT32_MAX);
+}
+
+/* Return the upper bound of maximum INTRODUCE2 cells per circuit before we
+ * rotate intro point (defined by a consensus parameter or the default
+ * value). */
+static int32_t
+get_intro_point_max_introduce2(void)
+{
+  /* The [0, 2147483647] range is quite large to accommodate anything we decide
+   * in the future. */
+  return networkstatus_get_param(NULL, "hs_intro_max_introduce2",
+                                 INTRO_POINT_MAX_LIFETIME_INTRODUCTIONS,
+                                 0, INT32_MAX);
+}
+
+/* Return the minimum lifetime in seconds of an introduction point defined by a
+ * consensus parameter or the default value. */
+static int32_t
+get_intro_point_min_lifetime(void)
+{
+#define MIN_INTRO_POINT_LIFETIME_TESTING 10
+  if (get_options()->TestingTorNetwork) {
+    return MIN_INTRO_POINT_LIFETIME_TESTING;
+  }
+
+  /* The [0, 2147483647] range is quite large to accommodate anything we decide
+   * in the future. */
+  return networkstatus_get_param(NULL, "hs_intro_min_lifetime",
+                                 INTRO_POINT_LIFETIME_MIN_SECONDS,
+                                 0, INT32_MAX);
+}
+
+/* Return the maximum lifetime in seconds of an introduction point defined by a
+ * consensus parameter or the default value. */
+static int32_t
+get_intro_point_max_lifetime(void)
+{
+#define MAX_INTRO_POINT_LIFETIME_TESTING 30
+  if (get_options()->TestingTorNetwork) {
+    return MAX_INTRO_POINT_LIFETIME_TESTING;
+  }
+
+  /* The [0, 2147483647] range is quite large to accommodate anything we decide
+   * in the future. */
+  return networkstatus_get_param(NULL, "hs_intro_max_lifetime",
+                                 INTRO_POINT_LIFETIME_MAX_SECONDS,
+                                 0, INT32_MAX);
+}
+
+/* Return the number of extra introduction point defined by a consensus
+ * parameter or the default value. */
+static int32_t
+get_intro_point_num_extra(void)
+{
+  /* The [0, 128] range bounds the number of extra introduction point allowed.
+   * Above 128 intro points, it's getting a bit crazy. */
+  return networkstatus_get_param(NULL, "hs_intro_num_extra",
+                                 NUM_INTRO_POINTS_EXTRA, 0, 128);
+}
+
+/* Helper: Function that needs to return 1 for the HT for each loop which
+ * frees every service in an hash map. */
+static int
+ht_free_service_(struct hs_service_t *service, void *data)
+{
+  (void) data;
+  hs_service_free(service);
+  /* This function MUST return 1 so the given object is then removed from the
+   * service map leading to this free of the object being safe. */
+  return 1;
+}
+
+/* Free every service that can be found in the global map. Once done, clear
+ * and free the global map. */
+static void
+service_free_all(void)
+{
+  if (hs_service_map) {
+    /* The free helper function returns 1 so this is safe. */
+    hs_service_ht_HT_FOREACH_FN(hs_service_map, ht_free_service_, NULL);
+    HT_CLEAR(hs_service_ht, hs_service_map);
+    tor_free(hs_service_map);
+    hs_service_map = NULL;
+  }
+
+  if (hs_service_staging_list) {
+    /* Cleanup staging list. */
+    SMARTLIST_FOREACH(hs_service_staging_list, hs_service_t *, s,
+                      hs_service_free(s));
+    smartlist_free(hs_service_staging_list);
+    hs_service_staging_list = NULL;
+  }
+}
+
+/* Free a given service intro point object. */
+STATIC void
+service_intro_point_free_(hs_service_intro_point_t *ip)
+{
+  if (!ip) {
+    return;
+  }
+  memwipe(&ip->auth_key_kp, 0, sizeof(ip->auth_key_kp));
+  memwipe(&ip->enc_key_kp, 0, sizeof(ip->enc_key_kp));
+  crypto_pk_free(ip->legacy_key);
+  replaycache_free(ip->replay_cache);
+  hs_intropoint_clear(&ip->base);
+  tor_free(ip);
+}
+
+/* Helper: free an hs_service_intro_point_t object. This function is used by
+ * digest256map_free() which requires a void * pointer. */
+static void
+service_intro_point_free_void(void *obj)
+{
+  service_intro_point_free_(obj);
+}
+
+/* Return a newly allocated service intro point and fully initialized from the
+ * given extend_info_t ei if non NULL.
+ * If is_legacy is true, we also generate the legacy key.
+ * If supports_ed25519_link_handshake_any is true, we add the relay's ed25519
+ * key to the link specifiers.
+ *
+ * If ei is NULL, returns a hs_service_intro_point_t with an empty link
+ * specifier list and no onion key. (This is used for testing.)
+ * On any other error, NULL is returned.
+ *
+ * ei must be an extend_info_t containing an IPv4 address. (We will add supoort
+ * for IPv6 in a later release.) When calling extend_info_from_node(), pass
+ * 0 in for_direct_connection to make sure ei always has an IPv4 address. */
+STATIC hs_service_intro_point_t *
+service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy,
+                        unsigned int supports_ed25519_link_handshake_any)
+{
+  hs_desc_link_specifier_t *ls;
+  hs_service_intro_point_t *ip;
+
+  ip = tor_malloc_zero(sizeof(*ip));
+  /* We'll create the key material. No need for extra strong, those are short
+   * term keys. */
+  ed25519_keypair_generate(&ip->auth_key_kp, 0);
+
+  { /* Set introduce2 max cells limit */
+    int32_t min_introduce2_cells = get_intro_point_min_introduce2();
+    int32_t max_introduce2_cells = get_intro_point_max_introduce2();
+    if (BUG(max_introduce2_cells < min_introduce2_cells)) {
+      goto err;
+    }
+    ip->introduce2_max = crypto_rand_int_range(min_introduce2_cells,
+                                               max_introduce2_cells);
+  }
+  { /* Set intro point lifetime */
+    int32_t intro_point_min_lifetime = get_intro_point_min_lifetime();
+    int32_t intro_point_max_lifetime = get_intro_point_max_lifetime();
+    if (BUG(intro_point_max_lifetime < intro_point_min_lifetime)) {
+      goto err;
+    }
+    ip->time_to_expire = approx_time() +
+      crypto_rand_int_range(intro_point_min_lifetime,intro_point_max_lifetime);
+  }
+
+  ip->replay_cache = replaycache_new(0, 0);
+
+  /* Initialize the base object. We don't need the certificate object. */
+  ip->base.link_specifiers = smartlist_new();
+
+  /* Generate the encryption key for this intro point. */
+  curve25519_keypair_generate(&ip->enc_key_kp, 0);
+  /* Figure out if this chosen node supports v3 or is legacy only. */
+  if (is_legacy) {
+    ip->base.is_only_legacy = 1;
+    /* Legacy mode that is doesn't support v3+ with ed25519 auth key. */
+    ip->legacy_key = crypto_pk_new();
+    if (crypto_pk_generate_key(ip->legacy_key) < 0) {
+      goto err;
+    }
+    if (crypto_pk_get_digest(ip->legacy_key,
+                             (char *) ip->legacy_key_digest) < 0) {
+      goto err;
+    }
+  }
+
+  if (ei == NULL) {
+    goto done;
+  }
+
+  /* We'll try to add all link specifiers. Legacy is mandatory.
+   * IPv4 or IPv6 is required, and we always send IPv4. */
+  ls = hs_desc_link_specifier_new(ei, LS_IPV4);
+  /* It is impossible to have an extend info object without a v4. */
+  if (BUG(!ls)) {
+    goto err;
+  }
+  smartlist_add(ip->base.link_specifiers, ls);
+
+  ls = hs_desc_link_specifier_new(ei, LS_LEGACY_ID);
+  /* It is impossible to have an extend info object without an identity
+   * digest. */
+  if (BUG(!ls)) {
+    goto err;
+  }
+  smartlist_add(ip->base.link_specifiers, ls);
+
+  /* ed25519 identity key is optional for intro points. If the node supports
+   * ed25519 link authentication, we include it. */
+  if (supports_ed25519_link_handshake_any) {
+    ls = hs_desc_link_specifier_new(ei, LS_ED25519_ID);
+    if (ls) {
+      smartlist_add(ip->base.link_specifiers, ls);
+    }
+  }
+
+  /* IPv6 is not supported in this release. */
+
+  /* Finally, copy onion key from the extend_info_t object. */
+  memcpy(&ip->onion_key, &ei->curve25519_onion_key, sizeof(ip->onion_key));
+
+ done:
+  return ip;
+ err:
+  service_intro_point_free(ip);
+  return NULL;
+}
+
+/* Add the given intro point object to the given intro point map. The intro
+ * point MUST have its RSA encryption key set if this is a legacy type or the
+ * authentication key set otherwise. */
+STATIC void
+service_intro_point_add(digest256map_t *map, hs_service_intro_point_t *ip)
+{
+  hs_service_intro_point_t *old_ip_entry;
+
+  tor_assert(map);
+  tor_assert(ip);
+
+  old_ip_entry = digest256map_set(map, ip->auth_key_kp.pubkey.pubkey, ip);
+  /* Make sure we didn't just try to double-add an intro point */
+  tor_assert_nonfatal(!old_ip_entry);
+}
+
+/* For a given service, remove the intro point from that service's descriptors
+ * (check both current and next descriptor) */
+STATIC void
+service_intro_point_remove(const hs_service_t *service,
+                           const hs_service_intro_point_t *ip)
+{
+  tor_assert(service);
+  tor_assert(ip);
+
+  /* Trying all descriptors. */
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    /* We'll try to remove the descriptor on both descriptors which is not
+     * very expensive to do instead of doing loopup + remove. */
+    digest256map_remove(desc->intro_points.map,
+                        ip->auth_key_kp.pubkey.pubkey);
+  } FOR_EACH_DESCRIPTOR_END;
+}
+
+/* For a given service and authentication key, return the intro point or NULL
+ * if not found. This will check both descriptors in the service. */
+STATIC hs_service_intro_point_t *
+service_intro_point_find(const hs_service_t *service,
+                         const ed25519_public_key_t *auth_key)
+{
+  hs_service_intro_point_t *ip = NULL;
+
+  tor_assert(service);
+  tor_assert(auth_key);
+
+  /* Trying all descriptors to find the right intro point.
+   *
+   * Even if we use the same node as intro point in both descriptors, the node
+   * will have a different intro auth key for each descriptor since we generate
+   * a new one everytime we pick an intro point.
+   *
+   * After #22893 gets implemented, intro points will be moved to be
+   * per-service instead of per-descriptor so this function will need to
+   * change.
+   */
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    if ((ip = digest256map_get(desc->intro_points.map,
+                               auth_key->pubkey)) != NULL) {
+      break;
+    }
+  } FOR_EACH_DESCRIPTOR_END;
+
+  return ip;
+}
+
+/* For a given service and intro point, return the descriptor for which the
+ * intro point is assigned to. NULL is returned if not found. */
+STATIC hs_service_descriptor_t *
+service_desc_find_by_intro(const hs_service_t *service,
+                           const hs_service_intro_point_t *ip)
+{
+  hs_service_descriptor_t *descp = NULL;
+
+  tor_assert(service);
+  tor_assert(ip);
+
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    if (digest256map_get(desc->intro_points.map,
+                         ip->auth_key_kp.pubkey.pubkey)) {
+      descp = desc;
+      break;
+    }
+  } FOR_EACH_DESCRIPTOR_END;
+
+  return descp;
+}
+
+/* From a circuit identifier, get all the possible objects associated with the
+ * ident. If not NULL, service, ip or desc are set if the object can be found.
+ * They are untouched if they can't be found.
+ *
+ * This is an helper function because we do those lookups often so it's more
+ * convenient to simply call this functions to get all the things at once. */
+STATIC void
+get_objects_from_ident(const hs_ident_circuit_t *ident,
+                       hs_service_t **service, hs_service_intro_point_t **ip,
+                       hs_service_descriptor_t **desc)
+{
+  hs_service_t *s;
+
+  tor_assert(ident);
+
+  /* Get service object from the circuit identifier. */
+  s = find_service(hs_service_map, &ident->identity_pk);
+  if (s && service) {
+    *service = s;
+  }
+
+  /* From the service object, get the intro point object of that circuit. The
+   * following will query both descriptors intro points list. */
+  if (s && ip) {
+    *ip = service_intro_point_find(s, &ident->intro_auth_pk);
+  }
+
+  /* Get the descriptor for this introduction point and service. */
+  if (s && ip && *ip && desc) {
+    *desc = service_desc_find_by_intro(s, *ip);
+  }
+}
+
+/* From a given intro point, return the first link specifier of type
+ * encountered in the link specifier list. Return NULL if it can't be found.
+ *
+ * The caller does NOT have ownership of the object, the intro point does. */
+static hs_desc_link_specifier_t *
+get_link_spec_by_type(const hs_service_intro_point_t *ip, uint8_t type)
+{
+  hs_desc_link_specifier_t *lnk_spec = NULL;
+
+  tor_assert(ip);
+
+  SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
+                          hs_desc_link_specifier_t *, ls) {
+    if (ls->type == type) {
+      lnk_spec = ls;
+      goto end;
+    }
+  } SMARTLIST_FOREACH_END(ls);
+
+ end:
+  return lnk_spec;
+}
+
+/* Given a service intro point, return the node_t associated to it. This can
+ * return NULL if the given intro point has no legacy ID or if the node can't
+ * be found in the consensus. */
+STATIC const node_t *
+get_node_from_intro_point(const hs_service_intro_point_t *ip)
+{
+  const hs_desc_link_specifier_t *ls;
+
+  tor_assert(ip);
+
+  ls = get_link_spec_by_type(ip, LS_LEGACY_ID);
+  if (BUG(!ls)) {
+    return NULL;
+  }
+  /* XXX In the future, we want to only use the ed25519 ID (#22173). */
+  return node_get_by_id((const char *) ls->u.legacy_id);
+}
+
+/* Given a service intro point, return the extend_info_t for it. This can
+ * return NULL if the node can't be found for the intro point or the extend
+ * info can't be created for the found node. If direct_conn is set, the extend
+ * info is validated on if we can connect directly. */
+static extend_info_t *
+get_extend_info_from_intro_point(const hs_service_intro_point_t *ip,
+                                 unsigned int direct_conn)
+{
+  extend_info_t *info = NULL;
+  const node_t *node;
+
+  tor_assert(ip);
+
+  node = get_node_from_intro_point(ip);
+  if (node == NULL) {
+    /* This can happen if the relay serving as intro point has been removed
+     * from the consensus. In that case, the intro point will be removed from
+     * the descriptor during the scheduled events. */
+    goto end;
+  }
+
+  /* In the case of a direct connection (single onion service), it is possible
+   * our firewall policy won't allow it so this can return a NULL value. */
+  info = extend_info_from_node(node, direct_conn);
+
+ end:
+  return info;
+}
+
+/* Return the number of introduction points that are established for the
+ * given descriptor. */
+static unsigned int
+count_desc_circuit_established(const hs_service_descriptor_t *desc)
+{
+  unsigned int count = 0;
+
+  tor_assert(desc);
+
+  DIGEST256MAP_FOREACH(desc->intro_points.map, key,
+                       const hs_service_intro_point_t *, ip) {
+    count += ip->circuit_established;
+  } DIGEST256MAP_FOREACH_END;
+
+  return count;
+}
+
+/* For a given service and descriptor of that service, close all active
+ * directory connections. */
+static void
+close_directory_connections(const hs_service_t *service,
+                            const hs_service_descriptor_t *desc)
+{
+  unsigned int count = 0;
+  smartlist_t *dir_conns;
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  /* Close pending HS desc upload connections for the blinded key of 'desc'. */
+  dir_conns = connection_list_by_type_purpose(CONN_TYPE_DIR,
+                                              DIR_PURPOSE_UPLOAD_HSDESC);
+  SMARTLIST_FOREACH_BEGIN(dir_conns, connection_t *, conn) {
+    dir_connection_t *dir_conn = TO_DIR_CONN(conn);
+    if (ed25519_pubkey_eq(&dir_conn->hs_ident->identity_pk,
+                          &service->keys.identity_pk) &&
+        ed25519_pubkey_eq(&dir_conn->hs_ident->blinded_pk,
+                          &desc->blinded_kp.pubkey)) {
+      connection_mark_for_close(conn);
+      count++;
+      continue;
+    }
+  } SMARTLIST_FOREACH_END(conn);
+
+  log_info(LD_REND, "Closed %u active service directory connections for "
+                    "descriptor %s of service %s",
+           count, safe_str_client(ed25519_fmt(&desc->blinded_kp.pubkey)),
+           safe_str_client(service->onion_address));
+  /* We don't have ownership of the objects in this list. */
+  smartlist_free(dir_conns);
+}
+
+/* Close all rendezvous circuits for the given service. */
+static void
+close_service_rp_circuits(hs_service_t *service)
+{
+  origin_circuit_t *ocirc = NULL;
+
+  tor_assert(service);
+
+  /* The reason we go over all circuit instead of using the circuitmap API is
+   * because most hidden service circuits are rendezvous circuits so there is
+   * no real improvement at getting all rendezvous circuits from the
+   * circuitmap and then going over them all to find the right ones.
+   * Furthermore, another option would have been to keep a list of RP cookies
+   * for a service but it creates an engineering complexity since we don't
+   * have a "RP circuit closed" event to clean it up properly so we avoid a
+   * memory DoS possibility. */
+
+  while ((ocirc = circuit_get_next_service_rp_circ(ocirc))) {
+    /* Only close circuits that are v3 and for this service. */
+    if (ocirc->hs_ident != NULL &&
+        ed25519_pubkey_eq(&ocirc->hs_ident->identity_pk,
+                          &service->keys.identity_pk)) {
+      /* Reason is FINISHED because service has been removed and thus the
+       * circuit is considered old/uneeded. When freed, it is removed from the
+       * hs circuitmap. */
+      circuit_mark_for_close(TO_CIRCUIT(ocirc), END_CIRC_REASON_FINISHED);
+    }
+  }
+}
+
+/* Close the circuit(s) for the given map of introduction points. */
+static void
+close_intro_circuits(hs_service_intropoints_t *intro_points)
+{
+  tor_assert(intro_points);
+
+  DIGEST256MAP_FOREACH(intro_points->map, key,
+                       const hs_service_intro_point_t *, ip) {
+    origin_circuit_t *ocirc = hs_circ_service_get_intro_circ(ip);
+    if (ocirc) {
+      /* Reason is FINISHED because service has been removed and thus the
+       * circuit is considered old/uneeded. When freed, the circuit is removed
+       * from the HS circuitmap. */
+      circuit_mark_for_close(TO_CIRCUIT(ocirc), END_CIRC_REASON_FINISHED);
+    }
+  } DIGEST256MAP_FOREACH_END;
+}
+
+/* Close all introduction circuits for the given service. */
+static void
+close_service_intro_circuits(hs_service_t *service)
+{
+  tor_assert(service);
+
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    close_intro_circuits(&desc->intro_points);
+  } FOR_EACH_DESCRIPTOR_END;
+}
+
+/* Close any circuits related to the given service. */
+static void
+close_service_circuits(hs_service_t *service)
+{
+  tor_assert(service);
+
+  /* Only support for version >= 3. */
+  if (BUG(service->config.version < HS_VERSION_THREE)) {
+    return;
+  }
+  /* Close intro points. */
+  close_service_intro_circuits(service);
+  /* Close rendezvous points. */
+  close_service_rp_circuits(service);
+}
+
+/* Move every ephemeral services from the src service map to the dst service
+ * map. It is possible that a service can't be register to the dst map which
+ * won't stop the process of moving them all but will trigger a log warn. */
+static void
+move_ephemeral_services(hs_service_ht *src, hs_service_ht *dst)
+{
+  hs_service_t **iter, **next;
+
+  tor_assert(src);
+  tor_assert(dst);
+
+  /* Iterate over the map to find ephemeral service and move them to the other
+   * map. We loop using this method to have a safe removal process. */
+  for (iter = HT_START(hs_service_ht, src); iter != NULL; iter = next) {
+    hs_service_t *s = *iter;
+    if (!s->config.is_ephemeral) {
+      /* Yeah, we are in a very manual loop :). */
+      next = HT_NEXT(hs_service_ht, src, iter);
+      continue;
+    }
+    /* Remove service from map and then register to it to the other map.
+     * Reminder that "*iter" and "s" are the same thing. */
+    next = HT_NEXT_RMV(hs_service_ht, src, iter);
+    if (register_service(dst, s) < 0) {
+      log_warn(LD_BUG, "Ephemeral service key is already being used. "
+                       "Skipping.");
+    }
+  }
+}
+
+/* Return a const string of the directory path escaped. If this is an
+ * ephemeral service, it returns "[EPHEMERAL]". This can only be called from
+ * the main thread because escaped() uses a static variable. */
+static const char *
+service_escaped_dir(const hs_service_t *s)
+{
+  return (s->config.is_ephemeral) ? "[EPHEMERAL]" :
+                                    escaped(s->config.directory_path);
+}
+
+/** Move the hidden service state from <b>src</b> to <b>dst</b>. We do this
+ *  when we receive a SIGHUP: <b>dst</b> is the post-HUP service */
+static void
+move_hs_state(hs_service_t *src_service, hs_service_t *dst_service)
+{
+  tor_assert(src_service);
+  tor_assert(dst_service);
+
+  hs_service_state_t *src = &src_service->state;
+  hs_service_state_t *dst = &dst_service->state;
+
+  /* Let's do a shallow copy */
+  dst->intro_circ_retry_started_time = src->intro_circ_retry_started_time;
+  dst->num_intro_circ_launched = src->num_intro_circ_launched;
+  /* Freeing a NULL replaycache triggers an info LD_BUG. */
+  if (dst->replay_cache_rend_cookie != NULL) {
+    replaycache_free(dst->replay_cache_rend_cookie);
+  }
+  dst->replay_cache_rend_cookie = src->replay_cache_rend_cookie;
+  dst->next_rotation_time = src->next_rotation_time;
+
+  src->replay_cache_rend_cookie = NULL; /* steal pointer reference */
+}
+
+/* Register services that are in the staging list. Once this function returns,
+ * the global service map will be set with the right content and all non
+ * surviving services will be cleaned up. */
+static void
+register_all_services(void)
+{
+  struct hs_service_ht *new_service_map;
+
+  tor_assert(hs_service_staging_list);
+
+  /* Allocate a new map that will replace the current one. */
+  new_service_map = tor_malloc_zero(sizeof(*new_service_map));
+  HT_INIT(hs_service_ht, new_service_map);
+
+  /* First step is to transfer all ephemeral services from the current global
+   * map to the new one we are constructing. We do not prune ephemeral
+   * services as the only way to kill them is by deleting it from the control
+   * port or stopping the tor daemon. */
+  move_ephemeral_services(hs_service_map, new_service_map);
+
+  SMARTLIST_FOREACH_BEGIN(hs_service_staging_list, hs_service_t *, snew) {
+    hs_service_t *s;
+
+    /* Check if that service is already in our global map and if so, we'll
+     * transfer the intro points to it. */
+    s = find_service(hs_service_map, &snew->keys.identity_pk);
+    if (s) {
+      /* Pass ownership of the descriptors from s (the current service) to
+       * snew (the newly configured one). */
+      move_descriptors(s, snew);
+      move_hs_state(s, snew);
+      /* Remove the service from the global map because after this, we need to
+       * go over the remaining service in that map that aren't surviving the
+       * reload to close their circuits. */
+      remove_service(hs_service_map, s);
+      hs_service_free(s);
+    }
+    /* Great, this service is now ready to be added to our new map. */
+    if (BUG(register_service(new_service_map, snew) < 0)) {
+      /* This should never happen because prior to registration, we validate
+       * every service against the entire set. Not being able to register a
+       * service means we failed to validate correctly. In that case, don't
+       * break tor and ignore the service but tell user. */
+      log_warn(LD_BUG, "Unable to register service with directory %s",
+               service_escaped_dir(snew));
+      SMARTLIST_DEL_CURRENT(hs_service_staging_list, snew);
+      hs_service_free(snew);
+    }
+  } SMARTLIST_FOREACH_END(snew);
+
+  /* Close any circuits associated with the non surviving services. Every
+   * service in the current global map are roaming. */
+  FOR_EACH_SERVICE_BEGIN(service) {
+    close_service_circuits(service);
+  } FOR_EACH_SERVICE_END;
+
+  /* Time to make the switch. We'll clear the staging list because its content
+   * has now changed ownership to the map. */
+  smartlist_clear(hs_service_staging_list);
+  service_free_all();
+  hs_service_map = new_service_map;
+  /* We've just register services into the new map and now we've replaced the
+   * global map with it so we have to notify that the change happened. When
+   * registering a service, the notify is only triggered if the destination
+   * map is the global map for which in here it was not. */
+  hs_service_map_has_changed();
+}
+
+/* Write the onion address of a given service to the given filename fname_ in
+ * the service directory. Return 0 on success else -1 on error. */
+STATIC int
+write_address_to_file(const hs_service_t *service, const char *fname_)
+{
+  int ret = -1;
+  char *fname = NULL;
+  char *addr_buf = NULL;
+
+  tor_assert(service);
+  tor_assert(fname_);
+
+  /* Construct the full address with the onion tld and write the hostname file
+   * to disk. */
+  tor_asprintf(&addr_buf, "%s.%s\n", service->onion_address, address_tld);
+  /* Notice here that we use the given "fname_". */
+  fname = hs_path_from_filename(service->config.directory_path, fname_);
+  if (write_str_to_file(fname, addr_buf, 0) < 0) {
+    log_warn(LD_REND, "Could not write onion address to hostname file %s",
+             escaped(fname));
+    goto end;
+  }
+
+#ifndef _WIN32
+  if (service->config.dir_group_readable) {
+    /* Mode to 0640. */
+    if (chmod(fname, S_IRUSR | S_IWUSR | S_IRGRP) < 0) {
+      log_warn(LD_FS, "Unable to make onion service hostname file %s "
+                      "group-readable.", escaped(fname));
+    }
+  }
+#endif /* !defined(_WIN32) */
+
+  /* Success. */
+  ret = 0;
+ end:
+  tor_free(fname);
+  tor_free(addr_buf);
+  return ret;
+}
+
+/* Load and/or generate private keys for the given service. On success, the
+ * hostname file will be written to disk along with the master private key iff
+ * the service is not configured for offline keys. Return 0 on success else -1
+ * on failure. */
+static int
+load_service_keys(hs_service_t *service)
+{
+  int ret = -1;
+  char *fname = NULL;
+  ed25519_keypair_t *kp;
+  const hs_service_config_t *config;
+
+  tor_assert(service);
+
+  config = &service->config;
+
+  /* Create and fix permission on service directory. We are about to write
+   * files to that directory so make sure it exists and has the right
+   * permissions. We do this here because at this stage we know that Tor is
+   * actually running and the service we have has been validated. */
+  if (hs_check_service_private_dir(get_options()->User,
+                                   config->directory_path,
+                                   config->dir_group_readable, 1) < 0) {
+    goto end;
+  }
+
+  /* Try to load the keys from file or generate it if not found. */
+  fname = hs_path_from_filename(config->directory_path, fname_keyfile_prefix);
+  /* Don't ask for key creation, we want to know if we were able to load it or
+   * we had to generate it. Better logging! */
+  kp = ed_key_init_from_file(fname, INIT_ED_KEY_SPLIT, LOG_INFO, NULL, 0, 0,
+                             0, NULL, NULL);
+  if (!kp) {
+    log_info(LD_REND, "Unable to load keys from %s. Generating it...", fname);
+    /* We'll now try to generate the keys and for it we want the strongest
+     * randomness for it. The keypair will be written in different files. */
+    uint32_t key_flags = INIT_ED_KEY_CREATE | INIT_ED_KEY_EXTRA_STRONG |
+                         INIT_ED_KEY_SPLIT;
+    kp = ed_key_init_from_file(fname, key_flags, LOG_WARN, NULL, 0, 0, 0,
+                               NULL, NULL);
+    if (!kp) {
+      log_warn(LD_REND, "Unable to generate keys and save in %s.", fname);
+      goto end;
+    }
+  }
+
+  /* Copy loaded or generated keys to service object. */
+  ed25519_pubkey_copy(&service->keys.identity_pk, &kp->pubkey);
+  memcpy(&service->keys.identity_sk, &kp->seckey,
+         sizeof(service->keys.identity_sk));
+  /* This does a proper memory wipe. */
+  ed25519_keypair_free(kp);
+
+  /* Build onion address from the newly loaded keys. */
+  tor_assert(service->config.version <= UINT8_MAX);
+  hs_build_address(&service->keys.identity_pk,
+                   (uint8_t) service->config.version,
+                   service->onion_address);
+
+  /* Write onion address to hostname file. */
+  if (write_address_to_file(service, fname_hostname) < 0) {
+    goto end;
+  }
+
+  /* Load all client authorization keys in the service. */
+  if (load_client_keys(service) < 0) {
+    goto end;
+  }
+
+  /* Succes. */
+  ret = 0;
+ end:
+  tor_free(fname);
+  return ret;
+}
+
+/* Check if the client file name is valid or not. Return 1 if valid,
+ * otherwise return 0. */
+STATIC int
+client_filename_is_valid(const char *filename)
+{
+  int ret = 1;
+  const char *valid_extension = ".auth";
+
+  tor_assert(filename);
+
+  /* The file extension must match and the total filename length can't be the
+   * length of the extension else we do not have a filename. */
+  if (!strcmpend(filename, valid_extension) &&
+      strlen(filename) != strlen(valid_extension)) {
+    ret = 1;
+  } else {
+    ret = 0;
+  }
+
+  return ret;
+}
+
+/* Parse an authorized client from a string. The format of a client string
+ * looks like (see rend-spec-v3.txt):
+ *
+ *  <auth-type>:<key-type>:<base32-encoded-public-key>
+ *
+ * The <auth-type> can only be "descriptor".
+ * The <key-type> can only be "x25519".
+ *
+ * Return the key on success, return NULL, otherwise. */
+STATIC hs_service_authorized_client_t *
+parse_authorized_client(const char *client_key_str)
+{
+  char *auth_type = NULL;
+  char *key_type = NULL;
+  char *pubkey_b32 = NULL;
+  hs_service_authorized_client_t *client = NULL;
+  smartlist_t *fields = smartlist_new();
+
+  tor_assert(client_key_str);
+
+  smartlist_split_string(fields, client_key_str, ":",
+                         SPLIT_SKIP_SPACE, 0);
+  /* Wrong number of fields. */
+  if (smartlist_len(fields) != 3) {
+    log_warn(LD_REND, "Unknown format of client authorization file.");
+    goto err;
+  }
+
+  auth_type = smartlist_get(fields, 0);
+  key_type = smartlist_get(fields, 1);
+  pubkey_b32 = smartlist_get(fields, 2);
+
+  /* Currently, the only supported auth type is "descriptor". */
+  if (strcmp(auth_type, "descriptor")) {
+    log_warn(LD_REND, "Client authorization auth type '%s' not supported.",
+             auth_type);
+    goto err;
+  }
+
+  /* Currently, the only supported key type is "x25519". */
+  if (strcmp(key_type, "x25519")) {
+    log_warn(LD_REND, "Client authorization key type '%s' not supported.",
+             key_type);
+    goto err;
+  }
+
+  /* We expect a specific length of the base32 encoded key so make sure we
+   * have that so we don't successfully decode a value with a different length
+   * and end up in trouble when copying the decoded key into a fixed length
+   * buffer. */
+  if (strlen(pubkey_b32) != BASE32_NOPAD_LEN(CURVE25519_PUBKEY_LEN)) {
+    log_warn(LD_REND, "Client authorization encoded base32 public key "
+                      "length is invalid: %s", pubkey_b32);
+    goto err;
+  }
+
+  client = tor_malloc_zero(sizeof(hs_service_authorized_client_t));
+  if (base32_decode((char *) client->client_pk.public_key,
+                    sizeof(client->client_pk.public_key),
+                    pubkey_b32, strlen(pubkey_b32)) < 0) {
+    log_warn(LD_REND, "Client authorization public key cannot be decoded: %s",
+             pubkey_b32);
+    goto err;
+  }
+
+  /* Success. */
+  goto done;
+
+ err:
+  service_authorized_client_free(client);
+ done:
+  /* It is also a good idea to wipe the public key. */
+  if (pubkey_b32) {
+    memwipe(pubkey_b32, 0, strlen(pubkey_b32));
+  }
+  tor_assert(fields);
+  SMARTLIST_FOREACH(fields, char *, s, tor_free(s));
+  smartlist_free(fields);
+  return client;
+}
+
+/* Load all the client public keys for the given service. Return 0 on
+ * success else -1 on failure. */
+static int
+load_client_keys(hs_service_t *service)
+{
+  int ret = -1;
+  char *client_key_str = NULL;
+  char *client_key_file_path = NULL;
+  char *client_keys_dir_path = NULL;
+  hs_service_config_t *config;
+  smartlist_t *file_list = NULL;
+
+  tor_assert(service);
+
+  config = &service->config;
+
+  /* Before calling this function, we already call load_service_keys to make
+   * sure that the directory exists with the right permission. So, if we
+   * cannot create a client pubkey key directory, we consider it as a bug. */
+  client_keys_dir_path = hs_path_from_filename(config->directory_path,
+                                               dname_client_pubkeys);
+  if (BUG(hs_check_service_private_dir(get_options()->User,
+                                       client_keys_dir_path,
+                                       config->dir_group_readable, 1) < 0)) {
+    goto end;
+  }
+
+  /* If the list of clients already exists, we must clear it first. */
+  if (config->clients) {
+    SMARTLIST_FOREACH(config->clients, hs_service_authorized_client_t *, p,
+                      service_authorized_client_free(p));
+    smartlist_free(config->clients);
+  }
+
+  config->clients = smartlist_new();
+
+  file_list = tor_listdir(client_keys_dir_path);
+  if (file_list == NULL) {
+    log_warn(LD_REND, "Client authorization directory %s can't be listed.",
+             client_keys_dir_path);
+    goto end;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(file_list, const char *, filename) {
+    hs_service_authorized_client_t *client = NULL;
+    log_info(LD_REND, "Loading a client authorization key file %s...",
+             filename);
+
+    if (!client_filename_is_valid(filename)) {
+      log_warn(LD_REND, "Client authorization unrecognized filename %s. "
+                        "File must end in .auth. Ignoring.", filename);
+      continue;
+    }
+
+    /* Create a full path for a file. */
+    client_key_file_path = hs_path_from_filename(client_keys_dir_path,
+                                                 filename);
+    client_key_str = read_file_to_str(client_key_file_path, 0, NULL);
+
+    /* If we cannot read the file, continue with the next file. */
+    if (!client_key_str)  {
+      log_warn(LD_REND, "Client authorization file %s can't be read. "
+                        "Corrupted or verify permission? Ignoring.",
+               client_key_file_path);
+      tor_free(client_key_file_path);
+      continue;
+    }
+    tor_free(client_key_file_path);
+
+    client = parse_authorized_client(client_key_str);
+    /* Wipe and free immediately after using it. */
+    memwipe(client_key_str, 0, strlen(client_key_str));
+    tor_free(client_key_str);
+
+    if (client) {
+      smartlist_add(config->clients, client);
+      log_info(LD_REND, "Loaded a client authorization key file %s.",
+               filename);
+    }
+
+  } SMARTLIST_FOREACH_END(filename);
+
+  /* If the number of clients is greater than zero, set the flag to be true. */
+  if (smartlist_len(config->clients) > 0) {
+    config->is_client_auth_enabled = 1;
+  }
+
+  /* Success. */
+  ret = 0;
+ end:
+  if (client_key_str) {
+    memwipe(client_key_str, 0, strlen(client_key_str));
+  }
+  if (file_list) {
+    SMARTLIST_FOREACH(file_list, char *, s, tor_free(s));
+    smartlist_free(file_list);
+  }
+  tor_free(client_key_str);
+  tor_free(client_key_file_path);
+  tor_free(client_keys_dir_path);
+  return ret;
+}
+
+STATIC void
+service_authorized_client_free_(hs_service_authorized_client_t *client)
+{
+  if (!client) {
+    return;
+  }
+  memwipe(&client->client_pk, 0, sizeof(client->client_pk));
+  tor_free(client);
+}
+
+/* Free a given service descriptor object and all key material is wiped. */
+STATIC void
+service_descriptor_free_(hs_service_descriptor_t *desc)
+{
+  if (!desc) {
+    return;
+  }
+  hs_descriptor_free(desc->desc);
+  memwipe(&desc->signing_kp, 0, sizeof(desc->signing_kp));
+  memwipe(&desc->blinded_kp, 0, sizeof(desc->blinded_kp));
+  /* Cleanup all intro points. */
+  digest256map_free(desc->intro_points.map, service_intro_point_free_void);
+  digestmap_free(desc->intro_points.failed_id, tor_free_);
+  if (desc->previous_hsdirs) {
+    SMARTLIST_FOREACH(desc->previous_hsdirs, char *, s, tor_free(s));
+    smartlist_free(desc->previous_hsdirs);
+  }
+  crypto_ope_free(desc->ope_cipher);
+  tor_free(desc);
+}
+
+/* Return a newly allocated service descriptor object. */
+STATIC hs_service_descriptor_t *
+service_descriptor_new(void)
+{
+  hs_service_descriptor_t *sdesc = tor_malloc_zero(sizeof(*sdesc));
+  sdesc->desc = tor_malloc_zero(sizeof(hs_descriptor_t));
+  /* Initialize the intro points map. */
+  sdesc->intro_points.map = digest256map_new();
+  sdesc->intro_points.failed_id = digestmap_new();
+  sdesc->previous_hsdirs = smartlist_new();
+  return sdesc;
+}
+
+/* Allocate and return a deep copy of client. */
+static hs_service_authorized_client_t *
+service_authorized_client_dup(const hs_service_authorized_client_t *client)
+{
+  hs_service_authorized_client_t *client_dup = NULL;
+
+  tor_assert(client);
+
+  client_dup = tor_malloc_zero(sizeof(hs_service_authorized_client_t));
+  /* Currently, the public key is the only component of
+   * hs_service_authorized_client_t. */
+  memcpy(client_dup->client_pk.public_key,
+         client->client_pk.public_key,
+         CURVE25519_PUBKEY_LEN);
+
+  return client_dup;
+}
+
+/* If two authorized clients are equal, return 0. If the first one should come
+ * before the second, return less than zero. If the first should come after
+ * the second, return greater than zero. */
+static int
+service_authorized_client_cmp(const hs_service_authorized_client_t *client1,
+                              const hs_service_authorized_client_t *client2)
+{
+  tor_assert(client1);
+  tor_assert(client2);
+
+  /* Currently, the public key is the only component of
+   * hs_service_authorized_client_t. */
+  return tor_memcmp(client1->client_pk.public_key,
+                    client2->client_pk.public_key,
+                    CURVE25519_PUBKEY_LEN);
+}
+
+/* Helper for sorting authorized clients. */
+static int
+compare_service_authorzized_client_(const void **_a, const void **_b)
+{
+  const hs_service_authorized_client_t *a = *_a, *b = *_b;
+  return service_authorized_client_cmp(a, b);
+}
+
+/* If the list of hs_service_authorized_client_t's is different between
+ * src and dst, return 1. Otherwise, return 0. */
+STATIC int
+service_authorized_client_config_equal(const hs_service_config_t *config1,
+                                       const hs_service_config_t *config2)
+{
+  int ret = 0;
+  int i;
+  smartlist_t *sl1 = smartlist_new();
+  smartlist_t *sl2 = smartlist_new();
+
+  tor_assert(config1);
+  tor_assert(config2);
+  tor_assert(config1->clients);
+  tor_assert(config2->clients);
+
+  /* If the number of clients is different, it is obvious that the list
+   * changes. */
+  if (smartlist_len(config1->clients) != smartlist_len(config2->clients)) {
+    goto done;
+  }
+
+  /* We do not want to mutate config1 and config2, so we will duplicate both
+   * entire client lists here. */
+  SMARTLIST_FOREACH(config1->clients,
+              hs_service_authorized_client_t *, client,
+              smartlist_add(sl1, service_authorized_client_dup(client)));
+
+  SMARTLIST_FOREACH(config2->clients,
+              hs_service_authorized_client_t *, client,
+              smartlist_add(sl2, service_authorized_client_dup(client)));
+
+  smartlist_sort(sl1, compare_service_authorzized_client_);
+  smartlist_sort(sl2, compare_service_authorzized_client_);
+
+  for (i = 0; i < smartlist_len(sl1); i++) {
+    /* If the clients at index i in both lists differ, the whole configs
+     * differ. */
+    if (service_authorized_client_cmp(smartlist_get(sl1, i),
+                                      smartlist_get(sl2, i))) {
+      goto done;
+    }
+  }
+
+  /* Success. */
+  ret = 1;
+
+ done:
+  if (sl1) {
+    SMARTLIST_FOREACH(sl1, hs_service_authorized_client_t *, p,
+                      service_authorized_client_free(p));
+    smartlist_free(sl1);
+  }
+  if (sl2) {
+    SMARTLIST_FOREACH(sl2, hs_service_authorized_client_t *, p,
+                      service_authorized_client_free(p));
+    smartlist_free(sl2);
+  }
+  return ret;
+}
+
+/* Move descriptor(s) from the src service to the dst service and modify their
+ * content if necessary. We do this during SIGHUP when we re-create our
+ * hidden services. */
+static void
+move_descriptors(hs_service_t *src, hs_service_t *dst)
+{
+  tor_assert(src);
+  tor_assert(dst);
+
+  if (src->desc_current) {
+    /* Nothing should be there, but clean it up just in case */
+    if (BUG(dst->desc_current)) {
+      service_descriptor_free(dst->desc_current);
+    }
+    dst->desc_current = src->desc_current;
+    src->desc_current = NULL;
+  }
+
+  if (src->desc_next) {
+    /* Nothing should be there, but clean it up just in case */
+    if (BUG(dst->desc_next)) {
+      service_descriptor_free(dst->desc_next);
+    }
+    dst->desc_next = src->desc_next;
+    src->desc_next = NULL;
+  }
+
+  /* If the client authorization changes, we must rebuild the superencrypted
+   * section and republish the descriptors. */
+  int client_auth_changed =
+    !service_authorized_client_config_equal(&src->config, &dst->config);
+  if (client_auth_changed && dst->desc_current) {
+    /* We have to clear the superencrypted content first. */
+    hs_desc_superencrypted_data_free_contents(
+                                &dst->desc_current->desc->superencrypted_data);
+    if (build_service_desc_superencrypted(dst, dst->desc_current) < 0) {
+      goto err;
+    }
+    service_desc_schedule_upload(dst->desc_current, time(NULL), 1);
+  }
+  if (client_auth_changed && dst->desc_next) {
+    /* We have to clear the superencrypted content first. */
+    hs_desc_superencrypted_data_free_contents(
+                                &dst->desc_next->desc->superencrypted_data);
+    if (build_service_desc_superencrypted(dst, dst->desc_next) < 0) {
+      goto err;
+    }
+    service_desc_schedule_upload(dst->desc_next, time(NULL), 1);
+  }
+
+  return;
+
+ err:
+  /* If there is an error, free all descriptors to make it clean and generate
+   * them later. */
+  service_descriptor_free(dst->desc_current);
+  service_descriptor_free(dst->desc_next);
+}
+
+/* From the given service, remove all expired failing intro points for each
+ * descriptor. */
+static void
+remove_expired_failing_intro(hs_service_t *service, time_t now)
+{
+  tor_assert(service);
+
+  /* For both descriptors, cleanup the failing intro points list. */
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    DIGESTMAP_FOREACH_MODIFY(desc->intro_points.failed_id, key, time_t *, t) {
+      time_t failure_time = *t;
+      if ((failure_time + INTRO_CIRC_RETRY_PERIOD) <= now) {
+        MAP_DEL_CURRENT(key);
+        tor_free(t);
+      }
+    } DIGESTMAP_FOREACH_END;
+  } FOR_EACH_DESCRIPTOR_END;
+}
+
+/* For the given descriptor desc, put all node_t object found from its failing
+ * intro point list and put them in the given node_list. */
+static void
+setup_intro_point_exclude_list(const hs_service_descriptor_t *desc,
+                               smartlist_t *node_list)
+{
+  tor_assert(desc);
+  tor_assert(node_list);
+
+  DIGESTMAP_FOREACH(desc->intro_points.failed_id, key, time_t *, t) {
+    (void) t; /* Make gcc happy. */
+    const node_t *node = node_get_by_id(key);
+    if (node) {
+      smartlist_add(node_list, (void *) node);
+    }
+  } DIGESTMAP_FOREACH_END;
+}
+
+/* For the given failing intro point ip, we add its time of failure to the
+ * failed map and index it by identity digest (legacy ID) in the descriptor
+ * desc failed id map. */
+static void
+remember_failing_intro_point(const hs_service_intro_point_t *ip,
+                             hs_service_descriptor_t *desc, time_t now)
+{
+  time_t *time_of_failure, *prev_ptr;
+  const hs_desc_link_specifier_t *legacy_ls;
+
+  tor_assert(ip);
+  tor_assert(desc);
+
+  time_of_failure = tor_malloc_zero(sizeof(time_t));
+  *time_of_failure = now;
+  legacy_ls = get_link_spec_by_type(ip, LS_LEGACY_ID);
+  tor_assert(legacy_ls);
+  prev_ptr = digestmap_set(desc->intro_points.failed_id,
+                           (const char *) legacy_ls->u.legacy_id,
+                           time_of_failure);
+  tor_free(prev_ptr);
+}
+
+/* Copy the descriptor link specifier object from src to dst. */
+static void
+link_specifier_copy(hs_desc_link_specifier_t *dst,
+                    const hs_desc_link_specifier_t *src)
+{
+  tor_assert(dst);
+  tor_assert(src);
+  memcpy(dst, src, sizeof(hs_desc_link_specifier_t));
+}
+
+/* Using a given descriptor signing keypair signing_kp, a service intro point
+ * object ip and the time now, setup the content of an already allocated
+ * descriptor intro desc_ip.
+ *
+ * Return 0 on success else a negative value. */
+static int
+setup_desc_intro_point(const ed25519_keypair_t *signing_kp,
+                       const hs_service_intro_point_t *ip,
+                       time_t now, hs_desc_intro_point_t *desc_ip)
+{
+  int ret = -1;
+  time_t nearest_hour = now - (now % 3600);
+
+  tor_assert(signing_kp);
+  tor_assert(ip);
+  tor_assert(desc_ip);
+
+  /* Copy the onion key. */
+  memcpy(&desc_ip->onion_key, &ip->onion_key, sizeof(desc_ip->onion_key));
+
+  /* Key and certificate material. */
+  desc_ip->auth_key_cert = tor_cert_create(signing_kp,
+                                           CERT_TYPE_AUTH_HS_IP_KEY,
+                                           &ip->auth_key_kp.pubkey,
+                                           nearest_hour,
+                                           HS_DESC_CERT_LIFETIME,
+                                           CERT_FLAG_INCLUDE_SIGNING_KEY);
+  if (desc_ip->auth_key_cert == NULL) {
+    log_warn(LD_REND, "Unable to create intro point auth-key certificate");
+    goto done;
+  }
+
+  /* Copy link specifier(s). */
+  SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
+                          const hs_desc_link_specifier_t *, ls) {
+    hs_desc_link_specifier_t *copy = tor_malloc_zero(sizeof(*copy));
+    link_specifier_copy(copy, ls);
+    smartlist_add(desc_ip->link_specifiers, copy);
+  } SMARTLIST_FOREACH_END(ls);
+
+  /* For a legacy intro point, we'll use an RSA/ed cross certificate. */
+  if (ip->base.is_only_legacy) {
+    desc_ip->legacy.key = crypto_pk_dup_key(ip->legacy_key);
+    /* Create cross certification cert. */
+    ssize_t cert_len = tor_make_rsa_ed25519_crosscert(
+                                    &signing_kp->pubkey,
+                                    desc_ip->legacy.key,
+                                    nearest_hour + HS_DESC_CERT_LIFETIME,
+                                    &desc_ip->legacy.cert.encoded);
+    if (cert_len < 0) {
+      log_warn(LD_REND, "Unable to create enc key legacy cross cert.");
+      goto done;
+    }
+    desc_ip->legacy.cert.len = cert_len;
+  }
+
+  /* Encryption key and its cross certificate. */
+  {
+    ed25519_public_key_t ed25519_pubkey;
+
+    /* Use the public curve25519 key. */
+    memcpy(&desc_ip->enc_key, &ip->enc_key_kp.pubkey,
+           sizeof(desc_ip->enc_key));
+    /* The following can't fail. */
+    ed25519_public_key_from_curve25519_public_key(&ed25519_pubkey,
+                                                  &ip->enc_key_kp.pubkey,
+                                                  0);
+    desc_ip->enc_key_cert = tor_cert_create(signing_kp,
+                                            CERT_TYPE_CROSS_HS_IP_KEYS,
+                                            &ed25519_pubkey, nearest_hour,
+                                            HS_DESC_CERT_LIFETIME,
+                                            CERT_FLAG_INCLUDE_SIGNING_KEY);
+    if (desc_ip->enc_key_cert == NULL) {
+      log_warn(LD_REND, "Unable to create enc key curve25519 cross cert.");
+      goto done;
+    }
+  }
+  /* Success. */
+  ret = 0;
+
+ done:
+  return ret;
+}
+
+/* Using the given descriptor from the given service, build the descriptor
+ * intro point list so we can then encode the descriptor for publication. This
+ * function does not pick intro points, they have to be in the descriptor
+ * current map. Cryptographic material (keys) must be initialized in the
+ * descriptor for this function to make sense. */
+static void
+build_desc_intro_points(const hs_service_t *service,
+                        hs_service_descriptor_t *desc, time_t now)
+{
+  hs_desc_encrypted_data_t *encrypted;
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  /* Ease our life. */
+  encrypted = &desc->desc->encrypted_data;
+  /* Cleanup intro points, we are about to set them from scratch. */
+  hs_descriptor_clear_intro_points(desc->desc);
+
+  DIGEST256MAP_FOREACH(desc->intro_points.map, key,
+                       const hs_service_intro_point_t *, ip) {
+    if (!ip->circuit_established) {
+      /* Ignore un-established intro points. They can linger in that list
+       * because their circuit has not opened and they haven't been removed
+       * yet even though we have enough intro circuits.
+       *
+       * Due to #31561, it can stay in that list until rotation so this check
+       * prevents to publish an intro point without a circuit. */
+      continue;
+    }
+    hs_desc_intro_point_t *desc_ip = hs_desc_intro_point_new();
+    if (setup_desc_intro_point(&desc->signing_kp, ip, now, desc_ip) < 0) {
+      hs_desc_intro_point_free(desc_ip);
+      continue;
+    }
+    /* We have a valid descriptor intro point. Add it to the list. */
+    smartlist_add(encrypted->intro_points, desc_ip);
+  } DIGEST256MAP_FOREACH_END;
+}
+
+/* Build the descriptor signing key certificate. */
+static void
+build_desc_signing_key_cert(hs_service_descriptor_t *desc, time_t now)
+{
+  hs_desc_plaintext_data_t *plaintext;
+
+  tor_assert(desc);
+  tor_assert(desc->desc);
+
+  /* Ease our life a bit. */
+  plaintext = &desc->desc->plaintext_data;
+
+  /* Get rid of what we have right now. */
+  tor_cert_free(plaintext->signing_key_cert);
+
+  /* Fresh certificate for the signing key. */
+  plaintext->signing_key_cert =
+    tor_cert_create(&desc->blinded_kp, CERT_TYPE_SIGNING_HS_DESC,
+                    &desc->signing_kp.pubkey, now, HS_DESC_CERT_LIFETIME,
+                    CERT_FLAG_INCLUDE_SIGNING_KEY);
+  /* If the cert creation fails, the descriptor encoding will fail and thus
+   * ultimately won't be uploaded. We'll get a stack trace to help us learn
+   * where the call came from and the tor_cert_create() will log the error. */
+  tor_assert_nonfatal(plaintext->signing_key_cert);
+}
+
+/* Populate the descriptor encrypted section from the given service object.
+ * This will generate a valid list of introduction points that can be used
+ * after for circuit creation. Return 0 on success else -1 on error. */
+static int
+build_service_desc_encrypted(const hs_service_t *service,
+                             hs_service_descriptor_t *desc)
+{
+  hs_desc_encrypted_data_t *encrypted;
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  encrypted = &desc->desc->encrypted_data;
+
+  encrypted->create2_ntor = 1;
+  encrypted->single_onion_service = service->config.is_single_onion;
+
+  /* Setup introduction points from what we have in the service. */
+  if (encrypted->intro_points == NULL) {
+    encrypted->intro_points = smartlist_new();
+  }
+  /* We do NOT build introduction point yet, we only do that once the circuit
+   * have been opened. Until we have the right number of introduction points,
+   * we do not encode anything in the descriptor. */
+
+  /* XXX: Support client authorization (#20700). */
+  encrypted->intro_auth_types = NULL;
+  return 0;
+}
+
+/* Populate the descriptor superencrypted section from the given service
+ * object. This will generate a valid list of hs_desc_authorized_client_t
+ * of clients that are authorized to use the service. Return 0 on success
+ * else -1 on error. */
+static int
+build_service_desc_superencrypted(const hs_service_t *service,
+                                  hs_service_descriptor_t *desc)
+{
+  const hs_service_config_t *config;
+  int i;
+  hs_desc_superencrypted_data_t *superencrypted;
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  superencrypted = &desc->desc->superencrypted_data;
+  config = &service->config;
+
+  /* The ephemeral key pair is already generated, so this should not give
+   * an error. */
+  if (BUG(!curve25519_public_key_is_ok(&desc->auth_ephemeral_kp.pubkey))) {
+    return -1;
+  }
+  memcpy(&superencrypted->auth_ephemeral_pubkey,
+         &desc->auth_ephemeral_kp.pubkey,
+         sizeof(curve25519_public_key_t));
+
+  /* Test that subcred is not zero because we might use it below */
+  if (BUG(tor_mem_is_zero((char*)desc->desc->subcredential, DIGEST256_LEN))) {
+    return -1;
+  }
+
+  /* Create a smartlist to store clients */
+  superencrypted->clients = smartlist_new();
+
+  /* We do not need to build the desc authorized client if the client
+   * authorization is disabled */
+  if (config->is_client_auth_enabled) {
+    SMARTLIST_FOREACH_BEGIN(config->clients,
+                            hs_service_authorized_client_t *, client) {
+      hs_desc_authorized_client_t *desc_client;
+      desc_client = tor_malloc_zero(sizeof(hs_desc_authorized_client_t));
+
+      /* Prepare the client for descriptor and then add to the list in the
+       * superencrypted part of the descriptor */
+      hs_desc_build_authorized_client(desc->desc->subcredential,
+                                      &client->client_pk,
+                                      &desc->auth_ephemeral_kp.seckey,
+                                      desc->descriptor_cookie, desc_client);
+      smartlist_add(superencrypted->clients, desc_client);
+
+    } SMARTLIST_FOREACH_END(client);
+  }
+
+  /* We cannot let the number of auth-clients to be zero, so we need to
+   * make it be 16. If it is already a multiple of 16, we do not need to
+   * do anything. Otherwise, add the additional ones to make it a
+   * multiple of 16. */
+  int num_clients = smartlist_len(superencrypted->clients);
+  int num_clients_to_add;
+  if (num_clients == 0) {
+    num_clients_to_add = HS_DESC_AUTH_CLIENT_MULTIPLE;
+  } else if (num_clients % HS_DESC_AUTH_CLIENT_MULTIPLE == 0) {
+    num_clients_to_add = 0;
+  } else {
+    num_clients_to_add =
+      HS_DESC_AUTH_CLIENT_MULTIPLE
+      - (num_clients % HS_DESC_AUTH_CLIENT_MULTIPLE);
+  }
+
+  for (i = 0; i < num_clients_to_add; i++) {
+    hs_desc_authorized_client_t *desc_client =
+      hs_desc_build_fake_authorized_client();
+    smartlist_add(superencrypted->clients, desc_client);
+  }
+
+  /* Shuffle the list to prevent the client know the position in the
+   * config. */
+  smartlist_shuffle(superencrypted->clients);
+
+  return 0;
+}
+
+/* Populate the descriptor plaintext section from the given service object.
+ * The caller must make sure that the keys in the descriptors are valid that
+ * is are non-zero. This can't fail. */
+static void
+build_service_desc_plaintext(const hs_service_t *service,
+                             hs_service_descriptor_t *desc)
+{
+  hs_desc_plaintext_data_t *plaintext;
+
+  tor_assert(service);
+  tor_assert(desc);
+  tor_assert(!tor_mem_is_zero((char *) &desc->blinded_kp,
+                              sizeof(desc->blinded_kp)));
+  tor_assert(!tor_mem_is_zero((char *) &desc->signing_kp,
+                              sizeof(desc->signing_kp)));
+
+  /* Set the subcredential. */
+  hs_get_subcredential(&service->keys.identity_pk, &desc->blinded_kp.pubkey,
+                       desc->desc->subcredential);
+
+  plaintext = &desc->desc->plaintext_data;
+
+  plaintext->version = service->config.version;
+  plaintext->lifetime_sec = HS_DESC_DEFAULT_LIFETIME;
+  /* Copy public key material to go in the descriptor. */
+  ed25519_pubkey_copy(&plaintext->signing_pubkey, &desc->signing_kp.pubkey);
+  ed25519_pubkey_copy(&plaintext->blinded_pubkey, &desc->blinded_kp.pubkey);
+
+  /* Create the signing key certificate. This will be updated before each
+   * upload but we create it here so we don't complexify our unit tests. */
+  build_desc_signing_key_cert(desc, approx_time());
+}
+
+/** Compute the descriptor's OPE cipher for encrypting revision counters. */
+static crypto_ope_t *
+generate_ope_cipher_for_desc(const hs_service_descriptor_t *hs_desc)
+{
+  /* Compute OPE key as H("rev-counter-generation" | blinded privkey) */
+  uint8_t key[DIGEST256_LEN];
+  crypto_digest_t *digest = crypto_digest256_new(DIGEST_SHA3_256);
+  const char ope_key_prefix[] = "rev-counter-generation";
+  const ed25519_secret_key_t *eph_privkey = &hs_desc->blinded_kp.seckey;
+  crypto_digest_add_bytes(digest, ope_key_prefix, sizeof(ope_key_prefix));
+  crypto_digest_add_bytes(digest, (char*)eph_privkey->seckey,
+                          sizeof(eph_privkey->seckey));
+  crypto_digest_get_digest(digest, (char *)key, sizeof(key));
+  crypto_digest_free(digest);
+
+  return crypto_ope_new(key);
+}
+
+/* For the given service and descriptor object, create the key material which
+ * is the blinded keypair, the descriptor signing keypair, the ephemeral
+ * keypair, and the descriptor cookie. Return 0 on success else -1 on error
+ * where the generated keys MUST be ignored. */
+static int
+build_service_desc_keys(const hs_service_t *service,
+                        hs_service_descriptor_t *desc)
+{
+  int ret = -1;
+  ed25519_keypair_t kp;
+
+  tor_assert(desc);
+  tor_assert(!tor_mem_is_zero((char *) &service->keys.identity_pk,
+             ED25519_PUBKEY_LEN));
+
+  /* XXX: Support offline key feature (#18098). */
+
+  /* Copy the identity keys to the keypair so we can use it to create the
+   * blinded key. */
+  memcpy(&kp.pubkey, &service->keys.identity_pk, sizeof(kp.pubkey));
+  memcpy(&kp.seckey, &service->keys.identity_sk, sizeof(kp.seckey));
+  /* Build blinded keypair for this time period. */
+  hs_build_blinded_keypair(&kp, NULL, 0, desc->time_period_num,
+                           &desc->blinded_kp);
+  /* Let's not keep too much traces of our keys in memory. */
+  memwipe(&kp, 0, sizeof(kp));
+
+  /* Compute the OPE cipher struct (it's tied to the current blinded key) */
+  log_info(LD_GENERAL,
+           "Getting OPE for TP#%u", (unsigned) desc->time_period_num);
+  tor_assert_nonfatal(!desc->ope_cipher);
+  desc->ope_cipher = generate_ope_cipher_for_desc(desc);
+
+  /* No need for extra strong, this is a temporary key only for this
+   * descriptor. Nothing long term. */
+  if (ed25519_keypair_generate(&desc->signing_kp, 0) < 0) {
+    log_warn(LD_REND, "Can't generate descriptor signing keypair for "
+                      "service %s",
+             safe_str_client(service->onion_address));
+    goto end;
+  }
+
+  /* No need for extra strong, this is a temporary key only for this
+   * descriptor. Nothing long term. */
+  if (curve25519_keypair_generate(&desc->auth_ephemeral_kp, 0) < 0) {
+    log_warn(LD_REND, "Can't generate auth ephemeral keypair for "
+                      "service %s",
+             safe_str_client(service->onion_address));
+    goto end;
+  }
+
+  /* Random descriptor cookie to be used as a part of a key to encrypt the
+   * descriptor, only if the client auth is enabled will it be used. */
+  crypto_strongest_rand(desc->descriptor_cookie,
+                        sizeof(desc->descriptor_cookie));
+
+  /* Success. */
+  ret = 0;
+ end:
+  return ret;
+}
+
+/* Given a service and the current time, build a descriptor for the service.
+ * This function does not pick introduction point, this needs to be done by
+ * the update function. On success, desc_out will point to the newly allocated
+ * descriptor object.
+ *
+ * This can error if we are unable to create keys or certificate. */
+static void
+build_service_descriptor(hs_service_t *service, uint64_t time_period_num,
+                         hs_service_descriptor_t **desc_out)
+{
+  char *encoded_desc;
+  hs_service_descriptor_t *desc;
+
+  tor_assert(service);
+  tor_assert(desc_out);
+
+  desc = service_descriptor_new();
+
+  /* Set current time period */
+  desc->time_period_num = time_period_num;
+
+  /* Create the needed keys so we can setup the descriptor content. */
+  if (build_service_desc_keys(service, desc) < 0) {
+    goto err;
+  }
+  /* Setup plaintext descriptor content. */
+  build_service_desc_plaintext(service, desc);
+
+  /* Setup superencrypted descriptor content. */
+  if (build_service_desc_superencrypted(service, desc) < 0) {
+    goto err;
+  }
+  /* Setup encrypted descriptor content. */
+  if (build_service_desc_encrypted(service, desc) < 0) {
+    goto err;
+  }
+
+  /* Let's make sure that we've created a descriptor that can actually be
+   * encoded properly. This function also checks if the encoded output is
+   * decodable after. */
+  if (BUG(service_encode_descriptor(service, desc, &desc->signing_kp,
+                                    &encoded_desc) < 0)) {
+    goto err;
+  }
+  tor_free(encoded_desc);
+
+  /* Assign newly built descriptor to the next slot. */
+  *desc_out = desc;
+  /* Fire a CREATED control port event. */
+  hs_control_desc_event_created(service->onion_address,
+                                &desc->blinded_kp.pubkey);
+  return;
+
+ err:
+  service_descriptor_free(desc);
+}
+
+/* Build both descriptors for the given service that has just booted up.
+ * Because it's a special case, it deserves its special function ;). */
+static void
+build_descriptors_for_new_service(hs_service_t *service, time_t now)
+{
+  uint64_t current_desc_tp, next_desc_tp;
+
+  tor_assert(service);
+  /* These are the conditions for a new service. */
+  tor_assert(!service->desc_current);
+  tor_assert(!service->desc_next);
+
+  /*
+   * +------------------------------------------------------------------+
+   * |                                                                  |
+   * | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   * | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   * |                                                                  |
+   * |  $==========|-----------$===========|-----------$===========|    |
+   * |                             ^         ^                          |
+   * |                             A         B                          |
+   * +------------------------------------------------------------------+
+   *
+   * Case A: The service boots up before a new time period, the current time
+   * period is thus TP#1 and the next is TP#2 which for both we have access to
+   * their SRVs.
+   *
+   * Case B: The service boots up inside TP#2, we can't use the TP#3 for the
+   * next descriptor because we don't have the SRV#3 so the current should be
+   * TP#1 and next TP#2.
+   */
+
+  if (hs_in_period_between_tp_and_srv(NULL, now)) {
+    /* Case B from the above, inside of the new time period. */
+    current_desc_tp = hs_get_previous_time_period_num(0); /* TP#1 */
+    next_desc_tp = hs_get_time_period_num(0);             /* TP#2 */
+  } else {
+    /* Case A from the above, outside of the new time period. */
+    current_desc_tp = hs_get_time_period_num(0);    /* TP#1 */
+    next_desc_tp = hs_get_next_time_period_num(0);  /* TP#2 */
+  }
+
+  /* Build descriptors. */
+  build_service_descriptor(service, current_desc_tp, &service->desc_current);
+  build_service_descriptor(service, next_desc_tp, &service->desc_next);
+  log_info(LD_REND, "Hidden service %s has just started. Both descriptors "
+                    "built. Now scheduled for upload.",
+           safe_str_client(service->onion_address));
+}
+
+/* Build descriptors for each service if needed. There are conditions to build
+ * a descriptor which are details in the function. */
+STATIC void
+build_all_descriptors(time_t now)
+{
+  FOR_EACH_SERVICE_BEGIN(service) {
+
+    /* A service booting up will have both descriptors to NULL. No other cases
+     * makes both descriptor non existent. */
+    if (service->desc_current == NULL && service->desc_next == NULL) {
+      build_descriptors_for_new_service(service, now);
+      continue;
+    }
+
+    /* Reaching this point means we are pass bootup so at runtime. We should
+     * *never* have an empty current descriptor. If the next descriptor is
+     * empty, we'll try to build it for the next time period. This only
+     * happens when we rotate meaning that we are guaranteed to have a new SRV
+     * at that point for the next time period. */
+    if (BUG(service->desc_current == NULL)) {
+      continue;
+    }
+
+    if (service->desc_next == NULL) {
+      build_service_descriptor(service, hs_get_next_time_period_num(0),
+                               &service->desc_next);
+      log_info(LD_REND, "Hidden service %s next descriptor successfully "
+                        "built. Now scheduled for upload.",
+               safe_str_client(service->onion_address));
+    }
+  } FOR_EACH_DESCRIPTOR_END;
+}
+
+/* Randomly pick a node to become an introduction point but not present in the
+ * given exclude_nodes list. The chosen node is put in the exclude list
+ * regardless of success or not because in case of failure, the node is simply
+ * unsusable from that point on.
+ *
+ * If direct_conn is set, try to pick a node that our local firewall/policy
+ * allows us to connect to directly. If we can't find any, return NULL.
+ * This function supports selecting dual-stack nodes for direct single onion
+ * service IPv6 connections. But it does not send IPv6 addresses in link
+ * specifiers. (Current clients don't use IPv6 addresses to extend, and
+ * direct client connections to intro points are not supported.)
+ *
+ * Return a newly allocated service intro point ready to be used for encoding.
+ * Return NULL on error. */
+static hs_service_intro_point_t *
+pick_intro_point(unsigned int direct_conn, smartlist_t *exclude_nodes)
+{
+  const or_options_t *options = get_options();
+  const node_t *node;
+  extend_info_t *info = NULL;
+  hs_service_intro_point_t *ip = NULL;
+  /* Normal 3-hop introduction point flags. */
+  router_crn_flags_t flags = CRN_NEED_UPTIME | CRN_NEED_DESC;
+  /* Single onion flags. */
+  router_crn_flags_t direct_flags = flags | CRN_PREF_ADDR | CRN_DIRECT_CONN;
+
+  node = router_choose_random_node(exclude_nodes, options->ExcludeNodes,
+                                   direct_conn ? direct_flags : flags);
+
+  /* If we are in single onion mode, retry node selection for a 3-hop
+   * path */
+  if (direct_conn && !node) {
+    log_info(LD_REND,
+             "Unable to find an intro point that we can connect to "
+             "directly, falling back to a 3-hop path.");
+    node = router_choose_random_node(exclude_nodes, options->ExcludeNodes,
+                                     flags);
+  }
+
+  if (!node) {
+    goto err;
+  }
+
+  /* We have a suitable node, add it to the exclude list. We do this *before*
+   * we can validate the extend information because even in case of failure,
+   * we don't want to use that node anymore. */
+  smartlist_add(exclude_nodes, (void *) node);
+
+  /* We do this to ease our life but also this call makes appropriate checks
+   * of the node object such as validating ntor support for instance.
+   *
+   * We must provide an extend_info for clients to connect over a 3-hop path,
+   * so we don't pass direct_conn here. */
+  info = extend_info_from_node(node, 0);
+  if (BUG(info == NULL)) {
+    goto err;
+  }
+
+  /* Let's do a basic sanity check here so that we don't end up advertising the
+   * ed25519 identity key of relays that don't actually support the link
+   * protocol */
+  if (!node_supports_ed25519_link_authentication(node, 0)) {
+    tor_assert_nonfatal(ed25519_public_key_is_zero(&info->ed_identity));
+  } else {
+    /* Make sure we *do* have an ed key if we support the link authentication.
+     * Sending an empty key would result in a failure to extend. */
+    tor_assert_nonfatal(!ed25519_public_key_is_zero(&info->ed_identity));
+  }
+
+  /* Create our objects and populate them with the node information.
+   * We don't care if the intro's link auth is compatible with us, because
+   * we are sending the ed25519 key to a remote client via the descriptor. */
+  ip = service_intro_point_new(info, !node_supports_ed25519_hs_intro(node),
+                               node_supports_ed25519_link_authentication(node,
+                                                                         0));
+  if (ip == NULL) {
+    goto err;
+  }
+
+  log_info(LD_REND, "Picked intro point: %s", extend_info_describe(info));
+  extend_info_free(info);
+  return ip;
+ err:
+  service_intro_point_free(ip);
+  extend_info_free(info);
+  return NULL;
+}
+
+/* For a given descriptor from the given service, pick any needed intro points
+ * and update the current map with those newly picked intro points. Return the
+ * number node that might have been added to the descriptor current map. */
+static unsigned int
+pick_needed_intro_points(hs_service_t *service,
+                         hs_service_descriptor_t *desc)
+{
+  int i = 0, num_needed_ip;
+  smartlist_t *exclude_nodes = smartlist_new();
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  /* Compute how many intro points we actually need to open. */
+  num_needed_ip = service->config.num_intro_points -
+                  digest256map_size(desc->intro_points.map);
+  if (BUG(num_needed_ip < 0)) {
+    /* Let's not make tor freak out here and just skip this. */
+    goto done;
+  }
+
+  /* We want to end up with config.num_intro_points intro points, but if we
+   * have no intro points at all (chances are they all cycled or we are
+   * starting up), we launch get_intro_point_num_extra() extra circuits and
+   * use the first config.num_intro_points that complete. See proposal #155,
+   * section 4 for the rationale of this which is purely for performance.
+   *
+   * The ones after the first config.num_intro_points will be converted to
+   * 'General' internal circuits and then we'll drop them from the list of
+   * intro points. */
+  if (digest256map_size(desc->intro_points.map) == 0) {
+    num_needed_ip += get_intro_point_num_extra();
+  }
+
+  /* Build an exclude list of nodes of our intro point(s). The expiring intro
+   * points are OK to pick again because this is afterall a concept of round
+   * robin so they are considered valid nodes to pick again. */
+  DIGEST256MAP_FOREACH(desc->intro_points.map, key,
+                       hs_service_intro_point_t *, ip) {
+    const node_t *intro_node = get_node_from_intro_point(ip);
+    if (intro_node) {
+      smartlist_add(exclude_nodes, (void*)intro_node);
+    }
+  } DIGEST256MAP_FOREACH_END;
+  /* Also, add the failing intro points that our descriptor encounteered in
+   * the exclude node list. */
+  setup_intro_point_exclude_list(desc, exclude_nodes);
+
+  for (i = 0; i < num_needed_ip; i++) {
+    hs_service_intro_point_t *ip;
+
+    /* This function will add the picked intro point node to the exclude nodes
+     * list so we don't pick the same one at the next iteration. */
+    ip = pick_intro_point(service->config.is_single_onion, exclude_nodes);
+    if (ip == NULL) {
+      /* If we end up unable to pick an introduction point it is because we
+       * can't find suitable node and calling this again is highly unlikely to
+       * give us a valid node all of the sudden. */
+      log_info(LD_REND, "Unable to find a suitable node to be an "
+                        "introduction point for service %s.",
+               safe_str_client(service->onion_address));
+      goto done;
+    }
+    /* Valid intro point object, add it to the descriptor current map. */
+    service_intro_point_add(desc->intro_points.map, ip);
+  }
+  /* We've successfully picked all our needed intro points thus none are
+   * missing which will tell our upload process to expect the number of
+   * circuits to be the number of configured intro points circuits and not the
+   * number of intro points object that we have. */
+  desc->missing_intro_points = 0;
+
+  /* Success. */
+ done:
+  /* We don't have ownership of the node_t object in this list. */
+  smartlist_free(exclude_nodes);
+  return i;
+}
+
+/** Clear previous cached HSDirs in <b>desc</b>. */
+static void
+service_desc_clear_previous_hsdirs(hs_service_descriptor_t *desc)
+{
+  if (BUG(!desc->previous_hsdirs)) {
+    return;
+  }
+
+  SMARTLIST_FOREACH(desc->previous_hsdirs, char*, s, tor_free(s));
+  smartlist_clear(desc->previous_hsdirs);
+}
+
+/** Note that we attempted to upload <b>desc</b> to <b>hsdir</b>. */
+static void
+service_desc_note_upload(hs_service_descriptor_t *desc, const node_t *hsdir)
+{
+  char b64_digest[BASE64_DIGEST_LEN+1] = {0};
+  digest_to_base64(b64_digest, hsdir->identity);
+
+  if (BUG(!desc->previous_hsdirs)) {
+    return;
+  }
+
+  if (!smartlist_contains_string(desc->previous_hsdirs, b64_digest)) {
+    smartlist_add_strdup(desc->previous_hsdirs, b64_digest);
+  }
+}
+
+/** Schedule an upload of <b>desc</b>. If <b>descriptor_changed</b> is set, it
+ *  means that this descriptor is dirty. */
+STATIC void
+service_desc_schedule_upload(hs_service_descriptor_t *desc,
+                             time_t now,
+                             int descriptor_changed)
+
+{
+  desc->next_upload_time = now;
+
+  /* If the descriptor changed, clean up the old HSDirs list. We want to
+   * re-upload no matter what. */
+  if (descriptor_changed) {
+    service_desc_clear_previous_hsdirs(desc);
+  }
+}
+
+/* Pick missing intro points for this descriptor if needed. */
+static void
+update_service_descriptor_intro_points(hs_service_t *service,
+                          hs_service_descriptor_t *desc, time_t now)
+{
+  unsigned int num_intro_points;
+
+  tor_assert(service);
+  tor_assert(desc);
+  tor_assert(desc->desc);
+
+  num_intro_points = digest256map_size(desc->intro_points.map);
+
+  /* Pick any missing introduction point(s). */
+  if (num_intro_points < service->config.num_intro_points) {
+    unsigned int num_new_intro_points = pick_needed_intro_points(service,
+                                                                 desc);
+    if (num_new_intro_points != 0) {
+      log_info(LD_REND, "Service %s just picked %u intro points and wanted "
+                        "%u for %s descriptor. It currently has %d intro "
+                        "points. Launching ESTABLISH_INTRO circuit shortly.",
+               safe_str_client(service->onion_address),
+               num_new_intro_points,
+               service->config.num_intro_points - num_intro_points,
+               (desc == service->desc_current) ? "current" : "next",
+               num_intro_points);
+      /* We'll build those introduction point into the descriptor once we have
+       * confirmation that the circuits are opened and ready. However,
+       * indicate that this descriptor should be uploaded from now on. */
+      service_desc_schedule_upload(desc, now, 1);
+    }
+    /* Were we able to pick all the intro points we needed? If not, we'll
+     * flag the descriptor that it's missing intro points because it
+     * couldn't pick enough which will trigger a descriptor upload. */
+    if ((num_new_intro_points + num_intro_points) <
+        service->config.num_intro_points) {
+      desc->missing_intro_points = 1;
+    }
+  }
+}
+
+/* Update descriptor intro points for each service if needed. We do this as
+ * part of the periodic event because we need to establish intro point circuits
+ * before we publish descriptors. */
+STATIC void
+update_all_descriptors_intro_points(time_t now)
+{
+  FOR_EACH_SERVICE_BEGIN(service) {
+    /* We'll try to update each descriptor that is if certain conditions apply
+     * in order for the descriptor to be updated. */
+    FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+      update_service_descriptor_intro_points(service, desc, now);
+    } FOR_EACH_DESCRIPTOR_END;
+  } FOR_EACH_SERVICE_END;
+}
+
+/* Return true iff the given intro point has expired that is it has been used
+ * for too long or we've reached our max seen INTRODUCE2 cell. */
+STATIC int
+intro_point_should_expire(const hs_service_intro_point_t *ip,
+                          time_t now)
+{
+  tor_assert(ip);
+
+  if (ip->introduce2_count >= ip->introduce2_max) {
+    goto expired;
+  }
+
+  if (ip->time_to_expire <= now) {
+    goto expired;
+  }
+
+  /* Not expiring. */
+  return 0;
+ expired:
+  return 1;
+}
+
+/* Go over the given set of intro points for each service and remove any
+ * invalid ones. The conditions for removal are:
+ *
+ *    - The node doesn't exists anymore (not in consensus)
+ *                          OR
+ *    - The intro point maximum circuit retry count has been reached and no
+ *      circuit can be found associated with it.
+ *                          OR
+ *    - The intro point has expired and we should pick a new one.
+ *
+ * If an intro point is removed, the circuit (if any) is immediately close.
+ * If a circuit can't be found, the intro point is kept if it hasn't reached
+ * its maximum circuit retry value and thus should be retried.  */
+static void
+cleanup_intro_points(hs_service_t *service, time_t now)
+{
+  /* List of intro points to close. We can't mark the intro circuits for close
+   * in the modify loop because doing so calls
+   * hs_service_intro_circ_has_closed() which does a digest256map_get() on the
+   * intro points map (that we are iterating over). This can't be done in a
+   * single iteration after a MAP_DEL_CURRENT, the object will still be
+   * returned leading to a use-after-free. So, we close the circuits and free
+   * the intro points after the loop if any. */
+  smartlist_t *ips_to_free = smartlist_new();
+
+  tor_assert(service);
+
+  /* For both descriptors, cleanup the intro points. */
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    /* Go over the current intro points we have, make sure they are still
+     * valid and remove any of them that aren't. */
+    DIGEST256MAP_FOREACH_MODIFY(desc->intro_points.map, key,
+                                hs_service_intro_point_t *, ip) {
+      const node_t *node = get_node_from_intro_point(ip);
+      int has_expired = intro_point_should_expire(ip, now);
+
+      /* We cleanup an intro point if it has expired or if we do not know the
+       * node_t anymore (removed from our latest consensus) or if we've
+       * reached the maximum number of retry with a non existing circuit. */
+      if (has_expired || node == NULL ||
+          ip->circuit_retries > MAX_INTRO_POINT_CIRCUIT_RETRIES) {
+        log_info(LD_REND, "Intro point %s%s (retried: %u times). "
+                          "Removing it.",
+                 describe_intro_point(ip),
+                 has_expired ? " has expired" :
+                    (node == NULL) ?  " fell off the consensus" : "",
+                 ip->circuit_retries);
+
+        /* We've retried too many times, remember it as a failed intro point
+         * so we don't pick it up again for INTRO_CIRC_RETRY_PERIOD sec. */
+        if (ip->circuit_retries > MAX_INTRO_POINT_CIRCUIT_RETRIES) {
+          remember_failing_intro_point(ip, desc, approx_time());
+        }
+
+        /* Remove intro point from descriptor map and add it to the list of
+         * ips to free for which we'll also try to close the intro circuit. */
+        MAP_DEL_CURRENT(key);
+        smartlist_add(ips_to_free, ip);
+      }
+    } DIGEST256MAP_FOREACH_END;
+  } FOR_EACH_DESCRIPTOR_END;
+
+  /* Go over the intro points to free and close their circuit if any. */
+  SMARTLIST_FOREACH_BEGIN(ips_to_free, hs_service_intro_point_t *, ip) {
+    /* See if we need to close the intro point circuit as well */
+
+    /* XXX: Legacy code does NOT close circuits like this: it keeps the circuit
+     * open until a new descriptor is uploaded and then closed all expiring
+     * intro point circuit. Here, we close immediately and because we just
+     * discarded the intro point, a new one will be selected, a new descriptor
+     * created and uploaded. There is no difference to an attacker between the
+     * timing of a new consensus and intro point rotation (possibly?). */
+    origin_circuit_t *ocirc = hs_circ_service_get_intro_circ(ip);
+    if (ocirc && !TO_CIRCUIT(ocirc)->marked_for_close) {
+      circuit_mark_for_close(TO_CIRCUIT(ocirc), END_CIRC_REASON_FINISHED);
+    }
+
+    /* Cleanup the intro point */
+    service_intro_point_free(ip);
+  } SMARTLIST_FOREACH_END(ip);
+
+  smartlist_free(ips_to_free);
+}
+
+/* Set the next rotation time of the descriptors for the given service for the
+ * time now. */
+static void
+set_rotation_time(hs_service_t *service)
+{
+  tor_assert(service);
+
+  service->state.next_rotation_time =
+    sr_state_get_start_time_of_current_protocol_run() +
+    sr_state_get_protocol_run_duration();
+
+  {
+    char fmt_time[ISO_TIME_LEN + 1];
+    format_local_iso_time(fmt_time, service->state.next_rotation_time);
+    log_info(LD_REND, "Next descriptor rotation time set to %s for %s",
+             fmt_time, safe_str_client(service->onion_address));
+  }
+}
+
+/* Return true iff the service should rotate its descriptor. The time now is
+ * only used to fetch the live consensus and if none can be found, this
+ * returns false. */
+static unsigned int
+should_rotate_descriptors(hs_service_t *service, time_t now)
+{
+  const networkstatus_t *ns;
+
+  tor_assert(service);
+
+  ns = networkstatus_get_live_consensus(now);
+  if (ns == NULL) {
+    goto no_rotation;
+  }
+
+  if (ns->valid_after >= service->state.next_rotation_time) {
+    /* In theory, we should never get here with no descriptors. We can never
+     * have a NULL current descriptor except when tor starts up. The next
+     * descriptor can be NULL after a rotation but we build a new one right
+     * after.
+     *
+     * So, when tor starts, the next rotation time is set to the start of the
+     * next SRV period using the consensus valid after time so it should
+     * always be set to a future time value. This means that we should never
+     * reach this point at bootup that is this check safeguards tor in never
+     * allowing a rotation if the valid after time is smaller than the next
+     * rotation time.
+     *
+     * This is all good in theory but we've had a NULL descriptor issue here
+     * so this is why we BUG() on both with extra logging to try to understand
+     * how this can possibly happens. We'll simply ignore and tor should
+     * recover from this by skipping rotation and building the missing
+     * descriptors just after this. */
+    if (BUG(service->desc_current == NULL || service->desc_next == NULL)) {
+      log_warn(LD_BUG, "Service descriptor is NULL (%p/%p). Next rotation "
+                       "time is %ld (now: %ld). Valid after time from "
+                       "consensus is %ld",
+               service->desc_current, service->desc_next,
+               (long)service->state.next_rotation_time,
+               (long)now,
+               (long)ns->valid_after);
+      goto no_rotation;
+    }
+    goto rotation;
+  }
+
+ no_rotation:
+  return 0;
+ rotation:
+  return 1;
+}
+
+/* Rotate the service descriptors of the given service. The current descriptor
+ * will be freed, the next one put in as the current and finally the next
+ * descriptor pointer is NULLified. */
+static void
+rotate_service_descriptors(hs_service_t *service)
+{
+  if (service->desc_current) {
+    /* Close all IP circuits for the descriptor. */
+    close_intro_circuits(&service->desc_current->intro_points);
+    /* We don't need this one anymore, we won't serve any clients coming with
+     * this service descriptor. */
+    service_descriptor_free(service->desc_current);
+  }
+  /* The next one become the current one and emptying the next will trigger
+   * a descriptor creation for it. */
+  service->desc_current = service->desc_next;
+  service->desc_next = NULL;
+
+  /* We've just rotated, set the next time for the rotation. */
+  set_rotation_time(service);
+}
+
+/* Rotate descriptors for each service if needed. A non existing current
+ * descriptor will trigger a descriptor build for the next time period. */
+STATIC void
+rotate_all_descriptors(time_t now)
+{
+  /* XXX We rotate all our service descriptors at once. In the future it might
+   *     be wise, to rotate service descriptors independently to hide that all
+   *     those descriptors are on the same tor instance */
+
+  FOR_EACH_SERVICE_BEGIN(service) {
+
+    /* Note for a service booting up: Both descriptors are NULL in that case
+     * so this function might return true if we are in the timeframe for a
+     * rotation leading to basically swapping two NULL pointers which is
+     * harmless. However, the side effect is that triggering a rotation will
+     * update the service state and avoid doing anymore rotations after the
+     * two descriptors have been built. */
+    if (!should_rotate_descriptors(service, now)) {
+      continue;
+    }
+
+    log_info(LD_REND, "Time to rotate our descriptors (%p / %p) for %s",
+             service->desc_current, service->desc_next,
+             safe_str_client(service->onion_address));
+
+    rotate_service_descriptors(service);
+  } FOR_EACH_SERVICE_END;
+}
+
+/* Scheduled event run from the main loop. Make sure all our services are up
+ * to date and ready for the other scheduled events. This includes looking at
+ * the introduction points status and descriptor rotation time. */
+STATIC void
+run_housekeeping_event(time_t now)
+{
+  /* Note that nothing here opens circuit(s) nor uploads descriptor(s). We are
+   * simply moving things around or removing unneeded elements. */
+
+  FOR_EACH_SERVICE_BEGIN(service) {
+
+    /* If the service is starting off, set the rotation time. We can't do that
+     * at configure time because the get_options() needs to be set for setting
+     * that time that uses the voting interval. */
+    if (service->state.next_rotation_time == 0) {
+      /* Set the next rotation time of the descriptors. If it's Oct 25th
+       * 23:47:00, the next rotation time is when the next SRV is computed
+       * which is at Oct 26th 00:00:00 that is in 13 minutes. */
+      set_rotation_time(service);
+    }
+
+    /* Cleanup invalid intro points from the service descriptor. */
+    cleanup_intro_points(service, now);
+
+    /* Remove expired failing intro point from the descriptor failed list. We
+     * reset them at each INTRO_CIRC_RETRY_PERIOD. */
+    remove_expired_failing_intro(service, now);
+
+    /* At this point, the service is now ready to go through the scheduled
+     * events guaranteeing a valid state. Intro points might be missing from
+     * the descriptors after the cleanup but the update/build process will
+     * make sure we pick those missing ones. */
+  } FOR_EACH_SERVICE_END;
+}
+
+/* Scheduled event run from the main loop. Make sure all descriptors are up to
+ * date. Once this returns, each service descriptor needs to be considered for
+ * new introduction circuits and then for upload. */
+static void
+run_build_descriptor_event(time_t now)
+{
+  /* For v2 services, this step happens in the upload event. */
+
+  /* Run v3+ events. */
+  /* We start by rotating the descriptors only if needed. */
+  rotate_all_descriptors(now);
+
+  /* Then, we'll try to build  new descriptors that we might need. The
+   * condition is that the next descriptor is non existing because it has
+   * been rotated or we just started up. */
+  build_all_descriptors(now);
+
+  /* Finally, we'll check if we should update the descriptors' intro
+   * points. Missing introduction points will be picked in this function which
+   * is useful for newly built descriptors. */
+  update_all_descriptors_intro_points(now);
+}
+
+/* For the given service, launch any intro point circuits that could be
+ * needed. This considers every descriptor of the service. */
+static void
+launch_intro_point_circuits(hs_service_t *service)
+{
+  tor_assert(service);
+
+  /* For both descriptors, try to launch any missing introduction point
+   * circuits using the current map. */
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    /* Keep a ref on if we need a direct connection. We use this often. */
+    bool direct_conn = service->config.is_single_onion;
+
+    DIGEST256MAP_FOREACH_MODIFY(desc->intro_points.map, key,
+                                hs_service_intro_point_t *, ip) {
+      extend_info_t *ei;
+
+      /* Skip the intro point that already has an existing circuit
+       * (established or not). */
+      if (hs_circ_service_get_intro_circ(ip)) {
+        continue;
+      }
+      ei = get_extend_info_from_intro_point(ip, direct_conn);
+
+      /* If we can't connect directly to the intro point, get an extend_info
+       * for a multi-hop path instead. */
+      if (ei == NULL && direct_conn) {
+        direct_conn = false;
+        ei = get_extend_info_from_intro_point(ip, 0);
+      }
+
+      if (ei == NULL) {
+        /* This is possible if we can get a node_t but not the extend info out
+         * of it. In this case, we remove the intro point and a new one will
+         * be picked at the next main loop callback. */
+        MAP_DEL_CURRENT(key);
+        service_intro_point_free(ip);
+        continue;
+      }
+
+      /* Launch a circuit to the intro point. */
+      ip->circuit_retries++;
+      if (hs_circ_launch_intro_point(service, ip, ei, direct_conn) < 0) {
+        log_info(LD_REND, "Unable to launch intro circuit to node %s "
+                          "for service %s.",
+                 safe_str_client(extend_info_describe(ei)),
+                 safe_str_client(service->onion_address));
+        /* Intro point will be retried if possible after this. */
+      }
+      extend_info_free(ei);
+    } DIGEST256MAP_FOREACH_END;
+  } FOR_EACH_DESCRIPTOR_END;
+}
+
+/* Don't try to build more than this many circuits before giving up for a
+ * while. Dynamically calculated based on the configured number of intro
+ * points for the given service and how many descriptor exists. The default
+ * use case of 3 introduction points and two descriptors will allow 28
+ * circuits for a retry period (((3 + 2) + (3 * 3)) * 2). */
+static unsigned int
+get_max_intro_circ_per_period(const hs_service_t *service)
+{
+  unsigned int count = 0;
+  unsigned int multiplier = 0;
+  unsigned int num_wanted_ip;
+
+  tor_assert(service);
+  tor_assert(service->config.num_intro_points <=
+             HS_CONFIG_V3_MAX_INTRO_POINTS);
+
+/* For a testing network, allow to do it for the maximum amount so circuit
+ * creation and rotation and so on can actually be tested without limit. */
+#define MAX_INTRO_POINT_CIRCUIT_RETRIES_TESTING -1
+  if (get_options()->TestingTorNetwork) {
+    return MAX_INTRO_POINT_CIRCUIT_RETRIES_TESTING;
+  }
+
+  num_wanted_ip = service->config.num_intro_points;
+
+  /* The calculation is as follow. We have a number of intro points that we
+   * want configured as a torrc option (num_intro_points). We then add an
+   * extra value so we can launch multiple circuits at once and pick the
+   * quickest ones. For instance, we want 3 intros, we add 2 extra so we'll
+   * pick 5 intros and launch 5 circuits. */
+  count += (num_wanted_ip + get_intro_point_num_extra());
+
+  /* Then we add the number of retries that is possible to do for each intro
+   * point. If we want 3 intros, we'll allow 3 times the number of possible
+   * retry. */
+  count += (num_wanted_ip * MAX_INTRO_POINT_CIRCUIT_RETRIES);
+
+  /* Then, we multiply by a factor of 2 if we have both descriptor or 0 if we
+   * have none.  */
+  multiplier += (service->desc_current) ? 1 : 0;
+  multiplier += (service->desc_next) ? 1 : 0;
+
+  return (count * multiplier);
+}
+
+/* For the given service, return 1 if the service is allowed to launch more
+ * introduction circuits else 0 if the maximum has been reached for the retry
+ * period of INTRO_CIRC_RETRY_PERIOD. */
+STATIC int
+can_service_launch_intro_circuit(hs_service_t *service, time_t now)
+{
+  tor_assert(service);
+
+  /* Consider the intro circuit retry period of the service. */
+  if (now > (service->state.intro_circ_retry_started_time +
+             INTRO_CIRC_RETRY_PERIOD)) {
+    service->state.intro_circ_retry_started_time = now;
+    service->state.num_intro_circ_launched = 0;
+    goto allow;
+  }
+  /* Check if we can still launch more circuits in this period. */
+  if (service->state.num_intro_circ_launched <=
+      get_max_intro_circ_per_period(service)) {
+    goto allow;
+  }
+
+  /* Rate limit log that we've reached our circuit creation limit. */
+  {
+    char *msg;
+    time_t elapsed_time = now - service->state.intro_circ_retry_started_time;
+    static ratelim_t rlimit = RATELIM_INIT(INTRO_CIRC_RETRY_PERIOD);
+    if ((msg = rate_limit_log(&rlimit, now))) {
+      log_info(LD_REND, "Hidden service %s exceeded its circuit launch limit "
+                        "of %u per %d seconds. It launched %u circuits in "
+                        "the last %ld seconds. Will retry in %ld seconds.",
+               safe_str_client(service->onion_address),
+               get_max_intro_circ_per_period(service),
+               INTRO_CIRC_RETRY_PERIOD,
+               service->state.num_intro_circ_launched,
+               (long int) elapsed_time,
+               (long int) (INTRO_CIRC_RETRY_PERIOD - elapsed_time));
+      tor_free(msg);
+    }
+  }
+
+  /* Not allow. */
+  return 0;
+ allow:
+  return 1;
+}
+
+/* Scheduled event run from the main loop. Make sure we have all the circuits
+ * we need for each service. */
+static void
+run_build_circuit_event(time_t now)
+{
+  /* Make sure we can actually have enough information or able to build
+   * internal circuits as required by services. */
+  if (router_have_consensus_path() == CONSENSUS_PATH_UNKNOWN ||
+      !have_completed_a_circuit()) {
+    return;
+  }
+
+  /* Run v2 check. */
+  if (rend_num_services() > 0) {
+    rend_consider_services_intro_points(now);
+  }
+
+  /* Run v3+ check. */
+  FOR_EACH_SERVICE_BEGIN(service) {
+    /* For introduction circuit, we need to make sure we don't stress too much
+     * circuit creation so make sure this service is respecting that limit. */
+    if (can_service_launch_intro_circuit(service, now)) {
+      /* Launch intro point circuits if needed. */
+      launch_intro_point_circuits(service);
+      /* Once the circuits have opened, we'll make sure to update the
+       * descriptor intro point list and cleanup any extraneous. */
+    }
+  } FOR_EACH_SERVICE_END;
+}
+
+/* Encode and sign the service descriptor desc and upload it to the given
+ * hidden service directory.  This does nothing if PublishHidServDescriptors
+ * is false. */
+static void
+upload_descriptor_to_hsdir(const hs_service_t *service,
+                           hs_service_descriptor_t *desc, const node_t *hsdir)
+{
+  char *encoded_desc = NULL;
+
+  tor_assert(service);
+  tor_assert(desc);
+  tor_assert(hsdir);
+
+  /* Let's avoid doing that if tor is configured to not publish. */
+  if (!get_options()->PublishHidServDescriptors) {
+    log_info(LD_REND, "Service %s not publishing descriptor. "
+                      "PublishHidServDescriptors is set to 1.",
+             safe_str_client(service->onion_address));
+    goto end;
+  }
+
+  /* First of all, we'll encode the descriptor. This should NEVER fail but
+   * just in case, let's make sure we have an actual usable descriptor. */
+  if (BUG(service_encode_descriptor(service, desc, &desc->signing_kp,
+                                    &encoded_desc) < 0)) {
+    goto end;
+  }
+
+  /* Time to upload the descriptor to the directory. */
+  hs_service_upload_desc_to_dir(encoded_desc, service->config.version,
+                                &service->keys.identity_pk,
+                                &desc->blinded_kp.pubkey, hsdir->rs);
+
+  /* Add this node to previous_hsdirs list */
+  service_desc_note_upload(desc, hsdir);
+
+  /* Logging so we know where it was sent. */
+  {
+    int is_next_desc = (service->desc_next == desc);
+    const uint8_t *idx = (is_next_desc) ? hsdir->hsdir_index.store_second:
+                                          hsdir->hsdir_index.store_first;
+    char *blinded_pubkey_log_str =
+      tor_strdup(hex_str((char*)&desc->blinded_kp.pubkey.pubkey, 32));
+    log_info(LD_REND, "Service %s %s descriptor of revision %" PRIu64
+                      " initiated upload request to %s with index %s (%s)",
+             safe_str_client(service->onion_address),
+             (is_next_desc) ? "next" : "current",
+             desc->desc->plaintext_data.revision_counter,
+             safe_str_client(node_describe(hsdir)),
+             safe_str_client(hex_str((const char *) idx, 32)),
+             safe_str_client(blinded_pubkey_log_str));
+    tor_free(blinded_pubkey_log_str);
+
+    /* Fire a UPLOAD control port event. */
+    hs_control_desc_event_upload(service->onion_address, hsdir->identity,
+                                 &desc->blinded_kp.pubkey, idx);
+  }
+
+ end:
+  tor_free(encoded_desc);
+  return;
+}
+
+/** Set the revision counter in <b>hs_desc</b>. We do this by encrypting a
+ *  timestamp using an OPE scheme and using the ciphertext as our revision
+ *  counter.
+ *
+ *  If <b>is_current</b> is true, then this is the current HS descriptor,
+ *  otherwise it's the next one. */
+static void
+set_descriptor_revision_counter(hs_service_descriptor_t *hs_desc, time_t now,
+                                bool is_current)
+{
+  uint64_t rev_counter = 0;
+
+  /* Get current time */
+  time_t srv_start = 0;
+
+  /* As our revision counter plaintext value, we use the seconds since the
+   * start of the SR protocol run that is relevant to this descriptor. This is
+   * guaranteed to be a positive value since we need the SRV to start making a
+   * descriptor (so that we know where to upload it).
+   *
+   * Depending on whether we are building the current or the next descriptor,
+   * services use a different SRV value. See [SERVICEUPLOAD] in
+   * rend-spec-v3.txt:
+   *
+   * In particular, for the current descriptor (aka first descriptor), Tor
+   * always uses the previous SRV for uploading the descriptor, and hence we
+   * should use the start time of the previous protocol run here.
+   *
+   * Whereas for the next descriptor (aka second descriptor), Tor always uses
+   * the current SRV for uploading the descriptor.  and hence we use the start
+   * time of the current protocol run.
+   */
+  if (is_current) {
+    srv_start = sr_state_get_start_time_of_previous_protocol_run();
+  } else {
+    srv_start = sr_state_get_start_time_of_current_protocol_run();
+  }
+
+  log_info(LD_REND, "Setting rev counter for TP #%u: "
+           "SRV started at %d, now %d (%s)",
+           (unsigned) hs_desc->time_period_num, (int)srv_start,
+           (int)now, is_current ? "current" : "next");
+
+  tor_assert_nonfatal(now >= srv_start);
+
+  /* Compute seconds elapsed since the start of the time period. That's the
+   * number of seconds of how long this blinded key has been active. */
+  time_t seconds_since_start_of_srv = now - srv_start;
+
+  /* Increment by one so that we are definitely sure this is strictly
+   * positive and not zero. */
+  seconds_since_start_of_srv++;
+
+  /* Check for too big inputs. */
+  if (BUG(seconds_since_start_of_srv > OPE_INPUT_MAX)) {
+    seconds_since_start_of_srv = OPE_INPUT_MAX;
+  }
+
+  /* Now we compute the final revision counter value by encrypting the
+     plaintext using our OPE cipher: */
+  tor_assert(hs_desc->ope_cipher);
+  rev_counter = crypto_ope_encrypt(hs_desc->ope_cipher,
+                                   (int) seconds_since_start_of_srv);
+
+  /* The OPE module returns CRYPTO_OPE_ERROR in case of errors. */
+  tor_assert_nonfatal(rev_counter < CRYPTO_OPE_ERROR);
+
+  log_info(LD_REND, "Encrypted revision counter %d to %ld",
+           (int) seconds_since_start_of_srv, (long int) rev_counter);
+
+  hs_desc->desc->plaintext_data.revision_counter = rev_counter;
+}
+
+/* Encode and sign the service descriptor desc and upload it to the
+ * responsible hidden service directories. If for_next_period is true, the set
+ * of directories are selected using the next hsdir_index. This does nothing
+ * if PublishHidServDescriptors is false. */
+STATIC void
+upload_descriptor_to_all(const hs_service_t *service,
+                         hs_service_descriptor_t *desc)
+{
+  smartlist_t *responsible_dirs = NULL;
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  /* We'll first cancel any directory request that are ongoing for this
+   * descriptor. It is possible that we can trigger multiple uploads in a
+   * short time frame which can lead to a race where the second upload arrives
+   * before the first one leading to a 400 malformed descriptor response from
+   * the directory. Closing all pending requests avoids that. */
+  close_directory_connections(service, desc);
+
+  /* Get our list of responsible HSDir. */
+  responsible_dirs = smartlist_new();
+  /* The parameter 0 means that we aren't a client so tell the function to use
+   * the spread store consensus paremeter. */
+  hs_get_responsible_hsdirs(&desc->blinded_kp.pubkey, desc->time_period_num,
+                            service->desc_next == desc, 0, responsible_dirs);
+
+  /** Clear list of previous hsdirs since we are about to upload to a new
+   *  list. Let's keep it up to date. */
+  service_desc_clear_previous_hsdirs(desc);
+
+  /* For each responsible HSDir we have, initiate an upload command. */
+  SMARTLIST_FOREACH_BEGIN(responsible_dirs, const routerstatus_t *,
+                          hsdir_rs) {
+    const node_t *hsdir_node = node_get_by_id(hsdir_rs->identity_digest);
+    /* Getting responsible hsdir implies that the node_t object exists for the
+     * routerstatus_t found in the consensus else we have a problem. */
+    tor_assert(hsdir_node);
+    /* Upload this descriptor to the chosen directory. */
+    upload_descriptor_to_hsdir(service, desc, hsdir_node);
+  } SMARTLIST_FOREACH_END(hsdir_rs);
+
+  /* Set the next upload time for this descriptor. Even if we are configured
+   * to not upload, we still want to follow the right cycle of life for this
+   * descriptor. */
+  desc->next_upload_time =
+    (time(NULL) + crypto_rand_int_range(HS_SERVICE_NEXT_UPLOAD_TIME_MIN,
+                                        HS_SERVICE_NEXT_UPLOAD_TIME_MAX));
+  {
+    char fmt_next_time[ISO_TIME_LEN+1];
+    format_local_iso_time(fmt_next_time, desc->next_upload_time);
+    log_debug(LD_REND, "Service %s set to upload a descriptor at %s",
+              safe_str_client(service->onion_address), fmt_next_time);
+  }
+
+  smartlist_free(responsible_dirs);
+  return;
+}
+
+/** The set of HSDirs have changed: check if the change affects our descriptor
+ *  HSDir placement, and if it does, reupload the desc. */
+STATIC int
+service_desc_hsdirs_changed(const hs_service_t *service,
+                            const hs_service_descriptor_t *desc)
+{
+  int should_reupload = 0;
+  smartlist_t *responsible_dirs = smartlist_new();
+
+  /* No desc upload has happened yet: it will happen eventually */
+  if (!desc->previous_hsdirs || !smartlist_len(desc->previous_hsdirs)) {
+    goto done;
+  }
+
+  /* Get list of responsible hsdirs */
+  hs_get_responsible_hsdirs(&desc->blinded_kp.pubkey, desc->time_period_num,
+                            service->desc_next == desc, 0, responsible_dirs);
+
+  /* Check if any new hsdirs have been added to the responsible hsdirs set:
+   * Iterate over the list of new hsdirs, and reupload if any of them is not
+   * present in the list of previous hsdirs.
+   */
+  SMARTLIST_FOREACH_BEGIN(responsible_dirs, const routerstatus_t *, hsdir_rs) {
+    char b64_digest[BASE64_DIGEST_LEN+1] = {0};
+    digest_to_base64(b64_digest, hsdir_rs->identity_digest);
+
+    if (!smartlist_contains_string(desc->previous_hsdirs, b64_digest)) {
+      should_reupload = 1;
+      break;
+    }
+  } SMARTLIST_FOREACH_END(hsdir_rs);
+
+ done:
+  smartlist_free(responsible_dirs);
+
+  return should_reupload;
+}
+
+/* Return 1 if the given descriptor from the given service can be uploaded
+ * else return 0 if it can not. */
+static int
+should_service_upload_descriptor(const hs_service_t *service,
+                              const hs_service_descriptor_t *desc, time_t now)
+{
+  unsigned int num_intro_points;
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  /* If this descriptors has missing intro points that is that it couldn't get
+   * them all when it was time to pick them, it means that we should upload
+   * instead of waiting an arbitrary amount of time breaking the service.
+   * Else, if we have no missing intro points, we use the value taken from the
+   * service configuration. */
+  if (desc->missing_intro_points) {
+    num_intro_points = digest256map_size(desc->intro_points.map);
+  } else {
+    num_intro_points = service->config.num_intro_points;
+  }
+
+  /* This means we tried to pick intro points but couldn't get any so do not
+   * upload descriptor in this case. We need at least one for the service to
+   * be reachable. */
+  if (desc->missing_intro_points && num_intro_points == 0) {
+    goto cannot;
+  }
+
+  /* Check if all our introduction circuit have been established for all the
+   * intro points we have selected. */
+  if (count_desc_circuit_established(desc) != num_intro_points) {
+    goto cannot;
+  }
+
+  /* Is it the right time to upload? */
+  if (desc->next_upload_time > now) {
+    goto cannot;
+  }
+
+  /* Don't upload desc if we don't have a live consensus */
+  if (!networkstatus_get_live_consensus(now)) {
+    goto cannot;
+  }
+
+  /* Do we know enough router descriptors to have adequate vision of the HSDir
+     hash ring? */
+  if (!router_have_minimum_dir_info()) {
+    goto cannot;
+  }
+
+  /* Can upload! */
+  return 1;
+ cannot:
+  return 0;
+}
+
+/* Refresh the given service descriptor meaning this will update every mutable
+ * field that needs to be updated before we upload.
+ *
+ * This should ONLY be called before uploading a descriptor. It assumes that
+ * the descriptor has been built (desc->desc) and that all intro point
+ * circuits have been established.  */
+static void
+refresh_service_descriptor(const hs_service_t *service,
+                           hs_service_descriptor_t *desc, time_t now)
+{
+  /* There are few fields that we consider "mutable" in the descriptor meaning
+   * we need to update them regurlarly over the lifetime fo the descriptor.
+   * The rest are set once and should not be modified.
+   *
+   *  - Signing key certificate.
+   *  - Revision counter.
+   *  - Introduction points which includes many thing. See
+   *    hs_desc_intro_point_t. and the setup_desc_intro_point() function.
+   */
+
+  /* Create the signing key certificate. */
+  build_desc_signing_key_cert(desc, now);
+
+  /* Build the intro points descriptor section. The refresh step is just
+   * before we upload so all circuits have been properly established. */
+  build_desc_intro_points(service, desc, now);
+
+  /* Set the desc revision counter right before uploading */
+  set_descriptor_revision_counter(desc, now, service->desc_current == desc);
+}
+
+/* Scheduled event run from the main loop. Try to upload the descriptor for
+ * each service. */
+STATIC void
+run_upload_descriptor_event(time_t now)
+{
+  /* v2 services use the same function for descriptor creation and upload so
+   * we do everything here because the intro circuits were checked before. */
+  if (rend_num_services() > 0) {
+    rend_consider_services_upload(now);
+    rend_consider_descriptor_republication();
+  }
+
+  /* Run v3+ check. */
+  FOR_EACH_SERVICE_BEGIN(service) {
+    FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+      /* If we were asked to re-examine the hash ring, and it changed, then
+         schedule an upload */
+      if (consider_republishing_hs_descriptors &&
+          service_desc_hsdirs_changed(service, desc)) {
+        service_desc_schedule_upload(desc, now, 0);
+      }
+
+      /* Can this descriptor be uploaded? */
+      if (!should_service_upload_descriptor(service, desc, now)) {
+        continue;
+      }
+
+      log_info(LD_REND, "Initiating upload for hidden service %s descriptor "
+                        "for service %s with %u/%u introduction points%s.",
+               (desc == service->desc_current) ? "current" : "next",
+               safe_str_client(service->onion_address),
+               digest256map_size(desc->intro_points.map),
+               service->config.num_intro_points,
+               (desc->missing_intro_points) ? " (couldn't pick more)" : "");
+
+      /* We are about to upload so we need to do one last step which is to
+       * update the service's descriptor mutable fields in order to upload a
+       * coherent descriptor. */
+      refresh_service_descriptor(service, desc, now);
+
+      /* Proceed with the upload, the descriptor is ready to be encoded. */
+      upload_descriptor_to_all(service, desc);
+    } FOR_EACH_DESCRIPTOR_END;
+  } FOR_EACH_SERVICE_END;
+
+  /* We are done considering whether to republish rend descriptors */
+  consider_republishing_hs_descriptors = 0;
+}
+
+/* Called when the introduction point circuit is done building and ready to be
+ * used. */
+static void
+service_intro_circ_has_opened(origin_circuit_t *circ)
+{
+  hs_service_t *service = NULL;
+  hs_service_intro_point_t *ip = NULL;
+  hs_service_descriptor_t *desc = NULL;
+
+  tor_assert(circ);
+
+  /* Let's do some basic sanity checking of the circ state */
+  if (BUG(!circ->cpath)) {
+    return;
+  }
+  if (BUG(TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO)) {
+    return;
+  }
+  if (BUG(!circ->hs_ident)) {
+    return;
+  }
+
+  /* Get the corresponding service and intro point. */
+  get_objects_from_ident(circ->hs_ident, &service, &ip, &desc);
+
+  if (service == NULL) {
+    log_warn(LD_REND, "Unknown service identity key %s on the introduction "
+                      "circuit %u. Can't find onion service.",
+             safe_str_client(ed25519_fmt(&circ->hs_ident->identity_pk)),
+             TO_CIRCUIT(circ)->n_circ_id);
+    goto err;
+  }
+  if (ip == NULL) {
+    log_warn(LD_REND, "Unknown introduction point auth key on circuit %u "
+                      "for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto err;
+  }
+  /* We can't have an IP object without a descriptor. */
+  tor_assert(desc);
+
+  if (hs_circ_service_intro_has_opened(service, ip, desc, circ)) {
+    /* Getting here means that the circuit has been re-purposed because we
+     * have enough intro circuit opened. Remove the IP from the service. */
+    service_intro_point_remove(service, ip);
+    service_intro_point_free(ip);
+  }
+
+  goto done;
+
+ err:
+  /* Close circuit, we can't use it. */
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_NOSUCHSERVICE);
+ done:
+  return;
+}
+
+/* Called when a rendezvous circuit is done building and ready to be used. */
+static void
+service_rendezvous_circ_has_opened(origin_circuit_t *circ)
+{
+  hs_service_t *service = NULL;
+
+  tor_assert(circ);
+  tor_assert(circ->cpath);
+  /* Getting here means this is a v3 rendezvous circuit. */
+  tor_assert(circ->hs_ident);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
+
+  /* Declare the circuit dirty to avoid reuse, and for path-bias. We set the
+   * timestamp regardless of its content because that circuit could have been
+   * cannibalized so in any cases, we are about to use that circuit more. */
+  TO_CIRCUIT(circ)->timestamp_dirty = time(NULL);
+  pathbias_count_use_attempt(circ);
+
+  /* Get the corresponding service and intro point. */
+  get_objects_from_ident(circ->hs_ident, &service, NULL, NULL);
+  if (service == NULL) {
+    log_warn(LD_REND, "Unknown service identity key %s on the rendezvous "
+                      "circuit %u with cookie %s. Can't find onion service.",
+             safe_str_client(ed25519_fmt(&circ->hs_ident->identity_pk)),
+             TO_CIRCUIT(circ)->n_circ_id,
+             hex_str((const char *) circ->hs_ident->rendezvous_cookie,
+                     REND_COOKIE_LEN));
+    goto err;
+  }
+
+  /* If the cell can't be sent, the circuit will be closed within this
+   * function. */
+  hs_circ_service_rp_has_opened(service, circ);
+  goto done;
+
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_NOSUCHSERVICE);
+ done:
+  return;
+}
+
+/* We've been expecting an INTRO_ESTABLISHED cell on this circuit and it just
+ * arrived. Handle the INTRO_ESTABLISHED cell arriving on the given
+ * introduction circuit. Return 0 on success else a negative value. */
+static int
+service_handle_intro_established(origin_circuit_t *circ,
+                                 const uint8_t *payload,
+                                 size_t payload_len)
+{
+  hs_service_t *service = NULL;
+  hs_service_intro_point_t *ip = NULL;
+
+  tor_assert(circ);
+  tor_assert(payload);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO);
+
+  /* We need the service and intro point for this cell. */
+  get_objects_from_ident(circ->hs_ident, &service, &ip, NULL);
+
+  /* Get service object from the circuit identifier. */
+  if (service == NULL) {
+    log_warn(LD_REND, "Unknown service identity key %s on the introduction "
+                      "circuit %u. Can't find onion service.",
+             safe_str_client(ed25519_fmt(&circ->hs_ident->identity_pk)),
+             TO_CIRCUIT(circ)->n_circ_id);
+    goto err;
+  }
+  if (ip == NULL) {
+    /* We don't recognize the key. */
+    log_warn(LD_REND, "Introduction circuit established without an intro "
+                      "point object on circuit %u for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto err;
+  }
+
+  /* Try to parse the payload into a cell making sure we do actually have a
+   * valid cell. On success, the ip object and circuit purpose is updated to
+   * reflect the fact that the introduction circuit is established. */
+  if (hs_circ_handle_intro_established(service, ip, circ, payload,
+                                       payload_len) < 0) {
+    goto err;
+  }
+
+  /* Flag that we have an established circuit for this intro point. This value
+   * is what indicates the upload scheduled event if we are ready to build the
+   * intro point into the descriptor and upload. */
+  ip->circuit_established = 1;
+
+  log_info(LD_REND, "Successfully received an INTRO_ESTABLISHED cell "
+                    "on circuit %u for service %s",
+           TO_CIRCUIT(circ)->n_circ_id,
+           safe_str_client(service->onion_address));
+  return 0;
+
+ err:
+  return -1;
+}
+
+/* We just received an INTRODUCE2 cell on the established introduction circuit
+ * circ. Handle the cell and return 0 on success else a negative value. */
+static int
+service_handle_introduce2(origin_circuit_t *circ, const uint8_t *payload,
+                          size_t payload_len)
+{
+  hs_service_t *service = NULL;
+  hs_service_intro_point_t *ip = NULL;
+  hs_service_descriptor_t *desc = NULL;
+
+  tor_assert(circ);
+  tor_assert(payload);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_INTRO);
+
+  /* We'll need every object associated with this circuit. */
+  get_objects_from_ident(circ->hs_ident, &service, &ip, &desc);
+
+  /* Get service object from the circuit identifier. */
+  if (service == NULL) {
+    log_warn(LD_BUG, "Unknown service identity key %s when handling "
+                     "an INTRODUCE2 cell on circuit %u",
+             safe_str_client(ed25519_fmt(&circ->hs_ident->identity_pk)),
+             TO_CIRCUIT(circ)->n_circ_id);
+    goto err;
+  }
+  if (ip == NULL) {
+    /* We don't recognize the key. */
+    log_warn(LD_BUG, "Unknown introduction auth key when handling "
+                     "an INTRODUCE2 cell on circuit %u for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto err;
+  }
+  /* If we have an IP object, we MUST have a descriptor object. */
+  tor_assert(desc);
+
+  /* The following will parse, decode and launch the rendezvous point circuit.
+   * Both current and legacy cells are handled. */
+  if (hs_circ_handle_introduce2(service, circ, ip, desc->desc->subcredential,
+                                payload, payload_len) < 0) {
+    goto err;
+  }
+
+  return 0;
+ err:
+  return -1;
+}
+
+/* Add to list every filename used by service. This is used by the sandbox
+ * subsystem. */
+static void
+service_add_fnames_to_list(const hs_service_t *service, smartlist_t *list)
+{
+  const char *s_dir;
+  char fname[128] = {0};
+
+  tor_assert(service);
+  tor_assert(list);
+
+  /* Ease our life. */
+  s_dir = service->config.directory_path;
+  /* The hostname file. */
+  smartlist_add(list, hs_path_from_filename(s_dir, fname_hostname));
+  /* The key files splitted in two. */
+  tor_snprintf(fname, sizeof(fname), "%s_secret_key", fname_keyfile_prefix);
+  smartlist_add(list, hs_path_from_filename(s_dir, fname));
+  tor_snprintf(fname, sizeof(fname), "%s_public_key", fname_keyfile_prefix);
+  smartlist_add(list, hs_path_from_filename(s_dir, fname));
+}
+
+/* Return true iff the given service identity key is present on disk. */
+static int
+service_key_on_disk(const char *directory_path)
+{
+  int ret = 0;
+  char *fname;
+  ed25519_keypair_t *kp = NULL;
+
+  tor_assert(directory_path);
+
+  /* Build the v3 key path name and then try to load it. */
+  fname = hs_path_from_filename(directory_path, fname_keyfile_prefix);
+  kp = ed_key_init_from_file(fname, INIT_ED_KEY_SPLIT,
+                             LOG_DEBUG, NULL, 0, 0, 0, NULL, NULL);
+  if (kp) {
+    ret = 1;
+  }
+
+  ed25519_keypair_free(kp);
+  tor_free(fname);
+
+  return ret;
+}
+
+/* This is a proxy function before actually calling hs_desc_encode_descriptor
+ * because we need some preprocessing here */
+static int
+service_encode_descriptor(const hs_service_t *service,
+                          const hs_service_descriptor_t *desc,
+                          const ed25519_keypair_t *signing_kp,
+                          char **encoded_out)
+{
+  int ret;
+  const uint8_t *descriptor_cookie = NULL;
+
+  tor_assert(service);
+  tor_assert(desc);
+  tor_assert(encoded_out);
+
+  /* If the client authorization is enabled, send the descriptor cookie to
+   * hs_desc_encode_descriptor. Otherwise, send NULL */
+  if (service->config.is_client_auth_enabled) {
+    descriptor_cookie = desc->descriptor_cookie;
+  }
+
+  ret = hs_desc_encode_descriptor(desc->desc, signing_kp,
+                                  descriptor_cookie, encoded_out);
+
+  return ret;
+}
+
+/* ========== */
+/* Public API */
+/* ========== */
+
+/* This is called everytime the service map (v2 or v3) changes that is if an
+ * element is added or removed. */
+void
+hs_service_map_has_changed(void)
+{
+  /* If we now have services where previously we had not, we need to enable
+   * the HS service main loop event. If we changed to having no services, we
+   * need to disable the event. */
+  rescan_periodic_events(get_options());
+}
+
+/* Upload an encoded descriptor in encoded_desc of the given version. This
+ * descriptor is for the service identity_pk and blinded_pk used to setup the
+ * directory connection identifier. It is uploaded to the directory hsdir_rs
+ * routerstatus_t object.
+ *
+ * NOTE: This function does NOT check for PublishHidServDescriptors because it
+ * is only used by the control port command HSPOST outside of this subsystem.
+ * Inside this code, upload_descriptor_to_hsdir() should be used. */
+void
+hs_service_upload_desc_to_dir(const char *encoded_desc,
+                              const uint8_t version,
+                              const ed25519_public_key_t *identity_pk,
+                              const ed25519_public_key_t *blinded_pk,
+                              const routerstatus_t *hsdir_rs)
+{
+  char version_str[4] = {0};
+  directory_request_t *dir_req;
+  hs_ident_dir_conn_t ident;
+
+  tor_assert(encoded_desc);
+  tor_assert(identity_pk);
+  tor_assert(blinded_pk);
+  tor_assert(hsdir_rs);
+
+  /* Setup the connection identifier. */
+  memset(&ident, 0, sizeof(ident));
+  hs_ident_dir_conn_init(identity_pk, blinded_pk, &ident);
+
+  /* This is our resource when uploading which is used to construct the URL
+   * with the version number: "/tor/hs/<version>/publish". */
+  tor_snprintf(version_str, sizeof(version_str), "%u", version);
+
+  /* Build the directory request for this HSDir. */
+  dir_req = directory_request_new(DIR_PURPOSE_UPLOAD_HSDESC);
+  directory_request_set_routerstatus(dir_req, hsdir_rs);
+  directory_request_set_indirection(dir_req, DIRIND_ANONYMOUS);
+  directory_request_set_resource(dir_req, version_str);
+  directory_request_set_payload(dir_req, encoded_desc,
+                                strlen(encoded_desc));
+  /* The ident object is copied over the directory connection object once
+   * the directory request is initiated. */
+  directory_request_upload_set_hs_ident(dir_req, &ident);
+
+  /* Initiate the directory request to the hsdir.*/
+  directory_initiate_request(dir_req);
+  directory_request_free(dir_req);
+}
+
+/* Add the ephemeral service using the secret key sk and ports. Both max
+ * streams parameter will be set in the newly created service.
+ *
+ * Ownership of sk and ports is passed to this routine.  Regardless of
+ * success/failure, callers should not touch these values after calling this
+ * routine, and may assume that correct cleanup has been done on failure.
+ *
+ * Return an appropriate hs_service_add_ephemeral_status_t. */
+hs_service_add_ephemeral_status_t
+hs_service_add_ephemeral(ed25519_secret_key_t *sk, smartlist_t *ports,
+                         int max_streams_per_rdv_circuit,
+                         int max_streams_close_circuit, char **address_out)
+{
+  hs_service_add_ephemeral_status_t ret;
+  hs_service_t *service = NULL;
+
+  tor_assert(sk);
+  tor_assert(ports);
+  tor_assert(address_out);
+
+  service = hs_service_new(get_options());
+
+  /* Setup the service configuration with specifics. A default service is
+   * HS_VERSION_TWO so explicitly set it. */
+  service->config.version = HS_VERSION_THREE;
+  service->config.max_streams_per_rdv_circuit = max_streams_per_rdv_circuit;
+  service->config.max_streams_close_circuit = !!max_streams_close_circuit;
+  service->config.is_ephemeral = 1;
+  smartlist_free(service->config.ports);
+  service->config.ports = ports;
+
+  /* Handle the keys. */
+  memcpy(&service->keys.identity_sk, sk, sizeof(service->keys.identity_sk));
+  if (ed25519_public_key_generate(&service->keys.identity_pk,
+                                  &service->keys.identity_sk) < 0) {
+    log_warn(LD_CONFIG, "Unable to generate ed25519 public key"
+                        "for v3 service.");
+    ret = RSAE_BADPRIVKEY;
+    goto err;
+  }
+
+  /* Make sure we have at least one port. */
+  if (smartlist_len(service->config.ports) == 0) {
+    log_warn(LD_CONFIG, "At least one VIRTPORT/TARGET must be specified "
+                        "for v3 service.");
+    ret = RSAE_BADVIRTPORT;
+    goto err;
+  }
+
+  /* Build the onion address for logging purposes but also the control port
+   * uses it for the HS_DESC event. */
+  hs_build_address(&service->keys.identity_pk,
+                   (uint8_t) service->config.version,
+                   service->onion_address);
+
+  /* The only way the registration can fail is if the service public key
+   * already exists. */
+  if (BUG(register_service(hs_service_map, service) < 0)) {
+    log_warn(LD_CONFIG, "Onion Service private key collides with an "
+                        "existing v3 service.");
+    ret = RSAE_ADDREXISTS;
+    goto err;
+  }
+
+  log_info(LD_CONFIG, "Added ephemeral v3 onion service: %s",
+           safe_str_client(service->onion_address));
+
+  *address_out = tor_strdup(service->onion_address);
+  ret = RSAE_OKAY;
+  goto end;
+
+ err:
+  hs_service_free(service);
+
+ end:
+  memwipe(sk, 0, sizeof(ed25519_secret_key_t));
+  tor_free(sk);
+  return ret;
+}
+
+/* For the given onion address, delete the ephemeral service. Return 0 on
+ * success else -1 on error. */
+int
+hs_service_del_ephemeral(const char *address)
+{
+  uint8_t version;
+  ed25519_public_key_t pk;
+  hs_service_t *service = NULL;
+
+  tor_assert(address);
+
+  if (hs_parse_address(address, &pk, NULL, &version) < 0) {
+    log_warn(LD_CONFIG, "Requested malformed v3 onion address for removal.");
+    goto err;
+  }
+
+  if (version != HS_VERSION_THREE) {
+    log_warn(LD_CONFIG, "Requested version of onion address for removal "
+                        "is not supported.");
+    goto err;
+  }
+
+  service = find_service(hs_service_map, &pk);
+  if (service == NULL) {
+    log_warn(LD_CONFIG, "Requested non-existent v3 hidden service for "
+                        "removal.");
+    goto err;
+  }
+
+  if (!service->config.is_ephemeral) {
+    log_warn(LD_CONFIG, "Requested non-ephemeral v3 hidden service for "
+                        "removal.");
+    goto err;
+  }
+
+  /* Close introduction circuits, remove from map and finally free. Notice
+   * that the rendezvous circuits aren't closed in order for any existing
+   * connections to finish. We let the application terminate them. */
+  close_service_intro_circuits(service);
+  remove_service(hs_service_map, service);
+  hs_service_free(service);
+
+  log_info(LD_CONFIG, "Removed ephemeral v3 hidden service: %s",
+           safe_str_client(address));
+  return 0;
+
+ err:
+  return -1;
+}
+
+/* Using the ed25519 public key pk, find a service for that key and return the
+ * current encoded descriptor as a newly allocated string or NULL if not
+ * found. This is used by the control port subsystem. */
+char *
+hs_service_lookup_current_desc(const ed25519_public_key_t *pk)
+{
+  const hs_service_t *service;
+
+  tor_assert(pk);
+
+  service = find_service(hs_service_map, pk);
+  if (service && service->desc_current) {
+    char *encoded_desc = NULL;
+    /* No matter what is the result (which should never be a failure), return
+     * the encoded variable, if success it will contain the right thing else
+     * it will be NULL. */
+    service_encode_descriptor(service,
+                              service->desc_current,
+                              &service->desc_current->signing_kp,
+                              &encoded_desc);
+    return encoded_desc;
+  }
+
+  return NULL;
+}
+
+/* Return the number of service we have configured and usable. */
+unsigned int
+hs_service_get_num_services(void)
+{
+  if (hs_service_map == NULL) {
+    return 0;
+  }
+  return HT_SIZE(hs_service_map);
+}
+
+/* Called once an introduction circuit is closed. If the circuit doesn't have
+ * a v3 identifier, it is ignored. */
+void
+hs_service_intro_circ_has_closed(origin_circuit_t *circ)
+{
+  hs_service_t *service = NULL;
+  hs_service_intro_point_t *ip = NULL;
+  hs_service_descriptor_t *desc = NULL;
+
+  tor_assert(circ);
+
+  if (circ->hs_ident == NULL) {
+    /* This is not a v3 circuit, ignore. */
+    goto end;
+  }
+
+  get_objects_from_ident(circ->hs_ident, &service, &ip, &desc);
+  if (service == NULL) {
+    /* This is possible if the circuits are closed and the service is
+     * immediately deleted. */
+    log_info(LD_REND, "Unable to find any hidden service associated "
+                      "identity key %s on intro circuit %u.",
+             ed25519_fmt(&circ->hs_ident->identity_pk),
+             TO_CIRCUIT(circ)->n_circ_id);
+    goto end;
+  }
+  if (ip == NULL) {
+    /* The introduction point object has already been removed probably by our
+     * cleanup process so ignore. */
+    goto end;
+  }
+  /* Can't have an intro point object without a descriptor. */
+  tor_assert(desc);
+
+  /* Circuit disappeared so make sure the intro point is updated. By
+   * keeping the object in the descriptor, we'll be able to retry. */
+  ip->circuit_established = 0;
+
+ end:
+  return;
+}
+
+/* Given conn, a rendezvous edge connection acting as an exit stream, look up
+ * the hidden service for the circuit circ, and look up the port and address
+ * based on the connection port. Assign the actual connection address.
+ *
+ * Return 0 on success. Return -1 on failure and the caller should NOT close
+ * the circuit. Return -2 on failure and the caller MUST close the circuit for
+ * security reasons. */
+int
+hs_service_set_conn_addr_port(const origin_circuit_t *circ,
+                              edge_connection_t *conn)
+{
+  hs_service_t *service = NULL;
+
+  tor_assert(circ);
+  tor_assert(conn);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_REND_JOINED);
+  tor_assert(circ->hs_ident);
+
+  get_objects_from_ident(circ->hs_ident, &service, NULL, NULL);
+
+  if (service == NULL) {
+    log_warn(LD_REND, "Unable to find any hidden service associated "
+                      "identity key %s on rendezvous circuit %u.",
+             ed25519_fmt(&circ->hs_ident->identity_pk),
+             TO_CIRCUIT(circ)->n_circ_id);
+    /* We want the caller to close the circuit because it's not a valid
+     * service so no danger. Attempting to bruteforce the entire key space by
+     * opening circuits to learn which service is being hosted here is
+     * impractical. */
+    goto err_close;
+  }
+
+  /* Enforce the streams-per-circuit limit, and refuse to provide a mapping if
+   * this circuit will exceed the limit. */
+  if (service->config.max_streams_per_rdv_circuit > 0 &&
+      (circ->hs_ident->num_rdv_streams >=
+       service->config.max_streams_per_rdv_circuit)) {
+#define MAX_STREAM_WARN_INTERVAL 600
+    static struct ratelim_t stream_ratelim =
+      RATELIM_INIT(MAX_STREAM_WARN_INTERVAL);
+    log_fn_ratelim(&stream_ratelim, LOG_WARN, LD_REND,
+                   "Maximum streams per circuit limit reached on "
+                   "rendezvous circuit %u for service %s. Circuit has "
+                   "%" PRIu64 " out of %" PRIu64 " streams. %s.",
+                   TO_CIRCUIT(circ)->n_circ_id,
+                   service->onion_address,
+                   circ->hs_ident->num_rdv_streams,
+                   service->config.max_streams_per_rdv_circuit,
+                   service->config.max_streams_close_circuit ?
+                    "Closing circuit" : "Ignoring open stream request");
+    if (service->config.max_streams_close_circuit) {
+      /* Service explicitly configured to close immediately. */
+      goto err_close;
+    }
+    /* Exceeding the limit makes tor silently ignore the stream creation
+     * request and keep the circuit open. */
+    goto err_no_close;
+  }
+
+  /* Find a virtual port of that service mathcing the one in the connection if
+   * successful, set the address in the connection. */
+  if (hs_set_conn_addr_port(service->config.ports, conn) < 0) {
+    log_info(LD_REND, "No virtual port mapping exists for port %d for "
+                      "hidden service %s.",
+             TO_CONN(conn)->port, service->onion_address);
+    if (service->config.allow_unknown_ports) {
+      /* Service explicitly allow connection to unknown ports so close right
+       * away because we do not care about port mapping. */
+      goto err_close;
+    }
+    /* If the service didn't explicitly allow it, we do NOT close the circuit
+     * here to raise the bar in terms of performance for port mapping. */
+    goto err_no_close;
+  }
+
+  /* Success. */
+  return 0;
+ err_close:
+  /* Indicate the caller that the circuit should be closed. */
+  return -2;
+ err_no_close:
+  /* Indicate the caller to NOT close the circuit. */
+  return -1;
+}
+
+/** Does the service with identity pubkey <b>pk</b> export the circuit IDs of
+ *  its clients?  */
+hs_circuit_id_protocol_t
+hs_service_exports_circuit_id(const ed25519_public_key_t *pk)
+{
+  hs_service_t *service = find_service(hs_service_map, pk);
+  if (!service) {
+    return HS_CIRCUIT_ID_PROTOCOL_NONE;
+  }
+
+  return service->config.circuit_id_protocol;
+}
+
+/* Add to file_list every filename used by a configured hidden service, and to
+ * dir_list every directory path used by a configured hidden service. This is
+ * used by the sandbox subsystem to whitelist those. */
+void
+hs_service_lists_fnames_for_sandbox(smartlist_t *file_list,
+                                    smartlist_t *dir_list)
+{
+  tor_assert(file_list);
+  tor_assert(dir_list);
+
+  /* Add files and dirs for legacy services. */
+  rend_services_add_filenames_to_lists(file_list, dir_list);
+
+  /* Add files and dirs for v3+. */
+  FOR_EACH_SERVICE_BEGIN(service) {
+    /* Skip ephemeral service, they don't touch the disk. */
+    if (service->config.is_ephemeral) {
+      continue;
+    }
+    service_add_fnames_to_list(service, file_list);
+    smartlist_add_strdup(dir_list, service->config.directory_path);
+    smartlist_add_strdup(dir_list, dname_client_pubkeys);
+  } FOR_EACH_DESCRIPTOR_END;
+}
+
+/* Called when our internal view of the directory has changed. We might have
+ * received a new batch of descriptors which might affect the shape of the
+ * HSDir hash ring. Signal that we should reexamine the hash ring and
+ * re-upload our HS descriptors if needed. */
+void
+hs_service_dir_info_changed(void)
+{
+  if (hs_service_get_num_services() > 0) {
+    /* New directory information usually goes every consensus so rate limit
+     * every 30 minutes to not be too conservative. */
+    static struct ratelim_t dir_info_changed_ratelim = RATELIM_INIT(30 * 60);
+    log_fn_ratelim(&dir_info_changed_ratelim, LOG_INFO, LD_REND,
+                   "New dirinfo arrived: consider reuploading descriptor");
+    consider_republishing_hs_descriptors = 1;
+  }
+}
+
+/* Called when we get an INTRODUCE2 cell on the circ. Respond to the cell and
+ * launch a circuit to the rendezvous point. */
+int
+hs_service_receive_introduce2(origin_circuit_t *circ, const uint8_t *payload,
+                              size_t payload_len)
+{
+  int ret = -1;
+
+  tor_assert(circ);
+  tor_assert(payload);
+
+  /* Do some initial validation and logging before we parse the cell */
+  if (TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_S_INTRO) {
+    log_warn(LD_PROTOCOL, "Received an INTRODUCE2 cell on a "
+                          "non introduction circuit of purpose %d",
+             TO_CIRCUIT(circ)->purpose);
+    goto done;
+  }
+
+  if (circ->hs_ident) {
+    ret = service_handle_introduce2(circ, payload, payload_len);
+    hs_stats_note_introduce2_cell(1);
+  } else {
+    ret = rend_service_receive_introduction(circ, payload, payload_len);
+    hs_stats_note_introduce2_cell(0);
+  }
+
+ done:
+  return ret;
+}
+
+/* Called when we get an INTRO_ESTABLISHED cell. Mark the circuit as an
+ * established introduction point. Return 0 on success else a negative value
+ * and the circuit is closed. */
+int
+hs_service_receive_intro_established(origin_circuit_t *circ,
+                                     const uint8_t *payload,
+                                     size_t payload_len)
+{
+  int ret = -1;
+
+  tor_assert(circ);
+  tor_assert(payload);
+
+  if (TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) {
+    log_warn(LD_PROTOCOL, "Received an INTRO_ESTABLISHED cell on a "
+                          "non introduction circuit of purpose %d",
+             TO_CIRCUIT(circ)->purpose);
+    goto err;
+  }
+
+  /* Handle both version. v2 uses rend_data and v3 uses the hs circuit
+   * identifier hs_ident. Can't be both. */
+  if (circ->hs_ident) {
+    ret = service_handle_intro_established(circ, payload, payload_len);
+  } else {
+    ret = rend_service_intro_established(circ, payload, payload_len);
+  }
+
+  if (ret < 0) {
+    goto err;
+  }
+  return 0;
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+  return -1;
+}
+
+/* Called when any kind of hidden service circuit is done building thus
+ * opened. This is the entry point from the circuit subsystem. */
+void
+hs_service_circuit_has_opened(origin_circuit_t *circ)
+{
+  tor_assert(circ);
+
+  /* Handle both version. v2 uses rend_data and v3 uses the hs circuit
+   * identifier hs_ident. Can't be both. */
+  switch (TO_CIRCUIT(circ)->purpose) {
+  case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO:
+    if (circ->hs_ident) {
+      service_intro_circ_has_opened(circ);
+    } else {
+      rend_service_intro_has_opened(circ);
+    }
+    break;
+  case CIRCUIT_PURPOSE_S_CONNECT_REND:
+    if (circ->hs_ident) {
+      service_rendezvous_circ_has_opened(circ);
+    } else {
+      rend_service_rendezvous_has_opened(circ);
+    }
+    break;
+  default:
+    tor_assert(0);
+  }
+}
+
+/* Return the service version by looking at the key in the service directory.
+ * If the key is not found or unrecognized, -1 is returned. Else, the service
+ * version is returned. */
+int
+hs_service_get_version_from_key(const hs_service_t *service)
+{
+  int version = -1; /* Unknown version. */
+  const char *directory_path;
+
+  tor_assert(service);
+
+  /* We'll try to load the key for version 3. If not found, we'll try version
+   * 2 and if not found, we'll send back an unknown version (-1). */
+  directory_path = service->config.directory_path;
+
+  /* Version 3 check. */
+  if (service_key_on_disk(directory_path)) {
+    version = HS_VERSION_THREE;
+    goto end;
+  }
+  /* Version 2 check. */
+  if (rend_service_key_on_disk(directory_path)) {
+    version = HS_VERSION_TWO;
+    goto end;
+  }
+
+ end:
+  return version;
+}
+
+/* Load and/or generate keys for all onion services including the client
+ * authorization if any. Return 0 on success, -1 on failure. */
+int
+hs_service_load_all_keys(void)
+{
+  /* Load v2 service keys if we have v2. */
+  if (rend_num_services() != 0) {
+    if (rend_service_load_all_keys(NULL) < 0) {
+      goto err;
+    }
+  }
+
+  /* Load or/and generate them for v3+. */
+  SMARTLIST_FOREACH_BEGIN(hs_service_staging_list, hs_service_t *, service) {
+    /* Ignore ephemeral service, they already have their keys set. */
+    if (service->config.is_ephemeral) {
+      continue;
+    }
+    log_info(LD_REND, "Loading v3 onion service keys from %s",
+             service_escaped_dir(service));
+    if (load_service_keys(service) < 0) {
+      goto err;
+    }
+  } SMARTLIST_FOREACH_END(service);
+
+  /* Final step, the staging list contains service in a quiescent state that
+   * is ready to be used. Register them to the global map. Once this is over,
+   * the staging list will be cleaned up. */
+  register_all_services();
+
+  /* All keys have been loaded successfully. */
+  return 0;
+ err:
+  return -1;
+}
+
+/* Put all service object in the given service list. After this, the caller
+ * looses ownership of every elements in the list and responsible to free the
+ * list pointer. */
+void
+hs_service_stage_services(const smartlist_t *service_list)
+{
+  tor_assert(service_list);
+  /* This list is freed at registration time but this function can be called
+   * multiple time. */
+  if (hs_service_staging_list == NULL) {
+    hs_service_staging_list = smartlist_new();
+  }
+  /* Add all service object to our staging list. Caller is responsible for
+   * freeing the service_list. */
+  smartlist_add_all(hs_service_staging_list, service_list);
+}
+
+/* Allocate and initilize a service object. The service configuration will
+ * contain the default values. Return the newly allocated object pointer. This
+ * function can't fail. */
+hs_service_t *
+hs_service_new(const or_options_t *options)
+{
+  hs_service_t *service = tor_malloc_zero(sizeof(hs_service_t));
+  /* Set default configuration value. */
+  set_service_default_config(&service->config, options);
+  /* Set the default service version. */
+  service->config.version = HS_SERVICE_DEFAULT_VERSION;
+  /* Allocate the CLIENT_PK replay cache in service state. */
+  service->state.replay_cache_rend_cookie =
+    replaycache_new(REND_REPLAY_TIME_INTERVAL, REND_REPLAY_TIME_INTERVAL);
+
+  return service;
+}
+
+/* Free the given <b>service</b> object and all its content. This function
+ * also takes care of wiping service keys from memory. It is safe to pass a
+ * NULL pointer. */
+void
+hs_service_free_(hs_service_t *service)
+{
+  if (service == NULL) {
+    return;
+  }
+
+  /* Free descriptors. Go over both descriptor with this loop. */
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    service_descriptor_free(desc);
+  } FOR_EACH_DESCRIPTOR_END;
+
+  /* Free service configuration. */
+  service_clear_config(&service->config);
+
+  /* Free replay cache from state. */
+  if (service->state.replay_cache_rend_cookie) {
+    replaycache_free(service->state.replay_cache_rend_cookie);
+  }
+
+  /* Wipe service keys. */
+  memwipe(&service->keys.identity_sk, 0, sizeof(service->keys.identity_sk));
+
+  tor_free(service);
+}
+
+/* Periodic callback. Entry point from the main loop to the HS service
+ * subsystem. This is call every second. This is skipped if tor can't build a
+ * circuit or the network is disabled. */
+void
+hs_service_run_scheduled_events(time_t now)
+{
+  /* First thing we'll do here is to make sure our services are in a
+   * quiescent state for the scheduled events. */
+  run_housekeeping_event(now);
+
+  /* Order matters here. We first make sure the descriptor object for each
+   * service contains the latest data. Once done, we check if we need to open
+   * new introduction circuit. Finally, we try to upload the descriptor for
+   * each service. */
+
+  /* Make sure descriptors are up to date. */
+  run_build_descriptor_event(now);
+  /* Make sure services have enough circuits. */
+  run_build_circuit_event(now);
+  /* Upload the descriptors if needed/possible. */
+  run_upload_descriptor_event(now);
+}
+
+/* Initialize the service HS subsystem. */
+void
+hs_service_init(void)
+{
+  /* Should never be called twice. */
+  tor_assert(!hs_service_map);
+  tor_assert(!hs_service_staging_list);
+
+  /* v2 specific. */
+  rend_service_init();
+
+  hs_service_map = tor_malloc_zero(sizeof(struct hs_service_ht));
+  HT_INIT(hs_service_ht, hs_service_map);
+
+  hs_service_staging_list = smartlist_new();
+}
+
+/* Release all global storage of the hidden service subsystem. */
+void
+hs_service_free_all(void)
+{
+  rend_service_free_all();
+  service_free_all();
+}
+
+#ifdef TOR_UNIT_TESTS
+
+/* Return the global service map size. Only used by unit test. */
+STATIC unsigned int
+get_hs_service_map_size(void)
+{
+  return HT_SIZE(hs_service_map);
+}
+
+/* Return the staging list size. Only used by unit test. */
+STATIC int
+get_hs_service_staging_list_size(void)
+{
+  return smartlist_len(hs_service_staging_list);
+}
+
+STATIC hs_service_ht *
+get_hs_service_map(void)
+{
+  return hs_service_map;
+}
+
+STATIC hs_service_t *
+get_first_service(void)
+{
+  hs_service_t **obj = HT_START(hs_service_ht, hs_service_map);
+  if (obj == NULL) {
+    return NULL;
+  }
+  return *obj;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/feature/hs/hs_service.h b/src/feature/hs/hs_service.h
new file mode 100644
index 0000000000..5f43233ea1
--- /dev/null
+++ b/src/feature/hs/hs_service.h
@@ -0,0 +1,444 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_service.h
+ * \brief Header file containing service data for the HS subsytem.
+ **/
+
+#ifndef TOR_HS_SERVICE_H
+#define TOR_HS_SERVICE_H
+
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "feature/hs_common/replaycache.h"
+
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_descriptor.h"
+#include "feature/hs/hs_ident.h"
+#include "feature/hs/hs_intropoint.h"
+
+/* Trunnel */
+#include "trunnel/hs/cell_establish_intro.h"
+
+/* When loading and configuring a service, this is the default version it will
+ * be configured for as it is possible that no HiddenServiceVersion is
+ * present. */
+#define HS_SERVICE_DEFAULT_VERSION HS_VERSION_THREE
+
+/* As described in the specification, service publishes their next descriptor
+ * at a random time between those two values (in seconds). */
+#define HS_SERVICE_NEXT_UPLOAD_TIME_MIN (60 * 60)
+#define HS_SERVICE_NEXT_UPLOAD_TIME_MAX (120 * 60)
+
+/* Service side introduction point. */
+typedef struct hs_service_intro_point_t {
+  /* Top level intropoint "shared" data between client/service. */
+  hs_intropoint_t base;
+
+  /* Onion key of the introduction point used to extend to it for the ntor
+   * handshake. */
+  curve25519_public_key_t onion_key;
+
+  /* Authentication keypair used to create the authentication certificate
+   * which is published in the descriptor. */
+  ed25519_keypair_t auth_key_kp;
+
+  /* Encryption keypair for the "ntor" type. */
+  curve25519_keypair_t enc_key_kp;
+
+  /* Legacy key if that intro point doesn't support v3. This should be used if
+   * the base object legacy flag is set. */
+  crypto_pk_t *legacy_key;
+  /* Legacy key SHA1 public key digest. This should be used only if the base
+   * object legacy flag is set. */
+  uint8_t legacy_key_digest[DIGEST_LEN];
+
+  /* Amount of INTRODUCE2 cell accepted from this intro point. */
+  uint64_t introduce2_count;
+
+  /* Maximum number of INTRODUCE2 cell this intro point should accept. */
+  uint64_t introduce2_max;
+
+  /* The time at which this intro point should expire and stop being used. */
+  time_t time_to_expire;
+
+  /* The amount of circuit creation we've made to this intro point. This is
+   * incremented every time we do a circuit relaunch on this intro point which
+   * is triggered when the circuit dies but the node is still in the
+   * consensus. After MAX_INTRO_POINT_CIRCUIT_RETRIES, we give up on it. */
+  uint32_t circuit_retries;
+
+  /* Set if this intro point has an established circuit. */
+  unsigned int circuit_established : 1;
+
+  /* Replay cache recording the encrypted part of an INTRODUCE2 cell that the
+   * circuit associated with this intro point has received. This is used to
+   * prevent replay attacks. */
+  replaycache_t *replay_cache;
+} hs_service_intro_point_t;
+
+/* Object handling introduction points of a service. */
+typedef struct hs_service_intropoints_t {
+  /* The time at which we've started our retry period to build circuits. We
+   * don't want to stress circuit creation so we can only retry for a certain
+   * time and then after we stop and wait. */
+  time_t retry_period_started;
+
+  /* Number of circuit we've launched during a single retry period. */
+  unsigned int num_circuits_launched;
+
+  /* Contains the current hs_service_intro_point_t objects indexed by
+   * authentication public key. */
+  digest256map_t *map;
+
+  /* Contains node's identity key digest that were introduction point for this
+   * descriptor but were retried to many times. We keep those so we avoid
+   * re-picking them over and over for a circuit retry period.
+   * XXX: Once we have #22173, change this to only use ed25519 identity. */
+  digestmap_t *failed_id;
+} hs_service_intropoints_t;
+
+/* Representation of a service descriptor.
+ *
+ * Some elements of the descriptor are mutable whereas others are immutable:
+
+ * Immutable elements are initialized once when the descriptor is built (when
+ * service descriptors gets rotated). This means that these elements are
+ * initialized once and then they don't change for the lifetime of the
+ * descriptor. See build_service_descriptor().
+ *
+ * Mutable elements are initialized when we build the descriptor but they are
+ * also altered during the lifetime of the descriptor. They could be
+ * _refreshed_ everytime we upload the descriptor (which happens multiple times
+ * over the lifetime of the descriptor), or through periodic events. We do this
+ * for elements like the descriptor revision counter and various
+ * certificates. See refresh_service_descriptor() and
+ * update_service_descriptor_intro_points().
+ */
+typedef struct hs_service_descriptor_t {
+  /* Immutable: Client authorization ephemeral keypair. */
+  curve25519_keypair_t auth_ephemeral_kp;
+
+  /* Immutable: Descriptor cookie used to encrypt the descriptor, when the
+   * client authorization is enabled */
+  uint8_t descriptor_cookie[HS_DESC_DESCRIPTOR_COOKIE_LEN];
+
+  /* Immutable: Descriptor signing keypair. */
+  ed25519_keypair_t signing_kp;
+
+  /* Immutable: Blinded keypair derived from the master identity public key. */
+  ed25519_keypair_t blinded_kp;
+
+  /* Immutable: The time period number this descriptor has been created for. */
+  uint64_t time_period_num;
+
+  /** Immutable: The OPE cipher for encrypting revision counters for this
+   *  descriptor.  Tied to the descriptor blinded key. */
+  struct crypto_ope_t *ope_cipher;
+
+  /* Mutable: Decoded descriptor. This object is used for encoding when the
+   * service publishes the descriptor. */
+  hs_descriptor_t *desc;
+
+  /* Mutable: When is the next time when we should upload the descriptor. */
+  time_t next_upload_time;
+
+  /* Mutable: Introduction points assign to this descriptor which contains
+   * hs_service_intropoints_t object indexed by authentication key (the RSA key
+   * if the node is legacy). */
+  hs_service_intropoints_t intro_points;
+
+  /* Mutable: True iff we have missing intro points for this descriptor because
+   * we couldn't pick any nodes. */
+  unsigned int missing_intro_points : 1;
+
+  /** Mutable: List of the responsible HSDirs (their b64ed identity digest)
+   *  last time we uploaded this descriptor. If the set of responsible HSDirs
+   *  is different from this list, this means we received new dirinfo and we
+   *  need to reupload our descriptor. */
+  smartlist_t *previous_hsdirs;
+} hs_service_descriptor_t;
+
+/* Service key material. */
+typedef struct hs_service_keys_t {
+  /* Master identify public key. */
+  ed25519_public_key_t identity_pk;
+  /* Master identity private key. */
+  ed25519_secret_key_t identity_sk;
+  /* True iff the key is kept offline which means the identity_sk MUST not be
+   * used in that case. */
+  unsigned int is_identify_key_offline : 1;
+} hs_service_keys_t;
+
+/** Service side configuration of client authorization. */
+typedef struct hs_service_authorized_client_t {
+  /* The client auth public key used to encrypt the descriptor cookie. */
+  curve25519_public_key_t client_pk;
+} hs_service_authorized_client_t;
+
+/** Which protocol to use for exporting HS client circuit ID. */
+typedef enum {
+  /** Don't expose the circuit id. */
+  HS_CIRCUIT_ID_PROTOCOL_NONE,
+
+  /** Use the HAProxy proxy protocol. */
+  HS_CIRCUIT_ID_PROTOCOL_HAPROXY
+} hs_circuit_id_protocol_t;
+
+/* Service configuration. The following are set from the torrc options either
+ * set by the configuration file or by the control port. Nothing else should
+ * change those values. */
+typedef struct hs_service_config_t {
+  /* Protocol version of the service. Specified by HiddenServiceVersion
+   * option. */
+  uint32_t version;
+
+  /* Have we explicitly set HiddenServiceVersion? */
+  unsigned int hs_version_explicitly_set : 1;
+
+  /* List of rend_service_port_config_t */
+  smartlist_t *ports;
+
+  /* Path on the filesystem where the service persistent data is stored. NULL
+   * if the service is ephemeral. Specified by HiddenServiceDir option. */
+  char *directory_path;
+
+  /* The maximum number of simultaneous streams per rendezvous circuit that
+   * are allowed to be created. No limit if 0. Specified by
+   * HiddenServiceMaxStreams option. */
+  uint64_t max_streams_per_rdv_circuit;
+
+  /* If true, we close circuits that exceed the max_streams_per_rdv_circuit
+   * limit. Specified by HiddenServiceMaxStreamsCloseCircuit option. */
+  unsigned int max_streams_close_circuit : 1;
+
+  /* How many introduction points this service has. Specified by
+   * HiddenServiceNumIntroductionPoints option. */
+  unsigned int num_intro_points;
+
+  /* True iff the client auth is enabled. */
+  unsigned int is_client_auth_enabled : 1;
+
+  /* List of hs_service_authorized_client_t's of clients that may access this
+   * service. Specified by HiddenServiceAuthorizeClient option. */
+  smartlist_t *clients;
+
+  /* True iff we allow request made on unknown ports. Specified by
+   * HiddenServiceAllowUnknownPorts option. */
+  unsigned int allow_unknown_ports : 1;
+
+  /* If true, this service is a Single Onion Service. Specified by
+   * HiddenServiceSingleHopMode and HiddenServiceNonAnonymousMode options. */
+  unsigned int is_single_onion : 1;
+
+  /* If true, allow group read permissions on the directory_path. Specified by
+   * HiddenServiceDirGroupReadable option. */
+  unsigned int dir_group_readable : 1;
+
+  /* Is this service ephemeral? */
+  unsigned int is_ephemeral : 1;
+
+  /* Does this service export the circuit ID of its clients? */
+  hs_circuit_id_protocol_t circuit_id_protocol;
+} hs_service_config_t;
+
+/* Service state. */
+typedef struct hs_service_state_t {
+  /* The time at which we've started our retry period to build circuits. We
+   * don't want to stress circuit creation so we can only retry for a certain
+   * time and then after we stop and wait. */
+  time_t intro_circ_retry_started_time;
+
+  /* Number of circuit we've launched during a single retry period. This
+   * should never go over MAX_INTRO_CIRCS_PER_PERIOD. */
+  unsigned int num_intro_circ_launched;
+
+  /* Replay cache tracking the REND_COOKIE found in INTRODUCE2 cell to detect
+   * repeats. Clients may send INTRODUCE1 cells for the same rendezvous point
+   * through two or more different introduction points; when they do, this
+   * keeps us from launching multiple simultaneous attempts to connect to the
+   * same rend point. */
+  replaycache_t *replay_cache_rend_cookie;
+
+  /* When is the next time we should rotate our descriptors. This is has to be
+   * done at the start time of the next SRV protocol run. */
+  time_t next_rotation_time;
+} hs_service_state_t;
+
+/* Representation of a service running on this tor instance. */
+typedef struct hs_service_t {
+  /* Onion address base32 encoded and NUL terminated. We keep it for logging
+   * purposes so we don't have to build it everytime. */
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+
+  /* Hashtable node: use to look up the service by its master public identity
+   * key in the service global map. */
+  HT_ENTRY(hs_service_t) hs_service_node;
+
+  /* Service state which contains various flags and counters. */
+  hs_service_state_t state;
+
+  /* Key material of the service. */
+  hs_service_keys_t keys;
+
+  /* Configuration of the service. */
+  hs_service_config_t config;
+
+  /* Current descriptor. */
+  hs_service_descriptor_t *desc_current;
+  /* Next descriptor. */
+  hs_service_descriptor_t *desc_next;
+
+  /* XXX: Credential (client auth.) #20700. */
+
+} hs_service_t;
+
+/* For the service global hash map, we define a specific type for it which
+ * will make it safe to use and specific to some controlled parameters such as
+ * the hashing function and how to compare services. */
+typedef HT_HEAD(hs_service_ht, hs_service_t) hs_service_ht;
+
+/* API */
+
+/* Global initializer and cleanup function. */
+void hs_service_init(void);
+void hs_service_free_all(void);
+
+/* Service new/free functions. */
+hs_service_t *hs_service_new(const or_options_t *options);
+void hs_service_free_(hs_service_t *service);
+#define hs_service_free(s) FREE_AND_NULL(hs_service_t, hs_service_free_, (s))
+
+unsigned int hs_service_get_num_services(void);
+void hs_service_stage_services(const smartlist_t *service_list);
+int hs_service_load_all_keys(void);
+int hs_service_get_version_from_key(const hs_service_t *service);
+void hs_service_lists_fnames_for_sandbox(smartlist_t *file_list,
+                                         smartlist_t *dir_list);
+int hs_service_set_conn_addr_port(const origin_circuit_t *circ,
+                                  edge_connection_t *conn);
+
+void hs_service_map_has_changed(void);
+void hs_service_dir_info_changed(void);
+void hs_service_run_scheduled_events(time_t now);
+void hs_service_circuit_has_opened(origin_circuit_t *circ);
+int hs_service_receive_intro_established(origin_circuit_t *circ,
+                                         const uint8_t *payload,
+                                         size_t payload_len);
+int hs_service_receive_introduce2(origin_circuit_t *circ,
+                                  const uint8_t *payload,
+                                  size_t payload_len);
+
+void hs_service_intro_circ_has_closed(origin_circuit_t *circ);
+
+char *hs_service_lookup_current_desc(const ed25519_public_key_t *pk);
+
+hs_service_add_ephemeral_status_t
+hs_service_add_ephemeral(ed25519_secret_key_t *sk, smartlist_t *ports,
+                         int max_streams_per_rdv_circuit,
+                         int max_streams_close_circuit, char **address_out);
+int hs_service_del_ephemeral(const char *address);
+
+/* Used outside of the HS subsystem by the control port command HSPOST. */
+void hs_service_upload_desc_to_dir(const char *encoded_desc,
+                                   const uint8_t version,
+                                   const ed25519_public_key_t *identity_pk,
+                                   const ed25519_public_key_t *blinded_pk,
+                                   const routerstatus_t *hsdir_rs);
+
+hs_circuit_id_protocol_t
+hs_service_exports_circuit_id(const ed25519_public_key_t *pk);
+
+#ifdef HS_SERVICE_PRIVATE
+
+#ifdef TOR_UNIT_TESTS
+/* Useful getters for unit tests. */
+STATIC unsigned int get_hs_service_map_size(void);
+STATIC int get_hs_service_staging_list_size(void);
+STATIC hs_service_ht *get_hs_service_map(void);
+STATIC hs_service_t *get_first_service(void);
+STATIC hs_service_intro_point_t *service_intro_point_find_by_ident(
+                                         const hs_service_t *service,
+                                         const hs_ident_circuit_t *ident);
+#endif
+
+/* Service accessors. */
+STATIC hs_service_t *find_service(hs_service_ht *map,
+                                  const ed25519_public_key_t *pk);
+STATIC void remove_service(hs_service_ht *map, hs_service_t *service);
+STATIC int register_service(hs_service_ht *map, hs_service_t *service);
+/* Service introduction point functions. */
+STATIC hs_service_intro_point_t *service_intro_point_new(
+                            const extend_info_t *ei,
+                            unsigned int is_legacy,
+                            unsigned int supports_ed25519_link_handshake_any);
+STATIC void service_intro_point_free_(hs_service_intro_point_t *ip);
+#define service_intro_point_free(ip)                            \
+  FREE_AND_NULL(hs_service_intro_point_t,             \
+                          service_intro_point_free_, (ip))
+STATIC void service_intro_point_add(digest256map_t *map,
+                                    hs_service_intro_point_t *ip);
+STATIC void service_intro_point_remove(const hs_service_t *service,
+                                       const hs_service_intro_point_t *ip);
+STATIC hs_service_intro_point_t *service_intro_point_find(
+                                 const hs_service_t *service,
+                                 const ed25519_public_key_t *auth_key);
+/* Service descriptor functions. */
+STATIC hs_service_descriptor_t *service_descriptor_new(void);
+STATIC hs_service_descriptor_t *service_desc_find_by_intro(
+                                         const hs_service_t *service,
+                                         const hs_service_intro_point_t *ip);
+/* Helper functions. */
+STATIC int client_filename_is_valid(const char *filename);
+STATIC hs_service_authorized_client_t *
+parse_authorized_client(const char *client_key_str);
+STATIC void get_objects_from_ident(const hs_ident_circuit_t *ident,
+                                   hs_service_t **service,
+                                   hs_service_intro_point_t **ip,
+                                   hs_service_descriptor_t **desc);
+STATIC const node_t *
+get_node_from_intro_point(const hs_service_intro_point_t *ip);
+STATIC int can_service_launch_intro_circuit(hs_service_t *service,
+                                            time_t now);
+STATIC int intro_point_should_expire(const hs_service_intro_point_t *ip,
+                                     time_t now);
+STATIC void run_housekeeping_event(time_t now);
+STATIC void rotate_all_descriptors(time_t now);
+STATIC void build_all_descriptors(time_t now);
+STATIC void update_all_descriptors_intro_points(time_t now);
+STATIC void run_upload_descriptor_event(time_t now);
+
+STATIC void service_descriptor_free_(hs_service_descriptor_t *desc);
+#define service_descriptor_free(d) \
+  FREE_AND_NULL(hs_service_descriptor_t, \
+                           service_descriptor_free_, (d))
+
+STATIC void
+service_authorized_client_free_(hs_service_authorized_client_t *client);
+#define service_authorized_client_free(c) \
+  FREE_AND_NULL(hs_service_authorized_client_t, \
+                           service_authorized_client_free_, (c))
+
+STATIC int
+write_address_to_file(const hs_service_t *service, const char *fname_);
+
+STATIC void upload_descriptor_to_all(const hs_service_t *service,
+                                     hs_service_descriptor_t *desc);
+
+STATIC void service_desc_schedule_upload(hs_service_descriptor_t *desc,
+                                         time_t now,
+                                         int descriptor_changed);
+
+STATIC int service_desc_hsdirs_changed(const hs_service_t *service,
+                                const hs_service_descriptor_t *desc);
+
+STATIC int service_authorized_client_config_equal(
+                                         const hs_service_config_t *config1,
+                                         const hs_service_config_t *config2);
+
+STATIC void service_clear_config(hs_service_config_t *config);
+
+#endif /* defined(HS_SERVICE_PRIVATE) */
+
+#endif /* !defined(TOR_HS_SERVICE_H) */
diff --git a/src/feature/hs/hs_stats.c b/src/feature/hs/hs_stats.c
new file mode 100644
index 0000000000..f24b731328
--- /dev/null
+++ b/src/feature/hs/hs_stats.c
@@ -0,0 +1,58 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_stats.c
+ * \brief Keeps stats about the activity of our onion service(s).
+ **/
+
+#include "core/or/or.h"
+#include "feature/hs/hs_stats.h"
+#include "feature/hs/hs_service.h"
+
+/** Number of v3 INTRODUCE2 cells received */
+static uint32_t n_introduce2_v3 = 0;
+/** Number of v2 INTRODUCE2 cells received */
+static uint32_t n_introduce2_v2 = 0;
+/** Number of attempts to make a circuit to a rendezvous point */
+static uint32_t n_rendezvous_launches = 0;
+
+/** Note that we received another INTRODUCE2 cell. */
+void
+hs_stats_note_introduce2_cell(int is_hsv3)
+{
+  if (is_hsv3) {
+    n_introduce2_v3++;
+  } else {
+    n_introduce2_v2++;
+  }
+}
+
+/** Return the number of v3 INTRODUCE2 cells we have received. */
+uint32_t
+hs_stats_get_n_introduce2_v3_cells(void)
+{
+  return n_introduce2_v3;
+}
+
+/** Return the number of v2 INTRODUCE2 cells we have received. */
+uint32_t
+hs_stats_get_n_introduce2_v2_cells(void)
+{
+  return n_introduce2_v2;
+}
+
+/** Note that we attempted to launch another circuit to a rendezvous point. */
+void
+hs_stats_note_service_rendezvous_launch(void)
+{
+  n_rendezvous_launches++;
+}
+
+/** Return the number of rendezvous circuits we have attempted to launch. */
+uint32_t
+hs_stats_get_n_rendezvous_launches(void)
+{
+  return n_rendezvous_launches;
+}
+
diff --git a/src/feature/hs/hs_stats.h b/src/feature/hs/hs_stats.h
new file mode 100644
index 0000000000..d89440faca
--- /dev/null
+++ b/src/feature/hs/hs_stats.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_stats.h
+ * \brief Header file for hs_stats.c
+ **/
+
+void hs_stats_note_introduce2_cell(int is_hsv3);
+uint32_t hs_stats_get_n_introduce2_v3_cells(void);
+uint32_t hs_stats_get_n_introduce2_v2_cells(void);
+void hs_stats_note_service_rendezvous_launch(void);
+uint32_t hs_stats_get_n_rendezvous_launches(void);
+
diff --git a/src/feature/hs/hsdir_index_st.h b/src/feature/hs/hsdir_index_st.h
new file mode 100644
index 0000000000..7d4116d8bb
--- /dev/null
+++ b/src/feature/hs/hsdir_index_st.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef HSDIR_INDEX_ST_H
+#define HSDIR_INDEX_ST_H
+
+/* Hidden service directory index used in a node_t which is set once we set
+ * the consensus. */
+struct hsdir_index_t {
+  /* HSDir index to use when fetching a descriptor. */
+  uint8_t fetch[DIGEST256_LEN];
+
+  /* HSDir index used by services to store their first and second
+   * descriptor. The first descriptor is chronologically older than the second
+   * one and uses older TP and SRV values. */
+  uint8_t store_first[DIGEST256_LEN];
+  uint8_t store_second[DIGEST256_LEN];
+};
+
+#endif
+
diff --git a/src/or/replaycache.c b/src/feature/hs_common/replaycache.c
index 8290fa6964..9e8c13b1c5 100644
--- a/src/or/replaycache.c
+++ b/src/feature/hs_common/replaycache.c
@@ -1,4 +1,4 @@
- /* Copyright (c) 2012-2016, The Tor Project, Inc. */
+ /* Copyright (c) 2012-2019, The Tor Project, Inc. */
  /* See LICENSE for licensing information */
 
 /**
@@ -21,14 +21,13 @@
 
 #define REPLAYCACHE_PRIVATE
 
-#include "or.h"
-#include "replaycache.h"
+#include "core/or/or.h"
+#include "feature/hs_common/replaycache.h"
 
 /** Free the replaycache r and all of its entries.
  */
-
 void
-replaycache_free(replaycache_t *r)
+replaycache_free_(replaycache_t *r)
 {
   if (!r) {
     log_info(LD_BUG, "replaycache_free() called on NULL");
@@ -44,7 +43,6 @@ replaycache_free(replaycache_t *r)
  * for entries to age out and interval is the time after which the cache
  * should be scrubbed for old entries.
  */
-
 replaycache_t *
 replaycache_new(time_t horizon, time_t interval)
 {
@@ -72,9 +70,8 @@ replaycache_new(time_t horizon, time_t interval)
   return r;
 }
 
-/** See documentation for replaycache_add_and_test()
+/** See documentation for replaycache_add_and_test().
  */
-
 STATIC int
 replaycache_add_and_test_internal(
     time_t present, replaycache_t *r, const void *data, size_t len,
@@ -136,9 +133,8 @@ replaycache_add_and_test_internal(
   return rv;
 }
 
-/** See documentation for replaycache_scrub_if_needed()
+/** See documentation for replaycache_scrub_if_needed().
  */
-
 STATIC void
 replaycache_scrub_if_needed_internal(time_t present, replaycache_t *r)
 {
@@ -186,7 +182,6 @@ replaycache_scrub_if_needed_internal(time_t present, replaycache_t *r)
  * and the function will return 1 if it was already seen within the cache's
  * horizon, or 0 otherwise.
  */
-
 int
 replaycache_add_and_test(replaycache_t *r, const void *data, size_t len)
 {
@@ -196,7 +191,6 @@ replaycache_add_and_test(replaycache_t *r, const void *data, size_t len)
 /** Like replaycache_add_and_test(), but if it's a hit also return the time
  * elapsed since this digest was last seen.
  */
-
 int
 replaycache_add_test_and_elapsed(
     replaycache_t *r, const void *data, size_t len, time_t *elapsed)
@@ -207,7 +201,6 @@ replaycache_add_test_and_elapsed(
 /** Scrub aged entries out of r if sufficiently long has elapsed since r was
  * last scrubbed.
  */
-
 void
 replaycache_scrub_if_needed(replaycache_t *r)
 {
diff --git a/src/or/replaycache.h b/src/feature/hs_common/replaycache.h
index 64a6caf5f5..01f5e600c2 100644
--- a/src/or/replaycache.h
+++ b/src/feature/hs_common/replaycache.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,11 +9,11 @@
 #ifndef TOR_REPLAYCACHE_H
 #define TOR_REPLAYCACHE_H
 
-typedef struct replaycache_s replaycache_t;
+typedef struct replaycache_t replaycache_t;
 
 #ifdef REPLAYCACHE_PRIVATE
 
-struct replaycache_s {
+struct replaycache_t {
   /* Scrub interval */
   time_t scrub_interval;
   /* Last scrubbed */
@@ -29,11 +29,13 @@ struct replaycache_s {
   digest256map_t *digests_seen;
 };
 
-#endif /* REPLAYCACHE_PRIVATE */
+#endif /* defined(REPLAYCACHE_PRIVATE) */
 
 /* replaycache_t free/new */
 
-void replaycache_free(replaycache_t *r);
+void replaycache_free_(replaycache_t *r);
+#define replaycache_free(r) \
+  FREE_AND_NULL(replaycache_t, replaycache_free_, (r))
 replaycache_t * replaycache_new(time_t horizon, time_t interval);
 
 #ifdef REPLAYCACHE_PRIVATE
@@ -51,7 +53,7 @@ STATIC int replaycache_add_and_test_internal(
 STATIC void replaycache_scrub_if_needed_internal(
     time_t present, replaycache_t *r);
 
-#endif /* REPLAYCACHE_PRIVATE */
+#endif /* defined(REPLAYCACHE_PRIVATE) */
 
 /*
  * replaycache_t methods
@@ -62,5 +64,4 @@ int replaycache_add_test_and_elapsed(
     replaycache_t *r, const void *data, size_t len, time_t *elapsed);
 void replaycache_scrub_if_needed(replaycache_t *r);
 
-#endif
-
+#endif /* !defined(TOR_REPLAYCACHE_H) */
diff --git a/src/feature/hs_common/shared_random_client.c b/src/feature/hs_common/shared_random_client.c
new file mode 100644
index 0000000000..5772034c6d
--- /dev/null
+++ b/src/feature/hs_common/shared_random_client.c
@@ -0,0 +1,293 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file shared_random_client.c
+ * \brief This file contains functions that are from the shared random
+ *        subsystem but used by many part of tor. The full feature is built
+ *        as part of the dirauth module.
+ **/
+
+#define SHARED_RANDOM_CLIENT_PRIVATE
+#include "feature/hs_common/shared_random_client.h"
+
+#include "app/config/config.h"
+#include "feature/dircommon/voting_schedule.h"
+#include "feature/nodelist/networkstatus.h"
+#include "lib/encoding/binascii.h"
+
+#include "feature/nodelist/networkstatus_st.h"
+
+/* Convert a given srv object to a string for the control port. This doesn't
+ * fail and the srv object MUST be valid. */
+static char *
+srv_to_control_string(const sr_srv_t *srv)
+{
+  char *srv_str;
+  char srv_hash_encoded[SR_SRV_VALUE_BASE64_LEN + 1];
+  tor_assert(srv);
+
+  sr_srv_encode(srv_hash_encoded, sizeof(srv_hash_encoded), srv);
+  tor_asprintf(&srv_str, "%s", srv_hash_encoded);
+  return srv_str;
+}
+
+/* Return the voting interval of the tor vote subsystem. */
+int
+get_voting_interval(void)
+{
+  int interval;
+  networkstatus_t *consensus = networkstatus_get_live_consensus(time(NULL));
+
+  if (consensus) {
+    interval = (int)(consensus->fresh_until - consensus->valid_after);
+  } else {
+    /* Same for both a testing and real network. We voluntarily ignore the
+     * InitialVotingInterval since it complexifies things and it doesn't
+     * affect the SR protocol. */
+    interval = get_options()->V3AuthVotingInterval;
+  }
+  tor_assert(interval > 0);
+  return interval;
+}
+
+/* Given the current consensus, return the start time of the current round of
+ * the SR protocol. For example, if it's 23:47:08, the current round thus
+ * started at 23:47:00 for a voting interval of 10 seconds.
+ *
+ * This function uses the consensus voting schedule to derive its results,
+ * instead of the actual consensus we are currently using, so it should be used
+ * for voting purposes. */
+time_t
+get_start_time_of_current_round(void)
+{
+  const or_options_t *options = get_options();
+  int voting_interval = get_voting_interval();
+  /* First, get the start time of the next round */
+  time_t next_start = voting_schedule_get_next_valid_after_time();
+  /* Now roll back next_start by a voting interval to find the start time of
+     the current round. */
+  time_t curr_start = voting_schedule_get_start_of_next_interval(
+                                     next_start - voting_interval - 1,
+                                     voting_interval,
+                                     options->TestingV3AuthVotingStartOffset);
+  return curr_start;
+}
+
+/*
+ * Public API
+ */
+
+/* Encode the given shared random value and put it in dst. Destination
+ * buffer must be at least SR_SRV_VALUE_BASE64_LEN plus the NULL byte. */
+void
+sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv)
+{
+  int ret;
+  /* Extra byte for the NULL terminated char. */
+  char buf[SR_SRV_VALUE_BASE64_LEN + 1];
+
+  tor_assert(dst);
+  tor_assert(srv);
+  tor_assert(dst_len >= sizeof(buf));
+
+  ret = base64_encode(buf, sizeof(buf), (const char *) srv->value,
+                      sizeof(srv->value), 0);
+  /* Always expect the full length without the NULL byte. */
+  tor_assert(ret == (sizeof(buf) - 1));
+  tor_assert(ret <= (int) dst_len);
+  strlcpy(dst, buf, dst_len);
+}
+
+/* Return the current SRV string representation for the control port. Return a
+ * newly allocated string on success containing the value else "" if not found
+ * or if we don't have a valid consensus yet. */
+char *
+sr_get_current_for_control(void)
+{
+  char *srv_str;
+  const networkstatus_t *c = networkstatus_get_latest_consensus();
+  if (c && c->sr_info.current_srv) {
+    srv_str = srv_to_control_string(c->sr_info.current_srv);
+  } else {
+    srv_str = tor_strdup("");
+  }
+  return srv_str;
+}
+
+/* Return the previous SRV string representation for the control port. Return
+ * a newly allocated string on success containing the value else "" if not
+ * found or if we don't have a valid consensus yet. */
+char *
+sr_get_previous_for_control(void)
+{
+  char *srv_str;
+  const networkstatus_t *c = networkstatus_get_latest_consensus();
+  if (c && c->sr_info.previous_srv) {
+    srv_str = srv_to_control_string(c->sr_info.previous_srv);
+  } else {
+    srv_str = tor_strdup("");
+  }
+  return srv_str;
+}
+
+/* Return current shared random value from the latest consensus. Caller can
+ * NOT keep a reference to the returned pointer. Return NULL if none. */
+const sr_srv_t *
+sr_get_current(const networkstatus_t *ns)
+{
+  const networkstatus_t *consensus;
+
+  /* Use provided ns else get a live one */
+  if (ns) {
+    consensus = ns;
+  } else {
+    consensus = networkstatus_get_live_consensus(approx_time());
+  }
+  /* Ideally we would never be asked for an SRV without a live consensus. Make
+   * sure this assumption is correct. */
+  tor_assert_nonfatal(consensus);
+
+  if (consensus) {
+    return consensus->sr_info.current_srv;
+  }
+  return NULL;
+}
+
+/* Return previous shared random value from the latest consensus. Caller can
+ * NOT keep a reference to the returned pointer. Return NULL if none. */
+const sr_srv_t *
+sr_get_previous(const networkstatus_t *ns)
+{
+  const networkstatus_t *consensus;
+
+  /* Use provided ns else get a live one */
+  if (ns) {
+    consensus = ns;
+  } else {
+    consensus = networkstatus_get_live_consensus(approx_time());
+  }
+  /* Ideally we would never be asked for an SRV without a live consensus. Make
+   * sure this assumption is correct. */
+  tor_assert_nonfatal(consensus);
+
+  if (consensus) {
+    return consensus->sr_info.previous_srv;
+  }
+  return NULL;
+}
+
+/* Parse a list of arguments from a SRV value either from a vote, consensus
+ * or from our disk state and return a newly allocated srv object. NULL is
+ * returned on error.
+ *
+ * The arguments' order:
+ *    num_reveals, value
+ */
+sr_srv_t *
+sr_parse_srv(const smartlist_t *args)
+{
+  char *value;
+  int ok, ret;
+  uint64_t num_reveals;
+  sr_srv_t *srv = NULL;
+
+  tor_assert(args);
+
+  if (smartlist_len(args) < 2) {
+    goto end;
+  }
+
+  /* First argument is the number of reveal values */
+  num_reveals = tor_parse_uint64(smartlist_get(args, 0),
+                                 10, 0, UINT64_MAX, &ok, NULL);
+  if (!ok) {
+    goto end;
+  }
+  /* Second and last argument is the shared random value it self. */
+  value = smartlist_get(args, 1);
+  if (strlen(value) != SR_SRV_VALUE_BASE64_LEN) {
+    goto end;
+  }
+
+  srv = tor_malloc_zero(sizeof(*srv));
+  srv->num_reveals = num_reveals;
+  /* We subtract one byte from the srclen because the function ignores the
+   * '=' character in the given buffer. This is broken but it's a documented
+   * behavior of the implementation. */
+  ret = base64_decode((char *) srv->value, sizeof(srv->value), value,
+                      SR_SRV_VALUE_BASE64_LEN - 1);
+  if (ret != sizeof(srv->value)) {
+    tor_free(srv);
+    srv = NULL;
+    goto end;
+  }
+ end:
+  return srv;
+}
+
+/** Return the start time of the current SR protocol run using the times from
+ *  the current consensus. For example, if the latest consensus valid-after is
+ *  23/06/2017 23:00:00 and a full SR protocol run is 24 hours, this function
+ *  returns 23/06/2017 00:00:00. */
+time_t
+sr_state_get_start_time_of_current_protocol_run(void)
+{
+  int total_rounds = SHARED_RANDOM_N_ROUNDS * SHARED_RANDOM_N_PHASES;
+  int voting_interval = get_voting_interval();
+  time_t beginning_of_curr_round;
+
+  /* This function is not used for voting purposes, so if we have a live
+     consensus, use its valid-after as the beginning of the current round,
+     otherwise resort to the voting schedule which should always exist. */
+  networkstatus_t *ns = networkstatus_get_live_consensus(approx_time());
+  if (ns) {
+    beginning_of_curr_round = ns->valid_after;
+  } else {
+    beginning_of_curr_round = get_start_time_of_current_round();
+  }
+
+  /* Get current SR protocol round */
+  int curr_round_slot;
+  curr_round_slot = (beginning_of_curr_round / voting_interval) % total_rounds;
+
+  /* Get start time by subtracting the time elapsed from the beginning of the
+     protocol run */
+  time_t time_elapsed_since_start_of_run = curr_round_slot * voting_interval;
+
+  log_debug(LD_GENERAL, "Current SRV proto run: Start of current round: %u. "
+            "Time elapsed: %u (%d)", (unsigned) beginning_of_curr_round,
+            (unsigned) time_elapsed_since_start_of_run, voting_interval);
+
+  return beginning_of_curr_round - time_elapsed_since_start_of_run;
+}
+
+/** Return the start time of the previous SR protocol run. See
+ *  sr_state_get_start_time_of_current_protocol_run() for more details.  */
+time_t
+sr_state_get_start_time_of_previous_protocol_run(void)
+{
+  time_t start_time_of_current_run =
+    sr_state_get_start_time_of_current_protocol_run();
+
+  /* We get the start time of previous protocol run, by getting the start time
+   * of current run and the subtracting a full protocol run from that. */
+  return start_time_of_current_run - sr_state_get_protocol_run_duration();
+}
+
+/** Return the time (in seconds) it takes to complete a full SR protocol phase
+ *  (e.g. the commit phase). */
+unsigned int
+sr_state_get_phase_duration(void)
+{
+  return SHARED_RANDOM_N_ROUNDS * get_voting_interval();
+}
+
+/** Return the time (in seconds) it takes to complete a full SR protocol run */
+unsigned int
+sr_state_get_protocol_run_duration(void)
+{
+  int total_protocol_rounds = SHARED_RANDOM_N_ROUNDS * SHARED_RANDOM_N_PHASES;
+  return total_protocol_rounds * get_voting_interval();
+}
+
diff --git a/src/feature/hs_common/shared_random_client.h b/src/feature/hs_common/shared_random_client.h
new file mode 100644
index 0000000000..95fe2c65ab
--- /dev/null
+++ b/src/feature/hs_common/shared_random_client.h
@@ -0,0 +1,48 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file shared_random_client.h
+ * \brief Header file for shared_random_client.c.
+ **/
+
+#ifndef TOR_SHARED_RANDOM_CLIENT_H
+#define TOR_SHARED_RANDOM_CLIENT_H
+
+/* Dirauth module. */
+#include "feature/dirauth/shared_random.h"
+
+/* Helper functions. */
+void sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv);
+int get_voting_interval(void);
+
+/* Control port functions. */
+char *sr_get_current_for_control(void);
+char *sr_get_previous_for_control(void);
+
+/* SRV functions. */
+const sr_srv_t *sr_get_current(const networkstatus_t *ns);
+const sr_srv_t *sr_get_previous(const networkstatus_t *ns);
+sr_srv_t *sr_parse_srv(const smartlist_t *args);
+
+/*
+ * Shared Random State API
+ */
+
+/* Each protocol phase has 12 rounds  */
+#define SHARED_RANDOM_N_ROUNDS 12
+/* Number of phase we have in a protocol. */
+#define SHARED_RANDOM_N_PHASES 2
+
+time_t sr_state_get_start_time_of_current_protocol_run(void);
+time_t sr_state_get_start_time_of_previous_protocol_run(void);
+unsigned int sr_state_get_phase_duration(void);
+unsigned int sr_state_get_protocol_run_duration(void);
+time_t get_start_time_of_current_round(void);
+
+#ifdef TOR_UNIT_TESTS
+
+#endif /* TOR_UNIT_TESTS */
+
+#endif /* TOR_SHARED_RANDOM_CLIENT_H */
+
diff --git a/src/or/routerkeys.c b/src/feature/keymgt/loadkey.c
index ca32228fc7..a8cbf0e582 100644
--- a/src/or/routerkeys.c
+++ b/src/feature/keymgt/loadkey.c
@@ -1,24 +1,119 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file routerkeys.c
+ * \file loadkey.c
+ * \brief Read keys from disk, creating as needed
  *
- * \brief Functions and structures to handle generating and maintaining the
- *  set of keypairs necessary to be an OR. (Some of the code in router.c
- *  belongs here.)
- */
+ * This code is shared by relays and onion services, which both need
+ * this functionality.
+ **/
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "app/main/main.h"
+#include "feature/keymgt/loadkey.h"
+#include "feature/nodelist/torcert.h"
 
-#include "or.h"
-#include "config.h"
-#include "router.h"
-#include "crypto_pwbox.h"
-#include "routerkeys.h"
-#include "torcert.h"
+#include "lib/crypt_ops/crypto_pwbox.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/term/getpass.h"
+#include "lib/crypt_ops/crypto_format.h"
 
 #define ENC_KEY_HEADER "Boxed Ed25519 key"
 #define ENC_KEY_TAG "master"
 
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/** Try to read an RSA key from <b>fname</b>.  If <b>fname</b> doesn't exist
+ * and <b>generate</b> is true, create a new RSA key and save it in
+ * <b>fname</b>.  Return the read/created key, or NULL on error.  Log all
+ * errors at level <b>severity</b>. If <b>created_out/b> is non-NULL and a
+ * new key was created, set *<b>created_out</b> to true.
+ */
+crypto_pk_t *
+init_key_from_file(const char *fname, int generate, int severity,
+                   bool *created_out)
+{
+  crypto_pk_t *prkey = NULL;
+
+  if (created_out) {
+    *created_out = false;
+  }
+
+  if (!(prkey = crypto_pk_new())) {
+    tor_log(severity, LD_GENERAL,"Error constructing key");
+    goto error;
+  }
+
+  switch (file_status(fname)) {
+    case FN_DIR:
+    case FN_ERROR:
+      tor_log(severity, LD_FS,"Can't read key from \"%s\"", fname);
+      goto error;
+    /* treat empty key files as if the file doesn't exist, and,
+     * if generate is set, replace the empty file in
+     * crypto_pk_write_private_key_to_filename() */
+    case FN_NOENT:
+    case FN_EMPTY:
+      if (generate) {
+        if (!have_lockfile()) {
+          if (try_locking(get_options(), 0)<0) {
+            /* Make sure that --list-fingerprint only creates new keys
+             * if there is no possibility for a deadlock. */
+            tor_log(severity, LD_FS, "Another Tor process has locked \"%s\". "
+                    "Not writing any new keys.", fname);
+            /*XXXX The 'other process' might make a key in a second or two;
+             * maybe we should wait for it. */
+            goto error;
+          }
+        }
+        log_info(LD_GENERAL, "No key found in \"%s\"; generating fresh key.",
+                 fname);
+        if (crypto_pk_generate_key(prkey)) {
+          tor_log(severity, LD_GENERAL,"Error generating onion key");
+          goto error;
+        }
+        if (! crypto_pk_is_valid_private_key(prkey)) {
+          tor_log(severity, LD_GENERAL,"Generated key seems invalid");
+          goto error;
+        }
+        log_info(LD_GENERAL, "Generated key seems valid");
+        if (created_out) {
+          *created_out = true;
+        }
+        if (crypto_pk_write_private_key_to_filename(prkey, fname)) {
+          tor_log(severity, LD_FS,
+              "Couldn't write generated key to \"%s\".", fname);
+          goto error;
+        }
+      } else {
+        tor_log(severity, LD_GENERAL, "No key found in \"%s\"", fname);
+        goto error;
+      }
+      return prkey;
+    case FN_FILE:
+      if (crypto_pk_read_private_key_from_filename(prkey, fname)) {
+        tor_log(severity, LD_GENERAL,"Error loading private key.");
+        goto error;
+      }
+      return prkey;
+    default:
+      tor_assert(0);
+  }
+
+ error:
+  if (prkey)
+    crypto_pk_free(prkey);
+  return NULL;
+}
+
+/* DOCDOC */
 static ssize_t
 do_getpass(const char *prompt, char *buf, size_t buflen,
            int twice, const or_options_t *options)
@@ -37,7 +132,7 @@ do_getpass(const char *prompt, char *buf, size_t buflen,
   if (options->use_keygen_passphrase_fd) {
     twice = 0;
     fd = options->keygen_passphrase_fd;
-    length = read_all(fd, buf, buflen-1, 0);
+    length = read_all_from_fd(fd, buf, buflen-1);
     if (length >= 0)
       buf[length] = 0;
     goto done_reading;
@@ -85,6 +180,7 @@ do_getpass(const char *prompt, char *buf, size_t buflen,
   return length;
 }
 
+/* DOCDOC */
 int
 read_encrypted_secret_key(ed25519_secret_key_t *out,
                           const char *fname)
@@ -157,6 +253,7 @@ read_encrypted_secret_key(ed25519_secret_key_t *out,
   return r;
 }
 
+/* DOCDOC */
 int
 write_encrypted_secret_key(const ed25519_secret_key_t *key,
                            const char *fname)
@@ -200,6 +297,7 @@ write_encrypted_secret_key(const ed25519_secret_key_t *key,
   return r;
 }
 
+/* DOCDOC */
 static int
 write_secret_key(const ed25519_secret_key_t *key, int encrypted,
                  const char *fname,
@@ -230,6 +328,9 @@ write_secret_key(const ed25519_secret_key_t *key, int encrypted,
  * <b>fname</b>, with certificate type <b>cert_type</b>.  On failure, return
  * NULL; on success return the keypair.
  *
+ * The <b>options</b> is used to look at the change_key_passphrase value when
+ * writing to disk a secret key. It is safe to be NULL even in that case.
+ *
  * If INIT_ED_KEY_CREATE is set in <b>flags</b>, then create the key (and
  * certificate if requested) if it doesn't exist, and save it to disk.
  *
@@ -258,9 +359,6 @@ write_secret_key(const ed25519_secret_key_t *key, int encrypted,
  * secret key unless no public key is found.  Do not return a secret key. (but
  * create and save one if needed).
  *
- * If INIT_ED_KEY_NO_LOAD_SECRET is set in <b>flags</b>, don't try to load
- * a secret key, no matter what.
- *
  * If INIT_ED_KEY_TRY_ENCRYPTED is set, we look for an encrypted secret key
  * and consider encrypting any new secret key.
  *
@@ -273,6 +371,9 @@ write_secret_key(const ed25519_secret_key_t *key, int encrypted,
  *
  * If INIT_ED_KEY_EXPLICIT_FNAME is set, use the provided file name for the
  * secret key file, encrypted or not.
+ *
+ * If INIT_ED_KEY_OFFLINE_SECRET is set, we won't try to load the master
+ * secret key and we log a message at <b>severity</b> that we've done so.
  */
 ed25519_keypair_t *
 ed_key_init_from_file(const char *fname, uint32_t flags,
@@ -281,7 +382,8 @@ ed_key_init_from_file(const char *fname, uint32_t flags,
                       time_t now,
                       time_t lifetime,
                       uint8_t cert_type,
-                      struct tor_cert_st **cert_out)
+                      struct tor_cert_st **cert_out,
+                      const or_options_t *options)
 {
   char *secret_fname = NULL;
   char *encrypted_secret_fname = NULL;
@@ -485,7 +587,8 @@ ed_key_init_from_file(const char *fname, uint32_t flags,
 
   /* Write it to disk if we're supposed to do with a new passphrase, or if
    * we just created it. */
-  if (created_sk || (have_secret && get_options()->change_key_passphrase)) {
+  if (created_sk || (have_secret && options != NULL &&
+                     options->change_key_passphrase)) {
     if (write_secret_key(&keypair->seckey,
                          encrypt_key,
                          secret_fname, tag, encrypted_secret_fname) < 0
@@ -527,7 +630,8 @@ ed_key_init_from_file(const char *fname, uint32_t flags,
     bad_cert = 1;
   } else if (signing_key &&
              tor_cert_checksig(cert, &signing_key->pubkey, now) < 0) {
-    tor_log(severity, LD_OR, "Can't check certificate");
+    tor_log(severity, LD_OR, "Can't check certificate: %s",
+            tor_cert_describe_signature_status(cert));
     bad_cert = 1;
   } else if (cert->cert_expired) {
     tor_log(severity, LD_OR, "Certificate is expired");
@@ -649,499 +753,3 @@ ed_key_new(const ed25519_keypair_t *signing_key,
   tor_free(keypair);
   return NULL;
 }
-
-static ed25519_keypair_t *master_identity_key = NULL;
-static ed25519_keypair_t *master_signing_key = NULL;
-static ed25519_keypair_t *current_auth_key = NULL;
-static tor_cert_t *signing_key_cert = NULL;
-static tor_cert_t *link_cert_cert = NULL;
-static tor_cert_t *auth_key_cert = NULL;
-
-static uint8_t *rsa_ed_crosscert = NULL;
-static size_t rsa_ed_crosscert_len = 0;
-
-/**
- * Running as a server: load, reload, or refresh our ed25519 keys and
- * certificates, creating and saving new ones as needed.
- */
-int
-load_ed_keys(const or_options_t *options, time_t now)
-{
-  ed25519_keypair_t *id = NULL;
-  ed25519_keypair_t *sign = NULL;
-  ed25519_keypair_t *auth = NULL;
-  const ed25519_keypair_t *sign_signing_key_with_id = NULL;
-  const ed25519_keypair_t *use_signing = NULL;
-  const tor_cert_t *check_signing_cert = NULL;
-  tor_cert_t *sign_cert = NULL;
-  tor_cert_t *auth_cert = NULL;
-
-#define FAIL(msg) do {                          \
-    log_warn(LD_OR, (msg));                     \
-    goto err;                                   \
-  } while (0)
-#define SET_KEY(key, newval) do {               \
-    if ((key) != (newval))                      \
-      ed25519_keypair_free(key);                \
-    key = (newval);                             \
-  } while (0)
-#define SET_CERT(cert, newval) do {             \
-    if ((cert) != (newval))                     \
-      tor_cert_free(cert);                      \
-    cert = (newval);                            \
-  } while (0)
-#define EXPIRES_SOON(cert, interval)            \
-  (!(cert) || (cert)->valid_until < now + (interval))
-
-  /* XXXX support encrypted identity keys fully */
-
-  /* First try to get the signing key to see how it is. */
-  {
-    char *fname =
-      options_get_datadir_fname2(options, "keys", "ed25519_signing");
-    sign = ed_key_init_from_file(
-               fname,
-               INIT_ED_KEY_NEEDCERT|
-               INIT_ED_KEY_INCLUDE_SIGNING_KEY_IN_CERT,
-               LOG_INFO,
-               NULL, 0, 0, CERT_TYPE_ID_SIGNING, &sign_cert);
-    tor_free(fname);
-    check_signing_cert = sign_cert;
-    use_signing = sign;
-  }
-
-  if (!use_signing && master_signing_key) {
-    check_signing_cert = signing_key_cert;
-    use_signing = master_signing_key;
-  }
-
-  const int offline_master =
-    options->OfflineMasterKey && options->command != CMD_KEYGEN;
-  const int need_new_signing_key =
-    NULL == use_signing ||
-    EXPIRES_SOON(check_signing_cert, 0) ||
-    (options->command == CMD_KEYGEN && ! options->change_key_passphrase);
-  const int want_new_signing_key =
-    need_new_signing_key ||
-    EXPIRES_SOON(check_signing_cert, options->TestingSigningKeySlop);
-
-  /* We can only create a master key if we haven't been told that the
-   * master key will always be offline.  Also, if we have a signing key,
-   * then we shouldn't make a new master ID key. */
-  const int can_make_master_id_key = !offline_master &&
-    NULL == use_signing;
-
-  if (need_new_signing_key) {
-    log_notice(LD_OR, "It looks like I need to generate and sign a new "
-               "medium-term signing key, because %s. To do that, I need to "
-               "load%s the permanent master identity key.",
-            (NULL == use_signing) ? "I don't have one" :
-            EXPIRES_SOON(check_signing_cert, 0) ? "the one I have is expired" :
-               "you asked me to make one with --keygen",
-            can_make_master_id_key ? " (or create)" : "");
-  } else if (want_new_signing_key && !offline_master) {
-    log_notice(LD_OR, "It looks like I should try to generate and sign a "
-               "new medium-term signing key, because the one I have is "
-               "going to expire soon. To do that, I'm going to have to try to "
-               "load the permanent master identity key.");
-  } else if (want_new_signing_key) {
-    log_notice(LD_OR, "It looks like I should try to generate and sign a "
-               "new medium-term signing key, because the one I have is "
-               "going to expire soon. But OfflineMasterKey is set, so I "
-               "won't try to load a permanent master identity key is set. "
-               "You will need to use 'tor --keygen' make a new signing key "
-               "and certificate.");
-  }
-
-  {
-    uint32_t flags =
-      (INIT_ED_KEY_SPLIT|
-       INIT_ED_KEY_EXTRA_STRONG|INIT_ED_KEY_NO_REPAIR);
-    if (can_make_master_id_key)
-      flags |= INIT_ED_KEY_CREATE;
-    if (! need_new_signing_key)
-      flags |= INIT_ED_KEY_MISSING_SECRET_OK;
-    if (! want_new_signing_key || offline_master)
-      flags |= INIT_ED_KEY_OMIT_SECRET;
-    if (offline_master)
-      flags |= INIT_ED_KEY_OFFLINE_SECRET;
-    if (options->command == CMD_KEYGEN)
-      flags |= INIT_ED_KEY_TRY_ENCRYPTED;
-
-    /* Check the key directory */
-    if (check_private_dir(options->DataDirectory, CPD_CREATE, options->User)) {
-      log_err(LD_OR, "Can't create/check datadirectory %s",
-              options->DataDirectory);
-      goto err;
-    }
-    char *fname = get_datadir_fname("keys");
-    if (check_private_dir(fname, CPD_CREATE, options->User) < 0) {
-      log_err(LD_OR, "Problem creating/checking key directory %s", fname);
-      tor_free(fname);
-      goto err;
-    }
-    tor_free(fname);
-    if (options->master_key_fname) {
-      fname = tor_strdup(options->master_key_fname);
-      flags |= INIT_ED_KEY_EXPLICIT_FNAME;
-    } else {
-      fname = options_get_datadir_fname2(options, "keys", "ed25519_master_id");
-    }
-    id = ed_key_init_from_file(
-             fname,
-             flags,
-             LOG_WARN, NULL, 0, 0, 0, NULL);
-    tor_free(fname);
-    if (!id) {
-      if (need_new_signing_key) {
-        if (offline_master)
-          FAIL("Can't load master identity key; OfflineMasterKey is set.");
-        else
-          FAIL("Missing identity key");
-      } else {
-        log_warn(LD_OR, "Master public key was absent; inferring from "
-                 "public key in signing certificate and saving to disk.");
-        tor_assert(check_signing_cert);
-        id = tor_malloc_zero(sizeof(*id));
-        memcpy(&id->pubkey, &check_signing_cert->signing_key,
-               sizeof(ed25519_public_key_t));
-        fname = options_get_datadir_fname2(options, "keys",
-                                           "ed25519_master_id_public_key");
-        if (ed25519_pubkey_write_to_file(&id->pubkey, fname, "type0") < 0) {
-          log_warn(LD_OR, "Error while attempting to write master public key "
-                   "to disk");
-          tor_free(fname);
-          goto err;
-        }
-        tor_free(fname);
-      }
-    }
-    if (tor_mem_is_zero((char*)id->seckey.seckey, sizeof(id->seckey)))
-      sign_signing_key_with_id = NULL;
-    else
-      sign_signing_key_with_id = id;
-  }
-
-  if (master_identity_key &&
-      !ed25519_pubkey_eq(&id->pubkey, &master_identity_key->pubkey)) {
-    FAIL("Identity key on disk does not match key we loaded earlier!");
-  }
-
-  if (need_new_signing_key && NULL == sign_signing_key_with_id)
-    FAIL("Can't load master key make a new signing key.");
-
-  if (sign_cert) {
-    if (! sign_cert->signing_key_included)
-      FAIL("Loaded a signing cert with no key included!");
-    if (! ed25519_pubkey_eq(&sign_cert->signing_key, &id->pubkey))
-      FAIL("The signing cert we have was not signed with the master key "
-           "we loaded!");
-    if (tor_cert_checksig(sign_cert, &id->pubkey, 0) < 0)
-      FAIL("The signing cert we loaded was not signed correctly!");
-  }
-
-  if (want_new_signing_key && sign_signing_key_with_id) {
-    uint32_t flags = (INIT_ED_KEY_CREATE|
-                      INIT_ED_KEY_REPLACE|
-                      INIT_ED_KEY_EXTRA_STRONG|
-                      INIT_ED_KEY_NEEDCERT|
-                      INIT_ED_KEY_INCLUDE_SIGNING_KEY_IN_CERT);
-    char *fname =
-      options_get_datadir_fname2(options, "keys", "ed25519_signing");
-    ed25519_keypair_free(sign);
-    tor_cert_free(sign_cert);
-    sign = ed_key_init_from_file(fname,
-                                 flags, LOG_WARN,
-                                 sign_signing_key_with_id, now,
-                                 options->SigningKeyLifetime,
-                                 CERT_TYPE_ID_SIGNING, &sign_cert);
-    tor_free(fname);
-    if (!sign)
-      FAIL("Missing signing key");
-    use_signing = sign;
-
-    tor_assert(sign_cert->signing_key_included);
-    tor_assert(ed25519_pubkey_eq(&sign_cert->signing_key, &id->pubkey));
-    tor_assert(ed25519_pubkey_eq(&sign_cert->signed_key, &sign->pubkey));
-  } else if (want_new_signing_key) {
-    static ratelim_t missing_master = RATELIM_INIT(3600);
-    log_fn_ratelim(&missing_master, LOG_WARN, LD_OR,
-                   "Signing key will expire soon, but I can't load the "
-                   "master key to sign a new one!");
-  }
-
-  tor_assert(use_signing);
-
-  /* At this point we no longer need our secret identity key.  So wipe
-   * it, if we loaded it in the first place. */
-  memwipe(id->seckey.seckey, 0, sizeof(id->seckey));
-
-  if (options->command == CMD_KEYGEN)
-    goto end;
-
-  if (!rsa_ed_crosscert && server_mode(options)) {
-    uint8_t *crosscert;
-    ssize_t crosscert_len = tor_make_rsa_ed25519_crosscert(&id->pubkey,
-                                                   get_server_identity_key(),
-                                                   now+10*365*86400,/*XXXX*/
-                                                   &crosscert);
-    rsa_ed_crosscert_len = crosscert_len;
-    rsa_ed_crosscert = crosscert;
-  }
-
-  if (!current_auth_key ||
-      EXPIRES_SOON(auth_key_cert, options->TestingAuthKeySlop)) {
-    auth = ed_key_new(use_signing, INIT_ED_KEY_NEEDCERT,
-                      now,
-                      options->TestingAuthKeyLifetime,
-                      CERT_TYPE_SIGNING_AUTH, &auth_cert);
-
-    if (!auth)
-      FAIL("Can't create auth key");
-  }
-
-  /* We've generated or loaded everything.  Put them in memory. */
-
- end:
-  if (! master_identity_key) {
-    SET_KEY(master_identity_key, id);
-  } else {
-    tor_free(id);
-  }
-  if (sign) {
-    SET_KEY(master_signing_key, sign);
-    SET_CERT(signing_key_cert, sign_cert);
-  }
-  if (auth) {
-    SET_KEY(current_auth_key, auth);
-    SET_CERT(auth_key_cert, auth_cert);
-  }
-
-  return 0;
- err:
-  ed25519_keypair_free(id);
-  ed25519_keypair_free(sign);
-  ed25519_keypair_free(auth);
-  tor_cert_free(sign_cert);
-  tor_cert_free(auth_cert);
-  return -1;
-}
-
-/* DOCDOC */
-int
-generate_ed_link_cert(const or_options_t *options, time_t now)
-{
-  const tor_x509_cert_t *link_ = NULL, *id = NULL;
-  tor_cert_t *link_cert = NULL;
-
-  if (tor_tls_get_my_certs(1, &link_, &id) < 0 || link_ == NULL) {
-    log_warn(LD_OR, "Can't get my x509 link cert.");
-    return -1;
-  }
-
-  const common_digests_t *digests = tor_x509_cert_get_cert_digests(link_);
-
-  if (link_cert_cert &&
-      ! EXPIRES_SOON(link_cert_cert, options->TestingLinkKeySlop) &&
-      fast_memeq(digests->d[DIGEST_SHA256], link_cert_cert->signed_key.pubkey,
-                 DIGEST256_LEN)) {
-    return 0;
-  }
-
-  ed25519_public_key_t dummy_key;
-  memcpy(dummy_key.pubkey, digests->d[DIGEST_SHA256], DIGEST256_LEN);
-
-  link_cert = tor_cert_create(get_master_signing_keypair(),
-                              CERT_TYPE_SIGNING_LINK,
-                              &dummy_key,
-                              now,
-                              options->TestingLinkCertLifetime, 0);
-
-  if (link_cert) {
-    SET_CERT(link_cert_cert, link_cert);
-  }
-  return 0;
-}
-
-#undef FAIL
-#undef SET_KEY
-#undef SET_CERT
-
-int
-should_make_new_ed_keys(const or_options_t *options, const time_t now)
-{
-  if (!master_identity_key ||
-      !master_signing_key ||
-      !current_auth_key ||
-      !link_cert_cert ||
-      EXPIRES_SOON(signing_key_cert, options->TestingSigningKeySlop) ||
-      EXPIRES_SOON(auth_key_cert, options->TestingAuthKeySlop) ||
-      EXPIRES_SOON(link_cert_cert, options->TestingLinkKeySlop))
-    return 1;
-
-  const tor_x509_cert_t *link_ = NULL, *id = NULL;
-
-  if (tor_tls_get_my_certs(1, &link_, &id) < 0 || link_ == NULL)
-    return 1;
-
-  const common_digests_t *digests = tor_x509_cert_get_cert_digests(link_);
-
-  if (!fast_memeq(digests->d[DIGEST_SHA256],
-                  link_cert_cert->signed_key.pubkey,
-                  DIGEST256_LEN)) {
-    return 1;
-  }
-
-  return 0;
-}
-
-#undef EXPIRES_SOON
-
-const ed25519_public_key_t *
-get_master_identity_key(void)
-{
-  if (!master_identity_key)
-    return NULL;
-  return &master_identity_key->pubkey;
-}
-
-const ed25519_keypair_t *
-get_master_signing_keypair(void)
-{
-  return master_signing_key;
-}
-
-const struct tor_cert_st *
-get_master_signing_key_cert(void)
-{
-  return signing_key_cert;
-}
-
-const ed25519_keypair_t *
-get_current_auth_keypair(void)
-{
-  return current_auth_key;
-}
-
-const tor_cert_t *
-get_current_link_cert_cert(void)
-{
-  return link_cert_cert;
-}
-
-const tor_cert_t *
-get_current_auth_key_cert(void)
-{
-  return auth_key_cert;
-}
-
-void
-get_master_rsa_crosscert(const uint8_t **cert_out,
-                         size_t *size_out)
-{
-  *cert_out = rsa_ed_crosscert;
-  *size_out = rsa_ed_crosscert_len;
-}
-
-/** Construct cross-certification for the master identity key with
- * the ntor onion key. Store the sign of the corresponding ed25519 public key
- * in *<b>sign_out</b>. */
-tor_cert_t *
-make_ntor_onion_key_crosscert(const curve25519_keypair_t *onion_key,
-      const ed25519_public_key_t *master_id_key, time_t now, time_t lifetime,
-      int *sign_out)
-{
-  tor_cert_t *cert = NULL;
-  ed25519_keypair_t ed_onion_key;
-
-  if (ed25519_keypair_from_curve25519_keypair(&ed_onion_key, sign_out,
-                                              onion_key) < 0)
-    goto end;
-
-  cert = tor_cert_create(&ed_onion_key, CERT_TYPE_ONION_ID, master_id_key,
-      now, lifetime, 0);
-
- end:
-  memwipe(&ed_onion_key, 0, sizeof(ed_onion_key));
-  return cert;
-}
-
-/** Construct and return an RSA signature for the TAP onion key to
- * cross-certify the RSA and Ed25519 identity keys. Set <b>len_out</b> to its
- * length. */
-uint8_t *
-make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
-                             const ed25519_public_key_t *master_id_key,
-                             const crypto_pk_t *rsa_id_key,
-                             int *len_out)
-{
-  uint8_t signature[PK_BYTES];
-  uint8_t signed_data[DIGEST_LEN + ED25519_PUBKEY_LEN];
-
-  *len_out = 0;
-  crypto_pk_get_digest(rsa_id_key, (char*)signed_data);
-  memcpy(signed_data + DIGEST_LEN, master_id_key->pubkey, ED25519_PUBKEY_LEN);
-
-  int r = crypto_pk_private_sign(onion_key,
-                               (char*)signature, sizeof(signature),
-                               (const char*)signed_data, sizeof(signed_data));
-  if (r < 0)
-    return NULL;
-
-  *len_out = r;
-
-  return tor_memdup(signature, r);
-}
-
-/** Check whether an RSA-TAP cross-certification is correct. Return 0 if it
- * is, -1 if it isn't. */
-int
-check_tap_onion_key_crosscert(const uint8_t *crosscert,
-                              int crosscert_len,
-                              const crypto_pk_t *onion_pkey,
-                              const ed25519_public_key_t *master_id_pkey,
-                              const uint8_t *rsa_id_digest)
-{
-  uint8_t *cc = tor_malloc(crypto_pk_keysize(onion_pkey));
-  int cc_len =
-    crypto_pk_public_checksig(onion_pkey,
-                              (char*)cc,
-                              crypto_pk_keysize(onion_pkey),
-                              (const char*)crosscert,
-                              crosscert_len);
-  if (cc_len < 0) {
-    goto err;
-  }
-  if (cc_len < DIGEST_LEN + ED25519_PUBKEY_LEN) {
-    log_warn(LD_DIR, "Short signature on cross-certification with TAP key");
-    goto err;
-  }
-  if (tor_memneq(cc, rsa_id_digest, DIGEST_LEN) ||
-      tor_memneq(cc + DIGEST_LEN, master_id_pkey->pubkey,
-                 ED25519_PUBKEY_LEN)) {
-    log_warn(LD_DIR, "Incorrect cross-certification with TAP key");
-    goto err;
-  }
-
-  tor_free(cc);
-  return 0;
- err:
-  tor_free(cc);
-  return -1;
-}
-
-void
-routerkeys_free_all(void)
-{
-  ed25519_keypair_free(master_identity_key);
-  ed25519_keypair_free(master_signing_key);
-  ed25519_keypair_free(current_auth_key);
-  tor_cert_free(signing_key_cert);
-  tor_cert_free(link_cert_cert);
-  tor_cert_free(auth_key_cert);
-
-  master_identity_key = master_signing_key = NULL;
-  current_auth_key = NULL;
-  signing_key_cert = link_cert_cert = auth_key_cert = NULL;
-}
-
diff --git a/src/or/routerkeys.h b/src/feature/keymgt/loadkey.h
index be9b19aea8..8beee57a20 100644
--- a/src/or/routerkeys.h
+++ b/src/feature/keymgt/loadkey.h
@@ -1,10 +1,21 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#ifndef TOR_ROUTERKEYS_H
-#define TOR_ROUTERKEYS_H
+/**
+ * \file loadkey.h
+ * \brief Header file for loadkey.c
+ **/
 
-#include "crypto_ed25519.h"
+#ifndef TOR_LOADKEY_H
+#define TOR_LOADKEY_H
+
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+crypto_pk_t *init_key_from_file(const char *fname, int generate,
+                                int severity, bool *created_out);
 
 #define INIT_ED_KEY_CREATE                      (1u<<0)
 #define INIT_ED_KEY_REPLACE                     (1u<<1)
@@ -27,51 +38,18 @@ ed25519_keypair_t *ed_key_init_from_file(const char *fname, uint32_t flags,
                                          time_t now,
                                          time_t lifetime,
                                          uint8_t cert_type,
-                                         struct tor_cert_st **cert_out);
+                                         struct tor_cert_st **cert_out,
+                                         const or_options_t *options);
 ed25519_keypair_t *ed_key_new(const ed25519_keypair_t *signing_key,
                               uint32_t flags,
                               time_t now,
                               time_t lifetime,
                               uint8_t cert_type,
                               struct tor_cert_st **cert_out);
-const ed25519_public_key_t *get_master_identity_key(void);
-const ed25519_keypair_t *get_master_signing_keypair(void);
-const struct tor_cert_st *get_master_signing_key_cert(void);
-
-const ed25519_keypair_t *get_current_auth_keypair(void);
-const struct tor_cert_st *get_current_link_cert_cert(void);
-const struct tor_cert_st *get_current_auth_key_cert(void);
-
-void get_master_rsa_crosscert(const uint8_t **cert_out,
-                              size_t *size_out);
-
-struct tor_cert_st *make_ntor_onion_key_crosscert(
-                                  const curve25519_keypair_t *onion_key,
-                                  const ed25519_public_key_t *master_id_key,
-                                  time_t now, time_t lifetime,
-                                  int *sign_out);
-uint8_t *make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
-                                  const ed25519_public_key_t *master_id_key,
-                                  const crypto_pk_t *rsa_id_key,
-                                  int *len_out);
-
-int check_tap_onion_key_crosscert(const uint8_t *crosscert,
-                                  int crosscert_len,
-                                  const crypto_pk_t *onion_pkey,
-                                  const ed25519_public_key_t *master_id_pkey,
-                                  const uint8_t *rsa_id_digest);
-
-int load_ed_keys(const or_options_t *options, time_t now);
-int should_make_new_ed_keys(const or_options_t *options, const time_t now);
-
-int generate_ed_link_cert(const or_options_t *options, time_t now);
 
 int read_encrypted_secret_key(ed25519_secret_key_t *out,
                               const char *fname);
 int write_encrypted_secret_key(const ed25519_secret_key_t *out,
                                const char *fname);
 
-void routerkeys_free_all(void);
-
 #endif
-
diff --git a/src/feature/nodelist/authcert.c b/src/feature/nodelist/authcert.c
new file mode 100644
index 0000000000..7a065662a7
--- /dev/null
+++ b/src/feature/nodelist/authcert.c
@@ -0,0 +1,1208 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file authcert.c
+ * \brief Code to maintain directory authorities' certificates.
+ *
+ * Authority certificates are signed with authority identity keys; they
+ * are used to authenticate shorter-term authority signing keys. We
+ * fetch them when we find a consensus or a vote that has been signed
+ * with a signing key we don't recognize.  We cache them on disk and
+ * load them on startup.  Authority operators generate them with the
+ * "tor-gencert" utility.
+ */
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/policies.h"
+#include "feature/client/bridges.h"
+#include "feature/dirauth/authmode.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dircommon/fp_pair.h"
+#include "feature/dirparse/authcert_parse.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/node_select.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/routermode.h"
+
+#include "core/or/connection_st.h"
+#include "feature/dirclient/dir_server_st.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/nodelist/authority_cert_st.h"
+#include "feature/nodelist/document_signature_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/networkstatus_voter_info_st.h"
+#include "feature/nodelist/node_st.h"
+
+DECLARE_TYPED_DIGESTMAP_FNS(dsmap_, digest_ds_map_t, download_status_t)
+#define DSMAP_FOREACH(map, keyvar, valvar) \
+  DIGESTMAP_FOREACH(dsmap_to_digestmap(map), keyvar, download_status_t *, \
+                    valvar)
+#define dsmap_free(map, fn) MAP_FREE_AND_NULL(dsmap, (map), (fn))
+
+/* Forward declaration for cert_list_t */
+typedef struct cert_list_t cert_list_t;
+
+static void download_status_reset_by_sk_in_cl(cert_list_t *cl,
+                                              const char *digest);
+static int download_status_is_ready_by_sk_in_cl(cert_list_t *cl,
+                                                const char *digest,
+                                                time_t now);
+static void list_pending_fpsk_downloads(fp_pair_map_t *result);
+
+/** List of certificates for a single authority, and download status for
+ * latest certificate.
+ */
+struct cert_list_t {
+  /*
+   * The keys of download status map are cert->signing_key_digest for pending
+   * downloads by (identity digest/signing key digest) pair; functions such
+   * as authority_cert_get_by_digest() already assume these are unique.
+   */
+  struct digest_ds_map_t *dl_status_map;
+  /* There is also a dlstatus for the download by identity key only */
+  download_status_t dl_status_by_id;
+  smartlist_t *certs;
+};
+/** Map from v3 identity key digest to cert_list_t. */
+static digestmap_t *trusted_dir_certs = NULL;
+
+/** True iff any key certificate in at least one member of
+ * <b>trusted_dir_certs</b> has changed since we last flushed the
+ * certificates to disk. */
+static int trusted_dir_servers_certs_changed = 0;
+
+/** Initialise schedule, want_authority, and increment_on in the download
+ * status dlstatus, then call download_status_reset() on it.
+ * It is safe to call this function or download_status_reset() multiple times
+ * on a new dlstatus. But it should *not* be called after a dlstatus has been
+ * used to count download attempts or failures. */
+static void
+download_status_cert_init(download_status_t *dlstatus)
+{
+  dlstatus->schedule = DL_SCHED_CONSENSUS;
+  dlstatus->want_authority = DL_WANT_ANY_DIRSERVER;
+  dlstatus->increment_on = DL_SCHED_INCREMENT_FAILURE;
+  dlstatus->last_backoff_position = 0;
+  dlstatus->last_delay_used = 0;
+
+  /* Use the new schedule to set next_attempt_at */
+  download_status_reset(dlstatus);
+}
+
+/** Reset the download status of a specified element in a dsmap */
+static void
+download_status_reset_by_sk_in_cl(cert_list_t *cl, const char *digest)
+{
+  download_status_t *dlstatus = NULL;
+
+  tor_assert(cl);
+  tor_assert(digest);
+
+  /* Make sure we have a dsmap */
+  if (!(cl->dl_status_map)) {
+    cl->dl_status_map = dsmap_new();
+  }
+  /* Look for a download_status_t in the map with this digest */
+  dlstatus = dsmap_get(cl->dl_status_map, digest);
+  /* Got one? */
+  if (!dlstatus) {
+    /* Insert before we reset */
+    dlstatus = tor_malloc_zero(sizeof(*dlstatus));
+    dsmap_set(cl->dl_status_map, digest, dlstatus);
+    download_status_cert_init(dlstatus);
+  }
+  tor_assert(dlstatus);
+  /* Go ahead and reset it */
+  download_status_reset(dlstatus);
+}
+
+/**
+ * Return true if the download for this signing key digest in cl is ready
+ * to be re-attempted.
+ */
+static int
+download_status_is_ready_by_sk_in_cl(cert_list_t *cl,
+                                     const char *digest,
+                                     time_t now)
+{
+  int rv = 0;
+  download_status_t *dlstatus = NULL;
+
+  tor_assert(cl);
+  tor_assert(digest);
+
+  /* Make sure we have a dsmap */
+  if (!(cl->dl_status_map)) {
+    cl->dl_status_map = dsmap_new();
+  }
+  /* Look for a download_status_t in the map with this digest */
+  dlstatus = dsmap_get(cl->dl_status_map, digest);
+  /* Got one? */
+  if (dlstatus) {
+    /* Use download_status_is_ready() */
+    rv = download_status_is_ready(dlstatus, now);
+  } else {
+    /*
+     * If we don't know anything about it, return 1, since we haven't
+     * tried this one before.  We need to create a new entry here,
+     * too.
+     */
+    dlstatus = tor_malloc_zero(sizeof(*dlstatus));
+    download_status_cert_init(dlstatus);
+    dsmap_set(cl->dl_status_map, digest, dlstatus);
+    rv = 1;
+  }
+
+  return rv;
+}
+
+/** Helper: Return the cert_list_t for an authority whose authority ID is
+ * <b>id_digest</b>, allocating a new list if necessary. */
+static cert_list_t *
+get_cert_list(const char *id_digest)
+{
+  cert_list_t *cl;
+  if (!trusted_dir_certs)
+    trusted_dir_certs = digestmap_new();
+  cl = digestmap_get(trusted_dir_certs, id_digest);
+  if (!cl) {
+    cl = tor_malloc_zero(sizeof(cert_list_t));
+    download_status_cert_init(&cl->dl_status_by_id);
+    cl->certs = smartlist_new();
+    cl->dl_status_map = dsmap_new();
+    digestmap_set(trusted_dir_certs, id_digest, cl);
+  }
+  return cl;
+}
+
+/** Return a list of authority ID digests with potentially enumerable lists
+ * of download_status_t objects; used by controller GETINFO queries.
+ */
+
+MOCK_IMPL(smartlist_t *,
+list_authority_ids_with_downloads, (void))
+{
+  smartlist_t *ids = smartlist_new();
+  digestmap_iter_t *i;
+  const char *digest;
+  char *tmp;
+  void *cl;
+
+  if (trusted_dir_certs) {
+    for (i = digestmap_iter_init(trusted_dir_certs);
+         !(digestmap_iter_done(i));
+         i = digestmap_iter_next(trusted_dir_certs, i)) {
+      /*
+       * We always have at least dl_status_by_id to query, so no need to
+       * probe deeper than the existence of a cert_list_t.
+       */
+      digestmap_iter_get(i, &digest, &cl);
+      tmp = tor_malloc(DIGEST_LEN);
+      memcpy(tmp, digest, DIGEST_LEN);
+      smartlist_add(ids, tmp);
+    }
+  }
+  /* else definitely no downloads going since nothing even has a cert list */
+
+  return ids;
+}
+
+/** Given an authority ID digest, return a pointer to the default download
+ * status, or NULL if there is no such entry in trusted_dir_certs */
+
+MOCK_IMPL(download_status_t *,
+id_only_download_status_for_authority_id, (const char *digest))
+{
+  download_status_t *dl = NULL;
+  cert_list_t *cl;
+
+  if (trusted_dir_certs) {
+    cl = digestmap_get(trusted_dir_certs, digest);
+    if (cl) {
+      dl = &(cl->dl_status_by_id);
+    }
+  }
+
+  return dl;
+}
+
+/** Given an authority ID digest, return a smartlist of signing key digests
+ * for which download_status_t is potentially queryable, or NULL if no such
+ * authority ID digest is known. */
+
+MOCK_IMPL(smartlist_t *,
+list_sk_digests_for_authority_id, (const char *digest))
+{
+  smartlist_t *sks = NULL;
+  cert_list_t *cl;
+  dsmap_iter_t *i;
+  const char *sk_digest;
+  char *tmp;
+  download_status_t *dl;
+
+  if (trusted_dir_certs) {
+    cl = digestmap_get(trusted_dir_certs, digest);
+    if (cl) {
+      sks = smartlist_new();
+      if (cl->dl_status_map) {
+        for (i = dsmap_iter_init(cl->dl_status_map);
+             !(dsmap_iter_done(i));
+             i = dsmap_iter_next(cl->dl_status_map, i)) {
+          /* Pull the digest out and add it to the list */
+          dsmap_iter_get(i, &sk_digest, &dl);
+          tmp = tor_malloc(DIGEST_LEN);
+          memcpy(tmp, sk_digest, DIGEST_LEN);
+          smartlist_add(sks, tmp);
+        }
+      }
+    }
+  }
+
+  return sks;
+}
+
+/** Given an authority ID digest and a signing key digest, return the
+ * download_status_t or NULL if none exists. */
+
+MOCK_IMPL(download_status_t *,
+download_status_for_authority_id_and_sk,(const char *id_digest,
+                                         const char *sk_digest))
+{
+  download_status_t *dl = NULL;
+  cert_list_t *cl = NULL;
+
+  if (trusted_dir_certs) {
+    cl = digestmap_get(trusted_dir_certs, id_digest);
+    if (cl && cl->dl_status_map) {
+      dl = dsmap_get(cl->dl_status_map, sk_digest);
+    }
+  }
+
+  return dl;
+}
+
+#define cert_list_free(val) \
+  FREE_AND_NULL(cert_list_t, cert_list_free_, (val))
+
+/** Release all space held by a cert_list_t */
+static void
+cert_list_free_(cert_list_t *cl)
+{
+  if (!cl)
+    return;
+
+  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+                    authority_cert_free(cert));
+  smartlist_free(cl->certs);
+  dsmap_free(cl->dl_status_map, tor_free_);
+  tor_free(cl);
+}
+
+/** Wrapper for cert_list_free so we can pass it to digestmap_free */
+static void
+cert_list_free_void(void *cl)
+{
+  cert_list_free_(cl);
+}
+
+/** Reload the cached v3 key certificates from the cached-certs file in
+ * the data directory. Return 0 on success, -1 on failure. */
+int
+trusted_dirs_reload_certs(void)
+{
+  char *filename;
+  char *contents;
+  int r;
+
+  filename = get_cachedir_fname("cached-certs");
+  contents = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+  tor_free(filename);
+  if (!contents)
+    return 0;
+  r = trusted_dirs_load_certs_from_string(
+        contents,
+        TRUSTED_DIRS_CERTS_SRC_FROM_STORE, 1, NULL);
+  tor_free(contents);
+  return r;
+}
+
+/** Helper: return true iff we already have loaded the exact cert
+ * <b>cert</b>. */
+static inline int
+already_have_cert(authority_cert_t *cert)
+{
+  cert_list_t *cl = get_cert_list(cert->cache_info.identity_digest);
+
+  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, c,
+  {
+    if (tor_memeq(c->cache_info.signed_descriptor_digest,
+                cert->cache_info.signed_descriptor_digest,
+                DIGEST_LEN))
+      return 1;
+  });
+  return 0;
+}
+
+/** Load a bunch of new key certificates from the string <b>contents</b>.  If
+ * <b>source</b> is TRUSTED_DIRS_CERTS_SRC_FROM_STORE, the certificates are
+ * from the cache, and we don't need to flush them to disk.  If we are a
+ * dirauth loading our own cert, source is TRUSTED_DIRS_CERTS_SRC_SELF.
+ * Otherwise, source is download type: TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST
+ * or TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST.  If <b>flush</b> is true, we
+ * need to flush any changed certificates to disk now.  Return 0 on success,
+ * -1 if any certs fail to parse.
+ *
+ * If source_dir is non-NULL, it's the identity digest for a directory that
+ * we've just successfully retrieved certificates from, so try it first to
+ * fetch any missing certificates.
+ */
+int
+trusted_dirs_load_certs_from_string(const char *contents, int source,
+                                    int flush, const char *source_dir)
+{
+  dir_server_t *ds;
+  const char *s, *eos;
+  int failure_code = 0;
+  int from_store = (source == TRUSTED_DIRS_CERTS_SRC_FROM_STORE);
+  int added_trusted_cert = 0;
+
+  for (s = contents; *s; s = eos) {
+    authority_cert_t *cert = authority_cert_parse_from_string(s, &eos);
+    cert_list_t *cl;
+    if (!cert) {
+      failure_code = -1;
+      break;
+    }
+    ds = trusteddirserver_get_by_v3_auth_digest(
+                                       cert->cache_info.identity_digest);
+    log_debug(LD_DIR, "Parsed certificate for %s",
+              ds ? ds->nickname : "unknown authority");
+
+    if (already_have_cert(cert)) {
+      /* we already have this one. continue. */
+      log_info(LD_DIR, "Skipping %s certificate for %s that we "
+               "already have.",
+               from_store ? "cached" : "downloaded",
+               ds ? ds->nickname : "an old or new authority");
+
+      /*
+       * A duplicate on download should be treated as a failure, so we call
+       * authority_cert_dl_failed() to reset the download status to make sure
+       * we can't try again.  Since we've implemented the fp-sk mechanism
+       * to download certs by signing key, this should be much rarer than it
+       * was and is perhaps cause for concern.
+       */
+      if (!from_store) {
+        if (authdir_mode(get_options())) {
+          log_warn(LD_DIR,
+                   "Got a certificate for %s, but we already have it. "
+                   "Maybe they haven't updated it. Waiting for a while.",
+                   ds ? ds->nickname : "an old or new authority");
+        } else {
+          log_info(LD_DIR,
+                   "Got a certificate for %s, but we already have it. "
+                   "Maybe they haven't updated it. Waiting for a while.",
+                   ds ? ds->nickname : "an old or new authority");
+        }
+
+        /*
+         * This is where we care about the source; authority_cert_dl_failed()
+         * needs to know whether the download was by fp or (fp,sk) pair to
+         * twiddle the right bit in the download map.
+         */
+        if (source == TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST) {
+          authority_cert_dl_failed(cert->cache_info.identity_digest,
+                                   NULL, 404);
+        } else if (source == TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST) {
+          authority_cert_dl_failed(cert->cache_info.identity_digest,
+                                   cert->signing_key_digest, 404);
+        }
+      }
+
+      authority_cert_free(cert);
+      continue;
+    }
+
+    if (ds) {
+      added_trusted_cert = 1;
+      log_info(LD_DIR, "Adding %s certificate for directory authority %s with "
+               "signing key %s", from_store ? "cached" : "downloaded",
+               ds->nickname, hex_str(cert->signing_key_digest,DIGEST_LEN));
+    } else {
+      int adding = we_want_to_fetch_unknown_auth_certs(get_options());
+      log_info(LD_DIR, "%s %s certificate for unrecognized directory "
+               "authority with signing key %s",
+               adding ? "Adding" : "Not adding",
+               from_store ? "cached" : "downloaded",
+               hex_str(cert->signing_key_digest,DIGEST_LEN));
+      if (!adding) {
+        authority_cert_free(cert);
+        continue;
+      }
+    }
+
+    cl = get_cert_list(cert->cache_info.identity_digest);
+    smartlist_add(cl->certs, cert);
+    if (ds && cert->cache_info.published_on > ds->addr_current_at) {
+      /* Check to see whether we should update our view of the authority's
+       * address. */
+      if (cert->addr && cert->dir_port &&
+          (ds->addr != cert->addr ||
+           ds->dir_port != cert->dir_port)) {
+        char *a = tor_dup_ip(cert->addr);
+        log_notice(LD_DIR, "Updating address for directory authority %s "
+                   "from %s:%d to %s:%d based on certificate.",
+                   ds->nickname, ds->address, (int)ds->dir_port,
+                   a, cert->dir_port);
+        tor_free(a);
+        ds->addr = cert->addr;
+        ds->dir_port = cert->dir_port;
+      }
+      ds->addr_current_at = cert->cache_info.published_on;
+    }
+
+    if (!from_store)
+      trusted_dir_servers_certs_changed = 1;
+  }
+
+  if (flush)
+    trusted_dirs_flush_certs_to_disk();
+
+  /* call this even if failure_code is <0, since some certs might have
+   * succeeded, but only pass source_dir if there were no failures,
+   * and at least one more authority certificate was added to the store.
+   * This avoids retrying a directory that's serving bad or entirely duplicate
+   * certificates. */
+  if (failure_code == 0 && added_trusted_cert) {
+    networkstatus_note_certs_arrived(source_dir);
+  } else {
+    networkstatus_note_certs_arrived(NULL);
+  }
+
+  return failure_code;
+}
+
+/** Save all v3 key certificates to the cached-certs file. */
+void
+trusted_dirs_flush_certs_to_disk(void)
+{
+  char *filename;
+  smartlist_t *chunks;
+
+  if (!trusted_dir_servers_certs_changed || !trusted_dir_certs)
+    return;
+
+  chunks = smartlist_new();
+  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
+    SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+          {
+            sized_chunk_t *c = tor_malloc(sizeof(sized_chunk_t));
+            c->bytes = cert->cache_info.signed_descriptor_body;
+            c->len = cert->cache_info.signed_descriptor_len;
+            smartlist_add(chunks, c);
+          });
+  } DIGESTMAP_FOREACH_END;
+
+  filename = get_cachedir_fname("cached-certs");
+  if (write_chunks_to_file(filename, chunks, 0, 0)) {
+    log_warn(LD_FS, "Error writing certificates to disk.");
+  }
+  tor_free(filename);
+  SMARTLIST_FOREACH(chunks, sized_chunk_t *, c, tor_free(c));
+  smartlist_free(chunks);
+
+  trusted_dir_servers_certs_changed = 0;
+}
+
+static int
+compare_certs_by_pubdates(const void **_a, const void **_b)
+{
+  const authority_cert_t *cert1 = *_a, *cert2=*_b;
+
+  if (cert1->cache_info.published_on < cert2->cache_info.published_on)
+    return -1;
+  else if (cert1->cache_info.published_on >  cert2->cache_info.published_on)
+    return 1;
+  else
+    return 0;
+}
+
+/** Remove all expired v3 authority certificates that have been superseded for
+ * more than 48 hours or, if not expired, that were published more than 7 days
+ * before being superseded. (If the most recent cert was published more than 48
+ * hours ago, then we aren't going to get any consensuses signed with older
+ * keys.) */
+void
+trusted_dirs_remove_old_certs(void)
+{
+  time_t now = time(NULL);
+#define DEAD_CERT_LIFETIME (2*24*60*60)
+#define SUPERSEDED_CERT_LIFETIME (2*24*60*60)
+  if (!trusted_dir_certs)
+    return;
+
+  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
+    /* Sort the list from first-published to last-published */
+    smartlist_sort(cl->certs, compare_certs_by_pubdates);
+
+    SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) {
+      if (cert_sl_idx == smartlist_len(cl->certs) - 1) {
+        /* This is the most recently published cert.  Keep it. */
+        continue;
+      }
+      authority_cert_t *next_cert = smartlist_get(cl->certs, cert_sl_idx+1);
+      const time_t next_cert_published = next_cert->cache_info.published_on;
+      if (next_cert_published > now) {
+        /* All later certs are published in the future. Keep everything
+         * we didn't discard. */
+        break;
+      }
+      int should_remove = 0;
+      if (cert->expires + DEAD_CERT_LIFETIME < now) {
+        /* Certificate has been expired for at least DEAD_CERT_LIFETIME.
+         * Remove it. */
+        should_remove = 1;
+      } else if (next_cert_published + SUPERSEDED_CERT_LIFETIME < now) {
+        /* Certificate has been superseded for OLD_CERT_LIFETIME.
+         * Remove it.
+         */
+        should_remove = 1;
+      }
+      if (should_remove) {
+        SMARTLIST_DEL_CURRENT_KEEPORDER(cl->certs, cert);
+        authority_cert_free(cert);
+        trusted_dir_servers_certs_changed = 1;
+      }
+    } SMARTLIST_FOREACH_END(cert);
+
+  } DIGESTMAP_FOREACH_END;
+#undef DEAD_CERT_LIFETIME
+#undef OLD_CERT_LIFETIME
+
+  trusted_dirs_flush_certs_to_disk();
+}
+
+/** Return the newest v3 authority certificate whose v3 authority identity key
+ * has digest <b>id_digest</b>.  Return NULL if no such authority is known,
+ * or it has no certificate. */
+authority_cert_t *
+authority_cert_get_newest_by_id(const char *id_digest)
+{
+  cert_list_t *cl;
+  authority_cert_t *best = NULL;
+  if (!trusted_dir_certs ||
+      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
+    return NULL;
+
+  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+  {
+    if (!best || cert->cache_info.published_on > best->cache_info.published_on)
+      best = cert;
+  });
+  return best;
+}
+
+/** Return the newest v3 authority certificate whose directory signing key has
+ * digest <b>sk_digest</b>. Return NULL if no such certificate is known.
+ */
+authority_cert_t *
+authority_cert_get_by_sk_digest(const char *sk_digest)
+{
+  authority_cert_t *c;
+  if (!trusted_dir_certs)
+    return NULL;
+
+  if ((c = get_my_v3_authority_cert()) &&
+      tor_memeq(c->signing_key_digest, sk_digest, DIGEST_LEN))
+    return c;
+  if ((c = get_my_v3_legacy_cert()) &&
+      tor_memeq(c->signing_key_digest, sk_digest, DIGEST_LEN))
+    return c;
+
+  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
+    SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+    {
+      if (tor_memeq(cert->signing_key_digest, sk_digest, DIGEST_LEN))
+        return cert;
+    });
+  } DIGESTMAP_FOREACH_END;
+  return NULL;
+}
+
+/** Return the v3 authority certificate with signing key matching
+ * <b>sk_digest</b>, for the authority with identity digest <b>id_digest</b>.
+ * Return NULL if no such authority is known. */
+authority_cert_t *
+authority_cert_get_by_digests(const char *id_digest,
+                              const char *sk_digest)
+{
+  cert_list_t *cl;
+  if (!trusted_dir_certs ||
+      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
+    return NULL;
+  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+    if (tor_memeq(cert->signing_key_digest, sk_digest, DIGEST_LEN))
+      return cert; );
+
+  return NULL;
+}
+
+/** Add every known authority_cert_t to <b>certs_out</b>. */
+void
+authority_cert_get_all(smartlist_t *certs_out)
+{
+  tor_assert(certs_out);
+  if (!trusted_dir_certs)
+    return;
+
+  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
+    SMARTLIST_FOREACH(cl->certs, authority_cert_t *, c,
+                      smartlist_add(certs_out, c));
+  } DIGESTMAP_FOREACH_END;
+}
+
+/** Called when an attempt to download a certificate with the authority with
+ * ID <b>id_digest</b> and, if not NULL, signed with key signing_key_digest
+ * fails with HTTP response code <b>status</b>: remember the failure, so we
+ * don't try again immediately. */
+void
+authority_cert_dl_failed(const char *id_digest,
+                         const char *signing_key_digest, int status)
+{
+  cert_list_t *cl;
+  download_status_t *dlstatus = NULL;
+  char id_digest_str[2*DIGEST_LEN+1];
+  char sk_digest_str[2*DIGEST_LEN+1];
+
+  if (!trusted_dir_certs ||
+      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
+    return;
+
+  /*
+   * Are we noting a failed download of the latest cert for the id digest,
+   * or of a download by (id, signing key) digest pair?
+   */
+  if (!signing_key_digest) {
+    /* Just by id digest */
+    download_status_failed(&cl->dl_status_by_id, status);
+  } else {
+    /* Reset by (id, signing key) digest pair
+     *
+     * Look for a download_status_t in the map with this digest
+     */
+    dlstatus = dsmap_get(cl->dl_status_map, signing_key_digest);
+    /* Got one? */
+    if (dlstatus) {
+      download_status_failed(dlstatus, status);
+    } else {
+      /*
+       * Do this rather than hex_str(), since hex_str clobbers
+       * old results and we call twice in the param list.
+       */
+      base16_encode(id_digest_str, sizeof(id_digest_str),
+                    id_digest, DIGEST_LEN);
+      base16_encode(sk_digest_str, sizeof(sk_digest_str),
+                    signing_key_digest, DIGEST_LEN);
+      log_warn(LD_BUG,
+               "Got failure for cert fetch with (fp,sk) = (%s,%s), with "
+               "status %d, but knew nothing about the download.",
+               id_digest_str, sk_digest_str, status);
+    }
+  }
+}
+
+static const char *BAD_SIGNING_KEYS[] = {
+  "09CD84F751FD6E955E0F8ADB497D5401470D697E", // Expires 2015-01-11 16:26:31
+  "0E7E9C07F0969D0468AD741E172A6109DC289F3C", // Expires 2014-08-12 10:18:26
+  "57B85409891D3FB32137F642FDEDF8B7F8CDFDCD", // Expires 2015-02-11 17:19:09
+  "87326329007AF781F587AF5B594E540B2B6C7630", // Expires 2014-07-17 11:10:09
+  "98CC82342DE8D298CF99D3F1A396475901E0D38E", // Expires 2014-11-10 13:18:56
+  "9904B52336713A5ADCB13E4FB14DC919E0D45571", // Expires 2014-04-20 20:01:01
+  "9DCD8E3F1DD1597E2AD476BBA28A1A89F3095227", // Expires 2015-01-16 03:52:30
+  "A61682F34B9BB9694AC98491FE1ABBFE61923941", // Expires 2014-06-11 09:25:09
+  "B59F6E99C575113650C99F1C425BA7B20A8C071D", // Expires 2014-07-31 13:22:10
+  "D27178388FA75B96D37FA36E0B015227DDDBDA51", // Expires 2014-08-04 04:01:57
+  NULL,
+};
+
+/** Return true iff <b>cert</b> authenticates some atuhority signing key
+ * which, because of the old openssl heartbleed vulnerability, should
+ * never be trusted. */
+int
+authority_cert_is_blacklisted(const authority_cert_t *cert)
+{
+  char hex_digest[HEX_DIGEST_LEN+1];
+  int i;
+  base16_encode(hex_digest, sizeof(hex_digest),
+                cert->signing_key_digest, sizeof(cert->signing_key_digest));
+
+  for (i = 0; BAD_SIGNING_KEYS[i]; ++i) {
+    if (!strcasecmp(hex_digest, BAD_SIGNING_KEYS[i])) {
+      return 1;
+    }
+  }
+  return 0;
+}
+
+/** Return true iff when we've been getting enough failures when trying to
+ * download the certificate with ID digest <b>id_digest</b> that we're willing
+ * to start bugging the user about it. */
+int
+authority_cert_dl_looks_uncertain(const char *id_digest)
+{
+#define N_AUTH_CERT_DL_FAILURES_TO_BUG_USER 2
+  cert_list_t *cl;
+  int n_failures;
+  if (!trusted_dir_certs ||
+      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
+    return 0;
+
+  n_failures = download_status_get_n_failures(&cl->dl_status_by_id);
+  return n_failures >= N_AUTH_CERT_DL_FAILURES_TO_BUG_USER;
+}
+
+/* Fetch the authority certificates specified in resource.
+ * If we are a bridge client, and node is a configured bridge, fetch from node
+ * using dir_hint as the fingerprint. Otherwise, if rs is not NULL, fetch from
+ * rs. Otherwise, fetch from a random directory mirror. */
+static void
+authority_certs_fetch_resource_impl(const char *resource,
+                                    const char *dir_hint,
+                                    const node_t *node,
+                                    const routerstatus_t *rs)
+{
+  const or_options_t *options = get_options();
+  int get_via_tor = purpose_needs_anonymity(DIR_PURPOSE_FETCH_CERTIFICATE, 0,
+                                            resource);
+
+  /* Make sure bridge clients never connect to anything but a bridge */
+  if (options->UseBridges) {
+    if (node && !node_is_a_configured_bridge(node)) {
+      /* If we're using bridges, and node is not a bridge, use a 3-hop path. */
+      get_via_tor = 1;
+    } else if (!node) {
+      /* If we're using bridges, and there's no node, use a 3-hop path. */
+      get_via_tor = 1;
+    }
+  }
+
+  const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS
+                                                    : DIRIND_ONEHOP;
+
+  directory_request_t *req = NULL;
+  /* If we've just downloaded a consensus from a bridge, re-use that
+   * bridge */
+  if (options->UseBridges && node && node->ri && !get_via_tor) {
+    /* clients always make OR connections to bridges */
+    tor_addr_port_t or_ap;
+    /* we are willing to use a non-preferred address if we need to */
+    fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0,
+                                         &or_ap);
+
+    req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE);
+    directory_request_set_or_addr_port(req, &or_ap);
+    if (dir_hint)
+      directory_request_set_directory_id_digest(req, dir_hint);
+  } else if (rs) {
+    /* And if we've just downloaded a consensus from a directory, re-use that
+     * directory */
+    req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE);
+    directory_request_set_routerstatus(req, rs);
+  }
+
+  if (req) {
+    /* We've set up a request object -- fill in the other request fields, and
+     * send the request.  */
+    directory_request_set_indirection(req, indirection);
+    directory_request_set_resource(req, resource);
+    directory_initiate_request(req);
+    directory_request_free(req);
+    return;
+  }
+
+  /* Otherwise, we want certs from a random fallback or directory
+   * mirror, because they will almost always succeed. */
+  directory_get_from_dirserver(DIR_PURPOSE_FETCH_CERTIFICATE, 0,
+                               resource, PDS_RETRY_IF_NO_SERVERS,
+                               DL_WANT_ANY_DIRSERVER);
+}
+
+/** Try to download any v3 authority certificates that we may be missing.  If
+ * <b>status</b> is provided, try to get all the ones that were used to sign
+ * <b>status</b>.  Additionally, try to have a non-expired certificate for
+ * every V3 authority in trusted_dir_servers.  Don't fetch certificates we
+ * already have.
+ *
+ * If dir_hint is non-NULL, it's the identity digest for a directory that
+ * we've just successfully retrieved a consensus or certificates from, so try
+ * it first to fetch any missing certificates.
+ **/
+void
+authority_certs_fetch_missing(networkstatus_t *status, time_t now,
+                              const char *dir_hint)
+{
+  /*
+   * The pending_id digestmap tracks pending certificate downloads by
+   * identity digest; the pending_cert digestmap tracks pending downloads
+   * by (identity digest, signing key digest) pairs.
+   */
+  digestmap_t *pending_id;
+  fp_pair_map_t *pending_cert;
+  /*
+   * The missing_id_digests smartlist will hold a list of id digests
+   * we want to fetch the newest cert for; the missing_cert_digests
+   * smartlist will hold a list of fp_pair_t with an identity and
+   * signing key digest.
+   */
+  smartlist_t *missing_cert_digests, *missing_id_digests;
+  char *resource = NULL;
+  cert_list_t *cl;
+  const or_options_t *options = get_options();
+  const int keep_unknown = we_want_to_fetch_unknown_auth_certs(options);
+  fp_pair_t *fp_tmp = NULL;
+  char id_digest_str[2*DIGEST_LEN+1];
+  char sk_digest_str[2*DIGEST_LEN+1];
+
+  if (should_delay_dir_fetches(options, NULL))
+    return;
+
+  pending_cert = fp_pair_map_new();
+  pending_id = digestmap_new();
+  missing_cert_digests = smartlist_new();
+  missing_id_digests = smartlist_new();
+
+  /*
+   * First, we get the lists of already pending downloads so we don't
+   * duplicate effort.
+   */
+  list_pending_downloads(pending_id, NULL,
+                         DIR_PURPOSE_FETCH_CERTIFICATE, "fp/");
+  list_pending_fpsk_downloads(pending_cert);
+
+  /*
+   * Now, we download any trusted authority certs we don't have by
+   * identity digest only.  This gets the latest cert for that authority.
+   */
+  SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
+                          dir_server_t *, ds) {
+    int found = 0;
+    if (!(ds->type & V3_DIRINFO))
+      continue;
+    if (smartlist_contains_digest(missing_id_digests,
+                                  ds->v3_identity_digest))
+      continue;
+    cl = get_cert_list(ds->v3_identity_digest);
+    SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) {
+      if (now < cert->expires) {
+        /* It's not expired, and we weren't looking for something to
+         * verify a consensus with.  Call it done. */
+        download_status_reset(&(cl->dl_status_by_id));
+        /* No sense trying to download it specifically by signing key hash */
+        download_status_reset_by_sk_in_cl(cl, cert->signing_key_digest);
+        found = 1;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(cert);
+    if (!found &&
+        download_status_is_ready(&(cl->dl_status_by_id), now) &&
+        !digestmap_get(pending_id, ds->v3_identity_digest)) {
+      log_info(LD_DIR,
+               "No current certificate known for authority %s "
+               "(ID digest %s); launching request.",
+               ds->nickname, hex_str(ds->v3_identity_digest, DIGEST_LEN));
+      smartlist_add(missing_id_digests, ds->v3_identity_digest);
+    }
+  } SMARTLIST_FOREACH_END(ds);
+
+  /*
+   * Next, if we have a consensus, scan through it and look for anything
+   * signed with a key from a cert we don't have.  Those get downloaded
+   * by (fp,sk) pair, but if we don't know any certs at all for the fp
+   * (identity digest), and it's one of the trusted dir server certs
+   * we started off above or a pending download in pending_id, don't
+   * try to get it yet.  Most likely, the one we'll get for that will
+   * have the right signing key too, and we'd just be downloading
+   * redundantly.
+   */
+  if (status) {
+    SMARTLIST_FOREACH_BEGIN(status->voters, networkstatus_voter_info_t *,
+                            voter) {
+      if (!smartlist_len(voter->sigs))
+        continue; /* This authority never signed this consensus, so don't
+                   * go looking for a cert with key digest 0000000000. */
+      if (!keep_unknown &&
+          !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest))
+        continue; /* We don't want unknown certs, and we don't know this
+                   * authority.*/
+
+      /*
+       * If we don't know *any* cert for this authority, and a download by ID
+       * is pending or we added it to missing_id_digests above, skip this
+       * one for now to avoid duplicate downloads.
+       */
+      cl = get_cert_list(voter->identity_digest);
+      if (smartlist_len(cl->certs) == 0) {
+        /* We have no certs at all for this one */
+
+        /* Do we have a download of one pending? */
+        if (digestmap_get(pending_id, voter->identity_digest))
+          continue;
+
+        /*
+         * Are we about to launch a download of one due to the trusted
+         * dir server check above?
+         */
+        if (smartlist_contains_digest(missing_id_digests,
+                                      voter->identity_digest))
+          continue;
+      }
+
+      SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) {
+        authority_cert_t *cert =
+          authority_cert_get_by_digests(voter->identity_digest,
+                                        sig->signing_key_digest);
+        if (cert) {
+          if (now < cert->expires)
+            download_status_reset_by_sk_in_cl(cl, sig->signing_key_digest);
+          continue;
+        }
+        if (download_status_is_ready_by_sk_in_cl(
+              cl, sig->signing_key_digest, now) &&
+            !fp_pair_map_get_by_digests(pending_cert,
+                                        voter->identity_digest,
+                                        sig->signing_key_digest)) {
+          /*
+           * Do this rather than hex_str(), since hex_str clobbers
+           * old results and we call twice in the param list.
+           */
+          base16_encode(id_digest_str, sizeof(id_digest_str),
+                        voter->identity_digest, DIGEST_LEN);
+          base16_encode(sk_digest_str, sizeof(sk_digest_str),
+                        sig->signing_key_digest, DIGEST_LEN);
+
+          if (voter->nickname) {
+            log_info(LD_DIR,
+                     "We're missing a certificate from authority %s "
+                     "(ID digest %s) with signing key %s: "
+                     "launching request.",
+                     voter->nickname, id_digest_str, sk_digest_str);
+          } else {
+            log_info(LD_DIR,
+                     "We're missing a certificate from authority ID digest "
+                     "%s with signing key %s: launching request.",
+                     id_digest_str, sk_digest_str);
+          }
+
+          /* Allocate a new fp_pair_t to append */
+          fp_tmp = tor_malloc(sizeof(*fp_tmp));
+          memcpy(fp_tmp->first, voter->identity_digest, sizeof(fp_tmp->first));
+          memcpy(fp_tmp->second, sig->signing_key_digest,
+                 sizeof(fp_tmp->second));
+          smartlist_add(missing_cert_digests, fp_tmp);
+        }
+      } SMARTLIST_FOREACH_END(sig);
+    } SMARTLIST_FOREACH_END(voter);
+  }
+
+  /* Bridge clients look up the node for the dir_hint */
+  const node_t *node = NULL;
+  /* All clients, including bridge clients, look up the routerstatus for the
+   * dir_hint */
+  const routerstatus_t *rs = NULL;
+
+  /* If we still need certificates, try the directory that just successfully
+   * served us a consensus or certificates.
+   * As soon as the directory fails to provide additional certificates, we try
+   * another, randomly selected directory. This avoids continual retries.
+   * (We only ever have one outstanding request per certificate.)
+   */
+  if (dir_hint) {
+    if (options->UseBridges) {
+      /* Bridge clients try the nodelist. If the dir_hint is from an authority,
+       * or something else fetched over tor, we won't find the node here, but
+       * we will find the rs. */
+      node = node_get_by_id(dir_hint);
+    }
+
+    /* All clients try the consensus routerstatus, then the fallback
+     * routerstatus */
+    rs = router_get_consensus_status_by_id(dir_hint);
+    if (!rs) {
+      /* This will also find authorities */
+      const dir_server_t *ds = router_get_fallback_dirserver_by_digest(
+                                                                    dir_hint);
+      if (ds) {
+        rs = &ds->fake_status;
+      }
+    }
+
+    if (!node && !rs) {
+      log_warn(LD_BUG, "Directory %s delivered a consensus, but %s"
+               "no routerstatus could be found for it.",
+               options->UseBridges ? "no node and " : "",
+               hex_str(dir_hint, DIGEST_LEN));
+    }
+  }
+
+  /* Do downloads by identity digest */
+  if (smartlist_len(missing_id_digests) > 0) {
+    int need_plus = 0;
+    smartlist_t *fps = smartlist_new();
+
+    smartlist_add_strdup(fps, "fp/");
+
+    SMARTLIST_FOREACH_BEGIN(missing_id_digests, const char *, d) {
+      char *fp = NULL;
+
+      if (digestmap_get(pending_id, d))
+        continue;
+
+      base16_encode(id_digest_str, sizeof(id_digest_str),
+                    d, DIGEST_LEN);
+
+      if (need_plus) {
+        tor_asprintf(&fp, "+%s", id_digest_str);
+      } else {
+        /* No need for tor_asprintf() in this case; first one gets no '+' */
+        fp = tor_strdup(id_digest_str);
+        need_plus = 1;
+      }
+
+      smartlist_add(fps, fp);
+    } SMARTLIST_FOREACH_END(d);
+
+    if (smartlist_len(fps) > 1) {
+      resource = smartlist_join_strings(fps, "", 0, NULL);
+      /* node and rs are directories that just gave us a consensus or
+       * certificates */
+      authority_certs_fetch_resource_impl(resource, dir_hint, node, rs);
+      tor_free(resource);
+    }
+    /* else we didn't add any: they were all pending */
+
+    SMARTLIST_FOREACH(fps, char *, cp, tor_free(cp));
+    smartlist_free(fps);
+  }
+
+  /* Do downloads by identity digest/signing key pair */
+  if (smartlist_len(missing_cert_digests) > 0) {
+    int need_plus = 0;
+    smartlist_t *fp_pairs = smartlist_new();
+
+    smartlist_add_strdup(fp_pairs, "fp-sk/");
+
+    SMARTLIST_FOREACH_BEGIN(missing_cert_digests, const fp_pair_t *, d) {
+      char *fp_pair = NULL;
+
+      if (fp_pair_map_get(pending_cert, d))
+        continue;
+
+      /* Construct string encodings of the digests */
+      base16_encode(id_digest_str, sizeof(id_digest_str),
+                    d->first, DIGEST_LEN);
+      base16_encode(sk_digest_str, sizeof(sk_digest_str),
+                    d->second, DIGEST_LEN);
+
+      /* Now tor_asprintf() */
+      if (need_plus) {
+        tor_asprintf(&fp_pair, "+%s-%s", id_digest_str, sk_digest_str);
+      } else {
+        /* First one in the list doesn't get a '+' */
+        tor_asprintf(&fp_pair, "%s-%s", id_digest_str, sk_digest_str);
+        need_plus = 1;
+      }
+
+      /* Add it to the list of pairs to request */
+      smartlist_add(fp_pairs, fp_pair);
+    } SMARTLIST_FOREACH_END(d);
+
+    if (smartlist_len(fp_pairs) > 1) {
+      resource = smartlist_join_strings(fp_pairs, "", 0, NULL);
+      /* node and rs are directories that just gave us a consensus or
+       * certificates */
+      authority_certs_fetch_resource_impl(resource, dir_hint, node, rs);
+      tor_free(resource);
+    }
+    /* else they were all pending */
+
+    SMARTLIST_FOREACH(fp_pairs, char *, p, tor_free(p));
+    smartlist_free(fp_pairs);
+  }
+
+  smartlist_free(missing_id_digests);
+  SMARTLIST_FOREACH(missing_cert_digests, fp_pair_t *, p, tor_free(p));
+  smartlist_free(missing_cert_digests);
+  digestmap_free(pending_id, NULL);
+  fp_pair_map_free(pending_cert, NULL);
+}
+
+void
+authcert_free_all(void)
+{
+  if (trusted_dir_certs) {
+    digestmap_free(trusted_dir_certs, cert_list_free_void);
+    trusted_dir_certs = NULL;
+  }
+}
+
+/** Free storage held in <b>cert</b>. */
+void
+authority_cert_free_(authority_cert_t *cert)
+{
+  if (!cert)
+    return;
+
+  tor_free(cert->cache_info.signed_descriptor_body);
+  crypto_pk_free(cert->signing_key);
+  crypto_pk_free(cert->identity_key);
+
+  tor_free(cert);
+}
+
+/** For every certificate we are currently downloading by (identity digest,
+ * signing key digest) pair, set result[fp_pair] to (void *1).
+ */
+static void
+list_pending_fpsk_downloads(fp_pair_map_t *result)
+{
+  const char *pfx = "fp-sk/";
+  smartlist_t *tmp;
+  smartlist_t *conns;
+  const char *resource;
+
+  tor_assert(result);
+
+  tmp = smartlist_new();
+  conns = get_connection_array();
+
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
+    if (conn->type == CONN_TYPE_DIR &&
+        conn->purpose == DIR_PURPOSE_FETCH_CERTIFICATE &&
+        !conn->marked_for_close) {
+      resource = TO_DIR_CONN(conn)->requested_resource;
+      if (!strcmpstart(resource, pfx))
+        dir_split_resource_into_fingerprint_pairs(resource + strlen(pfx),
+                                                  tmp);
+    }
+  } SMARTLIST_FOREACH_END(conn);
+
+  SMARTLIST_FOREACH_BEGIN(tmp, fp_pair_t *, fp) {
+    fp_pair_map_set(result, fp, (void*)1);
+    tor_free(fp);
+  } SMARTLIST_FOREACH_END(fp);
+
+  smartlist_free(tmp);
+}
diff --git a/src/feature/nodelist/authcert.h b/src/feature/nodelist/authcert.h
new file mode 100644
index 0000000000..2effdb06e6
--- /dev/null
+++ b/src/feature/nodelist/authcert.h
@@ -0,0 +1,60 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file authcert.h
+ * \brief Header file for authcert.c
+ **/
+
+#ifndef TOR_AUTHCERT_H
+#define TOR_AUTHCERT_H
+
+#include "lib/testsupport/testsupport.h"
+
+int trusted_dirs_reload_certs(void);
+
+/*
+ * Pass one of these as source to trusted_dirs_load_certs_from_string()
+ * to indicate whence string originates; this controls error handling
+ * behavior such as marking downloads as failed.
+ */
+
+#define TRUSTED_DIRS_CERTS_SRC_SELF 0
+#define TRUSTED_DIRS_CERTS_SRC_FROM_STORE 1
+#define TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST 2
+#define TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST 3
+#define TRUSTED_DIRS_CERTS_SRC_FROM_VOTE 4
+
+int trusted_dirs_load_certs_from_string(const char *contents, int source,
+                                        int flush, const char *source_dir);
+void trusted_dirs_remove_old_certs(void);
+void trusted_dirs_flush_certs_to_disk(void);
+authority_cert_t *authority_cert_get_newest_by_id(const char *id_digest);
+authority_cert_t *authority_cert_get_by_sk_digest(const char *sk_digest);
+authority_cert_t *authority_cert_get_by_digests(const char *id_digest,
+                                                const char *sk_digest);
+void authority_cert_get_all(smartlist_t *certs_out);
+void authority_cert_dl_failed(const char *id_digest,
+                              const char *signing_key_digest, int status);
+void authority_certs_fetch_missing(networkstatus_t *status, time_t now,
+                                   const char *dir_hint);
+int authority_cert_dl_looks_uncertain(const char *id_digest);
+int authority_cert_is_blacklisted(const authority_cert_t *cert);
+
+void authority_cert_free_(authority_cert_t *cert);
+#define authority_cert_free(cert) \
+  FREE_AND_NULL(authority_cert_t, authority_cert_free_, (cert))
+
+MOCK_DECL(smartlist_t *, list_authority_ids_with_downloads, (void));
+MOCK_DECL(download_status_t *, id_only_download_status_for_authority_id,
+          (const char *digest));
+MOCK_DECL(smartlist_t *, list_sk_digests_for_authority_id,
+          (const char *digest));
+MOCK_DECL(download_status_t *, download_status_for_authority_id_and_sk,
+    (const char *id_digest, const char *sk_digest));
+
+void authcert_free_all(void);
+
+#endif
diff --git a/src/feature/nodelist/authority_cert_st.h b/src/feature/nodelist/authority_cert_st.h
new file mode 100644
index 0000000000..68a84bc452
--- /dev/null
+++ b/src/feature/nodelist/authority_cert_st.h
@@ -0,0 +1,32 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef AUTHORITY_CERT_ST_H
+#define AUTHORITY_CERT_ST_H
+
+#include "feature/nodelist/signed_descriptor_st.h"
+
+/** Certificate for v3 directory protocol: binds long-term authority identity
+ * keys to medium-term authority signing keys. */
+struct authority_cert_t {
+  /** Information relating to caching this cert on disk and looking it up. */
+  signed_descriptor_t cache_info;
+  /** This authority's long-term authority identity key. */
+  crypto_pk_t *identity_key;
+  /** This authority's medium-term signing key. */
+  crypto_pk_t *signing_key;
+  /** The digest of <b>signing_key</b> */
+  char signing_key_digest[DIGEST_LEN];
+  /** The listed expiration time of this certificate. */
+  time_t expires;
+  /** This authority's IPv4 address, in host order. */
+  uint32_t addr;
+  /** This authority's directory port. */
+  uint16_t dir_port;
+};
+
+#endif
+
diff --git a/src/feature/nodelist/desc_store_st.h b/src/feature/nodelist/desc_store_st.h
new file mode 100644
index 0000000000..b04a1abc7d
--- /dev/null
+++ b/src/feature/nodelist/desc_store_st.h
@@ -0,0 +1,39 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef DESC_STORE_ST_H
+#define DESC_STORE_ST_H
+
+/** Allowable types of desc_store_t. */
+typedef enum store_type_t {
+  ROUTER_STORE = 0,
+  EXTRAINFO_STORE = 1
+} store_type_t;
+
+/** A 'store' is a set of descriptors saved on disk, with accompanying
+ * journal, mmaped as needed, rebuilt as needed. */
+struct desc_store_t {
+  /** Filename (within DataDir) for the store.  We append .tmp to this
+   * filename for a temporary file when rebuilding the store, and .new to this
+   * filename for the journal. */
+  const char *fname_base;
+  /** Human-readable description of what this store contains. */
+  const char *description;
+
+  tor_mmap_t *mmap; /**< A mmap for the main file in the store. */
+
+  store_type_t type; /**< What's stored in this store? */
+
+  /** The size of the router log, in bytes. */
+  size_t journal_len;
+  /** The size of the router store, in bytes. */
+  size_t store_len;
+  /** Total bytes dropped since last rebuild: this is space currently
+   * used in the cache and the journal that could be freed by a rebuild. */
+  size_t bytes_dropped;
+};
+
+#endif
diff --git a/src/feature/nodelist/describe.c b/src/feature/nodelist/describe.c
new file mode 100644
index 0000000000..5c376408c0
--- /dev/null
+++ b/src/feature/nodelist/describe.c
@@ -0,0 +1,183 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file describe.c
+ * \brief Format short descriptions of relays.
+ */
+
+#include "core/or/or.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/routerinfo.h"
+
+#include "core/or/extend_info_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+/**
+ * Longest allowed output of format_node_description, plus 1 character for
+ * NUL.  This allows space for:
+ * "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF~xxxxxxxxxxxxxxxxxxx at"
+ * " [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]"
+ * plus a terminating NUL.
+ */
+#define NODE_DESC_BUF_LEN (MAX_VERBOSE_NICKNAME_LEN+4+TOR_ADDR_BUF_LEN)
+
+/** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to
+ * hold a human-readable description of a node with identity digest
+ * <b>id_digest</b>, named-status <b>is_named</b>, nickname <b>nickname</b>,
+ * and address <b>addr</b> or <b>addr32h</b>.
+ *
+ * The <b>nickname</b> and <b>addr</b> fields are optional and may be set to
+ * NULL.  The <b>addr32h</b> field is optional and may be set to 0.
+ *
+ * Return a pointer to the front of <b>buf</b>.
+ */
+static const char *
+format_node_description(char *buf,
+                        const char *id_digest,
+                        int is_named,
+                        const char *nickname,
+                        const tor_addr_t *addr,
+                        uint32_t addr32h)
+{
+  char *cp;
+
+  if (!buf)
+    return "<NULL BUFFER>";
+
+  buf[0] = '$';
+  base16_encode(buf+1, HEX_DIGEST_LEN+1, id_digest, DIGEST_LEN);
+  cp = buf+1+HEX_DIGEST_LEN;
+  if (nickname) {
+    buf[1+HEX_DIGEST_LEN] = is_named ? '=' : '~';
+    strlcpy(buf+1+HEX_DIGEST_LEN+1, nickname, MAX_NICKNAME_LEN+1);
+    cp += strlen(cp);
+  }
+  if (addr32h || addr) {
+    memcpy(cp, " at ", 4);
+    cp += 4;
+    if (addr) {
+      tor_addr_to_str(cp, addr, TOR_ADDR_BUF_LEN, 0);
+    } else {
+      struct in_addr in;
+      in.s_addr = htonl(addr32h);
+      tor_inet_ntoa(&in, cp, INET_NTOA_BUF_LEN);
+    }
+  }
+  return buf;
+}
+
+/** Return a human-readable description of the routerinfo_t <b>ri</b>.
+ *
+ * This function is not thread-safe.  Each call to this function invalidates
+ * previous values returned by this function.
+ */
+const char *
+router_describe(const routerinfo_t *ri)
+{
+  static char buf[NODE_DESC_BUF_LEN];
+
+  if (!ri)
+    return "<null>";
+  return format_node_description(buf,
+                                 ri->cache_info.identity_digest,
+                                 0,
+                                 ri->nickname,
+                                 NULL,
+                                 ri->addr);
+}
+
+/** Return a human-readable description of the node_t <b>node</b>.
+ *
+ * This function is not thread-safe.  Each call to this function invalidates
+ * previous values returned by this function.
+ */
+const char *
+node_describe(const node_t *node)
+{
+  static char buf[NODE_DESC_BUF_LEN];
+  const char *nickname = NULL;
+  uint32_t addr32h = 0;
+  int is_named = 0;
+
+  if (!node)
+    return "<null>";
+
+  if (node->rs) {
+    nickname = node->rs->nickname;
+    is_named = node->rs->is_named;
+    addr32h = node->rs->addr;
+  } else if (node->ri) {
+    nickname = node->ri->nickname;
+    addr32h = node->ri->addr;
+  }
+
+  return format_node_description(buf,
+                                 node->identity,
+                                 is_named,
+                                 nickname,
+                                 NULL,
+                                 addr32h);
+}
+
+/** Return a human-readable description of the routerstatus_t <b>rs</b>.
+ *
+ * This function is not thread-safe.  Each call to this function invalidates
+ * previous values returned by this function.
+ */
+const char *
+routerstatus_describe(const routerstatus_t *rs)
+{
+  static char buf[NODE_DESC_BUF_LEN];
+
+  if (!rs)
+    return "<null>";
+  return format_node_description(buf,
+                                 rs->identity_digest,
+                                 rs->is_named,
+                                 rs->nickname,
+                                 NULL,
+                                 rs->addr);
+}
+
+/** Return a human-readable description of the extend_info_t <b>ei</b>.
+ *
+ * This function is not thread-safe.  Each call to this function invalidates
+ * previous values returned by this function.
+ */
+const char *
+extend_info_describe(const extend_info_t *ei)
+{
+  static char buf[NODE_DESC_BUF_LEN];
+
+  if (!ei)
+    return "<null>";
+  return format_node_description(buf,
+                                 ei->identity_digest,
+                                 0,
+                                 ei->nickname,
+                                 &ei->addr,
+                                 0);
+}
+
+/** Set <b>buf</b> (which must have MAX_VERBOSE_NICKNAME_LEN+1 bytes) to the
+ * verbose representation of the identity of <b>router</b>.  The format is:
+ *  A dollar sign.
+ *  The upper-case hexadecimal encoding of the SHA1 hash of router's identity.
+ *  A "=" if the router is named (no longer implemented); a "~" if it is not.
+ *  The router's nickname.
+ **/
+void
+router_get_verbose_nickname(char *buf, const routerinfo_t *router)
+{
+  buf[0] = '$';
+  base16_encode(buf+1, HEX_DIGEST_LEN+1, router->cache_info.identity_digest,
+                DIGEST_LEN);
+  buf[1+HEX_DIGEST_LEN] = '~';
+  strlcpy(buf+1+HEX_DIGEST_LEN+1, router->nickname, MAX_NICKNAME_LEN+1);
+}
diff --git a/src/feature/nodelist/describe.h b/src/feature/nodelist/describe.h
new file mode 100644
index 0000000000..018af6470e
--- /dev/null
+++ b/src/feature/nodelist/describe.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file describe.h
+ * \brief Header file for describe.c.
+ **/
+
+#ifndef TOR_DESCRIBE_H
+#define TOR_DESCRIBE_H
+
+struct extend_info_t;
+struct node_t;
+struct routerinfo_t;
+struct routerstatus_t;
+
+const char *extend_info_describe(const struct extend_info_t *ei);
+const char *node_describe(const struct node_t *node);
+const char *router_describe(const struct routerinfo_t *ri);
+const char *routerstatus_describe(const struct routerstatus_t *ri);
+
+#endif
diff --git a/src/feature/nodelist/dirlist.c b/src/feature/nodelist/dirlist.c
new file mode 100644
index 0000000000..93baa6e4e0
--- /dev/null
+++ b/src/feature/nodelist/dirlist.c
@@ -0,0 +1,422 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dirlist.c
+ * \brief Code to maintain our lists of directory authorities and
+ *    fallback directories.
+ *
+ * For the directory authorities, we have a list containing the public
+ * identity key, and contact points, for each authority.  The
+ * authorities receive descriptors from relays, and publish consensuses,
+ * descriptors, and microdescriptors.  This list is pre-configured.
+ *
+ * Fallback directories are well-known, stable, but untrusted directory
+ * caches that clients which have not yet bootstrapped can use to get
+ * their first networkstatus consensus, in order to find out where the
+ * Tor network really is.  This list is pre-configured in
+ * fallback_dirs.inc.  Every authority also serves as a fallback.
+ *
+ * Both fallback directories and directory authorities are are
+ * represented by a dir_server_t.
+ */
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/or/policies.h"
+#include "feature/control/control.h"
+#include "feature/dirauth/authmode.h"
+#include "feature/dircommon/directory.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/relay/router.h"
+#include "lib/net/resolve.h"
+
+#include "feature/dirclient/dir_server_st.h"
+#include "feature/nodelist/node_st.h"
+
+/** Global list of a dir_server_t object for each directory
+ * authority. */
+static smartlist_t *trusted_dir_servers = NULL;
+/** Global list of dir_server_t objects for all directory authorities
+ * and all fallback directory servers. */
+static smartlist_t *fallback_dir_servers = NULL;
+
+/** Return the number of directory authorities whose type matches some bit set
+ * in <b>type</b>  */
+int
+get_n_authorities(dirinfo_type_t type)
+{
+  int n = 0;
+  if (!trusted_dir_servers)
+    return 0;
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
+                    if (ds->type & type)
+                      ++n);
+  return n;
+}
+
+/** Return a smartlist containing a list of dir_server_t * for all
+ * known trusted dirservers.  Callers must not modify the list or its
+ * contents.
+ */
+smartlist_t *
+router_get_trusted_dir_servers_mutable(void)
+{
+  if (!trusted_dir_servers)
+    trusted_dir_servers = smartlist_new();
+
+  return trusted_dir_servers;
+}
+
+smartlist_t *
+router_get_fallback_dir_servers_mutable(void)
+{
+  if (!fallback_dir_servers)
+    fallback_dir_servers = smartlist_new();
+
+  return fallback_dir_servers;
+}
+
+const smartlist_t *
+router_get_trusted_dir_servers(void)
+{
+  return router_get_trusted_dir_servers_mutable();
+}
+
+const smartlist_t *
+router_get_fallback_dir_servers(void)
+{
+  return router_get_fallback_dir_servers_mutable();
+}
+
+/** Reset all internal variables used to count failed downloads of network
+ * status objects. */
+void
+router_reset_status_download_failures(void)
+{
+  mark_all_dirservers_up(fallback_dir_servers);
+}
+
+/** Return the dir_server_t for the directory authority whose identity
+ * key hashes to <b>digest</b>, or NULL if no such authority is known.
+ */
+dir_server_t *
+router_get_trusteddirserver_by_digest(const char *digest)
+{
+  if (!trusted_dir_servers)
+    return NULL;
+
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
+     {
+       if (tor_memeq(ds->digest, digest, DIGEST_LEN))
+         return ds;
+     });
+
+  return NULL;
+}
+
+/** Return the dir_server_t for the fallback dirserver whose identity
+ * key hashes to <b>digest</b>, or NULL if no such fallback is in the list of
+ * fallback_dir_servers. (fallback_dir_servers is affected by the FallbackDir
+ * and UseDefaultFallbackDirs torrc options.)
+ * The list of fallback directories includes the list of authorities.
+ */
+dir_server_t *
+router_get_fallback_dirserver_by_digest(const char *digest)
+{
+  if (!fallback_dir_servers)
+    return NULL;
+
+  if (!digest)
+    return NULL;
+
+  SMARTLIST_FOREACH(fallback_dir_servers, dir_server_t *, ds,
+     {
+       if (tor_memeq(ds->digest, digest, DIGEST_LEN))
+         return ds;
+     });
+
+  return NULL;
+}
+
+/** Return 1 if any fallback dirserver's identity key hashes to <b>digest</b>,
+ * or 0 if no such fallback is in the list of fallback_dir_servers.
+ * (fallback_dir_servers is affected by the FallbackDir and
+ * UseDefaultFallbackDirs torrc options.)
+ * The list of fallback directories includes the list of authorities.
+ */
+int
+router_digest_is_fallback_dir(const char *digest)
+{
+  return (router_get_fallback_dirserver_by_digest(digest) != NULL);
+}
+
+/** Return the dir_server_t for the directory authority whose
+ * v3 identity key hashes to <b>digest</b>, or NULL if no such authority
+ * is known.
+ */
+MOCK_IMPL(dir_server_t *,
+trusteddirserver_get_by_v3_auth_digest, (const char *digest))
+{
+  if (!trusted_dir_servers)
+    return NULL;
+
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
+     {
+       if (tor_memeq(ds->v3_identity_digest, digest, DIGEST_LEN) &&
+           (ds->type & V3_DIRINFO))
+         return ds;
+     });
+
+  return NULL;
+}
+
+/** Mark as running every dir_server_t in <b>server_list</b>. */
+void
+mark_all_dirservers_up(smartlist_t *server_list)
+{
+  if (server_list) {
+    SMARTLIST_FOREACH_BEGIN(server_list, dir_server_t *, dir) {
+      routerstatus_t *rs;
+      node_t *node;
+      dir->is_running = 1;
+      node = node_get_mutable_by_id(dir->digest);
+      if (node)
+        node->is_running = 1;
+      rs = router_get_mutable_consensus_status_by_id(dir->digest);
+      if (rs) {
+        rs->last_dir_503_at = 0;
+        control_event_networkstatus_changed_single(rs);
+      }
+    } SMARTLIST_FOREACH_END(dir);
+  }
+  router_dir_info_changed();
+}
+
+/** Return true iff <b>digest</b> is the digest of the identity key of a
+ * trusted directory matching at least one bit of <b>type</b>.  If <b>type</b>
+ * is zero (NO_DIRINFO), or ALL_DIRINFO, any authority is okay. */
+int
+router_digest_is_trusted_dir_type(const char *digest, dirinfo_type_t type)
+{
+  if (!trusted_dir_servers)
+    return 0;
+  if (authdir_mode(get_options()) && router_digest_is_me(digest))
+    return 1;
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ent,
+    if (tor_memeq(digest, ent->digest, DIGEST_LEN)) {
+      return (!type) || ((type & ent->type) != 0);
+    });
+  return 0;
+}
+
+/** Create a directory server at <b>address</b>:<b>port</b>, with OR identity
+ * key <b>digest</b> which has DIGEST_LEN bytes.  If <b>address</b> is NULL,
+ * add ourself.  If <b>is_authority</b>, this is a directory authority.  Return
+ * the new directory server entry on success or NULL on failure. */
+static dir_server_t *
+dir_server_new(int is_authority,
+               const char *nickname,
+               const tor_addr_t *addr,
+               const char *hostname,
+               uint16_t dir_port, uint16_t or_port,
+               const tor_addr_port_t *addrport_ipv6,
+               const char *digest, const char *v3_auth_digest,
+               dirinfo_type_t type,
+               double weight)
+{
+  dir_server_t *ent;
+  uint32_t a;
+  char *hostname_ = NULL;
+
+  tor_assert(digest);
+
+  if (weight < 0)
+    return NULL;
+
+  if (tor_addr_family(addr) == AF_INET)
+    a = tor_addr_to_ipv4h(addr);
+  else
+    return NULL;
+
+  if (!hostname)
+    hostname_ = tor_addr_to_str_dup(addr);
+  else
+    hostname_ = tor_strdup(hostname);
+
+  ent = tor_malloc_zero(sizeof(dir_server_t));
+  ent->nickname = nickname ? tor_strdup(nickname) : NULL;
+  ent->address = hostname_;
+  ent->addr = a;
+  ent->dir_port = dir_port;
+  ent->or_port = or_port;
+  ent->is_running = 1;
+  ent->is_authority = is_authority;
+  ent->type = type;
+  ent->weight = weight;
+  if (addrport_ipv6) {
+    if (tor_addr_family(&addrport_ipv6->addr) != AF_INET6) {
+      log_warn(LD_BUG, "Hey, I got a non-ipv6 addr as addrport_ipv6.");
+      tor_addr_make_unspec(&ent->ipv6_addr);
+    } else {
+      tor_addr_copy(&ent->ipv6_addr, &addrport_ipv6->addr);
+      ent->ipv6_orport = addrport_ipv6->port;
+    }
+  } else {
+    tor_addr_make_unspec(&ent->ipv6_addr);
+  }
+
+  memcpy(ent->digest, digest, DIGEST_LEN);
+  if (v3_auth_digest && (type & V3_DIRINFO))
+    memcpy(ent->v3_identity_digest, v3_auth_digest, DIGEST_LEN);
+
+  if (nickname)
+    tor_asprintf(&ent->description, "directory server \"%s\" at %s:%d",
+                 nickname, hostname_, (int)dir_port);
+  else
+    tor_asprintf(&ent->description, "directory server at %s:%d",
+                 hostname_, (int)dir_port);
+
+  ent->fake_status.addr = ent->addr;
+  tor_addr_copy(&ent->fake_status.ipv6_addr, &ent->ipv6_addr);
+  memcpy(ent->fake_status.identity_digest, digest, DIGEST_LEN);
+  if (nickname)
+    strlcpy(ent->fake_status.nickname, nickname,
+            sizeof(ent->fake_status.nickname));
+  else
+    ent->fake_status.nickname[0] = '\0';
+  ent->fake_status.dir_port = ent->dir_port;
+  ent->fake_status.or_port = ent->or_port;
+  ent->fake_status.ipv6_orport = ent->ipv6_orport;
+
+  return ent;
+}
+
+/** Create an authoritative directory server at
+ * <b>address</b>:<b>port</b>, with identity key <b>digest</b>.  If
+ * <b>address</b> is NULL, add ourself.  Return the new trusted directory
+ * server entry on success or NULL if we couldn't add it. */
+dir_server_t *
+trusted_dir_server_new(const char *nickname, const char *address,
+                       uint16_t dir_port, uint16_t or_port,
+                       const tor_addr_port_t *ipv6_addrport,
+                       const char *digest, const char *v3_auth_digest,
+                       dirinfo_type_t type, double weight)
+{
+  uint32_t a;
+  tor_addr_t addr;
+  char *hostname=NULL;
+  dir_server_t *result;
+
+  if (!address) { /* The address is us; we should guess. */
+    if (resolve_my_address(LOG_WARN, get_options(),
+                           &a, NULL, &hostname) < 0) {
+      log_warn(LD_CONFIG,
+               "Couldn't find a suitable address when adding ourself as a "
+               "trusted directory server.");
+      return NULL;
+    }
+    if (!hostname)
+      hostname = tor_dup_ip(a);
+  } else {
+    if (tor_lookup_hostname(address, &a)) {
+      log_warn(LD_CONFIG,
+               "Unable to lookup address for directory server at '%s'",
+               address);
+      return NULL;
+    }
+    hostname = tor_strdup(address);
+  }
+  tor_addr_from_ipv4h(&addr, a);
+
+  result = dir_server_new(1, nickname, &addr, hostname,
+                          dir_port, or_port,
+                          ipv6_addrport,
+                          digest,
+                          v3_auth_digest, type, weight);
+  tor_free(hostname);
+  return result;
+}
+
+/** Return a new dir_server_t for a fallback directory server at
+ * <b>addr</b>:<b>or_port</b>/<b>dir_port</b>, with identity key digest
+ * <b>id_digest</b> */
+dir_server_t *
+fallback_dir_server_new(const tor_addr_t *addr,
+                        uint16_t dir_port, uint16_t or_port,
+                        const tor_addr_port_t *addrport_ipv6,
+                        const char *id_digest, double weight)
+{
+  return dir_server_new(0, NULL, addr, NULL, dir_port, or_port,
+                        addrport_ipv6,
+                        id_digest,
+                        NULL, ALL_DIRINFO, weight);
+}
+
+/** Add a directory server to the global list(s). */
+void
+dir_server_add(dir_server_t *ent)
+{
+  if (!trusted_dir_servers)
+    trusted_dir_servers = smartlist_new();
+  if (!fallback_dir_servers)
+    fallback_dir_servers = smartlist_new();
+
+  if (ent->is_authority)
+    smartlist_add(trusted_dir_servers, ent);
+
+  smartlist_add(fallback_dir_servers, ent);
+  router_dir_info_changed();
+}
+
+#define dir_server_free(val) \
+  FREE_AND_NULL(dir_server_t, dir_server_free_, (val))
+
+/** Free storage held in <b>ds</b>. */
+static void
+dir_server_free_(dir_server_t *ds)
+{
+  if (!ds)
+    return;
+
+  tor_free(ds->nickname);
+  tor_free(ds->description);
+  tor_free(ds->address);
+  tor_free(ds);
+}
+
+/** Remove all members from the list of dir servers. */
+void
+clear_dir_servers(void)
+{
+  if (fallback_dir_servers) {
+    SMARTLIST_FOREACH(fallback_dir_servers, dir_server_t *, ent,
+                      dir_server_free(ent));
+    smartlist_clear(fallback_dir_servers);
+  } else {
+    fallback_dir_servers = smartlist_new();
+  }
+  if (trusted_dir_servers) {
+    smartlist_clear(trusted_dir_servers);
+  } else {
+    trusted_dir_servers = smartlist_new();
+  }
+  router_dir_info_changed();
+}
+
+void
+dirlist_free_all(void)
+{
+  clear_dir_servers();
+  smartlist_free(trusted_dir_servers);
+  smartlist_free(fallback_dir_servers);
+  trusted_dir_servers = fallback_dir_servers = NULL;
+}
diff --git a/src/feature/nodelist/dirlist.h b/src/feature/nodelist/dirlist.h
new file mode 100644
index 0000000000..9fabd0a44a
--- /dev/null
+++ b/src/feature/nodelist/dirlist.h
@@ -0,0 +1,47 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dirlist.h
+ * \brief Header file for dirlist.c
+ **/
+
+#ifndef TOR_DIRLIST_H
+#define TOR_DIRLIST_H
+
+int get_n_authorities(dirinfo_type_t type);
+const smartlist_t *router_get_trusted_dir_servers(void);
+const smartlist_t *router_get_fallback_dir_servers(void);
+smartlist_t *router_get_trusted_dir_servers_mutable(void);
+smartlist_t *router_get_fallback_dir_servers_mutable(void);
+void mark_all_dirservers_up(smartlist_t *server_list);
+
+dir_server_t *router_get_trusteddirserver_by_digest(const char *d);
+dir_server_t *router_get_fallback_dirserver_by_digest(
+                                                   const char *digest);
+int router_digest_is_fallback_dir(const char *digest);
+MOCK_DECL(dir_server_t *, trusteddirserver_get_by_v3_auth_digest,
+          (const char *d));
+
+int router_digest_is_trusted_dir_type(const char *digest,
+                                      dirinfo_type_t type);
+#define router_digest_is_trusted_dir(d) \
+  router_digest_is_trusted_dir_type((d), NO_DIRINFO)
+
+dir_server_t *trusted_dir_server_new(const char *nickname, const char *address,
+                       uint16_t dir_port, uint16_t or_port,
+                       const tor_addr_port_t *addrport_ipv6,
+                       const char *digest, const char *v3_auth_digest,
+                       dirinfo_type_t type, double weight);
+dir_server_t *fallback_dir_server_new(const tor_addr_t *addr,
+                                      uint16_t dir_port, uint16_t or_port,
+                                      const tor_addr_port_t *addrport_ipv6,
+                                      const char *id_digest, double weight);
+void dir_server_add(dir_server_t *ent);
+
+void clear_dir_servers(void);
+void dirlist_free_all(void);
+
+#endif
diff --git a/src/feature/nodelist/document_signature_st.h b/src/feature/nodelist/document_signature_st.h
new file mode 100644
index 0000000000..66e32c422f
--- /dev/null
+++ b/src/feature/nodelist/document_signature_st.h
@@ -0,0 +1,29 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef DOCUMENT_SIGNATURE_ST_H
+#define DOCUMENT_SIGNATURE_ST_H
+
+/** A signature of some document by an authority. */
+struct document_signature_t {
+  /** Declared SHA-1 digest of this voter's identity key */
+  char identity_digest[DIGEST_LEN];
+  /** Declared SHA-1 digest of signing key used by this voter. */
+  char signing_key_digest[DIGEST_LEN];
+  /** Algorithm used to compute the digest of the document. */
+  digest_algorithm_t alg;
+  /** Signature of the signed thing. */
+  char *signature;
+  /** Length of <b>signature</b> */
+  int signature_len;
+  unsigned int bad_signature : 1; /**< Set to true if we've tried to verify
+                                   * the sig, and we know it's bad. */
+  unsigned int good_signature : 1; /**< Set to true if we've verified the sig
+                                     * as good. */
+};
+
+#endif
+
diff --git a/src/feature/nodelist/extrainfo_st.h b/src/feature/nodelist/extrainfo_st.h
new file mode 100644
index 0000000000..c54277b05e
--- /dev/null
+++ b/src/feature/nodelist/extrainfo_st.h
@@ -0,0 +1,30 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef EXTRAINFO_ST_H
+#define EXTRAINFO_ST_H
+
+#include "feature/nodelist/signed_descriptor_st.h"
+
+/** Information needed to keep and cache a signed extra-info document. */
+struct extrainfo_t {
+  signed_descriptor_t cache_info;
+  /** SHA256 digest of this document */
+  uint8_t digest256[DIGEST256_LEN];
+  /** The router's nickname. */
+  char nickname[MAX_NICKNAME_LEN+1];
+  /** True iff we found the right key for this extra-info, verified the
+   * signature, and found it to be bad. */
+  unsigned int bad_sig : 1;
+  /** If present, we didn't have the right key to verify this extra-info,
+   * so this is a copy of the signature in the document. */
+  char *pending_sig;
+  /** Length of pending_sig. */
+  size_t pending_sig_len;
+};
+
+#endif
+
diff --git a/src/feature/nodelist/fmt_routerstatus.c b/src/feature/nodelist/fmt_routerstatus.c
new file mode 100644
index 0000000000..75cab7a0af
--- /dev/null
+++ b/src/feature/nodelist/fmt_routerstatus.c
@@ -0,0 +1,253 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file fmt_routerstatus.h
+ * \brief Format routerstatus entries for controller, vote, or consensus.
+ *
+ * (Because controllers consume this format, we can't make this
+ * code dirauth-only.)
+ **/
+
+#include "core/or/or.h"
+#include "feature/nodelist/fmt_routerstatus.h"
+
+/* #include "lib/container/buffers.h" */
+/* #include "app/config/config.h" */
+/* #include "app/config/confparse.h" */
+/* #include "core/or/channel.h" */
+/* #include "core/or/channeltls.h" */
+/* #include "core/or/command.h" */
+/* #include "core/mainloop/connection.h" */
+/* #include "core/or/connection_or.h" */
+/* #include "feature/dircache/conscache.h" */
+/* #include "feature/dircache/consdiffmgr.h" */
+/* #include "feature/control/control.h" */
+/* #include "feature/dircache/directory.h" */
+/* #include "feature/dircache/dirserv.h" */
+/* #include "feature/hibernate/hibernate.h" */
+/* #include "feature/dirauth/keypin.h" */
+/* #include "core/mainloop/mainloop.h" */
+/* #include "feature/nodelist/microdesc.h" */
+/* #include "feature/nodelist/networkstatus.h" */
+/* #include "feature/nodelist/nodelist.h" */
+#include "core/or/policies.h"
+/* #include "core/or/protover.h" */
+/* #include "feature/stats/rephist.h" */
+/* #include "feature/relay/router.h" */
+/* #include "feature/nodelist/dirlist.h" */
+#include "feature/nodelist/routerlist.h"
+
+/* #include "feature/nodelist/routerparse.h" */
+/* #include "feature/nodelist/routerset.h" */
+/* #include "feature/nodelist/torcert.h" */
+/* #include "feature/dircommon/voting_schedule.h" */
+
+#include "feature/dirauth/dirvote.h"
+
+/* #include "feature/dircache/cached_dir_st.h" */
+/* #include "feature/dircommon/dir_connection_st.h" */
+/* #include "feature/nodelist/extrainfo_st.h" */
+/* #include "feature/nodelist/microdesc_st.h" */
+/* #include "feature/nodelist/node_st.h" */
+#include "feature/nodelist/routerinfo_st.h"
+/* #include "feature/nodelist/routerlist_st.h" */
+/* #include "core/or/tor_version_st.h" */
+#include "feature/nodelist/vote_routerstatus_st.h"
+
+/* #include "lib/compress/compress.h" */
+/* #include "lib/container/order.h" */
+#include "lib/crypt_ops/crypto_format.h"
+/* #include "lib/encoding/confline.h" */
+
+/* #include "lib/encoding/keyval.h" */
+
+/** Helper: write the router-status information in <b>rs</b> into a newly
+ * allocated character buffer.  Use the same format as in network-status
+ * documents.  If <b>version</b> is non-NULL, add a "v" line for the platform.
+ *
+ * consensus_method is the current consensus method when format is
+ * NS_V3_CONSENSUS or NS_V3_CONSENSUS_MICRODESC. It is ignored for other
+ * formats: pass ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD.
+ *
+ * Return 0 on success, -1 on failure.
+ *
+ * The format argument has one of the following values:
+ *   NS_V2 - Output an entry suitable for a V2 NS opinion document
+ *   NS_V3_CONSENSUS - Output the first portion of a V3 NS consensus entry
+ *        for consensus_method.
+ *   NS_V3_CONSENSUS_MICRODESC - Output the first portion of a V3 microdesc
+ *        consensus entry for consensus_method.
+ *   NS_V3_VOTE - Output a complete V3 NS vote. If <b>vrs</b> is present,
+ *        it contains additional information for the vote.
+ *   NS_CONTROL_PORT - Output a NS document for the control port.
+ */
+char *
+routerstatus_format_entry(const routerstatus_t *rs, const char *version,
+                          const char *protocols,
+                          routerstatus_format_type_t format,
+                          int consensus_method,
+                          const vote_routerstatus_t *vrs)
+{
+  char *summary;
+  char *result = NULL;
+
+  char published[ISO_TIME_LEN+1];
+  char identity64[BASE64_DIGEST_LEN+1];
+  char digest64[BASE64_DIGEST_LEN+1];
+  smartlist_t *chunks = smartlist_new();
+
+  format_iso_time(published, rs->published_on);
+  digest_to_base64(identity64, rs->identity_digest);
+  digest_to_base64(digest64, rs->descriptor_digest);
+
+  smartlist_add_asprintf(chunks,
+                   "r %s %s %s%s%s %s %d %d\n",
+                   rs->nickname,
+                   identity64,
+                   (format==NS_V3_CONSENSUS_MICRODESC)?"":digest64,
+                   (format==NS_V3_CONSENSUS_MICRODESC)?"":" ",
+                   published,
+                   fmt_addr32(rs->addr),
+                   (int)rs->or_port,
+                   (int)rs->dir_port);
+
+  /* TODO: Maybe we want to pass in what we need to build the rest of
+   * this here, instead of in the caller. Then we could use the
+   * networkstatus_type_t values, with an additional control port value
+   * added -MP */
+
+  /* V3 microdesc consensuses only have "a" lines in later consensus methods
+   */
+  if (format == NS_V3_CONSENSUS_MICRODESC &&
+      consensus_method < MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS)
+    goto done;
+
+  /* Possible "a" line. At most one for now. */
+  if (!tor_addr_is_null(&rs->ipv6_addr)) {
+    smartlist_add_asprintf(chunks, "a %s\n",
+                           fmt_addrport(&rs->ipv6_addr, rs->ipv6_orport));
+  }
+
+  if (format == NS_V3_CONSENSUS || format == NS_V3_CONSENSUS_MICRODESC)
+    goto done;
+
+  smartlist_add_asprintf(chunks,
+                   "s%s%s%s%s%s%s%s%s%s%s\n",
+                  /* These must stay in alphabetical order. */
+                   rs->is_authority?" Authority":"",
+                   rs->is_bad_exit?" BadExit":"",
+                   rs->is_exit?" Exit":"",
+                   rs->is_fast?" Fast":"",
+                   rs->is_possible_guard?" Guard":"",
+                   rs->is_hs_dir?" HSDir":"",
+                   rs->is_flagged_running?" Running":"",
+                   rs->is_stable?" Stable":"",
+                   rs->is_v2_dir?" V2Dir":"",
+                   rs->is_valid?" Valid":"");
+
+  /* length of "opt v \n" */
+#define V_LINE_OVERHEAD 7
+  if (version && strlen(version) < MAX_V_LINE_LEN - V_LINE_OVERHEAD) {
+    smartlist_add_asprintf(chunks, "v %s\n", version);
+  }
+  if (protocols) {
+    smartlist_add_asprintf(chunks, "pr %s\n", protocols);
+  }
+
+  if (format != NS_V2) {
+    const routerinfo_t* desc = router_get_by_id_digest(rs->identity_digest);
+    uint32_t bw_kb;
+
+    if (format != NS_CONTROL_PORT) {
+      /* Blow up more or less nicely if we didn't get anything or not the
+       * thing we expected.
+       */
+      if (!desc) {
+        char id[HEX_DIGEST_LEN+1];
+        char dd[HEX_DIGEST_LEN+1];
+
+        base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN);
+        base16_encode(dd, sizeof(dd), rs->descriptor_digest, DIGEST_LEN);
+        log_warn(LD_BUG, "Cannot get any descriptor for %s "
+            "(wanted descriptor %s).",
+            id, dd);
+        goto err;
+      }
+
+      /* This assert could fire for the control port, because
+       * it can request NS documents before all descriptors
+       * have been fetched. Therefore, we only do this test when
+       * format != NS_CONTROL_PORT. */
+      if (tor_memneq(desc->cache_info.signed_descriptor_digest,
+            rs->descriptor_digest,
+            DIGEST_LEN)) {
+        char rl_d[HEX_DIGEST_LEN+1];
+        char rs_d[HEX_DIGEST_LEN+1];
+        char id[HEX_DIGEST_LEN+1];
+
+        base16_encode(rl_d, sizeof(rl_d),
+            desc->cache_info.signed_descriptor_digest, DIGEST_LEN);
+        base16_encode(rs_d, sizeof(rs_d), rs->descriptor_digest, DIGEST_LEN);
+        base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN);
+        log_err(LD_BUG, "descriptor digest in routerlist does not match "
+            "the one in routerstatus: %s vs %s "
+            "(router %s)\n",
+            rl_d, rs_d, id);
+
+        tor_assert(tor_memeq(desc->cache_info.signed_descriptor_digest,
+              rs->descriptor_digest,
+              DIGEST_LEN));
+      }
+    }
+
+    if (format == NS_CONTROL_PORT && rs->has_bandwidth) {
+      bw_kb = rs->bandwidth_kb;
+    } else {
+      tor_assert(desc);
+      bw_kb = router_get_advertised_bandwidth_capped(desc) / 1000;
+    }
+    smartlist_add_asprintf(chunks,
+                     "w Bandwidth=%d", bw_kb);
+
+    if (format == NS_V3_VOTE && vrs && vrs->has_measured_bw) {
+      smartlist_add_asprintf(chunks,
+                       " Measured=%d", vrs->measured_bw_kb);
+    }
+    /* Write down guardfraction information if we have it. */
+    if (format == NS_V3_VOTE && vrs && vrs->status.has_guardfraction) {
+      smartlist_add_asprintf(chunks,
+                             " GuardFraction=%d",
+                             vrs->status.guardfraction_percentage);
+    }
+
+    smartlist_add_strdup(chunks, "\n");
+
+    if (desc) {
+      summary = policy_summarize(desc->exit_policy, AF_INET);
+      smartlist_add_asprintf(chunks, "p %s\n", summary);
+      tor_free(summary);
+    }
+
+    if (format == NS_V3_VOTE && vrs) {
+      if (tor_mem_is_zero((char*)vrs->ed25519_id, ED25519_PUBKEY_LEN)) {
+        smartlist_add_strdup(chunks, "id ed25519 none\n");
+      } else {
+        char ed_b64[BASE64_DIGEST256_LEN+1];
+        digest256_to_base64(ed_b64, (const char*)vrs->ed25519_id);
+        smartlist_add_asprintf(chunks, "id ed25519 %s\n", ed_b64);
+      }
+    }
+  }
+
+ done:
+  result = smartlist_join_strings(chunks, "", 0, NULL);
+
+ err:
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_free(chunks);
+
+  return result;
+}
diff --git a/src/feature/nodelist/fmt_routerstatus.h b/src/feature/nodelist/fmt_routerstatus.h
new file mode 100644
index 0000000000..ddd7a7cf37
--- /dev/null
+++ b/src/feature/nodelist/fmt_routerstatus.h
@@ -0,0 +1,41 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file fmt_routerstatus.h
+ * \brief Header file for fmt_routerstatus.c.
+ **/
+
+#ifndef TOR_FMT_ROUTERSTATUS_H
+#define TOR_FMT_ROUTERSTATUS_H
+
+/** An enum to describe what format we're generating a routerstatus line in.
+ */
+typedef enum {
+  /** For use in a v2 opinion */
+  NS_V2,
+  /** For use in a consensus networkstatus document (ns flavor) */
+  NS_V3_CONSENSUS,
+  /** For use in a vote networkstatus document */
+  NS_V3_VOTE,
+  /** For passing to the controlport in response to a GETINFO request */
+  NS_CONTROL_PORT,
+  /** For use in a consensus networkstatus document (microdesc flavor) */
+  NS_V3_CONSENSUS_MICRODESC
+} routerstatus_format_type_t;
+
+/** Maximum allowable length of a version line in a networkstatus. */
+#define MAX_V_LINE_LEN 128
+
+char *routerstatus_format_entry(
+                              const routerstatus_t *rs,
+                              const char *version,
+                              const char *protocols,
+                              routerstatus_format_type_t format,
+                              int consensus_method,
+                              const vote_routerstatus_t *vrs);
+
+#endif /* !defined(TOR_FMT_ROUTERSTATUS_H) */
diff --git a/src/or/microdesc.c b/src/feature/nodelist/microdesc.c
index a81dc54628..dafaabb5e5 100644
--- a/src/or/microdesc.c
+++ b/src/feature/nodelist/microdesc.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2009-2016, The Tor Project, Inc. */
+/* Copyright (c) 2009-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -8,19 +8,36 @@
  *  less-frequently-changing router information.
  */
 
-#include "or.h"
-#include "circuitbuild.h"
-#include "config.h"
-#include "directory.h"
-#include "dirserv.h"
-#include "entrynodes.h"
-#include "microdesc.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "policies.h"
-#include "router.h"
-#include "routerlist.h"
-#include "routerparse.h"
+#include "core/or/or.h"
+
+#include "lib/fdio/fdio.h"
+
+#include "app/config/config.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/policies.h"
+#include "feature/client/entrynodes.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dlstatus.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirparse/microdesc_parse.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/router.h"
+
+#include "feature/nodelist/microdesc_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
 
 /** A data structure to hold a bunch of cached microdescriptors.  There are
  * two active files in the cache: a "cache file" that we mmap, and a "journal
@@ -74,6 +91,104 @@ HT_GENERATE2(microdesc_map, microdesc_t, node,
              microdesc_hash_, microdesc_eq_, 0.6,
              tor_reallocarray_, tor_free_)
 
+/************************* md fetch fail cache *****************************/
+
+/* If we end up with too many outdated dirservers, something probably went
+ * wrong so clean up the list. */
+#define TOO_MANY_OUTDATED_DIRSERVERS 30
+
+/** List of dirservers with outdated microdesc information. The smartlist is
+ *  filled with the hex digests of outdated dirservers. */
+static smartlist_t *outdated_dirserver_list = NULL;
+
+/** Note that we failed to fetch a microdescriptor from the relay with
+ *  <b>relay_digest</b> (of size DIGEST_LEN). */
+void
+microdesc_note_outdated_dirserver(const char *relay_digest)
+{
+  char relay_hexdigest[HEX_DIGEST_LEN+1];
+
+  /* If we have a reasonably live consensus, then most of our dirservers should
+   * still be caching all the microdescriptors in it. Reasonably live
+   * consensuses are up to a day old. But microdescriptors expire 7 days after
+   * the last consensus that referenced them. */
+  if (!networkstatus_get_reasonably_live_consensus(approx_time(),
+                                                   FLAV_MICRODESC)) {
+    return;
+  }
+
+  if (!outdated_dirserver_list) {
+    outdated_dirserver_list = smartlist_new();
+  }
+
+  tor_assert(outdated_dirserver_list);
+
+  /* If the list grows too big, clean it up */
+  if (BUG(smartlist_len(outdated_dirserver_list) >
+          TOO_MANY_OUTDATED_DIRSERVERS)) {
+    microdesc_reset_outdated_dirservers_list();
+  }
+
+  /* Turn the binary relay digest to a hex since smartlists have better support
+   * for strings than digests. */
+  base16_encode(relay_hexdigest,sizeof(relay_hexdigest),
+                relay_digest, DIGEST_LEN);
+
+  /* Make sure we don't add a dirauth as an outdated dirserver */
+  if (router_get_trusteddirserver_by_digest(relay_digest)) {
+    log_info(LD_GENERAL, "Auth %s gave us outdated dirinfo.", relay_hexdigest);
+    return;
+  }
+
+  /* Don't double-add outdated dirservers */
+  if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
+    return;
+  }
+
+  /* Add it to the list of outdated dirservers */
+  smartlist_add_strdup(outdated_dirserver_list, relay_hexdigest);
+
+  log_info(LD_GENERAL, "Noted %s as outdated md dirserver", relay_hexdigest);
+}
+
+/** Return True if the relay with <b>relay_digest</b> (size DIGEST_LEN) is an
+ *  outdated dirserver */
+int
+microdesc_relay_is_outdated_dirserver(const char *relay_digest)
+{
+  char relay_hexdigest[HEX_DIGEST_LEN+1];
+
+  if (!outdated_dirserver_list) {
+    return 0;
+  }
+
+  /* Convert identity digest to hex digest */
+  base16_encode(relay_hexdigest, sizeof(relay_hexdigest),
+                relay_digest, DIGEST_LEN);
+
+  /* Last time we tried to fetch microdescs, was this directory mirror missing
+   * any mds we asked for? */
+  if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
+    return 1;
+  }
+
+  return 0;
+}
+
+/** Reset the list of outdated dirservers. */
+void
+microdesc_reset_outdated_dirservers_list(void)
+{
+  if (!outdated_dirserver_list) {
+    return;
+  }
+
+  SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp));
+  smartlist_clear(outdated_dirserver_list);
+}
+
+/****************************************************************************/
+
 /** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations.
  * On success, return the total number of bytes written, and set
  * *<b>annotation_len_out</b> to the number of bytes written as
@@ -93,7 +208,7 @@ dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
     char annotation[ISO_TIME_LEN+32];
     format_iso_time(buf, md->last_listed);
     tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf);
-    if (write_all(fd, annotation, strlen(annotation), 0) < 0) {
+    if (write_all_to_fd(fd, annotation, strlen(annotation)) < 0) {
       log_warn(LD_DIR,
                "Couldn't write microdescriptor annotation: %s",
                strerror(errno));
@@ -106,7 +221,7 @@ dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
   }
 
   md->off = tor_fd_getpos(fd);
-  written = write_all(fd, md->body, md->bodylen, 0);
+  written = write_all_to_fd(fd, md->body, md->bodylen);
   if (written != (ssize_t)md->bodylen) {
     written = written < 0 ? 0 : written;
     log_warn(LD_DIR,
@@ -142,8 +257,8 @@ get_microdesc_cache_noload(void)
   if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) {
     microdesc_cache_t *cache = tor_malloc_zero(sizeof(*cache));
     HT_INIT(microdesc_map, &cache->map);
-    cache->cache_fname = get_datadir_fname("cached-microdescs");
-    cache->journal_fname = get_datadir_fname("cached-microdescs.new");
+    cache->cache_fname = get_cachedir_fname("cached-microdescs");
+    cache->journal_fname = get_cachedir_fname("cached-microdescs.new");
     the_microdesc_cache = cache;
   }
   return the_microdesc_cache;
@@ -610,9 +725,9 @@ microdesc_cache_rebuild(microdesc_cache_t *cache, int force)
     off_real = tor_fd_getpos(fd);
     if (off_real != off) {
       log_warn(LD_BUG, "Discontinuity in position in microdescriptor cache."
-               "By my count, I'm at "I64_FORMAT
-               ", but I should be at "I64_FORMAT,
-               I64_PRINTF_ARG(off), I64_PRINTF_ARG(off_real));
+               "By my count, I'm at %"PRId64
+               ", but I should be at %"PRId64,
+               (int64_t)(off), (int64_t)(off_real));
       if (off_real >= 0)
         off = off_real;
     }
@@ -763,7 +878,7 @@ microdesc_free_(microdesc_t *md, const char *fname, int lineno)
   //tor_assert(md->held_by_nodes == 0);
 
   if (md->onion_pkey)
-    crypto_pk_free(md->onion_pkey);
+    tor_free(md->onion_pkey);
   tor_free(md->onion_curve25519_pkey);
   tor_free(md->ed25519_identity_pkey);
   if (md->body && md->saved_location != SAVED_IN_CACHE)
@@ -789,6 +904,11 @@ microdesc_free_all(void)
     tor_free(the_microdesc_cache->journal_fname);
     tor_free(the_microdesc_cache);
   }
+
+  if (outdated_dirserver_list) {
+    SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp));
+    smartlist_free(outdated_dirserver_list);
+  }
 }
 
 /** If there is a microdescriptor in <b>cache</b> whose sha256 digest is
@@ -804,18 +924,6 @@ microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d)
   return md;
 }
 
-/** Return the mean size of decriptors added to <b>cache</b> since it was last
- * cleared.  Used to estimate the size of large downloads. */
-size_t
-microdesc_average_size(microdesc_cache_t *cache)
-{
-  if (!cache)
-    cache = get_microdesc_cache();
-  if (!cache->n_seen)
-    return 512;
-  return (size_t)(cache->total_len_seen / cache->n_seen);
-}
-
 /** Return a smartlist of all the sha256 digest of the microdescriptors that
  * are listed in <b>ns</b> but not present in <b>cache</b>. Returns pointers
  * to internals of <b>ns</b>; you should not free the members of the resulting
@@ -831,8 +939,7 @@ microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache,
     if (microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest))
       continue;
     if (downloadable_only &&
-        !download_status_is_ready(&rs->dl_status, now,
-                  get_options()->TestingMicrodescMaxDownloadTries))
+        !download_status_is_ready(&rs->dl_status, now))
       continue;
     if (skip && digest256map_get(skip, (const uint8_t*)rs->descriptor_digest))
       continue;
@@ -888,7 +995,7 @@ update_microdesc_downloads(time_t now)
   smartlist_free(missing);
 }
 
-/** For every microdescriptor listed in the current microdecriptor consensus,
+/** For every microdescriptor listed in the current microdescriptor consensus,
  * update its last_listed field to be at least as recent as the publication
  * time of the current microdescriptor consensus.
  */
@@ -917,20 +1024,9 @@ update_microdescs_from_networkstatus(time_t now)
 int
 we_use_microdescriptors_for_circuits(const or_options_t *options)
 {
-  int ret = options->UseMicrodescriptors;
-  if (ret == -1) {
-    /* UseMicrodescriptors is "auto"; we need to decide: */
-    /* If we are configured to use bridges and none of our bridges
-     * know what a microdescriptor is, the answer is no. */
-    if (options->UseBridges && !any_bridge_supports_microdescriptors())
-      return 0;
-    /* Otherwise, we decide that we'll use microdescriptors iff we are
-     * not a server, and we're not autofetching everything. */
-    /* XXXX++ what does not being a server have to do with it? also there's
-     * a partitioning issue here where bridges differ from clients. */
-    ret = !server_mode(options) && !options->FetchUselessDescriptors;
-  }
-  return ret;
+  if (options->UseMicrodescriptors == 0)
+    return 0; /* the user explicitly picked no */
+  return 1; /* yes and auto both mean yes */
 }
 
 /** Return true iff we should try to download microdescriptors at all. */
@@ -965,4 +1061,3 @@ usable_consensus_flavor,(void))
     return FLAV_NS;
   }
 }
-
diff --git a/src/or/microdesc.h b/src/feature/nodelist/microdesc.h
index 40c83139e9..c18099d540 100644
--- a/src/or/microdesc.h
+++ b/src/feature/nodelist/microdesc.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -32,16 +32,16 @@ void microdesc_cache_clear(microdesc_cache_t *cache);
 microdesc_t *microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache,
                                                  const char *d);
 
-size_t microdesc_average_size(microdesc_cache_t *cache);
-
 smartlist_t *microdesc_list_missing_digest256(networkstatus_t *ns,
                                               microdesc_cache_t *cache,
                                               int downloadable_only,
                                               digest256map_t *skip);
 
 void microdesc_free_(microdesc_t *md, const char *fname, int line);
-#define microdesc_free(md) \
-  microdesc_free_((md), __FILE__, __LINE__)
+#define microdesc_free(md) do {                 \
+    microdesc_free_((md), __FILE__, __LINE__);  \
+    (md) = NULL;                                \
+  } while (0)
 void microdesc_free_all(void);
 
 void update_microdesc_downloads(time_t now);
@@ -52,5 +52,9 @@ int we_fetch_microdescriptors(const or_options_t *options);
 int we_fetch_router_descriptors(const or_options_t *options);
 int we_use_microdescriptors_for_circuits(const or_options_t *options);
 
-#endif
+void microdesc_note_outdated_dirserver(const char *relay_digest);
+int microdesc_relay_is_outdated_dirserver(const char *relay_digest);
+void microdesc_reset_outdated_dirservers_list(void);
+
+#endif /* !defined(TOR_MICRODESC_H) */
 
diff --git a/src/feature/nodelist/microdesc_st.h b/src/feature/nodelist/microdesc_st.h
new file mode 100644
index 0000000000..bb8b23d664
--- /dev/null
+++ b/src/feature/nodelist/microdesc_st.h
@@ -0,0 +1,80 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef MICRODESC_ST_H
+#define MICRODESC_ST_H
+
+struct curve25519_public_key_t;
+struct ed25519_public_key_t;
+struct short_policy_t;
+
+/** A microdescriptor is the smallest amount of information needed to build a
+ * circuit through a router.  They are generated by the directory authorities,
+ * using information from the uploaded routerinfo documents.  They are not
+ * self-signed, but are rather authenticated by having their hash in a signed
+ * networkstatus document. */
+struct microdesc_t {
+  /** Hashtable node, used to look up the microdesc by its digest. */
+  HT_ENTRY(microdesc_t) node;
+
+  /* Cache information */
+
+  /**  When was this microdescriptor last listed in a consensus document?
+   * Once a microdesc has been unlisted long enough, we can drop it.
+   */
+  time_t last_listed;
+  /** Where is this microdescriptor currently stored? */
+  saved_location_bitfield_t saved_location : 3;
+  /** If true, do not attempt to cache this microdescriptor on disk. */
+  unsigned int no_save : 1;
+  /** If true, this microdesc has an entry in the microdesc_map */
+  unsigned int held_in_map : 1;
+  /** Reference count: how many node_ts have a reference to this microdesc? */
+  unsigned int held_by_nodes;
+
+  /** If saved_location == SAVED_IN_CACHE, this field holds the offset of the
+   * microdescriptor in the cache. */
+  off_t off;
+
+  /* The string containing the microdesc. */
+
+  /** A pointer to the encoded body of the microdescriptor.  If the
+   * saved_location is SAVED_IN_CACHE, then the body is a pointer into an
+   * mmap'd region.  Otherwise, it is a malloc'd string.  The string might not
+   * be NUL-terminated; take the length from <b>bodylen</b>. */
+  char *body;
+  /** The length of the microdescriptor in <b>body</b>. */
+  size_t bodylen;
+  /** A SHA256-digest of the microdescriptor. */
+  char digest[DIGEST256_LEN];
+
+  /* Fields in the microdescriptor. */
+
+  /**
+   * Public RSA TAP key for onions, ASN.1 encoded.  We store this
+   * in its encoded format since storing it as a crypto_pk_t uses
+   * significantly more memory. */
+  char *onion_pkey;
+  /** Length of onion_pkey, in bytes. */
+  size_t onion_pkey_len;
+
+  /** As routerinfo_t.onion_curve25519_pkey */
+  struct curve25519_public_key_t *onion_curve25519_pkey;
+  /** Ed25519 identity key, if included. */
+  struct ed25519_public_key_t *ed25519_identity_pkey;
+  /** As routerinfo_t.ipv6_addr */
+  tor_addr_t ipv6_addr;
+  /** As routerinfo_t.ipv6_orport */
+  uint16_t ipv6_orport;
+  /** As routerinfo_t.family */
+  smartlist_t *family;
+  /** IPv4 exit policy summary */
+  struct short_policy_t *exit_policy;
+  /** IPv6 exit policy summary */
+  struct short_policy_t *ipv6_exit_policy;
+};
+
+#endif
diff --git a/src/or/networkstatus.c b/src/feature/nodelist/networkstatus.c
index d8e2c00273..c74acd8b74 100644
--- a/src/or/networkstatus.c
+++ b/src/feature/nodelist/networkstatus.c
@@ -1,56 +1,115 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file networkstatus.c
- * \brief Functions and structures for handling network status documents as a
- * client or cache.
+ * \brief Functions and structures for handling networkstatus documents as a
+ * client or as a directory cache.
+ *
+ * A consensus networkstatus object is created by the directory
+ * authorities.  It authenticates a set of network parameters--most
+ * importantly, the list of all the relays in the network.  This list
+ * of relays is represented as an array of routerstatus_t objects.
+ *
+ * There are currently two flavors of consensus.  With the older "NS"
+ * flavor, each relay is associated with a digest of its router
+ * descriptor. Tor instances that use this consensus keep the list of
+ * router descriptors as routerinfo_t objects stored and managed in
+ * routerlist.c.  With the newer "microdesc" flavor, each relay is
+ * associated with a digest of the microdescriptor that the authorities
+ * made for it.  These are stored and managed in microdesc.c. Information
+ * about the router is divided between the the networkstatus and the
+ * microdescriptor according to the general rule that microdescriptors
+ * should hold information that changes much less frequently than the
+ * information in the networkstatus.
+ *
+ * Modern clients use microdescriptor networkstatuses.  Directory caches
+ * need to keep both kinds of networkstatus document, so they can serve them.
+ *
+ * This module manages fetching, holding, storing, updating, and
+ * validating networkstatus objects.  The download-and-validate process
+ * is slightly complicated by the fact that the keys you need to
+ * validate a consensus are stored in the authority certificates, which
+ * you might not have yet when you download the consensus.
  */
 
 #define NETWORKSTATUS_PRIVATE
-#include "or.h"
-#include "channel.h"
-#include "circuitmux.h"
-#include "circuitmux_ewma.h"
-#include "circuitstats.h"
-#include "config.h"
-#include "connection.h"
-#include "connection_or.h"
-#include "control.h"
-#include "directory.h"
-#include "dirserv.h"
-#include "dirvote.h"
-#include "dos.h"
-#include "entrynodes.h"
-#include "main.h"
-#include "microdesc.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "protover.h"
-#include "relay.h"
-#include "router.h"
-#include "routerlist.h"
-#include "routerparse.h"
-#include "shared_random.h"
-#include "transports.h"
-#include "torcert.h"
-
-/** Map from lowercase nickname to identity digest of named server, if any. */
-static strmap_t *named_server_map = NULL;
-/** Map from lowercase nickname to (void*)1 for all names that are listed
- * as unnamed for some server in the consensus. */
-static strmap_t *unnamed_server_map = NULL;
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
+#include "core/or/channel.h"
+#include "core/or/channelpadding.h"
+#include "core/or/circuitmux.h"
+#include "core/or/circuitmux_ewma.h"
+#include "core/or/circuitstats.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "core/or/dos.h"
+#include "core/or/protover.h"
+#include "core/or/relay.h"
+#include "core/or/scheduler.h"
+#include "core/or/versions.h"
+#include "feature/client/bridges.h"
+#include "feature/client/entrynodes.h"
+#include "feature/client/transports.h"
+#include "feature/control/control.h"
+#include "feature/dirauth/reachability.h"
+#include "feature/dircache/consdiffmgr.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirparse/ns_parse.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/fmt_routerstatus.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/node_select.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerinfo.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/torcert.h"
+#include "feature/relay/routermode.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/authmode.h"
+#include "feature/dirauth/shared_random.h"
+#include "feature/dirauth/voteflags.h"
+
+#include "feature/nodelist/authority_cert_st.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/dirclient/dir_server_st.h"
+#include "feature/nodelist/document_signature_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/networkstatus_voter_info_st.h"
+#include "feature/dirauth/ns_detached_signatures_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerlist_st.h"
+#include "feature/dirauth/vote_microdesc_hash_st.h"
+#include "feature/nodelist/vote_routerstatus_st.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
 
 /** Most recently received and validated v3 "ns"-flavored consensus network
  * status. */
-static networkstatus_t *current_ns_consensus = NULL;
+STATIC networkstatus_t *current_ns_consensus = NULL;
 
-/** Most recently received and validated v3 "microdec"-flavored consensus
+/** Most recently received and validated v3 "microdesc"-flavored consensus
  * network status. */
-static networkstatus_t *current_md_consensus = NULL;
+STATIC networkstatus_t *current_md_consensus = NULL;
 
 /** A v3 consensus networkstatus that we've received, but which we don't
  * have enough certificates to be happy about. */
@@ -82,9 +141,9 @@ static time_t time_to_download_next_consensus[N_CONSENSUS_FLAVORS];
 static download_status_t consensus_dl_status[N_CONSENSUS_FLAVORS] =
   {
     { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER,
-      DL_SCHED_INCREMENT_FAILURE, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 },
+      DL_SCHED_INCREMENT_FAILURE, 0, 0 },
     { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER,
-      DL_SCHED_INCREMENT_FAILURE, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 },
+      DL_SCHED_INCREMENT_FAILURE, 0, 0 },
   };
 
 #define N_CONSENSUS_BOOTSTRAP_SCHEDULES 2
@@ -101,10 +160,10 @@ static download_status_t
               consensus_bootstrap_dl_status[N_CONSENSUS_BOOTSTRAP_SCHEDULES] =
   {
     { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_AUTHORITY,
-      DL_SCHED_INCREMENT_ATTEMPT, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 },
+      DL_SCHED_INCREMENT_ATTEMPT, 0, 0 },
     /* During bootstrap, DL_WANT_ANY_DIRSERVER means "use fallbacks". */
     { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER,
-      DL_SCHED_INCREMENT_ATTEMPT, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 },
+      DL_SCHED_INCREMENT_ATTEMPT, 0, 0 },
   };
 
 /** True iff we have logged a warning about this OR's version being older than
@@ -114,7 +173,6 @@ static int have_warned_about_old_version = 0;
  * listed by the authorities. */
 static int have_warned_about_new_version = 0;
 
-static void routerstatus_list_update_named_server_map(void);
 static void update_consensus_bootstrap_multiple_downloads(
                                                   time_t now,
                                                   const or_options_t *options);
@@ -152,60 +210,74 @@ networkstatus_reset_download_failures(void)
     download_status_reset(&consensus_bootstrap_dl_status[i]);
 }
 
+/**
+ * Read and and return the cached consensus of type <b>flavorname</b>.  If
+ * <b>unverified</b> is true, get the one we haven't verified. Return NULL if
+ * the file isn't there. */
+static char *
+networkstatus_read_cached_consensus_impl(int flav,
+                                         const char *flavorname,
+                                         int unverified_consensus)
+{
+  char buf[128];
+  const char *prefix;
+  if (unverified_consensus) {
+    prefix = "unverified";
+  } else {
+    prefix = "cached";
+  }
+  if (flav == FLAV_NS) {
+    tor_snprintf(buf, sizeof(buf), "%s-consensus", prefix);
+  } else {
+    tor_snprintf(buf, sizeof(buf), "%s-%s-consensus", prefix, flavorname);
+  }
+
+  char *filename = get_cachedir_fname(buf);
+  char *result = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+  tor_free(filename);
+  return result;
+}
+
+/** Return a new string containing the current cached consensus of flavor
+ * <b>flavorname</b>. */
+char *
+networkstatus_read_cached_consensus(const char *flavorname)
+ {
+  int flav = networkstatus_parse_flavor_name(flavorname);
+  if (flav < 0)
+    return NULL;
+  return networkstatus_read_cached_consensus_impl(flav, flavorname, 0);
+}
+
 /** Read every cached v3 consensus networkstatus from the disk. */
 int
 router_reload_consensus_networkstatus(void)
 {
-  char *filename;
-  char *s;
   const unsigned int flags = NSSET_FROM_CACHE | NSSET_DONT_DOWNLOAD_CERTS;
   int flav;
 
   /* FFFF Suppress warnings if cached consensus is bad? */
   for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
-    char buf[128];
     const char *flavor = networkstatus_get_flavor_name(flav);
-    if (flav == FLAV_NS) {
-      filename = get_datadir_fname("cached-consensus");
-    } else {
-      tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor);
-      filename = get_datadir_fname(buf);
-    }
-    s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+    char *s = networkstatus_read_cached_consensus_impl(flav, flavor, 0);
     if (s) {
       if (networkstatus_set_current_consensus(s, flavor, flags, NULL) < -1) {
-        log_warn(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"",
-                 flavor, filename);
+        log_warn(LD_FS, "Couldn't load consensus %s networkstatus from cache",
+                 flavor);
       }
       tor_free(s);
     }
-    tor_free(filename);
-
-    if (flav == FLAV_NS) {
-      filename = get_datadir_fname("unverified-consensus");
-    } else {
-      tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor);
-      filename = get_datadir_fname(buf);
-    }
 
-    s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+    s = networkstatus_read_cached_consensus_impl(flav, flavor, 1);
     if (s) {
       if (networkstatus_set_current_consensus(s, flavor,
-                                     flags|NSSET_WAS_WAITING_FOR_CERTS,
+                                     flags | NSSET_WAS_WAITING_FOR_CERTS,
                                      NULL)) {
-      log_info(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"",
-               flavor, filename);
-    }
+        log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus "
+                 "from cache", flavor);
+      }
       tor_free(s);
     }
-    tor_free(filename);
-  }
-
-  if (!networkstatus_get_latest_consensus()) {
-    if (!named_server_map)
-      named_server_map = strmap_new();
-    if (!unnamed_server_map)
-      unnamed_server_map = strmap_new();
   }
 
   update_certificate_downloads(time(NULL));
@@ -217,8 +289,8 @@ router_reload_consensus_networkstatus(void)
 }
 
 /** Free all storage held by the vote_routerstatus object <b>rs</b>. */
-STATIC void
-vote_routerstatus_free(vote_routerstatus_t *rs)
+void
+vote_routerstatus_free_(vote_routerstatus_t *rs)
 {
   vote_microdesc_hash_t *h, *next;
   if (!rs)
@@ -236,7 +308,7 @@ vote_routerstatus_free(vote_routerstatus_t *rs)
 
 /** Free all storage held by the routerstatus object <b>rs</b>. */
 void
-routerstatus_free(routerstatus_t *rs)
+routerstatus_free_(routerstatus_t *rs)
 {
   if (!rs)
     return;
@@ -246,7 +318,7 @@ routerstatus_free(routerstatus_t *rs)
 
 /** Free all storage held in <b>sig</b> */
 void
-document_signature_free(document_signature_t *sig)
+document_signature_free_(document_signature_t *sig)
 {
   tor_free(sig->signature);
   tor_free(sig);
@@ -264,7 +336,7 @@ document_signature_dup(const document_signature_t *sig)
 
 /** Free all storage held in <b>ns</b>. */
 void
-networkstatus_vote_free(networkstatus_t *ns)
+networkstatus_vote_free_(networkstatus_t *ns)
 {
   if (!ns)
     return;
@@ -324,12 +396,15 @@ networkstatus_vote_free(networkstatus_t *ns)
     smartlist_free(ns->routerstatus_list);
   }
 
+  if (ns->bw_file_headers) {
+    SMARTLIST_FOREACH(ns->bw_file_headers, char *, c, tor_free(c));
+    smartlist_free(ns->bw_file_headers);
+  }
+
   digestmap_free(ns->desc_digest_map, NULL);
 
   if (ns->sr_info.commits) {
-    SMARTLIST_FOREACH(ns->sr_info.commits, sr_commit_t *, c,
-                      sr_commit_free(c));
-    smartlist_free(ns->sr_info.commits);
+    dirvote_clear_commits(ns);
   }
   tor_free(ns->sr_info.previous_srv);
   tor_free(ns->sr_info.current_srv);
@@ -353,6 +428,20 @@ networkstatus_get_voter_by_id(networkstatus_t *vote,
   return NULL;
 }
 
+/** Return the signature made by <b>voter</b> using the algorithm
+ * <b>alg</b>, or NULL if none is found. */
+document_signature_t *
+networkstatus_get_voter_sig_by_alg(const networkstatus_voter_info_t *voter,
+                                   digest_algorithm_t alg)
+{
+  if (!voter->sigs)
+    return NULL;
+  SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig,
+                    if (sig->alg == alg)
+                    return sig);
+  return NULL;
+}
+
 /** Check whether the signature <b>sig</b> is correctly signed with the
  * signing key in <b>cert</b>.  Return -1 if <b>cert</b> doesn't match the
  * signing key; otherwise set the good_signature or bad_signature flag on
@@ -538,8 +627,11 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus,
       char *tmp = smartlist_join_strings(list_good, " ", 0, NULL);
       smartlist_add_asprintf(sl,
                    "A consensus needs %d good signatures from recognized "
-                   "authorities for us to accept it. This one has %d (%s).",
-                   n_required, n_good, tmp);
+                   "authorities for us to accept it. "
+                   "This %s one has %d (%s).",
+                   n_required,
+                   networkstatus_get_flavor_name(consensus->flavor),
+                   n_good, tmp);
       tor_free(tmp);
       if (n_no_signature) {
         tmp = smartlist_join_strings(list_no_signature, " ", 0, NULL);
@@ -745,41 +837,6 @@ router_get_consensus_status_by_id(const char *digest)
   return router_get_mutable_consensus_status_by_id(digest);
 }
 
-/** Given a nickname (possibly verbose, possibly a hexadecimal digest), return
- * the corresponding routerstatus_t, or NULL if none exists.  Warn the
- * user if <b>warn_if_unnamed</b> is set, and they have specified a router by
- * nickname, but the Named flag isn't set for that router. */
-const routerstatus_t *
-router_get_consensus_status_by_nickname(const char *nickname,
-                                        int warn_if_unnamed)
-{
-  const node_t *node = node_get_by_nickname(nickname, warn_if_unnamed);
-  if (node)
-    return node->rs;
-  else
-    return NULL;
-}
-
-/** Return the identity digest that's mapped to officially by
- * <b>nickname</b>. */
-const char *
-networkstatus_get_router_digest_by_nickname(const char *nickname)
-{
-  if (!named_server_map)
-    return NULL;
-  return strmap_get_lc(named_server_map, nickname);
-}
-
-/** Return true iff <b>nickname</b> is disallowed from being the nickname
- * of any server. */
-int
-networkstatus_nickname_is_unnamed(const char *nickname)
-{
-  if (!unnamed_server_map)
-    return 0;
-  return strmap_get_lc(unnamed_server_map, nickname) != NULL;
-}
-
 /** How frequently do directory authorities re-download fresh networkstatus
  * documents? */
 #define AUTHORITY_NS_CACHE_INTERVAL (10*60)
@@ -789,8 +846,11 @@ networkstatus_nickname_is_unnamed(const char *nickname)
 #define NONAUTHORITY_NS_CACHE_INTERVAL (60*60)
 
 /** Return true iff, given the options listed in <b>options</b>, <b>flavor</b>
- *  is the flavor of a consensus networkstatus that we would like to fetch. */
-static int
+ *  is the flavor of a consensus networkstatus that we would like to fetch.
+ *
+ * For certificate fetches, use we_want_to_fetch_unknown_auth_certs, and
+ * for serving fetched documents, use directory_caches_dir_info. */
+int
 we_want_to_fetch_flavor(const or_options_t *options, int flavor)
 {
   if (flavor < 0 || flavor > N_CONSENSUS_FLAVORS) {
@@ -812,6 +872,29 @@ we_want_to_fetch_flavor(const or_options_t *options, int flavor)
   return flavor == usable_consensus_flavor();
 }
 
+/** Return true iff, given the options listed in <b>options</b>, we would like
+ * to fetch and store unknown authority certificates.
+ *
+ * For consensus and descriptor fetches, use we_want_to_fetch_flavor, and
+ * for serving fetched certificates, use directory_caches_unknown_auth_certs.
+ */
+int
+we_want_to_fetch_unknown_auth_certs(const or_options_t *options)
+{
+  if (authdir_mode_v3(options) ||
+      directory_caches_unknown_auth_certs((options))) {
+    /* We want to serve all certs to others, regardless if we would use
+     * them ourselves. */
+    return 1;
+  }
+  if (options->FetchUselessDescriptors) {
+    /* Unknown certificates are definitely useless. */
+    return 1;
+  }
+  /* Otherwise, don't fetch unknown certificates. */
+  return 0;
+}
+
 /** How long will we hang onto a possibly live consensus for which we're
  * fetching certs before we check whether there is a better one? */
 #define DELAY_WHILE_FETCHING_CERTS (20*60)
@@ -914,20 +997,20 @@ update_consensus_networkstatus_downloads(time_t now)
       update_consensus_bootstrap_multiple_downloads(now, options);
     } else {
       /* Check if we failed downloading a consensus too recently */
-      int max_dl_tries = options->TestingConsensusMaxDownloadTries;
 
       /* Let's make sure we remembered to update consensus_dl_status */
       tor_assert(consensus_dl_status[i].schedule == DL_SCHED_CONSENSUS);
 
-      if (!download_status_is_ready(&consensus_dl_status[i],
-                                    now,
-                                    max_dl_tries)) {
+      if (!download_status_is_ready(&consensus_dl_status[i], now)) {
         continue;
       }
 
-      /* Check if we're waiting for certificates to download */
-      if (check_consensus_waiting_for_certs(i, now, &consensus_dl_status[i]))
+      /** Check if we're waiting for certificates to download. If we are,
+       * launch download for missing directory authority certificates. */
+      if (check_consensus_waiting_for_certs(i, now, &consensus_dl_status[i])) {
+        update_certificate_downloads(now);
         continue;
+      }
 
       /* Try the requested attempt */
       log_info(LD_DIR, "Launching %s standard networkstatus consensus "
@@ -948,17 +1031,9 @@ update_consensus_networkstatus_downloads(time_t now)
 static void
 update_consensus_bootstrap_attempt_downloads(
                                       time_t now,
-                                      const or_options_t *options,
                                       download_status_t *dls,
                                       download_want_authority_t want_authority)
 {
-  int use_fallbacks = networkstatus_consensus_can_use_extra_fallbacks(options);
-  int max_dl_tries = options->ClientBootstrapConsensusMaxDownloadTries;
-  if (!use_fallbacks) {
-    max_dl_tries =
-              options->ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries;
-  }
-
   const char *resource = networkstatus_get_flavor_name(
                                                   usable_consensus_flavor());
 
@@ -967,7 +1042,7 @@ update_consensus_bootstrap_attempt_downloads(
 
   /* Allow for multiple connections in the same second, if the schedule value
    * is 0. */
-  while (download_status_is_ready(dls, now, max_dl_tries)) {
+  while (download_status_is_ready(dls, now)) {
     log_info(LD_DIR, "Launching %s bootstrap %s networkstatus consensus "
              "download.", resource, (want_authority == DL_WANT_AUTHORITY
                                      ? "authority"
@@ -1018,7 +1093,7 @@ update_consensus_bootstrap_multiple_downloads(time_t now,
 
     if (!check_consensus_waiting_for_certs(usable_flavor, now, dls_f)) {
       /* During bootstrap, DL_WANT_ANY_DIRSERVER means "use fallbacks". */
-      update_consensus_bootstrap_attempt_downloads(now, options, dls_f,
+      update_consensus_bootstrap_attempt_downloads(now, dls_f,
                                                    DL_WANT_ANY_DIRSERVER);
     }
   }
@@ -1028,7 +1103,7 @@ update_consensus_bootstrap_multiple_downloads(time_t now,
     &consensus_bootstrap_dl_status[CONSENSUS_BOOTSTRAP_SOURCE_AUTHORITY];
 
   if (!check_consensus_waiting_for_certs(usable_flavor, now, dls_a)) {
-    update_consensus_bootstrap_attempt_downloads(now, options, dls_a,
+    update_consensus_bootstrap_attempt_downloads(now, dls_a,
                                                  DL_WANT_AUTHORITY);
   }
 }
@@ -1181,8 +1256,18 @@ should_delay_dir_fetches(const or_options_t *options, const char **msg_out)
     return 1;
   }
 
+  if (we_are_hibernating()) {
+    if (msg_out) {
+      *msg_out = "We are hibernating or shutting down.";
+    }
+    log_info(LD_DIR, "Delaying dir fetches (Hibernating or shutting down)");
+    return 1;
+  }
+
   if (options->UseBridges) {
-    if (!any_bridge_descriptors_known()) {
+    /* If we know that none of our bridges can possibly work, avoid fetching
+     * directory documents. But if some of them might work, try again. */
+    if (num_bridges_usable(1) == 0) {
       if (msg_out) {
         *msg_out = "No running bridges";
       }
@@ -1202,16 +1287,20 @@ should_delay_dir_fetches(const or_options_t *options, const char **msg_out)
   return 0;
 }
 
-/** Launch requests for networkstatus documents and authority certificates as
- * appropriate. */
+/** Launch requests for networkstatus documents as appropriate. This is called
+ * when we retry all the connections on a SIGHUP and periodically by a Periodic
+ * event which checks whether we want to download any networkstatus documents.
+ */
 void
 update_networkstatus_downloads(time_t now)
 {
   const or_options_t *options = get_options();
   if (should_delay_dir_fetches(options, NULL))
     return;
+  /** Launch a consensus download request, we will wait for the consensus to
+   * download and when it completes we will launch a certificate download
+   * request. */
   update_consensus_networkstatus_downloads(now);
-  update_certificate_downloads(now);
 }
 
 /** Launch requests as appropriate for missing directory authority
@@ -1318,28 +1407,57 @@ networkstatus_get_latest_consensus_by_flavor,(consensus_flavor_t f))
 MOCK_IMPL(networkstatus_t *,
 networkstatus_get_live_consensus,(time_t now))
 {
-  if (networkstatus_get_latest_consensus() &&
-      networkstatus_get_latest_consensus()->valid_after <= now &&
-      now <= networkstatus_get_latest_consensus()->valid_until)
-    return networkstatus_get_latest_consensus();
+  networkstatus_t *ns = networkstatus_get_latest_consensus();
+  if (ns && networkstatus_is_live(ns, now))
+    return ns;
   else
     return NULL;
 }
 
-/* XXXX remove this in favor of get_live_consensus. But actually,
- * leave something like it for bridge users, who need to not totally
- * lose if they spend a while fetching a new consensus. */
+/** Given a consensus in <b>ns</b>, return true iff currently live and
+ *  unexpired. */
+int
+networkstatus_is_live(const networkstatus_t *ns, time_t now)
+{
+  return (ns->valid_after <= now && now <= ns->valid_until);
+}
+
+/** Determine if <b>consensus</b> is valid or expired recently enough that
+ * we can still use it.
+ *
+ * Return 1 if the consensus is reasonably live, or 0 if it is too old.
+ */
+int
+networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
+                                        time_t now)
+{
+  if (BUG(!consensus))
+    return 0;
+
+  return networkstatus_valid_until_is_reasonably_live(consensus->valid_until,
+                                                      now);
+}
+
+/** As networkstatus_consensus_reasonably_live, but takes a valid_until
+ * time rather than an entire consensus. */
+int
+networkstatus_valid_until_is_reasonably_live(time_t valid_until,
+                                             time_t now)
+{
+#define REASONABLY_LIVE_TIME (24*60*60)
+  return (now <= valid_until + REASONABLY_LIVE_TIME);
+}
+
 /** As networkstatus_get_live_consensus(), but is way more tolerant of expired
  * consensuses. */
-networkstatus_t *
-networkstatus_get_reasonably_live_consensus(time_t now, int flavor)
+MOCK_IMPL(networkstatus_t *,
+networkstatus_get_reasonably_live_consensus,(time_t now, int flavor))
 {
-#define REASONABLY_LIVE_TIME (24*60*60)
   networkstatus_t *consensus =
     networkstatus_get_latest_consensus_by_flavor(flavor);
   if (consensus &&
       consensus->valid_after <= now &&
-      now <= consensus->valid_until+REASONABLY_LIVE_TIME)
+      networkstatus_consensus_reasonably_live(consensus, now))
     return consensus;
   else
     return NULL;
@@ -1439,6 +1557,32 @@ networkstatus_consensus_is_already_downloading(const char *resource)
   return answer;
 }
 
+/* Does the current, reasonably live consensus have IPv6 addresses?
+ * Returns 1 if there is a reasonably live consensus and its consensus method
+ * includes IPv6 addresses in the consensus.
+ * Otherwise, if there is no consensus, or the method does not include IPv6
+ * addresses, returns 0. */
+int
+networkstatus_consensus_has_ipv6(const or_options_t* options)
+{
+  const networkstatus_t *cons = networkstatus_get_reasonably_live_consensus(
+                                                    approx_time(),
+                                                    usable_consensus_flavor());
+
+  /* If we have no consensus, we have no IPv6 in it */
+  if (!cons) {
+    return 0;
+  }
+
+  /* Different flavours of consensus gained IPv6 at different times */
+  if (we_use_microdescriptors_for_circuits(options)) {
+    return
+       cons->consensus_method >= MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS;
+  } else {
+    return 1;
+  }
+}
+
 /** Given two router status entries for the same router identity, return 1 if
  * if the contents have changed between them. Otherwise, return 0. */
 static int
@@ -1503,13 +1647,22 @@ notify_control_networkstatus_changed(const networkstatus_t *old_c,
   smartlist_free(changed);
 }
 
-/* Called when the consensus has changed from old_c to new_c. */
+/* Called before the consensus changes from old_c to new_c. */
 static void
-notify_networkstatus_changed(const networkstatus_t *old_c,
-                             const networkstatus_t *new_c)
+notify_before_networkstatus_changes(const networkstatus_t *old_c,
+                                    const networkstatus_t *new_c)
 {
   notify_control_networkstatus_changed(old_c, new_c);
   dos_consensus_has_changed(new_c);
+  relay_consensus_has_changed(new_c);
+}
+
+/* Called after a new consensus has been put in the global state. It is safe
+ * to use the consensus getters in this function. */
+static void
+notify_after_networkstatus_changes(void)
+{
+  scheduler_notify_networkstatus_changed();
 }
 
 /** Copy all the ancillary information (like router download status and so on)
@@ -1569,23 +1722,7 @@ networkstatus_set_current_consensus_from_ns(networkstatus_t *c,
   }
   return current_md_consensus ? 0 : -1;
 }
-#endif //TOR_UNIT_TESTS
-
-/**
- * Return true if any option is set in <b>options</b> to make us behave
- * as a client.
- *
- * XXXX If we need this elsewhere at any point, we should make it nonstatic
- * XXXX and move it into another file.
- */
-static int
-any_client_port_set(const or_options_t *options)
-{
-  return (options->SocksPort_set ||
-          options->TransPort_set ||
-          options->NATDPort_set ||
-          options->DNSPort_set);
-}
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /**
  * Helper for handle_missing_protocol_warning: handles either the
@@ -1610,7 +1747,7 @@ handle_missing_protocol_warning_impl(const networkstatus_t *c,
   }
   tor_free(protocol_warning);
   if (should_exit)
-    exit(1);
+    exit(1); // XXXX bad exit: should return from main.
 }
 
 /** Called when we have received a networkstatus <b>c</b>. If there are
@@ -1622,7 +1759,7 @@ handle_missing_protocol_warning(const networkstatus_t *c,
                                 const or_options_t *options)
 {
   const int is_server = server_mode(options);
-  const int is_client = any_client_port_set(options) || !is_server;
+  const int is_client = options_any_client_port_set(options) || !is_server;
 
   if (is_server)
     handle_missing_protocol_warning_impl(c, 0);
@@ -1630,6 +1767,57 @@ handle_missing_protocol_warning(const networkstatus_t *c,
     handle_missing_protocol_warning_impl(c, 1);
 }
 
+/**
+ * Check whether we received a consensus that appears to be coming
+ * from the future.  Because we implicitly trust the directory
+ * authorities' idea of the current time, we produce a warning if we
+ * get an early consensus.
+ *
+ * If we got a consensus that is time stamped far in the past, that
+ * could simply have come from a stale cache.  Possible ways to get a
+ * consensus from the future can include:
+ *
+ * - enough directory authorities have wrong clocks
+ * - directory authorities collude to produce misleading time stamps
+ * - our own clock is wrong (this is by far the most likely)
+ *
+ * We neglect highly improbable scenarios that involve actual time
+ * travel.
+ */
+STATIC void
+warn_early_consensus(const networkstatus_t *c, const char *flavor,
+                     time_t now)
+{
+  char tbuf[ISO_TIME_LEN+1];
+  char dbuf[64];
+  long delta = now - c->valid_after;
+  char *flavormsg = NULL;
+
+/** If a consensus appears more than this many seconds before it could
+ * possibly be a sufficiently-signed consensus, declare that our clock
+ * is skewed. */
+#define EARLY_CONSENSUS_NOTICE_SKEW 60
+
+  /* We assume that if a majority of dirauths have accurate clocks,
+   * the earliest that a dirauth with a skewed clock could possibly
+   * publish a sufficiently-signed consensus is (valid_after -
+   * dist_seconds).  Before that time, the skewed dirauth would be
+   * unable to obtain enough authority signatures for the consensus to
+   * be valid. */
+  if (now >= c->valid_after - c->dist_seconds - EARLY_CONSENSUS_NOTICE_SKEW)
+    return;
+
+  format_iso_time(tbuf, c->valid_after);
+  format_time_interval(dbuf, sizeof(dbuf), delta);
+  log_warn(LD_GENERAL, "Our clock is %s behind the time published in the "
+           "consensus network status document (%s UTC).  Tor needs an "
+           "accurate clock to work correctly. Please check your time and "
+           "date settings!", dbuf, tbuf);
+  tor_asprintf(&flavormsg, "%s flavor consensus", flavor);
+  clock_skew_warning(NULL, delta, 1, LD_GENERAL, flavormsg, "CONSENSUS");
+  tor_free(flavormsg);
+}
+
 /** Try to replace the current cached v3 networkstatus with the one in
  * <b>consensus</b>.  If we don't have enough certificates to validate it,
  * store it in consensus_waiting_for_certs and launch a certificate fetch.
@@ -1659,7 +1847,7 @@ networkstatus_set_current_consensus(const char *consensus,
 {
   networkstatus_t *c=NULL;
   int r, result = -1;
-  time_t now = time(NULL);
+  time_t now = approx_time();
   const or_options_t *options = get_options();
   char *unverified_fname = NULL, *consensus_fname = NULL;
   int flav = networkstatus_parse_flavor_name(flavor);
@@ -1672,7 +1860,6 @@ networkstatus_set_current_consensus(const char *consensus,
   consensus_waiting_for_certs_t *waiting = NULL;
   time_t current_valid_after = 0;
   int free_consensus = 1; /* Free 'c' at the end of the function */
-  int old_ewma_enabled;
   int checked_protocols_already = 0;
 
   if (flav < 0) {
@@ -1691,7 +1878,7 @@ networkstatus_set_current_consensus(const char *consensus,
 
   if (from_cache && !was_waiting_for_certs) {
     /* We previously stored this; check _now_ to make sure that version-kills
-     * really work.  This happens even before we check signatures: we did so
+     * really work. This happens even before we check signatures: we did so
      * before when we stored this to disk. This does mean an attacker who can
      * write to the datadir can make us not start: such an attacker could
      * already harm us by replacing our guards, which would be worse. */
@@ -1711,9 +1898,9 @@ networkstatus_set_current_consensus(const char *consensus,
   }
 
   if (flav != usable_consensus_flavor() &&
-      !directory_caches_dir_info(options)) {
-    /* This consensus is totally boring to us: we won't use it, and we won't
-     * serve it.  Drop it. */
+      !we_want_to_fetch_flavor(options, flav)) {
+    /* This consensus is totally boring to us: we won't use it, we didn't want
+     * it, and we won't serve it.  Drop it. */
     goto done;
   }
 
@@ -1724,31 +1911,23 @@ networkstatus_set_current_consensus(const char *consensus,
   }
 
   if (!strcmp(flavor, "ns")) {
-    consensus_fname = get_datadir_fname("cached-consensus");
-    unverified_fname = get_datadir_fname("unverified-consensus");
+    consensus_fname = get_cachedir_fname("cached-consensus");
+    unverified_fname = get_cachedir_fname("unverified-consensus");
     if (current_ns_consensus) {
       current_digests = &current_ns_consensus->digests;
       current_valid_after = current_ns_consensus->valid_after;
     }
   } else if (!strcmp(flavor, "microdesc")) {
-    consensus_fname = get_datadir_fname("cached-microdesc-consensus");
-    unverified_fname = get_datadir_fname("unverified-microdesc-consensus");
+    consensus_fname = get_cachedir_fname("cached-microdesc-consensus");
+    unverified_fname = get_cachedir_fname("unverified-microdesc-consensus");
     if (current_md_consensus) {
       current_digests = &current_md_consensus->digests;
       current_valid_after = current_md_consensus->valid_after;
     }
   } else {
-    cached_dir_t *cur;
-    char buf[128];
-    tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor);
-    consensus_fname = get_datadir_fname(buf);
-    tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor);
-    unverified_fname = get_datadir_fname(buf);
-    cur = dirserv_get_consensus(flavor);
-    if (cur) {
-      current_digests = &cur->digests;
-      current_valid_after = cur->published;
-    }
+    tor_assert_nonfatal_unreached();
+    result = -2;
+    goto done;
   }
 
   if (current_digests &&
@@ -1821,6 +2000,15 @@ networkstatus_set_current_consensus(const char *consensus,
     }
   }
 
+  /* Signatures from the consensus are verified */
+  if (from_cache && was_waiting_for_certs) {
+    /* We check if the consensus is loaded from disk cache and that it
+     * it is an unverified consensus. If it is unverified, rename it to
+     * cached-*-consensus since it has been verified. */
+    log_info(LD_DIR, "Unverified consensus signatures verified.");
+    tor_rename(unverified_fname, consensus_fname);
+  }
+
   if (!from_cache && flav == usable_consensus_flavor())
     control_event_client_status(LOG_NOTICE, "CONSENSUS_ARRIVED");
 
@@ -1834,8 +2022,11 @@ networkstatus_set_current_consensus(const char *consensus,
 
   const int is_usable_flavor = flav == usable_consensus_flavor();
 
+  /* Before we switch to the new consensus, notify that we are about to change
+   * it using the old consensus and the new one. */
   if (is_usable_flavor) {
-    notify_networkstatus_changed(networkstatus_get_latest_consensus(), c);
+    notify_before_networkstatus_changes(networkstatus_get_latest_consensus(),
+                                        c);
   }
   if (flav == FLAV_NS) {
     if (current_ns_consensus) {
@@ -1878,25 +2069,24 @@ networkstatus_set_current_consensus(const char *consensus,
   }
 
   if (is_usable_flavor) {
+    /* Notify that we just changed the consensus so the current global value
+     * can be looked at. */
+    notify_after_networkstatus_changes();
+
+    /* The "current" consensus has just been set and it is a usable flavor so
+     * the first thing we need to do is recalculate the voting schedule static
+     * object so we can use the timings in there needed by some subsystems
+     * such as hidden service and shared random. */
+    voting_schedule_recalculate_timing(options, now);
+    reschedule_dirvote(options);
+
     nodelist_set_consensus(c);
 
     /* XXXXNM Microdescs: needs a non-ns variant. ???? NM*/
     update_consensus_networkstatus_fetch_time(now);
 
-    dirvote_recalculate_timing(options, now);
-    routerstatus_list_update_named_server_map();
-
-    /* Update ewma and adjust policy if needed; first cache the old value */
-    old_ewma_enabled = cell_ewma_enabled();
     /* Change the cell EWMA settings */
-    cell_ewma_set_scale_factor(options, c);
-    /* If we just enabled ewma, set the cmux policy on all active channels */
-    if (cell_ewma_enabled() && !old_ewma_enabled) {
-      channel_set_cmux_policy_everywhere(&ewma_policy);
-    } else if (!cell_ewma_enabled() && old_ewma_enabled) {
-      /* Turn it off everywhere */
-      channel_set_cmux_policy_everywhere(NULL);
-    }
+    cmux_ewma_set_options(options, c);
 
     /* XXXX this call might be unnecessary here: can changing the
      * current consensus really alter our view of any OR's rate limits? */
@@ -1904,6 +2094,7 @@ networkstatus_set_current_consensus(const char *consensus,
 
     circuit_build_times_new_consensus_params(
                                get_circuit_build_times_mutable(), c);
+    channelpadding_new_consensus_params(c);
   }
 
   /* Reset the failure count only if this consensus is actually valid. */
@@ -1914,34 +2105,26 @@ networkstatus_set_current_consensus(const char *consensus,
       download_status_failed(&consensus_dl_status[flav], 0);
   }
 
-  if (directory_caches_dir_info(options)) {
-    dirserv_set_cached_consensus_networkstatus(consensus,
-                                               flavor,
-                                               &c->digests,
-                                               c->valid_after);
+  if (we_want_to_fetch_flavor(options, flav)) {
+    if (dir_server_mode(get_options())) {
+      dirserv_set_cached_consensus_networkstatus(consensus,
+                                                 flavor,
+                                                 &c->digests,
+                                                 c->digest_sha3_as_signed,
+                                                 c->valid_after);
+
+      consdiffmgr_add_consensus(consensus, c);
+    }
   }
 
   if (!from_cache) {
     write_str_to_file(consensus_fname, consensus, 0);
   }
 
-/** If a consensus appears more than this many seconds before its declared
- * valid-after time, declare that our clock is skewed. */
-#define EARLY_CONSENSUS_NOTICE_SKEW 60
+  warn_early_consensus(c, flavor, now);
 
-  if (now < c->valid_after - EARLY_CONSENSUS_NOTICE_SKEW) {
-    char tbuf[ISO_TIME_LEN+1];
-    char dbuf[64];
-    long delta = now - c->valid_after;
-    format_iso_time(tbuf, c->valid_after);
-    format_time_interval(dbuf, sizeof(dbuf), delta);
-    log_warn(LD_GENERAL, "Our clock is %s behind the time published in the "
-             "consensus network status document (%s UTC).  Tor needs an "
-             "accurate clock to work correctly. Please check your time and "
-             "date settings!", dbuf, tbuf);
-    control_event_general_status(LOG_WARN,
-                    "CLOCK_SKEW MIN_SKEW=%ld SOURCE=CONSENSUS", delta);
-  }
+  /* We got a new consesus. Reset our md fetch fail cache */
+  microdesc_reset_outdated_dirservers_list();
 
   router_dir_info_changed();
 
@@ -1966,6 +2149,7 @@ networkstatus_note_certs_arrived(const char *source_dir)
 {
   int i;
   for (i=0; i<N_CONSENSUS_FLAVORS; ++i) {
+    const char *flavor_name = networkstatus_get_flavor_name(i);
     consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i];
     if (!waiting->consensus)
       continue;
@@ -1973,7 +2157,7 @@ networkstatus_note_certs_arrived(const char *source_dir)
       char *waiting_body = waiting->body;
       if (!networkstatus_set_current_consensus(
                                  waiting_body,
-                                 networkstatus_get_flavor_name(i),
+                                 flavor_name,
                                  NSSET_WAS_WAITING_FOR_CERTS,
                                  source_dir)) {
         tor_free(waiting_body);
@@ -2048,31 +2232,6 @@ routers_update_all_from_networkstatus(time_t now, int dir_version)
   }
 }
 
-/** Update our view of the list of named servers from the most recently
- * retrieved networkstatus consensus. */
-static void
-routerstatus_list_update_named_server_map(void)
-{
-  networkstatus_t *ns = networkstatus_get_latest_consensus();
-  if (!ns)
-    return;
-
-  strmap_free(named_server_map, tor_free_);
-  named_server_map = strmap_new();
-  strmap_free(unnamed_server_map, NULL);
-  unnamed_server_map = strmap_new();
-  smartlist_t *rslist = ns->routerstatus_list;
-  SMARTLIST_FOREACH_BEGIN(rslist, const routerstatus_t *, rs) {
-      if (rs->is_named) {
-        strmap_set_lc(named_server_map, rs->nickname,
-                      tor_memdup(rs->identity_digest, DIGEST_LEN));
-      }
-      if (rs->is_unnamed) {
-        strmap_set_lc(unnamed_server_map, rs->nickname, (void*)1);
-      }
-  } SMARTLIST_FOREACH_END(rs);
-}
-
 /** Given a list <b>routers</b> of routerinfo_t *, update each status field
  * according to our current consensus networkstatus.  May re-order
  * <b>routers</b>. */
@@ -2151,7 +2310,9 @@ signed_descs_update_status_from_consensus_networkstatus(smartlist_t *descs)
 char *
 networkstatus_getinfo_helper_single(const routerstatus_t *rs)
 {
-  return routerstatus_format_entry(rs, NULL, NULL, NS_CONTROL_PORT, NULL);
+  return routerstatus_format_entry(rs, NULL, NULL, NS_CONTROL_PORT,
+                                   ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD,
+                                   NULL);
 }
 
 /** Alloc and return a string describing routerstatuses for the most
@@ -2164,13 +2325,13 @@ networkstatus_getinfo_helper_single(const routerstatus_t *rs)
 char *
 networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now)
 {
-  time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
+  const time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
   char *answer;
   routerlist_t *rl = router_get_routerlist();
   smartlist_t *statuses;
-  uint8_t purpose = router_purpose_from_string(purpose_string);
+  const uint8_t purpose = router_purpose_from_string(purpose_string);
   routerstatus_t rs;
-  int bridge_auth = authdir_mode_bridge(get_options());
+  const int bridge_auth = authdir_mode_bridge(get_options());
 
   if (purpose == ROUTER_PURPOSE_UNKNOWN) {
     log_info(LD_DIR, "Unrecognized purpose '%s' when listing router statuses.",
@@ -2187,6 +2348,7 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now)
       continue;
     if (ri->purpose != purpose)
       continue;
+    /* TODO: modifying the running flag in a getinfo is a bad idea */
     if (bridge_auth && ri->purpose == ROUTER_PURPOSE_BRIDGE)
       dirserv_set_router_is_running(ri, now);
     /* then generate and write out status lines for each of them */
@@ -2205,25 +2367,34 @@ void
 networkstatus_dump_bridge_status_to_file(time_t now)
 {
   char *status = networkstatus_getinfo_by_purpose("bridge", now);
-  const or_options_t *options = get_options();
   char *fname = NULL;
   char *thresholds = NULL;
   char *published_thresholds_and_status = NULL;
   char published[ISO_TIME_LEN+1];
+  const routerinfo_t *me = router_get_my_routerinfo();
+  char fingerprint[FINGERPRINT_LEN+1];
+  char *fingerprint_line = NULL;
 
+  if (me && crypto_pk_get_fingerprint(me->identity_pkey,
+                                      fingerprint, 0) >= 0) {
+    tor_asprintf(&fingerprint_line, "fingerprint %s\n", fingerprint);
+  } else {
+    log_warn(LD_BUG, "Error computing fingerprint for bridge status.");
+  }
   format_iso_time(published, now);
   dirserv_compute_bridge_flag_thresholds();
   thresholds = dirserv_get_flag_thresholds_line();
   tor_asprintf(&published_thresholds_and_status,
-               "published %s\nflag-thresholds %s\n%s",
-               published, thresholds, status);
-  tor_asprintf(&fname, "%s"PATH_SEPARATOR"networkstatus-bridges",
-               options->DataDirectory);
+               "published %s\nflag-thresholds %s\n%s%s",
+               published, thresholds, fingerprint_line ? fingerprint_line : "",
+               status);
+  fname = get_datadir_fname("networkstatus-bridges");
   write_str_to_file(fname,published_thresholds_and_status,0);
   tor_free(thresholds);
   tor_free(published_thresholds_and_status);
   tor_free(fname);
   tor_free(status);
+  tor_free(fingerprint_line);
 }
 
 /* DOCDOC get_net_param_from_list */
@@ -2260,6 +2431,8 @@ get_net_param_from_list(smartlist_t *net_params, const char *param_name,
     res = max_val;
   }
 
+  tor_assert(res >= min_val);
+  tor_assert(res <= max_val);
   return res;
 }
 
@@ -2270,9 +2443,9 @@ get_net_param_from_list(smartlist_t *net_params, const char *param_name,
  * Make sure the value parsed from the consensus is at least
  * <b>min_val</b> and at most <b>max_val</b> and raise/cap the parsed value
  * if necessary. */
-int32_t
-networkstatus_get_param(const networkstatus_t *ns, const char *param_name,
-                        int32_t default_val, int32_t min_val, int32_t max_val)
+MOCK_IMPL(int32_t,
+networkstatus_get_param, (const networkstatus_t *ns, const char *param_name,
+                        int32_t default_val, int32_t min_val, int32_t max_val))
 {
   if (!ns) /* if they pass in null, go find it ourselves */
     ns = networkstatus_get_latest_consensus();
@@ -2285,6 +2458,25 @@ networkstatus_get_param(const networkstatus_t *ns, const char *param_name,
 }
 
 /**
+ * As networkstatus_get_param(), but check torrc_value before checking the
+ * consensus. If torrc_value is in-range, then return it instead of the
+ * value from the consensus.
+ */
+int32_t
+networkstatus_get_overridable_param(const networkstatus_t *ns,
+                                    int32_t torrc_value,
+                                    const char *param_name,
+                                    int32_t default_val,
+                                    int32_t min_val, int32_t max_val)
+{
+  if (torrc_value >= min_val && torrc_value <= max_val)
+    return torrc_value;
+  else
+    return networkstatus_get_param(
+                         ns, param_name, default_val, min_val, max_val);
+}
+
+/**
  * Retrieve the consensus parameter that governs the
  * fixed-point precision of our network balancing 'bandwidth-weights'
  * (which are themselves integer consensus values). We divide them
@@ -2360,12 +2552,11 @@ networkstatus_parse_flavor_name(const char *flavname)
  * running, or otherwise not a descriptor that we would make any
  * use of even if we had it. Else return 1. */
 int
-client_would_use_router(const routerstatus_t *rs, time_t now,
-                        const or_options_t *options)
+client_would_use_router(const routerstatus_t *rs, time_t now)
 {
-  if (!rs->is_flagged_running && !options->FetchUselessDescriptors) {
+  if (!rs->is_flagged_running) {
     /* If we had this router descriptor, we wouldn't even bother using it.
-     * But, if we want to have a complete list, fetch it anyway. */
+     * (Fetching and storing depends on by we_want_to_fetch_flavor().) */
     return 0;
   }
   if (rs->published_on + OLD_ROUTER_DESC_MAX_AGE < now) {
@@ -2421,7 +2612,8 @@ getinfo_helper_networkstatus(control_connection_t *conn,
     }
     status = router_get_consensus_status_by_id(d);
   } else if (!strcmpstart(question, "ns/name/")) {
-    status = router_get_consensus_status_by_nickname(question+8, 0);
+    const node_t *n = node_get_by_nickname(question+8, 0);
+    status = n ? n->rs : NULL;
   } else if (!strcmpstart(question, "ns/purpose/")) {
     *answer = networkstatus_getinfo_by_purpose(question+11, time(NULL));
     return *answer ? 0 : -1;
@@ -2528,8 +2720,4 @@ networkstatus_free_all(void)
     }
     tor_free(waiting->body);
   }
-
-  strmap_free(named_server_map, tor_free_);
-  strmap_free(unnamed_server_map, NULL);
 }
-
diff --git a/src/or/networkstatus.h b/src/feature/nodelist/networkstatus.h
index 71f36b69ed..9e7b0f1bb0 100644
--- a/src/or/networkstatus.h
+++ b/src/feature/nodelist/networkstatus.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,16 +12,25 @@
 #ifndef TOR_NETWORKSTATUS_H
 #define TOR_NETWORKSTATUS_H
 
-#include "testsupport.h"
+#include "lib/testsupport/testsupport.h"
 
 void networkstatus_reset_warnings(void);
 void networkstatus_reset_download_failures(void);
+char *networkstatus_read_cached_consensus(const char *flavorname);
 int router_reload_consensus_networkstatus(void);
-void routerstatus_free(routerstatus_t *rs);
-void networkstatus_vote_free(networkstatus_t *ns);
+void routerstatus_free_(routerstatus_t *rs);
+#define routerstatus_free(rs) \
+  FREE_AND_NULL(routerstatus_t, routerstatus_free_, (rs))
+void networkstatus_vote_free_(networkstatus_t *ns);
+#define networkstatus_vote_free(ns) \
+  FREE_AND_NULL(networkstatus_t, networkstatus_vote_free_, (ns))
 networkstatus_voter_info_t *networkstatus_get_voter_by_id(
                                        networkstatus_t *vote,
                                        const char *identity);
+document_signature_t *networkstatus_get_voter_sig_by_alg(
+                                    const networkstatus_voter_info_t *voter,
+                                    digest_algorithm_t alg);
+
 int networkstatus_check_consensus_signature(networkstatus_t *consensus,
                                             int warn);
 int networkstatus_check_document_signature(const networkstatus_t *consensus,
@@ -61,11 +70,8 @@ const routerstatus_t *router_get_consensus_status_by_descriptor_digest(
 MOCK_DECL(routerstatus_t *,
           router_get_mutable_consensus_status_by_descriptor_digest,
           (networkstatus_t *consensus, const char *digest));
-const routerstatus_t *router_get_consensus_status_by_nickname(
-                                   const char *nickname,
-                                   int warn_if_unnamed);
-const char *networkstatus_get_router_digest_by_nickname(const char *nickname);
-int networkstatus_nickname_is_unnamed(const char *nickname);
+int we_want_to_fetch_flavor(const or_options_t *options, int flavor);
+int we_want_to_fetch_unknown_auth_certs(const or_options_t *options);
 void networkstatus_consensus_download_failed(int status_code,
                                              const char *flavname);
 void update_consensus_networkstatus_fetch_time(time_t now);
@@ -73,20 +79,26 @@ int should_delay_dir_fetches(const or_options_t *options,const char **msg_out);
 void update_networkstatus_downloads(time_t now);
 void update_certificate_downloads(time_t now);
 int consensus_is_waiting_for_certs(void);
-int client_would_use_router(const routerstatus_t *rs, time_t now,
-                            const or_options_t *options);
+int client_would_use_router(const routerstatus_t *rs, time_t now);
 MOCK_DECL(networkstatus_t *,networkstatus_get_latest_consensus,(void));
 MOCK_DECL(networkstatus_t *,networkstatus_get_latest_consensus_by_flavor,
           (consensus_flavor_t f));
 MOCK_DECL(networkstatus_t *, networkstatus_get_live_consensus,(time_t now));
-networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now,
-                                                             int flavor);
+int networkstatus_is_live(const networkstatus_t *ns, time_t now);
+int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
+                                            time_t now);
+int networkstatus_valid_until_is_reasonably_live(time_t valid_until,
+                                                 time_t now);
+MOCK_DECL(networkstatus_t *,networkstatus_get_reasonably_live_consensus,
+                                                        (time_t now,
+                                                         int flavor));
 MOCK_DECL(int, networkstatus_consensus_is_bootstrapping,(time_t now));
 int networkstatus_consensus_can_use_multiple_directories(
                                                 const or_options_t *options);
 MOCK_DECL(int, networkstatus_consensus_can_use_extra_fallbacks,(
                                                 const or_options_t *options));
 int networkstatus_consensus_is_already_downloading(const char *resource);
+int networkstatus_consensus_has_ipv6(const or_options_t* options);
 
 #define NSSET_FROM_CACHE 1
 #define NSSET_WAS_WAITING_FOR_CERTS 2
@@ -107,10 +119,14 @@ void signed_descs_update_status_from_consensus_networkstatus(
 char *networkstatus_getinfo_helper_single(const routerstatus_t *rs);
 char *networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now);
 void networkstatus_dump_bridge_status_to_file(time_t now);
-int32_t networkstatus_get_param(const networkstatus_t *ns,
-                                const char *param_name,
-                                int32_t default_val, int32_t min_val,
-                                int32_t max_val);
+MOCK_DECL(int32_t, networkstatus_get_param,
+          (const networkstatus_t *ns, const char *param_name,
+           int32_t default_val, int32_t min_val, int32_t max_val));
+int32_t networkstatus_get_overridable_param(const networkstatus_t *ns,
+                                            int32_t torrc_value,
+                                            const char *param_name,
+                                            int32_t default_val,
+                                            int32_t min_val, int32_t max_val);
 int getinfo_helper_networkstatus(control_connection_t *conn,
                                  const char *question, char **answer,
                                  const char **errmsg);
@@ -118,18 +134,27 @@ int32_t networkstatus_get_bw_weight(networkstatus_t *ns, const char *weight,
                                     int32_t default_val);
 const char *networkstatus_get_flavor_name(consensus_flavor_t flav);
 int networkstatus_parse_flavor_name(const char *flavname);
-void document_signature_free(document_signature_t *sig);
+void document_signature_free_(document_signature_t *sig);
+#define document_signature_free(sig) \
+  FREE_AND_NULL(document_signature_t, document_signature_free_, (sig))
 document_signature_t *document_signature_dup(const document_signature_t *sig);
 void networkstatus_free_all(void);
 int networkstatus_get_weight_scale_param(networkstatus_t *ns);
 
+void vote_routerstatus_free_(vote_routerstatus_t *rs);
+#define vote_routerstatus_free(rs) \
+  FREE_AND_NULL(vote_routerstatus_t, vote_routerstatus_free_, (rs))
+
 #ifdef NETWORKSTATUS_PRIVATE
-STATIC void vote_routerstatus_free(vote_routerstatus_t *rs);
 #ifdef TOR_UNIT_TESTS
 STATIC int networkstatus_set_current_consensus_from_ns(networkstatus_t *c,
                                                 const char *flavor);
-#endif // TOR_UNIT_TESTS
-#endif
+STATIC void warn_early_consensus(const networkstatus_t *c, const char *flavor,
+                                 time_t now);
+extern networkstatus_t *current_ns_consensus;
+extern networkstatus_t *current_md_consensus;
+#endif /* defined(TOR_UNIT_TESTS) */
+#endif /* defined(NETWORKSTATUS_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_NETWORKSTATUS_H) */
 
diff --git a/src/feature/nodelist/networkstatus_sr_info_st.h b/src/feature/nodelist/networkstatus_sr_info_st.h
new file mode 100644
index 0000000000..677d8ed811
--- /dev/null
+++ b/src/feature/nodelist/networkstatus_sr_info_st.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef NETWORKSTATUS_SR_INFO_ST_H
+#define NETWORKSTATUS_SR_INFO_ST_H
+
+struct networkstatus_sr_info_t {
+  /* Indicate if the dirauth partitipates in the SR protocol with its vote.
+   * This is tied to the SR flag in the vote. */
+  unsigned int participate:1;
+  /* Both vote and consensus: Current and previous SRV. If list is empty,
+   * this means none were found in either the consensus or vote. */
+  struct sr_srv_t *previous_srv;
+  struct sr_srv_t *current_srv;
+  /* Vote only: List of commitments. */
+  smartlist_t *commits;
+};
+
+#endif
+
diff --git a/src/feature/nodelist/networkstatus_st.h b/src/feature/nodelist/networkstatus_st.h
new file mode 100644
index 0000000000..6160f12361
--- /dev/null
+++ b/src/feature/nodelist/networkstatus_st.h
@@ -0,0 +1,104 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef NETWORKSTATUS_ST_H
+#define NETWORKSTATUS_ST_H
+
+#include "feature/nodelist/networkstatus_sr_info_st.h"
+
+/** Enumerates the possible seriousness values of a networkstatus document. */
+typedef enum networkstatus_type_t {
+  NS_TYPE_VOTE,
+  NS_TYPE_CONSENSUS,
+  NS_TYPE_OPINION,
+} networkstatus_type_t;
+
+/** A common structure to hold a v3 network status vote, or a v3 network
+ * status consensus. */
+struct networkstatus_t {
+  networkstatus_type_t type; /**< Vote, consensus, or opinion? */
+  consensus_flavor_t flavor; /**< If a consensus, what kind? */
+  unsigned int has_measured_bws : 1;/**< True iff this networkstatus contains
+                                     * measured= bandwidth values. */
+
+  time_t published; /**< Vote only: Time when vote was written. */
+  time_t valid_after; /**< Time after which this vote or consensus applies. */
+  time_t fresh_until; /**< Time before which this is the most recent vote or
+                       * consensus. */
+  time_t valid_until; /**< Time after which this vote or consensus should not
+                       * be used. */
+
+  /** Consensus only: what method was used to produce this consensus? */
+  int consensus_method;
+  /** Vote only: what methods is this voter willing to use? */
+  smartlist_t *supported_methods;
+
+  /** List of 'package' lines describing hashes of downloadable packages */
+  smartlist_t *package_lines;
+
+  /** How long does this vote/consensus claim that authorities take to
+   * distribute their votes to one another? */
+  int vote_seconds;
+  /** How long does this vote/consensus claim that authorities take to
+   * distribute their consensus signatures to one another? */
+  int dist_seconds;
+
+  /** Comma-separated list of recommended client software, or NULL if this
+   * voter has no opinion. */
+  char *client_versions;
+  char *server_versions;
+
+  /** Lists of subprotocol versions which are _recommended_ for relays and
+   * clients, or which are _require_ for relays and clients. Tor shouldn't
+   * make any more network connections if a required protocol is missing.
+   */
+  char *recommended_relay_protocols;
+  char *recommended_client_protocols;
+  char *required_relay_protocols;
+  char *required_client_protocols;
+
+  /** List of flags that this vote/consensus applies to routers.  If a flag is
+   * not listed here, the voter has no opinion on what its value should be. */
+  smartlist_t *known_flags;
+
+  /** List of key=value strings for the parameters in this vote or
+   * consensus, sorted by key. */
+  smartlist_t *net_params;
+
+  /** List of key=value strings for the bw weight parameters in the
+   * consensus. */
+  smartlist_t *weight_params;
+
+  /** List of networkstatus_voter_info_t.  For a vote, only one element
+   * is included.  For a consensus, one element is included for every voter
+   * whose vote contributed to the consensus. */
+  smartlist_t *voters;
+
+  struct authority_cert_t *cert; /**< Vote only: the voter's certificate. */
+
+  /** Digests of this document, as signed. */
+  common_digests_t digests;
+  /** A SHA3-256 digest of the document, not including signatures: used for
+   * consensus diffs */
+  uint8_t digest_sha3_as_signed[DIGEST256_LEN];
+
+  /** List of router statuses, sorted by identity digest.  For a vote,
+   * the elements are vote_routerstatus_t; for a consensus, the elements
+   * are routerstatus_t. */
+  smartlist_t *routerstatus_list;
+
+  /** If present, a map from descriptor digest to elements of
+   * routerstatus_list. */
+  digestmap_t *desc_digest_map;
+
+  /** Contains the shared random protocol data from a vote or consensus. */
+  networkstatus_sr_info_t sr_info;
+
+  /** List of key=value strings from the headers of the bandwidth list file */
+  smartlist_t *bw_file_headers;
+};
+
+#endif
diff --git a/src/feature/nodelist/networkstatus_voter_info_st.h b/src/feature/nodelist/networkstatus_voter_info_st.h
new file mode 100644
index 0000000000..4037fcdeca
--- /dev/null
+++ b/src/feature/nodelist/networkstatus_voter_info_st.h
@@ -0,0 +1,30 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef NETWORKSTATUS_VOTER_INFO_ST_H
+#define NETWORKSTATUS_VOTER_INFO_ST_H
+
+/** Information about a single voter in a vote or a consensus. */
+struct networkstatus_voter_info_t {
+  /** Declared SHA-1 digest of this voter's identity key */
+  char identity_digest[DIGEST_LEN];
+  char *nickname; /**< Nickname of this voter */
+  /** Digest of this voter's "legacy" identity key, if any.  In vote only; for
+   * consensuses, we treat legacy keys as additional signers. */
+  char legacy_id_digest[DIGEST_LEN];
+  char *address; /**< Address of this voter, in string format. */
+  uint32_t addr; /**< Address of this voter, in IPv4, in host order. */
+  uint16_t dir_port; /**< Directory port of this voter */
+  uint16_t or_port; /**< OR port of this voter */
+  char *contact; /**< Contact information for this voter. */
+  char vote_digest[DIGEST_LEN]; /**< Digest of this voter's vote, as signed. */
+
+  /* Nothing from here on is signed. */
+  /** The signature of the document and the signature's status. */
+  smartlist_t *sigs;
+};
+
+#endif
diff --git a/src/feature/nodelist/nickname.c b/src/feature/nodelist/nickname.c
new file mode 100644
index 0000000000..5378b749ca
--- /dev/null
+++ b/src/feature/nodelist/nickname.c
@@ -0,0 +1,62 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file nickname.c
+ * \brief Check and manipulate relay nicknames.
+ */
+
+#include "core/or/or.h"
+#include "feature/nodelist/nickname.h"
+
+/** Return true iff <b>s</b> is a valid server nickname. (That is, a string
+ * containing between 1 and MAX_NICKNAME_LEN characters from
+ * LEGAL_NICKNAME_CHARACTERS.) */
+int
+is_legal_nickname(const char *s)
+{
+  size_t len;
+  tor_assert(s);
+  len = strlen(s);
+  return len > 0 && len <= MAX_NICKNAME_LEN &&
+    strspn(s,LEGAL_NICKNAME_CHARACTERS) == len;
+}
+
+/** Return true iff <b>s</b> is a valid server nickname or
+ * hex-encoded identity-key digest. */
+int
+is_legal_nickname_or_hexdigest(const char *s)
+{
+  if (*s!='$')
+    return is_legal_nickname(s);
+  else
+    return is_legal_hexdigest(s);
+}
+
+/** Return true iff <b>s</b> is a valid hex-encoded identity-key
+ * digest. (That is, an optional $, followed by 40 hex characters,
+ * followed by either nothing, or = or ~ followed by a nickname, or
+ * a character other than =, ~, or a hex character.)
+ */
+int
+is_legal_hexdigest(const char *s)
+{
+  size_t len;
+  tor_assert(s);
+  if (s[0] == '$') s++;
+  len = strlen(s);
+  if (len > HEX_DIGEST_LEN) {
+    if (s[HEX_DIGEST_LEN] == '=' ||
+        s[HEX_DIGEST_LEN] == '~') {
+      if (!is_legal_nickname(s+HEX_DIGEST_LEN+1))
+        return 0;
+    } else {
+      return 0;
+    }
+  }
+  return (len >= HEX_DIGEST_LEN &&
+          strspn(s,HEX_CHARACTERS)==HEX_DIGEST_LEN);
+}
diff --git a/src/feature/nodelist/nickname.h b/src/feature/nodelist/nickname.h
new file mode 100644
index 0000000000..9bdc6b50e8
--- /dev/null
+++ b/src/feature/nodelist/nickname.h
@@ -0,0 +1,19 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file nickname.h
+ * \brief Header file for nickname.c.
+ **/
+
+#ifndef TOR_NICKNAME_H
+#define TOR_NICKNAME_H
+
+int is_legal_nickname(const char *s);
+int is_legal_nickname_or_hexdigest(const char *s);
+int is_legal_hexdigest(const char *s);
+
+#endif
diff --git a/src/feature/nodelist/node_select.c b/src/feature/nodelist/node_select.c
new file mode 100644
index 0000000000..e31abb247f
--- /dev/null
+++ b/src/feature/nodelist/node_select.c
@@ -0,0 +1,1111 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file node_select.c
+ * \brief Code to choose nodes randomly based on restrictions and
+ *   weighted probabilities.
+ **/
+
+#define NODE_SELECT_PRIVATE
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/policies.h"
+#include "core/or/reasons.h"
+#include "feature/client/entrynodes.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dircommon/directory.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/node_select.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/math/fp.h"
+
+#include "feature/dirclient/dir_server_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+static int compute_weighted_bandwidths(const smartlist_t *sl,
+                                       bandwidth_weight_rule_t rule,
+                                       double **bandwidths_out,
+                                       double *total_bandwidth_out);
+static const routerstatus_t *router_pick_trusteddirserver_impl(
+                const smartlist_t *sourcelist, dirinfo_type_t auth,
+                int flags, int *n_busy_out);
+static const routerstatus_t *router_pick_dirserver_generic(
+                              smartlist_t *sourcelist,
+                              dirinfo_type_t type, int flags);
+
+/** Try to find a running dirserver that supports operations of <b>type</b>.
+ *
+ * If there are no running dirservers in our routerlist and the
+ * <b>PDS_RETRY_IF_NO_SERVERS</b> flag is set, set all the fallback ones
+ * (including authorities) as running again, and pick one.
+ *
+ * If the <b>PDS_IGNORE_FASCISTFIREWALL</b> flag is set, then include
+ * dirservers that we can't reach.
+ *
+ * If the <b>PDS_ALLOW_SELF</b> flag is not set, then don't include ourself
+ * (if we're a dirserver).
+ *
+ * Don't pick a fallback directory mirror if any non-fallback is viable;
+ * (the fallback directory mirrors include the authorities)
+ * try to avoid using servers that have returned 503 recently.
+ */
+const routerstatus_t *
+router_pick_directory_server(dirinfo_type_t type, int flags)
+{
+  int busy = 0;
+  const routerstatus_t *choice;
+
+  choice = router_pick_directory_server_impl(type, flags, &busy);
+  if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS))
+    return choice;
+
+  if (busy) {
+    /* If the reason that we got no server is that servers are "busy",
+     * we must be excluding good servers because we already have serverdesc
+     * fetches with them.  Do not mark down servers up because of this. */
+    tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
+                         PDS_NO_EXISTING_MICRODESC_FETCH)));
+    return NULL;
+  }
+
+  log_info(LD_DIR,
+           "No reachable router entries for dirservers. "
+           "Trying them all again.");
+  /* mark all fallback directory mirrors as up again */
+  router_reset_status_download_failures();
+  /* try again */
+  choice = router_pick_directory_server_impl(type, flags, NULL);
+  return choice;
+}
+
+/** Try to find a running fallback directory. Flags are as for
+ * router_pick_directory_server.
+ */
+const routerstatus_t *
+router_pick_dirserver_generic(smartlist_t *sourcelist,
+                              dirinfo_type_t type, int flags)
+{
+  const routerstatus_t *choice;
+  int busy = 0;
+
+  if (smartlist_len(sourcelist) == 1) {
+    /* If there's only one choice, then we should disable the logic that
+     * would otherwise prevent us from choosing ourself. */
+    flags |= PDS_ALLOW_SELF;
+  }
+
+  choice = router_pick_trusteddirserver_impl(sourcelist, type, flags, &busy);
+  if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS))
+    return choice;
+  if (busy) {
+    /* If the reason that we got no server is that servers are "busy",
+     * we must be excluding good servers because we already have serverdesc
+     * fetches with them.  Do not mark down servers up because of this. */
+    tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
+                         PDS_NO_EXISTING_MICRODESC_FETCH)));
+    return NULL;
+  }
+
+  log_info(LD_DIR,
+           "No dirservers are reachable. Trying them all again.");
+  mark_all_dirservers_up(sourcelist);
+  return router_pick_trusteddirserver_impl(sourcelist, type, flags, NULL);
+}
+
+/* Common retry code for router_pick_directory_server_impl and
+ * router_pick_trusteddirserver_impl. Retry with the non-preferred IP version.
+ * Must be called before RETRY_WITHOUT_EXCLUDE().
+ *
+ * If we got no result, and we are applying IP preferences, and we are a
+ * client that could use an alternate IP version, try again with the
+ * opposite preferences. */
+#define RETRY_ALTERNATE_IP_VERSION(retry_label)                               \
+  STMT_BEGIN                                                                  \
+    if (result == NULL && try_ip_pref && options->ClientUseIPv4               \
+        && fascist_firewall_use_ipv6(options) && !server_mode(options)        \
+        && !n_busy) {                                                         \
+      n_excluded = 0;                                                         \
+      n_busy = 0;                                                             \
+      try_ip_pref = 0;                                                        \
+      goto retry_label;                                                       \
+    }                                                                         \
+  STMT_END                                                                    \
+
+/* Common retry code for router_pick_directory_server_impl and
+ * router_pick_trusteddirserver_impl. Retry without excluding nodes, but with
+ * the preferred IP version. Must be called after RETRY_ALTERNATE_IP_VERSION().
+ *
+ * If we got no result, and we are excluding nodes, and StrictNodes is
+ * not set, try again without excluding nodes. */
+#define RETRY_WITHOUT_EXCLUDE(retry_label)                                    \
+  STMT_BEGIN                                                                  \
+    if (result == NULL && try_excluding && !options->StrictNodes              \
+        && n_excluded && !n_busy) {                                           \
+      try_excluding = 0;                                                      \
+      n_excluded = 0;                                                         \
+      n_busy = 0;                                                             \
+      try_ip_pref = 1;                                                        \
+      goto retry_label;                                                       \
+    }                                                                         \
+  STMT_END
+
+/* Common code used in the loop within router_pick_directory_server_impl and
+ * router_pick_trusteddirserver_impl.
+ *
+ * Check if the given <b>identity</b> supports extrainfo. If not, skip further
+ * checks.
+ */
+#define SKIP_MISSING_TRUSTED_EXTRAINFO(type, identity)                        \
+  STMT_BEGIN                                                                  \
+    int is_trusted_extrainfo = router_digest_is_trusted_dir_type(             \
+                               (identity), EXTRAINFO_DIRINFO);                \
+    if (((type) & EXTRAINFO_DIRINFO) &&                                       \
+        !router_supports_extrainfo((identity), is_trusted_extrainfo))         \
+      continue;                                                               \
+  STMT_END
+
+#ifndef LOG_FALSE_POSITIVES_DURING_BOOTSTRAP
+#define LOG_FALSE_POSITIVES_DURING_BOOTSTRAP 0
+#endif
+
+/* Log a message if rs is not found or not a preferred address */
+static void
+router_picked_poor_directory_log(const routerstatus_t *rs)
+{
+  const networkstatus_t *usable_consensus;
+  usable_consensus = networkstatus_get_reasonably_live_consensus(time(NULL),
+                                                 usable_consensus_flavor());
+
+#if !LOG_FALSE_POSITIVES_DURING_BOOTSTRAP
+  /* Don't log early in the bootstrap process, it's normal to pick from a
+   * small pool of nodes. Of course, this won't help if we're trying to
+   * diagnose bootstrap issues. */
+  if (!smartlist_len(nodelist_get_list()) || !usable_consensus
+      || !router_have_minimum_dir_info()) {
+    return;
+  }
+#endif /* !LOG_FALSE_POSITIVES_DURING_BOOTSTRAP */
+
+  /* We couldn't find a node, or the one we have doesn't fit our preferences.
+   * Sometimes this is normal, sometimes it can be a reachability issue. */
+  if (!rs) {
+    /* This happens a lot, so it's at debug level */
+    log_debug(LD_DIR, "Wanted to make an outgoing directory connection, but "
+              "we couldn't find a directory that fit our criteria. "
+              "Perhaps we will succeed next time with less strict criteria.");
+  } else if (!fascist_firewall_allows_rs(rs, FIREWALL_OR_CONNECTION, 1)
+             && !fascist_firewall_allows_rs(rs, FIREWALL_DIR_CONNECTION, 1)
+             ) {
+    /* This is rare, and might be interesting to users trying to diagnose
+     * connection issues on dual-stack machines. */
+    log_info(LD_DIR, "Selected a directory %s with non-preferred OR and Dir "
+             "addresses for launching an outgoing connection: "
+             "IPv4 %s OR %d Dir %d IPv6 %s OR %d Dir %d",
+             routerstatus_describe(rs),
+             fmt_addr32(rs->addr), rs->or_port,
+             rs->dir_port, fmt_addr(&rs->ipv6_addr),
+             rs->ipv6_orport, rs->dir_port);
+  }
+}
+
+#undef LOG_FALSE_POSITIVES_DURING_BOOTSTRAP
+
+/* Check if we already have a directory fetch from ap, for serverdesc
+ * (including extrainfo) or microdesc documents.
+ * If so, return 1, if not, return 0.
+ * Also returns 0 if addr is NULL, tor_addr_is_null(addr), or dir_port is 0.
+ */
+STATIC int
+router_is_already_dir_fetching(const tor_addr_port_t *ap, int serverdesc,
+                               int microdesc)
+{
+  if (!ap || tor_addr_is_null(&ap->addr) || !ap->port) {
+    return 0;
+  }
+
+  /* XX/teor - we're not checking tunnel connections here, see #17848
+   */
+  if (serverdesc && (
+     connection_get_by_type_addr_port_purpose(
+       CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_SERVERDESC)
+  || connection_get_by_type_addr_port_purpose(
+       CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_EXTRAINFO))) {
+    return 1;
+  }
+
+  if (microdesc && (
+     connection_get_by_type_addr_port_purpose(
+       CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_MICRODESC))) {
+    return 1;
+  }
+
+  return 0;
+}
+
+/* Check if we already have a directory fetch from the ipv4 or ipv6
+ * router, for serverdesc (including extrainfo) or microdesc documents.
+ * If so, return 1, if not, return 0.
+ */
+static int
+router_is_already_dir_fetching_(uint32_t ipv4_addr,
+                                const tor_addr_t *ipv6_addr,
+                                uint16_t dir_port,
+                                int serverdesc,
+                                int microdesc)
+{
+  tor_addr_port_t ipv4_dir_ap, ipv6_dir_ap;
+
+  /* Assume IPv6 DirPort is the same as IPv4 DirPort */
+  tor_addr_from_ipv4h(&ipv4_dir_ap.addr, ipv4_addr);
+  ipv4_dir_ap.port = dir_port;
+  tor_addr_copy(&ipv6_dir_ap.addr, ipv6_addr);
+  ipv6_dir_ap.port = dir_port;
+
+  return (router_is_already_dir_fetching(&ipv4_dir_ap, serverdesc, microdesc)
+       || router_is_already_dir_fetching(&ipv6_dir_ap, serverdesc, microdesc));
+}
+
+/** Pick a random running valid directory server/mirror from our
+ * routerlist.  Arguments are as for router_pick_directory_server(), except:
+ *
+ * If <b>n_busy_out</b> is provided, set *<b>n_busy_out</b> to the number of
+ * directories that we excluded for no other reason than
+ * PDS_NO_EXISTING_SERVERDESC_FETCH or PDS_NO_EXISTING_MICRODESC_FETCH.
+ */
+STATIC const routerstatus_t *
+router_pick_directory_server_impl(dirinfo_type_t type, int flags,
+                                  int *n_busy_out)
+{
+  const or_options_t *options = get_options();
+  const node_t *result;
+  smartlist_t *direct, *tunnel;
+  smartlist_t *trusted_direct, *trusted_tunnel;
+  smartlist_t *overloaded_direct, *overloaded_tunnel;
+  time_t now = time(NULL);
+  const networkstatus_t *consensus = networkstatus_get_latest_consensus();
+  const int requireother = ! (flags & PDS_ALLOW_SELF);
+  const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL);
+  const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH);
+  const int no_microdesc_fetching = (flags & PDS_NO_EXISTING_MICRODESC_FETCH);
+  int try_excluding = 1, n_excluded = 0, n_busy = 0;
+  int try_ip_pref = 1;
+
+  if (!consensus)
+    return NULL;
+
+ retry_search:
+
+  direct = smartlist_new();
+  tunnel = smartlist_new();
+  trusted_direct = smartlist_new();
+  trusted_tunnel = smartlist_new();
+  overloaded_direct = smartlist_new();
+  overloaded_tunnel = smartlist_new();
+
+  const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref);
+  const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref);
+  const int must_have_or = directory_must_use_begindir(options);
+
+  /* Find all the running dirservers we know about. */
+  SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) {
+    int is_trusted;
+    int is_overloaded;
+    const routerstatus_t *status = node->rs;
+    const country_t country = node->country;
+    if (!status)
+      continue;
+
+    if (!node->is_running || !node_is_dir(node) || !node->is_valid)
+      continue;
+    if (requireother && router_digest_is_me(node->identity))
+      continue;
+
+    SKIP_MISSING_TRUSTED_EXTRAINFO(type, node->identity);
+
+    if (try_excluding &&
+        routerset_contains_routerstatus(options->ExcludeNodes, status,
+                                        country)) {
+      ++n_excluded;
+      continue;
+    }
+
+    if (router_is_already_dir_fetching_(status->addr,
+                                        &status->ipv6_addr,
+                                        status->dir_port,
+                                        no_serverdesc_fetching,
+                                        no_microdesc_fetching)) {
+      ++n_busy;
+      continue;
+    }
+
+    is_overloaded = status->last_dir_503_at + DIR_503_TIMEOUT > now;
+    is_trusted = router_digest_is_trusted_dir(node->identity);
+
+    /* Clients use IPv6 addresses if the server has one and the client
+     * prefers IPv6.
+     * Add the router if its preferred address and port are reachable.
+     * If we don't get any routers, we'll try again with the non-preferred
+     * address for each router (if any). (To ensure correct load-balancing
+     * we try routers that only have one address both times.)
+     */
+    if (!fascistfirewall || skip_or_fw ||
+        fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION,
+                                     try_ip_pref))
+      smartlist_add(is_trusted ? trusted_tunnel :
+                    is_overloaded ? overloaded_tunnel : tunnel, (void*)node);
+    else if (!must_have_or && (skip_dir_fw ||
+             fascist_firewall_allows_node(node, FIREWALL_DIR_CONNECTION,
+                                          try_ip_pref)))
+      smartlist_add(is_trusted ? trusted_direct :
+                    is_overloaded ? overloaded_direct : direct, (void*)node);
+  } SMARTLIST_FOREACH_END(node);
+
+  if (smartlist_len(tunnel)) {
+    result = node_sl_choose_by_bandwidth(tunnel, WEIGHT_FOR_DIR);
+  } else if (smartlist_len(overloaded_tunnel)) {
+    result = node_sl_choose_by_bandwidth(overloaded_tunnel,
+                                                 WEIGHT_FOR_DIR);
+  } else if (smartlist_len(trusted_tunnel)) {
+    /* FFFF We don't distinguish between trusteds and overloaded trusteds
+     * yet. Maybe one day we should. */
+    /* FFFF We also don't load balance over authorities yet. I think this
+     * is a feature, but it could easily be a bug. -RD */
+    result = smartlist_choose(trusted_tunnel);
+  } else if (smartlist_len(direct)) {
+    result = node_sl_choose_by_bandwidth(direct, WEIGHT_FOR_DIR);
+  } else if (smartlist_len(overloaded_direct)) {
+    result = node_sl_choose_by_bandwidth(overloaded_direct,
+                                         WEIGHT_FOR_DIR);
+  } else {
+    result = smartlist_choose(trusted_direct);
+  }
+  smartlist_free(direct);
+  smartlist_free(tunnel);
+  smartlist_free(trusted_direct);
+  smartlist_free(trusted_tunnel);
+  smartlist_free(overloaded_direct);
+  smartlist_free(overloaded_tunnel);
+
+  RETRY_ALTERNATE_IP_VERSION(retry_search);
+
+  RETRY_WITHOUT_EXCLUDE(retry_search);
+
+  if (n_busy_out)
+    *n_busy_out = n_busy;
+
+  router_picked_poor_directory_log(result ? result->rs : NULL);
+
+  return result ? result->rs : NULL;
+}
+
+/** Given an array of double/uint64_t unions that are currently being used as
+ * doubles, convert them to uint64_t, and try to scale them linearly so as to
+ * much of the range of uint64_t. If <b>total_out</b> is provided, set it to
+ * the sum of all elements in the array _before_ scaling. */
+STATIC void
+scale_array_elements_to_u64(uint64_t *entries_out, const double *entries_in,
+                            int n_entries,
+                            uint64_t *total_out)
+{
+  double total = 0.0;
+  double scale_factor = 0.0;
+  int i;
+
+  for (i = 0; i < n_entries; ++i)
+    total += entries_in[i];
+
+  if (total > 0.0) {
+    scale_factor = ((double)INT64_MAX) / total;
+    scale_factor /= 4.0; /* make sure we're very far away from overflowing */
+  }
+
+  for (i = 0; i < n_entries; ++i)
+    entries_out[i] = tor_llround(entries_in[i] * scale_factor);
+
+  if (total_out)
+    *total_out = (uint64_t) total;
+}
+
+/** Pick a random element of <b>n_entries</b>-element array <b>entries</b>,
+ * choosing each element with a probability proportional to its (uint64_t)
+ * value, and return the index of that element.  If all elements are 0, choose
+ * an index at random. Return -1 on error.
+ */
+STATIC int
+choose_array_element_by_weight(const uint64_t *entries, int n_entries)
+{
+  int i;
+  uint64_t rand_val;
+  uint64_t total = 0;
+
+  for (i = 0; i < n_entries; ++i)
+    total += entries[i];
+
+  if (n_entries < 1)
+    return -1;
+
+  if (total == 0)
+    return crypto_rand_int(n_entries);
+
+  tor_assert(total < INT64_MAX);
+
+  rand_val = crypto_rand_uint64(total);
+
+  return select_array_member_cumulative_timei(
+                           entries, n_entries, total, rand_val);
+}
+
+/** Return bw*1000, unless bw*1000 would overflow, in which case return
+ * INT32_MAX. */
+static inline int32_t
+kb_to_bytes(uint32_t bw)
+{
+  return (bw > (INT32_MAX/1000)) ? INT32_MAX : bw*1000;
+}
+
+/** Helper function:
+ * choose a random element of smartlist <b>sl</b> of nodes, weighted by
+ * the advertised bandwidth of each element using the consensus
+ * bandwidth weights.
+ *
+ * If <b>rule</b>==WEIGHT_FOR_EXIT. we're picking an exit node: consider all
+ * nodes' bandwidth equally regardless of their Exit status, since there may
+ * be some in the list because they exit to obscure ports. If
+ * <b>rule</b>==NO_WEIGHTING, we're picking a non-exit node: weight
+ * exit-node's bandwidth less depending on the smallness of the fraction of
+ * Exit-to-total bandwidth.  If <b>rule</b>==WEIGHT_FOR_GUARD, we're picking a
+ * guard node: consider all guard's bandwidth equally. Otherwise, weight
+ * guards proportionally less.
+ */
+static const node_t *
+smartlist_choose_node_by_bandwidth_weights(const smartlist_t *sl,
+                                           bandwidth_weight_rule_t rule)
+{
+  double *bandwidths_dbl=NULL;
+  uint64_t *bandwidths_u64=NULL;
+
+  if (compute_weighted_bandwidths(sl, rule, &bandwidths_dbl, NULL) < 0)
+    return NULL;
+
+  bandwidths_u64 = tor_calloc(smartlist_len(sl), sizeof(uint64_t));
+  scale_array_elements_to_u64(bandwidths_u64, bandwidths_dbl,
+                              smartlist_len(sl), NULL);
+
+  {
+    int idx = choose_array_element_by_weight(bandwidths_u64,
+                                             smartlist_len(sl));
+    tor_free(bandwidths_dbl);
+    tor_free(bandwidths_u64);
+    return idx < 0 ? NULL : smartlist_get(sl, idx);
+  }
+}
+
+/** When weighting bridges, enforce these values as lower and upper
+ * bound for believable bandwidth, because there is no way for us
+ * to verify a bridge's bandwidth currently. */
+#define BRIDGE_MIN_BELIEVABLE_BANDWIDTH 20000  /* 20 kB/sec */
+#define BRIDGE_MAX_BELIEVABLE_BANDWIDTH 100000 /* 100 kB/sec */
+
+/** Return the smaller of the router's configured BandwidthRate
+ * and its advertised capacity, making sure to stay within the
+ * interval between bridge-min-believe-bw and
+ * bridge-max-believe-bw. */
+static uint32_t
+bridge_get_advertised_bandwidth_bounded(routerinfo_t *router)
+{
+  uint32_t result = router->bandwidthcapacity;
+  if (result > router->bandwidthrate)
+    result = router->bandwidthrate;
+  if (result > BRIDGE_MAX_BELIEVABLE_BANDWIDTH)
+    result = BRIDGE_MAX_BELIEVABLE_BANDWIDTH;
+  else if (result < BRIDGE_MIN_BELIEVABLE_BANDWIDTH)
+    result = BRIDGE_MIN_BELIEVABLE_BANDWIDTH;
+  return result;
+}
+
+/** Given a list of routers and a weighting rule as in
+ * smartlist_choose_node_by_bandwidth_weights, compute weighted bandwidth
+ * values for each node and store them in a freshly allocated
+ * *<b>bandwidths_out</b> of the same length as <b>sl</b>, and holding results
+ * as doubles. If <b>total_bandwidth_out</b> is non-NULL, set it to the total
+ * of all the bandwidths.
+ * Return 0 on success, -1 on failure. */
+static int
+compute_weighted_bandwidths(const smartlist_t *sl,
+                            bandwidth_weight_rule_t rule,
+                            double **bandwidths_out,
+                            double *total_bandwidth_out)
+{
+  int64_t weight_scale;
+  double Wg = -1, Wm = -1, We = -1, Wd = -1;
+  double Wgb = -1, Wmb = -1, Web = -1, Wdb = -1;
+  guardfraction_bandwidth_t guardfraction_bw;
+  double *bandwidths = NULL;
+  double total_bandwidth = 0.0;
+
+  tor_assert(sl);
+  tor_assert(bandwidths_out);
+
+  /* Can't choose exit and guard at same time */
+  tor_assert(rule == NO_WEIGHTING ||
+             rule == WEIGHT_FOR_EXIT ||
+             rule == WEIGHT_FOR_GUARD ||
+             rule == WEIGHT_FOR_MID ||
+             rule == WEIGHT_FOR_DIR);
+
+  *bandwidths_out = NULL;
+
+  if (total_bandwidth_out) {
+    *total_bandwidth_out = 0.0;
+  }
+
+  if (smartlist_len(sl) == 0) {
+    log_info(LD_CIRC,
+             "Empty routerlist passed in to consensus weight node "
+             "selection for rule %s",
+             bandwidth_weight_rule_to_string(rule));
+    return -1;
+  }
+
+  weight_scale = networkstatus_get_weight_scale_param(NULL);
+
+  if (rule == WEIGHT_FOR_GUARD) {
+    Wg = networkstatus_get_bw_weight(NULL, "Wgg", -1);
+    Wm = networkstatus_get_bw_weight(NULL, "Wgm", -1); /* Bridges */
+    We = 0;
+    Wd = networkstatus_get_bw_weight(NULL, "Wgd", -1);
+
+    Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1);
+    Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1);
+    Web = networkstatus_get_bw_weight(NULL, "Web", -1);
+    Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1);
+  } else if (rule == WEIGHT_FOR_MID) {
+    Wg = networkstatus_get_bw_weight(NULL, "Wmg", -1);
+    Wm = networkstatus_get_bw_weight(NULL, "Wmm", -1);
+    We = networkstatus_get_bw_weight(NULL, "Wme", -1);
+    Wd = networkstatus_get_bw_weight(NULL, "Wmd", -1);
+
+    Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1);
+    Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1);
+    Web = networkstatus_get_bw_weight(NULL, "Web", -1);
+    Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1);
+  } else if (rule == WEIGHT_FOR_EXIT) {
+    // Guards CAN be exits if they have weird exit policies
+    // They are d then I guess...
+    We = networkstatus_get_bw_weight(NULL, "Wee", -1);
+    Wm = networkstatus_get_bw_weight(NULL, "Wem", -1); /* Odd exit policies */
+    Wd = networkstatus_get_bw_weight(NULL, "Wed", -1);
+    Wg = networkstatus_get_bw_weight(NULL, "Weg", -1); /* Odd exit policies */
+
+    Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1);
+    Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1);
+    Web = networkstatus_get_bw_weight(NULL, "Web", -1);
+    Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1);
+  } else if (rule == WEIGHT_FOR_DIR) {
+    We = networkstatus_get_bw_weight(NULL, "Wbe", -1);
+    Wm = networkstatus_get_bw_weight(NULL, "Wbm", -1);
+    Wd = networkstatus_get_bw_weight(NULL, "Wbd", -1);
+    Wg = networkstatus_get_bw_weight(NULL, "Wbg", -1);
+
+    Wgb = Wmb = Web = Wdb = weight_scale;
+  } else if (rule == NO_WEIGHTING) {
+    Wg = Wm = We = Wd = weight_scale;
+    Wgb = Wmb = Web = Wdb = weight_scale;
+  }
+
+  if (Wg < 0 || Wm < 0 || We < 0 || Wd < 0 || Wgb < 0 || Wmb < 0 || Wdb < 0
+      || Web < 0) {
+    log_debug(LD_CIRC,
+              "Got negative bandwidth weights. Defaulting to naive selection"
+              " algorithm.");
+    Wg = Wm = We = Wd = weight_scale;
+    Wgb = Wmb = Web = Wdb = weight_scale;
+  }
+
+  Wg /= weight_scale;
+  Wm /= weight_scale;
+  We /= weight_scale;
+  Wd /= weight_scale;
+
+  Wgb /= weight_scale;
+  Wmb /= weight_scale;
+  Web /= weight_scale;
+  Wdb /= weight_scale;
+
+  bandwidths = tor_calloc(smartlist_len(sl), sizeof(double));
+
+  // Cycle through smartlist and total the bandwidth.
+  static int warned_missing_bw = 0;
+  SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) {
+    int is_exit = 0, is_guard = 0, is_dir = 0, this_bw = 0;
+    double weight = 1;
+    double weight_without_guard_flag = 0; /* Used for guardfraction */
+    double final_weight = 0;
+    is_exit = node->is_exit && ! node->is_bad_exit;
+    is_guard = node->is_possible_guard;
+    is_dir = node_is_dir(node);
+    if (node->rs) {
+      if (!node->rs->has_bandwidth) {
+        /* This should never happen, unless all the authorities downgrade
+         * to 0.2.0 or rogue routerstatuses get inserted into our consensus. */
+        if (! warned_missing_bw) {
+          log_warn(LD_BUG,
+                 "Consensus is missing some bandwidths. Using a naive "
+                 "router selection algorithm");
+          warned_missing_bw = 1;
+        }
+        this_bw = 30000; /* Chosen arbitrarily */
+      } else {
+        this_bw = kb_to_bytes(node->rs->bandwidth_kb);
+      }
+    } else if (node->ri) {
+      /* bridge or other descriptor not in our consensus */
+      this_bw = bridge_get_advertised_bandwidth_bounded(node->ri);
+    } else {
+      /* We can't use this one. */
+      continue;
+    }
+
+    if (is_guard && is_exit) {
+      weight = (is_dir ? Wdb*Wd : Wd);
+      weight_without_guard_flag = (is_dir ? Web*We : We);
+    } else if (is_guard) {
+      weight = (is_dir ? Wgb*Wg : Wg);
+      weight_without_guard_flag = (is_dir ? Wmb*Wm : Wm);
+    } else if (is_exit) {
+      weight = (is_dir ? Web*We : We);
+    } else { // middle
+      weight = (is_dir ? Wmb*Wm : Wm);
+    }
+    /* These should be impossible; but overflows here would be bad, so let's
+     * make sure. */
+    if (this_bw < 0)
+      this_bw = 0;
+    if (weight < 0.0)
+      weight = 0.0;
+    if (weight_without_guard_flag < 0.0)
+      weight_without_guard_flag = 0.0;
+
+    /* If guardfraction information is available in the consensus, we
+     * want to calculate this router's bandwidth according to its
+     * guardfraction. Quoting from proposal236:
+     *
+     *    Let Wpf denote the weight from the 'bandwidth-weights' line a
+     *    client would apply to N for position p if it had the guard
+     *    flag, Wpn the weight if it did not have the guard flag, and B the
+     *    measured bandwidth of N in the consensus.  Then instead of choosing
+     *    N for position p proportionally to Wpf*B or Wpn*B, clients should
+     *    choose N proportionally to F*Wpf*B + (1-F)*Wpn*B.
+     */
+    if (node->rs && node->rs->has_guardfraction && rule != WEIGHT_FOR_GUARD) {
+      /* XXX The assert should actually check for is_guard. However,
+       * that crashes dirauths because of #13297. This should be
+       * equivalent: */
+      tor_assert(node->rs->is_possible_guard);
+
+      guard_get_guardfraction_bandwidth(&guardfraction_bw,
+                                        this_bw,
+                                        node->rs->guardfraction_percentage);
+
+      /* Calculate final_weight = F*Wpf*B + (1-F)*Wpn*B */
+      final_weight =
+        guardfraction_bw.guard_bw * weight +
+        guardfraction_bw.non_guard_bw * weight_without_guard_flag;
+
+      log_debug(LD_GENERAL, "%s: Guardfraction weight %f instead of %f (%s)",
+                node->rs->nickname, final_weight, weight*this_bw,
+                bandwidth_weight_rule_to_string(rule));
+    } else { /* no guardfraction information. calculate the weight normally. */
+      final_weight = weight*this_bw;
+    }
+
+    bandwidths[node_sl_idx] = final_weight;
+    total_bandwidth += final_weight;
+  } SMARTLIST_FOREACH_END(node);
+
+  log_debug(LD_CIRC, "Generated weighted bandwidths for rule %s based "
+            "on weights "
+            "Wg=%f Wm=%f We=%f Wd=%f with total bw %f",
+            bandwidth_weight_rule_to_string(rule),
+            Wg, Wm, We, Wd, total_bandwidth);
+
+  *bandwidths_out = bandwidths;
+
+  if (total_bandwidth_out) {
+    *total_bandwidth_out = total_bandwidth;
+  }
+
+  return 0;
+}
+
+/** For all nodes in <b>sl</b>, return the fraction of those nodes, weighted
+ * by their weighted bandwidths with rule <b>rule</b>, for which we have
+ * descriptors.
+ *
+ * If <b>for_direct_connect</b> is true, we intend to connect to the node
+ * directly, as the first hop of a circuit; otherwise, we intend to connect
+ * to it indirectly, or use it as if we were connecting to it indirectly. */
+double
+frac_nodes_with_descriptors(const smartlist_t *sl,
+                            bandwidth_weight_rule_t rule,
+                            int for_direct_conn)
+{
+  double *bandwidths = NULL;
+  double total, present;
+
+  if (smartlist_len(sl) == 0)
+    return 0.0;
+
+  if (compute_weighted_bandwidths(sl, rule, &bandwidths, &total) < 0 ||
+      total <= 0.0) {
+    int n_with_descs = 0;
+    SMARTLIST_FOREACH(sl, const node_t *, node, {
+      if (node_has_preferred_descriptor(node, for_direct_conn))
+        n_with_descs++;
+    });
+    tor_free(bandwidths);
+    return ((double)n_with_descs) / smartlist_len(sl);
+  }
+
+  present = 0.0;
+  SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) {
+    if (node_has_preferred_descriptor(node, for_direct_conn))
+      present += bandwidths[node_sl_idx];
+  } SMARTLIST_FOREACH_END(node);
+
+  tor_free(bandwidths);
+
+  return present / total;
+}
+
+/** Choose a random element of status list <b>sl</b>, weighted by
+ * the advertised bandwidth of each node */
+const node_t *
+node_sl_choose_by_bandwidth(const smartlist_t *sl,
+                            bandwidth_weight_rule_t rule)
+{ /*XXXX MOVE */
+  return smartlist_choose_node_by_bandwidth_weights(sl, rule);
+}
+
+/** Given a <b>router</b>, add every node_t in its family (including the
+ * node itself!) to <b>sl</b>.
+ *
+ * Note the type mismatch: This function takes a routerinfo, but adds nodes
+ * to the smartlist!
+ */
+static void
+routerlist_add_node_and_family(smartlist_t *sl, const routerinfo_t *router)
+{
+  /* XXXX MOVE ? */
+  node_t fake_node;
+  const node_t *node = node_get_by_id(router->cache_info.identity_digest);
+  if (node == NULL) {
+    memset(&fake_node, 0, sizeof(fake_node));
+    fake_node.ri = (routerinfo_t *)router;
+    memcpy(fake_node.identity, router->cache_info.identity_digest, DIGEST_LEN);
+    node = &fake_node;
+  }
+  nodelist_add_node_and_family(sl, node);
+}
+
+/** Return a random running node from the nodelist. Never
+ * pick a node that is in
+ * <b>excludedsmartlist</b>, or which matches <b>excludedset</b>,
+ * even if they are the only nodes available.
+ * If <b>CRN_NEED_UPTIME</b> is set in flags and any router has more than
+ * a minimum uptime, return one of those.
+ * If <b>CRN_NEED_CAPACITY</b> is set in flags, weight your choice by the
+ * advertised capacity of each router.
+ * If <b>CRN_NEED_GUARD</b> is set in flags, consider only Guard routers.
+ * If <b>CRN_WEIGHT_AS_EXIT</b> is set in flags, we weight bandwidths as if
+ * picking an exit node, otherwise we weight bandwidths for picking a relay
+ * node (that is, possibly discounting exit nodes).
+ * If <b>CRN_NEED_DESC</b> is set in flags, we only consider nodes that
+ * have a routerinfo or microdescriptor -- that is, enough info to be
+ * used to build a circuit.
+ * If <b>CRN_PREF_ADDR</b> is set in flags, we only consider nodes that
+ * have an address that is preferred by the ClientPreferIPv6ORPort setting
+ * (regardless of this flag, we exclude nodes that aren't allowed by the
+ * firewall, including ClientUseIPv4 0 and fascist_firewall_use_ipv6() == 0).
+ */
+const node_t *
+router_choose_random_node(smartlist_t *excludedsmartlist,
+                          routerset_t *excludedset,
+                          router_crn_flags_t flags)
+{ /* XXXX MOVE */
+  const int need_uptime = (flags & CRN_NEED_UPTIME) != 0;
+  const int need_capacity = (flags & CRN_NEED_CAPACITY) != 0;
+  const int need_guard = (flags & CRN_NEED_GUARD) != 0;
+  const int weight_for_exit = (flags & CRN_WEIGHT_AS_EXIT) != 0;
+  const int need_desc = (flags & CRN_NEED_DESC) != 0;
+  const int pref_addr = (flags & CRN_PREF_ADDR) != 0;
+  const int direct_conn = (flags & CRN_DIRECT_CONN) != 0;
+  const int rendezvous_v3 = (flags & CRN_RENDEZVOUS_V3) != 0;
+
+  smartlist_t *sl=smartlist_new(),
+    *excludednodes=smartlist_new();
+  const node_t *choice = NULL;
+  const routerinfo_t *r;
+  bandwidth_weight_rule_t rule;
+
+  tor_assert(!(weight_for_exit && need_guard));
+  rule = weight_for_exit ? WEIGHT_FOR_EXIT :
+    (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID);
+
+  SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) {
+    if (node_allows_single_hop_exits(node)) {
+      /* Exclude relays that allow single hop exit circuits. This is an
+       * obsolete option since 0.2.9.2-alpha and done by default in
+       * 0.3.1.0-alpha. */
+      smartlist_add(excludednodes, node);
+    } else if (rendezvous_v3 &&
+               !node_supports_v3_rendezvous_point(node)) {
+      /* Exclude relays that do not support to rendezvous for a hidden service
+       * version 3. */
+      smartlist_add(excludednodes, node);
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  /* If the node_t is not found we won't be to exclude ourself but we
+   * won't be able to pick ourself in router_choose_random_node() so
+   * this is fine to at least try with our routerinfo_t object. */
+  if ((r = router_get_my_routerinfo()))
+    routerlist_add_node_and_family(excludednodes, r);
+
+  router_add_running_nodes_to_smartlist(sl, need_uptime, need_capacity,
+                                        need_guard, need_desc, pref_addr,
+                                        direct_conn);
+  log_debug(LD_CIRC,
+           "We found %d running nodes.",
+            smartlist_len(sl));
+
+  smartlist_subtract(sl,excludednodes);
+  log_debug(LD_CIRC,
+            "We removed %d excludednodes, leaving %d nodes.",
+            smartlist_len(excludednodes),
+            smartlist_len(sl));
+
+  if (excludedsmartlist) {
+    smartlist_subtract(sl,excludedsmartlist);
+    log_debug(LD_CIRC,
+              "We removed %d excludedsmartlist, leaving %d nodes.",
+              smartlist_len(excludedsmartlist),
+              smartlist_len(sl));
+  }
+  if (excludedset) {
+    routerset_subtract_nodes(sl,excludedset);
+    log_debug(LD_CIRC,
+              "We removed excludedset, leaving %d nodes.",
+              smartlist_len(sl));
+  }
+
+  // Always weight by bandwidth
+  choice = node_sl_choose_by_bandwidth(sl, rule);
+
+  smartlist_free(sl);
+  if (!choice && (need_uptime || need_capacity || need_guard || pref_addr)) {
+    /* try once more -- recurse but with fewer restrictions. */
+    log_info(LD_CIRC,
+             "We couldn't find any live%s%s%s routers; falling back "
+             "to list of all routers.",
+             need_capacity?", fast":"",
+             need_uptime?", stable":"",
+             need_guard?", guard":"");
+    flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD|
+                CRN_PREF_ADDR);
+    choice = router_choose_random_node(
+                     excludedsmartlist, excludedset, flags);
+  }
+  smartlist_free(excludednodes);
+  if (!choice) {
+    log_warn(LD_CIRC,
+             "No available nodes when trying to choose node. Failing.");
+  }
+  return choice;
+}
+
+/** Try to find a running directory authority. Flags are as for
+ * router_pick_directory_server.
+ */
+const routerstatus_t *
+router_pick_trusteddirserver(dirinfo_type_t type, int flags)
+{
+  return router_pick_dirserver_generic(
+                                  router_get_trusted_dir_servers_mutable(),
+                                  type, flags);
+}
+
+/** Try to find a running fallback directory. Flags are as for
+ * router_pick_directory_server.
+ */
+const routerstatus_t *
+router_pick_fallback_dirserver(dirinfo_type_t type, int flags)
+{
+  return router_pick_dirserver_generic(
+                                  router_get_fallback_dir_servers_mutable(),
+                                  type, flags);
+}
+
+/** Pick a random element from a list of dir_server_t, weighting by their
+ * <b>weight</b> field. */
+static const dir_server_t *
+dirserver_choose_by_weight(const smartlist_t *servers, double authority_weight)
+{
+  int n = smartlist_len(servers);
+  int i;
+  double *weights_dbl;
+  uint64_t *weights_u64;
+  const dir_server_t *ds;
+
+  weights_dbl = tor_calloc(n, sizeof(double));
+  weights_u64 = tor_calloc(n, sizeof(uint64_t));
+  for (i = 0; i < n; ++i) {
+    ds = smartlist_get(servers, i);
+    weights_dbl[i] = ds->weight;
+    if (ds->is_authority)
+      weights_dbl[i] *= authority_weight;
+  }
+
+  scale_array_elements_to_u64(weights_u64, weights_dbl, n, NULL);
+  i = choose_array_element_by_weight(weights_u64, n);
+  tor_free(weights_dbl);
+  tor_free(weights_u64);
+  return (i < 0) ? NULL : smartlist_get(servers, i);
+}
+
+/** Choose randomly from among the dir_server_ts in sourcelist that
+ * are up. Flags are as for router_pick_directory_server_impl().
+ */
+static const routerstatus_t *
+router_pick_trusteddirserver_impl(const smartlist_t *sourcelist,
+                                  dirinfo_type_t type, int flags,
+                                  int *n_busy_out)
+{
+  const or_options_t *options = get_options();
+  smartlist_t *direct, *tunnel;
+  smartlist_t *overloaded_direct, *overloaded_tunnel;
+  const routerinfo_t *me = router_get_my_routerinfo();
+  const routerstatus_t *result = NULL;
+  time_t now = time(NULL);
+  const int requireother = ! (flags & PDS_ALLOW_SELF);
+  const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL);
+  const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH);
+  const int no_microdesc_fetching =(flags & PDS_NO_EXISTING_MICRODESC_FETCH);
+  const double auth_weight =
+    (sourcelist == router_get_fallback_dir_servers()) ?
+    options->DirAuthorityFallbackRate : 1.0;
+  smartlist_t *pick_from;
+  int n_busy = 0;
+  int try_excluding = 1, n_excluded = 0;
+  int try_ip_pref = 1;
+
+  if (!sourcelist)
+    return NULL;
+
+ retry_search:
+
+  direct = smartlist_new();
+  tunnel = smartlist_new();
+  overloaded_direct = smartlist_new();
+  overloaded_tunnel = smartlist_new();
+
+  const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref);
+  const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref);
+  const int must_have_or = directory_must_use_begindir(options);
+
+  SMARTLIST_FOREACH_BEGIN(sourcelist, const dir_server_t *, d)
+    {
+      int is_overloaded =
+          d->fake_status.last_dir_503_at + DIR_503_TIMEOUT > now;
+      if (!d->is_running) continue;
+      if ((type & d->type) == 0)
+        continue;
+
+      SKIP_MISSING_TRUSTED_EXTRAINFO(type, d->digest);
+
+      if (requireother && me && router_digest_is_me(d->digest))
+        continue;
+      if (try_excluding &&
+          routerset_contains_routerstatus(options->ExcludeNodes,
+                                          &d->fake_status, -1)) {
+        ++n_excluded;
+        continue;
+      }
+
+      if (router_is_already_dir_fetching_(d->addr,
+                                          &d->ipv6_addr,
+                                          d->dir_port,
+                                          no_serverdesc_fetching,
+                                          no_microdesc_fetching)) {
+        ++n_busy;
+        continue;
+      }
+
+      /* Clients use IPv6 addresses if the server has one and the client
+       * prefers IPv6.
+       * Add the router if its preferred address and port are reachable.
+       * If we don't get any routers, we'll try again with the non-preferred
+       * address for each router (if any). (To ensure correct load-balancing
+       * we try routers that only have one address both times.)
+       */
+      if (!fascistfirewall || skip_or_fw ||
+          fascist_firewall_allows_dir_server(d, FIREWALL_OR_CONNECTION,
+                                             try_ip_pref))
+        smartlist_add(is_overloaded ? overloaded_tunnel : tunnel, (void*)d);
+      else if (!must_have_or && (skip_dir_fw ||
+               fascist_firewall_allows_dir_server(d, FIREWALL_DIR_CONNECTION,
+                                                  try_ip_pref)))
+        smartlist_add(is_overloaded ? overloaded_direct : direct, (void*)d);
+    }
+  SMARTLIST_FOREACH_END(d);
+
+  if (smartlist_len(tunnel)) {
+    pick_from = tunnel;
+  } else if (smartlist_len(overloaded_tunnel)) {
+    pick_from = overloaded_tunnel;
+  } else if (smartlist_len(direct)) {
+    pick_from = direct;
+  } else {
+    pick_from = overloaded_direct;
+  }
+
+  {
+    const dir_server_t *selection =
+      dirserver_choose_by_weight(pick_from, auth_weight);
+
+    if (selection)
+      result = &selection->fake_status;
+  }
+
+  smartlist_free(direct);
+  smartlist_free(tunnel);
+  smartlist_free(overloaded_direct);
+  smartlist_free(overloaded_tunnel);
+
+  RETRY_ALTERNATE_IP_VERSION(retry_search);
+
+  RETRY_WITHOUT_EXCLUDE(retry_search);
+
+  router_picked_poor_directory_log(result);
+
+  if (n_busy_out)
+    *n_busy_out = n_busy;
+  return result;
+}
diff --git a/src/feature/nodelist/node_select.h b/src/feature/nodelist/node_select.h
new file mode 100644
index 0000000000..ed7450b92c
--- /dev/null
+++ b/src/feature/nodelist/node_select.h
@@ -0,0 +1,102 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file node_select.h
+ * \brief Header file for node_select.c
+ **/
+
+#ifndef TOR_NODE_SELECT_H
+#define TOR_NODE_SELECT_H
+
+/** Flags to be passed to control router_choose_random_node() to indicate what
+ * kind of nodes to pick according to what algorithm. */
+typedef enum router_crn_flags_t {
+  CRN_NEED_UPTIME = 1<<0,
+  CRN_NEED_CAPACITY = 1<<1,
+  CRN_NEED_GUARD = 1<<2,
+  /* XXXX not used, apparently. */
+  CRN_WEIGHT_AS_EXIT = 1<<5,
+  CRN_NEED_DESC = 1<<6,
+  /* On clients, only provide nodes that satisfy ClientPreferIPv6OR */
+  CRN_PREF_ADDR = 1<<7,
+  /* On clients, only provide nodes that we can connect to directly, based on
+   * our firewall rules */
+  CRN_DIRECT_CONN = 1<<8,
+  /* On clients, only provide nodes with HSRend >= 2 protocol version which
+   * is required for hidden service version >= 3. */
+  CRN_RENDEZVOUS_V3 = 1<<9,
+} router_crn_flags_t;
+
+/** Possible ways to weight routers when choosing one randomly.  See
+ * routerlist_sl_choose_by_bandwidth() for more information.*/
+typedef enum bandwidth_weight_rule_t {
+  NO_WEIGHTING, WEIGHT_FOR_EXIT, WEIGHT_FOR_MID, WEIGHT_FOR_GUARD,
+  WEIGHT_FOR_DIR
+} bandwidth_weight_rule_t;
+
+/* Flags for pick_directory_server() and pick_trusteddirserver(). */
+/** Flag to indicate that we should not automatically be willing to use
+ * ourself to answer a directory request.
+ * Passed to router_pick_directory_server (et al).*/
+#define PDS_ALLOW_SELF                 (1<<0)
+/** Flag to indicate that if no servers seem to be up, we should mark all
+ * directory servers as up and try again.
+ * Passed to router_pick_directory_server (et al).*/
+#define PDS_RETRY_IF_NO_SERVERS        (1<<1)
+/** Flag to indicate that we should not exclude directory servers that
+ * our ReachableAddress settings would exclude.  This usually means that
+ * we're going to connect to the server over Tor, and so we don't need to
+ * worry about our firewall telling us we can't.
+ * Passed to router_pick_directory_server (et al).*/
+#define PDS_IGNORE_FASCISTFIREWALL     (1<<2)
+/** Flag to indicate that we should not use any directory authority to which
+ * we have an existing directory connection for downloading server descriptors
+ * or extrainfo documents.
+ *
+ * Passed to router_pick_directory_server (et al)
+ */
+#define PDS_NO_EXISTING_SERVERDESC_FETCH (1<<3)
+/** Flag to indicate that we should not use any directory authority to which
+ * we have an existing directory connection for downloading microdescs.
+ *
+ * Passed to router_pick_directory_server (et al)
+ */
+#define PDS_NO_EXISTING_MICRODESC_FETCH (1<<4)
+
+const routerstatus_t *router_pick_directory_server(dirinfo_type_t type,
+                                                   int flags);
+
+int router_get_my_share_of_directory_requests(double *v3_share_out);
+
+const node_t *node_sl_choose_by_bandwidth(const smartlist_t *sl,
+                                          bandwidth_weight_rule_t rule);
+double frac_nodes_with_descriptors(const smartlist_t *sl,
+                                   bandwidth_weight_rule_t rule,
+                                   int for_direct_conn);
+const node_t *router_choose_random_node(smartlist_t *excludedsmartlist,
+                                        struct routerset_t *excludedset,
+                                        router_crn_flags_t flags);
+
+const routerstatus_t *router_pick_trusteddirserver(dirinfo_type_t type,
+                                                   int flags);
+const routerstatus_t *router_pick_fallback_dirserver(dirinfo_type_t type,
+                                                     int flags);
+
+#ifdef NODE_SELECT_PRIVATE
+STATIC int choose_array_element_by_weight(const uint64_t *entries,
+                                          int n_entries);
+STATIC void scale_array_elements_to_u64(uint64_t *entries_out,
+                                        const double *entries_in,
+                                        int n_entries,
+                                        uint64_t *total_out);
+STATIC const routerstatus_t *router_pick_directory_server_impl(
+                                           dirinfo_type_t auth, int flags,
+                                           int *n_busy_out);
+STATIC int router_is_already_dir_fetching(const tor_addr_port_t *ap,
+                                          int serverdesc, int microdesc);
+#endif
+
+#endif
diff --git a/src/feature/nodelist/node_st.h b/src/feature/nodelist/node_st.h
new file mode 100644
index 0000000000..53ffde29e4
--- /dev/null
+++ b/src/feature/nodelist/node_st.h
@@ -0,0 +1,102 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef NODE_ST_H
+#define NODE_ST_H
+
+#include "feature/hs/hsdir_index_st.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+/** A node_t represents a Tor router.
+ *
+ * Specifically, a node_t is a Tor router as we are using it: a router that
+ * we are considering for circuits, connections, and so on.  A node_t is a
+ * thin wrapper around the routerstatus, routerinfo, and microdesc for a
+ * single router, and provides a consistent interface for all of them.
+ *
+ * Also, a node_t has mutable state.  While a routerinfo, a routerstatus,
+ * and a microdesc have[*] only the information read from a router
+ * descriptor, a consensus entry, and a microdescriptor (respectively)...
+ * a node_t has flags based on *our own current opinion* of the node.
+ *
+ * [*] Actually, there is some leftover information in each that is mutable.
+ *  We should try to excise that.
+ */
+struct node_t {
+  /* Indexing information */
+
+  /** Used to look up the node_t by its identity digest. */
+  HT_ENTRY(node_t) ht_ent;
+  /** Used to look up the node_t by its ed25519 identity digest. */
+  HT_ENTRY(node_t) ed_ht_ent;
+  /** Position of the node within the list of nodes */
+  int nodelist_idx;
+
+  /** The identity digest of this node_t.  No more than one node_t per
+   * identity may exist at a time. */
+  char identity[DIGEST_LEN];
+
+  /** The ed25519 identity of this node_t. This field is nonzero iff we
+   * currently have an ed25519 identity for this node in either md or ri,
+   * _and_ this node has been inserted to the ed25519-to-node map in the
+   * nodelist.
+   */
+  ed25519_public_key_t ed25519_id;
+
+  microdesc_t *md;
+  routerinfo_t *ri;
+  routerstatus_t *rs;
+
+  /* local info: copied from routerstatus, then possibly frobbed based
+   * on experience.  Authorities set this stuff directly.  Note that
+   * these reflect knowledge of the primary (IPv4) OR port only.  */
+
+  unsigned int is_running:1; /**< As far as we know, is this OR currently
+                              * running? */
+  unsigned int is_valid:1; /**< Has a trusted dirserver validated this OR?
+                            *  (For Authdir: Have we validated this OR?) */
+  unsigned int is_fast:1; /** Do we think this is a fast OR? */
+  unsigned int is_stable:1; /** Do we think this is a stable OR? */
+  unsigned int is_possible_guard:1; /**< Do we think this is an OK guard? */
+  unsigned int is_exit:1; /**< Do we think this is an OK exit? */
+  unsigned int is_bad_exit:1; /**< Do we think this exit is censored, borked,
+                               * or otherwise nasty? */
+  unsigned int is_hs_dir:1; /**< True iff this router is a hidden service
+                             * directory according to the authorities. */
+
+  /* Local info: warning state. */
+
+  unsigned int name_lookup_warned:1; /**< Have we warned the user for referring
+                                      * to this (unnamed) router by nickname?
+                                      */
+
+  /** Local info: we treat this node as if it rejects everything */
+  unsigned int rejects_all:1;
+
+  /* Local info: derived. */
+
+  /** True if the IPv6 OR port is preferred over the IPv4 OR port.
+   * XX/teor - can this become out of date if the torrc changes? */
+  unsigned int ipv6_preferred:1;
+
+  /** According to the geoip db what country is this router in? */
+  /* XXXprop186 what is this suppose to mean with multiple OR ports? */
+  country_t country;
+
+  /* The below items are used only by authdirservers for
+   * reachability testing. */
+
+  /** When was the last time we could reach this OR? */
+  time_t last_reachable;        /* IPv4. */
+  time_t last_reachable6;       /* IPv6. */
+
+  /* Hidden service directory index data. This is used by a service or client
+   * in order to know what's the hs directory index for this node at the time
+   * the consensus is set. */
+  struct hsdir_index_t hsdir_index;
+};
+
+#endif
diff --git a/src/or/nodelist.c b/src/feature/nodelist/nodelist.c
index 26f990b08c..99d7f746a8 100644
--- a/src/or/nodelist.c
+++ b/src/feature/nodelist/nodelist.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -10,43 +10,101 @@
  * \brief Structures and functions for tracking what we know about the routers
  *   on the Tor network, and correlating information from networkstatus,
  *   routerinfo, and microdescs.
+ *
+ * The key structure here is node_t: that's the canonical way to refer
+ * to a Tor relay that we might want to build a circuit through.  Every
+ * node_t has either a routerinfo_t, or a routerstatus_t from the current
+ * networkstatus consensus.  If it has a routerstatus_t, it will also
+ * need to have a microdesc_t before you can use it for circuits.
+ *
+ * The nodelist_t is a global singleton that maps identities to node_t
+ * objects.  Access them with the node_get_*() functions.  The nodelist_t
+ * is maintained by calls throughout the codebase
+ *
+ * Generally, other code should not have to reach inside a node_t to
+ * see what information it has.  Instead, you should call one of the
+ * many accessor functions that works on a generic node_t.  If there
+ * isn't one that does what you need, it's better to make such a function,
+ * and then use it.
+ *
+ * For historical reasons, some of the functions that select a node_t
+ * from the list of all usable node_t objects are in the routerlist.c
+ * module, since they originally selected a routerinfo_t. (TODO: They
+ * should move!)
+ *
+ * (TODO: Perhaps someday we should abstract the remaining ways of
+ * talking about a relay to also be node_t instances. Those would be
+ * routerstatus_t as used for directory requests, and dir_server_t as
+ * used for authorities and fallback directories.)
  */
 
-#include "or.h"
-#include "address.h"
-#include "address_set.h"
-#include "config.h"
-#include "control.h"
-#include "dirserv.h"
-#include "geoip.h"
-#include "main.h"
-#include "microdesc.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "policies.h"
-#include "rendservice.h"
-#include "router.h"
-#include "routerlist.h"
-#include "routerset.h"
+#define NODELIST_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
+#include "core/or/address_set.h"
+#include "core/or/policies.h"
+#include "core/or/protover.h"
+#include "feature/client/bridges.h"
+#include "feature/client/entrynodes.h"
+#include "feature/control/control.h"
+#include "feature/dirauth/process_descs.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/hs/hs_client.h"
+#include "feature/hs/hs_common.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/node_select.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/nodelist/torcert.h"
+#include "feature/rend/rendservice.h"
+#include "lib/encoding/binascii.h"
+#include "lib/err/backtrace.h"
+#include "lib/geoip/geoip.h"
+#include "lib/net/address.h"
 
 #include <string.h>
 
+#include "feature/dirauth/authmode.h"
+
+#include "feature/dirclient/dir_server_st.h"
+#include "feature/nodelist/microdesc_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerlist_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
 static void nodelist_drop_node(node_t *node, int remove_from_ht);
-static void node_free(node_t *node);
+#define node_free(val) \
+  FREE_AND_NULL(node_t, node_free_, (val))
+static void node_free_(node_t *node);
 
 /** count_usable_descriptors counts descriptors with these flag(s)
  */
 typedef enum {
-  /* All descriptors regardless of flags */
-  USABLE_DESCRIPTOR_ALL = 0,
-  /* Only descriptors with the Exit flag */
-  USABLE_DESCRIPTOR_EXIT_ONLY = 1
+  /* All descriptors regardless of flags or exit policies */
+  USABLE_DESCRIPTOR_ALL         = 0U,
+  /* Only count descriptors with an exit policy that allows at least one port
+   */
+  USABLE_DESCRIPTOR_EXIT_POLICY = 1U << 0,
+  /* Only count descriptors for relays that have the exit flag in the
+   * consensus */
+  USABLE_DESCRIPTOR_EXIT_FLAG   = 1U << 1,
+  /* Only count descriptors for relays that have the policy and the flag */
+  USABLE_DESCRIPTOR_EXIT_POLICY_AND_FLAG = (USABLE_DESCRIPTOR_EXIT_POLICY |
+                                            USABLE_DESCRIPTOR_EXIT_FLAG)
 } usable_descriptor_t;
 static void count_usable_descriptors(int *num_present,
                                      int *num_usable,
                                      smartlist_t *descs_out,
                                      const networkstatus_t *consensus,
-                                     const or_options_t *options,
                                      time_t now,
                                      routerset_t *in_set,
                                      usable_descriptor_t exit_only);
@@ -64,9 +122,22 @@ typedef struct nodelist_t {
   smartlist_t *nodes;
   /* Hash table to map from node ID digest to node. */
   HT_HEAD(nodelist_map, node_t) nodes_by_id;
+  /* Hash table to map from node Ed25519 ID to node.
+   *
+   * Whenever a node's routerinfo or microdescriptor is about to change,
+   * you should remove it from this map with node_remove_from_ed25519_map().
+   * Whenever a node's routerinfo or microdescriptor has just chaned,
+   * you should add it to this map with node_add_to_ed25519_map().
+   */
+  HT_HEAD(nodelist_ed_map, node_t) nodes_by_ed_id;
 
   /* Set of addresses that belong to nodes we believe in. */
   address_set_t *node_addrs;
+
+  /* The valid-after time of the last live consensus that initialized the
+   * nodelist.  We use this to detect outdated nodelists that need to be
+   * rebuilt using a newer consensus. */
+  time_t live_consensus_valid_after;
 } nodelist_t;
 
 static inline unsigned int
@@ -85,6 +156,23 @@ HT_PROTOTYPE(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq)
 HT_GENERATE2(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq,
              0.6, tor_reallocarray_, tor_free_)
 
+static inline unsigned int
+node_ed_id_hash(const node_t *node)
+{
+  return (unsigned) siphash24g(node->ed25519_id.pubkey, ED25519_PUBKEY_LEN);
+}
+
+static inline unsigned int
+node_ed_id_eq(const node_t *node1, const node_t *node2)
+{
+  return ed25519_pubkey_eq(&node1->ed25519_id, &node2->ed25519_id);
+}
+
+HT_PROTOTYPE(nodelist_ed_map, node_t, ed_ht_ent, node_ed_id_hash,
+             node_ed_id_eq)
+HT_GENERATE2(nodelist_ed_map, node_t, ed_ht_ent, node_ed_id_hash,
+             node_ed_id_eq, 0.6, tor_reallocarray_, tor_free_)
+
 /** The global nodelist. */
 static nodelist_t *the_nodelist=NULL;
 
@@ -95,13 +183,14 @@ init_nodelist(void)
   if (PREDICT_UNLIKELY(the_nodelist == NULL)) {
     the_nodelist = tor_malloc_zero(sizeof(nodelist_t));
     HT_INIT(nodelist_map, &the_nodelist->nodes_by_id);
+    HT_INIT(nodelist_ed_map, &the_nodelist->nodes_by_ed_id);
     the_nodelist->nodes = smartlist_new();
   }
 }
 
 /** As node_get_by_id, but returns a non-const pointer */
-node_t *
-node_get_mutable_by_id(const char *identity_digest)
+MOCK_IMPL(node_t *,
+node_get_mutable_by_id,(const char *identity_digest))
 {
   node_t search, *node;
   if (PREDICT_UNLIKELY(the_nodelist == NULL))
@@ -112,6 +201,21 @@ node_get_mutable_by_id(const char *identity_digest)
   return node;
 }
 
+/** As node_get_by_ed25519_id, but returns a non-const pointer */
+node_t *
+node_get_mutable_by_ed25519_id(const ed25519_public_key_t *ed_id)
+{
+  node_t search, *node;
+  if (PREDICT_UNLIKELY(the_nodelist == NULL))
+    return NULL;
+  if (BUG(ed_id == NULL) || BUG(ed25519_public_key_is_zero(ed_id)))
+    return NULL;
+
+  memcpy(&search.ed25519_id, ed_id, sizeof(search.ed25519_id));
+  node = HT_FIND(nodelist_ed_map, &the_nodelist->nodes_by_ed_id, &search);
+  return node;
+}
+
 /** Return the node_t whose identity is <b>identity_digest</b>, or NULL
  * if no such node exists. */
 MOCK_IMPL(const node_t *,
@@ -120,6 +224,14 @@ node_get_by_id,(const char *identity_digest))
   return node_get_mutable_by_id(identity_digest);
 }
 
+/** Return the node_t whose ed25519 identity is <b>ed_id</b>, or NULL
+ * if no such node exists. */
+MOCK_IMPL(const node_t *,
+node_get_by_ed25519_id,(const ed25519_public_key_t *ed_id))
+{
+  return node_get_mutable_by_ed25519_id(ed_id);
+}
+
 /** Internal: return the node_t whose identity_digest is
  * <b>identity_digest</b>.  If none exists, create a new one, add it to the
  * nodelist, and return it.
@@ -146,6 +258,180 @@ node_get_or_create(const char *identity_digest)
   return node;
 }
 
+/** Remove <b>node</b> from the ed25519 map (if it present), and
+ * set its ed25519_id field to zero. */
+static int
+node_remove_from_ed25519_map(node_t *node)
+{
+  tor_assert(the_nodelist);
+  tor_assert(node);
+
+  if (ed25519_public_key_is_zero(&node->ed25519_id)) {
+    return 0;
+  }
+
+  int rv = 0;
+  node_t *search =
+    HT_FIND(nodelist_ed_map, &the_nodelist->nodes_by_ed_id, node);
+  if (BUG(search != node)) {
+    goto clear_and_return;
+  }
+
+  search = HT_REMOVE(nodelist_ed_map, &the_nodelist->nodes_by_ed_id, node);
+  tor_assert(search == node);
+  rv = 1;
+
+ clear_and_return:
+  memset(&node->ed25519_id, 0, sizeof(node->ed25519_id));
+  return rv;
+}
+
+/** Helper function to log details of duplicated ed2559_ids */
+static void
+node_log_dup_ed_id(const node_t *old, const node_t *node, const char *ed_id)
+{
+  char *s;
+  char *olddesc = tor_strdup(node_describe(old));
+
+  tor_asprintf(&s, "Reused ed25519_id %s: old %s new %s", ed_id,
+               olddesc, node_describe(node));
+  log_backtrace(LOG_NOTICE, LD_DIR, s);
+  tor_free(olddesc);
+  tor_free(s);
+}
+
+/** If <b>node</b> has an ed25519 id, and it is not already in the ed25519 id
+ * map, set its ed25519_id field, and add it to the ed25519 map.
+ */
+static int
+node_add_to_ed25519_map(node_t *node)
+{
+  tor_assert(the_nodelist);
+  tor_assert(node);
+
+  if (! ed25519_public_key_is_zero(&node->ed25519_id)) {
+    return 0;
+  }
+
+  const ed25519_public_key_t *key = node_get_ed25519_id(node);
+  if (!key) {
+    return 0;
+  }
+
+  node_t *old;
+  memcpy(&node->ed25519_id, key, sizeof(node->ed25519_id));
+  old = HT_FIND(nodelist_ed_map, &the_nodelist->nodes_by_ed_id, node);
+  if (old) {
+    char ed_id[BASE32_BUFSIZE(sizeof(key->pubkey))];
+
+    base32_encode(ed_id, sizeof(ed_id), (const char *)key->pubkey,
+                  sizeof(key->pubkey));
+    if (BUG(old == node)) {
+      /* Actual bug: all callers of this function call
+       * node_remove_from_ed25519_map first. */
+      log_err(LD_BUG,
+              "Unexpectedly found deleted node with ed25519_id %s", ed_id);
+    } else {
+      /* Distinct nodes sharing a ed25519 id, possibly due to relay
+       * misconfiguration.  The key pinning might not catch this,
+       * possibly due to downloading a missing descriptor during
+       * consensus voting. */
+      node_log_dup_ed_id(old, node, ed_id);
+      memset(&node->ed25519_id, 0, sizeof(node->ed25519_id));
+    }
+    return 0;
+  }
+
+  HT_INSERT(nodelist_ed_map, &the_nodelist->nodes_by_ed_id, node);
+  return 1;
+}
+
+/* For a given <b>node</b> for the consensus <b>ns</b>, set the hsdir index
+ * for the node, both current and next if possible. This can only fails if the
+ * node_t ed25519 identity key can't be found which would be a bug. */
+STATIC void
+node_set_hsdir_index(node_t *node, const networkstatus_t *ns)
+{
+  time_t now = approx_time();
+  const ed25519_public_key_t *node_identity_pk;
+  uint8_t *fetch_srv = NULL, *store_first_srv = NULL, *store_second_srv = NULL;
+  uint64_t next_time_period_num, current_time_period_num;
+  uint64_t fetch_tp, store_first_tp, store_second_tp;
+
+  tor_assert(node);
+  tor_assert(ns);
+
+  if (!networkstatus_is_live(ns, now)) {
+    static struct ratelim_t live_consensus_ratelim = RATELIM_INIT(30 * 60);
+    log_fn_ratelim(&live_consensus_ratelim, LOG_INFO, LD_GENERAL,
+                   "Not setting hsdir index with a non-live consensus.");
+    goto done;
+  }
+
+  node_identity_pk = node_get_ed25519_id(node);
+  if (node_identity_pk == NULL) {
+    log_debug(LD_GENERAL, "ed25519 identity public key not found when "
+                          "trying to build the hsdir indexes for node %s",
+              node_describe(node));
+    goto done;
+  }
+
+  /* Get the current and next time period number. */
+  current_time_period_num = hs_get_time_period_num(0);
+  next_time_period_num = hs_get_next_time_period_num(0);
+
+  /* We always use the current time period for fetching descs */
+  fetch_tp = current_time_period_num;
+
+  /* Now extract the needed SRVs and time periods for building hsdir indices */
+  if (hs_in_period_between_tp_and_srv(ns, now)) {
+    fetch_srv = hs_get_current_srv(fetch_tp, ns);
+
+    store_first_tp = hs_get_previous_time_period_num(0);
+    store_second_tp = current_time_period_num;
+  } else {
+    fetch_srv = hs_get_previous_srv(fetch_tp, ns);
+
+    store_first_tp = current_time_period_num;
+    store_second_tp = next_time_period_num;
+  }
+
+  /* We always use the old SRV for storing the first descriptor and the latest
+   * SRV for storing the second descriptor */
+  store_first_srv = hs_get_previous_srv(store_first_tp, ns);
+  store_second_srv = hs_get_current_srv(store_second_tp, ns);
+
+  /* Build the fetch index. */
+  hs_build_hsdir_index(node_identity_pk, fetch_srv, fetch_tp,
+                       node->hsdir_index.fetch);
+
+  /* If we are in the time segment between SRV#N and TP#N, the fetch index is
+     the same as the first store index */
+  if (!hs_in_period_between_tp_and_srv(ns, now)) {
+    memcpy(node->hsdir_index.store_first, node->hsdir_index.fetch,
+           sizeof(node->hsdir_index.store_first));
+  } else {
+    hs_build_hsdir_index(node_identity_pk, store_first_srv, store_first_tp,
+                         node->hsdir_index.store_first);
+  }
+
+  /* If we are in the time segment between TP#N and SRV#N+1, the fetch index is
+     the same as the second store index */
+  if (hs_in_period_between_tp_and_srv(ns, now)) {
+    memcpy(node->hsdir_index.store_second, node->hsdir_index.fetch,
+           sizeof(node->hsdir_index.store_second));
+  } else {
+    hs_build_hsdir_index(node_identity_pk, store_second_srv, store_second_tp,
+                         node->hsdir_index.store_second);
+  }
+
+ done:
+  tor_free(fetch_srv);
+  tor_free(store_first_srv);
+  tor_free(store_second_srv);
+  return;
+}
+
 /** Called when a node's address changes. */
 static void
 node_addrs_changed(node_t *node)
@@ -214,6 +500,8 @@ nodelist_set_routerinfo(routerinfo_t *ri, routerinfo_t **ri_old_out)
   id_digest = ri->cache_info.identity_digest;
   node = node_get_or_create(id_digest);
 
+  node_remove_from_ed25519_map(node);
+
   if (node->ri) {
     if (!routers_have_same_or_addrs(node->ri, ri)) {
       node_addrs_changed(node);
@@ -227,6 +515,8 @@ nodelist_set_routerinfo(routerinfo_t *ri, routerinfo_t **ri_old_out)
   }
   node->ri = ri;
 
+  node_add_to_ed25519_map(node);
+
   if (node->country == -1)
     node_set_country(node);
 
@@ -236,6 +526,14 @@ nodelist_set_routerinfo(routerinfo_t *ri, routerinfo_t **ri_old_out)
     dirserv_set_node_flags_from_authoritative_status(node, status);
   }
 
+  /* Setting the HSDir index requires the ed25519 identity key which can
+   * only be found either in the ri or md. This is why this is called here.
+   * Only nodes supporting HSDir=2 protocol version needs this index. */
+  if (node->rs && node->rs->pv.supports_v3_hsdir) {
+    node_set_hsdir_index(node,
+                         networkstatus_get_latest_consensus());
+  }
+
   node_add_to_address_set(node);
 
   return node;
@@ -265,10 +563,20 @@ nodelist_add_microdesc(microdesc_t *md)
   node = node_get_mutable_by_id(rs->identity_digest);
   if (node == NULL)
     return NULL;
+
+  node_remove_from_ed25519_map(node);
   if (node->md)
     node->md->held_by_nodes--;
+
   node->md = md;
   md->held_by_nodes++;
+  /* Setting the HSDir index requires the ed25519 identity key which can
+   * only be found either in the ri or md. This is why this is called here.
+   * Only nodes supporting HSDir=2 protocol version needs this index. */
+  if (rs->pv.supports_v3_hsdir) {
+    node_set_hsdir_index(node, ns);
+  }
+  node_add_to_ed25519_map(node);
   node_add_to_address_set(node);
 
   return node;
@@ -315,15 +623,20 @@ nodelist_set_consensus(networkstatus_t *ns)
     if (ns->flavor == FLAV_MICRODESC) {
       if (node->md == NULL ||
           tor_memneq(node->md->digest,rs->descriptor_digest,DIGEST256_LEN)) {
+        node_remove_from_ed25519_map(node);
         if (node->md)
           node->md->held_by_nodes--;
         node->md = microdesc_cache_lookup_by_digest256(NULL,
                                                        rs->descriptor_digest);
         if (node->md)
           node->md->held_by_nodes++;
+        node_add_to_ed25519_map(node);
       }
     }
 
+    if (rs->pv.supports_v3_hsdir) {
+      node_set_hsdir_index(node, ns);
+    }
     node_set_country(node);
 
     /* If we're not an authdir, believe others. */
@@ -369,6 +682,21 @@ nodelist_set_consensus(networkstatus_t *ns)
       }
     } SMARTLIST_FOREACH_END(node);
   }
+
+  /* If the consensus is live, note down the consensus valid-after that formed
+   * the nodelist. */
+  if (networkstatus_is_live(ns, approx_time())) {
+    the_nodelist->live_consensus_valid_after = ns->valid_after;
+  }
+}
+
+/** Return 1 iff <b>node</b> has Exit flag and no BadExit flag.
+ * Otherwise, return 0.
+ */
+int
+node_is_good_exit(const node_t *node)
+{
+  return node->is_exit && ! node->is_bad_exit;
 }
 
 /** Helper: return true iff a node has a usable amount of information*/
@@ -387,6 +715,9 @@ nodelist_remove_microdesc(const char *identity_digest, microdesc_t *md)
   if (node && node->md == md) {
     node->md = NULL;
     md->held_by_nodes--;
+    if (! node_get_ed25519_id(node)) {
+      node_remove_from_ed25519_map(node);
+    }
   }
 }
 
@@ -415,6 +746,7 @@ nodelist_drop_node(node_t *node, int remove_from_ht)
     tmp = HT_REMOVE(nodelist_map, &the_nodelist->nodes_by_id, node);
     tor_assert(tmp == node);
   }
+  node_remove_from_ed25519_map(node);
 
   idx = node->nodelist_idx;
   tor_assert(idx >= 0);
@@ -449,7 +781,7 @@ nodelist_find_nodes_with_microdesc(const microdesc_t *md)
 
 /** Release storage held by <b>node</b>  */
 static void
-node_free(node_t *node)
+node_free_(node_t *node)
 {
   if (!node)
     return;
@@ -497,6 +829,7 @@ nodelist_free_all(void)
     return;
 
   HT_CLEAR(nodelist_map, &the_nodelist->nodes_by_id);
+  HT_CLEAR(nodelist_ed_map, &the_nodelist->nodes_by_ed_id);
   SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
     node->nodelist_idx = -1;
     node_free(node);
@@ -564,12 +897,49 @@ nodelist_assert_ok(void)
     tor_assert(node_sl_idx == node->nodelist_idx);
   } SMARTLIST_FOREACH_END(node);
 
+  /* Every node listed with an ed25519 identity should be listed by that
+   * identity.
+   */
+  SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
+    if (!ed25519_public_key_is_zero(&node->ed25519_id)) {
+      tor_assert(node == node_get_by_ed25519_id(&node->ed25519_id));
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  node_t **idx;
+  HT_FOREACH(idx, nodelist_ed_map, &the_nodelist->nodes_by_ed_id) {
+    node_t *node = *idx;
+    tor_assert(node == node_get_by_ed25519_id(&node->ed25519_id));
+  }
+
   tor_assert((long)smartlist_len(the_nodelist->nodes) ==
              (long)HT_SIZE(&the_nodelist->nodes_by_id));
 
+  tor_assert((long)smartlist_len(the_nodelist->nodes) >=
+             (long)HT_SIZE(&the_nodelist->nodes_by_ed_id));
+
   digestmap_free(dm, NULL);
 }
 
+/** Ensure that the nodelist has been created with the most recent consensus.
+ *  If that's not the case, make it so.  */
+void
+nodelist_ensure_freshness(networkstatus_t *ns)
+{
+  tor_assert(ns);
+
+  /* We don't even have a nodelist: this is a NOP. */
+  if (!the_nodelist) {
+    return;
+  }
+
+  if (the_nodelist->live_consensus_valid_after != ns->valid_after) {
+    log_info(LD_GENERAL, "Nodelist was not fresh: rebuilding. (%d / %d)",
+             (int) the_nodelist->live_consensus_valid_after,
+             (int) ns->valid_after);
+    nodelist_set_consensus(ns);
+  }
+}
 /** Return a list of a node_t * for every node we know about.  The caller
  * MUST NOT modify the list. (You can set and clear flags in the nodes if
  * you must, but you must not add or remove nodes.) */
@@ -583,28 +953,23 @@ nodelist_get_list,(void))
 /** Given a hex-encoded nickname of the format DIGEST, $DIGEST, $DIGEST=name,
  * or $DIGEST~name, return the node with the matching identity digest and
  * nickname (if any).  Return NULL if no such node exists, or if <b>hex_id</b>
- * is not well-formed. */
+ * is not well-formed. DOCDOC flags */
 const node_t *
-node_get_by_hex_id(const char *hex_id)
+node_get_by_hex_id(const char *hex_id, unsigned flags)
 {
   char digest_buf[DIGEST_LEN];
   char nn_buf[MAX_NICKNAME_LEN+1];
   char nn_char='\0';
 
+  (void) flags; // XXXX
+
   if (hex_digest_nickname_decode(hex_id, digest_buf, &nn_char, nn_buf)==0) {
     const node_t *node = node_get_by_id(digest_buf);
     if (!node)
       return NULL;
-    if (nn_char) {
-      const char *real_name = node_get_nickname(node);
-      if (!real_name || strcasecmp(real_name, nn_buf))
-        return NULL;
-      if (nn_char == '=') {
-        const char *named_id =
-          networkstatus_get_router_digest_by_nickname(nn_buf);
-        if (!named_id || tor_memneq(named_id, digest_buf, DIGEST_LEN))
-          return NULL;
-      }
+    if (nn_char == '=') {
+      /* "=" indicates a Named relay, but there aren't any of those now. */
+      return NULL;
     }
     return node;
   }
@@ -613,42 +978,27 @@ node_get_by_hex_id(const char *hex_id)
 }
 
 /** Given a nickname (possibly verbose, possibly a hexadecimal digest), return
- * the corresponding node_t, or NULL if none exists.  Warn the user if
- * <b>warn_if_unnamed</b> is set, and they have specified a router by
- * nickname, but the Named flag isn't set for that router. */
+ * the corresponding node_t, or NULL if none exists. Warn the user if they
+ * have specified a router by nickname, unless the NNF_NO_WARN_UNNAMED bit is
+ * set in <b>flags</b>. */
 MOCK_IMPL(const node_t *,
-node_get_by_nickname,(const char *nickname, int warn_if_unnamed))
+node_get_by_nickname,(const char *nickname, unsigned flags))
 {
+  const int warn_if_unnamed = !(flags & NNF_NO_WARN_UNNAMED);
+
   if (!the_nodelist)
     return NULL;
 
   /* Handle these cases: DIGEST, $DIGEST, $DIGEST=name, $DIGEST~name. */
   {
     const node_t *node;
-    if ((node = node_get_by_hex_id(nickname)) != NULL)
+    if ((node = node_get_by_hex_id(nickname, flags)) != NULL)
       return node;
   }
 
   if (!strcasecmp(nickname, UNNAMED_ROUTER_NICKNAME))
     return NULL;
 
-  /* Okay, so if we get here, the nickname is just a nickname.  Is there
-   * a binding for it in the consensus? */
-  {
-    const char *named_id =
-      networkstatus_get_router_digest_by_nickname(nickname);
-    if (named_id)
-      return node_get_by_id(named_id);
-  }
-
-  /* Is it marked as owned-by-someone-else? */
-  if (networkstatus_nickname_is_unnamed(nickname)) {
-    log_info(LD_GENERAL, "The name %s is listed as Unnamed: there is some "
-             "router that holds it, but not one listed in the current "
-             "consensus.", escaped(nickname));
-    return NULL;
-  }
-
   /* Okay, so the name is not canonical for anybody. */
   {
     smartlist_t *matches = smartlist_new();
@@ -669,8 +1019,7 @@ node_get_by_nickname,(const char *nickname, int warn_if_unnamed))
       } SMARTLIST_FOREACH_END(node);
 
       if (any_unwarned) {
-        log_warn(LD_CONFIG, "There are multiple matches for the name %s, "
-                 "but none is listed as Named in the directory consensus. "
+        log_warn(LD_CONFIG, "There are multiple matches for the name %s. "
                  "Choosing one arbitrarily.", nickname);
       }
     } else if (smartlist_len(matches)==1 && warn_if_unnamed) {
@@ -679,11 +1028,9 @@ node_get_by_nickname,(const char *nickname, int warn_if_unnamed))
       if (! node->name_lookup_warned) {
         base16_encode(fp, sizeof(fp), node->identity, DIGEST_LEN);
         log_warn(LD_CONFIG,
-                 "You specified a server \"%s\" by name, but the directory "
-                 "authorities do not have any key registered for this "
-                 "nickname -- so it could be used by any server, not just "
-                 "the one you meant. "
-                 "To make sure you get the same server in the future, refer "
+                 "You specified a relay \"%s\" by name, but nicknames can be "
+                 "used by any relay, not just the one you meant. "
+                 "To make sure you get the same relay in the future, refer "
                  "to it by key, as \"$%s\".", nickname, fp);
         node->name_lookup_warned = 1;
       }
@@ -697,6 +1044,150 @@ node_get_by_nickname,(const char *nickname, int warn_if_unnamed))
   }
 }
 
+/** Return the Ed25519 identity key for the provided node, or NULL if it
+ * doesn't have one. */
+const ed25519_public_key_t *
+node_get_ed25519_id(const node_t *node)
+{
+  const ed25519_public_key_t *ri_pk = NULL;
+  const ed25519_public_key_t *md_pk = NULL;
+
+  if (node->ri) {
+    if (node->ri->cache_info.signing_key_cert) {
+      ri_pk = &node->ri->cache_info.signing_key_cert->signing_key;
+      /* Checking whether routerinfo ed25519 is all zero.
+       * Our descriptor parser should make sure this never happens. */
+      if (BUG(ed25519_public_key_is_zero(ri_pk)))
+        ri_pk = NULL;
+    }
+  }
+
+  if (node->md) {
+    if (node->md->ed25519_identity_pkey) {
+      md_pk = node->md->ed25519_identity_pkey;
+      /* Checking whether microdesc ed25519 is all zero.
+       * Our descriptor parser should make sure this never happens. */
+      if (BUG(ed25519_public_key_is_zero(md_pk)))
+        md_pk = NULL;
+    }
+  }
+
+  if (ri_pk && md_pk) {
+    if (ed25519_pubkey_eq(ri_pk, md_pk)) {
+      return ri_pk;
+    } else {
+      /* This can happen if the relay gets flagged NoEdConsensus which will be
+       * triggered on all relays of the network. Thus a protocol warning. */
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "Inconsistent ed25519 identities in the nodelist");
+      return NULL;
+    }
+  } else if (ri_pk) {
+    return ri_pk;
+  } else {
+    return md_pk;
+  }
+}
+
+/** Return true iff this node's Ed25519 identity matches <b>id</b>.
+ * (An absent Ed25519 identity matches NULL or zero.) */
+int
+node_ed25519_id_matches(const node_t *node, const ed25519_public_key_t *id)
+{
+  const ed25519_public_key_t *node_id = node_get_ed25519_id(node);
+  if (node_id == NULL || ed25519_public_key_is_zero(node_id)) {
+    return id == NULL || ed25519_public_key_is_zero(id);
+  } else {
+    return id && ed25519_pubkey_eq(node_id, id);
+  }
+}
+
+/** Dummy object that should be unreturnable.  Used to ensure that
+ * node_get_protover_summary_flags() always returns non-NULL. */
+static const protover_summary_flags_t zero_protover_flags = {
+  0,0,0,0,0,0,0
+};
+
+/** Return the protover_summary_flags for a given node. */
+static const protover_summary_flags_t *
+node_get_protover_summary_flags(const node_t *node)
+{
+  if (node->rs) {
+    return &node->rs->pv;
+  } else if (node->ri) {
+    return &node->ri->pv;
+  } else {
+    /* This should be impossible: every node should have a routerstatus or a
+     * router descriptor or both. But just in case we've messed up somehow,
+     * return a nice empty set of flags to indicate "this node supports
+     * nothing." */
+    tor_assert_nonfatal_unreached_once();
+    return &zero_protover_flags;
+  }
+}
+
+/** Return true iff <b>node</b> supports authenticating itself
+ * by ed25519 ID during the link handshake.  If <b>compatible_with_us</b>,
+ * it needs to be using a link authentication method that we understand.
+ * If not, any plausible link authentication method will do. */
+int
+node_supports_ed25519_link_authentication(const node_t *node,
+                                          int compatible_with_us)
+{
+  if (! node_get_ed25519_id(node))
+    return 0;
+
+  const protover_summary_flags_t *pv = node_get_protover_summary_flags(node);
+
+  if (compatible_with_us)
+    return pv->supports_ed25519_link_handshake_compat;
+  else
+    return pv->supports_ed25519_link_handshake_any;
+}
+
+/** Return true iff <b>node</b> supports the hidden service directory version
+ * 3 protocol (proposal 224). */
+int
+node_supports_v3_hsdir(const node_t *node)
+{
+  tor_assert(node);
+
+  return node_get_protover_summary_flags(node)->supports_v3_hsdir;
+}
+
+/** Return true iff <b>node</b> supports ed25519 authentication as an hidden
+ * service introduction point.*/
+int
+node_supports_ed25519_hs_intro(const node_t *node)
+{
+  tor_assert(node);
+
+  return node_get_protover_summary_flags(node)->supports_ed25519_hs_intro;
+}
+
+/** Return true iff <b>node</b> supports to be a rendezvous point for hidden
+ * service version 3 (HSRend=2). */
+int
+node_supports_v3_rendezvous_point(const node_t *node)
+{
+  tor_assert(node);
+
+  /* We can't use a v3 rendezvous point without the curve25519 onion pk. */
+  if (!node_get_curve25519_onion_key(node)) {
+    return 0;
+  }
+
+  return node_get_protover_summary_flags(node)->supports_v3_rendezvous_point;
+}
+
+/** Return the RSA ID key's SHA1 digest for the provided node. */
+const uint8_t *
+node_get_rsa_id_digest(const node_t *node)
+{
+  tor_assert(node);
+  return (const uint8_t*)node->identity;
+}
+
 /** Return the nickname of <b>node</b>, or NULL if we can't find one. */
 const char *
 node_get_nickname(const node_t *node)
@@ -710,21 +1201,6 @@ node_get_nickname(const node_t *node)
     return NULL;
 }
 
-/** Return true iff the nickname of <b>node</b> is canonical, based on the
- * latest consensus. */
-int
-node_is_named(const node_t *node)
-{
-  const char *named_id;
-  const char *nickname = node_get_nickname(node);
-  if (!nickname)
-    return 0;
-  named_id = networkstatus_get_router_digest_by_nickname(nickname);
-  if (!named_id)
-    return 0;
-  return tor_memeq(named_id, node->identity, DIGEST_LEN);
-}
-
 /** Return true iff <b>node</b> appears to be a directory authority or
  * directory cache */
 int
@@ -745,15 +1221,44 @@ node_is_dir(const node_t *node)
   }
 }
 
-/** Return true iff <b>node</b> has either kind of usable descriptor -- that
- * is, a routerdescriptor or a microdescriptor. */
+/** Return true iff <b>node</b> has either kind of descriptor -- that
+ * is, a routerdescriptor or a microdescriptor.
+ *
+ * You should probably use node_has_preferred_descriptor() instead.
+ **/
 int
-node_has_descriptor(const node_t *node)
+node_has_any_descriptor(const node_t *node)
 {
   return (node->ri ||
           (node->rs && node->md));
 }
 
+/** Return true iff <b>node</b> has the kind of descriptor we would prefer to
+ * use for it, given our configuration and how we intend to use the node.
+ *
+ * If <b>for_direct_connect</b> is true, we intend to connect to the node
+ * directly, as the first hop of a circuit; otherwise, we intend to connect to
+ * it indirectly, or use it as if we were connecting to it indirectly. */
+int
+node_has_preferred_descriptor(const node_t *node,
+                              int for_direct_connect)
+{
+  const int is_bridge = node_is_a_configured_bridge(node);
+  const int we_use_mds = we_use_microdescriptors_for_circuits(get_options());
+
+  if ((is_bridge && for_direct_connect) || !we_use_mds) {
+    /* We need an ri in this case. */
+    if (!node->ri)
+      return 0;
+  } else {
+    /* Otherwise we need an rs and an md. */
+    if (node->rs == NULL || node->md == NULL)
+      return 0;
+  }
+
+  return 1;
+}
+
 /** Return the router_purpose of <b>node</b>. */
 int
 node_get_purpose(const node_t *node)
@@ -772,13 +1277,12 @@ node_get_verbose_nickname(const node_t *node,
                           char *verbose_name_out)
 {
   const char *nickname = node_get_nickname(node);
-  int is_named = node_is_named(node);
   verbose_name_out[0] = '$';
   base16_encode(verbose_name_out+1, HEX_DIGEST_LEN+1, node->identity,
                 DIGEST_LEN);
   if (!nickname)
     return;
-  verbose_name_out[1+HEX_DIGEST_LEN] = is_named ? '=' : '~';
+  verbose_name_out[1+HEX_DIGEST_LEN] = '~';
   strlcpy(verbose_name_out+1+HEX_DIGEST_LEN+1, nickname, MAX_NICKNAME_LEN+1);
 }
 
@@ -992,16 +1496,6 @@ node_get_platform(const node_t *node)
     return NULL;
 }
 
-/** Return <b>node</b>'s time of publication, or 0 if we don't have one. */
-time_t
-node_get_published_on(const node_t *node)
-{
-  if (node->ri)
-    return node->ri->cache_info.published_on;
-  else
-    return 0;
-}
-
 /** Return true iff <b>node</b> is one representing this router. */
 int
 node_is_me(const node_t *node)
@@ -1082,9 +1576,11 @@ node_ipv6_or_preferred(const node_t *node)
   /* XX/teor - node->ipv6_preferred is set from
    * fascist_firewall_prefer_ipv6_orport() each time the consensus is loaded.
    */
+  node_get_prim_orport(node, &ipv4_addr);
   if (!fascist_firewall_use_ipv6(options)) {
     return 0;
-  } else if (node->ipv6_preferred || node_get_prim_orport(node, &ipv4_addr)) {
+  } else if (node->ipv6_preferred ||
+             !tor_addr_port_is_valid_ap(&ipv4_addr, 0)) {
     return node_has_ipv6_orport(node);
   }
   return 0;
@@ -1095,14 +1591,12 @@ node_ipv6_or_preferred(const node_t *node)
     if (r && tor_addr_port_is_valid_ipv4h((r)->addr, (r)->port_field, 0)) { \
       tor_addr_from_ipv4h(&(ap_out)->addr, (r)->addr); \
       (ap_out)->port = (r)->port_field; \
-      return 0; \
     } \
   STMT_END
 
-/** Copy the primary (IPv4) OR port (IP address and TCP port) for
- * <b>node</b> into *<b>ap_out</b>. Return 0 if a valid address and
- * port was copied, else return non-zero.*/
-int
+/** Copy the primary (IPv4) OR port (IP address and TCP port) for <b>node</b>
+ * into *<b>ap_out</b>. */
+void
 node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out)
 {
   node_assert_ok(node);
@@ -1119,8 +1613,6 @@ node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out)
   RETURN_IPV4_AP(node->ri, or_port, ap_out);
   RETURN_IPV4_AP(node->rs, or_port, ap_out);
   /* Microdescriptors only have an IPv6 address */
-
-  return -1;
 }
 
 /** Copy the preferred OR port (IP address and TCP port) for
@@ -1145,10 +1637,13 @@ node_get_pref_ipv6_orport(const node_t *node, tor_addr_port_t *ap_out)
 {
   node_assert_ok(node);
   tor_assert(ap_out);
+  memset(ap_out, 0, sizeof(*ap_out));
 
-  /* Prefer routerstatus over microdesc for consistency with the
-   * fascist_firewall_* functions. Also check if the address or port are valid,
-   * and try another alternative if they are not. */
+  /* Check ri first, because rewrite_node_address_for_bridge() updates
+   * node->ri with the configured bridge address.
+   * Prefer rs over md for consistency with the fascist_firewall_* functions.
+   * Check if the address or port are valid, and try another alternative
+   * if they are not. */
 
   if (node->ri && tor_addr_port_is_valid(&node->ri->ipv6_addr,
                                          node->ri->ipv6_orport, 0)) {
@@ -1190,29 +1685,35 @@ node_ipv6_dir_preferred(const node_t *node)
    * so we can't use it to determine DirPort IPv6 preference.
    * This means that bridge clients will use IPv4 DirPorts by default.
    */
+  node_get_prim_dirport(node, &ipv4_addr);
   if (!fascist_firewall_use_ipv6(options)) {
     return 0;
-  } else if (node_get_prim_dirport(node, &ipv4_addr)
+  } else if (!tor_addr_port_is_valid_ap(&ipv4_addr, 0)
       || fascist_firewall_prefer_ipv6_dirport(get_options())) {
     return node_has_ipv6_dirport(node);
   }
   return 0;
 }
 
-/** Copy the primary (IPv4) Dir port (IP address and TCP port) for
- * <b>node</b> into *<b>ap_out</b>. Return 0 if a valid address and
- * port was copied, else return non-zero.*/
-int
+/** Copy the primary (IPv4) Dir port (IP address and TCP port) for <b>node</b>
+ * into *<b>ap_out</b>. */
+void
 node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out)
 {
   node_assert_ok(node);
   tor_assert(ap_out);
 
+  /* Clear the address, as a safety precaution if calling functions ignore the
+   * return value */
+  tor_addr_make_null(&ap_out->addr, AF_INET);
+  ap_out->port = 0;
+
+  /* Check ri first, because rewrite_node_address_for_bridge() updates
+   * node->ri with the configured bridge address. */
+
   RETURN_IPV4_AP(node->ri, dir_port, ap_out);
   RETURN_IPV4_AP(node->rs, dir_port, ap_out);
   /* Microdescriptors only have an IPv6 address */
-
-  return -1;
 }
 
 #undef RETURN_IPV4_AP
@@ -1240,8 +1741,11 @@ node_get_pref_ipv6_dirport(const node_t *node, tor_addr_port_t *ap_out)
   node_assert_ok(node);
   tor_assert(ap_out);
 
-  /* Check if the address or port are valid, and try another alternative if
-   * they are not. Note that microdescriptors have no dir_port. */
+  /* Check ri first, because rewrite_node_address_for_bridge() updates
+   * node->ri with the configured bridge address.
+   * Prefer rs over md for consistency with the fascist_firewall_* functions.
+   * Check if the address or port are valid, and try another alternative
+   * if they are not. */
 
   /* Assume IPv4 and IPv6 dirports are the same */
   if (node->ri && tor_addr_port_is_valid(&node->ri->ipv6_addr,
@@ -1283,15 +1787,52 @@ microdesc_has_curve25519_onion_key(const microdesc_t *md)
 int
 node_has_curve25519_onion_key(const node_t *node)
 {
-  if (!node)
-    return 0;
+  return node_get_curve25519_onion_key(node) != NULL;
+}
 
-  if (node->ri)
-    return routerinfo_has_curve25519_onion_key(node->ri);
-  else if (node->md)
-    return microdesc_has_curve25519_onion_key(node->md);
+/** Return the curve25519 key of <b>node</b>, or NULL if none. */
+const curve25519_public_key_t *
+node_get_curve25519_onion_key(const node_t *node)
+{
+  if (!node)
+    return NULL;
+  if (routerinfo_has_curve25519_onion_key(node->ri))
+    return node->ri->onion_curve25519_pkey;
+  else if (microdesc_has_curve25519_onion_key(node->md))
+    return node->md->onion_curve25519_pkey;
   else
-    return 0;
+    return NULL;
+}
+
+/* Return a newly allocacted RSA onion public key taken from the given node.
+ *
+ * Return NULL if node is NULL or no RSA onion public key can be found. It is
+ * the caller responsability to free the returned object. */
+crypto_pk_t *
+node_get_rsa_onion_key(const node_t *node)
+{
+  crypto_pk_t *pk = NULL;
+  const char *onion_pkey;
+  size_t onion_pkey_len;
+
+  if (!node) {
+    goto end;
+  }
+
+  if (node->ri) {
+    onion_pkey = node->ri->onion_pkey;
+    onion_pkey_len = node->ri->onion_pkey_len;
+  } else if (node->rs && node->md) {
+    onion_pkey = node->md->onion_pkey;
+    onion_pkey_len = node->md->onion_pkey_len;
+  } else {
+    /* No descriptor or microdescriptor. */
+    goto end;
+  }
+  pk = router_get_rsa_onion_pkey(onion_pkey, onion_pkey_len);
+
+ end:
+  return pk;
 }
 
 /** Refresh the country code of <b>ri</b>.  This function MUST be called on
@@ -1321,11 +1862,19 @@ nodelist_refresh_countries(void)
 
 /** Return true iff router1 and router2 have similar enough network addresses
  * that we should treat them as being in the same family */
-static inline int
+int
 addrs_in_same_network_family(const tor_addr_t *a1,
                              const tor_addr_t *a2)
 {
-  return 0 == tor_addr_compare_masked(a1, a2, 16, CMP_SEMANTIC);
+  switch (tor_addr_family(a1)) {
+    case AF_INET:
+      return 0 == tor_addr_compare_masked(a1, a2, 16, CMP_SEMANTIC);
+    case AF_INET6:
+      return 0 == tor_addr_compare_masked(a1, a2, 32, CMP_SEMANTIC);
+    default:
+      /* If not IPv4 or IPv6, return 0. */
+      return 0;
+  }
 }
 
 /** Return true if <b>node</b>'s nickname matches <b>nickname</b>
@@ -1340,8 +1889,7 @@ node_nickname_matches(const node_t *node, const char *nickname)
     return 1;
   return hex_digest_nickname_matches(nickname,
                                      node->identity,
-                                     n,
-                                     node_is_named(node));
+                                     n);
 }
 
 /** Return true iff <b>node</b> is named by some nickname in <b>lst</b>. */
@@ -1383,7 +1931,7 @@ nodes_in_same_family(const node_t *node1, const node_t *node2)
       return 1;
   }
 
-  /* Are they in the same option because the user says they are? */
+  /* Are they in the same family because the user says they are? */
   if (options->NodeFamilySets) {
     SMARTLIST_FOREACH(options->NodeFamilySets, const routerset_t *, rs, {
         if (routerset_contains_node(rs, node1) &&
@@ -1443,7 +1991,7 @@ nodelist_add_node_and_family(smartlist_t *sl, const node_t *node)
     SMARTLIST_FOREACH_BEGIN(declared_family, const char *, name) {
       const node_t *node2;
       const smartlist_t *family2;
-      if (!(node2 = node_get_by_nickname(name, 0)))
+      if (!(node2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED)))
         continue;
       if (!(family2 = node_get_declared_family(node2)))
         continue;
@@ -1595,8 +2143,8 @@ static char dir_info_status[512] = "";
  * no exits in the consensus."
  * To obtain the final weighted bandwidth, we multiply the
  * weighted bandwidth fraction for each position (guard, middle, exit). */
-int
-router_have_minimum_dir_info(void)
+MOCK_IMPL(int,
+router_have_minimum_dir_info,(void))
 {
   static int logged_delay=0;
   const char *delay_fetches_msg = NULL;
@@ -1628,8 +2176,8 @@ router_have_minimum_dir_info(void)
  * this can cause router_have_consensus_path() to be set to
  * CONSENSUS_PATH_EXIT, even if there are no nodes with accept exit policies.
  */
-consensus_path_type_t
-router_have_consensus_path(void)
+MOCK_IMPL(consensus_path_type_t,
+router_have_consensus_path, (void))
 {
   return have_consensus_path;
 }
@@ -1643,6 +2191,8 @@ router_dir_info_changed(void)
 {
   need_to_update_have_min_dir_info = 1;
   rend_hsdir_routers_changed();
+  hs_service_dir_info_changed();
+  hs_client_dir_info_changed();
 }
 
 /** Return a string describing what we're missing before we have enough
@@ -1659,8 +2209,11 @@ get_dir_info_status_string(void)
  * *<b>num_present</b>).
  *
  * If <b>in_set</b> is non-NULL, only consider those routers in <b>in_set</b>.
- * If <b>exit_only</b> is USABLE_DESCRIPTOR_EXIT_ONLY, only consider nodes
- * with the Exit flag.
+ * If <b>exit_only</b> & USABLE_DESCRIPTOR_EXIT_POLICY, only consider nodes
+ * present if they have an exit policy that accepts at least one port.
+ * If <b>exit_only</b> & USABLE_DESCRIPTOR_EXIT_FLAG, only consider nodes
+ * usable if they have the exit flag in the consensus.
+ *
  * If *<b>descs_out</b> is present, add a node_t for each usable descriptor
  * to it.
  */
@@ -1668,7 +2221,7 @@ static void
 count_usable_descriptors(int *num_present, int *num_usable,
                          smartlist_t *descs_out,
                          const networkstatus_t *consensus,
-                         const or_options_t *options, time_t now,
+                         time_t now,
                          routerset_t *in_set,
                          usable_descriptor_t exit_only)
 {
@@ -1681,11 +2234,11 @@ count_usable_descriptors(int *num_present, int *num_usable,
        if (!node)
          continue; /* This would be a bug: every entry in the consensus is
                     * supposed to have a node. */
-       if (exit_only == USABLE_DESCRIPTOR_EXIT_ONLY && ! rs->is_exit)
+       if ((exit_only & USABLE_DESCRIPTOR_EXIT_FLAG) && ! rs->is_exit)
          continue;
        if (in_set && ! routerset_contains_routerstatus(in_set, rs, -1))
          continue;
-       if (client_would_use_router(rs, now, options)) {
+       if (client_would_use_router(rs, now)) {
          const char * const digest = rs->descriptor_digest;
          int present;
          ++*num_usable; /* the consensus says we want it. */
@@ -1694,7 +2247,14 @@ count_usable_descriptors(int *num_present, int *num_usable,
          else
            present = NULL != router_get_by_descriptor_digest(digest);
          if (present) {
-           /* we have the descriptor listed in the consensus. */
+           /* Do the policy check last, because it requires a descriptor,
+            * and is potentially expensive */
+           if ((exit_only & USABLE_DESCRIPTOR_EXIT_POLICY) &&
+               node_exit_policy_rejects_all(node)) {
+               continue;
+           }
+           /* we have the descriptor listed in the consensus, and it
+            * satisfies our exit constraints (if any) */
            ++*num_present;
          }
          if (descs_out)
@@ -1703,10 +2263,17 @@ count_usable_descriptors(int *num_present, int *num_usable,
      }
   SMARTLIST_FOREACH_END(rs);
 
-  log_debug(LD_DIR, "%d usable, %d present (%s%s).",
+  log_debug(LD_DIR, "%d usable, %d present (%s%s%s%s%s).",
             *num_usable, *num_present,
             md ? "microdesc" : "desc",
-            exit_only == USABLE_DESCRIPTOR_EXIT_ONLY ? " exits" : "s");
+            (exit_only & USABLE_DESCRIPTOR_EXIT_POLICY_AND_FLAG) ?
+              " exit"     : "s",
+            (exit_only & USABLE_DESCRIPTOR_EXIT_POLICY) ?
+              " policies" : "" ,
+            (exit_only == USABLE_DESCRIPTOR_EXIT_POLICY_AND_FLAG) ?
+              " and" : "" ,
+            (exit_only & USABLE_DESCRIPTOR_EXIT_FLAG) ?
+              " flags"    : "" );
 }
 
 /** Return an estimate of which fraction of usable paths through the Tor
@@ -1718,9 +2285,9 @@ count_usable_descriptors(int *num_present, int *num_usable,
  * If **<b>status_out</b> is present, allocate a new string and print the
  * available percentages of guard, middle, and exit nodes to it, noting
  * whether there are exits in the consensus.
- * If there are no guards in the consensus,
- * we treat the exit fraction as 100%.
- */
+ * If there are no exits in the consensus, we treat the exit fraction as 100%,
+ * but set router_have_consensus_path() so that we can only build internal
+ * paths. */
 static double
 compute_frac_paths_available(const networkstatus_t *consensus,
                              const or_options_t *options, time_t now,
@@ -1739,11 +2306,22 @@ compute_frac_paths_available(const networkstatus_t *consensus,
   const int authdir = authdir_mode_v3(options);
 
   count_usable_descriptors(num_present_out, num_usable_out,
-                           mid, consensus, options, now, NULL,
+                           mid, consensus, now, NULL,
                            USABLE_DESCRIPTOR_ALL);
+  log_debug(LD_NET,
+            "%s: %d present, %d usable",
+            "mid",
+            np,
+            nu);
+
   if (options->EntryNodes) {
-    count_usable_descriptors(&np, &nu, guards, consensus, options, now,
+    count_usable_descriptors(&np, &nu, guards, consensus, now,
                              options->EntryNodes, USABLE_DESCRIPTOR_ALL);
+    log_debug(LD_NET,
+              "%s: %d present, %d usable",
+              "guard",
+              np,
+              nu);
   } else {
     SMARTLIST_FOREACH(mid, const node_t *, node, {
       if (authdir) {
@@ -1754,47 +2332,55 @@ compute_frac_paths_available(const networkstatus_t *consensus,
           smartlist_add(guards, (node_t*)node);
       }
     });
+    log_debug(LD_NET,
+              "%s: %d possible",
+              "guard",
+              smartlist_len(guards));
   }
 
-  /* All nodes with exit flag
-   * If we're in a network with TestingDirAuthVoteExit set,
-   * this can cause false positives on have_consensus_path,
-   * incorrectly setting it to CONSENSUS_PATH_EXIT. This is
-   * an unavoidable feature of forcing authorities to declare
-   * certain nodes as exits.
-   */
-  count_usable_descriptors(&np, &nu, exits, consensus, options, now,
-                           NULL, USABLE_DESCRIPTOR_EXIT_ONLY);
+  /* All nodes with exit policy and flag */
+  count_usable_descriptors(&np, &nu, exits, consensus, now,
+                           NULL, USABLE_DESCRIPTOR_EXIT_POLICY_AND_FLAG);
   log_debug(LD_NET,
             "%s: %d present, %d usable",
             "exits",
             np,
             nu);
 
-  /* We need at least 1 exit present in the consensus to consider
+  /* We need at least 1 exit (flag and policy) in the consensus to consider
    * building exit paths */
   /* Update our understanding of whether the consensus has exits */
   consensus_path_type_t old_have_consensus_path = have_consensus_path;
-  have_consensus_path = ((nu > 0) ?
+  have_consensus_path = ((np > 0) ?
                          CONSENSUS_PATH_EXIT :
                          CONSENSUS_PATH_INTERNAL);
 
-  if (have_consensus_path == CONSENSUS_PATH_INTERNAL
-      && old_have_consensus_path != have_consensus_path) {
-    log_notice(LD_NET,
-               "The current consensus has no exit nodes. "
-               "Tor can only build internal paths, "
-               "such as paths to hidden services.");
-
-    /* However, exit nodes can reachability self-test using this consensus,
-     * join the network, and appear in a later consensus. This will allow
-     * the network to build exit paths, such as paths for world wide web
-     * browsing (as distinct from hidden service web browsing). */
+  if (old_have_consensus_path != have_consensus_path) {
+    if (have_consensus_path == CONSENSUS_PATH_INTERNAL) {
+      log_notice(LD_NET,
+                 "The current consensus has no exit nodes. "
+                 "Tor can only build internal paths, "
+                 "such as paths to onion services.");
+
+      /* However, exit nodes can reachability self-test using this consensus,
+       * join the network, and appear in a later consensus. This will allow
+       * the network to build exit paths, such as paths for world wide web
+       * browsing (as distinct from hidden service web browsing). */
+    } else if (old_have_consensus_path == CONSENSUS_PATH_INTERNAL) {
+      log_notice(LD_NET,
+                 "The current consensus contains exit nodes. "
+                 "Tor can build exit and internal paths.");
+    }
   }
 
-  f_guard = frac_nodes_with_descriptors(guards, WEIGHT_FOR_GUARD);
-  f_mid   = frac_nodes_with_descriptors(mid,    WEIGHT_FOR_MID);
-  f_exit  = frac_nodes_with_descriptors(exits,  WEIGHT_FOR_EXIT);
+  f_guard = frac_nodes_with_descriptors(guards, WEIGHT_FOR_GUARD, 1);
+  f_mid   = frac_nodes_with_descriptors(mid,    WEIGHT_FOR_MID,   0);
+  f_exit  = frac_nodes_with_descriptors(exits,  WEIGHT_FOR_EXIT,  0);
+
+  /* If we are using bridges and have at least one bridge with a full
+   * descriptor, assume f_guard is 1.0. */
+  if (options->UseBridges && num_bridges_usable(0) > 0)
+    f_guard = 1.0;
 
   log_debug(LD_NET,
             "f_guard: %.2f, f_mid: %.2f, f_exit: %.2f",
@@ -1811,45 +2397,32 @@ compute_frac_paths_available(const networkstatus_t *consensus,
     smartlist_t *myexits= smartlist_new();
     smartlist_t *myexits_unflagged = smartlist_new();
 
-    /* All nodes with exit flag in ExitNodes option */
-    count_usable_descriptors(&np, &nu, myexits, consensus, options, now,
-                             options->ExitNodes, USABLE_DESCRIPTOR_EXIT_ONLY);
+    /* All nodes with exit policy and flag in ExitNodes option */
+    count_usable_descriptors(&np, &nu, myexits, consensus, now,
+                             options->ExitNodes,
+                             USABLE_DESCRIPTOR_EXIT_POLICY_AND_FLAG);
     log_debug(LD_NET,
               "%s: %d present, %d usable",
               "myexits",
               np,
               nu);
 
-    /* Now compute the nodes in the ExitNodes option where which we don't know
-     * what their exit policy is, or we know it permits something. */
+    /* Now compute the nodes in the ExitNodes option where we know their exit
+     * policy permits something. */
     count_usable_descriptors(&np, &nu, myexits_unflagged,
-                             consensus, options, now,
-                             options->ExitNodes, USABLE_DESCRIPTOR_ALL);
+                             consensus, now,
+                             options->ExitNodes,
+                             USABLE_DESCRIPTOR_EXIT_POLICY);
     log_debug(LD_NET,
               "%s: %d present, %d usable",
               "myexits_unflagged (initial)",
               np,
               nu);
 
-    SMARTLIST_FOREACH_BEGIN(myexits_unflagged, const node_t *, node) {
-      if (node_has_descriptor(node) && node_exit_policy_rejects_all(node)) {
-        SMARTLIST_DEL_CURRENT(myexits_unflagged, node);
-        /* this node is not actually an exit */
-        np--;
-        /* this node is unusable as an exit */
-        nu--;
-      }
-    } SMARTLIST_FOREACH_END(node);
-
-    log_debug(LD_NET,
-              "%s: %d present, %d usable",
-              "myexits_unflagged (final)",
-              np,
-              nu);
-
-    f_myexit= frac_nodes_with_descriptors(myexits,WEIGHT_FOR_EXIT);
+    f_myexit= frac_nodes_with_descriptors(myexits,WEIGHT_FOR_EXIT, 0);
     f_myexit_unflagged=
-              frac_nodes_with_descriptors(myexits_unflagged,WEIGHT_FOR_EXIT);
+              frac_nodes_with_descriptors(myexits_unflagged,
+                                          WEIGHT_FOR_EXIT, 0);
 
     log_debug(LD_NET,
               "f_exit: %.2f, f_myexit: %.2f, f_myexit_unflagged: %.2f",
@@ -1876,9 +2449,20 @@ compute_frac_paths_available(const networkstatus_t *consensus,
       f_exit = f_myexit;
   }
 
-  /* if the consensus has no exits, treat the exit fraction as 100% */
+  /* If the consensus has no exits that pass flag, descriptor, and policy
+   * checks, we can only build onion service paths, which are G - M - M. */
   if (router_have_consensus_path() != CONSENSUS_PATH_EXIT) {
-    f_exit = 1.0;
+    /* If the exit bandwidth weight fraction is not zero, we need to wait for
+     * descriptors for those exits. (The bandwidth weight fraction does not
+     * check for descriptors.)
+     * If the exit bandwidth fraction is zero, there are no exits in the
+     * consensus at all. So it is safe to replace f_exit with f_mid.
+     *
+     * f_exit is non-negative, but some compilers complain about float and ==
+     */
+    if (f_exit <= 0.0) {
+      f_exit = f_mid;
+    }
   }
 
   f_path = f_guard * f_mid * f_exit;
@@ -1887,14 +2471,14 @@ compute_frac_paths_available(const networkstatus_t *consensus,
     tor_asprintf(status_out,
                  "%d%% of guards bw, "
                  "%d%% of midpoint bw, and "
-                 "%d%% of exit bw%s = "
+                 "%d%% of %s = "
                  "%d%% of path bw",
                  (int)(f_guard*100),
                  (int)(f_mid*100),
                  (int)(f_exit*100),
                  (router_have_consensus_path() == CONSENSUS_PATH_EXIT ?
-                  "" :
-                  " (no exits in consensus)"),
+                  "exit bw" :
+                  "end bw (no exits in consensus, using mid)"),
                  (int)(f_path*100));
 
   return f_path;
@@ -1952,6 +2536,7 @@ update_router_have_minimum_dir_info(void)
 {
   time_t now = time(NULL);
   int res;
+  int num_present=0, num_usable=0;
   const or_options_t *options = get_options();
   const networkstatus_t *consensus =
     networkstatus_get_reasonably_live_consensus(now,usable_consensus_flavor());
@@ -1973,7 +2558,6 @@ update_router_have_minimum_dir_info(void)
   /* Check fraction of available paths */
   {
     char *status = NULL;
-    int num_present=0, num_usable=0;
     double paths = compute_frac_paths_available(consensus, options, now,
                                                 &num_present, &num_usable,
                                                 &status);
@@ -1986,7 +2570,7 @@ update_router_have_minimum_dir_info(void)
                    (int)(paths*100), status);
       tor_free(status);
       res = 0;
-      control_event_bootstrap(BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, 0);
+      control_event_boot_dir(BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, 0);
       goto done;
     }
 
@@ -1994,15 +2578,26 @@ update_router_have_minimum_dir_info(void)
     res = 1;
   }
 
+  { /* Check entry guard dirinfo status */
+    char *guard_error = entry_guards_get_err_str_if_dir_info_missing(using_md,
+                                                             num_present,
+                                                             num_usable);
+    if (guard_error) {
+      strlcpy(dir_info_status, guard_error, sizeof(dir_info_status));
+      tor_free(guard_error);
+      res = 0;
+      goto done;
+    }
+  }
+
  done:
 
   /* If paths have just become available in this update. */
   if (res && !have_min_dir_info) {
     control_event_client_status(LOG_NOTICE, "ENOUGH_DIR_INFO");
-    if (control_event_bootstrap(BOOTSTRAP_STATUS_CONN_OR, 0) == 0) {
-      log_notice(LD_DIR,
-              "We now have enough directory information to build circuits.");
-    }
+    control_event_boot_dir(BOOTSTRAP_STATUS_CONN_OR, 0);
+    log_info(LD_DIR,
+             "We now have enough directory information to build circuits.");
   }
 
   /* If paths have just become unavailable in this update. */
@@ -2023,4 +2618,3 @@ update_router_have_minimum_dir_info(void)
   have_min_dir_info = res;
   need_to_update_have_min_dir_info = 0;
 }
-
diff --git a/src/or/nodelist.h b/src/feature/nodelist/nodelist.h
index 098f1d1555..c430f497d5 100644
--- a/src/or/nodelist.h
+++ b/src/feature/nodelist/nodelist.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,16 +12,28 @@
 #ifndef TOR_NODELIST_H
 #define TOR_NODELIST_H
 
+struct ed25519_public_key_t;
+struct curve25519_public_key_t;
+
 #define node_assert_ok(n) STMT_BEGIN {                          \
     tor_assert((n)->ri || (n)->rs);                             \
   } STMT_END
 
-node_t *node_get_mutable_by_id(const char *identity_digest);
+MOCK_DECL(node_t *, node_get_mutable_by_id,(const char *identity_digest));
 MOCK_DECL(const node_t *, node_get_by_id, (const char *identity_digest));
-const node_t *node_get_by_hex_id(const char *identity_digest);
+node_t *node_get_mutable_by_ed25519_id(
+                            const struct ed25519_public_key_t *ed_id);
+MOCK_DECL(const node_t *, node_get_by_ed25519_id,
+          (const struct ed25519_public_key_t *ed_id));
+
+#define NNF_NO_WARN_UNNAMED (1u<<0)
+
+const node_t *node_get_by_hex_id(const char *identity_digest,
+                                 unsigned flags);
 node_t *nodelist_set_routerinfo(routerinfo_t *ri, routerinfo_t **ri_old_out);
 node_t *nodelist_add_microdesc(microdesc_t *md);
 void nodelist_set_consensus(networkstatus_t *ns);
+void nodelist_ensure_freshness(networkstatus_t *ns);
 int nodelist_probably_contains_address(const tor_addr_t *addr);
 
 void nodelist_remove_microdesc(const char *identity_digest, microdesc_t *md);
@@ -33,14 +45,16 @@ void nodelist_free_all(void);
 void nodelist_assert_ok(void);
 
 MOCK_DECL(const node_t *, node_get_by_nickname,
-    (const char *nickname, int warn_if_unnamed));
+          (const char *nickname, unsigned flags));
 void node_get_verbose_nickname(const node_t *node,
                                char *verbose_name_out);
 void node_get_verbose_nickname_by_id(const char *id_digest,
                                 char *verbose_name_out);
-int node_is_named(const node_t *node);
 int node_is_dir(const node_t *node);
-int node_has_descriptor(const node_t *node);
+int node_is_good_exit(const node_t *node);
+int node_has_any_descriptor(const node_t *node);
+int node_has_preferred_descriptor(const node_t *node,
+                                  int for_direct_connect);
 int node_get_purpose(const node_t *node);
 #define node_is_bridge(node) \
   (node_get_purpose((node)) == ROUTER_PURPOSE_BRIDGE)
@@ -54,8 +68,16 @@ const char *node_get_platform(const node_t *node);
 uint32_t node_get_prim_addr_ipv4h(const node_t *node);
 void node_get_address_string(const node_t *node, char *cp, size_t len);
 long node_get_declared_uptime(const node_t *node);
-time_t node_get_published_on(const node_t *node);
 const smartlist_t *node_get_declared_family(const node_t *node);
+const struct ed25519_public_key_t *node_get_ed25519_id(const node_t *node);
+int node_ed25519_id_matches(const node_t *node,
+                            const struct ed25519_public_key_t *id);
+int node_supports_ed25519_link_authentication(const node_t *node,
+                                              int compatible_with_us);
+int node_supports_v3_hsdir(const node_t *node);
+int node_supports_ed25519_hs_intro(const node_t *node);
+int node_supports_v3_rendezvous_point(const node_t *node);
+const uint8_t *node_get_rsa_id_digest(const node_t *node);
 
 int node_has_ipv6_addr(const node_t *node);
 int node_has_ipv6_orport(const node_t *node);
@@ -63,14 +85,17 @@ int node_has_ipv6_dirport(const node_t *node);
 /* Deprecated - use node_ipv6_or_preferred or node_ipv6_dir_preferred */
 #define node_ipv6_preferred(node) node_ipv6_or_preferred(node)
 int node_ipv6_or_preferred(const node_t *node);
-int node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out);
+void node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out);
 void node_get_pref_orport(const node_t *node, tor_addr_port_t *ap_out);
 void node_get_pref_ipv6_orport(const node_t *node, tor_addr_port_t *ap_out);
 int node_ipv6_dir_preferred(const node_t *node);
-int node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out);
+void node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out);
 void node_get_pref_dirport(const node_t *node, tor_addr_port_t *ap_out);
 void node_get_pref_ipv6_dirport(const node_t *node, tor_addr_port_t *ap_out);
 int node_has_curve25519_onion_key(const node_t *node);
+const struct curve25519_public_key_t *node_get_curve25519_onion_key(
+                                  const node_t *node);
+crypto_pk_t *node_get_rsa_onion_key(const node_t *node);
 
 MOCK_DECL(smartlist_t *, nodelist_get_list, (void));
 
@@ -90,6 +115,8 @@ int node_is_unreliable(const node_t *router, int need_uptime,
 int router_exit_policy_all_nodes_reject(const tor_addr_t *addr, uint16_t port,
                                         int need_uptime);
 void router_set_status(const char *digest, int up);
+int addrs_in_same_network_family(const tor_addr_t *a1,
+                                 const tor_addr_t *a2);
 
 /** router_have_minimum_dir_info tests to see if we have enough
  * descriptor information to create circuits.
@@ -98,7 +125,7 @@ void router_set_status(const char *digest, int up);
  * no exits in the consensus, we wait for enough info to create internal
  * paths, and should avoid creating exit paths, as they will simply fail.
  * We make sure we create all available circuit types at the same time. */
-int router_have_minimum_dir_info(void);
+MOCK_DECL(int, router_have_minimum_dir_info,(void));
 
 /** Set to CONSENSUS_PATH_EXIT if there is at least one exit node
  * in the consensus. We update this flag in compute_frac_paths_available if
@@ -119,13 +146,24 @@ typedef enum {
    * create exit and internal paths, circuits, streams, ... */
   CONSENSUS_PATH_EXIT = 1
 } consensus_path_type_t;
-consensus_path_type_t router_have_consensus_path(void);
+
+MOCK_DECL(consensus_path_type_t, router_have_consensus_path, (void));
 
 void router_dir_info_changed(void);
 const char *get_dir_info_status_string(void);
 int count_loading_descriptors_progress(void);
 
-MOCK_DECL(int, get_estimated_address_per_node, (void));
+#ifdef NODELIST_PRIVATE
 
-#endif
+#ifdef TOR_UNIT_TESTS
+
+STATIC void
+node_set_hsdir_index(node_t *node, const networkstatus_t *ns);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(NODELIST_PRIVATE) */
+
+MOCK_DECL(int, get_estimated_address_per_node, (void));
 
+#endif /* !defined(TOR_NODELIST_H) */
diff --git a/src/feature/nodelist/routerinfo.c b/src/feature/nodelist/routerinfo.c
new file mode 100644
index 0000000000..975b503615
--- /dev/null
+++ b/src/feature/nodelist/routerinfo.c
@@ -0,0 +1,79 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerinfo.h"
+
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+
+/** Copy the primary (IPv4) OR port (IP address and TCP port) for
+ * <b>router</b> into *<b>ap_out</b>. */
+void
+router_get_prim_orport(const routerinfo_t *router, tor_addr_port_t *ap_out)
+{
+  tor_assert(ap_out != NULL);
+  tor_addr_from_ipv4h(&ap_out->addr, router->addr);
+  ap_out->port = router->or_port;
+}
+
+int
+router_has_orport(const routerinfo_t *router, const tor_addr_port_t *orport)
+{
+  return
+    (tor_addr_eq_ipv4h(&orport->addr, router->addr) &&
+     orport->port == router->or_port) ||
+    (tor_addr_eq(&orport->addr, &router->ipv6_addr) &&
+     orport->port == router->ipv6_orport);
+}
+
+/** Return a smartlist of tor_addr_port_t's with all the OR ports of
+    <b>ri</b>. Note that freeing of the items in the list as well as
+    the smartlist itself is the callers responsibility. */
+smartlist_t *
+router_get_all_orports(const routerinfo_t *ri)
+{
+  tor_assert(ri);
+  node_t fake_node;
+  memset(&fake_node, 0, sizeof(fake_node));
+  /* we don't modify ri, fake_node is passed as a const node_t *
+   */
+  fake_node.ri = (routerinfo_t *)ri;
+  return node_get_all_orports(&fake_node);
+}
+
+/** Given a router purpose, convert it to a string.  Don't call this on
+ * ROUTER_PURPOSE_UNKNOWN: The whole point of that value is that we don't
+ * know its string representation. */
+const char *
+router_purpose_to_string(uint8_t p)
+{
+  switch (p)
+    {
+    case ROUTER_PURPOSE_GENERAL: return "general";
+    case ROUTER_PURPOSE_BRIDGE: return "bridge";
+    case ROUTER_PURPOSE_CONTROLLER: return "controller";
+    default:
+      tor_assert(0);
+    }
+  return NULL;
+}
+
+/** Given a string, convert it to a router purpose. */
+uint8_t
+router_purpose_from_string(const char *s)
+{
+  if (!strcmp(s, "general"))
+    return ROUTER_PURPOSE_GENERAL;
+  else if (!strcmp(s, "bridge"))
+    return ROUTER_PURPOSE_BRIDGE;
+  else if (!strcmp(s, "controller"))
+    return ROUTER_PURPOSE_CONTROLLER;
+  else
+    return ROUTER_PURPOSE_UNKNOWN;
+}
diff --git a/src/feature/nodelist/routerinfo.h b/src/feature/nodelist/routerinfo.h
new file mode 100644
index 0000000000..bfa28c7754
--- /dev/null
+++ b/src/feature/nodelist/routerinfo.h
@@ -0,0 +1,27 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file routerinfo.h
+ * \brief Header file for routerinfo.c.
+ **/
+
+#ifndef TOR_ROUTERINFO_H
+#define TOR_ROUTERINFO_H
+
+void router_get_prim_orport(const routerinfo_t *router,
+                            tor_addr_port_t *addr_port_out);
+int router_has_orport(const routerinfo_t *router,
+                      const tor_addr_port_t *orport);
+
+void router_get_verbose_nickname(char *buf, const routerinfo_t *router);
+
+smartlist_t *router_get_all_orports(const routerinfo_t *ri);
+
+const char *router_purpose_to_string(uint8_t p);
+uint8_t router_purpose_from_string(const char *s);
+
+#endif
diff --git a/src/feature/nodelist/routerinfo_st.h b/src/feature/nodelist/routerinfo_st.h
new file mode 100644
index 0000000000..59656818c1
--- /dev/null
+++ b/src/feature/nodelist/routerinfo_st.h
@@ -0,0 +1,115 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef ROUTERINFO_ST_H
+#define ROUTERINFO_ST_H
+
+#include "feature/nodelist/signed_descriptor_st.h"
+
+struct curve25519_public_key_t;
+
+/** Information about another onion router in the network. */
+struct routerinfo_t {
+  signed_descriptor_t cache_info;
+  char *nickname; /**< Human-readable OR name. */
+
+  uint32_t addr; /**< IPv4 address of OR, in host order. */
+  uint16_t or_port; /**< Port for TLS connections. */
+  uint16_t dir_port; /**< Port for HTTP directory connections. */
+
+  /** A router's IPv6 address, if it has one. */
+  /* XXXXX187 Actually these should probably be part of a list of addresses,
+   * not just a special case.  Use abstractions to access these; don't do it
+   * directly. */
+  tor_addr_t ipv6_addr;
+  uint16_t ipv6_orport;
+
+  /**
+   * Public RSA TAP key for onions, ASN.1 encoded.  We store this
+   * in its encoded format since storing it as a crypto_pk_t uses
+   * significantly more memory. */
+  char *onion_pkey;
+  /** Length of onion_pkey, in bytes. */
+  size_t onion_pkey_len;
+
+  crypto_pk_t *identity_pkey;  /**< Public RSA key for signing. */
+  /** Public curve25519 key for onions */
+  struct curve25519_public_key_t *onion_curve25519_pkey;
+  /** What's the earliest expiration time on all the certs in this
+   * routerinfo? */
+  time_t cert_expiration_time;
+
+  char *platform; /**< What software/operating system is this OR using? */
+
+  char *protocol_list; /**< Encoded list of subprotocol versions supported
+                        * by this OR */
+
+  /* link info */
+  uint32_t bandwidthrate; /**< How many bytes does this OR add to its token
+                           * bucket per second? */
+  uint32_t bandwidthburst; /**< How large is this OR's token bucket? */
+  /** How many bytes/s is this router known to handle? */
+  uint32_t bandwidthcapacity;
+  smartlist_t *exit_policy; /**< What streams will this OR permit
+                             * to exit on IPv4?  NULL for 'reject *:*'. */
+  /** What streams will this OR permit to exit on IPv6?
+   * NULL for 'reject *:*' */
+  struct short_policy_t *ipv6_exit_policy;
+  long uptime; /**< How many seconds the router claims to have been up */
+  smartlist_t *declared_family; /**< Nicknames of router which this router
+                                 * claims are its family. */
+  char *contact_info; /**< Declared contact info for this router. */
+  unsigned int is_hibernating:1; /**< Whether the router claims to be
+                                  * hibernating */
+  unsigned int caches_extra_info:1; /**< Whether the router says it caches and
+                                     * serves extrainfo documents. */
+  unsigned int allow_single_hop_exits:1;  /**< Whether the router says
+                                           * it allows single hop exits. */
+
+  unsigned int wants_to_be_hs_dir:1; /**< True iff this router claims to be
+                                      * a hidden service directory. */
+  unsigned int policy_is_reject_star:1; /**< True iff the exit policy for this
+                                         * router rejects everything. */
+  /** True if, after we have added this router, we should re-launch
+   * tests for it. */
+  unsigned int needs_retest_if_added:1;
+
+  /** True iff this router included "tunnelled-dir-server" in its descriptor,
+   * implying it accepts tunnelled directory requests, or it advertised
+   * dir_port > 0. */
+  unsigned int supports_tunnelled_dir_requests:1;
+
+  /** Used during voting to indicate that we should not include an entry for
+   * this routerinfo. Used only during voting. */
+  unsigned int omit_from_vote:1;
+
+  /** Flags to summarize the protocol versions for this routerinfo_t. */
+  protover_summary_flags_t pv;
+
+/** Tor can use this router for general positions in circuits; we got it
+ * from a directory server as usual, or we're an authority and a server
+ * uploaded it. */
+#define ROUTER_PURPOSE_GENERAL 0
+/** Tor should avoid using this router for circuit-building: we got it
+ * from a controller.  If the controller wants to use it, it'll have to
+ * ask for it by identity. */
+#define ROUTER_PURPOSE_CONTROLLER 1
+/** Tor should use this router only for bridge positions in circuits: we got
+ * it via a directory request from the bridge itself, or a bridge
+ * authority. */
+#define ROUTER_PURPOSE_BRIDGE 2
+/** Tor should not use this router; it was marked in cached-descriptors with
+ * a purpose we didn't recognize. */
+#define ROUTER_PURPOSE_UNKNOWN 255
+
+  /** In what way did we find out about this router?  One of ROUTER_PURPOSE_*.
+   * Routers of different purposes are kept segregated and used for different
+   * things; see notes on ROUTER_PURPOSE_* macros above.
+   */
+  uint8_t purpose;
+};
+
+#endif
diff --git a/src/or/routerlist.c b/src/feature/nodelist/routerlist.c
index f3b298006c..e48675aada 100644
--- a/src/or/routerlist.c
+++ b/src/feature/nodelist/routerlist.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -57,69 +57,57 @@
  * nodes according to different rules and weights.  Historically, they
  * were all in this module.  Now, they are spread across this module,
  * nodelist.c, and networkstatus.c.  (TODO: Fix that.)
- *
- * <br>
- *
- * (For historical reasons) this module also contains code for handling
- * the list of fallback directories, the list of directory authorities,
- * and the list of authority certificates.
- *
- * For the directory authorities, we have a list containing the public
- * identity key, and contact points, for each authority.  The
- * authorities receive descriptors from relays, and publish consensuses,
- * descriptors, and microdescriptors.  This list is pre-configured.
- *
- * Fallback directories are well-known, stable, but untrusted directory
- * caches that clients which have not yet bootstrapped can use to get
- * their first networkstatus consensus, in order to find out where the
- * Tor network really is.  This list is pre-configured in
- * fallback_dirs.inc.  Every authority also serves as a fallback.
- *
- * Both fallback directories and directory authorities are are
- * represented by a dir_server_t.
- *
- * Authority certificates are signed with authority identity keys; they
- * are used to authenticate shorter-term authority signing keys. We
- * fetch them when we find a consensus or a vote that has been signed
- * with a signing key we don't recognize.  We cache them on disk and
- * load them on startup.  Authority operators generate them with the
- * "tor-gencert" utility.
- *
- * TODO: Authority certificates should be a separate module.
- *
- * TODO: dir_server_t stuff should be in a separate module.
  **/
 
 #define ROUTERLIST_PRIVATE
-#include "or.h"
-#include "backtrace.h"
-#include "crypto_ed25519.h"
-#include "circuitstats.h"
-#include "config.h"
-#include "connection.h"
-#include "control.h"
-#include "directory.h"
-#include "dirserv.h"
-#include "dirvote.h"
-#include "entrynodes.h"
-#include "fp_pair.h"
-#include "geoip.h"
-#include "hibernate.h"
-#include "main.h"
-#include "microdesc.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "policies.h"
-#include "reasons.h"
-#include "rendcommon.h"
-#include "rendservice.h"
-#include "rephist.h"
-#include "router.h"
-#include "routerlist.h"
-#include "routerparse.h"
-#include "routerset.h"
-#include "sandbox.h"
-#include "torcert.h"
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/policies.h"
+#include "feature/client/bridges.h"
+#include "feature/control/control.h"
+#include "feature/dirauth/authmode.h"
+#include "feature/dirauth/process_descs.h"
+#include "feature/dirauth/reachability.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
+#include "feature/dircommon/directory.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/node_select.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerinfo.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/dirparse/routerparse.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/nodelist/torcert.h"
+#include "feature/relay/routermode.h"
+#include "feature/stats/rephist.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/dirclient/dir_server_st.h"
+#include "feature/nodelist/document_signature_st.h"
+#include "feature/nodelist/extrainfo_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/networkstatus_voter_info_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerlist_st.h"
+#include "feature/nodelist/vote_routerstatus_st.h"
+
+#include "lib/crypt_ops/digestset.h"
+
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
 
 // #define DEBUG_ROUTERLIST
 
@@ -131,7 +119,6 @@
 DECLARE_TYPED_DIGESTMAP_FNS(sdmap_, digest_sd_map_t, signed_descriptor_t)
 DECLARE_TYPED_DIGESTMAP_FNS(rimap_, digest_ri_map_t, routerinfo_t)
 DECLARE_TYPED_DIGESTMAP_FNS(eimap_, digest_ei_map_t, extrainfo_t)
-DECLARE_TYPED_DIGESTMAP_FNS(dsmap_, digest_ds_map_t, download_status_t)
 #define SDMAP_FOREACH(map, keyvar, valvar)                              \
   DIGESTMAP_FOREACH(sdmap_to_digestmap(map), keyvar, signed_descriptor_t *, \
                     valvar)
@@ -139,72 +126,20 @@ DECLARE_TYPED_DIGESTMAP_FNS(dsmap_, digest_ds_map_t, download_status_t)
   DIGESTMAP_FOREACH(rimap_to_digestmap(map), keyvar, routerinfo_t *, valvar)
 #define EIMAP_FOREACH(map, keyvar, valvar) \
   DIGESTMAP_FOREACH(eimap_to_digestmap(map), keyvar, extrainfo_t *, valvar)
-#define DSMAP_FOREACH(map, keyvar, valvar) \
-  DIGESTMAP_FOREACH(dsmap_to_digestmap(map), keyvar, download_status_t *, \
-                    valvar)
-
-/* Forward declaration for cert_list_t */
-typedef struct cert_list_t cert_list_t;
+#define eimap_free(map, fn) MAP_FREE_AND_NULL(eimap, (map), (fn))
+#define rimap_free(map, fn) MAP_FREE_AND_NULL(rimap, (map), (fn))
+#define sdmap_free(map, fn) MAP_FREE_AND_NULL(sdmap, (map), (fn))
 
 /* static function prototypes */
-static int compute_weighted_bandwidths(const smartlist_t *sl,
-                                       bandwidth_weight_rule_t rule,
-                                       double **bandwidths_out,
-                                       double *total_bandwidth_out);
-static const routerstatus_t *router_pick_trusteddirserver_impl(
-                const smartlist_t *sourcelist, dirinfo_type_t auth,
-                int flags, int *n_busy_out);
-static const routerstatus_t *router_pick_dirserver_generic(
-                              smartlist_t *sourcelist,
-                              dirinfo_type_t type, int flags);
-static void mark_all_dirservers_up(smartlist_t *server_list);
-static void dir_server_free(dir_server_t *ds);
 static int signed_desc_digest_is_recognized(signed_descriptor_t *desc);
 static const char *signed_descriptor_get_body_impl(
                                               const signed_descriptor_t *desc,
                                               int with_annotations);
-static void list_pending_downloads(digestmap_t *result,
-                                   digest256map_t *result256,
-                                   int purpose, const char *prefix);
-static void list_pending_fpsk_downloads(fp_pair_map_t *result);
 static void launch_dummy_descriptor_download_as_needed(time_t now,
                                    const or_options_t *options);
-static void download_status_reset_by_sk_in_cl(cert_list_t *cl,
-                                              const char *digest);
-static int download_status_is_ready_by_sk_in_cl(cert_list_t *cl,
-                                                const char *digest,
-                                                time_t now, int max_failures);
 
 /****************************************************************************/
 
-/** Global list of a dir_server_t object for each directory
- * authority. */
-static smartlist_t *trusted_dir_servers = NULL;
-/** Global list of dir_server_t objects for all directory authorities
- * and all fallback directory servers. */
-static smartlist_t *fallback_dir_servers = NULL;
-
-/** List of certificates for a single authority, and download status for
- * latest certificate.
- */
-struct cert_list_t {
-  /*
-   * The keys of download status map are cert->signing_key_digest for pending
-   * downloads by (identity digest/signing key digest) pair; functions such
-   * as authority_cert_get_by_digest() already assume these are unique.
-   */
-  struct digest_ds_map_t *dl_status_map;
-  /* There is also a dlstatus for the download by identity key only */
-  download_status_t dl_status_by_id;
-  smartlist_t *certs;
-};
-/** Map from v3 identity key digest to cert_list_t. */
-static digestmap_t *trusted_dir_certs = NULL;
-/** True iff any key certificate in at least one member of
- * <b>trusted_dir_certs</b> has changed since we last flushed the
- * certificates to disk. */
-static int trusted_dir_servers_certs_changed = 0;
-
 /** Global list of all of the routers that we know about. */
 static routerlist_t *routerlist = NULL;
 
@@ -217,1079 +152,6 @@ static smartlist_t *warned_nicknames = NULL;
  * download is low. */
 static time_t last_descriptor_download_attempted = 0;
 
-/** Return the number of directory authorities whose type matches some bit set
- * in <b>type</b>  */
-int
-get_n_authorities(dirinfo_type_t type)
-{
-  int n = 0;
-  if (!trusted_dir_servers)
-    return 0;
-  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
-                    if (ds->type & type)
-                      ++n);
-  return n;
-}
-
-/** Initialise schedule, want_authority, and increment on in the download
- * status dlstatus, then call download_status_reset() on it.
- * It is safe to call this function or download_status_reset() multiple times
- * on a new dlstatus. But it should *not* be called after a dlstatus has been
- * used to count download attempts or failures. */
-static void
-download_status_cert_init(download_status_t *dlstatus)
-{
-  dlstatus->schedule = DL_SCHED_CONSENSUS;
-  dlstatus->want_authority = DL_WANT_ANY_DIRSERVER;
-  dlstatus->increment_on = DL_SCHED_INCREMENT_FAILURE;
-  dlstatus->backoff = DL_SCHED_RANDOM_EXPONENTIAL;
-  dlstatus->last_backoff_position = 0;
-  dlstatus->last_delay_used = 0;
-
-  /* Use the new schedule to set next_attempt_at */
-  download_status_reset(dlstatus);
-}
-
-/** Reset the download status of a specified element in a dsmap */
-static void
-download_status_reset_by_sk_in_cl(cert_list_t *cl, const char *digest)
-{
-  download_status_t *dlstatus = NULL;
-
-  tor_assert(cl);
-  tor_assert(digest);
-
-  /* Make sure we have a dsmap */
-  if (!(cl->dl_status_map)) {
-    cl->dl_status_map = dsmap_new();
-  }
-  /* Look for a download_status_t in the map with this digest */
-  dlstatus = dsmap_get(cl->dl_status_map, digest);
-  /* Got one? */
-  if (!dlstatus) {
-    /* Insert before we reset */
-    dlstatus = tor_malloc_zero(sizeof(*dlstatus));
-    dsmap_set(cl->dl_status_map, digest, dlstatus);
-    download_status_cert_init(dlstatus);
-  }
-  tor_assert(dlstatus);
-  /* Go ahead and reset it */
-  download_status_reset(dlstatus);
-}
-
-/**
- * Return true if the download for this signing key digest in cl is ready
- * to be re-attempted.
- */
-static int
-download_status_is_ready_by_sk_in_cl(cert_list_t *cl,
-                                     const char *digest,
-                                     time_t now, int max_failures)
-{
-  int rv = 0;
-  download_status_t *dlstatus = NULL;
-
-  tor_assert(cl);
-  tor_assert(digest);
-
-  /* Make sure we have a dsmap */
-  if (!(cl->dl_status_map)) {
-    cl->dl_status_map = dsmap_new();
-  }
-  /* Look for a download_status_t in the map with this digest */
-  dlstatus = dsmap_get(cl->dl_status_map, digest);
-  /* Got one? */
-  if (dlstatus) {
-    /* Use download_status_is_ready() */
-    rv = download_status_is_ready(dlstatus, now, max_failures);
-  } else {
-    /*
-     * If we don't know anything about it, return 1, since we haven't
-     * tried this one before.  We need to create a new entry here,
-     * too.
-     */
-    dlstatus = tor_malloc_zero(sizeof(*dlstatus));
-    download_status_cert_init(dlstatus);
-    dsmap_set(cl->dl_status_map, digest, dlstatus);
-    rv = 1;
-  }
-
-  return rv;
-}
-
-/** Helper: Return the cert_list_t for an authority whose authority ID is
- * <b>id_digest</b>, allocating a new list if necessary. */
-static cert_list_t *
-get_cert_list(const char *id_digest)
-{
-  cert_list_t *cl;
-  if (!trusted_dir_certs)
-    trusted_dir_certs = digestmap_new();
-  cl = digestmap_get(trusted_dir_certs, id_digest);
-  if (!cl) {
-    cl = tor_malloc_zero(sizeof(cert_list_t));
-    download_status_cert_init(&cl->dl_status_by_id);
-    cl->certs = smartlist_new();
-    cl->dl_status_map = dsmap_new();
-    digestmap_set(trusted_dir_certs, id_digest, cl);
-  }
-  return cl;
-}
-
-/** Return a list of authority ID digests with potentially enumerable lists
- * of download_status_t objects; used by controller GETINFO queries.
- */
-
-MOCK_IMPL(smartlist_t *,
-list_authority_ids_with_downloads, (void))
-{
-  smartlist_t *ids = smartlist_new();
-  digestmap_iter_t *i;
-  const char *digest;
-  char *tmp;
-  void *cl;
-
-  if (trusted_dir_certs) {
-    for (i = digestmap_iter_init(trusted_dir_certs);
-         !(digestmap_iter_done(i));
-         i = digestmap_iter_next(trusted_dir_certs, i)) {
-      /*
-       * We always have at least dl_status_by_id to query, so no need to
-       * probe deeper than the existence of a cert_list_t.
-       */
-      digestmap_iter_get(i, &digest, &cl);
-      tmp = tor_malloc(DIGEST_LEN);
-      memcpy(tmp, digest, DIGEST_LEN);
-      smartlist_add(ids, tmp);
-    }
-  }
-  /* else definitely no downlaods going since nothing even has a cert list */
-
-  return ids;
-}
-
-/** Given an authority ID digest, return a pointer to the default download
- * status, or NULL if there is no such entry in trusted_dir_certs */
-
-MOCK_IMPL(download_status_t *,
-id_only_download_status_for_authority_id, (const char *digest))
-{
-  download_status_t *dl = NULL;
-  cert_list_t *cl;
-
-  if (trusted_dir_certs) {
-    cl = digestmap_get(trusted_dir_certs, digest);
-    if (cl) {
-      dl = &(cl->dl_status_by_id);
-    }
-  }
-
-  return dl;
-}
-
-/** Given an authority ID digest, return a smartlist of signing key digests
- * for which download_status_t is potentially queryable, or NULL if no such
- * authority ID digest is known. */
-
-MOCK_IMPL(smartlist_t *,
-list_sk_digests_for_authority_id, (const char *digest))
-{
-  smartlist_t *sks = NULL;
-  cert_list_t *cl;
-  dsmap_iter_t *i;
-  const char *sk_digest;
-  char *tmp;
-  download_status_t *dl;
-
-  if (trusted_dir_certs) {
-    cl = digestmap_get(trusted_dir_certs, digest);
-    if (cl) {
-      sks = smartlist_new();
-      if (cl->dl_status_map) {
-        for (i = dsmap_iter_init(cl->dl_status_map);
-             !(dsmap_iter_done(i));
-             i = dsmap_iter_next(cl->dl_status_map, i)) {
-          /* Pull the digest out and add it to the list */
-          dsmap_iter_get(i, &sk_digest, &dl);
-          tmp = tor_malloc(DIGEST_LEN);
-          memcpy(tmp, sk_digest, DIGEST_LEN);
-          smartlist_add(sks, tmp);
-        }
-      }
-    }
-  }
-
-  return sks;
-}
-
-/** Given an authority ID digest and a signing key digest, return the
- * download_status_t or NULL if none exists. */
-
-MOCK_IMPL(download_status_t *,
-  download_status_for_authority_id_and_sk,
-  (const char *id_digest, const char *sk_digest))
-{
-  download_status_t *dl = NULL;
-  cert_list_t *cl = NULL;
-
-  if (trusted_dir_certs) {
-    cl = digestmap_get(trusted_dir_certs, id_digest);
-    if (cl && cl->dl_status_map) {
-      dl = dsmap_get(cl->dl_status_map, sk_digest);
-    }
-  }
-
-  return dl;
-}
-
-/** Release all space held by a cert_list_t */
-static void
-cert_list_free(cert_list_t *cl)
-{
-  if (!cl)
-    return;
-
-  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
-                    authority_cert_free(cert));
-  smartlist_free(cl->certs);
-  dsmap_free(cl->dl_status_map, tor_free_);
-  tor_free(cl);
-}
-
-/** Wrapper for cert_list_free so we can pass it to digestmap_free */
-static void
-cert_list_free_(void *cl)
-{
-  cert_list_free(cl);
-}
-
-/** Reload the cached v3 key certificates from the cached-certs file in
- * the data directory. Return 0 on success, -1 on failure. */
-int
-trusted_dirs_reload_certs(void)
-{
-  char *filename;
-  char *contents;
-  int r;
-
-  filename = get_datadir_fname("cached-certs");
-  contents = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
-  tor_free(filename);
-  if (!contents)
-    return 0;
-  r = trusted_dirs_load_certs_from_string(
-        contents,
-        TRUSTED_DIRS_CERTS_SRC_FROM_STORE, 1, NULL);
-  tor_free(contents);
-  return r;
-}
-
-/** Helper: return true iff we already have loaded the exact cert
- * <b>cert</b>. */
-static inline int
-already_have_cert(authority_cert_t *cert)
-{
-  cert_list_t *cl = get_cert_list(cert->cache_info.identity_digest);
-
-  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, c,
-  {
-    if (tor_memeq(c->cache_info.signed_descriptor_digest,
-                cert->cache_info.signed_descriptor_digest,
-                DIGEST_LEN))
-      return 1;
-  });
-  return 0;
-}
-
-/** Load a bunch of new key certificates from the string <b>contents</b>.  If
- * <b>source</b> is TRUSTED_DIRS_CERTS_SRC_FROM_STORE, the certificates are
- * from the cache, and we don't need to flush them to disk.  If we are a
- * dirauth loading our own cert, source is TRUSTED_DIRS_CERTS_SRC_SELF.
- * Otherwise, source is download type: TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST
- * or TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST.  If <b>flush</b> is true, we
- * need to flush any changed certificates to disk now.  Return 0 on success,
- * -1 if any certs fail to parse.
- *
- * If source_dir is non-NULL, it's the identity digest for a directory that
- * we've just successfully retrieved certificates from, so try it first to
- * fetch any missing certificates.
- */
-int
-trusted_dirs_load_certs_from_string(const char *contents, int source,
-                                    int flush, const char *source_dir)
-{
-  dir_server_t *ds;
-  const char *s, *eos;
-  int failure_code = 0;
-  int from_store = (source == TRUSTED_DIRS_CERTS_SRC_FROM_STORE);
-  int added_trusted_cert = 0;
-
-  for (s = contents; *s; s = eos) {
-    authority_cert_t *cert = authority_cert_parse_from_string(s, &eos);
-    cert_list_t *cl;
-    if (!cert) {
-      failure_code = -1;
-      break;
-    }
-    ds = trusteddirserver_get_by_v3_auth_digest(
-                                       cert->cache_info.identity_digest);
-    log_debug(LD_DIR, "Parsed certificate for %s",
-              ds ? ds->nickname : "unknown authority");
-
-    if (already_have_cert(cert)) {
-      /* we already have this one. continue. */
-      log_info(LD_DIR, "Skipping %s certificate for %s that we "
-               "already have.",
-               from_store ? "cached" : "downloaded",
-               ds ? ds->nickname : "an old or new authority");
-
-      /*
-       * A duplicate on download should be treated as a failure, so we call
-       * authority_cert_dl_failed() to reset the download status to make sure
-       * we can't try again.  Since we've implemented the fp-sk mechanism
-       * to download certs by signing key, this should be much rarer than it
-       * was and is perhaps cause for concern.
-       */
-      if (!from_store) {
-        if (authdir_mode(get_options())) {
-          log_warn(LD_DIR,
-                   "Got a certificate for %s, but we already have it. "
-                   "Maybe they haven't updated it. Waiting for a while.",
-                   ds ? ds->nickname : "an old or new authority");
-        } else {
-          log_info(LD_DIR,
-                   "Got a certificate for %s, but we already have it. "
-                   "Maybe they haven't updated it. Waiting for a while.",
-                   ds ? ds->nickname : "an old or new authority");
-        }
-
-        /*
-         * This is where we care about the source; authority_cert_dl_failed()
-         * needs to know whether the download was by fp or (fp,sk) pair to
-         * twiddle the right bit in the download map.
-         */
-        if (source == TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST) {
-          authority_cert_dl_failed(cert->cache_info.identity_digest,
-                                   NULL, 404);
-        } else if (source == TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST) {
-          authority_cert_dl_failed(cert->cache_info.identity_digest,
-                                   cert->signing_key_digest, 404);
-        }
-      }
-
-      authority_cert_free(cert);
-      continue;
-    }
-
-    if (ds) {
-      added_trusted_cert = 1;
-      log_info(LD_DIR, "Adding %s certificate for directory authority %s with "
-               "signing key %s", from_store ? "cached" : "downloaded",
-               ds->nickname, hex_str(cert->signing_key_digest,DIGEST_LEN));
-    } else {
-      int adding = directory_caches_unknown_auth_certs(get_options());
-      log_info(LD_DIR, "%s %s certificate for unrecognized directory "
-               "authority with signing key %s",
-               adding ? "Adding" : "Not adding",
-               from_store ? "cached" : "downloaded",
-               hex_str(cert->signing_key_digest,DIGEST_LEN));
-      if (!adding) {
-        authority_cert_free(cert);
-        continue;
-      }
-    }
-
-    cl = get_cert_list(cert->cache_info.identity_digest);
-    smartlist_add(cl->certs, cert);
-    if (ds && cert->cache_info.published_on > ds->addr_current_at) {
-      /* Check to see whether we should update our view of the authority's
-       * address. */
-      if (cert->addr && cert->dir_port &&
-          (ds->addr != cert->addr ||
-           ds->dir_port != cert->dir_port)) {
-        char *a = tor_dup_ip(cert->addr);
-        log_notice(LD_DIR, "Updating address for directory authority %s "
-                   "from %s:%d to %s:%d based on certificate.",
-                   ds->nickname, ds->address, (int)ds->dir_port,
-                   a, cert->dir_port);
-        tor_free(a);
-        ds->addr = cert->addr;
-        ds->dir_port = cert->dir_port;
-      }
-      ds->addr_current_at = cert->cache_info.published_on;
-    }
-
-    if (!from_store)
-      trusted_dir_servers_certs_changed = 1;
-  }
-
-  if (flush)
-    trusted_dirs_flush_certs_to_disk();
-
-  /* call this even if failure_code is <0, since some certs might have
-   * succeeded, but only pass source_dir if there were no failures,
-   * and at least one more authority certificate was added to the store.
-   * This avoids retrying a directory that's serving bad or entirely duplicate
-   * certificates. */
-  if (failure_code == 0 && added_trusted_cert) {
-    networkstatus_note_certs_arrived(source_dir);
-  } else {
-    networkstatus_note_certs_arrived(NULL);
-  }
-
-  return failure_code;
-}
-
-/** Save all v3 key certificates to the cached-certs file. */
-void
-trusted_dirs_flush_certs_to_disk(void)
-{
-  char *filename;
-  smartlist_t *chunks;
-
-  if (!trusted_dir_servers_certs_changed || !trusted_dir_certs)
-    return;
-
-  chunks = smartlist_new();
-  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
-    SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
-          {
-            sized_chunk_t *c = tor_malloc(sizeof(sized_chunk_t));
-            c->bytes = cert->cache_info.signed_descriptor_body;
-            c->len = cert->cache_info.signed_descriptor_len;
-            smartlist_add(chunks, c);
-          });
-  } DIGESTMAP_FOREACH_END;
-
-  filename = get_datadir_fname("cached-certs");
-  if (write_chunks_to_file(filename, chunks, 0, 0)) {
-    log_warn(LD_FS, "Error writing certificates to disk.");
-  }
-  tor_free(filename);
-  SMARTLIST_FOREACH(chunks, sized_chunk_t *, c, tor_free(c));
-  smartlist_free(chunks);
-
-  trusted_dir_servers_certs_changed = 0;
-}
-
-static int
-compare_certs_by_pubdates(const void **_a, const void **_b)
-{
-  const authority_cert_t *cert1 = *_a, *cert2=*_b;
-
-  if (cert1->cache_info.published_on < cert2->cache_info.published_on)
-    return -1;
-  else if (cert1->cache_info.published_on >  cert2->cache_info.published_on)
-    return 1;
-  else
-    return 0;
-}
-
-/** Remove all expired v3 authority certificates that have been superseded for
- * more than 48 hours or, if not expired, that were published more than 7 days
- * before being superseded. (If the most recent cert was published more than 48
- * hours ago, then we aren't going to get any consensuses signed with older
- * keys.) */
-static void
-trusted_dirs_remove_old_certs(void)
-{
-  time_t now = time(NULL);
-#define DEAD_CERT_LIFETIME (2*24*60*60)
-#define SUPERSEDED_CERT_LIFETIME (2*24*60*60)
-  if (!trusted_dir_certs)
-    return;
-
-  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
-    /* Sort the list from first-published to last-published */
-    smartlist_sort(cl->certs, compare_certs_by_pubdates);
-
-    SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) {
-      if (cert_sl_idx == smartlist_len(cl->certs) - 1) {
-        /* This is the most recently published cert.  Keep it. */
-        continue;
-      }
-      authority_cert_t *next_cert = smartlist_get(cl->certs, cert_sl_idx+1);
-      const time_t next_cert_published = next_cert->cache_info.published_on;
-      if (next_cert_published > now) {
-        /* All later certs are published in the future. Keep everything
-         * we didn't discard. */
-        break;
-      }
-      int should_remove = 0;
-      if (cert->expires + DEAD_CERT_LIFETIME < now) {
-        /* Certificate has been expired for at least DEAD_CERT_LIFETIME.
-         * Remove it. */
-        should_remove = 1;
-      } else if (next_cert_published + SUPERSEDED_CERT_LIFETIME < now) {
-        /* Certificate has been superseded for OLD_CERT_LIFETIME.
-         * Remove it.
-         */
-        should_remove = 1;
-      }
-      if (should_remove) {
-        SMARTLIST_DEL_CURRENT_KEEPORDER(cl->certs, cert);
-        authority_cert_free(cert);
-        trusted_dir_servers_certs_changed = 1;
-      }
-    } SMARTLIST_FOREACH_END(cert);
-
-  } DIGESTMAP_FOREACH_END;
-#undef DEAD_CERT_LIFETIME
-#undef OLD_CERT_LIFETIME
-
-  trusted_dirs_flush_certs_to_disk();
-}
-
-/** Return the newest v3 authority certificate whose v3 authority identity key
- * has digest <b>id_digest</b>.  Return NULL if no such authority is known,
- * or it has no certificate. */
-authority_cert_t *
-authority_cert_get_newest_by_id(const char *id_digest)
-{
-  cert_list_t *cl;
-  authority_cert_t *best = NULL;
-  if (!trusted_dir_certs ||
-      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
-    return NULL;
-
-  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
-  {
-    if (!best || cert->cache_info.published_on > best->cache_info.published_on)
-      best = cert;
-  });
-  return best;
-}
-
-/** Return the newest v3 authority certificate whose directory signing key has
- * digest <b>sk_digest</b>. Return NULL if no such certificate is known.
- */
-authority_cert_t *
-authority_cert_get_by_sk_digest(const char *sk_digest)
-{
-  authority_cert_t *c;
-  if (!trusted_dir_certs)
-    return NULL;
-
-  if ((c = get_my_v3_authority_cert()) &&
-      tor_memeq(c->signing_key_digest, sk_digest, DIGEST_LEN))
-    return c;
-  if ((c = get_my_v3_legacy_cert()) &&
-      tor_memeq(c->signing_key_digest, sk_digest, DIGEST_LEN))
-    return c;
-
-  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
-    SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
-    {
-      if (tor_memeq(cert->signing_key_digest, sk_digest, DIGEST_LEN))
-        return cert;
-    });
-  } DIGESTMAP_FOREACH_END;
-  return NULL;
-}
-
-/** Return the v3 authority certificate with signing key matching
- * <b>sk_digest</b>, for the authority with identity digest <b>id_digest</b>.
- * Return NULL if no such authority is known. */
-authority_cert_t *
-authority_cert_get_by_digests(const char *id_digest,
-                              const char *sk_digest)
-{
-  cert_list_t *cl;
-  if (!trusted_dir_certs ||
-      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
-    return NULL;
-  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
-    if (tor_memeq(cert->signing_key_digest, sk_digest, DIGEST_LEN))
-      return cert; );
-
-  return NULL;
-}
-
-/** Add every known authority_cert_t to <b>certs_out</b>. */
-void
-authority_cert_get_all(smartlist_t *certs_out)
-{
-  tor_assert(certs_out);
-  if (!trusted_dir_certs)
-    return;
-
-  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
-    SMARTLIST_FOREACH(cl->certs, authority_cert_t *, c,
-                      smartlist_add(certs_out, c));
-  } DIGESTMAP_FOREACH_END;
-}
-
-/** Called when an attempt to download a certificate with the authority with
- * ID <b>id_digest</b> and, if not NULL, signed with key signing_key_digest
- * fails with HTTP response code <b>status</b>: remember the failure, so we
- * don't try again immediately. */
-void
-authority_cert_dl_failed(const char *id_digest,
-                         const char *signing_key_digest, int status)
-{
-  cert_list_t *cl;
-  download_status_t *dlstatus = NULL;
-  char id_digest_str[2*DIGEST_LEN+1];
-  char sk_digest_str[2*DIGEST_LEN+1];
-
-  if (!trusted_dir_certs ||
-      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
-    return;
-
-  /*
-   * Are we noting a failed download of the latest cert for the id digest,
-   * or of a download by (id, signing key) digest pair?
-   */
-  if (!signing_key_digest) {
-    /* Just by id digest */
-    download_status_failed(&cl->dl_status_by_id, status);
-  } else {
-    /* Reset by (id, signing key) digest pair
-     *
-     * Look for a download_status_t in the map with this digest
-     */
-    dlstatus = dsmap_get(cl->dl_status_map, signing_key_digest);
-    /* Got one? */
-    if (dlstatus) {
-      download_status_failed(dlstatus, status);
-    } else {
-      /*
-       * Do this rather than hex_str(), since hex_str clobbers
-       * old results and we call twice in the param list.
-       */
-      base16_encode(id_digest_str, sizeof(id_digest_str),
-                    id_digest, DIGEST_LEN);
-      base16_encode(sk_digest_str, sizeof(sk_digest_str),
-                    signing_key_digest, DIGEST_LEN);
-      log_warn(LD_BUG,
-               "Got failure for cert fetch with (fp,sk) = (%s,%s), with "
-               "status %d, but knew nothing about the download.",
-               id_digest_str, sk_digest_str, status);
-    }
-  }
-}
-
-static const char *BAD_SIGNING_KEYS[] = {
-  "09CD84F751FD6E955E0F8ADB497D5401470D697E", // Expires 2015-01-11 16:26:31
-  "0E7E9C07F0969D0468AD741E172A6109DC289F3C", // Expires 2014-08-12 10:18:26
-  "57B85409891D3FB32137F642FDEDF8B7F8CDFDCD", // Expires 2015-02-11 17:19:09
-  "87326329007AF781F587AF5B594E540B2B6C7630", // Expires 2014-07-17 11:10:09
-  "98CC82342DE8D298CF99D3F1A396475901E0D38E", // Expires 2014-11-10 13:18:56
-  "9904B52336713A5ADCB13E4FB14DC919E0D45571", // Expires 2014-04-20 20:01:01
-  "9DCD8E3F1DD1597E2AD476BBA28A1A89F3095227", // Expires 2015-01-16 03:52:30
-  "A61682F34B9BB9694AC98491FE1ABBFE61923941", // Expires 2014-06-11 09:25:09
-  "B59F6E99C575113650C99F1C425BA7B20A8C071D", // Expires 2014-07-31 13:22:10
-  "D27178388FA75B96D37FA36E0B015227DDDBDA51", // Expires 2014-08-04 04:01:57
-  NULL,
-};
-
-/** Return true iff <b>cert</b> authenticates some atuhority signing key
- * which, because of the old openssl heartbleed vulnerability, should
- * never be trusted. */
-int
-authority_cert_is_blacklisted(const authority_cert_t *cert)
-{
-  char hex_digest[HEX_DIGEST_LEN+1];
-  int i;
-  base16_encode(hex_digest, sizeof(hex_digest),
-                cert->signing_key_digest, sizeof(cert->signing_key_digest));
-
-  for (i = 0; BAD_SIGNING_KEYS[i]; ++i) {
-    if (!strcasecmp(hex_digest, BAD_SIGNING_KEYS[i])) {
-      return 1;
-    }
-  }
-  return 0;
-}
-
-/** Return true iff when we've been getting enough failures when trying to
- * download the certificate with ID digest <b>id_digest</b> that we're willing
- * to start bugging the user about it. */
-int
-authority_cert_dl_looks_uncertain(const char *id_digest)
-{
-#define N_AUTH_CERT_DL_FAILURES_TO_BUG_USER 2
-  cert_list_t *cl;
-  int n_failures;
-  if (!trusted_dir_certs ||
-      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
-    return 0;
-
-  n_failures = download_status_get_n_failures(&cl->dl_status_by_id);
-  return n_failures >= N_AUTH_CERT_DL_FAILURES_TO_BUG_USER;
-}
-
-/* Fetch the authority certificates specified in resource.
- * If we are a bridge client, and node is a configured bridge, fetch from node
- * using dir_hint as the fingerprint. Otherwise, if rs is not NULL, fetch from
- * rs. Otherwise, fetch from a random directory mirror. */
-static void
-authority_certs_fetch_resource_impl(const char *resource,
-                                    const char *dir_hint,
-                                    const node_t *node,
-                                    const routerstatus_t *rs)
-{
-  const or_options_t *options = get_options();
-  int get_via_tor = purpose_needs_anonymity(DIR_PURPOSE_FETCH_CERTIFICATE, 0);
-
-  /* Make sure bridge clients never connect to anything but a bridge */
-  if (options->UseBridges) {
-    if (node && !node_is_a_configured_bridge(node)) {
-      /* If we're using bridges, and node is not a bridge, use a 3-hop path. */
-      get_via_tor = 1;
-    } else if (!node) {
-      /* If we're using bridges, and there's no node, use a 3-hop path. */
-      get_via_tor = 1;
-    }
-  }
-
-  const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS
-                                                    : DIRIND_ONEHOP;
-
-  /* If we've just downloaded a consensus from a bridge, re-use that
-   * bridge */
-  if (options->UseBridges && node && !get_via_tor) {
-    /* clients always make OR connections to bridges */
-    tor_addr_port_t or_ap;
-    /* we are willing to use a non-preferred address if we need to */
-    fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0,
-                                         &or_ap);
-    directory_initiate_command(&or_ap.addr, or_ap.port,
-                               NULL, 0, /*no dirport*/
-                               dir_hint,
-                               DIR_PURPOSE_FETCH_CERTIFICATE,
-                               0,
-                               indirection,
-                               resource, NULL, 0, 0);
-    return;
-  }
-
-  if (rs) {
-    /* If we've just downloaded a consensus from a directory, re-use that
-     * directory */
-    directory_initiate_command_routerstatus(rs,
-                                            DIR_PURPOSE_FETCH_CERTIFICATE,
-                                            0, indirection, resource, NULL,
-                                            0, 0);
-    return;
-  }
-
-  /* Otherwise, we want certs from a random fallback or directory
-   * mirror, because they will almost always succeed. */
-  directory_get_from_dirserver(DIR_PURPOSE_FETCH_CERTIFICATE, 0,
-                               resource, PDS_RETRY_IF_NO_SERVERS,
-                               DL_WANT_ANY_DIRSERVER);
-}
-
-/** Try to download any v3 authority certificates that we may be missing.  If
- * <b>status</b> is provided, try to get all the ones that were used to sign
- * <b>status</b>.  Additionally, try to have a non-expired certificate for
- * every V3 authority in trusted_dir_servers.  Don't fetch certificates we
- * already have.
- *
- * If dir_hint is non-NULL, it's the identity digest for a directory that
- * we've just successfully retrieved a consensus or certificates from, so try
- * it first to fetch any missing certificates.
- **/
-void
-authority_certs_fetch_missing(networkstatus_t *status, time_t now,
-                              const char *dir_hint)
-{
-  /*
-   * The pending_id digestmap tracks pending certificate downloads by
-   * identity digest; the pending_cert digestmap tracks pending downloads
-   * by (identity digest, signing key digest) pairs.
-   */
-  digestmap_t *pending_id;
-  fp_pair_map_t *pending_cert;
-  /*
-   * The missing_id_digests smartlist will hold a list of id digests
-   * we want to fetch the newest cert for; the missing_cert_digests
-   * smartlist will hold a list of fp_pair_t with an identity and
-   * signing key digest.
-   */
-  smartlist_t *missing_cert_digests, *missing_id_digests;
-  char *resource = NULL;
-  cert_list_t *cl;
-  const or_options_t *options = get_options();
-  const int cache = directory_caches_unknown_auth_certs(options);
-  fp_pair_t *fp_tmp = NULL;
-  char id_digest_str[2*DIGEST_LEN+1];
-  char sk_digest_str[2*DIGEST_LEN+1];
-
-  if (should_delay_dir_fetches(options, NULL))
-    return;
-
-  pending_cert = fp_pair_map_new();
-  pending_id = digestmap_new();
-  missing_cert_digests = smartlist_new();
-  missing_id_digests = smartlist_new();
-
-  /*
-   * First, we get the lists of already pending downloads so we don't
-   * duplicate effort.
-   */
-  list_pending_downloads(pending_id, NULL,
-                         DIR_PURPOSE_FETCH_CERTIFICATE, "fp/");
-  list_pending_fpsk_downloads(pending_cert);
-
-  /*
-   * Now, we download any trusted authority certs we don't have by
-   * identity digest only.  This gets the latest cert for that authority.
-   */
-  SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, dir_server_t *, ds) {
-    int found = 0;
-    if (!(ds->type & V3_DIRINFO))
-      continue;
-    if (smartlist_contains_digest(missing_id_digests,
-                                  ds->v3_identity_digest))
-      continue;
-    cl = get_cert_list(ds->v3_identity_digest);
-    SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) {
-      if (now < cert->expires) {
-        /* It's not expired, and we weren't looking for something to
-         * verify a consensus with.  Call it done. */
-        download_status_reset(&(cl->dl_status_by_id));
-        /* No sense trying to download it specifically by signing key hash */
-        download_status_reset_by_sk_in_cl(cl, cert->signing_key_digest);
-        found = 1;
-        break;
-      }
-    } SMARTLIST_FOREACH_END(cert);
-    if (!found &&
-        download_status_is_ready(&(cl->dl_status_by_id), now,
-                                 options->TestingCertMaxDownloadTries) &&
-        !digestmap_get(pending_id, ds->v3_identity_digest)) {
-      log_info(LD_DIR,
-               "No current certificate known for authority %s "
-               "(ID digest %s); launching request.",
-               ds->nickname, hex_str(ds->v3_identity_digest, DIGEST_LEN));
-      smartlist_add(missing_id_digests, ds->v3_identity_digest);
-    }
-  } SMARTLIST_FOREACH_END(ds);
-
-  /*
-   * Next, if we have a consensus, scan through it and look for anything
-   * signed with a key from a cert we don't have.  Those get downloaded
-   * by (fp,sk) pair, but if we don't know any certs at all for the fp
-   * (identity digest), and it's one of the trusted dir server certs
-   * we started off above or a pending download in pending_id, don't
-   * try to get it yet.  Most likely, the one we'll get for that will
-   * have the right signing key too, and we'd just be downloading
-   * redundantly.
-   */
-  if (status) {
-    SMARTLIST_FOREACH_BEGIN(status->voters, networkstatus_voter_info_t *,
-                            voter) {
-      if (!smartlist_len(voter->sigs))
-        continue; /* This authority never signed this consensus, so don't
-                   * go looking for a cert with key digest 0000000000. */
-      if (!cache &&
-          !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest))
-        continue; /* We are not a cache, and we don't know this authority.*/
-
-      /*
-       * If we don't know *any* cert for this authority, and a download by ID
-       * is pending or we added it to missing_id_digests above, skip this
-       * one for now to avoid duplicate downloads.
-       */
-      cl = get_cert_list(voter->identity_digest);
-      if (smartlist_len(cl->certs) == 0) {
-        /* We have no certs at all for this one */
-
-        /* Do we have a download of one pending? */
-        if (digestmap_get(pending_id, voter->identity_digest))
-          continue;
-
-        /*
-         * Are we about to launch a download of one due to the trusted
-         * dir server check above?
-         */
-        if (smartlist_contains_digest(missing_id_digests,
-                                      voter->identity_digest))
-          continue;
-      }
-
-      SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) {
-        authority_cert_t *cert =
-          authority_cert_get_by_digests(voter->identity_digest,
-                                        sig->signing_key_digest);
-        if (cert) {
-          if (now < cert->expires)
-            download_status_reset_by_sk_in_cl(cl, sig->signing_key_digest);
-          continue;
-        }
-        if (download_status_is_ready_by_sk_in_cl(
-              cl, sig->signing_key_digest,
-              now, options->TestingCertMaxDownloadTries) &&
-            !fp_pair_map_get_by_digests(pending_cert,
-                                        voter->identity_digest,
-                                        sig->signing_key_digest)) {
-          /*
-           * Do this rather than hex_str(), since hex_str clobbers
-           * old results and we call twice in the param list.
-           */
-          base16_encode(id_digest_str, sizeof(id_digest_str),
-                        voter->identity_digest, DIGEST_LEN);
-          base16_encode(sk_digest_str, sizeof(sk_digest_str),
-                        sig->signing_key_digest, DIGEST_LEN);
-
-          if (voter->nickname) {
-            log_info(LD_DIR,
-                     "We're missing a certificate from authority %s "
-                     "(ID digest %s) with signing key %s: "
-                     "launching request.",
-                     voter->nickname, id_digest_str, sk_digest_str);
-          } else {
-            log_info(LD_DIR,
-                     "We're missing a certificate from authority ID digest "
-                     "%s with signing key %s: launching request.",
-                     id_digest_str, sk_digest_str);
-          }
-
-          /* Allocate a new fp_pair_t to append */
-          fp_tmp = tor_malloc(sizeof(*fp_tmp));
-          memcpy(fp_tmp->first, voter->identity_digest, sizeof(fp_tmp->first));
-          memcpy(fp_tmp->second, sig->signing_key_digest,
-                 sizeof(fp_tmp->second));
-          smartlist_add(missing_cert_digests, fp_tmp);
-        }
-      } SMARTLIST_FOREACH_END(sig);
-    } SMARTLIST_FOREACH_END(voter);
-  }
-
-  /* Bridge clients look up the node for the dir_hint  */
-  const node_t *node = NULL;
-  /* All clients, including bridge clients, look up the routerstatus for the
-   * dir_hint */
-  const routerstatus_t *rs = NULL;
-
-  /* If we still need certificates, try the directory that just successfully
-   * served us a consensus or certificates.
-   * As soon as the directory fails to provide additional certificates, we try
-   * another, randomly selected directory. This avoids continual retries.
-   * (We only ever have one outstanding request per certificate.)
-   */
-  if (dir_hint) {
-    if (options->UseBridges) {
-      /* Bridge clients try the nodelist. If the dir_hint is from an authority,
-       * or something else fetched over tor, we won't find the node here, but
-       * we will find the rs. */
-      node = node_get_by_id(dir_hint);
-    }
-
-    /* All clients try the consensus routerstatus, then the fallback
-     * routerstatus */
-    rs = router_get_consensus_status_by_id(dir_hint);
-    if (!rs) {
-      /* This will also find authorities */
-      const dir_server_t *ds = router_get_fallback_dirserver_by_digest(
-                                                                    dir_hint);
-      if (ds) {
-        rs = &ds->fake_status;
-      }
-    }
-
-    if (!node && !rs) {
-      log_warn(LD_BUG, "Directory %s delivered a consensus, but %s"
-               "no routerstatus could be found for it.",
-               options->UseBridges ? "no node and " : "",
-               hex_str(dir_hint, DIGEST_LEN));
-    }
-  }
-
-  /* Do downloads by identity digest */
-  if (smartlist_len(missing_id_digests) > 0) {
-    int need_plus = 0;
-    smartlist_t *fps = smartlist_new();
-
-    smartlist_add(fps, tor_strdup("fp/"));
-
-    SMARTLIST_FOREACH_BEGIN(missing_id_digests, const char *, d) {
-      char *fp = NULL;
-
-      if (digestmap_get(pending_id, d))
-        continue;
-
-      base16_encode(id_digest_str, sizeof(id_digest_str),
-                    d, DIGEST_LEN);
-
-      if (need_plus) {
-        tor_asprintf(&fp, "+%s", id_digest_str);
-      } else {
-        /* No need for tor_asprintf() in this case; first one gets no '+' */
-        fp = tor_strdup(id_digest_str);
-        need_plus = 1;
-      }
-
-      smartlist_add(fps, fp);
-    } SMARTLIST_FOREACH_END(d);
-
-    if (smartlist_len(fps) > 1) {
-      resource = smartlist_join_strings(fps, "", 0, NULL);
-      /* node and rs are directories that just gave us a consensus or
-       * certificates */
-      authority_certs_fetch_resource_impl(resource, dir_hint, node, rs);
-      tor_free(resource);
-    }
-    /* else we didn't add any: they were all pending */
-
-    SMARTLIST_FOREACH(fps, char *, cp, tor_free(cp));
-    smartlist_free(fps);
-  }
-
-  /* Do downloads by identity digest/signing key pair */
-  if (smartlist_len(missing_cert_digests) > 0) {
-    int need_plus = 0;
-    smartlist_t *fp_pairs = smartlist_new();
-
-    smartlist_add(fp_pairs, tor_strdup("fp-sk/"));
-
-    SMARTLIST_FOREACH_BEGIN(missing_cert_digests, const fp_pair_t *, d) {
-      char *fp_pair = NULL;
-
-      if (fp_pair_map_get(pending_cert, d))
-        continue;
-
-      /* Construct string encodings of the digests */
-      base16_encode(id_digest_str, sizeof(id_digest_str),
-                    d->first, DIGEST_LEN);
-      base16_encode(sk_digest_str, sizeof(sk_digest_str),
-                    d->second, DIGEST_LEN);
-
-      /* Now tor_asprintf() */
-      if (need_plus) {
-        tor_asprintf(&fp_pair, "+%s-%s", id_digest_str, sk_digest_str);
-      } else {
-        /* First one in the list doesn't get a '+' */
-        tor_asprintf(&fp_pair, "%s-%s", id_digest_str, sk_digest_str);
-        need_plus = 1;
-      }
-
-      /* Add it to the list of pairs to request */
-      smartlist_add(fp_pairs, fp_pair);
-    } SMARTLIST_FOREACH_END(d);
-
-    if (smartlist_len(fp_pairs) > 1) {
-      resource = smartlist_join_strings(fp_pairs, "", 0, NULL);
-      /* node and rs are directories that just gave us a consensus or
-       * certificates */
-      authority_certs_fetch_resource_impl(resource, dir_hint, node, rs);
-      tor_free(resource);
-    }
-    /* else they were all pending */
-
-    SMARTLIST_FOREACH(fp_pairs, char *, p, tor_free(p));
-    smartlist_free(fp_pairs);
-  }
-
-  smartlist_free(missing_id_digests);
-  SMARTLIST_FOREACH(missing_cert_digests, fp_pair_t *, p, tor_free(p));
-  smartlist_free(missing_cert_digests);
-  digestmap_free(pending_id, NULL);
-  fp_pair_map_free(pending_cert, NULL);
-}
-
 /* Router descriptor storage.
  *
  * Routerdescs are stored in a big file, named "cached-descriptors".  As new
@@ -1333,7 +195,7 @@ static int
 signed_desc_append_to_journal(signed_descriptor_t *desc,
                               desc_store_t *store)
 {
-  char *fname = get_datadir_fname_suffix(store->fname_base, ".new");
+  char *fname = get_cachedir_fname_suffix(store->fname_base, ".new");
   const char *body = signed_descriptor_get_body_impl(desc,1);
   size_t len = desc->signed_descriptor_len + desc->annotations_len;
 
@@ -1404,8 +266,8 @@ router_rebuild_store(int flags, desc_store_t *store)
 
   log_info(LD_DIR, "Rebuilding %s cache", store->description);
 
-  fname = get_datadir_fname(store->fname_base);
-  fname_tmp = get_datadir_fname_suffix(store->fname_base, ".tmp");
+  fname = get_cachedir_fname(store->fname_base);
+  fname_tmp = get_cachedir_fname_suffix(store->fname_base, ".tmp");
 
   chunk_list = smartlist_new();
 
@@ -1502,7 +364,7 @@ router_rebuild_store(int flags, desc_store_t *store)
   } SMARTLIST_FOREACH_END(sd);
 
   tor_free(fname);
-  fname = get_datadir_fname_suffix(store->fname_base, ".new");
+  fname = get_cachedir_fname_suffix(store->fname_base, ".new");
   write_str_to_file(fname, "", 1);
 
   r = 0;
@@ -1532,7 +394,7 @@ router_reload_router_list_impl(desc_store_t *store)
   int extrainfo = (store->type == EXTRAINFO_STORE);
   store->journal_len = store->store_len = 0;
 
-  fname = get_datadir_fname(store->fname_base);
+  fname = get_cachedir_fname(store->fname_base);
 
   if (store->mmap) {
     /* get rid of it first */
@@ -1559,7 +421,7 @@ router_reload_router_list_impl(desc_store_t *store)
   }
 
   tor_free(fname);
-  fname = get_datadir_fname_suffix(store->fname_base, ".new");
+  fname = get_cachedir_fname_suffix(store->fname_base, ".new");
   /* don't load empty files - we wouldn't get any data, even if we tried */
   if (file_status(fname) == FN_FILE)
     contents = read_file_to_str(fname, RFTS_BIN|RFTS_IGNORE_MISSING, &st);
@@ -1602,356 +464,6 @@ router_reload_router_list(void)
   return 0;
 }
 
-/** Return a smartlist containing a list of dir_server_t * for all
- * known trusted dirservers.  Callers must not modify the list or its
- * contents.
- */
-const smartlist_t *
-router_get_trusted_dir_servers(void)
-{
-  if (!trusted_dir_servers)
-    trusted_dir_servers = smartlist_new();
-
-  return trusted_dir_servers;
-}
-
-const smartlist_t *
-router_get_fallback_dir_servers(void)
-{
-  if (!fallback_dir_servers)
-    fallback_dir_servers = smartlist_new();
-
-  return fallback_dir_servers;
-}
-
-/** Try to find a running dirserver that supports operations of <b>type</b>.
- *
- * If there are no running dirservers in our routerlist and the
- * <b>PDS_RETRY_IF_NO_SERVERS</b> flag is set, set all the fallback ones
- * (including authorities) as running again, and pick one.
- *
- * If the <b>PDS_IGNORE_FASCISTFIREWALL</b> flag is set, then include
- * dirservers that we can't reach.
- *
- * If the <b>PDS_ALLOW_SELF</b> flag is not set, then don't include ourself
- * (if we're a dirserver).
- *
- * Don't pick a fallback directory mirror if any non-fallback is viable;
- * (the fallback directory mirrors include the authorities)
- * try to avoid using servers that have returned 503 recently.
- */
-const routerstatus_t *
-router_pick_directory_server(dirinfo_type_t type, int flags)
-{
-  int busy = 0;
-  const routerstatus_t *choice;
-
-  if (!routerlist)
-    return NULL;
-
-  choice = router_pick_directory_server_impl(type, flags, &busy);
-  if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS))
-    return choice;
-
-  if (busy) {
-    /* If the reason that we got no server is that servers are "busy",
-     * we must be excluding good servers because we already have serverdesc
-     * fetches with them.  Do not mark down servers up because of this. */
-    tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
-                         PDS_NO_EXISTING_MICRODESC_FETCH)));
-    return NULL;
-  }
-
-  log_info(LD_DIR,
-           "No reachable router entries for dirservers. "
-           "Trying them all again.");
-  /* mark all fallback directory mirrors as up again */
-  mark_all_dirservers_up(fallback_dir_servers);
-  /* try again */
-  choice = router_pick_directory_server_impl(type, flags, NULL);
-  return choice;
-}
-
-/** Return the dir_server_t for the directory authority whose identity
- * key hashes to <b>digest</b>, or NULL if no such authority is known.
- */
-dir_server_t *
-router_get_trusteddirserver_by_digest(const char *digest)
-{
-  if (!trusted_dir_servers)
-    return NULL;
-
-  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
-     {
-       if (tor_memeq(ds->digest, digest, DIGEST_LEN))
-         return ds;
-     });
-
-  return NULL;
-}
-
-/** Return the dir_server_t for the fallback dirserver whose identity
- * key hashes to <b>digest</b>, or NULL if no such fallback is in the list of
- * fallback_dir_servers. (fallback_dir_servers is affected by the FallbackDir
- * and UseDefaultFallbackDirs torrc options.)
- * The list of fallback directories includes the list of authorities.
- */
-dir_server_t *
-router_get_fallback_dirserver_by_digest(const char *digest)
-{
-  if (!fallback_dir_servers)
-    return NULL;
-
-  if (!digest)
-    return NULL;
-
-  SMARTLIST_FOREACH(fallback_dir_servers, dir_server_t *, ds,
-     {
-       if (tor_memeq(ds->digest, digest, DIGEST_LEN))
-         return ds;
-     });
-
-  return NULL;
-}
-
-/** Return 1 if any fallback dirserver's identity key hashes to <b>digest</b>,
- * or 0 if no such fallback is in the list of fallback_dir_servers.
- * (fallback_dir_servers is affected by the FallbackDir and
- * UseDefaultFallbackDirs torrc options.)
- * The list of fallback directories includes the list of authorities.
- */
-int
-router_digest_is_fallback_dir(const char *digest)
-{
-  return (router_get_fallback_dirserver_by_digest(digest) != NULL);
-}
-
-/** Return the dir_server_t for the directory authority whose
- * v3 identity key hashes to <b>digest</b>, or NULL if no such authority
- * is known.
- */
-MOCK_IMPL(dir_server_t *,
-trusteddirserver_get_by_v3_auth_digest, (const char *digest))
-{
-  if (!trusted_dir_servers)
-    return NULL;
-
-  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
-     {
-       if (tor_memeq(ds->v3_identity_digest, digest, DIGEST_LEN) &&
-           (ds->type & V3_DIRINFO))
-         return ds;
-     });
-
-  return NULL;
-}
-
-/** Try to find a running directory authority. Flags are as for
- * router_pick_directory_server.
- */
-const routerstatus_t *
-router_pick_trusteddirserver(dirinfo_type_t type, int flags)
-{
-  return router_pick_dirserver_generic(trusted_dir_servers, type, flags);
-}
-
-/** Try to find a running fallback directory. Flags are as for
- * router_pick_directory_server.
- */
-const routerstatus_t *
-router_pick_fallback_dirserver(dirinfo_type_t type, int flags)
-{
-  return router_pick_dirserver_generic(fallback_dir_servers, type, flags);
-}
-
-/** Try to find a running fallback directory. Flags are as for
- * router_pick_directory_server.
- */
-static const routerstatus_t *
-router_pick_dirserver_generic(smartlist_t *sourcelist,
-                              dirinfo_type_t type, int flags)
-{
-  const routerstatus_t *choice;
-  int busy = 0;
-
-  choice = router_pick_trusteddirserver_impl(sourcelist, type, flags, &busy);
-  if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS))
-    return choice;
-  if (busy) {
-    /* If the reason that we got no server is that servers are "busy",
-     * we must be excluding good servers because we already have serverdesc
-     * fetches with them.  Do not mark down servers up because of this. */
-    tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
-                         PDS_NO_EXISTING_MICRODESC_FETCH)));
-    return NULL;
-  }
-
-  log_info(LD_DIR,
-           "No dirservers are reachable. Trying them all again.");
-  mark_all_dirservers_up(sourcelist);
-  return router_pick_trusteddirserver_impl(sourcelist, type, flags, NULL);
-}
-
-/* Check if we already have a directory fetch from ap, for serverdesc
- * (including extrainfo) or microdesc documents.
- * If so, return 1, if not, return 0.
- * Also returns 0 if addr is NULL, tor_addr_is_null(addr), or dir_port is 0.
- */
-STATIC int
-router_is_already_dir_fetching(const tor_addr_port_t *ap, int serverdesc,
-                               int microdesc)
-{
-  if (!ap || tor_addr_is_null(&ap->addr) || !ap->port) {
-    return 0;
-  }
-
-  /* XX/teor - we're not checking tunnel connections here, see #17848
-   */
-  if (serverdesc && (
-     connection_get_by_type_addr_port_purpose(
-       CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_SERVERDESC)
-  || connection_get_by_type_addr_port_purpose(
-       CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_EXTRAINFO))) {
-    return 1;
-  }
-
-  if (microdesc && (
-     connection_get_by_type_addr_port_purpose(
-       CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_MICRODESC))) {
-    return 1;
-  }
-
-  return 0;
-}
-
-/* Check if we already have a directory fetch from ds, for serverdesc
- * (including extrainfo) or microdesc documents.
- * If so, return 1, if not, return 0.
- */
-static int
-router_is_already_dir_fetching_ds(const dir_server_t *ds,
-                                  int serverdesc,
-                                  int microdesc)
-{
-  tor_addr_port_t ipv4_dir_ap, ipv6_dir_ap;
-
-  /* Assume IPv6 DirPort is the same as IPv4 DirPort */
-  tor_addr_from_ipv4h(&ipv4_dir_ap.addr, ds->addr);
-  ipv4_dir_ap.port = ds->dir_port;
-  tor_addr_copy(&ipv6_dir_ap.addr, &ds->ipv6_addr);
-  ipv6_dir_ap.port = ds->dir_port;
-
-  return (router_is_already_dir_fetching(&ipv4_dir_ap, serverdesc, microdesc)
-       || router_is_already_dir_fetching(&ipv6_dir_ap, serverdesc, microdesc));
-}
-
-/* Check if we already have a directory fetch from rs, for serverdesc
- * (including extrainfo) or microdesc documents.
- * If so, return 1, if not, return 0.
- */
-static int
-router_is_already_dir_fetching_rs(const routerstatus_t *rs,
-                                  int serverdesc,
-                                  int microdesc)
-{
-  tor_addr_port_t ipv4_dir_ap, ipv6_dir_ap;
-
-  /* Assume IPv6 DirPort is the same as IPv4 DirPort */
-  tor_addr_from_ipv4h(&ipv4_dir_ap.addr, rs->addr);
-  ipv4_dir_ap.port = rs->dir_port;
-  tor_addr_copy(&ipv6_dir_ap.addr, &rs->ipv6_addr);
-  ipv6_dir_ap.port = rs->dir_port;
-
-  return (router_is_already_dir_fetching(&ipv4_dir_ap, serverdesc, microdesc)
-       || router_is_already_dir_fetching(&ipv6_dir_ap, serverdesc, microdesc));
-}
-
-#ifndef LOG_FALSE_POSITIVES_DURING_BOOTSTRAP
-#define LOG_FALSE_POSITIVES_DURING_BOOTSTRAP 0
-#endif
-
-/* Log a message if rs is not found or not a preferred address */
-static void
-router_picked_poor_directory_log(const routerstatus_t *rs)
-{
-  const networkstatus_t *usable_consensus;
-  usable_consensus = networkstatus_get_reasonably_live_consensus(time(NULL),
-                                                 usable_consensus_flavor());
-
-#if !LOG_FALSE_POSITIVES_DURING_BOOTSTRAP
-  /* Don't log early in the bootstrap process, it's normal to pick from a
-   * small pool of nodes. Of course, this won't help if we're trying to
-   * diagnose bootstrap issues. */
-  if (!smartlist_len(nodelist_get_list()) || !usable_consensus
-      || !router_have_minimum_dir_info()) {
-    return;
-  }
-#endif
-
-  /* We couldn't find a node, or the one we have doesn't fit our preferences.
-   * Sometimes this is normal, sometimes it can be a reachability issue. */
-  if (!rs) {
-    /* This happens a lot, so it's at debug level */
-    log_debug(LD_DIR, "Wanted to make an outgoing directory connection, but "
-              "we couldn't find a directory that fit our criteria. "
-              "Perhaps we will succeed next time with less strict criteria.");
-  } else if (!fascist_firewall_allows_rs(rs, FIREWALL_OR_CONNECTION, 1)
-             && !fascist_firewall_allows_rs(rs, FIREWALL_DIR_CONNECTION, 1)
-             ) {
-    /* This is rare, and might be interesting to users trying to diagnose
-     * connection issues on dual-stack machines. */
-    log_info(LD_DIR, "Selected a directory %s with non-preferred OR and Dir "
-             "addresses for launching an outgoing connection: "
-             "IPv4 %s OR %d Dir %d IPv6 %s OR %d Dir %d",
-             routerstatus_describe(rs),
-             fmt_addr32(rs->addr), rs->or_port,
-             rs->dir_port, fmt_addr(&rs->ipv6_addr),
-             rs->ipv6_orport, rs->dir_port);
-  }
-}
-
-#undef LOG_FALSE_POSITIVES_DURING_BOOTSTRAP
-
-/** How long do we avoid using a directory server after it's given us a 503? */
-#define DIR_503_TIMEOUT (60*60)
-
-/* Common retry code for router_pick_directory_server_impl and
- * router_pick_trusteddirserver_impl. Retry with the non-preferred IP version.
- * Must be called before RETRY_WITHOUT_EXCLUDE().
- *
- * If we got no result, and we are applying IP preferences, and we are a
- * client that could use an alternate IP version, try again with the
- * opposite preferences. */
-#define RETRY_ALTERNATE_IP_VERSION(retry_label)                               \
-  STMT_BEGIN                                                                  \
-    if (result == NULL && try_ip_pref && options->ClientUseIPv4               \
-        && fascist_firewall_use_ipv6(options) && !server_mode(options)        \
-        && !n_busy) {                                                         \
-      n_excluded = 0;                                                         \
-      n_busy = 0;                                                             \
-      try_ip_pref = 0;                                                        \
-      goto retry_label;                                                       \
-    }                                                                         \
-  STMT_END                                                                    \
-
-/* Common retry code for router_pick_directory_server_impl and
- * router_pick_trusteddirserver_impl. Retry without excluding nodes, but with
- * the preferred IP version. Must be called after RETRY_ALTERNATE_IP_VERSION().
- *
- * If we got no result, and we are excluding nodes, and StrictNodes is
- * not set, try again without excluding nodes. */
-#define RETRY_WITHOUT_EXCLUDE(retry_label)                                    \
-  STMT_BEGIN                                                                  \
-    if (result == NULL && try_excluding && !options->StrictNodes              \
-        && n_excluded && !n_busy) {                                           \
-      try_excluding = 0;                                                      \
-      n_excluded = 0;                                                         \
-      n_busy = 0;                                                             \
-      try_ip_pref = 1;                                                        \
-      goto retry_label;                                                       \
-    }                                                                         \
-  STMT_END
-
 /* When iterating through the routerlist, can OR address/port preference
  * and reachability checks be skipped?
  */
@@ -1967,7 +479,7 @@ router_skip_or_reachability(const or_options_t *options, int try_ip_pref)
 /* When iterating through the routerlist, can Dir address/port preference
  * and reachability checks be skipped?
  */
-static int
+int
 router_skip_dir_reachability(const or_options_t *options, int try_ip_pref)
 {
   /* Servers always have and prefer IPv4.
@@ -1976,315 +488,6 @@ router_skip_dir_reachability(const or_options_t *options, int try_ip_pref)
   return server_mode(options) || (!try_ip_pref && !firewall_is_fascist_dir());
 }
 
-/** Pick a random running valid directory server/mirror from our
- * routerlist.  Arguments are as for router_pick_directory_server(), except:
- *
- * If <b>n_busy_out</b> is provided, set *<b>n_busy_out</b> to the number of
- * directories that we excluded for no other reason than
- * PDS_NO_EXISTING_SERVERDESC_FETCH or PDS_NO_EXISTING_MICRODESC_FETCH.
- */
-STATIC const routerstatus_t *
-router_pick_directory_server_impl(dirinfo_type_t type, int flags,
-                                  int *n_busy_out)
-{
-  const or_options_t *options = get_options();
-  const node_t *result;
-  smartlist_t *direct, *tunnel;
-  smartlist_t *trusted_direct, *trusted_tunnel;
-  smartlist_t *overloaded_direct, *overloaded_tunnel;
-  time_t now = time(NULL);
-  const networkstatus_t *consensus = networkstatus_get_latest_consensus();
-  const int requireother = ! (flags & PDS_ALLOW_SELF);
-  const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL);
-  const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH);
-  const int no_microdesc_fetching = (flags & PDS_NO_EXISTING_MICRODESC_FETCH);
-  const int for_guard = (flags & PDS_FOR_GUARD);
-  int try_excluding = 1, n_excluded = 0, n_busy = 0;
-  int try_ip_pref = 1;
-
-  if (!consensus)
-    return NULL;
-
- retry_search:
-
-  direct = smartlist_new();
-  tunnel = smartlist_new();
-  trusted_direct = smartlist_new();
-  trusted_tunnel = smartlist_new();
-  overloaded_direct = smartlist_new();
-  overloaded_tunnel = smartlist_new();
-
-  const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref);
-  const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref);
-  const int must_have_or = directory_must_use_begindir(options);
-
-  /* Find all the running dirservers we know about. */
-  SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) {
-    int is_trusted, is_trusted_extrainfo;
-    int is_overloaded;
-    const routerstatus_t *status = node->rs;
-    const country_t country = node->country;
-    if (!status)
-      continue;
-
-    if (!node->is_running || !node_is_dir(node) || !node->is_valid)
-      continue;
-    if (requireother && router_digest_is_me(node->identity))
-      continue;
-    is_trusted = router_digest_is_trusted_dir(node->identity);
-    is_trusted_extrainfo = router_digest_is_trusted_dir_type(
-                           node->identity, EXTRAINFO_DIRINFO);
-    if ((type & EXTRAINFO_DIRINFO) &&
-        !router_supports_extrainfo(node->identity, is_trusted_extrainfo))
-      continue;
-    /* Don't make the same node a guard twice */
-    if (for_guard && node->using_as_guard) {
-      continue;
-    }
-    /* Ensure that a directory guard is actually a guard node. */
-    if (for_guard && !node->is_possible_guard) {
-      continue;
-    }
-    if (try_excluding &&
-        routerset_contains_routerstatus(options->ExcludeNodes, status,
-                                        country)) {
-      ++n_excluded;
-      continue;
-    }
-
-    if (router_is_already_dir_fetching_rs(status,
-                                          no_serverdesc_fetching,
-                                          no_microdesc_fetching)) {
-      ++n_busy;
-      continue;
-    }
-
-    is_overloaded = status->last_dir_503_at + DIR_503_TIMEOUT > now;
-
-    /* Clients use IPv6 addresses if the server has one and the client
-     * prefers IPv6.
-     * Add the router if its preferred address and port are reachable.
-     * If we don't get any routers, we'll try again with the non-preferred
-     * address for each router (if any). (To ensure correct load-balancing
-     * we try routers that only have one address both times.)
-     */
-    if (!fascistfirewall || skip_or_fw ||
-        fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION,
-                                     try_ip_pref))
-      smartlist_add(is_trusted ? trusted_tunnel :
-                    is_overloaded ? overloaded_tunnel : tunnel, (void*)node);
-    else if (!must_have_or && (skip_dir_fw ||
-             fascist_firewall_allows_node(node, FIREWALL_DIR_CONNECTION,
-                                          try_ip_pref)))
-      smartlist_add(is_trusted ? trusted_direct :
-                    is_overloaded ? overloaded_direct : direct, (void*)node);
-  } SMARTLIST_FOREACH_END(node);
-
-  if (smartlist_len(tunnel)) {
-    result = node_sl_choose_by_bandwidth(tunnel, WEIGHT_FOR_DIR);
-  } else if (smartlist_len(overloaded_tunnel)) {
-    result = node_sl_choose_by_bandwidth(overloaded_tunnel,
-                                                 WEIGHT_FOR_DIR);
-  } else if (smartlist_len(trusted_tunnel)) {
-    /* FFFF We don't distinguish between trusteds and overloaded trusteds
-     * yet. Maybe one day we should. */
-    /* FFFF We also don't load balance over authorities yet. I think this
-     * is a feature, but it could easily be a bug. -RD */
-    result = smartlist_choose(trusted_tunnel);
-  } else if (smartlist_len(direct)) {
-    result = node_sl_choose_by_bandwidth(direct, WEIGHT_FOR_DIR);
-  } else if (smartlist_len(overloaded_direct)) {
-    result = node_sl_choose_by_bandwidth(overloaded_direct,
-                                         WEIGHT_FOR_DIR);
-  } else {
-    result = smartlist_choose(trusted_direct);
-  }
-  smartlist_free(direct);
-  smartlist_free(tunnel);
-  smartlist_free(trusted_direct);
-  smartlist_free(trusted_tunnel);
-  smartlist_free(overloaded_direct);
-  smartlist_free(overloaded_tunnel);
-
-  RETRY_ALTERNATE_IP_VERSION(retry_search);
-
-  RETRY_WITHOUT_EXCLUDE(retry_search);
-
-  if (n_busy_out)
-    *n_busy_out = n_busy;
-
-  router_picked_poor_directory_log(result ? result->rs : NULL);
-
-  return result ? result->rs : NULL;
-}
-
-/** Pick a random element from a list of dir_server_t, weighting by their
- * <b>weight</b> field. */
-static const dir_server_t *
-dirserver_choose_by_weight(const smartlist_t *servers, double authority_weight)
-{
-  int n = smartlist_len(servers);
-  int i;
-  double *weights_dbl;
-  uint64_t *weights_u64;
-  const dir_server_t *ds;
-
-  weights_dbl = tor_calloc(n, sizeof(double));
-  weights_u64 = tor_calloc(n, sizeof(uint64_t));
-  for (i = 0; i < n; ++i) {
-    ds = smartlist_get(servers, i);
-    weights_dbl[i] = ds->weight;
-    if (ds->is_authority)
-      weights_dbl[i] *= authority_weight;
-  }
-
-  scale_array_elements_to_u64(weights_u64, weights_dbl, n, NULL);
-  i = choose_array_element_by_weight(weights_u64, n);
-  tor_free(weights_dbl);
-  tor_free(weights_u64);
-  return (i < 0) ? NULL : smartlist_get(servers, i);
-}
-
-/** Choose randomly from among the dir_server_ts in sourcelist that
- * are up. Flags are as for router_pick_directory_server_impl().
- */
-static const routerstatus_t *
-router_pick_trusteddirserver_impl(const smartlist_t *sourcelist,
-                                  dirinfo_type_t type, int flags,
-                                  int *n_busy_out)
-{
-  const or_options_t *options = get_options();
-  smartlist_t *direct, *tunnel;
-  smartlist_t *overloaded_direct, *overloaded_tunnel;
-  const routerinfo_t *me = router_get_my_routerinfo();
-  const routerstatus_t *result = NULL;
-  time_t now = time(NULL);
-  const int requireother = ! (flags & PDS_ALLOW_SELF);
-  const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL);
-  const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH);
-  const int no_microdesc_fetching =(flags & PDS_NO_EXISTING_MICRODESC_FETCH);
-  const double auth_weight = (sourcelist == fallback_dir_servers) ?
-    options->DirAuthorityFallbackRate : 1.0;
-  smartlist_t *pick_from;
-  int n_busy = 0;
-  int try_excluding = 1, n_excluded = 0;
-  int try_ip_pref = 1;
-
-  if (!sourcelist)
-    return NULL;
-
- retry_search:
-
-  direct = smartlist_new();
-  tunnel = smartlist_new();
-  overloaded_direct = smartlist_new();
-  overloaded_tunnel = smartlist_new();
-
-  const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref);
-  const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref);
-  const int must_have_or = directory_must_use_begindir(options);
-
-  SMARTLIST_FOREACH_BEGIN(sourcelist, const dir_server_t *, d)
-    {
-      int is_overloaded =
-          d->fake_status.last_dir_503_at + DIR_503_TIMEOUT > now;
-      if (!d->is_running) continue;
-      if ((type & d->type) == 0)
-        continue;
-      int is_trusted_extrainfo = router_digest_is_trusted_dir_type(
-                                 d->digest, EXTRAINFO_DIRINFO);
-      if ((type & EXTRAINFO_DIRINFO) &&
-          !router_supports_extrainfo(d->digest, is_trusted_extrainfo))
-        continue;
-      if (requireother && me && router_digest_is_me(d->digest))
-        continue;
-      if (try_excluding &&
-          routerset_contains_routerstatus(options->ExcludeNodes,
-                                          &d->fake_status, -1)) {
-        ++n_excluded;
-        continue;
-      }
-
-      if (router_is_already_dir_fetching_ds(d, no_serverdesc_fetching,
-                                            no_microdesc_fetching)) {
-        ++n_busy;
-        continue;
-      }
-
-      /* Clients use IPv6 addresses if the server has one and the client
-       * prefers IPv6.
-       * Add the router if its preferred address and port are reachable.
-       * If we don't get any routers, we'll try again with the non-preferred
-       * address for each router (if any). (To ensure correct load-balancing
-       * we try routers that only have one address both times.)
-       */
-      if (!fascistfirewall || skip_or_fw ||
-          fascist_firewall_allows_dir_server(d, FIREWALL_OR_CONNECTION,
-                                             try_ip_pref))
-        smartlist_add(is_overloaded ? overloaded_tunnel : tunnel, (void*)d);
-      else if (!must_have_or && (skip_dir_fw ||
-               fascist_firewall_allows_dir_server(d, FIREWALL_DIR_CONNECTION,
-                                                  try_ip_pref)))
-        smartlist_add(is_overloaded ? overloaded_direct : direct, (void*)d);
-    }
-  SMARTLIST_FOREACH_END(d);
-
-  if (smartlist_len(tunnel)) {
-    pick_from = tunnel;
-  } else if (smartlist_len(overloaded_tunnel)) {
-    pick_from = overloaded_tunnel;
-  } else if (smartlist_len(direct)) {
-    pick_from = direct;
-  } else {
-    pick_from = overloaded_direct;
-  }
-
-  {
-    const dir_server_t *selection =
-      dirserver_choose_by_weight(pick_from, auth_weight);
-
-    if (selection)
-      result = &selection->fake_status;
-  }
-
-  smartlist_free(direct);
-  smartlist_free(tunnel);
-  smartlist_free(overloaded_direct);
-  smartlist_free(overloaded_tunnel);
-
-  RETRY_ALTERNATE_IP_VERSION(retry_search);
-
-  RETRY_WITHOUT_EXCLUDE(retry_search);
-
-  router_picked_poor_directory_log(result);
-
-  if (n_busy_out)
-    *n_busy_out = n_busy;
-  return result;
-}
-
-/** Mark as running every dir_server_t in <b>server_list</b>. */
-static void
-mark_all_dirservers_up(smartlist_t *server_list)
-{
-  if (server_list) {
-    SMARTLIST_FOREACH_BEGIN(server_list, dir_server_t *, dir) {
-      routerstatus_t *rs;
-      node_t *node;
-      dir->is_running = 1;
-      node = node_get_mutable_by_id(dir->digest);
-      if (node)
-        node->is_running = 1;
-      rs = router_get_mutable_consensus_status_by_id(dir->digest);
-      if (rs) {
-        rs->last_dir_503_at = 0;
-        control_event_networkstatus_changed_single(rs);
-      }
-    } SMARTLIST_FOREACH_END(dir);
-  }
-  router_dir_info_changed();
-}
-
 /** Return true iff r1 and r2 have the same address and OR port. */
 int
 routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2)
@@ -2294,52 +497,22 @@ routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2)
     r1->ipv6_orport == r2->ipv6_orport;
 }
 
-/** Reset all internal variables used to count failed downloads of network
- * status objects. */
-void
-router_reset_status_download_failures(void)
-{
-  mark_all_dirservers_up(fallback_dir_servers);
-}
-
-/** Given a <b>router</b>, add every node_t in its family (including the
- * node itself!) to <b>sl</b>.
- *
- * Note the type mismatch: This function takes a routerinfo, but adds nodes
- * to the smartlist!
- */
-static void
-routerlist_add_node_and_family(smartlist_t *sl, const routerinfo_t *router)
-{
-  /* XXXX MOVE ? */
-  node_t fake_node;
-  const node_t *node = node_get_by_id(router->cache_info.identity_digest);;
-  if (node == NULL) {
-    memset(&fake_node, 0, sizeof(fake_node));
-    fake_node.ri = (routerinfo_t *)router;
-    memcpy(fake_node.identity, router->cache_info.identity_digest, DIGEST_LEN);
-    node = &fake_node;
-  }
-  nodelist_add_node_and_family(sl, node);
-}
-
 /** Add every suitable node from our nodelist to <b>sl</b>, so that
  * we can pick a node for a circuit.
  */
 void
-router_add_running_nodes_to_smartlist(smartlist_t *sl, int allow_invalid,
-                                      int need_uptime, int need_capacity,
-                                      int need_guard, int need_desc,
-                                      int pref_addr, int direct_conn)
+router_add_running_nodes_to_smartlist(smartlist_t *sl, int need_uptime,
+                                      int need_capacity, int need_guard,
+                                      int need_desc, int pref_addr,
+                                      int direct_conn)
 {
   const int check_reach = !router_skip_or_reachability(get_options(),
                                                        pref_addr);
   /* XXXX MOVE */
   SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) {
-    if (!node->is_running ||
-        (!node->is_valid && !allow_invalid))
+    if (!node->is_running || !node->is_valid)
       continue;
-    if (need_desc && !(node->ri || (node->rs && node->md)))
+    if (need_desc && !node_has_preferred_descriptor(node, direct_conn))
       continue;
     if (node->ri && node->ri->purpose != ROUTER_PURPOSE_GENERAL)
       continue;
@@ -2405,505 +578,6 @@ router_get_advertised_bandwidth_capped(const routerinfo_t *router)
   return result;
 }
 
-/** Given an array of double/uint64_t unions that are currently being used as
- * doubles, convert them to uint64_t, and try to scale them linearly so as to
- * much of the range of uint64_t. If <b>total_out</b> is provided, set it to
- * the sum of all elements in the array _before_ scaling. */
-STATIC void
-scale_array_elements_to_u64(uint64_t *entries_out, const double *entries_in,
-                            int n_entries,
-                            uint64_t *total_out)
-{
-  double total = 0.0;
-  double scale_factor = 0.0;
-  int i;
-
-  for (i = 0; i < n_entries; ++i)
-    total += entries_in[i];
-
-  if (total > 0.0) {
-    scale_factor = ((double)INT64_MAX) / total;
-    scale_factor /= 4.0; /* make sure we're very far away from overflowing */
-  }
-
-  for (i = 0; i < n_entries; ++i)
-    entries_out[i] = tor_llround(entries_in[i] * scale_factor);
-
-  if (total_out)
-    *total_out = (uint64_t) total;
-}
-
-/** Pick a random element of <b>n_entries</b>-element array <b>entries</b>,
- * choosing each element with a probability proportional to its (uint64_t)
- * value, and return the index of that element.  If all elements are 0, choose
- * an index at random. Return -1 on error.
- */
-STATIC int
-choose_array_element_by_weight(const uint64_t *entries, int n_entries)
-{
-  int i;
-  uint64_t rand_val;
-  uint64_t total = 0;
-
-  for (i = 0; i < n_entries; ++i)
-    total += entries[i];
-
-  if (n_entries < 1)
-    return -1;
-
-  if (total == 0)
-    return crypto_rand_int(n_entries);
-
-  tor_assert(total < INT64_MAX);
-
-  rand_val = crypto_rand_uint64(total);
-
-  return select_array_member_cumulative_timei(
-                           entries, n_entries, total, rand_val);
-}
-
-/** When weighting bridges, enforce these values as lower and upper
- * bound for believable bandwidth, because there is no way for us
- * to verify a bridge's bandwidth currently. */
-#define BRIDGE_MIN_BELIEVABLE_BANDWIDTH 20000  /* 20 kB/sec */
-#define BRIDGE_MAX_BELIEVABLE_BANDWIDTH 100000 /* 100 kB/sec */
-
-/** Return the smaller of the router's configured BandwidthRate
- * and its advertised capacity, making sure to stay within the
- * interval between bridge-min-believe-bw and
- * bridge-max-believe-bw. */
-static uint32_t
-bridge_get_advertised_bandwidth_bounded(routerinfo_t *router)
-{
-  uint32_t result = router->bandwidthcapacity;
-  if (result > router->bandwidthrate)
-    result = router->bandwidthrate;
-  if (result > BRIDGE_MAX_BELIEVABLE_BANDWIDTH)
-    result = BRIDGE_MAX_BELIEVABLE_BANDWIDTH;
-  else if (result < BRIDGE_MIN_BELIEVABLE_BANDWIDTH)
-    result = BRIDGE_MIN_BELIEVABLE_BANDWIDTH;
-  return result;
-}
-
-/** Return bw*1000, unless bw*1000 would overflow, in which case return
- * INT32_MAX. */
-static inline int32_t
-kb_to_bytes(uint32_t bw)
-{
-  return (bw > (INT32_MAX/1000)) ? INT32_MAX : bw*1000;
-}
-
-/** Helper function:
- * choose a random element of smartlist <b>sl</b> of nodes, weighted by
- * the advertised bandwidth of each element using the consensus
- * bandwidth weights.
- *
- * If <b>rule</b>==WEIGHT_FOR_EXIT. we're picking an exit node: consider all
- * nodes' bandwidth equally regardless of their Exit status, since there may
- * be some in the list because they exit to obscure ports. If
- * <b>rule</b>==NO_WEIGHTING, we're picking a non-exit node: weight
- * exit-node's bandwidth less depending on the smallness of the fraction of
- * Exit-to-total bandwidth.  If <b>rule</b>==WEIGHT_FOR_GUARD, we're picking a
- * guard node: consider all guard's bandwidth equally. Otherwise, weight
- * guards proportionally less.
- */
-static const node_t *
-smartlist_choose_node_by_bandwidth_weights(const smartlist_t *sl,
-                                           bandwidth_weight_rule_t rule)
-{
-  double *bandwidths_dbl=NULL;
-  uint64_t *bandwidths_u64=NULL;
-
-  if (compute_weighted_bandwidths(sl, rule, &bandwidths_dbl, NULL) < 0)
-    return NULL;
-
-  bandwidths_u64 = tor_calloc(smartlist_len(sl), sizeof(uint64_t));
-  scale_array_elements_to_u64(bandwidths_u64, bandwidths_dbl,
-                              smartlist_len(sl), NULL);
-
-  {
-    int idx = choose_array_element_by_weight(bandwidths_u64,
-                                             smartlist_len(sl));
-    tor_free(bandwidths_dbl);
-    tor_free(bandwidths_u64);
-    return idx < 0 ? NULL : smartlist_get(sl, idx);
-  }
-}
-
-/** Given a list of routers and a weighting rule as in
- * smartlist_choose_node_by_bandwidth_weights, compute weighted bandwidth
- * values for each node and store them in a freshly allocated
- * *<b>bandwidths_out</b> of the same length as <b>sl</b>, and holding results
- * as doubles. If <b>total_bandwidth_out</b> is non-NULL, set it to the total
- * of all the bandwidths.
- * Return 0 on success, -1 on failure. */
-static int
-compute_weighted_bandwidths(const smartlist_t *sl,
-                            bandwidth_weight_rule_t rule,
-                            double **bandwidths_out,
-                            double *total_bandwidth_out)
-{
-  int64_t weight_scale;
-  double Wg = -1, Wm = -1, We = -1, Wd = -1;
-  double Wgb = -1, Wmb = -1, Web = -1, Wdb = -1;
-  guardfraction_bandwidth_t guardfraction_bw;
-  double *bandwidths = NULL;
-  double total_bandwidth = 0.0;
-
-  tor_assert(sl);
-  tor_assert(bandwidths_out);
-
-  /* Can't choose exit and guard at same time */
-  tor_assert(rule == NO_WEIGHTING ||
-             rule == WEIGHT_FOR_EXIT ||
-             rule == WEIGHT_FOR_GUARD ||
-             rule == WEIGHT_FOR_MID ||
-             rule == WEIGHT_FOR_DIR);
-
-  *bandwidths_out = NULL;
-
-  if (total_bandwidth_out) {
-    *total_bandwidth_out = 0.0;
-  }
-
-  if (smartlist_len(sl) == 0) {
-    log_info(LD_CIRC,
-             "Empty routerlist passed in to consensus weight node "
-             "selection for rule %s",
-             bandwidth_weight_rule_to_string(rule));
-    return -1;
-  }
-
-  weight_scale = networkstatus_get_weight_scale_param(NULL);
-
-  if (rule == WEIGHT_FOR_GUARD) {
-    Wg = networkstatus_get_bw_weight(NULL, "Wgg", -1);
-    Wm = networkstatus_get_bw_weight(NULL, "Wgm", -1); /* Bridges */
-    We = 0;
-    Wd = networkstatus_get_bw_weight(NULL, "Wgd", -1);
-
-    Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1);
-    Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1);
-    Web = networkstatus_get_bw_weight(NULL, "Web", -1);
-    Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1);
-  } else if (rule == WEIGHT_FOR_MID) {
-    Wg = networkstatus_get_bw_weight(NULL, "Wmg", -1);
-    Wm = networkstatus_get_bw_weight(NULL, "Wmm", -1);
-    We = networkstatus_get_bw_weight(NULL, "Wme", -1);
-    Wd = networkstatus_get_bw_weight(NULL, "Wmd", -1);
-
-    Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1);
-    Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1);
-    Web = networkstatus_get_bw_weight(NULL, "Web", -1);
-    Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1);
-  } else if (rule == WEIGHT_FOR_EXIT) {
-    // Guards CAN be exits if they have weird exit policies
-    // They are d then I guess...
-    We = networkstatus_get_bw_weight(NULL, "Wee", -1);
-    Wm = networkstatus_get_bw_weight(NULL, "Wem", -1); /* Odd exit policies */
-    Wd = networkstatus_get_bw_weight(NULL, "Wed", -1);
-    Wg = networkstatus_get_bw_weight(NULL, "Weg", -1); /* Odd exit policies */
-
-    Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1);
-    Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1);
-    Web = networkstatus_get_bw_weight(NULL, "Web", -1);
-    Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1);
-  } else if (rule == WEIGHT_FOR_DIR) {
-    We = networkstatus_get_bw_weight(NULL, "Wbe", -1);
-    Wm = networkstatus_get_bw_weight(NULL, "Wbm", -1);
-    Wd = networkstatus_get_bw_weight(NULL, "Wbd", -1);
-    Wg = networkstatus_get_bw_weight(NULL, "Wbg", -1);
-
-    Wgb = Wmb = Web = Wdb = weight_scale;
-  } else if (rule == NO_WEIGHTING) {
-    Wg = Wm = We = Wd = weight_scale;
-    Wgb = Wmb = Web = Wdb = weight_scale;
-  }
-
-  if (Wg < 0 || Wm < 0 || We < 0 || Wd < 0 || Wgb < 0 || Wmb < 0 || Wdb < 0
-      || Web < 0) {
-    log_debug(LD_CIRC,
-              "Got negative bandwidth weights. Defaulting to naive selection"
-              " algorithm.");
-    Wg = Wm = We = Wd = weight_scale;
-    Wgb = Wmb = Web = Wdb = weight_scale;
-  }
-
-  Wg /= weight_scale;
-  Wm /= weight_scale;
-  We /= weight_scale;
-  Wd /= weight_scale;
-
-  Wgb /= weight_scale;
-  Wmb /= weight_scale;
-  Web /= weight_scale;
-  Wdb /= weight_scale;
-
-  bandwidths = tor_calloc(smartlist_len(sl), sizeof(double));
-
-  // Cycle through smartlist and total the bandwidth.
-  static int warned_missing_bw = 0;
-  SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) {
-    int is_exit = 0, is_guard = 0, is_dir = 0, this_bw = 0;
-    double weight = 1;
-    double weight_without_guard_flag = 0; /* Used for guardfraction */
-    double final_weight = 0;
-    is_exit = node->is_exit && ! node->is_bad_exit;
-    is_guard = node->is_possible_guard;
-    is_dir = node_is_dir(node);
-    if (node->rs) {
-      if (!node->rs->has_bandwidth) {
-        /* This should never happen, unless all the authorites downgrade
-         * to 0.2.0 or rogue routerstatuses get inserted into our consensus. */
-        if (! warned_missing_bw) {
-          log_warn(LD_BUG,
-                 "Consensus is missing some bandwidths. Using a naive "
-                 "router selection algorithm");
-          warned_missing_bw = 1;
-        }
-        this_bw = 30000; /* Chosen arbitrarily */
-      } else {
-        this_bw = kb_to_bytes(node->rs->bandwidth_kb);
-      }
-    } else if (node->ri) {
-      /* bridge or other descriptor not in our consensus */
-      this_bw = bridge_get_advertised_bandwidth_bounded(node->ri);
-    } else {
-      /* We can't use this one. */
-      continue;
-    }
-
-    if (is_guard && is_exit) {
-      weight = (is_dir ? Wdb*Wd : Wd);
-      weight_without_guard_flag = (is_dir ? Web*We : We);
-    } else if (is_guard) {
-      weight = (is_dir ? Wgb*Wg : Wg);
-      weight_without_guard_flag = (is_dir ? Wmb*Wm : Wm);
-    } else if (is_exit) {
-      weight = (is_dir ? Web*We : We);
-    } else { // middle
-      weight = (is_dir ? Wmb*Wm : Wm);
-    }
-    /* These should be impossible; but overflows here would be bad, so let's
-     * make sure. */
-    if (this_bw < 0)
-      this_bw = 0;
-    if (weight < 0.0)
-      weight = 0.0;
-    if (weight_without_guard_flag < 0.0)
-      weight_without_guard_flag = 0.0;
-
-    /* If guardfraction information is available in the consensus, we
-     * want to calculate this router's bandwidth according to its
-     * guardfraction. Quoting from proposal236:
-     *
-     *    Let Wpf denote the weight from the 'bandwidth-weights' line a
-     *    client would apply to N for position p if it had the guard
-     *    flag, Wpn the weight if it did not have the guard flag, and B the
-     *    measured bandwidth of N in the consensus.  Then instead of choosing
-     *    N for position p proportionally to Wpf*B or Wpn*B, clients should
-     *    choose N proportionally to F*Wpf*B + (1-F)*Wpn*B.
-     */
-    if (node->rs && node->rs->has_guardfraction && rule != WEIGHT_FOR_GUARD) {
-      /* XXX The assert should actually check for is_guard. However,
-       * that crashes dirauths because of #13297. This should be
-       * equivalent: */
-      tor_assert(node->rs->is_possible_guard);
-
-      guard_get_guardfraction_bandwidth(&guardfraction_bw,
-                                        this_bw,
-                                        node->rs->guardfraction_percentage);
-
-      /* Calculate final_weight = F*Wpf*B + (1-F)*Wpn*B */
-      final_weight =
-        guardfraction_bw.guard_bw * weight +
-        guardfraction_bw.non_guard_bw * weight_without_guard_flag;
-
-      log_debug(LD_GENERAL, "%s: Guardfraction weight %f instead of %f (%s)",
-                node->rs->nickname, final_weight, weight*this_bw,
-                bandwidth_weight_rule_to_string(rule));
-    } else { /* no guardfraction information. calculate the weight normally. */
-      final_weight = weight*this_bw;
-    }
-
-    bandwidths[node_sl_idx] = final_weight;
-    total_bandwidth += final_weight;
-  } SMARTLIST_FOREACH_END(node);
-
-  log_debug(LD_CIRC, "Generated weighted bandwidths for rule %s based "
-            "on weights "
-            "Wg=%f Wm=%f We=%f Wd=%f with total bw %f",
-            bandwidth_weight_rule_to_string(rule),
-            Wg, Wm, We, Wd, total_bandwidth);
-
-  *bandwidths_out = bandwidths;
-
-  if (total_bandwidth_out) {
-    *total_bandwidth_out = total_bandwidth;
-  }
-
-  return 0;
-}
-
-/** For all nodes in <b>sl</b>, return the fraction of those nodes, weighted
- * by their weighted bandwidths with rule <b>rule</b>, for which we have
- * descriptors. */
-double
-frac_nodes_with_descriptors(const smartlist_t *sl,
-                            bandwidth_weight_rule_t rule)
-{
-  double *bandwidths = NULL;
-  double total, present;
-
-  if (smartlist_len(sl) == 0)
-    return 0.0;
-
-  if (compute_weighted_bandwidths(sl, rule, &bandwidths, &total) < 0 ||
-      total <= 0.0) {
-    int n_with_descs = 0;
-    SMARTLIST_FOREACH(sl, const node_t *, node, {
-      if (node_has_descriptor(node))
-        n_with_descs++;
-    });
-
-    tor_free(bandwidths);
-    return ((double)n_with_descs) / (double)smartlist_len(sl);
-  }
-
-  present = 0.0;
-  SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) {
-    if (node_has_descriptor(node))
-      present += bandwidths[node_sl_idx];
-  } SMARTLIST_FOREACH_END(node);
-
-  tor_free(bandwidths);
-
-  return present / total;
-}
-
-/** Choose a random element of status list <b>sl</b>, weighted by
- * the advertised bandwidth of each node */
-const node_t *
-node_sl_choose_by_bandwidth(const smartlist_t *sl,
-                            bandwidth_weight_rule_t rule)
-{ /*XXXX MOVE */
-  return smartlist_choose_node_by_bandwidth_weights(sl, rule);
-}
-
-/** Return a random running node from the nodelist. Never
- * pick a node that is in
- * <b>excludedsmartlist</b>, or which matches <b>excludedset</b>,
- * even if they are the only nodes available.
- * If <b>CRN_NEED_UPTIME</b> is set in flags and any router has more than
- * a minimum uptime, return one of those.
- * If <b>CRN_NEED_CAPACITY</b> is set in flags, weight your choice by the
- * advertised capacity of each router.
- * If <b>CRN_ALLOW_INVALID</b> is not set in flags, consider only Valid
- * routers.
- * If <b>CRN_NEED_GUARD</b> is set in flags, consider only Guard routers.
- * If <b>CRN_WEIGHT_AS_EXIT</b> is set in flags, we weight bandwidths as if
- * picking an exit node, otherwise we weight bandwidths for picking a relay
- * node (that is, possibly discounting exit nodes).
- * If <b>CRN_NEED_DESC</b> is set in flags, we only consider nodes that
- * have a routerinfo or microdescriptor -- that is, enough info to be
- * used to build a circuit.
- * If <b>CRN_PREF_ADDR</b> is set in flags, we only consider nodes that
- * have an address that is preferred by the ClientPreferIPv6ORPort setting
- * (regardless of this flag, we exclude nodes that aren't allowed by the
- * firewall, including ClientUseIPv4 0 and fascist_firewall_use_ipv6() == 0).
- */
-const node_t *
-router_choose_random_node(smartlist_t *excludedsmartlist,
-                          routerset_t *excludedset,
-                          router_crn_flags_t flags)
-{ /* XXXX MOVE */
-  const int need_uptime = (flags & CRN_NEED_UPTIME) != 0;
-  const int need_capacity = (flags & CRN_NEED_CAPACITY) != 0;
-  const int need_guard = (flags & CRN_NEED_GUARD) != 0;
-  const int allow_invalid = (flags & CRN_ALLOW_INVALID) != 0;
-  const int weight_for_exit = (flags & CRN_WEIGHT_AS_EXIT) != 0;
-  const int need_desc = (flags & CRN_NEED_DESC) != 0;
-  const int pref_addr = (flags & CRN_PREF_ADDR) != 0;
-  const int direct_conn = (flags & CRN_DIRECT_CONN) != 0;
-
-  smartlist_t *sl=smartlist_new(),
-    *excludednodes=smartlist_new();
-  const node_t *choice = NULL;
-  const routerinfo_t *r;
-  bandwidth_weight_rule_t rule;
-
-  tor_assert(!(weight_for_exit && need_guard));
-  rule = weight_for_exit ? WEIGHT_FOR_EXIT :
-    (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID);
-
-  /* Exclude relays that allow single hop exit circuits, if the user
-   * wants to (such relays might be risky) */
-  if (get_options()->ExcludeSingleHopRelays) {
-    SMARTLIST_FOREACH(nodelist_get_list(), node_t *, node,
-      if (node_allows_single_hop_exits(node)) {
-        smartlist_add(excludednodes, node);
-      });
-  }
-
-  /* If the node_t is not found we won't be to exclude ourself but we
-   * won't be able to pick ourself in router_choose_random_node() so
-   * this is fine to at least try with our routerinfo_t object. */
-  if ((r = router_get_my_routerinfo()))
-    routerlist_add_node_and_family(excludednodes, r);
-
-  router_add_running_nodes_to_smartlist(sl, allow_invalid,
-                                        need_uptime, need_capacity,
-                                        need_guard, need_desc, pref_addr,
-                                        direct_conn);
-  log_debug(LD_CIRC,
-           "We found %d running nodes.",
-            smartlist_len(sl));
-
-  smartlist_subtract(sl,excludednodes);
-  log_debug(LD_CIRC,
-            "We removed %d excludednodes, leaving %d nodes.",
-            smartlist_len(excludednodes),
-            smartlist_len(sl));
-
-  if (excludedsmartlist) {
-    smartlist_subtract(sl,excludedsmartlist);
-    log_debug(LD_CIRC,
-              "We removed %d excludedsmartlist, leaving %d nodes.",
-              smartlist_len(excludedsmartlist),
-              smartlist_len(sl));
-  }
-  if (excludedset) {
-    routerset_subtract_nodes(sl,excludedset);
-    log_debug(LD_CIRC,
-              "We removed excludedset, leaving %d nodes.",
-              smartlist_len(sl));
-  }
-
-  // Always weight by bandwidth
-  choice = node_sl_choose_by_bandwidth(sl, rule);
-
-  smartlist_free(sl);
-  if (!choice && (need_uptime || need_capacity || need_guard || pref_addr)) {
-    /* try once more -- recurse but with fewer restrictions. */
-    log_info(LD_CIRC,
-             "We couldn't find any live%s%s%s routers; falling back "
-             "to list of all routers.",
-             need_capacity?", fast":"",
-             need_uptime?", stable":"",
-             need_guard?", guard":"");
-    flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD|
-                CRN_PREF_ADDR);
-    choice = router_choose_random_node(
-                     excludedsmartlist, excludedset, flags);
-  }
-  smartlist_free(excludednodes);
-  if (!choice) {
-    log_warn(LD_CIRC,
-             "No available nodes when trying to choose node. Failing.");
-  }
-  return choice;
-}
-
 /** Helper: given an extended nickname in <b>hexdigest</b> try to decode it.
  * Return 0 on success, -1 on failure.  Store the result into the
  * DIGEST_LEN-byte buffer at <b>digest_out</b>, the single character at
@@ -2958,7 +632,7 @@ hex_digest_nickname_decode(const char *hexdigest,
  * <b>hexdigest</b> is malformed, or it doesn't match.  */
 int
 hex_digest_nickname_matches(const char *hexdigest, const char *identity_digest,
-                            const char *nickname, int is_named)
+                            const char *nickname)
 {
   char digest[DIGEST_LEN];
   char nn_char='\0';
@@ -2967,61 +641,20 @@ hex_digest_nickname_matches(const char *hexdigest, const char *identity_digest,
   if (hex_digest_nickname_decode(hexdigest, digest, &nn_char, nn_buf) == -1)
     return 0;
 
-  if (nn_char == '=' || nn_char == '~') {
-    if (!nickname)
+  if (nn_char == '=') {
+    return 0;
+  }
+
+  if (nn_char == '~') {
+    if (!nickname)     // XXX This seems wrong. -NM
       return 0;
     if (strcasecmp(nn_buf, nickname))
       return 0;
-    if (nn_char == '=' && !is_named)
-      return 0;
   }
 
   return tor_memeq(digest, identity_digest, DIGEST_LEN);
 }
 
-/** Return true iff <b>router</b> is listed as named in the current
- * consensus. */
-int
-router_is_named(const routerinfo_t *router)
-{
-  const char *digest =
-    networkstatus_get_router_digest_by_nickname(router->nickname);
-
-  return (digest &&
-          tor_memeq(digest, router->cache_info.identity_digest, DIGEST_LEN));
-}
-
-/** Return true iff <b>digest</b> is the digest of the identity key of a
- * trusted directory matching at least one bit of <b>type</b>.  If <b>type</b>
- * is zero (NO_DIRINFO), or ALL_DIRINFO, any authority is okay. */
-int
-router_digest_is_trusted_dir_type(const char *digest, dirinfo_type_t type)
-{
-  if (!trusted_dir_servers)
-    return 0;
-  if (authdir_mode(get_options()) && router_digest_is_me(digest))
-    return 1;
-  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ent,
-    if (tor_memeq(digest, ent->digest, DIGEST_LEN)) {
-      return (!type) || ((type & ent->type) != 0);
-    });
-  return 0;
-}
-
-/** Return true iff <b>addr</b> is the address of one of our trusted
- * directory authorities. */
-int
-router_addr_is_trusted_dir(uint32_t addr)
-{
-  if (!trusted_dir_servers)
-    return 0;
-  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ent,
-    if (ent->addr == addr)
-      return 1;
-    );
-  return 0;
-}
-
 /** If hexdigest is correctly formed, base16_decode it into
  * digest, which must have DIGEST_LEN space in it.
  * Return 0 on success, -1 on failure.
@@ -3087,8 +720,8 @@ router_get_by_extrainfo_digest,(const char *digest))
 /** Return the signed descriptor for the extrainfo_t in our routerlist whose
  * extra-info-digest is <b>digest</b>. Return NULL if no such extra-info
  * document is known. */
-signed_descriptor_t *
-extrainfo_get_by_descriptor_digest(const char *digest)
+MOCK_IMPL(signed_descriptor_t *,
+extrainfo_get_by_descriptor_digest,(const char *digest))
 {
   extrainfo_t *ei;
   tor_assert(digest);
@@ -3130,7 +763,7 @@ signed_descriptor_get_body_impl(const signed_descriptor_t *desc,
       log_err(LD_DIR, "We couldn't read a descriptor that is supposedly "
               "mmaped in our cache.  Is another process running in our data "
               "directory?  Exiting.");
-      exit(1);
+      exit(1); // XXXX bad exit: should recover.
     }
   }
   if (!r) /* no mmap, or not in cache. */
@@ -3144,7 +777,7 @@ signed_descriptor_get_body_impl(const signed_descriptor_t *desc,
       log_err(LD_DIR, "descriptor at %p begins with unexpected string %s.  "
               "Is another process running in our data directory?  Exiting.",
               desc, escaped(cp));
-      exit(1);
+      exit(1); // XXXX bad exit: should recover.
     }
   }
 
@@ -3198,7 +831,7 @@ router_get_routerlist(void)
 
 /** Free all storage held by <b>router</b>. */
 void
-routerinfo_free(routerinfo_t *router)
+routerinfo_free_(routerinfo_t *router)
 {
   if (!router)
     return;
@@ -3209,7 +842,7 @@ routerinfo_free(routerinfo_t *router)
   tor_free(router->protocol_list);
   tor_free(router->contact_info);
   if (router->onion_pkey)
-    crypto_pk_free(router->onion_pkey);
+    tor_free(router->onion_pkey);
   tor_free(router->onion_curve25519_pkey);
   if (router->identity_pkey)
     crypto_pk_free(router->identity_pkey);
@@ -3228,7 +861,7 @@ routerinfo_free(routerinfo_t *router)
 
 /** Release all storage held by <b>extrainfo</b> */
 void
-extrainfo_free(extrainfo_t *extrainfo)
+extrainfo_free_(extrainfo_t *extrainfo)
 {
   if (!extrainfo)
     return;
@@ -3240,9 +873,12 @@ extrainfo_free(extrainfo_t *extrainfo)
   tor_free(extrainfo);
 }
 
+#define signed_descriptor_free(val) \
+  FREE_AND_NULL(signed_descriptor_t, signed_descriptor_free_, (val))
+
 /** Release storage held by <b>sd</b>. */
 static void
-signed_descriptor_free(signed_descriptor_t *sd)
+signed_descriptor_free_(signed_descriptor_t *sd)
 {
   if (!sd)
     return;
@@ -3296,21 +932,21 @@ signed_descriptor_from_routerinfo(routerinfo_t *ri)
 
 /** Helper: free the storage held by the extrainfo_t in <b>e</b>. */
 static void
-extrainfo_free_(void *e)
+extrainfo_free_void(void *e)
 {
-  extrainfo_free(e);
+  extrainfo_free_(e);
 }
 
 /** Free all storage held by a routerlist <b>rl</b>. */
 void
-routerlist_free(routerlist_t *rl)
+routerlist_free_(routerlist_t *rl)
 {
   if (!rl)
     return;
   rimap_free(rl->identity_map, NULL);
   sdmap_free(rl->desc_digest_map, NULL);
   sdmap_free(rl->desc_by_eid_map, NULL);
-  eimap_free(rl->extra_info_map, extrainfo_free_);
+  eimap_free(rl->extra_info_map, extrainfo_free_void);
   SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r,
                     routerinfo_free(r));
   SMARTLIST_FOREACH(rl->old_routers, signed_descriptor_t *, sd,
@@ -3318,20 +954,18 @@ routerlist_free(routerlist_t *rl)
   smartlist_free(rl->routers);
   smartlist_free(rl->old_routers);
   if (rl->desc_store.mmap) {
-    int res = tor_munmap_file(routerlist->desc_store.mmap);
+    int res = tor_munmap_file(rl->desc_store.mmap);
     if (res != 0) {
       log_warn(LD_FS, "Failed to munmap routerlist->desc_store.mmap");
     }
   }
   if (rl->extrainfo_store.mmap) {
-    int res = tor_munmap_file(routerlist->extrainfo_store.mmap);
+    int res = tor_munmap_file(rl->extrainfo_store.mmap);
     if (res != 0) {
       log_warn(LD_FS, "Failed to munmap routerlist->extrainfo_store.mmap");
     }
   }
   tor_free(rl);
-
-  router_dir_info_changed();
 }
 
 /** Log information about how much memory is being used for routerlist,
@@ -3349,10 +983,10 @@ dump_routerlist_mem_usage(int severity)
                     olddescs += sd->signed_descriptor_len);
 
   tor_log(severity, LD_DIR,
-      "In %d live descriptors: "U64_FORMAT" bytes.  "
-      "In %d old descriptors: "U64_FORMAT" bytes.",
-      smartlist_len(routerlist->routers), U64_PRINTF_ARG(livedescs),
-      smartlist_len(routerlist->old_routers), U64_PRINTF_ARG(olddescs));
+      "In %d live descriptors: %"PRIu64" bytes.  "
+      "In %d old descriptors: %"PRIu64" bytes.",
+      smartlist_len(routerlist->routers), (livedescs),
+      smartlist_len(routerlist->old_routers), (olddescs));
 }
 
 /** Debugging helper: If <b>idx</b> is nonnegative, assert that <b>ri</b> is
@@ -3790,21 +1424,17 @@ routerlist_reparse_old(routerlist_t *rl, signed_descriptor_t *sd)
 void
 routerlist_free_all(void)
 {
-  routerlist_free(routerlist);
-  routerlist = NULL;
+  routerlist_t *rl = routerlist;
+  routerlist = NULL; // Prevent internals of routerlist_free() from using
+                     // routerlist.
+  routerlist_free(rl);
+  dirlist_free_all();
   if (warned_nicknames) {
     SMARTLIST_FOREACH(warned_nicknames, char *, cp, tor_free(cp));
     smartlist_free(warned_nicknames);
     warned_nicknames = NULL;
   }
-  clear_dir_servers();
-  smartlist_free(trusted_dir_servers);
-  smartlist_free(fallback_dir_servers);
-  trusted_dir_servers = fallback_dir_servers = NULL;
-  if (trusted_dir_certs) {
-    digestmap_free(trusted_dir_certs, cert_list_free_);
-    trusted_dir_certs = NULL;
-  }
+  authcert_free_all();
 }
 
 /** Forget that we have issued any router-related warnings, so that we'll
@@ -3926,7 +1556,7 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg,
                router_describe(router));
       *msg = "Router descriptor is not referenced by any network-status.";
 
-      /* Only journal this desc if we'll be serving it. */
+      /* Only journal this desc if we want to keep old descriptors */
       if (!from_cache && should_cache_old_descriptors())
         signed_desc_append_to_journal(&router->cache_info,
                                       &routerlist->desc_store);
@@ -4100,7 +1730,7 @@ routerlist_remove_old_cached_routers_with_id(time_t now,
     signed_descriptor_t *r = smartlist_get(lst, i);
     tor_assert(tor_memeq(ident, r->identity_digest, DIGEST_LEN));
   }
-#endif
+#endif /* 1 */
   /* Check whether we need to do anything at all. */
   {
     int mdpr = directory_caches_dir_info(get_options()) ? 2 : 1;
@@ -4119,7 +1749,8 @@ routerlist_remove_old_cached_routers_with_id(time_t now,
     signed_descriptor_t *r_next;
     lifespans[i-lo].idx = i;
     if (r->last_listed_as_valid_until >= now ||
-        (retain && digestset_contains(retain, r->signed_descriptor_digest))) {
+        (retain && digestset_probably_contains(retain,
+                                               r->signed_descriptor_digest))) {
       must_keep[i-lo] = 1;
     }
     if (i < hi) {
@@ -4214,7 +1845,7 @@ routerlist_remove_old_routers(void)
       router = smartlist_get(routerlist->routers, i);
       if (router->cache_info.published_on <= cutoff &&
           router->cache_info.last_listed_as_valid_until < now &&
-          !digestset_contains(retain,
+          !digestset_probably_contains(retain,
                           router->cache_info.signed_descriptor_digest)) {
         /* Too old: remove it.  (If we're a cache, just move it into
          * old_routers.) */
@@ -4235,7 +1866,7 @@ routerlist_remove_old_routers(void)
     sd = smartlist_get(routerlist->old_routers, i);
     if (sd->published_on <= cutoff &&
         sd->last_listed_as_valid_until < now &&
-        !digestset_contains(retain, sd->signed_descriptor_digest)) {
+        !digestset_probably_contains(retain, sd->signed_descriptor_digest)) {
       /* Too old.  Remove it. */
       routerlist_remove_old(routerlist, sd, i--);
     }
@@ -4513,7 +2144,7 @@ router_load_extrainfo_from_string(const char *s, const char *eos,
                         ei->cache_info.identity_digest,
                       DIGEST_LEN);
         smartlist_string_remove(requested_fingerprints, fp);
-        /* We silently let people stuff us with extrainfos we didn't ask for,
+        /* We silently let relays stuff us with extrainfos we didn't ask for,
          * so long as we would have wanted them anyway.  Since we always fetch
          * all the extrainfos we want, and we never actually act on them
          * inside Tor, this should be harmless. */
@@ -4556,13 +2187,14 @@ router_load_extrainfo_from_string(const char *s, const char *eos,
   smartlist_free(extrainfo_list);
 }
 
-/** Return true iff any networkstatus includes a descriptor whose digest
- * is that of <b>desc</b>. */
+/** Return true iff the latest ns-flavored consensus includes a descriptor
+ * whose digest is that of <b>desc</b>. */
 static int
 signed_desc_digest_is_recognized(signed_descriptor_t *desc)
 {
   const routerstatus_t *rs;
-  networkstatus_t *consensus = networkstatus_get_latest_consensus();
+  networkstatus_t *consensus = networkstatus_get_latest_consensus_by_flavor(
+                                                                      FLAV_NS);
 
   if (consensus) {
     rs = networkstatus_vote_find_entry(consensus, desc->identity_digest);
@@ -4578,7 +2210,7 @@ signed_desc_digest_is_recognized(signed_descriptor_t *desc)
 void
 update_all_descriptor_downloads(time_t now)
 {
-  if (get_options()->DisableNetwork)
+  if (should_delay_dir_fetches(get_options(), NULL))
     return;
   update_router_descriptor_downloads(now);
   update_microdesc_downloads(now);
@@ -4608,218 +2240,13 @@ router_exit_policy_rejects_all(const routerinfo_t *router)
   return router->policy_is_reject_star;
 }
 
-/** Create an directory server at <b>address</b>:<b>port</b>, with OR identity
- * key <b>digest</b> which has DIGEST_LEN bytes.  If <b>address</b> is NULL,
- * add ourself.  If <b>is_authority</b>, this is a directory authority.  Return
- * the new directory server entry on success or NULL on failure. */
-static dir_server_t *
-dir_server_new(int is_authority,
-               const char *nickname,
-               const tor_addr_t *addr,
-               const char *hostname,
-               uint16_t dir_port, uint16_t or_port,
-               const tor_addr_port_t *addrport_ipv6,
-               const char *digest, const char *v3_auth_digest,
-               dirinfo_type_t type,
-               double weight)
-{
-  dir_server_t *ent;
-  uint32_t a;
-  char *hostname_ = NULL;
-
-  tor_assert(digest);
-
-  if (weight < 0)
-    return NULL;
-
-  if (tor_addr_family(addr) == AF_INET)
-    a = tor_addr_to_ipv4h(addr);
-  else
-    return NULL;
-
-  if (!hostname)
-    hostname_ = tor_addr_to_str_dup(addr);
-  else
-    hostname_ = tor_strdup(hostname);
-
-  ent = tor_malloc_zero(sizeof(dir_server_t));
-  ent->nickname = nickname ? tor_strdup(nickname) : NULL;
-  ent->address = hostname_;
-  ent->addr = a;
-  ent->dir_port = dir_port;
-  ent->or_port = or_port;
-  ent->is_running = 1;
-  ent->is_authority = is_authority;
-  ent->type = type;
-  ent->weight = weight;
-  if (addrport_ipv6) {
-    if (tor_addr_family(&addrport_ipv6->addr) != AF_INET6) {
-      log_warn(LD_BUG, "Hey, I got a non-ipv6 addr as addrport_ipv6.");
-      tor_addr_make_unspec(&ent->ipv6_addr);
-    } else {
-      tor_addr_copy(&ent->ipv6_addr, &addrport_ipv6->addr);
-      ent->ipv6_orport = addrport_ipv6->port;
-    }
-  } else {
-    tor_addr_make_unspec(&ent->ipv6_addr);
-  }
-
-  memcpy(ent->digest, digest, DIGEST_LEN);
-  if (v3_auth_digest && (type & V3_DIRINFO))
-    memcpy(ent->v3_identity_digest, v3_auth_digest, DIGEST_LEN);
-
-  if (nickname)
-    tor_asprintf(&ent->description, "directory server \"%s\" at %s:%d",
-                 nickname, hostname_, (int)dir_port);
-  else
-    tor_asprintf(&ent->description, "directory server at %s:%d",
-                 hostname_, (int)dir_port);
-
-  ent->fake_status.addr = ent->addr;
-  tor_addr_copy(&ent->fake_status.ipv6_addr, &ent->ipv6_addr);
-  memcpy(ent->fake_status.identity_digest, digest, DIGEST_LEN);
-  if (nickname)
-    strlcpy(ent->fake_status.nickname, nickname,
-            sizeof(ent->fake_status.nickname));
-  else
-    ent->fake_status.nickname[0] = '\0';
-  ent->fake_status.dir_port = ent->dir_port;
-  ent->fake_status.or_port = ent->or_port;
-  ent->fake_status.ipv6_orport = ent->ipv6_orport;
-
-  return ent;
-}
-
-/** Create an authoritative directory server at
- * <b>address</b>:<b>port</b>, with identity key <b>digest</b>.  If
- * <b>address</b> is NULL, add ourself.  Return the new trusted directory
- * server entry on success or NULL if we couldn't add it. */
-dir_server_t *
-trusted_dir_server_new(const char *nickname, const char *address,
-                       uint16_t dir_port, uint16_t or_port,
-                       const tor_addr_port_t *ipv6_addrport,
-                       const char *digest, const char *v3_auth_digest,
-                       dirinfo_type_t type, double weight)
-{
-  uint32_t a;
-  tor_addr_t addr;
-  char *hostname=NULL;
-  dir_server_t *result;
-
-  if (!address) { /* The address is us; we should guess. */
-    if (resolve_my_address(LOG_WARN, get_options(),
-                           &a, NULL, &hostname) < 0) {
-      log_warn(LD_CONFIG,
-               "Couldn't find a suitable address when adding ourself as a "
-               "trusted directory server.");
-      return NULL;
-    }
-    if (!hostname)
-      hostname = tor_dup_ip(a);
-  } else {
-    if (tor_lookup_hostname(address, &a)) {
-      log_warn(LD_CONFIG,
-               "Unable to lookup address for directory server at '%s'",
-               address);
-      return NULL;
-    }
-    hostname = tor_strdup(address);
-  }
-  tor_addr_from_ipv4h(&addr, a);
-
-  result = dir_server_new(1, nickname, &addr, hostname,
-                          dir_port, or_port,
-                          ipv6_addrport,
-                          digest,
-                          v3_auth_digest, type, weight);
-  tor_free(hostname);
-  return result;
-}
-
-/** Return a new dir_server_t for a fallback directory server at
- * <b>addr</b>:<b>or_port</b>/<b>dir_port</b>, with identity key digest
- * <b>id_digest</b> */
-dir_server_t *
-fallback_dir_server_new(const tor_addr_t *addr,
-                        uint16_t dir_port, uint16_t or_port,
-                        const tor_addr_port_t *addrport_ipv6,
-                        const char *id_digest, double weight)
-{
-  return dir_server_new(0, NULL, addr, NULL, dir_port, or_port,
-                        addrport_ipv6,
-                        id_digest,
-                        NULL, ALL_DIRINFO, weight);
-}
-
-/** Add a directory server to the global list(s). */
-void
-dir_server_add(dir_server_t *ent)
-{
-  if (!trusted_dir_servers)
-    trusted_dir_servers = smartlist_new();
-  if (!fallback_dir_servers)
-    fallback_dir_servers = smartlist_new();
-
-  if (ent->is_authority)
-    smartlist_add(trusted_dir_servers, ent);
-
-  smartlist_add(fallback_dir_servers, ent);
-  router_dir_info_changed();
-}
-
-/** Free storage held in <b>cert</b>. */
-void
-authority_cert_free(authority_cert_t *cert)
-{
-  if (!cert)
-    return;
-
-  tor_free(cert->cache_info.signed_descriptor_body);
-  crypto_pk_free(cert->signing_key);
-  crypto_pk_free(cert->identity_key);
-
-  tor_free(cert);
-}
-
-/** Free storage held in <b>ds</b>. */
-static void
-dir_server_free(dir_server_t *ds)
-{
-  if (!ds)
-    return;
-
-  tor_free(ds->nickname);
-  tor_free(ds->description);
-  tor_free(ds->address);
-  tor_free(ds);
-}
-
-/** Remove all members from the list of dir servers. */
-void
-clear_dir_servers(void)
-{
-  if (fallback_dir_servers) {
-    SMARTLIST_FOREACH(fallback_dir_servers, dir_server_t *, ent,
-                      dir_server_free(ent));
-    smartlist_clear(fallback_dir_servers);
-  } else {
-    fallback_dir_servers = smartlist_new();
-  }
-  if (trusted_dir_servers) {
-    smartlist_clear(trusted_dir_servers);
-  } else {
-    trusted_dir_servers = smartlist_new();
-  }
-  router_dir_info_changed();
-}
-
 /** For every current directory connection whose purpose is <b>purpose</b>,
  * and where the resource being downloaded begins with <b>prefix</b>, split
  * rest of the resource into base16 fingerprints (or base64 fingerprints if
  * purpose==DIR_PURPOSE_FETCH_MICRODESC), decode them, and set the
  * corresponding elements of <b>result</b> to a nonzero value.
  */
-static void
+void
 list_pending_downloads(digestmap_t *result, digest256map_t *result256,
                        int purpose, const char *prefix)
 {
@@ -4879,49 +2306,15 @@ list_pending_microdesc_downloads(digest256map_t *result)
   list_pending_downloads(NULL, result, DIR_PURPOSE_FETCH_MICRODESC, "d/");
 }
 
-/** For every certificate we are currently downloading by (identity digest,
- * signing key digest) pair, set result[fp_pair] to (void *1).
- */
-static void
-list_pending_fpsk_downloads(fp_pair_map_t *result)
-{
-  const char *pfx = "fp-sk/";
-  smartlist_t *tmp;
-  smartlist_t *conns;
-  const char *resource;
-
-  tor_assert(result);
-
-  tmp = smartlist_new();
-  conns = get_connection_array();
-
-  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
-    if (conn->type == CONN_TYPE_DIR &&
-        conn->purpose == DIR_PURPOSE_FETCH_CERTIFICATE &&
-        !conn->marked_for_close) {
-      resource = TO_DIR_CONN(conn)->requested_resource;
-      if (!strcmpstart(resource, pfx))
-        dir_split_resource_into_fingerprint_pairs(resource + strlen(pfx),
-                                                  tmp);
-    }
-  } SMARTLIST_FOREACH_END(conn);
-
-  SMARTLIST_FOREACH_BEGIN(tmp, fp_pair_t *, fp) {
-    fp_pair_map_set(result, fp, (void*)1);
-    tor_free(fp);
-  } SMARTLIST_FOREACH_END(fp);
-
-  smartlist_free(tmp);
-}
-
 /** Launch downloads for all the descriptors whose digests or digests256
  * are listed as digests[i] for lo <= i < hi.  (Lo and hi may be out of
  * range.)  If <b>source</b> is given, download from <b>source</b>;
  * otherwise, download from an appropriate random directory server.
  */
-MOCK_IMPL(STATIC void, initiate_descriptor_downloads,
-          (const routerstatus_t *source, int purpose, smartlist_t *digests,
-           int lo, int hi, int pds_flags))
+MOCK_IMPL(STATIC void,
+initiate_descriptor_downloads,(const routerstatus_t *source,
+                               int purpose, smartlist_t *digests,
+                               int lo, int hi, int pds_flags))
 {
   char *resource, *cp;
   int digest_len, enc_digest_len;
@@ -4973,10 +2366,11 @@ MOCK_IMPL(STATIC void, initiate_descriptor_downloads,
 
   if (source) {
     /* We know which authority or directory mirror we want. */
-    directory_initiate_command_routerstatus(source, purpose,
-                                            ROUTER_PURPOSE_GENERAL,
-                                            DIRIND_ONEHOP,
-                                            resource, NULL, 0, 0);
+    directory_request_t *req = directory_request_new(purpose);
+    directory_request_set_routerstatus(req, source);
+    directory_request_set_resource(req, resource);
+    directory_initiate_request(req);
+    directory_request_free(req);
   } else {
     directory_get_from_dirserver(purpose, ROUTER_PURPOSE_GENERAL, resource,
                                  pds_flags, DL_WANT_ANY_DIRSERVER);
@@ -5014,8 +2408,9 @@ max_dl_per_request(const or_options_t *options, int purpose)
 }
 
 /** Don't split our requests so finely that we are requesting fewer than
- * this number per server. */
-#define MIN_DL_PER_REQUEST 4
+ * this number per server. (Grouping more than this at once leads to
+ * diminishing returns.) */
+#define MIN_DL_PER_REQUEST 32
 /** To prevent a single screwy cache from confusing us by selective reply,
  * try to split our requests into at least this many requests. */
 #define MIN_REQUESTS 3
@@ -5081,7 +2476,7 @@ launch_descriptor_downloads(int purpose,
     }
   }
 
-  if (!authdir_mode_any_nonhidserv(options)) {
+  if (!authdir_mode(options)) {
     /* If we wind up going to the authorities, we want to only open one
      * connection to each authority at a time, so that we don't overload
      * them.  We do this by setting PDS_NO_EXISTING_SERVERDESC_FETCH
@@ -5103,8 +2498,9 @@ launch_descriptor_downloads(int purpose,
   if (n_per_request > max_dl_per_req)
     n_per_request = max_dl_per_req;
 
-  if (n_per_request < MIN_DL_PER_REQUEST)
-    n_per_request = MIN_DL_PER_REQUEST;
+  if (n_per_request < MIN_DL_PER_REQUEST) {
+    n_per_request = MIN(MIN_DL_PER_REQUEST, n_downloadable);
+  }
 
   if (n_downloadable > n_per_request)
     req_plural = rtr_plural = "s";
@@ -5181,8 +2577,7 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote,
         ++n_inprogress;
         continue; /* We have an in-progress download. */
       }
-      if (!download_status_is_ready(&rs->dl_status, now,
-                          options->TestingDescriptorMaxDownloadTries)) {
+      if (!download_status_is_ready(&rs->dl_status, now)) {
         ++n_delayed; /* Not ready for retry. */
         continue;
       }
@@ -5190,8 +2585,8 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote,
         ++n_would_reject;
         continue; /* We would throw it out immediately. */
       }
-      if (!directory_caches_dir_info(options) &&
-          !client_would_use_router(rs, now, options)) {
+      if (!we_want_to_fetch_flavor(options, consensus->flavor) &&
+          !client_would_use_router(rs, now)) {
         ++n_wouldnt_use;
         continue; /* We would never use it ourself. */
       }
@@ -5212,7 +2607,7 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote,
       smartlist_add(downloadable, rs->descriptor_digest);
   } SMARTLIST_FOREACH_END(rsp);
 
-  if (!authdir_mode_handles_descs(options, ROUTER_PURPOSE_GENERAL)
+  if (!authdir_mode_v3(options)
       && smartlist_len(no_longer_old)) {
     routerlist_t *rl = router_get_routerlist();
     log_info(LD_DIR, "%d router descriptors listed in consensus are "
@@ -5221,15 +2616,30 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote,
     SMARTLIST_FOREACH_BEGIN(no_longer_old, signed_descriptor_t *, sd) {
         const char *msg;
         was_router_added_t r;
+        time_t tmp_cert_expiration_time;
         routerinfo_t *ri = routerlist_reparse_old(rl, sd);
         if (!ri) {
           log_warn(LD_BUG, "Failed to re-parse a router.");
           continue;
         }
+        /* need to remember for below, since add_to_routerlist may free. */
+        tmp_cert_expiration_time = ri->cert_expiration_time;
+
         r = router_add_to_routerlist(ri, &msg, 1, 0);
         if (WRA_WAS_OUTDATED(r)) {
-          log_warn(LD_DIR, "Couldn't add re-parsed router: %s",
+          log_warn(LD_DIR, "Couldn't add re-parsed router: %s. This isn't "
+                   "usually a big deal, but you should make sure that your "
+                   "clock and timezone are set correctly.",
                    msg?msg:"???");
+          if (r == ROUTER_CERTS_EXPIRED) {
+            char time_cons[ISO_TIME_LEN+1];
+            char time_cert_expires[ISO_TIME_LEN+1];
+            format_iso_time(time_cons, consensus->valid_after);
+            format_iso_time(time_cert_expires, tmp_cert_expiration_time);
+            log_warn(LD_DIR, "  (I'm looking at a consensus from %s; This "
+                     "router's certificates began expiring at %s.)",
+                     time_cons, time_cert_expires);
+          }
         }
     } SMARTLIST_FOREACH_END(sd);
     routerlist_assert_ok(rl);
@@ -5343,8 +2753,7 @@ update_extrainfo_downloads(time_t now)
         ++n_have;
         continue;
       }
-      if (!download_status_is_ready(&sd->ei_dl_status, now,
-                          options->TestingDescriptorMaxDownloadTries)) {
+      if (!download_status_is_ready(&sd->ei_dl_status, now)) {
         ++n_delay;
         continue;
       }
@@ -5406,8 +2815,10 @@ update_extrainfo_downloads(time_t now)
   smartlist_free(wanted);
 }
 
-/** Reset the descriptor download failure count on all routers, so that we
- * can retry any long-failed routers immediately.
+/** Reset the consensus and extra-info download failure count on all routers.
+ * When we get a new consensus,
+ * routers_update_status_from_consensus_networkstatus() will reset the
+ * download statuses on the descriptors in that consensus.
  */
 void
 router_reset_descriptor_download_failures(void)
@@ -5419,6 +2830,8 @@ router_reset_descriptor_download_failures(void)
   last_descriptor_download_attempted = 0;
   if (!routerlist)
     return;
+  /* We want to download *all* extra-info descriptors, not just those in
+   * the consensus we currently have (or are about to have) */
   SMARTLIST_FOREACH(routerlist->routers, routerinfo_t *, ri,
   {
     download_status_reset(&ri->cache_info.ei_dl_status);
@@ -5461,7 +2874,8 @@ router_differences_are_cosmetic(const routerinfo_t *r1, const routerinfo_t *r2)
       r1->ipv6_orport != r2->ipv6_orport ||
       r1->dir_port != r2->dir_port ||
       r1->purpose != r2->purpose ||
-      !crypto_pk_eq_keys(r1->onion_pkey, r2->onion_pkey) ||
+      r1->onion_pkey_len != r2->onion_pkey_len ||
+      !tor_memeq(r1->onion_pkey, r2->onion_pkey, r1->onion_pkey_len) ||
       !crypto_pk_eq_keys(r1->identity_pkey, r2->identity_pkey) ||
       strcasecmp(r1->platform, r2->platform) ||
       (r1->contact_info && !r2->contact_info) || /* contact_info is optional */
@@ -5662,11 +3076,11 @@ routerstatus_version_supports_extend2_cells(const routerstatus_t *rs,
     return allow_unknown_versions;
   }
 
-  if (!rs->protocols_known) {
+  if (!rs->pv.protocols_known) {
     return allow_unknown_versions;
   }
 
-  return rs->supports_extend2_cells;
+  return rs->pv.supports_extend2_cells;
 }
 
 /** Assert that the internal representation of <b>rl</b> is
diff --git a/src/or/routerlist.h b/src/feature/nodelist/routerlist.h
index 606e9085ce..5771ebb1ab 100644
--- a/src/or/routerlist.h
+++ b/src/feature/nodelist/routerlist.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,108 +11,83 @@
 #ifndef TOR_ROUTERLIST_H
 #define TOR_ROUTERLIST_H
 
-#include "testsupport.h"
+#include "lib/testsupport/testsupport.h"
+
+/** Return value for router_add_to_routerlist() and dirserv_add_descriptor() */
+typedef enum was_router_added_t {
+  /* Router was added successfully. */
+  ROUTER_ADDED_SUCCESSFULLY = 1,
+  /* Extrainfo document was rejected because no corresponding router
+   * descriptor was found OR router descriptor was rejected because
+   * it was incompatible with its extrainfo document. */
+  ROUTER_BAD_EI = -1,
+  /* Router descriptor was rejected because it is already known. */
+  ROUTER_IS_ALREADY_KNOWN = -2,
+  /* General purpose router was rejected, because it was not listed
+   * in consensus. */
+  ROUTER_NOT_IN_CONSENSUS = -3,
+  /* Router was neither in directory consensus nor in any of
+   * networkstatus documents. Caching it to access later.
+   * (Applies to fetched descriptors only.) */
+  ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS = -4,
+  /* Router was rejected by directory authority. */
+  ROUTER_AUTHDIR_REJECTS = -5,
+  /* Bridge descriptor was rejected because such bridge was not one
+   * of the bridges we have listed in our configuration. */
+  ROUTER_WAS_NOT_WANTED = -6,
+  /* Router descriptor was rejected because it was older than
+   * OLD_ROUTER_DESC_MAX_AGE. */
+  ROUTER_WAS_TOO_OLD = -7, /* note contrast with 'NOT_NEW' */
+  /* DOCDOC */
+  ROUTER_CERTS_EXPIRED = -8
+} was_router_added_t;
+
+/** How long do we avoid using a directory server after it's given us a 503? */
+#define DIR_503_TIMEOUT (60*60)
 
-int get_n_authorities(dirinfo_type_t type);
-int trusted_dirs_reload_certs(void);
-
-/*
- * Pass one of these as source to trusted_dirs_load_certs_from_string()
- * to indicate whence string originates; this controls error handling
- * behavior such as marking downloads as failed.
- */
-
-#define TRUSTED_DIRS_CERTS_SRC_SELF 0
-#define TRUSTED_DIRS_CERTS_SRC_FROM_STORE 1
-#define TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST 2
-#define TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST 3
-#define TRUSTED_DIRS_CERTS_SRC_FROM_VOTE 4
-
-int trusted_dirs_load_certs_from_string(const char *contents, int source,
-                                        int flush, const char *source_dir);
-void trusted_dirs_flush_certs_to_disk(void);
-authority_cert_t *authority_cert_get_newest_by_id(const char *id_digest);
-authority_cert_t *authority_cert_get_by_sk_digest(const char *sk_digest);
-authority_cert_t *authority_cert_get_by_digests(const char *id_digest,
-                                                const char *sk_digest);
-void authority_cert_get_all(smartlist_t *certs_out);
-void authority_cert_dl_failed(const char *id_digest,
-                              const char *signing_key_digest, int status);
-void authority_certs_fetch_missing(networkstatus_t *status, time_t now,
-                                   const char *dir_hint);
 int router_reload_router_list(void);
-int authority_cert_dl_looks_uncertain(const char *id_digest);
-const smartlist_t *router_get_trusted_dir_servers(void);
-const smartlist_t *router_get_fallback_dir_servers(void);
-int authority_cert_is_blacklisted(const authority_cert_t *cert);
 
-const routerstatus_t *router_pick_directory_server(dirinfo_type_t type,
-                                                   int flags);
-dir_server_t *router_get_trusteddirserver_by_digest(const char *d);
-dir_server_t *router_get_fallback_dirserver_by_digest(
-                                                   const char *digest);
-int router_digest_is_fallback_dir(const char *digest);
-MOCK_DECL(dir_server_t *, trusteddirserver_get_by_v3_auth_digest,
-          (const char *d));
-const routerstatus_t *router_pick_trusteddirserver(dirinfo_type_t type,
-                                                   int flags);
-const routerstatus_t *router_pick_fallback_dirserver(dirinfo_type_t type,
-                                                     int flags);
 int router_skip_or_reachability(const or_options_t *options, int try_ip_pref);
-int router_get_my_share_of_directory_requests(double *v3_share_out);
+int router_skip_dir_reachability(const or_options_t *options, int try_ip_pref);
 void router_reset_status_download_failures(void);
 int routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2);
-void router_add_running_nodes_to_smartlist(smartlist_t *sl, int allow_invalid,
-                                           int need_uptime, int need_capacity,
-                                           int need_guard, int need_desc,
-                                           int pref_addr, int direct_conn);
+void router_add_running_nodes_to_smartlist(smartlist_t *sl, int need_uptime,
+                                           int need_capacity, int need_guard,
+                                           int need_desc, int pref_addr,
+                                           int direct_conn);
 
 const routerinfo_t *routerlist_find_my_routerinfo(void);
 uint32_t router_get_advertised_bandwidth(const routerinfo_t *router);
 uint32_t router_get_advertised_bandwidth_capped(const routerinfo_t *router);
 
-const node_t *node_sl_choose_by_bandwidth(const smartlist_t *sl,
-                                          bandwidth_weight_rule_t rule);
-double frac_nodes_with_descriptors(const smartlist_t *sl,
-                                   bandwidth_weight_rule_t rule);
-
-const node_t *router_choose_random_node(smartlist_t *excludedsmartlist,
-                                        struct routerset_t *excludedset,
-                                        router_crn_flags_t flags);
-
-int router_is_named(const routerinfo_t *router);
-int router_digest_is_trusted_dir_type(const char *digest,
-                                      dirinfo_type_t type);
-#define router_digest_is_trusted_dir(d) \
-  router_digest_is_trusted_dir_type((d), NO_DIRINFO)
-
-int router_addr_is_trusted_dir(uint32_t addr);
 int hexdigest_to_digest(const char *hexdigest, char *digest);
 const routerinfo_t *router_get_by_id_digest(const char *digest);
 routerinfo_t *router_get_mutable_by_digest(const char *digest);
 signed_descriptor_t *router_get_by_descriptor_digest(const char *digest);
 MOCK_DECL(signed_descriptor_t *,router_get_by_extrainfo_digest,
           (const char *digest));
-signed_descriptor_t *extrainfo_get_by_descriptor_digest(const char *digest);
+MOCK_DECL(signed_descriptor_t *,extrainfo_get_by_descriptor_digest,
+          (const char *digest));
 const char *signed_descriptor_get_body(const signed_descriptor_t *desc);
 const char *signed_descriptor_get_annotations(const signed_descriptor_t *desc);
 routerlist_t *router_get_routerlist(void);
-void routerinfo_free(routerinfo_t *router);
-void extrainfo_free(extrainfo_t *extrainfo);
-void routerlist_free(routerlist_t *rl);
+void routerinfo_free_(routerinfo_t *router);
+#define routerinfo_free(router) \
+  FREE_AND_NULL(routerinfo_t, routerinfo_free_, (router))
+void extrainfo_free_(extrainfo_t *extrainfo);
+#define extrainfo_free(ei) FREE_AND_NULL(extrainfo_t, extrainfo_free_, (ei))
+void routerlist_free_(routerlist_t *rl);
+#define routerlist_free(rl) FREE_AND_NULL(routerlist_t, routerlist_free_, (rl))
 void dump_routerlist_mem_usage(int severity);
 void routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old,
                        time_t now);
 void routerlist_free_all(void);
 void routerlist_reset_warnings(void);
 
-MOCK_DECL(smartlist_t *, list_authority_ids_with_downloads, (void));
-MOCK_DECL(download_status_t *, id_only_download_status_for_authority_id,
-          (const char *digest));
-MOCK_DECL(smartlist_t *, list_sk_digests_for_authority_id,
-          (const char *digest));
-MOCK_DECL(download_status_t *, download_status_for_authority_id_and_sk,
-    (const char *id_digest, const char *sk_digest));
+/* XXXX move this */
+void list_pending_downloads(digestmap_t *result,
+                            digest256map_t *result256,
+                            int purpose, const char *prefix);
 
 static int WRA_WAS_ADDED(was_router_added_t s);
 static int WRA_WAS_OUTDATED(was_router_added_t s);
@@ -124,7 +99,7 @@ static int WRA_NEVER_DOWNLOADABLE(was_router_added_t s);
  */
 static inline int
 WRA_WAS_ADDED(was_router_added_t s) {
-  return s == ROUTER_ADDED_SUCCESSFULLY || s == ROUTER_ADDED_NOTIFY_GENERATOR;
+  return s == ROUTER_ADDED_SUCCESSFULLY;
 }
 /** Return true iff the outcome code in <b>s</b> indicates that the descriptor
  * was not added because it was either:
@@ -181,19 +156,6 @@ void routerlist_retry_directory_downloads(time_t now);
 
 int router_exit_policy_rejects_all(const routerinfo_t *router);
 
-dir_server_t *trusted_dir_server_new(const char *nickname, const char *address,
-                       uint16_t dir_port, uint16_t or_port,
-                       const tor_addr_port_t *addrport_ipv6,
-                       const char *digest, const char *v3_auth_digest,
-                       dirinfo_type_t type, double weight);
-dir_server_t *fallback_dir_server_new(const tor_addr_t *addr,
-                                      uint16_t dir_port, uint16_t or_port,
-                                      const tor_addr_port_t *addrport_ipv6,
-                                      const char *id_digest, double weight);
-void dir_server_add(dir_server_t *ent);
-
-void authority_cert_free(authority_cert_t *cert);
-void clear_dir_servers(void);
 void update_consensus_router_descriptor_downloads(time_t now, int is_vote,
                                                   networkstatus_t *consensus);
 void update_router_descriptor_downloads(time_t now);
@@ -228,19 +190,9 @@ int hex_digest_nickname_decode(const char *hexdigest,
                                char *nickname_out);
 int hex_digest_nickname_matches(const char *hexdigest,
                                 const char *identity_digest,
-                                const char *nickname, int is_named);
+                                const char *nickname);
 
 #ifdef ROUTERLIST_PRIVATE
-STATIC int choose_array_element_by_weight(const uint64_t *entries,
-                                          int n_entries);
-STATIC void scale_array_elements_to_u64(uint64_t *entries_out,
-                                        const double *entries_in,
-                                        int n_entries,
-                                        uint64_t *total_out);
-STATIC const routerstatus_t *router_pick_directory_server_impl(
-                                           dirinfo_type_t auth, int flags,
-                                           int *n_busy_out);
-
 MOCK_DECL(int, router_descriptor_is_older_than, (const routerinfo_t *router,
                                                  int seconds));
 MOCK_DECL(STATIC was_router_added_t, extrainfo_insert,
@@ -249,10 +201,7 @@ MOCK_DECL(STATIC was_router_added_t, extrainfo_insert,
 MOCK_DECL(STATIC void, initiate_descriptor_downloads,
           (const routerstatus_t *source, int purpose, smartlist_t *digests,
            int lo, int hi, int pds_flags));
-STATIC int router_is_already_dir_fetching(const tor_addr_port_t *ap,
-                                          int serverdesc, int microdesc);
-
-#endif
 
-#endif
+#endif /* defined(ROUTERLIST_PRIVATE) */
 
+#endif /* !defined(TOR_ROUTERLIST_H) */
diff --git a/src/feature/nodelist/routerlist_st.h b/src/feature/nodelist/routerlist_st.h
new file mode 100644
index 0000000000..7446ead3cb
--- /dev/null
+++ b/src/feature/nodelist/routerlist_st.h
@@ -0,0 +1,40 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef ROUTERLIST_ST_H
+#define ROUTERLIST_ST_H
+
+#include "feature/nodelist/desc_store_st.h"
+
+/** Contents of a directory of onion routers. */
+struct routerlist_t {
+  /** Map from server identity digest to a member of routers. */
+  struct digest_ri_map_t *identity_map;
+  /** Map from server descriptor digest to a signed_descriptor_t from
+   * routers or old_routers. */
+  struct digest_sd_map_t *desc_digest_map;
+  /** Map from extra-info digest to an extrainfo_t.  Only exists for
+   * routers in routers or old_routers. */
+  struct digest_ei_map_t *extra_info_map;
+  /** Map from extra-info digests to a signed_descriptor_t for a router
+   * descriptor having that extra-info digest.  Only exists for
+   * routers in routers or old_routers. */
+  struct digest_sd_map_t *desc_by_eid_map;
+  /** List of routerinfo_t for all currently live routers we know. */
+  smartlist_t *routers;
+  /** List of signed_descriptor_t for older router descriptors we're
+   * caching. */
+  smartlist_t *old_routers;
+  /** Store holding server descriptors.  If present, any router whose
+   * cache_info.saved_location == SAVED_IN_CACHE is stored in this file
+   * starting at cache_info.saved_offset */
+  desc_store_t desc_store;
+  /** Store holding extra-info documents. */
+  desc_store_t extrainfo_store;
+};
+
+#endif
+
diff --git a/src/or/routerset.c b/src/feature/nodelist/routerset.c
index 58b66ea777..55e2756959 100644
--- a/src/or/routerset.c
+++ b/src/feature/nodelist/routerset.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
+n * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,13 +27,20 @@
 
 #define ROUTERSET_PRIVATE
 
-#include "or.h"
-#include "geoip.h"
-#include "nodelist.h"
-#include "policies.h"
-#include "router.h"
-#include "routerparse.h"
-#include "routerset.h"
+#include "core/or/or.h"
+#include "core/or/policies.h"
+#include "feature/client/bridges.h"
+#include "feature/dirparse/policy_parse.h"
+#include "feature/nodelist/nickname.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerset.h"
+#include "lib/geoip/geoip.h"
+
+#include "core/or/addr_policy_st.h"
+#include "core/or/extend_info_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
 
 /** Return a new empty routerset. */
 routerset_t *
@@ -262,12 +269,12 @@ routerset_add_unknown_ccs(routerset_t **setp, int only_if_some_cc_set)
     geoip_get_country("A1") >= 0;
 
   if (add_unknown) {
-    smartlist_add(set->country_names, tor_strdup("??"));
-    smartlist_add(set->list, tor_strdup("{??}"));
+    smartlist_add_strdup(set->country_names, "??");
+    smartlist_add_strdup(set->list, "{??}");
   }
   if (add_a1) {
-    smartlist_add(set->country_names, tor_strdup("a1"));
-    smartlist_add(set->list, tor_strdup("{a1}"));
+    smartlist_add_strdup(set->country_names, "a1");
+    smartlist_add_strdup(set->list, "{a1}");
   }
 
   if (add_unknown || add_a1) {
@@ -334,6 +341,18 @@ routerset_contains_node(const routerset_t *set, const node_t *node)
     return 0;
 }
 
+/** Return true iff <b>routerset</b> contains the bridge <b>bridge</b>. */
+int
+routerset_contains_bridge(const routerset_t *set, const bridge_info_t *bridge)
+{
+  const char *id = (const char*)bridge_get_rsa_id_digest(bridge);
+  const tor_addr_port_t *addrport = bridge_get_addr_port(bridge);
+
+  tor_assert(addrport);
+  return routerset_contains(set, &addrport->addr, addrport->port,
+                            NULL, id, -1);
+}
+
 /** Add every known node_t that is a member of <b>routerset</b> to
  * <b>out</b>, but never add any that are part of <b>excludeset</b>.
  * If <b>running_only</b>, only add the running ones. */
@@ -349,7 +368,7 @@ routerset_get_all_nodes(smartlist_t *out, const routerset_t *routerset,
     /* No routers are specified by type; all are given by name or digest.
      * we can do a lookup in O(len(routerset)). */
     SMARTLIST_FOREACH(routerset->list, const char *, name, {
-        const node_t *node = node_get_by_nickname(name, 1);
+        const node_t *node = node_get_by_nickname(name, 0);
         if (node) {
           if (!running_only || node->is_running)
             if (!routerset_contains_node(excludeset, node))
@@ -424,7 +443,7 @@ routerset_equal(const routerset_t *old, const routerset_t *new)
 
 /** Free all storage held in <b>routerset</b>. */
 void
-routerset_free(routerset_t *routerset)
+routerset_free_(routerset_t *routerset)
 {
   if (!routerset)
     return;
@@ -442,4 +461,3 @@ routerset_free(routerset_t *routerset)
   bitarray_free(routerset->countries);
   tor_free(routerset);
 }
-
diff --git a/src/or/routerset.h b/src/feature/nodelist/routerset.h
index c2f7205c3e..ca8b6fed93 100644
--- a/src/or/routerset.h
+++ b/src/feature/nodelist/routerset.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -26,8 +26,11 @@ int routerset_contains_routerstatus(const routerset_t *set,
                                     country_t country);
 int routerset_contains_extendinfo(const routerset_t *set,
                                   const extend_info_t *ei);
-
+struct bridge_info_t;
+int routerset_contains_bridge(const routerset_t *set,
+                              const struct bridge_info_t *bridge);
 int routerset_contains_node(const routerset_t *set, const node_t *node);
+
 void routerset_get_all_nodes(smartlist_t *out, const routerset_t *routerset,
                              const routerset_t *excludeset,
                              int running_only);
@@ -37,10 +40,13 @@ void routerset_subtract_nodes(smartlist_t *out,
 
 char *routerset_to_string(const routerset_t *routerset);
 int routerset_equal(const routerset_t *old, const routerset_t *new);
-void routerset_free(routerset_t *routerset);
+void routerset_free_(routerset_t *routerset);
+#define routerset_free(rs) FREE_AND_NULL(routerset_t, routerset_free_, (rs))
 int routerset_len(const routerset_t *set);
 
 #ifdef ROUTERSET_PRIVATE
+#include "lib/container/bitarray.h"
+
 STATIC char * routerset_get_countryname(const char *c);
 STATIC int routerset_contains(const routerset_t *set, const tor_addr_t *addr,
                    uint16_t orport,
@@ -79,6 +85,5 @@ struct routerset_t {
    * reloaded. */
   bitarray_t *countries;
 };
-#endif
-#endif
-
+#endif /* defined(ROUTERSET_PRIVATE) */
+#endif /* !defined(TOR_ROUTERSET_H) */
diff --git a/src/feature/nodelist/routerstatus_st.h b/src/feature/nodelist/routerstatus_st.h
new file mode 100644
index 0000000000..288edf5982
--- /dev/null
+++ b/src/feature/nodelist/routerstatus_st.h
@@ -0,0 +1,80 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef ROUTERSTATUS_ST_H
+#define ROUTERSTATUS_ST_H
+
+#include "feature/dirclient/download_status_st.h"
+
+/** Contents of a single router entry in a network status object.
+ */
+struct routerstatus_t {
+  time_t published_on; /**< When was this router published? */
+  char nickname[MAX_NICKNAME_LEN+1]; /**< The nickname this router says it
+                                      * has. */
+  char identity_digest[DIGEST_LEN]; /**< Digest of the router's identity
+                                     * key. */
+  /** Digest of the router's most recent descriptor or microdescriptor.
+   * If it's a descriptor, we only use the first DIGEST_LEN bytes. */
+  char descriptor_digest[DIGEST256_LEN];
+  uint32_t addr; /**< IPv4 address for this router, in host order. */
+  uint16_t or_port; /**< IPv4 OR port for this router. */
+  uint16_t dir_port; /**< Directory port for this router. */
+  tor_addr_t ipv6_addr; /**< IPv6 address for this router. */
+  uint16_t ipv6_orport; /**< IPv6 OR port for this router. */
+  unsigned int is_authority:1; /**< True iff this router is an authority. */
+  unsigned int is_exit:1; /**< True iff this router is a good exit. */
+  unsigned int is_stable:1; /**< True iff this router stays up a long time. */
+  unsigned int is_fast:1; /**< True iff this router has good bandwidth. */
+  /** True iff this router is called 'running' in the consensus. We give it
+   * this funny name so that we don't accidentally use this bit as a view of
+   * whether we think the router is *currently* running.  If that's what you
+   * want to know, look at is_running in node_t. */
+  unsigned int is_flagged_running:1;
+  unsigned int is_named:1; /**< True iff "nickname" belongs to this router. */
+  unsigned int is_unnamed:1; /**< True iff "nickname" belongs to another
+                              * router. */
+  unsigned int is_valid:1; /**< True iff this router isn't invalid. */
+  unsigned int is_possible_guard:1; /**< True iff this router would be a good
+                                     * choice as an entry guard. */
+  unsigned int is_bad_exit:1; /**< True iff this node is a bad choice for
+                               * an exit node. */
+  unsigned int is_hs_dir:1; /**< True iff this router is a v2-or-later hidden
+                             * service directory. */
+  unsigned int is_v2_dir:1; /** True iff this router publishes an open DirPort
+                             * or it claims to accept tunnelled dir requests.
+                             */
+
+  unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */
+  unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */
+  unsigned int bw_is_unmeasured:1; /**< This is a consensus entry, with
+                                    * the Unmeasured flag set. */
+
+  /** Flags to summarize the protocol versions for this routerstatus_t. */
+  protover_summary_flags_t pv;
+
+  uint32_t bandwidth_kb; /**< Bandwidth (capacity) of the router as reported in
+                       * the vote/consensus, in kilobytes/sec. */
+
+  /** The consensus has guardfraction information for this router. */
+  unsigned int has_guardfraction:1;
+  /** The guardfraction value of this router. */
+  uint32_t guardfraction_percentage;
+
+  char *exitsummary; /**< exit policy summary -
+                      * XXX weasel: this probably should not stay a string. */
+
+  /* ---- The fields below aren't derived from the networkstatus; they
+   * hold local information only. */
+
+  time_t last_dir_503_at; /**< When did this router last tell us that it
+                           * was too busy to serve directory info? */
+  download_status_t dl_status;
+
+};
+
+#endif
+
diff --git a/src/feature/nodelist/signed_descriptor_st.h b/src/feature/nodelist/signed_descriptor_st.h
new file mode 100644
index 0000000000..bdcebf184a
--- /dev/null
+++ b/src/feature/nodelist/signed_descriptor_st.h
@@ -0,0 +1,61 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef SIGNED_DESCRIPTOR_ST_H
+#define SIGNED_DESCRIPTOR_ST_H
+
+#include "feature/dirclient/download_status_st.h"
+
+/** Information need to cache an onion router's descriptor. */
+struct signed_descriptor_t {
+  /** Pointer to the raw server descriptor, preceded by annotations.  Not
+   * necessarily NUL-terminated.  If saved_location is SAVED_IN_CACHE, this
+   * pointer is null. */
+  char *signed_descriptor_body;
+  /** Length of the annotations preceding the server descriptor. */
+  size_t annotations_len;
+  /** Length of the server descriptor. */
+  size_t signed_descriptor_len;
+  /** Digest of the server descriptor, computed as specified in
+   * dir-spec.txt. */
+  char signed_descriptor_digest[DIGEST_LEN];
+  /** Identity digest of the router. */
+  char identity_digest[DIGEST_LEN];
+  /** Declared publication time of the descriptor. */
+  time_t published_on;
+  /** For routerdescs only: digest of the corresponding extrainfo. */
+  char extra_info_digest[DIGEST_LEN];
+  /** For routerdescs only: A SHA256-digest of the extrainfo (if any) */
+  char extra_info_digest256[DIGEST256_LEN];
+  /** Certificate for ed25519 signing key. */
+  struct tor_cert_st *signing_key_cert;
+  /** For routerdescs only: Status of downloading the corresponding
+   * extrainfo. */
+  download_status_t ei_dl_status;
+  /** Where is the descriptor saved? */
+  saved_location_t saved_location;
+  /** If saved_location is SAVED_IN_CACHE or SAVED_IN_JOURNAL, the offset of
+   * this descriptor in the corresponding file. */
+  off_t saved_offset;
+  /** What position is this descriptor within routerlist->routers or
+   * routerlist->old_routers? -1 for none. */
+  int routerlist_index;
+  /** The valid-until time of the most recent consensus that listed this
+   * descriptor.  0 for "never listed in a consensus, so far as we know." */
+  time_t last_listed_as_valid_until;
+  /* If true, we do not ever try to save this object in the cache. */
+  unsigned int do_not_cache : 1;
+  /* If true, this item is meant to represent an extrainfo. */
+  unsigned int is_extrainfo : 1;
+  /* If true, we got an extrainfo for this item, and the digest was right,
+   * but it was incompatible. */
+  unsigned int extrainfo_is_bogus : 1;
+  /* If true, we are willing to transmit this item unencrypted. */
+  unsigned int send_unencrypted : 1;
+};
+
+#endif
+
diff --git a/src/feature/nodelist/torcert.c b/src/feature/nodelist/torcert.c
new file mode 100644
index 0000000000..b0197e9f13
--- /dev/null
+++ b/src/feature/nodelist/torcert.c
@@ -0,0 +1,764 @@
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file torcert.c
+ *
+ * \brief Implementation for ed25519-signed certificates as used in the Tor
+ * protocol.
+ *
+ * This certificate format is designed to be simple and compact; it's
+ * documented in tor-spec.txt in the torspec.git repository.  All of the
+ * certificates in this format are signed with an Ed25519 key; the
+ * contents themselves may be another Ed25519 key, a digest of a
+ * RSA key, or some other material.
+ *
+ * In this module there is also support for a crooss-certification of
+ * Ed25519 identities using (older) RSA1024 identities.
+ *
+ * Tor uses other types of certificate too, beyond those described in this
+ * module. Notably, our use of TLS requires us to touch X.509 certificates,
+ * even though sensible people would stay away from those. Our X.509
+ * certificates are represented with tor_x509_cert_t, and implemented in
+ * tortls.c.  We also have a separate certificate type that authorities
+ * use to authenticate their RSA signing keys with their RSA identity keys:
+ * that one is authority_cert_t, and it's mostly handled in routerlist.c.
+ */
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "feature/nodelist/torcert.h"
+#include "trunnel/ed25519_cert.h"
+#include "lib/log/log.h"
+#include "trunnel/link_handshake.h"
+#include "lib/tls/tortls.h"
+#include "lib/tls/x509.h"
+
+#include "core/or/or_handshake_certs_st.h"
+
+/** Helper for tor_cert_create(): signs any 32 bytes, not just an ed25519
+ * key.
+ */
+static tor_cert_t *
+tor_cert_sign_impl(const ed25519_keypair_t *signing_key,
+                      uint8_t cert_type,
+                      uint8_t signed_key_type,
+                      const uint8_t signed_key_info[32],
+                      time_t now, time_t lifetime,
+                      uint32_t flags)
+{
+  tor_cert_t *torcert = NULL;
+
+  ed25519_cert_t *cert = ed25519_cert_new();
+  tor_assert(cert); // Unlike Tor's, Trunnel's "new" functions can return NULL.
+  cert->cert_type = cert_type;
+  cert->exp_field = (uint32_t) CEIL_DIV(now + lifetime, 3600);
+  cert->cert_key_type = signed_key_type;
+  memcpy(cert->certified_key, signed_key_info, 32);
+
+  if (flags & CERT_FLAG_INCLUDE_SIGNING_KEY) {
+    ed25519_cert_extension_t *ext = ed25519_cert_extension_new();
+    ext->ext_type = CERTEXT_SIGNED_WITH_KEY;
+    memcpy(ext->un_signing_key, signing_key->pubkey.pubkey, 32);
+    ed25519_cert_add_ext(cert, ext);
+    ++cert->n_extensions;
+  }
+
+  const ssize_t alloc_len = ed25519_cert_encoded_len(cert);
+  tor_assert(alloc_len > 0);
+  uint8_t *encoded = tor_malloc(alloc_len);
+  const ssize_t real_len = ed25519_cert_encode(encoded, alloc_len, cert);
+  if (real_len < 0)
+    goto err;
+  tor_assert(real_len == alloc_len);
+  tor_assert(real_len > ED25519_SIG_LEN);
+  uint8_t *sig = encoded + (real_len - ED25519_SIG_LEN);
+  tor_assert(tor_mem_is_zero((char*)sig, ED25519_SIG_LEN));
+
+  ed25519_signature_t signature;
+  if (ed25519_sign(&signature, encoded,
+                   real_len-ED25519_SIG_LEN, signing_key)<0) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Can't sign certificate");
+    goto err;
+    /* LCOV_EXCL_STOP */
+  }
+  memcpy(sig, signature.sig, ED25519_SIG_LEN);
+
+  torcert = tor_cert_parse(encoded, real_len);
+  if (! torcert) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Generated a certificate we cannot parse");
+    goto err;
+    /* LCOV_EXCL_STOP */
+  }
+
+  if (tor_cert_checksig(torcert, &signing_key->pubkey, now) < 0) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Generated a certificate whose signature we can't "
+             "check: %s", tor_cert_describe_signature_status(torcert));
+    goto err;
+    /* LCOV_EXCL_STOP */
+  }
+
+  tor_free(encoded);
+
+  goto done;
+
+ /* LCOV_EXCL_START */
+ err:
+  tor_cert_free(torcert);
+  torcert = NULL;
+ /* LCOV_EXCL_STOP */
+
+ done:
+  ed25519_cert_free(cert);
+  tor_free(encoded);
+  return torcert;
+}
+
+/**
+ * Create and return a new new certificate of type <b>cert_type</b> to
+ * authenticate <b>signed_key</b> using the key <b>signing_key</b>.  The
+ * certificate should remain valid for at least <b>lifetime</b> seconds after
+ * <b>now</b>.
+ *
+ * If CERT_FLAG_INCLUDE_SIGNING_KEY is set in <b>flags</b>, embed
+ * the public part of <b>signing_key</b> in the certificate.
+ */
+tor_cert_t *
+tor_cert_create(const ed25519_keypair_t *signing_key,
+                uint8_t cert_type,
+                const ed25519_public_key_t *signed_key,
+                time_t now, time_t lifetime,
+                uint32_t flags)
+{
+  return tor_cert_sign_impl(signing_key, cert_type,
+                            SIGNED_KEY_TYPE_ED25519, signed_key->pubkey,
+                            now, lifetime, flags);
+}
+
+/** Release all storage held for <b>cert</b>. */
+void
+tor_cert_free_(tor_cert_t *cert)
+{
+  if (! cert)
+    return;
+
+  if (cert->encoded)
+    memwipe(cert->encoded, 0, cert->encoded_len);
+  tor_free(cert->encoded);
+
+  memwipe(cert, 0, sizeof(tor_cert_t));
+  tor_free(cert);
+}
+
+/** Parse a certificate encoded with <b>len</b> bytes in <b>encoded</b>. */
+tor_cert_t *
+tor_cert_parse(const uint8_t *encoded, const size_t len)
+{
+  tor_cert_t *cert = NULL;
+  ed25519_cert_t *parsed = NULL;
+  ssize_t got_len = ed25519_cert_parse(&parsed, encoded, len);
+  if (got_len < 0 || (size_t) got_len != len)
+    goto err;
+
+  cert = tor_malloc_zero(sizeof(tor_cert_t));
+  cert->encoded = tor_memdup(encoded, len);
+  cert->encoded_len = len;
+
+  memcpy(cert->signed_key.pubkey, parsed->certified_key, 32);
+  int64_t valid_until_64 = ((int64_t)parsed->exp_field) * 3600;
+#if SIZEOF_TIME_T < 8
+  if (valid_until_64 > TIME_MAX)
+    valid_until_64 = TIME_MAX - 1;
+#endif
+  cert->valid_until = (time_t) valid_until_64;
+  cert->cert_type = parsed->cert_type;
+
+  for (unsigned i = 0; i < ed25519_cert_getlen_ext(parsed); ++i) {
+    ed25519_cert_extension_t *ext = ed25519_cert_get_ext(parsed, i);
+    if (ext->ext_type == CERTEXT_SIGNED_WITH_KEY) {
+      if (cert->signing_key_included)
+        goto err;
+
+      cert->signing_key_included = 1;
+      memcpy(cert->signing_key.pubkey, ext->un_signing_key, 32);
+    } else if (ext->ext_flags & CERTEXT_FLAG_AFFECTS_VALIDATION) {
+      /* Unrecognized extension with affects_validation set */
+      goto err;
+    }
+  }
+
+  goto done;
+ err:
+  tor_cert_free(cert);
+  cert = NULL;
+ done:
+  ed25519_cert_free(parsed);
+  return cert;
+}
+
+/** Fill in <b>checkable_out</b> with the information needed to check
+ * the signature on <b>cert</b> with <b>pubkey</b>.
+ *
+ * On success, if <b>expiration_out</b> is provided, and it is some time
+ * _after_ the expiration time of this certificate, set it to the
+ * expiration time of this certificate.
+ */
+int
+tor_cert_get_checkable_sig(ed25519_checkable_t *checkable_out,
+                           const tor_cert_t *cert,
+                           const ed25519_public_key_t *pubkey,
+                           time_t *expiration_out)
+{
+  if (! pubkey) {
+    if (cert->signing_key_included)
+      pubkey = &cert->signing_key;
+    else
+      return -1;
+  }
+
+  checkable_out->msg = cert->encoded;
+  checkable_out->pubkey = pubkey;
+  tor_assert(cert->encoded_len > ED25519_SIG_LEN);
+  const size_t signed_len = cert->encoded_len - ED25519_SIG_LEN;
+  checkable_out->len = signed_len;
+  memcpy(checkable_out->signature.sig,
+         cert->encoded + signed_len, ED25519_SIG_LEN);
+
+  if (expiration_out) {
+    *expiration_out = MIN(*expiration_out, cert->valid_until);
+  }
+
+  return 0;
+}
+
+/** Validates the signature on <b>cert</b> with <b>pubkey</b> relative to the
+ * current time <b>now</b>.  (If <b>now</b> is 0, do not check the expiration
+ * time.) Return 0 on success, -1 on failure.  Sets flags in <b>cert</b> as
+ * appropriate.
+ */
+int
+tor_cert_checksig(tor_cert_t *cert,
+                  const ed25519_public_key_t *pubkey, time_t now)
+{
+  ed25519_checkable_t checkable;
+  int okay;
+  time_t expires = TIME_MAX;
+
+  if (tor_cert_get_checkable_sig(&checkable, cert, pubkey, &expires) < 0)
+    return -1;
+
+  if (now && now > expires) {
+    cert->cert_expired = 1;
+    return -1;
+  }
+
+  if (ed25519_checksig_batch(&okay, &checkable, 1) < 0) {
+    cert->sig_bad = 1;
+    return -1;
+  } else {
+    cert->sig_ok = 1;
+    /* Only copy the checkable public key when it is different from the signing
+     * key of the certificate to avoid undefined behavior. */
+    if (cert->signing_key.pubkey != checkable.pubkey->pubkey) {
+      memcpy(cert->signing_key.pubkey, checkable.pubkey->pubkey, 32);
+    }
+    cert->cert_valid = 1;
+    return 0;
+  }
+}
+
+/** Return a string describing the status of the signature on <b>cert</b>
+ *
+ * Will always be "unchecked" unless tor_cert_checksig has been called.
+ */
+const char *
+tor_cert_describe_signature_status(const tor_cert_t *cert)
+{
+  if (cert->cert_expired) {
+    return "expired";
+  } else if (cert->sig_bad) {
+    return "mis-signed";
+  } else if (cert->sig_ok) {
+    return "okay";
+  } else {
+    return "unchecked";
+  }
+}
+
+/** Return a new copy of <b>cert</b> */
+tor_cert_t *
+tor_cert_dup(const tor_cert_t *cert)
+{
+  tor_cert_t *newcert = tor_memdup(cert, sizeof(tor_cert_t));
+  if (cert->encoded)
+    newcert->encoded = tor_memdup(cert->encoded, cert->encoded_len);
+  return newcert;
+}
+
+/** Return true iff cert1 and cert2 are the same cert. */
+int
+tor_cert_eq(const tor_cert_t *cert1, const tor_cert_t *cert2)
+{
+  tor_assert(cert1);
+  tor_assert(cert2);
+  return cert1->encoded_len == cert2->encoded_len &&
+    tor_memeq(cert1->encoded, cert2->encoded, cert1->encoded_len);
+}
+
+/** Return true iff cert1 and cert2 are the same cert, or if they are both
+ * NULL. */
+int
+tor_cert_opt_eq(const tor_cert_t *cert1, const tor_cert_t *cert2)
+{
+  if (cert1 == NULL && cert2 == NULL)
+    return 1;
+  if (!cert1 || !cert2)
+    return 0;
+  return tor_cert_eq(cert1, cert2);
+}
+
+#define RSA_ED_CROSSCERT_PREFIX "Tor TLS RSA/Ed25519 cross-certificate"
+
+/** Create new cross-certification object to certify <b>ed_key</b> as the
+ * master ed25519 identity key for the RSA identity key <b>rsa_key</b>.
+ * Allocates and stores the encoded certificate in *<b>cert</b>, and returns
+ * the number of bytes stored. Returns negative on error.*/
+ssize_t
+tor_make_rsa_ed25519_crosscert(const ed25519_public_key_t *ed_key,
+                               const crypto_pk_t *rsa_key,
+                               time_t expires,
+                               uint8_t **cert)
+{
+  // It is later than 1985, since otherwise there would be no C89
+  // compilers. (Try to diagnose #22466.)
+  tor_assert_nonfatal(expires >= 15 * 365 * 86400);
+
+  uint8_t *res;
+
+  rsa_ed_crosscert_t *cc = rsa_ed_crosscert_new();
+  memcpy(cc->ed_key, ed_key->pubkey, ED25519_PUBKEY_LEN);
+  cc->expiration = (uint32_t) CEIL_DIV(expires, 3600);
+  cc->sig_len = crypto_pk_keysize(rsa_key);
+  rsa_ed_crosscert_setlen_sig(cc, crypto_pk_keysize(rsa_key));
+
+  ssize_t alloc_sz = rsa_ed_crosscert_encoded_len(cc);
+  tor_assert(alloc_sz > 0);
+  res = tor_malloc_zero(alloc_sz);
+  ssize_t sz = rsa_ed_crosscert_encode(res, alloc_sz, cc);
+  tor_assert(sz > 0 && sz <= alloc_sz);
+
+  crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA256);
+  crypto_digest_add_bytes(d, RSA_ED_CROSSCERT_PREFIX,
+                          strlen(RSA_ED_CROSSCERT_PREFIX));
+
+  const int signed_part_len = 32 + 4;
+  crypto_digest_add_bytes(d, (char*)res, signed_part_len);
+
+  uint8_t digest[DIGEST256_LEN];
+  crypto_digest_get_digest(d, (char*)digest, sizeof(digest));
+  crypto_digest_free(d);
+
+  int siglen = crypto_pk_private_sign(rsa_key,
+                                      (char*)rsa_ed_crosscert_getarray_sig(cc),
+                                      rsa_ed_crosscert_getlen_sig(cc),
+                                      (char*)digest, sizeof(digest));
+  tor_assert(siglen > 0 && siglen <= (int)crypto_pk_keysize(rsa_key));
+  tor_assert(siglen <= UINT8_MAX);
+  cc->sig_len = siglen;
+  rsa_ed_crosscert_setlen_sig(cc, siglen);
+
+  sz = rsa_ed_crosscert_encode(res, alloc_sz, cc);
+  rsa_ed_crosscert_free(cc);
+  *cert = res;
+  return sz;
+}
+
+/**
+ * Check whether the <b>crosscert_len</b> byte certificate in <b>crosscert</b>
+ * is in fact a correct cross-certification of <b>master_key</b> using
+ * the RSA key <b>rsa_id_key</b>.
+ *
+ * Also reject the certificate if it expired before
+ * <b>reject_if_expired_before</b>.
+ *
+ * Return 0 on success, negative on failure.
+ */
+MOCK_IMPL(int,
+rsa_ed25519_crosscert_check, (const uint8_t *crosscert,
+                              const size_t crosscert_len,
+                              const crypto_pk_t *rsa_id_key,
+                              const ed25519_public_key_t *master_key,
+                              const time_t reject_if_expired_before))
+{
+  rsa_ed_crosscert_t *cc = NULL;
+  int rv;
+
+#define ERR(code, s)                                            \
+  do {                                                          \
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,                      \
+           "Received a bad RSA->Ed25519 crosscert: %s",         \
+           (s));                                                \
+    rv = (code);                                                \
+    goto err;                                                   \
+  } while (0)
+
+  if (BUG(crypto_pk_keysize(rsa_id_key) > PK_BYTES))
+    return -1;
+
+  if (BUG(!crosscert))
+    return -1;
+
+  ssize_t parsed_len = rsa_ed_crosscert_parse(&cc, crosscert, crosscert_len);
+  if (parsed_len < 0 || crosscert_len != (size_t)parsed_len) {
+    ERR(-2, "Unparseable or overlong crosscert");
+  }
+
+  if (tor_memneq(rsa_ed_crosscert_getarray_ed_key(cc),
+                 master_key->pubkey,
+                 ED25519_PUBKEY_LEN)) {
+    ERR(-3, "Crosscert did not match Ed25519 key");
+  }
+
+  const uint32_t expiration_date = rsa_ed_crosscert_get_expiration(cc);
+  const uint64_t expiration_time = ((uint64_t)expiration_date) * 3600;
+
+  if (reject_if_expired_before < 0 ||
+      expiration_time < (uint64_t)reject_if_expired_before) {
+    ERR(-4, "Crosscert is expired");
+  }
+
+  const uint8_t *eos = rsa_ed_crosscert_get_end_of_signed(cc);
+  const uint8_t *sig = rsa_ed_crosscert_getarray_sig(cc);
+  const uint8_t siglen = rsa_ed_crosscert_get_sig_len(cc);
+  tor_assert(eos >= crosscert);
+  tor_assert((size_t)(eos - crosscert) <= crosscert_len);
+  tor_assert(siglen == rsa_ed_crosscert_getlen_sig(cc));
+
+  /* Compute the digest */
+  uint8_t digest[DIGEST256_LEN];
+  crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA256);
+  crypto_digest_add_bytes(d, RSA_ED_CROSSCERT_PREFIX,
+                          strlen(RSA_ED_CROSSCERT_PREFIX));
+  crypto_digest_add_bytes(d, (char*)crosscert, eos-crosscert);
+  crypto_digest_get_digest(d, (char*)digest, sizeof(digest));
+  crypto_digest_free(d);
+
+  /* Now check the signature */
+  uint8_t signed_[PK_BYTES];
+  int signed_len = crypto_pk_public_checksig(rsa_id_key,
+                                          (char*)signed_, sizeof(signed_),
+                                          (char*)sig, siglen);
+  if (signed_len < DIGEST256_LEN) {
+    ERR(-5, "Bad signature, or length of signed data not as expected");
+  }
+
+  if (tor_memneq(digest, signed_, DIGEST256_LEN)) {
+    ERR(-6, "The signature was good, but it didn't match the data");
+  }
+
+  rv = 0;
+ err:
+  rsa_ed_crosscert_free(cc);
+  return rv;
+}
+
+/** Construct and return a new empty or_handshake_certs object */
+or_handshake_certs_t *
+or_handshake_certs_new(void)
+{
+  return tor_malloc_zero(sizeof(or_handshake_certs_t));
+}
+
+/** Release all storage held in <b>certs</b> */
+void
+or_handshake_certs_free_(or_handshake_certs_t *certs)
+{
+  if (!certs)
+    return;
+
+  tor_x509_cert_free(certs->auth_cert);
+  tor_x509_cert_free(certs->link_cert);
+  tor_x509_cert_free(certs->id_cert);
+
+  tor_cert_free(certs->ed_id_sign);
+  tor_cert_free(certs->ed_sign_link);
+  tor_cert_free(certs->ed_sign_auth);
+  tor_free(certs->ed_rsa_crosscert);
+
+  memwipe(certs, 0xBD, sizeof(*certs));
+  tor_free(certs);
+}
+
+#undef ERR
+#define ERR(s)                                                  \
+  do {                                                          \
+    log_fn(severity, LD_PROTOCOL,                               \
+           "Received a bad CERTS cell: %s",                     \
+           (s));                                                \
+    return 0;                                                   \
+  } while (0)
+
+int
+or_handshake_certs_rsa_ok(int severity,
+                          or_handshake_certs_t *certs,
+                          tor_tls_t *tls,
+                          time_t now)
+{
+  tor_x509_cert_t *link_cert = certs->link_cert;
+  tor_x509_cert_t *auth_cert = certs->auth_cert;
+  tor_x509_cert_t *id_cert = certs->id_cert;
+
+  if (certs->started_here) {
+    if (! (id_cert && link_cert))
+      ERR("The certs we wanted (ID, Link) were missing");
+    if (! tor_tls_cert_matches_key(tls, link_cert))
+      ERR("The link certificate didn't match the TLS public key");
+    if (! tor_tls_cert_is_valid(severity, link_cert, id_cert, now, 0))
+      ERR("The link certificate was not valid");
+    if (! tor_tls_cert_is_valid(severity, id_cert, id_cert, now, 1))
+      ERR("The ID certificate was not valid");
+  } else {
+    if (! (id_cert && auth_cert))
+      ERR("The certs we wanted (ID, Auth) were missing");
+    if (! tor_tls_cert_is_valid(LOG_PROTOCOL_WARN, auth_cert, id_cert, now, 1))
+      ERR("The authentication certificate was not valid");
+    if (! tor_tls_cert_is_valid(LOG_PROTOCOL_WARN, id_cert, id_cert, now, 1))
+      ERR("The ID certificate was not valid");
+  }
+
+  return 1;
+}
+
+/** Check all the ed25519 certificates in <b>certs</b> against each other, and
+ * against the peer certificate in <b>tls</b> if appropriate.  On success,
+ * return 0; on failure, return a negative value and warn at level
+ * <b>severity</b> */
+int
+or_handshake_certs_ed25519_ok(int severity,
+                              or_handshake_certs_t *certs,
+                              tor_tls_t *tls,
+                              time_t now)
+{
+  ed25519_checkable_t check[10];
+  unsigned n_checkable = 0;
+  time_t expiration = TIME_MAX;
+
+#define ADDCERT(cert, pk)                                               \
+  do {                                                                  \
+    tor_assert(n_checkable < ARRAY_LENGTH(check));                      \
+    if (tor_cert_get_checkable_sig(&check[n_checkable++], cert, pk,     \
+                                   &expiration) < 0)                    \
+      ERR("Could not get checkable cert.");                             \
+  } while (0)
+
+  if (! certs->ed_id_sign || !certs->ed_id_sign->signing_key_included) {
+    ERR("No Ed25519 signing key");
+  }
+  ADDCERT(certs->ed_id_sign, NULL);
+
+  if (certs->started_here) {
+    if (! certs->ed_sign_link)
+      ERR("No Ed25519 link key");
+    {
+      /* check for a match with the TLS cert. */
+      tor_x509_cert_t *peer_cert = tor_tls_get_peer_cert(tls);
+      if (BUG(!peer_cert)) {
+        /* This is a bug, because if we got to this point, we are a connection
+         * that was initiated here, and we completed a TLS handshake. The
+         * other side *must* have given us a certificate! */
+        ERR("No x509 peer cert"); // LCOV_EXCL_LINE
+      }
+      const common_digests_t *peer_cert_digests =
+        tor_x509_cert_get_cert_digests(peer_cert);
+      int okay = tor_memeq(peer_cert_digests->d[DIGEST_SHA256],
+                           certs->ed_sign_link->signed_key.pubkey,
+                           DIGEST256_LEN);
+      tor_x509_cert_free(peer_cert);
+      if (!okay)
+        ERR("Link certificate does not match TLS certificate");
+    }
+
+    ADDCERT(certs->ed_sign_link, &certs->ed_id_sign->signed_key);
+
+  } else {
+    if (! certs->ed_sign_auth)
+      ERR("No Ed25519 link authentication key");
+    ADDCERT(certs->ed_sign_auth, &certs->ed_id_sign->signed_key);
+  }
+
+  if (expiration < now) {
+    ERR("At least one certificate expired.");
+  }
+
+  /* Okay, we've gotten ready to check all the Ed25519 certificates.
+   * Now, we are going to check the RSA certificate's cross-certification
+   * with the ED certificates.
+   *
+   * FFFF In the future, we might want to make this optional.
+   */
+
+  tor_x509_cert_t *rsa_id_cert = certs->id_cert;
+  if (!rsa_id_cert) {
+    ERR("Missing legacy RSA ID certificate");
+  }
+  if (! tor_tls_cert_is_valid(severity, rsa_id_cert, rsa_id_cert, now, 1)) {
+    ERR("The legacy RSA ID certificate was not valid");
+  }
+  if (! certs->ed_rsa_crosscert) {
+    ERR("Missing RSA->Ed25519 crosscert");
+  }
+  crypto_pk_t *rsa_id_key = tor_tls_cert_get_key(rsa_id_cert);
+  if (!rsa_id_key) {
+    ERR("RSA ID cert had no RSA key");
+  }
+
+  if (rsa_ed25519_crosscert_check(certs->ed_rsa_crosscert,
+                                  certs->ed_rsa_crosscert_len,
+                                  rsa_id_key,
+                                  &certs->ed_id_sign->signing_key,
+                                  now) < 0) {
+    crypto_pk_free(rsa_id_key);
+    ERR("Invalid RSA->Ed25519 crosscert");
+  }
+  crypto_pk_free(rsa_id_key);
+  rsa_id_key = NULL;
+
+  /* FFFF We could save a little time in the client case by queueing
+   * this batch to check it later, along with the signature from the
+   * AUTHENTICATE cell. That will change our data flow a bit, though,
+   * so I say "postpone". */
+
+  if (ed25519_checksig_batch(NULL, check, n_checkable) < 0) {
+    ERR("At least one Ed25519 certificate was badly signed");
+  }
+
+  return 1;
+}
+
+/** Check whether an RSA-TAP cross-certification is correct. Return 0 if it
+ * is, -1 if it isn't. */
+MOCK_IMPL(int,
+check_tap_onion_key_crosscert,(const uint8_t *crosscert,
+                               int crosscert_len,
+                               const crypto_pk_t *onion_pkey,
+                               const ed25519_public_key_t *master_id_pkey,
+                               const uint8_t *rsa_id_digest))
+{
+  uint8_t *cc = tor_malloc(crypto_pk_keysize(onion_pkey));
+  int cc_len =
+    crypto_pk_public_checksig(onion_pkey,
+                              (char*)cc,
+                              crypto_pk_keysize(onion_pkey),
+                              (const char*)crosscert,
+                              crosscert_len);
+  if (cc_len < 0) {
+    goto err;
+  }
+  if (cc_len < DIGEST_LEN + ED25519_PUBKEY_LEN) {
+    log_warn(LD_DIR, "Short signature on cross-certification with TAP key");
+    goto err;
+  }
+  if (tor_memneq(cc, rsa_id_digest, DIGEST_LEN) ||
+      tor_memneq(cc + DIGEST_LEN, master_id_pkey->pubkey,
+                 ED25519_PUBKEY_LEN)) {
+    log_warn(LD_DIR, "Incorrect cross-certification with TAP key");
+    goto err;
+  }
+
+  tor_free(cc);
+  return 0;
+ err:
+  tor_free(cc);
+  return -1;
+}
+
+/**
+ * Check the Ed certificates and/or the RSA certificates, as appropriate.  If
+ * we obtained an Ed25519 identity, set *ed_id_out. If we obtained an RSA
+ * identity, set *rs_id_out. Otherwise, set them both to NULL.
+ */
+void
+or_handshake_certs_check_both(int severity,
+                              or_handshake_certs_t *certs,
+                              tor_tls_t *tls,
+                              time_t now,
+                              const ed25519_public_key_t **ed_id_out,
+                              const common_digests_t **rsa_id_out)
+{
+  tor_assert(ed_id_out);
+  tor_assert(rsa_id_out);
+
+  *ed_id_out = NULL;
+  *rsa_id_out = NULL;
+
+  if (certs->ed_id_sign) {
+    if (or_handshake_certs_ed25519_ok(severity, certs, tls, now)) {
+      tor_assert(certs->ed_id_sign);
+      tor_assert(certs->id_cert);
+
+      *ed_id_out = &certs->ed_id_sign->signing_key;
+      *rsa_id_out = tor_x509_cert_get_id_digests(certs->id_cert);
+
+      /* If we reached this point, we did not look at any of the
+       * subsidiary RSA certificates, so we'd better just remove them.
+       */
+      tor_x509_cert_free(certs->link_cert);
+      tor_x509_cert_free(certs->auth_cert);
+      certs->link_cert = certs->auth_cert = NULL;
+    }
+    /* We do _not_ fall through here.  If you provided us Ed25519
+     * certificates, we expect to verify them! */
+  } else {
+    /* No ed25519 keys given in the CERTS cell */
+    if (or_handshake_certs_rsa_ok(severity, certs, tls, now)) {
+      *rsa_id_out = tor_x509_cert_get_id_digests(certs->id_cert);
+    }
+  }
+}
+
+/* === ENCODING === */
+
+/* Encode the ed25519 certificate <b>cert</b> and put the newly allocated
+ * string in <b>cert_str_out</b>. Return 0 on success else a negative value. */
+int
+tor_cert_encode_ed22519(const tor_cert_t *cert, char **cert_str_out)
+{
+  int ret = -1;
+  char *ed_cert_b64 = NULL;
+  size_t ed_cert_b64_len;
+
+  tor_assert(cert);
+  tor_assert(cert_str_out);
+
+  /* Get the encoded size and add the NUL byte. */
+  ed_cert_b64_len = base64_encode_size(cert->encoded_len,
+                                       BASE64_ENCODE_MULTILINE) + 1;
+  ed_cert_b64 = tor_malloc_zero(ed_cert_b64_len);
+
+  /* Base64 encode the encoded certificate. */
+  if (base64_encode(ed_cert_b64, ed_cert_b64_len,
+                    (const char *) cert->encoded, cert->encoded_len,
+                    BASE64_ENCODE_MULTILINE) < 0) {
+    /* LCOV_EXCL_START */
+    log_err(LD_BUG, "Couldn't base64-encode ed22519 cert!");
+    goto err;
+    /* LCOV_EXCL_STOP */
+  }
+
+  /* Put everything together in a NUL terminated string. */
+  tor_asprintf(cert_str_out,
+               "-----BEGIN ED25519 CERT-----\n"
+               "%s"
+               "-----END ED25519 CERT-----",
+               ed_cert_b64);
+  /* Success! */
+  ret = 0;
+
+ err:
+  tor_free(ed_cert_b64);
+  return ret;
+}
diff --git a/src/feature/nodelist/torcert.h b/src/feature/nodelist/torcert.h
new file mode 100644
index 0000000000..492275b514
--- /dev/null
+++ b/src/feature/nodelist/torcert.h
@@ -0,0 +1,116 @@
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TORCERT_H_INCLUDED
+#define TORCERT_H_INCLUDED
+
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+#define SIGNED_KEY_TYPE_ED25519     0x01
+
+#define CERT_TYPE_ID_SIGNING        0x04
+#define CERT_TYPE_SIGNING_LINK      0x05
+#define CERT_TYPE_SIGNING_AUTH      0x06
+#define CERT_TYPE_SIGNING_HS_DESC   0x08
+#define CERT_TYPE_AUTH_HS_IP_KEY    0x09
+#define CERT_TYPE_ONION_ID          0x0A
+#define CERT_TYPE_CROSS_HS_IP_KEYS  0x0B
+
+#define CERT_FLAG_INCLUDE_SIGNING_KEY 0x1
+
+/** An ed25519-signed certificate as used throughout the Tor protocol.
+ **/
+typedef struct tor_cert_st {
+  /** The key authenticated by this certificate */
+  ed25519_public_key_t signed_key;
+  /** The key that signed this certificate. This value may be unset if the
+   * certificate has never been checked, and didn't include its own key. */
+  ed25519_public_key_t signing_key;
+  /** A time after which this certificate will no longer be valid. */
+  time_t valid_until;
+
+  /** The encoded representation of this certificate */
+  uint8_t *encoded;
+  /** The length of <b>encoded</b> */
+  size_t encoded_len;
+
+  /** One of CERT_TYPE_... */
+  uint8_t cert_type;
+  /** True iff we received a signing key embedded in this certificate */
+  unsigned signing_key_included : 1;
+  /** True iff we checked the signature and found it bad */
+  unsigned sig_bad : 1;
+  /** True iff we checked the signature and found it correct */
+  unsigned sig_ok : 1;
+  /** True iff we checked the signature and first found that the cert
+   * had expired */
+  unsigned cert_expired : 1;
+  /** True iff we checked the signature and found the whole cert valid */
+  unsigned cert_valid : 1;
+} tor_cert_t;
+
+struct tor_tls_t;
+
+tor_cert_t *tor_cert_create(const ed25519_keypair_t *signing_key,
+                            uint8_t cert_type,
+                            const ed25519_public_key_t *signed_key,
+                            time_t now, time_t lifetime,
+                            uint32_t flags);
+
+tor_cert_t *tor_cert_parse(const uint8_t *cert, size_t certlen);
+
+void tor_cert_free_(tor_cert_t *cert);
+#define tor_cert_free(cert) FREE_AND_NULL(tor_cert_t, tor_cert_free_, (cert))
+
+int tor_cert_get_checkable_sig(ed25519_checkable_t *checkable_out,
+                               const tor_cert_t *out,
+                               const ed25519_public_key_t *pubkey,
+                               time_t *expiration_out);
+
+int tor_cert_checksig(tor_cert_t *cert,
+                      const ed25519_public_key_t *pubkey, time_t now);
+const char *tor_cert_describe_signature_status(const tor_cert_t *cert);
+
+tor_cert_t *tor_cert_dup(const tor_cert_t *cert);
+int tor_cert_eq(const tor_cert_t *cert1, const tor_cert_t *cert2);
+int tor_cert_opt_eq(const tor_cert_t *cert1, const tor_cert_t *cert2);
+
+ssize_t tor_make_rsa_ed25519_crosscert(const ed25519_public_key_t *ed_key,
+                                       const crypto_pk_t *rsa_key,
+                                       time_t expires,
+                                       uint8_t **cert);
+MOCK_DECL(int,
+rsa_ed25519_crosscert_check, (const uint8_t *crosscert,
+                              const size_t crosscert_len,
+                              const crypto_pk_t *rsa_id_key,
+                              const ed25519_public_key_t *master_key,
+                              const time_t reject_if_expired_before));
+
+or_handshake_certs_t *or_handshake_certs_new(void);
+void or_handshake_certs_free_(or_handshake_certs_t *certs);
+#define or_handshake_certs_free(certs) \
+  FREE_AND_NULL(or_handshake_certs_t, or_handshake_certs_free_, (certs))
+int or_handshake_certs_rsa_ok(int severity,
+                              or_handshake_certs_t *certs,
+                              struct tor_tls_t *tls,
+                              time_t now);
+int or_handshake_certs_ed25519_ok(int severity,
+                                  or_handshake_certs_t *certs,
+                                  struct tor_tls_t *tls,
+                                  time_t now);
+void or_handshake_certs_check_both(int severity,
+                              or_handshake_certs_t *certs,
+                              struct tor_tls_t *tls,
+                              time_t now,
+                              const ed25519_public_key_t **ed_id_out,
+                              const common_digests_t **rsa_id_out);
+
+int tor_cert_encode_ed22519(const tor_cert_t *cert, char **cert_str_out);
+
+MOCK_DECL(int, check_tap_onion_key_crosscert,(const uint8_t *crosscert,
+                                  int crosscert_len,
+                                  const crypto_pk_t *onion_pkey,
+                                  const ed25519_public_key_t *master_id_pkey,
+                                  const uint8_t *rsa_id_digest));
+
+#endif /* !defined(TORCERT_H_INCLUDED) */
diff --git a/src/feature/nodelist/vote_routerstatus_st.h b/src/feature/nodelist/vote_routerstatus_st.h
new file mode 100644
index 0000000000..366754c166
--- /dev/null
+++ b/src/feature/nodelist/vote_routerstatus_st.h
@@ -0,0 +1,41 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef VOTE_ROUTERSTATUS_ST_H
+#define VOTE_ROUTERSTATUS_ST_H
+
+#include "feature/nodelist/routerstatus_st.h"
+#include "lib/defs/x25519_sizes.h"
+
+/** The claim about a single router, made in a vote. */
+struct vote_routerstatus_t {
+  routerstatus_t status; /**< Underlying 'status' object for this router.
+                          * Flags are redundant. */
+  /** How many known-flags are allowed in a vote? This is the width of
+   * the flags field of vote_routerstatus_t */
+#define MAX_KNOWN_FLAGS_IN_VOTE 64
+  uint64_t flags; /**< Bit-field for all recognized flags; index into
+                   * networkstatus_t.known_flags. */
+  char *version; /**< The version that the authority says this router is
+                  * running. */
+  char *protocols; /**< The protocols that this authority says this router
+                    * provides. */
+  unsigned int has_measured_bw:1; /**< The vote had a measured bw */
+  /** True iff the vote included an entry for ed25519 ID, or included
+   * "id ed25519 none" to indicate that there was no ed25519 ID. */
+  unsigned int has_ed25519_listing:1;
+  /** True if the Ed25519 listing here is the consensus-opinion for the
+   * Ed25519 listing; false if there was no consensus on Ed25519 key status,
+   * or if this VRS doesn't reflect it. */
+  unsigned int ed25519_reflects_consensus:1;
+  uint32_t measured_bw_kb; /**< Measured bandwidth (capacity) of the router */
+  /** The hash or hashes that the authority claims this microdesc has. */
+  vote_microdesc_hash_t *microdesc;
+  /** Ed25519 identity for this router, or zero if it has none. */
+  uint8_t ed25519_id[ED25519_PUBKEY_LEN];
+};
+
+#endif
diff --git a/src/or/dns.c b/src/feature/relay/dns.c
index 33628373d5..cc9f4cf490 100644
--- a/src/or/dns.c
+++ b/src/feature/relay/dns.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,7 +19,7 @@
  *      dns_seems_to_be_broken().
  *    <li>When a client has asked the relay, in a RELAY_BEGIN cell, to connect
  *      to a given server by hostname.  This happens via dns_resolve().
- *    <li>When a client has asked the rela, in a RELAY_RESOLVE cell, to look
+ *    <li>When a client has asked the relay, in a RELAY_RESOLVE cell, to look
  *      up a given server's IP address(es) by hostname. This also happens via
  *      dns_resolve().
  *   </ol>
@@ -49,20 +49,33 @@
 
 #define DNS_PRIVATE
 
-#include "or.h"
-#include "circuitlist.h"
-#include "circuituse.h"
-#include "config.h"
-#include "connection.h"
-#include "connection_edge.h"
-#include "control.h"
-#include "dns.h"
-#include "main.h"
-#include "policies.h"
-#include "relay.h"
-#include "router.h"
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/connection_edge.h"
+#include "core/or/policies.h"
+#include "core/or/relay.h"
+#include "feature/control/control.h"
+#include "feature/relay/dns.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/sandbox/sandbox.h"
+
+#include "core/or/edge_connection_st.h"
+#include "core/or/or_circuit_st.h"
+
 #include "ht.h"
-#include "sandbox.h"
+
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
 #include <event2/event.h>
 #include <event2/dns.h>
 
@@ -102,7 +115,7 @@ static void assert_cache_ok_(void);
 #define assert_cache_ok() assert_cache_ok_()
 #else
 #define assert_cache_ok() STMT_NIL
-#endif
+#endif /* defined(DEBUG_DNS_CACHE) */
 static void assert_resolve_ok(cached_resolve_t *resolve);
 
 /** Hash table of cached_resolve objects. */
@@ -160,8 +173,9 @@ evdns_log_cb(int warn, const char *msg)
   }
   if (!strcmpstart(msg, "Nameserver ") && (cp=strstr(msg, " has failed: "))) {
     char *ns = tor_strndup(msg+11, cp-(msg+11));
-    const char *err = strchr(cp, ':')+2;
-    tor_assert(err);
+    const char *colon = strchr(cp, ':');
+    tor_assert(colon);
+    const char *err = colon+2;
     /* Don't warn about a single failed nameserver; we'll warn with 'all
      * nameservers have failed' if we're completely out of nameservers;
      * otherwise, the situation is tolerable. */
@@ -181,6 +195,18 @@ evdns_log_cb(int warn, const char *msg)
   } else if (!strcmp(msg, "All nameservers have failed")) {
     control_event_server_status(LOG_WARN, "NAMESERVER_ALL_DOWN");
     all_down = 1;
+  } else if (!strcmpstart(msg, "Address mismatch on received DNS")) {
+    static ratelim_t mismatch_limit = RATELIM_INIT(3600);
+    const char *src = strstr(msg, " Apparent source");
+    if (!src || get_options()->SafeLogging) {
+      src = "";
+    }
+    log_fn_ratelim(&mismatch_limit, severity, LD_EXIT,
+                   "eventdns: Received a DNS packet from "
+                   "an IP address to which we did not send a request. This "
+                   "could be a DNS spoofing attempt, or some kind of "
+                   "misconfiguration.%s", src);
+    return;
   }
   tor_log(severity, LD_EXIT, "eventdns: %s", msg);
 }
@@ -365,7 +391,7 @@ set_expiry(cached_resolve_t *resolve, time_t expires)
   resolve->expire = expires;
   smartlist_pqueue_add(cached_resolve_pqueue,
                        compare_cached_resolves_by_expiry_,
-                       STRUCT_OFFSET(cached_resolve_t, minheap_idx),
+                       offsetof(cached_resolve_t, minheap_idx),
                        resolve);
 }
 
@@ -412,7 +438,7 @@ purge_expired_resolves(time_t now)
       break;
     smartlist_pqueue_pop(cached_resolve_pqueue,
                          compare_cached_resolves_by_expiry_,
-                         STRUCT_OFFSET(cached_resolve_t, minheap_idx));
+                         offsetof(cached_resolve_t, minheap_idx));
 
     if (resolve->state == CACHE_STATE_PENDING) {
       log_debug(LD_EXIT,
@@ -444,7 +470,7 @@ purge_expired_resolves(time_t now)
         if (!pendconn->base_.marked_for_close) {
           connection_edge_end(pendconn, END_STREAM_REASON_TIMEOUT);
           circuit_detach_stream(circuit_get_by_edge_conn(pendconn), pendconn);
-          connection_free(TO_CONN(pendconn));
+          connection_free_(TO_CONN(pendconn));
         }
         tor_free(pend);
       }
@@ -560,8 +586,11 @@ send_resolved_hostname_cell,(edge_connection_t *conn,
   char buf[RELAY_PAYLOAD_SIZE];
   size_t buflen;
   uint32_t ttl;
+
+  if (BUG(!hostname))
+    return;
+
   size_t namelen = strlen(hostname);
-  tor_assert(hostname);
 
   tor_assert(namelen < 256);
   ttl = dns_clip_ttl(conn->address_ttl);
@@ -657,7 +686,7 @@ dns_resolve(edge_connection_t *exitconn)
         /* If we made the connection pending, then we freed it already in
          * dns_cancel_pending_resolve().  If we marked it for close, it'll
          * get freed from the main loop.  Otherwise, can free it now. */
-        connection_free(TO_CONN(exitconn));
+        connection_free_(TO_CONN(exitconn));
       }
       break;
     default:
@@ -948,14 +977,14 @@ assert_connection_edge_not_dns_pending(edge_connection_t *conn)
   for (pend = resolve->pending_connections; pend; pend = pend->next) {
     tor_assert(pend->conn != conn);
   }
-#else
+#else /* !(1) */
   cached_resolve_t **resolve;
   HT_FOREACH(resolve, cache_map, &cache_root) {
     for (pend = (*resolve)->pending_connections; pend; pend = pend->next) {
       tor_assert(pend->conn != conn);
     }
   }
-#endif
+#endif /* 1 */
 }
 
 /** Log an error and abort if any connection waiting for a DNS resolve is
@@ -1088,7 +1117,7 @@ dns_cancel_pending_resolve,(const char *address))
     if (circ)
       circuit_detach_stream(circ, pendconn);
     if (!pendconn->base_.marked_for_close)
-      connection_free(TO_CONN(pendconn));
+      connection_free_(TO_CONN(pendconn));
     resolve->pending_connections = pend->next;
     tor_free(pend);
   }
@@ -1217,7 +1246,7 @@ inform_pending_connections(cached_resolve_t *resolve)
         /* This detach must happen after we send the resolved cell. */
         circuit_detach_stream(circuit_get_by_edge_conn(pendconn), pendconn);
       }
-      connection_free(TO_CONN(pendconn));
+      connection_free_(TO_CONN(pendconn));
     } else {
       circuit_t *circ;
       if (pendconn->base_.purpose == EXIT_PURPOSE_CONNECT) {
@@ -1246,7 +1275,7 @@ inform_pending_connections(cached_resolve_t *resolve)
         circ = circuit_get_by_edge_conn(pendconn);
         tor_assert(circ);
         circuit_detach_stream(circ, pendconn);
-        connection_free(TO_CONN(pendconn));
+        connection_free_(TO_CONN(pendconn));
       }
     }
     resolve->pending_connections = pend->next;
@@ -1383,7 +1412,7 @@ configure_nameservers(int force)
       evdns_base_load_hosts(the_evdns_base,
           sandbox_intern_string("/etc/hosts"));
     }
-#endif
+#endif /* defined(DNS_OPTION_HOSTSFILE) && defined(USE_LIBSECCOMP) */
     log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname);
     if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags,
         sandbox_intern_string(conf_fname)))) {
@@ -1421,7 +1450,7 @@ configure_nameservers(int force)
     tor_free(resolv_conf_fname);
     resolv_conf_mtime = 0;
   }
-#endif
+#endif /* defined(_WIN32) */
 
 #define SET(k,v)  evdns_base_set_option(the_evdns_base, (k), (v))
 
@@ -1568,10 +1597,11 @@ evdns_callback(int result, char type, int count, int ttl, void *addresses,
                 escaped_safe_str(hostname));
       tor_free(escaped_address);
     } else if (count) {
-      log_warn(LD_EXIT, "eventdns returned only non-IPv4 answers for %s.",
+      log_info(LD_EXIT, "eventdns returned only unrecognized answer types "
+               " for %s.",
                escaped_safe_str(string_address));
     } else {
-      log_warn(LD_BUG, "eventdns returned no addresses or error for %s!",
+      log_info(LD_EXIT, "eventdns returned no addresses or error for %s.",
                escaped_safe_str(string_address));
     }
   }
@@ -1655,7 +1685,7 @@ launch_resolve,(cached_resolve_t *resolve))
   tor_addr_t a;
   int r;
 
-  if (get_options()->DisableNetwork)
+  if (net_is_disabled())
     return -1;
 
   /* What? Nameservers not configured?  Sounds like a bug. */
@@ -1762,7 +1792,7 @@ wildcard_increment_answer(const char *id)
         "invalid addresses. Apparently they are hijacking DNS failures. "
         "I'll try to correct for this by treating future occurrences of "
         "\"%s\" as 'not found'.", id, *ip, id);
-      smartlist_add(dns_wildcard_list, tor_strdup(id));
+      smartlist_add_strdup(dns_wildcard_list, id);
     }
     if (!dns_wildcard_notice_given)
       control_event_server_status(LOG_NOTICE, "DNS_HIJACKED");
@@ -1786,7 +1816,7 @@ add_wildcarded_test_address(const char *address)
   n_test_addrs = get_options()->ServerDNSTestAddresses ?
     smartlist_len(get_options()->ServerDNSTestAddresses) : 0;
 
-  smartlist_add(dns_wildcarded_test_address_list, tor_strdup(address));
+  smartlist_add_strdup(dns_wildcarded_test_address_list, address);
   n = smartlist_len(dns_wildcarded_test_address_list);
   if (n > n_test_addrs/2) {
     tor_log(dns_wildcarded_test_address_notice_given ? LOG_INFO : LOG_NOTICE,
@@ -1890,7 +1920,7 @@ launch_test_addresses(evutil_socket_t fd, short event, void *args)
   (void)event;
   (void)args;
 
-  if (options->DisableNetwork)
+  if (net_is_disabled())
     return;
 
   log_info(LD_EXIT, "Launching checks to see whether our nameservers like to "
@@ -1947,7 +1977,7 @@ dns_launch_wildcard_checks(void)
       launch_wildcard_check(8, 16, ipv6, ".com");
       launch_wildcard_check(8, 16, ipv6, ".org");
       launch_wildcard_check(8, 16, ipv6, ".net");
-  }
+    }
   }
 }
 
@@ -2040,7 +2070,7 @@ assert_resolve_ok(cached_resolve_t *resolve)
       tor_assert(!resolve->hostname);
     else
       tor_assert(!resolve->result_ipv4.addr_ipv4);
-#endif
+#endif /* 0 */
     /*XXXXX ADD MORE */
   }
 }
@@ -2052,14 +2082,21 @@ dns_cache_entry_count(void)
    return HT_SIZE(&cache_root);
 }
 
+/* Return the total size in bytes of the DNS cache. */
+size_t
+dns_cache_total_allocation(void)
+{
+  return sizeof(struct cached_resolve_t) * dns_cache_entry_count() +
+         HT_MEM_USAGE(&cache_root);
+}
+
 /** Log memory information about our internal DNS cache at level 'severity'. */
 void
 dump_dns_mem_usage(int severity)
 {
   /* This should never be larger than INT_MAX. */
   int hash_count = dns_cache_entry_count();
-  size_t hash_mem = sizeof(struct cached_resolve_t) * hash_count;
-  hash_mem += HT_MEM_USAGE(&cache_root);
+  size_t hash_mem = dns_cache_total_allocation();
 
   /* Print out the count and estimated size of our &cache_root.  It undercounts
      hostnames in cached reverse resolves.
@@ -2069,6 +2106,37 @@ dump_dns_mem_usage(int severity)
       (unsigned)hash_mem);
 }
 
+/* Do a round of OOM cleanup on all DNS entries. Return the amount of removed
+ * bytes. It is possible that the returned value is lower than min_remove_bytes
+ * if the caches get emptied out so the caller should be aware of this. */
+size_t
+dns_cache_handle_oom(time_t now, size_t min_remove_bytes)
+{
+  time_t time_inc = 0;
+  size_t total_bytes_removed = 0;
+  size_t current_size = dns_cache_total_allocation();
+
+  do {
+    /* If no DNS entries left, break loop. */
+    if (!dns_cache_entry_count())
+      break;
+
+    /* Get cutoff interval and remove entries. */
+    time_t cutoff = now + time_inc;
+    purge_expired_resolves(cutoff);
+
+    /* Update amount of bytes removed and array size. */
+    size_t bytes_removed = current_size - dns_cache_total_allocation();
+    current_size -= bytes_removed;
+    total_bytes_removed += bytes_removed;
+
+    /* Increase time_inc by a reasonable fraction. */
+    time_inc += (MAX_DNS_TTL_AT_EXIT / 4);
+  } while (total_bytes_removed < min_remove_bytes);
+
+  return total_bytes_removed;
+}
+
 #ifdef DEBUG_DNS_CACHE
 /** Exit with an assertion if the DNS cache is corrupt. */
 static void
@@ -2090,7 +2158,7 @@ assert_cache_ok_(void)
 
   smartlist_pqueue_assert_ok(cached_resolve_pqueue,
                              compare_cached_resolves_by_expiry_,
-                             STRUCT_OFFSET(cached_resolve_t, minheap_idx));
+                             offsetof(cached_resolve_t, minheap_idx));
 
   SMARTLIST_FOREACH(cached_resolve_pqueue, cached_resolve_t *, res,
     {
@@ -2104,10 +2172,10 @@ assert_cache_ok_(void)
     });
 }
 
-#endif
+#endif /* defined(DEBUG_DNS_CACHE) */
 
-cached_resolve_t
-*dns_get_cache_entry(cached_resolve_t *query)
+cached_resolve_t *
+dns_get_cache_entry(cached_resolve_t *query)
 {
   return HT_FIND(cache_map, &cache_root, query);
 }
@@ -2117,4 +2185,3 @@ dns_insert_cache_entry(cached_resolve_t *new_entry)
 {
   HT_INSERT(cache_map, &cache_root, new_entry);
 }
-
diff --git a/src/or/dns.h b/src/feature/relay/dns.h
index 951a2a3467..e4474cdf43 100644
--- a/src/or/dns.h
+++ b/src/feature/relay/dns.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -38,10 +38,12 @@ void dns_launch_correctness_checks(void);
 int dns_seems_to_be_broken(void);
 int dns_seems_to_be_broken_for_ipv6(void);
 void dns_reset_correctness_checks(void);
+size_t dns_cache_total_allocation(void);
 void dump_dns_mem_usage(int severity);
+size_t dns_cache_handle_oom(time_t now, size_t min_remove_bytes);
 
 #ifdef DNS_PRIVATE
-#include "dns_structs.h"
+#include "feature/relay/dns_structs.h"
 
 MOCK_DECL(STATIC int,dns_resolve_impl,(edge_connection_t *exitconn,
 int is_resolve,or_circuit_t *oncirc, char **hostname_out,
@@ -64,7 +66,7 @@ set_exitconn_info_from_resolve,(edge_connection_t *exitconn,
 MOCK_DECL(STATIC int,
 launch_resolve,(cached_resolve_t *resolve));
 
-#endif
+#endif /* defined(DNS_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_DNS_H) */
 
diff --git a/src/or/dns_structs.h b/src/feature/relay/dns_structs.h
index bc6067213d..e128746f81 100644
--- a/src/or/dns_structs.h
+++ b/src/feature/relay/dns_structs.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -98,5 +98,5 @@ typedef struct cached_resolve_t {
   int minheap_idx;
 } cached_resolve_t;
 
-#endif
+#endif /* !defined(TOR_DNS_STRUCTS_H) */
 
diff --git a/src/or/ext_orport.c b/src/feature/relay/ext_orport.c
index 676adfd8bf..56c5bb96f5 100644
--- a/src/or/ext_orport.c
+++ b/src/feature/relay/ext_orport.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,21 +17,25 @@
  */
 
 #define EXT_ORPORT_PRIVATE
-#include "or.h"
-#include "connection.h"
-#include "connection_or.h"
-#include "ext_orport.h"
-#include "control.h"
-#include "config.h"
-#include "util.h"
-#include "main.h"
+#include "core/or/or.h"
+#include "core/mainloop/connection.h"
+#include "core/or/connection_or.h"
+#include "feature/control/control.h"
+#include "app/config/config.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "feature/relay/ext_orport.h"
+#include "core/mainloop/mainloop.h"
+#include "core/proto/proto_ext_or.h"
+
+#include "core/or/or_connection_st.h"
 
 /** Allocate and return a structure capable of holding an Extended
  *  ORPort message of body length <b>len</b>. */
 ext_or_cmd_t *
 ext_or_cmd_new(uint16_t len)
 {
-  size_t size = STRUCT_OFFSET(ext_or_cmd_t, body) + len;
+  size_t size = offsetof(ext_or_cmd_t, body) + len;
   ext_or_cmd_t *cmd = tor_malloc(size);
   cmd->len = len;
   return cmd;
@@ -39,7 +43,7 @@ ext_or_cmd_new(uint16_t len)
 
 /** Deallocate the Extended ORPort message in <b>cmd</b>. */
 void
-ext_or_cmd_free(ext_or_cmd_t *cmd)
+ext_or_cmd_free_(ext_or_cmd_t *cmd)
 {
   tor_free(cmd);
 }
@@ -69,10 +73,10 @@ connection_write_ext_or_command(connection_t *conn,
     return -1;
   set_uint16(header, htons(command));
   set_uint16(header+2, htons(bodylen));
-  connection_write_to_buf(header, 4, conn);
+  connection_buf_add(header, 4, conn);
   if (bodylen) {
     tor_assert(body);
-    connection_write_to_buf(body, bodylen, conn);
+    connection_buf_add(body, bodylen, conn);
   }
   return 0;
 }
@@ -170,7 +174,7 @@ connection_ext_or_auth_neg_auth_type(connection_t *conn)
   if (connection_get_inbuf_len(conn) < 1)
     return 0;
 
-  if (connection_fetch_from_buf(authtype, 1, conn) < 0)
+  if (connection_buf_get_bytes(authtype, 1, conn) < 0)
     return -1;
 
   log_debug(LD_GENERAL, "Client wants us to use %d auth type", authtype[0]);
@@ -310,7 +314,7 @@ connection_ext_or_auth_handle_client_nonce(connection_t *conn)
   if (connection_get_inbuf_len(conn) < EXT_OR_PORT_AUTH_NONCE_LEN)
     return 0;
 
-  if (connection_fetch_from_buf(client_nonce,
+  if (connection_buf_get_bytes(client_nonce,
                                 EXT_OR_PORT_AUTH_NONCE_LEN, conn) < 0)
     return -1;
 
@@ -325,7 +329,7 @@ connection_ext_or_auth_handle_client_nonce(connection_t *conn)
                             &reply, &reply_len) < 0)
     return -1;
 
-  connection_write_to_buf(reply, reply_len, conn);
+  connection_buf_add(reply, reply_len, conn);
 
   memwipe(reply, 0, reply_len);
   tor_free(reply);
@@ -347,9 +351,9 @@ static void
 connection_ext_or_auth_send_result(connection_t *conn, int success)
 {
   if (success)
-    connection_write_to_buf("\x01", 1, conn);
+    connection_buf_add("\x01", 1, conn);
   else
-    connection_write_to_buf("\x00", 1, conn);
+    connection_buf_add("\x00", 1, conn);
 }
 
 /** Receive the client's hash from <b>conn</b>, validate that it's
@@ -367,7 +371,7 @@ connection_ext_or_auth_handle_client_hash(connection_t *conn)
   if (connection_get_inbuf_len(conn) < EXT_OR_PORT_AUTH_HASH_LEN)
     return 0;
 
-  if (connection_fetch_from_buf(provided_client_hash,
+  if (connection_buf_get_bytes(provided_client_hash,
                                 EXT_OR_PORT_AUTH_HASH_LEN, conn) < 0)
     return -1;
 
@@ -459,6 +463,11 @@ connection_ext_or_handle_cmd_useraddr(connection_t *conn,
   tor_free(addr_str);
   if (res<0)
     return -1;
+  if (port == 0) {
+    log_warn(LD_GENERAL, "Server transport proxy gave us an empty port "
+             "in ExtORPort UserAddr command.");
+    // return -1; // enable this if nothing breaks after a while.
+  }
 
   res = tor_addr_parse(&addr, address_part);
   tor_free(address_part);
@@ -637,7 +646,7 @@ connection_ext_or_start_auth(or_connection_t *or_conn)
   log_debug(LD_GENERAL,
            "ExtORPort authentication: Sending supported authentication types");
 
-  connection_write_to_buf((const char *)authtypes, sizeof(authtypes), conn);
+  connection_buf_add((const char *)authtypes, sizeof(authtypes), conn);
   conn->state = EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE;
 
   return 0;
diff --git a/src/or/ext_orport.h b/src/feature/relay/ext_orport.h
index 33d954e8d0..7313ebd03d 100644
--- a/src/or/ext_orport.h
+++ b/src/feature/relay/ext_orport.h
@@ -1,16 +1,39 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef EXT_ORPORT_H
 #define EXT_ORPORT_H
 
+/** States of the Extended ORPort protocol. Be careful before changing
+ *  the numbers: they matter. */
+#define EXT_OR_CONN_STATE_MIN_ 1
+/** Extended ORPort authentication is waiting for the authentication
+ *  type selected by the client. */
+#define EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE 1
+/** Extended ORPort authentication is waiting for the client nonce. */
+#define EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE 2
+/** Extended ORPort authentication is waiting for the client hash. */
+#define EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH 3
+#define EXT_OR_CONN_STATE_AUTH_MAX 3
+/** Authentication finished and the Extended ORPort is now accepting
+ *  traffic. */
+#define EXT_OR_CONN_STATE_OPEN 4
+/** Extended ORPort is flushing its last messages and preparing to
+ *  start accepting OR connections. */
+#define EXT_OR_CONN_STATE_FLUSHING 5
+#define EXT_OR_CONN_STATE_MAX_ 5
+
 int connection_ext_or_start_auth(or_connection_t *or_conn);
 
 ext_or_cmd_t *ext_or_cmd_new(uint16_t len);
-void ext_or_cmd_free(ext_or_cmd_t *cmd);
+
+#define ext_or_cmd_free(cmd)                            \
+  FREE_AND_NULL(ext_or_cmd_t, ext_or_cmd_free_, (cmd))
+
+void ext_or_cmd_free_(ext_or_cmd_t *cmd);
 void connection_or_set_ext_or_identifier(or_connection_t *conn);
 void connection_or_remove_from_ext_or_id_map(or_connection_t *conn);
 void connection_or_clear_ext_or_id_map(void);
@@ -36,7 +59,6 @@ STATIC int handle_client_auth_nonce(const char *client_nonce,
 extern uint8_t *ext_or_auth_cookie;
 extern int ext_or_auth_cookie_is_set;
 #endif
-#endif
-
-#endif
+#endif /* defined(EXT_ORPORT_PRIVATE) */
 
+#endif /* !defined(EXT_ORPORT_H) */
diff --git a/src/feature/relay/onion_queue.c b/src/feature/relay/onion_queue.c
new file mode 100644
index 0000000000..696905cf5e
--- /dev/null
+++ b/src/feature/relay/onion_queue.c
@@ -0,0 +1,361 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file onion_queue.c
+ * \brief Functions to queue create cells for processing.
+ *
+ * Relays invoke these functions when they receive a CREATE or EXTEND
+ * cell in command.c or relay.c, in order to queue the pending request.
+ * They also invoke them from cpuworker.c, which handles dispatching
+ * onionskin requests to different worker threads.
+ *
+ * <br>
+ *
+ * This module also handles:
+ *  <ul>
+ *  <li> Queueing incoming onionskins on the relay side before passing
+ *      them to worker threads.
+ *   <li>Expiring onionskins on the relay side if they have waited for
+ *     too long.
+ * </ul>
+ **/
+
+#include "core/or/or.h"
+
+#include "feature/relay/onion_queue.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/cpuworker.h"
+#include "core/or/circuitlist.h"
+#include "core/or/onion.h"
+#include "feature/nodelist/networkstatus.h"
+
+#include "core/or/or_circuit_st.h"
+
+/** Type for a linked list of circuits that are waiting for a free CPU worker
+ * to process a waiting onion handshake. */
+typedef struct onion_queue_t {
+  TOR_TAILQ_ENTRY(onion_queue_t) next;
+  or_circuit_t *circ;
+  uint16_t handshake_type;
+  create_cell_t *onionskin;
+  time_t when_added;
+} onion_queue_t;
+
+/** 5 seconds on the onion queue til we just send back a destroy */
+#define ONIONQUEUE_WAIT_CUTOFF 5
+
+/** Array of queues of circuits waiting for CPU workers. An element is NULL
+ * if that queue is empty.*/
+static TOR_TAILQ_HEAD(onion_queue_head_t, onion_queue_t)
+              ol_list[MAX_ONION_HANDSHAKE_TYPE+1] =
+{ TOR_TAILQ_HEAD_INITIALIZER(ol_list[0]), /* tap */
+  TOR_TAILQ_HEAD_INITIALIZER(ol_list[1]), /* fast */
+  TOR_TAILQ_HEAD_INITIALIZER(ol_list[2]), /* ntor */
+};
+
+/** Number of entries of each type currently in each element of ol_list[]. */
+static int ol_entries[MAX_ONION_HANDSHAKE_TYPE+1];
+
+static int num_ntors_per_tap(void);
+static void onion_queue_entry_remove(onion_queue_t *victim);
+
+/* XXXX Check lengths vs MAX_ONIONSKIN_{CHALLENGE,REPLY}_LEN.
+ *
+ * (By which I think I meant, "make sure that no
+ * X_ONIONSKIN_CHALLENGE/REPLY_LEN is greater than
+ * MAX_ONIONSKIN_CHALLENGE/REPLY_LEN."  Also, make sure that we can pass
+ * over-large values via EXTEND2/EXTENDED2, for future-compatibility.*/
+
+/** Return true iff we have room to queue another onionskin of type
+ * <b>type</b>. */
+static int
+have_room_for_onionskin(uint16_t type)
+{
+  const or_options_t *options = get_options();
+  int num_cpus;
+  uint64_t tap_usec, ntor_usec;
+  uint64_t ntor_during_tap_usec, tap_during_ntor_usec;
+
+  /* If we've got fewer than 50 entries, we always have room for one more. */
+  if (ol_entries[type] < 50)
+    return 1;
+  num_cpus = get_num_cpus(options);
+  /* Compute how many microseconds we'd expect to need to clear all
+   * onionskins in various combinations of the queues. */
+
+  /* How long would it take to process all the TAP cells in the queue? */
+  tap_usec  = estimated_usec_for_onionskins(
+                                    ol_entries[ONION_HANDSHAKE_TYPE_TAP],
+                                    ONION_HANDSHAKE_TYPE_TAP) / num_cpus;
+
+  /* How long would it take to process all the NTor cells in the queue? */
+  ntor_usec = estimated_usec_for_onionskins(
+                                    ol_entries[ONION_HANDSHAKE_TYPE_NTOR],
+                                    ONION_HANDSHAKE_TYPE_NTOR) / num_cpus;
+
+  /* How long would it take to process the tap cells that we expect to
+   * process while draining the ntor queue? */
+  tap_during_ntor_usec  = estimated_usec_for_onionskins(
+    MIN(ol_entries[ONION_HANDSHAKE_TYPE_TAP],
+        ol_entries[ONION_HANDSHAKE_TYPE_NTOR] / num_ntors_per_tap()),
+                                    ONION_HANDSHAKE_TYPE_TAP) / num_cpus;
+
+  /* How long would it take to process the ntor cells that we expect to
+   * process while draining the tap queue? */
+  ntor_during_tap_usec  = estimated_usec_for_onionskins(
+    MIN(ol_entries[ONION_HANDSHAKE_TYPE_NTOR],
+        ol_entries[ONION_HANDSHAKE_TYPE_TAP] * num_ntors_per_tap()),
+                                    ONION_HANDSHAKE_TYPE_NTOR) / num_cpus;
+
+  /* See whether that exceeds MaxOnionQueueDelay. If so, we can't queue
+   * this. */
+  if (type == ONION_HANDSHAKE_TYPE_NTOR &&
+      (ntor_usec + tap_during_ntor_usec) / 1000 >
+       (uint64_t)options->MaxOnionQueueDelay)
+    return 0;
+
+  if (type == ONION_HANDSHAKE_TYPE_TAP &&
+      (tap_usec + ntor_during_tap_usec) / 1000 >
+       (uint64_t)options->MaxOnionQueueDelay)
+    return 0;
+
+  /* If we support the ntor handshake, then don't let TAP handshakes use
+   * more than 2/3 of the space on the queue. */
+  if (type == ONION_HANDSHAKE_TYPE_TAP &&
+      tap_usec / 1000 > (uint64_t)options->MaxOnionQueueDelay * 2 / 3)
+    return 0;
+
+  return 1;
+}
+
+/** Add <b>circ</b> to the end of ol_list and return 0, except
+ * if ol_list is too long, in which case do nothing and return -1.
+ */
+int
+onion_pending_add(or_circuit_t *circ, create_cell_t *onionskin)
+{
+  onion_queue_t *tmp;
+  time_t now = time(NULL);
+
+  if (onionskin->handshake_type > MAX_ONION_HANDSHAKE_TYPE) {
+    /* LCOV_EXCL_START
+     * We should have rejected this far before this point */
+    log_warn(LD_BUG, "Handshake %d out of range! Dropping.",
+             onionskin->handshake_type);
+    return -1;
+    /* LCOV_EXCL_STOP */
+  }
+
+  tmp = tor_malloc_zero(sizeof(onion_queue_t));
+  tmp->circ = circ;
+  tmp->handshake_type = onionskin->handshake_type;
+  tmp->onionskin = onionskin;
+  tmp->when_added = now;
+
+  if (!have_room_for_onionskin(onionskin->handshake_type)) {
+#define WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL (60)
+    static ratelim_t last_warned =
+      RATELIM_INIT(WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL);
+    char *m;
+    if (onionskin->handshake_type == ONION_HANDSHAKE_TYPE_NTOR &&
+        (m = rate_limit_log(&last_warned, approx_time()))) {
+      log_warn(LD_GENERAL,
+               "Your computer is too slow to handle this many circuit "
+               "creation requests! Please consider using the "
+               "MaxAdvertisedBandwidth config option or choosing a more "
+               "restricted exit policy.%s",m);
+      tor_free(m);
+    }
+    tor_free(tmp);
+    return -1;
+  }
+
+  ++ol_entries[onionskin->handshake_type];
+  log_info(LD_OR, "New create (%s). Queues now ntor=%d and tap=%d.",
+    onionskin->handshake_type == ONION_HANDSHAKE_TYPE_NTOR ? "ntor" : "tap",
+    ol_entries[ONION_HANDSHAKE_TYPE_NTOR],
+    ol_entries[ONION_HANDSHAKE_TYPE_TAP]);
+
+  circ->onionqueue_entry = tmp;
+  TOR_TAILQ_INSERT_TAIL(&ol_list[onionskin->handshake_type], tmp, next);
+
+  /* cull elderly requests. */
+  while (1) {
+    onion_queue_t *head = TOR_TAILQ_FIRST(&ol_list[onionskin->handshake_type]);
+    if (now - head->when_added < (time_t)ONIONQUEUE_WAIT_CUTOFF)
+      break;
+
+    circ = head->circ;
+    circ->onionqueue_entry = NULL;
+    onion_queue_entry_remove(head);
+    log_info(LD_CIRC,
+             "Circuit create request is too old; canceling due to overload.");
+    if (! TO_CIRCUIT(circ)->marked_for_close) {
+      circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_RESOURCELIMIT);
+    }
+  }
+  return 0;
+}
+
+/** Return a fairness parameter, to prefer processing NTOR style
+ * handshakes but still slowly drain the TAP queue so we don't starve
+ * it entirely. */
+static int
+num_ntors_per_tap(void)
+{
+#define DEFAULT_NUM_NTORS_PER_TAP 10
+#define MIN_NUM_NTORS_PER_TAP 1
+#define MAX_NUM_NTORS_PER_TAP 100000
+
+  return networkstatus_get_param(NULL, "NumNTorsPerTAP",
+                                 DEFAULT_NUM_NTORS_PER_TAP,
+                                 MIN_NUM_NTORS_PER_TAP,
+                                 MAX_NUM_NTORS_PER_TAP);
+}
+
+/** Choose which onion queue we'll pull from next. If one is empty choose
+ * the other; if they both have elements, load balance across them but
+ * favoring NTOR. */
+static uint16_t
+decide_next_handshake_type(void)
+{
+  /* The number of times we've chosen ntor lately when both were available. */
+  static int recently_chosen_ntors = 0;
+
+  if (!ol_entries[ONION_HANDSHAKE_TYPE_NTOR])
+    return ONION_HANDSHAKE_TYPE_TAP; /* no ntors? try tap */
+
+  if (!ol_entries[ONION_HANDSHAKE_TYPE_TAP]) {
+
+    /* Nick wants us to prioritize new tap requests when there aren't
+     * any in the queue and we've processed k ntor cells since the last
+     * tap cell. This strategy is maybe a good idea, since it starves tap
+     * less in the case where tap is rare, or maybe a poor idea, since it
+     * makes the new tap cell unfairly jump in front of ntor cells that
+     * got here first. In any case this edge case will only become relevant
+     * once tap is rare. We should reevaluate whether we like this decision
+     * once tap gets more rare. */
+    if (ol_entries[ONION_HANDSHAKE_TYPE_NTOR] &&
+        recently_chosen_ntors <= num_ntors_per_tap())
+      ++recently_chosen_ntors;
+
+    return ONION_HANDSHAKE_TYPE_NTOR; /* no taps? try ntor */
+  }
+
+  /* They both have something queued. Pick ntor if we haven't done that
+   * too much lately. */
+  if (++recently_chosen_ntors <= num_ntors_per_tap()) {
+    return ONION_HANDSHAKE_TYPE_NTOR;
+  }
+
+  /* Else, it's time to let tap have its turn. */
+  recently_chosen_ntors = 0;
+  return ONION_HANDSHAKE_TYPE_TAP;
+}
+
+/** Remove the highest priority item from ol_list[] and return it, or
+ * return NULL if the lists are empty.
+ */
+or_circuit_t *
+onion_next_task(create_cell_t **onionskin_out)
+{
+  or_circuit_t *circ;
+  uint16_t handshake_to_choose = decide_next_handshake_type();
+  onion_queue_t *head = TOR_TAILQ_FIRST(&ol_list[handshake_to_choose]);
+
+  if (!head)
+    return NULL; /* no onions pending, we're done */
+
+  tor_assert(head->circ);
+  tor_assert(head->handshake_type <= MAX_ONION_HANDSHAKE_TYPE);
+//  tor_assert(head->circ->p_chan); /* make sure it's still valid */
+/* XXX I only commented out the above line to make the unit tests
+ * more manageable. That's probably not good long-term. -RD */
+  circ = head->circ;
+  if (head->onionskin)
+    --ol_entries[head->handshake_type];
+  log_info(LD_OR, "Processing create (%s). Queues now ntor=%d and tap=%d.",
+    head->handshake_type == ONION_HANDSHAKE_TYPE_NTOR ? "ntor" : "tap",
+    ol_entries[ONION_HANDSHAKE_TYPE_NTOR],
+    ol_entries[ONION_HANDSHAKE_TYPE_TAP]);
+
+  *onionskin_out = head->onionskin;
+  head->onionskin = NULL; /* prevent free. */
+  circ->onionqueue_entry = NULL;
+  onion_queue_entry_remove(head);
+  return circ;
+}
+
+/** Return the number of <b>handshake_type</b>-style create requests pending.
+ */
+int
+onion_num_pending(uint16_t handshake_type)
+{
+  return ol_entries[handshake_type];
+}
+
+/** Go through ol_list, find the onion_queue_t element which points to
+ * circ, remove and free that element. Leave circ itself alone.
+ */
+void
+onion_pending_remove(or_circuit_t *circ)
+{
+  onion_queue_t *victim;
+
+  if (!circ)
+    return;
+
+  victim = circ->onionqueue_entry;
+  if (victim)
+    onion_queue_entry_remove(victim);
+
+  cpuworker_cancel_circ_handshake(circ);
+}
+
+/** Remove a queue entry <b>victim</b> from the queue, unlinking it from
+ * its circuit and freeing it and any structures it owns.*/
+static void
+onion_queue_entry_remove(onion_queue_t *victim)
+{
+  if (victim->handshake_type > MAX_ONION_HANDSHAKE_TYPE) {
+    /* LCOV_EXCL_START
+     * We should have rejected this far before this point */
+    log_warn(LD_BUG, "Handshake %d out of range! Dropping.",
+             victim->handshake_type);
+    /* XXX leaks */
+    return;
+    /* LCOV_EXCL_STOP */
+  }
+
+  TOR_TAILQ_REMOVE(&ol_list[victim->handshake_type], victim, next);
+
+  if (victim->circ)
+    victim->circ->onionqueue_entry = NULL;
+
+  if (victim->onionskin)
+    --ol_entries[victim->handshake_type];
+
+  tor_free(victim->onionskin);
+  tor_free(victim);
+}
+
+/** Remove all circuits from the pending list.  Called from tor_free_all. */
+void
+clear_pending_onions(void)
+{
+  onion_queue_t *victim, *next;
+  int i;
+  for (i=0; i<=MAX_ONION_HANDSHAKE_TYPE; i++) {
+    for (victim = TOR_TAILQ_FIRST(&ol_list[i]); victim; victim = next) {
+      next = TOR_TAILQ_NEXT(victim,next);
+      onion_queue_entry_remove(victim);
+    }
+    tor_assert(TOR_TAILQ_EMPTY(&ol_list[i]));
+  }
+  memset(ol_entries, 0, sizeof(ol_entries));
+}
diff --git a/src/feature/relay/onion_queue.h b/src/feature/relay/onion_queue.h
new file mode 100644
index 0000000000..0df921e057
--- /dev/null
+++ b/src/feature/relay/onion_queue.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file onion_queue.h
+ * \brief Header file for onion_queue.c.
+ **/
+
+#ifndef TOR_ONION_QUEUE_H
+#define TOR_ONION_QUEUE_H
+
+struct create_cell_t;
+
+int onion_pending_add(or_circuit_t *circ, struct create_cell_t *onionskin);
+or_circuit_t *onion_next_task(struct create_cell_t **onionskin_out);
+int onion_num_pending(uint16_t handshake_type);
+void onion_pending_remove(or_circuit_t *circ);
+void clear_pending_onions(void);
+
+#endif
diff --git a/src/or/router.c b/src/feature/relay/router.c
index c416474226..1dbaf2ed66 100644
--- a/src/or/router.c
+++ b/src/feature/relay/router.c
@@ -1,39 +1,66 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define ROUTER_PRIVATE
 
-#include "or.h"
-#include "circuitbuild.h"
-#include "circuitlist.h"
-#include "circuituse.h"
-#include "config.h"
-#include "connection.h"
-#include "control.h"
-#include "crypto_curve25519.h"
-#include "directory.h"
-#include "dirserv.h"
-#include "dns.h"
-#include "geoip.h"
-#include "hibernate.h"
-#include "main.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "policies.h"
-#include "protover.h"
-#include "relay.h"
-#include "rephist.h"
-#include "router.h"
-#include "routerkeys.h"
-#include "routerlist.h"
-#include "routerparse.h"
-#include "statefile.h"
-#include "torcert.h"
-#include "transports.h"
-#include "routerset.h"
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "app/config/statefile.h"
+#include "app/main/main.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
+#include "core/or/policies.h"
+#include "core/or/protover.h"
+#include "feature/client/transports.h"
+#include "feature/control/control.h"
+#include "feature/dirauth/process_descs.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirparse/authcert_parse.h"
+#include "feature/dirparse/routerparse.h"
+#include "feature/dirparse/signing.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/keymgt/loadkey.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nickname.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/torcert.h"
+#include "feature/relay/dns.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routerkeys.h"
+#include "feature/relay/routermode.h"
+#include "feature/relay/selftest.h"
+#include "lib/geoip/geoip.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/stats/rephist.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_init.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/encoding/confline.h"
+#include "lib/osinfo/uname.h"
+#include "lib/tls/tortls.h"
+
+#include "feature/dirauth/authmode.h"
+
+#include "app/config/or_state_st.h"
+#include "core/or/port_cfg_st.h"
+#include "feature/dirclient/dir_server_st.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/nodelist/authority_cert_st.h"
+#include "feature/nodelist/extrainfo_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
 
 /**
  * \file router.c
@@ -103,6 +130,47 @@ static authority_cert_t *legacy_key_certificate = NULL;
  * used by tor-gencert to sign new signing keys and make new key
  * certificates. */
 
+/** Return a readonly string with human readable description
+ * of <b>err</b>.
+ */
+const char *
+routerinfo_err_to_string(int err)
+{
+  switch (err) {
+    case TOR_ROUTERINFO_ERROR_NO_EXT_ADDR:
+      return "No known exit address yet";
+    case TOR_ROUTERINFO_ERROR_CANNOT_PARSE:
+      return "Cannot parse descriptor";
+    case TOR_ROUTERINFO_ERROR_NOT_A_SERVER:
+      return "Not running in server mode";
+    case TOR_ROUTERINFO_ERROR_DIGEST_FAILED:
+      return "Key digest failed";
+    case TOR_ROUTERINFO_ERROR_CANNOT_GENERATE:
+      return "Cannot generate descriptor";
+    case TOR_ROUTERINFO_ERROR_DESC_REBUILDING:
+      return "Descriptor still rebuilding - not ready yet";
+  }
+
+  log_warn(LD_BUG, "unknown routerinfo error %d - shouldn't happen", err);
+  tor_assert_unreached();
+
+  return "Unknown error";
+}
+
+/** Return true if we expect given error to be transient.
+ * Return false otherwise.
+ */
+int
+routerinfo_err_is_transient(int err)
+{
+  /**
+   * For simplicity, we consider all errors other than
+   * "not a server" transient - see discussion on
+   * https://trac.torproject.org/projects/tor/ticket/27034
+   */
+  return err != TOR_ROUTERINFO_ERROR_NOT_A_SERVER;
+}
+
 /** Replace the current onion key with <b>k</b>.  Does not affect
  * lastonionkey; to update lastonionkey correctly, call rotate_onion_key().
  */
@@ -131,7 +199,8 @@ get_onion_key(void)
 }
 
 /** Store a full copy of the current onion key into *<b>key</b>, and a full
- * copy of the most recent onion key into *<b>last</b>.
+ * copy of the most recent onion key into *<b>last</b>.  Store NULL into
+ * a pointer if the corresponding key does not exist.
  */
 void
 dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last)
@@ -139,8 +208,10 @@ dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last)
   tor_assert(key);
   tor_assert(last);
   tor_mutex_acquire(key_lock);
-  tor_assert(onionkey);
-  *key = crypto_pk_copy_full(onionkey);
+  if (onionkey)
+    *key = crypto_pk_copy_full(onionkey);
+  else
+    *key = NULL;
   if (lastonionkey)
     *last = crypto_pk_copy_full(lastonionkey);
   else
@@ -148,6 +219,51 @@ dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last)
   tor_mutex_release(key_lock);
 }
 
+/** Expire our old set of onion keys. This is done by setting
+ * last_curve25519_onion_key and lastonionkey to all zero's and NULL
+ * respectively.
+ *
+ * This function does not perform any grace period checks for the old onion
+ * keys.
+ */
+void
+expire_old_onion_keys(void)
+{
+  char *fname = NULL;
+
+  tor_mutex_acquire(key_lock);
+
+  /* Free lastonionkey and set it to NULL. */
+  if (lastonionkey) {
+    crypto_pk_free(lastonionkey);
+    lastonionkey = NULL;
+  }
+
+  /* We zero out the keypair. See the tor_mem_is_zero() check made in
+   * construct_ntor_key_map() below. */
+  memset(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key));
+
+  tor_mutex_release(key_lock);
+
+  fname = get_keydir_fname("secret_onion_key.old");
+  if (file_status(fname) == FN_FILE) {
+    if (tor_unlink(fname) != 0) {
+      log_warn(LD_FS, "Couldn't unlink old onion key file %s: %s",
+               fname, strerror(errno));
+    }
+  }
+  tor_free(fname);
+
+  fname = get_keydir_fname("secret_onion_key_ntor.old");
+  if (file_status(fname) == FN_FILE) {
+    if (tor_unlink(fname) != 0) {
+      log_warn(LD_FS, "Couldn't unlink old ntor onion key file %s: %s",
+               fname, strerror(errno));
+    }
+  }
+  tor_free(fname);
+}
+
 /** Return the current secret onion key for the ntor handshake. Must only
  * be called from the main thread. */
 static const curve25519_keypair_t *
@@ -162,15 +278,17 @@ construct_ntor_key_map(void)
 {
   di_digest256_map_t *m = NULL;
 
-  dimap_add_entry(&m,
-                  curve25519_onion_key.pubkey.public_key,
-                  tor_memdup(&curve25519_onion_key,
-                             sizeof(curve25519_keypair_t)));
-  if (!tor_mem_is_zero((const char*)
-                          last_curve25519_onion_key.pubkey.public_key,
-                       CURVE25519_PUBKEY_LEN)) {
-    dimap_add_entry(&m,
-                    last_curve25519_onion_key.pubkey.public_key,
+  const uint8_t *cur_pk = curve25519_onion_key.pubkey.public_key;
+  const uint8_t *last_pk = last_curve25519_onion_key.pubkey.public_key;
+
+  if (!tor_mem_is_zero((const char *)cur_pk, CURVE25519_PUBKEY_LEN)) {
+    dimap_add_entry(&m, cur_pk,
+                    tor_memdup(&curve25519_onion_key,
+                               sizeof(curve25519_keypair_t)));
+  }
+  if (!tor_mem_is_zero((const char*)last_pk, CURVE25519_PUBKEY_LEN) &&
+      tor_memneq(cur_pk, last_pk, CURVE25519_PUBKEY_LEN)) {
+    dimap_add_entry(&m, last_pk,
                     tor_memdup(&last_curve25519_onion_key,
                                sizeof(curve25519_keypair_t)));
   }
@@ -188,7 +306,7 @@ ntor_key_map_free_helper(void *arg)
 }
 /** Release all storage from a keymap returned by construct_ntor_key_map. */
 void
-ntor_key_map_free(di_digest256_map_t *map)
+ntor_key_map_free_(di_digest256_map_t *map)
 {
   if (!map)
     return;
@@ -212,7 +330,11 @@ set_server_identity_key(crypto_pk_t *k)
 {
   crypto_pk_free(server_identitykey);
   server_identitykey = k;
-  crypto_pk_get_digest(server_identitykey, server_identitykey_digest);
+  if (crypto_pk_get_digest(server_identitykey,
+                           server_identitykey_digest) < 0) {
+    log_err(LD_BUG, "Couldn't compute our own identity key digest.");
+    tor_assert(0);
+  }
 }
 
 /** Make sure that we have set up our identity keys to match or not match as
@@ -329,8 +451,8 @@ rotate_onion_key(void)
   or_state_t *state = get_or_state();
   curve25519_keypair_t new_curve25519_keypair;
   time_t now;
-  fname = get_datadir_fname2("keys", "secret_onion_key");
-  fname_prev = get_datadir_fname2("keys", "secret_onion_key.old");
+  fname = get_keydir_fname("secret_onion_key");
+  fname_prev = get_keydir_fname("secret_onion_key.old");
   /* There isn't much point replacing an old key with an empty file */
   if (file_status(fname) == FN_FILE) {
     if (replace_file(fname, fname_prev))
@@ -350,8 +472,8 @@ rotate_onion_key(void)
   }
   tor_free(fname);
   tor_free(fname_prev);
-  fname = get_datadir_fname2("keys", "secret_onion_key_ntor");
-  fname_prev = get_datadir_fname2("keys", "secret_onion_key_ntor.old");
+  fname = get_keydir_fname("secret_onion_key_ntor");
+  fname_prev = get_keydir_fname("secret_onion_key_ntor.old");
   if (curve25519_keypair_generate(&new_curve25519_keypair, 1) < 0)
     goto error;
   /* There isn't much point replacing an old key with an empty file */
@@ -409,85 +531,6 @@ log_new_relay_greeting(void)
   already_logged = 1;
 }
 
-/** Try to read an RSA key from <b>fname</b>.  If <b>fname</b> doesn't exist
- * and <b>generate</b> is true, create a new RSA key and save it in
- * <b>fname</b>.  Return the read/created key, or NULL on error.  Log all
- * errors at level <b>severity</b>. If <b>log_greeting</b> is non-zero and a
- * new key was created, log_new_relay_greeting() is called.
- */
-crypto_pk_t *
-init_key_from_file(const char *fname, int generate, int severity,
-                   int log_greeting)
-{
-  crypto_pk_t *prkey = NULL;
-
-  if (!(prkey = crypto_pk_new())) {
-    tor_log(severity, LD_GENERAL,"Error constructing key");
-    goto error;
-  }
-
-  switch (file_status(fname)) {
-    case FN_DIR:
-    case FN_ERROR:
-      tor_log(severity, LD_FS,"Can't read key from \"%s\"", fname);
-      goto error;
-    /* treat empty key files as if the file doesn't exist, and,
-     * if generate is set, replace the empty file in
-     * crypto_pk_write_private_key_to_filename() */
-    case FN_NOENT:
-    case FN_EMPTY:
-      if (generate) {
-        if (!have_lockfile()) {
-          if (try_locking(get_options(), 0)<0) {
-            /* Make sure that --list-fingerprint only creates new keys
-             * if there is no possibility for a deadlock. */
-            tor_log(severity, LD_FS, "Another Tor process has locked \"%s\". "
-                    "Not writing any new keys.", fname);
-            /*XXXX The 'other process' might make a key in a second or two;
-             * maybe we should wait for it. */
-            goto error;
-          }
-        }
-        log_info(LD_GENERAL, "No key found in \"%s\"; generating fresh key.",
-                 fname);
-        if (crypto_pk_generate_key(prkey)) {
-          tor_log(severity, LD_GENERAL,"Error generating onion key");
-          goto error;
-        }
-        if (crypto_pk_check_key(prkey) <= 0) {
-          tor_log(severity, LD_GENERAL,"Generated key seems invalid");
-          goto error;
-        }
-        log_info(LD_GENERAL, "Generated key seems valid");
-        if (log_greeting) {
-            log_new_relay_greeting();
-        }
-        if (crypto_pk_write_private_key_to_filename(prkey, fname)) {
-          tor_log(severity, LD_FS,
-              "Couldn't write generated key to \"%s\".", fname);
-          goto error;
-        }
-      } else {
-        tor_log(severity, LD_GENERAL, "No key found in \"%s\"", fname);
-        goto error;
-      }
-      return prkey;
-    case FN_FILE:
-      if (crypto_pk_read_private_key_from_filename(prkey, fname)) {
-        tor_log(severity, LD_GENERAL,"Error loading private key.");
-        goto error;
-      }
-      return prkey;
-    default:
-      tor_assert(0);
-  }
-
- error:
-  if (prkey)
-    crypto_pk_free(prkey);
-  return NULL;
-}
-
 /** Load a curve25519 keypair from the file <b>fname</b>, writing it into
  * <b>keys_out</b>.  If the file isn't found, or is empty, and <b>generate</b>
  * is true, create a new keypair and write it into the file.  If there are
@@ -575,15 +618,15 @@ load_authority_keyset(int legacy, crypto_pk_t **key_out,
   crypto_pk_t *signing_key = NULL;
   authority_cert_t *parsed = NULL;
 
-  fname = get_datadir_fname2("keys",
+  fname = get_keydir_fname(
                  legacy ? "legacy_signing_key" : "authority_signing_key");
-  signing_key = init_key_from_file(fname, 0, LOG_ERR, 0);
+  signing_key = init_key_from_file(fname, 0, LOG_ERR, NULL);
   if (!signing_key) {
     log_warn(LD_DIR, "No version 3 directory key found in %s", fname);
     goto done;
   }
   tor_free(fname);
-  fname = get_datadir_fname2("keys",
+  fname = get_keydir_fname(
                legacy ? "legacy_certificate" : "authority_certificate");
   cert = read_file_to_str(fname, 0, NULL);
   if (!cert) {
@@ -683,6 +726,47 @@ v3_authority_check_key_expiry(void)
   last_warned = now;
 }
 
+/** Get the lifetime of an onion key in days. This value is defined by the
+ * network consesus parameter "onion-key-rotation-days". Always returns a value
+ * between <b>MIN_ONION_KEY_LIFETIME_DAYS</b> and
+ * <b>MAX_ONION_KEY_LIFETIME_DAYS</b>.
+ */
+static int
+get_onion_key_rotation_days_(void)
+{
+  return networkstatus_get_param(NULL,
+                                 "onion-key-rotation-days",
+                                 DEFAULT_ONION_KEY_LIFETIME_DAYS,
+                                 MIN_ONION_KEY_LIFETIME_DAYS,
+                                 MAX_ONION_KEY_LIFETIME_DAYS);
+}
+
+/** Get the current lifetime of an onion key in seconds. This value is defined
+ * by the network consesus parameter "onion-key-rotation-days", but the value
+ * is converted to seconds.
+ */
+int
+get_onion_key_lifetime(void)
+{
+  return get_onion_key_rotation_days_()*24*60*60;
+}
+
+/** Get the grace period of an onion key in seconds. This value is defined by
+ * the network consesus parameter "onion-key-grace-period-days", but the value
+ * is converted to seconds.
+ */
+int
+get_onion_key_grace_period(void)
+{
+  int grace_period;
+  grace_period = networkstatus_get_param(NULL,
+                                         "onion-key-grace-period-days",
+                                         DEFAULT_ONION_KEY_GRACE_PERIOD_DAYS,
+                                         MIN_ONION_KEY_GRACE_PERIOD_DAYS,
+                                         get_onion_key_rotation_days_());
+  return grace_period*24*60*60;
+}
+
 /** Set up Tor's TLS contexts, based on our configuration and keys. Return 0
  * on success, and -1 on failure. */
 int
@@ -693,12 +777,6 @@ router_initialize_tls_context(void)
   int lifetime = options->SSLKeyLifetime;
   if (public_server_mode(options))
     flags |= TOR_TLS_CTX_IS_PUBLIC_SERVER;
-  if (options->TLSECGroup) {
-    if (!strcasecmp(options->TLSECGroup, "P256"))
-      flags |= TOR_TLS_CTX_USE_ECDHE_P256;
-    else if (!strcasecmp(options->TLSECGroup, "P224"))
-      flags |= TOR_TLS_CTX_USE_ECDHE_P224;
-  }
   if (!lifetime) { /* we should guess a good ssl cert lifetime */
 
     /* choose between 5 and 365 days, and round to the day */
@@ -848,17 +926,9 @@ init_keys(void)
   }
   if (init_keys_common() < 0)
     return -1;
-  /* Make sure DataDirectory exists, and is private. */
-  if (check_private_dir(options->DataDirectory, CPD_CREATE, options->User)) {
-    return -1;
-  }
-  /* Check the key directory. */
-  keydir = get_datadir_fname("keys");
-  if (check_private_dir(keydir, CPD_CREATE, options->User)) {
-    tor_free(keydir);
+
+  if (create_keys_directory(options) < 0)
     return -1;
-  }
-  tor_free(keydir);
 
   /* 1a. Read v3 directory authority key/cert information. */
   memset(v3_digest, 0, sizeof(v3_digest));
@@ -871,18 +941,25 @@ init_keys(void)
     }
     cert = get_my_v3_authority_cert();
     if (cert) {
-      crypto_pk_get_digest(get_my_v3_authority_cert()->identity_key,
-                           v3_digest);
+      if (crypto_pk_get_digest(get_my_v3_authority_cert()->identity_key,
+                               v3_digest) < 0) {
+        log_err(LD_BUG, "Couldn't compute my v3 authority identity key "
+                "digest.");
+        return -1;
+      }
       v3_digest_set = 1;
     }
   }
 
   /* 1b. Read identity key. Make it if none is found. */
-  keydir = get_datadir_fname2("keys", "secret_id_key");
+  keydir = get_keydir_fname("secret_id_key");
   log_info(LD_GENERAL,"Reading/making identity key \"%s\"...",keydir);
-  prkey = init_key_from_file(keydir, 1, LOG_ERR, 1);
+  bool created = false;
+  prkey = init_key_from_file(keydir, 1, LOG_ERR, &created);
   tor_free(keydir);
   if (!prkey) return -1;
+  if (created)
+    log_new_relay_greeting();
   set_server_identity_key(prkey);
 
   /* 1c. If we are configured as a bridge, generate a client key;
@@ -901,13 +978,16 @@ init_keys(void)
   }
 
   /* 1d. Load all ed25519 keys */
-  if (load_ed_keys(options,now) < 0)
+  const int new_signing_key = load_ed_keys(options,now);
+  if (new_signing_key < 0)
     return -1;
 
   /* 2. Read onion key.  Make it if none is found. */
-  keydir = get_datadir_fname2("keys", "secret_onion_key");
+  keydir = get_keydir_fname("secret_onion_key");
   log_info(LD_GENERAL,"Reading/making onion key \"%s\"...",keydir);
-  prkey = init_key_from_file(keydir, 1, LOG_ERR, 1);
+  prkey = init_key_from_file(keydir, 1, LOG_ERR, &created);
+  if (created)
+    log_new_relay_greeting();
   tor_free(keydir);
   if (!prkey) return -1;
   set_onion_key(prkey);
@@ -923,14 +1003,14 @@ init_keys(void)
       /* We have no LastRotatedOnionKey set; either we just created the key
        * or it's a holdover from 0.1.2.4-alpha-dev or earlier.  In either case,
        * start the clock ticking now so that we will eventually rotate it even
-       * if we don't stay up for a full MIN_ONION_KEY_LIFETIME. */
+       * if we don't stay up for the full lifetime of an onion key. */
       state->LastRotatedOnionKey = onionkey_set_at = now;
       or_state_mark_dirty(state, options->AvoidDiskWrites ?
                                    time(NULL)+3600 : 0);
     }
   }
 
-  keydir = get_datadir_fname2("keys", "secret_onion_key.old");
+  keydir = get_keydir_fname("secret_onion_key.old");
   if (!lastonionkey && file_status(keydir) == FN_FILE) {
     /* Load keys from non-empty files only.
      * Missing old keys won't be replaced with freshly generated keys. */
@@ -943,14 +1023,14 @@ init_keys(void)
   {
     /* 2b. Load curve25519 onion keys. */
     int r;
-    keydir = get_datadir_fname2("keys", "secret_onion_key_ntor");
+    keydir = get_keydir_fname("secret_onion_key_ntor");
     r = init_curve25519_keypair_from_file(&curve25519_onion_key,
                                           keydir, 1, LOG_ERR, "onion");
     tor_free(keydir);
     if (r<0)
       return -1;
 
-    keydir = get_datadir_fname2("keys", "secret_onion_key_ntor.old");
+    keydir = get_keydir_fname("secret_onion_key_ntor.old");
     if (tor_mem_is_zero((const char *)
                            last_curve25519_onion_key.pubkey.public_key,
                         CURVE25519_PUBKEY_LEN) &&
@@ -971,7 +1051,7 @@ init_keys(void)
 
   /* 3b. Get an ed25519 link certificate.  Note that we need to do this
    * after we set up the TLS context */
-  if (generate_ed_link_cert(options, now) < 0) {
+  if (generate_ed_link_cert(options, now, new_signing_key > 0) < 0) {
     log_err(LD_GENERAL,"Couldn't make link cert");
     return -1;
   }
@@ -979,7 +1059,7 @@ init_keys(void)
   /* 4. Build our router descriptor. */
   /* Must be called after keys are initialized. */
   mydesc = router_get_my_descriptor();
-  if (authdir_mode_handles_descs(options, ROUTER_PURPOSE_GENERAL)) {
+  if (authdir_mode_v3(options)) {
     const char *m = NULL;
     routerinfo_t *ri;
     /* We need to add our own fingerprint so it gets recognized. */
@@ -1079,76 +1159,15 @@ init_keys(void)
   return 0; /* success */
 }
 
-/* Keep track of whether we should upload our server descriptor,
- * and what type of server we are.
- */
-
-/** Whether we can reach our ORPort from the outside. */
-static int can_reach_or_port = 0;
-/** Whether we can reach our DirPort from the outside. */
-static int can_reach_dir_port = 0;
-
-/** Forget what we have learned about our reachability status. */
-void
-router_reset_reachability(void)
-{
-  can_reach_or_port = can_reach_dir_port = 0;
-}
-
-/** Return 1 if we won't do reachability checks, because:
- *   - AssumeReachable is set, or
- *   - the network is disabled.
- * Otherwise, return 0.
- */
-static int
-router_reachability_checks_disabled(const or_options_t *options)
-{
-  return options->AssumeReachable ||
-         net_is_disabled();
-}
-
-/** Return 0 if we need to do an ORPort reachability check, because:
- *   - no reachability check has been done yet, or
- *   - we've initiated reachability checks, but none have succeeded.
- *  Return 1 if we don't need to do an ORPort reachability check, because:
- *   - we've seen a successful reachability check, or
- *   - AssumeReachable is set, or
- *   - the network is disabled.
- */
-int
-check_whether_orport_reachable(const or_options_t *options)
-{
-  int reach_checks_disabled = router_reachability_checks_disabled(options);
-  return reach_checks_disabled ||
-         can_reach_or_port;
-}
-
-/** Return 0 if we need to do a DirPort reachability check, because:
- *   - no reachability check has been done yet, or
- *   - we've initiated reachability checks, but none have succeeded.
- *  Return 1 if we don't need to do a DirPort reachability check, because:
- *   - we've seen a successful reachability check, or
- *   - there is no DirPort set, or
- *   - AssumeReachable is set, or
- *   - the network is disabled.
- */
-int
-check_whether_dirport_reachable(const or_options_t *options)
-{
-  int reach_checks_disabled = router_reachability_checks_disabled(options) ||
-                              !options->DirPort_set;
-  return reach_checks_disabled ||
-         can_reach_dir_port;
-}
-
 /** The lower threshold of remaining bandwidth required to advertise (or
  * automatically provide) directory services */
 /* XXX Should this be increased? */
 #define MIN_BW_TO_ADVERTISE_DIRSERVER 51200
 
-/** Return true iff we have enough configured bandwidth to cache directory
+/** Return true iff we have enough configured bandwidth to advertise or
+ * automatically provide directory services from cache directory
  * information. */
-static int
+int
 router_has_bandwidth_to_be_dirserver(const or_options_t *options)
 {
   if (options->BandwidthRate < MIN_BW_TO_ADVERTISE_DIRSERVER) {
@@ -1169,7 +1188,7 @@ router_has_bandwidth_to_be_dirserver(const or_options_t *options)
  * MIN_BW_TO_ADVERTISE_DIRSERVER, don't bother trying to serve requests.
  */
 static int
-router_should_be_directory_server(const or_options_t *options, int dir_port)
+router_should_be_dirserver(const or_options_t *options, int dir_port)
 {
   static int advertising=1; /* start out assuming we will advertise */
   int new_choice=1;
@@ -1178,9 +1197,9 @@ router_should_be_directory_server(const or_options_t *options, int dir_port)
   if (accounting_is_enabled(options) &&
     get_options()->AccountingRule != ACCT_IN) {
     /* Don't spend bytes for directory traffic if we could end up hibernating,
-     * but allow DirPort otherwise. Some people set AccountingMax because
-     * they're confused or to get statistics. Directory traffic has a much
-     * larger effect on output than input so there is no reason to turn it
+     * but allow DirPort otherwise. Some relay operators set AccountingMax
+     * because they're confused or to get statistics. Directory traffic has a
+     * much larger effect on output than input so there is no reason to turn it
      * off if using AccountingRule in. */
     int interval_length = accounting_get_interval_length();
     uint32_t effective_bw = get_effective_bwrate(options);
@@ -1191,10 +1210,10 @@ router_should_be_directory_server(const or_options_t *options, int dir_port)
       interval_length = 1;
     }
     log_info(LD_GENERAL, "Calculating whether to advertise %s: effective "
-                         "bwrate: %u, AccountingMax: "U64_FORMAT", "
+                         "bwrate: %u, AccountingMax: %"PRIu64", "
                          "accounting interval length %d",
                          dir_port ? "dirport" : "begindir",
-                         effective_bw, U64_PRINTF_ARG(options->AccountingMax),
+                         effective_bw, (options->AccountingMax),
                          interval_length);
 
     acc_bytes = options->AccountingMax;
@@ -1228,19 +1247,6 @@ router_should_be_directory_server(const or_options_t *options, int dir_port)
   return advertising;
 }
 
-/** Return 1 if we are configured to accept either relay or directory requests
- * from clients and we aren't at risk of exceeding our bandwidth limits, thus
- * we should be a directory server. If not, return 0.
- */
-int
-dir_server_mode(const or_options_t *options)
-{
-  if (!options->DirCache)
-    return 0;
-  return options->DirPort_set ||
-    (server_mode(options) && router_has_bandwidth_to_be_dirserver(options));
-}
-
 /** Look at a variety of factors, and return 0 if we don't want to
  * advertise the fact that we have a DirPort open or begindir support, else
  * return 1.
@@ -1274,15 +1280,15 @@ decide_to_advertise_dir_impl(const or_options_t *options,
 
   /* Part two: consider config options that could make us choose to
    * publish or not publish that the user might find surprising. */
-  return router_should_be_directory_server(options, dir_port);
+  return router_should_be_dirserver(options, dir_port);
 }
 
 /** Front-end to decide_to_advertise_dir_impl(): return 0 if we don't want to
  * advertise the fact that we have a DirPort open, else return the
  * DirPort we want to advertise.
  */
-static int
-decide_to_advertise_dirport(const or_options_t *options, uint16_t dir_port)
+int
+router_should_advertise_dirport(const or_options_t *options, uint16_t dir_port)
 {
   /* supports_tunnelled_dir_requests is not relevant, pass 0 */
   return decide_to_advertise_dir_impl(options, dir_port, 0) ? dir_port : 0;
@@ -1292,7 +1298,7 @@ decide_to_advertise_dirport(const or_options_t *options, uint16_t dir_port)
  * advertise the fact that we support begindir requests, else return 1.
  */
 static int
-decide_to_advertise_begindir(const or_options_t *options,
+router_should_advertise_begindir(const or_options_t *options,
                              int supports_tunnelled_dir_requests)
 {
   /* dir_port is not relevant, pass 0 */
@@ -1300,274 +1306,6 @@ decide_to_advertise_begindir(const or_options_t *options,
                                       supports_tunnelled_dir_requests);
 }
 
-/** Allocate and return a new extend_info_t that can be used to build
- * a circuit to or through the router <b>r</b>. Uses the primary
- * address of the router, so should only be called on a server. */
-static extend_info_t *
-extend_info_from_router(const routerinfo_t *r)
-{
-  tor_addr_port_t ap;
-  tor_assert(r);
-
-  /* Make sure we don't need to check address reachability */
-  tor_assert_nonfatal(router_skip_or_reachability(get_options(), 0));
-
-  router_get_prim_orport(r, &ap);
-  return extend_info_new(r->nickname, r->cache_info.identity_digest,
-                         r->onion_pkey, r->onion_curve25519_pkey,
-                         &ap.addr, ap.port);
-}
-
-/** Some time has passed, or we just got new directory information.
- * See if we currently believe our ORPort or DirPort to be
- * unreachable. If so, launch a new test for it.
- *
- * For ORPort, we simply try making a circuit that ends at ourselves.
- * Success is noticed in onionskin_answer().
- *
- * For DirPort, we make a connection via Tor to our DirPort and ask
- * for our own server descriptor.
- * Success is noticed in connection_dir_client_reached_eof().
- */
-void
-consider_testing_reachability(int test_or, int test_dir)
-{
-  const routerinfo_t *me = router_get_my_routerinfo();
-  const or_options_t *options = get_options();
-  int orport_reachable = check_whether_orport_reachable(options);
-  tor_addr_t addr;
-  if (!me)
-    return;
-
-  if (routerset_contains_router(options->ExcludeNodes, me, -1) &&
-      options->StrictNodes) {
-    /* If we've excluded ourself, and StrictNodes is set, we can't test
-     * ourself. */
-    if (test_or || test_dir) {
-#define SELF_EXCLUDED_WARN_INTERVAL 3600
-      static ratelim_t warning_limit=RATELIM_INIT(SELF_EXCLUDED_WARN_INTERVAL);
-      log_fn_ratelim(&warning_limit, LOG_WARN, LD_CIRC,
-                 "Can't peform self-tests for this relay: we have "
-                 "listed ourself in ExcludeNodes, and StrictNodes is set. "
-                 "We cannot learn whether we are usable, and will not "
-                 "be able to advertise ourself.");
-    }
-    return;
-  }
-
-  if (test_or && (!orport_reachable || !circuit_enough_testing_circs())) {
-    extend_info_t *ei = extend_info_from_router(me);
-    /* XXX IPv6 self testing */
-    log_info(LD_CIRC, "Testing %s of my ORPort: %s:%d.",
-             !orport_reachable ? "reachability" : "bandwidth",
-             fmt_addr32(me->addr), me->or_port);
-    circuit_launch_by_extend_info(CIRCUIT_PURPOSE_TESTING, ei,
-                            CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL);
-    extend_info_free(ei);
-  }
-
-  /* XXX IPv6 self testing */
-  tor_addr_from_ipv4h(&addr, me->addr);
-  if (test_dir && !check_whether_dirport_reachable(options) &&
-      !connection_get_by_type_addr_port_purpose(
-                CONN_TYPE_DIR, &addr, me->dir_port,
-                DIR_PURPOSE_FETCH_SERVERDESC)) {
-    /* ask myself, via tor, for my server descriptor. */
-    directory_initiate_command(&addr, me->or_port,
-                               &addr, me->dir_port,
-                               me->cache_info.identity_digest,
-                               DIR_PURPOSE_FETCH_SERVERDESC,
-                               ROUTER_PURPOSE_GENERAL,
-                               DIRIND_ANON_DIRPORT, "authority.z", NULL, 0, 0);
-  }
-}
-
-/** Annotate that we found our ORPort reachable. */
-void
-router_orport_found_reachable(void)
-{
-  const routerinfo_t *me = router_get_my_routerinfo();
-  const or_options_t *options = get_options();
-  if (!can_reach_or_port && me) {
-    char *address = tor_dup_ip(me->addr);
-    log_notice(LD_OR,"Self-testing indicates your ORPort is reachable from "
-               "the outside. Excellent.%s",
-               options->PublishServerDescriptor_ != NO_DIRINFO
-               && check_whether_dirport_reachable(options) ?
-                 " Publishing server descriptor." : "");
-    can_reach_or_port = 1;
-    mark_my_descriptor_dirty("ORPort found reachable");
-    /* This is a significant enough change to upload immediately,
-     * at least in a test network */
-    if (options->TestingTorNetwork == 1) {
-      reschedule_descriptor_update_check();
-    }
-    control_event_server_status(LOG_NOTICE,
-                                "REACHABILITY_SUCCEEDED ORADDRESS=%s:%d",
-                                address, me->or_port);
-    tor_free(address);
-  }
-}
-
-/** Annotate that we found our DirPort reachable. */
-void
-router_dirport_found_reachable(void)
-{
-  const routerinfo_t *me = router_get_my_routerinfo();
-  const or_options_t *options = get_options();
-  if (!can_reach_dir_port && me) {
-    char *address = tor_dup_ip(me->addr);
-    log_notice(LD_DIRSERV,"Self-testing indicates your DirPort is reachable "
-               "from the outside. Excellent.%s",
-               options->PublishServerDescriptor_ != NO_DIRINFO
-               && check_whether_orport_reachable(options) ?
-               " Publishing server descriptor." : "");
-    can_reach_dir_port = 1;
-    if (decide_to_advertise_dirport(options, me->dir_port)) {
-      mark_my_descriptor_dirty("DirPort found reachable");
-      /* This is a significant enough change to upload immediately,
-       * at least in a test network */
-      if (options->TestingTorNetwork == 1) {
-        reschedule_descriptor_update_check();
-      }
-    }
-    control_event_server_status(LOG_NOTICE,
-                                "REACHABILITY_SUCCEEDED DIRADDRESS=%s:%d",
-                                address, me->dir_port);
-    tor_free(address);
-  }
-}
-
-/** We have enough testing circuits open. Send a bunch of "drop"
- * cells down each of them, to exercise our bandwidth. */
-void
-router_perform_bandwidth_test(int num_circs, time_t now)
-{
-  int num_cells = (int)(get_options()->BandwidthRate * 10 /
-                        CELL_MAX_NETWORK_SIZE);
-  int max_cells = num_cells < CIRCWINDOW_START ?
-                    num_cells : CIRCWINDOW_START;
-  int cells_per_circuit = max_cells / num_circs;
-  origin_circuit_t *circ = NULL;
-
-  log_notice(LD_OR,"Performing bandwidth self-test...done.");
-  while ((circ = circuit_get_next_by_pk_and_purpose(circ, NULL,
-                                              CIRCUIT_PURPOSE_TESTING))) {
-    /* dump cells_per_circuit drop cells onto this circ */
-    int i = cells_per_circuit;
-    if (circ->base_.state != CIRCUIT_STATE_OPEN)
-      continue;
-    circ->base_.timestamp_dirty = now;
-    while (i-- > 0) {
-      if (relay_send_command_from_edge(0, TO_CIRCUIT(circ),
-                                       RELAY_COMMAND_DROP,
-                                       NULL, 0, circ->cpath->prev)<0) {
-        return; /* stop if error */
-      }
-    }
-  }
-}
-
-/** Return true iff our network is in some sense disabled: either we're
- * hibernating, entering hibernation, or the network is turned off with
- * DisableNetwork. */
-int
-net_is_disabled(void)
-{
-  return get_options()->DisableNetwork || we_are_hibernating();
-}
-
-/** Return true iff we believe ourselves to be an authoritative
- * directory server.
- */
-int
-authdir_mode(const or_options_t *options)
-{
-  return options->AuthoritativeDir != 0;
-}
-/** Return true iff we believe ourselves to be a v3 authoritative
- * directory server.
- */
-int
-authdir_mode_v3(const or_options_t *options)
-{
-  return authdir_mode(options) && options->V3AuthoritativeDir != 0;
-}
-/** Return true iff we are a v3 directory authority. */
-int
-authdir_mode_any_main(const or_options_t *options)
-{
-  return options->V3AuthoritativeDir;
-}
-/** Return true if we believe ourselves to be any kind of
- * authoritative directory beyond just a hidserv authority. */
-int
-authdir_mode_any_nonhidserv(const or_options_t *options)
-{
-  return options->BridgeAuthoritativeDir ||
-         authdir_mode_any_main(options);
-}
-/** Return true iff we are an authoritative directory server that is
- * authoritative about receiving and serving descriptors of type
- * <b>purpose</b> on its dirport.  Use -1 for "any purpose". */
-int
-authdir_mode_handles_descs(const or_options_t *options, int purpose)
-{
-  if (purpose < 0)
-    return authdir_mode_any_nonhidserv(options);
-  else if (purpose == ROUTER_PURPOSE_GENERAL)
-    return authdir_mode_any_main(options);
-  else if (purpose == ROUTER_PURPOSE_BRIDGE)
-    return (options->BridgeAuthoritativeDir);
-  else
-    return 0;
-}
-/** Return true iff we are an authoritative directory server that
- * publishes its own network statuses.
- */
-int
-authdir_mode_publishes_statuses(const or_options_t *options)
-{
-  if (authdir_mode_bridge(options))
-    return 0;
-  return authdir_mode_any_nonhidserv(options);
-}
-/** Return true iff we are an authoritative directory server that
- * tests reachability of the descriptors it learns about.
- */
-int
-authdir_mode_tests_reachability(const or_options_t *options)
-{
-  return authdir_mode_handles_descs(options, -1);
-}
-/** Return true iff we believe ourselves to be a bridge authoritative
- * directory server.
- */
-int
-authdir_mode_bridge(const or_options_t *options)
-{
-  return authdir_mode(options) && options->BridgeAuthoritativeDir != 0;
-}
-
-/** Return true iff we are trying to be a server.
- */
-MOCK_IMPL(int,
-server_mode,(const or_options_t *options))
-{
-  if (options->ClientOnly) return 0;
-  /* XXXX I believe we can kill off ORListenAddress here.*/
-  return (options->ORPort_set || options->ORListenAddress);
-}
-
-/** Return true iff we are trying to be a non-bridge server.
- */
-MOCK_IMPL(int,
-public_server_mode,(const or_options_t *options))
-{
-  if (!server_mode(options)) return 0;
-  return (!options->BridgeRelay);
-}
-
 /** Return true iff the combination of options in <b>options</b> and parameters
  * in the consensus mean that we don't want to allow exits from circuits
  * we got from addresses not known to be servers. */
@@ -1581,42 +1319,6 @@ should_refuse_unknown_exits(const or_options_t *options)
   }
 }
 
-/** Remember if we've advertised ourselves to the dirservers. */
-static int server_is_advertised=0;
-
-/** Return true iff we have published our descriptor lately.
- */
-MOCK_IMPL(int,
-advertised_server_mode,(void))
-{
-  return server_is_advertised;
-}
-
-/**
- * Called with a boolean: set whether we have recently published our
- * descriptor.
- */
-static void
-set_server_advertised(int s)
-{
-  server_is_advertised = s;
-}
-
-/** Return true iff we are trying to proxy client connections. */
-int
-proxy_mode(const or_options_t *options)
-{
-  (void)options;
-  SMARTLIST_FOREACH_BEGIN(get_configured_ports(), const port_cfg_t *, p) {
-    if (p->type == CONN_TYPE_AP_LISTENER ||
-        p->type == CONN_TYPE_AP_TRANS_LISTENER ||
-        p->type == CONN_TYPE_AP_DNS_LISTENER ||
-        p->type == CONN_TYPE_AP_NATD_LISTENER)
-      return 1;
-  } SMARTLIST_FOREACH_END(p);
-  return 0;
-}
-
 /** Decide if we're a publishable server. We are a publishable server if:
  * - We don't have the ClientOnly option set
  * and
@@ -1762,12 +1464,12 @@ static routerinfo_t *desc_routerinfo = NULL;
 static extrainfo_t *desc_extrainfo = NULL;
 /** Why did we most recently decide to regenerate our descriptor?  Used to
  * tell the authorities why we're sending it to them. */
-static const char *desc_gen_reason = NULL;
+static const char *desc_gen_reason = "uninitialized reason";
 /** Since when has our descriptor been "clean"?  0 if we need to regenerate it
  * now. */
 static time_t desc_clean_since = 0;
 /** Why did we mark the descriptor dirty? */
-static const char *desc_dirty_reason = NULL;
+static const char *desc_dirty_reason = "Tor just started";
 /** Boolean: do we need to regenerate the above? */
 static int desc_needs_upload = 0;
 
@@ -1848,7 +1550,7 @@ router_compare_to_my_exit_policy(const tor_addr_t *addr, uint16_t port)
       desc_routerinfo->ipv6_exit_policy &&
       compare_tor_addr_to_short_policy(addr, port,
                                me->ipv6_exit_policy) != ADDR_POLICY_ACCEPTED;
-#endif
+#endif /* 0 */
   } else {
     return -1;
   }
@@ -1908,10 +1610,43 @@ router_is_me(const routerinfo_t *router)
 MOCK_IMPL(const routerinfo_t *,
 router_get_my_routerinfo,(void))
 {
-  if (!server_mode(get_options()))
+  return router_get_my_routerinfo_with_err(NULL);
+}
+
+/** Return routerinfo of this OR. Rebuild it from
+ * scratch if needed. Set <b>*err</b> to 0 on success or to
+ * appropriate TOR_ROUTERINFO_ERROR_* value on failure.
+ */
+MOCK_IMPL(const routerinfo_t *,
+router_get_my_routerinfo_with_err,(int *err))
+{
+  if (!server_mode(get_options())) {
+    if (err)
+      *err = TOR_ROUTERINFO_ERROR_NOT_A_SERVER;
+
     return NULL;
-  if (router_rebuild_descriptor(0))
+  }
+
+  if (!desc_clean_since) {
+    int rebuild_err = router_rebuild_descriptor(0);
+    if (rebuild_err < 0) {
+      if (err)
+        *err = rebuild_err;
+
+      return NULL;
+    }
+  }
+
+  if (!desc_routerinfo) {
+    if (err)
+      *err = TOR_ROUTERINFO_ERROR_DESC_REBUILDING;
+
     return NULL;
+  }
+
+  if (err)
+    *err = 0;
+
   return desc_routerinfo;
 }
 
@@ -2088,7 +1823,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
 
   if (router_pick_published_address(options, &addr, 0) < 0) {
     log_warn(LD_CONFIG, "Don't know my address while generating descriptor");
-    return -1;
+    return TOR_ROUTERINFO_ERROR_NO_EXT_ADDR;
   }
 
   /* Log a message if the address in the descriptor doesn't match the ORPort
@@ -2104,8 +1839,10 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
   ri->supports_tunnelled_dir_requests =
     directory_permits_begindir_requests(options);
   ri->cache_info.published_on = time(NULL);
-  ri->onion_pkey = crypto_pk_dup_key(get_onion_key()); /* must invoke from
-                                                        * main thread */
+  /* get_onion_key() must invoke from main thread */
+  router_set_rsa_onion_pkey(get_onion_key(), &ri->onion_pkey,
+                            &ri->onion_pkey_len);
+
   ri->onion_curve25519_pkey =
     tor_memdup(&get_current_curve25519_keypair()->pubkey,
                sizeof(curve25519_public_key_t));
@@ -2141,10 +1878,10 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
   }
 
   ri->identity_pkey = crypto_pk_dup_key(get_server_identity_key());
-  if (crypto_pk_get_digest(ri->identity_pkey,
-                           ri->cache_info.identity_digest)<0) {
+  if (BUG(crypto_pk_get_digest(ri->identity_pkey,
+                           ri->cache_info.identity_digest) < 0)) {
     routerinfo_free(ri);
-    return -1;
+    return TOR_ROUTERINFO_ERROR_DIGEST_FAILED;
   }
   ri->cache_info.signing_key_cert =
     tor_cert_dup(get_master_signing_key_cert());
@@ -2160,6 +1897,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
   /* and compute ri->bandwidthburst similarly */
   ri->bandwidthburst = get_effective_bwburst(options);
 
+  /* Report bandwidth, unless we're hibernating or shutting down */
   ri->bandwidthcapacity = hibernating ? 0 : rep_hist_bandwidth_assess();
 
   if (dns_seems_to_be_broken() || has_dns_init_failed()) {
@@ -2181,19 +1919,17 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
   }
 
   if (options->MyFamily && ! options->BridgeRelay) {
-    smartlist_t *family;
     if (!warned_nonexistent_family)
       warned_nonexistent_family = smartlist_new();
-    family = smartlist_new();
     ri->declared_family = smartlist_new();
-    smartlist_split_string(family, options->MyFamily, ",",
-      SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK|SPLIT_STRIP_SPACE, 0);
-    SMARTLIST_FOREACH_BEGIN(family, char *, name) {
+    config_line_t *family;
+    for (family = options->MyFamily; family; family = family->next) {
+       char *name = family->value;
        const node_t *member;
        if (!strcasecmp(name, options->Nickname))
-         goto skip; /* Don't list ourself, that's redundant */
+         continue; /* Don't list ourself, that's redundant */
        else
-         member = node_get_by_nickname(name, 1);
+         member = node_get_by_nickname(name, 0);
        if (!member) {
          int is_legal = is_legal_nickname_or_hexdigest(name);
          if (!smartlist_contains_string(warned_nonexistent_family, name) &&
@@ -2207,11 +1943,10 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
              log_warn(LD_CONFIG, "There is a router named \"%s\" in my "
                       "declared family, but that isn't a legal nickname. "
                       "Skipping it.", escaped(name));
-           smartlist_add(warned_nonexistent_family, tor_strdup(name));
+           smartlist_add_strdup(warned_nonexistent_family, name);
          }
          if (is_legal) {
-           smartlist_add(ri->declared_family, name);
-           name = NULL;
+           smartlist_add_strdup(ri->declared_family, name);
          }
        } else if (router_digest_is_me(member->identity)) {
          /* Don't list ourself in our own family; that's redundant */
@@ -2225,15 +1960,11 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
          if (smartlist_contains_string(warned_nonexistent_family, name))
            smartlist_string_remove(warned_nonexistent_family, name);
        }
-    skip:
-       tor_free(name);
-    } SMARTLIST_FOREACH_END(name);
+    }
 
     /* remove duplicates from the list */
     smartlist_sort_strings(ri->declared_family);
     smartlist_uniq_strings(ri->declared_family);
-
-    smartlist_free(family);
   }
 
   /* Now generate the extrainfo. */
@@ -2284,7 +2015,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
     log_warn(LD_BUG, "Couldn't generate router descriptor.");
     routerinfo_free(ri);
     extrainfo_free(ei);
-    return -1;
+    return TOR_ROUTERINFO_ERROR_CANNOT_GENERATE;
   }
   ri->cache_info.signed_descriptor_len =
     strlen(ri->cache_info.signed_descriptor_body);
@@ -2327,6 +2058,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
 int
 router_rebuild_descriptor(int force)
 {
+  int err = 0;
   routerinfo_t *ri;
   extrainfo_t *ei;
   uint32_t addr;
@@ -2341,13 +2073,14 @@ router_rebuild_descriptor(int force)
      * learn that it's time to try again when ip_address_changed()
      * marks it dirty. */
     desc_clean_since = time(NULL);
-    return -1;
+    return TOR_ROUTERINFO_ERROR_DESC_REBUILDING;
   }
 
   log_info(LD_OR, "Rebuilding relay descriptor%s", force ? " (forced)" : "");
 
-  if (router_build_fresh_descriptor(&ri, &ei) < 0) {
-    return -1;
+  err = router_build_fresh_descriptor(&ri, &ei);
+  if (err < 0) {
+    return err;
   }
 
   routerinfo_free(desc_routerinfo);
@@ -2358,6 +2091,9 @@ router_rebuild_descriptor(int force)
   desc_clean_since = time(NULL);
   desc_needs_upload = 1;
   desc_gen_reason = desc_dirty_reason;
+  if (BUG(desc_gen_reason == NULL)) {
+    desc_gen_reason = "descriptor was marked dirty earlier, for no reason.";
+  }
   desc_dirty_reason = NULL;
   control_event_my_descriptor_changed();
   return 0;
@@ -2414,6 +2150,9 @@ void
 mark_my_descriptor_dirty(const char *reason)
 {
   const or_options_t *options = get_options();
+  if (BUG(reason == NULL)) {
+    reason = "marked descriptor dirty for unspecified reason";
+  }
   if (server_mode(options) && options->PublishServerDescriptor_)
     log_info(LD_OR, "Decided to publish new relay descriptor: %s", reason);
   desc_clean_since = 0;
@@ -2441,20 +2180,24 @@ check_descriptor_bandwidth_changed(time_t now)
 {
   static time_t last_changed = 0;
   uint64_t prev, cur;
-  const routerinfo_t *my_ri = router_get_my_routerinfo();
-
-  int hibernating = we_are_hibernating();
+  const int hibernating = we_are_hibernating();
 
   /* If the relay uptime is bigger than MAX_UPTIME_BANDWIDTH_CHANGE,
    * the next regularly scheduled descriptor update (18h) will be enough */
   if (get_uptime() > MAX_UPTIME_BANDWIDTH_CHANGE && !hibernating)
     return;
 
-  if (!my_ri) /* make sure routerinfo exists */
+  const routerinfo_t *my_ri = router_get_my_routerinfo();
+
+  if (!my_ri)
     return;
 
   prev = my_ri->bandwidthcapacity;
+
+  /* Consider ourselves to have zero bandwidth if we're hibernating or
+   * shutting down. */
   cur = hibernating ? 0 : rep_hist_bandwidth_assess();
+
   if ((prev != cur && (!prev || !cur)) ||
       cur > (prev * BANDWIDTH_CHANGE_FACTOR) ||
       cur < (prev / BANDWIDTH_CHANGE_FACTOR) ) {
@@ -2478,6 +2221,9 @@ log_addr_has_changed(int severity,
   char addrbuf_prev[TOR_ADDR_BUF_LEN];
   char addrbuf_cur[TOR_ADDR_BUF_LEN];
 
+  if (BUG(!server_mode(get_options())))
+    return;
+
   if (tor_addr_to_str(addrbuf_prev, prev, sizeof(addrbuf_prev), 1) == NULL)
     strlcpy(addrbuf_prev, "???", TOR_ADDR_BUF_LEN);
   if (tor_addr_to_str(addrbuf_cur, cur, sizeof(addrbuf_cur), 1) == NULL)
@@ -2648,6 +2394,7 @@ router_dump_router_to_string(routerinfo_t *router,
 {
   char *address = NULL;
   char *onion_pkey = NULL; /* Onion key, PEM-encoded. */
+  crypto_pk_t *rsa_pubkey = NULL;
   char *identity_pkey = NULL; /* Identity key, PEM-encoded. */
   char digest[DIGEST256_LEN];
   char published[ISO_TIME_LEN+1];
@@ -2714,7 +2461,9 @@ router_dump_router_to_string(routerinfo_t *router,
   }
 
   /* PEM-encode the onion key */
-  if (crypto_pk_write_public_key_to_string(router->onion_pkey,
+  rsa_pubkey = router_get_rsa_onion_pkey(router->onion_pkey,
+                                         router->onion_pkey_len);
+  if (crypto_pk_write_public_key_to_string(rsa_pubkey,
                                            &onion_pkey,&onion_pkeylen)<0) {
     log_warn(LD_BUG,"write onion_pkey to string failed!");
     goto err;
@@ -2767,7 +2516,7 @@ router_dump_router_to_string(routerinfo_t *router,
       make_ntor_onion_key_crosscert(ntor_keypair,
                          &router->cache_info.signing_key_cert->signing_key,
                          router->cache_info.published_on,
-                         MIN_ONION_KEY_LIFETIME, &sign);
+                         get_onion_key_lifetime(), &sign);
     if (!cert) {
       log_warn(LD_BUG,"make_ntor_onion_key_crosscert failed!");
       goto err;
@@ -2853,18 +2602,18 @@ router_dump_router_to_string(routerinfo_t *router,
                     "onion-key\n%s"
                     "signing-key\n%s"
                     "%s%s"
-                    "%s%s%s%s",
+                    "%s%s%s",
     router->nickname,
     address,
     router->or_port,
-    decide_to_advertise_dirport(options, router->dir_port),
+    router_should_advertise_dirport(options, router->dir_port),
     ed_cert_line ? ed_cert_line : "",
     extra_or_address ? extra_or_address : "",
     router->platform,
     proto_line,
     published,
     fingerprint,
-    stats_n_seconds_working,
+    get_uptime(),
     (int) router->bandwidthrate,
     (int) router->bandwidthburst,
     (int) router->bandwidthcapacity,
@@ -2876,8 +2625,7 @@ router_dump_router_to_string(routerinfo_t *router,
     ntor_cc_line ? ntor_cc_line : "",
     family_line,
     we_are_hibernating() ? "hibernating 1\n" : "",
-    "hidden-service-dir\n",
-    options->AllowSingleHopExits ? "allow-single-hop-exits\n" : "");
+    "hidden-service-dir\n");
 
   if (options->ContactInfo && strlen(options->ContactInfo)) {
     const char *ci = options->ContactInfo;
@@ -2888,10 +2636,13 @@ router_dump_router_to_string(routerinfo_t *router,
 
   if (options->BridgeRelay) {
     const char *bd;
-    if (options->PublishServerDescriptor_ & BRIDGE_DIRINFO)
+    if (options->BridgeDistribution && strlen(options->BridgeDistribution)) {
+      bd = options->BridgeDistribution;
+    } else {
       bd = "any";
-    else
-      bd = "none";
+    }
+    if (strchr(bd, '\n') || strchr(bd, '\r'))
+      bd = escaped(bd);
     smartlist_add_asprintf(chunks, "bridge-distribution-request %s\n", bd);
   }
 
@@ -2909,7 +2660,7 @@ router_dump_router_to_string(routerinfo_t *router,
 
   /* Write the exit policy to the end of 's'. */
   if (!router->exit_policy || !smartlist_len(router->exit_policy)) {
-    smartlist_add(chunks, tor_strdup("reject *:*\n"));
+    smartlist_add_strdup(chunks, "reject *:*\n");
   } else if (router->exit_policy) {
     char *exit_policy = router_dump_exit_policy_to_string(router,1,0);
 
@@ -2929,14 +2680,14 @@ router_dump_router_to_string(routerinfo_t *router,
     tor_free(p6);
   }
 
-  if (decide_to_advertise_begindir(options,
+  if (router_should_advertise_begindir(options,
                                    router->supports_tunnelled_dir_requests)) {
-    smartlist_add(chunks, tor_strdup("tunnelled-dir-server\n"));
+    smartlist_add_strdup(chunks, "tunnelled-dir-server\n");
   }
 
   /* Sign the descriptor with Ed25519 */
   if (emit_ed_sigs)  {
-    smartlist_add(chunks, tor_strdup("router-sig-ed25519 "));
+    smartlist_add_strdup(chunks, "router-sig-ed25519 ");
     crypto_digest_smartlist_prefix(digest, DIGEST256_LEN,
                                    ED_DESC_SIGNATURE_PREFIX,
                                    chunks, "", DIGEST_SHA256);
@@ -2952,11 +2703,10 @@ router_dump_router_to_string(routerinfo_t *router,
   }
 
   /* Sign the descriptor with RSA */
-  smartlist_add(chunks, tor_strdup("router-signature\n"));
+  smartlist_add_strdup(chunks, "router-signature\n");
 
   crypto_digest_smartlist(digest, DIGEST_LEN, chunks, "", DIGEST_SHA1);
 
-  note_crypto_pk_op(SIGN_RTR);
   {
     char *sig;
     if (!(sig = router_get_dirobj_signature(digest, DIGEST_LEN, ident_key))) {
@@ -2967,7 +2717,7 @@ router_dump_router_to_string(routerinfo_t *router,
   }
 
   /* include a last '\n' */
-  smartlist_add(chunks, tor_strdup("\n"));
+  smartlist_add_strdup(chunks, "\n");
 
   output = smartlist_join_strings(chunks, "", 0, NULL);
 
@@ -2987,7 +2737,7 @@ router_dump_router_to_string(routerinfo_t *router,
     tor_free(s_dup);
     routerinfo_free(ri_tmp);
   }
-#endif
+#endif /* defined(DEBUG_ROUTER_DUMP_ROUTER_TO_STRING) */
 
   goto done;
 
@@ -2998,6 +2748,7 @@ router_dump_router_to_string(routerinfo_t *router,
     SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
     smartlist_free(chunks);
   }
+  crypto_pk_free(rsa_pubkey);
   tor_free(address);
   tor_free(family_line);
   tor_free(onion_pkey);
@@ -3031,36 +2782,6 @@ router_dump_exit_policy_to_string(const routerinfo_t *router,
                                include_ipv6);
 }
 
-/** Copy the primary (IPv4) OR port (IP address and TCP port) for
- * <b>router</b> into *<b>ap_out</b>. */
-void
-router_get_prim_orport(const routerinfo_t *router, tor_addr_port_t *ap_out)
-{
-  tor_assert(ap_out != NULL);
-  tor_addr_from_ipv4h(&ap_out->addr, router->addr);
-  ap_out->port = router->or_port;
-}
-
-/** Return 1 if any of <b>router</b>'s addresses are <b>addr</b>.
- *   Otherwise return 0. */
-int
-router_has_addr(const routerinfo_t *router, const tor_addr_t *addr)
-{
-  return
-    tor_addr_eq_ipv4h(addr, router->addr) ||
-    tor_addr_eq(&router->ipv6_addr, addr);
-}
-
-int
-router_has_orport(const routerinfo_t *router, const tor_addr_port_t *orport)
-{
-  return
-    (tor_addr_eq_ipv4h(&orport->addr, router->addr) &&
-     orport->port == router->or_port) ||
-    (tor_addr_eq(&orport->addr, &router->ipv6_addr) &&
-     orport->port == router->ipv6_orport);
-}
-
 /** Load the contents of <b>filename</b>, find the last line starting with
  * <b>end_line</b>, ensure that its timestamp is not more than 25 hours in
  * the past or more than 1 hour in the future with respect to <b>now</b>,
@@ -3213,6 +2934,11 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
                         "conn-bi-direct", now, &contents) > 0) {
       smartlist_add(chunks, contents);
     }
+    if (options->PaddingStatistics) {
+      contents = rep_hist_get_padding_count_lines();
+      if (contents)
+        smartlist_add(chunks, contents);
+    }
   }
 
   /* Add information about the pluggable transports we support. */
@@ -3225,13 +2951,13 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
   if (should_record_bridge_info(options) && write_stats_to_extrainfo) {
     const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now);
     if (bridge_stats) {
-      smartlist_add(chunks, tor_strdup(bridge_stats));
+      smartlist_add_strdup(chunks, bridge_stats);
     }
   }
 
   if (emit_ed_sigs) {
     char sha256_digest[DIGEST256_LEN];
-    smartlist_add(chunks, tor_strdup("router-sig-ed25519 "));
+    smartlist_add_strdup(chunks, "router-sig-ed25519 ");
     crypto_digest_smartlist_prefix(sha256_digest, DIGEST256_LEN,
                                    ED_DESC_SIGNATURE_PREFIX,
                                    chunks, "", DIGEST_SHA256);
@@ -3246,7 +2972,7 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
     smartlist_add_asprintf(chunks, "%s\n", buf);
   }
 
-  smartlist_add(chunks, tor_strdup("router-signature\n"));
+  smartlist_add_strdup(chunks, "router-signature\n");
   s = smartlist_join_strings(chunks, "", 0, NULL);
 
   while (strlen(s) > MAX_EXTRAINFO_UPLOAD_SIZE - DIROBJ_MAX_SIG_LEN) {
@@ -3281,7 +3007,7 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
                      "descriptor.");
     goto err;
   }
-  smartlist_add(chunks, tor_strdup(sig));
+  smartlist_add_strdup(chunks, sig);
   tor_free(s);
   s = smartlist_join_strings(chunks, "", 0, NULL);
 
@@ -3324,257 +3050,6 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
   return result;
 }
 
-/** Return true iff <b>s</b> is a valid server nickname. (That is, a string
- * containing between 1 and MAX_NICKNAME_LEN characters from
- * LEGAL_NICKNAME_CHARACTERS.) */
-int
-is_legal_nickname(const char *s)
-{
-  size_t len;
-  tor_assert(s);
-  len = strlen(s);
-  return len > 0 && len <= MAX_NICKNAME_LEN &&
-    strspn(s,LEGAL_NICKNAME_CHARACTERS) == len;
-}
-
-/** Return true iff <b>s</b> is a valid server nickname or
- * hex-encoded identity-key digest. */
-int
-is_legal_nickname_or_hexdigest(const char *s)
-{
-  if (*s!='$')
-    return is_legal_nickname(s);
-  else
-    return is_legal_hexdigest(s);
-}
-
-/** Return true iff <b>s</b> is a valid hex-encoded identity-key
- * digest. (That is, an optional $, followed by 40 hex characters,
- * followed by either nothing, or = or ~ followed by a nickname, or
- * a character other than =, ~, or a hex character.)
- */
-int
-is_legal_hexdigest(const char *s)
-{
-  size_t len;
-  tor_assert(s);
-  if (s[0] == '$') s++;
-  len = strlen(s);
-  if (len > HEX_DIGEST_LEN) {
-    if (s[HEX_DIGEST_LEN] == '=' ||
-        s[HEX_DIGEST_LEN] == '~') {
-      if (!is_legal_nickname(s+HEX_DIGEST_LEN+1))
-        return 0;
-    } else {
-      return 0;
-    }
-  }
-  return (len >= HEX_DIGEST_LEN &&
-          strspn(s,HEX_CHARACTERS)==HEX_DIGEST_LEN);
-}
-
-/** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to
- * hold a human-readable description of a node with identity digest
- * <b>id_digest</b>, named-status <b>is_named</b>, nickname <b>nickname</b>,
- * and address <b>addr</b> or <b>addr32h</b>.
- *
- * The <b>nickname</b> and <b>addr</b> fields are optional and may be set to
- * NULL.  The <b>addr32h</b> field is optional and may be set to 0.
- *
- * Return a pointer to the front of <b>buf</b>.
- */
-const char *
-format_node_description(char *buf,
-                        const char *id_digest,
-                        int is_named,
-                        const char *nickname,
-                        const tor_addr_t *addr,
-                        uint32_t addr32h)
-{
-  char *cp;
-
-  if (!buf)
-    return "<NULL BUFFER>";
-
-  buf[0] = '$';
-  base16_encode(buf+1, HEX_DIGEST_LEN+1, id_digest, DIGEST_LEN);
-  cp = buf+1+HEX_DIGEST_LEN;
-  if (nickname) {
-    buf[1+HEX_DIGEST_LEN] = is_named ? '=' : '~';
-    strlcpy(buf+1+HEX_DIGEST_LEN+1, nickname, MAX_NICKNAME_LEN+1);
-    cp += strlen(cp);
-  }
-  if (addr32h || addr) {
-    memcpy(cp, " at ", 4);
-    cp += 4;
-    if (addr) {
-      tor_addr_to_str(cp, addr, TOR_ADDR_BUF_LEN, 0);
-    } else {
-      struct in_addr in;
-      in.s_addr = htonl(addr32h);
-      tor_inet_ntoa(&in, cp, INET_NTOA_BUF_LEN);
-    }
-  }
-  return buf;
-}
-
-/** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to
- * hold a human-readable description of <b>ri</b>.
- *
- *
- * Return a pointer to the front of <b>buf</b>.
- */
-const char *
-router_get_description(char *buf, const routerinfo_t *ri)
-{
-  if (!ri)
-    return "<null>";
-  return format_node_description(buf,
-                                 ri->cache_info.identity_digest,
-                                 router_is_named(ri),
-                                 ri->nickname,
-                                 NULL,
-                                 ri->addr);
-}
-
-/** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to
- * hold a human-readable description of <b>node</b>.
- *
- * Return a pointer to the front of <b>buf</b>.
- */
-const char *
-node_get_description(char *buf, const node_t *node)
-{
-  const char *nickname = NULL;
-  uint32_t addr32h = 0;
-  int is_named = 0;
-
-  if (!node)
-    return "<null>";
-
-  if (node->rs) {
-    nickname = node->rs->nickname;
-    is_named = node->rs->is_named;
-    addr32h = node->rs->addr;
-  } else if (node->ri) {
-    nickname = node->ri->nickname;
-    addr32h = node->ri->addr;
-  }
-
-  return format_node_description(buf,
-                                 node->identity,
-                                 is_named,
-                                 nickname,
-                                 NULL,
-                                 addr32h);
-}
-
-/** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to
- * hold a human-readable description of <b>rs</b>.
- *
- * Return a pointer to the front of <b>buf</b>.
- */
-const char *
-routerstatus_get_description(char *buf, const routerstatus_t *rs)
-{
-  if (!rs)
-    return "<null>";
-  return format_node_description(buf,
-                                 rs->identity_digest,
-                                 rs->is_named,
-                                 rs->nickname,
-                                 NULL,
-                                 rs->addr);
-}
-
-/** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to
- * hold a human-readable description of <b>ei</b>.
- *
- * Return a pointer to the front of <b>buf</b>.
- */
-const char *
-extend_info_get_description(char *buf, const extend_info_t *ei)
-{
-  if (!ei)
-    return "<null>";
-  return format_node_description(buf,
-                                 ei->identity_digest,
-                                 0,
-                                 ei->nickname,
-                                 &ei->addr,
-                                 0);
-}
-
-/** Return a human-readable description of the routerinfo_t <b>ri</b>.
- *
- * This function is not thread-safe.  Each call to this function invalidates
- * previous values returned by this function.
- */
-const char *
-router_describe(const routerinfo_t *ri)
-{
-  static char buf[NODE_DESC_BUF_LEN];
-  return router_get_description(buf, ri);
-}
-
-/** Return a human-readable description of the node_t <b>node</b>.
- *
- * This function is not thread-safe.  Each call to this function invalidates
- * previous values returned by this function.
- */
-const char *
-node_describe(const node_t *node)
-{
-  static char buf[NODE_DESC_BUF_LEN];
-  return node_get_description(buf, node);
-}
-
-/** Return a human-readable description of the routerstatus_t <b>rs</b>.
- *
- * This function is not thread-safe.  Each call to this function invalidates
- * previous values returned by this function.
- */
-const char *
-routerstatus_describe(const routerstatus_t *rs)
-{
-  static char buf[NODE_DESC_BUF_LEN];
-  return routerstatus_get_description(buf, rs);
-}
-
-/** Return a human-readable description of the extend_info_t <b>ri</b>.
- *
- * This function is not thread-safe.  Each call to this function invalidates
- * previous values returned by this function.
- */
-const char *
-extend_info_describe(const extend_info_t *ei)
-{
-  static char buf[NODE_DESC_BUF_LEN];
-  return extend_info_get_description(buf, ei);
-}
-
-/** Set <b>buf</b> (which must have MAX_VERBOSE_NICKNAME_LEN+1 bytes) to the
- * verbose representation of the identity of <b>router</b>.  The format is:
- *  A dollar sign.
- *  The upper-case hexadecimal encoding of the SHA1 hash of router's identity.
- *  A "=" if the router is named; a "~" if it is not.
- *  The router's nickname.
- **/
-void
-router_get_verbose_nickname(char *buf, const routerinfo_t *router)
-{
-  const char *good_digest = networkstatus_get_router_digest_by_nickname(
-                                                         router->nickname);
-  int is_named = good_digest && tor_memeq(good_digest,
-                                        router->cache_info.identity_digest,
-                                        DIGEST_LEN);
-  buf[0] = '$';
-  base16_encode(buf+1, HEX_DIGEST_LEN+1, router->cache_info.identity_digest,
-                DIGEST_LEN);
-  buf[1+HEX_DIGEST_LEN] = is_named ? '=' : '~';
-  strlcpy(buf+1+HEX_DIGEST_LEN+1, router->nickname, MAX_NICKNAME_LEN+1);
-}
-
 /** Forget that we have issued any router-related warnings, so that we'll
  * warn again if we see the same errors. */
 void
@@ -3586,37 +3061,6 @@ router_reset_warnings(void)
   }
 }
 
-/** Given a router purpose, convert it to a string.  Don't call this on
- * ROUTER_PURPOSE_UNKNOWN: The whole point of that value is that we don't
- * know its string representation. */
-const char *
-router_purpose_to_string(uint8_t p)
-{
-  switch (p)
-    {
-    case ROUTER_PURPOSE_GENERAL: return "general";
-    case ROUTER_PURPOSE_BRIDGE: return "bridge";
-    case ROUTER_PURPOSE_CONTROLLER: return "controller";
-    default:
-      tor_assert(0);
-    }
-  return NULL;
-}
-
-/** Given a string, convert it to a router purpose. */
-uint8_t
-router_purpose_from_string(const char *s)
-{
-  if (!strcmp(s, "general"))
-    return ROUTER_PURPOSE_GENERAL;
-  else if (!strcmp(s, "bridge"))
-    return ROUTER_PURPOSE_BRIDGE;
-  else if (!strcmp(s, "controller"))
-    return ROUTER_PURPOSE_CONTROLLER;
-  else
-    return ROUTER_PURPOSE_UNKNOWN;
-}
-
 /** Release all static resources held in router.c */
 void
 router_free_all(void)
@@ -3625,6 +3069,7 @@ router_free_all(void)
   crypto_pk_free(lastonionkey);
   crypto_pk_free(server_identitykey);
   crypto_pk_free(client_identitykey);
+
   tor_mutex_free(key_lock);
   routerinfo_free(desc_routerinfo);
   extrainfo_free(desc_extrainfo);
@@ -3641,18 +3086,41 @@ router_free_all(void)
     smartlist_free(warned_nonexistent_family);
   }
 }
+/* From the given RSA key object, convert it to ASN-1 encoded format and set
+ * the newly allocated object in onion_pkey_out. The length of the key is set
+ * in onion_pkey_len_out. */
+void
+router_set_rsa_onion_pkey(const crypto_pk_t *pk, char **onion_pkey_out,
+                          size_t *onion_pkey_len_out)
+{
+  int len;
+  char buf[1024];
+
+  tor_assert(pk);
+  tor_assert(onion_pkey_out);
+  tor_assert(onion_pkey_len_out);
+
+  len = crypto_pk_asn1_encode(pk, buf, sizeof(buf));
+  if (BUG(len < 0)) {
+    goto done;
+  }
+
+  *onion_pkey_out = tor_memdup(buf, len);
+  *onion_pkey_len_out = len;
+
+ done:
+  return;
+}
 
-/** Return a smartlist of tor_addr_port_t's with all the OR ports of
-    <b>ri</b>. Note that freeing of the items in the list as well as
-    the smartlist itself is the callers responsibility. */
-smartlist_t *
-router_get_all_orports(const routerinfo_t *ri)
+/* From an ASN-1 encoded onion pkey, return a newly allocated RSA key object.
+ * It is the caller responsability to free the returned object.
+ *
+ * Return NULL if the pkey is NULL, malformed or if the length is 0. */
+crypto_pk_t *
+router_get_rsa_onion_pkey(const char *pkey, size_t pkey_len)
 {
-  tor_assert(ri);
-  node_t fake_node;
-  memset(&fake_node, 0, sizeof(fake_node));
-  /* we don't modify ri, fake_node is passed as a const node_t *
-   */
-  fake_node.ri = (routerinfo_t *)ri;
-  return node_get_all_orports(&fake_node);
+  if (!pkey || pkey_len == 0) {
+    return NULL;
+  }
+  return crypto_pk_asn1_decode(pkey, pkey_len);
 }
diff --git a/src/feature/relay/router.h b/src/feature/relay/router.h
new file mode 100644
index 0000000000..bd6a8a012e
--- /dev/null
+++ b/src/feature/relay/router.h
@@ -0,0 +1,122 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file router.h
+ * \brief Header file for router.c.
+ **/
+
+#ifndef TOR_ROUTER_H
+#define TOR_ROUTER_H
+
+#include "lib/testsupport/testsupport.h"
+
+struct curve25519_keypair_t;
+struct ed25519_keypair_t;
+
+#define TOR_ROUTERINFO_ERROR_NO_EXT_ADDR     (-1)
+#define TOR_ROUTERINFO_ERROR_CANNOT_PARSE    (-2)
+#define TOR_ROUTERINFO_ERROR_NOT_A_SERVER    (-3)
+#define TOR_ROUTERINFO_ERROR_DIGEST_FAILED   (-4)
+#define TOR_ROUTERINFO_ERROR_CANNOT_GENERATE (-5)
+#define TOR_ROUTERINFO_ERROR_DESC_REBUILDING (-6)
+
+crypto_pk_t *get_onion_key(void);
+time_t get_onion_key_set_at(void);
+void set_server_identity_key(crypto_pk_t *k);
+crypto_pk_t *get_server_identity_key(void);
+int server_identity_key_is_set(void);
+void set_client_identity_key(crypto_pk_t *k);
+crypto_pk_t *get_tlsclient_identity_key(void);
+int client_identity_key_is_set(void);
+MOCK_DECL(authority_cert_t *, get_my_v3_authority_cert, (void));
+crypto_pk_t *get_my_v3_authority_signing_key(void);
+authority_cert_t *get_my_v3_legacy_cert(void);
+crypto_pk_t *get_my_v3_legacy_signing_key(void);
+void dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last);
+void expire_old_onion_keys(void);
+void rotate_onion_key(void);
+void v3_authority_check_key_expiry(void);
+int get_onion_key_lifetime(void);
+int get_onion_key_grace_period(void);
+
+crypto_pk_t *router_get_rsa_onion_pkey(const char *pkey, size_t pkey_len);
+void router_set_rsa_onion_pkey(const crypto_pk_t *pk, char **onion_pkey_out,
+                               size_t *onion_pkey_len);
+
+di_digest256_map_t *construct_ntor_key_map(void);
+void ntor_key_map_free_(di_digest256_map_t *map);
+#define ntor_key_map_free(map) \
+  FREE_AND_NULL(di_digest256_map_t, ntor_key_map_free_, (map))
+
+int router_initialize_tls_context(void);
+int init_keys(void);
+int init_keys_client(void);
+
+uint16_t router_get_active_listener_port_by_type_af(int listener_type,
+                                                    sa_family_t family);
+uint16_t router_get_advertised_or_port(const or_options_t *options);
+uint16_t router_get_advertised_or_port_by_af(const or_options_t *options,
+                                             sa_family_t family);
+uint16_t router_get_advertised_dir_port(const or_options_t *options,
+                                        uint16_t dirport);
+
+int router_should_advertise_dirport(const or_options_t *options,
+                                    uint16_t dir_port);
+
+void consider_publishable_server(int force);
+int should_refuse_unknown_exits(const or_options_t *options);
+
+void router_upload_dir_desc_to_dirservers(int force);
+void mark_my_descriptor_dirty_if_too_old(time_t now);
+void mark_my_descriptor_dirty(const char *reason);
+void check_descriptor_bandwidth_changed(time_t now);
+void check_descriptor_ipaddress_changed(time_t now);
+int router_has_bandwidth_to_be_dirserver(const or_options_t *options);
+void router_new_address_suggestion(const char *suggestion,
+                                   const dir_connection_t *d_conn);
+int router_compare_to_my_exit_policy(const tor_addr_t *addr, uint16_t port);
+MOCK_DECL(int, router_my_exit_policy_is_reject_star,(void));
+MOCK_DECL(const routerinfo_t *, router_get_my_routerinfo, (void));
+MOCK_DECL(const routerinfo_t *, router_get_my_routerinfo_with_err,(int *err));
+extrainfo_t *router_get_my_extrainfo(void);
+const char *router_get_my_descriptor(void);
+const char *router_get_descriptor_gen_reason(void);
+int router_digest_is_me(const char *digest);
+const uint8_t *router_get_my_id_digest(void);
+int router_extrainfo_digest_is_me(const char *digest);
+int router_is_me(const routerinfo_t *router);
+MOCK_DECL(int,router_pick_published_address,(const or_options_t *options,
+                                             uint32_t *addr,
+                                             int cache_only));
+int router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e);
+int router_rebuild_descriptor(int force);
+char *router_dump_router_to_string(routerinfo_t *router,
+                             const crypto_pk_t *ident_key,
+                             const crypto_pk_t *tap_key,
+                             const struct curve25519_keypair_t *ntor_keypair,
+                             const struct ed25519_keypair_t *signing_keypair);
+char *router_dump_exit_policy_to_string(const routerinfo_t *router,
+                                         int include_ipv4,
+                                         int include_ipv6);
+int extrainfo_dump_to_string(char **s, extrainfo_t *extrainfo,
+                             crypto_pk_t *ident_key,
+                             const struct ed25519_keypair_t *signing_keypair);
+
+const char *routerinfo_err_to_string(int err);
+int routerinfo_err_is_transient(int err);
+
+void router_reset_warnings(void);
+void router_reset_reachability(void);
+void router_free_all(void);
+
+#ifdef ROUTER_PRIVATE
+/* Used only by router.c and test.c */
+STATIC void get_platform_str(char *platform, size_t len);
+STATIC int router_write_fingerprint(int hashed);
+#endif
+
+#endif /* !defined(TOR_ROUTER_H) */
diff --git a/src/feature/relay/routerkeys.c b/src/feature/relay/routerkeys.c
new file mode 100644
index 0000000000..f639fc91e7
--- /dev/null
+++ b/src/feature/relay/routerkeys.c
@@ -0,0 +1,740 @@
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file routerkeys.c
+ *
+ * \brief Functions and structures to handle generating and maintaining the
+ *  set of keypairs necessary to be an OR.
+ *
+ * The keys handled here now are the Ed25519 keys that Tor relays use to sign
+ * descriptors, authenticate themselves on links, and identify one another
+ * uniquely.  Other keys are maintained in router.c and rendservice.c.
+ *
+ * (TODO: The keys in router.c should go here too.)
+ */
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routerkeys.h"
+#include "feature/relay/routermode.h"
+#include "feature/keymgt/loadkey.h"
+#include "feature/nodelist/torcert.h"
+
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/tls/tortls.h"
+#include "lib/tls/x509.h"
+
+#define ENC_KEY_HEADER "Boxed Ed25519 key"
+#define ENC_KEY_TAG "master"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+static ed25519_keypair_t *master_identity_key = NULL;
+static ed25519_keypair_t *master_signing_key = NULL;
+static ed25519_keypair_t *current_auth_key = NULL;
+static tor_cert_t *signing_key_cert = NULL;
+static tor_cert_t *link_cert_cert = NULL;
+static tor_cert_t *auth_key_cert = NULL;
+
+static uint8_t *rsa_ed_crosscert = NULL;
+static size_t rsa_ed_crosscert_len = 0;
+static time_t rsa_ed_crosscert_expiration = 0;
+
+/**
+ * Running as a server: load, reload, or refresh our ed25519 keys and
+ * certificates, creating and saving new ones as needed.
+ *
+ * Return -1 on failure; 0 on success if the signing key was not replaced;
+ * and 1 on success if the signing key was replaced.
+ */
+int
+load_ed_keys(const or_options_t *options, time_t now)
+{
+  ed25519_keypair_t *id = NULL;
+  ed25519_keypair_t *sign = NULL;
+  ed25519_keypair_t *auth = NULL;
+  const ed25519_keypair_t *sign_signing_key_with_id = NULL;
+  const ed25519_keypair_t *use_signing = NULL;
+  const tor_cert_t *check_signing_cert = NULL;
+  tor_cert_t *sign_cert = NULL;
+  tor_cert_t *auth_cert = NULL;
+  int signing_key_changed = 0;
+
+  // It is later than 1972, since otherwise there would be no C compilers.
+  // (Try to diagnose #22466.)
+  tor_assert_nonfatal(now >= 2 * 365 * 86400);
+
+#define FAIL(msg) do {                          \
+    log_warn(LD_OR, (msg));                     \
+    goto err;                                   \
+  } while (0)
+#define SET_KEY(key, newval) do {               \
+    if ((key) != (newval))                      \
+      ed25519_keypair_free(key);                \
+    key = (newval);                             \
+  } while (0)
+#define SET_CERT(cert, newval) do {             \
+    if ((cert) != (newval))                     \
+      tor_cert_free(cert);                      \
+    cert = (newval);                            \
+  } while (0)
+#define HAPPENS_SOON(when, interval)            \
+  ((when) < now + (interval))
+#define EXPIRES_SOON(cert, interval)            \
+  (!(cert) || HAPPENS_SOON((cert)->valid_until, (interval)))
+
+  /* XXXX support encrypted identity keys fully */
+
+  /* First try to get the signing key to see how it is. */
+  {
+    char *fname =
+      options_get_keydir_fname(options, "ed25519_signing");
+    sign = ed_key_init_from_file(
+               fname,
+               INIT_ED_KEY_NEEDCERT|
+               INIT_ED_KEY_INCLUDE_SIGNING_KEY_IN_CERT,
+               LOG_INFO,
+               NULL, 0, 0, CERT_TYPE_ID_SIGNING, &sign_cert, options);
+    tor_free(fname);
+    check_signing_cert = sign_cert;
+    use_signing = sign;
+  }
+
+  if (use_signing) {
+    /* We loaded a signing key with its certificate.  */
+    if (! master_signing_key) {
+      /* We didn't know one before! */
+      signing_key_changed = 1;
+    } else if (! ed25519_pubkey_eq(&use_signing->pubkey,
+                                   &master_signing_key->pubkey) ||
+               ! tor_memeq(use_signing->seckey.seckey,
+                           master_signing_key->seckey.seckey,
+                           ED25519_SECKEY_LEN)) {
+      /* We loaded a different signing key than the one we knew before. */
+      signing_key_changed = 1;
+    }
+  }
+
+  if (!use_signing && master_signing_key) {
+    /* We couldn't load a signing key, but we already had one loaded */
+    check_signing_cert = signing_key_cert;
+    use_signing = master_signing_key;
+  }
+
+  const int offline_master =
+    options->OfflineMasterKey && options->command != CMD_KEYGEN;
+  const int need_new_signing_key =
+    NULL == use_signing ||
+    EXPIRES_SOON(check_signing_cert, 0) ||
+    (options->command == CMD_KEYGEN && ! options->change_key_passphrase);
+  const int want_new_signing_key =
+    need_new_signing_key ||
+    EXPIRES_SOON(check_signing_cert, options->TestingSigningKeySlop);
+
+  /* We can only create a master key if we haven't been told that the
+   * master key will always be offline.  Also, if we have a signing key,
+   * then we shouldn't make a new master ID key. */
+  const int can_make_master_id_key = !offline_master &&
+    NULL == use_signing;
+
+  if (need_new_signing_key) {
+    log_notice(LD_OR, "It looks like I need to generate and sign a new "
+               "medium-term signing key, because %s. To do that, I "
+               "need to load%s the permanent master identity key. "
+               "If the master identity key was not moved or encrypted "
+               "with a passphrase, this will be done automatically and "
+               "no further action is required. Otherwise, provide the "
+               "necessary data using 'tor --keygen' to do it manually.",
+            (NULL == use_signing) ? "I don't have one" :
+            EXPIRES_SOON(check_signing_cert, 0) ? "the one I have is expired" :
+               "you asked me to make one with --keygen",
+            can_make_master_id_key ? " (or create)" : "");
+  } else if (want_new_signing_key && !offline_master) {
+    log_notice(LD_OR, "It looks like I should try to generate and sign a "
+               "new medium-term signing key, because the one I have is "
+               "going to expire soon. To do that, I'm going to have to "
+               "try to load the permanent master identity key. "
+               "If the master identity key was not moved or encrypted "
+               "with a passphrase, this will be done automatically and "
+               "no further action is required. Otherwise, provide the "
+               "necessary data using 'tor --keygen' to do it manually.");
+  } else if (want_new_signing_key) {
+    log_notice(LD_OR, "It looks like I should try to generate and sign a "
+               "new medium-term signing key, because the one I have is "
+               "going to expire soon. But OfflineMasterKey is set, so I "
+               "won't try to load a permanent master identity key. You "
+               "will need to use 'tor --keygen' to make a new signing "
+               "key and certificate.");
+  }
+
+  {
+    uint32_t flags =
+      (INIT_ED_KEY_SPLIT|
+       INIT_ED_KEY_EXTRA_STRONG|INIT_ED_KEY_NO_REPAIR);
+    if (can_make_master_id_key)
+      flags |= INIT_ED_KEY_CREATE;
+    if (! need_new_signing_key)
+      flags |= INIT_ED_KEY_MISSING_SECRET_OK;
+    if (! want_new_signing_key || offline_master)
+      flags |= INIT_ED_KEY_OMIT_SECRET;
+    if (offline_master)
+      flags |= INIT_ED_KEY_OFFLINE_SECRET;
+    if (options->command == CMD_KEYGEN)
+      flags |= INIT_ED_KEY_TRY_ENCRYPTED;
+
+    /* Check/Create the key directory */
+    if (create_keys_directory(options) < 0)
+      goto err;
+
+    char *fname;
+    if (options->master_key_fname) {
+      fname = tor_strdup(options->master_key_fname);
+      flags |= INIT_ED_KEY_EXPLICIT_FNAME;
+    } else {
+      fname = options_get_keydir_fname(options, "ed25519_master_id");
+    }
+    id = ed_key_init_from_file(
+             fname,
+             flags,
+             LOG_WARN, NULL, 0, 0, 0, NULL, options);
+    tor_free(fname);
+    if (!id) {
+      if (need_new_signing_key) {
+        if (offline_master)
+          FAIL("Can't load master identity key; OfflineMasterKey is set.");
+        else
+          FAIL("Missing identity key");
+      } else {
+        log_warn(LD_OR, "Master public key was absent; inferring from "
+                 "public key in signing certificate and saving to disk.");
+        tor_assert(check_signing_cert);
+        id = tor_malloc_zero(sizeof(*id));
+        memcpy(&id->pubkey, &check_signing_cert->signing_key,
+               sizeof(ed25519_public_key_t));
+        fname = options_get_keydir_fname(options,
+                                         "ed25519_master_id_public_key");
+        if (ed25519_pubkey_write_to_file(&id->pubkey, fname, "type0") < 0) {
+          log_warn(LD_OR, "Error while attempting to write master public key "
+                   "to disk");
+          tor_free(fname);
+          goto err;
+        }
+        tor_free(fname);
+      }
+    }
+    if (tor_mem_is_zero((char*)id->seckey.seckey, sizeof(id->seckey)))
+      sign_signing_key_with_id = NULL;
+    else
+      sign_signing_key_with_id = id;
+  }
+
+  if (master_identity_key &&
+      !ed25519_pubkey_eq(&id->pubkey, &master_identity_key->pubkey)) {
+    FAIL("Identity key on disk does not match key we loaded earlier!");
+  }
+
+  if (need_new_signing_key && NULL == sign_signing_key_with_id)
+    FAIL("Can't load master key make a new signing key.");
+
+  if (sign_cert) {
+    if (! sign_cert->signing_key_included)
+      FAIL("Loaded a signing cert with no key included!");
+    if (! ed25519_pubkey_eq(&sign_cert->signing_key, &id->pubkey))
+      FAIL("The signing cert we have was not signed with the master key "
+           "we loaded!");
+    if (tor_cert_checksig(sign_cert, &id->pubkey, 0) < 0) {
+      log_warn(LD_OR, "The signing cert we loaded was not signed "
+               "correctly: %s!",
+               tor_cert_describe_signature_status(sign_cert));
+      goto err;
+    }
+  }
+
+  if (want_new_signing_key && sign_signing_key_with_id) {
+    uint32_t flags = (INIT_ED_KEY_CREATE|
+                      INIT_ED_KEY_REPLACE|
+                      INIT_ED_KEY_EXTRA_STRONG|
+                      INIT_ED_KEY_NEEDCERT|
+                      INIT_ED_KEY_INCLUDE_SIGNING_KEY_IN_CERT);
+    char *fname =
+      options_get_keydir_fname(options, "ed25519_signing");
+    ed25519_keypair_free(sign);
+    tor_cert_free(sign_cert);
+    sign = ed_key_init_from_file(fname,
+                                 flags, LOG_WARN,
+                                 sign_signing_key_with_id, now,
+                                 options->SigningKeyLifetime,
+                                 CERT_TYPE_ID_SIGNING, &sign_cert, options);
+    tor_free(fname);
+    if (!sign)
+      FAIL("Missing signing key");
+    use_signing = sign;
+    signing_key_changed = 1;
+
+    tor_assert(sign_cert->signing_key_included);
+    tor_assert(ed25519_pubkey_eq(&sign_cert->signing_key, &id->pubkey));
+    tor_assert(ed25519_pubkey_eq(&sign_cert->signed_key, &sign->pubkey));
+  } else if (want_new_signing_key) {
+    static ratelim_t missing_master = RATELIM_INIT(3600);
+    log_fn_ratelim(&missing_master, LOG_WARN, LD_OR,
+                   "Signing key will expire soon, but I can't load the "
+                   "master key to sign a new one!");
+  }
+
+  tor_assert(use_signing);
+
+  /* At this point we no longer need our secret identity key.  So wipe
+   * it, if we loaded it in the first place. */
+  memwipe(id->seckey.seckey, 0, sizeof(id->seckey));
+
+  if (options->command == CMD_KEYGEN)
+    goto end;
+
+  if (server_mode(options) &&
+      (!rsa_ed_crosscert ||
+       HAPPENS_SOON(rsa_ed_crosscert_expiration, 30*86400))) {
+    uint8_t *crosscert;
+    time_t expiration = now+6*30*86400; /* 6 months in the future. */
+    ssize_t crosscert_len = tor_make_rsa_ed25519_crosscert(&id->pubkey,
+                                                   get_server_identity_key(),
+                                                   expiration,
+                                                   &crosscert);
+    tor_free(rsa_ed_crosscert);
+    rsa_ed_crosscert_len = crosscert_len;
+    rsa_ed_crosscert = crosscert;
+    rsa_ed_crosscert_expiration = expiration;
+  }
+
+  if (!current_auth_key ||
+      signing_key_changed ||
+      EXPIRES_SOON(auth_key_cert, options->TestingAuthKeySlop)) {
+    auth = ed_key_new(use_signing, INIT_ED_KEY_NEEDCERT,
+                      now,
+                      options->TestingAuthKeyLifetime,
+                      CERT_TYPE_SIGNING_AUTH, &auth_cert);
+
+    if (!auth)
+      FAIL("Can't create auth key");
+  }
+
+  /* We've generated or loaded everything.  Put them in memory. */
+
+ end:
+  if (! master_identity_key) {
+    SET_KEY(master_identity_key, id);
+  } else {
+    tor_free(id);
+  }
+  if (sign) {
+    SET_KEY(master_signing_key, sign);
+    SET_CERT(signing_key_cert, sign_cert);
+  }
+  if (auth) {
+    SET_KEY(current_auth_key, auth);
+    SET_CERT(auth_key_cert, auth_cert);
+  }
+
+  return signing_key_changed;
+ err:
+  ed25519_keypair_free(id);
+  ed25519_keypair_free(sign);
+  ed25519_keypair_free(auth);
+  tor_cert_free(sign_cert);
+  tor_cert_free(auth_cert);
+  return -1;
+}
+
+/**
+ * Retrieve our currently-in-use Ed25519 link certificate and id certificate,
+ * and, if they would expire soon (based on the time <b>now</b>, generate new
+ * certificates (without embedding the public part of the signing key inside).
+ * If <b>force</b> is true, always generate a new certificate.
+ *
+ * The signed_key from the current id->signing certificate will be used to
+ * sign the new key within newly generated X509 certificate.
+ *
+ * Returns -1 upon error.  Otherwise, returns 0 upon success (either when the
+ * current certificate is still valid, or when a new certificate was
+ * successfully generated, or no certificate was needed).
+ */
+int
+generate_ed_link_cert(const or_options_t *options, time_t now,
+                      int force)
+{
+  const tor_x509_cert_t *link_ = NULL, *id = NULL;
+  tor_cert_t *link_cert = NULL;
+
+  if (tor_tls_get_my_certs(1, &link_, &id) < 0 || link_ == NULL) {
+    if (!server_mode(options)) {
+        /* No need to make an Ed25519->Link cert: we are a client */
+      return 0;
+    }
+    log_warn(LD_OR, "Can't get my x509 link cert.");
+    return -1;
+  }
+
+  const common_digests_t *digests = tor_x509_cert_get_cert_digests(link_);
+
+  if (force == 0 &&
+      link_cert_cert &&
+      ! EXPIRES_SOON(link_cert_cert, options->TestingLinkKeySlop) &&
+      fast_memeq(digests->d[DIGEST_SHA256], link_cert_cert->signed_key.pubkey,
+                 DIGEST256_LEN)) {
+    return 0;
+  }
+
+  ed25519_public_key_t dummy_key;
+  memcpy(dummy_key.pubkey, digests->d[DIGEST_SHA256], DIGEST256_LEN);
+
+  link_cert = tor_cert_create(get_master_signing_keypair(),
+                              CERT_TYPE_SIGNING_LINK,
+                              &dummy_key,
+                              now,
+                              options->TestingLinkCertLifetime, 0);
+
+  if (link_cert) {
+    SET_CERT(link_cert_cert, link_cert);
+  }
+  return 0;
+}
+
+#undef FAIL
+#undef SET_KEY
+#undef SET_CERT
+
+/**
+ * Return 1 if any of the following are true:
+ *
+ *   - if one of our Ed25519 signing, auth, or link certificates would expire
+ *     soon w.r.t. the time <b>now</b>,
+ *   - if we do not currently have a link certificate, or
+ *   - if our cached Ed25519 link certificate is not same as the one we're
+ *     currently using.
+ *
+ * Otherwise, returns 0.
+ */
+int
+should_make_new_ed_keys(const or_options_t *options, const time_t now)
+{
+  if (!master_identity_key ||
+      !master_signing_key ||
+      !current_auth_key ||
+      !link_cert_cert ||
+      EXPIRES_SOON(signing_key_cert, options->TestingSigningKeySlop) ||
+      EXPIRES_SOON(auth_key_cert, options->TestingAuthKeySlop) ||
+      EXPIRES_SOON(link_cert_cert, options->TestingLinkKeySlop))
+    return 1;
+
+  const tor_x509_cert_t *link_ = NULL, *id = NULL;
+
+  if (tor_tls_get_my_certs(1, &link_, &id) < 0 || link_ == NULL)
+    return 1;
+
+  const common_digests_t *digests = tor_x509_cert_get_cert_digests(link_);
+
+  if (!fast_memeq(digests->d[DIGEST_SHA256],
+                  link_cert_cert->signed_key.pubkey,
+                  DIGEST256_LEN)) {
+    return 1;
+  }
+
+  return 0;
+}
+
+#undef EXPIRES_SOON
+#undef HAPPENS_SOON
+
+#ifdef TOR_UNIT_TESTS
+/* Helper for unit tests: populate the ed25519 keys without saving or
+ * loading */
+void
+init_mock_ed_keys(const crypto_pk_t *rsa_identity_key)
+{
+  routerkeys_free_all();
+
+#define MAKEKEY(k)                                      \
+  k = tor_malloc_zero(sizeof(*k));                      \
+  if (ed25519_keypair_generate(k, 0) < 0) {             \
+    log_warn(LD_BUG, "Couldn't make a keypair");        \
+    goto err;                                           \
+  }
+  MAKEKEY(master_identity_key);
+  MAKEKEY(master_signing_key);
+  MAKEKEY(current_auth_key);
+#define MAKECERT(cert, signing, signed_, type, flags)            \
+  cert = tor_cert_create(signing,                                \
+                         type,                                   \
+                         &signed_->pubkey,                       \
+                         time(NULL), 86400,                      \
+                         flags);                                 \
+  if (!cert) {                                                   \
+    log_warn(LD_BUG, "Couldn't make a %s certificate!", #cert);  \
+    goto err;                                                    \
+  }
+
+  MAKECERT(signing_key_cert,
+           master_identity_key, master_signing_key, CERT_TYPE_ID_SIGNING,
+           CERT_FLAG_INCLUDE_SIGNING_KEY);
+  MAKECERT(auth_key_cert,
+           master_signing_key, current_auth_key, CERT_TYPE_SIGNING_AUTH, 0);
+
+  if (generate_ed_link_cert(get_options(), time(NULL), 0) < 0) {
+    log_warn(LD_BUG, "Couldn't make link certificate");
+    goto err;
+  }
+
+  rsa_ed_crosscert_len = tor_make_rsa_ed25519_crosscert(
+                                     &master_identity_key->pubkey,
+                                     rsa_identity_key,
+                                     time(NULL)+86400,
+                                     &rsa_ed_crosscert);
+
+  return;
+
+ err:
+  routerkeys_free_all();
+  tor_assert_nonfatal_unreached();
+}
+#undef MAKEKEY
+#undef MAKECERT
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Print the ISO8601-formated <b>expiration</b> for a certificate with
+ * some <b>description</b> to stdout.
+ *
+ * For example, for a signing certificate, this might print out:
+ * signing-cert-expiry: 2017-07-25 08:30:15 UTC
+ */
+static void
+print_cert_expiration(const char *expiration,
+                      const char *description)
+{
+  fprintf(stderr, "%s-cert-expiry: %s\n", description, expiration);
+}
+
+/**
+ * Log when a certificate, <b>cert</b>, with some <b>description</b> and
+ * stored in a file named <b>fname</b>, is going to expire.
+ */
+static void
+log_ed_cert_expiration(const tor_cert_t *cert,
+                       const char *description,
+                       const char *fname) {
+  char expiration[ISO_TIME_LEN+1];
+
+  if (BUG(!cert)) { /* If the specified key hasn't been loaded */
+    log_warn(LD_OR, "No %s key loaded; can't get certificate expiration.",
+             description);
+  } else {
+    format_local_iso_time(expiration, cert->valid_until);
+    log_notice(LD_OR, "The %s certificate stored in %s is valid until %s.",
+               description, fname, expiration);
+    print_cert_expiration(expiration, description);
+  }
+}
+
+/**
+ * Log when our master signing key certificate expires.  Used when tor is given
+ * the --key-expiration command-line option.
+ *
+ * Returns 0 on success and 1 on failure.
+ */
+static int
+log_master_signing_key_cert_expiration(const or_options_t *options)
+{
+  const tor_cert_t *signing_key;
+  char *fn = NULL;
+  int failed = 0;
+  time_t now = approx_time();
+
+  fn = options_get_keydir_fname(options, "ed25519_signing_cert");
+
+  /* Try to grab our cached copy of the key. */
+  signing_key = get_master_signing_key_cert();
+
+  tor_assert(server_identity_key_is_set());
+
+  /* Load our keys from disk, if necessary. */
+  if (!signing_key) {
+    failed = load_ed_keys(options, now) < 0;
+    signing_key = get_master_signing_key_cert();
+  }
+
+  /* If we do have a signing key, log the expiration time. */
+  if (signing_key) {
+    log_ed_cert_expiration(signing_key, "signing", fn);
+  } else {
+    log_warn(LD_OR, "Could not load signing key certificate from %s, so " \
+             "we couldn't learn anything about certificate expiration.", fn);
+  }
+
+  tor_free(fn);
+
+  return failed;
+}
+
+/**
+ * Log when a key certificate expires.  Used when tor is given the
+ * --key-expiration command-line option.
+ *
+ * If an command argument is given, which should specify the type of
+ * key to get expiry information about (currently supported arguments
+ * are "sign"), get info about that type of certificate.  Otherwise,
+ * print info about the supported arguments.
+ *
+ * Returns 0 on success and -1 on failure.
+ */
+int
+log_cert_expiration(void)
+{
+  const or_options_t *options = get_options();
+  const char *arg = options->command_arg;
+
+  if (!strcmp(arg, "sign")) {
+    return log_master_signing_key_cert_expiration(options);
+  } else {
+    fprintf(stderr, "No valid argument to --key-expiration found!\n");
+    fprintf(stderr, "Currently recognised arguments are: 'sign'\n");
+
+    return -1;
+  }
+}
+
+const ed25519_public_key_t *
+get_master_identity_key(void)
+{
+  if (!master_identity_key)
+    return NULL;
+  return &master_identity_key->pubkey;
+}
+
+/** Return true iff <b>id</b> is our Ed25519 master identity key. */
+int
+router_ed25519_id_is_me(const ed25519_public_key_t *id)
+{
+  return id && master_identity_key &&
+    ed25519_pubkey_eq(id, &master_identity_key->pubkey);
+}
+
+#ifdef TOR_UNIT_TESTS
+/* only exists for the unit tests, since otherwise the identity key
+ * should be used to sign nothing but the signing key. */
+const ed25519_keypair_t *
+get_master_identity_keypair(void)
+{
+  return master_identity_key;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+const ed25519_keypair_t *
+get_master_signing_keypair(void)
+{
+  return master_signing_key;
+}
+
+const struct tor_cert_st *
+get_master_signing_key_cert(void)
+{
+  return signing_key_cert;
+}
+
+const ed25519_keypair_t *
+get_current_auth_keypair(void)
+{
+  return current_auth_key;
+}
+
+const tor_cert_t *
+get_current_link_cert_cert(void)
+{
+  return link_cert_cert;
+}
+
+const tor_cert_t *
+get_current_auth_key_cert(void)
+{
+  return auth_key_cert;
+}
+
+void
+get_master_rsa_crosscert(const uint8_t **cert_out,
+                         size_t *size_out)
+{
+  *cert_out = rsa_ed_crosscert;
+  *size_out = rsa_ed_crosscert_len;
+}
+
+/** Construct cross-certification for the master identity key with
+ * the ntor onion key. Store the sign of the corresponding ed25519 public key
+ * in *<b>sign_out</b>. */
+tor_cert_t *
+make_ntor_onion_key_crosscert(const curve25519_keypair_t *onion_key,
+      const ed25519_public_key_t *master_id_key, time_t now, time_t lifetime,
+      int *sign_out)
+{
+  tor_cert_t *cert = NULL;
+  ed25519_keypair_t ed_onion_key;
+
+  if (ed25519_keypair_from_curve25519_keypair(&ed_onion_key, sign_out,
+                                              onion_key) < 0)
+    goto end;
+
+  cert = tor_cert_create(&ed_onion_key, CERT_TYPE_ONION_ID, master_id_key,
+      now, lifetime, 0);
+
+ end:
+  memwipe(&ed_onion_key, 0, sizeof(ed_onion_key));
+  return cert;
+}
+
+/** Construct and return an RSA signature for the TAP onion key to
+ * cross-certify the RSA and Ed25519 identity keys. Set <b>len_out</b> to its
+ * length. */
+uint8_t *
+make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
+                             const ed25519_public_key_t *master_id_key,
+                             const crypto_pk_t *rsa_id_key,
+                             int *len_out)
+{
+  uint8_t signature[PK_BYTES];
+  uint8_t signed_data[DIGEST_LEN + ED25519_PUBKEY_LEN];
+
+  *len_out = 0;
+  if (crypto_pk_get_digest(rsa_id_key, (char*)signed_data) < 0) {
+    return NULL;
+  }
+  memcpy(signed_data + DIGEST_LEN, master_id_key->pubkey, ED25519_PUBKEY_LEN);
+
+  int r = crypto_pk_private_sign(onion_key,
+                               (char*)signature, sizeof(signature),
+                               (const char*)signed_data, sizeof(signed_data));
+  if (r < 0)
+    return NULL;
+
+  *len_out = r;
+
+  return tor_memdup(signature, r);
+}
+
+void
+routerkeys_free_all(void)
+{
+  ed25519_keypair_free(master_identity_key);
+  ed25519_keypair_free(master_signing_key);
+  ed25519_keypair_free(current_auth_key);
+  tor_cert_free(signing_key_cert);
+  tor_cert_free(link_cert_cert);
+  tor_cert_free(auth_key_cert);
+  tor_free(rsa_ed_crosscert);
+
+  master_identity_key = master_signing_key = NULL;
+  current_auth_key = NULL;
+  signing_key_cert = link_cert_cert = auth_key_cert = NULL;
+  rsa_ed_crosscert = NULL; // redundant
+  rsa_ed_crosscert_len = 0;
+}
diff --git a/src/feature/relay/routerkeys.h b/src/feature/relay/routerkeys.h
new file mode 100644
index 0000000000..0badd34191
--- /dev/null
+++ b/src/feature/relay/routerkeys.h
@@ -0,0 +1,45 @@
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_ROUTERKEYS_H
+#define TOR_ROUTERKEYS_H
+
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+const ed25519_public_key_t *get_master_identity_key(void);
+const ed25519_keypair_t *get_master_signing_keypair(void);
+const struct tor_cert_st *get_master_signing_key_cert(void);
+
+const ed25519_keypair_t *get_current_auth_keypair(void);
+const struct tor_cert_st *get_current_link_cert_cert(void);
+const struct tor_cert_st *get_current_auth_key_cert(void);
+
+void get_master_rsa_crosscert(const uint8_t **cert_out,
+                              size_t *size_out);
+
+int router_ed25519_id_is_me(const ed25519_public_key_t *id);
+
+struct tor_cert_st *make_ntor_onion_key_crosscert(
+                                  const curve25519_keypair_t *onion_key,
+                                  const ed25519_public_key_t *master_id_key,
+                                  time_t now, time_t lifetime,
+                                  int *sign_out);
+uint8_t *make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
+                                  const ed25519_public_key_t *master_id_key,
+                                  const crypto_pk_t *rsa_id_key,
+                                  int *len_out);
+
+int log_cert_expiration(void);
+int load_ed_keys(const or_options_t *options, time_t now);
+int should_make_new_ed_keys(const or_options_t *options, const time_t now);
+
+int generate_ed_link_cert(const or_options_t *options, time_t now, int force);
+
+void routerkeys_free_all(void);
+
+#ifdef TOR_UNIT_TESTS
+const ed25519_keypair_t *get_master_identity_keypair(void);
+void init_mock_ed_keys(const crypto_pk_t *rsa_identity_key);
+#endif
+
+#endif /* !defined(TOR_ROUTERKEYS_H) */
diff --git a/src/feature/relay/routermode.c b/src/feature/relay/routermode.c
new file mode 100644
index 0000000000..2a9ddeac4d
--- /dev/null
+++ b/src/feature/relay/routermode.c
@@ -0,0 +1,80 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/port_cfg_st.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+
+/** Return 1 if we are configured to accept either relay or directory requests
+ * from clients and we aren't at risk of exceeding our bandwidth limits, thus
+ * we should be a directory server. If not, return 0.
+ */
+int
+dir_server_mode(const or_options_t *options)
+{
+  if (!options->DirCache)
+    return 0;
+  return options->DirPort_set ||
+    (server_mode(options) && router_has_bandwidth_to_be_dirserver(options));
+}
+
+/** Return true iff we are trying to proxy client connections. */
+int
+proxy_mode(const or_options_t *options)
+{
+  (void)options;
+  SMARTLIST_FOREACH_BEGIN(get_configured_ports(), const port_cfg_t *, p) {
+    if (p->type == CONN_TYPE_AP_LISTENER ||
+        p->type == CONN_TYPE_AP_TRANS_LISTENER ||
+        p->type == CONN_TYPE_AP_DNS_LISTENER ||
+        p->type == CONN_TYPE_AP_NATD_LISTENER)
+      return 1;
+  } SMARTLIST_FOREACH_END(p);
+  return 0;
+}
+
+/** Return true iff we are trying to be a server.
+ */
+MOCK_IMPL(int,
+server_mode,(const or_options_t *options))
+{
+  if (options->ClientOnly) return 0;
+  return (options->ORPort_set);
+}
+
+/** Return true iff we are trying to be a non-bridge server.
+ */
+MOCK_IMPL(int,
+public_server_mode,(const or_options_t *options))
+{
+  if (!server_mode(options)) return 0;
+  return (!options->BridgeRelay);
+}
+
+/** Remember if we've advertised ourselves to the dirservers. */
+static int server_is_advertised=0;
+
+/** Return true iff we have published our descriptor lately.
+ */
+MOCK_IMPL(int,
+advertised_server_mode,(void))
+{
+  return server_is_advertised;
+}
+
+/**
+ * Called with a boolean: set whether we have recently published our
+ * descriptor.
+ */
+void
+set_server_advertised(int s)
+{
+  server_is_advertised = s;
+}
diff --git a/src/feature/relay/routermode.h b/src/feature/relay/routermode.h
new file mode 100644
index 0000000000..be535af478
--- /dev/null
+++ b/src/feature/relay/routermode.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file routermode.h
+ * \brief Header file for routermode.c.
+ **/
+
+#ifndef TOR_ROUTERMODE_H
+#define TOR_ROUTERMODE_H
+
+int dir_server_mode(const or_options_t *options);
+
+MOCK_DECL(int, server_mode, (const or_options_t *options));
+MOCK_DECL(int, public_server_mode, (const or_options_t *options));
+MOCK_DECL(int, advertised_server_mode, (void));
+int proxy_mode(const or_options_t *options);
+
+void set_server_advertised(int s);
+
+#endif /* !defined(TOR_ROUTERMODE_H) */
diff --git a/src/feature/relay/selftest.c b/src/feature/relay/selftest.c
new file mode 100644
index 0000000000..064eea6c46
--- /dev/null
+++ b/src/feature/relay/selftest.c
@@ -0,0 +1,301 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file selftest.c
+ * \brief Relay self-testing
+ *
+ * Relays need to make sure that their own ports are reasonable, and estimate
+ * their own bandwidth, before publishing.
+ */
+
+#define SELFTEST_PRIVATE
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/crypt_path_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/relay.h"
+#include "feature/control/control.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dircommon/directory.h"
+#include "feature/nodelist/authority_cert_st.h"
+#include "feature/nodelist/routerinfo.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerlist.h" // but...
+#include "feature/nodelist/routerset.h"
+#include "feature/nodelist/torcert.h"
+#include "feature/relay/router.h"
+#include "feature/relay/selftest.h"
+
+/** Whether we can reach our ORPort from the outside. */
+static int can_reach_or_port = 0;
+/** Whether we can reach our DirPort from the outside. */
+static int can_reach_dir_port = 0;
+
+/** Forget what we have learned about our reachability status. */
+void
+router_reset_reachability(void)
+{
+  can_reach_or_port = can_reach_dir_port = 0;
+}
+
+/** Return 1 if we won't do reachability checks, because:
+ *   - AssumeReachable is set, or
+ *   - the network is disabled.
+ * Otherwise, return 0.
+ */
+static int
+router_reachability_checks_disabled(const or_options_t *options)
+{
+  return options->AssumeReachable ||
+         net_is_disabled();
+}
+
+/** Return 0 if we need to do an ORPort reachability check, because:
+ *   - no reachability check has been done yet, or
+ *   - we've initiated reachability checks, but none have succeeded.
+ *  Return 1 if we don't need to do an ORPort reachability check, because:
+ *   - we've seen a successful reachability check, or
+ *   - AssumeReachable is set, or
+ *   - the network is disabled.
+ */
+int
+check_whether_orport_reachable(const or_options_t *options)
+{
+  int reach_checks_disabled = router_reachability_checks_disabled(options);
+  return reach_checks_disabled ||
+         can_reach_or_port;
+}
+
+/** Return 0 if we need to do a DirPort reachability check, because:
+ *   - no reachability check has been done yet, or
+ *   - we've initiated reachability checks, but none have succeeded.
+ *  Return 1 if we don't need to do a DirPort reachability check, because:
+ *   - we've seen a successful reachability check, or
+ *   - there is no DirPort set, or
+ *   - AssumeReachable is set, or
+ *   - the network is disabled.
+ */
+int
+check_whether_dirport_reachable(const or_options_t *options)
+{
+  int reach_checks_disabled = router_reachability_checks_disabled(options) ||
+                              !options->DirPort_set;
+  return reach_checks_disabled ||
+         can_reach_dir_port;
+}
+
+/** See if we currently believe our ORPort or DirPort to be
+ * unreachable. If so, return 1 else return 0.
+ */
+static int
+router_should_check_reachability(int test_or, int test_dir)
+{
+  const routerinfo_t *me = router_get_my_routerinfo();
+  const or_options_t *options = get_options();
+
+  if (!me)
+    return 0;
+
+  if (routerset_contains_router(options->ExcludeNodes, me, -1) &&
+      options->StrictNodes) {
+    /* If we've excluded ourself, and StrictNodes is set, we can't test
+     * ourself. */
+    if (test_or || test_dir) {
+#define SELF_EXCLUDED_WARN_INTERVAL 3600
+      static ratelim_t warning_limit=RATELIM_INIT(SELF_EXCLUDED_WARN_INTERVAL);
+      log_fn_ratelim(&warning_limit, LOG_WARN, LD_CIRC,
+                 "Can't peform self-tests for this relay: we have "
+                 "listed ourself in ExcludeNodes, and StrictNodes is set. "
+                 "We cannot learn whether we are usable, and will not "
+                 "be able to advertise ourself.");
+    }
+    return 0;
+  }
+  return 1;
+}
+
+/** Allocate and return a new extend_info_t that can be used to build
+ * a circuit to or through the router <b>r</b>. Uses the primary
+ * address of the router, so should only be called on a server. */
+static extend_info_t *
+extend_info_from_router(const routerinfo_t *r)
+{
+  crypto_pk_t *rsa_pubkey;
+  extend_info_t *info;
+  tor_addr_port_t ap;
+  tor_assert(r);
+
+  /* Make sure we don't need to check address reachability */
+  tor_assert_nonfatal(router_skip_or_reachability(get_options(), 0));
+
+  const ed25519_public_key_t *ed_id_key;
+  if (r->cache_info.signing_key_cert)
+    ed_id_key = &r->cache_info.signing_key_cert->signing_key;
+  else
+    ed_id_key = NULL;
+
+  router_get_prim_orport(r, &ap);
+  rsa_pubkey = router_get_rsa_onion_pkey(r->onion_pkey, r->onion_pkey_len);
+  info = extend_info_new(r->nickname, r->cache_info.identity_digest,
+                         ed_id_key,
+                         rsa_pubkey, r->onion_curve25519_pkey,
+                         &ap.addr, ap.port);
+  crypto_pk_free(rsa_pubkey);
+  return info;
+}
+
+/** Some time has passed, or we just got new directory information.
+ * See if we currently believe our ORPort or DirPort to be
+ * unreachable. If so, launch a new test for it.
+ *
+ * For ORPort, we simply try making a circuit that ends at ourselves.
+ * Success is noticed in onionskin_answer().
+ *
+ * For DirPort, we make a connection via Tor to our DirPort and ask
+ * for our own server descriptor.
+ * Success is noticed in connection_dir_client_reached_eof().
+ */
+void
+router_do_reachability_checks(int test_or, int test_dir)
+{
+  const routerinfo_t *me = router_get_my_routerinfo();
+  const or_options_t *options = get_options();
+  int orport_reachable = check_whether_orport_reachable(options);
+  tor_addr_t addr;
+
+  if (router_should_check_reachability(test_or, test_dir)) {
+    if (test_or && (!orport_reachable || !circuit_enough_testing_circs())) {
+      extend_info_t *ei = extend_info_from_router(me);
+      /* XXX IPv6 self testing */
+      log_info(LD_CIRC, "Testing %s of my ORPort: %s:%d.",
+               !orport_reachable ? "reachability" : "bandwidth",
+               fmt_addr32(me->addr), me->or_port);
+      circuit_launch_by_extend_info(CIRCUIT_PURPOSE_TESTING, ei,
+                              CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL);
+      extend_info_free(ei);
+    }
+
+    /* XXX IPv6 self testing */
+    tor_addr_from_ipv4h(&addr, me->addr);
+    if (test_dir && !check_whether_dirport_reachable(options) &&
+        !connection_get_by_type_addr_port_purpose(
+                  CONN_TYPE_DIR, &addr, me->dir_port,
+                  DIR_PURPOSE_FETCH_SERVERDESC)) {
+      tor_addr_port_t my_orport, my_dirport;
+      memcpy(&my_orport.addr, &addr, sizeof(addr));
+      memcpy(&my_dirport.addr, &addr, sizeof(addr));
+      my_orport.port = me->or_port;
+      my_dirport.port = me->dir_port;
+      /* ask myself, via tor, for my server descriptor. */
+      directory_request_t *req =
+        directory_request_new(DIR_PURPOSE_FETCH_SERVERDESC);
+      directory_request_set_or_addr_port(req, &my_orport);
+      directory_request_set_dir_addr_port(req, &my_dirport);
+      directory_request_set_directory_id_digest(req,
+                                              me->cache_info.identity_digest);
+      // ask via an anon circuit, connecting to our dirport.
+      directory_request_set_indirection(req, DIRIND_ANON_DIRPORT);
+      directory_request_set_resource(req, "authority.z");
+      directory_initiate_request(req);
+      directory_request_free(req);
+    }
+  }
+}
+
+/** Annotate that we found our ORPort reachable. */
+void
+router_orport_found_reachable(void)
+{
+  const routerinfo_t *me = router_get_my_routerinfo();
+  const or_options_t *options = get_options();
+  if (!can_reach_or_port && me) {
+    char *address = tor_dup_ip(me->addr);
+    log_notice(LD_OR,"Self-testing indicates your ORPort is reachable from "
+               "the outside. Excellent.%s",
+               options->PublishServerDescriptor_ != NO_DIRINFO
+               && check_whether_dirport_reachable(options) ?
+                 " Publishing server descriptor." : "");
+    can_reach_or_port = 1;
+    mark_my_descriptor_dirty("ORPort found reachable");
+    /* This is a significant enough change to upload immediately,
+     * at least in a test network */
+    if (options->TestingTorNetwork == 1) {
+      reschedule_descriptor_update_check();
+    }
+    control_event_server_status(LOG_NOTICE,
+                                "REACHABILITY_SUCCEEDED ORADDRESS=%s:%d",
+                                address, me->or_port);
+    tor_free(address);
+  }
+}
+
+/** Annotate that we found our DirPort reachable. */
+void
+router_dirport_found_reachable(void)
+{
+  const routerinfo_t *me = router_get_my_routerinfo();
+  const or_options_t *options = get_options();
+  if (!can_reach_dir_port && me) {
+    char *address = tor_dup_ip(me->addr);
+    log_notice(LD_DIRSERV,"Self-testing indicates your DirPort is reachable "
+               "from the outside. Excellent.%s",
+               options->PublishServerDescriptor_ != NO_DIRINFO
+               && check_whether_orport_reachable(options) ?
+               " Publishing server descriptor." : "");
+    can_reach_dir_port = 1;
+    if (router_should_advertise_dirport(options, me->dir_port)) {
+      mark_my_descriptor_dirty("DirPort found reachable");
+      /* This is a significant enough change to upload immediately,
+       * at least in a test network */
+      if (options->TestingTorNetwork == 1) {
+        reschedule_descriptor_update_check();
+      }
+    }
+    control_event_server_status(LOG_NOTICE,
+                                "REACHABILITY_SUCCEEDED DIRADDRESS=%s:%d",
+                                address, me->dir_port);
+    tor_free(address);
+  }
+}
+
+/** We have enough testing circuits open. Send a bunch of "drop"
+ * cells down each of them, to exercise our bandwidth. */
+void
+router_perform_bandwidth_test(int num_circs, time_t now)
+{
+  int num_cells = (int)(get_options()->BandwidthRate * 10 /
+                        CELL_MAX_NETWORK_SIZE);
+  int max_cells = num_cells < CIRCWINDOW_START ?
+                    num_cells : CIRCWINDOW_START;
+  int cells_per_circuit = max_cells / num_circs;
+  origin_circuit_t *circ = NULL;
+
+  log_notice(LD_OR,"Performing bandwidth self-test...done.");
+  while ((circ = circuit_get_next_by_pk_and_purpose(circ, NULL,
+                                              CIRCUIT_PURPOSE_TESTING))) {
+    /* dump cells_per_circuit drop cells onto this circ */
+    int i = cells_per_circuit;
+    if (circ->base_.state != CIRCUIT_STATE_OPEN)
+      continue;
+    circ->base_.timestamp_dirty = now;
+    while (i-- > 0) {
+      if (relay_send_command_from_edge(0, TO_CIRCUIT(circ),
+                                       RELAY_COMMAND_DROP,
+                                       NULL, 0, circ->cpath->prev)<0) {
+        return; /* stop if error */
+      }
+    }
+  }
+}
diff --git a/src/feature/relay/selftest.h b/src/feature/relay/selftest.h
new file mode 100644
index 0000000000..a80ec8936e
--- /dev/null
+++ b/src/feature/relay/selftest.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file selftest.h
+ * \brief Header file for selftest.c.
+ **/
+
+#ifndef TOR_SELFTEST_H
+#define TOR_SELFTEST_H
+
+struct or_options_t;
+int check_whether_orport_reachable(const struct or_options_t *options);
+int check_whether_dirport_reachable(const struct or_options_t *options);
+
+void router_do_reachability_checks(int test_or, int test_dir);
+void router_orport_found_reachable(void);
+void router_dirport_found_reachable(void);
+void router_perform_bandwidth_test(int num_circs, time_t now);
+
+#endif
diff --git a/src/feature/rend/rend_authorized_client_st.h b/src/feature/rend/rend_authorized_client_st.h
new file mode 100644
index 0000000000..7bd4f2fe8c
--- /dev/null
+++ b/src/feature/rend/rend_authorized_client_st.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef REND_AUTHORIZED_CLIENT_ST_H
+#define REND_AUTHORIZED_CLIENT_ST_H
+
+/** Hidden-service side configuration of client authorization. */
+struct rend_authorized_client_t {
+  char *client_name;
+  uint8_t descriptor_cookie[REND_DESC_COOKIE_LEN];
+  crypto_pk_t *client_key;
+};
+
+#endif
+
diff --git a/src/feature/rend/rend_encoded_v2_service_descriptor_st.h b/src/feature/rend/rend_encoded_v2_service_descriptor_st.h
new file mode 100644
index 0000000000..05ff145d53
--- /dev/null
+++ b/src/feature/rend/rend_encoded_v2_service_descriptor_st.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef REND_ENCODED_V2_SERVICE_DESCRIPTOR_ST_H
+#define REND_ENCODED_V2_SERVICE_DESCRIPTOR_ST_H
+
+/** ASCII-encoded v2 hidden service descriptor. */
+struct rend_encoded_v2_service_descriptor_t {
+  char desc_id[DIGEST_LEN]; /**< Descriptor ID. */
+  char *desc_str; /**< Descriptor string. */
+};
+
+#endif
+
diff --git a/src/feature/rend/rend_intro_point_st.h b/src/feature/rend/rend_intro_point_st.h
new file mode 100644
index 0000000000..de6987e569
--- /dev/null
+++ b/src/feature/rend/rend_intro_point_st.h
@@ -0,0 +1,76 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef REND_INTRO_POINT_ST_H
+#define REND_INTRO_POINT_ST_H
+
+struct replaycache_t;
+struct crypto_pk_t;
+
+/** Introduction point information.  Used both in rend_service_t (on
+ * the service side) and in rend_service_descriptor_t (on both the
+ * client and service side). */
+struct rend_intro_point_t {
+  extend_info_t *extend_info; /**< Extend info for connecting to this
+                               * introduction point via a multi-hop path. */
+  struct crypto_pk_t *intro_key; /**< Introduction key that replaces the
+                                  * service key, if this descriptor is V2. */
+
+  /** (Client side only) Flag indicating that a timeout has occurred
+   * after sending an INTRODUCE cell to this intro point.  After a
+   * timeout, an intro point should not be tried again during the same
+   * hidden service connection attempt, but it may be tried again
+   * during a future connection attempt. */
+  unsigned int timed_out : 1;
+
+  /** (Client side only) The number of times we have failed to build a
+   * circuit to this intro point for some reason other than our
+   * circuit-build timeout.  See also MAX_INTRO_POINT_REACHABILITY_FAILURES. */
+  unsigned int unreachable_count : 3;
+
+  /** (Service side only) Flag indicating that this intro point was
+   * included in the last HS descriptor we generated. */
+  unsigned int listed_in_last_desc : 1;
+
+  /** (Service side only) A replay cache recording the RSA-encrypted parts
+   * of INTRODUCE2 cells this intro point's circuit has received.  This is
+   * used to prevent replay attacks. */
+  struct replaycache_t *accepted_intro_rsa_parts;
+
+  /** (Service side only) Count of INTRODUCE2 cells accepted from this
+   * intro point.
+   */
+  int accepted_introduce2_count;
+
+  /** (Service side only) Maximum number of INTRODUCE2 cells that this IP
+   * will accept. This is a random value between
+   * INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS and
+   * INTRO_POINT_MAX_LIFETIME_INTRODUCTIONS. */
+  int max_introductions;
+
+  /** (Service side only) The time at which this intro point was first
+   * published, or -1 if this intro point has not yet been
+   * published. */
+  time_t time_published;
+
+  /** (Service side only) The time at which this intro point should
+   * (start to) expire, or -1 if we haven't decided when this intro
+   * point should expire. */
+  time_t time_to_expire;
+
+  /** (Service side only) The amount of circuit creation we've made to this
+   * intro point. This is incremented every time we do a circuit relaunch on
+   * this object which is triggered when the circuit dies but the node is
+   * still in the consensus. After MAX_INTRO_POINT_CIRCUIT_RETRIES, we give
+   * up on it. */
+  unsigned int circuit_retries;
+
+  /** (Service side only) Set if this intro point has an established circuit
+   * and unset if it doesn't. */
+  unsigned int circuit_established:1;
+};
+
+#endif
diff --git a/src/feature/rend/rend_service_descriptor_st.h b/src/feature/rend/rend_service_descriptor_st.h
new file mode 100644
index 0000000000..aeb3178064
--- /dev/null
+++ b/src/feature/rend/rend_service_descriptor_st.h
@@ -0,0 +1,34 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef REND_SERVICE_DESCRIPTOR_ST_H
+#define REND_SERVICE_DESCRIPTOR_ST_H
+
+#define REND_PROTOCOL_VERSION_BITMASK_WIDTH 16
+
+/** Information used to connect to a hidden service.  Used on both the
+ * service side and the client side. */
+struct rend_service_descriptor_t {
+  crypto_pk_t *pk; /**< This service's public key. */
+  int version; /**< Version of the descriptor format: 0 or 2. */
+  time_t timestamp; /**< Time when the descriptor was generated. */
+  /** Bitmask: which introduce/rendezvous protocols are supported?
+   * (We allow bits '0', '1', '2' and '3' to be set.) */
+  unsigned protocols : REND_PROTOCOL_VERSION_BITMASK_WIDTH;
+  /** List of the service's introduction points.  Elements are removed if
+   * introduction attempts fail. */
+  smartlist_t *intro_nodes;
+  /** Has descriptor been uploaded to all hidden service directories? */
+  int all_uploads_performed;
+  /** List of hidden service directories to which an upload request for
+   * this descriptor could be sent. Smartlist exists only when at least one
+   * of the previous upload requests failed (otherwise it's not important
+   * to know which uploads succeeded and which not). */
+  smartlist_t *successful_uploads;
+};
+
+#endif
+
diff --git a/src/or/rendcache.c b/src/feature/rend/rendcache.c
index aa69d735fe..1c3badaff3 100644
--- a/src/or/rendcache.c
+++ b/src/feature/rend/rendcache.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
+/* Copyright (c) 2015-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -7,13 +7,17 @@
  **/
 
 #define RENDCACHE_PRIVATE
-#include "rendcache.h"
+#include "feature/rend/rendcache.h"
 
-#include "config.h"
-#include "rephist.h"
-#include "routerlist.h"
-#include "routerparse.h"
-#include "rendcommon.h"
+#include "app/config/config.h"
+#include "feature/stats/rephist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/rend/rendparse.h"
+
+#include "core/or/extend_info_st.h"
+#include "feature/rend/rend_intro_point_st.h"
+#include "feature/rend/rend_service_descriptor_st.h"
 
 /** Map from service id (as generated by rend_get_service_id) to
  * rend_cache_entry_t. */
@@ -43,12 +47,12 @@ STATIC digestmap_t *rend_cache_v2_dir = NULL;
  * ID, that were NOT present in the descriptor are removed from this cache.
  * Which means that if at least one IP was not in this cache, thus usuable,
  * it's considered a new descriptor so we keep it. Else, if all IPs were in
- * this cache, we discard the descriptor as it's considered unsuable.
+ * this cache, we discard the descriptor as it's considered unusable.
  *
  * Once a descriptor is removed from the rend cache or expires, the entry
  * in this cache is also removed for the service ID.
  *
- * This scheme allows us to not realy on the descriptor's timestamp (which
+ * This scheme allows us to not rely on the descriptor's timestamp (which
  * is rounded down to the hour) to know if we have a newer descriptor. We
  * only rely on the usability of intro points from an internal state. */
 STATIC strmap_t *rend_cache_failure = NULL;
@@ -86,7 +90,7 @@ rend_cache_get_total_allocation(void)
 }
 
 /** Decrement the total bytes attributed to the rendezvous cache by n. */
-STATIC void
+void
 rend_cache_decrement_allocation(size_t n)
 {
   static int have_underflowed = 0;
@@ -103,7 +107,7 @@ rend_cache_decrement_allocation(size_t n)
 }
 
 /** Increase the total bytes attributed to the rendezvous cache by n. */
-STATIC void
+void
 rend_cache_increment_allocation(size_t n)
 {
   static int have_overflowed = 0;
@@ -120,7 +124,7 @@ rend_cache_increment_allocation(size_t n)
 
 /** Helper: free a rend cache failure intro object. */
 STATIC void
-rend_cache_failure_intro_entry_free(rend_cache_failure_intro_t *entry)
+rend_cache_failure_intro_entry_free_(rend_cache_failure_intro_t *entry)
 {
   if (entry == NULL) {
     return;
@@ -129,9 +133,9 @@ rend_cache_failure_intro_entry_free(rend_cache_failure_intro_t *entry)
 }
 
 static void
-rend_cache_failure_intro_entry_free_(void *entry)
+rend_cache_failure_intro_entry_free_void(void *entry)
 {
-  rend_cache_failure_intro_entry_free(entry);
+  rend_cache_failure_intro_entry_free_(entry);
 }
 
 /** Allocate a rend cache failure intro object and return it. <b>failure</b>
@@ -147,7 +151,7 @@ rend_cache_failure_intro_entry_new(rend_intro_point_failure_t failure)
 
 /** Helper: free a rend cache failure object. */
 STATIC void
-rend_cache_failure_entry_free(rend_cache_failure_t *entry)
+rend_cache_failure_entry_free_(rend_cache_failure_t *entry)
 {
   if (entry == NULL) {
     return;
@@ -155,7 +159,7 @@ rend_cache_failure_entry_free(rend_cache_failure_t *entry)
 
   /* Free and remove every intro failure object. */
   digestmap_free(entry->intro_failures,
-                 rend_cache_failure_intro_entry_free_);
+                 rend_cache_failure_intro_entry_free_void);
 
   tor_free(entry);
 }
@@ -163,9 +167,9 @@ rend_cache_failure_entry_free(rend_cache_failure_t *entry)
 /** Helper: deallocate a rend_cache_failure_t. (Used with strmap_free(),
  * which requires a function pointer whose argument is void*). */
 STATIC void
-rend_cache_failure_entry_free_(void *entry)
+rend_cache_failure_entry_free_void(void *entry)
 {
-  rend_cache_failure_entry_free(entry);
+  rend_cache_failure_entry_free_(entry);
 }
 
 /** Allocate a rend cache failure object and return it. This function can
@@ -201,7 +205,7 @@ rend_cache_failure_remove(rend_service_descriptor_t *desc)
 
 /** Helper: free storage held by a single service descriptor cache entry. */
 STATIC void
-rend_cache_entry_free(rend_cache_entry_t *e)
+rend_cache_entry_free_(rend_cache_entry_t *e)
 {
   if (!e)
     return;
@@ -217,19 +221,19 @@ rend_cache_entry_free(rend_cache_entry_t *e)
 /** Helper: deallocate a rend_cache_entry_t.  (Used with strmap_free(), which
  * requires a function pointer whose argument is void*). */
 static void
-rend_cache_entry_free_(void *p)
+rend_cache_entry_free_void(void *p)
 {
-  rend_cache_entry_free(p);
+  rend_cache_entry_free_(p);
 }
 
 /** Free all storage held by the service descriptor cache. */
 void
 rend_cache_free_all(void)
 {
-  strmap_free(rend_cache, rend_cache_entry_free_);
-  digestmap_free(rend_cache_v2_dir, rend_cache_entry_free_);
-  strmap_free(rend_cache_local_service, rend_cache_entry_free_);
-  strmap_free(rend_cache_failure, rend_cache_failure_entry_free_);
+  strmap_free(rend_cache, rend_cache_entry_free_void);
+  digestmap_free(rend_cache_v2_dir, rend_cache_entry_free_void);
+  strmap_free(rend_cache_local_service, rend_cache_entry_free_void);
+  strmap_free(rend_cache_failure, rend_cache_failure_entry_free_void);
   rend_cache = NULL;
   rend_cache_v2_dir = NULL;
   rend_cache_local_service = NULL;
@@ -303,8 +307,8 @@ void
 rend_cache_purge(void)
 {
   if (rend_cache) {
-    log_info(LD_REND, "Purging HS descriptor cache");
-    strmap_free(rend_cache, rend_cache_entry_free_);
+    log_info(LD_REND, "Purging HS v2 descriptor cache");
+    strmap_free(rend_cache, rend_cache_entry_free_void);
   }
   rend_cache = strmap_new();
 }
@@ -315,8 +319,8 @@ void
 rend_cache_failure_purge(void)
 {
   if (rend_cache_failure) {
-    log_info(LD_REND, "Purging HS failure cache");
-    strmap_free(rend_cache_failure, rend_cache_failure_entry_free_);
+    log_info(LD_REND, "Purging HS v2 failure cache");
+    strmap_free(rend_cache_failure, rend_cache_failure_entry_free_void);
   }
   rend_cache_failure = strmap_new();
 }
@@ -462,45 +466,36 @@ rend_cache_intro_failure_note(rend_intro_point_failure_t failure,
 }
 
 /** Remove all old v2 descriptors and those for which this hidden service
- * directory is not responsible for any more.
- *
- * If at all possible, remove at least <b>force_remove</b> bytes of data.
- */
-void
-rend_cache_clean_v2_descs_as_dir(time_t now, size_t force_remove)
+ * directory is not responsible for any more. The cutoff is the time limit for
+ * which we want to keep the cache entry. In other words, any entry created
+ * before will be removed. */
+size_t
+rend_cache_clean_v2_descs_as_dir(time_t cutoff)
 {
   digestmap_iter_t *iter;
-  time_t cutoff = now - REND_CACHE_MAX_AGE - REND_CACHE_MAX_SKEW;
-  const int LAST_SERVED_CUTOFF_STEP = 1800;
-  time_t last_served_cutoff = cutoff;
   size_t bytes_removed = 0;
-  do {
-    for (iter = digestmap_iter_init(rend_cache_v2_dir);
-         !digestmap_iter_done(iter); ) {
-      const char *key;
-      void *val;
-      rend_cache_entry_t *ent;
-      digestmap_iter_get(iter, &key, &val);
-      ent = val;
-      if (ent->parsed->timestamp < cutoff ||
-          ent->last_served < last_served_cutoff) {
-        char key_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
-        base32_encode(key_base32, sizeof(key_base32), key, DIGEST_LEN);
-        log_info(LD_REND, "Removing descriptor with ID '%s' from cache",
-                 safe_str_client(key_base32));
-        bytes_removed += rend_cache_entry_allocation(ent);
-        iter = digestmap_iter_next_rmv(rend_cache_v2_dir, iter);
-        rend_cache_entry_free(ent);
-      } else {
-        iter = digestmap_iter_next(rend_cache_v2_dir, iter);
-      }
+
+  for (iter = digestmap_iter_init(rend_cache_v2_dir);
+       !digestmap_iter_done(iter); ) {
+    const char *key;
+    void *val;
+    rend_cache_entry_t *ent;
+    digestmap_iter_get(iter, &key, &val);
+    ent = val;
+    if (ent->parsed->timestamp < cutoff) {
+      char key_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+      base32_encode(key_base32, sizeof(key_base32), key, DIGEST_LEN);
+      log_info(LD_REND, "Removing descriptor with ID '%s' from cache",
+               safe_str_client(key_base32));
+      bytes_removed += rend_cache_entry_allocation(ent);
+      iter = digestmap_iter_next_rmv(rend_cache_v2_dir, iter);
+      rend_cache_entry_free(ent);
+    } else {
+      iter = digestmap_iter_next(rend_cache_v2_dir, iter);
     }
+  }
 
-    /* In case we didn't remove enough bytes, advance the cutoff a little. */
-    last_served_cutoff += LAST_SERVED_CUTOFF_STEP;
-    if (last_served_cutoff > now)
-      break;
-  } while (bytes_removed < force_remove);
+  return bytes_removed;
 }
 
 /** Lookup in the client cache the given service ID <b>query</b> for
@@ -521,7 +516,7 @@ rend_cache_lookup_entry(const char *query, int version, rend_cache_entry_t **e)
   tor_assert(rend_cache);
   tor_assert(query);
 
-  if (!rend_valid_service_id(query)) {
+  if (!rend_valid_v2_service_id(query)) {
     ret = -EINVAL;
     goto end;
   }
@@ -567,7 +562,7 @@ rend_cache_lookup_v2_desc_as_service(const char *query, rend_cache_entry_t **e)
   tor_assert(rend_cache_local_service);
   tor_assert(query);
 
-  if (!rend_valid_service_id(query)) {
+  if (!rend_valid_v2_service_id(query)) {
     ret = -EINVAL;
     goto end;
   }
@@ -849,6 +844,8 @@ rend_cache_store_v2_desc_as_client(const char *desc,
   char want_desc_id[DIGEST_LEN];
   rend_cache_entry_t *e;
   int retval = -1;
+  rend_data_v2_t *rend_data = TO_REND_DATA_V2(rend_query);
+
   tor_assert(rend_cache);
   tor_assert(desc);
   tor_assert(desc_id_base32);
@@ -874,11 +871,11 @@ rend_cache_store_v2_desc_as_client(const char *desc,
     log_warn(LD_REND, "Couldn't compute service ID.");
     goto err;
   }
-  if (rend_query->onion_address[0] != '\0' &&
-      strcmp(rend_query->onion_address, service_id)) {
+  if (rend_data->onion_address[0] != '\0' &&
+      strcmp(rend_data->onion_address, service_id)) {
     log_warn(LD_REND, "Received service descriptor for service ID %s; "
              "expected descriptor for service ID %s.",
-             service_id, safe_str(rend_query->onion_address));
+             service_id, safe_str(rend_data->onion_address));
     goto err;
   }
   if (tor_memneq(desc_id, want_desc_id, DIGEST_LEN)) {
@@ -890,14 +887,14 @@ rend_cache_store_v2_desc_as_client(const char *desc,
   /* Decode/decrypt introduction points. */
   if (intro_content && intro_size > 0) {
     int n_intro_points;
-    if (rend_query->auth_type != REND_NO_AUTH &&
-        !tor_mem_is_zero(rend_query->descriptor_cookie,
-                         sizeof(rend_query->descriptor_cookie))) {
+    if (rend_data->auth_type != REND_NO_AUTH &&
+        !tor_mem_is_zero(rend_data->descriptor_cookie,
+                         sizeof(rend_data->descriptor_cookie))) {
       char *ipos_decrypted = NULL;
       size_t ipos_decrypted_size;
       if (rend_decrypt_introduction_points(&ipos_decrypted,
                                            &ipos_decrypted_size,
-                                           rend_query->descriptor_cookie,
+                                           rend_data->descriptor_cookie,
                                            intro_content,
                                            intro_size) < 0) {
         log_warn(LD_REND, "Failed to decrypt introduction points. We are "
@@ -915,9 +912,7 @@ rend_cache_store_v2_desc_as_client(const char *desc,
     if (n_intro_points <= 0) {
       log_warn(LD_REND, "Failed to parse introduction points. Either the "
                "service has published a corrupt descriptor or you have "
-               "provided invalid authorization data, or (maybe!) the "
-               "server is deliberately serving broken data in an attempt "
-               "to crash you with bug 21018.");
+               "provided invalid authorization data.");
       goto err;
     } else if (n_intro_points > MAX_INTRO_POINTS) {
       log_warn(LD_REND, "Found too many introduction points on a hidden "
diff --git a/src/or/rendcache.h b/src/feature/rend/rendcache.h
index 270b614c38..aec97eabb8 100644
--- a/src/or/rendcache.h
+++ b/src/feature/rend/rendcache.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
+/* Copyright (c) 2015-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,8 +9,8 @@
 #ifndef TOR_RENDCACHE_H
 #define TOR_RENDCACHE_H
 
-#include "or.h"
-#include "rendcommon.h"
+#include "core/or/or.h"
+#include "feature/rend/rendcommon.h"
 
 /** How old do we let hidden service descriptors get before discarding
  * them as too old? */
@@ -36,7 +36,7 @@ typedef struct rend_cache_entry_t {
 
 /* Introduction point failure type. */
 typedef struct rend_cache_failure_intro_t {
-  /* When this intro point failure occured thus we allocated this object and
+  /* When this intro point failure occurred thus we allocated this object and
    * cache it. */
   time_t created_ts;
   rend_intro_point_failure_t failure_type;
@@ -53,10 +53,17 @@ typedef enum {
   REND_CACHE_TYPE_SERVICE = 2,
 } rend_cache_type_t;
 
+/* Return maximum lifetime in seconds of a cache entry. */
+static inline time_t
+rend_cache_max_entry_lifetime(void)
+{
+  return REND_CACHE_MAX_AGE + REND_CACHE_MAX_SKEW;
+}
+
 void rend_cache_init(void);
 void rend_cache_clean(time_t now, rend_cache_type_t cache_type);
 void rend_cache_failure_clean(time_t now);
-void rend_cache_clean_v2_descs_as_dir(time_t now, size_t min_to_remove);
+size_t rend_cache_clean_v2_descs_as_dir(time_t cutoff);
 void rend_cache_purge(void);
 void rend_cache_free_all(void);
 int rend_cache_lookup_entry(const char *query, int version,
@@ -77,20 +84,28 @@ void rend_cache_intro_failure_note(rend_intro_point_failure_t failure,
                                    const uint8_t *identity,
                                    const char *service_id);
 void rend_cache_failure_purge(void);
+void rend_cache_decrement_allocation(size_t n);
+void rend_cache_increment_allocation(size_t n);
 
 #ifdef RENDCACHE_PRIVATE
 
 STATIC size_t rend_cache_entry_allocation(const rend_cache_entry_t *e);
-STATIC void rend_cache_entry_free(rend_cache_entry_t *e);
-STATIC void rend_cache_failure_intro_entry_free(rend_cache_failure_intro_t
-                                                *entry);
-STATIC void rend_cache_failure_entry_free(rend_cache_failure_t *entry);
+STATIC void rend_cache_entry_free_(rend_cache_entry_t *e);
+#define rend_cache_entry_free(e) \
+  FREE_AND_NULL(rend_cache_entry_t, rend_cache_entry_free_, (e))
+STATIC void rend_cache_failure_intro_entry_free_(rend_cache_failure_intro_t
+                                                 *entry);
+#define rend_cache_failure_intro_entry_free(e)                          \
+  FREE_AND_NULL(rend_cache_failure_intro_t,                   \
+                          rend_cache_failure_intro_entry_free_, (e))
+STATIC void rend_cache_failure_entry_free_(rend_cache_failure_t *entry);
+#define rend_cache_failure_entry_free(e)                        \
+  FREE_AND_NULL(rend_cache_failure_t,                 \
+                          rend_cache_failure_entry_free_, (e))
 STATIC int cache_failure_intro_lookup(const uint8_t *identity,
                                       const char *service_id,
                                       rend_cache_failure_intro_t
                                       **intro_entry);
-STATIC void rend_cache_decrement_allocation(size_t n);
-STATIC void rend_cache_increment_allocation(size_t n);
 STATIC rend_cache_failure_intro_t *rend_cache_failure_intro_entry_new(
                                       rend_intro_point_failure_t failure);
 STATIC rend_cache_failure_t *rend_cache_failure_entry_new(void);
@@ -101,15 +116,15 @@ STATIC void cache_failure_intro_add(const uint8_t *identity,
 STATIC void validate_intro_point_failure(const rend_service_descriptor_t *desc,
                                         const char *service_id);
 
-STATIC void rend_cache_failure_entry_free_(void *entry);
+STATIC void rend_cache_failure_entry_free_void(void *entry);
 
 #ifdef TOR_UNIT_TESTS
 extern strmap_t *rend_cache;
 extern strmap_t *rend_cache_failure;
 extern digestmap_t *rend_cache_v2_dir;
 extern size_t rend_cache_total_allocation;
-#endif
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
+#endif /* defined(RENDCACHE_PRIVATE) */
 
-#endif /* TOR_RENDCACHE_H */
+#endif /* !defined(TOR_RENDCACHE_H) */
 
diff --git a/src/or/rendclient.c b/src/feature/rend/rendclient.c
index f0144b076f..4ca783c7c3 100644
--- a/src/or/rendclient.c
+++ b/src/feature/rend/rendclient.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -7,26 +7,44 @@
  * \brief Client code to access location-hidden services.
  **/
 
-#include "or.h"
-#include "circpathbias.h"
-#include "circuitbuild.h"
-#include "circuitlist.h"
-#include "circuituse.h"
-#include "config.h"
-#include "connection.h"
-#include "connection_edge.h"
-#include "directory.h"
-#include "main.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "relay.h"
-#include "rendclient.h"
-#include "rendcommon.h"
-#include "rephist.h"
-#include "router.h"
-#include "routerlist.h"
-#include "routerset.h"
-#include "control.h"
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/connection_edge.h"
+#include "core/or/relay.h"
+#include "feature/client/circpathbias.h"
+#include "feature/control/control.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dircommon/directory.h"
+#include "feature/hs/hs_circuit.h"
+#include "feature/hs/hs_client.h"
+#include "feature/hs/hs_common.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/rend/rendclient.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/stats/rephist.h"
+#include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/encoding/confline.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/crypt_path_st.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/extend_info_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "feature/rend/rend_intro_point_st.h"
+#include "feature/rend/rend_service_descriptor_st.h"
+#include "feature/nodelist/routerstatus_st.h"
 
 static extend_info_t *rend_client_get_random_intro_impl(
                           const rend_cache_entry_t *rend_query,
@@ -40,7 +58,7 @@ rend_client_purge_state(void)
   rend_cache_purge();
   rend_cache_failure_purge();
   rend_client_cancel_descriptor_fetches();
-  rend_client_purge_last_hid_serv_requests();
+  hs_purge_last_hid_serv_requests();
 }
 
 /** Called when we've established a circuit to an introduction point:
@@ -87,46 +105,6 @@ rend_client_send_establish_rendezvous(origin_circuit_t *circ)
   return 0;
 }
 
-/** Extend the introduction circuit <b>circ</b> to another valid
- * introduction point for the hidden service it is trying to connect
- * to, or mark it and launch a new circuit if we can't extend it.
- * Return 0 on success or possible success.  Return -1 and mark the
- * introduction circuit for close on permanent failure.
- *
- * On failure, the caller is responsible for marking the associated
- * rendezvous circuit for close. */
-static int
-rend_client_reextend_intro_circuit(origin_circuit_t *circ)
-{
-  extend_info_t *extend_info;
-  int result;
-  extend_info = rend_client_get_random_intro(circ->rend_data);
-  if (!extend_info) {
-    log_warn(LD_REND,
-             "No usable introduction points left for %s. Closing.",
-             safe_str_client(circ->rend_data->onion_address));
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
-    return -1;
-  }
-  // XXX: should we not re-extend if hs_circ_has_timed_out?
-  if (circ->remaining_relay_early_cells) {
-    log_info(LD_REND,
-             "Re-extending circ %u, this time to %s.",
-             (unsigned)circ->base_.n_circ_id,
-             safe_str_client(extend_info_describe(extend_info)));
-    result = circuit_extend_to_new_exit(circ, extend_info);
-  } else {
-    log_info(LD_REND,
-             "Closing intro circ %u (out of RELAY_EARLY cells).",
-             (unsigned)circ->base_.n_circ_id);
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED);
-    /* connection_ap_handshake_attach_circuit will launch a new intro circ. */
-    result = 0;
-  }
-  extend_info_free(extend_info);
-  return result;
-}
-
 /** Called when we're trying to connect an ap conn; sends an INTRODUCE1 cell
  * down introcirc if possible.
  */
@@ -144,18 +122,19 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
   off_t dh_offset;
   crypto_pk_t *intro_key = NULL;
   int status = 0;
+  const char *onion_address;
 
   tor_assert(introcirc->base_.purpose == CIRCUIT_PURPOSE_C_INTRODUCING);
   tor_assert(rendcirc->base_.purpose == CIRCUIT_PURPOSE_C_REND_READY);
   tor_assert(introcirc->rend_data);
   tor_assert(rendcirc->rend_data);
-  tor_assert(!rend_cmp_service_ids(introcirc->rend_data->onion_address,
-                                   rendcirc->rend_data->onion_address));
+  tor_assert(!rend_cmp_service_ids(rend_data_get_address(introcirc->rend_data),
+                                  rend_data_get_address(rendcirc->rend_data)));
   assert_circ_anonymity_ok(introcirc, options);
   assert_circ_anonymity_ok(rendcirc, options);
+  onion_address = rend_data_get_address(introcirc->rend_data);
 
-  r = rend_cache_lookup_entry(introcirc->rend_data->onion_address, -1,
-                              &entry);
+  r = rend_cache_lookup_entry(onion_address, -1, &entry);
   /* An invalid onion address is not possible else we have a big issue. */
   tor_assert(r != -EINVAL);
   if (r < 0 || !rend_client_any_intro_points_usable(entry)) {
@@ -164,16 +143,14 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
     log_info(LD_REND,
              "query %s didn't have valid rend desc in cache. "
              "Refetching descriptor.",
-             safe_str_client(introcirc->rend_data->onion_address));
+             safe_str_client(onion_address));
     rend_client_refetch_v2_renddesc(introcirc->rend_data);
     {
       connection_t *conn;
 
       while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP,
-                       AP_CONN_STATE_CIRCUIT_WAIT,
-                       introcirc->rend_data->onion_address))) {
-        connection_ap_mark_as_non_pending_circuit(TO_ENTRY_CONN(conn));
-        conn->state = AP_CONN_STATE_RENDDESC_WAIT;
+                       AP_CONN_STATE_CIRCUIT_WAIT, onion_address))) {
+        connection_ap_mark_as_waiting_for_renddesc(TO_ENTRY_CONN(conn));
       }
     }
 
@@ -195,12 +172,12 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
     log_info(LD_REND, "Could not find intro key for %s at %s; we "
              "have a v2 rend desc with %d intro points. "
              "Trying a different intro point...",
-             safe_str_client(introcirc->rend_data->onion_address),
+             safe_str_client(onion_address),
              safe_str_client(extend_info_describe(
                                    introcirc->build_state->chosen_exit)),
              smartlist_len(entry->parsed->intro_nodes));
 
-    if (rend_client_reextend_intro_circuit(introcirc)) {
+    if (hs_client_reextend_intro_circuit(introcirc)) {
       status = -2;
       goto perm_err;
     } else {
@@ -235,11 +212,12 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
   /* If version is 3, write (optional) auth data and timestamp. */
   if (entry->parsed->protocols & (1<<3)) {
     tmp[0] = 3; /* version 3 of the cell format */
-    tmp[1] = (uint8_t)introcirc->rend_data->auth_type; /* auth type, if any */
+    /* auth type, if any */
+    tmp[1] = (uint8_t) TO_REND_DATA_V2(introcirc->rend_data)->auth_type;
     v3_shift = 1;
-    if (introcirc->rend_data->auth_type != REND_NO_AUTH) {
+    if (tmp[1] != REND_NO_AUTH) {
       set_uint16(tmp+2, htons(REND_DESC_COOKIE_LEN));
-      memcpy(tmp+4, introcirc->rend_data->descriptor_cookie,
+      memcpy(tmp+4, TO_REND_DATA_V2(introcirc->rend_data)->descriptor_cookie,
              REND_DESC_COOKIE_LEN);
       v3_shift += 2+REND_DESC_COOKIE_LEN;
     }
@@ -263,6 +241,11 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
     klen = crypto_pk_asn1_encode(extend_info->onion_key,
                                  tmp+v3_shift+7+DIGEST_LEN+2,
                                  sizeof(tmp)-(v3_shift+7+DIGEST_LEN+2));
+    if (klen < 0) {
+      log_warn(LD_BUG,"Internal error: can't encode public key.");
+      status = -2;
+      goto perm_err;
+    }
     set_uint16(tmp+v3_shift+7+DIGEST_LEN, htons(klen));
     memcpy(tmp+v3_shift+7+DIGEST_LEN+2+klen, rendcirc->rend_data->rend_cookie,
            REND_COOKIE_LEN);
@@ -286,19 +269,18 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
   }
 
   if (crypto_dh_get_public(cpath->rend_dh_handshake_state, tmp+dh_offset,
-                           DH_KEY_LEN)<0) {
+                           DH1024_KEY_LEN)<0) {
     log_warn(LD_BUG, "Internal error: couldn't extract g^x.");
     status = -2;
     goto perm_err;
   }
 
-  note_crypto_pk_op(REND_CLIENT);
-  /*XXX maybe give crypto_pk_public_hybrid_encrypt a max_len arg,
+  /*XXX maybe give crypto_pk_obsolete_public_hybrid_encrypt a max_len arg,
    * to avoid buffer overflows? */
-  r = crypto_pk_public_hybrid_encrypt(intro_key, payload+DIGEST_LEN,
+  r = crypto_pk_obsolete_public_hybrid_encrypt(intro_key, payload+DIGEST_LEN,
                                       sizeof(payload)-DIGEST_LEN,
                                       tmp,
-                                      (int)(dh_offset+DH_KEY_LEN),
+                                      (int)(dh_offset+DH1024_KEY_LEN),
                                       PK_PKCS1_OAEP_PADDING, 0);
   if (r<0) {
     log_warn(LD_BUG,"Internal error: hybrid pk encrypt failed.");
@@ -368,7 +350,7 @@ rend_client_rendcirc_has_opened(origin_circuit_t *circ)
  * Called to close other intro circuits we launched in parallel.
  */
 static void
-rend_client_close_other_intros(const char *onion_address)
+rend_client_close_other_intros(const uint8_t *rend_pk_digest)
 {
   /* abort parallel intro circs, if any */
   SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, c) {
@@ -377,8 +359,7 @@ rend_client_close_other_intros(const char *onion_address)
         !c->marked_for_close && CIRCUIT_IS_ORIGIN(c)) {
       origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(c);
       if (oc->rend_data &&
-          !rend_cmp_service_ids(onion_address,
-                                oc->rend_data->onion_address)) {
+          rend_circuit_pk_digest_eq(oc, rend_pk_digest)) {
         log_info(LD_REND|LD_CIRC, "Closing introduction circuit %d that we "
                  "built in parallel (Purpose %d).", oc->global_identifier,
                  c->purpose);
@@ -399,23 +380,11 @@ rend_client_introduction_acked(origin_circuit_t *circ,
   origin_circuit_t *rendcirc;
   (void) request; // XXXX Use this.
 
-  if (circ->base_.purpose != CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT) {
-    log_warn(LD_PROTOCOL,
-             "Received REND_INTRODUCE_ACK on unexpected circuit %u.",
-             (unsigned)circ->base_.n_circ_id);
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
-    return -1;
-  }
-
   tor_assert(circ->build_state);
   tor_assert(circ->build_state->chosen_exit);
   assert_circ_anonymity_ok(circ, options);
   tor_assert(circ->rend_data);
 
-  /* For path bias: This circuit was used successfully. Valid
-   * nacks and acks count. */
-  pathbias_mark_use_success(circ);
-
   if (request_len == 0) {
     /* It's an ACK; the introduction point relayed our introduction request. */
     /* Locate the rend circ which is waiting to hear about this ack,
@@ -440,7 +409,8 @@ rend_client_introduction_acked(origin_circuit_t *circ,
     circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED);
 
     /* close any other intros launched in parallel */
-    rend_client_close_other_intros(circ->rend_data->onion_address);
+    rend_client_close_other_intros(rend_data_get_pk_digest(circ->rend_data,
+                                                           NULL));
   } else {
     /* It's a NAK; the introduction point didn't relay our request. */
     circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_INTRODUCING);
@@ -449,14 +419,14 @@ rend_client_introduction_acked(origin_circuit_t *circ,
      * If none remain, refetch the service descriptor.
      */
     log_info(LD_REND, "Got nack for %s from %s...",
-        safe_str_client(circ->rend_data->onion_address),
+        safe_str_client(rend_data_get_address(circ->rend_data)),
         safe_str_client(extend_info_describe(circ->build_state->chosen_exit)));
     if (rend_client_report_intro_point_failure(circ->build_state->chosen_exit,
                                              circ->rend_data,
                                              INTRO_POINT_FAILURE_GENERIC)>0) {
       /* There are introduction points left. Re-extend the circuit to
        * another intro point and try again. */
-      int result = rend_client_reextend_intro_circuit(circ);
+      int result = hs_client_reextend_intro_circuit(circ);
       /* XXXX If that call failed, should we close the rend circuit,
        * too? */
       return result;
@@ -472,230 +442,6 @@ rend_client_introduction_acked(origin_circuit_t *circ,
   return 0;
 }
 
-/** The period for which a hidden service directory cannot be queried for
- * the same descriptor ID again. */
-#define REND_HID_SERV_DIR_REQUERY_PERIOD (15 * 60)
-/** Test networks generate a new consensus every 5 or 10 seconds.
- * So allow them to requery HSDirs much faster. */
-#define REND_HID_SERV_DIR_REQUERY_PERIOD_TESTING (5)
-
-/** Return the period for which a hidden service directory cannot be queried
- * for the same descriptor ID again, taking TestingTorNetwork into account. */
-static time_t
-hsdir_requery_period(const or_options_t *options)
-{
-  tor_assert(options);
-
-  if (options->TestingTorNetwork) {
-    return REND_HID_SERV_DIR_REQUERY_PERIOD_TESTING;
-  } else {
-    return REND_HID_SERV_DIR_REQUERY_PERIOD;
-  }
-}
-
-/** Contains the last request times to hidden service directories for
- * certain queries; each key is a string consisting of the
- * concatenation of a base32-encoded HS directory identity digest and
- * base32-encoded HS descriptor ID; each value is a pointer to a time_t
- * holding the time of the last request for that descriptor ID to that
- * HS directory. */
-static strmap_t *last_hid_serv_requests_ = NULL;
-
-/** Returns last_hid_serv_requests_, initializing it to a new strmap if
- * necessary. */
-static strmap_t *
-get_last_hid_serv_requests(void)
-{
-  if (!last_hid_serv_requests_)
-    last_hid_serv_requests_ = strmap_new();
-  return last_hid_serv_requests_;
-}
-
-#define LAST_HID_SERV_REQUEST_KEY_LEN (REND_DESC_ID_V2_LEN_BASE32 + \
-                                       REND_DESC_ID_V2_LEN_BASE32)
-
-/** Look up the last request time to hidden service directory <b>hs_dir</b>
- * for descriptor ID <b>desc_id_base32</b>. If <b>set</b> is non-zero,
- * assign the current time <b>now</b> and return that. Otherwise, return the
- * most recent request time, or 0 if no such request has been sent before.
- */
-static time_t
-lookup_last_hid_serv_request(routerstatus_t *hs_dir,
-                             const char *desc_id_base32,
-                             time_t now, int set)
-{
-  char hsdir_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
-  char hsdir_desc_comb_id[LAST_HID_SERV_REQUEST_KEY_LEN + 1];
-  time_t *last_request_ptr;
-  strmap_t *last_hid_serv_requests = get_last_hid_serv_requests();
-  base32_encode(hsdir_id_base32, sizeof(hsdir_id_base32),
-                hs_dir->identity_digest, DIGEST_LEN);
-  tor_snprintf(hsdir_desc_comb_id, sizeof(hsdir_desc_comb_id), "%s%s",
-               hsdir_id_base32,
-               desc_id_base32);
-  /* XXX++?? tor_assert(strlen(hsdir_desc_comb_id) ==
-                       LAST_HID_SERV_REQUEST_KEY_LEN); */
-  if (set) {
-    time_t *oldptr;
-    last_request_ptr = tor_malloc_zero(sizeof(time_t));
-    *last_request_ptr = now;
-    oldptr = strmap_set(last_hid_serv_requests, hsdir_desc_comb_id,
-                        last_request_ptr);
-    tor_free(oldptr);
-  } else
-    last_request_ptr = strmap_get_lc(last_hid_serv_requests,
-                                     hsdir_desc_comb_id);
-  return (last_request_ptr) ? *last_request_ptr : 0;
-}
-
-/** Clean the history of request times to hidden service directories, so that
- * it does not contain requests older than REND_HID_SERV_DIR_REQUERY_PERIOD
- * seconds any more. */
-static void
-directory_clean_last_hid_serv_requests(time_t now)
-{
-  strmap_iter_t *iter;
-  time_t cutoff = now - hsdir_requery_period(get_options());
-  strmap_t *last_hid_serv_requests = get_last_hid_serv_requests();
-  for (iter = strmap_iter_init(last_hid_serv_requests);
-       !strmap_iter_done(iter); ) {
-    const char *key;
-    void *val;
-    time_t *ent;
-    strmap_iter_get(iter, &key, &val);
-    ent = (time_t *) val;
-    if (*ent < cutoff) {
-      iter = strmap_iter_next_rmv(last_hid_serv_requests, iter);
-      tor_free(ent);
-    } else {
-      iter = strmap_iter_next(last_hid_serv_requests, iter);
-    }
-  }
-}
-
-/** Remove all requests related to the descriptor ID <b>desc_id</b> from the
- * history of times of requests to hidden service directories.
- * <b>desc_id</b> is an unencoded descriptor ID of size DIGEST_LEN.
- *
- * This is called from rend_client_note_connection_attempt_ended(), which
- * must be idempotent, so any future changes to this function must leave it
- * idempotent too. */
-static void
-purge_hid_serv_from_last_hid_serv_requests(const char *desc_id)
-{
-  strmap_iter_t *iter;
-  strmap_t *last_hid_serv_requests = get_last_hid_serv_requests();
-  char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
-
-  /* Key is stored with the base32 encoded desc_id. */
-  base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
-                DIGEST_LEN);
-  for (iter = strmap_iter_init(last_hid_serv_requests);
-       !strmap_iter_done(iter); ) {
-    const char *key;
-    void *val;
-    strmap_iter_get(iter, &key, &val);
-    /* XXX++?? tor_assert(strlen(key) == LAST_HID_SERV_REQUEST_KEY_LEN); */
-    if (tor_memeq(key + LAST_HID_SERV_REQUEST_KEY_LEN -
-                  REND_DESC_ID_V2_LEN_BASE32,
-                  desc_id_base32,
-                  REND_DESC_ID_V2_LEN_BASE32)) {
-      iter = strmap_iter_next_rmv(last_hid_serv_requests, iter);
-      tor_free(val);
-    } else {
-      iter = strmap_iter_next(last_hid_serv_requests, iter);
-    }
-  }
-}
-
-/** Purge the history of request times to hidden service directories,
- * so that future lookups of an HS descriptor will not fail because we
- * accessed all of the HSDir relays responsible for the descriptor
- * recently. */
-void
-rend_client_purge_last_hid_serv_requests(void)
-{
-  /* Don't create the table if it doesn't exist yet (and it may very
-   * well not exist if the user hasn't accessed any HSes)... */
-  strmap_t *old_last_hid_serv_requests = last_hid_serv_requests_;
-  /* ... and let get_last_hid_serv_requests re-create it for us if
-   * necessary. */
-  last_hid_serv_requests_ = NULL;
-
-  if (old_last_hid_serv_requests != NULL) {
-    log_info(LD_REND, "Purging client last-HS-desc-request-time table");
-    strmap_free(old_last_hid_serv_requests, tor_free_);
-  }
-}
-
-/** This returns a good valid hs dir that should be used for the given
- * descriptor id.
- *
- * Return NULL on error else the hsdir node pointer. */
-static routerstatus_t *
-pick_hsdir(const char *desc_id, const char *desc_id_base32)
-{
-  smartlist_t *responsible_dirs = smartlist_new();
-  smartlist_t *usable_responsible_dirs = smartlist_new();
-  const or_options_t *options = get_options();
-  routerstatus_t *hs_dir;
-  time_t now = time(NULL);
-  int excluded_some;
-
-  tor_assert(desc_id);
-  tor_assert(desc_id_base32);
-
-  /* Determine responsible dirs. Even if we can't get all we want, work with
-   * the ones we have. If it's empty, we'll notice below. */
-  hid_serv_get_responsible_directories(responsible_dirs, desc_id);
-
-  /* Clean request history first. */
-  directory_clean_last_hid_serv_requests(now);
-
-  /* Only select those hidden service directories to which we did not send a
-   * request recently and for which we have a router descriptor here. */
-  SMARTLIST_FOREACH_BEGIN(responsible_dirs, routerstatus_t *, dir) {
-    time_t last = lookup_last_hid_serv_request(dir, desc_id_base32,
-                                               0, 0);
-    const node_t *node = node_get_by_id(dir->identity_digest);
-    if (last + hsdir_requery_period(options) >= now ||
-        !node || !node_has_descriptor(node)) {
-      SMARTLIST_DEL_CURRENT(responsible_dirs, dir);
-      continue;
-    }
-    if (!routerset_contains_node(options->ExcludeNodes, node)) {
-      smartlist_add(usable_responsible_dirs, dir);
-    }
-  } SMARTLIST_FOREACH_END(dir);
-
-  excluded_some =
-    smartlist_len(usable_responsible_dirs) < smartlist_len(responsible_dirs);
-
-  hs_dir = smartlist_choose(usable_responsible_dirs);
-  if (!hs_dir && !options->StrictNodes) {
-    hs_dir = smartlist_choose(responsible_dirs);
-  }
-
-  smartlist_free(responsible_dirs);
-  smartlist_free(usable_responsible_dirs);
-  if (!hs_dir) {
-    log_info(LD_REND, "Could not pick one of the responsible hidden "
-                      "service directories, because we requested them all "
-                      "recently without success.");
-    if (options->StrictNodes && excluded_some) {
-      log_warn(LD_REND, "Could not pick a hidden service directory for the "
-               "requested hidden service: they are all either down or "
-               "excluded, and StrictNodes is set.");
-    }
-  } else {
-    /* Remember that we are requesting a descriptor from this hidden service
-     * directory now. */
-    lookup_last_hid_serv_request(hs_dir, desc_id_base32, now, 1);
-  }
-
-  return hs_dir;
-}
-
 /** Determine the responsible hidden service directories for <b>desc_id</b>
  * and fetch the descriptor with that ID from one of them. Only
  * send a request to a hidden service directory that we have not yet tried
@@ -703,30 +449,38 @@ pick_hsdir(const char *desc_id, const char *desc_id_base32)
  * in the case that no hidden service directory is left to ask for the
  * descriptor, return 0, and in case of a failure -1.  */
 static int
-directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query,
+directory_get_from_hs_dir(const char *desc_id,
+                          const rend_data_t *rend_query,
                           routerstatus_t *rs_hsdir)
 {
   routerstatus_t *hs_dir = rs_hsdir;
   char *hsdir_fp;
   char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
   char descriptor_cookie_base64[3*REND_DESC_COOKIE_LEN_BASE64];
-#ifdef ENABLE_TOR2WEB_MODE
-  const int tor2web_mode = get_options()->Tor2webMode;
-  const int how_to_fetch = tor2web_mode ? DIRIND_ONEHOP : DIRIND_ANONYMOUS;
-#else
+  const rend_data_v2_t *rend_data;
   const int how_to_fetch = DIRIND_ANONYMOUS;
-#endif
 
   tor_assert(desc_id);
+  tor_assert(rend_query);
+  rend_data = TO_REND_DATA_V2(rend_query);
 
   base32_encode(desc_id_base32, sizeof(desc_id_base32),
                 desc_id, DIGEST_LEN);
 
   /* Automatically pick an hs dir if none given. */
   if (!rs_hsdir) {
-    hs_dir = pick_hsdir(desc_id, desc_id_base32);
+    /* Determine responsible dirs. Even if we can't get all we want, work with
+     * the ones we have. If it's empty, we'll notice in hs_pick_hsdir(). */
+    smartlist_t *responsible_dirs = smartlist_new();
+    hid_serv_get_responsible_directories(responsible_dirs, desc_id);
+
+    hs_dir = hs_pick_hsdir(responsible_dirs, desc_id_base32);
     if (!hs_dir) {
       /* No suitable hs dir can be found, stop right now. */
+      control_event_hsv2_descriptor_failed(rend_query, NULL,
+                                           "QUERY_NO_HSDIR");
+      control_event_hs_descriptor_content(rend_data_get_address(rend_query),
+                                          desc_id_base32, NULL, NULL);
       return 0;
     }
   }
@@ -740,12 +494,16 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query,
   /* Encode descriptor cookie for logging purposes. Also, if the cookie is
    * malformed, no fetch is triggered thus this needs to be done before the
    * fetch request. */
-  if (rend_query->auth_type != REND_NO_AUTH) {
+  if (rend_data->auth_type != REND_NO_AUTH) {
     if (base64_encode(descriptor_cookie_base64,
                       sizeof(descriptor_cookie_base64),
-                      rend_query->descriptor_cookie, REND_DESC_COOKIE_LEN,
+                      rend_data->descriptor_cookie,
+                      REND_DESC_COOKIE_LEN,
                       0)<0) {
       log_warn(LD_BUG, "Could not base64-encode descriptor cookie.");
+      control_event_hsv2_descriptor_failed(rend_query, hsdir_fp, "BAD_DESC");
+      control_event_hs_descriptor_content(rend_data_get_address(rend_query),
+                                          desc_id_base32, hsdir_fp, NULL);
       return 0;
     }
     /* Remove == signs. */
@@ -758,36 +516,53 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query,
   /* Send fetch request. (Pass query and possibly descriptor cookie so that
    * they can be written to the directory connection and be referred to when
    * the response arrives. */
-  directory_initiate_command_routerstatus_rend(hs_dir,
-                                          DIR_PURPOSE_FETCH_RENDDESC_V2,
-                                          ROUTER_PURPOSE_GENERAL,
-                                          how_to_fetch,
-                                          desc_id_base32,
-                                          NULL, 0, 0,
-                                          rend_query);
+  directory_request_t *req =
+    directory_request_new(DIR_PURPOSE_FETCH_RENDDESC_V2);
+  directory_request_set_routerstatus(req, hs_dir);
+  directory_request_set_indirection(req, how_to_fetch);
+  directory_request_set_resource(req, desc_id_base32);
+  directory_request_set_rend_query(req, rend_query);
+  directory_initiate_request(req);
+  directory_request_free(req);
+
   log_info(LD_REND, "Sending fetch request for v2 descriptor for "
                     "service '%s' with descriptor ID '%s', auth type %d, "
                     "and descriptor cookie '%s' to hidden service "
                     "directory %s",
-           rend_query->onion_address, desc_id_base32,
-           rend_query->auth_type,
-           (rend_query->auth_type == REND_NO_AUTH ? "[none]" :
+           rend_data->onion_address, desc_id_base32,
+           rend_data->auth_type,
+           (rend_data->auth_type == REND_NO_AUTH ? "[none]" :
             escaped_safe_str_client(descriptor_cookie_base64)),
            routerstatus_describe(hs_dir));
-  control_event_hs_descriptor_requested(rend_query,
+  control_event_hs_descriptor_requested(rend_data->onion_address,
+                                        rend_data->auth_type,
                                         hs_dir->identity_digest,
-                                        desc_id_base32);
+                                        desc_id_base32, NULL);
   return 1;
 }
 
+/** Remove tracked HSDir requests from our history for this hidden service
+ *  descriptor <b>desc_id</b> (of size DIGEST_LEN) */
+static void
+purge_v2_hidserv_req(const char *desc_id)
+{
+  char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+
+  /* The hsdir request tracker stores v2 keys using the base32 encoded
+     desc_id. Do it: */
+  base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
+                DIGEST_LEN);
+  hs_purge_hid_serv_from_last_hid_serv_requests(desc_id_base32);
+}
+
 /** Fetch a v2 descriptor using the given descriptor id. If any hsdir(s) are
  * given, they will be used instead.
  *
  * On success, 1 is returned. If no hidden service is left to ask, return 0.
  * On error, -1 is returned. */
 static int
-fetch_v2_desc_by_descid(const char *desc_id, const rend_data_t *rend_query,
-                        smartlist_t *hsdirs)
+fetch_v2_desc_by_descid(const char *desc_id,
+                        const rend_data_t *rend_query, smartlist_t *hsdirs)
 {
   int ret;
 
@@ -814,19 +589,18 @@ fetch_v2_desc_by_descid(const char *desc_id, const rend_data_t *rend_query,
 
 /** Fetch a v2 descriptor using the onion address in the given query object.
  * This will compute the descriptor id for each replicas and fetch it on the
- * given hsdir(s) if any or the responsible ones that are choosen
+ * given hsdir(s) if any or the responsible ones that are chosen
  * automatically.
  *
  * On success, 1 is returned. If no hidden service is left to ask, return 0.
  * On error, -1 is returned. */
 static int
-fetch_v2_desc_by_addr(rend_data_t *query, smartlist_t *hsdirs)
+fetch_v2_desc_by_addr(rend_data_t *rend_query, smartlist_t *hsdirs)
 {
   char descriptor_id[DIGEST_LEN];
   int replicas_left_to_try[REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS];
   int i, tries_left, ret;
-
-  tor_assert(query);
+  rend_data_v2_t *rend_data = TO_REND_DATA_V2(rend_query);
 
   /* Randomly iterate over the replicas until a descriptor can be fetched
    * from one of the consecutive nodes, or no options are left. */
@@ -840,9 +614,10 @@ fetch_v2_desc_by_addr(rend_data_t *query, smartlist_t *hsdirs)
     int chosen_replica = replicas_left_to_try[rand_val];
     replicas_left_to_try[rand_val] = replicas_left_to_try[--tries_left];
 
-    ret = rend_compute_v2_desc_id(descriptor_id, query->onion_address,
-                                  query->auth_type == REND_STEALTH_AUTH ?
-                                  query->descriptor_cookie : NULL,
+    ret = rend_compute_v2_desc_id(descriptor_id,
+                                  rend_data->onion_address,
+                                  rend_data->auth_type == REND_STEALTH_AUTH ?
+                                    rend_data->descriptor_cookie : NULL,
                                   time(NULL), chosen_replica);
     if (ret < 0) {
       /* Normally, on failure the descriptor_id is untouched but let's be
@@ -850,18 +625,17 @@ fetch_v2_desc_by_addr(rend_data_t *query, smartlist_t *hsdirs)
       goto end;
     }
 
-    if (tor_memcmp(descriptor_id, query->descriptor_id[chosen_replica],
+    if (tor_memcmp(descriptor_id, rend_data->descriptor_id[chosen_replica],
                    sizeof(descriptor_id)) != 0) {
       /* Not equal from what we currently have so purge the last hid serv
        * request cache and update the descriptor ID with the new value. */
-      purge_hid_serv_from_last_hid_serv_requests(
-                                        query->descriptor_id[chosen_replica]);
-      memcpy(query->descriptor_id[chosen_replica], descriptor_id,
-             sizeof(query->descriptor_id[chosen_replica]));
+      purge_v2_hidserv_req(rend_data->descriptor_id[chosen_replica]);
+      memcpy(rend_data->descriptor_id[chosen_replica], descriptor_id,
+             sizeof(rend_data->descriptor_id[chosen_replica]));
     }
 
     /* Trigger the fetch with the computed descriptor ID. */
-    ret = fetch_v2_desc_by_descid(descriptor_id, query, hsdirs);
+    ret = fetch_v2_desc_by_descid(descriptor_id, rend_query, hsdirs);
     if (ret != 0) {
       /* Either on success or failure, as long as we tried a fetch we are
        * done here. */
@@ -889,16 +663,23 @@ int
 rend_client_fetch_v2_desc(rend_data_t *query, smartlist_t *hsdirs)
 {
   int ret;
+  rend_data_v2_t *rend_data;
+  const char *onion_address;
 
   tor_assert(query);
 
+  /* Get the version 2 data structure of the query. */
+  rend_data = TO_REND_DATA_V2(query);
+  onion_address = rend_data_get_address(query);
+
   /* Depending on what's available in the rend data query object, we will
    * trigger a fetch by HS address or using a descriptor ID. */
 
-  if (query->onion_address[0] != '\0') {
+  if (onion_address[0] != '\0') {
     ret = fetch_v2_desc_by_addr(query, hsdirs);
-  } else if (!tor_digest_is_zero(query->desc_id_fetch)) {
-    ret = fetch_v2_desc_by_descid(query->desc_id_fetch, query, hsdirs);
+  } else if (!tor_digest_is_zero(rend_data->desc_id_fetch)) {
+    ret = fetch_v2_desc_by_descid(rend_data->desc_id_fetch, query,
+                                  hsdirs);
   } else {
     /* Query data is invalid. */
     ret = -1;
@@ -916,10 +697,11 @@ void
 rend_client_refetch_v2_renddesc(rend_data_t *rend_query)
 {
   rend_cache_entry_t *e = NULL;
+  const char *onion_address = rend_data_get_address(rend_query);
 
   tor_assert(rend_query);
   /* Before fetching, check if we already have a usable descriptor here. */
-  if (rend_cache_lookup_entry(rend_query->onion_address, -1, &e) == 0 &&
+  if (rend_cache_lookup_entry(onion_address, -1, &e) == 0 &&
       rend_client_any_intro_points_usable(e)) {
     log_info(LD_REND, "We would fetch a v2 rendezvous descriptor, but we "
                       "already have a usable descriptor here. Not fetching.");
@@ -932,7 +714,7 @@ rend_client_refetch_v2_renddesc(rend_data_t *rend_query)
     return;
   }
   log_debug(LD_REND, "Fetching v2 rendezvous descriptor for service %s",
-            safe_str_client(rend_query->onion_address));
+            safe_str_client(onion_address));
 
   rend_client_fetch_v2_desc(rend_query, NULL);
   /* We don't need to look the error code because either on failure or
@@ -968,7 +750,7 @@ rend_client_cancel_descriptor_fetches(void)
       } else {
         log_debug(LD_REND, "Marking for close dir conn fetching "
                   "rendezvous descriptor for service %s",
-                  safe_str(rd->onion_address));
+                  safe_str(rend_data_get_address(rd)));
       }
       connection_mark_for_close(conn);
     }
@@ -998,25 +780,26 @@ rend_client_cancel_descriptor_fetches(void)
  */
 int
 rend_client_report_intro_point_failure(extend_info_t *failed_intro,
-                                       rend_data_t *rend_query,
+                                       rend_data_t *rend_data,
                                        unsigned int failure_type)
 {
   int i, r;
   rend_cache_entry_t *ent;
   connection_t *conn;
+  const char *onion_address = rend_data_get_address(rend_data);
 
-  r = rend_cache_lookup_entry(rend_query->onion_address, -1, &ent);
+  r = rend_cache_lookup_entry(onion_address, -1, &ent);
   if (r < 0) {
     /* Either invalid onion address or cache entry not found. */
     switch (-r) {
     case EINVAL:
       log_warn(LD_BUG, "Malformed service ID %s.",
-          escaped_safe_str_client(rend_query->onion_address));
+               escaped_safe_str_client(onion_address));
       return -1;
     case ENOENT:
       log_info(LD_REND, "Unknown service %s. Re-fetching descriptor.",
-          escaped_safe_str_client(rend_query->onion_address));
-      rend_client_refetch_v2_renddesc(rend_query);
+               escaped_safe_str_client(onion_address));
+      rend_client_refetch_v2_renddesc(rend_data);
       return 0;
     default:
       log_warn(LD_BUG, "Unknown cache lookup returned code: %d", r);
@@ -1040,7 +823,7 @@ rend_client_report_intro_point_failure(extend_info_t *failed_intro,
       case INTRO_POINT_FAILURE_GENERIC:
         rend_cache_intro_failure_note(failure_type,
                                       (uint8_t *)failed_intro->identity_digest,
-                                      rend_query->onion_address);
+                                      onion_address);
         rend_intro_point_free(intro);
         smartlist_del(ent->parsed->intro_nodes, i);
         break;
@@ -1058,8 +841,7 @@ rend_client_report_intro_point_failure(extend_info_t *failed_intro,
           if (zap_intro_point) {
             rend_cache_intro_failure_note(
                 failure_type,
-                (uint8_t *) failed_intro->identity_digest,
-                rend_query->onion_address);
+                (uint8_t *) failed_intro->identity_digest, onion_address);
             rend_intro_point_free(intro);
             smartlist_del(ent->parsed->intro_nodes, i);
           }
@@ -1073,138 +855,43 @@ rend_client_report_intro_point_failure(extend_info_t *failed_intro,
   if (! rend_client_any_intro_points_usable(ent)) {
     log_info(LD_REND,
              "No more intro points remain for %s. Re-fetching descriptor.",
-             escaped_safe_str_client(rend_query->onion_address));
-    rend_client_refetch_v2_renddesc(rend_query);
+             escaped_safe_str_client(onion_address));
+    rend_client_refetch_v2_renddesc(rend_data);
 
     /* move all pending streams back to renddesc_wait */
     /* NOTE: We can now do this faster, if we use pending_entry_connections */
     while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP,
                                    AP_CONN_STATE_CIRCUIT_WAIT,
-                                   rend_query->onion_address))) {
-      connection_ap_mark_as_non_pending_circuit(TO_ENTRY_CONN(conn));
-      conn->state = AP_CONN_STATE_RENDDESC_WAIT;
+                                   onion_address))) {
+      connection_ap_mark_as_waiting_for_renddesc(TO_ENTRY_CONN(conn));
     }
 
     return 0;
   }
   log_info(LD_REND,"%d options left for %s.",
            smartlist_len(ent->parsed->intro_nodes),
-           escaped_safe_str_client(rend_query->onion_address));
+           escaped_safe_str_client(onion_address));
   return 1;
 }
 
-/** Called when we receive a RENDEZVOUS_ESTABLISHED cell; changes the state of
- * the circuit to C_REND_READY.
- */
-int
-rend_client_rendezvous_acked(origin_circuit_t *circ, const uint8_t *request,
-                             size_t request_len)
-{
-  (void) request;
-  (void) request_len;
-  /* we just got an ack for our establish-rendezvous. switch purposes. */
-  if (circ->base_.purpose != CIRCUIT_PURPOSE_C_ESTABLISH_REND) {
-    log_warn(LD_PROTOCOL,"Got a rendezvous ack when we weren't expecting one. "
-             "Closing circ.");
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
-    return -1;
-  }
-  log_info(LD_REND,"Got rendezvous ack. This circuit is now ready for "
-           "rendezvous.");
-  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_REND_READY);
-  /* Set timestamp_dirty, because circuit_expire_building expects it
-   * to specify when a circuit entered the _C_REND_READY state. */
-  circ->base_.timestamp_dirty = time(NULL);
-
-  /* From a path bias point of view, this circuit is now successfully used.
-   * Waiting any longer opens us up to attacks from malicious hidden services.
-   * They could induce the client to attempt to connect to their hidden
-   * service and never reply to the client's rend requests */
-  pathbias_mark_use_success(circ);
-
-  /* XXXX++ This is a pretty brute-force approach. It'd be better to
-   * attach only the connections that are waiting on this circuit, rather
-   * than trying to attach them all. See comments bug 743. */
-  /* If we already have the introduction circuit built, make sure we send
-   * the INTRODUCE cell _now_ */
-  connection_ap_attach_pending(1);
-  return 0;
-}
-
 /** The service sent us a rendezvous cell; join the circuits. */
 int
 rend_client_receive_rendezvous(origin_circuit_t *circ, const uint8_t *request,
                                size_t request_len)
 {
-  crypt_path_t *hop;
-  char keys[DIGEST_LEN+CPATH_KEY_MATERIAL_LEN];
-
-  if ((circ->base_.purpose != CIRCUIT_PURPOSE_C_REND_READY &&
-       circ->base_.purpose != CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED)
-      || !circ->build_state->pending_final_cpath) {
-    log_warn(LD_PROTOCOL,"Got rendezvous2 cell from hidden service, but not "
-             "expecting it. Closing.");
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
-    return -1;
-  }
-
-  if (request_len != DH_KEY_LEN+DIGEST_LEN) {
+  if (request_len != DH1024_KEY_LEN+DIGEST_LEN) {
     log_warn(LD_PROTOCOL,"Incorrect length (%d) on RENDEZVOUS2 cell.",
              (int)request_len);
     goto err;
   }
 
-  log_info(LD_REND,"Got RENDEZVOUS2 cell from hidden service.");
-
-  /* first DH_KEY_LEN bytes are g^y from the service. Finish the dh
-   * handshake...*/
-  tor_assert(circ->build_state);
-  tor_assert(circ->build_state->pending_final_cpath);
-  hop = circ->build_state->pending_final_cpath;
-  tor_assert(hop->rend_dh_handshake_state);
-  if (crypto_dh_compute_secret(LOG_PROTOCOL_WARN,
-                               hop->rend_dh_handshake_state, (char*)request,
-                               DH_KEY_LEN,
-                               keys, DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) {
-    log_warn(LD_GENERAL, "Couldn't complete DH handshake.");
+  if (hs_circuit_setup_e2e_rend_circ_legacy_client(circ, request) < 0) {
+    log_warn(LD_GENERAL, "Failed to setup circ");
     goto err;
   }
-  /* ... and set up cpath. */
-  if (circuit_init_cpath_crypto(hop, keys+DIGEST_LEN, 0)<0)
-    goto err;
-
-  /* Check whether the digest is right... */
-  if (tor_memneq(keys, request+DH_KEY_LEN, DIGEST_LEN)) {
-    log_warn(LD_PROTOCOL, "Incorrect digest of key material.");
-    goto err;
-  }
-
-  crypto_dh_free(hop->rend_dh_handshake_state);
-  hop->rend_dh_handshake_state = NULL;
-
-  /* All is well. Extend the circuit. */
-  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_REND_JOINED);
-  hop->state = CPATH_STATE_OPEN;
-  /* set the windows to default. these are the windows
-   * that the client thinks the service has.
-   */
-  hop->package_window = circuit_initial_package_window();
-  hop->deliver_window = CIRCWINDOW_START;
-
-  /* Now that this circuit has finished connecting to its destination,
-   * make sure circuit_get_open_circ_or_launch is willing to return it
-   * so we can actually use it. */
-  circ->hs_circ_has_timed_out = 0;
-
-  onion_append_to_cpath(&circ->cpath, hop);
-  circ->build_state->pending_final_cpath = NULL; /* prevent double-free */
-
-  circuit_try_attaching_streams(circ);
-
-  memwipe(keys, 0, sizeof(keys));
   return 0;
+
  err:
-  memwipe(keys, 0, sizeof(keys));
   circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
   return -1;
 }
@@ -1230,10 +917,11 @@ rend_client_desc_trynow(const char *query)
     rend_data = ENTRY_TO_EDGE_CONN(conn)->rend_data;
     if (!rend_data)
       continue;
-    if (rend_cmp_service_ids(query, rend_data->onion_address))
+    const char *onion_address = rend_data_get_address(rend_data);
+    if (rend_cmp_service_ids(query, onion_address))
       continue;
     assert_connection_ok(base_conn, now);
-    if (rend_cache_lookup_entry(rend_data->onion_address, -1,
+    if (rend_cache_lookup_entry(onion_address, -1,
                                 &entry) == 0 &&
         rend_client_any_intro_points_usable(entry)) {
       /* either this fetch worked, or it failed but there was a
@@ -1244,8 +932,8 @@ rend_client_desc_trynow(const char *query)
       /* restart their timeout values, so they get a fair shake at
        * connecting to the hidden service. */
       base_conn->timestamp_created = now;
-      base_conn->timestamp_lastread = now;
-      base_conn->timestamp_lastwritten = now;
+      base_conn->timestamp_last_read_allowed = now;
+      base_conn->timestamp_last_write_allowed = now;
 
       connection_ap_mark_as_pending_circuit(conn);
     } else { /* 404, or fetch didn't get that far */
@@ -1268,11 +956,12 @@ rend_client_note_connection_attempt_ended(const rend_data_t *rend_data)
 {
   unsigned int have_onion = 0;
   rend_cache_entry_t *cache_entry = NULL;
+  const char *onion_address = rend_data_get_address(rend_data);
+  rend_data_v2_t *rend_data_v2 = TO_REND_DATA_V2(rend_data);
 
-  if (*rend_data->onion_address != '\0') {
+  if (onion_address[0] != '\0') {
     /* Ignore return value; we find an entry, or we don't. */
-    (void) rend_cache_lookup_entry(rend_data->onion_address, -1,
-                                   &cache_entry);
+    (void) rend_cache_lookup_entry(onion_address, -1, &cache_entry);
     have_onion = 1;
   }
 
@@ -1286,17 +975,17 @@ rend_client_note_connection_attempt_ended(const rend_data_t *rend_data)
   /* Remove the HS's entries in last_hid_serv_requests. */
   if (have_onion) {
     unsigned int replica;
-    for (replica = 0; replica < ARRAY_LENGTH(rend_data->descriptor_id);
+    for (replica = 0; replica < ARRAY_LENGTH(rend_data_v2->descriptor_id);
          replica++) {
-      const char *desc_id = rend_data->descriptor_id[replica];
-      purge_hid_serv_from_last_hid_serv_requests(desc_id);
+      const char *desc_id = rend_data_v2->descriptor_id[replica];
+      purge_v2_hidserv_req(desc_id);
     }
     log_info(LD_REND, "Connection attempt for %s has ended; "
              "cleaning up temporary state.",
-             safe_str_client(rend_data->onion_address));
+             safe_str_client(onion_address));
   } else {
     /* We only have an ID for a fetch. Probably used by HSFETCH. */
-    purge_hid_serv_from_last_hid_serv_requests(rend_data->desc_id_fetch);
+    purge_v2_hidserv_req(rend_data_v2->desc_id_fetch);
   }
 }
 
@@ -1310,12 +999,13 @@ rend_client_get_random_intro(const rend_data_t *rend_query)
   int ret;
   extend_info_t *result;
   rend_cache_entry_t *entry;
+  const char *onion_address = rend_data_get_address(rend_query);
 
-  ret = rend_cache_lookup_entry(rend_query->onion_address, -1, &entry);
+  ret = rend_cache_lookup_entry(onion_address, -1, &entry);
   if (ret < 0 || !rend_client_any_intro_points_usable(entry)) {
     log_warn(LD_REND,
              "Query '%s' didn't have valid rend desc in cache. Failing.",
-             safe_str_client(rend_query->onion_address));
+             safe_str_client(onion_address));
     /* XXX: Should we refetch the descriptor here if the IPs are not usable
      * anymore ?. */
     return NULL;
@@ -1431,18 +1121,22 @@ rend_client_lookup_service_authorization(const char *onion_address)
   return strmap_get(auth_hid_servs, onion_address);
 }
 
+#define rend_service_authorization_free(val)                    \
+  FREE_AND_NULL(rend_service_authorization_t,                   \
+                rend_service_authorization_free_, (val))
+
 /** Helper: Free storage held by rend_service_authorization_t. */
 static void
-rend_service_authorization_free(rend_service_authorization_t *auth)
+rend_service_authorization_free_(rend_service_authorization_t *auth)
 {
   tor_free(auth);
 }
 
 /** Helper for strmap_free. */
 static void
-rend_service_authorization_strmap_item_free(void *service_auth)
+rend_service_authorization_free_void(void *service_auth)
 {
-  rend_service_authorization_free(service_auth);
+  rend_service_authorization_free_(service_auth);
 }
 
 /** Release all the storage held in auth_hid_servs.
@@ -1453,7 +1147,7 @@ rend_service_authorization_free_all(void)
   if (!auth_hid_servs) {
     return;
   }
-  strmap_free(auth_hid_servs, rend_service_authorization_strmap_item_free);
+  strmap_free(auth_hid_servs, rend_service_authorization_free_void);
   auth_hid_servs = NULL;
 }
 
@@ -1494,7 +1188,7 @@ rend_parse_service_authorization(const or_options_t *options,
       goto err;
     }
     strlcpy(auth->onion_address, onion_address, REND_SERVICE_ID_LEN_BASE32+1);
-    if (!rend_valid_service_id(auth->onion_address)) {
+    if (!rend_valid_v2_service_id(auth->onion_address)) {
       log_warn(LD_CONFIG, "Onion address has wrong format: '%s'",
                onion_address);
       goto err;
@@ -1528,40 +1222,7 @@ rend_parse_service_authorization(const or_options_t *options,
     rend_service_authorization_free_all();
     auth_hid_servs = parsed;
   } else {
-    strmap_free(parsed, rend_service_authorization_strmap_item_free);
+    strmap_free(parsed, rend_service_authorization_free_void);
   }
   return res;
 }
-
-/* Can Tor client code make direct (non-anonymous) connections to introduction
- * or rendezvous points?
- * Returns true if tor was compiled with NON_ANONYMOUS_MODE_ENABLED, and is
- * configured in Tor2web mode. */
-int
-rend_client_allow_non_anonymous_connection(const or_options_t *options)
-{
-  /* Tor2web support needs to be compiled in to a tor binary. */
-#ifdef NON_ANONYMOUS_MODE_ENABLED
-  /* Tor2web */
-  return options->Tor2webMode ? 1 : 0;
-#else
-  (void)options;
-  return 0;
-#endif
-}
-
-/* At compile-time, was non-anonymous mode enabled via
- * NON_ANONYMOUS_MODE_ENABLED ? */
-int
-rend_client_non_anonymous_mode_enabled(const or_options_t *options)
-{
-  (void)options;
-  /* Tor2web support needs to be compiled in to a tor binary. */
-#ifdef NON_ANONYMOUS_MODE_ENABLED
-  /* Tor2web */
-  return 1;
-#else
-  return 0;
-#endif
-}
-
diff --git a/src/or/rendclient.h b/src/feature/rend/rendclient.h
index b8f8c2f871..e5f333238e 100644
--- a/src/or/rendclient.h
+++ b/src/feature/rend/rendclient.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,7 +12,7 @@
 #ifndef TOR_RENDCLIENT_H
 #define TOR_RENDCLIENT_H
 
-#include "rendcache.h"
+#include "feature/rend/rendcache.h"
 
 void rend_client_purge_state(void);
 
@@ -24,15 +24,11 @@ int rend_client_introduction_acked(origin_circuit_t *circ,
 void rend_client_refetch_v2_renddesc(rend_data_t *rend_query);
 int rend_client_fetch_v2_desc(rend_data_t *query, smartlist_t *hsdirs);
 void rend_client_cancel_descriptor_fetches(void);
-void rend_client_purge_last_hid_serv_requests(void);
 
 int rend_client_report_intro_point_failure(extend_info_t *failed_intro,
-                                           rend_data_t *rend_query,
+                                           rend_data_t *rend_data,
                                            unsigned int failure_type);
 
-int rend_client_rendezvous_acked(origin_circuit_t *circ,
-                                 const uint8_t *request,
-                                 size_t request_len);
 int rend_client_receive_rendezvous(origin_circuit_t *circ,
                                    const uint8_t *request,
                                    size_t request_len);
@@ -51,8 +47,5 @@ rend_service_authorization_t *rend_client_lookup_service_authorization(
                                                 const char *onion_address);
 void rend_service_authorization_free_all(void);
 
-int rend_client_allow_non_anonymous_connection(const or_options_t *options);
-int rend_client_non_anonymous_mode_enabled(const or_options_t *options);
-
-#endif
+#endif /* !defined(TOR_RENDCLIENT_H) */
 
diff --git a/src/or/rendcommon.c b/src/feature/rend/rendcommon.c
index d9d39b1f19..de48af795f 100644
--- a/src/or/rendcommon.c
+++ b/src/feature/rend/rendcommon.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -8,19 +8,40 @@
  * introducers, services, clients, and rendezvous points.
  **/
 
-#include "or.h"
-#include "circuitbuild.h"
-#include "config.h"
-#include "control.h"
-#include "rendclient.h"
-#include "rendcommon.h"
-#include "rendmid.h"
-#include "rendservice.h"
-#include "rephist.h"
-#include "router.h"
-#include "routerlist.h"
-#include "routerparse.h"
-#include "networkstatus.h"
+#define RENDCOMMON_PRIVATE
+
+#include "core/or/or.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "app/config/config.h"
+#include "feature/control/control.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "feature/hs/hs_client.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_intropoint.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/rend/rendclient.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/rend/rendmid.h"
+#include "feature/rend/rendparse.h"
+#include "feature/rend/rendservice.h"
+#include "feature/stats/rephist.h"
+#include "feature/hs_common/replaycache.h"
+#include "feature/relay/router.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/dirparse/signing.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/crypt_path_st.h"
+#include "core/or/extend_info_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "feature/rend/rend_encoded_v2_service_descriptor_st.h"
+#include "feature/rend/rend_intro_point_st.h"
+#include "feature/rend/rend_service_descriptor_st.h"
+#include "feature/nodelist/routerstatus_st.h"
 
 /** Return 0 if one and two are the same service ids, else -1 or 1 */
 int
@@ -32,7 +53,7 @@ rend_cmp_service_ids(const char *one, const char *two)
 /** Free the storage held by the service descriptor <b>desc</b>.
  */
 void
-rend_service_descriptor_free(rend_service_descriptor_t *desc)
+rend_service_descriptor_free_(rend_service_descriptor_t *desc)
 {
   if (!desc)
     return;
@@ -393,7 +414,7 @@ rend_encrypt_v2_intro_points_stealth(char **encrypted_out,
 
 /** Attempt to parse the given <b>desc_str</b> and return true if this
  * succeeds, false otherwise. */
-static int
+STATIC int
 rend_desc_v2_is_parsable(rend_encoded_v2_service_descriptor_t *desc)
 {
   rend_service_descriptor_t *test_parsed = NULL;
@@ -414,7 +435,7 @@ rend_desc_v2_is_parsable(rend_encoded_v2_service_descriptor_t *desc)
 
 /** Free the storage held by an encoded v2 service descriptor. */
 void
-rend_encoded_v2_service_descriptor_free(
+rend_encoded_v2_service_descriptor_free_(
   rend_encoded_v2_service_descriptor_t *desc)
 {
   if (!desc)
@@ -425,7 +446,7 @@ rend_encoded_v2_service_descriptor_free(
 
 /** Free the storage held by an introduction point info. */
 void
-rend_intro_point_free(rend_intro_point_t *intro)
+rend_intro_point_free_(rend_intro_point_t *intro)
 {
   if (!intro)
     return;
@@ -475,7 +496,10 @@ rend_encode_v2_descriptors(smartlist_t *descs_out,
     tor_assert(descriptor_cookie);
   }
   /* Obtain service_id from public key. */
-  crypto_pk_get_digest(service_key, service_id);
+  if (crypto_pk_get_digest(service_key, service_id) < 0) {
+    log_warn(LD_BUG, "Couldn't compute service key digest.");
+    return -1;
+  }
   /* Calculate current time-period. */
   time_period = get_time_period(now, period, service_id);
   /* Determine how many seconds the descriptor will be valid. */
@@ -691,7 +715,7 @@ rend_get_service_id(crypto_pk_t *pk, char *out)
 /** Return true iff <b>query</b> is a syntactically valid service ID (as
  * generated by rend_get_service_id).  */
 int
-rend_valid_service_id(const char *query)
+rend_valid_v2_service_id(const char *query)
 {
   if (strlen(query) != REND_SERVICE_ID_LEN_BASE32)
     return 0;
@@ -761,7 +785,7 @@ rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint,
   switch (command) {
     case RELAY_COMMAND_ESTABLISH_INTRO:
       if (or_circ)
-        r = rend_mid_establish_intro(or_circ,payload,length);
+        r = hs_intro_received_establish_intro(or_circ,payload,length);
       break;
     case RELAY_COMMAND_ESTABLISH_RENDEZVOUS:
       if (or_circ)
@@ -769,15 +793,15 @@ rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint,
       break;
     case RELAY_COMMAND_INTRODUCE1:
       if (or_circ)
-        r = rend_mid_introduce(or_circ,payload,length);
+        r = hs_intro_received_introduce1(or_circ,payload,length);
       break;
     case RELAY_COMMAND_INTRODUCE2:
       if (origin_circ)
-        r = rend_service_receive_introduction(origin_circ,payload,length);
+        r = hs_service_receive_introduce2(origin_circ,payload,length);
       break;
     case RELAY_COMMAND_INTRODUCE_ACK:
       if (origin_circ)
-        r = rend_client_introduction_acked(origin_circ,payload,length);
+        r = hs_client_receive_introduce_ack(origin_circ,payload,length);
       break;
     case RELAY_COMMAND_RENDEZVOUS1:
       if (or_circ)
@@ -785,143 +809,30 @@ rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint,
       break;
     case RELAY_COMMAND_RENDEZVOUS2:
       if (origin_circ)
-        r = rend_client_receive_rendezvous(origin_circ,payload,length);
+        r = hs_client_receive_rendezvous2(origin_circ,payload,length);
       break;
     case RELAY_COMMAND_INTRO_ESTABLISHED:
       if (origin_circ)
-        r = rend_service_intro_established(origin_circ,payload,length);
+        r = hs_service_receive_intro_established(origin_circ,payload,length);
       break;
     case RELAY_COMMAND_RENDEZVOUS_ESTABLISHED:
       if (origin_circ)
-        r = rend_client_rendezvous_acked(origin_circ,payload,length);
+        r = hs_client_receive_rendezvous_acked(origin_circ,payload,length);
       break;
     default:
       tor_fragile_assert();
   }
 
+  if (r == 0 && origin_circ) {
+    /* This was a valid cell. Count it as delivered + overhead. */
+    circuit_read_valid_data(origin_circ, length);
+  }
+
   if (r == -2)
     log_info(LD_PROTOCOL, "Dropping cell (type %d) for wrong circuit type.",
              command);
 }
 
-/** Allocate and return a new rend_data_t with the same
- * contents as <b>query</b>. */
-rend_data_t *
-rend_data_dup(const rend_data_t *data)
-{
-  rend_data_t *data_dup;
-  tor_assert(data);
-  data_dup = tor_memdup(data, sizeof(rend_data_t));
-  data_dup->hsdirs_fp = smartlist_new();
-  SMARTLIST_FOREACH(data->hsdirs_fp, char *, fp,
-                    smartlist_add(data_dup->hsdirs_fp,
-                                  tor_memdup(fp, DIGEST_LEN)));
-  return data_dup;
-}
-
-/** Compute descriptor ID for each replicas and save them. A valid onion
- * address must be present in the <b>rend_data</b>.
- *
- * Return 0 on success else -1. */
-static int
-compute_desc_id(rend_data_t *rend_data)
-{
-  int ret = 0;
-  unsigned replica;
-  time_t now = time(NULL);
-
-  tor_assert(rend_data);
-
-  /* Compute descriptor ID for each replicas. */
-  for (replica = 0; replica < ARRAY_LENGTH(rend_data->descriptor_id);
-       replica++) {
-    ret = rend_compute_v2_desc_id(rend_data->descriptor_id[replica],
-                                  rend_data->onion_address,
-                                  rend_data->descriptor_cookie,
-                                  now, replica);
-    if (ret < 0) {
-      goto end;
-    }
-  }
-
- end:
-  return ret;
-}
-
-/** Allocate and initialize a rend_data_t object for a service using the
- * given arguments. Only the <b>onion_address</b> is not optional.
- *
- * Return a valid rend_data_t pointer. */
-rend_data_t *
-rend_data_service_create(const char *onion_address, const char *pk_digest,
-                         const uint8_t *cookie, rend_auth_type_t auth_type)
-{
-  rend_data_t *rend_data = tor_malloc_zero(sizeof(*rend_data));
-
-  /* We need at least one else the call is wrong. */
-  tor_assert(onion_address != NULL);
-
-  if (pk_digest) {
-    memcpy(rend_data->rend_pk_digest, pk_digest,
-           sizeof(rend_data->rend_pk_digest));
-  }
-  if (cookie) {
-    memcpy(rend_data->rend_cookie, cookie,
-           sizeof(rend_data->rend_cookie));
-  }
-
-  strlcpy(rend_data->onion_address, onion_address,
-          sizeof(rend_data->onion_address));
-  rend_data->auth_type = auth_type;
-  /* Won't be used but still need to initialize it for rend_data dup and
-   * free. */
-  rend_data->hsdirs_fp = smartlist_new();
-
-  return rend_data;
-}
-
-/** Allocate and initialize a rend_data_t object for a client request using
- * the given arguments.  Either an onion address or a descriptor ID is
- * needed. Both can be given but only the onion address will be used to make
- * the descriptor fetch.
- *
- * Return a valid rend_data_t pointer or NULL on error meaning the
- * descriptor IDs couldn't be computed from the given data. */
-rend_data_t *
-rend_data_client_create(const char *onion_address, const char *desc_id,
-                        const char *cookie, rend_auth_type_t auth_type)
-{
-  rend_data_t *rend_data = tor_malloc_zero(sizeof(*rend_data));
-
-  /* We need at least one else the call is wrong. */
-  tor_assert(onion_address != NULL || desc_id != NULL);
-
-  if (cookie) {
-    memcpy(rend_data->descriptor_cookie, cookie,
-           sizeof(rend_data->descriptor_cookie));
-  }
-  if (desc_id) {
-    memcpy(rend_data->desc_id_fetch, desc_id,
-           sizeof(rend_data->desc_id_fetch));
-  }
-  if (onion_address) {
-    strlcpy(rend_data->onion_address, onion_address,
-            sizeof(rend_data->onion_address));
-    if (compute_desc_id(rend_data) < 0) {
-      goto error;
-    }
-  }
-
-  rend_data->auth_type = auth_type;
-  rend_data->hsdirs_fp = smartlist_new();
-
-  return rend_data;
-
- error:
-  rend_data_free(rend_data);
-  return NULL;
-}
-
 /** Determine the routers that are responsible for <b>id</b> (binary) and
  * add pointers to those routers' routerstatus_t to <b>responsible_dirs</b>.
  * Return -1 if we're returning an empty smartlist, else return 0.
@@ -933,7 +844,7 @@ hid_serv_get_responsible_directories(smartlist_t *responsible_dirs,
   int start, found, n_added = 0, i;
   networkstatus_t *c = networkstatus_get_latest_consensus();
   if (!c || !smartlist_len(c->routerstatus_list)) {
-    log_warn(LD_REND, "We don't have a consensus, so we can't perform v2 "
+    log_info(LD_REND, "We don't have a consensus, so we can't perform v2 "
              "rendezvous operations.");
     return -1;
   }
@@ -1069,42 +980,32 @@ rend_auth_decode_cookie(const char *cookie_in, uint8_t *cookie_out,
 
 /* Is this a rend client or server that allows direct (non-anonymous)
  * connections?
- * Clients must be specifically compiled and configured in this mode.
- * Onion services can be configured to start in this mode.
- * Prefer rend_client_allow_non_anonymous_connection() or
- * rend_service_allow_non_anonymous_connection() whenever possible, so that
- * checks are specific to Single Onion Services or Tor2web. */
+ * Onion services can be configured to start in this mode for single onion. */
 int
 rend_allow_non_anonymous_connection(const or_options_t* options)
 {
-  return (rend_client_allow_non_anonymous_connection(options)
-          || rend_service_allow_non_anonymous_connection(options));
+  return rend_service_allow_non_anonymous_connection(options);
 }
 
 /* Is this a rend client or server in non-anonymous mode?
- * Clients must be specifically compiled in this mode.
- * Onion services can be configured to start in this mode.
- * Prefer rend_client_non_anonymous_mode_enabled() or
- * rend_service_non_anonymous_mode_enabled() whenever possible, so that checks
- * are specific to Single Onion Services or Tor2web. */
+ * Onion services can be configured to start in this mode for single onion. */
 int
 rend_non_anonymous_mode_enabled(const or_options_t *options)
 {
-  return (rend_client_non_anonymous_mode_enabled(options)
-          || rend_service_non_anonymous_mode_enabled(options));
+  return rend_service_non_anonymous_mode_enabled(options);
 }
 
 /* Make sure that tor only builds one-hop circuits when they would not
  * compromise user anonymity.
  *
- * One-hop circuits are permitted in Tor2web or Single Onion modes.
+ * One-hop circuits are permitted in Single Onion modes.
  *
- * Tor2web or Single Onion modes are also allowed to make multi-hop circuits.
+ * Single Onion modes are also allowed to make multi-hop circuits.
  * For example, single onion HSDir circuits are 3-hop to prevent denial of
  * service.
  */
 void
-assert_circ_anonymity_ok(origin_circuit_t *circ,
+assert_circ_anonymity_ok(const origin_circuit_t *circ,
                          const or_options_t *options)
 {
   tor_assert(options);
@@ -1116,3 +1017,31 @@ assert_circ_anonymity_ok(origin_circuit_t *circ,
   }
 }
 
+/* Return 1 iff the given <b>digest</b> of a permenanent hidden service key is
+ * equal to the digest in the origin circuit <b>ocirc</b> of its rend data .
+ * If the rend data doesn't exist, 0 is returned. This function is agnostic to
+ * the rend data version. */
+int
+rend_circuit_pk_digest_eq(const origin_circuit_t *ocirc,
+                          const uint8_t *digest)
+{
+  size_t rend_pk_digest_len;
+  const uint8_t *rend_pk_digest;
+
+  tor_assert(ocirc);
+  tor_assert(digest);
+
+  if (ocirc->rend_data == NULL) {
+    goto no_match;
+  }
+
+  rend_pk_digest = rend_data_get_pk_digest(ocirc->rend_data,
+                                           &rend_pk_digest_len);
+  if (tor_memeq(rend_pk_digest, digest, rend_pk_digest_len)) {
+    goto match;
+  }
+ no_match:
+  return 0;
+ match:
+  return 1;
+}
diff --git a/src/or/rendcommon.h b/src/feature/rend/rendcommon.h
index 090e6f25e0..f136863c7a 100644
--- a/src/or/rendcommon.h
+++ b/src/feature/rend/rendcommon.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,32 +18,27 @@ typedef enum rend_intro_point_failure_t {
   INTRO_POINT_FAILURE_UNREACHABLE = 2,
 } rend_intro_point_failure_t;
 
-/** Free all storage associated with <b>data</b> */
-static inline void
-rend_data_free(rend_data_t *data)
-{
-  if (!data) {
-    return;
-  }
-  /* Cleanup the HSDir identity digest. */
-  SMARTLIST_FOREACH(data->hsdirs_fp, char *, d, tor_free(d));
-  smartlist_free(data->hsdirs_fp);
-  tor_free(data);
-}
-
 int rend_cmp_service_ids(const char *one, const char *two);
 
 void rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint,
                              int command, size_t length,
                              const uint8_t *payload);
 
-void rend_service_descriptor_free(rend_service_descriptor_t *desc);
+void rend_service_descriptor_free_(rend_service_descriptor_t *desc);
+#define rend_service_descriptor_free(desc) \
+  FREE_AND_NULL(rend_service_descriptor_t, rend_service_descriptor_free_, \
+                (desc))
 int rend_get_service_id(crypto_pk_t *pk, char *out);
-void rend_encoded_v2_service_descriptor_free(
+void rend_encoded_v2_service_descriptor_free_(
                                rend_encoded_v2_service_descriptor_t *desc);
-void rend_intro_point_free(rend_intro_point_t *intro);
+#define rend_encoded_v2_service_descriptor_free(desc) \
+  FREE_AND_NULL(rend_encoded_v2_service_descriptor_t, \
+                rend_encoded_v2_service_descriptor_free_, (desc))
+void rend_intro_point_free_(rend_intro_point_t *intro);
+#define rend_intro_point_free(intro) \
+  FREE_AND_NULL(rend_intro_point_t, rend_intro_point_free_, (intro))
 
-int rend_valid_service_id(const char *query);
+int rend_valid_v2_service_id(const char *query);
 int rend_valid_descriptor_id(const char *query);
 int rend_valid_client_name(const char *client_name);
 int rend_encode_v2_descriptors(smartlist_t *descs_out,
@@ -60,15 +55,8 @@ void rend_get_descriptor_id_bytes(char *descriptor_id_out,
 int hid_serv_get_responsible_directories(smartlist_t *responsible_dirs,
                                          const char *id);
 
-rend_data_t *rend_data_dup(const rend_data_t *data);
-rend_data_t *rend_data_client_create(const char *onion_address,
-                                     const char *desc_id,
-                                     const char *cookie,
-                                     rend_auth_type_t auth_type);
-rend_data_t *rend_data_service_create(const char *onion_address,
-                                      const char *pk_digest,
-                                      const uint8_t *cookie,
-                                      rend_auth_type_t auth_type);
+int rend_circuit_pk_digest_eq(const origin_circuit_t *ocirc,
+                              const uint8_t *digest);
 
 char *rend_auth_encode_cookie(const uint8_t *cookie_in,
                               rend_auth_type_t auth_type);
@@ -80,8 +68,15 @@ int rend_auth_decode_cookie(const char *cookie_in,
 int rend_allow_non_anonymous_connection(const or_options_t* options);
 int rend_non_anonymous_mode_enabled(const or_options_t *options);
 
-void assert_circ_anonymity_ok(origin_circuit_t *circ,
+void assert_circ_anonymity_ok(const origin_circuit_t *circ,
                               const or_options_t *options);
 
-#endif
+#ifdef RENDCOMMON_PRIVATE
+
+STATIC int
+rend_desc_v2_is_parsable(rend_encoded_v2_service_descriptor_t *desc);
+
+#endif /* defined(RENDCOMMON_PRIVATE) */
+
+#endif /* !defined(TOR_RENDCOMMON_H) */
 
diff --git a/src/or/rendmid.c b/src/feature/rend/rendmid.c
index 08b2e8923a..3ba48f8858 100644
--- a/src/or/rendmid.c
+++ b/src/feature/rend/rendmid.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -7,22 +7,27 @@
  * \brief Implement introductions points and rendezvous points.
  **/
 
-#include "or.h"
-#include "channel.h"
-#include "circuitlist.h"
-#include "circuituse.h"
-#include "config.h"
-#include "dos.h"
-#include "relay.h"
-#include "rendmid.h"
-#include "rephist.h"
+#include "core/or/or.h"
+#include "core/or/channel.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "app/config/config.h"
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "core/or/dos.h"
+#include "core/or/relay.h"
+#include "feature/rend/rendmid.h"
+#include "feature/stats/rephist.h"
+#include "feature/hs/hs_circuitmap.h"
+#include "feature/hs/hs_intropoint.h"
+
+#include "core/or/or_circuit_st.h"
 
 /** Respond to an ESTABLISH_INTRO cell by checking the signed data and
  * setting the circuit's purpose and service pk digest.
  */
 int
-rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
-                         size_t request_len)
+rend_mid_establish_intro_legacy(or_circuit_t *circ, const uint8_t *request,
+                                size_t request_len)
 {
   crypto_pk_t *pk = NULL;
   char buf[DIGEST_LEN+9];
@@ -34,15 +39,14 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
   int reason = END_CIRC_REASON_INTERNAL;
 
   log_info(LD_REND,
-           "Received an ESTABLISH_INTRO request on circuit %u",
+           "Received a legacy ESTABLISH_INTRO request on circuit %u",
            (unsigned) circ->p_circ_id);
 
-  if (circ->base_.purpose != CIRCUIT_PURPOSE_OR || circ->base_.n_chan) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-         "Rejecting ESTABLISH_INTRO on non-OR or non-edge circuit.");
+  if (!hs_intro_circuit_is_suitable_for_establish_intro(circ)) {
     reason = END_CIRC_REASON_TORPROTOCOL;
     goto err;
   }
+
   if (request_len < 2+DIGEST_LEN)
     goto truncated;
   /* First 2 bytes: length of asn1-encoded key. */
@@ -66,12 +70,12 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
     goto err;
   }
   if (tor_memneq(expected_digest, request+2+asn1len, DIGEST_LEN)) {
-    log_warn(LD_PROTOCOL, "Hash of session info was not as expected.");
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Hash of session info was not as expected.");
     reason = END_CIRC_REASON_TORPROTOCOL;
     goto err;
   }
   /* Rest of body: signature of previous data */
-  note_crypto_pk_op(REND_MID);
   if (crypto_pk_public_checksig_digest(pk,
                                        (char*)request, 2+asn1len+DIGEST_LEN,
                                        (char*)(request+2+DIGEST_LEN+asn1len),
@@ -96,7 +100,8 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
 
   /* Close any other intro circuits with the same pk. */
   c = NULL;
-  while ((c = circuit_get_intro_point((const uint8_t *)pk_digest))) {
+  while ((c = hs_circuitmap_get_intro_circ_v2_relay_side(
+                                                (const uint8_t *)pk_digest))) {
     log_info(LD_REND, "Replacing old circuit for service %s",
              safe_str(serviceid));
     circuit_mark_for_close(TO_CIRCUIT(c), END_CIRC_REASON_FINISHED);
@@ -104,16 +109,14 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
   }
 
   /* Acknowledge the request. */
-  if (relay_send_command_from_edge(0, TO_CIRCUIT(circ),
-                                   RELAY_COMMAND_INTRO_ESTABLISHED,
-                                   "", 0, NULL)<0) {
+  if (hs_intro_send_intro_established_cell(circ) < 0) {
     log_info(LD_GENERAL, "Couldn't send INTRO_ESTABLISHED cell.");
-    goto err;
+    goto err_no_close;
   }
 
   /* Now, set up this circuit. */
   circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_INTRO_POINT);
-  circuit_set_intro_point_digest(circ, (uint8_t *)pk_digest);
+  hs_circuitmap_register_intro_circ_v2_relay_side(circ, (uint8_t *)pk_digest);
 
   log_info(LD_REND,
            "Established introduction point on circuit %u for service %s",
@@ -124,8 +127,9 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
   log_warn(LD_PROTOCOL, "Rejecting truncated ESTABLISH_INTRO cell.");
   reason = END_CIRC_REASON_TORPROTOCOL;
  err:
-  if (pk) crypto_pk_free(pk);
   circuit_mark_for_close(TO_CIRCUIT(circ), reason);
+ err_no_close:
+  if (pk) crypto_pk_free(pk);
   return -1;
 }
 
@@ -134,8 +138,8 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
  * INTRODUCE2 cell.
  */
 int
-rend_mid_introduce(or_circuit_t *circ, const uint8_t *request,
-                   size_t request_len)
+rend_mid_introduce_legacy(or_circuit_t *circ, const uint8_t *request,
+                          size_t request_len)
 {
   or_circuit_t *intro_circ;
   char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
@@ -144,33 +148,18 @@ rend_mid_introduce(or_circuit_t *circ, const uint8_t *request,
   log_info(LD_REND, "Received an INTRODUCE1 request on circuit %u",
            (unsigned)circ->p_circ_id);
 
-  if (circ->base_.purpose != CIRCUIT_PURPOSE_OR || circ->base_.n_chan) {
-    log_warn(LD_PROTOCOL,
-             "Rejecting INTRODUCE1 on non-OR or non-edge circuit %u.",
-             (unsigned)circ->p_circ_id);
-    goto err;
-  }
-
-  /* We have already done an introduction on this circuit but we just
-     received a request for another one. We block it since this might
-     be an attempt to DoS a hidden service (#15515). */
-  if (circ->already_received_introduce1) {
-    log_fn(LOG_PROTOCOL_WARN, LD_REND,
-           "Blocking multiple introductions on the same circuit. "
-           "Someone might be trying to attack a hidden service through "
-           "this relay.");
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
-    return -1;
-  }
-
-  circ->already_received_introduce1 = 1;
+  /* At this point, we know that the circuit is valid for an INTRODUCE1
+   * because the validation has been made before calling this function. */
+  tor_assert(circ->base_.purpose == CIRCUIT_PURPOSE_OR);
+  tor_assert(!circ->base_.n_chan);
 
   /* We could change this to MAX_HEX_NICKNAME_LEN now that 0.0.9.x is
    * obsolete; however, there isn't much reason to do so, and we're going
    * to revise this protocol anyway.
    */
   if (request_len < (DIGEST_LEN+(MAX_NICKNAME_LEN+1)+REND_COOKIE_LEN+
-                     DH_KEY_LEN+CIPHER_KEY_LEN+PKCS1_OAEP_PADDING_OVERHEAD)) {
+                     DH1024_KEY_LEN+CIPHER_KEY_LEN+
+                     PKCS1_OAEP_PADDING_OVERHEAD)) {
     log_warn(LD_PROTOCOL, "Impossibly short INTRODUCE1 cell on circuit %u; "
              "responding with nack.",
              (unsigned)circ->p_circ_id);
@@ -182,7 +171,8 @@ rend_mid_introduce(or_circuit_t *circ, const uint8_t *request,
 
   /* The first 20 bytes are all we look at: they have a hash of the service's
    * PK. */
-  intro_circ = circuit_get_intro_point((const uint8_t*)request);
+  intro_circ = hs_circuitmap_get_intro_circ_v2_relay_side(
+                                                      (const uint8_t*)request);
   if (!intro_circ) {
     log_info(LD_REND,
              "No intro circ found for INTRODUCE1 cell (%s) from circuit %u; "
@@ -203,14 +193,15 @@ rend_mid_introduce(or_circuit_t *circ, const uint8_t *request,
                                    (char*)request, request_len, NULL)) {
     log_warn(LD_GENERAL,
              "Unable to send INTRODUCE2 cell to Tor client.");
-    goto err;
+    /* Stop right now, the circuit has been closed. */
+    return -1;
   }
   /* And send an ack down the client's circuit.  Empty body means succeeded. */
   if (relay_send_command_from_edge(0,TO_CIRCUIT(circ),
                                    RELAY_COMMAND_INTRODUCE_ACK,
                                    NULL,0,NULL)) {
     log_warn(LD_GENERAL, "Unable to send INTRODUCE_ACK cell to Tor client.");
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
+    /* Stop right now, the circuit has been closed. */
     return -1;
   }
 
@@ -222,8 +213,6 @@ rend_mid_introduce(or_circuit_t *circ, const uint8_t *request,
                                    RELAY_COMMAND_INTRODUCE_ACK,
                                    nak_body, 1, NULL)) {
     log_warn(LD_GENERAL, "Unable to send NAK to Tor client.");
-    /* Is this right? */
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
   }
   return -1;
 }
@@ -249,7 +238,7 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request,
   }
 
   /* Check if we are configured to accept established rendezvous cells from
-   * client or in other words tor2web clients. */
+   * client or in other words Tor2Web clients. */
   if (channel_is_client(circ->p_chan) &&
       dos_should_refuse_single_hop_client()) {
     /* Note it down for the heartbeat log purposes. */
@@ -270,7 +259,7 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request,
     goto err;
   }
 
-  if (circuit_get_rendezvous(request)) {
+  if (hs_circuitmap_get_rend_circ_relay_side(request)) {
     log_warn(LD_PROTOCOL,
              "Duplicate rendezvous cookie in ESTABLISH_RENDEZVOUS.");
     goto err;
@@ -281,12 +270,12 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request,
                                    RELAY_COMMAND_RENDEZVOUS_ESTABLISHED,
                                    "", 0, NULL)<0) {
     log_warn(LD_PROTOCOL, "Couldn't send RENDEZVOUS_ESTABLISHED cell.");
-    reason = END_CIRC_REASON_INTERNAL;
-    goto err;
+    /* Stop right now, the circuit has been closed. */
+    return -1;
   }
 
   circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_REND_POINT_WAITING);
-  circuit_set_rendezvous_cookie(circ, request);
+  hs_circuitmap_register_rend_circ_relay_side(circ, request);
 
   base16_encode(hexid,9,(char*)request,4);
 
@@ -335,7 +324,7 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request,
            "Got request for rendezvous from circuit %u to cookie %s.",
            (unsigned)circ->p_circ_id, hexid);
 
-  rend_circ = circuit_get_rendezvous(request);
+  rend_circ = hs_circuitmap_get_rend_circ_relay_side(request);
   if (!rend_circ) {
     log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
          "Rejecting RENDEZVOUS1 cell with unrecognized rendezvous cookie %s.",
@@ -358,7 +347,8 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request,
     log_warn(LD_GENERAL,
              "Unable to send RENDEZVOUS2 cell to client on circuit %u.",
              (unsigned)rend_circ->p_circ_id);
-    goto err;
+    /* Stop right now, the circuit has been closed. */
+    return -1;
   }
 
   /* Join the circuits. */
@@ -369,7 +359,7 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request,
   circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_REND_ESTABLISHED);
   circuit_change_purpose(TO_CIRCUIT(rend_circ),
                          CIRCUIT_PURPOSE_REND_ESTABLISHED);
-  circuit_set_rendezvous_cookie(circ, NULL);
+  hs_circuitmap_remove_circuit(TO_CIRCUIT(circ));
 
   rend_circ->rend_splice = circ;
   circ->rend_splice = rend_circ;
@@ -379,4 +369,3 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request,
   circuit_mark_for_close(TO_CIRCUIT(circ), reason);
   return -1;
 }
-
diff --git a/src/or/rendmid.h b/src/feature/rend/rendmid.h
index 10d1287085..8ae1fa16b8 100644
--- a/src/or/rendmid.h
+++ b/src/feature/rend/rendmid.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,14 +12,14 @@
 #ifndef TOR_RENDMID_H
 #define TOR_RENDMID_H
 
-int rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
-                             size_t request_len);
-int rend_mid_introduce(or_circuit_t *circ, const uint8_t *request,
-                       size_t request_len);
+int rend_mid_establish_intro_legacy(or_circuit_t *circ, const uint8_t *request,
+                                    size_t request_len);
+int rend_mid_introduce_legacy(or_circuit_t *circ, const uint8_t *request,
+                              size_t request_len);
 int rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request,
                                   size_t request_len);
 int rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request,
                         size_t request_len);
 
-#endif
+#endif /* !defined(TOR_RENDMID_H) */
 
diff --git a/src/feature/rend/rendparse.c b/src/feature/rend/rendparse.c
new file mode 100644
index 0000000000..abd0feb448
--- /dev/null
+++ b/src/feature/rend/rendparse.c
@@ -0,0 +1,600 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rendparse.c
+ * \brief Code to parse and validate v2 hidden service descriptors.
+ **/
+
+#include "core/or/or.h"
+#include "feature/dirparse/parsecommon.h"
+#include "feature/dirparse/sigcommon.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/rend/rendparse.h"
+#include "lib/memarea/memarea.h"
+
+#include "core/or/extend_info_st.h"
+#include "feature/rend/rend_authorized_client_st.h"
+#include "feature/rend/rend_intro_point_st.h"
+#include "feature/rend/rend_service_descriptor_st.h"
+
+/** List of tokens recognized in rendezvous service descriptors */
+static token_rule_t desc_token_table[] = {
+  T1_START("rendezvous-service-descriptor", R_RENDEZVOUS_SERVICE_DESCRIPTOR,
+           EQ(1), NO_OBJ),
+  T1("version", R_VERSION, EQ(1), NO_OBJ),
+  T1("permanent-key", R_PERMANENT_KEY, NO_ARGS, NEED_KEY_1024),
+  T1("secret-id-part", R_SECRET_ID_PART, EQ(1), NO_OBJ),
+  T1("publication-time", R_PUBLICATION_TIME, CONCAT_ARGS, NO_OBJ),
+  T1("protocol-versions", R_PROTOCOL_VERSIONS, EQ(1), NO_OBJ),
+  T01("introduction-points", R_INTRODUCTION_POINTS, NO_ARGS, NEED_OBJ),
+  T1_END("signature", R_SIGNATURE, NO_ARGS, NEED_OBJ),
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in the (encrypted) list of introduction points of
+ * rendezvous service descriptors */
+static token_rule_t ipo_token_table[] = {
+  T1_START("introduction-point", R_IPO_IDENTIFIER, EQ(1), NO_OBJ),
+  T1("ip-address", R_IPO_IP_ADDRESS, EQ(1), NO_OBJ),
+  T1("onion-port", R_IPO_ONION_PORT, EQ(1), NO_OBJ),
+  T1("onion-key", R_IPO_ONION_KEY, NO_ARGS, NEED_KEY_1024),
+  T1("service-key", R_IPO_SERVICE_KEY, NO_ARGS, NEED_KEY_1024),
+  END_OF_TABLE
+};
+
+/** List of tokens recognized in the (possibly encrypted) list of introduction
+ * points of rendezvous service descriptors */
+static token_rule_t client_keys_token_table[] = {
+  T1_START("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ),
+  T1("descriptor-cookie", C_DESCRIPTOR_COOKIE, EQ(1), NO_OBJ),
+  T01("client-key", C_CLIENT_KEY, NO_ARGS, NEED_SKEY_1024),
+  END_OF_TABLE
+};
+
+/** Parse and validate the ASCII-encoded v2 descriptor in <b>desc</b>,
+ * write the parsed descriptor to the newly allocated *<b>parsed_out</b>, the
+ * binary descriptor ID of length DIGEST_LEN to <b>desc_id_out</b>, the
+ * encrypted introduction points to the newly allocated
+ * *<b>intro_points_encrypted_out</b>, their encrypted size to
+ * *<b>intro_points_encrypted_size_out</b>, the size of the encoded descriptor
+ * to *<b>encoded_size_out</b>, and a pointer to the possibly next
+ * descriptor to *<b>next_out</b>; return 0 for success (including validation)
+ * and -1 for failure.
+ *
+ * If <b>as_hsdir</b> is 1, we're parsing this as an HSDir, and we should
+ * be strict about time formats.
+ */
+int
+rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
+                                 char *desc_id_out,
+                                 char **intro_points_encrypted_out,
+                                 size_t *intro_points_encrypted_size_out,
+                                 size_t *encoded_size_out,
+                                 const char **next_out, const char *desc,
+                                 int as_hsdir)
+{
+  rend_service_descriptor_t *result =
+                            tor_malloc_zero(sizeof(rend_service_descriptor_t));
+  char desc_hash[DIGEST_LEN];
+  const char *eos;
+  smartlist_t *tokens = smartlist_new();
+  directory_token_t *tok;
+  char secret_id_part[DIGEST_LEN];
+  int i, version, num_ok=1;
+  smartlist_t *versions;
+  char public_key_hash[DIGEST_LEN];
+  char test_desc_id[DIGEST_LEN];
+  memarea_t *area = NULL;
+  const int strict_time_fmt = as_hsdir;
+
+  tor_assert(desc);
+  /* Check if desc starts correctly. */
+  if (strcmpstart(desc, "rendezvous-service-descriptor ")) {
+    log_info(LD_REND, "Descriptor does not start correctly.");
+    goto err;
+  }
+  /* Compute descriptor hash for later validation. */
+  if (router_get_hash_impl(desc, strlen(desc), desc_hash,
+                           "rendezvous-service-descriptor ",
+                           "\nsignature", '\n', DIGEST_SHA1) < 0) {
+    log_warn(LD_REND, "Couldn't compute descriptor hash.");
+    goto err;
+  }
+  /* Determine end of string. */
+  eos = strstr(desc, "\nrendezvous-service-descriptor ");
+  if (!eos)
+    eos = desc + strlen(desc);
+  else
+    eos = eos + 1;
+  /* Check length. */
+  if (eos-desc > REND_DESC_MAX_SIZE) {
+    /* XXXX+ If we are parsing this descriptor as a server, this
+     * should be a protocol warning. */
+    log_warn(LD_REND, "Descriptor length is %d which exceeds "
+             "maximum rendezvous descriptor size of %d bytes.",
+             (int)(eos-desc), REND_DESC_MAX_SIZE);
+    goto err;
+  }
+  /* Tokenize descriptor. */
+  area = memarea_new();
+  if (tokenize_string(area, desc, eos, tokens, desc_token_table, 0)) {
+    log_warn(LD_REND, "Error tokenizing descriptor.");
+    goto err;
+  }
+  /* Set next to next descriptor, if available. */
+  *next_out = eos;
+  /* Set length of encoded descriptor. */
+  *encoded_size_out = eos - desc;
+  /* Check min allowed length of token list. */
+  if (smartlist_len(tokens) < 7) {
+    log_warn(LD_REND, "Impossibly short descriptor.");
+    goto err;
+  }
+  /* Parse base32-encoded descriptor ID. */
+  tok = find_by_keyword(tokens, R_RENDEZVOUS_SERVICE_DESCRIPTOR);
+  tor_assert(tok == smartlist_get(tokens, 0));
+  tor_assert(tok->n_args == 1);
+  if (!rend_valid_descriptor_id(tok->args[0])) {
+    log_warn(LD_REND, "Invalid descriptor ID: '%s'", tok->args[0]);
+    goto err;
+  }
+  if (base32_decode(desc_id_out, DIGEST_LEN,
+                    tok->args[0], REND_DESC_ID_V2_LEN_BASE32) < 0) {
+    log_warn(LD_REND, "Descriptor ID contains illegal characters: %s",
+             tok->args[0]);
+    goto err;
+  }
+  /* Parse descriptor version. */
+  tok = find_by_keyword(tokens, R_VERSION);
+  tor_assert(tok->n_args == 1);
+  result->version =
+    (int) tor_parse_long(tok->args[0], 10, 0, INT_MAX, &num_ok, NULL);
+  if (result->version != 2 || !num_ok) {
+    /* If it's <2, it shouldn't be under this format.  If the number
+     * is greater than 2, we bumped it because we broke backward
+     * compatibility.  See how version numbers in our other formats
+     * work. */
+    log_warn(LD_REND, "Unrecognized descriptor version: %s",
+             escaped(tok->args[0]));
+    goto err;
+  }
+  /* Parse public key. */
+  tok = find_by_keyword(tokens, R_PERMANENT_KEY);
+  result->pk = tok->key;
+  tok->key = NULL; /* Prevent free */
+  /* Parse secret ID part. */
+  tok = find_by_keyword(tokens, R_SECRET_ID_PART);
+  tor_assert(tok->n_args == 1);
+  if (strlen(tok->args[0]) != REND_SECRET_ID_PART_LEN_BASE32 ||
+      strspn(tok->args[0], BASE32_CHARS) != REND_SECRET_ID_PART_LEN_BASE32) {
+    log_warn(LD_REND, "Invalid secret ID part: '%s'", tok->args[0]);
+    goto err;
+  }
+  if (base32_decode(secret_id_part, DIGEST_LEN, tok->args[0], 32) < 0) {
+    log_warn(LD_REND, "Secret ID part contains illegal characters: %s",
+             tok->args[0]);
+    goto err;
+  }
+  /* Parse publication time -- up-to-date check is done when storing the
+   * descriptor. */
+  tok = find_by_keyword(tokens, R_PUBLICATION_TIME);
+  tor_assert(tok->n_args == 1);
+  if (parse_iso_time_(tok->args[0], &result->timestamp,
+                      strict_time_fmt, 0) < 0) {
+    log_warn(LD_REND, "Invalid publication time: '%s'", tok->args[0]);
+    goto err;
+  }
+  /* Parse protocol versions. */
+  tok = find_by_keyword(tokens, R_PROTOCOL_VERSIONS);
+  tor_assert(tok->n_args == 1);
+  versions = smartlist_new();
+  smartlist_split_string(versions, tok->args[0], ",",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  for (i = 0; i < smartlist_len(versions); i++) {
+    version = (int) tor_parse_long(smartlist_get(versions, i),
+                                   10, 0, INT_MAX, &num_ok, NULL);
+    if (!num_ok) /* It's a string; let's ignore it. */
+      continue;
+    if (version >= REND_PROTOCOL_VERSION_BITMASK_WIDTH)
+      /* Avoid undefined left-shift behaviour. */
+      continue;
+    result->protocols |= 1 << version;
+  }
+  SMARTLIST_FOREACH(versions, char *, cp, tor_free(cp));
+  smartlist_free(versions);
+  /* Parse encrypted introduction points. Don't verify. */
+  tok = find_opt_by_keyword(tokens, R_INTRODUCTION_POINTS);
+  if (tok) {
+    if (strcmp(tok->object_type, "MESSAGE")) {
+      log_warn(LD_DIR, "Bad object type: introduction points should be of "
+               "type MESSAGE");
+      goto err;
+    }
+    *intro_points_encrypted_out = tor_memdup(tok->object_body,
+                                             tok->object_size);
+    *intro_points_encrypted_size_out = tok->object_size;
+  } else {
+    *intro_points_encrypted_out = NULL;
+    *intro_points_encrypted_size_out = 0;
+  }
+  /* Parse and verify signature. */
+  tok = find_by_keyword(tokens, R_SIGNATURE);
+  if (check_signature_token(desc_hash, DIGEST_LEN, tok, result->pk, 0,
+                            "v2 rendezvous service descriptor") < 0)
+    goto err;
+  /* Verify that descriptor ID belongs to public key and secret ID part. */
+  if (crypto_pk_get_digest(result->pk, public_key_hash) < 0) {
+    log_warn(LD_REND, "Unable to compute rend descriptor public key digest");
+    goto err;
+  }
+  rend_get_descriptor_id_bytes(test_desc_id, public_key_hash,
+                               secret_id_part);
+  if (tor_memneq(desc_id_out, test_desc_id, DIGEST_LEN)) {
+    log_warn(LD_REND, "Parsed descriptor ID does not match "
+             "computed descriptor ID.");
+    goto err;
+  }
+  goto done;
+ err:
+  rend_service_descriptor_free(result);
+  result = NULL;
+ done:
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  if (area)
+    memarea_drop_all(area);
+  *parsed_out = result;
+  if (result)
+    return 0;
+  return -1;
+}
+
+/** Decrypt the encrypted introduction points in <b>ipos_encrypted</b> of
+ * length <b>ipos_encrypted_size</b> using <b>descriptor_cookie</b> and
+ * write the result to a newly allocated string that is pointed to by
+ * <b>ipos_decrypted</b> and its length to <b>ipos_decrypted_size</b>.
+ * Return 0 if decryption was successful and -1 otherwise. */
+int
+rend_decrypt_introduction_points(char **ipos_decrypted,
+                                 size_t *ipos_decrypted_size,
+                                 const char *descriptor_cookie,
+                                 const char *ipos_encrypted,
+                                 size_t ipos_encrypted_size)
+{
+  tor_assert(ipos_encrypted);
+  tor_assert(descriptor_cookie);
+  if (ipos_encrypted_size < 2) {
+    log_warn(LD_REND, "Size of encrypted introduction points is too "
+                      "small.");
+    return -1;
+  }
+  if (ipos_encrypted[0] == (int)REND_BASIC_AUTH) {
+    char iv[CIPHER_IV_LEN], client_id[REND_BASIC_AUTH_CLIENT_ID_LEN],
+         session_key[CIPHER_KEY_LEN], *dec;
+    int declen, client_blocks;
+    size_t pos = 0, len, client_entries_len;
+    crypto_digest_t *digest;
+    crypto_cipher_t *cipher;
+    client_blocks = (int) ipos_encrypted[1];
+    client_entries_len = client_blocks * REND_BASIC_AUTH_CLIENT_MULTIPLE *
+                         REND_BASIC_AUTH_CLIENT_ENTRY_LEN;
+    if (ipos_encrypted_size < 2 + client_entries_len + CIPHER_IV_LEN + 1) {
+      log_warn(LD_REND, "Size of encrypted introduction points is too "
+                        "small.");
+      return -1;
+    }
+    memcpy(iv, ipos_encrypted + 2 + client_entries_len, CIPHER_IV_LEN);
+    digest = crypto_digest_new();
+    crypto_digest_add_bytes(digest, descriptor_cookie, REND_DESC_COOKIE_LEN);
+    crypto_digest_add_bytes(digest, iv, CIPHER_IV_LEN);
+    crypto_digest_get_digest(digest, client_id,
+                             REND_BASIC_AUTH_CLIENT_ID_LEN);
+    crypto_digest_free(digest);
+    for (pos = 2; pos < 2 + client_entries_len;
+         pos += REND_BASIC_AUTH_CLIENT_ENTRY_LEN) {
+      if (tor_memeq(ipos_encrypted + pos, client_id,
+                  REND_BASIC_AUTH_CLIENT_ID_LEN)) {
+        /* Attempt to decrypt introduction points. */
+        cipher = crypto_cipher_new(descriptor_cookie);
+        if (crypto_cipher_decrypt(cipher, session_key, ipos_encrypted
+                                  + pos + REND_BASIC_AUTH_CLIENT_ID_LEN,
+                                  CIPHER_KEY_LEN) < 0) {
+          log_warn(LD_REND, "Could not decrypt session key for client.");
+          crypto_cipher_free(cipher);
+          return -1;
+        }
+        crypto_cipher_free(cipher);
+
+        len = ipos_encrypted_size - 2 - client_entries_len - CIPHER_IV_LEN;
+        dec = tor_malloc_zero(len + 1);
+        declen = crypto_cipher_decrypt_with_iv(session_key, dec, len,
+            ipos_encrypted + 2 + client_entries_len,
+            ipos_encrypted_size - 2 - client_entries_len);
+
+        if (declen < 0) {
+          log_warn(LD_REND, "Could not decrypt introduction point string.");
+          tor_free(dec);
+          return -1;
+        }
+        if (fast_memcmpstart(dec, declen, "introduction-point ")) {
+          log_warn(LD_REND, "Decrypted introduction points don't "
+                            "look like we could parse them.");
+          tor_free(dec);
+          continue;
+        }
+        *ipos_decrypted = dec;
+        *ipos_decrypted_size = declen;
+        return 0;
+      }
+    }
+    log_warn(LD_REND, "Could not decrypt introduction points. Please "
+             "check your authorization for this service!");
+    return -1;
+  } else if (ipos_encrypted[0] == (int)REND_STEALTH_AUTH) {
+    char *dec;
+    int declen;
+    if (ipos_encrypted_size < CIPHER_IV_LEN + 2) {
+      log_warn(LD_REND, "Size of encrypted introduction points is too "
+                        "small.");
+      return -1;
+    }
+    dec = tor_malloc_zero(ipos_encrypted_size - CIPHER_IV_LEN - 1 + 1);
+
+    declen = crypto_cipher_decrypt_with_iv(descriptor_cookie, dec,
+                                           ipos_encrypted_size -
+                                               CIPHER_IV_LEN - 1,
+                                           ipos_encrypted + 1,
+                                           ipos_encrypted_size - 1);
+
+    if (declen < 0) {
+      log_warn(LD_REND, "Decrypting introduction points failed!");
+      tor_free(dec);
+      return -1;
+    }
+    *ipos_decrypted = dec;
+    *ipos_decrypted_size = declen;
+    return 0;
+  } else {
+    log_warn(LD_REND, "Unknown authorization type number: %d",
+             ipos_encrypted[0]);
+    return -1;
+  }
+}
+
+/** Parse the encoded introduction points in <b>intro_points_encoded</b> of
+ * length <b>intro_points_encoded_size</b> and write the result to the
+ * descriptor in <b>parsed</b>; return the number of successfully parsed
+ * introduction points or -1 in case of a failure. */
+int
+rend_parse_introduction_points(rend_service_descriptor_t *parsed,
+                               const char *intro_points_encoded,
+                               size_t intro_points_encoded_size)
+{
+  const char *current_ipo, *end_of_intro_points;
+  smartlist_t *tokens = NULL;
+  directory_token_t *tok;
+  rend_intro_point_t *intro;
+  extend_info_t *info;
+  int result, num_ok=1;
+  memarea_t *area = NULL;
+  tor_assert(parsed);
+  /** Function may only be invoked once. */
+  tor_assert(!parsed->intro_nodes);
+  if (!intro_points_encoded || intro_points_encoded_size == 0) {
+    log_warn(LD_REND, "Empty or zero size introduction point list");
+    goto err;
+  }
+  /* Consider one intro point after the other. */
+  current_ipo = intro_points_encoded;
+  end_of_intro_points = intro_points_encoded + intro_points_encoded_size;
+  tokens = smartlist_new();
+  parsed->intro_nodes = smartlist_new();
+  area = memarea_new();
+
+  while (!fast_memcmpstart(current_ipo, end_of_intro_points-current_ipo,
+                      "introduction-point ")) {
+    /* Determine end of string. */
+    const char *eos = tor_memstr(current_ipo, end_of_intro_points-current_ipo,
+                                 "\nintroduction-point ");
+    if (!eos)
+      eos = end_of_intro_points;
+    else
+      eos = eos+1;
+    tor_assert(eos <= intro_points_encoded+intro_points_encoded_size);
+    /* Free tokens and clear token list. */
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_clear(tokens);
+    memarea_clear(area);
+    /* Tokenize string. */
+    if (tokenize_string(area, current_ipo, eos, tokens, ipo_token_table, 0)) {
+      log_warn(LD_REND, "Error tokenizing introduction point");
+      goto err;
+    }
+    /* Advance to next introduction point, if available. */
+    current_ipo = eos;
+    /* Check minimum allowed length of introduction point. */
+    if (smartlist_len(tokens) < 5) {
+      log_warn(LD_REND, "Impossibly short introduction point.");
+      goto err;
+    }
+    /* Allocate new intro point and extend info. */
+    intro = tor_malloc_zero(sizeof(rend_intro_point_t));
+    info = intro->extend_info = tor_malloc_zero(sizeof(extend_info_t));
+    /* Parse identifier. */
+    tok = find_by_keyword(tokens, R_IPO_IDENTIFIER);
+    if (base32_decode(info->identity_digest, DIGEST_LEN,
+                      tok->args[0], REND_INTRO_POINT_ID_LEN_BASE32) < 0) {
+      log_warn(LD_REND, "Identity digest contains illegal characters: %s",
+               tok->args[0]);
+      rend_intro_point_free(intro);
+      goto err;
+    }
+    /* Write identifier to nickname. */
+    info->nickname[0] = '$';
+    base16_encode(info->nickname + 1, sizeof(info->nickname) - 1,
+                  info->identity_digest, DIGEST_LEN);
+    /* Parse IP address. */
+    tok = find_by_keyword(tokens, R_IPO_IP_ADDRESS);
+    if (tor_addr_parse(&info->addr, tok->args[0])<0) {
+      log_warn(LD_REND, "Could not parse introduction point address.");
+      rend_intro_point_free(intro);
+      goto err;
+    }
+    if (tor_addr_family(&info->addr) != AF_INET) {
+      log_warn(LD_REND, "Introduction point address was not ipv4.");
+      rend_intro_point_free(intro);
+      goto err;
+    }
+
+    /* Parse onion port. */
+    tok = find_by_keyword(tokens, R_IPO_ONION_PORT);
+    info->port = (uint16_t) tor_parse_long(tok->args[0],10,1,65535,
+                                           &num_ok,NULL);
+    if (!info->port || !num_ok) {
+      log_warn(LD_REND, "Introduction point onion port %s is invalid",
+               escaped(tok->args[0]));
+      rend_intro_point_free(intro);
+      goto err;
+    }
+    /* Parse onion key. */
+    tok = find_by_keyword(tokens, R_IPO_ONION_KEY);
+    if (!crypto_pk_public_exponent_ok(tok->key)) {
+      log_warn(LD_REND,
+               "Introduction point's onion key had invalid exponent.");
+      rend_intro_point_free(intro);
+      goto err;
+    }
+    info->onion_key = tok->key;
+    tok->key = NULL; /* Prevent free */
+    /* Parse service key. */
+    tok = find_by_keyword(tokens, R_IPO_SERVICE_KEY);
+    if (!crypto_pk_public_exponent_ok(tok->key)) {
+      log_warn(LD_REND,
+               "Introduction point key had invalid exponent.");
+      rend_intro_point_free(intro);
+      goto err;
+    }
+    intro->intro_key = tok->key;
+    tok->key = NULL; /* Prevent free */
+    /* Add extend info to list of introduction points. */
+    smartlist_add(parsed->intro_nodes, intro);
+  }
+  result = smartlist_len(parsed->intro_nodes);
+  goto done;
+
+ err:
+  result = -1;
+
+ done:
+  /* Free tokens and clear token list. */
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  if (area)
+    memarea_drop_all(area);
+
+  return result;
+}
+
+/** Parse the content of a client_key file in <b>ckstr</b> and add
+ * rend_authorized_client_t's for each parsed client to
+ * <b>parsed_clients</b>. Return the number of parsed clients as result
+ * or -1 for failure. */
+int
+rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr)
+{
+  int result = -1;
+  smartlist_t *tokens;
+  directory_token_t *tok;
+  const char *current_entry = NULL;
+  memarea_t *area = NULL;
+  char *err_msg = NULL;
+  if (!ckstr || strlen(ckstr) == 0)
+    return -1;
+  tokens = smartlist_new();
+  /* Begin parsing with first entry, skipping comments or whitespace at the
+   * beginning. */
+  area = memarea_new();
+  current_entry = eat_whitespace(ckstr);
+  while (!strcmpstart(current_entry, "client-name ")) {
+    rend_authorized_client_t *parsed_entry;
+    /* Determine end of string. */
+    const char *eos = strstr(current_entry, "\nclient-name ");
+    if (!eos)
+      eos = current_entry + strlen(current_entry);
+    else
+      eos = eos + 1;
+    /* Free tokens and clear token list. */
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_clear(tokens);
+    memarea_clear(area);
+    /* Tokenize string. */
+    if (tokenize_string(area, current_entry, eos, tokens,
+                        client_keys_token_table, 0)) {
+      log_warn(LD_REND, "Error tokenizing client keys file.");
+      goto err;
+    }
+    /* Advance to next entry, if available. */
+    current_entry = eos;
+    /* Check minimum allowed length of token list. */
+    if (smartlist_len(tokens) < 2) {
+      log_warn(LD_REND, "Impossibly short client key entry.");
+      goto err;
+    }
+    /* Parse client name. */
+    tok = find_by_keyword(tokens, C_CLIENT_NAME);
+    tor_assert(tok == smartlist_get(tokens, 0));
+    tor_assert(tok->n_args == 1);
+
+    if (!rend_valid_client_name(tok->args[0])) {
+      log_warn(LD_CONFIG, "Illegal client name: %s. (Length must be "
+               "between 1 and %d, and valid characters are "
+               "[A-Za-z0-9+-_].)", tok->args[0], REND_CLIENTNAME_MAX_LEN);
+      goto err;
+    }
+    /* Check if client name is duplicate. */
+    if (strmap_get(parsed_clients, tok->args[0])) {
+      log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains a "
+               "duplicate client name: '%s'. Ignoring.", tok->args[0]);
+      goto err;
+    }
+    parsed_entry = tor_malloc_zero(sizeof(rend_authorized_client_t));
+    parsed_entry->client_name = tor_strdup(tok->args[0]);
+    strmap_set(parsed_clients, parsed_entry->client_name, parsed_entry);
+    /* Parse client key. */
+    tok = find_opt_by_keyword(tokens, C_CLIENT_KEY);
+    if (tok) {
+      parsed_entry->client_key = tok->key;
+      tok->key = NULL; /* Prevent free */
+    }
+
+    /* Parse descriptor cookie. */
+    tok = find_by_keyword(tokens, C_DESCRIPTOR_COOKIE);
+    tor_assert(tok->n_args == 1);
+    if (rend_auth_decode_cookie(tok->args[0], parsed_entry->descriptor_cookie,
+                                NULL, &err_msg) < 0) {
+      tor_assert(err_msg);
+      log_warn(LD_REND, "%s", err_msg);
+      tor_free(err_msg);
+      goto err;
+    }
+  }
+  result = strmap_size(parsed_clients);
+  goto done;
+ err:
+  result = -1;
+ done:
+  /* Free tokens and clear token list. */
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_free(tokens);
+  if (area)
+    memarea_drop_all(area);
+  return result;
+}
diff --git a/src/feature/rend/rendparse.h b/src/feature/rend/rendparse.h
new file mode 100644
index 0000000000..0cef931e90
--- /dev/null
+++ b/src/feature/rend/rendparse.h
@@ -0,0 +1,32 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rend_parse.h
+ * \brief Header file for rend_parse.c.
+ **/
+
+#ifndef TOR_REND_PARSE_H
+#define TOR_REND_PARSE_H
+
+int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
+                                     char *desc_id_out,
+                                     char **intro_points_encrypted_out,
+                                     size_t *intro_points_encrypted_size_out,
+                                     size_t *encoded_size_out,
+                                     const char **next_out, const char *desc,
+                                     int as_hsdir);
+int rend_decrypt_introduction_points(char **ipos_decrypted,
+                                     size_t *ipos_decrypted_size,
+                                     const char *descriptor_cookie,
+                                     const char *ipos_encrypted,
+                                     size_t ipos_encrypted_size);
+int rend_parse_introduction_points(rend_service_descriptor_t *parsed,
+                                   const char *intro_points_encoded,
+                                   size_t intro_points_encoded_size);
+int rend_parse_client_keys(strmap_t *parsed_clients, const char *str);
+
+#endif
diff --git a/src/or/rendservice.c b/src/feature/rend/rendservice.c
index 32b856452d..c96ecec308 100644
--- a/src/or/rendservice.c
+++ b/src/feature/rend/rendservice.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,28 +9,62 @@
 
 #define RENDSERVICE_PRIVATE
 
-#include "or.h"
-#include "circpathbias.h"
-#include "circuitbuild.h"
-#include "circuitlist.h"
-#include "circuituse.h"
-#include "config.h"
-#include "control.h"
-#include "directory.h"
-#include "main.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "policies.h"
-#include "rendclient.h"
-#include "rendcommon.h"
-#include "rendservice.h"
-#include "router.h"
-#include "relay.h"
-#include "rephist.h"
-#include "replaycache.h"
-#include "routerlist.h"
-#include "routerparse.h"
-#include "routerset.h"
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/policies.h"
+#include "core/or/relay.h"
+#include "feature/client/circpathbias.h"
+#include "feature/control/control.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dircommon/directory.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_config.h"
+#include "feature/hs_common/replaycache.h"
+#include "feature/keymgt/loadkey.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nickname.h"
+#include "feature/nodelist/node_select.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/rend/rendclient.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/rend/rendparse.h"
+#include "feature/rend/rendservice.h"
+#include "feature/stats/predict_ports.h"
+#include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/encoding/confline.h"
+#include "lib/net/resolve.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/crypt_path_st.h"
+#include "core/or/crypt_path_reference_st.h"
+#include "core/or/edge_connection_st.h"
+#include "core/or/extend_info_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "feature/rend/rend_authorized_client_st.h"
+#include "feature/rend/rend_encoded_v2_service_descriptor_st.h"
+#include "feature/rend/rend_intro_point_st.h"
+#include "feature/rend/rend_service_descriptor_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
 
 struct rend_service_t;
 static origin_circuit_t *find_intro_circuit(rend_intro_point_t *intro,
@@ -75,58 +109,11 @@ static ssize_t rend_service_parse_intro_for_v3(
 static int rend_service_check_private_dir(const or_options_t *options,
                                           const rend_service_t *s,
                                           int create);
-static int rend_service_check_private_dir_impl(const or_options_t *options,
-                                               const rend_service_t *s,
-                                               int create);
-
-/** Represents the mapping from a virtual port of a rendezvous service to
- * a real port on some IP.
- */
-struct rend_service_port_config_s {
-  /* The incoming HS virtual port we're mapping */
-  uint16_t virtual_port;
-  /* Is this an AF_UNIX port? */
-  unsigned int is_unix_addr:1;
-  /* The outgoing TCP port to use, if !is_unix_addr */
-  uint16_t real_port;
-  /* The outgoing IPv4 or IPv6 address to use, if !is_unix_addr */
-  tor_addr_t real_addr;
-  /* The socket path to connect to, if is_unix_addr */
-  char unix_addr[FLEXIBLE_ARRAY_MEMBER];
-};
-
-/** Try to maintain this many intro points per service by default. */
-#define NUM_INTRO_POINTS_DEFAULT 3
-/** Maximum number of intro points per service. */
-#define NUM_INTRO_POINTS_MAX 10
-/** Number of extra intro points we launch if our set of intro nodes is
- * empty. See proposal 155, section 4. */
-#define NUM_INTRO_POINTS_EXTRA 2
-
-/** If we can't build our intro circuits, don't retry for this long. */
-#define INTRO_CIRC_RETRY_PERIOD (60*5)
-/** Don't try to build more than this many circuits before giving up
- * for a while.*/
-#define MAX_INTRO_CIRCS_PER_PERIOD 10
-/** How many seconds should we spend trying to connect to a requested
- * rendezvous point before giving up? */
-#define MAX_REND_TIMEOUT 30
-/* Default, minimum and maximum values for the maximum rendezvous failures
- * consensus parameter. */
-#define MAX_REND_FAILURES_DEFAULT 2
-#define MAX_REND_FAILURES_MIN 1
-#define MAX_REND_FAILURES_MAX 10
-
-/** How many times will a hidden service operator attempt to connect to
- * a requested rendezvous point before giving up? */
-static int
-get_max_rend_failures(void)
-{
-  return networkstatus_get_param(NULL, "hs_service_max_rdv_failures",
-                                 MAX_REND_FAILURES_DEFAULT,
-                                 MAX_REND_FAILURES_MIN,
-                                 MAX_REND_FAILURES_MAX);
-}
+static const smartlist_t* rend_get_service_list(
+                                  const smartlist_t* substitute_service_list);
+static smartlist_t* rend_get_service_list_mutable(
+                                  smartlist_t* substitute_service_list);
+static int rend_max_intro_circs_per_period(unsigned int n_intro_points_wanted);
 
 /* Hidden service directory file names:
  * new file names should be added to rend_service_add_filenames_to_list()
@@ -136,21 +123,64 @@ static const char *hostname_fname = "hostname";
 static const char *client_keys_fname = "client_keys";
 static const char *sos_poison_fname = "onion_service_non_anonymous";
 
+/** A list of rend_service_t's for services run on this OP. */
+static smartlist_t *rend_service_list = NULL;
+/** A list of rend_service_t's for services run on this OP which is used as a
+ * staging area before they are put in the main list in order to prune dying
+ * service on config reload. */
+static smartlist_t *rend_service_staging_list = NULL;
+
+/* Like rend_get_service_list_mutable, but returns a read-only list. */
+static const smartlist_t*
+rend_get_service_list(const smartlist_t* substitute_service_list)
+{
+  /* It is safe to cast away the const here, because
+   * rend_get_service_list_mutable does not actually modify the list */
+  return rend_get_service_list_mutable((smartlist_t*)substitute_service_list);
+}
+
+/* Return a mutable list of hidden services.
+ * If substitute_service_list is not NULL, return it.
+ * Otherwise, check if the global rend_service_list is non-NULL, and if so,
+ * return it.
+ * Otherwise, log a BUG message and return NULL.
+ * */
+static smartlist_t*
+rend_get_service_list_mutable(smartlist_t* substitute_service_list)
+{
+  if (substitute_service_list) {
+    return substitute_service_list;
+  }
+
+  /* If no special service list is provided, then just use the global one. */
+
+  if (BUG(!rend_service_list)) {
+    /* No global HS list, which is a programmer error. */
+    return NULL;
+  }
+
+  return rend_service_list;
+}
+
+/** Tells if onion service <b>s</b> is ephemeral.
+ */
+static unsigned int
+rend_service_is_ephemeral(const struct rend_service_t *s)
+{
+  return (s->directory == NULL);
+}
+
 /** Returns a escaped string representation of the service, <b>s</b>.
  */
 static const char *
 rend_service_escaped_dir(const struct rend_service_t *s)
 {
-  return (s->directory) ? escaped(s->directory) : "[EPHEMERAL]";
+  return rend_service_is_ephemeral(s) ? "[EPHEMERAL]" : escaped(s->directory);
 }
 
-/** A list of rend_service_t's for services run on this OP.
- */
-static smartlist_t *rend_service_list = NULL;
-
 /** Return the number of rendezvous services we have configured. */
 int
-num_rend_services(void)
+rend_num_services(void)
 {
   if (!rend_service_list)
     return 0;
@@ -159,7 +189,7 @@ num_rend_services(void)
 
 /** Helper: free storage held by a single service authorized client entry. */
 void
-rend_authorized_client_free(rend_authorized_client_t *client)
+rend_authorized_client_free_(rend_authorized_client_t *client)
 {
   if (!client)
     return;
@@ -174,15 +204,15 @@ rend_authorized_client_free(rend_authorized_client_t *client)
 
 /** Helper for strmap_free. */
 static void
-rend_authorized_client_strmap_item_free(void *authorized_client)
+rend_authorized_client_free_void(void *authorized_client)
 {
-  rend_authorized_client_free(authorized_client);
+  rend_authorized_client_free_(authorized_client);
 }
 
 /** Release the storage held by <b>service</b>.
  */
 STATIC void
-rend_service_free(rend_service_t *service)
+rend_service_free_(rend_service_t *service)
 {
   if (!service)
     return;
@@ -218,137 +248,146 @@ rend_service_free(rend_service_t *service)
   tor_free(service);
 }
 
-/** Release all the storage held in rend_service_list.
- */
+/* Release all the storage held in rend_service_staging_list. */
 void
-rend_service_free_all(void)
+rend_service_free_staging_list(void)
 {
-  if (!rend_service_list)
-    return;
-
-  SMARTLIST_FOREACH(rend_service_list, rend_service_t*, ptr,
-                    rend_service_free(ptr));
-  smartlist_free(rend_service_list);
-  rend_service_list = NULL;
+  if (rend_service_staging_list) {
+    SMARTLIST_FOREACH(rend_service_staging_list, rend_service_t*, ptr,
+                      rend_service_free(ptr));
+    smartlist_free(rend_service_staging_list);
+    rend_service_staging_list = NULL;
+  }
 }
 
-/** Validate <b>service</b> and add it to <b>service_list</b>, or to
- * the global rend_service_list if <b>service_list</b> is NULL.
- * Return 0 on success.  On failure, free <b>service</b> and return -1.
- * Takes ownership of <b>service</b>.
- */
-static int
-rend_add_service(smartlist_t *service_list, rend_service_t *service)
+/** Release all the storage held in both rend_service_list and
+ * rend_service_staging_list. */
+void
+rend_service_free_all(void)
 {
-  int i;
-  rend_service_port_config_t *p;
+  if (rend_service_list) {
+    SMARTLIST_FOREACH(rend_service_list, rend_service_t*, ptr,
+                      rend_service_free(ptr));
+    smartlist_free(rend_service_list);
+    rend_service_list = NULL;
+  }
+  rend_service_free_staging_list();
+}
 
-  smartlist_t *s_list;
-  /* If no special service list is provided, then just use the global one. */
-  if (!service_list) {
-    if (BUG(!rend_service_list)) {
-      /* No global HS list, which is a failure. */
-      return -1;
-    }
+/* Initialize the subsystem. */
+void
+rend_service_init(void)
+{
+  tor_assert(!rend_service_list);
+  tor_assert(!rend_service_staging_list);
 
-    s_list = rend_service_list;
-  } else {
-    s_list = service_list;
-  }
+  rend_service_list = smartlist_new();
+  rend_service_staging_list = smartlist_new();
+}
 
-  service->intro_nodes = smartlist_new();
-  service->expiring_nodes = smartlist_new();
+/* Validate a <b>service</b>. Use the <b>service_list</b> to make sure there
+ * is no duplicate entry for the given service object. Return 0 if valid else
+ * -1 if not.*/
+static int
+rend_validate_service(const smartlist_t *service_list,
+                      const rend_service_t *service)
+{
+  tor_assert(service_list);
+  tor_assert(service);
 
   if (service->max_streams_per_circuit < 0) {
     log_warn(LD_CONFIG, "Hidden service (%s) configured with negative max "
-                        "streams per circuit; ignoring.",
+                        "streams per circuit.",
              rend_service_escaped_dir(service));
-    rend_service_free(service);
-    return -1;
+    goto invalid;
   }
 
   if (service->max_streams_close_circuit < 0 ||
       service->max_streams_close_circuit > 1) {
     log_warn(LD_CONFIG, "Hidden service (%s) configured with invalid "
-                        "max streams handling; ignoring.",
+                        "max streams handling.",
              rend_service_escaped_dir(service));
-    rend_service_free(service);
-    return -1;
+    goto invalid;
   }
 
   if (service->auth_type != REND_NO_AUTH &&
-      (!service->clients ||
-       smartlist_len(service->clients) == 0)) {
-    log_warn(LD_CONFIG, "Hidden service (%s) with client authorization but no "
-                        "clients; ignoring.",
+      (!service->clients || smartlist_len(service->clients) == 0)) {
+    log_warn(LD_CONFIG, "Hidden service (%s) with client authorization but "
+                        "no clients.",
              rend_service_escaped_dir(service));
-    rend_service_free(service);
-    return -1;
+    goto invalid;
   }
 
   if (!service->ports || !smartlist_len(service->ports)) {
-    log_warn(LD_CONFIG, "Hidden service (%s) with no ports configured; "
-             "ignoring.",
+    log_warn(LD_CONFIG, "Hidden service (%s) with no ports configured.",
              rend_service_escaped_dir(service));
+    goto invalid;
+  }
+
+  /* Valid. */
+  return 0;
+ invalid:
+  return -1;
+}
+
+/** Add it to <b>service_list</b>, or to the global rend_service_list if
+ * <b>service_list</b> is NULL.  Return 0 on success.  On failure, free
+ * <b>service</b> and return -1. Takes ownership of <b>service</b>. */
+static int
+rend_add_service(smartlist_t *service_list, rend_service_t *service)
+{
+  int i;
+  rend_service_port_config_t *p;
+
+  tor_assert(service);
+
+  smartlist_t *s_list = rend_get_service_list_mutable(service_list);
+  /* We must have a service list, even if it's a temporary one, so we can
+   * check for duplicate services */
+  if (BUG(!s_list)) {
     rend_service_free(service);
     return -1;
-  } else {
-    int dupe = 0;
-    /* XXX This duplicate check has two problems:
-     *
-     * a) It's O(n^2), but the same comment from the bottom of
-     *    rend_config_services() should apply.
-     *
-     * b) We only compare directory paths as strings, so we can't
-     *    detect two distinct paths that specify the same directory
-     *    (which can arise from symlinks, case-insensitivity, bind
-     *    mounts, etc.).
-     *
-     * It also can't detect that two separate Tor instances are trying
-     * to use the same HiddenServiceDir; for that, we would need a
-     * lock file.  But this is enough to detect a simple mistake that
-     * at least one person has actually made.
-     */
-    if (service->directory != NULL) {
-      /* Skip dupe for ephemeral services. */
-      SMARTLIST_FOREACH(s_list, rend_service_t*, ptr,
-                        dupe = dupe ||
-                               !strcmp(ptr->directory, service->directory));
-      if (dupe) {
-        log_warn(LD_REND, "Another hidden service is already configured for "
-                 "directory %s, ignoring.",
-                 rend_service_escaped_dir(service));
-        rend_service_free(service);
-        return -1;
-      }
-    }
-    smartlist_add(s_list, service);
-    log_debug(LD_REND,"Configuring service with directory \"%s\"",
-              service->directory);
-    for (i = 0; i < smartlist_len(service->ports); ++i) {
-      p = smartlist_get(service->ports, i);
-      if (!(p->is_unix_addr)) {
-        log_debug(LD_REND,
-                  "Service maps port %d to %s",
-                  p->virtual_port,
-                  fmt_addrport(&p->real_addr, p->real_port));
-      } else {
+  }
+
+  service->intro_nodes = smartlist_new();
+  service->expiring_nodes = smartlist_new();
+
+  log_debug(LD_REND,"Configuring service with directory %s",
+            rend_service_escaped_dir(service));
+  for (i = 0; i < smartlist_len(service->ports); ++i) {
+    p = smartlist_get(service->ports, i);
+    if (!(p->is_unix_addr)) {
+      log_debug(LD_REND,
+                "Service maps port %d to %s",
+                p->virtual_port,
+                fmt_addrport(&p->real_addr, p->real_port));
+    } else {
 #ifdef HAVE_SYS_UN_H
-        log_debug(LD_REND,
-                  "Service maps port %d to socket at \"%s\"",
-                  p->virtual_port, p->unix_addr);
+      log_debug(LD_REND,
+                "Service maps port %d to socket at \"%s\"",
+                p->virtual_port, p->unix_addr);
 #else
-        log_debug(LD_REND,
-                  "Service maps port %d to an AF_UNIX socket, but we "
-                  "have no AF_UNIX support on this platform.  This is "
-                  "probably a bug.",
-                  p->virtual_port);
+      log_warn(LD_BUG,
+               "Service maps port %d to an AF_UNIX socket, but we "
+               "have no AF_UNIX support on this platform.  This is "
+               "probably a bug.",
+               p->virtual_port);
+      rend_service_free(service);
+      return -1;
 #endif /* defined(HAVE_SYS_UN_H) */
-      }
     }
-    return 0;
   }
-  /* NOTREACHED */
+  /* The service passed all the checks */
+  tor_assert(s_list);
+  smartlist_add(s_list, service);
+
+  /* Notify that our global service list has changed only if this new service
+   * went into our global list. If not, when we move service from the staging
+   * list to the new list, a notify is triggered. */
+  if (s_list == rend_service_list) {
+    hs_service_map_has_changed();
+  }
+  return 0;
 }
 
 /** Return a new rend_service_port_config_t with its path set to
@@ -367,9 +406,9 @@ rend_service_port_config_new(const char *socket_path)
   return conf;
 }
 
-/** Parses a real-port to virtual-port mapping separated by the provided
- * separator and returns a new rend_service_port_config_t, or NULL and an
- * optional error string on failure.
+/** Parses a virtual-port to real-port/socket mapping separated by
+ * the provided separator and returns a new rend_service_port_config_t,
+ * or NULL and an optional error string on failure.
  *
  * The format is: VirtualPort SEP (IP|RealPort|IP:RealPort|'socket':path)?
  *
@@ -394,14 +433,12 @@ rend_service_parse_port_config(const char *string, const char *sep,
   smartlist_split_string(sl, string, sep,
                          SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2);
   if (smartlist_len(sl) < 1 || BUG(smartlist_len(sl) > 2)) {
-    if (err_msg_out)
-      err_msg = tor_strdup("Bad syntax in hidden service port configuration.");
+    err_msg = tor_strdup("Bad syntax in hidden service port configuration.");
     goto err;
   }
   virtport = (int)tor_parse_long(smartlist_get(sl,0), 10, 1, 65535, NULL,NULL);
   if (!virtport) {
-    if (err_msg_out)
-      tor_asprintf(&err_msg, "Missing or invalid port %s in hidden service "
+    tor_asprintf(&err_msg, "Missing or invalid port %s in hidden service "
                    "port configuration", escaped(smartlist_get(sl,0)));
 
     goto err;
@@ -418,21 +455,27 @@ rend_service_parse_port_config(const char *string, const char *sep,
     int is_unix;
     ret = port_cfg_line_extract_addrport(addrport_element, &addrport,
                                          &is_unix, &rest);
+
     if (ret < 0) {
       tor_asprintf(&err_msg, "Couldn't process address <%s> from hidden "
                    "service configuration", addrport_element);
       goto err;
     }
+
+    if (rest && strlen(rest)) {
+      err_msg = tor_strdup("HiddenServicePort parse error: invalid port "
+                           "mapping");
+      goto err;
+    }
+
     if (is_unix) {
       socket_path = addrport;
       is_unix_addr = 1;
     } else if (strchr(addrport, ':') || strchr(addrport, '.')) {
       /* else try it as an IP:port pair if it has a : or . in it */
       if (tor_addr_port_lookup(addrport, &addr, &p)<0) {
-        if (err_msg_out)
-          err_msg = tor_strdup("Unparseable address in hidden service port "
-                               "configuration.");
-
+        err_msg = tor_strdup("Unparseable address in hidden service port "
+                             "configuration.");
         goto err;
       }
       realport = p?p:virtport;
@@ -440,11 +483,9 @@ rend_service_parse_port_config(const char *string, const char *sep,
       /* No addr:port, no addr -- must be port. */
       realport = (int)tor_parse_long(addrport, 10, 1, 65535, NULL, NULL);
       if (!realport) {
-        if (err_msg_out)
-          tor_asprintf(&err_msg, "Unparseable or out-of-range port %s in "
-                       "hidden service port configuration.",
-                       escaped(addrport));
-
+        tor_asprintf(&err_msg, "Unparseable or out-of-range port %s in "
+                     "hidden service port configuration.",
+                     escaped(addrport));
         goto err;
       }
       tor_addr_from_ipv4h(&addr, 0x7F000001u); /* Default to 127.0.0.1 */
@@ -463,7 +504,11 @@ rend_service_parse_port_config(const char *string, const char *sep,
 
  err:
   tor_free(addrport);
-  if (err_msg_out) *err_msg_out = err_msg;
+  if (err_msg_out != NULL) {
+    *err_msg_out = err_msg;
+  } else {
+    tor_free(err_msg);
+  }
   SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
   smartlist_free(sl);
 
@@ -472,66 +517,11 @@ rend_service_parse_port_config(const char *string, const char *sep,
 
 /** Release all storage held in a rend_service_port_config_t. */
 void
-rend_service_port_config_free(rend_service_port_config_t *p)
+rend_service_port_config_free_(rend_service_port_config_t *p)
 {
   tor_free(p);
 }
 
-/* Check the directory for <b>service</b>, and add the service to
- * <b>service_list</b>, or to the global list if <b>service_list</b> is NULL.
- * Only add the service to the list if <b>validate_only</b> is false.
- * If <b>validate_only</b> is true, free the service.
- * If <b>service</b> is NULL, ignore it, and return 0.
- * Returns 0 on success, and -1 on failure.
- * Takes ownership of <b>service</b>, either freeing it, or adding it to the
- * global service list.
- */
-STATIC int
-rend_service_check_dir_and_add(smartlist_t *service_list,
-                               const or_options_t *options,
-                               rend_service_t *service,
-                               int validate_only)
-{
-  if (!service) {
-    /* It is ok for a service to be NULL, this means there are no services */
-    return 0;
-  }
-
-  if (rend_service_check_private_dir(options, service, !validate_only)
-      < 0) {
-    rend_service_free(service);
-    return -1;
-  }
-
-  if (validate_only) {
-    rend_service_free(service);
-    return 0;
-  } else {
-    /* Use service_list for unit tests */
-    smartlist_t *s_list = NULL;
-    /* If no special service list is provided, then just use the global one. */
-    if (!service_list) {
-      if (BUG(!rend_service_list)) {
-        /* No global HS list, which is a failure, because we plan on adding to
-         * it */
-        return -1;
-      }
-      s_list = rend_service_list;
-    } else {
-      s_list = service_list;
-    }
-    /* s_list can not be NULL here - if both service_list and rend_service_list
-     * are NULL, and validate_only is false, we exit earlier in the function
-     */
-    if (BUG(!s_list)) {
-      return -1;
-    }
-    /* Ignore service failures until 030 */
-    rend_add_service(s_list, service);
-    return 0;
-  }
-}
-
 /* Copy relevant data from service src to dst while pruning the service lists.
  * This should only be called during the pruning process which takes existing
  * services and copy their data to the newly configured services. The src
@@ -556,141 +546,238 @@ copy_service_on_prunning(rend_service_t *dst, rend_service_t *src)
   dst->n_intro_points_wanted = src->n_intro_points_wanted;
 }
 
-/** Set up rend_service_list, based on the values of HiddenServiceDir and
- * HiddenServicePort in <b>options</b>.  Return 0 on success and -1 on
- * failure.  (If <b>validate_only</b> is set, parse, warn and return as
- * normal, but don't actually change the configured services.)
- */
+/* Helper: Actual implementation of the pruning on reload which we've
+ * decoupled in order to make the unit test workeable without ugly hacks.
+ * Furthermore, this function does NOT free any memory but will nullify the
+ * temporary list pointer whatever happens. */
+STATIC void
+rend_service_prune_list_impl_(void)
+{
+  origin_circuit_t *ocirc = NULL;
+  smartlist_t *surviving_services, *old_service_list, *new_service_list;
+
+  /* When pruning our current service list, we must have a staging list that
+   * contains what we want to check else it's a code flow error. */
+  tor_assert(rend_service_staging_list);
+
+  /* We are about to prune the current list of its dead service so set the
+   * semantic for that list to be the "old" one. */
+  old_service_list = rend_service_list;
+  /* The staging list is now the "new" list so set this semantic. */
+  new_service_list = rend_service_staging_list;
+  /* After this, whatever happens, we'll use our new list. */
+  rend_service_list = new_service_list;
+  /* Finally, nullify the staging list pointer as we don't need it anymore
+   * and it needs to be NULL before the next reload. */
+  rend_service_staging_list = NULL;
+  /* Nothing to prune if we have no service list so stop right away. */
+  if (!old_service_list) {
+    return;
+  }
+
+  /* This contains all _existing_ services that survives the relaod that is
+   * that haven't been removed from the configuration. The difference between
+   * this list and the new service list is that the new list can possibly
+   * contain newly configured service that have no introduction points opened
+   * yet nor key material loaded or generated. */
+  surviving_services = smartlist_new();
+
+  /* Preserve the existing ephemeral services.
+   *
+   * This is the ephemeral service equivalent of the "Copy introduction
+   * points to new services" block, except there's no copy required since
+   * the service structure isn't regenerated.
+   *
+   * After this is done, all ephemeral services will be:
+   *  * Removed from old_service_list, so the equivalent non-ephemeral code
+   *    will not attempt to preserve them.
+   *  * Added to the new_service_list (that previously only had the
+   *    services listed in the configuration).
+   *  * Added to surviving_services, which is the list of services that
+   *    will NOT have their intro point closed.
+   */
+  SMARTLIST_FOREACH_BEGIN(old_service_list, rend_service_t *, old) {
+    if (rend_service_is_ephemeral(old)) {
+      SMARTLIST_DEL_CURRENT(old_service_list, old);
+      smartlist_add(surviving_services, old);
+      smartlist_add(new_service_list, old);
+    }
+  } SMARTLIST_FOREACH_END(old);
+
+  /* Copy introduction points to new services. This is O(n^2), but it's only
+   * called on reconfigure, so it's ok performance wise. */
+  SMARTLIST_FOREACH_BEGIN(new_service_list, rend_service_t *, new) {
+    SMARTLIST_FOREACH_BEGIN(old_service_list, rend_service_t *, old) {
+      /* Skip ephemeral services as we only want to copy introduction points
+       * from current services to newly configured one that already exists.
+       * The same directory means it's the same service. */
+      if (rend_service_is_ephemeral(new) || rend_service_is_ephemeral(old) ||
+          strcmp(old->directory, new->directory)) {
+        continue;
+      }
+      smartlist_add_all(new->intro_nodes, old->intro_nodes);
+      smartlist_clear(old->intro_nodes);
+      smartlist_add_all(new->expiring_nodes, old->expiring_nodes);
+      smartlist_clear(old->expiring_nodes);
+
+      /* Copy needed information from old to new. */
+      copy_service_on_prunning(new, old);
+
+      /* This regular service will survive the closing IPs step after. */
+      smartlist_add(surviving_services, old);
+      break;
+    } SMARTLIST_FOREACH_END(old);
+  } SMARTLIST_FOREACH_END(new);
+
+  /* For every service introduction circuit we can find, see if we have a
+   * matching surviving configured service. If not, close the circuit. */
+  while ((ocirc = circuit_get_next_intro_circ(ocirc, false))) {
+    int keep_it = 0;
+    if (ocirc->rend_data == NULL) {
+      /* This is a v3 circuit, ignore it. */
+      continue;
+    }
+    SMARTLIST_FOREACH_BEGIN(surviving_services, const rend_service_t *, s) {
+      if (rend_circuit_pk_digest_eq(ocirc, (uint8_t *) s->pk_digest)) {
+        /* Keep this circuit as we have a matching configured service. */
+        keep_it = 1;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(s);
+    if (keep_it) {
+      continue;
+    }
+    log_info(LD_REND, "Closing intro point %s for service %s.",
+             safe_str_client(extend_info_describe(
+                                        ocirc->build_state->chosen_exit)),
+             safe_str_client(rend_data_get_address(ocirc->rend_data)));
+    /* Reason is FINISHED because service has been removed and thus the
+     * circuit is considered old/uneeded. */
+    circuit_mark_for_close(TO_CIRCUIT(ocirc), END_CIRC_REASON_FINISHED);
+  }
+  smartlist_free(surviving_services);
+  /* Notify that our global service list has changed. */
+  hs_service_map_has_changed();
+}
+
+/* Try to prune our main service list using the temporary one that we just
+ * loaded and parsed successfully. The pruning process decides which onion
+ * services to keep and which to discard after a reload. */
+void
+rend_service_prune_list(void)
+{
+  smartlist_t *old_service_list = rend_service_list;
+
+  if (!rend_service_staging_list) {
+    rend_service_staging_list = smartlist_new();
+  }
+
+  rend_service_prune_list_impl_();
+  if (old_service_list) {
+    /* Every remaining service in the old list have been removed from the
+     * configuration so clean them up safely. */
+    SMARTLIST_FOREACH(old_service_list, rend_service_t *, s,
+                      rend_service_free(s));
+    smartlist_free(old_service_list);
+  }
+}
+
+/* Copy all the relevant data that the hs_service object contains over to the
+ * rend_service_t object. The reason to do so is because when configuring a
+ * service, we go through a generic handler that creates an hs_service_t
+ * object which so we have to copy the parsed values to a rend service object
+ * which is version 2 specific. */
+static void
+service_config_shadow_copy(rend_service_t *service,
+                           hs_service_config_t *config)
+{
+  tor_assert(service);
+  tor_assert(config);
+
+  service->directory = tor_strdup(config->directory_path);
+  service->dir_group_readable = config->dir_group_readable;
+  service->allow_unknown_ports = config->allow_unknown_ports;
+  /* This value can't go above HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT (65535)
+   * if the code flow is right so this cast is safe. But just in case, we'll
+   * check it. */
+  service->max_streams_per_circuit = (int) config->max_streams_per_rdv_circuit;
+  if (BUG(config->max_streams_per_rdv_circuit >
+          HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT)) {
+    service->max_streams_per_circuit = HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT;
+  }
+  service->max_streams_close_circuit = config->max_streams_close_circuit;
+  service->n_intro_points_wanted = config->num_intro_points;
+  /* Switching ownership of the ports to the rend service object. */
+  smartlist_add_all(service->ports, config->ports);
+  smartlist_free(config->ports);
+  config->ports = NULL;
+}
+
+/* Parse the hidden service configuration starting at <b>line_</b> using the
+ * already configured generic service configuration in <b>config</b>. This
+ * function will translate the config object to a rend_service_t and add it to
+ * the temporary list if valid. If <b>validate_only</b> is set, parse, warn
+ * and return as normal but don't actually add the service to the list. */
 int
-rend_config_services(const or_options_t *options, int validate_only)
+rend_config_service(const config_line_t *line_,
+                    const or_options_t *options,
+                    hs_service_config_t *config)
 {
-  config_line_t *line;
+  const config_line_t *line;
   rend_service_t *service = NULL;
-  rend_service_port_config_t *portcfg;
-  smartlist_t *old_service_list = NULL;
-  int ok = 0;
 
-  if (!validate_only) {
-    old_service_list = rend_service_list;
-    rend_service_list = smartlist_new();
+  /* line_ can be NULL which would mean that the service configuration only
+   * have one line that is the directory directive. */
+  tor_assert(options);
+  tor_assert(config);
+
+  /* Use the staging service list so that we can check then do the pruning
+   * process using the main list at the end. */
+  if (rend_service_staging_list == NULL) {
+    rend_service_staging_list = smartlist_new();
   }
 
-  for (line = options->RendConfigLines; line; line = line->next) {
+  /* Initialize service. */
+  service = tor_malloc_zero(sizeof(rend_service_t));
+  service->intro_period_started = time(NULL);
+  service->ports = smartlist_new();
+  /* From the hs_service object which has been used to load the generic
+   * options, we'll copy over the useful data to the rend_service_t object. */
+  service_config_shadow_copy(service, config);
+
+  for (line = line_; line; line = line->next) {
     if (!strcasecmp(line->key, "HiddenServiceDir")) {
-      /* register the service we just finished parsing
-       * this code registers every service except the last one parsed,
-       * which is registered below the loop */
-      if (rend_service_check_dir_and_add(NULL, options, service,
-                                         validate_only) < 0) {
-          return -1;
-      }
-      service = tor_malloc_zero(sizeof(rend_service_t));
-      service->directory = tor_strdup(line->value);
-      service->ports = smartlist_new();
-      service->intro_period_started = time(NULL);
-      service->n_intro_points_wanted = NUM_INTRO_POINTS_DEFAULT;
-      continue;
-    }
-    if (!service) {
-      log_warn(LD_CONFIG, "%s with no preceding HiddenServiceDir directive",
-               line->key);
-      rend_service_free(service);
-      return -1;
+      /* We just hit the next hidden service, stop right now. */
+      break;
     }
-    if (!strcasecmp(line->key, "HiddenServicePort")) {
-      char *err_msg = NULL;
-      portcfg = rend_service_parse_port_config(line->value, " ", &err_msg);
-      if (!portcfg) {
-        if (err_msg)
-          log_warn(LD_CONFIG, "%s", err_msg);
-        tor_free(err_msg);
-        rend_service_free(service);
-        return -1;
-      }
-      tor_assert(!err_msg);
-      smartlist_add(service->ports, portcfg);
-    } else if (!strcasecmp(line->key, "HiddenServiceAllowUnknownPorts")) {
-      service->allow_unknown_ports = (int)tor_parse_long(line->value,
-                                                        10, 0, 1, &ok, NULL);
-      if (!ok) {
-        log_warn(LD_CONFIG,
-                 "HiddenServiceAllowUnknownPorts should be 0 or 1, not %s",
-                 line->value);
-        rend_service_free(service);
-        return -1;
-      }
-      log_info(LD_CONFIG,
-               "HiddenServiceAllowUnknownPorts=%d for %s",
-               (int)service->allow_unknown_ports, service->directory);
-    } else if (!strcasecmp(line->key,
-                           "HiddenServiceDirGroupReadable")) {
-        service->dir_group_readable = (int)tor_parse_long(line->value,
-                                                        10, 0, 1, &ok, NULL);
-        if (!ok) {
-            log_warn(LD_CONFIG,
-                     "HiddenServiceDirGroupReadable should be 0 or 1, not %s",
-                     line->value);
-            rend_service_free(service);
-            return -1;
-        }
-        log_info(LD_CONFIG,
-                 "HiddenServiceDirGroupReadable=%d for %s",
-                 service->dir_group_readable, service->directory);
-    } else if (!strcasecmp(line->key, "HiddenServiceMaxStreams")) {
-      service->max_streams_per_circuit = (int)tor_parse_long(line->value,
-                                                    10, 0, 65535, &ok, NULL);
-      if (!ok) {
-        log_warn(LD_CONFIG,
-                 "HiddenServiceMaxStreams should be between 0 and %d, not %s",
-                 65535, line->value);
-        rend_service_free(service);
-        return -1;
-      }
-      log_info(LD_CONFIG,
-               "HiddenServiceMaxStreams=%d for %s",
-               service->max_streams_per_circuit, service->directory);
-    } else if (!strcasecmp(line->key, "HiddenServiceMaxStreamsCloseCircuit")) {
-      service->max_streams_close_circuit = (int)tor_parse_long(line->value,
-                                                        10, 0, 1, &ok, NULL);
-      if (!ok) {
-        log_warn(LD_CONFIG,
-                 "HiddenServiceMaxStreamsCloseCircuit should be 0 or 1, "
-                 "not %s",
-                 line->value);
-        rend_service_free(service);
-        return -1;
-      }
-      log_info(LD_CONFIG,
-               "HiddenServiceMaxStreamsCloseCircuit=%d for %s",
-               (int)service->max_streams_close_circuit, service->directory);
-    } else if (!strcasecmp(line->key, "HiddenServiceNumIntroductionPoints")) {
+    /* Number of introduction points. */
+    if (!strcasecmp(line->key, "HiddenServiceNumIntroductionPoints")) {
+      int ok = 0;
+      /* Those are specific defaults for version 2. */
       service->n_intro_points_wanted =
         (unsigned int) tor_parse_long(line->value, 10,
-                                      NUM_INTRO_POINTS_DEFAULT,
-                                      NUM_INTRO_POINTS_MAX, &ok, NULL);
+                                      0, NUM_INTRO_POINTS_MAX, &ok, NULL);
       if (!ok) {
         log_warn(LD_CONFIG,
                  "HiddenServiceNumIntroductionPoints "
                  "should be between %d and %d, not %s",
-                 NUM_INTRO_POINTS_DEFAULT, NUM_INTRO_POINTS_MAX,
-                 line->value);
-        rend_service_free(service);
-        return -1;
+                 0, NUM_INTRO_POINTS_MAX, line->value);
+        goto err;
       }
       log_info(LD_CONFIG, "HiddenServiceNumIntroductionPoints=%d for %s",
-               service->n_intro_points_wanted, service->directory);
-    } else if (!strcasecmp(line->key, "HiddenServiceAuthorizeClient")) {
+               service->n_intro_points_wanted, escaped(service->directory));
+      continue;
+    }
+    if (!strcasecmp(line->key, "HiddenServiceAuthorizeClient")) {
       /* Parse auth type and comma-separated list of client names and add a
        * rend_authorized_client_t for each client to the service's list
        * of authorized clients. */
       smartlist_t *type_names_split, *clients;
       const char *authname;
-      int num_clients;
       if (service->auth_type != REND_NO_AUTH) {
         log_warn(LD_CONFIG, "Got multiple HiddenServiceAuthorizeClient "
                  "lines for a single service.");
-        rend_service_free(service);
-        return -1;
+        goto err;
       }
       type_names_split = smartlist_new();
       smartlist_split_string(type_names_split, line->value, " ", 0, 2);
@@ -699,8 +786,7 @@ rend_config_services(const or_options_t *options, int validate_only)
                          "should have been prevented when parsing the "
                          "configuration.");
         smartlist_free(type_names_split);
-        rend_service_free(service);
-        return -1;
+        goto err;
       }
       authname = smartlist_get(type_names_split, 0);
       if (!strcasecmp(authname, "basic")) {
@@ -714,8 +800,7 @@ rend_config_services(const or_options_t *options, int validate_only)
                  (char *) smartlist_get(type_names_split, 0));
         SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp));
         smartlist_free(type_names_split);
-        rend_service_free(service);
-        return -1;
+        goto err;
       }
       service->clients = smartlist_new();
       if (smartlist_len(type_names_split) < 2) {
@@ -732,14 +817,15 @@ rend_config_services(const or_options_t *options, int validate_only)
       SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp));
       smartlist_free(type_names_split);
       /* Remove duplicate client names. */
-      num_clients = smartlist_len(clients);
-      smartlist_sort_strings(clients);
-      smartlist_uniq_strings(clients);
-      if (smartlist_len(clients) < num_clients) {
-        log_info(LD_CONFIG, "HiddenServiceAuthorizeClient contains %d "
-                            "duplicate client name(s); removing.",
-                 num_clients - smartlist_len(clients));
-        num_clients = smartlist_len(clients);
+      {
+        int num_clients = smartlist_len(clients);
+        smartlist_sort_strings(clients);
+        smartlist_uniq_strings(clients);
+        if (smartlist_len(clients) < num_clients) {
+          log_info(LD_CONFIG, "HiddenServiceAuthorizeClient contains %d "
+                   "duplicate client name(s); removing.",
+                   num_clients - smartlist_len(clients));
+        }
       }
       SMARTLIST_FOREACH_BEGIN(clients, const char *, client_name)
       {
@@ -752,8 +838,7 @@ rend_config_services(const or_options_t *options, int validate_only)
                    client_name, REND_CLIENTNAME_MAX_LEN);
           SMARTLIST_FOREACH(clients, char *, cp, tor_free(cp));
           smartlist_free(clients);
-          rend_service_free(service);
-          return -1;
+          goto err;
         }
         client = tor_malloc_zero(sizeof(rend_authorized_client_t));
         client->client_name = tor_strdup(client_name);
@@ -775,112 +860,29 @@ rend_config_services(const or_options_t *options, int validate_only)
                  smartlist_len(service->clients),
                  service->auth_type == REND_BASIC_AUTH ? 512 : 16,
                  service->auth_type == REND_BASIC_AUTH ? "basic" : "stealth");
-        rend_service_free(service);
-        return -1;
-      }
-    } else {
-      tor_assert(!strcasecmp(line->key, "HiddenServiceVersion"));
-      if (strcmp(line->value, "2")) {
-        log_warn(LD_CONFIG,
-                 "The only supported HiddenServiceVersion is 2.");
-        rend_service_free(service);
-        return -1;
+        goto err;
       }
+      continue;
     }
   }
-  /* register the final service after we have finished parsing all services
-   * this code only registers the last service, other services are registered
-   * within the loop. It is ok for this service to be NULL, it is ignored. */
-  if (rend_service_check_dir_and_add(NULL, options, service,
-                                     validate_only) < 0) {
-    return -1;
+  /* Validate the service just parsed. */
+  if (rend_validate_service(rend_service_staging_list, service) < 0) {
+    /* Service is in the staging list so don't try to free it. */
+    goto err;
   }
 
-  /* If this is a reload and there were hidden services configured before,
-   * keep the introduction points that are still needed and close the
-   * other ones. */
-  if (old_service_list && !validate_only) {
-    smartlist_t *surviving_services = smartlist_new();
-
-    /* Preserve the existing ephemeral services.
-     *
-     * This is the ephemeral service equivalent of the "Copy introduction
-     * points to new services" block, except there's no copy required since
-     * the service structure isn't regenerated.
-     *
-     * After this is done, all ephemeral services will be:
-     *  * Removed from old_service_list, so the equivalent non-ephemeral code
-     *    will not attempt to preserve them.
-     *  * Added to the new rend_service_list (that previously only had the
-     *    services listed in the configuration).
-     *  * Added to surviving_services, which is the list of services that
-     *    will NOT have their intro point closed.
-     */
-    SMARTLIST_FOREACH(old_service_list, rend_service_t *, old, {
-      if (!old->directory) {
-        SMARTLIST_DEL_CURRENT(old_service_list, old);
-        smartlist_add(surviving_services, old);
-        smartlist_add(rend_service_list, old);
-      }
-    });
-
-    /* Copy introduction points to new services. */
-    /* XXXX This is O(n^2), but it's only called on reconfigure, so it's
-     * probably ok? */
-    SMARTLIST_FOREACH_BEGIN(rend_service_list, rend_service_t *, new) {
-      SMARTLIST_FOREACH_BEGIN(old_service_list, rend_service_t *, old) {
-        if (new->directory && old->directory &&
-            !strcmp(old->directory, new->directory)) {
-          smartlist_add_all(new->intro_nodes, old->intro_nodes);
-          smartlist_clear(old->intro_nodes);
-          smartlist_add_all(new->expiring_nodes, old->expiring_nodes);
-          smartlist_clear(old->expiring_nodes);
-          smartlist_add(surviving_services, old);
-
-          /* Copy service flags to the new service object. */
-          copy_service_on_prunning(new, old);
-          break;
-        }
-      } SMARTLIST_FOREACH_END(old);
-    } SMARTLIST_FOREACH_END(new);
-
-    /* Close introduction circuits of services we don't serve anymore. */
-    /* XXXX it would be nicer if we had a nicer abstraction to use here,
-     * so we could just iterate over the list of services to close, but
-     * once again, this isn't critical-path code. */
-    SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
-      if (!circ->marked_for_close &&
-          circ->state == CIRCUIT_STATE_OPEN &&
-          (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
-           circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) {
-        origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ);
-        int keep_it = 0;
-        tor_assert(oc->rend_data);
-        SMARTLIST_FOREACH(surviving_services, rend_service_t *, ptr, {
-          if (tor_memeq(ptr->pk_digest, oc->rend_data->rend_pk_digest,
-                      DIGEST_LEN)) {
-            keep_it = 1;
-            break;
-          }
-        });
-        if (keep_it)
-          continue;
-        log_info(LD_REND, "Closing intro point %s for service %s.",
-                 safe_str_client(extend_info_describe(
-                                            oc->build_state->chosen_exit)),
-                 oc->rend_data->onion_address);
-        circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED);
-        /* XXXX Is there another reason we should use here? */
-      }
-    }
-    SMARTLIST_FOREACH_END(circ);
-    smartlist_free(surviving_services);
-    SMARTLIST_FOREACH(old_service_list, rend_service_t *, ptr,
-                      rend_service_free(ptr));
-    smartlist_free(old_service_list);
+  /* Add it to the temporary list which we will use to prune our current
+   * list if any after configuring all services. */
+  if (rend_add_service(rend_service_staging_list, service) < 0) {
+    /* The object has been freed on error already. */
+    service = NULL;
+    goto err;
   }
 
   return 0;
+ err:
+  rend_service_free(service);
+  return -1;
 }
 
 /** Add the ephemeral service <b>pk</b>/<b>ports</b> if possible, using
@@ -895,9 +897,9 @@ rend_config_services(const or_options_t *options, int validate_only)
  * after calling this routine, and may assume that correct cleanup has
  * been done on failure.
  *
- * Return an appropriate rend_service_add_ephemeral_status_t.
+ * Return an appropriate hs_service_add_ephemeral_status_t.
  */
-rend_service_add_ephemeral_status_t
+hs_service_add_ephemeral_status_t
 rend_service_add_ephemeral(crypto_pk_t *pk,
                            smartlist_t *ports,
                            int max_streams_per_circuit,
@@ -969,7 +971,7 @@ int
 rend_service_del_ephemeral(const char *service_id)
 {
   rend_service_t *s;
-  if (!rend_valid_service_id(service_id)) {
+  if (!rend_valid_v2_service_id(service_id)) {
     log_warn(LD_CONFIG, "Requested malformed Onion Service id for removal.");
     return -1;
   }
@@ -978,7 +980,7 @@ rend_service_del_ephemeral(const char *service_id)
              "removal.");
     return -1;
   }
-  if (s->directory) {
+  if (!rend_service_is_ephemeral(s)) {
     log_warn(LD_CONFIG, "Requested non-ephemeral Onion Service for removal.");
     return -1;
   }
@@ -994,17 +996,20 @@ rend_service_del_ephemeral(const char *service_id)
         (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
          circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) {
       origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ);
-      tor_assert(oc->rend_data);
-      if (!tor_memeq(s->pk_digest, oc->rend_data->rend_pk_digest, DIGEST_LEN))
+      if (oc->rend_data == NULL ||
+          !rend_circuit_pk_digest_eq(oc, (uint8_t *) s->pk_digest)) {
         continue;
+      }
       log_debug(LD_REND, "Closing intro point %s for service %s.",
                 safe_str_client(extend_info_describe(
                                           oc->build_state->chosen_exit)),
-                oc->rend_data->onion_address);
+                rend_data_get_address(oc->rend_data));
       circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED);
     }
   } SMARTLIST_FOREACH_END(circ);
   smartlist_remove(rend_service_list, s);
+  /* Notify that we just removed a service from our global list. */
+  hs_service_map_has_changed();
   rend_service_free(s);
 
   log_debug(LD_CONFIG, "Removed ephemeral Onion Service: %s", service_id);
@@ -1012,6 +1017,45 @@ rend_service_del_ephemeral(const char *service_id)
   return 0;
 }
 
+/* There can be 1 second's delay due to second_elapsed_callback, and perhaps
+ * another few seconds due to blocking calls. */
+#define INTRO_CIRC_RETRY_PERIOD_SLOP 10
+
+/** Log information about the intro point creation rate and current intro
+ * points for service, upgrading the log level from min_severity to warn if
+ * we have stopped launching new intro point circuits. */
+static void
+rend_log_intro_limit(const rend_service_t *service, int min_severity)
+{
+  int exceeded_limit = (service->n_intro_circuits_launched >=
+                        rend_max_intro_circs_per_period(
+                                              service->n_intro_points_wanted));
+  int severity = min_severity;
+  /* We stopped creating circuits */
+  if (exceeded_limit) {
+    severity = LOG_WARN;
+  }
+  time_t intro_period_elapsed = time(NULL) - service->intro_period_started;
+  tor_assert_nonfatal(intro_period_elapsed >= 0);
+  {
+    char *msg;
+    static ratelim_t rlimit = RATELIM_INIT(INTRO_CIRC_RETRY_PERIOD);
+    if ((msg = rate_limit_log(&rlimit, approx_time()))) {
+      log_fn(severity, LD_REND,
+             "Hidden service %s %s %d intro points in the last %d seconds. "
+             "Intro circuit launches are limited to %d per %d seconds.%s",
+             service->service_id,
+             exceeded_limit ? "exceeded launch limit with" : "launched",
+             service->n_intro_circuits_launched,
+             (int)intro_period_elapsed,
+             rend_max_intro_circs_per_period(service->n_intro_points_wanted),
+             INTRO_CIRC_RETRY_PERIOD, msg);
+      rend_service_dump_stats(severity);
+      tor_free(msg);
+    }
+  }
+}
+
 /** Replace the old value of <b>service</b>-\>desc with one that reflects
  * the other fields in service.
  */
@@ -1019,7 +1063,6 @@ static void
 rend_service_update_descriptor(rend_service_t *service)
 {
   rend_service_descriptor_t *d;
-  origin_circuit_t *circ;
   int i;
 
   rend_service_descriptor_free(service->desc);
@@ -1040,9 +1083,10 @@ rend_service_update_descriptor(rend_service_t *service)
     /* This intro point won't be listed in the descriptor... */
     intro_svc->listed_in_last_desc = 0;
 
-    circ = find_intro_circuit(intro_svc, service->pk_digest);
-    if (!circ || circ->base_.purpose != CIRCUIT_PURPOSE_S_INTRO) {
-      /* This intro point's circuit isn't finished yet.  Don't list it. */
+    /* circuit_established is set in rend_service_intro_established(), and
+     * checked every second in rend_consider_services_intro_points(), so it's
+     * safe to use it here */
+    if (!intro_svc->circuit_established) {
       continue;
     }
 
@@ -1064,6 +1108,26 @@ rend_service_update_descriptor(rend_service_t *service)
       intro_svc->time_published = time(NULL);
     }
   }
+
+  /* Check that we have the right number of intro points */
+  unsigned int have_intro = (unsigned int)smartlist_len(d->intro_nodes);
+  if (have_intro != service->n_intro_points_wanted) {
+    int severity;
+    /* Getting less than we wanted or more than we're allowed is serious */
+    if (have_intro < service->n_intro_points_wanted ||
+        have_intro > NUM_INTRO_POINTS_MAX) {
+      severity = LOG_WARN;
+    } else {
+      /* Getting more than we wanted is weird, but less of a problem */
+      severity = LOG_NOTICE;
+    }
+    log_fn(severity, LD_REND, "Hidden service %s wanted %d intro points, but "
+           "descriptor was updated with %d instead.",
+           service->service_id,
+           service->n_intro_points_wanted, have_intro);
+    /* Now log an informative message about how we might have got here. */
+    rend_log_intro_limit(service, severity);
+  }
 }
 
 /* Allocate and return a string containing the path to file_name in
@@ -1073,15 +1137,8 @@ rend_service_update_descriptor(rend_service_t *service)
 static char *
 rend_service_path(const rend_service_t *service, const char *file_name)
 {
-  char *file_path = NULL;
-
   tor_assert(service->directory);
-
-  /* Can never fail: asserts rather than leaving file_path NULL. */
-  tor_asprintf(&file_path, "%s%s%s",
-               service->directory, PATH_SEPARATOR, file_name);
-
-  return file_path;
+  return hs_path_from_filename(service->directory, file_name);
 }
 
 /* Allocate and return a string containing the path to the single onion
@@ -1094,7 +1151,7 @@ rend_service_sos_poison_path(const rend_service_t *service)
   return rend_service_path(service, sos_poison_fname);
 }
 
-/** Return True if hidden services <b>service> has been poisoned by single
+/** Return True if hidden services <b>service</b> has been poisoned by single
  * onion mode. */
 static int
 service_is_single_onion_poisoned(const rend_service_t *service)
@@ -1107,7 +1164,7 @@ service_is_single_onion_poisoned(const rend_service_t *service)
     return 0;
   }
 
-  if (!service->directory) {
+  if (rend_service_is_ephemeral(service)) {
     return 0;
   }
 
@@ -1159,8 +1216,13 @@ rend_service_verify_single_onion_poison(const rend_service_t* s,
   }
 
   /* Ephemeral services are checked at ADD_ONION time */
-  if (!s->directory) {
-    return 0;
+  if (BUG(rend_service_is_ephemeral(s))) {
+    return -1;
+  }
+
+  /* Service is expected to have a directory */
+  if (BUG(!s->directory)) {
+    return -1;
   }
 
   /* Services without keys are always ok - their keys will only ever be used
@@ -1203,7 +1265,7 @@ poison_new_single_onion_hidden_service_dir_impl(const rend_service_t *service,
   int retval = -1;
   char *poison_fname = NULL;
 
-  if (!service->directory) {
+  if (rend_service_is_ephemeral(service)) {
     log_info(LD_REND, "Ephemeral HS started in non-anonymous mode.");
     return 0;
   }
@@ -1216,7 +1278,8 @@ poison_new_single_onion_hidden_service_dir_impl(const rend_service_t *service,
   }
 
   /* Make sure the directory was created before calling this function. */
-  if (BUG(rend_service_check_private_dir_impl(options, service, 0) < 0))
+  if (BUG(hs_check_service_private_dir(options->User, service->directory,
+                                       service->dir_group_readable, 0) < 0))
     return -1;
 
   poison_fname = rend_service_sos_poison_path(service);
@@ -1253,7 +1316,7 @@ poison_new_single_onion_hidden_service_dir_impl(const rend_service_t *service,
   return retval;
 }
 
-/** We just got launched in Single Onion Mode. That's a non-anoymous mode for
+/** We just got launched in Single Onion Mode. That's a non-anonymous mode for
  * hidden services. If s is new, we should mark its hidden service
  * directory appropriately so that it is never launched as a location-private
  * hidden service. (New directories don't have private key files.)
@@ -1270,6 +1333,16 @@ rend_service_poison_new_single_onion_dir(const rend_service_t *s,
   /* We must only poison directories if we're in Single Onion mode */
   tor_assert(rend_service_non_anonymous_mode_enabled(options));
 
+  /* Ephemeral services aren't allowed in non-anonymous mode */
+  if (BUG(rend_service_is_ephemeral(s))) {
+    return -1;
+  }
+
+  /* Service is expected to have a directory */
+  if (BUG(!s->directory)) {
+    return -1;
+  }
+
   if (!rend_service_private_key_exists(s)) {
     if (poison_new_single_onion_hidden_service_dir_impl(s, options)
         < 0) {
@@ -1280,6 +1353,29 @@ rend_service_poison_new_single_onion_dir(const rend_service_t *s,
   return 0;
 }
 
+/* Return true iff the given service identity key is present on disk. This is
+ * used to try to learn the service version during configuration time. */
+int
+rend_service_key_on_disk(const char *directory_path)
+{
+  int ret = 0;
+  char *fname;
+  crypto_pk_t *pk = NULL;
+
+  tor_assert(directory_path);
+
+  /* Load key */
+  fname = hs_path_from_filename(directory_path, private_key_fname);
+  pk = init_key_from_file(fname, 0, LOG_DEBUG, NULL);
+  if (pk) {
+    ret = 1;
+  }
+
+  crypto_pk_free(pk);
+  tor_free(fname);
+  return ret;
+}
+
 /** Load and/or generate private keys for all hidden services, possibly
  * including keys for client authorization.
  * If a <b>service_list</b> is provided, treat it as the list of hidden
@@ -1289,22 +1385,17 @@ rend_service_poison_new_single_onion_dir(const rend_service_t *s,
 int
 rend_service_load_all_keys(const smartlist_t *service_list)
 {
-  const smartlist_t *s_list = NULL;
-  /* If no special service list is provided, then just use the global one. */
-  if (!service_list) {
-    if (BUG(!rend_service_list)) {
-      return -1;
-    }
-    s_list = rend_service_list;
-  } else {
-    s_list = service_list;
+  /* Use service_list for unit tests */
+  const smartlist_t *s_list = rend_get_service_list(service_list);
+  if (BUG(!s_list)) {
+    return -1;
   }
 
   SMARTLIST_FOREACH_BEGIN(s_list, rend_service_t *, s) {
     if (s->private_key)
       continue;
-    log_info(LD_REND, "Loading hidden-service keys from \"%s\"",
-             s->directory);
+    log_info(LD_REND, "Loading hidden-service keys from %s",
+             rend_service_escaped_dir(s));
 
     if (rend_service_load_keys(s) < 0)
       return -1;
@@ -1336,15 +1427,15 @@ rend_services_add_filenames_to_lists(smartlist_t *open_lst,
   if (!rend_service_list)
     return;
   SMARTLIST_FOREACH_BEGIN(rend_service_list, rend_service_t *, s) {
-    if (s->directory) {
+    if (!rend_service_is_ephemeral(s)) {
       rend_service_add_filenames_to_list(open_lst, s);
-      smartlist_add(stat_lst, tor_strdup(s->directory));
+      smartlist_add_strdup(stat_lst, s->directory);
     }
   } SMARTLIST_FOREACH_END(s);
 }
 
 /** Derive all rend_service_t internal material based on the service's key.
- * Returns 0 on sucess, -1 on failure.
+ * Returns 0 on success, -1 on failure.
  */
 static int
 rend_service_derive_key_digests(struct rend_service_t *s)
@@ -1361,32 +1452,6 @@ rend_service_derive_key_digests(struct rend_service_t *s)
   return 0;
 }
 
-/* Implements the directory check from rend_service_check_private_dir,
- * without doing the single onion poison checks. */
-static int
-rend_service_check_private_dir_impl(const or_options_t *options,
-                                    const rend_service_t *s,
-                                    int create)
-{
-  cpd_check_t  check_opts = CPD_NONE;
-  if (create) {
-    check_opts |= CPD_CREATE;
-  } else {
-    check_opts |= CPD_CHECK_MODE_ONLY;
-    check_opts |= CPD_CHECK;
-  }
-  if (s->dir_group_readable) {
-    check_opts |= CPD_GROUP_READ;
-  }
-  /* Check/create directory */
-  if (check_private_dir(s->directory, check_opts, options->User) < 0) {
-    log_warn(LD_REND, "Checking service directory %s failed.", s->directory);
-    return -1;
-  }
-
-  return 0;
-}
-
 /** Make sure that the directory for <b>s</b> is private, using the config in
  * <b>options</b>.
  * If <b>create</b> is true:
@@ -1407,7 +1472,8 @@ rend_service_check_private_dir(const or_options_t *options,
   }
 
   /* Check/create directory */
-  if (rend_service_check_private_dir_impl(options, s, create) < 0) {
+  if (hs_check_service_private_dir(options->User, s->directory,
+                                   s->dir_group_readable, create) < 0) {
     return -1;
   }
 
@@ -1465,14 +1531,14 @@ rend_service_load_keys(rend_service_t *s)
   char *fname = NULL;
   char buf[128];
 
-  /* Make sure the directory was created and single onion poisoning was
-   * checked before calling this function */
-  if (BUG(rend_service_check_private_dir(get_options(), s, 0) < 0))
+  /* Create the directory if needed which will also poison it in case of
+   * single onion service. */
+  if (rend_service_check_private_dir(get_options(), s, 1) < 0)
     goto err;
 
   /* Load key */
   fname = rend_service_path(s, private_key_fname);
-  s->private_key = init_key_from_file(fname, 1, LOG_ERR, 0);
+  s->private_key = init_key_from_file(fname, 1, LOG_ERR, NULL);
 
   if (!s->private_key)
     goto err;
@@ -1496,7 +1562,7 @@ rend_service_load_keys(rend_service_t *s)
       log_warn(LD_FS,"Unable to make hidden hostname file %s group-readable.",
                fname);
   }
-#endif
+#endif /* !defined(_WIN32) */
 
   /* If client authorization is configured, load or generate keys. */
   if (s->auth_type != REND_NO_AUTH) {
@@ -1598,7 +1664,7 @@ rend_service_load_auth_keys(rend_service_t *s, const char *hfname)
         crypto_pk_free(prkey);
         goto err;
       }
-      if (crypto_pk_check_key(prkey) <= 0) {
+      if (! crypto_pk_is_valid_private_key(prkey)) {
         log_warn(LD_BUG,"Generated client key seems invalid");
         crypto_pk_free(prkey);
         goto err;
@@ -1684,7 +1750,7 @@ rend_service_load_auth_keys(rend_service_t *s, const char *hfname)
     memwipe(client_keys_str, 0, strlen(client_keys_str));
     tor_free(client_keys_str);
   }
-  strmap_free(parsed_clients, rend_authorized_client_strmap_item_free);
+  strmap_free(parsed_clients, rend_authorized_client_free_void);
 
   if (cfname) {
     memwipe(cfname, 0, strlen(cfname));
@@ -1725,24 +1791,6 @@ rend_service_get_by_service_id(const char *id)
   return NULL;
 }
 
-/** Return 1 if any virtual port in <b>service</b> wants a circuit
- * to have good uptime. Else return 0.
- */
-static int
-rend_service_requires_uptime(rend_service_t *service)
-{
-  int i;
-  rend_service_port_config_t *p;
-
-  for (i=0; i < smartlist_len(service->ports); ++i) {
-    p = smartlist_get(service->ports, i);
-    if (smartlist_contains_int_as_string(get_options()->LongLivedPorts,
-                                  p->virtual_port))
-      return 1;
-  }
-  return 0;
-}
-
 /** Check client authorization of a given <b>descriptor_cookie</b> of
  * length <b>cookie_len</b> for <b>service</b>. Return 1 for success
  * and 0 for failure. */
@@ -1833,7 +1881,7 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
   const or_options_t *options = get_options();
   char *err_msg = NULL;
   int err_msg_severity = LOG_WARN;
-  const char *stage_descr = NULL;
+  const char *stage_descr = NULL, *rend_pk_digest;
   int reason = END_CIRC_REASON_TORPROTOCOL;
   /* Service/circuit/key stuff we can learn before parsing */
   char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
@@ -1868,14 +1916,15 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
 
   assert_circ_anonymity_ok(circuit, options);
   tor_assert(circuit->rend_data);
+  /* XXX: This is version 2 specific (only one supported). */
+  rend_pk_digest = (char *) rend_data_get_pk_digest(circuit->rend_data, NULL);
 
   /* We'll use this in a bazillion log messages */
   base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
-                circuit->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN);
+                rend_pk_digest, REND_SERVICE_ID_LEN);
 
   /* look up service depending on circuit. */
-  service =
-    rend_service_get_by_pk_digest(circuit->rend_data->rend_pk_digest);
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
   if (!service) {
     log_warn(LD_BUG,
              "Internal error: Got an INTRODUCE2 cell on an intro "
@@ -2004,7 +2053,7 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
    * part 1. */
   replay = replaycache_add_test_and_elapsed(
       service->accepted_intro_dh_parts,
-      parsed_req->dh, DH_KEY_LEN,
+      parsed_req->dh, DH1024_KEY_LEN,
       &elapsed);
 
   if (replay) {
@@ -2054,27 +2103,31 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
   }
   if (crypto_dh_compute_secret(LOG_PROTOCOL_WARN, dh,
                                (char *)(parsed_req->dh),
-                               DH_KEY_LEN, keys,
+                               DH1024_KEY_LEN, keys,
                                DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) {
     log_warn(LD_BUG, "Internal error: couldn't complete DH handshake");
     reason = END_CIRC_REASON_INTERNAL;
     goto err;
   }
 
-  circ_needs_uptime = rend_service_requires_uptime(service);
+  circ_needs_uptime = hs_service_requires_uptime_circ(service->ports);
 
   /* help predict this next time */
   rep_hist_note_used_internal(now, circ_needs_uptime, 1);
 
   /* Launch a circuit to the client's chosen rendezvous point.
    */
-  int max_rend_failures=get_max_rend_failures();
+  int max_rend_failures=hs_get_service_max_rend_failures();
   for (i=0;i<max_rend_failures;i++) {
     int flags = CIRCLAUNCH_NEED_CAPACITY | CIRCLAUNCH_IS_INTERNAL;
     if (circ_needs_uptime) flags |= CIRCLAUNCH_NEED_UPTIME;
     /* A Single Onion Service only uses a direct connection if its
-     * firewall rules permit direct connections to the address. */
-    if (rend_service_use_direct_connection(options, rp)) {
+     * firewall rules permit direct connections to the address.
+     *
+     * We only use a one-hop path on the first attempt. If the first attempt
+     * fails, we use a 3-hop path for reachability / reliability.
+     * See the comment in rend_service_relauch_rendezvous() for details. */
+    if (rend_service_use_direct_connection(options, rp) && i == 0) {
       flags = flags | CIRCLAUNCH_ONEHOP_TUNNEL;
     }
     launched = circuit_launch_by_extend_info(
@@ -2100,8 +2153,7 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
   /* Fill in the circuit's state. */
 
   launched->rend_data =
-    rend_data_service_create(service->service_id,
-                             circuit->rend_data->rend_pk_digest,
+    rend_data_service_create(service->service_id, rend_pk_digest,
                              parsed_req->rc, service->auth_type);
 
   launched->build_state->service_pending_final_cpath_ref =
@@ -2115,7 +2167,9 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
 
   cpath->rend_dh_handshake_state = dh;
   dh = NULL;
-  if (circuit_init_cpath_crypto(cpath,keys+DIGEST_LEN,1)<0)
+  if (circuit_init_cpath_crypto(cpath,
+                                keys+DIGEST_LEN, sizeof(keys)-DIGEST_LEN,
+                                1, 0)<0)
     goto err;
   memcpy(cpath->rend_circ_nonce, keys, DIGEST_LEN);
 
@@ -2178,7 +2232,7 @@ find_rp_for_intro(const rend_intro_cell_t *intro,
   if (intro->version == 0 || intro->version == 1) {
     rp_nickname = (const char *)(intro->u.v0_v1.rp);
 
-    node = node_get_by_nickname(rp_nickname, 0);
+    node = node_get_by_nickname(rp_nickname, NNF_NO_WARN_UNNAMED);
     if (!node) {
       if (err_msg_out) {
         tor_asprintf(&err_msg,
@@ -2249,7 +2303,7 @@ find_rp_for_intro(const rend_intro_cell_t *intro,
  * rend_service_parse_intro().
  */
 void
-rend_service_free_intro(rend_intro_cell_t *request)
+rend_service_free_intro_(rend_intro_cell_t *request)
 {
   if (!request) {
     return;
@@ -2334,7 +2388,7 @@ rend_service_begin_parse_intro(const uint8_t *request,
   /* min key length plus digest length plus nickname length */
   if (request_len <
         (DIGEST_LEN + REND_COOKIE_LEN + (MAX_NICKNAME_LEN + 1) +
-         DH_KEY_LEN + 42)) {
+         DH1024_KEY_LEN + 42)) {
     if (err_msg_out) {
       tor_asprintf(&err_msg,
                    "got a truncated INTRODUCE%d cell",
@@ -2732,7 +2786,14 @@ rend_service_decrypt_intro(
   /* Check that this cell actually matches this service key */
 
   /* first DIGEST_LEN bytes of request is intro or service pk digest */
-  crypto_pk_get_digest(key, (char *)key_digest);
+  if (crypto_pk_get_digest(key, (char *)key_digest) < 0) {
+    if (err_msg_out)
+      *err_msg_out = tor_strdup("Couldn't compute RSA digest.");
+    log_warn(LD_BUG, "Couldn't compute key digest.");
+    status = -7;
+    goto err;
+  }
+
   if (tor_memneq(key_digest, intro->pk, DIGEST_LEN)) {
     if (err_msg_out) {
       base32_encode(service_id, REND_SERVICE_ID_LEN_BASE32 + 1,
@@ -2763,10 +2824,8 @@ rend_service_decrypt_intro(
   }
 
   /* Decrypt the encrypted part */
-
-  note_crypto_pk_op(REND_SERVER);
   result =
-    crypto_pk_private_hybrid_decrypt(
+    crypto_pk_obsolete_private_hybrid_decrypt(
        key, (char *)buf, sizeof(buf),
        (const char *)(intro->ciphertext), intro->ciphertext_len,
        PK_PKCS1_OAEP_PADDING, 1);
@@ -2865,14 +2924,14 @@ rend_service_parse_intro_plaintext(
    */
 
   ver_invariant_len = intro->plaintext_len - ver_specific_len;
-  if (ver_invariant_len < REND_COOKIE_LEN + DH_KEY_LEN) {
+  if (ver_invariant_len < REND_COOKIE_LEN + DH1024_KEY_LEN) {
     tor_asprintf(&err_msg,
         "decrypted plaintext of INTRODUCE%d cell was truncated (%ld bytes)",
         (int)(intro->type),
         (long)(intro->plaintext_len));
     status = -5;
     goto err;
-  } else if (ver_invariant_len > REND_COOKIE_LEN + DH_KEY_LEN) {
+  } else if (ver_invariant_len > REND_COOKIE_LEN + DH1024_KEY_LEN) {
     tor_asprintf(&err_msg,
         "decrypted plaintext of INTRODUCE%d cell was too long (%ld bytes)",
         (int)(intro->type),
@@ -2885,7 +2944,7 @@ rend_service_parse_intro_plaintext(
            REND_COOKIE_LEN);
     memcpy(intro->dh,
            intro->plaintext + ver_specific_len + REND_COOKIE_LEN,
-           DH_KEY_LEN);
+           DH1024_KEY_LEN);
   }
 
   /* Flag it as being fully parsed */
@@ -2959,35 +3018,6 @@ rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc)
   cpath_build_state_t *newstate, *oldstate;
 
   tor_assert(oldcirc->base_.purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
-
-  /* Don't relaunch the same rend circ twice. */
-  if (oldcirc->hs_service_side_rend_circ_has_been_relaunched) {
-    log_info(LD_REND, "Rendezvous circuit to %s has already been relaunched; "
-             "not relaunching it again.",
-             oldcirc->build_state ?
-             safe_str(extend_info_describe(oldcirc->build_state->chosen_exit))
-             : "*unknown*");
-    return;
-  }
-  oldcirc->hs_service_side_rend_circ_has_been_relaunched = 1;
-
-  /* We check failure_count >= get_max_rend_failures()-1 below, and the -1
-   * is because we increment the failure count for our current failure
-   * *after* this clause. */
-  int max_rend_failures = get_max_rend_failures() - 1;
-
-  if (!oldcirc->build_state ||
-      oldcirc->build_state->failure_count >= max_rend_failures ||
-      oldcirc->build_state->expiry_time < time(NULL)) {
-    log_info(LD_REND,
-             "Attempt to build circuit to %s for rendezvous has failed "
-             "too many times or expired; giving up.",
-             oldcirc->build_state ?
-             safe_str(extend_info_describe(oldcirc->build_state->chosen_exit))
-             : "*unknown*");
-    return;
-  }
-
   oldstate = oldcirc->build_state;
   tor_assert(oldstate);
 
@@ -3037,8 +3067,15 @@ rend_service_launch_establish_intro(rend_service_t *service,
   extend_info_t *launch_ei = intro->extend_info;
   extend_info_t *direct_ei = NULL;
 
-  /* Are we in single onion mode? */
-  if (rend_service_allow_non_anonymous_connection(options)) {
+  /* Are we in single onion mode?
+   *
+   * We only use a one-hop path on the first attempt. If the first attempt
+   * fails, we use a 3-hop path for reachability / reliability.
+   * (Unlike v3, retries is incremented by the caller after it calls this
+   * function.)
+   */
+  if (rend_service_allow_non_anonymous_connection(options) &&
+      intro->circuit_retries == 0) {
     /* Do we have a descriptor for the node?
      * We've either just chosen it from the consensus, or we've just reviewed
      * our intro points to see which ones are still valid, and deleted the ones
@@ -3140,15 +3177,67 @@ count_intro_point_circuits(const rend_service_t *service)
          circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) {
       origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ);
       if (oc->rend_data &&
-          !rend_cmp_service_ids(service->service_id,
-                                oc->rend_data->onion_address))
+          rend_circuit_pk_digest_eq(oc, (uint8_t *) service->pk_digest)) {
         num_ipos++;
+      }
     }
   }
   SMARTLIST_FOREACH_END(circ);
   return num_ipos;
 }
 
+/* Given a buffer of at least RELAY_PAYLOAD_SIZE bytes in <b>cell_body_out</b>,
+   write the body of a legacy ESTABLISH_INTRO cell in it. Use <b>intro_key</b>
+   as the intro point auth key, and <b>rend_circ_nonce</b> as the circuit
+   crypto material. On success, fill <b>cell_body_out</b> and return the number
+   of bytes written. On fail, return -1.
+ */
+ssize_t
+rend_service_encode_establish_intro_cell(char *cell_body_out,
+                                         size_t cell_body_out_len,
+                                         crypto_pk_t *intro_key,
+                                         const char *rend_circ_nonce)
+{
+  int retval = -1;
+  int r;
+  int len = 0;
+  char auth[DIGEST_LEN + 9];
+
+  tor_assert(intro_key);
+  tor_assert(rend_circ_nonce);
+
+  /* Build the payload for a RELAY_ESTABLISH_INTRO cell. */
+  r = crypto_pk_asn1_encode(intro_key, cell_body_out+2,
+                            RELAY_PAYLOAD_SIZE-2);
+  if (r < 0) {
+    log_warn(LD_BUG, "Internal error; failed to establish intro point.");
+    goto err;
+  }
+  len = r;
+  set_uint16(cell_body_out, htons((uint16_t)len));
+  len += 2;
+  memcpy(auth, rend_circ_nonce, DIGEST_LEN);
+  memcpy(auth+DIGEST_LEN, "INTRODUCE", 9);
+  if (crypto_digest(cell_body_out+len, auth, DIGEST_LEN+9))
+    goto err;
+  len += 20;
+  r = crypto_pk_private_sign_digest(intro_key, cell_body_out+len,
+                                    cell_body_out_len - len,
+                                    cell_body_out, len);
+  if (r<0) {
+    log_warn(LD_BUG, "Internal error: couldn't sign introduction request.");
+    goto err;
+  }
+  len += r;
+
+  retval = len;
+
+ err:
+  memwipe(auth, 0, sizeof(auth));
+
+  return retval;
+}
+
 /** Called when we're done building a circuit to an introduction point:
  *  sends a RELAY_ESTABLISH_INTRO cell.
  */
@@ -3156,23 +3245,23 @@ void
 rend_service_intro_has_opened(origin_circuit_t *circuit)
 {
   rend_service_t *service;
-  size_t len;
-  int r;
   char buf[RELAY_PAYLOAD_SIZE];
-  char auth[DIGEST_LEN + 9];
   char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
+  unsigned int expiring_nodes_len, num_ip_circuits, valid_ip_circuits = 0;
   int reason = END_CIRC_REASON_TORPROTOCOL;
+  const char *rend_pk_digest;
 
   tor_assert(circuit->base_.purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO);
   assert_circ_anonymity_ok(circuit, get_options());
   tor_assert(circuit->cpath);
   tor_assert(circuit->rend_data);
+  /* XXX: This is version 2 specific (only on supported). */
+  rend_pk_digest = (char *) rend_data_get_pk_digest(circuit->rend_data, NULL);
 
   base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
-                circuit->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN);
+                rend_pk_digest, REND_SERVICE_ID_LEN);
 
-  service = rend_service_get_by_pk_digest(
-                circuit->rend_data->rend_pk_digest);
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
   if (!service) {
     log_warn(LD_REND, "Unrecognized service ID %s on introduction circuit %u.",
              safe_str_client(serviceid), (unsigned)circuit->base_.n_circ_id);
@@ -3180,13 +3269,22 @@ rend_service_intro_has_opened(origin_circuit_t *circuit)
     goto err;
   }
 
+  /* Take the current amount of expiring nodes and the current amount of IP
+   * circuits and compute how many valid IP circuits we have. */
+  expiring_nodes_len = (unsigned int) smartlist_len(service->expiring_nodes);
+  num_ip_circuits = count_intro_point_circuits(service);
+  /* Let's avoid an underflow. The valid_ip_circuits is initialized to 0 in
+   * case this condition turns out false because it means that all circuits
+   * are expiring so we need to keep this circuit. */
+  if (num_ip_circuits > expiring_nodes_len) {
+    valid_ip_circuits = num_ip_circuits - expiring_nodes_len;
+  }
+
   /* If we already have enough introduction circuits for this service,
    * redefine this one as a general circuit or close it, depending.
-   * Substract the amount of expiring nodes here since the circuits are
+   * Subtract the amount of expiring nodes here because the circuits are
    * still opened. */
-  if ((count_intro_point_circuits(service) -
-       smartlist_len(service->expiring_nodes)) >
-      service->n_intro_points_wanted) {
+  if (valid_ip_circuits > service->n_intro_points_wanted) {
     const or_options_t *options = get_options();
     /* Remove the intro point associated with this circuit, it's being
      * repurposed or closed thus cleanup memory. */
@@ -3210,12 +3308,16 @@ rend_service_intro_has_opened(origin_circuit_t *circuit)
                "circuit, but we already have enough. Redefining purpose to "
                "general; leaving as internal.");
 
-      circuit_change_purpose(TO_CIRCUIT(circuit), CIRCUIT_PURPOSE_C_GENERAL);
+      if (circuit_should_use_vanguards(TO_CIRCUIT(circuit)->purpose)) {
+        circuit_change_purpose(TO_CIRCUIT(circuit),
+                CIRCUIT_PURPOSE_HS_VANGUARDS);
+      } else {
+        circuit_change_purpose(TO_CIRCUIT(circuit), CIRCUIT_PURPOSE_C_GENERAL);
+      }
 
       {
-        rend_data_t *rend_data = circuit->rend_data;
+        rend_data_free(circuit->rend_data);
         circuit->rend_data = NULL;
-        rend_data_free(rend_data);
       }
       {
         crypto_pk_t *intro_key = circuit->intro_key;
@@ -3233,42 +3335,25 @@ rend_service_intro_has_opened(origin_circuit_t *circuit)
            (unsigned)circuit->base_.n_circ_id, serviceid);
   circuit_log_path(LOG_INFO, LD_REND, circuit);
 
-  /* Use the intro key instead of the service key in ESTABLISH_INTRO. */
-  crypto_pk_t *intro_key = circuit->intro_key;
-  /* Build the payload for a RELAY_ESTABLISH_INTRO cell. */
-  r = crypto_pk_asn1_encode(intro_key, buf+2,
-                            RELAY_PAYLOAD_SIZE-2);
-  if (r < 0) {
-    log_warn(LD_BUG, "Internal error; failed to establish intro point.");
-    reason = END_CIRC_REASON_INTERNAL;
-    goto err;
-  }
-  len = r;
-  set_uint16(buf, htons((uint16_t)len));
-  len += 2;
-  memcpy(auth, circuit->cpath->prev->rend_circ_nonce, DIGEST_LEN);
-  memcpy(auth+DIGEST_LEN, "INTRODUCE", 9);
-  if (crypto_digest(buf+len, auth, DIGEST_LEN+9))
-    goto err;
-  len += 20;
-  note_crypto_pk_op(REND_SERVER);
-  r = crypto_pk_private_sign_digest(intro_key, buf+len, sizeof(buf)-len,
-                                    buf, len);
-  if (r<0) {
-    log_warn(LD_BUG, "Internal error: couldn't sign introduction request.");
-    reason = END_CIRC_REASON_INTERNAL;
-    goto err;
-  }
-  len += r;
+  /* Send the ESTABLISH_INTRO cell */
+  {
+    ssize_t len;
+    len = rend_service_encode_establish_intro_cell(buf, sizeof(buf),
+                                      circuit->intro_key,
+                                      circuit->cpath->prev->rend_circ_nonce);
+    if (len < 0) {
+      reason = END_CIRC_REASON_INTERNAL;
+      goto err;
+    }
 
-  if (relay_send_command_from_edge(0, TO_CIRCUIT(circuit),
-                                   RELAY_COMMAND_ESTABLISH_INTRO,
-                                   buf, len, circuit->cpath->prev)<0) {
-    log_info(LD_GENERAL,
+    if (relay_send_command_from_edge(0, TO_CIRCUIT(circuit),
+                                     RELAY_COMMAND_ESTABLISH_INTRO,
+                                     buf, len, circuit->cpath->prev)<0) {
+      log_info(LD_GENERAL,
              "Couldn't send introduction request for service %s on circuit %u",
              serviceid, (unsigned)circuit->base_.n_circ_id);
-    reason = END_CIRC_REASON_INTERNAL;
-    goto err;
+      goto done;
+    }
   }
 
   /* We've attempted to use this circuit */
@@ -3280,7 +3365,6 @@ rend_service_intro_has_opened(origin_circuit_t *circuit)
   circuit_mark_for_close(TO_CIRCUIT(circuit), reason);
  done:
   memwipe(buf, 0, sizeof(buf));
-  memwipe(auth, 0, sizeof(auth));
   memwipe(serviceid, 0, sizeof(serviceid));
 
   return;
@@ -3299,22 +3383,24 @@ rend_service_intro_established(origin_circuit_t *circuit,
   char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
   (void) request;
   (void) request_len;
+  tor_assert(circuit->rend_data);
+  /* XXX: This is version 2 specific (only supported one for now). */
+  const char *rend_pk_digest =
+    (char *) rend_data_get_pk_digest(circuit->rend_data, NULL);
 
   if (circuit->base_.purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) {
     log_warn(LD_PROTOCOL,
              "received INTRO_ESTABLISHED cell on non-intro circuit.");
     goto err;
   }
-  tor_assert(circuit->rend_data);
-  service = rend_service_get_by_pk_digest(
-                circuit->rend_data->rend_pk_digest);
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
   if (!service) {
     log_warn(LD_REND, "Unknown service on introduction circuit %u.",
              (unsigned)circuit->base_.n_circ_id);
     goto err;
   }
   base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32 + 1,
-                circuit->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN);
+                rend_pk_digest, REND_SERVICE_ID_LEN);
   /* We've just successfully established a intro circuit to one of our
    * introduction point, account for it. */
   intro = find_intro_point(circuit);
@@ -3357,6 +3443,7 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit)
   char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
   char hexcookie[9];
   int reason;
+  const char *rend_cookie, *rend_pk_digest;
 
   tor_assert(circuit->base_.purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
   tor_assert(circuit->cpath);
@@ -3364,18 +3451,24 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit)
   assert_circ_anonymity_ok(circuit, get_options());
   tor_assert(circuit->rend_data);
 
-  /* Declare the circuit dirty to avoid reuse, and for path-bias */
-  if (!circuit->base_.timestamp_dirty)
-    circuit->base_.timestamp_dirty = time(NULL);
+  /* XXX: This is version 2 specific (only one supported). */
+  rend_pk_digest = (char *) rend_data_get_pk_digest(circuit->rend_data,
+                                                    NULL);
+  rend_cookie = circuit->rend_data->rend_cookie;
+
+  /* Declare the circuit dirty to avoid reuse, and for path-bias. We set the
+   * timestamp regardless of its content because that circuit could have been
+   * cannibalized so in any cases, we are about to use that circuit more. */
+  circuit->base_.timestamp_dirty = time(NULL);
 
   /* This may be redundant */
   pathbias_count_use_attempt(circuit);
 
   hop = circuit->build_state->service_pending_final_cpath_ref->cpath;
 
-  base16_encode(hexcookie,9,circuit->rend_data->rend_cookie,4);
+  base16_encode(hexcookie,9, rend_cookie,4);
   base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
-                circuit->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN);
+                rend_pk_digest, REND_SERVICE_ID_LEN);
 
   log_info(LD_REND,
            "Done building circuit %u to rendezvous with "
@@ -3404,8 +3497,7 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit)
   circuit->build_state->pending_final_cpath = hop;
   circuit->build_state->service_pending_final_cpath_ref->cpath = NULL;
 
-  service = rend_service_get_by_pk_digest(
-                circuit->rend_data->rend_pk_digest);
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
   if (!service) {
     log_warn(LD_GENERAL, "Internal error: unrecognized service ID on "
              "rendezvous circuit.");
@@ -3414,24 +3506,23 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit)
   }
 
   /* All we need to do is send a RELAY_RENDEZVOUS1 cell... */
-  memcpy(buf, circuit->rend_data->rend_cookie, REND_COOKIE_LEN);
+  memcpy(buf, rend_cookie, REND_COOKIE_LEN);
   if (crypto_dh_get_public(hop->rend_dh_handshake_state,
-                           buf+REND_COOKIE_LEN, DH_KEY_LEN)<0) {
+                           buf+REND_COOKIE_LEN, DH1024_KEY_LEN)<0) {
     log_warn(LD_GENERAL,"Couldn't get DH public key.");
     reason = END_CIRC_REASON_INTERNAL;
     goto err;
   }
-  memcpy(buf+REND_COOKIE_LEN+DH_KEY_LEN, hop->rend_circ_nonce,
+  memcpy(buf+REND_COOKIE_LEN+DH1024_KEY_LEN, hop->rend_circ_nonce,
          DIGEST_LEN);
 
   /* Send the cell */
   if (relay_send_command_from_edge(0, TO_CIRCUIT(circuit),
                                    RELAY_COMMAND_RENDEZVOUS1,
-                                   buf, REND_COOKIE_LEN+DH_KEY_LEN+DIGEST_LEN,
+                                   buf, HS_LEGACY_RENDEZVOUS_CELL_SIZE,
                                    circuit->cpath->prev)<0) {
     log_warn(LD_GENERAL, "Couldn't send RENDEZVOUS1 cell.");
-    reason = END_CIRC_REASON_INTERNAL;
-    goto err;
+    goto done;
   }
 
   crypto_dh_free(hop->rend_dh_handshake_state);
@@ -3478,8 +3569,8 @@ find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest)
   origin_circuit_t *circ = NULL;
 
   tor_assert(intro);
-  while ((circ = circuit_get_next_by_pk_and_purpose(circ,pk_digest,
-                                                  CIRCUIT_PURPOSE_S_INTRO))) {
+  while ((circ = circuit_get_next_by_pk_and_purpose(circ,
+                         (uint8_t *) pk_digest, CIRCUIT_PURPOSE_S_INTRO))) {
     if (tor_memeq(circ->build_state->chosen_exit->identity_digest,
                 intro->extend_info->identity_digest, DIGEST_LEN) &&
         circ->rend_data) {
@@ -3488,8 +3579,9 @@ find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest)
   }
 
   circ = NULL;
-  while ((circ = circuit_get_next_by_pk_and_purpose(circ,pk_digest,
-                                        CIRCUIT_PURPOSE_S_ESTABLISH_INTRO))) {
+  while ((circ = circuit_get_next_by_pk_and_purpose(circ,
+                         (uint8_t *) pk_digest,
+                         CIRCUIT_PURPOSE_S_ESTABLISH_INTRO))) {
     if (tor_memeq(circ->build_state->chosen_exit->identity_digest,
                 intro->extend_info->identity_digest, DIGEST_LEN) &&
         circ->rend_data) {
@@ -3528,7 +3620,7 @@ find_intro_point(origin_circuit_t *circ)
   tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
              TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_INTRO);
   tor_assert(circ->rend_data);
-  serviceid = circ->rend_data->onion_address;
+  serviceid = rend_data_get_address(circ->rend_data);
 
   SMARTLIST_FOREACH(rend_service_list, rend_service_t *, s,
     if (tor_memeq(s->service_id, serviceid, REND_SERVICE_ID_LEN_BASE32)) {
@@ -3575,7 +3667,7 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc,
                           "directories to post descriptors to.");
         control_event_hs_descriptor_upload(service_id,
                                            "UNKNOWN",
-                                           "UNKNOWN");
+                                           "UNKNOWN", NULL);
         goto done;
       }
     }
@@ -3590,7 +3682,7 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc,
         /* Don't upload descriptor if we succeeded in doing so last time. */
         continue;
       node = node_get_by_id(hs_dir->identity_digest);
-      if (!node || !node_has_descriptor(node)) {
+      if (!node || !node_has_preferred_descriptor(node,0)) {
         log_info(LD_REND, "Not launching upload for for v2 descriptor to "
                           "hidden service directory %s; we don't have its "
                           "router descriptor. Queuing for later upload.",
@@ -3604,13 +3696,16 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc,
        * request. Lookup is made in rend_service_desc_has_uploaded(). */
       rend_data = rend_data_client_create(service_id, desc->desc_id, NULL,
                                           REND_NO_AUTH);
-      directory_initiate_command_routerstatus_rend(hs_dir,
-                                              DIR_PURPOSE_UPLOAD_RENDDESC_V2,
-                                                   ROUTER_PURPOSE_GENERAL,
-                                                   DIRIND_ANONYMOUS, NULL,
-                                                   desc->desc_str,
-                                                   strlen(desc->desc_str),
-                                                   0, rend_data);
+      directory_request_t *req =
+        directory_request_new(DIR_PURPOSE_UPLOAD_RENDDESC_V2);
+      directory_request_set_routerstatus(req, hs_dir);
+      directory_request_set_indirection(req, DIRIND_ANONYMOUS);
+      directory_request_set_payload(req,
+                                    desc->desc_str, strlen(desc->desc_str));
+      directory_request_set_rend_query(req, rend_data);
+      directory_initiate_request(req);
+      directory_request_free(req);
+
       rend_data_free(rend_data);
       base32_encode(desc_id_base32, sizeof(desc_id_base32),
                     desc->desc_id, DIGEST_LEN);
@@ -3627,7 +3722,7 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc,
                hs_dir->or_port);
       control_event_hs_descriptor_upload(service_id,
                                          hs_dir->identity_digest,
-                                         desc_id_base32);
+                                         desc_id_base32, NULL);
       tor_free(hs_dir_ip);
       /* Remember successful upload to this router for next time. */
       if (!smartlist_contains_digest(successful_uploads,
@@ -3724,7 +3819,7 @@ upload_service_descriptor(rend_service_t *service)
       }
       /* Free memory for descriptors. */
       for (i = 0; i < smartlist_len(descs); i++)
-        rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i));
+        rend_encoded_v2_service_descriptor_free_(smartlist_get(descs, i));
       smartlist_clear(descs);
       /* Update next upload time. */
       if (seconds_valid - REND_TIME_PERIOD_OVERLAPPING_V2_DESCS
@@ -3755,7 +3850,7 @@ upload_service_descriptor(rend_service_t *service)
         }
         /* Free memory for descriptors. */
         for (i = 0; i < smartlist_len(descs); i++)
-          rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i));
+          rend_encoded_v2_service_descriptor_free_(smartlist_get(descs, i));
         smartlist_clear(descs);
       }
     }
@@ -3842,6 +3937,19 @@ remove_invalid_intro_points(rend_service_t *service,
 {
   tor_assert(service);
 
+  /* Remove any expired nodes that doesn't have a circuit. */
+  SMARTLIST_FOREACH_BEGIN(service->expiring_nodes, rend_intro_point_t *,
+                          intro) {
+    origin_circuit_t *intro_circ =
+      find_intro_circuit(intro, service->pk_digest);
+    if (intro_circ) {
+      continue;
+    }
+    /* No more circuit, cleanup the into point object. */
+    SMARTLIST_DEL_CURRENT(service->expiring_nodes, intro);
+    rend_intro_point_free(intro);
+  } SMARTLIST_FOREACH_END(intro);
+
   SMARTLIST_FOREACH_BEGIN(service->intro_nodes, rend_intro_point_t *,
                           intro) {
     /* Find the introduction point node object. */
@@ -3917,10 +4025,13 @@ void
 rend_service_desc_has_uploaded(const rend_data_t *rend_data)
 {
   rend_service_t *service;
+  const char *onion_address;
 
   tor_assert(rend_data);
 
-  service = rend_service_get_by_service_id(rend_data->onion_address);
+  onion_address = rend_data_get_address(rend_data);
+
+  service = rend_service_get_by_service_id(onion_address);
   if (service == NULL) {
     return;
   }
@@ -3938,6 +4049,23 @@ rend_service_desc_has_uploaded(const rend_data_t *rend_data)
   } SMARTLIST_FOREACH_END(intro);
 }
 
+/** Don't try to build more than this many circuits before giving up
+ * for a while. Dynamically calculated based on the configured number of
+ * introduction points for the service, n_intro_points_wanted. */
+static int
+rend_max_intro_circs_per_period(unsigned int n_intro_points_wanted)
+{
+  /* Allow all but one of the initial connections to fail and be
+   * retried. (If all fail, we *want* to wait, because something is broken.) */
+  tor_assert(n_intro_points_wanted <= NUM_INTRO_POINTS_MAX);
+
+  /* For the normal use case, 3 intro points plus 2 extra for performance and
+   * allow that twice because once every 24h or so, we can do it twice for two
+   * descriptors that is the current one and the next one. So (3 + 2) * 2 ==
+   * 12 allowed attempts for one period. */
+  return ((n_intro_points_wanted + NUM_INTRO_POINTS_EXTRA) * 2);
+}
+
 /** For every service, check how many intro points it currently has, and:
  *  - Invalidate introdution points based on specific criteria, see
  *  remove_invalid_intro_points comments.
@@ -3947,10 +4075,9 @@ rend_service_desc_has_uploaded(const rend_data_t *rend_data)
  * This is called once a second by the main loop.
  */
 void
-rend_consider_services_intro_points(void)
+rend_consider_services_intro_points(time_t now)
 {
   int i;
-  time_t now;
   const or_options_t *options = get_options();
   /* Are we in single onion mode? */
   const int allow_direct = rend_service_allow_non_anonymous_connection(
@@ -3967,7 +4094,6 @@ rend_consider_services_intro_points(void)
 
   exclude_nodes = smartlist_new();
   retry_nodes = smartlist_new();
-  now = time(NULL);
 
   SMARTLIST_FOREACH_BEGIN(rend_service_list, rend_service_t *, service) {
     int r;
@@ -3982,23 +4108,29 @@ rend_consider_services_intro_points(void)
     smartlist_clear(exclude_nodes);
     smartlist_clear(retry_nodes);
 
+    /* Cleanup the invalid intro points and save the node objects, if any,
+     * in the exclude_nodes and retry_nodes lists. */
+    remove_invalid_intro_points(service, exclude_nodes, retry_nodes, now);
+
     /* This retry period is important here so we don't stress circuit
      * creation. */
+
     if (now > service->intro_period_started + INTRO_CIRC_RETRY_PERIOD) {
-      /* One period has elapsed; we can try building circuits again. */
+      /* One period has elapsed:
+       *  - if we stopped, we can try building circuits again,
+       *  - if we haven't, we reset the circuit creation counts. */
+      rend_log_intro_limit(service, LOG_INFO);
       service->intro_period_started = now;
       service->n_intro_circuits_launched = 0;
     } else if (service->n_intro_circuits_launched >=
-               MAX_INTRO_CIRCS_PER_PERIOD) {
+               rend_max_intro_circs_per_period(
+                                      service->n_intro_points_wanted)) {
       /* We have failed too many times in this period; wait for the next
-       * one before we try again. */
+       * one before we try to initiate any more connections. */
+      rend_log_intro_limit(service, LOG_WARN);
       continue;
     }
 
-    /* Cleanup the invalid intro points and save the node objects, if apply,
-     * in the exclude_nodes and retry_nodes list. */
-    remove_invalid_intro_points(service, exclude_nodes, retry_nodes, now);
-
     /* Let's try to rebuild circuit on the nodes we want to retry on. */
     SMARTLIST_FOREACH_BEGIN(retry_nodes, rend_intro_point_t *, intro) {
       r = rend_service_launch_establish_intro(service, intro);
@@ -4018,17 +4150,17 @@ rend_consider_services_intro_points(void)
     /* Avoid mismatched signed comparaison below. */
     intro_nodes_len = (unsigned int) smartlist_len(service->intro_nodes);
 
-    /* Quiescent state, no node expiring and we have more or the amount of
-     * wanted node for this service. Proceed to the next service. Could be
-     * more because we launch two preemptive circuits if our intro nodes
-     * list is empty. */
-    if (smartlist_len(service->expiring_nodes) == 0 &&
-        intro_nodes_len >= service->n_intro_points_wanted) {
+    /* Quiescent state, we have more or the equal amount of wanted node for
+     * this service. Proceed to the next service. We can have more nodes
+     * because we launch extra preemptive circuits if our intro nodes list was
+     * originally empty for performance reasons. */
+    if (intro_nodes_len >= service->n_intro_points_wanted) {
       continue;
     }
 
-    /* Number of intro points we want to open which is the wanted amount
-     * minus the current amount of valid nodes. */
+    /* Number of intro points we want to open which is the wanted amount minus
+     * the current amount of valid nodes. We know that this won't underflow
+     * because of the check above. */
     n_intro_points_to_open = service->n_intro_points_wanted - intro_nodes_len;
     if (intro_nodes_len == 0) {
       /* We want to end up with n_intro_points_wanted intro points, but if
@@ -4048,8 +4180,6 @@ rend_consider_services_intro_points(void)
       const node_t *node;
       rend_intro_point_t *intro;
       router_crn_flags_t flags = CRN_NEED_UPTIME|CRN_NEED_DESC;
-      if (get_options()->AllowInvalid_ & ALLOW_INVALID_INTRODUCTION)
-        flags |= CRN_ALLOW_INVALID;
       router_crn_flags_t direct_flags = flags;
       direct_flags |= CRN_PREF_ADDR;
       direct_flags |= CRN_DIRECT_CONN;
@@ -4077,7 +4207,7 @@ rend_consider_services_intro_points(void)
                  n_intro_points_to_open);
         break;
       }
-      /* Add the choosen node to the exclusion list in order to avoid picking
+      /* Add the chosen node to the exclusion list in order to avoid picking
        * it again in the next iteration. */
       smartlist_add(exclude_nodes, (void*)node);
       intro = tor_malloc_zero(sizeof(rend_intro_point_t));
@@ -4085,6 +4215,9 @@ rend_consider_services_intro_points(void)
        * even if we are a single onion service and intend to connect to it
        * directly ourselves. */
       intro->extend_info = extend_info_from_node(node, 0);
+      if (BUG(intro->extend_info == NULL)) {
+        break;
+      }
       intro->intro_key = crypto_pk_new();
       const int fail = crypto_pk_generate_key(intro->intro_key);
       tor_assert(!fail);
@@ -4220,8 +4353,8 @@ rend_service_dump_stats(int severity)
 
   for (i=0; i < smartlist_len(rend_service_list); ++i) {
     service = smartlist_get(rend_service_list, i);
-    tor_log(severity, LD_GENERAL, "Service configured in \"%s\":",
-        service->directory);
+    tor_log(severity, LD_GENERAL, "Service configured in %s:",
+            rend_service_escaped_dir(service));
     for (j=0; j < smartlist_len(service->intro_nodes); ++j) {
       intro = smartlist_get(service->intro_nodes, j);
       safe_name = safe_str_client(intro->extend_info->nickname);
@@ -4238,62 +4371,8 @@ rend_service_dump_stats(int severity)
   }
 }
 
-#ifdef HAVE_SYS_UN_H
-
-/** Given <b>ports</b>, a smarlist containing rend_service_port_config_t,
- * add the given <b>p</b>, a AF_UNIX port to the list. Return 0 on success
- * else return -ENOSYS if AF_UNIX is not supported (see function in the
- * #else statement below). */
-static int
-add_unix_port(smartlist_t *ports, rend_service_port_config_t *p)
-{
-  tor_assert(ports);
-  tor_assert(p);
-  tor_assert(p->is_unix_addr);
-
-  smartlist_add(ports, p);
-  return 0;
-}
-
-/** Given <b>conn</b> set it to use the given port <b>p</b> values. Return 0
- * on success else return -ENOSYS if AF_UNIX is not supported (see function
- * in the #else statement below). */
-static int
-set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p)
-{
-  tor_assert(conn);
-  tor_assert(p);
-  tor_assert(p->is_unix_addr);
-
-  conn->base_.socket_family = AF_UNIX;
-  tor_addr_make_unspec(&conn->base_.addr);
-  conn->base_.port = 1;
-  conn->base_.address = tor_strdup(p->unix_addr);
-  return 0;
-}
-
-#else /* defined(HAVE_SYS_UN_H) */
-
-static int
-set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p)
-{
-  (void) conn;
-  (void) p;
-  return -ENOSYS;
-}
-
-static int
-add_unix_port(smartlist_t *ports, rend_service_port_config_t *p)
-{
-  (void) ports;
-  (void) p;
-  return -ENOSYS;
-}
-
-#endif /* HAVE_SYS_UN_H */
-
 /** Given <b>conn</b>, a rendezvous exit stream, look up the hidden service for
- * 'circ', and look up the port and address based on conn-\>port.
+ * <b>circ</b>, and look up the port and address based on conn-\>port.
  * Assign the actual conn-\>addr and conn-\>port. Return -2 on failure
  * for which the circuit should be closed, -1 on other failure,
  * or 0 for success.
@@ -4304,17 +4383,15 @@ rend_service_set_connection_addr_port(edge_connection_t *conn,
 {
   rend_service_t *service;
   char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
-  smartlist_t *matching_ports;
-  rend_service_port_config_t *chosen_port;
-  unsigned int warn_once = 0;
+  const char *rend_pk_digest;
 
   tor_assert(circ->base_.purpose == CIRCUIT_PURPOSE_S_REND_JOINED);
   tor_assert(circ->rend_data);
   log_debug(LD_REND,"beginning to hunt for addr/port");
+  rend_pk_digest = (char *) rend_data_get_pk_digest(circ->rend_data, NULL);
   base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
-                circ->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN);
-  service = rend_service_get_by_pk_digest(
-                circ->rend_data->rend_pk_digest);
+                rend_pk_digest, REND_SERVICE_ID_LEN);
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
   if (!service) {
     log_warn(LD_REND, "Couldn't find any service associated with pk %s on "
              "rendezvous circuit %u; closing.",
@@ -4340,41 +4417,9 @@ rend_service_set_connection_addr_port(edge_connection_t *conn,
       return service->max_streams_close_circuit ? -2 : -1;
     }
   }
-  matching_ports = smartlist_new();
-  SMARTLIST_FOREACH(service->ports, rend_service_port_config_t *, p,
-  {
-    if (conn->base_.port != p->virtual_port) {
-      continue;
-    }
-    if (!(p->is_unix_addr)) {
-      smartlist_add(matching_ports, p);
-    } else {
-      if (add_unix_port(matching_ports, p)) {
-        if (!warn_once) {
-         /* Unix port not supported so warn only once. */
-          log_warn(LD_REND,
-              "Saw AF_UNIX virtual port mapping for port %d on service "
-              "%s, which is unsupported on this platform. Ignoring it.",
-              conn->base_.port, serviceid);
-        }
-        warn_once++;
-      }
-    }
-  });
-  chosen_port = smartlist_choose(matching_ports);
-  smartlist_free(matching_ports);
-  if (chosen_port) {
-    if (!(chosen_port->is_unix_addr)) {
-      /* Get a non-AF_UNIX connection ready for connection_exit_connect() */
-      tor_addr_copy(&conn->base_.addr, &chosen_port->real_addr);
-      conn->base_.port = chosen_port->real_port;
-    } else {
-      if (set_unix_port(conn, chosen_port)) {
-        /* Simply impossible to end up here else we were able to add a Unix
-         * port without AF_UNIX support... ? */
-        tor_assert(0);
-      }
-    }
+
+  if (hs_set_conn_addr_port(service->ports, conn) == 0) {
+    /* Successfully set the port to the connection. We are done. */
     return 0;
   }
 
@@ -4436,3 +4481,18 @@ rend_service_non_anonymous_mode_enabled(const or_options_t *options)
   return options->HiddenServiceNonAnonymousMode ? 1 : 0;
 }
 
+#ifdef TOR_UNIT_TESTS
+
+STATIC void
+set_rend_service_list(smartlist_t *new_list)
+{
+  rend_service_list = new_list;
+}
+
+STATIC void
+set_rend_rend_service_staging_list(smartlist_t *new_list)
+{
+  rend_service_staging_list = new_list;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/or/rendservice.h b/src/feature/rend/rendservice.h
index 3b185672f6..a8eb28bee2 100644
--- a/src/or/rendservice.h
+++ b/src/feature/rend/rendservice.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,16 +12,15 @@
 #ifndef TOR_RENDSERVICE_H
 #define TOR_RENDSERVICE_H
 
-#include "or.h"
+#include "core/or/or.h"
+#include "feature/hs/hs_service.h"
 
-typedef struct rend_intro_cell_s rend_intro_cell_t;
-typedef struct rend_service_port_config_s rend_service_port_config_t;
-
-#ifdef RENDSERVICE_PRIVATE
+typedef struct rend_intro_cell_t rend_intro_cell_t;
+struct config_line_t;
 
 /* This can be used for both INTRODUCE1 and INTRODUCE2 */
 
-struct rend_intro_cell_s {
+struct rend_intro_cell_t {
   /* Is this an INTRODUCE1 or INTRODUCE2? (set to 1 or 2) */
   uint8_t type;
   /* Public key digest */
@@ -60,9 +59,11 @@ struct rend_intro_cell_s {
   /* Rendezvous cookie */
   uint8_t rc[REND_COOKIE_LEN];
   /* Diffie-Hellman data */
-  uint8_t dh[DH_KEY_LEN];
+  uint8_t dh[DH1024_KEY_LEN];
 };
 
+#ifdef RENDSERVICE_PRIVATE
+
 /** Represents a single hidden service running at this OP. */
 typedef struct rend_service_t {
   /* Fields specified in config file */
@@ -108,7 +109,7 @@ typedef struct rend_service_t {
   /** If true, we don't close circuits for making requests to unsupported
    * ports. */
   int allow_unknown_ports;
-  /** The maximum number of simultanious streams-per-circuit that are allowed
+  /** The maximum number of simultaneous streams-per-circuit that are allowed
    * to be established, or 0 if no limit is set.
    */
   int max_streams_per_circuit;
@@ -117,26 +118,37 @@ typedef struct rend_service_t {
   int max_streams_close_circuit;
 } rend_service_t;
 
-STATIC void rend_service_free(rend_service_t *service);
+STATIC void rend_service_free_(rend_service_t *service);
+#define rend_service_free(s) \
+  FREE_AND_NULL(rend_service_t, rend_service_free_, (s))
 STATIC char *rend_service_sos_poison_path(const rend_service_t *service);
-STATIC int rend_service_check_dir_and_add(smartlist_t *service_list,
-                                          const or_options_t *options,
-                                          rend_service_t *service,
-                                          int validate_only);
 STATIC int rend_service_verify_single_onion_poison(
                                                   const rend_service_t *s,
                                                   const or_options_t *options);
 STATIC int rend_service_poison_new_single_onion_dir(
                                                   const rend_service_t *s,
                                                   const or_options_t* options);
-#endif
+#ifdef TOR_UNIT_TESTS
+
+STATIC void set_rend_service_list(smartlist_t *new_list);
+STATIC void set_rend_rend_service_staging_list(smartlist_t *new_list);
+STATIC void rend_service_prune_list_impl_(void);
 
-int num_rend_services(void);
-int rend_config_services(const or_options_t *options, int validate_only);
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(RENDSERVICE_PRIVATE) */
+
+int rend_num_services(void);
+int rend_config_service(const struct config_line_t *line_,
+                        const or_options_t *options,
+                        hs_service_config_t *config);
+void rend_service_prune_list(void);
+void rend_service_free_staging_list(void);
 int rend_service_load_all_keys(const smartlist_t *service_list);
+int rend_service_key_on_disk(const char *directory_path);
 void rend_services_add_filenames_to_lists(smartlist_t *open_lst,
                                           smartlist_t *stat_lst);
-void rend_consider_services_intro_points(void);
+void rend_consider_services_intro_points(time_t now);
 void rend_consider_services_upload(time_t now);
 void rend_hsdir_routers_changed(void);
 void rend_consider_descriptor_republication(void);
@@ -152,13 +164,21 @@ int rend_service_receive_introduction(origin_circuit_t *circuit,
 int rend_service_decrypt_intro(rend_intro_cell_t *request,
                                crypto_pk_t *key,
                                char **err_msg_out);
-void rend_service_free_intro(rend_intro_cell_t *request);
+void rend_service_free_intro_(rend_intro_cell_t *request);
+#define rend_service_free_intro(req) do {       \
+    rend_service_free_intro_(req);              \
+    (req) = NULL;                               \
+  } while (0)
 rend_intro_cell_t * rend_service_begin_parse_intro(const uint8_t *request,
                                                    size_t request_len,
                                                    uint8_t type,
                                                    char **err_msg_out);
 int rend_service_parse_intro_plaintext(rend_intro_cell_t *intro,
                                        char **err_msg_out);
+ssize_t rend_service_encode_establish_intro_cell(char *cell_body_out,
+                                                 size_t cell_body_out_len,
+                                                 crypto_pk_t *intro_key,
+                                                 const char *rend_circ_nonce);
 int rend_service_validate_intro_late(const rend_intro_cell_t *intro,
                                      char **err_msg_out);
 void rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc);
@@ -166,24 +186,22 @@ int rend_service_set_connection_addr_port(edge_connection_t *conn,
                                           origin_circuit_t *circ);
 void rend_service_dump_stats(int severity);
 void rend_service_free_all(void);
+void rend_service_init(void);
 
 rend_service_port_config_t *rend_service_parse_port_config(const char *string,
                                                            const char *sep,
                                                            char **err_msg_out);
-void rend_service_port_config_free(rend_service_port_config_t *p);
-
-void rend_authorized_client_free(rend_authorized_client_t *client);
-
-/** Return value from rend_service_add_ephemeral. */
-typedef enum {
-  RSAE_BADAUTH = -5, /**< Invalid auth_type/auth_clients */
-  RSAE_BADVIRTPORT = -4, /**< Invalid VIRTPORT/TARGET(s) */
-  RSAE_ADDREXISTS = -3, /**< Onion address collision */
-  RSAE_BADPRIVKEY = -2, /**< Invalid public key */
-  RSAE_INTERNAL = -1, /**< Internal error */
-  RSAE_OKAY = 0 /**< Service added as expected */
-} rend_service_add_ephemeral_status_t;
-rend_service_add_ephemeral_status_t rend_service_add_ephemeral(crypto_pk_t *pk,
+void rend_service_port_config_free_(rend_service_port_config_t *p);
+#define rend_service_port_config_free(p) \
+  FREE_AND_NULL(rend_service_port_config_t, rend_service_port_config_free_, \
+                (p))
+
+void rend_authorized_client_free_(rend_authorized_client_t *client);
+#define rend_authorized_client_free(client) \
+  FREE_AND_NULL(rend_authorized_client_t, rend_authorized_client_free_, \
+                (client))
+
+hs_service_add_ephemeral_status_t rend_service_add_ephemeral(crypto_pk_t *pk,
                                smartlist_t *ports,
                                int max_streams_per_circuit,
                                int max_streams_close_circuit,
@@ -201,5 +219,4 @@ int rend_service_allow_non_anonymous_connection(const or_options_t *options);
 int rend_service_reveal_startup_time(const or_options_t *options);
 int rend_service_non_anonymous_mode_enabled(const or_options_t *options);
 
-#endif
-
+#endif /* !defined(TOR_RENDSERVICE_H) */
diff --git a/src/or/geoip.c b/src/feature/stats/geoip_stats.c
index a39366ed13..a54b589eb6 100644
--- a/src/or/geoip.c
+++ b/src/feature/stats/geoip_stats.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2016, The Tor Project, Inc. */
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,51 +27,26 @@
  * for each country.
  */
 
-#define GEOIP_PRIVATE
-#include "or.h"
+#include "core/or/or.h"
+
 #include "ht.h"
-#include "config.h"
-#include "control.h"
-#include "dnsserv.h"
-#include "dos.h"
-#include "geoip.h"
-#include "routerlist.h"
-
-static void init_geoip_countries(void);
-
-/** An entry from the GeoIP IPv4 file: maps an IPv4 range to a country. */
-typedef struct geoip_ipv4_entry_t {
-  uint32_t ip_low; /**< The lowest IP in the range, in host order */
-  uint32_t ip_high; /**< The highest IP in the range, in host order */
-  intptr_t country; /**< An index into geoip_countries */
-} geoip_ipv4_entry_t;
-
-/** An entry from the GeoIP IPv6 file: maps an IPv6 range to a country. */
-typedef struct geoip_ipv6_entry_t {
-  struct in6_addr ip_low; /**< The lowest IP in the range, in host order */
-  struct in6_addr ip_high; /**< The highest IP in the range, in host order */
-  intptr_t country; /**< An index into geoip_countries */
-} geoip_ipv6_entry_t;
-
-/** A per-country record for GeoIP request history. */
-typedef struct geoip_country_t {
-  char countrycode[3];
-  uint32_t n_v3_ns_requests;
-} geoip_country_t;
-
-/** A list of geoip_country_t */
-static smartlist_t *geoip_countries = NULL;
-/** A map from lowercased country codes to their position in geoip_countries.
- * The index is encoded in the pointer, and 1 is added so that NULL can mean
- * not found. */
-static strmap_t *country_idxplus1_by_lc_code = NULL;
-/** Lists of all known geoip_ipv4_entry_t and geoip_ipv6_entry_t, sorted
- * by their respective ip_low. */
-static smartlist_t *geoip_ipv4_entries = NULL, *geoip_ipv6_entries = NULL;
-
-/** SHA1 digest of the GeoIP files to include in extra-info descriptors. */
-static char geoip_digest[DIGEST_LEN];
-static char geoip6_digest[DIGEST_LEN];
+#include "lib/container/buffers.h"
+#include "app/config/config.h"
+#include "feature/control/control.h"
+#include "feature/client/dnsserv.h"
+#include "core/or/dos.h"
+#include "lib/geoip/geoip.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/nodelist/routerlist.h"
+
+#include "lib/container/order.h"
+#include "lib/time/tvdiff.h"
+
+/** Number of entries in n_v3_ns_requests */
+static size_t n_v3_ns_requests_len = 0;
+/** Array, indexed by country index, of number of v3 networkstatus requests
+ * received from that country */
+static uint32_t *n_v3_ns_requests;
 
 /* Total size in bytes of the geoip client history cache. Used by the OOM
  * handler. */
@@ -105,194 +80,30 @@ geoip_decrement_client_history_cache_size(size_t bytes)
   geoip_client_history_cache_size -= bytes;
 }
 
-/** Return the index of the <b>country</b>'s entry in the GeoIP
- * country list if it is a valid 2-letter country code, otherwise
- * return -1. */
-MOCK_IMPL(country_t,
-geoip_get_country,(const char *country))
-{
-  void *idxplus1_;
-  intptr_t idx;
-
-  idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country);
-  if (!idxplus1_)
-    return -1;
-
-  idx = ((uintptr_t)idxplus1_)-1;
-  return (country_t)idx;
-}
-
-/** Add an entry to a GeoIP table, mapping all IP addresses between <b>low</b>
- * and <b>high</b>, inclusive, to the 2-letter country code <b>country</b>. */
+/** Add 1 to the count of v3 ns requests received from <b>country</b>. */
 static void
-geoip_add_entry(const tor_addr_t *low, const tor_addr_t *high,
-                const char *country)
+increment_v3_ns_request(country_t country)
 {
-  intptr_t idx;
-  void *idxplus1_;
-
-  IF_BUG_ONCE(tor_addr_family(low) != tor_addr_family(high))
-    return;
-  IF_BUG_ONCE(tor_addr_compare(high, low, CMP_EXACT) < 0)
+  if (country < 0)
     return;
 
-  idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country);
-
-  if (!idxplus1_) {
-    geoip_country_t *c = tor_malloc_zero(sizeof(geoip_country_t));
-    strlcpy(c->countrycode, country, sizeof(c->countrycode));
-    tor_strlower(c->countrycode);
-    smartlist_add(geoip_countries, c);
-    idx = smartlist_len(geoip_countries) - 1;
-    strmap_set_lc(country_idxplus1_by_lc_code, country, (void*)(idx+1));
-  } else {
-    idx = ((uintptr_t)idxplus1_)-1;
-  }
-  {
-    geoip_country_t *c = smartlist_get(geoip_countries, idx);
-    tor_assert(!strcasecmp(c->countrycode, country));
-  }
-
-  if (tor_addr_family(low) == AF_INET) {
-    geoip_ipv4_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv4_entry_t));
-    ent->ip_low = tor_addr_to_ipv4h(low);
-    ent->ip_high = tor_addr_to_ipv4h(high);
-    ent->country = idx;
-    smartlist_add(geoip_ipv4_entries, ent);
-  } else if (tor_addr_family(low) == AF_INET6) {
-    geoip_ipv6_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv6_entry_t));
-    ent->ip_low = *tor_addr_to_in6_assert(low);
-    ent->ip_high = *tor_addr_to_in6_assert(high);
-    ent->country = idx;
-    smartlist_add(geoip_ipv6_entries, ent);
-  }
-}
-
-/** Add an entry to the GeoIP table indicated by <b>family</b>,
- * parsing it from <b>line</b>. The format is as for geoip_load_file(). */
-STATIC int
-geoip_parse_entry(const char *line, sa_family_t family)
-{
-  tor_addr_t low_addr, high_addr;
-  char c[3];
-  char *country = NULL;
-
-  if (!geoip_countries)
-    init_geoip_countries();
-  if (family == AF_INET) {
-    if (!geoip_ipv4_entries)
-      geoip_ipv4_entries = smartlist_new();
-  } else if (family == AF_INET6) {
-    if (!geoip_ipv6_entries)
-      geoip_ipv6_entries = smartlist_new();
-  } else {
-    log_warn(LD_GENERAL, "Unsupported family: %d", family);
-    return -1;
+  if ((size_t)country >= n_v3_ns_requests_len) {
+    /* We need to reallocate the array. */
+    size_t new_len;
+    if (n_v3_ns_requests_len == 0)
+      new_len = 256;
+    else
+      new_len = n_v3_ns_requests_len * 2;
+    if (new_len <= (size_t)country)
+      new_len = ((size_t)country)+1;
+    n_v3_ns_requests = tor_reallocarray(n_v3_ns_requests, new_len,
+                                        sizeof(uint32_t));
+    memset(n_v3_ns_requests + n_v3_ns_requests_len, 0,
+           sizeof(uint32_t)*(new_len - n_v3_ns_requests_len));
+    n_v3_ns_requests_len = new_len;
   }
 
-  while (TOR_ISSPACE(*line))
-    ++line;
-  if (*line == '#')
-    return 0;
-
-  char buf[512];
-  if (family == AF_INET) {
-    unsigned int low, high;
-    if (tor_sscanf(line,"%u,%u,%2s", &low, &high, c) == 3 ||
-        tor_sscanf(line,"\"%u\",\"%u\",\"%2s\",", &low, &high, c) == 3) {
-      tor_addr_from_ipv4h(&low_addr, low);
-      tor_addr_from_ipv4h(&high_addr, high);
-    } else
-      goto fail;
-    country = c;
-  } else {                      /* AF_INET6 */
-    char *low_str, *high_str;
-    struct in6_addr low, high;
-    char *strtok_state;
-    strlcpy(buf, line, sizeof(buf));
-    low_str = tor_strtok_r(buf, ",", &strtok_state);
-    if (!low_str)
-      goto fail;
-    high_str = tor_strtok_r(NULL, ",", &strtok_state);
-    if (!high_str)
-      goto fail;
-    country = tor_strtok_r(NULL, "\n", &strtok_state);
-    if (!country)
-      goto fail;
-    if (strlen(country) != 2)
-      goto fail;
-    if (tor_inet_pton(AF_INET6, low_str, &low) <= 0)
-      goto fail;
-    tor_addr_from_in6(&low_addr, &low);
-    if (tor_inet_pton(AF_INET6, high_str, &high) <= 0)
-      goto fail;
-    tor_addr_from_in6(&high_addr, &high);
-  }
-  geoip_add_entry(&low_addr, &high_addr, country);
-  return 0;
-
-  fail:
-  log_warn(LD_GENERAL, "Unable to parse line from GEOIP %s file: %s",
-           family == AF_INET ? "IPv4" : "IPv6", escaped(line));
-  return -1;
-}
-
-/** Sorting helper: return -1, 1, or 0 based on comparison of two
- * geoip_ipv4_entry_t */
-static int
-geoip_ipv4_compare_entries_(const void **_a, const void **_b)
-{
-  const geoip_ipv4_entry_t *a = *_a, *b = *_b;
-  if (a->ip_low < b->ip_low)
-    return -1;
-  else if (a->ip_low > b->ip_low)
-    return 1;
-  else
-    return 0;
-}
-
-/** bsearch helper: return -1, 1, or 0 based on comparison of an IP (a pointer
- * to a uint32_t in host order) to a geoip_ipv4_entry_t */
-static int
-geoip_ipv4_compare_key_to_entry_(const void *_key, const void **_member)
-{
-  /* No alignment issue here, since _key really is a pointer to uint32_t */
-  const uint32_t addr = *(uint32_t *)_key;
-  const geoip_ipv4_entry_t *entry = *_member;
-  if (addr < entry->ip_low)
-    return -1;
-  else if (addr > entry->ip_high)
-    return 1;
-  else
-    return 0;
-}
-
-/** Sorting helper: return -1, 1, or 0 based on comparison of two
- * geoip_ipv6_entry_t */
-static int
-geoip_ipv6_compare_entries_(const void **_a, const void **_b)
-{
-  const geoip_ipv6_entry_t *a = *_a, *b = *_b;
-  return fast_memcmp(a->ip_low.s6_addr, b->ip_low.s6_addr,
-                     sizeof(struct in6_addr));
-}
-
-/** bsearch helper: return -1, 1, or 0 based on comparison of an IPv6
- * (a pointer to a in6_addr) to a geoip_ipv6_entry_t */
-static int
-geoip_ipv6_compare_key_to_entry_(const void *_key, const void **_member)
-{
-  const struct in6_addr *addr = (struct in6_addr *)_key;
-  const geoip_ipv6_entry_t *entry = *_member;
-
-  if (fast_memcmp(addr->s6_addr, entry->ip_low.s6_addr,
-             sizeof(struct in6_addr)) < 0)
-    return -1;
-  else if (fast_memcmp(addr->s6_addr, entry->ip_high.s6_addr,
-                  sizeof(struct in6_addr)) > 0)
-    return 1;
-  else
-    return 0;
+  n_v3_ns_requests[country] += 1;
 }
 
 /** Return 1 if we should collect geoip stats on bridge users, and
@@ -303,208 +114,6 @@ should_record_bridge_info(const or_options_t *options)
   return options->BridgeRelay && options->BridgeRecordUsageByCountry;
 }
 
-/** Set up a new list of geoip countries with no countries (yet) set in it,
- * except for the unknown country.
- */
-static void
-init_geoip_countries(void)
-{
-  geoip_country_t *geoip_unresolved;
-  geoip_countries = smartlist_new();
-  /* Add a geoip_country_t for requests that could not be resolved to a
-   * country as first element (index 0) to geoip_countries. */
-  geoip_unresolved = tor_malloc_zero(sizeof(geoip_country_t));
-  strlcpy(geoip_unresolved->countrycode, "??",
-          sizeof(geoip_unresolved->countrycode));
-  smartlist_add(geoip_countries, geoip_unresolved);
-  country_idxplus1_by_lc_code = strmap_new();
-  strmap_set_lc(country_idxplus1_by_lc_code, "??", (void*)(1));
-}
-
-/** Clear appropriate GeoIP database, based on <b>family</b>, and
- * reload it from the file <b>filename</b>. Return 0 on success, -1 on
- * failure.
- *
- * Recognized line formats for IPv4 are:
- *   INTIPLOW,INTIPHIGH,CC
- * and
- *   "INTIPLOW","INTIPHIGH","CC","CC3","COUNTRY NAME"
- * where INTIPLOW and INTIPHIGH are IPv4 addresses encoded as 4-byte unsigned
- * integers, and CC is a country code.
- *
- * Recognized line format for IPv6 is:
- *   IPV6LOW,IPV6HIGH,CC
- * where IPV6LOW and IPV6HIGH are IPv6 addresses and CC is a country code.
- *
- * It also recognizes, and skips over, blank lines and lines that start
- * with '#' (comments).
- */
-int
-geoip_load_file(sa_family_t family, const char *filename)
-{
-  FILE *f;
-  const char *msg = "";
-  const or_options_t *options = get_options();
-  int severity = options_need_geoip_info(options, &msg) ? LOG_WARN : LOG_INFO;
-  crypto_digest_t *geoip_digest_env = NULL;
-
-  tor_assert(family == AF_INET || family == AF_INET6);
-
-  if (!(f = tor_fopen_cloexec(filename, "r"))) {
-    log_fn(severity, LD_GENERAL, "Failed to open GEOIP file %s.  %s",
-           filename, msg);
-    return -1;
-  }
-  if (!geoip_countries)
-    init_geoip_countries();
-
-  if (family == AF_INET) {
-    if (geoip_ipv4_entries) {
-      SMARTLIST_FOREACH(geoip_ipv4_entries, geoip_ipv4_entry_t *, e,
-                        tor_free(e));
-      smartlist_free(geoip_ipv4_entries);
-    }
-    geoip_ipv4_entries = smartlist_new();
-  } else { /* AF_INET6 */
-    if (geoip_ipv6_entries) {
-      SMARTLIST_FOREACH(geoip_ipv6_entries, geoip_ipv6_entry_t *, e,
-                        tor_free(e));
-      smartlist_free(geoip_ipv6_entries);
-    }
-    geoip_ipv6_entries = smartlist_new();
-  }
-  geoip_digest_env = crypto_digest_new();
-
-  log_notice(LD_GENERAL, "Parsing GEOIP %s file %s.",
-             (family == AF_INET) ? "IPv4" : "IPv6", filename);
-  while (!feof(f)) {
-    char buf[512];
-    if (fgets(buf, (int)sizeof(buf), f) == NULL)
-      break;
-    crypto_digest_add_bytes(geoip_digest_env, buf, strlen(buf));
-    /* FFFF track full country name. */
-    geoip_parse_entry(buf, family);
-  }
-  /*XXXX abort and return -1 if no entries/illformed?*/
-  fclose(f);
-
-  /* Sort list and remember file digests so that we can include it in
-   * our extra-info descriptors. */
-  if (family == AF_INET) {
-    smartlist_sort(geoip_ipv4_entries, geoip_ipv4_compare_entries_);
-    /* Okay, now we need to maybe change our mind about what is in
-     * which country. We do this for IPv4 only since that's what we
-     * store in node->country. */
-    refresh_all_country_info();
-    crypto_digest_get_digest(geoip_digest_env, geoip_digest, DIGEST_LEN);
-  } else {
-    /* AF_INET6 */
-    smartlist_sort(geoip_ipv6_entries, geoip_ipv6_compare_entries_);
-    crypto_digest_get_digest(geoip_digest_env, geoip6_digest, DIGEST_LEN);
-  }
-  crypto_digest_free(geoip_digest_env);
-
-  return 0;
-}
-
-/** Given an IP address in host order, return a number representing the
- * country to which that address belongs, -1 for "No geoip information
- * available", or 0 for the 'unknown country'.  The return value will always
- * be less than geoip_get_n_countries().  To decode it, call
- * geoip_get_country_name().
- */
-STATIC int
-geoip_get_country_by_ipv4(uint32_t ipaddr)
-{
-  geoip_ipv4_entry_t *ent;
-  if (!geoip_ipv4_entries)
-    return -1;
-  ent = smartlist_bsearch(geoip_ipv4_entries, &ipaddr,
-                          geoip_ipv4_compare_key_to_entry_);
-  return ent ? (int)ent->country : 0;
-}
-
-/** Given an IPv6 address, return a number representing the country to
- * which that address belongs, -1 for "No geoip information available", or
- * 0 for the 'unknown country'.  The return value will always be less than
- * geoip_get_n_countries().  To decode it, call geoip_get_country_name().
- */
-STATIC int
-geoip_get_country_by_ipv6(const struct in6_addr *addr)
-{
-  geoip_ipv6_entry_t *ent;
-
-  if (!geoip_ipv6_entries)
-    return -1;
-  ent = smartlist_bsearch(geoip_ipv6_entries, addr,
-                          geoip_ipv6_compare_key_to_entry_);
-  return ent ? (int)ent->country : 0;
-}
-
-/** Given an IP address, return a number representing the country to which
- * that address belongs, -1 for "No geoip information available", or 0 for
- * the 'unknown country'.  The return value will always be less than
- * geoip_get_n_countries().  To decode it, call geoip_get_country_name().
- */
-MOCK_IMPL(int,
-geoip_get_country_by_addr,(const tor_addr_t *addr))
-{
-  if (tor_addr_family(addr) == AF_INET) {
-    return geoip_get_country_by_ipv4(tor_addr_to_ipv4h(addr));
-  } else if (tor_addr_family(addr) == AF_INET6) {
-    return geoip_get_country_by_ipv6(tor_addr_to_in6(addr));
-  } else {
-    return -1;
-  }
-}
-
-/** Return the number of countries recognized by the GeoIP country list. */
-MOCK_IMPL(int,
-geoip_get_n_countries,(void))
-{
-  if (!geoip_countries)
-    init_geoip_countries();
-  return (int) smartlist_len(geoip_countries);
-}
-
-/** Return the two-letter country code associated with the number <b>num</b>,
- * or "??" for an unknown value. */
-const char *
-geoip_get_country_name(country_t num)
-{
-  if (geoip_countries && num >= 0 && num < smartlist_len(geoip_countries)) {
-    geoip_country_t *c = smartlist_get(geoip_countries, num);
-    return c->countrycode;
-  } else
-    return "??";
-}
-
-/** Return true iff we have loaded a GeoIP database.*/
-MOCK_IMPL(int,
-geoip_is_loaded,(sa_family_t family))
-{
-  tor_assert(family == AF_INET || family == AF_INET6);
-  if (geoip_countries == NULL)
-    return 0;
-  if (family == AF_INET)
-    return geoip_ipv4_entries != NULL;
-  else                          /* AF_INET6 */
-    return geoip_ipv6_entries != NULL;
-}
-
-/** Return the hex-encoded SHA1 digest of the loaded GeoIP file. The
- * result does not need to be deallocated, but will be overwritten by the
- * next call of hex_str(). */
-const char *
-geoip_db_digest(sa_family_t family)
-{
-  tor_assert(family == AF_INET || family == AF_INET6);
-  if (family == AF_INET)
-    return hex_str(geoip_digest, DIGEST_LEN);
-  else                          /* AF_INET6 */
-    return hex_str(geoip6_digest, DIGEST_LEN);
-}
-
 /** Largest allowable value for last_seen_in_minutes.  (It's a 30-bit field,
  * so it can hold up to (1u<<30)-1, or 0x3fffffffu.
  */
@@ -541,6 +150,9 @@ HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
 HT_GENERATE2(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
              clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
 
+#define clientmap_entry_free(ent) \
+  FREE_AND_NULL(clientmap_entry_t, clientmap_entry_free_, ent)
+
 /** Return the size of a client map entry. */
 static inline size_t
 clientmap_entry_size(const clientmap_entry_t *ent)
@@ -552,7 +164,7 @@ clientmap_entry_size(const clientmap_entry_t *ent)
 
 /** Free all storage held by <b>ent</b>. */
 static void
-clientmap_entry_free(clientmap_entry_t *ent)
+clientmap_entry_free_(clientmap_entry_t *ent)
 {
   if (!ent)
     return;
@@ -624,8 +236,7 @@ geoip_note_client_seen(geoip_client_action_t action,
     /* Only remember statistics if the DoS mitigation subsystem is enabled. If
      * not, only if as entry guard or as bridge. */
     if (!dos_enabled()) {
-      if (!options->EntryStatistics &&
-          (!(options->BridgeRelay && options->BridgeRecordUsageByCountry))) {
+      if (!options->EntryStatistics && !should_record_bridge_info(options)) {
         return;
       }
     }
@@ -654,10 +265,10 @@ geoip_note_client_seen(geoip_client_action_t action,
     int country_idx = geoip_get_country_by_addr(addr);
     if (country_idx < 0)
       country_idx = 0; /** unresolved requests are stored at index 0. */
-    if (country_idx >= 0 && country_idx < smartlist_len(geoip_countries)) {
-      geoip_country_t *country = smartlist_get(geoip_countries, country_idx);
-      ++country->n_v3_ns_requests;
+    IF_BUG_ONCE(country_idx > COUNTRY_MAX) {
+      return;
     }
+    increment_v3_ns_request((country_t) country_idx);
   }
 }
 
@@ -1017,7 +628,7 @@ geoip_get_transport_history(void)
 
     /* If it's the first time we see this transport, note it. */
     if (val == 1)
-      smartlist_add(transports_used, tor_strdup(transport_name));
+      smartlist_add_strdup(transports_used, transport_name);
 
     log_debug(LD_GENERAL, "Client from '%s' with transport '%s'. "
               "I've now seen %d clients.",
@@ -1034,12 +645,12 @@ geoip_get_transport_history(void)
     void *transport_count_ptr = strmap_get(transport_counts, transport_name);
     uintptr_t transport_count = (uintptr_t) transport_count_ptr;
 
-    log_debug(LD_GENERAL, "We got "U64_FORMAT" clients with transport '%s'.",
-              U64_PRINTF_ARG((uint64_t)transport_count), transport_name);
+    log_debug(LD_GENERAL, "We got %"PRIu64" clients with transport '%s'.",
+              ((uint64_t)transport_count), transport_name);
 
-    smartlist_add_asprintf(string_chunks, "%s="U64_FORMAT,
+    smartlist_add_asprintf(string_chunks, "%s=%"PRIu64,
                            transport_name,
-                           U64_PRINTF_ARG(round_uint64_to_next_multiple_of(
+                           (round_uint64_to_next_multiple_of(
                                                (uint64_t)transport_count,
                                                granularity)));
   } SMARTLIST_FOREACH_END(transport_name);
@@ -1067,9 +678,9 @@ static char *
 geoip_get_dirreq_history(dirreq_type_t type)
 {
   char *result = NULL;
+  buf_t *buf = NULL;
   smartlist_t *dirreq_completed = NULL;
   uint32_t complete = 0, timeouts = 0, running = 0;
-  int bufsize = 1024, written;
   dirreq_map_entry_t **ptr, **next;
   struct timeval now;
 
@@ -1102,13 +713,9 @@ geoip_get_dirreq_history(dirreq_type_t type)
                                               DIR_REQ_GRANULARITY);
   running = round_uint32_to_next_multiple_of(running,
                                              DIR_REQ_GRANULARITY);
-  result = tor_malloc_zero(bufsize);
-  written = tor_snprintf(result, bufsize, "complete=%u,timeout=%u,"
-                         "running=%u", complete, timeouts, running);
-  if (written < 0) {
-    tor_free(result);
-    goto done;
-  }
+  buf = buf_new_with_capacity(1024);
+  buf_add_printf(buf, "complete=%u,timeout=%u,"
+                 "running=%u", complete, timeouts, running);
 
 #define MIN_DIR_REQ_RESPONSES 16
   if (complete >= MIN_DIR_REQ_RESPONSES) {
@@ -1119,17 +726,17 @@ geoip_get_dirreq_history(dirreq_type_t type)
     dltimes = tor_calloc(complete, sizeof(uint32_t));
     SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) {
       uint32_t bytes_per_second;
-      uint32_t time_diff = (uint32_t) tv_mdiff(&ent->request_time,
+      uint32_t time_diff_ = (uint32_t) tv_mdiff(&ent->request_time,
                                                &ent->completion_time);
-      if (time_diff == 0)
-        time_diff = 1; /* Avoid DIV/0; "instant" answers are impossible
+      if (time_diff_ == 0)
+        time_diff_ = 1; /* Avoid DIV/0; "instant" answers are impossible
                         * by law of nature or something, but a millisecond
                         * is a bit greater than "instantly" */
-      bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff);
+      bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff_);
       dltimes[ent_sl_idx] = bytes_per_second;
     } SMARTLIST_FOREACH_END(ent);
     median_uint32(dltimes, complete); /* sorts as a side effect. */
-    written = tor_snprintf(result + written, bufsize - written,
+    buf_add_printf(buf,
                            ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u,"
                            "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u",
                            dltimes[0],
@@ -1145,14 +752,15 @@ geoip_get_dirreq_history(dirreq_type_t type)
                            dltimes[8*complete/10-1],
                            dltimes[9*complete/10-1],
                            dltimes[complete-1]);
-    if (written<0)
-      tor_free(result);
     tor_free(dltimes);
   }
- done:
+
+  result = buf_extract(buf, NULL);
+
   SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent,
                     tor_free(ent));
   smartlist_free(dirreq_completed);
+  buf_free(buf);
   return result;
 }
 
@@ -1272,14 +880,14 @@ geoip_get_request_history(void)
   char *result;
   unsigned granularity = IP_GRANULARITY;
 
-  if (!geoip_countries)
-    return NULL;
-
   entries = smartlist_new();
-  SMARTLIST_FOREACH_BEGIN(geoip_countries, geoip_country_t *, c) {
+  SMARTLIST_FOREACH_BEGIN(geoip_get_countries(), const geoip_country_t *, c) {
       uint32_t tot = 0;
       c_hist_t *ent;
-      tot = c->n_v3_ns_requests;
+      if ((size_t)c_sl_idx < n_v3_ns_requests_len)
+        tot = n_v3_ns_requests[c_sl_idx];
+      else
+        tot = 0;
       if (!tot)
         continue;
       ent = tor_malloc_zero(sizeof(c_hist_t));
@@ -1316,9 +924,8 @@ geoip_dirreq_stats_init(time_t now)
 void
 geoip_reset_dirreq_stats(time_t now)
 {
-  SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, {
-      c->n_v3_ns_requests = 0;
-  });
+  memset(n_v3_ns_requests, 0,
+         n_v3_ns_requests_len * sizeof(uint32_t));
   {
     clientmap_entry_t **ent, **next, *this;
     for (ent = HT_START(clientmap, &client_history); ent != NULL;
@@ -1790,65 +1397,9 @@ geoip_entry_stats_write(time_t now)
   return start_of_entry_stats_interval + WRITE_STATS_INTERVAL;
 }
 
-/** Helper used to implement GETINFO ip-to-country/... controller command. */
-int
-getinfo_helper_geoip(control_connection_t *control_conn,
-                     const char *question, char **answer,
-                     const char **errmsg)
-{
-  (void)control_conn;
-  if (!strcmpstart(question, "ip-to-country/")) {
-    int c;
-    sa_family_t family;
-    tor_addr_t addr;
-    question += strlen("ip-to-country/");
-    family = tor_addr_parse(&addr, question);
-    if (family != AF_INET && family != AF_INET6) {
-      *errmsg = "Invalid address family";
-      return -1;
-    }
-    if (!geoip_is_loaded(family)) {
-      *errmsg = "GeoIP data not loaded";
-      return -1;
-    }
-    if (family == AF_INET)
-      c = geoip_get_country_by_ipv4(tor_addr_to_ipv4h(&addr));
-    else                      /* AF_INET6 */
-      c = geoip_get_country_by_ipv6(tor_addr_to_in6(&addr));
-    *answer = tor_strdup(geoip_get_country_name(c));
-  }
-  return 0;
-}
-
-/** Release all storage held by the GeoIP databases and country list. */
-STATIC void
-clear_geoip_db(void)
-{
-  if (geoip_countries) {
-    SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, tor_free(c));
-    smartlist_free(geoip_countries);
-  }
-
-  strmap_free(country_idxplus1_by_lc_code, NULL);
-  if (geoip_ipv4_entries) {
-    SMARTLIST_FOREACH(geoip_ipv4_entries, geoip_ipv4_entry_t *, ent,
-                      tor_free(ent));
-    smartlist_free(geoip_ipv4_entries);
-  }
-  if (geoip_ipv6_entries) {
-    SMARTLIST_FOREACH(geoip_ipv6_entries, geoip_ipv6_entry_t *, ent,
-                      tor_free(ent));
-    smartlist_free(geoip_ipv6_entries);
-  }
-  geoip_countries = NULL;
-  country_idxplus1_by_lc_code = NULL;
-  geoip_ipv4_entries = NULL;
-  geoip_ipv6_entries = NULL;
-}
-
 /** Release all storage held in this file. */
 void
-geoip_free_all(void)
+geoip_stats_free_all(void)
 {
   {
     clientmap_entry_t **ent, **next, *this;
@@ -1869,7 +1420,6 @@ geoip_free_all(void)
     HT_CLEAR(dirreqmap, &dirreq_map);
   }
 
-  clear_geoip_db();
   tor_free(bridge_stats_extrainfo);
+  tor_free(n_v3_ns_requests);
 }
-
diff --git a/src/or/geoip.h b/src/feature/stats/geoip_stats.h
index c8ea9f85ea..2fc62b5466 100644
--- a/src/or/geoip.h
+++ b/src/feature/stats/geoip_stats.h
@@ -1,26 +1,76 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file geoip.h
- * \brief Header file for geoip.c.
+ * \file geoip_stats.h
+ * \brief Header file for geoip_stats.c.
  **/
 
-#ifndef TOR_GEOIP_H
-#define TOR_GEOIP_H
+#ifndef TOR_GEOIP_STATS_H
+#define TOR_GEOIP_STATS_H
 
-#include "testsupport.h"
-#include "dos.h"
+#include "core/or/dos.h"
 
-#ifdef GEOIP_PRIVATE
-STATIC int geoip_parse_entry(const char *line, sa_family_t family);
-STATIC int geoip_get_country_by_ipv4(uint32_t ipaddr);
-STATIC int geoip_get_country_by_ipv6(const struct in6_addr *addr);
-STATIC void clear_geoip_db(void);
-#endif
+/** Indicates an action that we might be noting geoip statistics on.
+ * Note that if we're noticing CONNECT, we're a bridge, and if we're noticing
+ * the others, we're not.
+ */
+typedef enum {
+  /** We've noticed a connection as a bridge relay or entry guard. */
+  GEOIP_CLIENT_CONNECT = 0,
+  /** We've served a networkstatus consensus as a directory server. */
+  GEOIP_CLIENT_NETWORKSTATUS = 1,
+} geoip_client_action_t;
+/** Indicates either a positive reply or a reason for rejectng a network
+ * status request that will be included in geoip statistics. */
+typedef enum {
+  /** Request is answered successfully. */
+  GEOIP_SUCCESS = 0,
+  /** V3 network status is not signed by a sufficient number of requested
+   * authorities. */
+  GEOIP_REJECT_NOT_ENOUGH_SIGS = 1,
+  /** Requested network status object is unavailable. */
+  GEOIP_REJECT_UNAVAILABLE = 2,
+  /** Requested network status not found. */
+  GEOIP_REJECT_NOT_FOUND = 3,
+  /** Network status has not been modified since If-Modified-Since time. */
+  GEOIP_REJECT_NOT_MODIFIED = 4,
+  /** Directory is busy. */
+  GEOIP_REJECT_BUSY = 5,
+} geoip_ns_response_t;
+#define GEOIP_NS_RESPONSE_NUM 6
+
+/** Directory requests that we are measuring can be either direct or
+ * tunneled. */
+typedef enum {
+  DIRREQ_DIRECT = 0,
+  DIRREQ_TUNNELED = 1,
+} dirreq_type_t;
+
+/** Possible states for either direct or tunneled directory requests that
+ * are relevant for determining network status download times. */
+typedef enum {
+  /** Found that the client requests a network status; applies to both
+   * direct and tunneled requests; initial state of a request that we are
+   * measuring. */
+  DIRREQ_IS_FOR_NETWORK_STATUS = 0,
+  /** Finished writing a network status to the directory connection;
+   * applies to both direct and tunneled requests; completes a direct
+   * request. */
+  DIRREQ_FLUSHING_DIR_CONN_FINISHED = 1,
+  /** END cell sent to circuit that initiated a tunneled request. */
+  DIRREQ_END_CELL_SENT = 2,
+  /** Flushed last cell from queue of the circuit that initiated a
+    * tunneled request to the outbuf of the OR connection. */
+  DIRREQ_CIRC_QUEUE_FLUSHED = 3,
+  /** Flushed last byte from buffer of the channel belonging to the
+    * circuit that initiated a tunneled request; completes a tunneled
+    * request. */
+  DIRREQ_CHANNEL_BUFFER_FLUSHED = 4
+} dirreq_state_t;
 
 /** Entry in a map from IP address to the last time we've seen an incoming
  * connection from that IP address. Used by bridges only to track which
@@ -45,13 +95,6 @@ typedef struct clientmap_entry_t {
 } clientmap_entry_t;
 
 int should_record_bridge_info(const or_options_t *options);
-int geoip_load_file(sa_family_t family, const char *filename);
-MOCK_DECL(int, geoip_get_country_by_addr, (const tor_addr_t *addr));
-MOCK_DECL(int, geoip_get_n_countries, (void));
-const char *geoip_get_country_name(country_t num);
-MOCK_DECL(int, geoip_is_loaded, (sa_family_t family));
-const char *geoip_db_digest(sa_family_t family);
-MOCK_DECL(country_t, geoip_get_country, (const char *countrycode));
 
 void geoip_note_client_seen(geoip_client_action_t action,
                             const tor_addr_t *addr, const char *transport_name,
@@ -68,10 +111,7 @@ char *geoip_get_transport_history(void);
 int geoip_get_client_history(geoip_client_action_t action,
                              char **country_str, char **ipver_str);
 char *geoip_get_request_history(void);
-int getinfo_helper_geoip(control_connection_t *control_conn,
-                         const char *question, char **answer,
-                         const char **errmsg);
-void geoip_free_all(void);
+void geoip_stats_free_all(void);
 
 void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size,
                         dirreq_type_t type);
@@ -96,5 +136,4 @@ const char *geoip_get_bridge_stats_extrainfo(time_t);
 char *geoip_get_bridge_stats_controller(time_t);
 char *format_client_stats_heartbeat(time_t now);
 
-#endif
-
+#endif /* !defined(TOR_GEOIP_STATS_H) */
diff --git a/src/feature/stats/predict_ports.c b/src/feature/stats/predict_ports.c
new file mode 100644
index 0000000000..3cbba2c831
--- /dev/null
+++ b/src/feature/stats/predict_ports.c
@@ -0,0 +1,313 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file predict_ports.c
+ * \brief Remember what ports we've needed so we can have circuits ready.
+ *
+ * Predicted ports are used by clients to remember how long it's been
+ * since they opened an exit connection to each given target
+ * port. Clients use this information in order to try to keep circuits
+ * open to exit nodes that can connect to the ports that they care
+ * about.  (The predicted ports mechanism also handles predicted circuit
+ * usage that _isn't_ port-specific, such as resolves, internal circuits,
+ * and so on.)
+ **/
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/or/channelpadding.h"
+#include "core/or/circuituse.h"
+#include "feature/relay/routermode.h"
+#include "feature/relay/selftest.h"
+#include "feature/stats/predict_ports.h"
+#include "lib/container/bitarray.h"
+#include "lib/time/tvdiff.h"
+
+static size_t predicted_ports_total_alloc = 0;
+
+static void predicted_ports_alloc(void);
+
+/** A single predicted port: used to remember which ports we've made
+ * connections to, so that we can try to keep making circuits that can handle
+ * those ports. */
+typedef struct predicted_port_t {
+  /** The port we connected to */
+  uint16_t port;
+  /** The time at which we last used it */
+  time_t time;
+} predicted_port_t;
+
+/** A list of port numbers that have been used recently. */
+static smartlist_t *predicted_ports_list=NULL;
+/** How long do we keep predicting circuits? */
+static time_t prediction_timeout=0;
+/** When was the last time we added a prediction entry (HS or port) */
+static time_t last_prediction_add_time=0;
+
+/**
+ * How much time left until we stop predicting circuits?
+ */
+int
+predicted_ports_prediction_time_remaining(time_t now)
+{
+  time_t seconds_waited;
+  time_t seconds_left;
+
+  /* Protect against overflow of return value. This can happen if the clock
+   * jumps backwards in time. Update the last prediction time (aka last
+   * active time) to prevent it. This update is preferable to using monotonic
+   * time because it prevents clock jumps into the past from simply causing
+   * very long idle timeouts while the monotonic time stands still. */
+  seconds_waited = time_diff(last_prediction_add_time, now);
+  if (seconds_waited == TIME_MAX) {
+    last_prediction_add_time = now;
+    seconds_waited = 0;
+  }
+
+  /* Protect against underflow of the return value. This can happen for very
+   * large periods of inactivity/system sleep. */
+  if (seconds_waited > prediction_timeout)
+    return 0;
+
+  seconds_left = time_diff(seconds_waited, prediction_timeout);
+  if (BUG(seconds_left == TIME_MAX))
+    return INT_MAX;
+
+  return (int)(seconds_left);
+}
+
+/** We just got an application request for a connection with
+ * port <b>port</b>. Remember it for the future, so we can keep
+ * some circuits open that will exit to this port.
+ */
+static void
+add_predicted_port(time_t now, uint16_t port)
+{
+  predicted_port_t *pp = tor_malloc(sizeof(predicted_port_t));
+
+  //  If the list is empty, re-randomize predicted ports lifetime
+  if (!any_predicted_circuits(now)) {
+    prediction_timeout =
+     (time_t)channelpadding_get_circuits_available_timeout();
+  }
+
+  last_prediction_add_time = now;
+
+  log_info(LD_CIRC,
+          "New port prediction added. Will continue predictive circ building "
+          "for %d more seconds.",
+          predicted_ports_prediction_time_remaining(now));
+
+  pp->port = port;
+  pp->time = now;
+  predicted_ports_total_alloc += sizeof(*pp);
+  smartlist_add(predicted_ports_list, pp);
+}
+
+/** Remember that <b>port</b> has been asked for as of time <b>now</b>.
+ * This is used for predicting what sorts of streams we'll make in the
+ * future and making exit circuits to anticipate that.
+ */
+void
+rep_hist_note_used_port(time_t now, uint16_t port)
+{
+  tor_assert(predicted_ports_list);
+
+  if (!port) /* record nothing */
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) {
+    if (pp->port == port) {
+      pp->time = now;
+
+      last_prediction_add_time = now;
+      log_info(LD_CIRC,
+               "New port prediction added. Will continue predictive circ "
+               "building for %d more seconds.",
+               predicted_ports_prediction_time_remaining(now));
+      return;
+    }
+  } SMARTLIST_FOREACH_END(pp);
+  /* it's not there yet; we need to add it */
+  add_predicted_port(now, port);
+}
+
+/** Return a newly allocated pointer to a list of uint16_t * for ports that
+ * are likely to be asked for in the near future.
+ */
+smartlist_t *
+rep_hist_get_predicted_ports(time_t now)
+{
+  int predicted_circs_relevance_time;
+  smartlist_t *out = smartlist_new();
+  tor_assert(predicted_ports_list);
+
+  predicted_circs_relevance_time = (int)prediction_timeout;
+
+  /* clean out obsolete entries */
+  SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) {
+    if (pp->time + predicted_circs_relevance_time < now) {
+      log_debug(LD_CIRC, "Expiring predicted port %d", pp->port);
+
+      predicted_ports_total_alloc -= sizeof(predicted_port_t);
+      tor_free(pp);
+      SMARTLIST_DEL_CURRENT(predicted_ports_list, pp);
+    } else {
+      smartlist_add(out, tor_memdup(&pp->port, sizeof(uint16_t)));
+    }
+  } SMARTLIST_FOREACH_END(pp);
+  return out;
+}
+
+/**
+ * Take a list of uint16_t *, and remove every port in the list from the
+ * current list of predicted ports.
+ */
+void
+rep_hist_remove_predicted_ports(const smartlist_t *rmv_ports)
+{
+  /* Let's do this on O(N), not O(N^2). */
+  bitarray_t *remove_ports = bitarray_init_zero(UINT16_MAX);
+  SMARTLIST_FOREACH(rmv_ports, const uint16_t *, p,
+                    bitarray_set(remove_ports, *p));
+  SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) {
+    if (bitarray_is_set(remove_ports, pp->port)) {
+      tor_free(pp);
+      predicted_ports_total_alloc -= sizeof(*pp);
+      SMARTLIST_DEL_CURRENT(predicted_ports_list, pp);
+    }
+  } SMARTLIST_FOREACH_END(pp);
+  bitarray_free(remove_ports);
+}
+
+/** The user asked us to do a resolve. Rather than keeping track of
+ * timings and such of resolves, we fake it for now by treating
+ * it the same way as a connection to port 80. This way we will continue
+ * to have circuits lying around if the user only uses Tor for resolves.
+ */
+void
+rep_hist_note_used_resolve(time_t now)
+{
+  rep_hist_note_used_port(now, 80);
+}
+
+/** The last time at which we needed an internal circ. */
+static time_t predicted_internal_time = 0;
+/** The last time we needed an internal circ with good uptime. */
+static time_t predicted_internal_uptime_time = 0;
+/** The last time we needed an internal circ with good capacity. */
+static time_t predicted_internal_capacity_time = 0;
+
+/** Remember that we used an internal circ at time <b>now</b>. */
+void
+rep_hist_note_used_internal(time_t now, int need_uptime, int need_capacity)
+{
+  // If the list is empty, re-randomize predicted ports lifetime
+  if (!any_predicted_circuits(now)) {
+    prediction_timeout = channelpadding_get_circuits_available_timeout();
+  }
+
+  last_prediction_add_time = now;
+
+  log_info(LD_CIRC,
+          "New port prediction added. Will continue predictive circ building "
+          "for %d more seconds.",
+          predicted_ports_prediction_time_remaining(now));
+
+  predicted_internal_time = now;
+  if (need_uptime)
+    predicted_internal_uptime_time = now;
+  if (need_capacity)
+    predicted_internal_capacity_time = now;
+}
+
+/** Return 1 if we've used an internal circ recently; else return 0. */
+int
+rep_hist_get_predicted_internal(time_t now, int *need_uptime,
+                                int *need_capacity)
+{
+  int predicted_circs_relevance_time;
+
+  predicted_circs_relevance_time = (int)prediction_timeout;
+
+  if (!predicted_internal_time) { /* initialize it */
+    predicted_internal_time = now;
+    predicted_internal_uptime_time = now;
+    predicted_internal_capacity_time = now;
+  }
+  if (predicted_internal_time + predicted_circs_relevance_time < now)
+    return 0; /* too long ago */
+  if (predicted_internal_uptime_time + predicted_circs_relevance_time >= now)
+    *need_uptime = 1;
+  // Always predict that we need capacity.
+  *need_capacity = 1;
+  return 1;
+}
+
+/** Any ports used lately? These are pre-seeded if we just started
+ * up or if we're running a hidden service. */
+int
+any_predicted_circuits(time_t now)
+{
+  int predicted_circs_relevance_time;
+  predicted_circs_relevance_time = (int)prediction_timeout;
+
+  return smartlist_len(predicted_ports_list) ||
+         predicted_internal_time + predicted_circs_relevance_time >= now;
+}
+
+/** Return 1 if we have no need for circuits currently, else return 0. */
+int
+rep_hist_circbuilding_dormant(time_t now)
+{
+  const or_options_t *options = get_options();
+
+  if (any_predicted_circuits(now))
+    return 0;
+
+  /* see if we'll still need to build testing circuits */
+  if (server_mode(options) &&
+      (!check_whether_orport_reachable(options) ||
+       !circuit_enough_testing_circs()))
+    return 0;
+  if (!check_whether_dirport_reachable(options))
+    return 0;
+
+  return 1;
+}
+
+/**
+ * Allocate whatever memory and structs are needed for predicting
+ * which ports will be used. Also seed it with port 80, so we'll build
+ * circuits on start-up.
+ */
+static void
+predicted_ports_alloc(void)
+{
+  predicted_ports_list = smartlist_new();
+}
+
+void
+predicted_ports_init(void)
+{
+  predicted_ports_alloc();
+  add_predicted_port(time(NULL), 443); // Add a port to get us started
+}
+
+/** Free whatever memory is needed for predicting which ports will
+ * be used.
+ */
+void
+predicted_ports_free_all(void)
+{
+  if (!predicted_ports_list)
+    return;
+  predicted_ports_total_alloc -=
+    smartlist_len(predicted_ports_list)*sizeof(predicted_port_t);
+  SMARTLIST_FOREACH(predicted_ports_list, predicted_port_t *,
+                    pp, tor_free(pp));
+  smartlist_free(predicted_ports_list);
+}
diff --git a/src/feature/stats/predict_ports.h b/src/feature/stats/predict_ports.h
new file mode 100644
index 0000000000..272344da2f
--- /dev/null
+++ b/src/feature/stats/predict_ports.h
@@ -0,0 +1,30 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file predict_portst.h
+ * \brief Header file for predict_ports.c.
+ **/
+
+#ifndef TOR_PREDICT_PORTS_H
+#define TOR_PREDICT_PORTS_H
+
+void predicted_ports_init(void);
+void rep_hist_note_used_port(time_t now, uint16_t port);
+smartlist_t *rep_hist_get_predicted_ports(time_t now);
+void rep_hist_remove_predicted_ports(const smartlist_t *rmv_ports);
+void rep_hist_note_used_resolve(time_t now);
+void rep_hist_note_used_internal(time_t now, int need_uptime,
+                                 int need_capacity);
+int rep_hist_get_predicted_internal(time_t now, int *need_uptime,
+                                    int *need_capacity);
+
+int any_predicted_circuits(time_t now);
+int rep_hist_circbuilding_dormant(time_t now);
+int predicted_ports_prediction_time_remaining(time_t now);
+void predicted_ports_free_all(void);
+
+#endif
diff --git a/src/or/rephist.c b/src/feature/stats/rephist.c
index dc86fad1d0..3f560fbce7 100644
--- a/src/or/rephist.c
+++ b/src/feature/stats/rephist.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2018, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -74,24 +74,31 @@
  * (The "rephist" name originally stood for "reputation and history". )
  **/
 
-#include "or.h"
-#include "circuitlist.h"
-#include "circuituse.h"
-#include "config.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "rephist.h"
-#include "router.h"
-#include "routerlist.h"
-#include "ht.h"
+#define REPHIST_PRIVATE
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "app/config/statefile.h"
+#include "core/or/circuitlist.h"
+#include "core/or/connection_or.h"
+#include "feature/dirauth/authmode.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/relay/routermode.h"
+#include "feature/stats/predict_ports.h"
+#include "feature/stats/rephist.h"
+#include "lib/container/order.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/math/laplace.h"
+
+#include "feature/nodelist/networkstatus_st.h"
+#include "core/or/or_circuit_st.h"
+#include "app/config/or_state_st.h"
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
 
 static void bw_arrays_init(void);
-static void predicted_ports_init(void);
-
-typedef struct bw_array_t bw_array_t;
-STATIC uint64_t find_largest_max(bw_array_t *b);
-STATIC void commit_max(bw_array_t *b);
-STATIC void advance_obs(bw_array_t *b);
 
 /** Total number of bytes currently allocated in fields used by rephist.c. */
 uint64_t rephist_total_alloc=0;
@@ -111,38 +118,12 @@ uint32_t rephist_total_num=0;
  * 20X as much as one that ended a month ago, and routers that have had no
  * uptime data for about half a year will get forgotten.) */
 
-/** History of an OR-\>OR link. */
-typedef struct link_history_t {
-  /** When did we start tracking this list? */
-  time_t since;
-  /** When did we most recently note a change to this link */
-  time_t changed;
-  /** How many times did extending from OR1 to OR2 succeed? */
-  unsigned long n_extend_ok;
-  /** How many times did extending from OR1 to OR2 fail? */
-  unsigned long n_extend_fail;
-} link_history_t;
-
 /** History of an OR. */
 typedef struct or_history_t {
   /** When did we start tracking this OR? */
   time_t since;
   /** When did we most recently note a change to this OR? */
   time_t changed;
-  /** How many times did we successfully connect? */
-  unsigned long n_conn_ok;
-  /** How many times did we try to connect and fail?*/
-  unsigned long n_conn_fail;
-  /** How many seconds have we been connected to this OR before
-   * 'up_since'? */
-  unsigned long uptime;
-  /** How many seconds have we been unable to connect to this OR before
-   * 'down_since'? */
-  unsigned long downtime;
-  /** If nonzero, we have been connected since this time. */
-  time_t up_since;
-  /** If nonzero, we have been unable to connect since this time. */
-  time_t down_since;
 
   /** The address at which we most recently connected to this OR
    * successfully. */
@@ -164,12 +145,46 @@ typedef struct or_history_t {
   time_t start_of_downtime;
   unsigned long weighted_uptime;
   unsigned long total_weighted_time;
-
-  /** Map from hex OR2 identity digest to a link_history_t for the link
-   * from this OR to OR2. */
-  digestmap_t *link_history_map;
 } or_history_t;
 
+/**
+ * This structure holds accounting needed to calculate the padding overhead.
+ */
+typedef struct padding_counts_t {
+  /** Total number of cells we have received, including padding */
+  uint64_t read_cell_count;
+  /** Total number of cells we have sent, including padding */
+  uint64_t write_cell_count;
+  /** Total number of CELL_PADDING cells we have received */
+  uint64_t read_pad_cell_count;
+  /** Total number of CELL_PADDING cells we have sent */
+  uint64_t write_pad_cell_count;
+  /** Total number of read cells on padding-enabled conns */
+  uint64_t enabled_read_cell_count;
+  /** Total number of sent cells on padding-enabled conns */
+  uint64_t enabled_write_cell_count;
+  /** Total number of read CELL_PADDING cells on padding-enabled cons */
+  uint64_t enabled_read_pad_cell_count;
+  /** Total number of sent CELL_PADDING cells on padding-enabled cons */
+  uint64_t enabled_write_pad_cell_count;
+  /** Total number of RELAY_DROP cells we have received */
+  uint64_t read_drop_cell_count;
+  /** Total number of RELAY_DROP cells we have sent */
+  uint64_t write_drop_cell_count;
+  /** The maximum number of padding timers we've seen in 24 hours */
+  uint64_t maximum_chanpad_timers;
+  /** When did we first copy padding_current into padding_published? */
+  char first_published_at[ISO_TIME_LEN+1];
+} padding_counts_t;
+
+/** Holds the current values of our padding statistics.
+ * It is not published until it is transferred to padding_published. */
+static padding_counts_t padding_current;
+
+/** Remains fixed for a 24 hour period, and then is replaced
+ * by a redacted copy of padding_current */
+static padding_counts_t padding_published;
+
 /** When did we last multiply all routers' weighted_run_length and
  * total_run_weights by STABILITY_ALPHA? */
 static time_t stability_last_downrated = 0;
@@ -195,7 +210,6 @@ get_or_history(const char* id)
     hist = tor_malloc_zero(sizeof(or_history_t));
     rephist_total_alloc += sizeof(or_history_t);
     rephist_total_num++;
-    hist->link_history_map = digestmap_new();
     hist->since = hist->changed = time(NULL);
     tor_addr_make_unspec(&hist->last_reached_addr);
     digestmap_set(history_map, id, hist);
@@ -203,166 +217,22 @@ get_or_history(const char* id)
   return hist;
 }
 
-/** Return the link_history_t for the link from the first named OR to
- * the second, creating it if necessary. (ORs are identified by
- * identity digest.)
- */
-static link_history_t *
-get_link_history(const char *from_id, const char *to_id)
-{
-  or_history_t *orhist;
-  link_history_t *lhist;
-  orhist = get_or_history(from_id);
-  if (!orhist)
-    return NULL;
-  if (tor_digest_is_zero(to_id))
-    return NULL;
-  lhist = digestmap_get(orhist->link_history_map, to_id);
-  if (!lhist) {
-    lhist = tor_malloc_zero(sizeof(link_history_t));
-    rephist_total_alloc += sizeof(link_history_t);
-    lhist->since = lhist->changed = time(NULL);
-    digestmap_set(orhist->link_history_map, to_id, lhist);
-  }
-  return lhist;
-}
-
-/** Helper: free storage held by a single link history entry. */
-static void
-free_link_history_(void *val)
-{
-  rephist_total_alloc -= sizeof(link_history_t);
-  tor_free(val);
-}
-
 /** Helper: free storage held by a single OR history entry. */
 static void
 free_or_history(void *_hist)
 {
   or_history_t *hist = _hist;
-  digestmap_free(hist->link_history_map, free_link_history_);
   rephist_total_alloc -= sizeof(or_history_t);
   rephist_total_num--;
   tor_free(hist);
 }
 
-/** Update an or_history_t object <b>hist</b> so that its uptime/downtime
- * count is up-to-date as of <b>when</b>.
- */
-static void
-update_or_history(or_history_t *hist, time_t when)
-{
-  tor_assert(hist);
-  if (hist->up_since) {
-    tor_assert(!hist->down_since);
-    hist->uptime += (when - hist->up_since);
-    hist->up_since = when;
-  } else if (hist->down_since) {
-    hist->downtime += (when - hist->down_since);
-    hist->down_since = when;
-  }
-}
-
 /** Initialize the static data structures for tracking history. */
 void
 rep_hist_init(void)
 {
   history_map = digestmap_new();
   bw_arrays_init();
-  predicted_ports_init();
-}
-
-/** Helper: note that we are no longer connected to the router with history
- * <b>hist</b>.  If <b>failed</b>, the connection failed; otherwise, it was
- * closed correctly. */
-static void
-mark_or_down(or_history_t *hist, time_t when, int failed)
-{
-  if (hist->up_since) {
-    hist->uptime += (when - hist->up_since);
-    hist->up_since = 0;
-  }
-  if (failed && !hist->down_since) {
-    hist->down_since = when;
-  }
-}
-
-/** Helper: note that we are connected to the router with history
- * <b>hist</b>. */
-static void
-mark_or_up(or_history_t *hist, time_t when)
-{
-  if (hist->down_since) {
-    hist->downtime += (when - hist->down_since);
-    hist->down_since = 0;
-  }
-  if (!hist->up_since) {
-    hist->up_since = when;
-  }
-}
-
-/** Remember that an attempt to connect to the OR with identity digest
- * <b>id</b> failed at <b>when</b>.
- */
-void
-rep_hist_note_connect_failed(const char* id, time_t when)
-{
-  or_history_t *hist;
-  hist = get_or_history(id);
-  if (!hist)
-    return;
-  ++hist->n_conn_fail;
-  mark_or_down(hist, when, 1);
-  hist->changed = when;
-}
-
-/** Remember that an attempt to connect to the OR with identity digest
- * <b>id</b> succeeded at <b>when</b>.
- */
-void
-rep_hist_note_connect_succeeded(const char* id, time_t when)
-{
-  or_history_t *hist;
-  hist = get_or_history(id);
-  if (!hist)
-    return;
-  ++hist->n_conn_ok;
-  mark_or_up(hist, when);
-  hist->changed = when;
-}
-
-/** Remember that we intentionally closed our connection to the OR
- * with identity digest <b>id</b> at <b>when</b>.
- */
-void
-rep_hist_note_disconnect(const char* id, time_t when)
-{
-  or_history_t *hist;
-  hist = get_or_history(id);
-  if (!hist)
-    return;
-  mark_or_down(hist, when, 0);
-  hist->changed = when;
-}
-
-/** Remember that our connection to the OR with identity digest
- * <b>id</b> had an error and stopped working at <b>when</b>.
- */
-void
-rep_hist_note_connection_died(const char* id, time_t when)
-{
-  or_history_t *hist;
-  if (!id) {
-    /* If conn has no identity, it didn't complete its handshake, or something
-     * went wrong.  Ignore it.
-     */
-    return;
-  }
-  hist = get_or_history(id);
-  if (!hist)
-    return;
-  mark_or_down(hist, when, 1);
-  hist->changed = when;
 }
 
 /** We have just decided that this router with identity digest <b>id</b> is
@@ -501,7 +371,6 @@ rep_hist_make_router_pessimal(const char *id, time_t when)
   tor_assert(hist);
 
   rep_hist_note_router_unreachable(id, when);
-  mark_or_down(hist, when, 1);
 
   hist->weighted_run_length = 0;
   hist->weighted_uptime = 0;
@@ -678,57 +547,17 @@ rep_hist_have_measured_enough_stability(void)
   return started_tracking_stability < time(NULL) - 4*60*60;
 }
 
-/** Remember that we successfully extended from the OR with identity
- * digest <b>from_id</b> to the OR with identity digest
- * <b>to_name</b>.
- */
-void
-rep_hist_note_extend_succeeded(const char *from_id, const char *to_id)
-{
-  link_history_t *hist;
-  /* log_fn(LOG_WARN, "EXTEND SUCCEEDED: %s->%s",from_name,to_name); */
-  hist = get_link_history(from_id, to_id);
-  if (!hist)
-    return;
-  ++hist->n_extend_ok;
-  hist->changed = time(NULL);
-}
-
-/** Remember that we tried to extend from the OR with identity digest
- * <b>from_id</b> to the OR with identity digest <b>to_name</b>, but
- * failed.
- */
-void
-rep_hist_note_extend_failed(const char *from_id, const char *to_id)
-{
-  link_history_t *hist;
-  /* log_fn(LOG_WARN, "EXTEND FAILED: %s->%s",from_name,to_name); */
-  hist = get_link_history(from_id, to_id);
-  if (!hist)
-    return;
-  ++hist->n_extend_fail;
-  hist->changed = time(NULL);
-}
-
 /** Log all the reliability data we have remembered, with the chosen
  * severity.
  */
 void
 rep_hist_dump_stats(time_t now, int severity)
 {
-  digestmap_iter_t *lhist_it;
   digestmap_iter_t *orhist_it;
-  const char *name1, *name2, *digest1, *digest2;
+  const char *name1, *digest1;
   char hexdigest1[HEX_DIGEST_LEN+1];
-  char hexdigest2[HEX_DIGEST_LEN+1];
   or_history_t *or_history;
-  link_history_t *link_history;
-  void *or_history_p, *link_history_p;
-  double uptime;
-  char buffer[2048];
-  size_t len;
-  int ret;
-  unsigned long upt, downt;
+  void *or_history_p;
   const node_t *node;
 
   rep_history_clean(now - get_options()->RephistTrackTime);
@@ -748,52 +577,12 @@ rep_hist_dump_stats(time_t now, int severity)
     else
       name1 = "(unknown)";
     base16_encode(hexdigest1, sizeof(hexdigest1), digest1, DIGEST_LEN);
-    update_or_history(or_history, now);
-    upt = or_history->uptime;
-    downt = or_history->downtime;
     s = get_stability(or_history, now);
     stability = (long)s;
-    if (upt+downt) {
-      uptime = ((double)upt) / (upt+downt);
-    } else {
-      uptime=1.0;
-    }
     tor_log(severity, LD_HIST,
-        "OR %s [%s]: %ld/%ld good connections; uptime %ld/%ld sec (%.2f%%); "
-        "wmtbf %lu:%02lu:%02lu",
+        "OR %s [%s]: wmtbf %lu:%02lu:%02lu",
         name1, hexdigest1,
-        or_history->n_conn_ok, or_history->n_conn_fail+or_history->n_conn_ok,
-        upt, upt+downt, uptime*100.0,
         stability/3600, (stability/60)%60, stability%60);
-
-    if (!digestmap_isempty(or_history->link_history_map)) {
-      strlcpy(buffer, "    Extend attempts: ", sizeof(buffer));
-      len = strlen(buffer);
-      for (lhist_it = digestmap_iter_init(or_history->link_history_map);
-           !digestmap_iter_done(lhist_it);
-           lhist_it = digestmap_iter_next(or_history->link_history_map,
-                                          lhist_it)) {
-        digestmap_iter_get(lhist_it, &digest2, &link_history_p);
-        if ((node = node_get_by_id(digest2)) && node_get_nickname(node))
-          name2 = node_get_nickname(node);
-        else
-          name2 = "(unknown)";
-
-        link_history = (link_history_t*) link_history_p;
-
-        base16_encode(hexdigest2, sizeof(hexdigest2), digest2, DIGEST_LEN);
-        ret = tor_snprintf(buffer+len, 2048-len, "%s [%s](%ld/%ld); ",
-                        name2,
-                        hexdigest2,
-                        link_history->n_extend_ok,
-                        link_history->n_extend_ok+link_history->n_extend_fail);
-        if (ret<0)
-          break;
-        else
-          len += ret;
-      }
-      tor_log(severity, LD_HIST, "%s", buffer);
-    }
   }
 }
 
@@ -805,10 +594,9 @@ rep_history_clean(time_t before)
 {
   int authority = authdir_mode(get_options());
   or_history_t *or_history;
-  link_history_t *link_history;
-  void *or_history_p, *link_history_p;
-  digestmap_iter_t *orhist_it, *lhist_it;
-  const char *d1, *d2;
+  void *or_history_p;
+  digestmap_iter_t *orhist_it;
+  const char *d1;
 
   orhist_it = digestmap_iter_init(history_map);
   while (!digestmap_iter_done(orhist_it)) {
@@ -825,19 +613,6 @@ rep_history_clean(time_t before)
       free_or_history(or_history);
       continue;
     }
-    for (lhist_it = digestmap_iter_init(or_history->link_history_map);
-         !digestmap_iter_done(lhist_it); ) {
-      digestmap_iter_get(lhist_it, &d2, &link_history_p);
-      link_history = link_history_p;
-      if (link_history->changed < before) {
-        lhist_it = digestmap_iter_next_rmv(or_history->link_history_map,
-                                           lhist_it);
-        rephist_total_alloc -= sizeof(link_history_t);
-        tor_free(link_history);
-        continue;
-      }
-      lhist_it = digestmap_iter_next(or_history->link_history_map,lhist_it);
-    }
     orhist_it = digestmap_iter_next(history_map, orhist_it);
   }
 }
@@ -910,9 +685,9 @@ rep_hist_record_mtbf_data(time_t now, int missing_means_down)
     base16_encode(dbuf, sizeof(dbuf), digest, DIGEST_LEN);
 
     if (missing_means_down && hist->start_of_run &&
-        !router_get_by_id_digest(digest)) {
+        !connection_or_digest_is_known_relay(digest)) {
       /* We think this relay is running, but it's not listed in our
-       * routerlist. Somehow it fell out without telling us it went
+       * consensus. Somehow it fell out without telling us it went
        * down. Complain and also correct it. */
       log_info(LD_HIST,
                "Relay '%s' is listed as up in rephist, but it's not in "
@@ -1321,9 +1096,12 @@ bw_array_new(void)
   return b;
 }
 
+#define bw_array_free(val) \
+  FREE_AND_NULL(bw_array_t, bw_array_free_, (val))
+
 /** Free storage held by bandwidth array <b>b</b>. */
 static void
-bw_array_free(bw_array_t *b)
+bw_array_free_(bw_array_t *b)
 {
   if (!b) {
     return;
@@ -1439,9 +1217,9 @@ rep_hist_bandwidth_assess,(void))
   r = find_largest_max(read_array);
   w = find_largest_max(write_array);
   if (r>w)
-    return (int)(U64_TO_DBL(w)/NUM_SECS_ROLLING_MEASURE);
+    return (int)(((double)w)/NUM_SECS_ROLLING_MEASURE);
   else
-    return (int)(U64_TO_DBL(r)/NUM_SECS_ROLLING_MEASURE);
+    return (int)(((double)r)/NUM_SECS_ROLLING_MEASURE);
 }
 
 /** Print the bandwidth history of b (either [dir-]read_array or
@@ -1487,9 +1265,9 @@ rep_hist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b)
       total = cutoff;
 
     if (n==(b->num_maxes_set-1))
-      tor_snprintf(cp, len-(cp-buf), U64_FORMAT, U64_PRINTF_ARG(total));
+      tor_snprintf(cp, len-(cp-buf), "%"PRIu64, (total));
     else
-      tor_snprintf(cp, len-(cp-buf), U64_FORMAT",", U64_PRINTF_ARG(total));
+      tor_snprintf(cp, len-(cp-buf), "%"PRIu64",", (total));
     cp += strlen(cp);
   }
   return cp-buf;
@@ -1601,17 +1379,17 @@ rep_hist_update_bwhist_state_section(or_state_t *state,
   for (j=0; j < b->num_maxes_set; ++j,++i) {
     if (i >= NUM_TOTALS)
       i = 0;
-    smartlist_add_asprintf(*s_values, U64_FORMAT,
-                           U64_PRINTF_ARG(b->totals[i] & ~0x3ff));
+    smartlist_add_asprintf(*s_values, "%"PRIu64,
+                           (b->totals[i] & ~0x3ff));
     maxval = b->maxima[i] / NUM_SECS_ROLLING_MEASURE;
-    smartlist_add_asprintf(*s_maxima, U64_FORMAT,
-                           U64_PRINTF_ARG(maxval & ~0x3ff));
+    smartlist_add_asprintf(*s_maxima, "%"PRIu64,
+                           (maxval & ~0x3ff));
   }
-  smartlist_add_asprintf(*s_values, U64_FORMAT,
-                         U64_PRINTF_ARG(b->total_in_period & ~0x3ff));
+  smartlist_add_asprintf(*s_values, "%"PRIu64,
+                         (b->total_in_period & ~0x3ff));
   maxval = b->max_total / NUM_SECS_ROLLING_MEASURE;
-  smartlist_add_asprintf(*s_maxima, U64_FORMAT,
-                         U64_PRINTF_ARG(maxval & ~0x3ff));
+  smartlist_add_asprintf(*s_maxima, "%"PRIu64,
+                         (maxval & ~0x3ff));
 }
 
 /** Update <b>state</b> with the newest bandwidth history. Done before
@@ -1749,310 +1527,6 @@ rep_hist_load_state(or_state_t *state, char **err)
   return 0;
 }
 
-/*********************************************************************/
-
-/** A single predicted port: used to remember which ports we've made
- * connections to, so that we can try to keep making circuits that can handle
- * those ports. */
-typedef struct predicted_port_t {
-  /** The port we connected to */
-  uint16_t port;
-  /** The time at which we last used it */
-  time_t time;
-} predicted_port_t;
-
-/** A list of port numbers that have been used recently. */
-static smartlist_t *predicted_ports_list=NULL;
-
-/** We just got an application request for a connection with
- * port <b>port</b>. Remember it for the future, so we can keep
- * some circuits open that will exit to this port.
- */
-static void
-add_predicted_port(time_t now, uint16_t port)
-{
-  predicted_port_t *pp = tor_malloc(sizeof(predicted_port_t));
-  pp->port = port;
-  pp->time = now;
-  rephist_total_alloc += sizeof(*pp);
-  smartlist_add(predicted_ports_list, pp);
-}
-
-/** Initialize whatever memory and structs are needed for predicting
- * which ports will be used. Also seed it with port 80, so we'll build
- * circuits on start-up.
- */
-static void
-predicted_ports_init(void)
-{
-  predicted_ports_list = smartlist_new();
-  add_predicted_port(time(NULL), 80); /* add one to kickstart us */
-}
-
-/** Free whatever memory is needed for predicting which ports will
- * be used.
- */
-static void
-predicted_ports_free(void)
-{
-  rephist_total_alloc -=
-    smartlist_len(predicted_ports_list)*sizeof(predicted_port_t);
-  SMARTLIST_FOREACH(predicted_ports_list, predicted_port_t *,
-                    pp, tor_free(pp));
-  smartlist_free(predicted_ports_list);
-}
-
-/** Remember that <b>port</b> has been asked for as of time <b>now</b>.
- * This is used for predicting what sorts of streams we'll make in the
- * future and making exit circuits to anticipate that.
- */
-void
-rep_hist_note_used_port(time_t now, uint16_t port)
-{
-  tor_assert(predicted_ports_list);
-
-  if (!port) /* record nothing */
-    return;
-
-  SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) {
-    if (pp->port == port) {
-      pp->time = now;
-      return;
-    }
-  } SMARTLIST_FOREACH_END(pp);
-  /* it's not there yet; we need to add it */
-  add_predicted_port(now, port);
-}
-
-/** Return a newly allocated pointer to a list of uint16_t * for ports that
- * are likely to be asked for in the near future.
- */
-smartlist_t *
-rep_hist_get_predicted_ports(time_t now)
-{
-  int predicted_circs_relevance_time;
-  smartlist_t *out = smartlist_new();
-  tor_assert(predicted_ports_list);
-  predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime;
-
-  /* clean out obsolete entries */
-  SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) {
-    if (pp->time + predicted_circs_relevance_time < now) {
-      log_debug(LD_CIRC, "Expiring predicted port %d", pp->port);
-
-      rephist_total_alloc -= sizeof(predicted_port_t);
-      tor_free(pp);
-      SMARTLIST_DEL_CURRENT(predicted_ports_list, pp);
-    } else {
-      smartlist_add(out, tor_memdup(&pp->port, sizeof(uint16_t)));
-    }
-  } SMARTLIST_FOREACH_END(pp);
-  return out;
-}
-
-/**
- * Take a list of uint16_t *, and remove every port in the list from the
- * current list of predicted ports.
- */
-void
-rep_hist_remove_predicted_ports(const smartlist_t *rmv_ports)
-{
-  /* Let's do this on O(N), not O(N^2). */
-  bitarray_t *remove_ports = bitarray_init_zero(UINT16_MAX);
-  SMARTLIST_FOREACH(rmv_ports, const uint16_t *, p,
-                    bitarray_set(remove_ports, *p));
-  SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) {
-    if (bitarray_is_set(remove_ports, pp->port)) {
-      tor_free(pp);
-      rephist_total_alloc -= sizeof(*pp);
-      SMARTLIST_DEL_CURRENT(predicted_ports_list, pp);
-    }
-  } SMARTLIST_FOREACH_END(pp);
-  bitarray_free(remove_ports);
-}
-
-/** The user asked us to do a resolve. Rather than keeping track of
- * timings and such of resolves, we fake it for now by treating
- * it the same way as a connection to port 80. This way we will continue
- * to have circuits lying around if the user only uses Tor for resolves.
- */
-void
-rep_hist_note_used_resolve(time_t now)
-{
-  rep_hist_note_used_port(now, 80);
-}
-
-/** The last time at which we needed an internal circ. */
-static time_t predicted_internal_time = 0;
-/** The last time we needed an internal circ with good uptime. */
-static time_t predicted_internal_uptime_time = 0;
-/** The last time we needed an internal circ with good capacity. */
-static time_t predicted_internal_capacity_time = 0;
-
-/** Remember that we used an internal circ at time <b>now</b>. */
-void
-rep_hist_note_used_internal(time_t now, int need_uptime, int need_capacity)
-{
-  predicted_internal_time = now;
-  if (need_uptime)
-    predicted_internal_uptime_time = now;
-  if (need_capacity)
-    predicted_internal_capacity_time = now;
-}
-
-/** Return 1 if we've used an internal circ recently; else return 0. */
-int
-rep_hist_get_predicted_internal(time_t now, int *need_uptime,
-                                int *need_capacity)
-{
-  int predicted_circs_relevance_time;
-  predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime;
-
-  if (!predicted_internal_time) { /* initialize it */
-    predicted_internal_time = now;
-    predicted_internal_uptime_time = now;
-    predicted_internal_capacity_time = now;
-  }
-  if (predicted_internal_time + predicted_circs_relevance_time < now)
-    return 0; /* too long ago */
-  if (predicted_internal_uptime_time + predicted_circs_relevance_time >= now)
-    *need_uptime = 1;
-  // Always predict that we need capacity.
-  *need_capacity = 1;
-  return 1;
-}
-
-/** Any ports used lately? These are pre-seeded if we just started
- * up or if we're running a hidden service. */
-int
-any_predicted_circuits(time_t now)
-{
-  int predicted_circs_relevance_time;
-  predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime;
-
-  return smartlist_len(predicted_ports_list) ||
-         predicted_internal_time + predicted_circs_relevance_time >= now;
-}
-
-/** Return 1 if we have no need for circuits currently, else return 0. */
-int
-rep_hist_circbuilding_dormant(time_t now)
-{
-  const or_options_t *options = get_options();
-
-  if (any_predicted_circuits(now))
-    return 0;
-
-  /* see if we'll still need to build testing circuits */
-  if (server_mode(options) &&
-      (!check_whether_orport_reachable(options) ||
-       !circuit_enough_testing_circs()))
-    return 0;
-  if (!check_whether_dirport_reachable(options))
-    return 0;
-
-  return 1;
-}
-
-/** Structure to track how many times we've done each public key operation. */
-static struct {
-  /** How many directory objects have we signed? */
-  unsigned long n_signed_dir_objs;
-  /** How many routerdescs have we signed? */
-  unsigned long n_signed_routerdescs;
-  /** How many directory objects have we verified? */
-  unsigned long n_verified_dir_objs;
-  /** How many routerdescs have we verified */
-  unsigned long n_verified_routerdescs;
-  /** How many onionskins have we encrypted to build circuits? */
-  unsigned long n_onionskins_encrypted;
-  /** How many onionskins have we decrypted to do circuit build requests? */
-  unsigned long n_onionskins_decrypted;
-  /** How many times have we done the TLS handshake as a client? */
-  unsigned long n_tls_client_handshakes;
-  /** How many times have we done the TLS handshake as a server? */
-  unsigned long n_tls_server_handshakes;
-  /** How many PK operations have we done as a hidden service client? */
-  unsigned long n_rend_client_ops;
-  /** How many PK operations have we done as a hidden service midpoint? */
-  unsigned long n_rend_mid_ops;
-  /** How many PK operations have we done as a hidden service provider? */
-  unsigned long n_rend_server_ops;
-} pk_op_counts = {0,0,0,0,0,0,0,0,0,0,0};
-
-/** Increment the count of the number of times we've done <b>operation</b>. */
-void
-note_crypto_pk_op(pk_op_t operation)
-{
-  switch (operation)
-    {
-    case SIGN_DIR:
-      pk_op_counts.n_signed_dir_objs++;
-      break;
-    case SIGN_RTR:
-      pk_op_counts.n_signed_routerdescs++;
-      break;
-    case VERIFY_DIR:
-      pk_op_counts.n_verified_dir_objs++;
-      break;
-    case VERIFY_RTR:
-      pk_op_counts.n_verified_routerdescs++;
-      break;
-    case ENC_ONIONSKIN:
-      pk_op_counts.n_onionskins_encrypted++;
-      break;
-    case DEC_ONIONSKIN:
-      pk_op_counts.n_onionskins_decrypted++;
-      break;
-    case TLS_HANDSHAKE_C:
-      pk_op_counts.n_tls_client_handshakes++;
-      break;
-    case TLS_HANDSHAKE_S:
-      pk_op_counts.n_tls_server_handshakes++;
-      break;
-    case REND_CLIENT:
-      pk_op_counts.n_rend_client_ops++;
-      break;
-    case REND_MID:
-      pk_op_counts.n_rend_mid_ops++;
-      break;
-    case REND_SERVER:
-      pk_op_counts.n_rend_server_ops++;
-      break;
-    default:
-      log_warn(LD_BUG, "Unknown pk operation %d", operation);
-  }
-}
-
-/** Log the number of times we've done each public/private-key operation. */
-void
-dump_pk_ops(int severity)
-{
-  tor_log(severity, LD_HIST,
-      "PK operations: %lu directory objects signed, "
-      "%lu directory objects verified, "
-      "%lu routerdescs signed, "
-      "%lu routerdescs verified, "
-      "%lu onionskins encrypted, "
-      "%lu onionskins decrypted, "
-      "%lu client-side TLS handshakes, "
-      "%lu server-side TLS handshakes, "
-      "%lu rendezvous client operations, "
-      "%lu rendezvous middle operations, "
-      "%lu rendezvous server operations.",
-      pk_op_counts.n_signed_dir_objs,
-      pk_op_counts.n_verified_dir_objs,
-      pk_op_counts.n_signed_routerdescs,
-      pk_op_counts.n_verified_routerdescs,
-      pk_op_counts.n_onionskins_encrypted,
-      pk_op_counts.n_onionskins_decrypted,
-      pk_op_counts.n_tls_client_handshakes,
-      pk_op_counts.n_tls_server_handshakes,
-      pk_op_counts.n_rend_client_ops,
-      pk_op_counts.n_rend_mid_ops,
-      pk_op_counts.n_rend_server_ops);
-}
-
 /*** Exit port statistics ***/
 
 /* Some constants */
@@ -2209,8 +1683,8 @@ rep_hist_format_exit_stats(time_t now)
                      exit_bytes_written[cur_port],
                      EXIT_STATS_ROUND_UP_BYTES);
       num /= 1024;
-      smartlist_add_asprintf(written_strings, "%d="U64_FORMAT,
-                             cur_port, U64_PRINTF_ARG(num));
+      smartlist_add_asprintf(written_strings, "%d=%"PRIu64,
+                             cur_port, (num));
       other_written -= exit_bytes_written[cur_port];
     }
     if (exit_bytes_read[cur_port] > 0) {
@@ -2218,8 +1692,8 @@ rep_hist_format_exit_stats(time_t now)
                      exit_bytes_read[cur_port],
                      EXIT_STATS_ROUND_UP_BYTES);
       num /= 1024;
-      smartlist_add_asprintf(read_strings, "%d="U64_FORMAT,
-                             cur_port, U64_PRINTF_ARG(num));
+      smartlist_add_asprintf(read_strings, "%d=%"PRIu64,
+                             cur_port, (num));
       other_read -= exit_bytes_read[cur_port];
     }
     if (exit_streams[cur_port] > 0) {
@@ -2235,13 +1709,13 @@ rep_hist_format_exit_stats(time_t now)
   other_written = round_uint64_to_next_multiple_of(other_written,
                   EXIT_STATS_ROUND_UP_BYTES);
   other_written /= 1024;
-  smartlist_add_asprintf(written_strings, "other="U64_FORMAT,
-                         U64_PRINTF_ARG(other_written));
+  smartlist_add_asprintf(written_strings, "other=%"PRIu64,
+                         (other_written));
   other_read = round_uint64_to_next_multiple_of(other_read,
                EXIT_STATS_ROUND_UP_BYTES);
   other_read /= 1024;
-  smartlist_add_asprintf(read_strings, "other="U64_FORMAT,
-                         U64_PRINTF_ARG(other_read));
+  smartlist_add_asprintf(read_strings, "other=%"PRIu64,
+                         (other_read));
   other_streams = round_uint32_to_next_multiple_of(other_streams,
                   EXIT_STATS_ROUND_UP_STREAMS);
   smartlist_add_asprintf(streams_strings, "other=%u", other_streams);
@@ -2501,8 +1975,8 @@ rep_hist_format_buffer_stats(time_t now)
   time_in_queue_strings = smartlist_new();
   for (i = 0; i < SHARES; i++) {
     smartlist_add_asprintf(processed_cells_strings,
-                           U64_FORMAT, !circs_in_share[i] ? 0 :
-                           U64_PRINTF_ARG(processed_cells[i] /
+                           "%"PRIu64, !circs_in_share[i] ? 0 :
+                           (processed_cells[i] /
                            circs_in_share[i]));
   }
   for (i = 0; i < SHARES; i++) {
@@ -2541,7 +2015,7 @@ rep_hist_format_buffer_stats(time_t now)
                processed_cells_string,
                queued_cells_string,
                time_in_queue_string,
-               (number_of_circuits + SHARES - 1) / SHARES);
+               CEIL_DIV(number_of_circuits, SHARES));
   tor_free(processed_cells_string);
   tor_free(queued_cells_string);
   tor_free(time_in_queue_string);
@@ -2720,7 +2194,7 @@ rep_hist_desc_stats_write(time_t now)
 }
 
 /** Called to note that we've served a given descriptor (by
- * digest). Incrememnts the count of descriptors served, and the number
+ * digest). Increments the count of descriptors served, and the number
  * of times we've served this descriptor. */
 void
 rep_hist_note_desc_served(const char * desc)
@@ -2817,7 +2291,7 @@ HT_GENERATE2(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash,
 
 /* DOCDOC bidi_map_free */
 static void
-bidi_map_free(void)
+bidi_map_free_all(void)
 {
   bidi_map_entry_t **ptr, **next, *ent;
   for (ptr = HT_START(bidimap, &bidi_map); ptr; ptr = next) {
@@ -2837,7 +2311,7 @@ rep_hist_reset_conn_stats(time_t now)
   mostly_read = 0;
   mostly_written = 0;
   both_read_and_written = 0;
-  bidi_map_free();
+  bidi_map_free_all();
 }
 
 /** Stop collecting connection stats in a way that we can re-start doing
@@ -3026,9 +2500,12 @@ hs_stats_new(void)
   return new_hs_stats;
 }
 
+#define hs_stats_free(val) \
+  FREE_AND_NULL(hs_stats_t, hs_stats_free_, (val))
+
 /** Free an hs_stats_t structure. */
 static void
-hs_stats_free(hs_stats_t *victim_hs_stats)
+hs_stats_free_(hs_stats_t *victim_hs_stats)
 {
   if (!victim_hs_stats) {
     return;
@@ -3166,14 +2643,14 @@ rep_hist_format_hs_stats(time_t now)
 
   format_iso_time(t, now);
   tor_asprintf(&hs_stats_string, "hidserv-stats-end %s (%d s)\n"
-               "hidserv-rend-relayed-cells "I64_FORMAT" delta_f=%d "
+               "hidserv-rend-relayed-cells %"PRId64" delta_f=%d "
                                            "epsilon=%.2f bin_size=%d\n"
-               "hidserv-dir-onions-seen "I64_FORMAT" delta_f=%d "
+               "hidserv-dir-onions-seen %"PRId64" delta_f=%d "
                                         "epsilon=%.2f bin_size=%d\n",
                t, (unsigned) (now - start_of_hs_stats_interval),
-               I64_PRINTF_ARG(obfuscated_cells_seen), REND_CELLS_DELTA_F,
+               (obfuscated_cells_seen), REND_CELLS_DELTA_F,
                REND_CELLS_EPSILON, REND_CELLS_BIN_SIZE,
-               I64_PRINTF_ARG(obfuscated_onions_seen),
+               (obfuscated_onions_seen),
                ONIONS_SEEN_DELTA_F,
                ONIONS_SEEN_EPSILON, ONIONS_SEEN_BIN_SIZE);
 
@@ -3215,8 +2692,7 @@ rep_hist_hs_stats_write(time_t now)
   return start_of_hs_stats_interval + WRITE_STATS_INTERVAL;
 }
 
-#define MAX_LINK_PROTO_TO_LOG 4
-static uint64_t link_proto_count[MAX_LINK_PROTO_TO_LOG+1][2];
+static uint64_t link_proto_count[MAX_LINK_PROTO+1][2];
 
 /** Note that we negotiated link protocol version <b>link_proto</b>, on
  * a connection that started here iff <b>started_here</b> is true.
@@ -3225,7 +2701,7 @@ void
 rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here)
 {
   started_here = !!started_here; /* force to 0 or 1 */
-  if (link_proto > MAX_LINK_PROTO_TO_LOG) {
+  if (link_proto > MAX_LINK_PROTO) {
     log_warn(LD_BUG, "Can't log link protocol %u", link_proto);
     return;
   }
@@ -3233,30 +2709,188 @@ rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here)
   link_proto_count[link_proto][started_here]++;
 }
 
+/**
+ * Update the maximum count of total pending channel padding timers
+ * in this period.
+ */
+void
+rep_hist_padding_count_timers(uint64_t num_timers)
+{
+  if (num_timers > padding_current.maximum_chanpad_timers) {
+    padding_current.maximum_chanpad_timers = num_timers;
+  }
+}
+
+/**
+ * Count a cell that we sent for padding overhead statistics.
+ *
+ * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be
+ * counted for PADDING_TYPE_TOTAL.
+ */
+void
+rep_hist_padding_count_write(padding_type_t type)
+{
+  switch (type) {
+    case PADDING_TYPE_DROP:
+      padding_current.write_drop_cell_count++;
+      break;
+    case PADDING_TYPE_CELL:
+      padding_current.write_pad_cell_count++;
+      break;
+    case PADDING_TYPE_TOTAL:
+      padding_current.write_cell_count++;
+      break;
+    case PADDING_TYPE_ENABLED_TOTAL:
+      padding_current.enabled_write_cell_count++;
+      break;
+    case PADDING_TYPE_ENABLED_CELL:
+      padding_current.enabled_write_pad_cell_count++;
+      break;
+  }
+}
+
+/**
+ * Count a cell that we've received for padding overhead statistics.
+ *
+ * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be
+ * counted for PADDING_TYPE_TOTAL.
+ */
+void
+rep_hist_padding_count_read(padding_type_t type)
+{
+  switch (type) {
+    case PADDING_TYPE_DROP:
+      padding_current.read_drop_cell_count++;
+      break;
+    case PADDING_TYPE_CELL:
+      padding_current.read_pad_cell_count++;
+      break;
+    case PADDING_TYPE_TOTAL:
+      padding_current.read_cell_count++;
+      break;
+    case PADDING_TYPE_ENABLED_TOTAL:
+      padding_current.enabled_read_cell_count++;
+      break;
+    case PADDING_TYPE_ENABLED_CELL:
+      padding_current.enabled_read_pad_cell_count++;
+      break;
+  }
+}
+
+/**
+ * Reset our current padding statistics. Called once every 24 hours.
+ */
+void
+rep_hist_reset_padding_counts(void)
+{
+  memset(&padding_current, 0, sizeof(padding_current));
+}
+
+/**
+ * Copy our current cell counts into a structure for listing in our
+ * extra-info descriptor. Also perform appropriate rounding and redaction.
+ *
+ * This function is called once every 24 hours.
+ */
+#define MIN_CELL_COUNTS_TO_PUBLISH 1
+#define ROUND_CELL_COUNTS_TO 10000
+void
+rep_hist_prep_published_padding_counts(time_t now)
+{
+  memcpy(&padding_published, &padding_current, sizeof(padding_published));
+
+  if (padding_published.read_cell_count < MIN_CELL_COUNTS_TO_PUBLISH ||
+      padding_published.write_cell_count < MIN_CELL_COUNTS_TO_PUBLISH) {
+    memset(&padding_published, 0, sizeof(padding_published));
+    return;
+  }
+
+  format_iso_time(padding_published.first_published_at, now);
+#define ROUND_AND_SET_COUNT(x) (x) = round_uint64_to_next_multiple_of((x), \
+                                      ROUND_CELL_COUNTS_TO)
+  ROUND_AND_SET_COUNT(padding_published.read_pad_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.write_pad_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.read_drop_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.write_drop_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.write_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.read_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.enabled_read_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.enabled_read_pad_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.enabled_write_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.enabled_write_pad_cell_count);
+#undef ROUND_AND_SET_COUNT
+}
+
+/**
+ * Returns an allocated string for extra-info documents for publishing
+ * padding statistics from the last 24 hour interval.
+ */
+char *
+rep_hist_get_padding_count_lines(void)
+{
+  char *result = NULL;
+
+  if (!padding_published.read_cell_count ||
+          !padding_published.write_cell_count) {
+    return NULL;
+  }
+
+  tor_asprintf(&result, "padding-counts %s (%d s)"
+                        " bin-size=%"PRIu64
+                        " write-drop=%"PRIu64
+                        " write-pad=%"PRIu64
+                        " write-total=%"PRIu64
+                        " read-drop=%"PRIu64
+                        " read-pad=%"PRIu64
+                        " read-total=%"PRIu64
+                        " enabled-read-pad=%"PRIu64
+                        " enabled-read-total=%"PRIu64
+                        " enabled-write-pad=%"PRIu64
+                        " enabled-write-total=%"PRIu64
+                        " max-chanpad-timers=%"PRIu64
+                        "\n",
+               padding_published.first_published_at,
+               REPHIST_CELL_PADDING_COUNTS_INTERVAL,
+               (uint64_t)ROUND_CELL_COUNTS_TO,
+               (padding_published.write_drop_cell_count),
+               (padding_published.write_pad_cell_count),
+               (padding_published.write_cell_count),
+               (padding_published.read_drop_cell_count),
+               (padding_published.read_pad_cell_count),
+               (padding_published.read_cell_count),
+               (padding_published.enabled_read_pad_cell_count),
+               (padding_published.enabled_read_cell_count),
+               (padding_published.enabled_write_pad_cell_count),
+               (padding_published.enabled_write_cell_count),
+               (padding_published.maximum_chanpad_timers)
+               );
+
+  return result;
+}
+
 /** Log a heartbeat message explaining how many connections of each link
  * protocol version we have used.
  */
 void
 rep_hist_log_link_protocol_counts(void)
 {
-  log_notice(LD_HEARTBEAT,
-             "Since startup, we have initiated "
-             U64_FORMAT" v1 connections, "
-             U64_FORMAT" v2 connections, "
-             U64_FORMAT" v3 connections, and "
-             U64_FORMAT" v4 connections; and received "
-             U64_FORMAT" v1 connections, "
-             U64_FORMAT" v2 connections, "
-             U64_FORMAT" v3 connections, and "
-             U64_FORMAT" v4 connections.",
-             U64_PRINTF_ARG(link_proto_count[1][1]),
-             U64_PRINTF_ARG(link_proto_count[2][1]),
-             U64_PRINTF_ARG(link_proto_count[3][1]),
-             U64_PRINTF_ARG(link_proto_count[4][1]),
-             U64_PRINTF_ARG(link_proto_count[1][0]),
-             U64_PRINTF_ARG(link_proto_count[2][0]),
-             U64_PRINTF_ARG(link_proto_count[3][0]),
-             U64_PRINTF_ARG(link_proto_count[4][0]));
+  smartlist_t *lines = smartlist_new();
+
+  for (int i = 1; i <= MAX_LINK_PROTO; i++) {
+     char *line = NULL;
+     tor_asprintf(&line, "initiated %"PRIu64" and received "
+                  "%"PRIu64" v%d connections", link_proto_count[i][1],
+                  link_proto_count[i][0], i);
+     smartlist_add(lines, line);
+  }
+
+  char *log_line = smartlist_join_strings(lines, "; ", 0, NULL);
+
+  log_notice(LD_HEARTBEAT, "Since startup we %s.", log_line);
+
+  SMARTLIST_FOREACH(lines, char *, s, tor_free(s));
+  smartlist_free(lines);
+  tor_free(log_line);
 }
 
 /** Free all storage held by the OR/link history caches, by the
@@ -3282,8 +2916,8 @@ rep_hist_free_all(void)
   tor_free(exit_bytes_read);
   tor_free(exit_bytes_written);
   tor_free(exit_streams);
-  predicted_ports_free();
-  bidi_map_free();
+  predicted_ports_free_all();
+  bidi_map_free_all();
 
   if (circuits_for_buffer_stats) {
     SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, s,
diff --git a/src/or/rephist.h b/src/feature/stats/rephist.h
index 303cd74f7a..3accc8c610 100644
--- a/src/or/rephist.h
+++ b/src/feature/stats/rephist.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2018, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,13 +13,6 @@
 #define TOR_REPHIST_H
 
 void rep_hist_init(void);
-void rep_hist_note_connect_failed(const char* nickname, time_t when);
-void rep_hist_note_connect_succeeded(const char* nickname, time_t when);
-void rep_hist_note_disconnect(const char* nickname, time_t when);
-void rep_hist_note_connection_died(const char* nickname, time_t when);
-void rep_hist_note_extend_succeeded(const char *from_name,
-                                    const char *to_name);
-void rep_hist_note_extend_failed(const char *from_name, const char *to_name);
 void rep_hist_dump_stats(time_t now, int severity);
 void rep_hist_note_bytes_read(uint64_t num_bytes, time_t when);
 void rep_hist_note_bytes_written(uint64_t num_bytes, time_t when);
@@ -48,21 +41,6 @@ double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when);
 long rep_hist_get_weighted_time_known(const char *id, time_t when);
 int rep_hist_have_measured_enough_stability(void);
 
-void rep_hist_note_used_port(time_t now, uint16_t port);
-smartlist_t *rep_hist_get_predicted_ports(time_t now);
-void rep_hist_remove_predicted_ports(const smartlist_t *rmv_ports);
-void rep_hist_note_used_resolve(time_t now);
-void rep_hist_note_used_internal(time_t now, int need_uptime,
-                                 int need_capacity);
-int rep_hist_get_predicted_internal(time_t now, int *need_uptime,
-                                    int *need_capacity);
-
-int any_predicted_circuits(time_t now);
-int rep_hist_circbuilding_dormant(time_t now);
-
-void note_crypto_pk_op(pk_op_t operation);
-void dump_pk_ops(int severity);
-
 void rep_hist_exit_stats_init(time_t now);
 void rep_hist_reset_exit_stats(time_t now);
 void rep_hist_exit_stats_term(void);
@@ -120,4 +98,36 @@ extern int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1];
 extern struct bw_array_t *write_array;
 #endif
 
+#ifdef REPHIST_PRIVATE
+typedef struct bw_array_t bw_array_t;
+STATIC uint64_t find_largest_max(bw_array_t *b);
+STATIC void commit_max(bw_array_t *b);
+STATIC void advance_obs(bw_array_t *b);
 #endif
+
+/**
+ * Represents the type of a cell for padding accounting
+ */
+typedef enum padding_type_t {
+    /** A RELAY_DROP cell */
+    PADDING_TYPE_DROP,
+    /** A CELL_PADDING cell */
+    PADDING_TYPE_CELL,
+    /** Total counts of padding and non-padding together */
+    PADDING_TYPE_TOTAL,
+    /** Total cell counts for all padding-enabled channels */
+    PADDING_TYPE_ENABLED_TOTAL,
+    /** CELL_PADDING counts for all padding-enabled channels */
+    PADDING_TYPE_ENABLED_CELL
+} padding_type_t;
+
+/** The amount of time over which the padding cell counts were counted */
+#define REPHIST_CELL_PADDING_COUNTS_INTERVAL (24*60*60)
+void rep_hist_padding_count_read(padding_type_t type);
+void rep_hist_padding_count_write(padding_type_t type);
+char *rep_hist_get_padding_count_lines(void);
+void rep_hist_reset_padding_counts(void);
+void rep_hist_prep_published_padding_counts(time_t now);
+void rep_hist_padding_count_timers(uint64_t num_timers);
+
+#endif /* !defined(TOR_REPHIST_H) */
diff --git a/src/include.am b/src/include.am
index c468af3649..d2f83da814 100644
--- a/src/include.am
+++ b/src/include.am
@@ -1,9 +1,46 @@
 include src/ext/include.am
+include src/lib/arch/include.am
+include src/lib/err/include.am
+include src/lib/cc/include.am
+include src/lib/ctime/include.am
+include src/lib/compress/include.am
+include src/lib/container/include.am
+include src/lib/crypt_ops/include.am
+include src/lib/defs/include.am
+include src/lib/encoding/include.am
+include src/lib/evloop/include.am
+include src/lib/fdio/include.am
+include src/lib/fs/include.am
+include src/lib/geoip/include.am
+include src/lib/include.libdonna.am
+include src/lib/intmath/include.am
+include src/lib/lock/include.am
+include src/lib/log/include.am
+include src/lib/math/include.am
+include src/lib/memarea/include.am
+include src/lib/meminfo/include.am
+include src/lib/malloc/include.am
+include src/lib/net/include.am
+include src/lib/osinfo/include.am
+include src/lib/process/include.am
+include src/lib/sandbox/include.am
+include src/lib/string/include.am
+include src/lib/smartlist_core/include.am
+include src/lib/term/include.am
+include src/lib/testsupport/include.am
+include src/lib/thread/include.am
+include src/lib/time/include.am
+include src/lib/tls/include.am
+include src/lib/trace/include.am
+include src/lib/wallclock/include.am
 include src/trunnel/include.am
-include src/common/include.am
-include src/or/include.am
+
+include src/core/include.am
+include src/app/include.am
+
+include src/rust/include.am
 include src/test/include.am
 include src/tools/include.am
 include src/win32/include.am
 include src/config/include.am
-
+include src/test/fuzz/include.am
diff --git a/src/lib/arch/.may_include b/src/lib/arch/.may_include
new file mode 100644
index 0000000000..4285c3dcb8
--- /dev/null
+++ b/src/lib/arch/.may_include
@@ -0,0 +1,2 @@
+orconfig.h
+lib/cc/*.h
diff --git a/src/lib/arch/bytes.h b/src/lib/arch/bytes.h
new file mode 100644
index 0000000000..fa82241b28
--- /dev/null
+++ b/src/lib/arch/bytes.h
@@ -0,0 +1,182 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_BYTES_H
+#define TOR_BYTES_H
+
+/**
+ * \file bytes.h
+ *
+ * \brief Inline functions for reading and writing multibyte values from
+ *  the middle of strings, and for manipulating byte order.
+ **/
+
+#include <string.h>
+#include "lib/cc/torint.h"
+
+/* The uint8 variants are defined to make the code more uniform. */
+static inline uint8_t
+get_uint8(const void *cp)
+{
+  return *(const uint8_t*)(cp);
+}
+static inline void
+set_uint8(void *cp, uint8_t v)
+{
+  *(uint8_t*)cp = v;
+}
+
+/**
+ * Read a 16-bit value beginning at <b>cp</b>.  Equivalent to
+ * *(uint16_t*)(cp), but will not cause segfaults on platforms that forbid
+ * unaligned memory access.
+ */
+static inline uint16_t
+get_uint16(const void *cp)
+{
+  uint16_t v;
+  memcpy(&v,cp,2);
+  return v;
+}
+/**
+ * Read a 32-bit value beginning at <b>cp</b>.  Equivalent to
+ * *(uint32_t*)(cp), but will not cause segfaults on platforms that forbid
+ * unaligned memory access.
+ */
+static inline uint32_t
+get_uint32(const void *cp)
+{
+  uint32_t v;
+  memcpy(&v,cp,4);
+  return v;
+}
+/**
+ * Read a 64-bit value beginning at <b>cp</b>.  Equivalent to
+ * *(uint64_t*)(cp), but will not cause segfaults on platforms that forbid
+ * unaligned memory access.
+ */
+static inline uint64_t
+get_uint64(const void *cp)
+{
+  uint64_t v;
+  memcpy(&v,cp,8);
+  return v;
+}
+
+/**
+ * Set a 16-bit value beginning at <b>cp</b> to <b>v</b>. Equivalent to
+ * *(uint16_t*)(cp) = v, but will not cause segfaults on platforms that forbid
+ * unaligned memory access. */
+static inline void
+set_uint16(void *cp, uint16_t v)
+{
+  memcpy(cp,&v,2);
+}
+/**
+ * Set a 32-bit value beginning at <b>cp</b> to <b>v</b>. Equivalent to
+ * *(uint32_t*)(cp) = v, but will not cause segfaults on platforms that forbid
+ * unaligned memory access. */
+static inline void
+set_uint32(void *cp, uint32_t v)
+{
+  memcpy(cp,&v,4);
+}
+/**
+ * Set a 64-bit value beginning at <b>cp</b> to <b>v</b>. Equivalent to
+ * *(uint64_t*)(cp) = v, but will not cause segfaults on platforms that forbid
+ * unaligned memory access. */
+static inline void
+set_uint64(void *cp, uint64_t v)
+{
+  memcpy(cp,&v,8);
+}
+
+#ifdef WORDS_BIGENDIAN
+static inline uint16_t
+tor_htons(uint32_t a)
+{
+  return a;
+}
+
+static inline uint16_t
+tor_ntohs(uint64_t a)
+{
+  return a;
+}
+
+static inline uint32_t
+tor_htonl(uint32_t a)
+{
+  return a;
+}
+
+static inline uint32_t
+tor_ntohl(uint64_t a)
+{
+  return a;
+}
+
+static inline uint64_t
+tor_htonll(uint64_t a)
+{
+  return a;
+}
+
+static inline uint64_t
+tor_ntohll(uint64_t a)
+{
+  return a;
+}
+#else
+static inline uint16_t
+tor_htons(uint16_t a)
+{
+  /* Our compilers will indeed recognize this as bswap. */
+  return
+    ((a & 0x00ff) << 8) |
+    ((a & 0xff00) >> 8);
+}
+
+static inline uint16_t
+tor_ntohs(uint16_t a)
+{
+  return tor_htons(a);
+}
+
+static inline uint32_t
+tor_htonl(uint32_t a)
+{
+  /* Our compilers will indeed recognize this as bswap. */
+  return
+    ((a & 0x000000ff) <<24) |
+    ((a & 0x0000ff00) << 8) |
+    ((a & 0x00ff0000) >> 8) |
+    ((a & 0xff000000) >>24);
+}
+
+static inline uint32_t
+tor_ntohl(uint32_t a)
+{
+  return tor_htonl(a);
+}
+
+/** Return a uint64_t value from <b>a</b> in network byte order. */
+static inline uint64_t
+tor_htonll(uint64_t a)
+{
+  /* Little endian. The worst... */
+  return tor_htonl((uint32_t)(a>>32)) |
+    (((uint64_t)tor_htonl((uint32_t)a))<<32);
+}
+
+/** Return a uint64_t value from <b>a</b> in host byte order. */
+static inline uint64_t
+tor_ntohll(uint64_t a)
+{
+  return tor_htonll(a);
+}
+#endif
+
+#endif
diff --git a/src/lib/arch/include.am b/src/lib/arch/include.am
new file mode 100644
index 0000000000..f92ee9222f
--- /dev/null
+++ b/src/lib/arch/include.am
@@ -0,0 +1,3 @@
+
+noinst_HEADERS += \
+	src/lib/arch/bytes.h
diff --git a/src/lib/cc/.may_include b/src/lib/cc/.may_include
new file mode 100644
index 0000000000..2b06e8519c
--- /dev/null
+++ b/src/lib/cc/.may_include
@@ -0,0 +1 @@
+orconfig.h
diff --git a/src/lib/cc/compat_compiler.h b/src/lib/cc/compat_compiler.h
new file mode 100644
index 0000000000..3a0f307186
--- /dev/null
+++ b/src/lib/cc/compat_compiler.h
@@ -0,0 +1,220 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compat_compiler.h
+ * \brief Utility macros to handle different features and behavior in different
+ *    compilers.
+ **/
+
+#ifndef TOR_COMPAT_COMPILER_H
+#define TOR_COMPAT_COMPILER_H
+
+#include "orconfig.h"
+#include <inttypes.h>
+
+#if defined(__has_feature)
+#  if __has_feature(address_sanitizer)
+/* Some of the fancy glibc strcmp() macros include references to memory that
+ * clang rejects because it is off the end of a less-than-3. Clang hates this,
+ * even though those references never actually happen. */
+#    undef strcmp
+#endif /* __has_feature(address_sanitizer) */
+#endif /* defined(__has_feature) */
+
+#ifndef NULL_REP_IS_ZERO_BYTES
+#error "It seems your platform does not represent NULL as zero. We can't cope."
+#endif
+
+#ifndef DOUBLE_0_REP_IS_ZERO_BYTES
+#error "It seems your platform does not represent 0.0 as zeros. We can't cope."
+#endif
+
+#if 'a'!=97 || 'z'!=122 || 'A'!=65 || ' '!=32
+#error "It seems that you encode characters in something other than ASCII."
+#endif
+
+/* GCC can check printf and scanf types on arbitrary functions. */
+#ifdef __GNUC__
+#define CHECK_PRINTF(formatIdx, firstArg) \
+   __attribute__ ((format(printf, formatIdx, firstArg)))
+#else
+#define CHECK_PRINTF(formatIdx, firstArg)
+#endif /* defined(__GNUC__) */
+#ifdef __GNUC__
+#define CHECK_SCANF(formatIdx, firstArg) \
+   __attribute__ ((format(scanf, formatIdx, firstArg)))
+#else
+#define CHECK_SCANF(formatIdx, firstArg)
+#endif /* defined(__GNUC__) */
+
+/* What GCC do we have? */
+#ifdef __GNUC__
+#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+#else
+#define GCC_VERSION 0
+#endif
+
+/* Temporarily enable and disable warnings. */
+#ifdef __GNUC__
+#  define PRAGMA_STRINGIFY_(s) #s
+#  define PRAGMA_JOIN_STRINGIFY_(a,b) PRAGMA_STRINGIFY_(a ## b)
+/* Support for macro-generated pragmas (c99) */
+#  define PRAGMA_(x) _Pragma (#x)
+#  ifdef __clang__
+#    define PRAGMA_DIAGNOSTIC_(x) PRAGMA_(clang diagnostic x)
+#  else
+#    define PRAGMA_DIAGNOSTIC_(x) PRAGMA_(GCC diagnostic x)
+#  endif
+#  if defined(__clang__) || GCC_VERSION >= 406
+/* we have push/pop support */
+#    define DISABLE_GCC_WARNING(warningopt) \
+          PRAGMA_DIAGNOSTIC_(push) \
+          PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warningopt))
+#    define ENABLE_GCC_WARNING(warningopt) \
+          PRAGMA_DIAGNOSTIC_(pop)
+#else /* !(defined(__clang__) || GCC_VERSION >= 406) */
+/* older version of gcc: no push/pop support. */
+#    define DISABLE_GCC_WARNING(warningopt) \
+         PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warningopt))
+#    define ENABLE_GCC_WARNING(warningopt) \
+         PRAGMA_DIAGNOSTIC_(warning PRAGMA_JOIN_STRINGIFY_(-W,warningopt))
+#endif /* defined(__clang__) || GCC_VERSION >= 406 */
+#else /* !(defined(__GNUC__)) */
+/* not gcc at all */
+# define DISABLE_GCC_WARNING(warning)
+# define ENABLE_GCC_WARNING(warning)
+#endif /* defined(__GNUC__) */
+
+/* inline is __inline on windows. */
+#ifdef _WIN32
+#define inline __inline
+#endif
+
+/* Try to get a reasonable __func__ substitute in place. */
+#if defined(_MSC_VER)
+
+#define __func__ __FUNCTION__
+
+#else
+/* For platforms where autoconf works, make sure __func__ is defined
+ * sanely. */
+#ifndef HAVE_MACRO__func__
+#ifdef HAVE_MACRO__FUNCTION__
+#define __func__ __FUNCTION__
+#elif HAVE_MACRO__FUNC__
+#define __func__ __FUNC__
+#else
+#define __func__ "???"
+#endif /* defined(HAVE_MACRO__FUNCTION__) || ... */
+#endif /* !defined(HAVE_MACRO__func__) */
+#endif /* defined(_MSC_VER) */
+
+#ifdef ENUM_VALS_ARE_SIGNED
+#define ENUM_BF(t) unsigned
+#else
+/** Wrapper for having a bitfield of an enumerated type. Where possible, we
+ * just use the enumerated type (so the compiler can help us and notice
+ * problems), but if enumerated types are unsigned, we must use unsigned,
+ * so that the loss of precision doesn't make large values negative. */
+#define ENUM_BF(t) t
+#endif /* defined(ENUM_VALS_ARE_SIGNED) */
+
+/* GCC has several useful attributes. */
+#if defined(__GNUC__) && __GNUC__ >= 3
+#define ATTR_NORETURN __attribute__((noreturn))
+#define ATTR_CONST __attribute__((const))
+#define ATTR_MALLOC __attribute__((malloc))
+#define ATTR_NORETURN __attribute__((noreturn))
+#define ATTR_WUR __attribute__((warn_unused_result))
+#define ATTR_UNUSED __attribute__ ((unused))
+
+/** Macro: Evaluates to <b>exp</b> and hints the compiler that the value
+ * of <b>exp</b> will probably be true.
+ *
+ * In other words, "if (PREDICT_LIKELY(foo))" is the same as "if (foo)",
+ * except that it tells the compiler that the branch will be taken most of the
+ * time.  This can generate slightly better code with some CPUs.
+ */
+#define PREDICT_LIKELY(exp) __builtin_expect(!!(exp), 1)
+/** Macro: Evaluates to <b>exp</b> and hints the compiler that the value
+ * of <b>exp</b> will probably be false.
+ *
+ * In other words, "if (PREDICT_UNLIKELY(foo))" is the same as "if (foo)",
+ * except that it tells the compiler that the branch will usually not be
+ * taken.  This can generate slightly better code with some CPUs.
+ */
+#define PREDICT_UNLIKELY(exp) __builtin_expect(!!(exp), 0)
+#else /* !(defined(__GNUC__) && __GNUC__ >= 3) */
+#define ATTR_NORETURN
+#define ATTR_CONST
+#define ATTR_MALLOC
+#define ATTR_NORETURN
+#define ATTR_UNUSED
+#define ATTR_WUR
+#define PREDICT_LIKELY(exp) (exp)
+#define PREDICT_UNLIKELY(exp) (exp)
+#endif /* defined(__GNUC__) && __GNUC__ >= 3 */
+
+/** Expands to a syntactically valid empty statement.  */
+#define STMT_NIL (void)0
+
+/** Expands to a syntactically valid empty statement, explicitly (void)ing its
+ * argument. */
+#define STMT_VOID(a) while (0) { (void)(a); }
+
+#ifdef __GNUC__
+/** STMT_BEGIN and STMT_END are used to wrap blocks inside macros so that
+ * the macro can be used as if it were a single C statement. */
+#define STMT_BEGIN (void) ({
+#define STMT_END })
+#elif defined(sun) || defined(__sun__)
+#define STMT_BEGIN if (1) {
+#define STMT_END } else STMT_NIL
+#else
+#define STMT_BEGIN do {
+#define STMT_END } while (0)
+#endif /* defined(__GNUC__) || ... */
+
+/* Some tools (like coccinelle) don't like to see operators as macro
+ * arguments. */
+#define OP_LT <
+#define OP_GT >
+#define OP_GE >=
+#define OP_LE <=
+#define OP_EQ ==
+#define OP_NE !=
+
+#if defined(__MINGW32__) || defined(__MINGW64__)
+#define MINGW_ANY
+#endif
+
+/** Macro: yield a pointer to the field at position <b>off</b> within the
+ * structure <b>st</b>.  Example:
+ * <pre>
+ *   struct a { int foo; int bar; } x;
+ *   off_t bar_offset = offsetof(struct a, bar);
+ *   int *bar_p = STRUCT_VAR_P(&x, bar_offset);
+ *   *bar_p = 3;
+ * </pre>
+ */
+#define STRUCT_VAR_P(st, off) ((void*) ( ((char*)(st)) + (off) ) )
+
+/** Macro: yield a pointer to an enclosing structure given a pointer to
+ * a substructure at offset <b>off</b>. Example:
+ * <pre>
+ *   struct base { ... };
+ *   struct subtype { int x; struct base b; } x;
+ *   struct base *bp = &x.base;
+ *   struct *sp = SUBTYPE_P(bp, struct subtype, b);
+ * </pre>
+ */
+#define SUBTYPE_P(p, subtype, basemember) \
+  ((void*) ( ((char*)(p)) - offsetof(subtype, basemember) ))
+
+/** Macro: Yields the number of elements in array x. */
+#define ARRAY_LENGTH(x) ((sizeof(x)) / sizeof(x[0]))
+
+#endif /* !defined(TOR_COMPAT_H) */
diff --git a/src/lib/cc/include.am b/src/lib/cc/include.am
new file mode 100644
index 0000000000..2ae90f97dd
--- /dev/null
+++ b/src/lib/cc/include.am
@@ -0,0 +1,4 @@
+
+noinst_HEADERS += \
+	src/lib/cc/compat_compiler.h \
+	src/lib/cc/torint.h
diff --git a/src/lib/cc/torint.h b/src/lib/cc/torint.h
new file mode 100644
index 0000000000..c9b2d329f2
--- /dev/null
+++ b/src/lib/cc/torint.h
@@ -0,0 +1,128 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file torint.h
+ *
+ * \brief Integer definitions used throughout Tor.
+ **/
+
+#ifndef TOR_TORINT_H
+#define TOR_TORINT_H
+
+#include "orconfig.h"
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <limits.h>
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_LIMITS_H
+#include <sys/limits.h>
+#endif
+
+#ifndef SIZE_MAX
+#if SIZEOF_SIZE_T == 8
+#define SIZE_MAX UINT64_MAX
+#elif  SIZEOF_SIZE_T == 4
+#define SIZE_MAX UINT32_MAX
+#else
+#error "Can't define SIZE_MAX"
+#endif /* SIZEOF_SIZE_T == 8 || ... */
+#endif /* !defined(SIZE_MAX) */
+
+#ifndef HAVE_SSIZE_T
+#if SIZEOF_SIZE_T == 8
+typedef int64_t ssize_t;
+#elif SIZEOF_SIZE_T == 4
+typedef int32_t ssize_t;
+#else
+#error "Can't define ssize_t."
+#endif /* SIZEOF_SIZE_T == 8 || ... */
+#endif /* !defined(HAVE_SSIZE_T) */
+
+/* This assumes a sane (2's-complement) representation.  But if you
+ * aren't 2's complement, and you don't define LONG_MAX, then you're so
+ * bizarre that I want nothing to do with you. */
+#ifndef USING_TWOS_COMPLEMENT
+#error "Seems that your platform doesn't use 2's complement arithmetic. Argh."
+#endif
+
+#ifndef TIME_MAX
+
+#if (SIZEOF_TIME_T == SIZEOF_INT)
+#define TIME_MAX ((time_t)INT_MAX)
+#elif (SIZEOF_TIME_T == SIZEOF_LONG)
+#define TIME_MAX ((time_t)LONG_MAX)
+#elif (SIZEOF_TIME_T == 8)
+#define TIME_MAX ((time_t)INT64_MAX)
+#else
+#error "Can't define TIME_MAX"
+#endif /* (SIZEOF_TIME_T == SIZEOF_INT) || ... */
+
+#endif /* !defined(TIME_MAX) */
+
+#ifndef TIME_MIN
+
+#if (SIZEOF_TIME_T == SIZEOF_INT)
+#define TIME_MIN ((time_t)INT_MIN)
+#elif (SIZEOF_TIME_T == SIZEOF_LONG)
+#define TIME_MIN ((time_t)LONG_MIN)
+#elif (SIZEOF_TIME_T == 8)
+#define TIME_MIN ((time_t)INT64_MIN)
+#else
+#error "Can't define TIME_MIN"
+#endif /* (SIZEOF_TIME_T == SIZEOF_INT) || ... */
+
+#endif /* !defined(TIME_MIN) */
+
+#ifndef SIZE_MAX
+#if (SIZEOF_SIZE_T == 4)
+#define SIZE_MAX UINT32_MAX
+#elif (SIZEOF_SIZE_T == 8)
+#define SIZE_MAX UINT64_MAX
+#else
+#error "Can't define SIZE_MAX"
+#endif /* (SIZEOF_SIZE_T == 4) || ... */
+#endif /* !defined(SIZE_MAX) */
+
+#ifdef _WIN32
+#  ifdef _WIN64
+#    define TOR_PRIuSZ PRIu64
+#  else
+#    define TOR_PRIuSZ PRIu32
+#  endif
+#else
+#  define TOR_PRIuSZ "zu"
+#endif
+
+#ifdef _WIN32
+#  ifdef _WIN64
+#    define TOR_PRIdSZ PRId64
+#  else
+#    define TOR_PRIdSZ PRId32
+#  endif
+#else
+#  define TOR_PRIdSZ "zd"
+#endif
+
+#ifndef SSIZE_MAX
+#if (SIZEOF_SIZE_T == 4)
+#define SSIZE_MAX INT32_MAX
+#elif (SIZEOF_SIZE_T == 8)
+#define SSIZE_MAX INT64_MAX
+#else
+#error "Can't define SSIZE_MAX"
+#endif /* (SIZEOF_SIZE_T == 4) || ... */
+#endif /* !defined(SSIZE_MAX) */
+
+/** Any ssize_t larger than this amount is likely to be an underflow. */
+#define SSIZE_T_CEILING ((ssize_t)(SSIZE_MAX-16))
+/** Any size_t larger than this amount is likely to be an underflow. */
+#define SIZE_T_CEILING  ((size_t)(SSIZE_MAX-16))
+
+#endif /* !defined(TOR_TORINT_H) */
diff --git a/src/lib/compress/.may_include b/src/lib/compress/.may_include
new file mode 100644
index 0000000000..68fe9f1c54
--- /dev/null
+++ b/src/lib/compress/.may_include
@@ -0,0 +1,12 @@
+orconfig.h
+lib/arch/*.h
+lib/cc/*.h
+lib/compress/*.h
+lib/container/*.h
+lib/ctime/*.h
+lib/intmath/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/string/*.h
+lib/testsupport/*.h
+lib/thread/*.h
diff --git a/src/lib/compress/compress.c b/src/lib/compress/compress.c
new file mode 100644
index 0000000000..95fd73bb32
--- /dev/null
+++ b/src/lib/compress/compress.c
@@ -0,0 +1,681 @@
+/* Copyright (c) 2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress.c
+ * \brief Common compression API implementation.
+ *
+ * This file provides a unified interface to all the compression libraries Tor
+ * knows how to use.
+ **/
+
+#include "orconfig.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include "lib/cc/torint.h"
+
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/arch/bytes.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/compress/compress.h"
+#include "lib/compress/compress_lzma.h"
+#include "lib/compress/compress_none.h"
+#include "lib/compress/compress_zlib.h"
+#include "lib/compress/compress_zstd.h"
+#include "lib/intmath/cmp.h"
+#include "lib/malloc/malloc.h"
+#include "lib/thread/threads.h"
+
+/** Total number of bytes allocated for compression state overhead. */
+static atomic_counter_t total_compress_allocation;
+
+/** @{ */
+/* These macros define the maximum allowable compression factor.  Anything of
+ * size greater than CHECK_FOR_COMPRESSION_BOMB_AFTER is not allowed to
+ * have an uncompression factor (uncompressed size:compressed size ratio) of
+ * any greater than MAX_UNCOMPRESSION_FACTOR.
+ *
+ * Picking a value for MAX_UNCOMPRESSION_FACTOR is a trade-off: we want it to
+ * be small to limit the attack multiplier, but we also want it to be large
+ * enough so that no legitimate document --even ones we might invent in the
+ * future -- ever compresses by a factor of greater than
+ * MAX_UNCOMPRESSION_FACTOR. Within those parameters, there's a reasonably
+ * large range of possible values. IMO, anything over 8 is probably safe; IMO
+ * anything under 50 is probably sufficient.
+ */
+#define MAX_UNCOMPRESSION_FACTOR 25
+#define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64)
+/** @} */
+
+/** Return true if uncompressing an input of size <b>in_size</b> to an input of
+ * size at least <b>size_out</b> looks like a compression bomb. */
+MOCK_IMPL(int,
+tor_compress_is_compression_bomb,(size_t size_in, size_t size_out))
+{
+  if (size_in == 0 || size_out < CHECK_FOR_COMPRESSION_BOMB_AFTER)
+    return 0;
+
+  return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR);
+}
+
+/** Guess the size that <b>in_len</b> will be after compression or
+ * decompression. */
+static size_t
+guess_compress_size(int compress, compress_method_t method,
+                    compression_level_t compression_level,
+                    size_t in_len)
+{
+  // ignore these for now.
+  (void)compression_level;
+  if (method == NO_METHOD) {
+    /* Guess that we'll need an extra byte, to avoid a needless realloc
+     * for nul-termination */
+    return (in_len < SIZE_MAX) ? in_len + 1 : in_len;
+  }
+
+  /* Always guess a factor of 2. */
+  if (compress) {
+    in_len /= 2;
+  } else {
+    if (in_len < SIZE_T_CEILING/2)
+      in_len *= 2;
+  }
+  return MAX(in_len, 1024);
+}
+
+/** Internal function to implement tor_compress/tor_uncompress, depending on
+ * whether <b>compress</b> is set.  All arguments are as for tor_compress or
+ * tor_uncompress. */
+static int
+tor_compress_impl(int compress,
+                  char **out, size_t *out_len,
+                  const char *in, size_t in_len,
+                  compress_method_t method,
+                  compression_level_t compression_level,
+                  int complete_only,
+                  int protocol_warn_level)
+{
+  tor_compress_state_t *stream;
+  int rv;
+
+  stream = tor_compress_new(compress, method, compression_level);
+
+  if (stream == NULL) {
+    log_warn(LD_GENERAL, "NULL stream while %scompressing",
+             compress?"":"de");
+    log_debug(LD_GENERAL, "method: %d level: %d at len: %lu",
+              method, compression_level, (unsigned long)in_len);
+    return -1;
+  }
+
+  size_t in_len_orig = in_len;
+  size_t out_remaining, out_alloc;
+  char *outptr;
+
+  out_remaining = out_alloc =
+    guess_compress_size(compress, method, compression_level, in_len);
+  *out = outptr = tor_malloc(out_remaining);
+
+  const int finish = complete_only || compress;
+
+  while (1) {
+    switch (tor_compress_process(stream,
+                                 &outptr, &out_remaining,
+                                 &in, &in_len, finish)) {
+      case TOR_COMPRESS_DONE:
+        if (in_len == 0 || compress) {
+          goto done;
+        } else {
+          // More data is present, and we're decompressing.  So we may need to
+          // reinitialize the stream if we are handling multiple concatenated
+          // inputs.
+          tor_compress_free(stream);
+          stream = tor_compress_new(compress, method, compression_level);
+          if (stream == NULL) {
+            log_warn(LD_GENERAL, "NULL stream while %scompressing",
+                     compress?"":"de");
+            goto err;
+          }
+        }
+        break;
+      case TOR_COMPRESS_OK:
+        if (compress || complete_only) {
+          log_fn(protocol_warn_level, LD_PROTOCOL,
+                 "Unexpected %s while %scompressing",
+                 complete_only?"end of input":"result",
+                 compress?"":"de");
+          log_debug(LD_GENERAL, "method: %d level: %d at len: %lu",
+                    method, compression_level, (unsigned long)in_len);
+          goto err;
+        } else {
+          if (in_len == 0) {
+            goto done;
+          }
+        }
+        break;
+      case TOR_COMPRESS_BUFFER_FULL: {
+        if (!compress && outptr < *out+out_alloc) {
+          // A buffer error in this case means that we have a problem
+          // with our input.
+          log_fn(protocol_warn_level, LD_PROTOCOL,
+                 "Possible truncated or corrupt compressed data");
+          goto err;
+        }
+        if (out_alloc >= SIZE_T_CEILING / 2) {
+          log_warn(LD_GENERAL, "While %scompressing data: ran out of space.",
+                   compress?"":"un");
+          goto err;
+        }
+        if (!compress &&
+            tor_compress_is_compression_bomb(in_len_orig, out_alloc)) {
+          // This should already have been caught down in the backend logic.
+          // LCOV_EXCL_START
+          tor_assert_nonfatal_unreached();
+          goto err;
+          // LCOV_EXCL_STOP
+        }
+        const size_t offset = outptr - *out;
+        out_alloc *= 2;
+        *out = tor_realloc(*out, out_alloc);
+        outptr = *out + offset;
+        out_remaining = out_alloc - offset;
+        break;
+      }
+      case TOR_COMPRESS_ERROR:
+        log_fn(protocol_warn_level, LD_GENERAL,
+               "Error while %scompressing data: bad input?",
+               compress?"":"un");
+        goto err; // bad data.
+
+        // LCOV_EXCL_START
+      default:
+        tor_assert_nonfatal_unreached();
+        goto err;
+        // LCOV_EXCL_STOP
+    }
+  }
+ done:
+  *out_len = outptr - *out;
+  if (compress && tor_compress_is_compression_bomb(*out_len, in_len_orig)) {
+    log_warn(LD_BUG, "We compressed something and got an insanely high "
+             "compression factor; other Tors would think this was a "
+             "compression bomb.");
+    goto err;
+  }
+  if (!compress) {
+    // NUL-terminate our output.
+    if (out_alloc == *out_len)
+      *out = tor_realloc(*out, out_alloc + 1);
+    (*out)[*out_len] = '\0';
+  }
+  rv = 0;
+  goto out;
+
+ err:
+  tor_free(*out);
+  *out_len = 0;
+  rv = -1;
+  goto out;
+
+ out:
+  tor_compress_free(stream);
+  return rv;
+}
+
+/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly
+ * allocated buffer, using the method described in <b>method</b>.  Store the
+ * compressed string in *<b>out</b>, and its length in *<b>out_len</b>.
+ * Return 0 on success, -1 on failure.
+ */
+int
+tor_compress(char **out, size_t *out_len,
+             const char *in, size_t in_len,
+             compress_method_t method)
+{
+  return tor_compress_impl(1, out, out_len, in, in_len, method,
+                           BEST_COMPRESSION,
+                           1, LOG_WARN);
+}
+
+/** Given zero or more compressed strings of total length <b>in_len</b> bytes
+ * at <b>in</b>, uncompress them into a newly allocated buffer, using the
+ * method described in <b>method</b>.  Store the uncompressed string in
+ * *<b>out</b>, and its length in *<b>out_len</b>.  Return 0 on success, -1 on
+ * failure.
+ *
+ * If any bytes are written to <b>out</b>, an extra byte NUL is always
+ * written at the end, but not counted in <b>out_len</b>.  This is a
+ * safety feature to ensure that the output can be treated as a
+ * NUL-terminated string -- though of course, callers should check
+ * out_len anyway.
+ *
+ * If <b>complete_only</b> is true, we consider a truncated input as a
+ * failure; otherwise we decompress as much as we can.  Warn about truncated
+ * or corrupt inputs at <b>protocol_warn_level</b>.
+ */
+int
+tor_uncompress(char **out, size_t *out_len,
+               const char *in, size_t in_len,
+               compress_method_t method,
+               int complete_only,
+               int protocol_warn_level)
+{
+  return tor_compress_impl(0, out, out_len, in, in_len, method,
+                           BEST_COMPRESSION,
+                           complete_only, protocol_warn_level);
+}
+
+/** Try to tell whether the <b>in_len</b>-byte string in <b>in</b> is likely
+ * to be compressed or not.  If it is, return the likeliest compression method.
+ * Otherwise, return UNKNOWN_METHOD.
+ */
+compress_method_t
+detect_compression_method(const char *in, size_t in_len)
+{
+  if (in_len > 2 && fast_memeq(in, "\x1f\x8b", 2)) {
+    return GZIP_METHOD;
+  } else if (in_len > 2 && (in[0] & 0x0f) == 8 &&
+             (tor_ntohs(get_uint16(in)) % 31) == 0) {
+    return ZLIB_METHOD;
+  } else if (in_len > 2 &&
+             fast_memeq(in, "\x5d\x00\x00", 3)) {
+    return LZMA_METHOD;
+  } else if (in_len > 3 &&
+             fast_memeq(in, "\x28\xb5\x2f\xfd", 4)) {
+    return ZSTD_METHOD;
+  } else {
+    return UNKNOWN_METHOD;
+  }
+}
+
+/** Return 1 if a given <b>method</b> is supported; otherwise 0. */
+int
+tor_compress_supports_method(compress_method_t method)
+{
+  switch (method) {
+    case GZIP_METHOD:
+    case ZLIB_METHOD:
+      return tor_zlib_method_supported();
+    case LZMA_METHOD:
+      return tor_lzma_method_supported();
+    case ZSTD_METHOD:
+      return tor_zstd_method_supported();
+    case NO_METHOD:
+      return 1;
+    case UNKNOWN_METHOD:
+    default:
+      return 0;
+  }
+}
+
+/**
+ * Return a bitmask of the supported compression types, where 1&lt;&lt;m is
+ * set in the bitmask if and only if compression with method <b>m</b> is
+ * supported.
+ */
+unsigned
+tor_compress_get_supported_method_bitmask(void)
+{
+  static unsigned supported = 0;
+  if (supported == 0) {
+    compress_method_t m;
+    for (m = NO_METHOD; m <= UNKNOWN_METHOD; ++m) {
+      if (tor_compress_supports_method(m)) {
+        supported |= (1u << m);
+      }
+    }
+  }
+  return supported;
+}
+
+/** Table of compression method names.  These should have an "x-" prefix,
+ * if they are not listed in the IANA content coding registry. */
+static const struct {
+  const char *name;
+  compress_method_t method;
+} compression_method_names[] = {
+  { "gzip", GZIP_METHOD },
+  { "deflate", ZLIB_METHOD },
+  // We call this "x-tor-lzma" rather than "x-lzma", because we impose a
+  // lower maximum memory usage on the decoding side.
+  { "x-tor-lzma", LZMA_METHOD },
+  { "x-zstd" , ZSTD_METHOD },
+  { "identity", NO_METHOD },
+
+  /* Later entries in this table are not canonical; these are recognized but
+   * not emitted. */
+  { "x-gzip", GZIP_METHOD },
+};
+
+/** Return the canonical string representation of the compression method
+ * <b>method</b>, or NULL if the method isn't recognized. */
+const char *
+compression_method_get_name(compress_method_t method)
+{
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(compression_method_names); ++i) {
+    if (method == compression_method_names[i].method)
+      return compression_method_names[i].name;
+  }
+  return NULL;
+}
+
+/** Table of compression human readable method names. */
+static const struct {
+  compress_method_t method;
+  const char *name;
+} compression_method_human_names[] = {
+  { NO_METHOD, "uncompressed" },
+  { GZIP_METHOD, "gzipped" },
+  { ZLIB_METHOD, "deflated" },
+  { LZMA_METHOD, "LZMA compressed" },
+  { ZSTD_METHOD, "Zstandard compressed" },
+  { UNKNOWN_METHOD, "unknown encoding" },
+};
+
+/** Return a human readable string representation of the compression method
+ * <b>method</b>, or NULL if the method isn't recognized. */
+const char *
+compression_method_get_human_name(compress_method_t method)
+{
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(compression_method_human_names); ++i) {
+    if (method == compression_method_human_names[i].method)
+      return compression_method_human_names[i].name;
+  }
+  return NULL;
+}
+
+/** Return the compression method represented by the string <b>name</b>, or
+ * UNKNOWN_METHOD if the string isn't recognized. */
+compress_method_t
+compression_method_get_by_name(const char *name)
+{
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(compression_method_names); ++i) {
+    if (!strcmp(compression_method_names[i].name, name))
+      return compression_method_names[i].method;
+  }
+  return UNKNOWN_METHOD;
+}
+
+/** Return a string representation of the version of the library providing the
+ * compression method given in <b>method</b>. Returns NULL if <b>method</b> is
+ * unknown or unsupported. */
+const char *
+tor_compress_version_str(compress_method_t method)
+{
+  switch (method) {
+    case GZIP_METHOD:
+    case ZLIB_METHOD:
+      return tor_zlib_get_version_str();
+    case LZMA_METHOD:
+      return tor_lzma_get_version_str();
+    case ZSTD_METHOD:
+      return tor_zstd_get_version_str();
+    case NO_METHOD:
+    case UNKNOWN_METHOD:
+    default:
+      return NULL;
+  }
+}
+
+/** Return a string representation of the version of the library, found at
+ * compile time, providing the compression method given in <b>method</b>.
+ * Returns NULL if <b>method</b> is unknown or unsupported. */
+const char *
+tor_compress_header_version_str(compress_method_t method)
+{
+  switch (method) {
+    case GZIP_METHOD:
+    case ZLIB_METHOD:
+      return tor_zlib_get_header_version_str();
+    case LZMA_METHOD:
+      return tor_lzma_get_header_version_str();
+    case ZSTD_METHOD:
+      return tor_zstd_get_header_version_str();
+    case NO_METHOD:
+    case UNKNOWN_METHOD:
+    default:
+      return NULL;
+  }
+}
+
+/** Return the approximate number of bytes allocated for all
+ * supported compression schemas. */
+size_t
+tor_compress_get_total_allocation(void)
+{
+  return atomic_counter_get(&total_compress_allocation) +
+         tor_zlib_get_total_allocation() +
+         tor_lzma_get_total_allocation() +
+         tor_zstd_get_total_allocation();
+}
+
+/** Internal state for an incremental compression/decompression.  The body of
+ * this struct is not exposed. */
+struct tor_compress_state_t {
+  compress_method_t method; /**< The compression method. */
+
+  union {
+    tor_zlib_compress_state_t *zlib_state;
+    tor_lzma_compress_state_t *lzma_state;
+    tor_zstd_compress_state_t *zstd_state;
+  } u; /**< Compression backend state. */
+};
+
+/** Construct and return a tor_compress_state_t object using <b>method</b>.  If
+ * <b>compress</b>, it's for compression; otherwise it's for decompression. */
+tor_compress_state_t *
+tor_compress_new(int compress, compress_method_t method,
+                 compression_level_t compression_level)
+{
+  tor_compress_state_t *state;
+
+  state = tor_malloc_zero(sizeof(tor_compress_state_t));
+  state->method = method;
+
+  switch (method) {
+    case GZIP_METHOD:
+    case ZLIB_METHOD: {
+      tor_zlib_compress_state_t *zlib_state =
+        tor_zlib_compress_new(compress, method, compression_level);
+
+      if (zlib_state == NULL)
+        goto err;
+
+      state->u.zlib_state = zlib_state;
+      break;
+    }
+    case LZMA_METHOD: {
+      tor_lzma_compress_state_t *lzma_state =
+        tor_lzma_compress_new(compress, method, compression_level);
+
+      if (lzma_state == NULL)
+        goto err;
+
+      state->u.lzma_state = lzma_state;
+      break;
+    }
+    case ZSTD_METHOD: {
+      tor_zstd_compress_state_t *zstd_state =
+        tor_zstd_compress_new(compress, method, compression_level);
+
+      if (zstd_state == NULL)
+        goto err;
+
+      state->u.zstd_state = zstd_state;
+      break;
+    }
+    case NO_METHOD: {
+      break;
+    }
+    case UNKNOWN_METHOD:
+      goto err;
+  }
+
+  atomic_counter_add(&total_compress_allocation,
+                     sizeof(tor_compress_state_t));
+  return state;
+
+ err:
+  tor_free(state);
+  return NULL;
+}
+
+/** Compress/decompress some bytes using <b>state</b>.  Read up to
+ * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
+ * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
+ * we've reached the end of the input.
+ *
+ * Return TOR_COMPRESS_DONE if we've finished the entire
+ * compression/decompression.
+ * Return TOR_COMPRESS_OK if we're processed everything from the input.
+ * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
+ * Return TOR_COMPRESS_ERROR if the stream is corrupt.
+ */
+tor_compress_output_t
+tor_compress_process(tor_compress_state_t *state,
+                     char **out, size_t *out_len,
+                     const char **in, size_t *in_len,
+                     int finish)
+{
+  tor_assert(state != NULL);
+  const size_t in_len_orig = *in_len;
+  const size_t out_len_orig = *out_len;
+  tor_compress_output_t rv;
+
+  if (*out_len == 0 && (*in_len > 0 || finish)) {
+    // If we still have input data, but no space for output data, we might as
+    // well return early and let the caller do the reallocation of the out
+    // variable.
+    return TOR_COMPRESS_BUFFER_FULL;
+  }
+
+  switch (state->method) {
+    case GZIP_METHOD:
+    case ZLIB_METHOD:
+      rv = tor_zlib_compress_process(state->u.zlib_state,
+                                     out, out_len, in, in_len,
+                                     finish);
+      break;
+    case LZMA_METHOD:
+      rv = tor_lzma_compress_process(state->u.lzma_state,
+                                     out, out_len, in, in_len,
+                                     finish);
+      break;
+    case ZSTD_METHOD:
+      rv = tor_zstd_compress_process(state->u.zstd_state,
+                                     out, out_len, in, in_len,
+                                     finish);
+      break;
+    case NO_METHOD:
+      rv = tor_cnone_compress_process(out, out_len, in, in_len,
+                                      finish);
+      break;
+    default:
+    case UNKNOWN_METHOD:
+      goto err;
+  }
+  if (BUG((rv == TOR_COMPRESS_OK) &&
+          *in_len == in_len_orig &&
+          *out_len == out_len_orig)) {
+    log_warn(LD_GENERAL,
+             "More info on the bug: method == %s, finish == %d, "
+             " *in_len == in_len_orig == %lu, "
+             "*out_len == out_len_orig == %lu",
+             compression_method_get_human_name(state->method), finish,
+             (unsigned long)in_len_orig, (unsigned long)out_len_orig);
+    return TOR_COMPRESS_ERROR;
+  }
+
+  return rv;
+ err:
+  return TOR_COMPRESS_ERROR;
+}
+
+/** Deallocate <b>state</b>. */
+void
+tor_compress_free_(tor_compress_state_t *state)
+{
+  if (state == NULL)
+    return;
+
+  switch (state->method) {
+    case GZIP_METHOD:
+    case ZLIB_METHOD:
+      tor_zlib_compress_free(state->u.zlib_state);
+      break;
+    case LZMA_METHOD:
+      tor_lzma_compress_free(state->u.lzma_state);
+      break;
+    case ZSTD_METHOD:
+      tor_zstd_compress_free(state->u.zstd_state);
+      break;
+    case NO_METHOD:
+      break;
+    case UNKNOWN_METHOD:
+      break;
+  }
+
+  atomic_counter_sub(&total_compress_allocation,
+                     sizeof(tor_compress_state_t));
+  tor_free(state);
+}
+
+/** Return the approximate number of bytes allocated for <b>state</b>. */
+size_t
+tor_compress_state_size(const tor_compress_state_t *state)
+{
+  tor_assert(state != NULL);
+
+  size_t size = sizeof(tor_compress_state_t);
+
+  switch (state->method) {
+    case GZIP_METHOD:
+    case ZLIB_METHOD:
+      size += tor_zlib_compress_state_size(state->u.zlib_state);
+      break;
+    case LZMA_METHOD:
+      size += tor_lzma_compress_state_size(state->u.lzma_state);
+      break;
+    case ZSTD_METHOD:
+      size += tor_zstd_compress_state_size(state->u.zstd_state);
+      break;
+    case NO_METHOD:
+    case UNKNOWN_METHOD:
+      break;
+  }
+
+  return size;
+}
+
+/** Initialize all compression modules. */
+void
+tor_compress_init(void)
+{
+  atomic_counter_init(&total_compress_allocation);
+
+  tor_zlib_init();
+  tor_lzma_init();
+  tor_zstd_init();
+}
+
+/** Warn if we had any problems while setting up our compression libraries.
+ *
+ * (This isn't part of tor_compress_init, since the logs aren't set up yet.)
+ */
+void
+tor_compress_log_init_warnings(void)
+{
+  tor_zstd_warn_if_version_mismatched();
+}
diff --git a/src/lib/compress/compress.h b/src/lib/compress/compress.h
new file mode 100644
index 0000000000..5f16a2ab27
--- /dev/null
+++ b/src/lib/compress/compress.h
@@ -0,0 +1,99 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress.h
+ * \brief Headers for compress.c
+ **/
+
+#ifndef TOR_COMPRESS_H
+#define TOR_COMPRESS_H
+
+#include <stddef.h>
+#include "lib/testsupport/testsupport.h"
+
+/** Enumeration of what kind of compression to use.  Only ZLIB_METHOD and
+ * GZIP_METHOD is guaranteed to be supported by the compress/uncompress
+ * functions here. Call tor_compress_supports_method() to check if a given
+ * compression schema is supported by Tor. */
+typedef enum compress_method_t {
+  NO_METHOD=0, // This method must be first.
+  GZIP_METHOD=1,
+  ZLIB_METHOD=2,
+  LZMA_METHOD=3,
+  ZSTD_METHOD=4,
+  UNKNOWN_METHOD=5, // This method must be last. Add new ones in the middle.
+} compress_method_t;
+
+/**
+ * Enumeration to define tradeoffs between memory usage and compression level.
+ * BEST_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most
+ * memory.
+ **/
+typedef enum compression_level_t {
+  BEST_COMPRESSION, HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION
+} compression_level_t;
+
+int tor_compress(char **out, size_t *out_len,
+                 const char *in, size_t in_len,
+                 compress_method_t method);
+
+int tor_uncompress(char **out, size_t *out_len,
+                   const char *in, size_t in_len,
+                   compress_method_t method,
+                   int complete_only,
+                   int protocol_warn_level);
+
+compress_method_t detect_compression_method(const char *in, size_t in_len);
+
+MOCK_DECL(int,tor_compress_is_compression_bomb,(size_t size_in,
+                                                size_t size_out));
+
+int tor_compress_supports_method(compress_method_t method);
+unsigned tor_compress_get_supported_method_bitmask(void);
+const char *compression_method_get_name(compress_method_t method);
+const char *compression_method_get_human_name(compress_method_t method);
+compress_method_t compression_method_get_by_name(const char *name);
+
+const char *tor_compress_version_str(compress_method_t method);
+
+const char *tor_compress_header_version_str(compress_method_t method);
+
+size_t tor_compress_get_total_allocation(void);
+
+/** Return values from tor_compress_process; see that function's documentation
+ * for details. */
+typedef enum {
+  TOR_COMPRESS_OK,
+  TOR_COMPRESS_DONE,
+  TOR_COMPRESS_BUFFER_FULL,
+  TOR_COMPRESS_ERROR
+} tor_compress_output_t;
+
+/** Internal state for an incremental compression/decompression. */
+typedef struct tor_compress_state_t tor_compress_state_t;
+
+tor_compress_state_t *tor_compress_new(int compress,
+                                       compress_method_t method,
+                                       compression_level_t level);
+
+tor_compress_output_t tor_compress_process(tor_compress_state_t *state,
+                                           char **out, size_t *out_len,
+                                           const char **in, size_t *in_len,
+                                           int finish);
+void tor_compress_free_(tor_compress_state_t *state);
+#define tor_compress_free(st) \
+  FREE_AND_NULL(tor_compress_state_t, tor_compress_free_, (st))
+
+size_t tor_compress_state_size(const tor_compress_state_t *state);
+
+void tor_compress_init(void);
+void tor_compress_log_init_warnings(void);
+
+struct buf_t;
+int buf_add_compress(struct buf_t *buf, struct tor_compress_state_t *state,
+                     const char *data, size_t data_len, int done);
+
+#endif /* !defined(TOR_COMPRESS_H) */
diff --git a/src/lib/compress/compress_buf.c b/src/lib/compress/compress_buf.c
new file mode 100644
index 0000000000..198128b261
--- /dev/null
+++ b/src/lib/compress/compress_buf.c
@@ -0,0 +1,83 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_buf.c
+ *
+ * \brief Working with compressed data in buffers.
+ **/
+
+#define BUFFERS_PRIVATE
+#include "lib/cc/compat_compiler.h"
+#include "lib/container/buffers.h"
+#include "lib/compress/compress.h"
+#include "lib/log/util_bug.h"
+
+#ifdef PARANOIA
+/** Helper: If PARANOIA is defined, assert that the buffer in local variable
+ * <b>buf</b> is well-formed. */
+#define check() STMT_BEGIN buf_assert_ok(buf); STMT_END
+#else
+#define check() STMT_NIL
+#endif /* defined(PARANOIA) */
+
+/** Compress or uncompress the <b>data_len</b> bytes in <b>data</b> using the
+ * compression state <b>state</b>, appending the result to <b>buf</b>.  If
+ * <b>done</b> is true, flush the data in the state and finish the
+ * compression/uncompression.  Return -1 on failure, 0 on success. */
+int
+buf_add_compress(buf_t *buf, tor_compress_state_t *state,
+                 const char *data, size_t data_len,
+                 const int done)
+{
+  char *next;
+  size_t old_avail, avail;
+  int over = 0;
+
+  do {
+    int need_new_chunk = 0;
+    if (!buf->tail || ! CHUNK_REMAINING_CAPACITY(buf->tail)) {
+      size_t cap = data_len / 4;
+      buf_add_chunk_with_capacity(buf, cap, 1);
+    }
+    next = CHUNK_WRITE_PTR(buf->tail);
+    avail = old_avail = CHUNK_REMAINING_CAPACITY(buf->tail);
+    switch (tor_compress_process(state, &next, &avail,
+                                 &data, &data_len, done)) {
+      case TOR_COMPRESS_DONE:
+        over = 1;
+        break;
+      case TOR_COMPRESS_ERROR:
+        return -1;
+      case TOR_COMPRESS_OK:
+        if (data_len == 0) {
+          tor_assert_nonfatal(!done);
+          over = 1;
+        }
+        break;
+      case TOR_COMPRESS_BUFFER_FULL:
+        if (avail) {
+          /* The compression module says we need more room
+           * (TOR_COMPRESS_BUFFER_FULL).  Start a new chunk automatically,
+           * whether were going to or not. */
+          need_new_chunk = 1;
+        }
+        if (data_len == 0 && !done) {
+          /* We've consumed all the input data, though, so there's no
+           * point in forging ahead right now. */
+          over = 1;
+        }
+        break;
+    }
+    buf->datalen += old_avail - avail;
+    buf->tail->datalen += old_avail - avail;
+    if (need_new_chunk) {
+      buf_add_chunk_with_capacity(buf, data_len/4, 1);
+    }
+
+  } while (!over);
+  check();
+  return 0;
+}
diff --git a/src/lib/compress/compress_lzma.c b/src/lib/compress/compress_lzma.c
new file mode 100644
index 0000000000..2dab37e433
--- /dev/null
+++ b/src/lib/compress/compress_lzma.c
@@ -0,0 +1,362 @@
+/* Copyright (c) 2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_lzma.c
+ * \brief Compression backend for LZMA.
+ *
+ * This module should never be invoked directly. Use the compress module
+ * instead.
+ **/
+
+#include "orconfig.h"
+
+#include "lib/compress/compress.h"
+#include "lib/compress/compress_lzma.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/thread/threads.h"
+
+#ifdef HAVE_LZMA
+#include <lzma.h>
+#endif
+
+/** The maximum amount of memory we allow the LZMA decoder to use, in bytes. */
+#define MEMORY_LIMIT (16 * 1024 * 1024)
+
+/** Total number of bytes allocated for LZMA state. */
+static atomic_counter_t total_lzma_allocation;
+
+#ifdef HAVE_LZMA
+/** Given <b>level</b> return the memory level. */
+static int
+memory_level(compression_level_t level)
+{
+  switch (level) {
+    default:
+    case BEST_COMPRESSION:
+    case HIGH_COMPRESSION: return 6;
+    case MEDIUM_COMPRESSION: return 4;
+    case LOW_COMPRESSION: return 2;
+  }
+}
+
+/** Convert a given <b>error</b> to a human readable error string. */
+static const char *
+lzma_error_str(lzma_ret error)
+{
+  switch (error) {
+    case LZMA_OK:
+      return "Operation completed successfully";
+    case LZMA_STREAM_END:
+      return "End of stream";
+    case LZMA_NO_CHECK:
+      return "Input stream lacks integrity check";
+    case LZMA_UNSUPPORTED_CHECK:
+      return "Unable to calculate integrity check";
+    case LZMA_GET_CHECK:
+      return "Integrity check available";
+    case LZMA_MEM_ERROR:
+      return "Unable to allocate memory";
+    case LZMA_MEMLIMIT_ERROR:
+      return "Memory limit reached";
+    case LZMA_FORMAT_ERROR:
+      return "Unknown file format";
+    case LZMA_OPTIONS_ERROR:
+      return "Unsupported options";
+    case LZMA_DATA_ERROR:
+      return "Corrupt input data";
+    case LZMA_BUF_ERROR:
+      return "Unable to progress";
+    case LZMA_PROG_ERROR:
+      return "Programming error";
+    default:
+      return "Unknown LZMA error";
+  }
+}
+#endif /* defined(HAVE_LZMA) */
+
+/** Return 1 if LZMA compression is supported; otherwise 0. */
+int
+tor_lzma_method_supported(void)
+{
+#ifdef HAVE_LZMA
+  return 1;
+#else
+  return 0;
+#endif
+}
+
+/** Return a string representation of the version of the currently running
+ * version of liblzma. Returns NULL if LZMA is unsupported. */
+const char *
+tor_lzma_get_version_str(void)
+{
+#ifdef HAVE_LZMA
+  return lzma_version_string();
+#else
+  return NULL;
+#endif
+}
+
+/** Return a string representation of the version of liblzma used at
+ * compilation time. Returns NULL if LZMA is unsupported. */
+const char *
+tor_lzma_get_header_version_str(void)
+{
+#ifdef HAVE_LZMA
+  return LZMA_VERSION_STRING;
+#else
+  return NULL;
+#endif
+}
+
+/** Internal LZMA state for incremental compression/decompression.
+ * The body of this struct is not exposed. */
+struct tor_lzma_compress_state_t {
+#ifdef HAVE_LZMA
+  lzma_stream stream; /**< The LZMA stream. */
+#endif
+
+  int compress; /**< True if we are compressing; false if we are inflating */
+
+  /** Number of bytes read so far.  Used to detect compression bombs. */
+  size_t input_so_far;
+  /** Number of bytes written so far.  Used to detect compression bombs. */
+  size_t output_so_far;
+
+  /** Approximate number of bytes allocated for this object. */
+  size_t allocation;
+};
+
+#ifdef HAVE_LZMA
+/** Return an approximate number of bytes stored in memory to hold the LZMA
+ * encoder/decoder state. */
+static size_t
+tor_lzma_state_size_precalc(int compress, compression_level_t level)
+{
+  uint64_t memory_usage;
+
+  if (compress)
+    memory_usage = lzma_easy_encoder_memusage(memory_level(level));
+  else
+    memory_usage = lzma_easy_decoder_memusage(memory_level(level));
+
+  if (memory_usage == UINT64_MAX) {
+    // LCOV_EXCL_START
+    log_warn(LD_GENERAL, "Unsupported compression level passed to LZMA %s",
+                         compress ? "encoder" : "decoder");
+    goto err;
+    // LCOV_EXCL_STOP
+  }
+
+  if (memory_usage + sizeof(tor_lzma_compress_state_t) > SIZE_MAX)
+    memory_usage = SIZE_MAX;
+  else
+    memory_usage += sizeof(tor_lzma_compress_state_t);
+
+  return (size_t)memory_usage;
+
+ // LCOV_EXCL_START
+ err:
+  return 0;
+ // LCOV_EXCL_STOP
+}
+#endif /* defined(HAVE_LZMA) */
+
+/** Construct and return a tor_lzma_compress_state_t object using
+ * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for
+ * decompression. */
+tor_lzma_compress_state_t *
+tor_lzma_compress_new(int compress,
+                      compress_method_t method,
+                      compression_level_t level)
+{
+  tor_assert(method == LZMA_METHOD);
+
+#ifdef HAVE_LZMA
+  tor_lzma_compress_state_t *result;
+  lzma_ret retval;
+  lzma_options_lzma stream_options;
+
+  // Note that we do not explicitly initialize the lzma_stream object here,
+  // since the LZMA_STREAM_INIT "just" initializes all members to 0, which is
+  // also what `tor_malloc_zero()` does.
+  result = tor_malloc_zero(sizeof(tor_lzma_compress_state_t));
+  result->compress = compress;
+  result->allocation = tor_lzma_state_size_precalc(compress, level);
+
+  if (compress) {
+    lzma_lzma_preset(&stream_options, memory_level(level));
+
+    retval = lzma_alone_encoder(&result->stream, &stream_options);
+
+    if (retval != LZMA_OK) {
+      // LCOV_EXCL_START
+      log_warn(LD_GENERAL, "Error from LZMA encoder: %s (%u).",
+               lzma_error_str(retval), retval);
+      goto err;
+      // LCOV_EXCL_STOP
+    }
+  } else {
+    retval = lzma_alone_decoder(&result->stream, MEMORY_LIMIT);
+
+    if (retval != LZMA_OK) {
+      // LCOV_EXCL_START
+      log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).",
+               lzma_error_str(retval), retval);
+      goto err;
+      // LCOV_EXCL_STOP
+    }
+  }
+
+  atomic_counter_add(&total_lzma_allocation, result->allocation);
+  return result;
+
+ /* LCOV_EXCL_START */
+ err:
+  tor_free(result);
+  return NULL;
+ /* LCOV_EXCL_STOP */
+#else /* !(defined(HAVE_LZMA)) */
+  (void)compress;
+  (void)method;
+  (void)level;
+
+  return NULL;
+#endif /* defined(HAVE_LZMA) */
+}
+
+/** Compress/decompress some bytes using <b>state</b>.  Read up to
+ * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
+ * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
+ * we've reached the end of the input.
+ *
+ * Return TOR_COMPRESS_DONE if we've finished the entire
+ * compression/decompression.
+ * Return TOR_COMPRESS_OK if we're processed everything from the input.
+ * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
+ * Return TOR_COMPRESS_ERROR if the stream is corrupt.
+ */
+tor_compress_output_t
+tor_lzma_compress_process(tor_lzma_compress_state_t *state,
+                          char **out, size_t *out_len,
+                          const char **in, size_t *in_len,
+                          int finish)
+{
+#ifdef HAVE_LZMA
+  lzma_ret retval;
+  lzma_action action;
+
+  tor_assert(state != NULL);
+  tor_assert(*in_len <= UINT_MAX);
+  tor_assert(*out_len <= UINT_MAX);
+
+  state->stream.next_in = (unsigned char *)*in;
+  state->stream.avail_in = *in_len;
+  state->stream.next_out = (unsigned char *)*out;
+  state->stream.avail_out = *out_len;
+
+  action = finish ? LZMA_FINISH : LZMA_RUN;
+
+  retval = lzma_code(&state->stream, action);
+
+  state->input_so_far += state->stream.next_in - ((unsigned char *)*in);
+  state->output_so_far += state->stream.next_out - ((unsigned char *)*out);
+
+  *out = (char *)state->stream.next_out;
+  *out_len = state->stream.avail_out;
+  *in = (const char *)state->stream.next_in;
+  *in_len = state->stream.avail_in;
+
+  if (! state->compress &&
+      tor_compress_is_compression_bomb(state->input_so_far,
+                                       state->output_so_far)) {
+    log_warn(LD_DIR, "Possible compression bomb; abandoning stream.");
+    return TOR_COMPRESS_ERROR;
+  }
+
+  switch (retval) {
+    case LZMA_OK:
+      if (state->stream.avail_out == 0 || finish)
+        return TOR_COMPRESS_BUFFER_FULL;
+
+      return TOR_COMPRESS_OK;
+
+    case LZMA_BUF_ERROR:
+      if (state->stream.avail_in == 0 && !finish)
+        return TOR_COMPRESS_OK;
+
+      return TOR_COMPRESS_BUFFER_FULL;
+
+    case LZMA_STREAM_END:
+      return TOR_COMPRESS_DONE;
+
+    // We list all the possible values of `lzma_ret` here to silence the
+    // `switch-enum` warning and to detect if a new member was added.
+    case LZMA_NO_CHECK:
+    case LZMA_UNSUPPORTED_CHECK:
+    case LZMA_GET_CHECK:
+    case LZMA_MEM_ERROR:
+    case LZMA_MEMLIMIT_ERROR:
+    case LZMA_FORMAT_ERROR:
+    case LZMA_OPTIONS_ERROR:
+    case LZMA_DATA_ERROR:
+    case LZMA_PROG_ERROR:
+    default:
+      log_warn(LD_GENERAL, "LZMA %s didn't finish: %s.",
+               state->compress ? "compression" : "decompression",
+               lzma_error_str(retval));
+      return TOR_COMPRESS_ERROR;
+  }
+#else /* !(defined(HAVE_LZMA)) */
+  (void)state;
+  (void)out;
+  (void)out_len;
+  (void)in;
+  (void)in_len;
+  (void)finish;
+  return TOR_COMPRESS_ERROR;
+#endif /* defined(HAVE_LZMA) */
+}
+
+/** Deallocate <b>state</b>. */
+void
+tor_lzma_compress_free_(tor_lzma_compress_state_t *state)
+{
+  if (state == NULL)
+    return;
+
+  atomic_counter_sub(&total_lzma_allocation, state->allocation);
+
+#ifdef HAVE_LZMA
+  lzma_end(&state->stream);
+#endif
+
+  tor_free(state);
+}
+
+/** Return the approximate number of bytes allocated for <b>state</b>. */
+size_t
+tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state)
+{
+  tor_assert(state != NULL);
+  return state->allocation;
+}
+
+/** Return the approximate number of bytes allocated for all LZMA states. */
+size_t
+tor_lzma_get_total_allocation(void)
+{
+  return atomic_counter_get(&total_lzma_allocation);
+}
+
+/** Initialize the lzma module */
+void
+tor_lzma_init(void)
+{
+  atomic_counter_init(&total_lzma_allocation);
+}
diff --git a/src/lib/compress/compress_lzma.h b/src/lib/compress/compress_lzma.h
new file mode 100644
index 0000000000..556ab437dc
--- /dev/null
+++ b/src/lib/compress/compress_lzma.h
@@ -0,0 +1,46 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_lzma.h
+ * \brief Header for compress_lzma.c
+ **/
+
+#ifndef TOR_COMPRESS_LZMA_H
+#define TOR_COMPRESS_LZMA_H
+
+int tor_lzma_method_supported(void);
+
+const char *tor_lzma_get_version_str(void);
+
+const char *tor_lzma_get_header_version_str(void);
+
+/** Internal state for an incremental LZMA compression/decompression. */
+typedef struct tor_lzma_compress_state_t tor_lzma_compress_state_t;
+
+tor_lzma_compress_state_t *
+tor_lzma_compress_new(int compress,
+                      compress_method_t method,
+                      compression_level_t compression_level);
+
+tor_compress_output_t
+tor_lzma_compress_process(tor_lzma_compress_state_t *state,
+                          char **out, size_t *out_len,
+                          const char **in, size_t *in_len,
+                          int finish);
+
+void tor_lzma_compress_free_(tor_lzma_compress_state_t *state);
+#define tor_lzma_compress_free(st)                      \
+  FREE_AND_NULL(tor_lzma_compress_state_t,   \
+                           tor_lzma_compress_free_, (st))
+
+size_t tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state);
+
+size_t tor_lzma_get_total_allocation(void);
+
+void tor_lzma_init(void);
+
+#endif /* !defined(TOR_COMPRESS_LZMA_H) */
+
diff --git a/src/lib/compress/compress_none.c b/src/lib/compress/compress_none.c
new file mode 100644
index 0000000000..0b5760773a
--- /dev/null
+++ b/src/lib/compress/compress_none.c
@@ -0,0 +1,54 @@
+/* Copyright (c) 2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_none.c
+ * \brief Compression backend for identity compression.
+ *
+ * We actually define this backend so that we can treat the identity transform
+ * as another case of compression.
+ *
+ * This module should never be invoked directly. Use the compress module
+ * instead.
+ **/
+
+#include "orconfig.h"
+
+#include "lib/log/log.h"
+#include "lib/compress/compress.h"
+#include "lib/compress/compress_none.h"
+#include "lib/intmath/cmp.h"
+
+#include <string.h>
+
+/** Transfer some bytes using the identity transformation.  Read up to
+ * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
+ * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
+ * we've reached the end of the input.
+ *
+ * Return TOR_COMPRESS_DONE if we've finished the entire
+ * compression/decompression.
+ * Return TOR_COMPRESS_OK if we're processed everything from the input.
+ * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
+ * Return TOR_COMPRESS_ERROR if the stream is corrupt.
+ */
+tor_compress_output_t
+tor_cnone_compress_process(char **out, size_t *out_len,
+                           const char **in, size_t *in_len,
+                           int finish)
+{
+  size_t n_to_copy = MIN(*in_len, *out_len);
+
+  memcpy(*out, *in, n_to_copy);
+  *out += n_to_copy;
+  *in += n_to_copy;
+  *out_len -= n_to_copy;
+  *in_len -= n_to_copy;
+  if (*in_len == 0) {
+    return finish ? TOR_COMPRESS_DONE : TOR_COMPRESS_OK;
+  } else {
+    return TOR_COMPRESS_BUFFER_FULL;
+  }
+}
diff --git a/src/lib/compress/compress_none.h b/src/lib/compress/compress_none.h
new file mode 100644
index 0000000000..2bb9c3d66c
--- /dev/null
+++ b/src/lib/compress/compress_none.h
@@ -0,0 +1,20 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_none.h
+ * \brief Header for compress_none.c
+ **/
+
+#ifndef TOR_COMPRESS_NONE_H
+#define TOR_COMPRESS_NONE_H
+
+tor_compress_output_t
+tor_cnone_compress_process(char **out, size_t *out_len,
+                           const char **in, size_t *in_len,
+                           int finish);
+
+#endif /* !defined(TOR_COMPRESS_NONE_H) */
+
diff --git a/src/lib/compress/compress_zlib.c b/src/lib/compress/compress_zlib.c
new file mode 100644
index 0000000000..df0d1bff5f
--- /dev/null
+++ b/src/lib/compress/compress_zlib.c
@@ -0,0 +1,304 @@
+/* Copyright (c) 2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_zlib.c
+ * \brief Compression backend for gzip and zlib.
+ *
+ * This module should never be invoked directly. Use the compress module
+ * instead.
+ **/
+
+#include "orconfig.h"
+
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/compress/compress.h"
+#include "lib/compress/compress_zlib.h"
+#include "lib/thread/threads.h"
+
+/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros.  Instead of
+   saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory
+   that nobody will care if the compile outputs a no-such-identifier warning.
+
+   Sorry, but we like -Werror over here, so I guess we need to define these.
+   I hope that zlib 1.2.6 doesn't break these too.
+*/
+#ifndef _LARGEFILE64_SOURCE
+#define _LARGEFILE64_SOURCE 0
+#endif
+#ifndef _LFS64_LARGEFILE
+#define _LFS64_LARGEFILE 0
+#endif
+#ifndef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 0
+#endif
+#ifndef off64_t
+#define off64_t int64_t
+#endif
+
+#include <zlib.h>
+
+#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200
+#error "We require zlib version 1.2 or later."
+#endif
+
+static size_t tor_zlib_state_size_precalc(int inflate,
+                                          int windowbits, int memlevel);
+
+/** Total number of bytes allocated for zlib state */
+static atomic_counter_t total_zlib_allocation;
+
+/** Given <b>level</b> return the memory level. */
+static int
+memory_level(compression_level_t level)
+{
+  switch (level) {
+    default:
+    case BEST_COMPRESSION: return 9;
+    case HIGH_COMPRESSION: return 8;
+    case MEDIUM_COMPRESSION: return 7;
+    case LOW_COMPRESSION: return 6;
+  }
+}
+
+/** Return the 'bits' value to tell zlib to use <b>method</b>.*/
+static inline int
+method_bits(compress_method_t method, compression_level_t level)
+{
+  /* Bits+16 means "use gzip" in zlib >= 1.2 */
+  const int flag = method == GZIP_METHOD ? 16 : 0;
+  switch (level) {
+    default:
+    case BEST_COMPRESSION:
+    case HIGH_COMPRESSION: return flag + 15;
+    case MEDIUM_COMPRESSION: return flag + 13;
+    case LOW_COMPRESSION: return flag + 11;
+  }
+}
+
+/** Return 1 if zlib/gzip compression is supported; otherwise 0. */
+int
+tor_zlib_method_supported(void)
+{
+  /* We currently always support zlib/gzip, but we keep this function around in
+   * case we some day decide to deprecate zlib/gzip support.
+   */
+  return 1;
+}
+
+/** Return a string representation of the version of the currently running
+ * version of zlib. */
+const char *
+tor_zlib_get_version_str(void)
+{
+  return zlibVersion();
+}
+
+/** Return a string representation of the version of the version of zlib
+* used at compilation. */
+const char *
+tor_zlib_get_header_version_str(void)
+{
+  return ZLIB_VERSION;
+}
+
+/** Internal zlib state for an incremental compression/decompression.
+ * The body of this struct is not exposed. */
+struct tor_zlib_compress_state_t {
+  struct z_stream_s stream; /**< The zlib stream */
+  int compress; /**< True if we are compressing; false if we are inflating */
+
+  /** Number of bytes read so far.  Used to detect zlib bombs. */
+  size_t input_so_far;
+  /** Number of bytes written so far.  Used to detect zlib bombs. */
+  size_t output_so_far;
+
+  /** Approximate number of bytes allocated for this object. */
+  size_t allocation;
+};
+
+/** Return an approximate number of bytes used in RAM to hold a state with
+ * window bits <b>windowBits</b> and compression level 'memlevel' */
+static size_t
+tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel)
+{
+  windowbits &= 15;
+
+#define A_FEW_KILOBYTES 2048
+
+  if (inflate_) {
+    /* From zconf.h:
+
+       "The memory requirements for inflate are (in bytes) 1 << windowBits
+       that is, 32K for windowBits=15 (default value) plus a few kilobytes
+       for small objects."
+    */
+    return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) +
+      (1 << 15) + A_FEW_KILOBYTES;
+  } else {
+    /* Also from zconf.h:
+
+       "The memory requirements for deflate are (in bytes):
+            (1 << (windowBits+2)) +  (1 << (memLevel+9))
+        ... plus a few kilobytes for small objects."
+    */
+    return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) +
+      (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES;
+  }
+#undef A_FEW_KILOBYTES
+}
+
+/** Construct and return a tor_zlib_compress_state_t object using
+ * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for
+ * decompression. */
+tor_zlib_compress_state_t *
+tor_zlib_compress_new(int compress_,
+                      compress_method_t method,
+                      compression_level_t compression_level)
+{
+  tor_zlib_compress_state_t *out;
+  int bits, memlevel;
+
+  if (! compress_) {
+    /* use this setting for decompression, since we might have the
+     * max number of window bits */
+    compression_level = BEST_COMPRESSION;
+  }
+
+  out = tor_malloc_zero(sizeof(tor_zlib_compress_state_t));
+  out->stream.zalloc = Z_NULL;
+  out->stream.zfree = Z_NULL;
+  out->stream.opaque = NULL;
+  out->compress = compress_;
+  bits = method_bits(method, compression_level);
+  memlevel = memory_level(compression_level);
+  if (compress_) {
+    if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,
+                     bits, memlevel,
+                     Z_DEFAULT_STRATEGY) != Z_OK)
+    goto err; // LCOV_EXCL_LINE
+  } else {
+    if (inflateInit2(&out->stream, bits) != Z_OK)
+      goto err; // LCOV_EXCL_LINE
+  }
+  out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel);
+
+  atomic_counter_add(&total_zlib_allocation, out->allocation);
+
+  return out;
+
+ err:
+  tor_free(out);
+  return NULL;
+}
+
+/** Compress/decompress some bytes using <b>state</b>.  Read up to
+ * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
+ * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
+ * we've reached the end of the input.
+ *
+ * Return TOR_COMPRESS_DONE if we've finished the entire
+ * compression/decompression.
+ * Return TOR_COMPRESS_OK if we're processed everything from the input.
+ * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
+ * Return TOR_COMPRESS_ERROR if the stream is corrupt.
+ */
+tor_compress_output_t
+tor_zlib_compress_process(tor_zlib_compress_state_t *state,
+                          char **out, size_t *out_len,
+                          const char **in, size_t *in_len,
+                          int finish)
+{
+  int err;
+  tor_assert(state != NULL);
+  if (*in_len > UINT_MAX ||
+      *out_len > UINT_MAX) {
+    return TOR_COMPRESS_ERROR;
+  }
+
+  state->stream.next_in = (unsigned char*) *in;
+  state->stream.avail_in = (unsigned int)*in_len;
+  state->stream.next_out = (unsigned char*) *out;
+  state->stream.avail_out = (unsigned int)*out_len;
+
+  if (state->compress) {
+    err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH);
+  } else {
+    err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH);
+  }
+
+  state->input_so_far += state->stream.next_in - ((unsigned char*)*in);
+  state->output_so_far += state->stream.next_out - ((unsigned char*)*out);
+
+  *out = (char*) state->stream.next_out;
+  *out_len = state->stream.avail_out;
+  *in = (const char *) state->stream.next_in;
+  *in_len = state->stream.avail_in;
+
+  if (! state->compress &&
+      tor_compress_is_compression_bomb(state->input_so_far,
+                                       state->output_so_far)) {
+    log_warn(LD_DIR, "Possible zlib bomb; abandoning stream.");
+    return TOR_COMPRESS_ERROR;
+  }
+
+  switch (err)
+    {
+    case Z_STREAM_END:
+      return TOR_COMPRESS_DONE;
+    case Z_BUF_ERROR:
+      if (state->stream.avail_in == 0 && !finish)
+        return TOR_COMPRESS_OK;
+      return TOR_COMPRESS_BUFFER_FULL;
+    case Z_OK:
+      if (state->stream.avail_out == 0 || finish)
+        return TOR_COMPRESS_BUFFER_FULL;
+      return TOR_COMPRESS_OK;
+    default:
+      log_warn(LD_GENERAL, "Gzip returned an error: %s",
+               state->stream.msg ? state->stream.msg : "<no message>");
+      return TOR_COMPRESS_ERROR;
+    }
+}
+
+/** Deallocate <b>state</b>. */
+void
+tor_zlib_compress_free_(tor_zlib_compress_state_t *state)
+{
+  if (state == NULL)
+    return;
+
+  atomic_counter_sub(&total_zlib_allocation, state->allocation);
+
+  if (state->compress)
+    deflateEnd(&state->stream);
+  else
+    inflateEnd(&state->stream);
+
+  tor_free(state);
+}
+
+/** Return the approximate number of bytes allocated for <b>state</b>. */
+size_t
+tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state)
+{
+  tor_assert(state != NULL);
+  return state->allocation;
+}
+
+/** Return the approximate number of bytes allocated for all zlib states. */
+size_t
+tor_zlib_get_total_allocation(void)
+{
+  return atomic_counter_get(&total_zlib_allocation);
+}
+
+/** Set up global state for the zlib module */
+void
+tor_zlib_init(void)
+{
+  atomic_counter_init(&total_zlib_allocation);
+}
diff --git a/src/lib/compress/compress_zlib.h b/src/lib/compress/compress_zlib.h
new file mode 100644
index 0000000000..e4f248cd9b
--- /dev/null
+++ b/src/lib/compress/compress_zlib.h
@@ -0,0 +1,46 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_zlib.h
+ * \brief Header for compress_zlib.c
+ **/
+
+#ifndef TOR_COMPRESS_ZLIB_H
+#define TOR_COMPRESS_ZLIB_H
+
+int tor_zlib_method_supported(void);
+
+const char *tor_zlib_get_version_str(void);
+
+const char *tor_zlib_get_header_version_str(void);
+
+/** Internal state for an incremental zlib/gzip compression/decompression. */
+typedef struct tor_zlib_compress_state_t tor_zlib_compress_state_t;
+
+tor_zlib_compress_state_t *
+tor_zlib_compress_new(int compress,
+                      compress_method_t method,
+                      compression_level_t compression_level);
+
+tor_compress_output_t
+tor_zlib_compress_process(tor_zlib_compress_state_t *state,
+                          char **out, size_t *out_len,
+                          const char **in, size_t *in_len,
+                          int finish);
+
+void tor_zlib_compress_free_(tor_zlib_compress_state_t *state);
+#define tor_zlib_compress_free(st)                      \
+  FREE_AND_NULL(tor_zlib_compress_state_t,   \
+                           tor_zlib_compress_free_, (st))
+
+size_t tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state);
+
+size_t tor_zlib_get_total_allocation(void);
+
+void tor_zlib_init(void);
+
+#endif /* !defined(TOR_COMPRESS_ZLIB_H) */
+
diff --git a/src/lib/compress/compress_zstd.c b/src/lib/compress/compress_zstd.c
new file mode 100644
index 0000000000..45d0d4d602
--- /dev/null
+++ b/src/lib/compress/compress_zstd.c
@@ -0,0 +1,541 @@
+/* Copyright (c) 2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_zstd.c
+ * \brief Compression backend for Zstandard.
+ *
+ * This module should never be invoked directly. Use the compress module
+ * instead.
+ **/
+
+#include "orconfig.h"
+
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/compress/compress.h"
+#include "lib/compress/compress_zstd.h"
+#include "lib/string/printf.h"
+#include "lib/thread/threads.h"
+
+#ifdef ENABLE_ZSTD_ADVANCED_APIS
+/* This is a lie, but we make sure it doesn't get us in trouble by wrapping
+ * all invocations of zstd's static-only functions in a check to make sure
+ * that the compile-time version matches the run-time version. */
+#define ZSTD_STATIC_LINKING_ONLY
+#endif
+
+#ifdef HAVE_ZSTD
+#ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE
+DISABLE_GCC_WARNING(unused-const-variable)
+#endif
+#include <zstd.h>
+#ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE
+ENABLE_GCC_WARNING(unused-const-variable)
+#endif
+#endif
+
+/** Total number of bytes allocated for Zstandard state. */
+static atomic_counter_t total_zstd_allocation;
+
+#ifdef HAVE_ZSTD
+/** Given <b>level</b> return the memory level. */
+static int
+memory_level(compression_level_t level)
+{
+  switch (level) {
+    default:
+    case BEST_COMPRESSION:
+    case HIGH_COMPRESSION: return 9;
+    case MEDIUM_COMPRESSION: return 8;
+    case LOW_COMPRESSION: return 7;
+  }
+}
+#endif /* defined(HAVE_ZSTD) */
+
+/** Return 1 if Zstandard compression is supported; otherwise 0. */
+int
+tor_zstd_method_supported(void)
+{
+#ifdef HAVE_ZSTD
+  return 1;
+#else
+  return 0;
+#endif
+}
+
+#ifdef HAVE_ZSTD
+/** Format a zstd version number as a string in <b>buf</b>. */
+static void
+tor_zstd_format_version(char *buf, size_t buflen, unsigned version_number)
+{
+  tor_snprintf(buf, buflen,
+               "%u.%u.%u",
+               version_number / 10000 % 100,
+               version_number / 100 % 100,
+               version_number % 100);
+}
+#endif
+
+#define VERSION_STR_MAX_LEN 16 /* more than enough space for 99.99.99 */
+
+/** Return a string representation of the version of the currently running
+ * version of libzstd. Returns NULL if Zstandard is unsupported. */
+const char *
+tor_zstd_get_version_str(void)
+{
+#ifdef HAVE_ZSTD
+  static char version_str[VERSION_STR_MAX_LEN];
+
+  tor_zstd_format_version(version_str, sizeof(version_str),
+                          ZSTD_versionNumber());
+
+  return version_str;
+#else /* !(defined(HAVE_ZSTD)) */
+  return NULL;
+#endif /* defined(HAVE_ZSTD) */
+}
+
+/** Return a string representation of the version of the version of libzstd
+ * used at compilation time. Returns NULL if Zstandard is unsupported. */
+const char *
+tor_zstd_get_header_version_str(void)
+{
+#ifdef HAVE_ZSTD
+  return ZSTD_VERSION_STRING;
+#else
+  return NULL;
+#endif
+}
+
+#ifdef TOR_UNIT_TESTS
+static int static_apis_disable_for_testing = 0;
+#endif
+
+/** Return true iff we can use the "static-only" APIs. */
+int
+tor_zstd_can_use_static_apis(void)
+{
+#if defined(ZSTD_STATIC_LINKING_ONLY) && defined(HAVE_ZSTD)
+#ifdef TOR_UNIT_TESTS
+  if (static_apis_disable_for_testing) {
+    return 0;
+  }
+#endif
+  return (ZSTD_VERSION_NUMBER == ZSTD_versionNumber());
+#else
+  return 0;
+#endif
+}
+
+/** Internal Zstandard state for incremental compression/decompression.
+ * The body of this struct is not exposed. */
+struct tor_zstd_compress_state_t {
+#ifdef HAVE_ZSTD
+  union {
+    /** Compression stream. Used when <b>compress</b> is true. */
+    ZSTD_CStream *compress_stream;
+    /** Decompression stream. Used when <b>compress</b> is false. */
+    ZSTD_DStream *decompress_stream;
+  } u; /**< Zstandard stream objects. */
+#endif /* defined(HAVE_ZSTD) */
+
+  int compress; /**< True if we are compressing; false if we are inflating */
+  int have_called_end; /**< True if we are compressing and we've called
+                        * ZSTD_endStream */
+
+  /** Number of bytes read so far.  Used to detect compression bombs. */
+  size_t input_so_far;
+  /** Number of bytes written so far.  Used to detect compression bombs. */
+  size_t output_so_far;
+
+  /** Approximate number of bytes allocated for this object. */
+  size_t allocation;
+};
+
+#ifdef HAVE_ZSTD
+/** Return an approximate number of bytes stored in memory to hold the
+ * Zstandard compression/decompression state. This is a fake estimate
+ * based on inspecting the zstd source: tor_zstd_state_size_precalc() is
+ * more accurate when it's allowed to use "static-only" functions */
+static size_t
+tor_zstd_state_size_precalc_fake(int compress, int preset)
+{
+  tor_assert(preset > 0);
+
+  size_t memory_usage = sizeof(tor_zstd_compress_state_t);
+
+  // The Zstandard library provides a number of functions that would be useful
+  // here, but they are, unfortunately, still considered experimental and are
+  // thus only available in libzstd if we link against the library statically.
+  //
+  // The code in this function tries to approximate the calculations without
+  // being able to use the following:
+  //
+  // - We do not have access to neither the internal members of ZSTD_CStream
+  //   and ZSTD_DStream and their internal context objects.
+  //
+  // - We cannot use ZSTD_sizeof_CStream() and ZSTD_sizeof_DStream() since they
+  //   are unexposed.
+  //
+  // In the future it might be useful to check if libzstd have started
+  // providing these functions in a stable manner and simplify this function.
+  if (compress) {
+    // We try to approximate the ZSTD_sizeof_CStream(ZSTD_CStream *stream)
+    // function here. This function uses the following fields to make its
+    // estimate:
+
+    // - sizeof(ZSTD_CStream): Around 192 bytes on a 64-bit machine:
+    memory_usage += 192;
+
+    // - ZSTD_sizeof_CCtx(stream->cctx): This function requires access to
+    // variables that are not exposed via the public API. We use a _very_
+    // simplified function to calculate the estimated amount of bytes used in
+    // this struct.
+    // memory_usage += (preset - 0.5) * 1024 * 1024;
+    memory_usage += (preset * 1024 * 1024) - (512 * 1024);
+    // - ZSTD_sizeof_CDict(stream->cdictLocal): Unused in Tor: 0 bytes.
+    // - stream->outBuffSize: 128 KB:
+    memory_usage += 128 * 1024;
+    // - stream->inBuffSize: 2048 KB:
+    memory_usage += 2048 * 1024;
+  } else {
+    // We try to approximate the ZSTD_sizeof_DStream(ZSTD_DStream *stream)
+    // function here. This function uses the following fields to make its
+    // estimate:
+
+    // - sizeof(ZSTD_DStream): Around 208 bytes on a 64-bit machine:
+    memory_usage += 208;
+    // - ZSTD_sizeof_DCtx(stream->dctx): Around 150 KB.
+    memory_usage += 150 * 1024;
+
+    // - ZSTD_sizeof_DDict(stream->ddictLocal): Unused in Tor: 0 bytes.
+    // - stream->inBuffSize: 0 KB.
+    // - stream->outBuffSize: 0 KB.
+  }
+
+  return memory_usage;
+}
+
+/** Return an approximate number of bytes stored in memory to hold the
+ * Zstandard compression/decompression state. */
+static size_t
+tor_zstd_state_size_precalc(int compress, int preset)
+{
+#ifdef ZSTD_STATIC_LINKING_ONLY
+  if (tor_zstd_can_use_static_apis()) {
+    if (compress) {
+#ifdef HAVE_ZSTD_ESTIMATECSTREAMSIZE
+      return ZSTD_estimateCStreamSize(preset);
+#endif
+    } else {
+#ifdef HAVE_ZSTD_ESTIMATEDCTXSIZE
+      /* Could use DStream, but that takes a windowSize. */
+      return ZSTD_estimateDCtxSize();
+#endif
+    }
+  }
+#endif
+  return tor_zstd_state_size_precalc_fake(compress, preset);
+}
+#endif /* defined(HAVE_ZSTD) */
+
+/** Construct and return a tor_zstd_compress_state_t object using
+ * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for
+ * decompression. */
+tor_zstd_compress_state_t *
+tor_zstd_compress_new(int compress,
+                      compress_method_t method,
+                      compression_level_t level)
+{
+  tor_assert(method == ZSTD_METHOD);
+
+#ifdef HAVE_ZSTD
+  const int preset = memory_level(level);
+  tor_zstd_compress_state_t *result;
+  size_t retval;
+
+  result = tor_malloc_zero(sizeof(tor_zstd_compress_state_t));
+  result->compress = compress;
+  result->allocation = tor_zstd_state_size_precalc(compress, preset);
+
+  if (compress) {
+    result->u.compress_stream = ZSTD_createCStream();
+
+    if (result->u.compress_stream == NULL) {
+      // LCOV_EXCL_START
+      log_warn(LD_GENERAL, "Error while creating Zstandard compression "
+               "stream");
+      goto err;
+      // LCOV_EXCL_STOP
+    }
+
+    retval = ZSTD_initCStream(result->u.compress_stream, preset);
+
+    if (ZSTD_isError(retval)) {
+      // LCOV_EXCL_START
+      log_warn(LD_GENERAL, "Zstandard stream initialization error: %s",
+               ZSTD_getErrorName(retval));
+      goto err;
+      // LCOV_EXCL_STOP
+    }
+  } else {
+    result->u.decompress_stream = ZSTD_createDStream();
+
+    if (result->u.decompress_stream == NULL) {
+      // LCOV_EXCL_START
+      log_warn(LD_GENERAL, "Error while creating Zstandard decompression "
+               "stream");
+      goto err;
+      // LCOV_EXCL_STOP
+    }
+
+    retval = ZSTD_initDStream(result->u.decompress_stream);
+
+    if (ZSTD_isError(retval)) {
+      // LCOV_EXCL_START
+      log_warn(LD_GENERAL, "Zstandard stream initialization error: %s",
+               ZSTD_getErrorName(retval));
+      goto err;
+      // LCOV_EXCL_STOP
+    }
+  }
+
+  atomic_counter_add(&total_zstd_allocation, result->allocation);
+  return result;
+
+ err:
+  // LCOV_EXCL_START
+  if (compress) {
+    ZSTD_freeCStream(result->u.compress_stream);
+  } else {
+    ZSTD_freeDStream(result->u.decompress_stream);
+  }
+
+  tor_free(result);
+  return NULL;
+  // LCOV_EXCL_STOP
+#else /* !(defined(HAVE_ZSTD)) */
+  (void)compress;
+  (void)method;
+  (void)level;
+
+  return NULL;
+#endif /* defined(HAVE_ZSTD) */
+}
+
+/** Compress/decompress some bytes using <b>state</b>.  Read up to
+ * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
+ * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
+ * we've reached the end of the input.
+ *
+ * Return TOR_COMPRESS_DONE if we've finished the entire
+ * compression/decompression.
+ * Return TOR_COMPRESS_OK if we're processed everything from the input.
+ * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
+ * Return TOR_COMPRESS_ERROR if the stream is corrupt.
+ */
+tor_compress_output_t
+tor_zstd_compress_process(tor_zstd_compress_state_t *state,
+                          char **out, size_t *out_len,
+                          const char **in, size_t *in_len,
+                          int finish)
+{
+#ifdef HAVE_ZSTD
+  size_t retval;
+
+  tor_assert(state != NULL);
+  tor_assert(*in_len <= UINT_MAX);
+  tor_assert(*out_len <= UINT_MAX);
+
+  ZSTD_inBuffer input = { *in, *in_len, 0 };
+  ZSTD_outBuffer output = { *out, *out_len, 0 };
+
+  if (BUG(finish == 0 && state->have_called_end)) {
+    finish = 1;
+  }
+
+  if (state->compress) {
+    if (! state->have_called_end)
+      retval = ZSTD_compressStream(state->u.compress_stream,
+                                   &output, &input);
+    else
+      retval = 0;
+  } else {
+    retval = ZSTD_decompressStream(state->u.decompress_stream,
+                                   &output, &input);
+  }
+
+  state->input_so_far += input.pos;
+  state->output_so_far += output.pos;
+
+  *out = (char *)output.dst + output.pos;
+  *out_len = output.size - output.pos;
+  *in = (char *)input.src + input.pos;
+  *in_len = input.size - input.pos;
+
+  if (! state->compress &&
+      tor_compress_is_compression_bomb(state->input_so_far,
+                                       state->output_so_far)) {
+    log_warn(LD_DIR, "Possible compression bomb; abandoning stream.");
+    return TOR_COMPRESS_ERROR;
+  }
+
+  if (ZSTD_isError(retval)) {
+    log_warn(LD_GENERAL, "Zstandard %s didn't finish: %s.",
+             state->compress ? "compression" : "decompression",
+             ZSTD_getErrorName(retval));
+    return TOR_COMPRESS_ERROR;
+  }
+
+  if (state->compress && !state->have_called_end) {
+    retval = ZSTD_flushStream(state->u.compress_stream, &output);
+
+    *out = (char *)output.dst + output.pos;
+    *out_len = output.size - output.pos;
+
+    if (ZSTD_isError(retval)) {
+      log_warn(LD_GENERAL, "Zstandard compression unable to flush: %s.",
+               ZSTD_getErrorName(retval));
+      return TOR_COMPRESS_ERROR;
+    }
+
+    // ZSTD_flushStream returns 0 if the frame is done, or >0 if it
+    // is incomplete.
+    if (retval > 0) {
+      return TOR_COMPRESS_BUFFER_FULL;
+    }
+  }
+
+  if (!finish) {
+    // The caller says we're not done with the input, so no need to write an
+    // epilogue.
+    return TOR_COMPRESS_OK;
+  } else if (state->compress) {
+    if (*in_len) {
+      // We say that we're not done with the input, so we can't write an
+      // epilogue.
+      return TOR_COMPRESS_OK;
+    }
+
+    retval = ZSTD_endStream(state->u.compress_stream, &output);
+    state->have_called_end = 1;
+    *out = (char *)output.dst + output.pos;
+    *out_len = output.size - output.pos;
+
+    if (ZSTD_isError(retval)) {
+      log_warn(LD_GENERAL, "Zstandard compression unable to write "
+               "epilogue: %s.",
+               ZSTD_getErrorName(retval));
+      return TOR_COMPRESS_ERROR;
+    }
+
+    // endStream returns the number of bytes that is needed to write the
+    // epilogue.
+    if (retval > 0)
+      return TOR_COMPRESS_BUFFER_FULL;
+
+    return TOR_COMPRESS_DONE;
+  } else /* if (!state->compress) */ {
+    // ZSTD_decompressStream returns 0 if the frame is done, or >0 if it
+    // is incomplete.
+    // We check this above.
+    tor_assert_nonfatal(!ZSTD_isError(retval));
+    // Start a new frame if this frame is done
+    if (retval == 0)
+      return TOR_COMPRESS_DONE;
+    // Don't check out_len, it might have some space left if the next output
+    // chunk is larger than the remaining space
+    else if (*in_len > 0)
+      return  TOR_COMPRESS_BUFFER_FULL;
+    else
+      return TOR_COMPRESS_OK;
+  }
+
+#else /* !(defined(HAVE_ZSTD)) */
+  (void)state;
+  (void)out;
+  (void)out_len;
+  (void)in;
+  (void)in_len;
+  (void)finish;
+
+  return TOR_COMPRESS_ERROR;
+#endif /* defined(HAVE_ZSTD) */
+}
+
+/** Deallocate <b>state</b>. */
+void
+tor_zstd_compress_free_(tor_zstd_compress_state_t *state)
+{
+  if (state == NULL)
+    return;
+
+  atomic_counter_sub(&total_zstd_allocation, state->allocation);
+
+#ifdef HAVE_ZSTD
+  if (state->compress) {
+    ZSTD_freeCStream(state->u.compress_stream);
+  } else {
+    ZSTD_freeDStream(state->u.decompress_stream);
+  }
+#endif /* defined(HAVE_ZSTD) */
+
+  tor_free(state);
+}
+
+/** Return the approximate number of bytes allocated for <b>state</b>. */
+size_t
+tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state)
+{
+  tor_assert(state != NULL);
+  return state->allocation;
+}
+
+/** Return the approximate number of bytes allocated for all Zstandard
+ * states. */
+size_t
+tor_zstd_get_total_allocation(void)
+{
+  return atomic_counter_get(&total_zstd_allocation);
+}
+
+/** Initialize the zstd module */
+void
+tor_zstd_init(void)
+{
+  atomic_counter_init(&total_zstd_allocation);
+}
+
+/** Warn if the header and library versions don't match. */
+void
+tor_zstd_warn_if_version_mismatched(void)
+{
+#if defined(HAVE_ZSTD) && defined(ENABLE_ZSTD_ADVANCED_APIS)
+  if (! tor_zstd_can_use_static_apis()) {
+    char header_version[VERSION_STR_MAX_LEN];
+    char runtime_version[VERSION_STR_MAX_LEN];
+    tor_zstd_format_version(header_version, sizeof(header_version),
+                            ZSTD_VERSION_NUMBER);
+    tor_zstd_format_version(runtime_version, sizeof(runtime_version),
+                            ZSTD_versionNumber());
+
+    log_warn(LD_GENERAL,
+             "Tor was compiled with zstd %s, but is running with zstd %s. "
+             "For safety, we'll avoid using advanced zstd functionality.",
+             header_version, runtime_version);
+  }
+#endif
+}
+
+#ifdef TOR_UNIT_TESTS
+/** Testing only: disable usage of static-only APIs, so we can make sure that
+ * we still work without them. */
+void
+tor_zstd_set_static_apis_disabled_for_testing(int disabled)
+{
+  static_apis_disable_for_testing = disabled;
+}
+#endif
diff --git a/src/lib/compress/compress_zstd.h b/src/lib/compress/compress_zstd.h
new file mode 100644
index 0000000000..47f950b9e0
--- /dev/null
+++ b/src/lib/compress/compress_zstd.h
@@ -0,0 +1,53 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_zstd.h
+ * \brief Header for compress_zstd.c
+ **/
+
+#ifndef TOR_COMPRESS_ZSTD_H
+#define TOR_COMPRESS_ZSTD_H
+
+int tor_zstd_method_supported(void);
+
+const char *tor_zstd_get_version_str(void);
+
+const char *tor_zstd_get_header_version_str(void);
+
+int tor_zstd_can_use_static_apis(void);
+
+/** Internal state for an incremental Zstandard compression/decompression. */
+typedef struct tor_zstd_compress_state_t tor_zstd_compress_state_t;
+
+tor_zstd_compress_state_t *
+tor_zstd_compress_new(int compress,
+                      compress_method_t method,
+                      compression_level_t compression_level);
+
+tor_compress_output_t
+tor_zstd_compress_process(tor_zstd_compress_state_t *state,
+                          char **out, size_t *out_len,
+                          const char **in, size_t *in_len,
+                          int finish);
+
+void tor_zstd_compress_free_(tor_zstd_compress_state_t *state);
+#define tor_zstd_compress_free(st)                      \
+  FREE_AND_NULL(tor_zstd_compress_state_t,   \
+                           tor_zstd_compress_free_, (st))
+
+size_t tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state);
+
+size_t tor_zstd_get_total_allocation(void);
+
+void tor_zstd_init(void);
+void tor_zstd_warn_if_version_mismatched(void);
+
+#ifdef TOR_UNIT_TESTS
+void tor_zstd_set_static_apis_disabled_for_testing(int disabled);
+#endif
+
+#endif /* !defined(TOR_COMPRESS_ZSTD_H) */
+
diff --git a/src/lib/compress/include.am b/src/lib/compress/include.am
new file mode 100644
index 0000000000..75c9032bd2
--- /dev/null
+++ b/src/lib/compress/include.am
@@ -0,0 +1,26 @@
+
+noinst_LIBRARIES += src/lib/libtor-compress.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-compress-testing.a
+endif
+
+src_lib_libtor_compress_a_SOURCES =		\
+	src/lib/compress/compress.c		\
+	src/lib/compress/compress_buf.c		\
+	src/lib/compress/compress_lzma.c	\
+	src/lib/compress/compress_none.c	\
+	src/lib/compress/compress_zlib.c	\
+	src/lib/compress/compress_zstd.c
+
+src_lib_libtor_compress_testing_a_SOURCES = \
+	$(src_lib_libtor_compress_a_SOURCES)
+src_lib_libtor_compress_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_compress_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS	+=				\
+	src/lib/compress/compress.h		\
+	src/lib/compress/compress_lzma.h	\
+	src/lib/compress/compress_none.h	\
+	src/lib/compress/compress_zlib.h	\
+	src/lib/compress/compress_zstd.h
diff --git a/src/lib/container/.may_include b/src/lib/container/.may_include
new file mode 100644
index 0000000000..90de5eda40
--- /dev/null
+++ b/src/lib/container/.may_include
@@ -0,0 +1,18 @@
+orconfig.h
+lib/cc/*.h
+lib/container/*.h
+lib/ctime/*.h
+lib/defs/*.h
+lib/malloc/*.h
+lib/err/*.h
+lib/smartlist_core/*.h
+lib/string/*.h
+lib/testsupport/testsupport.h
+lib/intmath/*.h
+lib/log/*.h
+
+# XXXX I am unsure about this one. It's only here for buffers.c
+lib/time/*.h
+
+ht.h
+siphash.h
diff --git a/src/lib/container/bitarray.h b/src/lib/container/bitarray.h
new file mode 100644
index 0000000000..910d5fea65
--- /dev/null
+++ b/src/lib/container/bitarray.h
@@ -0,0 +1,86 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_BITARRAY_H
+#define TOR_BITARRAY_H
+
+/**
+ * \file bitarray.h
+ *
+ * \brief Implements a variable-sized (but non-resizeable) bit-array.
+ **/
+
+#include "orconfig.h"
+#include <string.h>
+#include "lib/cc/torint.h"
+#include "lib/malloc/malloc.h"
+
+#if SIZEOF_INT == 4
+#define BITARRAY_SHIFT 5
+#elif SIZEOF_INT == 8
+#define BITARRAY_SHIFT 6
+#else
+#error "int is neither 4 nor 8 bytes. I can't deal with that."
+#endif /* SIZEOF_INT == 4 || ... */
+#define BITARRAY_MASK ((1u<<BITARRAY_SHIFT)-1)
+
+/** A random-access array of one-bit-wide elements. */
+typedef unsigned int bitarray_t;
+/** Create a new bit array that can hold <b>n_bits</b> bits. */
+static inline bitarray_t *
+bitarray_init_zero(unsigned int n_bits)
+{
+  /* round up to the next int. */
+  size_t sz = (n_bits+BITARRAY_MASK) >> BITARRAY_SHIFT;
+  return tor_calloc(sz, sizeof(unsigned int));
+}
+/** Expand <b>ba</b> from holding <b>n_bits_old</b> to <b>n_bits_new</b>,
+ * clearing all new bits.  Returns a possibly changed pointer to the
+ * bitarray. */
+static inline bitarray_t *
+bitarray_expand(bitarray_t *ba,
+                unsigned int n_bits_old, unsigned int n_bits_new)
+{
+  size_t sz_old = (n_bits_old+BITARRAY_MASK) >> BITARRAY_SHIFT;
+  size_t sz_new = (n_bits_new+BITARRAY_MASK) >> BITARRAY_SHIFT;
+  char *ptr;
+  if (sz_new <= sz_old)
+    return ba;
+  ptr = tor_reallocarray(ba, sz_new, sizeof(unsigned int));
+  /* This memset does nothing to the older excess bytes.  But they were
+   * already set to 0 by bitarry_init_zero. */
+  memset(ptr+sz_old*sizeof(unsigned int), 0,
+         (sz_new-sz_old)*sizeof(unsigned int));
+  return (bitarray_t*) ptr;
+}
+/** Free the bit array <b>ba</b>. */
+static inline void
+bitarray_free_(bitarray_t *ba)
+{
+  tor_free(ba);
+}
+#define bitarray_free(ba) FREE_AND_NULL(bitarray_t, bitarray_free_, (ba))
+
+/** Set the <b>bit</b>th bit in <b>b</b> to 1. */
+static inline void
+bitarray_set(bitarray_t *b, int bit)
+{
+  b[bit >> BITARRAY_SHIFT] |= (1u << (bit & BITARRAY_MASK));
+}
+/** Set the <b>bit</b>th bit in <b>b</b> to 0. */
+static inline void
+bitarray_clear(bitarray_t *b, int bit)
+{
+  b[bit >> BITARRAY_SHIFT] &= ~ (1u << (bit & BITARRAY_MASK));
+}
+/** Return true iff <b>bit</b>th bit in <b>b</b> is nonzero.  NOTE: does
+ * not necessarily return 1 on true. */
+static inline unsigned int
+bitarray_is_set(bitarray_t *b, int bit)
+{
+  return b[bit >> BITARRAY_SHIFT] & (1u << (bit & BITARRAY_MASK));
+}
+
+#endif /* !defined(TOR_CONTAINER_H) */
diff --git a/src/lib/container/bloomfilt.c b/src/lib/container/bloomfilt.c
new file mode 100644
index 0000000000..9aa9b1ee56
--- /dev/null
+++ b/src/lib/container/bloomfilt.c
@@ -0,0 +1,113 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file bloomfilt.c
+ * \brief Uses bitarray_t to implement a bloom filter.
+ **/
+
+#include <stdlib.h>
+
+#include "lib/malloc/malloc.h"
+#include "lib/container/bloomfilt.h"
+#include "lib/intmath/bits.h"
+#include "lib/log/util_bug.h"
+#include "siphash.h"
+
+/** How many bloom-filter bits we set per address. This is twice the
+ * BLOOMFILT_N_HASHES value, since we split the siphash output into two 32-bit
+ * values. */
+#define N_BITS_PER_ITEM (BLOOMFILT_N_HASHES * 2)
+
+struct bloomfilt_t {
+  /** siphash keys to make BLOOMFILT_N_HASHES independent hashes for each
+   * items. */
+  struct sipkey key[BLOOMFILT_N_HASHES];
+  bloomfilt_hash_fn hashfn; /**< Function used to generate hashes */
+  uint32_t mask; /**< One less than the number of bits in <b>ba</b>; always
+                  * one less than a power of two. */
+  bitarray_t *ba; /**< A bit array to implement the Bloom filter. */
+};
+
+#define BIT(set, n) ((n) & (set)->mask)
+
+/** Add the element <b>item</b> to <b>set</b>. */
+void
+bloomfilt_add(bloomfilt_t *set,
+              const void *item)
+{
+  int i;
+  for (i = 0; i < BLOOMFILT_N_HASHES; ++i) {
+    uint64_t h = set->hashfn(&set->key[i], item);
+    uint32_t high_bits = (uint32_t)(h >> 32);
+    uint32_t low_bits = (uint32_t)(h);
+    bitarray_set(set->ba, BIT(set, high_bits));
+    bitarray_set(set->ba, BIT(set, low_bits));
+  }
+}
+
+/** If <b>item</b> is in <b>set</b>, return nonzero.  Otherwise,
+ * <em>probably</em> return zero. */
+int
+bloomfilt_probably_contains(const bloomfilt_t *set,
+                            const void *item)
+{
+  int i, matches = 0;
+  for (i = 0; i < BLOOMFILT_N_HASHES; ++i) {
+    uint64_t h = set->hashfn(&set->key[i], item);
+    uint32_t high_bits = (uint32_t)(h >> 32);
+    uint32_t low_bits = (uint32_t)(h);
+    // Note that !! is necessary here, since bitarray_is_set does not
+    // necessarily return 1 on true.
+    matches += !! bitarray_is_set(set->ba, BIT(set, high_bits));
+    matches += !! bitarray_is_set(set->ba, BIT(set, low_bits));
+  }
+  return matches == N_BITS_PER_ITEM;
+}
+
+/** Return a newly allocated bloomfilt_t, optimized to hold a total of
+ * <b>max_elements</b> elements with a reasonably low false positive weight.
+ *
+ * Uses the siphash-based function <b>hashfn</b> to compute hard-to-collide
+ * functions of the items, and the key material <b>random_key</b> to
+ * key the hash.  There must be BLOOMFILT_KEY_LEN bytes in the supplied key.
+ **/
+bloomfilt_t *
+bloomfilt_new(int max_elements,
+              bloomfilt_hash_fn hashfn,
+              const uint8_t *random_key)
+{
+  /* The probability of false positives is about P=(1 - exp(-kn/m))^k, where k
+   * is the number of hash functions per entry, m is the bits in the array,
+   * and n is the number of elements inserted.  For us, k==4, n<=max_elements,
+   * and m==n_bits= approximately max_elements*32.  This gives
+   *   P<(1-exp(-4*n/(32*n)))^4 == (1-exp(1/-8))^4 == .00019
+   *
+   * It would be more optimal in space vs false positives to get this false
+   * positive rate by going for k==13, and m==18.5n, but we also want to
+   * conserve CPU, and k==13 is pretty big.
+   */
+  int n_bits = 1u << (tor_log2(max_elements)+5);
+  bloomfilt_t *r = tor_malloc(sizeof(bloomfilt_t));
+  r->mask = n_bits - 1;
+  r->ba = bitarray_init_zero(n_bits);
+
+  tor_assert(sizeof(r->key) == BLOOMFILT_KEY_LEN);
+  memcpy(r->key, random_key, sizeof(r->key));
+
+  r->hashfn = hashfn;
+
+  return r;
+}
+
+/** Free all storage held in <b>set</b>. */
+void
+bloomfilt_free_(bloomfilt_t *set)
+{
+  if (!set)
+    return;
+  bitarray_free(set->ba);
+  tor_free(set);
+}
diff --git a/src/lib/container/bloomfilt.h b/src/lib/container/bloomfilt.h
new file mode 100644
index 0000000000..0ce18bd3ec
--- /dev/null
+++ b/src/lib/container/bloomfilt.h
@@ -0,0 +1,41 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_BLOOMFILT_H
+#define TOR_BLOOMFILT_H
+
+/**
+ * \file bloomfilt.h
+ *
+ * \brief Header for bloomfilt.c
+ **/
+
+#include "orconfig.h"
+#include "lib/cc/torint.h"
+#include "lib/container/bitarray.h"
+
+/** A set of elements, implemented as a Bloom filter. */
+typedef struct bloomfilt_t bloomfilt_t;
+
+/** How many 64-bit siphash values to extract per item. */
+#define BLOOMFILT_N_HASHES 2
+
+/** How much key material do we need to randomize hashes? */
+#define BLOOMFILT_KEY_LEN (BLOOMFILT_N_HASHES * 16)
+
+struct sipkey;
+typedef uint64_t (*bloomfilt_hash_fn)(const struct sipkey *key,
+                                      const void *item);
+
+void bloomfilt_add(bloomfilt_t *set, const void *item);
+int bloomfilt_probably_contains(const bloomfilt_t *set, const void *item);
+
+bloomfilt_t *bloomfilt_new(int max_elements,
+                           bloomfilt_hash_fn hashfn,
+                           const uint8_t *random_key);
+void bloomfilt_free_(bloomfilt_t* set);
+#define bloomfilt_free(set) FREE_AND_NULL(bloomfilt_t, bloomfilt_free_, (set))
+
+#endif /* !defined(TOR_BLOOMFILT_H) */
diff --git a/src/lib/container/buffers.c b/src/lib/container/buffers.c
new file mode 100644
index 0000000000..67887f2f30
--- /dev/null
+++ b/src/lib/container/buffers.c
@@ -0,0 +1,932 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file buffers.c
+ * \brief Implements a generic buffer interface.
+ *
+ * A buf_t is a (fairly) opaque byte-oriented FIFO that can read to or flush
+ * from memory, sockets, file descriptors, TLS connections, or another buf_t.
+ * Buffers are implemented as linked lists of memory chunks.
+ *
+ * All socket-backed and TLS-based connection_t objects have a pair of
+ * buffers: one for incoming data, and one for outcoming data.  These are fed
+ * and drained from functions in connection.c, trigged by events that are
+ * monitored in main.c.
+ *
+ * This module only handles the buffer implementation itself. To use a buffer
+ * with the network, a compressor, or a TLS connection, see the other buffer_*
+ * modules.
+ **/
+
+#define BUFFERS_PRIVATE
+#include "orconfig.h"
+#include <stddef.h>
+#include "lib/container/buffers.h"
+#include "lib/cc/torint.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+#include "lib/time/compat_time.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+
+//#define PARANOIA
+
+#ifdef PARANOIA
+/** Helper: If PARANOIA is defined, assert that the buffer in local variable
+ * <b>buf</b> is well-formed. */
+#define check() STMT_BEGIN buf_assert_ok(buf); STMT_END
+#else
+#define check() STMT_NIL
+#endif /* defined(PARANOIA) */
+
+/* Implementation notes:
+ *
+ * After flirting with memmove, and dallying with ring-buffers, we're finally
+ * getting up to speed with the 1970s and implementing buffers as a linked
+ * list of small chunks.  Each buffer has such a list; data is removed from
+ * the head of the list, and added at the tail.  The list is singly linked,
+ * and the buffer keeps a pointer to the head and the tail.
+ *
+ * Every chunk, except the tail, contains at least one byte of data.  Data in
+ * each chunk is contiguous.
+ *
+ * When you need to treat the first N characters on a buffer as a contiguous
+ * string, use the buf_pullup function to make them so.  Don't do this more
+ * than necessary.
+ *
+ * The major free Unix kernels have handled buffers like this since, like,
+ * forever.
+ */
+
+/* Chunk manipulation functions */
+
+#define CHUNK_HEADER_LEN offsetof(chunk_t, mem[0])
+
+/* We leave this many NUL bytes at the end of the buffer. */
+#ifdef DISABLE_MEMORY_SENTINELS
+#define SENTINEL_LEN 0
+#else
+#define SENTINEL_LEN 4
+#endif
+
+/* Header size plus NUL bytes at the end */
+#define CHUNK_OVERHEAD (CHUNK_HEADER_LEN + SENTINEL_LEN)
+
+/** Return the number of bytes needed to allocate a chunk to hold
+ * <b>memlen</b> bytes. */
+#define CHUNK_ALLOC_SIZE(memlen) (CHUNK_OVERHEAD + (memlen))
+/** Return the number of usable bytes in a chunk allocated with
+ * malloc(<b>memlen</b>). */
+#define CHUNK_SIZE_WITH_ALLOC(memlen) ((memlen) - CHUNK_OVERHEAD)
+
+#define DEBUG_SENTINEL
+
+#if defined(DEBUG_SENTINEL) && !defined(DISABLE_MEMORY_SENTINELS)
+#define DBG_S(s) s
+#else
+#define DBG_S(s) (void)0
+#endif
+
+#ifdef DISABLE_MEMORY_SENTINELS
+#define CHUNK_SET_SENTINEL(chunk, alloclen) STMT_NIL
+#else
+#define CHUNK_SET_SENTINEL(chunk, alloclen) do {                        \
+    uint8_t *a = (uint8_t*) &(chunk)->mem[(chunk)->memlen];             \
+    DBG_S(uint8_t *b = &((uint8_t*)(chunk))[(alloclen)-SENTINEL_LEN]);  \
+    DBG_S(tor_assert(a == b));                                          \
+    memset(a,0,SENTINEL_LEN);                                           \
+  } while (0)
+#endif /* defined(DISABLE_MEMORY_SENTINELS) */
+
+/** Move all bytes stored in <b>chunk</b> to the front of <b>chunk</b>->mem,
+ * to free up space at the end. */
+static inline void
+chunk_repack(chunk_t *chunk)
+{
+  if (chunk->datalen && chunk->data != &chunk->mem[0]) {
+    memmove(chunk->mem, chunk->data, chunk->datalen);
+  }
+  chunk->data = &chunk->mem[0];
+}
+
+/** Keep track of total size of allocated chunks for consistency asserts */
+static size_t total_bytes_allocated_in_chunks = 0;
+static void
+buf_chunk_free_unchecked(chunk_t *chunk)
+{
+  if (!chunk)
+    return;
+#ifdef DEBUG_CHUNK_ALLOC
+  tor_assert(CHUNK_ALLOC_SIZE(chunk->memlen) == chunk->DBG_alloc);
+#endif
+  tor_assert(total_bytes_allocated_in_chunks >=
+             CHUNK_ALLOC_SIZE(chunk->memlen));
+  total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen);
+  tor_free(chunk);
+}
+static inline chunk_t *
+chunk_new_with_alloc_size(size_t alloc)
+{
+  chunk_t *ch;
+  ch = tor_malloc(alloc);
+  ch->next = NULL;
+  ch->datalen = 0;
+#ifdef DEBUG_CHUNK_ALLOC
+  ch->DBG_alloc = alloc;
+#endif
+  ch->memlen = CHUNK_SIZE_WITH_ALLOC(alloc);
+  total_bytes_allocated_in_chunks += alloc;
+  ch->data = &ch->mem[0];
+  CHUNK_SET_SENTINEL(ch, alloc);
+  return ch;
+}
+
+/** Expand <b>chunk</b> until it can hold <b>sz</b> bytes, and return a
+ * new pointer to <b>chunk</b>.  Old pointers are no longer valid. */
+static inline chunk_t *
+chunk_grow(chunk_t *chunk, size_t sz)
+{
+  off_t offset;
+  const size_t memlen_orig = chunk->memlen;
+  const size_t orig_alloc = CHUNK_ALLOC_SIZE(memlen_orig);
+  const size_t new_alloc = CHUNK_ALLOC_SIZE(sz);
+  tor_assert(sz > chunk->memlen);
+  offset = chunk->data - chunk->mem;
+  chunk = tor_realloc(chunk, new_alloc);
+  chunk->memlen = sz;
+  chunk->data = chunk->mem + offset;
+#ifdef DEBUG_CHUNK_ALLOC
+  tor_assert(chunk->DBG_alloc == orig_alloc);
+  chunk->DBG_alloc = new_alloc;
+#endif
+  total_bytes_allocated_in_chunks += new_alloc - orig_alloc;
+  CHUNK_SET_SENTINEL(chunk, new_alloc);
+  return chunk;
+}
+
+/** Every chunk should take up at least this many bytes. */
+#define MIN_CHUNK_ALLOC 256
+/** No chunk should take up more than this many bytes. */
+#define MAX_CHUNK_ALLOC 65536
+
+/** Return the allocation size we'd like to use to hold <b>target</b>
+ * bytes. */
+size_t
+buf_preferred_chunk_size(size_t target)
+{
+  tor_assert(target <= SIZE_T_CEILING - CHUNK_OVERHEAD);
+  if (CHUNK_ALLOC_SIZE(target) >= MAX_CHUNK_ALLOC)
+    return CHUNK_ALLOC_SIZE(target);
+  size_t sz = MIN_CHUNK_ALLOC;
+  while (CHUNK_SIZE_WITH_ALLOC(sz) < target) {
+    sz <<= 1;
+  }
+  return sz;
+}
+
+/** Collapse data from the first N chunks from <b>buf</b> into buf->head,
+ * growing it as necessary, until buf->head has the first <b>bytes</b> bytes
+ * of data from the buffer, or until buf->head has all the data in <b>buf</b>.
+ *
+ * Set *<b>head_out</b> to point to the first byte of available data, and
+ * *<b>len_out</b> to the number of bytes of data available at
+ * *<b>head_out</b>. Note that *<b>len_out</b> may be more or less than
+ * <b>bytes</b>, depending on the number of bytes available.
+ */
+void
+buf_pullup(buf_t *buf, size_t bytes, const char **head_out, size_t *len_out)
+{
+  chunk_t *dest, *src;
+  size_t capacity;
+  if (!buf->head) {
+    *head_out = NULL;
+    *len_out = 0;
+    return;
+  }
+
+  check();
+  if (buf->datalen < bytes)
+    bytes = buf->datalen;
+
+  capacity = bytes;
+  if (buf->head->datalen >= bytes) {
+    *head_out = buf->head->data;
+    *len_out = buf->head->datalen;
+    return;
+  }
+
+  if (buf->head->memlen >= capacity) {
+    /* We don't need to grow the first chunk, but we might need to repack it.*/
+    size_t needed = capacity - buf->head->datalen;
+    if (CHUNK_REMAINING_CAPACITY(buf->head) < needed)
+      chunk_repack(buf->head);
+    tor_assert(CHUNK_REMAINING_CAPACITY(buf->head) >= needed);
+  } else {
+    chunk_t *newhead;
+    size_t newsize;
+    /* We need to grow the chunk. */
+    chunk_repack(buf->head);
+    newsize = CHUNK_SIZE_WITH_ALLOC(buf_preferred_chunk_size(capacity));
+    newhead = chunk_grow(buf->head, newsize);
+    tor_assert(newhead->memlen >= capacity);
+    if (newhead != buf->head) {
+      if (buf->tail == buf->head)
+        buf->tail = newhead;
+      buf->head = newhead;
+    }
+  }
+
+  dest = buf->head;
+  while (dest->datalen < bytes) {
+    size_t n = bytes - dest->datalen;
+    src = dest->next;
+    tor_assert(src);
+    if (n >= src->datalen) {
+      memcpy(CHUNK_WRITE_PTR(dest), src->data, src->datalen);
+      dest->datalen += src->datalen;
+      dest->next = src->next;
+      if (buf->tail == src)
+        buf->tail = dest;
+      buf_chunk_free_unchecked(src);
+    } else {
+      memcpy(CHUNK_WRITE_PTR(dest), src->data, n);
+      dest->datalen += n;
+      src->data += n;
+      src->datalen -= n;
+      tor_assert(dest->datalen == bytes);
+    }
+  }
+
+  check();
+  *head_out = buf->head->data;
+  *len_out = buf->head->datalen;
+}
+
+#ifdef TOR_UNIT_TESTS
+/* Write sz bytes from cp into a newly allocated buffer buf.
+ * Returns NULL when passed a NULL cp or zero sz.
+ * Asserts on failure: only for use in unit tests.
+ * buf must be freed using buf_free(). */
+buf_t *
+buf_new_with_data(const char *cp, size_t sz)
+{
+  /* Validate arguments */
+  if (!cp || sz <= 0 || sz >= INT_MAX) {
+    return NULL;
+  }
+
+  tor_assert(sz < SSIZE_T_CEILING);
+
+  /* Allocate a buffer */
+  buf_t *buf = buf_new_with_capacity(sz);
+  tor_assert(buf);
+  buf_assert_ok(buf);
+  tor_assert(!buf->head);
+
+  /* Allocate a chunk that is sz bytes long */
+  buf->head = chunk_new_with_alloc_size(CHUNK_ALLOC_SIZE(sz));
+  buf->tail = buf->head;
+  tor_assert(buf->head);
+  buf_assert_ok(buf);
+  tor_assert(buf_allocation(buf) >= sz);
+
+  /* Copy the data and size the buffers */
+  tor_assert(sz <= buf_slack(buf));
+  tor_assert(sz <= CHUNK_REMAINING_CAPACITY(buf->head));
+  memcpy(&buf->head->mem[0], cp, sz);
+  buf->datalen = sz;
+  buf->head->datalen = sz;
+  buf->head->data = &buf->head->mem[0];
+  buf_assert_ok(buf);
+
+  /* Make sure everything is large enough */
+  tor_assert(buf_allocation(buf) >= sz);
+  tor_assert(buf_allocation(buf) >= buf_datalen(buf) + buf_slack(buf));
+  /* Does the buffer implementation allocate more than the requested size?
+   * (for example, by rounding up). If so, these checks will fail. */
+  tor_assert(buf_datalen(buf) == sz);
+  tor_assert(buf_slack(buf) == 0);
+
+  return buf;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/** Remove the first <b>n</b> bytes from buf. */
+void
+buf_drain(buf_t *buf, size_t n)
+{
+  tor_assert(buf->datalen >= n);
+  while (n) {
+    tor_assert(buf->head);
+    if (buf->head->datalen > n) {
+      buf->head->datalen -= n;
+      buf->head->data += n;
+      buf->datalen -= n;
+      return;
+    } else {
+      chunk_t *victim = buf->head;
+      n -= victim->datalen;
+      buf->datalen -= victim->datalen;
+      buf->head = victim->next;
+      if (buf->tail == victim)
+        buf->tail = NULL;
+      buf_chunk_free_unchecked(victim);
+    }
+  }
+  check();
+}
+
+/** Create and return a new buf with default chunk capacity <b>size</b>.
+ */
+buf_t *
+buf_new_with_capacity(size_t size)
+{
+  buf_t *b = buf_new();
+  b->default_chunk_size = buf_preferred_chunk_size(size);
+  return b;
+}
+
+/** Allocate and return a new buffer with default capacity. */
+buf_t *
+buf_new(void)
+{
+  buf_t *buf = tor_malloc_zero(sizeof(buf_t));
+  buf->magic = BUFFER_MAGIC;
+  buf->default_chunk_size = 4096;
+  return buf;
+}
+
+size_t
+buf_get_default_chunk_size(const buf_t *buf)
+{
+  return buf->default_chunk_size;
+}
+
+/** Remove all data from <b>buf</b>. */
+void
+buf_clear(buf_t *buf)
+{
+  chunk_t *chunk, *next;
+  buf->datalen = 0;
+  for (chunk = buf->head; chunk; chunk = next) {
+    next = chunk->next;
+    buf_chunk_free_unchecked(chunk);
+  }
+  buf->head = buf->tail = NULL;
+}
+
+/** Return the number of bytes stored in <b>buf</b> */
+MOCK_IMPL(size_t,
+buf_datalen, (const buf_t *buf))
+{
+  return buf->datalen;
+}
+
+/** Return the total length of all chunks used in <b>buf</b>. */
+size_t
+buf_allocation(const buf_t *buf)
+{
+  size_t total = 0;
+  const chunk_t *chunk;
+  for (chunk = buf->head; chunk; chunk = chunk->next) {
+    total += CHUNK_ALLOC_SIZE(chunk->memlen);
+  }
+  return total;
+}
+
+/** Return the number of bytes that can be added to <b>buf</b> without
+ * performing any additional allocation. */
+size_t
+buf_slack(const buf_t *buf)
+{
+  if (!buf->tail)
+    return 0;
+  else
+    return CHUNK_REMAINING_CAPACITY(buf->tail);
+}
+
+/** Release storage held by <b>buf</b>. */
+void
+buf_free_(buf_t *buf)
+{
+  if (!buf)
+    return;
+
+  buf_clear(buf);
+  buf->magic = 0xdeadbeef;
+  tor_free(buf);
+}
+
+/** Return a new copy of <b>in_chunk</b> */
+static chunk_t *
+chunk_copy(const chunk_t *in_chunk)
+{
+  chunk_t *newch = tor_memdup(in_chunk, CHUNK_ALLOC_SIZE(in_chunk->memlen));
+  total_bytes_allocated_in_chunks += CHUNK_ALLOC_SIZE(in_chunk->memlen);
+#ifdef DEBUG_CHUNK_ALLOC
+  newch->DBG_alloc = CHUNK_ALLOC_SIZE(in_chunk->memlen);
+#endif
+  newch->next = NULL;
+  if (in_chunk->data) {
+    off_t offset = in_chunk->data - in_chunk->mem;
+    newch->data = newch->mem + offset;
+  }
+  return newch;
+}
+
+/** Return a new copy of <b>buf</b> */
+buf_t *
+buf_copy(const buf_t *buf)
+{
+  chunk_t *ch;
+  buf_t *out = buf_new();
+  out->default_chunk_size = buf->default_chunk_size;
+  for (ch = buf->head; ch; ch = ch->next) {
+    chunk_t *newch = chunk_copy(ch);
+    if (out->tail) {
+      out->tail->next = newch;
+      out->tail = newch;
+    } else {
+      out->head = out->tail = newch;
+    }
+  }
+  out->datalen = buf->datalen;
+  return out;
+}
+
+/** Append a new chunk with enough capacity to hold <b>capacity</b> bytes to
+ * the tail of <b>buf</b>.  If <b>capped</b>, don't allocate a chunk bigger
+ * than MAX_CHUNK_ALLOC. */
+chunk_t *
+buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped)
+{
+  chunk_t *chunk;
+
+  if (CHUNK_ALLOC_SIZE(capacity) < buf->default_chunk_size) {
+    chunk = chunk_new_with_alloc_size(buf->default_chunk_size);
+  } else if (capped && CHUNK_ALLOC_SIZE(capacity) > MAX_CHUNK_ALLOC) {
+    chunk = chunk_new_with_alloc_size(MAX_CHUNK_ALLOC);
+  } else {
+    chunk = chunk_new_with_alloc_size(buf_preferred_chunk_size(capacity));
+  }
+
+  chunk->inserted_time = monotime_coarse_get_stamp();
+
+  if (buf->tail) {
+    tor_assert(buf->head);
+    buf->tail->next = chunk;
+    buf->tail = chunk;
+  } else {
+    tor_assert(!buf->head);
+    buf->head = buf->tail = chunk;
+  }
+  check();
+  return chunk;
+}
+
+/** Return the age of the oldest chunk in the buffer <b>buf</b>, in
+ * timestamp units.  Requires the current monotonic timestamp as its
+ * input <b>now</b>.
+ */
+uint32_t
+buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now)
+{
+  if (buf->head) {
+    return now - buf->head->inserted_time;
+  } else {
+    return 0;
+  }
+}
+
+size_t
+buf_get_total_allocation(void)
+{
+  return total_bytes_allocated_in_chunks;
+}
+
+/** Append <b>string_len</b> bytes from <b>string</b> to the end of
+ * <b>buf</b>.
+ *
+ * Return the new length of the buffer on success, -1 on failure.
+ */
+int
+buf_add(buf_t *buf, const char *string, size_t string_len)
+{
+  if (!string_len)
+    return (int)buf->datalen;
+  check();
+
+  if (BUG(buf->datalen >= INT_MAX))
+    return -1;
+  if (BUG(buf->datalen >= INT_MAX - string_len))
+    return -1;
+
+  while (string_len) {
+    size_t copy;
+    if (!buf->tail || !CHUNK_REMAINING_CAPACITY(buf->tail))
+      buf_add_chunk_with_capacity(buf, string_len, 1);
+
+    copy = CHUNK_REMAINING_CAPACITY(buf->tail);
+    if (copy > string_len)
+      copy = string_len;
+    memcpy(CHUNK_WRITE_PTR(buf->tail), string, copy);
+    string_len -= copy;
+    string += copy;
+    buf->datalen += copy;
+    buf->tail->datalen += copy;
+  }
+
+  check();
+  tor_assert(buf->datalen < INT_MAX);
+  return (int)buf->datalen;
+}
+
+/** Add a nul-terminated <b>string</b> to <b>buf</b>, not including the
+ * terminating NUL. */
+void
+buf_add_string(buf_t *buf, const char *string)
+{
+  buf_add(buf, string, strlen(string));
+}
+
+/** As tor_snprintf, but write the results into a buf_t */
+void
+buf_add_printf(buf_t *buf, const char *format, ...)
+{
+  va_list ap;
+  va_start(ap,format);
+  buf_add_vprintf(buf, format, ap);
+  va_end(ap);
+}
+
+/** As tor_vsnprintf, but write the results into a buf_t. */
+void
+buf_add_vprintf(buf_t *buf, const char *format, va_list args)
+{
+  /* XXXX Faster implementations are easy enough, but let's optimize later */
+  char *tmp;
+  tor_vasprintf(&tmp, format, args);
+  buf_add(buf, tmp, strlen(tmp));
+  tor_free(tmp);
+}
+
+/** Return a heap-allocated string containing the contents of <b>buf</b>, plus
+ * a NUL byte. If <b>sz_out</b> is provided, set *<b>sz_out</b> to the length
+ * of the returned string, not including the terminating NUL. */
+char *
+buf_extract(buf_t *buf, size_t *sz_out)
+{
+  tor_assert(buf);
+
+  size_t sz = buf_datalen(buf);
+  char *result;
+  result = tor_malloc(sz+1);
+  buf_peek(buf, result, sz);
+  result[sz] = 0;
+  if (sz_out)
+    *sz_out = sz;
+  return result;
+}
+
+/** Helper: copy the first <b>string_len</b> bytes from <b>buf</b>
+ * onto <b>string</b>.
+ */
+void
+buf_peek(const buf_t *buf, char *string, size_t string_len)
+{
+  chunk_t *chunk;
+
+  tor_assert(string);
+  /* make sure we don't ask for too much */
+  tor_assert(string_len <= buf->datalen);
+  /* buf_assert_ok(buf); */
+
+  chunk = buf->head;
+  while (string_len) {
+    size_t copy = string_len;
+    tor_assert(chunk);
+    if (chunk->datalen < copy)
+      copy = chunk->datalen;
+    memcpy(string, chunk->data, copy);
+    string_len -= copy;
+    string += copy;
+    chunk = chunk->next;
+  }
+}
+
+/** Remove <b>string_len</b> bytes from the front of <b>buf</b>, and store
+ * them into <b>string</b>.  Return the new buffer size.  <b>string_len</b>
+ * must be \<= the number of bytes on the buffer.
+ */
+int
+buf_get_bytes(buf_t *buf, char *string, size_t string_len)
+{
+  /* There must be string_len bytes in buf; write them onto string,
+   * then memmove buf back (that is, remove them from buf).
+   *
+   * Return the number of bytes still on the buffer. */
+
+  check();
+  buf_peek(buf, string, string_len);
+  buf_drain(buf, string_len);
+  check();
+  tor_assert(buf->datalen < INT_MAX);
+  return (int)buf->datalen;
+}
+
+/** Move up to *<b>buf_flushlen</b> bytes from <b>buf_in</b> to
+ * <b>buf_out</b>, and modify *<b>buf_flushlen</b> appropriately.
+ * Return the number of bytes actually copied.
+ */
+int
+buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen)
+{
+  /* We can do way better here, but this doesn't turn up in any profiles. */
+  char b[4096];
+  size_t cp, len;
+
+  if (BUG(buf_out->datalen >= INT_MAX || *buf_flushlen >= INT_MAX))
+    return -1;
+  if (BUG(buf_out->datalen >= INT_MAX - *buf_flushlen))
+    return -1;
+
+  len = *buf_flushlen;
+  if (len > buf_in->datalen)
+    len = buf_in->datalen;
+
+  cp = len; /* Remember the number of bytes we intend to copy. */
+  tor_assert(cp < INT_MAX);
+  while (len) {
+    /* This isn't the most efficient implementation one could imagine, since
+     * it does two copies instead of 1, but I kinda doubt that this will be
+     * critical path. */
+    size_t n = len > sizeof(b) ? sizeof(b) : len;
+    buf_get_bytes(buf_in, b, n);
+    buf_add(buf_out, b, n);
+    len -= n;
+  }
+  *buf_flushlen -= cp;
+  return (int)cp;
+}
+
+/** Moves all data from <b>buf_in</b> to <b>buf_out</b>, without copying.
+ */
+void
+buf_move_all(buf_t *buf_out, buf_t *buf_in)
+{
+  tor_assert(buf_out);
+  if (!buf_in)
+    return;
+  if (BUG(buf_out->datalen >= INT_MAX || buf_in->datalen >= INT_MAX))
+    return;
+  if (BUG(buf_out->datalen >= INT_MAX - buf_in->datalen))
+    return;
+
+  if (buf_out->head == NULL) {
+    buf_out->head = buf_in->head;
+    buf_out->tail = buf_in->tail;
+  } else {
+    buf_out->tail->next = buf_in->head;
+    buf_out->tail = buf_in->tail;
+  }
+
+  buf_out->datalen += buf_in->datalen;
+  buf_in->head = buf_in->tail = NULL;
+  buf_in->datalen = 0;
+}
+
+/** Internal structure: represents a position in a buffer. */
+typedef struct buf_pos_t {
+  const chunk_t *chunk; /**< Which chunk are we pointing to? */
+  int pos;/**< Which character inside the chunk's data are we pointing to? */
+  size_t chunk_pos; /**< Total length of all previous chunks. */
+} buf_pos_t;
+
+/** Initialize <b>out</b> to point to the first character of <b>buf</b>.*/
+static void
+buf_pos_init(const buf_t *buf, buf_pos_t *out)
+{
+  out->chunk = buf->head;
+  out->pos = 0;
+  out->chunk_pos = 0;
+}
+
+/** Advance <b>out</b> to the first appearance of <b>ch</b> at the current
+ * position of <b>out</b>, or later.  Return -1 if no instances are found;
+ * otherwise returns the absolute position of the character. */
+static off_t
+buf_find_pos_of_char(char ch, buf_pos_t *out)
+{
+  const chunk_t *chunk;
+  int pos;
+  tor_assert(out);
+  if (out->chunk) {
+    if (out->chunk->datalen) {
+      tor_assert(out->pos < (off_t)out->chunk->datalen);
+    } else {
+      tor_assert(out->pos == 0);
+    }
+  }
+  pos = out->pos;
+  for (chunk = out->chunk; chunk; chunk = chunk->next) {
+    char *cp = memchr(chunk->data+pos, ch, chunk->datalen - pos);
+    if (cp) {
+      out->chunk = chunk;
+      tor_assert(cp - chunk->data < INT_MAX);
+      out->pos = (int)(cp - chunk->data);
+      return out->chunk_pos + out->pos;
+    } else {
+      out->chunk_pos += chunk->datalen;
+      pos = 0;
+    }
+  }
+  return -1;
+}
+
+/** Advance <b>pos</b> by a single character, if there are any more characters
+ * in the buffer.  Returns 0 on success, -1 on failure. */
+static inline int
+buf_pos_inc(buf_pos_t *pos)
+{
+  tor_assert(pos->pos < INT_MAX - 1);
+  ++pos->pos;
+  if (pos->pos == (off_t)pos->chunk->datalen) {
+    if (!pos->chunk->next)
+      return -1;
+    pos->chunk_pos += pos->chunk->datalen;
+    pos->chunk = pos->chunk->next;
+    pos->pos = 0;
+  }
+  return 0;
+}
+
+/** Return true iff the <b>n</b>-character string in <b>s</b> appears
+ * (verbatim) at <b>pos</b>. */
+static int
+buf_matches_at_pos(const buf_pos_t *pos, const char *s, size_t n)
+{
+  buf_pos_t p;
+  if (!n)
+    return 1;
+
+  memcpy(&p, pos, sizeof(p));
+
+  while (1) {
+    char ch = p.chunk->data[p.pos];
+    if (ch != *s)
+      return 0;
+    ++s;
+    /* If we're out of characters that don't match, we match.  Check this
+     * _before_ we test incrementing pos, in case we're at the end of the
+     * string. */
+    if (--n == 0)
+      return 1;
+    if (buf_pos_inc(&p)<0)
+      return 0;
+  }
+}
+
+/** Return the first position in <b>buf</b> at which the <b>n</b>-character
+ * string <b>s</b> occurs, or -1 if it does not occur. */
+int
+buf_find_string_offset(const buf_t *buf, const char *s, size_t n)
+{
+  buf_pos_t pos;
+  buf_pos_init(buf, &pos);
+  while (buf_find_pos_of_char(*s, &pos) >= 0) {
+    if (buf_matches_at_pos(&pos, s, n)) {
+      tor_assert(pos.chunk_pos + pos.pos < INT_MAX);
+      return (int)(pos.chunk_pos + pos.pos);
+    } else {
+      if (buf_pos_inc(&pos)<0)
+        return -1;
+    }
+  }
+  return -1;
+}
+
+/** Return 1 iff <b>buf</b> starts with <b>cmd</b>. <b>cmd</b> must be a null
+ * terminated string, of no more than PEEK_BUF_STARTSWITH_MAX bytes. */
+int
+buf_peek_startswith(const buf_t *buf, const char *cmd)
+{
+  char tmp[PEEK_BUF_STARTSWITH_MAX];
+  size_t clen = strlen(cmd);
+  if (clen == 0)
+    return 1;
+  if (BUG(clen > sizeof(tmp)))
+    return 0;
+  if (buf->datalen < clen)
+    return 0;
+  buf_peek(buf, tmp, clen);
+  return fast_memeq(tmp, cmd, clen);
+}
+
+/** Return the index within <b>buf</b> at which <b>ch</b> first appears,
+ * or -1 if <b>ch</b> does not appear on buf. */
+static off_t
+buf_find_offset_of_char(buf_t *buf, char ch)
+{
+  chunk_t *chunk;
+  off_t offset = 0;
+  tor_assert(buf->datalen < INT_MAX);
+  for (chunk = buf->head; chunk; chunk = chunk->next) {
+    char *cp = memchr(chunk->data, ch, chunk->datalen);
+    if (cp)
+      return offset + (cp - chunk->data);
+    else
+      offset += chunk->datalen;
+  }
+  return -1;
+}
+
+/** Try to read a single LF-terminated line from <b>buf</b>, and write it
+ * (including the LF), NUL-terminated, into the *<b>data_len</b> byte buffer
+ * at <b>data_out</b>.  Set *<b>data_len</b> to the number of bytes in the
+ * line, not counting the terminating NUL.  Return 1 if we read a whole line,
+ * return 0 if we don't have a whole line yet, and return -1 if the line
+ * length exceeds *<b>data_len</b>.
+ */
+int
+buf_get_line(buf_t *buf, char *data_out, size_t *data_len)
+{
+  size_t sz;
+  off_t offset;
+
+  if (!buf->head)
+    return 0;
+
+  offset = buf_find_offset_of_char(buf, '\n');
+  if (offset < 0)
+    return 0;
+  sz = (size_t) offset;
+  if (sz+2 > *data_len) {
+    *data_len = sz + 2;
+    return -1;
+  }
+  buf_get_bytes(buf, data_out, sz+1);
+  data_out[sz+1] = '\0';
+  *data_len = sz+1;
+  return 1;
+}
+
+/** Set *<b>output</b> to contain a copy of the data in *<b>input</b> */
+int
+buf_set_to_copy(buf_t **output,
+                const buf_t *input)
+{
+  if (*output)
+    buf_free(*output);
+  *output = buf_copy(input);
+  return 0;
+}
+
+/** Log an error and exit if <b>buf</b> is corrupted.
+ */
+void
+buf_assert_ok(buf_t *buf)
+{
+  tor_assert(buf);
+  tor_assert(buf->magic == BUFFER_MAGIC);
+
+  if (! buf->head) {
+    tor_assert(!buf->tail);
+    tor_assert(buf->datalen == 0);
+  } else {
+    chunk_t *ch;
+    size_t total = 0;
+    tor_assert(buf->tail);
+    for (ch = buf->head; ch; ch = ch->next) {
+      total += ch->datalen;
+      tor_assert(ch->datalen <= ch->memlen);
+      tor_assert(ch->datalen < INT_MAX);
+      tor_assert(ch->data >= &ch->mem[0]);
+      tor_assert(ch->data <= &ch->mem[0]+ch->memlen);
+      if (ch->data == &ch->mem[0]+ch->memlen) {
+        /* LCOV_EXCL_START */
+        static int warned = 0;
+        if (! warned) {
+          log_warn(LD_BUG, "Invariant violation in buf.c related to #15083");
+          warned = 1;
+        }
+        /* LCOV_EXCL_STOP */
+      }
+      tor_assert(ch->data+ch->datalen <= &ch->mem[0] + ch->memlen);
+      if (!ch->next)
+        tor_assert(ch == buf->tail);
+    }
+    tor_assert(buf->datalen == total);
+  }
+}
diff --git a/src/lib/container/buffers.h b/src/lib/container/buffers.h
new file mode 100644
index 0000000000..c103b93a82
--- /dev/null
+++ b/src/lib/container/buffers.h
@@ -0,0 +1,122 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file buffers.h
+ *
+ * \brief Header file for buffers.c.
+ **/
+
+#ifndef TOR_BUFFERS_H
+#define TOR_BUFFERS_H
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+#include "lib/testsupport/testsupport.h"
+
+#include <stdarg.h>
+
+typedef struct buf_t buf_t;
+
+buf_t *buf_new(void);
+buf_t *buf_new_with_capacity(size_t size);
+size_t buf_get_default_chunk_size(const buf_t *buf);
+void buf_free_(buf_t *buf);
+#define buf_free(b) FREE_AND_NULL(buf_t, buf_free_, (b))
+void buf_clear(buf_t *buf);
+buf_t *buf_copy(const buf_t *buf);
+
+MOCK_DECL(size_t, buf_datalen, (const buf_t *buf));
+size_t buf_allocation(const buf_t *buf);
+size_t buf_slack(const buf_t *buf);
+
+uint32_t buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now);
+size_t buf_get_total_allocation(void);
+
+int buf_add(buf_t *buf, const char *string, size_t string_len);
+void buf_add_string(buf_t *buf, const char *string);
+void buf_add_printf(buf_t *buf, const char *format, ...)
+  CHECK_PRINTF(2, 3);
+void buf_add_vprintf(buf_t *buf, const char *format, va_list args)
+  CHECK_PRINTF(2, 0);
+int buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen);
+void buf_move_all(buf_t *buf_out, buf_t *buf_in);
+void buf_peek(const buf_t *buf, char *string, size_t string_len);
+void buf_drain(buf_t *buf, size_t n);
+int buf_get_bytes(buf_t *buf, char *string, size_t string_len);
+int buf_get_line(buf_t *buf, char *data_out, size_t *data_len);
+
+#define PEEK_BUF_STARTSWITH_MAX 16
+int buf_peek_startswith(const buf_t *buf, const char *cmd);
+
+int buf_set_to_copy(buf_t **output,
+                    const buf_t *input);
+
+void buf_assert_ok(buf_t *buf);
+
+int buf_find_string_offset(const buf_t *buf, const char *s, size_t n);
+void buf_pullup(buf_t *buf, size_t bytes,
+                const char **head_out, size_t *len_out);
+char *buf_extract(buf_t *buf, size_t *sz_out);
+
+#ifdef BUFFERS_PRIVATE
+#ifdef TOR_UNIT_TESTS
+buf_t *buf_new_with_data(const char *cp, size_t sz);
+#endif
+size_t buf_preferred_chunk_size(size_t target);
+
+#define DEBUG_CHUNK_ALLOC
+/** A single chunk on a buffer. */
+typedef struct chunk_t {
+  struct chunk_t *next; /**< The next chunk on the buffer. */
+  size_t datalen; /**< The number of bytes stored in this chunk */
+  size_t memlen; /**< The number of usable bytes of storage in <b>mem</b>. */
+#ifdef DEBUG_CHUNK_ALLOC
+  size_t DBG_alloc;
+#endif
+  char *data; /**< A pointer to the first byte of data stored in <b>mem</b>. */
+  uint32_t inserted_time; /**< Timestamp when this chunk was inserted. */
+  char mem[FLEXIBLE_ARRAY_MEMBER]; /**< The actual memory used for storage in
+                * this chunk. */
+} chunk_t;
+
+/** Magic value for buf_t.magic, to catch pointer errors. */
+#define BUFFER_MAGIC 0xB0FFF312u
+/** A resizeable buffer, optimized for reading and writing. */
+struct buf_t {
+  uint32_t magic; /**< Magic cookie for debugging: Must be set to
+                   *   BUFFER_MAGIC. */
+  size_t datalen; /**< How many bytes is this buffer holding right now? */
+  size_t default_chunk_size; /**< Don't allocate any chunks smaller than
+                              * this for this buffer. */
+  chunk_t *head; /**< First chunk in the list, or NULL for none. */
+  chunk_t *tail; /**< Last chunk in the list, or NULL for none. */
+};
+
+chunk_t *buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped);
+/** If a read onto the end of a chunk would be smaller than this number, then
+ * just start a new chunk. */
+#define MIN_READ_LEN 8
+
+/** Return the number of bytes that can be written onto <b>chunk</b> without
+ * running out of space. */
+static inline size_t
+CHUNK_REMAINING_CAPACITY(const chunk_t *chunk)
+{
+  return (chunk->mem + chunk->memlen) - (chunk->data + chunk->datalen);
+}
+
+/** Return the next character in <b>chunk</b> onto which data can be appended.
+ * If the chunk is full, this might be off the end of chunk->mem. */
+static inline char *
+CHUNK_WRITE_PTR(chunk_t *chunk)
+{
+  return chunk->data + chunk->datalen;
+}
+
+#endif /* defined(BUFFERS_PRIVATE) */
+
+#endif /* !defined(TOR_BUFFERS_H) */
diff --git a/src/common/handles.h b/src/lib/container/handles.h
index 1ee2322579..ca7c94559e 100644
--- a/src/common/handles.h
+++ b/src/lib/container/handles.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -50,8 +50,9 @@
 #define TOR_HANDLE_H
 
 #include "orconfig.h"
-#include "tor_queue.h"
-#include "util.h"
+
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
 
 #define HANDLE_ENTRY(name, structname)         \
   struct name ## _handle_head_t *handle_head
@@ -59,7 +60,7 @@
 #define HANDLE_DECL(name, structname, linkage)                          \
   typedef struct name ## _handle_t name ## _handle_t;                   \
   linkage  name ## _handle_t *name ## _handle_new(struct structname *object); \
-  linkage void name ## _handle_free(name ## _handle_t *);               \
+  linkage void name ## _handle_free_(name ## _handle_t *);              \
   linkage struct structname *name ## _handle_get(name ## _handle_t *);  \
   linkage void name ## _handles_clear(struct structname *object);
 
@@ -113,7 +114,7 @@
   }                                                                     \
                                                                         \
   linkage void                                                          \
-  name ## _handle_free(struct name ## _handle_t *ref)                   \
+  name ## _handle_free_(struct name ## _handle_t *ref)                   \
   {                                                                     \
     if (! ref) return;                                                  \
     name ## _handle_head_t *head = ref->head;                           \
@@ -149,5 +150,4 @@
     }                                                                   \
   }
 
-#endif /* TOR_HANDLE_H */
-
+#endif /* !defined(TOR_HANDLE_H) */
diff --git a/src/lib/container/include.am b/src/lib/container/include.am
new file mode 100644
index 0000000000..e6492098b5
--- /dev/null
+++ b/src/lib/container/include.am
@@ -0,0 +1,27 @@
+
+noinst_LIBRARIES += src/lib/libtor-container.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-container-testing.a
+endif
+
+src_lib_libtor_container_a_SOURCES =			\
+	src/lib/container/bloomfilt.c			\
+	src/lib/container/buffers.c			\
+	src/lib/container/map.c				\
+	src/lib/container/order.c			\
+	src/lib/container/smartlist.c
+
+src_lib_libtor_container_testing_a_SOURCES = \
+	$(src_lib_libtor_container_a_SOURCES)
+src_lib_libtor_container_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_container_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/container/bitarray.h			\
+	src/lib/container/bloomfilt.h			\
+	src/lib/container/buffers.h			\
+	src/lib/container/handles.h			\
+	src/lib/container/map.h				\
+	src/lib/container/order.h			\
+	src/lib/container/smartlist.h
diff --git a/src/lib/container/map.c b/src/lib/container/map.c
new file mode 100644
index 0000000000..d213ad50bf
--- /dev/null
+++ b/src/lib/container/map.c
@@ -0,0 +1,413 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file map.c
+ *
+ * \brief Hash-table implementations of a string-to-void* map, and of
+ * a digest-to-void* map.
+ **/
+
+#include "lib/container/map.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/defs/digest_sizes.h"
+#include "lib/string/util_string.h"
+#include "lib/malloc/malloc.h"
+
+#include "lib/log/util_bug.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "ht.h"
+
+/** Helper: Declare an entry type and a map type to implement a mapping using
+ * ht.h.  The map type will be called <b>maptype</b>.  The key part of each
+ * entry is declared using the C declaration <b>keydecl</b>.  All functions
+ * and types associated with the map get prefixed with <b>prefix</b> */
+#define DEFINE_MAP_STRUCTS(maptype, keydecl, prefix)      \
+  typedef struct prefix ## entry_t {                      \
+    HT_ENTRY(prefix ## entry_t) node;                     \
+    void *val;                                            \
+    keydecl;                                              \
+  } prefix ## entry_t;                                    \
+  struct maptype {                                        \
+    HT_HEAD(prefix ## impl, prefix ## entry_t) head;      \
+  }
+
+DEFINE_MAP_STRUCTS(strmap_t, char *key, strmap_);
+DEFINE_MAP_STRUCTS(digestmap_t, char key[DIGEST_LEN], digestmap_);
+DEFINE_MAP_STRUCTS(digest256map_t, uint8_t key[DIGEST256_LEN], digest256map_);
+
+/** Helper: compare strmap_entry_t objects by key value. */
+static inline int
+strmap_entries_eq(const strmap_entry_t *a, const strmap_entry_t *b)
+{
+  return !strcmp(a->key, b->key);
+}
+
+/** Helper: return a hash value for a strmap_entry_t. */
+static inline unsigned int
+strmap_entry_hash(const strmap_entry_t *a)
+{
+  return (unsigned) siphash24g(a->key, strlen(a->key));
+}
+
+/** Helper: compare digestmap_entry_t objects by key value. */
+static inline int
+digestmap_entries_eq(const digestmap_entry_t *a, const digestmap_entry_t *b)
+{
+  return tor_memeq(a->key, b->key, DIGEST_LEN);
+}
+
+/** Helper: return a hash value for a digest_map_t. */
+static inline unsigned int
+digestmap_entry_hash(const digestmap_entry_t *a)
+{
+  return (unsigned) siphash24g(a->key, DIGEST_LEN);
+}
+
+/** Helper: compare digestmap_entry_t objects by key value. */
+static inline int
+digest256map_entries_eq(const digest256map_entry_t *a,
+                        const digest256map_entry_t *b)
+{
+  return tor_memeq(a->key, b->key, DIGEST256_LEN);
+}
+
+/** Helper: return a hash value for a digest_map_t. */
+static inline unsigned int
+digest256map_entry_hash(const digest256map_entry_t *a)
+{
+  return (unsigned) siphash24g(a->key, DIGEST256_LEN);
+}
+
+HT_PROTOTYPE(strmap_impl, strmap_entry_t, node, strmap_entry_hash,
+             strmap_entries_eq)
+HT_GENERATE2(strmap_impl, strmap_entry_t, node, strmap_entry_hash,
+             strmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
+
+HT_PROTOTYPE(digestmap_impl, digestmap_entry_t, node, digestmap_entry_hash,
+             digestmap_entries_eq)
+HT_GENERATE2(digestmap_impl, digestmap_entry_t, node, digestmap_entry_hash,
+             digestmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
+
+HT_PROTOTYPE(digest256map_impl, digest256map_entry_t, node,
+             digest256map_entry_hash,
+             digest256map_entries_eq)
+HT_GENERATE2(digest256map_impl, digest256map_entry_t, node,
+             digest256map_entry_hash,
+             digest256map_entries_eq, 0.6, tor_reallocarray_, tor_free_)
+
+#define strmap_entry_free(ent) \
+  FREE_AND_NULL(strmap_entry_t, strmap_entry_free_, (ent))
+#define digestmap_entry_free(ent) \
+  FREE_AND_NULL(digestmap_entry_t, digestmap_entry_free_, (ent))
+#define digest256map_entry_free(ent) \
+  FREE_AND_NULL(digest256map_entry_t, digest256map_entry_free_, (ent))
+
+static inline void
+strmap_entry_free_(strmap_entry_t *ent)
+{
+  tor_free(ent->key);
+  tor_free(ent);
+}
+static inline void
+digestmap_entry_free_(digestmap_entry_t *ent)
+{
+  tor_free(ent);
+}
+static inline void
+digest256map_entry_free_(digest256map_entry_t *ent)
+{
+  tor_free(ent);
+}
+
+static inline void
+strmap_assign_tmp_key(strmap_entry_t *ent, const char *key)
+{
+  ent->key = (char*)key;
+}
+static inline void
+digestmap_assign_tmp_key(digestmap_entry_t *ent, const char *key)
+{
+  memcpy(ent->key, key, DIGEST_LEN);
+}
+static inline void
+digest256map_assign_tmp_key(digest256map_entry_t *ent, const uint8_t *key)
+{
+  memcpy(ent->key, key, DIGEST256_LEN);
+}
+static inline void
+strmap_assign_key(strmap_entry_t *ent, const char *key)
+{
+  ent->key = tor_strdup(key);
+}
+static inline void
+digestmap_assign_key(digestmap_entry_t *ent, const char *key)
+{
+  memcpy(ent->key, key, DIGEST_LEN);
+}
+static inline void
+digest256map_assign_key(digest256map_entry_t *ent, const uint8_t *key)
+{
+  memcpy(ent->key, key, DIGEST256_LEN);
+}
+
+/**
+ * Macro: implement all the functions for a map that are declared in
+ * map.h by the DECLARE_MAP_FNS() macro.  You must additionally define a
+ * prefix_entry_free_() function to free entries (and their keys), a
+ * prefix_assign_tmp_key() function to temporarily set a stack-allocated
+ * entry to hold a key, and a prefix_assign_key() function to set a
+ * heap-allocated entry to hold a key.
+ */
+#define IMPLEMENT_MAP_FNS(maptype, keytype, prefix)                     \
+  /** Create and return a new empty map. */                             \
+  MOCK_IMPL(maptype *,                                                  \
+  prefix##_new,(void))                                                  \
+  {                                                                     \
+    maptype *result;                                                    \
+    result = tor_malloc(sizeof(maptype));                               \
+    HT_INIT(prefix##_impl, &result->head);                              \
+    return result;                                                      \
+  }                                                                     \
+                                                                        \
+  /** Return the item from <b>map</b> whose key matches <b>key</b>, or  \
+   * NULL if no such value exists. */                                   \
+  void *                                                                \
+  prefix##_get(const maptype *map, const keytype key)                   \
+  {                                                                     \
+    prefix ##_entry_t *resolve;                                         \
+    prefix ##_entry_t search;                                           \
+    tor_assert(map);                                                    \
+    tor_assert(key);                                                    \
+    prefix ##_assign_tmp_key(&search, key);                             \
+    resolve = HT_FIND(prefix ##_impl, &map->head, &search);             \
+    if (resolve) {                                                      \
+      return resolve->val;                                              \
+    } else {                                                            \
+      return NULL;                                                      \
+    }                                                                   \
+  }                                                                     \
+                                                                        \
+  /** Add an entry to <b>map</b> mapping <b>key</b> to <b>val</b>;      \
+   * return the previous value, or NULL if no such value existed. */     \
+  void *                                                                \
+  prefix##_set(maptype *map, const keytype key, void *val)              \
+  {                                                                     \
+    prefix##_entry_t search;                                            \
+    void *oldval;                                                       \
+    tor_assert(map);                                                    \
+    tor_assert(key);                                                    \
+    tor_assert(val);                                                    \
+    prefix##_assign_tmp_key(&search, key);                              \
+    /* We a lot of our time in this function, so the code below is */   \
+    /* meant to optimize the check/alloc/set cycle by avoiding the two */\
+    /* trips to the hash table that we would do in the unoptimized */   \
+    /* version of this code. (Each of HT_INSERT and HT_FIND calls */     \
+    /* HT_SET_HASH and HT_FIND_P.) */                                   \
+    HT_FIND_OR_INSERT_(prefix##_impl, node, prefix##_entry_hash,        \
+                       &(map->head),                                    \
+                       prefix##_entry_t, &search, ptr,                  \
+                       {                                                \
+                         /* we found an entry. */                       \
+                         oldval = (*ptr)->val;                          \
+                         (*ptr)->val = val;                             \
+                         return oldval;                                 \
+                       },                                               \
+                       {                                                \
+                         /* We didn't find the entry. */                \
+                         prefix##_entry_t *newent =                     \
+                           tor_malloc_zero(sizeof(prefix##_entry_t));   \
+                         prefix##_assign_key(newent, key);              \
+                         newent->val = val;                             \
+                         HT_FOI_INSERT_(node, &(map->head),             \
+                            &search, newent, ptr);                      \
+                         return NULL;                                   \
+    });                                                                 \
+  }                                                                     \
+                                                                        \
+  /** Remove the value currently associated with <b>key</b> from the map. \
+   * Return the value if one was set, or NULL if there was no entry for \
+   * <b>key</b>.                                                        \
+   *                                                                    \
+   * Note: you must free any storage associated with the returned value. \
+   */                                                                   \
+  void *                                                                \
+  prefix##_remove(maptype *map, const keytype key)                      \
+  {                                                                     \
+    prefix##_entry_t *resolve;                                          \
+    prefix##_entry_t search;                                            \
+    void *oldval;                                                       \
+    tor_assert(map);                                                    \
+    tor_assert(key);                                                    \
+    prefix##_assign_tmp_key(&search, key);                              \
+    resolve = HT_REMOVE(prefix##_impl, &map->head, &search);            \
+    if (resolve) {                                                      \
+      oldval = resolve->val;                                            \
+      prefix##_entry_free(resolve);                                     \
+      return oldval;                                                    \
+    } else {                                                            \
+      return NULL;                                                      \
+    }                                                                   \
+  }                                                                     \
+                                                                        \
+  /** Return the number of elements in <b>map</b>. */                   \
+  int                                                                   \
+  prefix##_size(const maptype *map)                                     \
+  {                                                                     \
+    return HT_SIZE(&map->head);                                         \
+  }                                                                     \
+                                                                        \
+  /** Return true iff <b>map</b> has no entries. */                     \
+  int                                                                   \
+  prefix##_isempty(const maptype *map)                                  \
+  {                                                                     \
+    return HT_EMPTY(&map->head);                                        \
+  }                                                                     \
+                                                                        \
+  /** Assert that <b>map</b> is not corrupt. */                         \
+  void                                                                  \
+  prefix##_assert_ok(const maptype *map)                                \
+  {                                                                     \
+    tor_assert(!prefix##_impl_HT_REP_IS_BAD_(&map->head));              \
+  }                                                                     \
+                                                                        \
+  /** Remove all entries from <b>map</b>, and deallocate storage for    \
+   * those entries.  If free_val is provided, invoked it every value in \
+   * <b>map</b>. */                                                     \
+  MOCK_IMPL(void,                                                       \
+  prefix##_free_, (maptype *map, void (*free_val)(void*)))              \
+  {                                                                     \
+    prefix##_entry_t **ent, **next, *this;                              \
+    if (!map)                                                           \
+      return;                                                           \
+    for (ent = HT_START(prefix##_impl, &map->head); ent != NULL;        \
+         ent = next) {                                                  \
+      this = *ent;                                                      \
+      next = HT_NEXT_RMV(prefix##_impl, &map->head, ent);               \
+      if (free_val)                                                     \
+        free_val(this->val);                                            \
+      prefix##_entry_free(this);                                        \
+    }                                                                   \
+    tor_assert(HT_EMPTY(&map->head));                                   \
+    HT_CLEAR(prefix##_impl, &map->head);                                \
+    tor_free(map);                                                      \
+  }                                                                     \
+                                                                        \
+  /** return an <b>iterator</b> pointer to the front of a map.          \
+   *                                                                    \
+   * Iterator example:                                                  \
+   *                                                                    \
+   * \code                                                              \
+   * // uppercase values in "map", removing empty values.               \
+   *                                                                    \
+   * strmap_iter_t *iter;                                               \
+   * const char *key;                                                   \
+   * void *val;                                                         \
+   * char *cp;                                                          \
+   *                                                                    \
+   * for (iter = strmap_iter_init(map); !strmap_iter_done(iter); ) {    \
+   *    strmap_iter_get(iter, &key, &val);                              \
+   *    cp = (char*)val;                                                \
+   *    if (!*cp) {                                                     \
+   *       iter = strmap_iter_next_rmv(map,iter);                       \
+   *       free(val);                                                   \
+   *    } else {                                                        \
+   *       for (;*cp;cp++) *cp = TOR_TOUPPER(*cp);                      \
+   */                                                                   \
+  prefix##_iter_t *                                                     \
+  prefix##_iter_init(maptype *map)                                      \
+  {                                                                     \
+    tor_assert(map);                                                    \
+    return HT_START(prefix##_impl, &map->head);                         \
+  }                                                                     \
+                                                                        \
+  /** Advance <b>iter</b> a single step to the next entry, and return   \
+   * its new value. */                                                  \
+  prefix##_iter_t *                                                     \
+  prefix##_iter_next(maptype *map, prefix##_iter_t *iter)               \
+  {                                                                     \
+    tor_assert(map);                                                    \
+    tor_assert(iter);                                                   \
+    return HT_NEXT(prefix##_impl, &map->head, iter);                    \
+  }                                                                     \
+  /** Advance <b>iter</b> a single step to the next entry, removing the \
+   * current entry, and return its new value. */                        \
+  prefix##_iter_t *                                                     \
+  prefix##_iter_next_rmv(maptype *map, prefix##_iter_t *iter)           \
+  {                                                                     \
+    prefix##_entry_t *rmv;                                              \
+    tor_assert(map);                                                    \
+    tor_assert(iter);                                                   \
+    tor_assert(*iter);                                                  \
+    rmv = *iter;                                                        \
+    iter = HT_NEXT_RMV(prefix##_impl, &map->head, iter);                \
+    prefix##_entry_free(rmv);                                           \
+    return iter;                                                        \
+  }                                                                     \
+  /** Set *<b>keyp</b> and *<b>valp</b> to the current entry pointed    \
+   * to by iter. */                                                     \
+  void                                                                  \
+  prefix##_iter_get(prefix##_iter_t *iter, const keytype *keyp,         \
+                    void **valp)                                        \
+  {                                                                     \
+    tor_assert(iter);                                                   \
+    tor_assert(*iter);                                                  \
+    tor_assert(keyp);                                                   \
+    tor_assert(valp);                                                   \
+    *keyp = (*iter)->key;                                               \
+    *valp = (*iter)->val;                                               \
+  }                                                                     \
+  /** Return true iff <b>iter</b> has advanced past the last entry of   \
+   * <b>map</b>. */                                                     \
+  int                                                                   \
+  prefix##_iter_done(prefix##_iter_t *iter)                             \
+  {                                                                     \
+    return iter == NULL;                                                \
+  }
+
+IMPLEMENT_MAP_FNS(strmap_t, char *, strmap)
+IMPLEMENT_MAP_FNS(digestmap_t, char *, digestmap)
+IMPLEMENT_MAP_FNS(digest256map_t, uint8_t *, digest256map)
+
+/** Same as strmap_set, but first converts <b>key</b> to lowercase. */
+void *
+strmap_set_lc(strmap_t *map, const char *key, void *val)
+{
+  /* We could be a little faster by using strcasecmp instead, and a separate
+   * type, but I don't think it matters. */
+  void *v;
+  char *lc_key = tor_strdup(key);
+  tor_strlower(lc_key);
+  v = strmap_set(map,lc_key,val);
+  tor_free(lc_key);
+  return v;
+}
+
+/** Same as strmap_get, but first converts <b>key</b> to lowercase. */
+void *
+strmap_get_lc(const strmap_t *map, const char *key)
+{
+  void *v;
+  char *lc_key = tor_strdup(key);
+  tor_strlower(lc_key);
+  v = strmap_get(map,lc_key);
+  tor_free(lc_key);
+  return v;
+}
+
+/** Same as strmap_remove, but first converts <b>key</b> to lowercase */
+void *
+strmap_remove_lc(strmap_t *map, const char *key)
+{
+  void *v;
+  char *lc_key = tor_strdup(key);
+  tor_strlower(lc_key);
+  v = strmap_remove(map,lc_key);
+  tor_free(lc_key);
+  return v;
+}
diff --git a/src/lib/container/map.h b/src/lib/container/map.h
new file mode 100644
index 0000000000..a2d1b01d12
--- /dev/null
+++ b/src/lib/container/map.h
@@ -0,0 +1,261 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_MAP_H
+#define TOR_MAP_H
+
+/**
+ * \file map.h
+ *
+ * \brief Headers for map.c.
+ **/
+
+#include "lib/testsupport/testsupport.h"
+#include "lib/cc/torint.h"
+
+#include "siphash.h"
+
+#define DECLARE_MAP_FNS(maptype, keytype, prefix)                       \
+  typedef struct maptype maptype;                                       \
+  typedef struct prefix##entry_t *prefix##iter_t;                       \
+  MOCK_DECL(maptype*, prefix##new, (void));                             \
+  void* prefix##set(maptype *map, keytype key, void *val);              \
+  void* prefix##get(const maptype *map, keytype key);                   \
+  void* prefix##remove(maptype *map, keytype key);                      \
+  MOCK_DECL(void, prefix##free_, (maptype *map, void (*free_val)(void*))); \
+  int prefix##isempty(const maptype *map);                              \
+  int prefix##size(const maptype *map);                                 \
+  prefix##iter_t *prefix##iter_init(maptype *map);                      \
+  prefix##iter_t *prefix##iter_next(maptype *map, prefix##iter_t *iter); \
+  prefix##iter_t *prefix##iter_next_rmv(maptype *map, prefix##iter_t *iter); \
+  void prefix##iter_get(prefix##iter_t *iter, keytype *keyp, void **valp); \
+  int prefix##iter_done(prefix##iter_t *iter);                          \
+  void prefix##assert_ok(const maptype *map)
+
+/* Map from const char * to void *. Implemented with a hash table. */
+DECLARE_MAP_FNS(strmap_t, const char *, strmap_);
+/* Map from const char[DIGEST_LEN] to void *. Implemented with a hash table. */
+DECLARE_MAP_FNS(digestmap_t, const char *, digestmap_);
+/* Map from const uint8_t[DIGEST256_LEN] to void *. Implemented with a hash
+ * table. */
+DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_);
+
+#define MAP_FREE_AND_NULL(maptype, map, fn)     \
+  do {                                          \
+    maptype ## _free_((map), (fn));             \
+    (map) = NULL;                               \
+  } while (0)
+
+#define strmap_free(map, fn) MAP_FREE_AND_NULL(strmap, (map), (fn))
+#define digestmap_free(map, fn) MAP_FREE_AND_NULL(digestmap, (map), (fn))
+#define digest256map_free(map, fn) MAP_FREE_AND_NULL(digest256map, (map), (fn))
+
+#undef DECLARE_MAP_FNS
+
+/** Iterates over the key-value pairs in a map <b>map</b> in order.
+ * <b>prefix</b> is as for DECLARE_MAP_FNS (i.e., strmap_ or digestmap_).
+ * The map's keys and values are of type keytype and valtype respectively;
+ * each iteration assigns them to keyvar and valvar.
+ *
+ * Example use:
+ *   MAP_FOREACH(digestmap_, m, const char *, k, routerinfo_t *, r) {
+ *     // use k and r
+ *   } MAP_FOREACH_END.
+ */
+/* Unpacks to, approximately:
+ * {
+ *   digestmap_iter_t *k_iter;
+ *   for (k_iter = digestmap_iter_init(m); !digestmap_iter_done(k_iter);
+ *        k_iter = digestmap_iter_next(m, k_iter)) {
+ *     const char *k;
+ *     void *r_voidp;
+ *     routerinfo_t *r;
+ *     digestmap_iter_get(k_iter, &k, &r_voidp);
+ *     r = r_voidp;
+ *     // use k and r
+ *   }
+ * }
+ */
+#define MAP_FOREACH(prefix, map, keytype, keyvar, valtype, valvar)      \
+  STMT_BEGIN                                                            \
+    prefix##iter_t *keyvar##_iter;                                      \
+    for (keyvar##_iter = prefix##iter_init(map);                        \
+         !prefix##iter_done(keyvar##_iter);                             \
+         keyvar##_iter = prefix##iter_next(map, keyvar##_iter)) {       \
+      keytype keyvar;                                                   \
+      void *valvar##_voidp;                                             \
+      valtype valvar;                                                   \
+      prefix##iter_get(keyvar##_iter, &keyvar, &valvar##_voidp);        \
+      valvar = valvar##_voidp;
+
+/** As MAP_FOREACH, except allows members to be removed from the map
+ * during the iteration via MAP_DEL_CURRENT.  Example use:
+ *
+ * Example use:
+ *   MAP_FOREACH(digestmap_, m, const char *, k, routerinfo_t *, r) {
+ *      if (is_very_old(r))
+ *       MAP_DEL_CURRENT(k);
+ *   } MAP_FOREACH_END.
+ **/
+/* Unpacks to, approximately:
+ * {
+ *   digestmap_iter_t *k_iter;
+ *   int k_del=0;
+ *   for (k_iter = digestmap_iter_init(m); !digestmap_iter_done(k_iter);
+ *        k_iter = k_del ? digestmap_iter_next(m, k_iter)
+ *                       : digestmap_iter_next_rmv(m, k_iter)) {
+ *     const char *k;
+ *     void *r_voidp;
+ *     routerinfo_t *r;
+ *     k_del=0;
+ *     digestmap_iter_get(k_iter, &k, &r_voidp);
+ *     r = r_voidp;
+ *     if (is_very_old(r)) {
+ *       k_del = 1;
+ *     }
+ *   }
+ * }
+ */
+#define MAP_FOREACH_MODIFY(prefix, map, keytype, keyvar, valtype, valvar) \
+  STMT_BEGIN                                                            \
+    prefix##iter_t *keyvar##_iter;                                      \
+    int keyvar##_del=0;                                                 \
+    for (keyvar##_iter = prefix##iter_init(map);                        \
+         !prefix##iter_done(keyvar##_iter);                             \
+         keyvar##_iter = keyvar##_del ?                                 \
+           prefix##iter_next_rmv(map, keyvar##_iter) :                  \
+           prefix##iter_next(map, keyvar##_iter)) {                     \
+      keytype keyvar;                                                   \
+      void *valvar##_voidp;                                             \
+      valtype valvar;                                                   \
+      keyvar##_del=0;                                                   \
+      prefix##iter_get(keyvar##_iter, &keyvar, &valvar##_voidp);        \
+      valvar = valvar##_voidp;
+
+/** Used with MAP_FOREACH_MODIFY to remove the currently-iterated-upon
+ * member of the map.  */
+#define MAP_DEL_CURRENT(keyvar)                   \
+  STMT_BEGIN                                      \
+    keyvar##_del = 1;                             \
+  STMT_END
+
+/** Used to end a MAP_FOREACH() block. */
+#define MAP_FOREACH_END } STMT_END ;
+
+/** As MAP_FOREACH, but does not require declaration of prefix or keytype.
+ * Example use:
+ *   DIGESTMAP_FOREACH(m, k, routerinfo_t *, r) {
+ *     // use k and r
+ *   } DIGESTMAP_FOREACH_END.
+ */
+#define DIGESTMAP_FOREACH(map, keyvar, valtype, valvar)                 \
+  MAP_FOREACH(digestmap_, map, const char *, keyvar, valtype, valvar)
+
+/** As MAP_FOREACH_MODIFY, but does not require declaration of prefix or
+ * keytype.
+ * Example use:
+ *   DIGESTMAP_FOREACH_MODIFY(m, k, routerinfo_t *, r) {
+ *      if (is_very_old(r))
+ *       MAP_DEL_CURRENT(k);
+ *   } DIGESTMAP_FOREACH_END.
+ */
+#define DIGESTMAP_FOREACH_MODIFY(map, keyvar, valtype, valvar)          \
+  MAP_FOREACH_MODIFY(digestmap_, map, const char *, keyvar, valtype, valvar)
+/** Used to end a DIGESTMAP_FOREACH() block. */
+#define DIGESTMAP_FOREACH_END MAP_FOREACH_END
+
+#define DIGEST256MAP_FOREACH(map, keyvar, valtype, valvar)               \
+  MAP_FOREACH(digest256map_, map, const uint8_t *, keyvar, valtype, valvar)
+#define DIGEST256MAP_FOREACH_MODIFY(map, keyvar, valtype, valvar)       \
+  MAP_FOREACH_MODIFY(digest256map_, map, const uint8_t *,               \
+                     keyvar, valtype, valvar)
+#define DIGEST256MAP_FOREACH_END MAP_FOREACH_END
+
+#define STRMAP_FOREACH(map, keyvar, valtype, valvar)                 \
+  MAP_FOREACH(strmap_, map, const char *, keyvar, valtype, valvar)
+#define STRMAP_FOREACH_MODIFY(map, keyvar, valtype, valvar)          \
+  MAP_FOREACH_MODIFY(strmap_, map, const char *, keyvar, valtype, valvar)
+#define STRMAP_FOREACH_END MAP_FOREACH_END
+
+void* strmap_set_lc(strmap_t *map, const char *key, void *val);
+void* strmap_get_lc(const strmap_t *map, const char *key);
+void* strmap_remove_lc(strmap_t *map, const char *key);
+
+#define DECLARE_TYPED_DIGESTMAP_FNS(prefix, maptype, valtype)           \
+  typedef struct maptype maptype;                                       \
+  typedef struct prefix##iter_t *prefix##iter_t;                        \
+  ATTR_UNUSED static inline maptype*                                    \
+  prefix##new(void)                                                     \
+  {                                                                     \
+    return (maptype*)digestmap_new();                                   \
+  }                                                                     \
+  ATTR_UNUSED static inline digestmap_t*                                \
+  prefix##to_digestmap(maptype *map)                                    \
+  {                                                                     \
+    return (digestmap_t*)map;                                           \
+  }                                                                     \
+  ATTR_UNUSED static inline valtype*                                    \
+  prefix##get(maptype *map, const char *key)     \
+  {                                                                     \
+    return (valtype*)digestmap_get((digestmap_t*)map, key);             \
+  }                                                                     \
+  ATTR_UNUSED static inline valtype*                                    \
+  prefix##set(maptype *map, const char *key, valtype *val)              \
+  {                                                                     \
+    return (valtype*)digestmap_set((digestmap_t*)map, key, val);        \
+  }                                                                     \
+  ATTR_UNUSED static inline valtype*                                    \
+  prefix##remove(maptype *map, const char *key)                         \
+  {                                                                     \
+    return (valtype*)digestmap_remove((digestmap_t*)map, key);          \
+  }                                                                     \
+  ATTR_UNUSED static inline void                                        \
+  prefix##f##ree_(maptype *map, void (*free_val)(void*))                \
+  {                                                                     \
+    digestmap_free_((digestmap_t*)map, free_val);                       \
+  }                                                                     \
+  ATTR_UNUSED static inline int                                         \
+  prefix##isempty(maptype *map)                                         \
+  {                                                                     \
+    return digestmap_isempty((digestmap_t*)map);                        \
+  }                                                                     \
+  ATTR_UNUSED static inline int                                         \
+  prefix##size(maptype *map)                                            \
+  {                                                                     \
+    return digestmap_size((digestmap_t*)map);                           \
+  }                                                                     \
+  ATTR_UNUSED static inline                                             \
+  prefix##iter_t *prefix##iter_init(maptype *map)                       \
+  {                                                                     \
+    return (prefix##iter_t*) digestmap_iter_init((digestmap_t*)map);    \
+  }                                                                     \
+  ATTR_UNUSED static inline                                             \
+  prefix##iter_t *prefix##iter_next(maptype *map, prefix##iter_t *iter) \
+  {                                                                     \
+    return (prefix##iter_t*) digestmap_iter_next(                       \
+                       (digestmap_t*)map, (digestmap_iter_t*)iter);     \
+  }                                                                     \
+  ATTR_UNUSED static inline prefix##iter_t*                             \
+  prefix##iter_next_rmv(maptype *map, prefix##iter_t *iter)             \
+  {                                                                     \
+    return (prefix##iter_t*) digestmap_iter_next_rmv(                   \
+                       (digestmap_t*)map, (digestmap_iter_t*)iter);     \
+  }                                                                     \
+  ATTR_UNUSED static inline void                                        \
+  prefix##iter_get(prefix##iter_t *iter,                                \
+                   const char **keyp,                                   \
+                   valtype **valp)                                      \
+  {                                                                     \
+    void *v;                                                            \
+    digestmap_iter_get((digestmap_iter_t*) iter, keyp, &v);             \
+    *valp = v;                                                          \
+  }                                                                     \
+  ATTR_UNUSED static inline int                                         \
+  prefix##iter_done(prefix##iter_t *iter)                               \
+  {                                                                     \
+    return digestmap_iter_done((digestmap_iter_t*)iter);                \
+  }
+
+#endif /* !defined(TOR_CONTAINER_H) */
diff --git a/src/lib/container/order.c b/src/lib/container/order.c
new file mode 100644
index 0000000000..f6503a124e
--- /dev/null
+++ b/src/lib/container/order.c
@@ -0,0 +1,48 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file order.c
+ * \brief Functions for finding the n'th element of an array.
+ **/
+
+#include <stdlib.h>
+
+#include "lib/container/order.h"
+#include "lib/log/util_bug.h"
+
+/** Declare a function called <b>funcname</b> that acts as a find_nth_FOO
+ * function for an array of type <b>elt_t</b>*.
+ *
+ * NOTE: The implementation kind of sucks: It's O(n log n), whereas finding
+ * the kth element of an n-element list can be done in O(n).  Then again, this
+ * implementation is not in critical path, and it is obviously correct. */
+#define IMPLEMENT_ORDER_FUNC(funcname, elt_t)                   \
+  static int                                                    \
+  _cmp_ ## elt_t(const void *_a, const void *_b)                \
+  {                                                             \
+    const elt_t *a = _a, *b = _b;                               \
+    if (*a<*b)                                                  \
+      return -1;                                                \
+    else if (*a>*b)                                             \
+      return 1;                                                 \
+    else                                                        \
+      return 0;                                                 \
+  }                                                             \
+  elt_t                                                         \
+  funcname(elt_t *array, int n_elements, int nth)               \
+  {                                                             \
+    tor_assert(nth >= 0);                                       \
+    tor_assert(nth < n_elements);                               \
+    qsort(array, n_elements, sizeof(elt_t), _cmp_ ##elt_t);     \
+    return array[nth];                                          \
+  }
+
+IMPLEMENT_ORDER_FUNC(find_nth_int, int)
+IMPLEMENT_ORDER_FUNC(find_nth_time, time_t)
+IMPLEMENT_ORDER_FUNC(find_nth_double, double)
+IMPLEMENT_ORDER_FUNC(find_nth_uint32, uint32_t)
+IMPLEMENT_ORDER_FUNC(find_nth_int32, int32_t)
+IMPLEMENT_ORDER_FUNC(find_nth_long, long)
diff --git a/src/lib/container/order.h b/src/lib/container/order.h
new file mode 100644
index 0000000000..a176d6d8a6
--- /dev/null
+++ b/src/lib/container/order.h
@@ -0,0 +1,60 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_ORDER_H
+#define TOR_ORDER_H
+
+/**
+ * \file order.h
+ *
+ * \brief Header for order.c.
+ **/
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+
+/* These functions, given an <b>array</b> of <b>n_elements</b>, return the
+ * <b>nth</b> lowest element. <b>nth</b>=0 gives the lowest element;
+ * <b>n_elements</b>-1 gives the highest; and (<b>n_elements</b>-1) / 2 gives
+ * the median.  As a side effect, the elements of <b>array</b> are sorted. */
+int find_nth_int(int *array, int n_elements, int nth);
+time_t find_nth_time(time_t *array, int n_elements, int nth);
+double find_nth_double(double *array, int n_elements, int nth);
+int32_t find_nth_int32(int32_t *array, int n_elements, int nth);
+uint32_t find_nth_uint32(uint32_t *array, int n_elements, int nth);
+long find_nth_long(long *array, int n_elements, int nth);
+static inline int
+median_int(int *array, int n_elements)
+{
+  return find_nth_int(array, n_elements, (n_elements-1)/2);
+}
+static inline time_t
+median_time(time_t *array, int n_elements)
+{
+  return find_nth_time(array, n_elements, (n_elements-1)/2);
+}
+static inline double
+median_double(double *array, int n_elements)
+{
+  return find_nth_double(array, n_elements, (n_elements-1)/2);
+}
+static inline uint32_t
+median_uint32(uint32_t *array, int n_elements)
+{
+  return find_nth_uint32(array, n_elements, (n_elements-1)/2);
+}
+static inline int32_t
+median_int32(int32_t *array, int n_elements)
+{
+  return find_nth_int32(array, n_elements, (n_elements-1)/2);
+}
+
+static inline uint32_t
+third_quartile_uint32(uint32_t *array, int n_elements)
+{
+  return find_nth_uint32(array, n_elements, (n_elements*3)/4);
+}
+
+#endif /* !defined(TOR_CONTAINER_H) */
diff --git a/src/lib/container/smartlist.c b/src/lib/container/smartlist.c
new file mode 100644
index 0000000000..3ab2797d68
--- /dev/null
+++ b/src/lib/container/smartlist.c
@@ -0,0 +1,866 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file smartlist.c
+ *
+ * \brief Higher-level functions for the "smartlist" resizeable array
+ * abstraction.
+ *
+ * The functions declared here use higher-level functionality than those in
+ * smartlist_core.c, and handle things like smartlists of different types,
+ * sorting, searching, heap-structured smartlists, and other convenience
+ * functions.
+ **/
+
+#include "lib/container/smartlist.h"
+#include "lib/err/torerr.h"
+#include "lib/malloc/malloc.h"
+#include "lib/defs/digest_sizes.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/string/compat_string.h"
+#include "lib/string/util_string.h"
+#include "lib/string/printf.h"
+
+#include "lib/log/util_bug.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+/** Append the string produced by tor_asprintf(<b>pattern</b>, <b>...</b>)
+ * to <b>sl</b>. */
+void
+smartlist_add_asprintf(struct smartlist_t *sl, const char *pattern, ...)
+{
+  va_list ap;
+  va_start(ap, pattern);
+  smartlist_add_vasprintf(sl, pattern, ap);
+  va_end(ap);
+}
+
+/** va_list-based backend of smartlist_add_asprintf. */
+void
+smartlist_add_vasprintf(struct smartlist_t *sl, const char *pattern,
+                        va_list args)
+{
+  char *str = NULL;
+
+  tor_vasprintf(&str, pattern, args);
+  tor_assert(str != NULL);
+
+  smartlist_add(sl, str);
+}
+
+/** Reverse the order of the items in <b>sl</b>. */
+void
+smartlist_reverse(smartlist_t *sl)
+{
+  int i, j;
+  void *tmp;
+  tor_assert(sl);
+  for (i = 0, j = sl->num_used-1; i < j; ++i, --j) {
+    tmp = sl->list[i];
+    sl->list[i] = sl->list[j];
+    sl->list[j] = tmp;
+  }
+}
+
+/** If there are any strings in sl equal to element, remove and free them.
+ * Does not preserve order. */
+void
+smartlist_string_remove(smartlist_t *sl, const char *element)
+{
+  int i;
+  tor_assert(sl);
+  tor_assert(element);
+  for (i = 0; i < sl->num_used; ++i) {
+    if (!strcmp(element, sl->list[i])) {
+      tor_free(sl->list[i]);
+      sl->list[i] = sl->list[--sl->num_used]; /* swap with the end */
+      i--; /* so we process the new i'th element */
+      sl->list[sl->num_used] = NULL;
+    }
+  }
+}
+
+/** Return true iff <b>sl</b> has some element E such that
+ * !strcmp(E,<b>element</b>)
+ */
+int
+smartlist_contains_string(const smartlist_t *sl, const char *element)
+{
+  int i;
+  if (!sl) return 0;
+  for (i=0; i < sl->num_used; i++)
+    if (strcmp((const char*)sl->list[i],element)==0)
+      return 1;
+  return 0;
+}
+
+/** If <b>element</b> is equal to an element of <b>sl</b>, return that
+ * element's index.  Otherwise, return -1. */
+int
+smartlist_string_pos(const smartlist_t *sl, const char *element)
+{
+  int i;
+  if (!sl) return -1;
+  for (i=0; i < sl->num_used; i++)
+    if (strcmp((const char*)sl->list[i],element)==0)
+      return i;
+  return -1;
+}
+
+/** If <b>element</b> is the same pointer as an element of <b>sl</b>, return
+ * that element's index.  Otherwise, return -1. */
+int
+smartlist_pos(const smartlist_t *sl, const void *element)
+{
+  int i;
+  if (!sl) return -1;
+  for (i=0; i < sl->num_used; i++)
+    if (element == sl->list[i])
+      return i;
+  return -1;
+}
+
+/** Return true iff <b>sl</b> has some element E such that
+ * !strcasecmp(E,<b>element</b>)
+ */
+int
+smartlist_contains_string_case(const smartlist_t *sl, const char *element)
+{
+  int i;
+  if (!sl) return 0;
+  for (i=0; i < sl->num_used; i++)
+    if (strcasecmp((const char*)sl->list[i],element)==0)
+      return 1;
+  return 0;
+}
+
+/** Return true iff <b>sl</b> has some element E such that E is equal
+ * to the decimal encoding of <b>num</b>.
+ */
+int
+smartlist_contains_int_as_string(const smartlist_t *sl, int num)
+{
+  char buf[32]; /* long enough for 64-bit int, and then some. */
+  tor_snprintf(buf,sizeof(buf),"%d", num);
+  return smartlist_contains_string(sl, buf);
+}
+
+/** Return true iff the two lists contain the same strings in the same
+ * order, or if they are both NULL. */
+int
+smartlist_strings_eq(const smartlist_t *sl1, const smartlist_t *sl2)
+{
+  if (sl1 == NULL)
+    return sl2 == NULL;
+  if (sl2 == NULL)
+    return 0;
+  if (smartlist_len(sl1) != smartlist_len(sl2))
+    return 0;
+  SMARTLIST_FOREACH(sl1, const char *, cp1, {
+      const char *cp2 = smartlist_get(sl2, cp1_sl_idx);
+      if (strcmp(cp1, cp2))
+        return 0;
+    });
+  return 1;
+}
+
+/** Return true iff the two lists contain the same int pointer values in
+ * the same order, or if they are both NULL. */
+int
+smartlist_ints_eq(const smartlist_t *sl1, const smartlist_t *sl2)
+{
+  if (sl1 == NULL)
+    return sl2 == NULL;
+  if (sl2 == NULL)
+    return 0;
+  if (smartlist_len(sl1) != smartlist_len(sl2))
+    return 0;
+  SMARTLIST_FOREACH(sl1, int *, cp1, {
+      int *cp2 = smartlist_get(sl2, cp1_sl_idx);
+      if (*cp1 != *cp2)
+        return 0;
+    });
+  return 1;
+}
+
+/**
+ * Return true if there is shallow equality between smartlists -
+ * i.e. all indices correspond to exactly same object (pointer
+ * values are matching). Otherwise, return false.
+ */
+int
+smartlist_ptrs_eq(const smartlist_t *s1, const smartlist_t *s2)
+{
+  if (s1 == s2)
+    return 1;
+
+  // Note: pointers cannot both be NULL at this point, because
+  // above check.
+  if (s1 == NULL || s2 == NULL)
+    return 0;
+
+  if (smartlist_len(s1) != smartlist_len(s2))
+    return 0;
+
+  for (int i = 0; i < smartlist_len(s1); i++) {
+    if (smartlist_get(s1, i) != smartlist_get(s2, i))
+      return 0;
+  }
+
+  return 1;
+}
+
+/** Return true iff <b>sl</b> has some element E such that
+ * tor_memeq(E,<b>element</b>,DIGEST_LEN)
+ */
+int
+smartlist_contains_digest(const smartlist_t *sl, const char *element)
+{
+  int i;
+  if (!sl) return 0;
+  for (i=0; i < sl->num_used; i++)
+    if (tor_memeq((const char*)sl->list[i],element,DIGEST_LEN))
+      return 1;
+  return 0;
+}
+
+/** Return true iff some element E of sl2 has smartlist_contains(sl1,E).
+ */
+int
+smartlist_overlap(const smartlist_t *sl1, const smartlist_t *sl2)
+{
+  int i;
+  for (i=0; i < sl2->num_used; i++)
+    if (smartlist_contains(sl1, sl2->list[i]))
+      return 1;
+  return 0;
+}
+
+/** Remove every element E of sl1 such that !smartlist_contains(sl2,E).
+ * Does not preserve the order of sl1.
+ */
+void
+smartlist_intersect(smartlist_t *sl1, const smartlist_t *sl2)
+{
+  int i;
+  for (i=0; i < sl1->num_used; i++)
+    if (!smartlist_contains(sl2, sl1->list[i])) {
+      sl1->list[i] = sl1->list[--sl1->num_used]; /* swap with the end */
+      i--; /* so we process the new i'th element */
+      sl1->list[sl1->num_used] = NULL;
+    }
+}
+
+/** Remove every element E of sl1 such that smartlist_contains(sl2,E).
+ * Does not preserve the order of sl1.
+ */
+void
+smartlist_subtract(smartlist_t *sl1, const smartlist_t *sl2)
+{
+  int i;
+  for (i=0; i < sl2->num_used; i++)
+    smartlist_remove(sl1, sl2->list[i]);
+}
+
+/** Allocate and return a new string containing the concatenation of
+ * the elements of <b>sl</b>, in order, separated by <b>join</b>.  If
+ * <b>terminate</b> is true, also terminate the string with <b>join</b>.
+ * If <b>len_out</b> is not NULL, set <b>len_out</b> to the length of
+ * the returned string. Requires that every element of <b>sl</b> is
+ * NUL-terminated string.
+ */
+char *
+smartlist_join_strings(smartlist_t *sl, const char *join,
+                       int terminate, size_t *len_out)
+{
+  return smartlist_join_strings2(sl,join,strlen(join),terminate,len_out);
+}
+
+/** As smartlist_join_strings, but instead of separating/terminated with a
+ * NUL-terminated string <b>join</b>, uses the <b>join_len</b>-byte sequence
+ * at <b>join</b>.  (Useful for generating a sequence of NUL-terminated
+ * strings.)
+ */
+char *
+smartlist_join_strings2(smartlist_t *sl, const char *join,
+                        size_t join_len, int terminate, size_t *len_out)
+{
+  int i;
+  size_t n = 0;
+  char *r = NULL, *dst, *src;
+
+  tor_assert(sl);
+  tor_assert(join);
+
+  if (terminate)
+    n = join_len;
+
+  for (i = 0; i < sl->num_used; ++i) {
+    n += strlen(sl->list[i]);
+    if (i+1 < sl->num_used) /* avoid double-counting the last one */
+      n += join_len;
+  }
+  dst = r = tor_malloc(n+1);
+  for (i = 0; i < sl->num_used; ) {
+    for (src = sl->list[i]; *src; )
+      *dst++ = *src++;
+    if (++i < sl->num_used) {
+      memcpy(dst, join, join_len);
+      dst += join_len;
+    }
+  }
+  if (terminate) {
+    memcpy(dst, join, join_len);
+    dst += join_len;
+  }
+  *dst = '\0';
+
+  if (len_out)
+    *len_out = dst-r;
+  return r;
+}
+
+/** Sort the members of <b>sl</b> into an order defined by
+ * the ordering function <b>compare</b>, which returns less then 0 if a
+ * precedes b, greater than 0 if b precedes a, and 0 if a 'equals' b.
+ */
+void
+smartlist_sort(smartlist_t *sl, int (*compare)(const void **a, const void **b))
+{
+  if (!sl->num_used)
+    return;
+  qsort(sl->list, sl->num_used, sizeof(void*),
+        (int (*)(const void *,const void*))compare);
+}
+
+/** Given a smartlist <b>sl</b> sorted with the function <b>compare</b>,
+ * return the most frequent member in the list.  Break ties in favor of
+ * later elements.  If the list is empty, return NULL.  If count_out is
+ * non-null, set it to the count of the most frequent member.
+ */
+void *
+smartlist_get_most_frequent_(const smartlist_t *sl,
+                             int (*compare)(const void **a, const void **b),
+                             int *count_out)
+{
+  const void *most_frequent = NULL;
+  int most_frequent_count = 0;
+
+  const void *cur = NULL;
+  int i, count=0;
+
+  if (!sl->num_used) {
+    if (count_out)
+      *count_out = 0;
+    return NULL;
+  }
+  for (i = 0; i < sl->num_used; ++i) {
+    const void *item = sl->list[i];
+    if (cur && 0 == compare(&cur, &item)) {
+      ++count;
+    } else {
+      if (cur && count >= most_frequent_count) {
+        most_frequent = cur;
+        most_frequent_count = count;
+      }
+      cur = item;
+      count = 1;
+    }
+  }
+  if (cur && count >= most_frequent_count) {
+    most_frequent = cur;
+    most_frequent_count = count;
+  }
+  if (count_out)
+    *count_out = most_frequent_count;
+  return (void*)most_frequent;
+}
+
+/** Given a sorted smartlist <b>sl</b> and the comparison function used to
+ * sort it, remove all duplicate members.  If free_fn is provided, calls
+ * free_fn on each duplicate.  Otherwise, just removes them.  Preserves order.
+ */
+void
+smartlist_uniq(smartlist_t *sl,
+               int (*compare)(const void **a, const void **b),
+               void (*free_fn)(void *a))
+{
+  int i;
+  for (i=1; i < sl->num_used; ++i) {
+    if (compare((const void **)&(sl->list[i-1]),
+                (const void **)&(sl->list[i])) == 0) {
+      if (free_fn)
+        free_fn(sl->list[i]);
+      smartlist_del_keeporder(sl, i--);
+    }
+  }
+}
+
+/** Assuming the members of <b>sl</b> are in order, return a pointer to the
+ * member that matches <b>key</b>.  Ordering and matching are defined by a
+ * <b>compare</b> function that returns 0 on a match; less than 0 if key is
+ * less than member, and greater than 0 if key is greater then member.
+ */
+void *
+smartlist_bsearch(const smartlist_t *sl, const void *key,
+                  int (*compare)(const void *key, const void **member))
+{
+  int found, idx;
+  idx = smartlist_bsearch_idx(sl, key, compare, &found);
+  return found ? smartlist_get(sl, idx) : NULL;
+}
+
+/** Assuming the members of <b>sl</b> are in order, return the index of the
+ * member that matches <b>key</b>.  If no member matches, return the index of
+ * the first member greater than <b>key</b>, or smartlist_len(sl) if no member
+ * is greater than <b>key</b>.  Set <b>found_out</b> to true on a match, to
+ * false otherwise.  Ordering and matching are defined by a <b>compare</b>
+ * function that returns 0 on a match; less than 0 if key is less than member,
+ * and greater than 0 if key is greater then member.
+ */
+int
+smartlist_bsearch_idx(const smartlist_t *sl, const void *key,
+                      int (*compare)(const void *key, const void **member),
+                      int *found_out)
+{
+  int hi, lo, cmp, mid, len, diff;
+
+  tor_assert(sl);
+  tor_assert(compare);
+  tor_assert(found_out);
+
+  len = smartlist_len(sl);
+
+  /* Check for the trivial case of a zero-length list */
+  if (len == 0) {
+    *found_out = 0;
+    /* We already know smartlist_len(sl) is 0 in this case */
+    return 0;
+  }
+
+  /* Okay, we have a real search to do */
+  tor_assert(len > 0);
+  lo = 0;
+  hi = len - 1;
+
+  /*
+   * These invariants are always true:
+   *
+   * For all i such that 0 <= i < lo, sl[i] < key
+   * For all i such that hi < i <= len, sl[i] > key
+   */
+
+  while (lo <= hi) {
+    diff = hi - lo;
+    /*
+     * We want mid = (lo + hi) / 2, but that could lead to overflow, so
+     * instead diff = hi - lo (non-negative because of loop condition), and
+     * then hi = lo + diff, mid = (lo + lo + diff) / 2 = lo + (diff / 2).
+     */
+    mid = lo + (diff / 2);
+    cmp = compare(key, (const void**) &(sl->list[mid]));
+    if (cmp == 0) {
+      /* sl[mid] == key; we found it */
+      *found_out = 1;
+      return mid;
+    } else if (cmp > 0) {
+      /*
+       * key > sl[mid] and an index i such that sl[i] == key must
+       * have i > mid if it exists.
+       */
+
+      /*
+       * Since lo <= mid <= hi, hi can only decrease on each iteration (by
+       * being set to mid - 1) and hi is initially len - 1, mid < len should
+       * always hold, and this is not symmetric with the left end of list
+       * mid > 0 test below.  A key greater than the right end of the list
+       * should eventually lead to lo == hi == mid == len - 1, and then
+       * we set lo to len below and fall out to the same exit we hit for
+       * a key in the middle of the list but not matching.  Thus, we just
+       * assert for consistency here rather than handle a mid == len case.
+       */
+      tor_assert(mid < len);
+      /* Move lo to the element immediately after sl[mid] */
+      lo = mid + 1;
+    } else {
+      /* This should always be true in this case */
+      tor_assert(cmp < 0);
+
+      /*
+       * key < sl[mid] and an index i such that sl[i] == key must
+       * have i < mid if it exists.
+       */
+
+      if (mid > 0) {
+        /* Normal case, move hi to the element immediately before sl[mid] */
+        hi = mid - 1;
+      } else {
+        /* These should always be true in this case */
+        tor_assert(mid == lo);
+        tor_assert(mid == 0);
+        /*
+         * We were at the beginning of the list and concluded that every
+         * element e compares e > key.
+         */
+        *found_out = 0;
+        return 0;
+      }
+    }
+  }
+
+  /*
+   * lo > hi; we have no element matching key but we have elements falling
+   * on both sides of it.  The lo index points to the first element > key.
+   */
+  tor_assert(lo == hi + 1); /* All other cases should have been handled */
+  tor_assert(lo >= 0);
+  tor_assert(lo <= len);
+  tor_assert(hi >= 0);
+  tor_assert(hi <= len);
+
+  if (lo < len) {
+    cmp = compare(key, (const void **) &(sl->list[lo]));
+    tor_assert(cmp < 0);
+  } else {
+    cmp = compare(key, (const void **) &(sl->list[len-1]));
+    tor_assert(cmp > 0);
+  }
+
+  *found_out = 0;
+  return lo;
+}
+
+/** Helper: compare two const char **s. */
+static int
+compare_string_ptrs_(const void **_a, const void **_b)
+{
+  return strcmp((const char*)*_a, (const char*)*_b);
+}
+
+/** Sort a smartlist <b>sl</b> containing strings into lexically ascending
+ * order. */
+void
+smartlist_sort_strings(smartlist_t *sl)
+{
+  smartlist_sort(sl, compare_string_ptrs_);
+}
+
+/** Return the most frequent string in the sorted list <b>sl</b> */
+const char *
+smartlist_get_most_frequent_string(smartlist_t *sl)
+{
+  return smartlist_get_most_frequent(sl, compare_string_ptrs_);
+}
+
+/** Return the most frequent string in the sorted list <b>sl</b>.
+ * If <b>count_out</b> is provided, set <b>count_out</b> to the
+ * number of times that string appears.
+ */
+const char *
+smartlist_get_most_frequent_string_(smartlist_t *sl, int *count_out)
+{
+  return smartlist_get_most_frequent_(sl, compare_string_ptrs_, count_out);
+}
+
+/** Remove duplicate strings from a sorted list, and free them with tor_free().
+ */
+void
+smartlist_uniq_strings(smartlist_t *sl)
+{
+  smartlist_uniq(sl, compare_string_ptrs_, tor_free_);
+}
+
+/** Helper: compare two pointers. */
+static int
+compare_ptrs_(const void **_a, const void **_b)
+{
+  const void *a = *_a, *b = *_b;
+  if (a<b)
+    return -1;
+  else if (a==b)
+    return 0;
+  else
+    return 1;
+}
+
+/** Sort <b>sl</b> in ascending order of the pointers it contains. */
+void
+smartlist_sort_pointers(smartlist_t *sl)
+{
+  smartlist_sort(sl, compare_ptrs_);
+}
+
+/* Heap-based priority queue implementation for O(lg N) insert and remove.
+ * Recall that the heap property is that, for every index I, h[I] <
+ * H[LEFT_CHILD[I]] and h[I] < H[RIGHT_CHILD[I]].
+ *
+ * For us to remove items other than the topmost item, each item must store
+ * its own index within the heap.  When calling the pqueue functions, tell
+ * them about the offset of the field that stores the index within the item.
+ *
+ * Example:
+ *
+ *   typedef struct timer_t {
+ *     struct timeval tv;
+ *     int heap_index;
+ *   } timer_t;
+ *
+ *   static int compare(const void *p1, const void *p2) {
+ *     const timer_t *t1 = p1, *t2 = p2;
+ *     if (t1->tv.tv_sec < t2->tv.tv_sec) {
+ *        return -1;
+ *     } else if (t1->tv.tv_sec > t2->tv.tv_sec) {
+ *        return 1;
+ *     } else {
+ *        return t1->tv.tv_usec - t2->tv_usec;
+ *     }
+ *   }
+ *
+ *   void timer_heap_insert(smartlist_t *heap, timer_t *timer) {
+ *      smartlist_pqueue_add(heap, compare, offsetof(timer_t, heap_index),
+ *         timer);
+ *   }
+ *
+ *   void timer_heap_pop(smartlist_t *heap) {
+ *      return smartlist_pqueue_pop(heap, compare,
+ *         offsetof(timer_t, heap_index));
+ *   }
+ */
+
+/** @{ */
+/** Functions to manipulate heap indices to find a node's parent and children.
+ *
+ * For a 1-indexed array, we would use LEFT_CHILD[x] = 2*x and RIGHT_CHILD[x]
+ *   = 2*x + 1.  But this is C, so we have to adjust a little. */
+
+/* MAX_PARENT_IDX is the largest IDX in the smartlist which might have
+ * children whose indices fit inside an int.
+ * LEFT_CHILD(MAX_PARENT_IDX) == INT_MAX-2;
+ * RIGHT_CHILD(MAX_PARENT_IDX) == INT_MAX-1;
+ * LEFT_CHILD(MAX_PARENT_IDX + 1) == INT_MAX // impossible, see max list size.
+ */
+#define MAX_PARENT_IDX ((INT_MAX - 2) / 2)
+/* If this is true, then i is small enough to potentially have children
+ * in the smartlist, and it is save to use LEFT_CHILD/RIGHT_CHILD on it. */
+#define IDX_MAY_HAVE_CHILDREN(i) ((i) <= MAX_PARENT_IDX)
+#define LEFT_CHILD(i)  ( 2*(i) + 1 )
+#define RIGHT_CHILD(i) ( 2*(i) + 2 )
+#define PARENT(i)      ( ((i)-1) / 2 )
+/** }@ */
+
+/** @{ */
+/** Helper macros for heaps: Given a local variable <b>idx_field_offset</b>
+ * set to the offset of an integer index within the heap element structure,
+ * IDX_OF_ITEM(p) gives you the index of p, and IDXP(p) gives you a pointer to
+ * where p's index is stored.  Given additionally a local smartlist <b>sl</b>,
+ * UPDATE_IDX(i) sets the index of the element at <b>i</b> to the correct
+ * value (that is, to <b>i</b>).
+ */
+#define IDXP(p) ((int*)STRUCT_VAR_P(p, idx_field_offset))
+
+#define UPDATE_IDX(i)  do {                            \
+    void *updated = sl->list[i];                       \
+    *IDXP(updated) = i;                                \
+  } while (0)
+
+#define IDX_OF_ITEM(p) (*IDXP(p))
+/** @} */
+
+/** Helper. <b>sl</b> may have at most one violation of the heap property:
+ * the item at <b>idx</b> may be greater than one or both of its children.
+ * Restore the heap property. */
+static inline void
+smartlist_heapify(smartlist_t *sl,
+                  int (*compare)(const void *a, const void *b),
+                  int idx_field_offset,
+                  int idx)
+{
+  while (1) {
+    if (! IDX_MAY_HAVE_CHILDREN(idx)) {
+      /* idx is so large that it cannot have any children, since doing so
+       * would mean the smartlist was over-capacity. Therefore it cannot
+       * violate the heap property by being greater than a child (since it
+       * doesn't have any). */
+      return;
+    }
+
+    int left_idx = LEFT_CHILD(idx);
+    int best_idx;
+
+    if (left_idx >= sl->num_used)
+      return;
+    if (compare(sl->list[idx],sl->list[left_idx]) < 0)
+      best_idx = idx;
+    else
+      best_idx = left_idx;
+    if (left_idx+1 < sl->num_used &&
+        compare(sl->list[left_idx+1],sl->list[best_idx]) < 0)
+      best_idx = left_idx + 1;
+
+    if (best_idx == idx) {
+      return;
+    } else {
+      void *tmp = sl->list[idx];
+      sl->list[idx] = sl->list[best_idx];
+      sl->list[best_idx] = tmp;
+      UPDATE_IDX(idx);
+      UPDATE_IDX(best_idx);
+
+      idx = best_idx;
+    }
+  }
+}
+
+/** Insert <b>item</b> into the heap stored in <b>sl</b>, where order is
+ * determined by <b>compare</b> and the offset of the item in the heap is
+ * stored in an int-typed field at position <b>idx_field_offset</b> within
+ * item.
+ */
+void
+smartlist_pqueue_add(smartlist_t *sl,
+                     int (*compare)(const void *a, const void *b),
+                     int idx_field_offset,
+                     void *item)
+{
+  int idx;
+  smartlist_add(sl,item);
+  UPDATE_IDX(sl->num_used-1);
+
+  for (idx = sl->num_used - 1; idx; ) {
+    int parent = PARENT(idx);
+    if (compare(sl->list[idx], sl->list[parent]) < 0) {
+      void *tmp = sl->list[parent];
+      sl->list[parent] = sl->list[idx];
+      sl->list[idx] = tmp;
+      UPDATE_IDX(parent);
+      UPDATE_IDX(idx);
+      idx = parent;
+    } else {
+      return;
+    }
+  }
+}
+
+/** Remove and return the top-priority item from the heap stored in <b>sl</b>,
+ * where order is determined by <b>compare</b> and the item's position is
+ * stored at position <b>idx_field_offset</b> within the item.  <b>sl</b> must
+ * not be empty. */
+void *
+smartlist_pqueue_pop(smartlist_t *sl,
+                     int (*compare)(const void *a, const void *b),
+                     int idx_field_offset)
+{
+  void *top;
+  tor_assert(sl->num_used);
+
+  top = sl->list[0];
+  *IDXP(top)=-1;
+  if (--sl->num_used) {
+    sl->list[0] = sl->list[sl->num_used];
+    sl->list[sl->num_used] = NULL;
+    UPDATE_IDX(0);
+    smartlist_heapify(sl, compare, idx_field_offset, 0);
+  }
+  sl->list[sl->num_used] = NULL;
+  return top;
+}
+
+/** Remove the item <b>item</b> from the heap stored in <b>sl</b>,
+ * where order is determined by <b>compare</b> and the item's position is
+ * stored at position <b>idx_field_offset</b> within the item.  <b>sl</b> must
+ * not be empty. */
+void
+smartlist_pqueue_remove(smartlist_t *sl,
+                        int (*compare)(const void *a, const void *b),
+                        int idx_field_offset,
+                        void *item)
+{
+  int idx = IDX_OF_ITEM(item);
+  tor_assert(idx >= 0);
+  tor_assert(sl->list[idx] == item);
+  --sl->num_used;
+  *IDXP(item) = -1;
+  if (idx == sl->num_used) {
+    sl->list[sl->num_used] = NULL;
+    return;
+  } else {
+    sl->list[idx] = sl->list[sl->num_used];
+    sl->list[sl->num_used] = NULL;
+    UPDATE_IDX(idx);
+    smartlist_heapify(sl, compare, idx_field_offset, idx);
+  }
+}
+
+/** Assert that the heap property is correctly maintained by the heap stored
+ * in <b>sl</b>, where order is determined by <b>compare</b>. */
+void
+smartlist_pqueue_assert_ok(smartlist_t *sl,
+                           int (*compare)(const void *a, const void *b),
+                           int idx_field_offset)
+{
+  int i;
+  for (i = sl->num_used - 1; i >= 0; --i) {
+    if (i>0)
+      tor_assert(compare(sl->list[PARENT(i)], sl->list[i]) <= 0);
+    tor_assert(IDX_OF_ITEM(sl->list[i]) == i);
+  }
+}
+
+/** Helper: compare two DIGEST_LEN digests. */
+static int
+compare_digests_(const void **_a, const void **_b)
+{
+  return tor_memcmp((const char*)*_a, (const char*)*_b, DIGEST_LEN);
+}
+
+/** Sort the list of DIGEST_LEN-byte digests into ascending order. */
+void
+smartlist_sort_digests(smartlist_t *sl)
+{
+  smartlist_sort(sl, compare_digests_);
+}
+
+/** Remove duplicate digests from a sorted list, and free them with tor_free().
+ */
+void
+smartlist_uniq_digests(smartlist_t *sl)
+{
+  smartlist_uniq(sl, compare_digests_, tor_free_);
+}
+
+/** Helper: compare two DIGEST256_LEN digests. */
+static int
+compare_digests256_(const void **_a, const void **_b)
+{
+  return tor_memcmp((const char*)*_a, (const char*)*_b, DIGEST256_LEN);
+}
+
+/** Sort the list of DIGEST256_LEN-byte digests into ascending order. */
+void
+smartlist_sort_digests256(smartlist_t *sl)
+{
+  smartlist_sort(sl, compare_digests256_);
+}
+
+/** Return the most frequent member of the sorted list of DIGEST256_LEN
+ * digests in <b>sl</b> */
+const uint8_t *
+smartlist_get_most_frequent_digest256(smartlist_t *sl)
+{
+  return smartlist_get_most_frequent(sl, compare_digests256_);
+}
+
+/** Remove duplicate 256-bit digests from a sorted list, and free them with
+ * tor_free().
+ */
+void
+smartlist_uniq_digests256(smartlist_t *sl)
+{
+  smartlist_uniq(sl, compare_digests256_, tor_free_);
+}
diff --git a/src/lib/container/smartlist.h b/src/lib/container/smartlist.h
new file mode 100644
index 0000000000..77682db03e
--- /dev/null
+++ b/src/lib/container/smartlist.h
@@ -0,0 +1,168 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_SMARTLIST_H
+#define TOR_SMARTLIST_H
+
+/**
+ * \file smartlist.h
+ *
+ * \brief Header for smartlist.c
+ **/
+
+#include <stdarg.h>
+
+#include "lib/smartlist_core/smartlist_core.h"
+#include "lib/smartlist_core/smartlist_foreach.h"
+#include "lib/smartlist_core/smartlist_split.h"
+
+void smartlist_add_asprintf(struct smartlist_t *sl, const char *pattern, ...)
+  CHECK_PRINTF(2, 3);
+void smartlist_add_vasprintf(struct smartlist_t *sl, const char *pattern,
+                             va_list args)
+  CHECK_PRINTF(2, 0);
+void smartlist_reverse(smartlist_t *sl);
+void smartlist_string_remove(smartlist_t *sl, const char *element);
+int smartlist_contains_string(const smartlist_t *sl, const char *element);
+int smartlist_pos(const smartlist_t *sl, const void *element);
+int smartlist_string_pos(const smartlist_t *, const char *elt);
+int smartlist_contains_string_case(const smartlist_t *sl, const char *element);
+int smartlist_contains_int_as_string(const smartlist_t *sl, int num);
+int smartlist_strings_eq(const smartlist_t *sl1, const smartlist_t *sl2);
+int smartlist_contains_digest(const smartlist_t *sl, const char *element);
+int smartlist_ints_eq(const smartlist_t *sl1, const smartlist_t *sl2);
+int smartlist_overlap(const smartlist_t *sl1, const smartlist_t *sl2);
+void smartlist_intersect(smartlist_t *sl1, const smartlist_t *sl2);
+void smartlist_subtract(smartlist_t *sl1, const smartlist_t *sl2);
+
+int smartlist_ptrs_eq(const smartlist_t *s1,
+                      const smartlist_t *s2);
+
+void smartlist_sort(smartlist_t *sl,
+                    int (*compare)(const void **a, const void **b));
+void *smartlist_get_most_frequent_(const smartlist_t *sl,
+                    int (*compare)(const void **a, const void **b),
+                    int *count_out);
+#define smartlist_get_most_frequent(sl, compare) \
+  smartlist_get_most_frequent_((sl), (compare), NULL)
+void smartlist_uniq(smartlist_t *sl,
+                    int (*compare)(const void **a, const void **b),
+                    void (*free_fn)(void *elt));
+
+void smartlist_sort_strings(smartlist_t *sl);
+void smartlist_sort_digests(smartlist_t *sl);
+void smartlist_sort_digests256(smartlist_t *sl);
+void smartlist_sort_pointers(smartlist_t *sl);
+
+const char *smartlist_get_most_frequent_string(smartlist_t *sl);
+const char *smartlist_get_most_frequent_string_(smartlist_t *sl,
+                                                int *count_out);
+const uint8_t *smartlist_get_most_frequent_digest256(smartlist_t *sl);
+
+void smartlist_uniq_strings(smartlist_t *sl);
+void smartlist_uniq_digests(smartlist_t *sl);
+void smartlist_uniq_digests256(smartlist_t *sl);
+void *smartlist_bsearch(const smartlist_t *sl, const void *key,
+                        int (*compare)(const void *key, const void **member));
+int smartlist_bsearch_idx(const smartlist_t *sl, const void *key,
+                          int (*compare)(const void *key, const void **member),
+                          int *found_out);
+
+void smartlist_pqueue_add(smartlist_t *sl,
+                          int (*compare)(const void *a, const void *b),
+                          int idx_field_offset,
+                          void *item);
+void *smartlist_pqueue_pop(smartlist_t *sl,
+                           int (*compare)(const void *a, const void *b),
+                           int idx_field_offset);
+void smartlist_pqueue_remove(smartlist_t *sl,
+                             int (*compare)(const void *a, const void *b),
+                             int idx_field_offset,
+                             void *item);
+void smartlist_pqueue_assert_ok(smartlist_t *sl,
+                                int (*compare)(const void *a, const void *b),
+                                int idx_field_offset);
+
+char *smartlist_join_strings(smartlist_t *sl, const char *join, int terminate,
+                             size_t *len_out) ATTR_MALLOC;
+char *smartlist_join_strings2(smartlist_t *sl, const char *join,
+                              size_t join_len, int terminate, size_t *len_out)
+  ATTR_MALLOC;
+
+/* Helper: Given two lists of items, possibly of different types, such that
+ * both lists are sorted on some common field (as determined by a comparison
+ * expression <b>cmpexpr</b>), and such that one list (<b>sl1</b>) has no
+ * duplicates on the common field, loop through the lists in lockstep, and
+ * execute <b>unmatched_var2</b> on items in var2 that do not appear in
+ * var1.
+ *
+ * WARNING: It isn't safe to add remove elements from either list while the
+ * loop is in progress.
+ *
+ * Example use:
+ *  SMARTLIST_FOREACH_JOIN(routerstatus_list, routerstatus_t *, rs,
+ *                     routerinfo_list, routerinfo_t *, ri,
+ *                    tor_memcmp(rs->identity_digest, ri->identity_digest, 20),
+ *                     log_info(LD_GENERAL,"No match for %s", ri->nickname)) {
+ *    log_info(LD_GENERAL, "%s matches routerstatus %p", ri->nickname, rs);
+ * } SMARTLIST_FOREACH_JOIN_END(rs, ri);
+ **/
+/* The example above unpacks (approximately) to:
+ *  int rs_sl_idx = 0, rs_sl_len = smartlist_len(routerstatus_list);
+ *  int ri_sl_idx, ri_sl_len = smartlist_len(routerinfo_list);
+ *  int rs_ri_cmp;
+ *  routerstatus_t *rs;
+ *  routerinfo_t *ri;
+ *  for (; ri_sl_idx < ri_sl_len; ++ri_sl_idx) {
+ *    ri = smartlist_get(routerinfo_list, ri_sl_idx);
+ *    while (rs_sl_idx < rs_sl_len) {
+ *      rs = smartlist_get(routerstatus_list, rs_sl_idx);
+ *      rs_ri_cmp = tor_memcmp(rs->identity_digest, ri->identity_digest, 20);
+ *      if (rs_ri_cmp > 0) {
+ *        break;
+ *      } else if (rs_ri_cmp == 0) {
+ *        goto matched_ri;
+ *      } else {
+ *        ++rs_sl_idx;
+ *      }
+ *    }
+ *    log_info(LD_GENERAL,"No match for %s", ri->nickname);
+ *    continue;
+ *   matched_ri: {
+ *    log_info(LD_GENERAL,"%s matches with routerstatus %p",ri->nickname,rs);
+ *    }
+ *  }
+ */
+#define SMARTLIST_FOREACH_JOIN(sl1, type1, var1, sl2, type2, var2,      \
+                                cmpexpr, unmatched_var2)                \
+  STMT_BEGIN                                                            \
+  int var1 ## _sl_idx = 0, var1 ## _sl_len=(sl1)->num_used;             \
+  int var2 ## _sl_idx = 0, var2 ## _sl_len=(sl2)->num_used;             \
+  int var1 ## _ ## var2 ## _cmp;                                        \
+  type1 var1;                                                           \
+  type2 var2;                                                           \
+  for (; var2##_sl_idx < var2##_sl_len; ++var2##_sl_idx) {              \
+    var2 = (sl2)->list[var2##_sl_idx];                                  \
+    while (var1##_sl_idx < var1##_sl_len) {                             \
+      var1 = (sl1)->list[var1##_sl_idx];                                \
+      var1##_##var2##_cmp = (cmpexpr);                                  \
+      if (var1##_##var2##_cmp > 0) {                                    \
+        break;                                                          \
+      } else if (var1##_##var2##_cmp == 0) {                            \
+        goto matched_##var2;                                            \
+      } else {                                                          \
+        ++var1##_sl_idx;                                                \
+      }                                                                 \
+    }                                                                   \
+    /* Ran out of v1, or no match for var2. */                          \
+    unmatched_var2;                                                     \
+    continue;                                                           \
+    matched_##var2: ;                                                   \
+
+#define SMARTLIST_FOREACH_JOIN_END(var1, var2)  \
+  }                                             \
+  STMT_END
+
+#endif /* !defined(TOR_CONTAINER_H) */
diff --git a/src/lib/crypt_ops/.may_include b/src/lib/crypt_ops/.may_include
new file mode 100644
index 0000000000..a0fa4ec05c
--- /dev/null
+++ b/src/lib/crypt_ops/.may_include
@@ -0,0 +1,24 @@
+orconfig.h
+lib/arch/*.h
+lib/cc/*.h
+lib/container/*.h
+lib/crypt_ops/*.h
+lib/ctime/*.h
+lib/defs/*.h
+lib/encoding/*.h
+lib/fs/*.h
+lib/lock/*.h
+lib/malloc/*.h
+lib/intmath/*.h
+lib/sandbox/*.h
+lib/string/*.h
+lib/testsupport/testsupport.h
+lib/thread/*.h
+lib/log/*.h
+
+trunnel/pwbox.h
+
+keccak-tiny/*.h
+ed25519/*.h
+
+siphash.h
diff --git a/src/common/aes.h b/src/lib/crypt_ops/aes.h
index 1cda53f2fa..7c774062d9 100644
--- a/src/common/aes.h
+++ b/src/lib/crypt_ops/aes.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /* Implements a minimal interface to counter-mode AES. */
@@ -13,15 +13,19 @@
  * \brief Headers for aes.c
  */
 
+#include "lib/cc/torint.h"
+#include "lib/malloc/malloc.h"
+
 typedef struct aes_cnt_cipher aes_cnt_cipher_t;
 
 aes_cnt_cipher_t* aes_new_cipher(const uint8_t *key, const uint8_t *iv,
                                  int key_bits);
-void aes_cipher_free(aes_cnt_cipher_t *cipher);
+void aes_cipher_free_(aes_cnt_cipher_t *cipher);
+#define aes_cipher_free(cipher) \
+  FREE_AND_NULL(aes_cnt_cipher_t, aes_cipher_free_, (cipher))
 void aes_crypt_inplace(aes_cnt_cipher_t *cipher, char *data, size_t len);
 
 int evaluate_evp_for_aes(int force_value);
 int evaluate_ctr_for_aes(void);
 
-#endif
-
+#endif /* !defined(TOR_AES_H) */
diff --git a/src/lib/crypt_ops/aes_nss.c b/src/lib/crypt_ops/aes_nss.c
new file mode 100644
index 0000000000..4eda5e5902
--- /dev/null
+++ b/src/lib/crypt_ops/aes_nss.c
@@ -0,0 +1,106 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file aes_nss.c
+ * \brief Use NSS to implement AES_CTR.
+ **/
+
+#include "orconfig.h"
+#include "lib/crypt_ops/aes.h"
+#include "lib/crypt_ops/crypto_nss_mgt.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/log/util_bug.h"
+
+DISABLE_GCC_WARNING(strict-prototypes)
+#include <pk11pub.h>
+#include <secerr.h>
+ENABLE_GCC_WARNING(strict-prototypes)
+
+aes_cnt_cipher_t *
+aes_new_cipher(const uint8_t *key, const uint8_t *iv,
+               int key_bits)
+{
+  const CK_MECHANISM_TYPE ckm = CKM_AES_CTR;
+  SECItem keyItem = { .type = siBuffer,
+                      .data = (unsigned char *)key,
+                      .len = (key_bits / 8) };
+  CK_AES_CTR_PARAMS params;
+  params.ulCounterBits = 128;
+  memcpy(params.cb, iv, 16);
+  SECItem ivItem = { .type = siBuffer,
+                     .data = (unsigned char *)&params,
+                     .len = sizeof(params) };
+  PK11SlotInfo *slot = NULL;
+  PK11SymKey *keyObj = NULL;
+  SECItem *ivObj = NULL;
+  PK11Context *result = NULL;
+
+  slot = PK11_GetBestSlot(ckm, NULL);
+  if (!slot)
+    goto err;
+
+  keyObj = PK11_ImportSymKey(slot, ckm, PK11_OriginUnwrap,
+                             CKA_ENCRYPT, &keyItem, NULL);
+  if (!keyObj)
+    goto err;
+
+  ivObj = PK11_ParamFromIV(ckm, &ivItem);
+  if (!ivObj)
+    goto err;
+
+  PORT_SetError(SEC_ERROR_IO);
+  result = PK11_CreateContextBySymKey(ckm, CKA_ENCRYPT, keyObj, ivObj);
+
+ err:
+  memwipe(&params, 0, sizeof(params));
+  if (ivObj)
+    SECITEM_FreeItem(ivObj, PR_TRUE);
+  if (keyObj)
+    PK11_FreeSymKey(keyObj);
+  if (slot)
+    PK11_FreeSlot(slot);
+
+  tor_assert(result);
+  return (aes_cnt_cipher_t *)result;
+}
+
+void
+aes_cipher_free_(aes_cnt_cipher_t *cipher)
+{
+  if (!cipher)
+    return;
+  PK11_DestroyContext((PK11Context*) cipher, PR_TRUE);
+}
+
+void
+aes_crypt_inplace(aes_cnt_cipher_t *cipher, char *data_, size_t len_)
+{
+  tor_assert(len_ <= INT_MAX);
+
+  SECStatus s;
+  PK11Context *ctx = (PK11Context*)cipher;
+  unsigned char *data = (unsigned char *)data_;
+  int len = (int) len_;
+  int result_len = 0;
+
+  s = PK11_CipherOp(ctx, data, &result_len, len, data, len);
+  tor_assert(s == SECSuccess);
+  tor_assert(result_len == len);
+}
+
+int
+evaluate_evp_for_aes(int force_value)
+{
+  (void)force_value;
+  return 0;
+}
+
+int
+evaluate_ctr_for_aes(void)
+{
+  return 0;
+}
diff --git a/src/common/aes.c b/src/lib/crypt_ops/aes_openssl.c
index 8ab2d2fc6e..2f985d4512 100644
--- a/src/common/aes.c
+++ b/src/lib/crypt_ops/aes_openssl.c
@@ -1,23 +1,28 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file aes.c
- * \brief Implements a counter-mode stream cipher on top of AES.
+ * \file aes_openssl.c
+ * \brief Use OpenSSL to implement AES_CTR.
  **/
 
 #include "orconfig.h"
+#include "lib/crypt_ops/aes.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/log/util_bug.h"
+#include "lib/arch/bytes.h"
 
 #ifdef _WIN32 /*wrkard for dtls1.h >= 0.9.8m of "#include <winsock.h>"*/
   #include <winsock2.h>
   #include <ws2tcpip.h>
 #endif
 
+#include "lib/crypt_ops/compat_openssl.h"
 #include <openssl/opensslv.h>
-#include "crypto.h"
+#include "lib/crypt_ops/crypto_openssl_mgt.h"
 
 #if OPENSSL_VERSION_NUMBER < OPENSSL_V_SERIES(1,0,0)
 #error "We require OpenSSL >= 1.0.0"
@@ -25,7 +30,6 @@
 
 DISABLE_GCC_WARNING(redundant-decls)
 
-#include <assert.h>
 #include <stdlib.h>
 #include <string.h>
 #include <openssl/aes.h>
@@ -35,13 +39,11 @@ DISABLE_GCC_WARNING(redundant-decls)
 
 ENABLE_GCC_WARNING(redundant-decls)
 
-#include "compat.h"
-#include "aes.h"
-#include "util.h"
-#include "torlog.h"
-#include "di_ops.h"
+#include "lib/crypt_ops/aes.h"
+#include "lib/log/log.h"
+#include "lib/ctime/di_ops.h"
 
-#ifdef ANDROID
+#ifdef OPENSSL_NO_ENGINE
 /* Android's OpenSSL seems to have removed all of its Engine support. */
 #define DISABLE_ENGINES
 #endif
@@ -66,11 +68,11 @@ ENABLE_GCC_WARNING(redundant-decls)
 #elif OPENSSL_VERSION_NUMBER >= OPENSSL_V_NOPATCH(1,0,1) &&               \
   (defined(__i386) || defined(__i386__) || defined(_M_IX86) ||          \
    defined(__x86_64) || defined(__x86_64__) ||                          \
-   defined(_M_AMD64) || defined(_M_X64) || defined(__INTEL__))          \
+   defined(_M_AMD64) || defined(_M_X64) || defined(__INTEL__))
 
 #define USE_EVP_AES_CTR
 
-#endif
+#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_V_NOPATCH(1,1,0) || ... */
 
 /* We have 2 strategies for getting the AES block cipher: Via OpenSSL's
  * AES_encrypt function, or via OpenSSL's EVP_EncryptUpdate function.
@@ -110,12 +112,16 @@ aes_new_cipher(const uint8_t *key, const uint8_t *iv, int key_bits)
   return (aes_cnt_cipher_t *) cipher;
 }
 void
-aes_cipher_free(aes_cnt_cipher_t *cipher_)
+aes_cipher_free_(aes_cnt_cipher_t *cipher_)
 {
   if (!cipher_)
     return;
   EVP_CIPHER_CTX *cipher = (EVP_CIPHER_CTX *) cipher_;
+#ifdef OPENSSL_1_1_API
+  EVP_CIPHER_CTX_reset(cipher);
+#else
   EVP_CIPHER_CTX_cleanup(cipher);
+#endif
   EVP_CIPHER_CTX_free(cipher);
 }
 void
@@ -142,7 +148,7 @@ evaluate_ctr_for_aes(void)
 {
   return 0;
 }
-#else
+#else /* !(defined(USE_EVP_AES_CTR)) */
 
 /*======================================================================*/
 /* Interface to AES code, and counter implementation */
@@ -163,7 +169,7 @@ struct aes_cnt_cipher {
   uint32_t counter2;
   uint32_t counter1;
   uint32_t counter0;
-#endif
+#endif /* !defined(WORDS_BIGENDIAN) */
 
   union {
     /** The counter, in big-endian order, as bytes. */
@@ -212,7 +218,7 @@ evaluate_evp_for_aes(int force_val)
     log_info(LD_CRYPTO, "No AES engine found; using AES_* functions.");
     should_use_EVP = 0;
   }
-#endif
+#endif /* defined(DISABLE_ENGINES) */
 
   return 0;
 }
@@ -254,7 +260,7 @@ evaluate_ctr_for_aes(void)
     /* LCOV_EXCL_START */
     log_err(LD_CRYPTO, "This OpenSSL has a buggy version of counter mode; "
                   "quitting tor.");
-    exit(1);
+    exit(1); // exit ok: openssl is broken.
     /* LCOV_EXCL_STOP */
   }
   return 0;
@@ -312,7 +318,7 @@ aes_set_key(aes_cnt_cipher_t *cipher, const uint8_t *key, int key_bits)
   cipher->counter1 = 0;
   cipher->counter2 = 0;
   cipher->counter3 = 0;
-#endif
+#endif /* defined(USING_COUNTER_VARS) */
 
   memset(cipher->ctr_buf.buf, 0, sizeof(cipher->ctr_buf.buf));
 
@@ -324,7 +330,7 @@ aes_set_key(aes_cnt_cipher_t *cipher, const uint8_t *key, int key_bits)
 /** Release storage held by <b>cipher</b>
  */
 void
-aes_cipher_free(aes_cnt_cipher_t *cipher)
+aes_cipher_free_(aes_cnt_cipher_t *cipher)
 {
   if (!cipher)
     return;
@@ -341,7 +347,7 @@ aes_cipher_free(aes_cnt_cipher_t *cipher)
   STMT_END
 #else
 #define UPDATE_CTR_BUF(c, n)
-#endif
+#endif /* defined(USING_COUNTER_VARS) */
 
 /* Helper function to use EVP with openssl's counter-mode wrapper. */
 static void
@@ -392,13 +398,13 @@ static void
 aes_set_iv(aes_cnt_cipher_t *cipher, const uint8_t *iv)
 {
 #ifdef USING_COUNTER_VARS
-  cipher->counter3 = ntohl(get_uint32(iv));
-  cipher->counter2 = ntohl(get_uint32(iv+4));
-  cipher->counter1 = ntohl(get_uint32(iv+8));
-  cipher->counter0 = ntohl(get_uint32(iv+12));
-#endif
+  cipher->counter3 = tor_ntohl(get_uint32(iv));
+  cipher->counter2 = tor_ntohl(get_uint32(iv+4));
+  cipher->counter1 = tor_ntohl(get_uint32(iv+8));
+  cipher->counter0 = tor_ntohl(get_uint32(iv+12));
+#endif /* defined(USING_COUNTER_VARS) */
   cipher->pos = 0;
   memcpy(cipher->ctr_buf.buf, iv, 16);
 }
 
-#endif
+#endif /* defined(USE_EVP_AES_CTR) */
diff --git a/src/common/compat_openssl.h b/src/lib/crypt_ops/compat_openssl.h
index 1bfe188075..9c10386c34 100644
--- a/src/common/compat_openssl.h
+++ b/src/lib/crypt_ops/compat_openssl.h
@@ -1,18 +1,23 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_COMPAT_OPENSSL_H
 #define TOR_COMPAT_OPENSSL_H
 
+#include "orconfig.h"
+
+#ifdef ENABLE_OPENSSL
+
 #include <openssl/opensslv.h>
+#include "lib/crypt_ops/crypto_openssl_mgt.h"
 
 /**
  * \file compat_openssl.h
  *
- * \brief compatability definitions for working with different openssl forks
+ * \brief compatibility definitions for working with different openssl forks
  **/
 
 #if !defined(LIBRESSL_VERSION_NUMBER) && \
@@ -25,10 +30,13 @@
 /* We define this macro if we're trying to build with the majorly refactored
  * API in OpenSSL 1.1 */
 #define OPENSSL_1_1_API
+#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) && ... */
+
+#ifndef OPENSSL_VERSION
+#define OPENSSL_VERSION SSLEAY_VERSION
 #endif
 
 #ifndef OPENSSL_1_1_API
-#define OPENSSL_VERSION SSLEAY_VERSION
 #define OpenSSL_version(v) SSLeay_version(v)
 #define OpenSSL_version_num() SSLeay()
 #define RAND_OpenSSL() RAND_SSLeay()
@@ -37,11 +45,13 @@
    ((st) == SSL3_ST_SW_SRVR_HELLO_B))
 #define OSSL_HANDSHAKE_STATE int
 #define CONST_IF_OPENSSL_1_1_API
-#else
+#else /* !(!defined(OPENSSL_1_1_API)) */
 #define STATE_IS_SW_SERVER_HELLO(st) \
   ((st) == TLS_ST_SW_SRVR_HELLO)
 #define CONST_IF_OPENSSL_1_1_API const
-#endif
+#endif /* !defined(OPENSSL_1_1_API) */
 
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
+
+#endif /* !defined(TOR_COMPAT_OPENSSL_H) */
 
diff --git a/src/lib/crypt_ops/crypto_cipher.c b/src/lib/crypt_ops/crypto_cipher.c
new file mode 100644
index 0000000000..7bc2edad54
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_cipher.c
@@ -0,0 +1,190 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_cipher.c
+ * \brief Symmetric cryptography (low-level) with AES.
+ **/
+
+#include "orconfig.h"
+
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/cc/torint.h"
+#include "lib/crypt_ops/aes.h"
+
+#include <string.h>
+
+/** Allocate and return a new symmetric cipher using the provided key and iv.
+ * The key is <b>bits</b> bits long; the IV is CIPHER_IV_LEN bytes.  Both
+ * must be provided. Key length must be 128, 192, or 256 */
+crypto_cipher_t *
+crypto_cipher_new_with_iv_and_bits(const uint8_t *key,
+                                   const uint8_t *iv,
+                                   int bits)
+{
+  tor_assert(key);
+  tor_assert(iv);
+
+  return aes_new_cipher((const uint8_t*)key, (const uint8_t*)iv, bits);
+}
+
+/** Allocate and return a new symmetric cipher using the provided key and iv.
+ * The key is CIPHER_KEY_LEN bytes; the IV is CIPHER_IV_LEN bytes.  Both
+ * must be provided.
+ */
+crypto_cipher_t *
+crypto_cipher_new_with_iv(const char *key, const char *iv)
+{
+  return crypto_cipher_new_with_iv_and_bits((uint8_t*)key, (uint8_t*)iv,
+                                            128);
+}
+
+/** Return a new crypto_cipher_t with the provided <b>key</b> and an IV of all
+ * zero bytes and key length <b>bits</b>.  Key length must be 128, 192, or
+ * 256. */
+crypto_cipher_t *
+crypto_cipher_new_with_bits(const char *key, int bits)
+{
+  char zeroiv[CIPHER_IV_LEN];
+  memset(zeroiv, 0, sizeof(zeroiv));
+  return crypto_cipher_new_with_iv_and_bits((uint8_t*)key, (uint8_t*)zeroiv,
+                                            bits);
+}
+
+/** Return a new crypto_cipher_t with the provided <b>key</b> (of
+ * CIPHER_KEY_LEN bytes) and an IV of all zero bytes.  */
+crypto_cipher_t *
+crypto_cipher_new(const char *key)
+{
+  return crypto_cipher_new_with_bits(key, 128);
+}
+
+/** Free a symmetric cipher.
+ */
+void
+crypto_cipher_free_(crypto_cipher_t *env)
+{
+  if (!env)
+    return;
+
+  aes_cipher_free(env);
+}
+
+/* symmetric crypto */
+
+/** Encrypt <b>fromlen</b> bytes from <b>from</b> using the cipher
+ * <b>env</b>; on success, store the result to <b>to</b> and return 0.
+ * Does not check for failure.
+ */
+int
+crypto_cipher_encrypt(crypto_cipher_t *env, char *to,
+                      const char *from, size_t fromlen)
+{
+  tor_assert(env);
+  tor_assert(env);
+  tor_assert(from);
+  tor_assert(fromlen);
+  tor_assert(to);
+  tor_assert(fromlen < SIZE_T_CEILING);
+
+  memcpy(to, from, fromlen);
+  aes_crypt_inplace(env, to, fromlen);
+  return 0;
+}
+
+/** Decrypt <b>fromlen</b> bytes from <b>from</b> using the cipher
+ * <b>env</b>; on success, store the result to <b>to</b> and return 0.
+ * Does not check for failure.
+ */
+int
+crypto_cipher_decrypt(crypto_cipher_t *env, char *to,
+                      const char *from, size_t fromlen)
+{
+  tor_assert(env);
+  tor_assert(from);
+  tor_assert(to);
+  tor_assert(fromlen < SIZE_T_CEILING);
+
+  memcpy(to, from, fromlen);
+  aes_crypt_inplace(env, to, fromlen);
+  return 0;
+}
+
+/** Encrypt <b>len</b> bytes on <b>from</b> using the cipher in <b>env</b>;
+ * on success. Does not check for failure.
+ */
+void
+crypto_cipher_crypt_inplace(crypto_cipher_t *env, char *buf, size_t len)
+{
+  tor_assert(len < SIZE_T_CEILING);
+  aes_crypt_inplace(env, buf, len);
+}
+
+/** Encrypt <b>fromlen</b> bytes (at least 1) from <b>from</b> with the key in
+ * <b>key</b> to the buffer in <b>to</b> of length
+ * <b>tolen</b>. <b>tolen</b> must be at least <b>fromlen</b> plus
+ * CIPHER_IV_LEN bytes for the initialization vector. On success, return the
+ * number of bytes written, on failure, return -1.
+ */
+int
+crypto_cipher_encrypt_with_iv(const char *key,
+                              char *to, size_t tolen,
+                              const char *from, size_t fromlen)
+{
+  crypto_cipher_t *cipher;
+  tor_assert(from);
+  tor_assert(to);
+  tor_assert(fromlen < INT_MAX);
+
+  if (fromlen < 1)
+    return -1;
+  if (tolen < fromlen + CIPHER_IV_LEN)
+    return -1;
+
+  char iv[CIPHER_IV_LEN];
+  crypto_rand(iv, sizeof(iv));
+  cipher = crypto_cipher_new_with_iv(key, iv);
+
+  memcpy(to, iv, CIPHER_IV_LEN);
+  crypto_cipher_encrypt(cipher, to+CIPHER_IV_LEN, from, fromlen);
+  crypto_cipher_free(cipher);
+  memwipe(iv, 0, sizeof(iv));
+  return (int)(fromlen + CIPHER_IV_LEN);
+}
+
+/** Decrypt <b>fromlen</b> bytes (at least 1+CIPHER_IV_LEN) from <b>from</b>
+ * with the key in <b>key</b> to the buffer in <b>to</b> of length
+ * <b>tolen</b>. <b>tolen</b> must be at least <b>fromlen</b> minus
+ * CIPHER_IV_LEN bytes for the initialization vector. On success, return the
+ * number of bytes written, on failure, return -1.
+ */
+int
+crypto_cipher_decrypt_with_iv(const char *key,
+                              char *to, size_t tolen,
+                              const char *from, size_t fromlen)
+{
+  crypto_cipher_t *cipher;
+  tor_assert(key);
+  tor_assert(from);
+  tor_assert(to);
+  tor_assert(fromlen < INT_MAX);
+
+  if (fromlen <= CIPHER_IV_LEN)
+    return -1;
+  if (tolen < fromlen - CIPHER_IV_LEN)
+    return -1;
+
+  cipher = crypto_cipher_new_with_iv(key, from);
+
+  crypto_cipher_encrypt(cipher, to, from+CIPHER_IV_LEN, fromlen-CIPHER_IV_LEN);
+  crypto_cipher_free(cipher);
+  return (int)(fromlen - CIPHER_IV_LEN);
+}
diff --git a/src/lib/crypt_ops/crypto_cipher.h b/src/lib/crypt_ops/crypto_cipher.h
new file mode 100644
index 0000000000..cc4fbf7a41
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_cipher.h
@@ -0,0 +1,57 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_cipher.h
+ *
+ * \brief Headers for crypto_cipher.c
+ **/
+
+#ifndef TOR_CRYPTO_CIPHER_H
+#define TOR_CRYPTO_CIPHER_H
+
+#include "orconfig.h"
+
+#include <stdio.h>
+#include "lib/cc/torint.h"
+
+/** Length of our symmetric cipher's keys of 128-bit. */
+#define CIPHER_KEY_LEN 16
+/** Length of our symmetric cipher's IV of 128-bit. */
+#define CIPHER_IV_LEN 16
+/** Length of our symmetric cipher's keys of 256-bit. */
+#define CIPHER256_KEY_LEN 32
+
+typedef struct aes_cnt_cipher crypto_cipher_t;
+
+/* environment setup */
+crypto_cipher_t *crypto_cipher_new(const char *key);
+crypto_cipher_t *crypto_cipher_new_with_bits(const char *key, int bits);
+crypto_cipher_t *crypto_cipher_new_with_iv(const char *key, const char *iv);
+crypto_cipher_t *crypto_cipher_new_with_iv_and_bits(const uint8_t *key,
+                                                    const uint8_t *iv,
+                                                    int bits);
+void crypto_cipher_free_(crypto_cipher_t *env);
+#define crypto_cipher_free(c) \
+  FREE_AND_NULL(crypto_cipher_t, crypto_cipher_free_, (c))
+
+/* symmetric crypto */
+const char *crypto_cipher_get_key(crypto_cipher_t *env);
+
+int crypto_cipher_encrypt(crypto_cipher_t *env, char *to,
+                          const char *from, size_t fromlen);
+int crypto_cipher_decrypt(crypto_cipher_t *env, char *to,
+                          const char *from, size_t fromlen);
+void crypto_cipher_crypt_inplace(crypto_cipher_t *env, char *d, size_t len);
+
+int crypto_cipher_encrypt_with_iv(const char *key,
+                                  char *to, size_t tolen,
+                                  const char *from, size_t fromlen);
+int crypto_cipher_decrypt_with_iv(const char *key,
+                                  char *to, size_t tolen,
+                                  const char *from, size_t fromlen);
+
+#endif /* !defined(TOR_CRYPTO_H) */
diff --git a/src/common/crypto_curve25519.c b/src/lib/crypt_ops/crypto_curve25519.c
index fcbee3aba2..de4e17a296 100644
--- a/src/common/crypto_curve25519.c
+++ b/src/lib/crypt_ops/crypto_curve25519.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,15 +20,19 @@
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
 #endif
-#include "container.h"
-#include "crypto.h"
-#include "crypto_curve25519.h"
-#include "crypto_format.h"
-#include "util.h"
-#include "torlog.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
 
 #include "ed25519/donna/ed25519_donna_tor.h"
 
+#include <string.h>
+
 /* ==============================
    Part 1: wrap a suitable curve25519 implementation as curve25519_impl
    ============================== */
@@ -43,7 +47,7 @@ int curve25519_donna(uint8_t *mypublic,
 #elif defined(HAVE_NACL_CRYPTO_SCALARMULT_CURVE25519_H)
 #include <nacl/crypto_scalarmult_curve25519.h>
 #endif
-#endif
+#endif /* defined(USE_CURVE25519_NACL) */
 
 static void pick_curve25519_basepoint_impl(void);
 
@@ -72,7 +76,7 @@ curve25519_impl(uint8_t *output, const uint8_t *secret,
   r = crypto_scalarmult_curve25519(output, secret, bp);
 #else
 #error "No implementation of curve25519 is available."
-#endif
+#endif /* defined(USE_CURVE25519_DONNA) || ... */
   memwipe(bp, 0, sizeof(bp));
   return r;
 }
@@ -80,7 +84,7 @@ curve25519_impl(uint8_t *output, const uint8_t *secret,
 /**
  * Helper function: Multiply the scalar "secret" by the Curve25519
  * basepoint (X=9), and store the result in "output".  Return 0 on
- * success, -1 on false.
+ * success, -1 on failure.
  */
 STATIC int
 curve25519_basepoint_impl(uint8_t *output, const uint8_t *secret)
@@ -285,14 +289,20 @@ curve25519_basepoint_spot_check(void)
     0x0d,0xbf,0x3a,0x0d,0x26,0x38,0x1a,0xf4,
     0xeb,0xa4,0xa9,0x8e,0xaa,0x9b,0x4e,0x6a
   };
-  const int loop_max=200;
+  const int loop_max=8;
   int save_use_ed = curve25519_use_ed;
-  unsigned char e1[32] = { 5 };
-  unsigned char e2[32] = { 5 };
+  unsigned char e1[32], e2[32];
   unsigned char x[32],y[32];
   int i;
   int r=0;
 
+  memset(x, 0, sizeof(x));
+  memset(y, 0, sizeof(y));
+  memset(e1, 0, sizeof(e1));
+  memset(e2, 0, sizeof(e2));
+  e1[0]=5;
+  e2[0]=5;
+
   /* Check the most basic possible sanity via the test secret/public key pair
    * used in "Cryptography in NaCl - 2. Secret keys and public keys".  This
    * may catch catastrophic failures on systems where Curve25519 is expensive,
@@ -318,8 +328,11 @@ curve25519_basepoint_spot_check(void)
   }
 
   goto end;
+ // LCOV_EXCL_START -- we can only hit this code if there is a bug in our
+ // curve25519-basepoint implementation.
  fail:
   r = -1;
+ // LCOV_EXCL_STOP
  end:
   curve25519_use_ed = save_use_ed;
   return r;
@@ -351,4 +364,3 @@ curve25519_init(void)
 {
   pick_curve25519_basepoint_impl();
 }
-
diff --git a/src/common/crypto_curve25519.h b/src/lib/crypt_ops/crypto_curve25519.h
index 4011820949..061a7a3505 100644
--- a/src/common/crypto_curve25519.h
+++ b/src/lib/crypt_ops/crypto_curve25519.h
@@ -1,18 +1,19 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * \file crypto_curve25519.h
+ * \brief Header for crypto_curve25519.c
+ **/
+
 #ifndef TOR_CRYPTO_CURVE25519_H
 #define TOR_CRYPTO_CURVE25519_H
 
-#include "testsupport.h"
-#include "torint.h"
-
-/** Length of a curve25519 public key when encoded. */
-#define CURVE25519_PUBKEY_LEN 32
-/** Length of a curve25519 secret key when encoded. */
-#define CURVE25519_SECKEY_LEN 32
-/** Length of the result of a curve25519 handshake. */
-#define CURVE25519_OUTPUT_LEN 32
+#include "lib/testsupport/testsupport.h"
+#include "lib/cc/torint.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_openssl_mgt.h"
+#include "lib/defs/x25519_sizes.h"
 
 /** Wrapper type for a curve25519 public key.
  *
@@ -71,9 +72,7 @@ STATIC int curve25519_impl(uint8_t *output, const uint8_t *secret,
                            const uint8_t *basepoint);
 
 STATIC int curve25519_basepoint_impl(uint8_t *output, const uint8_t *secret);
-#endif
-
-#define CURVE25519_BASE64_PADDED_LEN 44
+#endif /* defined(CRYPTO_CURVE25519_PRIVATE) */
 
 int curve25519_public_from_base64(curve25519_public_key_t *pkey,
                                   const char *input);
@@ -83,5 +82,4 @@ int curve25519_public_to_base64(char *output,
 void curve25519_set_impl_params(int use_ed);
 void curve25519_init(void);
 
-#endif
-
+#endif /* !defined(TOR_CRYPTO_CURVE25519_H) */
diff --git a/src/lib/crypt_ops/crypto_dh.c b/src/lib/crypt_ops/crypto_dh.c
new file mode 100644
index 0000000000..4be7948761
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_dh.c
@@ -0,0 +1,113 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_dh.c
+ * \brief Block of functions related with DH utilities and operations.
+ *    over Z_p.  We aren't using this for any new crypto -- EC is more
+ *    efficient.
+ **/
+
+#include "lib/crypt_ops/compat_openssl.h"
+#include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_hkdf.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+
+/** Our DH 'g' parameter */
+const unsigned DH_GENERATOR = 2;
+/** This is the 1024-bit safe prime that Apache uses for its DH stuff; see
+ * modules/ssl/ssl_engine_dh.c; Apache also uses a generator of 2 with this
+ * prime.
+ */
+const char TLS_DH_PRIME[] =
+  "D67DE440CBBBDC1936D693D34AFD0AD50C84D239A45F520BB88174CB98"
+  "BCE951849F912E639C72FB13B4B4D7177E16D55AC179BA420B2A29FE324A"
+  "467A635E81FF5901377BEDDCFD33168A461AAD3B72DAE8860078045B07A7"
+  "DBCA7874087D1510EA9FCC9DDD330507DD62DB88AEAA747DE0F4D6E2BD68"
+  "B0E7393E0F24218EB3";
+/**
+ * This is from rfc2409, section 6.2.  It's a safe prime, and
+ * supposedly it equals:
+ * 2^1024 - 2^960 - 1 + 2^64 * { [2^894 pi] + 129093 }.
+ */
+const char OAKLEY_PRIME_2[] =
+  "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E08"
+  "8A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B"
+  "302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9"
+  "A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE6"
+  "49286651ECE65381FFFFFFFFFFFFFFFF";
+
+void
+crypto_dh_init(void)
+{
+#ifdef ENABLE_OPENSSL
+  crypto_dh_init_openssl();
+#endif
+#ifdef ENABLE_NSS
+  crypto_dh_init_nss();
+#endif
+}
+
+void
+crypto_dh_free_all(void)
+{
+#ifdef ENABLE_OPENSSL
+  crypto_dh_free_all_openssl();
+#endif
+#ifdef ENABLE_NSS
+  crypto_dh_free_all_nss();
+#endif
+}
+
+/** Given a DH key exchange object, and our peer's value of g^y (as a
+ * <b>pubkey_len</b>-byte value in <b>pubkey</b>) generate
+ * <b>secret_bytes_out</b> bytes of shared key material and write them
+ * to <b>secret_out</b>.  Return the number of bytes generated on success,
+ * or -1 on failure.
+ *
+ * (We generate key material by computing
+ *         SHA1( g^xy || "\x00" ) || SHA1( g^xy || "\x01" ) || ...
+ * where || is concatenation.)
+ */
+ssize_t
+crypto_dh_compute_secret(int severity, crypto_dh_t *dh,
+                         const char *pubkey, size_t pubkey_len,
+                         char *secret_out, size_t secret_bytes_out)
+{
+  tor_assert(secret_bytes_out/DIGEST_LEN <= 255);
+
+  unsigned char *secret_tmp = NULL;
+  size_t secret_len=0, secret_tmp_len=0;
+  secret_tmp_len = crypto_dh_get_bytes(dh);
+  secret_tmp = tor_malloc(secret_tmp_len);
+
+  ssize_t result = crypto_dh_handshake(severity, dh, pubkey, pubkey_len,
+                                   secret_tmp, secret_tmp_len);
+  if (result < 0)
+    goto error;
+
+  secret_len = result;
+  if (crypto_expand_key_material_TAP(secret_tmp, secret_len,
+                                     (uint8_t*)secret_out, secret_bytes_out)<0)
+    goto error;
+  secret_len = secret_bytes_out;
+
+  goto done;
+ error:
+  result = -1;
+ done:
+  if (secret_tmp) {
+    memwipe(secret_tmp, 0, secret_tmp_len);
+    tor_free(secret_tmp);
+  }
+  if (result < 0)
+    return result;
+  else
+    return secret_len;
+}
diff --git a/src/lib/crypt_ops/crypto_dh.h b/src/lib/crypt_ops/crypto_dh.h
new file mode 100644
index 0000000000..850d50c7ae
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_dh.h
@@ -0,0 +1,64 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_dh.h
+ *
+ * \brief Headers for crypto_dh.c
+ **/
+
+#ifndef TOR_CRYPTO_DH_H
+#define TOR_CRYPTO_DH_H
+
+#include "orconfig.h"
+#include "lib/cc/torint.h"
+#include "lib/defs/dh_sizes.h"
+
+typedef struct crypto_dh_t crypto_dh_t;
+
+extern const unsigned DH_GENERATOR;
+extern const char TLS_DH_PRIME[];
+extern const char OAKLEY_PRIME_2[];
+
+/* Key negotiation */
+#define DH_TYPE_CIRCUIT 1
+#define DH_TYPE_REND 2
+#define DH_TYPE_TLS 3
+void crypto_dh_init(void);
+crypto_dh_t *crypto_dh_new(int dh_type);
+crypto_dh_t *crypto_dh_dup(const crypto_dh_t *dh);
+int crypto_dh_get_bytes(crypto_dh_t *dh);
+int crypto_dh_generate_public(crypto_dh_t *dh);
+int crypto_dh_get_public(crypto_dh_t *dh, char *pubkey_out,
+                         size_t pubkey_out_len);
+ssize_t crypto_dh_compute_secret(int severity, crypto_dh_t *dh,
+                             const char *pubkey, size_t pubkey_len,
+                             char *secret_out, size_t secret_out_len);
+void crypto_dh_free_(crypto_dh_t *dh);
+#define crypto_dh_free(dh) FREE_AND_NULL(crypto_dh_t, crypto_dh_free_, (dh))
+
+ssize_t crypto_dh_handshake(int severity, crypto_dh_t *dh,
+                            const char *pubkey, size_t pubkey_len,
+                            unsigned char *secret_out,
+                            size_t secret_bytes_out);
+
+void crypto_dh_free_all(void);
+
+/* Prototypes for private functions only used by tortls.c, crypto.c, and the
+ * unit tests. */
+struct dh_st;
+struct dh_st *crypto_dh_new_openssl_tls(void);
+
+#ifdef ENABLE_OPENSSL
+void crypto_dh_init_openssl(void);
+void crypto_dh_free_all_openssl(void);
+#endif
+#ifdef ENABLE_NSS
+void crypto_dh_init_nss(void);
+void crypto_dh_free_all_nss(void);
+#endif
+
+#endif /* !defined(TOR_CRYPTO_DH_H) */
diff --git a/src/lib/crypt_ops/crypto_dh_nss.c b/src/lib/crypt_ops/crypto_dh_nss.c
new file mode 100644
index 0000000000..379eb84a4f
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_dh_nss.c
@@ -0,0 +1,209 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_dh_nss.h
+ *
+ * \brief NSS implementation of Diffie-Hellman over Z_p.
+ **/
+
+#include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_nss_mgt.h"
+
+#include "lib/encoding/binascii.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+
+#include <cryptohi.h>
+#include <keyhi.h>
+#include <pk11pub.h>
+
+static int dh_initialized = 0;
+static SECKEYDHParams tls_dh_param, circuit_dh_param;
+static unsigned char tls_dh_prime_data[DH1024_KEY_LEN];
+static unsigned char circuit_dh_prime_data[DH1024_KEY_LEN];
+static unsigned char dh_generator_data[1];
+
+void
+crypto_dh_init_nss(void)
+{
+  if (dh_initialized)
+    return;
+
+  int r;
+  r = base16_decode((char*)tls_dh_prime_data,
+                    sizeof(tls_dh_prime_data),
+                    TLS_DH_PRIME, strlen(TLS_DH_PRIME));
+  tor_assert(r == DH1024_KEY_LEN);
+  r = base16_decode((char*)circuit_dh_prime_data,
+                    sizeof(circuit_dh_prime_data),
+                    OAKLEY_PRIME_2, strlen(OAKLEY_PRIME_2));
+  tor_assert(r == DH1024_KEY_LEN);
+  dh_generator_data[0] = DH_GENERATOR;
+
+  tls_dh_param.prime.data = tls_dh_prime_data;
+  tls_dh_param.prime.len = DH1024_KEY_LEN;
+  tls_dh_param.base.data = dh_generator_data;
+  tls_dh_param.base.len = 1;
+
+  circuit_dh_param.prime.data = circuit_dh_prime_data;
+  circuit_dh_param.prime.len = DH1024_KEY_LEN;
+  circuit_dh_param.base.data = dh_generator_data;
+  circuit_dh_param.base.len = 1;
+
+  dh_initialized = 1;
+}
+
+void
+crypto_dh_free_all_nss(void)
+{
+  dh_initialized = 0;
+}
+
+struct crypto_dh_t {
+  int dh_type; // XXXX let's remove this later on.
+  SECKEYPrivateKey *seckey;
+  SECKEYPublicKey *pubkey;
+};
+
+crypto_dh_t *
+crypto_dh_new(int dh_type)
+{
+  crypto_dh_t *r = tor_malloc_zero(sizeof(crypto_dh_t));
+  r->dh_type = dh_type;
+  return r;
+}
+
+crypto_dh_t *
+crypto_dh_dup(const crypto_dh_t *dh)
+{
+  tor_assert(dh);
+  crypto_dh_t *r = crypto_dh_new(dh->dh_type);
+  if (dh->seckey)
+    r->seckey = SECKEY_CopyPrivateKey(dh->seckey);
+  if (dh->pubkey)
+    r->pubkey = SECKEY_CopyPublicKey(dh->pubkey);
+  return r;
+}
+
+int
+crypto_dh_get_bytes(crypto_dh_t *dh)
+{
+  (void)dh;
+  return DH1024_KEY_LEN;
+}
+
+int
+crypto_dh_generate_public(crypto_dh_t *dh)
+{
+  tor_assert(dh);
+  SECKEYDHParams *p;
+  if (dh->dh_type == DH_TYPE_TLS)
+    p = &tls_dh_param;
+  else
+    p = &circuit_dh_param;
+
+  dh->seckey = SECKEY_CreateDHPrivateKey(p, &dh->pubkey, NULL);
+  if (!dh->seckey || !dh->pubkey)
+    return -1;
+  else
+    return 0;
+}
+int
+crypto_dh_get_public(crypto_dh_t *dh, char *pubkey_out,
+                     size_t pubkey_out_len)
+{
+  tor_assert(dh);
+  tor_assert(pubkey_out);
+  if (!dh->pubkey) {
+    if (crypto_dh_generate_public(dh) < 0)
+      return -1;
+  }
+
+  const SECItem *item = &dh->pubkey->u.dh.publicValue;
+
+  if (item->len > pubkey_out_len)
+    return -1;
+
+  /* Left-pad the result with 0s. */
+  memset(pubkey_out, 0, pubkey_out_len);
+  memcpy(pubkey_out + pubkey_out_len - item->len,
+         item->data,
+         item->len);
+
+  return 0;
+}
+
+void
+crypto_dh_free_(crypto_dh_t *dh)
+{
+  if (!dh)
+    return;
+  if (dh->seckey)
+    SECKEY_DestroyPrivateKey(dh->seckey);
+  if (dh->pubkey)
+    SECKEY_DestroyPublicKey(dh->pubkey);
+  tor_free(dh);
+}
+
+ssize_t
+crypto_dh_handshake(int severity, crypto_dh_t *dh,
+                    const char *pubkey, size_t pubkey_len,
+                    unsigned char *secret_out,
+                    size_t secret_bytes_out)
+{
+  tor_assert(dh);
+  if (pubkey_len > DH1024_KEY_LEN)
+    return -1;
+  if (!dh->pubkey || !dh->seckey)
+    return -1;
+  if (secret_bytes_out < DH1024_KEY_LEN)
+    return -1;
+
+  SECKEYPublicKey peer_key;
+  memset(&peer_key, 0, sizeof(peer_key));
+  peer_key.keyType = dhKey;
+  peer_key.pkcs11ID = CK_INVALID_HANDLE;
+
+  if (dh->dh_type == DH_TYPE_TLS)
+    peer_key.u.dh.prime.data = tls_dh_prime_data; // should never use this code
+  else
+    peer_key.u.dh.prime.data = circuit_dh_prime_data;
+  peer_key.u.dh.prime.len = DH1024_KEY_LEN;
+  peer_key.u.dh.base.data = dh_generator_data;
+  peer_key.u.dh.base.len = 1;
+  peer_key.u.dh.publicValue.data = (unsigned char *)pubkey;
+  peer_key.u.dh.publicValue.len = (int) pubkey_len;
+
+  PK11SymKey *sym = PK11_PubDerive(dh->seckey, &peer_key,
+                       PR_FALSE, NULL, NULL, CKM_DH_PKCS_DERIVE,
+                       CKM_GENERIC_SECRET_KEY_GEN /* ??? */,
+                       CKA_DERIVE, 0, NULL);
+  if (! sym) {
+    crypto_nss_log_errors(severity, "deriving a DH shared secret");
+    return -1;
+  }
+
+  SECStatus s = PK11_ExtractKeyValue(sym);
+  if (s != SECSuccess) {
+    crypto_nss_log_errors(severity, "extracting a DH shared secret");
+    PK11_FreeSymKey(sym);
+    return -1;
+  }
+
+  SECItem *result = PK11_GetKeyData(sym);
+  tor_assert(result); // This cannot fail.
+  if (BUG(result->len > secret_bytes_out)) {
+    PK11_FreeSymKey(sym);
+    return -1;
+  }
+
+  ssize_t len = result->len;
+  memcpy(secret_out, result->data, len);
+  PK11_FreeSymKey(sym);
+
+  return len;
+}
diff --git a/src/lib/crypt_ops/crypto_dh_openssl.c b/src/lib/crypt_ops/crypto_dh_openssl.c
new file mode 100644
index 0000000000..8c6388fd5d
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_dh_openssl.c
@@ -0,0 +1,477 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_dh_openssl.c
+ * \brief Implement Tor's Z_p diffie-hellman stuff for OpenSSL.
+ **/
+
+#include "lib/crypt_ops/compat_openssl.h"
+#include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_hkdf.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+
+DISABLE_GCC_WARNING(redundant-decls)
+
+#include <openssl/dh.h>
+
+ENABLE_GCC_WARNING(redundant-decls)
+
+#include <openssl/bn.h>
+#include <string.h>
+
+#ifndef ENABLE_NSS
+static int tor_check_dh_key(int severity, const BIGNUM *bn);
+
+/** A structure to hold the first half (x, g^x) of a Diffie-Hellman handshake
+ * while we're waiting for the second.*/
+struct crypto_dh_t {
+  DH *dh; /**< The openssl DH object */
+};
+#endif
+
+static DH *new_openssl_dh_from_params(BIGNUM *p, BIGNUM *g);
+
+/** Shared P parameter for our circuit-crypto DH key exchanges. */
+static BIGNUM *dh_param_p = NULL;
+/** Shared P parameter for our TLS DH key exchanges. */
+static BIGNUM *dh_param_p_tls = NULL;
+/** Shared G parameter for our DH key exchanges. */
+static BIGNUM *dh_param_g = NULL;
+
+/* This function is disabled unless we change the DH parameters. */
+#if 0
+/** Validate a given set of Diffie-Hellman parameters.  This is moderately
+ * computationally expensive (milliseconds), so should only be called when
+ * the DH parameters change. Returns 0 on success, * -1 on failure.
+ */
+static int
+crypto_validate_dh_params(const BIGNUM *p, const BIGNUM *g)
+{
+  DH *dh = NULL;
+  int ret = -1;
+
+  /* Copy into a temporary DH object, just so that DH_check() can be called. */
+  if (!(dh = DH_new()))
+      goto out;
+#ifdef OPENSSL_1_1_API
+  BIGNUM *dh_p, *dh_g;
+  if (!(dh_p = BN_dup(p)))
+    goto out;
+  if (!(dh_g = BN_dup(g)))
+    goto out;
+  if (!DH_set0_pqg(dh, dh_p, NULL, dh_g))
+    goto out;
+#else /* !(defined(OPENSSL_1_1_API)) */
+  if (!(dh->p = BN_dup(p)))
+    goto out;
+  if (!(dh->g = BN_dup(g)))
+    goto out;
+#endif /* defined(OPENSSL_1_1_API) */
+
+  /* Perform the validation. */
+  int codes = 0;
+  if (!DH_check(dh, &codes))
+    goto out;
+  if (BN_is_word(g, DH_GENERATOR_2)) {
+    /* Per https://wiki.openssl.org/index.php/Diffie-Hellman_parameters
+     *
+     * OpenSSL checks the prime is congruent to 11 when g = 2; while the
+     * IETF's primes are congruent to 23 when g = 2.
+     */
+    BN_ULONG residue = BN_mod_word(p, 24);
+    if (residue == 11 || residue == 23)
+      codes &= ~DH_NOT_SUITABLE_GENERATOR;
+  }
+  if (codes != 0) /* Specifics on why the params suck is irrelevant. */
+    goto out;
+
+  /* Things are probably not evil. */
+  ret = 0;
+
+ out:
+  if (dh)
+    DH_free(dh);
+  return ret;
+}
+#endif
+
+/**
+ * Helper: convert <b>hex<b> to a bignum, and return it.  Assert that the
+ * operation was successful.
+ */
+static BIGNUM *
+bignum_from_hex(const char *hex)
+{
+  BIGNUM *result = BN_new();
+  tor_assert(result);
+
+  int r = BN_hex2bn(&result, hex);
+  tor_assert(r);
+  tor_assert(result);
+  return result;
+}
+
+/** Set the global Diffie-Hellman generator, used for both TLS and internal
+ * DH stuff.
+ */
+static void
+crypto_set_dh_generator(void)
+{
+  BIGNUM *generator;
+  int r;
+
+  if (dh_param_g)
+    return;
+
+  generator = BN_new();
+  tor_assert(generator);
+
+  r = BN_set_word(generator, DH_GENERATOR);
+  tor_assert(r);
+
+  dh_param_g = generator;
+}
+
+/** Initialize our DH parameters. Idempotent. */
+void
+crypto_dh_init_openssl(void)
+{
+  if (dh_param_p && dh_param_g && dh_param_p_tls)
+    return;
+
+  tor_assert(dh_param_g == NULL);
+  tor_assert(dh_param_p == NULL);
+  tor_assert(dh_param_p_tls == NULL);
+
+  crypto_set_dh_generator();
+  dh_param_p = bignum_from_hex(OAKLEY_PRIME_2);
+  dh_param_p_tls = bignum_from_hex(TLS_DH_PRIME);
+
+  /* Checks below are disabled unless we change the hardcoded DH parameters. */
+#if 0
+  tor_assert(0 == crypto_validate_dh_params(dh_param_p, dh_param_g));
+  tor_assert(0 == crypto_validate_dh_params(dh_param_p_tls, dh_param_g));
+#endif
+}
+
+/** Number of bits to use when choosing the x or y value in a Diffie-Hellman
+ * handshake.  Since we exponentiate by this value, choosing a smaller one
+ * lets our handhake go faster.
+ */
+#define DH_PRIVATE_KEY_BITS 320
+
+/** Used by tortls.c: Get the DH* for use with TLS.
+ */
+DH *
+crypto_dh_new_openssl_tls(void)
+{
+  return new_openssl_dh_from_params(dh_param_p_tls, dh_param_g);
+}
+
+#ifndef ENABLE_NSS
+/** Allocate and return a new DH object for a key exchange. Returns NULL on
+ * failure.
+ */
+crypto_dh_t *
+crypto_dh_new(int dh_type)
+{
+  crypto_dh_t *res = tor_malloc_zero(sizeof(crypto_dh_t));
+
+  tor_assert(dh_type == DH_TYPE_CIRCUIT || dh_type == DH_TYPE_TLS ||
+             dh_type == DH_TYPE_REND);
+
+  if (!dh_param_p)
+    crypto_dh_init();
+
+  BIGNUM *dh_p = NULL;
+  if (dh_type == DH_TYPE_TLS) {
+    dh_p = dh_param_p_tls;
+  } else {
+    dh_p = dh_param_p;
+  }
+
+  res->dh = new_openssl_dh_from_params(dh_p, dh_param_g);
+  if (res->dh == NULL)
+    tor_free(res); // sets res to NULL.
+  return res;
+}
+#endif
+
+/** Create and return a new openssl DH from a given prime and generator. */
+static DH *
+new_openssl_dh_from_params(BIGNUM *p, BIGNUM *g)
+{
+  DH *res_dh;
+  if (!(res_dh = DH_new()))
+    goto err;
+
+  BIGNUM *dh_p = NULL, *dh_g = NULL;
+  dh_p = BN_dup(p);
+  if (!dh_p)
+    goto err;
+
+  dh_g = BN_dup(g);
+  if (!dh_g) {
+    BN_free(dh_p);
+    goto err;
+  }
+
+#ifdef OPENSSL_1_1_API
+
+  if (!DH_set0_pqg(res_dh, dh_p, NULL, dh_g)) {
+    goto err;
+  }
+
+  if (!DH_set_length(res_dh, DH_PRIVATE_KEY_BITS))
+    goto err;
+#else /* !(defined(OPENSSL_1_1_API)) */
+  res_dh->p = dh_p;
+  res_dh->g = dh_g;
+  res_dh->length = DH_PRIVATE_KEY_BITS;
+#endif /* defined(OPENSSL_1_1_API) */
+
+  return res_dh;
+
+  /* LCOV_EXCL_START
+   * This error condition is only reached when an allocation fails */
+ err:
+  crypto_openssl_log_errors(LOG_WARN, "creating DH object");
+  if (res_dh) DH_free(res_dh); /* frees p and g too */
+  return NULL;
+  /* LCOV_EXCL_STOP */
+}
+
+#ifndef ENABLE_NSS
+/** Return a copy of <b>dh</b>, sharing its internal state. */
+crypto_dh_t *
+crypto_dh_dup(const crypto_dh_t *dh)
+{
+  crypto_dh_t *dh_new = tor_malloc_zero(sizeof(crypto_dh_t));
+  tor_assert(dh);
+  tor_assert(dh->dh);
+  dh_new->dh = dh->dh;
+  DH_up_ref(dh->dh);
+  return dh_new;
+}
+
+/** Return the length of the DH key in <b>dh</b>, in bytes.
+ */
+int
+crypto_dh_get_bytes(crypto_dh_t *dh)
+{
+  tor_assert(dh);
+  return DH_size(dh->dh);
+}
+
+/** Generate \<x,g^x\> for our part of the key exchange.  Return 0 on
+ * success, -1 on failure.
+ */
+int
+crypto_dh_generate_public(crypto_dh_t *dh)
+{
+#ifndef OPENSSL_1_1_API
+ again:
+#endif
+  if (!DH_generate_key(dh->dh)) {
+    /* LCOV_EXCL_START
+     * To test this we would need some way to tell openssl to break DH. */
+    crypto_openssl_log_errors(LOG_WARN, "generating DH key");
+    return -1;
+    /* LCOV_EXCL_STOP */
+  }
+#ifdef OPENSSL_1_1_API
+  /* OpenSSL 1.1.x doesn't appear to let you regenerate a DH key, without
+   * recreating the DH object.  I have no idea what sort of aliasing madness
+   * can occur here, so do the check, and just bail on failure.
+   */
+  const BIGNUM *pub_key, *priv_key;
+  DH_get0_key(dh->dh, &pub_key, &priv_key);
+  if (tor_check_dh_key(LOG_WARN, pub_key)<0) {
+    log_warn(LD_CRYPTO, "Weird! Our own DH key was invalid.  I guess once-in-"
+             "the-universe chances really do happen.  Treating as a failure.");
+    return -1;
+  }
+#else /* !(defined(OPENSSL_1_1_API)) */
+  if (tor_check_dh_key(LOG_WARN, dh->dh->pub_key)<0) {
+    /* LCOV_EXCL_START
+     * If this happens, then openssl's DH implementation is busted. */
+    log_warn(LD_CRYPTO, "Weird! Our own DH key was invalid.  I guess once-in-"
+             "the-universe chances really do happen.  Trying again.");
+    /* Free and clear the keys, so OpenSSL will actually try again. */
+    BN_clear_free(dh->dh->pub_key);
+    BN_clear_free(dh->dh->priv_key);
+    dh->dh->pub_key = dh->dh->priv_key = NULL;
+    goto again;
+    /* LCOV_EXCL_STOP */
+  }
+#endif /* defined(OPENSSL_1_1_API) */
+  return 0;
+}
+
+/** Generate g^x as necessary, and write the g^x for the key exchange
+ * as a <b>pubkey_len</b>-byte value into <b>pubkey</b>. Return 0 on
+ * success, -1 on failure.  <b>pubkey_len</b> must be \>= DH1024_KEY_LEN.
+ */
+int
+crypto_dh_get_public(crypto_dh_t *dh, char *pubkey, size_t pubkey_len)
+{
+  int bytes;
+  tor_assert(dh);
+
+  const BIGNUM *dh_pub;
+
+#ifdef OPENSSL_1_1_API
+  const BIGNUM *dh_priv;
+  DH_get0_key(dh->dh, &dh_pub, &dh_priv);
+#else
+  dh_pub = dh->dh->pub_key;
+#endif /* defined(OPENSSL_1_1_API) */
+
+  if (!dh_pub) {
+    if (crypto_dh_generate_public(dh)<0)
+      return -1;
+    else {
+#ifdef OPENSSL_1_1_API
+      DH_get0_key(dh->dh, &dh_pub, &dh_priv);
+#else
+      dh_pub = dh->dh->pub_key;
+#endif
+    }
+  }
+
+  tor_assert(dh_pub);
+  bytes = BN_num_bytes(dh_pub);
+  tor_assert(bytes >= 0);
+  if (pubkey_len < (size_t)bytes) {
+    log_warn(LD_CRYPTO,
+             "Weird! pubkey_len (%d) was smaller than DH1024_KEY_LEN (%d)",
+             (int) pubkey_len, bytes);
+    return -1;
+  }
+
+  memset(pubkey, 0, pubkey_len);
+  BN_bn2bin(dh_pub, (unsigned char*)(pubkey+(pubkey_len-bytes)));
+
+  return 0;
+}
+
+/** Check for bad Diffie-Hellman public keys (g^x).  Return 0 if the key is
+ * okay (in the subgroup [2,p-2]), or -1 if it's bad.
+ * See http://www.cl.cam.ac.uk/ftp/users/rja14/psandqs.ps.gz for some tips.
+ */
+static int
+tor_check_dh_key(int severity, const BIGNUM *bn)
+{
+  BIGNUM *x;
+  char *s;
+  tor_assert(bn);
+  x = BN_new();
+  tor_assert(x);
+  if (BUG(!dh_param_p))
+    crypto_dh_init(); //LCOV_EXCL_LINE we already checked whether we did this.
+  BN_set_word(x, 1);
+  if (BN_cmp(bn,x)<=0) {
+    log_fn(severity, LD_CRYPTO, "DH key must be at least 2.");
+    goto err;
+  }
+  BN_copy(x,dh_param_p);
+  BN_sub_word(x, 1);
+  if (BN_cmp(bn,x)>=0) {
+    log_fn(severity, LD_CRYPTO, "DH key must be at most p-2.");
+    goto err;
+  }
+  BN_clear_free(x);
+  return 0;
+ err:
+  BN_clear_free(x);
+  s = BN_bn2hex(bn);
+  log_fn(severity, LD_CRYPTO, "Rejecting insecure DH key [%s]", s);
+  OPENSSL_free(s);
+  return -1;
+}
+
+/** Given a DH key exchange object, and our peer's value of g^y (as a
+ * <b>pubkey_len</b>-byte value in <b>pubkey</b>) generate
+ * g^xy as a big-endian integer in <b>secret_out</b>.
+ * Return the number of bytes generated on success,
+ * or -1 on failure.
+ *
+ * This function MUST validate that g^y is actually in the group.
+ */
+ssize_t
+crypto_dh_handshake(int severity, crypto_dh_t *dh,
+                    const char *pubkey, size_t pubkey_len,
+                    unsigned char *secret_out, size_t secret_bytes_out)
+{
+  BIGNUM *pubkey_bn = NULL;
+  size_t secret_len=0;
+  int result=0;
+
+  tor_assert(dh);
+  tor_assert(secret_bytes_out/DIGEST_LEN <= 255);
+  tor_assert(pubkey_len < INT_MAX);
+
+  if (BUG(crypto_dh_get_bytes(dh) > (int)secret_bytes_out)) {
+    goto error;
+  }
+
+  if (!(pubkey_bn = BN_bin2bn((const unsigned char*)pubkey,
+                              (int)pubkey_len, NULL)))
+    goto error;
+  if (tor_check_dh_key(severity, pubkey_bn)<0) {
+    /* Check for invalid public keys. */
+    log_fn(severity, LD_CRYPTO,"Rejected invalid g^x");
+    goto error;
+  }
+  result = DH_compute_key(secret_out, pubkey_bn, dh->dh);
+  if (result < 0) {
+    log_warn(LD_CRYPTO,"DH_compute_key() failed.");
+    goto error;
+  }
+  secret_len = result;
+
+  goto done;
+ error:
+  result = -1;
+ done:
+  crypto_openssl_log_errors(LOG_WARN, "completing DH handshake");
+  if (pubkey_bn)
+    BN_clear_free(pubkey_bn);
+  if (result < 0)
+    return result;
+  else
+    return secret_len;
+}
+
+/** Free a DH key exchange object.
+ */
+void
+crypto_dh_free_(crypto_dh_t *dh)
+{
+  if (!dh)
+    return;
+  tor_assert(dh->dh);
+  DH_free(dh->dh);
+  tor_free(dh);
+}
+#endif
+
+void
+crypto_dh_free_all_openssl(void)
+{
+  if (dh_param_p)
+    BN_clear_free(dh_param_p);
+  if (dh_param_p_tls)
+    BN_clear_free(dh_param_p_tls);
+  if (dh_param_g)
+    BN_clear_free(dh_param_g);
+
+  dh_param_p = dh_param_p_tls = dh_param_g = NULL;
+}
diff --git a/src/lib/crypt_ops/crypto_digest.c b/src/lib/crypt_ops/crypto_digest.c
new file mode 100644
index 0000000000..26f06c6c79
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_digest.c
@@ -0,0 +1,828 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_digest.c
+ * \brief Block of functions related with digest and xof utilities and
+ * operations.
+ **/
+
+#include "lib/container/smartlist.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+
+#include "keccak-tiny/keccak-tiny.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "lib/arch/bytes.h"
+
+#ifdef ENABLE_NSS
+DISABLE_GCC_WARNING(strict-prototypes)
+#include <pk11pub.h>
+ENABLE_GCC_WARNING(strict-prototypes)
+#else
+
+#include "lib/crypt_ops/crypto_openssl_mgt.h"
+
+DISABLE_GCC_WARNING(redundant-decls)
+
+#include <openssl/hmac.h>
+#include <openssl/sha.h>
+
+ENABLE_GCC_WARNING(redundant-decls)
+#endif
+
+#ifdef ENABLE_NSS
+/**
+ * Convert a digest_algorithm_t (used by tor) to a HashType (used by NSS).
+ * On failure, return SEC_OID_UNKNOWN. */
+static SECOidTag
+digest_alg_to_nss_oid(digest_algorithm_t alg)
+{
+  switch (alg) {
+    case DIGEST_SHA1: return SEC_OID_SHA1;
+    case DIGEST_SHA256: return SEC_OID_SHA256;
+    case DIGEST_SHA512: return SEC_OID_SHA512;
+    case DIGEST_SHA3_256: /* Fall through */
+    case DIGEST_SHA3_512: /* Fall through */
+    default:
+      return SEC_OID_UNKNOWN;
+  }
+}
+
+/* Helper: get an unkeyed digest via pk11wrap */
+static int
+digest_nss_internal(SECOidTag alg,
+                    char *digest, unsigned len_out,
+                    const char *msg, size_t msg_len)
+{
+  if (alg == SEC_OID_UNKNOWN)
+    return -1;
+  tor_assert(msg_len <= UINT_MAX);
+
+  int rv = -1;
+  SECStatus s;
+  PK11Context *ctx = PK11_CreateDigestContext(alg);
+  if (!ctx)
+    return -1;
+
+  s = PK11_DigestBegin(ctx);
+  if (s != SECSuccess)
+    goto done;
+
+  s = PK11_DigestOp(ctx, (const unsigned char *)msg, (unsigned int)msg_len);
+  if (s != SECSuccess)
+    goto done;
+
+  unsigned int len = 0;
+  s = PK11_DigestFinal(ctx, (unsigned char *)digest, &len, len_out);
+  if (s != SECSuccess)
+    goto done;
+
+  rv = 0;
+ done:
+  PK11_DestroyContext(ctx, PR_TRUE);
+  return rv;
+}
+
+/** True iff alg is implemented in our crypto library, and we want to use that
+ * implementation */
+static bool
+library_supports_digest(digest_algorithm_t alg)
+{
+  switch (alg) {
+    case DIGEST_SHA1: /* Fall through */
+    case DIGEST_SHA256: /* Fall through */
+    case DIGEST_SHA512: /* Fall through */
+      return true;
+    case DIGEST_SHA3_256: /* Fall through */
+    case DIGEST_SHA3_512: /* Fall through */
+    default:
+      return false;
+  }
+}
+#endif
+
+/* Crypto digest functions */
+
+/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in
+ * <b>m</b>.  Write the DIGEST_LEN byte result into <b>digest</b>.
+ * Return 0 on success, -1 on failure.
+ */
+MOCK_IMPL(int,
+crypto_digest,(char *digest, const char *m, size_t len))
+{
+  tor_assert(m);
+  tor_assert(digest);
+#ifdef ENABLE_NSS
+  return digest_nss_internal(SEC_OID_SHA1, digest, DIGEST_LEN, m, len);
+#else
+  if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) {
+    return -1;
+  }
+#endif
+  return 0;
+}
+
+/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
+ * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN256-byte result
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_digest256(char *digest, const char *m, size_t len,
+                 digest_algorithm_t algorithm)
+{
+  tor_assert(m);
+  tor_assert(digest);
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+
+  int ret = 0;
+  if (algorithm == DIGEST_SHA256) {
+#ifdef ENABLE_NSS
+    return digest_nss_internal(SEC_OID_SHA256, digest, DIGEST256_LEN, m, len);
+#else
+    ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL);
+#endif
+  } else {
+    ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len)
+           > -1);
+  }
+
+  if (!ret)
+    return -1;
+  return 0;
+}
+
+/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
+ * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN512-byte result
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_digest512(char *digest, const char *m, size_t len,
+                 digest_algorithm_t algorithm)
+{
+  tor_assert(m);
+  tor_assert(digest);
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+
+  int ret = 0;
+  if (algorithm == DIGEST_SHA512) {
+#ifdef ENABLE_NSS
+    return digest_nss_internal(SEC_OID_SHA512, digest, DIGEST512_LEN, m, len);
+#else
+    ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest)
+           != NULL);
+#endif
+  } else {
+    ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len)
+           > -1);
+  }
+
+  if (!ret)
+    return -1;
+  return 0;
+}
+
+/** Set the common_digests_t in <b>ds_out</b> to contain every digest on the
+ * <b>len</b> bytes in <b>m</b> that we know how to compute.  Return 0 on
+ * success, -1 on failure. */
+int
+crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len)
+{
+  tor_assert(ds_out);
+  memset(ds_out, 0, sizeof(*ds_out));
+  if (crypto_digest(ds_out->d[DIGEST_SHA1], m, len) < 0)
+    return -1;
+  if (crypto_digest256(ds_out->d[DIGEST_SHA256], m, len, DIGEST_SHA256) < 0)
+    return -1;
+
+  return 0;
+}
+
+/** Return the name of an algorithm, as used in directory documents. */
+const char *
+crypto_digest_algorithm_get_name(digest_algorithm_t alg)
+{
+  switch (alg) {
+    case DIGEST_SHA1:
+      return "sha1";
+    case DIGEST_SHA256:
+      return "sha256";
+    case DIGEST_SHA512:
+      return "sha512";
+    case DIGEST_SHA3_256:
+      return "sha3-256";
+    case DIGEST_SHA3_512:
+      return "sha3-512";
+      // LCOV_EXCL_START
+    default:
+      tor_fragile_assert();
+      return "??unknown_digest??";
+      // LCOV_EXCL_STOP
+  }
+}
+
+/** Given the name of a digest algorithm, return its integer value, or -1 if
+ * the name is not recognized. */
+int
+crypto_digest_algorithm_parse_name(const char *name)
+{
+  if (!strcmp(name, "sha1"))
+    return DIGEST_SHA1;
+  else if (!strcmp(name, "sha256"))
+    return DIGEST_SHA256;
+  else if (!strcmp(name, "sha512"))
+    return DIGEST_SHA512;
+  else if (!strcmp(name, "sha3-256"))
+    return DIGEST_SHA3_256;
+  else if (!strcmp(name, "sha3-512"))
+    return DIGEST_SHA3_512;
+  else
+    return -1;
+}
+
+/** Given an algorithm, return the digest length in bytes. */
+size_t
+crypto_digest_algorithm_get_length(digest_algorithm_t alg)
+{
+  switch (alg) {
+    case DIGEST_SHA1:
+      return DIGEST_LEN;
+    case DIGEST_SHA256:
+      return DIGEST256_LEN;
+    case DIGEST_SHA512:
+      return DIGEST512_LEN;
+    case DIGEST_SHA3_256:
+      return DIGEST256_LEN;
+    case DIGEST_SHA3_512:
+      return DIGEST512_LEN;
+    default:
+      tor_assert(0);              // LCOV_EXCL_LINE
+      return 0; /* Unreachable */ // LCOV_EXCL_LINE
+  }
+}
+
+/** Intermediate information about the digest of a stream of data. */
+struct crypto_digest_t {
+  digest_algorithm_t algorithm; /**< Which algorithm is in use? */
+   /** State for the digest we're using.  Only one member of the
+    * union is usable, depending on the value of <b>algorithm</b>. Note also
+    * that space for other members might not even be allocated!
+    */
+  union {
+#ifdef ENABLE_NSS
+    PK11Context *ctx;
+#else
+    SHA_CTX sha1; /**< state for SHA1 */
+    SHA256_CTX sha2; /**< state for SHA256 */
+    SHA512_CTX sha512; /**< state for SHA512 */
+#endif
+    keccak_state sha3; /**< state for SHA3-[256,512] */
+  } d;
+};
+
+#ifdef TOR_UNIT_TESTS
+
+digest_algorithm_t
+crypto_digest_get_algorithm(crypto_digest_t *digest)
+{
+  tor_assert(digest);
+
+  return digest->algorithm;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Return the number of bytes we need to malloc in order to get a
+ * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe
+ * when we free one.
+ */
+static size_t
+crypto_digest_alloc_bytes(digest_algorithm_t alg)
+{
+  /* Helper: returns the number of bytes in the 'f' field of 'st' */
+#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f ))
+  /* Gives the length of crypto_digest_t through the end of the field 'd' */
+#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \
+                         STRUCT_FIELD_SIZE(crypto_digest_t, f))
+  switch (alg) {
+#ifdef ENABLE_NSS
+    case DIGEST_SHA1: /* Fall through */
+    case DIGEST_SHA256: /* Fall through */
+    case DIGEST_SHA512:
+      return END_OF_FIELD(d.ctx);
+#else
+    case DIGEST_SHA1:
+      return END_OF_FIELD(d.sha1);
+    case DIGEST_SHA256:
+      return END_OF_FIELD(d.sha2);
+    case DIGEST_SHA512:
+      return END_OF_FIELD(d.sha512);
+#endif
+    case DIGEST_SHA3_256:
+    case DIGEST_SHA3_512:
+      return END_OF_FIELD(d.sha3);
+    default:
+      tor_assert(0); // LCOV_EXCL_LINE
+      return 0;      // LCOV_EXCL_LINE
+  }
+#undef END_OF_FIELD
+#undef STRUCT_FIELD_SIZE
+}
+
+/**
+ * Internal function: create and return a new digest object for 'algorithm'.
+ * Does not typecheck the algorithm.
+ */
+static crypto_digest_t *
+crypto_digest_new_internal(digest_algorithm_t algorithm)
+{
+  crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm));
+  r->algorithm = algorithm;
+
+  switch (algorithm)
+    {
+#ifdef ENABLE_NSS
+    case DIGEST_SHA1: /* fall through */
+    case DIGEST_SHA256: /* fall through */
+    case DIGEST_SHA512:
+      r->d.ctx = PK11_CreateDigestContext(digest_alg_to_nss_oid(algorithm));
+      if (BUG(!r->d.ctx)) {
+        tor_free(r);
+        return NULL;
+      }
+      if (BUG(SECSuccess != PK11_DigestBegin(r->d.ctx))) {
+        crypto_digest_free(r);
+        return NULL;
+      }
+      break;
+#else
+    case DIGEST_SHA1:
+      SHA1_Init(&r->d.sha1);
+      break;
+    case DIGEST_SHA256:
+      SHA256_Init(&r->d.sha2);
+      break;
+    case DIGEST_SHA512:
+      SHA512_Init(&r->d.sha512);
+      break;
+#endif
+    case DIGEST_SHA3_256:
+      keccak_digest_init(&r->d.sha3, 256);
+      break;
+    case DIGEST_SHA3_512:
+      keccak_digest_init(&r->d.sha3, 512);
+      break;
+    default:
+      tor_assert_unreached();
+    }
+
+  return r;
+}
+
+/** Allocate and return a new digest object to compute SHA1 digests.
+ */
+crypto_digest_t *
+crypto_digest_new(void)
+{
+  return crypto_digest_new_internal(DIGEST_SHA1);
+}
+
+/** Allocate and return a new digest object to compute 256-bit digests
+ * using <b>algorithm</b>.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new`
+ * C_RUST_COUPLED: `crypto::digest::Sha256::default`
+ */
+crypto_digest_t *
+crypto_digest256_new(digest_algorithm_t algorithm)
+{
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+  return crypto_digest_new_internal(algorithm);
+}
+
+/** Allocate and return a new digest object to compute 512-bit digests
+ * using <b>algorithm</b>. */
+crypto_digest_t *
+crypto_digest512_new(digest_algorithm_t algorithm)
+{
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+  return crypto_digest_new_internal(algorithm);
+}
+
+/** Deallocate a digest object.
+ */
+void
+crypto_digest_free_(crypto_digest_t *digest)
+{
+  if (!digest)
+    return;
+#ifdef ENABLE_NSS
+  if (library_supports_digest(digest->algorithm)) {
+    PK11_DestroyContext(digest->d.ctx, PR_TRUE);
+  }
+#endif
+  size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  memwipe(digest, 0, bytes);
+  tor_free(digest);
+}
+
+/** Add <b>len</b> bytes from <b>data</b> to the digest object.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess`
+ * C_RUST_COUPLED: `crypto::digest::Sha256::process`
+ */
+void
+crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
+                        size_t len)
+{
+  tor_assert(digest);
+  tor_assert(data);
+  /* Using the SHA*_*() calls directly means we don't support doing
+   * SHA in hardware. But so far the delay of getting the question
+   * to the hardware, and hearing the answer, is likely higher than
+   * just doing it ourselves. Hashes are fast.
+   */
+  switch (digest->algorithm) {
+#ifdef ENABLE_NSS
+    case DIGEST_SHA1: /* fall through */
+    case DIGEST_SHA256: /* fall through */
+    case DIGEST_SHA512:
+      tor_assert(len <= UINT_MAX);
+      SECStatus s = PK11_DigestOp(digest->d.ctx,
+                                  (const unsigned char *)data,
+                                  (unsigned int)len);
+      tor_assert(s == SECSuccess);
+      break;
+#else
+    case DIGEST_SHA1:
+      SHA1_Update(&digest->d.sha1, (void*)data, len);
+      break;
+    case DIGEST_SHA256:
+      SHA256_Update(&digest->d.sha2, (void*)data, len);
+      break;
+    case DIGEST_SHA512:
+      SHA512_Update(&digest->d.sha512, (void*)data, len);
+      break;
+#endif
+    case DIGEST_SHA3_256: /* FALLSTHROUGH */
+    case DIGEST_SHA3_512:
+      keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len);
+      break;
+    default:
+      /* LCOV_EXCL_START */
+      tor_fragile_assert();
+      break;
+      /* LCOV_EXCL_STOP */
+  }
+}
+
+/** Compute the hash of the data that has been passed to the digest
+ * object; write the first out_len bytes of the result to <b>out</b>.
+ * <b>out_len</b> must be \<= DIGEST512_LEN.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest`
+ * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256`
+ */
+void
+crypto_digest_get_digest(crypto_digest_t *digest,
+                         char *out, size_t out_len)
+{
+  unsigned char r[DIGEST512_LEN];
+  tor_assert(digest);
+  tor_assert(out);
+  tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm));
+
+  /* The SHA-3 code handles copying into a temporary ctx, and also can handle
+   * short output buffers by truncating appropriately. */
+  if (digest->algorithm == DIGEST_SHA3_256 ||
+      digest->algorithm == DIGEST_SHA3_512) {
+    keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len);
+    return;
+  }
+
+#ifdef ENABLE_NSS
+  /* Copy into a temporary buffer since DigestFinal (alters) the context */
+  unsigned char buf[1024];
+  unsigned int saved_len = 0;
+  unsigned rlen;
+  unsigned char *saved = PK11_SaveContextAlloc(digest->d.ctx,
+                                               buf, sizeof(buf),
+                                               &saved_len);
+  tor_assert(saved);
+  SECStatus s = PK11_DigestFinal(digest->d.ctx, r, &rlen, sizeof(r));
+  tor_assert(s == SECSuccess);
+  tor_assert(rlen >= out_len);
+  s = PK11_RestoreContext(digest->d.ctx, saved, saved_len);
+  tor_assert(s == SECSuccess);
+  if (saved != buf) {
+    PORT_ZFree(saved, saved_len);
+  }
+#else
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  crypto_digest_t tmpenv;
+  /* memcpy into a temporary ctx, since SHA*_Final clears the context */
+  memcpy(&tmpenv, digest, alloc_bytes);
+  switch (digest->algorithm) {
+    case DIGEST_SHA1:
+      SHA1_Final(r, &tmpenv.d.sha1);
+      break;
+    case DIGEST_SHA256:
+      SHA256_Final(r, &tmpenv.d.sha2);
+      break;
+    case DIGEST_SHA512:
+      SHA512_Final(r, &tmpenv.d.sha512);
+      break;
+//LCOV_EXCL_START
+    case DIGEST_SHA3_256: /* FALLSTHROUGH */
+    case DIGEST_SHA3_512:
+    default:
+      log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm);
+      /* This is fatal, because it should never happen. */
+      tor_assert_unreached();
+      break;
+//LCOV_EXCL_STOP
+  }
+#endif
+  memcpy(out, r, out_len);
+  memwipe(r, 0, sizeof(r));
+}
+
+/** Allocate and return a new digest object with the same state as
+ * <b>digest</b>
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup`
+ * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256`
+ */
+crypto_digest_t *
+crypto_digest_dup(const crypto_digest_t *digest)
+{
+  tor_assert(digest);
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  crypto_digest_t *result = tor_memdup(digest, alloc_bytes);
+#ifdef ENABLE_NSS
+  if (library_supports_digest(digest->algorithm)) {
+    result->d.ctx = PK11_CloneContext(digest->d.ctx);
+  }
+#endif
+  return result;
+}
+
+/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>.
+ * Asserts that <b>digest</b> is a SHA1 digest object.
+ */
+void
+crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint,
+                         const crypto_digest_t *digest)
+{
+  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  tor_assert(bytes <= sizeof(checkpoint->mem));
+#ifdef ENABLE_NSS
+  if (library_supports_digest(digest->algorithm)) {
+    unsigned char *allocated;
+    allocated = PK11_SaveContextAlloc(digest->d.ctx,
+                                      (unsigned char *)checkpoint->mem,
+                                      sizeof(checkpoint->mem),
+                                      &checkpoint->bytes_used);
+    /* No allocation is allowed here. */
+    tor_assert(allocated == checkpoint->mem);
+    return;
+  }
+#endif
+  memcpy(checkpoint->mem, digest, bytes);
+}
+
+/** Restore the state of  <b>digest</b> from <b>checkpoint</b>.
+ * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the
+ * state was previously stored with crypto_digest_checkpoint() */
+void
+crypto_digest_restore(crypto_digest_t *digest,
+                      const crypto_digest_checkpoint_t *checkpoint)
+{
+  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+#ifdef ENABLE_NSS
+  if (library_supports_digest(digest->algorithm)) {
+    SECStatus s = PK11_RestoreContext(digest->d.ctx,
+                                      (unsigned char *)checkpoint->mem,
+                                      checkpoint->bytes_used);
+    tor_assert(s == SECSuccess);
+    return;
+  }
+#endif
+  memcpy(digest, checkpoint->mem, bytes);
+}
+
+/** Replace the state of the digest object <b>into</b> with the state
+ * of the digest object <b>from</b>.  Requires that 'into' and 'from'
+ * have the same digest type.
+ */
+void
+crypto_digest_assign(crypto_digest_t *into,
+                     const crypto_digest_t *from)
+{
+  tor_assert(into);
+  tor_assert(from);
+  tor_assert(into->algorithm == from->algorithm);
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm);
+#ifdef ENABLE_NSS
+  if (library_supports_digest(from->algorithm)) {
+    PK11_DestroyContext(into->d.ctx, PR_TRUE);
+    into->d.ctx = PK11_CloneContext(from->d.ctx);
+    return;
+  }
+#endif
+  memcpy(into,from,alloc_bytes);
+}
+
+/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
+ * at <b>digest_out</b> to the hash of the concatenation of those strings,
+ * plus the optional string <b>append</b>, computed with the algorithm
+ * <b>alg</b>.
+ * <b>out_len</b> must be \<= DIGEST512_LEN. */
+void
+crypto_digest_smartlist(char *digest_out, size_t len_out,
+                        const smartlist_t *lst,
+                        const char *append,
+                        digest_algorithm_t alg)
+{
+  crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg);
+}
+
+/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
+ * at <b>digest_out</b> to the hash of the concatenation of: the
+ * optional string <b>prepend</b>, those strings,
+ * and the optional string <b>append</b>, computed with the algorithm
+ * <b>alg</b>.
+ * <b>len_out</b> must be \<= DIGEST512_LEN. */
+void
+crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
+                        const char *prepend,
+                        const smartlist_t *lst,
+                        const char *append,
+                        digest_algorithm_t alg)
+{
+  crypto_digest_t *d = crypto_digest_new_internal(alg);
+  if (prepend)
+    crypto_digest_add_bytes(d, prepend, strlen(prepend));
+  SMARTLIST_FOREACH(lst, const char *, cp,
+                    crypto_digest_add_bytes(d, cp, strlen(cp)));
+  if (append)
+    crypto_digest_add_bytes(d, append, strlen(append));
+  crypto_digest_get_digest(d, digest_out, len_out);
+  crypto_digest_free(d);
+}
+
+/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using
+ * the <b>key</b> of length <b>key_len</b>.  Store the DIGEST256_LEN-byte
+ * result in <b>hmac_out</b>. Asserts on failure.
+ */
+void
+crypto_hmac_sha256(char *hmac_out,
+                   const char *key, size_t key_len,
+                   const char *msg, size_t msg_len)
+{
+  /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */
+  tor_assert(key_len < INT_MAX);
+  tor_assert(msg_len < INT_MAX);
+  tor_assert(hmac_out);
+#ifdef ENABLE_NSS
+  PK11SlotInfo *slot = NULL;
+  PK11SymKey *symKey = NULL;
+  PK11Context *hmac = NULL;
+
+  int ok = 0;
+  SECStatus s;
+  SECItem keyItem, paramItem;
+  keyItem.data = (unsigned char *)key;
+  keyItem.len = (unsigned)key_len;
+  paramItem.type = siBuffer;
+  paramItem.data = NULL;
+  paramItem.len = 0;
+
+  slot = PK11_GetBestSlot(CKM_SHA256_HMAC, NULL);
+  if (!slot)
+    goto done;
+  symKey = PK11_ImportSymKey(slot, CKM_SHA256_HMAC,
+                             PK11_OriginUnwrap, CKA_SIGN, &keyItem, NULL);
+  if (!symKey)
+    goto done;
+
+  hmac = PK11_CreateContextBySymKey(CKM_SHA256_HMAC, CKA_SIGN, symKey,
+                                    &paramItem);
+  if (!hmac)
+    goto done;
+  s = PK11_DigestBegin(hmac);
+  if (s != SECSuccess)
+    goto done;
+  s = PK11_DigestOp(hmac, (const unsigned char *)msg, (unsigned int)msg_len);
+  if (s != SECSuccess)
+    goto done;
+  unsigned int len=0;
+  s = PK11_DigestFinal(hmac, (unsigned char *)hmac_out, &len, DIGEST256_LEN);
+  if (s != SECSuccess || len != DIGEST256_LEN)
+    goto done;
+  ok = 1;
+
+ done:
+  if (hmac)
+    PK11_DestroyContext(hmac, PR_TRUE);
+  if (symKey)
+    PK11_FreeSymKey(symKey);
+  if (slot)
+    PK11_FreeSlot(slot);
+
+  tor_assert(ok);
+#else
+  unsigned char *rv = NULL;
+  rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len,
+            (unsigned char*)hmac_out, NULL);
+  tor_assert(rv);
+#endif
+}
+
+/** Compute a MAC using SHA3-256 of <b>msg_len</b> bytes in <b>msg</b> using a
+ * <b>key</b> of length <b>key_len</b> and a <b>salt</b> of length
+ * <b>salt_len</b>. Store the result of <b>len_out</b> bytes in in
+ * <b>mac_out</b>. This function can't fail. */
+void
+crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out,
+                    const uint8_t *key, size_t key_len,
+                    const uint8_t *msg, size_t msg_len)
+{
+  crypto_digest_t *digest;
+
+  const uint64_t key_len_netorder = tor_htonll(key_len);
+
+  tor_assert(mac_out);
+  tor_assert(key);
+  tor_assert(msg);
+
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+
+  /* Order matters here that is any subsystem using this function should
+   * expect this very precise ordering in the MAC construction. */
+  crypto_digest_add_bytes(digest, (const char *) &key_len_netorder,
+                          sizeof(key_len_netorder));
+  crypto_digest_add_bytes(digest, (const char *) key, key_len);
+  crypto_digest_add_bytes(digest, (const char *) msg, msg_len);
+  crypto_digest_get_digest(digest, (char *) mac_out, len_out);
+  crypto_digest_free(digest);
+}
+
+/* xof functions  */
+
+/** Internal state for a eXtendable-Output Function (XOF). */
+struct crypto_xof_t {
+  keccak_state s;
+};
+
+/** Allocate a new XOF object backed by SHAKE-256.  The security level
+ * provided is a function of the length of the output used.  Read and
+ * understand FIPS-202 A.2 "Additional Consideration for Extendable-Output
+ * Functions" before using this construct.
+ */
+crypto_xof_t *
+crypto_xof_new(void)
+{
+  crypto_xof_t *xof;
+  xof = tor_malloc(sizeof(crypto_xof_t));
+  keccak_xof_init(&xof->s, 256);
+  return xof;
+}
+
+/** Absorb bytes into a XOF object.  Must not be called after a call to
+ * crypto_xof_squeeze_bytes() for the same instance, and will assert
+ * if attempted.
+ */
+void
+crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len)
+{
+  int i = keccak_xof_absorb(&xof->s, data, len);
+  tor_assert(i == 0);
+}
+
+/** Squeeze bytes out of a XOF object.  Calling this routine will render
+ * the XOF instance ineligible to absorb further data.
+ */
+void
+crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len)
+{
+  int i = keccak_xof_squeeze(&xof->s, out, len);
+  tor_assert(i == 0);
+}
+
+/** Cleanse and deallocate a XOF object. */
+void
+crypto_xof_free_(crypto_xof_t *xof)
+{
+  if (!xof)
+    return;
+  memwipe(xof, 0, sizeof(crypto_xof_t));
+  tor_free(xof);
+}
diff --git a/src/lib/crypt_ops/crypto_digest.h b/src/lib/crypt_ops/crypto_digest.h
new file mode 100644
index 0000000000..47e60ce617
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_digest.h
@@ -0,0 +1,132 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_digest.h
+ *
+ * \brief Headers for crypto_digest.c
+ **/
+
+#ifndef TOR_CRYPTO_DIGEST_H
+#define TOR_CRYPTO_DIGEST_H
+
+#include "lib/cc/torint.h"
+#include "lib/defs/digest_sizes.h"
+#include "lib/malloc/malloc.h"
+#include "lib/testsupport/testsupport.h"
+
+/** Length of a sha1 message digest when encoded in base32 with trailing =
+ * signs removed. */
+#define BASE32_DIGEST_LEN 32
+/** Length of a sha1 message digest when encoded in base64 with trailing =
+ * signs removed. */
+#define BASE64_DIGEST_LEN 27
+/** Length of a sha256 message digest when encoded in base64 with trailing =
+ * signs removed. */
+#define BASE64_DIGEST256_LEN 43
+/** Length of a sha512 message digest when encoded in base64 with trailing =
+ * signs removed. */
+#define BASE64_DIGEST512_LEN 86
+
+/** Length of hex encoding of SHA1 digest, not including final NUL. */
+#define HEX_DIGEST_LEN 40
+/** Length of hex encoding of SHA256 digest, not including final NUL. */
+#define HEX_DIGEST256_LEN 64
+/** Length of hex encoding of SHA512 digest, not including final NUL. */
+#define HEX_DIGEST512_LEN 128
+
+typedef enum {
+  DIGEST_SHA1 = 0,
+  DIGEST_SHA256 = 1,
+  DIGEST_SHA512 = 2,
+  DIGEST_SHA3_256 = 3,
+  DIGEST_SHA3_512 = 4,
+} digest_algorithm_t;
+#define  N_DIGEST_ALGORITHMS (DIGEST_SHA3_512+1)
+#define  N_COMMON_DIGEST_ALGORITHMS (DIGEST_SHA256+1)
+
+#define DIGEST_CHECKPOINT_BYTES (SIZEOF_VOID_P + 512)
+/** Structure used to temporarily save the a digest object. Only implemented
+ * for SHA1 digest for now. */
+typedef struct crypto_digest_checkpoint_t {
+#ifdef ENABLE_NSS
+  unsigned int bytes_used;
+#endif
+  uint8_t mem[DIGEST_CHECKPOINT_BYTES];
+} crypto_digest_checkpoint_t;
+
+/** A set of all the digests we commonly compute, taken on a single
+ * string.  Any digests that are shorter than 512 bits are right-padded
+ * with 0 bits.
+ *
+ * Note that this representation wastes 44 bytes for the SHA1 case, so
+ * don't use it for anything where we need to allocate a whole bunch at
+ * once.
+ **/
+typedef struct {
+  char d[N_COMMON_DIGEST_ALGORITHMS][DIGEST256_LEN];
+} common_digests_t;
+
+typedef struct crypto_digest_t crypto_digest_t;
+typedef struct crypto_xof_t crypto_xof_t;
+
+struct smartlist_t;
+
+/* SHA-1 and other digests */
+MOCK_DECL(int, crypto_digest,(char *digest, const char *m, size_t len));
+int crypto_digest256(char *digest, const char *m, size_t len,
+                     digest_algorithm_t algorithm);
+int crypto_digest512(char *digest, const char *m, size_t len,
+                     digest_algorithm_t algorithm);
+int crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len);
+void crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
+                                    const char *prepend,
+                                    const struct smartlist_t *lst,
+                                    const char *append,
+                                    digest_algorithm_t alg);
+void crypto_digest_smartlist(char *digest_out, size_t len_out,
+                             const struct smartlist_t *lst, const char *append,
+                             digest_algorithm_t alg);
+const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg);
+size_t crypto_digest_algorithm_get_length(digest_algorithm_t alg);
+int crypto_digest_algorithm_parse_name(const char *name);
+crypto_digest_t *crypto_digest_new(void);
+crypto_digest_t *crypto_digest256_new(digest_algorithm_t algorithm);
+crypto_digest_t *crypto_digest512_new(digest_algorithm_t algorithm);
+void crypto_digest_free_(crypto_digest_t *digest);
+#define crypto_digest_free(d) \
+  FREE_AND_NULL(crypto_digest_t, crypto_digest_free_, (d))
+void crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
+                             size_t len);
+void crypto_digest_get_digest(crypto_digest_t *digest,
+                              char *out, size_t out_len);
+crypto_digest_t *crypto_digest_dup(const crypto_digest_t *digest);
+void crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint,
+                              const crypto_digest_t *digest);
+void crypto_digest_restore(crypto_digest_t *digest,
+                           const crypto_digest_checkpoint_t *checkpoint);
+void crypto_digest_assign(crypto_digest_t *into,
+                          const crypto_digest_t *from);
+void crypto_hmac_sha256(char *hmac_out,
+                        const char *key, size_t key_len,
+                        const char *msg, size_t msg_len);
+void crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out,
+                         const uint8_t *key, size_t key_len,
+                         const uint8_t *msg, size_t msg_len);
+
+/* xof functions*/
+crypto_xof_t *crypto_xof_new(void);
+void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len);
+void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len);
+void crypto_xof_free_(crypto_xof_t *xof);
+#define crypto_xof_free(xof) \
+  FREE_AND_NULL(crypto_xof_t, crypto_xof_free_, (xof))
+
+#ifdef TOR_UNIT_TESTS
+digest_algorithm_t crypto_digest_get_algorithm(crypto_digest_t *digest);
+#endif
+
+#endif /* !defined(TOR_CRYPTO_DIGEST_H) */
diff --git a/src/common/crypto_ed25519.c b/src/lib/crypt_ops/crypto_ed25519.c
index 30ed772274..400f963898 100644
--- a/src/common/crypto_ed25519.c
+++ b/src/lib/crypt_ops/crypto_ed25519.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,26 +15,31 @@
  * keys to and from the corresponding Curve25519 keys.
  */
 
+#define CRYPTO_ED25519_PRIVATE
 #include "orconfig.h"
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
 #endif
 
-#include "crypto.h"
-
-#include "crypto_curve25519.h"
-#include "crypto_ed25519.h"
-#include "crypto_format.h"
-#include "torlog.h"
-#include "util.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/encoding/binascii.h"
+#include "lib/string/util_string.h"
 
 #include "ed25519/ref10/ed25519_ref10.h"
 #include "ed25519/donna/ed25519_donna_tor.h"
 
-#include <openssl/sha.h>
+#include <string.h>
+#include <errno.h>
 
 static void pick_ed25519_impl(void);
-static int ed25519_impl_spot_check(void);
 
 /** An Ed25519 implementation, as a set of function pointers. */
 typedef struct {
@@ -59,6 +64,9 @@ typedef struct {
 
   int (*pubkey_from_curve25519_pubkey)(unsigned char *, const unsigned char *,
                                        int);
+
+  int (*ed25519_scalarmult_with_group_order)(unsigned char *,
+                                             const unsigned char *);
 } ed25519_impl_t;
 
 /** The Ref10 Ed25519 implementation. This one is pure C and lightly
@@ -79,6 +87,7 @@ static const ed25519_impl_t impl_ref10 = {
   ed25519_ref10_blind_public_key,
 
   ed25519_ref10_pubkey_from_curve25519_pubkey,
+  ed25519_ref10_scalarmult_with_group_order,
 };
 
 /** The Ref10 Ed25519 implementation. This one is heavily optimized, but still
@@ -99,6 +108,7 @@ static const ed25519_impl_t impl_donna = {
   ed25519_donna_blind_public_key,
 
   ed25519_donna_pubkey_from_curve25519_pubkey,
+  ed25519_donna_scalarmult_with_group_order,
 };
 
 /** Which Ed25519 implementation are we using?  NULL if we haven't decided
@@ -147,7 +157,7 @@ crypto_ed25519_testing_restore_impl(void)
   ed25519_impl = saved_ed25519_impl;
   saved_ed25519_impl = NULL;
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /**
  * Initialize a new ed25519 secret key in <b>seckey_out</b>.  If
@@ -211,10 +221,18 @@ ed25519_keypair_generate(ed25519_keypair_t *keypair_out, int extra_strong)
   return 0;
 }
 
+/** Return true iff 'pubkey' is set to zero (eg to indicate that it is not
+ * set). */
+int
+ed25519_public_key_is_zero(const ed25519_public_key_t *pubkey)
+{
+  return tor_mem_is_zero((char*)pubkey->pubkey, ED25519_PUBKEY_LEN);
+}
+
 /* Return a heap-allocated array that contains <b>msg</b> prefixed by the
  * string <b>prefix_str</b>. Set <b>final_msg_len_out</b> to the size of the
- * final array. If an error occured, return NULL. It's the resonsibility of the
- * caller to free the returned array. */
+ * final array. If an error occurred, return NULL. It's the responsibility of
+ * the caller to free the returned array. */
 static uint8_t *
 get_prefixed_msg(const uint8_t *msg, size_t msg_len,
                  const char *prefix_str,
@@ -247,7 +265,7 @@ get_prefixed_msg(const uint8_t *msg, size_t msg_len,
  * Set <b>signature_out</b> to a signature of the <b>len</b>-byte message
  * <b>msg</b>, using the secret and public key in <b>keypair</b>.
  *
- * Return 0 if we successfuly signed the message, otherwise return -1.
+ * Return 0 if we successfully signed the message, otherwise return -1.
  */
 int
 ed25519_sign(ed25519_signature_t *signature_out,
@@ -267,11 +285,11 @@ ed25519_sign(ed25519_signature_t *signature_out,
  * Like ed25519_sign(), but also prefix <b>msg</b> with <b>prefix_str</b>
  * before signing. <b>prefix_str</b> must be a NUL-terminated string.
  */
-int
-ed25519_sign_prefixed(ed25519_signature_t *signature_out,
-                      const uint8_t *msg, size_t msg_len,
-                      const char *prefix_str,
-                      const ed25519_keypair_t *keypair)
+MOCK_IMPL(int,
+ed25519_sign_prefixed,(ed25519_signature_t *signature_out,
+                       const uint8_t *msg, size_t msg_len,
+                       const char *prefix_str,
+                       const ed25519_keypair_t *keypair))
 {
   int retval;
   size_t prefixed_msg_len;
@@ -281,9 +299,12 @@ ed25519_sign_prefixed(ed25519_signature_t *signature_out,
 
   prefixed_msg = get_prefixed_msg(msg, msg_len, prefix_str,
                                   &prefixed_msg_len);
-  if (!prefixed_msg) {
+  if (BUG(!prefixed_msg)) {
+    /* LCOV_EXCL_START -- only possible when the message and prefix are
+     * ridiculously huge */
     log_warn(LD_GENERAL, "Failed to get prefixed msg.");
     return -1;
+    /* LCOV_EXCL_STOP */
   }
 
   retval = ed25519_sign(signature_out,
@@ -300,10 +321,10 @@ ed25519_sign_prefixed(ed25519_signature_t *signature_out,
  *
  * Return 0 if the signature is valid; -1 if it isn't.
  */
-int
-ed25519_checksig(const ed25519_signature_t *signature,
-                 const uint8_t *msg, size_t len,
-                 const ed25519_public_key_t *pubkey)
+MOCK_IMPL(int,
+ed25519_checksig,(const ed25519_signature_t *signature,
+                  const uint8_t *msg, size_t len,
+                  const ed25519_public_key_t *pubkey))
 {
   return
     get_ed_impl()->open(signature->sig, msg, len, pubkey->pubkey) < 0 ? -1 : 0;
@@ -326,9 +347,12 @@ ed25519_checksig_prefixed(const ed25519_signature_t *signature,
 
   prefixed_msg = get_prefixed_msg(msg, msg_len, prefix_str,
                                   &prefixed_msg_len);
-  if (!prefixed_msg) {
+  if (BUG(!prefixed_msg)) {
+    /* LCOV_EXCL_START -- only possible when the message and prefix are
+     * ridiculously huge */
     log_warn(LD_GENERAL, "Failed to get prefixed msg.");
     return -1;
+    /* LCOV_EXCL_STOP */
   }
 
   retval = ed25519_checksig(signature,
@@ -346,10 +370,10 @@ ed25519_checksig_prefixed(const ed25519_signature_t *signature,
  * was valid. Otherwise return -N, where N is the number of invalid
  * signatures.
  */
-int
-ed25519_checksig_batch(int *okay_out,
-                       const ed25519_checkable_t *checkable,
-                       int n_checkable)
+MOCK_IMPL(int,
+ed25519_checksig_batch,(int *okay_out,
+                        const ed25519_checkable_t *checkable,
+                        int n_checkable))
 {
   int i, res;
   const ed25519_impl_t *impl = get_ed_impl();
@@ -434,14 +458,16 @@ ed25519_keypair_from_curve25519_keypair(ed25519_keypair_t *out,
 {
   const char string[] = "Derive high part of ed25519 key from curve25519 key";
   ed25519_public_key_t pubkey_check;
-  SHA512_CTX ctx;
-  uint8_t sha512_output[64];
+  crypto_digest_t *ctx;
+  uint8_t sha512_output[DIGEST512_LEN];
 
   memcpy(out->seckey.seckey, inp->seckey.secret_key, 32);
-  SHA512_Init(&ctx);
-  SHA512_Update(&ctx, out->seckey.seckey, 32);
-  SHA512_Update(&ctx, string, sizeof(string));
-  SHA512_Final(sha512_output, &ctx);
+
+  ctx = crypto_digest512_new(DIGEST_SHA512);
+  crypto_digest_add_bytes(ctx, (const char*)out->seckey.seckey, 32);
+  crypto_digest_add_bytes(ctx, (const char*)string, sizeof(string));
+  crypto_digest_get_digest(ctx, (char *)sha512_output, sizeof(sha512_output));
+  crypto_digest_free(ctx);
   memcpy(out->seckey.seckey + 32, sha512_output, 32);
 
   ed25519_public_key_generate(&out->pubkey, &out->seckey);
@@ -454,7 +480,6 @@ ed25519_keypair_from_curve25519_keypair(ed25519_keypair_t *out,
   tor_assert(fast_memeq(pubkey_check.pubkey, out->pubkey.pubkey, 32));
 
   memwipe(&pubkey_check, 0, sizeof(pubkey_check));
-  memwipe(&ctx, 0, sizeof(ctx));
   memwipe(sha512_output, 0, sizeof(sha512_output));
 
   return 0;
@@ -483,7 +508,8 @@ ed25519_public_key_from_curve25519_public_key(ed25519_public_key_t *pubkey,
  * service descriptors are encrypted with a key derived from the service's
  * long-term public key, and then signed with (and stored at a position
  * indexed by) a short-term key derived by blinding the long-term keys.
- */
+ *
+ * Return 0 if blinding was successful, else return -1. */
 int
 ed25519_keypair_blind(ed25519_keypair_t *out,
                       const ed25519_keypair_t *inp,
@@ -494,7 +520,9 @@ ed25519_keypair_blind(ed25519_keypair_t *out,
   get_ed_impl()->blind_secret_key(out->seckey.seckey,
                                   inp->seckey.seckey, param);
 
-  ed25519_public_blind(&pubkey_check, &inp->pubkey, param);
+  if (ed25519_public_blind(&pubkey_check, &inp->pubkey, param) < 0) {
+    return -1;
+  }
   ed25519_public_key_generate(&out->pubkey, &out->seckey);
 
   tor_assert(fast_memeq(pubkey_check.pubkey, out->pubkey.pubkey, 32));
@@ -507,15 +535,14 @@ ed25519_keypair_blind(ed25519_keypair_t *out,
 /**
  * Given an ed25519 public key in <b>inp</b>, generate a corresponding blinded
  * public key in <b>out</b>, blinded with the 32-byte parameter in
- * <b>param</b>.  Return 0 on sucess, -1 on railure.
+ * <b>param</b>.  Return 0 on success, -1 on railure.
  */
 int
 ed25519_public_blind(ed25519_public_key_t *out,
                      const ed25519_public_key_t *inp,
                      const uint8_t *param)
 {
-  get_ed_impl()->blind_public_key(out->pubkey, inp->pubkey, param);
-  return 0;
+  return get_ed_impl()->blind_public_key(out->pubkey, inp->pubkey, param);
 }
 
 /**
@@ -601,7 +628,7 @@ ed25519_pubkey_read_from_file(ed25519_public_key_t *pubkey_out,
 
 /** Release all storage held for <b>kp</b>. */
 void
-ed25519_keypair_free(ed25519_keypair_t *kp)
+ed25519_keypair_free_(ed25519_keypair_t *kp)
 {
   if (! kp)
     return;
@@ -620,10 +647,22 @@ ed25519_pubkey_eq(const ed25519_public_key_t *key1,
   return tor_memeq(key1->pubkey, key2->pubkey, ED25519_PUBKEY_LEN);
 }
 
+/**
+ * Set <b>dest</b> to contain the same key as <b>src</b>.
+ */
+void
+ed25519_pubkey_copy(ed25519_public_key_t *dest,
+                    const ed25519_public_key_t *src)
+{
+  tor_assert(dest);
+  tor_assert(src);
+  memcpy(dest, src, sizeof(ed25519_public_key_t));
+}
+
 /** Check whether the given Ed25519 implementation seems to be working.
  * If so, return 0; otherwise return -1. */
-static int
-ed25519_impl_spot_check(void)
+MOCK_IMPL(STATIC int,
+ed25519_impl_spot_check,(void))
 {
   static const uint8_t alicesk[32] = {
     0xc5,0xaa,0x8d,0xf4,0x3f,0x9f,0x83,0x7b,
@@ -686,13 +725,16 @@ ed25519_impl_spot_check(void)
     return -1;
 
   /* XXX/yawning: Someone that's more paranoid than I am, can write "Assume
-   * ref0 is cannonical, and fuzz impl against it" if they want, but I doubt
+   * ref0 is canonical, and fuzz impl against it" if they want, but I doubt
    * that will catch anything that the known answer tests won't.
    */
   goto end;
 
+ // LCOV_EXCL_START -- We can only reach this if our ed25519 implementation is
+ // broken.
  fail:
   r = -1;
+ // LCOV_EXCL_STOP
  end:
   return r;
 }
@@ -726,7 +768,7 @@ pick_ed25519_impl(void)
   /* LCOV_EXCL_STOP */
 }
 
-/* Initialize the Ed25519 implementation. This is neccessary if you're
+/* Initialize the Ed25519 implementation. This is necessary if you're
  * going to use them in a multithreaded setting, and not otherwise. */
 void
 ed25519_init(void)
@@ -734,3 +776,46 @@ ed25519_init(void)
   pick_ed25519_impl();
 }
 
+/* Return true if <b>point</b> is the identity element of the ed25519 group. */
+static int
+ed25519_point_is_identity_element(const uint8_t *point)
+{
+  /* The identity element in ed25159 is the point with coordinates (0,1). */
+  static const uint8_t ed25519_identity[32] = {
+    0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
+  tor_assert(sizeof(ed25519_identity) == ED25519_PUBKEY_LEN);
+  return tor_memeq(point, ed25519_identity, sizeof(ed25519_identity));
+}
+
+/** Validate <b>pubkey</b> to ensure that it has no torsion component.
+ *  Return 0 if <b>pubkey</b> is valid, else return -1. */
+int
+ed25519_validate_pubkey(const ed25519_public_key_t *pubkey)
+{
+  uint8_t result[32] = {9};
+
+  /* First check that we were not given the identity element */
+  if (ed25519_point_is_identity_element(pubkey->pubkey)) {
+    log_warn(LD_CRYPTO, "ed25519 pubkey is the identity");
+    return -1;
+  }
+
+  /* For any point on the curve, doing l*point should give the identity element
+   * (where l is the group order). Do the computation and check that the
+   * identity element is returned. */
+  if (get_ed_impl()->ed25519_scalarmult_with_group_order(result,
+                                                         pubkey->pubkey) < 0) {
+    log_warn(LD_CRYPTO, "ed25519 group order scalarmult failed");
+    return -1;
+  }
+
+  if (!ed25519_point_is_identity_element(result)) {
+    log_warn(LD_CRYPTO, "ed25519 validation failed");
+    return -1;
+  }
+
+  return 0;
+}
diff --git a/src/common/crypto_ed25519.h b/src/lib/crypt_ops/crypto_ed25519.h
index 31afc49ccc..325b28244d 100644
--- a/src/common/crypto_ed25519.h
+++ b/src/lib/crypt_ops/crypto_ed25519.h
@@ -1,30 +1,31 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * \file crypto_ed25519.h
+ * \brief Header for crypto_ed25519.c
+ **/
+
 #ifndef TOR_CRYPTO_ED25519_H
 #define TOR_CRYPTO_ED25519_H
 
-#include "testsupport.h"
-#include "torint.h"
-#include "crypto_curve25519.h"
-
-#define ED25519_PUBKEY_LEN 32
-#define ED25519_SECKEY_LEN 64
-#define ED25519_SECKEY_SEED_LEN 32
-#define ED25519_SIG_LEN 64
+#include "lib/testsupport/testsupport.h"
+#include "lib/cc/torint.h"
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "lib/defs/x25519_sizes.h"
 
 /** An Ed25519 signature. */
-typedef struct {
+typedef struct ed25519_signature_t {
   uint8_t sig[ED25519_SIG_LEN];
 } ed25519_signature_t;
 
 /** An Ed25519 public key */
-typedef struct {
+typedef struct ed25519_public_key_t {
   uint8_t pubkey[ED25519_PUBKEY_LEN];
 } ed25519_public_key_t;
 
 /** An Ed25519 secret key */
-typedef struct {
+typedef struct ed25519_secret_key_t {
   /** Note that we store secret keys in an expanded format that doesn't match
    * the format from standard ed25519.  Ed25519 stores a 32-byte value k and
    * expands it into a 64-byte H(k), using the first 32 bytes for a multiplier
@@ -35,7 +36,7 @@ typedef struct {
 } ed25519_secret_key_t;
 
 /** An Ed25519 keypair. */
-typedef struct {
+typedef struct ed25519_keypair_t {
   ed25519_public_key_t pubkey;
   ed25519_secret_key_t seckey;
 } ed25519_keypair_t;
@@ -51,21 +52,24 @@ int ed25519_keypair_generate(ed25519_keypair_t *keypair_out, int extra_strong);
 int ed25519_sign(ed25519_signature_t *signature_out,
                  const uint8_t *msg, size_t len,
                  const ed25519_keypair_t *key);
-int ed25519_checksig(const ed25519_signature_t *signature,
-                     const uint8_t *msg, size_t len,
-                     const ed25519_public_key_t *pubkey);
+MOCK_DECL(int,ed25519_checksig,(const ed25519_signature_t *signature,
+                                const uint8_t *msg, size_t len,
+                                const ed25519_public_key_t *pubkey));
+
+MOCK_DECL(int,
+ed25519_sign_prefixed,(ed25519_signature_t *signature_out,
+                       const uint8_t *msg, size_t len,
+                       const char *prefix_str,
+                       const ed25519_keypair_t *keypair));
 
-int
-ed25519_sign_prefixed(ed25519_signature_t *signature_out,
-                      const uint8_t *msg, size_t len,
-                      const char *prefix_str,
-                      const ed25519_keypair_t *keypair);
 int
 ed25519_checksig_prefixed(const ed25519_signature_t *signature,
                           const uint8_t *msg, size_t len,
                           const char *prefix_str,
                           const ed25519_public_key_t *pubkey);
 
+int ed25519_public_key_is_zero(const ed25519_public_key_t *pubkey);
+
 /**
  * A collection of information necessary to check an Ed25519 signature. Used
  * for batch verification.
@@ -81,9 +85,9 @@ typedef struct {
   size_t len;
 } ed25519_checkable_t;
 
-int ed25519_checksig_batch(int *okay_out,
-                           const ed25519_checkable_t *checkable,
-                           int n_checkable);
+MOCK_DECL(int, ed25519_checksig_batch,(int *okay_out,
+                                       const ed25519_checkable_t *checkable,
+                                       int n_checkable));
 
 int ed25519_keypair_from_curve25519_keypair(ed25519_keypair_t *out,
                                             int *signbit_out,
@@ -114,18 +118,27 @@ int ed25519_pubkey_read_from_file(ed25519_public_key_t *pubkey_out,
                                   char **tag_out,
                                   const char *filename);
 
-void ed25519_keypair_free(ed25519_keypair_t *kp);
+void ed25519_keypair_free_(ed25519_keypair_t *kp);
+#define ed25519_keypair_free(kp) \
+  FREE_AND_NULL(ed25519_keypair_t, ed25519_keypair_free_, (kp))
 
 int ed25519_pubkey_eq(const ed25519_public_key_t *key1,
                       const ed25519_public_key_t *key2);
+void ed25519_pubkey_copy(ed25519_public_key_t *dest,
+                         const ed25519_public_key_t *src);
 
 void ed25519_set_impl_params(int use_donna);
 void ed25519_init(void);
 
+int ed25519_validate_pubkey(const ed25519_public_key_t *pubkey);
+
 #ifdef TOR_UNIT_TESTS
 void crypto_ed25519_testing_force_impl(const char *name);
 void crypto_ed25519_testing_restore_impl(void);
 #endif
 
+#ifdef CRYPTO_ED25519_PRIVATE
+MOCK_DECL(STATIC int, ed25519_impl_spot_check, (void));
 #endif
 
+#endif /* !defined(TOR_CRYPTO_ED25519_H) */
diff --git a/src/common/crypto_format.c b/src/lib/crypt_ops/crypto_format.c
index 2f6d847c83..84f73e5272 100644
--- a/src/common/crypto_format.c
+++ b/src/lib/crypt_ops/crypto_format.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,14 +14,22 @@
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
 #endif
-#include "container.h"
-#include "crypto.h"
-#include "crypto_curve25519.h"
-#include "crypto_ed25519.h"
-#include "crypto_format.h"
-#include "util.h"
-#include "util_format.h"
-#include "torlog.h"
+#include "lib/container/smartlist.h"
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/string/compat_string.h"
+#include "lib/string/util_string.h"
+#include "lib/string/printf.h"
+#include "lib/encoding/binascii.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/fs/files.h"
+
+#include <string.h>
+#include <errno.h>
 
 /** Write the <b>datalen</b> bytes from <b>data</b> to the file named
  * <b>fname</b> in the tagged-data format.  This format contains a
@@ -161,6 +169,27 @@ curve25519_public_from_base64(curve25519_public_key_t *pkey,
   }
 }
 
+/** For logging convenience: Convert <b>pkey</b> to a statically allocated
+ * base64 string and return it. Not threadsafe. Format not meant to be
+ * computer-readable; it may change in the future. Subsequent calls invalidate
+ * previous returns. */
+const char *
+ed25519_fmt(const ed25519_public_key_t *pkey)
+{
+  static char formatted[ED25519_BASE64_LEN+1];
+  if (pkey) {
+    if (ed25519_public_key_is_zero(pkey)) {
+      strlcpy(formatted, "<unset>", sizeof(formatted));
+    } else {
+      int r = ed25519_public_to_base64(formatted, pkey);
+      tor_assert(!r);
+    }
+  } else {
+    strlcpy(formatted, "<null>", sizeof(formatted));
+  }
+  return formatted;
+}
+
 /** Try to decode the string <b>input</b> into an ed25519 public key. On
  * success, store the value in <b>pkey</b> and return 0. Otherwise return
  * -1. */
@@ -274,4 +303,3 @@ digest256_from_base64(char *digest, const char *d64)
   else
     return -1;
 }
-
diff --git a/src/common/crypto_format.h b/src/lib/crypt_ops/crypto_format.h
index 012e228cc4..fe852e6a61 100644
--- a/src/common/crypto_format.h
+++ b/src/lib/crypt_ops/crypto_format.h
@@ -1,15 +1,23 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * \file crypto_format.h
+ * \brief Header for crypto_format.c
+ **/
+
 #ifndef TOR_CRYPTO_FORMAT_H
 #define TOR_CRYPTO_FORMAT_H
 
-#include "testsupport.h"
-#include "torint.h"
-#include "crypto_ed25519.h"
+#include "lib/testsupport/testsupport.h"
+#include "lib/cc/torint.h"
+#include "lib/defs/x25519_sizes.h"
+
+struct ed25519_public_key_t;
+struct ed25519_signature_t;
 
 int crypto_write_tagged_contents_to_file(const char *fname,
                                          const char *typestring,
@@ -23,24 +31,20 @@ ssize_t crypto_read_tagged_contents_from_file(const char *fname,
                                               uint8_t *data_out,
                                               ssize_t data_out_len);
 
-#define ED25519_BASE64_LEN 43
-int ed25519_public_from_base64(ed25519_public_key_t *pkey,
+int ed25519_public_from_base64(struct ed25519_public_key_t *pkey,
                                const char *input);
 int ed25519_public_to_base64(char *output,
-                             const ed25519_public_key_t *pkey);
+                             const struct ed25519_public_key_t *pkey);
+const char *ed25519_fmt(const struct ed25519_public_key_t *pkey);
 
-/* XXXX move these to crypto_format.h */
-#define ED25519_SIG_BASE64_LEN 86
-
-int ed25519_signature_from_base64(ed25519_signature_t *sig,
+int ed25519_signature_from_base64(struct ed25519_signature_t *sig,
                                   const char *input);
 int ed25519_signature_to_base64(char *output,
-                                const ed25519_signature_t *sig);
+                                const struct ed25519_signature_t *sig);
 
 int digest_to_base64(char *d64, const char *digest);
 int digest_from_base64(char *digest, const char *d64);
 int digest256_to_base64(char *d64, const char *digest);
 int digest256_from_base64(char *digest, const char *d64);
 
-#endif
-
+#endif /* !defined(TOR_CRYPTO_FORMAT_H) */
diff --git a/src/lib/crypt_ops/crypto_hkdf.c b/src/lib/crypt_ops/crypto_hkdf.c
new file mode 100644
index 0000000000..fd2e701651
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_hkdf.c
@@ -0,0 +1,201 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_hkdf.c
+ * \brief Block of functions related with HKDF utilities and operations.
+ **/
+
+#include "lib/crypt_ops/crypto_hkdf.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/crypt_ops/crypto_digest.h"
+
+#include "lib/crypt_ops/crypto_openssl_mgt.h"
+#include "lib/intmath/cmp.h"
+#include "lib/log/util_bug.h"
+
+#ifdef ENABLE_OPENSSL
+#include <openssl/evp.h>
+#include <openssl/opensslv.h>
+
+#if defined(HAVE_ERR_LOAD_KDF_STRINGS)
+#include <openssl/kdf.h>
+#define HAVE_OPENSSL_HKDF 1
+#endif
+#endif
+
+#include <string.h>
+
+/** Given <b>key_in_len</b> bytes of negotiated randomness in <b>key_in</b>
+ * ("K"), expand it into <b>key_out_len</b> bytes of negotiated key material in
+ * <b>key_out</b> by taking the first <b>key_out_len</b> bytes of
+ *    H(K | [00]) | H(K | [01]) | ....
+ *
+ * This is the key expansion algorithm used in the "TAP" circuit extension
+ * mechanism; it shouldn't be used for new protocols.
+ *
+ * Return 0 on success, -1 on failure.
+ */
+int
+crypto_expand_key_material_TAP(const uint8_t *key_in, size_t key_in_len,
+                               uint8_t *key_out, size_t key_out_len)
+{
+  int i, r = -1;
+  uint8_t *cp, *tmp = tor_malloc(key_in_len+1);
+  uint8_t digest[DIGEST_LEN];
+
+  /* If we try to get more than this amount of key data, we'll repeat blocks.*/
+  tor_assert(key_out_len <= DIGEST_LEN*256);
+
+  memcpy(tmp, key_in, key_in_len);
+  for (cp = key_out, i=0; cp < key_out+key_out_len;
+       ++i, cp += DIGEST_LEN) {
+    tmp[key_in_len] = i;
+    if (crypto_digest((char*)digest, (const char *)tmp, key_in_len+1) < 0)
+      goto exit;
+    memcpy(cp, digest, MIN(DIGEST_LEN, key_out_len-(cp-key_out)));
+  }
+
+  r = 0;
+ exit:
+  memwipe(tmp, 0, key_in_len+1);
+  tor_free(tmp);
+  memwipe(digest, 0, sizeof(digest));
+  return r;
+}
+
+#ifdef HAVE_OPENSSL_HKDF
+/**
+ * Perform RFC5869 HKDF computation using OpenSSL (only to be called from
+ * crypto_expand_key_material_rfc5869_sha256_openssl). Note that OpenSSL
+ * requires input key to be nonempty and salt length to be equal or less
+ * than 1024.
+ */
+static int
+crypto_expand_key_material_rfc5869_sha256_openssl(
+                                    const uint8_t *key_in, size_t key_in_len,
+                                    const uint8_t *salt_in, size_t salt_in_len,
+                                    const uint8_t *info_in, size_t info_in_len,
+                                    uint8_t *key_out, size_t key_out_len)
+{
+  int r;
+  EVP_PKEY_CTX *evp_pkey_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_HKDF, NULL);
+  tor_assert(evp_pkey_ctx);
+  tor_assert(key_in_len != 0);
+  tor_assert(salt_in_len <= 1024);
+
+  r = EVP_PKEY_derive_init(evp_pkey_ctx);
+  tor_assert(r == 1);
+
+  r = EVP_PKEY_CTX_set_hkdf_md(evp_pkey_ctx, EVP_sha256());
+  tor_assert(r == 1);
+
+  r = EVP_PKEY_CTX_set1_hkdf_salt(evp_pkey_ctx, salt_in, (int)salt_in_len);
+  tor_assert(r == 1);
+
+  r = EVP_PKEY_CTX_set1_hkdf_key(evp_pkey_ctx, key_in, (int)key_in_len);
+  tor_assert(r == 1);
+
+  r = EVP_PKEY_CTX_add1_hkdf_info(evp_pkey_ctx, info_in, (int)info_in_len);
+  tor_assert(r == 1);
+
+  r = EVP_PKEY_derive(evp_pkey_ctx, key_out, &key_out_len);
+  tor_assert(r == 1);
+
+  EVP_PKEY_CTX_free(evp_pkey_ctx);
+  return 0;
+}
+
+#else
+
+/**
+ * Perform RFC5869 HKDF computation using our own legacy implementation.
+ * Only to be called from crypto_expand_key_material_rfc5869_sha256_openssl.
+ */
+static int
+crypto_expand_key_material_rfc5869_sha256_legacy(
+                                    const uint8_t *key_in, size_t key_in_len,
+                                    const uint8_t *salt_in, size_t salt_in_len,
+                                    const uint8_t *info_in, size_t info_in_len,
+                                    uint8_t *key_out, size_t key_out_len)
+{
+  uint8_t prk[DIGEST256_LEN];
+  uint8_t tmp[DIGEST256_LEN + 128 + 1];
+  uint8_t mac[DIGEST256_LEN];
+  int i;
+  uint8_t *outp;
+  size_t tmp_len;
+
+  crypto_hmac_sha256((char*)prk,
+                     (const char*)salt_in, salt_in_len,
+                     (const char*)key_in, key_in_len);
+
+  /* If we try to get more than this amount of key data, we'll repeat blocks.*/
+  tor_assert(key_out_len <= DIGEST256_LEN * 256);
+  tor_assert(info_in_len <= 128);
+  memset(tmp, 0, sizeof(tmp));
+  outp = key_out;
+  i = 1;
+
+  while (key_out_len) {
+    size_t n;
+    if (i > 1) {
+      memcpy(tmp, mac, DIGEST256_LEN);
+      memcpy(tmp+DIGEST256_LEN, info_in, info_in_len);
+      tmp[DIGEST256_LEN+info_in_len] = i;
+      tmp_len = DIGEST256_LEN + info_in_len + 1;
+    } else {
+      memcpy(tmp, info_in, info_in_len);
+      tmp[info_in_len] = i;
+      tmp_len = info_in_len + 1;
+    }
+    crypto_hmac_sha256((char*)mac,
+                       (const char*)prk, DIGEST256_LEN,
+                       (const char*)tmp, tmp_len);
+    n = key_out_len < DIGEST256_LEN ? key_out_len : DIGEST256_LEN;
+    memcpy(outp, mac, n);
+    key_out_len -= n;
+    outp += n;
+    ++i;
+  }
+
+  memwipe(tmp, 0, sizeof(tmp));
+  memwipe(mac, 0, sizeof(mac));
+  return 0;
+}
+#endif
+
+/** Expand some secret key material according to RFC5869, using SHA256 as the
+ * underlying hash.  The <b>key_in_len</b> bytes at <b>key_in</b> are the
+ * secret key material; the <b>salt_in_len</b> bytes at <b>salt_in</b> and the
+ * <b>info_in_len</b> bytes in <b>info_in_len</b> are the algorithm's "salt"
+ * and "info" parameters respectively.  On success, write <b>key_out_len</b>
+ * bytes to <b>key_out</b> and return 0.  Assert on failure.
+ */
+int
+crypto_expand_key_material_rfc5869_sha256(
+                                    const uint8_t *key_in, size_t key_in_len,
+                                    const uint8_t *salt_in, size_t salt_in_len,
+                                    const uint8_t *info_in, size_t info_in_len,
+                                    uint8_t *key_out, size_t key_out_len)
+{
+  tor_assert(key_in);
+  tor_assert(key_in_len > 0);
+
+#ifdef HAVE_OPENSSL_HKDF
+  return crypto_expand_key_material_rfc5869_sha256_openssl(key_in,
+                                             key_in_len, salt_in,
+                                             salt_in_len, info_in,
+                                             info_in_len,
+                                             key_out, key_out_len);
+#else
+  return crypto_expand_key_material_rfc5869_sha256_legacy(key_in,
+                                               key_in_len, salt_in,
+                                               salt_in_len, info_in,
+                                               info_in_len,
+                                               key_out, key_out_len);
+#endif
+}
diff --git a/src/lib/crypt_ops/crypto_hkdf.h b/src/lib/crypt_ops/crypto_hkdf.h
new file mode 100644
index 0000000000..2994d18e3d
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_hkdf.h
@@ -0,0 +1,27 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_hkdf.h
+ *
+ * \brief Headers for crypto_hkdf.h
+ **/
+
+#ifndef TOR_CRYPTO_HKDF_H
+#define TOR_CRYPTO_HKDF_H
+
+#include "lib/cc/torint.h"
+
+int crypto_expand_key_material_TAP(const uint8_t *key_in,
+                                   size_t key_in_len,
+                                   uint8_t *key_out, size_t key_out_len);
+int crypto_expand_key_material_rfc5869_sha256(
+                                    const uint8_t *key_in, size_t key_in_len,
+                                    const uint8_t *salt_in, size_t salt_in_len,
+                                    const uint8_t *info_in, size_t info_in_len,
+                                    uint8_t *key_out, size_t key_out_len);
+
+#endif /* !defined(TOR_CRYPTO_HKDF_H) */
diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c
new file mode 100644
index 0000000000..329c264af6
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_init.c
@@ -0,0 +1,204 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_init.c
+ *
+ * \brief Initialize and shut down Tor's crypto library and subsystem.
+ **/
+
+#include "orconfig.h"
+
+#include "lib/crypt_ops/crypto_init.h"
+
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "lib/crypt_ops/crypto_openssl_mgt.h"
+#include "lib/crypt_ops/crypto_nss_mgt.h"
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "siphash.h"
+
+/** Boolean: has our crypto library been initialized? (early phase) */
+static int crypto_early_initialized_ = 0;
+
+/** Boolean: has our crypto library been initialized? (late phase) */
+static int crypto_global_initialized_ = 0;
+
+static int have_seeded_siphash = 0;
+
+/** Set up the siphash key if we haven't already done so. */
+int
+crypto_init_siphash_key(void)
+{
+  struct sipkey key;
+  if (have_seeded_siphash)
+    return 0;
+
+  crypto_rand((char*) &key, sizeof(key));
+  siphash_set_global_key(&key);
+  have_seeded_siphash = 1;
+  return 0;
+}
+
+/** Initialize the crypto library.  Return 0 on success, -1 on failure.
+ */
+int
+crypto_early_init(void)
+{
+  if (!crypto_early_initialized_) {
+
+    crypto_early_initialized_ = 1;
+
+#ifdef ENABLE_OPENSSL
+    crypto_openssl_early_init();
+#endif
+#ifdef ENABLE_NSS
+    crypto_nss_early_init(0);
+#endif
+
+    if (crypto_seed_rng() < 0)
+      return -1;
+    if (crypto_init_siphash_key() < 0)
+      return -1;
+
+    curve25519_init();
+    ed25519_init();
+  }
+  return 0;
+}
+
+/** Initialize the crypto library.  Return 0 on success, -1 on failure.
+ */
+int
+crypto_global_init(int useAccel, const char *accelName, const char *accelDir)
+{
+  if (!crypto_global_initialized_) {
+    if (crypto_early_init() < 0)
+      return -1;
+
+    crypto_global_initialized_ = 1;
+
+    crypto_dh_init();
+
+#ifdef ENABLE_OPENSSL
+    if (crypto_openssl_late_init(useAccel, accelName, accelDir) < 0)
+      return -1;
+#else
+    (void)useAccel;
+    (void)accelName;
+    (void)accelDir;
+#endif
+#ifdef ENABLE_NSS
+    if (crypto_nss_late_init() < 0)
+      return -1;
+#endif
+  }
+  return 0;
+}
+
+/** Free crypto resources held by this thread. */
+void
+crypto_thread_cleanup(void)
+{
+#ifdef ENABLE_OPENSSL
+  crypto_openssl_thread_cleanup();
+#endif
+}
+
+/**
+ * Uninitialize the crypto library. Return 0 on success. Does not detect
+ * failure.
+ */
+int
+crypto_global_cleanup(void)
+{
+  crypto_dh_free_all();
+
+#ifdef ENABLE_OPENSSL
+  crypto_openssl_global_cleanup();
+#endif
+#ifdef ENABLE_NSS
+  crypto_nss_global_cleanup();
+#endif
+
+  crypto_early_initialized_ = 0;
+  crypto_global_initialized_ = 0;
+  have_seeded_siphash = 0;
+  siphash_unset_global_key();
+
+  return 0;
+}
+
+/** Run operations that the crypto library requires to be happy again
+ * after forking. */
+void
+crypto_prefork(void)
+{
+#ifdef ENABLE_NSS
+  crypto_nss_prefork();
+#endif
+}
+
+/** Run operations that the crypto library requires to be happy again
+ * after forking. */
+void
+crypto_postfork(void)
+{
+#ifdef ENABLE_NSS
+  crypto_nss_postfork();
+#endif
+}
+
+/** Return the name of the crypto library we're using. */
+const char *
+crypto_get_library_name(void)
+{
+#ifdef ENABLE_OPENSSL
+  return "OpenSSL";
+#endif
+#ifdef ENABLE_NSS
+  return "NSS";
+#endif
+}
+
+/** Return the version of the crypto library we are using, as given in the
+ * library. */
+const char *
+crypto_get_library_version_string(void)
+{
+#ifdef ENABLE_OPENSSL
+  return crypto_openssl_get_version_str();
+#endif
+#ifdef ENABLE_NSS
+  return crypto_nss_get_version_str();
+#endif
+}
+
+/** Return the version of the crypto library we're using, as given in the
+ * headers. */
+const char *
+crypto_get_header_version_string(void)
+{
+#ifdef ENABLE_OPENSSL
+  return crypto_openssl_get_header_version_str();
+#endif
+#ifdef ENABLE_NSS
+  return crypto_nss_get_header_version_str();
+#endif
+}
+
+/** Return true iff Tor is using the NSS library. */
+int
+tor_is_using_nss(void)
+{
+#ifdef ENABLE_NSS
+  return 1;
+#else
+  return 0;
+#endif
+}
diff --git a/src/lib/crypt_ops/crypto_init.h b/src/lib/crypt_ops/crypto_init.h
new file mode 100644
index 0000000000..540d08eb56
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_init.h
@@ -0,0 +1,36 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_init.h
+ *
+ * \brief Headers for crypto_init.c
+ **/
+
+#ifndef TOR_CRYPTO_INIT_H
+#define TOR_CRYPTO_INIT_H
+
+#include "orconfig.h"
+#include "lib/cc/compat_compiler.h"
+
+int crypto_init_siphash_key(void);
+int crypto_early_init(void) ATTR_WUR;
+int crypto_global_init(int hardwareAccel,
+                       const char *accelName,
+                       const char *accelPath) ATTR_WUR;
+
+void crypto_thread_cleanup(void);
+int crypto_global_cleanup(void);
+void crypto_prefork(void);
+void crypto_postfork(void);
+
+const char *crypto_get_library_name(void);
+const char *crypto_get_library_version_string(void);
+const char *crypto_get_header_version_string(void);
+
+int tor_is_using_nss(void);
+
+#endif /* !defined(TOR_CRYPTO_H) */
diff --git a/src/lib/crypt_ops/crypto_nss_mgt.c b/src/lib/crypt_ops/crypto_nss_mgt.c
new file mode 100644
index 0000000000..0179126e38
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_nss_mgt.c
@@ -0,0 +1,132 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_nss_mgt.c
+ *
+ * \brief Manage the NSS library (if used)
+ **/
+
+#include "lib/crypt_ops/crypto_nss_mgt.h"
+
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/string/printf.h"
+
+DISABLE_GCC_WARNING(strict-prototypes)
+#include <nss.h>
+#include <pk11func.h>
+#include <ssl.h>
+
+#include <prerror.h>
+#include <prtypes.h>
+#include <prinit.h>
+ENABLE_GCC_WARNING(strict-prototypes)
+
+const char *
+crypto_nss_get_version_str(void)
+{
+  return NSS_GetVersion();
+}
+const char *
+crypto_nss_get_header_version_str(void)
+{
+  return NSS_VERSION;
+}
+
+/** A password function that always returns NULL. */
+static char *
+nss_password_func_always_fail(PK11SlotInfo *slot,
+                              PRBool retry,
+                              void *arg)
+{
+  (void) slot;
+  (void) retry;
+  (void) arg;
+  return NULL;
+}
+
+void
+crypto_nss_early_init(int nss_only)
+{
+  if (! nss_only) {
+    PR_Init(PR_USER_THREAD, PR_PRIORITY_NORMAL, 0);
+    PK11_SetPasswordFunc(nss_password_func_always_fail);
+  }
+
+  /* Eventually we should use NSS_Init() instead -- but that wants a
+     directory. The documentation says that we can't use this if we want
+     to use OpenSSL. */
+  if (NSS_NoDB_Init(NULL) == SECFailure) {
+    log_err(LD_CRYPTO, "Unable to initialize NSS.");
+    crypto_nss_log_errors(LOG_ERR, "initializing NSS");
+    tor_assert_unreached();
+  }
+
+  if (NSS_SetDomesticPolicy() == SECFailure) {
+    log_err(LD_CRYPTO, "Unable to set NSS cipher policy.");
+    crypto_nss_log_errors(LOG_ERR, "setting cipher policy");
+    tor_assert_unreached();
+  }
+
+  /* We need to override the default here, or NSS will reject all the
+   * legacy Tor certificates. */
+  SECStatus rv = NSS_OptionSet(NSS_RSA_MIN_KEY_SIZE, 1024);
+  if (rv != SECSuccess) {
+    log_err(LD_CRYPTO, "Unable to set NSS min RSA key size");
+    crypto_nss_log_errors(LOG_ERR, "setting cipher option.");
+    tor_assert_unreached();
+  }
+}
+
+void
+crypto_nss_log_errors(int severity, const char *doing)
+{
+  PRErrorCode code = PR_GetError();
+  const char *string = PORT_ErrorToString(code);
+  const char *name = PORT_ErrorToName(code);
+  char buf[16];
+  if (!string)
+    string = "<unrecognized>";
+  if (!name) {
+    tor_snprintf(buf, sizeof(buf), "%d", code);
+    name = buf;
+  }
+  if (doing) {
+    tor_log(severity, LD_CRYPTO, "NSS error %s while %s: %s",
+            name, doing, string);
+  } else {
+    tor_log(severity, LD_CRYPTO, "NSS error %s: %s", name, string);
+  }
+}
+
+int
+crypto_nss_late_init(void)
+{
+  /* Possibly, SSL_OptionSetDefault? */
+
+  return 0;
+}
+
+void
+crypto_nss_global_cleanup(void)
+{
+  NSS_Shutdown();
+  PL_ArenaFinish();
+  PR_Cleanup();
+}
+
+void
+crypto_nss_prefork(void)
+{
+  NSS_Shutdown();
+}
+
+void
+crypto_nss_postfork(void)
+{
+  crypto_nss_early_init(1);
+}
diff --git a/src/lib/crypt_ops/crypto_nss_mgt.h b/src/lib/crypt_ops/crypto_nss_mgt.h
new file mode 100644
index 0000000000..72fd2a1229
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_nss_mgt.h
@@ -0,0 +1,34 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_nss_mgt.h
+ *
+ * \brief Headers for crypto_nss_mgt.c
+ **/
+
+#ifndef TOR_CRYPTO_NSS_MGT_H
+#define TOR_CRYPTO_NSS_MGT_H
+
+#include "orconfig.h"
+
+#ifdef ENABLE_NSS
+/* global nss state */
+const char *crypto_nss_get_version_str(void);
+const char *crypto_nss_get_header_version_str(void);
+
+void crypto_nss_log_errors(int severity, const char *doing);
+
+void crypto_nss_early_init(int nss_only);
+int crypto_nss_late_init(void);
+
+void crypto_nss_global_cleanup(void);
+
+void crypto_nss_prefork(void);
+void crypto_nss_postfork(void);
+#endif
+
+#endif /* !defined(TOR_CRYPTO_NSS_H) */
diff --git a/src/lib/crypt_ops/crypto_ope.c b/src/lib/crypt_ops/crypto_ope.c
new file mode 100644
index 0000000000..2186d2a939
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_ope.c
@@ -0,0 +1,185 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * A rudimentary order-preserving encryption scheme.
+ *
+ * To compute the encryption of N, this scheme uses an AES-CTR stream to
+ * generate M-byte values, and adds the first N of them together. (+1 each to
+ * insure that the ciphertexts are strictly decreasing.)
+ *
+ * We use this for generating onion service revision counters based on the
+ * current time, without leaking the amount of skew in our view of the current
+ * time.  MUCH more analysis would be needed before using it for anything
+ * else!
+ */
+
+#include "orconfig.h"
+
+#define CRYPTO_OPE_PRIVATE
+#include "lib/crypt_ops/crypto_ope.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/arch/bytes.h"
+
+#include <string.h>
+
+/**
+ * How infrequent should the precomputed values be for this encryption?
+ * The choice of this value creates a space/time tradeoff.
+ *
+ * Note that this value must be a multiple of 16; see
+ * ope_get_cipher()
+ */
+#define SAMPLE_INTERVAL 1024
+/** Number of precomputed samples to make for each OPE key. */
+#define N_SAMPLES (OPE_INPUT_MAX / SAMPLE_INTERVAL)
+
+struct crypto_ope_t {
+  /** An AES key for use with this object. */
+  uint8_t key[OPE_KEY_LEN];
+  /** Cached intermediate encryption values at SAMPLE_INTERVAL,
+   * SAMPLE_INTERVAL*2,...SAMPLE_INTERVAL*N_SAMPLES */
+  uint64_t samples[N_SAMPLES];
+};
+
+/** The type to add up in order to produce our OPE ciphertexts */
+typedef uint16_t ope_val_t;
+
+#ifdef WORDS_BIGENDIAN
+/** Convert an OPE value from little-endian. */
+static inline ope_val_t
+ope_val_from_le(ope_val_t x)
+{
+  return
+    ((x) >> 8) |
+    (((x)&0xff) << 8);
+}
+#else
+#define ope_val_from_le(x) (x)
+#endif
+
+/**
+ * Return a new AES256-CTR stream cipher object for <b>ope</b>, ready to yield
+ * bytes from the stream at position <b>initial_idx</b>.
+ *
+ * Note that because the index is converted directly to an IV, it must be a
+ * multiple of the AES block size (16).
+ */
+STATIC crypto_cipher_t *
+ope_get_cipher(const crypto_ope_t *ope, uint32_t initial_idx)
+{
+  uint8_t iv[CIPHER_IV_LEN];
+  tor_assert((initial_idx & 0xf) == 0);
+  uint32_t n = tor_htonl(initial_idx >> 4);
+  memset(iv, 0, sizeof(iv));
+  memcpy(iv + CIPHER_IV_LEN - sizeof(n), &n, sizeof(n));
+
+  return crypto_cipher_new_with_iv_and_bits(ope->key,
+                                            iv,
+                                            OPE_KEY_LEN * 8);
+}
+
+/**
+ * Retrieve and add the next <b>n</b> values from the stream cipher <b>c</b>,
+ * and return their sum.
+ *
+ * Note that values are taken in little-endian order (for performance on
+ * prevalent hardware), and are mapped from range 0..2^n-1 to range 1..2^n (so
+ * that each input encrypts to a different output).
+ *
+ * NOTE: this function is not constant-time.
+ */
+STATIC uint64_t
+sum_values_from_cipher(crypto_cipher_t *c, size_t n)
+{
+#define BUFSZ 256
+  ope_val_t buf[BUFSZ];
+  uint64_t total = 0;
+  unsigned i;
+  while (n >= BUFSZ) {
+    memset(buf, 0, sizeof(buf));
+    crypto_cipher_crypt_inplace(c, (char*)buf, BUFSZ*sizeof(ope_val_t));
+
+    for (i = 0; i < BUFSZ; ++i) {
+      total += ope_val_from_le(buf[i]);
+      total += 1;
+    }
+    n -= BUFSZ;
+  }
+
+  memset(buf, 0, n*sizeof(ope_val_t));
+  crypto_cipher_crypt_inplace(c, (char*)buf, n*sizeof(ope_val_t));
+  for (i = 0; i < n; ++i) {
+    total += ope_val_from_le(buf[i]);
+    total += 1;
+  }
+
+  memset(buf, 0, sizeof(buf));
+  return total;
+}
+
+/**
+ * Return a new crypto_ope_t object, using the provided 256-bit key.
+ */
+crypto_ope_t *
+crypto_ope_new(const uint8_t *key)
+{
+  crypto_ope_t *ope = tor_malloc_zero(sizeof(crypto_ope_t));
+  memcpy(ope->key, key, OPE_KEY_LEN);
+
+  crypto_cipher_t *cipher = ope_get_cipher(ope, 0);
+
+  uint64_t v = 0;
+  int i;
+  for (i = 0; i < N_SAMPLES; ++i) {
+    v += sum_values_from_cipher(cipher, SAMPLE_INTERVAL);
+    ope->samples[i] = v;
+  }
+
+  crypto_cipher_free(cipher);
+  return ope;
+}
+
+/** Free all storage held in <>ope</b>. */
+void
+crypto_ope_free_(crypto_ope_t *ope)
+{
+  if (!ope)
+    return;
+  memwipe(ope, 0, sizeof(*ope));
+  tor_free(ope);
+}
+
+/**
+ * Return the encrypted value corresponding to <b>input</b>.  The input value
+ * must be in range 1..OPE_INPUT_MAX.  Returns CRYPTO_OPE_ERROR on an invalid
+ * input.
+ *
+ * NOTE: this function is not constant-time.
+ */
+uint64_t
+crypto_ope_encrypt(const crypto_ope_t *ope, int plaintext)
+{
+  if (plaintext <= 0 || plaintext > OPE_INPUT_MAX)
+    return CRYPTO_OPE_ERROR;
+
+  const int sample_idx = (plaintext / SAMPLE_INTERVAL);
+  const int starting_iv = sample_idx * SAMPLE_INTERVAL;
+  const int remaining_values =  plaintext - starting_iv;
+  uint64_t v;
+  if (sample_idx == 0) {
+    v = 0;
+  } else {
+    v = ope->samples[sample_idx - 1];
+  }
+  crypto_cipher_t *cipher = ope_get_cipher(ope, starting_iv*sizeof(ope_val_t));
+
+  v += sum_values_from_cipher(cipher, remaining_values);
+
+  crypto_cipher_free(cipher);
+
+  return v;
+}
diff --git a/src/lib/crypt_ops/crypto_ope.h b/src/lib/crypt_ops/crypto_ope.h
new file mode 100644
index 0000000000..610d956335
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_ope.h
@@ -0,0 +1,46 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef CRYPTO_OPE_H
+#define CRYPTO_OPE_H
+
+#include "orconfig.h"
+#include "lib/cc/torint.h"
+#include "lib/crypt_ops/crypto_ope.h"
+#include "lib/testsupport/testsupport.h"
+
+/** Length of OPE key, in bytes. */
+#define OPE_KEY_LEN 32
+
+/** Largest value that can be passed to crypto_ope_encrypt().
+ *
+ *  Expressed as 2^18 because the OPE system prefers powers of two.
+ *
+ *  The current max value stands for about 70 hours. The rationale here is as
+ *  follows: The rev counter is the time of seconds since the start of an SRV
+ *  period. SRVs are useful for about 48 hours (that's how long they stick
+ *  around on the consensus). Let's also add 12 hours of drift for clock skewed
+ *  services that might be using an old consensus and we arrive to 60
+ *  hours. The max value should be beyond that.
+ */
+#define OPE_INPUT_MAX (1<<18)
+
+#define CRYPTO_OPE_ERROR UINT64_MAX
+
+typedef struct crypto_ope_t crypto_ope_t;
+
+crypto_ope_t *crypto_ope_new(const uint8_t *key);
+void crypto_ope_free_(crypto_ope_t *ope);
+#define crypto_ope_free(ope) \
+  FREE_AND_NULL(crypto_ope_t, crypto_ope_free_, (ope))
+
+uint64_t crypto_ope_encrypt(const crypto_ope_t *ope, int plaintext);
+
+#ifdef CRYPTO_OPE_PRIVATE
+struct aes_cnt_cipher;
+STATIC struct aes_cnt_cipher *ope_get_cipher(const crypto_ope_t *ope,
+                                              uint32_t initial_idx);
+STATIC uint64_t sum_values_from_cipher(struct aes_cnt_cipher *c, size_t n);
+#endif
+
+#endif
diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.c b/src/lib/crypt_ops/crypto_openssl_mgt.c
new file mode 100644
index 0000000000..c97815f9a4
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_openssl_mgt.c
@@ -0,0 +1,398 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_openssl_mgt.c
+ *
+ * \brief Block of functions related to operations from OpenSSL.
+ **/
+
+#include "lib/crypt_ops/compat_openssl.h"
+#include "lib/crypt_ops/crypto_openssl_mgt.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/aes.h"
+#include "lib/string/util_string.h"
+#include "lib/lock/compat_mutex.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/testsupport/testsupport.h"
+#include "lib/thread/threads.h"
+
+DISABLE_GCC_WARNING(redundant-decls)
+
+#include <openssl/err.h>
+#include <openssl/rsa.h>
+#include <openssl/pem.h>
+#include <openssl/evp.h>
+#include <openssl/engine.h>
+#include <openssl/rand.h>
+#include <openssl/bn.h>
+#include <openssl/dh.h>
+#include <openssl/conf.h>
+#include <openssl/hmac.h>
+#include <openssl/crypto.h>
+#include <openssl/ssl.h>
+
+ENABLE_GCC_WARNING(redundant-decls)
+
+#include <string.h>
+
+#ifndef NEW_THREAD_API
+/** A number of preallocated mutexes for use by OpenSSL. */
+static tor_mutex_t **openssl_mutexes_ = NULL;
+/** How many mutexes have we allocated for use by OpenSSL? */
+static int n_openssl_mutexes_ = 0;
+#endif /* !defined(NEW_THREAD_API) */
+
+/** Declare STATIC functions */
+STATIC char * parse_openssl_version_str(const char *raw_version);
+#ifndef NEW_THREAD_API
+STATIC void openssl_locking_cb_(int mode, int n, const char *file, int line);
+STATIC void tor_set_openssl_thread_id(CRYPTO_THREADID *threadid);
+#endif
+
+/** Log all pending crypto errors at level <b>severity</b>.  Use
+ * <b>doing</b> to describe our current activities.
+ */
+void
+crypto_openssl_log_errors(int severity, const char *doing)
+{
+  unsigned long err;
+  const char *msg, *lib, *func;
+  while ((err = ERR_get_error()) != 0) {
+    msg = (const char*)ERR_reason_error_string(err);
+    lib = (const char*)ERR_lib_error_string(err);
+    func = (const char*)ERR_func_error_string(err);
+    if (!msg) msg = "(null)";
+    if (!lib) lib = "(null)";
+    if (!func) func = "(null)";
+    if (BUG(!doing)) doing = "(null)";
+    tor_log(severity, LD_CRYPTO, "crypto error while %s: %s (in %s:%s)",
+              doing, msg, lib, func);
+  }
+}
+
+/* Returns a trimmed and human-readable version of an openssl version string
+* <b>raw_version</b>. They are usually in the form of 'OpenSSL 1.0.0b 10
+* May 2012' and this will parse them into a form similar to '1.0.0b' */
+STATIC char *
+parse_openssl_version_str(const char *raw_version)
+{
+  const char *end_of_version = NULL;
+  /* The output should be something like "OpenSSL 1.0.0b 10 May 2012. Let's
+     trim that down. */
+  if (!strcmpstart(raw_version, "OpenSSL ")) {
+    raw_version += strlen("OpenSSL ");
+    end_of_version = strchr(raw_version, ' ');
+  }
+
+  if (end_of_version)
+    return tor_strndup(raw_version,
+                      end_of_version-raw_version);
+  else
+    return tor_strdup(raw_version);
+}
+
+static char *crypto_openssl_version_str = NULL;
+/* Return a human-readable version of the run-time openssl version number. */
+const char *
+crypto_openssl_get_version_str(void)
+{
+  if (crypto_openssl_version_str == NULL) {
+    const char *raw_version = OpenSSL_version(OPENSSL_VERSION);
+    crypto_openssl_version_str = parse_openssl_version_str(raw_version);
+  }
+  return crypto_openssl_version_str;
+}
+
+static char *crypto_openssl_header_version_str = NULL;
+/* Return a human-readable version of the compile-time openssl version
+* number. */
+const char *
+crypto_openssl_get_header_version_str(void)
+{
+  if (crypto_openssl_header_version_str == NULL) {
+    crypto_openssl_header_version_str =
+                        parse_openssl_version_str(OPENSSL_VERSION_TEXT);
+  }
+  return crypto_openssl_header_version_str;
+}
+
+#ifndef OPENSSL_THREADS
+#error OpenSSL has been built without thread support. Tor requires an \
+ OpenSSL library with thread support enabled.
+#endif
+
+#ifndef NEW_THREAD_API
+/** Helper: OpenSSL uses this callback to manipulate mutexes. */
+STATIC void
+openssl_locking_cb_(int mode, int n, const char *file, int line)
+{
+  (void)file;
+  (void)line;
+  if (!openssl_mutexes_)
+    /* This is not a really good fix for the
+     * "release-freed-lock-from-separate-thread-on-shutdown" problem, but
+     * it can't hurt. */
+    return;
+  if (mode & CRYPTO_LOCK)
+    tor_mutex_acquire(openssl_mutexes_[n]);
+  else
+    tor_mutex_release(openssl_mutexes_[n]);
+}
+
+STATIC void
+tor_set_openssl_thread_id(CRYPTO_THREADID *threadid)
+{
+  CRYPTO_THREADID_set_numeric(threadid, tor_get_thread_id());
+}
+#endif /* !defined(NEW_THREAD_API) */
+
+/** Helper: Construct mutexes, and set callbacks to help OpenSSL handle being
+ * multithreaded. Returns 0. */
+static int
+setup_openssl_threading(void)
+{
+#ifndef NEW_THREAD_API
+  int i;
+  int n = CRYPTO_num_locks();
+  n_openssl_mutexes_ = n;
+  openssl_mutexes_ = tor_calloc(n, sizeof(tor_mutex_t *));
+  for (i=0; i < n; ++i)
+    openssl_mutexes_[i] = tor_mutex_new();
+  CRYPTO_set_locking_callback(openssl_locking_cb_);
+  CRYPTO_THREADID_set_callback(tor_set_openssl_thread_id);
+#endif /* !defined(NEW_THREAD_API) */
+  return 0;
+}
+
+/** free OpenSSL variables */
+static void
+crypto_openssl_free_all(void)
+{
+  tor_free(crypto_openssl_version_str);
+  tor_free(crypto_openssl_header_version_str);
+
+#ifndef NEW_THREAD_API
+  if (n_openssl_mutexes_) {
+    int n = n_openssl_mutexes_;
+    tor_mutex_t **ms = openssl_mutexes_;
+    int i;
+    openssl_mutexes_ = NULL;
+    n_openssl_mutexes_ = 0;
+    for (i=0;i<n;++i) {
+      tor_mutex_free(ms[i]);
+    }
+    tor_free(ms);
+  }
+#endif /* !defined(NEW_THREAD_API) */
+}
+
+/** Perform early (pre-configuration) initialization tasks for OpenSSL. */
+void
+crypto_openssl_early_init(void)
+{
+#ifdef OPENSSL_1_1_API
+    OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS |
+                     OPENSSL_INIT_LOAD_CRYPTO_STRINGS |
+                     OPENSSL_INIT_ADD_ALL_CIPHERS |
+                     OPENSSL_INIT_ADD_ALL_DIGESTS, NULL);
+#else
+    ERR_load_crypto_strings();
+    OpenSSL_add_all_algorithms();
+#endif
+
+    setup_openssl_threading();
+
+    unsigned long version_num = OpenSSL_version_num();
+    const char *version_str = OpenSSL_version(OPENSSL_VERSION);
+    if (version_num == OPENSSL_VERSION_NUMBER &&
+        !strcmp(version_str, OPENSSL_VERSION_TEXT)) {
+      log_info(LD_CRYPTO, "OpenSSL version matches version from headers "
+                 "(%lx: %s).", version_num, version_str);
+    } else if ((version_num & 0xffff0000) ==
+               (OPENSSL_VERSION_NUMBER & 0xffff0000)) {
+      log_notice(LD_CRYPTO,
+               "We compiled with OpenSSL %lx: %s and we "
+               "are running with OpenSSL %lx: %s. "
+               "These two versions should be binary compatible.",
+               (unsigned long)OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_TEXT,
+               version_num, version_str);
+    } else {
+      log_warn(LD_CRYPTO, "OpenSSL version from headers does not match the "
+               "version we're running with. If you get weird crashes, that "
+               "might be why. (Compiled with %lx: %s; running with %lx: %s).",
+               (unsigned long)OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_TEXT,
+               version_num, version_str);
+    }
+
+    crypto_force_rand_ssleay();
+}
+
+#ifndef DISABLE_ENGINES
+/** Try to load an engine in a shared library via fully qualified path.
+ */
+static ENGINE *
+try_load_engine(const char *path, const char *engine)
+{
+  ENGINE *e = ENGINE_by_id("dynamic");
+  if (e) {
+    if (!ENGINE_ctrl_cmd_string(e, "ID", engine, 0) ||
+        !ENGINE_ctrl_cmd_string(e, "DIR_LOAD", "2", 0) ||
+        !ENGINE_ctrl_cmd_string(e, "DIR_ADD", path, 0) ||
+        !ENGINE_ctrl_cmd_string(e, "LOAD", NULL, 0)) {
+      ENGINE_free(e);
+      e = NULL;
+    }
+  }
+  return e;
+}
+#endif /* !defined(DISABLE_ENGINES) */
+
+#ifndef DISABLE_ENGINES
+/** Log any OpenSSL engines we're using at NOTICE. */
+static void
+log_engine(const char *fn, ENGINE *e)
+{
+  if (e) {
+    const char *name, *id;
+    name = ENGINE_get_name(e);
+    id = ENGINE_get_id(e);
+    log_notice(LD_CRYPTO, "Default OpenSSL engine for %s is %s [%s]",
+               fn, name?name:"?", id?id:"?");
+  } else {
+    log_info(LD_CRYPTO, "Using default implementation for %s", fn);
+  }
+}
+#endif /* !defined(DISABLE_ENGINES) */
+
+/** Initialize engines for openssl (if enabled). */
+static void
+crypto_openssl_init_engines(const char *accelName,
+                            const char *accelDir)
+{
+#ifdef DISABLE_ENGINES
+  (void)accelName;
+  (void)accelDir;
+  log_warn(LD_CRYPTO, "No OpenSSL hardware acceleration support enabled.");
+#else
+  ENGINE *e = NULL;
+
+  log_info(LD_CRYPTO, "Initializing OpenSSL engine support.");
+  ENGINE_load_builtin_engines();
+  ENGINE_register_all_complete();
+
+  if (accelName) {
+    if (accelDir) {
+      log_info(LD_CRYPTO, "Trying to load dynamic OpenSSL engine \"%s\""
+               " via path \"%s\".", accelName, accelDir);
+      e = try_load_engine(accelName, accelDir);
+    } else {
+      log_info(LD_CRYPTO, "Initializing dynamic OpenSSL engine \"%s\""
+               " acceleration support.", accelName);
+      e = ENGINE_by_id(accelName);
+    }
+    if (!e) {
+      log_warn(LD_CRYPTO, "Unable to load dynamic OpenSSL engine \"%s\".",
+               accelName);
+    } else {
+      log_info(LD_CRYPTO, "Loaded dynamic OpenSSL engine \"%s\".",
+               accelName);
+    }
+  }
+  if (e) {
+    log_info(LD_CRYPTO, "Loaded OpenSSL hardware acceleration engine,"
+             " setting default ciphers.");
+    ENGINE_set_default(e, ENGINE_METHOD_ALL);
+  }
+  /* Log, if available, the intersection of the set of algorithms
+     used by Tor and the set of algorithms available in the engine */
+  log_engine("RSA", ENGINE_get_default_RSA());
+  log_engine("DH", ENGINE_get_default_DH());
+#ifdef OPENSSL_1_1_API
+  log_engine("EC", ENGINE_get_default_EC());
+#else
+  log_engine("ECDH", ENGINE_get_default_ECDH());
+  log_engine("ECDSA", ENGINE_get_default_ECDSA());
+#endif /* defined(OPENSSL_1_1_API) */
+  log_engine("RAND", ENGINE_get_default_RAND());
+  log_engine("RAND (which we will not use)", ENGINE_get_default_RAND());
+  log_engine("SHA1", ENGINE_get_digest_engine(NID_sha1));
+  log_engine("3DES-CBC", ENGINE_get_cipher_engine(NID_des_ede3_cbc));
+  log_engine("AES-128-ECB", ENGINE_get_cipher_engine(NID_aes_128_ecb));
+  log_engine("AES-128-CBC", ENGINE_get_cipher_engine(NID_aes_128_cbc));
+#ifdef NID_aes_128_ctr
+  log_engine("AES-128-CTR", ENGINE_get_cipher_engine(NID_aes_128_ctr));
+#endif
+#ifdef NID_aes_128_gcm
+  log_engine("AES-128-GCM", ENGINE_get_cipher_engine(NID_aes_128_gcm));
+#endif
+  log_engine("AES-256-CBC", ENGINE_get_cipher_engine(NID_aes_256_cbc));
+#ifdef NID_aes_256_gcm
+  log_engine("AES-256-GCM", ENGINE_get_cipher_engine(NID_aes_256_gcm));
+#endif
+
+#endif /* defined(DISABLE_ENGINES) */
+}
+
+/** Perform late (post-init) initialization tasks for OpenSSL */
+int
+crypto_openssl_late_init(int useAccel, const char *accelName,
+                         const char *accelDir)
+{
+  if (useAccel > 0) {
+    crypto_openssl_init_engines(accelName, accelDir);
+  } else {
+    log_info(LD_CRYPTO, "NOT using OpenSSL engine support.");
+  }
+
+  if (crypto_force_rand_ssleay()) {
+    if (crypto_seed_rng() < 0)
+      return -1;
+  }
+
+  evaluate_evp_for_aes(-1);
+  evaluate_ctr_for_aes();
+
+  return 0;
+}
+
+/** Free crypto resources held by this thread. */
+void
+crypto_openssl_thread_cleanup(void)
+{
+#ifndef NEW_THREAD_API
+  ERR_remove_thread_state(NULL);
+#endif
+}
+
+/** Clean up global resources held by openssl. */
+void
+crypto_openssl_global_cleanup(void)
+{
+  #ifndef OPENSSL_1_1_API
+  EVP_cleanup();
+#endif
+#ifndef NEW_THREAD_API
+  ERR_remove_thread_state(NULL);
+#endif
+#ifndef OPENSSL_1_1_API
+  ERR_free_strings();
+#endif
+
+#ifndef DISABLE_ENGINES
+#ifndef OPENSSL_1_1_API
+  ENGINE_cleanup();
+#endif
+#endif
+
+  CONF_modules_unload(1);
+#ifndef OPENSSL_1_1_API
+  CRYPTO_cleanup_all_ex_data();
+#endif
+
+  crypto_openssl_free_all();
+}
diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.h b/src/lib/crypt_ops/crypto_openssl_mgt.h
new file mode 100644
index 0000000000..a3dd03aa04
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_openssl_mgt.h
@@ -0,0 +1,89 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_openssl_mgt.h
+ *
+ * \brief Headers for crypto_openssl_mgt.c
+ **/
+
+#ifndef TOR_CRYPTO_OPENSSL_H
+#define TOR_CRYPTO_OPENSSL_H
+
+#include "orconfig.h"
+
+#ifdef ENABLE_OPENSSL
+#include <openssl/engine.h>
+
+/*
+  Macro to create an arbitrary OpenSSL version number as used by
+  OPENSSL_VERSION_NUMBER or SSLeay(), since the actual numbers are a bit hard
+  to read.
+
+  Don't use this directly, instead use one of the other OPENSSL_V macros
+  below.
+
+  The format is: 4 bits major, 8 bits minor, 8 bits fix, 8 bits patch, 4 bit
+  status.
+ */
+#define OPENSSL_VER(a,b,c,d,e)                                \
+  (((a)<<28) |                                                \
+   ((b)<<20) |                                                \
+   ((c)<<12) |                                                \
+   ((d)<< 4) |                                                \
+    (e))
+/** An openssl release number.  For example, OPENSSL_V(0,9,8,'j') is the
+ * version for the released version of 0.9.8j */
+#define OPENSSL_V(a,b,c,d) \
+  OPENSSL_VER((a),(b),(c),(d)-'a'+1,0xf)
+/** An openssl release number for the first release in the series.  For
+ * example, OPENSSL_V_NOPATCH(1,0,0) is the first released version of OpenSSL
+ * 1.0.0. */
+#define OPENSSL_V_NOPATCH(a,b,c) \
+  OPENSSL_VER((a),(b),(c),0,0xf)
+/** The first version that would occur for any alpha or beta in an openssl
+ * series. For example, OPENSSL_V_SERIES(0,9,8) is greater than any released
+ * 0.9.7, and less than any released 0.9.8. */
+#define OPENSSL_V_SERIES(a,b,c) \
+  OPENSSL_VER((a),(b),(c),0,0)
+
+#ifdef OPENSSL_NO_ENGINE
+/* Android's OpenSSL seems to have removed all of its Engine support. */
+#define DISABLE_ENGINES
+#endif
+
+#if OPENSSL_VERSION_NUMBER >= OPENSSL_VER(1,1,0,0,5) && \
+  !defined(LIBRESSL_VERSION_NUMBER)
+/* OpenSSL as of 1.1.0pre4 has an "new" thread API, which doesn't require
+ * seting up various callbacks.
+ *
+ * OpenSSL 1.1.0pre4 has a messed up `ERR_remove_thread_state()` prototype,
+ * while the previous one was restored in pre5, and the function made a no-op
+ * (along with a deprecated annotation, which produces a compiler warning).
+ *
+ * While it is possible to support all three versions of the thread API,
+ * a version that existed only for one snapshot pre-release is kind of
+ * pointless, so let's not.
+ */
+#define NEW_THREAD_API
+#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_VER(1,1,0,0,5) && ... */
+
+void crypto_openssl_log_errors(int severity, const char *doing);
+
+/* global openssl state */
+const char * crypto_openssl_get_version_str(void);
+const char * crypto_openssl_get_header_version_str(void);
+
+void crypto_openssl_early_init(void);
+int crypto_openssl_late_init(int useAccel, const char *accelName,
+                             const char *accelDir);
+
+void crypto_openssl_thread_cleanup(void);
+void crypto_openssl_global_cleanup(void);
+
+#endif /* ENABLE_OPENSSL */
+
+#endif /* !defined(TOR_CRYPTO_OPENSSL_H) */
diff --git a/src/common/crypto_pwbox.c b/src/lib/crypt_ops/crypto_pwbox.c
index 31e37c007d..a8db08f7b7 100644
--- a/src/common/crypto_pwbox.c
+++ b/src/lib/crypt_ops/crypto_pwbox.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -8,12 +8,19 @@
  * them to disk.
  */
 
-#include "crypto.h"
-#include "crypto_s2k.h"
-#include "crypto_pwbox.h"
-#include "di_ops.h"
-#include "util.h"
-#include "pwbox.h"
+#include <string.h>
+
+#include "lib/arch/bytes.h"
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_pwbox.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_s2k.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/intmath/muldiv.h"
+#include "trunnel/pwbox.h"
+#include "lib/log/util_bug.h"
 
 /* 8 bytes "TORBOX00"
    1 byte: header len (H)
@@ -54,6 +61,7 @@ crypto_pwbox(uint8_t **out, size_t *outlen_out,
   int rv;
 
   enc = pwbox_encoded_new();
+  tor_assert(enc);
 
   pwbox_encoded_setlen_skey_header(enc, S2K_MAXLEN);
 
@@ -71,7 +79,7 @@ crypto_pwbox(uint8_t **out, size_t *outlen_out,
   pwbox_encoded_setlen_data(enc, encrypted_len);
   encrypted_portion = pwbox_encoded_getarray_data(enc);
 
-  set_uint32(encrypted_portion, htonl((uint32_t)input_len));
+  set_uint32(encrypted_portion, tor_htonl((uint32_t)input_len));
   memcpy(encrypted_portion+4, input, input_len);
 
   /* Now that all the data is in position, derive some keys, encrypt, and
@@ -107,7 +115,6 @@ crypto_pwbox(uint8_t **out, size_t *outlen_out,
   rv = 0;
   goto out;
 
- err:
   /* LCOV_EXCL_START
 
      This error case is often unreachable if we're correctly coded, unless
@@ -123,6 +130,7 @@ crypto_pwbox(uint8_t **out, size_t *outlen_out,
        - pwbox_encoded_encode can't fail unless we're using trunnel wrong,
          or it's buggy.
    */
+ err:
   tor_free(result);
   rv = -1;
   /* LCOV_EXCL_STOP */
@@ -186,7 +194,7 @@ crypto_unpwbox(uint8_t **out, size_t *outlen_out,
 
   cipher = crypto_cipher_new_with_iv((char*)keys, (char*)enc->iv);
   crypto_cipher_decrypt(cipher, (char*)&result_len, (char*)encrypted, 4);
-  result_len = ntohl(result_len);
+  result_len = tor_ntohl(result_len);
   if (encrypted_len < result_len + 4)
     goto err;
 
@@ -209,4 +217,3 @@ crypto_unpwbox(uint8_t **out, size_t *outlen_out,
   memwipe(keys, 0, sizeof(keys));
   return rv;
 }
-
diff --git a/src/common/crypto_pwbox.h b/src/lib/crypt_ops/crypto_pwbox.h
index aadd477078..5a26889fb2 100644
--- a/src/common/crypto_pwbox.h
+++ b/src/lib/crypt_ops/crypto_pwbox.h
@@ -1,7 +1,16 @@
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_pwbox.h
+ *
+ * \brief Header for crypto_pwbox.c
+ **/
+
 #ifndef CRYPTO_PWBOX_H_INCLUDED_
 #define CRYPTO_PWBOX_H_INCLUDED_
 
-#include "torint.h"
+#include "lib/cc/torint.h"
 
 #define UNPWBOX_OKAY 0
 #define UNPWBOX_BAD_SECRET -1
@@ -16,5 +25,4 @@ int crypto_unpwbox(uint8_t **out, size_t *outlen_out,
                    const uint8_t *inp, size_t input_len,
                    const char *secret, size_t secret_len);
 
-#endif
-
+#endif /* !defined(CRYPTO_PWBOX_H_INCLUDED_) */
diff --git a/src/lib/crypt_ops/crypto_rand.c b/src/lib/crypt_ops/crypto_rand.c
new file mode 100644
index 0000000000..915fe0870d
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_rand.c
@@ -0,0 +1,731 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_rand.c
+ *
+ * \brief Functions for initialising and seeding (pseudo-)random
+ * number generators, and working with randomness.
+ **/
+
+#ifndef CRYPTO_RAND_PRIVATE
+#define CRYPTO_RAND_PRIVATE
+
+#include "lib/crypt_ops/crypto_rand.h"
+
+#ifdef _WIN32
+#include <windows.h>
+#include <wincrypt.h>
+#endif /* defined(_WIN32) */
+
+#include "lib/container/smartlist.h"
+#include "lib/crypt_ops/compat_openssl.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/encoding/binascii.h"
+#include "lib/intmath/weakrng.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/sandbox/sandbox.h"
+#include "lib/string/compat_string.h"
+#include "lib/string/util_string.h"
+#include "lib/testsupport/testsupport.h"
+#include "lib/fs/files.h"
+
+#include "lib/defs/digest_sizes.h"
+#include "lib/crypt_ops/crypto_digest.h"
+
+#ifdef ENABLE_NSS
+#include "lib/crypt_ops/crypto_nss_mgt.h"
+#endif
+
+#ifdef ENABLE_OPENSSL
+DISABLE_GCC_WARNING(redundant-decls)
+#include <openssl/rand.h>
+#include <openssl/sha.h>
+ENABLE_GCC_WARNING(redundant-decls)
+#endif
+
+#ifdef ENABLE_NSS
+#include <pk11pub.h>
+#include <secerr.h>
+#include <prerror.h>
+#endif
+
+#if __GNUC__ && GCC_VERSION >= 402
+#if GCC_VERSION >= 406
+#pragma GCC diagnostic pop
+#else
+#pragma GCC diagnostic warning "-Wredundant-decls"
+#endif
+#endif /* __GNUC__ && GCC_VERSION >= 402 */
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_SYS_FCNTL_H
+#include <sys/fcntl.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_SYSCALL_H
+#include <sys/syscall.h>
+#endif
+#ifdef HAVE_SYS_RANDOM_H
+#include <sys/random.h>
+#endif
+
+#include <string.h>
+#include <errno.h>
+
+/**
+ * How many bytes of entropy we add at once.
+ *
+ * This is how much entropy OpenSSL likes to add right now, so maybe it will
+ * work for us too.
+ **/
+#define ADD_ENTROPY 32
+
+/**
+ * Longest recognized DNS query.
+ **/
+#define MAX_DNS_LABEL_SIZE 63
+
+/**
+ * Largest strong entropy request permitted.
+ **/
+#define MAX_STRONGEST_RAND_SIZE 256
+
+/**
+ * Set the seed of the weak RNG to a random value.
+ **/
+void
+crypto_seed_weak_rng(tor_weak_rng_t *rng)
+{
+  unsigned seed;
+  crypto_rand((void*)&seed, sizeof(seed));
+  tor_init_weak_random(rng, seed);
+}
+
+#ifdef TOR_UNIT_TESTS
+int break_strongest_rng_syscall = 0;
+int break_strongest_rng_fallback = 0;
+#endif
+
+/**
+ * Try to get <b>out_len</b> bytes of the strongest entropy we can generate,
+ * via system calls, storing it into <b>out</b>. Return 0 on success, -1 on
+ * failure.  A maximum request size of 256 bytes is imposed.
+ **/
+static int
+crypto_strongest_rand_syscall(uint8_t *out, size_t out_len)
+{
+  tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE);
+
+  /* We only log at notice-level here because in the case that this function
+   * fails the crypto_strongest_rand_raw() caller will log with a warning-level
+   * message and let crypto_strongest_rand() error out and finally terminating
+   * Tor with an assertion error.
+   */
+
+#ifdef TOR_UNIT_TESTS
+  if (break_strongest_rng_syscall)
+    return -1;
+#endif
+
+#if defined(_WIN32)
+  static int provider_set = 0;
+  static HCRYPTPROV provider;
+
+  if (!provider_set) {
+    if (!CryptAcquireContext(&provider, NULL, NULL, PROV_RSA_FULL,
+                             CRYPT_VERIFYCONTEXT)) {
+      log_notice(LD_CRYPTO, "Unable to set Windows CryptoAPI provider [1].");
+      return -1;
+    }
+    provider_set = 1;
+  }
+  if (!CryptGenRandom(provider, out_len, out)) {
+    log_notice(LD_CRYPTO, "Unable get entropy from the Windows CryptoAPI.");
+    return -1;
+  }
+
+  return 0;
+#elif defined(__linux__) && defined(SYS_getrandom)
+  static int getrandom_works = 1; /* Be optimistic about our chances... */
+
+  /* getrandom() isn't as straightforward as getentropy(), and has
+   * no glibc wrapper.
+   *
+   * As far as I can tell from getrandom(2) and the source code, the
+   * requests we issue will always succeed (though it will block on the
+   * call if /dev/urandom isn't seeded yet), since we are NOT specifying
+   * GRND_NONBLOCK and the request is <= 256 bytes.
+   *
+   * The manpage is unclear on what happens if a signal interrupts the call
+   * while the request is blocked due to lack of entropy....
+   *
+   * We optimistically assume that getrandom() is available and functional
+   * because it is the way of the future, and 2 branch mispredicts pale in
+   * comparison to the overheads involved with failing to open
+   * /dev/srandom followed by opening and reading from /dev/urandom.
+   */
+  if (PREDICT_LIKELY(getrandom_works)) {
+    long ret;
+    /* A flag of '0' here means to read from '/dev/urandom', and to
+     * block if insufficient entropy is available to service the
+     * request.
+     */
+    const unsigned int flags = 0;
+    do {
+      ret = syscall(SYS_getrandom, out, out_len, flags);
+    } while (ret == -1 && ((errno == EINTR) ||(errno == EAGAIN)));
+
+    if (PREDICT_UNLIKELY(ret == -1)) {
+      /* LCOV_EXCL_START we can't actually make the syscall fail in testing. */
+      tor_assert(errno != EAGAIN);
+      tor_assert(errno != EINTR);
+
+      /* Useful log message for errno. */
+      if (errno == ENOSYS) {
+        log_notice(LD_CRYPTO, "Can't get entropy from getrandom()."
+                   " You are running a version of Tor built to support"
+                   " getrandom(), but the kernel doesn't implement this"
+                   " function--probably because it is too old?"
+                   " Trying fallback method instead.");
+      } else {
+        log_notice(LD_CRYPTO, "Can't get entropy from getrandom(): %s."
+                              " Trying fallback method instead.",
+                   strerror(errno));
+      }
+
+      getrandom_works = 0; /* Don't bother trying again. */
+      return -1;
+      /* LCOV_EXCL_STOP */
+    }
+
+    tor_assert(ret == (long)out_len);
+    return 0;
+  }
+
+  return -1; /* getrandom() previously failed unexpectedly. */
+#elif defined(HAVE_GETENTROPY)
+  /* getentropy() is what Linux's getrandom() wants to be when it grows up.
+   * the only gotcha is that requests are limited to 256 bytes.
+   */
+  return getentropy(out, out_len);
+#else
+  (void) out;
+#endif /* defined(_WIN32) || ... */
+
+  /* This platform doesn't have a supported syscall based random. */
+  return -1;
+}
+
+/**
+ * Try to get <b>out_len</b> bytes of the strongest entropy we can generate,
+ * via the per-platform fallback mechanism, storing it into <b>out</b>.
+ * Return 0 on success, -1 on failure.  A maximum request size of 256 bytes
+ * is imposed.
+ **/
+static int
+crypto_strongest_rand_fallback(uint8_t *out, size_t out_len)
+{
+#ifdef TOR_UNIT_TESTS
+  if (break_strongest_rng_fallback)
+    return -1;
+#endif
+
+#ifdef _WIN32
+  /* Windows exclusively uses crypto_strongest_rand_syscall(). */
+  (void)out;
+  (void)out_len;
+  return -1;
+#else /* !(defined(_WIN32)) */
+  static const char *filenames[] = {
+    "/dev/srandom", "/dev/urandom", "/dev/random", NULL
+  };
+  int fd, i;
+  size_t n;
+
+  for (i = 0; filenames[i]; ++i) {
+    log_debug(LD_FS, "Considering %s as entropy source", filenames[i]);
+    fd = open(sandbox_intern_string(filenames[i]), O_RDONLY, 0);
+    if (fd<0) continue;
+    log_info(LD_CRYPTO, "Reading entropy from \"%s\"", filenames[i]);
+    n = read_all_from_fd(fd, (char*)out, out_len);
+    close(fd);
+    if (n != out_len) {
+      /* LCOV_EXCL_START
+       * We can't make /dev/foorandom actually fail. */
+      log_notice(LD_CRYPTO,
+                 "Error reading from entropy source %s (read only %lu bytes).",
+                 filenames[i],
+                 (unsigned long)n);
+      return -1;
+      /* LCOV_EXCL_STOP */
+    }
+
+    return 0;
+  }
+
+  return -1;
+#endif /* defined(_WIN32) */
+}
+
+/**
+ * Try to get <b>out_len</b> bytes of the strongest entropy we can generate,
+ * storing it into <b>out</b>. Return 0 on success, -1 on failure.  A maximum
+ * request size of 256 bytes is imposed.
+ **/
+STATIC int
+crypto_strongest_rand_raw(uint8_t *out, size_t out_len)
+{
+  static const size_t sanity_min_size = 16;
+  static const int max_attempts = 3;
+  tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE);
+
+  /* For buffers >= 16 bytes (128 bits), we sanity check the output by
+   * zero filling the buffer and ensuring that it actually was at least
+   * partially modified.
+   *
+   * Checking that any individual byte is non-zero seems like it would
+   * fail too often (p = out_len * 1/256) for comfort, but this is an
+   * "adjust according to taste" sort of check.
+   */
+  memwipe(out, 0, out_len);
+  for (int i = 0; i < max_attempts; i++) {
+    /* Try to use the syscall/OS favored mechanism to get strong entropy. */
+    if (crypto_strongest_rand_syscall(out, out_len) != 0) {
+      /* Try to use the less-favored mechanism to get strong entropy. */
+      if (crypto_strongest_rand_fallback(out, out_len) != 0) {
+        /* Welp, we tried.  Hopefully the calling code terminates the process
+         * since we're basically boned without good entropy.
+         */
+        log_warn(LD_CRYPTO,
+                 "Cannot get strong entropy: no entropy source found.");
+        return -1;
+      }
+    }
+
+    if ((out_len < sanity_min_size) || !tor_mem_is_zero((char*)out, out_len))
+      return 0;
+  }
+
+  /* LCOV_EXCL_START
+   *
+   * We tried max_attempts times to fill a buffer >= 128 bits long,
+   * and each time it returned all '0's.  Either the system entropy
+   * source is busted, or the user should go out and buy a ticket to
+   * every lottery on the planet.
+   */
+  log_warn(LD_CRYPTO, "Strong OS entropy returned all zero buffer.");
+
+  return -1;
+  /* LCOV_EXCL_STOP */
+}
+
+/**
+ * Try to get <b>out_len</b> bytes of the strongest entropy we can generate,
+ * storing it into <b>out</b>.
+ **/
+void
+crypto_strongest_rand(uint8_t *out, size_t out_len)
+{
+  crypto_strongest_rand_(out, out_len);
+}
+
+/**
+ * Try to get <b>out_len</b> bytes of the strongest entropy we can generate,
+ * storing it into <b>out</b>. (Mockable version.)
+ **/
+MOCK_IMPL(void,
+crypto_strongest_rand_,(uint8_t *out, size_t out_len))
+{
+#define DLEN DIGEST512_LEN
+
+  /* We're going to hash DLEN bytes from the system RNG together with some
+   * bytes from the PRNGs from our crypto librar(y/ies), in order to yield
+   * DLEN bytes.
+   */
+  uint8_t inp[DLEN*3];
+  uint8_t tmp[DLEN];
+  tor_assert(out);
+  while (out_len) {
+    memset(inp, 0, sizeof(inp));
+#ifdef ENABLE_OPENSSL
+    RAND_bytes(inp, DLEN);
+#endif
+#ifdef ENABLE_NSS
+    PK11_GenerateRandom(inp+DLEN, DLEN);
+#endif
+    if (crypto_strongest_rand_raw(inp+DLEN*2, DLEN) < 0) {
+      // LCOV_EXCL_START
+      log_err(LD_CRYPTO, "Failed to load strong entropy when generating an "
+              "important key. Exiting.");
+      /* Die with an assertion so we get a stack trace. */
+      tor_assert(0);
+      // LCOV_EXCL_STOP
+    }
+    if (out_len >= DLEN) {
+      crypto_digest512((char*)out, (char*)inp, sizeof(inp), DIGEST_SHA512);
+      out += DLEN;
+      out_len -= DLEN;
+    } else {
+      crypto_digest512((char*)tmp, (char*)inp, sizeof(inp), DIGEST_SHA512);
+      memcpy(out, tmp, out_len);
+      break;
+    }
+  }
+  memwipe(tmp, 0, sizeof(tmp));
+  memwipe(inp, 0, sizeof(inp));
+#undef DLEN
+}
+
+#ifdef ENABLE_OPENSSL
+/**
+ * Seed OpenSSL's random number generator with bytes from the operating
+ * system.  Return 0 on success, -1 on failure.
+ **/
+static int
+crypto_seed_openssl_rng(void)
+{
+  int rand_poll_ok = 0, load_entropy_ok = 0;
+  uint8_t buf[ADD_ENTROPY];
+
+  /* OpenSSL has a RAND_poll function that knows about more kinds of
+   * entropy than we do.  We'll try calling that, *and* calling our own entropy
+   * functions.  If one succeeds, we'll accept the RNG as seeded. */
+  rand_poll_ok = RAND_poll();
+  if (rand_poll_ok == 0)
+    log_warn(LD_CRYPTO, "RAND_poll() failed."); // LCOV_EXCL_LINE
+
+  load_entropy_ok = !crypto_strongest_rand_raw(buf, sizeof(buf));
+  if (load_entropy_ok) {
+    RAND_seed(buf, sizeof(buf));
+  }
+
+  memwipe(buf, 0, sizeof(buf));
+
+  if ((rand_poll_ok || load_entropy_ok) && RAND_status() == 1)
+    return 0;
+  else
+    return -1;
+}
+#endif
+
+#ifdef ENABLE_NSS
+/**
+ * Seed OpenSSL's random number generator with bytes from the operating
+ * system.  Return 0 on success, -1 on failure.
+ **/
+static int
+crypto_seed_nss_rng(void)
+{
+  uint8_t buf[ADD_ENTROPY];
+
+  int load_entropy_ok = !crypto_strongest_rand_raw(buf, sizeof(buf));
+  if (load_entropy_ok) {
+    if (PK11_RandomUpdate(buf, sizeof(buf)) != SECSuccess) {
+      load_entropy_ok = 0;
+    }
+  }
+
+  memwipe(buf, 0, sizeof(buf));
+
+  return load_entropy_ok ? 0 : -1;
+}
+#endif
+
+/**
+ * Seed the RNG for any and all crypto libraries that we're using with bytes
+ * from the operating system.  Return 0 on success, -1 on failure.
+ */
+int
+crypto_seed_rng(void)
+{
+  int seeded = 0;
+#ifdef ENABLE_NSS
+  if (crypto_seed_nss_rng() < 0)
+    return -1;
+  ++seeded;
+#endif
+#ifdef ENABLE_OPENSSL
+  if (crypto_seed_openssl_rng() < 0)
+    return -1;
+  ++seeded;
+#endif
+  tor_assert(seeded);
+  return 0;
+}
+
+/**
+ * Write <b>n</b> bytes of strong random data to <b>to</b>. Supports mocking
+ * for unit tests.
+ *
+ * This function is not allowed to fail; if it would fail to generate strong
+ * entropy, it must terminate the process instead.
+ **/
+MOCK_IMPL(void,
+crypto_rand, (char *to, size_t n))
+{
+  crypto_rand_unmocked(to, n);
+}
+
+/**
+ * Write <b>n</b> bytes of strong random data to <b>to</b>.  Most callers
+ * will want crypto_rand instead.
+ *
+ * This function is not allowed to fail; if it would fail to generate strong
+ * entropy, it must terminate the process instead.
+ **/
+void
+crypto_rand_unmocked(char *to, size_t n)
+{
+  if (n == 0)
+    return;
+
+  tor_assert(n < INT_MAX);
+  tor_assert(to);
+
+#ifdef ENABLE_NSS
+  SECStatus s = PK11_GenerateRandom((unsigned char*)to, (int)n);
+  if (s != SECSuccess) {
+    /* NSS rather sensibly might refuse to generate huge amounts of random
+     * data at once.  Unfortunately, our unit test do this in a couple of
+     * places.  To solve this issue, we use our XOF to stretch a shorter
+     * output when a longer one is needed.
+     *
+     * Yes, this is secure. */
+
+    /* This is longer than it needs to be; 1600 bits == 200 bytes is the
+     * state-size of SHA3. */
+#define BUFLEN 512
+    tor_assert(PR_GetError() == SEC_ERROR_INVALID_ARGS && n > BUFLEN);
+    unsigned char buf[BUFLEN];
+    s = PK11_GenerateRandom(buf, BUFLEN);
+    tor_assert(s == SECSuccess);
+    crypto_xof_t *xof = crypto_xof_new();
+    crypto_xof_add_bytes(xof, buf, BUFLEN);
+    crypto_xof_squeeze_bytes(xof, (unsigned char *)to, n);
+    crypto_xof_free(xof);
+    memwipe(buf, 0, BUFLEN);
+
+#undef BUFLEN
+  }
+#else
+  int r = RAND_bytes((unsigned char*)to, (int)n);
+  /* We consider a PRNG failure non-survivable. Let's assert so that we get a
+   * stack trace about where it happened.
+   */
+  tor_assert(r >= 0);
+#endif
+}
+
+/**
+ * Return a pseudorandom integer, chosen uniformly from the values
+ * between 0 and <b>max</b>-1 inclusive.  <b>max</b> must be between 1 and
+ * INT_MAX+1, inclusive.
+ */
+int
+crypto_rand_int(unsigned int max)
+{
+  unsigned int val;
+  unsigned int cutoff;
+  tor_assert(max <= ((unsigned int)INT_MAX)+1);
+  tor_assert(max > 0); /* don't div by 0 */
+
+  /* We ignore any values that are >= 'cutoff,' to avoid biasing the
+   * distribution with clipping at the upper end of unsigned int's
+   * range.
+   */
+  cutoff = UINT_MAX - (UINT_MAX%max);
+  while (1) {
+    crypto_rand((char*)&val, sizeof(val));
+    if (val < cutoff)
+      return val % max;
+  }
+}
+
+/**
+ * Return a pseudorandom integer, chosen uniformly from the values i such
+ * that min <= i < max.
+ *
+ * <b>min</b> MUST be in range [0, <b>max</b>).
+ * <b>max</b> MUST be in range (min, INT_MAX].
+ **/
+int
+crypto_rand_int_range(unsigned int min, unsigned int max)
+{
+  tor_assert(min < max);
+  tor_assert(max <= INT_MAX);
+
+  /* The overflow is avoided here because crypto_rand_int() returns a value
+   * between 0 and (max - min) inclusive. */
+  return min + crypto_rand_int(max - min);
+}
+
+/**
+ * As crypto_rand_int_range, but supports uint64_t.
+ **/
+uint64_t
+crypto_rand_uint64_range(uint64_t min, uint64_t max)
+{
+  tor_assert(min < max);
+  return min + crypto_rand_uint64(max - min);
+}
+
+/**
+ * As crypto_rand_int_range, but supports time_t.
+ **/
+time_t
+crypto_rand_time_range(time_t min, time_t max)
+{
+  tor_assert(min < max);
+  return min + (time_t)crypto_rand_uint64(max - min);
+}
+
+/**
+ * Return a pseudorandom 64-bit integer, chosen uniformly from the values
+ * between 0 and <b>max</b>-1 inclusive.
+ **/
+uint64_t
+crypto_rand_uint64(uint64_t max)
+{
+  uint64_t val;
+  uint64_t cutoff;
+  tor_assert(max < UINT64_MAX);
+  tor_assert(max > 0); /* don't div by 0 */
+
+  /* We ignore any values that are >= 'cutoff,' to avoid biasing the
+   * distribution with clipping at the upper end of unsigned int's
+   * range.
+   */
+  cutoff = UINT64_MAX - (UINT64_MAX%max);
+  while (1) {
+    crypto_rand((char*)&val, sizeof(val));
+    if (val < cutoff)
+      return val % max;
+  }
+}
+
+/**
+ * Return a pseudorandom double d, chosen uniformly from the range
+ * 0.0 <= d < 1.0.
+ **/
+double
+crypto_rand_double(void)
+{
+  /* We just use an unsigned int here; we don't really care about getting
+   * more than 32 bits of resolution */
+  unsigned int u;
+  crypto_rand((char*)&u, sizeof(u));
+#if SIZEOF_INT == 4
+#define UINT_MAX_AS_DOUBLE 4294967296.0
+#elif SIZEOF_INT == 8
+#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19
+#else
+#error SIZEOF_INT is neither 4 nor 8
+#endif /* SIZEOF_INT == 4 || ... */
+  return ((double)u) / UINT_MAX_AS_DOUBLE;
+}
+
+/**
+ * Generate and return a new random hostname starting with <b>prefix</b>,
+ * ending with <b>suffix</b>, and containing no fewer than
+ * <b>min_rand_len</b> and no more than <b>max_rand_len</b> random base32
+ * characters. Does not check for failure.
+ *
+ * Clip <b>max_rand_len</b> to MAX_DNS_LABEL_SIZE.
+ **/
+char *
+crypto_random_hostname(int min_rand_len, int max_rand_len, const char *prefix,
+                       const char *suffix)
+{
+  char *result, *rand_bytes;
+  int randlen, rand_bytes_len;
+  size_t resultlen, prefixlen;
+
+  if (max_rand_len > MAX_DNS_LABEL_SIZE)
+    max_rand_len = MAX_DNS_LABEL_SIZE;
+  if (min_rand_len > max_rand_len)
+    min_rand_len = max_rand_len;
+
+  randlen = crypto_rand_int_range(min_rand_len, max_rand_len+1);
+
+  prefixlen = strlen(prefix);
+  resultlen = prefixlen + strlen(suffix) + randlen + 16;
+
+  rand_bytes_len = ((randlen*5)+7)/8;
+  if (rand_bytes_len % 5)
+    rand_bytes_len += 5 - (rand_bytes_len%5);
+  rand_bytes = tor_malloc(rand_bytes_len);
+  crypto_rand(rand_bytes, rand_bytes_len);
+
+  result = tor_malloc(resultlen);
+  memcpy(result, prefix, prefixlen);
+  base32_encode(result+prefixlen, resultlen-prefixlen,
+                rand_bytes, rand_bytes_len);
+  tor_free(rand_bytes);
+  strlcpy(result+prefixlen+randlen, suffix, resultlen-(prefixlen+randlen));
+
+  return result;
+}
+
+/**
+ * Return a randomly chosen element of <b>sl</b>; or NULL if <b>sl</b>
+ * is empty.
+ **/
+void *
+smartlist_choose(const smartlist_t *sl)
+{
+  int len = smartlist_len(sl);
+  if (len)
+    return smartlist_get(sl,crypto_rand_int(len));
+  return NULL; /* no elements to choose from */
+}
+
+/**
+ * Scramble the elements of <b>sl</b> into a random order.
+ **/
+void
+smartlist_shuffle(smartlist_t *sl)
+{
+  int i;
+  /* From the end of the list to the front, choose at random from the
+     positions we haven't looked at yet, and swap that position into the
+     current position.  Remember to give "no swap" the same probability as
+     any other swap. */
+  for (i = smartlist_len(sl)-1; i > 0; --i) {
+    int j = crypto_rand_int(i+1);
+    smartlist_swap(sl, i, j);
+  }
+}
+
+/** Make sure that openssl is using its default PRNG. Return 1 if we had to
+ * adjust it; 0 otherwise. */
+int
+crypto_force_rand_ssleay(void)
+{
+#ifdef ENABLE_OPENSSL
+  RAND_METHOD *default_method;
+  default_method = RAND_OpenSSL();
+  if (RAND_get_rand_method() != default_method) {
+    log_notice(LD_CRYPTO, "It appears that one of our engines has provided "
+               "a replacement the OpenSSL RNG. Resetting it to the default "
+               "implementation.");
+    RAND_set_rand_method(default_method);
+    return 1;
+  }
+#endif
+  return 0;
+}
+
+#endif /* !defined(CRYPTO_RAND_PRIVATE) */
diff --git a/src/lib/crypt_ops/crypto_rand.h b/src/lib/crypt_ops/crypto_rand.h
new file mode 100644
index 0000000000..86fa20faa3
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_rand.h
@@ -0,0 +1,53 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_rand.h
+ *
+ * \brief Common functions for using (pseudo-)random number generators.
+ **/
+
+#ifndef TOR_CRYPTO_RAND_H
+#define TOR_CRYPTO_RAND_H
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+#include "lib/testsupport/testsupport.h"
+
+/* random numbers */
+int crypto_seed_rng(void) ATTR_WUR;
+MOCK_DECL(void,crypto_rand,(char *to, size_t n));
+void crypto_rand_unmocked(char *to, size_t n);
+void crypto_strongest_rand(uint8_t *out, size_t out_len);
+MOCK_DECL(void,crypto_strongest_rand_,(uint8_t *out, size_t out_len));
+int crypto_rand_int(unsigned int max);
+int crypto_rand_int_range(unsigned int min, unsigned int max);
+uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max);
+time_t crypto_rand_time_range(time_t min, time_t max);
+uint64_t crypto_rand_uint64(uint64_t max);
+double crypto_rand_double(void);
+struct tor_weak_rng_t;
+void crypto_seed_weak_rng(struct tor_weak_rng_t *rng);
+
+char *crypto_random_hostname(int min_rand_len, int max_rand_len,
+                             const char *prefix, const char *suffix);
+
+struct smartlist_t;
+void *smartlist_choose(const struct smartlist_t *sl);
+void smartlist_shuffle(struct smartlist_t *sl);
+int crypto_force_rand_ssleay(void);
+
+#ifdef CRYPTO_RAND_PRIVATE
+
+STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len);
+
+#ifdef TOR_UNIT_TESTS
+extern int break_strongest_rng_syscall;
+extern int break_strongest_rng_fallback;
+#endif
+#endif /* defined(CRYPTO_RAND_PRIVATE) */
+
+#endif /* !defined(TOR_CRYPTO_RAND_H) */
diff --git a/src/lib/crypt_ops/crypto_rsa.c b/src/lib/crypt_ops/crypto_rsa.c
new file mode 100644
index 0000000000..c9189b0dfc
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_rsa.c
@@ -0,0 +1,672 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_rsa.c
+ * \brief Block of functions related with RSA utilities and operations.
+ **/
+
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/compat_openssl.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_rsa.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/log/util_bug.h"
+#include "lib/fs/files.h"
+
+#include "lib/log/escape.h"
+#include "lib/log/log.h"
+#include "lib/encoding/binascii.h"
+#include "lib/encoding/pem.h"
+
+#include <string.h>
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
+#ifdef ENABLE_OPENSSL
+#include <openssl/rsa.h>
+#endif
+
+/** Return the number of bytes added by padding method <b>padding</b>.
+ */
+int
+crypto_get_rsa_padding_overhead(int padding)
+{
+  switch (padding)
+    {
+    case PK_PKCS1_OAEP_PADDING: return PKCS1_OAEP_PADDING_OVERHEAD;
+    default: tor_assert(0); return -1; // LCOV_EXCL_LINE
+    }
+}
+
+#ifdef ENABLE_OPENSSL
+/** Given a padding method <b>padding</b>, return the correct OpenSSL constant.
+ */
+int
+crypto_get_rsa_padding(int padding)
+{
+  switch (padding)
+    {
+    case PK_PKCS1_OAEP_PADDING: return RSA_PKCS1_OAEP_PADDING;
+    default: tor_assert(0); return -1; // LCOV_EXCL_LINE
+    }
+}
+#endif
+
+/** Compare the public-key components of a and b.  Return non-zero iff
+ * a==b.  A NULL key is considered to be distinct from all non-NULL
+ * keys, and equal to itself.
+ *
+ *  Note that this may leak information about the keys through timing.
+ */
+int
+crypto_pk_eq_keys(const crypto_pk_t *a, const crypto_pk_t *b)
+{
+  return (crypto_pk_cmp_keys(a, b) == 0);
+}
+
+/** Perform a hybrid (public/secret) encryption on <b>fromlen</b>
+ * bytes of data from <b>from</b>, with padding type 'padding',
+ * storing the results on <b>to</b>.
+ *
+ * Returns the number of bytes written on success, -1 on failure.
+ *
+ * The encrypted data consists of:
+ *   - The source data, padded and encrypted with the public key, if the
+ *     padded source data is no longer than the public key, and <b>force</b>
+ *     is false, OR
+ *   - The beginning of the source data prefixed with a 16-byte symmetric key,
+ *     padded and encrypted with the public key; followed by the rest of
+ *     the source data encrypted in AES-CTR mode with the symmetric key.
+ *
+ * NOTE that this format does not authenticate the symmetrically encrypted
+ * part of the data, and SHOULD NOT BE USED for new protocols.
+ */
+int
+crypto_pk_obsolete_public_hybrid_encrypt(crypto_pk_t *env,
+                                char *to, size_t tolen,
+                                const char *from,
+                                size_t fromlen,
+                                int padding, int force)
+{
+  int overhead, outlen, r;
+  size_t pkeylen, symlen;
+  crypto_cipher_t *cipher = NULL;
+  char *buf = NULL;
+
+  tor_assert(env);
+  tor_assert(from);
+  tor_assert(to);
+  tor_assert(fromlen < SIZE_T_CEILING);
+
+  overhead = crypto_get_rsa_padding_overhead(padding);
+  pkeylen = crypto_pk_keysize(env);
+
+  if (!force && fromlen+overhead <= pkeylen) {
+    /* It all fits in a single encrypt. */
+    return crypto_pk_public_encrypt(env,to,
+                                    tolen,
+                                    from,fromlen,padding);
+  }
+  tor_assert(tolen >= fromlen + overhead + CIPHER_KEY_LEN);
+  tor_assert(tolen >= pkeylen);
+
+  char key[CIPHER_KEY_LEN];
+  crypto_rand(key, sizeof(key)); /* generate a new key. */
+  cipher = crypto_cipher_new(key);
+
+  buf = tor_malloc(pkeylen+1);
+  memcpy(buf, key, CIPHER_KEY_LEN);
+  memcpy(buf+CIPHER_KEY_LEN, from, pkeylen-overhead-CIPHER_KEY_LEN);
+
+  /* Length of symmetrically encrypted data. */
+  symlen = fromlen-(pkeylen-overhead-CIPHER_KEY_LEN);
+
+  outlen = crypto_pk_public_encrypt(env,to,tolen,buf,pkeylen-overhead,padding);
+  if (outlen!=(int)pkeylen) {
+    goto err;
+  }
+  r = crypto_cipher_encrypt(cipher, to+outlen,
+                            from+pkeylen-overhead-CIPHER_KEY_LEN, symlen);
+
+  if (r<0) goto err;
+  memwipe(buf, 0, pkeylen);
+  memwipe(key, 0, sizeof(key));
+  tor_free(buf);
+  crypto_cipher_free(cipher);
+  tor_assert(outlen+symlen < INT_MAX);
+  return (int)(outlen + symlen);
+ err:
+
+  memwipe(buf, 0, pkeylen);
+  memwipe(key, 0, sizeof(key));
+  tor_free(buf);
+  crypto_cipher_free(cipher);
+  return -1;
+}
+
+/** Invert crypto_pk_obsolete_public_hybrid_encrypt. Returns the number of
+ * bytes written on success, -1 on failure.
+ *
+ * NOTE that this format does not authenticate the symmetrically encrypted
+ * part of the data, and SHOULD NOT BE USED for new protocols.
+ */
+int
+crypto_pk_obsolete_private_hybrid_decrypt(crypto_pk_t *env,
+                                 char *to,
+                                 size_t tolen,
+                                 const char *from,
+                                 size_t fromlen,
+                                 int padding, int warnOnFailure)
+{
+  int outlen, r;
+  size_t pkeylen;
+  crypto_cipher_t *cipher = NULL;
+  char *buf = NULL;
+
+  tor_assert(fromlen < SIZE_T_CEILING);
+  pkeylen = crypto_pk_keysize(env);
+
+  if (fromlen <= pkeylen) {
+    return crypto_pk_private_decrypt(env,to,tolen,from,fromlen,padding,
+                                     warnOnFailure);
+  }
+
+  buf = tor_malloc(pkeylen);
+  outlen = crypto_pk_private_decrypt(env,buf,pkeylen,from,pkeylen,padding,
+                                     warnOnFailure);
+  if (outlen<0) {
+    log_fn(warnOnFailure?LOG_WARN:LOG_DEBUG, LD_CRYPTO,
+           "Error decrypting public-key data");
+    goto err;
+  }
+  if (outlen < CIPHER_KEY_LEN) {
+    log_fn(warnOnFailure?LOG_WARN:LOG_INFO, LD_CRYPTO,
+           "No room for a symmetric key");
+    goto err;
+  }
+  cipher = crypto_cipher_new(buf);
+  if (!cipher) {
+    goto err;
+  }
+  memcpy(to,buf+CIPHER_KEY_LEN,outlen-CIPHER_KEY_LEN);
+  outlen -= CIPHER_KEY_LEN;
+  tor_assert(tolen - outlen >= fromlen - pkeylen);
+  r = crypto_cipher_decrypt(cipher, to+outlen, from+pkeylen, fromlen-pkeylen);
+  if (r<0)
+    goto err;
+  memwipe(buf,0,pkeylen);
+  tor_free(buf);
+  crypto_cipher_free(cipher);
+  tor_assert(outlen + fromlen < INT_MAX);
+  return (int)(outlen + (fromlen-pkeylen));
+ err:
+  memwipe(buf,0,pkeylen);
+  tor_free(buf);
+  crypto_cipher_free(cipher);
+  return -1;
+}
+
+/** Given a private or public key <b>pk</b>, put a fingerprint of the
+ * public key into <b>fp_out</b> (must have at least FINGERPRINT_LEN+1 bytes of
+ * space).  Return 0 on success, -1 on failure.
+ *
+ * Fingerprints are computed as the SHA1 digest of the ASN.1 encoding
+ * of the public key, converted to hexadecimal, in upper case, with a
+ * space after every four digits.
+ *
+ * If <b>add_space</b> is false, omit the spaces.
+ */
+int
+crypto_pk_get_fingerprint(crypto_pk_t *pk, char *fp_out, int add_space)
+{
+  char digest[DIGEST_LEN];
+  char hexdigest[HEX_DIGEST_LEN+1];
+  if (crypto_pk_get_digest(pk, digest)) {
+    return -1;
+  }
+  base16_encode(hexdigest,sizeof(hexdigest),digest,DIGEST_LEN);
+  if (add_space) {
+    crypto_add_spaces_to_fp(fp_out, FINGERPRINT_LEN+1, hexdigest);
+  } else {
+    strncpy(fp_out, hexdigest, HEX_DIGEST_LEN+1);
+  }
+  return 0;
+}
+
+/** Given a private or public key <b>pk</b>, put a hashed fingerprint of
+ * the public key into <b>fp_out</b> (must have at least FINGERPRINT_LEN+1
+ * bytes of space).  Return 0 on success, -1 on failure.
+ *
+ * Hashed fingerprints are computed as the SHA1 digest of the SHA1 digest
+ * of the ASN.1 encoding of the public key, converted to hexadecimal, in
+ * upper case.
+ */
+int
+crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out)
+{
+  char digest[DIGEST_LEN], hashed_digest[DIGEST_LEN];
+  if (crypto_pk_get_digest(pk, digest)) {
+    return -1;
+  }
+  if (crypto_digest(hashed_digest, digest, DIGEST_LEN) < 0) {
+    return -1;
+  }
+  base16_encode(fp_out, FINGERPRINT_LEN + 1, hashed_digest, DIGEST_LEN);
+  return 0;
+}
+
+/** Copy <b>in</b> to the <b>outlen</b>-byte buffer <b>out</b>, adding spaces
+ * every four characters. */
+void
+crypto_add_spaces_to_fp(char *out, size_t outlen, const char *in)
+{
+  int n = 0;
+  char *end = out+outlen;
+  tor_assert(outlen < SIZE_T_CEILING);
+
+  while (*in && out<end) {
+    *out++ = *in++;
+    if (++n == 4 && *in && out<end) {
+      n = 0;
+      *out++ = ' ';
+    }
+  }
+  tor_assert(out<end);
+  *out = '\0';
+}
+
+/** Check a siglen-byte long signature at <b>sig</b> against
+ * <b>datalen</b> bytes of data at <b>data</b>, using the public key
+ * in <b>env</b>. Return 0 if <b>sig</b> is a correct signature for
+ * SHA1(data).  Else return -1.
+ */
+MOCK_IMPL(int,
+crypto_pk_public_checksig_digest,(crypto_pk_t *env, const char *data,
+                                  size_t datalen, const char *sig,
+                                  size_t siglen))
+{
+  char digest[DIGEST_LEN];
+  char *buf;
+  size_t buflen;
+  int r;
+
+  tor_assert(env);
+  tor_assert(data);
+  tor_assert(sig);
+  tor_assert(datalen < SIZE_T_CEILING);
+  tor_assert(siglen < SIZE_T_CEILING);
+
+  if (crypto_digest(digest,data,datalen)<0) {
+    log_warn(LD_BUG, "couldn't compute digest");
+    return -1;
+  }
+  buflen = crypto_pk_keysize(env);
+  buf = tor_malloc(buflen);
+  r = crypto_pk_public_checksig(env,buf,buflen,sig,siglen);
+  if (r != DIGEST_LEN) {
+    log_warn(LD_CRYPTO, "Invalid signature");
+    tor_free(buf);
+    return -1;
+  }
+  if (tor_memneq(buf, digest, DIGEST_LEN)) {
+    log_warn(LD_CRYPTO, "Signature mismatched with digest.");
+    tor_free(buf);
+    return -1;
+  }
+  tor_free(buf);
+
+  return 0;
+}
+
+/** Compute a SHA1 digest of <b>fromlen</b> bytes of data stored at
+ * <b>from</b>; sign the data with the private key in <b>env</b>, and
+ * store it in <b>to</b>.  Return the number of bytes written on
+ * success, and -1 on failure.
+ *
+ * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
+ * at least the length of the modulus of <b>env</b>.
+ */
+int
+crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen,
+                              const char *from, size_t fromlen)
+{
+  int r;
+  char digest[DIGEST_LEN];
+  if (crypto_digest(digest,from,fromlen)<0)
+    return -1;
+  r = crypto_pk_private_sign(env,to,tolen,digest,DIGEST_LEN);
+  memwipe(digest, 0, sizeof(digest));
+  return r;
+}
+
+/** Given a private or public key <b>pk</b>, put a SHA1 hash of the
+ * public key into <b>digest_out</b> (must have DIGEST_LEN bytes of space).
+ * Return 0 on success, -1 on failure.
+ */
+int
+crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out)
+{
+  char *buf;
+  size_t buflen;
+  int len;
+  int rv = -1;
+
+  buflen = crypto_pk_keysize(pk)*2;
+  buf = tor_malloc(buflen);
+  len = crypto_pk_asn1_encode(pk, buf, buflen);
+  if (len < 0)
+    goto done;
+
+  if (crypto_digest(digest_out, buf, len) < 0)
+    goto done;
+
+  rv = 0;
+  done:
+  tor_free(buf);
+  return rv;
+}
+
+/** Compute all digests of the DER encoding of <b>pk</b>, and store them
+ * in <b>digests_out</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_pk_get_common_digests(crypto_pk_t *pk, common_digests_t *digests_out)
+{
+  char *buf;
+  size_t buflen;
+  int len;
+  int rv = -1;
+
+  buflen = crypto_pk_keysize(pk)*2;
+  buf = tor_malloc(buflen);
+  len = crypto_pk_asn1_encode(pk, buf, buflen);
+  if (len < 0)
+    goto done;
+
+  if (crypto_common_digests(digests_out, (char*)buf, len) < 0)
+    goto done;
+
+  rv = 0;
+ done:
+  tor_free(buf);
+  return rv;
+}
+
+static const char RSA_PUBLIC_TAG[] = "RSA PUBLIC KEY";
+static const char RSA_PRIVATE_TAG[] = "RSA PRIVATE KEY";
+
+/* These are overestimates for how many extra bytes we might need to encode
+ * a key in DER */
+#define PRIVATE_ASN_MAX_OVERHEAD_FACTOR 16
+#define PUBLIC_ASN_MAX_OVERHEAD_FACTOR 3
+
+/** Helper: PEM-encode <b>env</b> and write it to a newly allocated string.
+ * If <b>private_key</b>, write the private part of <b>env</b>; otherwise
+ * write only the public portion. On success, set *<b>dest</b> to the new
+ * string, *<b>len</b> to the string's length, and return 0.  On failure,
+ * return -1.
+ */
+static int
+crypto_pk_write_to_string_generic(crypto_pk_t *env,
+                                  char **dest, size_t *len,
+                                  bool private_key)
+{
+  const int factor =
+    private_key ? PRIVATE_ASN_MAX_OVERHEAD_FACTOR
+                : PUBLIC_ASN_MAX_OVERHEAD_FACTOR;
+  size_t buflen = crypto_pk_keysize(env) * factor;
+  const char *tag =
+    private_key ? RSA_PRIVATE_TAG : RSA_PUBLIC_TAG;
+  char *buf = tor_malloc(buflen);
+  char *result = NULL;
+  size_t resultlen = 0;
+  int rv = -1;
+
+  int n = private_key
+    ? crypto_pk_asn1_encode_private(env, buf, buflen)
+    : crypto_pk_asn1_encode(env, buf, buflen);
+  if (n < 0)
+    goto done;
+
+  resultlen = pem_encoded_size(n, tag);
+  result = tor_malloc(resultlen);
+  if (pem_encode(result, resultlen,
+                 (const unsigned char *)buf, n, tag) < 0) {
+    goto done;
+  }
+
+  *dest = result;
+  *len = resultlen;
+  rv = 0;
+
+ done:
+  if (rv < 0 && result) {
+    memwipe(result, 0, resultlen);
+    tor_free(result);
+  }
+  memwipe(buf, 0, buflen);
+  tor_free(buf);
+  return rv;
+}
+
+/** PEM-encode the public key portion of <b>env</b> and write it to a
+ * newly allocated string.  On success, set *<b>dest</b> to the new
+ * string, *<b>len</b> to the string's length, and return 0.  On
+ * failure, return -1.
+ */
+int
+crypto_pk_write_public_key_to_string(crypto_pk_t *env,
+                                     char **dest, size_t *len)
+{
+  return crypto_pk_write_to_string_generic(env, dest, len, false);
+}
+
+/** PEM-encode the private key portion of <b>env</b> and write it to a
+ * newly allocated string.  On success, set *<b>dest</b> to the new
+ * string, *<b>len</b> to the string's length, and return 0.  On
+ * failure, return -1.
+ */
+int
+crypto_pk_write_private_key_to_string(crypto_pk_t *env,
+                                          char **dest, size_t *len)
+{
+  return crypto_pk_write_to_string_generic(env, dest, len, true);
+}
+
+/**
+ * Helper. Read a PEM-encoded RSA from the first <b>len</b> characters of
+ * <b>src</b>, and store the result in <b>env</b>.  If <b>private_key</b>,
+ * expect a private key; otherwise expect a public key. Return 0 on success,
+ * -1 on failure.  If len is -1, the string is nul-terminated.
+ */
+static int
+crypto_pk_read_from_string_generic(crypto_pk_t *env, const char *src,
+                                   size_t len, int severity,
+                                   bool private_key)
+{
+  if (len == (size_t)-1) // "-1" indicates "use the length of the string."
+    len = strlen(src);
+
+  const char *ktype = private_key ? "private key" : "public key";
+  const char *tag =
+    private_key ? RSA_PRIVATE_TAG : RSA_PUBLIC_TAG;
+  size_t buflen = len;
+  uint8_t *buf = tor_malloc(buflen);
+  int rv = -1;
+
+  int n = pem_decode(buf, buflen, src, len, tag);
+  if (n < 0) {
+    log_fn(severity, LD_CRYPTO,
+           "Error decoding PEM wrapper while reading %s", ktype);
+    goto done;
+  }
+
+  crypto_pk_t *pk = private_key
+    ? crypto_pk_asn1_decode_private((const char*)buf, n)
+    : crypto_pk_asn1_decode((const char*)buf, n);
+  if (! pk) {
+    log_fn(severity, LD_CRYPTO,
+           "Error decoding ASN.1 while reading %s", ktype);
+    goto done;
+  }
+
+  if (private_key)
+    crypto_pk_assign_private(env, pk);
+  else
+    crypto_pk_assign_public(env, pk);
+  crypto_pk_free(pk);
+  rv = 0;
+
+ done:
+  memwipe(buf, 0, buflen);
+  tor_free(buf);
+  return rv;
+}
+
+/** Read a PEM-encoded public key from the first <b>len</b> characters of
+ * <b>src</b>, and store the result in <b>env</b>.  Return 0 on success, -1 on
+ * failure.  If len is -1, the string is nul-terminated.
+ */
+int
+crypto_pk_read_public_key_from_string(crypto_pk_t *env,
+                                      const char *src, size_t len)
+{
+  return crypto_pk_read_from_string_generic(env, src, len, LOG_INFO, false);
+}
+
+/** Read a PEM-encoded private key from the <b>len</b>-byte string <b>src</b>
+ * into <b>env</b>.  Return 0 on success, -1 on failure.  If len is -1,
+ * the string is nul-terminated.
+ */
+int
+crypto_pk_read_private_key_from_string(crypto_pk_t *env,
+                                       const char *src, ssize_t len)
+{
+  return crypto_pk_read_from_string_generic(env, src, len, LOG_INFO, true);
+}
+
+/** If a file is longer than this, we won't try to decode its private key */
+#define MAX_PRIVKEY_FILE_LEN (16*1024*1024)
+
+/** Read a PEM-encoded private key from the file named by
+ * <b>keyfile</b> into <b>env</b>.  Return 0 on success, -1 on failure.
+ */
+int
+crypto_pk_read_private_key_from_filename(crypto_pk_t *env,
+                                         const char *keyfile)
+{
+  struct stat st;
+  char *buf = read_file_to_str(keyfile, 0, &st);
+  if (!buf) {
+    log_warn(LD_CRYPTO, "Unable to read file for private key in %s",
+             escaped(keyfile));
+    return -1;
+  }
+  if (st.st_size > MAX_PRIVKEY_FILE_LEN) {
+    log_warn(LD_CRYPTO, "Private key file %s was far too large.",
+             escaped(keyfile));
+    tor_free(buf);
+    return -1;
+  }
+
+  int rv = crypto_pk_read_from_string_generic(env, buf, (ssize_t)st.st_size,
+                                              LOG_WARN, true);
+  if (rv < 0) {
+    log_warn(LD_CRYPTO, "Unable to decode private key from file %s",
+             escaped(keyfile));
+  }
+  memwipe(buf, 0, (size_t)st.st_size);
+  tor_free(buf);
+  return rv;
+}
+
+/** Write the private key from <b>env</b> into the file named by <b>fname</b>,
+ * PEM-encoded.  Return 0 on success, -1 on failure.
+ */
+int
+crypto_pk_write_private_key_to_filename(crypto_pk_t *env,
+                                        const char *fname)
+{
+  char *s = NULL;
+  size_t n = 0;
+
+  if (crypto_pk_write_private_key_to_string(env, &s, &n) < 0)
+    return -1;
+
+  int rv = write_bytes_to_file(fname, s, n, 0);
+  memwipe(s, 0, n);
+  tor_free(s);
+  return rv;
+}
+
+/** Given a crypto_pk_t <b>pk</b>, allocate a new buffer containing the
+ * Base64 encoding of the DER representation of the private key as a NUL
+ * terminated string, and return it via <b>priv_out</b>.  Return 0 on
+ * success, -1 on failure.
+ *
+ * It is the caller's responsibility to sanitize and free the resulting buffer.
+ */
+int
+crypto_pk_base64_encode_private(const crypto_pk_t *pk, char **priv_out)
+{
+  size_t buflen = crypto_pk_keysize(pk)*16;
+  char *buf = tor_malloc(buflen);
+  char *result = NULL;
+  size_t reslen = 0;
+  bool ok = false;
+
+  int n = crypto_pk_asn1_encode_private(pk, buf, buflen);
+
+  if (n < 0)
+    goto done;
+
+  reslen = base64_encode_size(n, 0)+1;
+  result = tor_malloc(reslen);
+  if (base64_encode(result, reslen, buf, n, 0) < 0)
+    goto done;
+
+  ok = true;
+
+ done:
+  memwipe(buf, 0, buflen);
+  tor_free(buf);
+  if (result && ! ok) {
+    memwipe(result, 0, reslen);
+    tor_free(result);
+  }
+  *priv_out = result;
+  return ok ? 0 : -1;
+}
+
+/** Given a string containing the Base64 encoded DER representation of the
+ * private key <b>str</b>, decode and return the result on success, or NULL
+ * on failure.
+ */
+crypto_pk_t *
+crypto_pk_base64_decode_private(const char *str, size_t len)
+{
+  crypto_pk_t *pk = NULL;
+
+  char *der = tor_malloc_zero(len + 1);
+  int der_len = base64_decode(der, len, str, len);
+  if (der_len <= 0) {
+    log_warn(LD_CRYPTO, "Stored RSA private key seems corrupted (base64).");
+    goto out;
+  }
+
+  pk = crypto_pk_asn1_decode_private(der, der_len);
+
+ out:
+  memwipe(der, 0, len+1);
+  tor_free(der);
+
+  return pk;
+}
diff --git a/src/lib/crypt_ops/crypto_rsa.h b/src/lib/crypt_ops/crypto_rsa.h
new file mode 100644
index 0000000000..c1ea767f85
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_rsa.h
@@ -0,0 +1,145 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_rsa.h
+ *
+ * \brief Headers for crypto_rsa.c
+ **/
+
+#ifndef TOR_CRYPTO_RSA_H
+#define TOR_CRYPTO_RSA_H
+
+#include "orconfig.h"
+
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/cc/torint.h"
+#include "lib/testsupport/testsupport.h"
+#include "lib/log/log.h"
+
+/** Length of our public keys. */
+#define PK_BYTES (1024/8)
+
+/** Constant used to indicate OAEP padding for public-key encryption */
+#define PK_PKCS1_OAEP_PADDING 60002
+
+/** Number of bytes added for PKCS1-OAEP padding. */
+#define PKCS1_OAEP_PADDING_OVERHEAD 42
+
+/** Length of encoded public key fingerprints, including space; but not
+ * including terminating NUL. */
+#define FINGERPRINT_LEN 49
+
+/** Value of 'e' to use in our public keys */
+#define TOR_RSA_EXPONENT 65537
+
+/** A public key, or a public/private key-pair. */
+typedef struct crypto_pk_t crypto_pk_t;
+
+/* RSA environment setup */
+MOCK_DECL(crypto_pk_t *,crypto_pk_new,(void));
+void crypto_pk_free_(crypto_pk_t *env);
+#define crypto_pk_free(pk) FREE_AND_NULL(crypto_pk_t, crypto_pk_free_, (pk))
+int crypto_get_rsa_padding_overhead(int padding);
+int crypto_get_rsa_padding(int padding);
+
+/* public key crypto */
+MOCK_DECL(int, crypto_pk_generate_key_with_bits,(crypto_pk_t *env, int bits));
+#define crypto_pk_generate_key(env)                     \
+  crypto_pk_generate_key_with_bits((env), (PK_BYTES*8))
+
+int crypto_pk_read_private_key_from_filename(crypto_pk_t *env,
+                                             const char *keyfile);
+int crypto_pk_write_public_key_to_string(crypto_pk_t *env,
+                                         char **dest, size_t *len);
+int crypto_pk_write_private_key_to_string(crypto_pk_t *env,
+                                          char **dest, size_t *len);
+int crypto_pk_read_public_key_from_string(crypto_pk_t *env,
+                                          const char *src, size_t len);
+int crypto_pk_read_private_key_from_string(crypto_pk_t *env,
+                                           const char *s, ssize_t len);
+int crypto_pk_write_private_key_to_filename(crypto_pk_t *env,
+                                            const char *fname);
+
+int crypto_pk_is_valid_private_key(const crypto_pk_t *env);
+int crypto_pk_cmp_keys(const crypto_pk_t *a, const crypto_pk_t *b);
+int crypto_pk_eq_keys(const crypto_pk_t *a, const crypto_pk_t *b);
+size_t crypto_pk_keysize(const crypto_pk_t *env);
+int crypto_pk_num_bits(crypto_pk_t *env);
+crypto_pk_t *crypto_pk_dup_key(crypto_pk_t *orig);
+crypto_pk_t *crypto_pk_copy_full(crypto_pk_t *orig);
+int crypto_pk_key_is_private(const crypto_pk_t *key);
+int crypto_pk_public_exponent_ok(const crypto_pk_t *env);
+int crypto_pk_obsolete_public_hybrid_encrypt(crypto_pk_t *env, char *to,
+                                    size_t tolen,
+                                    const char *from, size_t fromlen,
+                                    int padding, int force);
+int crypto_pk_obsolete_private_hybrid_decrypt(crypto_pk_t *env, char *to,
+                                     size_t tolen,
+                                     const char *from, size_t fromlen,
+                                     int padding, int warnOnFailure);
+int crypto_pk_public_encrypt(crypto_pk_t *env, char *to, size_t tolen,
+                             const char *from, size_t fromlen, int padding);
+int crypto_pk_private_decrypt(crypto_pk_t *env, char *to, size_t tolen,
+                              const char *from, size_t fromlen,
+                              int padding, int warnOnFailure);
+MOCK_DECL(int, crypto_pk_public_checksig,(const crypto_pk_t *env,
+                                          char *to, size_t tolen,
+                                          const char *from, size_t fromlen));
+int crypto_pk_private_sign(const crypto_pk_t *env, char *to, size_t tolen,
+                           const char *from, size_t fromlen);
+int crypto_pk_asn1_encode(const crypto_pk_t *pk, char *dest, size_t dest_len);
+crypto_pk_t *crypto_pk_asn1_decode(const char *str, size_t len);
+int crypto_pk_asn1_encode_private(const crypto_pk_t *pk,
+                                  char *dest, size_t dest_len);
+crypto_pk_t *crypto_pk_asn1_decode_private(const char *str, size_t len);
+int crypto_pk_get_fingerprint(crypto_pk_t *pk, char *fp_out,int add_space);
+int crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out);
+void crypto_add_spaces_to_fp(char *out, size_t outlen, const char *in);
+
+MOCK_DECL(int, crypto_pk_public_checksig_digest,(crypto_pk_t *env,
+          const char *data, size_t datalen, const char *sig, size_t siglen));
+int crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen,
+                                  const char *from, size_t fromlen);
+int crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out);
+int crypto_pk_get_common_digests(crypto_pk_t *pk,
+                                 common_digests_t *digests_out);
+int crypto_pk_base64_encode_private(const crypto_pk_t *pk, char **priv_out);
+crypto_pk_t *crypto_pk_base64_decode_private(const char *str, size_t len);
+
+#ifdef ENABLE_OPENSSL
+/* Prototypes for private functions only used by tortls.c, crypto.c, and the
+ * unit tests. */
+struct rsa_st;
+struct evp_pkey_st;
+struct rsa_st *crypto_pk_get_openssl_rsa_(crypto_pk_t *env);
+crypto_pk_t *crypto_new_pk_from_openssl_rsa_(struct rsa_st *rsa);
+MOCK_DECL(struct evp_pkey_st *, crypto_pk_get_openssl_evp_pkey_,(
+                                 crypto_pk_t *env,int private));
+#endif
+
+#ifdef ENABLE_NSS
+struct SECKEYPublicKeyStr;
+struct SECKEYPrivateKeyStr;
+crypto_pk_t *crypto_pk_new_from_nss_pubkey(struct SECKEYPublicKeyStr *pub);
+const struct SECKEYPublicKeyStr *crypto_pk_get_nss_pubkey(
+                                           const crypto_pk_t *key);
+const struct SECKEYPrivateKeyStr *crypto_pk_get_nss_privkey(
+                                           const crypto_pk_t *key);
+#endif
+
+void crypto_pk_assign_public(crypto_pk_t *dest, const crypto_pk_t *src);
+void crypto_pk_assign_private(crypto_pk_t *dest, const crypto_pk_t *src);
+
+#ifdef TOR_UNIT_TESTS
+#ifdef ENABLE_NSS
+struct SECItemStr;
+STATIC int secitem_uint_cmp(const struct SECItemStr *a,
+                            const struct SECItemStr *b);
+#endif
+#endif
+
+#endif
diff --git a/src/lib/crypt_ops/crypto_rsa_nss.c b/src/lib/crypt_ops/crypto_rsa_nss.c
new file mode 100644
index 0000000000..ad2ad38b66
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_rsa_nss.c
@@ -0,0 +1,738 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_rsa.c
+ * \brief NSS implementations of our RSA code.
+ **/
+
+#include "lib/crypt_ops/crypto_rsa.h"
+
+#include "lib/crypt_ops/crypto_nss_mgt.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/encoding/binascii.h"
+#include "lib/fs/files.h"
+#include "lib/intmath/cmp.h"
+#include "lib/intmath/muldiv.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+
+#include <string.h>
+
+#include <keyhi.h>
+#include <pk11pub.h>
+#include <secder.h>
+
+#ifdef ENABLE_OPENSSL
+#include <openssl/rsa.h>
+#include <openssl/evp.h>
+#endif
+
+/** Declaration for crypto_pk_t structure. */
+struct crypto_pk_t
+{
+  SECKEYPrivateKey *seckey;
+  SECKEYPublicKey *pubkey;
+};
+
+/** Return true iff <b>key</b> contains the private-key portion of the RSA
+ * key. */
+int
+crypto_pk_key_is_private(const crypto_pk_t *key)
+{
+  return key && key->seckey;
+}
+
+/** used by tortls.c: wrap a SecKEYPublicKey in a crypto_pk_t. Take ownership
+ * of the RSA object. */
+crypto_pk_t *
+crypto_pk_new_from_nss_pubkey(struct SECKEYPublicKeyStr *pub)
+{
+  crypto_pk_t *result = tor_malloc_zero(sizeof(crypto_pk_t));
+  result->pubkey = pub;
+  return result;
+}
+
+/** Return the SECKEYPublicKey for the provided crypto_pk_t. */
+const SECKEYPublicKey *
+crypto_pk_get_nss_pubkey(const crypto_pk_t *key)
+{
+  tor_assert(key);
+  return key->pubkey;
+}
+
+/** Return the SECKEYPrivateKey for the provided crypto_pk_t, or NULL if it
+ * does not exist. */
+const SECKEYPrivateKey *
+crypto_pk_get_nss_privkey(const crypto_pk_t *key)
+{
+  tor_assert(key);
+  return key->seckey;
+}
+
+#ifdef ENABLE_OPENSSL
+/** used by tortls.c: wrap an RSA* in a crypto_pk_t. Take ownership of the
+ * RSA object. */
+crypto_pk_t *
+crypto_new_pk_from_openssl_rsa_(RSA *rsa)
+{
+  crypto_pk_t *pk = NULL;
+  unsigned char *buf = NULL;
+  int len = i2d_RSAPublicKey(rsa, &buf);
+  RSA_free(rsa);
+
+  if (len < 0 || buf == NULL)
+    goto end;
+
+  pk = crypto_pk_asn1_decode((const char *)buf, len);
+
+ end:
+  if (buf)
+    OPENSSL_free(buf);
+  return pk;
+}
+
+/** Helper, used by tor-gencert.c.  Return the RSA from a
+ * crypto_pk_t. */
+struct rsa_st *
+crypto_pk_get_openssl_rsa_(crypto_pk_t *pk)
+{
+  size_t buflen = crypto_pk_keysize(pk)*16;
+  unsigned char *buf = tor_malloc_zero(buflen);
+  const unsigned char *cp = buf;
+  RSA *rsa = NULL;
+
+  int used = crypto_pk_asn1_encode_private(pk, (char*)buf, buflen);
+  if (used < 0)
+    goto end;
+  rsa = d2i_RSAPrivateKey(NULL, &cp, used);
+
+ end:
+  memwipe(buf, 0, buflen);
+  tor_free(buf);
+  return rsa;
+}
+
+/** used by tortls.c: get an equivalent EVP_PKEY* for a crypto_pk_t.  Iff
+ * private is set, include the private-key portion of the key. Return a valid
+ * pointer on success, and NULL on failure. */
+MOCK_IMPL(struct evp_pkey_st *,
+crypto_pk_get_openssl_evp_pkey_,(crypto_pk_t *pk, int private))
+{
+  size_t buflen = crypto_pk_keysize(pk)*16;
+  unsigned char *buf = tor_malloc_zero(buflen);
+  const unsigned char *cp = buf;
+  RSA *rsa = NULL;
+  EVP_PKEY *result = NULL;
+
+  if (private) {
+    int len = crypto_pk_asn1_encode_private(pk, (char*)buf, buflen);
+    if (len < 0)
+      goto end;
+    rsa = d2i_RSAPrivateKey(NULL, &cp, len);
+  } else {
+    int len = crypto_pk_asn1_encode(pk, (char*)buf, buflen);
+    if (len < 0)
+      goto end;
+    rsa = d2i_RSAPublicKey(NULL, &cp, len);
+  }
+  if (!rsa)
+    goto end;
+
+  if (!(result = EVP_PKEY_new()))
+    goto end;
+  if (!(EVP_PKEY_assign_RSA(result, rsa))) {
+    EVP_PKEY_free(result);
+    RSA_free(rsa);
+    result = NULL;
+  }
+
+ end:
+  memwipe(buf, 0, buflen);
+  tor_free(buf);
+  return result;
+}
+#endif
+
+/** Allocate and return storage for a public key.  The key itself will not yet
+ * be set.
+ */
+MOCK_IMPL(crypto_pk_t *,
+crypto_pk_new,(void))
+{
+  crypto_pk_t *result = tor_malloc_zero(sizeof(crypto_pk_t));
+  return result;
+}
+
+/** Release the NSS objects held in <b>key</b> */
+static void
+crypto_pk_clear(crypto_pk_t *key)
+{
+  if (key->pubkey)
+    SECKEY_DestroyPublicKey(key->pubkey);
+  if (key->seckey)
+    SECKEY_DestroyPrivateKey(key->seckey);
+  memset(key, 0, sizeof(crypto_pk_t));
+}
+
+/** Release a reference to an asymmetric key; when all the references
+ * are released, free the key.
+ */
+void
+crypto_pk_free_(crypto_pk_t *key)
+{
+  if (!key)
+    return;
+
+  crypto_pk_clear(key);
+
+  tor_free(key);
+}
+
+/** Generate a <b>bits</b>-bit new public/private keypair in <b>env</b>.
+ * Return 0 on success, -1 on failure.
+ */
+MOCK_IMPL(int,
+crypto_pk_generate_key_with_bits,(crypto_pk_t *key, int bits))
+{
+  tor_assert(key);
+
+  PK11RSAGenParams params = {
+    .keySizeInBits = bits,
+    .pe = TOR_RSA_EXPONENT
+  };
+
+  int result = -1;
+  PK11SlotInfo *slot = PK11_GetBestSlot(CKM_RSA_PKCS_KEY_PAIR_GEN, NULL);
+  SECKEYPrivateKey *seckey = NULL;
+  SECKEYPublicKey *pubkey = NULL;
+
+  if (!slot) {
+    crypto_nss_log_errors(LOG_WARN, "getting slot for RSA keygen");
+    goto done;
+  }
+
+  seckey = PK11_GenerateKeyPair(slot, CKM_RSA_PKCS_KEY_PAIR_GEN, &params,
+                                &pubkey,
+                                PR_FALSE /*isPerm */,
+                                PR_FALSE /*isSensitive*/,
+                                NULL);
+  if (seckey == NULL || pubkey == NULL) {
+    crypto_nss_log_errors(LOG_WARN, "generating an RSA key");
+    goto done;
+  }
+
+  crypto_pk_clear(key);
+  key->seckey = seckey;
+  key->pubkey = pubkey;
+  seckey = NULL;
+  pubkey = NULL;
+
+  result = 0;
+ done:
+  if (slot)
+    PK11_FreeSlot(slot);
+  if (pubkey)
+    SECKEY_DestroyPublicKey(pubkey);
+  if (seckey)
+    SECKEY_DestroyPrivateKey(seckey);
+
+  return result;
+}
+
+/** Return true iff <b>env</b> is a valid private key.
+ */
+int
+crypto_pk_is_valid_private_key(const crypto_pk_t *key)
+{
+  /* We don't need to do validation here, since unlike OpenSSL, NSS won't let
+   * us load private keys without validating them. */
+  return key && key->seckey;
+}
+
+/** Return true iff <b>env</b> contains a public key whose public exponent
+ * equals 65537.
+ */
+int
+crypto_pk_public_exponent_ok(const crypto_pk_t *key)
+{
+  return key &&
+    key->pubkey &&
+    key->pubkey->keyType == rsaKey &&
+    DER_GetUInteger(&key->pubkey->u.rsa.publicExponent) == TOR_RSA_EXPONENT;
+}
+
+/** Compare two big-endian integers stored in a and b; return a tristate.
+ */
+STATIC int
+secitem_uint_cmp(const SECItem *a, const SECItem *b)
+{
+  const unsigned abits = SECKEY_BigIntegerBitLength(a);
+  const unsigned bbits = SECKEY_BigIntegerBitLength(b);
+
+  if (abits < bbits)
+    return -1;
+  else if (abits > bbits)
+    return 1;
+
+  /* okay, they have the same number of bits set. Get a pair of aligned
+   * pointers to their bytes that are set... */
+  const unsigned nbytes = CEIL_DIV(abits, 8);
+  tor_assert(nbytes <= a->len);
+  tor_assert(nbytes <= b->len);
+
+  const unsigned char *aptr = a->data + (a->len - nbytes);
+  const unsigned char *bptr = b->data + (b->len - nbytes);
+
+  /* And compare them. */
+  return fast_memcmp(aptr, bptr, nbytes);
+}
+
+/** Compare the public-key components of a and b.  Return less than 0
+ * if a\<b, 0 if a==b, and greater than 0 if a\>b.  A NULL key is
+ * considered to be less than all non-NULL keys, and equal to itself.
+ *
+ * Note that this may leak information about the keys through timing.
+ */
+int
+crypto_pk_cmp_keys(const crypto_pk_t *a, const crypto_pk_t *b)
+{
+  int result;
+  char a_is_non_null = (a != NULL) && (a->pubkey != NULL);
+  char b_is_non_null = (b != NULL) && (b->pubkey != NULL);
+  char an_argument_is_null = !a_is_non_null | !b_is_non_null;
+
+  result = tor_memcmp(&a_is_non_null, &b_is_non_null, sizeof(a_is_non_null));
+  if (an_argument_is_null)
+    return result;
+
+  // This is all Tor uses with this structure.
+  tor_assert(a->pubkey->keyType == rsaKey);
+  tor_assert(b->pubkey->keyType == rsaKey);
+
+  const SECItem *a_n, *a_e, *b_n, *b_e;
+  a_n = &a->pubkey->u.rsa.modulus;
+  b_n = &b->pubkey->u.rsa.modulus;
+  a_e = &a->pubkey->u.rsa.publicExponent;
+  b_e = &b->pubkey->u.rsa.publicExponent;
+
+  result = secitem_uint_cmp(a_n, b_n);
+  if (result)
+    return result;
+  return secitem_uint_cmp(a_e, b_e);
+}
+
+/** Return the size of the public key modulus in <b>env</b>, in bytes. */
+size_t
+crypto_pk_keysize(const crypto_pk_t *key)
+{
+  tor_assert(key);
+  tor_assert(key->pubkey);
+  return SECKEY_PublicKeyStrength(key->pubkey);
+}
+
+/** Return the size of the public key modulus of <b>env</b>, in bits. */
+int
+crypto_pk_num_bits(crypto_pk_t *key)
+{
+  tor_assert(key);
+  tor_assert(key->pubkey);
+  return SECKEY_PublicKeyStrengthInBits(key->pubkey);
+}
+
+/**
+ * Make a copy of <b>key</b> and return it.
+ */
+crypto_pk_t *
+crypto_pk_dup_key(crypto_pk_t *key)
+{
+  crypto_pk_t *result = crypto_pk_new();
+  if (key->pubkey)
+    result->pubkey = SECKEY_CopyPublicKey(key->pubkey);
+  if (key->seckey)
+    result->seckey = SECKEY_CopyPrivateKey(key->seckey);
+  return result;
+}
+
+/** For testing: replace dest with src.  (Dest must have a refcount
+ * of 1) */
+void
+crypto_pk_assign_public(crypto_pk_t *dest, const crypto_pk_t *src)
+{
+  crypto_pk_clear(dest);
+  if (src->pubkey)
+    dest->pubkey = SECKEY_CopyPublicKey(src->pubkey);
+}
+
+/** For testing: replace dest with src.  (Dest must have a refcount
+ * of 1) */
+void
+crypto_pk_assign_private(crypto_pk_t *dest, const crypto_pk_t *src)
+{
+  crypto_pk_clear(dest);
+  if (src->pubkey)
+    dest->pubkey = SECKEY_CopyPublicKey(src->pubkey);
+  if (src->seckey)
+    dest->seckey = SECKEY_CopyPrivateKey(src->seckey);
+}
+
+/** Make a real honest-to-goodness copy of <b>env</b>, and return it.
+ * Returns NULL on failure. */
+crypto_pk_t *
+crypto_pk_copy_full(crypto_pk_t *key)
+{
+  // These aren't reference-counted is nss, so it's fine to just
+  // use the same function.
+  return crypto_pk_dup_key(key);
+}
+
+static const CK_RSA_PKCS_OAEP_PARAMS oaep_params = {
+            .hashAlg = CKM_SHA_1,
+            .mgf = CKG_MGF1_SHA1,
+            .source = CKZ_DATA_SPECIFIED,
+            .pSourceData = NULL,
+            .ulSourceDataLen = 0
+};
+static const SECItem oaep_item = {
+            .type = siBuffer,
+            .data = (unsigned char *) &oaep_params,
+            .len = sizeof(oaep_params)
+};
+
+/** Return the mechanism code and parameters for a given padding method when
+ * used with RSA */
+static CK_MECHANISM_TYPE
+padding_to_mechanism(int padding, SECItem **item_out)
+{
+  switch (padding) {
+    case PK_PKCS1_OAEP_PADDING:
+      *item_out = (SECItem *)&oaep_item;
+      return CKM_RSA_PKCS_OAEP;
+    default:
+      tor_assert_unreached();
+      *item_out = NULL;
+      return CKM_INVALID_MECHANISM;
+  }
+}
+
+/** Encrypt <b>fromlen</b> bytes from <b>from</b> with the public key
+ * in <b>env</b>, using the padding method <b>padding</b>.  On success,
+ * write the result to <b>to</b>, and return the number of bytes
+ * written.  On failure, return -1.
+ *
+ * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
+ * at least the length of the modulus of <b>env</b>.
+ */
+int
+crypto_pk_public_encrypt(crypto_pk_t *env, char *to, size_t tolen,
+                         const char *from, size_t fromlen, int padding)
+{
+  tor_assert(env);
+  tor_assert(to);
+  tor_assert(from);
+  tor_assert(tolen < INT_MAX);
+  tor_assert(fromlen < INT_MAX);
+
+  if (BUG(! env->pubkey))
+    return -1;
+
+  unsigned int result_len = 0;
+  SECItem *item = NULL;
+  CK_MECHANISM_TYPE m = padding_to_mechanism(padding, &item);
+
+  SECStatus s = PK11_PubEncrypt(env->pubkey, m, item,
+                                (unsigned char *)to, &result_len,
+                                (unsigned int)tolen,
+                                (const unsigned char *)from,
+                                (unsigned int)fromlen,
+                                NULL);
+  if (s != SECSuccess) {
+    crypto_nss_log_errors(LOG_WARN, "encrypting to an RSA key");
+    return -1;
+  }
+
+  return (int)result_len;
+}
+
+/** Decrypt <b>fromlen</b> bytes from <b>from</b> with the private key
+ * in <b>env</b>, using the padding method <b>padding</b>.  On success,
+ * write the result to <b>to</b>, and return the number of bytes
+ * written.  On failure, return -1.
+ *
+ * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
+ * at least the length of the modulus of <b>key</b>.
+ */
+int
+crypto_pk_private_decrypt(crypto_pk_t *key, char *to,
+                          size_t tolen,
+                          const char *from, size_t fromlen,
+                          int padding, int warnOnFailure)
+{
+  tor_assert(key);
+  tor_assert(to);
+  tor_assert(from);
+  tor_assert(tolen < INT_MAX);
+  tor_assert(fromlen < INT_MAX);
+
+  if (!crypto_pk_key_is_private(key))
+    return -1; /* Not a private key. */
+
+  unsigned int result_len = 0;
+  SECItem *item = NULL;
+  CK_MECHANISM_TYPE m = padding_to_mechanism(padding, &item);
+  SECStatus s = PK11_PrivDecrypt(key->seckey, m, item,
+                                 (unsigned char *)to, &result_len,
+                                 (unsigned int)tolen,
+                                 (const unsigned char *)from,
+                                 (unsigned int)fromlen);
+
+  if (s != SECSuccess) {
+    const int severity = warnOnFailure ? LOG_WARN : LOG_INFO;
+    crypto_nss_log_errors(severity, "decrypting with an RSA key");
+    return -1;
+  }
+
+  return (int)result_len;
+}
+
+/** Check the signature in <b>from</b> (<b>fromlen</b> bytes long) with the
+ * public key in <b>key</b>, using PKCS1 padding.  On success, write the
+ * signed data to <b>to</b>, and return the number of bytes written.
+ * On failure, return -1.
+ *
+ * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
+ * at least the length of the modulus of <b>key</b>.
+ */
+MOCK_IMPL(int,
+crypto_pk_public_checksig,(const crypto_pk_t *key, char *to,
+                           size_t tolen,
+                           const char *from, size_t fromlen))
+{
+  tor_assert(key);
+  tor_assert(to);
+  tor_assert(from);
+  tor_assert(tolen < INT_MAX);
+  tor_assert(fromlen < INT_MAX);
+  tor_assert(key->pubkey);
+
+  SECItem sig = {
+                 .type = siBuffer,
+                 .data = (unsigned char *) from,
+                 .len = (unsigned int) fromlen,
+  };
+  SECItem dsig = {
+                  .type = siBuffer,
+                  .data = (unsigned char *) to,
+                  .len = (unsigned int) tolen
+  };
+  SECStatus s;
+  s = PK11_VerifyRecover(key->pubkey, &sig, &dsig, NULL);
+  if (s != SECSuccess)
+    return -1;
+
+  return (int)dsig.len;
+}
+
+/** Sign <b>fromlen</b> bytes of data from <b>from</b> with the private key in
+ * <b>env</b>, using PKCS1 padding.  On success, write the signature to
+ * <b>to</b>, and return the number of bytes written.  On failure, return
+ * -1.
+ *
+ * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
+ * at least the length of the modulus of <b>env</b>.
+ */
+int
+crypto_pk_private_sign(const crypto_pk_t *key, char *to, size_t tolen,
+                       const char *from, size_t fromlen)
+{
+  tor_assert(key);
+  tor_assert(to);
+  tor_assert(from);
+  tor_assert(tolen < INT_MAX);
+  tor_assert(fromlen < INT_MAX);
+
+  if (BUG(!crypto_pk_key_is_private(key)))
+    return -1;
+
+  SECItem sig = {
+                 .type = siBuffer,
+                 .data = (unsigned char *)to,
+                 .len = (unsigned int) tolen
+  };
+  SECItem hash = {
+                 .type = siBuffer,
+                 .data = (unsigned char *)from,
+                 .len = (unsigned int) fromlen
+  };
+  CK_MECHANISM_TYPE m = CKM_RSA_PKCS;
+  SECStatus s = PK11_SignWithMechanism(key->seckey, m, NULL,
+                                       &sig, &hash);
+
+  if (s != SECSuccess) {
+    crypto_nss_log_errors(LOG_WARN, "signing with an RSA key");
+    return -1;
+  }
+
+  return (int)sig.len;
+}
+
+/* "This has lead to people trading hard-to-find object identifiers and ASN.1
+ * definitions like baseball cards" - Peter Gutmann, "X.509 Style Guide". */
+static const unsigned char RSA_OID[] = {
+  /* RSADSI */ 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d,
+  /* PKCS1 */ 0x01, 0x01,
+  /* RSA */ 0x01
+};
+
+/** ASN.1-encode the public portion of <b>pk</b> into <b>dest</b>.
+ * Return -1 on error, or the number of characters used on success.
+ */
+int
+crypto_pk_asn1_encode(const crypto_pk_t *pk, char *dest, size_t dest_len)
+{
+  tor_assert(pk);
+  if (pk->pubkey == NULL)
+    return -1;
+
+  CERTSubjectPublicKeyInfo *info;
+  info = SECKEY_CreateSubjectPublicKeyInfo(pk->pubkey);
+  if (! info)
+    return -1;
+
+  const SECItem *item = &info->subjectPublicKey;
+  size_t actual_len = (item->len) >> 3; /* bits to bytes */
+  size_t n_used = MIN(actual_len, dest_len);
+  memcpy(dest, item->data, n_used);
+
+  SECKEY_DestroySubjectPublicKeyInfo(info);
+  return (int) n_used;
+}
+
+/** Decode an ASN.1-encoded public key from <b>str</b>; return the result on
+ * success and NULL on failure.
+ */
+crypto_pk_t *
+crypto_pk_asn1_decode(const char *str, size_t len)
+{
+  tor_assert(str);
+  if (len >= INT_MAX)
+    return NULL;
+  CERTSubjectPublicKeyInfo info = {
+             .algorithm = {
+                           .algorithm = {
+                                         .type = siDEROID,
+                                         .data = (unsigned char *)RSA_OID,
+                                         .len = sizeof(RSA_OID)
+                                           }
+                           },
+             .subjectPublicKey = {
+                   .type = siBuffer,
+                   .data = (unsigned char *)str,
+                   .len = (unsigned int)(len << 3) /* bytes to bits */
+                                  }
+  };
+
+  SECKEYPublicKey *pub = SECKEY_ExtractPublicKey(&info);
+  if (pub == NULL)
+    return NULL;
+
+  crypto_pk_t *result = crypto_pk_new();
+  result->pubkey = pub;
+  return result;
+}
+
+DISABLE_GCC_WARNING(unused-parameter)
+
+/** Given a crypto_pk_t <b>pk</b>, allocate a new buffer containing the Base64
+ * encoding of the DER representation of the private key into the
+ * <b>dest_len</b>-byte buffer in <b>dest</b>.
+ * Return the number of bytes written on success, -1 on failure.
+ */
+int
+crypto_pk_asn1_encode_private(const crypto_pk_t *pk,
+                              char *dest, size_t destlen)
+{
+  tor_assert(destlen <= INT_MAX);
+  if (!crypto_pk_key_is_private(pk))
+    return -1;
+
+  SECKEYPrivateKeyInfo *info = PK11_ExportPrivKeyInfo(pk->seckey, NULL);
+  if (!info)
+    return -1;
+  SECItem *item = &info->privateKey;
+
+  if (destlen < item->len) {
+    SECKEY_DestroyPrivateKeyInfo(info, PR_TRUE);
+    return -1;
+  }
+  int result = (int)item->len;
+  memcpy(dest, item->data, item->len);
+  SECKEY_DestroyPrivateKeyInfo(info, PR_TRUE);
+
+  return result;
+}
+
+/** Given a buffer containing the DER representation of the
+ * private key <b>str</b>, decode and return the result on success, or NULL
+ * on failure.
+ */
+crypto_pk_t *
+crypto_pk_asn1_decode_private(const char *str, size_t len)
+{
+  tor_assert(str);
+  tor_assert(len < INT_MAX);
+  PK11SlotInfo *slot = PK11_GetBestSlot(CKM_RSA_PKCS, NULL);
+  if (!slot)
+    return NULL;
+
+  SECKEYPrivateKeyInfo info = {
+             .algorithm = {
+                           .algorithm = {
+                                         .type = siBuffer,
+                                         .data = (unsigned char *)RSA_OID,
+                                         .len = sizeof(RSA_OID)
+                                           }
+                           },
+             .privateKey = {
+                            .type = siBuffer,
+                            .data = (unsigned char *)str,
+                            .len = (int)len,
+                            }
+  };
+
+  SECStatus s;
+  SECKEYPrivateKey *seckey = NULL;
+
+  s = PK11_ImportPrivateKeyInfoAndReturnKey(slot, &info,
+                                            NULL /* nickname */,
+                                            NULL /* publicValue */,
+                                            PR_FALSE /* isPerm */,
+                                            PR_FALSE /* isPrivate */,
+                                            KU_ALL /* keyUsage */,
+                                            &seckey, NULL);
+
+  crypto_pk_t *output = NULL;
+
+  if (s == SECSuccess && seckey) {
+    output = crypto_pk_new();
+    output->seckey = seckey;
+    output->pubkey = SECKEY_ConvertToPublicKey(seckey);
+    tor_assert(output->pubkey);
+  } else {
+    crypto_nss_log_errors(LOG_WARN, "decoding an RSA private key");
+  }
+
+  if (! crypto_pk_is_valid_private_key(output)) {
+    crypto_pk_free(output);
+    output = NULL;
+  }
+
+  if (slot)
+    PK11_FreeSlot(slot);
+
+  return output;
+}
diff --git a/src/lib/crypt_ops/crypto_rsa_openssl.c b/src/lib/crypt_ops/crypto_rsa_openssl.c
new file mode 100644
index 0000000000..fbdc76ccd6
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_rsa_openssl.c
@@ -0,0 +1,590 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_rsa.c
+ * \brief OpenSSL implementations of our RSA code.
+ **/
+
+#include "lib/crypt_ops/compat_openssl.h"
+#include "lib/crypt_ops/crypto_rsa.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/log/util_bug.h"
+#include "lib/fs/files.h"
+
+DISABLE_GCC_WARNING(redundant-decls)
+
+#include <openssl/err.h>
+#include <openssl/rsa.h>
+#include <openssl/pem.h>
+#include <openssl/evp.h>
+#include <openssl/engine.h>
+#include <openssl/rand.h>
+#include <openssl/bn.h>
+#include <openssl/conf.h>
+
+ENABLE_GCC_WARNING(redundant-decls)
+
+#include "lib/log/log.h"
+#include "lib/encoding/binascii.h"
+
+#include <string.h>
+
+/** Declaration for crypto_pk_t structure. */
+struct crypto_pk_t
+{
+  int refs; /**< reference count, so we don't have to copy keys */
+  RSA *key; /**< The key itself */
+};
+
+/** Return true iff <b>key</b> contains the private-key portion of the RSA
+ * key. */
+int
+crypto_pk_key_is_private(const crypto_pk_t *k)
+{
+#ifdef OPENSSL_1_1_API
+  if (!k || !k->key)
+    return 0;
+
+  const BIGNUM *p, *q;
+  RSA_get0_factors(k->key, &p, &q);
+  return p != NULL; /* XXX/yawning: Should we check q? */
+#else /* !(defined(OPENSSL_1_1_API)) */
+  return k && k->key && k->key->p;
+#endif /* defined(OPENSSL_1_1_API) */
+}
+
+/** used by tortls.c: wrap an RSA* in a crypto_pk_t. Takes ownership of
+ * its argument. */
+crypto_pk_t *
+crypto_new_pk_from_openssl_rsa_(RSA *rsa)
+{
+  crypto_pk_t *env;
+  tor_assert(rsa);
+  env = tor_malloc(sizeof(crypto_pk_t));
+  env->refs = 1;
+  env->key = rsa;
+  return env;
+}
+
+/** Helper, used by tor-gencert.c.  Return a copy of the private RSA from a
+ * crypto_pk_t. */
+RSA *
+crypto_pk_get_openssl_rsa_(crypto_pk_t *env)
+{
+  return RSAPrivateKey_dup(env->key);
+}
+
+/** used by tortls.c: get an equivalent EVP_PKEY* for a crypto_pk_t.  Iff
+ * private is set, include the private-key portion of the key. Return a valid
+ * pointer on success, and NULL on failure. */
+MOCK_IMPL(EVP_PKEY *,
+crypto_pk_get_openssl_evp_pkey_,(crypto_pk_t *env, int private))
+{
+  RSA *key = NULL;
+  EVP_PKEY *pkey = NULL;
+  tor_assert(env->key);
+  if (private) {
+    if (!(key = RSAPrivateKey_dup(env->key)))
+      goto error;
+  } else {
+    if (!(key = RSAPublicKey_dup(env->key)))
+      goto error;
+  }
+  if (!(pkey = EVP_PKEY_new()))
+    goto error;
+  if (!(EVP_PKEY_assign_RSA(pkey, key)))
+    goto error;
+  return pkey;
+ error:
+  if (pkey)
+    EVP_PKEY_free(pkey);
+  if (key)
+    RSA_free(key);
+  return NULL;
+}
+
+/** Allocate and return storage for a public key.  The key itself will not yet
+ * be set.
+ */
+MOCK_IMPL(crypto_pk_t *,
+crypto_pk_new,(void))
+{
+  RSA *rsa;
+
+  rsa = RSA_new();
+  tor_assert(rsa);
+  return crypto_new_pk_from_openssl_rsa_(rsa);
+}
+
+/** Release a reference to an asymmetric key; when all the references
+ * are released, free the key.
+ */
+void
+crypto_pk_free_(crypto_pk_t *env)
+{
+  if (!env)
+    return;
+
+  if (--env->refs > 0)
+    return;
+  tor_assert(env->refs == 0);
+
+  if (env->key)
+    RSA_free(env->key);
+
+  tor_free(env);
+}
+
+/** Generate a <b>bits</b>-bit new public/private keypair in <b>env</b>.
+ * Return 0 on success, -1 on failure.
+ */
+MOCK_IMPL(int,
+crypto_pk_generate_key_with_bits,(crypto_pk_t *env, int bits))
+{
+  tor_assert(env);
+
+  if (env->key) {
+    RSA_free(env->key);
+    env->key = NULL;
+  }
+
+  {
+    BIGNUM *e = BN_new();
+    RSA *r = NULL;
+    if (!e)
+      goto done;
+    if (! BN_set_word(e, TOR_RSA_EXPONENT))
+      goto done;
+    r = RSA_new();
+    if (!r)
+      goto done;
+    if (RSA_generate_key_ex(r, bits, e, NULL) == -1)
+      goto done;
+
+    env->key = r;
+    r = NULL;
+  done:
+    if (e)
+      BN_clear_free(e);
+    if (r)
+      RSA_free(r);
+  }
+
+  if (!env->key) {
+    crypto_openssl_log_errors(LOG_WARN, "generating RSA key");
+    return -1;
+  }
+
+  return 0;
+}
+
+/** Return true if <b>env</b> has a valid key; false otherwise.
+ */
+int
+crypto_pk_is_valid_private_key(const crypto_pk_t *env)
+{
+  int r;
+  tor_assert(env);
+
+  r = RSA_check_key(env->key);
+  if (r <= 0) {
+    crypto_openssl_log_errors(LOG_WARN,"checking RSA key");
+    return 0;
+  } else {
+    return 1;
+  }
+}
+
+/** Return true iff <b>env</b> contains a public key whose public exponent
+ * equals TOR_RSA_EXPONENT.
+ */
+int
+crypto_pk_public_exponent_ok(const crypto_pk_t *env)
+{
+  tor_assert(env);
+  tor_assert(env->key);
+
+  const BIGNUM *e;
+
+#ifdef OPENSSL_1_1_API
+  const BIGNUM *n, *d;
+  RSA_get0_key(env->key, &n, &e, &d);
+#else
+  e = env->key->e;
+#endif /* defined(OPENSSL_1_1_API) */
+  return BN_is_word(e, TOR_RSA_EXPONENT);
+}
+
+/** Compare the public-key components of a and b.  Return less than 0
+ * if a\<b, 0 if a==b, and greater than 0 if a\>b.  A NULL key is
+ * considered to be less than all non-NULL keys, and equal to itself.
+ *
+ * Note that this may leak information about the keys through timing.
+ */
+int
+crypto_pk_cmp_keys(const crypto_pk_t *a, const crypto_pk_t *b)
+{
+  int result;
+  char a_is_non_null = (a != NULL) && (a->key != NULL);
+  char b_is_non_null = (b != NULL) && (b->key != NULL);
+  char an_argument_is_null = !a_is_non_null | !b_is_non_null;
+
+  result = tor_memcmp(&a_is_non_null, &b_is_non_null, sizeof(a_is_non_null));
+  if (an_argument_is_null)
+    return result;
+
+  const BIGNUM *a_n, *a_e;
+  const BIGNUM *b_n, *b_e;
+
+#ifdef OPENSSL_1_1_API
+  const BIGNUM *a_d, *b_d;
+  RSA_get0_key(a->key, &a_n, &a_e, &a_d);
+  RSA_get0_key(b->key, &b_n, &b_e, &b_d);
+#else
+  a_n = a->key->n;
+  a_e = a->key->e;
+  b_n = b->key->n;
+  b_e = b->key->e;
+#endif /* defined(OPENSSL_1_1_API) */
+
+  tor_assert(a_n != NULL && a_e != NULL);
+  tor_assert(b_n != NULL && b_e != NULL);
+
+  result = BN_cmp(a_n, b_n);
+  if (result)
+    return result;
+  return BN_cmp(a_e, b_e);
+}
+
+/** Return the size of the public key modulus in <b>env</b>, in bytes. */
+size_t
+crypto_pk_keysize(const crypto_pk_t *env)
+{
+  tor_assert(env);
+  tor_assert(env->key);
+
+  return (size_t) RSA_size((RSA*)env->key);
+}
+
+/** Return the size of the public key modulus of <b>env</b>, in bits. */
+int
+crypto_pk_num_bits(crypto_pk_t *env)
+{
+  tor_assert(env);
+  tor_assert(env->key);
+
+#ifdef OPENSSL_1_1_API
+  /* It's so stupid that there's no other way to check that n is valid
+   * before calling RSA_bits().
+   */
+  const BIGNUM *n, *e, *d;
+  RSA_get0_key(env->key, &n, &e, &d);
+  tor_assert(n != NULL);
+
+  return RSA_bits(env->key);
+#else /* !(defined(OPENSSL_1_1_API)) */
+  tor_assert(env->key->n);
+  return BN_num_bits(env->key->n);
+#endif /* defined(OPENSSL_1_1_API) */
+}
+
+/** Increase the reference count of <b>env</b>, and return it.
+ */
+crypto_pk_t *
+crypto_pk_dup_key(crypto_pk_t *env)
+{
+  tor_assert(env);
+  tor_assert(env->key);
+
+  env->refs++;
+  return env;
+}
+
+/** Replace dest with src (private key only).  (Dest must have a refcount
+ * of 1)
+ */
+void
+crypto_pk_assign_private(crypto_pk_t *dest, const crypto_pk_t *src)
+{
+  tor_assert(dest);
+  tor_assert(dest->refs == 1);
+  tor_assert(src);
+  RSA_free(dest->key);
+  dest->key = RSAPrivateKey_dup(src->key);
+}
+
+/** Replace dest with src (public key only).  (Dest must have a refcount
+ * of 1)
+ */
+void
+crypto_pk_assign_public(crypto_pk_t *dest, const crypto_pk_t *src)
+{
+  tor_assert(dest);
+  tor_assert(dest->refs == 1);
+  tor_assert(src);
+  RSA_free(dest->key);
+  dest->key = RSAPublicKey_dup(src->key);
+}
+
+/** Make a real honest-to-goodness copy of <b>env</b>, and return it.
+ * Returns NULL on failure. */
+crypto_pk_t *
+crypto_pk_copy_full(crypto_pk_t *env)
+{
+  RSA *new_key;
+  int privatekey = 0;
+  tor_assert(env);
+  tor_assert(env->key);
+
+  if (crypto_pk_key_is_private(env)) {
+    new_key = RSAPrivateKey_dup(env->key);
+    privatekey = 1;
+  } else {
+    new_key = RSAPublicKey_dup(env->key);
+  }
+  if (!new_key) {
+    /* LCOV_EXCL_START
+     *
+     * We can't cause RSA*Key_dup() to fail, so we can't really test this.
+     */
+    log_err(LD_CRYPTO, "Unable to duplicate a %s key: openssl failed.",
+            privatekey?"private":"public");
+    crypto_openssl_log_errors(LOG_ERR,
+                      privatekey ? "Duplicating a private key" :
+                      "Duplicating a public key");
+    tor_fragile_assert();
+    return NULL;
+    /* LCOV_EXCL_STOP */
+  }
+
+  return crypto_new_pk_from_openssl_rsa_(new_key);
+}
+
+/** Encrypt <b>fromlen</b> bytes from <b>from</b> with the public key
+ * in <b>env</b>, using the padding method <b>padding</b>.  On success,
+ * write the result to <b>to</b>, and return the number of bytes
+ * written.  On failure, return -1.
+ *
+ * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
+ * at least the length of the modulus of <b>env</b>.
+ */
+int
+crypto_pk_public_encrypt(crypto_pk_t *env, char *to, size_t tolen,
+                         const char *from, size_t fromlen, int padding)
+{
+  int r;
+  tor_assert(env);
+  tor_assert(from);
+  tor_assert(to);
+  tor_assert(fromlen<INT_MAX);
+  tor_assert(tolen >= crypto_pk_keysize(env));
+
+  r = RSA_public_encrypt((int)fromlen,
+                         (unsigned char*)from, (unsigned char*)to,
+                         env->key, crypto_get_rsa_padding(padding));
+  if (r<0) {
+    crypto_openssl_log_errors(LOG_WARN, "performing RSA encryption");
+    return -1;
+  }
+  return r;
+}
+
+/** Decrypt <b>fromlen</b> bytes from <b>from</b> with the private key
+ * in <b>env</b>, using the padding method <b>padding</b>.  On success,
+ * write the result to <b>to</b>, and return the number of bytes
+ * written.  On failure, return -1.
+ *
+ * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
+ * at least the length of the modulus of <b>env</b>.
+ */
+int
+crypto_pk_private_decrypt(crypto_pk_t *env, char *to,
+                          size_t tolen,
+                          const char *from, size_t fromlen,
+                          int padding, int warnOnFailure)
+{
+  int r;
+  tor_assert(env);
+  tor_assert(from);
+  tor_assert(to);
+  tor_assert(env->key);
+  tor_assert(fromlen<INT_MAX);
+  tor_assert(tolen >= crypto_pk_keysize(env));
+  if (!crypto_pk_key_is_private(env))
+    /* Not a private key */
+    return -1;
+
+  r = RSA_private_decrypt((int)fromlen,
+                          (unsigned char*)from, (unsigned char*)to,
+                          env->key, crypto_get_rsa_padding(padding));
+
+  if (r<0) {
+    crypto_openssl_log_errors(warnOnFailure?LOG_WARN:LOG_DEBUG,
+                      "performing RSA decryption");
+    return -1;
+  }
+  return r;
+}
+
+/** Check the signature in <b>from</b> (<b>fromlen</b> bytes long) with the
+ * public key in <b>env</b>, using PKCS1 padding.  On success, write the
+ * signed data to <b>to</b>, and return the number of bytes written.
+ * On failure, return -1.
+ *
+ * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
+ * at least the length of the modulus of <b>env</b>.
+ */
+MOCK_IMPL(int,
+crypto_pk_public_checksig,(const crypto_pk_t *env, char *to,
+                           size_t tolen,
+                           const char *from, size_t fromlen))
+{
+  int r;
+  tor_assert(env);
+  tor_assert(from);
+  tor_assert(to);
+  tor_assert(fromlen < INT_MAX);
+  tor_assert(tolen >= crypto_pk_keysize(env));
+  r = RSA_public_decrypt((int)fromlen,
+                         (unsigned char*)from, (unsigned char*)to,
+                         env->key, RSA_PKCS1_PADDING);
+
+  if (r<0) {
+    crypto_openssl_log_errors(LOG_INFO, "checking RSA signature");
+    return -1;
+  }
+  return r;
+}
+
+/** Sign <b>fromlen</b> bytes of data from <b>from</b> with the private key in
+ * <b>env</b>, using PKCS1 padding.  On success, write the signature to
+ * <b>to</b>, and return the number of bytes written.  On failure, return
+ * -1.
+ *
+ * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
+ * at least the length of the modulus of <b>env</b>.
+ */
+int
+crypto_pk_private_sign(const crypto_pk_t *env, char *to, size_t tolen,
+                       const char *from, size_t fromlen)
+{
+  int r;
+  tor_assert(env);
+  tor_assert(from);
+  tor_assert(to);
+  tor_assert(fromlen < INT_MAX);
+  tor_assert(tolen >= crypto_pk_keysize(env));
+  if (!crypto_pk_key_is_private(env))
+    /* Not a private key */
+    return -1;
+
+  r = RSA_private_encrypt((int)fromlen,
+                          (unsigned char*)from, (unsigned char*)to,
+                          (RSA*)env->key, RSA_PKCS1_PADDING);
+  if (r<0) {
+    crypto_openssl_log_errors(LOG_WARN, "generating RSA signature");
+    return -1;
+  }
+  return r;
+}
+
+/** ASN.1-encode the public portion of <b>pk</b> into <b>dest</b>.
+ * Return -1 on error, or the number of characters used on success.
+ */
+int
+crypto_pk_asn1_encode(const crypto_pk_t *pk, char *dest, size_t dest_len)
+{
+  int len;
+  unsigned char *buf = NULL;
+
+  len = i2d_RSAPublicKey(pk->key, &buf);
+  if (len < 0 || buf == NULL)
+    return -1;
+
+  if ((size_t)len > dest_len || dest_len > SIZE_T_CEILING) {
+    OPENSSL_free(buf);
+    return -1;
+  }
+  /* We don't encode directly into 'dest', because that would be illegal
+   * type-punning.  (C99 is smarter than me, C99 is smarter than me...)
+   */
+  memcpy(dest,buf,len);
+  OPENSSL_free(buf);
+  return len;
+}
+
+/** Decode an ASN.1-encoded public key from <b>str</b>; return the result on
+ * success and NULL on failure.
+ */
+crypto_pk_t *
+crypto_pk_asn1_decode(const char *str, size_t len)
+{
+  RSA *rsa;
+  unsigned char *buf;
+  const unsigned char *cp;
+  cp = buf = tor_malloc(len);
+  memcpy(buf,str,len);
+  rsa = d2i_RSAPublicKey(NULL, &cp, len);
+  tor_free(buf);
+  if (!rsa) {
+    crypto_openssl_log_errors(LOG_WARN,"decoding public key");
+    return NULL;
+  }
+  return crypto_new_pk_from_openssl_rsa_(rsa);
+}
+
+/** ASN.1-encode the private portion of <b>pk</b> into <b>dest</b>.
+ * Return -1 on error, or the number of characters used on success.
+ */
+int
+crypto_pk_asn1_encode_private(const crypto_pk_t *pk, char *dest,
+                              size_t dest_len)
+{
+  int len;
+  unsigned char *buf = NULL;
+
+  len = i2d_RSAPrivateKey(pk->key, &buf);
+  if (len < 0 || buf == NULL)
+    return -1;
+
+  if ((size_t)len > dest_len || dest_len > SIZE_T_CEILING) {
+    OPENSSL_free(buf);
+    return -1;
+  }
+  /* We don't encode directly into 'dest', because that would be illegal
+   * type-punning.  (C99 is smarter than me, C99 is smarter than me...)
+   */
+  memcpy(dest,buf,len);
+  OPENSSL_free(buf);
+  return len;
+}
+
+/** Decode an ASN.1-encoded private key from <b>str</b>; return the result on
+ * success and NULL on failure.
+ */
+crypto_pk_t *
+crypto_pk_asn1_decode_private(const char *str, size_t len)
+{
+  RSA *rsa;
+  unsigned char *buf;
+  const unsigned char *cp;
+  cp = buf = tor_malloc(len);
+  memcpy(buf,str,len);
+  rsa = d2i_RSAPrivateKey(NULL, &cp, len);
+  tor_free(buf);
+  if (!rsa) {
+    crypto_openssl_log_errors(LOG_WARN,"decoding public key");
+    return NULL;
+  }
+  crypto_pk_t *result = crypto_new_pk_from_openssl_rsa_(rsa);
+  if (! crypto_pk_is_valid_private_key(result)) {
+    crypto_pk_free(result);
+    return NULL;
+  }
+  return result;
+}
diff --git a/src/common/crypto_s2k.c b/src/lib/crypt_ops/crypto_s2k.c
index 5dbd2ad91f..42276597d4 100644
--- a/src/common/crypto_s2k.c
+++ b/src/lib/crypt_ops/crypto_s2k.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,18 +12,30 @@
 
 #define CRYPTO_S2K_PRIVATE
 
-#include "crypto.h"
-#include "util.h"
-#include "compat.h"
-#include "crypto_s2k.h"
-
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_hkdf.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_s2k.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/log/util_bug.h"
+#include "lib/intmath/cmp.h"
+
+#ifdef ENABLE_OPENSSL
 #include <openssl/evp.h>
+#endif
+#ifdef ENABLE_NSS
+#include <pk11pub.h>
+#endif
 
 #if defined(HAVE_LIBSCRYPT_H) && defined(HAVE_LIBSCRYPT_SCRYPT)
 #define HAVE_SCRYPT
 #include <libscrypt.h>
 #endif
 
+#include <string.h>
+
 /* Encoded secrets take the form:
 
      u8 type;
@@ -86,9 +98,11 @@ secret_to_key_key_len(uint8_t type)
       return DIGEST_LEN;
     case S2K_TYPE_SCRYPT:
       return DIGEST256_LEN;
+    // LCOV_EXCL_START
     default:
-      tor_fragile_assert(); // LCOV_EXCL_LINE
-      return -1;            // LCOV_EXCL_LINE
+      tor_fragile_assert();
+      return -1;
+    // LCOV_EXCL_STOP
   }
 }
 
@@ -169,9 +183,11 @@ make_specifier(uint8_t *spec_out, uint8_t type, unsigned flags)
       /* r = 8; p = 2. */
       spec_out[SCRYPT_SPEC_LEN-1] = (3u << 4) | (1u << 0);
       break;
+    // LCOV_EXCL_START - we should have returned above.
     default:
-      tor_fragile_assert(); // LCOV_EXCL_LINE - we should have returned above.
+      tor_fragile_assert();
       return S2K_BAD_ALGORITHM;
+    // LCOV_EXCL_STOP
   }
 
   return speclen;
@@ -261,6 +277,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len,
       log_iters = spec[spec_len-1];
       if (log_iters > 31)
         return S2K_BAD_PARAMS;
+#ifdef ENABLE_OPENSSL
       rv = PKCS5_PBKDF2_HMAC_SHA1(secret, (int)secret_len,
                                   spec, (int)spec_len-1,
                                   (1<<log_iters),
@@ -268,6 +285,47 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len,
       if (rv < 0)
         return S2K_FAILED;
       return (int)key_out_len;
+#else
+      SECItem passItem = { .type = siBuffer,
+                           .data = (unsigned char *) secret,
+                           .len = (int)secret_len };
+      SECItem saltItem = { .type = siBuffer,
+                           .data = (unsigned char *) spec,
+                           .len = (int)spec_len - 1 };
+      SECAlgorithmID *alg = NULL;
+      PK11SymKey *key = NULL;
+
+      rv = S2K_FAILED;
+      alg = PK11_CreatePBEV2AlgorithmID(
+                  SEC_OID_PKCS5_PBKDF2, SEC_OID_HMAC_SHA1, SEC_OID_HMAC_SHA1,
+                  (int)key_out_len, (1<<log_iters), &saltItem);
+      if (alg == NULL)
+        return S2K_FAILED;
+
+      key = PK11_PBEKeyGen(NULL /* slot */,
+                           alg,
+                           &passItem,
+                           false,
+                           NULL);
+
+      SECStatus st = PK11_ExtractKeyValue(key);
+      if (st != SECSuccess)
+        goto nss_pbkdf_err;
+
+      const SECItem *iptr = PK11_GetKeyData(key);
+      if (iptr == NULL)
+        goto nss_pbkdf_err;
+
+      rv = MIN((int)iptr->len, (int)key_out_len);
+      memcpy(key_out, iptr->data, rv);
+
+    nss_pbkdf_err:
+      if (key)
+        PK11_FreeSymKey(key);
+      if (alg)
+        SECOID_DestroyAlgorithmID(alg, PR_TRUE);
+      return rv;
+#endif
     }
 
     case S2K_TYPE_SCRYPT: {
@@ -290,9 +348,9 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len,
       if (rv != 0)
         return S2K_FAILED;
       return (int)key_out_len;
-#else
+#else /* !(defined(HAVE_SCRYPT)) */
       return S2K_NO_SCRYPT_SUPPORT;
-#endif
+#endif /* defined(HAVE_SCRYPT) */
     }
     default:
       return S2K_BAD_ALGORITHM;
@@ -465,4 +523,3 @@ secret_to_key_check(const uint8_t *spec_and_key, size_t spec_and_key_len,
   memwipe(buf, 0, sizeof(buf));
   return rv;
 }
-
diff --git a/src/common/crypto_s2k.h b/src/lib/crypt_ops/crypto_s2k.h
index 9b186450b1..a16a3d781e 100644
--- a/src/common/crypto_s2k.h
+++ b/src/lib/crypt_ops/crypto_s2k.h
@@ -1,14 +1,20 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * \file crypto_s2k.h
+ *
+ * \brief Header for crypto_s2k.c
+ **/
+
 #ifndef TOR_CRYPTO_S2K_H_INCLUDED
 #define TOR_CRYPTO_S2K_H_INCLUDED
 
 #include <stdio.h>
-#include "torint.h"
+#include "lib/cc/torint.h"
 
 /** Length of RFC2440-style S2K specifier: the first 8 bytes are a salt, the
  * 9th describes how much iteration to do. */
@@ -67,7 +73,6 @@ STATIC int secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len,
                                      const uint8_t *spec, size_t spec_len,
                                      const char *secret, size_t secret_len,
                                      int type);
-#endif
-
-#endif
+#endif /* defined(CRYPTO_S2K_PRIVATE) */
 
+#endif /* !defined(TOR_CRYPTO_S2K_H_INCLUDED) */
diff --git a/src/lib/crypt_ops/crypto_util.c b/src/lib/crypt_ops/crypto_util.c
new file mode 100644
index 0000000000..67a1a9eb92
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_util.c
@@ -0,0 +1,111 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_util.c
+ *
+ * \brief Common cryptographic utilities.
+ **/
+
+#define CRYPTO_UTIL_PRIVATE
+
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/cc/compat_compiler.h"
+
+#include <string.h>
+
+#ifdef _WIN32
+#include <winsock2.h>
+#include <windows.h>
+#include <wincrypt.h>
+#endif /* defined(_WIN32) */
+
+#include <stdlib.h>
+
+#ifdef ENABLE_OPENSSL
+DISABLE_GCC_WARNING(redundant-decls)
+#include <openssl/err.h>
+#include <openssl/crypto.h>
+ENABLE_GCC_WARNING(redundant-decls)
+#endif
+
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+
+/**
+ * Destroy the <b>sz</b> bytes of data stored at <b>mem</b>, setting them to
+ * the value <b>byte</b>.
+ * If <b>mem</b> is NULL or <b>sz</b> is zero, nothing happens.
+ *
+ * This function is preferable to memset, since many compilers will happily
+ * optimize out memset() when they can convince themselves that the data being
+ * cleared will never be read.
+ *
+ * Right now, our convention is to use this function when we are wiping data
+ * that's about to become inaccessible, such as stack buffers that are about
+ * to go out of scope or structures that are about to get freed.  (In
+ * practice, it appears that the compilers we're currently using will optimize
+ * out the memset()s for stack-allocated buffers, but not those for
+ * about-to-be-freed structures. That could change, though, so we're being
+ * wary.)  If there are live reads for the data, then you can just use
+ * memset().
+ */
+void
+memwipe(void *mem, uint8_t byte, size_t sz)
+{
+  if (sz == 0) {
+    return;
+  }
+  /* If sz is nonzero, then mem must not be NULL. */
+  tor_assert(mem != NULL);
+
+  /* Data this large is likely to be an underflow. */
+  tor_assert(sz < SIZE_T_CEILING);
+
+  /* Because whole-program-optimization exists, we may not be able to just
+   * have this function call "memset".  A smart compiler could inline it, then
+   * eliminate dead memsets, and declare itself to be clever. */
+
+#if defined(SecureZeroMemory) || defined(HAVE_SECUREZEROMEMORY)
+  /* Here's what you do on windows. */
+  SecureZeroMemory(mem,sz);
+#elif defined(HAVE_RTLSECUREZEROMEMORY)
+  RtlSecureZeroMemory(mem,sz);
+#elif defined(HAVE_EXPLICIT_BZERO)
+  /* The BSDs provide this. */
+  explicit_bzero(mem, sz);
+#elif defined(HAVE_MEMSET_S)
+  /* This is in the C99 standard. */
+  memset_s(mem, sz, 0, sz);
+#elif defined(ENABLE_OPENSSL)
+  /* This is a slow and ugly function from OpenSSL that fills 'mem' with junk
+   * based on the pointer value, then uses that junk to update a global
+   * variable.  It's an elaborate ruse to trick the compiler into not
+   * optimizing out the "wipe this memory" code.  Read it if you like zany
+   * programming tricks! In later versions of Tor, we should look for better
+   * not-optimized-out memory wiping stuff...
+   *
+   * ...or maybe not.  In practice, there are pure-asm implementations of
+   * OPENSSL_cleanse() on most platforms, which ought to do the job.
+   **/
+
+  OPENSSL_cleanse(mem, sz);
+#else
+  memset(mem, 0, sz);
+  asm volatile("" ::: "memory");
+#endif /* defined(SecureZeroMemory) || defined(HAVE_SECUREZEROMEMORY) || ... */
+
+  /* Just in case some caller of memwipe() is relying on getting a buffer
+   * filled with a particular value, fill the buffer.
+   *
+   * If this function gets inlined, this memset might get eliminated, but
+   * that's okay: We only care about this particular memset in the case where
+   * the caller should have been using memset(), and the memset() wouldn't get
+   * eliminated.  In other words, this is here so that we won't break anything
+   * if somebody accidentally calls memwipe() instead of memset().
+   **/
+  memset(mem, byte, sz);
+}
diff --git a/src/lib/crypt_ops/crypto_util.h b/src/lib/crypt_ops/crypto_util.h
new file mode 100644
index 0000000000..613a1bd0dd
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_util.h
@@ -0,0 +1,21 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_util.h
+ *
+ * \brief Common functions for cryptographic routines.
+ **/
+
+#ifndef TOR_CRYPTO_UTIL_H
+#define TOR_CRYPTO_UTIL_H
+
+#include "lib/cc/torint.h"
+
+/** OpenSSL-based utility functions. */
+void memwipe(void *mem, uint8_t byte, size_t sz);
+
+#endif /* !defined(TOR_CRYPTO_UTIL_H) */
diff --git a/src/lib/crypt_ops/digestset.c b/src/lib/crypt_ops/digestset.c
new file mode 100644
index 0000000000..0dba64d595
--- /dev/null
+++ b/src/lib/crypt_ops/digestset.c
@@ -0,0 +1,58 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file digestset.c
+ * \brief Implementation for a set of digests
+ **/
+
+#include "orconfig.h"
+#include "lib/container/bloomfilt.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/defs/digest_sizes.h"
+#include "lib/crypt_ops/digestset.h"
+#include "siphash.h"
+
+/* Wrap our hash function to have the signature that the bloom filter
+ * needs. */
+static uint64_t
+bloomfilt_digest_hash(const struct sipkey *key,
+                      const void *item)
+{
+  return siphash24(item, DIGEST_LEN, key);
+}
+
+/**
+ * Allocate and return an digestset, suitable for holding up to
+ * <b>max_guess</b> distinct values.
+ */
+digestset_t *
+digestset_new(int max_guess)
+{
+  uint8_t k[BLOOMFILT_KEY_LEN];
+  crypto_rand((void*)k, sizeof(k));
+  return bloomfilt_new(max_guess, bloomfilt_digest_hash, k);
+}
+
+/**
+ * Add <b>digest</b> to <b>set</b>.
+ *
+ * All future queries for <b>digest</b> in set will return true. Removing
+ * items is not possible.
+ */
+void
+digestset_add(digestset_t *set, const char *digest)
+{
+  bloomfilt_add(set, digest);
+}
+
+/**
+ * Return true if <b>digest</b> is a member of <b>set</b>.  (And probably,
+ * return false if <b>digest</b> is not a member of set.)
+ */
+int
+digestset_probably_contains(const digestset_t *set,
+                            const char *digest)
+{
+  return bloomfilt_probably_contains(set, digest);
+}
diff --git a/src/lib/crypt_ops/digestset.h b/src/lib/crypt_ops/digestset.h
new file mode 100644
index 0000000000..91d53a0542
--- /dev/null
+++ b/src/lib/crypt_ops/digestset.h
@@ -0,0 +1,29 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file digestset.h
+ * \brief Types to handle sets of digests, based on bloom filters.
+ **/
+
+#ifndef TOR_DIGESTSET_H
+#define TOR_DIGESTSET_H
+
+#include "orconfig.h"
+#include "lib/cc/torint.h"
+#include "lib/container/bloomfilt.h"
+
+/**
+ * An digestset_t represents a set of 20-byte digest values. The
+ * implementation is probabilistic: false negatives cannot occur but false
+ * positives are possible.
+ */
+typedef struct bloomfilt_t digestset_t;
+
+digestset_t *digestset_new(int max_addresses_guess);
+#define digestset_free(set) bloomfilt_free(set)
+void digestset_add(digestset_t *set, const char *addr);
+int digestset_probably_contains(const digestset_t *set,
+                                const char *addr);
+
+#endif
diff --git a/src/lib/crypt_ops/include.am b/src/lib/crypt_ops/include.am
new file mode 100644
index 0000000000..1022096fdc
--- /dev/null
+++ b/src/lib/crypt_ops/include.am
@@ -0,0 +1,70 @@
+
+noinst_LIBRARIES += src/lib/libtor-crypt-ops.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-crypt-ops-testing.a
+endif
+
+src_lib_libtor_crypt_ops_a_SOURCES =			\
+	src/lib/crypt_ops/crypto_cipher.c		\
+	src/lib/crypt_ops/crypto_curve25519.c		\
+	src/lib/crypt_ops/crypto_dh.c			\
+	src/lib/crypt_ops/crypto_digest.c		\
+	src/lib/crypt_ops/crypto_ed25519.c		\
+	src/lib/crypt_ops/crypto_format.c		\
+	src/lib/crypt_ops/crypto_hkdf.c			\
+	src/lib/crypt_ops/crypto_init.c			\
+	src/lib/crypt_ops/crypto_ope.c          	\
+	src/lib/crypt_ops/crypto_pwbox.c		\
+	src/lib/crypt_ops/crypto_rand.c			\
+	src/lib/crypt_ops/crypto_rsa.c			\
+	src/lib/crypt_ops/crypto_s2k.c			\
+	src/lib/crypt_ops/crypto_util.c                 \
+	src/lib/crypt_ops/digestset.c
+
+if USE_NSS
+src_lib_libtor_crypt_ops_a_SOURCES +=			\
+	src/lib/crypt_ops/aes_nss.c			\
+	src/lib/crypt_ops/crypto_dh_nss.c		\
+	src/lib/crypt_ops/crypto_nss_mgt.c		\
+	src/lib/crypt_ops/crypto_rsa_nss.c
+else
+src_lib_libtor_crypt_ops_a_SOURCES +=			\
+	src/lib/crypt_ops/aes_openssl.c			\
+	src/lib/crypt_ops/crypto_rsa_openssl.c
+endif
+
+if USE_OPENSSL
+src_lib_libtor_crypt_ops_a_SOURCES +=			\
+	src/lib/crypt_ops/crypto_dh_openssl.c		\
+	src/lib/crypt_ops/crypto_openssl_mgt.c
+endif
+
+src_lib_libtor_crypt_ops_a_CFLAGS = $(AM_CFLAGS) $(TOR_CFLAGS_CRYPTLIB)
+
+src_lib_libtor_crypt_ops_testing_a_SOURCES = \
+	$(src_lib_libtor_crypt_ops_a_SOURCES)
+src_lib_libtor_crypt_ops_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_crypt_ops_testing_a_CFLAGS = \
+	$(AM_CFLAGS) $(TOR_CFLAGS_CRYPTLIB) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/crypt_ops/aes.h				\
+	src/lib/crypt_ops/compat_openssl.h		\
+	src/lib/crypt_ops/crypto_curve25519.h		\
+	src/lib/crypt_ops/crypto_dh.h			\
+	src/lib/crypt_ops/crypto_digest.h		\
+	src/lib/crypt_ops/crypto_ed25519.h		\
+	src/lib/crypt_ops/crypto_format.h		\
+	src/lib/crypt_ops/crypto_cipher.h		\
+	src/lib/crypt_ops/crypto_hkdf.h			\
+	src/lib/crypt_ops/crypto_init.h			\
+	src/lib/crypt_ops/crypto_nss_mgt.h		\
+	src/lib/crypt_ops/crypto_openssl_mgt.h		\
+	src/lib/crypt_ops/crypto_ope.h          	\
+	src/lib/crypt_ops/crypto_pwbox.h		\
+	src/lib/crypt_ops/crypto_rand.h			\
+	src/lib/crypt_ops/crypto_rsa.h			\
+	src/lib/crypt_ops/crypto_s2k.h			\
+	src/lib/crypt_ops/crypto_util.h                 \
+	src/lib/crypt_ops/digestset.h
diff --git a/src/lib/ctime/.may_include b/src/lib/ctime/.may_include
new file mode 100644
index 0000000000..e74669bce1
--- /dev/null
+++ b/src/lib/ctime/.may_include
@@ -0,0 +1,5 @@
+orconfig.h
+lib/cc/*.h
+lib/ctime/*.h
+lib/err/*.h
+lib/malloc/*.h
diff --git a/src/common/di_ops.c b/src/lib/ctime/di_ops.c
index 4ed49e1164..89e0837ae9 100644
--- a/src/common/di_ops.c
+++ b/src/lib/ctime/di_ops.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2016, The Tor Project, Inc. */
+/* Copyright (c) 2011-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -7,9 +7,11 @@
  **/
 
 #include "orconfig.h"
-#include "di_ops.h"
-#include "torlog.h"
-#include "util.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/err/torerr.h"
+#include "lib/malloc/malloc.h"
+
+#include <string.h>
 
 /**
  * Timing-safe version of memcmp.  As memcmp, compare the <b>sz</b> bytes at
@@ -21,6 +23,9 @@
  * This implementation differs from memcmp in that its timing behavior is not
  * data-dependent: it should return in the same amount of time regardless of
  * the contents of <b>a</b> and <b>b</b>.
+ *
+ * Note that if all you care about is equality, this implementation is
+ * overkill: it would be better to use tor_memeq() or tor_memneq().
  */
 int
 tor_memcmp(const void *a, const void *b, size_t len)
@@ -86,7 +91,7 @@ tor_memcmp(const void *a, const void *b, size_t len)
   }
 
   return retval;
-#endif /* timingsafe_memcmp */
+#endif /* defined(HAVE_TIMINGSAFE_MEMCMP) */
 }
 
 /**
@@ -148,7 +153,7 @@ struct di_digest256_map_t {
 /** Release all storage held in <b>map</b>, calling free_fn on each value
  * as we go. */
 void
-dimap_free(di_digest256_map_t *map, dimap_free_fn free_fn)
+dimap_free_(di_digest256_map_t *map, dimap_free_fn free_fn)
 {
   while (map) {
     di_digest256_map_t *victim = map;
@@ -171,8 +176,8 @@ dimap_add_entry(di_digest256_map_t **map,
   di_digest256_map_t *new_ent;
   {
     void *old_val = dimap_search(*map, key, NULL);
-    tor_assert(! old_val);
-    tor_assert(val);
+    raw_assert(! old_val);
+    raw_assert(val);
   }
   new_ent = tor_malloc_zero(sizeof(di_digest256_map_t));
   new_ent->next = *map;
@@ -238,7 +243,7 @@ gt_i64_timei(uint64_t a, uint64_t b)
   int res = diff >> 63;
   return res & 1;
 }
-#endif
+#endif /* SIZEOF_VOID_P == 8 */
 
 /**
  * Given an array of list of <b>n_entries</b> uint64_t values, whose sum is
@@ -264,11 +269,10 @@ select_array_member_cumulative_timei(const uint64_t *entries, int n_entries,
       rand_val = INT64_MAX;
     }
   }
-  tor_assert(total_so_far == total);
-  tor_assert(n_chosen == 1);
-  tor_assert(i_chosen >= 0);
-  tor_assert(i_chosen < n_entries);
+  raw_assert(total_so_far == total);
+  raw_assert(n_chosen == 1);
+  raw_assert(i_chosen >= 0);
+  raw_assert(i_chosen < n_entries);
 
   return i_chosen;
 }
-
diff --git a/src/common/di_ops.h b/src/lib/ctime/di_ops.h
index 0a154302bf..264b56a8c1 100644
--- a/src/common/di_ops.h
+++ b/src/lib/ctime/di_ops.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,7 +12,7 @@
 #define TOR_DI_OPS_H
 
 #include "orconfig.h"
-#include "torint.h"
+#include "lib/cc/torint.h"
 
 int tor_memcmp(const void *a, const void *b, size_t sz);
 int tor_memeq(const void *a, const void *b, size_t sz);
@@ -37,7 +37,12 @@ int safe_mem_is_zero(const void *mem, size_t sz);
 typedef struct di_digest256_map_t di_digest256_map_t;
 typedef void (*dimap_free_fn)(void *);
 
-void dimap_free(di_digest256_map_t *map, dimap_free_fn free_fn);
+void dimap_free_(di_digest256_map_t *map, dimap_free_fn free_fn);
+#define dimap_free(map, free_fn)                \
+  do {                                          \
+    dimap_free_((map), (free_fn));              \
+    (map) = NULL;                               \
+  } while (0)
 void dimap_add_entry(di_digest256_map_t **map,
                      const uint8_t *key, void *val);
 void *dimap_search(const di_digest256_map_t *map, const uint8_t *key,
@@ -46,5 +51,5 @@ int select_array_member_cumulative_timei(const uint64_t *entries,
                                          int n_entries,
                                          uint64_t total, uint64_t rand_val);
 
-#endif
+#endif /* !defined(TOR_DI_OPS_H) */
 
diff --git a/src/lib/ctime/include.am b/src/lib/ctime/include.am
new file mode 100644
index 0000000000..b46c43ba0c
--- /dev/null
+++ b/src/lib/ctime/include.am
@@ -0,0 +1,25 @@
+
+noinst_LIBRARIES += src/lib/libtor-ctime.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-ctime-testing.a
+endif
+
+if ADD_MULODI4
+mulodi4_source=src/ext/mulodi/mulodi4.c
+else
+mulodi4_source=
+endif
+
+src_lib_libtor_ctime_a_SOURCES =			\
+	$(mulodi4_source) 				\
+	src/ext/csiphash.c   				\
+	src/lib/ctime/di_ops.c
+
+src_lib_libtor_ctime_testing_a_SOURCES = \
+	$(src_lib_libtor_ctime_a_SOURCES)
+src_lib_libtor_ctime_a_CFLAGS = @CFLAGS_CONSTTIME@
+src_lib_libtor_ctime_testing_a_CFLAGS = @CFLAGS_CONSTTIME@ $(TEST_CFLAGS)
+
+noinst_HEADERS += \
+	src/lib/ctime/di_ops.h
diff --git a/src/lib/defs/.may_include b/src/lib/defs/.may_include
new file mode 100644
index 0000000000..2b06e8519c
--- /dev/null
+++ b/src/lib/defs/.may_include
@@ -0,0 +1 @@
+orconfig.h
diff --git a/src/lib/defs/dh_sizes.h b/src/lib/defs/dh_sizes.h
new file mode 100644
index 0000000000..a2ffbc51c2
--- /dev/null
+++ b/src/lib/defs/dh_sizes.h
@@ -0,0 +1,22 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dh_sizes.h
+
+ * \brief Definitions for sizes of Diffie-Hellman groups elements in Z_p.
+ *
+ * Tor uses these definitions throughout its codebase, even in parts that
+ * don't actually do any Diffie-Hellman calculations.
+ **/
+
+#ifndef TOR_DH_SIZES_H
+#define TOR_DH_SIZES_H
+
+/** Length of our legacy DH keys. */
+#define DH1024_KEY_LEN (1024/8)
+
+#endif
diff --git a/src/lib/defs/digest_sizes.h b/src/lib/defs/digest_sizes.h
new file mode 100644
index 0000000000..525e5209d6
--- /dev/null
+++ b/src/lib/defs/digest_sizes.h
@@ -0,0 +1,27 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_DIGEST_SIZES_H
+#define TOR_DIGEST_SIZES_H
+
+/**
+ * \file digest_sizes.h
+ *
+ * \brief Definitions for common sizes of cryptographic digests.
+ *
+ * Tor uses digests throughout its codebase, even in parts that don't actually
+ * calculate the digests.
+ **/
+
+/** Length of the output of our message digest. */
+#define DIGEST_LEN 20
+/** Length of the output of our second (improved) message digests.  (For now
+ * this is just sha256, but it could be any other 256-bit digest.) */
+#define DIGEST256_LEN 32
+/** Length of the output of our 64-bit optimized message digests (SHA512). */
+#define DIGEST512_LEN 64
+
+#endif
diff --git a/src/lib/defs/include.am b/src/lib/defs/include.am
new file mode 100644
index 0000000000..48ee7f29fc
--- /dev/null
+++ b/src/lib/defs/include.am
@@ -0,0 +1,5 @@
+
+noinst_HEADERS += 			\
+	src/lib/defs/dh_sizes.h 	\
+	src/lib/defs/digest_sizes.h	\
+	src/lib/defs/x25519_sizes.h
diff --git a/src/lib/defs/x25519_sizes.h b/src/lib/defs/x25519_sizes.h
new file mode 100644
index 0000000000..8933a8866b
--- /dev/null
+++ b/src/lib/defs/x25519_sizes.h
@@ -0,0 +1,36 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file x25519_sizes.h
+
+ * \brief Definitions for sizes of x25519 keys and elements.
+ *
+ * Tor uses these definitions throughout its codebase, even in parts that
+ * don't actually do any x25519 calculations.
+ **/
+
+#ifndef TOR_X25519_SIZES_H
+#define TOR_X25519_SIZES_H
+
+/** Length of a curve25519 public key when encoded. */
+#define CURVE25519_PUBKEY_LEN 32
+/** Length of a curve25519 secret key when encoded. */
+#define CURVE25519_SECKEY_LEN 32
+/** Length of the result of a curve25519 handshake. */
+#define CURVE25519_OUTPUT_LEN 32
+
+#define ED25519_PUBKEY_LEN 32
+#define ED25519_SECKEY_LEN 64
+#define ED25519_SECKEY_SEED_LEN 32
+#define ED25519_SIG_LEN 64
+
+#define CURVE25519_BASE64_PADDED_LEN 44
+
+#define ED25519_BASE64_LEN 43
+#define ED25519_SIG_BASE64_LEN 86
+
+#endif
diff --git a/src/lib/encoding/.may_include b/src/lib/encoding/.may_include
new file mode 100644
index 0000000000..7c2ef36929
--- /dev/null
+++ b/src/lib/encoding/.may_include
@@ -0,0 +1,10 @@
+orconfig.h
+lib/cc/*.h
+lib/ctime/*.h
+lib/encoding/*.h
+lib/intmath/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/string/*.h
+lib/testsupport/*.h
+lib/wallclock/*.h
diff --git a/src/common/util_format.c b/src/lib/encoding/binascii.c
index aef9db85c8..bd063440d6 100644
--- a/src/common/util_format.c
+++ b/src/lib/encoding/binascii.c
@@ -1,34 +1,56 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file util_format.c
+ * \file binascii.c
  *
  * \brief Miscellaneous functions for encoding and decoding various things
  *   in base{16,32,64}.
  */
 
 #include "orconfig.h"
-#include "torlog.h"
-#include "util.h"
-#include "util_format.h"
-#include "torint.h"
+
+#include "lib/encoding/binascii.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/cc/torint.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/intmath/muldiv.h"
+#include "lib/malloc/malloc.h"
 
 #include <stddef.h>
 #include <string.h>
 #include <stdlib.h>
 
+/** Return a pointer to a NUL-terminated hexadecimal string encoding
+ * the first <b>fromlen</b> bytes of <b>from</b>. (fromlen must be \<= 32.) The
+ * result does not need to be deallocated, but repeated calls to
+ * hex_str will trash old results.
+ */
+const char *
+hex_str(const char *from, size_t fromlen)
+{
+  static char buf[65];
+  if (fromlen>(sizeof(buf)-1)/2)
+    fromlen = (sizeof(buf)-1)/2;
+  base16_encode(buf,sizeof(buf),from,fromlen);
+  return buf;
+}
+
 /* Return the base32 encoded size in bytes using the source length srclen.
- * The NUL terminated byte is added as well since every base32 encoding
- * requires enough space for it. */
+ *
+ * (WATCH OUT: This API counts the terminating NUL byte, but
+ * base64_encode_size does not.)
+ */
 size_t
 base32_encoded_size(size_t srclen)
 {
   size_t enclen;
-  enclen = CEIL_DIV(srclen*8, 5) + 1;
+  tor_assert(srclen < SIZE_T_CEILING / 8);
+  enclen = BASE32_NOPAD_BUFSIZE(srclen);
   tor_assert(enclen < INT_MAX && enclen > srclen);
   return enclen;
 }
@@ -41,7 +63,6 @@ base32_encode(char *dest, size_t destlen, const char *src, size_t srclen)
   size_t nbits = srclen * 8;
   size_t bit;
 
-  tor_assert(srclen < SIZE_T_CEILING/8);
   /* We need enough space for the encoded data and the extra NUL byte. */
   tor_assert(base32_encoded_size(srclen) <= destlen);
   tor_assert(destlen < SIZE_T_CEILING);
@@ -51,9 +72,10 @@ base32_encode(char *dest, size_t destlen, const char *src, size_t srclen)
 
   for (i=0,bit=0; bit < nbits; ++i, bit+=5) {
     /* set v to the 16-bit value starting at src[bits/8], 0-padded. */
-    v = ((uint8_t)src[bit/8]) << 8;
-    if (bit+5<nbits)
-      v += (uint8_t)src[(bit/8)+1];
+    size_t idx = bit / 8;
+    v = ((uint8_t)src[idx]) << 8;
+    if (idx+1 < srclen)
+      v += (uint8_t)src[idx+1];
     /* set u to the 5-bit value at the bit'th bit of buf. */
     u = (v >> (11-(bit%8))) & 0x1F;
     dest[i] = BASE32_CHARS[u];
@@ -133,6 +155,9 @@ base32_decode(char *dest, size_t destlen, const char *src, size_t srclen)
 /** Return the Base64 encoded size of <b>srclen</b> bytes of data in
  * bytes.
  *
+ * (WATCH OUT: This API <em>does not</em> count the terminating NUL byte,
+ * but base32_encoded_size does.)
+ *
  * If <b>flags</b>&amp;BASE64_ENCODE_MULTILINE is true, return the size
  * of the encoded output as multiline output (64 character, `\n' terminated
  * lines).
@@ -141,19 +166,16 @@ size_t
 base64_encode_size(size_t srclen, int flags)
 {
   size_t enclen;
+
+  /* Use INT_MAX for overflow checking because base64_encode() returns int. */
   tor_assert(srclen < INT_MAX);
+  tor_assert(CEIL_DIV(srclen, 3) < INT_MAX / 4);
 
-  if (srclen == 0)
-    return 0;
+  enclen = BASE64_LEN(srclen);
+  if (flags & BASE64_ENCODE_MULTILINE)
+    enclen += CEIL_DIV(enclen, BASE64_OPENSSL_LINELEN);
 
-  enclen = ((srclen - 1) / 3) * 4 + 4;
-  if (flags & BASE64_ENCODE_MULTILINE) {
-    size_t remainder = enclen % BASE64_OPENSSL_LINELEN;
-    enclen += enclen / BASE64_OPENSSL_LINELEN;
-    if (remainder)
-      enclen++;
-  }
-  tor_assert(enclen < INT_MAX && enclen > srclen);
+  tor_assert(enclen < INT_MAX && (enclen == 0 || enclen > srclen));
   return enclen;
 }
 
@@ -263,10 +285,13 @@ base64_encode(char *dest, size_t destlen, const char *src, size_t srclen,
     ENCODE_N(3);
     ENCODE_PAD();
     break;
+  // LCOV_EXCL_START -- we can't reach this point, because we enforce
+  // 0 <= ncov_idx < 3 in the loop above.
   default:
     /* Something went catastrophically wrong. */
-    tor_fragile_assert(); // LCOV_EXCL_LINE
+    tor_fragile_assert();
     return -1;
+  // LCOV_EXCL_STOP
   }
 
 #undef ENCODE_N
@@ -310,39 +335,6 @@ base64_encode_nopad(char *dest, size_t destlen,
   return (int)(out - dest);
 }
 
-/** As base64_decode, but do not require any padding on the input */
-int
-base64_decode_nopad(uint8_t *dest, size_t destlen,
-                    const char *src, size_t srclen)
-{
-  if (srclen > SIZE_T_CEILING - 4)
-    return -1;
-  char *buf = tor_malloc(srclen + 4);
-  memcpy(buf, src, srclen+1);
-  size_t buflen;
-  switch (srclen % 4)
-    {
-    case 0:
-    default:
-      buflen = srclen;
-      break;
-    case 1:
-      tor_free(buf);
-      return -1;
-    case 2:
-      memcpy(buf+srclen, "==", 3);
-      buflen = srclen + 2;
-      break;
-    case 3:
-      memcpy(buf+srclen, "=", 2);
-      buflen = srclen + 1;
-      break;
-  }
-  int n = base64_decode((char*)dest, destlen, buf, buflen);
-  tor_free(buf);
-  return n;
-}
-
 #undef BASE64_OPENSSL_LINELEN
 
 /** @{ */
@@ -392,15 +384,9 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen)
   const char *eos = src+srclen;
   uint32_t n=0;
   int n_idx=0;
-  char *dest_orig = dest;
+  size_t di = 0;
 
-  /* Max number of bits == srclen*6.
-   * Number of bytes required to hold all bits == (srclen*6)/8.
-   * Yes, we want to round down: anything that hangs over the end of a
-   * byte is padding. */
-  if (destlen < (srclen*3)/4)
-    return -1;
-  if (destlen > SIZE_T_CEILING)
+  if (destlen > INT_MAX)
     return -1;
 
   /* Make sure we leave no uninitialized data in the destination buffer. */
@@ -428,9 +414,11 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen)
         n = (n<<6) | v;
         if ((++n_idx) == 4) {
           /* We've accumulated 24 bits in n. Flush them. */
-          *dest++ = (n>>16);
-          *dest++ = (n>>8) & 0xff;
-          *dest++ = (n) & 0xff;
+          if (destlen < 3 || di > destlen - 3)
+            return -1;
+          dest[di++] = (n>>16);
+          dest[di++] = (n>>8) & 0xff;
+          dest[di++] = (n) & 0xff;
           n_idx = 0;
           n = 0;
         }
@@ -448,18 +436,21 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen)
       return -1;
     case 2:
       /* 12 leftover bits: The last 4 are padding and the first 8 are data. */
-      *dest++ = n >> 4;
+      if (destlen < 1 || di > destlen - 1)
+        return -1;
+      dest[di++] = n >> 4;
       break;
     case 3:
       /* 18 leftover bits: The last 2 are padding and the first 16 are data. */
-      *dest++ = n >> 10;
-      *dest++ = n >> 2;
+      if (destlen < 2 || di > destlen - 2)
+        return -1;
+      dest[di++] = n >> 10;
+      dest[di++] = n >> 2;
   }
 
-  tor_assert((dest-dest_orig) <= (ssize_t)destlen);
-  tor_assert((dest-dest_orig) <= INT_MAX);
+  tor_assert(di <= destlen);
 
-  return (int)(dest-dest_orig);
+  return (int)di;
 }
 #undef X
 #undef SP
@@ -475,7 +466,8 @@ base16_encode(char *dest, size_t destlen, const char *src, size_t srclen)
   const char *end;
   char *cp;
 
-  tor_assert(destlen >= srclen*2+1);
+  tor_assert(srclen < SIZE_T_CEILING / 2 - 1);
+  tor_assert(destlen >= BASE16_BUFSIZE(srclen));
   tor_assert(destlen < SIZE_T_CEILING);
 
   /* Make sure we leave no uninitialized data in the destination buffer. */
@@ -491,39 +483,6 @@ base16_encode(char *dest, size_t destlen, const char *src, size_t srclen)
   *cp = '\0';
 }
 
-/** Helper: given a hex digit, return its value, or -1 if it isn't hex. */
-static inline int
-hex_decode_digit_(char c)
-{
-  switch (c) {
-    case '0': return 0;
-    case '1': return 1;
-    case '2': return 2;
-    case '3': return 3;
-    case '4': return 4;
-    case '5': return 5;
-    case '6': return 6;
-    case '7': return 7;
-    case '8': return 8;
-    case '9': return 9;
-    case 'A': case 'a': return 10;
-    case 'B': case 'b': return 11;
-    case 'C': case 'c': return 12;
-    case 'D': case 'd': return 13;
-    case 'E': case 'e': return 14;
-    case 'F': case 'f': return 15;
-    default:
-      return -1;
-  }
-}
-
-/** Helper: given a hex digit, return its value, or -1 if it isn't hex. */
-int
-hex_decode_digit(char c)
-{
-  return hex_decode_digit_(c);
-}
-
 /** Given a hexadecimal string of <b>srclen</b> bytes in <b>src</b>, decode
  * it and store the result in the <b>destlen</b>-byte buffer at <b>dest</b>.
  * Return the number of bytes decoded on success, -1 on failure. If
@@ -546,8 +505,8 @@ base16_decode(char *dest, size_t destlen, const char *src, size_t srclen)
 
   end = src+srclen;
   while (src<end) {
-    v1 = hex_decode_digit_(*src);
-    v2 = hex_decode_digit_(*(src+1));
+    v1 = hex_decode_digit(*src);
+    v2 = hex_decode_digit(*(src+1));
     if (v1<0||v2<0)
       return -1;
     *(uint8_t*)dest = (v1<<4)|v2;
@@ -559,4 +518,3 @@ base16_decode(char *dest, size_t destlen, const char *src, size_t srclen)
 
   return (int) (dest-dest_orig);
 }
-
diff --git a/src/lib/encoding/binascii.h b/src/lib/encoding/binascii.h
new file mode 100644
index 0000000000..7e3cc04f09
--- /dev/null
+++ b/src/lib/encoding/binascii.h
@@ -0,0 +1,60 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file binascii.h
+ *
+ * \brief Header for binascii.c
+ **/
+
+#ifndef TOR_BINASCII_H
+#define TOR_BINASCII_H
+
+#include "orconfig.h"
+#include <stddef.h>
+#include "lib/testsupport/testsupport.h"
+#include "lib/cc/torint.h"
+
+const char *hex_str(const char *from, size_t fromlen);
+
+/** @{ */
+/** These macros don't check for overflow.  Use them only for constant inputs
+ * (like array declarations).  The *_LEN macros are the raw encoding lengths
+ * (without terminating NUL), while the *_BUFSIZE macros count the terminating
+ * NUL. */
+#define BASE64_LEN(n) (CEIL_DIV((n), 3) * 4)
+#define BASE32_LEN(n) (CEIL_DIV((n), 5) * 8)
+#define BASE16_LEN(n) ((n) * 2)
+
+#define BASE64_BUFSIZE(n) (BASE64_LEN(n) + 1)
+#define BASE32_BUFSIZE(n) (BASE32_LEN(n) + 1)
+#define BASE16_BUFSIZE(n) (BASE16_LEN(n) + 1)
+
+#define BASE64_NOPAD_LEN(n) (CEIL_DIV((n) * 4, 3))
+#define BASE32_NOPAD_LEN(n) (CEIL_DIV((n) * 8, 5))
+
+#define BASE64_NOPAD_BUFSIZE(n) (BASE64_NOPAD_LEN(n) + 1)
+#define BASE32_NOPAD_BUFSIZE(n) (BASE32_NOPAD_LEN(n) + 1)
+/** @} */
+
+#define BASE64_ENCODE_MULTILINE 1
+size_t base64_encode_size(size_t srclen, int flags);
+int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen,
+                  int flags);
+int base64_decode(char *dest, size_t destlen, const char *src, size_t srclen);
+int base64_encode_nopad(char *dest, size_t destlen,
+                        const uint8_t *src, size_t srclen);
+
+/** Characters that can appear (case-insensitively) in a base32 encoding. */
+#define BASE32_CHARS "abcdefghijklmnopqrstuvwxyz234567"
+void base32_encode(char *dest, size_t destlen, const char *src, size_t srclen);
+int base32_decode(char *dest, size_t destlen, const char *src, size_t srclen);
+size_t base32_encoded_size(size_t srclen);
+
+void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen);
+int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen);
+
+#endif /* !defined(TOR_UTIL_FORMAT_H) */
diff --git a/src/lib/encoding/confline.c b/src/lib/encoding/confline.c
new file mode 100644
index 0000000000..8110f3dd9c
--- /dev/null
+++ b/src/lib/encoding/confline.c
@@ -0,0 +1,402 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file confline.c
+ *
+ * \brief Functions to manipulate a linked list of key-value pairs, of the
+ *   type used in Tor's configuration files.
+ *
+ * Tor uses the config_line_t type and its associated serialized format for
+ * human-readable key-value pairs in many places, including its configuration,
+ * its state files, its consensus cache, and so on.
+ **/
+
+#include "lib/encoding/confline.h"
+#include "lib/encoding/cstring.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/string/compat_string.h"
+#include "lib/string/util_string.h"
+
+#include <string.h>
+
+/** Helper: allocate a new configuration option mapping 'key' to 'val',
+ * append it to *<b>lst</b>. */
+void
+config_line_append(config_line_t **lst,
+                   const char *key,
+                   const char *val)
+{
+  tor_assert(lst);
+
+  config_line_t *newline;
+
+  newline = tor_malloc_zero(sizeof(config_line_t));
+  newline->key = tor_strdup(key);
+  newline->value = tor_strdup(val);
+  newline->next = NULL;
+  while (*lst)
+    lst = &((*lst)->next);
+
+  (*lst) = newline;
+}
+
+/** Helper: allocate a new configuration option mapping 'key' to 'val',
+ * and prepend it to *<b>lst</b> */
+void
+config_line_prepend(config_line_t **lst,
+                    const char *key,
+                    const char *val)
+{
+  tor_assert(lst);
+
+  config_line_t *newline;
+
+  newline = tor_malloc_zero(sizeof(config_line_t));
+  newline->key = tor_strdup(key);
+  newline->value = tor_strdup(val);
+  newline->next = *lst;
+  *lst = newline;
+}
+
+/** Return the first line in <b>lines</b> whose key is exactly <b>key</b>, or
+ * NULL if no such key exists.
+ *
+ * (In options parsing, this is for handling commandline-only options only;
+ * other options should be looked up in the appropriate data structure.) */
+const config_line_t *
+config_line_find(const config_line_t *lines,
+                 const char *key)
+{
+  const config_line_t *cl;
+  for (cl = lines; cl; cl = cl->next) {
+    if (!strcmp(cl->key, key))
+      return cl;
+  }
+  return NULL;
+}
+
+/** Auxiliary function that does all the work of config_get_lines.
+ * <b>recursion_level</b> is the count of how many nested %includes we have.
+ * <b>opened_lst</b> will have a list of opened files if provided.
+ * Returns the a pointer to the last element of the <b>result</b> in
+ * <b>last</b>. */
+int
+config_get_lines_aux(const char *string, config_line_t **result, int extended,
+                     int allow_include, int *has_include,
+                     struct smartlist_t *opened_lst, int recursion_level,
+                     config_line_t **last,
+                     include_handler_fn handle_include)
+{
+  config_line_t *list = NULL, **next, *list_last = NULL;
+  char *k, *v;
+  const char *parse_err;
+  int include_used = 0;
+
+  if (recursion_level > MAX_INCLUDE_RECURSION_LEVEL) {
+    log_warn(LD_CONFIG, "Error while parsing configuration: more than %d "
+             "nested %%includes.", MAX_INCLUDE_RECURSION_LEVEL);
+    return -1;
+  }
+
+  next = &list;
+  do {
+    k = v = NULL;
+    string = parse_config_line_from_str_verbose(string, &k, &v, &parse_err);
+    if (!string) {
+      log_warn(LD_CONFIG, "Error while parsing configuration: %s",
+               parse_err?parse_err:"<unknown>");
+      config_free_lines(list);
+      tor_free(k);
+      tor_free(v);
+      return -1;
+    }
+    if (k && v) {
+      unsigned command = CONFIG_LINE_NORMAL;
+      if (extended) {
+        if (k[0] == '+') {
+          char *k_new = tor_strdup(k+1);
+          tor_free(k);
+          k = k_new;
+          command = CONFIG_LINE_APPEND;
+        } else if (k[0] == '/') {
+          char *k_new = tor_strdup(k+1);
+          tor_free(k);
+          k = k_new;
+          tor_free(v);
+          v = tor_strdup("");
+          command = CONFIG_LINE_CLEAR;
+        }
+      }
+
+      if (allow_include && !strcmp(k, "%include") && handle_include) {
+        tor_free(k);
+        include_used = 1;
+
+        config_line_t *include_list;
+        if (handle_include(v, recursion_level, extended, &include_list,
+                           &list_last, opened_lst) < 0) {
+          log_warn(LD_CONFIG, "Error reading included configuration "
+                   "file or directory: \"%s\".", v);
+          config_free_lines(list);
+          tor_free(v);
+          return -1;
+        }
+        log_notice(LD_CONFIG, "Included configuration file or "
+                   "directory at recursion level %d: \"%s\".",
+                   recursion_level, v);
+        *next = include_list;
+        if (list_last)
+          next = &list_last->next;
+        tor_free(v);
+      } else {
+        /* This list can get long, so we keep a pointer to the end of it
+         * rather than using config_line_append over and over and getting
+         * n^2 performance. */
+        *next = tor_malloc_zero(sizeof(**next));
+        (*next)->key = k;
+        (*next)->value = v;
+        (*next)->next = NULL;
+        (*next)->command = command;
+        list_last = *next;
+        next = &((*next)->next);
+      }
+    } else {
+      tor_free(k);
+      tor_free(v);
+    }
+  } while (*string);
+
+  if (last) {
+    *last = list_last;
+  }
+  if (has_include) {
+    *has_include = include_used;
+  }
+  *result = list;
+  return 0;
+}
+
+/** Same as config_get_lines_include but does not allow %include */
+int
+config_get_lines(const char *string, config_line_t **result, int extended)
+{
+  return config_get_lines_aux(string, result, extended, 0, NULL, NULL, 1,
+                              NULL, NULL);
+}
+
+/**
+ * Free all the configuration lines on the linked list <b>front</b>.
+ */
+void
+config_free_lines_(config_line_t *front)
+{
+  config_line_t *tmp;
+
+  while (front) {
+    tmp = front;
+    front = tmp->next;
+
+    tor_free(tmp->key);
+    tor_free(tmp->value);
+    tor_free(tmp);
+  }
+}
+
+/** Return a newly allocated deep copy of the lines in <b>inp</b>. */
+config_line_t *
+config_lines_dup(const config_line_t *inp)
+{
+  return config_lines_dup_and_filter(inp, NULL);
+}
+
+/** Return a newly allocated deep copy of the lines in <b>inp</b>,
+ * but only the ones whose keys begin with <b>key</b> (case-insensitive).
+ * If <b>key</b> is NULL, do not filter. */
+config_line_t *
+config_lines_dup_and_filter(const config_line_t *inp,
+                            const char *key)
+{
+  config_line_t *result = NULL;
+  config_line_t **next_out = &result;
+  while (inp) {
+    if (key && strcasecmpstart(inp->key, key)) {
+      inp = inp->next;
+      continue;
+    }
+    *next_out = tor_malloc_zero(sizeof(config_line_t));
+    (*next_out)->key = tor_strdup(inp->key);
+    (*next_out)->value = tor_strdup(inp->value);
+    inp = inp->next;
+    next_out = &((*next_out)->next);
+  }
+  (*next_out) = NULL;
+  return result;
+}
+
+/** Return true iff a and b contain identical keys and values in identical
+ * order. */
+int
+config_lines_eq(config_line_t *a, config_line_t *b)
+{
+  while (a && b) {
+    if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value))
+      return 0;
+    a = a->next;
+    b = b->next;
+  }
+  if (a || b)
+    return 0;
+  return 1;
+}
+
+/** Return the number of lines in <b>a</b> whose key is <b>key</b>. */
+int
+config_count_key(const config_line_t *a, const char *key)
+{
+  int n = 0;
+  while (a) {
+    if (!strcasecmp(a->key, key)) {
+      ++n;
+    }
+    a = a->next;
+  }
+  return n;
+}
+
+/** Given a string containing part of a configuration file or similar format,
+ * advance past comments and whitespace and try to parse a single line.  If we
+ * parse a line successfully, set *<b>key_out</b> to a new string holding the
+ * key portion and *<b>value_out</b> to a new string holding the value portion
+ * of the line, and return a pointer to the start of the next line.  If we run
+ * out of data, return a pointer to the end of the string.  If we encounter an
+ * error, return NULL and set *<b>err_out</b> (if provided) to an error
+ * message.
+ */
+const char *
+parse_config_line_from_str_verbose(const char *line, char **key_out,
+                                   char **value_out,
+                                   const char **err_out)
+{
+  /*
+    See torrc_format.txt for a description of the (silly) format this parses.
+   */
+  const char *key, *val, *cp;
+  int continuation = 0;
+
+  tor_assert(key_out);
+  tor_assert(value_out);
+
+  *key_out = *value_out = NULL;
+  key = val = NULL;
+  /* Skip until the first keyword. */
+  while (1) {
+    while (TOR_ISSPACE(*line))
+      ++line;
+    if (*line == '#') {
+      while (*line && *line != '\n')
+        ++line;
+    } else {
+      break;
+    }
+  }
+
+  if (!*line) { /* End of string? */
+    *key_out = *value_out = NULL;
+    return line;
+  }
+
+  /* Skip until the next space or \ followed by newline. */
+  key = line;
+  while (*line && !TOR_ISSPACE(*line) && *line != '#' &&
+         ! (line[0] == '\\' && line[1] == '\n'))
+    ++line;
+  *key_out = tor_strndup(key, line-key);
+
+  /* Skip until the value. */
+  while (*line == ' ' || *line == '\t')
+    ++line;
+
+  val = line;
+
+  /* Find the end of the line. */
+  if (*line == '\"') { // XXX No continuation handling is done here
+    if (!(line = unescape_string(line, value_out, NULL))) {
+      if (err_out)
+        *err_out = "Invalid escape sequence in quoted string";
+      return NULL;
+    }
+    while (*line == ' ' || *line == '\t')
+      ++line;
+    if (*line == '\r' && *(++line) == '\n')
+      ++line;
+    if (*line && *line != '#' && *line != '\n') {
+      if (err_out)
+        *err_out = "Excess data after quoted string";
+      return NULL;
+    }
+  } else {
+    /* Look for the end of the line. */
+    while (*line && *line != '\n' && (*line != '#' || continuation)) {
+      if (*line == '\\' && line[1] == '\n') {
+        continuation = 1;
+        line += 2;
+      } else if (*line == '#') {
+        do {
+          ++line;
+        } while (*line && *line != '\n');
+        if (*line == '\n')
+          ++line;
+      } else {
+        ++line;
+      }
+    }
+
+    if (*line == '\n') {
+      cp = line++;
+    } else {
+      cp = line;
+    }
+    /* Now back cp up to be the last nonspace character */
+    while (cp>val && TOR_ISSPACE(*(cp-1)))
+      --cp;
+
+    tor_assert(cp >= val);
+
+    /* Now copy out and decode the value. */
+    *value_out = tor_strndup(val, cp-val);
+    if (continuation) {
+      char *v_out, *v_in;
+      v_out = v_in = *value_out;
+      while (*v_in) {
+        if (*v_in == '#') {
+          do {
+            ++v_in;
+          } while (*v_in && *v_in != '\n');
+          if (*v_in == '\n')
+            ++v_in;
+        } else if (v_in[0] == '\\' && v_in[1] == '\n') {
+          v_in += 2;
+        } else {
+          *v_out++ = *v_in++;
+        }
+      }
+      *v_out = '\0';
+    }
+  }
+
+  if (*line == '#') {
+    do {
+      ++line;
+    } while (*line && *line != '\n');
+  }
+  while (TOR_ISSPACE(*line)) ++line;
+
+  return line;
+}
diff --git a/src/lib/encoding/confline.h b/src/lib/encoding/confline.h
new file mode 100644
index 0000000000..3d9ae8a662
--- /dev/null
+++ b/src/lib/encoding/confline.h
@@ -0,0 +1,78 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file confline.h
+ *
+ * \brief Header for confline.c
+ **/
+
+#ifndef TOR_CONFLINE_H
+#define TOR_CONFLINE_H
+
+struct smartlist_t;
+
+/** Ordinary configuration line. */
+#define CONFIG_LINE_NORMAL 0
+/** Appends to previous configuration for the same option, even if we
+ * would ordinary replace it. */
+#define CONFIG_LINE_APPEND 1
+/* Removes all previous configuration for an option. */
+#define CONFIG_LINE_CLEAR 2
+
+#define MAX_INCLUDE_RECURSION_LEVEL 31
+
+/** A linked list of lines in a config file, or elsewhere */
+typedef struct config_line_t {
+  char *key;
+  char *value;
+  struct config_line_t *next;
+
+  /** What special treatment (if any) does this line require? */
+  unsigned int command:2;
+  /** If true, subsequent assignments to this linelist should replace
+   * it, not extend it.  Set only on the first item in a linelist in an
+   * or_options_t. */
+  unsigned int fragile:1;
+} config_line_t;
+
+void config_line_append(config_line_t **lst,
+                        const char *key, const char *val);
+void config_line_prepend(config_line_t **lst,
+                         const char *key, const char *val);
+config_line_t *config_lines_dup(const config_line_t *inp);
+config_line_t *config_lines_dup_and_filter(const config_line_t *inp,
+                                           const char *key);
+const config_line_t *config_line_find(const config_line_t *lines,
+                                      const char *key);
+int config_lines_eq(config_line_t *a, config_line_t *b);
+int config_count_key(const config_line_t *a, const char *key);
+void config_free_lines_(config_line_t *front);
+#define config_free_lines(front) \
+  do {                           \
+    config_free_lines_(front);   \
+    (front) = NULL;              \
+  } while (0)
+const char *parse_config_line_from_str_verbose(const char *line,
+                                       char **key_out, char **value_out,
+                                       const char **err_out);
+
+int config_get_lines(const char *string, struct config_line_t **result,
+                     int extended);
+
+typedef int (*include_handler_fn)(const char *, int, int,
+                                  struct config_line_t **,
+                                  struct config_line_t **,
+                                  struct smartlist_t *);
+
+int config_get_lines_aux(const char *string, struct config_line_t **result,
+                         int extended,
+                         int allow_include, int *has_include,
+                         struct smartlist_t *opened_lst, int recursion_level,
+                         config_line_t **last,
+                         include_handler_fn handle_include);
+
+#endif /* !defined(TOR_CONFLINE_H) */
diff --git a/src/lib/encoding/cstring.c b/src/lib/encoding/cstring.c
new file mode 100644
index 0000000000..29d3714126
--- /dev/null
+++ b/src/lib/encoding/cstring.c
@@ -0,0 +1,138 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file cstring.c
+ *
+ * \brief Decode data that has been written as a C literal.
+ **/
+
+#include "lib/encoding/cstring.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/compat_ctype.h"
+
+#include <string.h>
+
+#define TOR_ISODIGIT(c) ('0' <= (c) && (c) <= '7')
+
+/** Given a c-style double-quoted escaped string in <b>s</b>, extract and
+ * decode its contents into a newly allocated string.  On success, assign this
+ * string to *<b>result</b>, assign its length to <b>size_out</b> (if
+ * provided), and return a pointer to the position in <b>s</b> immediately
+ * after the string.  On failure, return NULL.
+ */
+const char *
+unescape_string(const char *s, char **result, size_t *size_out)
+{
+  const char *cp;
+  char *out;
+  if (s[0] != '\"')
+    return NULL;
+  cp = s+1;
+  while (1) {
+    switch (*cp) {
+      case '\0':
+      case '\n':
+        return NULL;
+      case '\"':
+        goto end_of_loop;
+      case '\\':
+        if (cp[1] == 'x' || cp[1] == 'X') {
+          if (!(TOR_ISXDIGIT(cp[2]) && TOR_ISXDIGIT(cp[3])))
+            return NULL;
+          cp += 4;
+        } else if (TOR_ISODIGIT(cp[1])) {
+          cp += 2;
+          if (TOR_ISODIGIT(*cp)) ++cp;
+          if (TOR_ISODIGIT(*cp)) ++cp;
+        } else if (cp[1] == 'n' || cp[1] == 'r' || cp[1] == 't' || cp[1] == '"'
+                   || cp[1] == '\\' || cp[1] == '\'') {
+          cp += 2;
+        } else {
+          return NULL;
+        }
+        break;
+      default:
+        ++cp;
+        break;
+    }
+  }
+ end_of_loop:
+  out = *result = tor_malloc(cp-s + 1);
+  cp = s+1;
+  while (1) {
+    switch (*cp)
+      {
+      case '\"':
+        *out = '\0';
+        if (size_out) *size_out = out - *result;
+        return cp+1;
+
+        /* LCOV_EXCL_START -- we caught this in parse_config_from_line. */
+      case '\0':
+        tor_fragile_assert();
+        tor_free(*result);
+        return NULL;
+        /* LCOV_EXCL_STOP */
+      case '\\':
+        switch (cp[1])
+          {
+          case 'n': *out++ = '\n'; cp += 2; break;
+          case 'r': *out++ = '\r'; cp += 2; break;
+          case 't': *out++ = '\t'; cp += 2; break;
+          case 'x': case 'X':
+            {
+              int x1, x2;
+
+              x1 = hex_decode_digit(cp[2]);
+              x2 = hex_decode_digit(cp[3]);
+              if (x1 == -1 || x2 == -1) {
+                /* LCOV_EXCL_START */
+                /* we caught this above in the initial loop. */
+                tor_assert_nonfatal_unreached();
+                tor_free(*result);
+                return NULL;
+                /* LCOV_EXCL_STOP */
+              }
+
+              *out++ = ((x1<<4) + x2);
+              cp += 4;
+            }
+            break;
+          case '0': case '1': case '2': case '3': case '4': case '5':
+          case '6': case '7':
+            {
+              int n = cp[1]-'0';
+              cp += 2;
+              if (TOR_ISODIGIT(*cp)) { n = n*8 + *cp-'0'; cp++; }
+              if (TOR_ISODIGIT(*cp)) { n = n*8 + *cp-'0'; cp++; }
+              if (n > 255) { tor_free(*result); return NULL; }
+              *out++ = (char)n;
+            }
+            break;
+          case '\'':
+          case '\"':
+          case '\\':
+          case '\?':
+            *out++ = cp[1];
+            cp += 2;
+            break;
+
+            /* LCOV_EXCL_START */
+          default:
+            /* we caught this above in the initial loop. */
+            tor_assert_nonfatal_unreached();
+            tor_free(*result); return NULL;
+            /* LCOV_EXCL_STOP */
+          }
+        break;
+      default:
+        *out++ = *cp++;
+      }
+  }
+}
diff --git a/src/lib/encoding/cstring.h b/src/lib/encoding/cstring.h
new file mode 100644
index 0000000000..904a2c9c1c
--- /dev/null
+++ b/src/lib/encoding/cstring.h
@@ -0,0 +1,19 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file cstring.h
+ *
+ * \brief Header for cstring.c
+ **/
+
+#ifndef TOR_CSTRING_H
+#define TOR_CSTRING_H
+
+#include <stddef.h>
+const char *unescape_string(const char *s, char **result, size_t *size_out);
+
+#endif /* !defined(TOR_CSTRING_H) */
diff --git a/src/lib/encoding/include.am b/src/lib/encoding/include.am
new file mode 100644
index 0000000000..2d2aa3988a
--- /dev/null
+++ b/src/lib/encoding/include.am
@@ -0,0 +1,26 @@
+noinst_LIBRARIES += src/lib/libtor-encoding.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-encoding-testing.a
+endif
+
+src_lib_libtor_encoding_a_SOURCES =			\
+	src/lib/encoding/binascii.c			\
+	src/lib/encoding/confline.c			\
+	src/lib/encoding/cstring.c			\
+	src/lib/encoding/keyval.c			\
+	src/lib/encoding/pem.c				\
+	src/lib/encoding/time_fmt.c
+
+src_lib_libtor_encoding_testing_a_SOURCES = \
+	$(src_lib_libtor_encoding_a_SOURCES)
+src_lib_libtor_encoding_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_encoding_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/encoding/binascii.h			\
+	src/lib/encoding/confline.h			\
+	src/lib/encoding/cstring.h			\
+	src/lib/encoding/keyval.h			\
+	src/lib/encoding/pem.h				\
+	src/lib/encoding/time_fmt.h
diff --git a/src/lib/encoding/keyval.c b/src/lib/encoding/keyval.c
new file mode 100644
index 0000000000..c5da5a0bfc
--- /dev/null
+++ b/src/lib/encoding/keyval.c
@@ -0,0 +1,52 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file keyval.c
+ *
+ * \brief Handle data encoded as a key=value pair.
+ **/
+
+#include "orconfig.h"
+#include "lib/encoding/keyval.h"
+#include "lib/log/escape.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+/** Return true if <b>string</b> is a valid 'key=[value]' string.
+ *  "value" is optional, to indicate the empty string. Log at logging
+ *  <b>severity</b> if something ugly happens. */
+int
+string_is_key_value(int severity, const char *string)
+{
+  /* position of equal sign in string */
+  const char *equal_sign_pos = NULL;
+
+  tor_assert(string);
+
+  if (strlen(string) < 2) { /* "x=" is shortest args string */
+    tor_log(severity, LD_GENERAL, "'%s' is too short to be a k=v value.",
+            escaped(string));
+    return 0;
+  }
+
+  equal_sign_pos = strchr(string, '=');
+  if (!equal_sign_pos) {
+    tor_log(severity, LD_GENERAL, "'%s' is not a k=v value.", escaped(string));
+    return 0;
+  }
+
+  /* validate that the '=' is not in the beginning of the string. */
+  if (equal_sign_pos == string) {
+    tor_log(severity, LD_GENERAL, "'%s' is not a valid k=v value.",
+            escaped(string));
+    return 0;
+  }
+
+  return 1;
+}
diff --git a/src/lib/encoding/keyval.h b/src/lib/encoding/keyval.h
new file mode 100644
index 0000000000..cd327b7a82
--- /dev/null
+++ b/src/lib/encoding/keyval.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file keyval.h
+ *
+ * \brief Header for keyval.c
+ **/
+
+#ifndef TOR_KEYVAL_H
+#define TOR_KEYVAL_H
+
+int string_is_key_value(int severity, const char *string);
+
+#endif
diff --git a/src/lib/encoding/pem.c b/src/lib/encoding/pem.c
new file mode 100644
index 0000000000..24b238b130
--- /dev/null
+++ b/src/lib/encoding/pem.c
@@ -0,0 +1,106 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file pem.c
+ *
+ * \brief Implement a trivial version of PEM encoding, for use with NSS.
+ *
+ * We deliberately do not support any encryption here.
+ **/
+
+#include "orconfig.h"
+
+#include "lib/encoding/pem.h"
+
+#include "lib/ctime/di_ops.h"
+#include "lib/encoding/binascii.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+
+#include <string.h>
+
+/**
+ * Return the length of a <b>src_len</b>-byte object when tagged with
+ * <b>objtype</b> and PEM-encoded.  Includes terminating NUL.
+ */
+size_t
+pem_encoded_size(size_t src_len, const char *objtype)
+{
+  return
+    strlen("-----BEGIN -----\n") +
+    strlen("-----END -----\n") +
+    strlen(objtype) * 2 +
+    base64_encode_size(src_len, BASE64_ENCODE_MULTILINE)
+    + 1;
+}
+
+/**
+ * PEM-encode the <b>srclen</b>-byte object at <b>src</b> into the
+ * <b>destlen<\b>-byte buffer at <b>dest</b>, tagging it with <b>objtype</b>.
+ * Return 0 on success and -1 on failure.
+ */
+int
+pem_encode(char *dest, size_t destlen, const uint8_t *src, size_t srclen,
+           const char *objtype)
+{
+  if (tor_snprintf(dest, destlen, "-----BEGIN %s-----\n", objtype) < 0)
+    return -1;
+
+  size_t offset = strlen(dest);
+
+  int n = base64_encode(dest + offset, destlen - offset,
+                        (const char *)src, srclen, BASE64_ENCODE_MULTILINE);
+  if (n < 0)
+    return -1;
+  offset += n;
+  if (BUG(offset > destlen))
+    return -1;
+
+  if (tor_snprintf(dest + offset, destlen - offset,
+                   "-----END %s-----\n", objtype) < 0)
+    return -1;
+
+  tor_assert(strlen(dest) + 1 <= pem_encoded_size(srclen, objtype));
+  return 0;
+}
+
+/**
+ * Given a PEM-encoded block of size <b>srclen</b> in <b>src</b>, if it has
+ * object type <b>objtype</b>, decode it into the <b>destlen</b>-byte buffer
+ * at <b>dest</b>.  Return the number of characters decoded on success, or -1
+ * on failure.
+ */
+int
+pem_decode(uint8_t *dest, size_t destlen, const char *src, size_t srclen,
+           const char *objtype)
+{
+  const char *eos = src + srclen;
+
+  src = eat_whitespace_eos(src, eos);
+
+  char *tag = NULL;
+  tor_asprintf(&tag, "-----BEGIN %s-----\n", objtype);
+  if ((size_t)(eos-src) < strlen(tag) || fast_memneq(src, tag, strlen(tag))) {
+    tor_free(tag);
+    return -1;
+  }
+  src += strlen(tag);
+  tor_free(tag);
+
+  // NOTE lack of trailing \n.  We do not enforce its presence.
+  tor_asprintf(&tag, "\n-----END %s-----", objtype);
+  const char *end_of_base64 = tor_memstr(src, eos-src, tag);
+  tor_free(tag);
+  if (end_of_base64 == NULL)
+    return -1;
+
+  /* Should we actually allow extra stuff at the end? */
+
+  return base64_decode((char*)dest, destlen, src, end_of_base64-src);
+}
diff --git a/src/lib/encoding/pem.h b/src/lib/encoding/pem.h
new file mode 100644
index 0000000000..0bbb06a794
--- /dev/null
+++ b/src/lib/encoding/pem.h
@@ -0,0 +1,26 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file pem.h
+ *
+ * \brief Header for pem.c
+ **/
+
+#ifndef TOR_PEM_H
+#define TOR_PEM_H
+
+#include "orconfig.h"
+#include <stddef.h>
+#include "lib/cc/torint.h"
+
+size_t pem_encoded_size(size_t src_len, const char *objtype);
+int pem_encode(char *dest, size_t destlen, const uint8_t *src, size_t srclen,
+               const char *objtype);
+int pem_decode(uint8_t *dest, size_t destlen, const char *src, size_t srclen,
+               const char *objtype);
+
+#endif
diff --git a/src/lib/encoding/time_fmt.c b/src/lib/encoding/time_fmt.c
new file mode 100644
index 0000000000..40543d41e0
--- /dev/null
+++ b/src/lib/encoding/time_fmt.c
@@ -0,0 +1,516 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file time_fmt.c
+ *
+ * \brief Encode and decode time in various formats.
+ *
+ * This module is higher-level than the conversion functions in "wallclock",
+ * and handles a larger variety of types.  It converts between different time
+ * formats, and encodes and decodes them from strings.
+ **/
+
+#include "lib/encoding/time_fmt.h"
+#include "lib/log/log.h"
+#include "lib/log/escape.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+#include "lib/string/scanf.h"
+#include "lib/wallclock/time_to_tm.h"
+
+#include <string.h>
+#include <time.h>
+
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+
+#ifdef _WIN32
+/* For struct timeval */
+#include <winsock2.h>
+#endif
+
+/** As localtime_r, but defined for platforms that don't have it:
+ *
+ * Convert *<b>timep</b> to a struct tm in local time, and store the value in
+ * *<b>result</b>.  Return the result on success, or NULL on failure.
+ *
+ * Treat malformatted inputs localtime outputs as a BUG.
+ */
+struct tm *
+tor_localtime_r(const time_t *timep, struct tm *result)
+{
+  char *err = NULL;
+  struct tm *r = tor_localtime_r_msg(timep, result, &err);
+  if (err) {
+    log_warn(LD_BUG, "%s", err);
+    tor_free(err);
+  }
+  return r;
+}
+
+/** As gmtime_r, but defined for platforms that don't have it:
+ *
+ * Convert *<b>timep</b> to a struct tm in UTC, and store the value in
+ * *<b>result</b>.  Return the result on success, or NULL on failure.
+ *
+ * Treat malformatted inputs or gmtime outputs as a BUG.
+ */
+struct tm *
+tor_gmtime_r(const time_t *timep, struct tm *result)
+{
+  char *err = NULL;
+  struct tm *r = tor_gmtime_r_msg(timep, result, &err);
+  if (err) {
+    log_warn(LD_BUG, "%s", err);
+    tor_free(err);
+  }
+  return r;
+}
+
+/** Yield true iff <b>y</b> is a leap-year. */
+#define IS_LEAPYEAR(y) (!(y % 4) && ((y % 100) || !(y % 400)))
+/** Helper: Return the number of leap-days between Jan 1, y1 and Jan 1, y2. */
+static int
+n_leapdays(int year1, int year2)
+{
+  --year1;
+  --year2;
+  return (year2/4 - year1/4) - (year2/100 - year1/100)
+    + (year2/400 - year1/400);
+}
+/** Number of days per month in non-leap year; used by tor_timegm and
+ * parse_rfc1123_time. */
+static const int days_per_month[] =
+  { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
+
+/** Compute a time_t given a struct tm.  The result is given in UTC, and
+ * does not account for leap seconds.  Return 0 on success, -1 on failure.
+ */
+int
+tor_timegm(const struct tm *tm, time_t *time_out)
+{
+  /* This is a pretty ironclad timegm implementation, snarfed from Python2.2.
+   * It's way more brute-force than fiddling with tzset().
+   *
+   * We use int64_t rather than time_t to avoid overflow on multiplication on
+   * platforms with 32-bit time_t. Since year is clipped to INT32_MAX, and
+   * since 365 * 24 * 60 * 60 is approximately 31 million, it's not possible
+   * for INT32_MAX years to overflow int64_t when converted to seconds. */
+  int64_t year, days, hours, minutes, seconds;
+  int i, invalid_year, dpm;
+
+  /* Initialize time_out to 0 for now, to avoid bad usage in case this function
+     fails and the caller ignores the return value. */
+  tor_assert(time_out);
+  *time_out = 0;
+
+  /* avoid int overflow on addition */
+  if (tm->tm_year < INT32_MAX-1900) {
+    year = tm->tm_year + 1900;
+  } else {
+    /* clamp year */
+    year = INT32_MAX;
+  }
+  invalid_year = (year < 1970 || tm->tm_year >= INT32_MAX-1900);
+
+  if (tm->tm_mon >= 0 && tm->tm_mon <= 11) {
+    dpm = days_per_month[tm->tm_mon];
+    if (tm->tm_mon == 1 && !invalid_year && IS_LEAPYEAR(tm->tm_year)) {
+      dpm = 29;
+    }
+  } else {
+    /* invalid month - default to 0 days per month */
+    dpm = 0;
+  }
+
+  if (invalid_year ||
+      tm->tm_mon < 0 || tm->tm_mon > 11 ||
+      tm->tm_mday < 1 || tm->tm_mday > dpm ||
+      tm->tm_hour < 0 || tm->tm_hour > 23 ||
+      tm->tm_min < 0 || tm->tm_min > 59 ||
+      tm->tm_sec < 0 || tm->tm_sec > 60) {
+    log_warn(LD_BUG, "Out-of-range argument to tor_timegm");
+    return -1;
+  }
+  days = 365 * (year-1970) + n_leapdays(1970,(int)year);
+  for (i = 0; i < tm->tm_mon; ++i)
+    days += days_per_month[i];
+  if (tm->tm_mon > 1 && IS_LEAPYEAR(year))
+    ++days;
+  days += tm->tm_mday - 1;
+  hours = days*24 + tm->tm_hour;
+
+  minutes = hours*60 + tm->tm_min;
+  seconds = minutes*60 + tm->tm_sec;
+  /* Check that "seconds" will fit in a time_t. On platforms where time_t is
+   * 32-bit, this check will fail for dates in and after 2038.
+   *
+   * We already know that "seconds" can't be negative because "year" >= 1970 */
+#if SIZEOF_TIME_T < 8
+  if (seconds < TIME_MIN || seconds > TIME_MAX) {
+    log_warn(LD_BUG, "Result does not fit in tor_timegm");
+    return -1;
+  }
+#endif /* SIZEOF_TIME_T < 8 */
+  *time_out = (time_t)seconds;
+  return 0;
+}
+
+/* strftime is locale-specific, so we need to replace those parts */
+
+/** A c-locale array of 3-letter names of weekdays, starting with Sun. */
+static const char *WEEKDAY_NAMES[] =
+  { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" };
+/** A c-locale array of 3-letter names of months, starting with Jan. */
+static const char *MONTH_NAMES[] =
+  { "Jan", "Feb", "Mar", "Apr", "May", "Jun",
+    "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" };
+
+/** Set <b>buf</b> to the RFC1123 encoding of the UTC value of <b>t</b>.
+ * The buffer must be at least RFC1123_TIME_LEN+1 bytes long.
+ *
+ * (RFC1123 format is "Fri, 29 Sep 2006 15:54:20 GMT". Note the "GMT"
+ * rather than "UTC".)
+ */
+void
+format_rfc1123_time(char *buf, time_t t)
+{
+  struct tm tm;
+
+  tor_gmtime_r(&t, &tm);
+
+  strftime(buf, RFC1123_TIME_LEN+1, "___, %d ___ %Y %H:%M:%S GMT", &tm);
+  tor_assert(tm.tm_wday >= 0);
+  tor_assert(tm.tm_wday <= 6);
+  memcpy(buf, WEEKDAY_NAMES[tm.tm_wday], 3);
+  tor_assert(tm.tm_mon >= 0);
+  tor_assert(tm.tm_mon <= 11);
+  memcpy(buf+8, MONTH_NAMES[tm.tm_mon], 3);
+}
+
+/** Parse the (a subset of) the RFC1123 encoding of some time (in UTC) from
+ * <b>buf</b>, and store the result in *<b>t</b>.
+ *
+ * Note that we only accept the subset generated by format_rfc1123_time above,
+ * not the full range of formats suggested by RFC 1123.
+ *
+ * Return 0 on success, -1 on failure.
+*/
+int
+parse_rfc1123_time(const char *buf, time_t *t)
+{
+  struct tm tm;
+  char month[4];
+  char weekday[4];
+  int i, m, invalid_year;
+  unsigned tm_mday, tm_year, tm_hour, tm_min, tm_sec;
+  unsigned dpm;
+
+  if (strlen(buf) != RFC1123_TIME_LEN)
+    return -1;
+  memset(&tm, 0, sizeof(tm));
+  if (tor_sscanf(buf, "%3s, %2u %3s %u %2u:%2u:%2u GMT", weekday,
+             &tm_mday, month, &tm_year, &tm_hour,
+             &tm_min, &tm_sec) < 7) {
+    char *esc = esc_for_log(buf);
+    log_warn(LD_GENERAL, "Got invalid RFC1123 time %s", esc);
+    tor_free(esc);
+    return -1;
+  }
+
+  m = -1;
+  for (i = 0; i < 12; ++i) {
+    if (!strcmp(month, MONTH_NAMES[i])) {
+      m = i;
+      break;
+    }
+  }
+  if (m<0) {
+    char *esc = esc_for_log(buf);
+    log_warn(LD_GENERAL, "Got invalid RFC1123 time %s: No such month", esc);
+    tor_free(esc);
+    return -1;
+  }
+  tm.tm_mon = m;
+
+  invalid_year = (tm_year >= INT32_MAX || tm_year < 1970);
+  tor_assert(m >= 0 && m <= 11);
+  dpm = days_per_month[m];
+  if (m == 1 && !invalid_year && IS_LEAPYEAR(tm_year)) {
+    dpm = 29;
+  }
+
+  if (invalid_year || tm_mday < 1 || tm_mday > dpm ||
+      tm_hour > 23 || tm_min > 59 || tm_sec > 60) {
+    char *esc = esc_for_log(buf);
+    log_warn(LD_GENERAL, "Got invalid RFC1123 time %s", esc);
+    tor_free(esc);
+    return -1;
+  }
+  tm.tm_mday = (int)tm_mday;
+  tm.tm_year = (int)tm_year;
+  tm.tm_hour = (int)tm_hour;
+  tm.tm_min = (int)tm_min;
+  tm.tm_sec = (int)tm_sec;
+
+  if (tm.tm_year < 1970) {
+    /* LCOV_EXCL_START
+     * XXXX I think this is dead code; we already checked for
+     *      invalid_year above. */
+    tor_assert_nonfatal_unreached();
+    char *esc = esc_for_log(buf);
+    log_warn(LD_GENERAL,
+             "Got invalid RFC1123 time %s. (Before 1970)", esc);
+    tor_free(esc);
+    return -1;
+    /* LCOV_EXCL_STOP */
+  }
+  tm.tm_year -= 1900;
+
+  return tor_timegm(&tm, t);
+}
+
+/** Set <b>buf</b> to the ISO8601 encoding of the local value of <b>t</b>.
+ * The buffer must be at least ISO_TIME_LEN+1 bytes long.
+ *
+ * (ISO8601 format is 2006-10-29 10:57:20)
+ */
+void
+format_local_iso_time(char *buf, time_t t)
+{
+  struct tm tm;
+  strftime(buf, ISO_TIME_LEN+1, "%Y-%m-%d %H:%M:%S", tor_localtime_r(&t, &tm));
+}
+
+/** Set <b>buf</b> to the ISO8601 encoding of the GMT value of <b>t</b>.
+ * The buffer must be at least ISO_TIME_LEN+1 bytes long.
+ */
+void
+format_iso_time(char *buf, time_t t)
+{
+  struct tm tm;
+  strftime(buf, ISO_TIME_LEN+1, "%Y-%m-%d %H:%M:%S", tor_gmtime_r(&t, &tm));
+}
+
+/** As format_local_iso_time, but use the yyyy-mm-ddThh:mm:ss format to avoid
+ * embedding an internal space. */
+void
+format_local_iso_time_nospace(char *buf, time_t t)
+{
+  format_local_iso_time(buf, t);
+  buf[10] = 'T';
+}
+
+/** As format_iso_time, but use the yyyy-mm-ddThh:mm:ss format to avoid
+ * embedding an internal space. */
+void
+format_iso_time_nospace(char *buf, time_t t)
+{
+  format_iso_time(buf, t);
+  buf[10] = 'T';
+}
+
+/** As format_iso_time_nospace, but include microseconds in decimal
+ * fixed-point format.  Requires that buf be at least ISO_TIME_USEC_LEN+1
+ * bytes long. */
+void
+format_iso_time_nospace_usec(char *buf, const struct timeval *tv)
+{
+  tor_assert(tv);
+  format_iso_time_nospace(buf, (time_t)tv->tv_sec);
+  tor_snprintf(buf+ISO_TIME_LEN, 8, ".%06d", (int)tv->tv_usec);
+}
+
+/** Given an ISO-formatted UTC time value (after the epoch) in <b>cp</b>,
+ * parse it and store its value in *<b>t</b>.  Return 0 on success, -1 on
+ * failure.  Ignore extraneous stuff in <b>cp</b> after the end of the time
+ * string, unless <b>strict</b> is set. If <b>nospace</b> is set,
+ * expect the YYYY-MM-DDTHH:MM:SS format. */
+int
+parse_iso_time_(const char *cp, time_t *t, int strict, int nospace)
+{
+  struct tm st_tm;
+  unsigned int year=0, month=0, day=0, hour=0, minute=0, second=0;
+  int n_fields;
+  char extra_char, separator_char;
+  n_fields = tor_sscanf(cp, "%u-%2u-%2u%c%2u:%2u:%2u%c",
+                        &year, &month, &day,
+                        &separator_char,
+                        &hour, &minute, &second, &extra_char);
+  if (strict ? (n_fields != 7) : (n_fields < 7)) {
+    char *esc = esc_for_log(cp);
+    log_warn(LD_GENERAL, "ISO time %s was unparseable", esc);
+    tor_free(esc);
+    return -1;
+  }
+  if (separator_char != (nospace ? 'T' : ' ')) {
+    char *esc = esc_for_log(cp);
+    log_warn(LD_GENERAL, "ISO time %s was unparseable", esc);
+    tor_free(esc);
+    return -1;
+  }
+  if (year < 1970 || month < 1 || month > 12 || day < 1 || day > 31 ||
+          hour > 23 || minute > 59 || second > 60 || year >= INT32_MAX) {
+    char *esc = esc_for_log(cp);
+    log_warn(LD_GENERAL, "ISO time %s was nonsensical", esc);
+    tor_free(esc);
+    return -1;
+  }
+  st_tm.tm_year = (int)year-1900;
+  st_tm.tm_mon = month-1;
+  st_tm.tm_mday = day;
+  st_tm.tm_hour = hour;
+  st_tm.tm_min = minute;
+  st_tm.tm_sec = second;
+  st_tm.tm_wday = 0; /* Should be ignored. */
+
+  if (st_tm.tm_year < 70) {
+    /* LCOV_EXCL_START
+     * XXXX I think this is dead code; we already checked for
+     *      year < 1970 above. */
+    tor_assert_nonfatal_unreached();
+    char *esc = esc_for_log(cp);
+    log_warn(LD_GENERAL, "Got invalid ISO time %s. (Before 1970)", esc);
+    tor_free(esc);
+    return -1;
+    /* LCOV_EXCL_STOP */
+  }
+  return tor_timegm(&st_tm, t);
+}
+
+/** Given an ISO-formatted UTC time value (after the epoch) in <b>cp</b>,
+ * parse it and store its value in *<b>t</b>.  Return 0 on success, -1 on
+ * failure. Reject the string if any characters are present after the time.
+ */
+int
+parse_iso_time(const char *cp, time_t *t)
+{
+  return parse_iso_time_(cp, t, 1, 0);
+}
+
+/**
+ * As parse_iso_time, but parses a time encoded by format_iso_time_nospace().
+ */
+int
+parse_iso_time_nospace(const char *cp, time_t *t)
+{
+  return parse_iso_time_(cp, t, 1, 1);
+}
+
+/** Given a <b>date</b> in one of the three formats allowed by HTTP (ugh),
+ * parse it into <b>tm</b>.  Return 0 on success, negative on failure. */
+int
+parse_http_time(const char *date, struct tm *tm)
+{
+  const char *cp;
+  char month[4];
+  char wkday[4];
+  int i;
+  unsigned tm_mday, tm_year, tm_hour, tm_min, tm_sec;
+
+  tor_assert(tm);
+  memset(tm, 0, sizeof(*tm));
+
+  /* First, try RFC1123 or RFC850 format: skip the weekday.  */
+  if ((cp = strchr(date, ','))) {
+    ++cp;
+    if (*cp != ' ')
+      return -1;
+    ++cp;
+    if (tor_sscanf(cp, "%2u %3s %4u %2u:%2u:%2u GMT",
+               &tm_mday, month, &tm_year,
+               &tm_hour, &tm_min, &tm_sec) == 6) {
+      /* rfc1123-date */
+      tm_year -= 1900;
+    } else if (tor_sscanf(cp, "%2u-%3s-%2u %2u:%2u:%2u GMT",
+                      &tm_mday, month, &tm_year,
+                      &tm_hour, &tm_min, &tm_sec) == 6) {
+      /* rfc850-date */
+    } else {
+      return -1;
+    }
+  } else {
+    /* No comma; possibly asctime() format. */
+    if (tor_sscanf(date, "%3s %3s %2u %2u:%2u:%2u %4u",
+               wkday, month, &tm_mday,
+               &tm_hour, &tm_min, &tm_sec, &tm_year) == 7) {
+      tm_year -= 1900;
+    } else {
+      return -1;
+    }
+  }
+  tm->tm_mday = (int)tm_mday;
+  tm->tm_year = (int)tm_year;
+  tm->tm_hour = (int)tm_hour;
+  tm->tm_min = (int)tm_min;
+  tm->tm_sec = (int)tm_sec;
+  tm->tm_wday = 0; /* Leave this unset. */
+
+  month[3] = '\0';
+  /* Okay, now decode the month. */
+  /* set tm->tm_mon to dummy value so the check below fails. */
+  tm->tm_mon = -1;
+  for (i = 0; i < 12; ++i) {
+    if (!strcasecmp(MONTH_NAMES[i], month)) {
+      tm->tm_mon = i;
+    }
+  }
+
+  if (tm->tm_year < 0 ||
+      tm->tm_mon < 0  || tm->tm_mon > 11 ||
+      tm->tm_mday < 1 || tm->tm_mday > 31 ||
+      tm->tm_hour < 0 || tm->tm_hour > 23 ||
+      tm->tm_min < 0  || tm->tm_min > 59 ||
+      tm->tm_sec < 0  || tm->tm_sec > 60)
+    return -1; /* Out of range, or bad month. */
+
+  return 0;
+}
+
+/** Given an <b>interval</b> in seconds, try to write it to the
+ * <b>out_len</b>-byte buffer in <b>out</b> in a human-readable form.
+ * Returns a non-negative integer on success, -1 on failure.
+ */
+int
+format_time_interval(char *out, size_t out_len, long interval)
+{
+  /* We only report seconds if there's no hours. */
+  long sec = 0, min = 0, hour = 0, day = 0;
+
+  /* -LONG_MIN is LONG_MAX + 1, which causes signed overflow */
+  if (interval < -LONG_MAX)
+    interval = LONG_MAX;
+  else if (interval < 0)
+    interval = -interval;
+
+  if (interval >= 86400) {
+    day = interval / 86400;
+    interval %= 86400;
+  }
+  if (interval >= 3600) {
+    hour = interval / 3600;
+    interval %= 3600;
+  }
+  if (interval >= 60) {
+    min = interval / 60;
+    interval %= 60;
+  }
+  sec = interval;
+
+  if (day) {
+    return tor_snprintf(out, out_len, "%ld days, %ld hours, %ld minutes",
+                        day, hour, min);
+  } else if (hour) {
+    return tor_snprintf(out, out_len, "%ld hours, %ld minutes", hour, min);
+  } else if (min) {
+    return tor_snprintf(out, out_len, "%ld minutes, %ld seconds", min, sec);
+  } else {
+    return tor_snprintf(out, out_len, "%ld seconds", sec);
+  }
+}
diff --git a/src/lib/encoding/time_fmt.h b/src/lib/encoding/time_fmt.h
new file mode 100644
index 0000000000..0ddeca57fc
--- /dev/null
+++ b/src/lib/encoding/time_fmt.h
@@ -0,0 +1,44 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file time_fmt.h
+ *
+ * \brief Header for time_fmt.c
+ **/
+
+#ifndef TOR_TIME_FMT_H
+#define TOR_TIME_FMT_H
+
+#include "orconfig.h"
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+
+struct tm;
+struct timeval;
+
+struct tm *tor_localtime_r(const time_t *timep, struct tm *result);
+struct tm *tor_gmtime_r(const time_t *timep, struct tm *result);
+int tor_timegm(const struct tm *tm, time_t *time_out);
+
+#define RFC1123_TIME_LEN 29
+void format_rfc1123_time(char *buf, time_t t);
+int parse_rfc1123_time(const char *buf, time_t *t);
+#define ISO_TIME_LEN 19
+#define ISO_TIME_USEC_LEN (ISO_TIME_LEN+7)
+void format_local_iso_time(char *buf, time_t t);
+void format_iso_time(char *buf, time_t t);
+void format_local_iso_time_nospace(char *buf, time_t t);
+void format_iso_time_nospace(char *buf, time_t t);
+void format_iso_time_nospace_usec(char *buf, const struct timeval *tv);
+int parse_iso_time_(const char *cp, time_t *t, int strict, int nospace);
+int parse_iso_time(const char *buf, time_t *t);
+int parse_iso_time_nospace(const char *cp, time_t *t);
+int parse_http_time(const char *buf, struct tm *tm);
+int format_time_interval(char *out, size_t out_len, long interval);
+
+#endif
diff --git a/src/lib/err/.may_include b/src/lib/err/.may_include
new file mode 100644
index 0000000000..48cc0ef088
--- /dev/null
+++ b/src/lib/err/.may_include
@@ -0,0 +1,3 @@
+orconfig.h
+lib/cc/*.h
+lib/err/*.h
diff --git a/src/common/backtrace.c b/src/lib/err/backtrace.c
index 81e04e94eb..8606f42177 100644
--- a/src/common/backtrace.c
+++ b/src/lib/err/backtrace.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,12 +11,14 @@
  * family of functions, which are sometimes provided by libc and sometimes
  * provided by libexecinfo.  We tie into the sigaction() backend in order to
  * detect crashes.
+ *
+ * This is one of the lowest-level modules, since nearly everything needs to
+ * be able to log an error.  As such, it doesn't call the log module or any
+ * other higher-level modules directly.
  */
 
 #include "orconfig.h"
-#include "compat.h"
-#include "util.h"
-#include "torlog.h"
+#include "lib/err/torerr.h"
 
 #ifdef HAVE_EXECINFO_H
 #include <execinfo.h>
@@ -30,6 +32,13 @@
 #ifdef HAVE_SIGNAL_H
 #include <signal.h>
 #endif
+#ifdef HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
 
 #ifdef HAVE_CYGWIN_SIGNAL_H
 #include <cygwin/signal.h>
@@ -37,10 +46,15 @@
 #include <sys/ucontext.h>
 #elif defined(HAVE_UCONTEXT_H)
 #include <ucontext.h>
+#endif /* defined(HAVE_CYGWIN_SIGNAL_H) || ... */
+
+#ifdef HAVE_PTHREAD_H
+#include <pthread.h>
 #endif
 
 #define EXPOSE_CLEAN_BACKTRACE
-#include "backtrace.h"
+#include "lib/err/backtrace.h"
+#include "lib/err/torerr.h"
 
 #if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \
   defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION)
@@ -51,8 +65,11 @@
 #define NO_BACKTRACE_IMPL
 #endif
 
+// Redundant with util.h, but doing it here so we can avoid that dependency.
+#define raw_free free
+
 /** Version of Tor to report in backtrace messages. */
-static char *bt_version = NULL;
+static char bt_version[128] = "";
 
 #ifdef USE_BACKTRACE
 /** Largest stack depth to try to dump. */
@@ -60,8 +77,9 @@ static char *bt_version = NULL;
 /** Static allocation of stack to dump. This is static so we avoid stack
  * pressure. */
 static void *cb_buf[MAX_DEPTH];
-/** Protects cb_buf from concurrent access */
-static tor_mutex_t cb_buf_mutex;
+/** Protects cb_buf from concurrent access. Pthreads, since this code
+ * is Unix-only, and since this code needs to be lowest-level. */
+static pthread_mutex_t cb_buf_mutex = PTHREAD_MUTEX_INITIALIZER;
 
 /** Change a stacktrace in <b>stack</b> of depth <b>depth</b> so that it will
  * log the correct function from which a signal was received with context
@@ -76,51 +94,53 @@ clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx)
 #ifdef PC_FROM_UCONTEXT
 #if defined(__linux__)
   const size_t n = 1;
-#elif defined(__darwin__) || defined(__APPLE__) || defined(__OpenBSD__) \
+#elif defined(__darwin__) || defined(__APPLE__) || defined(OpenBSD) \
   || defined(__FreeBSD__)
   const size_t n = 2;
 #else
   const size_t n = 1;
-#endif
+#endif /* defined(__linux__) || ... */
   if (depth <= n)
     return;
 
   stack[n] = (void*) ctx->PC_FROM_UCONTEXT;
-#else
+#else /* !(defined(PC_FROM_UCONTEXT)) */
   (void) depth;
   (void) ctx;
   (void) stack;
-#endif
+#endif /* defined(PC_FROM_UCONTEXT) */
 }
 
 /** Log a message <b>msg</b> at <b>severity</b> in <b>domain</b>, and follow
- * that with a backtrace log. */
+ * that with a backtrace log.  Send messages via the tor_log function at
+ * logger". */
 void
-log_backtrace(int severity, int domain, const char *msg)
+log_backtrace_impl(int severity, int domain, const char *msg,
+                   tor_log_fn logger)
 {
   size_t depth;
   char **symbols;
   size_t i;
 
-  tor_mutex_acquire(&cb_buf_mutex);
+  pthread_mutex_lock(&cb_buf_mutex);
 
   depth = backtrace(cb_buf, MAX_DEPTH);
   symbols = backtrace_symbols(cb_buf, (int)depth);
 
-  tor_log(severity, domain, "%s. Stack trace:", msg);
+  logger(severity, domain, "%s: %s. Stack trace:", bt_version, msg);
   if (!symbols) {
     /* LCOV_EXCL_START -- we can't provoke this. */
-    tor_log(severity, domain, "    Unable to generate backtrace.");
+    logger(severity, domain, "    Unable to generate backtrace.");
     goto done;
     /* LCOV_EXCL_STOP */
   }
   for (i=0; i < depth; ++i) {
-    tor_log(severity, domain, "    %s", symbols[i]);
+    logger(severity, domain, "    %s", symbols[i]);
   }
   raw_free(symbols);
 
  done:
-  tor_mutex_release(&cb_buf_mutex);
+  pthread_mutex_unlock(&cb_buf_mutex);
 }
 
 static void crash_handler(int sig, siginfo_t *si, void *ctx_)
@@ -155,8 +175,23 @@ crash_handler(int sig, siginfo_t *si, void *ctx_)
   abort();
 }
 
+/** Write a backtrace to all of the emergency-error fds. */
+void
+dump_stack_symbols_to_error_fds(void)
+{
+  int n_fds, i;
+  const int *fds = NULL;
+  size_t depth;
+
+  depth = backtrace(cb_buf, MAX_DEPTH);
+
+  n_fds = tor_log_get_sigsafe_err_fds(&fds);
+  for (i=0; i < n_fds; ++i)
+    backtrace_symbols_fd(cb_buf, (int)depth, fds[i]);
+}
+
 /** Install signal handlers as needed so that when we crash, we produce a
- * useful stack trace. Return 0 on success, -1 on failure. */
+ * useful stack trace. Return 0 on success, -errno on failure. */
 static int
 install_bt_handler(void)
 {
@@ -166,8 +201,6 @@ install_bt_handler(void)
 
   struct sigaction sa;
 
-  tor_mutex_init(&cb_buf_mutex);
-
   memset(&sa, 0, sizeof(sa));
   sa.sa_sigaction = crash_handler;
   sa.sa_flags = SA_SIGINFO;
@@ -176,8 +209,7 @@ install_bt_handler(void)
   for (i = 0; trap_signals[i] >= 0; ++i) {
     if (sigaction(trap_signals[i], &sa, NULL) == -1) {
       /* LCOV_EXCL_START */
-      log_warn(LD_BUG, "Sigaction failed: %s", strerror(errno));
-      rv = -1;
+      rv = -errno;
       /* LCOV_EXCL_STOP */
     }
   }
@@ -200,15 +232,16 @@ install_bt_handler(void)
 static void
 remove_bt_handler(void)
 {
-  tor_mutex_uninit(&cb_buf_mutex);
 }
-#endif
+#endif /* defined(USE_BACKTRACE) */
 
 #ifdef NO_BACKTRACE_IMPL
 void
-log_backtrace(int severity, int domain, const char *msg)
+log_backtrace_impl(int severity, int domain, const char *msg,
+                   tor_log_fn logger)
 {
-  tor_log(severity, domain, "%s. (Stack trace not available)", msg);
+  logger(severity, domain, "%s: %s. (Stack trace not available)",
+         bt_version, msg);
 }
 
 static int
@@ -221,17 +254,45 @@ static void
 remove_bt_handler(void)
 {
 }
-#endif
+
+void
+dump_stack_symbols_to_error_fds(void)
+{
+}
+#endif /* defined(NO_BACKTRACE_IMPL) */
+
+/** Return the tor version used for error messages on crashes.
+ * Signal-safe: returns a pointer to a static array. */
+const char *
+get_tor_backtrace_version(void)
+{
+  return bt_version;
+}
 
 /** Set up code to handle generating error messages on crashes. */
 int
 configure_backtrace_handler(const char *tor_version)
 {
-  tor_free(bt_version);
-  if (tor_version)
-    tor_asprintf(&bt_version, "Tor %s", tor_version);
-  else
-    tor_asprintf(&bt_version, "Tor");
+  char version[128] = "Tor\0";
+
+  if (tor_version) {
+    int snp_rv = 0;
+    /* We can't use strlcat() here, because it is defined in
+     * string/compat_string.h on some platforms, and string uses torerr. */
+    snp_rv = snprintf(version, sizeof(version), "Tor %s", tor_version);
+    /* It's safe to call raw_assert() here, because raw_assert() does not
+     * call configure_backtrace_handler(). */
+    raw_assert(snp_rv < (int)sizeof(version));
+    raw_assert(snp_rv >= 0);
+  }
+
+  char *str_rv = NULL;
+  /* We can't use strlcpy() here, see the note about strlcat() above. */
+  str_rv = strncpy(bt_version, version, sizeof(bt_version) - 1);
+  /* We must terminate bt_version, then raw_assert(), because raw_assert()
+   * uses bt_version. */
+  bt_version[sizeof(bt_version) - 1] = 0;
+  raw_assert(str_rv == bt_version);
 
   return install_bt_handler();
 }
@@ -242,7 +303,4 @@ void
 clean_up_backtrace_handler(void)
 {
   remove_bt_handler();
-
-  tor_free(bt_version);
 }
-
diff --git a/src/lib/err/backtrace.h b/src/lib/err/backtrace.h
new file mode 100644
index 0000000000..48b41fca02
--- /dev/null
+++ b/src/lib/err/backtrace.h
@@ -0,0 +1,36 @@
+/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_BACKTRACE_H
+#define TOR_BACKTRACE_H
+
+/**
+ * \file backtrace.h
+ *
+ * \brief Header for backtrace.c
+ **/
+
+#include "orconfig.h"
+#include "lib/cc/compat_compiler.h"
+
+typedef void (*tor_log_fn)(int, unsigned, const char *fmt, ...)
+  CHECK_PRINTF(3,4);
+
+void log_backtrace_impl(int severity, int domain, const char *msg,
+                        tor_log_fn logger);
+int configure_backtrace_handler(const char *tor_version);
+void clean_up_backtrace_handler(void);
+void dump_stack_symbols_to_error_fds(void);
+const char *get_tor_backtrace_version(void);
+
+#define log_backtrace(sev, dom, msg) \
+  log_backtrace_impl((sev), (dom), (msg), tor_log)
+
+#ifdef EXPOSE_CLEAN_BACKTRACE
+#if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \
+  defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION)
+void clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx);
+#endif
+#endif /* defined(EXPOSE_CLEAN_BACKTRACE) */
+
+#endif /* !defined(TOR_BACKTRACE_H) */
diff --git a/src/lib/err/include.am b/src/lib/err/include.am
new file mode 100644
index 0000000000..f2a409c51e
--- /dev/null
+++ b/src/lib/err/include.am
@@ -0,0 +1,19 @@
+
+noinst_LIBRARIES += src/lib/libtor-err.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-err-testing.a
+endif
+
+src_lib_libtor_err_a_SOURCES =			\
+	src/lib/err/backtrace.c				\
+	src/lib/err/torerr.c
+
+src_lib_libtor_err_testing_a_SOURCES = \
+	$(src_lib_libtor_err_a_SOURCES)
+src_lib_libtor_err_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_err_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/err/backtrace.h				\
+	src/lib/err/torerr.h
diff --git a/src/lib/err/torerr.c b/src/lib/err/torerr.c
new file mode 100644
index 0000000000..6b5224273a
--- /dev/null
+++ b/src/lib/err/torerr.c
@@ -0,0 +1,245 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file torerr.c
+ *
+ * \brief Handling code for unrecoverable emergencies, at a lower level
+ *   than the logging code.
+ *
+ * There are plenty of places that things can go wrong in Tor's backend
+ * libraries: the allocator can fail, the locking subsystem can fail, and so
+ * on.  But since these subsystems are used themselves by the logging module,
+ * they can't use the logging code directly to report their errors.
+ *
+ * As a workaround, the logging code provides this module with a set of raw
+ * fds to be used for reporting errors in the lowest-level Tor code.
+ */
+
+#include "orconfig.h"
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+#ifdef HAVE_TIME_H
+#include <time.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+
+#include "lib/err/torerr.h"
+#include "lib/err/backtrace.h"
+
+/** Array of fds to log crash-style warnings to. */
+static int sigsafe_log_fds[TOR_SIGSAFE_LOG_MAX_FDS] = { STDERR_FILENO };
+/** The number of elements used in sigsafe_log_fds */
+static int n_sigsafe_log_fds = 1;
+/** Log granularity in milliseconds. */
+static int log_granularity = 1000;
+
+/** Write <b>s</b> to each element of sigsafe_log_fds. Return 0 on success, -1
+ * on failure. */
+static int
+tor_log_err_sigsafe_write(const char *s)
+{
+  int i;
+  ssize_t r;
+  size_t len = strlen(s);
+  int err = 0;
+  for (i=0; i < n_sigsafe_log_fds; ++i) {
+    r = write(sigsafe_log_fds[i], s, len);
+    err += (r != (ssize_t)len);
+  }
+  return err ? -1 : 0;
+}
+
+/** Given a list of string arguments ending with a NULL, writes them
+ * to our logs and to stderr (if possible).  This function is safe to call
+ * from within a signal handler. */
+void
+tor_log_err_sigsafe(const char *m, ...)
+{
+  va_list ap;
+  const char *x;
+  char timebuf[33];
+  time_t now = time(NULL);
+
+  if (!m)
+    return;
+  if (log_granularity >= 2000) {
+    int g = log_granularity / 1000;
+    now -= now % g;
+  }
+  timebuf[0] = now < 0 ? '-' : ' ';
+  if (now < 0) now = -now;
+  timebuf[1] = '\0';
+  format_dec_number_sigsafe(now, timebuf+1, sizeof(timebuf)-1);
+  tor_log_err_sigsafe_write("\n=========================================="
+                             "================== T=");
+  tor_log_err_sigsafe_write(timebuf);
+  tor_log_err_sigsafe_write("\n");
+  tor_log_err_sigsafe_write(m);
+  va_start(ap, m);
+  while ((x = va_arg(ap, const char*))) {
+    tor_log_err_sigsafe_write(x);
+  }
+  va_end(ap);
+}
+
+/** Set *<b>out</b> to a pointer to an array of the fds to log errors to from
+ * inside a signal handler or other emergency condition. Return the number of
+ * elements in the array. */
+int
+tor_log_get_sigsafe_err_fds(const int **out)
+{
+  *out = sigsafe_log_fds;
+  return n_sigsafe_log_fds;
+}
+
+/**
+ * Update the list of fds that get errors from inside a signal handler or
+ * other emergency condition. Ignore any beyond the first
+ * TOR_SIGSAFE_LOG_MAX_FDS.
+ */
+void
+tor_log_set_sigsafe_err_fds(const int *fds, int n)
+{
+  if (n > TOR_SIGSAFE_LOG_MAX_FDS) {
+    n = TOR_SIGSAFE_LOG_MAX_FDS;
+  }
+
+  memcpy(sigsafe_log_fds, fds, n * sizeof(int));
+  n_sigsafe_log_fds = n;
+}
+
+/**
+ * Set the granularity (in ms) to use when reporting fatal errors outside
+ * the logging system.
+ */
+void
+tor_log_sigsafe_err_set_granularity(int ms)
+{
+  log_granularity = ms;
+}
+
+/**
+ * Log an emergency assertion failure message.
+ *
+ * This kind of message is safe to send from within a log handler,
+ * a signal handler, or other emergency situation.
+ */
+void
+tor_raw_assertion_failed_msg_(const char *file, int line, const char *expr,
+                              const char *msg)
+{
+  char linebuf[16];
+  format_dec_number_sigsafe(line, linebuf, sizeof(linebuf));
+  tor_log_err_sigsafe("INTERNAL ERROR: Raw assertion failed in ",
+                      get_tor_backtrace_version(), " at ",
+                      file, ":", linebuf, ": ", expr, "\n", NULL);
+  if (msg) {
+    tor_log_err_sigsafe_write(msg);
+    tor_log_err_sigsafe_write("\n");
+  }
+
+  dump_stack_symbols_to_error_fds();
+
+  /* Some platforms (macOS, maybe others?) can swallow the last write before an
+   * abort. This issue is probably caused by a race condition between write
+   * buffer cache flushing, and process termination. So we write an extra
+   * newline, to make sure that the message always gets through. */
+  tor_log_err_sigsafe_write("\n");
+}
+
+/* As format_{hex,dex}_number_sigsafe, but takes a <b>radix</b> argument
+ * in range 2..16 inclusive. */
+static int
+format_number_sigsafe(unsigned long x, char *buf, int buf_len,
+                      unsigned int radix)
+{
+  unsigned long tmp;
+  int len;
+  char *cp;
+
+  /* NOT tor_assert. This needs to be safe to run from within a signal
+   * handler, and from within the 'tor_assert() has failed' code.  Not even
+   * raw_assert(), since raw_assert() calls this function on failure. */
+  if (radix < 2 || radix > 16)
+    return 0;
+
+  /* Count how many digits we need. */
+  tmp = x;
+  len = 1;
+  while (tmp >= radix) {
+    tmp /= radix;
+    ++len;
+  }
+
+  /* Not long enough */
+  if (!buf || len >= buf_len)
+    return 0;
+
+  cp = buf + len;
+  *cp = '\0';
+  do {
+    unsigned digit = (unsigned) (x % radix);
+    if (cp <= buf) {
+      /* Not tor_assert(); see above. */
+      abort();
+    }
+    --cp;
+    *cp = "0123456789ABCDEF"[digit];
+    x /= radix;
+  } while (x);
+
+  /* NOT tor_assert; see above. */
+  if (cp != buf) {
+    abort(); // LCOV_EXCL_LINE
+  }
+
+  return len;
+}
+
+/**
+ * Helper function to output hex numbers from within a signal handler.
+ *
+ * Writes the nul-terminated hexadecimal digits of <b>x</b> into a buffer
+ * <b>buf</b> of size <b>buf_len</b>, and return the actual number of digits
+ * written, not counting the terminal NUL.
+ *
+ * If there is insufficient space, write nothing and return 0.
+ *
+ * This accepts an unsigned int because format_helper_exit_status() needs to
+ * call it with a signed int and an unsigned char, and since the C standard
+ * does not guarantee that an int is wider than a char (an int must be at
+ * least 16 bits but it is permitted for a char to be that wide as well), we
+ * can't assume a signed int is sufficient to accommodate an unsigned char.
+ * Thus, format_helper_exit_status() will still need to emit any require '-'
+ * on its own.
+ *
+ * For most purposes, you'd want to use tor_snprintf("%x") instead of this
+ * function; it's designed to be used in code paths where you can't call
+ * arbitrary C functions.
+ */
+int
+format_hex_number_sigsafe(unsigned long x, char *buf, int buf_len)
+{
+  return format_number_sigsafe(x, buf, buf_len, 16);
+}
+
+/** As format_hex_number_sigsafe, but format the number in base 10. */
+int
+format_dec_number_sigsafe(unsigned long x, char *buf, int buf_len)
+{
+  return format_number_sigsafe(x, buf, buf_len, 10);
+}
diff --git a/src/lib/err/torerr.h b/src/lib/err/torerr.h
new file mode 100644
index 0000000000..6ae91fbe85
--- /dev/null
+++ b/src/lib/err/torerr.h
@@ -0,0 +1,47 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file torerr.h
+ *
+ * \brief Headers for torerr.c.
+ **/
+
+#ifndef TOR_TORERR_H
+#define TOR_TORERR_H
+
+#include "lib/cc/compat_compiler.h"
+
+/* The raw_assert...() variants are for use within code that can't call
+ * tor_assertion_failed_() because of call circularity issues. */
+#define raw_assert(expr) STMT_BEGIN                                     \
+    if (!(expr)) {                                                      \
+      tor_raw_assertion_failed_msg_(__FILE__, __LINE__, #expr, NULL);   \
+      abort();                                                          \
+    }                                                                   \
+  STMT_END
+#define raw_assert_unreached(expr) raw_assert(0)
+#define raw_assert_unreached_msg(msg) STMT_BEGIN                    \
+    tor_raw_assertion_failed_msg_(__FILE__, __LINE__, "0", (msg));  \
+    abort();                                                        \
+  STMT_END
+
+void tor_raw_assertion_failed_msg_(const char *file, int line,
+                                   const char *expr,
+                                   const char *msg);
+
+/** Maximum number of fds that will get notifications if we crash */
+#define TOR_SIGSAFE_LOG_MAX_FDS 8
+
+void tor_log_err_sigsafe(const char *m, ...);
+int tor_log_get_sigsafe_err_fds(const int **out);
+void tor_log_set_sigsafe_err_fds(const int *fds, int n);
+void tor_log_sigsafe_err_set_granularity(int ms);
+
+int format_hex_number_sigsafe(unsigned long x, char *buf, int max_len);
+int format_dec_number_sigsafe(unsigned long x, char *buf, int max_len);
+
+#endif /* !defined(TOR_TORLOG_H) */
diff --git a/src/lib/evloop/.may_include b/src/lib/evloop/.may_include
new file mode 100644
index 0000000000..30af508914
--- /dev/null
+++ b/src/lib/evloop/.may_include
@@ -0,0 +1,16 @@
+orconfig.h
+
+lib/cc/*.h
+lib/crypt_ops/*.h
+lib/evloop/*.h
+lib/intmath/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/net/*.h
+lib/string/*.h
+lib/testsupport/*.h
+lib/thread/*.h
+lib/time/*.h
+
+src/ext/timeouts/timeout.c
+tor_queue.h
\ No newline at end of file
diff --git a/src/lib/evloop/compat_libevent.c b/src/lib/evloop/compat_libevent.c
new file mode 100644
index 0000000000..91eacb9938
--- /dev/null
+++ b/src/lib/evloop/compat_libevent.c
@@ -0,0 +1,535 @@
+/* Copyright (c) 2009-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compat_libevent.c
+ * \brief Wrappers and utility functions for Libevent.
+ */
+
+#include "orconfig.h"
+#define COMPAT_LIBEVENT_PRIVATE
+#include "lib/evloop/compat_libevent.h"
+
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/string/compat_string.h"
+
+#include <event2/event.h>
+#include <event2/thread.h>
+#include <string.h>
+
+/** A string which, if it appears in a libevent log, should be ignored. */
+static const char *suppress_msg = NULL;
+/** Callback function passed to event_set_log() so we can intercept
+ * log messages from libevent. */
+STATIC void
+libevent_logging_callback(int severity, const char *msg)
+{
+  char buf[1024];
+  size_t n;
+  if (suppress_msg && strstr(msg, suppress_msg))
+    return;
+  n = strlcpy(buf, msg, sizeof(buf));
+  if (n && n < sizeof(buf) && buf[n-1] == '\n') {
+    buf[n-1] = '\0';
+  }
+  switch (severity) {
+    case _EVENT_LOG_DEBUG:
+      log_debug(LD_NOCB|LD_NET, "Message from libevent: %s", buf);
+      break;
+    case _EVENT_LOG_MSG:
+      log_info(LD_NOCB|LD_NET, "Message from libevent: %s", buf);
+      break;
+    case _EVENT_LOG_WARN:
+      log_warn(LD_NOCB|LD_GENERAL, "Warning from libevent: %s", buf);
+      break;
+    case _EVENT_LOG_ERR:
+      log_err(LD_NOCB|LD_GENERAL, "Error from libevent: %s", buf);
+      break;
+    default:
+      log_warn(LD_NOCB|LD_GENERAL, "Message [%d] from libevent: %s",
+          severity, buf);
+      break;
+  }
+}
+/** Set hook to intercept log messages from libevent. */
+void
+configure_libevent_logging(void)
+{
+  event_set_log_callback(libevent_logging_callback);
+}
+
+/** Ignore any libevent log message that contains <b>msg</b>. */
+void
+suppress_libevent_log_msg(const char *msg)
+{
+  suppress_msg = msg;
+}
+
+/* Wrapper for event_free() that tolerates tor_event_free(NULL) */
+void
+tor_event_free_(struct event *ev)
+{
+  if (ev == NULL)
+    return;
+  event_free(ev);
+}
+
+/** Global event base for use by the main thread. */
+static struct event_base *the_event_base = NULL;
+
+/**
+ * @defgroup postloop post-loop event helpers
+ *
+ * If we're not careful, Libevent can susceptible to infinite event chains:
+ * one event can activate another, whose callback activates another, whose
+ * callback activates another, ad infinitum.  While this is happening,
+ * Libevent won't be checking timeouts, socket-based events, signals, and so
+ * on.
+ *
+ * We solve this problem by marking some events as "post-loop".  A post-loop
+ * event behaves like any ordinary event, but any events that _it_ activates
+ * cannot run until Libevent has checked for other events at least once.
+ *
+ * @{ */
+
+/**
+ * An event that stops Libevent from running any more events on the current
+ * iteration of its loop, until it has re-checked for socket events, signal
+ * events, timeouts, etc.
+ */
+static struct event *rescan_mainloop_ev = NULL;
+
+/**
+ * Callback to implement rescan_mainloop_ev: it simply exits the mainloop,
+ * and relies on Tor to re-enter the mainloop since no error has occurred.
+ */
+static void
+rescan_mainloop_cb(evutil_socket_t fd, short events, void *arg)
+{
+  (void)fd;
+  (void)events;
+  struct event_base *the_base = arg;
+  event_base_loopbreak(the_base);
+}
+
+/** @} */
+
+/* This is what passes for version detection on OSX.  We set
+ * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before
+ * 10.4.0 (aka 1040). */
+#ifdef __APPLE__
+#ifdef __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__
+#define MACOSX_KQUEUE_IS_BROKEN \
+  (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1040)
+#else
+#define MACOSX_KQUEUE_IS_BROKEN 0
+#endif /* defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) */
+#endif /* defined(__APPLE__) */
+
+/** Initialize the Libevent library and set up the event base. */
+void
+tor_libevent_initialize(tor_libevent_cfg *torcfg)
+{
+  tor_assert(the_event_base == NULL);
+  /* some paths below don't use torcfg, so avoid unused variable warnings */
+  (void)torcfg;
+
+  {
+    int attempts = 0;
+    struct event_config *cfg;
+
+    ++attempts;
+    cfg = event_config_new();
+    tor_assert(cfg);
+
+    /* Telling Libevent not to try to turn locking on can avoid a needless
+     * socketpair() attempt. */
+    event_config_set_flag(cfg, EVENT_BASE_FLAG_NOLOCK);
+
+    if (torcfg->num_cpus > 0)
+      event_config_set_num_cpus_hint(cfg, torcfg->num_cpus);
+
+    /* We can enable changelist support with epoll, since we don't give
+     * Libevent any dup'd fds.  This lets us avoid some syscalls. */
+    event_config_set_flag(cfg, EVENT_BASE_FLAG_EPOLL_USE_CHANGELIST);
+
+    the_event_base = event_base_new_with_config(cfg);
+
+    event_config_free(cfg);
+  }
+
+  if (!the_event_base) {
+    /* LCOV_EXCL_START */
+    log_err(LD_GENERAL, "Unable to initialize Libevent: cannot continue.");
+    exit(1); // exit ok: libevent is broken.
+    /* LCOV_EXCL_STOP */
+  }
+
+  rescan_mainloop_ev = event_new(the_event_base, -1, 0,
+                                 rescan_mainloop_cb, the_event_base);
+  if (!rescan_mainloop_ev) {
+    /* LCOV_EXCL_START */
+    log_err(LD_GENERAL, "Unable to create rescan event: cannot continue.");
+    exit(1); // exit ok: libevent is broken.
+    /* LCOV_EXCL_STOP */
+  }
+
+  log_info(LD_GENERAL,
+      "Initialized libevent version %s using method %s. Good.",
+      event_get_version(), tor_libevent_get_method());
+}
+
+/** Return the current Libevent event base that we're set up to use. */
+MOCK_IMPL(struct event_base *,
+tor_libevent_get_base, (void))
+{
+  tor_assert(the_event_base != NULL);
+  return the_event_base;
+}
+
+/** Return the name of the Libevent backend we're using. */
+const char *
+tor_libevent_get_method(void)
+{
+  return event_base_get_method(the_event_base);
+}
+
+/** Return a string representation of the version of the currently running
+ * version of Libevent. */
+const char *
+tor_libevent_get_version_str(void)
+{
+  return event_get_version();
+}
+
+/** Return a string representation of the version of Libevent that was used
+* at compilation time. */
+const char *
+tor_libevent_get_header_version_str(void)
+{
+  return LIBEVENT_VERSION;
+}
+
+/** Represents a timer that's run every N microseconds by Libevent. */
+struct periodic_timer_t {
+  /** Underlying event used to implement this periodic event. */
+  struct event *ev;
+  /** The callback we'll be invoking whenever the event triggers */
+  void (*cb)(struct periodic_timer_t *, void *);
+  /** User-supplied data for the callback */
+  void *data;
+};
+
+/** Libevent callback to implement a periodic event. */
+static void
+periodic_timer_cb(evutil_socket_t fd, short what, void *arg)
+{
+  periodic_timer_t *timer = arg;
+  (void) what;
+  (void) fd;
+  timer->cb(timer, timer->data);
+}
+
+/** Create and schedule a new timer that will run every <b>tv</b> in
+ * the event loop of <b>base</b>.  When the timer fires, it will
+ * run the timer in <b>cb</b> with the user-supplied data in <b>data</b>. */
+periodic_timer_t *
+periodic_timer_new(struct event_base *base,
+                   const struct timeval *tv,
+                   void (*cb)(periodic_timer_t *timer, void *data),
+                   void *data)
+{
+  periodic_timer_t *timer;
+  tor_assert(base);
+  tor_assert(tv);
+  tor_assert(cb);
+  timer = tor_malloc_zero(sizeof(periodic_timer_t));
+  if (!(timer->ev = tor_event_new(base, -1, EV_PERSIST,
+                                  periodic_timer_cb, timer))) {
+    tor_free(timer);
+    return NULL;
+  }
+  timer->cb = cb;
+  timer->data = data;
+  periodic_timer_launch(timer, tv);
+  return timer;
+}
+
+/**
+ * Launch the timer <b>timer</b> to run at <b>tv</b> from now, and every
+ * <b>tv</b> thereafter.
+ *
+ * If the timer is already enabled, this function does nothing.
+ */
+void
+periodic_timer_launch(periodic_timer_t *timer, const struct timeval *tv)
+{
+  tor_assert(timer);
+  if (event_pending(timer->ev, EV_TIMEOUT, NULL))
+    return;
+  event_add(timer->ev, tv);
+}
+
+/**
+ * Disable the provided <b>timer</b>, but do not free it.
+ *
+ * You can reenable the same timer later with periodic_timer_launch.
+ *
+ * If the timer is already disabled, this function does nothing.
+ */
+void
+periodic_timer_disable(periodic_timer_t *timer)
+{
+  tor_assert(timer);
+  (void) event_del(timer->ev);
+}
+
+/** Stop and free a periodic timer */
+void
+periodic_timer_free_(periodic_timer_t *timer)
+{
+  if (!timer)
+    return;
+  tor_event_free(timer->ev);
+  tor_free(timer);
+}
+
+/**
+ * Type used to represent events that run directly from the main loop,
+ * either because they are activated from elsewhere in the code, or
+ * because they have a simple timeout.
+ *
+ * We use this type to avoid exposing Libevent's API throughout the rest
+ * of the codebase.
+ *
+ * This type can't be used for all events: it doesn't handle events that
+ * are triggered by signals or by sockets.
+ */
+struct mainloop_event_t {
+  struct event *ev;
+  void (*cb)(mainloop_event_t *, void *);
+  void *userdata;
+};
+
+/**
+ * Internal: Implements mainloop event using a libevent event.
+ */
+static void
+mainloop_event_cb(evutil_socket_t fd, short what, void *arg)
+{
+  (void)fd;
+  (void)what;
+  mainloop_event_t *mev = arg;
+  mev->cb(mev, mev->userdata);
+}
+
+/**
+ * As mainloop_event_cb, but implements a post-loop event.
+ */
+static void
+mainloop_event_postloop_cb(evutil_socket_t fd, short what, void *arg)
+{
+  (void)fd;
+  (void)what;
+
+  /* Note that if rescan_mainloop_ev is already activated,
+   * event_active() will do nothing: only the first post-loop event that
+   * happens each time through the event loop will cause it to be
+   * activated.
+   *
+   * Because event_active() puts events on a FIFO queue, every event
+   * that is made active _after_ rescan_mainloop_ev will get its
+   * callback run after rescan_mainloop_cb is called -- that is, on the
+   * next iteration of the loop.
+   */
+  event_active(rescan_mainloop_ev, EV_READ, 1);
+
+  mainloop_event_t *mev = arg;
+  mev->cb(mev, mev->userdata);
+}
+
+/**
+ * Helper for mainloop_event_new() and mainloop_event_postloop_new().
+ */
+static mainloop_event_t *
+mainloop_event_new_impl(int postloop,
+                        void (*cb)(mainloop_event_t *, void *),
+                        void *userdata)
+{
+  tor_assert(cb);
+
+  struct event_base *base = tor_libevent_get_base();
+  mainloop_event_t *mev = tor_malloc_zero(sizeof(mainloop_event_t));
+  mev->ev = tor_event_new(base, -1, 0,
+                  postloop ? mainloop_event_postloop_cb : mainloop_event_cb,
+                  mev);
+  tor_assert(mev->ev);
+  mev->cb = cb;
+  mev->userdata = userdata;
+  return mev;
+}
+
+/**
+ * Create and return a new mainloop_event_t to run the function <b>cb</b>.
+ *
+ * When run, the callback function will be passed the mainloop_event_t
+ * and <b>userdata</b> as its arguments.  The <b>userdata</b> pointer
+ * must remain valid for as long as the mainloop_event_t event exists:
+ * it is your responsibility to free it.
+ *
+ * The event is not scheduled by default: Use mainloop_event_activate()
+ * or mainloop_event_schedule() to make it run.
+ */
+mainloop_event_t *
+mainloop_event_new(void (*cb)(mainloop_event_t *, void *),
+                   void *userdata)
+{
+  return mainloop_event_new_impl(0, cb, userdata);
+}
+
+/**
+ * As mainloop_event_new(), but create a post-loop event.
+ *
+ * A post-loop event behaves like any ordinary event, but any events
+ * that _it_ activates cannot run until Libevent has checked for other
+ * events at least once.
+ */
+mainloop_event_t *
+mainloop_event_postloop_new(void (*cb)(mainloop_event_t *, void *),
+                            void *userdata)
+{
+  return mainloop_event_new_impl(1, cb, userdata);
+}
+
+/**
+ * Schedule <b>event</b> to run in the main loop, immediately.  If it is
+ * not scheduled, it will run anyway. If it is already scheduled to run
+ * later, it will run now instead.  This function will have no effect if
+ * the event is already scheduled to run.
+ *
+ * This function may only be called from the main thread.
+ */
+void
+mainloop_event_activate(mainloop_event_t *event)
+{
+  tor_assert(event);
+  event_active(event->ev, EV_READ, 1);
+}
+
+/** Schedule <b>event</b> to run in the main loop, after a delay of <b>tv</b>.
+ *
+ * If the event is scheduled for a different time, cancel it and run
+ * after this delay instead.  If the event is currently pending to run
+ * <em>now</b>, has no effect.
+ *
+ * Do not call this function with <b>tv</b> == NULL -- use
+ * mainloop_event_activate() instead.
+ *
+ * This function may only be called from the main thread.
+ */
+int
+mainloop_event_schedule(mainloop_event_t *event, const struct timeval *tv)
+{
+  tor_assert(event);
+  if (BUG(tv == NULL)) {
+    // LCOV_EXCL_START
+    mainloop_event_activate(event);
+    return 0;
+    // LCOV_EXCL_STOP
+  }
+  return event_add(event->ev, tv);
+}
+
+/** Cancel <b>event</b> if it is currently active or pending. (Do nothing if
+ * the event is not currently active or pending.) */
+void
+mainloop_event_cancel(mainloop_event_t *event)
+{
+  if (!event)
+    return;
+  (void) event_del(event->ev);
+}
+
+/** Cancel <b>event</b> and release all storage associated with it. */
+void
+mainloop_event_free_(mainloop_event_t *event)
+{
+  if (!event)
+    return;
+  tor_event_free(event->ev);
+  memset(event, 0xb8, sizeof(*event));
+  tor_free(event);
+}
+
+int
+tor_init_libevent_rng(void)
+{
+  int rv = 0;
+  char buf[256];
+  if (evutil_secure_rng_init() < 0) {
+    rv = -1;
+  }
+  crypto_rand(buf, 32);
+#ifdef HAVE_EVUTIL_SECURE_RNG_ADD_BYTES
+  evutil_secure_rng_add_bytes(buf, 32);
+#endif
+  evutil_secure_rng_get_bytes(buf, sizeof(buf));
+  return rv;
+}
+
+/**
+ * Un-initialize libevent in preparation for an exit
+ */
+void
+tor_libevent_free_all(void)
+{
+  tor_event_free(rescan_mainloop_ev);
+  if (the_event_base)
+    event_base_free(the_event_base);
+  the_event_base = NULL;
+}
+
+/**
+ * Run the event loop for the provided event_base, handling events until
+ * something stops it.  If <b>once</b> is set, then just poll-and-run
+ * once, then exit.  Return 0 on success, -1 if an error occurred, or 1
+ * if we exited because no events were pending or active.
+ *
+ * This isn't reentrant or multithreaded.
+ */
+int
+tor_libevent_run_event_loop(struct event_base *base, int once)
+{
+  const int flags = once ? EVLOOP_ONCE : 0;
+  return event_base_loop(base, flags);
+}
+
+/** Tell the event loop to exit after <b>delay</b>.  If <b>delay</b> is NULL,
+ * instead exit after we're done running the currently active events. */
+void
+tor_libevent_exit_loop_after_delay(struct event_base *base,
+                                   const struct timeval *delay)
+{
+  event_base_loopexit(base, delay);
+}
+
+/** Tell the event loop to exit after running whichever callback is currently
+ * active. */
+void
+tor_libevent_exit_loop_after_callback(struct event_base *base)
+{
+  event_base_loopbreak(base);
+}
+
+#if defined(TOR_UNIT_TESTS)
+/** For testing: called post-fork to make libevent reinitialize
+ * kernel structures. */
+void
+tor_libevent_postfork(void)
+{
+  int r = event_reinit(tor_libevent_get_base());
+  tor_assert(r == 0);
+}
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/common/compat_libevent.h b/src/lib/evloop/compat_libevent.h
index c2e34764e4..afe887a013 100644
--- a/src/common/compat_libevent.h
+++ b/src/lib/evloop/compat_libevent.h
@@ -1,13 +1,17 @@
-/* Copyright (c) 2009-2016, The Tor Project, Inc. */
+/* Copyright (c) 2009-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * \file compat_libevent.h
+ * \brief Header for compat_libevent.c
+ **/
+
 #ifndef TOR_COMPAT_LIBEVENT_H
 #define TOR_COMPAT_LIBEVENT_H
 
 #include "orconfig.h"
-#include "testsupport.h"
-
-#include <event2/event.h>
+#include "lib/testsupport/testsupport.h"
+#include "lib/malloc/malloc.h"
 
 void configure_libevent_logging(void);
 void suppress_libevent_log_msg(const char *msg);
@@ -19,7 +23,13 @@ void suppress_libevent_log_msg(const char *msg);
   evdns_add_server_port_with_base(tor_libevent_get_base(), \
   (sock),(tcp),(cb),(data));
 
-void tor_event_free(struct event *ev);
+struct event;
+struct event_base;
+struct timeval;
+
+void tor_event_free_(struct event *ev);
+#define tor_event_free(ev) \
+  FREE_AND_NULL(struct event, tor_event_free_, (ev))
 
 typedef struct periodic_timer_t periodic_timer_t;
 
@@ -27,9 +37,25 @@ periodic_timer_t *periodic_timer_new(struct event_base *base,
              const struct timeval *tv,
              void (*cb)(periodic_timer_t *timer, void *data),
              void *data);
-void periodic_timer_free(periodic_timer_t *);
-
-#define tor_event_base_loopexit event_base_loopexit
+void periodic_timer_free_(periodic_timer_t *);
+void periodic_timer_launch(periodic_timer_t *, const struct timeval *tv);
+void periodic_timer_disable(periodic_timer_t *);
+#define periodic_timer_free(t) \
+  FREE_AND_NULL(periodic_timer_t, periodic_timer_free_, (t))
+
+typedef struct mainloop_event_t mainloop_event_t;
+mainloop_event_t *mainloop_event_new(void (*cb)(mainloop_event_t *, void *),
+                                     void *userdata);
+mainloop_event_t * mainloop_event_postloop_new(
+                                     void (*cb)(mainloop_event_t *, void *),
+                                     void *userdata);
+void mainloop_event_activate(mainloop_event_t *event);
+int mainloop_event_schedule(mainloop_event_t *event,
+                            const struct timeval *delay);
+void mainloop_event_cancel(mainloop_event_t *event);
+void mainloop_event_free_(mainloop_event_t *event);
+#define mainloop_event_free(event) \
+  FREE_AND_NULL(mainloop_event_t, mainloop_event_free_, (event))
 
 /** Defines a configuration for using libevent with Tor: passed as an argument
  * to tor_libevent_initialize() to describe how we want to set up. */
@@ -47,15 +73,19 @@ const char *tor_libevent_get_method(void);
 void tor_check_libevent_header_compatibility(void);
 const char *tor_libevent_get_version_str(void);
 const char *tor_libevent_get_header_version_str(void);
+void tor_libevent_free_all(void);
 
 int tor_init_libevent_rng(void);
 
-void tor_gettimeofday_cached(struct timeval *tv);
-void tor_gettimeofday_cache_clear(void);
 #ifdef TOR_UNIT_TESTS
-void tor_gettimeofday_cache_set(const struct timeval *tv);
+void tor_libevent_postfork(void);
 #endif
 
+int tor_libevent_run_event_loop(struct event_base *base, int once);
+void tor_libevent_exit_loop_after_delay(struct event_base *base,
+                                        const struct timeval *delay);
+void tor_libevent_exit_loop_after_callback(struct event_base *base);
+
 #ifdef COMPAT_LIBEVENT_PRIVATE
 
 /** Macro: returns the number of a Libevent version as a 4-byte number,
@@ -69,7 +99,6 @@ void tor_gettimeofday_cache_set(const struct timeval *tv);
 
 STATIC void
 libevent_logging_callback(int severity, const char *msg);
-#endif
-
-#endif
+#endif /* defined(COMPAT_LIBEVENT_PRIVATE) */
 
+#endif /* !defined(TOR_COMPAT_LIBEVENT_H) */
diff --git a/src/lib/evloop/include.am b/src/lib/evloop/include.am
new file mode 100644
index 0000000000..6b0076272a
--- /dev/null
+++ b/src/lib/evloop/include.am
@@ -0,0 +1,26 @@
+
+noinst_LIBRARIES += src/lib/libtor-evloop.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-evloop-testing.a
+endif
+
+src_lib_libtor_evloop_a_SOURCES =			\
+	src/lib/evloop/compat_libevent.c		\
+	src/lib/evloop/procmon.c			\
+	src/lib/evloop/timers.c				\
+	src/lib/evloop/token_bucket.c			\
+	src/lib/evloop/workqueue.c
+
+
+src_lib_libtor_evloop_testing_a_SOURCES = \
+	$(src_lib_libtor_evloop_a_SOURCES)
+src_lib_libtor_evloop_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_evloop_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/evloop/compat_libevent.h		\
+	src/lib/evloop/procmon.h			\
+	src/lib/evloop/timers.h				\
+	src/lib/evloop/token_bucket.h			\
+	src/lib/evloop/workqueue.h
diff --git a/src/common/procmon.c b/src/lib/evloop/procmon.c
index c485c760c7..52469fa5fc 100644
--- a/src/common/procmon.c
+++ b/src/lib/evloop/procmon.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2016, The Tor Project, Inc. */
+/* Copyright (c) 2011-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -6,11 +6,13 @@
  * \brief Process-termination monitor functions
  **/
 
-#include "procmon.h"
+#include "lib/evloop/procmon.h"
 
-#include "util.h"
-
-#include <event2/event.h>
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/log/win32err.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/parse_int.h"
 
 #ifdef HAVE_SIGNAL_H
 #include <signal.h>
@@ -18,8 +20,12 @@
 #ifdef HAVE_ERRNO_H
 #include <errno.h>
 #endif
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
 
 #ifdef _WIN32
+#include <winsock2.h>
 #include <windows.h>
 #endif
 
@@ -32,11 +38,11 @@ typedef int pid_t;
 #define PID_T_FORMAT "%d"
 #elif (SIZEOF_PID_T == SIZEOF_LONG)
 #define PID_T_FORMAT "%ld"
-#elif (SIZEOF_PID_T == SIZEOF_INT64_T)
-#define PID_T_FORMAT I64_FORMAT
+#elif (SIZEOF_PID_T == 8)
+#define PID_T_FORMAT "%"PRId64
 #else
 #error Unknown: SIZEOF_PID_T
-#endif
+#endif /* (0 == SIZEOF_PID_T) && defined(_WIN32) || ... */
 
 /* Define to 1 if process-termination monitors on this OS and Libevent
    version must poll for process termination themselves. */
@@ -44,7 +50,7 @@ typedef int pid_t;
 /* Currently we need to poll in some way on all systems. */
 
 #ifdef PROCMON_POLLS
-static void tor_process_monitor_poll_cb(evutil_socket_t unused1, short unused2,
+static void tor_process_monitor_poll_cb(periodic_timer_t *ev,
                                         void *procmon_);
 #endif
 
@@ -71,7 +77,7 @@ parse_process_specifier(const char *process_spec,
 
   /* If we're lucky, long will turn out to be large enough to hold a
    * PID everywhere that Tor runs. */
-  pid_l = tor_parse_long(process_spec, 0, 1, LONG_MAX, &pid_ok, &pspec_next);
+  pid_l = tor_parse_long(process_spec, 10, 1, LONG_MAX, &pid_ok, &pspec_next);
 
   /* Reserve room in the ‘process specifier’ for additional
    * (platform-specific) identifying information beyond the PID, to
@@ -114,7 +120,7 @@ struct tor_process_monitor_t {
   HANDLE hproc;
   /* XXXX We should have Libevent watch hproc for us,
    * if/when some version of Libevent can be told to do so. */
-#endif
+#endif /* defined(_WIN32) */
 
   /* XXXX On Linux, we can and should receive the 22nd
    * (space-delimited) field (‘starttime’) of /proc/$PID/stat from the
@@ -136,7 +142,7 @@ struct tor_process_monitor_t {
 
   /** A Libevent event structure, to either poll for the process's
    * existence or receive a notification when the process ends. */
-  struct event *e;
+  periodic_timer_t *e;
 
   /** A callback to be called when the process ends. */
   tor_procmon_callback_t cb;
@@ -159,9 +165,6 @@ tor_validate_process_specifier(const char *process_spec,
   return parse_process_specifier(process_spec, &ppspec, msg);
 }
 
-/* XXXX we should use periodic_timer_new() for this stuff */
-#define PERIODIC_TIMER_FLAGS EV_PERSIST
-
 /* DOCDOC poll_interval_tv */
 static const struct timeval poll_interval_tv = {15, 0};
 
@@ -219,22 +222,18 @@ tor_process_monitor_new(struct event_base *base,
              "try again later.",
              procmon->pid);
   }
-#endif
+#endif /* defined(_WIN32) */
 
   procmon->cb = cb;
   procmon->cb_arg = cb_arg;
 
 #ifdef PROCMON_POLLS
-  procmon->e = tor_event_new(base, -1 /* no FD */, PERIODIC_TIMER_FLAGS,
-                             tor_process_monitor_poll_cb, procmon);
-  /* Note: If you port this file to plain Libevent 2, check that
-   * procmon->e is non-NULL.  We don't need to here because
-   * tor_evtimer_new never returns NULL. */
-
-  evtimer_add(procmon->e, &poll_interval_tv);
-#else
+  procmon->e = periodic_timer_new(base,
+                                  &poll_interval_tv,
+                                  tor_process_monitor_poll_cb, procmon);
+#else /* !(defined(PROCMON_POLLS)) */
 #error OOPS?
-#endif
+#endif /* defined(PROCMON_POLLS) */
 
   return procmon;
  err:
@@ -246,14 +245,12 @@ tor_process_monitor_new(struct event_base *base,
 /** Libevent callback to poll for the existence of the process
  * monitored by <b>procmon_</b>. */
 static void
-tor_process_monitor_poll_cb(evutil_socket_t unused1, short unused2,
-                            void *procmon_)
+tor_process_monitor_poll_cb(periodic_timer_t *event, void *procmon_)
 {
+  (void)event;
   tor_process_monitor_t *procmon = (tor_process_monitor_t *)(procmon_);
   int its_dead_jim;
 
-  (void)unused1; (void)unused2;
-
   tor_assert(procmon != NULL);
 
 #ifdef _WIN32
@@ -306,11 +303,11 @@ tor_process_monitor_poll_cb(evutil_socket_t unused1, short unused2,
       tor_free(errmsg);
     }
   }
-#else
+#else /* !(defined(_WIN32)) */
   /* Unix makes this part easy, if a bit racy. */
   its_dead_jim = kill(procmon->pid, 0);
   its_dead_jim = its_dead_jim && (errno == ESRCH);
-#endif
+#endif /* defined(_WIN32) */
 
   tor_log(its_dead_jim ? LOG_NOTICE : LOG_INFO,
       procmon->log_domain, "Monitored process "PID_T_FORMAT" is %s.",
@@ -321,11 +318,11 @@ tor_process_monitor_poll_cb(evutil_socket_t unused1, short unused2,
     procmon->cb(procmon->cb_arg);
   }
 }
-#endif
+#endif /* defined(PROCMON_POLLS) */
 
 /** Free the process-termination monitor <b>procmon</b>. */
 void
-tor_process_monitor_free(tor_process_monitor_t *procmon)
+tor_process_monitor_free_(tor_process_monitor_t *procmon)
 {
   if (procmon == NULL)
     return;
@@ -336,8 +333,7 @@ tor_process_monitor_free(tor_process_monitor_t *procmon)
 #endif
 
   if (procmon->e != NULL)
-    tor_event_free(procmon->e);
+    periodic_timer_free(procmon->e);
 
   tor_free(procmon);
 }
-
diff --git a/src/common/procmon.h b/src/lib/evloop/procmon.h
index 49ead24092..6caae5be86 100644
--- a/src/common/procmon.h
+++ b/src/lib/evloop/procmon.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2016, The Tor Project, Inc. */
+/* Copyright (c) 2011-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,10 +9,9 @@
 #ifndef TOR_PROCMON_H
 #define TOR_PROCMON_H
 
-#include "compat.h"
-#include "compat_libevent.h"
+#include "lib/evloop/compat_libevent.h"
 
-#include "torlog.h"
+#include "lib/log/log.h"
 
 typedef struct tor_process_monitor_t tor_process_monitor_t;
 
@@ -27,7 +26,9 @@ tor_process_monitor_t *tor_process_monitor_new(struct event_base *base,
                                                tor_procmon_callback_t cb,
                                                void *cb_arg,
                                                const char **msg);
-void tor_process_monitor_free(tor_process_monitor_t *procmon);
+void tor_process_monitor_free_(tor_process_monitor_t *procmon);
+#define tor_process_monitor_free(procmon) \
+  FREE_AND_NULL(tor_process_monitor_t, tor_process_monitor_free_, (procmon))
 
-#endif
+#endif /* !defined(TOR_PROCMON_H) */
 
diff --git a/src/common/timers.c b/src/lib/evloop/timers.c
index 41b2008ac4..e46d2635a8 100644
--- a/src/common/timers.c
+++ b/src/lib/evloop/timers.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,13 +29,24 @@
 
 #include "orconfig.h"
 
-#include "compat.h"
-#include "compat_libevent.h"
-#include "timers.h"
-#include "torlog.h"
-#include "util.h"
+#define TOR_TIMERS_PRIVATE
 
-#include <event2/event.h>
+#include "lib/evloop/compat_libevent.h"
+#include "lib/evloop/timers.h"
+#include "lib/intmath/muldiv.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/time/compat_time.h"
+
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+
+#ifdef _WIN32
+// For struct timeval.
+#include <winsock2.h>
+#endif
 
 struct timeout_cb {
   timer_cb_fn_t cb;
@@ -51,7 +62,7 @@ struct timeout_cb {
 #else
 /* We're not exposing any of the functions outside this file. */
 #define TIMEOUT_PUBLIC static
-#endif
+#endif /* defined(__GNUC__) */
 /* We're not using periodic events. */
 #define TIMEOUT_DISABLE_INTERVALS
 /* We always know the global_timeouts object, so we don't need each timeout
@@ -61,13 +72,18 @@ struct timeout_cb {
 #define TIMEOUT_CB_OVERRIDE
 /* We're going to support timers that are pretty far out in advance. Making
  * this big can be inefficient, but having a significant number of timers
- * above TIMEOUT_MAX can also be super-inefficent. Choosing 5 here sets
+ * above TIMEOUT_MAX can also be super-inefficient. Choosing 5 here sets
  * timeout_max to 2^30 ticks, or 29 hours with our value for USEC_PER_TICK */
 #define WHEEL_NUM 5
+#if SIZEOF_VOID_P == 4
+/* On 32-bit platforms, we want to override wheel_bit, so that timeout.c will
+ * use 32-bit math. */
+#define WHEEL_BIT 5
+#endif
 #include "src/ext/timeouts/timeout.c"
 
 static struct timeouts *global_timeouts = NULL;
-static struct event *global_timer_event = NULL;
+static struct mainloop_event_t *global_timer_event = NULL;
 
 static monotime_t start_of_time;
 
@@ -145,20 +161,14 @@ libevent_timer_reschedule(void)
   if (delay > MIN_CHECK_TICKS)
     delay = MIN_CHECK_TICKS;
   timeout_to_tv(delay, &d);
-  event_add(global_timer_event, &d);
+  mainloop_event_schedule(global_timer_event, &d);
 }
 
-/**
- * Invoked when the libevent timer has expired: see which tor_timer_t events
- * have fired, activate their callbacks, and reschedule the libevent timer.
- */
-static void
-libevent_timer_callback(evutil_socket_t fd, short what, void *arg)
+/** Run the callback of every timer that has expired, based on the current
+ * output of monotime_get(). */
+STATIC void
+timers_run_pending(void)
 {
-  (void)fd;
-  (void)what;
-  (void)arg;
-
   monotime_t now;
   monotime_get(&now);
   timer_advance_to_cur_time(&now);
@@ -167,6 +177,19 @@ libevent_timer_callback(evutil_socket_t fd, short what, void *arg)
   while ((t = timeouts_get(global_timeouts))) {
     t->callback.cb(t, t->callback.arg, &now);
   }
+}
+
+/**
+ * Invoked when the libevent timer has expired: see which tor_timer_t events
+ * have fired, activate their callbacks, and reschedule the libevent timer.
+ */
+static void
+libevent_timer_callback(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  (void)arg;
+
+  timers_run_pending();
 
   libevent_timer_reschedule();
 }
@@ -181,7 +204,7 @@ timers_initialize(void)
   if (BUG(global_timeouts))
     return; // LCOV_EXCL_LINE
 
-  timeout_error_t err;
+  timeout_error_t err = 0;
   global_timeouts = timeouts_open(0, &err);
   if (!global_timeouts) {
     // LCOV_EXCL_START -- this can only fail on malloc failure.
@@ -193,9 +216,8 @@ timers_initialize(void)
   monotime_init();
   monotime_get(&start_of_time);
 
-  struct event *timer_event;
-  timer_event = tor_event_new(tor_libevent_get_base(),
-                              -1, 0, libevent_timer_callback, NULL);
+  mainloop_event_t *timer_event;
+  timer_event = mainloop_event_new(libevent_timer_callback, NULL);
   tor_assert(timer_event);
   global_timer_event = timer_event;
 
@@ -209,7 +231,7 @@ void
 timers_shutdown(void)
 {
   if (global_timer_event) {
-    tor_event_free(global_timer_event);
+    mainloop_event_free(global_timer_event);
     global_timer_event = NULL;
   }
   if (global_timeouts) {
@@ -235,7 +257,7 @@ timer_new(timer_cb_fn_t cb, void *arg)
  * scheduled.
  */
 void
-timer_free(tor_timer_t *t)
+timer_free_(tor_timer_t *t)
 {
   if (! t)
     return;
@@ -255,6 +277,20 @@ timer_set_cb(tor_timer_t *t, timer_cb_fn_t cb, void *arg)
 }
 
 /**
+ * Set *<b>cb_out</b> (if provided) to this timer's callback function,
+ * and *<b>arg_out</b> (if provided) to this timer's callback argument.
+ */
+void
+timer_get_cb(const tor_timer_t *t,
+             timer_cb_fn_t *cb_out, void **arg_out)
+{
+  if (cb_out)
+    *cb_out = t->callback.cb;
+  if (arg_out)
+    *arg_out = t->callback.arg;
+}
+
+/**
  * Schedule the timer t to fire at the current time plus a delay of
  * <b>delay</b> microseconds.  All times are relative to monotime_get().
  */
@@ -290,4 +326,3 @@ timer_disable(tor_timer_t *t)
   /* We don't reschedule the libevent timer here, since it's okay if it fires
    * early. */
 }
-
diff --git a/src/common/timers.h b/src/lib/evloop/timers.h
index 5f918f8e15..7595554204 100644
--- a/src/common/timers.h
+++ b/src/lib/evloop/timers.h
@@ -1,11 +1,16 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * \file timers.h
+ * \brief Header for timers.c
+ **/
+
 #ifndef TOR_TIMERS_H
 #define TOR_TIMERS_H
 
 #include "orconfig.h"
-#include "testsupport.h"
+#include "lib/testsupport/testsupport.h"
 
 struct monotime_t;
 typedef struct timeout tor_timer_t;
@@ -13,12 +18,18 @@ typedef void (*timer_cb_fn_t)(tor_timer_t *, void *,
                               const struct monotime_t *);
 tor_timer_t *timer_new(timer_cb_fn_t cb, void *arg);
 void timer_set_cb(tor_timer_t *t, timer_cb_fn_t cb, void *arg);
+void timer_get_cb(const tor_timer_t *t,
+                  timer_cb_fn_t *cb_out, void **arg_out);
 void timer_schedule(tor_timer_t *t, const struct timeval *delay);
 void timer_disable(tor_timer_t *t);
-void timer_free(tor_timer_t *t);
+void timer_free_(tor_timer_t *t);
+#define timer_free(t) FREE_AND_NULL(tor_timer_t, timer_free_, (t))
 
 void timers_initialize(void);
 void timers_shutdown(void);
 
+#ifdef TOR_TIMERS_PRIVATE
+STATIC void timers_run_pending(void);
 #endif
 
+#endif /* !defined(TOR_TIMERS_H) */
diff --git a/src/lib/evloop/token_bucket.c b/src/lib/evloop/token_bucket.c
new file mode 100644
index 0000000000..ee6d631e3b
--- /dev/null
+++ b/src/lib/evloop/token_bucket.c
@@ -0,0 +1,258 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file token_bucket.c
+ * \brief Functions to use and manipulate token buckets, used for
+ *    rate-limiting on connections and globally.
+ *
+ * Tor uses these token buckets to keep track of bandwidth usage, and
+ * sometimes other things too.
+ *
+ * There are two layers of abstraction here: "raw" token buckets, in which all
+ * the pieces are decoupled, and "read-write" token buckets, which combine all
+ * the moving parts into one.
+ *
+ * Token buckets may become negative.
+ **/
+
+#define TOKEN_BUCKET_PRIVATE
+
+#include "lib/evloop/token_bucket.h"
+#include "lib/log/util_bug.h"
+#include "lib/intmath/cmp.h"
+#include "lib/time/compat_time.h"
+
+#include <string.h>
+
+/**
+ * Set the <b>rate</b> and <b>burst</b> value in a token_bucket_cfg.
+ *
+ * Note that the <b>rate</b> value is in arbitrary units, but those units will
+ * determine the units of token_bucket_raw_dec(), token_bucket_raw_refill, and
+ * so on.
+ */
+void
+token_bucket_cfg_init(token_bucket_cfg_t *cfg,
+                      uint32_t rate,
+                      uint32_t burst)
+{
+  tor_assert_nonfatal(rate > 0);
+  tor_assert_nonfatal(burst > 0);
+  if (burst > TOKEN_BUCKET_MAX_BURST)
+    burst = TOKEN_BUCKET_MAX_BURST;
+
+  cfg->rate = rate;
+  cfg->burst = burst;
+}
+
+/**
+ * Initialize a raw token bucket and its associated timestamp to the "full"
+ * state, according to <b>cfg</b>.
+ */
+void
+token_bucket_raw_reset(token_bucket_raw_t *bucket,
+                       const token_bucket_cfg_t *cfg)
+{
+  bucket->bucket = cfg->burst;
+}
+
+/**
+ * Adust a preexisting token bucket to respect the new configuration
+ * <b>cfg</b>, by decreasing its current level if needed. */
+void
+token_bucket_raw_adjust(token_bucket_raw_t *bucket,
+                        const token_bucket_cfg_t *cfg)
+{
+  bucket->bucket = MIN(bucket->bucket, cfg->burst);
+}
+
+/**
+ * Given an amount of <b>elapsed</b> time units, and a bucket configuration
+ * <b>cfg</b>, refill the level of <b>bucket</b> accordingly.  Note that the
+ * units of time in <b>elapsed</b> must correspond to those used to set the
+ * rate in <b>cfg</b>, or the result will be illogical.
+ */
+int
+token_bucket_raw_refill_steps(token_bucket_raw_t *bucket,
+                              const token_bucket_cfg_t *cfg,
+                              const uint32_t elapsed)
+{
+  const int was_empty = (bucket->bucket <= 0);
+  /* The casts here prevent an underflow.
+   *
+   * Note that even if the bucket value is negative, subtracting it from
+   * "burst" will still produce a correct result.  If this result is
+   * ridiculously high, then the "elapsed > gap / rate" check below
+   * should catch it. */
+  const size_t gap = ((size_t)cfg->burst) - ((size_t)bucket->bucket);
+
+  if (elapsed > gap / cfg->rate) {
+    bucket->bucket = cfg->burst;
+  } else {
+    bucket->bucket += cfg->rate * elapsed;
+  }
+
+  return was_empty && bucket->bucket > 0;
+}
+
+/**
+ * Decrement a provided bucket by <b>n</b> units.  Note that <b>n</b>
+ * must be nonnegative.
+ */
+int
+token_bucket_raw_dec(token_bucket_raw_t *bucket,
+                     ssize_t n)
+{
+  if (BUG(n < 0))
+    return 0;
+  const int becomes_empty = bucket->bucket > 0 && n >= bucket->bucket;
+  bucket->bucket -= n;
+  return becomes_empty;
+}
+
+/** Convert a rate in bytes per second to a rate in bytes per step */
+STATIC uint32_t
+rate_per_sec_to_rate_per_step(uint32_t rate)
+{
+  /*
+    The precise calculation we'd want to do is
+
+    (rate / 1000) * to_approximate_msec(TICKS_PER_STEP).  But to minimize
+    rounding error, we do it this way instead, and divide last.
+  */
+  uint64_t units = (uint64_t) rate * TICKS_PER_STEP;
+  uint32_t val = (uint32_t)
+    (monotime_coarse_stamp_units_to_approx_msec(units) / 1000);
+  return val ? val : 1;
+}
+
+/**
+ * Initialize a token bucket in *<b>bucket</b>, set up to allow <b>rate</b>
+ * bytes per second, with a maximum burst of <b>burst</b> bytes. The bucket
+ * is created such that <b>now_ts</b> is the current timestamp.  The bucket
+ * starts out full.
+ */
+void
+token_bucket_rw_init(token_bucket_rw_t *bucket,
+                     uint32_t rate,
+                     uint32_t burst,
+                     uint32_t now_ts)
+{
+  memset(bucket, 0, sizeof(token_bucket_rw_t));
+  token_bucket_rw_adjust(bucket, rate, burst);
+  token_bucket_rw_reset(bucket, now_ts);
+}
+
+/**
+ * Change the configured rate (in bytes per second) and burst (in bytes)
+ * for the token bucket in *<b>bucket</b>.
+ */
+void
+token_bucket_rw_adjust(token_bucket_rw_t *bucket,
+                       uint32_t rate,
+                       uint32_t burst)
+{
+  token_bucket_cfg_init(&bucket->cfg,
+                        rate_per_sec_to_rate_per_step(rate),
+                        burst);
+  token_bucket_raw_adjust(&bucket->read_bucket, &bucket->cfg);
+  token_bucket_raw_adjust(&bucket->write_bucket, &bucket->cfg);
+}
+
+/**
+ * Reset <b>bucket</b> to be full, as of timestamp <b>now_ts</b>.
+ */
+void
+token_bucket_rw_reset(token_bucket_rw_t *bucket,
+                      uint32_t now_ts)
+{
+  token_bucket_raw_reset(&bucket->read_bucket, &bucket->cfg);
+  token_bucket_raw_reset(&bucket->write_bucket, &bucket->cfg);
+  bucket->last_refilled_at_timestamp = now_ts;
+}
+
+/**
+ * Refill <b>bucket</b> as appropriate, given that the current timestamp
+ * is <b>now_ts</b>.
+ *
+ * Return a bitmask containing TB_READ iff read bucket was empty and became
+ * nonempty, and TB_WRITE iff the write bucket was empty and became nonempty.
+ */
+int
+token_bucket_rw_refill(token_bucket_rw_t *bucket,
+                       uint32_t now_ts)
+{
+  const uint32_t elapsed_ticks =
+    (now_ts - bucket->last_refilled_at_timestamp);
+  if (elapsed_ticks > UINT32_MAX-(300*1000)) {
+    /* Either about 48 days have passed since the last refill, or the
+     * monotonic clock has somehow moved backwards. (We're looking at you,
+     * Windows.).  We accept up to a 5 minute jump backwards as
+     * "unremarkable".
+     */
+    return 0;
+  }
+  const uint32_t elapsed_steps = elapsed_ticks / TICKS_PER_STEP;
+
+  if (!elapsed_steps) {
+    /* Note that if less than one whole step elapsed, we don't advance the
+     * time in last_refilled_at. That's intentional: we want to make sure
+     * that we add some bytes to it eventually. */
+    return 0;
+  }
+
+  int flags = 0;
+  if (token_bucket_raw_refill_steps(&bucket->read_bucket,
+                                    &bucket->cfg, elapsed_steps))
+    flags |= TB_READ;
+  if (token_bucket_raw_refill_steps(&bucket->write_bucket,
+                                    &bucket->cfg, elapsed_steps))
+    flags |= TB_WRITE;
+
+  bucket->last_refilled_at_timestamp = now_ts;
+  return flags;
+}
+
+/**
+ * Decrement the read token bucket in <b>bucket</b> by <b>n</b> bytes.
+ *
+ * Return true if the bucket was nonempty and became empty; return false
+ * otherwise.
+ */
+int
+token_bucket_rw_dec_read(token_bucket_rw_t *bucket,
+                         ssize_t n)
+{
+  return token_bucket_raw_dec(&bucket->read_bucket, n);
+}
+
+/**
+ * Decrement the write token bucket in <b>bucket</b> by <b>n</b> bytes.
+ *
+ * Return true if the bucket was nonempty and became empty; return false
+ * otherwise.
+ */
+int
+token_bucket_rw_dec_write(token_bucket_rw_t *bucket,
+                          ssize_t n)
+{
+  return token_bucket_raw_dec(&bucket->write_bucket, n);
+}
+
+/**
+ * As token_bucket_rw_dec_read and token_bucket_rw_dec_write, in a single
+ * operation.  Return a bitmask of TB_READ and TB_WRITE to indicate
+ * which buckets became empty.
+ */
+int
+token_bucket_rw_dec(token_bucket_rw_t *bucket,
+                    ssize_t n_read, ssize_t n_written)
+{
+  int flags = 0;
+  if (token_bucket_rw_dec_read(bucket, n_read))
+    flags |= TB_READ;
+  if (token_bucket_rw_dec_write(bucket, n_written))
+    flags |= TB_WRITE;
+  return flags;
+}
diff --git a/src/lib/evloop/token_bucket.h b/src/lib/evloop/token_bucket.h
new file mode 100644
index 0000000000..9398d2baa3
--- /dev/null
+++ b/src/lib/evloop/token_bucket.h
@@ -0,0 +1,117 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file token_bucket.h
+ * \brief Headers for token_bucket.c
+ **/
+
+#ifndef TOR_TOKEN_BUCKET_H
+#define TOR_TOKEN_BUCKET_H
+
+#include "lib/cc/torint.h"
+#include "lib/testsupport/testsupport.h"
+
+/** Largest allowable burst value for a token buffer. */
+#define TOKEN_BUCKET_MAX_BURST INT32_MAX
+
+/** A generic token buffer configuration: determines the number of tokens
+ * added to the bucket in each time unit (the "rate"), and the maximum number
+ * of tokens in the bucket (the "burst") */
+typedef struct token_bucket_cfg_t {
+  uint32_t rate;
+  int32_t burst;
+} token_bucket_cfg_t;
+
+/** A raw token bucket, decoupled from its configuration and timestamp. */
+typedef struct token_bucket_raw_t {
+  int32_t bucket;
+} token_bucket_raw_t;
+
+void token_bucket_cfg_init(token_bucket_cfg_t *cfg,
+                           uint32_t rate,
+                           uint32_t burst);
+
+void token_bucket_raw_adjust(token_bucket_raw_t *bucket,
+                             const token_bucket_cfg_t *cfg);
+
+void token_bucket_raw_reset(token_bucket_raw_t *bucket,
+                            const token_bucket_cfg_t *cfg);
+
+int token_bucket_raw_dec(token_bucket_raw_t *bucket,
+                         ssize_t n);
+
+int token_bucket_raw_refill_steps(token_bucket_raw_t *bucket,
+                                  const token_bucket_cfg_t *cfg,
+                                  const uint32_t elapsed_steps);
+
+static inline size_t token_bucket_raw_get(const token_bucket_raw_t *bucket);
+/** Return the current number of bytes set in a token bucket. */
+static inline size_t
+token_bucket_raw_get(const token_bucket_raw_t *bucket)
+{
+  return bucket->bucket >= 0 ? bucket->bucket : 0;
+}
+
+/** A convenience type containing all the pieces needed for a coupled
+ * read-bucket and write-bucket that have the same rate limit, and which use
+ * "timestamp units" (see compat_time.h) for their time. */
+typedef struct token_bucket_rw_t {
+  token_bucket_cfg_t cfg;
+  token_bucket_raw_t read_bucket;
+  token_bucket_raw_t write_bucket;
+  uint32_t last_refilled_at_timestamp;
+} token_bucket_rw_t;
+
+void token_bucket_rw_init(token_bucket_rw_t *bucket,
+                          uint32_t rate,
+                          uint32_t burst,
+                          uint32_t now_ts);
+
+void token_bucket_rw_adjust(token_bucket_rw_t *bucket,
+                            uint32_t rate, uint32_t burst);
+
+void token_bucket_rw_reset(token_bucket_rw_t *bucket,
+                           uint32_t now_ts);
+
+#define TB_READ 1
+#define TB_WRITE 2
+
+int token_bucket_rw_refill(token_bucket_rw_t *bucket,
+                           uint32_t now_ts);
+
+int token_bucket_rw_dec_read(token_bucket_rw_t *bucket,
+                             ssize_t n);
+int token_bucket_rw_dec_write(token_bucket_rw_t *bucket,
+                              ssize_t n);
+
+int token_bucket_rw_dec(token_bucket_rw_t *bucket,
+                        ssize_t n_read, ssize_t n_written);
+
+static inline size_t token_bucket_rw_get_read(const token_bucket_rw_t *bucket);
+static inline size_t
+token_bucket_rw_get_read(const token_bucket_rw_t *bucket)
+{
+  return token_bucket_raw_get(&bucket->read_bucket);
+}
+
+static inline size_t token_bucket_rw_get_write(
+                                            const token_bucket_rw_t *bucket);
+static inline size_t
+token_bucket_rw_get_write(const token_bucket_rw_t *bucket)
+{
+  return token_bucket_raw_get(&bucket->write_bucket);
+}
+
+#ifdef TOKEN_BUCKET_PRIVATE
+
+/* To avoid making the rates too small, we consider units of "steps",
+ * where a "step" is defined as this many timestamp ticks.  Keep this
+ * a power of two if you can. */
+#define TICKS_PER_STEP 16
+
+STATIC uint32_t rate_per_sec_to_rate_per_step(uint32_t rate);
+
+#endif
+
+#endif /* TOR_TOKEN_BUCKET_H */
diff --git a/src/common/workqueue.c b/src/lib/evloop/workqueue.c
index e1fb663a2a..931f65e710 100644
--- a/src/common/workqueue.c
+++ b/src/lib/evloop/workqueue.c
@@ -1,3 +1,4 @@
+
 /* copyright (c) 2013-2015, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
@@ -23,12 +24,28 @@
  */
 
 #include "orconfig.h"
-#include "compat.h"
-#include "compat_threads.h"
-#include "util.h"
-#include "workqueue.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/evloop/workqueue.h"
+
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/intmath/weakrng.h"
+#include "lib/log/ratelim.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/net/alertsock.h"
+#include "lib/net/socket.h"
+#include "lib/thread/threads.h"
+
 #include "tor_queue.h"
-#include "torlog.h"
+#include <event2/event.h>
+#include <string.h>
+
+#define WORKQUEUE_PRIORITY_FIRST WQ_PRI_HIGH
+#define WORKQUEUE_PRIORITY_LAST WQ_PRI_LOW
+#define WORKQUEUE_N_PRIORITIES (((int) WORKQUEUE_PRIORITY_LAST)+1)
+
+TOR_TAILQ_HEAD(work_tailq_t, workqueue_entry_s);
+typedef struct work_tailq_t work_tailq_t;
 
 struct threadpool_s {
   /** An array of pointers to workerthread_t: one for each running worker
@@ -38,8 +55,12 @@ struct threadpool_s {
   /** Condition variable that we wait on when we have no work, and which
    * gets signaled when our queue becomes nonempty. */
   tor_cond_t condition;
-  /** Queue of pending work that we have to do. */
-  TOR_TAILQ_HEAD(, workqueue_entry_s) work;
+  /** Queues of pending work that we have to do. The queue with priority
+   * <b>p</b> is work[p]. */
+  work_tailq_t work[WORKQUEUE_N_PRIORITIES];
+
+  /** Weak RNG, used to decide when to ignore priority. */
+  tor_weak_rng_t weak_rng;
 
   /** The current 'update generation' of the threadpool.  Any thread that is
    * at an earlier generation needs to run the update function. */
@@ -51,6 +72,9 @@ struct threadpool_s {
   void (*free_update_arg_fn)(void *);
   /** Array of n_threads update arguments. */
   void **update_args;
+  /** Event to notice when another thread has sent a reply. */
+  struct event *reply_event;
+  void (*reply_cb)(threadpool_t *);
 
   /** Number of elements in threads. */
   int n_threads;
@@ -66,6 +90,11 @@ struct threadpool_s {
   void *new_thread_state_arg;
 };
 
+/** Used to put a workqueue_priority_t value into a bitfield. */
+#define workqueue_priority_bitfield_t ENUM_BF(workqueue_priority_t)
+/** Number of bits needed to hold all legal values of workqueue_priority_t */
+#define WORKQUEUE_PRIORITY_BITS 2
+
 struct workqueue_entry_s {
   /** The next workqueue_entry_t that's pending on the same thread or
    * reply queue. */
@@ -76,6 +105,8 @@ struct workqueue_entry_s {
   struct threadpool_s *on_pool;
   /** True iff this entry is waiting for a worker to start processing it. */
   uint8_t pending;
+  /** Priority of this entry. */
+  workqueue_priority_bitfield_t priority : WORKQUEUE_PRIORITY_BITS;
   /** Function to run in the worker thread. */
   workqueue_reply_t (*fn)(void *state, void *arg);
   /** Function to run while processing the reply queue. */
@@ -94,9 +125,7 @@ struct replyqueue_s {
   alert_sockets_t alert;
 };
 
-/** A worker thread represents a single thread in a thread pool.  To avoid
- * contention, each gets its own queue. This breaks the guarantee that that
- * queued work will get executed strictly in order. */
+/** A worker thread represents a single thread in a thread pool. */
 typedef struct workerthread_s {
   /** Which thread it this?  In range 0..in_pool->n_threads-1 */
   int index;
@@ -109,6 +138,8 @@ typedef struct workerthread_s {
   replyqueue_t *reply_queue;
   /** The current update generation of this thread */
   unsigned generation;
+  /** One over the probability of taking work from a lower-priority queue. */
+  int32_t lower_priority_chance;
 } workerthread_t;
 
 static void queue_reply(replyqueue_t *queue, workqueue_entry_t *work);
@@ -125,15 +156,19 @@ workqueue_entry_new(workqueue_reply_t (*fn)(void*, void*),
   ent->fn = fn;
   ent->reply_fn = reply_fn;
   ent->arg = arg;
+  ent->priority = WQ_PRI_HIGH;
   return ent;
 }
 
+#define workqueue_entry_free(ent) \
+  FREE_AND_NULL(workqueue_entry_t, workqueue_entry_free_, (ent))
+
 /**
  * Release all storage held in <b>ent</b>. Call only when <b>ent</b> is not on
  * any queue.
  */
 static void
-workqueue_entry_free(workqueue_entry_t *ent)
+workqueue_entry_free_(workqueue_entry_t *ent)
 {
   if (!ent)
     return;
@@ -161,8 +196,9 @@ workqueue_entry_cancel(workqueue_entry_t *ent)
   int cancelled = 0;
   void *result = NULL;
   tor_mutex_acquire(&ent->on_pool->lock);
+  workqueue_priority_t prio = ent->priority;
   if (ent->pending) {
-    TOR_TAILQ_REMOVE(&ent->on_pool->work, ent, next_work);
+    TOR_TAILQ_REMOVE(&ent->on_pool->work[prio], ent, next_work);
     cancelled = 1;
     result = ent->arg;
   }
@@ -180,8 +216,46 @@ workqueue_entry_cancel(workqueue_entry_t *ent)
 static int
 worker_thread_has_work(workerthread_t *thread)
 {
-  return !TOR_TAILQ_EMPTY(&thread->in_pool->work) ||
-    thread->generation != thread->in_pool->generation;
+  unsigned i;
+  for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) {
+    if (!TOR_TAILQ_EMPTY(&thread->in_pool->work[i]))
+        return 1;
+  }
+  return thread->generation != thread->in_pool->generation;
+}
+
+/** Extract the next workqueue_entry_t from the the thread's pool, removing
+ * it from the relevant queues and marking it as non-pending.
+ *
+ * The caller must hold the lock. */
+static workqueue_entry_t *
+worker_thread_extract_next_work(workerthread_t *thread)
+{
+  threadpool_t *pool = thread->in_pool;
+  work_tailq_t *queue = NULL, *this_queue;
+  unsigned i;
+  for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) {
+    this_queue = &pool->work[i];
+    if (!TOR_TAILQ_EMPTY(this_queue)) {
+      queue = this_queue;
+      if (! tor_weak_random_one_in_n(&pool->weak_rng,
+                                     thread->lower_priority_chance)) {
+        /* Usually we'll just break now, so that we can get out of the loop
+         * and use the queue where we found work. But with a small
+         * probability, we'll keep looking for lower priority work, so that
+         * we don't ignore our low-priority queues entirely. */
+        break;
+      }
+    }
+  }
+
+  if (queue == NULL)
+    return NULL;
+
+  workqueue_entry_t *work = TOR_TAILQ_FIRST(queue);
+  TOR_TAILQ_REMOVE(queue, work, next_work);
+  work->pending = 0;
+  return work;
 }
 
 /**
@@ -217,9 +291,9 @@ worker_thread_main(void *thread_)
         tor_mutex_acquire(&pool->lock);
         continue;
       }
-      work = TOR_TAILQ_FIRST(&pool->work);
-      TOR_TAILQ_REMOVE(&pool->work, work, next_work);
-      work->pending = 0;
+      work = worker_thread_extract_next_work(thread);
+      if (BUG(work == NULL))
+        break;
       tor_mutex_release(&pool->lock);
 
       /* We run the work function without holding the thread lock. This
@@ -268,12 +342,14 @@ queue_reply(replyqueue_t *queue, workqueue_entry_t *work)
 /** Allocate and start a new worker thread to use state object <b>state</b>,
  * and send responses to <b>replyqueue</b>. */
 static workerthread_t *
-workerthread_new(void *state, threadpool_t *pool, replyqueue_t *replyqueue)
+workerthread_new(int32_t lower_priority_chance,
+                 void *state, threadpool_t *pool, replyqueue_t *replyqueue)
 {
   workerthread_t *thr = tor_malloc_zero(sizeof(workerthread_t));
   thr->state = state;
   thr->reply_queue = replyqueue;
   thr->in_pool = pool;
+  thr->lower_priority_chance = lower_priority_chance;
 
   if (spawn_func(worker_thread_main, thr) < 0) {
     //LCOV_EXCL_START
@@ -299,24 +375,34 @@ workerthread_new(void *state, threadpool_t *pool, replyqueue_t *replyqueue)
  * function's responsibility to free the work object.
  *
  * On success, return a workqueue_entry_t object that can be passed to
- * workqueue_entry_cancel(). On failure, return NULL.
+ * workqueue_entry_cancel(). On failure, return NULL.  (Failure is not
+ * currently possible, but callers should check anyway.)
+ *
+ * Items are executed in a loose priority order -- each thread will usually
+ * take from the queued work with the highest prioirity, but will occasionally
+ * visit lower-priority queues to keep them from starving completely.
  *
- * Note that because each thread has its own work queue, work items may not
+ * Note that because of priorities and thread behavior, work items may not
  * be executed strictly in order.
  */
 workqueue_entry_t *
-threadpool_queue_work(threadpool_t *pool,
-                      workqueue_reply_t (*fn)(void *, void *),
-                      void (*reply_fn)(void *),
-                      void *arg)
+threadpool_queue_work_priority(threadpool_t *pool,
+                               workqueue_priority_t prio,
+                               workqueue_reply_t (*fn)(void *, void *),
+                               void (*reply_fn)(void *),
+                               void *arg)
 {
+  tor_assert(((int)prio) >= WORKQUEUE_PRIORITY_FIRST &&
+             ((int)prio) <= WORKQUEUE_PRIORITY_LAST);
+
   workqueue_entry_t *ent = workqueue_entry_new(fn, reply_fn, arg);
   ent->on_pool = pool;
   ent->pending = 1;
+  ent->priority = prio;
 
   tor_mutex_acquire(&pool->lock);
 
-  TOR_TAILQ_INSERT_TAIL(&pool->work, ent, next_work);
+  TOR_TAILQ_INSERT_TAIL(&pool->work[prio], ent, next_work);
 
   tor_cond_signal_one(&pool->condition);
 
@@ -325,6 +411,16 @@ threadpool_queue_work(threadpool_t *pool,
   return ent;
 }
 
+/** As threadpool_queue_work_priority(), but assumes WQ_PRI_HIGH */
+workqueue_entry_t *
+threadpool_queue_work(threadpool_t *pool,
+                      workqueue_reply_t (*fn)(void *, void *),
+                      void (*reply_fn)(void *),
+                      void *arg)
+{
+  return threadpool_queue_work_priority(pool, WQ_PRI_HIGH, fn, reply_fn, arg);
+}
+
 /**
  * Queue a copy of a work item for every thread in a pool.  This can be used,
  * for example, to tell the threads to update some parameter in their states.
@@ -388,6 +484,14 @@ threadpool_queue_update(threadpool_t *pool,
 /** Don't have more than this many threads per pool. */
 #define MAX_THREADS 1024
 
+/** For half of our threads, choose lower priority queues with probability
+ * 1/N for each of these values. Both are chosen somewhat arbitrarily.  If
+ * CHANCE_PERMISSIVE is too low, then we have a risk of low-priority tasks
+ * stalling forever.  If it's too high, we have a risk of low-priority tasks
+ * grabbing half of the threads. */
+#define CHANCE_PERMISSIVE 37
+#define CHANCE_STRICT INT32_MAX
+
 /** Launch threads until we have <b>n</b>. */
 static int
 threadpool_start_threads(threadpool_t *pool, int n)
@@ -404,8 +508,14 @@ threadpool_start_threads(threadpool_t *pool, int n)
                                      sizeof(workerthread_t*), n);
 
   while (pool->n_threads < n) {
+    /* For half of our threads, we'll choose lower priorities permissively;
+     * for the other half, we'll stick more strictly to higher priorities.
+     * This keeps slow low-priority tasks from taking over completely. */
+    int32_t chance = (pool->n_threads & 1) ? CHANCE_STRICT : CHANCE_PERMISSIVE;
+
     void *state = pool->new_thread_state_fn(pool->new_thread_state_arg);
-    workerthread_t *thr = workerthread_new(state, pool, pool->reply_queue);
+    workerthread_t *thr = workerthread_new(chance,
+                                           state, pool, pool->reply_queue);
 
     if (!thr) {
       //LCOV_EXCL_START
@@ -441,7 +551,15 @@ threadpool_new(int n_threads,
   pool = tor_malloc_zero(sizeof(threadpool_t));
   tor_mutex_init_nonrecursive(&pool->lock);
   tor_cond_init(&pool->condition);
-  TOR_TAILQ_INIT(&pool->work);
+  unsigned i;
+  for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) {
+    TOR_TAILQ_INIT(&pool->work[i]);
+  }
+  {
+    unsigned seed;
+    crypto_rand((void*)&seed, sizeof(seed));
+    tor_init_weak_random(&pool->weak_rng, seed);
+  }
 
   pool->new_thread_state_fn = new_thread_state_fn;
   pool->new_thread_state_arg = arg;
@@ -491,15 +609,41 @@ replyqueue_new(uint32_t alertsocks_flags)
   return rq;
 }
 
-/**
- * Return the "read socket" for a given reply queue.  The main thread should
- * listen for read events on this socket, and call replyqueue_process() every
- * time it triggers.
+/** Internal: Run from the libevent mainloop when there is work to handle in
+ * the reply queue handler. */
+static void
+reply_event_cb(evutil_socket_t sock, short events, void *arg)
+{
+  threadpool_t *tp = arg;
+  (void) sock;
+  (void) events;
+  replyqueue_process(tp->reply_queue);
+  if (tp->reply_cb)
+    tp->reply_cb(tp);
+}
+
+/** Register the threadpool <b>tp</b>'s reply queue with the libevent
+ * mainloop of <b>base</b>. If <b>tp</b> is provided, it is run after
+ * each time there is work to process from the reply queue. Return 0 on
+ * success, -1 on failure.
  */
-tor_socket_t
-replyqueue_get_socket(replyqueue_t *rq)
+int
+threadpool_register_reply_event(threadpool_t *tp,
+                                void (*cb)(threadpool_t *tp))
 {
-  return rq->alert.read_fd;
+  struct event_base *base = tor_libevent_get_base();
+
+  if (tp->reply_event) {
+    tor_event_free(tp->reply_event);
+  }
+  tp->reply_event = tor_event_new(base,
+                                  tp->reply_queue->alert.read_fd,
+                                  EV_READ|EV_PERSIST,
+                                  reply_event_cb,
+                                  tp);
+  tor_assert(tp->reply_event);
+  tp->reply_cb = cb;
+  return event_add(tp->reply_event, NULL);
 }
 
 /**
@@ -510,12 +654,13 @@ replyqueue_get_socket(replyqueue_t *rq)
 void
 replyqueue_process(replyqueue_t *queue)
 {
-  if (queue->alert.drain_fn(queue->alert.read_fd) < 0) {
+  int r = queue->alert.drain_fn(queue->alert.read_fd);
+  if (r < 0) {
     //LCOV_EXCL_START
     static ratelim_t warn_limit = RATELIM_INIT(7200);
     log_fn_ratelim(&warn_limit, LOG_WARN, LD_GENERAL,
                  "Failure from drain_fd: %s",
-                 tor_socket_strerror(tor_socket_errno(queue->alert.read_fd)));
+                   tor_socket_strerror(-r));
     //LCOV_EXCL_STOP
   }
 
@@ -535,4 +680,3 @@ replyqueue_process(replyqueue_t *queue)
 
   tor_mutex_release(&queue->lock);
 }
-
diff --git a/src/common/workqueue.h b/src/lib/evloop/workqueue.h
index 54276767b0..333a3f6dde 100644
--- a/src/common/workqueue.h
+++ b/src/lib/evloop/workqueue.h
@@ -1,10 +1,15 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * \file workqueue.h
+ * \brief Header for workqueue.c
+ **/
+
 #ifndef TOR_WORKQUEUE_H
 #define TOR_WORKQUEUE_H
 
-#include "compat.h"
+#include "lib/cc/torint.h"
 
 /** A replyqueue is used to tell the main thread about the outcome of
  * work that we queued for the workers. */
@@ -16,12 +21,26 @@ typedef struct threadpool_s threadpool_t;
 typedef struct workqueue_entry_s workqueue_entry_t;
 
 /** Possible return value from a work function: */
-typedef enum {
+typedef enum workqueue_reply_t {
   WQ_RPL_REPLY = 0, /** indicates success */
   WQ_RPL_ERROR = 1, /** indicates fatal error */
   WQ_RPL_SHUTDOWN = 2, /** indicates thread is shutting down */
 } workqueue_reply_t;
 
+/** Possible priorities for work.  Lower numeric values are more important. */
+typedef enum workqueue_priority_t {
+  WQ_PRI_HIGH = 0,
+  WQ_PRI_MED  = 1,
+  WQ_PRI_LOW  = 2,
+} workqueue_priority_t;
+
+workqueue_entry_t *threadpool_queue_work_priority(threadpool_t *pool,
+                                    workqueue_priority_t prio,
+                                    workqueue_reply_t (*fn)(void *,
+                                                            void *),
+                                    void (*reply_fn)(void *),
+                                    void *arg);
+
 workqueue_entry_t *threadpool_queue_work(threadpool_t *pool,
                                          workqueue_reply_t (*fn)(void *,
                                                                  void *),
@@ -42,8 +61,10 @@ threadpool_t *threadpool_new(int n_threads,
 replyqueue_t *threadpool_get_replyqueue(threadpool_t *tp);
 
 replyqueue_t *replyqueue_new(uint32_t alertsocks_flags);
-tor_socket_t replyqueue_get_socket(replyqueue_t *rq);
 void replyqueue_process(replyqueue_t *queue);
 
-#endif
+struct event_base;
+int threadpool_register_reply_event(threadpool_t *tp,
+                                    void (*cb)(threadpool_t *tp));
 
+#endif /* !defined(TOR_WORKQUEUE_H) */
diff --git a/src/lib/fdio/.may_include b/src/lib/fdio/.may_include
new file mode 100644
index 0000000000..30fd277b81
--- /dev/null
+++ b/src/lib/fdio/.may_include
@@ -0,0 +1,4 @@
+orconfig.h
+lib/cc/*.h
+lib/err/*.h
+lib/fdio/*.h
diff --git a/src/lib/fdio/fdio.c b/src/lib/fdio/fdio.c
new file mode 100644
index 0000000000..6c87af791d
--- /dev/null
+++ b/src/lib/fdio/fdio.c
@@ -0,0 +1,115 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file fdio.c
+ *
+ * \brief Low-level compatibility wrappers for fd-based IO.
+ **/
+
+#include "orconfig.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+#include "lib/fdio/fdio.h"
+#include "lib/cc/torint.h"
+#include "lib/err/torerr.h"
+
+#include <stdlib.h>
+
+/** @{ */
+/** Some old versions of Unix didn't define constants for these values,
+ * and instead expect you to say 0, 1, or 2. */
+#ifndef SEEK_SET
+#define SEEK_SET 0
+#endif
+#ifndef SEEK_CUR
+#define SEEK_CUR 1
+#endif
+#ifndef SEEK_END
+#define SEEK_END 2
+#endif
+/** @} */
+
+/** Return the position of <b>fd</b> with respect to the start of the file. */
+off_t
+tor_fd_getpos(int fd)
+{
+#ifdef _WIN32
+  return (off_t) _lseek(fd, 0, SEEK_CUR);
+#else
+  return (off_t) lseek(fd, 0, SEEK_CUR);
+#endif
+}
+
+/** Move <b>fd</b> to the end of the file. Return -1 on error, 0 on success.
+ * If the file is a pipe, do nothing and succeed.
+ **/
+int
+tor_fd_seekend(int fd)
+{
+#ifdef _WIN32
+  return _lseek(fd, 0, SEEK_END) < 0 ? -1 : 0;
+#else
+  off_t rc = lseek(fd, 0, SEEK_END) < 0 ? -1 : 0;
+#ifdef ESPIPE
+  /* If we get an error and ESPIPE, then it's a pipe or a socket of a fifo:
+   * no need to worry. */
+  if (rc < 0 && errno == ESPIPE)
+    rc = 0;
+#endif /* defined(ESPIPE) */
+  return (rc < 0) ? -1 : 0;
+#endif /* defined(_WIN32) */
+}
+
+/** Move <b>fd</b> to position <b>pos</b> in the file. Return -1 on error, 0
+ * on success. */
+int
+tor_fd_setpos(int fd, off_t pos)
+{
+#ifdef _WIN32
+  return _lseek(fd, pos, SEEK_SET) < 0 ? -1 : 0;
+#else
+  return lseek(fd, pos, SEEK_SET) < 0 ? -1 : 0;
+#endif
+}
+
+/** Replacement for ftruncate(fd, 0): move to the front of the file and remove
+ * all the rest of the file. Return -1 on error, 0 on success. */
+int
+tor_ftruncate(int fd)
+{
+  /* Rumor has it that some versions of ftruncate do not move the file pointer.
+   */
+  if (tor_fd_setpos(fd, 0) < 0)
+    return -1;
+
+#ifdef _WIN32
+  return _chsize(fd, 0);
+#else
+  return ftruncate(fd, 0);
+#endif
+}
+
+/** Minimal version of write_all, for use by logging. */
+int
+write_all_to_fd_minimal(int fd, const char *buf, size_t count)
+{
+  size_t written = 0;
+  raw_assert(count < SSIZE_MAX);
+
+  while (written < count) {
+    ssize_t result = write(fd, buf+written, count-written);
+    if (result<0)
+      return -1;
+    written += result;
+  }
+  return 0;
+}
diff --git a/src/lib/fdio/fdio.h b/src/lib/fdio/fdio.h
new file mode 100644
index 0000000000..8395af353b
--- /dev/null
+++ b/src/lib/fdio/fdio.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file fdio.h
+ *
+ * \brief Header for fdio.c
+ **/
+
+#ifndef TOR_FDIO_H
+#define TOR_FDIO_H
+
+#include <stddef.h>
+
+off_t tor_fd_getpos(int fd);
+int tor_fd_setpos(int fd, off_t pos);
+int tor_fd_seekend(int fd);
+int tor_ftruncate(int fd);
+int write_all_to_fd_minimal(int fd, const char *buf, size_t count);
+
+#endif /* !defined(TOR_FDIO_H) */
diff --git a/src/lib/fdio/include.am b/src/lib/fdio/include.am
new file mode 100644
index 0000000000..6c18f00a0d
--- /dev/null
+++ b/src/lib/fdio/include.am
@@ -0,0 +1,17 @@
+
+noinst_LIBRARIES += src/lib/libtor-fdio.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-fdio-testing.a
+endif
+
+src_lib_libtor_fdio_a_SOURCES =			\
+	src/lib/fdio/fdio.c
+
+src_lib_libtor_fdio_testing_a_SOURCES = \
+	$(src_lib_libtor_fdio_a_SOURCES)
+src_lib_libtor_fdio_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_fdio_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/fdio/fdio.h
diff --git a/src/lib/fs/.may_include b/src/lib/fs/.may_include
new file mode 100644
index 0000000000..b1e49fc891
--- /dev/null
+++ b/src/lib/fs/.may_include
@@ -0,0 +1,16 @@
+orconfig.h
+
+ext/getdelim.c
+
+lib/cc/*.h
+lib/container/*.h
+lib/encoding/*.h
+lib/err/*.h
+lib/fdio/*.h
+lib/fs/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/memarea/*.h
+lib/sandbox/*.h
+lib/string/*.h
+lib/testsupport/testsupport.h
diff --git a/src/lib/fs/conffile.c b/src/lib/fs/conffile.c
new file mode 100644
index 0000000000..0d5d56b335
--- /dev/null
+++ b/src/lib/fs/conffile.c
@@ -0,0 +1,176 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file conffile.h
+ *
+ * \brief Read configuration files from disk, with full `%include` support.
+ **/
+
+#include "lib/fs/conffile.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/encoding/confline.h"
+#include "lib/fs/dir.h"
+#include "lib/fs/files.h"
+#include "lib/fs/path.h"
+#include "lib/log/log.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+static smartlist_t *config_get_file_list(const char *path,
+                                         smartlist_t *opened_files);
+static int config_get_included_config(const char *path, int recursion_level,
+                                      int extended, config_line_t **config,
+                                      config_line_t **config_last,
+                                      smartlist_t *opened_lst);
+static int config_process_include(const char *path, int recursion_level,
+                                  int extended, config_line_t **list,
+                                  config_line_t **list_last,
+                                  smartlist_t *opened_lst);
+
+/** Helper: parse the config string and strdup into key/value
+ * strings. Set *result to the list, or NULL if parsing the string
+ * failed. Set *has_include to 1 if <b>result</b> has values from
+ * %included files. <b>opened_lst</b> will have a list of opened files if
+ * provided. Return 0 on success, -1 on failure. Warn and ignore any
+ * misformatted lines.
+ *
+ * If <b>extended</b> is set, then treat keys beginning with / and with + as
+ * indicating "clear" and "append" respectively. */
+int
+config_get_lines_include(const char *string, config_line_t **result,
+                         int extended, int *has_include,
+                         smartlist_t *opened_lst)
+{
+  return config_get_lines_aux(string, result, extended, 1, has_include,
+                              opened_lst, 1, NULL, config_process_include);
+}
+
+/** Adds a list of configuration files present on <b>path</b> to
+ * <b>file_list</b>. <b>path</b> can be a file or a directory. If it is a file,
+ * only that file will be added to <b>file_list</b>. If it is a directory,
+ * all paths for files on that directory root (no recursion) except for files
+ * whose name starts with a dot will be added to <b>file_list</b>.
+ * <b>opened_files</b> will have a list of files opened by this function
+ * if provided. Return 0 on success, -1 on failure. Ignores empty files.
+ */
+static smartlist_t *
+config_get_file_list(const char *path, smartlist_t *opened_files)
+{
+  smartlist_t *file_list = smartlist_new();
+
+  if (opened_files) {
+    smartlist_add_strdup(opened_files, path);
+  }
+
+  file_status_t file_type = file_status(path);
+  if (file_type == FN_FILE) {
+    smartlist_add_strdup(file_list, path);
+    return file_list;
+  } else if (file_type == FN_DIR) {
+    smartlist_t *all_files = tor_listdir(path);
+    if (!all_files) {
+      smartlist_free(file_list);
+      return NULL;
+    }
+    smartlist_sort_strings(all_files);
+    SMARTLIST_FOREACH_BEGIN(all_files, char *, f) {
+      if (f[0] == '.') {
+        tor_free(f);
+        continue;
+      }
+
+      char *fullname;
+      tor_asprintf(&fullname, "%s"PATH_SEPARATOR"%s", path, f);
+      tor_free(f);
+
+      if (opened_files) {
+        smartlist_add_strdup(opened_files, fullname);
+      }
+
+      if (file_status(fullname) != FN_FILE) {
+        tor_free(fullname);
+        continue;
+      }
+      smartlist_add(file_list, fullname);
+    } SMARTLIST_FOREACH_END(f);
+    smartlist_free(all_files);
+    return file_list;
+  } else if (file_type == FN_EMPTY) {
+      return file_list;
+  } else {
+    smartlist_free(file_list);
+    return NULL;
+  }
+}
+
+/** Creates a list of config lines present on included <b>path</b>.
+ * Set <b>config</b> to the list and <b>config_last</b> to the last element of
+ * <b>config</b>. <b>opened_lst</b> will have a list of opened files if
+ * provided. Return 0 on success, -1 on failure. */
+static int
+config_get_included_config(const char *path, int recursion_level, int extended,
+                           config_line_t **config, config_line_t **config_last,
+                           smartlist_t *opened_lst)
+{
+  char *included_conf = read_file_to_str(path, 0, NULL);
+  if (!included_conf) {
+    return -1;
+  }
+
+  if (config_get_lines_aux(included_conf, config, extended, 1, NULL,
+                           opened_lst, recursion_level+1, config_last,
+                           config_process_include) < 0) {
+    tor_free(included_conf);
+    return -1;
+  }
+
+  tor_free(included_conf);
+  return 0;
+}
+
+/** Process an %include <b>path</b> in a config file. Set <b>list</b> to the
+ * list of configuration settings obtained and <b>list_last</b> to the last
+ * element of the same list. <b>opened_lst</b> will have a list of opened
+ * files if provided. Return 0 on success, -1 on failure. */
+static int
+config_process_include(const char *path, int recursion_level, int extended,
+                       config_line_t **list, config_line_t **list_last,
+                       smartlist_t *opened_lst)
+{
+  config_line_t *ret_list = NULL;
+  config_line_t **next = &ret_list;
+
+  smartlist_t *config_files = config_get_file_list(path, opened_lst);
+  if (!config_files) {
+    return -1;
+  }
+
+  int rv = -1;
+  SMARTLIST_FOREACH_BEGIN(config_files, const char *, config_file) {
+    config_line_t *included_config = NULL;
+    config_line_t *included_config_last = NULL;
+    if (config_get_included_config(config_file, recursion_level, extended,
+                                   &included_config, &included_config_last,
+                                   opened_lst) < 0) {
+      goto done;
+    }
+
+    *next = included_config;
+    if (included_config_last) {
+      next = &included_config_last->next;
+      *list_last = included_config_last;
+    }
+  } SMARTLIST_FOREACH_END(config_file);
+  *list = ret_list;
+  rv = 0;
+
+ done:
+  SMARTLIST_FOREACH(config_files, char *, f, tor_free(f));
+  smartlist_free(config_files);
+  return rv;
+}
diff --git a/src/lib/fs/conffile.h b/src/lib/fs/conffile.h
new file mode 100644
index 0000000000..7af9119dbb
--- /dev/null
+++ b/src/lib/fs/conffile.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_CONFFILE_H
+#define TOR_CONFFILE_H
+
+/**
+ * \file conffile.h
+ *
+ * \brief Header for conffile.c
+ **/
+
+struct smartlist_t;
+struct config_line_t;
+
+int config_get_lines_include(const char *string, struct config_line_t **result,
+                             int extended, int *has_include,
+                             struct smartlist_t *opened_lst);
+
+#endif /* !defined(TOR_CONFLINE_H) */
diff --git a/src/lib/fs/dir.c b/src/lib/fs/dir.c
new file mode 100644
index 0000000000..3c31e00d99
--- /dev/null
+++ b/src/lib/fs/dir.c
@@ -0,0 +1,367 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dir.c
+ *
+ * \brief Read directories, and create directories with restrictive
+ * permissions.
+ **/
+
+#include "lib/fs/dir.h"
+#include "lib/fs/path.h"
+#include "lib/fs/userdb.h"
+
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/log/win32err.h"
+#include "lib/container/smartlist.h"
+#include "lib/sandbox/sandbox.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+#include "lib/string/compat_string.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+#ifdef _WIN32
+#include <io.h>
+#include <direct.h>
+#include <windows.h>
+#else /* !(defined(_WIN32)) */
+#include <dirent.h>
+#include <pwd.h>
+#include <grp.h>
+#endif /* defined(_WIN32) */
+
+#include <errno.h>
+#include <string.h>
+
+/** Check whether <b>dirname</b> exists and is private.  If yes return 0.
+ * If <b>dirname</b> does not exist:
+ *  - if <b>check</b>&CPD_CREATE, try to create it and return 0 on success.
+ *  - if <b>check</b>&CPD_CHECK, and we think we can create it, return 0.
+ *  - if <b>check</b>&CPD_CHECK is false, and the directory exists, return 0.
+ *  - otherwise, return -1.
+ * If CPD_GROUP_OK is set, then it's okay if the directory
+ * is group-readable, but in all cases we create the directory mode 0700.
+ * If CPD_GROUP_READ is set, existing directory behaves as CPD_GROUP_OK and
+ * if the directory is created it will use mode 0750 with group read
+ * permission. Group read privileges also assume execute permission
+ * as norm for directories. If CPD_CHECK_MODE_ONLY is set, then we don't
+ * alter the directory permissions if they are too permissive:
+ * we just return -1.
+ * When effective_user is not NULL, check permissions against the given user
+ * and its primary group.
+ */
+MOCK_IMPL(int,
+check_private_dir,(const char *dirname, cpd_check_t check,
+                   const char *effective_user))
+{
+  int r;
+  struct stat st;
+
+  tor_assert(dirname);
+
+#ifndef _WIN32
+  int fd;
+  const struct passwd *pw = NULL;
+  uid_t running_uid;
+  gid_t running_gid;
+
+  /*
+   * Goal is to harden the implementation by removing any
+   * potential for race between stat() and chmod().
+   * chmod() accepts filename as argument. If an attacker can move
+   * the file between stat() and chmod(), a potential race exists.
+   *
+   * Several suggestions taken from:
+   * https://developer.apple.com/library/mac/documentation/
+   *     Security/Conceptual/SecureCodingGuide/Articles/RaceConditions.html
+   */
+
+  /* Open directory.
+   * O_NOFOLLOW to ensure that it does not follow symbolic links */
+  fd = open(sandbox_intern_string(dirname), O_NOFOLLOW);
+
+  /* Was there an error? Maybe the directory does not exist? */
+  if (fd == -1) {
+
+    if (errno != ENOENT) {
+      /* Other directory error */
+      log_warn(LD_FS, "Directory %s cannot be read: %s", dirname,
+               strerror(errno));
+      return -1;
+    }
+
+    /* Received ENOENT: Directory does not exist */
+
+    /* Should we create the directory? */
+    if (check & CPD_CREATE) {
+      log_info(LD_GENERAL, "Creating directory %s", dirname);
+      if (check & CPD_GROUP_READ) {
+        r = mkdir(dirname, 0750);
+      } else {
+        r = mkdir(dirname, 0700);
+      }
+
+      /* check for mkdir() error */
+      if (r) {
+        log_warn(LD_FS, "Error creating directory %s: %s", dirname,
+            strerror(errno));
+        return -1;
+      }
+
+      /* we just created the directory. try to open it again.
+       * permissions on the directory will be checked again below.*/
+      fd = open(sandbox_intern_string(dirname), O_NOFOLLOW);
+
+      if (fd == -1) {
+        log_warn(LD_FS, "Could not reopen recently created directory %s: %s",
+                 dirname,
+                 strerror(errno));
+        return -1;
+      } else {
+        close(fd);
+      }
+
+    } else if (!(check & CPD_CHECK)) {
+      log_warn(LD_FS, "Directory %s does not exist.", dirname);
+      return -1;
+    }
+
+    /* XXXX In the case where check==CPD_CHECK, we should look at the
+     * parent directory a little harder. */
+    return 0;
+  }
+
+  tor_assert(fd >= 0);
+
+  //f = tor_strdup(dirname);
+  //clean_name_for_stat(f);
+  log_debug(LD_FS, "stat()ing %s", dirname);
+  //r = stat(sandbox_intern_string(f), &st);
+  r = fstat(fd, &st);
+  if (r == -1) {
+      log_warn(LD_FS, "fstat() on directory %s failed.", dirname);
+      close(fd);
+      return -1;
+  }
+  //tor_free(f);
+
+  /* check that dirname is a directory */
+  if (!(st.st_mode & S_IFDIR)) {
+    log_warn(LD_FS, "%s is not a directory", dirname);
+    close(fd);
+    return -1;
+  }
+
+  if (effective_user) {
+    /* Look up the user and group information.
+     * If we have a problem, bail out. */
+    pw = tor_getpwnam(effective_user);
+    if (pw == NULL) {
+      log_warn(LD_CONFIG, "Error setting configured user: %s not found",
+               effective_user);
+      close(fd);
+      return -1;
+    }
+    running_uid = pw->pw_uid;
+    running_gid = pw->pw_gid;
+  } else {
+    running_uid = getuid();
+    running_gid = getgid();
+  }
+  if (st.st_uid != running_uid) {
+    char *process_ownername = NULL, *file_ownername = NULL;
+
+    {
+      const struct passwd *pw_running = tor_getpwuid(running_uid);
+      process_ownername = pw_running ? tor_strdup(pw_running->pw_name) :
+        tor_strdup("<unknown>");
+    }
+
+    {
+      const struct passwd *pw_stat = tor_getpwuid(st.st_uid);
+      file_ownername = pw_stat ? tor_strdup(pw_stat->pw_name) :
+        tor_strdup("<unknown>");
+    }
+
+    log_warn(LD_FS, "%s is not owned by this user (%s, %d) but by "
+        "%s (%d). Perhaps you are running Tor as the wrong user?",
+             dirname, process_ownername, (int)running_uid,
+             file_ownername, (int)st.st_uid);
+
+    tor_free(process_ownername);
+    tor_free(file_ownername);
+    close(fd);
+    return -1;
+  }
+  if ( (check & (CPD_GROUP_OK|CPD_GROUP_READ))
+       && (st.st_gid != running_gid) && (st.st_gid != 0)) {
+    struct group *gr;
+    char *process_groupname = NULL;
+    gr = getgrgid(running_gid);
+    process_groupname = gr ? tor_strdup(gr->gr_name) : tor_strdup("<unknown>");
+    gr = getgrgid(st.st_gid);
+
+    log_warn(LD_FS, "%s is not owned by this group (%s, %d) but by group "
+             "%s (%d).  Are you running Tor as the wrong user?",
+             dirname, process_groupname, (int)running_gid,
+             gr ?  gr->gr_name : "<unknown>", (int)st.st_gid);
+
+    tor_free(process_groupname);
+    close(fd);
+    return -1;
+  }
+  unsigned unwanted_bits = 0;
+  if (check & (CPD_GROUP_OK|CPD_GROUP_READ)) {
+    unwanted_bits = 0027;
+  } else {
+    unwanted_bits = 0077;
+  }
+  unsigned check_bits_filter = ~0;
+  if (check & CPD_RELAX_DIRMODE_CHECK) {
+    check_bits_filter = 0022;
+  }
+  if ((st.st_mode & unwanted_bits & check_bits_filter) != 0) {
+    unsigned new_mode;
+    if (check & CPD_CHECK_MODE_ONLY) {
+      log_warn(LD_FS, "Permissions on directory %s are too permissive.",
+               dirname);
+      close(fd);
+      return -1;
+    }
+    log_warn(LD_FS, "Fixing permissions on directory %s", dirname);
+    new_mode = st.st_mode;
+    new_mode |= 0700; /* Owner should have rwx */
+    if (check & CPD_GROUP_READ) {
+      new_mode |= 0050; /* Group should have rx */
+    }
+    new_mode &= ~unwanted_bits; /* Clear the bits that we didn't want set...*/
+    if (fchmod(fd, new_mode)) {
+      log_warn(LD_FS, "Could not chmod directory %s: %s", dirname,
+               strerror(errno));
+      close(fd);
+      return -1;
+    } else {
+      close(fd);
+      return 0;
+    }
+  }
+  close(fd);
+#else /* !(!defined(_WIN32)) */
+  /* Win32 case: we can't open() a directory. */
+  (void)effective_user;
+
+  char *f = tor_strdup(dirname);
+  clean_fname_for_stat(f);
+  log_debug(LD_FS, "stat()ing %s", f);
+  r = stat(sandbox_intern_string(f), &st);
+  tor_free(f);
+  if (r) {
+    if (errno != ENOENT) {
+      log_warn(LD_FS, "Directory %s cannot be read: %s", dirname,
+               strerror(errno));
+      return -1;
+    }
+    if (check & CPD_CREATE) {
+      log_info(LD_GENERAL, "Creating directory %s", dirname);
+      r = mkdir(dirname);
+      if (r) {
+        log_warn(LD_FS, "Error creating directory %s: %s", dirname,
+                 strerror(errno));
+        return -1;
+      }
+    } else if (!(check & CPD_CHECK)) {
+      log_warn(LD_FS, "Directory %s does not exist.", dirname);
+      return -1;
+    }
+    return 0;
+  }
+  if (!(st.st_mode & S_IFDIR)) {
+    log_warn(LD_FS, "%s is not a directory", dirname);
+    return -1;
+  }
+
+#endif /* !defined(_WIN32) */
+  return 0;
+}
+
+/** Return a new list containing the filenames in the directory <b>dirname</b>.
+ * Return NULL on error or if <b>dirname</b> is not a directory.
+ */
+MOCK_IMPL(smartlist_t *,
+tor_listdir, (const char *dirname))
+{
+  smartlist_t *result;
+#ifdef _WIN32
+  char *pattern=NULL;
+  TCHAR tpattern[MAX_PATH] = {0};
+  char name[MAX_PATH*2+1] = {0};
+  HANDLE handle;
+  WIN32_FIND_DATA findData;
+  tor_asprintf(&pattern, "%s\\*", dirname);
+#ifdef UNICODE
+  mbstowcs(tpattern,pattern,MAX_PATH);
+#else
+  strlcpy(tpattern, pattern, MAX_PATH);
+#endif
+  if (INVALID_HANDLE_VALUE == (handle = FindFirstFile(tpattern, &findData))) {
+    tor_free(pattern);
+    return NULL;
+  }
+  result = smartlist_new();
+  while (1) {
+#ifdef UNICODE
+    wcstombs(name,findData.cFileName,MAX_PATH);
+    name[sizeof(name)-1] = '\0';
+#else
+    strlcpy(name,findData.cFileName,sizeof(name));
+#endif /* defined(UNICODE) */
+    if (strcmp(name, ".") &&
+        strcmp(name, "..")) {
+      smartlist_add_strdup(result, name);
+    }
+    if (!FindNextFile(handle, &findData)) {
+      DWORD err;
+      if ((err = GetLastError()) != ERROR_NO_MORE_FILES) {
+        char *errstr = format_win32_error(err);
+        log_warn(LD_FS, "Error reading directory '%s': %s", dirname, errstr);
+        tor_free(errstr);
+      }
+      break;
+    }
+  }
+  FindClose(handle);
+  tor_free(pattern);
+#else /* !(defined(_WIN32)) */
+  const char *prot_dname = sandbox_intern_string(dirname);
+  DIR *d;
+  struct dirent *de;
+  if (!(d = opendir(prot_dname)))
+    return NULL;
+
+  result = smartlist_new();
+  while ((de = readdir(d))) {
+    if (!strcmp(de->d_name, ".") ||
+        !strcmp(de->d_name, ".."))
+      continue;
+    smartlist_add_strdup(result, de->d_name);
+  }
+  closedir(d);
+#endif /* defined(_WIN32) */
+  return result;
+}
diff --git a/src/lib/fs/dir.h b/src/lib/fs/dir.h
new file mode 100644
index 0000000000..826bc2dfc5
--- /dev/null
+++ b/src/lib/fs/dir.h
@@ -0,0 +1,33 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_DIR_H
+#define TOR_DIR_H
+
+/**
+ * \file dir.h
+ *
+ * \brief Header for dir.c
+ **/
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/testsupport/testsupport.h"
+
+/** Possible behaviors for check_private_dir() on encountering a nonexistent
+ * directory; see that function's documentation for details. */
+typedef unsigned int cpd_check_t;
+#define CPD_NONE                 0
+#define CPD_CREATE               (1u << 0)
+#define CPD_CHECK                (1u << 1)
+#define CPD_GROUP_OK             (1u << 2)
+#define CPD_GROUP_READ           (1u << 3)
+#define CPD_CHECK_MODE_ONLY      (1u << 4)
+#define CPD_RELAX_DIRMODE_CHECK  (1u << 5)
+MOCK_DECL(int, check_private_dir, (const char *dirname, cpd_check_t check,
+                                   const char *effective_user));
+
+MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname));
+
+#endif
diff --git a/src/lib/fs/files.c b/src/lib/fs/files.c
new file mode 100644
index 0000000000..b98a51a287
--- /dev/null
+++ b/src/lib/fs/files.c
@@ -0,0 +1,721 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file files.h
+ *
+ * \brief Wrappers for reading and writing data to files on disk.
+ **/
+
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+#include "lib/fs/files.h"
+#include "lib/fs/path.h"
+#include "lib/container/smartlist.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/log/escape.h"
+#include "lib/err/torerr.h"
+#include "lib/malloc/malloc.h"
+#include "lib/sandbox/sandbox.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+#include "lib/fdio/fdio.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_UTIME_H
+#include <utime.h>
+#endif
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+
+/** As open(path, flags, mode), but return an fd with the close-on-exec mode
+ * set. */
+int
+tor_open_cloexec(const char *path, int flags, unsigned mode)
+{
+  int fd;
+  const char *p = sandbox_intern_string(path);
+#ifdef O_CLOEXEC
+  fd = open(p, flags|O_CLOEXEC, mode);
+  if (fd >= 0)
+    return fd;
+  /* If we got an error, see if it is EINVAL. EINVAL might indicate that,
+   * even though we were built on a system with O_CLOEXEC support, we
+   * are running on one without. */
+  if (errno != EINVAL)
+    return -1;
+#endif /* defined(O_CLOEXEC) */
+
+  log_debug(LD_FS, "Opening %s with flags %x", p, flags);
+  fd = open(p, flags, mode);
+#ifdef FD_CLOEXEC
+  if (fd >= 0) {
+    if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1) {
+      log_warn(LD_FS,"Couldn't set FD_CLOEXEC: %s", strerror(errno));
+      close(fd);
+      return -1;
+    }
+  }
+#endif /* defined(FD_CLOEXEC) */
+  return fd;
+}
+
+/** As fopen(path,mode), but ensures that the O_CLOEXEC bit is set on the
+ * underlying file handle. */
+FILE *
+tor_fopen_cloexec(const char *path, const char *mode)
+{
+  FILE *result = fopen(path, mode);
+#ifdef FD_CLOEXEC
+  if (result != NULL) {
+    if (fcntl(fileno(result), F_SETFD, FD_CLOEXEC) == -1) {
+      log_warn(LD_FS,"Couldn't set FD_CLOEXEC: %s", strerror(errno));
+      fclose(result);
+      return NULL;
+    }
+  }
+#endif /* defined(FD_CLOEXEC) */
+  return result;
+}
+
+/** As rename(), but work correctly with the sandbox. */
+int
+tor_rename(const char *path_old, const char *path_new)
+{
+  log_debug(LD_FS, "Renaming %s to %s", path_old, path_new);
+  return rename(sandbox_intern_string(path_old),
+                sandbox_intern_string(path_new));
+}
+
+/**
+ * Rename the file <b>from</b> to the file <b>to</b>.  On Unix, this is
+ * the same as rename(2).  On windows, this removes <b>to</b> first if
+ * it already exists.
+ * Returns 0 on success.  Returns -1 and sets errno on failure.
+ */
+int
+replace_file(const char *from, const char *to)
+{
+#ifndef _WIN32
+  return tor_rename(from, to);
+#else
+  switch (file_status(to))
+    {
+    case FN_NOENT:
+      break;
+    case FN_FILE:
+    case FN_EMPTY:
+      if (unlink(to)) return -1;
+      break;
+    case FN_ERROR:
+      return -1;
+    case FN_DIR:
+      errno = EISDIR;
+      return -1;
+    }
+  return tor_rename(from,to);
+#endif /* !defined(_WIN32) */
+}
+
+/** Change <b>fname</b>'s modification time to now. */
+int
+touch_file(const char *fname)
+{
+  if (utime(fname, NULL)!=0)
+    return -1;
+  return 0;
+}
+
+/** Wrapper for unlink() to make it mockable for the test suite; returns 0
+ * if unlinking the file succeeded, -1 and sets errno if unlinking fails.
+ */
+
+MOCK_IMPL(int,
+tor_unlink,(const char *pathname))
+{
+  return unlink(pathname);
+}
+
+/** Write <b>count</b> bytes from <b>buf</b> to <b>fd</b>. Return the number
+ * of bytes written, or -1 on error.  Only use if fd is a blocking fd.  */
+ssize_t
+write_all_to_fd(int fd, const char *buf, size_t count)
+{
+  size_t written = 0;
+  ssize_t result;
+  raw_assert(count < SSIZE_MAX);
+
+  while (written != count) {
+    result = write(fd, buf+written, count-written);
+    if (result<0)
+      return -1;
+    written += result;
+  }
+  return (ssize_t)count;
+}
+
+/** Read from <b>fd</b> to <b>buf</b>, until we get <b>count</b> bytes or
+ * reach the end of the file.  Return the number of bytes read, or -1 on
+ * error. Only use if fd is a blocking fd. */
+ssize_t
+read_all_from_fd(int fd, char *buf, size_t count)
+{
+  size_t numread = 0;
+  ssize_t result;
+
+  if (count > SIZE_T_CEILING || count > SSIZE_MAX) {
+    errno = EINVAL;
+    return -1;
+  }
+
+  while (numread < count) {
+    result = read(fd, buf+numread, count-numread);
+    if (result<0)
+      return -1;
+    else if (result == 0)
+      break;
+    numread += result;
+  }
+  return (ssize_t)numread;
+}
+
+/** Return:
+ * FN_ERROR if filename can't be read, is NULL, or is zero-length,
+ * FN_NOENT if it doesn't exist,
+ * FN_FILE if it is a non-empty regular file, or a FIFO on unix-like systems,
+ * FN_EMPTY for zero-byte regular files,
+ * FN_DIR if it's a directory, and
+ * FN_ERROR for any other file type.
+ * On FN_ERROR and FN_NOENT, sets errno.  (errno is not set when FN_ERROR
+ * is returned due to an unhandled file type.) */
+file_status_t
+file_status(const char *fname)
+{
+  struct stat st;
+  char *f;
+  int r;
+  if (!fname || strlen(fname) == 0) {
+    return FN_ERROR;
+  }
+  f = tor_strdup(fname);
+  clean_fname_for_stat(f);
+  log_debug(LD_FS, "stat()ing %s", f);
+  r = stat(sandbox_intern_string(f), &st);
+  tor_free(f);
+  if (r) {
+    if (errno == ENOENT) {
+      return FN_NOENT;
+    }
+    return FN_ERROR;
+  }
+  if (st.st_mode & S_IFDIR) {
+    return FN_DIR;
+  } else if (st.st_mode & S_IFREG) {
+    if (st.st_size > 0) {
+      return FN_FILE;
+    } else if (st.st_size == 0) {
+      return FN_EMPTY;
+    } else {
+      return FN_ERROR;
+    }
+#ifndef _WIN32
+  } else if (st.st_mode & S_IFIFO) {
+    return FN_FILE;
+#endif
+  } else {
+    return FN_ERROR;
+  }
+}
+
+/** Create a file named <b>fname</b> with the contents <b>str</b>.  Overwrite
+ * the previous <b>fname</b> if possible.  Return 0 on success, -1 on failure.
+ *
+ * This function replaces the old file atomically, if possible.  This
+ * function, and all other functions in util.c that create files, create them
+ * with mode 0600.
+ */
+MOCK_IMPL(int,
+write_str_to_file,(const char *fname, const char *str, int bin))
+{
+#ifdef _WIN32
+  if (!bin && strchr(str, '\r')) {
+    log_warn(LD_BUG,
+             "We're writing a text string that already contains a CR to %s",
+             escaped(fname));
+  }
+#endif /* defined(_WIN32) */
+  return write_bytes_to_file(fname, str, strlen(str), bin);
+}
+
+/** Represents a file that we're writing to, with support for atomic commit:
+ * we can write into a temporary file, and either remove the file on
+ * failure, or replace the original file on success. */
+struct open_file_t {
+  char *tempname; /**< Name of the temporary file. */
+  char *filename; /**< Name of the original file. */
+  unsigned rename_on_close:1; /**< Are we using the temporary file or not? */
+  unsigned binary:1; /**< Did we open in binary mode? */
+  int fd; /**< fd for the open file. */
+  FILE *stdio_file; /**< stdio wrapper for <b>fd</b>. */
+};
+
+/** Try to start writing to the file in <b>fname</b>, passing the flags
+ * <b>open_flags</b> to the open() syscall, creating the file (if needed) with
+ * access value <b>mode</b>.  If the O_APPEND flag is set, we append to the
+ * original file.  Otherwise, we open a new temporary file in the same
+ * directory, and either replace the original or remove the temporary file
+ * when we're done.
+ *
+ * Return the fd for the newly opened file, and store working data in
+ * *<b>data_out</b>.  The caller should not close the fd manually:
+ * instead, call finish_writing_to_file() or abort_writing_to_file().
+ * Returns -1 on failure.
+ *
+ * NOTE: When not appending, the flags O_CREAT and O_TRUNC are treated
+ * as true and the flag O_EXCL is treated as false.
+ *
+ * NOTE: Ordinarily, O_APPEND means "seek to the end of the file before each
+ * write()".  We don't do that.
+ */
+int
+start_writing_to_file(const char *fname, int open_flags, int mode,
+                      open_file_t **data_out)
+{
+  open_file_t *new_file = tor_malloc_zero(sizeof(open_file_t));
+  const char *open_name;
+  int append = 0;
+
+  tor_assert(fname);
+  tor_assert(data_out);
+#if (O_BINARY != 0 && O_TEXT != 0)
+  tor_assert((open_flags & (O_BINARY|O_TEXT)) != 0);
+#endif
+  new_file->fd = -1;
+  new_file->filename = tor_strdup(fname);
+  if (open_flags & O_APPEND) {
+    open_name = fname;
+    new_file->rename_on_close = 0;
+    append = 1;
+    open_flags &= ~O_APPEND;
+  } else {
+    tor_asprintf(&new_file->tempname, "%s.tmp", fname);
+    open_name = new_file->tempname;
+    /* We always replace an existing temporary file if there is one. */
+    open_flags |= O_CREAT|O_TRUNC;
+    open_flags &= ~O_EXCL;
+    new_file->rename_on_close = 1;
+  }
+#if O_BINARY != 0
+  if (open_flags & O_BINARY)
+    new_file->binary = 1;
+#endif
+
+  new_file->fd = tor_open_cloexec(open_name, open_flags, mode);
+  if (new_file->fd < 0) {
+    log_warn(LD_FS, "Couldn't open \"%s\" (%s) for writing: %s",
+        open_name, fname, strerror(errno));
+    goto err;
+  }
+  if (append) {
+    if (tor_fd_seekend(new_file->fd) < 0) {
+      log_warn(LD_FS, "Couldn't seek to end of file \"%s\": %s", open_name,
+               strerror(errno));
+      goto err;
+    }
+  }
+
+  *data_out = new_file;
+
+  return new_file->fd;
+
+ err:
+  if (new_file->fd >= 0)
+    close(new_file->fd);
+  *data_out = NULL;
+  tor_free(new_file->filename);
+  tor_free(new_file->tempname);
+  tor_free(new_file);
+  return -1;
+}
+
+/** Given <b>file_data</b> from start_writing_to_file(), return a stdio FILE*
+ * that can be used to write to the same file.  The caller should not mix
+ * stdio calls with non-stdio calls. */
+FILE *
+fdopen_file(open_file_t *file_data)
+{
+  tor_assert(file_data);
+  if (file_data->stdio_file)
+    return file_data->stdio_file;
+  tor_assert(file_data->fd >= 0);
+  if (!(file_data->stdio_file = fdopen(file_data->fd,
+                                       file_data->binary?"ab":"a"))) {
+    log_warn(LD_FS, "Couldn't fdopen \"%s\" [%d]: %s", file_data->filename,
+             file_data->fd, strerror(errno));
+  }
+  return file_data->stdio_file;
+}
+
+/** Combines start_writing_to_file with fdopen_file(): arguments are as
+ * for start_writing_to_file, but  */
+FILE *
+start_writing_to_stdio_file(const char *fname, int open_flags, int mode,
+                            open_file_t **data_out)
+{
+  FILE *res;
+  if (start_writing_to_file(fname, open_flags, mode, data_out)<0)
+    return NULL;
+  if (!(res = fdopen_file(*data_out))) {
+    abort_writing_to_file(*data_out);
+    *data_out = NULL;
+  }
+  return res;
+}
+
+/** Helper function: close and free the underlying file and memory in
+ * <b>file_data</b>.  If we were writing into a temporary file, then delete
+ * that file (if abort_write is true) or replaces the target file with
+ * the temporary file (if abort_write is false). */
+static int
+finish_writing_to_file_impl(open_file_t *file_data, int abort_write)
+{
+  int r = 0;
+
+  tor_assert(file_data && file_data->filename);
+  if (file_data->stdio_file) {
+    if (fclose(file_data->stdio_file)) {
+      log_warn(LD_FS, "Error closing \"%s\": %s", file_data->filename,
+               strerror(errno));
+      abort_write = r = -1;
+    }
+  } else if (file_data->fd >= 0 && close(file_data->fd) < 0) {
+    log_warn(LD_FS, "Error flushing \"%s\": %s", file_data->filename,
+             strerror(errno));
+    abort_write = r = -1;
+  }
+
+  if (file_data->rename_on_close) {
+    tor_assert(file_data->tempname && file_data->filename);
+    if (!abort_write) {
+      tor_assert(strcmp(file_data->filename, file_data->tempname));
+      if (replace_file(file_data->tempname, file_data->filename)) {
+        log_warn(LD_FS, "Error replacing \"%s\": %s", file_data->filename,
+                 strerror(errno));
+        abort_write = r = -1;
+      }
+    }
+    if (abort_write) {
+      int res = unlink(file_data->tempname);
+      if (res != 0) {
+        /* We couldn't unlink and we'll leave a mess behind */
+        log_warn(LD_FS, "Failed to unlink %s: %s",
+                 file_data->tempname, strerror(errno));
+        r = -1;
+      }
+    }
+  }
+
+  tor_free(file_data->filename);
+  tor_free(file_data->tempname);
+  tor_free(file_data);
+
+  return r;
+}
+
+/** Finish writing to <b>file_data</b>: close the file handle, free memory as
+ * needed, and if using a temporary file, replace the original file with
+ * the temporary file. */
+int
+finish_writing_to_file(open_file_t *file_data)
+{
+  return finish_writing_to_file_impl(file_data, 0);
+}
+
+/** Finish writing to <b>file_data</b>: close the file handle, free memory as
+ * needed, and if using a temporary file, delete it. */
+int
+abort_writing_to_file(open_file_t *file_data)
+{
+  return finish_writing_to_file_impl(file_data, 1);
+}
+
+/** Helper: given a set of flags as passed to open(2), open the file
+ * <b>fname</b> and write all the sized_chunk_t structs in <b>chunks</b> to
+ * the file.  Do so as atomically as possible e.g. by opening temp files and
+ * renaming. */
+static int
+write_chunks_to_file_impl(const char *fname, const smartlist_t *chunks,
+                          int open_flags)
+{
+  open_file_t *file = NULL;
+  int fd;
+  ssize_t result;
+  fd = start_writing_to_file(fname, open_flags, 0600, &file);
+  if (fd<0)
+    return -1;
+  SMARTLIST_FOREACH(chunks, sized_chunk_t *, chunk,
+  {
+    result = write_all_to_fd(fd, chunk->bytes, chunk->len);
+    if (result < 0) {
+      log_warn(LD_FS, "Error writing to \"%s\": %s", fname,
+          strerror(errno));
+      goto err;
+    }
+    tor_assert((size_t)result == chunk->len);
+  });
+
+  return finish_writing_to_file(file);
+ err:
+  abort_writing_to_file(file);
+  return -1;
+}
+
+/** Given a smartlist of sized_chunk_t, write them to a file
+ * <b>fname</b>, overwriting or creating the file as necessary.
+ * If <b>no_tempfile</b> is 0 then the file will be written
+ * atomically. */
+int
+write_chunks_to_file(const char *fname, const smartlist_t *chunks, int bin,
+                     int no_tempfile)
+{
+  int flags = OPEN_FLAGS_REPLACE|(bin?O_BINARY:O_TEXT);
+
+  if (no_tempfile) {
+    /* O_APPEND stops write_chunks_to_file from using tempfiles */
+    flags |= O_APPEND;
+  }
+  return write_chunks_to_file_impl(fname, chunks, flags);
+}
+
+/** Write <b>len</b> bytes, starting at <b>str</b>, to <b>fname</b>
+    using the open() flags passed in <b>flags</b>. */
+static int
+write_bytes_to_file_impl(const char *fname, const char *str, size_t len,
+                         int flags)
+{
+  int r;
+  sized_chunk_t c = { str, len };
+  smartlist_t *chunks = smartlist_new();
+  smartlist_add(chunks, &c);
+  r = write_chunks_to_file_impl(fname, chunks, flags);
+  smartlist_free(chunks);
+  return r;
+}
+
+/** As write_str_to_file, but does not assume a NUL-terminated
+ * string. Instead, we write <b>len</b> bytes, starting at <b>str</b>. */
+MOCK_IMPL(int,
+write_bytes_to_file,(const char *fname, const char *str, size_t len,
+                     int bin))
+{
+  return write_bytes_to_file_impl(fname, str, len,
+                                  OPEN_FLAGS_REPLACE|(bin?O_BINARY:O_TEXT));
+}
+
+/** As write_bytes_to_file, but if the file already exists, append the bytes
+ * to the end of the file instead of overwriting it. */
+int
+append_bytes_to_file(const char *fname, const char *str, size_t len,
+                     int bin)
+{
+  return write_bytes_to_file_impl(fname, str, len,
+                                  OPEN_FLAGS_APPEND|(bin?O_BINARY:O_TEXT));
+}
+
+/** Like write_str_to_file(), but also return -1 if there was a file
+    already residing in <b>fname</b>. */
+int
+write_bytes_to_new_file(const char *fname, const char *str, size_t len,
+                        int bin)
+{
+  return write_bytes_to_file_impl(fname, str, len,
+                                  OPEN_FLAGS_DONT_REPLACE|
+                                  (bin?O_BINARY:O_TEXT));
+}
+
+/**
+ * Read the contents of the open file <b>fd</b> presuming it is a FIFO
+ * (or similar) file descriptor for which the size of the file isn't
+ * known ahead of time. Return NULL on failure, and a NUL-terminated
+ * string on success.  On success, set <b>sz_out</b> to the number of
+ * bytes read.
+ */
+char *
+read_file_to_str_until_eof(int fd, size_t max_bytes_to_read, size_t *sz_out)
+{
+  ssize_t r;
+  size_t pos = 0;
+  char *string = NULL;
+  size_t string_max = 0;
+
+  if (max_bytes_to_read+1 >= SIZE_T_CEILING) {
+    errno = EINVAL;
+    return NULL;
+  }
+
+  do {
+    /* XXXX This "add 1K" approach is a little goofy; if we care about
+     * performance here, we should be doubling.  But in practice we shouldn't
+     * be using this function on big files anyway. */
+    string_max = pos + 1024;
+    if (string_max > max_bytes_to_read)
+      string_max = max_bytes_to_read + 1;
+    string = tor_realloc(string, string_max);
+    r = read(fd, string + pos, string_max - pos - 1);
+    if (r < 0) {
+      int save_errno = errno;
+      tor_free(string);
+      errno = save_errno;
+      return NULL;
+    }
+
+    pos += r;
+  } while (r > 0 && pos < max_bytes_to_read);
+
+  tor_assert(pos < string_max);
+  *sz_out = pos;
+  string[pos] = '\0';
+  return string;
+}
+
+/** Read the contents of <b>filename</b> into a newly allocated
+ * string; return the string on success or NULL on failure.
+ *
+ * If <b>stat_out</b> is provided, store the result of stat()ing the
+ * file into <b>stat_out</b>.
+ *
+ * If <b>flags</b> &amp; RFTS_BIN, open the file in binary mode.
+ * If <b>flags</b> &amp; RFTS_IGNORE_MISSING, don't warn if the file
+ * doesn't exist.
+ */
+/*
+ * This function <em>may</em> return an erroneous result if the file
+ * is modified while it is running, but must not crash or overflow.
+ * Right now, the error case occurs when the file length grows between
+ * the call to stat and the call to read_all: the resulting string will
+ * be truncated.
+ */
+MOCK_IMPL(char *,
+read_file_to_str, (const char *filename, int flags, struct stat *stat_out))
+{
+  int fd; /* router file */
+  struct stat statbuf;
+  char *string;
+  ssize_t r;
+  int bin = flags & RFTS_BIN;
+
+  tor_assert(filename);
+
+  fd = tor_open_cloexec(filename,O_RDONLY|(bin?O_BINARY:O_TEXT),0);
+  if (fd<0) {
+    int severity = LOG_WARN;
+    int save_errno = errno;
+    if (errno == ENOENT && (flags & RFTS_IGNORE_MISSING))
+      severity = LOG_INFO;
+    log_fn(severity, LD_FS,"Could not open \"%s\": %s",filename,
+           strerror(errno));
+    errno = save_errno;
+    return NULL;
+  }
+
+  if (fstat(fd, &statbuf)<0) {
+    int save_errno = errno;
+    close(fd);
+    log_warn(LD_FS,"Could not fstat \"%s\".",filename);
+    errno = save_errno;
+    return NULL;
+  }
+
+#ifndef _WIN32
+/** When we detect that we're reading from a FIFO, don't read more than
+ * this many bytes.  It's insane overkill for most uses. */
+#define FIFO_READ_MAX (1024*1024)
+  if (S_ISFIFO(statbuf.st_mode)) {
+    size_t sz = 0;
+    string = read_file_to_str_until_eof(fd, FIFO_READ_MAX, &sz);
+    int save_errno = errno;
+    if (string && stat_out) {
+      statbuf.st_size = sz;
+      memcpy(stat_out, &statbuf, sizeof(struct stat));
+    }
+    close(fd);
+    if (!string)
+      errno = save_errno;
+    return string;
+  }
+#endif /* !defined(_WIN32) */
+
+  if ((uint64_t)(statbuf.st_size)+1 >= SIZE_T_CEILING) {
+    close(fd);
+    errno = EINVAL;
+    return NULL;
+  }
+
+  string = tor_malloc((size_t)(statbuf.st_size+1));
+
+  r = read_all_from_fd(fd,string,(size_t)statbuf.st_size);
+  if (r<0) {
+    int save_errno = errno;
+    log_warn(LD_FS,"Error reading from file \"%s\": %s", filename,
+             strerror(errno));
+    tor_free(string);
+    close(fd);
+    errno = save_errno;
+    return NULL;
+  }
+  string[r] = '\0'; /* NUL-terminate the result. */
+
+#if defined(_WIN32) || defined(__CYGWIN__)
+  if (!bin && strchr(string, '\r')) {
+    log_debug(LD_FS, "We didn't convert CRLF to LF as well as we hoped "
+              "when reading %s. Coping.",
+              filename);
+    tor_strstrip(string, "\r");
+    r = strlen(string);
+  }
+  if (!bin) {
+    statbuf.st_size = (size_t) r;
+  } else
+#endif /* defined(_WIN32) || defined(__CYGWIN__) */
+    if (r != statbuf.st_size) {
+      /* Unless we're using text mode on win32, we'd better have an exact
+       * match for size. */
+      int save_errno = errno;
+      log_warn(LD_FS,"Could read only %d of %ld bytes of file \"%s\".",
+               (int)r, (long)statbuf.st_size,filename);
+      tor_free(string);
+      close(fd);
+      errno = save_errno;
+      return NULL;
+    }
+  close(fd);
+  if (stat_out) {
+    memcpy(stat_out, &statbuf, sizeof(struct stat));
+  }
+
+  return string;
+}
+
+#if !defined(HAVE_GETDELIM) || defined(TOR_UNIT_TESTS)
+#include "ext/getdelim.c"
+#endif
diff --git a/src/lib/fs/files.h b/src/lib/fs/files.h
new file mode 100644
index 0000000000..52c94c914f
--- /dev/null
+++ b/src/lib/fs/files.h
@@ -0,0 +1,145 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file files.h
+ *
+ * \brief Header for files.c
+ **/
+
+#ifndef TOR_FS_H
+#define TOR_FS_H
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+#include "lib/testsupport/testsupport.h"
+
+#include <stddef.h>
+#include <stdio.h>
+
+#ifdef _WIN32
+/* We need these for struct stat to work */
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#endif
+
+#ifndef O_BINARY
+#define O_BINARY 0
+#endif
+#ifndef O_TEXT
+#define O_TEXT 0
+#endif
+#ifndef O_NOFOLLOW
+#define O_NOFOLLOW 0
+#endif
+
+struct stat;
+
+int tor_open_cloexec(const char *path, int flags, unsigned mode);
+FILE *tor_fopen_cloexec(const char *path, const char *mode);
+int tor_rename(const char *path_old, const char *path_new);
+
+int replace_file(const char *from, const char *to);
+int touch_file(const char *fname);
+
+MOCK_DECL(int,tor_unlink,(const char *pathname));
+
+/** Return values from file_status(); see that function's documentation
+ * for details. */
+typedef enum { FN_ERROR, FN_NOENT, FN_FILE, FN_DIR, FN_EMPTY } file_status_t;
+
+file_status_t file_status(const char *filename);
+
+int64_t tor_get_avail_disk_space(const char *path);
+
+ssize_t write_all_to_fd(int fd, const char *buf, size_t count);
+ssize_t read_all_from_fd(int fd, char *buf, size_t count);
+
+#define OPEN_FLAGS_REPLACE (O_WRONLY|O_CREAT|O_TRUNC)
+#define OPEN_FLAGS_APPEND (O_WRONLY|O_CREAT|O_APPEND)
+#define OPEN_FLAGS_DONT_REPLACE (O_CREAT|O_EXCL|O_APPEND|O_WRONLY)
+typedef struct open_file_t open_file_t;
+int start_writing_to_file(const char *fname, int open_flags, int mode,
+                          open_file_t **data_out);
+FILE *start_writing_to_stdio_file(const char *fname, int open_flags, int mode,
+                                  open_file_t **data_out);
+FILE *fdopen_file(open_file_t *file_data);
+int finish_writing_to_file(open_file_t *file_data);
+int abort_writing_to_file(open_file_t *file_data);
+MOCK_DECL(int, write_str_to_file,(const char *fname, const char *str,
+                                  int bin));
+MOCK_DECL(int, write_bytes_to_file,(const char *fname, const char *str,
+                                    size_t len,int bin));
+
+/** An ad-hoc type to hold a string of characters and a count; used by
+ * write_chunks_to_file. */
+typedef struct sized_chunk_t {
+  const char *bytes;
+  size_t len;
+} sized_chunk_t;
+struct smartlist_t;
+int write_chunks_to_file(const char *fname, const struct smartlist_t *chunks,
+                         int bin, int no_tempfile);
+int append_bytes_to_file(const char *fname, const char *str, size_t len,
+                         int bin);
+int write_bytes_to_new_file(const char *fname, const char *str, size_t len,
+                            int bin);
+
+/** Flag for read_file_to_str: open the file in binary mode. */
+#define RFTS_BIN            1
+/** Flag for read_file_to_str: it's okay if the file doesn't exist. */
+#define RFTS_IGNORE_MISSING 2
+
+MOCK_DECL_ATTR(char *, read_file_to_str,(const char *filename, int flags,
+                                         struct stat *stat_out),
+               ATTR_MALLOC);
+char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read,
+                                 size_t *sz_out)
+  ATTR_MALLOC;
+
+#if !defined(HAVE_GETDELIM) || defined(TOR_UNIT_TESTS)
+/** Internal back-end function to implement getdelim(): only exists when
+ * Tor is built for unit tests, or when Tor is built on an operating system
+ * without its own getdelim(). */
+ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream);
+#endif
+
+#ifdef HAVE_GETDELIM
+/**
+ * Cross-platform wrapper for getdelim(): behaves as the POSIX-standard
+ * getdelim() function.
+ *
+ * See `getdelim(3)` for more information.
+ *
+ * Note that this function will use the libc memory allocator -- so any memory
+ * passed to this function must come from raw_malloc(), and must be freed by
+ * raw_free() -- don't use tor_malloc() and tor_free() with this.
+ */
+#define tor_getdelim(lineptr, n, delim, stream) \
+  getdelim((lineptr), (n), (delim), (stream))
+#else
+#define tor_getdelim(lineptr, n, delim, stream) \
+  compat_getdelim_((lineptr), (n), (delim), (stream))
+#endif
+
+#ifdef HAVE_GETLINE
+/**
+ * Cross-platform wrapper for getline(): behaves as the POSIX-standard
+ * getline() function.
+ *
+ * See tor_getdelim() for usage notes.
+ */
+#define tor_getline(lineptr, n, stream) \
+  getline((lineptr), (n), (stream))
+#else
+#define tor_getline(lineptr, n, stream) \
+  tor_getdelim((lineptr), (n), '\n', (stream))
+#endif
+
+#endif
diff --git a/src/lib/fs/freespace.c b/src/lib/fs/freespace.c
new file mode 100644
index 0000000000..ee0f93073d
--- /dev/null
+++ b/src/lib/fs/freespace.c
@@ -0,0 +1,63 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file freespace.c
+ *
+ * \brief Find the available disk space on the current volume.
+ **/
+
+#include "lib/fs/files.h"
+#include "lib/cc/torint.h"
+
+#ifdef HAVE_SYS_STATVFS_H
+#include <sys/statvfs.h>
+#endif
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+#include <errno.h>
+#include <string.h>
+
+/** Return the amount of free disk space we have permission to use, in
+ * bytes. Return -1 if the amount of free space can't be determined. */
+int64_t
+tor_get_avail_disk_space(const char *path)
+{
+#ifdef HAVE_STATVFS
+  struct statvfs st;
+  int r;
+  memset(&st, 0, sizeof(st));
+
+  r = statvfs(path, &st);
+  if (r < 0)
+    return -1;
+
+  int64_t result = st.f_bavail;
+  if (st.f_frsize) {
+    result *= st.f_frsize;
+  } else if (st.f_bsize) {
+    result *= st.f_bsize;
+  } else {
+    return -1;
+  }
+
+  return result;
+#elif defined(_WIN32)
+  ULARGE_INTEGER freeBytesAvail;
+  BOOL ok;
+
+  ok = GetDiskFreeSpaceEx(path, &freeBytesAvail, NULL, NULL);
+  if (!ok) {
+    return -1;
+  }
+  return (int64_t)freeBytesAvail.QuadPart;
+#else
+  (void)path;
+  errno = ENOSYS;
+  return -1;
+#endif /* defined(HAVE_STATVFS) || ... */
+}
diff --git a/src/lib/fs/include.am b/src/lib/fs/include.am
new file mode 100644
index 0000000000..f33e4d6430
--- /dev/null
+++ b/src/lib/fs/include.am
@@ -0,0 +1,37 @@
+
+noinst_LIBRARIES += src/lib/libtor-fs.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-fs-testing.a
+endif
+
+src_lib_libtor_fs_a_SOURCES =			\
+	src/lib/fs/conffile.c			\
+	src/lib/fs/dir.c			\
+	src/lib/fs/files.c			\
+	src/lib/fs/freespace.c			\
+	src/lib/fs/lockfile.c			\
+	src/lib/fs/mmap.c			\
+	src/lib/fs/path.c			\
+	src/lib/fs/storagedir.c			\
+	src/lib/fs/userdb.c
+
+if WIN32
+src_lib_libtor_fs_a_SOURCES += src/lib/fs/winlib.c
+endif
+
+src_lib_libtor_fs_testing_a_SOURCES = \
+	$(src_lib_libtor_fs_a_SOURCES)
+src_lib_libtor_fs_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_fs_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/fs/conffile.h				\
+	src/lib/fs/dir.h				\
+	src/lib/fs/files.h				\
+	src/lib/fs/lockfile.h				\
+	src/lib/fs/mmap.h				\
+	src/lib/fs/path.h				\
+	src/lib/fs/storagedir.h				\
+	src/lib/fs/userdb.h				\
+	src/lib/fs/winlib.h
diff --git a/src/lib/fs/lockfile.c b/src/lib/fs/lockfile.c
new file mode 100644
index 0000000000..933ff1e02f
--- /dev/null
+++ b/src/lib/fs/lockfile.c
@@ -0,0 +1,145 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file lockfile.c
+ *
+ * \brief Implements lock files to prevent two Tor processes from using the
+ * same data directory at the same time.
+ **/
+
+#include "orconfig.h"
+#include "lib/fs/files.h"
+#include "lib/fs/lockfile.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+
+#ifdef HAVE_SYS_FILE_H
+#include <sys/file.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef _WIN32
+#include <windows.h>
+#include <sys/locking.h>
+#endif
+
+#include <errno.h>
+#include <string.h>
+
+/** Represents a lockfile on which we hold the lock. */
+struct tor_lockfile_t {
+  /** Name of the file */
+  char *filename;
+  /** File descriptor used to hold the file open */
+  int fd;
+};
+
+/** Try to get a lock on the lockfile <b>filename</b>, creating it as
+ * necessary.  If someone else has the lock and <b>blocking</b> is true,
+ * wait until the lock is available.  Otherwise return immediately whether
+ * we succeeded or not.
+ *
+ * Set *<b>locked_out</b> to true if somebody else had the lock, and to false
+ * otherwise.
+ *
+ * Return a <b>tor_lockfile_t</b> on success, NULL on failure.
+ *
+ * (Implementation note: because we need to fall back to fcntl on some
+ *  platforms, these locks are per-process, not per-thread.  If you want
+ *  to do in-process locking, use tor_mutex_t like a normal person.
+ *  On Windows, when <b>blocking</b> is true, the maximum time that
+ *  is actually waited is 10 seconds, after which NULL is returned
+ *  and <b>locked_out</b> is set to 1.)
+ */
+tor_lockfile_t *
+tor_lockfile_lock(const char *filename, int blocking, int *locked_out)
+{
+  tor_lockfile_t *result;
+  int fd;
+  *locked_out = 0;
+
+  log_info(LD_FS, "Locking \"%s\"", filename);
+  fd = tor_open_cloexec(filename, O_RDWR|O_CREAT|O_TRUNC, 0600);
+  if (fd < 0) {
+    log_warn(LD_FS,"Couldn't open \"%s\" for locking: %s", filename,
+             strerror(errno));
+    return NULL;
+  }
+
+#ifdef _WIN32
+  _lseek(fd, 0, SEEK_SET);
+  if (_locking(fd, blocking ? _LK_LOCK : _LK_NBLCK, 1) < 0) {
+    if (errno != EACCES && errno != EDEADLOCK)
+      log_warn(LD_FS,"Couldn't lock \"%s\": %s", filename, strerror(errno));
+    else
+      *locked_out = 1;
+    close(fd);
+    return NULL;
+  }
+#elif defined(HAVE_FLOCK)
+  if (flock(fd, LOCK_EX|(blocking ? 0 : LOCK_NB)) < 0) {
+    if (errno != EWOULDBLOCK)
+      log_warn(LD_FS,"Couldn't lock \"%s\": %s", filename, strerror(errno));
+    else
+      *locked_out = 1;
+    close(fd);
+    return NULL;
+  }
+#else
+  {
+    struct flock lock;
+    memset(&lock, 0, sizeof(lock));
+    lock.l_type = F_WRLCK;
+    lock.l_whence = SEEK_SET;
+    if (fcntl(fd, blocking ? F_SETLKW : F_SETLK, &lock) < 0) {
+      if (errno != EACCES && errno != EAGAIN)
+        log_warn(LD_FS, "Couldn't lock \"%s\": %s", filename, strerror(errno));
+      else
+        *locked_out = 1;
+      close(fd);
+      return NULL;
+    }
+  }
+#endif /* defined(_WIN32) || ... */
+
+  result = tor_malloc(sizeof(tor_lockfile_t));
+  result->filename = tor_strdup(filename);
+  result->fd = fd;
+  return result;
+}
+
+/** Release the lock held as <b>lockfile</b>. */
+void
+tor_lockfile_unlock(tor_lockfile_t *lockfile)
+{
+  tor_assert(lockfile);
+
+  log_info(LD_FS, "Unlocking \"%s\"", lockfile->filename);
+#ifdef _WIN32
+  _lseek(lockfile->fd, 0, SEEK_SET);
+  if (_locking(lockfile->fd, _LK_UNLCK, 1) < 0) {
+    log_warn(LD_FS,"Error unlocking \"%s\": %s", lockfile->filename,
+             strerror(errno));
+  }
+#elif defined(HAVE_FLOCK)
+  if (flock(lockfile->fd, LOCK_UN) < 0) {
+    log_warn(LD_FS, "Error unlocking \"%s\": %s", lockfile->filename,
+             strerror(errno));
+  }
+#else
+  /* Closing the lockfile is sufficient. */
+#endif /* defined(_WIN32) || ... */
+
+  close(lockfile->fd);
+  lockfile->fd = -1;
+  tor_free(lockfile->filename);
+  tor_free(lockfile);
+}
diff --git a/src/lib/fs/lockfile.h b/src/lib/fs/lockfile.h
new file mode 100644
index 0000000000..8aeee4cc7f
--- /dev/null
+++ b/src/lib/fs/lockfile.h
@@ -0,0 +1,20 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file lockfile.h
+ *
+ * \brief Header for lockfile.c
+ **/
+
+#ifndef TOR_LOCKFILE_H
+#define TOR_LOCKFILE_H
+
+typedef struct tor_lockfile_t tor_lockfile_t;
+tor_lockfile_t *tor_lockfile_lock(const char *filename, int blocking,
+                                  int *locked_out);
+void tor_lockfile_unlock(tor_lockfile_t *lockfile);
+
+#endif
diff --git a/src/lib/fs/mmap.c b/src/lib/fs/mmap.c
new file mode 100644
index 0000000000..daaee1f9b1
--- /dev/null
+++ b/src/lib/fs/mmap.c
@@ -0,0 +1,240 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file mmap.c
+ *
+ * \brief Cross-platform support for mapping files into our address space.
+ **/
+
+#include "lib/fs/mmap.h"
+#include "lib/fs/files.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/log/win32err.h"
+#include "lib/string/compat_string.h"
+#include "lib/malloc/malloc.h"
+
+#ifdef HAVE_MMAP
+#include <sys/mman.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+#include <errno.h>
+#include <string.h>
+
+#if defined(HAVE_MMAP) || defined(RUNNING_DOXYGEN)
+/** Try to create a memory mapping for <b>filename</b> and return it.  On
+ * failure, return NULL. Sets errno properly, using ERANGE to mean
+ * "empty file". Must only be called on trusted Tor-owned files, as changing
+ * the underlying file's size causes unspecified behavior. */
+tor_mmap_t *
+tor_mmap_file(const char *filename)
+{
+  int fd; /* router file */
+  char *string;
+  int result;
+  tor_mmap_t *res;
+  size_t size, filesize;
+  struct stat st;
+
+  tor_assert(filename);
+
+  fd = tor_open_cloexec(filename, O_RDONLY, 0);
+  if (fd<0) {
+    int save_errno = errno;
+    int severity = (errno == ENOENT) ? LOG_INFO : LOG_WARN;
+    log_fn(severity, LD_FS,"Could not open \"%s\" for mmap(): %s",filename,
+           strerror(errno));
+    errno = save_errno;
+    return NULL;
+  }
+
+  /* Get the size of the file */
+  result = fstat(fd, &st);
+  if (result != 0) {
+    int save_errno = errno;
+    log_warn(LD_FS,
+             "Couldn't fstat opened descriptor for \"%s\" during mmap: %s",
+             filename, strerror(errno));
+    close(fd);
+    errno = save_errno;
+    return NULL;
+  }
+  size = filesize = (size_t)(st.st_size);
+
+  if (st.st_size > SSIZE_T_CEILING || (off_t)size < st.st_size) {
+    log_warn(LD_FS, "File \"%s\" is too large. Ignoring.",filename);
+    errno = EFBIG;
+    close(fd);
+    return NULL;
+  }
+  if (!size) {
+    /* Zero-length file. If we call mmap on it, it will succeed but
+     * return NULL, and bad things will happen. So just fail. */
+    log_info(LD_FS,"File \"%s\" is empty. Ignoring.",filename);
+    errno = ERANGE;
+    close(fd);
+    return NULL;
+  }
+
+  string = mmap(0, size, PROT_READ, MAP_PRIVATE, fd, 0);
+  close(fd);
+  if (string == MAP_FAILED) {
+    int save_errno = errno;
+    log_warn(LD_FS,"Could not mmap file \"%s\": %s", filename,
+             strerror(errno));
+    errno = save_errno;
+    return NULL;
+  }
+
+  res = tor_malloc_zero(sizeof(tor_mmap_t));
+  res->data = string;
+  res->size = filesize;
+  res->mapping_size = size;
+
+  return res;
+}
+/** Release storage held for a memory mapping; returns 0 on success,
+ * or -1 on failure (and logs a warning). */
+int
+tor_munmap_file(tor_mmap_t *handle)
+{
+  int res;
+
+  if (handle == NULL)
+    return 0;
+
+  res = munmap((char*)handle->data, handle->mapping_size);
+  if (res == 0) {
+    /* munmap() succeeded */
+    tor_free(handle);
+  } else {
+    log_warn(LD_FS, "Failed to munmap() in tor_munmap_file(): %s",
+             strerror(errno));
+    res = -1;
+  }
+
+  return res;
+}
+#elif defined(_WIN32)
+tor_mmap_t *
+tor_mmap_file(const char *filename)
+{
+  TCHAR tfilename[MAX_PATH]= {0};
+  tor_mmap_t *res = tor_malloc_zero(sizeof(tor_mmap_t));
+  int empty = 0;
+  HANDLE file_handle = INVALID_HANDLE_VALUE;
+  DWORD size_low, size_high;
+  uint64_t real_size;
+  res->mmap_handle = NULL;
+#ifdef UNICODE
+  mbstowcs(tfilename,filename,MAX_PATH);
+#else
+  strlcpy(tfilename,filename,MAX_PATH);
+#endif
+  file_handle = CreateFile(tfilename,
+                           GENERIC_READ, FILE_SHARE_READ,
+                           NULL,
+                           OPEN_EXISTING,
+                           FILE_ATTRIBUTE_NORMAL,
+                           0);
+
+  if (file_handle == INVALID_HANDLE_VALUE)
+    goto win_err;
+
+  size_low = GetFileSize(file_handle, &size_high);
+
+  if (size_low == INVALID_FILE_SIZE && GetLastError() != NO_ERROR) {
+    log_warn(LD_FS,"Error getting size of \"%s\".",filename);
+    goto win_err;
+  }
+  if (size_low == 0 && size_high == 0) {
+    log_info(LD_FS,"File \"%s\" is empty. Ignoring.",filename);
+    empty = 1;
+    goto err;
+  }
+  real_size = (((uint64_t)size_high)<<32) | size_low;
+  if (real_size > SIZE_MAX) {
+    log_warn(LD_FS,"File \"%s\" is too big to map; not trying.",filename);
+    goto err;
+  }
+  res->size = real_size;
+
+  res->mmap_handle = CreateFileMapping(file_handle,
+                                       NULL,
+                                       PAGE_READONLY,
+                                       size_high,
+                                       size_low,
+                                       NULL);
+  if (res->mmap_handle == NULL)
+    goto win_err;
+  res->data = (char*) MapViewOfFile(res->mmap_handle,
+                                    FILE_MAP_READ,
+                                    0, 0, 0);
+  if (!res->data)
+    goto win_err;
+
+  CloseHandle(file_handle);
+  return res;
+ win_err: {
+    DWORD e = GetLastError();
+    int severity = (e == ERROR_FILE_NOT_FOUND || e == ERROR_PATH_NOT_FOUND) ?
+      LOG_INFO : LOG_WARN;
+    char *msg = format_win32_error(e);
+    log_fn(severity, LD_FS, "Couldn't mmap file \"%s\": %s", filename, msg);
+    tor_free(msg);
+    if (e == ERROR_FILE_NOT_FOUND || e == ERROR_PATH_NOT_FOUND)
+      errno = ENOENT;
+    else
+      errno = EINVAL;
+  }
+ err:
+  if (empty)
+    errno = ERANGE;
+  if (file_handle != INVALID_HANDLE_VALUE)
+    CloseHandle(file_handle);
+  tor_munmap_file(res);
+  return NULL;
+}
+
+/* Unmap the file, and return 0 for success or -1 for failure */
+int
+tor_munmap_file(tor_mmap_t *handle)
+{
+  if (handle == NULL)
+    return 0;
+
+  if (handle->data) {
+    /* This is an ugly cast, but without it, "data" in struct tor_mmap_t would
+       have to be redefined as non-const. */
+    BOOL ok = UnmapViewOfFile( (LPVOID) handle->data);
+    if (!ok) {
+      log_warn(LD_FS, "Failed to UnmapViewOfFile() in tor_munmap_file(): %d",
+               (int)GetLastError());
+    }
+  }
+
+  if (handle->mmap_handle != NULL)
+    CloseHandle(handle->mmap_handle);
+  tor_free(handle);
+
+  return 0;
+}
+#else
+#error "cannot implement tor_mmap_file"
+#endif /* defined(HAVE_MMAP) || ... || ... */
diff --git a/src/lib/fs/mmap.h b/src/lib/fs/mmap.h
new file mode 100644
index 0000000000..18fb18a13c
--- /dev/null
+++ b/src/lib/fs/mmap.h
@@ -0,0 +1,41 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file mmap.h
+ *
+ * \brief Header for mmap.c
+ **/
+
+#ifndef TOR_MMAP_H
+#define TOR_MMAP_H
+
+#include "lib/cc/compat_compiler.h"
+#include <stddef.h>
+
+#ifdef _WIN32
+#include <windef.h>
+#endif
+
+/** Represents an mmaped file. Allocated via tor_mmap_file; freed with
+ * tor_munmap_file. */
+typedef struct tor_mmap_t {
+  const char *data; /**< Mapping of the file's contents. */
+  size_t size; /**< Size of the file. */
+
+  /* None of the fields below should be accessed from outside compat.c */
+#ifdef HAVE_MMAP
+  size_t mapping_size; /**< Size of the actual mapping. (This is this file
+                        * size, rounded up to the nearest page.) */
+#elif defined _WIN32
+  HANDLE mmap_handle;
+#endif /* defined(HAVE_MMAP) || ... */
+
+} tor_mmap_t;
+
+tor_mmap_t *tor_mmap_file(const char *filename);
+int tor_munmap_file(tor_mmap_t *handle);
+
+#endif
diff --git a/src/lib/fs/path.c b/src/lib/fs/path.c
new file mode 100644
index 0000000000..b3ef61979d
--- /dev/null
+++ b/src/lib/fs/path.c
@@ -0,0 +1,295 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file path.c
+ *
+ * \brief Manipulate strings that contain filesystem paths.
+ **/
+
+#include "lib/fs/path.h"
+#include "lib/malloc/malloc.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/fs/userdb.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#include <errno.h>
+#include <string.h>
+
+/** Removes enclosing quotes from <b>path</b> and unescapes quotes between the
+ * enclosing quotes. Backslashes are not unescaped. Return the unquoted
+ * <b>path</b> on success or 0 if <b>path</b> is not quoted correctly. */
+char *
+get_unquoted_path(const char *path)
+{
+  size_t len = strlen(path);
+
+  if (len == 0) {
+    return tor_strdup("");
+  }
+
+  int has_start_quote = (path[0] == '\"');
+  int has_end_quote = (len > 0 && path[len-1] == '\"');
+  if (has_start_quote != has_end_quote || (len == 1 && has_start_quote)) {
+    return NULL;
+  }
+
+  char *unquoted_path = tor_malloc(len - has_start_quote - has_end_quote + 1);
+  char *s = unquoted_path;
+  size_t i;
+  for (i = has_start_quote; i < len - has_end_quote; i++) {
+    if (path[i] == '\"' && (i > 0 && path[i-1] == '\\')) {
+      *(s-1) = path[i];
+    } else if (path[i] != '\"') {
+      *s++ = path[i];
+    } else {  /* unescaped quote */
+      tor_free(unquoted_path);
+      return NULL;
+    }
+  }
+  *s = '\0';
+  return unquoted_path;
+}
+
+/** Expand any homedir prefix on <b>filename</b>; return a newly allocated
+ * string. */
+char *
+expand_filename(const char *filename)
+{
+  tor_assert(filename);
+#ifdef _WIN32
+  /* Might consider using GetFullPathName() as described here:
+   * http://etutorials.org/Programming/secure+programming/
+   *     Chapter+3.+Input+Validation/3.7+Validating+Filenames+and+Paths/
+   */
+  return tor_strdup(filename);
+#else /* !(defined(_WIN32)) */
+  if (*filename == '~') {
+    char *home, *result=NULL;
+    const char *rest;
+
+    if (filename[1] == '/' || filename[1] == '\0') {
+      home = getenv("HOME");
+      if (!home) {
+        log_warn(LD_CONFIG, "Couldn't find $HOME environment variable while "
+                 "expanding \"%s\"; defaulting to \"\".", filename);
+        home = tor_strdup("");
+      } else {
+        home = tor_strdup(home);
+      }
+      rest = strlen(filename)>=2?(filename+2):"";
+    } else {
+#ifdef HAVE_PWD_H
+      char *username, *slash;
+      slash = strchr(filename, '/');
+      if (slash)
+        username = tor_strndup(filename+1,slash-filename-1);
+      else
+        username = tor_strdup(filename+1);
+      if (!(home = get_user_homedir(username))) {
+        log_warn(LD_CONFIG,"Couldn't get homedir for \"%s\"",username);
+        tor_free(username);
+        return NULL;
+      }
+      tor_free(username);
+      rest = slash ? (slash+1) : "";
+#else /* !(defined(HAVE_PWD_H)) */
+      log_warn(LD_CONFIG, "Couldn't expand homedir on system without pwd.h");
+      return tor_strdup(filename);
+#endif /* defined(HAVE_PWD_H) */
+    }
+    tor_assert(home);
+    /* Remove trailing slash. */
+    if (strlen(home)>1 && !strcmpend(home,PATH_SEPARATOR)) {
+      home[strlen(home)-1] = '\0';
+    }
+    tor_asprintf(&result,"%s"PATH_SEPARATOR"%s",home,rest);
+    tor_free(home);
+    return result;
+  } else {
+    return tor_strdup(filename);
+  }
+#endif /* defined(_WIN32) */
+}
+
+/** Return true iff <b>filename</b> is a relative path. */
+int
+path_is_relative(const char *filename)
+{
+  if (filename && filename[0] == '/')
+    return 0;
+#ifdef _WIN32
+  else if (filename && filename[0] == '\\')
+    return 0;
+  else if (filename && strlen(filename)>3 && TOR_ISALPHA(filename[0]) &&
+           filename[1] == ':' && filename[2] == '\\')
+    return 0;
+#endif /* defined(_WIN32) */
+  else
+    return 1;
+}
+
+/** Clean up <b>name</b> so that we can use it in a call to "stat".  On Unix,
+ * we do nothing.  On Windows, we remove a trailing slash, unless the path is
+ * the root of a disk. */
+void
+clean_fname_for_stat(char *name)
+{
+#ifdef _WIN32
+  size_t len = strlen(name);
+  if (!len)
+    return;
+  if (name[len-1]=='\\' || name[len-1]=='/') {
+    if (len == 1 || (len==3 && name[1]==':'))
+      return;
+    name[len-1]='\0';
+  }
+#else /* !(defined(_WIN32)) */
+  (void)name;
+#endif /* defined(_WIN32) */
+}
+
+/** Modify <b>fname</b> to contain the name of its parent directory.  Doesn't
+ * actually examine the filesystem; does a purely syntactic modification.
+ *
+ * The parent of the root director is considered to be iteself.
+ *
+ * Path separators are the forward slash (/) everywhere and additionally
+ * the backslash (\) on Win32.
+ *
+ * Cuts off any number of trailing path separators but otherwise ignores
+ * them for purposes of finding the parent directory.
+ *
+ * Returns 0 if a parent directory was successfully found, -1 otherwise (fname
+ * did not have any path separators or only had them at the end).
+ * */
+int
+get_parent_directory(char *fname)
+{
+  char *cp;
+  int at_end = 1;
+  tor_assert(fname);
+#ifdef _WIN32
+  /* If we start with, say, c:, then don't consider that the start of the path
+   */
+  if (fname[0] && fname[1] == ':') {
+    fname += 2;
+  }
+#endif /* defined(_WIN32) */
+  /* Now we want to remove all path-separators at the end of the string,
+   * and to remove the end of the string starting with the path separator
+   * before the last non-path-separator.  In perl, this would be
+   *   s#[/]*$##; s#/[^/]*$##;
+   * on a unixy platform.
+   */
+  cp = fname + strlen(fname);
+  at_end = 1;
+  while (--cp >= fname) {
+    int is_sep = (*cp == '/'
+#ifdef _WIN32
+                  || *cp == '\\'
+#endif
+                  );
+    if (is_sep) {
+      if (cp == fname) {
+        /* This is the first separator in the file name; don't remove it! */
+        cp[1] = '\0';
+        return 0;
+      }
+      *cp = '\0';
+      if (! at_end)
+        return 0;
+    } else {
+      at_end = 0;
+    }
+  }
+  return -1;
+}
+
+#ifndef _WIN32
+/** Return a newly allocated string containing the output of getcwd(). Return
+ * NULL on failure. (We can't just use getcwd() into a PATH_MAX buffer, since
+ * Hurd hasn't got a PATH_MAX.)
+ */
+static char *
+alloc_getcwd(void)
+{
+#ifdef HAVE_GET_CURRENT_DIR_NAME
+  /* Glibc makes this nice and simple for us. */
+  char *cwd = get_current_dir_name();
+  char *result = NULL;
+  if (cwd) {
+    /* We make a copy here, in case tor_malloc() is not malloc(). */
+    result = tor_strdup(cwd);
+    raw_free(cwd); // alias for free to avoid tripping check-spaces.
+  }
+  return result;
+#else /* !(defined(HAVE_GET_CURRENT_DIR_NAME)) */
+  size_t size = 1024;
+  char *buf = NULL;
+  char *ptr = NULL;
+
+  while (ptr == NULL) {
+    buf = tor_realloc(buf, size);
+    ptr = getcwd(buf, size);
+
+    if (ptr == NULL && errno != ERANGE) {
+      tor_free(buf);
+      return NULL;
+    }
+
+    size *= 2;
+  }
+  return buf;
+#endif /* defined(HAVE_GET_CURRENT_DIR_NAME) */
+}
+#endif /* !defined(_WIN32) */
+
+/** Expand possibly relative path <b>fname</b> to an absolute path.
+ * Return a newly allocated string, possibly equal to <b>fname</b>. */
+char *
+make_path_absolute(char *fname)
+{
+#ifdef _WIN32
+  char *absfname_malloced = _fullpath(NULL, fname, 1);
+
+  /* We don't want to assume that tor_free can free a string allocated
+   * with malloc.  On failure, return fname (it's better than nothing). */
+  char *absfname = tor_strdup(absfname_malloced ? absfname_malloced : fname);
+  if (absfname_malloced) raw_free(absfname_malloced);
+
+  return absfname;
+#else /* !(defined(_WIN32)) */
+  char *absfname = NULL, *path = NULL;
+
+  tor_assert(fname);
+
+  if (fname[0] == '/') {
+    absfname = tor_strdup(fname);
+  } else {
+    path = alloc_getcwd();
+    if (path) {
+      tor_asprintf(&absfname, "%s/%s", path, fname);
+      tor_free(path);
+    } else {
+      /* LCOV_EXCL_START Can't make getcwd fail. */
+      /* If getcwd failed, the best we can do here is keep using the
+       * relative path.  (Perhaps / isn't readable by this UID/GID.) */
+      log_warn(LD_GENERAL, "Unable to find current working directory: %s",
+               strerror(errno));
+      absfname = tor_strdup(fname);
+      /* LCOV_EXCL_STOP */
+    }
+  }
+  return absfname;
+#endif /* defined(_WIN32) */
+}
diff --git a/src/lib/fs/path.h b/src/lib/fs/path.h
new file mode 100644
index 0000000000..4675ac84e8
--- /dev/null
+++ b/src/lib/fs/path.h
@@ -0,0 +1,30 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file path.h
+ *
+ * \brief Header for path.c
+ **/
+
+#ifndef TOR_PATH_H
+#define TOR_PATH_H
+
+#include "lib/cc/compat_compiler.h"
+
+#ifdef _WIN32
+#define PATH_SEPARATOR "\\"
+#else
+#define PATH_SEPARATOR "/"
+#endif
+
+char *get_unquoted_path(const char *path);
+char *expand_filename(const char *filename);
+int path_is_relative(const char *filename);
+void clean_fname_for_stat(char *name);
+int get_parent_directory(char *fname);
+char *make_path_absolute(char *fname);
+
+#endif
diff --git a/src/lib/fs/storagedir.c b/src/lib/fs/storagedir.c
new file mode 100644
index 0000000000..2caddf1ad9
--- /dev/null
+++ b/src/lib/fs/storagedir.c
@@ -0,0 +1,606 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file storagedir.c
+ *
+ * \brief An abstraction for a directory full of similar files.
+ *
+ * Storagedirs are used by our consensus cache code, and may someday also get
+ * used for unparseable objects. A large part of the need for this type is to
+ * work around the limitations in our sandbox code, where all filenames need
+ * to be registered in advance.
+ **/
+
+#include "lib/fs/storagedir.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/encoding/confline.h"
+#include "lib/fs/dir.h"
+#include "lib/fs/files.h"
+#include "lib/fs/mmap.h"
+#include "lib/log/escape.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/memarea/memarea.h"
+#include "lib/sandbox/sandbox.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <stdlib.h>
+#include <errno.h>
+#include <string.h>
+
+#define FNAME_MIN_NUM 1000
+
+/** A storage_dir_t represents a directory full of similar cached
+ * files. Filenames are decimal integers. Files can be cleaned as needed
+ * to limit total disk usage. */
+struct storage_dir_t {
+  /** Directory holding the files for this storagedir. */
+  char *directory;
+  /** Either NULL, or a directory listing of the directory (as a smartlist
+   * of strings */
+  smartlist_t *contents;
+  /** The largest number of non-temporary files we'll place in the
+   * directory. */
+  int max_files;
+  /** If true, then 'usage' has been computed. */
+  int usage_known;
+  /** The total number of bytes used in this directory */
+  uint64_t usage;
+};
+
+/** Create or open a new storage directory at <b>dirname</b>, with
+ * capacity for up to <b>max_files</b> files.
+ */
+storage_dir_t *
+storage_dir_new(const char *dirname, int max_files)
+{
+  if (check_private_dir(dirname, CPD_CREATE, NULL) < 0)
+    return NULL;
+
+  storage_dir_t *d = tor_malloc_zero(sizeof(storage_dir_t));
+  d->directory = tor_strdup(dirname);
+  d->max_files = max_files;
+  return d;
+}
+
+/**
+ * Drop all in-RAM storage for <b>d</b>.  Does not delete any files.
+ */
+void
+storage_dir_free_(storage_dir_t *d)
+{
+  if (d == NULL)
+    return;
+  tor_free(d->directory);
+  if (d->contents) {
+    SMARTLIST_FOREACH(d->contents, char *, cp, tor_free(cp));
+    smartlist_free(d->contents);
+  }
+  tor_free(d);
+}
+
+/**
+ * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
+ * operations that <b>d</b> will need.
+ *
+ * The presence of this function is why we need an upper limit on the
+ * number of files in a storage_dir_t: we need to approve file operations
+ * one by one.
+ */
+int
+storage_dir_register_with_sandbox(storage_dir_t *d, sandbox_cfg_t **cfg)
+{
+  int problems = 0;
+  int idx;
+  for (idx = FNAME_MIN_NUM; idx < FNAME_MIN_NUM + d->max_files; ++idx) {
+    char *path = NULL, *tmppath = NULL;
+    tor_asprintf(&path, "%s/%d", d->directory, idx);
+    tor_asprintf(&tmppath, "%s/%d.tmp", d->directory, idx);
+
+    problems += sandbox_cfg_allow_open_filename(cfg, tor_strdup(path));
+    problems += sandbox_cfg_allow_open_filename(cfg, tor_strdup(tmppath));
+    problems += sandbox_cfg_allow_stat_filename(cfg, tor_strdup(path));
+    problems += sandbox_cfg_allow_stat_filename(cfg, tor_strdup(tmppath));
+    problems += sandbox_cfg_allow_rename(cfg,
+                                      tor_strdup(tmppath), tor_strdup(path));
+
+    tor_free(path);
+    tor_free(tmppath);
+  }
+
+  return problems ? -1 : 0;
+}
+
+/**
+ * Remove all files in <b>d</b> whose names end with ".tmp".
+ *
+ * Requires that the contents field of <b>d</b> is set.
+ */
+static void
+storage_dir_clean_tmpfiles(storage_dir_t *d)
+{
+  if (!d->contents)
+    return;
+  SMARTLIST_FOREACH_BEGIN(d->contents, char *, fname) {
+    if (strcmpend(fname, ".tmp"))
+      continue;
+    char *path = NULL;
+    tor_asprintf(&path, "%s/%s", d->directory, fname);
+    if (unlink(sandbox_intern_string(path))) {
+      log_warn(LD_FS, "Unable to unlink %s while cleaning "
+               "temporary files: %s", escaped(path), strerror(errno));
+      tor_free(path);
+      continue;
+    }
+    tor_free(path);
+    SMARTLIST_DEL_CURRENT(d->contents, fname);
+    tor_free(fname);
+  } SMARTLIST_FOREACH_END(fname);
+
+  d->usage_known = 0;
+}
+
+/**
+ * Re-scan the directory <b>d</b> to learn its contents.
+ */
+static int
+storage_dir_rescan(storage_dir_t *d)
+{
+  if (d->contents) {
+    SMARTLIST_FOREACH(d->contents, char *, cp, tor_free(cp));
+    smartlist_free(d->contents);
+  }
+  d->usage = 0;
+  d->usage_known = 0;
+  if (NULL == (d->contents = tor_listdir(d->directory))) {
+    return -1;
+  }
+  storage_dir_clean_tmpfiles(d);
+  return 0;
+}
+
+/**
+ * Return a smartlist containing the filenames within <b>d</b>.
+ */
+const smartlist_t *
+storage_dir_list(storage_dir_t *d)
+{
+  if (! d->contents)
+    storage_dir_rescan(d);
+  return d->contents;
+}
+
+/**
+ * Return the total number of bytes used for storage in <b>d</b>.
+ */
+uint64_t
+storage_dir_get_usage(storage_dir_t *d)
+{
+  if (d->usage_known)
+    return d->usage;
+
+  uint64_t total = 0;
+  SMARTLIST_FOREACH_BEGIN(storage_dir_list(d), const char *, cp) {
+    char *path = NULL;
+    struct stat st;
+    tor_asprintf(&path, "%s/%s", d->directory, cp);
+    if (stat(sandbox_intern_string(path), &st) == 0) {
+      total += st.st_size;
+    }
+    tor_free(path);
+  } SMARTLIST_FOREACH_END(cp);
+
+  d->usage = total;
+  d->usage_known = 1;
+  return d->usage;
+}
+
+/** Mmap a specified file within <b>d</b>.
+ *
+ * On failure, return NULL and set errno as for tor_mmap_file(). */
+tor_mmap_t *
+storage_dir_map(storage_dir_t *d, const char *fname)
+{
+  char *path = NULL;
+  tor_asprintf(&path, "%s/%s", d->directory, fname);
+  tor_mmap_t *result = tor_mmap_file(path);
+  int errval = errno;
+  tor_free(path);
+  if (result == NULL)
+    errno = errval;
+  return result;
+}
+
+/** Read a file within <b>d</b> into a newly allocated buffer.  Set
+ * *<b>sz_out</b> to its size. */
+uint8_t *
+storage_dir_read(storage_dir_t *d, const char *fname, int bin, size_t *sz_out)
+{
+  const int flags = bin ? RFTS_BIN : 0;
+
+  char *path = NULL;
+  tor_asprintf(&path, "%s/%s", d->directory, fname);
+  struct stat st;
+  char *contents = read_file_to_str(path, flags, &st);
+  if (contents && sz_out) {
+    // it fits in RAM, so we know its size is less than SIZE_MAX
+#if UINT64_MAX > SIZE_MAX
+    tor_assert((uint64_t)st.st_size <= SIZE_MAX);
+#endif
+    *sz_out = (size_t) st.st_size;
+  }
+
+  tor_free(path);
+  return (uint8_t *) contents;
+}
+
+/** Helper: Find an unused filename within the directory */
+static char *
+find_unused_fname(storage_dir_t *d)
+{
+  if (!d->contents) {
+    if (storage_dir_rescan(d) < 0)
+      return NULL;
+  }
+
+  char buf[16];
+  int i;
+  /* Yuck; this is quadratic.  Fortunately, that shouldn't matter much,
+   * since disk writes are more expensive by a lot. */
+  for (i = FNAME_MIN_NUM; i < FNAME_MIN_NUM + d->max_files; ++i) {
+    tor_snprintf(buf, sizeof(buf), "%d", i);
+    if (!smartlist_contains_string(d->contents, buf)) {
+      return tor_strdup(buf);
+    }
+  }
+  return NULL;
+}
+
+/** Helper: As storage_dir_save_bytes_to_file, but store a smartlist of
+ * sized_chunk_t rather than a single byte array. */
+static int
+storage_dir_save_chunks_to_file(storage_dir_t *d,
+                                const smartlist_t *chunks,
+                                int binary,
+                                char **fname_out)
+{
+  uint64_t total_length = 0;
+  char *fname = find_unused_fname(d);
+  if (!fname)
+    return -1;
+
+  SMARTLIST_FOREACH(chunks, const sized_chunk_t *, ch,
+                    total_length += ch->len);
+
+  char *path = NULL;
+  tor_asprintf(&path, "%s/%s", d->directory, fname);
+
+  int r = write_chunks_to_file(path, chunks, binary, 0);
+  if (r == 0) {
+    if (d->usage_known)
+      d->usage += total_length;
+    if (fname_out) {
+      *fname_out = tor_strdup(fname);
+    }
+    if (d->contents)
+      smartlist_add(d->contents, tor_strdup(fname));
+  }
+  tor_free(fname);
+  tor_free(path);
+  return r;
+}
+
+/** Try to write the <b>length</b> bytes at <b>data</b> into a new file
+ * in <b>d</b>.  On success, return 0 and set *<b>fname_out</b> to a
+ * newly allocated string containing the filename.  On failure, return
+ * -1. */
+int
+storage_dir_save_bytes_to_file(storage_dir_t *d,
+                               const uint8_t *data,
+                               size_t length,
+                               int binary,
+                               char **fname_out)
+{
+  smartlist_t *chunks = smartlist_new();
+  sized_chunk_t chunk = { (const char *)data, length };
+  smartlist_add(chunks, &chunk);
+  int r = storage_dir_save_chunks_to_file(d, chunks, binary, fname_out);
+  smartlist_free(chunks);
+  return r;
+}
+
+/**
+ * As storage_dir_save_bytes_to_file, but saves a NUL-terminated string
+ * <b>str</b>.
+ */
+int
+storage_dir_save_string_to_file(storage_dir_t *d,
+                                const char *str,
+                                int binary,
+                                char **fname_out)
+{
+  return storage_dir_save_bytes_to_file(d,
+                (const uint8_t*)str, strlen(str), binary, fname_out);
+}
+
+/**
+ * As storage_dir_save_bytes_to_file, but associates the data with the
+ * key-value pairs in <b>labels</b>. Files stored in this format can be
+ * recovered with storage_dir_map_labeled() or storage_dir_read_labeled().
+ */
+int
+storage_dir_save_labeled_to_file(storage_dir_t *d,
+                                  const config_line_t *labels,
+                                  const uint8_t *data,
+                                  size_t length,
+                                  char **fname_out)
+{
+  /*
+   * The storage format is to prefix the data with the key-value pairs in
+   * <b>labels</b>, and a single NUL separator.  But code outside this module
+   * MUST NOT rely on that format.
+   */
+
+  smartlist_t *chunks = smartlist_new();
+  memarea_t *area = memarea_new();
+  const config_line_t *line;
+  for (line = labels; line; line = line->next) {
+    sized_chunk_t *sz = memarea_alloc(area, sizeof(sized_chunk_t));
+    sz->len = strlen(line->key) + 1 + strlen(line->value) + 1;
+    const size_t allocated = sz->len + 1;
+    char *bytes = memarea_alloc(area, allocated);
+    tor_snprintf(bytes, allocated, "%s %s\n", line->key, line->value);
+    sz->bytes = bytes;
+    smartlist_add(chunks, sz);
+  }
+
+  sized_chunk_t *nul = memarea_alloc(area, sizeof(sized_chunk_t));
+  nul->len = 1;
+  nul->bytes = "\0";
+  smartlist_add(chunks, nul);
+
+  sized_chunk_t *datachunk = memarea_alloc(area, sizeof(sized_chunk_t));
+  datachunk->bytes = (const char *)data;
+  datachunk->len = length;
+  smartlist_add(chunks, datachunk);
+
+  int r = storage_dir_save_chunks_to_file(d, chunks, 1, fname_out);
+  smartlist_free(chunks);
+  memarea_drop_all(area);
+  return r;
+}
+
+/**
+ * Map a file that was created with storage_dir_save_labeled_to_file().  On
+ * failure, return NULL.  On success, write a set of newly allocated labels
+ * into *<b>labels_out</b>, a pointer to the data into *<b>data_out</b>, and
+ * the data's size into *<b>sz_out</b>. On success, also return a tor_mmap_t
+ * object whose contents should not be used -- it needs to be kept around,
+ * though, for as long as <b>data_out</b> is going to be valid.
+ *
+ * On failure, set errno as for tor_mmap_file() if the file was missing or
+ * empty, and set errno to EINVAL if the file was not in the labeled
+ * format expected.
+ */
+tor_mmap_t *
+storage_dir_map_labeled(storage_dir_t *dir,
+                         const char *fname,
+                         config_line_t **labels_out,
+                         const uint8_t **data_out,
+                         size_t *sz_out)
+{
+  tor_mmap_t *m = storage_dir_map(dir, fname);
+  int errval;
+  if (! m) {
+    errval = errno;
+    goto err;
+  }
+  const char *nulp = memchr(m->data, '\0', m->size);
+  if (! nulp) {
+    errval = EINVAL;
+    goto err;
+  }
+  if (labels_out && config_get_lines(m->data, labels_out, 0) < 0) {
+    errval = EINVAL;
+    goto err;
+  }
+  size_t offset = nulp - m->data + 1;
+  tor_assert(offset <= m->size);
+  *data_out = (const uint8_t *)(m->data + offset);
+  *sz_out = m->size - offset;
+
+  return m;
+ err:
+  tor_munmap_file(m);
+  errno = errval;
+  return NULL;
+}
+
+/** As storage_dir_map_labeled, but return a new byte array containing the
+ * data. */
+uint8_t *
+storage_dir_read_labeled(storage_dir_t *dir,
+                          const char *fname,
+                          config_line_t **labels_out,
+                          size_t *sz_out)
+{
+  const uint8_t *data = NULL;
+  tor_mmap_t *m = storage_dir_map_labeled(dir, fname, labels_out,
+                                           &data, sz_out);
+  if (m == NULL)
+    return NULL;
+  uint8_t *result = tor_memdup(data, *sz_out);
+  tor_munmap_file(m);
+  return result;
+}
+
+/* Reduce the cached usage amount in <b>d</b> by <b>removed_file_size</b>.
+ * This function is a no-op if <b>d->usage_known</b> is 0. */
+static void
+storage_dir_reduce_usage(storage_dir_t *d, uint64_t removed_file_size)
+{
+  if (d->usage_known) {
+    if (! BUG(d->usage < removed_file_size)) {
+      /* This bug can also be triggered if an external process resized a file
+       * between the call to storage_dir_get_usage() that last checked
+       * actual usage (rather than relaying on cached usage), and the call to
+       * this function. */
+      d->usage -= removed_file_size;
+    } else {
+      /* If we underflowed the cached directory size, re-check the sizes of all
+       * the files in the directory. This makes storage_dir_shrink() quadratic,
+       * but only if a process is continually changing file sizes in the
+       * storage directory (in which case, we have bigger issues).
+       *
+       * We can't just reset usage_known, because storage_dir_shrink() relies
+       * on knowing the usage. */
+      storage_dir_rescan(d);
+      (void)storage_dir_get_usage(d);
+    }
+  }
+}
+
+/**
+ * Remove the file called <b>fname</b> from <b>d</b>.
+ */
+void
+storage_dir_remove_file(storage_dir_t *d,
+                        const char *fname)
+{
+  char *path = NULL;
+  tor_asprintf(&path, "%s/%s", d->directory, fname);
+  const char *ipath = sandbox_intern_string(path);
+
+  uint64_t size = 0;
+  if (d->usage_known) {
+    struct stat st;
+    if (stat(ipath, &st) == 0) {
+      size = st.st_size;
+    }
+  }
+  if (unlink(ipath) == 0) {
+    storage_dir_reduce_usage(d, size);
+  } else {
+    log_warn(LD_FS, "Unable to unlink %s while removing file: %s",
+             escaped(path), strerror(errno));
+    tor_free(path);
+    return;
+  }
+  if (d->contents) {
+    smartlist_string_remove(d->contents, fname);
+  }
+
+  tor_free(path);
+}
+
+/** Helper type: used to sort the members of storage directory by mtime. */
+typedef struct shrinking_dir_entry_t {
+  time_t mtime;
+  uint64_t size;
+  char *path;
+} shrinking_dir_entry_t;
+
+/** Helper: use with qsort to sort shrinking_dir_entry_t structs. */
+static int
+shrinking_dir_entry_compare(const void *a_, const void *b_)
+{
+  const shrinking_dir_entry_t *a = a_;
+  const shrinking_dir_entry_t *b = b_;
+
+  if (a->mtime < b->mtime)
+    return -1;
+  else if (a->mtime > b->mtime)
+    return 1;
+  else
+    return 0;
+}
+
+/**
+ * Try to free space by removing the oldest files in <b>d</b>. Delete
+ * until no more than <b>target_size</b> bytes are left, and at least
+ * <b>min_to_remove</b> files have been removed... or until there is
+ * nothing left to remove.
+ *
+ * Return 0 on success; -1 on failure.
+ */
+int
+storage_dir_shrink(storage_dir_t *d,
+                   uint64_t target_size,
+                   int min_to_remove)
+{
+  if (d->usage_known && d->usage <= target_size && !min_to_remove) {
+    /* Already small enough. */
+    return 0;
+  }
+
+  if (storage_dir_rescan(d) < 0)
+    return -1;
+
+  const uint64_t orig_usage = storage_dir_get_usage(d);
+  if (orig_usage <= target_size && !min_to_remove) {
+    /* Okay, small enough after rescan! */
+    return 0;
+  }
+
+  const int n = smartlist_len(d->contents);
+  shrinking_dir_entry_t *ents = tor_calloc(n, sizeof(shrinking_dir_entry_t));
+  SMARTLIST_FOREACH_BEGIN(d->contents, const char *, fname) {
+    shrinking_dir_entry_t *ent = &ents[fname_sl_idx];
+    struct stat st;
+    tor_asprintf(&ent->path, "%s/%s", d->directory, fname);
+    if (stat(sandbox_intern_string(ent->path), &st) == 0) {
+      ent->mtime = st.st_mtime;
+      ent->size = st.st_size;
+    }
+  } SMARTLIST_FOREACH_END(fname);
+
+  qsort(ents, n, sizeof(shrinking_dir_entry_t), shrinking_dir_entry_compare);
+
+  int idx = 0;
+  while ((d->usage > target_size || min_to_remove > 0) && idx < n) {
+    if (unlink(sandbox_intern_string(ents[idx].path)) == 0) {
+      storage_dir_reduce_usage(d, ents[idx].size);
+      --min_to_remove;
+    }
+    ++idx;
+  }
+
+  for (idx = 0; idx < n; ++idx) {
+    tor_free(ents[idx].path);
+  }
+  tor_free(ents);
+
+  storage_dir_rescan(d);
+
+  return 0;
+}
+
+/** Remove all files in <b>d</b>. */
+int
+storage_dir_remove_all(storage_dir_t *d)
+{
+  return storage_dir_shrink(d, 0, d->max_files);
+}
+
+/**
+ * Return the largest number of non-temporary files we're willing to
+ * store in <b>d</b>.
+ */
+int
+storage_dir_get_max_files(storage_dir_t *d)
+{
+  return d->max_files;
+}
diff --git a/src/lib/fs/storagedir.h b/src/lib/fs/storagedir.h
new file mode 100644
index 0000000000..7e6633a0bb
--- /dev/null
+++ b/src/lib/fs/storagedir.h
@@ -0,0 +1,64 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file storagedir.h
+ *
+ * \brief Header for storagedir.c
+ **/
+
+#ifndef TOR_STORAGEDIR_H
+#define TOR_STORAGEDIR_H
+
+#include "lib/cc/torint.h"
+#include <stddef.h>
+
+typedef struct storage_dir_t storage_dir_t;
+struct config_line_t;
+struct sandbox_cfg_elem;
+struct tor_mmap_t;
+struct smartlist_t;
+
+storage_dir_t * storage_dir_new(const char *dirname, int n_files);
+void storage_dir_free_(storage_dir_t *d);
+#define storage_dir_free(d) \
+  FREE_AND_NULL(storage_dir_t, storage_dir_free_, (d))
+
+int storage_dir_register_with_sandbox(storage_dir_t *d,
+                                      struct sandbox_cfg_elem **cfg);
+const struct smartlist_t *storage_dir_list(storage_dir_t *d);
+uint64_t storage_dir_get_usage(storage_dir_t *d);
+struct tor_mmap_t *storage_dir_map(storage_dir_t *d, const char *fname);
+uint8_t *storage_dir_read(storage_dir_t *d, const char *fname, int bin,
+                          size_t *sz_out);
+int storage_dir_save_bytes_to_file(storage_dir_t *d,
+                                   const uint8_t *data,
+                                   size_t length,
+                                   int binary,
+                                   char **fname_out);
+int storage_dir_save_string_to_file(storage_dir_t *d,
+                                    const char *data,
+                                    int binary,
+                                    char **fname_out);
+int storage_dir_save_labeled_to_file(storage_dir_t *d,
+                                      const struct config_line_t *labels,
+                                      const uint8_t *data,
+                                      size_t length,
+                                      char **fname_out);
+struct tor_mmap_t *storage_dir_map_labeled(storage_dir_t *dir,
+                                     const char *fname,
+                                     struct config_line_t **labels_out,
+                                     const uint8_t **data_out,
+                                     size_t *size_out);
+uint8_t *storage_dir_read_labeled(storage_dir_t *d, const char *fname,
+                                   struct config_line_t **labels_out,
+                                   size_t *sz_out);
+void storage_dir_remove_file(storage_dir_t *d,
+                             const char *fname);
+int storage_dir_shrink(storage_dir_t *d,
+                       uint64_t target_size,
+                       int min_to_remove);
+int storage_dir_remove_all(storage_dir_t *d);
+int storage_dir_get_max_files(storage_dir_t *d);
+
+#endif /* !defined(TOR_STORAGEDIR_H) */
diff --git a/src/lib/fs/userdb.c b/src/lib/fs/userdb.c
new file mode 100644
index 0000000000..95205c670e
--- /dev/null
+++ b/src/lib/fs/userdb.c
@@ -0,0 +1,138 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file userdb.c
+ *
+ * \brief Access the POSIX user database.
+ **/
+
+#include "lib/fs/userdb.h"
+
+#ifndef _WIN32
+#include "lib/malloc/malloc.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+
+#include <pwd.h>
+#include <stddef.h>
+#include <string.h>
+
+/** Cached struct from the last getpwname() call we did successfully. */
+static struct passwd *passwd_cached = NULL;
+
+/** Helper: copy a struct passwd object.
+ *
+ * We only copy the fields pw_uid, pw_gid, pw_name, pw_dir.  Tor doesn't use
+ * any others, and I don't want to run into incompatibilities.
+ */
+static struct passwd *
+tor_passwd_dup(const struct passwd *pw)
+{
+  struct passwd *new_pw = tor_malloc_zero(sizeof(struct passwd));
+  if (pw->pw_name)
+    new_pw->pw_name = tor_strdup(pw->pw_name);
+  if (pw->pw_dir)
+    new_pw->pw_dir = tor_strdup(pw->pw_dir);
+  new_pw->pw_uid = pw->pw_uid;
+  new_pw->pw_gid = pw->pw_gid;
+
+  return new_pw;
+}
+
+#define tor_passwd_free(pw) \
+  FREE_AND_NULL(struct passwd, tor_passwd_free_, (pw))
+
+/** Helper: free one of our cached 'struct passwd' values. */
+static void
+tor_passwd_free_(struct passwd *pw)
+{
+  if (!pw)
+    return;
+
+  tor_free(pw->pw_name);
+  tor_free(pw->pw_dir);
+  tor_free(pw);
+}
+
+/** Wrapper around getpwnam() that caches result. Used so that we don't need
+ * to give the sandbox access to /etc/passwd.
+ *
+ * The following fields alone will definitely be copied in the output: pw_uid,
+ * pw_gid, pw_name, pw_dir.  Other fields are not present in cached values.
+ *
+ * When called with a NULL argument, this function clears storage associated
+ * with static variables it uses.
+ **/
+const struct passwd *
+tor_getpwnam(const char *username)
+{
+  struct passwd *pw;
+
+  if (username == NULL) {
+    tor_passwd_free(passwd_cached);
+    passwd_cached = NULL;
+    return NULL;
+  }
+
+  if ((pw = getpwnam(username))) {
+    tor_passwd_free(passwd_cached);
+    passwd_cached = tor_passwd_dup(pw);
+    log_info(LD_GENERAL, "Caching new entry %s for %s",
+             passwd_cached->pw_name, username);
+    return pw;
+  }
+
+  /* Lookup failed */
+  if (! passwd_cached || ! passwd_cached->pw_name)
+    return NULL;
+
+  if (! strcmp(username, passwd_cached->pw_name))
+    return passwd_cached; // LCOV_EXCL_LINE - would need to make getpwnam flaky
+
+  return NULL;
+}
+
+/** Wrapper around getpwnam() that can use cached result from
+ * tor_getpwnam(). Used so that we don't need to give the sandbox access to
+ * /etc/passwd.
+ *
+ * The following fields alone will definitely be copied in the output: pw_uid,
+ * pw_gid, pw_name, pw_dir.  Other fields are not present in cached values.
+ */
+const struct passwd *
+tor_getpwuid(uid_t uid)
+{
+  struct passwd *pw;
+
+  if ((pw = getpwuid(uid))) {
+    return pw;
+  }
+
+  /* Lookup failed */
+  if (! passwd_cached)
+    return NULL;
+
+  if (uid == passwd_cached->pw_uid)
+    return passwd_cached; // LCOV_EXCL_LINE - would need to make getpwnam flaky
+
+  return NULL;
+}
+
+/** Allocate and return a string containing the home directory for the
+ * user <b>username</b>. Only works on posix-like systems. */
+char *
+get_user_homedir(const char *username)
+{
+  const struct passwd *pw;
+  tor_assert(username);
+
+  if (!(pw = tor_getpwnam(username))) {
+    log_err(LD_CONFIG,"User \"%s\" not found.", username);
+    return NULL;
+  }
+  return tor_strdup(pw->pw_dir);
+}
+#endif /* !defined(_WIN32) */
diff --git a/src/lib/fs/userdb.h b/src/lib/fs/userdb.h
new file mode 100644
index 0000000000..5c39794873
--- /dev/null
+++ b/src/lib/fs/userdb.h
@@ -0,0 +1,26 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file userdb.h
+ *
+ * \brief Header for userdb.c
+ **/
+
+#ifndef TOR_USERDB_H
+#define TOR_USERDB_H
+
+#include "orconfig.h"
+
+#ifndef _WIN32
+#include <sys/types.h>
+
+struct passwd;
+const struct passwd *tor_getpwnam(const char *username);
+const struct passwd *tor_getpwuid(uid_t uid);
+char *get_user_homedir(const char *username);
+#endif
+
+#endif
diff --git a/src/lib/fs/winlib.c b/src/lib/fs/winlib.c
new file mode 100644
index 0000000000..b7302bd4ca
--- /dev/null
+++ b/src/lib/fs/winlib.c
@@ -0,0 +1,30 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file winlib.c
+ *
+ * \brief Find and load windows system libraries.
+ *
+ * We use this function to dynamically load code at runtime that might not be
+ * available on all versions of Windows that we support.
+ **/
+
+#ifdef _WIN32
+#include "lib/fs/winlib.h"
+
+HANDLE
+load_windows_system_library(const TCHAR *library_name)
+{
+  TCHAR path[MAX_PATH];
+  unsigned n;
+  n = GetSystemDirectory(path, MAX_PATH);
+  if (n == 0 || n + _tcslen(library_name) + 2 >= MAX_PATH)
+    return 0;
+  _tcscat(path, TEXT("\\"));
+  _tcscat(path, library_name);
+  return LoadLibrary(path);
+}
+#endif /* defined(_WIN32) */
diff --git a/src/lib/fs/winlib.h b/src/lib/fs/winlib.h
new file mode 100644
index 0000000000..64a22439e5
--- /dev/null
+++ b/src/lib/fs/winlib.h
@@ -0,0 +1,22 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file winlib.h
+ *
+ * \brief Header for winlib.c
+ **/
+
+#ifndef TOR_WINLIB_H
+#define TOR_WINLIB_H
+
+#ifdef _WIN32
+#include <windows.h>
+#include <tchar.h>
+
+HANDLE load_windows_system_library(const TCHAR *library_name);
+#endif
+
+#endif
diff --git a/src/lib/geoip/.may_include b/src/lib/geoip/.may_include
new file mode 100644
index 0000000000..b1ee2dcfe9
--- /dev/null
+++ b/src/lib/geoip/.may_include
@@ -0,0 +1,13 @@
+orconfig.h
+lib/cc/*.h
+lib/container/*.h
+lib/crypt_ops/*.h
+lib/ctime/*.h
+lib/encoding/*.h
+lib/fs/*.h
+lib/geoip/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/net/*.h
+lib/string/*.h
+lib/testsupport/*.h
diff --git a/src/lib/geoip/country.h b/src/lib/geoip/country.h
new file mode 100644
index 0000000000..9a8911d494
--- /dev/null
+++ b/src/lib/geoip/country.h
@@ -0,0 +1,16 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_COUNTRY_H
+#define TOR_COUNTRY_H
+
+#include "lib/cc/torint.h"
+/** A signed integer representing a country code. */
+typedef int16_t country_t;
+
+#define COUNTRY_MAX INT16_MAX
+
+#endif
diff --git a/src/lib/geoip/geoip.c b/src/lib/geoip/geoip.c
new file mode 100644
index 0000000000..70b1c2dc8c
--- /dev/null
+++ b/src/lib/geoip/geoip.c
@@ -0,0 +1,510 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file geoip.c
+ * \brief Functions related to maintaining an IP-to-country database;
+ * to summarizing client connections by country to entry guards, bridges,
+ * and directory servers; and for statistics on answering network status
+ * requests.
+ *
+ * There are two main kinds of functions in this module: geoip functions,
+ * which map groups of IPv4 and IPv6 addresses to country codes, and
+ * statistical functions, which collect statistics about different kinds of
+ * per-country usage.
+ *
+ * The geoip lookup tables are implemented as sorted lists of disjoint address
+ * ranges, each mapping to a singleton geoip_country_t.  These country objects
+ * are also indexed by their names in a hashtable.
+ *
+ * The tables are populated from disk at startup by the geoip_load_file()
+ * function.  For more information on the file format they read, see that
+ * function.  See the scripts and the README file in src/config for more
+ * information about how those files are generated.
+ *
+ * Tor uses GeoIP information in order to implement user requests (such as
+ * ExcludeNodes {cc}), and to keep track of how much usage relays are getting
+ * for each country.
+ */
+
+#define GEOIP_PRIVATE
+#include "lib/geoip/geoip.h"
+#include "lib/container/map.h"
+#include "lib/container/order.h"
+#include "lib/container/smartlist.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/encoding/binascii.h"
+#include "lib/fs/files.h"
+#include "lib/log/escape.h"
+#include "lib/malloc/malloc.h"
+#include "lib/net/address.h" //????
+#include "lib/net/inaddr.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/string/compat_string.h"
+#include "lib/string/scanf.h"
+#include "lib/string/util_string.h"
+
+#include <stdio.h>
+#include <string.h>
+
+static void init_geoip_countries(void);
+
+/** An entry from the GeoIP IPv4 file: maps an IPv4 range to a country. */
+typedef struct geoip_ipv4_entry_t {
+  uint32_t ip_low; /**< The lowest IP in the range, in host order */
+  uint32_t ip_high; /**< The highest IP in the range, in host order */
+  intptr_t country; /**< An index into geoip_countries */
+} geoip_ipv4_entry_t;
+
+/** An entry from the GeoIP IPv6 file: maps an IPv6 range to a country. */
+typedef struct geoip_ipv6_entry_t {
+  struct in6_addr ip_low; /**< The lowest IP in the range, in host order */
+  struct in6_addr ip_high; /**< The highest IP in the range, in host order */
+  intptr_t country; /**< An index into geoip_countries */
+} geoip_ipv6_entry_t;
+
+/** A list of geoip_country_t */
+static smartlist_t *geoip_countries = NULL;
+/** A map from lowercased country codes to their position in geoip_countries.
+ * The index is encoded in the pointer, and 1 is added so that NULL can mean
+ * not found. */
+static strmap_t *country_idxplus1_by_lc_code = NULL;
+/** Lists of all known geoip_ipv4_entry_t and geoip_ipv6_entry_t, sorted
+ * by their respective ip_low. */
+static smartlist_t *geoip_ipv4_entries = NULL, *geoip_ipv6_entries = NULL;
+
+/** SHA1 digest of the GeoIP files to include in extra-info descriptors. */
+static char geoip_digest[DIGEST_LEN];
+static char geoip6_digest[DIGEST_LEN];
+
+/** Return a list of geoip_country_t for all known countries. */
+const smartlist_t *
+geoip_get_countries(void)
+{
+  if (geoip_countries == NULL) {
+    init_geoip_countries();
+  }
+  return geoip_countries;
+}
+
+/** Return the index of the <b>country</b>'s entry in the GeoIP
+ * country list if it is a valid 2-letter country code, otherwise
+ * return -1. */
+MOCK_IMPL(country_t,
+geoip_get_country,(const char *country))
+{
+  void *idxplus1_;
+  intptr_t idx;
+
+  idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country);
+  if (!idxplus1_)
+    return -1;
+
+  idx = ((uintptr_t)idxplus1_)-1;
+  return (country_t)idx;
+}
+
+/** Add an entry to a GeoIP table, mapping all IP addresses between <b>low</b>
+ * and <b>high</b>, inclusive, to the 2-letter country code <b>country</b>. */
+static void
+geoip_add_entry(const tor_addr_t *low, const tor_addr_t *high,
+                const char *country)
+{
+  intptr_t idx;
+  void *idxplus1_;
+
+  IF_BUG_ONCE(tor_addr_family(low) != tor_addr_family(high))
+    return;
+  IF_BUG_ONCE(tor_addr_compare(high, low, CMP_EXACT) < 0)
+    return;
+
+  idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country);
+
+  if (!idxplus1_) {
+    geoip_country_t *c = tor_malloc_zero(sizeof(geoip_country_t));
+    strlcpy(c->countrycode, country, sizeof(c->countrycode));
+    tor_strlower(c->countrycode);
+    smartlist_add(geoip_countries, c);
+    idx = smartlist_len(geoip_countries) - 1;
+    strmap_set_lc(country_idxplus1_by_lc_code, country, (void*)(idx+1));
+  } else {
+    idx = ((uintptr_t)idxplus1_)-1;
+  }
+  {
+    geoip_country_t *c = smartlist_get(geoip_countries, (int)idx);
+    tor_assert(!strcasecmp(c->countrycode, country));
+  }
+
+  if (tor_addr_family(low) == AF_INET) {
+    geoip_ipv4_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv4_entry_t));
+    ent->ip_low = tor_addr_to_ipv4h(low);
+    ent->ip_high = tor_addr_to_ipv4h(high);
+    ent->country = idx;
+    smartlist_add(geoip_ipv4_entries, ent);
+  } else if (tor_addr_family(low) == AF_INET6) {
+    geoip_ipv6_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv6_entry_t));
+    ent->ip_low = *tor_addr_to_in6_assert(low);
+    ent->ip_high = *tor_addr_to_in6_assert(high);
+    ent->country = idx;
+    smartlist_add(geoip_ipv6_entries, ent);
+  }
+}
+
+/** Add an entry to the GeoIP table indicated by <b>family</b>,
+ * parsing it from <b>line</b>. The format is as for geoip_load_file(). */
+STATIC int
+geoip_parse_entry(const char *line, sa_family_t family)
+{
+  tor_addr_t low_addr, high_addr;
+  char c[3];
+  char *country = NULL;
+
+  if (!geoip_countries)
+    init_geoip_countries();
+  if (family == AF_INET) {
+    if (!geoip_ipv4_entries)
+      geoip_ipv4_entries = smartlist_new();
+  } else if (family == AF_INET6) {
+    if (!geoip_ipv6_entries)
+      geoip_ipv6_entries = smartlist_new();
+  } else {
+    log_warn(LD_GENERAL, "Unsupported family: %d", family);
+    return -1;
+  }
+
+  while (TOR_ISSPACE(*line))
+    ++line;
+  if (*line == '#')
+    return 0;
+
+  char buf[512];
+  if (family == AF_INET) {
+    unsigned int low, high;
+    if (tor_sscanf(line,"%u,%u,%2s", &low, &high, c) == 3 ||
+        tor_sscanf(line,"\"%u\",\"%u\",\"%2s\",", &low, &high, c) == 3) {
+      tor_addr_from_ipv4h(&low_addr, low);
+      tor_addr_from_ipv4h(&high_addr, high);
+    } else
+      goto fail;
+    country = c;
+  } else {                      /* AF_INET6 */
+    char *low_str, *high_str;
+    struct in6_addr low, high;
+    char *strtok_state;
+    strlcpy(buf, line, sizeof(buf));
+    low_str = tor_strtok_r(buf, ",", &strtok_state);
+    if (!low_str)
+      goto fail;
+    high_str = tor_strtok_r(NULL, ",", &strtok_state);
+    if (!high_str)
+      goto fail;
+    country = tor_strtok_r(NULL, "\n", &strtok_state);
+    if (!country)
+      goto fail;
+    if (strlen(country) != 2)
+      goto fail;
+    if (tor_inet_pton(AF_INET6, low_str, &low) <= 0)
+      goto fail;
+    tor_addr_from_in6(&low_addr, &low);
+    if (tor_inet_pton(AF_INET6, high_str, &high) <= 0)
+      goto fail;
+    tor_addr_from_in6(&high_addr, &high);
+  }
+  geoip_add_entry(&low_addr, &high_addr, country);
+  return 0;
+
+  fail:
+  log_warn(LD_GENERAL, "Unable to parse line from GEOIP %s file: %s",
+           family == AF_INET ? "IPv4" : "IPv6", escaped(line));
+  return -1;
+}
+
+/** Sorting helper: return -1, 1, or 0 based on comparison of two
+ * geoip_ipv4_entry_t */
+static int
+geoip_ipv4_compare_entries_(const void **_a, const void **_b)
+{
+  const geoip_ipv4_entry_t *a = *_a, *b = *_b;
+  if (a->ip_low < b->ip_low)
+    return -1;
+  else if (a->ip_low > b->ip_low)
+    return 1;
+  else
+    return 0;
+}
+
+/** bsearch helper: return -1, 1, or 0 based on comparison of an IP (a pointer
+ * to a uint32_t in host order) to a geoip_ipv4_entry_t */
+static int
+geoip_ipv4_compare_key_to_entry_(const void *_key, const void **_member)
+{
+  /* No alignment issue here, since _key really is a pointer to uint32_t */
+  const uint32_t addr = *(uint32_t *)_key;
+  const geoip_ipv4_entry_t *entry = *_member;
+  if (addr < entry->ip_low)
+    return -1;
+  else if (addr > entry->ip_high)
+    return 1;
+  else
+    return 0;
+}
+
+/** Sorting helper: return -1, 1, or 0 based on comparison of two
+ * geoip_ipv6_entry_t */
+static int
+geoip_ipv6_compare_entries_(const void **_a, const void **_b)
+{
+  const geoip_ipv6_entry_t *a = *_a, *b = *_b;
+  return fast_memcmp(a->ip_low.s6_addr, b->ip_low.s6_addr,
+                     sizeof(struct in6_addr));
+}
+
+/** bsearch helper: return -1, 1, or 0 based on comparison of an IPv6
+ * (a pointer to a in6_addr) to a geoip_ipv6_entry_t */
+static int
+geoip_ipv6_compare_key_to_entry_(const void *_key, const void **_member)
+{
+  const struct in6_addr *addr = (struct in6_addr *)_key;
+  const geoip_ipv6_entry_t *entry = *_member;
+
+  if (fast_memcmp(addr->s6_addr, entry->ip_low.s6_addr,
+             sizeof(struct in6_addr)) < 0)
+    return -1;
+  else if (fast_memcmp(addr->s6_addr, entry->ip_high.s6_addr,
+                  sizeof(struct in6_addr)) > 0)
+    return 1;
+  else
+    return 0;
+}
+
+/** Set up a new list of geoip countries with no countries (yet) set in it,
+ * except for the unknown country.
+ */
+static void
+init_geoip_countries(void)
+{
+  geoip_country_t *geoip_unresolved;
+  geoip_countries = smartlist_new();
+  /* Add a geoip_country_t for requests that could not be resolved to a
+   * country as first element (index 0) to geoip_countries. */
+  geoip_unresolved = tor_malloc_zero(sizeof(geoip_country_t));
+  strlcpy(geoip_unresolved->countrycode, "??",
+          sizeof(geoip_unresolved->countrycode));
+  smartlist_add(geoip_countries, geoip_unresolved);
+  country_idxplus1_by_lc_code = strmap_new();
+  strmap_set_lc(country_idxplus1_by_lc_code, "??", (void*)(1));
+}
+
+/** Clear appropriate GeoIP database, based on <b>family</b>, and
+ * reload it from the file <b>filename</b>. Return 0 on success, -1 on
+ * failure.
+ *
+ * Recognized line formats for IPv4 are:
+ *   INTIPLOW,INTIPHIGH,CC
+ * and
+ *   "INTIPLOW","INTIPHIGH","CC","CC3","COUNTRY NAME"
+ * where INTIPLOW and INTIPHIGH are IPv4 addresses encoded as 4-byte unsigned
+ * integers, and CC is a country code.
+ *
+ * Recognized line format for IPv6 is:
+ *   IPV6LOW,IPV6HIGH,CC
+ * where IPV6LOW and IPV6HIGH are IPv6 addresses and CC is a country code.
+ *
+ * It also recognizes, and skips over, blank lines and lines that start
+ * with '#' (comments).
+ */
+int
+geoip_load_file(sa_family_t family, const char *filename, int severity)
+{
+  FILE *f;
+  crypto_digest_t *geoip_digest_env = NULL;
+
+  tor_assert(family == AF_INET || family == AF_INET6);
+
+  if (!(f = tor_fopen_cloexec(filename, "r"))) {
+    log_fn(severity, LD_GENERAL, "Failed to open GEOIP file %s.",
+           filename);
+    return -1;
+  }
+  if (!geoip_countries)
+    init_geoip_countries();
+
+  if (family == AF_INET) {
+    if (geoip_ipv4_entries) {
+      SMARTLIST_FOREACH(geoip_ipv4_entries, geoip_ipv4_entry_t *, e,
+                        tor_free(e));
+      smartlist_free(geoip_ipv4_entries);
+    }
+    geoip_ipv4_entries = smartlist_new();
+  } else { /* AF_INET6 */
+    if (geoip_ipv6_entries) {
+      SMARTLIST_FOREACH(geoip_ipv6_entries, geoip_ipv6_entry_t *, e,
+                        tor_free(e));
+      smartlist_free(geoip_ipv6_entries);
+    }
+    geoip_ipv6_entries = smartlist_new();
+  }
+  geoip_digest_env = crypto_digest_new();
+
+  log_notice(LD_GENERAL, "Parsing GEOIP %s file %s.",
+             (family == AF_INET) ? "IPv4" : "IPv6", filename);
+  while (!feof(f)) {
+    char buf[512];
+    if (fgets(buf, (int)sizeof(buf), f) == NULL)
+      break;
+    crypto_digest_add_bytes(geoip_digest_env, buf, strlen(buf));
+    /* FFFF track full country name. */
+    geoip_parse_entry(buf, family);
+  }
+  /*XXXX abort and return -1 if no entries/illformed?*/
+  fclose(f);
+
+  /* Sort list and remember file digests so that we can include it in
+   * our extra-info descriptors. */
+  if (family == AF_INET) {
+    smartlist_sort(geoip_ipv4_entries, geoip_ipv4_compare_entries_);
+    crypto_digest_get_digest(geoip_digest_env, geoip_digest, DIGEST_LEN);
+  } else {
+    /* AF_INET6 */
+    smartlist_sort(geoip_ipv6_entries, geoip_ipv6_compare_entries_);
+    crypto_digest_get_digest(geoip_digest_env, geoip6_digest, DIGEST_LEN);
+  }
+  crypto_digest_free(geoip_digest_env);
+
+  return 0;
+}
+
+/** Given an IP address in host order, return a number representing the
+ * country to which that address belongs, -1 for "No geoip information
+ * available", or 0 for the 'unknown country'.  The return value will always
+ * be less than geoip_get_n_countries().  To decode it, call
+ * geoip_get_country_name().
+ */
+int
+geoip_get_country_by_ipv4(uint32_t ipaddr)
+{
+  geoip_ipv4_entry_t *ent;
+  if (!geoip_ipv4_entries)
+    return -1;
+  ent = smartlist_bsearch(geoip_ipv4_entries, &ipaddr,
+                          geoip_ipv4_compare_key_to_entry_);
+  return ent ? (int)ent->country : 0;
+}
+
+/** Given an IPv6 address, return a number representing the country to
+ * which that address belongs, -1 for "No geoip information available", or
+ * 0 for the 'unknown country'.  The return value will always be less than
+ * geoip_get_n_countries().  To decode it, call geoip_get_country_name().
+ */
+int
+geoip_get_country_by_ipv6(const struct in6_addr *addr)
+{
+  geoip_ipv6_entry_t *ent;
+
+  if (!geoip_ipv6_entries)
+    return -1;
+  ent = smartlist_bsearch(geoip_ipv6_entries, addr,
+                          geoip_ipv6_compare_key_to_entry_);
+  return ent ? (int)ent->country : 0;
+}
+
+/** Given an IP address, return a number representing the country to which
+ * that address belongs, -1 for "No geoip information available", or 0 for
+ * the 'unknown country'.  The return value will always be less than
+ * geoip_get_n_countries().  To decode it, call geoip_get_country_name().
+ */
+MOCK_IMPL(int,
+geoip_get_country_by_addr,(const tor_addr_t *addr))
+{
+  if (tor_addr_family(addr) == AF_INET) {
+    return geoip_get_country_by_ipv4(tor_addr_to_ipv4h(addr));
+  } else if (tor_addr_family(addr) == AF_INET6) {
+    return geoip_get_country_by_ipv6(tor_addr_to_in6(addr));
+  } else {
+    return -1;
+  }
+}
+
+/** Return the number of countries recognized by the GeoIP country list. */
+MOCK_IMPL(int,
+geoip_get_n_countries,(void))
+{
+  if (!geoip_countries)
+    init_geoip_countries();
+  return (int) smartlist_len(geoip_countries);
+}
+
+/** Return the two-letter country code associated with the number <b>num</b>,
+ * or "??" for an unknown value. */
+const char *
+geoip_get_country_name(country_t num)
+{
+  if (geoip_countries && num >= 0 && num < smartlist_len(geoip_countries)) {
+    geoip_country_t *c = smartlist_get(geoip_countries, num);
+    return c->countrycode;
+  } else
+    return "??";
+}
+
+/** Return true iff we have loaded a GeoIP database.*/
+MOCK_IMPL(int,
+geoip_is_loaded,(sa_family_t family))
+{
+  tor_assert(family == AF_INET || family == AF_INET6);
+  if (geoip_countries == NULL)
+    return 0;
+  if (family == AF_INET)
+    return geoip_ipv4_entries != NULL;
+  else                          /* AF_INET6 */
+    return geoip_ipv6_entries != NULL;
+}
+
+/** Return the hex-encoded SHA1 digest of the loaded GeoIP file. The
+ * result does not need to be deallocated, but will be overwritten by the
+ * next call of hex_str(). */
+const char *
+geoip_db_digest(sa_family_t family)
+{
+  tor_assert(family == AF_INET || family == AF_INET6);
+  if (family == AF_INET)
+    return hex_str(geoip_digest, DIGEST_LEN);
+  else                          /* AF_INET6 */
+    return hex_str(geoip6_digest, DIGEST_LEN);
+}
+
+/** Release all storage held by the GeoIP databases and country list. */
+STATIC void
+clear_geoip_db(void)
+{
+  if (geoip_countries) {
+    SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, tor_free(c));
+    smartlist_free(geoip_countries);
+  }
+
+  strmap_free(country_idxplus1_by_lc_code, NULL);
+  if (geoip_ipv4_entries) {
+    SMARTLIST_FOREACH(geoip_ipv4_entries, geoip_ipv4_entry_t *, ent,
+                      tor_free(ent));
+    smartlist_free(geoip_ipv4_entries);
+  }
+  if (geoip_ipv6_entries) {
+    SMARTLIST_FOREACH(geoip_ipv6_entries, geoip_ipv6_entry_t *, ent,
+                      tor_free(ent));
+    smartlist_free(geoip_ipv6_entries);
+  }
+  geoip_countries = NULL;
+  country_idxplus1_by_lc_code = NULL;
+  geoip_ipv4_entries = NULL;
+  geoip_ipv6_entries = NULL;
+}
+
+/** Release all storage held in this file. */
+void
+geoip_free_all(void)
+{
+  clear_geoip_db();
+
+  memset(geoip_digest, 0, sizeof(geoip_digest));
+  memset(geoip6_digest, 0, sizeof(geoip6_digest));
+}
diff --git a/src/lib/geoip/geoip.h b/src/lib/geoip/geoip.h
new file mode 100644
index 0000000000..f872ebd25f
--- /dev/null
+++ b/src/lib/geoip/geoip.h
@@ -0,0 +1,50 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file geoip.h
+ * \brief Header file for geoip.c.
+ **/
+
+#ifndef TOR_GEOIP_H
+#define TOR_GEOIP_H
+
+#include "orconfig.h"
+#include "lib/net/nettypes.h"
+#include "lib/testsupport/testsupport.h"
+#include "lib/net/inaddr_st.h"
+#include "lib/geoip/country.h"
+
+#ifdef GEOIP_PRIVATE
+STATIC int geoip_parse_entry(const char *line, sa_family_t family);
+STATIC void clear_geoip_db(void);
+#endif /* defined(GEOIP_PRIVATE) */
+
+struct in6_addr;
+struct tor_addr_t;
+
+int geoip_get_country_by_ipv4(uint32_t ipaddr);
+int geoip_get_country_by_ipv6(const struct in6_addr *addr);
+
+/** A per-country GeoIP record. */
+typedef struct geoip_country_t {
+  char countrycode[3];
+} geoip_country_t;
+
+struct smartlist_t;
+const struct smartlist_t *geoip_get_countries(void);
+
+int geoip_load_file(sa_family_t family, const char *filename, int severity);
+MOCK_DECL(int, geoip_get_country_by_addr, (const struct tor_addr_t *addr));
+MOCK_DECL(int, geoip_get_n_countries, (void));
+const char *geoip_get_country_name(country_t num);
+MOCK_DECL(int, geoip_is_loaded, (sa_family_t family));
+const char *geoip_db_digest(sa_family_t family);
+MOCK_DECL(country_t, geoip_get_country, (const char *countrycode));
+
+void geoip_free_all(void);
+
+#endif /* !defined(TOR_GEOIP_H) */
diff --git a/src/lib/geoip/include.am b/src/lib/geoip/include.am
new file mode 100644
index 0000000000..9710d75ac7
--- /dev/null
+++ b/src/lib/geoip/include.am
@@ -0,0 +1,17 @@
+noinst_LIBRARIES += src/lib/libtor-geoip.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-geoip-testing.a
+endif
+
+src_lib_libtor_geoip_a_SOURCES =			\
+	src/lib/geoip/geoip.c
+
+src_lib_libtor_geoip_testing_a_SOURCES = \
+	$(src_lib_libtor_geoip_a_SOURCES)
+src_lib_libtor_geoip_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_geoip_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/geoip/geoip.h   			\
+	src/lib/geoip/country.h
diff --git a/src/lib/include.libdonna.am b/src/lib/include.libdonna.am
new file mode 100644
index 0000000000..60a3f5097e
--- /dev/null
+++ b/src/lib/include.libdonna.am
@@ -0,0 +1,24 @@
+src_lib_libcurve25519_donna_a_CFLAGS=$(TOR_CFLAGS_CRYPTO)
+
+if BUILD_CURVE25519_DONNA
+src_lib_libcurve25519_donna_a_SOURCES=\
+	src/ext/curve25519_donna/curve25519-donna.c
+# See bug 13538 -- this code is known to have signed overflow issues.
+src_lib_libcurve25519_donna_a_CFLAGS+=\
+	@F_OMIT_FRAME_POINTER@ @CFLAGS_CONSTTIME@
+noinst_LIBRARIES+=src/lib/libcurve25519_donna.a
+LIBDONNA=src/lib/libcurve25519_donna.a
+else
+if BUILD_CURVE25519_DONNA_C64
+src_lib_libcurve25519_donna_a_CFLAGS+=@CFLAGS_CONSTTIME@
+src_lib_libcurve25519_donna_a_SOURCES=\
+	src/ext/curve25519_donna/curve25519-donna-c64.c
+noinst_LIBRARIES+=src/lib/libcurve25519_donna.a
+LIBDONNA=src/lib/libcurve25519_donna.a
+else
+LIBDONNA=
+endif
+endif
+
+LIBDONNA += $(LIBED25519_REF10)
+LIBDONNA += $(LIBED25519_DONNA)
diff --git a/src/lib/intmath/.may_include b/src/lib/intmath/.may_include
new file mode 100644
index 0000000000..d96c112220
--- /dev/null
+++ b/src/lib/intmath/.may_include
@@ -0,0 +1,4 @@
+orconfig.h
+lib/cc/*.h
+lib/err/*.h
+lib/intmath/*.h
diff --git a/src/lib/intmath/addsub.c b/src/lib/intmath/addsub.c
new file mode 100644
index 0000000000..12146f4e72
--- /dev/null
+++ b/src/lib/intmath/addsub.c
@@ -0,0 +1,28 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file addsub.c
+ *
+ * \brief Helpers for addition and subtraction.
+ *
+ * Currently limited to non-wrapping (saturating) addition.
+ **/
+
+#include "lib/intmath/addsub.h"
+#include "lib/cc/compat_compiler.h"
+
+/* Helper: safely add two uint32_t's, capping at UINT32_MAX rather
+ * than overflow */
+uint32_t
+tor_add_u32_nowrap(uint32_t a, uint32_t b)
+{
+  /* a+b > UINT32_MAX check, without overflow */
+  if (PREDICT_UNLIKELY(a > UINT32_MAX - b)) {
+    return UINT32_MAX;
+  } else {
+    return a+b;
+  }
+}
diff --git a/src/lib/intmath/addsub.h b/src/lib/intmath/addsub.h
new file mode 100644
index 0000000000..83efa82919
--- /dev/null
+++ b/src/lib/intmath/addsub.h
@@ -0,0 +1,19 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file addsub.h
+ *
+ * \brief Header for addsub.c
+ **/
+
+#ifndef TOR_INTMATH_ADDSUB_H
+#define TOR_INTMATH_ADDSUB_H
+
+#include "lib/cc/torint.h"
+
+uint32_t tor_add_u32_nowrap(uint32_t a, uint32_t b);
+
+#endif /* !defined(TOR_INTMATH_MULDIV_H) */
diff --git a/src/lib/intmath/bits.c b/src/lib/intmath/bits.c
new file mode 100644
index 0000000000..2158790e3f
--- /dev/null
+++ b/src/lib/intmath/bits.c
@@ -0,0 +1,94 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file bits.c
+ *
+ * \brief Count the bits in an integer, manipulate powers of 2, etc.
+ **/
+
+#include "lib/intmath/bits.h"
+
+/** Returns floor(log2(u64)).  If u64 is 0, (incorrectly) returns 0. */
+int
+tor_log2(uint64_t u64)
+{
+  int r = 0;
+  if (u64 >= (UINT64_C(1)<<32)) {
+    u64 >>= 32;
+    r = 32;
+  }
+  if (u64 >= (UINT64_C(1)<<16)) {
+    u64 >>= 16;
+    r += 16;
+  }
+  if (u64 >= (UINT64_C(1)<<8)) {
+    u64 >>= 8;
+    r += 8;
+  }
+  if (u64 >= (UINT64_C(1)<<4)) {
+    u64 >>= 4;
+    r += 4;
+  }
+  if (u64 >= (UINT64_C(1)<<2)) {
+    u64 >>= 2;
+    r += 2;
+  }
+  if (u64 >= (UINT64_C(1)<<1)) {
+    // u64 >>= 1; // not using this any more.
+    r += 1;
+  }
+  return r;
+}
+
+/** Return the power of 2 in range [1,UINT64_MAX] closest to <b>u64</b>.  If
+ * there are two powers of 2 equally close, round down. */
+uint64_t
+round_to_power_of_2(uint64_t u64)
+{
+  int lg2;
+  uint64_t low;
+  uint64_t high;
+  if (u64 == 0)
+    return 1;
+
+  lg2 = tor_log2(u64);
+  low = UINT64_C(1) << lg2;
+
+  if (lg2 == 63)
+    return low;
+
+  high = UINT64_C(1) << (lg2+1);
+  if (high - u64 < u64 - low)
+    return high;
+  else
+    return low;
+}
+
+/** Return the number of bits set in <b>v</b>. */
+int
+n_bits_set_u8(uint8_t v)
+{
+  static const int nybble_table[] = {
+    0, /* 0000 */
+    1, /* 0001 */
+    1, /* 0010 */
+    2, /* 0011 */
+    1, /* 0100 */
+    2, /* 0101 */
+    2, /* 0110 */
+    3, /* 0111 */
+    1, /* 1000 */
+    2, /* 1001 */
+    2, /* 1010 */
+    3, /* 1011 */
+    2, /* 1100 */
+    3, /* 1101 */
+    3, /* 1110 */
+    4, /* 1111 */
+  };
+
+  return nybble_table[v & 15] + nybble_table[v>>4];
+}
diff --git a/src/lib/intmath/bits.h b/src/lib/intmath/bits.h
new file mode 100644
index 0000000000..c1483a18b8
--- /dev/null
+++ b/src/lib/intmath/bits.h
@@ -0,0 +1,22 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file bits.h
+ *
+ * \brief Header for bits.c
+ **/
+
+#ifndef TOR_INTMATH_BITS_H
+#define TOR_INTMATH_BITS_H
+
+#include "lib/cc/torint.h"
+#include "lib/cc/compat_compiler.h"
+
+int tor_log2(uint64_t u64) ATTR_CONST;
+uint64_t round_to_power_of_2(uint64_t u64);
+int n_bits_set_u8(uint8_t v);
+
+#endif /* !defined(TOR_INTMATH_BITS_H) */
diff --git a/src/lib/intmath/cmp.h b/src/lib/intmath/cmp.h
new file mode 100644
index 0000000000..d0b0e8b954
--- /dev/null
+++ b/src/lib/intmath/cmp.h
@@ -0,0 +1,39 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file cmp.h
+ *
+ * \brief Macro definitions for MIN, MAX, and CLAMP.
+ **/
+
+#ifndef TOR_INTMATH_CMP_H
+#define TOR_INTMATH_CMP_H
+
+/** Macros for MIN/MAX.  Never use these when the arguments could have
+ * side-effects.
+ * {With GCC extensions we could probably define a safer MIN/MAX.  But
+ * depending on that safety would be dangerous, since not every platform
+ * has it.}
+ **/
+#ifndef MAX
+#define MAX(a,b) ( ((a)<(b)) ? (b) : (a) )
+#endif
+#ifndef MIN
+#define MIN(a,b) ( ((a)>(b)) ? (b) : (a) )
+#endif
+
+/* Return <b>v</b> if it's between <b>min</b> and <b>max</b>.  Otherwise
+ * return <b>min</b> if <b>v</b> is smaller than <b>min</b>, or <b>max</b> if
+ * <b>b</b> is larger than <b>max</b>.
+ *
+ * Requires that <b>min</b> is no more than <b>max</b>. May evaluate any of
+ * its arguments more than once! */
+#define CLAMP(min,v,max)                        \
+  ( ((v) < (min)) ? (min) :                     \
+    ((v) > (max)) ? (max) :                     \
+    (v) )
+
+#endif /* !defined(TOR_INTMATH_CMP_H) */
diff --git a/src/lib/intmath/include.am b/src/lib/intmath/include.am
new file mode 100644
index 0000000000..45ee3bd53b
--- /dev/null
+++ b/src/lib/intmath/include.am
@@ -0,0 +1,25 @@
+
+noinst_LIBRARIES += src/lib/libtor-intmath.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-intmath-testing.a
+endif
+
+src_lib_libtor_intmath_a_SOURCES =			\
+	src/lib/intmath/addsub.c			\
+	src/lib/intmath/bits.c				\
+	src/lib/intmath/muldiv.c			\
+	src/lib/intmath/weakrng.c
+
+src_lib_libtor_intmath_testing_a_SOURCES = \
+	$(src_lib_libtor_intmath_a_SOURCES)
+src_lib_libtor_intmath_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_intmath_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/intmath/addsub.h			\
+	src/lib/intmath/cmp.h				\
+	src/lib/intmath/bits.h				\
+	src/lib/intmath/logic.h				\
+	src/lib/intmath/muldiv.h			\
+	src/lib/intmath/weakrng.h
diff --git a/src/lib/intmath/logic.h b/src/lib/intmath/logic.h
new file mode 100644
index 0000000000..a4cecd69cc
--- /dev/null
+++ b/src/lib/intmath/logic.h
@@ -0,0 +1,20 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file logic.h
+ *
+ * \brief Macros for comparing the boolean value of integers.
+ **/
+
+#ifndef HAVE_TOR_LOGIC_H
+#define HAVE_TOR_LOGIC_H
+
+/** Macro: true if two values have the same boolean value. */
+#define bool_eq(a,b) (!(a)==!(b))
+/** Macro: true if two values have different boolean values. */
+#define bool_neq(a,b) (!(a)!=!(b))
+
+#endif
diff --git a/src/lib/intmath/muldiv.c b/src/lib/intmath/muldiv.c
new file mode 100644
index 0000000000..6a292db7ba
--- /dev/null
+++ b/src/lib/intmath/muldiv.c
@@ -0,0 +1,81 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file muldiv.c
+ *
+ * \brief Integer math related to multiplication, division, and rounding.
+ **/
+
+#include "lib/intmath/muldiv.h"
+#include "lib/err/torerr.h"
+
+#include <stdlib.h>
+
+/** Return the lowest x such that x is at least <b>number</b>, and x modulo
+ * <b>divisor</b> == 0.  If no such x can be expressed as an unsigned, return
+ * UINT_MAX. Asserts if divisor is zero. */
+unsigned
+round_to_next_multiple_of(unsigned number, unsigned divisor)
+{
+  raw_assert(divisor > 0);
+  if (UINT_MAX - divisor + 1 < number)
+    return UINT_MAX;
+  number += divisor - 1;
+  number -= number % divisor;
+  return number;
+}
+
+/** Return the lowest x such that x is at least <b>number</b>, and x modulo
+ * <b>divisor</b> == 0. If no such x can be expressed as a uint32_t, return
+ * UINT32_MAX. Asserts if divisor is zero. */
+uint32_t
+round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor)
+{
+  raw_assert(divisor > 0);
+  if (UINT32_MAX - divisor + 1 < number)
+    return UINT32_MAX;
+
+  number += divisor - 1;
+  number -= number % divisor;
+  return number;
+}
+
+/** Return the lowest x such that x is at least <b>number</b>, and x modulo
+ * <b>divisor</b> == 0. If no such x can be expressed as a uint64_t, return
+ * UINT64_MAX. Asserts if divisor is zero. */
+uint64_t
+round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor)
+{
+  raw_assert(divisor > 0);
+  if (UINT64_MAX - divisor + 1 < number)
+    return UINT64_MAX;
+  number += divisor - 1;
+  number -= number % divisor;
+  return number;
+}
+
+/* Helper: return greatest common divisor of a,b */
+static uint64_t
+gcd64(uint64_t a, uint64_t b)
+{
+  while (b) {
+    uint64_t t = b;
+    b = a % b;
+    a = t;
+  }
+  return a;
+}
+
+/* Given a fraction *<b>numer</b> / *<b>denom</b>, simplify it.
+ * Requires that the denominator is greater than 0. */
+void
+simplify_fraction64(uint64_t *numer, uint64_t *denom)
+{
+  raw_assert(denom);
+  uint64_t gcd = gcd64(*numer, *denom);
+  *numer /= gcd;
+  *denom /= gcd;
+}
diff --git a/src/lib/intmath/muldiv.h b/src/lib/intmath/muldiv.h
new file mode 100644
index 0000000000..64500b6dce
--- /dev/null
+++ b/src/lib/intmath/muldiv.h
@@ -0,0 +1,28 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file muldiv.h
+ *
+ * \brief Header for muldiv.c
+ **/
+
+#ifndef TOR_INTMATH_MULDIV_H
+#define TOR_INTMATH_MULDIV_H
+
+#include "lib/cc/torint.h"
+
+unsigned round_to_next_multiple_of(unsigned number, unsigned divisor);
+uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor);
+uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor);
+
+void simplify_fraction64(uint64_t *numer, uint64_t *denom);
+
+/* Compute the CEIL of <b>a</b> divided by <b>b</b>, for nonnegative <b>a</b>
+ * and positive <b>b</b>.  Works on integer types only. Not defined if a+(b-1)
+ * can overflow. */
+#define CEIL_DIV(a,b) (((a)+((b)-1))/(b))
+
+#endif /* !defined(TOR_INTMATH_MULDIV_H) */
diff --git a/src/lib/intmath/weakrng.c b/src/lib/intmath/weakrng.c
new file mode 100644
index 0000000000..99c9252c2b
--- /dev/null
+++ b/src/lib/intmath/weakrng.c
@@ -0,0 +1,60 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file weakrng.c
+ *
+ * \brief A weak but fast PRNG based on a linear congruential generator.
+ *
+ * We don't want to use the platform random(), since some of them are even
+ * worse than this.
+ **/
+
+#include "lib/intmath/weakrng.h"
+#include "lib/err/torerr.h"
+
+#include <stdlib.h>
+
+/** Initialize the insecure RNG <b>rng</b> from a seed value <b>seed</b>. */
+void
+tor_init_weak_random(tor_weak_rng_t *rng, unsigned seed)
+{
+  rng->state = (uint32_t)(seed & 0x7fffffff);
+}
+
+/** Return a randomly chosen value in the range 0..TOR_WEAK_RANDOM_MAX based
+ * on the RNG state of <b>rng</b>.  This entropy will not be cryptographically
+ * strong; do not rely on it for anything an adversary should not be able to
+ * predict. */
+int32_t
+tor_weak_random(tor_weak_rng_t *rng)
+{
+  /* Here's a linear congruential generator. OpenBSD and glibc use these
+   * parameters; they aren't too bad, and should have maximal period over the
+   * range 0..INT32_MAX. We don't want to use the platform rand() or random(),
+   * since some platforms have bad weak RNGs that only return values in the
+   * range 0..INT16_MAX, which just isn't enough. */
+  rng->state = (rng->state * 1103515245 + 12345) & 0x7fffffff;
+  return (int32_t) rng->state;
+}
+
+/** Return a random number in the range [0 , <b>top</b>). {That is, the range
+ * of integers i such that 0 <= i < top.}  Chooses uniformly.  Requires that
+ * top is greater than 0. This randomness is not cryptographically strong; do
+ * not rely on it for anything an adversary should not be able to predict. */
+int32_t
+tor_weak_random_range(tor_weak_rng_t *rng, int32_t top)
+{
+  /* We don't want to just do tor_weak_random() % top, since random() is often
+   * implemented with an LCG whose modulus is a power of 2, and those are
+   * cyclic in their low-order bits. */
+  int divisor, result;
+  raw_assert(top > 0);
+  divisor = TOR_WEAK_RANDOM_MAX / top;
+  do {
+    result = (int32_t)(tor_weak_random(rng) / divisor);
+  } while (result >= top);
+  return result;
+}
diff --git a/src/lib/intmath/weakrng.h b/src/lib/intmath/weakrng.h
new file mode 100644
index 0000000000..e26bf58cbb
--- /dev/null
+++ b/src/lib/intmath/weakrng.h
@@ -0,0 +1,31 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file weakrng.h
+ *
+ * \brief Header for weakrng.c
+ **/
+
+#ifndef TOR_WEAKRNG_H
+#define TOR_WEAKRNG_H
+
+#include "lib/cc/torint.h"
+
+/* ===== Insecure rng */
+typedef struct tor_weak_rng_t {
+  uint32_t state;
+} tor_weak_rng_t;
+
+#define TOR_WEAK_RNG_INIT {383745623}
+#define TOR_WEAK_RANDOM_MAX (INT_MAX)
+void tor_init_weak_random(tor_weak_rng_t *weak_rng, unsigned seed);
+int32_t tor_weak_random(tor_weak_rng_t *weak_rng);
+int32_t tor_weak_random_range(tor_weak_rng_t *rng, int32_t top);
+/** Randomly return true according to <b>rng</b> with probability 1 in
+ * <b>n</b> */
+#define tor_weak_random_one_in_n(rng, n) (0==tor_weak_random_range((rng),(n)))
+
+#endif
diff --git a/src/lib/lock/.may_include b/src/lib/lock/.may_include
new file mode 100644
index 0000000000..9522e3af49
--- /dev/null
+++ b/src/lib/lock/.may_include
@@ -0,0 +1,5 @@
+orconfig.h
+lib/cc/*.h
+lib/err/*.h
+lib/lock/*.h
+lib/malloc/*.h
diff --git a/src/lib/lock/compat_mutex.c b/src/lib/lock/compat_mutex.c
new file mode 100644
index 0000000000..4ad5929715
--- /dev/null
+++ b/src/lib/lock/compat_mutex.c
@@ -0,0 +1,40 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compat_mutex.c
+ *
+ * \brief Portable wrapper for platform mutex implementations.
+ **/
+
+#include "lib/lock/compat_mutex.h"
+#include "lib/malloc/malloc.h"
+
+/** Return a newly allocated, ready-for-use mutex. */
+tor_mutex_t *
+tor_mutex_new(void)
+{
+  tor_mutex_t *m = tor_malloc_zero(sizeof(tor_mutex_t));
+  tor_mutex_init(m);
+  return m;
+}
+/** Return a newly allocated, ready-for-use mutex.  This one might be
+ * non-recursive, if that's faster. */
+tor_mutex_t *
+tor_mutex_new_nonrecursive(void)
+{
+  tor_mutex_t *m = tor_malloc_zero(sizeof(tor_mutex_t));
+  tor_mutex_init_nonrecursive(m);
+  return m;
+}
+/** Release all storage and system resources held by <b>m</b>. */
+void
+tor_mutex_free_(tor_mutex_t *m)
+{
+  if (!m)
+    return;
+  tor_mutex_uninit(m);
+  tor_free(m);
+}
diff --git a/src/lib/lock/compat_mutex.h b/src/lib/lock/compat_mutex.h
new file mode 100644
index 0000000000..b63ce24024
--- /dev/null
+++ b/src/lib/lock/compat_mutex.h
@@ -0,0 +1,66 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compat_mutex.h
+ *
+ * \brief Header for compat_mutex.c
+ **/
+
+#ifndef TOR_COMPAT_MUTEX_H
+#define TOR_COMPAT_MUTEX_H
+
+#include "orconfig.h"
+#include "lib/cc/torint.h"
+#include "lib/malloc/malloc.h"
+
+#if defined(HAVE_PTHREAD_H) && !defined(_WIN32)
+#include <pthread.h>
+#endif
+
+#if defined(_WIN32)
+#include <windows.h>
+#endif
+
+#if defined(_WIN32)
+#define USE_WIN32_THREADS
+#elif defined(HAVE_PTHREAD_H) && defined(HAVE_PTHREAD_CREATE)
+#define USE_PTHREADS
+#else
+#error "No threading system was found"
+#endif /* defined(_WIN32) || ... */
+
+/* Because we use threads instead of processes on most platforms (Windows,
+ * Linux, etc), we need locking for them.  On platforms with poor thread
+ * support or broken gethostbyname_r, these functions are no-ops. */
+
+/** A generic lock structure for multithreaded builds. */
+typedef struct tor_mutex_t {
+#if defined(USE_WIN32_THREADS)
+  /** Windows-only: on windows, we implement locks with CRITICAL_SECTIONS. */
+  CRITICAL_SECTION mutex;
+#elif defined(USE_PTHREADS)
+  /** Pthreads-only: with pthreads, we implement locks with
+   * pthread_mutex_t. */
+  pthread_mutex_t mutex;
+#else
+  /** No-threads only: Dummy variable so that tor_mutex_t takes up space. */
+  int _unused;
+#endif /* defined(USE_WIN32_MUTEX) || ... */
+} tor_mutex_t;
+
+tor_mutex_t *tor_mutex_new(void);
+tor_mutex_t *tor_mutex_new_nonrecursive(void);
+void tor_mutex_init(tor_mutex_t *m);
+void tor_mutex_init_nonrecursive(tor_mutex_t *m);
+void tor_mutex_acquire(tor_mutex_t *m);
+void tor_mutex_release(tor_mutex_t *m);
+void tor_mutex_free_(tor_mutex_t *m);
+#define tor_mutex_free(m) FREE_AND_NULL(tor_mutex_t, tor_mutex_free_, (m))
+void tor_mutex_uninit(tor_mutex_t *m);
+
+void tor_locking_init(void);
+
+#endif /* !defined(TOR_COMPAT_MUTEX_H) */
diff --git a/src/lib/lock/compat_mutex_pthreads.c b/src/lib/lock/compat_mutex_pthreads.c
new file mode 100644
index 0000000000..ee5f520cd0
--- /dev/null
+++ b/src/lib/lock/compat_mutex_pthreads.c
@@ -0,0 +1,103 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compat_mutex_pthreads.c
+ *
+ * \brief Implement the tor_mutex API using pthread_mutex_t.
+ **/
+
+#include "lib/lock/compat_mutex.h"
+#include "lib/cc/compat_compiler.h"
+#include "lib/err/torerr.h"
+
+/** A mutex attribute that we're going to use to tell pthreads that we want
+ * "recursive" mutexes (i.e., once we can re-lock if we're already holding
+ * them.) */
+static pthread_mutexattr_t attr_recursive;
+static int attr_initialized = 0;
+
+void
+tor_locking_init(void)
+{
+  if (!attr_initialized) {
+    pthread_mutexattr_init(&attr_recursive);
+    pthread_mutexattr_settype(&attr_recursive, PTHREAD_MUTEX_RECURSIVE);
+    attr_initialized = 1;
+  }
+}
+
+/** Initialize <b>mutex</b> so it can be locked.  Every mutex must be set
+ * up with tor_mutex_init() or tor_mutex_new(); not both. */
+void
+tor_mutex_init(tor_mutex_t *mutex)
+{
+  if (PREDICT_UNLIKELY(!attr_initialized))
+    tor_locking_init(); // LCOV_EXCL_LINE
+  const int err = pthread_mutex_init(&mutex->mutex, &attr_recursive);
+  if (PREDICT_UNLIKELY(err)) {
+    // LCOV_EXCL_START
+    raw_assert_unreached_msg("Error creating a mutex.");
+    // LCOV_EXCL_STOP
+  }
+}
+
+/** As tor_mutex_init, but initialize a mutex suitable that may be
+ * non-recursive, if the OS supports that. */
+void
+tor_mutex_init_nonrecursive(tor_mutex_t *mutex)
+{
+  int err;
+  if (!attr_initialized)
+    tor_locking_init(); // LCOV_EXCL_LINE
+  err = pthread_mutex_init(&mutex->mutex, NULL);
+  if (PREDICT_UNLIKELY(err)) {
+    // LCOV_EXCL_START
+    raw_assert_unreached_msg("Error creating a mutex.");
+    // LCOV_EXCL_STOP
+  }
+}
+
+/** Wait until <b>m</b> is free, then acquire it. */
+void
+tor_mutex_acquire(tor_mutex_t *m)
+{
+  int err;
+  raw_assert(m);
+  err = pthread_mutex_lock(&m->mutex);
+  if (PREDICT_UNLIKELY(err)) {
+    // LCOV_EXCL_START
+    raw_assert_unreached_msg("Error locking a mutex.");
+    // LCOV_EXCL_STOP
+  }
+}
+/** Release the lock <b>m</b> so another thread can have it. */
+void
+tor_mutex_release(tor_mutex_t *m)
+{
+  int err;
+  raw_assert(m);
+  err = pthread_mutex_unlock(&m->mutex);
+  if (PREDICT_UNLIKELY(err)) {
+    // LCOV_EXCL_START
+    raw_assert_unreached_msg("Error unlocking a mutex.");
+    // LCOV_EXCL_STOP
+  }
+}
+/** Clean up the mutex <b>m</b> so that it no longer uses any system
+ * resources.  Does not free <b>m</b>.  This function must only be called on
+ * mutexes from tor_mutex_init(). */
+void
+tor_mutex_uninit(tor_mutex_t *m)
+{
+  int err;
+  raw_assert(m);
+  err = pthread_mutex_destroy(&m->mutex);
+  if (PREDICT_UNLIKELY(err)) {
+    // LCOV_EXCL_START
+    raw_assert_unreached_msg("Error destroying a mutex.");
+    // LCOV_EXCL_STOP
+  }
+}
diff --git a/src/lib/lock/compat_mutex_winthreads.c b/src/lib/lock/compat_mutex_winthreads.c
new file mode 100644
index 0000000000..b0f5999e42
--- /dev/null
+++ b/src/lib/lock/compat_mutex_winthreads.c
@@ -0,0 +1,46 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compat_mutex_winthreads.c
+ *
+ * \brief Implement the tor_mutex API using CRITICAL_SECTION.
+ **/
+
+#include "lib/lock/compat_mutex.h"
+#include "lib/err/torerr.h"
+
+void
+tor_locking_init(void)
+{
+}
+
+void
+tor_mutex_init(tor_mutex_t *m)
+{
+  InitializeCriticalSection(&m->mutex);
+}
+void
+tor_mutex_init_nonrecursive(tor_mutex_t *m)
+{
+  InitializeCriticalSection(&m->mutex);
+}
+
+void
+tor_mutex_uninit(tor_mutex_t *m)
+{
+  DeleteCriticalSection(&m->mutex);
+}
+void
+tor_mutex_acquire(tor_mutex_t *m)
+{
+  raw_assert(m);
+  EnterCriticalSection(&m->mutex);
+}
+void
+tor_mutex_release(tor_mutex_t *m)
+{
+  LeaveCriticalSection(&m->mutex);
+}
diff --git a/src/lib/lock/include.am b/src/lib/lock/include.am
new file mode 100644
index 0000000000..4e6f444347
--- /dev/null
+++ b/src/lib/lock/include.am
@@ -0,0 +1,24 @@
+
+noinst_LIBRARIES += src/lib/libtor-lock.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-lock-testing.a
+endif
+
+src_lib_libtor_lock_a_SOURCES =			\
+	src/lib/lock/compat_mutex.c
+
+if THREADS_PTHREADS
+src_lib_libtor_lock_a_SOURCES += src/lib/lock/compat_mutex_pthreads.c
+endif
+if THREADS_WIN32
+src_lib_libtor_lock_a_SOURCES += src/lib/lock/compat_mutex_winthreads.c
+endif
+
+src_lib_libtor_lock_testing_a_SOURCES = \
+	$(src_lib_libtor_lock_a_SOURCES)
+src_lib_libtor_lock_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_lock_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/lock/compat_mutex.h
diff --git a/src/lib/log/.may_include b/src/lib/log/.may_include
new file mode 100644
index 0000000000..852173aab3
--- /dev/null
+++ b/src/lib/log/.may_include
@@ -0,0 +1,15 @@
+orconfig.h
+
+lib/cc/*.h
+lib/smartlist_core/*.h
+lib/err/*.h
+lib/fdio/*.h
+lib/intmath/*.h
+lib/lock/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/string/*.h
+lib/testsupport/*.h
+lib/wallclock/*.h
+
+micro-revision.i
\ No newline at end of file
diff --git a/src/lib/log/escape.c b/src/lib/log/escape.c
new file mode 100644
index 0000000000..6ca01c6963
--- /dev/null
+++ b/src/lib/log/escape.c
@@ -0,0 +1,137 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file escape.c
+ * \brief Escape untrusted strings before sending them to the log.
+ **/
+
+#include "lib/log/escape.h"
+#include "lib/log/util_bug.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/string/printf.h"
+#include "lib/malloc/malloc.h"
+
+/** Allocate and return a new string representing the contents of <b>s</b>,
+ * surrounded by quotes and using standard C escapes.
+ *
+ * Generally, we use this for logging values that come in over the network to
+ * keep them from tricking users, and for sending certain values to the
+ * controller.
+ *
+ * We trust values from the resolver, OS, configuration file, and command line
+ * to not be maliciously ill-formed.  We validate incoming routerdescs and
+ * SOCKS requests and addresses from BEGIN cells as they're parsed;
+ * afterwards, we trust them as non-malicious.
+ */
+char *
+esc_for_log(const char *s)
+{
+  const char *cp;
+  char *result, *outp;
+  size_t len = 3;
+  if (!s) {
+    return tor_strdup("(null)");
+  }
+
+  for (cp = s; *cp; ++cp) {
+    switch (*cp) {
+      case '\\':
+      case '\"':
+      case '\'':
+      case '\r':
+      case '\n':
+      case '\t':
+        len += 2;
+        break;
+      default:
+        if (TOR_ISPRINT(*cp) && ((uint8_t)*cp)<127)
+          ++len;
+        else
+          len += 4;
+        break;
+    }
+  }
+
+  tor_assert(len <= SSIZE_MAX);
+
+  result = outp = tor_malloc(len);
+  *outp++ = '\"';
+  for (cp = s; *cp; ++cp) {
+    /* This assertion should always succeed, since we will write at least
+     * one char here, and two chars for closing quote and nul later */
+    tor_assert((outp-result) < (ssize_t)len-2);
+    switch (*cp) {
+      case '\\':
+      case '\"':
+      case '\'':
+        *outp++ = '\\';
+        *outp++ = *cp;
+        break;
+      case '\n':
+        *outp++ = '\\';
+        *outp++ = 'n';
+        break;
+      case '\t':
+        *outp++ = '\\';
+        *outp++ = 't';
+        break;
+      case '\r':
+        *outp++ = '\\';
+        *outp++ = 'r';
+        break;
+      default:
+        if (TOR_ISPRINT(*cp) && ((uint8_t)*cp)<127) {
+          *outp++ = *cp;
+        } else {
+          tor_assert((outp-result) < (ssize_t)len-4);
+          tor_snprintf(outp, 5, "\\%03o", (int)(uint8_t) *cp);
+          outp += 4;
+        }
+        break;
+    }
+  }
+
+  tor_assert((outp-result) <= (ssize_t)len-2);
+  *outp++ = '\"';
+  *outp++ = 0;
+
+  return result;
+}
+
+/** Similar to esc_for_log. Allocate and return a new string representing
+ * the first n characters in <b>chars</b>, surround by quotes and using
+ * standard C escapes. If a NUL character is encountered in <b>chars</b>,
+ * the resulting string will be terminated there.
+ */
+char *
+esc_for_log_len(const char *chars, size_t n)
+{
+  char *string = tor_strndup(chars, n);
+  char *string_escaped = esc_for_log(string);
+  tor_free(string);
+  return string_escaped;
+}
+
+/** Allocate and return a new string representing the contents of <b>s</b>,
+ * surrounded by quotes and using standard C escapes.
+ *
+ * THIS FUNCTION IS NOT REENTRANT.  Don't call it from outside the main
+ * thread.  Also, each call invalidates the last-returned value, so don't
+ * try log_warn(LD_GENERAL, "%s %s", escaped(a), escaped(b));
+ */
+const char *
+escaped(const char *s)
+{
+  static char *escaped_val_ = NULL;
+  tor_free(escaped_val_);
+
+  if (s)
+    escaped_val_ = esc_for_log(s);
+  else
+    escaped_val_ = NULL;
+
+  return escaped_val_;
+}
diff --git a/src/lib/log/escape.h b/src/lib/log/escape.h
new file mode 100644
index 0000000000..2f726186c5
--- /dev/null
+++ b/src/lib/log/escape.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file escape.h
+ * \brief Header for escape.c
+ **/
+
+#ifndef TOR_ESCAPE_H
+#define TOR_ESCAPE_H
+
+#include "orconfig.h"
+#include "lib/cc/compat_compiler.h"
+#include <stddef.h>
+
+char *esc_for_log(const char *string) ATTR_MALLOC;
+char *esc_for_log_len(const char *chars, size_t n) ATTR_MALLOC;
+const char *escaped(const char *string);
+
+#endif /* !defined(TOR_TORLOG_H) */
diff --git a/src/lib/log/git_revision.c b/src/lib/log/git_revision.c
new file mode 100644
index 0000000000..7d27549cad
--- /dev/null
+++ b/src/lib/log/git_revision.c
@@ -0,0 +1,24 @@
+/* Copyright 2001-2004 Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "lib/log/git_revision.h"
+
+/** String describing which Tor Git repository version the source was
+ * built from.  This string is generated by a bit of shell kludging in
+ * src/core/include.am, and is usually right.
+ */
+const char tor_git_revision[] =
+#ifndef _MSC_VER
+#include "micro-revision.i"
+#endif
+  "";
+
+const char tor_bug_suffix[] = " (on Tor " VERSION
+#ifndef _MSC_VER
+  " "
+#include "micro-revision.i"
+#endif
+  ")";
diff --git a/src/lib/log/git_revision.h b/src/lib/log/git_revision.h
new file mode 100644
index 0000000000..79e3c6684b
--- /dev/null
+++ b/src/lib/log/git_revision.h
@@ -0,0 +1,12 @@
+/* Copyright 2001-2004 Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_GIT_REVISION_H
+#define TOR_GIT_REVISION_H
+
+extern const char tor_git_revision[];
+extern const char tor_bug_suffix[];
+
+#endif /* !defined(TOR_GIT_REVISION_H) */
diff --git a/src/lib/log/include.am b/src/lib/log/include.am
new file mode 100644
index 0000000000..4a6c9b3686
--- /dev/null
+++ b/src/lib/log/include.am
@@ -0,0 +1,36 @@
+
+noinst_LIBRARIES += src/lib/libtor-log.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-log-testing.a
+endif
+
+src_lib_libtor_log_a_SOURCES =			\
+	src/lib/log/escape.c			\
+        src/lib/log/git_revision.c              \
+	src/lib/log/ratelim.c			\
+	src/lib/log/log.c			\
+	src/lib/log/util_bug.c
+
+if WIN32
+src_lib_libtor_log_a_SOURCES += src/lib/log/win32err.c
+endif
+
+src_lib_libtor_log_testing_a_SOURCES = \
+	$(src_lib_libtor_log_a_SOURCES)
+src_lib_libtor_log_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_log_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# Declare that these object files depend on micro-revision.i.  Without this
+# rule, we could try to build them before micro-revision.i was created.
+src/lib/log/git_revision.$(OBJEXT) \
+  src/lib/log/src_lib_libtor_log_testing_a-git_revision.$(OBJEXT): \
+	micro-revision.i
+
+noinst_HEADERS +=					\
+	src/lib/log/escape.h				\
+        src/lib/log/git_revision.h                      \
+	src/lib/log/ratelim.h				\
+	src/lib/log/log.h				\
+	src/lib/log/util_bug.h				\
+	src/lib/log/win32err.h
diff --git a/src/common/log.c b/src/lib/log/log.c
index 4db1c9f0d0..a9ad38fb25 100644
--- a/src/common/log.c
+++ b/src/lib/log/log.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,8 +11,7 @@
 
 #include "orconfig.h"
 #include <stdarg.h>
-#include <assert.h>
-// #include <stdio.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #ifdef HAVE_SYS_TIME_H
@@ -30,11 +29,29 @@
 #ifdef HAVE_FCNTL_H
 #include <fcntl.h>
 #endif
-#include "compat.h"
-#include "util.h"
+
 #define LOG_PRIVATE
-#include "torlog.h"
-#include "container.h"
+#include "lib/log/log.h"
+#include "lib/log/git_revision.h"
+#include "lib/log/ratelim.h"
+#include "lib/lock/compat_mutex.h"
+#include "lib/smartlist_core/smartlist_core.h"
+#include "lib/smartlist_core/smartlist_foreach.h"
+#include "lib/smartlist_core/smartlist_split.h"
+#include "lib/err/torerr.h"
+#include "lib/intmath/bits.h"
+#include "lib/string/compat_string.h"
+#include "lib/string/printf.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/util_string.h"
+#include "lib/wallclock/tor_gettimeofday.h"
+#include "lib/wallclock/approx_time.h"
+#include "lib/wallclock/time_to_tm.h"
+#include "lib/fdio/fdio.h"
+
+#ifdef HAVE_ANDROID_LOG_H
+#include <android/log.h>
+#endif // HAVE_ANDROID_LOG_H.
 
 /** Given a severity, yields an index into log_severity_list_t.masks to use
  * for that severity. */
@@ -47,7 +64,12 @@
 #define TRUNCATED_STR_LEN 14
 /** @} */
 
-#define raw_assert(x) assert(x) // assert OK
+/** Defining compile-time constants for Tor log levels (used by the Rust
+ * log wrapper at src/rust/tor_log) */
+const int LOG_WARN_ = LOG_WARN;
+const int LOG_NOTICE_ = LOG_NOTICE;
+const log_domain_mask_t LD_GENERAL_ = LD_GENERAL;
+const log_domain_mask_t LD_NET_ = LD_NET;
 
 /** Information for a single logfile; only used in log.c */
 typedef struct logfile_t {
@@ -58,12 +80,16 @@ typedef struct logfile_t {
   int needs_close; /**< Boolean: true if the stream gets closed on shutdown. */
   int is_temporary; /**< Boolean: close after initializing logging subsystem.*/
   int is_syslog; /**< Boolean: send messages to syslog. */
+  int is_android; /**< Boolean: send messages to Android's log subsystem. */
+  char *android_tag; /**< Identity Tag used in Android's log subsystem. */
   log_callback callback; /**< If not NULL, send messages to this function. */
   log_severity_list_t *severities; /**< Which severity of messages should we
                                     * log for each log domain? */
 } logfile_t;
 
-static void log_free(logfile_t *victim);
+static void log_free_(logfile_t *victim);
+#define log_free(lg)    \
+  FREE_AND_NULL(logfile_t, log_free_, (lg))
 
 /** Helper: map a log severity to descriptive string. */
 static inline const char *
@@ -75,9 +101,9 @@ sev_to_string(int severity)
     case LOG_NOTICE:  return "notice";
     case LOG_WARN:    return "warn";
     case LOG_ERR:     return "err";
-    default:          /* Call assert, not tor_assert, since tor_assert
-                       * calls log on failure. */
-                      raw_assert(0); return "UNKNOWN"; // LCOV_EXCL_LINE
+    default:     /* Call raw_assert, not tor_assert, since tor_assert
+                  * calls log on failure. */
+                 raw_assert_unreached(); return "UNKNOWN"; // LCOV_EXCL_LINE
   }
 }
 
@@ -96,11 +122,39 @@ should_log_function_name(log_domain_mask_t domain, int severity)
       /* We care about places where bugs occur. */
       return (domain & (LD_BUG|LD_NOFUNCNAME)) == LD_BUG;
     default:
-      /* Call assert, not tor_assert, since tor_assert calls log on failure. */
+      /* Call raw_assert, not tor_assert, since tor_assert calls
+       * log on failure. */
       raw_assert(0); return 0; // LCOV_EXCL_LINE
   }
 }
 
+#ifdef HAVE_ANDROID_LOG_H
+/** Helper function to convert Tor's log severity into the matching
+ * Android log priority.
+ */
+static int
+severity_to_android_log_priority(int severity)
+{
+  switch (severity) {
+    case LOG_DEBUG:
+      return ANDROID_LOG_VERBOSE;
+    case LOG_INFO:
+      return ANDROID_LOG_DEBUG;
+    case LOG_NOTICE:
+      return ANDROID_LOG_INFO;
+    case LOG_WARN:
+      return ANDROID_LOG_WARN;
+    case LOG_ERR:
+      return ANDROID_LOG_ERROR;
+    default:
+      // LCOV_EXCL_START
+      raw_assert(0);
+      return 0;
+      // LCOV_EXCL_STOP
+  }
+}
+#endif // HAVE_ANDROID_LOG_H.
+
 /** A mutex to guard changes to logfiles and logging. */
 static tor_mutex_t log_mutex;
 /** True iff we have initialized log_mutex */
@@ -129,6 +183,9 @@ typedef struct pending_log_message_t {
 /** Log messages waiting to be replayed onto callback-based logs */
 static smartlist_t *pending_cb_messages = NULL;
 
+/** Callback to invoke when pending_cb_messages becomes nonempty. */
+static pending_callback_callback pending_cb_cb = NULL;
+
 /** Log messages waiting to be replayed once the logging system is initialized.
  */
 static smartlist_t *pending_startup_messages = NULL;
@@ -151,12 +208,12 @@ static int pretty_fn_has_parens = 0;
 
 /** Lock the log_mutex to prevent others from changing the logfile_t list */
 #define LOCK_LOGS() STMT_BEGIN                                          \
-  tor_assert(log_mutex_initialized);                                    \
+  raw_assert(log_mutex_initialized);                                    \
   tor_mutex_acquire(&log_mutex);                                        \
   STMT_END
 /** Unlock the log_mutex */
 #define UNLOCK_LOGS() STMT_BEGIN                                        \
-  tor_assert(log_mutex_initialized);                                    \
+  raw_assert(log_mutex_initialized);                                    \
   tor_mutex_release(&log_mutex);                                        \
   STMT_END
 
@@ -191,6 +248,30 @@ log_set_application_name(const char *name)
   appname = name ? tor_strdup(name) : NULL;
 }
 
+/** Return true if some of the running logs might be interested in a log
+ * message of the given severity in the given domains. If this function
+ * returns true, the log message might be ignored anyway, but if it returns
+ * false, it is definitely_ safe not to log the message. */
+int
+log_message_is_interesting(int severity, log_domain_mask_t domain)
+{
+  (void) domain;
+  return (severity <= log_global_min_severity_);
+}
+
+/**
+ * As tor_log, but takes an optional function name, and does not treat its
+ * <b>string</b> as a printf format.
+ *
+ * For use by Rust integration.
+ */
+void
+tor_log_string(int severity, log_domain_mask_t domain,
+               const char *function, const char *string)
+{
+  log_fn_(severity, domain, function, "%s", string);
+}
+
 /** Log time granularity in milliseconds. */
 static int log_time_granularity = 1;
 
@@ -200,6 +281,7 @@ void
 set_log_time_granularity(int granularity_msec)
 {
   log_time_granularity = granularity_msec;
+  tor_log_sigsafe_err_set_granularity(granularity_msec);
 }
 
 /** Helper: Write the standard prefix for log lines to a
@@ -224,7 +306,8 @@ log_prefix_(char *buf, size_t buf_len, int severity)
     ms -= ((int)now.tv_usec / 1000) % log_time_granularity;
   }
 
-  n = strftime(buf, buf_len, "%b %d %H:%M:%S", tor_localtime_r(&t, &tm));
+  n = strftime(buf, buf_len, "%b %d %H:%M:%S",
+               tor_localtime_r_msg(&t, &tm, NULL));
   r = tor_snprintf(buf+n, buf_len-n, ".%.3i [%s] ", ms,
                    sev_to_string(severity));
 
@@ -267,18 +350,11 @@ log_tor_version(logfile_t *lf, int reset)
     tor_snprintf(buf+n, sizeof(buf)-n,
                  "Tor %s opening %slog file.\n", VERSION, is_new?"new ":"");
   }
-  if (write_all(lf->fd, buf, strlen(buf), 0) < 0) /* error */
+  if (write_all_to_fd_minimal(lf->fd, buf, strlen(buf)) < 0) /* error */
     return -1; /* failed */
   return 0;
 }
 
-static const char bug_suffix[] = " (on Tor " VERSION
-#ifndef _MSC_VER
-  " "
-#include "micro-revision.i"
-#endif
-  ")";
-
 /** Helper: Format a log message into a fixed-sized buffer. (This is
  * factored out of <b>logv</b> so that we never format a message more
  * than once.)  Return a pointer to the first character of the message
@@ -361,9 +437,9 @@ format_msg(char *buf, size_t buf_len,
   }
 
   if (domain == LD_BUG &&
-      buf_len - n > strlen(bug_suffix)+1) {
-    memcpy(buf+n, bug_suffix, strlen(bug_suffix));
-    n += strlen(bug_suffix);
+      buf_len - n > strlen(tor_bug_suffix)+1) {
+    memcpy(buf+n, tor_bug_suffix, strlen(tor_bug_suffix));
+    n += strlen(tor_bug_suffix);
   }
 
   buf[n]='\n';
@@ -385,9 +461,12 @@ pending_log_message_new(int severity, log_domain_mask_t domain,
   return m;
 }
 
+#define pending_log_message_free(msg) \
+  FREE_AND_NULL(pending_log_message_t, pending_log_message_free_, (msg))
+
 /** Release all storage held by <b>msg</b>. */
 static void
-pending_log_message_free(pending_log_message_t *msg)
+pending_log_message_free_(pending_log_message_t *msg)
 {
   if (!msg)
     return;
@@ -396,6 +475,16 @@ pending_log_message_free(pending_log_message_t *msg)
   tor_free(msg);
 }
 
+/** Helper function: returns true iff the log file, given in <b>lf</b>, is
+ * handled externally via the system log API, the Android logging API, or is an
+ * external callback function. */
+static inline int
+logfile_is_external(const logfile_t *lf)
+{
+  raw_assert(lf);
+  return lf->is_syslog || lf->is_android || lf->callback;
+}
+
 /** Return true iff <b>lf</b> would like to receive a message with the
  * specified <b>severity</b> in the specified <b>domain</b>.
  */
@@ -406,7 +495,7 @@ logfile_wants_message(const logfile_t *lf, int severity,
   if (! (lf->severities->masks[SEVERITY_MASK_IDX(severity)] & domain)) {
     return 0;
   }
-  if (! (lf->fd >= 0 || lf->is_syslog || lf->callback)) {
+  if (! (lf->fd >= 0 || logfile_is_external(lf))) {
     return 0;
   }
   if (lf->seems_dead) {
@@ -444,23 +533,31 @@ logfile_deliver(logfile_t *lf, const char *buf, size_t msg_len,
     if (m != msg_after_prefix) {
       tor_free(m);
     }
-#else
+#else /* !(defined(MAXLINE)) */
     /* We have syslog but not MAXLINE.  That's promising! */
     syslog(severity, "%s", msg_after_prefix);
-#endif
-#endif
+#endif /* defined(MAXLINE) */
+#endif /* defined(HAVE_SYSLOG_H) */
+  } else if (lf->is_android) {
+#ifdef HAVE_ANDROID_LOG_H
+    int priority = severity_to_android_log_priority(severity);
+    __android_log_write(priority, lf->android_tag, msg_after_prefix);
+#endif // HAVE_ANDROID_LOG_H.
   } else if (lf->callback) {
     if (domain & LD_NOCB) {
       if (!*callbacks_deferred && pending_cb_messages) {
         smartlist_add(pending_cb_messages,
             pending_log_message_new(severity,domain,NULL,msg_after_prefix));
         *callbacks_deferred = 1;
+        if (smartlist_len(pending_cb_messages) == 1 && pending_cb_cb) {
+          pending_cb_cb();
+        }
       }
     } else {
       lf->callback(severity, domain, msg_after_prefix);
     }
   } else {
-    if (write_all(lf->fd, buf, msg_len, 0) < 0) { /* error */
+    if (write_all_to_fd_minimal(lf->fd, buf, msg_len) < 0) { /* error */
       /* don't log the error! mark this log entry to be blown away, and
        * continue. */
       lf->seems_dead = 1;
@@ -483,7 +580,7 @@ logv,(int severity, log_domain_mask_t domain, const char *funcname,
   char *end_of_prefix=NULL;
   int callbacks_deferred = 0;
 
-  /* Call assert, not tor_assert, since tor_assert calls log on failure. */
+  /* Call raw_assert, not tor_assert, since tor_assert calls log on failure. */
   raw_assert(format);
   /* check that severity is sane.  Overrunning the masks array leads to
    * interesting and hard to diagnose effects */
@@ -545,71 +642,6 @@ tor_log(int severity, log_domain_mask_t domain, const char *format, ...)
   va_end(ap);
 }
 
-/** Maximum number of fds that will get notifications if we crash */
-#define MAX_SIGSAFE_FDS 8
-/** Array of fds to log crash-style warnings to. */
-static int sigsafe_log_fds[MAX_SIGSAFE_FDS] = { STDERR_FILENO };
-/** The number of elements used in sigsafe_log_fds */
-static int n_sigsafe_log_fds = 1;
-
-/** Write <b>s</b> to each element of sigsafe_log_fds. Return 0 on success, -1
- * on failure. */
-static int
-tor_log_err_sigsafe_write(const char *s)
-{
-  int i;
-  ssize_t r;
-  size_t len = strlen(s);
-  int err = 0;
-  for (i=0; i < n_sigsafe_log_fds; ++i) {
-    r = write(sigsafe_log_fds[i], s, len);
-    err += (r != (ssize_t)len);
-  }
-  return err ? -1 : 0;
-}
-
-/** Given a list of string arguments ending with a NULL, writes them
- * to our logs and to stderr (if possible).  This function is safe to call
- * from within a signal handler. */
-void
-tor_log_err_sigsafe(const char *m, ...)
-{
-  va_list ap;
-  const char *x;
-  char timebuf[33];
-  time_t now = time(NULL);
-
-  if (!m)
-    return;
-  if (log_time_granularity >= 2000) {
-    int g = log_time_granularity / 1000;
-    now -= now % g;
-  }
-  timebuf[0] = now < 0 ? '-' : ' ';
-  if (now < 0) now = -now;
-  timebuf[1] = '\0';
-  format_dec_number_sigsafe(now, timebuf+1, sizeof(timebuf)-1);
-  tor_log_err_sigsafe_write("\n=========================================="
-                             "================== T=");
-  tor_log_err_sigsafe_write(timebuf);
-  tor_log_err_sigsafe_write("\n");
-  tor_log_err_sigsafe_write(m);
-  va_start(ap, m);
-  while ((x = va_arg(ap, const char*))) {
-    tor_log_err_sigsafe_write(x);
-  }
-  va_end(ap);
-}
-
-/** Set *<b>out</b> to a pointer to an array of the fds to log errors to from
- * inside a signal handler. Return the number of elements in the array. */
-int
-tor_log_get_sigsafe_err_fds(const int **out)
-{
-  *out = sigsafe_log_fds;
-  return n_sigsafe_log_fds;
-}
-
 /** Helper function; return true iff the <b>n</b>-element array <b>array</b>
  * contains <b>item</b>. */
 static int
@@ -631,40 +663,45 @@ tor_log_update_sigsafe_err_fds(void)
   const logfile_t *lf;
   int found_real_stderr = 0;
 
+  int fds[TOR_SIGSAFE_LOG_MAX_FDS];
+  int n_fds;
+
   LOCK_LOGS();
   /* Reserve the first one for stderr. This is safe because when we daemonize,
    * we dup2 /dev/null to stderr, */
-  sigsafe_log_fds[0] = STDERR_FILENO;
-  n_sigsafe_log_fds = 1;
+  fds[0] = STDERR_FILENO;
+  n_fds = 1;
 
   for (lf = logfiles; lf; lf = lf->next) {
      /* Don't try callback to the control port, or syslogs: We can't
       * do them from a signal handler. Don't try stdout: we always do stderr.
       */
-    if (lf->is_temporary || lf->is_syslog ||
-        lf->callback || lf->seems_dead || lf->fd < 0)
+    if (lf->is_temporary || logfile_is_external(lf)
+        || lf->seems_dead || lf->fd < 0)
       continue;
     if (lf->severities->masks[SEVERITY_MASK_IDX(LOG_ERR)] &
         (LD_BUG|LD_GENERAL)) {
       if (lf->fd == STDERR_FILENO)
         found_real_stderr = 1;
       /* Avoid duplicates */
-      if (int_array_contains(sigsafe_log_fds, n_sigsafe_log_fds, lf->fd))
+      if (int_array_contains(fds, n_fds, lf->fd))
         continue;
-      sigsafe_log_fds[n_sigsafe_log_fds++] = lf->fd;
-      if (n_sigsafe_log_fds == MAX_SIGSAFE_FDS)
+      fds[n_fds++] = lf->fd;
+      if (n_fds == TOR_SIGSAFE_LOG_MAX_FDS)
         break;
     }
   }
 
   if (!found_real_stderr &&
-      int_array_contains(sigsafe_log_fds, n_sigsafe_log_fds, STDOUT_FILENO)) {
+      int_array_contains(fds, n_fds, STDOUT_FILENO)) {
     /* Don't use a virtual stderr when we're also logging to stdout. */
-    raw_assert(n_sigsafe_log_fds >= 2); /* Don't tor_assert inside log fns */
-    sigsafe_log_fds[0] = sigsafe_log_fds[--n_sigsafe_log_fds];
+    raw_assert(n_fds >= 2); /* Don't tor_assert inside log fns */
+    fds[0] = fds[--n_fds];
   }
 
   UNLOCK_LOGS();
+
+  tor_log_set_sigsafe_err_fds(fds, n_fds);
 }
 
 /** Add to <b>out</b> a copy of every currently configured log file name. Used
@@ -673,16 +710,16 @@ void
 tor_log_get_logfile_names(smartlist_t *out)
 {
   logfile_t *lf;
-  tor_assert(out);
+  raw_assert(out);
 
   LOCK_LOGS();
 
   for (lf = logfiles; lf; lf = lf->next) {
-    if (lf->is_temporary || lf->is_syslog || lf->callback)
+    if (lf->is_temporary || logfile_is_external(lf))
       continue;
     if (lf->filename == NULL)
       continue;
-    smartlist_add(out, tor_strdup(lf->filename));
+    smartlist_add_strdup(out, lf->filename);
   }
 
   UNLOCK_LOGS();
@@ -721,12 +758,13 @@ log_fn_ratelim_(ratelim_t *ratelim, int severity, log_domain_mask_t domain,
 
 /** Free all storage held by <b>victim</b>. */
 static void
-log_free(logfile_t *victim)
+log_free_(logfile_t *victim)
 {
   if (!victim)
     return;
   tor_free(victim->severities);
   tor_free(victim->filename);
+  tor_free(victim->android_tag);
   tor_free(victim);
 }
 
@@ -741,6 +779,7 @@ logs_free_all(void)
   logfiles = NULL;
   messages = pending_cb_messages;
   pending_cb_messages = NULL;
+  pending_cb_cb = NULL;
   messages2 = pending_startup_messages;
   pending_startup_messages = NULL;
   UNLOCK_LOGS();
@@ -784,8 +823,8 @@ delete_log(logfile_t *victim)
     logfiles = victim->next;
   else {
     for (tmpl = logfiles; tmpl && tmpl->next != victim; tmpl=tmpl->next) ;
-//    tor_assert(tmpl);
-//    tor_assert(tmpl->next == victim);
+//    raw_assert(tmpl);
+//    raw_assert(tmpl->next == victim);
     if (!tmpl)
       return;
     tmpl->next = victim->next;
@@ -807,7 +846,7 @@ close_log(logfile_t *victim)
       /* There are no other syslogs; close the logging facility. */
       closelog();
     }
-#endif
+#endif /* defined(HAVE_SYSLOG_H) */
   }
 }
 
@@ -819,9 +858,9 @@ set_log_severity_config(int loglevelMin, int loglevelMax,
                         log_severity_list_t *severity_out)
 {
   int i;
-  tor_assert(loglevelMin >= loglevelMax);
-  tor_assert(loglevelMin >= LOG_ERR && loglevelMin <= LOG_DEBUG);
-  tor_assert(loglevelMax >= LOG_ERR && loglevelMax <= LOG_DEBUG);
+  raw_assert(loglevelMin >= loglevelMax);
+  raw_assert(loglevelMin >= LOG_ERR && loglevelMin <= LOG_DEBUG);
+  raw_assert(loglevelMax >= LOG_ERR && loglevelMax <= LOG_DEBUG);
   memset(severity_out, 0, sizeof(log_severity_list_t));
   for (i = loglevelMin; i >= loglevelMax; --i) {
     severity_out->masks[SEVERITY_MASK_IDX(i)] = ~0u;
@@ -904,6 +943,24 @@ add_temp_log(int min_severity)
 }
 
 /**
+ * Register "cb" as the callback to call when there are new pending log
+ * callbacks to be flushed with flush_pending_log_callbacks().
+ *
+ * Note that this callback, if present, can be invoked from any thread.
+ *
+ * This callback must not log.
+ *
+ * It is intentional that this function contains the name "callback" twice: it
+ * sets a "callback" to be called on the condition that there is a "pending
+ * callback".
+ **/
+void
+logs_set_pending_callback_callback(pending_callback_callback cb)
+{
+  pending_cb_cb = cb;
+}
+
+/**
  * Add a log handler to send messages in <b>severity</b>
  * to the function <b>cb</b>.
  */
@@ -1073,20 +1130,17 @@ mark_logs_temp(void)
 }
 
 /**
- * Add a log handler to send messages to <b>filename</b>. If opening the
- * logfile fails, -1 is returned and errno is set appropriately (by open(2)).
+ * Add a log handler to send messages to <b>filename</b> via <b>fd</b>. If
+ * opening the logfile failed, -1 is returned and errno is set appropriately
+ * (by open(2)).  Takes ownership of fd.
  */
 int
-add_file_log(const log_severity_list_t *severity, const char *filename,
-             const int truncate_log)
+add_file_log(const log_severity_list_t *severity,
+             const char *filename,
+             int fd)
 {
-  int fd;
   logfile_t *lf;
 
-  int open_flags = O_WRONLY|O_CREAT;
-  open_flags |= truncate_log ? O_TRUNC : O_APPEND;
-
-  fd = tor_open_cloexec(filename, open_flags, 0644);
   if (fd<0)
     return -1;
   if (tor_fd_seekend(fd)<0) {
@@ -1144,7 +1198,40 @@ add_syslog_log(const log_severity_list_t *severity,
   UNLOCK_LOGS();
   return 0;
 }
-#endif
+#endif /* defined(HAVE_SYSLOG_H) */
+
+#ifdef HAVE_ANDROID_LOG_H
+/**
+ * Add a log handler to send messages to the Android platform log facility.
+ */
+int
+add_android_log(const log_severity_list_t *severity,
+                const char *android_tag)
+{
+  logfile_t *lf = NULL;
+
+  lf = tor_malloc_zero(sizeof(logfile_t));
+  lf->fd = -1;
+  lf->severities = tor_memdup(severity, sizeof(log_severity_list_t));
+  lf->filename = tor_strdup("<android>");
+  lf->is_android = 1;
+
+  if (android_tag == NULL)
+    lf->android_tag = tor_strdup("Tor");
+  else {
+    char buf[256];
+    tor_snprintf(buf, sizeof(buf), "Tor-%s", android_tag);
+    lf->android_tag = tor_strdup(buf);
+  }
+
+  LOCK_LOGS();
+  lf->next = logfiles;
+  logfiles = lf;
+  log_global_min_severity_ = get_min_log_level();
+  UNLOCK_LOGS();
+  return 0;
+}
+#endif // HAVE_ANDROID_LOG_H.
 
 /** If <b>level</b> is a valid log severity, return the corresponding
  * numeric value.  Otherwise, return -1. */
@@ -1172,12 +1259,15 @@ log_level_to_string(int level)
 }
 
 /** NULL-terminated array of names for log domains such that domain_list[dom]
- * is a description of <b>dom</b>. */
+ * is a description of <b>dom</b>.
+ *
+ * Remember to update doc/tor.1.txt if you modify this list.
+ * */
 static const char *domain_list[] = {
   "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM",
   "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV",
   "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL",
-  "SCHED", "DOS", NULL
+  "SCHED", "GUARD", "CONSDIFF", "DOS", NULL
 };
 
 /** Return a bitmask for the log domain for which <b>domain</b> is the name,
@@ -1313,16 +1403,15 @@ parse_log_severity_config(const char **cfg_ptr,
     if (!strcasecmpstart(cfg, "file") ||
         !strcasecmpstart(cfg, "stderr") ||
         !strcasecmpstart(cfg, "stdout") ||
-        !strcasecmpstart(cfg, "syslog")) {
+        !strcasecmpstart(cfg, "syslog") ||
+        !strcasecmpstart(cfg, "android")) {
       goto done;
     }
     if (got_an_unqualified_range > 1)
       return -1;
 
-    space = strchr(cfg, ' ');
+    space = find_whitespace(cfg);
     dash = strchr(cfg, '-');
-    if (!space)
-      space = strchr(cfg, '\0');
     if (dash && dash < space) {
       sev_lo = tor_strndup(cfg, dash-cfg);
       sev_hi = tor_strndup(dash+1, space-(dash+1));
@@ -1392,4 +1481,3 @@ truncate_logs(void)
     }
   }
 }
-
diff --git a/src/common/torlog.h b/src/lib/log/log.h
index 20b7d938f0..d7a5070610 100644
--- a/src/common/torlog.h
+++ b/src/lib/log/log.h
@@ -1,19 +1,21 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file torlog.h
+ * \file log.h
  *
  * \brief Headers for log.c
  **/
 
 #ifndef TOR_TORLOG_H
 
-#include "compat.h"
-#include "testsupport.h"
+#include <stdarg.h>
+#include "lib/cc/torint.h"
+#include "lib/cc/compat_compiler.h"
+#include "lib/testsupport/testsupport.h"
 
 #ifdef HAVE_SYSLOG_H
 #include <syslog.h>
@@ -22,7 +24,7 @@
 #error "Your syslog.h thinks high numbers are more important.  " \
        "We aren't prepared to deal with that."
 #endif
-#else
+#else /* !(defined(HAVE_SYSLOG_H)) */
 /* Note: Syslog's logging code refers to priorities, with 0 being the most
  * important.  Thus, all our comparisons needed to be reversed when we added
  * syslog support.
@@ -48,7 +50,7 @@
 /** Error-level severity: for messages that only appear when something has gone
  * very wrong, and the Tor process can no longer proceed. */
 #define LOG_ERR     3
-#endif
+#endif /* defined(HAVE_SYSLOG_H) */
 
 /* Logging domains */
 
@@ -99,10 +101,14 @@
 #define LD_CHANNEL   (1u<<21)
 /** Scheduler */
 #define LD_SCHED     (1u<<22)
+/** Guard nodes */
+#define LD_GUARD     (1u<<23)
+/** Generation and application of consensus diffs. */
+#define LD_CONSDIFF  (1u<<24)
 /** Denial of Service mitigation. */
-#define LD_DOS       (1u<<23)
+#define LD_DOS       (1u<<25)
 /** Number of logging domains in the code. */
-#define N_LOGGING_DOMAINS 24
+#define N_LOGGING_DOMAINS 26
 
 /** This log message is not safe to send to a callback-based logger
  * immediately.  Used as a flag, not a log domain. */
@@ -139,13 +145,21 @@ void set_log_severity_config(int minSeverity, int maxSeverity,
                              log_severity_list_t *severity_out);
 void add_stream_log(const log_severity_list_t *severity, const char *name,
                     int fd);
-int add_file_log(const log_severity_list_t *severity, const char *filename,
-                 const int truncate);
+int add_file_log(const log_severity_list_t *severity,
+                 const char *filename,
+                 int fd);
+
 #ifdef HAVE_SYSLOG_H
 int add_syslog_log(const log_severity_list_t *severity,
                    const char* syslog_identity_tag);
-#endif
+#endif // HAVE_SYSLOG_H.
+#ifdef HAVE_ANDROID_LOG_H
+int add_android_log(const log_severity_list_t *severity,
+                    const char *android_identity_tag);
+#endif // HAVE_ANDROID_LOG_H.
 int add_callback_log(const log_severity_list_t *severity, log_callback cb);
+typedef void (*pending_callback_callback)(void);
+void logs_set_pending_callback_callback(pending_callback_callback cb);
 void logs_set_domain_logging(int enabled);
 int get_min_log_level(void);
 void switch_logs_debug(void);
@@ -165,8 +179,6 @@ void truncate_logs(void);
 void tor_log(int severity, log_domain_mask_t domain, const char *format, ...)
   CHECK_PRINTF(3,4);
 
-void tor_log_err_sigsafe(const char *m, ...);
-int tor_log_get_sigsafe_err_fds(const int **out);
 void tor_log_update_sigsafe_err_fds(void);
 
 struct smartlist_t;
@@ -183,6 +195,10 @@ void log_fn_ratelim_(struct ratelim_t *ratelim, int severity,
                      const char *format, ...)
   CHECK_PRINTF(5,6);
 
+int log_message_is_interesting(int severity, log_domain_mask_t domain);
+void tor_log_string(int severity, log_domain_mask_t domain,
+                    const char *function, const char *string);
+
 #if defined(__GNUC__) && __GNUC__ <= 3
 
 /* These are the GCC varidaic macros, so that older versions of GCC don't
@@ -211,7 +227,7 @@ void log_fn_ratelim_(struct ratelim_t *ratelim, int severity,
 #define log_err(domain, args...)                            \
   log_fn_(LOG_ERR, domain, __FUNCTION__, args)
 
-#else /* ! defined(__GNUC__) */
+#else /* !(defined(__GNUC__) && __GNUC__ <= 3) */
 
 /* Here are the c99 variadic macros, to work with non-GCC compilers */
 
@@ -238,7 +254,17 @@ void log_fn_ratelim_(struct ratelim_t *ratelim, int severity,
 #define log_fn_ratelim(ratelim, severity, domain, args,...)      \
   log_fn_ratelim_(ratelim, severity, domain, __FUNCTION__, \
                   args, ##__VA_ARGS__)
-#endif
+#endif /* defined(__GNUC__) && __GNUC__ <= 3 */
+
+/** This defines log levels that are linked in the Rust log module, rather
+ * than re-defining these in both Rust and C.
+ *
+ * C_RUST_COUPLED src/rust/tor_log LogSeverity, LogDomain
+ */
+extern const int LOG_WARN_;
+extern const int LOG_NOTICE_;
+extern const log_domain_mask_t LD_NET_;
+extern const log_domain_mask_t LD_GENERAL_;
 
 #ifdef LOG_PRIVATE
 MOCK_DECL(STATIC void, logv, (int severity, log_domain_mask_t domain,
@@ -247,5 +273,4 @@ MOCK_DECL(STATIC void, logv, (int severity, log_domain_mask_t domain,
 #endif
 
 # define TOR_TORLOG_H
-#endif
-
+#endif /* !defined(TOR_TORLOG_H) */
diff --git a/src/lib/log/ratelim.c b/src/lib/log/ratelim.c
new file mode 100644
index 0000000000..5eec742aa7
--- /dev/null
+++ b/src/lib/log/ratelim.c
@@ -0,0 +1,60 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file ratelim.c
+ * \brief Summarize similar messages that would otherwise flood the logs.
+ **/
+
+#include "lib/log/ratelim.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+/** If the rate-limiter <b>lim</b> is ready at <b>now</b>, return the number
+ * of calls to rate_limit_is_ready (including this one!) since the last time
+ * rate_limit_is_ready returned nonzero.  Otherwise return 0.
+ * If the call number hits <b>RATELIM_TOOMANY</b> limit, drop a warning
+ * about this event and stop counting. */
+static int
+rate_limit_is_ready(ratelim_t *lim, time_t now)
+{
+  if (lim->rate + lim->last_allowed <= now) {
+    int res = lim->n_calls_since_last_time + 1;
+    lim->last_allowed = now;
+    lim->n_calls_since_last_time = 0;
+    return res;
+  } else {
+    if (lim->n_calls_since_last_time <= RATELIM_TOOMANY) {
+      ++lim->n_calls_since_last_time;
+    }
+
+    return 0;
+  }
+}
+
+/** If the rate-limiter <b>lim</b> is ready at <b>now</b>, return a newly
+ * allocated string indicating how many messages were suppressed, suitable to
+ * append to a log message.  Otherwise return NULL. */
+char *
+rate_limit_log(ratelim_t *lim, time_t now)
+{
+  int n;
+  if ((n = rate_limit_is_ready(lim, now))) {
+    if (n == 1) {
+      return tor_strdup("");
+    } else {
+      char *cp=NULL;
+      const char *opt_over = (n >= RATELIM_TOOMANY) ? "over " : "";
+      /* XXXX this is not exactly correct: the messages could have occurred
+       * any time between the old value of lim->allowed and now. */
+      tor_asprintf(&cp,
+                   " [%s%d similar message(s) suppressed in last %d seconds]",
+                   opt_over, n-1, lim->rate);
+      return cp;
+    }
+  } else {
+    return NULL;
+  }
+}
diff --git a/src/lib/log/ratelim.h b/src/lib/log/ratelim.h
new file mode 100644
index 0000000000..48edd7c849
--- /dev/null
+++ b/src/lib/log/ratelim.h
@@ -0,0 +1,53 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file ratelim.h
+ * \brief Summarize similar messages that would otherwise flood the logs.
+ **/
+
+#ifndef TOR_RATELIM_H
+#define TOR_RATELIM_H
+
+#include <time.h>
+
+/* Rate-limiter */
+
+/** A ratelim_t remembers how often an event is occurring, and how often
+ * it's allowed to occur.  Typical usage is something like:
+ *
+   <pre>
+    if (possibly_very_frequent_event()) {
+      const int INTERVAL = 300;
+      static ratelim_t warning_limit = RATELIM_INIT(INTERVAL);
+      char *m;
+      if ((m = rate_limit_log(&warning_limit, approx_time()))) {
+        log_warn(LD_GENERAL, "The event occurred!%s", m);
+        tor_free(m);
+      }
+    }
+   </pre>
+
+   As a convenience wrapper for logging, you can replace the above with:
+   <pre>
+   if (possibly_very_frequent_event()) {
+     static ratelim_t warning_limit = RATELIM_INIT(300);
+     log_fn_ratelim(&warning_limit, LOG_WARN, LD_GENERAL,
+                    "The event occurred!");
+   }
+   </pre>
+ */
+typedef struct ratelim_t {
+  int rate;
+  time_t last_allowed;
+  int n_calls_since_last_time;
+} ratelim_t;
+
+#define RATELIM_INIT(r) { (r), 0, 0 }
+#define RATELIM_TOOMANY (16*1000*1000)
+
+char *rate_limit_log(ratelim_t *lim, time_t now);
+
+#endif
diff --git a/src/common/util_bug.c b/src/lib/log/util_bug.c
index 08aba47974..c65a91ae9e 100644
--- a/src/common/util_bug.c
+++ b/src/lib/log/util_bug.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -8,10 +8,18 @@
  **/
 
 #include "orconfig.h"
-#include "util_bug.h"
-#include "torlog.h"
-#include "backtrace.h"
-#include "container.h"
+#include "lib/log/util_bug.h"
+#include "lib/log/log.h"
+#include "lib/err/backtrace.h"
+#ifdef TOR_UNIT_TESTS
+#include "lib/smartlist_core/smartlist_core.h"
+#include "lib/smartlist_core/smartlist_foreach.h"
+#endif
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+#include <string.h>
+#include <stdlib.h>
 
 #ifdef TOR_UNIT_TESTS
 static void (*failed_assertion_cb)(void) = NULL;
@@ -44,7 +52,7 @@ static void
 add_captured_bug(const char *s)
 {
   --n_bugs_to_capture;
-  smartlist_add(bug_messages, tor_strdup(s));
+  smartlist_add_strdup(bug_messages, s);
 }
 /** Set a callback to be invoked when we get any tor_bug_occurred_
  * invocation. We use this in the unit tests so that a nonfatal
@@ -55,10 +63,10 @@ tor_set_failed_assertion_callback(void (*fn)(void))
 {
   failed_assertion_cb = fn;
 }
-#else
+#else /* !(defined(TOR_UNIT_TESTS)) */
 #define capturing_bugs() (0)
 #define add_captured_bug(s) do { } while (0)
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /** Helper for tor_assert: report the assertion failure. */
 void
@@ -113,3 +121,41 @@ tor_bug_occurred_(const char *fname, unsigned int line,
 #endif
 }
 
+/**
+ * Call the abort() function to kill the current process with a fatal
+ * error.
+ *
+ * (This is a separate function so that we declare it in util_bug.h without
+ * including stdlib in all the users of util_bug.h)
+ **/
+void
+tor_abort_(void)
+{
+  abort();
+}
+
+#ifdef _WIN32
+/** Take a filename and return a pointer to its final element.  This
+ * function is called on __FILE__ to fix a MSVC nit where __FILE__
+ * contains the full path to the file.  This is bad, because it
+ * confuses users to find the home directory of the person who
+ * compiled the binary in their warning messages.
+ */
+const char *
+tor_fix_source_file(const char *fname)
+{
+  const char *cp1, *cp2, *r;
+  cp1 = strrchr(fname, '/');
+  cp2 = strrchr(fname, '\\');
+  if (cp1 && cp2) {
+    r = (cp1<cp2)?(cp2+1):(cp1+1);
+  } else if (cp1) {
+    r = cp1+1;
+  } else if (cp2) {
+    r = cp2+1;
+  } else {
+    r = fname;
+  }
+  return r;
+}
+#endif /* defined(_WIN32) */
diff --git a/src/common/util_bug.h b/src/lib/log/util_bug.h
index 0db6bb6ab0..2a4d68127e 100644
--- a/src/common/util_bug.h
+++ b/src/lib/log/util_bug.h
@@ -1,18 +1,45 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file util_bug.h
+ *
+ * \brief Macros to manage assertions, fatal and non-fatal.
+ *
+ * Guidelines: All the different kinds of assertion in this file are for
+ * bug-checking only. Don't write code that can assert based on bad inputs.
+ *
+ * We provide two kinds of assertion here: "fatal" and "nonfatal". Use
+ * nonfatal assertions for any bug you can reasonably recover from -- and
+ * please, try to recover!  Many severe bugs in Tor have been caused by using
+ * a regular assertion when a nonfatal assertion would have been better.
+ *
+ * If you need to check a condition with a nonfatal assertion, AND recover
+ * from that same condition, consider using the BUG() macro inside a
+ * conditional.  For example:
+ *
+ * <code>
+ *  // wrong -- use tor_assert_nonfatal() if you just want an assertion.
+ *  BUG(ptr == NULL);
+ *
+ *  // okay, but needlessly verbose
+ *  tor_assert_nonfatal(ptr != NULL);
+ *  if (ptr == NULL) { ... }
+ *
+ *  // this is how we do it:
+ *  if (BUG(ptr == NULL)) { ... }
+ * </code>
  **/
 
 #ifndef TOR_UTIL_BUG_H
 #define TOR_UTIL_BUG_H
 
 #include "orconfig.h"
-#include "compat.h"
-#include "testsupport.h"
+#include "lib/cc/compat_compiler.h"
+#include "lib/log/log.h"
+#include "lib/testsupport/testsupport.h"
 
 /* Replace assert() with a variant that sends failures to the log before
  * calling assert() normally.
@@ -27,7 +54,7 @@
  * security-critical properties.
  */
 #error "Sorry; we don't support building with NDEBUG."
-#endif
+#endif /* defined(NDEBUG) */
 
 #if defined(TOR_UNIT_TESTS) && defined(__GNUC__)
 /* We define this GCC macro as a replacement for PREDICT_UNLIKELY() in this
@@ -72,11 +99,16 @@
   if (ASSERT_PREDICT_LIKELY_(expr)) {                                   \
   } else {                                                              \
     tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr);     \
-    abort();                                                            \
+    tor_abort_();                                                       \
   } STMT_END
-#endif
+#endif /* defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) */
 
-#define tor_assert_unreached() tor_assert(0)
+#define tor_assert_unreached()                                  \
+  STMT_BEGIN {                                                  \
+    tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__,     \
+                          "line should be unreached");          \
+    tor_abort_();                                               \
+  } STMT_END
 
 /* Non-fatal bug assertions. The "unreached" variants mean "this line should
  * never be reached." The "once" variants mean "Don't log a warning more than
@@ -92,8 +124,8 @@
 #undef BUG
 // Coverity defines this in global headers; let's override it.  This is a
 // magic coverity-only preprocessor thing.
-#nodef BUG(x) ((x)?(__coverity_panic__(),1):0)
-#endif
+#nodef BUG(x) (x)
+#endif /* defined(__COVERITY__) */
 
 #if defined(__COVERITY__) || defined(__clang_analyzer__)
 // We're running with a static analysis tool: let's treat even nonfatal
@@ -109,7 +141,7 @@
 #define BUG(cond)                                                       \
   (ASSERT_PREDICT_UNLIKELY_(cond) ?                                     \
    (tor_assertion_failed_(SHORT_FILE__,__LINE__,__func__,"!("#cond")"), \
-    abort(), 1)                                                         \
+    tor_abort_(), 1)                                                    \
    : 0)
 #elif defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS)
 #define tor_assert_nonfatal_unreached() STMT_NIL
@@ -146,7 +178,7 @@
   (ASSERT_PREDICT_UNLIKELY_(cond) ?                                     \
    (tor_bug_occurred_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",0), 1) \
    : 0)
-#endif
+#endif /* defined(ALL_BUGS_ARE_FATAL) || ... */
 
 #ifdef __GNUC__
 #define IF_BUG_ONCE__(cond,var)                                         \
@@ -159,7 +191,7 @@
                           "!("#cond")", 1);                             \
       }                                                                 \
       bool_result; } ))
-#else
+#else /* !(defined(__GNUC__)) */
 #define IF_BUG_ONCE__(cond,var)                                         \
   static int var = 0;                                                   \
   if ((cond) ?                                                          \
@@ -169,7 +201,7 @@
                            "!("#cond")", 1),                            \
         1))                                                             \
       : 0)
-#endif
+#endif /* defined(__GNUC__) */
 #define IF_BUG_ONCE_VARNAME_(a)               \
   warning_logged_on_ ## a ## __
 #define IF_BUG_ONCE_VARNAME__(a)              \
@@ -194,11 +226,21 @@ void tor_bug_occurred_(const char *fname, unsigned int line,
                        const char *func, const char *expr,
                        int once);
 
+void tor_abort_(void) ATTR_NORETURN;
+
+#ifdef _WIN32
+#define SHORT_FILE__ (tor_fix_source_file(__FILE__))
+const char *tor_fix_source_file(const char *fname);
+#else
+#define SHORT_FILE__ (__FILE__)
+#define tor_fix_source_file(s) (s)
+#endif /* defined(_WIN32) */
+
 #ifdef TOR_UNIT_TESTS
 void tor_capture_bugs_(int n);
 void tor_end_capture_bugs_(void);
 const struct smartlist_t *tor_get_captured_bug_log_(void);
 void tor_set_failed_assertion_callback(void (*fn)(void));
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
-#endif
+#endif /* !defined(TOR_UTIL_BUG_H) */
diff --git a/src/lib/log/win32err.c b/src/lib/log/win32err.c
new file mode 100644
index 0000000000..dc45cb4c3d
--- /dev/null
+++ b/src/lib/log/win32err.c
@@ -0,0 +1,61 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file win32err.c
+ * \brief Convert windows error codes to useful C strings.
+ **/
+
+#ifdef _WIN32
+#include "orconfig.h"
+#include "lib/log/win32err.h"
+#include "lib/malloc/malloc.h"
+
+#include <tchar.h>
+#include <windows.h>
+
+/** Return a newly allocated string describing the windows system error code
+ * <b>err</b>.  Note that error codes are different from errno.  Error codes
+ * come from GetLastError() when a winapi call fails.  errno is set only when
+ * ANSI functions fail.  Whee. */
+char *
+format_win32_error(DWORD err)
+{
+  TCHAR *str = NULL;
+  char *result;
+  DWORD n;
+
+  /* Somebody once decided that this interface was better than strerror(). */
+  n = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER |
+                 FORMAT_MESSAGE_FROM_SYSTEM |
+                 FORMAT_MESSAGE_IGNORE_INSERTS,
+                 NULL, err,
+                 MAKELANGID(LANG_ENGLISH, SUBLANG_DEFAULT),
+                 (LPVOID)&str,
+                 0, NULL);
+
+  if (str && n) {
+#ifdef UNICODE
+    size_t len;
+    if (n > 128*1024)
+      len = (128 * 1024) * 2 + 1; /* This shouldn't be possible, but let's
+                                   * make sure. */
+    else
+      len = n * 2 + 1;
+    result = tor_malloc(len);
+    wcstombs(result,str,len);
+    result[len-1] = '\0';
+#else /* !(defined(UNICODE)) */
+    result = tor_strdup(str);
+#endif /* defined(UNICODE) */
+  } else {
+    result = tor_strdup("<unformattable error>");
+  }
+  if (str) {
+    LocalFree(str); /* LocalFree != free() */
+  }
+  return result;
+}
+#endif /* defined(_WIN32) */
diff --git a/src/lib/log/win32err.h b/src/lib/log/win32err.h
new file mode 100644
index 0000000000..33413dfd15
--- /dev/null
+++ b/src/lib/log/win32err.h
@@ -0,0 +1,22 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file win32err.h
+ * \brief Header for win32err.c
+ **/
+
+#ifndef TOR_WIN32ERR_H
+#define TOR_WIN32ERR_H
+
+#include "orconfig.h"
+
+/* Platform-specific helpers. */
+#ifdef _WIN32
+#include <windef.h>
+char *format_win32_error(DWORD err);
+#endif
+
+#endif
diff --git a/src/lib/malloc/.may_include b/src/lib/malloc/.may_include
new file mode 100644
index 0000000000..cc62bb1013
--- /dev/null
+++ b/src/lib/malloc/.may_include
@@ -0,0 +1,6 @@
+orconfig.h
+
+lib/cc/*.h
+lib/err/*.h
+lib/malloc/*.h
+lib/testsupport/testsupport.h
diff --git a/src/lib/malloc/include.am b/src/lib/malloc/include.am
new file mode 100644
index 0000000000..502cc1c6b7
--- /dev/null
+++ b/src/lib/malloc/include.am
@@ -0,0 +1,21 @@
+
+noinst_LIBRARIES += src/lib/libtor-malloc.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-malloc-testing.a
+endif
+
+src_lib_libtor_malloc_a_SOURCES =			\
+	src/lib/malloc/malloc.c
+
+if USE_OPENBSD_MALLOC
+src_lib_libtor_malloc_a_SOURCES += src/ext/OpenBSD_malloc_Linux.c
+endif
+
+src_lib_libtor_malloc_testing_a_SOURCES = \
+	$(src_lib_libtor_malloc_a_SOURCES)
+src_lib_libtor_malloc_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_malloc_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/malloc/malloc.h
diff --git a/src/lib/malloc/malloc.c b/src/lib/malloc/malloc.c
new file mode 100644
index 0000000000..8628acfc97
--- /dev/null
+++ b/src/lib/malloc/malloc.c
@@ -0,0 +1,230 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file malloc.c
+ * \brief Wrappers for C malloc code, and replacements for items that
+ *   may be missing.
+ **/
+
+#include "orconfig.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "lib/testsupport/testsupport.h"
+#define UTIL_MALLOC_PRIVATE
+#include "lib/malloc/malloc.h"
+#include "lib/cc/torint.h"
+#include "lib/err/torerr.h"
+
+#ifdef __clang_analyzer__
+#undef MALLOC_ZERO_WORKS
+#endif
+
+/** Allocate a chunk of <b>size</b> bytes of memory, and return a pointer to
+ * result.  On error, log and terminate the process.  (Same as malloc(size),
+ * but never returns NULL.)
+ */
+void *
+tor_malloc_(size_t size)
+{
+  void *result;
+
+  raw_assert(size < SIZE_T_CEILING);
+
+#ifndef MALLOC_ZERO_WORKS
+  /* Some libc mallocs don't work when size==0. Override them. */
+  if (size==0) {
+    size=1;
+  }
+#endif /* !defined(MALLOC_ZERO_WORKS) */
+
+  result = raw_malloc(size);
+
+  if (PREDICT_UNLIKELY(result == NULL)) {
+    /* LCOV_EXCL_START */
+    /* If these functions die within a worker process, they won't call
+     * spawn_exit, but that's ok, since the parent will run out of memory soon
+     * anyway. */
+    raw_assert_unreached_msg("Out of memory on malloc(). Dying.");
+    /* LCOV_EXCL_STOP */
+  }
+  return result;
+}
+
+/** Allocate a chunk of <b>size</b> bytes of memory, fill the memory with
+ * zero bytes, and return a pointer to the result.  Log and terminate
+ * the process on error.  (Same as calloc(size,1), but never returns NULL.)
+ */
+void *
+tor_malloc_zero_(size_t size)
+{
+  /* You may ask yourself, "wouldn't it be smart to use calloc instead of
+   * malloc+memset?  Perhaps libc's calloc knows some nifty optimization trick
+   * we don't!"  Indeed it does, but its optimizations are only a big win when
+   * we're allocating something very big (it knows if it just got the memory
+   * from the OS in a pre-zeroed state).  We don't want to use tor_malloc_zero
+   * for big stuff, so we don't bother with calloc. */
+  void *result = tor_malloc_(size);
+  memset(result, 0, size);
+  return result;
+}
+
+/* The square root of SIZE_MAX + 1.  If a is less than this, and b is less
+ * than this, then a*b is less than SIZE_MAX.  (For example, if size_t is
+ * 32 bits, then SIZE_MAX is 0xffffffff and this value is 0x10000.  If a and
+ * b are less than this, then their product is at most (65535*65535) ==
+ * 0xfffe0001. */
+#define SQRT_SIZE_MAX_P1 (((size_t)1) << (sizeof(size_t)*4))
+
+/** Return non-zero if and only if the product of the arguments is exact,
+ * and cannot overflow. */
+STATIC int
+size_mul_check(const size_t x, const size_t y)
+{
+  /* This first check is equivalent to
+     (x < SQRT_SIZE_MAX_P1 && y < SQRT_SIZE_MAX_P1)
+
+     Rationale: if either one of x or y is >= SQRT_SIZE_MAX_P1, then it
+     will have some bit set in its most significant half.
+   */
+  return ((x|y) < SQRT_SIZE_MAX_P1 ||
+          y == 0 ||
+          x <= SIZE_MAX / y);
+}
+
+/** Allocate a chunk of <b>nmemb</b>*<b>size</b> bytes of memory, fill
+ * the memory with zero bytes, and return a pointer to the result.
+ * Log and terminate the process on error.  (Same as
+ * calloc(<b>nmemb</b>,<b>size</b>), but never returns NULL.)
+ * The second argument (<b>size</b>) should preferably be non-zero
+ * and a compile-time constant.
+ */
+void *
+tor_calloc_(size_t nmemb, size_t size)
+{
+  raw_assert(size_mul_check(nmemb, size));
+  return tor_malloc_zero_((nmemb * size));
+}
+
+/** Change the size of the memory block pointed to by <b>ptr</b> to <b>size</b>
+ * bytes long; return the new memory block.  On error, log and
+ * terminate. (Like realloc(ptr,size), but never returns NULL.)
+ */
+void *
+tor_realloc_(void *ptr, size_t size)
+{
+  void *result;
+
+  raw_assert(size < SIZE_T_CEILING);
+
+#ifndef MALLOC_ZERO_WORKS
+  /* Some libc mallocs don't work when size==0. Override them. */
+  if (size==0) {
+    size=1;
+  }
+#endif /* !defined(MALLOC_ZERO_WORKS) */
+
+  result = raw_realloc(ptr, size);
+
+  if (PREDICT_UNLIKELY(result == NULL)) {
+    /* LCOV_EXCL_START */
+    raw_assert_unreached_msg("Out of memory on realloc(). Dying.");
+    /* LCOV_EXCL_STOP */
+  }
+  return result;
+}
+
+/**
+ * Try to realloc <b>ptr</b> so that it takes up sz1 * sz2 bytes.  Check for
+ * overflow. Unlike other allocation functions, return NULL on overflow.
+ */
+void *
+tor_reallocarray_(void *ptr, size_t sz1, size_t sz2)
+{
+  /* XXXX we can make this return 0, but we would need to check all the
+   * reallocarray users. */
+  raw_assert(size_mul_check(sz1, sz2));
+
+  return tor_realloc(ptr, (sz1 * sz2));
+}
+
+/** Return a newly allocated copy of the NUL-terminated string s. On
+ * error, log and terminate.  (Like strdup(s), but never returns
+ * NULL.)
+ */
+char *
+tor_strdup_(const char *s)
+{
+  char *duplicate;
+  raw_assert(s);
+
+  duplicate = raw_strdup(s);
+
+  if (PREDICT_UNLIKELY(duplicate == NULL)) {
+    /* LCOV_EXCL_START */
+    raw_assert_unreached_msg("Out of memory on strdup(). Dying.");
+    /* LCOV_EXCL_STOP */
+  }
+  return duplicate;
+}
+
+/** Allocate and return a new string containing the first <b>n</b>
+ * characters of <b>s</b>.  If <b>s</b> is longer than <b>n</b>
+ * characters, only the first <b>n</b> are copied.  The result is
+ * always NUL-terminated.  (Like strndup(s,n), but never returns
+ * NULL.)
+ */
+char *
+tor_strndup_(const char *s, size_t n)
+{
+  char *duplicate;
+  raw_assert(s);
+  raw_assert(n < SIZE_T_CEILING);
+  duplicate = tor_malloc_((n+1));
+  /* Performance note: Ordinarily we prefer strlcpy to strncpy.  But
+   * this function gets called a whole lot, and platform strncpy is
+   * much faster than strlcpy when strlen(s) is much longer than n.
+   */
+  strncpy(duplicate, s, n);
+  duplicate[n]='\0';
+  return duplicate;
+}
+
+/** Allocate a chunk of <b>len</b> bytes, with the same contents as the
+ * <b>len</b> bytes starting at <b>mem</b>. */
+void *
+tor_memdup_(const void *mem, size_t len)
+{
+  char *duplicate;
+  raw_assert(len < SIZE_T_CEILING);
+  raw_assert(mem);
+  duplicate = tor_malloc_(len);
+  memcpy(duplicate, mem, len);
+  return duplicate;
+}
+
+/** As tor_memdup(), but add an extra 0 byte at the end of the resulting
+ * memory. */
+void *
+tor_memdup_nulterm_(const void *mem, size_t len)
+{
+  char *duplicate;
+  raw_assert(len < SIZE_T_CEILING+1);
+  raw_assert(mem);
+  duplicate = tor_malloc_(len+1);
+  memcpy(duplicate, mem, len);
+  duplicate[len] = '\0';
+  return duplicate;
+}
+
+/** Helper for places that need to take a function pointer to the right
+ * spelling of "free()". */
+void
+tor_free_(void *mem)
+{
+  tor_free(mem);
+}
diff --git a/src/lib/malloc/malloc.h b/src/lib/malloc/malloc.h
new file mode 100644
index 0000000000..ef6b509ca4
--- /dev/null
+++ b/src/lib/malloc/malloc.h
@@ -0,0 +1,92 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file malloc.h
+ * \brief Headers for util_malloc.c
+ **/
+
+#ifndef TOR_UTIL_MALLOC_H
+#define TOR_UTIL_MALLOC_H
+
+#include <stddef.h>
+#include <stdlib.h>
+#include "lib/cc/compat_compiler.h"
+
+/* Memory management */
+void *tor_malloc_(size_t size) ATTR_MALLOC;
+void *tor_malloc_zero_(size_t size) ATTR_MALLOC;
+void *tor_calloc_(size_t nmemb, size_t size) ATTR_MALLOC;
+void *tor_realloc_(void *ptr, size_t size);
+void *tor_reallocarray_(void *ptr, size_t size1, size_t size2);
+char *tor_strdup_(const char *s) ATTR_MALLOC;
+char *tor_strndup_(const char *s, size_t n)
+  ATTR_MALLOC;
+void *tor_memdup_(const void *mem, size_t len)
+  ATTR_MALLOC;
+void *tor_memdup_nulterm_(const void *mem, size_t len)
+  ATTR_MALLOC;
+void tor_free_(void *mem);
+
+/** Release memory allocated by tor_malloc, tor_realloc, tor_strdup,
+ * etc.  Unlike the free() function, the tor_free() macro sets the
+ * pointer value to NULL after freeing it.
+ *
+ * This is a macro.  If you need a function pointer to release memory from
+ * tor_malloc(), use tor_free_().
+ *
+ * Note that this macro takes the address of the pointer it is going to
+ * free and clear.  If that pointer is stored with a nonstandard
+ * alignment (eg because of a "packed" pragma) it is not correct to use
+ * tor_free().
+ */
+#ifdef __GNUC__
+#define tor_free(p) STMT_BEGIN                                 \
+    typeof(&(p)) tor_free__tmpvar = &(p);                      \
+    raw_free(*tor_free__tmpvar);                               \
+    *tor_free__tmpvar=NULL;                                    \
+  STMT_END
+#else
+#define tor_free(p) STMT_BEGIN                                 \
+  raw_free(p);                                                 \
+  (p)=NULL;                                                    \
+  STMT_END
+#endif
+
+#define tor_malloc(size)       tor_malloc_(size)
+#define tor_malloc_zero(size)  tor_malloc_zero_(size)
+#define tor_calloc(nmemb,size) tor_calloc_(nmemb, size)
+#define tor_realloc(ptr, size) tor_realloc_(ptr, size)
+#define tor_reallocarray(ptr, sz1, sz2) \
+  tor_reallocarray_((ptr), (sz1), (sz2))
+#define tor_strdup(s)          tor_strdup_(s)
+#define tor_strndup(s, n)      tor_strndup_(s, n)
+#define tor_memdup(s, n)       tor_memdup_(s, n)
+#define tor_memdup_nulterm(s, n)       tor_memdup_nulterm_(s, n)
+
+/* Aliases for the underlying system malloc/realloc/free. Only use
+ * them to indicate "I really want the underlying system function, I know
+ * what I'm doing." */
+#define raw_malloc  malloc
+#define raw_realloc realloc
+#define raw_free    free
+#define raw_strdup  strdup
+
+/* Helper macro: free a variable of type 'typename' using freefn, and
+ * set the variable to NULL.
+ */
+#define FREE_AND_NULL(typename, freefn, var)                            \
+  do {                                                                  \
+    /* only evaluate (var) once. */                                     \
+    typename **tmp__free__ptr ## freefn = &(var);                       \
+    freefn(*tmp__free__ptr ## freefn);                                  \
+    (*tmp__free__ptr ## freefn) = NULL;                                 \
+  } while (0)
+
+#ifdef UTIL_MALLOC_PRIVATE
+STATIC int size_mul_check(const size_t x, const size_t y);
+#endif
+
+#endif /* !defined(TOR_UTIL_MALLOC_H) */
diff --git a/src/lib/math/.may_include b/src/lib/math/.may_include
new file mode 100644
index 0000000000..1fd26864dc
--- /dev/null
+++ b/src/lib/math/.may_include
@@ -0,0 +1,5 @@
+orconfig.h
+
+lib/cc/*.h
+lib/log/*.h
+lib/math/*.h
diff --git a/src/lib/math/fp.c b/src/lib/math/fp.c
new file mode 100644
index 0000000000..eafad358c3
--- /dev/null
+++ b/src/lib/math/fp.c
@@ -0,0 +1,123 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file fp.c
+ *
+ * \brief Basic floating-point compatibility and convenience code.
+ **/
+
+#include "orconfig.h"
+#include "lib/math/fp.h"
+
+#include <math.h>
+
+/**
+ * Returns the natural logarithm of d base e.  We defined this wrapper here so
+ * to avoid conflicts with old versions of tor_log(), which were named log().
+ */
+double
+tor_mathlog(double d)
+{
+  return log(d);
+}
+
+/** Return the long integer closest to <b>d</b>. We define this wrapper
+ * here so that not all users of math.h need to use the right incantations
+ * to get the c99 functions. */
+long
+tor_lround(double d)
+{
+#if defined(HAVE_LROUND)
+  return lround(d);
+#elif defined(HAVE_RINT)
+  return (long)rint(d);
+#else
+  return (long)(d > 0 ? d + 0.5 : ceil(d - 0.5));
+#endif /* defined(HAVE_LROUND) || ... */
+}
+
+/** Return the 64-bit integer closest to d.  We define this wrapper here so
+ * that not all users of math.h need to use the right incantations to get the
+ * c99 functions. */
+int64_t
+tor_llround(double d)
+{
+#if defined(HAVE_LLROUND)
+  return (int64_t)llround(d);
+#elif defined(HAVE_RINT)
+  return (int64_t)rint(d);
+#else
+  return (int64_t)(d > 0 ? d + 0.5 : ceil(d - 0.5));
+#endif /* defined(HAVE_LLROUND) || ... */
+}
+
+/** Cast a given double value to a int64_t. Return 0 if number is NaN.
+ * Returns either INT64_MIN or INT64_MAX if number is outside of the int64_t
+ * range. */
+int64_t
+clamp_double_to_int64(double number)
+{
+  int exponent;
+
+#if (defined(MINGW_ANY)||defined(__FreeBSD__)) && GCC_VERSION >= 409
+/*
+  Mingw's math.h uses gcc's __builtin_choose_expr() facility to declare
+  isnan, isfinite, and signbit.  But as implemented in at least some
+  versions of gcc, __builtin_choose_expr() can generate type warnings
+  even from branches that are not taken.  So, suppress those warnings.
+
+  FreeBSD's math.h uses an __fp_type_select() macro, which dispatches
+  based on sizeof -- again, this can generate type warnings from
+  branches that are not taken.
+*/
+#define PROBLEMATIC_FLOAT_CONVERSION_WARNING
+DISABLE_GCC_WARNING(float-conversion)
+#endif /* defined(MINGW_ANY) && GCC_VERSION >= 409 */
+
+/*
+  With clang 4.0 we apparently run into "double promotion" warnings here,
+  since clang thinks we're promoting a double to a long double.
+ */
+#if defined(__clang__)
+#if __has_warning("-Wdouble-promotion")
+#define PROBLEMATIC_DOUBLE_PROMOTION_WARNING
+DISABLE_GCC_WARNING(double-promotion)
+#endif
+#endif /* defined(__clang__) */
+
+  /* NaN is a special case that can't be used with the logic below. */
+  if (isnan(number)) {
+    return 0;
+  }
+
+  /* Time to validate if result can overflows a int64_t value. Fun with
+   * float! Find that exponent exp such that
+   *    number == x * 2^exp
+   * for some x with abs(x) in [0.5, 1.0). Note that this implies that the
+   * magnitude of number is strictly less than 2^exp.
+   *
+   * If number is infinite, the call to frexp is legal but the contents of
+   * are exponent unspecified. */
+  frexp(number, &exponent);
+
+  /* If the magnitude of number is strictly less than 2^63, the truncated
+   * version of number is guaranteed to be representable. The only
+   * representable integer for which this is not the case is INT64_MIN, but
+   * it is covered by the logic below. */
+  if (isfinite(number) && exponent <= 63) {
+    return (int64_t)number;
+  }
+
+  /* Handle infinities and finite numbers with magnitude >= 2^63. */
+  return signbit(number) ? INT64_MIN : INT64_MAX;
+
+#ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING
+ENABLE_GCC_WARNING(double-promotion)
+#endif
+#ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING
+ENABLE_GCC_WARNING(float-conversion)
+#endif
+}
diff --git a/src/lib/math/fp.h b/src/lib/math/fp.h
new file mode 100644
index 0000000000..6f07152e92
--- /dev/null
+++ b/src/lib/math/fp.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file fp.h
+ *
+ * \brief Header for fp.c
+ **/
+
+#ifndef TOR_FP_H
+#define TOR_FP_H
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+
+double tor_mathlog(double d) ATTR_CONST;
+long tor_lround(double d) ATTR_CONST;
+int64_t tor_llround(double d) ATTR_CONST;
+int64_t clamp_double_to_int64(double number);
+
+#endif
diff --git a/src/lib/math/include.am b/src/lib/math/include.am
new file mode 100644
index 0000000000..b088b3f3cc
--- /dev/null
+++ b/src/lib/math/include.am
@@ -0,0 +1,20 @@
+
+noinst_LIBRARIES += src/lib/libtor-math.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-math-testing.a
+endif
+
+src_lib_libtor_math_a_SOURCES =	\
+		src/lib/math/fp.c		\
+		src/lib/math/laplace.c
+
+
+src_lib_libtor_math_testing_a_SOURCES = \
+	$(src_lib_libtor_math_a_SOURCES)
+src_lib_libtor_math_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_math_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=				\
+		src/lib/math/fp.h		\
+		src/lib/math/laplace.h
diff --git a/src/lib/math/laplace.c b/src/lib/math/laplace.c
new file mode 100644
index 0000000000..302edb20b8
--- /dev/null
+++ b/src/lib/math/laplace.c
@@ -0,0 +1,73 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file laplace.c
+ *
+ * \brief Implements a Laplace distribution, used for adding noise to things.
+ **/
+
+#include "orconfig.h"
+#include "lib/math/laplace.h"
+#include "lib/math/fp.h"
+
+#include "lib/log/util_bug.h"
+
+#include <math.h>
+#include <stdlib.h>
+
+/** Transform a random value <b>p</b> from the uniform distribution in
+ * [0.0, 1.0[ into a Laplace distributed value with location parameter
+ * <b>mu</b> and scale parameter <b>b</b>. Truncate the final result
+ * to be an integer in [INT64_MIN, INT64_MAX]. */
+int64_t
+sample_laplace_distribution(double mu, double b, double p)
+{
+  double result;
+  tor_assert(p >= 0.0 && p < 1.0);
+
+  /* This is the "inverse cumulative distribution function" from:
+   * http://en.wikipedia.org/wiki/Laplace_distribution */
+  if (p <= 0.0) {
+    /* Avoid taking log(0.0) == -INFINITY, as some processors or compiler
+     * options can cause the program to trap. */
+    return INT64_MIN;
+  }
+
+  result = mu - b * (p > 0.5 ? 1.0 : -1.0)
+                  * tor_mathlog(1.0 - 2.0 * fabs(p - 0.5));
+
+  return clamp_double_to_int64(result);
+}
+
+/** Add random noise between INT64_MIN and INT64_MAX coming from a Laplace
+ * distribution with mu = 0 and b = <b>delta_f</b>/<b>epsilon</b> to
+ * <b>signal</b> based on the provided <b>random</b> value in [0.0, 1.0[.
+ * The epsilon value must be between ]0.0, 1.0]. delta_f must be greater
+ * than 0. */
+int64_t
+add_laplace_noise(int64_t signal_, double random_, double delta_f,
+                  double epsilon)
+{
+  int64_t noise;
+
+  /* epsilon MUST be between ]0.0, 1.0] */
+  tor_assert(epsilon > 0.0 && epsilon <= 1.0);
+  /* delta_f MUST be greater than 0. */
+  tor_assert(delta_f > 0.0);
+
+  /* Just add noise, no further signal */
+  noise = sample_laplace_distribution(0.0,
+                                      delta_f / epsilon,
+                                      random_);
+
+  /* Clip (signal + noise) to [INT64_MIN, INT64_MAX] */
+  if (noise > 0 && INT64_MAX - noise < signal_)
+    return INT64_MAX;
+  else if (noise < 0 && INT64_MIN - noise > signal_)
+    return INT64_MIN;
+  else
+    return signal_ + noise;
+}
diff --git a/src/lib/math/laplace.h b/src/lib/math/laplace.h
new file mode 100644
index 0000000000..e8651e5197
--- /dev/null
+++ b/src/lib/math/laplace.h
@@ -0,0 +1,22 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file laplace.h
+ *
+ * \brief Header for laplace.c
+ **/
+
+#ifndef TOR_LAPLACE_H
+#define TOR_LAPLACE_H
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+
+int64_t sample_laplace_distribution(double mu, double b, double p);
+int64_t add_laplace_noise(int64_t signal, double random, double delta_f,
+                          double epsilon);
+
+#endif
diff --git a/src/lib/memarea/.may_include b/src/lib/memarea/.may_include
new file mode 100644
index 0000000000..814652a93c
--- /dev/null
+++ b/src/lib/memarea/.may_include
@@ -0,0 +1,7 @@
+orconfig.h
+lib/arch/*.h
+lib/cc/*.h
+lib/container/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/memarea/*.h
diff --git a/src/lib/memarea/include.am b/src/lib/memarea/include.am
new file mode 100644
index 0000000000..94343dcead
--- /dev/null
+++ b/src/lib/memarea/include.am
@@ -0,0 +1,17 @@
+
+noinst_LIBRARIES += src/lib/libtor-memarea.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-memarea-testing.a
+endif
+
+src_lib_libtor_memarea_a_SOURCES =			\
+	src/lib/memarea/memarea.c
+
+src_lib_libtor_memarea_testing_a_SOURCES = \
+	$(src_lib_libtor_memarea_a_SOURCES)
+src_lib_libtor_memarea_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_memarea_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/memarea/memarea.h
diff --git a/src/common/memarea.c b/src/lib/memarea/memarea.c
index 7d16b702e3..486673116c 100644
--- a/src/common/memarea.c
+++ b/src/lib/memarea/memarea.c
@@ -1,17 +1,27 @@
-/* Copyright (c) 2008-2016, The Tor Project, Inc. */
+/* Copyright (c) 2008-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-/** \file memarea.c
+/**
+ * \file memarea.c
+ *
  * \brief Implementation for memarea_t, an allocator for allocating lots of
  * small objects that will be freed all at once.
  */
 
 #include "orconfig.h"
+#include "lib/memarea/memarea.h"
+
 #include <stdlib.h>
-#include "memarea.h"
-#include "util.h"
-#include "compat.h"
-#include "torlog.h"
+#include <string.h>
+
+#include "lib/arch/bytes.h"
+#include "lib/cc/torint.h"
+#include "lib/container/smartlist.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+
+#ifndef DISABLE_MEMORY_SENTINELS
 
 /** If true, we try to detect any attempts to write beyond the length of a
  * memarea. */
@@ -29,7 +39,7 @@
 #define MEMAREA_ALIGN_MASK ((uintptr_t)7)
 #else
 #error "void* is neither 4 nor 8 bytes long. I don't know how to align stuff."
-#endif
+#endif /* MEMAREA_ALIGN == 4 || ... */
 
 #if defined(__GNUC__) && defined(FLEXIBLE_ARRAY_MEMBER)
 #define USE_ALIGNED_ATTRIBUTE
@@ -37,7 +47,7 @@
 #define U_MEM mem
 #else
 #define U_MEM u.mem
-#endif
+#endif /* defined(__GNUC__) && defined(FLEXIBLE_ARRAY_MEMBER) */
 
 #ifdef USE_SENTINELS
 /** Magic value that we stick at the end of a memarea so we can make sure
@@ -57,11 +67,11 @@
   uint32_t sent_val = get_uint32(&(chunk)->U_MEM[chunk->mem_size]);     \
   tor_assert(sent_val == SENTINEL_VAL);                                 \
   STMT_END
-#else
+#else /* !(defined(USE_SENTINELS)) */
 #define SENTINEL_LEN 0
 #define SET_SENTINEL(chunk) STMT_NIL
 #define CHECK_SENTINEL(chunk) STMT_NIL
-#endif
+#endif /* defined(USE_SENTINELS) */
 
 /** Increment <b>ptr</b> until it is aligned to MEMAREA_ALIGN. */
 static inline void *
@@ -93,12 +103,12 @@ typedef struct memarea_chunk_t {
     void *void_for_alignment_; /**< Dummy; used to make sure mem is aligned. */
   } u; /**< Union used to enforce alignment when we don't have support for
         * doing it right. */
-#endif
+#endif /* defined(USE_ALIGNED_ATTRIBUTE) */
 } memarea_chunk_t;
 
 /** How many bytes are needed for overhead before we get to the memory part
  * of a chunk? */
-#define CHUNK_HEADER_SIZE STRUCT_OFFSET(memarea_chunk_t, U_MEM)
+#define CHUNK_HEADER_SIZE offsetof(memarea_chunk_t, U_MEM)
 
 /** What's the smallest that we'll allocate a chunk? */
 #define CHUNK_SIZE 4096
@@ -149,7 +159,7 @@ memarea_new(void)
 /** Free <b>area</b>, invalidating all pointers returned from memarea_alloc()
  * and friends for this area */
 void
-memarea_drop_all(memarea_t *area)
+memarea_drop_all_(memarea_t *area)
 {
   memarea_chunk_t *chunk, *next;
   for (chunk = area->first; chunk; chunk = next) {
@@ -304,3 +314,90 @@ memarea_assert_ok(memarea_t *area)
   }
 }
 
+#else /* !(!defined(DISABLE_MEMORY_SENTINELS)) */
+
+struct memarea_t {
+  smartlist_t *pieces;
+};
+
+memarea_t *
+memarea_new(void)
+{
+  memarea_t *ma = tor_malloc_zero(sizeof(memarea_t));
+  ma->pieces = smartlist_new();
+  return ma;
+}
+void
+memarea_drop_all_(memarea_t *area)
+{
+  memarea_clear(area);
+  smartlist_free(area->pieces);
+  tor_free(area);
+}
+void
+memarea_clear(memarea_t *area)
+{
+  SMARTLIST_FOREACH(area->pieces, void *, p, tor_free_(p));
+  smartlist_clear(area->pieces);
+}
+int
+memarea_owns_ptr(const memarea_t *area, const void *ptr)
+{
+  SMARTLIST_FOREACH(area->pieces, const void *, p, if (ptr == p) return 1;);
+  return 0;
+}
+
+void *
+memarea_alloc(memarea_t *area, size_t sz)
+{
+  void *result = tor_malloc(sz);
+  smartlist_add(area->pieces, result);
+  return result;
+}
+
+void *
+memarea_alloc_zero(memarea_t *area, size_t sz)
+{
+  void *result = tor_malloc_zero(sz);
+  smartlist_add(area->pieces, result);
+  return result;
+}
+void *
+memarea_memdup(memarea_t *area, const void *s, size_t n)
+{
+  void *r = memarea_alloc(area, n);
+  memcpy(r, s, n);
+  return r;
+}
+char *
+memarea_strdup(memarea_t *area, const char *s)
+{
+  size_t n = strlen(s);
+  char *r = memarea_alloc(area, n+1);
+  memcpy(r, s, n);
+  r[n] = 0;
+  return r;
+}
+char *
+memarea_strndup(memarea_t *area, const char *s, size_t n)
+{
+  size_t ln = strnlen(s, n);
+  char *r = memarea_alloc(area, ln+1);
+  memcpy(r, s, ln);
+  r[ln] = 0;
+  return r;
+}
+void
+memarea_get_stats(memarea_t *area,
+                  size_t *allocated_out, size_t *used_out)
+{
+  (void)area;
+  *allocated_out = *used_out = 128;
+}
+void
+memarea_assert_ok(memarea_t *area)
+{
+  (void)area;
+}
+
+#endif /* !defined(DISABLE_MEMORY_SENTINELS) */
diff --git a/src/common/memarea.h b/src/lib/memarea/memarea.h
index 85bca51ad3..9c23cf62e9 100644
--- a/src/common/memarea.h
+++ b/src/lib/memarea/memarea.h
@@ -1,14 +1,26 @@
-/* Copyright (c) 2008-2016, The Tor Project, Inc. */
+/* Copyright (c) 2008-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
-/* Tor dependencies */
+
+/**
+ * \file memarea.h
+ *
+ * \brief Header for memarea.c
+ **/
 
 #ifndef TOR_MEMAREA_H
 #define TOR_MEMAREA_H
 
+#include <stddef.h>
+
 typedef struct memarea_t memarea_t;
 
 memarea_t *memarea_new(void);
-void memarea_drop_all(memarea_t *area);
+void memarea_drop_all_(memarea_t *area);
+#define memarea_drop_all(area) \
+  do {                                          \
+    memarea_drop_all_(area);                    \
+    (area) = NULL;                              \
+  } while (0)
 void memarea_clear(memarea_t *area);
 int memarea_owns_ptr(const memarea_t *area, const void *ptr);
 void *memarea_alloc(memarea_t *area, size_t sz);
@@ -20,5 +32,4 @@ void memarea_get_stats(memarea_t *area,
                        size_t *allocated_out, size_t *used_out);
 void memarea_assert_ok(memarea_t *area);
 
-#endif
-
+#endif /* !defined(TOR_MEMAREA_H) */
diff --git a/src/lib/meminfo/.may_include b/src/lib/meminfo/.may_include
new file mode 100644
index 0000000000..9e4d25fd6a
--- /dev/null
+++ b/src/lib/meminfo/.may_include
@@ -0,0 +1,8 @@
+orconfig.h
+
+lib/cc/*.h
+lib/fs/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/meminfo/*.h
+lib/testsupport/*.h
diff --git a/src/lib/meminfo/include.am b/src/lib/meminfo/include.am
new file mode 100644
index 0000000000..d1fdde6313
--- /dev/null
+++ b/src/lib/meminfo/include.am
@@ -0,0 +1,17 @@
+
+noinst_LIBRARIES += src/lib/libtor-meminfo.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-meminfo-testing.a
+endif
+
+src_lib_libtor_meminfo_a_SOURCES =			\
+	src/lib/meminfo/meminfo.c
+
+src_lib_libtor_meminfo_testing_a_SOURCES = \
+	$(src_lib_libtor_meminfo_a_SOURCES)
+src_lib_libtor_meminfo_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_meminfo_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/meminfo/meminfo.h
diff --git a/src/lib/meminfo/meminfo.c b/src/lib/meminfo/meminfo.c
new file mode 100644
index 0000000000..f4fa45167e
--- /dev/null
+++ b/src/lib/meminfo/meminfo.c
@@ -0,0 +1,181 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file meminfo.c
+ *
+ * \brief Functions to query total memory, and access meta-information about
+ * the allocator.
+ **/
+
+#include "lib/meminfo/meminfo.h"
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+#include "lib/fs/files.h"
+#include "lib/log/log.h"
+#include "lib/malloc/malloc.h"
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_MALLOC_H
+#include <malloc.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifdef _WIN32
+#include <windows.h>
+#endif
+#include <string.h>
+
+#if defined(HAVE_SYS_SYSCTL_H) && !defined(_WIN32) && !defined(__linux__)
+#include <sys/sysctl.h>
+#endif
+
+DISABLE_GCC_WARNING(aggregate-return)
+/** Call the platform malloc info function, and dump the results to the log at
+ * level <b>severity</b>.  If no such function exists, do nothing. */
+void
+tor_log_mallinfo(int severity)
+{
+#ifdef HAVE_MALLINFO
+  struct mallinfo mi;
+  memset(&mi, 0, sizeof(mi));
+  mi = mallinfo();
+  tor_log(severity, LD_MM,
+      "mallinfo() said: arena=%d, ordblks=%d, smblks=%d, hblks=%d, "
+      "hblkhd=%d, usmblks=%d, fsmblks=%d, uordblks=%d, fordblks=%d, "
+      "keepcost=%d",
+      mi.arena, mi.ordblks, mi.smblks, mi.hblks,
+      mi.hblkhd, mi.usmblks, mi.fsmblks, mi.uordblks, mi.fordblks,
+      mi.keepcost);
+#else /* !(defined(HAVE_MALLINFO)) */
+  (void)severity;
+#endif /* defined(HAVE_MALLINFO) */
+}
+ENABLE_GCC_WARNING(aggregate-return)
+
+#if defined(HW_PHYSMEM64)
+/* OpenBSD and NetBSD define this */
+#define INT64_HW_MEM HW_PHYSMEM64
+#elif defined(HW_MEMSIZE)
+/* OSX defines this one */
+#define INT64_HW_MEM HW_MEMSIZE
+#endif /* defined(HW_PHYSMEM64) || ... */
+
+/**
+ * Helper: try to detect the total system memory, and return it. On failure,
+ * return 0.
+ */
+static uint64_t
+get_total_system_memory_impl(void)
+{
+#if defined(__linux__)
+  /* On linux, sysctl is deprecated. Because proc is so awesome that you
+   * shouldn't _want_ to write portable code, I guess? */
+  unsigned long long result=0;
+  int fd = -1;
+  char *s = NULL;
+  const char *cp;
+  size_t file_size=0;
+  if (-1 == (fd = tor_open_cloexec("/proc/meminfo",O_RDONLY,0)))
+    return 0;
+  s = read_file_to_str_until_eof(fd, 65536, &file_size);
+  if (!s)
+    goto err;
+  cp = strstr(s, "MemTotal:");
+  if (!cp)
+    goto err;
+  /* Use the system sscanf so that space will match a wider number of space */
+  if (sscanf(cp, "MemTotal: %llu kB\n", &result) != 1)
+    goto err;
+
+  close(fd);
+  tor_free(s);
+  return result * 1024;
+
+  /* LCOV_EXCL_START Can't reach this unless proc is broken. */
+ err:
+  tor_free(s);
+  close(fd);
+  return 0;
+  /* LCOV_EXCL_STOP */
+#elif defined (_WIN32)
+  /* Windows has MEMORYSTATUSEX; pretty straightforward. */
+  MEMORYSTATUSEX ms;
+  memset(&ms, 0, sizeof(ms));
+  ms.dwLength = sizeof(ms);
+  if (! GlobalMemoryStatusEx(&ms))
+    return 0;
+
+  return ms.ullTotalPhys;
+
+#elif defined(HAVE_SYSCTL) && defined(INT64_HW_MEM)
+  /* On many systems, HW_PHYSMEM is clipped to 32 bits; let's use a better
+   * variant if we know about it. */
+  uint64_t memsize = 0;
+  size_t len = sizeof(memsize);
+  int mib[2] = {CTL_HW, INT64_HW_MEM};
+  if (sysctl(mib,2,&memsize,&len,NULL,0))
+    return 0;
+
+  return memsize;
+
+#elif defined(HAVE_SYSCTL) && defined(HW_PHYSMEM)
+  /* On some systems (like FreeBSD I hope) you can use a size_t with
+   * HW_PHYSMEM. */
+  size_t memsize=0;
+  size_t len = sizeof(memsize);
+  int mib[2] = {CTL_HW, HW_PHYSMEM};
+  if (sysctl(mib,2,&memsize,&len,NULL,0))
+    return 0;
+
+  return memsize;
+
+#else
+  /* I have no clue. */
+  return 0;
+#endif /* defined(__linux__) || ... */
+}
+
+/**
+ * Try to find out how much physical memory the system has. On success,
+ * return 0 and set *<b>mem_out</b> to that value. On failure, return -1.
+ */
+MOCK_IMPL(int,
+get_total_system_memory, (size_t *mem_out))
+{
+  static size_t mem_cached=0;
+  uint64_t m = get_total_system_memory_impl();
+  if (0 == m) {
+    /* LCOV_EXCL_START -- can't make this happen without mocking. */
+    /* We couldn't find our memory total */
+    if (0 == mem_cached) {
+      /* We have no cached value either */
+      *mem_out = 0;
+      return -1;
+    }
+
+    *mem_out = mem_cached;
+    return 0;
+    /* LCOV_EXCL_STOP */
+  }
+
+#if SIZE_MAX != UINT64_MAX
+  if (m > SIZE_MAX) {
+    /* I think this could happen if we're a 32-bit Tor running on a 64-bit
+     * system: we could have more system memory than would fit in a
+     * size_t. */
+    m = SIZE_MAX;
+  }
+#endif /* SIZE_MAX != UINT64_MAX */
+
+  *mem_out = mem_cached = (size_t) m;
+
+  return 0;
+}
diff --git a/src/lib/meminfo/meminfo.h b/src/lib/meminfo/meminfo.h
new file mode 100644
index 0000000000..2d64e1ab06
--- /dev/null
+++ b/src/lib/meminfo/meminfo.h
@@ -0,0 +1,21 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file meminfo.h
+ *
+ * \brief Header for meminfo.c
+ **/
+
+#ifndef TOR_MEMINFO_H
+#define TOR_MEMINFO_H
+
+#include "lib/testsupport/testsupport.h"
+#include <stddef.h>
+
+void tor_log_mallinfo(int severity);
+MOCK_DECL(int, get_total_system_memory, (size_t *mem_out));
+
+#endif
diff --git a/src/lib/net/.may_include b/src/lib/net/.may_include
new file mode 100644
index 0000000000..13b209bbed
--- /dev/null
+++ b/src/lib/net/.may_include
@@ -0,0 +1,15 @@
+orconfig.h
+siphash.h
+ht.h
+
+lib/arch/*.h
+lib/cc/*.h
+lib/container/*.h
+lib/ctime/*.h
+lib/err/*.h
+lib/lock/*.h
+lib/log/*.h
+lib/net/*.h
+lib/string/*.h
+lib/testsupport/*.h
+lib/malloc/*.h
\ No newline at end of file
diff --git a/src/common/address.c b/src/lib/net/address.c
index 71d3805386..a2d234b742 100644
--- a/src/common/address.c
+++ b/src/lib/net/address.c
@@ -1,11 +1,14 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file address.c
  * \brief Functions to use and manipulate the tor_addr_t structure.
+ *
+ * This module doesn't have any support for the libc resolver: that is all in
+ * resolve.c.
  **/
 
 #define ADDRESS_PRIVATE
@@ -33,15 +36,23 @@
 #include <process.h>
 #include <windows.h>
 #include <iphlpapi.h>
-#endif
-
-#include "compat.h"
-#include "util.h"
-#include "util_format.h"
-#include "address.h"
-#include "torlog.h"
-#include "container.h"
-#include "sandbox.h"
+#endif /* defined(_WIN32) */
+
+#include "lib/net/address.h"
+#include "lib/net/socket.h"
+#include "lib/container/smartlist.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/log/log.h"
+#include "lib/log/escape.h"
+#include "lib/malloc/malloc.h"
+#include "lib/net/inaddr.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/string/compat_string.h"
+#include "lib/string/parse_int.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+
+#include "siphash.h"
 
 #ifdef HAVE_SYS_TIME_H
 #include <sys/time.h>
@@ -52,9 +63,6 @@
 #ifdef HAVE_ERRNO_H
 #include <errno.h>
 #endif
-#ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
-#endif
 #ifdef HAVE_ARPA_INET_H
 #include <arpa/inet.h>
 #endif
@@ -83,7 +91,6 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <assert.h>
 
 /* tor_addr_is_null() and maybe other functions rely on AF_UNSPEC being 0 to
  * work correctly. Bail out here if we've found a platform where AF_UNSPEC
@@ -159,6 +166,8 @@ tor_addr_from_sockaddr(tor_addr_t *a, const struct sockaddr *sa,
   tor_assert(a);
   tor_assert(sa);
 
+  /* This memset is redundant; leaving it in to avoid any future accidents,
+     however. */
   memset(a, 0, sizeof(*a));
 
   if (sa->sa_family == AF_INET) {
@@ -196,7 +205,7 @@ tor_sockaddr_to_str(const struct sockaddr *sa)
     tor_asprintf(&result, "unix:%s", s_un->sun_path);
     return result;
   }
-#endif
+#endif /* defined(HAVE_SYS_UN_H) */
   if (sa->sa_family == AF_UNSPEC)
     return tor_strdup("unspec");
 
@@ -227,127 +236,6 @@ tor_addr_make_null(tor_addr_t *a, sa_family_t family)
   a->family = family;
 }
 
-/** Similar behavior to Unix gethostbyname: resolve <b>name</b>, and set
- * *<b>addr</b> to the proper IP address and family. The <b>family</b>
- * argument (which must be AF_INET, AF_INET6, or AF_UNSPEC) declares a
- * <i>preferred</i> family, though another one may be returned if only one
- * family is implemented for this address.
- *
- * Return 0 on success, -1 on failure; 1 on transient failure.
- */
-int
-tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr)
-{
-  /* Perhaps eventually this should be replaced by a tor_getaddrinfo or
-   * something.
-   */
-  struct in_addr iaddr;
-  struct in6_addr iaddr6;
-  tor_assert(name);
-  tor_assert(addr);
-  tor_assert(family == AF_INET || family == AF_INET6 || family == AF_UNSPEC);
-  if (!*name) {
-    /* Empty address is an error. */
-    return -1;
-  } else if (tor_inet_pton(AF_INET, name, &iaddr)) {
-    /* It's an IPv4 IP. */
-    if (family == AF_INET6)
-      return -1;
-    tor_addr_from_in(addr, &iaddr);
-    return 0;
-  } else if (tor_inet_pton(AF_INET6, name, &iaddr6)) {
-    if (family == AF_INET)
-      return -1;
-    tor_addr_from_in6(addr, &iaddr6);
-    return 0;
-  } else {
-#ifdef HAVE_GETADDRINFO
-    int err;
-    struct addrinfo *res=NULL, *res_p;
-    struct addrinfo *best=NULL;
-    struct addrinfo hints;
-    int result = -1;
-    memset(&hints, 0, sizeof(hints));
-    hints.ai_family = family;
-    hints.ai_socktype = SOCK_STREAM;
-    err = sandbox_getaddrinfo(name, NULL, &hints, &res);
-    /* The check for 'res' here shouldn't be necessary, but it makes static
-     * analysis tools happy. */
-    if (!err && res) {
-      best = NULL;
-      for (res_p = res; res_p; res_p = res_p->ai_next) {
-        if (family == AF_UNSPEC) {
-          if (res_p->ai_family == AF_INET) {
-            best = res_p;
-            break;
-          } else if (res_p->ai_family == AF_INET6 && !best) {
-            best = res_p;
-          }
-        } else if (family == res_p->ai_family) {
-          best = res_p;
-          break;
-        }
-      }
-      if (!best)
-        best = res;
-      if (best->ai_family == AF_INET) {
-        tor_addr_from_in(addr,
-                         &((struct sockaddr_in*)best->ai_addr)->sin_addr);
-        result = 0;
-      } else if (best->ai_family == AF_INET6) {
-        tor_addr_from_in6(addr,
-                          &((struct sockaddr_in6*)best->ai_addr)->sin6_addr);
-        result = 0;
-      }
-      sandbox_freeaddrinfo(res);
-      return result;
-    }
-    return (err == EAI_AGAIN) ? 1 : -1;
-#else
-    struct hostent *ent;
-    int err;
-#ifdef HAVE_GETHOSTBYNAME_R_6_ARG
-    char buf[2048];
-    struct hostent hostent;
-    int r;
-    r = gethostbyname_r(name, &hostent, buf, sizeof(buf), &ent, &err);
-#elif defined(HAVE_GETHOSTBYNAME_R_5_ARG)
-    char buf[2048];
-    struct hostent hostent;
-    ent = gethostbyname_r(name, &hostent, buf, sizeof(buf), &err);
-#elif defined(HAVE_GETHOSTBYNAME_R_3_ARG)
-    struct hostent_data data;
-    struct hostent hent;
-    memset(&data, 0, sizeof(data));
-    err = gethostbyname_r(name, &hent, &data);
-    ent = err ? NULL : &hent;
-#else
-    ent = gethostbyname(name);
-#ifdef _WIN32
-    err = WSAGetLastError();
-#else
-    err = h_errno;
-#endif
-#endif /* endif HAVE_GETHOSTBYNAME_R_6_ARG. */
-    if (ent) {
-      if (ent->h_addrtype == AF_INET) {
-        tor_addr_from_in(addr, (struct in_addr*) ent->h_addr);
-      } else if (ent->h_addrtype == AF_INET6) {
-        tor_addr_from_in6(addr, (struct in6_addr*) ent->h_addr);
-      } else {
-        tor_assert(0); // LCOV_EXCL_LINE: gethostbyname() returned bizarre type
-      }
-      return 0;
-    }
-#ifdef _WIN32
-    return (err == WSATRY_AGAIN) ? 1 : -1;
-#else
-    return (err == TRY_AGAIN) ? 1 : -1;
-#endif
-#endif
-  }
-}
-
 /** Return true iff <b>ip</b> is an IP reserved to localhost or local networks.
  *
  * If <b>ip</b> is in RFC1918 or RFC4193 or RFC4291, we will return true.
@@ -573,8 +461,8 @@ tor_addr_parse_PTR_name(tor_addr_t *result, const char *address,
 
 /** Convert <b>addr</b> to an in-addr.arpa name or a .ip6.arpa name,
  * and store the result in the <b>outlen</b>-byte buffer at
- * <b>out</b>.  Return the number of chars written to <b>out</b>, not
- * including the trailing \0, on success. Returns -1 on failure. */
+ * <b>out</b>.  Returns a non-negative integer on success.
+ * Returns -1 on failure. */
 int
 tor_addr_to_PTR_name(char *out, size_t outlen,
                      const tor_addr_t *addr)
@@ -916,8 +804,8 @@ tor_addr_is_loopback(const tor_addr_t *addr)
       return (tor_addr_to_ipv4h(addr) & 0xff000000) == 0x7f000000;
     case AF_UNSPEC:
       return 0;
-    default:
       /* LCOV_EXCL_START */
+    default:
       tor_fragile_assert();
       return 0;
       /* LCOV_EXCL_STOP */
@@ -1040,8 +928,10 @@ tor_addr_copy_tight(tor_addr_t *dest, const tor_addr_t *src)
       memcpy(dest->addr.in6_addr.s6_addr, src->addr.in6_addr.s6_addr, 16);
     case AF_UNSPEC:
       break;
+      // LCOV_EXCL_START
     default:
-      tor_fragile_assert(); // LCOV_EXCL_LINE
+      tor_fragile_assert();
+      // LCOV_EXCL_STOP
     }
 }
 
@@ -1132,7 +1022,7 @@ tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2,
       case AF_UNIX:
         /* HACKHACKHACKHACKHACK:
          * tor_addr_t doesn't contain a copy of sun_path, so it's not
-         * possible to comapre this at all.
+         * possible to compare this at all.
          *
          * Since the only time we currently actually should be comparing
          * 2 AF_UNIX addresses is when dealing with ISO_CLIENTADDR (which
@@ -1147,8 +1037,8 @@ tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2,
           return 0;
         else
           return 1;
-      default:
         /* LCOV_EXCL_START */
+      default:
         tor_fragile_assert();
         return 0;
         /* LCOV_EXCL_STOP */
@@ -1206,8 +1096,8 @@ tor_addr_hash(const tor_addr_t *addr)
     return siphash24g(unspec_hash_input, sizeof(unspec_hash_input));
   case AF_INET6:
     return siphash24g(&addr->addr.in6_addr.s6_addr, 16);
-  default:
     /* LCOV_EXCL_START */
+  default:
     tor_fragile_assert();
     return 0;
     /* LCOV_EXCL_STOP */
@@ -1326,69 +1216,6 @@ tor_addr_parse(tor_addr_t *addr, const char *src)
   return result;
 }
 
-/** Parse an address or address-port combination from <b>s</b>, resolve the
- * address as needed, and put the result in <b>addr_out</b> and (optionally)
- * <b>port_out</b>.  Return 0 on success, negative on failure. */
-int
-tor_addr_port_lookup(const char *s, tor_addr_t *addr_out, uint16_t *port_out)
-{
-  const char *port;
-  tor_addr_t addr;
-  uint16_t portval;
-  char *tmp = NULL;
-
-  tor_assert(s);
-  tor_assert(addr_out);
-
-  s = eat_whitespace(s);
-
-  if (*s == '[') {
-    port = strstr(s, "]");
-    if (!port)
-      goto err;
-    tmp = tor_strndup(s+1, port-(s+1));
-    port = port+1;
-    if (*port == ':')
-      port++;
-    else
-      port = NULL;
-  } else {
-    port = strchr(s, ':');
-    if (port)
-      tmp = tor_strndup(s, port-s);
-    else
-      tmp = tor_strdup(s);
-    if (port)
-      ++port;
-  }
-
-  if (tor_addr_lookup(tmp, AF_UNSPEC, &addr) != 0)
-    goto err;
-  tor_free(tmp);
-
-  if (port) {
-    portval = (int) tor_parse_long(port, 10, 1, 65535, NULL, NULL);
-    if (!portval)
-      goto err;
-  } else {
-    portval = 0;
-  }
-
-  if (port_out)
-    *port_out = portval;
-  tor_addr_copy(addr_out, &addr);
-
-  return 0;
- err:
-  tor_free(tmp);
-  return -1;
-}
-
-#ifdef _WIN32
-typedef ULONG (WINAPI *GetAdaptersAddresses_fn_t)(
-              ULONG, ULONG, PVOID, PIP_ADAPTER_ADDRESSES, PULONG);
-#endif
-
 #ifdef HAVE_IFADDRS_TO_SMARTLIST
 /*
  * Convert a linked list consisting of <b>ifaddrs</b> structures
@@ -1443,7 +1270,7 @@ get_interface_addresses_ifaddrs(int severity, sa_family_t family)
 
   return result;
 }
-#endif
+#endif /* defined(HAVE_IFADDRS_TO_SMARTLIST) */
 
 #ifdef HAVE_IP_ADAPTER_TO_SMARTLIST
 
@@ -1474,21 +1301,14 @@ ip_adapter_addresses_to_smartlist(const IP_ADAPTER_ADDRESSES *addresses)
   return result;
 }
 
-/** Windows only: use GetAdaptersInfo() function to retrieve network interface
- * addresses of current machine and return them to caller as smartlist of
- * <b>tor_addr_t</b>  structures.
+/** Windows only: use GetAdaptersAddresses() to retrieve the network interface
+ * addresses of the current machine.
+ * Returns a smartlist of <b>tor_addr_t</b>  structures.
  */
 STATIC smartlist_t *
 get_interface_addresses_win32(int severity, sa_family_t family)
 {
-
-  /* Windows XP began to provide GetAdaptersAddresses. Windows 2000 had a
-     "GetAdaptersInfo", but that's deprecated; let's just try
-     GetAdaptersAddresses and fall back to connect+getsockname.
-  */
-  HANDLE lib = load_windows_system_library(TEXT("iphlpapi.dll"));
   smartlist_t *result = NULL;
-  GetAdaptersAddresses_fn_t fn;
   ULONG size, res;
   IP_ADAPTER_ADDRESSES *addresses = NULL;
 
@@ -1498,30 +1318,16 @@ get_interface_addresses_win32(int severity, sa_family_t family)
                GAA_FLAG_SKIP_MULTICAST | \
                GAA_FLAG_SKIP_DNS_SERVER)
 
-  if (!lib) {
-    log_fn(severity, LD_NET, "Unable to load iphlpapi.dll");
-    goto done;
-  }
-
-  /* Cast through a void function pointer, to silence a spurious compiler
-   * warning on 64-bit Windows. This cast is safe, because we are casting to
-   * the correct type for GetAdaptersAddresses(). */
-  if (!(fn = (GetAdaptersAddresses_fn_t)(void(*)(void))
-                  GetProcAddress(lib, "GetAdaptersAddresses"))) {
-    log_fn(severity, LD_NET, "Unable to obtain pointer to "
-           "GetAdaptersAddresses");
-    goto done;
-  }
-
   /* Guess how much space we need. */
   size = 15*1024;
   addresses = tor_malloc(size);
-  res = fn(family, FLAGS, NULL, addresses, &size);
+  /* Exists in windows XP and later. */
+  res = GetAdaptersAddresses(family, FLAGS, NULL, addresses, &size);
   if (res == ERROR_BUFFER_OVERFLOW) {
     /* we didn't guess that we needed enough space; try again */
     tor_free(addresses);
     addresses = tor_malloc(size);
-    res = fn(AF_UNSPEC, FLAGS, NULL, addresses, &size);
+    res = GetAdaptersAddresses(AF_UNSPEC, FLAGS, NULL, addresses, &size);
   }
   if (res != NO_ERROR) {
     log_fn(severity, LD_NET, "GetAdaptersAddresses failed (result: %lu)", res);
@@ -1531,13 +1337,11 @@ get_interface_addresses_win32(int severity, sa_family_t family)
   result = ip_adapter_addresses_to_smartlist(addresses);
 
  done:
-  if (lib)
-    FreeLibrary(lib);
   tor_free(addresses);
   return result;
 }
 
-#endif
+#endif /* defined(HAVE_IP_ADAPTER_TO_SMARTLIST) */
 
 #ifdef HAVE_IFCONF_TO_SMARTLIST
 
@@ -1550,6 +1354,18 @@ get_interface_addresses_win32(int severity, sa_family_t family)
 #define _SIZEOF_ADDR_IFREQ sizeof
 #endif
 
+/* Free ifc->ifc_buf safely. */
+static void
+ifconf_free_ifc_buf(struct ifconf *ifc)
+{
+  /* On macOS, tor_free() takes the address of ifc.ifc_buf, which leads to
+   * undefined behaviour, because pointer-to-pointers are expected to be
+   * aligned at 8-bytes, but the ifconf structure is packed.  So we use
+   * raw_free() instead. */
+  raw_free(ifc->ifc_buf);
+  ifc->ifc_buf = NULL;
+}
+
 /** Convert <b>*buf</b>, an ifreq structure array of size <b>buflen</b>,
  * into smartlist of <b>tor_addr_t</b> structures.
  */
@@ -1636,10 +1452,10 @@ get_interface_addresses_ioctl(int severity, sa_family_t family)
  done:
   if (fd >= 0)
     close(fd);
-  tor_free(ifc.ifc_buf);
+  ifconf_free_ifc_buf(&ifc);
   return result;
 }
-#endif
+#endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */
 
 /** Try to ask our network interfaces what addresses they are bound to.
  * Return a new smartlist of tor_addr_t on success, and NULL on failure.
@@ -1695,7 +1511,7 @@ get_interface_address6_via_udp_socket_hack,(int severity,
                                             sa_family_t family,
                                             tor_addr_t *addr))
 {
-  struct sockaddr_storage my_addr, target_addr;
+  struct sockaddr_storage target_addr;
   int sock=-1, r=-1;
   socklen_t addr_len;
 
@@ -1738,21 +1554,19 @@ get_interface_address6_via_udp_socket_hack,(int severity,
     goto err;
   }
 
-  if (tor_getsockname(sock,(struct sockaddr*)&my_addr, &addr_len)) {
+  if (tor_addr_from_getsockname(addr, sock) < 0) {
     int e = tor_socket_errno(sock);
     log_fn(severity, LD_NET, "getsockname() to determine interface failed: %s",
            tor_socket_strerror(e));
     goto err;
   }
 
- if (tor_addr_from_sockaddr(addr, (struct sockaddr*)&my_addr, NULL) == 0) {
-    if (tor_addr_is_loopback(addr) || tor_addr_is_multicast(addr)) {
-      log_fn(severity, LD_NET, "Address that we determined via UDP socket"
-                               " magic is unsuitable for public comms.");
-    } else {
-      r=0;
-    }
- }
+  if (tor_addr_is_loopback(addr) || tor_addr_is_multicast(addr)) {
+    log_fn(severity, LD_NET, "Address that we determined via UDP socket"
+           " magic is unsuitable for public comms.");
+  } else {
+    r=0;
+  }
 
  err:
   if (sock >= 0)
@@ -1794,14 +1608,14 @@ get_interface_address6,(int severity, sa_family_t family, tor_addr_t *addr))
       break;
   } SMARTLIST_FOREACH_END(a);
 
-  free_interface_address6_list(addrs);
+  interface_address6_list_free(addrs);
   return rv;
 }
 
 /** Free a smartlist of IP addresses returned by get_interface_address6_list.
  */
 void
-free_interface_address6_list(smartlist_t *addrs)
+interface_address6_list_free_(smartlist_t *addrs)
 {
   if (addrs != NULL) {
     SMARTLIST_FOREACH(addrs, tor_addr_t *, a, tor_free(a));
@@ -1816,11 +1630,12 @@ free_interface_address6_list(smartlist_t *addrs)
  * An empty smartlist means that there are no addresses of the selected type
  * matching these criteria.
  * Returns NULL on failure.
- * Use free_interface_address6_list to free the returned list.
+ * Use interface_address6_list_free to free the returned list.
  */
-MOCK_IMPL(smartlist_t *,get_interface_address6_list,(int severity,
-                                                     sa_family_t family,
-                                                     int include_internal))
+MOCK_IMPL(smartlist_t *,
+get_interface_address6_list,(int severity,
+                             sa_family_t family,
+                             int include_internal))
 {
   smartlist_t *addrs;
   tor_addr_t addr;
@@ -1941,7 +1756,7 @@ tor_addr_port_split(int severity, const char *addrport,
   tor_assert(addrport);
   tor_assert(address_out);
   tor_assert(port_out);
-  /* We need to check for IPv6 manually because addr_port_lookup() doesn't
+  /* We need to check for IPv6 manually because the logic below doesn't
    * do a good job on IPv6 addresses that lack a port. */
   if (tor_addr_parse(&a_tmp, addrport) == AF_INET6) {
     *port_out = 0;
@@ -1949,30 +1764,11 @@ tor_addr_port_split(int severity, const char *addrport,
     return 0;
   }
 
-  return addr_port_lookup(severity, addrport, address_out, NULL, port_out);
-}
-
-/** Parse a string of the form "host[:port]" from <b>addrport</b>.  If
- * <b>address</b> is provided, set *<b>address</b> to a copy of the
- * host portion of the string.  If <b>addr</b> is provided, try to
- * resolve the host portion of the string and store it into
- * *<b>addr</b> (in host byte order).  If <b>port_out</b> is provided,
- * store the port number into *<b>port_out</b>, or 0 if no port is given.
- * If <b>port_out</b> is NULL, then there must be no port number in
- * <b>addrport</b>.
- * Return 0 on success, -1 on failure.
- */
-int
-addr_port_lookup(int severity, const char *addrport, char **address,
-                uint32_t *addr, uint16_t *port_out)
-{
   const char *colon;
   char *address_ = NULL;
   int port_;
   int ok = 1;
 
-  tor_assert(addrport);
-
   colon = strrchr(addrport, ':');
   if (colon) {
     address_ = tor_strndup(addrport, colon-addrport);
@@ -1994,22 +1790,13 @@ addr_port_lookup(int severity, const char *addrport, char **address,
     port_ = 0;
   }
 
-  if (addr) {
-    /* There's an addr pointer, so we need to resolve the hostname. */
-    if (tor_lookup_hostname(address_,addr)) {
-      log_fn(severity, LD_NET, "Couldn't look up %s", escaped(address_));
-      ok = 0;
-      *addr = 0;
-    }
-  }
-
-  if (address && ok) {
-    *address = address_;
+  if (ok) {
+    *address_out = address_;
   } else {
-    if (address)
-      *address = NULL;
+    *address_out = NULL;
     tor_free(address_);
   }
+
   if (port_out)
     *port_out = ok ? ((uint16_t) port_) : 0;
 
@@ -2086,21 +1873,6 @@ parse_port_range(const char *port, uint16_t *port_min_out,
   return 0;
 }
 
-/** Given an IPv4 in_addr struct *<b>in</b> (in network order, as usual),
- *  write it as a string into the <b>buf_len</b>-byte buffer in
- *  <b>buf</b>.
- */
-int
-tor_inet_ntoa(const struct in_addr *in, char *buf, size_t buf_len)
-{
-  uint32_t a = ntohl(in->s_addr);
-  return tor_snprintf(buf, buf_len, "%d.%d.%d.%d",
-                      (int)(uint8_t)((a>>24)&0xff),
-                      (int)(uint8_t)((a>>16)&0xff),
-                      (int)(uint8_t)((a>>8 )&0xff),
-                      (int)(uint8_t)((a    )&0xff));
-}
-
 /** Given a host-order <b>addr</b>, call tor_inet_ntop() on it
  *  and return a strdup of the resulting address.
  */
@@ -2139,7 +1911,8 @@ get_interface_address,(int severity, uint32_t *addr))
 }
 
 /** Return true if we can tell that <b>name</b> is a canonical name for the
- * loopback address. */
+ * loopback address.  Return true also for *.local hostnames, which are
+ * multicast DNS names for hosts on the local network. */
 int
 tor_addr_hostname_is_local(const char *name)
 {
@@ -2160,3 +1933,125 @@ tor_addr_port_new(const tor_addr_t *addr, uint16_t port)
   return ap;
 }
 
+/** Return true iff <a>a</b> and <b>b</b> are the same address and port */
+int
+tor_addr_port_eq(const tor_addr_port_t *a,
+                 const tor_addr_port_t *b)
+{
+  return tor_addr_eq(&a->addr, &b->addr) && a->port == b->port;
+}
+
+/** Return true if <b>string</b> represents a valid IPv4 adddress in
+ * 'a.b.c.d' form.
+ */
+int
+string_is_valid_ipv4_address(const char *string)
+{
+  struct in_addr addr;
+
+  return (tor_inet_pton(AF_INET,string,&addr) == 1);
+}
+
+/** Return true if <b>string</b> represents a valid IPv6 address in
+ * a form that inet_pton() can parse.
+ */
+int
+string_is_valid_ipv6_address(const char *string)
+{
+  struct in6_addr addr;
+
+  return (tor_inet_pton(AF_INET6,string,&addr) == 1);
+}
+
+/** Return true iff <b>string</b> is a valid destination address,
+ * i.e. either a DNS hostname or IPv4/IPv6 address string.
+ */
+int
+string_is_valid_dest(const char *string)
+{
+  char *tmp = NULL;
+  int retval;
+  size_t len;
+
+  if (string == NULL)
+    return 0;
+
+  len = strlen(string);
+
+  if (len == 0)
+    return 0;
+
+  if (string[0] == '[' && string[len - 1] == ']')
+    string = tmp = tor_strndup(string + 1, len - 2);
+
+  retval = string_is_valid_ipv4_address(string) ||
+    string_is_valid_ipv6_address(string) ||
+    string_is_valid_nonrfc_hostname(string);
+
+  tor_free(tmp);
+
+  return retval;
+}
+
+/** Return true iff <b>string</b> matches a pattern of DNS names
+ * that we allow Tor clients to connect to.
+ *
+ * Note: This allows certain technically invalid characters ('_') to cope
+ * with misconfigured zones that have been encountered in the wild.
+ */
+int
+string_is_valid_nonrfc_hostname(const char *string)
+{
+  int result = 1;
+  int has_trailing_dot;
+  char *last_label;
+  smartlist_t *components;
+
+  if (!string || strlen(string) == 0)
+    return 0;
+
+  if (string_is_valid_ipv4_address(string))
+    return 0;
+
+  components = smartlist_new();
+
+  smartlist_split_string(components,string,".",0,0);
+
+  if (BUG(smartlist_len(components) == 0))
+    return 0; // LCOV_EXCL_LINE should be impossible given the earlier checks.
+
+  /* Allow a single terminating '.' used rarely to indicate domains
+   * are FQDNs rather than relative. */
+  last_label = (char *)smartlist_get(components,
+                                     smartlist_len(components) - 1);
+  has_trailing_dot = (last_label[0] == '\0');
+  if (has_trailing_dot) {
+    smartlist_pop_last(components);
+    tor_free(last_label);
+    last_label = NULL;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(components, char *, c) {
+    if ((c[0] == '-') || (*c == '_')) {
+      result = 0;
+      break;
+    }
+
+    do {
+      result = (TOR_ISALNUM(*c) || (*c == '-') || (*c == '_'));
+      c++;
+    } while (result && *c);
+
+    if (result == 0) {
+      break;
+    }
+  } SMARTLIST_FOREACH_END(c);
+
+  SMARTLIST_FOREACH_BEGIN(components, char *, c) {
+    tor_free(c);
+  } SMARTLIST_FOREACH_END(c);
+
+  smartlist_free(components);
+
+  return result;
+}
diff --git a/src/common/address.h b/src/lib/net/address.h
index d57abd0d9e..9b826c8359 100644
--- a/src/common/address.h
+++ b/src/lib/net/address.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,11 +11,22 @@
 #ifndef TOR_ADDRESS_H
 #define TOR_ADDRESS_H
 
-//#include <sys/sockio.h>
 #include "orconfig.h"
-#include "torint.h"
-#include "compat.h"
-#include "container.h"
+#include "lib/cc/torint.h"
+#include "lib/log/util_bug.h"
+#include "lib/net/inaddr_st.h"
+#include "lib/net/nettypes.h"
+
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+#ifdef _WIN32
+#include <winsock2.h>
+#include <windows.h>
+#endif
+
+#include <stddef.h>
+#include <stdlib.h>
 
 #ifdef ADDRESS_PRIVATE
 
@@ -44,7 +55,7 @@
 #endif
 
 // TODO win32 specific includes
-#endif // ADDRESS_PRIVATE
+#endif /* defined(ADDRESS_PRIVATE) */
 
 /** The number of bits from an address to consider while doing a masked
  * comparison. */
@@ -73,6 +84,9 @@ typedef struct tor_addr_port_t
 
 #define TOR_ADDR_NULL {AF_UNSPEC, {0}}
 
+/* XXXX To do: extract all of the functions here that can possibly invoke
+ * XXXX resolver, and make sure they have distinctive names. */
+
 static inline const struct in6_addr *tor_addr_to_in6(const tor_addr_t *a);
 static inline const struct in6_addr *tor_addr_to_in6_assert(
     const tor_addr_t *a);
@@ -190,7 +204,6 @@ tor_addr_eq_ipv4h(const tor_addr_t *a, uint32_t u)
  */
 #define TOR_ADDR_BUF_LEN 48
 
-int tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr_out);
 char *tor_addr_to_str_dup(const tor_addr_t *addr) ATTR_MALLOC;
 
 /** Wrapper function of fmt_addr_impl(). It does not decorate IPv6
@@ -205,8 +218,11 @@ const char * fmt_addr32(uint32_t addr);
 
 MOCK_DECL(int,get_interface_address6,(int severity, sa_family_t family,
 tor_addr_t *addr));
-void free_interface_address6_list(smartlist_t * addrs);
-MOCK_DECL(smartlist_t *,get_interface_address6_list,(int severity,
+struct smartlist_t;
+void interface_address6_list_free_(struct smartlist_t * addrs);// XXXX
+#define interface_address6_list_free(addrs) \
+  FREE_AND_NULL(struct smartlist_t, interface_address6_list_free_, (addrs))
+MOCK_DECL(struct smartlist_t *,get_interface_address6_list,(int severity,
                                                      sa_family_t family,
                                                      int include_internal));
 
@@ -245,9 +261,6 @@ int tor_addr_to_PTR_name(char *out, size_t outlen,
 int tor_addr_parse_PTR_name(tor_addr_t *result, const char *address,
                                        int family, int accept_regular);
 
-int tor_addr_port_lookup(const char *s, tor_addr_t *addr_out,
-                        uint16_t *port_out);
-
 /* Does the address * yield an AF_UNSPEC wildcard address (1),
  * which expands to corresponding wildcard IPv4 and IPv6 rules, and do we
  * allow *4 and *6 for IPv4 and IPv6 wildcards, respectively;
@@ -312,23 +325,13 @@ int tor_addr_port_parse(int severity, const char *addrport,
 int tor_addr_hostname_is_local(const char *name);
 
 /* IPv4 helpers */
-int addr_port_lookup(int severity, const char *addrport, char **address,
-                    uint32_t *addr, uint16_t *port_out);
 int parse_port_range(const char *port, uint16_t *port_min_out,
                      uint16_t *port_max_out);
 int addr_mask_get_bits(uint32_t mask);
-/** Length of a buffer to allocate to hold the results of tor_inet_ntoa.*/
-#define INET_NTOA_BUF_LEN 16
-int tor_inet_ntoa(const struct in_addr *in, char *buf, size_t buf_len);
 char *tor_dup_ip(uint32_t addr) ATTR_MALLOC;
 MOCK_DECL(int,get_interface_address,(int severity, uint32_t *addr));
-/** Free a smartlist of IP addresses returned by get_interface_address_list.
- */
-static inline void
-free_interface_address_list(smartlist_t *addrs)
-{
-  free_interface_address6_list(addrs);
-}
+#define interface_address_list_free(lst)\
+  interface_address6_list_free(lst)
 /** Return a smartlist of the IPv4 addresses of all interfaces on the server.
  * Excludes loopback and multicast addresses. Only includes internal addresses
  * if include_internal is true. (Note that a relay behind NAT may use an
@@ -337,43 +340,49 @@ free_interface_address_list(smartlist_t *addrs)
  * Returns NULL on failure.
  * Use free_interface_address_list to free the returned list.
  */
-static inline smartlist_t *
+static inline struct smartlist_t *
 get_interface_address_list(int severity, int include_internal)
 {
   return get_interface_address6_list(severity, AF_INET, include_internal);
 }
 
 tor_addr_port_t *tor_addr_port_new(const tor_addr_t *addr, uint16_t port);
+int tor_addr_port_eq(const tor_addr_port_t *a,
+                     const tor_addr_port_t *b);
+
+int string_is_valid_dest(const char *string);
+int string_is_valid_nonrfc_hostname(const char *string);
+int string_is_valid_ipv4_address(const char *string);
+int string_is_valid_ipv6_address(const char *string);
 
 #ifdef ADDRESS_PRIVATE
-MOCK_DECL(smartlist_t *,get_interface_addresses_raw,(int severity,
+MOCK_DECL(struct smartlist_t *,get_interface_addresses_raw,(int severity,
                                                      sa_family_t family));
 MOCK_DECL(int,get_interface_address6_via_udp_socket_hack,(int severity,
                                                           sa_family_t family,
                                                           tor_addr_t *addr));
 
 #ifdef HAVE_IFADDRS_TO_SMARTLIST
-STATIC smartlist_t *ifaddrs_to_smartlist(const struct ifaddrs *ifa,
+STATIC struct smartlist_t *ifaddrs_to_smartlist(const struct ifaddrs *ifa,
                                          sa_family_t family);
-STATIC smartlist_t *get_interface_addresses_ifaddrs(int severity,
+STATIC struct smartlist_t *get_interface_addresses_ifaddrs(int severity,
                                                     sa_family_t family);
-#endif
+#endif /* defined(HAVE_IFADDRS_TO_SMARTLIST) */
 
 #ifdef HAVE_IP_ADAPTER_TO_SMARTLIST
-STATIC smartlist_t *ip_adapter_addresses_to_smartlist(
+STATIC struct smartlist_t *ip_adapter_addresses_to_smartlist(
                                         const IP_ADAPTER_ADDRESSES *addresses);
-STATIC smartlist_t *get_interface_addresses_win32(int severity,
+STATIC struct smartlist_t *get_interface_addresses_win32(int severity,
                                                   sa_family_t family);
-#endif
+#endif /* defined(HAVE_IP_ADAPTER_TO_SMARTLIST) */
 
 #ifdef HAVE_IFCONF_TO_SMARTLIST
-STATIC smartlist_t *ifreq_to_smartlist(char *ifr,
+STATIC struct smartlist_t *ifreq_to_smartlist(char *ifr,
                                        size_t buflen);
-STATIC smartlist_t *get_interface_addresses_ioctl(int severity,
+STATIC struct smartlist_t *get_interface_addresses_ioctl(int severity,
                                                   sa_family_t family);
-#endif
+#endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */
 
-#endif // ADDRESS_PRIVATE
-
-#endif
+#endif /* defined(ADDRESS_PRIVATE) */
 
+#endif /* !defined(TOR_ADDRESS_H) */
diff --git a/src/common/compat_threads.c b/src/lib/net/alertsock.c
index f4809060d6..cc59d7d893 100644
--- a/src/common/compat_threads.c
+++ b/src/lib/net/alertsock.c
@@ -1,23 +1,23 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file compat_threads.c
+ * \file alertsock.c
  *
- * \brief Cross-platform threading and inter-thread communication logic.
- *  (Platform-specific parts are written in the other compat_*threads
- *  modules.)
- */
+ * \brief Use a socket to alert the main thread from a worker thread.
+ *
+ * Because our main loop spends all of its time in select, epoll, kqueue, or
+ * etc, we need a way to wake up the main loop from another thread.  This code
+ * tries to provide the fastest reasonable way to do that, depending on our
+ * platform.
+ **/
 
 #include "orconfig.h"
-#include <stdlib.h>
-#include "compat.h"
-#include "compat_threads.h"
-
-#include "util.h"
-#include "torlog.h"
+#include "lib/net/alertsock.h"
+#include "lib/net/socket.h"
+#include "lib/log/util_bug.h"
 
 #ifdef HAVE_SYS_EVENTFD_H
 #include <sys/eventfd.h>
@@ -28,117 +28,81 @@
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
-
-/** Return a newly allocated, ready-for-use mutex. */
-tor_mutex_t *
-tor_mutex_new(void)
-{
-  tor_mutex_t *m = tor_malloc_zero(sizeof(tor_mutex_t));
-  tor_mutex_init(m);
-  return m;
-}
-/** Return a newly allocated, ready-for-use mutex.  This one might be
- * non-recursive, if that's faster. */
-tor_mutex_t *
-tor_mutex_new_nonrecursive(void)
-{
-  tor_mutex_t *m = tor_malloc_zero(sizeof(tor_mutex_t));
-  tor_mutex_init_nonrecursive(m);
-  return m;
-}
-/** Release all storage and system resources held by <b>m</b>. */
-void
-tor_mutex_free(tor_mutex_t *m)
-{
-  if (!m)
-    return;
-  tor_mutex_uninit(m);
-  tor_free(m);
-}
-
-/** Allocate and return a new condition variable. */
-tor_cond_t *
-tor_cond_new(void)
-{
-  tor_cond_t *cond = tor_malloc(sizeof(tor_cond_t));
-  if (BUG(tor_cond_init(cond)<0))
-    tor_free(cond); // LCOV_EXCL_LINE
-  return cond;
-}
-
-/** Free all storage held in <b>c</b>. */
-void
-tor_cond_free(tor_cond_t *c)
-{
-  if (!c)
-    return;
-  tor_cond_uninit(c);
-  tor_free(c);
-}
-
-/** Identity of the "main" thread */
-static unsigned long main_thread_id = -1;
-
-/** Start considering the current thread to be the 'main thread'.  This has
- * no effect on anything besides in_main_thread(). */
-void
-set_main_thread(void)
-{
-  main_thread_id = tor_get_thread_id();
-}
-/** Return true iff called from the main thread. */
-int
-in_main_thread(void)
-{
-  return main_thread_id == tor_get_thread_id();
-}
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#ifdef _WIN32
+#include <winsock2.h>
+#endif
 
 #if defined(HAVE_EVENTFD) || defined(HAVE_PIPE)
-/* As write(), but retry on EINTR */
+/* As write(), but retry on EINTR, and return the negative error code on
+ * error. */
 static int
 write_ni(int fd, const void *buf, size_t n)
 {
   int r;
  again:
   r = (int) write(fd, buf, n);
-  if (r < 0 && errno == EINTR)
-    goto again;
+  if (r < 0) {
+    if (errno == EINTR)
+      goto again;
+    else
+      return -errno;
+  }
   return r;
 }
-/* As read(), but retry on EINTR */
+/* As read(), but retry on EINTR, and return the negative error code on error.
+ */
 static int
 read_ni(int fd, void *buf, size_t n)
 {
   int r;
  again:
   r = (int) read(fd, buf, n);
-  if (r < 0 && errno == EINTR)
-    goto again;
+  if (r < 0) {
+    if (errno == EINTR)
+      goto again;
+    else
+      return -errno;
+  }
   return r;
 }
-#endif
+#endif /* defined(HAVE_EVENTFD) || defined(HAVE_PIPE) */
 
-/** As send(), but retry on EINTR. */
+/** As send(), but retry on EINTR, and return the negative error code on
+ * error. */
 static int
 send_ni(int fd, const void *buf, size_t n, int flags)
 {
   int r;
  again:
   r = (int) send(fd, buf, n, flags);
-  if (r < 0 && ERRNO_IS_EINTR(tor_socket_errno(fd)))
-    goto again;
+  if (r < 0) {
+    int error = tor_socket_errno(fd);
+    if (ERRNO_IS_EINTR(error))
+      goto again;
+    else
+      return -error;
+  }
   return r;
 }
 
-/** As recv(), but retry on EINTR. */
+/** As recv(), but retry on EINTR, and return the negative error code on
+ * error. */
 static int
 recv_ni(int fd, void *buf, size_t n, int flags)
 {
   int r;
  again:
   r = (int) recv(fd, buf, n, flags);
-  if (r < 0 && ERRNO_IS_EINTR(tor_socket_errno(fd)))
-    goto again;
+  if (r < 0) {
+    int error = tor_socket_errno(fd);
+    if (ERRNO_IS_EINTR(error))
+      goto again;
+    else
+      return -error;
+  }
   return r;
 }
 
@@ -149,7 +113,7 @@ eventfd_alert(int fd)
 {
   uint64_t u = 1;
   int r = write_ni(fd, (void*)&u, sizeof(u));
-  if (r < 0 && errno != EAGAIN)
+  if (r < 0 && -r != EAGAIN)
     return -1;
   return 0;
 }
@@ -160,11 +124,11 @@ eventfd_drain(int fd)
 {
   uint64_t u = 0;
   int r = read_ni(fd, (void*)&u, sizeof(u));
-  if (r < 0 && errno != EAGAIN)
-    return -1;
+  if (r < 0 && -r != EAGAIN)
+    return r;
   return 0;
 }
-#endif
+#endif /* defined(HAVE_EVENTFD) */
 
 #ifdef HAVE_PIPE
 /** Send a byte over a pipe. Return 0 on success or EAGAIN; -1 on error */
@@ -172,8 +136,8 @@ static int
 pipe_alert(int fd)
 {
   ssize_t r = write_ni(fd, "x", 1);
-  if (r < 0 && errno != EAGAIN)
-    return -1;
+  if (r < 0 && -r != EAGAIN)
+    return (int)r;
   return 0;
 }
 
@@ -188,11 +152,11 @@ pipe_drain(int fd)
     r = read_ni(fd, buf, sizeof(buf));
   } while (r > 0);
   if (r < 0 && errno != EAGAIN)
-    return -1;
+    return -errno;
   /* A value of r = 0 means EOF on the fd so successfully drained. */
   return 0;
 }
-#endif
+#endif /* defined(HAVE_PIPE) */
 
 /** Send a byte on socket <b>fd</b>t.  Return 0 on success or EAGAIN,
  * -1 on error. */
@@ -200,13 +164,13 @@ static int
 sock_alert(tor_socket_t fd)
 {
   ssize_t r = send_ni(fd, "x", 1, 0);
-  if (r < 0 && !ERRNO_IS_EAGAIN(tor_socket_errno(fd)))
-    return -1;
+  if (r < 0 && !ERRNO_IS_EAGAIN(-r))
+    return (int)r;
   return 0;
 }
 
 /** Drain all the input from a socket <b>fd</b>, and ignore it.  Return 0 on
- * success, -1 on error. */
+ * success, -errno on error. */
 static int
 sock_drain(tor_socket_t fd)
 {
@@ -215,8 +179,8 @@ sock_drain(tor_socket_t fd)
   do {
     r = recv_ni(fd, buf, sizeof(buf), 0);
   } while (r > 0);
-  if (r < 0 && !ERRNO_IS_EAGAIN(tor_socket_errno(fd)))
-    return -1;
+  if (r < 0 && !ERRNO_IS_EAGAIN(-r))
+    return (int)r;
   /* A value of r = 0 means EOF on the fd so successfully drained. */
   return 0;
 }
@@ -254,7 +218,7 @@ alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags)
     socks_out->drain_fn = eventfd_drain;
     return 0;
   }
-#endif
+#endif /* defined(HAVE_EVENTFD) */
 
 #ifdef HAVE_PIPE2
   /* Now we're going to try pipes. First type the pipe2() syscall, if we
@@ -267,7 +231,7 @@ alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags)
     socks_out->drain_fn = pipe_drain;
     return 0;
   }
-#endif
+#endif /* defined(HAVE_PIPE2) */
 
 #ifdef HAVE_PIPE
   /* Now try the regular pipe() syscall.  Pipes have a bit lower overhead than
@@ -291,7 +255,7 @@ alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags)
     socks_out->drain_fn = pipe_drain;
     return 0;
   }
-#endif
+#endif /* defined(HAVE_PIPE) */
 
   /* If nothing else worked, fall back on socketpair(). */
   if (!(flags & ASOCKS_NOSOCKETPAIR) &&
@@ -329,4 +293,3 @@ alert_sockets_close(alert_sockets_t *socks)
   }
   socks->read_fd = socks->write_fd = -1;
 }
-
diff --git a/src/lib/net/alertsock.h b/src/lib/net/alertsock.h
new file mode 100644
index 0000000000..c45f42be81
--- /dev/null
+++ b/src/lib/net/alertsock.h
@@ -0,0 +1,45 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file alertsock.h
+ *
+ * \brief Header for alertsock.c
+ **/
+
+#ifndef TOR_ALERTSOCK_H
+#define TOR_ALERTSOCK_H
+
+#include "orconfig.h"
+#include "lib/net/nettypes.h"
+#include "lib/cc/torint.h"
+
+/** Helper type used to manage waking up the main thread while it's in
+ * the libevent main loop.  Used by the work queue code. */
+typedef struct alert_sockets_t {
+  /* XXXX This structure needs a better name. */
+  /** Socket that the main thread should listen for EV_READ events on.
+   * Note that this socket may be a regular fd on a non-Windows platform.
+   */
+  tor_socket_t read_fd;
+  /** Socket to use when alerting the main thread. */
+  tor_socket_t write_fd;
+  /** Function to alert the main thread */
+  int (*alert_fn)(tor_socket_t write_fd);
+  /** Function to make the main thread no longer alerted. */
+  int (*drain_fn)(tor_socket_t read_fd);
+} alert_sockets_t;
+
+/* Flags to disable one or more alert_sockets backends. */
+#define ASOCKS_NOEVENTFD2   (1u<<0)
+#define ASOCKS_NOEVENTFD    (1u<<1)
+#define ASOCKS_NOPIPE2      (1u<<2)
+#define ASOCKS_NOPIPE       (1u<<3)
+#define ASOCKS_NOSOCKETPAIR (1u<<4)
+
+int alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags);
+void alert_sockets_close(alert_sockets_t *socks);
+
+#endif
diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c
new file mode 100644
index 0000000000..3eb0a033d5
--- /dev/null
+++ b/src/lib/net/buffers_net.c
@@ -0,0 +1,202 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file buffers_net.c
+ * \brief Read and write data on a buf_t object.
+ **/
+
+#define BUFFERS_PRIVATE
+#include "lib/net/buffers_net.h"
+#include "lib/container/buffers.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/net/nettypes.h"
+
+#ifdef _WIN32
+#include <winsock2.h>
+#endif
+
+#include <stdlib.h>
+
+#ifdef PARANOIA
+/** Helper: If PARANOIA is defined, assert that the buffer in local variable
+ * <b>buf</b> is well-formed. */
+#define check() STMT_BEGIN buf_assert_ok(buf); STMT_END
+#else
+#define check() STMT_NIL
+#endif /* defined(PARANOIA) */
+
+/** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into
+ * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set
+ * *<b>reached_eof</b> to 1.  Return -1 on error, 0 on eof or blocking,
+ * and the number of bytes read otherwise. */
+static inline int
+read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most,
+              int *reached_eof, int *socket_error)
+{
+  ssize_t read_result;
+  if (at_most > CHUNK_REMAINING_CAPACITY(chunk))
+    at_most = CHUNK_REMAINING_CAPACITY(chunk);
+  read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0);
+
+  if (read_result < 0) {
+    int e = tor_socket_errno(fd);
+    if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
+#ifdef _WIN32
+      if (e == WSAENOBUFS)
+        log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?");
+#endif
+      *socket_error = e;
+      return -1;
+    }
+    return 0; /* would block. */
+  } else if (read_result == 0) {
+    log_debug(LD_NET,"Encountered eof on fd %d", (int)fd);
+    *reached_eof = 1;
+    return 0;
+  } else { /* actually got bytes. */
+    buf->datalen += read_result;
+    chunk->datalen += read_result;
+    log_debug(LD_NET,"Read %ld bytes. %d on inbuf.", (long)read_result,
+              (int)buf->datalen);
+    tor_assert(read_result < INT_MAX);
+    return (int)read_result;
+  }
+}
+
+/** Read from socket <b>s</b>, writing onto end of <b>buf</b>.  Read at most
+ * <b>at_most</b> bytes, growing the buffer as necessary.  If recv() returns 0
+ * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on
+ * error; else return the number of bytes read.
+ */
+/* XXXX indicate "read blocked" somehow? */
+int
+buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
+                     int *reached_eof,
+                     int *socket_error)
+{
+  /* XXXX It's stupid to overload the return values for these functions:
+   * "error status" and "number of bytes read" are not mutually exclusive.
+   */
+  int r = 0;
+  size_t total_read = 0;
+
+  check();
+  tor_assert(reached_eof);
+  tor_assert(SOCKET_OK(s));
+
+  if (BUG(buf->datalen >= INT_MAX))
+    return -1;
+  if (BUG(buf->datalen >= INT_MAX - at_most))
+    return -1;
+
+  while (at_most > total_read) {
+    size_t readlen = at_most - total_read;
+    chunk_t *chunk;
+    if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) {
+      chunk = buf_add_chunk_with_capacity(buf, at_most, 1);
+      if (readlen > chunk->memlen)
+        readlen = chunk->memlen;
+    } else {
+      size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail);
+      chunk = buf->tail;
+      if (cap < readlen)
+        readlen = cap;
+    }
+
+    r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error);
+    check();
+    if (r < 0)
+      return r; /* Error */
+    tor_assert(total_read+r < INT_MAX);
+    total_read += r;
+    if ((size_t)r < readlen) { /* eof, block, or no more to read. */
+      break;
+    }
+  }
+  return (int)total_read;
+}
+
+/** Helper for buf_flush_to_socket(): try to write <b>sz</b> bytes from chunk
+ * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>.  On success, deduct
+ * the bytes written from *<b>buf_flushlen</b>.  Return the number of bytes
+ * written on success, 0 on blocking, -1 on failure.
+ */
+static inline int
+flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz,
+            size_t *buf_flushlen)
+{
+  ssize_t write_result;
+
+  if (sz > chunk->datalen)
+    sz = chunk->datalen;
+  write_result = tor_socket_send(s, chunk->data, sz, 0);
+
+  if (write_result < 0) {
+    int e = tor_socket_errno(s);
+    if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
+#ifdef _WIN32
+      if (e == WSAENOBUFS)
+        log_warn(LD_NET,"write() failed: WSAENOBUFS. Not enough ram?");
+#endif
+      return -1;
+    }
+    log_debug(LD_NET,"write() would block, returning.");
+    return 0;
+  } else {
+    *buf_flushlen -= write_result;
+    buf_drain(buf, write_result);
+    tor_assert(write_result < INT_MAX);
+    return (int)write_result;
+  }
+}
+
+/** Write data from <b>buf</b> to the socket <b>s</b>.  Write at most
+ * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by
+ * the number of bytes actually written, and remove the written bytes
+ * from the buffer.  Return the number of bytes written on success,
+ * -1 on failure.  Return 0 if write() would block.
+ */
+int
+buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
+                    size_t *buf_flushlen)
+{
+  /* XXXX It's stupid to overload the return values for these functions:
+   * "error status" and "number of bytes flushed" are not mutually exclusive.
+   */
+  int r;
+  size_t flushed = 0;
+  tor_assert(buf_flushlen);
+  tor_assert(SOCKET_OK(s));
+  if (BUG(*buf_flushlen > buf->datalen)) {
+    *buf_flushlen = buf->datalen;
+  }
+  if (BUG(sz > *buf_flushlen)) {
+    sz = *buf_flushlen;
+  }
+
+  check();
+  while (sz) {
+    size_t flushlen0;
+    tor_assert(buf->head);
+    if (buf->head->datalen >= sz)
+      flushlen0 = sz;
+    else
+      flushlen0 = buf->head->datalen;
+
+    r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen);
+    check();
+    if (r < 0)
+      return r;
+    flushed += r;
+    sz -= r;
+    if (r == 0 || (size_t)r < flushlen0) /* can't flush any more now. */
+      break;
+  }
+  tor_assert(flushed < INT_MAX);
+  return (int)flushed;
+}
diff --git a/src/lib/net/buffers_net.h b/src/lib/net/buffers_net.h
new file mode 100644
index 0000000000..5f69bebedf
--- /dev/null
+++ b/src/lib/net/buffers_net.h
@@ -0,0 +1,27 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file buffers_net.h
+ *
+ * \brief Header file for buffers_net.c.
+ **/
+
+#ifndef TOR_BUFFERS_NET_H
+#define TOR_BUFFERS_NET_H
+
+#include <stddef.h>
+#include "lib/net/socket.h"
+
+struct buf_t;
+int buf_read_from_socket(struct buf_t *buf, tor_socket_t s, size_t at_most,
+                         int *reached_eof,
+                         int *socket_error);
+
+int buf_flush_to_socket(struct buf_t *buf, tor_socket_t s, size_t sz,
+                        size_t *buf_flushlen);
+
+#endif /* !defined(TOR_BUFFERS_H) */
diff --git a/src/lib/net/gethostname.c b/src/lib/net/gethostname.c
new file mode 100644
index 0000000000..e54a1ea16e
--- /dev/null
+++ b/src/lib/net/gethostname.c
@@ -0,0 +1,30 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file gethostname.c
+ * \brief Mockable wrapper for gethostname().
+ */
+
+#include "orconfig.h"
+#include "lib/net/gethostname.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef _WIN32
+#include <winsock2.h>
+#endif
+
+/** Get name of current host and write it to <b>name</b> array, whose
+ * length is specified by <b>namelen</b> argument. Return 0 upon
+ * successful completion; otherwise return return -1. (Currently,
+ * this function is merely a mockable wrapper for POSIX gethostname().)
+ */
+MOCK_IMPL(int,
+tor_gethostname,(char *name, size_t namelen))
+{
+   return gethostname(name,namelen);
+}
diff --git a/src/lib/net/gethostname.h b/src/lib/net/gethostname.h
new file mode 100644
index 0000000000..69b0528bc0
--- /dev/null
+++ b/src/lib/net/gethostname.h
@@ -0,0 +1,19 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file gethostname.h
+ * \brief Header for gethostname.c
+ **/
+
+#ifndef TOR_GETHOSTNAME_H
+#define TOR_GETHOSTNAME_H
+
+#include "lib/testsupport/testsupport.h"
+#include <stddef.h>
+
+MOCK_DECL(int,tor_gethostname,(char *name, size_t namelen));
+
+#endif
diff --git a/src/lib/net/inaddr.c b/src/lib/net/inaddr.c
new file mode 100644
index 0000000000..1a2406ce5f
--- /dev/null
+++ b/src/lib/net/inaddr.c
@@ -0,0 +1,267 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file inaddr.c
+ * \brief Convert in_addr and in6_addr to and from strings.
+ **/
+
+#include "lib/net/inaddr.h"
+
+#include "lib/cc/torint.h"
+#include "lib/log/util_bug.h"
+#include "lib/net/inaddr_st.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/string/compat_string.h"
+#include "lib/string/printf.h"
+#include "lib/string/scanf.h"
+#include "lib/string/util_string.h"
+
+#ifdef HAVE_ARPA_INET_H
+#include <arpa/inet.h>
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef _WIN32
+#include <winsock2.h>
+#endif
+
+/** Set *addr to the IP address (in dotted-quad notation) stored in *str.
+ * Return 1 on success, 0 if *str is badly formatted.
+ * (Like inet_aton(str,addr), but works on Windows and Solaris.)
+ */
+int
+tor_inet_aton(const char *str, struct in_addr* addr)
+{
+  unsigned a,b,c,d;
+  char more;
+  if (tor_sscanf(str, "%3u.%3u.%3u.%3u%c", &a,&b,&c,&d,&more) != 4)
+    return 0;
+  if (a > 255) return 0;
+  if (b > 255) return 0;
+  if (c > 255) return 0;
+  if (d > 255) return 0;
+  addr->s_addr = htonl((a<<24) | (b<<16) | (c<<8) | d);
+  return 1;
+}
+
+/** Given an IPv4 in_addr struct *<b>in</b> (in network order, as usual),
+ *  write it as a string into the <b>buf_len</b>-byte buffer in
+ *  <b>buf</b>. Returns a non-negative integer on success.
+ *  Returns -1 on failure.
+ */
+int
+tor_inet_ntoa(const struct in_addr *in, char *buf, size_t buf_len)
+{
+  uint32_t a = ntohl(in->s_addr);
+  return tor_snprintf(buf, buf_len, "%d.%d.%d.%d",
+                      (int)(uint8_t)((a>>24)&0xff),
+                      (int)(uint8_t)((a>>16)&0xff),
+                      (int)(uint8_t)((a>>8 )&0xff),
+                      (int)(uint8_t)((a    )&0xff));
+}
+
+/** Given <b>af</b>==AF_INET and <b>src</b> a struct in_addr, or
+ * <b>af</b>==AF_INET6 and <b>src</b> a struct in6_addr, try to format the
+ * address and store it in the <b>len</b>-byte buffer <b>dst</b>.  Returns
+ * <b>dst</b> on success, NULL on failure.
+ *
+ * (Like inet_ntop(af,src,dst,len), but works on platforms that don't have it:
+ * Tor sometimes needs to format ipv6 addresses even on platforms without ipv6
+ * support.) */
+const char *
+tor_inet_ntop(int af, const void *src, char *dst, size_t len)
+{
+  if (af == AF_INET) {
+    if (tor_inet_ntoa(src, dst, len) < 0)
+      return NULL;
+    else
+      return dst;
+  } else if (af == AF_INET6) {
+    const struct in6_addr *addr = src;
+    char buf[64], *cp;
+    int longestGapLen = 0, longestGapPos = -1, i,
+      curGapPos = -1, curGapLen = 0;
+    uint16_t words[8];
+    for (i = 0; i < 8; ++i) {
+      words[i] = (((uint16_t)addr->s6_addr[2*i])<<8) + addr->s6_addr[2*i+1];
+    }
+    if (words[0] == 0 && words[1] == 0 && words[2] == 0 && words[3] == 0 &&
+        words[4] == 0 && ((words[5] == 0 && words[6] && words[7]) ||
+                          (words[5] == 0xffff))) {
+      /* This is an IPv4 address. */
+      if (words[5] == 0) {
+        tor_snprintf(buf, sizeof(buf), "::%d.%d.%d.%d",
+                     addr->s6_addr[12], addr->s6_addr[13],
+                     addr->s6_addr[14], addr->s6_addr[15]);
+      } else {
+        tor_snprintf(buf, sizeof(buf), "::%x:%d.%d.%d.%d", words[5],
+                     addr->s6_addr[12], addr->s6_addr[13],
+                     addr->s6_addr[14], addr->s6_addr[15]);
+      }
+      if ((strlen(buf) + 1) > len) /* +1 for \0 */
+        return NULL;
+      strlcpy(dst, buf, len);
+      return dst;
+    }
+    i = 0;
+    while (i < 8) {
+      if (words[i] == 0) {
+        curGapPos = i++;
+        curGapLen = 1;
+        while (i<8 && words[i] == 0) {
+          ++i; ++curGapLen;
+        }
+        if (curGapLen > longestGapLen) {
+          longestGapPos = curGapPos;
+          longestGapLen = curGapLen;
+        }
+      } else {
+        ++i;
+      }
+    }
+    if (longestGapLen<=1)
+      longestGapPos = -1;
+
+    cp = buf;
+    for (i = 0; i < 8; ++i) {
+      if (words[i] == 0 && longestGapPos == i) {
+        if (i == 0)
+          *cp++ = ':';
+        *cp++ = ':';
+        while (i < 8 && words[i] == 0)
+          ++i;
+        --i; /* to compensate for loop increment. */
+      } else {
+        tor_snprintf(cp, sizeof(buf)-(cp-buf), "%x", (unsigned)words[i]);
+        cp += strlen(cp);
+        if (i != 7)
+          *cp++ = ':';
+      }
+    }
+    *cp = '\0';
+    if ((strlen(buf) + 1) > len) /* +1 for \0 */
+      return NULL;
+    strlcpy(dst, buf, len);
+    return dst;
+  } else {
+    return NULL;
+  }
+}
+
+/** Given <b>af</b>==AF_INET or <b>af</b>==AF_INET6, and a string <b>src</b>
+ * encoding an IPv4 address or IPv6 address correspondingly, try to parse the
+ * address and store the result in <b>dst</b> (which must have space for a
+ * struct in_addr or a struct in6_addr, as appropriate).  Return 1 on success,
+ * 0 on a bad parse, and -1 on a bad <b>af</b>.
+ *
+ * (Like inet_pton(af,src,dst) but works on platforms that don't have it: Tor
+ * sometimes needs to format ipv6 addresses even on platforms without ipv6
+ * support.) */
+int
+tor_inet_pton(int af, const char *src, void *dst)
+{
+  if (af == AF_INET) {
+    return tor_inet_aton(src, dst);
+  } else if (af == AF_INET6) {
+    struct in6_addr *out = dst;
+    uint16_t words[8];
+    int gapPos = -1, i, setWords=0;
+    const char *dot = strchr(src, '.');
+    const char *eow; /* end of words. */
+    memset(words, 0xf8, sizeof(words));
+    if (dot == src)
+      return 0;
+    else if (!dot)
+      eow = src+strlen(src);
+    else {
+      unsigned byte1,byte2,byte3,byte4;
+      char more;
+      for (eow = dot-1; eow > src && TOR_ISDIGIT(*eow); --eow)
+        ;
+      if (*eow != ':')
+        return 0;
+      ++eow;
+
+      /* We use "scanf" because some platform inet_aton()s are too lax
+       * about IPv4 addresses of the form "1.2.3" */
+      if (tor_sscanf(eow, "%3u.%3u.%3u.%3u%c",
+                     &byte1,&byte2,&byte3,&byte4,&more) != 4)
+        return 0;
+
+      if (byte1 > 255 || byte2 > 255 || byte3 > 255 || byte4 > 255)
+        return 0;
+
+      words[6] = (byte1<<8) | byte2;
+      words[7] = (byte3<<8) | byte4;
+      setWords += 2;
+    }
+
+    i = 0;
+    while (src < eow) {
+      if (i > 7)
+        return 0;
+      if (TOR_ISXDIGIT(*src)) {
+        char *next;
+        ssize_t len;
+        long r = strtol(src, &next, 16);
+        if (next == NULL || next == src) {
+          /* The 'next == src' error case can happen on versions of openbsd
+           * which treat "0xfoo" as an error, rather than as "0" followed by
+           * "xfoo". */
+          return 0;
+        }
+
+        len = *next == '\0' ? eow - src : next - src;
+        if (len > 4)
+          return 0;
+        if (len > 1 && !TOR_ISXDIGIT(src[1]))
+          return 0; /* 0x is not valid */
+
+        tor_assert(r >= 0);
+        tor_assert(r < 65536);
+        words[i++] = (uint16_t)r;
+        setWords++;
+        src = next;
+        if (*src != ':' && src != eow)
+          return 0;
+        ++src;
+      } else if (*src == ':' && i > 0 && gapPos == -1) {
+        gapPos = i;
+        ++src;
+      } else if (*src == ':' && i == 0 && src+1 < eow && src[1] == ':' &&
+                 gapPos == -1) {
+        gapPos = i;
+        src += 2;
+      } else {
+        return 0;
+      }
+    }
+
+    if (setWords > 8 ||
+        (setWords == 8 && gapPos != -1) ||
+        (setWords < 8 && gapPos == -1))
+      return 0;
+
+    if (gapPos >= 0) {
+      int nToMove = setWords - (dot ? 2 : 0) - gapPos;
+      int gapLen = 8 - setWords;
+      tor_assert(nToMove >= 0);
+      memmove(&words[gapPos+gapLen], &words[gapPos],
+              sizeof(uint16_t)*nToMove);
+      memset(&words[gapPos], 0, sizeof(uint16_t)*gapLen);
+    }
+    for (i = 0; i < 8; ++i) {
+      out->s6_addr[2*i  ] = words[i] >> 8;
+      out->s6_addr[2*i+1] = words[i] & 0xff;
+    }
+
+    return 1;
+  } else {
+    return -1;
+  }
+}
diff --git a/src/lib/net/inaddr.h b/src/lib/net/inaddr.h
new file mode 100644
index 0000000000..36352b65ea
--- /dev/null
+++ b/src/lib/net/inaddr.h
@@ -0,0 +1,27 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file inaddr.h
+ * \brief Header for inaddr.c.
+ **/
+
+#ifndef TOR_INADDR_H
+#define TOR_INADDR_H
+
+#include "orconfig.h"
+#include <stddef.h>
+
+struct in_addr;
+
+int tor_inet_aton(const char *str, struct in_addr *addr);
+/** Length of a buffer to allocate to hold the results of tor_inet_ntoa.*/
+#define INET_NTOA_BUF_LEN 16
+int tor_inet_ntoa(const struct in_addr *in, char *buf, size_t buf_len);
+
+const char *tor_inet_ntop(int af, const void *src, char *dst, size_t len);
+int tor_inet_pton(int af, const char *src, void *dst);
+
+#endif
diff --git a/src/lib/net/inaddr_st.h b/src/lib/net/inaddr_st.h
new file mode 100644
index 0000000000..806f2c096a
--- /dev/null
+++ b/src/lib/net/inaddr_st.h
@@ -0,0 +1,107 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file inaddr_st.h
+ *
+ * \brief Define in6_addr, its members, and related types on platforms that
+ *    lack it.
+ **/
+
+#ifndef TOR_INADDR_ST_H
+#define TOR_INADDR_ST_H
+
+#include "orconfig.h"
+#include <stddef.h>
+
+#ifdef HAVE_ARPA_INET_H
+#include <arpa/inet.h>
+#endif
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+#ifdef HAVE_NETINET_IN6_H
+#include <netinet/in6.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#ifdef HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
+
+#ifdef _WIN32
+#include <winsock2.h>
+#include <ws2tcpip.h>
+#include <windows.h>
+#endif
+
+#include "lib/cc/torint.h"
+
+struct in_addr;
+
+/** Implementation of struct in6_addr for platforms that do not have it.
+ * Generally, these platforms are ones without IPv6 support, but we want to
+ * have a working in6_addr there anyway, so we can use it to parse IPv6
+ * addresses. */
+#if !defined(HAVE_STRUCT_IN6_ADDR)
+struct in6_addr
+{
+  union {
+    uint8_t u6_addr8[16];
+    uint16_t u6_addr16[8];
+    uint32_t u6_addr32[4];
+  } in6_u;
+#define s6_addr   in6_u.u6_addr8
+#define s6_addr16 in6_u.u6_addr16
+#define s6_addr32 in6_u.u6_addr32
+};
+#endif /* !defined(HAVE_STRUCT_IN6_ADDR) */
+
+/** @{ */
+/** Many BSD variants seem not to define these. */
+#if defined(__APPLE__) || defined(__darwin__) || \
+  defined(__FreeBSD__) || defined(__NetBSD__) || defined(OpenBSD)
+#ifndef s6_addr16
+#define s6_addr16 __u6_addr.__u6_addr16
+#endif
+#ifndef s6_addr32
+#define s6_addr32 __u6_addr.__u6_addr32
+#endif
+#endif /* defined(__APPLE__) || defined(__darwin__) || ... */
+/** @} */
+
+#ifndef HAVE_SA_FAMILY_T
+typedef uint16_t sa_family_t;
+#endif
+
+/** @{ */
+/** Apparently, MS and Solaris don't define s6_addr16 or s6_addr32; these
+ * macros get you a pointer to s6_addr32 or local equivalent. */
+#ifdef HAVE_STRUCT_IN6_ADDR_S6_ADDR32
+#define S6_ADDR32(x) ((uint32_t*)(x).s6_addr32)
+#else
+#define S6_ADDR32(x) ((uint32_t*)((char*)&(x).s6_addr))
+#endif
+#ifdef HAVE_STRUCT_IN6_ADDR_S6_ADDR16
+#define S6_ADDR16(x) ((uint16_t*)(x).s6_addr16)
+#else
+#define S6_ADDR16(x) ((uint16_t*)((char*)&(x).s6_addr))
+#endif
+/** @} */
+
+/** Implementation of struct sockaddr_in6 on platforms that do not have
+ * it. See notes on struct in6_addr. */
+#if !defined(HAVE_STRUCT_SOCKADDR_IN6)
+struct sockaddr_in6 {
+  sa_family_t sin6_family;
+  uint16_t sin6_port;
+  // uint32_t sin6_flowinfo;
+  struct in6_addr sin6_addr;
+  // uint32_t sin6_scope_id;
+};
+#endif /* !defined(HAVE_STRUCT_SOCKADDR_IN6) */
+
+#endif /* TOR_INADDR_ST_H */
diff --git a/src/lib/net/include.am b/src/lib/net/include.am
new file mode 100644
index 0000000000..ff0967e786
--- /dev/null
+++ b/src/lib/net/include.am
@@ -0,0 +1,34 @@
+
+noinst_LIBRARIES += src/lib/libtor-net.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-net-testing.a
+endif
+
+src_lib_libtor_net_a_SOURCES =			\
+	src/lib/net/address.c			\
+	src/lib/net/alertsock.c                 \
+	src/lib/net/buffers_net.c		\
+	src/lib/net/gethostname.c		\
+	src/lib/net/inaddr.c			\
+	src/lib/net/resolve.c			\
+	src/lib/net/socket.c			\
+	src/lib/net/socketpair.c
+
+src_lib_libtor_net_testing_a_SOURCES = \
+	$(src_lib_libtor_net_a_SOURCES)
+src_lib_libtor_net_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_net_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=				\
+	src/lib/net/address.h			\
+	src/lib/net/alertsock.h                 \
+	src/lib/net/buffers_net.h		\
+	src/lib/net/gethostname.h		\
+	src/lib/net/inaddr.h			\
+	src/lib/net/inaddr_st.h			\
+	src/lib/net/nettypes.h			\
+	src/lib/net/resolve.h			\
+	src/lib/net/socket.h			\
+	src/lib/net/socketpair.h		\
+	src/lib/net/socks5_status.h
diff --git a/src/lib/net/nettypes.h b/src/lib/net/nettypes.h
new file mode 100644
index 0000000000..6209bbe18a
--- /dev/null
+++ b/src/lib/net/nettypes.h
@@ -0,0 +1,44 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file nettypes.h
+ * \brief Declarations for types used throughout the Tor networking system
+ **/
+
+#ifndef TOR_NET_TYPES_H
+#define TOR_NET_TYPES_H
+
+#include "orconfig.h"
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+
+#if (SIZEOF_SOCKLEN_T == 0)
+typedef int socklen_t;
+#endif
+
+#ifdef _WIN32
+/* XXX Actually, this should arguably be SOCKET; we use intptr_t here so that
+ * any inadvertent checks for the socket being <= 0 or > 0 will probably
+ * still work. */
+#define tor_socket_t intptr_t
+#define TOR_SOCKET_T_FORMAT "%"PRIuPTR
+#define SOCKET_OK(s) ((SOCKET)(s) != INVALID_SOCKET)
+#define TOR_INVALID_SOCKET INVALID_SOCKET
+#else /* !(defined(_WIN32)) */
+/** Type used for a network socket. */
+#define tor_socket_t int
+#define TOR_SOCKET_T_FORMAT "%d"
+/** Macro: true iff 's' is a possible value for a valid initialized socket. */
+#define SOCKET_OK(s) ((s) >= 0)
+/** Error/uninitialized value for a tor_socket_t. */
+#define TOR_INVALID_SOCKET (-1)
+#endif /* defined(_WIN32) */
+
+#endif
diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c
new file mode 100644
index 0000000000..8cee29df37
--- /dev/null
+++ b/src/lib/net/resolve.c
@@ -0,0 +1,424 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file resolve.c
+ * \brief Use the libc DNS resolver to convert hostnames into addresses.
+ **/
+
+#include "lib/net/resolve.h"
+
+#include "lib/net/address.h"
+#include "lib/net/inaddr.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/parse_int.h"
+#include "lib/string/util_string.h"
+
+#include "siphash.h"
+#include "ht.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#ifdef HAVE_NETDB_H
+#include <netdb.h>
+#endif
+
+#include <string.h>
+
+/** Similar behavior to Unix gethostbyname: resolve <b>name</b>, and set
+ * *<b>addr</b> to the proper IP address, in host byte order.  Returns 0
+ * on success, -1 on failure; 1 on transient failure.
+ *
+ * (This function exists because standard windows gethostbyname
+ * doesn't treat raw IP addresses properly.)
+ */
+
+MOCK_IMPL(int,
+tor_lookup_hostname,(const char *name, uint32_t *addr))
+{
+  tor_addr_t myaddr;
+  int ret;
+
+  if ((ret = tor_addr_lookup(name, AF_INET, &myaddr)))
+    return ret;
+
+  if (tor_addr_family(&myaddr) == AF_INET) {
+    *addr = tor_addr_to_ipv4h(&myaddr);
+    return ret;
+  }
+
+  return -1;
+}
+
+/** Similar behavior to Unix gethostbyname: resolve <b>name</b>, and set
+ * *<b>addr</b> to the proper IP address and family. The <b>family</b>
+ * argument (which must be AF_INET, AF_INET6, or AF_UNSPEC) declares a
+ * <i>preferred</i> family, though another one may be returned if only one
+ * family is implemented for this address.
+ *
+ * Return 0 on success, -1 on failure; 1 on transient failure.
+ */
+MOCK_IMPL(int,
+tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr))
+{
+  /* Perhaps eventually this should be replaced by a tor_getaddrinfo or
+   * something.
+   */
+  struct in_addr iaddr;
+  struct in6_addr iaddr6;
+  tor_assert(name);
+  tor_assert(addr);
+  tor_assert(family == AF_INET || family == AF_INET6 || family == AF_UNSPEC);
+  if (!*name) {
+    /* Empty address is an error. */
+    return -1;
+  } else if (tor_inet_pton(AF_INET, name, &iaddr)) {
+    /* It's an IPv4 IP. */
+    if (family == AF_INET6)
+      return -1;
+    tor_addr_from_in(addr, &iaddr);
+    return 0;
+  } else if (tor_inet_pton(AF_INET6, name, &iaddr6)) {
+    if (family == AF_INET)
+      return -1;
+    tor_addr_from_in6(addr, &iaddr6);
+    return 0;
+  } else {
+#ifdef HAVE_GETADDRINFO
+    int err;
+    struct addrinfo *res=NULL, *res_p;
+    struct addrinfo *best=NULL;
+    struct addrinfo hints;
+    int result = -1;
+    memset(&hints, 0, sizeof(hints));
+    hints.ai_family = family;
+    hints.ai_socktype = SOCK_STREAM;
+    err = tor_getaddrinfo(name, NULL, &hints, &res);
+    /* The check for 'res' here shouldn't be necessary, but it makes static
+     * analysis tools happy. */
+    if (!err && res) {
+      best = NULL;
+      for (res_p = res; res_p; res_p = res_p->ai_next) {
+        if (family == AF_UNSPEC) {
+          if (res_p->ai_family == AF_INET) {
+            best = res_p;
+            break;
+          } else if (res_p->ai_family == AF_INET6 && !best) {
+            best = res_p;
+          }
+        } else if (family == res_p->ai_family) {
+          best = res_p;
+          break;
+        }
+      }
+      if (!best)
+        best = res;
+      if (best->ai_family == AF_INET) {
+        tor_addr_from_in(addr,
+                         &((struct sockaddr_in*)best->ai_addr)->sin_addr);
+        result = 0;
+      } else if (best->ai_family == AF_INET6) {
+        tor_addr_from_in6(addr,
+                          &((struct sockaddr_in6*)best->ai_addr)->sin6_addr);
+        result = 0;
+      }
+      tor_freeaddrinfo(res);
+      return result;
+    }
+    return (err == EAI_AGAIN) ? 1 : -1;
+#else /* !(defined(HAVE_GETADDRINFO)) */
+    struct hostent *ent;
+    int err;
+#ifdef HAVE_GETHOSTBYNAME_R_6_ARG
+    char buf[2048];
+    struct hostent hostent;
+    int r;
+    r = gethostbyname_r(name, &hostent, buf, sizeof(buf), &ent, &err);
+#elif defined(HAVE_GETHOSTBYNAME_R_5_ARG)
+    char buf[2048];
+    struct hostent hostent;
+    ent = gethostbyname_r(name, &hostent, buf, sizeof(buf), &err);
+#elif defined(HAVE_GETHOSTBYNAME_R_3_ARG)
+    struct hostent_data data;
+    struct hostent hent;
+    memset(&data, 0, sizeof(data));
+    err = gethostbyname_r(name, &hent, &data);
+    ent = err ? NULL : &hent;
+#else
+    ent = gethostbyname(name);
+#ifdef _WIN32
+    err = WSAGetLastError();
+#else
+    err = h_errno;
+#endif
+#endif /* defined(HAVE_GETHOSTBYNAME_R_6_ARG) || ... */
+    if (ent) {
+      if (ent->h_addrtype == AF_INET) {
+        tor_addr_from_in(addr, (struct in_addr*) ent->h_addr);
+      } else if (ent->h_addrtype == AF_INET6) {
+        tor_addr_from_in6(addr, (struct in6_addr*) ent->h_addr);
+      } else {
+        tor_assert(0); // LCOV_EXCL_LINE: gethostbyname() returned bizarre type
+      }
+      return 0;
+    }
+#ifdef _WIN32
+    return (err == WSATRY_AGAIN) ? 1 : -1;
+#else
+    return (err == TRY_AGAIN) ? 1 : -1;
+#endif
+#endif /* defined(HAVE_GETADDRINFO) */
+  }
+}
+
+/** Parse an address or address-port combination from <b>s</b>, resolve the
+ * address as needed, and put the result in <b>addr_out</b> and (optionally)
+ * <b>port_out</b>.  Return 0 on success, negative on failure. */
+int
+tor_addr_port_lookup(const char *s, tor_addr_t *addr_out, uint16_t *port_out)
+{
+  const char *port;
+  tor_addr_t addr;
+  uint16_t portval;
+  char *tmp = NULL;
+
+  tor_assert(s);
+  tor_assert(addr_out);
+
+  s = eat_whitespace(s);
+
+  if (*s == '[') {
+    port = strstr(s, "]");
+    if (!port)
+      goto err;
+    tmp = tor_strndup(s+1, port-(s+1));
+    port = port+1;
+    if (*port == ':')
+      port++;
+    else
+      port = NULL;
+  } else {
+    port = strchr(s, ':');
+    if (port)
+      tmp = tor_strndup(s, port-s);
+    else
+      tmp = tor_strdup(s);
+    if (port)
+      ++port;
+  }
+
+  if (tor_addr_lookup(tmp, AF_UNSPEC, &addr) != 0)
+    goto err;
+  tor_free(tmp);
+
+  if (port) {
+    portval = (int) tor_parse_long(port, 10, 1, 65535, NULL, NULL);
+    if (!portval)
+      goto err;
+  } else {
+    portval = 0;
+  }
+
+  if (port_out)
+    *port_out = portval;
+  tor_addr_copy(addr_out, &addr);
+
+  return 0;
+ err:
+  tor_free(tmp);
+  return -1;
+}
+
+#ifdef USE_SANDBOX_GETADDRINFO
+/** True if we should only return cached values */
+static int sandbox_getaddrinfo_is_active = 0;
+
+/** Cache entry for getaddrinfo results; used when sandboxing is implemented
+ * so that we can consult the cache when the sandbox prevents us from doing
+ * getaddrinfo.
+ *
+ * We support only a limited range of getaddrinfo calls, where servname is null
+ * and hints contains only socktype=SOCK_STREAM, family in INET,INET6,UNSPEC.
+ */
+typedef struct cached_getaddrinfo_item_t {
+  HT_ENTRY(cached_getaddrinfo_item_t) node;
+  char *name;
+  int family;
+  /** set if no error; otherwise NULL */
+  struct addrinfo *res;
+  /** 0 for no error; otherwise an EAI_* value */
+  int err;
+} cached_getaddrinfo_item_t;
+
+static unsigned
+cached_getaddrinfo_item_hash(const cached_getaddrinfo_item_t *item)
+{
+  return (unsigned)siphash24g(item->name, strlen(item->name)) + item->family;
+}
+
+static unsigned
+cached_getaddrinfo_items_eq(const cached_getaddrinfo_item_t *a,
+                            const cached_getaddrinfo_item_t *b)
+{
+  return (a->family == b->family) && 0 == strcmp(a->name, b->name);
+}
+
+#define cached_getaddrinfo_item_free(item)              \
+  FREE_AND_NULL(cached_getaddrinfo_item_t,              \
+                cached_getaddrinfo_item_free_, (item))
+
+static void
+cached_getaddrinfo_item_free_(cached_getaddrinfo_item_t *item)
+{
+  if (item == NULL)
+    return;
+
+  tor_free(item->name);
+  if (item->res)
+    freeaddrinfo(item->res);
+  tor_free(item);
+}
+
+static HT_HEAD(getaddrinfo_cache, cached_getaddrinfo_item_t)
+     getaddrinfo_cache = HT_INITIALIZER();
+
+HT_PROTOTYPE(getaddrinfo_cache, cached_getaddrinfo_item_t, node,
+             cached_getaddrinfo_item_hash,
+             cached_getaddrinfo_items_eq)
+HT_GENERATE2(getaddrinfo_cache, cached_getaddrinfo_item_t, node,
+             cached_getaddrinfo_item_hash,
+             cached_getaddrinfo_items_eq,
+             0.6, tor_reallocarray_, tor_free_)
+
+/** If true, don't try to cache getaddrinfo results. */
+static int sandbox_getaddrinfo_cache_disabled = 0;
+
+/** Tell the sandbox layer not to try to cache getaddrinfo results. Used as in
+ * tor-resolve, when we have no intention of initializing crypto or of
+ * installing the sandbox.*/
+void
+sandbox_disable_getaddrinfo_cache(void)
+{
+  sandbox_getaddrinfo_cache_disabled = 1;
+}
+
+void
+tor_freeaddrinfo(struct addrinfo *ai)
+{
+  if (sandbox_getaddrinfo_cache_disabled)
+    freeaddrinfo(ai);
+}
+
+int
+tor_getaddrinfo(const char *name, const char *servname,
+                const struct addrinfo *hints,
+                struct addrinfo **res)
+{
+  int err;
+  struct cached_getaddrinfo_item_t search, *item;
+
+  if (sandbox_getaddrinfo_cache_disabled) {
+    return getaddrinfo(name, NULL, hints, res);
+  }
+
+  if (servname != NULL) {
+    log_warn(LD_BUG, "called with non-NULL servname");
+    return EAI_NONAME;
+  }
+  if (name == NULL) {
+    log_warn(LD_BUG, "called with NULL name");
+    return EAI_NONAME;
+  }
+
+  *res = NULL;
+
+  memset(&search, 0, sizeof(search));
+  search.name = (char *) name;
+  search.family = hints ? hints->ai_family : AF_UNSPEC;
+  item = HT_FIND(getaddrinfo_cache, &getaddrinfo_cache, &search);
+
+  if (! sandbox_getaddrinfo_is_active) {
+    /* If the sandbox is not turned on yet, then getaddrinfo and store the
+       result. */
+
+    err = getaddrinfo(name, NULL, hints, res);
+    log_info(LD_NET,"(Sandbox) getaddrinfo %s.", err ? "failed" : "succeeded");
+
+    if (! item) {
+      item = tor_malloc_zero(sizeof(*item));
+      item->name = tor_strdup(name);
+      item->family = hints ? hints->ai_family : AF_UNSPEC;
+      HT_INSERT(getaddrinfo_cache, &getaddrinfo_cache, item);
+    }
+
+    if (item->res) {
+      freeaddrinfo(item->res);
+      item->res = NULL;
+    }
+    item->res = *res;
+    item->err = err;
+    return err;
+  }
+
+  /* Otherwise, the sandbox is on.  If we have an item, yield its cached
+     result. */
+  if (item) {
+    *res = item->res;
+    return item->err;
+  }
+
+  /* getting here means something went wrong */
+  log_err(LD_BUG,"(Sandbox) failed to get address %s!", name);
+  return EAI_NONAME;
+}
+
+int
+tor_add_addrinfo(const char *name)
+{
+  struct addrinfo *res;
+  struct addrinfo hints;
+  int i;
+  static const int families[] = { AF_INET, AF_INET6, AF_UNSPEC };
+
+  memset(&hints, 0, sizeof(hints));
+  hints.ai_socktype = SOCK_STREAM;
+  for (i = 0; i < 3; ++i) {
+    hints.ai_family = families[i];
+
+    res = NULL;
+    (void) tor_getaddrinfo(name, NULL, &hints, &res);
+    if (res)
+      tor_freeaddrinfo(res);
+  }
+
+  return 0;
+}
+
+void
+tor_free_getaddrinfo_cache(void)
+{
+  cached_getaddrinfo_item_t **next, **item, *this;
+
+  for (item = HT_START(getaddrinfo_cache, &getaddrinfo_cache);
+       item;
+       item = next) {
+    this = *item;
+    next = HT_NEXT_RMV(getaddrinfo_cache, &getaddrinfo_cache, item);
+    cached_getaddrinfo_item_free(this);
+  }
+
+  HT_CLEAR(getaddrinfo_cache, &getaddrinfo_cache);
+}
+
+void
+tor_make_getaddrinfo_cache_active(void)
+{
+  sandbox_getaddrinfo_is_active = 1;
+}
+#endif
diff --git a/src/lib/net/resolve.h b/src/lib/net/resolve.h
new file mode 100644
index 0000000000..47a283c81c
--- /dev/null
+++ b/src/lib/net/resolve.h
@@ -0,0 +1,58 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file resolve.h
+ * \brief Header for resolve.c
+ **/
+
+#ifndef TOR_RESOLVE_H
+#define TOR_RESOLVE_H
+
+#include "orconfig.h"
+#include "lib/cc/torint.h"
+#include "lib/testsupport/testsupport.h"
+#ifdef _WIN32
+#include <winsock2.h>
+#endif
+
+#if defined(HAVE_SECCOMP_H) && defined(__linux__)
+#define USE_SANDBOX_GETADDRINFO
+#endif
+
+struct tor_addr_t;
+
+MOCK_DECL(int, tor_lookup_hostname,(const char *name, uint32_t *addr));
+MOCK_DECL(int, tor_addr_lookup,(const char *name, uint16_t family,
+                                struct tor_addr_t *addr_out));
+int tor_addr_port_lookup(const char *s, struct tor_addr_t *addr_out,
+                         uint16_t *port_out);
+
+struct addrinfo;
+#ifdef USE_SANDBOX_GETADDRINFO
+/** Pre-calls getaddrinfo in order to pre-record result. */
+int tor_add_addrinfo(const char *addr);
+
+struct addrinfo;
+/** Replacement for getaddrinfo(), using pre-recorded results. */
+int tor_getaddrinfo(const char *name, const char *servname,
+                        const struct addrinfo *hints,
+                        struct addrinfo **res);
+void tor_freeaddrinfo(struct addrinfo *addrinfo);
+void tor_free_getaddrinfo_cache(void);
+void tor_make_getaddrinfo_cache_active(void);
+#else /* !(defined(USE_SANDBOX_GETADDRINFO)) */
+#define tor_getaddrinfo(name, servname, hints, res)  \
+  getaddrinfo((name),(servname), (hints),(res))
+#define tor_add_addrinfo(name) \
+  ((void)(name))
+#define tor_freeaddrinfo(addrinfo) \
+  freeaddrinfo((addrinfo))
+#define tor_free_getaddrinfo_cache()
+#endif /* defined(USE_SANDBOX_GETADDRINFO) */
+
+void sandbox_disable_getaddrinfo_cache(void);
+
+#endif
diff --git a/src/lib/net/socket.c b/src/lib/net/socket.c
new file mode 100644
index 0000000000..fba90b7506
--- /dev/null
+++ b/src/lib/net/socket.c
@@ -0,0 +1,697 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file socket.c
+ * \brief Compatibility and utility functions for working with network
+ *    sockets.
+ **/
+
+#define SOCKET_PRIVATE
+#include "lib/net/socket.h"
+#include "lib/net/socketpair.h"
+#include "lib/net/address.h"
+#include "lib/cc/compat_compiler.h"
+#include "lib/err/torerr.h"
+#include "lib/lock/compat_mutex.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+
+#ifdef _WIN32
+#include <winsock2.h>
+#include <windows.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#include <stddef.h>
+#include <string.h>
+
+/** Called before we make any calls to network-related functions.
+ * (Some operating systems require their network libraries to be
+ * initialized.) */
+int
+network_init(void)
+{
+#ifdef _WIN32
+  /* This silly exercise is necessary before windows will allow
+   * gethostbyname to work. */
+  WSADATA WSAData;
+  int r;
+  r = WSAStartup(0x101,&WSAData);
+  if (r) {
+    log_warn(LD_NET,"Error initializing windows network layer: code was %d",r);
+    return -1;
+  }
+  if (sizeof(SOCKET) != sizeof(tor_socket_t)) {
+    log_warn(LD_BUG,"The tor_socket_t type does not match SOCKET in size; Tor "
+             "might not work. (Sizes are %d and %d respectively.)",
+             (int)sizeof(tor_socket_t), (int)sizeof(SOCKET));
+  }
+  /* WSAData.iMaxSockets might show the max sockets we're allowed to use.
+   * We might use it to complain if we're trying to be a server but have
+   * too few sockets available. */
+#endif /* defined(_WIN32) */
+  return 0;
+}
+
+/* When set_max_file_sockets() is called, update this with the max file
+ * descriptor value so we can use it to check the limit when opening a new
+ * socket. Default value is what Debian sets as the default hard limit. */
+static int max_sockets = 1024;
+
+/** Return the maximum number of allowed sockets. */
+int
+get_max_sockets(void)
+{
+  return max_sockets;
+}
+
+/** Set the maximum number of allowed sockets to <b>n</b> */
+void
+set_max_sockets(int n)
+{
+  max_sockets = n;
+}
+
+#undef DEBUG_SOCKET_COUNTING
+#ifdef DEBUG_SOCKET_COUNTING
+#include "lib/container/bitarray.h"
+
+/** A bitarray of all fds that should be passed to tor_socket_close(). Only
+ * used if DEBUG_SOCKET_COUNTING is defined. */
+static bitarray_t *open_sockets = NULL;
+/** The size of <b>open_sockets</b>, in bits. */
+static int max_socket = -1;
+#endif /* defined(DEBUG_SOCKET_COUNTING) */
+
+/** Count of number of sockets currently open.  (Undercounts sockets opened by
+ * eventdns and libevent.) */
+static int n_sockets_open = 0;
+
+/** Mutex to protect open_sockets, max_socket, and n_sockets_open. */
+static tor_mutex_t *socket_accounting_mutex = NULL;
+
+/** Helper: acquire the socket accounting lock. */
+static inline void
+socket_accounting_lock(void)
+{
+  if (PREDICT_UNLIKELY(!socket_accounting_mutex))
+    socket_accounting_mutex = tor_mutex_new();
+  tor_mutex_acquire(socket_accounting_mutex);
+}
+
+/** Helper: release the socket accounting lock. */
+static inline void
+socket_accounting_unlock(void)
+{
+  tor_mutex_release(socket_accounting_mutex);
+}
+
+/** As close(), but guaranteed to work for sockets across platforms (including
+ * Windows, where close()ing a socket doesn't work.  Returns 0 on success and
+ * the socket error code on failure. */
+int
+tor_close_socket_simple(tor_socket_t s)
+{
+  int r = 0;
+
+  /* On Windows, you have to call close() on fds returned by open(),
+  * and closesocket() on fds returned by socket().  On Unix, everything
+  * gets close()'d.  We abstract this difference by always using
+  * tor_close_socket to close sockets, and always using close() on
+  * files.
+  */
+  #if defined(_WIN32)
+    r = closesocket(s);
+  #else
+    r = close(s);
+  #endif
+
+  if (r != 0) {
+    int err = tor_socket_errno(-1);
+    log_info(LD_NET, "Close returned an error: %s", tor_socket_strerror(err));
+    return err;
+  }
+
+  return r;
+}
+
+/** @{ */
+#ifdef DEBUG_SOCKET_COUNTING
+/** Helper: if DEBUG_SOCKET_COUNTING is enabled, remember that <b>s</b> is
+ * now an open socket. */
+static inline void
+mark_socket_open(tor_socket_t s)
+{
+  /* XXXX This bitarray business will NOT work on windows: sockets aren't
+     small ints there. */
+  if (s > max_socket) {
+    if (max_socket == -1) {
+      open_sockets = bitarray_init_zero(s+128);
+      max_socket = s+128;
+    } else {
+      open_sockets = bitarray_expand(open_sockets, max_socket, s+128);
+      max_socket = s+128;
+    }
+  }
+  if (bitarray_is_set(open_sockets, s)) {
+    log_warn(LD_BUG, "I thought that %d was already open, but socket() just "
+             "gave it to me!", s);
+  }
+  bitarray_set(open_sockets, s);
+}
+static inline void
+mark_socket_closed(tor_socket_t s)
+{
+  if (s > max_socket || ! bitarray_is_set(open_sockets, s)) {
+    log_warn(LD_BUG, "Closing a socket (%d) that wasn't returned by tor_open_"
+             "socket(), or that was already closed or something.", s);
+  } else {
+    tor_assert(open_sockets && s <= max_socket);
+    bitarray_clear(open_sockets, s);
+  }
+}
+#else /* !(defined(DEBUG_SOCKET_COUNTING)) */
+#define mark_socket_open(s) ((void) (s))
+#define mark_socket_closed(s) ((void) (s))
+#endif /* defined(DEBUG_SOCKET_COUNTING) */
+/** @} */
+
+/** As tor_close_socket_simple(), but keeps track of the number
+ * of open sockets. Returns 0 on success, -1 on failure. */
+MOCK_IMPL(int,
+tor_close_socket,(tor_socket_t s))
+{
+  int r = tor_close_socket_simple(s);
+
+  socket_accounting_lock();
+  mark_socket_closed(s);
+  if (r == 0) {
+    --n_sockets_open;
+  } else {
+#ifdef _WIN32
+    if (r != WSAENOTSOCK)
+      --n_sockets_open;
+#else
+    if (r != EBADF)
+      --n_sockets_open; // LCOV_EXCL_LINE -- EIO and EINTR too hard to force.
+#endif /* defined(_WIN32) */
+    r = -1;
+  }
+
+  tor_assert_nonfatal(n_sockets_open >= 0);
+  socket_accounting_unlock();
+  return r;
+}
+
+/** As socket(), but counts the number of open sockets. */
+MOCK_IMPL(tor_socket_t,
+tor_open_socket,(int domain, int type, int protocol))
+{
+  return tor_open_socket_with_extensions(domain, type, protocol, 1, 0);
+}
+
+/** Mockable wrapper for connect(). */
+MOCK_IMPL(tor_socket_t,
+tor_connect_socket,(tor_socket_t sock, const struct sockaddr *address,
+                     socklen_t address_len))
+{
+  return connect(sock,address,address_len);
+}
+
+/** As socket(), but creates a nonblocking socket and
+ * counts the number of open sockets. */
+tor_socket_t
+tor_open_socket_nonblocking(int domain, int type, int protocol)
+{
+  return tor_open_socket_with_extensions(domain, type, protocol, 1, 1);
+}
+
+/** As socket(), but counts the number of open sockets and handles
+ * socket creation with either of SOCK_CLOEXEC and SOCK_NONBLOCK specified.
+ * <b>cloexec</b> and <b>nonblock</b> should be either 0 or 1 to indicate
+ * if the corresponding extension should be used.*/
+tor_socket_t
+tor_open_socket_with_extensions(int domain, int type, int protocol,
+                                int cloexec, int nonblock)
+{
+  tor_socket_t s;
+
+  /* We are about to create a new file descriptor so make sure we have
+   * enough of them. */
+  if (get_n_open_sockets() >= max_sockets - 1) {
+#ifdef _WIN32
+    WSASetLastError(WSAEMFILE);
+#else
+    errno = EMFILE;
+#endif
+    return TOR_INVALID_SOCKET;
+  }
+
+#if defined(SOCK_CLOEXEC) && defined(SOCK_NONBLOCK)
+  int ext_flags = (cloexec ? SOCK_CLOEXEC : 0) |
+                  (nonblock ? SOCK_NONBLOCK : 0);
+  s = socket(domain, type|ext_flags, protocol);
+  if (SOCKET_OK(s))
+    goto socket_ok;
+  /* If we got an error, see if it is EINVAL. EINVAL might indicate that,
+   * even though we were built on a system with SOCK_CLOEXEC and SOCK_NONBLOCK
+   * support, we are running on one without. */
+  if (errno != EINVAL)
+    return s;
+#endif /* defined(SOCK_CLOEXEC) && defined(SOCK_NONBLOCK) */
+
+  s = socket(domain, type, protocol);
+  if (! SOCKET_OK(s))
+    return s;
+
+#if defined(FD_CLOEXEC)
+  if (cloexec) {
+    if (fcntl(s, F_SETFD, FD_CLOEXEC) == -1) {
+      log_warn(LD_FS,"Couldn't set FD_CLOEXEC: %s", strerror(errno));
+      tor_close_socket_simple(s);
+      return TOR_INVALID_SOCKET;
+    }
+  }
+#else /* !(defined(FD_CLOEXEC)) */
+  (void)cloexec;
+#endif /* defined(FD_CLOEXEC) */
+
+  if (nonblock) {
+    if (set_socket_nonblocking(s) == -1) {
+      tor_close_socket_simple(s);
+      return TOR_INVALID_SOCKET;
+    }
+  }
+
+  goto socket_ok; /* So that socket_ok will not be unused. */
+
+ socket_ok:
+  tor_take_socket_ownership(s);
+  return s;
+}
+
+/**
+ * For socket accounting: remember that we are the owner of the socket
+ * <b>s</b>. This will prevent us from overallocating sockets, and prevent us
+ * from asserting later when we close the socket <b>s</b>.
+ */
+void
+tor_take_socket_ownership(tor_socket_t s)
+{
+  socket_accounting_lock();
+  ++n_sockets_open;
+  mark_socket_open(s);
+  socket_accounting_unlock();
+}
+
+/**
+ * For socket accounting: declare that we are no longer the owner of the
+ * socket <b>s</b>. This will prevent us from overallocating sockets, and
+ * prevent us from asserting later when we close the socket <b>s</b>.
+ */
+void
+tor_release_socket_ownership(tor_socket_t s)
+{
+  socket_accounting_lock();
+  --n_sockets_open;
+  mark_socket_closed(s);
+  socket_accounting_unlock();
+}
+
+/** As accept(), but counts the number of open sockets. */
+tor_socket_t
+tor_accept_socket(tor_socket_t sockfd, struct sockaddr *addr, socklen_t *len)
+{
+  return tor_accept_socket_with_extensions(sockfd, addr, len, 1, 0);
+}
+
+/** As accept(), but returns a nonblocking socket and
+ * counts the number of open sockets. */
+tor_socket_t
+tor_accept_socket_nonblocking(tor_socket_t sockfd, struct sockaddr *addr,
+                              socklen_t *len)
+{
+  return tor_accept_socket_with_extensions(sockfd, addr, len, 1, 1);
+}
+
+/** As accept(), but counts the number of open sockets and handles
+ * socket creation with either of SOCK_CLOEXEC and SOCK_NONBLOCK specified.
+ * <b>cloexec</b> and <b>nonblock</b> should be either 0 or 1 to indicate
+ * if the corresponding extension should be used.*/
+tor_socket_t
+tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr,
+                                 socklen_t *len, int cloexec, int nonblock)
+{
+  tor_socket_t s;
+
+  /* We are about to create a new file descriptor so make sure we have
+   * enough of them. */
+  if (get_n_open_sockets() >= max_sockets - 1) {
+#ifdef _WIN32
+    WSASetLastError(WSAEMFILE);
+#else
+    errno = EMFILE;
+#endif
+    return TOR_INVALID_SOCKET;
+  }
+
+#if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) \
+  && defined(SOCK_NONBLOCK)
+  int ext_flags = (cloexec ? SOCK_CLOEXEC : 0) |
+                  (nonblock ? SOCK_NONBLOCK : 0);
+  s = accept4(sockfd, addr, len, ext_flags);
+  if (SOCKET_OK(s))
+    goto socket_ok;
+  /* If we got an error, see if it is ENOSYS. ENOSYS indicates that,
+   * even though we were built on a system with accept4 support, we
+   * are running on one without. Also, check for EINVAL, which indicates that
+   * we are missing SOCK_CLOEXEC/SOCK_NONBLOCK support. */
+  if (errno != EINVAL && errno != ENOSYS)
+    return s;
+#endif /* defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) ... */
+
+  s = accept(sockfd, addr, len);
+  if (!SOCKET_OK(s))
+    return s;
+
+#if defined(FD_CLOEXEC)
+  if (cloexec) {
+    if (fcntl(s, F_SETFD, FD_CLOEXEC) == -1) {
+      log_warn(LD_NET, "Couldn't set FD_CLOEXEC: %s", strerror(errno));
+      tor_close_socket_simple(s);
+      return TOR_INVALID_SOCKET;
+    }
+  }
+#else /* !(defined(FD_CLOEXEC)) */
+  (void)cloexec;
+#endif /* defined(FD_CLOEXEC) */
+
+  if (nonblock) {
+    if (set_socket_nonblocking(s) == -1) {
+      tor_close_socket_simple(s);
+      return TOR_INVALID_SOCKET;
+    }
+  }
+
+  goto socket_ok; /* So that socket_ok will not be unused. */
+
+ socket_ok:
+  tor_take_socket_ownership(s);
+  return s;
+}
+
+/** Return the number of sockets we currently have opened. */
+int
+get_n_open_sockets(void)
+{
+  int n;
+  socket_accounting_lock();
+  n = n_sockets_open;
+  socket_accounting_unlock();
+  return n;
+}
+
+/**
+ * Allocate a pair of connected sockets.  (Like socketpair(family,
+ * type,protocol,fd), but works on systems that don't have
+ * socketpair.)
+ *
+ * Currently, only (AF_UNIX, SOCK_STREAM, 0) sockets are supported.
+ *
+ * Note that on systems without socketpair, this call will fail if
+ * localhost is inaccessible (for example, if the networking
+ * stack is down). And even if it succeeds, the socket pair will not
+ * be able to read while localhost is down later (the socket pair may
+ * even close, depending on OS-specific timeouts).
+ *
+ * Returns 0 on success and -errno on failure; do not rely on the value
+ * of errno or WSAGetLastError().
+ **/
+/* It would be nicer just to set errno, but that won't work for windows. */
+int
+tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2])
+{
+  int r;
+//don't use win32 socketpairs (they are always bad)
+#if defined(HAVE_SOCKETPAIR) && !defined(_WIN32)
+
+#ifdef SOCK_CLOEXEC
+  r = socketpair(family, type|SOCK_CLOEXEC, protocol, fd);
+  if (r == 0)
+    goto sockets_ok;
+  /* If we got an error, see if it is EINVAL. EINVAL might indicate that,
+   * even though we were built on a system with SOCK_CLOEXEC support, we
+   * are running on one without. */
+  if (errno != EINVAL)
+    return -errno;
+#endif /* defined(SOCK_CLOEXEC) */
+
+  r = socketpair(family, type, protocol, fd);
+  if (r < 0)
+    return -errno;
+#else
+  r = tor_ersatz_socketpair(family, type, protocol, fd);
+  if (r < 0)
+    return -r;
+#endif
+
+#if defined(FD_CLOEXEC)
+  if (SOCKET_OK(fd[0])) {
+    r = fcntl(fd[0], F_SETFD, FD_CLOEXEC);
+    if (r == -1) {
+      close(fd[0]);
+      close(fd[1]);
+      return -errno;
+    }
+  }
+  if (SOCKET_OK(fd[1])) {
+    r = fcntl(fd[1], F_SETFD, FD_CLOEXEC);
+    if (r == -1) {
+      close(fd[0]);
+      close(fd[1]);
+      return -errno;
+    }
+  }
+#endif /* defined(FD_CLOEXEC) */
+  goto sockets_ok; /* So that sockets_ok will not be unused. */
+
+ sockets_ok:
+  socket_accounting_lock();
+  if (SOCKET_OK(fd[0])) {
+    ++n_sockets_open;
+    mark_socket_open(fd[0]);
+  }
+  if (SOCKET_OK(fd[1])) {
+    ++n_sockets_open;
+    mark_socket_open(fd[1]);
+  }
+  socket_accounting_unlock();
+
+  return 0;
+}
+
+/** Mockable wrapper for getsockname(). */
+MOCK_IMPL(int,
+tor_getsockname,(tor_socket_t sock, struct sockaddr *address,
+                 socklen_t *address_len))
+{
+   return getsockname(sock, address, address_len);
+}
+
+/**
+ * Find the local address associated with the socket <b>sock</b>, and
+ * place it in *<b>addr_out</b>.  Return 0 on success, -1 on failure.
+ *
+ * (As tor_getsockname, but instead places the result in a tor_addr_t.) */
+int
+tor_addr_from_getsockname(struct tor_addr_t *addr_out, tor_socket_t sock)
+{
+  struct sockaddr_storage ss;
+  socklen_t ss_len = sizeof(ss);
+  memset(&ss, 0, sizeof(ss));
+
+  if (tor_getsockname(sock, (struct sockaddr *) &ss, &ss_len) < 0)
+    return -1;
+
+  return tor_addr_from_sockaddr(addr_out, (struct sockaddr *)&ss, NULL);
+}
+
+/** Turn <b>socket</b> into a nonblocking socket. Return 0 on success, -1
+ * on failure.
+ */
+int
+set_socket_nonblocking(tor_socket_t sock)
+{
+#if defined(_WIN32)
+  unsigned long nonblocking = 1;
+  ioctlsocket(sock, FIONBIO, (unsigned long*) &nonblocking);
+#else
+  int flags;
+
+  flags = fcntl(sock, F_GETFL, 0);
+  if (flags == -1) {
+    log_warn(LD_NET, "Couldn't get file status flags: %s", strerror(errno));
+    return -1;
+  }
+  flags |= O_NONBLOCK;
+  if (fcntl(sock, F_SETFL, flags) == -1) {
+    log_warn(LD_NET, "Couldn't set file status flags: %s", strerror(errno));
+    return -1;
+  }
+#endif /* defined(_WIN32) */
+
+  return 0;
+}
+
+/** Read from <b>sock</b> to <b>buf</b>, until we get <b>count</b> bytes or
+ * reach the end of the file.  Return the number of bytes read, or -1 on
+ * error. Only use if fd is a blocking fd. */
+ssize_t
+read_all_from_socket(tor_socket_t sock, char *buf, size_t count)
+{
+  size_t numread = 0;
+  ssize_t result;
+
+  if (count > SIZE_T_CEILING || count > SSIZE_MAX) {
+    errno = EINVAL;
+    return -1;
+  }
+
+  while (numread < count) {
+    result = tor_socket_recv(sock, buf+numread, count-numread, 0);
+    if (result<0)
+      return -1;
+    else if (result == 0)
+      break;
+    numread += result;
+  }
+  return (ssize_t)numread;
+}
+
+/** Write <b>count</b> bytes from <b>buf</b> to <b>sock</b>. Return the number
+ * of bytes written, or -1 on error.  Only use if fd is a blocking fd.  */
+ssize_t
+write_all_to_socket(tor_socket_t fd, const char *buf, size_t count)
+{
+  size_t written = 0;
+  ssize_t result;
+  raw_assert(count < SSIZE_MAX);
+
+  while (written != count) {
+    result = tor_socket_send(fd, buf+written, count-written, 0);
+    if (result<0)
+      return -1;
+    written += result;
+  }
+  return (ssize_t)count;
+}
+
+/**
+ * On Windows, WSAEWOULDBLOCK is not always correct: when you see it,
+ * you need to ask the socket for its actual errno.  Also, you need to
+ * get your errors from WSAGetLastError, not errno.  (If you supply a
+ * socket of -1, we check WSAGetLastError, but don't correct
+ * WSAEWOULDBLOCKs.)
+ *
+ * The upshot of all of this is that when a socket call fails, you
+ * should call tor_socket_errno <em>at most once</em> on the failing
+ * socket to get the error.
+ */
+#if defined(_WIN32)
+int
+tor_socket_errno(tor_socket_t sock)
+{
+  int optval, optvallen=sizeof(optval);
+  int err = WSAGetLastError();
+  if (err == WSAEWOULDBLOCK && SOCKET_OK(sock)) {
+    if (getsockopt(sock, SOL_SOCKET, SO_ERROR, (void*)&optval, &optvallen))
+      return err;
+    if (optval)
+      return optval;
+  }
+  return err;
+}
+#endif /* defined(_WIN32) */
+
+#if defined(_WIN32)
+#define E(code, s) { code, (s " [" #code " ]") }
+struct { int code; const char *msg; } windows_socket_errors[] = {
+  E(WSAEINTR, "Interrupted function call"),
+  E(WSAEACCES, "Permission denied"),
+  E(WSAEFAULT, "Bad address"),
+  E(WSAEINVAL, "Invalid argument"),
+  E(WSAEMFILE, "Too many open files"),
+  E(WSAEWOULDBLOCK,  "Resource temporarily unavailable"),
+  E(WSAEINPROGRESS, "Operation now in progress"),
+  E(WSAEALREADY, "Operation already in progress"),
+  E(WSAENOTSOCK, "Socket operation on nonsocket"),
+  E(WSAEDESTADDRREQ, "Destination address required"),
+  E(WSAEMSGSIZE, "Message too long"),
+  E(WSAEPROTOTYPE, "Protocol wrong for socket"),
+  E(WSAENOPROTOOPT, "Bad protocol option"),
+  E(WSAEPROTONOSUPPORT, "Protocol not supported"),
+  E(WSAESOCKTNOSUPPORT, "Socket type not supported"),
+  /* What's the difference between NOTSUPP and NOSUPPORT? :) */
+  E(WSAEOPNOTSUPP, "Operation not supported"),
+  E(WSAEPFNOSUPPORT,  "Protocol family not supported"),
+  E(WSAEAFNOSUPPORT, "Address family not supported by protocol family"),
+  E(WSAEADDRINUSE, "Address already in use"),
+  E(WSAEADDRNOTAVAIL, "Cannot assign requested address"),
+  E(WSAENETDOWN, "Network is down"),
+  E(WSAENETUNREACH, "Network is unreachable"),
+  E(WSAENETRESET, "Network dropped connection on reset"),
+  E(WSAECONNABORTED, "Software caused connection abort"),
+  E(WSAECONNRESET, "Connection reset by peer"),
+  E(WSAENOBUFS, "No buffer space available"),
+  E(WSAEISCONN, "Socket is already connected"),
+  E(WSAENOTCONN, "Socket is not connected"),
+  E(WSAESHUTDOWN, "Cannot send after socket shutdown"),
+  E(WSAETIMEDOUT, "Connection timed out"),
+  E(WSAECONNREFUSED, "Connection refused"),
+  E(WSAEHOSTDOWN, "Host is down"),
+  E(WSAEHOSTUNREACH, "No route to host"),
+  E(WSAEPROCLIM, "Too many processes"),
+  /* Yes, some of these start with WSA, not WSAE. No, I don't know why. */
+  E(WSASYSNOTREADY, "Network subsystem is unavailable"),
+  E(WSAVERNOTSUPPORTED, "Winsock.dll out of range"),
+  E(WSANOTINITIALISED, "Successful WSAStartup not yet performed"),
+  E(WSAEDISCON, "Graceful shutdown now in progress"),
+#ifdef WSATYPE_NOT_FOUND
+  E(WSATYPE_NOT_FOUND, "Class type not found"),
+#endif
+  E(WSAHOST_NOT_FOUND, "Host not found"),
+  E(WSATRY_AGAIN, "Nonauthoritative host not found"),
+  E(WSANO_RECOVERY, "This is a nonrecoverable error"),
+  E(WSANO_DATA, "Valid name, no data record of requested type)"),
+
+  /* There are some more error codes whose numeric values are marked
+   * <b>OS dependent</b>. They start with WSA_, apparently for the same
+   * reason that practitioners of some craft traditions deliberately
+   * introduce imperfections into their baskets and rugs "to allow the
+   * evil spirits to escape."  If we catch them, then our binaries
+   * might not report consistent results across versions of Windows.
+   * Thus, I'm going to let them all fall through.
+   */
+  { -1, NULL },
+};
+/** There does not seem to be a strerror equivalent for Winsock errors.
+ * Naturally, we have to roll our own.
+ */
+const char *
+tor_socket_strerror(int e)
+{
+  int i;
+  for (i=0; windows_socket_errors[i].code >= 0; ++i) {
+    if (e == windows_socket_errors[i].code)
+      return windows_socket_errors[i].msg;
+  }
+  return strerror(e);
+}
+#endif /* defined(_WIN32) */
diff --git a/src/lib/net/socket.h b/src/lib/net/socket.h
new file mode 100644
index 0000000000..0909619510
--- /dev/null
+++ b/src/lib/net/socket.h
@@ -0,0 +1,118 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file socket.h
+ * \brief Header for socket.c
+ **/
+
+#ifndef TOR_SOCKET_H
+#define TOR_SOCKET_H
+
+#include "orconfig.h"
+#include "lib/cc/torint.h"
+#include "lib/net/nettypes.h"
+#include "lib/testsupport/testsupport.h"
+
+#include <errno.h>
+
+struct sockaddr;
+
+int tor_close_socket_simple(tor_socket_t s);
+MOCK_DECL(int, tor_close_socket, (tor_socket_t s));
+void tor_take_socket_ownership(tor_socket_t s);
+void tor_release_socket_ownership(tor_socket_t s);
+tor_socket_t tor_open_socket_with_extensions(
+                                           int domain, int type, int protocol,
+                                           int cloexec, int nonblock);
+MOCK_DECL(tor_socket_t,tor_open_socket,(int domain, int type, int protocol));
+tor_socket_t tor_open_socket_nonblocking(int domain, int type, int protocol);
+tor_socket_t tor_accept_socket(tor_socket_t sockfd, struct sockaddr *addr,
+                                  socklen_t *len);
+tor_socket_t tor_accept_socket_nonblocking(tor_socket_t sockfd,
+                                           struct sockaddr *addr,
+                                           socklen_t *len);
+tor_socket_t tor_accept_socket_with_extensions(tor_socket_t sockfd,
+                                               struct sockaddr *addr,
+                                               socklen_t *len,
+                                               int cloexec, int nonblock);
+MOCK_DECL(tor_socket_t, tor_connect_socket,(tor_socket_t socket,
+                                            const struct sockaddr *address,
+                                            socklen_t address_len));
+int get_n_open_sockets(void);
+
+MOCK_DECL(int,tor_getsockname,(tor_socket_t socket, struct sockaddr *address,
+                 socklen_t *address_len));
+struct tor_addr_t;
+int tor_addr_from_getsockname(struct tor_addr_t *addr_out, tor_socket_t sock);
+
+#define tor_socket_send(s, buf, len, flags) send(s, buf, len, flags)
+#define tor_socket_recv(s, buf, len, flags) recv(s, buf, len, flags)
+
+int set_socket_nonblocking(tor_socket_t socket);
+int tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]);
+int network_init(void);
+
+int get_max_sockets(void);
+void set_max_sockets(int);
+
+ssize_t write_all_to_socket(tor_socket_t fd, const char *buf, size_t count);
+ssize_t read_all_from_socket(tor_socket_t fd, char *buf, size_t count);
+
+/* For stupid historical reasons, windows sockets have an independent
+ * set of errnos, and an independent way to get them.  Also, you can't
+ * always believe WSAEWOULDBLOCK.  Use the macros below to compare
+ * errnos against expected values, and use tor_socket_errno to find
+ * the actual errno after a socket operation fails.
+ */
+#if defined(_WIN32)
+/** Expands to WSA<b>e</b> on Windows, and to <b>e</b> elsewhere. */
+#define SOCK_ERRNO(e) WSA##e
+/** Return true if e is EAGAIN or the local equivalent. */
+#define ERRNO_IS_EAGAIN(e)           ((e) == EAGAIN || (e) == WSAEWOULDBLOCK)
+/** Return true if e is EINPROGRESS or the local equivalent. */
+#define ERRNO_IS_EINPROGRESS(e)      ((e) == WSAEINPROGRESS)
+/** Return true if e is EINPROGRESS or the local equivalent as returned by
+ * a call to connect(). */
+#define ERRNO_IS_CONN_EINPROGRESS(e) \
+  ((e) == WSAEINPROGRESS || (e)== WSAEINVAL || (e) == WSAEWOULDBLOCK)
+/** Return true if e is EAGAIN or another error indicating that a call to
+ * accept() has no pending connections to return. */
+#define ERRNO_IS_ACCEPT_EAGAIN(e)    ERRNO_IS_EAGAIN(e)
+/** Return true if e is EMFILE or another error indicating that a call to
+ * accept() has failed because we're out of fds or something. */
+#define ERRNO_IS_RESOURCE_LIMIT(e) \
+  ((e) == WSAEMFILE || (e) == WSAENOBUFS)
+/** Return true if e is EADDRINUSE or the local equivalent. */
+#define ERRNO_IS_EADDRINUSE(e)      ((e) == WSAEADDRINUSE)
+/** Return true if e is EINTR  or the local equivalent */
+#define ERRNO_IS_EINTR(e)            ((e) == WSAEINTR || 0)
+int tor_socket_errno(tor_socket_t sock);
+const char *tor_socket_strerror(int e);
+#else /* !(defined(_WIN32)) */
+#define SOCK_ERRNO(e) e
+#if EAGAIN == EWOULDBLOCK
+/* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */
+#define ERRNO_IS_EAGAIN(e)           ((e) == EAGAIN || 0)
+#else
+#define ERRNO_IS_EAGAIN(e)           ((e) == EAGAIN || (e) == EWOULDBLOCK)
+#endif /* EAGAIN == EWOULDBLOCK */
+#define ERRNO_IS_EINTR(e)            ((e) == EINTR || 0)
+#define ERRNO_IS_EINPROGRESS(e)      ((e) == EINPROGRESS || 0)
+#define ERRNO_IS_CONN_EINPROGRESS(e) ((e) == EINPROGRESS || 0)
+#define ERRNO_IS_ACCEPT_EAGAIN(e) \
+  (ERRNO_IS_EAGAIN(e) || (e) == ECONNABORTED)
+#define ERRNO_IS_RESOURCE_LIMIT(e) \
+  ((e) == EMFILE || (e) == ENFILE || (e) == ENOBUFS || (e) == ENOMEM)
+#define ERRNO_IS_EADDRINUSE(e)       (((e) == EADDRINUSE) || 0)
+#define tor_socket_errno(sock)       (errno)
+#define tor_socket_strerror(e)       strerror(e)
+#endif /* defined(_WIN32) */
+
+#if defined(_WIN32) && !defined(SIO_IDEAL_SEND_BACKLOG_QUERY)
+#define SIO_IDEAL_SEND_BACKLOG_QUERY 0x4004747b
+#endif
+
+#endif
diff --git a/src/lib/net/socketpair.c b/src/lib/net/socketpair.c
new file mode 100644
index 0000000000..10eb749735
--- /dev/null
+++ b/src/lib/net/socketpair.c
@@ -0,0 +1,214 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+
+#include "lib/cc/torint.h"
+#include "lib/net/socketpair.h"
+#include "lib/net/inaddr_st.h"
+#include "lib/arch/bytes.h"
+
+#include <errno.h>
+#include <string.h>
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+
+#ifdef _WIN32
+#include <winsock2.h>
+#include <windows.h>
+#define socket_errno() (WSAGetLastError())
+#define SOCKET_EPROTONOSUPPORT WSAEPROTONOSUPPORT
+#else
+#define closesocket(x) close(x)
+#define socket_errno() (errno)
+#define SOCKET_EPROTONOSUPPORT EPROTONOSUPPORT
+#endif
+
+#ifdef NEED_ERSATZ_SOCKETPAIR
+
+// Avoid warning about call to memcmp.
+#define raw_memcmp memcmp
+
+/**
+ * Return a new socket that is bound and listening on the loopback interface
+ * of family <b>family</b> for a socket of type <b>type</b>. On failure return
+ * TOR_INVALID_SOCKET.
+ */
+static tor_socket_t
+get_local_listener(int family, int type)
+{
+  struct sockaddr_in sin;
+  struct sockaddr_in6 sin6;
+  struct sockaddr *sa;
+  int len;
+
+  memset(&sin, 0, sizeof(sin));
+  memset(&sin6, 0, sizeof(sin6));
+
+  tor_socket_t sock = TOR_INVALID_SOCKET;
+  sock = socket(family, type, 0);
+  if (!SOCKET_OK(sock)) {
+    return TOR_INVALID_SOCKET;
+  }
+
+  if (family == AF_INET) {
+    sa = (struct sockaddr *) &sin;
+    sin.sin_family = AF_INET;
+    sin.sin_addr.s_addr = tor_htonl(0x7f000001);
+    len = sizeof(sin);
+  } else {
+    sa = (struct sockaddr *) &sin6;
+    sin6.sin6_family = AF_INET6;
+    sin6.sin6_addr.s6_addr[15] = 1;
+    len = sizeof(sin6);
+  }
+
+  if (bind(sock, sa, len) == -1)
+    goto err;
+  if (listen(sock, 1) == -1)
+    goto err;
+
+  return sock;
+ err:
+  closesocket(sock);
+  return TOR_INVALID_SOCKET;
+}
+
+/**
+ * Return true iff sa1 and sa2 are equivalent AF_INET or AF_INET6 addresses.
+ */
+static int
+sockaddr_eq(struct sockaddr *sa1, struct sockaddr *sa2)
+{
+  if (sa1->sa_family != sa2->sa_family)
+    return 0;
+
+  if (sa1->sa_family == AF_INET6) {
+    struct sockaddr_in6 *sin6_1 = (struct sockaddr_in6 *) sa1;
+    struct sockaddr_in6 *sin6_2 = (struct sockaddr_in6 *) sa2;
+    return sin6_1->sin6_port == sin6_2->sin6_port &&
+      0==raw_memcmp(sin6_1->sin6_addr.s6_addr, sin6_2->sin6_addr.s6_addr, 16);
+  } else if (sa1->sa_family == AF_INET) {
+    struct sockaddr_in *sin_1 = (struct sockaddr_in *) sa1;
+    struct sockaddr_in *sin_2 = (struct sockaddr_in *) sa2;
+    return sin_1->sin_port == sin_2->sin_port &&
+      sin_1->sin_addr.s_addr == sin_2->sin_addr.s_addr;
+  } else {
+    return 0;
+  }
+}
+
+/**
+ * Helper used to implement socketpair on systems that lack it, by
+ * making a direct connection to localhost.
+ */
+int
+tor_ersatz_socketpair(int family, int type, int protocol, tor_socket_t fd[2])
+{
+  /* This socketpair does not work when localhost is down. So
+   * it's really not the same thing at all. But it's close enough
+   * for now, and really, when localhost is down sometimes, we
+   * have other problems too.
+   */
+  tor_socket_t listener = TOR_INVALID_SOCKET;
+  tor_socket_t connector = TOR_INVALID_SOCKET;
+  tor_socket_t acceptor = TOR_INVALID_SOCKET;
+  struct sockaddr_storage accepted_addr_ss;
+  struct sockaddr_storage connect_addr_ss;
+  struct sockaddr *connect_addr = (struct sockaddr *) &connect_addr_ss;
+  struct sockaddr *accepted_addr = (struct sockaddr *) &accepted_addr_ss;
+  socklen_t size;
+  int saved_errno = -1;
+  int ersatz_domain = AF_INET;
+  socklen_t addrlen = sizeof(struct sockaddr_in);
+
+  memset(&accepted_addr_ss, 0, sizeof(accepted_addr_ss));
+  memset(&connect_addr_ss, 0, sizeof(connect_addr_ss));
+
+  if (protocol
+#ifdef AF_UNIX
+      || family != AF_UNIX
+#endif
+      ) {
+#ifdef _WIN32
+    return -WSAEAFNOSUPPORT;
+#else
+    return -EAFNOSUPPORT;
+#endif
+  }
+  if (!fd) {
+    return -EINVAL;
+  }
+
+  listener = get_local_listener(ersatz_domain, type);
+  if (!SOCKET_OK(listener)) {
+    int first_errno = socket_errno();
+    if (first_errno == SOCKET_EPROTONOSUPPORT) {
+      /* Assume we're on an IPv6-only system */
+      ersatz_domain = AF_INET6;
+      addrlen = sizeof(struct sockaddr_in6);
+      listener = get_local_listener(ersatz_domain, type);
+    }
+    if (!SOCKET_OK(listener)) {
+      /* Keep the previous behaviour, which was to return the IPv4 error.
+       * (This may be less informative on IPv6-only systems.)
+       * XX/teor - is there a better way to decide which errno to return?
+       * (I doubt we care much either way, once there is an error.)
+       */
+      return -first_errno;
+    }
+  }
+
+  connector = socket(ersatz_domain, type, 0);
+  if (!SOCKET_OK(connector))
+    goto tidy_up_and_fail;
+  /* We want to find out the port number to connect to.  */
+  size = sizeof(connect_addr_ss);
+  if (getsockname(listener, connect_addr, &size) == -1)
+    goto tidy_up_and_fail;
+  if (size != addrlen)
+    goto abort_tidy_up_and_fail;
+  if (connect(connector, connect_addr, size) == -1)
+    goto tidy_up_and_fail;
+
+  size = sizeof(accepted_addr_ss);
+  acceptor = accept(listener, accepted_addr, &size);
+  if (!SOCKET_OK(acceptor))
+    goto tidy_up_and_fail;
+  if (size != addrlen)
+    goto abort_tidy_up_and_fail;
+  /* Now check we are talking to ourself by matching port and host on the
+     two sockets.  */
+  if (getsockname(connector, connect_addr, &size) == -1)
+    goto tidy_up_and_fail;
+  /* Set *_tor_addr and *_port to the address and port that was used */
+  if (!sockaddr_eq(accepted_addr, connect_addr))
+    goto abort_tidy_up_and_fail;
+  closesocket(listener);
+  fd[0] = connector;
+  fd[1] = acceptor;
+  return 0;
+
+ abort_tidy_up_and_fail:
+#ifdef _WIN32
+  saved_errno = WSAECONNABORTED;
+#else
+  saved_errno = ECONNABORTED; /* I hope this is portable and appropriate.  */
+#endif
+ tidy_up_and_fail:
+  if (saved_errno < 0)
+    saved_errno = errno;
+  if (SOCKET_OK(listener))
+    closesocket(listener);
+  if (SOCKET_OK(connector))
+    closesocket(connector);
+  if (SOCKET_OK(acceptor))
+    closesocket(acceptor);
+  return -saved_errno;
+}
+
+#endif /* defined(NEED_ERSATZ_SOCKETPAIR) */
diff --git a/src/lib/net/socketpair.h b/src/lib/net/socketpair.h
new file mode 100644
index 0000000000..6be0803881
--- /dev/null
+++ b/src/lib/net/socketpair.h
@@ -0,0 +1,19 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_SOCKETPAIR_H
+#define TOR_SOCKETPAIR_H
+
+#include "orconfig.h"
+#include "lib/testsupport/testsupport.h"
+#include "lib/net/nettypes.h"
+
+#if !defined(HAVE_SOCKETPAIR) || defined(_WIN32) || defined(TOR_UNIT_TESTS)
+#define NEED_ERSATZ_SOCKETPAIR
+int tor_ersatz_socketpair(int family, int type, int protocol,
+                          tor_socket_t fd[2]);
+#endif
+
+#endif
diff --git a/src/lib/net/socks5_status.h b/src/lib/net/socks5_status.h
new file mode 100644
index 0000000000..e55242ce66
--- /dev/null
+++ b/src/lib/net/socks5_status.h
@@ -0,0 +1,32 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file socks5_status.h
+ * \brief Status codes used by the SOCKS5 protocol.
+ **/
+
+/* NOTE: it probably isn't necessary to put this header in lib/net, but
+ * we need it in _some_ lower-level layer for now, since it is used by
+ * tools/tor-resolve.c.
+ */
+
+#ifndef TOR_SOCKS5_STATUS_H
+#define TOR_SOCKS5_STATUS_H
+
+/** Specified SOCKS5 status codes. */
+typedef enum {
+  SOCKS5_SUCCEEDED                  = 0x00,
+  SOCKS5_GENERAL_ERROR              = 0x01,
+  SOCKS5_NOT_ALLOWED                = 0x02,
+  SOCKS5_NET_UNREACHABLE            = 0x03,
+  SOCKS5_HOST_UNREACHABLE           = 0x04,
+  SOCKS5_CONNECTION_REFUSED         = 0x05,
+  SOCKS5_TTL_EXPIRED                = 0x06,
+  SOCKS5_COMMAND_NOT_SUPPORTED      = 0x07,
+  SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED = 0x08,
+} socks5_reply_status_t;
+
+#endif
diff --git a/src/lib/osinfo/.may_include b/src/lib/osinfo/.may_include
new file mode 100644
index 0000000000..058f6eb89c
--- /dev/null
+++ b/src/lib/osinfo/.may_include
@@ -0,0 +1,5 @@
+orconfig.h
+
+lib/osinfo/*.h
+lib/string/*.h
+lib/testsupport/*.h
diff --git a/src/lib/osinfo/include.am b/src/lib/osinfo/include.am
new file mode 100644
index 0000000000..16c5812604
--- /dev/null
+++ b/src/lib/osinfo/include.am
@@ -0,0 +1,17 @@
+
+noinst_LIBRARIES += src/lib/libtor-osinfo.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-osinfo-testing.a
+endif
+
+src_lib_libtor_osinfo_a_SOURCES =			\
+	src/lib/osinfo/uname.c
+
+src_lib_libtor_osinfo_testing_a_SOURCES = \
+	$(src_lib_libtor_osinfo_a_SOURCES)
+src_lib_libtor_osinfo_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_osinfo_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/osinfo/uname.h
diff --git a/src/lib/osinfo/uname.c b/src/lib/osinfo/uname.c
new file mode 100644
index 0000000000..2b37ff136c
--- /dev/null
+++ b/src/lib/osinfo/uname.c
@@ -0,0 +1,149 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file uname.c
+ * \brief Look up a description of the operating system.
+ **/
+
+#include "orconfig.h"
+#include "lib/osinfo/uname.h"
+
+#include "lib/string/compat_string.h"
+#include "lib/string/printf.h"
+
+#ifdef HAVE_UNAME
+#include <sys/utsname.h>
+#endif
+#ifdef _WIN32
+#include <windows.h>
+#endif
+#include <string.h>
+
+/** Hold the result of our call to <b>uname</b>. */
+static char uname_result[256];
+/** True iff uname_result is set. */
+static int uname_result_is_set = 0;
+
+/** Return a pointer to a description of our platform.
+ */
+MOCK_IMPL(const char *,
+get_uname,(void))
+{
+#ifdef HAVE_UNAME
+  struct utsname u;
+#endif
+  if (!uname_result_is_set) {
+#ifdef HAVE_UNAME
+    if (uname(&u) != -1) {
+      /* (Linux says 0 is success, Solaris says 1 is success) */
+      strlcpy(uname_result, u.sysname, sizeof(uname_result));
+    } else
+#endif /* defined(HAVE_UNAME) */
+      {
+#ifdef _WIN32
+        OSVERSIONINFOEX info;
+        int i;
+        int is_client = 0;
+        int is_server = 0;
+        const char *plat = NULL;
+        static struct {
+          unsigned major; unsigned minor;
+          const char *client_version; const char *server_version;
+        } win_version_table[] = {
+    /* This table must be sorted in descending order.
+     * Sources:
+     *   https://en.wikipedia.org/wiki/List_of_Microsoft_Windows_versions
+     *   https://docs.microsoft.com/en-us/windows/desktop/api/winnt/
+     *     ns-winnt-_osversioninfoexa#remarks
+     */
+    /* Windows Server 2019 is indistinguishable from Windows Server 2016
+     * using GetVersionEx().
+    { 10,  0, NULL,                        "Windows Server 2019" }, */
+    { 10,  0, "Windows 10",                "Windows Server 2016" },
+    {  6,  3, "Windows 8.1",               "Windows Server 2012 R2" },
+    {  6,  2, "Windows 8",                 "Windows Server 2012" },
+    {  6,  1, "Windows 7",                 "Windows Server 2008 R2" },
+    {  6,  0, "Windows Vista",             "Windows Server 2008" },
+    {  5,  2, "Windows XP Professional",   "Windows Server 2003" },
+    /* Windows XP did not have a server version, but we need something here */
+    {  5,  1, "Windows XP",                "Windows XP Server" },
+    {  5,  0, "Windows 2000 Professional", "Windows 2000 Server" },
+    /* Earlier versions are not supported by GetVersionEx(). */
+    {  0,  0, NULL,                        NULL }
+        };
+        memset(&info, 0, sizeof(info));
+        info.dwOSVersionInfoSize = sizeof(info);
+        if (! GetVersionEx((LPOSVERSIONINFO)&info)) {
+          strlcpy(uname_result, "Bizarre version of Windows where GetVersionEx"
+                  " doesn't work.", sizeof(uname_result));
+          uname_result_is_set = 1;
+          return uname_result;
+        }
+#ifdef VER_NT_SERVER
+        if (info.wProductType == VER_NT_SERVER ||
+            info.wProductType == VER_NT_DOMAIN_CONTROLLER) {
+          is_server = 1;
+        } else {
+          is_client = 1;
+        }
+#endif /* defined(VER_NT_SERVER) */
+        /* Search the version table for a matching version */
+        for (i=0; win_version_table[i].major>0; ++i) {
+          if (win_version_table[i].major == info.dwMajorVersion &&
+              win_version_table[i].minor == info.dwMinorVersion) {
+            if (is_server) {
+              plat = win_version_table[i].server_version;
+            } else {
+              /* Use client versions for clients, and when we don't know if it
+              * is a client or a server. */
+              plat = win_version_table[i].client_version;
+            }
+            break;
+          }
+        }
+        if (plat) {
+          strlcpy(uname_result, plat, sizeof(uname_result));
+        } else {
+          if (info.dwMajorVersion > win_version_table[0].major ||
+              (info.dwMajorVersion == win_version_table[0].major &&
+               info.dwMinorVersion > win_version_table[0].minor))
+            tor_snprintf(uname_result, sizeof(uname_result),
+                         "Very recent version of Windows [major=%d,minor=%d]",
+                         (int)info.dwMajorVersion,(int)info.dwMinorVersion);
+          else
+            tor_snprintf(uname_result, sizeof(uname_result),
+                         "Unrecognized version of Windows [major=%d,minor=%d]",
+                         (int)info.dwMajorVersion,(int)info.dwMinorVersion);
+        }
+        /* Now append extra information to the name.
+         *
+         * Microsoft's API documentation says that on Windows 8.1 and later,
+         * GetVersionEx returns Windows 8 (6.2) for applications without an
+         * app compatibility manifest (including tor's default build).
+         *
+         * But in our testing, we have seen the actual Windows version on
+         * Windows Server 2012 R2, even without a manifest. */
+        if (info.dwMajorVersion > 6 ||
+            (info.dwMajorVersion == 6 && info.dwMinorVersion >= 2)) {
+          /* When GetVersionEx() returns Windows 8, the actual OS may be any
+           * later version. */
+          strlcat(uname_result, " [or later]", sizeof(uname_result));
+        }
+        /* When we don't know if the OS is a client or server version, we use
+         * the client version, and this qualifier. */
+        if (!is_server && !is_client) {
+          strlcat(uname_result, " [client or server]", sizeof(uname_result));
+        }
+#else /* !(defined(_WIN32)) */
+        /* LCOV_EXCL_START -- can't provoke uname failure */
+        strlcpy(uname_result, "Unknown platform", sizeof(uname_result));
+        /* LCOV_EXCL_STOP */
+#endif /* defined(_WIN32) */
+      }
+    uname_result_is_set = 1;
+  }
+  return uname_result;
+}
diff --git a/src/lib/osinfo/uname.h b/src/lib/osinfo/uname.h
new file mode 100644
index 0000000000..fcce629074
--- /dev/null
+++ b/src/lib/osinfo/uname.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file uname.h
+ * \brief Header for uname.c
+ **/
+
+#ifndef HAVE_TOR_UNAME_H
+#define HAVE_TOR_UNAME_H
+
+#include "lib/testsupport/testsupport.h"
+
+MOCK_DECL(const char *, get_uname,(void));
+
+#endif
diff --git a/src/lib/process/.may_include b/src/lib/process/.may_include
new file mode 100644
index 0000000000..05414d2a96
--- /dev/null
+++ b/src/lib/process/.may_include
@@ -0,0 +1,17 @@
+orconfig.h
+
+lib/cc/*.h
+lib/container/*.h
+lib/ctime/*.h
+lib/err/*.h
+lib/intmath/*.h
+lib/fs/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/net/*.h
+lib/process/*.h
+lib/string/*.h
+lib/testsupport/*.h
+lib/thread/*.h
+
+ht.h
\ No newline at end of file
diff --git a/src/lib/process/daemon.c b/src/lib/process/daemon.c
new file mode 100644
index 0000000000..3b90bef671
--- /dev/null
+++ b/src/lib/process/daemon.c
@@ -0,0 +1,187 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file daemon.c
+ * \brief Run the tor process in the background (unix only)
+ **/
+
+#include "orconfig.h"
+#include "lib/process/daemon.h"
+
+#ifndef _WIN32
+
+#include "lib/fs/files.h"
+#include "lib/log/log.h"
+#include "lib/thread/threads.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* Based on code contributed by christian grothoff */
+/** True iff we've called start_daemon(). */
+static int start_daemon_called = 0;
+/** True iff we've called finish_daemon(). */
+static int finish_daemon_called = 0;
+/** Socketpair used to communicate between parent and child process while
+ * daemonizing. */
+static int daemon_filedes[2];
+
+/**
+ * Return true iff we've called start_daemon() at least once.
+ */
+bool
+start_daemon_has_been_called(void)
+{
+  return start_daemon_called != 0;
+}
+
+/** Start putting the process into daemon mode: fork and drop all resources
+ * except standard fds.  The parent process never returns, but stays around
+ * until finish_daemon is called.  (Note: it's safe to call this more
+ * than once: calls after the first are ignored.)  Return true if we actually
+ * forked and this is the child; false otherwise.
+ */
+int
+start_daemon(void)
+{
+  pid_t pid;
+
+  if (start_daemon_called)
+    return 0;
+  start_daemon_called = 1;
+
+  if (pipe(daemon_filedes)) {
+    /* LCOV_EXCL_START */
+    log_err(LD_GENERAL,"pipe failed; exiting. Error was %s", strerror(errno));
+    exit(1); // exit ok: during daemonize, pipe failed.
+    /* LCOV_EXCL_STOP */
+  }
+  pid = fork();
+  if (pid < 0) {
+    /* LCOV_EXCL_START */
+    log_err(LD_GENERAL,"fork failed. Exiting.");
+    exit(1); // exit ok: during daemonize, fork failed
+    /* LCOV_EXCL_STOP */
+  }
+  if (pid) {  /* Parent */
+    int ok;
+    char c;
+
+    close(daemon_filedes[1]); /* we only read */
+    ok = -1;
+    while (0 < read(daemon_filedes[0], &c, sizeof(char))) {
+      if (c == '.')
+        ok = 1;
+    }
+    fflush(stdout);
+    if (ok == 1)
+      exit(0); // exit ok: during daemonize, daemonizing.
+    else
+      exit(1); /* child reported error. exit ok: daemonize failed. */
+    return 0; // LCOV_EXCL_LINE unreachable
+  } else { /* Child */
+    close(daemon_filedes[0]); /* we only write */
+
+    (void) setsid(); /* Detach from controlling terminal */
+    /*
+     * Fork one more time, so the parent (the session group leader) can exit.
+     * This means that we, as a non-session group leader, can never regain a
+     * controlling terminal.   This part is recommended by Stevens's
+     * _Advanced Programming in the Unix Environment_.
+     */
+    if (fork() != 0) {
+      exit(0); // exit ok: during daemonize, fork failed (2)
+    }
+    set_main_thread(); /* We are now the main thread. */
+
+    return 1;
+  }
+}
+
+/** Finish putting the process into daemon mode: drop standard fds, and tell
+ * the parent process to exit.  (Note: it's safe to call this more than once:
+ * calls after the first are ignored.  Calls start_daemon first if it hasn't
+ * been called already.) Return true if we actually did a fork; false if we
+ * didn't.
+ */
+int
+finish_daemon(const char *desired_cwd)
+{
+  int nullfd;
+  char c = '.';
+  if (finish_daemon_called)
+    return 0;
+  if (!start_daemon_called)
+    start_daemon();
+  finish_daemon_called = 1;
+
+  if (!desired_cwd)
+    desired_cwd = "/";
+   /* Don't hold the wrong FS mounted */
+  if (chdir(desired_cwd) < 0) {
+    log_err(LD_GENERAL,"chdir to \"%s\" failed. Exiting.",desired_cwd);
+    exit(1); // exit ok: during daemonize, chdir failed.
+  }
+
+  nullfd = tor_open_cloexec("/dev/null", O_RDWR, 0);
+  if (nullfd < 0) {
+    /* LCOV_EXCL_START */
+    log_err(LD_GENERAL,"/dev/null can't be opened. Exiting.");
+    exit(1); // exit ok: during daemonize, couldn't open /dev/null
+    /* LCOV_EXCL_STOP */
+  }
+  /* close fds linking to invoking terminal, but
+   * close usual incoming fds, but redirect them somewhere
+   * useful so the fds don't get reallocated elsewhere.
+   */
+  if (dup2(nullfd,0) < 0 ||
+      dup2(nullfd,1) < 0 ||
+      dup2(nullfd,2) < 0) {
+    /* LCOV_EXCL_START */
+    log_err(LD_GENERAL,"dup2 failed. Exiting.");
+    exit(1); // exit ok: during daemonize, dup2 failed.
+    /* LCOV_EXCL_STOP */
+  }
+  if (nullfd > 2)
+    close(nullfd);
+  /* signal success */
+  if (write(daemon_filedes[1], &c, sizeof(char)) != sizeof(char)) {
+    log_err(LD_GENERAL,"write failed. Exiting.");
+  }
+  close(daemon_filedes[1]);
+
+  return 0;
+}
+#else /* !(!defined(_WIN32)) */
+/* defined(_WIN32) */
+int
+start_daemon(void)
+{
+  return 0;
+}
+int
+finish_daemon(const char *cp)
+{
+  (void)cp;
+  return 0;
+}
+bool
+start_daemon_has_been_called(void)
+{
+  return false;
+}
+
+#endif /* !defined(_WIN32) */
diff --git a/src/lib/process/daemon.h b/src/lib/process/daemon.h
new file mode 100644
index 0000000000..20920e0aae
--- /dev/null
+++ b/src/lib/process/daemon.h
@@ -0,0 +1,21 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file daemon.h
+ * \brief Header for daemon.c
+ **/
+
+#ifndef TOR_DAEMON_H
+#define TOR_DAEMON_H
+
+#include <stdbool.h>
+
+int start_daemon(void);
+int finish_daemon(const char *desired_cwd);
+
+bool start_daemon_has_been_called(void);
+
+#endif
diff --git a/src/lib/process/env.c b/src/lib/process/env.c
new file mode 100644
index 0000000000..0060200ba1
--- /dev/null
+++ b/src/lib/process/env.c
@@ -0,0 +1,224 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file env.c
+ * \brief Inspect and manipulate the environment variables.
+ **/
+
+#include "orconfig.h"
+#include "lib/process/env.h"
+
+#include "lib/malloc/malloc.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/container/smartlist.h"
+#include "lib/log/util_bug.h"
+#include "lib/log/log.h"
+#include "lib/malloc/malloc.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <stdlib.h>
+#include <string.h>
+#ifdef HAVE_CRT_EXTERNS_H
+/* For _NSGetEnviron on macOS */
+#include <crt_externs.h>
+#endif
+
+#ifndef HAVE__NSGETENVIRON
+#ifndef HAVE_EXTERN_ENVIRON_DECLARED
+/* Some platforms declare environ under some circumstances, others don't. */
+#ifndef RUNNING_DOXYGEN
+extern char **environ;
+#endif
+#endif /* !defined(HAVE_EXTERN_ENVIRON_DECLARED) */
+#endif /* !defined(HAVE__NSGETENVIRON) */
+
+/** Return the current environment. This is a portable replacement for
+ * 'environ'. */
+char **
+get_environment(void)
+{
+#ifdef HAVE__NSGETENVIRON
+  /* This is for compatibility between OSX versions.  Otherwise (for example)
+   * when we do a mostly-static build on OSX 10.7, the resulting binary won't
+   * work on OSX 10.6. */
+  return *_NSGetEnviron();
+#else /* !(defined(HAVE__NSGETENVIRON)) */
+  return environ;
+#endif /* defined(HAVE__NSGETENVIRON) */
+}
+
+/** Helper: return the number of characters in <b>s</b> preceding the first
+ * occurrence of <b>ch</b>. If <b>ch</b> does not occur in <b>s</b>, return
+ * the length of <b>s</b>. Should be equivalent to strspn(s, "ch"). */
+static inline size_t
+str_num_before(const char *s, char ch)
+{
+  const char *cp = strchr(s, ch);
+  if (cp)
+    return cp - s;
+  else
+    return strlen(s);
+}
+
+/** Return non-zero iff getenv would consider <b>s1</b> and <b>s2</b>
+ * to have the same name as strings in a process's environment. */
+int
+environment_variable_names_equal(const char *s1, const char *s2)
+{
+  size_t s1_name_len = str_num_before(s1, '=');
+  size_t s2_name_len = str_num_before(s2, '=');
+
+  return (s1_name_len == s2_name_len &&
+          tor_memeq(s1, s2, s1_name_len));
+}
+
+/** Free <b>env</b> (assuming it was produced by
+ * process_environment_make). */
+void
+process_environment_free_(process_environment_t *env)
+{
+  if (env == NULL) return;
+
+  /* As both an optimization hack to reduce consing on Unixoid systems
+   * and a nice way to ensure that some otherwise-Windows-specific
+   * code will always get tested before changes to it get merged, the
+   * strings which env->unixoid_environment_block points to are packed
+   * into env->windows_environment_block. */
+  tor_free(env->unixoid_environment_block);
+  tor_free(env->windows_environment_block);
+
+  tor_free(env);
+}
+
+/** Make a process_environment_t containing the environment variables
+ * specified in <b>env_vars</b> (as C strings of the form
+ * "NAME=VALUE"). */
+process_environment_t *
+process_environment_make(struct smartlist_t *env_vars)
+{
+  process_environment_t *env = tor_malloc_zero(sizeof(process_environment_t));
+  int n_env_vars = smartlist_len(env_vars);
+  int i;
+  size_t total_env_length;
+  smartlist_t *env_vars_sorted;
+
+  tor_assert(n_env_vars + 1 != 0);
+  env->unixoid_environment_block = tor_calloc(n_env_vars + 1, sizeof(char *));
+  /* env->unixoid_environment_block is already NULL-terminated,
+   * because we assume that NULL == 0 (and check that during compilation). */
+
+  total_env_length = 1; /* terminating NUL of terminating empty string */
+  for (i = 0; i < n_env_vars; ++i) {
+    const char *s = smartlist_get(env_vars, (int)i);
+    size_t slen = strlen(s);
+
+    tor_assert(slen + 1 != 0);
+    tor_assert(slen + 1 < SIZE_MAX - total_env_length);
+    total_env_length += slen + 1;
+  }
+
+  env->windows_environment_block = tor_malloc_zero(total_env_length);
+  /* env->windows_environment_block is already
+   * (NUL-terminated-empty-string)-terminated. */
+
+  /* Some versions of Windows supposedly require that environment
+   * blocks be sorted.  Or maybe some Windows programs (or their
+   * runtime libraries) fail to look up strings in non-sorted
+   * environment blocks.
+   *
+   * Also, sorting strings makes it easy to find duplicate environment
+   * variables and environment-variable strings without an '=' on all
+   * OSes, and they can cause badness.  Let's complain about those. */
+  env_vars_sorted = smartlist_new();
+  smartlist_add_all(env_vars_sorted, env_vars);
+  smartlist_sort_strings(env_vars_sorted);
+
+  /* Now copy the strings into the environment blocks. */
+  {
+    char *cp = env->windows_environment_block;
+    const char *prev_env_var = NULL;
+
+    for (i = 0; i < n_env_vars; ++i) {
+      const char *s = smartlist_get(env_vars_sorted, (int)i);
+      size_t slen = strlen(s);
+      size_t s_name_len = str_num_before(s, '=');
+
+      if (s_name_len == slen) {
+        log_warn(LD_GENERAL,
+                 "Preparing an environment containing a variable "
+                 "without a value: %s",
+                 s);
+      }
+      if (prev_env_var != NULL &&
+          environment_variable_names_equal(s, prev_env_var)) {
+        log_warn(LD_GENERAL,
+                 "Preparing an environment containing two variables "
+                 "with the same name: %s and %s",
+                 prev_env_var, s);
+      }
+
+      prev_env_var = s;
+
+      /* Actually copy the string into the environment. */
+      memcpy(cp, s, slen+1);
+      env->unixoid_environment_block[i] = cp;
+      cp += slen+1;
+    }
+
+    tor_assert(cp == env->windows_environment_block + total_env_length - 1);
+  }
+
+  smartlist_free(env_vars_sorted);
+
+  return env;
+}
+
+/** Return a newly allocated smartlist containing every variable in
+ * this process's environment, as a NUL-terminated string of the form
+ * "NAME=VALUE".  Note that on some/many/most/all OSes, the parent
+ * process can put strings not of that form in our environment;
+ * callers should try to not get crashed by that.
+ *
+ * The returned strings are heap-allocated, and must be freed by the
+ * caller. */
+struct smartlist_t *
+get_current_process_environment_variables(void)
+{
+  smartlist_t *sl = smartlist_new();
+
+  char **environ_tmp; /* Not const char ** ? Really? */
+  for (environ_tmp = get_environment(); *environ_tmp; ++environ_tmp) {
+    smartlist_add_strdup(sl, *environ_tmp);
+  }
+
+  return sl;
+}
+
+/** For each string s in <b>env_vars</b> such that
+ * environment_variable_names_equal(s, <b>new_var</b>), remove it; if
+ * <b>free_p</b> is non-zero, call <b>free_old</b>(s).  If
+ * <b>new_var</b> contains '=', insert it into <b>env_vars</b>. */
+void
+set_environment_variable_in_smartlist(struct smartlist_t *env_vars,
+                                      const char *new_var,
+                                      void (*free_old)(void*),
+                                      int free_p)
+{
+  SMARTLIST_FOREACH_BEGIN(env_vars, const char *, s) {
+    if (environment_variable_names_equal(s, new_var)) {
+      SMARTLIST_DEL_CURRENT(env_vars, s);
+      if (free_p) {
+        free_old((void *)s);
+      }
+    }
+  } SMARTLIST_FOREACH_END(s);
+
+  if (strchr(new_var, '=') != NULL) {
+    smartlist_add(env_vars, (void *)new_var);
+  }
+}
diff --git a/src/lib/process/env.h b/src/lib/process/env.h
new file mode 100644
index 0000000000..15d59351e0
--- /dev/null
+++ b/src/lib/process/env.h
@@ -0,0 +1,41 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file env.h
+ * \brief Header for env.c
+ **/
+
+#ifndef TOR_ENV_H
+#define TOR_ENV_H
+
+char **get_environment(void);
+
+struct smartlist_t;
+
+int environment_variable_names_equal(const char *s1, const char *s2);
+
+/* DOCDOC process_environment_t */
+typedef struct process_environment_t {
+  /** A pointer to a sorted empty-string-terminated sequence of
+   * NUL-terminated strings of the form "NAME=VALUE". */
+  char *windows_environment_block;
+  /** A pointer to a NULL-terminated array of pointers to
+   * NUL-terminated strings of the form "NAME=VALUE". */
+  char **unixoid_environment_block;
+} process_environment_t;
+
+process_environment_t *process_environment_make(struct smartlist_t *env_vars);
+void process_environment_free_(process_environment_t *env);
+#define process_environment_free(env) \
+  FREE_AND_NULL(process_environment_t, process_environment_free_, (env))
+
+struct smartlist_t *get_current_process_environment_variables(void);
+
+void set_environment_variable_in_smartlist(struct smartlist_t *env_vars,
+                                           const char *new_var,
+                                           void (*free_old)(void*),
+                                           int free_p);
+#endif
diff --git a/src/lib/process/include.am b/src/lib/process/include.am
new file mode 100644
index 0000000000..c6cc3a6699
--- /dev/null
+++ b/src/lib/process/include.am
@@ -0,0 +1,29 @@
+
+noinst_LIBRARIES += src/lib/libtor-process.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-process-testing.a
+endif
+
+src_lib_libtor_process_a_SOURCES =		\
+	src/lib/process/daemon.c		\
+	src/lib/process/env.c			\
+	src/lib/process/pidfile.c		\
+	src/lib/process/restrict.c		\
+	src/lib/process/setuid.c		\
+	src/lib/process/subprocess.c		\
+	src/lib/process/waitpid.c
+
+src_lib_libtor_process_testing_a_SOURCES = \
+	$(src_lib_libtor_process_a_SOURCES)
+src_lib_libtor_process_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_process_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=				\
+	src/lib/process/daemon.h		\
+	src/lib/process/env.h			\
+	src/lib/process/pidfile.h		\
+	src/lib/process/restrict.h		\
+	src/lib/process/setuid.h		\
+	src/lib/process/subprocess.h		\
+	src/lib/process/waitpid.h
diff --git a/src/lib/process/pidfile.c b/src/lib/process/pidfile.c
new file mode 100644
index 0000000000..1b9d1c6d25
--- /dev/null
+++ b/src/lib/process/pidfile.c
@@ -0,0 +1,52 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file pidfile.c
+ * \brief Record this process's PID to disk.
+ **/
+
+#include "orconfig.h"
+#include "lib/process/pidfile.h"
+
+#include "lib/log/log.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+
+/** Write the current process ID, followed by NL, into <b>filename</b>.
+ * Return 0 on success, -1 on failure.
+ */
+int
+write_pidfile(const char *filename)
+{
+  FILE *pidfile;
+
+  if ((pidfile = fopen(filename, "w")) == NULL) {
+    log_warn(LD_FS, "Unable to open \"%s\" for writing: %s", filename,
+             strerror(errno));
+    return -1;
+  } else {
+#ifdef _WIN32
+    int pid = (int)_getpid();
+#else
+    int pid = (int)getpid();
+#endif
+    int rv = 0;
+    if (fprintf(pidfile, "%d\n", pid) < 0)
+      rv = -1;
+    if (fclose(pidfile) < 0)
+      rv = -1;
+    return rv;
+  }
+}
diff --git a/src/lib/process/pidfile.h b/src/lib/process/pidfile.h
new file mode 100644
index 0000000000..dfeb39e046
--- /dev/null
+++ b/src/lib/process/pidfile.h
@@ -0,0 +1,16 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file pidfile.h
+ * \brief Header for pidfile.c
+ **/
+
+#ifndef TOR_PIDFILE_H
+#define TOR_PIDFILE_H
+
+int write_pidfile(const char *filename);
+
+#endif
diff --git a/src/lib/process/restrict.c b/src/lib/process/restrict.c
new file mode 100644
index 0000000000..534b39d101
--- /dev/null
+++ b/src/lib/process/restrict.c
@@ -0,0 +1,285 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file restrict.c
+ * \brief Drop privileges from the current process.
+ **/
+
+#include "orconfig.h"
+#include "lib/process/restrict.h"
+#include "lib/intmath/cmp.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/net/socket.h"
+
+#ifdef HAVE_SYS_MMAN_H
+#include <sys/mman.h>
+#endif
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* We only use the linux prctl for now. There is no Win32 support; this may
+ * also work on various BSD systems and Mac OS X - send testing feedback!
+ *
+ * On recent Gnu/Linux kernels it is possible to create a system-wide policy
+ * that will prevent non-root processes from attaching to other processes
+ * unless they are the parent process; thus gdb can attach to programs that
+ * they execute but they cannot attach to other processes running as the same
+ * user. The system wide policy may be set with the sysctl
+ * kernel.yama.ptrace_scope or by inspecting
+ * /proc/sys/kernel/yama/ptrace_scope and it is 1 by default on Ubuntu 11.04.
+ *
+ * This ptrace scope will be ignored on Gnu/Linux for users with
+ * CAP_SYS_PTRACE and so it is very likely that root will still be able to
+ * attach to the Tor process.
+ */
+/** Attempt to disable debugger attachment: return 1 on success, -1 on
+ * failure, and 0 if we don't know how to try on this platform. */
+int
+tor_disable_debugger_attach(void)
+{
+  int r = -1;
+  log_debug(LD_CONFIG,
+            "Attemping to disable debugger attachment to Tor for "
+            "unprivileged users.");
+#if defined(__linux__) && defined(HAVE_SYS_PRCTL_H) \
+  && defined(HAVE_PRCTL) && defined(PR_SET_DUMPABLE)
+#define TRIED_TO_DISABLE
+  r = prctl(PR_SET_DUMPABLE, 0);
+#elif defined(__APPLE__) && defined(PT_DENY_ATTACH)
+#define TRIED_TO_ATTACH
+  r = ptrace(PT_DENY_ATTACH, 0, 0, 0);
+#endif /* defined(__linux__) && defined(HAVE_SYS_PRCTL_H) ... || ... */
+
+  // XXX: TODO - Mac OS X has dtrace and this may be disabled.
+  // XXX: TODO - Windows probably has something similar
+#ifdef TRIED_TO_DISABLE
+  if (r == 0) {
+    log_debug(LD_CONFIG,"Debugger attachment disabled for "
+              "unprivileged users.");
+    return 1;
+  } else {
+    log_warn(LD_CONFIG, "Unable to disable debugger attaching: %s",
+             strerror(errno));
+  }
+#endif /* defined(TRIED_TO_DISABLE) */
+#undef TRIED_TO_DISABLE
+  return r;
+}
+
+#if defined(HAVE_MLOCKALL) && HAVE_DECL_MLOCKALL && defined(RLIMIT_MEMLOCK)
+#define HAVE_UNIX_MLOCKALL
+#endif
+
+#ifdef HAVE_UNIX_MLOCKALL
+/** Attempt to raise the current and max rlimit to infinity for our process.
+ * This only needs to be done once and can probably only be done when we have
+ * not already dropped privileges.
+ */
+static int
+tor_set_max_memlock(void)
+{
+  /* Future consideration for Windows is probably SetProcessWorkingSetSize
+   * This is similar to setting the memory rlimit of RLIMIT_MEMLOCK
+   * http://msdn.microsoft.com/en-us/library/ms686234(VS.85).aspx
+   */
+
+  struct rlimit limit;
+
+  /* RLIM_INFINITY is -1 on some platforms. */
+  limit.rlim_cur = RLIM_INFINITY;
+  limit.rlim_max = RLIM_INFINITY;
+
+  if (setrlimit(RLIMIT_MEMLOCK, &limit) == -1) {
+    if (errno == EPERM) {
+      log_warn(LD_GENERAL, "You appear to lack permissions to change memory "
+                           "limits. Are you root?");
+    }
+    log_warn(LD_GENERAL, "Unable to raise RLIMIT_MEMLOCK: %s",
+             strerror(errno));
+    return -1;
+  }
+
+  return 0;
+}
+#endif /* defined(HAVE_UNIX_MLOCKALL) */
+
+/** Attempt to lock all current and all future memory pages.
+ * This should only be called once and while we're privileged.
+ * Like mlockall() we return 0 when we're successful and -1 when we're not.
+ * Unlike mlockall() we return 1 if we've already attempted to lock memory.
+ */
+int
+tor_mlockall(void)
+{
+  static int memory_lock_attempted = 0;
+
+  if (memory_lock_attempted) {
+    return 1;
+  }
+
+  memory_lock_attempted = 1;
+
+  /*
+   * Future consideration for Windows may be VirtualLock
+   * VirtualLock appears to implement mlock() but not mlockall()
+   *
+   * http://msdn.microsoft.com/en-us/library/aa366895(VS.85).aspx
+   */
+
+#ifdef HAVE_UNIX_MLOCKALL
+  if (tor_set_max_memlock() == 0) {
+    log_debug(LD_GENERAL, "RLIMIT_MEMLOCK is now set to RLIM_INFINITY.");
+  }
+
+  if (mlockall(MCL_CURRENT|MCL_FUTURE) == 0) {
+    log_info(LD_GENERAL, "Insecure OS paging is effectively disabled.");
+    return 0;
+  } else {
+    if (errno == ENOSYS) {
+      /* Apple - it's 2009! I'm looking at you. Grrr. */
+      log_notice(LD_GENERAL, "It appears that mlockall() is not available on "
+                             "your platform.");
+    } else if (errno == EPERM) {
+      log_notice(LD_GENERAL, "It appears that you lack the permissions to "
+                             "lock memory. Are you root?");
+    }
+    log_notice(LD_GENERAL, "Unable to lock all current and future memory "
+                           "pages: %s", strerror(errno));
+    return -1;
+  }
+#else /* !(defined(HAVE_UNIX_MLOCKALL)) */
+  log_warn(LD_GENERAL, "Unable to lock memory pages. mlockall() unsupported?");
+  return -1;
+#endif /* defined(HAVE_UNIX_MLOCKALL) */
+}
+
+/** Number of extra file descriptors to keep in reserve beyond those that we
+ * tell Tor it's allowed to use. */
+#define ULIMIT_BUFFER 32 /* keep 32 extra fd's beyond ConnLimit_ */
+
+/** Learn the maximum allowed number of file descriptors, and tell the
+ * system we want to use up to that number. (Some systems have a low soft
+ * limit, and let us set it higher.)  We compute this by finding the largest
+ * number that we can use.
+ *
+ * If the limit is below the reserved file descriptor value (ULIMIT_BUFFER),
+ * return -1 and <b>max_out</b> is untouched.
+ *
+ * If we can't find a number greater than or equal to <b>limit</b>, then we
+ * fail by returning -1 and <b>max_out</b> is untouched.
+ *
+ * If we are unable to set the limit value because of setrlimit() failing,
+ * return 0 and <b>max_out</b> is set to the current maximum value returned
+ * by getrlimit().
+ *
+ * Otherwise, return 0 and store the maximum we found inside <b>max_out</b>
+ * and set <b>max_sockets</b> with that value as well.*/
+int
+set_max_file_descriptors(rlim_t limit, int *max_out)
+{
+  if (limit < ULIMIT_BUFFER) {
+    log_warn(LD_CONFIG,
+             "ConnLimit must be at least %d. Failing.", ULIMIT_BUFFER);
+    return -1;
+  }
+
+  /* Define some maximum connections values for systems where we cannot
+   * automatically determine a limit. Re Cygwin, see
+   * http://archives.seul.org/or/talk/Aug-2006/msg00210.html
+   * For an iPhone, 9999 should work. For Windows and all other unknown
+   * systems we use 15000 as the default. */
+#ifndef HAVE_GETRLIMIT
+#if defined(CYGWIN) || defined(__CYGWIN__)
+  const char *platform = "Cygwin";
+  const unsigned long MAX_CONNECTIONS = 3200;
+#elif defined(_WIN32)
+  const char *platform = "Windows";
+  const unsigned long MAX_CONNECTIONS = 15000;
+#else
+  const char *platform = "unknown platforms with no getrlimit()";
+  const unsigned long MAX_CONNECTIONS = 15000;
+#endif /* defined(CYGWIN) || defined(__CYGWIN__) || ... */
+  log_fn(LOG_INFO, LD_NET,
+         "This platform is missing getrlimit(). Proceeding.");
+  if (limit > MAX_CONNECTIONS) {
+    log_warn(LD_CONFIG,
+             "We do not support more than %lu file descriptors "
+             "on %s. Tried to raise to %lu.",
+             (unsigned long)MAX_CONNECTIONS, platform, (unsigned long)limit);
+    return -1;
+  }
+  limit = MAX_CONNECTIONS;
+#else /* !(!defined(HAVE_GETRLIMIT)) */
+  struct rlimit rlim;
+
+  if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) {
+    log_warn(LD_NET, "Could not get maximum number of file descriptors: %s",
+             strerror(errno));
+    return -1;
+  }
+  if (rlim.rlim_max < limit) {
+    log_warn(LD_CONFIG,"We need %lu file descriptors available, and we're "
+             "limited to %lu. Please change your ulimit -n.",
+             (unsigned long)limit, (unsigned long)rlim.rlim_max);
+    return -1;
+  }
+
+  if (rlim.rlim_max > rlim.rlim_cur) {
+    log_info(LD_NET,"Raising max file descriptors from %lu to %lu.",
+             (unsigned long)rlim.rlim_cur, (unsigned long)rlim.rlim_max);
+  }
+  /* Set the current limit value so if the attempt to set the limit to the
+   * max fails at least we'll have a valid value of maximum sockets. */
+  *max_out = (int)rlim.rlim_cur - ULIMIT_BUFFER;
+  set_max_sockets(*max_out);
+  rlim.rlim_cur = rlim.rlim_max;
+
+  if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) {
+    int couldnt_set = 1;
+    const int setrlimit_errno = errno;
+#ifdef OPEN_MAX
+    uint64_t try_limit = OPEN_MAX - ULIMIT_BUFFER;
+    if (errno == EINVAL && try_limit < (uint64_t) rlim.rlim_cur) {
+      /* On some platforms, OPEN_MAX is the real limit, and getrlimit() is
+       * full of nasty lies.  I'm looking at you, OSX 10.5.... */
+      rlim.rlim_cur = MIN((rlim_t) try_limit, rlim.rlim_cur);
+      if (setrlimit(RLIMIT_NOFILE, &rlim) == 0) {
+        if (rlim.rlim_cur < (rlim_t)limit) {
+          log_warn(LD_CONFIG, "We are limited to %lu file descriptors by "
+                   "OPEN_MAX (%lu), and ConnLimit is %lu.  Changing "
+                   "ConnLimit; sorry.",
+                   (unsigned long)try_limit, (unsigned long)OPEN_MAX,
+                   (unsigned long)limit);
+        } else {
+          log_info(LD_CONFIG, "Dropped connection limit to %lu based on "
+                   "OPEN_MAX (%lu); Apparently, %lu was too high and rlimit "
+                   "lied to us.",
+                   (unsigned long)try_limit, (unsigned long)OPEN_MAX,
+                   (unsigned long)rlim.rlim_max);
+        }
+        couldnt_set = 0;
+      }
+    }
+#endif /* defined(OPEN_MAX) */
+    if (couldnt_set) {
+      log_warn(LD_CONFIG,"Couldn't set maximum number of file descriptors: %s",
+               strerror(setrlimit_errno));
+    }
+  }
+  /* leave some overhead for logs, etc, */
+  limit = rlim.rlim_cur;
+#endif /* !defined(HAVE_GETRLIMIT) */
+
+  if (limit > INT_MAX)
+    limit = INT_MAX;
+  tor_assert(max_out);
+  *max_out = (int)limit - ULIMIT_BUFFER;
+  set_max_sockets(*max_out);
+
+  return 0;
+}
diff --git a/src/lib/process/restrict.h b/src/lib/process/restrict.h
new file mode 100644
index 0000000000..8491c99044
--- /dev/null
+++ b/src/lib/process/restrict.h
@@ -0,0 +1,27 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file restrict.h
+ * \brief Header for restrict.c
+ **/
+
+#ifndef TOR_RESTRICT_H
+#define TOR_RESTRICT_H
+
+#include "orconfig.h"
+#ifdef HAVE_SYS_RESOURCE_H
+#include <sys/resource.h>
+#endif
+
+int tor_disable_debugger_attach(void);
+int tor_mlockall(void);
+
+#if !defined(HAVE_RLIM_T)
+typedef unsigned long rlim_t;
+#endif
+int set_max_file_descriptors(rlim_t limit, int *max_out);
+
+#endif /* !defined(TOR_RESTRICT_H) */
diff --git a/src/lib/process/setuid.c b/src/lib/process/setuid.c
new file mode 100644
index 0000000000..6e8258f279
--- /dev/null
+++ b/src/lib/process/setuid.c
@@ -0,0 +1,386 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file setuid.c
+ * \brief Change the user ID after Tor has started (Unix only)
+ **/
+
+#include "orconfig.h"
+#include "lib/process/setuid.h"
+
+#if defined(HAVE_SYS_CAPABILITY_H) && defined(HAVE_CAP_SET_PROC)
+#define HAVE_LINUX_CAPABILITIES
+#endif
+
+#include "lib/container/smartlist.h"
+#include "lib/fs/userdb.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_GRP_H
+#include <grp.h>
+#endif
+#ifdef HAVE_PWD_H
+#include <pwd.h>
+#endif
+#ifdef HAVE_SYS_CAPABILITY_H
+#include <sys/capability.h>
+#endif
+#ifdef HAVE_SYS_PRCTL_H
+#include <sys/prctl.h>
+#endif
+
+#include <errno.h>
+#include <string.h>
+
+#ifndef _WIN32
+/** Log details of current user and group credentials. Return 0 on
+ * success. Logs and return -1 on failure.
+ */
+static int
+log_credential_status(void)
+{
+/** Log level to use when describing non-error UID/GID status. */
+#define CREDENTIAL_LOG_LEVEL LOG_INFO
+  /* Real, effective and saved UIDs */
+  uid_t ruid, euid, suid;
+  /* Read, effective and saved GIDs */
+  gid_t rgid, egid, sgid;
+  /* Supplementary groups */
+  gid_t *sup_gids = NULL;
+  int sup_gids_size;
+  /* Number of supplementary groups */
+  int ngids;
+
+  /* log UIDs */
+#ifdef HAVE_GETRESUID
+  if (getresuid(&ruid, &euid, &suid) != 0 ) {
+    log_warn(LD_GENERAL, "Error getting changed UIDs: %s", strerror(errno));
+    return -1;
+  } else {
+    log_fn(CREDENTIAL_LOG_LEVEL, LD_GENERAL,
+           "UID is %u (real), %u (effective), %u (saved)",
+           (unsigned)ruid, (unsigned)euid, (unsigned)suid);
+  }
+#else /* !(defined(HAVE_GETRESUID)) */
+  /* getresuid is not present on MacOS X, so we can't get the saved (E)UID */
+  ruid = getuid();
+  euid = geteuid();
+  (void)suid;
+
+  log_fn(CREDENTIAL_LOG_LEVEL, LD_GENERAL,
+         "UID is %u (real), %u (effective), unknown (saved)",
+         (unsigned)ruid, (unsigned)euid);
+#endif /* defined(HAVE_GETRESUID) */
+
+  /* log GIDs */
+#ifdef HAVE_GETRESGID
+  if (getresgid(&rgid, &egid, &sgid) != 0 ) {
+    log_warn(LD_GENERAL, "Error getting changed GIDs: %s", strerror(errno));
+    return -1;
+  } else {
+    log_fn(CREDENTIAL_LOG_LEVEL, LD_GENERAL,
+           "GID is %u (real), %u (effective), %u (saved)",
+           (unsigned)rgid, (unsigned)egid, (unsigned)sgid);
+  }
+#else /* !(defined(HAVE_GETRESGID)) */
+  /* getresgid is not present on MacOS X, so we can't get the saved (E)GID */
+  rgid = getgid();
+  egid = getegid();
+  (void)sgid;
+  log_fn(CREDENTIAL_LOG_LEVEL, LD_GENERAL,
+         "GID is %u (real), %u (effective), unknown (saved)",
+         (unsigned)rgid, (unsigned)egid);
+#endif /* defined(HAVE_GETRESGID) */
+
+  /* log supplementary groups */
+  sup_gids_size = 64;
+  sup_gids = tor_calloc(64, sizeof(gid_t));
+  while ((ngids = getgroups(sup_gids_size, sup_gids)) < 0 &&
+         errno == EINVAL &&
+         sup_gids_size < NGROUPS_MAX) {
+    sup_gids_size *= 2;
+    sup_gids = tor_reallocarray(sup_gids, sizeof(gid_t), sup_gids_size);
+  }
+
+  if (ngids < 0) {
+    log_warn(LD_GENERAL, "Error getting supplementary GIDs: %s",
+             strerror(errno));
+    tor_free(sup_gids);
+    return -1;
+  } else {
+    int i, retval = 0;
+    char *s = NULL;
+    smartlist_t *elts = smartlist_new();
+
+    for (i = 0; i<ngids; i++) {
+      smartlist_add_asprintf(elts, "%u", (unsigned)sup_gids[i]);
+    }
+
+    s = smartlist_join_strings(elts, " ", 0, NULL);
+
+    log_fn(CREDENTIAL_LOG_LEVEL, LD_GENERAL, "Supplementary groups are: %s",s);
+
+    tor_free(s);
+    SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp));
+    smartlist_free(elts);
+    tor_free(sup_gids);
+
+    return retval;
+  }
+
+  return 0;
+}
+#endif /* !defined(_WIN32) */
+
+/** Return true iff we were compiled with capability support, and capabilities
+ * seem to work. **/
+int
+have_capability_support(void)
+{
+#ifdef HAVE_LINUX_CAPABILITIES
+  cap_t caps = cap_get_proc();
+  if (caps == NULL)
+    return 0;
+  cap_free(caps);
+  return 1;
+#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */
+  return 0;
+#endif /* defined(HAVE_LINUX_CAPABILITIES) */
+}
+
+#ifdef HAVE_LINUX_CAPABILITIES
+/** Helper. Drop all capabilities but a small set, and set PR_KEEPCAPS as
+ * appropriate.
+ *
+ * If pre_setuid, retain only CAP_NET_BIND_SERVICE, CAP_SETUID, and
+ * CAP_SETGID, and use PR_KEEPCAPS to ensure that capabilities persist across
+ * setuid().
+ *
+ * If not pre_setuid, retain only CAP_NET_BIND_SERVICE, and disable
+ * PR_KEEPCAPS.
+ *
+ * Return 0 on success, and -1 on failure.
+ */
+static int
+drop_capabilities(int pre_setuid)
+{
+  /* We keep these three capabilities, and these only, as we setuid.
+   * After we setuid, we drop all but the first. */
+  const cap_value_t caplist[] = {
+    CAP_NET_BIND_SERVICE, CAP_SETUID, CAP_SETGID
+  };
+  const char *where = pre_setuid ? "pre-setuid" : "post-setuid";
+  const int n_effective = pre_setuid ? 3 : 1;
+  const int n_permitted = pre_setuid ? 3 : 1;
+  const int n_inheritable = 1;
+  const int keepcaps = pre_setuid ? 1 : 0;
+
+  /* Sets whether we keep capabilities across a setuid. */
+  if (prctl(PR_SET_KEEPCAPS, keepcaps) < 0) {
+    log_warn(LD_CONFIG, "Unable to call prctl() %s: %s",
+             where, strerror(errno));
+    return -1;
+  }
+
+  cap_t caps = cap_get_proc();
+  if (!caps) {
+    log_warn(LD_CONFIG, "Unable to call cap_get_proc() %s: %s",
+             where, strerror(errno));
+    return -1;
+  }
+  cap_clear(caps);
+
+  cap_set_flag(caps, CAP_EFFECTIVE, n_effective, caplist, CAP_SET);
+  cap_set_flag(caps, CAP_PERMITTED, n_permitted, caplist, CAP_SET);
+  cap_set_flag(caps, CAP_INHERITABLE, n_inheritable, caplist, CAP_SET);
+
+  int r = cap_set_proc(caps);
+  cap_free(caps);
+  if (r < 0) {
+    log_warn(LD_CONFIG, "No permission to set capabilities %s: %s",
+             where, strerror(errno));
+    return -1;
+  }
+
+  return 0;
+}
+#endif /* defined(HAVE_LINUX_CAPABILITIES) */
+
+/** Call setuid and setgid to run as <b>user</b> and switch to their
+ * primary group.  Return 0 on success.  On failure, log and return -1.
+ *
+ * If SWITCH_ID_KEEP_BINDLOW is set in 'flags', try to use the capability
+ * system to retain the abilitity to bind low ports.
+ *
+ * If SWITCH_ID_WARN_IF_NO_CAPS is set in flags, also warn if we have
+ * don't have capability support.
+ */
+int
+switch_id(const char *user, const unsigned flags)
+{
+#ifndef _WIN32
+  const struct passwd *pw = NULL;
+  uid_t old_uid;
+  gid_t old_gid;
+  static int have_already_switched_id = 0;
+  const int keep_bindlow = !!(flags & SWITCH_ID_KEEP_BINDLOW);
+  const int warn_if_no_caps = !!(flags & SWITCH_ID_WARN_IF_NO_CAPS);
+
+  tor_assert(user);
+
+  if (have_already_switched_id)
+    return 0;
+
+  /* Log the initial credential state */
+  if (log_credential_status())
+    return -1;
+
+  log_fn(CREDENTIAL_LOG_LEVEL, LD_GENERAL, "Changing user and groups");
+
+  /* Get old UID/GID to check if we changed correctly */
+  old_uid = getuid();
+  old_gid = getgid();
+
+  /* Lookup the user and group information, if we have a problem, bail out. */
+  pw = tor_getpwnam(user);
+  if (pw == NULL) {
+    log_warn(LD_CONFIG, "Error setting configured user: %s not found", user);
+    return -1;
+  }
+
+#ifdef HAVE_LINUX_CAPABILITIES
+  (void) warn_if_no_caps;
+  if (keep_bindlow) {
+    if (drop_capabilities(1))
+      return -1;
+  }
+#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */
+  (void) keep_bindlow;
+  if (warn_if_no_caps) {
+    log_warn(LD_CONFIG, "KeepBindCapabilities set, but no capability support "
+             "on this system.");
+  }
+#endif /* defined(HAVE_LINUX_CAPABILITIES) */
+
+  /* Properly switch egid,gid,euid,uid here or bail out */
+  if (setgroups(1, &pw->pw_gid)) {
+    log_warn(LD_GENERAL, "Error setting groups to gid %d: \"%s\".",
+             (int)pw->pw_gid, strerror(errno));
+    if (old_uid == pw->pw_uid) {
+      log_warn(LD_GENERAL, "Tor is already running as %s.  You do not need "
+               "the \"User\" option if you are already running as the user "
+               "you want to be.  (If you did not set the User option in your "
+               "torrc, check whether it was specified on the command line "
+               "by a startup script.)", user);
+    } else {
+      log_warn(LD_GENERAL, "If you set the \"User\" option, you must start Tor"
+               " as root.");
+    }
+    return -1;
+  }
+
+  if (setegid(pw->pw_gid)) {
+    log_warn(LD_GENERAL, "Error setting egid to %d: %s",
+             (int)pw->pw_gid, strerror(errno));
+    return -1;
+  }
+
+  if (setgid(pw->pw_gid)) {
+    log_warn(LD_GENERAL, "Error setting gid to %d: %s",
+             (int)pw->pw_gid, strerror(errno));
+    return -1;
+  }
+
+  if (setuid(pw->pw_uid)) {
+    log_warn(LD_GENERAL, "Error setting configured uid to %s (%d): %s",
+             user, (int)pw->pw_uid, strerror(errno));
+    return -1;
+  }
+
+  if (seteuid(pw->pw_uid)) {
+    log_warn(LD_GENERAL, "Error setting configured euid to %s (%d): %s",
+             user, (int)pw->pw_uid, strerror(errno));
+    return -1;
+  }
+
+  /* This is how OpenBSD rolls:
+  if (setgroups(1, &pw->pw_gid) || setegid(pw->pw_gid) ||
+      setgid(pw->pw_gid) || setuid(pw->pw_uid) || seteuid(pw->pw_uid)) {
+      setgid(pw->pw_gid) || seteuid(pw->pw_uid) || setuid(pw->pw_uid)) {
+    log_warn(LD_GENERAL, "Error setting configured UID/GID: %s",
+    strerror(errno));
+    return -1;
+  }
+  */
+
+  /* We've properly switched egid, gid, euid, uid, and supplementary groups if
+   * we're here. */
+#ifdef HAVE_LINUX_CAPABILITIES
+  if (keep_bindlow) {
+    if (drop_capabilities(0))
+      return -1;
+  }
+#endif /* defined(HAVE_LINUX_CAPABILITIES) */
+
+#if !defined(CYGWIN) && !defined(__CYGWIN__)
+  /* If we tried to drop privilege to a group/user other than root, attempt to
+   * restore root (E)(U|G)ID, and abort if the operation succeeds */
+
+  /* Only check for privilege dropping if we were asked to be non-root */
+  if (pw->pw_uid) {
+    /* Try changing GID/EGID */
+    if (pw->pw_gid != old_gid &&
+        (setgid(old_gid) != -1 || setegid(old_gid) != -1)) {
+      log_warn(LD_GENERAL, "Was able to restore group credentials even after "
+               "switching GID: this means that the setgid code didn't work.");
+      return -1;
+    }
+
+    /* Try changing UID/EUID */
+    if (pw->pw_uid != old_uid &&
+        (setuid(old_uid) != -1 || seteuid(old_uid) != -1)) {
+      log_warn(LD_GENERAL, "Was able to restore user credentials even after "
+               "switching UID: this means that the setuid code didn't work.");
+      return -1;
+    }
+  }
+#endif /* !defined(CYGWIN) && !defined(__CYGWIN__) */
+
+  /* Check what really happened */
+  if (log_credential_status()) {
+    return -1;
+  }
+
+  have_already_switched_id = 1; /* mark success so we never try again */
+
+#if defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && \
+  defined(HAVE_PRCTL) && defined(PR_SET_DUMPABLE)
+  if (pw->pw_uid) {
+    /* Re-enable core dumps if we're not running as root. */
+    log_info(LD_CONFIG, "Re-enabling coredumps");
+    if (prctl(PR_SET_DUMPABLE, 1)) {
+      log_warn(LD_CONFIG, "Unable to re-enable coredumps: %s",strerror(errno));
+    }
+  }
+#endif /* defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && ... */
+  return 0;
+
+#else /* !(!defined(_WIN32)) */
+  (void)user;
+  (void)flags;
+
+  log_warn(LD_CONFIG, "Switching users is unsupported on your OS.");
+  return -1;
+#endif /* !defined(_WIN32) */
+}
diff --git a/src/lib/process/setuid.h b/src/lib/process/setuid.h
new file mode 100644
index 0000000000..7d03e1f025
--- /dev/null
+++ b/src/lib/process/setuid.h
@@ -0,0 +1,22 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file setuid.h
+ * \brief Header for setuid.c
+ **/
+
+#ifndef TOR_SETUID_H
+#define TOR_SETUID_H
+
+int have_capability_support(void);
+
+/** Flag for switch_id; see switch_id() for documentation */
+#define SWITCH_ID_KEEP_BINDLOW    (1<<0)
+/** Flag for switch_id; see switch_id() for documentation */
+#define SWITCH_ID_WARN_IF_NO_CAPS (1<<1)
+int switch_id(const char *user, unsigned flags);
+
+#endif
diff --git a/src/lib/process/subprocess.c b/src/lib/process/subprocess.c
new file mode 100644
index 0000000000..f4429d7f76
--- /dev/null
+++ b/src/lib/process/subprocess.c
@@ -0,0 +1,1236 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file subprocess.c
+ * \brief Launch and monitor other processes.
+ **/
+
+#define SUBPROCESS_PRIVATE
+#include "lib/process/subprocess.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/err/torerr.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/log/win32err.h"
+#include "lib/malloc/malloc.h"
+#include "lib/process/env.h"
+#include "lib/process/waitpid.h"
+#include "lib/string/compat_ctype.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_PRCTL_H
+#include <sys/prctl.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SIGNAL_H
+#include <signal.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
+#include <errno.h>
+#include <string.h>
+
+/** Format a single argument for being put on a Windows command line.
+ * Returns a newly allocated string */
+static char *
+format_win_cmdline_argument(const char *arg)
+{
+  char *formatted_arg;
+  char need_quotes;
+  const char *c;
+  int i;
+  int bs_counter = 0;
+  /* Backslash we can point to when one is inserted into the string */
+  const char backslash = '\\';
+
+  /* Smartlist of *char */
+  smartlist_t *arg_chars;
+  arg_chars = smartlist_new();
+
+  /* Quote string if it contains whitespace or is empty */
+  need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]);
+
+  /* Build up smartlist of *chars */
+  for (c=arg; *c != '\0'; c++) {
+    if ('"' == *c) {
+      /* Double up backslashes preceding a quote */
+      for (i=0; i<(bs_counter*2); i++)
+        smartlist_add(arg_chars, (void*)&backslash);
+      bs_counter = 0;
+      /* Escape the quote */
+      smartlist_add(arg_chars, (void*)&backslash);
+      smartlist_add(arg_chars, (void*)c);
+    } else if ('\\' == *c) {
+      /* Count backslashes until we know whether to double up */
+      bs_counter++;
+    } else {
+      /* Don't double up slashes preceding a non-quote */
+      for (i=0; i<bs_counter; i++)
+        smartlist_add(arg_chars, (void*)&backslash);
+      bs_counter = 0;
+      smartlist_add(arg_chars, (void*)c);
+    }
+  }
+  /* Don't double up trailing backslashes */
+  for (i=0; i<bs_counter; i++)
+    smartlist_add(arg_chars, (void*)&backslash);
+
+  /* Allocate space for argument, quotes (if needed), and terminator */
+  const size_t formatted_arg_len = smartlist_len(arg_chars) +
+    (need_quotes ? 2 : 0) + 1;
+  formatted_arg = tor_malloc_zero(formatted_arg_len);
+
+  /* Add leading quote */
+  i=0;
+  if (need_quotes)
+    formatted_arg[i++] = '"';
+
+  /* Add characters */
+  SMARTLIST_FOREACH(arg_chars, char*, ch,
+  {
+    formatted_arg[i++] = *ch;
+  });
+
+  /* Add trailing quote */
+  if (need_quotes)
+    formatted_arg[i++] = '"';
+  formatted_arg[i] = '\0';
+
+  smartlist_free(arg_chars);
+  return formatted_arg;
+}
+
+/** Format a command line for use on Windows, which takes the command as a
+ * string rather than string array. Follows the rules from "Parsing C++
+ * Command-Line Arguments" in MSDN. Algorithm based on list2cmdline in the
+ * Python subprocess module. Returns a newly allocated string */
+char *
+tor_join_win_cmdline(const char *argv[])
+{
+  smartlist_t *argv_list;
+  char *joined_argv;
+  int i;
+
+  /* Format each argument and put the result in a smartlist */
+  argv_list = smartlist_new();
+  for (i=0; argv[i] != NULL; i++) {
+    smartlist_add(argv_list, (void *)format_win_cmdline_argument(argv[i]));
+  }
+
+  /* Join the arguments with whitespace */
+  joined_argv = smartlist_join_strings(argv_list, " ", 0, NULL);
+
+  /* Free the newly allocated arguments, and the smartlist */
+  SMARTLIST_FOREACH(argv_list, char *, arg,
+  {
+    tor_free(arg);
+  });
+  smartlist_free(argv_list);
+
+  return joined_argv;
+}
+
+#ifndef _WIN32
+/** Format <b>child_state</b> and <b>saved_errno</b> as a hex string placed in
+ * <b>hex_errno</b>.  Called between fork and _exit, so must be signal-handler
+ * safe.
+ *
+ * <b>hex_errno</b> must have at least HEX_ERRNO_SIZE+1 bytes available.
+ *
+ * The format of <b>hex_errno</b> is: "CHILD_STATE/ERRNO\n", left-padded
+ * with spaces. CHILD_STATE indicates where
+ * in the process of starting the child process did the failure occur (see
+ * CHILD_STATE_* macros for definition), and SAVED_ERRNO is the value of
+ * errno when the failure occurred.
+ *
+ * On success return the number of characters added to hex_errno, not counting
+ * the terminating NUL; return -1 on error.
+ */
+STATIC int
+format_helper_exit_status(unsigned char child_state, int saved_errno,
+                          char *hex_errno)
+{
+  unsigned int unsigned_errno;
+  int written, left;
+  char *cur;
+  size_t i;
+  int res = -1;
+
+  /* Fill hex_errno with spaces, and a trailing newline (memset may
+     not be signal handler safe, so we can't use it) */
+  for (i = 0; i < (HEX_ERRNO_SIZE - 1); i++)
+    hex_errno[i] = ' ';
+  hex_errno[HEX_ERRNO_SIZE - 1] = '\n';
+
+  /* Convert errno to be unsigned for hex conversion */
+  if (saved_errno < 0) {
+    // Avoid overflow on the cast to unsigned int when result is INT_MIN
+    // by adding 1 to the signed int negative value,
+    // then, after it has been negated and cast to unsigned,
+    // adding the original 1 back (the double-addition is intentional).
+    // Otherwise, the cast to signed could cause a temporary int
+    // to equal INT_MAX + 1, which is undefined.
+    unsigned_errno = ((unsigned int) -(saved_errno + 1)) + 1;
+  } else {
+    unsigned_errno = (unsigned int) saved_errno;
+  }
+
+  /*
+   * Count how many chars of space we have left, and keep a pointer into the
+   * current point in the buffer.
+   */
+  left = HEX_ERRNO_SIZE+1;
+  cur = hex_errno;
+
+  /* Emit child_state */
+  written = format_hex_number_sigsafe(child_state, cur, left);
+
+  if (written <= 0)
+    goto err;
+
+  /* Adjust left and cur */
+  left -= written;
+  cur += written;
+  if (left <= 0)
+    goto err;
+
+  /* Now the '/' */
+  *cur = '/';
+
+  /* Adjust left and cur */
+  ++cur;
+  --left;
+  if (left <= 0)
+    goto err;
+
+  /* Need minus? */
+  if (saved_errno < 0) {
+    *cur = '-';
+    ++cur;
+    --left;
+    if (left <= 0)
+      goto err;
+  }
+
+  /* Emit unsigned_errno */
+  written = format_hex_number_sigsafe(unsigned_errno, cur, left);
+
+  if (written <= 0)
+    goto err;
+
+  /* Adjust left and cur */
+  left -= written;
+  cur += written;
+
+  /* Check that we have enough space left for a newline and a NUL */
+  if (left <= 1)
+    goto err;
+
+  /* Emit the newline and NUL */
+  *cur++ = '\n';
+  *cur++ = '\0';
+
+  res = (int)(cur - hex_errno - 1);
+
+  goto done;
+
+ err:
+  /*
+   * In error exit, just write a '\0' in the first char so whatever called
+   * this at least won't fall off the end.
+   */
+  *hex_errno = '\0';
+
+ done:
+  return res;
+}
+#endif /* !defined(_WIN32) */
+
+/* Maximum number of file descriptors, if we cannot get it via sysconf() */
+#define DEFAULT_MAX_FD 256
+
+/** Terminate the process of <b>process_handle</b>, if that process has not
+ * already exited.
+ *
+ * Return 0 if we succeeded in terminating the process (or if the process
+ * already exited), and -1 if we tried to kill the process but failed.
+ *
+ * Based on code originally borrowed from Python's os.kill. */
+int
+tor_terminate_process(process_handle_t *process_handle)
+{
+#ifdef _WIN32
+  if (tor_get_exit_code(process_handle, 0, NULL) == PROCESS_EXIT_RUNNING) {
+    HANDLE handle = process_handle->pid.hProcess;
+
+    if (!TerminateProcess(handle, 0))
+      return -1;
+    else
+      return 0;
+  }
+#else /* !(defined(_WIN32)) */
+  if (process_handle->waitpid_cb) {
+    /* We haven't got a waitpid yet, so we can just kill off the process. */
+    return kill(process_handle->pid, SIGTERM);
+  }
+#endif /* defined(_WIN32) */
+
+  return 0; /* We didn't need to kill the process, so report success */
+}
+
+/** Return the Process ID of <b>process_handle</b>. */
+int
+tor_process_get_pid(process_handle_t *process_handle)
+{
+#ifdef _WIN32
+  return (int) process_handle->pid.dwProcessId;
+#else
+  return (int) process_handle->pid;
+#endif
+}
+
+#ifdef _WIN32
+HANDLE
+tor_process_get_stdout_pipe(process_handle_t *process_handle)
+{
+  return process_handle->stdout_pipe;
+}
+#else /* !(defined(_WIN32)) */
+/* DOCDOC tor_process_get_stdout_pipe */
+int
+tor_process_get_stdout_pipe(process_handle_t *process_handle)
+{
+  return process_handle->stdout_pipe;
+}
+#endif /* defined(_WIN32) */
+
+/* DOCDOC process_handle_new */
+static process_handle_t *
+process_handle_new(void)
+{
+  process_handle_t *out = tor_malloc_zero(sizeof(process_handle_t));
+
+#ifdef _WIN32
+  out->stdin_pipe = INVALID_HANDLE_VALUE;
+  out->stdout_pipe = INVALID_HANDLE_VALUE;
+  out->stderr_pipe = INVALID_HANDLE_VALUE;
+#else
+  out->stdin_pipe = -1;
+  out->stdout_pipe = -1;
+  out->stderr_pipe = -1;
+#endif /* defined(_WIN32) */
+
+  return out;
+}
+
+#ifndef _WIN32
+/** Invoked when a process that we've launched via tor_spawn_background() has
+ * been found to have terminated.
+ */
+static void
+process_handle_waitpid_cb(int status, void *arg)
+{
+  process_handle_t *process_handle = arg;
+
+  process_handle->waitpid_exit_status = status;
+  clear_waitpid_callback(process_handle->waitpid_cb);
+  if (process_handle->status == PROCESS_STATUS_RUNNING)
+    process_handle->status = PROCESS_STATUS_NOTRUNNING;
+  process_handle->waitpid_cb = 0;
+}
+#endif /* !defined(_WIN32) */
+
+/**
+ * @name child-process states
+ *
+ * Each of these values represents a possible state that a child process can
+ * be in.  They're used to determine what to say when telling the parent how
+ * far along we were before failure.
+ *
+ * @{
+ */
+#define CHILD_STATE_INIT 0
+#define CHILD_STATE_PIPE 1
+#define CHILD_STATE_MAXFD 2
+#define CHILD_STATE_FORK 3
+#define CHILD_STATE_DUPOUT 4
+#define CHILD_STATE_DUPERR 5
+#define CHILD_STATE_DUPIN 6
+#define CHILD_STATE_CLOSEFD 7
+#define CHILD_STATE_EXEC 8
+#define CHILD_STATE_FAILEXEC 9
+/** @} */
+/**
+ * Boolean.  If true, then Tor may call execve or CreateProcess via
+ * tor_spawn_background.
+ **/
+static int may_spawn_background_process = 1;
+/**
+ * Turn off may_spawn_background_process, so that all future calls to
+ * tor_spawn_background are guaranteed to fail.
+ **/
+void
+tor_disable_spawning_background_processes(void)
+{
+  may_spawn_background_process = 0;
+}
+/** Start a program in the background. If <b>filename</b> contains a '/', then
+ * it will be treated as an absolute or relative path.  Otherwise, on
+ * non-Windows systems, the system path will be searched for <b>filename</b>.
+ * On Windows, only the current directory will be searched. Here, to search the
+ * system path (as well as the application directory, current working
+ * directory, and system directories), set filename to NULL.
+ *
+ * The strings in <b>argv</b> will be passed as the command line arguments of
+ * the child program (following convention, argv[0] should normally be the
+ * filename of the executable, and this must be the case if <b>filename</b> is
+ * NULL). The last element of argv must be NULL. A handle to the child process
+ * will be returned in process_handle (which must be non-NULL). Read
+ * process_handle.status to find out if the process was successfully launched.
+ * For convenience, process_handle.status is returned by this function.
+ *
+ * Some parts of this code are based on the POSIX subprocess module from
+ * Python, and example code from
+ * http://msdn.microsoft.com/en-us/library/ms682499%28v=vs.85%29.aspx.
+ */
+int
+tor_spawn_background(const char *const filename, const char **argv,
+                     process_environment_t *env,
+                     process_handle_t **process_handle_out)
+{
+  if (BUG(may_spawn_background_process == 0)) {
+    /* We should never reach this point if we're forbidden to spawn
+     * processes. Instead we should have caught the attempt earlier. */
+    return PROCESS_STATUS_ERROR;
+  }
+
+#ifdef _WIN32
+  HANDLE stdout_pipe_read = NULL;
+  HANDLE stdout_pipe_write = NULL;
+  HANDLE stderr_pipe_read = NULL;
+  HANDLE stderr_pipe_write = NULL;
+  HANDLE stdin_pipe_read = NULL;
+  HANDLE stdin_pipe_write = NULL;
+  process_handle_t *process_handle;
+  int status;
+
+  STARTUPINFOA siStartInfo;
+  BOOL retval = FALSE;
+
+  SECURITY_ATTRIBUTES saAttr;
+  char *joined_argv;
+
+  saAttr.nLength = sizeof(SECURITY_ATTRIBUTES);
+  saAttr.bInheritHandle = TRUE;
+  /* TODO: should we set explicit security attributes? (#2046, comment 5) */
+  saAttr.lpSecurityDescriptor = NULL;
+
+  /* Assume failure to start process */
+  status = PROCESS_STATUS_ERROR;
+
+  /* Set up pipe for stdout */
+  if (!CreatePipe(&stdout_pipe_read, &stdout_pipe_write, &saAttr, 0)) {
+    log_warn(LD_GENERAL,
+      "Failed to create pipe for stdout communication with child process: %s",
+      format_win32_error(GetLastError()));
+    return status;
+  }
+  if (!SetHandleInformation(stdout_pipe_read, HANDLE_FLAG_INHERIT, 0)) {
+    log_warn(LD_GENERAL,
+      "Failed to configure pipe for stdout communication with child "
+      "process: %s", format_win32_error(GetLastError()));
+    return status;
+  }
+
+  /* Set up pipe for stderr */
+  if (!CreatePipe(&stderr_pipe_read, &stderr_pipe_write, &saAttr, 0)) {
+    log_warn(LD_GENERAL,
+      "Failed to create pipe for stderr communication with child process: %s",
+      format_win32_error(GetLastError()));
+    return status;
+  }
+  if (!SetHandleInformation(stderr_pipe_read, HANDLE_FLAG_INHERIT, 0)) {
+    log_warn(LD_GENERAL,
+      "Failed to configure pipe for stderr communication with child "
+      "process: %s", format_win32_error(GetLastError()));
+    return status;
+  }
+
+  /* Set up pipe for stdin */
+  if (!CreatePipe(&stdin_pipe_read, &stdin_pipe_write, &saAttr, 0)) {
+    log_warn(LD_GENERAL,
+      "Failed to create pipe for stdin communication with child process: %s",
+      format_win32_error(GetLastError()));
+    return status;
+  }
+  if (!SetHandleInformation(stdin_pipe_write, HANDLE_FLAG_INHERIT, 0)) {
+    log_warn(LD_GENERAL,
+      "Failed to configure pipe for stdin communication with child "
+      "process: %s", format_win32_error(GetLastError()));
+    return status;
+  }
+
+  /* Create the child process */
+
+  /* Windows expects argv to be a whitespace delimited string, so join argv up
+   */
+  joined_argv = tor_join_win_cmdline(argv);
+
+  process_handle = process_handle_new();
+  process_handle->status = status;
+
+  ZeroMemory(&(process_handle->pid), sizeof(PROCESS_INFORMATION));
+  ZeroMemory(&siStartInfo, sizeof(STARTUPINFO));
+  siStartInfo.cb = sizeof(STARTUPINFO);
+  siStartInfo.hStdError = stderr_pipe_write;
+  siStartInfo.hStdOutput = stdout_pipe_write;
+  siStartInfo.hStdInput = stdin_pipe_read;
+  siStartInfo.dwFlags |= STARTF_USESTDHANDLES;
+
+  /* Create the child process */
+
+  retval = CreateProcessA(filename,      // module name
+                 joined_argv,   // command line
+  /* TODO: should we set explicit security attributes? (#2046, comment 5) */
+                 NULL,          // process security attributes
+                 NULL,          // primary thread security attributes
+                 TRUE,          // handles are inherited
+  /*(TODO: set CREATE_NEW CONSOLE/PROCESS_GROUP to make GetExitCodeProcess()
+   * work?) */
+                 CREATE_NO_WINDOW,             // creation flags
+                 (env==NULL) ? NULL : env->windows_environment_block,
+                 NULL,          // use parent's current directory
+                 &siStartInfo,  // STARTUPINFO pointer
+                 &(process_handle->pid));  // receives PROCESS_INFORMATION
+
+  tor_free(joined_argv);
+
+  if (!retval) {
+    log_warn(LD_GENERAL,
+      "Failed to create child process %s: %s", filename?filename:argv[0],
+      format_win32_error(GetLastError()));
+    tor_free(process_handle);
+  } else  {
+    /* TODO: Close hProcess and hThread in process_handle->pid? */
+    process_handle->stdout_pipe = stdout_pipe_read;
+    process_handle->stderr_pipe = stderr_pipe_read;
+    process_handle->stdin_pipe = stdin_pipe_write;
+    status = process_handle->status = PROCESS_STATUS_RUNNING;
+  }
+
+  /* TODO: Close pipes on exit */
+  *process_handle_out = process_handle;
+  return status;
+#else /* !(defined(_WIN32)) */
+  pid_t pid;
+  int stdout_pipe[2];
+  int stderr_pipe[2];
+  int stdin_pipe[2];
+  int fd, retval;
+  process_handle_t *process_handle;
+  int status;
+
+  const char *error_message = SPAWN_ERROR_MESSAGE;
+  size_t error_message_length;
+
+  /* Represents where in the process of spawning the program is;
+     this is used for printing out the error message */
+  unsigned char child_state = CHILD_STATE_INIT;
+
+  char hex_errno[HEX_ERRNO_SIZE + 2]; /* + 1 should be sufficient actually */
+
+  static int max_fd = -1;
+
+  status = PROCESS_STATUS_ERROR;
+
+  /* We do the strlen here because strlen() is not signal handler safe,
+     and we are not allowed to use unsafe functions between fork and exec */
+  error_message_length = strlen(error_message);
+
+  // child_state = CHILD_STATE_PIPE;
+
+  /* Set up pipe for redirecting stdout, stderr, and stdin of child */
+  retval = pipe(stdout_pipe);
+  if (-1 == retval) {
+    log_warn(LD_GENERAL,
+      "Failed to set up pipe for stdout communication with child process: %s",
+       strerror(errno));
+    return status;
+  }
+
+  retval = pipe(stderr_pipe);
+  if (-1 == retval) {
+    log_warn(LD_GENERAL,
+      "Failed to set up pipe for stderr communication with child process: %s",
+      strerror(errno));
+
+    close(stdout_pipe[0]);
+    close(stdout_pipe[1]);
+
+    return status;
+  }
+
+  retval = pipe(stdin_pipe);
+  if (-1 == retval) {
+    log_warn(LD_GENERAL,
+      "Failed to set up pipe for stdin communication with child process: %s",
+       strerror(errno));
+
+    close(stdout_pipe[0]);
+    close(stdout_pipe[1]);
+    close(stderr_pipe[0]);
+    close(stderr_pipe[1]);
+
+    return status;
+  }
+
+  // child_state = CHILD_STATE_MAXFD;
+
+#ifdef _SC_OPEN_MAX
+  if (-1 == max_fd) {
+    max_fd = (int) sysconf(_SC_OPEN_MAX);
+    if (max_fd == -1) {
+      max_fd = DEFAULT_MAX_FD;
+      log_warn(LD_GENERAL,
+               "Cannot find maximum file descriptor, assuming %d", max_fd);
+    }
+  }
+#else /* !(defined(_SC_OPEN_MAX)) */
+  max_fd = DEFAULT_MAX_FD;
+#endif /* defined(_SC_OPEN_MAX) */
+
+  // child_state = CHILD_STATE_FORK;
+
+  pid = fork();
+  if (0 == pid) {
+    /* In child */
+
+#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__)
+    /* Attempt to have the kernel issue a SIGTERM if the parent
+     * goes away. Certain attributes of the binary being execve()ed
+     * will clear this during the execve() call, but it's better
+     * than nothing.
+     */
+    prctl(PR_SET_PDEATHSIG, SIGTERM);
+#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */
+
+    child_state = CHILD_STATE_DUPOUT;
+
+    /* Link child stdout to the write end of the pipe */
+    retval = dup2(stdout_pipe[1], STDOUT_FILENO);
+    if (-1 == retval)
+        goto error;
+
+    child_state = CHILD_STATE_DUPERR;
+
+    /* Link child stderr to the write end of the pipe */
+    retval = dup2(stderr_pipe[1], STDERR_FILENO);
+    if (-1 == retval)
+        goto error;
+
+    child_state = CHILD_STATE_DUPIN;
+
+    /* Link child stdin to the read end of the pipe */
+    retval = dup2(stdin_pipe[0], STDIN_FILENO);
+    if (-1 == retval)
+      goto error;
+
+    // child_state = CHILD_STATE_CLOSEFD;
+
+    close(stderr_pipe[0]);
+    close(stderr_pipe[1]);
+    close(stdout_pipe[0]);
+    close(stdout_pipe[1]);
+    close(stdin_pipe[0]);
+    close(stdin_pipe[1]);
+
+    /* Close all other fds, including the read end of the pipe */
+    /* XXX: We should now be doing enough FD_CLOEXEC setting to make
+     * this needless. */
+    for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) {
+      close(fd);
+    }
+
+    // child_state = CHILD_STATE_EXEC;
+
+    /* Call the requested program. We need the cast because
+       execvp doesn't define argv as const, even though it
+       does not modify the arguments */
+    if (env)
+      execve(filename, (char *const *) argv, env->unixoid_environment_block);
+    else {
+      static char *new_env[] = { NULL };
+      execve(filename, (char *const *) argv, new_env);
+    }
+
+    /* If we got here, the exec or open(/dev/null) failed */
+
+    child_state = CHILD_STATE_FAILEXEC;
+
+  error:
+    {
+      /* XXX: are we leaking fds from the pipe? */
+      int n, err=0;
+      ssize_t nbytes;
+
+      n = format_helper_exit_status(child_state, errno, hex_errno);
+
+      if (n >= 0) {
+        /* Write the error message. GCC requires that we check the return
+           value, but there is nothing we can do if it fails */
+        /* TODO: Don't use STDOUT, use a pipe set up just for this purpose */
+        nbytes = write(STDOUT_FILENO, error_message, error_message_length);
+        err = (nbytes < 0);
+        nbytes = write(STDOUT_FILENO, hex_errno, n);
+        err += (nbytes < 0);
+      }
+
+      _exit(err?254:255); // exit ok: in child.
+    }
+
+    /* Never reached, but avoids compiler warning */
+    return status; // LCOV_EXCL_LINE
+  }
+
+  /* In parent */
+
+  if (-1 == pid) {
+    log_warn(LD_GENERAL, "Failed to fork child process: %s", strerror(errno));
+    close(stdin_pipe[0]);
+    close(stdin_pipe[1]);
+    close(stdout_pipe[0]);
+    close(stdout_pipe[1]);
+    close(stderr_pipe[0]);
+    close(stderr_pipe[1]);
+    return status;
+  }
+
+  process_handle = process_handle_new();
+  process_handle->status = status;
+  process_handle->pid = pid;
+
+  /* TODO: If the child process forked but failed to exec, waitpid it */
+
+  /* Return read end of the pipes to caller, and close write end */
+  process_handle->stdout_pipe = stdout_pipe[0];
+  retval = close(stdout_pipe[1]);
+
+  if (-1 == retval) {
+    log_warn(LD_GENERAL,
+            "Failed to close write end of stdout pipe in parent process: %s",
+            strerror(errno));
+  }
+
+  process_handle->waitpid_cb = set_waitpid_callback(pid,
+                                                    process_handle_waitpid_cb,
+                                                    process_handle);
+
+  process_handle->stderr_pipe = stderr_pipe[0];
+  retval = close(stderr_pipe[1]);
+
+  if (-1 == retval) {
+    log_warn(LD_GENERAL,
+            "Failed to close write end of stderr pipe in parent process: %s",
+            strerror(errno));
+  }
+
+  /* Return write end of the stdin pipe to caller, and close the read end */
+  process_handle->stdin_pipe = stdin_pipe[1];
+  retval = close(stdin_pipe[0]);
+
+  if (-1 == retval) {
+    log_warn(LD_GENERAL,
+            "Failed to close read end of stdin pipe in parent process: %s",
+            strerror(errno));
+  }
+
+  status = process_handle->status = PROCESS_STATUS_RUNNING;
+  /* Set stdin/stdout/stderr pipes to be non-blocking */
+  if (fcntl(process_handle->stdout_pipe, F_SETFL, O_NONBLOCK) < 0 ||
+      fcntl(process_handle->stderr_pipe, F_SETFL, O_NONBLOCK) < 0 ||
+      fcntl(process_handle->stdin_pipe, F_SETFL, O_NONBLOCK) < 0) {
+    log_warn(LD_GENERAL, "Failed to set stderror/stdout/stdin pipes "
+             "nonblocking in parent process: %s", strerror(errno));
+  }
+
+  *process_handle_out = process_handle;
+  return status;
+#endif /* defined(_WIN32) */
+}
+
+/** Destroy all resources allocated by the process handle in
+ *  <b>process_handle</b>.
+ *  If <b>also_terminate_process</b> is true, also terminate the
+ *  process of the process handle. */
+MOCK_IMPL(void,
+tor_process_handle_destroy,(process_handle_t *process_handle,
+                            int also_terminate_process))
+{
+  if (!process_handle)
+    return;
+
+  if (also_terminate_process) {
+    if (tor_terminate_process(process_handle) < 0) {
+      const char *errstr =
+#ifdef _WIN32
+        format_win32_error(GetLastError());
+#else
+        strerror(errno);
+#endif
+      log_notice(LD_GENERAL, "Failed to terminate process with "
+                 "PID '%d' ('%s').", tor_process_get_pid(process_handle),
+                 errstr);
+    } else {
+      log_info(LD_GENERAL, "Terminated process with PID '%d'.",
+               tor_process_get_pid(process_handle));
+    }
+  }
+
+  process_handle->status = PROCESS_STATUS_NOTRUNNING;
+
+#ifdef _WIN32
+  if (process_handle->stdout_pipe)
+    CloseHandle(process_handle->stdout_pipe);
+
+  if (process_handle->stderr_pipe)
+    CloseHandle(process_handle->stderr_pipe);
+
+  if (process_handle->stdin_pipe)
+    CloseHandle(process_handle->stdin_pipe);
+#else /* !(defined(_WIN32)) */
+  close(process_handle->stdout_pipe);
+  close(process_handle->stderr_pipe);
+  close(process_handle->stdin_pipe);
+
+  clear_waitpid_callback(process_handle->waitpid_cb);
+#endif /* defined(_WIN32) */
+
+  memset(process_handle, 0x0f, sizeof(process_handle_t));
+  tor_free(process_handle);
+}
+
+/** Get the exit code of a process specified by <b>process_handle</b> and store
+ * it in <b>exit_code</b>, if set to a non-NULL value.  If <b>block</b> is set
+ * to true, the call will block until the process has exited.  Otherwise if
+ * the process is still running, the function will return
+ * PROCESS_EXIT_RUNNING, and exit_code will be left unchanged. Returns
+ * PROCESS_EXIT_EXITED if the process did exit. If there is a failure,
+ * PROCESS_EXIT_ERROR will be returned and the contents of exit_code (if
+ * non-NULL) will be undefined. N.B. Under *nix operating systems, this will
+ * probably not work in Tor, because waitpid() is called in main.c to reap any
+ * terminated child processes.*/
+int
+tor_get_exit_code(process_handle_t *process_handle,
+                  int block, int *exit_code)
+{
+#ifdef _WIN32
+  DWORD retval;
+  BOOL success;
+
+  if (block) {
+    /* Wait for the process to exit */
+    retval = WaitForSingleObject(process_handle->pid.hProcess, INFINITE);
+    if (retval != WAIT_OBJECT_0) {
+      log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s",
+              (int)retval, format_win32_error(GetLastError()));
+      return PROCESS_EXIT_ERROR;
+    }
+  } else {
+    retval = WaitForSingleObject(process_handle->pid.hProcess, 0);
+    if (WAIT_TIMEOUT == retval) {
+      /* Process has not exited */
+      return PROCESS_EXIT_RUNNING;
+    } else if (retval != WAIT_OBJECT_0) {
+      log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s",
+               (int)retval, format_win32_error(GetLastError()));
+      return PROCESS_EXIT_ERROR;
+    }
+  }
+
+  if (exit_code != NULL) {
+    success = GetExitCodeProcess(process_handle->pid.hProcess,
+                                 (PDWORD)exit_code);
+    if (!success) {
+      log_warn(LD_GENERAL, "GetExitCodeProcess() failed: %s",
+               format_win32_error(GetLastError()));
+      return PROCESS_EXIT_ERROR;
+    }
+  }
+#else /* !(defined(_WIN32)) */
+  int stat_loc;
+  int retval;
+
+  if (process_handle->waitpid_cb) {
+    /* We haven't processed a SIGCHLD yet. */
+    retval = waitpid(process_handle->pid, &stat_loc, block?0:WNOHANG);
+    if (retval == process_handle->pid) {
+      clear_waitpid_callback(process_handle->waitpid_cb);
+      process_handle->waitpid_cb = NULL;
+      process_handle->waitpid_exit_status = stat_loc;
+    }
+  } else {
+    /* We already got a SIGCHLD for this process, and handled it. */
+    retval = process_handle->pid;
+    stat_loc = process_handle->waitpid_exit_status;
+  }
+
+  if (!block && 0 == retval) {
+    /* Process has not exited */
+    return PROCESS_EXIT_RUNNING;
+  } else if (retval != process_handle->pid) {
+    log_warn(LD_GENERAL, "waitpid() failed for PID %d: %s",
+             (int)process_handle->pid, strerror(errno));
+    return PROCESS_EXIT_ERROR;
+  }
+
+  if (!WIFEXITED(stat_loc)) {
+    log_warn(LD_GENERAL, "Process %d did not exit normally",
+             (int)process_handle->pid);
+    return PROCESS_EXIT_ERROR;
+  }
+
+  if (exit_code != NULL)
+    *exit_code = WEXITSTATUS(stat_loc);
+#endif /* defined(_WIN32) */
+
+  return PROCESS_EXIT_EXITED;
+}
+
+#ifdef _WIN32
+/** Read from a handle <b>h</b> into <b>buf</b>, up to <b>count</b> bytes.  If
+ * <b>hProcess</b> is NULL, the function will return immediately if there is
+ * nothing more to read. Otherwise <b>hProcess</b> should be set to the handle
+ * to the process owning the <b>h</b>. In this case, the function will exit
+ * only once the process has exited, or <b>count</b> bytes are read. Returns
+ * the number of bytes read, or -1 on error. */
+ssize_t
+tor_read_all_handle(HANDLE h, char *buf, size_t count,
+                    const process_handle_t *process)
+{
+  size_t numread = 0;
+  BOOL retval;
+  DWORD byte_count;
+  BOOL process_exited = FALSE;
+
+  if (count > SIZE_T_CEILING || count > SSIZE_MAX)
+    return -1;
+
+  while (numread < count) {
+    /* Check if there is anything to read */
+    retval = PeekNamedPipe(h, NULL, 0, NULL, &byte_count, NULL);
+    if (!retval) {
+      log_warn(LD_GENERAL,
+        "Failed to peek from handle: %s",
+        format_win32_error(GetLastError()));
+      return -1;
+    } else if (0 == byte_count) {
+      /* Nothing available: process exited or it is busy */
+
+      /* Exit if we don't know whether the process is running */
+      if (NULL == process)
+        break;
+
+      /* The process exited and there's nothing left to read from it */
+      if (process_exited)
+        break;
+
+      /* If process is not running, check for output one more time in case
+         it wrote something after the peek was performed. Otherwise keep on
+         waiting for output */
+      tor_assert(process != NULL);
+      byte_count = WaitForSingleObject(process->pid.hProcess, 0);
+      if (WAIT_TIMEOUT != byte_count)
+        process_exited = TRUE;
+
+      continue;
+    }
+
+    /* There is data to read; read it */
+    retval = ReadFile(h, buf+numread, count-numread, &byte_count, NULL);
+    tor_assert(byte_count + numread <= count);
+    if (!retval) {
+      log_warn(LD_GENERAL, "Failed to read from handle: %s",
+        format_win32_error(GetLastError()));
+      return -1;
+    } else if (0 == byte_count) {
+      /* End of file */
+      break;
+    }
+    numread += byte_count;
+  }
+  return (ssize_t)numread;
+}
+#else /* !(defined(_WIN32)) */
+/** Read from a handle <b>fd</b> into <b>buf</b>, up to <b>count</b> bytes.  If
+ * <b>process</b> is NULL, the function will return immediately if there is
+ * nothing more to read. Otherwise data will be read until end of file, or
+ * <b>count</b> bytes are read.  Returns the number of bytes read, or -1 on
+ * error. Sets <b>eof</b> to true if <b>eof</b> is not NULL and the end of the
+ * file has been reached. */
+ssize_t
+tor_read_all_handle(int fd, char *buf, size_t count,
+                    const process_handle_t *process,
+                    int *eof)
+{
+  size_t numread = 0;
+  ssize_t result;
+
+  if (eof)
+    *eof = 0;
+
+  if (count > SIZE_T_CEILING || count > SSIZE_MAX)
+    return -1;
+
+  while (numread < count) {
+    result = read(fd, buf+numread, count-numread);
+
+    if (result == 0) {
+      log_debug(LD_GENERAL, "read() reached end of file");
+      if (eof)
+        *eof = 1;
+      break;
+    } else if (result < 0 && errno == EAGAIN) {
+      if (process)
+        continue;
+      else
+        break;
+    } else if (result < 0) {
+      log_warn(LD_GENERAL, "read() failed: %s", strerror(errno));
+      return -1;
+    }
+
+    numread += result;
+  }
+
+  log_debug(LD_GENERAL, "read() read %d bytes from handle", (int)numread);
+  return (ssize_t)numread;
+}
+#endif /* defined(_WIN32) */
+
+/** Read from stdout of a process until the process exits. */
+ssize_t
+tor_read_all_from_process_stdout(const process_handle_t *process_handle,
+                                 char *buf, size_t count)
+{
+#ifdef _WIN32
+  return tor_read_all_handle(process_handle->stdout_pipe, buf, count,
+                             process_handle);
+#else
+  return tor_read_all_handle(process_handle->stdout_pipe, buf, count,
+                             process_handle, NULL);
+#endif /* defined(_WIN32) */
+}
+
+/** Read from stdout of a process until the process exits. */
+ssize_t
+tor_read_all_from_process_stderr(const process_handle_t *process_handle,
+                                 char *buf, size_t count)
+{
+#ifdef _WIN32
+  return tor_read_all_handle(process_handle->stderr_pipe, buf, count,
+                             process_handle);
+#else
+  return tor_read_all_handle(process_handle->stderr_pipe, buf, count,
+                             process_handle, NULL);
+#endif /* defined(_WIN32) */
+}
+
+/** Return a string corresponding to <b>stream_status</b>. */
+const char *
+stream_status_to_string(enum stream_status stream_status)
+{
+  switch (stream_status) {
+    case IO_STREAM_OKAY:
+      return "okay";
+    case IO_STREAM_EAGAIN:
+      return "temporarily unavailable";
+    case IO_STREAM_TERM:
+      return "terminated";
+    case IO_STREAM_CLOSED:
+      return "closed";
+    default:
+      tor_fragile_assert();
+      return "unknown";
+  }
+}
+
+/** Split buf into lines, and add to smartlist. The buffer <b>buf</b> will be
+ * modified. The resulting smartlist will consist of pointers to buf, so there
+ * is no need to free the contents of sl. <b>buf</b> must be a NUL-terminated
+ * string. <b>len</b> should be set to the length of the buffer excluding the
+ * NUL. Non-printable characters (including NUL) will be replaced with "." */
+int
+tor_split_lines(smartlist_t *sl, char *buf, int len)
+{
+  /* Index in buf of the start of the current line */
+  int start = 0;
+  /* Index in buf of the current character being processed */
+  int cur = 0;
+  /* Are we currently in a line */
+  char in_line = 0;
+
+  /* Loop over string */
+  while (cur < len) {
+    /* Loop until end of line or end of string */
+    for (; cur < len; cur++) {
+      if (in_line) {
+        if ('\r' == buf[cur] || '\n' == buf[cur]) {
+          /* End of line */
+          buf[cur] = '\0';
+          /* Point cur to the next line */
+          cur++;
+          /* Line starts at start and ends with a nul */
+          break;
+        } else {
+          if (!TOR_ISPRINT(buf[cur]))
+            buf[cur] = '.';
+        }
+      } else {
+        if ('\r' == buf[cur] || '\n' == buf[cur]) {
+          /* Skip leading vertical space */
+          ;
+        } else {
+          in_line = 1;
+          start = cur;
+          if (!TOR_ISPRINT(buf[cur]))
+            buf[cur] = '.';
+        }
+      }
+    }
+    /* We are at the end of the line or end of string. If in_line is true there
+     * is a line which starts at buf+start and ends at a NUL. cur points to
+     * the character after the NUL. */
+    if (in_line)
+      smartlist_add(sl, (void *)(buf+start));
+    in_line = 0;
+  }
+  return smartlist_len(sl);
+}
+
+#ifdef _WIN32
+
+/** Return a smartlist containing lines outputted from
+ *  <b>handle</b>. Return NULL on error, and set
+ *  <b>stream_status_out</b> appropriately. */
+MOCK_IMPL(smartlist_t *,
+tor_get_lines_from_handle, (HANDLE *handle,
+                            enum stream_status *stream_status_out))
+{
+  int pos;
+  char stdout_buf[600] = {0};
+  smartlist_t *lines = NULL;
+
+  tor_assert(stream_status_out);
+
+  *stream_status_out = IO_STREAM_TERM;
+
+  pos = tor_read_all_handle(handle, stdout_buf, sizeof(stdout_buf) - 1, NULL);
+  if (pos < 0) {
+    *stream_status_out = IO_STREAM_TERM;
+    return NULL;
+  }
+  if (pos == 0) {
+    *stream_status_out = IO_STREAM_EAGAIN;
+    return NULL;
+  }
+
+  /* End with a null even if there isn't a \r\n at the end */
+  /* TODO: What if this is a partial line? */
+  stdout_buf[pos] = '\0';
+
+  /* Split up the buffer */
+  lines = smartlist_new();
+  tor_split_lines(lines, stdout_buf, pos);
+
+  /* Currently 'lines' is populated with strings residing on the
+     stack. Replace them with their exact copies on the heap: */
+  SMARTLIST_FOREACH(lines, char *, line,
+                    SMARTLIST_REPLACE_CURRENT(lines, line, tor_strdup(line)));
+
+  *stream_status_out = IO_STREAM_OKAY;
+
+  return lines;
+}
+
+#else /* !(defined(_WIN32)) */
+
+/** Return a smartlist containing lines outputted from
+ *  <b>fd</b>. Return NULL on error, and set
+ *  <b>stream_status_out</b> appropriately. */
+MOCK_IMPL(smartlist_t *,
+tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out))
+{
+  enum stream_status stream_status;
+  char stdout_buf[400];
+  smartlist_t *lines = NULL;
+
+  while (1) {
+    memset(stdout_buf, 0, sizeof(stdout_buf));
+
+    stream_status = get_string_from_pipe(fd,
+                                         stdout_buf, sizeof(stdout_buf) - 1);
+    if (stream_status != IO_STREAM_OKAY)
+      goto done;
+
+    if (!lines) lines = smartlist_new();
+    smartlist_split_string(lines, stdout_buf, "\n", 0, 0);
+  }
+
+ done:
+  *stream_status_out = stream_status;
+  return lines;
+}
+
+#endif /* defined(_WIN32) */
+
+/** Reads from <b>fd</b> and stores input in <b>buf_out</b> making
+ *  sure it's below <b>count</b> bytes.
+ *  If the string has a trailing newline, we strip it off.
+ *
+ * This function is specifically created to handle input from managed
+ * proxies, according to the pluggable transports spec. Make sure it
+ * fits your needs before using it.
+ *
+ * Returns:
+ * IO_STREAM_CLOSED: If the stream is closed.
+ * IO_STREAM_EAGAIN: If there is nothing to read and we should check back
+ *  later.
+ * IO_STREAM_TERM: If something is wrong with the stream.
+ * IO_STREAM_OKAY: If everything went okay and we got a string
+ *  in <b>buf_out</b>. */
+enum stream_status
+get_string_from_pipe(int fd, char *buf_out, size_t count)
+{
+  ssize_t ret;
+
+  tor_assert(count <= INT_MAX);
+
+  ret = read(fd, buf_out, count);
+
+  if (ret == 0)
+    return IO_STREAM_CLOSED;
+  else if (ret < 0 && errno == EAGAIN)
+    return IO_STREAM_EAGAIN;
+  else if (ret < 0)
+    return IO_STREAM_TERM;
+
+  if (buf_out[ret - 1] == '\n') {
+    /* Remove the trailing newline */
+    buf_out[ret - 1] = '\0';
+  } else
+    buf_out[ret] = '\0';
+
+  return IO_STREAM_OKAY;
+}
diff --git a/src/lib/process/subprocess.h b/src/lib/process/subprocess.h
new file mode 100644
index 0000000000..aa3127d62d
--- /dev/null
+++ b/src/lib/process/subprocess.h
@@ -0,0 +1,134 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file subprocess.h
+ * \brief Header for subprocess.c
+ **/
+
+#ifndef TOR_SUBPROCESS_H
+#define TOR_SUBPROCESS_H
+
+#include "lib/cc/torint.h"
+#include "lib/testsupport/testsupport.h"
+#include <stddef.h>
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+struct smartlist_t;
+
+void tor_disable_spawning_background_processes(void);
+
+typedef struct process_handle_t process_handle_t;
+struct process_environment_t;
+int tor_spawn_background(const char *const filename, const char **argv,
+                         struct process_environment_t *env,
+                         process_handle_t **process_handle_out);
+
+#define SPAWN_ERROR_MESSAGE "ERR: Failed to spawn background process - code "
+
+/** Status of an I/O stream. */
+enum stream_status {
+  IO_STREAM_OKAY,
+  IO_STREAM_EAGAIN,
+  IO_STREAM_TERM,
+  IO_STREAM_CLOSED
+};
+
+const char *stream_status_to_string(enum stream_status stream_status);
+
+enum stream_status get_string_from_pipe(int fd, char *buf, size_t count);
+
+/* Values of process_handle_t.status. */
+#define PROCESS_STATUS_NOTRUNNING 0
+#define PROCESS_STATUS_RUNNING 1
+#define PROCESS_STATUS_ERROR -1
+
+#ifdef SUBPROCESS_PRIVATE
+struct waitpid_callback_t;
+
+/** Structure to represent the state of a process with which Tor is
+ * communicating. The contents of this structure are private to util.c */
+struct process_handle_t {
+  /** One of the PROCESS_STATUS_* values */
+  int status;
+#ifdef _WIN32
+  HANDLE stdin_pipe;
+  HANDLE stdout_pipe;
+  HANDLE stderr_pipe;
+  PROCESS_INFORMATION pid;
+#else /* !(defined(_WIN32)) */
+  int stdin_pipe;
+  int stdout_pipe;
+  int stderr_pipe;
+  pid_t pid;
+  /** If the process has not given us a SIGCHLD yet, this has the
+   * waitpid_callback_t that gets invoked once it has. Otherwise this
+   * contains NULL. */
+  struct waitpid_callback_t *waitpid_cb;
+  /** The exit status reported by waitpid. */
+  int waitpid_exit_status;
+#endif /* defined(_WIN32) */
+};
+#endif /* defined(SUBPROCESS_PRIVATE) */
+
+/* Return values of tor_get_exit_code() */
+#define PROCESS_EXIT_RUNNING 1
+#define PROCESS_EXIT_EXITED 0
+#define PROCESS_EXIT_ERROR -1
+int tor_get_exit_code(process_handle_t *process_handle,
+                      int block, int *exit_code);
+int tor_split_lines(struct smartlist_t *sl, char *buf, int len);
+#ifdef _WIN32
+ssize_t tor_read_all_handle(HANDLE h, char *buf, size_t count,
+                            const process_handle_t *process);
+#else
+ssize_t tor_read_all_handle(int fd, char *buf, size_t count,
+                            const process_handle_t *process,
+                            int *eof);
+#endif /* defined(_WIN32) */
+ssize_t tor_read_all_from_process_stdout(
+    const process_handle_t *process_handle, char *buf, size_t count);
+ssize_t tor_read_all_from_process_stderr(
+    const process_handle_t *process_handle, char *buf, size_t count);
+char *tor_join_win_cmdline(const char *argv[]);
+
+int tor_process_get_pid(process_handle_t *process_handle);
+#ifdef _WIN32
+HANDLE tor_process_get_stdout_pipe(process_handle_t *process_handle);
+#else
+int tor_process_get_stdout_pipe(process_handle_t *process_handle);
+#endif
+
+#ifdef _WIN32
+MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(HANDLE *handle,
+                           enum stream_status *stream_status));
+#else
+MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(int fd,
+                                       enum stream_status *stream_status));
+#endif /* defined(_WIN32) */
+
+int tor_terminate_process(process_handle_t *process_handle);
+
+MOCK_DECL(void, tor_process_handle_destroy,(process_handle_t *process_handle,
+                                            int also_terminate_process));
+
+#ifdef SUBPROCESS_PRIVATE
+/* Prototypes for private functions only used by util.c (and unit tests) */
+
+#ifndef _WIN32
+STATIC int format_helper_exit_status(unsigned char child_state,
+                              int saved_errno, char *hex_errno);
+
+/* Space for hex values of child state, a slash, saved_errno (with
+   leading minus) and newline (no null) */
+#define HEX_ERRNO_SIZE (sizeof(char) * 2 + 1 + \
+                        1 + sizeof(int) * 2 + 1)
+#endif /* !defined(_WIN32) */
+
+#endif /* defined(SUBPROCESS_PRIVATE) */
+
+#endif
diff --git a/src/common/util_process.c b/src/lib/process/waitpid.c
index abda63720c..9b626394d2 100644
--- a/src/common/util_process.c
+++ b/src/lib/process/waitpid.c
@@ -1,29 +1,28 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file util_process.c
- * \brief utility functions for launching processes and checking their
- *    status. These functions are kept separately from procmon so that they
- *    won't require linking against libevent.
+ * \file waitpid.c
+ * \brief Convenience structures for handlers for handling waitpid().
  **/
 
 #include "orconfig.h"
 
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
+#ifndef _WIN32
+
+#include "lib/process/waitpid.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "ht.h"
+
 #ifdef HAVE_SYS_WAIT_H
 #include <sys/wait.h>
 #endif
 
-#include "compat.h"
-#include "util.h"
-#include "torlog.h"
-#include "util_process.h"
-#include "ht.h"
+#include <string.h>
 
 /* ================================================== */
 /* Convenience structures for handlers for waitpid().
@@ -32,8 +31,6 @@
  * monitoring a non-child process.
  */
 
-#ifndef _WIN32
-
 /** Mapping from a PID to a userfn/userdata pair. */
 struct waitpid_callback_t {
   HT_ENTRY(waitpid_callback_t) node;
@@ -154,5 +151,4 @@ notify_pending_waitpid_callbacks(void)
   }
 }
 
-#endif
-
+#endif /* !defined(_WIN32) */
diff --git a/src/common/util_process.h b/src/lib/process/waitpid.h
index d38301a354..5faef468c1 100644
--- a/src/common/util_process.h
+++ b/src/lib/process/waitpid.h
@@ -1,15 +1,19 @@
-/* Copyright (c) 2011-2016, The Tor Project, Inc. */
+/* Copyright (c) 2011-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file util_process.h
- * \brief Headers for util_process.c
+ * \file waitpid.h
+ * \brief Headers for waitpid.c
  **/
 
-#ifndef TOR_UTIL_PROCESS_H
-#define TOR_UTIL_PROCESS_H
+#ifndef TOR_WAITPID_H
+#define TOR_WAITPID_H
 
 #ifndef _WIN32
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+
 /** A callback structure waiting for us to get a SIGCHLD informing us that a
  * PID has been closed. Created by set_waitpid_callback. Cancelled or cleaned-
  * up from clear_waitpid_callback().  Do not access outside of the main thread;
@@ -20,7 +24,6 @@ waitpid_callback_t *set_waitpid_callback(pid_t pid,
                                          void (*fn)(int, void *), void *arg);
 void clear_waitpid_callback(waitpid_callback_t *ent);
 void notify_pending_waitpid_callbacks(void);
-#endif
-
-#endif
+#endif /* !defined(_WIN32) */
 
+#endif /* !defined(TOR_WAITPID_H) */
diff --git a/src/lib/sandbox/.may_include b/src/lib/sandbox/.may_include
new file mode 100644
index 0000000000..84906dfb3d
--- /dev/null
+++ b/src/lib/sandbox/.may_include
@@ -0,0 +1,15 @@
+orconfig.h
+
+lib/cc/*.h
+lib/container/*.h
+lib/err/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/net/*.h
+lib/sandbox/*.h
+lib/sandbox/*.inc
+lib/string/*.h
+
+ht.h
+siphash.h
+tor_queue.h
diff --git a/src/lib/sandbox/include.am b/src/lib/sandbox/include.am
new file mode 100644
index 0000000000..adfda6bde5
--- /dev/null
+++ b/src/lib/sandbox/include.am
@@ -0,0 +1,18 @@
+
+noinst_LIBRARIES += src/lib/libtor-sandbox.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-sandbox-testing.a
+endif
+
+src_lib_libtor_sandbox_a_SOURCES =			\
+	src/lib/sandbox/sandbox.c
+
+src_lib_libtor_sandbox_testing_a_SOURCES = \
+	$(src_lib_libtor_sandbox_a_SOURCES)
+src_lib_libtor_sandbox_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_sandbox_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/sandbox/linux_syscalls.inc		\
+	src/lib/sandbox/sandbox.h
diff --git a/src/common/linux_syscalls.inc b/src/lib/sandbox/linux_syscalls.inc
index cf47c73809..cf47c73809 100644
--- a/src/common/linux_syscalls.inc
+++ b/src/lib/sandbox/linux_syscalls.inc
diff --git a/src/common/sandbox.c b/src/lib/sandbox/sandbox.c
index 0a972d496b..e2356a1720 100644
--- a/src/common/sandbox.c
+++ b/src/lib/sandbox/sandbox.c
@@ -1,7 +1,7 @@
- /* Copyright (c) 2001 Matej Pfajfar.
+/* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,23 +17,34 @@
  * with the libevent fix.
  */
 #define _LARGEFILE64_SOURCE
-#endif
+#endif /* !defined(_LARGEFILE64_SOURCE) */
 
-/** Malloc mprotect limit in bytes. */
-#define MALLOC_MP_LIM 1048576
+/** Malloc mprotect limit in bytes.
+ *
+ * 28/06/2017: This value was increased from 16 MB to 20 MB after we introduced
+ * LZMA support in Tor (0.3.1.1-alpha). We limit our LZMA coder to 16 MB, but
+ * liblzma have a small overhead that we need to compensate for to avoid being
+ * killed by the sandbox.
+ */
+#define MALLOC_MP_LIM (20*1024*1024)
 
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
+#include <errno.h>
 
-#include "sandbox.h"
-#include "container.h"
-#include "torlog.h"
-#include "torint.h"
-#include "util.h"
-#include "tor_queue.h"
+#include "lib/sandbox/sandbox.h"
+#include "lib/container/map.h"
+#include "lib/err/torerr.h"
+#include "lib/log/log.h"
+#include "lib/cc/torint.h"
+#include "lib/net/resolve.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/scanf.h"
 
+#include "tor_queue.h"
 #include "ht.h"
+#include "siphash.h"
 
 #define DEBUGGING_CLOSE
 
@@ -73,8 +84,8 @@
   defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION)
 #define USE_BACKTRACE
 #define EXPOSE_CLEAN_BACKTRACE
-#include "backtrace.h"
-#endif
+#include "lib/err/backtrace.h"
+#endif /* defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && ... */
 
 #ifdef USE_BACKTRACE
 #include <execinfo.h>
@@ -100,7 +111,12 @@
 
 #define M_SYSCALL arm_r7
 
-#endif
+#elif defined(__aarch64__) && defined(__LP64__)
+
+#define REG_SYSCALL 8
+#define M_SYSCALL regs[REG_SYSCALL]
+
+#endif /* defined(__i386__) || ... */
 
 /**Determines if at least one sandbox is active.*/
 static int sandbox_active = 0;
@@ -151,6 +167,7 @@ static int filter_nopar_gen[] = {
     SCMP_SYS(fstat64),
 #endif
     SCMP_SYS(futex),
+    SCMP_SYS(getdents),
     SCMP_SYS(getdents64),
     SCMP_SYS(getegid),
 #ifdef __NR_getegid32
@@ -205,6 +222,7 @@ static int filter_nopar_gen[] = {
 #ifdef __NR_setrlimit
     SCMP_SYS(setrlimit),
 #endif
+    SCMP_SYS(shutdown),
 #ifdef __NR_sigaltstack
     SCMP_SYS(sigaltstack),
 #endif
@@ -293,37 +311,6 @@ sb_rt_sigaction(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
   return rc;
 }
 
-#if 0
-/**
- * Function responsible for setting up the execve syscall for
- * the seccomp filter sandbox.
- */
-static int
-sb_execve(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
-{
-  int rc;
-  sandbox_cfg_t *elem = NULL;
-
-  // for each dynamic parameter filters
-  for (elem = filter; elem != NULL; elem = elem->next) {
-    smp_param_t *param = elem->param;
-
-    if (param != NULL && param->prot == 1 && param->syscall
-        == SCMP_SYS(execve)) {
-      rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(execve),
-               SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value));
-      if (rc != 0) {
-        log_err(LD_BUG,"(Sandbox) failed to add execve syscall, received "
-            "libseccomp error %d", rc);
-        return rc;
-      }
-    }
-  }
-
-  return 0;
-}
-#endif
-
 /**
  * Function responsible for setting up the time syscall for
  * the seccomp filter sandbox.
@@ -337,7 +324,7 @@ sb_time(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
        SCMP_CMP(0, SCMP_CMP_EQ, 0));
 #else
   return 0;
-#endif
+#endif /* defined(__NR_time) */
 }
 
 /**
@@ -356,7 +343,7 @@ sb_accept4(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
   if (rc) {
     return rc;
   }
-#endif
+#endif /* defined(__i386__) */
 
   rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(accept4),
                    SCMP_CMP_MASKED(3, SOCK_CLOEXEC|SOCK_NONBLOCK, 0));
@@ -429,7 +416,7 @@ sb_mmap2(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
 
   return 0;
 }
-#endif
+#endif /* defined(__NR_mmap2) */
 
 #ifdef HAVE_GNU_LIBC_VERSION_H
 #ifdef HAVE_GNU_GET_LIBC_VERSION
@@ -457,9 +444,9 @@ libc_uses_openat_for_everything(void)
     return 1;
   else
     return 0;
-#else
+#else /* !(defined(CHECK_LIBC_VERSION)) */
   return 0;
-#endif
+#endif /* defined(CHECK_LIBC_VERSION) */
 }
 
 /** Allow a single file to be opened.  If <b>use_openat</b> is true,
@@ -540,7 +527,7 @@ sb_chmod(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
       rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(chmod),
             SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value));
       if (rc != 0) {
-        log_err(LD_BUG,"(Sandbox) failed to add open syscall, received "
+        log_err(LD_BUG,"(Sandbox) failed to add chmod syscall, received "
             "libseccomp error %d", rc);
         return rc;
       }
@@ -565,7 +552,7 @@ sb_chown(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
       rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(chown),
             SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value));
       if (rc != 0) {
-        log_err(LD_BUG,"(Sandbox) failed to add open syscall, received "
+        log_err(LD_BUG,"(Sandbox) failed to add chown syscall, received "
             "libseccomp error %d", rc);
         return rc;
       }
@@ -744,6 +731,25 @@ sb_socketpair(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
   return 0;
 }
 
+#ifdef HAVE_KIST_SUPPORT
+
+#include <linux/sockios.h>
+
+static int
+sb_ioctl(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
+{
+  int rc;
+  (void) filter;
+
+  rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(ioctl),
+                          SCMP_CMP(1, SCMP_CMP_EQ, SIOCOUTQNSD));
+  if (rc)
+    return rc;
+  return 0;
+}
+
+#endif /* defined(HAVE_KIST_SUPPORT) */
+
 /**
  * Function responsible for setting up the setsockopt syscall for
  * the seccomp filter sandbox.
@@ -784,7 +790,7 @@ sb_setsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
       SCMP_CMP(2, SCMP_CMP_EQ, SO_SNDBUFFORCE));
   if (rc)
     return rc;
-#endif
+#endif /* defined(HAVE_SYSTEMD) */
 
 #ifdef IP_TRANSPARENT
   rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(setsockopt),
@@ -792,7 +798,7 @@ sb_setsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
       SCMP_CMP(2, SCMP_CMP_EQ, IP_TRANSPARENT));
   if (rc)
     return rc;
-#endif
+#endif /* defined(IP_TRANSPARENT) */
 
 #ifdef IPV6_V6ONLY
   rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(setsockopt),
@@ -800,7 +806,7 @@ sb_setsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
       SCMP_CMP(2, SCMP_CMP_EQ, IPV6_V6ONLY));
   if (rc)
     return rc;
-#endif
+#endif /* defined(IPV6_V6ONLY) */
 
   return 0;
 }
@@ -833,7 +839,7 @@ sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
       SCMP_CMP(2, SCMP_CMP_EQ, SO_SNDBUF));
   if (rc)
     return rc;
-#endif
+#endif /* defined(HAVE_SYSTEMD) */
 
 #ifdef HAVE_LINUX_NETFILTER_IPV4_H
   rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt),
@@ -841,7 +847,7 @@ sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
       SCMP_CMP(2, SCMP_CMP_EQ, SO_ORIGINAL_DST));
   if (rc)
     return rc;
-#endif
+#endif /* defined(HAVE_LINUX_NETFILTER_IPV4_H) */
 
 #ifdef HAVE_LINUX_NETFILTER_IPV6_IP6_TABLES_H
   rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt),
@@ -849,7 +855,16 @@ sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
       SCMP_CMP(2, SCMP_CMP_EQ, IP6T_SO_ORIGINAL_DST));
   if (rc)
     return rc;
-#endif
+#endif /* defined(HAVE_LINUX_NETFILTER_IPV6_IP6_TABLES_H) */
+
+#ifdef HAVE_KIST_SUPPORT
+#include <netinet/tcp.h>
+  rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt),
+      SCMP_CMP(1, SCMP_CMP_EQ, SOL_TCP),
+      SCMP_CMP(2, SCMP_CMP_EQ, TCP_INFO));
+  if (rc)
+    return rc;
+#endif /* defined(HAVE_KIST_SUPPORT) */
 
   return 0;
 }
@@ -889,7 +904,7 @@ sb_fcntl64(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
 
   return 0;
 }
-#endif
+#endif /* defined(__NR_fcntl64) */
 
 /**
  * Function responsible for setting up the epoll_ctl syscall for
@@ -1086,8 +1101,8 @@ sb_stat64(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
       rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(stat64),
           SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value));
       if (rc != 0) {
-        log_err(LD_BUG,"(Sandbox) failed to add open syscall, received "
-            "libseccomp  error %d", rc);
+        log_err(LD_BUG,"(Sandbox) failed to add stat64 syscall, received "
+            "libseccomp error %d", rc);
         return rc;
       }
     }
@@ -1095,7 +1110,7 @@ sb_stat64(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
 
   return 0;
 }
-#endif
+#endif /* defined(__NR_stat64) */
 
 static int
 sb_kill(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
@@ -1107,7 +1122,7 @@ sb_kill(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
        SCMP_CMP(1, SCMP_CMP_EQ, 0));
 #else
   return 0;
-#endif
+#endif /* defined(__NR_kill) */
 }
 
 /**
@@ -1117,9 +1132,6 @@ sb_kill(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
 static sandbox_filter_func_t filter_func[] = {
     sb_rt_sigaction,
     sb_rt_sigprocmask,
-#if 0
-    sb_execve,
-#endif
     sb_time,
     sb_accept4,
 #ifdef __NR_mmap2
@@ -1148,6 +1160,9 @@ static sandbox_filter_func_t filter_func[] = {
     sb_setsockopt,
     sb_getsockopt,
     sb_socketpair,
+#ifdef HAVE_KIST_SUPPORT
+    sb_ioctl,
+#endif
     sb_kill
 };
 
@@ -1374,10 +1389,6 @@ sandbox_cfg_allow_stat_filename(sandbox_cfg_t **cfg, char *file)
   sandbox_cfg_t *elem = NULL;
 
   elem = new_element(SCMP_stat, file);
-  if (!elem) {
-    log_err(LD_BUG,"(Sandbox) failed to register parameter!");
-    return -1;
-  }
 
   elem->next = *cfg;
   *cfg = elem;
@@ -1391,10 +1402,6 @@ sandbox_cfg_allow_open_filename(sandbox_cfg_t **cfg, char *file)
   sandbox_cfg_t *elem = NULL;
 
   elem = new_element(SCMP_SYS(open), file);
-  if (!elem) {
-    log_err(LD_BUG,"(Sandbox) failed to register parameter!");
-    return -1;
-  }
 
   elem->next = *cfg;
   *cfg = elem;
@@ -1408,10 +1415,6 @@ sandbox_cfg_allow_chmod_filename(sandbox_cfg_t **cfg, char *file)
   sandbox_cfg_t *elem = NULL;
 
   elem = new_element(SCMP_SYS(chmod), file);
-  if (!elem) {
-    log_err(LD_BUG,"(Sandbox) failed to register parameter!");
-    return -1;
-  }
 
   elem->next = *cfg;
   *cfg = elem;
@@ -1425,10 +1428,6 @@ sandbox_cfg_allow_chown_filename(sandbox_cfg_t **cfg, char *file)
   sandbox_cfg_t *elem = NULL;
 
   elem = new_element(SCMP_SYS(chown), file);
-  if (!elem) {
-    log_err(LD_BUG,"(Sandbox) failed to register parameter!");
-    return -1;
-  }
 
   elem->next = *cfg;
   *cfg = elem;
@@ -1443,11 +1442,6 @@ sandbox_cfg_allow_rename(sandbox_cfg_t **cfg, char *file1, char *file2)
 
   elem = new_element2(SCMP_SYS(rename), file1, file2);
 
-  if (!elem) {
-    log_err(LD_BUG,"(Sandbox) failed to register parameter!");
-    return -1;
-  }
-
   elem->next = *cfg;
   *cfg = elem;
 
@@ -1460,10 +1454,6 @@ sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file)
   sandbox_cfg_t *elem = NULL;
 
   elem = new_element(SCMP_SYS(openat), file);
-  if (!elem) {
-    log_err(LD_BUG,"(Sandbox) failed to register parameter!");
-    return -1;
-  }
 
   elem->next = *cfg;
   *cfg = elem;
@@ -1471,199 +1461,6 @@ sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file)
   return 0;
 }
 
-#if 0
-int
-sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com)
-{
-  sandbox_cfg_t *elem = NULL;
-
-  elem = new_element(SCMP_SYS(execve), com);
-  if (!elem) {
-    log_err(LD_BUG,"(Sandbox) failed to register parameter!");
-    return -1;
-  }
-
-  elem->next = *cfg;
-  *cfg = elem;
-
-  return 0;
-}
-
-#endif
-
-/** Cache entry for getaddrinfo results; used when sandboxing is implemented
- * so that we can consult the cache when the sandbox prevents us from doing
- * getaddrinfo.
- *
- * We support only a limited range of getaddrinfo calls, where servname is null
- * and hints contains only socktype=SOCK_STREAM, family in INET,INET6,UNSPEC.
- */
-typedef struct cached_getaddrinfo_item_t {
-  HT_ENTRY(cached_getaddrinfo_item_t) node;
-  char *name;
-  int family;
-  /** set if no error; otherwise NULL */
-  struct addrinfo *res;
-  /** 0 for no error; otherwise an EAI_* value */
-  int err;
-} cached_getaddrinfo_item_t;
-
-static unsigned
-cached_getaddrinfo_item_hash(const cached_getaddrinfo_item_t *item)
-{
-  return (unsigned)siphash24g(item->name, strlen(item->name)) + item->family;
-}
-
-static unsigned
-cached_getaddrinfo_items_eq(const cached_getaddrinfo_item_t *a,
-                            const cached_getaddrinfo_item_t *b)
-{
-  return (a->family == b->family) && 0 == strcmp(a->name, b->name);
-}
-
-static void
-cached_getaddrinfo_item_free(cached_getaddrinfo_item_t *item)
-{
-  if (item == NULL)
-    return;
-
-  tor_free(item->name);
-  if (item->res)
-    freeaddrinfo(item->res);
-  tor_free(item);
-}
-
-static HT_HEAD(getaddrinfo_cache, cached_getaddrinfo_item_t)
-     getaddrinfo_cache = HT_INITIALIZER();
-
-HT_PROTOTYPE(getaddrinfo_cache, cached_getaddrinfo_item_t, node,
-             cached_getaddrinfo_item_hash,
-             cached_getaddrinfo_items_eq)
-HT_GENERATE2(getaddrinfo_cache, cached_getaddrinfo_item_t, node,
-             cached_getaddrinfo_item_hash,
-             cached_getaddrinfo_items_eq,
-             0.6, tor_reallocarray_, tor_free_)
-
-/** If true, don't try to cache getaddrinfo results. */
-static int sandbox_getaddrinfo_cache_disabled = 0;
-
-/** Tell the sandbox layer not to try to cache getaddrinfo results. Used as in
- * tor-resolve, when we have no intention of initializing crypto or of
- * installing the sandbox.*/
-void
-sandbox_disable_getaddrinfo_cache(void)
-{
-  sandbox_getaddrinfo_cache_disabled = 1;
-}
-
-void
-sandbox_freeaddrinfo(struct addrinfo *ai)
-{
-  if (sandbox_getaddrinfo_cache_disabled)
-    freeaddrinfo(ai);
-}
-
-int
-sandbox_getaddrinfo(const char *name, const char *servname,
-                    const struct addrinfo *hints,
-                    struct addrinfo **res)
-{
-  int err;
-  struct cached_getaddrinfo_item_t search, *item;
-
-  if (sandbox_getaddrinfo_cache_disabled) {
-    return getaddrinfo(name, NULL, hints, res);
-  }
-
-  if (servname != NULL) {
-    log_warn(LD_BUG, "called with non-NULL servname");
-    return EAI_NONAME;
-  }
-  if (name == NULL) {
-    log_warn(LD_BUG, "called with NULL name");
-    return EAI_NONAME;
-  }
-
-  *res = NULL;
-
-  memset(&search, 0, sizeof(search));
-  search.name = (char *) name;
-  search.family = hints ? hints->ai_family : AF_UNSPEC;
-  item = HT_FIND(getaddrinfo_cache, &getaddrinfo_cache, &search);
-
-  if (! sandbox_is_active()) {
-    /* If the sandbox is not turned on yet, then getaddrinfo and store the
-       result. */
-
-    err = getaddrinfo(name, NULL, hints, res);
-    log_info(LD_NET,"(Sandbox) getaddrinfo %s.", err ? "failed" : "succeeded");
-
-    if (! item) {
-      item = tor_malloc_zero(sizeof(*item));
-      item->name = tor_strdup(name);
-      item->family = hints ? hints->ai_family : AF_UNSPEC;
-      HT_INSERT(getaddrinfo_cache, &getaddrinfo_cache, item);
-    }
-
-    if (item->res) {
-      freeaddrinfo(item->res);
-      item->res = NULL;
-    }
-    item->res = *res;
-    item->err = err;
-    return err;
-  }
-
-  /* Otherwise, the sanbox is on.  If we have an item, yield its cached
-     result. */
-  if (item) {
-    *res = item->res;
-    return item->err;
-  }
-
-  /* getting here means something went wrong */
-  log_err(LD_BUG,"(Sandbox) failed to get address %s!", name);
-  return EAI_NONAME;
-}
-
-int
-sandbox_add_addrinfo(const char *name)
-{
-  struct addrinfo *res;
-  struct addrinfo hints;
-  int i;
-  static const int families[] = { AF_INET, AF_INET6, AF_UNSPEC };
-
-  memset(&hints, 0, sizeof(hints));
-  hints.ai_socktype = SOCK_STREAM;
-  for (i = 0; i < 3; ++i) {
-    hints.ai_family = families[i];
-
-    res = NULL;
-    (void) sandbox_getaddrinfo(name, NULL, &hints, &res);
-    if (res)
-      sandbox_freeaddrinfo(res);
-  }
-
-  return 0;
-}
-
-void
-sandbox_free_getaddrinfo_cache(void)
-{
-  cached_getaddrinfo_item_t **next, **item, *this;
-
-  for (item = HT_START(getaddrinfo_cache, &getaddrinfo_cache);
-       item;
-       item = next) {
-    this = *item;
-    next = HT_NEXT_RMV(getaddrinfo_cache, &getaddrinfo_cache, item);
-    cached_getaddrinfo_item_free(this);
-  }
-
-  HT_CLEAR(getaddrinfo_cache, &getaddrinfo_cache);
-}
-
 /**
  * Function responsible for going through the parameter syscall filters and
  * call each function pointer in the list.
@@ -1747,20 +1544,24 @@ install_syscall_filter(sandbox_cfg_t* cfg)
 
   // loading the seccomp2 filter
   if ((rc = seccomp_load(ctx))) {
-    log_err(LD_BUG, "(Sandbox) failed to load: %d (%s)!", rc,
+    log_err(LD_BUG, "(Sandbox) failed to load: %d (%s)! "
+            "Are you sure that your kernel has seccomp2 support? The "
+            "sandbox won't work without it.", rc,
             strerror(-rc));
     goto end;
   }
 
   // marking the sandbox as active
   sandbox_active = 1;
+  tor_make_getaddrinfo_cache_active();
 
  end:
   seccomp_release(ctx);
   return (rc < 0 ? -rc : rc);
 }
 
-#include "linux_syscalls.inc"
+#include "lib/sandbox/linux_syscalls.inc"
+
 static const char *
 get_syscall_name(int syscall_num)
 {
@@ -1815,7 +1616,7 @@ sigsys_debugging(int nr, siginfo_t *info, void *void_context)
   /* Clean up the top stack frame so we get the real function
    * name for the most recently failing function. */
   clean_backtrace(syscall_cb_buf, depth, ctx);
-#endif
+#endif /* defined(USE_BACKTRACE) */
 
   syscall_name = get_syscall_name(syscall);
 
@@ -1831,7 +1632,7 @@ sigsys_debugging(int nr, siginfo_t *info, void *void_context)
 #endif
 
 #if defined(DEBUGGING_CLOSE)
-  _exit(1);
+  _exit(1); // exit ok: programming error has led to sandbox failure.
 #endif // DEBUGGING_CLOSE
 }
 
@@ -1889,7 +1690,7 @@ register_cfg(sandbox_cfg_t* cfg)
   return 0;
 }
 
-#endif // USE_LIBSECCOMP
+#endif /* defined(USE_LIBSECCOMP) */
 
 #ifdef USE_LIBSECCOMP
 /**
@@ -1919,7 +1720,7 @@ sandbox_is_active(void)
 {
   return sandbox_active != 0;
 }
-#endif // USE_LIBSECCOMP
+#endif /* defined(USE_LIBSECCOMP) */
 
 sandbox_cfg_t*
 sandbox_cfg_new(void)
@@ -1947,7 +1748,7 @@ sandbox_init(sandbox_cfg_t *cfg)
            "Currently, sandboxing is only implemented on Linux. The feature "
            "is disabled on your platform.");
   return 0;
-#endif
+#endif /* defined(USE_LIBSECCOMP) || ... */
 }
 
 #ifndef USE_LIBSECCOMP
@@ -1965,15 +1766,6 @@ sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file)
   return 0;
 }
 
-#if 0
-int
-sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com)
-{
-  (void)cfg; (void)com;
-  return 0;
-}
-#endif
-
 int
 sandbox_cfg_allow_stat_filename(sandbox_cfg_t **cfg, char *file)
 {
@@ -2012,5 +1804,5 @@ void
 sandbox_disable_getaddrinfo_cache(void)
 {
 }
-#endif
 
+#endif /* !defined(USE_LIBSECCOMP) */
diff --git a/src/common/sandbox.h b/src/lib/sandbox/sandbox.h
index c5963e3119..5bec09a36a 100644
--- a/src/common/sandbox.h
+++ b/src/lib/sandbox/sandbox.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,7 +13,7 @@
 #define SANDBOX_H_
 
 #include "orconfig.h"
-#include "torint.h"
+#include "lib/cc/torint.h"
 
 #ifndef SYS_SECCOMP
 
@@ -23,7 +23,7 @@
  */
 #define SYS_SECCOMP 1
 
-#endif
+#endif /* !defined(SYS_SECCOMP) */
 
 #if defined(HAVE_SECCOMP_H) && defined(__linux__)
 #define USE_LIBSECCOMP
@@ -101,37 +101,16 @@ typedef struct {
   sandbox_cfg_t *filter_dynamic;
 } sandbox_t;
 
-#endif // USE_LIBSECCOMP
-
-#ifdef USE_LIBSECCOMP
-/** Pre-calls getaddrinfo in order to pre-record result. */
-int sandbox_add_addrinfo(const char *addr);
-
-struct addrinfo;
-/** Replacement for getaddrinfo(), using pre-recorded results. */
-int sandbox_getaddrinfo(const char *name, const char *servname,
-                        const struct addrinfo *hints,
-                        struct addrinfo **res);
-void sandbox_freeaddrinfo(struct addrinfo *addrinfo);
-void sandbox_free_getaddrinfo_cache(void);
-#else
-#define sandbox_getaddrinfo(name, servname, hints, res)  \
-  getaddrinfo((name),(servname), (hints),(res))
-#define sandbox_add_addrinfo(name) \
-  ((void)(name))
-#define sandbox_freeaddrinfo(addrinfo) \
-  freeaddrinfo((addrinfo))
-#define sandbox_free_getaddrinfo_cache()
-#endif
+#endif /* defined(USE_LIBSECCOMP) */
 
 #ifdef USE_LIBSECCOMP
 /** Returns a registered protected string used with the sandbox, given that
  * it matches the parameter.
  */
 const char* sandbox_intern_string(const char *param);
-#else
+#else /* !(defined(USE_LIBSECCOMP)) */
 #define sandbox_intern_string(s) (s)
-#endif
+#endif /* defined(USE_LIBSECCOMP) */
 
 /** Creates an empty sandbox configuration file.*/
 sandbox_cfg_t * sandbox_cfg_new(void);
@@ -156,14 +135,6 @@ int sandbox_cfg_allow_rename(sandbox_cfg_t **cfg, char *file1, char *file2);
  */
 int sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file);
 
-#if 0
-/**
- * Function used to add a execve allowed filename to a supplied configuration.
- * The (char*) specifies the path to the allowed file; that pointer is stolen.
- */
-int sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com);
-#endif
-
 /**
  * Function used to add a stat/stat64 allowed filename to a configuration.
  * The (char*) specifies the path to the allowed file; that pointer is stolen.
@@ -176,7 +147,4 @@ int sandbox_init(sandbox_cfg_t* cfg);
 /** Return true iff the sandbox is turned on. */
 int sandbox_is_active(void);
 
-void sandbox_disable_getaddrinfo_cache(void);
-
-#endif /* SANDBOX_H_ */
-
+#endif /* !defined(SANDBOX_H_) */
diff --git a/src/lib/smartlist_core/.may_include b/src/lib/smartlist_core/.may_include
new file mode 100644
index 0000000000..a8507761a4
--- /dev/null
+++ b/src/lib/smartlist_core/.may_include
@@ -0,0 +1,7 @@
+orconfig.h
+lib/cc/*.h
+lib/malloc/*.h
+lib/err/*.h
+lib/string/*.h
+lib/smartlist_core/*.h
+lib/testsupport/testsupport.h
diff --git a/src/lib/smartlist_core/include.am b/src/lib/smartlist_core/include.am
new file mode 100644
index 0000000000..99d65f0b23
--- /dev/null
+++ b/src/lib/smartlist_core/include.am
@@ -0,0 +1,21 @@
+
+noinst_LIBRARIES += src/lib/libtor-smartlist-core.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-smartlist-core-testing.a
+endif
+
+src_lib_libtor_smartlist_core_a_SOURCES =			\
+	src/lib/smartlist_core/smartlist_core.c                 \
+	src/lib/smartlist_core/smartlist_split.c
+
+src_lib_libtor_smartlist_core_testing_a_SOURCES = \
+	$(src_lib_libtor_smartlist_core_a_SOURCES)
+src_lib_libtor_smartlist_core_testing_a_CPPFLAGS = \
+	$(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_smartlist_core_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/smartlist_core/smartlist_core.h         \
+	src/lib/smartlist_core/smartlist_foreach.h      \
+	src/lib/smartlist_core/smartlist_split.h
diff --git a/src/lib/smartlist_core/smartlist_core.c b/src/lib/smartlist_core/smartlist_core.c
new file mode 100644
index 0000000000..ac85a6cc84
--- /dev/null
+++ b/src/lib/smartlist_core/smartlist_core.c
@@ -0,0 +1,234 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file smartlist_core.c
+ * \brief Implements the core functionality of a smartlist (a resizeable
+ * dynamic array).  For more functionality and helper functions, see the
+ * container library.
+ **/
+
+#include "lib/err/torerr.h"
+#include "lib/malloc/malloc.h"
+#include "lib/smartlist_core/smartlist_core.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+/** All newly allocated smartlists have this capacity. */
+#define SMARTLIST_DEFAULT_CAPACITY 16
+
+/** Allocate and return an empty smartlist.
+ */
+MOCK_IMPL(smartlist_t *,
+smartlist_new,(void))
+{
+  smartlist_t *sl = tor_malloc(sizeof(smartlist_t));
+  sl->num_used = 0;
+  sl->capacity = SMARTLIST_DEFAULT_CAPACITY;
+  sl->list = tor_calloc(sizeof(void *), sl->capacity);
+  return sl;
+}
+
+/** Deallocate a smartlist.  Does not release storage associated with the
+ * list's elements.
+ */
+MOCK_IMPL(void,
+smartlist_free_,(smartlist_t *sl))
+{
+  if (!sl)
+    return;
+  tor_free(sl->list);
+  tor_free(sl);
+}
+
+/** Remove all elements from the list.
+ */
+void
+smartlist_clear(smartlist_t *sl)
+{
+  memset(sl->list, 0, sizeof(void *) * sl->num_used);
+  sl->num_used = 0;
+}
+
+#if SIZE_MAX < INT_MAX
+#error "We don't support systems where size_t is smaller than int."
+#endif
+
+/** Make sure that <b>sl</b> can hold at least <b>size</b> entries. */
+static inline void
+smartlist_ensure_capacity(smartlist_t *sl, size_t size)
+{
+  /* Set MAX_CAPACITY to MIN(INT_MAX, SIZE_MAX / sizeof(void*)) */
+#if (SIZE_MAX/SIZEOF_VOID_P) > INT_MAX
+#define MAX_CAPACITY (INT_MAX)
+#else
+#define MAX_CAPACITY (int)((SIZE_MAX / (sizeof(void*))))
+#endif
+
+  raw_assert(size <= MAX_CAPACITY);
+
+  if (size > (size_t) sl->capacity) {
+    size_t higher = (size_t) sl->capacity;
+    if (PREDICT_UNLIKELY(size > MAX_CAPACITY/2)) {
+      higher = MAX_CAPACITY;
+    } else {
+      while (size > higher)
+        higher *= 2;
+    }
+    sl->list = tor_reallocarray(sl->list, sizeof(void *),
+                                ((size_t)higher));
+    memset(sl->list + sl->capacity, 0,
+           sizeof(void *) * (higher - sl->capacity));
+    sl->capacity = (int) higher;
+  }
+#undef ASSERT_CAPACITY
+#undef MAX_CAPACITY
+}
+
+/** Append element to the end of the list. */
+void
+smartlist_add(smartlist_t *sl, void *element)
+{
+  smartlist_ensure_capacity(sl, ((size_t) sl->num_used)+1);
+  sl->list[sl->num_used++] = element;
+}
+
+/** Append each element from S2 to the end of S1. */
+void
+smartlist_add_all(smartlist_t *s1, const smartlist_t *s2)
+{
+  size_t new_size = (size_t)s1->num_used + (size_t)s2->num_used;
+  raw_assert(new_size >= (size_t) s1->num_used); /* check for overflow. */
+  smartlist_ensure_capacity(s1, new_size);
+  memcpy(s1->list + s1->num_used, s2->list, s2->num_used*sizeof(void*));
+  raw_assert(new_size <= INT_MAX); /* redundant. */
+  s1->num_used = (int) new_size;
+}
+
+/** Append a copy of string to sl */
+void
+smartlist_add_strdup(struct smartlist_t *sl, const char *string)
+{
+  char *copy;
+
+  copy = tor_strdup(string);
+
+  smartlist_add(sl, copy);
+}
+
+/** Remove all elements E from sl such that E==element.  Preserve
+ * the order of any elements before E, but elements after E can be
+ * rearranged.
+ */
+void
+smartlist_remove(smartlist_t *sl, const void *element)
+{
+  int i;
+  if (element == NULL)
+    return;
+  for (i=0; i < sl->num_used; i++)
+    if (sl->list[i] == element) {
+      sl->list[i] = sl->list[--sl->num_used]; /* swap with the end */
+      i--; /* so we process the new i'th element */
+      sl->list[sl->num_used] = NULL;
+    }
+}
+
+/** As <b>smartlist_remove</b>, but do not change the order of
+ * any elements not removed */
+void
+smartlist_remove_keeporder(smartlist_t *sl, const void *element)
+{
+  int i, j, num_used_orig = sl->num_used;
+  if (element == NULL)
+    return;
+
+  for (i=j=0; j < num_used_orig; ++j) {
+    if (sl->list[j] == element) {
+      --sl->num_used;
+    } else {
+      sl->list[i++] = sl->list[j];
+    }
+  }
+}
+
+/** If <b>sl</b> is nonempty, remove and return the final element.  Otherwise,
+ * return NULL. */
+void *
+smartlist_pop_last(smartlist_t *sl)
+{
+  raw_assert(sl);
+  if (sl->num_used) {
+    void *tmp = sl->list[--sl->num_used];
+    sl->list[sl->num_used] = NULL;
+    return tmp;
+  } else
+    return NULL;
+}
+
+/** Return true iff some element E of sl has E==element.
+ */
+int
+smartlist_contains(const smartlist_t *sl, const void *element)
+{
+  int i;
+  for (i=0; i < sl->num_used; i++)
+    if (sl->list[i] == element)
+      return 1;
+  return 0;
+}
+
+/** Remove the <b>idx</b>th element of sl; if idx is not the last
+ * element, swap the last element of sl into the <b>idx</b>th space.
+ */
+void
+smartlist_del(smartlist_t *sl, int idx)
+{
+  raw_assert(sl);
+  raw_assert(idx>=0);
+  raw_assert(idx < sl->num_used);
+  sl->list[idx] = sl->list[--sl->num_used];
+  sl->list[sl->num_used] = NULL;
+}
+
+/** Remove the <b>idx</b>th element of sl; if idx is not the last element,
+ * moving all subsequent elements back one space. Return the old value
+ * of the <b>idx</b>th element.
+ */
+void
+smartlist_del_keeporder(smartlist_t *sl, int idx)
+{
+  raw_assert(sl);
+  raw_assert(idx>=0);
+  raw_assert(idx < sl->num_used);
+  --sl->num_used;
+  if (idx < sl->num_used)
+    memmove(sl->list+idx, sl->list+idx+1, sizeof(void*)*(sl->num_used-idx));
+  sl->list[sl->num_used] = NULL;
+}
+
+/** Insert the value <b>val</b> as the new <b>idx</b>th element of
+ * <b>sl</b>, moving all items previously at <b>idx</b> or later
+ * forward one space.
+ */
+void
+smartlist_insert(smartlist_t *sl, int idx, void *val)
+{
+  raw_assert(sl);
+  raw_assert(idx>=0);
+  raw_assert(idx <= sl->num_used);
+  if (idx == sl->num_used) {
+    smartlist_add(sl, val);
+  } else {
+    smartlist_ensure_capacity(sl, ((size_t) sl->num_used)+1);
+    /* Move other elements away */
+    if (idx < sl->num_used)
+      memmove(sl->list + idx + 1, sl->list + idx,
+              sizeof(void*)*(sl->num_used-idx));
+    sl->num_used++;
+    sl->list[idx] = val;
+  }
+}
diff --git a/src/lib/smartlist_core/smartlist_core.h b/src/lib/smartlist_core/smartlist_core.h
new file mode 100644
index 0000000000..a7fbaa099b
--- /dev/null
+++ b/src/lib/smartlist_core/smartlist_core.h
@@ -0,0 +1,100 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file smartlist_core.h
+ * \brief Top-level declarations for the smartlist_t dynamic array type.
+ **/
+
+#ifndef TOR_SMARTLIST_CORE_H
+#define TOR_SMARTLIST_CORE_H
+
+#include <stddef.h>
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+#include "lib/testsupport/testsupport.h"
+
+/** A resizeable list of pointers, with associated helpful functionality.
+ *
+ * The members of this struct are exposed only so that macros and inlines can
+ * use them; all access to smartlist internals should go through the functions
+ * and macros defined here.
+ **/
+typedef struct smartlist_t {
+  /** @{ */
+  /** <b>list</b> has enough capacity to store exactly <b>capacity</b> elements
+   * before it needs to be resized.  Only the first <b>num_used</b> (\<=
+   * capacity) elements point to valid data.
+   */
+  void **list;
+  int num_used;
+  int capacity;
+  /** @} */
+} smartlist_t;
+
+MOCK_DECL(smartlist_t *, smartlist_new, (void));
+MOCK_DECL(void, smartlist_free_, (smartlist_t *sl));
+#define smartlist_free(sl) FREE_AND_NULL(smartlist_t, smartlist_free_, (sl))
+
+void smartlist_clear(smartlist_t *sl);
+void smartlist_add(smartlist_t *sl, void *element);
+void smartlist_add_all(smartlist_t *sl, const smartlist_t *s2);
+void smartlist_add_strdup(struct smartlist_t *sl, const char *string);
+
+void smartlist_remove(smartlist_t *sl, const void *element);
+void smartlist_remove_keeporder(smartlist_t *sl, const void *element);
+void *smartlist_pop_last(smartlist_t *sl);
+
+int smartlist_contains(const smartlist_t *sl, const void *element);
+
+/* smartlist_choose() is defined in crypto.[ch] */
+#ifdef DEBUG_SMARTLIST
+#include "lib/err/torerr.h"
+#include <stdlib.h>
+/** Return the number of items in sl.
+ */
+static inline int smartlist_len(const smartlist_t *sl);
+static inline int smartlist_len(const smartlist_t *sl) {
+  raw_assert(sl);
+  return (sl)->num_used;
+}
+/** Return the <b>idx</b>th element of sl.
+ */
+static inline void *smartlist_get(const smartlist_t *sl, int idx);
+static inline void *smartlist_get(const smartlist_t *sl, int idx) {
+  raw_assert(sl);
+  raw_assert(idx>=0);
+  raw_assert(sl->num_used > idx);
+  return sl->list[idx];
+}
+static inline void smartlist_set(smartlist_t *sl, int idx, void *val) {
+  raw_assert(sl);
+  raw_assert(idx>=0);
+  raw_assert(sl->num_used > idx);
+  sl->list[idx] = val;
+}
+#else /* !(defined(DEBUG_SMARTLIST)) */
+#define smartlist_len(sl) ((sl)->num_used)
+#define smartlist_get(sl, idx) ((sl)->list[idx])
+#define smartlist_set(sl, idx, val) ((sl)->list[idx] = (val))
+#endif /* defined(DEBUG_SMARTLIST) */
+
+/** Exchange the elements at indices <b>idx1</b> and <b>idx2</b> of the
+ * smartlist <b>sl</b>. */
+static inline void smartlist_swap(smartlist_t *sl, int idx1, int idx2)
+{
+  if (idx1 != idx2) {
+    void *elt = smartlist_get(sl, idx1);
+    smartlist_set(sl, idx1, smartlist_get(sl, idx2));
+    smartlist_set(sl, idx2, elt);
+  }
+}
+
+void smartlist_del(smartlist_t *sl, int idx);
+void smartlist_del_keeporder(smartlist_t *sl, int idx);
+void smartlist_insert(smartlist_t *sl, int idx, void *val);
+
+#endif /* !defined(TOR_CONTAINER_H) */
diff --git a/src/lib/smartlist_core/smartlist_foreach.h b/src/lib/smartlist_core/smartlist_foreach.h
new file mode 100644
index 0000000000..0f6fe30074
--- /dev/null
+++ b/src/lib/smartlist_core/smartlist_foreach.h
@@ -0,0 +1,133 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file smartlist_foreach.h
+ * \brief Macros for iterating over the elements of a smartlist_t.
+ **/
+
+#ifndef TOR_SMARTLIST_FOREACH_H
+#define TOR_SMARTLIST_FOREACH_H
+
+/** Iterate over the items in a smartlist <b>sl</b>, in order.  For each item,
+ * assign it to a new local variable of type <b>type</b> named <b>var</b>, and
+ * execute the statements inside the loop body.  Inside the loop, the loop
+ * index can be accessed as <b>var</b>_sl_idx and the length of the list can
+ * be accessed as <b>var</b>_sl_len.
+ *
+ * NOTE: Do not change the length of the list while the loop is in progress,
+ * unless you adjust the _sl_len variable correspondingly.  See second example
+ * below.
+ *
+ * Example use:
+ * <pre>
+ *   smartlist_t *list = smartlist_split("A:B:C", ":", 0, 0);
+ *   SMARTLIST_FOREACH_BEGIN(list, char *, cp) {
+ *     printf("%d: %s\n", cp_sl_idx, cp);
+ *     tor_free(cp);
+ *   } SMARTLIST_FOREACH_END(cp);
+ *   smartlist_free(list);
+ * </pre>
+ *
+ * Example use (advanced):
+ * <pre>
+ *   SMARTLIST_FOREACH_BEGIN(list, char *, cp) {
+ *     if (!strcmp(cp, "junk")) {
+ *       tor_free(cp);
+ *       SMARTLIST_DEL_CURRENT(list, cp);
+ *     }
+ *   } SMARTLIST_FOREACH_END(cp);
+ * </pre>
+ */
+/* Note: these macros use token pasting, and reach into smartlist internals.
+ * This can make them a little daunting. Here's the approximate unpacking of
+ * the above examples, for entertainment value:
+ *
+ * <pre>
+ * smartlist_t *list = smartlist_split("A:B:C", ":", 0, 0);
+ * {
+ *   int cp_sl_idx, cp_sl_len = smartlist_len(list);
+ *   char *cp;
+ *   for (cp_sl_idx = 0; cp_sl_idx < cp_sl_len; ++cp_sl_idx) {
+ *     cp = smartlist_get(list, cp_sl_idx);
+ *     printf("%d: %s\n", cp_sl_idx, cp);
+ *     tor_free(cp);
+ *   }
+ * }
+ * smartlist_free(list);
+ * </pre>
+ *
+ * <pre>
+ * {
+ *   int cp_sl_idx, cp_sl_len = smartlist_len(list);
+ *   char *cp;
+ *   for (cp_sl_idx = 0; cp_sl_idx < cp_sl_len; ++cp_sl_idx) {
+ *     cp = smartlist_get(list, cp_sl_idx);
+ *     if (!strcmp(cp, "junk")) {
+ *       tor_free(cp);
+ *       smartlist_del(list, cp_sl_idx);
+ *       --cp_sl_idx;
+ *       --cp_sl_len;
+ *     }
+ *   }
+ * }
+ * </pre>
+ */
+#define SMARTLIST_FOREACH_BEGIN(sl, type, var)  \
+  STMT_BEGIN                                                    \
+    int var ## _sl_idx, var ## _sl_len=(sl)->num_used;          \
+    type var;                                                   \
+    for (var ## _sl_idx = 0; var ## _sl_idx < var ## _sl_len;   \
+         ++var ## _sl_idx) {                                    \
+      var = (sl)->list[var ## _sl_idx];
+
+#define SMARTLIST_FOREACH_END(var)              \
+    var = NULL;                                 \
+    (void) var ## _sl_idx;                      \
+  } STMT_END
+
+/**
+ * An alias for SMARTLIST_FOREACH_BEGIN and SMARTLIST_FOREACH_END, using
+ * <b>cmd</b> as the loop body.  This wrapper is here for convenience with
+ * very short loops.
+ *
+ * By convention, we do not use this for loops which nest, or for loops over
+ * 10 lines or so.  Use SMARTLIST_FOREACH_{BEGIN,END} for those.
+ */
+#define SMARTLIST_FOREACH(sl, type, var, cmd)                   \
+  SMARTLIST_FOREACH_BEGIN(sl,type,var) {                        \
+    cmd;                                                        \
+  } SMARTLIST_FOREACH_END(var)
+
+/** Helper: While in a SMARTLIST_FOREACH loop over the list <b>sl</b> indexed
+ * with the variable <b>var</b>, remove the current element in a way that
+ * won't confuse the loop. */
+#define SMARTLIST_DEL_CURRENT(sl, var)          \
+  STMT_BEGIN                                    \
+    smartlist_del(sl, var ## _sl_idx);          \
+    --var ## _sl_idx;                           \
+    --var ## _sl_len;                           \
+  STMT_END
+
+/** Helper: While in a SMARTLIST_FOREACH loop over the list <b>sl</b> indexed
+ * with the variable <b>var</b>, remove the current element in a way that
+ * won't confuse the loop. */
+#define SMARTLIST_DEL_CURRENT_KEEPORDER(sl, var)          \
+  STMT_BEGIN                                              \
+     smartlist_del_keeporder(sl, var ## _sl_idx);         \
+     --var ## _sl_idx;                                    \
+     --var ## _sl_len;                                    \
+  STMT_END
+
+/** Helper: While in a SMARTLIST_FOREACH loop over the list <b>sl</b> indexed
+ * with the variable <b>var</b>, replace the current element with <b>val</b>.
+ * Does not deallocate the current value of <b>var</b>.
+ */
+#define SMARTLIST_REPLACE_CURRENT(sl, var, val) \
+  STMT_BEGIN                                    \
+    smartlist_set(sl, var ## _sl_idx, val);     \
+  STMT_END
+
+#endif /* !defined(TOR_SMARTLIST_FOREACH_H) */
diff --git a/src/lib/smartlist_core/smartlist_split.c b/src/lib/smartlist_core/smartlist_split.c
new file mode 100644
index 0000000000..c9cf59851f
--- /dev/null
+++ b/src/lib/smartlist_core/smartlist_split.c
@@ -0,0 +1,92 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file smartlist_split.c
+ * \brief Split a string into a smartlist_t of substrings.
+ **/
+
+#include "lib/smartlist_core/smartlist_core.h"
+#include "lib/smartlist_core/smartlist_split.h"
+
+#include "lib/err/torerr.h"
+#include "lib/string/util_string.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/malloc/malloc.h"
+
+#include <string.h>
+
+/**
+ * Split a string <b>str</b> along all occurrences of <b>sep</b>,
+ * appending the (newly allocated) split strings, in order, to
+ * <b>sl</b>.  Return the number of strings added to <b>sl</b>.
+ *
+ * If <b>flags</b>&amp;SPLIT_SKIP_SPACE is true, remove initial and
+ * trailing space from each entry.
+ * If <b>flags</b>&amp;SPLIT_IGNORE_BLANK is true, remove any entries
+ * of length 0.
+ * If <b>flags</b>&amp;SPLIT_STRIP_SPACE is true, strip spaces from each
+ * split string.
+ *
+ * If <b>max</b>\>0, divide the string into no more than <b>max</b> pieces. If
+ * <b>sep</b> is NULL, split on any sequence of horizontal space.
+ */
+int
+smartlist_split_string(smartlist_t *sl, const char *str, const char *sep,
+                       int flags, int max)
+{
+  const char *cp, *end, *next;
+  int n = 0;
+
+  raw_assert(sl);
+  raw_assert(str);
+
+  cp = str;
+  while (1) {
+    if (flags&SPLIT_SKIP_SPACE) {
+      while (TOR_ISSPACE(*cp)) ++cp;
+    }
+
+    if (max>0 && n == max-1) {
+      end = strchr(cp,'\0');
+    } else if (sep) {
+      end = strstr(cp,sep);
+      if (!end)
+        end = strchr(cp,'\0');
+    } else {
+      for (end = cp; *end && *end != '\t' && *end != ' '; ++end)
+        ;
+    }
+
+    raw_assert(end);
+
+    if (!*end) {
+      next = NULL;
+    } else if (sep) {
+      next = end+strlen(sep);
+    } else {
+      next = end+1;
+      while (*next == '\t' || *next == ' ')
+        ++next;
+    }
+
+    if (flags&SPLIT_SKIP_SPACE) {
+      while (end > cp && TOR_ISSPACE(*(end-1)))
+        --end;
+    }
+    if (end != cp || !(flags&SPLIT_IGNORE_BLANK)) {
+      char *string = tor_strndup(cp, end-cp);
+      if (flags&SPLIT_STRIP_SPACE)
+        tor_strstrip(string, " ");
+      smartlist_add(sl, string);
+      ++n;
+    }
+    if (!next)
+      break;
+    cp = next;
+  }
+
+  return n;
+}
diff --git a/src/lib/smartlist_core/smartlist_split.h b/src/lib/smartlist_core/smartlist_split.h
new file mode 100644
index 0000000000..4f72376125
--- /dev/null
+++ b/src/lib/smartlist_core/smartlist_split.h
@@ -0,0 +1,20 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file smartlist_split.h
+ * \brief Header for smartlist_split.c
+ **/
+
+#ifndef TOR_SMARTLIST_SPLIT_H
+#define TOR_SMARTLIST_SPLIT_H
+
+#define SPLIT_SKIP_SPACE   0x01
+#define SPLIT_IGNORE_BLANK 0x02
+#define SPLIT_STRIP_SPACE  0x04
+int smartlist_split_string(smartlist_t *sl, const char *str, const char *sep,
+                           int flags, int max);
+
+#endif
diff --git a/src/lib/string/.may_include b/src/lib/string/.may_include
new file mode 100644
index 0000000000..ec5c769831
--- /dev/null
+++ b/src/lib/string/.may_include
@@ -0,0 +1,10 @@
+orconfig.h
+lib/cc/*.h
+lib/defs/*.h
+lib/err/*.h
+lib/malloc/*.h
+lib/ctime/*.h
+lib/string/*.h
+
+strlcat.c
+strlcpy.c
diff --git a/src/lib/string/compat_ctype.c b/src/lib/string/compat_ctype.c
new file mode 100644
index 0000000000..f5d82be3ae
--- /dev/null
+++ b/src/lib/string/compat_ctype.c
@@ -0,0 +1,72 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compat_ctype.c
+ * \brief Locale-independent character-type inspection (backend)
+ **/
+
+#include "lib/string/compat_ctype.h"
+
+/**
+ * Tables to implement ctypes-replacement TOR_IS*() functions.  Each table
+ * has 256 bits to look up whether a character is in some set or not.  This
+ * fails on non-ASCII platforms, but it is hard to find a platform whose
+ * character set is not a superset of ASCII nowadays. */
+
+/**@{*/
+const uint32_t TOR_ISALPHA_TABLE[8] =
+  { 0, 0, 0x7fffffe, 0x7fffffe, 0, 0, 0, 0 };
+const uint32_t TOR_ISALNUM_TABLE[8] =
+  { 0, 0x3ff0000, 0x7fffffe, 0x7fffffe, 0, 0, 0, 0 };
+const uint32_t TOR_ISSPACE_TABLE[8] = { 0x3e00, 0x1, 0, 0, 0, 0, 0, 0 };
+const uint32_t TOR_ISXDIGIT_TABLE[8] =
+  { 0, 0x3ff0000, 0x7e, 0x7e, 0, 0, 0, 0 };
+const uint32_t TOR_ISDIGIT_TABLE[8] = { 0, 0x3ff0000, 0, 0, 0, 0, 0, 0 };
+const uint32_t TOR_ISPRINT_TABLE[8] =
+  { 0, 0xffffffff, 0xffffffff, 0x7fffffff, 0, 0, 0, 0x0 };
+const uint32_t TOR_ISUPPER_TABLE[8] = { 0, 0, 0x7fffffe, 0, 0, 0, 0, 0 };
+const uint32_t TOR_ISLOWER_TABLE[8] = { 0, 0, 0, 0x7fffffe, 0, 0, 0, 0 };
+
+/** Upper-casing and lowercasing tables to map characters to upper/lowercase
+ * equivalents.  Used by tor_toupper() and tor_tolower(). */
+/**@{*/
+const uint8_t TOR_TOUPPER_TABLE[256] = {
+  0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,
+  16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,
+  32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,
+  48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,
+  64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,
+  80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,
+  96,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,
+  80,81,82,83,84,85,86,87,88,89,90,123,124,125,126,127,
+  128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,
+  144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,
+  160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,
+  176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,
+  192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,
+  208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,
+  224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,
+  240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,
+};
+const uint8_t TOR_TOLOWER_TABLE[256] = {
+  0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,
+  16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,
+  32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,
+  48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,
+  64,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,
+  112,113,114,115,116,117,118,119,120,121,122,91,92,93,94,95,
+  96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,
+  112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,
+  128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,
+  144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,
+  160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,
+  176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,
+  192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,
+  208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,
+  224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,
+  240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,
+};
+/**@}*/
diff --git a/src/lib/string/compat_ctype.h b/src/lib/string/compat_ctype.h
new file mode 100644
index 0000000000..dbddd356c1
--- /dev/null
+++ b/src/lib/string/compat_ctype.h
@@ -0,0 +1,67 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compat_ctype.h
+ * \brief Locale-independent character-type inspection (header)
+ **/
+
+#ifndef TOR_COMPAT_CTYPE_H
+#define TOR_COMPAT_CTYPE_H
+
+#include "orconfig.h"
+#include "lib/cc/torint.h"
+
+/* Much of the time when we're checking ctypes, we're doing spec compliance,
+ * which all assumes we're doing ASCII. */
+#define DECLARE_CTYPE_FN(name)                                          \
+  static int TOR_##name(char c);                                        \
+  extern const uint32_t TOR_##name##_TABLE[];                           \
+  static inline int TOR_##name(char c) {                                \
+    uint8_t u = c;                                                      \
+    return !!(TOR_##name##_TABLE[(u >> 5) & 7] & (1u << (u & 31)));     \
+  }
+DECLARE_CTYPE_FN(ISALPHA)
+DECLARE_CTYPE_FN(ISALNUM)
+DECLARE_CTYPE_FN(ISSPACE)
+DECLARE_CTYPE_FN(ISDIGIT)
+DECLARE_CTYPE_FN(ISXDIGIT)
+DECLARE_CTYPE_FN(ISPRINT)
+DECLARE_CTYPE_FN(ISLOWER)
+DECLARE_CTYPE_FN(ISUPPER)
+extern const uint8_t TOR_TOUPPER_TABLE[];
+extern const uint8_t TOR_TOLOWER_TABLE[];
+#define TOR_TOLOWER(c) (TOR_TOLOWER_TABLE[(uint8_t)c])
+#define TOR_TOUPPER(c) (TOR_TOUPPER_TABLE[(uint8_t)c])
+
+static inline int hex_decode_digit(char c);
+
+/** Helper: given a hex digit, return its value, or -1 if it isn't hex. */
+static inline int
+hex_decode_digit(char c)
+{
+  switch (c) {
+    case '0': return 0;
+    case '1': return 1;
+    case '2': return 2;
+    case '3': return 3;
+    case '4': return 4;
+    case '5': return 5;
+    case '6': return 6;
+    case '7': return 7;
+    case '8': return 8;
+    case '9': return 9;
+    case 'A': case 'a': return 10;
+    case 'B': case 'b': return 11;
+    case 'C': case 'c': return 12;
+    case 'D': case 'd': return 13;
+    case 'E': case 'e': return 14;
+    case 'F': case 'f': return 15;
+    default:
+      return -1;
+  }
+}
+
+#endif /* !defined(TOR_COMPAT_CTYPE_H) */
diff --git a/src/lib/string/compat_string.c b/src/lib/string/compat_string.c
new file mode 100644
index 0000000000..e125c921a4
--- /dev/null
+++ b/src/lib/string/compat_string.c
@@ -0,0 +1,74 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compat_string.c
+ * \brief Useful string-processing functions that some platforms don't
+ * provide.
+ **/
+
+#include "lib/string/compat_string.h"
+#include "lib/err/torerr.h"
+
+/* Inline the strl functions if the platform doesn't have them. */
+#ifndef HAVE_STRLCPY
+#include "strlcpy.c"
+#endif
+#ifndef HAVE_STRLCAT
+#include "strlcat.c"
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+
+/** Helper for tor_strtok_r_impl: Advances cp past all characters in
+ * <b>sep</b>, and returns its new value. */
+static char *
+strtok_helper(char *cp, const char *sep)
+{
+  if (sep[1]) {
+    while (*cp && strchr(sep, *cp))
+      ++cp;
+  } else {
+    while (*cp && *cp == *sep)
+      ++cp;
+  }
+  return cp;
+}
+
+/** Implementation of strtok_r for platforms whose coders haven't figured out
+ * how to write one.  Hey, retrograde libc developers!  You can use this code
+ * here for free! */
+char *
+tor_strtok_r_impl(char *str, const char *sep, char **lasts)
+{
+  char *cp, *start;
+  raw_assert(*sep);
+  if (str) {
+    str = strtok_helper(str, sep);
+    if (!*str)
+      return NULL;
+    start = cp = *lasts = str;
+  } else if (!*lasts || !**lasts) {
+    return NULL;
+  } else {
+    start = cp = *lasts;
+  }
+
+  if (sep[1]) {
+    while (*cp && !strchr(sep, *cp))
+      ++cp;
+  } else {
+    cp = strchr(cp, *sep);
+  }
+
+  if (!cp || !*cp) {
+    *lasts = NULL;
+  } else {
+    *cp++ = '\0';
+    *lasts = strtok_helper(cp, sep);
+  }
+  return start;
+}
diff --git a/src/lib/string/compat_string.h b/src/lib/string/compat_string.h
new file mode 100644
index 0000000000..a0e37bb6dc
--- /dev/null
+++ b/src/lib/string/compat_string.h
@@ -0,0 +1,62 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compat_string.h
+ * \brief Header for compat_string.c
+ **/
+
+#ifndef TOR_COMPAT_STRING_H
+#define TOR_COMPAT_STRING_H
+
+#include "orconfig.h"
+#include "lib/cc/compat_compiler.h"
+
+#include <stddef.h>
+
+/* ===== String compatibility */
+#ifdef _WIN32
+/* Windows doesn't have str(n)casecmp, but mingw defines it: only define it
+ * ourselves if it's missing. */
+#ifndef HAVE_STRNCASECMP
+static inline int strncasecmp(const char *a, const char *b, size_t n);
+static inline int strncasecmp(const char *a, const char *b, size_t n) {
+  return _strnicmp(a,b,n);
+}
+#endif
+#ifndef HAVE_STRCASECMP
+static inline int strcasecmp(const char *a, const char *b);
+static inline int strcasecmp(const char *a, const char *b) {
+  return _stricmp(a,b);
+}
+#endif
+#endif
+
+#if defined __APPLE__
+/* On OSX 10.9 and later, the overlap-checking code for strlcat would
+ * appear to have a severe bug that can sometimes cause aborts in Tor.
+ * Instead, use the non-checking variants.  This is sad.
+ *
+ * See https://trac.torproject.org/projects/tor/ticket/15205
+ */
+#undef strlcat
+#undef strlcpy
+#endif /* defined __APPLE__ */
+
+#ifndef HAVE_STRLCAT
+size_t strlcat(char *dst, const char *src, size_t siz);
+#endif
+#ifndef HAVE_STRLCPY
+size_t strlcpy(char *dst, const char *src, size_t siz);
+#endif
+
+char *tor_strtok_r_impl(char *str, const char *sep, char **lasts);
+#ifdef HAVE_STRTOK_R
+#define tor_strtok_r(str, sep, lasts) strtok_r(str, sep, lasts)
+#else
+#define tor_strtok_r(str, sep, lasts) tor_strtok_r_impl(str, sep, lasts)
+#endif
+
+#endif
diff --git a/src/lib/string/include.am b/src/lib/string/include.am
new file mode 100644
index 0000000000..edd74b8a3e
--- /dev/null
+++ b/src/lib/string/include.am
@@ -0,0 +1,27 @@
+
+noinst_LIBRARIES += src/lib/libtor-string.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-string-testing.a
+endif
+
+src_lib_libtor_string_a_SOURCES =			\
+	src/lib/string/compat_ctype.c			\
+	src/lib/string/compat_string.c			\
+	src/lib/string/util_string.c			\
+	src/lib/string/parse_int.c			\
+	src/lib/string/printf.c				\
+	src/lib/string/scanf.c
+
+src_lib_libtor_string_testing_a_SOURCES = \
+	$(src_lib_libtor_string_a_SOURCES)
+src_lib_libtor_string_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_string_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/string/compat_ctype.h			\
+	src/lib/string/compat_string.h			\
+	src/lib/string/util_string.h			\
+	src/lib/string/parse_int.h			\
+	src/lib/string/printf.h				\
+	src/lib/string/scanf.h
diff --git a/src/lib/string/parse_int.c b/src/lib/string/parse_int.c
new file mode 100644
index 0000000000..fbdd554a47
--- /dev/null
+++ b/src/lib/string/parse_int.c
@@ -0,0 +1,131 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file parse_int.c
+ * \brief Convert strings into the integers they encode, with bounds checking.
+ **/
+
+#include "lib/string/parse_int.h"
+
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* Helper: common code to check whether the result of a strtol or strtoul or
+ * strtoll is correct. */
+#define CHECK_STRTOX_RESULT()                           \
+  /* Did an overflow occur? */                          \
+  if (errno == ERANGE)                                  \
+    goto err;                                           \
+  /* Was at least one character converted? */           \
+  if (endptr == s)                                      \
+    goto err;                                           \
+  /* Were there unexpected unconverted characters? */   \
+  if (!next && *endptr)                                 \
+    goto err;                                           \
+  /* Illogical (max, min) inputs? */                    \
+  if (max < min)                                        \
+    goto err;                                           \
+  /* Is r within limits? */                             \
+  if (r < min || r > max)                               \
+    goto err;                                           \
+  if (ok) *ok = 1;                                      \
+  if (next) *next = endptr;                             \
+  return r;                                             \
+ err:                                                   \
+  if (ok) *ok = 0;                                      \
+  if (next) *next = endptr;                             \
+  return 0
+
+/** Extract a long from the start of <b>s</b>, in the given numeric
+ * <b>base</b>.  If <b>base</b> is 0, <b>s</b> is parsed as a decimal,
+ * octal, or hex number in the syntax of a C integer literal.  If
+ * there is unconverted data and <b>next</b> is provided, set
+ * *<b>next</b> to the first unconverted character.  An error has
+ * occurred if no characters are converted; or if there are
+ * unconverted characters and <b>next</b> is NULL; or if the parsed
+ * value is not between <b>min</b> and <b>max</b>.  When no error
+ * occurs, return the parsed value and set *<b>ok</b> (if provided) to
+ * 1.  When an error occurs, return 0 and set *<b>ok</b> (if provided)
+ * to 0.
+ */
+long
+tor_parse_long(const char *s, int base, long min, long max,
+               int *ok, char **next)
+{
+  char *endptr;
+  long r;
+
+  if (base < 0) {
+    if (ok)
+      *ok = 0;
+    return 0;
+  }
+
+  errno = 0;
+  r = strtol(s, &endptr, base);
+  CHECK_STRTOX_RESULT();
+}
+
+/** As tor_parse_long(), but return an unsigned long. */
+unsigned long
+tor_parse_ulong(const char *s, int base, unsigned long min,
+                unsigned long max, int *ok, char **next)
+{
+  char *endptr;
+  unsigned long r;
+
+  if (base < 0) {
+    if (ok)
+      *ok = 0;
+    return 0;
+  }
+
+  errno = 0;
+  r = strtoul(s, &endptr, base);
+  CHECK_STRTOX_RESULT();
+}
+
+/** As tor_parse_long(), but return a double. */
+double
+tor_parse_double(const char *s, double min, double max, int *ok, char **next)
+{
+  char *endptr;
+  double r;
+
+  errno = 0;
+  r = strtod(s, &endptr);
+  CHECK_STRTOX_RESULT();
+}
+
+/** As tor_parse_long, but return a uint64_t.  Only base 10 is guaranteed to
+ * work for now. */
+uint64_t
+tor_parse_uint64(const char *s, int base, uint64_t min,
+                 uint64_t max, int *ok, char **next)
+{
+  char *endptr;
+  uint64_t r;
+
+  if (base < 0) {
+    if (ok)
+      *ok = 0;
+    return 0;
+  }
+
+  errno = 0;
+#ifdef HAVE_STRTOULL
+  r = (uint64_t)strtoull(s, &endptr, base);
+#elif defined(_WIN32)
+  r = (uint64_t)_strtoui64(s, &endptr, base);
+#elif SIZEOF_LONG == 8
+  r = (uint64_t)strtoul(s, &endptr, base);
+#else
+#error "I don't know how to parse 64-bit numbers."
+#endif /* defined(HAVE_STRTOULL) || ... */
+
+  CHECK_STRTOX_RESULT();
+}
diff --git a/src/lib/string/parse_int.h b/src/lib/string/parse_int.h
new file mode 100644
index 0000000000..925547942e
--- /dev/null
+++ b/src/lib/string/parse_int.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file parse_int.h
+ * \brief Header for parse_int.c
+ **/
+
+#ifndef TOR_PARSE_INT_H
+#define TOR_PARSE_INT_H
+
+#include "lib/cc/torint.h"
+
+long tor_parse_long(const char *s, int base, long min,
+                    long max, int *ok, char **next);
+unsigned long tor_parse_ulong(const char *s, int base, unsigned long min,
+                              unsigned long max, int *ok, char **next);
+double tor_parse_double(const char *s, double min, double max, int *ok,
+                        char **next);
+uint64_t tor_parse_uint64(const char *s, int base, uint64_t min,
+                         uint64_t max, int *ok, char **next);
+
+#endif
diff --git a/src/lib/string/printf.c b/src/lib/string/printf.c
new file mode 100644
index 0000000000..a5cb71ce09
--- /dev/null
+++ b/src/lib/string/printf.c
@@ -0,0 +1,167 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file printf.c
+ * \brief Compatibility wrappers around snprintf and its friends
+ **/
+
+#include "lib/string/printf.h"
+#include "lib/err/torerr.h"
+#include "lib/cc/torint.h"
+#include "lib/malloc/malloc.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+
+/** Replacement for snprintf.  Differs from platform snprintf in two
+ * ways: First, always NUL-terminates its output.  Second, always
+ * returns -1 if the result is truncated.  (Note that this return
+ * behavior does <i>not</i> conform to C99; it just happens to be
+ * easier to emulate "return -1" with conformant implementations than
+ * it is to emulate "return number that would be written" with
+ * non-conformant implementations.) */
+int
+tor_snprintf(char *str, size_t size, const char *format, ...)
+{
+  va_list ap;
+  int r;
+  va_start(ap,format);
+  r = tor_vsnprintf(str,size,format,ap);
+  va_end(ap);
+  return r;
+}
+
+/** Replacement for vsnprintf; behavior differs as tor_snprintf differs from
+ * snprintf.
+ */
+int
+tor_vsnprintf(char *str, size_t size, const char *format, va_list args)
+{
+  int r;
+  if (size == 0)
+    return -1; /* no place for the NUL */
+  if (size > SIZE_T_CEILING)
+    return -1;
+#ifdef _WIN32
+  r = _vsnprintf(str, size, format, args);
+#else
+  r = vsnprintf(str, size, format, args);
+#endif
+  str[size-1] = '\0';
+  if (r < 0 || r >= (ssize_t)size)
+    return -1;
+  return r;
+}
+
+/**
+ * Portable asprintf implementation.  Does a printf() into a newly malloc'd
+ * string.  Sets *<b>strp</b> to this string, and returns its length (not
+ * including the terminating NUL character).
+ *
+ * You can treat this function as if its implementation were something like
+   <pre>
+     char buf[_INFINITY_];
+     tor_snprintf(buf, sizeof(buf), fmt, args);
+     *strp = tor_strdup(buf);
+     return strlen(*strp):
+   </pre>
+ * Where _INFINITY_ is an imaginary constant so big that any string can fit
+ * into it.
+ */
+int
+tor_asprintf(char **strp, const char *fmt, ...)
+{
+  int r;
+  va_list args;
+  va_start(args, fmt);
+  r = tor_vasprintf(strp, fmt, args);
+  va_end(args);
+  if (!*strp || r < 0) {
+    /* LCOV_EXCL_START */
+    raw_assert_unreached_msg("Internal error in asprintf");
+    /* LCOV_EXCL_STOP */
+  }
+  return r;
+}
+
+/**
+ * Portable vasprintf implementation.  Does a printf() into a newly malloc'd
+ * string.  Differs from regular vasprintf in the same ways that
+ * tor_asprintf() differs from regular asprintf.
+ */
+int
+tor_vasprintf(char **strp, const char *fmt, va_list args)
+{
+  /* use a temporary variable in case *strp is in args. */
+  char *strp_tmp=NULL;
+#ifdef HAVE_VASPRINTF
+  /* If the platform gives us one, use it. */
+  int r = vasprintf(&strp_tmp, fmt, args);
+  if (r < 0)
+    *strp = NULL; // LCOV_EXCL_LINE -- no cross-platform way to force this
+  else
+    *strp = strp_tmp;
+  return r;
+#elif defined(HAVE__VSCPRINTF)
+  /* On Windows, _vsnprintf won't tell us the length of the string if it
+   * overflows, so we need to use _vcsprintf to tell how much to allocate */
+  int len, r;
+  va_list tmp_args;
+  va_copy(tmp_args, args);
+  len = _vscprintf(fmt, tmp_args);
+  va_end(tmp_args);
+  if (len < 0) {
+    *strp = NULL;
+    return -1;
+  }
+  strp_tmp = tor_malloc(len + 1);
+  r = _vsnprintf(strp_tmp, len+1, fmt, args);
+  if (r != len) {
+    tor_free(strp_tmp);
+    *strp = NULL;
+    return -1;
+  }
+  *strp = strp_tmp;
+  return len;
+#else
+  /* Everywhere else, we have a decent vsnprintf that tells us how many
+   * characters we need.  We give it a try on a short buffer first, since
+   * it might be nice to avoid the second vsnprintf call.
+   */
+  /* XXXX This code spent a number of years broken (see bug 30651). It is
+   * possible that no Tor users actually run on systems without vasprintf() or
+   * _vscprintf(). If so, we should consider removing this code. */
+  char buf[128];
+  int len, r;
+  va_list tmp_args;
+  va_copy(tmp_args, args);
+  /* Use vsnprintf to retrieve needed length.  tor_vsnprintf() is not an
+   * option here because it will simply return -1 if buf is not large enough
+   * to hold the complete string.
+   */
+  len = vsnprintf(buf, sizeof(buf), fmt, tmp_args);
+  va_end(tmp_args);
+  buf[sizeof(buf) - 1] = '\0';
+  if (len < 0) {
+    *strp = NULL;
+    return -1;
+  }
+  if (len < (int)sizeof(buf)) {
+    *strp = tor_strdup(buf);
+    return len;
+  }
+  strp_tmp = tor_malloc(len+1);
+  /* use of tor_vsnprintf() will ensure string is null terminated */
+  r = tor_vsnprintf(strp_tmp, len+1, fmt, args);
+  if (r != len) {
+    tor_free(strp_tmp);
+    *strp = NULL;
+    return -1;
+  }
+  *strp = strp_tmp;
+  return len;
+#endif /* defined(HAVE_VASPRINTF) || ... */
+}
diff --git a/src/lib/string/printf.h b/src/lib/string/printf.h
new file mode 100644
index 0000000000..2cc13d6bee
--- /dev/null
+++ b/src/lib/string/printf.h
@@ -0,0 +1,30 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file printf.h
+ * \brief Header for printf.c
+ **/
+
+#ifndef TOR_UTIL_PRINTF_H
+#define TOR_UTIL_PRINTF_H
+
+#include "orconfig.h"
+#include "lib/cc/compat_compiler.h"
+
+#include <stdarg.h>
+#include <stddef.h>
+
+int tor_snprintf(char *str, size_t size, const char *format, ...)
+  CHECK_PRINTF(3,4);
+int tor_vsnprintf(char *str, size_t size, const char *format, va_list args)
+  CHECK_PRINTF(3,0);
+
+int tor_asprintf(char **strp, const char *fmt, ...)
+  CHECK_PRINTF(2,3);
+int tor_vasprintf(char **strp, const char *fmt, va_list args)
+  CHECK_PRINTF(2,0);
+
+#endif /* !defined(TOR_UTIL_STRING_H) */
diff --git a/src/lib/string/scanf.c b/src/lib/string/scanf.c
new file mode 100644
index 0000000000..1bc39b5182
--- /dev/null
+++ b/src/lib/string/scanf.c
@@ -0,0 +1,317 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file scanf.c
+ * \brief Locale-independent minimal implementation of sscanf().
+ **/
+
+#include "lib/string/scanf.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/cc/torint.h"
+#include "lib/err/torerr.h"
+
+#include <stdlib.h>
+
+#define MAX_SCANF_WIDTH 9999
+
+/** Helper: given an ASCII-encoded decimal digit, return its numeric value.
+ * NOTE: requires that its input be in-bounds. */
+static int
+digit_to_num(char d)
+{
+  int num = ((int)d) - (int)'0';
+  raw_assert(num <= 9 && num >= 0);
+  return num;
+}
+
+/** Helper: Read an unsigned int from *<b>bufp</b> of up to <b>width</b>
+ * characters.  (Handle arbitrary width if <b>width</b> is less than 0.)  On
+ * success, store the result in <b>out</b>, advance bufp to the next
+ * character, and return 0.  On failure, return -1. */
+static int
+scan_unsigned(const char **bufp, unsigned long *out, int width, unsigned base)
+{
+  unsigned long result = 0;
+  int scanned_so_far = 0;
+  const int hex = base==16;
+  raw_assert(base == 10 || base == 16);
+  if (!bufp || !*bufp || !out)
+    return -1;
+  if (width<0)
+    width=MAX_SCANF_WIDTH;
+
+  while (**bufp && (hex?TOR_ISXDIGIT(**bufp):TOR_ISDIGIT(**bufp))
+         && scanned_so_far < width) {
+    unsigned digit = hex?hex_decode_digit(*(*bufp)++):digit_to_num(*(*bufp)++);
+    // Check for overflow beforehand, without actually causing any overflow
+    // This preserves functionality on compilers that don't wrap overflow
+    // (i.e. that trap or optimise away overflow)
+    // result * base + digit > ULONG_MAX
+    // result * base > ULONG_MAX - digit
+    if (result > (ULONG_MAX - digit)/base)
+      return -1; /* Processing this digit would overflow */
+    result = result * base + digit;
+    ++scanned_so_far;
+  }
+
+  if (!scanned_so_far) /* No actual digits scanned */
+    return -1;
+
+  *out = result;
+  return 0;
+}
+
+/** Helper: Read an signed int from *<b>bufp</b> of up to <b>width</b>
+ * characters.  (Handle arbitrary width if <b>width</b> is less than 0.)  On
+ * success, store the result in <b>out</b>, advance bufp to the next
+ * character, and return 0.  On failure, return -1. */
+static int
+scan_signed(const char **bufp, long *out, int width)
+{
+  int neg = 0;
+  unsigned long result = 0;
+
+  if (!bufp || !*bufp || !out)
+    return -1;
+  if (width<0)
+    width=MAX_SCANF_WIDTH;
+
+  if (**bufp == '-') {
+    neg = 1;
+    ++*bufp;
+    --width;
+  }
+
+  if (scan_unsigned(bufp, &result, width, 10) < 0)
+    return -1;
+
+  if (neg && result > 0) {
+    if (result > ((unsigned long)LONG_MAX) + 1)
+      return -1; /* Underflow */
+    else if (result == ((unsigned long)LONG_MAX) + 1)
+      *out = LONG_MIN;
+    else {
+      /* We once had a far more clever no-overflow conversion here, but
+       * some versions of GCC apparently ran it into the ground.  Now
+       * we just check for LONG_MIN explicitly.
+       */
+      *out = -(long)result;
+    }
+  } else {
+    if (result > LONG_MAX)
+      return -1; /* Overflow */
+    *out = (long)result;
+  }
+
+  return 0;
+}
+
+/** Helper: Read a decimal-formatted double from *<b>bufp</b> of up to
+ * <b>width</b> characters.  (Handle arbitrary width if <b>width</b> is less
+ * than 0.)  On success, store the result in <b>out</b>, advance bufp to the
+ * next character, and return 0.  On failure, return -1. */
+static int
+scan_double(const char **bufp, double *out, int width)
+{
+  int neg = 0;
+  double result = 0;
+  int scanned_so_far = 0;
+
+  if (!bufp || !*bufp || !out)
+    return -1;
+  if (width<0)
+    width=MAX_SCANF_WIDTH;
+
+  if (**bufp == '-') {
+    neg = 1;
+    ++*bufp;
+  }
+
+  while (**bufp && TOR_ISDIGIT(**bufp) && scanned_so_far < width) {
+    const int digit = digit_to_num(*(*bufp)++);
+    result = result * 10 + digit;
+    ++scanned_so_far;
+  }
+  if (**bufp == '.') {
+    double fracval = 0, denominator = 1;
+    ++*bufp;
+    ++scanned_so_far;
+    while (**bufp && TOR_ISDIGIT(**bufp) && scanned_so_far < width) {
+      const int digit = digit_to_num(*(*bufp)++);
+      fracval = fracval * 10 + digit;
+      denominator *= 10;
+      ++scanned_so_far;
+    }
+    result += fracval / denominator;
+  }
+
+  if (!scanned_so_far) /* No actual digits scanned */
+    return -1;
+
+  *out = neg ? -result : result;
+  return 0;
+}
+
+/** Helper: copy up to <b>width</b> non-space characters from <b>bufp</b> to
+ * <b>out</b>.  Make sure <b>out</b> is nul-terminated. Advance <b>bufp</b>
+ * to the next non-space character or the EOS. */
+static int
+scan_string(const char **bufp, char *out, int width)
+{
+  int scanned_so_far = 0;
+  if (!bufp || !out || width < 0)
+    return -1;
+  while (**bufp && ! TOR_ISSPACE(**bufp) && scanned_so_far < width) {
+    *out++ = *(*bufp)++;
+    ++scanned_so_far;
+  }
+  *out = '\0';
+  return 0;
+}
+
+/** Locale-independent, minimal, no-surprises scanf variant, accepting only a
+ * restricted pattern format.  For more info on what it supports, see
+ * tor_sscanf() documentation.  */
+int
+tor_vsscanf(const char *buf, const char *pattern, va_list ap)
+{
+  int n_matched = 0;
+
+  while (*pattern) {
+    if (*pattern != '%') {
+      if (*buf == *pattern) {
+        ++buf;
+        ++pattern;
+        continue;
+      } else {
+        return n_matched;
+      }
+    } else {
+      int width = -1;
+      int longmod = 0;
+      ++pattern;
+      if (TOR_ISDIGIT(*pattern)) {
+        width = digit_to_num(*pattern++);
+        while (TOR_ISDIGIT(*pattern)) {
+          width *= 10;
+          width += digit_to_num(*pattern++);
+          if (width > MAX_SCANF_WIDTH)
+            return -1;
+        }
+        if (!width) /* No zero-width things. */
+          return -1;
+      }
+      if (*pattern == 'l') {
+        longmod = 1;
+        ++pattern;
+      }
+      if (*pattern == 'u' || *pattern == 'x') {
+        unsigned long u;
+        const int base = (*pattern == 'u') ? 10 : 16;
+        if (!*buf)
+          return n_matched;
+        if (scan_unsigned(&buf, &u, width, base)<0)
+          return n_matched;
+        if (longmod) {
+          unsigned long *out = va_arg(ap, unsigned long *);
+          *out = u;
+        } else {
+          unsigned *out = va_arg(ap, unsigned *);
+          if (u > UINT_MAX)
+            return n_matched;
+          *out = (unsigned) u;
+        }
+        ++pattern;
+        ++n_matched;
+      } else if (*pattern == 'f') {
+        double *d = va_arg(ap, double *);
+        if (!longmod)
+          return -1; /* float not supported */
+        if (!*buf)
+          return n_matched;
+        if (scan_double(&buf, d, width)<0)
+          return n_matched;
+        ++pattern;
+        ++n_matched;
+      } else if (*pattern == 'd') {
+        long lng=0;
+        if (scan_signed(&buf, &lng, width)<0)
+          return n_matched;
+        if (longmod) {
+          long *out = va_arg(ap, long *);
+          *out = lng;
+        } else {
+          int *out = va_arg(ap, int *);
+#if LONG_MAX > INT_MAX
+          if (lng < INT_MIN || lng > INT_MAX)
+            return n_matched;
+#endif
+          *out = (int)lng;
+        }
+        ++pattern;
+        ++n_matched;
+      } else if (*pattern == 's') {
+        char *s = va_arg(ap, char *);
+        if (longmod)
+          return -1;
+        if (width < 0)
+          return -1;
+        if (scan_string(&buf, s, width)<0)
+          return n_matched;
+        ++pattern;
+        ++n_matched;
+      } else if (*pattern == 'c') {
+        char *ch = va_arg(ap, char *);
+        if (longmod)
+          return -1;
+        if (width != -1)
+          return -1;
+        if (!*buf)
+          return n_matched;
+        *ch = *buf++;
+        ++pattern;
+        ++n_matched;
+      } else if (*pattern == '%') {
+        if (*buf != '%')
+          return n_matched;
+        if (longmod)
+          return -1;
+        ++buf;
+        ++pattern;
+      } else {
+        return -1; /* Unrecognized pattern component. */
+      }
+    }
+  }
+
+  return n_matched;
+}
+
+/** Minimal sscanf replacement: parse <b>buf</b> according to <b>pattern</b>
+ * and store the results in the corresponding argument fields.  Differs from
+ * sscanf in that:
+ * <ul><li>It only handles %u, %lu, %x, %lx, %[NUM]s, %d, %ld, %lf, and %c.
+ *     <li>It only handles decimal inputs for %lf. (12.3, not 1.23e1)
+ *     <li>It does not handle arbitrarily long widths.
+ *     <li>Numbers do not consume any space characters.
+ *     <li>It is locale-independent.
+ *     <li>%u and %x do not consume any space.
+ *     <li>It returns -1 on malformed patterns.</ul>
+ *
+ * (As with other locale-independent functions, we need this to parse data that
+ * is in ASCII without worrying that the C library's locale-handling will make
+ * miscellaneous characters look like numbers, spaces, and so on.)
+ */
+int
+tor_sscanf(const char *buf, const char *pattern, ...)
+{
+  int r;
+  va_list ap;
+  va_start(ap, pattern);
+  r = tor_vsscanf(buf, pattern, ap);
+  va_end(ap);
+  return r;
+}
diff --git a/src/lib/string/scanf.h b/src/lib/string/scanf.h
new file mode 100644
index 0000000000..6673173de5
--- /dev/null
+++ b/src/lib/string/scanf.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file scanf.h
+ * \brief Header for scanf.c
+ **/
+
+#ifndef TOR_UTIL_SCANF_H
+#define TOR_UTIL_SCANF_H
+
+#include "orconfig.h"
+#include "lib/cc/compat_compiler.h"
+
+#include <stdarg.h>
+
+int tor_vsscanf(const char *buf, const char *pattern, va_list ap) \
+  CHECK_SCANF(2, 0);
+int tor_sscanf(const char *buf, const char *pattern, ...)
+  CHECK_SCANF(2, 3);
+
+#endif /* !defined(TOR_UTIL_STRING_H) */
diff --git a/src/lib/string/util_string.c b/src/lib/string/util_string.c
new file mode 100644
index 0000000000..f934f66f02
--- /dev/null
+++ b/src/lib/string/util_string.c
@@ -0,0 +1,543 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file util_string.c
+ * \brief Non-standard string functions used throughout Tor.
+ **/
+
+#include "lib/string/util_string.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/err/torerr.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/defs/digest_sizes.h"
+
+#include <string.h>
+#include <stdlib.h>
+
+/** Given <b>hlen</b> bytes at <b>haystack</b> and <b>nlen</b> bytes at
+ * <b>needle</b>, return a pointer to the first occurrence of the needle
+ * within the haystack, or NULL if there is no such occurrence.
+ *
+ * This function is <em>not</em> timing-safe.
+ *
+ * Requires that <b>nlen</b> be greater than zero.
+ */
+const void *
+tor_memmem(const void *_haystack, size_t hlen,
+           const void *_needle, size_t nlen)
+{
+#if defined(HAVE_MEMMEM) && (!defined(__GNUC__) || __GNUC__ >= 2)
+  raw_assert(nlen);
+  return memmem(_haystack, hlen, _needle, nlen);
+#else
+  /* This isn't as fast as the GLIBC implementation, but it doesn't need to
+   * be. */
+  const char *p, *last_possible_start;
+  const char *haystack = (const char*)_haystack;
+  const char *needle = (const char*)_needle;
+  char first;
+  raw_assert(nlen);
+
+  if (nlen > hlen)
+    return NULL;
+
+  p = haystack;
+  /* Last position at which the needle could start. */
+  last_possible_start = haystack + hlen - nlen;
+  first = *(const char*)needle;
+  while ((p = memchr(p, first, last_possible_start + 1 - p))) {
+    if (fast_memeq(p, needle, nlen))
+      return p;
+    if (++p > last_possible_start) {
+      /* This comparison shouldn't be necessary, since if p was previously
+       * equal to last_possible_start, the next memchr call would be
+       * "memchr(p, first, 0)", which will return NULL. But it clarifies the
+       * logic. */
+      return NULL;
+    }
+  }
+  return NULL;
+#endif /* defined(HAVE_MEMMEM) && (!defined(__GNUC__) || __GNUC__ >= 2) */
+}
+
+const void *
+tor_memstr(const void *haystack, size_t hlen, const char *needle)
+{
+  return tor_memmem(haystack, hlen, needle, strlen(needle));
+}
+
+/** Return true iff the 'len' bytes at 'mem' are all zero. */
+int
+tor_mem_is_zero(const char *mem, size_t len)
+{
+  static const char ZERO[] = {
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+  };
+  while (len >= sizeof(ZERO)) {
+    /* It's safe to use fast_memcmp here, since the very worst thing an
+     * attacker could learn is how many initial bytes of a secret were zero */
+    if (fast_memcmp(mem, ZERO, sizeof(ZERO)))
+      return 0;
+    len -= sizeof(ZERO);
+    mem += sizeof(ZERO);
+  }
+  /* Deal with leftover bytes. */
+  if (len)
+    return fast_memeq(mem, ZERO, len);
+
+  return 1;
+}
+
+/** Return true iff the DIGEST_LEN bytes in digest are all zero. */
+int
+tor_digest_is_zero(const char *digest)
+{
+  static const uint8_t ZERO_DIGEST[] = {
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
+  };
+  return tor_memeq(digest, ZERO_DIGEST, DIGEST_LEN);
+}
+
+/** Return true iff the DIGEST256_LEN bytes in digest are all zero. */
+int
+tor_digest256_is_zero(const char *digest)
+{
+  return tor_mem_is_zero(digest, DIGEST256_LEN);
+}
+
+/** Remove from the string <b>s</b> every character which appears in
+ * <b>strip</b>. */
+void
+tor_strstrip(char *s, const char *strip)
+{
+  char *readp = s;
+  while (*readp) {
+    if (strchr(strip, *readp)) {
+      ++readp;
+    } else {
+      *s++ = *readp++;
+    }
+  }
+  *s = '\0';
+}
+
+/** Convert all alphabetic characters in the nul-terminated string <b>s</b> to
+ * lowercase. */
+void
+tor_strlower(char *s)
+{
+  while (*s) {
+    *s = TOR_TOLOWER(*s);
+    ++s;
+  }
+}
+
+/** Convert all alphabetic characters in the nul-terminated string <b>s</b> to
+ * lowercase. */
+void
+tor_strupper(char *s)
+{
+  while (*s) {
+    *s = TOR_TOUPPER(*s);
+    ++s;
+  }
+}
+
+/** Return 1 if every character in <b>s</b> is printable, else return 0.
+ */
+int
+tor_strisprint(const char *s)
+{
+  while (*s) {
+    if (!TOR_ISPRINT(*s))
+      return 0;
+    s++;
+  }
+  return 1;
+}
+
+/** Return 1 if no character in <b>s</b> is uppercase, else return 0.
+ */
+int
+tor_strisnonupper(const char *s)
+{
+  while (*s) {
+    if (TOR_ISUPPER(*s))
+      return 0;
+    s++;
+  }
+  return 1;
+}
+
+/** Return true iff every character in <b>s</b> is whitespace space; else
+ * return false. */
+int
+tor_strisspace(const char *s)
+{
+  while (*s) {
+    if (!TOR_ISSPACE(*s))
+      return 0;
+    s++;
+  }
+  return 1;
+}
+
+/** As strcmp, except that either string may be NULL.  The NULL string is
+ * considered to be before any non-NULL string. */
+int
+strcmp_opt(const char *s1, const char *s2)
+{
+  if (!s1) {
+    if (!s2)
+      return 0;
+    else
+      return -1;
+  } else if (!s2) {
+    return 1;
+  } else {
+    return strcmp(s1, s2);
+  }
+}
+
+/** Compares the first strlen(s2) characters of s1 with s2.  Returns as for
+ * strcmp.
+ */
+int
+strcmpstart(const char *s1, const char *s2)
+{
+  size_t n = strlen(s2);
+  return strncmp(s1, s2, n);
+}
+
+/** Compare the s1_len-byte string <b>s1</b> with <b>s2</b>,
+ * without depending on a terminating nul in s1.  Sorting order is first by
+ * length, then lexically; return values are as for strcmp.
+ */
+int
+strcmp_len(const char *s1, const char *s2, size_t s1_len)
+{
+  size_t s2_len = strlen(s2);
+  if (s1_len < s2_len)
+    return -1;
+  if (s1_len > s2_len)
+    return 1;
+  return fast_memcmp(s1, s2, s2_len);
+}
+
+/** Compares the first strlen(s2) characters of s1 with s2.  Returns as for
+ * strcasecmp.
+ */
+int
+strcasecmpstart(const char *s1, const char *s2)
+{
+  size_t n = strlen(s2);
+  return strncasecmp(s1, s2, n);
+}
+
+/** Compare the value of the string <b>prefix</b> with the start of the
+ * <b>memlen</b>-byte memory chunk at <b>mem</b>.  Return as for strcmp.
+ *
+ * [As fast_memcmp(mem, prefix, strlen(prefix)) but returns -1 if memlen is
+ * less than strlen(prefix).]
+ */
+int
+fast_memcmpstart(const void *mem, size_t memlen,
+                const char *prefix)
+{
+  size_t plen = strlen(prefix);
+  if (memlen < plen)
+    return -1;
+  return fast_memcmp(mem, prefix, plen);
+}
+
+/** Compares the last strlen(s2) characters of s1 with s2.  Returns as for
+ * strcmp.
+ */
+int
+strcmpend(const char *s1, const char *s2)
+{
+  size_t n1 = strlen(s1), n2 = strlen(s2);
+  if (n2>n1)
+    return strcmp(s1,s2);
+  else
+    return strncmp(s1+(n1-n2), s2, n2);
+}
+
+/** Compares the last strlen(s2) characters of s1 with s2.  Returns as for
+ * strcasecmp.
+ */
+int
+strcasecmpend(const char *s1, const char *s2)
+{
+  size_t n1 = strlen(s1), n2 = strlen(s2);
+  if (n2>n1) /* then they can't be the same; figure out which is bigger */
+    return strcasecmp(s1,s2);
+  else
+    return strncasecmp(s1+(n1-n2), s2, n2);
+}
+
+/** Return a pointer to the first char of s that is not whitespace and
+ * not a comment, or to the terminating NUL if no such character exists.
+ */
+const char *
+eat_whitespace(const char *s)
+{
+  raw_assert(s);
+
+  while (1) {
+    switch (*s) {
+    case '\0':
+    default:
+      return s;
+    case ' ':
+    case '\t':
+    case '\n':
+    case '\r':
+      ++s;
+      break;
+    case '#':
+      ++s;
+      while (*s && *s != '\n')
+        ++s;
+    }
+  }
+}
+
+/** Return a pointer to the first char of s that is not whitespace and
+ * not a comment, or to the terminating NUL if no such character exists.
+ */
+const char *
+eat_whitespace_eos(const char *s, const char *eos)
+{
+  raw_assert(s);
+  raw_assert(eos && s <= eos);
+
+  while (s < eos) {
+    switch (*s) {
+    case '\0':
+    default:
+      return s;
+    case ' ':
+    case '\t':
+    case '\n':
+    case '\r':
+      ++s;
+      break;
+    case '#':
+      ++s;
+      while (s < eos && *s && *s != '\n')
+        ++s;
+    }
+  }
+  return s;
+}
+
+/** Return a pointer to the first char of s that is not a space or a tab
+ * or a \\r, or to the terminating NUL if no such character exists. */
+const char *
+eat_whitespace_no_nl(const char *s)
+{
+  while (*s == ' ' || *s == '\t' || *s == '\r')
+    ++s;
+  return s;
+}
+
+/** As eat_whitespace_no_nl, but stop at <b>eos</b> whether we have
+ * found a non-whitespace character or not. */
+const char *
+eat_whitespace_eos_no_nl(const char *s, const char *eos)
+{
+  while (s < eos && (*s == ' ' || *s == '\t' || *s == '\r'))
+    ++s;
+  return s;
+}
+
+/** Return a pointer to the first char of s that is whitespace or <b>#</b>,
+ * or to the terminating NUL if no such character exists.
+ */
+const char *
+find_whitespace(const char *s)
+{
+  /* tor_assert(s); */
+  while (1) {
+    switch (*s)
+    {
+    case '\0':
+    case '#':
+    case ' ':
+    case '\r':
+    case '\n':
+    case '\t':
+      return s;
+    default:
+      ++s;
+    }
+  }
+}
+
+/** As find_whitespace, but stop at <b>eos</b> whether we have found a
+ * whitespace or not. */
+const char *
+find_whitespace_eos(const char *s, const char *eos)
+{
+  /* tor_assert(s); */
+  while (s < eos) {
+    switch (*s)
+    {
+    case '\0':
+    case '#':
+    case ' ':
+    case '\r':
+    case '\n':
+    case '\t':
+      return s;
+    default:
+      ++s;
+    }
+  }
+  return s;
+}
+
+/** Return the first occurrence of <b>needle</b> in <b>haystack</b> that
+ * occurs at the start of a line (that is, at the beginning of <b>haystack</b>
+ * or immediately after a newline).  Return NULL if no such string is found.
+ */
+const char *
+find_str_at_start_of_line(const char *haystack, const char *needle)
+{
+  size_t needle_len = strlen(needle);
+
+  do {
+    if (!strncmp(haystack, needle, needle_len))
+      return haystack;
+
+    haystack = strchr(haystack, '\n');
+    if (!haystack)
+      return NULL;
+    else
+      ++haystack;
+  } while (*haystack);
+
+  return NULL;
+}
+
+/** Returns true if <b>string</b> could be a C identifier.
+    A C identifier must begin with a letter or an underscore and the
+    rest of its characters can be letters, numbers or underscores. No
+    length limit is imposed. */
+int
+string_is_C_identifier(const char *string)
+{
+  size_t iter;
+  size_t length = strlen(string);
+  if (!length)
+    return 0;
+
+  for (iter = 0; iter < length ; iter++) {
+    if (iter == 0) {
+      if (!(TOR_ISALPHA(string[iter]) ||
+            string[iter] == '_'))
+        return 0;
+    } else {
+      if (!(TOR_ISALPHA(string[iter]) ||
+            TOR_ISDIGIT(string[iter]) ||
+            string[iter] == '_'))
+        return 0;
+    }
+  }
+
+  return 1;
+}
+
+/** A byte with the top <b>x</b> bits set. */
+#define TOP_BITS(x) ((uint8_t)(0xFF << (8 - (x))))
+/** A byte with the lowest <b>x</b> bits set. */
+#define LOW_BITS(x) ((uint8_t)(0xFF >> (8 - (x))))
+
+/** Given the leading byte <b>b</b>, return the total number of bytes in the
+ * UTF-8 character. Returns 0 if it's an invalid leading byte.
+ */
+static uint8_t
+bytes_in_char(uint8_t b)
+{
+  if ((TOP_BITS(1) & b) == 0x00)
+    return 1; // a 1-byte UTF-8 char, aka ASCII
+  if ((TOP_BITS(3) & b) == TOP_BITS(2))
+    return 2; // a 2-byte UTF-8 char
+  if ((TOP_BITS(4) & b) == TOP_BITS(3))
+    return 3; // a 3-byte UTF-8 char
+  if ((TOP_BITS(5) & b) == TOP_BITS(4))
+    return 4; // a 4-byte UTF-8 char
+
+  // Invalid: either the top 2 bits are 10, or the top 5 bits are 11111.
+  return 0;
+}
+
+/** Returns true iff <b>b</b> is a UTF-8 continuation byte. */
+static bool
+is_continuation_byte(uint8_t b)
+{
+  uint8_t top2bits = b & TOP_BITS(2);
+  return top2bits == TOP_BITS(1);
+}
+
+/** Returns true iff the <b>len</b> bytes in <b>c</b> are a valid UTF-8
+ * character.
+ */
+static bool
+validate_char(const uint8_t *c, uint8_t len)
+{
+  if (len == 1)
+    return true; // already validated this is an ASCII char earlier.
+
+  uint8_t mask = LOW_BITS(7 - len); // bitmask for the leading byte.
+  uint32_t codepoint = c[0] & mask;
+
+  mask = LOW_BITS(6); // bitmask for continuation bytes.
+  for (uint8_t i = 1; i < len; i++) {
+    if (!is_continuation_byte(c[i]))
+      return false;
+    codepoint <<= 6;
+    codepoint |= (c[i] & mask);
+  }
+
+  if (len == 2 && codepoint <= 0x7f)
+    return false; // Invalid, overly long encoding, should have fit in 1 byte.
+
+  if (len == 3 && codepoint <= 0x7ff)
+    return false; // Invalid, overly long encoding, should have fit in 2 bytes.
+
+  if (len == 4 && codepoint <= 0xffff)
+    return false; // Invalid, overly long encoding, should have fit in 3 bytes.
+
+  if (codepoint >= 0xd800 && codepoint <= 0xdfff)
+    return false; // Invalid, reserved for UTF-16 surrogate pairs.
+
+  return codepoint <= 0x10ffff; // Check if within maximum.
+}
+
+/** Returns true iff the first <b>len</b> bytes in <b>str</b> are a
+    valid UTF-8 string. */
+int
+string_is_utf8(const char *str, size_t len)
+{
+  for (size_t i = 0; i < len;) {
+    uint8_t num_bytes = bytes_in_char(str[i]);
+    if (num_bytes == 0) // Invalid leading byte found.
+      return false;
+
+    size_t next_char = i + num_bytes;
+    if (next_char > len)
+      return false;
+
+    // Validate the continuation bytes in this multi-byte character,
+    // and advance to the next character in the string.
+    if (!validate_char((const uint8_t*)&str[i], num_bytes))
+      return false;
+    i = next_char;
+  }
+  return true;
+}
diff --git a/src/lib/string/util_string.h b/src/lib/string/util_string.h
new file mode 100644
index 0000000000..d9fbf8c61e
--- /dev/null
+++ b/src/lib/string/util_string.h
@@ -0,0 +1,57 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file util_string.h
+ * \brief Header for util_string.c
+ **/
+
+#ifndef TOR_UTIL_STRING_H
+#define TOR_UTIL_STRING_H
+
+#include "orconfig.h"
+#include "lib/cc/compat_compiler.h"
+
+#include <stddef.h>
+
+const void *tor_memmem(const void *haystack, size_t hlen, const void *needle,
+                       size_t nlen);
+const void *tor_memstr(const void *haystack, size_t hlen,
+                       const char *needle);
+int tor_mem_is_zero(const char *mem, size_t len);
+int tor_digest_is_zero(const char *digest);
+int tor_digest256_is_zero(const char *digest);
+
+/** Allowable characters in a hexadecimal string. */
+#define HEX_CHARACTERS "0123456789ABCDEFabcdef"
+void tor_strlower(char *s);
+void tor_strupper(char *s);
+int tor_strisprint(const char *s);
+int tor_strisnonupper(const char *s);
+int tor_strisspace(const char *s);
+int strcmp_opt(const char *s1, const char *s2);
+int strcmpstart(const char *s1, const char *s2);
+int strcmp_len(const char *s1, const char *s2, size_t len);
+int strcasecmpstart(const char *s1, const char *s2);
+int strcmpend(const char *s1, const char *s2);
+int strcasecmpend(const char *s1, const char *s2);
+int fast_memcmpstart(const void *mem, size_t memlen, const char *prefix);
+
+void tor_strstrip(char *s, const char *strip);
+
+const char *eat_whitespace(const char *s);
+const char *eat_whitespace_eos(const char *s, const char *eos);
+const char *eat_whitespace_no_nl(const char *s);
+const char *eat_whitespace_eos_no_nl(const char *s, const char *eos);
+const char *find_whitespace(const char *s);
+const char *find_whitespace_eos(const char *s, const char *eos);
+const char *find_str_at_start_of_line(const char *haystack,
+                                      const char *needle);
+
+int string_is_C_identifier(const char *string);
+
+int string_is_utf8(const char *str, size_t len);
+
+#endif /* !defined(TOR_UTIL_STRING_H) */
diff --git a/src/lib/term/.may_include b/src/lib/term/.may_include
new file mode 100644
index 0000000000..c93a06e59e
--- /dev/null
+++ b/src/lib/term/.may_include
@@ -0,0 +1,9 @@
+orconfig.h
+
+lib/cc/*.h
+lib/log/*.h
+lib/term/*.h
+lib/malloc/*.h
+
+# From src/ext
+tor_readpassphrase.h
diff --git a/src/lib/term/getpass.c b/src/lib/term/getpass.c
new file mode 100644
index 0000000000..c6459f250f
--- /dev/null
+++ b/src/lib/term/getpass.c
@@ -0,0 +1,120 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file getpass.c
+ * \brief Cross-platform wrapper to read passphrases from the terminal.
+ **/
+
+#include "lib/term/getpass.h"
+
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+
+#ifdef _WIN32
+#include <windows.h>
+#include <conio.h>
+#include <wchar.h>
+/* Some mingw headers lack these. :p */
+#if defined(HAVE_DECL__GETWCH) && !HAVE_DECL__GETWCH
+wint_t _getwch(void);
+#endif
+#ifndef WEOF
+#define WEOF (wchar_t)(0xFFFF)
+#endif
+#if defined(HAVE_DECL_SECUREZEROMEMORY) && !HAVE_DECL_SECUREZEROMEMORY
+static inline void
+SecureZeroMemory(PVOID ptr, SIZE_T cnt)
+{
+  volatile char *vcptr = (volatile char*)ptr;
+  while (cnt--)
+    *vcptr++ = 0;
+}
+#endif /* defined(HAVE_DECL_SECUREZEROMEMORY) && !HAVE_DECL_SECUREZEROMEMORY */
+#elif defined(HAVE_READPASSPHRASE_H)
+#include <readpassphrase.h>
+#else
+#include "tor_readpassphrase.h"
+#endif /* defined(_WIN32) || ... */
+
+#include <stdlib.h>
+#include <string.h>
+
+/** Emit the password prompt <b>prompt</b>, then read up to <b>buflen</b>
+ * bytes of passphrase into <b>output</b>. Return the number of bytes in
+ * the passphrase, excluding terminating NUL.
+ */
+ssize_t
+tor_getpass(const char *prompt, char *output, size_t buflen)
+{
+  tor_assert(buflen <= SSIZE_MAX);
+  tor_assert(buflen >= 1);
+#if defined(HAVE_READPASSPHRASE)
+  char *pwd = readpassphrase(prompt, output, buflen, RPP_ECHO_OFF);
+  if (pwd == NULL)
+    return -1;
+  return strlen(pwd);
+#elif defined(_WIN32)
+  int r = -1;
+  while (*prompt) {
+    _putch(*prompt++);
+  }
+
+  tor_assert(buflen <= INT_MAX);
+  wchar_t *buf = tor_calloc(buflen, sizeof(wchar_t));
+
+  wchar_t *ptr = buf, *lastch = buf + buflen - 1;
+  while (ptr < lastch) {
+    wint_t ch = _getwch();
+    switch (ch) {
+      case '\r':
+      case '\n':
+      case WEOF:
+        goto done_reading;
+      case 3:
+        goto done; /* Can't actually read ctrl-c this way. */
+      case '\b':
+        if (ptr > buf)
+          --ptr;
+        continue;
+      case 0:
+      case 0xe0:
+        ch = _getwch(); /* Ignore; this is a function or arrow key */
+        break;
+      default:
+        *ptr++ = ch;
+        break;
+    }
+  }
+ done_reading:
+  ;
+
+#ifndef WC_ERR_INVALID_CHARS
+#define WC_ERR_INVALID_CHARS 0x80
+#endif
+
+  /* Now convert it to UTF-8 */
+  r = WideCharToMultiByte(CP_UTF8,
+                          WC_NO_BEST_FIT_CHARS|WC_ERR_INVALID_CHARS,
+                          buf, (int)(ptr-buf),
+                          output, (int)(buflen-1),
+                          NULL, NULL);
+  if (r <= 0) {
+    r = -1;
+    goto done;
+  }
+
+  tor_assert(r < (int)buflen);
+
+  output[r] = 0;
+
+ done:
+  SecureZeroMemory(buf, sizeof(wchar_t)*buflen);
+  tor_free(buf);
+  return r;
+#else
+#error "No implementation for tor_getpass found!"
+#endif /* defined(HAVE_READPASSPHRASE) || ... */
+}
diff --git a/src/lib/term/getpass.h b/src/lib/term/getpass.h
new file mode 100644
index 0000000000..a9c146ea8f
--- /dev/null
+++ b/src/lib/term/getpass.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file getpass.h
+ * \brief Header for getpass.c
+ **/
+
+#ifndef TOR_GETPASS_H
+#define TOR_GETPASS_H
+
+#include "lib/cc/torint.h"
+
+ssize_t tor_getpass(const char *prompt, char *output, size_t buflen);
+
+#endif
diff --git a/src/lib/term/include.am b/src/lib/term/include.am
new file mode 100644
index 0000000000..55fe548ebc
--- /dev/null
+++ b/src/lib/term/include.am
@@ -0,0 +1,24 @@
+
+noinst_LIBRARIES += src/lib/libtor-term.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-term-testing.a
+endif
+
+if BUILD_READPASSPHRASE_C
+readpassphrase_source=src/ext/readpassphrase.c
+else
+readpassphrase_source=
+endif
+
+src_lib_libtor_term_a_SOURCES =			\
+	src/lib/term/getpass.c			\
+	$(readpassphrase_source)
+
+src_lib_libtor_term_testing_a_SOURCES = \
+	$(src_lib_libtor_term_a_SOURCES)
+src_lib_libtor_term_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_term_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/term/getpass.h
diff --git a/src/lib/testsupport/.may_include b/src/lib/testsupport/.may_include
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/lib/testsupport/.may_include
diff --git a/src/lib/testsupport/include.am b/src/lib/testsupport/include.am
new file mode 100644
index 0000000000..b2aa620985
--- /dev/null
+++ b/src/lib/testsupport/include.am
@@ -0,0 +1,3 @@
+
+noinst_HEADERS += \
+	src/lib/testsupport/testsupport.h
diff --git a/src/common/testsupport.h b/src/lib/testsupport/testsupport.h
index 9ad2ba77e0..9363a9ba66 100644
--- a/src/common/testsupport.h
+++ b/src/lib/testsupport/testsupport.h
@@ -1,16 +1,30 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * \file testsupport.h
+ *
+ * \brief Macros to implement mocking and selective exposure for the test code.
+ *
+ * Each Tor source file is built twice: once with TOR_UNIT_TESTS defined, and
+ * once with it undefined.  The only difference between these configurations
+ * should be that when building for the tests, more functions are exposed as
+ * non-static, and a number of functions are declared as mockable.
+ **/
+
 #ifndef TOR_TESTSUPPORT_H
 #define TOR_TESTSUPPORT_H
 
 #ifdef TOR_UNIT_TESTS
+/** The "STATIC" macro marks a function or variable that is static when
+ * building Tor for production, but non-static when building the unit
+ * tests. */
 #define STATIC
 #define EXTERN(type, name) extern type name;
 #else
 #define STATIC static
 #define EXTERN(type, name)
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /** Quick and dirty macros to implement test mocking.
  *
@@ -76,15 +90,14 @@
   do {                                          \
     func = func ##__real;                       \
   } while (0)
-#else
+#else /* !(defined(TOR_UNIT_TESTS)) */
 #define MOCK_DECL(rv, funcname, arglist) \
   rv funcname arglist
 #define MOCK_DECL_ATTR(rv, funcname, arglist, attr) \
   rv funcname arglist attr
 #define MOCK_IMPL(rv, funcname, arglist) \
   rv funcname arglist
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 /** @} */
 
-#endif
-
+#endif /* !defined(TOR_TESTSUPPORT_H) */
diff --git a/src/lib/thread/.may_include b/src/lib/thread/.may_include
new file mode 100644
index 0000000000..fc56f46836
--- /dev/null
+++ b/src/lib/thread/.may_include
@@ -0,0 +1,7 @@
+orconfig.h
+lib/cc/*.h
+lib/lock/*.h
+lib/log/*.h
+lib/testsupport/*.h
+lib/thread/*.h
+lib/wallclock/*.h
diff --git a/src/common/compat_pthreads.c b/src/lib/thread/compat_pthreads.c
index c1ae66c1d2..05efe9cfd0 100644
--- a/src/common/compat_pthreads.c
+++ b/src/lib/thread/compat_pthreads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,13 +11,17 @@
  */
 
 #include "orconfig.h"
+#include "lib/thread/threads.h"
+#include "lib/wallclock/timeval.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+
+#include <sys/time.h>
 #include <pthread.h>
 #include <signal.h>
 #include <time.h>
-
-#include "compat.h"
-#include "torlog.h"
-#include "util.h"
+#include <errno.h>
+#include <string.h>
 
 /** Wraps a void (*)(void*) function and its argument so we can
  * invoke them in a way pthreads would expect.
@@ -91,88 +95,6 @@ spawn_exit(void)
   pthread_exit(NULL);
 }
 
-/** A mutex attribute that we're going to use to tell pthreads that we want
- * "recursive" mutexes (i.e., once we can re-lock if we're already holding
- * them.) */
-static pthread_mutexattr_t attr_recursive;
-
-/** Initialize <b>mutex</b> so it can be locked.  Every mutex must be set
- * up with tor_mutex_init() or tor_mutex_new(); not both. */
-void
-tor_mutex_init(tor_mutex_t *mutex)
-{
-  if (PREDICT_UNLIKELY(!threads_initialized))
-    tor_threads_init(); // LCOV_EXCL_LINE
-  const int err = pthread_mutex_init(&mutex->mutex, &attr_recursive);
-  if (PREDICT_UNLIKELY(err)) {
-    // LCOV_EXCL_START
-    log_err(LD_GENERAL, "Error %d creating a mutex.", err);
-    tor_assert_unreached();
-    // LCOV_EXCL_STOP
-  }
-}
-
-/** As tor_mutex_init, but initialize a mutex suitable that may be
- * non-recursive, if the OS supports that. */
-void
-tor_mutex_init_nonrecursive(tor_mutex_t *mutex)
-{
-  int err;
-  if (!threads_initialized)
-    tor_threads_init(); // LCOV_EXCL_LINE
-  err = pthread_mutex_init(&mutex->mutex, NULL);
-  if (PREDICT_UNLIKELY(err)) {
-    // LCOV_EXCL_START
-    log_err(LD_GENERAL, "Error %d creating a mutex.", err);
-    tor_assert_unreached();
-    // LCOV_EXCL_STOP
-  }
-}
-
-/** Wait until <b>m</b> is free, then acquire it. */
-void
-tor_mutex_acquire(tor_mutex_t *m)
-{
-  int err;
-  tor_assert(m);
-  err = pthread_mutex_lock(&m->mutex);
-  if (PREDICT_UNLIKELY(err)) {
-    // LCOV_EXCL_START
-    log_err(LD_GENERAL, "Error %d locking a mutex.", err);
-    tor_assert_unreached();
-    // LCOV_EXCL_STOP
-  }
-}
-/** Release the lock <b>m</b> so another thread can have it. */
-void
-tor_mutex_release(tor_mutex_t *m)
-{
-  int err;
-  tor_assert(m);
-  err = pthread_mutex_unlock(&m->mutex);
-  if (PREDICT_UNLIKELY(err)) {
-    // LCOV_EXCL_START
-    log_err(LD_GENERAL, "Error %d unlocking a mutex.", err);
-    tor_assert_unreached();
-    // LCOV_EXCL_STOP
-  }
-}
-/** Clean up the mutex <b>m</b> so that it no longer uses any system
- * resources.  Does not free <b>m</b>.  This function must only be called on
- * mutexes from tor_mutex_init(). */
-void
-tor_mutex_uninit(tor_mutex_t *m)
-{
-  int err;
-  tor_assert(m);
-  err = pthread_mutex_destroy(&m->mutex);
-  if (PREDICT_UNLIKELY(err)) {
-    // LCOV_EXCL_START
-    log_err(LD_GENERAL, "Error %d destroying a mutex.", err);
-    tor_assert_unreached();
-    // LCOV_EXCL_STOP
-  }
-}
 /** Return an integer representing this thread. */
 unsigned long
 tor_get_thread_id(void)
@@ -201,20 +123,21 @@ tor_cond_init(tor_cond_t *cond)
   }
 
 #if defined(HAVE_CLOCK_GETTIME)
-#if defined(CLOCK_MONOTONIC) && defined(HAVE_PTHREAD_CONDATTR_SETCLOCK)
+#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && \
+  defined(CLOCK_MONOTONIC)
   /* Use monotonic time so when we timedwait() on it, any clock adjustment
    * won't affect the timeout value. */
   if (pthread_condattr_setclock(&condattr, CLOCK_MONOTONIC)) {
     return -1;
   }
 #define USE_COND_CLOCK CLOCK_MONOTONIC
-#else /* !defined HAVE_PTHREAD_CONDATTR_SETCLOCK */
+#else /* !(defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && ...) */
   /* On OSX Sierra, there is no pthread_condattr_setclock, so we are stuck
    * with the realtime clock.
    */
 #define USE_COND_CLOCK CLOCK_REALTIME
-#endif /* which clock to use */
-#endif /* HAVE_CLOCK_GETTIME */
+#endif /* defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && ... */
+#endif /* defined(HAVE_CLOCK_GETTIME) */
   if (pthread_cond_init(&cond->cond, &condattr)) {
     return -1;
   }
@@ -266,11 +189,11 @@ tor_cond_wait(tor_cond_t *cond, tor_mutex_t *mutex, const struct timeval *tv)
       tvnow.tv_sec = ts.tv_sec;
       tvnow.tv_usec = (int)(ts.tv_nsec / 1000);
       timeradd(tv, &tvnow, &tvsum);
-#else
+#else /* !(defined(HAVE_CLOCK_GETTIME) && defined(USE_COND_CLOCK)) */
       if (gettimeofday(&tvnow, NULL) < 0)
         return -1;
       timeradd(tv, &tvnow, &tvsum);
-#endif /* HAVE_CLOCK_GETTIME, CLOCK_MONOTONIC */
+#endif /* defined(HAVE_CLOCK_GETTIME) && defined(USE_COND_CLOCK) */
 
       ts.tv_sec = tvsum.tv_sec;
       ts.tv_nsec = tvsum.tv_usec * 1000;
@@ -332,8 +255,7 @@ void
 tor_threads_init(void)
 {
   if (!threads_initialized) {
-    pthread_mutexattr_init(&attr_recursive);
-    pthread_mutexattr_settype(&attr_recursive, PTHREAD_MUTEX_RECURSIVE);
+    tor_locking_init();
     const int ret1 = pthread_attr_init(&attr_detached);
     tor_assert(ret1 == 0);
 #ifndef PTHREAD_CREATE_DETACHED
@@ -346,4 +268,3 @@ tor_threads_init(void)
     set_main_thread();
   }
 }
-
diff --git a/src/lib/thread/compat_threads.c b/src/lib/thread/compat_threads.c
new file mode 100644
index 0000000000..94ab021c52
--- /dev/null
+++ b/src/lib/thread/compat_threads.c
@@ -0,0 +1,111 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compat_threads.c
+ *
+ * \brief Cross-platform threading and inter-thread communication logic.
+ *  (Platform-specific parts are written in the other compat_*threads
+ *  modules.)
+ */
+
+#include "orconfig.h"
+#include <stdlib.h>
+#include "lib/thread/threads.h"
+
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+
+#include <string.h>
+
+/** Allocate and return a new condition variable. */
+tor_cond_t *
+tor_cond_new(void)
+{
+  tor_cond_t *cond = tor_malloc(sizeof(tor_cond_t));
+  if (BUG(tor_cond_init(cond)<0))
+    tor_free(cond); // LCOV_EXCL_LINE
+  return cond;
+}
+
+/** Free all storage held in <b>c</b>. */
+void
+tor_cond_free_(tor_cond_t *c)
+{
+  if (!c)
+    return;
+  tor_cond_uninit(c);
+  tor_free(c);
+}
+
+/** Identity of the "main" thread */
+static unsigned long main_thread_id = -1;
+
+/** Start considering the current thread to be the 'main thread'.  This has
+ * no effect on anything besides in_main_thread(). */
+void
+set_main_thread(void)
+{
+  main_thread_id = tor_get_thread_id();
+}
+/** Return true iff called from the main thread. */
+int
+in_main_thread(void)
+{
+  return main_thread_id == tor_get_thread_id();
+}
+
+#ifndef HAVE_WORKING_STDATOMIC
+/** Initialize a new atomic counter with the value 0 */
+void
+atomic_counter_init(atomic_counter_t *counter)
+{
+  memset(counter, 0, sizeof(*counter));
+  tor_mutex_init_nonrecursive(&counter->mutex);
+}
+/** Clean up all resources held by an atomic counter. */
+void
+atomic_counter_destroy(atomic_counter_t *counter)
+{
+  tor_mutex_uninit(&counter->mutex);
+  memset(counter, 0, sizeof(*counter));
+}
+/** Add a value to an atomic counter. */
+void
+atomic_counter_add(atomic_counter_t *counter, size_t add)
+{
+  tor_mutex_acquire(&counter->mutex);
+  counter->val += add;
+  tor_mutex_release(&counter->mutex);
+}
+/** Subtract a value from an atomic counter. */
+void
+atomic_counter_sub(atomic_counter_t *counter, size_t sub)
+{
+  // this relies on unsigned overflow, but that's fine.
+  atomic_counter_add(counter, -sub);
+}
+/** Return the current value of an atomic counter */
+size_t
+atomic_counter_get(atomic_counter_t *counter)
+{
+  size_t val;
+  tor_mutex_acquire(&counter->mutex);
+  val = counter->val;
+  tor_mutex_release(&counter->mutex);
+  return val;
+}
+/** Replace the value of an atomic counter; return the old one. */
+size_t
+atomic_counter_exchange(atomic_counter_t *counter, size_t newval)
+{
+  size_t oldval;
+  tor_mutex_acquire(&counter->mutex);
+  oldval = counter->val;
+  counter->val = newval;
+  tor_mutex_release(&counter->mutex);
+  return oldval;
+}
+#endif /* !defined(HAVE_WORKING_STDATOMIC) */
diff --git a/src/common/compat_winthreads.c b/src/lib/thread/compat_winthreads.c
index 735be4ad17..f0b1430e84 100644
--- a/src/common/compat_winthreads.c
+++ b/src/lib/thread/compat_winthreads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,13 +12,12 @@
 
 #ifdef _WIN32
 
-#include "compat.h"
 #include <windows.h>
 #include <process.h>
-#include "util.h"
-#include "container.h"
-#include "torlog.h"
-#include <process.h>
+#include "lib/thread/threads.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/log/win32err.h"
 
 /* This value is more or less total cargo-cult */
 #define SPIN_COUNT 2000
@@ -48,39 +47,14 @@ void
 spawn_exit(void)
 {
   _endthread();
+  // LCOV_EXCL_START
   //we should never get here. my compiler thinks that _endthread returns, this
   //is an attempt to fool it.
   tor_assert(0);
-  _exit(0);
+  _exit(0); // exit ok: unreachable.
+  // LCOV_EXCL_STOP
 }
 
-void
-tor_mutex_init(tor_mutex_t *m)
-{
-  InitializeCriticalSection(&m->mutex);
-}
-void
-tor_mutex_init_nonrecursive(tor_mutex_t *m)
-{
-  InitializeCriticalSection(&m->mutex);
-}
-
-void
-tor_mutex_uninit(tor_mutex_t *m)
-{
-  DeleteCriticalSection(&m->mutex);
-}
-void
-tor_mutex_acquire(tor_mutex_t *m)
-{
-  tor_assert(m);
-  EnterCriticalSection(&m->mutex);
-}
-void
-tor_mutex_release(tor_mutex_t *m)
-{
-  LeaveCriticalSection(&m->mutex);
-}
 unsigned long
 tor_get_thread_id(void)
 {
@@ -246,5 +220,4 @@ tor_threads_init(void)
   set_main_thread();
 }
 
-#endif
-
+#endif /* defined(_WIN32) */
diff --git a/src/lib/thread/include.am b/src/lib/thread/include.am
new file mode 100644
index 0000000000..9ec23d166e
--- /dev/null
+++ b/src/lib/thread/include.am
@@ -0,0 +1,27 @@
+
+noinst_LIBRARIES += src/lib/libtor-thread.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-thread-testing.a
+endif
+
+if THREADS_PTHREADS
+threads_impl_source=src/lib/thread/compat_pthreads.c
+endif
+if THREADS_WIN32
+threads_impl_source=src/lib/thread/compat_winthreads.c
+endif
+
+src_lib_libtor_thread_a_SOURCES =			\
+	src/lib/thread/compat_threads.c			\
+	src/lib/thread/numcpus.c			\
+	$(threads_impl_source)
+
+src_lib_libtor_thread_testing_a_SOURCES = \
+	$(src_lib_libtor_thread_a_SOURCES)
+src_lib_libtor_thread_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_thread_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/thread/threads.h			\
+	src/lib/thread/numcpus.h
diff --git a/src/lib/thread/numcpus.c b/src/lib/thread/numcpus.c
new file mode 100644
index 0000000000..b293d965d2
--- /dev/null
+++ b/src/lib/thread/numcpus.c
@@ -0,0 +1,98 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file numcpus.c
+ * \brief Compute the number of CPUs configured on this system.
+ **/
+
+#include "orconfig.h"
+#include "lib/thread/numcpus.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+#include <stdlib.h>
+
+/** Implementation logic for compute_num_cpus(). */
+static int
+compute_num_cpus_impl(void)
+{
+#ifdef _WIN32
+  SYSTEM_INFO info;
+  memset(&info, 0, sizeof(info));
+  GetSystemInfo(&info);
+  if (info.dwNumberOfProcessors >= 1 && info.dwNumberOfProcessors < INT_MAX)
+    return (int)info.dwNumberOfProcessors;
+  else
+    return -1;
+#elif defined(HAVE_SYSCONF)
+#ifdef _SC_NPROCESSORS_CONF
+  long cpus_conf = sysconf(_SC_NPROCESSORS_CONF);
+#else
+  long cpus_conf = -1;
+#endif
+#ifdef _SC_NPROCESSORS_ONLN
+  long cpus_onln = sysconf(_SC_NPROCESSORS_ONLN);
+#else
+  long cpus_onln = -1;
+#endif
+  long cpus = -1;
+
+  if (cpus_conf > 0 && cpus_onln < 0) {
+    cpus = cpus_conf;
+  } else if (cpus_onln > 0 && cpus_conf < 0) {
+    cpus = cpus_onln;
+  } else if (cpus_onln > 0 && cpus_conf > 0) {
+    if (cpus_onln < cpus_conf) {
+      log_notice(LD_GENERAL, "I think we have %ld CPUS, but only %ld of them "
+                 "are available. Telling Tor to only use %ld. You can over"
+                 "ride this with the NumCPUs option",
+                 cpus_conf, cpus_onln, cpus_onln);
+    }
+    cpus = cpus_onln;
+  }
+
+  if (cpus >= 1 && cpus < INT_MAX)
+    return (int)cpus;
+  else
+    return -1;
+#else
+  return -1;
+#endif /* defined(_WIN32) || ... */
+}
+
+#define MAX_DETECTABLE_CPUS 16
+
+/** Return how many CPUs we are running with.  We assume that nobody is
+ * using hot-swappable CPUs, so we don't recompute this after the first
+ * time.  Return -1 if we don't know how to tell the number of CPUs on this
+ * system.
+ */
+int
+compute_num_cpus(void)
+{
+  static int num_cpus = -2;
+  if (num_cpus == -2) {
+    num_cpus = compute_num_cpus_impl();
+    tor_assert(num_cpus != -2);
+    if (num_cpus > MAX_DETECTABLE_CPUS) {
+      /* LCOV_EXCL_START */
+      log_notice(LD_GENERAL, "Wow!  I detected that you have %d CPUs. I "
+                 "will not autodetect any more than %d, though.  If you "
+                 "want to configure more, set NumCPUs in your torrc",
+                 num_cpus, MAX_DETECTABLE_CPUS);
+      num_cpus = MAX_DETECTABLE_CPUS;
+      /* LCOV_EXCL_STOP */
+    }
+  }
+  return num_cpus;
+}
diff --git a/src/lib/thread/numcpus.h b/src/lib/thread/numcpus.h
new file mode 100644
index 0000000000..3f0a29ce7c
--- /dev/null
+++ b/src/lib/thread/numcpus.h
@@ -0,0 +1,16 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file numcpus.h
+ * \brief Header for numcpus.c
+ **/
+
+#ifndef TOR_NUMCPUS_H
+#define TOR_NUMCPUS_H
+
+int compute_num_cpus(void);
+
+#endif
diff --git a/src/lib/thread/threads.h b/src/lib/thread/threads.h
new file mode 100644
index 0000000000..ecf60641b5
--- /dev/null
+++ b/src/lib/thread/threads.h
@@ -0,0 +1,168 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file threads.h
+ * \brief Header for threads.c
+ **/
+
+#ifndef TOR_COMPAT_THREADS_H
+#define TOR_COMPAT_THREADS_H
+
+#include "orconfig.h"
+#include "lib/cc/torint.h"
+#include "lib/testsupport/testsupport.h"
+#include "lib/lock/compat_mutex.h"
+
+#if defined(HAVE_STDATOMIC_H) && defined(STDATOMIC_WORKS)
+#define HAVE_WORKING_STDATOMIC
+#endif
+
+#ifdef HAVE_WORKING_STDATOMIC
+#include <stdatomic.h>
+#endif
+
+struct timeval;
+
+int spawn_func(void (*func)(void *), void *data);
+void spawn_exit(void) ATTR_NORETURN;
+
+unsigned long tor_get_thread_id(void);
+void tor_threads_init(void);
+
+/** Conditions need nonrecursive mutexes with pthreads. */
+#define tor_mutex_init_for_cond(m) tor_mutex_init_nonrecursive(m)
+
+void set_main_thread(void);
+int in_main_thread(void);
+
+typedef struct tor_cond_t {
+#ifdef USE_PTHREADS
+  pthread_cond_t cond;
+#elif defined(USE_WIN32_THREADS)
+  HANDLE event;
+
+  CRITICAL_SECTION lock;
+  int n_waiting;
+  int n_to_wake;
+  int generation;
+#else
+#error no known condition implementation.
+#endif /* defined(USE_PTHREADS) || ... */
+} tor_cond_t;
+
+tor_cond_t *tor_cond_new(void);
+void tor_cond_free_(tor_cond_t *cond);
+#define tor_cond_free(c) FREE_AND_NULL(tor_cond_t, tor_cond_free_, (c))
+int tor_cond_init(tor_cond_t *cond);
+void tor_cond_uninit(tor_cond_t *cond);
+int tor_cond_wait(tor_cond_t *cond, tor_mutex_t *mutex,
+                  const struct timeval *tv);
+void tor_cond_signal_one(tor_cond_t *cond);
+void tor_cond_signal_all(tor_cond_t *cond);
+
+typedef struct tor_threadlocal_s {
+#ifdef _WIN32
+  DWORD index;
+#else
+  pthread_key_t key;
+#endif
+} tor_threadlocal_t;
+
+/** Initialize a thread-local variable.
+ *
+ * After you call this function on a tor_threadlocal_t, you can call
+ * tor_threadlocal_set to change the current value of this variable for the
+ * current thread, and tor_threadlocal_get to retrieve the current value for
+ * the current thread.  Each thread has its own value.
+ **/
+int tor_threadlocal_init(tor_threadlocal_t *threadlocal);
+/**
+ * Release all resource associated with a thread-local variable.
+ */
+void tor_threadlocal_destroy(tor_threadlocal_t *threadlocal);
+/**
+ * Return the current value of a thread-local variable for this thread.
+ *
+ * It's undefined behavior to use this function if the threadlocal hasn't
+ * been initialized, or has been destroyed.
+ */
+void *tor_threadlocal_get(tor_threadlocal_t *threadlocal);
+/**
+ * Change the current value of a thread-local variable for this thread to
+ * <b>value</b>.
+ *
+ * It's undefined behavior to use this function if the threadlocal hasn't
+ * been initialized, or has been destroyed.
+ */
+void tor_threadlocal_set(tor_threadlocal_t *threadlocal, void *value);
+
+/**
+ * Atomic counter type; holds a size_t value.
+ */
+#ifdef HAVE_WORKING_STDATOMIC
+typedef struct atomic_counter_t {
+  atomic_size_t val;
+} atomic_counter_t;
+#define ATOMIC_LINKAGE static
+#else /* !(defined(HAVE_WORKING_STDATOMIC)) */
+typedef struct atomic_counter_t {
+  tor_mutex_t mutex;
+  size_t val;
+} atomic_counter_t;
+#define ATOMIC_LINKAGE
+#endif /* defined(HAVE_WORKING_STDATOMIC) */
+
+ATOMIC_LINKAGE void atomic_counter_init(atomic_counter_t *counter);
+ATOMIC_LINKAGE void atomic_counter_destroy(atomic_counter_t *counter);
+ATOMIC_LINKAGE void atomic_counter_add(atomic_counter_t *counter, size_t add);
+ATOMIC_LINKAGE void atomic_counter_sub(atomic_counter_t *counter, size_t sub);
+ATOMIC_LINKAGE size_t atomic_counter_get(atomic_counter_t *counter);
+ATOMIC_LINKAGE size_t atomic_counter_exchange(atomic_counter_t *counter,
+                                              size_t newval);
+#undef ATOMIC_LINKAGE
+
+#ifdef HAVE_WORKING_STDATOMIC
+/** Initialize a new atomic counter with the value 0 */
+static inline void
+atomic_counter_init(atomic_counter_t *counter)
+{
+  atomic_init(&counter->val, 0);
+}
+/** Clean up all resources held by an atomic counter. */
+static inline void
+atomic_counter_destroy(atomic_counter_t *counter)
+{
+  (void)counter;
+}
+/** Add a value to an atomic counter. */
+static inline void
+atomic_counter_add(atomic_counter_t *counter, size_t add)
+{
+  (void) atomic_fetch_add(&counter->val, add);
+}
+/** Subtract a value from an atomic counter. */
+static inline void
+atomic_counter_sub(atomic_counter_t *counter, size_t sub)
+{
+  (void) atomic_fetch_sub(&counter->val, sub);
+}
+/** Return the current value of an atomic counter */
+static inline size_t
+atomic_counter_get(atomic_counter_t *counter)
+{
+  return atomic_load(&counter->val);
+}
+/** Replace the value of an atomic counter; return the old one. */
+static inline size_t
+atomic_counter_exchange(atomic_counter_t *counter, size_t newval)
+{
+  return atomic_exchange(&counter->val, newval);
+}
+
+#else /* !(defined(HAVE_WORKING_STDATOMIC)) */
+#endif /* defined(HAVE_WORKING_STDATOMIC) */
+
+#endif /* !defined(TOR_COMPAT_THREADS_H) */
diff --git a/src/lib/time/.may_include b/src/lib/time/.may_include
new file mode 100644
index 0000000000..2c7e37a836
--- /dev/null
+++ b/src/lib/time/.may_include
@@ -0,0 +1,11 @@
+orconfig.h
+
+lib/cc/*.h
+lib/err/*.h
+lib/intmath/*.h
+lib/log/*.h
+lib/time/*.h
+lib/wallclock/*.h
+
+# For load_windows_system_lib.
+lib/fs/winlib.h
\ No newline at end of file
diff --git a/src/common/compat_time.c b/src/lib/time/compat_time.c
index 52da609db8..98854bad2c 100644
--- a/src/common/compat_time.c
+++ b/src/lib/time/compat_time.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -10,7 +10,15 @@
  **/
 
 #define COMPAT_TIME_PRIVATE
-#include "compat.h"
+#include "lib/time/compat_time.h"
+
+#include "lib/err/torerr.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/intmath/muldiv.h"
+#include "lib/intmath/bits.h"
+#include "lib/fs/winlib.h"
+#include "lib/wallclock/timeval.h"
 
 #ifdef _WIN32
 #include <winsock2.h>
@@ -20,6 +28,9 @@
 #ifdef HAVE_SYS_TYPES_H
 #include <sys/types.h>
 #endif
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
@@ -28,21 +39,15 @@
 /* as fallback implementation for tor_sleep_msec */
 #include <sys/select.h>
 #endif
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 #ifdef __APPLE__
 #include <mach/mach_time.h>
 #endif
 
-#include "torlog.h"
-#include "util.h"
-#include "container.h"
-
-#ifndef HAVE_GETTIMEOFDAY
-#ifdef HAVE_FTIME
-#include <sys/timeb.h>
-#endif
-#endif
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
 
 #ifdef _WIN32
 #undef HAVE_CLOCK_GETTIME
@@ -64,57 +69,9 @@ tor_sleep_msec(int msec)
   select(0, NULL, NULL, NULL, &tv);
 #else
   sleep(CEIL_DIV(msec, 1000));
-#endif
-}
-#endif
-
-/** Set *timeval to the current time of day.  On error, log and terminate.
- * (Same as gettimeofday(timeval,NULL), but never returns -1.)
- */
-void
-tor_gettimeofday(struct timeval *timeval)
-{
-#ifdef _WIN32
-  /* Epoch bias copied from perl: number of units between windows epoch and
-   * Unix epoch. */
-#define EPOCH_BIAS U64_LITERAL(116444736000000000)
-#define UNITS_PER_SEC U64_LITERAL(10000000)
-#define USEC_PER_SEC U64_LITERAL(1000000)
-#define UNITS_PER_USEC U64_LITERAL(10)
-  union {
-    uint64_t ft_64;
-    FILETIME ft_ft;
-  } ft;
-  /* number of 100-nsec units since Jan 1, 1601 */
-  GetSystemTimeAsFileTime(&ft.ft_ft);
-  if (ft.ft_64 < EPOCH_BIAS) {
-    /* LCOV_EXCL_START */
-    log_err(LD_GENERAL,"System time is before 1970; failing.");
-    exit(1);
-    /* LCOV_EXCL_STOP */
-  }
-  ft.ft_64 -= EPOCH_BIAS;
-  timeval->tv_sec = (unsigned) (ft.ft_64 / UNITS_PER_SEC);
-  timeval->tv_usec = (unsigned) ((ft.ft_64 / UNITS_PER_USEC) % USEC_PER_SEC);
-#elif defined(HAVE_GETTIMEOFDAY)
-  if (gettimeofday(timeval, NULL)) {
-    /* LCOV_EXCL_START */
-    log_err(LD_GENERAL,"gettimeofday failed.");
-    /* If gettimeofday dies, we have either given a bad timezone (we didn't),
-       or segfaulted.*/
-    exit(1);
-    /* LCOV_EXCL_STOP */
-  }
-#elif defined(HAVE_FTIME)
-  struct timeb tb;
-  ftime(&tb);
-  timeval->tv_sec = tb.time;
-  timeval->tv_usec = tb.millitm * 1000;
-#else
-#error "No way to get time."
-#endif
-  return;
+#endif /* defined(_WIN32) || ... */
 }
+#endif /* defined(TOR_UNIT_TESTS) */
 
 #define ONE_MILLION ((int64_t) (1000 * 1000))
 #define ONE_BILLION ((int64_t) (1000 * 1000 * 1000))
@@ -187,8 +144,8 @@ monotime_coarse_set_mock_time_nsec(int64_t nsec)
   tor_assert_nonfatal(monotime_mocking_enabled == 1);
   mock_time_nsec_coarse = nsec;
 }
-#endif
-#endif
+#endif /* defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /* "ratchet" functions for monotonic time. */
 
@@ -235,7 +192,7 @@ ratchet_coarse_performance_counter(const int64_t count_raw)
   last_tick_count = count;
   return count;
 }
-#endif
+#endif /* defined(_WIN32) || defined(TOR_UNIT_TESTS) */
 
 #if defined(MONOTIME_USING_GETTIMEOFDAY) || defined(TOR_UNIT_TESTS)
 static struct timeval last_timeofday = { 0, 0 };
@@ -251,7 +208,7 @@ ratchet_timeval(const struct timeval *timeval_raw, struct timeval *out)
 {
   /* must hold lock */
   timeradd(timeval_raw, &timeofday_offset, out);
-  if (PREDICT_UNLIKELY(timercmp(out, &last_timeofday, <))) {
+  if (PREDICT_UNLIKELY(timercmp(out, &last_timeofday, OP_LT))) {
     /* time ran backwards. Instead, declare that no time occurred. */
     timersub(&last_timeofday, timeval_raw, &timeofday_offset);
     memcpy(out, &last_timeofday, sizeof(struct timeval));
@@ -259,7 +216,7 @@ ratchet_timeval(const struct timeval *timeval_raw, struct timeval *out)
     memcpy(&last_timeofday, out, sizeof(struct timeval));
   }
 }
-#endif
+#endif /* defined(MONOTIME_USING_GETTIMEOFDAY) || defined(TOR_UNIT_TESTS) */
 
 #ifdef TOR_UNIT_TESTS
 /** For testing: reset all the ratchets */
@@ -271,7 +228,7 @@ monotime_reset_ratchets_for_testing(void)
   memset(&last_timeofday, 0, sizeof(struct timeval));
   memset(&timeofday_offset, 0, sizeof(struct timeval));
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 #ifdef __APPLE__
 
@@ -279,6 +236,9 @@ monotime_reset_ratchets_for_testing(void)
  * nanoseconds.
  */
 static struct mach_timebase_info mach_time_info;
+static struct mach_timebase_info mach_time_info_msec_cvt;
+static int32_t mach_time_msec_cvt_threshold;
+static int monotime_shift = 0;
 
 static void
 monotime_init_internal(void)
@@ -287,6 +247,26 @@ monotime_init_internal(void)
   int r = mach_timebase_info(&mach_time_info);
   tor_assert(r == 0);
   tor_assert(mach_time_info.denom != 0);
+
+  {
+    // approximate only.
+    uint64_t ns_per_tick = mach_time_info.numer / mach_time_info.denom;
+    uint64_t ms_per_tick = ns_per_tick * ONE_MILLION;
+    // requires that tor_log2(0) == 0.
+    monotime_shift = tor_log2(ms_per_tick);
+  }
+  {
+    // For converting ticks to milliseconds in a 32-bit-friendly way, we
+    // will first right-shift by 20, and then multiply by 2048/1953, since
+    // (1<<20) * 1953/2048 is about 1e6.  We precompute a new numerator and
+    // denominator here to avoid multiple multiplies.
+    mach_time_info_msec_cvt.numer = mach_time_info.numer * 2048;
+    mach_time_info_msec_cvt.denom = mach_time_info.denom * 1953;
+    // For any value above this amount, we should divide before multiplying,
+    // to avoid overflow.  For a value below this, we should multiply
+    // before dividing, to improve accuracy.
+    mach_time_msec_cvt_threshold = INT32_MAX / mach_time_info_msec_cvt.numer;
+  }
 }
 
 /**
@@ -301,10 +281,25 @@ monotime_get(monotime_t *out)
       / mach_time_info.numer;
     return;
   }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
   out->abstime_ = mach_absolute_time();
 }
 
+#if defined(HAVE_MACH_APPROXIMATE_TIME)
+void
+monotime_coarse_get(monotime_coarse_t *out)
+{
+#ifdef TOR_UNIT_TESTS
+  if (monotime_mocking_enabled) {
+    out->abstime_ = (mock_time_nsec_coarse * mach_time_info.denom)
+      / mach_time_info.numer;
+    return;
+  }
+#endif /* defined(TOR_UNIT_TESTS) */
+  out->abstime_ = mach_approximate_time();
+}
+#endif
+
 /**
  * Return the number of nanoseconds between <b>start</b> and <b>end</b>.
  */
@@ -321,6 +316,47 @@ monotime_diff_nsec(const monotime_t *start,
   return diff_nsec;
 }
 
+int32_t
+monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
+                             const monotime_coarse_t *end)
+{
+  if (BUG(mach_time_info.denom == 0)) {
+    monotime_init();
+  }
+  const int64_t diff_ticks = end->abstime_ - start->abstime_;
+
+  /* We already require in di_ops.c that right-shift performs a sign-extend. */
+  const int32_t diff_microticks = (int32_t)(diff_ticks >> 20);
+
+  if (diff_microticks >= mach_time_msec_cvt_threshold) {
+    return (diff_microticks / mach_time_info_msec_cvt.denom) *
+      mach_time_info_msec_cvt.numer;
+  } else {
+    return (diff_microticks * mach_time_info_msec_cvt.numer) /
+      mach_time_info_msec_cvt.denom;
+  }
+}
+
+uint32_t
+monotime_coarse_to_stamp(const monotime_coarse_t *t)
+{
+  return (uint32_t)(t->abstime_ >> monotime_shift);
+}
+
+int
+monotime_is_zero(const monotime_t *val)
+{
+  return val->abstime_ == 0;
+}
+
+void
+monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec)
+{
+  const uint64_t nsec = msec * ONE_MILLION;
+  const uint64_t ticks = (nsec * mach_time_info.denom) / mach_time_info.numer;
+  out->abstime_ = val->abstime_ + ticks;
+}
+
 /* end of "__APPLE__" */
 #elif defined(HAVE_CLOCK_GETTIME)
 
@@ -332,7 +368,7 @@ monotime_diff_nsec(const monotime_t *start,
  * an old Linux kernel. In that case, we will fall back to CLOCK_MONOTONIC.
  */
 static int clock_monotonic_coarse = CLOCK_MONOTONIC_COARSE;
-#endif
+#endif /* defined(CLOCK_MONOTONIC_COARSE) */
 
 static void
 monotime_init_internal(void)
@@ -344,7 +380,7 @@ monotime_init_internal(void)
              "falling back to CLOCK_MONOTONIC.", strerror(errno));
     clock_monotonic_coarse = CLOCK_MONOTONIC;
   }
-#endif
+#endif /* defined(CLOCK_MONOTONIC_COARSE) */
 }
 
 void
@@ -356,7 +392,7 @@ monotime_get(monotime_t *out)
     out->ts_.tv_nsec = (int) (mock_time_nsec % ONE_BILLION);
     return;
   }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
   int r = clock_gettime(CLOCK_MONOTONIC, &out->ts_);
   tor_assert(r == 0);
 }
@@ -371,7 +407,7 @@ monotime_coarse_get(monotime_coarse_t *out)
     out->ts_.tv_nsec = (int) (mock_time_nsec_coarse % ONE_BILLION);
     return;
   }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
   int r = clock_gettime(clock_monotonic_coarse, &out->ts_);
   if (PREDICT_UNLIKELY(r < 0) &&
       errno == EINVAL &&
@@ -386,7 +422,7 @@ monotime_coarse_get(monotime_coarse_t *out)
 
   tor_assert(r == 0);
 }
-#endif
+#endif /* defined(CLOCK_MONOTONIC_COARSE) */
 
 int64_t
 monotime_diff_nsec(const monotime_t *start,
@@ -399,6 +435,46 @@ monotime_diff_nsec(const monotime_t *start,
   return diff_nsec;
 }
 
+int32_t
+monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
+                             const monotime_coarse_t *end)
+{
+  const int32_t diff_sec = (int32_t)(end->ts_.tv_sec - start->ts_.tv_sec);
+  const int32_t diff_nsec = (int32_t)(end->ts_.tv_nsec - start->ts_.tv_nsec);
+  return diff_sec * 1000 + diff_nsec / ONE_MILLION;
+}
+
+/* This value is ONE_BILLION >> 20. */
+static const uint32_t STAMP_TICKS_PER_SECOND = 953;
+
+uint32_t
+monotime_coarse_to_stamp(const monotime_coarse_t *t)
+{
+  uint32_t nsec = (uint32_t)t->ts_.tv_nsec;
+  uint32_t sec = (uint32_t)t->ts_.tv_sec;
+
+  return (sec * STAMP_TICKS_PER_SECOND) + (nsec >> 20);
+}
+
+int
+monotime_is_zero(const monotime_t *val)
+{
+  return val->ts_.tv_sec == 0 && val->ts_.tv_nsec == 0;
+}
+
+void
+monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec)
+{
+  const uint32_t sec = msec / 1000;
+  const uint32_t msec_remainder = msec % 1000;
+  out->ts_.tv_sec = val->ts_.tv_sec + sec;
+  out->ts_.tv_nsec = val->ts_.tv_nsec + (msec_remainder * ONE_MILLION);
+  if (out->ts_.tv_nsec > ONE_BILLION) {
+    out->ts_.tv_nsec -= ONE_BILLION;
+    out->ts_.tv_sec += 1;
+  }
+}
+
 /* end of "HAVE_CLOCK_GETTIME" */
 #elif defined (_WIN32)
 
@@ -462,7 +538,7 @@ monotime_get(monotime_t *out)
       / nsec_per_tick_numer;
     return;
   }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
   /* Alas, QueryPerformanceCounter is not always monotonic: see bug list at
 
@@ -486,7 +562,7 @@ monotime_coarse_get(monotime_coarse_t *out)
     out->tick_count_ = mock_time_nsec_coarse / ONE_MILLION;
     return;
   }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
   if (GetTickCount64_fn) {
     out->tick_count_ = (int64_t)GetTickCount64_fn();
@@ -517,6 +593,13 @@ monotime_coarse_diff_msec(const monotime_coarse_t *start,
   return diff_ticks;
 }
 
+int32_t
+monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
+                             const monotime_coarse_t *end)
+{
+  return (int32_t)monotime_coarse_diff_msec(start, end);
+}
+
 int64_t
 monotime_coarse_diff_usec(const monotime_coarse_t *start,
                           const monotime_coarse_t *end)
@@ -531,6 +614,41 @@ monotime_coarse_diff_nsec(const monotime_coarse_t *start,
   return monotime_coarse_diff_msec(start, end) * ONE_MILLION;
 }
 
+static const uint32_t STAMP_TICKS_PER_SECOND = 1000;
+
+uint32_t
+monotime_coarse_to_stamp(const monotime_coarse_t *t)
+{
+  return (uint32_t) t->tick_count_;
+}
+
+int
+monotime_is_zero(const monotime_t *val)
+{
+  return val->pcount_ == 0;
+}
+
+int
+monotime_coarse_is_zero(const monotime_coarse_t *val)
+{
+  return val->tick_count_ == 0;
+}
+
+void
+monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec)
+{
+  const uint64_t nsec = msec * ONE_MILLION;
+  const uint64_t ticks = (nsec * nsec_per_tick_denom) / nsec_per_tick_numer;
+  out->pcount_ = val->pcount_ + ticks;
+}
+
+void
+monotime_coarse_add_msec(monotime_coarse_t *out, const monotime_coarse_t *val,
+                         uint32_t msec)
+{
+  out->tick_count_ = val->tick_count_ + msec;
+}
+
 /* end of "_WIN32" */
 #elif defined(MONOTIME_USING_GETTIMEOFDAY)
 
@@ -567,10 +685,49 @@ monotime_diff_nsec(const monotime_t *start,
   return (diff.tv_sec * ONE_BILLION + diff.tv_usec * 1000);
 }
 
+int32_t
+monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
+                             const monotime_coarse_t *end)
+{
+  struct timeval diff;
+  timersub(&end->tv_, &start->tv_, &diff);
+  return diff.tv_sec * 1000 + diff.tv_usec / 1000;
+}
+
+/* This value is ONE_MILLION >> 10. */
+static const uint32_t STAMP_TICKS_PER_SECOND = 976;
+
+uint32_t
+monotime_coarse_to_stamp(const monotime_coarse_t *t)
+{
+  const uint32_t usec = (uint32_t)t->tv_.tv_usec;
+  const uint32_t sec = (uint32_t)t->tv_.tv_sec;
+  return (sec * STAMP_TICKS_PER_SECOND) | (nsec >> 10);
+}
+
+int
+monotime_is_zero(const monotime_t *val)
+{
+  return val->tv_.tv_sec == 0 && val->tv_.tv_usec == 0;
+}
+
+void
+monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec)
+{
+  const uint32_t sec = msec / 1000;
+  const uint32_t msec_remainder = msec % 1000;
+  out->tv_.tv_sec = val->tv_.tv_sec + sec;
+  out->tv_.tv_usec = val->tv_.tv_nsec + (msec_remainder * 1000);
+  if (out->tv_.tv_usec > ONE_MILLION) {
+    out->tv_.tv_usec -= ONE_MILLION;
+    out->tv_.tv_sec += 1;
+  }
+}
+
 /* end of "MONOTIME_USING_GETTIMEOFDAY" */
 #else
 #error "No way to implement monotonic timers."
-#endif
+#endif /* defined(__APPLE__) || ... */
 
 /**
  * Initialize the monotonic timer subsystem. Must be called before any
@@ -589,6 +746,19 @@ monotime_init(void)
   }
 }
 
+void
+monotime_zero(monotime_t *out)
+{
+  memset(out, 0, sizeof(*out));
+}
+#ifdef MONOTIME_COARSE_TYPE_IS_DIFFERENT
+void
+monotime_coarse_zero(monotime_coarse_t *out)
+{
+  memset(out, 0, sizeof(*out));
+}
+#endif
+
 int64_t
 monotime_diff_usec(const monotime_t *start,
                    const monotime_t *end)
@@ -653,4 +823,47 @@ monotime_coarse_absolute_msec(void)
 {
   return monotime_coarse_absolute_nsec() / ONE_MILLION;
 }
+#else
+#define initialized_at_coarse initialized_at
+#endif /* defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */
+
+/**
+ * Return the current time "stamp" as described by monotime_coarse_to_stamp.
+ */
+uint32_t
+monotime_coarse_get_stamp(void)
+{
+  monotime_coarse_t now;
+  monotime_coarse_get(&now);
+  return monotime_coarse_to_stamp(&now);
+}
+
+#ifdef __APPLE__
+uint64_t
+monotime_coarse_stamp_units_to_approx_msec(uint64_t units)
+{
+  /* Recover as much precision as we can. */
+  uint64_t abstime_diff = (units << monotime_shift);
+  return (abstime_diff * mach_time_info.numer) /
+    (mach_time_info.denom * ONE_MILLION);
+}
+uint64_t
+monotime_msec_to_approx_coarse_stamp_units(uint64_t msec)
+{
+  uint64_t abstime_val =
+    (((uint64_t)msec) * ONE_MILLION * mach_time_info.denom) /
+    mach_time_info.numer;
+  return abstime_val >> monotime_shift;
+}
+#else
+uint64_t
+monotime_coarse_stamp_units_to_approx_msec(uint64_t units)
+{
+  return (units * 1000) / STAMP_TICKS_PER_SECOND;
+}
+uint64_t
+monotime_msec_to_approx_coarse_stamp_units(uint64_t msec)
+{
+  return (msec * STAMP_TICKS_PER_SECOND) / 1000;
+}
 #endif
diff --git a/src/common/compat_time.h b/src/lib/time/compat_time.h
index 2262446e57..480d426ac7 100644
--- a/src/common/compat_time.h
+++ b/src/lib/time/compat_time.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,6 +19,10 @@
 #define TOR_COMPAT_TIME_H
 
 #include "orconfig.h"
+#include "lib/cc/torint.h"
+
+#include "lib/wallclock/tor_gettimeofday.h"
+
 #ifdef _WIN32
 #undef HAVE_CLOCK_GETTIME
 #endif
@@ -28,13 +32,13 @@
 #include <time.h>
 #endif
 
-#if !defined(HAVE_GETTIMEOFDAY) && !defined(HAVE_STRUCT_TIMEVAL_TV_SEC)
+#if !defined(HAVE_STRUCT_TIMEVAL_TV_SEC)
 /** Implementation of timeval for platforms that don't have it. */
 struct timeval {
   time_t tv_sec;
   unsigned int tv_usec;
 };
-#endif
+#endif /* !defined(HAVE_STRUCT_TIMEVAL_TV_SEC) */
 
 /** Represents a monotonic timer in a platform-dependent way. */
 typedef struct monotime_t {
@@ -51,10 +55,11 @@ typedef struct monotime_t {
 #define MONOTIME_USING_GETTIMEOFDAY
   /* Otherwise, we will be stuck using gettimeofday. */
   struct timeval tv_;
-#endif
+#endif /* defined(__APPLE__) || ... */
 } monotime_t;
 
-#if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC_COARSE)
+#if defined(CLOCK_MONOTONIC_COARSE) && \
+  defined(HAVE_CLOCK_GETTIME)
 #define MONOTIME_COARSE_FN_IS_DIFFERENT
 #define monotime_coarse_t monotime_t
 #elif defined(_WIN32)
@@ -64,9 +69,12 @@ typedef struct monotime_t {
 typedef struct monotime_coarse_t {
   uint64_t tick_count_;
 } monotime_coarse_t;
+#elif defined(__APPLE__) && defined(HAVE_MACH_APPROXIMATE_TIME)
+#define MONOTIME_COARSE_FN_IS_DIFFERENT
+#define monotime_coarse_t monotime_t
 #else
 #define monotime_coarse_t monotime_t
-#endif
+#endif /* defined(CLOCK_MONOTONIC_COARSE) && ... || ... */
 
 /**
  * Initialize the timing subsystem. This function is idempotent.
@@ -101,6 +109,21 @@ uint64_t monotime_absolute_usec(void);
  */
 uint64_t monotime_absolute_msec(void);
 
+/**
+ * Set <b>out</b> to zero.
+ */
+void monotime_zero(monotime_t *out);
+/**
+ * Return true iff <b>out</b> is zero
+ */
+int monotime_is_zero(const monotime_t *out);
+
+/**
+ * Set <b>out</b> to N milliseconds after <b>val</b>.
+ */
+/* XXXX We should add a more generic function here if we ever need to */
+void monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec);
+
 #if defined(MONOTIME_COARSE_FN_IS_DIFFERENT)
 /**
  * Set <b>out</b> to the current coarse time.
@@ -109,12 +132,30 @@ void monotime_coarse_get(monotime_coarse_t *out);
 uint64_t monotime_coarse_absolute_nsec(void);
 uint64_t monotime_coarse_absolute_usec(void);
 uint64_t monotime_coarse_absolute_msec(void);
-#else
+#else /* !(defined(MONOTIME_COARSE_FN_IS_DIFFERENT)) */
 #define monotime_coarse_get monotime_get
 #define monotime_coarse_absolute_nsec monotime_absolute_nsec
 #define monotime_coarse_absolute_usec monotime_absolute_usec
 #define monotime_coarse_absolute_msec monotime_absolute_msec
-#endif
+#endif /* defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */
+
+/**
+ * Return a "timestamp" approximation for a coarse monotonic timer.
+ * This timestamp is meant to be fast to calculate and easy to
+ * compare, and have a unit of something roughly around 1 msec.
+ *
+ * It will wrap over from time to time.
+ *
+ * It has no defined zero point.
+ */
+uint32_t monotime_coarse_to_stamp(const monotime_coarse_t *t);
+/**
+ * Convert a difference, expressed in the units of monotime_coarse_to_stamp,
+ * into an approximate number of milliseconds.
+ */
+uint64_t monotime_coarse_stamp_units_to_approx_msec(uint64_t units);
+uint64_t monotime_msec_to_approx_coarse_stamp_units(uint64_t msec);
+uint32_t monotime_coarse_get_stamp(void);
 
 #if defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT)
 int64_t monotime_coarse_diff_nsec(const monotime_coarse_t *start,
@@ -123,13 +164,46 @@ int64_t monotime_coarse_diff_usec(const monotime_coarse_t *start,
     const monotime_coarse_t *end);
 int64_t monotime_coarse_diff_msec(const monotime_coarse_t *start,
     const monotime_coarse_t *end);
-#else
+void monotime_coarse_zero(monotime_coarse_t *out);
+int monotime_coarse_is_zero(const monotime_coarse_t *val);
+void monotime_coarse_add_msec(monotime_coarse_t *out,
+                              const monotime_coarse_t *val, uint32_t msec);
+#else /* !(defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT)) */
 #define monotime_coarse_diff_nsec monotime_diff_nsec
 #define monotime_coarse_diff_usec monotime_diff_usec
 #define monotime_coarse_diff_msec monotime_diff_msec
-#endif
+#define monotime_coarse_zero monotime_zero
+#define monotime_coarse_is_zero monotime_is_zero
+#define monotime_coarse_add_msec monotime_add_msec
+#endif /* defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */
 
-void tor_gettimeofday(struct timeval *timeval);
+/**
+ * As monotime_coarse_diff_msec, but avoid 64-bit division.
+ *
+ * Requires that the difference fit into an int32_t; not for use with
+ * large time differences.
+ */
+int32_t monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
+                                     const monotime_coarse_t *end);
+
+/**
+ * As monotime_coarse_diff_msec, but avoid 64-bit division if it is expensive.
+ *
+ * Requires that the difference fit into an int32_t; not for use with
+ * large time differences.
+ */
+static inline int32_t
+monotime_coarse_diff_msec32(const monotime_coarse_t *start,
+                            const monotime_coarse_t *end)
+{
+#if SIZEOF_VOID_P == 8
+  // on a 64-bit platform, let's assume 64/64 division is cheap.
+  return (int32_t) monotime_coarse_diff_msec(start, end);
+#else
+#define USING_32BIT_MSEC_HACK
+  return monotime_coarse_diff_msec32_(start, end);
+#endif
+}
 
 #ifdef TOR_UNIT_TESTS
 void tor_sleep_msec(int msec);
@@ -142,7 +216,7 @@ void monotime_coarse_set_mock_time_nsec(int64_t);
 #else
 #define monotime_coarse_set_mock_time_nsec monotime_set_mock_time_nsec
 #endif
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 #ifdef COMPAT_TIME_PRIVATE
 #if defined(_WIN32) || defined(TOR_UNIT_TESTS)
@@ -156,7 +230,6 @@ STATIC void ratchet_timeval(const struct timeval *timeval_raw,
 #ifdef TOR_UNIT_TESTS
 void monotime_reset_ratchets_for_testing(void);
 #endif
-#endif
-
-#endif
+#endif /* defined(COMPAT_TIME_PRIVATE) */
 
+#endif /* !defined(TOR_COMPAT_TIME_H) */
diff --git a/src/lib/time/include.am b/src/lib/time/include.am
new file mode 100644
index 0000000000..a3f93a3744
--- /dev/null
+++ b/src/lib/time/include.am
@@ -0,0 +1,19 @@
+
+noinst_LIBRARIES += src/lib/libtor-time.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-time-testing.a
+endif
+
+src_lib_libtor_time_a_SOURCES =	\
+		src/lib/time/compat_time.c	\
+		src/lib/time/tvdiff.c
+
+src_lib_libtor_time_testing_a_SOURCES = \
+	$(src_lib_libtor_time_a_SOURCES)
+src_lib_libtor_time_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_time_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=				\
+		src/lib/time/compat_time.h	\
+		src/lib/time/tvdiff.h
diff --git a/src/lib/time/tvdiff.c b/src/lib/time/tvdiff.c
new file mode 100644
index 0000000000..a87d0d96dc
--- /dev/null
+++ b/src/lib/time/tvdiff.c
@@ -0,0 +1,189 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file tvdiff.c
+ * \brief Compute the difference between timevals, in various units.
+ **/
+
+#include "lib/time/tvdiff.h"
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/log/log.h"
+
+#ifdef _WIN32
+#include <winsock2.h>
+#endif
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+
+#define TOR_USEC_PER_SEC 1000000
+
+/** Return the difference between start->tv_sec and end->tv_sec.
+ * Returns INT64_MAX on overflow and underflow.
+ */
+static int64_t
+tv_secdiff_impl(const struct timeval *start, const struct timeval *end)
+{
+  const int64_t s = (int64_t)start->tv_sec;
+  const int64_t e = (int64_t)end->tv_sec;
+
+  /* This may not be the most efficient way of implemeting this check,
+   * but it's easy to see that it's correct and doesn't overflow */
+
+  if (s > 0 && e < INT64_MIN + s) {
+    /* s is positive: equivalent to e - s < INT64_MIN, but without any
+     * overflow */
+    return INT64_MAX;
+  } else if (s < 0 && e > INT64_MAX + s) {
+    /* s is negative: equivalent to e - s > INT64_MAX, but without any
+     * overflow */
+    return INT64_MAX;
+  }
+
+  return e - s;
+}
+
+/** Return the number of microseconds elapsed between *start and *end.
+ * Returns LONG_MAX on overflow and underflow.
+ */
+long
+tv_udiff(const struct timeval *start, const struct timeval *end)
+{
+  /* Sanity check tv_usec */
+  if (start->tv_usec > TOR_USEC_PER_SEC || start->tv_usec < 0) {
+    log_warn(LD_GENERAL, "comparing times on microsecond detail with bad "
+             "start tv_usec: %"PRId64 " microseconds",
+             (int64_t)start->tv_usec);
+    return LONG_MAX;
+  }
+
+  if (end->tv_usec > TOR_USEC_PER_SEC || end->tv_usec < 0) {
+    log_warn(LD_GENERAL, "comparing times on microsecond detail with bad "
+             "end tv_usec: %"PRId64 " microseconds",
+             (int64_t)end->tv_usec);
+    return LONG_MAX;
+  }
+
+  /* Some BSDs have struct timeval.tv_sec 64-bit, but time_t (and long) 32-bit
+   */
+  int64_t udiff;
+  const int64_t secdiff = tv_secdiff_impl(start, end);
+
+  /* end->tv_usec - start->tv_usec can be up to 1 second either way */
+  if (secdiff > (int64_t)(LONG_MAX/1000000 - 1) ||
+      secdiff < (int64_t)(LONG_MIN/1000000 + 1)) {
+    log_warn(LD_GENERAL, "comparing times on microsecond detail too far "
+             "apart: %"PRId64 " seconds", (secdiff));
+    return LONG_MAX;
+  }
+
+  /* we'll never get an overflow here, because we check that both usecs are
+   * between 0 and TV_USEC_PER_SEC. */
+  udiff = secdiff*1000000 + ((int64_t)end->tv_usec - (int64_t)start->tv_usec);
+
+  /* Some compilers are smart enough to work out this is a no-op on L64 */
+#if SIZEOF_LONG < 8
+  if (udiff > (int64_t)LONG_MAX || udiff < (int64_t)LONG_MIN) {
+    return LONG_MAX;
+  }
+#endif
+
+  return (long)udiff;
+}
+
+/** Return the number of milliseconds elapsed between *start and *end.
+ * If the tv_usec difference is 500, rounds away from zero.
+ * Returns LONG_MAX on overflow and underflow.
+ */
+long
+tv_mdiff(const struct timeval *start, const struct timeval *end)
+{
+  /* Sanity check tv_usec */
+  if (start->tv_usec > TOR_USEC_PER_SEC || start->tv_usec < 0) {
+    log_warn(LD_GENERAL, "comparing times on millisecond detail with bad "
+             "start tv_usec: %"PRId64 " microseconds",
+             (int64_t)start->tv_usec);
+    return LONG_MAX;
+  }
+
+  if (end->tv_usec > TOR_USEC_PER_SEC || end->tv_usec < 0) {
+    log_warn(LD_GENERAL, "comparing times on millisecond detail with bad "
+             "end tv_usec: %"PRId64 " microseconds",
+             (int64_t)end->tv_usec);
+    return LONG_MAX;
+  }
+
+  /* Some BSDs have struct timeval.tv_sec 64-bit, but time_t (and long) 32-bit
+   */
+  int64_t mdiff;
+  const int64_t secdiff = tv_secdiff_impl(start, end);
+
+  /* end->tv_usec - start->tv_usec can be up to 1 second either way, but the
+   * mdiff calculation may add another temporary second for rounding.
+   * Whether this actually causes overflow depends on the compiler's constant
+   * folding and order of operations. */
+  if (secdiff > (int64_t)(LONG_MAX/1000 - 2) ||
+      secdiff < (int64_t)(LONG_MIN/1000 + 1)) {
+    log_warn(LD_GENERAL, "comparing times on millisecond detail too far "
+             "apart: %"PRId64 " seconds", (int64_t)secdiff);
+    return LONG_MAX;
+  }
+
+  /* Subtract and round */
+  mdiff = secdiff*1000 +
+      /* We add a million usec here to ensure that the result is positive,
+       * so that the round-towards-zero behavior of the division will give
+       * the right result for rounding to the nearest msec. Later we subtract
+       * 1000 in order to get the correct result.
+       * We'll never get an overflow here, because we check that both usecs are
+       * between 0 and TV_USEC_PER_SEC. */
+      ((int64_t)end->tv_usec - (int64_t)start->tv_usec + 500 + 1000000) / 1000
+      - 1000;
+
+  /* Some compilers are smart enough to work out this is a no-op on L64 */
+#if SIZEOF_LONG < 8
+  if (mdiff > (int64_t)LONG_MAX || mdiff < (int64_t)LONG_MIN) {
+    return LONG_MAX;
+  }
+#endif
+
+  return (long)mdiff;
+}
+
+/**
+ * Converts timeval to milliseconds.
+ */
+int64_t
+tv_to_msec(const struct timeval *tv)
+{
+  int64_t conv = ((int64_t)tv->tv_sec)*1000L;
+  /* Round ghetto-style */
+  conv += ((int64_t)tv->tv_usec+500)/1000L;
+  return conv;
+}
+
+/**
+ * Return duration in seconds between time_t values
+ * <b>t1</b> and <b>t2</b> iff <b>t1</b> is numerically
+ * less or equal than <b>t2</b>. Otherwise, return TIME_MAX.
+ *
+ * This provides a safe way to compute difference between
+ * two UNIX timestamps (<b>t2</b> can be assumed by calling
+ * code to be later than <b>t1</b>) or two durations measured
+ * in seconds (<b>t2</b> can be assumed to be longer than
+ * <b>t1</b>). Calling code is expected to check for TIME_MAX
+ * return value and interpret that as error condition.
+ */
+time_t
+time_diff(const time_t t1, const time_t t2)
+{
+  if (t1 <= t2)
+    return t2 - t1;
+
+  return TIME_MAX;
+}
+
diff --git a/src/lib/time/tvdiff.h b/src/lib/time/tvdiff.h
new file mode 100644
index 0000000000..724af1528a
--- /dev/null
+++ b/src/lib/time/tvdiff.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file tvdiff.h
+ * \brief Header for tvdiff.c
+ **/
+
+#ifndef TOR_TVDIFF_H
+#define TOR_TVDIFF_H
+
+#include "lib/cc/torint.h"
+struct timeval;
+
+long tv_udiff(const struct timeval *start, const struct timeval *end);
+long tv_mdiff(const struct timeval *start, const struct timeval *end);
+int64_t tv_to_msec(const struct timeval *tv);
+
+time_t time_diff(const time_t from, const time_t to);
+
+#endif
diff --git a/src/lib/tls/.may_include b/src/lib/tls/.may_include
new file mode 100644
index 0000000000..2840e590b8
--- /dev/null
+++ b/src/lib/tls/.may_include
@@ -0,0 +1,17 @@
+orconfig.h
+
+lib/arch/*.h
+lib/cc/*.h
+lib/container/*.h
+lib/crypt_ops/*.h
+lib/ctime/*.h
+lib/encoding/*.h
+lib/intmath/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/net/*.h
+lib/string/*.h
+lib/testsupport/testsupport.h
+lib/tls/*.h
+
+ciphers.inc
diff --git a/src/lib/tls/buffers_tls.c b/src/lib/tls/buffers_tls.c
new file mode 100644
index 0000000000..e92cb9163f
--- /dev/null
+++ b/src/lib/tls/buffers_tls.c
@@ -0,0 +1,182 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file buffers_tls.c
+ * \brief Read and write data on a tor_tls_t connection from a buf_t object.
+ **/
+
+#define BUFFERS_PRIVATE
+#include "orconfig.h"
+#include <stddef.h>
+#include "lib/container/buffers.h"
+#include "lib/tls/buffers_tls.h"
+#include "lib/cc/torint.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/tls/tortls.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/** As read_to_chunk(), but return (negative) error code on error, blocking,
+ * or TLS, and the number of bytes read otherwise. */
+static inline int
+read_to_chunk_tls(buf_t *buf, chunk_t *chunk, tor_tls_t *tls,
+                  size_t at_most)
+{
+  int read_result;
+
+  tor_assert(CHUNK_REMAINING_CAPACITY(chunk) >= at_most);
+  read_result = tor_tls_read(tls, CHUNK_WRITE_PTR(chunk), at_most);
+  if (read_result < 0)
+    return read_result;
+  buf->datalen += read_result;
+  chunk->datalen += read_result;
+  return read_result;
+}
+
+/** As read_to_buf, but reads from a TLS connection, and returns a TLS
+ * status value rather than the number of bytes read.
+ *
+ * Using TLS on OR connections complicates matters in two ways.
+ *
+ * First, a TLS stream has its own read buffer independent of the
+ * connection's read buffer.  (TLS needs to read an entire frame from
+ * the network before it can decrypt any data.  Thus, trying to read 1
+ * byte from TLS can require that several KB be read from the network
+ * and decrypted.  The extra data is stored in TLS's decrypt buffer.)
+ * Because the data hasn't been read by Tor (it's still inside the TLS),
+ * this means that sometimes a connection "has stuff to read" even when
+ * poll() didn't return POLLIN. The tor_tls_get_pending_bytes function is
+ * used in connection.c to detect TLS objects with non-empty internal
+ * buffers and read from them again.
+ *
+ * Second, the TLS stream's events do not correspond directly to network
+ * events: sometimes, before a TLS stream can read, the network must be
+ * ready to write -- or vice versa.
+ */
+int
+buf_read_from_tls(buf_t *buf, tor_tls_t *tls, size_t at_most)
+{
+  int r = 0;
+  size_t total_read = 0;
+
+  check_no_tls_errors();
+
+  IF_BUG_ONCE(buf->datalen >= INT_MAX)
+    return -1;
+  IF_BUG_ONCE(buf->datalen >= INT_MAX - at_most)
+    return -1;
+
+  while (at_most > total_read) {
+    size_t readlen = at_most - total_read;
+    chunk_t *chunk;
+    if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) {
+      chunk = buf_add_chunk_with_capacity(buf, at_most, 1);
+      if (readlen > chunk->memlen)
+        readlen = chunk->memlen;
+    } else {
+      size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail);
+      chunk = buf->tail;
+      if (cap < readlen)
+        readlen = cap;
+    }
+
+    r = read_to_chunk_tls(buf, chunk, tls, readlen);
+    if (r < 0)
+      return r; /* Error */
+    tor_assert(total_read+r < INT_MAX);
+    total_read += r;
+    if ((size_t)r < readlen) /* eof, block, or no more to read. */
+      break;
+  }
+  return (int)total_read;
+}
+
+/** Helper for buf_flush_to_tls(): try to write <b>sz</b> bytes from chunk
+ * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>.  (Tries to write
+ * more if there is a forced pending write size.)  On success, deduct the
+ * bytes written from *<b>buf_flushlen</b>.  Return the number of bytes
+ * written on success, and a TOR_TLS error code on failure or blocking.
+ */
+static inline int
+flush_chunk_tls(tor_tls_t *tls, buf_t *buf, chunk_t *chunk,
+                size_t sz, size_t *buf_flushlen)
+{
+  int r;
+  size_t forced;
+  char *data;
+
+  forced = tor_tls_get_forced_write_size(tls);
+  if (forced > sz)
+    sz = forced;
+  if (chunk) {
+    data = chunk->data;
+    tor_assert(sz <= chunk->datalen);
+  } else {
+    data = NULL;
+    tor_assert(sz == 0);
+  }
+  r = tor_tls_write(tls, data, sz);
+  if (r < 0)
+    return r;
+  if (*buf_flushlen > (size_t)r)
+    *buf_flushlen -= r;
+  else
+    *buf_flushlen = 0;
+  buf_drain(buf, r);
+  log_debug(LD_NET,"flushed %d bytes, %d ready to flush, %d remain.",
+            r,(int)*buf_flushlen,(int)buf->datalen);
+  return r;
+}
+
+/** As buf_flush_to_socket(), but writes data to a TLS connection.  Can write
+ * more than <b>flushlen</b> bytes.
+ */
+int
+buf_flush_to_tls(buf_t *buf, tor_tls_t *tls, size_t flushlen,
+              size_t *buf_flushlen)
+{
+  int r;
+  size_t flushed = 0;
+  ssize_t sz;
+  tor_assert(buf_flushlen);
+  if (BUG(*buf_flushlen > buf->datalen)) {
+    *buf_flushlen = buf->datalen;
+  }
+  if (BUG(flushlen > *buf_flushlen)) {
+    flushlen = *buf_flushlen;
+  }
+  sz = (ssize_t) flushlen;
+
+  /* we want to let tls write even if flushlen is zero, because it might
+   * have a partial record pending */
+  check_no_tls_errors();
+
+  do {
+    size_t flushlen0;
+    if (buf->head) {
+      if ((ssize_t)buf->head->datalen >= sz)
+        flushlen0 = sz;
+      else
+        flushlen0 = buf->head->datalen;
+    } else {
+      flushlen0 = 0;
+    }
+
+    r = flush_chunk_tls(tls, buf, buf->head, flushlen0, buf_flushlen);
+    if (r < 0)
+      return r;
+    flushed += r;
+    sz -= r;
+    if (r == 0) /* Can't flush any more now. */
+      break;
+  } while (sz > 0);
+  tor_assert(flushed < INT_MAX);
+  return (int)flushed;
+}
diff --git a/src/lib/tls/buffers_tls.h b/src/lib/tls/buffers_tls.h
new file mode 100644
index 0000000000..65788c3f34
--- /dev/null
+++ b/src/lib/tls/buffers_tls.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file buffers_tls.h
+ * \brief Header for buffers_tls.c
+ **/
+
+#ifndef TOR_BUFFERS_TLS_H
+#define TOR_BUFFERS_TLS_H
+
+struct buf_t;
+struct tor_tls_t;
+
+int buf_read_from_tls(struct buf_t *buf,
+                      struct tor_tls_t *tls, size_t at_most);
+int buf_flush_to_tls(struct buf_t *buf, struct tor_tls_t *tls,
+                     size_t sz, size_t *buf_flushlen);
+
+#endif /* !defined(TOR_BUFFERS_TLS_H) */
diff --git a/src/common/ciphers.inc b/src/lib/tls/ciphers.inc
index 23f5fd2da4..0084b3e325 100644
--- a/src/common/ciphers.inc
+++ b/src/lib/tls/ciphers.inc
@@ -33,6 +33,26 @@
 #else
    XCIPHER(0xc02f, TLS1_TXT_ECDHE_RSA_WITH_AES_128_GCM_SHA256)
 #endif
+#ifdef TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+    CIPHER(0xcca9, TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305)
+#else
+   XCIPHER(0xcca9, TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305)
+#endif
+#ifdef TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305
+    CIPHER(0xcca8, TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305)
+#else
+   XCIPHER(0xcca8, TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305)
+#endif
+#ifdef TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+    CIPHER(0xc02c, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384)
+#else
+   XCIPHER(0xc02c, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384)
+#endif
+#ifdef TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+    CIPHER(0xc030, TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384)
+#else
+   XCIPHER(0xc030, TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384)
+#endif
 #ifdef TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_CBC_SHA
     CIPHER(0xc00a, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_CBC_SHA)
 #else
@@ -53,88 +73,28 @@
 #else
    XCIPHER(0xc014, TLS1_TXT_ECDHE_RSA_WITH_AES_256_CBC_SHA)
 #endif
-#ifdef TLS1_TXT_ECDHE_RSA_WITH_DES_192_CBC3_SHA
-    CIPHER(0xc012, TLS1_TXT_ECDHE_RSA_WITH_DES_192_CBC3_SHA)
-#else
-   XCIPHER(0xc012, TLS1_TXT_ECDHE_RSA_WITH_DES_192_CBC3_SHA)
-#endif
-#ifdef TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA
-    CIPHER(0xc007, TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA)
-#else
-   XCIPHER(0xc007, TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA)
-#endif
-#ifdef TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA
-    CIPHER(0xc011, TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA)
-#else
-   XCIPHER(0xc011, TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA)
-#endif
 #ifdef TLS1_TXT_DHE_RSA_WITH_AES_128_SHA
     CIPHER(0x0033, TLS1_TXT_DHE_RSA_WITH_AES_128_SHA)
 #else
    XCIPHER(0x0033, TLS1_TXT_DHE_RSA_WITH_AES_128_SHA)
 #endif
-#ifdef TLS1_TXT_DHE_DSS_WITH_AES_128_SHA
-    CIPHER(0x0032, TLS1_TXT_DHE_DSS_WITH_AES_128_SHA)
-#else
-   XCIPHER(0x0032, TLS1_TXT_DHE_DSS_WITH_AES_128_SHA)
-#endif
-#ifdef TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA
-    CIPHER(0x0045, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA)
-#else
-   XCIPHER(0x0045, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA)
-#endif
 #ifdef TLS1_TXT_DHE_RSA_WITH_AES_256_SHA
     CIPHER(0x0039, TLS1_TXT_DHE_RSA_WITH_AES_256_SHA)
 #else
    XCIPHER(0x0039, TLS1_TXT_DHE_RSA_WITH_AES_256_SHA)
 #endif
-#ifdef TLS1_TXT_DHE_DSS_WITH_AES_256_SHA
-    CIPHER(0x0038, TLS1_TXT_DHE_DSS_WITH_AES_256_SHA)
-#else
-   XCIPHER(0x0038, TLS1_TXT_DHE_DSS_WITH_AES_256_SHA)
-#endif
-#ifdef TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA
-    CIPHER(0x0088, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA)
-#else
-   XCIPHER(0x0088, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA)
-#endif
-#ifdef SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA
-    CIPHER(0x0016, SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA)
-#else
-   XCIPHER(0x0016, SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA)
-#endif
 #ifdef TLS1_TXT_RSA_WITH_AES_128_SHA
     CIPHER(0x002f, TLS1_TXT_RSA_WITH_AES_128_SHA)
 #else
    XCIPHER(0x002f, TLS1_TXT_RSA_WITH_AES_128_SHA)
 #endif
-#ifdef TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA
-    CIPHER(0x0041, TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA)
-#else
-   XCIPHER(0x0041, TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA)
-#endif
 #ifdef TLS1_TXT_RSA_WITH_AES_256_SHA
     CIPHER(0x0035, TLS1_TXT_RSA_WITH_AES_256_SHA)
 #else
    XCIPHER(0x0035, TLS1_TXT_RSA_WITH_AES_256_SHA)
 #endif
-#ifdef TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA
-    CIPHER(0x0084, TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA)
-#else
-   XCIPHER(0x0084, TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA)
-#endif
 #ifdef SSL3_TXT_RSA_DES_192_CBC3_SHA
     CIPHER(0x000a, SSL3_TXT_RSA_DES_192_CBC3_SHA)
 #else
    XCIPHER(0x000a, SSL3_TXT_RSA_DES_192_CBC3_SHA)
 #endif
-#ifdef SSL3_TXT_RSA_RC4_128_SHA
-    CIPHER(0x0005, SSL3_TXT_RSA_RC4_128_SHA)
-#else
-   XCIPHER(0x0005, SSL3_TXT_RSA_RC4_128_SHA)
-#endif
-#ifdef SSL3_TXT_RSA_RC4_128_MD5
-    CIPHER(0x0004, SSL3_TXT_RSA_RC4_128_MD5)
-#else
-   XCIPHER(0x0004, SSL3_TXT_RSA_RC4_128_MD5)
-#endif
diff --git a/src/lib/tls/include.am b/src/lib/tls/include.am
new file mode 100644
index 0000000000..a664b29fb2
--- /dev/null
+++ b/src/lib/tls/include.am
@@ -0,0 +1,40 @@
+
+noinst_LIBRARIES += src/lib/libtor-tls.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-tls-testing.a
+endif
+
+src_lib_libtor_tls_a_SOURCES =			\
+	src/lib/tls/buffers_tls.c		\
+	src/lib/tls/tortls.c			\
+	src/lib/tls/x509.c
+
+if USE_NSS
+src_lib_libtor_tls_a_SOURCES +=			\
+	src/lib/tls/nss_countbytes.c		\
+	src/lib/tls/tortls_nss.c		\
+	src/lib/tls/x509_nss.c
+else
+src_lib_libtor_tls_a_SOURCES +=			\
+	src/lib/tls/tortls_openssl.c		\
+	src/lib/tls/x509_openssl.c
+endif
+
+src_lib_libtor_tls_a_CFLAGS = $(AM_CFLAGS) $(TOR_CFLAGS_CRYPTLIB)
+
+src_lib_libtor_tls_testing_a_SOURCES = \
+	$(src_lib_libtor_tls_a_SOURCES)
+src_lib_libtor_tls_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_tls_testing_a_CFLAGS = \
+	$(AM_CFLAGS) $(TOR_CFLAGS_CRYPTLIB) $(TEST_CFLAGS)
+
+noinst_HEADERS +=				\
+	src/lib/tls/ciphers.inc			\
+	src/lib/tls/buffers_tls.h		\
+	src/lib/tls/nss_countbytes.h		\
+	src/lib/tls/tortls.h			\
+	src/lib/tls/tortls_internal.h		\
+	src/lib/tls/tortls_st.h			\
+	src/lib/tls/x509.h			\
+	src/lib/tls/x509_internal.h
diff --git a/src/lib/tls/nss_countbytes.c b/src/lib/tls/nss_countbytes.c
new file mode 100644
index 0000000000..7761727acd
--- /dev/null
+++ b/src/lib/tls/nss_countbytes.c
@@ -0,0 +1,244 @@
+/* Copyright 2018-2019, The Tor Project Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file nss_countbytes.c
+ * \brief A PRFileDesc layer to let us count the number of bytes
+ *        bytes actually written on a PRFileDesc.
+ **/
+
+#include "orconfig.h"
+
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/tls/nss_countbytes.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <prio.h>
+
+/** Boolean: have we initialized this module */
+static bool countbytes_initialized = false;
+
+/** Integer to identity this layer. */
+static PRDescIdentity countbytes_layer_id = PR_INVALID_IO_LAYER;
+
+/** Table of methods for this layer.*/
+static PRIOMethods countbytes_methods;
+
+/** Default close function provided by NSPR.  We use this to help
+ *  implement our own close function.*/
+static PRStatus(*default_close_fn)(PRFileDesc *fd);
+
+static PRStatus countbytes_close_fn(PRFileDesc *fd);
+static PRInt32 countbytes_read_fn(PRFileDesc *fd, void *buf, PRInt32 amount);
+static PRInt32 countbytes_write_fn(PRFileDesc *fd, const void *buf,
+                                   PRInt32 amount);
+static PRInt32 countbytes_writev_fn(PRFileDesc *fd, const PRIOVec *iov,
+                                    PRInt32 size, PRIntervalTime timeout);
+static PRInt32 countbytes_send_fn(PRFileDesc *fd, const void *buf,
+                                  PRInt32 amount, PRIntn flags,
+                                  PRIntervalTime timeout);
+static PRInt32 countbytes_recv_fn(PRFileDesc *fd, void *buf, PRInt32 amount,
+                                  PRIntn flags, PRIntervalTime timeout);
+
+/** Private fields for the byte-counter layer.  We cast this to and from
+ * PRFilePrivate*, which is supposed to be allowed. */
+typedef struct tor_nss_bytecounts_t {
+  uint64_t n_read;
+  uint64_t n_written;
+} tor_nss_bytecounts_t;
+
+/**
+ * Initialize this module, if it is not already initialized.
+ **/
+void
+tor_nss_countbytes_init(void)
+{
+  if (countbytes_initialized)
+    return;
+
+  countbytes_layer_id = PR_GetUniqueIdentity("Tor byte-counting layer");
+  tor_assert(countbytes_layer_id != PR_INVALID_IO_LAYER);
+
+  memcpy(&countbytes_methods, PR_GetDefaultIOMethods(), sizeof(PRIOMethods));
+
+  default_close_fn = countbytes_methods.close;
+  countbytes_methods.close = countbytes_close_fn;
+  countbytes_methods.read = countbytes_read_fn;
+  countbytes_methods.write = countbytes_write_fn;
+  countbytes_methods.writev = countbytes_writev_fn;
+  countbytes_methods.send = countbytes_send_fn;
+  countbytes_methods.recv = countbytes_recv_fn;
+  /* NOTE: We aren't wrapping recvfrom, sendto, or sendfile, since I think
+   * NSS won't be using them for TLS connections. */
+
+  countbytes_initialized = true;
+}
+
+/**
+ * Return the tor_nss_bytecounts_t object for a given IO layer. Asserts that
+ * the IO layer is in fact a layer created by this module.
+ */
+static tor_nss_bytecounts_t *
+get_counts(PRFileDesc *fd)
+{
+  tor_assert(fd->identity == countbytes_layer_id);
+  return (tor_nss_bytecounts_t*) fd->secret;
+}
+
+/** Helper: increment the read-count of an fd by n. */
+#define INC_READ(fd, n) STMT_BEGIN                      \
+    get_counts(fd)->n_read += (n);                      \
+  STMT_END
+
+/** Helper: increment the write-count of an fd by n. */
+#define INC_WRITTEN(fd, n) STMT_BEGIN                      \
+    get_counts(fd)->n_written += (n);                      \
+  STMT_END
+
+/** Implementation for PR_Close: frees the 'secret' field, then passes control
+ * to the default close function */
+static PRStatus
+countbytes_close_fn(PRFileDesc *fd)
+{
+  tor_assert(fd);
+
+  tor_nss_bytecounts_t *counts = (tor_nss_bytecounts_t *)fd->secret;
+  tor_free(counts);
+  fd->secret = NULL;
+
+  return default_close_fn(fd);
+}
+
+/** Implementation for PR_Read: Calls the lower-level read function,
+ * and records what it said. */
+static PRInt32
+countbytes_read_fn(PRFileDesc *fd, void *buf, PRInt32 amount)
+{
+  tor_assert(fd);
+  tor_assert(fd->lower);
+
+  PRInt32 result = (fd->lower->methods->read)(fd->lower, buf, amount);
+  if (result > 0)
+    INC_READ(fd, result);
+  return result;
+}
+/** Implementation for PR_Write: Calls the lower-level write function,
+ * and records what it said. */
+static PRInt32
+countbytes_write_fn(PRFileDesc *fd, const void *buf, PRInt32 amount)
+{
+  tor_assert(fd);
+  tor_assert(fd->lower);
+
+  PRInt32 result = (fd->lower->methods->write)(fd->lower, buf, amount);
+  if (result > 0)
+    INC_WRITTEN(fd, result);
+  return result;
+}
+/** Implementation for PR_Writev: Calls the lower-level writev function,
+ * and records what it said. */
+static PRInt32
+countbytes_writev_fn(PRFileDesc *fd, const PRIOVec *iov,
+                     PRInt32 size, PRIntervalTime timeout)
+{
+  tor_assert(fd);
+  tor_assert(fd->lower);
+
+  PRInt32 result = (fd->lower->methods->writev)(fd->lower, iov, size, timeout);
+  if (result > 0)
+    INC_WRITTEN(fd, result);
+  return result;
+}
+/** Implementation for PR_Send: Calls the lower-level send function,
+ * and records what it said. */
+static PRInt32
+countbytes_send_fn(PRFileDesc *fd, const void *buf,
+                   PRInt32 amount, PRIntn flags, PRIntervalTime timeout)
+{
+  tor_assert(fd);
+  tor_assert(fd->lower);
+
+  PRInt32 result = (fd->lower->methods->send)(fd->lower, buf, amount, flags,
+                                              timeout);
+  if (result > 0)
+    INC_WRITTEN(fd, result);
+  return result;
+}
+/** Implementation for PR_Recv: Calls the lower-level recv function,
+ * and records what it said. */
+static PRInt32
+countbytes_recv_fn(PRFileDesc *fd, void *buf, PRInt32 amount,
+                                  PRIntn flags, PRIntervalTime timeout)
+{
+  tor_assert(fd);
+  tor_assert(fd->lower);
+
+  PRInt32 result = (fd->lower->methods->recv)(fd->lower, buf, amount, flags,
+                                              timeout);
+  if (result > 0)
+    INC_READ(fd, result);
+  return result;
+}
+
+/**
+ * Wrap a PRFileDesc from NSPR with a new PRFileDesc that will count the
+ * total number of bytes read and written.  Return the new PRFileDesc.
+ *
+ * This function takes ownership of its input.
+ */
+PRFileDesc *
+tor_wrap_prfiledesc_with_byte_counter(PRFileDesc *stack)
+{
+  if (BUG(! countbytes_initialized)) {
+    tor_nss_countbytes_init();
+  }
+
+  tor_nss_bytecounts_t *bytecounts = tor_malloc_zero(sizeof(*bytecounts));
+
+  PRFileDesc *newfd = PR_CreateIOLayerStub(countbytes_layer_id,
+                                           &countbytes_methods);
+  tor_assert(newfd);
+  newfd->secret = (PRFilePrivate *)bytecounts;
+
+  /* This does some complicated messing around with the headers of these
+     objects; see the NSPR documentation for more. The upshot is that
+     after PushIOLayer, "stack" will be the head of the stack.
+  */
+  PRStatus status = PR_PushIOLayer(stack, PR_TOP_IO_LAYER, newfd);
+  tor_assert(status == PR_SUCCESS);
+
+  return stack;
+}
+
+/**
+ * Given a PRFileDesc returned by tor_wrap_prfiledesc_with_byte_counter(),
+ * or another PRFileDesc wrapping that PRFileDesc, set the provided
+ * pointers to the number of bytes read and written on the descriptor since
+ * it was created.
+ *
+ * Return 0 on success, -1 on failure.
+ */
+int
+tor_get_prfiledesc_byte_counts(PRFileDesc *fd,
+                               uint64_t *n_read_out,
+                               uint64_t *n_written_out)
+{
+  if (BUG(! countbytes_initialized)) {
+    tor_nss_countbytes_init();
+  }
+
+  tor_assert(fd);
+  PRFileDesc *bclayer = PR_GetIdentitiesLayer(fd, countbytes_layer_id);
+  if (BUG(bclayer == NULL))
+    return -1;
+
+  tor_nss_bytecounts_t *counts = get_counts(bclayer);
+
+  *n_read_out = counts->n_read;
+  *n_written_out = counts->n_written;
+
+  return 0;
+}
diff --git a/src/lib/tls/nss_countbytes.h b/src/lib/tls/nss_countbytes.h
new file mode 100644
index 0000000000..8b31603923
--- /dev/null
+++ b/src/lib/tls/nss_countbytes.h
@@ -0,0 +1,25 @@
+/* Copyright 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file nss_countbytes.h
+ * \brief Header for nss_countbytes.c, which lets us count the number of
+ *        bytes actually written on a PRFileDesc.
+ **/
+
+#ifndef TOR_NSS_COUNTBYTES_H
+#define TOR_NSS_COUNTBYTES_H
+
+#include "lib/cc/torint.h"
+
+void tor_nss_countbytes_init(void);
+
+struct PRFileDesc;
+struct PRFileDesc *tor_wrap_prfiledesc_with_byte_counter(
+                                               struct PRFileDesc *stack);
+
+int tor_get_prfiledesc_byte_counts(struct PRFileDesc *fd,
+                                   uint64_t *n_read_out,
+                                   uint64_t *n_written_out);
+
+#endif
diff --git a/src/lib/tls/tortls.c b/src/lib/tls/tortls.c
new file mode 100644
index 0000000000..4ca7c7d9d3
--- /dev/null
+++ b/src/lib/tls/tortls.c
@@ -0,0 +1,442 @@
+/* Copyright (c) 2003, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define TORTLS_PRIVATE
+#define TOR_X509_PRIVATE
+#include "lib/tls/x509.h"
+#include "lib/tls/x509_internal.h"
+#include "lib/tls/tortls.h"
+#include "lib/tls/tortls_st.h"
+#include "lib/tls/tortls_internal.h"
+#include "lib/log/util_bug.h"
+#include "lib/intmath/cmp.h"
+#include "lib/crypt_ops/crypto_rsa.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/net/socket.h"
+
+#ifdef _WIN32
+  #include <winsock2.h>
+  #include <ws2tcpip.h>
+#endif
+
+#include <time.h>
+
+/** Global TLS contexts. We keep them here because nobody else needs
+ * to touch them.
+ *
+ * @{ */
+STATIC tor_tls_context_t *server_tls_context = NULL;
+STATIC tor_tls_context_t *client_tls_context = NULL;
+/**@}*/
+
+/**
+ * Return the appropriate TLS context.
+ */
+tor_tls_context_t *
+tor_tls_context_get(int is_server)
+{
+  return is_server ? server_tls_context : client_tls_context;
+}
+
+/** Convert an errno (or a WSAerrno on windows) into a TOR_TLS_* error
+ * code. */
+int
+tor_errno_to_tls_error(int e)
+{
+  switch (e) {
+    case SOCK_ERRNO(ECONNRESET): // most common
+      return TOR_TLS_ERROR_CONNRESET;
+    case SOCK_ERRNO(ETIMEDOUT):
+      return TOR_TLS_ERROR_TIMEOUT;
+    case SOCK_ERRNO(EHOSTUNREACH):
+    case SOCK_ERRNO(ENETUNREACH):
+      return TOR_TLS_ERROR_NO_ROUTE;
+    case SOCK_ERRNO(ECONNREFUSED):
+      return TOR_TLS_ERROR_CONNREFUSED; // least common
+    default:
+      return TOR_TLS_ERROR_MISC;
+  }
+}
+
+/** Set *<b>link_cert_out</b> and *<b>id_cert_out</b> to the link certificate
+ * and ID certificate that we're currently using for our V3 in-protocol
+ * handshake's certificate chain.  If <b>server</b> is true, provide the certs
+ * that we use in server mode (auth, ID); otherwise, provide the certs that we
+ * use in client mode. (link, ID) */
+int
+tor_tls_get_my_certs(int server,
+                     const tor_x509_cert_t **link_cert_out,
+                     const tor_x509_cert_t **id_cert_out)
+{
+  tor_tls_context_t *ctx = tor_tls_context_get(server);
+  int rv = -1;
+  const tor_x509_cert_t *link_cert = NULL;
+  const tor_x509_cert_t *id_cert = NULL;
+  if (ctx) {
+    rv = 0;
+    link_cert = server ? ctx->my_link_cert : ctx->my_auth_cert;
+    id_cert = ctx->my_id_cert;
+  }
+  if (link_cert_out)
+    *link_cert_out = link_cert;
+  if (id_cert_out)
+    *id_cert_out = id_cert;
+  return rv;
+}
+
+/**
+ * Return the authentication key that we use to authenticate ourselves as a
+ * client in the V3 in-protocol handshake.
+ */
+crypto_pk_t *
+tor_tls_get_my_client_auth_key(void)
+{
+  tor_tls_context_t *context = tor_tls_context_get(0);
+  if (! context)
+    return NULL;
+  return context->auth_key;
+}
+
+/** Increase the reference count of <b>ctx</b>. */
+void
+tor_tls_context_incref(tor_tls_context_t *ctx)
+{
+  ++ctx->refcnt;
+}
+
+/** Remove a reference to <b>ctx</b>, and free it if it has no more
+ * references. */
+void
+tor_tls_context_decref(tor_tls_context_t *ctx)
+{
+  tor_assert(ctx);
+  if (--ctx->refcnt == 0) {
+    tor_tls_context_impl_free(ctx->ctx);
+    tor_x509_cert_free(ctx->my_link_cert);
+    tor_x509_cert_free(ctx->my_id_cert);
+    tor_x509_cert_free(ctx->my_auth_cert);
+    crypto_pk_free(ctx->link_key);
+    crypto_pk_free(ctx->auth_key);
+    /* LCOV_EXCL_BR_START since ctx will never be NULL here */
+    tor_free(ctx);
+    /* LCOV_EXCL_BR_STOP */
+  }
+}
+
+/** Free all global TLS structures. */
+void
+tor_tls_free_all(void)
+{
+  check_no_tls_errors();
+
+  if (server_tls_context) {
+    tor_tls_context_t *ctx = server_tls_context;
+    server_tls_context = NULL;
+    tor_tls_context_decref(ctx);
+  }
+  if (client_tls_context) {
+    tor_tls_context_t *ctx = client_tls_context;
+    client_tls_context = NULL;
+    tor_tls_context_decref(ctx);
+  }
+}
+
+/** Given a TOR_TLS_* error code, return a string equivalent. */
+const char *
+tor_tls_err_to_string(int err)
+{
+  if (err >= 0)
+    return "[Not an error.]";
+  switch (err) {
+    case TOR_TLS_ERROR_MISC: return "misc error";
+    case TOR_TLS_ERROR_IO: return "unexpected close";
+    case TOR_TLS_ERROR_CONNREFUSED: return "connection refused";
+    case TOR_TLS_ERROR_CONNRESET: return "connection reset";
+    case TOR_TLS_ERROR_NO_ROUTE: return "host unreachable";
+    case TOR_TLS_ERROR_TIMEOUT: return "connection timed out";
+    case TOR_TLS_CLOSE: return "closed";
+    case TOR_TLS_WANTREAD: return "want to read";
+    case TOR_TLS_WANTWRITE: return "want to write";
+    default: return "(unknown error code)";
+  }
+}
+
+/** Create new global client and server TLS contexts.
+ *
+ * If <b>server_identity</b> is NULL, this will not generate a server
+ * TLS context. If TOR_TLS_CTX_IS_PUBLIC_SERVER is set in <b>flags</b>, use
+ * the same TLS context for incoming and outgoing connections, and
+ * ignore <b>client_identity</b>. If one of TOR_TLS_CTX_USE_ECDHE_P{224,256}
+ * is set in <b>flags</b>, use that ECDHE group if possible; otherwise use
+ * the default ECDHE group. */
+int
+tor_tls_context_init(unsigned flags,
+                     crypto_pk_t *client_identity,
+                     crypto_pk_t *server_identity,
+                     unsigned int key_lifetime)
+{
+  int rv1 = 0;
+  int rv2 = 0;
+  const int is_public_server = flags & TOR_TLS_CTX_IS_PUBLIC_SERVER;
+  check_no_tls_errors();
+
+  if (is_public_server) {
+    tor_tls_context_t *new_ctx;
+    tor_tls_context_t *old_ctx;
+
+    tor_assert(server_identity != NULL);
+
+    rv1 = tor_tls_context_init_one(&server_tls_context,
+                                   server_identity,
+                                   key_lifetime, flags, 0);
+
+    if (rv1 >= 0) {
+      new_ctx = server_tls_context;
+      tor_tls_context_incref(new_ctx);
+      old_ctx = client_tls_context;
+      client_tls_context = new_ctx;
+
+      if (old_ctx != NULL) {
+        tor_tls_context_decref(old_ctx);
+      }
+    } else {
+      tls_log_errors(NULL, LOG_WARN, LD_CRYPTO,
+                     "constructing a TLS context");
+    }
+  } else {
+    if (server_identity != NULL) {
+      rv1 = tor_tls_context_init_one(&server_tls_context,
+                                     server_identity,
+                                     key_lifetime,
+                                     flags,
+                                     0);
+      if (rv1 < 0)
+        tls_log_errors(NULL, LOG_WARN, LD_CRYPTO,
+                       "constructing a server TLS context");
+    } else {
+      tor_tls_context_t *old_ctx = server_tls_context;
+      server_tls_context = NULL;
+
+      if (old_ctx != NULL) {
+        tor_tls_context_decref(old_ctx);
+      }
+    }
+
+    rv2 = tor_tls_context_init_one(&client_tls_context,
+                                   client_identity,
+                                   key_lifetime,
+                                   flags,
+                                   1);
+    if (rv2 < 0)
+        tls_log_errors(NULL, LOG_WARN, LD_CRYPTO,
+                       "constructing a client TLS context");
+  }
+
+  return MIN(rv1, rv2);
+}
+
+/** Create a new global TLS context.
+ *
+ * You can call this function multiple times.  Each time you call it,
+ * it generates new certificates; all new connections will use
+ * the new SSL context.
+ */
+int
+tor_tls_context_init_one(tor_tls_context_t **ppcontext,
+                         crypto_pk_t *identity,
+                         unsigned int key_lifetime,
+                         unsigned int flags,
+                         int is_client)
+{
+  tor_tls_context_t *new_ctx = tor_tls_context_new(identity,
+                                                   key_lifetime,
+                                                   flags,
+                                                   is_client);
+  tor_tls_context_t *old_ctx = *ppcontext;
+
+  if (new_ctx != NULL) {
+    *ppcontext = new_ctx;
+
+    /* Free the old context if one existed. */
+    if (old_ctx != NULL) {
+      /* This is safe even if there are open connections: we reference-
+       * count tor_tls_context_t objects. */
+      tor_tls_context_decref(old_ctx);
+    }
+  }
+
+  return ((new_ctx != NULL) ? 0 : -1);
+}
+
+/** Size of the RSA key to use for our TLS link keys */
+#define RSA_LINK_KEY_BITS 2048
+
+/** How long do identity certificates live? (sec) */
+#define IDENTITY_CERT_LIFETIME  (365*24*60*60)
+
+/**
+ * Initialize the certificates and keys for a TLS context <b>result</b>
+ *
+ * Other arguments as for tor_tls_context_new().
+ */
+int
+tor_tls_context_init_certificates(tor_tls_context_t *result,
+                                  crypto_pk_t *identity,
+                                  unsigned key_lifetime,
+                                  unsigned flags)
+{
+  (void)flags;
+  int rv = -1;
+  char *nickname = NULL, *nn2 = NULL;
+  crypto_pk_t *rsa = NULL, *rsa_auth = NULL;
+  tor_x509_cert_impl_t *cert = NULL, *idcert = NULL, *authcert = NULL;
+
+  nickname = crypto_random_hostname(8, 20, "www.", ".net");
+
+#ifdef DISABLE_V3_LINKPROTO_SERVERSIDE
+  nn2 = crypto_random_hostname(8, 20, "www.", ".net");
+#else
+  nn2 = crypto_random_hostname(8, 20, "www.", ".com");
+#endif
+
+  /* Generate short-term RSA key for use with TLS. */
+  if (!(rsa = crypto_pk_new()))
+    goto error;
+  if (crypto_pk_generate_key_with_bits(rsa, RSA_LINK_KEY_BITS)<0)
+    goto error;
+
+  /* Generate short-term RSA key for use in the in-protocol ("v3")
+   * authentication handshake. */
+  if (!(rsa_auth = crypto_pk_new()))
+    goto error;
+  if (crypto_pk_generate_key(rsa_auth)<0)
+    goto error;
+
+  /* Create a link certificate signed by identity key. */
+  cert = tor_tls_create_certificate(rsa, identity, nickname, nn2,
+                                    key_lifetime);
+  /* Create self-signed certificate for identity key. */
+  idcert = tor_tls_create_certificate(identity, identity, nn2, nn2,
+                                      IDENTITY_CERT_LIFETIME);
+  /* Create an authentication certificate signed by identity key. */
+  authcert = tor_tls_create_certificate(rsa_auth, identity, nickname, nn2,
+                                        key_lifetime);
+  if (!cert || !idcert || !authcert) {
+    log_warn(LD_CRYPTO, "Error creating certificate");
+    goto error;
+  }
+
+  result->my_link_cert = tor_x509_cert_new(cert);
+  cert = NULL;
+  result->my_id_cert = tor_x509_cert_new(idcert);
+  idcert = NULL;
+  result->my_auth_cert = tor_x509_cert_new(authcert);
+  authcert = NULL;
+  if (!result->my_link_cert || !result->my_id_cert || !result->my_auth_cert)
+    goto error;
+  result->link_key = rsa;
+  rsa = NULL;
+  result->auth_key = rsa_auth;
+  rsa_auth = NULL;
+
+  rv = 0;
+ error:
+
+  tor_free(nickname);
+  tor_free(nn2);
+
+  tor_x509_cert_impl_free(cert);
+  tor_x509_cert_impl_free(idcert);
+  tor_x509_cert_impl_free(authcert);
+  crypto_pk_free(rsa);
+  crypto_pk_free(rsa_auth);
+
+  return rv;
+}
+/** Make future log messages about <b>tls</b> display the address
+ * <b>address</b>.
+ */
+void
+tor_tls_set_logged_address(tor_tls_t *tls, const char *address)
+{
+  tor_assert(tls);
+  tor_free(tls->address);
+  tls->address = tor_strdup(address);
+}
+
+/** Return whether this tls initiated the connect (client) or
+ * received it (server). */
+int
+tor_tls_is_server(tor_tls_t *tls)
+{
+  tor_assert(tls);
+  return tls->isServer;
+}
+
+/** Release resources associated with a TLS object.  Does not close the
+ * underlying file descriptor.
+ */
+void
+tor_tls_free_(tor_tls_t *tls)
+{
+  if (!tls)
+    return;
+  tor_assert(tls->ssl);
+  {
+    size_t r,w;
+    tor_tls_get_n_raw_bytes(tls,&r,&w); /* ensure written_by_tls is updated */
+  }
+  tor_tls_impl_free(tls->ssl);
+  tls->ssl = NULL;
+#ifdef ENABLE_OPENSSL
+  tls->negotiated_callback = NULL;
+#endif
+  if (tls->context)
+    tor_tls_context_decref(tls->context);
+  tor_free(tls->address);
+  tls->magic = 0x99999999;
+  tor_free(tls);
+}
+
+/** If the provided tls connection is authenticated and has a
+ * certificate chain that is currently valid and signed, then set
+ * *<b>identity_key</b> to the identity certificate's key and return
+ * 0.  Else, return -1 and log complaints with log-level <b>severity</b>.
+ */
+int
+tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity)
+{
+  tor_x509_cert_impl_t *cert = NULL, *id_cert = NULL;
+  tor_x509_cert_t *peer_x509 = NULL, *id_x509 = NULL;
+  tor_assert(tls);
+  tor_assert(identity);
+  int rv = -1;
+
+  try_to_extract_certs_from_tls(severity, tls, &cert, &id_cert);
+  if (!cert)
+    goto done;
+  if (!id_cert) {
+    log_fn(severity,LD_PROTOCOL,"No distinct identity certificate found");
+    goto done;
+  }
+  peer_x509 = tor_x509_cert_new(cert);
+  id_x509 = tor_x509_cert_new(id_cert);
+  cert = id_cert = NULL; /* Prevent double-free */
+
+  if (! tor_tls_cert_is_valid(severity, peer_x509, id_x509, time(NULL), 0)) {
+    goto done;
+  }
+
+  *identity = tor_tls_cert_get_key(id_x509);
+  rv = 0;
+
+ done:
+  tor_x509_cert_impl_free(cert);
+  tor_x509_cert_impl_free(id_cert);
+  tor_x509_cert_free(peer_x509);
+  tor_x509_cert_free(id_x509);
+
+  return rv;
+}
diff --git a/src/lib/tls/tortls.h b/src/lib/tls/tortls.h
new file mode 100644
index 0000000000..8efc7a1c98
--- /dev/null
+++ b/src/lib/tls/tortls.h
@@ -0,0 +1,160 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_TORTLS_H
+#define TOR_TORTLS_H
+
+/**
+ * \file tortls.h
+ * \brief Headers for tortls.c
+ **/
+
+#include "lib/crypt_ops/crypto_rsa.h"
+#include "lib/testsupport/testsupport.h"
+#include "lib/net/nettypes.h"
+
+/* Opaque structure to hold a TLS connection. */
+typedef struct tor_tls_t tor_tls_t;
+
+#ifdef TORTLS_PRIVATE
+#ifdef ENABLE_OPENSSL
+struct ssl_st;
+struct ssl_ctx_st;
+struct ssl_session_st;
+typedef struct ssl_ctx_st tor_tls_context_impl_t;
+typedef struct ssl_st tor_tls_impl_t;
+#else
+struct PRFileDesc;
+typedef struct PRFileDesc tor_tls_context_impl_t;
+typedef struct PRFileDesc tor_tls_impl_t;
+#endif
+#endif
+
+struct tor_x509_cert_t;
+
+/* Possible return values for most tor_tls_* functions. */
+#define MIN_TOR_TLS_ERROR_VAL_     -9
+#define TOR_TLS_ERROR_MISC         -9
+/* Rename to unexpected close or something. XXXX */
+#define TOR_TLS_ERROR_IO           -8
+#define TOR_TLS_ERROR_CONNREFUSED  -7
+#define TOR_TLS_ERROR_CONNRESET    -6
+#define TOR_TLS_ERROR_NO_ROUTE     -5
+#define TOR_TLS_ERROR_TIMEOUT      -4
+#define TOR_TLS_CLOSE              -3
+#define TOR_TLS_WANTREAD           -2
+#define TOR_TLS_WANTWRITE          -1
+#define TOR_TLS_DONE                0
+
+/** Collection of case statements for all TLS errors that are not due to
+ * underlying IO failure. */
+#define CASE_TOR_TLS_ERROR_ANY_NONIO            \
+  case TOR_TLS_ERROR_MISC:                      \
+  case TOR_TLS_ERROR_CONNREFUSED:               \
+  case TOR_TLS_ERROR_CONNRESET:                 \
+  case TOR_TLS_ERROR_NO_ROUTE:                  \
+  case TOR_TLS_ERROR_TIMEOUT
+
+/** Use this macro in a switch statement to catch _any_ TLS error.  That way,
+ * if more errors are added, your switches will still work. */
+#define CASE_TOR_TLS_ERROR_ANY                  \
+  CASE_TOR_TLS_ERROR_ANY_NONIO:                 \
+  case TOR_TLS_ERROR_IO
+
+#define TOR_TLS_IS_ERROR(rv) ((rv) < TOR_TLS_CLOSE)
+
+/** Holds a SSL_CTX object and related state used to configure TLS
+ * connections.
+ */
+typedef struct tor_tls_context_t tor_tls_context_t;
+
+const char *tor_tls_err_to_string(int err);
+void tor_tls_get_state_description(tor_tls_t *tls, char *buf, size_t sz);
+void tor_tls_free_all(void);
+
+#define TOR_TLS_CTX_IS_PUBLIC_SERVER (1u<<0)
+#define TOR_TLS_CTX_USE_ECDHE_P256   (1u<<1)
+#define TOR_TLS_CTX_USE_ECDHE_P224   (1u<<2)
+
+void tor_tls_init(void);
+void tls_log_errors(tor_tls_t *tls, int severity, int domain,
+                    const char *doing);
+int tor_tls_context_init(unsigned flags,
+                         crypto_pk_t *client_identity,
+                         crypto_pk_t *server_identity,
+                         unsigned int key_lifetime);
+void tor_tls_context_incref(tor_tls_context_t *ctx);
+void tor_tls_context_decref(tor_tls_context_t *ctx);
+tor_tls_context_t *tor_tls_context_get(int is_server);
+tor_tls_t *tor_tls_new(tor_socket_t sock, int is_server);
+void tor_tls_set_logged_address(tor_tls_t *tls, const char *address);
+void tor_tls_set_renegotiate_callback(tor_tls_t *tls,
+                                      void (*cb)(tor_tls_t *, void *arg),
+                                      void *arg);
+int tor_tls_is_server(tor_tls_t *tls);
+void tor_tls_release_socket(tor_tls_t *tls);
+void tor_tls_free_(tor_tls_t *tls);
+#define tor_tls_free(tls) FREE_AND_NULL(tor_tls_t, tor_tls_free_, (tls))
+int tor_tls_peer_has_cert(tor_tls_t *tls);
+MOCK_DECL(struct tor_x509_cert_t *,tor_tls_get_peer_cert,(tor_tls_t *tls));
+MOCK_DECL(struct tor_x509_cert_t *,tor_tls_get_own_cert,(tor_tls_t *tls));
+int tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity);
+MOCK_DECL(int, tor_tls_read, (tor_tls_t *tls, char *cp, size_t len));
+int tor_tls_write(tor_tls_t *tls, const char *cp, size_t n);
+int tor_tls_handshake(tor_tls_t *tls);
+int tor_tls_finish_handshake(tor_tls_t *tls);
+void tor_tls_unblock_renegotiation(tor_tls_t *tls);
+void tor_tls_block_renegotiation(tor_tls_t *tls);
+void tor_tls_assert_renegotiation_unblocked(tor_tls_t *tls);
+int tor_tls_get_pending_bytes(tor_tls_t *tls);
+size_t tor_tls_get_forced_write_size(tor_tls_t *tls);
+
+void tor_tls_get_n_raw_bytes(tor_tls_t *tls,
+                             size_t *n_read, size_t *n_written);
+
+int tor_tls_get_buffer_sizes(tor_tls_t *tls,
+                              size_t *rbuf_capacity, size_t *rbuf_bytes,
+                              size_t *wbuf_capacity, size_t *wbuf_bytes);
+
+MOCK_DECL(double, tls_get_write_overhead_ratio, (void));
+
+int tor_tls_used_v1_handshake(tor_tls_t *tls);
+int tor_tls_get_num_server_handshakes(tor_tls_t *tls);
+int tor_tls_server_got_renegotiate(tor_tls_t *tls);
+MOCK_DECL(int,tor_tls_cert_matches_key,(const tor_tls_t *tls,
+                                        const struct tor_x509_cert_t *cert));
+MOCK_DECL(int,tor_tls_get_tlssecrets,(tor_tls_t *tls, uint8_t *secrets_out));
+#ifdef ENABLE_OPENSSL
+/* OpenSSL lets us see these master secrets; NSS sensibly does not. */
+#define HAVE_WORKING_TOR_TLS_GET_TLSSECRETS
+#endif
+MOCK_DECL(int,tor_tls_export_key_material,(
+                     tor_tls_t *tls, uint8_t *secrets_out,
+                     const uint8_t *context,
+                     size_t context_len,
+                     const char *label));
+
+#ifdef ENABLE_OPENSSL
+/* Log and abort if there are unhandled TLS errors in OpenSSL's error stack.
+ */
+#define check_no_tls_errors() check_no_tls_errors_(__FILE__,__LINE__)
+void check_no_tls_errors_(const char *fname, int line);
+
+void tor_tls_log_one_error(tor_tls_t *tls, unsigned long err,
+                           int severity, int domain, const char *doing);
+#else
+#define check_no_tls_errors() STMT_NIL
+#endif
+
+int tor_tls_get_my_certs(int server,
+                         const struct tor_x509_cert_t **link_cert_out,
+                         const struct tor_x509_cert_t **id_cert_out);
+crypto_pk_t *tor_tls_get_my_client_auth_key(void);
+
+const char *tor_tls_get_ciphersuite_name(tor_tls_t *tls);
+
+int evaluate_ecgroup_for_tls(const char *ecgroup);
+
+#endif /* !defined(TOR_TORTLS_H) */
diff --git a/src/lib/tls/tortls_internal.h b/src/lib/tls/tortls_internal.h
new file mode 100644
index 0000000000..071c506561
--- /dev/null
+++ b/src/lib/tls/tortls_internal.h
@@ -0,0 +1,76 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TORTLS_INTERNAL_H
+#define TORTLS_INTERNAL_H
+
+int tor_errno_to_tls_error(int e);
+#ifdef ENABLE_OPENSSL
+int tor_tls_get_error(tor_tls_t *tls, int r, int extra,
+                  const char *doing, int severity, int domain);
+#endif
+MOCK_DECL(void, try_to_extract_certs_from_tls,
+          (int severity, tor_tls_t *tls,
+           tor_x509_cert_impl_t **cert_out,
+           tor_x509_cert_impl_t **id_cert_out));
+
+tor_tls_context_t *tor_tls_context_new(crypto_pk_t *identity,
+                   unsigned int key_lifetime, unsigned flags, int is_client);
+int tor_tls_context_init_one(tor_tls_context_t **ppcontext,
+                             crypto_pk_t *identity,
+                             unsigned int key_lifetime,
+                             unsigned int flags,
+                             int is_client);
+int tor_tls_context_init_certificates(tor_tls_context_t *result,
+                                      crypto_pk_t *identity,
+                                      unsigned key_lifetime,
+                                      unsigned flags);
+void tor_tls_impl_free_(tor_tls_impl_t *ssl);
+#define tor_tls_impl_free(tls) \
+  FREE_AND_NULL(tor_tls_impl_t, tor_tls_impl_free_, (tls))
+
+void tor_tls_context_impl_free_(tor_tls_context_impl_t *);
+#define tor_tls_context_impl_free(ctx) \
+  FREE_AND_NULL(tor_tls_context_impl_t, tor_tls_context_impl_free_, (ctx))
+
+#ifdef ENABLE_OPENSSL
+tor_tls_t *tor_tls_get_by_ssl(const struct ssl_st *ssl);
+int tor_tls_client_is_using_v2_ciphers(const struct ssl_st *ssl);
+void tor_tls_debug_state_callback(const struct ssl_st *ssl,
+                                         int type, int val);
+void tor_tls_server_info_callback(const struct ssl_st *ssl,
+                                         int type, int val);
+void tor_tls_allocate_tor_tls_object_ex_data_index(void);
+
+#if !defined(HAVE_SSL_SESSION_GET_MASTER_KEY)
+size_t SSL_SESSION_get_master_key(struct ssl_session_st *s,
+                                  uint8_t *out,
+                                  size_t len);
+#endif
+
+#ifdef TORTLS_OPENSSL_PRIVATE
+int always_accept_verify_cb(int preverify_ok, X509_STORE_CTX *x509_ctx);
+int tor_tls_classify_client_ciphers(const struct ssl_st *ssl,
+                                           STACK_OF(SSL_CIPHER) *peer_ciphers);
+STATIC int tor_tls_session_secret_cb(struct ssl_st *ssl, void *secret,
+                            int *secret_len,
+                            STACK_OF(SSL_CIPHER) *peer_ciphers,
+                            CONST_IF_OPENSSL_1_1_API SSL_CIPHER **cipher,
+                            void *arg);
+STATIC int find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m,
+                             uint16_t cipher);
+#endif
+#endif
+
+#ifdef TOR_UNIT_TESTS
+extern int tor_tls_object_ex_data_index;
+extern tor_tls_context_t *server_tls_context;
+extern tor_tls_context_t *client_tls_context;
+extern uint16_t v2_cipher_list[];
+extern uint64_t total_bytes_written_over_tls;
+extern uint64_t total_bytes_written_by_tls;
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(TORTLS_INTERNAL_H) */
diff --git a/src/lib/tls/tortls_nss.c b/src/lib/tls/tortls_nss.c
new file mode 100644
index 0000000000..3c62e98df1
--- /dev/null
+++ b/src/lib/tls/tortls_nss.c
@@ -0,0 +1,833 @@
+/* Copyright (c) 2003, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file tortls_nss.c
+ * \brief Wrapper functions to present a consistent interface to
+ * TLS and SSL X.509 functions from NSS.
+ **/
+
+#include "orconfig.h"
+
+#define TORTLS_PRIVATE
+#define TOR_X509_PRIVATE
+
+#ifdef _WIN32
+  #include <winsock2.h>
+  #include <ws2tcpip.h>
+#endif
+
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/crypt_ops/crypto_nss_mgt.h"
+#include "lib/string/printf.h"
+
+#include "lib/tls/x509.h"
+#include "lib/tls/x509_internal.h"
+#include "lib/tls/tortls.h"
+#include "lib/tls/tortls_st.h"
+#include "lib/tls/tortls_internal.h"
+#include "lib/tls/nss_countbytes.h"
+#include "lib/log/util_bug.h"
+
+DISABLE_GCC_WARNING(strict-prototypes)
+#include <prio.h>
+// For access to rar sockets.
+#include <private/pprio.h>
+#include <ssl.h>
+#include <sslt.h>
+#include <sslproto.h>
+#include <certt.h>
+ENABLE_GCC_WARNING(strict-prototypes)
+
+static SECStatus always_accept_cert_cb(void *, PRFileDesc *, PRBool, PRBool);
+
+MOCK_IMPL(void,
+try_to_extract_certs_from_tls,(int severity, tor_tls_t *tls,
+                               tor_x509_cert_impl_t **cert_out,
+                               tor_x509_cert_impl_t **id_cert_out))
+{
+  tor_assert(tls);
+  tor_assert(cert_out);
+  tor_assert(id_cert_out);
+  (void) severity;
+
+  *cert_out = *id_cert_out = NULL;
+
+  CERTCertificate *peer = SSL_PeerCertificate(tls->ssl);
+  if (!peer)
+    return;
+  *cert_out = peer; /* Now owns pointer. */
+
+  CERTCertList *chain = SSL_PeerCertificateChain(tls->ssl);
+  CERTCertListNode *c = CERT_LIST_HEAD(chain);
+  for (; !CERT_LIST_END(c, chain); c = CERT_LIST_NEXT(c)) {
+    if (CERT_CompareCerts(c->cert, peer) == PR_FALSE) {
+      *id_cert_out = CERT_DupCertificate(c->cert);
+      break;
+    }
+  }
+  CERT_DestroyCertList(chain);
+}
+
+static bool
+we_like_ssl_cipher(SSLCipherAlgorithm ca)
+{
+  switch (ca) {
+    case ssl_calg_null: return false;
+    case ssl_calg_rc4: return false;
+    case ssl_calg_rc2: return false;
+    case ssl_calg_des: return false;
+    case ssl_calg_3des: return false; /* ???? */
+    case ssl_calg_idea: return false;
+    case ssl_calg_fortezza: return false;
+    case ssl_calg_camellia: return false;
+    case ssl_calg_seed: return false;
+
+    case ssl_calg_aes: return true;
+    case ssl_calg_aes_gcm: return true;
+    case ssl_calg_chacha20: return true;
+    default: return true;
+  }
+}
+static bool
+we_like_ssl_kea(SSLKEAType kt)
+{
+  switch (kt) {
+    case ssl_kea_null: return false;
+    case ssl_kea_rsa: return false; /* ??? */
+    case ssl_kea_fortezza: return false;
+    case ssl_kea_ecdh_psk: return false;
+    case ssl_kea_dh_psk: return false;
+
+    case ssl_kea_dh: return true;
+    case ssl_kea_ecdh: return true;
+    case ssl_kea_tls13_any: return true;
+
+    case ssl_kea_size: return true; /* prevent a warning. */
+    default: return true;
+  }
+}
+
+static bool
+we_like_mac_algorithm(SSLMACAlgorithm ma)
+{
+  switch (ma) {
+    case ssl_mac_null: return false;
+    case ssl_mac_md5: return false;
+    case ssl_hmac_md5: return false;
+
+    case ssl_mac_sha: return true;
+    case ssl_hmac_sha: return true;
+    case ssl_hmac_sha256: return true;
+    case ssl_mac_aead: return true;
+    case ssl_hmac_sha384: return true;
+    default: return true;
+  }
+}
+
+static bool
+we_like_auth_type(SSLAuthType at)
+{
+  switch (at) {
+    case ssl_auth_null: return false;
+    case ssl_auth_rsa_decrypt: return false;
+    case ssl_auth_dsa: return false;
+    case ssl_auth_kea: return false;
+
+    case ssl_auth_ecdsa: return true;
+    case ssl_auth_ecdh_rsa: return true;
+    case ssl_auth_ecdh_ecdsa: return true;
+    case ssl_auth_rsa_sign: return true;
+    case ssl_auth_rsa_pss: return true;
+    case ssl_auth_psk: return true;
+    case ssl_auth_tls13_any: return true;
+
+    case ssl_auth_size: return true; /* prevent a warning. */
+    default: return true;
+  }
+}
+
+/**
+ * Return true iff this ciphersuite will be hit by a mozilla bug 1312976,
+ * which makes TLS key exporters not work with TLS 1.2 non-SHA256
+ * ciphersuites.
+ **/
+static bool
+ciphersuite_has_nss_export_bug(const SSLCipherSuiteInfo *info)
+{
+  /* For more information on the bug, see
+     https://bugzilla.mozilla.org/show_bug.cgi?id=1312976 */
+
+  /* This bug only exists in TLS 1.2. */
+  if (info->authType == ssl_auth_tls13_any)
+    return false;
+
+  /* Sadly, there's no way to get this information from the
+   * CipherSuiteInfo object itself other than by looking at the
+   * name.  */
+  if (strstr(info->cipherSuiteName, "_SHA384") ||
+      strstr(info->cipherSuiteName, "_SHA512")) {
+    return true;
+  }
+
+  return false;
+}
+
+tor_tls_context_t *
+tor_tls_context_new(crypto_pk_t *identity,
+                    unsigned int key_lifetime, unsigned flags, int is_client)
+{
+  SECStatus s;
+  tor_assert(identity);
+
+  tor_tls_init();
+
+  tor_tls_context_t *ctx = tor_malloc_zero(sizeof(tor_tls_context_t));
+  ctx->refcnt = 1;
+
+  if (! is_client) {
+    if (tor_tls_context_init_certificates(ctx, identity,
+                                          key_lifetime, flags) < 0) {
+      goto err;
+    }
+  }
+
+  {
+    /* Create the "model" PRFileDesc that we will use to base others on. */
+    PRFileDesc *tcp = PR_NewTCPSocket();
+    if (!tcp)
+      goto err;
+
+    ctx->ctx = SSL_ImportFD(NULL, tcp);
+    if (!ctx->ctx) {
+      PR_Close(tcp);
+      goto err;
+    }
+  }
+
+  // Configure the certificate.
+  if (!is_client) {
+    s = SSL_ConfigServerCert(ctx->ctx,
+                             ctx->my_link_cert->cert,
+                             (SECKEYPrivateKey *)
+                               crypto_pk_get_nss_privkey(ctx->link_key),
+                             NULL, /* ExtraServerCertData */
+                             0 /* DataLen */);
+    if (s != SECSuccess)
+      goto err;
+  }
+
+  // We need a certificate from the other side.
+  if (is_client) {
+    // XXXX does this do anything?
+    s = SSL_OptionSet(ctx->ctx, SSL_REQUIRE_CERTIFICATE, PR_TRUE);
+    if (s != SECSuccess)
+      goto err;
+  }
+
+  // Always accept other side's cert; we'll check it ourselves in goofy
+  // tor ways.
+  s = SSL_AuthCertificateHook(ctx->ctx, always_accept_cert_cb, NULL);
+
+  // We allow simultaneous read and write.
+  s = SSL_OptionSet(ctx->ctx, SSL_ENABLE_FDX, PR_TRUE);
+  if (s != SECSuccess)
+    goto err;
+  // XXXX SSL_ROLLBACK_DETECTION??
+  // XXXX SSL_ENABLE_ALPN??
+
+  // Force client-mode or server_mode.
+  s = SSL_OptionSet(ctx->ctx,
+                is_client ? SSL_HANDSHAKE_AS_CLIENT : SSL_HANDSHAKE_AS_SERVER,
+                PR_TRUE);
+  if (s != SECSuccess)
+    goto err;
+
+  // Disable everything before TLS 1.0; support everything else.
+  {
+    SSLVersionRange vrange;
+    memset(&vrange, 0, sizeof(vrange));
+    s = SSL_VersionRangeGetSupported(ssl_variant_stream, &vrange);
+    if (s != SECSuccess)
+      goto err;
+    if (vrange.min < SSL_LIBRARY_VERSION_TLS_1_0)
+      vrange.min = SSL_LIBRARY_VERSION_TLS_1_0;
+    s = SSL_VersionRangeSet(ctx->ctx, &vrange);
+    if (s != SECSuccess)
+      goto err;
+  }
+
+  // Only support strong ciphers.
+  {
+    const PRUint16 *ciphers = SSL_GetImplementedCiphers();
+    const PRUint16 n_ciphers = SSL_GetNumImplementedCiphers();
+    PRUint16 i;
+    for (i = 0; i < n_ciphers; ++i) {
+      SSLCipherSuiteInfo info;
+      memset(&info, 0, sizeof(info));
+      s = SSL_GetCipherSuiteInfo(ciphers[i], &info, sizeof(info));
+      if (s != SECSuccess)
+        goto err;
+      if (BUG(info.cipherSuite != ciphers[i]))
+        goto err;
+      int disable = info.effectiveKeyBits < 128 ||
+        info.macBits < 128 ||
+        !we_like_ssl_cipher(info.symCipher) ||
+        !we_like_ssl_kea(info.keaType) ||
+        !we_like_mac_algorithm(info.macAlgorithm) ||
+        !we_like_auth_type(info.authType)/* Requires NSS 3.24 */;
+
+      if (ciphersuite_has_nss_export_bug(&info)) {
+        /* SSL_ExportKeyingMaterial will fail; we can't use this cipher.
+         */
+        disable = 1;
+      }
+
+      s = SSL_CipherPrefSet(ctx->ctx, ciphers[i],
+                            disable ? PR_FALSE : PR_TRUE);
+      if (s != SECSuccess)
+        goto err;
+    }
+  }
+
+  // Only use DH and ECDH keys once.
+  s = SSL_OptionSet(ctx->ctx, SSL_REUSE_SERVER_ECDHE_KEY, PR_FALSE);
+  if (s != SECSuccess)
+    goto err;
+
+  // don't cache sessions.
+  s = SSL_OptionSet(ctx->ctx, SSL_NO_CACHE, PR_TRUE);
+  if (s != SECSuccess)
+    goto err;
+
+  // Enable DH.
+  s = SSL_OptionSet(ctx->ctx, SSL_ENABLE_SERVER_DHE, PR_TRUE);
+  if (s != SECSuccess)
+    goto err;
+
+  // Set DH and ECDH groups.
+  SSLNamedGroup groups[] = {
+      ssl_grp_ec_curve25519,
+      ssl_grp_ec_secp256r1,
+      ssl_grp_ec_secp224r1,
+      ssl_grp_ffdhe_2048,
+  };
+  s = SSL_NamedGroupConfig(ctx->ctx, groups, ARRAY_LENGTH(groups));
+  if (s != SECSuccess)
+    goto err;
+
+  // These features are off by default, so we don't need to disable them:
+  //   Session tickets
+  //   Renegotiation
+  //   Compression
+
+  goto done;
+ err:
+  tor_tls_context_decref(ctx);
+  ctx = NULL;
+ done:
+  return ctx;
+}
+
+void
+tor_tls_context_impl_free_(tor_tls_context_impl_t *ctx)
+{
+  if (!ctx)
+    return;
+  PR_Close(ctx);
+}
+
+void
+tor_tls_get_state_description(tor_tls_t *tls, char *buf, size_t sz)
+{
+  (void)tls;
+  (void)buf;
+  (void)sz;
+  // AFAICT, NSS doesn't expose its internal state.
+  buf[0]=0;
+}
+
+void
+tor_tls_init(void)
+{
+  tor_nss_countbytes_init();
+}
+
+void
+tls_log_errors(tor_tls_t *tls, int severity, int domain,
+               const char *doing)
+{
+  /* This implementation is a little different for NSS than it is for OpenSSL
+     -- it logs the last error whether anything actually failed or not. So we
+     have to only call it when something has gone wrong and we have a real
+     error to report. */
+
+  (void)tls;
+  PRErrorCode code = PORT_GetError();
+
+  const char *addr = tls ? tls->address : NULL;
+  const char *string = PORT_ErrorToString(code);
+  const char *name = PORT_ErrorToName(code);
+  char buf[16];
+  if (!string)
+    string = "<unrecognized>";
+  if (!name) {
+    tor_snprintf(buf, sizeof(buf), "%d", code);
+    name = buf;
+  }
+
+  const char *with = addr ? " with " : "";
+  addr = addr ? addr : "";
+  if (doing) {
+    log_fn(severity, domain, "TLS error %s while %s%s%s: %s",
+           name, doing, with, addr, string);
+  } else {
+    log_fn(severity, domain, "TLS error %s%s%s: %s", name, string,
+           with, addr);
+  }
+}
+
+tor_tls_t *
+tor_tls_new(tor_socket_t sock, int is_server)
+{
+  (void)sock;
+  tor_tls_context_t *ctx = tor_tls_context_get(is_server);
+
+  PRFileDesc *tcp = NULL;
+  if (SOCKET_OK(sock)) {
+    tcp = PR_ImportTCPSocket(sock);
+  } else {
+    tcp = PR_NewTCPSocket();
+  }
+
+  if (!tcp)
+    return NULL;
+
+  PRFileDesc *count = tor_wrap_prfiledesc_with_byte_counter(tcp);
+  if (! count)
+    return NULL;
+
+  PRFileDesc *ssl = SSL_ImportFD(ctx->ctx, count);
+  if (!ssl) {
+    PR_Close(tcp);
+    return NULL;
+  }
+
+  tor_tls_t *tls = tor_malloc_zero(sizeof(tor_tls_t));
+  tls->magic = TOR_TLS_MAGIC;
+  tls->context = ctx;
+  tor_tls_context_incref(ctx);
+  tls->ssl = ssl;
+  tls->socket = sock;
+  tls->state = TOR_TLS_ST_HANDSHAKE;
+  tls->isServer = !!is_server;
+
+  if (!is_server) {
+    /* Set a random SNI */
+    char *fake_hostname = crypto_random_hostname(4,25, "www.",".com");
+    SSL_SetURL(tls->ssl, fake_hostname);
+    tor_free(fake_hostname);
+  }
+  SECStatus s = SSL_ResetHandshake(ssl, is_server ? PR_TRUE : PR_FALSE);
+  if (s != SECSuccess) {
+    tls_log_errors(tls, LOG_WARN, LD_CRYPTO, "resetting handshake state");
+  }
+
+  return tls;
+}
+
+void
+tor_tls_set_renegotiate_callback(tor_tls_t *tls,
+                                 void (*cb)(tor_tls_t *, void *arg),
+                                 void *arg)
+{
+  tor_assert(tls);
+  (void)cb;
+  (void)arg;
+
+  /* We don't support renegotiation-based TLS with NSS. */
+}
+
+/**
+ * Tell the TLS library that the underlying socket for <b>tls</b> has been
+ * closed, and the library should not attempt to free that socket itself.
+ */
+void
+tor_tls_release_socket(tor_tls_t *tls)
+{
+  if (! tls)
+    return;
+
+  /* NSS doesn't have the equivalent of BIO_NO_CLOSE.  If you replace the
+   * fd with something that's invalid, it causes a memory leak in PR_Close.
+   *
+   * If there were a way to put the PRFileDesc into the CLOSED state, that
+   * would prevent it from closing its fd -- but there doesn't seem to be a
+   * supported way to do that either.
+   *
+   * So instead: we make a new sacrificial socket, and replace the original
+   * socket with that one. This seems to be the best we can do, until we
+   * redesign the mainloop code enough to make this function unnecessary.
+   */
+  tor_socket_t sock =
+    tor_open_socket_nonblocking(AF_INET, SOCK_STREAM, IPPROTO_TCP);
+  if (! SOCKET_OK(sock)) {
+    log_warn(LD_NET, "Out of sockets when trying to shut down an NSS "
+             "connection");
+    return;
+  }
+
+  PRFileDesc *tcp = PR_GetIdentitiesLayer(tls->ssl, PR_NSPR_IO_LAYER);
+  if (BUG(! tcp)) {
+    tor_close_socket(sock);
+    return;
+  }
+
+  PR_ChangeFileDescNativeHandle(tcp, sock);
+  /* Tell our socket accounting layer that we don't own this socket any more:
+   * NSS is about to free it for us. */
+  tor_release_socket_ownership(sock);
+}
+
+void
+tor_tls_impl_free_(tor_tls_impl_t *tls)
+{
+  // XXXX This will close the underlying fd, which our OpenSSL version does
+  // not do!
+  if (!tls)
+    return;
+
+  PR_Close(tls);
+}
+
+int
+tor_tls_peer_has_cert(tor_tls_t *tls)
+{
+  CERTCertificate *cert = SSL_PeerCertificate(tls->ssl);
+  int result = (cert != NULL);
+  CERT_DestroyCertificate(cert);
+  return result;
+}
+
+MOCK_IMPL(tor_x509_cert_t *,
+tor_tls_get_peer_cert,(tor_tls_t *tls))
+{
+  CERTCertificate *cert = SSL_PeerCertificate(tls->ssl);
+  if (cert)
+    return tor_x509_cert_new(cert);
+  else
+    return NULL;
+}
+
+MOCK_IMPL(tor_x509_cert_t *,
+tor_tls_get_own_cert,(tor_tls_t *tls))
+{
+  tor_assert(tls);
+  CERTCertificate *cert = SSL_LocalCertificate(tls->ssl);
+  if (cert)
+    return tor_x509_cert_new(cert);
+  else
+    return NULL;
+}
+
+MOCK_IMPL(int,
+tor_tls_read, (tor_tls_t *tls, char *cp, size_t len))
+{
+  tor_assert(tls);
+  tor_assert(cp);
+  tor_assert(len < INT_MAX);
+
+  PRInt32 rv = PR_Read(tls->ssl, cp, (int)len);
+  // log_debug(LD_NET, "PR_Read(%zu) returned %d", n, (int)rv);
+  if (rv > 0) {
+    return rv;
+  }
+  if (rv == 0)
+    return TOR_TLS_CLOSE;
+  PRErrorCode err = PORT_GetError();
+  if (err == PR_WOULD_BLOCK_ERROR) {
+    return TOR_TLS_WANTREAD; // XXXX ????
+  } else {
+    tls_log_errors(tls, LOG_NOTICE, LD_CRYPTO, "reading"); // XXXX
+    return TOR_TLS_ERROR_MISC; // ????
+  }
+}
+
+int
+tor_tls_write(tor_tls_t *tls, const char *cp, size_t n)
+{
+  tor_assert(tls);
+  tor_assert(cp || n == 0);
+  tor_assert(n < INT_MAX);
+
+  PRInt32 rv = PR_Write(tls->ssl, cp, (int)n);
+  // log_debug(LD_NET, "PR_Write(%zu) returned %d", n, (int)rv);
+  if (rv > 0) {
+    return rv;
+  }
+  if (rv == 0)
+    return TOR_TLS_ERROR_MISC;
+  PRErrorCode err = PORT_GetError();
+
+  if (err == PR_WOULD_BLOCK_ERROR) {
+    return TOR_TLS_WANTWRITE; // XXXX ????
+  } else {
+    tls_log_errors(tls, LOG_NOTICE, LD_CRYPTO, "writing"); // XXXX
+    return TOR_TLS_ERROR_MISC; // ????
+  }
+}
+
+int
+tor_tls_handshake(tor_tls_t *tls)
+{
+  tor_assert(tls);
+  tor_assert(tls->state == TOR_TLS_ST_HANDSHAKE);
+
+  SECStatus s = SSL_ForceHandshake(tls->ssl);
+  if (s == SECSuccess) {
+    tls->state = TOR_TLS_ST_OPEN;
+    log_debug(LD_NET, "SSL handshake is supposedly complete.");
+    return tor_tls_finish_handshake(tls);
+  }
+  if (PORT_GetError() == PR_WOULD_BLOCK_ERROR)
+    return TOR_TLS_WANTREAD; /* XXXX What about wantwrite? */
+
+  return TOR_TLS_ERROR_MISC; // XXXX
+}
+
+int
+tor_tls_finish_handshake(tor_tls_t *tls)
+{
+  tor_assert(tls);
+  // We don't need to do any of the weird handshake nonsense stuff on NSS,
+  // since we only support recent handshakes.
+  return TOR_TLS_DONE;
+}
+
+void
+tor_tls_unblock_renegotiation(tor_tls_t *tls)
+{
+  tor_assert(tls);
+  /* We don't support renegotiation with NSS. */
+}
+
+void
+tor_tls_block_renegotiation(tor_tls_t *tls)
+{
+  tor_assert(tls);
+  /* We don't support renegotiation with NSS. */
+}
+
+void
+tor_tls_assert_renegotiation_unblocked(tor_tls_t *tls)
+{
+  tor_assert(tls);
+  /* We don't support renegotiation with NSS. */
+}
+
+int
+tor_tls_get_pending_bytes(tor_tls_t *tls)
+{
+  tor_assert(tls);
+  int n = SSL_DataPending(tls->ssl);
+  if (n < 0) {
+    tls_log_errors(tls, LOG_WARN, LD_CRYPTO, "looking up pending bytes");
+    return 0;
+  }
+  return (int)n;
+}
+
+size_t
+tor_tls_get_forced_write_size(tor_tls_t *tls)
+{
+  tor_assert(tls);
+  /* NSS doesn't have the same "forced write" restriction as openssl. */
+  return 0;
+}
+
+void
+tor_tls_get_n_raw_bytes(tor_tls_t *tls,
+                        size_t *n_read, size_t *n_written)
+{
+  tor_assert(tls);
+  tor_assert(n_read);
+  tor_assert(n_written);
+  uint64_t r, w;
+  if (tor_get_prfiledesc_byte_counts(tls->ssl, &r, &w) < 0) {
+    *n_read = *n_written = 0;
+    return;
+  }
+
+  *n_read = (size_t)(r - tls->last_read_count);
+  *n_written = (size_t)(w - tls->last_write_count);
+
+  tls->last_read_count = r;
+  tls->last_write_count = w;
+}
+
+int
+tor_tls_get_buffer_sizes(tor_tls_t *tls,
+                         size_t *rbuf_capacity, size_t *rbuf_bytes,
+                         size_t *wbuf_capacity, size_t *wbuf_bytes)
+{
+  tor_assert(tls);
+  tor_assert(rbuf_capacity);
+  tor_assert(rbuf_bytes);
+  tor_assert(wbuf_capacity);
+  tor_assert(wbuf_bytes);
+
+  /* This is an acceptable way to say "we can't measure this." */
+  return -1;
+}
+
+MOCK_IMPL(double,
+tls_get_write_overhead_ratio, (void))
+{
+  /* XXX We don't currently have a way to measure this in NSS; we could do that
+   * XXX with a PRIO layer, but it'll take a little coding. */
+  return 0.95;
+}
+
+int
+tor_tls_used_v1_handshake(tor_tls_t *tls)
+{
+  tor_assert(tls);
+  /* We don't support or allow the V1 handshake with NSS.
+   */
+  return 0;
+}
+
+int
+tor_tls_server_got_renegotiate(tor_tls_t *tls)
+{
+  tor_assert(tls);
+  return 0; /* We don't support renegotiation with NSS */
+}
+
+MOCK_IMPL(int,
+tor_tls_cert_matches_key,(const tor_tls_t *tls,
+                          const struct tor_x509_cert_t *cert))
+{
+  tor_assert(tls);
+  tor_assert(cert);
+  int rv = 0;
+
+  CERTCertificate *peercert = SSL_PeerCertificate(tls->ssl);
+  if (!peercert)
+    goto done;
+  CERTSubjectPublicKeyInfo *peer_info = &peercert->subjectPublicKeyInfo;
+  CERTSubjectPublicKeyInfo *cert_info = &cert->cert->subjectPublicKeyInfo;
+  rv = SECOID_CompareAlgorithmID(&peer_info->algorithm,
+                                 &cert_info->algorithm) == 0 &&
+       SECITEM_ItemsAreEqual(&peer_info->subjectPublicKey,
+                             &cert_info->subjectPublicKey);
+
+ done:
+  if (peercert)
+    CERT_DestroyCertificate(peercert);
+  return rv;
+}
+
+MOCK_IMPL(int,
+tor_tls_get_tlssecrets,(tor_tls_t *tls, uint8_t *secrets_out))
+{
+  tor_assert(tls);
+  tor_assert(secrets_out);
+
+  /* There's no way to get this information out of NSS. */
+
+  return -1;
+}
+
+MOCK_IMPL(int,
+tor_tls_export_key_material,(tor_tls_t *tls, uint8_t *secrets_out,
+                             const uint8_t *context,
+                             size_t context_len,
+                             const char *label))
+{
+  tor_assert(tls);
+  tor_assert(secrets_out);
+  tor_assert(context);
+  tor_assert(label);
+  tor_assert(strlen(label) <= UINT_MAX);
+  tor_assert(context_len <= UINT_MAX);
+
+  SECStatus s;
+  /* Make sure that the error code is set here, so that we can be sure that
+   * any error code set after a failure was in fact caused by
+   * SSL_ExportKeyingMaterial. */
+  PR_SetError(PR_UNKNOWN_ERROR, 0);
+  s = SSL_ExportKeyingMaterial(tls->ssl,
+                               label, (unsigned)strlen(label),
+                               PR_TRUE, context, (unsigned)context_len,
+                               secrets_out, DIGEST256_LEN);
+  if (s != SECSuccess) {
+    tls_log_errors(tls, LOG_WARN, LD_CRYPTO,
+                   "exporting key material for a TLS handshake");
+  }
+
+  return (s == SECSuccess) ? 0 : -1;
+}
+
+const char *
+tor_tls_get_ciphersuite_name(tor_tls_t *tls)
+{
+  tor_assert(tls);
+
+  SSLChannelInfo channel_info;
+  SSLCipherSuiteInfo cipher_info;
+
+  memset(&channel_info, 0, sizeof(channel_info));
+  memset(&cipher_info, 0, sizeof(cipher_info));
+
+  SECStatus s = SSL_GetChannelInfo(tls->ssl,
+                                   &channel_info, sizeof(channel_info));
+  if (s != SECSuccess)
+    return NULL;
+
+  s = SSL_GetCipherSuiteInfo(channel_info.cipherSuite,
+                             &cipher_info, sizeof(cipher_info));
+  if (s != SECSuccess)
+    return NULL;
+
+  return cipher_info.cipherSuiteName;
+}
+
+/** The group we should use for ecdhe when none was selected. */
+#define SEC_OID_TOR_DEFAULT_ECDHE_GROUP SEC_OID_ANSIX962_EC_PRIME256V1
+
+int
+evaluate_ecgroup_for_tls(const char *ecgroup)
+{
+  SECOidTag tag;
+
+  if (!ecgroup)
+    tag = SEC_OID_TOR_DEFAULT_ECDHE_GROUP;
+  else if (!strcasecmp(ecgroup, "P256"))
+    tag = SEC_OID_ANSIX962_EC_PRIME256V1;
+  else if (!strcasecmp(ecgroup, "P224"))
+    tag = SEC_OID_SECG_EC_SECP224R1;
+  else
+    return 0;
+
+  /* I don't think we need any additional tests here for NSS */
+  (void) tag;
+
+  return 1;
+}
+
+static SECStatus
+always_accept_cert_cb(void *arg, PRFileDesc *ssl, PRBool checkSig,
+                      PRBool isServer)
+{
+  (void)arg;
+  (void)ssl;
+  (void)checkSig;
+  (void)isServer;
+  return SECSuccess;
+}
diff --git a/src/common/tortls.c b/src/lib/tls/tortls_openssl.c
index f79969d0da..80b0df301f 100644
--- a/src/common/tortls.c
+++ b/src/lib/tls/tortls_openssl.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,21 +17,29 @@
 #include "orconfig.h"
 
 #define TORTLS_PRIVATE
+#define TORTLS_OPENSSL_PRIVATE
+#define TOR_X509_PRIVATE
 
-#include <assert.h>
-#ifdef _WIN32 /*wrkard for dtls1.h >= 0.9.8m of "#include <winsock.h>"*/
+#ifdef _WIN32
+  /* We need to include these here, or else the dtls1.h header will include
+   * <winsock.h> and mess things up, in at least some openssl versions. */
   #include <winsock2.h>
   #include <ws2tcpip.h>
 #endif
 
-#include "compat.h"
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/crypt_ops/compat_openssl.h"
+#include "lib/tls/x509.h"
+#include "lib/tls/x509_internal.h"
 
 /* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in
  * srtp.h. Suppress the GCC warning so we can build with -Wredundant-decl. */
 DISABLE_GCC_WARNING(redundant-decls)
 
 #include <openssl/opensslv.h>
-#include "crypto.h"
 
 #ifdef OPENSSL_NO_EC
 #error "We require OpenSSL with ECC support"
@@ -48,25 +56,28 @@ DISABLE_GCC_WARNING(redundant-decls)
 
 ENABLE_GCC_WARNING(redundant-decls)
 
-#define TORTLS_PRIVATE
-#include "tortls.h"
-#include "util.h"
-#include "torlog.h"
-#include "container.h"
+#include "lib/tls/tortls.h"
+#include "lib/tls/tortls_st.h"
+#include "lib/tls/tortls_internal.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/container/smartlist.h"
+#include "lib/string/compat_string.h"
+#include "lib/string/printf.h"
+#include "lib/net/socket.h"
+#include "lib/intmath/cmp.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/encoding/time_fmt.h"
+
+#include <stdlib.h>
 #include <string.h>
 
-#define X509_get_notBefore_const(cert) \
-  ((const ASN1_TIME*) X509_get_notBefore((X509 *)cert))
-#define X509_get_notAfter_const(cert) \
-  ((const ASN1_TIME*) X509_get_notAfter((X509 *)cert))
+#include "lib/arch/bytes.h"
 
 /* Copied from or.h */
 #define LEGAL_NICKNAME_CHARACTERS \
   "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
 
-/** How long do identity certificates live? (sec) */
-#define IDENTITY_CERT_LIFETIME  (365*24*60*60)
-
 #define ADDR(tls) (((tls) && (tls)->address) ? tls->address : "peer")
 
 #if OPENSSL_VERSION_NUMBER <  OPENSSL_V(1,0,0,'f')
@@ -75,7 +86,7 @@ ENABLE_GCC_WARNING(redundant-decls)
  * SSL3 safely at the same time.
  */
 #define DISABLE_SSL3_HANDSHAKE
-#endif
+#endif /* OPENSSL_VERSION_NUMBER <  OPENSSL_V(1,0,0,'f') */
 
 /* We redefine these so that we can run correctly even if the vendor gives us
  * a version of OpenSSL that does not match its header files.  (Apple: I am
@@ -88,6 +99,9 @@ ENABLE_GCC_WARNING(redundant-decls)
 #define SSL3_FLAGS_ALLOW_UNSAFE_LEGACY_RENEGOTIATION 0x0010
 #endif
 
+/** Set to true iff openssl bug 7712 has been detected. */
+static int openssl_bug_7712_is_present = 0;
+
 /** Return values for tor_tls_classify_client_ciphers.
  *
  * @{
@@ -111,7 +125,7 @@ ENABLE_GCC_WARNING(redundant-decls)
 STATIC int tor_tls_object_ex_data_index = -1;
 
 /** Helper: Allocate tor_tls_object_ex_data_index. */
-STATIC void
+void
 tor_tls_allocate_tor_tls_object_ex_data_index(void)
 {
   if (tor_tls_object_ex_data_index == -1) {
@@ -123,7 +137,7 @@ tor_tls_allocate_tor_tls_object_ex_data_index(void)
 
 /** Helper: given a SSL* pointer, return the tor_tls_t object using that
  * pointer. */
-STATIC tor_tls_t *
+tor_tls_t *
 tor_tls_get_by_ssl(const SSL *ssl)
 {
   tor_tls_t *result = SSL_get_ex_data(ssl, tor_tls_object_ex_data_index);
@@ -132,20 +146,6 @@ tor_tls_get_by_ssl(const SSL *ssl)
   return result;
 }
 
-static void tor_tls_context_decref(tor_tls_context_t *ctx);
-static void tor_tls_context_incref(tor_tls_context_t *ctx);
-
-static int check_cert_lifetime_internal(int severity, const X509 *cert,
-                                   int past_tolerance, int future_tolerance);
-
-/** Global TLS contexts. We keep them here because nobody else needs
- * to touch them.
- *
- * @{ */
-STATIC tor_tls_context_t *server_tls_context = NULL;
-STATIC tor_tls_context_t *client_tls_context = NULL;
-/**@}*/
-
 /** True iff tor_tls_init() has been called. */
 static int tls_library_is_initialized = 0;
 
@@ -239,7 +239,7 @@ tor_tls_log_one_error(tor_tls_t *tls, unsigned long err,
 /** Log all pending tls errors at level <b>severity</b> in log domain
  * <b>domain</b>.  Use <b>doing</b> to describe our current activities.
  */
-STATIC void
+void
 tls_log_errors(tor_tls_t *tls, int severity, int domain, const char *doing)
 {
   unsigned long err;
@@ -249,46 +249,6 @@ tls_log_errors(tor_tls_t *tls, int severity, int domain, const char *doing)
   }
 }
 
-/** Convert an errno (or a WSAerrno on windows) into a TOR_TLS_* error
- * code. */
-STATIC int
-tor_errno_to_tls_error(int e)
-{
-  switch (e) {
-    case SOCK_ERRNO(ECONNRESET): // most common
-      return TOR_TLS_ERROR_CONNRESET;
-    case SOCK_ERRNO(ETIMEDOUT):
-      return TOR_TLS_ERROR_TIMEOUT;
-    case SOCK_ERRNO(EHOSTUNREACH):
-    case SOCK_ERRNO(ENETUNREACH):
-      return TOR_TLS_ERROR_NO_ROUTE;
-    case SOCK_ERRNO(ECONNREFUSED):
-      return TOR_TLS_ERROR_CONNREFUSED; // least common
-    default:
-      return TOR_TLS_ERROR_MISC;
-  }
-}
-
-/** Given a TOR_TLS_* error code, return a string equivalent. */
-const char *
-tor_tls_err_to_string(int err)
-{
-  if (err >= 0)
-    return "[Not an error.]";
-  switch (err) {
-    case TOR_TLS_ERROR_MISC: return "misc error";
-    case TOR_TLS_ERROR_IO: return "unexpected close";
-    case TOR_TLS_ERROR_CONNREFUSED: return "connection refused";
-    case TOR_TLS_ERROR_CONNRESET: return "connection reset";
-    case TOR_TLS_ERROR_NO_ROUTE: return "host unreachable";
-    case TOR_TLS_ERROR_TIMEOUT: return "connection timed out";
-    case TOR_TLS_CLOSE: return "closed";
-    case TOR_TLS_WANTREAD: return "want to read";
-    case TOR_TLS_WANTWRITE: return "want to write";
-    default: return "(unknown error code)";
-  }
-}
-
 #define CATCH_SYSCALL 1
 #define CATCH_ZERO    2
 
@@ -302,7 +262,7 @@ tor_tls_err_to_string(int err)
  * If an error has occurred, log it at level <b>severity</b> and describe the
  * current action as <b>doing</b>.
  */
-STATIC int
+int
 tor_tls_get_error(tor_tls_t *tls, int r, int extra,
                   const char *doing, int severity, int domain)
 {
@@ -347,14 +307,18 @@ tor_tls_get_error(tor_tls_t *tls, int r, int extra,
 
 /** Initialize OpenSSL, unless it has already been initialized.
  */
-static void
+void
 tor_tls_init(void)
 {
   check_no_tls_errors();
 
   if (!tls_library_is_initialized) {
+#ifdef OPENSSL_1_1_API
+    OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS, NULL);
+#else
     SSL_library_init();
     SSL_load_error_strings();
+#endif
 
 #if (SIZEOF_VOID_P >= 8 &&                              \
      OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,0,1))
@@ -388,7 +352,7 @@ tor_tls_init(void)
                    "when configuring it) would make ECDH much faster.");
     }
     /* LCOV_EXCL_STOP */
-#endif
+#endif /* (SIZEOF_VOID_P >= 8 &&                              ... */
 
     tor_tls_allocate_tor_tls_object_ex_data_index();
 
@@ -396,29 +360,11 @@ tor_tls_init(void)
   }
 }
 
-/** Free all global TLS structures. */
-void
-tor_tls_free_all(void)
-{
-  check_no_tls_errors();
-
-  if (server_tls_context) {
-    tor_tls_context_t *ctx = server_tls_context;
-    server_tls_context = NULL;
-    tor_tls_context_decref(ctx);
-  }
-  if (client_tls_context) {
-    tor_tls_context_t *ctx = client_tls_context;
-    client_tls_context = NULL;
-    tor_tls_context_decref(ctx);
-  }
-}
-
 /** We need to give OpenSSL a callback to verify certificates. This is
  * it: We always accept peer certs and complete the handshake.  We
  * don't validate them until later.
  */
-STATIC int
+int
 always_accept_verify_cb(int preverify_ok,
                         X509_STORE_CTX *x509_ctx)
 {
@@ -427,127 +373,6 @@ always_accept_verify_cb(int preverify_ok,
   return 1;
 }
 
-/** Return a newly allocated X509 name with commonName <b>cname</b>. */
-static X509_NAME *
-tor_x509_name_new(const char *cname)
-{
-  int nid;
-  X509_NAME *name;
-  /* LCOV_EXCL_BR_START : these branches will only fail on OOM errors */
-  if (!(name = X509_NAME_new()))
-    return NULL;
-  if ((nid = OBJ_txt2nid("commonName")) == NID_undef) goto error;
-  if (!(X509_NAME_add_entry_by_NID(name, nid, MBSTRING_ASC,
-                                   (unsigned char*)cname, -1, -1, 0)))
-    goto error;
-  /* LCOV_EXCL_BR_STOP */
-  return name;
- error:
-  /* LCOV_EXCL_START : these lines will only execute on out of memory errors*/
-  X509_NAME_free(name);
-  return NULL;
-  /* LCOV_EXCL_STOP */
-}
-
-/** Generate and sign an X509 certificate with the public key <b>rsa</b>,
- * signed by the private key <b>rsa_sign</b>.  The commonName of the
- * certificate will be <b>cname</b>; the commonName of the issuer will be
- * <b>cname_sign</b>. The cert will be valid for <b>cert_lifetime</b>
- * seconds, starting from some time in the past.
- *
- * Return a certificate on success, NULL on failure.
- */
-MOCK_IMPL(STATIC X509 *,
-          tor_tls_create_certificate,(crypto_pk_t *rsa,
-                                      crypto_pk_t *rsa_sign,
-                                      const char *cname,
-                                      const char *cname_sign,
-                                      unsigned int cert_lifetime))
-{
-  /* OpenSSL generates self-signed certificates with random 64-bit serial
-   * numbers, so let's do that too. */
-#define SERIAL_NUMBER_SIZE 8
-
-  time_t start_time, end_time;
-  BIGNUM *serial_number = NULL;
-  unsigned char serial_tmp[SERIAL_NUMBER_SIZE];
-  EVP_PKEY *sign_pkey = NULL, *pkey=NULL;
-  X509 *x509 = NULL;
-  X509_NAME *name = NULL, *name_issuer=NULL;
-
-  tor_tls_init();
-
-  /* Make sure we're part-way through the certificate lifetime, rather
-   * than having it start right now. Don't choose quite uniformly, since
-   * then we might pick a time where we're about to expire. Lastly, be
-   * sure to start on a day boundary. */
-  time_t now = time(NULL);
-  start_time = crypto_rand_time_range(now - cert_lifetime, now) + 2*24*3600;
-  start_time -= start_time % (24*3600);
-
-  tor_assert(rsa);
-  tor_assert(cname);
-  tor_assert(rsa_sign);
-  tor_assert(cname_sign);
-  if (!(sign_pkey = crypto_pk_get_evp_pkey_(rsa_sign,1)))
-    goto error;
-  if (!(pkey = crypto_pk_get_evp_pkey_(rsa,0)))
-    goto error;
-  if (!(x509 = X509_new()))
-    goto error;
-  if (!(X509_set_version(x509, 2)))
-    goto error;
-
-  { /* our serial number is 8 random bytes. */
-    crypto_rand((char *)serial_tmp, sizeof(serial_tmp));
-    if (!(serial_number = BN_bin2bn(serial_tmp, sizeof(serial_tmp), NULL)))
-      goto error;
-    if (!(BN_to_ASN1_INTEGER(serial_number, X509_get_serialNumber(x509))))
-      goto error;
-  }
-
-  if (!(name = tor_x509_name_new(cname)))
-    goto error;
-  if (!(X509_set_subject_name(x509, name)))
-    goto error;
-  if (!(name_issuer = tor_x509_name_new(cname_sign)))
-    goto error;
-  if (!(X509_set_issuer_name(x509, name_issuer)))
-    goto error;
-
-  if (!X509_time_adj(X509_get_notBefore(x509),0,&start_time))
-    goto error;
-  end_time = start_time + cert_lifetime;
-  if (!X509_time_adj(X509_get_notAfter(x509),0,&end_time))
-    goto error;
-  if (!X509_set_pubkey(x509, pkey))
-    goto error;
-  if (!X509_sign(x509, sign_pkey, EVP_sha1()))
-    goto error;
-
-  goto done;
- error:
-  if (x509) {
-    X509_free(x509);
-    x509 = NULL;
-  }
- done:
-  tls_log_errors(NULL, LOG_WARN, LD_NET, "generating certificate");
-  if (sign_pkey)
-    EVP_PKEY_free(sign_pkey);
-  if (pkey)
-    EVP_PKEY_free(pkey);
-  if (serial_number)
-    BN_clear_free(serial_number);
-  if (name)
-    X509_NAME_free(name);
-  if (name_issuer)
-    X509_NAME_free(name_issuer);
-  return x509;
-
-#undef SERIAL_NUMBER_SIZE
-}
-
 /** List of ciphers that servers should select from when the client might be
  * claiming extra unsupported ciphers in order to avoid fingerprinting.  */
 static const char SERVER_CIPHER_LIST[] =
@@ -605,6 +430,12 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] =
 #ifdef TLS1_TXT_DHE_RSA_WITH_AES_128_GCM_SHA256
        TLS1_TXT_DHE_RSA_WITH_AES_128_GCM_SHA256 ":"
 #endif
+#ifdef TLS1_TXT_DHE_RSA_WITH_AES_256_CCM
+       TLS1_TXT_DHE_RSA_WITH_AES_256_CCM ":"
+#endif
+#ifdef TLS1_TXT_DHE_RSA_WITH_AES_128_CCM
+       TLS1_TXT_DHE_RSA_WITH_AES_128_CCM ":"
+#endif
 #ifdef TLS1_TXT_DHE_RSA_WITH_AES_256_SHA256
        TLS1_TXT_DHE_RSA_WITH_AES_256_SHA256 ":"
 #endif
@@ -614,8 +445,14 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] =
        /* Required */
        TLS1_TXT_DHE_RSA_WITH_AES_256_SHA ":"
        /* Required */
-       TLS1_TXT_DHE_RSA_WITH_AES_128_SHA
-       ;
+       TLS1_TXT_DHE_RSA_WITH_AES_128_SHA ":"
+#ifdef TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305
+       TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305 ":"
+#endif
+#ifdef TLS1_TXT_DHE_RSA_WITH_CHACHA20_POLY1305
+       TLS1_TXT_DHE_RSA_WITH_CHACHA20_POLY1305
+#endif
+  ;
 
 /* Note: to set up your own private testing network with link crypto
  * disabled, set your Tors' cipher list to
@@ -635,222 +472,6 @@ static const char CLIENT_CIPHER_LIST[] =
 #undef CIPHER
 #undef XCIPHER
 
-/** Free all storage held in <b>cert</b> */
-void
-tor_x509_cert_free(tor_x509_cert_t *cert)
-{
-  if (! cert)
-    return;
-  if (cert->cert)
-    X509_free(cert->cert);
-  tor_free(cert->encoded);
-  memwipe(cert, 0x03, sizeof(*cert));
-  /* LCOV_EXCL_BR_START since cert will never be NULL here */
-  tor_free(cert);
-  /* LCOV_EXCL_BR_STOP */
-}
-
-/**
- * Allocate a new tor_x509_cert_t to hold the certificate "x509_cert".
- *
- * Steals a reference to x509_cert.
- */
-MOCK_IMPL(STATIC tor_x509_cert_t *,
-          tor_x509_cert_new,(X509 *x509_cert))
-{
-  tor_x509_cert_t *cert;
-  EVP_PKEY *pkey;
-  RSA *rsa;
-  int length;
-  unsigned char *buf = NULL;
-
-  if (!x509_cert)
-    return NULL;
-
-  length = i2d_X509(x509_cert, &buf);
-  cert = tor_malloc_zero(sizeof(tor_x509_cert_t));
-  if (length <= 0 || buf == NULL) {
-    /* LCOV_EXCL_START for the same reason as the exclusion above */
-    tor_free(cert);
-    log_err(LD_CRYPTO, "Couldn't get length of encoded x509 certificate");
-    X509_free(x509_cert);
-    return NULL;
-    /* LCOV_EXCL_STOP */
-  }
-  cert->encoded_len = (size_t) length;
-  cert->encoded = tor_malloc(length);
-  memcpy(cert->encoded, buf, length);
-  OPENSSL_free(buf);
-
-  cert->cert = x509_cert;
-
-  crypto_common_digests(&cert->cert_digests,
-                    (char*)cert->encoded, cert->encoded_len);
-
-  if ((pkey = X509_get_pubkey(x509_cert)) &&
-      (rsa = EVP_PKEY_get1_RSA(pkey))) {
-    crypto_pk_t *pk = crypto_new_pk_from_rsa_(rsa);
-    crypto_pk_get_common_digests(pk, &cert->pkey_digests);
-    cert->pkey_digests_set = 1;
-    crypto_pk_free(pk);
-    EVP_PKEY_free(pkey);
-  }
-
-  return cert;
-}
-
-/** Return a new copy of <b>cert</b>. */
-tor_x509_cert_t *
-tor_x509_cert_dup(const tor_x509_cert_t *cert)
-{
-  tor_assert(cert);
-  X509 *x509 = cert->cert;
-  return tor_x509_cert_new(X509_dup(x509));
-}
-
-/** Read a DER-encoded X509 cert, of length exactly <b>certificate_len</b>,
- * from a <b>certificate</b>.  Return a newly allocated tor_x509_cert_t on
- * success and NULL on failure. */
-tor_x509_cert_t *
-tor_x509_cert_decode(const uint8_t *certificate, size_t certificate_len)
-{
-  X509 *x509;
-  const unsigned char *cp = (const unsigned char *)certificate;
-  tor_x509_cert_t *newcert;
-  tor_assert(certificate);
-  check_no_tls_errors();
-
-  if (certificate_len > INT_MAX)
-    goto err;
-
-  x509 = d2i_X509(NULL, &cp, (int)certificate_len);
-
-  if (!x509)
-    goto err; /* Couldn't decode */
-  if (cp - certificate != (int)certificate_len) {
-    X509_free(x509);
-    goto err; /* Didn't use all the bytes */
-  }
-  newcert = tor_x509_cert_new(x509);
-  if (!newcert) {
-    goto err;
-  }
-  if (newcert->encoded_len != certificate_len ||
-      fast_memneq(newcert->encoded, certificate, certificate_len)) {
-    /* Cert wasn't in DER */
-    tor_x509_cert_free(newcert);
-    goto err;
-  }
-  return newcert;
- err:
-  tls_log_errors(NULL, LOG_INFO, LD_CRYPTO, "decoding a certificate");
-  return NULL;
-}
-
-/** Set *<b>encoded_out</b> and *<b>size_out</b> to <b>cert</b>'s encoded DER
- * representation and length, respectively. */
-void
-tor_x509_cert_get_der(const tor_x509_cert_t *cert,
-                 const uint8_t **encoded_out, size_t *size_out)
-{
-  tor_assert(cert);
-  tor_assert(encoded_out);
-  tor_assert(size_out);
-  *encoded_out = cert->encoded;
-  *size_out = cert->encoded_len;
-}
-
-/** Return a set of digests for the public key in <b>cert</b>, or NULL if this
- * cert's public key is not one we know how to take the digest of. */
-const common_digests_t *
-tor_x509_cert_get_id_digests(const tor_x509_cert_t *cert)
-{
-  if (cert->pkey_digests_set)
-    return &cert->pkey_digests;
-  else
-    return NULL;
-}
-
-/** Return a set of digests for the public key in <b>cert</b>. */
-const common_digests_t *
-tor_x509_cert_get_cert_digests(const tor_x509_cert_t *cert)
-{
-  return &cert->cert_digests;
-}
-
-/** Remove a reference to <b>ctx</b>, and free it if it has no more
- * references. */
-static void
-tor_tls_context_decref(tor_tls_context_t *ctx)
-{
-  tor_assert(ctx);
-  if (--ctx->refcnt == 0) {
-    SSL_CTX_free(ctx->ctx);
-    tor_x509_cert_free(ctx->my_link_cert);
-    tor_x509_cert_free(ctx->my_id_cert);
-    tor_x509_cert_free(ctx->my_auth_cert);
-    crypto_pk_free(ctx->link_key);
-    crypto_pk_free(ctx->auth_key);
-    /* LCOV_EXCL_BR_START since ctx will never be NULL here */
-    tor_free(ctx);
-    /* LCOV_EXCL_BR_STOP */
-  }
-}
-
-/** Set *<b>link_cert_out</b> and *<b>id_cert_out</b> to the link certificate
- * and ID certificate that we're currently using for our V3 in-protocol
- * handshake's certificate chain.  If <b>server</b> is true, provide the certs
- * that we use in server mode; otherwise, provide the certs that we use in
- * client mode. */
-int
-tor_tls_get_my_certs(int server,
-                     const tor_x509_cert_t **link_cert_out,
-                     const tor_x509_cert_t **id_cert_out)
-{
-  tor_tls_context_t *ctx = server ? server_tls_context : client_tls_context;
-  if (! ctx)
-    return -1;
-  if (link_cert_out)
-    *link_cert_out = server ? ctx->my_link_cert : ctx->my_auth_cert;
-  if (id_cert_out)
-    *id_cert_out = ctx->my_id_cert;
-  return 0;
-}
-
-/**
- * Return the authentication key that we use to authenticate ourselves as a
- * client in the V3 in-protocol handshake.
- */
-crypto_pk_t *
-tor_tls_get_my_client_auth_key(void)
-{
-  if (! client_tls_context)
-    return NULL;
-  return client_tls_context->auth_key;
-}
-
-/**
- * Return a newly allocated copy of the public key that a certificate
- * certifies.  Return NULL if the cert's key is not RSA.
- */
-crypto_pk_t *
-tor_tls_cert_get_key(tor_x509_cert_t *cert)
-{
-  crypto_pk_t *result = NULL;
-  EVP_PKEY *pkey = X509_get_pubkey(cert->cert);
-  RSA *rsa;
-  if (!pkey)
-    return NULL;
-  rsa = EVP_PKEY_get1_RSA(pkey);
-  if (!rsa) {
-    EVP_PKEY_free(pkey);
-    return NULL;
-  }
-  result = crypto_new_pk_from_rsa_(rsa);
-  EVP_PKEY_free(pkey);
-  return result;
-}
-
 /** Return true iff the other side of <b>tls</b> has authenticated to us, and
  * the key certified in <b>cert</b> is the same as the key they used to do it.
  */
@@ -879,174 +500,12 @@ tor_tls_cert_matches_key,(const tor_tls_t *tls, const tor_x509_cert_t *cert))
   return result;
 }
 
-/** Check whether <b>cert</b> is well-formed, currently live, and correctly
- * signed by the public key in <b>signing_cert</b>.  If <b>check_rsa_1024</b>,
- * make sure that it has an RSA key with 1024 bits; otherwise, just check that
- * the key is long enough. Return 1 if the cert is good, and 0 if it's bad or
- * we couldn't check it. */
-int
-tor_tls_cert_is_valid(int severity,
-                      const tor_x509_cert_t *cert,
-                      const tor_x509_cert_t *signing_cert,
-                      int check_rsa_1024)
-{
-  check_no_tls_errors();
-  EVP_PKEY *cert_key;
-  int r, key_ok = 0;
-
-  if (!signing_cert || !cert)
-    goto bad;
-
-  EVP_PKEY *signing_key = X509_get_pubkey(signing_cert->cert);
-  if (!signing_key)
-    goto bad;
-  r = X509_verify(cert->cert, signing_key);
-  EVP_PKEY_free(signing_key);
-  if (r <= 0)
-    goto bad;
-
-  /* okay, the signature checked out right.  Now let's check the check the
-   * lifetime. */
-  if (check_cert_lifetime_internal(severity, cert->cert,
-                                   48*60*60, 30*24*60*60) < 0)
-    goto bad;
-
-  cert_key = X509_get_pubkey(cert->cert);
-  if (check_rsa_1024 && cert_key) {
-    RSA *rsa = EVP_PKEY_get1_RSA(cert_key);
-#ifdef OPENSSL_1_1_API
-    if (rsa && RSA_bits(rsa) == 1024)
-#else
-    if (rsa && BN_num_bits(rsa->n) == 1024)
-#endif
-      key_ok = 1;
-    if (rsa)
-      RSA_free(rsa);
-  } else if (cert_key) {
-    int min_bits = 1024;
-#ifdef EVP_PKEY_EC
-    if (EVP_PKEY_base_id(cert_key) == EVP_PKEY_EC)
-      min_bits = 128;
-#endif
-    if (EVP_PKEY_bits(cert_key) >= min_bits)
-      key_ok = 1;
-  }
-  EVP_PKEY_free(cert_key);
-  if (!key_ok)
-    goto bad;
-
-  /* XXXX compare DNs or anything? */
-
-  return 1;
- bad:
-  tls_log_errors(NULL, LOG_INFO, LD_CRYPTO, "checking a certificate");
-  return 0;
-}
-
-/** Increase the reference count of <b>ctx</b>. */
-static void
-tor_tls_context_incref(tor_tls_context_t *ctx)
-{
-  ++ctx->refcnt;
-}
-
-/** Create new global client and server TLS contexts.
- *
- * If <b>server_identity</b> is NULL, this will not generate a server
- * TLS context. If TOR_TLS_CTX_IS_PUBLIC_SERVER is set in <b>flags</b>, use
- * the same TLS context for incoming and outgoing connections, and
- * ignore <b>client_identity</b>. If one of TOR_TLS_CTX_USE_ECDHE_P{224,256}
- * is set in <b>flags</b>, use that ECDHE group if possible; otherwise use
- * the default ECDHE group. */
-int
-tor_tls_context_init(unsigned flags,
-                     crypto_pk_t *client_identity,
-                     crypto_pk_t *server_identity,
-                     unsigned int key_lifetime)
-{
-  int rv1 = 0;
-  int rv2 = 0;
-  const int is_public_server = flags & TOR_TLS_CTX_IS_PUBLIC_SERVER;
-  check_no_tls_errors();
-
-  if (is_public_server) {
-    tor_tls_context_t *new_ctx;
-    tor_tls_context_t *old_ctx;
-
-    tor_assert(server_identity != NULL);
-
-    rv1 = tor_tls_context_init_one(&server_tls_context,
-                                   server_identity,
-                                   key_lifetime, flags, 0);
-
-    if (rv1 >= 0) {
-      new_ctx = server_tls_context;
-      tor_tls_context_incref(new_ctx);
-      old_ctx = client_tls_context;
-      client_tls_context = new_ctx;
-
-      if (old_ctx != NULL) {
-        tor_tls_context_decref(old_ctx);
-      }
-    }
-  } else {
-    if (server_identity != NULL) {
-      rv1 = tor_tls_context_init_one(&server_tls_context,
-                                     server_identity,
-                                     key_lifetime,
-                                     flags,
-                                     0);
-    } else {
-      tor_tls_context_t *old_ctx = server_tls_context;
-      server_tls_context = NULL;
-
-      if (old_ctx != NULL) {
-        tor_tls_context_decref(old_ctx);
-      }
-    }
-
-    rv2 = tor_tls_context_init_one(&client_tls_context,
-                                   client_identity,
-                                   key_lifetime,
-                                   flags,
-                                   1);
-  }
-
-  tls_log_errors(NULL, LOG_WARN, LD_CRYPTO, "constructing a TLS context");
-  return MIN(rv1, rv2);
-}
-
-/** Create a new global TLS context.
- *
- * You can call this function multiple times.  Each time you call it,
- * it generates new certificates; all new connections will use
- * the new SSL context.
- */
-STATIC int
-tor_tls_context_init_one(tor_tls_context_t **ppcontext,
-                         crypto_pk_t *identity,
-                         unsigned int key_lifetime,
-                         unsigned int flags,
-                         int is_client)
+void
+tor_tls_context_impl_free_(struct ssl_ctx_st *ctx)
 {
-  tor_tls_context_t *new_ctx = tor_tls_context_new(identity,
-                                                   key_lifetime,
-                                                   flags,
-                                                   is_client);
-  tor_tls_context_t *old_ctx = *ppcontext;
-
-  if (new_ctx != NULL) {
-    *ppcontext = new_ctx;
-
-    /* Free the old context if one existed. */
-    if (old_ctx != NULL) {
-      /* This is safe even if there are open connections: we reference-
-       * count tor_tls_context_t objects. */
-      tor_tls_context_decref(old_ctx);
-    }
-  }
-
-  return ((new_ctx != NULL) ? 0 : -1);
+  if (!ctx)
+    return;
+  SSL_CTX_free(ctx);
 }
 
 /** The group we should use for ecdhe when none was selected. */
@@ -1056,61 +515,23 @@ tor_tls_context_init_one(tor_tls_context_t **ppcontext,
  * <b>identity</b> should be set to the identity key used to sign the
  * certificate.
  */
-STATIC tor_tls_context_t *
+tor_tls_context_t *
 tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime,
                     unsigned flags, int is_client)
 {
-  crypto_pk_t *rsa = NULL, *rsa_auth = NULL;
   EVP_PKEY *pkey = NULL;
   tor_tls_context_t *result = NULL;
-  X509 *cert = NULL, *idcert = NULL, *authcert = NULL;
-  char *nickname = NULL, *nn2 = NULL;
 
   tor_tls_init();
-  nickname = crypto_random_hostname(8, 20, "www.", ".net");
-#ifdef DISABLE_V3_LINKPROTO_SERVERSIDE
-  nn2 = crypto_random_hostname(8, 20, "www.", ".net");
-#else
-  nn2 = crypto_random_hostname(8, 20, "www.", ".com");
-#endif
-
-  /* Generate short-term RSA key for use with TLS. */
-  if (!(rsa = crypto_pk_new()))
-    goto error;
-  if (crypto_pk_generate_key(rsa)<0)
-    goto error;
-  if (!is_client) {
-    /* Generate short-term RSA key for use in the in-protocol ("v3")
-     * authentication handshake. */
-    if (!(rsa_auth = crypto_pk_new()))
-      goto error;
-    if (crypto_pk_generate_key(rsa_auth)<0)
-      goto error;
-    /* Create a link certificate signed by identity key. */
-    cert = tor_tls_create_certificate(rsa, identity, nickname, nn2,
-                                      key_lifetime);
-    /* Create self-signed certificate for identity key. */
-    idcert = tor_tls_create_certificate(identity, identity, nn2, nn2,
-                                        IDENTITY_CERT_LIFETIME);
-    /* Create an authentication certificate signed by identity key. */
-    authcert = tor_tls_create_certificate(rsa_auth, identity, nickname, nn2,
-                                          key_lifetime);
-    if (!cert || !idcert || !authcert) {
-      log_warn(LD_CRYPTO, "Error creating certificate");
-      goto error;
-    }
-  }
 
   result = tor_malloc_zero(sizeof(tor_tls_context_t));
   result->refcnt = 1;
-  if (!is_client) {
-    result->my_link_cert = tor_x509_cert_new(X509_dup(cert));
-    result->my_id_cert = tor_x509_cert_new(X509_dup(idcert));
-    result->my_auth_cert = tor_x509_cert_new(X509_dup(authcert));
-    if (!result->my_link_cert || !result->my_id_cert || !result->my_auth_cert)
+
+  if (! is_client) {
+    if (tor_tls_context_init_certificates(result, identity, key_lifetime,
+                                          flags) < 0) {
       goto error;
-    result->link_key = crypto_pk_dup_key(rsa);
-    result->auth_key = crypto_pk_dup_key(rsa_auth);
+    }
   }
 
 #if 0
@@ -1120,7 +541,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime,
    * with existing Tors. */
   if (!(result->ctx = SSL_CTX_new(TLSv1_method())))
     goto error;
-#endif
+#endif /* 0 */
 
   /* Tell OpenSSL to use TLS 1.0 or later but not SSL2 or SSL3. */
 #ifdef HAVE_TLS_METHOD
@@ -1129,7 +550,8 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime,
 #else
   if (!(result->ctx = SSL_CTX_new(SSLv23_method())))
     goto error;
-#endif
+#endif /* defined(HAVE_TLS_METHOD) */
+
 #ifdef HAVE_SSL_CTX_SET_SECURITY_LEVEL
   /* Level 1 re-enables RSA1024 and DH1024 for compatibility with old tors */
   SSL_CTX_set_security_level(result->ctx, 1);
@@ -1172,37 +594,39 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime,
     SSL_CTX_set_options(result->ctx,
                         SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION);
   }
+
+  /* Don't actually allow compression; it uses RAM and time, it makes TLS
+   * vulnerable to CRIME-style attacks, and most of the data we transmit over
+   * TLS is encrypted (and therefore uncompressible) anyway. */
 #ifdef SSL_OP_NO_COMPRESSION
   SSL_CTX_set_options(result->ctx, SSL_OP_NO_COMPRESSION);
 #endif
 #if OPENSSL_VERSION_NUMBER < OPENSSL_V_SERIES(1,1,0)
 #ifndef OPENSSL_NO_COMP
-  /* Don't actually allow compression; it uses ram and time, but the data
-   * we transmit is all encrypted anyway. */
   if (result->ctx->comp_methods)
     result->ctx->comp_methods = NULL;
 #endif
-#endif
+#endif /* OPENSSL_VERSION_NUMBER < OPENSSL_V_SERIES(1,1,0) */
+
 #ifdef SSL_MODE_RELEASE_BUFFERS
   SSL_CTX_set_mode(result->ctx, SSL_MODE_RELEASE_BUFFERS);
 #endif
   if (! is_client) {
-    if (cert && !SSL_CTX_use_certificate(result->ctx,cert))
+    if (result->my_link_cert &&
+        !SSL_CTX_use_certificate(result->ctx,
+                                 result->my_link_cert->cert)) {
       goto error;
-    X509_free(cert); /* We just added a reference to cert. */
-    cert=NULL;
-    if (idcert) {
+    }
+    if (result->my_id_cert) {
       X509_STORE *s = SSL_CTX_get_cert_store(result->ctx);
       tor_assert(s);
-      X509_STORE_add_cert(s, idcert);
-      X509_free(idcert); /* The context now owns the reference to idcert */
-      idcert = NULL;
+      X509_STORE_add_cert(s, result->my_id_cert->cert);
     }
   }
   SSL_CTX_set_session_cache_mode(result->ctx, SSL_SESS_CACHE_OFF);
   if (!is_client) {
-    tor_assert(rsa);
-    if (!(pkey = crypto_pk_get_evp_pkey_(rsa,1)))
+    tor_assert(result->link_key);
+    if (!(pkey = crypto_pk_get_openssl_evp_pkey_(result->link_key,1)))
       goto error;
     if (!SSL_CTX_use_PrivateKey(result->ctx, pkey))
       goto error;
@@ -1211,11 +635,12 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime,
     if (!SSL_CTX_check_private_key(result->ctx))
       goto error;
   }
+
   {
-    crypto_dh_t *dh = crypto_dh_new(DH_TYPE_TLS);
+    DH *dh = crypto_dh_new_openssl_tls();
     tor_assert(dh);
-    SSL_CTX_set_tmp_dh(result->ctx, crypto_dh_get_dh_(dh));
-    crypto_dh_free(dh);
+    SSL_CTX_set_tmp_dh(result->ctx, dh);
+    DH_free(dh);
   }
 /* We check for this function in two ways, since it might be either a symbol
  * or a macro. */
@@ -1254,38 +679,18 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime,
   /* let us realloc bufs that we're writing from */
   SSL_CTX_set_mode(result->ctx, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
 
-  if (rsa)
-    crypto_pk_free(rsa);
-  if (rsa_auth)
-    crypto_pk_free(rsa_auth);
-  X509_free(authcert);
-  tor_free(nickname);
-  tor_free(nn2);
   return result;
 
  error:
   tls_log_errors(NULL, LOG_WARN, LD_NET, "creating TLS context");
-  tor_free(nickname);
-  tor_free(nn2);
   if (pkey)
     EVP_PKEY_free(pkey);
-  if (rsa)
-    crypto_pk_free(rsa);
-  if (rsa_auth)
-    crypto_pk_free(rsa_auth);
-  if (result)
-    tor_tls_context_decref(result);
-  if (cert)
-    X509_free(cert);
-  if (idcert)
-    X509_free(idcert);
-  if (authcert)
-    X509_free(authcert);
+  tor_tls_context_decref(result);
   return NULL;
 }
 
 /** Invoked when a TLS state changes: log the change at severity 'debug' */
-STATIC void
+void
 tor_tls_debug_state_callback(const SSL *ssl, int type, int val)
 {
   /* LCOV_EXCL_START since this depends on whether debug is captured or not */
@@ -1341,7 +746,7 @@ static int v2_cipher_list_pruned = 0;
 
 /** Return 0 if <b>m</b> does not support the cipher with ID <b>cipher</b>;
  * return 1 if it does support it, or if we have no way to tell. */
-STATIC int
+int
 find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher)
 {
   const SSL_CIPHER *c;
@@ -1350,7 +755,7 @@ find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher)
   {
     unsigned char cipherid[3];
     tor_assert(ssl);
-    set_uint16(cipherid, htons(cipher));
+    set_uint16(cipherid, tor_htons(cipher));
     cipherid[2] = 0; /* If ssl23_get_cipher_by_char finds no cipher starting
                       * with a two-byte 'cipherid', it may look for a v2
                       * cipher with the appropriate 3 bytes. */
@@ -1359,12 +764,12 @@ find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher)
       tor_assert((SSL_CIPHER_get_id(c) & 0xffff) == cipher);
     return c != NULL;
   }
-#else
+#else /* !(defined(HAVE_SSL_CIPHER_FIND)) */
 
 # if defined(HAVE_STRUCT_SSL_METHOD_ST_GET_CIPHER_BY_CHAR)
   if (m && m->get_cipher_by_char) {
     unsigned char cipherid[3];
-    set_uint16(cipherid, htons(cipher));
+    set_uint16(cipherid, tor_htons(cipher));
     cipherid[2] = 0; /* If ssl23_get_cipher_by_char finds no cipher starting
                       * with a two-byte 'cipherid', it may look for a v2
                       * cipher with the appropriate 3 bytes. */
@@ -1373,7 +778,7 @@ find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher)
       tor_assert((c->id & 0xffff) == cipher);
     return c != NULL;
   }
-# endif
+#endif /* defined(HAVE_STRUCT_SSL_METHOD_ST_GET_CIPHER_BY_CHAR) */
 # ifndef OPENSSL_1_1_API
   if (m && m->get_cipher && m->num_ciphers) {
     /* It would seem that some of the "let's-clean-up-openssl" forks have
@@ -1389,12 +794,12 @@ find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher)
     }
     return 0;
   }
-# endif
+#endif /* !defined(OPENSSL_1_1_API) */
   (void) ssl;
   (void) m;
   (void) cipher;
   return 1; /* No way to search */
-#endif
+#endif /* defined(HAVE_SSL_CIPHER_FIND) */
 }
 
 /** Remove from v2_cipher_list every cipher that we don't support, so that
@@ -1427,7 +832,7 @@ prune_v2_cipher_list(const SSL *ssl)
  * client it is.  Return one of CIPHERS_ERR, CIPHERS_V1, CIPHERS_V2,
  * CIPHERS_UNRESTRICTED.
  **/
-STATIC int
+int
 tor_tls_classify_client_ciphers(const SSL *ssl,
                                 STACK_OF(SSL_CIPHER) *peer_ciphers)
 {
@@ -1509,7 +914,7 @@ tor_tls_classify_client_ciphers(const SSL *ssl,
 /** Return true iff the cipher list suggested by the client for <b>ssl</b> is
  * a list that indicates that the client knows how to do the v2 TLS connection
  * handshake. */
-STATIC int
+int
 tor_tls_client_is_using_v2_ciphers(const SSL *ssl)
 {
   STACK_OF(SSL_CIPHER) *ciphers;
@@ -1522,7 +927,7 @@ tor_tls_client_is_using_v2_ciphers(const SSL *ssl)
     return CIPHERS_ERR;
   }
   ciphers = session->ciphers;
-#endif
+#endif /* defined(HAVE_SSL_GET_CLIENT_CIPHERS) */
 
   return tor_tls_classify_client_ciphers(ssl, ciphers) >= CIPHERS_V2;
 }
@@ -1533,7 +938,7 @@ tor_tls_client_is_using_v2_ciphers(const SSL *ssl)
  *         do not send or request extra certificates in v2 handshakes.</li>
  * <li>To detect renegotiation</li></ul>
  */
-STATIC void
+void
 tor_tls_server_info_callback(const SSL *ssl, int type, int val)
 {
   tor_tls_t *tls;
@@ -1556,8 +961,6 @@ tor_tls_server_info_callback(const SSL *ssl, int type, int val)
     /* Check whether we're watching for renegotiates.  If so, this is one! */
     if (tls->negotiated_callback)
       tls->got_renegotiate = 1;
-    if (tls->server_handshake_count < 127) /*avoid any overflow possibility*/
-      ++tls->server_handshake_count;
   } else {
     log_warn(LD_BUG, "Couldn't look up the tls for an SSL*. How odd!");
     return;
@@ -1599,7 +1002,7 @@ tor_tls_server_info_callback(const SSL *ssl, int type, int val)
  * authentication on the fly.  But as long as we return 0, we won't actually be
  * setting up a shared secret, and all will be fine.
  */
-STATIC int
+int
 tor_tls_session_secret_cb(SSL *ssl, void *secret, int *secret_len,
                           STACK_OF(SSL_CIPHER) *peer_ciphers,
                           CONST_IF_OPENSSL_1_1_API SSL_CIPHER **cipher,
@@ -1630,12 +1033,11 @@ tor_tls_setup_session_secret_cb(tor_tls_t *tls)
  * determine whether it is functioning as a server.
  */
 tor_tls_t *
-tor_tls_new(int sock, int isServer)
+tor_tls_new(tor_socket_t sock, int isServer)
 {
   BIO *bio = NULL;
   tor_tls_t *result = tor_malloc_zero(sizeof(tor_tls_t));
-  tor_tls_context_t *context = isServer ? server_tls_context :
-    client_tls_context;
+  tor_tls_context_t *context = tor_tls_context_get(isServer);
   result->magic = TOR_TLS_MAGIC;
 
   check_no_tls_errors();
@@ -1653,6 +1055,13 @@ tor_tls_new(int sock, int isServer)
     SSL_set_tlsext_host_name(result->ssl, fake_hostname);
     tor_free(fake_hostname);
   }
+#endif /* defined(SSL_set_tlsext_host_name) */
+
+#ifdef SSL_CTRL_SET_MAX_PROTO_VERSION
+  if (openssl_bug_7712_is_present) {
+    /* We can't actually use TLS 1.3 until this bug is fixed. */
+    SSL_set_max_proto_version(result->ssl, TLS1_2_VERSION);
+  }
 #endif
 
   if (!SSL_set_cipher_list(result->ssl,
@@ -1666,7 +1075,7 @@ tor_tls_new(int sock, int isServer)
     goto err;
   }
   result->socket = sock;
-  bio = BIO_new_socket(sock, BIO_NOCLOSE);
+  bio = BIO_new_socket(sock, BIO_CLOSE);
   if (! bio) {
     tls_log_errors(NULL, LOG_WARN, LD_NET, "opening BIO");
 #ifdef SSL_set_tlsext_host_name
@@ -1714,17 +1123,6 @@ tor_tls_new(int sock, int isServer)
   return result;
 }
 
-/** Make future log messages about <b>tls</b> display the address
- * <b>address</b>.
- */
-void
-tor_tls_set_logged_address(tor_tls_t *tls, const char *address)
-{
-  tor_assert(tls);
-  tor_free(tls->address);
-  tls->address = tor_strdup(address);
-}
-
 /** Set <b>cb</b> to be called with argument <b>arg</b> whenever <b>tls</b>
  * next gets a client-side renegotiate in the middle of a read.  Do not
  * invoke this function until <em>after</em> initial handshaking is done!
@@ -1780,42 +1178,41 @@ tor_tls_assert_renegotiation_unblocked(tor_tls_t *tls)
   tor_assert(0 != (options & SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION));
 #else
   (void) tls;
-#endif
+#endif /* defined(SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION) && ... */
 }
 
-/** Return whether this tls initiated the connect (client) or
- * received it (server). */
-int
-tor_tls_is_server(tor_tls_t *tls)
+/**
+ * Tell the TLS library that the underlying socket for <b>tls</b> has been
+ * closed, and the library should not attempt to free that socket itself.
+ */
+void
+tor_tls_release_socket(tor_tls_t *tls)
 {
-  tor_assert(tls);
-  return tls->isServer;
+  if (! tls)
+    return;
+
+  BIO *rbio, *wbio;
+  rbio = SSL_get_rbio(tls->ssl);
+  wbio = SSL_get_wbio(tls->ssl);
+
+  if (rbio) {
+    (void) BIO_set_close(rbio, BIO_NOCLOSE);
+  }
+  if (wbio && wbio != rbio) {
+    (void) BIO_set_close(wbio, BIO_NOCLOSE);
+  }
 }
 
-/** Release resources associated with a TLS object.  Does not close the
- * underlying file descriptor.
- */
 void
-tor_tls_free(tor_tls_t *tls)
+tor_tls_impl_free_(tor_tls_impl_t *ssl)
 {
-  if (!tls)
+  if (!ssl)
     return;
-  tor_assert(tls->ssl);
-  {
-    size_t r,w;
-    tor_tls_get_n_raw_bytes(tls,&r,&w); /* ensure written_by_tls is updated */
-  }
+
 #ifdef SSL_set_tlsext_host_name
-  SSL_set_tlsext_host_name(tls->ssl, NULL);
+  SSL_set_tlsext_host_name(ssl, NULL);
 #endif
-  SSL_free(tls->ssl);
-  tls->ssl = NULL;
-  tls->negotiated_callback = NULL;
-  if (tls->context)
-    tor_tls_context_decref(tls->context);
-  tor_free(tls->address);
-  tls->magic = 0x99999999;
-  tor_free(tls);
+  SSL_free(ssl);
 }
 
 /** Underlying function for TLS reading.  Reads up to <b>len</b>
@@ -1943,7 +1340,7 @@ tor_tls_handshake(tor_tls_t *tls)
   return r;
 }
 
-/** Perform the final part of the intial TLS handshake on <b>tls</b>.  This
+/** Perform the final part of the initial TLS handshake on <b>tls</b>.  This
  * should be called for the first handshake only: it determines whether the v1
  * or the v2 handshake was used, and adjusts things for the renegotiation
  * handshake as appropriate.
@@ -1987,69 +1384,6 @@ tor_tls_finish_handshake(tor_tls_t *tls)
   return r;
 }
 
-/** Shut down an open tls connection <b>tls</b>.  When finished, returns
- * TOR_TLS_DONE.  On failure, returns TOR_TLS_ERROR, TOR_TLS_WANTREAD,
- * or TOR_TLS_WANTWRITE.
- */
-int
-tor_tls_shutdown(tor_tls_t *tls)
-{
-  int r, err;
-  char buf[128];
-  tor_assert(tls);
-  tor_assert(tls->ssl);
-  check_no_tls_errors();
-
-  while (1) {
-    if (tls->state == TOR_TLS_ST_SENTCLOSE) {
-      /* If we've already called shutdown once to send a close message,
-       * we read until the other side has closed too.
-       */
-      do {
-        r = SSL_read(tls->ssl, buf, 128);
-      } while (r>0);
-      err = tor_tls_get_error(tls, r, CATCH_ZERO, "reading to shut down",
-                              LOG_INFO, LD_NET);
-      if (err == TOR_TLS_ZERORETURN_) {
-        tls->state = TOR_TLS_ST_GOTCLOSE;
-        /* fall through... */
-      } else {
-        return err;
-      }
-    }
-
-    r = SSL_shutdown(tls->ssl);
-    if (r == 1) {
-      /* If shutdown returns 1, the connection is entirely closed. */
-      tls->state = TOR_TLS_ST_CLOSED;
-      return TOR_TLS_DONE;
-    }
-    err = tor_tls_get_error(tls, r, CATCH_SYSCALL|CATCH_ZERO, "shutting down",
-                            LOG_INFO, LD_NET);
-    if (err == TOR_TLS_SYSCALL_) {
-      /* The underlying TCP connection closed while we were shutting down. */
-      tls->state = TOR_TLS_ST_CLOSED;
-      return TOR_TLS_DONE;
-    } else if (err == TOR_TLS_ZERORETURN_) {
-      /* The TLS connection says that it sent a shutdown record, but
-       * isn't done shutting down yet.  Make sure that this hasn't
-       * happened before, then go back to the start of the function
-       * and try to read.
-       */
-      if (tls->state == TOR_TLS_ST_GOTCLOSE ||
-         tls->state == TOR_TLS_ST_SENTCLOSE) {
-        log_warn(LD_NET,
-            "TLS returned \"half-closed\" value while already half-closed");
-        return TOR_TLS_ERROR_MISC;
-      }
-      tls->state = TOR_TLS_ST_SENTCLOSE;
-      /* fall through ... */
-    } else {
-      return err;
-    }
-  } /* end loop */
-}
-
 /** Return true iff this TLS connection is authenticated.
  */
 int
@@ -2095,71 +1429,13 @@ tor_tls_get_own_cert,(tor_tls_t *tls))
   return tor_x509_cert_new(duplicate);
 }
 
-/** Warn that a certificate lifetime extends through a certain range. */
-static void
-log_cert_lifetime(int severity, const X509 *cert, const char *problem)
-{
-  BIO *bio = NULL;
-  BUF_MEM *buf;
-  char *s1=NULL, *s2=NULL;
-  char mytime[33];
-  time_t now = time(NULL);
-  struct tm tm;
-  size_t n;
-
-  if (problem)
-    tor_log(severity, LD_GENERAL,
-        "Certificate %s. Either their clock is set wrong, or your clock "
-        "is wrong.",
-           problem);
-
-  if (!(bio = BIO_new(BIO_s_mem()))) {
-    log_warn(LD_GENERAL, "Couldn't allocate BIO!"); goto end;
-  }
-  if (!(ASN1_TIME_print(bio, X509_get_notBefore_const(cert)))) {
-    tls_log_errors(NULL, LOG_WARN, LD_NET, "printing certificate lifetime");
-    goto end;
-  }
-  BIO_get_mem_ptr(bio, &buf);
-  s1 = tor_strndup(buf->data, buf->length);
-
-  (void)BIO_reset(bio);
-  if (!(ASN1_TIME_print(bio, X509_get_notAfter_const(cert)))) {
-    tls_log_errors(NULL, LOG_WARN, LD_NET, "printing certificate lifetime");
-    goto end;
-  }
-  BIO_get_mem_ptr(bio, &buf);
-  s2 = tor_strndup(buf->data, buf->length);
-
-  n = strftime(mytime, 32, "%b %d %H:%M:%S %Y UTC", tor_gmtime_r(&now, &tm));
-  if (n > 0) {
-    tor_log(severity, LD_GENERAL,
-        "(certificate lifetime runs from %s through %s. Your time is %s.)",
-        s1,s2,mytime);
-  } else {
-    tor_log(severity, LD_GENERAL,
-        "(certificate lifetime runs from %s through %s. "
-        "Couldn't get your time.)",
-        s1, s2);
-  }
-
- end:
-  /* Not expected to get invoked */
-  tls_log_errors(NULL, LOG_WARN, LD_NET, "getting certificate lifetime");
-  if (bio)
-    BIO_free(bio);
-  tor_free(s1);
-  tor_free(s2);
-}
-
 /** Helper function: try to extract a link certificate and an identity
  * certificate from <b>tls</b>, and store them in *<b>cert_out</b> and
  * *<b>id_cert_out</b> respectively.  Log all messages at level
  * <b>severity</b>.
  *
- * Note that a reference is added to cert_out, so it needs to be
- * freed. id_cert_out doesn't. */
-MOCK_IMPL(STATIC void,
+ * Note that a reference is added both of the returned certificates. */
+MOCK_IMPL(void,
 try_to_extract_certs_from_tls,(int severity, tor_tls_t *tls,
                                X509 **cert_out, X509 **id_cert_out))
 {
@@ -2188,115 +1464,7 @@ try_to_extract_certs_from_tls,(int severity, tor_tls_t *tls,
     if (X509_cmp(id_cert, cert) != 0)
       break;
   }
-  *id_cert_out = id_cert;
-}
-
-/** If the provided tls connection is authenticated and has a
- * certificate chain that is currently valid and signed, then set
- * *<b>identity_key</b> to the identity certificate's key and return
- * 0.  Else, return -1 and log complaints with log-level <b>severity</b>.
- */
-int
-tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity_key)
-{
-  X509 *cert = NULL, *id_cert = NULL;
-  EVP_PKEY *id_pkey = NULL;
-  RSA *rsa;
-  int r = -1;
-
-  check_no_tls_errors();
-  *identity_key = NULL;
-
-  try_to_extract_certs_from_tls(severity, tls, &cert, &id_cert);
-  if (!cert)
-    goto done;
-  if (!id_cert) {
-    log_fn(severity,LD_PROTOCOL,"No distinct identity certificate found");
-    goto done;
-  }
-  tls_log_errors(tls, severity, LD_HANDSHAKE, "before verifying certificate");
-
-  if (!(id_pkey = X509_get_pubkey(id_cert)) ||
-      X509_verify(cert, id_pkey) <= 0) {
-    log_fn(severity,LD_PROTOCOL,"X509_verify on cert and pkey returned <= 0");
-    tls_log_errors(tls, severity, LD_HANDSHAKE, "verifying certificate");
-    goto done;
-  }
-
-  rsa = EVP_PKEY_get1_RSA(id_pkey);
-  if (!rsa)
-    goto done;
-  *identity_key = crypto_new_pk_from_rsa_(rsa);
-
-  r = 0;
-
- done:
-  if (cert)
-    X509_free(cert);
-  if (id_pkey)
-    EVP_PKEY_free(id_pkey);
-
-  /* This should never get invoked, but let's make sure in case OpenSSL
-   * acts unexpectedly. */
-  tls_log_errors(tls, LOG_WARN, LD_HANDSHAKE, "finishing tor_tls_verify");
-
-  return r;
-}
-
-/** Check whether the certificate set on the connection <b>tls</b> is expired
- * give or take <b>past_tolerance</b> seconds, or not-yet-valid give or take
- * <b>future_tolerance</b> seconds. Return 0 for valid, -1 for failure.
- *
- * NOTE: you should call tor_tls_verify before tor_tls_check_lifetime.
- */
-int
-tor_tls_check_lifetime(int severity, tor_tls_t *tls,
-                       int past_tolerance, int future_tolerance)
-{
-  X509 *cert;
-  int r = -1;
-
-  if (!(cert = SSL_get_peer_certificate(tls->ssl)))
-    goto done;
-
-  if (check_cert_lifetime_internal(severity, cert,
-                                   past_tolerance, future_tolerance) < 0)
-    goto done;
-
-  r = 0;
- done:
-  if (cert)
-    X509_free(cert);
-  /* Not expected to get invoked */
-  tls_log_errors(tls, LOG_WARN, LD_NET, "checking certificate lifetime");
-
-  return r;
-}
-
-/** Helper: check whether <b>cert</b> is expired give or take
- * <b>past_tolerance</b> seconds, or not-yet-valid give or take
- * <b>future_tolerance</b> seconds.  If it is live, return 0.  If it is not
- * live, log a message and return -1. */
-static int
-check_cert_lifetime_internal(int severity, const X509 *cert,
-                             int past_tolerance, int future_tolerance)
-{
-  time_t now, t;
-
-  now = time(NULL);
-
-  t = now + future_tolerance;
-  if (X509_cmp_time(X509_get_notBefore_const(cert), &t) > 0) {
-    log_cert_lifetime(severity, cert, "not yet valid");
-    return -1;
-  }
-  t = now - past_tolerance;
-  if (X509_cmp_time(X509_get_notAfter_const(cert), &t) < 0) {
-    log_cert_lifetime(severity, cert, "already expired");
-    return -1;
-  }
-
-  return 0;
+  *id_cert_out = id_cert ? X509_dup(id_cert) : NULL;
 }
 
 /** Return the number of bytes available for reading from <b>tls</b>.
@@ -2341,10 +1509,10 @@ tor_tls_get_n_raw_bytes(tor_tls_t *tls, size_t *n_read, size_t *n_written)
   if (BIO_method_type(wbio) == BIO_TYPE_BUFFER &&
         (tmpbio = BIO_next(wbio)) != NULL)
     wbio = tmpbio;
-#else
+#else /* !(OPENSSL_VERSION_NUMBER >= OPENSSL_VER(1,1,0,0,5)) */
   if (wbio->method == BIO_f_buffer() && (tmpbio = BIO_next(wbio)) != NULL)
     wbio = tmpbio;
-#endif
+#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_VER(1,1,0,0,5) */
   w = (unsigned long) BIO_number_written(wbio);
 
   /* We are ok with letting these unsigned ints go "negative" here:
@@ -2372,8 +1540,8 @@ tls_get_write_overhead_ratio,(void))
   if (total_bytes_written_over_tls == 0)
     return 1.0;
 
-  return U64_TO_DBL(total_bytes_written_by_tls) /
-    U64_TO_DBL(total_bytes_written_over_tls);
+  return ((double)total_bytes_written_by_tls) /
+    ((double)total_bytes_written_over_tls);
 }
 
 /** Implement check_no_tls_errors: If there are any pending OpenSSL
@@ -2396,14 +1564,6 @@ tor_tls_used_v1_handshake(tor_tls_t *tls)
   return ! tls->wasV2Handshake;
 }
 
-/** Return the number of server handshakes that we've noticed doing on
- * <b>tls</b>. */
-int
-tor_tls_get_num_server_handshakes(tor_tls_t *tls)
-{
-  return tls->server_handshake_count;
-}
-
 /** Return true iff the server TLS connection <b>tls</b> got the renegotiation
  * request it was waiting for. */
 int
@@ -2423,7 +1583,7 @@ SSL_get_client_random(SSL *s, uint8_t *out, size_t len)
   memcpy(out, s->s3->client_random, len);
   return len;
 }
-#endif
+#endif /* !defined(HAVE_SSL_GET_CLIENT_RANDOM) */
 
 #ifndef HAVE_SSL_GET_SERVER_RANDOM
 static size_t
@@ -2436,10 +1596,10 @@ SSL_get_server_random(SSL *s, uint8_t *out, size_t len)
   memcpy(out, s->s3->server_random, len);
   return len;
 }
-#endif
+#endif /* !defined(HAVE_SSL_GET_SERVER_RANDOM) */
 
 #ifndef HAVE_SSL_SESSION_GET_MASTER_KEY
-STATIC size_t
+size_t
 SSL_SESSION_get_master_key(SSL_SESSION *s, uint8_t *out, size_t len)
 {
   tor_assert(s);
@@ -2450,7 +1610,7 @@ SSL_SESSION_get_master_key(SSL_SESSION *s, uint8_t *out, size_t len)
   memcpy(out, s->master_key, len);
   return len;
 }
-#endif
+#endif /* !defined(HAVE_SSL_SESSION_GET_MASTER_KEY) */
 
 /** Set the DIGEST256_LEN buffer at <b>secrets_out</b> to the value used in
  * the v3 handshake to prove that the client knows the TLS secrets for the
@@ -2517,6 +1677,62 @@ tor_tls_get_tlssecrets,(tor_tls_t *tls, uint8_t *secrets_out))
   return 0;
 }
 
+/** Using the RFC5705 key material exporting construction, and the
+ * provided <b>context</b> (<b>context_len</b> bytes long) and
+ * <b>label</b> (a NUL-terminated string), compute a 32-byte secret in
+ * <b>secrets_out</b> that only the parties to this TLS session can
+ * compute.  Return 0 on success; -1 on failure; and -2 on failure
+ * caused by OpenSSL bug 7712.
+ */
+MOCK_IMPL(int,
+tor_tls_export_key_material,(tor_tls_t *tls, uint8_t *secrets_out,
+                             const uint8_t *context,
+                             size_t context_len,
+                             const char *label))
+{
+  tor_assert(tls);
+  tor_assert(tls->ssl);
+
+  int r = SSL_export_keying_material(tls->ssl,
+                                     secrets_out, DIGEST256_LEN,
+                                     label, strlen(label),
+                                     context, context_len, 1);
+
+  if (r != 1) {
+    int severity = openssl_bug_7712_is_present ? LOG_WARN : LOG_DEBUG;
+    tls_log_errors(tls, severity, LD_NET, "exporting keying material");
+  }
+
+#ifdef TLS1_3_VERSION
+  if (r != 1 &&
+      strlen(label) > 12 &&
+      SSL_version(tls->ssl) >= TLS1_3_VERSION) {
+
+    if (! openssl_bug_7712_is_present) {
+      /* We might have run into OpenSSL issue 7712, which caused OpenSSL
+       * 1.1.1a to not handle long labels.  Let's test to see if we have.
+       */
+      r = SSL_export_keying_material(tls->ssl, secrets_out, DIGEST256_LEN,
+                                     "short", 5, context, context_len, 1);
+      if (r == 1) {
+        /* A short label succeeds, but a long label fails. This was openssl
+         * issue 7712. */
+        openssl_bug_7712_is_present = 1;
+        log_warn(LD_GENERAL, "Detected OpenSSL bug 7712: disabling TLS 1.3 on "
+                 "future connections. A fix is expected to appear in OpenSSL "
+                 "1.1.1b.");
+      }
+    }
+    if (openssl_bug_7712_is_present)
+      return -2;
+    else
+      return -1;
+  }
+#endif
+
+  return (r == 1) ? 0 : -1;
+}
+
 /** Examine the amount of memory used and available for buffers in <b>tls</b>.
  * Set *<b>rbuf_capacity</b> to the amount of storage allocated for the read
  * buffer and *<b>rbuf_bytes</b> to the amount actually used.
@@ -2537,7 +1753,7 @@ tor_tls_get_buffer_sizes(tor_tls_t *tls,
   (void)wbuf_bytes;
 
   return -1;
-#else
+#else /* !(OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0)) */
   if (tls->ssl->s3->rbuf.buf)
     *rbuf_capacity = tls->ssl->s3->rbuf.len;
   else
@@ -2549,7 +1765,7 @@ tor_tls_get_buffer_sizes(tor_tls_t *tls,
   *rbuf_bytes = tls->ssl->s3->rbuf.left;
   *wbuf_bytes = tls->ssl->s3->wbuf.left;
   return 0;
-#endif
+#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) */
 }
 
 /** Check whether the ECC group requested is supported by the current OpenSSL
diff --git a/src/lib/tls/tortls_st.h b/src/lib/tls/tortls_st.h
new file mode 100644
index 0000000000..3f7ea8ac6a
--- /dev/null
+++ b/src/lib/tls/tortls_st.h
@@ -0,0 +1,75 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_TORTLS_ST_H
+#define TOR_TORTLS_ST_H
+
+#include "lib/net/socket.h"
+
+#define TOR_TLS_MAGIC 0x71571571
+
+typedef enum {
+    TOR_TLS_ST_HANDSHAKE, TOR_TLS_ST_OPEN, TOR_TLS_ST_GOTCLOSE,
+    TOR_TLS_ST_SENTCLOSE, TOR_TLS_ST_CLOSED, TOR_TLS_ST_RENEGOTIATE,
+    TOR_TLS_ST_BUFFEREVENT
+} tor_tls_state_t;
+#define tor_tls_state_bitfield_t ENUM_BF(tor_tls_state_t)
+
+struct tor_tls_context_t {
+  int refcnt;
+  tor_tls_context_impl_t *ctx;
+  struct tor_x509_cert_t *my_link_cert;
+  struct tor_x509_cert_t *my_id_cert;
+  struct tor_x509_cert_t *my_auth_cert;
+  crypto_pk_t *link_key;
+  crypto_pk_t *auth_key;
+};
+
+/** Holds a SSL object and its associated data.  Members are only
+ * accessed from within tortls.c.
+ */
+struct tor_tls_t {
+  uint32_t magic;
+  tor_tls_context_t *context; /** A link to the context object for this tls. */
+  tor_tls_impl_t *ssl; /**< An OpenSSL SSL object or NSS PRFileDesc. */
+  tor_socket_t socket; /**< The underlying file descriptor for this TLS
+                        * connection. */
+  char *address; /**< An address to log when describing this connection. */
+  tor_tls_state_bitfield_t state : 3; /**< The current SSL state,
+                                       * depending on which operations
+                                       * have completed successfully. */
+  unsigned int isServer:1; /**< True iff this is a server-side connection */
+  unsigned int wasV2Handshake:1; /**< True iff the original handshake for
+                                  * this connection used the updated version
+                                  * of the connection protocol (client sends
+                                  * different cipher list, server sends only
+                                  * one certificate). */
+  /** True iff we should call negotiated_callback when we're done reading. */
+  unsigned int got_renegotiate:1;
+#ifdef ENABLE_OPENSSL
+  /** Return value from tor_tls_classify_client_ciphers, or 0 if we haven't
+   * called that function yet. */
+  int8_t client_cipher_list_type;
+  size_t wantwrite_n; /**< 0 normally, >0 if we returned wantwrite last
+                       * time. */
+  /** Last values retrieved from BIO_number_read()/write(); see
+   * tor_tls_get_n_raw_bytes() for usage.
+   */
+  unsigned long last_write_count;
+  unsigned long last_read_count;
+  /** If set, a callback to invoke whenever the client tries to renegotiate
+   * the handshake. */
+  void (*negotiated_callback)(tor_tls_t *tls, void *arg);
+  /** Argument to pass to negotiated_callback. */
+  void *callback_arg;
+#endif
+#ifdef ENABLE_NSS
+  /** Last values retried from tor_get_prfiledesc_byte_counts(). */
+  uint64_t last_write_count;
+  uint64_t last_read_count;
+#endif
+};
+
+#endif
diff --git a/src/lib/tls/x509.c b/src/lib/tls/x509.c
new file mode 100644
index 0000000000..67a8b49b9d
--- /dev/null
+++ b/src/lib/tls/x509.c
@@ -0,0 +1,143 @@
+/* Copyright (c) 2003, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file x509_openssl.c
+ * \brief Wrapper functions to present a consistent interface to
+ * X.509 functions.
+ **/
+
+#define TOR_X509_PRIVATE
+#include "lib/tls/x509.h"
+#include "lib/tls/x509_internal.h"
+#include "lib/log/util_bug.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+
+/** Choose the start and end times for a certificate */
+void
+tor_tls_pick_certificate_lifetime(time_t now,
+                                  unsigned int cert_lifetime,
+                                  time_t *start_time_out,
+                                  time_t *end_time_out)
+{
+  time_t start_time, end_time;
+  /* Make sure we're part-way through the certificate lifetime, rather
+   * than having it start right now. Don't choose quite uniformly, since
+   * then we might pick a time where we're about to expire. Lastly, be
+   * sure to start on a day boundary. */
+  /* Our certificate lifetime will be cert_lifetime no matter what, but if we
+   * start cert_lifetime in the past, we'll have 0 real lifetime.  instead we
+   * start up to (cert_lifetime - min_real_lifetime - start_granularity) in
+   * the past. */
+  const time_t min_real_lifetime = 24*3600;
+  const time_t start_granularity = 24*3600;
+  time_t earliest_start_time;
+  /* Don't actually start in the future! */
+  if (cert_lifetime <= min_real_lifetime + start_granularity) {
+    earliest_start_time = now - 1;
+  } else {
+    earliest_start_time = now + min_real_lifetime + start_granularity
+      - cert_lifetime;
+  }
+  start_time = crypto_rand_time_range(earliest_start_time, now);
+  /* Round the start time back to the start of a day. */
+  start_time -= start_time % start_granularity;
+
+  end_time = start_time + cert_lifetime;
+
+  *start_time_out = start_time;
+  *end_time_out = end_time;
+}
+
+/** Return a set of digests for the public key in <b>cert</b>, or NULL if this
+ * cert's public key is not one we know how to take the digest of. */
+const common_digests_t *
+tor_x509_cert_get_id_digests(const tor_x509_cert_t *cert)
+{
+  if (cert->pkey_digests_set)
+    return &cert->pkey_digests;
+  else
+    return NULL;
+}
+
+/** Return a set of digests for the public key in <b>cert</b>. */
+const common_digests_t *
+tor_x509_cert_get_cert_digests(const tor_x509_cert_t *cert)
+{
+  return &cert->cert_digests;
+}
+
+/** Free all storage held in <b>cert</b> */
+void
+tor_x509_cert_free_(tor_x509_cert_t *cert)
+{
+  if (! cert)
+    return;
+  tor_x509_cert_impl_free(cert->cert);
+#ifdef ENABLE_OPENSSL
+  tor_free(cert->encoded);
+#endif
+  memwipe(cert, 0x03, sizeof(*cert));
+  /* LCOV_EXCL_BR_START since cert will never be NULL here */
+  tor_free(cert);
+  /* LCOV_EXCL_BR_STOP */
+}
+
+/**
+ * Allocate a new tor_x509_cert_t to hold the certificate "x509_cert".
+ *
+ * Steals a reference to x509_cert.
+ */
+MOCK_IMPL(tor_x509_cert_t *,
+tor_x509_cert_new,(tor_x509_cert_impl_t *x509_cert))
+{
+  tor_x509_cert_t *cert;
+
+  if (!x509_cert)
+    return NULL;
+
+  cert = tor_malloc_zero(sizeof(tor_x509_cert_t));
+  cert->cert = x509_cert;
+
+  if (tor_x509_cert_set_cached_der_encoding(cert) < 0)
+    goto err;
+
+  {
+    const uint8_t *encoded=NULL;
+    size_t encoded_len=0;
+    tor_x509_cert_get_der(cert, &encoded, &encoded_len);
+    tor_assert(encoded);
+    crypto_common_digests(&cert->cert_digests, (char *)encoded, encoded_len);
+  }
+
+  {
+    crypto_pk_t *pk = tor_tls_cert_get_key(cert);
+    if (pk) {
+      if (crypto_pk_get_common_digests(pk, &cert->pkey_digests) < 0) {
+        log_warn(LD_CRYPTO, "unable to compute digests of certificate key");
+        crypto_pk_free(pk);
+        goto err;
+      }
+    }
+    cert->pkey_digests_set = 1;
+    crypto_pk_free(pk);
+  }
+
+  return cert;
+ err:
+  log_err(LD_CRYPTO, "Couldn't wrap encoded X509 certificate.");
+  tor_x509_cert_free(cert);
+  return NULL;
+}
+
+/** Return a new copy of <b>cert</b>. */
+tor_x509_cert_t *
+tor_x509_cert_dup(const tor_x509_cert_t *cert)
+{
+  tor_assert(cert);
+  tor_assert(cert->cert);
+  return tor_x509_cert_new(tor_x509_cert_impl_dup_(cert->cert));
+}
diff --git a/src/lib/tls/x509.h b/src/lib/tls/x509.h
new file mode 100644
index 0000000000..5e6660de5c
--- /dev/null
+++ b/src/lib/tls/x509.h
@@ -0,0 +1,75 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_X509_H
+#define TOR_X509_H
+
+/**
+ * \file x509.h
+ * \brief Headers for tortls.c
+ **/
+
+#include "lib/crypt_ops/crypto_rsa.h"
+#include "lib/testsupport/testsupport.h"
+
+/* Opaque structure to hold an X509 certificate. */
+typedef struct tor_x509_cert_t tor_x509_cert_t;
+
+#ifdef ENABLE_NSS
+typedef struct CERTCertificateStr tor_x509_cert_impl_t;
+#elif defined(ENABLE_OPENSSL)
+typedef struct x509_st tor_x509_cert_impl_t;
+#endif
+
+#ifdef TOR_X509_PRIVATE
+/** Structure that we use for a single certificate. */
+struct tor_x509_cert_t {
+  tor_x509_cert_impl_t *cert;
+#ifdef ENABLE_OPENSSL
+  uint8_t *encoded;
+  size_t encoded_len;
+#endif
+  unsigned pkey_digests_set : 1;
+  common_digests_t cert_digests;
+  common_digests_t pkey_digests;
+};
+#endif
+
+void tor_tls_pick_certificate_lifetime(time_t now,
+                                       unsigned cert_lifetime,
+                                       time_t *start_time_out,
+                                       time_t *end_time_out);
+
+#ifdef TOR_UNIT_TESTS
+tor_x509_cert_t *tor_x509_cert_replace_expiration(
+                                               const tor_x509_cert_t *inp,
+                                               time_t new_expiration_time,
+                                               crypto_pk_t *signing_key);
+#endif
+
+tor_x509_cert_t *tor_x509_cert_dup(const tor_x509_cert_t *cert);
+
+void tor_x509_cert_free_(tor_x509_cert_t *cert);
+#define tor_x509_cert_free(c) \
+  FREE_AND_NULL(tor_x509_cert_t, tor_x509_cert_free_, (c))
+tor_x509_cert_t *tor_x509_cert_decode(const uint8_t *certificate,
+                            size_t certificate_len);
+void tor_x509_cert_get_der(const tor_x509_cert_t *cert,
+                      const uint8_t **encoded_out, size_t *size_out);
+
+const common_digests_t *tor_x509_cert_get_id_digests(
+                      const tor_x509_cert_t *cert);
+const common_digests_t *tor_x509_cert_get_cert_digests(
+                      const tor_x509_cert_t *cert);
+
+crypto_pk_t *tor_tls_cert_get_key(tor_x509_cert_t *cert);
+
+int tor_tls_cert_is_valid(int severity,
+                          const tor_x509_cert_t *cert,
+                          const tor_x509_cert_t *signing_cert,
+                          time_t now,
+                          int check_rsa_1024);
+
+#endif
diff --git a/src/lib/tls/x509_internal.h b/src/lib/tls/x509_internal.h
new file mode 100644
index 0000000000..bf2bec9689
--- /dev/null
+++ b/src/lib/tls/x509_internal.h
@@ -0,0 +1,53 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_X509_INTERNAL_H
+#define TOR_X509_INTERNAL_H
+
+/**
+ * \file x509.h
+ * \brief Internal headers for tortls.c
+ **/
+
+#include "lib/crypt_ops/crypto_rsa.h"
+#include "lib/testsupport/testsupport.h"
+
+/**
+ * How skewed do we allow our clock to be with respect to certificates that
+ * seem to be expired? (seconds)
+ */
+#define TOR_X509_PAST_SLOP (2*24*60*60)
+/**
+ * How skewed do we allow our clock to be with respect to certificates that
+ * seem to come from the future? (seconds)
+ */
+#define  TOR_X509_FUTURE_SLOP (30*24*60*60)
+
+MOCK_DECL(tor_x509_cert_impl_t *, tor_tls_create_certificate,
+                                                   (crypto_pk_t *rsa,
+                                                    crypto_pk_t *rsa_sign,
+                                                    const char *cname,
+                                                    const char *cname_sign,
+                                                  unsigned int cert_lifetime));
+MOCK_DECL(tor_x509_cert_t *, tor_x509_cert_new,
+          (tor_x509_cert_impl_t *x509_cert));
+
+int tor_x509_check_cert_lifetime_internal(int severity,
+                                          const tor_x509_cert_impl_t *cert,
+                                          time_t now,
+                                          int past_tolerance,
+                                          int future_tolerance);
+
+void tor_x509_cert_impl_free_(tor_x509_cert_impl_t *cert);
+#define tor_x509_cert_impl_free(cert) \
+  FREE_AND_NULL(tor_x509_cert_impl_t, tor_x509_cert_impl_free_, (cert))
+tor_x509_cert_impl_t *tor_x509_cert_impl_dup_(tor_x509_cert_impl_t *cert);
+#ifdef ENABLE_OPENSSL
+int tor_x509_cert_set_cached_der_encoding(tor_x509_cert_t *cert);
+#else
+#define tor_x509_cert_set_cached_der_encoding(cert) (0)
+#endif
+
+#endif
diff --git a/src/lib/tls/x509_nss.c b/src/lib/tls/x509_nss.c
new file mode 100644
index 0000000000..fb4af54c52
--- /dev/null
+++ b/src/lib/tls/x509_nss.c
@@ -0,0 +1,458 @@
+/* Copyright (c) 2003, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file x509_nss.c
+ * \brief Wrapper functions to present a consistent interface to
+ * X.509 functions from NSS.
+ **/
+
+#define TOR_X509_PRIVATE
+#include "lib/tls/x509.h"
+#include "lib/tls/x509_internal.h"
+#include "lib/tls/tortls.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/crypt_ops/crypto_nss_mgt.h"
+#include "lib/log/util_bug.h"
+#include "lib/encoding/time_fmt.h"
+#include "lib/string/printf.h"
+
+#include <pk11pub.h>
+#include <cryptohi.h>
+#include <cert.h>
+#include <keyhi.h>
+#include <time.h>
+
+/* Units of PRTime per second.
+ *
+ * (PRTime is based in microseconds since the Unix
+ * epoch.) */
+#define PRTIME_PER_SEC (1000*1000)
+
+static tor_x509_cert_impl_t *tor_x509_cert_decode_internal(
+                      const uint8_t *certificate, int certificate_len);
+
+static tor_x509_cert_impl_t *
+tor_tls_create_certificate_internal(crypto_pk_t *rsa,
+                                    crypto_pk_t *rsa_sign,
+                                    CERTName *subject_dn,
+                                    CERTName *issuer_dn,
+                                    time_t start_time,
+                                    time_t end_time)
+{
+  if (! crypto_pk_key_is_private(rsa_sign)) {
+    return NULL;
+  }
+
+  const SECKEYPublicKey *subject_key = crypto_pk_get_nss_pubkey(rsa);
+  const SECKEYPrivateKey *signing_key = crypto_pk_get_nss_privkey(rsa_sign);
+  SECStatus s;
+
+  CERTSubjectPublicKeyInfo *subject_spki = NULL;
+  CERTCertificateRequest *request = NULL;
+  CERTValidity *validity = NULL;
+  CERTCertificate *cert = NULL;
+  SECItem der = { .data = NULL, .len = 0 };
+  SECItem signed_der = { .data = NULL, .len = 0 };
+
+  CERTCertificate *result_cert = NULL;
+
+  validity = CERT_CreateValidity(((PRTime)start_time) * PRTIME_PER_SEC,
+                                 ((PRTime)end_time) * PRTIME_PER_SEC);
+  if (BUG(! validity)) {
+    /* LCOV_EXCL_START */
+    crypto_nss_log_errors(LOG_WARN, "creating a validity object");
+    goto err;
+    /* LCOV_EXCL_STOP */
+  }
+
+  unsigned long serial_number;
+  crypto_rand((char*)&serial_number, sizeof(serial_number));
+
+  subject_spki = SECKEY_CreateSubjectPublicKeyInfo(subject_key);
+  if (!subject_spki)
+    goto err;
+
+  /* Make a CSR ... */
+  // XXX do we need to set any attributes?
+  request = CERT_CreateCertificateRequest(subject_dn,
+                                          subject_spki,
+                                          NULL /* attributes */);
+  if (!request)
+    goto err;
+
+  /* Put it into a certificate ... */
+  cert = CERT_CreateCertificate(serial_number,
+                                issuer_dn,
+                                validity,
+                                request);
+  if (!cert)
+    goto err;
+
+  /* version 3 cert */
+  *cert->version.data = 2; /* 2 means version 3. */
+  cert->version.len = 1;
+
+  // XXX do we need to set anything else on the cert?
+
+  /* Sign it. */
+  KeyType privkey_type = SECKEY_GetPrivateKeyType(signing_key);
+  SECOidTag oid_tag = SEC_GetSignatureAlgorithmOidTag(privkey_type,
+                                                      SEC_OID_SHA256);
+  if (oid_tag == SEC_OID_UNKNOWN)
+    goto err;
+  s = SECOID_SetAlgorithmID(cert->arena, &cert->signature, oid_tag, NULL);
+  if (s != SECSuccess)
+    goto err;
+
+  void *tmp;
+  tmp = SEC_ASN1EncodeItem(cert->arena, &der, cert,
+                           SEC_ASN1_GET(CERT_CertificateTemplate));
+  if (!tmp)
+    goto err;
+
+#if 0
+  s = SEC_DerSignDataWithAlgorithmID(cert->arena,
+                                     &signed_der,
+                                     der.data, der.len,
+                                     (SECKEYPrivateKey *)signing_key,//const
+                                     &cert->signature);
+#else
+  s = SEC_DerSignData(cert->arena,
+                      &signed_der,
+                      der.data, der.len,
+                      (SECKEYPrivateKey *)signing_key,//const
+                      SEC_OID_PKCS1_SHA256_WITH_RSA_ENCRYPTION);
+#endif
+
+  if (s != SECSuccess)
+    goto err;
+
+  /* Re-parse it, to make sure all the certificates we actually use
+   * appear via being decoded. */
+  result_cert = tor_x509_cert_decode_internal(signed_der.data, signed_der.len);
+
+#if 1
+  {
+    // Can we check the cert we just signed?
+    tor_assert(result_cert);
+    SECKEYPublicKey *issuer_pk = (SECKEYPublicKey *)
+      crypto_pk_get_nss_pubkey(rsa_sign);
+    SECStatus cert_ok = CERT_VerifySignedDataWithPublicKey(
+                               &result_cert->signatureWrap, issuer_pk, NULL);
+    tor_assert(cert_ok == SECSuccess);
+  }
+#endif
+
+ err:
+  if (subject_spki)
+    SECKEY_DestroySubjectPublicKeyInfo(subject_spki);
+  if (request)
+    CERT_DestroyCertificateRequest(request);
+  if (validity)
+    CERT_DestroyValidity(validity);
+
+  // unnecessary, since these are allocated in the cert's arena.
+  //SECITEM_FreeItem(&der, PR_FALSE);
+  //SECITEM_FreeItem(&signed_der, PR_FALSE);
+  if (cert)
+    CERT_DestroyCertificate(cert);
+
+  return result_cert;
+}
+
+MOCK_IMPL(tor_x509_cert_impl_t *,
+tor_tls_create_certificate,(crypto_pk_t *rsa,
+                            crypto_pk_t *rsa_sign,
+                            const char *cname,
+                            const char *cname_sign,
+                            unsigned int cert_lifetime))
+{
+  tor_assert(rsa);
+  tor_assert(rsa_sign);
+  tor_assert(cname);
+  tor_assert(cname_sign);
+
+  char *cname_rfc_1485 = NULL, *cname_sign_rfc_1485 = NULL;
+  CERTName *subject_dn = NULL, *issuer_dn = NULL;
+  time_t start_time;
+  time_t end_time;
+  CERTCertificate *result = NULL;
+
+  tor_asprintf(&cname_rfc_1485, "CN=%s", cname);
+  tor_asprintf(&cname_sign_rfc_1485, "CN=%s", cname_sign);
+
+  subject_dn = CERT_AsciiToName(cname_rfc_1485);
+  issuer_dn = CERT_AsciiToName(cname_sign_rfc_1485);
+  if (!subject_dn || !issuer_dn)
+    goto err;
+
+  tor_tls_pick_certificate_lifetime(time(NULL), cert_lifetime,
+                                    &start_time, &end_time);
+
+  result = tor_tls_create_certificate_internal(rsa,
+                                               rsa_sign,
+                                               subject_dn,
+                                               issuer_dn,
+                                               start_time,
+                                               end_time);
+ err:
+  tor_free(cname_rfc_1485);
+  tor_free(cname_sign_rfc_1485);
+  if (subject_dn)
+    CERT_DestroyName(subject_dn);
+  if (issuer_dn)
+    CERT_DestroyName(issuer_dn);
+
+  return result;
+}
+
+/** Set *<b>encoded_out</b> and *<b>size_out</b> to <b>cert</b>'s encoded DER
+ * representation and length, respectively. */
+void
+tor_x509_cert_get_der(const tor_x509_cert_t *cert,
+                 const uint8_t **encoded_out, size_t *size_out)
+{
+  tor_assert(cert);
+  tor_assert(cert->cert);
+  tor_assert(encoded_out);
+  tor_assert(size_out);
+
+  const SECItem *item = &cert->cert->derCert;
+  *encoded_out = item->data;
+  *size_out = (size_t)item->len;
+}
+
+void
+tor_x509_cert_impl_free_(tor_x509_cert_impl_t *cert)
+{
+  if (cert)
+    CERT_DestroyCertificate(cert);
+}
+
+tor_x509_cert_impl_t *
+tor_x509_cert_impl_dup_(tor_x509_cert_impl_t *cert)
+{
+  if (cert)
+    return CERT_DupCertificate(cert);
+  else
+    return NULL;
+}
+
+/**
+ * As tor_x509_cert_decode, but return the NSS certificate type
+*/
+static tor_x509_cert_impl_t *
+tor_x509_cert_decode_internal(const uint8_t *certificate,
+                              int certificate_len)
+{
+  tor_assert(certificate);
+  if (certificate_len > INT_MAX)
+    return NULL;
+
+  SECItem der = { .type = siBuffer,
+                  .data = (unsigned char *)certificate,
+                  .len = certificate_len };
+  CERTCertDBHandle *certdb = CERT_GetDefaultCertDB();
+  tor_assert(certdb);
+  return CERT_NewTempCertificate(certdb,
+                                 &der,
+                                 NULL /* nickname */,
+                                 PR_FALSE, /* isPerm */
+                                 PR_TRUE /* CopyDER */);
+}
+
+tor_x509_cert_t *
+tor_x509_cert_decode(const uint8_t *certificate,
+                     size_t certificate_len)
+{
+  CERTCertificate *cert = tor_x509_cert_decode_internal(certificate,
+                                                        (int)certificate_len);
+  if (! cert) {
+    crypto_nss_log_errors(LOG_INFO, "decoding certificate");
+    return NULL;
+  }
+
+  tor_x509_cert_t *newcert = tor_x509_cert_new(cert);
+
+  return newcert;
+}
+
+crypto_pk_t *
+tor_tls_cert_get_key(tor_x509_cert_t *cert)
+{
+  tor_assert(cert);
+  CERTSubjectPublicKeyInfo *spki = &cert->cert->subjectPublicKeyInfo;
+  SECKEYPublicKey *pub = SECKEY_ExtractPublicKey(spki); // we own this pointer
+  if (pub == NULL)
+    return NULL;
+
+  if (SECKEY_GetPublicKeyType(pub) != rsaKey) {
+    SECKEY_DestroyPublicKey(pub);
+    return NULL;
+  }
+
+  return crypto_pk_new_from_nss_pubkey(pub);
+}
+
+int
+tor_tls_cert_is_valid(int severity,
+                      const tor_x509_cert_t *cert,
+                      const tor_x509_cert_t *signing_cert,
+                      time_t now,
+                      int check_rsa_1024)
+{
+  int result = 0;
+
+  tor_assert(cert);
+  tor_assert(signing_cert);
+
+  SECKEYPublicKey *pk = CERT_ExtractPublicKey(signing_cert->cert);
+  if (pk == NULL) {
+    log_fn(severity, LD_CRYPTO,
+           "Invalid certificate: could not extract issuer key");
+    goto fail;
+  }
+
+  SECStatus s = CERT_VerifySignedDataWithPublicKey(&cert->cert->signatureWrap,
+                                                   pk, NULL);
+  if (s != SECSuccess) {
+    log_fn(severity, LD_CRYPTO,
+           "Invalid certificate: could not validate signature.");
+    goto fail;
+  }
+
+  if (tor_x509_check_cert_lifetime_internal(severity,
+                                            cert->cert,
+                                            now,
+                                            TOR_X509_PAST_SLOP,
+                                            TOR_X509_FUTURE_SLOP) < 0)
+    goto fail;
+
+  if (check_rsa_1024) {
+    /* We require that this is a 1024-bit RSA key, for legacy reasons .:p */
+    if (SECKEY_GetPublicKeyType(pk) != rsaKey ||
+        SECKEY_PublicKeyStrengthInBits(pk) != 1024) {
+      log_fn(severity, LD_CRYPTO, "Invalid certificate: Key is not RSA1024.");
+      goto fail;
+    }
+  } else {
+    /* We require that this key is at least minimally strong. */
+    unsigned min_bits = (SECKEY_GetPublicKeyType(pk) == ecKey) ? 128: 1024;
+    if (SECKEY_PublicKeyStrengthInBits(pk) < min_bits) {
+      log_fn(severity, LD_CRYPTO, "Invalid certificate: Key is too weak.");
+      goto fail;
+    }
+  }
+
+  /* The certificate is valid. */
+  result = 1;
+
+ fail:
+  if (pk)
+    SECKEY_DestroyPublicKey(pk);
+  return result;
+}
+
+static void
+log_cert_lifetime(int severity,
+                  const char *status,
+                  time_t now,
+                  PRTime notBefore,
+                  PRTime notAfter)
+{
+  log_fn(severity, LD_GENERAL,
+         "Certificate %s. Either their clock is set wrong, or your clock "
+         "is incorrect.", status);
+
+  char nowbuf[ISO_TIME_LEN+1];
+  char nbbuf[ISO_TIME_LEN+1];
+  char nabuf[ISO_TIME_LEN+1];
+
+  format_iso_time(nowbuf, now);
+  format_iso_time(nbbuf, notBefore / PRTIME_PER_SEC);
+  format_iso_time(nabuf, notAfter / PRTIME_PER_SEC);
+
+  log_fn(severity, LD_GENERAL,
+         "(The certificate is valid from %s until %s. Your time is %s.)",
+         nbbuf, nabuf, nowbuf);
+}
+
+int
+tor_x509_check_cert_lifetime_internal(int severity,
+                                      const tor_x509_cert_impl_t *cert,
+                                      time_t now,
+                                      int past_tolerance,
+                                      int future_tolerance)
+{
+  tor_assert(cert);
+
+  PRTime notBefore=0, notAfter=0;
+  int64_t t;
+  SECStatus r = CERT_GetCertTimes(cert, &notBefore, &notAfter);
+  if (r != SECSuccess) {
+    log_fn(severity, LD_CRYPTO,
+           "Couldn't get validity times from certificate");
+    return -1;
+  }
+
+  t = ((int64_t)now) + future_tolerance;
+  t *= PRTIME_PER_SEC;
+  if (notBefore > t) {
+    log_cert_lifetime(severity, "not yet valid", now,
+                      notBefore, notAfter);
+    return -1;
+  }
+
+  t = ((int64_t)now) - past_tolerance;
+  t *= PRTIME_PER_SEC;
+  if (notAfter < t) {
+    log_cert_lifetime(severity, "already expired", now,
+                      notBefore, notAfter);
+    return -1;
+  }
+
+  return 0;
+}
+
+#ifdef TOR_UNIT_TESTS
+tor_x509_cert_t *
+tor_x509_cert_replace_expiration(const tor_x509_cert_t *inp,
+                                 time_t new_expiration_time,
+                                 crypto_pk_t *signing_key)
+{
+  tor_assert(inp);
+  tor_assert(signing_key);
+
+  PRTime notBefore=0, notAfter=0;
+  SECStatus r = CERT_GetCertTimes(inp->cert, &notBefore, &notAfter);
+  if (r != SECSuccess)
+    return NULL;
+
+  time_t start_time = notBefore / PRTIME_PER_SEC;
+  if (new_expiration_time < start_time) {
+    /* This prevents an NSS error. */
+    start_time = new_expiration_time - 10;
+  }
+
+  crypto_pk_t *subject_key = tor_tls_cert_get_key((tor_x509_cert_t *)inp);
+  if (!subject_key)
+    return NULL;
+
+  CERTCertificate *newcert;
+
+  newcert = tor_tls_create_certificate_internal(subject_key,
+                                                signing_key,
+                                                &inp->cert->subject,
+                                                &inp->cert->issuer,
+                                                start_time,
+                                                new_expiration_time);
+
+  crypto_pk_free(subject_key);
+
+  return newcert ? tor_x509_cert_new(newcert) : NULL;
+}
+#endif
diff --git a/src/lib/tls/x509_openssl.c b/src/lib/tls/x509_openssl.c
new file mode 100644
index 0000000000..a344279c22
--- /dev/null
+++ b/src/lib/tls/x509_openssl.c
@@ -0,0 +1,464 @@
+/* Copyright (c) 2003, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file x509_openssl.c
+ * \brief Wrapper functions to present a consistent interface to
+ * X.509 functions from OpenSSL.
+ **/
+
+#define TOR_X509_PRIVATE
+#include "lib/tls/x509.h"
+#include "lib/tls/x509_internal.h"
+#include "lib/tls/tortls.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/crypt_ops/compat_openssl.h"
+
+/* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in
+ * srtp.h. Suppress the GCC warning so we can build with -Wredundant-decl. */
+DISABLE_GCC_WARNING(redundant-decls)
+
+#include <openssl/opensslv.h>
+
+#ifdef OPENSSL_NO_EC
+#error "We require OpenSSL with ECC support"
+#endif
+
+#include <openssl/err.h>
+#include <openssl/asn1.h>
+#include <openssl/bio.h>
+#include <openssl/bn.h>
+#include <openssl/evp.h>
+#include <openssl/objects.h>
+#include <openssl/rsa.h>
+#include <openssl/x509.h>
+
+ENABLE_GCC_WARNING(redundant-decls)
+
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/encoding/time_fmt.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef OPENSSL_1_1_API
+#define X509_get_notBefore_const(cert) \
+    X509_get0_notBefore(cert)
+#define X509_get_notAfter_const(cert) \
+    X509_get0_notAfter(cert)
+#ifndef X509_get_notBefore
+#define X509_get_notBefore(cert) \
+    X509_getm_notBefore(cert)
+#endif
+#ifndef X509_get_notAfter
+#define X509_get_notAfter(cert) \
+    X509_getm_notAfter(cert)
+#endif
+#else /* ! OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) */
+#define X509_get_notBefore_const(cert) \
+  ((const ASN1_TIME*) X509_get_notBefore((X509 *)cert))
+#define X509_get_notAfter_const(cert) \
+  ((const ASN1_TIME*) X509_get_notAfter((X509 *)cert))
+#endif
+
+/** Return a newly allocated X509 name with commonName <b>cname</b>. */
+static X509_NAME *
+tor_x509_name_new(const char *cname)
+{
+  int nid;
+  X509_NAME *name;
+  /* LCOV_EXCL_BR_START : these branches will only fail on OOM errors */
+  if (!(name = X509_NAME_new()))
+    return NULL;
+  if ((nid = OBJ_txt2nid("commonName")) == NID_undef) goto error;
+  if (!(X509_NAME_add_entry_by_NID(name, nid, MBSTRING_ASC,
+                                   (unsigned char*)cname, -1, -1, 0)))
+    goto error;
+  /* LCOV_EXCL_BR_STOP */
+  return name;
+
+  /* LCOV_EXCL_START : these lines will only execute on out of memory errors*/
+ error:
+  X509_NAME_free(name);
+  return NULL;
+  /* LCOV_EXCL_STOP */
+}
+
+/** Generate and sign an X509 certificate with the public key <b>rsa</b>,
+ * signed by the private key <b>rsa_sign</b>.  The commonName of the
+ * certificate will be <b>cname</b>; the commonName of the issuer will be
+ * <b>cname_sign</b>. The cert will be valid for <b>cert_lifetime</b>
+ * seconds, starting from some time in the past.
+ *
+ * Return a certificate on success, NULL on failure.
+ */
+MOCK_IMPL(X509 *,
+tor_tls_create_certificate,(crypto_pk_t *rsa,
+                            crypto_pk_t *rsa_sign,
+                            const char *cname,
+                            const char *cname_sign,
+                            unsigned int cert_lifetime))
+{
+  /* OpenSSL generates self-signed certificates with random 64-bit serial
+   * numbers, so let's do that too. */
+#define SERIAL_NUMBER_SIZE 8
+
+  time_t start_time, end_time;
+  BIGNUM *serial_number = NULL;
+  unsigned char serial_tmp[SERIAL_NUMBER_SIZE];
+  EVP_PKEY *sign_pkey = NULL, *pkey=NULL;
+  X509 *x509 = NULL;
+  X509_NAME *name = NULL, *name_issuer=NULL;
+
+  tor_tls_init();
+
+  time_t now = time(NULL);
+
+  tor_tls_pick_certificate_lifetime(now, cert_lifetime,
+                                    &start_time, &end_time);
+
+  tor_assert(rsa);
+  tor_assert(cname);
+  tor_assert(rsa_sign);
+  tor_assert(cname_sign);
+  if (!(sign_pkey = crypto_pk_get_openssl_evp_pkey_(rsa_sign,1)))
+    goto error;
+  if (!(pkey = crypto_pk_get_openssl_evp_pkey_(rsa,0)))
+    goto error;
+  if (!(x509 = X509_new()))
+    goto error;
+  if (!(X509_set_version(x509, 2)))
+    goto error;
+
+  { /* our serial number is 8 random bytes. */
+    crypto_rand((char *)serial_tmp, sizeof(serial_tmp));
+    if (!(serial_number = BN_bin2bn(serial_tmp, sizeof(serial_tmp), NULL)))
+      goto error;
+    if (!(BN_to_ASN1_INTEGER(serial_number, X509_get_serialNumber(x509))))
+      goto error;
+  }
+
+  if (!(name = tor_x509_name_new(cname)))
+    goto error;
+  if (!(X509_set_subject_name(x509, name)))
+    goto error;
+  if (!(name_issuer = tor_x509_name_new(cname_sign)))
+    goto error;
+  if (!(X509_set_issuer_name(x509, name_issuer)))
+    goto error;
+
+  if (!X509_time_adj(X509_get_notBefore(x509),0,&start_time))
+    goto error;
+  if (!X509_time_adj(X509_get_notAfter(x509),0,&end_time))
+    goto error;
+  if (!X509_set_pubkey(x509, pkey))
+    goto error;
+
+  if (!X509_sign(x509, sign_pkey, EVP_sha256()))
+    goto error;
+
+  goto done;
+ error:
+  if (x509) {
+    X509_free(x509);
+    x509 = NULL;
+  }
+ done:
+  tls_log_errors(NULL, LOG_WARN, LD_NET, "generating certificate");
+  if (sign_pkey)
+    EVP_PKEY_free(sign_pkey);
+  if (pkey)
+    EVP_PKEY_free(pkey);
+  if (serial_number)
+    BN_clear_free(serial_number);
+  if (name)
+    X509_NAME_free(name);
+  if (name_issuer)
+    X509_NAME_free(name_issuer);
+  return x509;
+
+#undef SERIAL_NUMBER_SIZE
+}
+
+/** Set the 'encoded' and 'encoded_len' fields of "cert" from cert->cert. */
+int
+tor_x509_cert_set_cached_der_encoding(tor_x509_cert_t *cert)
+{
+  unsigned char *buf = NULL;
+  int length = i2d_X509(cert->cert, &buf);
+
+  if (length <= 0 || buf == NULL) {
+    return -1;
+  }
+  cert->encoded_len = (size_t) length;
+  cert->encoded = tor_malloc(length);
+  memcpy(cert->encoded, buf, length);
+  OPENSSL_free(buf);
+  return 0;
+}
+
+void
+tor_x509_cert_impl_free_(tor_x509_cert_impl_t *cert)
+{
+  if (cert)
+    X509_free(cert);
+}
+
+tor_x509_cert_impl_t *
+tor_x509_cert_impl_dup_(tor_x509_cert_impl_t *cert)
+{
+  if (cert)
+    return X509_dup(cert);
+  else
+    return NULL;
+}
+
+/** Set *<b>encoded_out</b> and *<b>size_out</b> to <b>cert</b>'s encoded DER
+ * representation and length, respectively. */
+void
+tor_x509_cert_get_der(const tor_x509_cert_t *cert,
+                 const uint8_t **encoded_out, size_t *size_out)
+{
+  tor_assert(cert);
+  tor_assert(encoded_out);
+  tor_assert(size_out);
+  *encoded_out = cert->encoded;
+  *size_out = cert->encoded_len;
+}
+
+/** Read a DER-encoded X509 cert, of length exactly <b>certificate_len</b>,
+ * from a <b>certificate</b>.  Return a newly allocated tor_x509_cert_t on
+ * success and NULL on failure. */
+tor_x509_cert_t *
+tor_x509_cert_decode(const uint8_t *certificate, size_t certificate_len)
+{
+  X509 *x509;
+  const unsigned char *cp = (const unsigned char *)certificate;
+  tor_x509_cert_t *newcert;
+  tor_assert(certificate);
+  check_no_tls_errors();
+
+  if (certificate_len > INT_MAX)
+    goto err;
+
+  x509 = d2i_X509(NULL, &cp, (int)certificate_len);
+
+  if (!x509)
+    goto err; /* Couldn't decode */
+  if (cp - certificate != (int)certificate_len) {
+    X509_free(x509);
+    goto err; /* Didn't use all the bytes */
+  }
+  newcert = tor_x509_cert_new(x509);
+  if (!newcert) {
+    goto err;
+  }
+  if (newcert->encoded_len != certificate_len ||
+      fast_memneq(newcert->encoded, certificate, certificate_len)) {
+    /* Cert wasn't in DER */
+    tor_x509_cert_free(newcert);
+    goto err;
+  }
+  return newcert;
+ err:
+  tls_log_errors(NULL, LOG_INFO, LD_CRYPTO, "decoding a certificate");
+  return NULL;
+}
+
+/**
+ * Return a newly allocated copy of the public key that a certificate
+ * certifies. Watch out! This returns NULL if the cert's key is not RSA.
+ */
+crypto_pk_t *
+tor_tls_cert_get_key(tor_x509_cert_t *cert)
+{
+  crypto_pk_t *result = NULL;
+  EVP_PKEY *pkey = X509_get_pubkey(cert->cert);
+  RSA *rsa;
+  if (!pkey)
+    return NULL;
+  rsa = EVP_PKEY_get1_RSA(pkey);
+  if (!rsa) {
+    EVP_PKEY_free(pkey);
+    return NULL;
+  }
+  result = crypto_new_pk_from_openssl_rsa_(rsa);
+  EVP_PKEY_free(pkey);
+  return result;
+}
+
+/** Check whether <b>cert</b> is well-formed, currently live, and correctly
+ * signed by the public key in <b>signing_cert</b>.  If <b>check_rsa_1024</b>,
+ * make sure that it has an RSA key with 1024 bits; otherwise, just check that
+ * the key is long enough. Return 1 if the cert is good, and 0 if it's bad or
+ * we couldn't check it. */
+int
+tor_tls_cert_is_valid(int severity,
+                      const tor_x509_cert_t *cert,
+                      const tor_x509_cert_t *signing_cert,
+                      time_t now,
+                      int check_rsa_1024)
+{
+  check_no_tls_errors();
+  EVP_PKEY *cert_key;
+  int r, key_ok = 0;
+
+  if (!signing_cert || !cert)
+    goto bad;
+
+  EVP_PKEY *signing_key = X509_get_pubkey(signing_cert->cert);
+  if (!signing_key)
+    goto bad;
+  r = X509_verify(cert->cert, signing_key);
+  EVP_PKEY_free(signing_key);
+  if (r <= 0)
+    goto bad;
+
+  /* okay, the signature checked out right.  Now let's check the check the
+   * lifetime. */
+  if (tor_x509_check_cert_lifetime_internal(severity, cert->cert, now,
+                                            TOR_X509_PAST_SLOP,
+                                            TOR_X509_FUTURE_SLOP) < 0)
+    goto bad;
+
+  cert_key = X509_get_pubkey(cert->cert);
+  if (check_rsa_1024 && cert_key) {
+    RSA *rsa = EVP_PKEY_get1_RSA(cert_key);
+#ifdef OPENSSL_1_1_API
+    if (rsa && RSA_bits(rsa) == 1024) {
+#else
+    if (rsa && BN_num_bits(rsa->n) == 1024) {
+#endif
+      key_ok = 1;
+    } else {
+      log_fn(severity, LD_CRYPTO, "Invalid certificate: Key is not RSA1024.");
+    }
+
+    if (rsa)
+      RSA_free(rsa);
+  } else if (cert_key) {
+    int min_bits = 1024;
+#ifdef EVP_PKEY_EC
+    if (EVP_PKEY_base_id(cert_key) == EVP_PKEY_EC)
+      min_bits = 128;
+#endif
+    if (EVP_PKEY_bits(cert_key) >= min_bits)
+      key_ok = 1;
+  }
+  EVP_PKEY_free(cert_key);
+  if (!key_ok)
+    goto bad;
+
+  /* XXXX compare DNs or anything? */
+
+  return 1;
+ bad:
+  tls_log_errors(NULL, LOG_INFO, LD_CRYPTO, "checking a certificate");
+  return 0;
+}
+
+/** Warn that a certificate lifetime extends through a certain range. */
+static void
+log_cert_lifetime(int severity, const X509 *cert, const char *problem,
+                  time_t now)
+{
+  BIO *bio = NULL;
+  BUF_MEM *buf;
+  char *s1=NULL, *s2=NULL;
+  char mytime[33];
+  struct tm tm;
+  size_t n;
+
+  if (problem)
+    tor_log(severity, LD_GENERAL,
+        "Certificate %s. Either their clock is set wrong, or your clock "
+        "is wrong.",
+           problem);
+
+  if (!(bio = BIO_new(BIO_s_mem()))) {
+    log_warn(LD_GENERAL, "Couldn't allocate BIO!"); goto end;
+  }
+  if (!(ASN1_TIME_print(bio, X509_get_notBefore_const(cert)))) {
+    tls_log_errors(NULL, LOG_WARN, LD_NET, "printing certificate lifetime");
+    goto end;
+  }
+  BIO_get_mem_ptr(bio, &buf);
+  s1 = tor_strndup(buf->data, buf->length);
+
+  (void)BIO_reset(bio);
+  if (!(ASN1_TIME_print(bio, X509_get_notAfter_const(cert)))) {
+    tls_log_errors(NULL, LOG_WARN, LD_NET, "printing certificate lifetime");
+    goto end;
+  }
+  BIO_get_mem_ptr(bio, &buf);
+  s2 = tor_strndup(buf->data, buf->length);
+
+  n = strftime(mytime, 32, "%b %d %H:%M:%S %Y UTC", tor_gmtime_r(&now, &tm));
+  if (n > 0) {
+    tor_log(severity, LD_GENERAL,
+        "(certificate lifetime runs from %s through %s. Your time is %s.)",
+        s1,s2,mytime);
+  } else {
+    tor_log(severity, LD_GENERAL,
+        "(certificate lifetime runs from %s through %s. "
+        "Couldn't get your time.)",
+        s1, s2);
+  }
+
+ end:
+  /* Not expected to get invoked */
+  tls_log_errors(NULL, LOG_WARN, LD_NET, "getting certificate lifetime");
+  if (bio)
+    BIO_free(bio);
+  tor_free(s1);
+  tor_free(s2);
+}
+
+/** Helper: check whether <b>cert</b> is expired give or take
+ * <b>past_tolerance</b> seconds, or not-yet-valid give or take
+ * <b>future_tolerance</b> seconds.  (Relative to the current time
+ * <b>now</b>.)  If it is live, return 0.  If it is not live, log a message
+ * and return -1. */
+int
+tor_x509_check_cert_lifetime_internal(int severity, const X509 *cert,
+                                      time_t now,
+                                      int past_tolerance, int future_tolerance)
+{
+  time_t t;
+
+  t = now + future_tolerance;
+  if (X509_cmp_time(X509_get_notBefore_const(cert), &t) > 0) {
+    log_cert_lifetime(severity, cert, "not yet valid", now);
+    return -1;
+  }
+  t = now - past_tolerance;
+  if (X509_cmp_time(X509_get_notAfter_const(cert), &t) < 0) {
+    log_cert_lifetime(severity, cert, "already expired", now);
+    return -1;
+  }
+
+  return 0;
+}
+
+#ifdef TOR_UNIT_TESTS
+/* Testing only: return a new x509 cert with the same contents as <b>inp</b>,
+   but with the expiration time <b>new_expiration_time</b>, signed with
+   <b>signing_key</b>. */
+STATIC tor_x509_cert_t *
+tor_x509_cert_replace_expiration(const tor_x509_cert_t *inp,
+                                 time_t new_expiration_time,
+                                 crypto_pk_t *signing_key)
+{
+  X509 *newc = X509_dup(inp->cert);
+  X509_time_adj(X509_get_notAfter(newc), 0, &new_expiration_time);
+  EVP_PKEY *pk = crypto_pk_get_openssl_evp_pkey_(signing_key, 1);
+  tor_assert(X509_sign(newc, pk, EVP_sha256()));
+  EVP_PKEY_free(pk);
+  return tor_x509_cert_new(newc);
+}
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/lib/trace/.may_include b/src/lib/trace/.may_include
new file mode 100644
index 0000000000..45cd13676b
--- /dev/null
+++ b/src/lib/trace/.may_include
@@ -0,0 +1,3 @@
+orconfig.h
+lib/log/*.h
+lib/trace/*.h
diff --git a/src/lib/trace/debug.h b/src/lib/trace/debug.h
new file mode 100644
index 0000000000..e35616cf50
--- /dev/null
+++ b/src/lib/trace/debug.h
@@ -0,0 +1,30 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file debug.h
+ * \brief Macros for debugging our event-trace support.
+ **/
+
+#ifndef TOR_TRACE_LOG_DEBUG_H
+#define TOR_TRACE_LOG_DEBUG_H
+
+#include "lib/log/log.h"
+
+/* Stringify pre-processor trick. */
+#define XSTR(d) STR(d)
+#define STR(s) #s
+
+/* Send every event to a debug log level. This is useful to debug new trace
+ * events without implementing them for a specific event tracing framework.
+ * Note that the arguments are ignored since at this step we do not know the
+ * types and amount there is. */
+
+/* Example on how to map a tracepoint to log_debug(). */
+#undef tor_trace
+#define tor_trace(subsystem, name, args...) \
+  log_debug(LD_GENERAL, "Trace event \"" XSTR(name) "\" from " \
+                        "\"" XSTR(subsystem) "\" hit. " \
+                        "(line "XSTR(__LINE__) ")")
+
+#endif /* TOR_TRACE_LOG_DEBUG_H */
diff --git a/src/lib/trace/events.h b/src/lib/trace/events.h
new file mode 100644
index 0000000000..1e1e7b9d16
--- /dev/null
+++ b/src/lib/trace/events.h
@@ -0,0 +1,45 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file events.h
+ * \brief Header file for Tor event tracing.
+ **/
+
+#ifndef TOR_TRACE_EVENTS_H
+#define TOR_TRACE_EVENTS_H
+
+/*
+ * The following defines a generic event tracing function name that has to be
+ * used to trace events in the code base.
+ *
+ * That generic function is then defined by a event tracing framework. For
+ * instance, the "log debug" framework sends all trace events to log_debug()
+ * which is defined in src/trace/debug.h which can only be enabled at compile
+ * time (--enable-event-tracing-debug).
+ *
+ * By default, every trace events in the code base are replaced by a NOP. See
+ * doc/HACKING/Tracing.md for more information on how to use event tracing or
+ * add events.
+ */
+
+#ifdef TOR_EVENT_TRACING_ENABLED
+/* Map every trace event to a per subsystem macro. */
+#define tor_trace(subsystem, name, ...) \
+  tor_trace_##subsystem(name, __VA_ARGS__)
+
+/* Enable event tracing for the debug framework where all trace events are
+ * mapped to a log_debug(). */
+#ifdef USE_EVENT_TRACING_DEBUG
+#include "lib/trace/debug.h"
+#endif
+
+#else /* TOR_EVENT_TRACING_ENABLED */
+
+/* Reaching this point, we NOP every event declaration because event tracing
+ * is not been enabled at compile time. */
+#define tor_trace(subsystem, name, args...)
+
+#endif /* TOR_EVENT_TRACING_ENABLED */
+
+#endif /* TOR_TRACE_EVENTS_H */
diff --git a/src/lib/trace/include.am b/src/lib/trace/include.am
new file mode 100644
index 0000000000..6f10c98744
--- /dev/null
+++ b/src/lib/trace/include.am
@@ -0,0 +1,18 @@
+
+noinst_LIBRARIES += \
+	src/lib/libtor-trace.a
+
+TRACEHEADERS = \
+	src/lib/trace/trace.h \
+	src/lib/trace/events.h
+
+if USE_EVENT_TRACING_DEBUG
+TRACEHEADERS += \
+	src/lib/trace/debug.h
+endif
+
+# Library source files.
+src_lib_libtor_trace_a_SOURCES = \
+	src/lib/trace/trace.c
+
+noinst_HEADERS+= $(TRACEHEADERS)
diff --git a/src/lib/trace/trace.c b/src/lib/trace/trace.c
new file mode 100644
index 0000000000..18be63c5a8
--- /dev/null
+++ b/src/lib/trace/trace.c
@@ -0,0 +1,17 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file trace.c
+ * \brief Common functions for event-tracing implementation
+ *
+ * See trace.h and doc/HACKING/Tracing.md for more information.
+ **/
+
+#include "lib/trace/trace.h"
+
+/** Initialize the tracing library. */
+void
+tor_trace_init(void)
+{
+}
diff --git a/src/lib/trace/trace.h b/src/lib/trace/trace.h
new file mode 100644
index 0000000000..606d435568
--- /dev/null
+++ b/src/lib/trace/trace.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file trace.h
+ * \brief Header for trace.c
+ **/
+
+#ifndef TOR_TRACE_TRACE_H
+#define TOR_TRACE_TRACE_H
+
+void tor_trace_init(void);
+
+#endif // TOR_TRACE_TRACE_H
diff --git a/src/lib/wallclock/.may_include b/src/lib/wallclock/.may_include
new file mode 100644
index 0000000000..dc010da063
--- /dev/null
+++ b/src/lib/wallclock/.may_include
@@ -0,0 +1,6 @@
+orconfig.h
+lib/cc/*.h
+lib/err/*.h
+lib/wallclock/*.h
+lib/string/*.h
+lib/testsupport/*.h
diff --git a/src/lib/wallclock/approx_time.c b/src/lib/wallclock/approx_time.c
new file mode 100644
index 0000000000..ee498702d5
--- /dev/null
+++ b/src/lib/wallclock/approx_time.c
@@ -0,0 +1,43 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file approx_time.c
+ * \brief Cache the last result of time(), for performance and testing.
+ **/
+
+#include "orconfig.h"
+#include "lib/wallclock/approx_time.h"
+
+/* =====
+ * Cached time
+ * ===== */
+
+#ifndef TIME_IS_FAST
+/** Cached estimate of the current time.  Updated around once per second;
+ * may be a few seconds off if we are really busy.  This is a hack to avoid
+ * calling time(NULL) (which not everybody has optimized) on critical paths.
+ */
+static time_t cached_approx_time = 0;
+
+/** Return a cached estimate of the current time from when
+ * update_approx_time() was last called.  This is a hack to avoid calling
+ * time(NULL) on critical paths: please do not even think of calling it
+ * anywhere else. */
+time_t
+approx_time(void)
+{
+  return cached_approx_time;
+}
+
+/** Update the cached estimate of the current time.  This function SHOULD be
+ * called once per second, and MUST be called before the first call to
+ * get_approx_time. */
+void
+update_approx_time(time_t now)
+{
+  cached_approx_time = now;
+}
+#endif /* !defined(TIME_IS_FAST) */
diff --git a/src/lib/wallclock/approx_time.h b/src/lib/wallclock/approx_time.h
new file mode 100644
index 0000000000..e6b53f2c27
--- /dev/null
+++ b/src/lib/wallclock/approx_time.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file approx_time.h
+ * \brief Header for approx_time.c
+ **/
+
+#ifndef TOR_APPROX_TIME_H
+#define TOR_APPROX_TIME_H
+
+#include <time.h>
+
+/* Cached time */
+#ifdef TIME_IS_FAST
+#define approx_time() time(NULL)
+#define update_approx_time(t) STMT_NIL
+#else
+time_t approx_time(void);
+void update_approx_time(time_t now);
+#endif /* defined(TIME_IS_FAST) */
+
+#endif
diff --git a/src/lib/wallclock/include.am b/src/lib/wallclock/include.am
new file mode 100644
index 0000000000..1961639bd7
--- /dev/null
+++ b/src/lib/wallclock/include.am
@@ -0,0 +1,22 @@
+
+noinst_LIBRARIES += src/lib/libtor-wallclock.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-wallclock-testing.a
+endif
+
+src_lib_libtor_wallclock_a_SOURCES =			\
+	src/lib/wallclock/approx_time.c			\
+	src/lib/wallclock/time_to_tm.c			\
+	src/lib/wallclock/tor_gettimeofday.c
+
+src_lib_libtor_wallclock_testing_a_SOURCES = \
+	$(src_lib_libtor_wallclock_a_SOURCES)
+src_lib_libtor_wallclock_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_wallclock_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/wallclock/approx_time.h			\
+	src/lib/wallclock/timeval.h			\
+	src/lib/wallclock/time_to_tm.h			\
+	src/lib/wallclock/tor_gettimeofday.h
diff --git a/src/lib/wallclock/time_to_tm.c b/src/lib/wallclock/time_to_tm.c
new file mode 100644
index 0000000000..f7cb21827b
--- /dev/null
+++ b/src/lib/wallclock/time_to_tm.c
@@ -0,0 +1,200 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file time_to_tm.c
+ * \brief Convert to struct tm, portably.
+ **/
+
+#include "orconfig.h"
+#include "lib/cc/torint.h"
+#include "lib/cc/compat_compiler.h"
+#include "lib/wallclock/time_to_tm.h"
+#include "lib/string/printf.h"
+#include "lib/err/torerr.h"
+
+#include <errno.h>
+#include <time.h>
+#include <string.h>
+#include <stdlib.h>
+
+#if !defined(_WIN32)
+/** Defined iff we need to add locks when defining fake versions of reentrant
+ * versions of time-related functions. */
+#define TIME_FNS_NEED_LOCKS
+#endif
+
+/** Helper: Deal with confused or out-of-bounds values from localtime_r and
+ * friends.  (On some platforms, they can give out-of-bounds values or can
+ * return NULL.)  If <b>islocal</b>, this is a localtime result; otherwise
+ * it's from gmtime.  The function returns <b>r</b>, when given <b>timep</b>
+ * as its input. If we need to store new results, store them in
+ * <b>resultbuf</b>. */
+static struct tm *
+correct_tm(int islocal, const time_t *timep, struct tm *resultbuf,
+           struct tm *r, char **err_out)
+{
+  const char *outcome;
+
+  if (PREDICT_LIKELY(r)) {
+    /* We can't strftime dates after 9999 CE, and we want to avoid dates
+     * before 1 CE (avoiding the year 0 issue and negative years). */
+    if (r->tm_year > 8099) {
+      r->tm_year = 8099;
+      r->tm_mon = 11;
+      r->tm_mday = 31;
+      r->tm_yday = 364;
+      r->tm_wday = 6;
+      r->tm_hour = 23;
+      r->tm_min = 59;
+      r->tm_sec = 59;
+    } else if (r->tm_year < (1-1900)) {
+      r->tm_year = (1-1900);
+      r->tm_mon = 0;
+      r->tm_mday = 1;
+      r->tm_yday = 0;
+      r->tm_wday = 0;
+      r->tm_hour = 0;
+      r->tm_min = 0;
+      r->tm_sec = 0;
+    }
+    return r;
+  }
+
+  /* If we get here, gmtime or localtime returned NULL. It might have done
+   * this because of overrun or underrun, or it might have done it because of
+   * some other weird issue. */
+  if (timep) {
+    if (*timep < 0) {
+      r = resultbuf;
+      r->tm_year = 70; /* 1970 CE */
+      r->tm_mon = 0;
+      r->tm_mday = 1;
+      r->tm_yday = 0;
+      r->tm_wday = 0;
+      r->tm_hour = 0;
+      r->tm_min = 0 ;
+      r->tm_sec = 0;
+      outcome = "Rounding up to 1970";
+      goto done;
+    } else if (*timep >= INT32_MAX) {
+      /* Rounding down to INT32_MAX isn't so great, but keep in mind that we
+       * only do it if gmtime/localtime tells us NULL. */
+      r = resultbuf;
+      r->tm_year = 137; /* 2037 CE */
+      r->tm_mon = 11;
+      r->tm_mday = 31;
+      r->tm_yday = 364;
+      r->tm_wday = 6;
+      r->tm_hour = 23;
+      r->tm_min = 59;
+      r->tm_sec = 59;
+      outcome = "Rounding down to 2037";
+      goto done;
+    }
+  }
+
+  /* If we get here, then gmtime/localtime failed without getting an extreme
+   * value for *timep */
+  /* LCOV_EXCL_START */
+  r = resultbuf;
+  memset(resultbuf, 0, sizeof(struct tm));
+  outcome="can't recover";
+  /* LCOV_EXCL_STOP */
+ done:
+  if (err_out) {
+    tor_asprintf(err_out, "%s(%"PRId64") failed with error %s: %s",
+                 islocal?"localtime":"gmtime",
+                 timep?((int64_t)*timep):0,
+                 strerror(errno),
+                 outcome);
+  }
+  return r;
+}
+
+/** @{ */
+/** As localtime_r, but defined for platforms that don't have it:
+ *
+ * Convert *<b>timep</b> to a struct tm in local time, and store the value in
+ * *<b>result</b>.  Return the result on success, or NULL on failure.
+ */
+#ifdef HAVE_LOCALTIME_R
+struct tm *
+tor_localtime_r_msg(const time_t *timep, struct tm *result, char **err_out)
+{
+  struct tm *r;
+  r = localtime_r(timep, result);
+  return correct_tm(1, timep, result, r, err_out);
+}
+#elif defined(TIME_FNS_NEED_LOCKS)
+struct tm *
+tor_localtime_r_msg(const time_t *timep, struct tm *result, char **err_out)
+{
+  struct tm *r;
+  static tor_mutex_t *m=NULL;
+  if (!m) { m=tor_mutex_new(); }
+  raw_assert(result);
+  tor_mutex_acquire(m);
+  r = localtime(timep);
+  if (r)
+    memcpy(result, r, sizeof(struct tm));
+  tor_mutex_release(m);
+  return correct_tm(1, timep, result, r, err_out);
+}
+#else
+struct tm *
+tor_localtime_r_msg(const time_t *timep, struct tm *result, char **err_out)
+{
+  struct tm *r;
+  raw_assert(result);
+  r = localtime(timep);
+  if (r)
+    memcpy(result, r, sizeof(struct tm));
+  return correct_tm(1, timep, result, r, err_out);
+}
+#endif /* defined(HAVE_LOCALTIME_R) || ... */
+/** @} */
+
+/** @{ */
+/** As gmtime_r, but defined for platforms that don't have it:
+ *
+ * Convert *<b>timep</b> to a struct tm in UTC, and store the value in
+ * *<b>result</b>.  Return the result on success, or NULL on failure.
+ */
+#ifdef HAVE_GMTIME_R
+struct tm *
+tor_gmtime_r_msg(const time_t *timep, struct tm *result, char **err_out)
+{
+  struct tm *r;
+  r = gmtime_r(timep, result);
+  return correct_tm(0, timep, result, r, err_out);
+}
+#elif defined(TIME_FNS_NEED_LOCKS)
+struct tm *
+tor_gmtime_r_msg(const time_t *timep, struct tm *result, char **err_out)
+{
+  struct tm *r;
+  static tor_mutex_t *m=NULL;
+  if (!m) { m=tor_mutex_new(); }
+  raw_assert(result);
+  tor_mutex_acquire(m);
+  r = gmtime(timep);
+  if (r)
+    memcpy(result, r, sizeof(struct tm));
+  tor_mutex_release(m);
+  return correct_tm(0, timep, result, r, err_out);
+}
+#else
+struct tm *
+tor_gmtime_r_msg(const time_t *timep, struct tm *result, char **err_out)
+{
+  struct tm *r;
+  raw_assert(result);
+  r = gmtime(timep);
+  if (r)
+    memcpy(result, r, sizeof(struct tm));
+  return correct_tm(0, timep, result, r, err_out);
+}
+#endif /* defined(HAVE_GMTIME_R) || ... */
diff --git a/src/lib/wallclock/time_to_tm.h b/src/lib/wallclock/time_to_tm.h
new file mode 100644
index 0000000000..abe78c0efe
--- /dev/null
+++ b/src/lib/wallclock/time_to_tm.h
@@ -0,0 +1,22 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file time_to_tm.h
+ * \brief Header for time_to_tm.c
+ **/
+
+#ifndef TOR_WALLCLOCK_TIME_TO_TM_H
+#define TOR_WALLCLOCK_TIME_TO_TM_H
+
+#include <sys/types.h>
+
+struct tm;
+struct tm *tor_localtime_r_msg(const time_t *timep, struct tm *result,
+                               char **err_out);
+struct tm *tor_gmtime_r_msg(const time_t *timep, struct tm *result,
+                            char **err_out);
+
+#endif
diff --git a/src/lib/wallclock/timeval.h b/src/lib/wallclock/timeval.h
new file mode 100644
index 0000000000..4967e939bf
--- /dev/null
+++ b/src/lib/wallclock/timeval.h
@@ -0,0 +1,65 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file timeval.h
+ *
+ * \brief Declarations for timeval-related macros that some platforms
+ * are missing.
+ **/
+
+#ifndef TOR_TIMEVAL_H
+#define TOR_TIMEVAL_H
+
+#include "orconfig.h"
+#include "lib/cc/torint.h"
+
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+
+#ifndef timeradd
+/** Replacement for timeradd on platforms that do not have it: sets tvout to
+ * the sum of tv1 and tv2. */
+#define timeradd(tv1,tv2,tvout) \
+  do {                                                  \
+    (tvout)->tv_sec = (tv1)->tv_sec + (tv2)->tv_sec;    \
+    (tvout)->tv_usec = (tv1)->tv_usec + (tv2)->tv_usec; \
+    if ((tvout)->tv_usec >= 1000000) {                  \
+      (tvout)->tv_usec -= 1000000;                      \
+      (tvout)->tv_sec++;                                \
+    }                                                   \
+  } while (0)
+#endif /* !defined(timeradd) */
+
+#ifndef timersub
+/** Replacement for timersub on platforms that do not have it: sets tvout to
+ * tv1 minus tv2. */
+#define timersub(tv1,tv2,tvout) \
+  do {                                                  \
+    (tvout)->tv_sec = (tv1)->tv_sec - (tv2)->tv_sec;    \
+    (tvout)->tv_usec = (tv1)->tv_usec - (tv2)->tv_usec; \
+    if ((tvout)->tv_usec < 0) {                         \
+      (tvout)->tv_usec += 1000000;                      \
+      (tvout)->tv_sec--;                                \
+    }                                                   \
+  } while (0)
+#endif /* !defined(timersub) */
+
+#ifndef timercmp
+/** Replacement for timercmp on platforms that do not have it: returns true
+ * iff the relational operator "op" makes the expression tv1 op tv2 true.
+ *
+ * Note that while this definition should work for all boolean operators, some
+ * platforms' native timercmp definitions do not support >=, <=, or ==.  So
+ * don't use those.
+ */
+#define timercmp(tv1,tv2,op)                    \
+  (((tv1)->tv_sec == (tv2)->tv_sec) ?           \
+   ((tv1)->tv_usec op (tv2)->tv_usec) :         \
+   ((tv1)->tv_sec op (tv2)->tv_sec))
+#endif /* !defined(timercmp) */
+
+#endif
diff --git a/src/lib/wallclock/tor_gettimeofday.c b/src/lib/wallclock/tor_gettimeofday.c
new file mode 100644
index 0000000000..63538f3b81
--- /dev/null
+++ b/src/lib/wallclock/tor_gettimeofday.c
@@ -0,0 +1,82 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file tor_gettimeofday.c
+ * \brief Implementat gettimeofday() for windows, and other platforms without
+ *   it.
+ **/
+
+#include "orconfig.h"
+#include "lib/err/torerr.h"
+#include "lib/wallclock/tor_gettimeofday.h"
+#include "lib/cc/torint.h"
+
+#include <stddef.h>
+#include <stdlib.h>
+
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+
+#ifndef HAVE_GETTIMEOFDAY
+#ifdef HAVE_FTIME
+#include <sys/timeb.h>
+#endif
+#endif
+
+/** Set *timeval to the current time of day.  On error, log and terminate.
+ * (Same as gettimeofday(timeval,NULL), but never returns -1.)
+ */
+MOCK_IMPL(void,
+tor_gettimeofday, (struct timeval *timeval))
+{
+#ifdef _WIN32
+  /* Epoch bias copied from perl: number of units between windows epoch and
+   * Unix epoch. */
+#define EPOCH_BIAS UINT64_C(116444736000000000)
+#define UNITS_PER_SEC UINT64_C(10000000)
+#define USEC_PER_SEC UINT64_C(1000000)
+#define UNITS_PER_USEC UINT64_C(10)
+  union {
+    uint64_t ft_64;
+    FILETIME ft_ft;
+  } ft;
+  /* number of 100-nsec units since Jan 1, 1601 */
+  GetSystemTimeAsFileTime(&ft.ft_ft);
+  if (ft.ft_64 < EPOCH_BIAS) {
+    /* LCOV_EXCL_START */
+    raw_assert_unreached_msg("System time is before 1970; failing.");
+    /* LCOV_EXCL_STOP */
+  }
+  ft.ft_64 -= EPOCH_BIAS;
+  timeval->tv_sec = (unsigned) (ft.ft_64 / UNITS_PER_SEC);
+  timeval->tv_usec = (unsigned) ((ft.ft_64 / UNITS_PER_USEC) % USEC_PER_SEC);
+#elif defined(HAVE_GETTIMEOFDAY)
+  if (gettimeofday(timeval, NULL)) {
+    /* LCOV_EXCL_START */
+    /* If gettimeofday dies, we have either given a bad timezone (we didn't),
+       or segfaulted.*/
+    raw_assert_unreached_msg("gettimeofday failed");
+    /* LCOV_EXCL_STOP */
+  }
+#elif defined(HAVE_FTIME)
+  struct timeb tb;
+  ftime(&tb);
+  timeval->tv_sec = tb.time;
+  timeval->tv_usec = tb.millitm * 1000;
+#else
+#error "No way to get time."
+#endif /* defined(_WIN32) || ... */
+  return;
+}
diff --git a/src/lib/wallclock/tor_gettimeofday.h b/src/lib/wallclock/tor_gettimeofday.h
new file mode 100644
index 0000000000..c7fff9747a
--- /dev/null
+++ b/src/lib/wallclock/tor_gettimeofday.h
@@ -0,0 +1,20 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file tor_gettimeofday.h
+ * \brief Header for tor_gettimeofday.c
+ **/
+
+#ifndef TOR_GETTIMEOFDAY_H
+#define TOR_GETTIMEOFDAY_H
+
+#include "lib/testsupport/testsupport.h"
+
+struct timeval;
+
+MOCK_DECL(void, tor_gettimeofday, (struct timeval *timeval));
+
+#endif
diff --git a/src/or/Makefile.nmake b/src/or/Makefile.nmake
deleted file mode 100644
index 2ac98cd372..0000000000
--- a/src/or/Makefile.nmake
+++ /dev/null
@@ -1,78 +0,0 @@
-all: tor.exe
-
-CFLAGS = /O2 /MT /I ..\win32 /I ..\..\..\build-alpha\include /I ..\common \
-    /I ..\ext
-
-LIBS = ..\..\..\build-alpha\lib\libevent.lib \
- ..\..\..\build-alpha\lib\libcrypto.lib \
- ..\..\..\build-alpha\lib\libssl.lib \
- ..\..\..\build-alpha\lib\libz.lib \
- ws2_32.lib advapi32.lib shell32.lib \
- crypt32.lib gdi32.lib user32.lib
-
-LIBTOR_OBJECTS = \
-  addressmap.obj \
-  buffers.obj \
-  channel.obj \
-  channeltls.obj \
-  circpathbias.obj \
-  circuitbuild.obj \
-  circuitlist.obj \
-  circuitmux.obj \
-  circuitmux_ewma.obj \
-  circuitstats.obj \
-  circuituse.obj \
-  command.obj \
-  config.obj \
-  config_codedigest.obj \
-  confparse.obj \
-  connection.obj \
-  connection_edge.obj \
-  connection_or.obj \
-  control.obj \
-  cpuworker.obj \
-  directory.obj \
-  dirserv.obj \
-  dirvote.obj \
-  dns.obj \
-  dnsserv.obj \
-  ext_orport.obj \
-  fp_pair.obj \
-  entrynodes.obj \
-  geoip.obj \
-  hibernate.obj \
-  main.obj \
-  microdesc.obj \
-  networkstatus.obj \
-  nodelist.obj \
-  ntmain.obj \
-  onion.obj \
-  onion_fast.obj \
-  onion_ntor.obj \
-  onion_tap.obj \
-  policies.obj \
-  reasons.obj \
-  relay.obj \
-  rendclient.obj \
-  rendcommon.obj \
-  rendmid.obj \
-  rendservice.obj \
-  rephist.obj \
-  replaycache.obj \
-  router.obj \
-  routerlist.obj \
-  routerparse.obj \
-  routerset.obj \
-  scheduler.obj \
-  statefile.obj \
-  status.obj \
-  transports.obj
-
-libtor.lib: $(LIBTOR_OBJECTS)
-	lib $(LIBTOR_OBJECTS) /out:$@
-
-tor.exe: libtor.lib tor_main.obj
-	$(CC) $(CFLAGS) $(LIBS) libtor.lib ..\common\*.lib ..\ext\*.lib tor_main.obj /Fe$@
-
-clean:
-	del $(LIBTOR_OBJECTS) tor_main.obj *.lib tor.exe
diff --git a/src/or/buffers.c b/src/or/buffers.c
deleted file mode 100644
index b36e4ab509..0000000000
--- a/src/or/buffers.c
+++ /dev/null
@@ -1,2065 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file buffers.c
- * \brief Implements a generic buffer interface.
- *
- * A buf_t is a (fairly) opaque byte-oriented FIFO that can read to or flush
- * from memory, sockets, file descriptors, TLS connections, or another buf_t.
- * Buffers are implemented as linked lists of memory chunks.
- *
- * All socket-backed and TLS-based connection_t objects have a pair of
- * buffers: one for incoming data, and one for outcoming data.  These are fed
- * and drained from functions in connection.c, trigged by events that are
- * monitored in main.c.
- *
- * This module has basic support for reading and writing on buf_t objects. It
- * also contains specialized functions for handling particular protocols
- * on a buf_t backend, including SOCKS (used in connection_edge.c), Tor cells
- * (used in connection_or.c and channeltls.c), HTTP (used in directory.c), and
- * line-oriented communication (used in control.c).
- **/
-#define BUFFERS_PRIVATE
-#include "or.h"
-#include "addressmap.h"
-#include "buffers.h"
-#include "config.h"
-#include "connection_edge.h"
-#include "connection_or.h"
-#include "control.h"
-#include "reasons.h"
-#include "ext_orport.h"
-#include "util.h"
-#include "torlog.h"
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-//#define PARANOIA
-
-#ifdef PARANOIA
-/** Helper: If PARANOIA is defined, assert that the buffer in local variable
- * <b>buf</b> is well-formed. */
-#define check() STMT_BEGIN assert_buf_ok(buf); STMT_END
-#else
-#define check() STMT_NIL
-#endif
-
-/* Implementation notes:
- *
- * After flirting with memmove, and dallying with ring-buffers, we're finally
- * getting up to speed with the 1970s and implementing buffers as a linked
- * list of small chunks.  Each buffer has such a list; data is removed from
- * the head of the list, and added at the tail.  The list is singly linked,
- * and the buffer keeps a pointer to the head and the tail.
- *
- * Every chunk, except the tail, contains at least one byte of data.  Data in
- * each chunk is contiguous.
- *
- * When you need to treat the first N characters on a buffer as a contiguous
- * string, use the buf_pullup function to make them so.  Don't do this more
- * than necessary.
- *
- * The major free Unix kernels have handled buffers like this since, like,
- * forever.
- */
-
-static void socks_request_set_socks5_error(socks_request_t *req,
-                              socks5_reply_status_t reason);
-
-static int parse_socks(const char *data, size_t datalen, socks_request_t *req,
-                       int log_sockstype, int safe_socks, ssize_t *drain_out,
-                       size_t *want_length_out);
-static int parse_socks_client(const uint8_t *data, size_t datalen,
-                              int state, char **reason,
-                              ssize_t *drain_out);
-
-/* Chunk manipulation functions */
-
-#define CHUNK_HEADER_LEN STRUCT_OFFSET(chunk_t, mem[0])
-
-/* We leave this many NUL bytes at the end of the buffer. */
-#define SENTINEL_LEN 4
-
-/* Header size plus NUL bytes at the end */
-#define CHUNK_OVERHEAD (CHUNK_HEADER_LEN + SENTINEL_LEN)
-
-/** Return the number of bytes needed to allocate a chunk to hold
- * <b>memlen</b> bytes. */
-#define CHUNK_ALLOC_SIZE(memlen) (CHUNK_OVERHEAD + (memlen))
-/** Return the number of usable bytes in a chunk allocated with
- * malloc(<b>memlen</b>). */
-#define CHUNK_SIZE_WITH_ALLOC(memlen) ((memlen) - CHUNK_OVERHEAD)
-
-#define DEBUG_SENTINEL
-
-#ifdef DEBUG_SENTINEL
-#define DBG_S(s) s
-#else
-#define DBG_S(s) (void)0
-#endif
-
-#define CHUNK_SET_SENTINEL(chunk, alloclen) do {                        \
-    uint8_t *a = (uint8_t*) &(chunk)->mem[(chunk)->memlen];             \
-    DBG_S(uint8_t *b = &((uint8_t*)(chunk))[(alloclen)-SENTINEL_LEN]);  \
-    DBG_S(tor_assert(a == b));                                          \
-    memset(a,0,SENTINEL_LEN);                                           \
-  } while (0)
-
-/** Return the next character in <b>chunk</b> onto which data can be appended.
- * If the chunk is full, this might be off the end of chunk->mem. */
-static inline char *
-CHUNK_WRITE_PTR(chunk_t *chunk)
-{
-  return chunk->data + chunk->datalen;
-}
-
-/** Return the number of bytes that can be written onto <b>chunk</b> without
- * running out of space. */
-static inline size_t
-CHUNK_REMAINING_CAPACITY(const chunk_t *chunk)
-{
-  return (chunk->mem + chunk->memlen) - (chunk->data + chunk->datalen);
-}
-
-/** Move all bytes stored in <b>chunk</b> to the front of <b>chunk</b>->mem,
- * to free up space at the end. */
-static inline void
-chunk_repack(chunk_t *chunk)
-{
-  if (chunk->datalen && chunk->data != &chunk->mem[0]) {
-    memmove(chunk->mem, chunk->data, chunk->datalen);
-  }
-  chunk->data = &chunk->mem[0];
-}
-
-/** Keep track of total size of allocated chunks for consistency asserts */
-static size_t total_bytes_allocated_in_chunks = 0;
-static void
-buf_chunk_free_unchecked(chunk_t *chunk)
-{
-  if (!chunk)
-    return;
-#ifdef DEBUG_CHUNK_ALLOC
-  tor_assert(CHUNK_ALLOC_SIZE(chunk->memlen) == chunk->DBG_alloc);
-#endif
-  tor_assert(total_bytes_allocated_in_chunks >=
-             CHUNK_ALLOC_SIZE(chunk->memlen));
-  total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen);
-  tor_free(chunk);
-}
-static inline chunk_t *
-chunk_new_with_alloc_size(size_t alloc)
-{
-  chunk_t *ch;
-  ch = tor_malloc(alloc);
-  ch->next = NULL;
-  ch->datalen = 0;
-#ifdef DEBUG_CHUNK_ALLOC
-  ch->DBG_alloc = alloc;
-#endif
-  ch->memlen = CHUNK_SIZE_WITH_ALLOC(alloc);
-  total_bytes_allocated_in_chunks += alloc;
-  ch->data = &ch->mem[0];
-  CHUNK_SET_SENTINEL(ch, alloc);
-  return ch;
-}
-
-/** Expand <b>chunk</b> until it can hold <b>sz</b> bytes, and return a
- * new pointer to <b>chunk</b>.  Old pointers are no longer valid. */
-static inline chunk_t *
-chunk_grow(chunk_t *chunk, size_t sz)
-{
-  off_t offset;
-  const size_t memlen_orig = chunk->memlen;
-  const size_t orig_alloc = CHUNK_ALLOC_SIZE(memlen_orig);
-  const size_t new_alloc = CHUNK_ALLOC_SIZE(sz);
-  tor_assert(sz > chunk->memlen);
-  offset = chunk->data - chunk->mem;
-  chunk = tor_realloc(chunk, new_alloc);
-  chunk->memlen = sz;
-  chunk->data = chunk->mem + offset;
-#ifdef DEBUG_CHUNK_ALLOC
-  tor_assert(chunk->DBG_alloc == orig_alloc);
-  chunk->DBG_alloc = new_alloc;
-#endif
-  total_bytes_allocated_in_chunks += new_alloc - orig_alloc;
-  CHUNK_SET_SENTINEL(chunk, new_alloc);
-  return chunk;
-}
-
-/** If a read onto the end of a chunk would be smaller than this number, then
- * just start a new chunk. */
-#define MIN_READ_LEN 8
-/** Every chunk should take up at least this many bytes. */
-#define MIN_CHUNK_ALLOC 256
-/** No chunk should take up more than this many bytes. */
-#define MAX_CHUNK_ALLOC 65536
-
-/** Return the allocation size we'd like to use to hold <b>target</b>
- * bytes. */
-STATIC size_t
-preferred_chunk_size(size_t target)
-{
-  tor_assert(target <= SIZE_T_CEILING - CHUNK_OVERHEAD);
-  if (CHUNK_ALLOC_SIZE(target) >= MAX_CHUNK_ALLOC)
-    return CHUNK_ALLOC_SIZE(target);
-  size_t sz = MIN_CHUNK_ALLOC;
-  while (CHUNK_SIZE_WITH_ALLOC(sz) < target) {
-    sz <<= 1;
-  }
-  return sz;
-}
-
-/** Collapse data from the first N chunks from <b>buf</b> into buf->head,
- * growing it as necessary, until buf->head has the first <b>bytes</b> bytes
- * of data from the buffer, or until buf->head has all the data in <b>buf</b>.
- */
-STATIC void
-buf_pullup(buf_t *buf, size_t bytes)
-{
-  chunk_t *dest, *src;
-  size_t capacity;
-  if (!buf->head)
-    return;
-
-  check();
-  if (buf->datalen < bytes)
-    bytes = buf->datalen;
-
-  capacity = bytes;
-  if (buf->head->datalen >= bytes)
-    return;
-
-  if (buf->head->memlen >= capacity) {
-    /* We don't need to grow the first chunk, but we might need to repack it.*/
-    size_t needed = capacity - buf->head->datalen;
-    if (CHUNK_REMAINING_CAPACITY(buf->head) < needed)
-      chunk_repack(buf->head);
-    tor_assert(CHUNK_REMAINING_CAPACITY(buf->head) >= needed);
-  } else {
-    chunk_t *newhead;
-    size_t newsize;
-    /* We need to grow the chunk. */
-    chunk_repack(buf->head);
-    newsize = CHUNK_SIZE_WITH_ALLOC(preferred_chunk_size(capacity));
-    newhead = chunk_grow(buf->head, newsize);
-    tor_assert(newhead->memlen >= capacity);
-    if (newhead != buf->head) {
-      if (buf->tail == buf->head)
-        buf->tail = newhead;
-      buf->head = newhead;
-    }
-  }
-
-  dest = buf->head;
-  while (dest->datalen < bytes) {
-    size_t n = bytes - dest->datalen;
-    src = dest->next;
-    tor_assert(src);
-    if (n >= src->datalen) {
-      memcpy(CHUNK_WRITE_PTR(dest), src->data, src->datalen);
-      dest->datalen += src->datalen;
-      dest->next = src->next;
-      if (buf->tail == src)
-        buf->tail = dest;
-      buf_chunk_free_unchecked(src);
-    } else {
-      memcpy(CHUNK_WRITE_PTR(dest), src->data, n);
-      dest->datalen += n;
-      src->data += n;
-      src->datalen -= n;
-      tor_assert(dest->datalen == bytes);
-    }
-  }
-
-  check();
-}
-
-#ifdef TOR_UNIT_TESTS
-void
-buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz)
-{
-  if (!buf || !buf->head) {
-    *cp = NULL;
-    *sz = 0;
-  } else {
-    *cp = buf->head->data;
-    *sz = buf->head->datalen;
-  }
-}
-#endif
-
-/** Remove the first <b>n</b> bytes from buf. */
-static inline void
-buf_remove_from_front(buf_t *buf, size_t n)
-{
-  tor_assert(buf->datalen >= n);
-  while (n) {
-    tor_assert(buf->head);
-    if (buf->head->datalen > n) {
-      buf->head->datalen -= n;
-      buf->head->data += n;
-      buf->datalen -= n;
-      return;
-    } else {
-      chunk_t *victim = buf->head;
-      n -= victim->datalen;
-      buf->datalen -= victim->datalen;
-      buf->head = victim->next;
-      if (buf->tail == victim)
-        buf->tail = NULL;
-      buf_chunk_free_unchecked(victim);
-    }
-  }
-  check();
-}
-
-/** Create and return a new buf with default chunk capacity <b>size</b>.
- */
-buf_t *
-buf_new_with_capacity(size_t size)
-{
-  buf_t *b = buf_new();
-  b->default_chunk_size = preferred_chunk_size(size);
-  return b;
-}
-
-/** Allocate and return a new buffer with default capacity. */
-buf_t *
-buf_new(void)
-{
-  buf_t *buf = tor_malloc_zero(sizeof(buf_t));
-  buf->magic = BUFFER_MAGIC;
-  buf->default_chunk_size = 4096;
-  return buf;
-}
-
-size_t
-buf_get_default_chunk_size(const buf_t *buf)
-{
-  return buf->default_chunk_size;
-}
-
-/** Remove all data from <b>buf</b>. */
-void
-buf_clear(buf_t *buf)
-{
-  chunk_t *chunk, *next;
-  buf->datalen = 0;
-  for (chunk = buf->head; chunk; chunk = next) {
-    next = chunk->next;
-    buf_chunk_free_unchecked(chunk);
-  }
-  buf->head = buf->tail = NULL;
-}
-
-/** Return the number of bytes stored in <b>buf</b> */
-MOCK_IMPL(size_t,
-buf_datalen, (const buf_t *buf))
-{
-  return buf->datalen;
-}
-
-/** Return the total length of all chunks used in <b>buf</b>. */
-size_t
-buf_allocation(const buf_t *buf)
-{
-  size_t total = 0;
-  const chunk_t *chunk;
-  for (chunk = buf->head; chunk; chunk = chunk->next) {
-    total += CHUNK_ALLOC_SIZE(chunk->memlen);
-  }
-  return total;
-}
-
-/** Return the number of bytes that can be added to <b>buf</b> without
- * performing any additional allocation. */
-size_t
-buf_slack(const buf_t *buf)
-{
-  if (!buf->tail)
-    return 0;
-  else
-    return CHUNK_REMAINING_CAPACITY(buf->tail);
-}
-
-/** Release storage held by <b>buf</b>. */
-void
-buf_free(buf_t *buf)
-{
-  if (!buf)
-    return;
-
-  buf_clear(buf);
-  buf->magic = 0xdeadbeef;
-  tor_free(buf);
-}
-
-/** Return a new copy of <b>in_chunk</b> */
-static chunk_t *
-chunk_copy(const chunk_t *in_chunk)
-{
-  chunk_t *newch = tor_memdup(in_chunk, CHUNK_ALLOC_SIZE(in_chunk->memlen));
-  total_bytes_allocated_in_chunks += CHUNK_ALLOC_SIZE(in_chunk->memlen);
-#ifdef DEBUG_CHUNK_ALLOC
-  newch->DBG_alloc = CHUNK_ALLOC_SIZE(in_chunk->memlen);
-#endif
-  newch->next = NULL;
-  if (in_chunk->data) {
-    off_t offset = in_chunk->data - in_chunk->mem;
-    newch->data = newch->mem + offset;
-  }
-  return newch;
-}
-
-/** Return a new copy of <b>buf</b> */
-buf_t *
-buf_copy(const buf_t *buf)
-{
-  chunk_t *ch;
-  buf_t *out = buf_new();
-  out->default_chunk_size = buf->default_chunk_size;
-  for (ch = buf->head; ch; ch = ch->next) {
-    chunk_t *newch = chunk_copy(ch);
-    if (out->tail) {
-      out->tail->next = newch;
-      out->tail = newch;
-    } else {
-      out->head = out->tail = newch;
-    }
-  }
-  out->datalen = buf->datalen;
-  return out;
-}
-
-/** Append a new chunk with enough capacity to hold <b>capacity</b> bytes to
- * the tail of <b>buf</b>.  If <b>capped</b>, don't allocate a chunk bigger
- * than MAX_CHUNK_ALLOC. */
-static chunk_t *
-buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped)
-{
-  chunk_t *chunk;
-
-  if (CHUNK_ALLOC_SIZE(capacity) < buf->default_chunk_size) {
-    chunk = chunk_new_with_alloc_size(buf->default_chunk_size);
-  } else if (capped && CHUNK_ALLOC_SIZE(capacity) > MAX_CHUNK_ALLOC) {
-    chunk = chunk_new_with_alloc_size(MAX_CHUNK_ALLOC);
-  } else {
-    chunk = chunk_new_with_alloc_size(preferred_chunk_size(capacity));
-  }
-
-  chunk->inserted_time = (uint32_t)monotime_coarse_absolute_msec();
-
-  if (buf->tail) {
-    tor_assert(buf->head);
-    buf->tail->next = chunk;
-    buf->tail = chunk;
-  } else {
-    tor_assert(!buf->head);
-    buf->head = buf->tail = chunk;
-  }
-  check();
-  return chunk;
-}
-
-/** Return the age of the oldest chunk in the buffer <b>buf</b>, in
- * milliseconds.  Requires the current monotonic time, in truncated msec,
- * as its input <b>now</b>.
- */
-uint32_t
-buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now)
-{
-  if (buf->head) {
-    return now - buf->head->inserted_time;
-  } else {
-    return 0;
-  }
-}
-
-size_t
-buf_get_total_allocation(void)
-{
-  return total_bytes_allocated_in_chunks;
-}
-
-/** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into
- * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set
- * *<b>reached_eof</b> to 1.  Return -1 on error, 0 on eof or blocking,
- * and the number of bytes read otherwise. */
-static inline int
-read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most,
-              int *reached_eof, int *socket_error)
-{
-  ssize_t read_result;
-  if (at_most > CHUNK_REMAINING_CAPACITY(chunk))
-    at_most = CHUNK_REMAINING_CAPACITY(chunk);
-  read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0);
-
-  if (read_result < 0) {
-    int e = tor_socket_errno(fd);
-    if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
-#ifdef _WIN32
-      if (e == WSAENOBUFS)
-        log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?");
-#endif
-      *socket_error = e;
-      return -1;
-    }
-    return 0; /* would block. */
-  } else if (read_result == 0) {
-    log_debug(LD_NET,"Encountered eof on fd %d", (int)fd);
-    *reached_eof = 1;
-    return 0;
-  } else { /* actually got bytes. */
-    buf->datalen += read_result;
-    chunk->datalen += read_result;
-    log_debug(LD_NET,"Read %ld bytes. %d on inbuf.", (long)read_result,
-              (int)buf->datalen);
-    tor_assert(read_result < INT_MAX);
-    return (int)read_result;
-  }
-}
-
-/** As read_to_chunk(), but return (negative) error code on error, blocking,
- * or TLS, and the number of bytes read otherwise. */
-static inline int
-read_to_chunk_tls(buf_t *buf, chunk_t *chunk, tor_tls_t *tls,
-                  size_t at_most)
-{
-  int read_result;
-
-  tor_assert(CHUNK_REMAINING_CAPACITY(chunk) >= at_most);
-  read_result = tor_tls_read(tls, CHUNK_WRITE_PTR(chunk), at_most);
-  if (read_result < 0)
-    return read_result;
-  buf->datalen += read_result;
-  chunk->datalen += read_result;
-  return read_result;
-}
-
-/** Read from socket <b>s</b>, writing onto end of <b>buf</b>.  Read at most
- * <b>at_most</b> bytes, growing the buffer as necessary.  If recv() returns 0
- * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on
- * error; else return the number of bytes read.
- */
-/* XXXX indicate "read blocked" somehow? */
-int
-read_to_buf(tor_socket_t s, size_t at_most, buf_t *buf, int *reached_eof,
-            int *socket_error)
-{
-  /* XXXX It's stupid to overload the return values for these functions:
-   * "error status" and "number of bytes read" are not mutually exclusive.
-   */
-  int r = 0;
-  size_t total_read = 0;
-
-  check();
-  tor_assert(reached_eof);
-  tor_assert(SOCKET_OK(s));
-
-  while (at_most > total_read) {
-    size_t readlen = at_most - total_read;
-    chunk_t *chunk;
-    if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) {
-      chunk = buf_add_chunk_with_capacity(buf, at_most, 1);
-      if (readlen > chunk->memlen)
-        readlen = chunk->memlen;
-    } else {
-      size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail);
-      chunk = buf->tail;
-      if (cap < readlen)
-        readlen = cap;
-    }
-
-    r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error);
-    check();
-    if (r < 0)
-      return r; /* Error */
-    tor_assert(total_read+r < INT_MAX);
-    total_read += r;
-    if ((size_t)r < readlen) { /* eof, block, or no more to read. */
-      break;
-    }
-  }
-  return (int)total_read;
-}
-
-/** As read_to_buf, but reads from a TLS connection, and returns a TLS
- * status value rather than the number of bytes read.
- *
- * Using TLS on OR connections complicates matters in two ways.
- *
- * First, a TLS stream has its own read buffer independent of the
- * connection's read buffer.  (TLS needs to read an entire frame from
- * the network before it can decrypt any data.  Thus, trying to read 1
- * byte from TLS can require that several KB be read from the network
- * and decrypted.  The extra data is stored in TLS's decrypt buffer.)
- * Because the data hasn't been read by Tor (it's still inside the TLS),
- * this means that sometimes a connection "has stuff to read" even when
- * poll() didn't return POLLIN. The tor_tls_get_pending_bytes function is
- * used in connection.c to detect TLS objects with non-empty internal
- * buffers and read from them again.
- *
- * Second, the TLS stream's events do not correspond directly to network
- * events: sometimes, before a TLS stream can read, the network must be
- * ready to write -- or vice versa.
- */
-int
-read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf)
-{
-  int r = 0;
-  size_t total_read = 0;
-
-  check_no_tls_errors();
-
-  check();
-
-  while (at_most > total_read) {
-    size_t readlen = at_most - total_read;
-    chunk_t *chunk;
-    if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) {
-      chunk = buf_add_chunk_with_capacity(buf, at_most, 1);
-      if (readlen > chunk->memlen)
-        readlen = chunk->memlen;
-    } else {
-      size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail);
-      chunk = buf->tail;
-      if (cap < readlen)
-        readlen = cap;
-    }
-
-    r = read_to_chunk_tls(buf, chunk, tls, readlen);
-    check();
-    if (r < 0)
-      return r; /* Error */
-    tor_assert(total_read+r < INT_MAX);
-    total_read += r;
-    if ((size_t)r < readlen) /* eof, block, or no more to read. */
-      break;
-  }
-  return (int)total_read;
-}
-
-/** Helper for flush_buf(): try to write <b>sz</b> bytes from chunk
- * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>.  On success, deduct
- * the bytes written from *<b>buf_flushlen</b>.  Return the number of bytes
- * written on success, 0 on blocking, -1 on failure.
- */
-static inline int
-flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz,
-            size_t *buf_flushlen)
-{
-  ssize_t write_result;
-
-  if (sz > chunk->datalen)
-    sz = chunk->datalen;
-  write_result = tor_socket_send(s, chunk->data, sz, 0);
-
-  if (write_result < 0) {
-    int e = tor_socket_errno(s);
-    if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
-#ifdef _WIN32
-      if (e == WSAENOBUFS)
-        log_warn(LD_NET,"write() failed: WSAENOBUFS. Not enough ram?");
-#endif
-      return -1;
-    }
-    log_debug(LD_NET,"write() would block, returning.");
-    return 0;
-  } else {
-    *buf_flushlen -= write_result;
-    buf_remove_from_front(buf, write_result);
-    tor_assert(write_result < INT_MAX);
-    return (int)write_result;
-  }
-}
-
-/** Helper for flush_buf_tls(): try to write <b>sz</b> bytes from chunk
- * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>.  (Tries to write
- * more if there is a forced pending write size.)  On success, deduct the
- * bytes written from *<b>buf_flushlen</b>.  Return the number of bytes
- * written on success, and a TOR_TLS error code on failure or blocking.
- */
-static inline int
-flush_chunk_tls(tor_tls_t *tls, buf_t *buf, chunk_t *chunk,
-                size_t sz, size_t *buf_flushlen)
-{
-  int r;
-  size_t forced;
-  char *data;
-
-  forced = tor_tls_get_forced_write_size(tls);
-  if (forced > sz)
-    sz = forced;
-  if (chunk) {
-    data = chunk->data;
-    tor_assert(sz <= chunk->datalen);
-  } else {
-    data = NULL;
-    tor_assert(sz == 0);
-  }
-  r = tor_tls_write(tls, data, sz);
-  if (r < 0)
-    return r;
-  if (*buf_flushlen > (size_t)r)
-    *buf_flushlen -= r;
-  else
-    *buf_flushlen = 0;
-  buf_remove_from_front(buf, r);
-  log_debug(LD_NET,"flushed %d bytes, %d ready to flush, %d remain.",
-            r,(int)*buf_flushlen,(int)buf->datalen);
-  return r;
-}
-
-/** Write data from <b>buf</b> to the socket <b>s</b>.  Write at most
- * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by
- * the number of bytes actually written, and remove the written bytes
- * from the buffer.  Return the number of bytes written on success,
- * -1 on failure.  Return 0 if write() would block.
- */
-int
-flush_buf(tor_socket_t s, buf_t *buf, size_t sz, size_t *buf_flushlen)
-{
-  /* XXXX It's stupid to overload the return values for these functions:
-   * "error status" and "number of bytes flushed" are not mutually exclusive.
-   */
-  int r;
-  size_t flushed = 0;
-  tor_assert(buf_flushlen);
-  tor_assert(SOCKET_OK(s));
-  tor_assert(*buf_flushlen <= buf->datalen);
-  tor_assert(sz <= *buf_flushlen);
-
-  check();
-  while (sz) {
-    size_t flushlen0;
-    tor_assert(buf->head);
-    if (buf->head->datalen >= sz)
-      flushlen0 = sz;
-    else
-      flushlen0 = buf->head->datalen;
-
-    r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen);
-    check();
-    if (r < 0)
-      return r;
-    flushed += r;
-    sz -= r;
-    if (r == 0 || (size_t)r < flushlen0) /* can't flush any more now. */
-      break;
-  }
-  tor_assert(flushed < INT_MAX);
-  return (int)flushed;
-}
-
-/** As flush_buf(), but writes data to a TLS connection.  Can write more than
- * <b>flushlen</b> bytes.
- */
-int
-flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t flushlen,
-              size_t *buf_flushlen)
-{
-  int r;
-  size_t flushed = 0;
-  ssize_t sz;
-  tor_assert(buf_flushlen);
-  tor_assert(*buf_flushlen <= buf->datalen);
-  tor_assert(flushlen <= *buf_flushlen);
-  sz = (ssize_t) flushlen;
-
-  /* we want to let tls write even if flushlen is zero, because it might
-   * have a partial record pending */
-  check_no_tls_errors();
-
-  check();
-  do {
-    size_t flushlen0;
-    if (buf->head) {
-      if ((ssize_t)buf->head->datalen >= sz)
-        flushlen0 = sz;
-      else
-        flushlen0 = buf->head->datalen;
-    } else {
-      flushlen0 = 0;
-    }
-
-    r = flush_chunk_tls(tls, buf, buf->head, flushlen0, buf_flushlen);
-    check();
-    if (r < 0)
-      return r;
-    flushed += r;
-    sz -= r;
-    if (r == 0) /* Can't flush any more now. */
-      break;
-  } while (sz > 0);
-  tor_assert(flushed < INT_MAX);
-  return (int)flushed;
-}
-
-/** Append <b>string_len</b> bytes from <b>string</b> to the end of
- * <b>buf</b>.
- *
- * Return the new length of the buffer on success, -1 on failure.
- */
-int
-write_to_buf(const char *string, size_t string_len, buf_t *buf)
-{
-  if (!string_len)
-    return (int)buf->datalen;
-  check();
-
-  while (string_len) {
-    size_t copy;
-    if (!buf->tail || !CHUNK_REMAINING_CAPACITY(buf->tail))
-      buf_add_chunk_with_capacity(buf, string_len, 1);
-
-    copy = CHUNK_REMAINING_CAPACITY(buf->tail);
-    if (copy > string_len)
-      copy = string_len;
-    memcpy(CHUNK_WRITE_PTR(buf->tail), string, copy);
-    string_len -= copy;
-    string += copy;
-    buf->datalen += copy;
-    buf->tail->datalen += copy;
-  }
-
-  check();
-  tor_assert(buf->datalen < INT_MAX);
-  return (int)buf->datalen;
-}
-
-/** Helper: copy the first <b>string_len</b> bytes from <b>buf</b>
- * onto <b>string</b>.
- */
-static inline void
-peek_from_buf(char *string, size_t string_len, const buf_t *buf)
-{
-  chunk_t *chunk;
-
-  tor_assert(string);
-  /* make sure we don't ask for too much */
-  tor_assert(string_len <= buf->datalen);
-  /* assert_buf_ok(buf); */
-
-  chunk = buf->head;
-  while (string_len) {
-    size_t copy = string_len;
-    tor_assert(chunk);
-    if (chunk->datalen < copy)
-      copy = chunk->datalen;
-    memcpy(string, chunk->data, copy);
-    string_len -= copy;
-    string += copy;
-    chunk = chunk->next;
-  }
-}
-
-/** Remove <b>string_len</b> bytes from the front of <b>buf</b>, and store
- * them into <b>string</b>.  Return the new buffer size.  <b>string_len</b>
- * must be \<= the number of bytes on the buffer.
- */
-int
-fetch_from_buf(char *string, size_t string_len, buf_t *buf)
-{
-  /* There must be string_len bytes in buf; write them onto string,
-   * then memmove buf back (that is, remove them from buf).
-   *
-   * Return the number of bytes still on the buffer. */
-
-  check();
-  peek_from_buf(string, string_len, buf);
-  buf_remove_from_front(buf, string_len);
-  check();
-  tor_assert(buf->datalen < INT_MAX);
-  return (int)buf->datalen;
-}
-
-/** True iff the cell command <b>command</b> is one that implies a
- * variable-length cell in Tor link protocol <b>linkproto</b>. */
-static inline int
-cell_command_is_var_length(uint8_t command, int linkproto)
-{
-  /* If linkproto is v2 (2), CELL_VERSIONS is the only variable-length cells
-   * work as implemented here. If it's 1, there are no variable-length cells.
-   * Tor does not support other versions right now, and so can't negotiate
-   * them.
-   */
-  switch (linkproto) {
-  case 1:
-    /* Link protocol version 1 has no variable-length cells. */
-    return 0;
-  case 2:
-    /* In link protocol version 2, VERSIONS is the only variable-length cell */
-    return command == CELL_VERSIONS;
-  case 0:
-  case 3:
-  default:
-    /* In link protocol version 3 and later, and in version "unknown",
-     * commands 128 and higher indicate variable-length. VERSIONS is
-     * grandfathered in. */
-    return command == CELL_VERSIONS || command >= 128;
-  }
-}
-
-/** Check <b>buf</b> for a variable-length cell according to the rules of link
- * protocol version <b>linkproto</b>.  If one is found, pull it off the buffer
- * and assign a newly allocated var_cell_t to *<b>out</b>, and return 1.
- * Return 0 if whatever is on the start of buf_t is not a variable-length
- * cell.  Return 1 and set *<b>out</b> to NULL if there seems to be the start
- * of a variable-length cell on <b>buf</b>, but the whole thing isn't there
- * yet. */
-int
-fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto)
-{
-  char hdr[VAR_CELL_MAX_HEADER_SIZE];
-  var_cell_t *result;
-  uint8_t command;
-  uint16_t length;
-  const int wide_circ_ids = linkproto >= MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS;
-  const int circ_id_len = get_circ_id_size(wide_circ_ids);
-  const unsigned header_len = get_var_cell_header_size(wide_circ_ids);
-  check();
-  *out = NULL;
-  if (buf->datalen < header_len)
-    return 0;
-  peek_from_buf(hdr, header_len, buf);
-
-  command = get_uint8(hdr + circ_id_len);
-  if (!(cell_command_is_var_length(command, linkproto)))
-    return 0;
-
-  length = ntohs(get_uint16(hdr + circ_id_len + 1));
-  if (buf->datalen < (size_t)(header_len+length))
-    return 1;
-  result = var_cell_new(length);
-  result->command = command;
-  if (wide_circ_ids)
-    result->circ_id = ntohl(get_uint32(hdr));
-  else
-    result->circ_id = ntohs(get_uint16(hdr));
-
-  buf_remove_from_front(buf, header_len);
-  peek_from_buf((char*) result->payload, length, buf);
-  buf_remove_from_front(buf, length);
-  check();
-
-  *out = result;
-  return 1;
-}
-
-/** Move up to *<b>buf_flushlen</b> bytes from <b>buf_in</b> to
- * <b>buf_out</b>, and modify *<b>buf_flushlen</b> appropriately.
- * Return the number of bytes actually copied.
- */
-int
-move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen)
-{
-  /* We can do way better here, but this doesn't turn up in any profiles. */
-  char b[4096];
-  size_t cp, len;
-  len = *buf_flushlen;
-  if (len > buf_in->datalen)
-    len = buf_in->datalen;
-
-  cp = len; /* Remember the number of bytes we intend to copy. */
-  tor_assert(cp < INT_MAX);
-  while (len) {
-    /* This isn't the most efficient implementation one could imagine, since
-     * it does two copies instead of 1, but I kinda doubt that this will be
-     * critical path. */
-    size_t n = len > sizeof(b) ? sizeof(b) : len;
-    fetch_from_buf(b, n, buf_in);
-    write_to_buf(b, n, buf_out);
-    len -= n;
-  }
-  *buf_flushlen -= cp;
-  return (int)cp;
-}
-
-/** Internal structure: represents a position in a buffer. */
-typedef struct buf_pos_t {
-  const chunk_t *chunk; /**< Which chunk are we pointing to? */
-  int pos;/**< Which character inside the chunk's data are we pointing to? */
-  size_t chunk_pos; /**< Total length of all previous chunks. */
-} buf_pos_t;
-
-/** Initialize <b>out</b> to point to the first character of <b>buf</b>.*/
-static void
-buf_pos_init(const buf_t *buf, buf_pos_t *out)
-{
-  out->chunk = buf->head;
-  out->pos = 0;
-  out->chunk_pos = 0;
-}
-
-/** Advance <b>out</b> to the first appearance of <b>ch</b> at the current
- * position of <b>out</b>, or later.  Return -1 if no instances are found;
- * otherwise returns the absolute position of the character. */
-static off_t
-buf_find_pos_of_char(char ch, buf_pos_t *out)
-{
-  const chunk_t *chunk;
-  int pos;
-  tor_assert(out);
-  if (out->chunk) {
-    if (out->chunk->datalen) {
-      tor_assert(out->pos < (off_t)out->chunk->datalen);
-    } else {
-      tor_assert(out->pos == 0);
-    }
-  }
-  pos = out->pos;
-  for (chunk = out->chunk; chunk; chunk = chunk->next) {
-    char *cp = memchr(chunk->data+pos, ch, chunk->datalen - pos);
-    if (cp) {
-      out->chunk = chunk;
-      tor_assert(cp - chunk->data < INT_MAX);
-      out->pos = (int)(cp - chunk->data);
-      return out->chunk_pos + out->pos;
-    } else {
-      out->chunk_pos += chunk->datalen;
-      pos = 0;
-    }
-  }
-  return -1;
-}
-
-/** Advance <b>pos</b> by a single character, if there are any more characters
- * in the buffer.  Returns 0 on success, -1 on failure. */
-static inline int
-buf_pos_inc(buf_pos_t *pos)
-{
-  tor_assert(pos->pos < INT_MAX - 1);
-  ++pos->pos;
-  if (pos->pos == (off_t)pos->chunk->datalen) {
-    if (!pos->chunk->next)
-      return -1;
-    pos->chunk_pos += pos->chunk->datalen;
-    pos->chunk = pos->chunk->next;
-    pos->pos = 0;
-  }
-  return 0;
-}
-
-/** Return true iff the <b>n</b>-character string in <b>s</b> appears
- * (verbatim) at <b>pos</b>. */
-static int
-buf_matches_at_pos(const buf_pos_t *pos, const char *s, size_t n)
-{
-  buf_pos_t p;
-  if (!n)
-    return 1;
-
-  memcpy(&p, pos, sizeof(p));
-
-  while (1) {
-    char ch = p.chunk->data[p.pos];
-    if (ch != *s)
-      return 0;
-    ++s;
-    /* If we're out of characters that don't match, we match.  Check this
-     * _before_ we test incrementing pos, in case we're at the end of the
-     * string. */
-    if (--n == 0)
-      return 1;
-    if (buf_pos_inc(&p)<0)
-      return 0;
-  }
-}
-
-/** Return the first position in <b>buf</b> at which the <b>n</b>-character
- * string <b>s</b> occurs, or -1 if it does not occur. */
-STATIC int
-buf_find_string_offset(const buf_t *buf, const char *s, size_t n)
-{
-  buf_pos_t pos;
-  buf_pos_init(buf, &pos);
-  while (buf_find_pos_of_char(*s, &pos) >= 0) {
-    if (buf_matches_at_pos(&pos, s, n)) {
-      tor_assert(pos.chunk_pos + pos.pos < INT_MAX);
-      return (int)(pos.chunk_pos + pos.pos);
-    } else {
-      if (buf_pos_inc(&pos)<0)
-        return -1;
-    }
-  }
-  return -1;
-}
-
-/** There is a (possibly incomplete) http statement on <b>buf</b>, of the
- * form "\%s\\r\\n\\r\\n\%s", headers, body. (body may contain NULs.)
- * If a) the headers include a Content-Length field and all bytes in
- * the body are present, or b) there's no Content-Length field and
- * all headers are present, then:
- *
- *  - strdup headers into <b>*headers_out</b>, and NUL-terminate it.
- *  - memdup body into <b>*body_out</b>, and NUL-terminate it.
- *  - Then remove them from <b>buf</b>, and return 1.
- *
- *  - If headers or body is NULL, discard that part of the buf.
- *  - If a headers or body doesn't fit in the arg, return -1.
- *  (We ensure that the headers or body don't exceed max len,
- *   _even if_ we're planning to discard them.)
- *  - If force_complete is true, then succeed even if not all of the
- *    content has arrived.
- *
- * Else, change nothing and return 0.
- */
-int
-fetch_from_buf_http(buf_t *buf,
-                    char **headers_out, size_t max_headerlen,
-                    char **body_out, size_t *body_used, size_t max_bodylen,
-                    int force_complete)
-{
-  char *headers, *p;
-  size_t headerlen, bodylen, contentlen;
-  int crlf_offset;
-
-  check();
-  if (!buf->head)
-    return 0;
-
-  crlf_offset = buf_find_string_offset(buf, "\r\n\r\n", 4);
-  if (crlf_offset > (int)max_headerlen ||
-      (crlf_offset < 0 && buf->datalen > max_headerlen)) {
-    log_debug(LD_HTTP,"headers too long.");
-    return -1;
-  } else if (crlf_offset < 0) {
-    log_debug(LD_HTTP,"headers not all here yet.");
-    return 0;
-  }
-  /* Okay, we have a full header.  Make sure it all appears in the first
-   * chunk. */
-  if ((int)buf->head->datalen < crlf_offset + 4)
-    buf_pullup(buf, crlf_offset+4);
-  headerlen = crlf_offset + 4;
-
-  headers = buf->head->data;
-  bodylen = buf->datalen - headerlen;
-  log_debug(LD_HTTP,"headerlen %d, bodylen %d.", (int)headerlen, (int)bodylen);
-
-  if (max_headerlen <= headerlen) {
-    log_warn(LD_HTTP,"headerlen %d larger than %d. Failing.",
-             (int)headerlen, (int)max_headerlen-1);
-    return -1;
-  }
-  if (max_bodylen <= bodylen) {
-    log_warn(LD_HTTP,"bodylen %d larger than %d. Failing.",
-             (int)bodylen, (int)max_bodylen-1);
-    return -1;
-  }
-
-#define CONTENT_LENGTH "\r\nContent-Length: "
-  p = (char*) tor_memstr(headers, headerlen, CONTENT_LENGTH);
-  if (p) {
-    int i;
-    i = atoi(p+strlen(CONTENT_LENGTH));
-    if (i < 0) {
-      log_warn(LD_PROTOCOL, "Content-Length is less than zero; it looks like "
-               "someone is trying to crash us.");
-      return -1;
-    }
-    contentlen = i;
-    /* if content-length is malformed, then our body length is 0. fine. */
-    log_debug(LD_HTTP,"Got a contentlen of %d.",(int)contentlen);
-    if (bodylen < contentlen) {
-      if (!force_complete) {
-        log_debug(LD_HTTP,"body not all here yet.");
-        return 0; /* not all there yet */
-      }
-    }
-    if (bodylen > contentlen) {
-      bodylen = contentlen;
-      log_debug(LD_HTTP,"bodylen reduced to %d.",(int)bodylen);
-    }
-  }
-  /* all happy. copy into the appropriate places, and return 1 */
-  if (headers_out) {
-    *headers_out = tor_malloc(headerlen+1);
-    fetch_from_buf(*headers_out, headerlen, buf);
-    (*headers_out)[headerlen] = 0; /* NUL terminate it */
-  }
-  if (body_out) {
-    tor_assert(body_used);
-    *body_used = bodylen;
-    *body_out = tor_malloc(bodylen+1);
-    fetch_from_buf(*body_out, bodylen, buf);
-    (*body_out)[bodylen] = 0; /* NUL terminate it */
-  }
-  check();
-  return 1;
-}
-
-/**
- * Wait this many seconds before warning the user about using SOCKS unsafely
- * again (requires that WarnUnsafeSocks is turned on). */
-#define SOCKS_WARN_INTERVAL 5
-
-/** Warn that the user application has made an unsafe socks request using
- * protocol <b>socks_protocol</b> on port <b>port</b>.  Don't warn more than
- * once per SOCKS_WARN_INTERVAL, unless <b>safe_socks</b> is set. */
-static void
-log_unsafe_socks_warning(int socks_protocol, const char *address,
-                         uint16_t port, int safe_socks)
-{
-  static ratelim_t socks_ratelim = RATELIM_INIT(SOCKS_WARN_INTERVAL);
-
-  const or_options_t *options = get_options();
-  if (! options->WarnUnsafeSocks)
-    return;
-  if (safe_socks) {
-    log_fn_ratelim(&socks_ratelim, LOG_WARN, LD_APP,
-             "Your application (using socks%d to port %d) is giving "
-             "Tor only an IP address. Applications that do DNS resolves "
-             "themselves may leak information. Consider using Socks4A "
-             "(e.g. via privoxy or socat) instead. For more information, "
-             "please see https://wiki.torproject.org/TheOnionRouter/"
-             "TorFAQ#SOCKSAndDNS.%s",
-             socks_protocol,
-             (int)port,
-             safe_socks ? " Rejecting." : "");
-  }
-  control_event_client_status(LOG_WARN,
-                              "DANGEROUS_SOCKS PROTOCOL=SOCKS%d ADDRESS=%s:%d",
-                              socks_protocol, address, (int)port);
-}
-
-/** Do not attempt to parse socks messages longer than this.  This value is
- * actually significantly higher than the longest possible socks message. */
-#define MAX_SOCKS_MESSAGE_LEN 512
-
-/** Return a new socks_request_t. */
-socks_request_t *
-socks_request_new(void)
-{
-  return tor_malloc_zero(sizeof(socks_request_t));
-}
-
-/** Free all storage held in the socks_request_t <b>req</b>. */
-void
-socks_request_free(socks_request_t *req)
-{
-  if (!req)
-    return;
-  if (req->username) {
-    memwipe(req->username, 0x10, req->usernamelen);
-    tor_free(req->username);
-  }
-  if (req->password) {
-    memwipe(req->password, 0x04, req->passwordlen);
-    tor_free(req->password);
-  }
-  memwipe(req, 0xCC, sizeof(socks_request_t));
-  tor_free(req);
-}
-
-/** There is a (possibly incomplete) socks handshake on <b>buf</b>, of one
- * of the forms
- *  - socks4: "socksheader username\\0"
- *  - socks4a: "socksheader username\\0 destaddr\\0"
- *  - socks5 phase one: "version #methods methods"
- *  - socks5 phase two: "version command 0 addresstype..."
- * If it's a complete and valid handshake, and destaddr fits in
- *   MAX_SOCKS_ADDR_LEN bytes, then pull the handshake off the buf,
- *   assign to <b>req</b>, and return 1.
- *
- * If it's invalid or too big, return -1.
- *
- * Else it's not all there yet, leave buf alone and return 0.
- *
- * If you want to specify the socks reply, write it into <b>req->reply</b>
- *   and set <b>req->replylen</b>, else leave <b>req->replylen</b> alone.
- *
- * If <b>log_sockstype</b> is non-zero, then do a notice-level log of whether
- * the connection is possibly leaking DNS requests locally or not.
- *
- * If <b>safe_socks</b> is true, then reject unsafe socks protocols.
- *
- * If returning 0 or -1, <b>req->address</b> and <b>req->port</b> are
- * undefined.
- */
-int
-fetch_from_buf_socks(buf_t *buf, socks_request_t *req,
-                     int log_sockstype, int safe_socks)
-{
-  int res;
-  ssize_t n_drain;
-  size_t want_length = 128;
-
-  if (buf->datalen < 2) /* version and another byte */
-    return 0;
-
-  do {
-    n_drain = 0;
-    buf_pullup(buf, want_length);
-    tor_assert(buf->head && buf->head->datalen >= 2);
-    want_length = 0;
-
-    res = parse_socks(buf->head->data, buf->head->datalen, req, log_sockstype,
-                      safe_socks, &n_drain, &want_length);
-
-    if (n_drain < 0)
-      buf_clear(buf);
-    else if (n_drain > 0)
-      buf_remove_from_front(buf, n_drain);
-
-  } while (res == 0 && buf->head && want_length < buf->datalen &&
-           buf->datalen >= 2);
-
-  return res;
-}
-
-/** The size of the header of an Extended ORPort message: 2 bytes for
- *  COMMAND, 2 bytes for BODYLEN */
-#define EXT_OR_CMD_HEADER_SIZE 4
-
-/** Read <b>buf</b>, which should contain an Extended ORPort message
- *  from a transport proxy. If well-formed, create and populate
- *  <b>out</b> with the Extended ORport message. Return 0 if the
- *  buffer was incomplete, 1 if it was well-formed and -1 if we
- *  encountered an error while parsing it.  */
-int
-fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out)
-{
-  char hdr[EXT_OR_CMD_HEADER_SIZE];
-  uint16_t len;
-
-  check();
-  if (buf->datalen < EXT_OR_CMD_HEADER_SIZE)
-    return 0;
-  peek_from_buf(hdr, sizeof(hdr), buf);
-  len = ntohs(get_uint16(hdr+2));
-  if (buf->datalen < (unsigned)len + EXT_OR_CMD_HEADER_SIZE)
-    return 0;
-  *out = ext_or_cmd_new(len);
-  (*out)->cmd = ntohs(get_uint16(hdr));
-  (*out)->len = len;
-  buf_remove_from_front(buf, EXT_OR_CMD_HEADER_SIZE);
-  fetch_from_buf((*out)->body, len, buf);
-  return 1;
-}
-
-/** Create a SOCKS5 reply message with <b>reason</b> in its REP field and
- * have Tor send it as error response to <b>req</b>.
- */
-static void
-socks_request_set_socks5_error(socks_request_t *req,
-                  socks5_reply_status_t reason)
-{
-   req->replylen = 10;
-   memset(req->reply,0,10);
-
-   req->reply[0] = 0x05;   // VER field.
-   req->reply[1] = reason; // REP field.
-   req->reply[3] = 0x01;   // ATYP field.
-}
-
-/** Implementation helper to implement fetch_from_*_socks.  Instead of looking
- * at a buffer's contents, we look at the <b>datalen</b> bytes of data in
- * <b>data</b>. Instead of removing data from the buffer, we set
- * <b>drain_out</b> to the amount of data that should be removed (or -1 if the
- * buffer should be cleared).  Instead of pulling more data into the first
- * chunk of the buffer, we set *<b>want_length_out</b> to the number of bytes
- * we'd like to see in the input buffer, if they're available. */
-static int
-parse_socks(const char *data, size_t datalen, socks_request_t *req,
-            int log_sockstype, int safe_socks, ssize_t *drain_out,
-            size_t *want_length_out)
-{
-  unsigned int len;
-  char tmpbuf[TOR_ADDR_BUF_LEN+1];
-  tor_addr_t destaddr;
-  uint32_t destip;
-  uint8_t socksver;
-  char *next, *startaddr;
-  unsigned char usernamelen, passlen;
-  struct in_addr in;
-
-  if (datalen < 2) {
-    /* We always need at least 2 bytes. */
-    *want_length_out = 2;
-    return 0;
-  }
-
-  if (req->socks_version == 5 && !req->got_auth) {
-    /* See if we have received authentication.  Strictly speaking, we should
-       also check whether we actually negotiated username/password
-       authentication.  But some broken clients will send us authentication
-       even if we negotiated SOCKS_NO_AUTH. */
-    if (*data == 1) { /* username/pass version 1 */
-      /* Format is: authversion [1 byte] == 1
-                    usernamelen [1 byte]
-                    username    [usernamelen bytes]
-                    passlen     [1 byte]
-                    password    [passlen bytes] */
-      usernamelen = (unsigned char)*(data + 1);
-      if (datalen < 2u + usernamelen + 1u) {
-        *want_length_out = 2u + usernamelen + 1u;
-        return 0;
-      }
-      passlen = (unsigned char)*(data + 2u + usernamelen);
-      if (datalen < 2u + usernamelen + 1u + passlen) {
-        *want_length_out = 2u + usernamelen + 1u + passlen;
-        return 0;
-      }
-      req->replylen = 2; /* 2 bytes of response */
-      req->reply[0] = 1; /* authversion == 1 */
-      req->reply[1] = 0; /* authentication successful */
-      log_debug(LD_APP,
-               "socks5: Accepted username/password without checking.");
-      if (usernamelen) {
-        req->username = tor_memdup(data+2u, usernamelen);
-        req->usernamelen = usernamelen;
-      }
-      if (passlen) {
-        req->password = tor_memdup(data+3u+usernamelen, passlen);
-        req->passwordlen = passlen;
-      }
-      *drain_out = 2u + usernamelen + 1u + passlen;
-      req->got_auth = 1;
-      *want_length_out = 7; /* Minimal socks5 command. */
-      return 0;
-    } else if (req->auth_type == SOCKS_USER_PASS) {
-      /* unknown version byte */
-      log_warn(LD_APP, "Socks5 username/password version %d not recognized; "
-               "rejecting.", (int)*data);
-      return -1;
-    }
-  }
-
-  socksver = *data;
-
-  switch (socksver) { /* which version of socks? */
-    case 5: /* socks5 */
-
-      if (req->socks_version != 5) { /* we need to negotiate a method */
-        unsigned char nummethods = (unsigned char)*(data+1);
-        int have_user_pass, have_no_auth;
-        int r=0;
-        tor_assert(!req->socks_version);
-        if (datalen < 2u+nummethods) {
-          *want_length_out = 2u+nummethods;
-          return 0;
-        }
-        if (!nummethods)
-          return -1;
-        req->replylen = 2; /* 2 bytes of response */
-        req->reply[0] = 5; /* socks5 reply */
-        have_user_pass = (memchr(data+2, SOCKS_USER_PASS, nummethods) !=NULL);
-        have_no_auth   = (memchr(data+2, SOCKS_NO_AUTH,   nummethods) !=NULL);
-        if (have_user_pass && !(have_no_auth && req->socks_prefer_no_auth)) {
-          req->auth_type = SOCKS_USER_PASS;
-          req->reply[1] = SOCKS_USER_PASS; /* tell client to use "user/pass"
-                                              auth method */
-          req->socks_version = 5; /* remember we've already negotiated auth */
-          log_debug(LD_APP,"socks5: accepted method 2 (username/password)");
-          r=0;
-        } else if (have_no_auth) {
-          req->reply[1] = SOCKS_NO_AUTH; /* tell client to use "none" auth
-                                            method */
-          req->socks_version = 5; /* remember we've already negotiated auth */
-          log_debug(LD_APP,"socks5: accepted method 0 (no authentication)");
-          r=0;
-        } else {
-          log_warn(LD_APP,
-                    "socks5: offered methods don't include 'no auth' or "
-                    "username/password. Rejecting.");
-          req->reply[1] = '\xFF'; /* reject all methods */
-          r=-1;
-        }
-        /* Remove packet from buf. Some SOCKS clients will have sent extra
-         * junk at this point; let's hope it's an authentication message. */
-        *drain_out = 2u + nummethods;
-
-        return r;
-      }
-      if (req->auth_type != SOCKS_NO_AUTH && !req->got_auth) {
-        log_warn(LD_APP,
-                 "socks5: negotiated authentication, but none provided");
-        return -1;
-      }
-      /* we know the method; read in the request */
-      log_debug(LD_APP,"socks5: checking request");
-      if (datalen < 7) {/* basic info plus >=1 for addr plus 2 for port */
-        *want_length_out = 7;
-        return 0; /* not yet */
-      }
-      req->command = (unsigned char) *(data+1);
-      if (req->command != SOCKS_COMMAND_CONNECT &&
-          req->command != SOCKS_COMMAND_RESOLVE &&
-          req->command != SOCKS_COMMAND_RESOLVE_PTR) {
-        /* not a connect or resolve or a resolve_ptr? we don't support it. */
-        socks_request_set_socks5_error(req,SOCKS5_COMMAND_NOT_SUPPORTED);
-
-        log_warn(LD_APP,"socks5: command %d not recognized. Rejecting.",
-                 req->command);
-        return -1;
-      }
-      switch (*(data+3)) { /* address type */
-        case 1: /* IPv4 address */
-        case 4: /* IPv6 address */ {
-          const int is_v6 = *(data+3) == 4;
-          const unsigned addrlen = is_v6 ? 16 : 4;
-          log_debug(LD_APP,"socks5: ipv4 address type");
-          if (datalen < 6+addrlen) {/* ip/port there? */
-            *want_length_out = 6+addrlen;
-            return 0; /* not yet */
-          }
-
-          if (is_v6)
-            tor_addr_from_ipv6_bytes(&destaddr, data+4);
-          else
-            tor_addr_from_ipv4n(&destaddr, get_uint32(data+4));
-
-          tor_addr_to_str(tmpbuf, &destaddr, sizeof(tmpbuf), 1);
-
-          if (strlen(tmpbuf)+1 > MAX_SOCKS_ADDR_LEN) {
-            socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR);
-            log_warn(LD_APP,
-                     "socks5 IP takes %d bytes, which doesn't fit in %d. "
-                     "Rejecting.",
-                     (int)strlen(tmpbuf)+1,(int)MAX_SOCKS_ADDR_LEN);
-            return -1;
-          }
-          strlcpy(req->address,tmpbuf,sizeof(req->address));
-          req->port = ntohs(get_uint16(data+4+addrlen));
-          *drain_out = 6+addrlen;
-          if (req->command != SOCKS_COMMAND_RESOLVE_PTR &&
-              !addressmap_have_mapping(req->address,0)) {
-            log_unsafe_socks_warning(5, req->address, req->port, safe_socks);
-            if (safe_socks) {
-              socks_request_set_socks5_error(req, SOCKS5_NOT_ALLOWED);
-              return -1;
-            }
-          }
-          return 1;
-        }
-        case 3: /* fqdn */
-          log_debug(LD_APP,"socks5: fqdn address type");
-          if (req->command == SOCKS_COMMAND_RESOLVE_PTR) {
-            socks_request_set_socks5_error(req,
-                                           SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED);
-            log_warn(LD_APP, "socks5 received RESOLVE_PTR command with "
-                     "hostname type. Rejecting.");
-            return -1;
-          }
-          len = (unsigned char)*(data+4);
-          if (datalen < 7+len) { /* addr/port there? */
-            *want_length_out = 7+len;
-            return 0; /* not yet */
-          }
-          if (len+1 > MAX_SOCKS_ADDR_LEN) {
-            socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR);
-            log_warn(LD_APP,
-                     "socks5 hostname is %d bytes, which doesn't fit in "
-                     "%d. Rejecting.", len+1,MAX_SOCKS_ADDR_LEN);
-            return -1;
-          }
-          memcpy(req->address,data+5,len);
-          req->address[len] = 0;
-          req->port = ntohs(get_uint16(data+5+len));
-          *drain_out = 5+len+2;
-
-          if (string_is_valid_ipv4_address(req->address) ||
-              string_is_valid_ipv6_address(req->address)) {
-            log_unsafe_socks_warning(5,req->address,req->port,safe_socks);
-
-            if (safe_socks) {
-              socks_request_set_socks5_error(req, SOCKS5_NOT_ALLOWED);
-              return -1;
-            }
-          } else if (!string_is_valid_hostname(req->address)) {
-            socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR);
-
-            log_warn(LD_PROTOCOL,
-                     "Your application (using socks5 to port %d) gave Tor "
-                     "a malformed hostname: %s. Rejecting the connection.",
-                     req->port, escaped_safe_str_client(req->address));
-            return -1;
-          }
-          if (log_sockstype)
-            log_notice(LD_APP,
-                  "Your application (using socks5 to port %d) instructed "
-                  "Tor to take care of the DNS resolution itself if "
-                  "necessary. This is good.", req->port);
-          return 1;
-        default: /* unsupported */
-          socks_request_set_socks5_error(req,
-                                         SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED);
-          log_warn(LD_APP,"socks5: unsupported address type %d. Rejecting.",
-                   (int) *(data+3));
-          return -1;
-      }
-      tor_assert(0);
-      break;
-    case 4: { /* socks4 */
-      enum {socks4, socks4a} socks4_prot = socks4a;
-      const char *authstart, *authend;
-      /* http://ss5.sourceforge.net/socks4.protocol.txt */
-      /* http://ss5.sourceforge.net/socks4A.protocol.txt */
-
-      req->socks_version = 4;
-      if (datalen < SOCKS4_NETWORK_LEN) {/* basic info available? */
-        *want_length_out = SOCKS4_NETWORK_LEN;
-        return 0; /* not yet */
-      }
-      // buf_pullup(buf, 1280);
-      req->command = (unsigned char) *(data+1);
-      if (req->command != SOCKS_COMMAND_CONNECT &&
-          req->command != SOCKS_COMMAND_RESOLVE) {
-        /* not a connect or resolve? we don't support it. (No resolve_ptr with
-         * socks4.) */
-        log_warn(LD_APP,"socks4: command %d not recognized. Rejecting.",
-                 req->command);
-        return -1;
-      }
-
-      req->port = ntohs(get_uint16(data+2));
-      destip = ntohl(get_uint32(data+4));
-      if ((!req->port && req->command!=SOCKS_COMMAND_RESOLVE) || !destip) {
-        log_warn(LD_APP,"socks4: Port or DestIP is zero. Rejecting.");
-        return -1;
-      }
-      if (destip >> 8) {
-        log_debug(LD_APP,"socks4: destip not in form 0.0.0.x.");
-        in.s_addr = htonl(destip);
-        tor_inet_ntoa(&in,tmpbuf,sizeof(tmpbuf));
-        if (strlen(tmpbuf)+1 > MAX_SOCKS_ADDR_LEN) {
-          log_debug(LD_APP,"socks4 addr (%d bytes) too long. Rejecting.",
-                    (int)strlen(tmpbuf));
-          return -1;
-        }
-        log_debug(LD_APP,
-                  "socks4: successfully read destip (%s)",
-                  safe_str_client(tmpbuf));
-        socks4_prot = socks4;
-      }
-
-      authstart = data + SOCKS4_NETWORK_LEN;
-      next = memchr(authstart, 0,
-                    datalen-SOCKS4_NETWORK_LEN);
-      if (!next) {
-        if (datalen >= 1024) {
-          log_debug(LD_APP, "Socks4 user name too long; rejecting.");
-          return -1;
-        }
-        log_debug(LD_APP,"socks4: Username not here yet.");
-        *want_length_out = datalen+1024; /* More than we need, but safe */
-        return 0;
-      }
-      authend = next;
-      tor_assert(next < data+datalen);
-
-      startaddr = NULL;
-      if (socks4_prot != socks4a &&
-          !addressmap_have_mapping(tmpbuf,0)) {
-        log_unsafe_socks_warning(4, tmpbuf, req->port, safe_socks);
-
-        if (safe_socks)
-          return -1;
-      }
-      if (socks4_prot == socks4a) {
-        if (next+1 == data+datalen) {
-          log_debug(LD_APP,"socks4: No part of destaddr here yet.");
-          *want_length_out = datalen + 1024; /* More than we need, but safe */
-          return 0;
-        }
-        startaddr = next+1;
-        next = memchr(startaddr, 0, data + datalen - startaddr);
-        if (!next) {
-          if (datalen >= 1024) {
-            log_debug(LD_APP,"socks4: Destaddr too long.");
-            return -1;
-          }
-          log_debug(LD_APP,"socks4: Destaddr not all here yet.");
-          *want_length_out = datalen + 1024; /* More than we need, but safe */
-          return 0;
-        }
-        if (MAX_SOCKS_ADDR_LEN <= next-startaddr) {
-          log_warn(LD_APP,"socks4: Destaddr too long. Rejecting.");
-          return -1;
-        }
-        // tor_assert(next < buf->cur+buf->datalen);
-
-        if (log_sockstype)
-          log_notice(LD_APP,
-                     "Your application (using socks4a to port %d) instructed "
-                     "Tor to take care of the DNS resolution itself if "
-                     "necessary. This is good.", req->port);
-      }
-      log_debug(LD_APP,"socks4: Everything is here. Success.");
-      strlcpy(req->address, startaddr ? startaddr : tmpbuf,
-              sizeof(req->address));
-      if (!tor_strisprint(req->address) || strchr(req->address,'\"')) {
-        log_warn(LD_PROTOCOL,
-                 "Your application (using socks4 to port %d) gave Tor "
-                 "a malformed hostname: %s. Rejecting the connection.",
-                 req->port, escaped_safe_str_client(req->address));
-        return -1;
-      }
-      if (authend != authstart) {
-        req->got_auth = 1;
-        req->usernamelen = authend - authstart;
-        req->username = tor_memdup(authstart, authend - authstart);
-      }
-      /* next points to the final \0 on inbuf */
-      *drain_out = next - data + 1;
-      return 1;
-    }
-    case 'G': /* get */
-    case 'H': /* head */
-    case 'P': /* put/post */
-    case 'C': /* connect */
-      strlcpy((char*)req->reply,
-"HTTP/1.0 501 Tor is not an HTTP Proxy\r\n"
-"Content-Type: text/html; charset=iso-8859-1\r\n\r\n"
-"<html>\n"
-"<head>\n"
-"<title>Tor is not an HTTP Proxy</title>\n"
-"</head>\n"
-"<body>\n"
-"<h1>Tor is not an HTTP Proxy</h1>\n"
-"<p>\n"
-"It appears you have configured your web browser to use Tor as an HTTP proxy."
-"\n"
-"This is not correct: Tor is a SOCKS proxy, not an HTTP proxy.\n"
-"Please configure your client accordingly.\n"
-"</p>\n"
-"<p>\n"
-"See <a href=\"https://www.torproject.org/documentation.html\">"
-           "https://www.torproject.org/documentation.html</a> for more "
-           "information.\n"
-"<!-- Plus this comment, to make the body response more than 512 bytes, so "
-"     IE will be willing to display it. Comment comment comment comment "
-"     comment comment comment comment comment comment comment comment.-->\n"
-"</p>\n"
-"</body>\n"
-"</html>\n"
-             , MAX_SOCKS_REPLY_LEN);
-      req->replylen = strlen((char*)req->reply)+1;
-      /* fall through */
-    default: /* version is not socks4 or socks5 */
-      log_warn(LD_APP,
-               "Socks version %d not recognized. (Tor is not an http proxy.)",
-               *(data));
-      {
-        /* Tell the controller the first 8 bytes. */
-        char *tmp = tor_strndup(data, datalen < 8 ? datalen : 8);
-        control_event_client_status(LOG_WARN,
-                                    "SOCKS_UNKNOWN_PROTOCOL DATA=\"%s\"",
-                                    escaped(tmp));
-        tor_free(tmp);
-      }
-      return -1;
-  }
-}
-
-/** Inspect a reply from SOCKS server stored in <b>buf</b> according
- * to <b>state</b>, removing the protocol data upon success. Return 0 on
- * incomplete response, 1 on success and -1 on error, in which case
- * <b>reason</b> is set to a descriptive message (free() when finished
- * with it).
- *
- * As a special case, 2 is returned when user/pass is required
- * during SOCKS5 handshake and user/pass is configured.
- */
-int
-fetch_from_buf_socks_client(buf_t *buf, int state, char **reason)
-{
-  ssize_t drain = 0;
-  int r;
-  if (buf->datalen < 2)
-    return 0;
-
-  buf_pullup(buf, MAX_SOCKS_MESSAGE_LEN);
-  tor_assert(buf->head && buf->head->datalen >= 2);
-
-  r = parse_socks_client((uint8_t*)buf->head->data, buf->head->datalen,
-                         state, reason, &drain);
-  if (drain > 0)
-    buf_remove_from_front(buf, drain);
-  else if (drain < 0)
-    buf_clear(buf);
-
-  return r;
-}
-
-/** Implementation logic for fetch_from_*_socks_client. */
-static int
-parse_socks_client(const uint8_t *data, size_t datalen,
-                   int state, char **reason,
-                   ssize_t *drain_out)
-{
-  unsigned int addrlen;
-  *drain_out = 0;
-  if (datalen < 2)
-    return 0;
-
-  switch (state) {
-    case PROXY_SOCKS4_WANT_CONNECT_OK:
-      /* Wait for the complete response */
-      if (datalen < 8)
-        return 0;
-
-      if (data[1] != 0x5a) {
-        *reason = tor_strdup(socks4_response_code_to_string(data[1]));
-        return -1;
-      }
-
-      /* Success */
-      *drain_out = 8;
-      return 1;
-
-    case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE:
-      /* we don't have any credentials */
-      if (data[1] != 0x00) {
-        *reason = tor_strdup("server doesn't support any of our "
-                             "available authentication methods");
-        return -1;
-      }
-
-      log_info(LD_NET, "SOCKS 5 client: continuing without authentication");
-      *drain_out = -1;
-      return 1;
-
-    case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929:
-      /* we have a username and password. return 1 if we can proceed without
-       * providing authentication, or 2 otherwise. */
-      switch (data[1]) {
-        case 0x00:
-          log_info(LD_NET, "SOCKS 5 client: we have auth details but server "
-                            "doesn't require authentication.");
-          *drain_out = -1;
-          return 1;
-        case 0x02:
-          log_info(LD_NET, "SOCKS 5 client: need authentication.");
-          *drain_out = -1;
-          return 2;
-        /* fall through */
-      }
-
-      *reason = tor_strdup("server doesn't support any of our available "
-                           "authentication methods");
-      return -1;
-
-    case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK:
-      /* handle server reply to rfc1929 authentication */
-      if (data[1] != 0x00) {
-        *reason = tor_strdup("authentication failed");
-        return -1;
-      }
-
-      log_info(LD_NET, "SOCKS 5 client: authentication successful.");
-      *drain_out = -1;
-      return 1;
-
-    case PROXY_SOCKS5_WANT_CONNECT_OK:
-      /* response is variable length. BND.ADDR, etc, isn't needed
-       * (don't bother with buf_pullup()), but make sure to eat all
-       * the data used */
-
-      /* wait for address type field to arrive */
-      if (datalen < 4)
-        return 0;
-
-      switch (data[3]) {
-        case 0x01: /* ip4 */
-          addrlen = 4;
-          break;
-        case 0x04: /* ip6 */
-          addrlen = 16;
-          break;
-        case 0x03: /* fqdn (can this happen here?) */
-          if (datalen < 5)
-            return 0;
-          addrlen = 1 + data[4];
-          break;
-        default:
-          *reason = tor_strdup("invalid response to connect request");
-          return -1;
-      }
-
-      /* wait for address and port */
-      if (datalen < 6 + addrlen)
-        return 0;
-
-      if (data[1] != 0x00) {
-        *reason = tor_strdup(socks5_response_code_to_string(data[1]));
-        return -1;
-      }
-
-      *drain_out = 6 + addrlen;
-      return 1;
-  }
-
-  /* shouldn't get here... */
-  tor_assert(0);
-
-  return -1;
-}
-
-/** Return 1 iff buf looks more like it has an (obsolete) v0 controller
- * command on it than any valid v1 controller command. */
-int
-peek_buf_has_control0_command(buf_t *buf)
-{
-  if (buf->datalen >= 4) {
-    char header[4];
-    uint16_t cmd;
-    peek_from_buf(header, sizeof(header), buf);
-    cmd = ntohs(get_uint16(header+2));
-    if (cmd <= 0x14)
-      return 1; /* This is definitely not a v1 control command. */
-  }
-  return 0;
-}
-
-/** Return the index within <b>buf</b> at which <b>ch</b> first appears,
- * or -1 if <b>ch</b> does not appear on buf. */
-static off_t
-buf_find_offset_of_char(buf_t *buf, char ch)
-{
-  chunk_t *chunk;
-  off_t offset = 0;
-  tor_assert(buf->datalen < INT_MAX);
-  for (chunk = buf->head; chunk; chunk = chunk->next) {
-    char *cp = memchr(chunk->data, ch, chunk->datalen);
-    if (cp)
-      return offset + (cp - chunk->data);
-    else
-      offset += chunk->datalen;
-  }
-  return -1;
-}
-
-/** Try to read a single LF-terminated line from <b>buf</b>, and write it
- * (including the LF), NUL-terminated, into the *<b>data_len</b> byte buffer
- * at <b>data_out</b>.  Set *<b>data_len</b> to the number of bytes in the
- * line, not counting the terminating NUL.  Return 1 if we read a whole line,
- * return 0 if we don't have a whole line yet, and return -1 if the line
- * length exceeds *<b>data_len</b>.
- */
-int
-fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len)
-{
-  size_t sz;
-  off_t offset;
-
-  if (!buf->head)
-    return 0;
-
-  offset = buf_find_offset_of_char(buf, '\n');
-  if (offset < 0)
-    return 0;
-  sz = (size_t) offset;
-  if (sz+2 > *data_len) {
-    *data_len = sz + 2;
-    return -1;
-  }
-  fetch_from_buf(data_out, sz+1, buf);
-  data_out[sz+1] = '\0';
-  *data_len = sz+1;
-  return 1;
-}
-
-/** Compress on uncompress the <b>data_len</b> bytes in <b>data</b> using the
- * zlib state <b>state</b>, appending the result to <b>buf</b>.  If
- * <b>done</b> is true, flush the data in the state and finish the
- * compression/uncompression.  Return -1 on failure, 0 on success. */
-int
-write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state,
-                  const char *data, size_t data_len,
-                  int done)
-{
-  char *next;
-  size_t old_avail, avail;
-  int over = 0;
-
-  do {
-    int need_new_chunk = 0;
-    if (!buf->tail || ! CHUNK_REMAINING_CAPACITY(buf->tail)) {
-      size_t cap = data_len / 4;
-      buf_add_chunk_with_capacity(buf, cap, 1);
-    }
-    next = CHUNK_WRITE_PTR(buf->tail);
-    avail = old_avail = CHUNK_REMAINING_CAPACITY(buf->tail);
-    switch (tor_zlib_process(state, &next, &avail, &data, &data_len, done)) {
-      case TOR_ZLIB_DONE:
-        over = 1;
-        break;
-      case TOR_ZLIB_ERR:
-        return -1;
-      case TOR_ZLIB_OK:
-        if (data_len == 0)
-          over = 1;
-        break;
-      case TOR_ZLIB_BUF_FULL:
-        if (avail) {
-          /* Zlib says we need more room (ZLIB_BUF_FULL).  Start a new chunk
-           * automatically, whether were going to or not. */
-          need_new_chunk = 1;
-        }
-        break;
-    }
-    buf->datalen += old_avail - avail;
-    buf->tail->datalen += old_avail - avail;
-    if (need_new_chunk) {
-      buf_add_chunk_with_capacity(buf, data_len/4, 1);
-    }
-
-  } while (!over);
-  check();
-  return 0;
-}
-
-/** Set *<b>output</b> to contain a copy of the data in *<b>input</b> */
-int
-buf_set_to_copy(buf_t **output,
-                const buf_t *input)
-{
-  if (*output)
-    buf_free(*output);
-  *output = buf_copy(input);
-  return 0;
-}
-
-/** Log an error and exit if <b>buf</b> is corrupted.
- */
-void
-assert_buf_ok(buf_t *buf)
-{
-  tor_assert(buf);
-  tor_assert(buf->magic == BUFFER_MAGIC);
-
-  if (! buf->head) {
-    tor_assert(!buf->tail);
-    tor_assert(buf->datalen == 0);
-  } else {
-    chunk_t *ch;
-    size_t total = 0;
-    tor_assert(buf->tail);
-    for (ch = buf->head; ch; ch = ch->next) {
-      total += ch->datalen;
-      tor_assert(ch->datalen <= ch->memlen);
-      tor_assert(ch->datalen < INT_MAX);
-      tor_assert(ch->data >= &ch->mem[0]);
-      tor_assert(ch->data <= &ch->mem[0]+ch->memlen);
-      if (ch->data == &ch->mem[0]+ch->memlen) {
-        static int warned = 0;
-        if (! warned) {
-          log_warn(LD_BUG, "Invariant violation in buf.c related to #15083");
-          warned = 1;
-        }
-      }
-      tor_assert(ch->data+ch->datalen <= &ch->mem[0] + ch->memlen);
-      if (!ch->next)
-        tor_assert(ch == buf->tail);
-    }
-    tor_assert(buf->datalen == total);
-  }
-}
diff --git a/src/or/buffers.h b/src/or/buffers.h
deleted file mode 100644
index 52b21d5885..0000000000
--- a/src/or/buffers.h
+++ /dev/null
@@ -1,101 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file buffers.h
- * \brief Header file for buffers.c.
- **/
-
-#ifndef TOR_BUFFERS_H
-#define TOR_BUFFERS_H
-
-#include "testsupport.h"
-
-buf_t *buf_new(void);
-buf_t *buf_new_with_capacity(size_t size);
-size_t buf_get_default_chunk_size(const buf_t *buf);
-void buf_free(buf_t *buf);
-void buf_clear(buf_t *buf);
-buf_t *buf_copy(const buf_t *buf);
-
-MOCK_DECL(size_t, buf_datalen, (const buf_t *buf));
-size_t buf_allocation(const buf_t *buf);
-size_t buf_slack(const buf_t *buf);
-
-uint32_t buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now);
-size_t buf_get_total_allocation(void);
-
-int read_to_buf(tor_socket_t s, size_t at_most, buf_t *buf, int *reached_eof,
-                int *socket_error);
-int read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf);
-
-int flush_buf(tor_socket_t s, buf_t *buf, size_t sz, size_t *buf_flushlen);
-int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen);
-
-int write_to_buf(const char *string, size_t string_len, buf_t *buf);
-int write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state,
-                      const char *data, size_t data_len, int done);
-int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen);
-int fetch_from_buf(char *string, size_t string_len, buf_t *buf);
-int fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto);
-int fetch_from_buf_http(buf_t *buf,
-                        char **headers_out, size_t max_headerlen,
-                        char **body_out, size_t *body_used, size_t max_bodylen,
-                        int force_complete);
-socks_request_t *socks_request_new(void);
-void socks_request_free(socks_request_t *req);
-int fetch_from_buf_socks(buf_t *buf, socks_request_t *req,
-                         int log_sockstype, int safe_socks);
-int fetch_from_buf_socks_client(buf_t *buf, int state, char **reason);
-int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len);
-
-int peek_buf_has_control0_command(buf_t *buf);
-
-int fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out);
-
-int buf_set_to_copy(buf_t **output,
-                    const buf_t *input);
-
-void assert_buf_ok(buf_t *buf);
-
-#ifdef BUFFERS_PRIVATE
-STATIC int buf_find_string_offset(const buf_t *buf, const char *s, size_t n);
-STATIC void buf_pullup(buf_t *buf, size_t bytes);
-void buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz);
-STATIC size_t preferred_chunk_size(size_t target);
-
-#define DEBUG_CHUNK_ALLOC
-/** A single chunk on a buffer. */
-typedef struct chunk_t {
-  struct chunk_t *next; /**< The next chunk on the buffer. */
-  size_t datalen; /**< The number of bytes stored in this chunk */
-  size_t memlen; /**< The number of usable bytes of storage in <b>mem</b>. */
-#ifdef DEBUG_CHUNK_ALLOC
-  size_t DBG_alloc;
-#endif
-  char *data; /**< A pointer to the first byte of data stored in <b>mem</b>. */
-  uint32_t inserted_time; /**< Timestamp in truncated ms since epoch
-                           * when this chunk was inserted. */
-  char mem[FLEXIBLE_ARRAY_MEMBER]; /**< The actual memory used for storage in
-                * this chunk. */
-} chunk_t;
-
-/** Magic value for buf_t.magic, to catch pointer errors. */
-#define BUFFER_MAGIC 0xB0FFF312u
-/** A resizeable buffer, optimized for reading and writing. */
-struct buf_t {
-  uint32_t magic; /**< Magic cookie for debugging: Must be set to
-                   *   BUFFER_MAGIC. */
-  size_t datalen; /**< How many bytes is this buffer holding right now? */
-  size_t default_chunk_size; /**< Don't allocate any chunks smaller than
-                              * this for this buffer. */
-  chunk_t *head; /**< First chunk in the list, or NULL for none. */
-  chunk_t *tail; /**< Last chunk in the list, or NULL for none. */
-};
-#endif
-
-#endif
-
diff --git a/src/or/circuitlist.h b/src/or/circuitlist.h
deleted file mode 100644
index 2ede6f76cd..0000000000
--- a/src/or/circuitlist.h
+++ /dev/null
@@ -1,91 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file circuitlist.h
- * \brief Header file for circuitlist.c.
- **/
-
-#ifndef TOR_CIRCUITLIST_H
-#define TOR_CIRCUITLIST_H
-
-#include "testsupport.h"
-
-MOCK_DECL(smartlist_t *, circuit_get_global_list, (void));
-const char *circuit_state_to_string(int state);
-const char *circuit_purpose_to_controller_string(uint8_t purpose);
-const char *circuit_purpose_to_controller_hs_state_string(uint8_t purpose);
-const char *circuit_purpose_to_string(uint8_t purpose);
-void circuit_dump_by_conn(connection_t *conn, int severity);
-void circuit_set_p_circid_chan(or_circuit_t *circ, circid_t id,
-                               channel_t *chan);
-void circuit_set_n_circid_chan(circuit_t *circ, circid_t id,
-                               channel_t *chan);
-void channel_mark_circid_unusable(channel_t *chan, circid_t id);
-void channel_mark_circid_usable(channel_t *chan, circid_t id);
-time_t circuit_id_when_marked_unusable_on_channel(circid_t circ_id,
-                                                  channel_t *chan);
-void circuit_set_state(circuit_t *circ, uint8_t state);
-void circuit_close_all_marked(void);
-int32_t circuit_initial_package_window(void);
-origin_circuit_t *origin_circuit_new(void);
-or_circuit_t *or_circuit_new(circid_t p_circ_id, channel_t *p_chan);
-circuit_t *circuit_get_by_circid_channel(circid_t circ_id,
-                                         channel_t *chan);
-circuit_t *
-circuit_get_by_circid_channel_even_if_marked(circid_t circ_id,
-                                             channel_t *chan);
-int circuit_id_in_use_on_channel(circid_t circ_id, channel_t *chan);
-circuit_t *circuit_get_by_edge_conn(edge_connection_t *conn);
-void circuit_unlink_all_from_channel(channel_t *chan, int reason);
-origin_circuit_t *circuit_get_by_global_id(uint32_t id);
-origin_circuit_t *circuit_get_ready_rend_circ_by_rend_data(
-  const rend_data_t *rend_data);
-origin_circuit_t *circuit_get_next_by_pk_and_purpose(origin_circuit_t *start,
-                                         const char *digest, uint8_t purpose);
-or_circuit_t *circuit_get_rendezvous(const uint8_t *cookie);
-or_circuit_t *circuit_get_intro_point(const uint8_t *digest);
-void circuit_set_rendezvous_cookie(or_circuit_t *circ, const uint8_t *cookie);
-void circuit_set_intro_point_digest(or_circuit_t *circ, const uint8_t *digest);
-origin_circuit_t *circuit_find_to_cannibalize(uint8_t purpose,
-                                              extend_info_t *info, int flags);
-void circuit_mark_all_unused_circs(void);
-void circuit_mark_all_dirty_circs_as_unusable(void);
-MOCK_DECL(void, circuit_mark_for_close_, (circuit_t *circ, int reason,
-                                          int line, const char *file));
-int circuit_get_cpath_len(origin_circuit_t *circ);
-void circuit_clear_cpath(origin_circuit_t *circ);
-crypt_path_t *circuit_get_cpath_hop(origin_circuit_t *circ, int hopnum);
-void circuit_get_all_pending_on_channel(smartlist_t *out,
-                                        channel_t *chan);
-int circuit_count_pending_on_channel(channel_t *chan);
-void circuit_synchronize_written_or_bandwidth(const circuit_t *c,
-                                              circuit_channel_direction_t dir);
-
-#define circuit_mark_for_close(c, reason)                               \
-  circuit_mark_for_close_((c), (reason), __LINE__, SHORT_FILE__)
-
-void assert_cpath_layer_ok(const crypt_path_t *cp);
-void assert_circuit_ok(const circuit_t *c);
-void circuit_free_all(void);
-void circuits_handle_oom(size_t current_allocation);
-
-void circuit_clear_testing_cell_stats(circuit_t *circ);
-
-void channel_note_destroy_pending(channel_t *chan, circid_t id);
-MOCK_DECL(void, channel_note_destroy_not_pending,
-          (channel_t *chan, circid_t id));
-
-#ifdef CIRCUITLIST_PRIVATE
-STATIC void circuit_free(circuit_t *circ);
-STATIC size_t n_cells_in_circ_queues(const circuit_t *c);
-STATIC uint32_t circuit_max_queued_data_age(const circuit_t *c, uint32_t now);
-STATIC uint32_t circuit_max_queued_cell_age(const circuit_t *c, uint32_t now);
-STATIC uint32_t circuit_max_queued_item_age(const circuit_t *c, uint32_t now);
-#endif
-
-#endif
-
diff --git a/src/or/circuitmux_ewma.h b/src/or/circuitmux_ewma.h
deleted file mode 100644
index a7b8961ac6..0000000000
--- a/src/or/circuitmux_ewma.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file circuitmux_ewma.h
- * \brief Header file for circuitmux_ewma.c
- **/
-
-#ifndef TOR_CIRCUITMUX_EWMA_H
-#define TOR_CIRCUITMUX_EWMA_H
-
-#include "or.h"
-#include "circuitmux.h"
-
-extern circuitmux_policy_t ewma_policy;
-
-/* Externally visible EWMA functions */
-int cell_ewma_enabled(void);
-unsigned int cell_ewma_get_tick(void);
-void cell_ewma_set_scale_factor(const or_options_t *options,
-                                const networkstatus_t *consensus);
-
-#endif /* TOR_CIRCUITMUX_EWMA_H */
-
diff --git a/src/or/circuitstats.h b/src/or/circuitstats.h
deleted file mode 100644
index 72b160983f..0000000000
--- a/src/or/circuitstats.h
+++ /dev/null
@@ -1,98 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file circuitstats.h
- * \brief Header file for circuitstats.c
- **/
-
-#ifndef TOR_CIRCUITSTATS_H
-#define TOR_CIRCUITSTATS_H
-
-const circuit_build_times_t *get_circuit_build_times(void);
-circuit_build_times_t *get_circuit_build_times_mutable(void);
-double get_circuit_build_close_time_ms(void);
-double get_circuit_build_timeout_ms(void);
-
-int circuit_build_times_disabled(void);
-int circuit_build_times_enough_to_compute(const circuit_build_times_t *cbt);
-void circuit_build_times_update_state(const circuit_build_times_t *cbt,
-                                      or_state_t *state);
-int circuit_build_times_parse_state(circuit_build_times_t *cbt,
-                                    or_state_t *state);
-void circuit_build_times_count_timeout(circuit_build_times_t *cbt,
-                                       int did_onehop);
-int circuit_build_times_count_close(circuit_build_times_t *cbt,
-                                    int did_onehop, time_t start_time);
-void circuit_build_times_set_timeout(circuit_build_times_t *cbt);
-int circuit_build_times_add_time(circuit_build_times_t *cbt,
-                                 build_time_t time);
-int circuit_build_times_needs_circuits(const circuit_build_times_t *cbt);
-
-int circuit_build_times_needs_circuits_now(const circuit_build_times_t *cbt);
-void circuit_build_times_init(circuit_build_times_t *cbt);
-void circuit_build_times_free_timeouts(circuit_build_times_t *cbt);
-void circuit_build_times_new_consensus_params(circuit_build_times_t *cbt,
-                                              networkstatus_t *ns);
-double circuit_build_times_timeout_rate(const circuit_build_times_t *cbt);
-double circuit_build_times_close_rate(const circuit_build_times_t *cbt);
-
-void circuit_build_times_update_last_circ(circuit_build_times_t *cbt);
-
-#ifdef CIRCUITSTATS_PRIVATE
-STATIC double circuit_build_times_calculate_timeout(circuit_build_times_t *cbt,
-                                             double quantile);
-STATIC int circuit_build_times_update_alpha(circuit_build_times_t *cbt);
-STATIC void circuit_build_times_reset(circuit_build_times_t *cbt);
-
-/* Network liveness functions */
-STATIC int circuit_build_times_network_check_changed(
-                                             circuit_build_times_t *cbt);
-#endif
-
-#ifdef TOR_UNIT_TESTS
-build_time_t circuit_build_times_generate_sample(circuit_build_times_t *cbt,
-                                                 double q_lo, double q_hi);
-double circuit_build_times_cdf(circuit_build_times_t *cbt, double x);
-void circuit_build_times_initial_alpha(circuit_build_times_t *cbt,
-                                       double quantile, double time_ms);
-void circuitbuild_running_unit_tests(void);
-#endif
-
-/* Network liveness functions */
-void circuit_build_times_network_is_live(circuit_build_times_t *cbt);
-int circuit_build_times_network_check_live(const circuit_build_times_t *cbt);
-void circuit_build_times_network_circ_success(circuit_build_times_t *cbt);
-
-#ifdef CIRCUITSTATS_PRIVATE
-/** Structure for circuit build times history */
-struct circuit_build_times_s {
-  /** The circular array of recorded build times in milliseconds */
-  build_time_t circuit_build_times[CBT_NCIRCUITS_TO_OBSERVE];
-  /** Current index in the circuit_build_times circular array */
-  int build_times_idx;
-  /** Total number of build times accumulated. Max CBT_NCIRCUITS_TO_OBSERVE */
-  int total_build_times;
-  /** Information about the state of our local network connection */
-  network_liveness_t liveness;
-  /** Last time we built a circuit. Used to decide to build new test circs */
-  time_t last_circ_at;
-  /** "Minimum" value of our pareto distribution (actually mode) */
-  build_time_t Xm;
-  /** alpha exponent for pareto dist. */
-  double alpha;
-  /** Have we computed a timeout? */
-  int have_computed_timeout;
-  /** The exact value for that timeout in milliseconds. Stored as a double
-   * to maintain precision from calculations to and from quantile value. */
-  double timeout_ms;
-  /** How long we wait before actually closing the circuit. */
-  double close_ms;
-};
-#endif
-
-#endif
-
diff --git a/src/or/directory.c b/src/or/directory.c
deleted file mode 100644
index f285e4c6ed..0000000000
--- a/src/or/directory.c
+++ /dev/null
@@ -1,4316 +0,0 @@
-/* Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#include "or.h"
-#include "backtrace.h"
-#include "buffers.h"
-#include "circuitbuild.h"
-#include "config.h"
-#include "connection.h"
-#include "connection_edge.h"
-#include "control.h"
-#include "directory.h"
-#include "dirserv.h"
-#include "dirvote.h"
-#include "entrynodes.h"
-#include "geoip.h"
-#include "main.h"
-#include "microdesc.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "policies.h"
-#include "relay.h"
-#include "rendclient.h"
-#include "rendcommon.h"
-#include "rendservice.h"
-#include "rephist.h"
-#include "router.h"
-#include "routerlist.h"
-#include "routerparse.h"
-#include "routerset.h"
-#include "shared_random.h"
-
-#if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO)
-#ifndef OPENBSD
-#include <malloc.h>
-#endif
-#endif
-
-/**
- * \file directory.c
- * \brief Code to send and fetch directories and router
- * descriptors via HTTP.  Directories use dirserv.c to generate the
- * results; clients use routers.c to parse them.
- **/
-
-/* In-points to directory.c:
- *
- * - directory_post_to_dirservers(), called from
- *   router_upload_dir_desc_to_dirservers() in router.c
- *   upload_service_descriptor() in rendservice.c
- * - directory_get_from_dirserver(), called from
- *   rend_client_refetch_renddesc() in rendclient.c
- *   run_scheduled_events() in main.c
- *   do_hup() in main.c
- * - connection_dir_process_inbuf(), called from
- *   connection_process_inbuf() in connection.c
- * - connection_dir_finished_flushing(), called from
- *   connection_finished_flushing() in connection.c
- * - connection_dir_finished_connecting(), called from
- *   connection_finished_connecting() in connection.c
- */
-static void directory_send_command(dir_connection_t *conn,
-                             int purpose, int direct, const char *resource,
-                             const char *payload, size_t payload_len,
-                             time_t if_modified_since);
-static int directory_handle_command(dir_connection_t *conn);
-static int body_is_plausible(const char *body, size_t body_len, int purpose);
-static char *http_get_header(const char *headers, const char *which);
-static void http_set_address_origin(const char *headers, connection_t *conn);
-static void connection_dir_download_routerdesc_failed(dir_connection_t *conn);
-static void connection_dir_bridge_routerdesc_failed(dir_connection_t *conn);
-static void connection_dir_download_cert_failed(
-                               dir_connection_t *conn, int status_code);
-static void connection_dir_retry_bridges(smartlist_t *descs);
-static void dir_routerdesc_download_failed(smartlist_t *failed,
-                                           int status_code,
-                                           int router_purpose,
-                                           int was_extrainfo,
-                                           int was_descriptor_digests);
-static void dir_microdesc_download_failed(smartlist_t *failed,
-                                          int status_code);
-static int client_likes_consensus(networkstatus_t *v, const char *want_url);
-
-static void directory_initiate_command_rend(
-                                          const tor_addr_port_t *or_addr_port,
-                                          const tor_addr_port_t *dir_addr_port,
-                                          const char *digest,
-                                          uint8_t dir_purpose,
-                                          uint8_t router_purpose,
-                                          dir_indirection_t indirection,
-                                          const char *resource,
-                                          const char *payload,
-                                          size_t payload_len,
-                                          time_t if_modified_since,
-                                          const rend_data_t *rend_query);
-
-static void connection_dir_close_consensus_fetches(
-                   dir_connection_t *except_this_one, const char *resource);
-
-/********* START VARIABLES **********/
-
-/** How far in the future do we allow a directory server to tell us it is
- * before deciding that one of us has the wrong time? */
-#define ALLOW_DIRECTORY_TIME_SKEW (30*60)
-
-#define X_ADDRESS_HEADER "X-Your-Address-Is: "
-
-/** HTTP cache control: how long do we tell proxies they can cache each
- * kind of document we serve? */
-#define FULL_DIR_CACHE_LIFETIME (60*60)
-#define RUNNINGROUTERS_CACHE_LIFETIME (20*60)
-#define DIRPORTFRONTPAGE_CACHE_LIFETIME (20*60)
-#define NETWORKSTATUS_CACHE_LIFETIME (5*60)
-#define ROUTERDESC_CACHE_LIFETIME (30*60)
-#define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
-#define ROBOTS_CACHE_LIFETIME (24*60*60)
-#define MICRODESC_CACHE_LIFETIME (48*60*60)
-
-/********* END VARIABLES ************/
-
-/** Return true iff the directory purpose <b>dir_purpose</b> (and if it's
- * fetching descriptors, it's fetching them for <b>router_purpose</b>)
- * must use an anonymous connection to a directory. */
-int
-purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose)
-{
-  if (get_options()->AllDirActionsPrivate)
-    return 1;
-  if (router_purpose == ROUTER_PURPOSE_BRIDGE)
-    return 1; /* if no circuits yet, this might break bootstrapping, but it's
-               * needed to be safe. */
-  if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR ||
-      dir_purpose == DIR_PURPOSE_UPLOAD_VOTE ||
-      dir_purpose == DIR_PURPOSE_UPLOAD_SIGNATURES ||
-      dir_purpose == DIR_PURPOSE_FETCH_STATUS_VOTE ||
-      dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES ||
-      dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS ||
-      dir_purpose == DIR_PURPOSE_FETCH_CERTIFICATE ||
-      dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
-      dir_purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
-      dir_purpose == DIR_PURPOSE_FETCH_MICRODESC)
-    return 0;
-  return 1;
-}
-
-/** Return a newly allocated string describing <b>auth</b>. Only describes
- * authority features. */
-STATIC char *
-authdir_type_to_string(dirinfo_type_t auth)
-{
-  char *result;
-  smartlist_t *lst = smartlist_new();
-  if (auth & V3_DIRINFO)
-    smartlist_add(lst, (void*)"V3");
-  if (auth & BRIDGE_DIRINFO)
-    smartlist_add(lst, (void*)"Bridge");
-  if (smartlist_len(lst)) {
-    result = smartlist_join_strings(lst, ", ", 0, NULL);
-  } else {
-    result = tor_strdup("[Not an authority]");
-  }
-  smartlist_free(lst);
-  return result;
-}
-
-/** Return a string describing a given directory connection purpose. */
-STATIC const char *
-dir_conn_purpose_to_string(int purpose)
-{
-  switch (purpose)
-    {
-    case DIR_PURPOSE_UPLOAD_DIR:
-      return "server descriptor upload";
-    case DIR_PURPOSE_UPLOAD_VOTE:
-      return "server vote upload";
-    case DIR_PURPOSE_UPLOAD_SIGNATURES:
-      return "consensus signature upload";
-    case DIR_PURPOSE_FETCH_SERVERDESC:
-      return "server descriptor fetch";
-    case DIR_PURPOSE_FETCH_EXTRAINFO:
-      return "extra-info fetch";
-    case DIR_PURPOSE_FETCH_CONSENSUS:
-      return "consensus network-status fetch";
-    case DIR_PURPOSE_FETCH_CERTIFICATE:
-      return "authority cert fetch";
-    case DIR_PURPOSE_FETCH_STATUS_VOTE:
-      return "status vote fetch";
-    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
-      return "consensus signature fetch";
-    case DIR_PURPOSE_FETCH_RENDDESC_V2:
-      return "hidden-service v2 descriptor fetch";
-    case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
-      return "hidden-service v2 descriptor upload";
-    case DIR_PURPOSE_FETCH_MICRODESC:
-      return "microdescriptor fetch";
-    }
-
-  log_warn(LD_BUG, "Called with unknown purpose %d", purpose);
-  return "(unknown)";
-}
-
-/** Return the requisite directory information types. */
-STATIC dirinfo_type_t
-dir_fetch_type(int dir_purpose, int router_purpose, const char *resource)
-{
-  dirinfo_type_t type;
-  switch (dir_purpose) {
-    case DIR_PURPOSE_FETCH_EXTRAINFO:
-      type = EXTRAINFO_DIRINFO;
-      if (router_purpose == ROUTER_PURPOSE_BRIDGE)
-        type |= BRIDGE_DIRINFO;
-      else
-        type |= V3_DIRINFO;
-      break;
-    case DIR_PURPOSE_FETCH_SERVERDESC:
-      if (router_purpose == ROUTER_PURPOSE_BRIDGE)
-        type = BRIDGE_DIRINFO;
-      else
-        type = V3_DIRINFO;
-      break;
-    case DIR_PURPOSE_FETCH_STATUS_VOTE:
-    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
-    case DIR_PURPOSE_FETCH_CERTIFICATE:
-      type = V3_DIRINFO;
-      break;
-    case DIR_PURPOSE_FETCH_CONSENSUS:
-      type = V3_DIRINFO;
-      if (resource && !strcmp(resource, "microdesc"))
-        type |= MICRODESC_DIRINFO;
-      break;
-    case DIR_PURPOSE_FETCH_MICRODESC:
-      type = MICRODESC_DIRINFO;
-      break;
-    default:
-      log_warn(LD_BUG, "Unexpected purpose %d", (int)dir_purpose);
-      type = NO_DIRINFO;
-      break;
-  }
-  return type;
-}
-
-/** Return true iff <b>identity_digest</b> is the digest of a router which
- * says that it caches extrainfos.  (If <b>is_authority</b> we always
- * believe that to be true.) */
-int
-router_supports_extrainfo(const char *identity_digest, int is_authority)
-{
-  const node_t *node = node_get_by_id(identity_digest);
-
-  if (node && node->ri) {
-    if (node->ri->caches_extra_info)
-      return 1;
-  }
-  if (is_authority) {
-    return 1;
-  }
-  return 0;
-}
-
-/** Return true iff any trusted directory authority has accepted our
- * server descriptor.
- *
- * We consider any authority sufficient because waiting for all of
- * them means it never happens while any authority is down; we don't
- * go for something more complex in the middle (like \>1/3 or \>1/2 or
- * \>=1/2) because that doesn't seem necessary yet.
- */
-int
-directories_have_accepted_server_descriptor(void)
-{
-  const smartlist_t *servers = router_get_trusted_dir_servers();
-  const or_options_t *options = get_options();
-  SMARTLIST_FOREACH(servers, dir_server_t *, d, {
-    if ((d->type & options->PublishServerDescriptor_) &&
-        d->has_accepted_serverdesc) {
-      return 1;
-    }
-  });
-  return 0;
-}
-
-/** Start a connection to every suitable directory authority, using
- * connection purpose <b>dir_purpose</b> and uploading <b>payload</b>
- * (of length <b>payload_len</b>). The dir_purpose should be one of
- * 'DIR_PURPOSE_UPLOAD_{DIR|VOTE|SIGNATURES}'.
- *
- * <b>router_purpose</b> describes the type of descriptor we're
- * publishing, if we're publishing a descriptor -- e.g. general or bridge.
- *
- * <b>type</b> specifies what sort of dir authorities (V3,
- * BRIDGE, etc) we should upload to.
- *
- * If <b>extrainfo_len</b> is nonzero, the first <b>payload_len</b> bytes of
- * <b>payload</b> hold a router descriptor, and the next <b>extrainfo_len</b>
- * bytes of <b>payload</b> hold an extra-info document.  Upload the descriptor
- * to all authorities, and the extra-info document to all authorities that
- * support it.
- */
-void
-directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
-                             dirinfo_type_t type,
-                             const char *payload,
-                             size_t payload_len, size_t extrainfo_len)
-{
-  const or_options_t *options = get_options();
-  dir_indirection_t indirection;
-  const smartlist_t *dirservers = router_get_trusted_dir_servers();
-  int found = 0;
-  const int exclude_self = (dir_purpose == DIR_PURPOSE_UPLOAD_VOTE ||
-                            dir_purpose == DIR_PURPOSE_UPLOAD_SIGNATURES);
-  tor_assert(dirservers);
-  /* This tries dirservers which we believe to be down, but ultimately, that's
-   * harmless, and we may as well err on the side of getting things uploaded.
-   */
-  SMARTLIST_FOREACH_BEGIN(dirservers, dir_server_t *, ds) {
-      routerstatus_t *rs = &(ds->fake_status);
-      size_t upload_len = payload_len;
-
-      if ((type & ds->type) == 0)
-        continue;
-
-      if (exclude_self && router_digest_is_me(ds->digest)) {
-        /* we don't upload to ourselves, but at least there's now at least
-         * one authority of this type that has what we wanted to upload. */
-        found = 1;
-        continue;
-      }
-
-      if (options->StrictNodes &&
-          routerset_contains_routerstatus(options->ExcludeNodes, rs, -1)) {
-        log_warn(LD_DIR, "Wanted to contact authority '%s' for %s, but "
-                 "it's in our ExcludedNodes list and StrictNodes is set. "
-                 "Skipping.",
-                 ds->nickname,
-                 dir_conn_purpose_to_string(dir_purpose));
-        continue;
-      }
-
-      found = 1; /* at least one authority of this type was listed */
-      if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR)
-        ds->has_accepted_serverdesc = 0;
-
-      if (extrainfo_len && router_supports_extrainfo(ds->digest, 1)) {
-        upload_len += extrainfo_len;
-        log_info(LD_DIR, "Uploading an extrainfo too (length %d)",
-                 (int) extrainfo_len);
-      }
-      if (purpose_needs_anonymity(dir_purpose, router_purpose)) {
-        indirection = DIRIND_ANONYMOUS;
-      } else if (!fascist_firewall_allows_dir_server(ds,
-                                                     FIREWALL_DIR_CONNECTION,
-                                                     0)) {
-        if (fascist_firewall_allows_dir_server(ds, FIREWALL_OR_CONNECTION, 0))
-          indirection = DIRIND_ONEHOP;
-        else
-          indirection = DIRIND_ANONYMOUS;
-      } else {
-        indirection = DIRIND_DIRECT_CONN;
-      }
-      directory_initiate_command_routerstatus(rs, dir_purpose,
-                                              router_purpose,
-                                              indirection,
-                                              NULL, payload, upload_len, 0);
-  } SMARTLIST_FOREACH_END(ds);
-  if (!found) {
-    char *s = authdir_type_to_string(type);
-    log_warn(LD_DIR, "Publishing server descriptor to directory authorities "
-             "of type '%s', but no authorities of that type listed!", s);
-    tor_free(s);
-  }
-}
-
-/** Return true iff, according to the values in <b>options</b>, we should be
- * using directory guards for direct downloads of directory information. */
-STATIC int
-should_use_directory_guards(const or_options_t *options)
-{
-  /* Public (non-bridge) servers never use directory guards. */
-  if (public_server_mode(options))
-    return 0;
-  /* If guards are disabled, or directory guards are disabled, we can't
-   * use directory guards.
-   */
-  if (!options->UseEntryGuards || !options->UseEntryGuardsAsDirGuards)
-    return 0;
-  /* If we're configured to fetch directory info aggressively or of a
-   * nonstandard type, don't use directory guards. */
-  if (options->DownloadExtraInfo || options->FetchDirInfoEarly ||
-      options->FetchDirInfoExtraEarly || options->FetchUselessDescriptors)
-    return 0;
-  return 1;
-}
-
-/** Pick an unconstrained directory server from among our guards, the latest
- * networkstatus, or the fallback dirservers, for use in downloading
- * information of type <b>type</b>, and return its routerstatus. */
-static const routerstatus_t *
-directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags,
-                                 uint8_t dir_purpose)
-{
-  const routerstatus_t *rs = NULL;
-  const or_options_t *options = get_options();
-
-  if (options->UseBridges)
-    log_warn(LD_BUG, "Called when we have UseBridges set.");
-
-  if (should_use_directory_guards(options)) {
-    const node_t *node = choose_random_dirguard(type);
-    if (node)
-      rs = node->rs;
-  } else {
-    /* anybody with a non-zero dirport will do */
-    rs = router_pick_directory_server(type, pds_flags);
-  }
-  if (!rs) {
-    log_info(LD_DIR, "No router found for %s; falling back to "
-             "dirserver list.", dir_conn_purpose_to_string(dir_purpose));
-    rs = router_pick_fallback_dirserver(type, pds_flags);
-  }
-
-  return rs;
-}
-
-/** Start a connection to a random running directory server, using
- * connection purpose <b>dir_purpose</b>, intending to fetch descriptors
- * of purpose <b>router_purpose</b>, and requesting <b>resource</b>.
- * Use <b>pds_flags</b> as arguments to router_pick_directory_server()
- * or router_pick_trusteddirserver().
- */
-MOCK_IMPL(void, directory_get_from_dirserver, (
-                            uint8_t dir_purpose,
-                            uint8_t router_purpose,
-                            const char *resource,
-                            int pds_flags,
-                            download_want_authority_t want_authority))
-{
-  const routerstatus_t *rs = NULL;
-  const or_options_t *options = get_options();
-  int prefer_authority = (directory_fetches_from_authorities(options)
-                          || want_authority == DL_WANT_AUTHORITY);
-  int require_authority = 0;
-  int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose);
-  dirinfo_type_t type = dir_fetch_type(dir_purpose, router_purpose, resource);
-  time_t if_modified_since = 0;
-
-  if (type == NO_DIRINFO)
-    return;
-
-  if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
-    int flav = FLAV_NS;
-    networkstatus_t *v;
-    if (resource)
-      flav = networkstatus_parse_flavor_name(resource);
-
-    /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus
-     * period of 1 hour.
-     */
-#define DEFAULT_IF_MODIFIED_SINCE_DELAY (180)
-    if (flav != -1) {
-      /* IF we have a parsed consensus of this type, we can do an
-       * if-modified-time based on it. */
-      v = networkstatus_get_latest_consensus_by_flavor(flav);
-      if (v) {
-        /* In networks with particularly short V3AuthVotingIntervals,
-         * ask for the consensus if it's been modified since half the
-         * V3AuthVotingInterval of the most recent consensus. */
-        time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY;
-        if (v->fresh_until > v->valid_after
-            && ims_delay > (v->fresh_until - v->valid_after)/2) {
-          ims_delay = (v->fresh_until - v->valid_after)/2;
-        }
-        if_modified_since = v->valid_after + ims_delay;
-      }
-    } else {
-      /* Otherwise it might be a consensus we don't parse, but which we
-       * do cache.  Look at the cached copy, perhaps. */
-      cached_dir_t *cd = dirserv_get_consensus(resource);
-      /* We have no method of determining the voting interval from an
-       * unparsed consensus, so we use the default. */
-      if (cd)
-        if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY;
-    }
-  }
-
-  if (!options->FetchServerDescriptors)
-    return;
-
-  if (!get_via_tor) {
-    if (options->UseBridges && !(type & BRIDGE_DIRINFO)) {
-      /* We want to ask a running bridge for which we have a descriptor.
-       *
-       * When we ask choose_random_entry() for a bridge, we specify what
-       * sort of dir fetch we'll be doing, so it won't return a bridge
-       * that can't answer our question.
-       */
-      const node_t *node = choose_random_dirguard(type);
-      if (node && node->ri) {
-        /* every bridge has a routerinfo. */
-        routerinfo_t *ri = node->ri;
-        /* clients always make OR connections to bridges */
-        tor_addr_port_t or_ap;
-        /* we are willing to use a non-preferred address if we need to */
-        fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0,
-                                             &or_ap);
-        directory_initiate_command(&or_ap.addr, or_ap.port,
-                                   NULL, 0, /*no dirport*/
-                                   ri->cache_info.identity_digest,
-                                   dir_purpose,
-                                   router_purpose,
-                                   DIRIND_ONEHOP,
-                                   resource, NULL, 0, if_modified_since);
-      } else
-        log_notice(LD_DIR, "Ignoring directory request, since no bridge "
-                           "nodes are available yet.");
-      return;
-    } else {
-      if (prefer_authority || (type & BRIDGE_DIRINFO)) {
-        /* only ask authdirservers, and don't ask myself */
-        rs = router_pick_trusteddirserver(type, pds_flags);
-        if (rs == NULL && (pds_flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
-                                        PDS_NO_EXISTING_MICRODESC_FETCH))) {
-          /* We don't want to fetch from any authorities that we're currently
-           * fetching server descriptors from, and we got no match.  Did we
-           * get no match because all the authorities have connections
-           * fetching server descriptors (in which case we should just
-           * return,) or because all the authorities are down or on fire or
-           * unreachable or something (in which case we should go on with
-           * our fallback code)? */
-          pds_flags &= ~(PDS_NO_EXISTING_SERVERDESC_FETCH|
-                         PDS_NO_EXISTING_MICRODESC_FETCH);
-          rs = router_pick_trusteddirserver(type, pds_flags);
-          if (rs) {
-            log_debug(LD_DIR, "Deferring serverdesc fetch: all authorities "
-                      "are in use.");
-            return;
-          }
-        }
-        if (rs == NULL && require_authority) {
-          log_info(LD_DIR, "No authorities were available for %s: will try "
-                   "later.", dir_conn_purpose_to_string(dir_purpose));
-          return;
-        }
-      }
-      if (!rs && !(type & BRIDGE_DIRINFO)) {
-        /* */
-        rs = directory_pick_generic_dirserver(type, pds_flags,
-                                              dir_purpose);
-        if (!rs)
-          get_via_tor = 1; /* last resort: try routing it via Tor */
-      }
-    }
-  }
-
-  if (get_via_tor) {
-    /* Never use fascistfirewall; we're going via Tor. */
-    pds_flags |= PDS_IGNORE_FASCISTFIREWALL;
-    rs = router_pick_directory_server(type, pds_flags);
-  }
-
-  /* If we have any hope of building an indirect conn, we know some router
-   * descriptors.  If (rs==NULL), we can't build circuits anyway, so
-   * there's no point in falling back to the authorities in this case. */
-  if (rs) {
-    const dir_indirection_t indirection =
-      get_via_tor ? DIRIND_ANONYMOUS : DIRIND_ONEHOP;
-    directory_initiate_command_routerstatus(rs, dir_purpose,
-                                            router_purpose,
-                                            indirection,
-                                            resource, NULL, 0,
-                                            if_modified_since);
-  } else {
-    log_notice(LD_DIR,
-               "While fetching directory info, "
-               "no running dirservers known. Will try again later. "
-               "(purpose %d)", dir_purpose);
-    if (!purpose_needs_anonymity(dir_purpose, router_purpose)) {
-      /* remember we tried them all and failed. */
-      directory_all_unreachable(time(NULL));
-    }
-  }
-}
-
-/** As directory_get_from_dirserver, but initiates a request to <i>every</i>
- * directory authority other than ourself.  Only for use by authorities when
- * searching for missing information while voting. */
-void
-directory_get_from_all_authorities(uint8_t dir_purpose,
-                                   uint8_t router_purpose,
-                                   const char *resource)
-{
-  tor_assert(dir_purpose == DIR_PURPOSE_FETCH_STATUS_VOTE ||
-             dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES);
-
-  SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
-                          dir_server_t *, ds) {
-      routerstatus_t *rs;
-      if (router_digest_is_me(ds->digest))
-        continue;
-      if (!(ds->type & V3_DIRINFO))
-        continue;
-      rs = &ds->fake_status;
-      directory_initiate_command_routerstatus(rs, dir_purpose, router_purpose,
-                                              DIRIND_ONEHOP, resource, NULL,
-                                              0, 0);
-  } SMARTLIST_FOREACH_END(ds);
-}
-
-/** Return true iff <b>ind</b> requires a multihop circuit. */
-static int
-dirind_is_anon(dir_indirection_t ind)
-{
-  return ind == DIRIND_ANON_DIRPORT || ind == DIRIND_ANONYMOUS;
-}
-
-/* Choose reachable OR and Dir addresses and ports from status, copying them
- * into use_or_ap and use_dir_ap. If indirection is anonymous, then we're
- * connecting via another relay, so choose the primary IPv4 address and ports.
- *
- * status should have at least one reachable address, if we can't choose a
- * reachable address, warn and return -1. Otherwise, return 0.
- */
-static int
-directory_choose_address_routerstatus(const routerstatus_t *status,
-                                      dir_indirection_t indirection,
-                                      tor_addr_port_t *use_or_ap,
-                                      tor_addr_port_t *use_dir_ap)
-{
-  tor_assert(status != NULL);
-  tor_assert(use_or_ap != NULL);
-  tor_assert(use_dir_ap != NULL);
-
-  const or_options_t *options = get_options();
-  int have_or = 0, have_dir = 0;
-
-  /* We expect status to have at least one reachable address if we're
-   * connecting to it directly.
-   *
-   * Therefore, we can simply use the other address if the one we want isn't
-   * allowed by the firewall.
-   *
-   * (When Tor uploads and downloads a hidden service descriptor, it uses
-   * DIRIND_ANONYMOUS, except for Tor2Web, which uses DIRIND_ONEHOP.
-   * So this code will only modify the address for Tor2Web's HS descriptor
-   * fetches. Even Single Onion Servers (NYI) use DIRIND_ANONYMOUS, to avoid
-   * HSDirs denying service by rejecting descriptors.)
-   */
-
-  /* Initialise the OR / Dir addresses */
-  tor_addr_make_null(&use_or_ap->addr, AF_UNSPEC);
-  use_or_ap->port = 0;
-  tor_addr_make_null(&use_dir_ap->addr, AF_UNSPEC);
-  use_dir_ap->port = 0;
-
-  /* ORPort connections */
-  if (indirection == DIRIND_ANONYMOUS) {
-    if (status->addr) {
-      /* Since we're going to build a 3-hop circuit and ask the 2nd relay
-       * to extend to this address, always use the primary (IPv4) OR address */
-      tor_addr_from_ipv4h(&use_or_ap->addr, status->addr);
-      use_or_ap->port = status->or_port;
-      have_or = 1;
-    }
-  } else if (indirection == DIRIND_ONEHOP) {
-    /* We use an IPv6 address if we have one and we prefer it.
-     * Use the preferred address and port if they are reachable, otherwise,
-     * use the alternate address and port (if any).
-     */
-    have_or = fascist_firewall_choose_address_rs(status,
-                                                 FIREWALL_OR_CONNECTION, 0,
-                                                 use_or_ap);
-  }
-
-  /* DirPort connections
-   * DIRIND_ONEHOP uses ORPort, but may fall back to the DirPort on relays */
-  if (indirection == DIRIND_DIRECT_CONN ||
-      indirection == DIRIND_ANON_DIRPORT ||
-      (indirection == DIRIND_ONEHOP
-       && !directory_must_use_begindir(options))) {
-    have_dir = fascist_firewall_choose_address_rs(status,
-                                                  FIREWALL_DIR_CONNECTION, 0,
-                                                  use_dir_ap);
-  }
-
-  /* We rejected all addresses in the relay's status. This means we can't
-   * connect to it. */
-  if (!have_or && !have_dir) {
-    static int logged_backtrace = 0;
-    log_info(LD_BUG, "Rejected all OR and Dir addresses from %s when "
-             "launching an outgoing directory connection to: IPv4 %s OR %d "
-             "Dir %d IPv6 %s OR %d Dir %d", routerstatus_describe(status),
-             fmt_addr32(status->addr), status->or_port,
-             status->dir_port, fmt_addr(&status->ipv6_addr),
-             status->ipv6_orport, status->dir_port);
-    if (!logged_backtrace) {
-      log_backtrace(LOG_INFO, LD_BUG, "Addresses came from");
-      logged_backtrace = 1;
-    }
-    return -1;
-  }
-
-  return 0;
-}
-
-/** Same as directory_initiate_command_routerstatus(), but accepts
- * rendezvous data to fetch a hidden service descriptor. */
-void
-directory_initiate_command_routerstatus_rend(const routerstatus_t *status,
-                                             uint8_t dir_purpose,
-                                             uint8_t router_purpose,
-                                             dir_indirection_t indirection,
-                                             const char *resource,
-                                             const char *payload,
-                                             size_t payload_len,
-                                             time_t if_modified_since,
-                                             const rend_data_t *rend_query)
-{
-  const or_options_t *options = get_options();
-  const node_t *node;
-  tor_addr_port_t use_or_ap, use_dir_ap;
-  const int anonymized_connection = dirind_is_anon(indirection);
-
-  tor_assert(status != NULL);
-
-  node = node_get_by_id(status->identity_digest);
-
-  /* XXX The below check is wrong: !node means it's not in the consensus,
-   * but we haven't checked if we have a descriptor for it -- and also,
-   * we only care about the descriptor if it's a begindir-style anonymized
-   * connection. */
-  if (!node && anonymized_connection) {
-    log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we "
-             "don't have its router descriptor.",
-             routerstatus_describe(status));
-    return;
-  }
-
-  if (options->ExcludeNodes && options->StrictNodes &&
-      routerset_contains_routerstatus(options->ExcludeNodes, status, -1)) {
-    log_warn(LD_DIR, "Wanted to contact directory mirror %s for %s, but "
-             "it's in our ExcludedNodes list and StrictNodes is set. "
-             "Skipping. This choice might make your Tor not work.",
-             routerstatus_describe(status),
-             dir_conn_purpose_to_string(dir_purpose));
-    return;
-  }
-
-  /* At this point, if we are a client making a direct connection to a
-   * directory server, we have selected a server that has at least one address
-   * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This
-   * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if
-   * possible. (If UseBridges is set, clients always use IPv6, and prefer it
-   * by default.)
-   *
-   * Now choose an address that we can use to connect to the directory server.
-   */
-  if (directory_choose_address_routerstatus(status, indirection, &use_or_ap,
-                                            &use_dir_ap) < 0) {
-    return;
-  }
-
-  /* We don't retry the alternate OR/Dir address for the same directory if
-   * the address we choose fails (#6772).
-   * Instead, we'll retry another directory on failure. */
-
-  directory_initiate_command_rend(&use_or_ap, &use_dir_ap,
-                                  status->identity_digest,
-                                  dir_purpose, router_purpose,
-                                  indirection, resource,
-                                  payload, payload_len, if_modified_since,
-                                  rend_query);
-}
-
-/** Launch a new connection to the directory server <b>status</b> to
- * upload or download a server or rendezvous
- * descriptor. <b>dir_purpose</b> determines what
- * kind of directory connection we're launching, and must be one of
- * DIR_PURPOSE_{FETCH|UPLOAD}_{DIR|RENDDESC_V2}. <b>router_purpose</b>
- * specifies the descriptor purposes we have in mind (currently only
- * used for FETCH_DIR).
- *
- * When uploading, <b>payload</b> and <b>payload_len</b> determine the content
- * of the HTTP post.  Otherwise, <b>payload</b> should be NULL.
- *
- * When fetching a rendezvous descriptor, <b>resource</b> is the service ID we
- * want to fetch.
- */
-MOCK_IMPL(void, directory_initiate_command_routerstatus,
-                (const routerstatus_t *status,
-                 uint8_t dir_purpose,
-                 uint8_t router_purpose,
-                 dir_indirection_t indirection,
-                 const char *resource,
-                 const char *payload,
-                 size_t payload_len,
-                 time_t if_modified_since))
-{
-  directory_initiate_command_routerstatus_rend(status, dir_purpose,
-                                          router_purpose,
-                                          indirection, resource,
-                                          payload, payload_len,
-                                          if_modified_since, NULL);
-}
-
-/** Return true iff <b>conn</b> is the client side of a directory connection
- * we launched to ourself in order to determine the reachability of our
- * dir_port. */
-static int
-directory_conn_is_self_reachability_test(dir_connection_t *conn)
-{
-  if (conn->requested_resource &&
-      !strcmpstart(conn->requested_resource,"authority")) {
-    const routerinfo_t *me = router_get_my_routerinfo();
-    if (me &&
-        router_digest_is_me(conn->identity_digest) &&
-        tor_addr_eq_ipv4h(&conn->base_.addr, me->addr) && /*XXXX prop 118*/
-        me->dir_port == conn->base_.port)
-      return 1;
-  }
-  return 0;
-}
-
-/** Called when we are unable to complete the client's request to a directory
- * server due to a network error: Mark the router as down and try again if
- * possible.
- */
-static void
-connection_dir_request_failed(dir_connection_t *conn)
-{
-  if (directory_conn_is_self_reachability_test(conn)) {
-    return; /* this was a test fetch. don't retry. */
-  }
-  if (!entry_list_is_constrained(get_options()))
-    router_set_status(conn->identity_digest, 0); /* don't try this one again */
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
-             conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) {
-    log_info(LD_DIR, "Giving up on serverdesc/extrainfo fetch from "
-             "directory server at '%s'; retrying",
-             conn->base_.address);
-    if (conn->router_purpose == ROUTER_PURPOSE_BRIDGE)
-      connection_dir_bridge_routerdesc_failed(conn);
-    connection_dir_download_routerdesc_failed(conn);
-  } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
-    if (conn->requested_resource)
-      networkstatus_consensus_download_failed(0, conn->requested_resource);
-  } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) {
-    log_info(LD_DIR, "Giving up on certificate fetch from directory server "
-             "at '%s'; retrying",
-             conn->base_.address);
-    connection_dir_download_cert_failed(conn, 0);
-  } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) {
-    log_info(LD_DIR, "Giving up downloading detached signatures from '%s'",
-             conn->base_.address);
-  } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) {
-    log_info(LD_DIR, "Giving up downloading votes from '%s'",
-             conn->base_.address);
-  } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC) {
-    log_info(LD_DIR, "Giving up on downloading microdescriptors from "
-             "directory server at '%s'; will retry", conn->base_.address);
-    connection_dir_download_routerdesc_failed(conn);
-  }
-}
-
-/** Helper: Attempt to fetch directly the descriptors of each bridge
- * listed in <b>failed</b>.
- */
-static void
-connection_dir_retry_bridges(smartlist_t *descs)
-{
-  char digest[DIGEST_LEN];
-  SMARTLIST_FOREACH(descs, const char *, cp,
-  {
-    if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp)) != DIGEST_LEN) {
-      log_warn(LD_BUG, "Malformed fingerprint in list: %s",
-              escaped(cp));
-      continue;
-    }
-    retry_bridge_descriptor_fetch_directly(digest);
-  });
-}
-
-/** Called when an attempt to download one or more router descriptors
- * or extra-info documents on connection <b>conn</b> failed.
- */
-static void
-connection_dir_download_routerdesc_failed(dir_connection_t *conn)
-{
-  /* No need to increment the failure count for routerdescs, since
-   * it's not their fault. */
-
-  /* No need to relaunch descriptor downloads here: we already do it
-   * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
-  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
-             conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
-             conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
-
-  (void) conn;
-}
-
-/** Called when an attempt to download a bridge's routerdesc from
- * one of the authorities failed due to a network error. If
- * possible attempt to download descriptors from the bridge directly.
- */
-static void
-connection_dir_bridge_routerdesc_failed(dir_connection_t *conn)
-{
-  smartlist_t *which = NULL;
-
-  /* Requests for bridge descriptors are in the form 'fp/', so ignore
-     anything else. */
-  if (!conn->requested_resource || strcmpstart(conn->requested_resource,"fp/"))
-    return;
-
-  which = smartlist_new();
-  dir_split_resource_into_fingerprints(conn->requested_resource
-                                        + strlen("fp/"),
-                                       which, NULL, 0);
-
-  tor_assert(conn->base_.purpose != DIR_PURPOSE_FETCH_EXTRAINFO);
-  if (smartlist_len(which)) {
-    connection_dir_retry_bridges(which);
-    SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
-  }
-  smartlist_free(which);
-}
-
-/** Called when an attempt to fetch a certificate fails. */
-static void
-connection_dir_download_cert_failed(dir_connection_t *conn, int status)
-{
-  const char *fp_pfx = "fp/";
-  const char *fpsk_pfx = "fp-sk/";
-  smartlist_t *failed;
-  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE);
-
-  if (!conn->requested_resource)
-    return;
-  failed = smartlist_new();
-  /*
-   * We have two cases download by fingerprint (resource starts
-   * with "fp/") or download by fingerprint/signing key pair
-   * (resource starts with "fp-sk/").
-   */
-  if (!strcmpstart(conn->requested_resource, fp_pfx)) {
-    /* Download by fingerprint case */
-    dir_split_resource_into_fingerprints(conn->requested_resource +
-                                         strlen(fp_pfx),
-                                         failed, NULL, DSR_HEX);
-    SMARTLIST_FOREACH_BEGIN(failed, char *, cp) {
-      /* Null signing key digest indicates download by fp only */
-      authority_cert_dl_failed(cp, NULL, status);
-      tor_free(cp);
-    } SMARTLIST_FOREACH_END(cp);
-  } else if (!strcmpstart(conn->requested_resource, fpsk_pfx)) {
-    /* Download by (fp,sk) pairs */
-    dir_split_resource_into_fingerprint_pairs(conn->requested_resource +
-                                              strlen(fpsk_pfx), failed);
-    SMARTLIST_FOREACH_BEGIN(failed, fp_pair_t *, cp) {
-      authority_cert_dl_failed(cp->first, cp->second, status);
-      tor_free(cp);
-    } SMARTLIST_FOREACH_END(cp);
-  } else {
-    log_warn(LD_DIR,
-             "Don't know what to do with failure for cert fetch %s",
-             conn->requested_resource);
-  }
-
-  smartlist_free(failed);
-
-  update_certificate_downloads(time(NULL));
-}
-
-/* Should this tor instance only use begindir for all its directory requests?
- */
-int
-directory_must_use_begindir(const or_options_t *options)
-{
-  /* Clients, onion services, and bridges must use begindir,
-   * relays and authorities do not have to */
-  return !public_server_mode(options);
-}
-
-/** Evaluate the situation and decide if we should use an encrypted
- * "begindir-style" connection for this directory request.
- * 1) If or_port is 0, or it's a direct conn and or_port is firewalled
- *    or we're a dir mirror, no.
- * 2) If we prefer to avoid begindir conns, and we're not fetching or
- *    publishing a bridge relay descriptor, no.
- * 3) Else yes.
- * If returning 0, return in *reason why we can't use begindir.
- * reason must not be NULL.
- */
-static int
-directory_command_should_use_begindir(const or_options_t *options,
-                                      const tor_addr_t *addr,
-                                      int or_port, uint8_t router_purpose,
-                                      dir_indirection_t indirection,
-                                      const char **reason)
-{
-  (void) router_purpose;
-  tor_assert(reason);
-  *reason = NULL;
-
-  /* Reasons why we can't possibly use begindir */
-  if (!or_port) {
-    *reason = "directory with unknown ORPort";
-    return 0; /* We don't know an ORPort -- no chance. */
-  }
-  if (indirection == DIRIND_DIRECT_CONN ||
-      indirection == DIRIND_ANON_DIRPORT) {
-    *reason = "DirPort connection";
-    return 0;
-  }
-  if (indirection == DIRIND_ONEHOP) {
-    /* We're firewalled and want a direct OR connection */
-    if (!fascist_firewall_allows_address_addr(addr, or_port,
-                                              FIREWALL_OR_CONNECTION, 0, 0)) {
-      *reason = "ORPort not reachable";
-      return 0;
-    }
-  }
-  /* Reasons why we want to avoid using begindir */
-  if (indirection == DIRIND_ONEHOP) {
-    if (!directory_must_use_begindir(options)) {
-      *reason = "in relay mode";
-      return 0;
-    }
-  }
-  /* DIRIND_ONEHOP on a client, or DIRIND_ANONYMOUS
-   */
-  *reason = "(using begindir)";
-  return 1;
-}
-
-/** Helper for directory_initiate_command_rend: send the
- * command to a server whose OR address/port is <b>or_addr</b>/<b>or_port</b>,
- * whose directory address/port is <b>dir_addr</b>/<b>dir_port</b>, whose
- * identity key digest is <b>digest</b>, with purposes <b>dir_purpose</b> and
- * <b>router_purpose</b>, making an (in)direct connection as specified in
- * <b>indirection</b>, with command <b>resource</b>, <b>payload</b> of
- * <b>payload_len</b>, and asking for a result only <b>if_modified_since</b>.
- */
-void
-directory_initiate_command(const tor_addr_t *or_addr, uint16_t or_port,
-                           const tor_addr_t *dir_addr, uint16_t dir_port,
-                           const char *digest,
-                           uint8_t dir_purpose, uint8_t router_purpose,
-                           dir_indirection_t indirection, const char *resource,
-                           const char *payload, size_t payload_len,
-                           time_t if_modified_since)
-{
-  tor_addr_port_t or_ap, dir_ap;
-
-  /* Use the null tor_addr and 0 port if the address or port isn't valid. */
-  if (tor_addr_port_is_valid(or_addr, or_port, 0)) {
-    tor_addr_copy(&or_ap.addr, or_addr);
-    or_ap.port = or_port;
-  } else {
-    /* the family doesn't matter here, so make it IPv4 */
-    tor_addr_make_null(&or_ap.addr, AF_INET);
-    or_ap.port = or_port = 0;
-  }
-
-  if (tor_addr_port_is_valid(dir_addr, dir_port, 0)) {
-    tor_addr_copy(&dir_ap.addr, dir_addr);
-    dir_ap.port = dir_port;
-  } else {
-    /* the family doesn't matter here, so make it IPv4 */
-    tor_addr_make_null(&dir_ap.addr, AF_INET);
-    dir_ap.port = dir_port = 0;
-  }
-
-  directory_initiate_command_rend(&or_ap, &dir_ap,
-                             digest, dir_purpose,
-                             router_purpose, indirection,
-                             resource, payload, payload_len,
-                             if_modified_since, NULL);
-}
-
-/** Return non-zero iff a directory connection with purpose
- * <b>dir_purpose</b> reveals sensitive information about a Tor
- * instance's client activities.  (Such connections must be performed
- * through normal three-hop Tor circuits.) */
-int
-is_sensitive_dir_purpose(uint8_t dir_purpose)
-{
-  return ((dir_purpose == DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2) ||
-          (dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) ||
-          (dir_purpose == DIR_PURPOSE_FETCH_RENDDESC_V2));
-}
-
-/** Same as directory_initiate_command(), but accepts rendezvous data to
- * fetch a hidden service descriptor, and takes its address & port arguments
- * as tor_addr_port_t. */
-static void
-directory_initiate_command_rend(const tor_addr_port_t *or_addr_port,
-                                const tor_addr_port_t *dir_addr_port,
-                                const char *digest,
-                                uint8_t dir_purpose, uint8_t router_purpose,
-                                dir_indirection_t indirection,
-                                const char *resource,
-                                const char *payload, size_t payload_len,
-                                time_t if_modified_since,
-                                const rend_data_t *rend_query)
-{
-  tor_assert(or_addr_port);
-  tor_assert(dir_addr_port);
-  tor_assert(or_addr_port->port || dir_addr_port->port);
-  tor_assert(digest);
-
-  dir_connection_t *conn;
-  const or_options_t *options = get_options();
-  int socket_error = 0;
-  const char *begindir_reason = NULL;
-  /* Should the connection be to a relay's OR port (and inside that we will
-   * send our directory request)? */
-  const int use_begindir = directory_command_should_use_begindir(options,
-                                     &or_addr_port->addr, or_addr_port->port,
-                                     router_purpose, indirection,
-                                     &begindir_reason);
-  /* Will the connection go via a three-hop Tor circuit? Note that this
-   * is separate from whether it will use_begindir. */
-  const int anonymized_connection = dirind_is_anon(indirection);
-
-  /* What is the address we want to make the directory request to? If
-   * we're making a begindir request this is the ORPort of the relay
-   * we're contacting; if not a begindir request, this is its DirPort.
-   * Note that if anonymized_connection is true, we won't be initiating
-   * a connection directly to this address. */
-  tor_addr_t addr;
-  tor_addr_copy(&addr, &(use_begindir ? or_addr_port : dir_addr_port)->addr);
-  uint16_t port = (use_begindir ? or_addr_port : dir_addr_port)->port;
-
-  log_debug(LD_DIR, "anonymized %d, use_begindir %d.",
-            anonymized_connection, use_begindir);
-
-  log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose));
-
-  if (is_sensitive_dir_purpose(dir_purpose)) {
-    tor_assert(anonymized_connection ||
-               rend_non_anonymous_mode_enabled(options));
-  }
-
-  /* use encrypted begindir connections for everything except relays
-   * this provides better protection for directory fetches */
-  if (!use_begindir && directory_must_use_begindir(options)) {
-    log_warn(LD_BUG, "Client could not use begindir connection: %s",
-             begindir_reason ? begindir_reason : "(NULL)");
-    return;
-  }
-
-  /* ensure that we don't make direct connections when a SOCKS server is
-   * configured. */
-  if (!anonymized_connection && !use_begindir && !options->HTTPProxy &&
-      (options->Socks4Proxy || options->Socks5Proxy)) {
-    log_warn(LD_DIR, "Cannot connect to a directory server through a "
-                     "SOCKS proxy!");
-    return;
-  }
-
-  /* Make sure that the destination addr and port we picked is viable. */
-  if (!port || tor_addr_is_null(&addr)) {
-    static int logged_backtrace = 0;
-    log_warn(LD_DIR,
-             "Cannot make an outgoing %sconnection without %sPort.",
-             use_begindir ? "begindir " : "",
-             use_begindir ? "an OR" : "a Dir");
-    if (!logged_backtrace) {
-      log_backtrace(LOG_INFO, LD_BUG, "Address came from");
-      logged_backtrace = 1;
-    }
-    return;
-  }
-
-  conn = dir_connection_new(tor_addr_family(&addr));
-
-  /* set up conn so it's got all the data we need to remember */
-  tor_addr_copy(&conn->base_.addr, &addr);
-  conn->base_.port = port;
-  conn->base_.address = tor_addr_to_str_dup(&addr);
-  memcpy(conn->identity_digest, digest, DIGEST_LEN);
-
-  conn->base_.purpose = dir_purpose;
-  conn->router_purpose = router_purpose;
-
-  /* give it an initial state */
-  conn->base_.state = DIR_CONN_STATE_CONNECTING;
-
-  /* decide whether we can learn our IP address from this conn */
-  /* XXXX This is a bad name for this field now. */
-  conn->dirconn_direct = !anonymized_connection;
-
-  /* copy rendezvous data, if any */
-  if (rend_query)
-    conn->rend_data = rend_data_dup(rend_query);
-
-  if (!anonymized_connection && !use_begindir) {
-    /* then we want to connect to dirport directly */
-
-    if (options->HTTPProxy) {
-      tor_addr_copy(&addr, &options->HTTPProxyAddr);
-      port = options->HTTPProxyPort;
-    }
-
-    switch (connection_connect(TO_CONN(conn), conn->base_.address, &addr,
-                               port, &socket_error)) {
-      case -1:
-        connection_mark_for_close(TO_CONN(conn));
-        return;
-      case 1:
-        /* start flushing conn */
-        conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING;
-        /* fall through */
-      case 0:
-        /* queue the command on the outbuf */
-        directory_send_command(conn, dir_purpose, 1, resource,
-                               payload, payload_len,
-                               if_modified_since);
-        connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT);
-        /* writable indicates finish, readable indicates broken link,
-           error indicates broken link in windowsland. */
-    }
-  } else {
-    /* We will use a Tor circuit (maybe 1-hop, maybe 3-hop, maybe with
-     * begindir, maybe not with begindir) */
-
-    entry_connection_t *linked_conn;
-
-    /* Anonymized tunneled connections can never share a circuit.
-     * One-hop directory connections can share circuits with each other
-     * but nothing else. */
-    int iso_flags = anonymized_connection ? ISO_STREAM : ISO_SESSIONGRP;
-
-    /* If it's an anonymized connection, remember the fact that we
-     * wanted it for later: maybe we'll want it again soon. */
-    if (anonymized_connection && use_begindir)
-      rep_hist_note_used_internal(time(NULL), 0, 1);
-    else if (anonymized_connection && !use_begindir)
-      rep_hist_note_used_port(time(NULL), conn->base_.port);
-
-    /* make an AP connection
-     * populate it and add it at the right state
-     * hook up both sides
-     */
-    linked_conn =
-      connection_ap_make_link(TO_CONN(conn),
-                              conn->base_.address, conn->base_.port,
-                              digest,
-                              SESSION_GROUP_DIRCONN, iso_flags,
-                              use_begindir, !anonymized_connection);
-    if (!linked_conn) {
-      log_warn(LD_NET,"Making tunnel to dirserver failed.");
-      connection_mark_for_close(TO_CONN(conn));
-      return;
-    }
-
-    if (connection_add(TO_CONN(conn)) < 0) {
-      log_warn(LD_NET,"Unable to add connection for link to dirserver.");
-      connection_mark_for_close(TO_CONN(conn));
-      return;
-    }
-    conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING;
-    /* queue the command on the outbuf */
-    directory_send_command(conn, dir_purpose, 0, resource,
-                           payload, payload_len,
-                           if_modified_since);
-
-    connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT);
-    connection_start_reading(ENTRY_TO_CONN(linked_conn));
-  }
-}
-
-/** Return true iff anything we say on <b>conn</b> is being encrypted before
- * we send it to the client/server. */
-int
-connection_dir_is_encrypted(dir_connection_t *conn)
-{
-  /* Right now it's sufficient to see if conn is or has been linked, since
-   * the only thing it could be linked to is an edge connection on a
-   * circuit, and the only way it could have been unlinked is at the edge
-   * connection getting closed.
-   */
-  return TO_CONN(conn)->linked;
-}
-
-/** Helper for sorting
- *
- * sort strings alphabetically
- */
-static int
-compare_strs_(const void **a, const void **b)
-{
-  const char *s1 = *a, *s2 = *b;
-  return strcmp(s1, s2);
-}
-
-#define CONDITIONAL_CONSENSUS_FPR_LEN 3
-#if (CONDITIONAL_CONSENSUS_FPR_LEN > DIGEST_LEN)
-#error "conditional consensus fingerprint length is larger than digest length"
-#endif
-
-/** Return the URL we should use for a consensus download.
- *
- * Use the "conditional consensus downloading" feature described in
- * dir-spec.txt, i.e.
- * GET .../consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>
- *
- * If 'resource' is provided, it is the name of a consensus flavor to request.
- */
-static char *
-directory_get_consensus_url(const char *resource)
-{
-  char *url = NULL;
-  const char *hyphen, *flavor;
-  if (resource==NULL || strcmp(resource, "ns")==0) {
-    flavor = ""; /* Request ns consensuses as "", so older servers will work*/
-    hyphen = "";
-  } else {
-    flavor = resource;
-    hyphen = "-";
-  }
-
-  {
-    char *authority_id_list;
-    smartlist_t *authority_digests = smartlist_new();
-
-    SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
-                            dir_server_t *, ds) {
-        char *hex;
-        if (!(ds->type & V3_DIRINFO))
-          continue;
-
-        hex = tor_malloc(2*CONDITIONAL_CONSENSUS_FPR_LEN+1);
-        base16_encode(hex, 2*CONDITIONAL_CONSENSUS_FPR_LEN+1,
-                      ds->v3_identity_digest, CONDITIONAL_CONSENSUS_FPR_LEN);
-        smartlist_add(authority_digests, hex);
-    } SMARTLIST_FOREACH_END(ds);
-    smartlist_sort(authority_digests, compare_strs_);
-    authority_id_list = smartlist_join_strings(authority_digests,
-                                               "+", 0, NULL);
-
-    tor_asprintf(&url, "/tor/status-vote/current/consensus%s%s/%s.z",
-                 hyphen, flavor, authority_id_list);
-
-    SMARTLIST_FOREACH(authority_digests, char *, cp, tor_free(cp));
-    smartlist_free(authority_digests);
-    tor_free(authority_id_list);
-  }
-  return url;
-}
-
-/**
- * Copies the ipv6 from source to destination, subject to buffer size limit
- * size. If decorate is true, makes sure the copied address is decorated.
- */
-static void
-copy_ipv6_address(char* destination, const char* source, size_t len,
-                  int decorate) {
-  tor_assert(destination);
-  tor_assert(source);
-
-  if (decorate && source[0] != '[') {
-    tor_snprintf(destination, len, "[%s]", source);
-  } else {
-    strlcpy(destination, source, len);
-  }
-}
-
-/** Queue an appropriate HTTP command on conn-\>outbuf.  The other args
- * are as in directory_initiate_command().
- */
-static void
-directory_send_command(dir_connection_t *conn,
-                       int purpose, int direct, const char *resource,
-                       const char *payload, size_t payload_len,
-                       time_t if_modified_since)
-{
-  char proxystring[256];
-  char hoststring[128];
-  /* NEEDS to be the same size hoststring.
-   Will be decorated with brackets around it if it is ipv6. */
-  char decorated_address[128];
-  smartlist_t *headers = smartlist_new();
-  char *url;
-  char request[8192];
-  const char *httpcommand = NULL;
-
-  tor_assert(conn);
-  tor_assert(conn->base_.type == CONN_TYPE_DIR);
-
-  tor_free(conn->requested_resource);
-  if (resource)
-    conn->requested_resource = tor_strdup(resource);
-
-  /* decorate the ip address if it is ipv6 */
-  if (strchr(conn->base_.address, ':')) {
-    copy_ipv6_address(decorated_address, conn->base_.address,
-                      sizeof(decorated_address), 1);
-  } else {
-    strlcpy(decorated_address, conn->base_.address, sizeof(decorated_address));
-  }
-
-  /* come up with a string for which Host: we want */
-  if (conn->base_.port == 80) {
-    strlcpy(hoststring, decorated_address, sizeof(hoststring));
-  } else {
-    tor_snprintf(hoststring, sizeof(hoststring), "%s:%d",
-                 decorated_address, conn->base_.port);
-  }
-
-  /* Format if-modified-since */
-  if (if_modified_since) {
-    char b[RFC1123_TIME_LEN+1];
-    format_rfc1123_time(b, if_modified_since);
-    smartlist_add_asprintf(headers, "If-Modified-Since: %s\r\n", b);
-  }
-
-  /* come up with some proxy lines, if we're using one. */
-  if (direct && get_options()->HTTPProxy) {
-    char *base64_authenticator=NULL;
-    const char *authenticator = get_options()->HTTPProxyAuthenticator;
-
-    tor_snprintf(proxystring, sizeof(proxystring),"http://%s", hoststring);
-    if (authenticator) {
-      base64_authenticator = alloc_http_authenticator(authenticator);
-      if (!base64_authenticator)
-        log_warn(LD_BUG, "Encoding http authenticator failed");
-    }
-    if (base64_authenticator) {
-      smartlist_add_asprintf(headers,
-                   "Proxy-Authorization: Basic %s\r\n",
-                   base64_authenticator);
-      tor_free(base64_authenticator);
-    }
-  } else {
-    proxystring[0] = 0;
-  }
-
-  switch (purpose) {
-    case DIR_PURPOSE_FETCH_CONSENSUS:
-      /* resource is optional.  If present, it's a flavor name */
-      tor_assert(!payload);
-      httpcommand = "GET";
-      url = directory_get_consensus_url(resource);
-      log_info(LD_DIR, "Downloading consensus from %s using %s",
-               hoststring, url);
-      break;
-    case DIR_PURPOSE_FETCH_CERTIFICATE:
-      tor_assert(resource);
-      tor_assert(!payload);
-      httpcommand = "GET";
-      tor_asprintf(&url, "/tor/keys/%s", resource);
-      break;
-    case DIR_PURPOSE_FETCH_STATUS_VOTE:
-      tor_assert(resource);
-      tor_assert(!payload);
-      httpcommand = "GET";
-      tor_asprintf(&url, "/tor/status-vote/next/%s.z", resource);
-      break;
-    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
-      tor_assert(!resource);
-      tor_assert(!payload);
-      httpcommand = "GET";
-      url = tor_strdup("/tor/status-vote/next/consensus-signatures.z");
-      break;
-    case DIR_PURPOSE_FETCH_SERVERDESC:
-      tor_assert(resource);
-      httpcommand = "GET";
-      tor_asprintf(&url, "/tor/server/%s", resource);
-      break;
-    case DIR_PURPOSE_FETCH_EXTRAINFO:
-      tor_assert(resource);
-      httpcommand = "GET";
-      tor_asprintf(&url, "/tor/extra/%s", resource);
-      break;
-    case DIR_PURPOSE_FETCH_MICRODESC:
-      tor_assert(resource);
-      httpcommand = "GET";
-      tor_asprintf(&url, "/tor/micro/%s", resource);
-      break;
-    case DIR_PURPOSE_UPLOAD_DIR: {
-      const char *why = router_get_descriptor_gen_reason();
-      tor_assert(!resource);
-      tor_assert(payload);
-      httpcommand = "POST";
-      url = tor_strdup("/tor/");
-      if (why) {
-        smartlist_add_asprintf(headers, "X-Desc-Gen-Reason: %s\r\n", why);
-      }
-      break;
-    }
-    case DIR_PURPOSE_UPLOAD_VOTE:
-      tor_assert(!resource);
-      tor_assert(payload);
-      httpcommand = "POST";
-      url = tor_strdup("/tor/post/vote");
-      break;
-    case DIR_PURPOSE_UPLOAD_SIGNATURES:
-      tor_assert(!resource);
-      tor_assert(payload);
-      httpcommand = "POST";
-      url = tor_strdup("/tor/post/consensus-signature");
-      break;
-    case DIR_PURPOSE_FETCH_RENDDESC_V2:
-      tor_assert(resource);
-      tor_assert(strlen(resource) <= REND_DESC_ID_V2_LEN_BASE32);
-      tor_assert(!payload);
-      httpcommand = "GET";
-      tor_asprintf(&url, "/tor/rendezvous2/%s", resource);
-      break;
-    case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
-      tor_assert(!resource);
-      tor_assert(payload);
-      httpcommand = "POST";
-      url = tor_strdup("/tor/rendezvous2/publish");
-      break;
-    default:
-      tor_assert(0);
-      return;
-  }
-
-  /* warn in the non-tunneled case */
-  if (direct && (strlen(proxystring) + strlen(url) >= 4096)) {
-    log_warn(LD_BUG,
-             "Squid does not like URLs longer than 4095 bytes, and this "
-             "one is %d bytes long: %s%s",
-             (int)(strlen(proxystring) + strlen(url)), proxystring, url);
-  }
-
-  tor_snprintf(request, sizeof(request), "%s %s", httpcommand, proxystring);
-  connection_write_to_buf(request, strlen(request), TO_CONN(conn));
-  connection_write_to_buf(url, strlen(url), TO_CONN(conn));
-  tor_free(url);
-
-  if (!strcmp(httpcommand, "POST") || payload) {
-    smartlist_add_asprintf(headers, "Content-Length: %lu\r\n",
-                 payload ? (unsigned long)payload_len : 0);
-  }
-
-  {
-    char *header = smartlist_join_strings(headers, "", 0, NULL);
-    tor_snprintf(request, sizeof(request), " HTTP/1.0\r\nHost: %s\r\n%s\r\n",
-                 hoststring, header);
-    tor_free(header);
-  }
-
-  connection_write_to_buf(request, strlen(request), TO_CONN(conn));
-
-  if (payload) {
-    /* then send the payload afterwards too */
-    connection_write_to_buf(payload, payload_len, TO_CONN(conn));
-  }
-
-  SMARTLIST_FOREACH(headers, char *, h, tor_free(h));
-  smartlist_free(headers);
-}
-
-/** Parse an HTTP request string <b>headers</b> of the form
- * \verbatim
- * "\%s [http[s]://]\%s HTTP/1..."
- * \endverbatim
- * If it's well-formed, strdup the second \%s into *<b>url</b>, and
- * nul-terminate it. If the url doesn't start with "/tor/", rewrite it
- * so it does. Return 0.
- * Otherwise, return -1.
- */
-STATIC int
-parse_http_url(const char *headers, char **url)
-{
-  char *s, *start, *tmp;
-
-  s = (char *)eat_whitespace_no_nl(headers);
-  if (!*s) return -1;
-  s = (char *)find_whitespace(s); /* get past GET/POST */
-  if (!*s) return -1;
-  s = (char *)eat_whitespace_no_nl(s);
-  if (!*s) return -1;
-  start = s; /* this is it, assuming it's valid */
-  s = (char *)find_whitespace(start);
-  if (!*s) return -1;
-
-  /* tolerate the http[s] proxy style of putting the hostname in the url */
-  if (s-start >= 4 && !strcmpstart(start,"http")) {
-    tmp = start + 4;
-    if (*tmp == 's')
-      tmp++;
-    if (s-tmp >= 3 && !strcmpstart(tmp,"://")) {
-      tmp = strchr(tmp+3, '/');
-      if (tmp && tmp < s) {
-        log_debug(LD_DIR,"Skipping over 'http[s]://hostname/' string");
-        start = tmp;
-      }
-    }
-  }
-
-  /* Check if the header is well formed (next sequence
-   * should be HTTP/1.X\r\n). Assumes we're supporting 1.0? */
-  {
-    unsigned minor_ver;
-    char ch;
-    char *e = (char *)eat_whitespace_no_nl(s);
-    if (2 != tor_sscanf(e, "HTTP/1.%u%c", &minor_ver, &ch)) {
-      return -1;
-    }
-    if (ch != '\r')
-      return -1;
-  }
-
-  if (s-start < 5 || strcmpstart(start,"/tor/")) { /* need to rewrite it */
-    *url = tor_malloc(s - start + 5);
-    strlcpy(*url,"/tor", s-start+5);
-    strlcat((*url)+4, start, s-start+1);
-  } else {
-    *url = tor_strndup(start, s-start);
-  }
-  return 0;
-}
-
-/** Return a copy of the first HTTP header in <b>headers</b> whose key is
- * <b>which</b>.  The key should be given with a terminating colon and space;
- * this function copies everything after, up to but not including the
- * following \\r\\n. */
-static char *
-http_get_header(const char *headers, const char *which)
-{
-  const char *cp = headers;
-  while (cp) {
-    if (!strcasecmpstart(cp, which)) {
-      char *eos;
-      cp += strlen(which);
-      if ((eos = strchr(cp,'\r')))
-        return tor_strndup(cp, eos-cp);
-      else
-        return tor_strdup(cp);
-    }
-    cp = strchr(cp, '\n');
-    if (cp)
-      ++cp;
-  }
-  return NULL;
-}
-
-/** If <b>headers</b> indicates that a proxy was involved, then rewrite
- * <b>conn</b>-\>address to describe our best guess of the address that
- * originated this HTTP request. */
-static void
-http_set_address_origin(const char *headers, connection_t *conn)
-{
-  char *fwd;
-
-  fwd = http_get_header(headers, "Forwarded-For: ");
-  if (!fwd)
-    fwd = http_get_header(headers, "X-Forwarded-For: ");
-  if (fwd) {
-    tor_addr_t toraddr;
-    if (tor_addr_parse(&toraddr,fwd) == -1 ||
-        tor_addr_is_internal(&toraddr,0)) {
-      log_debug(LD_DIR, "Ignoring local/internal IP %s", escaped(fwd));
-      tor_free(fwd);
-      return;
-    }
-
-    tor_free(conn->address);
-    conn->address = tor_strdup(fwd);
-    tor_free(fwd);
-  }
-}
-
-/** Parse an HTTP response string <b>headers</b> of the form
- * \verbatim
- * "HTTP/1.\%d \%d\%s\r\n...".
- * \endverbatim
- *
- * If it's well-formed, assign the status code to *<b>code</b> and
- * return 0.  Otherwise, return -1.
- *
- * On success: If <b>date</b> is provided, set *date to the Date
- * header in the http headers, or 0 if no such header is found.  If
- * <b>compression</b> is provided, set *<b>compression</b> to the
- * compression method given in the Content-Encoding header, or 0 if no
- * such header is found, or -1 if the value of the header is not
- * recognized.  If <b>reason</b> is provided, strdup the reason string
- * into it.
- */
-int
-parse_http_response(const char *headers, int *code, time_t *date,
-                    compress_method_t *compression, char **reason)
-{
-  unsigned n1, n2;
-  char datestr[RFC1123_TIME_LEN+1];
-  smartlist_t *parsed_headers;
-  tor_assert(headers);
-  tor_assert(code);
-
-  while (TOR_ISSPACE(*headers)) headers++; /* tolerate leading whitespace */
-
-  if (tor_sscanf(headers, "HTTP/1.%u %u", &n1, &n2) < 2 ||
-      (n1 != 0 && n1 != 1) ||
-      (n2 < 100 || n2 >= 600)) {
-    log_warn(LD_HTTP,"Failed to parse header %s",escaped(headers));
-    return -1;
-  }
-  *code = n2;
-
-  parsed_headers = smartlist_new();
-  smartlist_split_string(parsed_headers, headers, "\n",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
-  if (reason) {
-    smartlist_t *status_line_elements = smartlist_new();
-    tor_assert(smartlist_len(parsed_headers));
-    smartlist_split_string(status_line_elements,
-                           smartlist_get(parsed_headers, 0),
-                           " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3);
-    tor_assert(smartlist_len(status_line_elements) <= 3);
-    if (smartlist_len(status_line_elements) == 3) {
-      *reason = smartlist_get(status_line_elements, 2);
-      smartlist_set(status_line_elements, 2, NULL); /* Prevent free */
-    }
-    SMARTLIST_FOREACH(status_line_elements, char *, cp, tor_free(cp));
-    smartlist_free(status_line_elements);
-  }
-  if (date) {
-    *date = 0;
-    SMARTLIST_FOREACH(parsed_headers, const char *, s,
-      if (!strcmpstart(s, "Date: ")) {
-        strlcpy(datestr, s+6, sizeof(datestr));
-        /* This will do nothing on failure, so we don't need to check
-           the result.   We shouldn't warn, since there are many other valid
-           date formats besides the one we use. */
-        parse_rfc1123_time(datestr, date);
-        break;
-      });
-  }
-  if (compression) {
-    const char *enc = NULL;
-    SMARTLIST_FOREACH(parsed_headers, const char *, s,
-      if (!strcmpstart(s, "Content-Encoding: ")) {
-        enc = s+18; break;
-      });
-    if (!enc || !strcmp(enc, "identity")) {
-      *compression = NO_METHOD;
-    } else if (!strcmp(enc, "deflate") || !strcmp(enc, "x-deflate")) {
-      *compression = ZLIB_METHOD;
-    } else if (!strcmp(enc, "gzip") || !strcmp(enc, "x-gzip")) {
-      *compression = GZIP_METHOD;
-    } else {
-      log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.",
-               escaped(enc));
-      *compression = UNKNOWN_METHOD;
-    }
-  }
-  SMARTLIST_FOREACH(parsed_headers, char *, s, tor_free(s));
-  smartlist_free(parsed_headers);
-
-  return 0;
-}
-
-/** Return true iff <b>body</b> doesn't start with a plausible router or
- * network-status or microdescriptor opening.  This is a sign of possible
- * compression. */
-static int
-body_is_plausible(const char *body, size_t len, int purpose)
-{
-  int i;
-  if (len == 0)
-    return 1; /* empty bodies don't need decompression */
-  if (len < 32)
-    return 0;
-  if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
-    return (!strcmpstart(body,"onion-key"));
-  }
-  if (1) {
-    if (!strcmpstart(body,"router") ||
-        !strcmpstart(body,"network-status"))
-      return 1;
-    for (i=0;i<32;++i) {
-      if (!TOR_ISPRINT(body[i]) && !TOR_ISSPACE(body[i]))
-        return 0;
-    }
-  }
-  return 1;
-}
-
-/** Called when we've just fetched a bunch of router descriptors in
- * <b>body</b>.  The list <b>which</b>, if present, holds digests for
- * descriptors we requested: descriptor digests if <b>descriptor_digests</b>
- * is true, or identity digests otherwise.  Parse the descriptors, validate
- * them, and annotate them as having purpose <b>purpose</b> and as having been
- * downloaded from <b>source</b>.
- *
- * Return the number of routers actually added. */
-static int
-load_downloaded_routers(const char *body, smartlist_t *which,
-                        int descriptor_digests,
-                        int router_purpose,
-                        const char *source)
-{
-  char buf[256];
-  char time_buf[ISO_TIME_LEN+1];
-  int added = 0;
-  int general = router_purpose == ROUTER_PURPOSE_GENERAL;
-  format_iso_time(time_buf, time(NULL));
-  tor_assert(source);
-
-  if (tor_snprintf(buf, sizeof(buf),
-                   "@downloaded-at %s\n"
-                   "@source %s\n"
-                   "%s%s%s", time_buf, escaped(source),
-                   !general ? "@purpose " : "",
-                   !general ? router_purpose_to_string(router_purpose) : "",
-                   !general ? "\n" : "")<0)
-    return added;
-
-  added = router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which,
-                                  descriptor_digests, buf);
-  if (added && general)
-    control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS,
-                            count_loading_descriptors_progress());
-  return added;
-}
-
-/** We are a client, and we've finished reading the server's
- * response. Parse it and act appropriately.
- *
- * If we're still happy with using this directory server in the future, return
- * 0. Otherwise return -1; and the caller should consider trying the request
- * again.
- *
- * The caller will take care of marking the connection for close.
- */
-static int
-connection_dir_client_reached_eof(dir_connection_t *conn)
-{
-  char *body;
-  char *headers;
-  char *reason = NULL;
-  size_t body_len = 0;
-  int status_code;
-  time_t date_header = 0;
-  long apparent_skew;
-  compress_method_t compression;
-  int plausible;
-  int skewed = 0;
-  int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
-                       conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
-                       conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
-  time_t now = time(NULL);
-  int src_code;
-
-  switch (connection_fetch_from_buf_http(TO_CONN(conn),
-                              &headers, MAX_HEADERS_SIZE,
-                              &body, &body_len, MAX_DIR_DL_SIZE,
-                              allow_partial)) {
-    case -1: /* overflow */
-      log_warn(LD_PROTOCOL,
-               "'fetch' response too large (server '%s:%d'). Closing.",
-               conn->base_.address, conn->base_.port);
-      return -1;
-    case 0:
-      log_info(LD_HTTP,
-               "'fetch' response not all here, but we're at eof. Closing.");
-      return -1;
-    /* case 1, fall through */
-  }
-
-  if (parse_http_response(headers, &status_code, &date_header,
-                          &compression, &reason) < 0) {
-    log_warn(LD_HTTP,"Unparseable headers (server '%s:%d'). Closing.",
-             conn->base_.address, conn->base_.port);
-    tor_free(body); tor_free(headers);
-    return -1;
-  }
-  if (!reason) reason = tor_strdup("[no reason given]");
-
-  log_debug(LD_DIR,
-            "Received response from directory server '%s:%d': %d %s "
-            "(purpose: %d)",
-            conn->base_.address, conn->base_.port, status_code,
-            escaped(reason),
-            conn->base_.purpose);
-
-  /* now check if it's got any hints for us about our IP address. */
-  if (conn->dirconn_direct) {
-    char *guess = http_get_header(headers, X_ADDRESS_HEADER);
-    if (guess) {
-      router_new_address_suggestion(guess, conn);
-      tor_free(guess);
-    }
-  }
-
-  if (date_header > 0) {
-    /* The date header was written very soon after we sent our request,
-     * so compute the skew as the difference between sending the request
-     * and the date header.  (We used to check now-date_header, but that's
-     * inaccurate if we spend a lot of time downloading.)
-     */
-    apparent_skew = conn->base_.timestamp_lastwritten - date_header;
-    if (labs(apparent_skew)>ALLOW_DIRECTORY_TIME_SKEW) {
-      int trusted = router_digest_is_trusted_dir(conn->identity_digest);
-      clock_skew_warning(TO_CONN(conn), apparent_skew, trusted, LD_HTTP,
-                         "directory", "DIRSERV");
-      skewed = 1; /* don't check the recommended-versions line */
-    } else {
-      log_debug(LD_HTTP, "Time on received directory is within tolerance; "
-                "we are %ld seconds skewed.  (That's okay.)", apparent_skew);
-    }
-  }
-  (void) skewed; /* skewed isn't used yet. */
-
-  if (status_code == 503) {
-    routerstatus_t *rs;
-    dir_server_t *ds;
-    const char *id_digest = conn->identity_digest;
-    log_info(LD_DIR,"Received http status code %d (%s) from server "
-             "'%s:%d'. I'll try again soon.",
-             status_code, escaped(reason), conn->base_.address,
-             conn->base_.port);
-    if ((rs = router_get_mutable_consensus_status_by_id(id_digest)))
-      rs->last_dir_503_at = now;
-    if ((ds = router_get_fallback_dirserver_by_digest(id_digest)))
-      ds->fake_status.last_dir_503_at = now;
-
-    tor_free(body); tor_free(headers); tor_free(reason);
-    return -1;
-  }
-
-  plausible = body_is_plausible(body, body_len, conn->base_.purpose);
-  if (compression != NO_METHOD || !plausible) {
-    char *new_body = NULL;
-    size_t new_len = 0;
-    compress_method_t guessed = detect_compression_method(body, body_len);
-    if (compression == UNKNOWN_METHOD || guessed != compression) {
-      /* Tell the user if we don't believe what we're told about compression.*/
-      const char *description1, *description2;
-      if (compression == ZLIB_METHOD)
-        description1 = "as deflated";
-      else if (compression == GZIP_METHOD)
-        description1 = "as gzipped";
-      else if (compression == NO_METHOD)
-        description1 = "as uncompressed";
-      else
-        description1 = "with an unknown Content-Encoding";
-      if (guessed == ZLIB_METHOD)
-        description2 = "deflated";
-      else if (guessed == GZIP_METHOD)
-        description2 = "gzipped";
-      else if (!plausible)
-        description2 = "confusing binary junk";
-      else
-        description2 = "uncompressed";
-
-      log_info(LD_HTTP, "HTTP body from server '%s:%d' was labeled %s, "
-               "but it seems to be %s.%s",
-               conn->base_.address, conn->base_.port, description1,
-               description2,
-               (compression>0 && guessed>0)?"  Trying both.":"");
-    }
-    /* Try declared compression first if we can. */
-    if (compression == GZIP_METHOD  || compression == ZLIB_METHOD)
-      tor_gzip_uncompress(&new_body, &new_len, body, body_len, compression,
-                          !allow_partial, LOG_PROTOCOL_WARN);
-    /* Okay, if that didn't work, and we think that it was compressed
-     * differently, try that. */
-    if (!new_body &&
-        (guessed == GZIP_METHOD || guessed == ZLIB_METHOD) &&
-        compression != guessed)
-      tor_gzip_uncompress(&new_body, &new_len, body, body_len, guessed,
-                          !allow_partial, LOG_PROTOCOL_WARN);
-    /* If we're pretty sure that we have a compressed directory, and
-     * we didn't manage to uncompress it, then warn and bail. */
-    if (!plausible && !new_body) {
-      log_fn(LOG_PROTOCOL_WARN, LD_HTTP,
-             "Unable to decompress HTTP body (server '%s:%d').",
-             conn->base_.address, conn->base_.port);
-      tor_free(body); tor_free(headers); tor_free(reason);
-      return -1;
-    }
-    if (new_body) {
-      tor_free(body);
-      body = new_body;
-      body_len = new_len;
-    }
-  }
-
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
-    int r;
-    const char *flavname = conn->requested_resource;
-    if (status_code != 200) {
-      int severity = (status_code == 304) ? LOG_INFO : LOG_WARN;
-      tor_log(severity, LD_DIR,
-          "Received http status code %d (%s) from server "
-          "'%s:%d' while fetching consensus directory.",
-           status_code, escaped(reason), conn->base_.address,
-           conn->base_.port);
-      tor_free(body); tor_free(headers); tor_free(reason);
-      networkstatus_consensus_download_failed(status_code, flavname);
-      return -1;
-    }
-    log_info(LD_DIR,"Received consensus directory (size %d) from server "
-             "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port);
-    if ((r=networkstatus_set_current_consensus(body, flavname, 0,
-                                               conn->identity_digest))<0) {
-      log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
-             "Unable to load %s consensus directory downloaded from "
-             "server '%s:%d'. I'll try again soon.",
-             flavname, conn->base_.address, conn->base_.port);
-      tor_free(body); tor_free(headers); tor_free(reason);
-      networkstatus_consensus_download_failed(0, flavname);
-      return -1;
-    }
-
-    /* If we launched other fetches for this consensus, cancel them. */
-    connection_dir_close_consensus_fetches(conn, flavname);
-
-    /* launches router downloads as needed */
-    routers_update_all_from_networkstatus(now, 3);
-    update_microdescs_from_networkstatus(now);
-    update_microdesc_downloads(now);
-    directory_info_has_arrived(now, 0, 0);
-    if (authdir_mode_v3(get_options())) {
-      sr_act_post_consensus(
-                   networkstatus_get_latest_consensus_by_flavor(FLAV_NS));
-    }
-    log_info(LD_DIR, "Successfully loaded consensus.");
-  }
-
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) {
-    if (status_code != 200) {
-      log_warn(LD_DIR,
-          "Received http status code %d (%s) from server "
-          "'%s:%d' while fetching \"/tor/keys/%s\".",
-           status_code, escaped(reason), conn->base_.address,
-           conn->base_.port, conn->requested_resource);
-      connection_dir_download_cert_failed(conn, status_code);
-      tor_free(body); tor_free(headers); tor_free(reason);
-      return -1;
-    }
-    log_info(LD_DIR,"Received authority certificates (size %d) from server "
-             "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port);
-
-    /*
-     * Tell trusted_dirs_load_certs_from_string() whether it was by fp
-     * or fp-sk pair.
-     */
-    src_code = -1;
-    if (!strcmpstart(conn->requested_resource, "fp/")) {
-      src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST;
-    } else if (!strcmpstart(conn->requested_resource, "fp-sk/")) {
-      src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST;
-    }
-
-    if (src_code != -1) {
-      if (trusted_dirs_load_certs_from_string(body, src_code, 1,
-                                              conn->identity_digest)<0) {
-        log_warn(LD_DIR, "Unable to parse fetched certificates");
-        /* if we fetched more than one and only some failed, the successful
-         * ones got flushed to disk so it's safe to call this on them */
-        connection_dir_download_cert_failed(conn, status_code);
-      } else {
-        directory_info_has_arrived(now, 0, 0);
-        log_info(LD_DIR, "Successfully loaded certificates from fetch.");
-      }
-    } else {
-      log_warn(LD_DIR,
-               "Couldn't figure out what to do with fetched certificates for "
-               "unknown resource %s",
-               conn->requested_resource);
-      connection_dir_download_cert_failed(conn, status_code);
-    }
-  }
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) {
-    const char *msg;
-    int st;
-    log_info(LD_DIR,"Got votes (size %d) from server %s:%d",
-             (int)body_len, conn->base_.address, conn->base_.port);
-    if (status_code != 200) {
-      log_warn(LD_DIR,
-             "Received http status code %d (%s) from server "
-             "'%s:%d' while fetching \"/tor/status-vote/next/%s.z\".",
-             status_code, escaped(reason), conn->base_.address,
-             conn->base_.port, conn->requested_resource);
-      tor_free(body); tor_free(headers); tor_free(reason);
-      return -1;
-    }
-    dirvote_add_vote(body, &msg, &st);
-    if (st > 299) {
-      log_warn(LD_DIR, "Error adding retrieved vote: %s", msg);
-    } else {
-      log_info(LD_DIR, "Added vote(s) successfully [msg: %s]", msg);
-    }
-  }
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) {
-    const char *msg = NULL;
-    log_info(LD_DIR,"Got detached signatures (size %d) from server %s:%d",
-             (int)body_len, conn->base_.address, conn->base_.port);
-    if (status_code != 200) {
-      log_warn(LD_DIR,
-        "Received http status code %d (%s) from server '%s:%d' while fetching "
-        "\"/tor/status-vote/next/consensus-signatures.z\".",
-             status_code, escaped(reason), conn->base_.address,
-             conn->base_.port);
-      tor_free(body); tor_free(headers); tor_free(reason);
-      return -1;
-    }
-    if (dirvote_add_signatures(body, conn->base_.address, &msg)<0) {
-      log_warn(LD_DIR, "Problem adding detached signatures from %s:%d: %s",
-               conn->base_.address, conn->base_.port, msg?msg:"???");
-    }
-  }
-
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
-      conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) {
-    int was_ei = conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO;
-    smartlist_t *which = NULL;
-    int n_asked_for = 0;
-    int descriptor_digests = conn->requested_resource &&
-                             !strcmpstart(conn->requested_resource,"d/");
-    log_info(LD_DIR,"Received %s (size %d) from server '%s:%d'",
-             was_ei ? "extra server info" : "server info",
-             (int)body_len, conn->base_.address, conn->base_.port);
-    if (conn->requested_resource &&
-        (!strcmpstart(conn->requested_resource,"d/") ||
-         !strcmpstart(conn->requested_resource,"fp/"))) {
-      which = smartlist_new();
-      dir_split_resource_into_fingerprints(conn->requested_resource +
-                                             (descriptor_digests ? 2 : 3),
-                                           which, NULL, 0);
-      n_asked_for = smartlist_len(which);
-    }
-    if (status_code != 200) {
-      int dir_okay = status_code == 404 ||
-        (status_code == 400 && !strcmp(reason, "Servers unavailable."));
-      /* 404 means that it didn't have them; no big deal.
-       * Older (pre-0.1.1.8) servers said 400 Servers unavailable instead. */
-      log_fn(dir_okay ? LOG_INFO : LOG_WARN, LD_DIR,
-             "Received http status code %d (%s) from server '%s:%d' "
-             "while fetching \"/tor/server/%s\". I'll try again soon.",
-             status_code, escaped(reason), conn->base_.address,
-             conn->base_.port, conn->requested_resource);
-      if (!which) {
-        connection_dir_download_routerdesc_failed(conn);
-      } else {
-        dir_routerdesc_download_failed(which, status_code,
-                                       conn->router_purpose,
-                                       was_ei, descriptor_digests);
-        SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
-        smartlist_free(which);
-      }
-      tor_free(body); tor_free(headers); tor_free(reason);
-      return dir_okay ? 0 : -1;
-    }
-    /* Learn the routers, assuming we requested by fingerprint or "all"
-     * or "authority".
-     *
-     * We use "authority" to fetch our own descriptor for
-     * testing, and to fetch bridge descriptors for bootstrapping. Ignore
-     * the output of "authority" requests unless we are using bridges,
-     * since otherwise they'll be the response from reachability tests,
-     * and we don't really want to add that to our routerlist. */
-    if (which || (conn->requested_resource &&
-                  (!strcmpstart(conn->requested_resource, "all") ||
-                   (!strcmpstart(conn->requested_resource, "authority") &&
-                    get_options()->UseBridges)))) {
-      /* as we learn from them, we remove them from 'which' */
-      if (was_ei) {
-        router_load_extrainfo_from_string(body, NULL, SAVED_NOWHERE, which,
-                                          descriptor_digests);
-      } else {
-        //router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which,
-        //                       descriptor_digests, conn->router_purpose);
-        if (load_downloaded_routers(body, which, descriptor_digests,
-                                conn->router_purpose,
-                                conn->base_.address))
-          directory_info_has_arrived(now, 0, 0);
-      }
-    }
-    if (which) { /* mark remaining ones as failed */
-      log_info(LD_DIR, "Received %d/%d %s requested from %s:%d",
-               n_asked_for-smartlist_len(which), n_asked_for,
-               was_ei ? "extra-info documents" : "router descriptors",
-               conn->base_.address, (int)conn->base_.port);
-      if (smartlist_len(which)) {
-        dir_routerdesc_download_failed(which, status_code,
-                                       conn->router_purpose,
-                                       was_ei, descriptor_digests);
-      }
-      SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
-      smartlist_free(which);
-    }
-    if (directory_conn_is_self_reachability_test(conn))
-      router_dirport_found_reachable();
-  }
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC) {
-    smartlist_t *which = NULL;
-    log_info(LD_DIR,"Received answer to microdescriptor request (status %d, "
-             "size %d) from server '%s:%d'",
-             status_code, (int)body_len, conn->base_.address,
-             conn->base_.port);
-    tor_assert(conn->requested_resource &&
-               !strcmpstart(conn->requested_resource, "d/"));
-    which = smartlist_new();
-    dir_split_resource_into_fingerprints(conn->requested_resource+2,
-                                         which, NULL,
-                                         DSR_DIGEST256|DSR_BASE64);
-    if (status_code != 200) {
-      log_info(LD_DIR, "Received status code %d (%s) from server "
-               "'%s:%d' while fetching \"/tor/micro/%s\".  I'll try again "
-               "soon.",
-               status_code, escaped(reason), conn->base_.address,
-               (int)conn->base_.port, conn->requested_resource);
-      dir_microdesc_download_failed(which, status_code);
-      SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
-      smartlist_free(which);
-      tor_free(body); tor_free(headers); tor_free(reason);
-      return 0;
-    } else {
-      smartlist_t *mds;
-      mds = microdescs_add_to_cache(get_microdesc_cache(),
-                                    body, body+body_len, SAVED_NOWHERE, 0,
-                                    now, which);
-      if (smartlist_len(which)) {
-        /* Mark remaining ones as failed. */
-        dir_microdesc_download_failed(which, status_code);
-      }
-      if (mds && smartlist_len(mds)) {
-        control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS,
-                                count_loading_descriptors_progress());
-        directory_info_has_arrived(now, 0, 1);
-      }
-      SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
-      smartlist_free(which);
-      smartlist_free(mds);
-    }
-  }
-
-  if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_DIR) {
-    switch (status_code) {
-      case 200: {
-          dir_server_t *ds =
-            router_get_trusteddirserver_by_digest(conn->identity_digest);
-          char *rejected_hdr = http_get_header(headers,
-                                               "X-Descriptor-Not-New: ");
-          if (rejected_hdr) {
-            if (!strcmp(rejected_hdr, "Yes")) {
-              log_info(LD_GENERAL,
-                       "Authority '%s' declined our descriptor (not new)",
-                       ds->nickname);
-              /* XXXX use this information; be sure to upload next one
-               * sooner. -NM */
-              /* XXXX++ On further thought, the task above implies that we're
-               * basing our regenerate-descriptor time on when we uploaded the
-               * last descriptor, not on the published time of the last
-               * descriptor.  If those are different, that's a bad thing to
-               * do. -NM */
-            }
-            tor_free(rejected_hdr);
-          }
-          log_info(LD_GENERAL,"eof (status 200) after uploading server "
-                   "descriptor: finished.");
-          control_event_server_status(
-                      LOG_NOTICE, "ACCEPTED_SERVER_DESCRIPTOR DIRAUTH=%s:%d",
-                      conn->base_.address, conn->base_.port);
-
-          ds->has_accepted_serverdesc = 1;
-          if (directories_have_accepted_server_descriptor())
-            control_event_server_status(LOG_NOTICE, "GOOD_SERVER_DESCRIPTOR");
-        }
-        break;
-      case 400:
-        log_warn(LD_GENERAL,"http status 400 (%s) response from "
-                 "dirserver '%s:%d'. Please correct.",
-                 escaped(reason), conn->base_.address, conn->base_.port);
-        control_event_server_status(LOG_WARN,
-                      "BAD_SERVER_DESCRIPTOR DIRAUTH=%s:%d REASON=\"%s\"",
-                      conn->base_.address, conn->base_.port, escaped(reason));
-        break;
-      default:
-        log_warn(LD_GENERAL,
-             "http status %d (%s) reason unexpected while uploading "
-             "descriptor to server '%s:%d').",
-             status_code, escaped(reason), conn->base_.address,
-             conn->base_.port);
-        break;
-    }
-    /* return 0 in all cases, since we don't want to mark any
-     * dirservers down just because they don't like us. */
-  }
-
-  if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_VOTE) {
-    switch (status_code) {
-      case 200: {
-        log_notice(LD_DIR,"Uploaded a vote to dirserver %s:%d",
-                   conn->base_.address, conn->base_.port);
-        }
-        break;
-      case 400:
-        log_warn(LD_DIR,"http status 400 (%s) response after uploading "
-                 "vote to dirserver '%s:%d'. Please correct.",
-                 escaped(reason), conn->base_.address, conn->base_.port);
-        break;
-      default:
-        log_warn(LD_GENERAL,
-             "http status %d (%s) reason unexpected while uploading "
-             "vote to server '%s:%d').",
-             status_code, escaped(reason), conn->base_.address,
-             conn->base_.port);
-        break;
-    }
-    /* return 0 in all cases, since we don't want to mark any
-     * dirservers down just because they don't like us. */
-  }
-
-  if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_SIGNATURES) {
-    switch (status_code) {
-      case 200: {
-        log_notice(LD_DIR,"Uploaded signature(s) to dirserver %s:%d",
-                   conn->base_.address, conn->base_.port);
-        }
-        break;
-      case 400:
-        log_warn(LD_DIR,"http status 400 (%s) response after uploading "
-                 "signatures to dirserver '%s:%d'. Please correct.",
-                 escaped(reason), conn->base_.address, conn->base_.port);
-        break;
-      default:
-        log_warn(LD_GENERAL,
-             "http status %d (%s) reason unexpected while uploading "
-             "signatures to server '%s:%d').",
-             status_code, escaped(reason), conn->base_.address,
-             conn->base_.port);
-        break;
-    }
-    /* return 0 in all cases, since we don't want to mark any
-     * dirservers down just because they don't like us. */
-  }
-
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2) {
-    #define SEND_HS_DESC_FAILED_EVENT(reason) ( \
-      control_event_hs_descriptor_failed(conn->rend_data, \
-                                         conn->identity_digest, \
-                                         reason) )
-    #define SEND_HS_DESC_FAILED_CONTENT() ( \
-      control_event_hs_descriptor_content(conn->rend_data->onion_address, \
-                                          conn->requested_resource, \
-                                          conn->identity_digest, \
-                                          NULL) )
-    tor_assert(conn->rend_data);
-    log_info(LD_REND,"Received rendezvous descriptor (size %d, status %d "
-             "(%s))",
-             (int)body_len, status_code, escaped(reason));
-    switch (status_code) {
-      case 200:
-      {
-        rend_cache_entry_t *entry = NULL;
-
-        if (rend_cache_store_v2_desc_as_client(body,
-            conn->requested_resource, conn->rend_data, &entry) < 0) {
-          log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. "
-                   "Retrying at another directory.");
-          /* We'll retry when connection_about_to_close_connection()
-           * cleans this dir conn up. */
-          SEND_HS_DESC_FAILED_EVENT("BAD_DESC");
-          SEND_HS_DESC_FAILED_CONTENT();
-        } else {
-          char service_id[REND_SERVICE_ID_LEN_BASE32 + 1];
-          /* Should never be NULL here if we found the descriptor. */
-          tor_assert(entry);
-          rend_get_service_id(entry->parsed->pk, service_id);
-
-          /* success. notify pending connections about this. */
-          log_info(LD_REND, "Successfully fetched v2 rendezvous "
-                   "descriptor.");
-          control_event_hs_descriptor_received(service_id,
-                                               conn->rend_data,
-                                               conn->identity_digest);
-          control_event_hs_descriptor_content(service_id,
-                                              conn->requested_resource,
-                                              conn->identity_digest,
-                                              body);
-          conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2;
-          rend_client_desc_trynow(service_id);
-          memwipe(service_id, 0, sizeof(service_id));
-        }
-        break;
-      }
-      case 404:
-        /* Not there. We'll retry when
-         * connection_about_to_close_connection() cleans this conn up. */
-        log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: "
-                         "Retrying at another directory.");
-        SEND_HS_DESC_FAILED_EVENT("NOT_FOUND");
-        SEND_HS_DESC_FAILED_CONTENT();
-        break;
-      case 400:
-        log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: "
-                 "http status 400 (%s). Dirserver didn't like our "
-                 "v2 rendezvous query? Retrying at another directory.",
-                 escaped(reason));
-        SEND_HS_DESC_FAILED_EVENT("QUERY_REJECTED");
-        SEND_HS_DESC_FAILED_CONTENT();
-        break;
-      default:
-        log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: "
-                 "http status %d (%s) response unexpected while "
-                 "fetching v2 hidden service descriptor (server '%s:%d'). "
-                 "Retrying at another directory.",
-                 status_code, escaped(reason), conn->base_.address,
-                 conn->base_.port);
-        SEND_HS_DESC_FAILED_EVENT("UNEXPECTED");
-        SEND_HS_DESC_FAILED_CONTENT();
-        break;
-    }
-  }
-
-  if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) {
-    #define SEND_HS_DESC_UPLOAD_FAILED_EVENT(reason) ( \
-      control_event_hs_descriptor_upload_failed( \
-        conn->identity_digest, \
-        conn->rend_data->onion_address, \
-        reason) )
-    log_info(LD_REND,"Uploaded rendezvous descriptor (status %d "
-             "(%s))",
-             status_code, escaped(reason));
-    /* Without the rend data, we'll have a problem identifying what has been
-     * uploaded for which service. */
-    tor_assert(conn->rend_data);
-    switch (status_code) {
-      case 200:
-        log_info(LD_REND,
-                 "Uploading rendezvous descriptor: finished with status "
-                 "200 (%s)", escaped(reason));
-        control_event_hs_descriptor_uploaded(conn->identity_digest,
-                                             conn->rend_data->onion_address);
-        rend_service_desc_has_uploaded(conn->rend_data);
-        break;
-      case 400:
-        log_warn(LD_REND,"http status 400 (%s) response from dirserver "
-                 "'%s:%d'. Malformed rendezvous descriptor?",
-                 escaped(reason), conn->base_.address, conn->base_.port);
-        SEND_HS_DESC_UPLOAD_FAILED_EVENT("UPLOAD_REJECTED");
-        break;
-      default:
-        log_warn(LD_REND,"http status %d (%s) response unexpected (server "
-                 "'%s:%d').",
-                 status_code, escaped(reason), conn->base_.address,
-                 conn->base_.port);
-        SEND_HS_DESC_UPLOAD_FAILED_EVENT("UNEXPECTED");
-        break;
-    }
-  }
-  tor_free(body); tor_free(headers); tor_free(reason);
-  return 0;
-}
-
-/** Called when a directory connection reaches EOF. */
-int
-connection_dir_reached_eof(dir_connection_t *conn)
-{
-  int retval;
-  if (conn->base_.state != DIR_CONN_STATE_CLIENT_READING) {
-    log_info(LD_HTTP,"conn reached eof, not reading. [state=%d] Closing.",
-             conn->base_.state);
-    connection_close_immediate(TO_CONN(conn)); /* error: give up on flushing */
-    connection_mark_for_close(TO_CONN(conn));
-    return -1;
-  }
-
-  retval = connection_dir_client_reached_eof(conn);
-  if (retval == 0) /* success */
-    conn->base_.state = DIR_CONN_STATE_CLIENT_FINISHED;
-  connection_mark_for_close(TO_CONN(conn));
-  return retval;
-}
-
-/** If any directory object is arriving, and it's over 10MB large, we're
- * getting DoS'd.  (As of 0.1.2.x, raw directories are about 1MB, and we never
- * ask for more than 96 router descriptors at a time.)
- */
-#define MAX_DIRECTORY_OBJECT_SIZE (10*(1<<20))
-
-#define MAX_VOTE_DL_SIZE (MAX_DIRECTORY_OBJECT_SIZE * 5)
-
-/** Read handler for directory connections.  (That's connections <em>to</em>
- * directory servers and connections <em>at</em> directory servers.)
- */
-int
-connection_dir_process_inbuf(dir_connection_t *conn)
-{
-  size_t max_size;
-  tor_assert(conn);
-  tor_assert(conn->base_.type == CONN_TYPE_DIR);
-
-  /* Directory clients write, then read data until they receive EOF;
-   * directory servers read data until they get an HTTP command, then
-   * write their response (when it's finished flushing, they mark for
-   * close).
-   */
-
-  /* If we're on the dirserver side, look for a command. */
-  if (conn->base_.state == DIR_CONN_STATE_SERVER_COMMAND_WAIT) {
-    if (directory_handle_command(conn) < 0) {
-      connection_mark_for_close(TO_CONN(conn));
-      return -1;
-    }
-    return 0;
-  }
-
-  max_size =
-    (TO_CONN(conn)->purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) ?
-    MAX_VOTE_DL_SIZE : MAX_DIRECTORY_OBJECT_SIZE;
-
-  if (connection_get_inbuf_len(TO_CONN(conn)) > max_size) {
-    log_warn(LD_HTTP,
-             "Too much data received from directory connection (%s): "
-             "denial of service attempt, or you need to upgrade?",
-             conn->base_.address);
-    connection_mark_for_close(TO_CONN(conn));
-    return -1;
-  }
-
-  if (!conn->base_.inbuf_reached_eof)
-    log_debug(LD_HTTP,"Got data, not eof. Leaving on inbuf.");
-  return 0;
-}
-
-/** Called when we're about to finally unlink and free a directory connection:
- * perform necessary accounting and cleanup */
-void
-connection_dir_about_to_close(dir_connection_t *dir_conn)
-{
-  connection_t *conn = TO_CONN(dir_conn);
-
-  if (conn->state < DIR_CONN_STATE_CLIENT_FINISHED) {
-    /* It's a directory connection and connecting or fetching
-     * failed: forget about this router, and maybe try again. */
-    connection_dir_request_failed(dir_conn);
-  }
-  /* If we were trying to fetch a v2 rend desc and did not succeed,
-   * retry as needed. (If a fetch is successful, the connection state
-   * is changed to DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 to mark that
-   * refetching is unnecessary.) */
-  if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 &&
-      dir_conn->rend_data &&
-      strlen(dir_conn->rend_data->onion_address) == REND_SERVICE_ID_LEN_BASE32)
-    rend_client_refetch_v2_renddesc(dir_conn->rend_data);
-}
-
-/** Create an http response for the client <b>conn</b> out of
- * <b>status</b> and <b>reason_phrase</b>. Write it to <b>conn</b>.
- */
-static void
-write_http_status_line(dir_connection_t *conn, int status,
-                       const char *reason_phrase)
-{
-  char buf[256];
-  if (tor_snprintf(buf, sizeof(buf), "HTTP/1.0 %d %s\r\n\r\n",
-      status, reason_phrase ? reason_phrase : "OK") < 0) {
-    log_warn(LD_BUG,"status line too long.");
-    return;
-  }
-  log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase);
-  connection_write_to_buf(buf, strlen(buf), TO_CONN(conn));
-}
-
-/** Write the header for an HTTP/1.0 response onto <b>conn</b>-\>outbuf,
- * with <b>type</b> as the Content-Type.
- *
- * If <b>length</b> is nonnegative, it is the Content-Length.
- * If <b>encoding</b> is provided, it is the Content-Encoding.
- * If <b>cache_lifetime</b> is greater than 0, the content may be cached for
- * up to cache_lifetime seconds.  Otherwise, the content may not be cached. */
-static void
-write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
-                           const char *type, const char *encoding,
-                           const char *extra_headers,
-                           long cache_lifetime)
-{
-  char date[RFC1123_TIME_LEN+1];
-  char tmp[1024];
-  char *cp;
-  time_t now = time(NULL);
-
-  tor_assert(conn);
-
-  format_rfc1123_time(date, now);
-  cp = tmp;
-  tor_snprintf(cp, sizeof(tmp),
-               "HTTP/1.0 200 OK\r\nDate: %s\r\n",
-               date);
-  cp += strlen(tmp);
-  if (type) {
-    tor_snprintf(cp, sizeof(tmp)-(cp-tmp), "Content-Type: %s\r\n", type);
-    cp += strlen(cp);
-  }
-  if (!is_local_addr(&conn->base_.addr)) {
-    /* Don't report the source address for a nearby/private connection.
-     * Otherwise we tend to mis-report in cases where incoming ports are
-     * being forwarded to a Tor server running behind the firewall. */
-    tor_snprintf(cp, sizeof(tmp)-(cp-tmp),
-                 X_ADDRESS_HEADER "%s\r\n", conn->base_.address);
-    cp += strlen(cp);
-  }
-  if (encoding) {
-    tor_snprintf(cp, sizeof(tmp)-(cp-tmp),
-                 "Content-Encoding: %s\r\n", encoding);
-    cp += strlen(cp);
-  }
-  if (length >= 0) {
-    tor_snprintf(cp, sizeof(tmp)-(cp-tmp),
-                 "Content-Length: %ld\r\n", (long)length);
-    cp += strlen(cp);
-  }
-  if (cache_lifetime > 0) {
-    char expbuf[RFC1123_TIME_LEN+1];
-    format_rfc1123_time(expbuf, (time_t)(now + cache_lifetime));
-    /* We could say 'Cache-control: max-age=%d' here if we start doing
-     * http/1.1 */
-    tor_snprintf(cp, sizeof(tmp)-(cp-tmp),
-                 "Expires: %s\r\n", expbuf);
-    cp += strlen(cp);
-  } else if (cache_lifetime == 0) {
-    /* We could say 'Cache-control: no-cache' here if we start doing
-     * http/1.1 */
-    strlcpy(cp, "Pragma: no-cache\r\n", sizeof(tmp)-(cp-tmp));
-    cp += strlen(cp);
-  }
-  if (extra_headers) {
-    strlcpy(cp, extra_headers, sizeof(tmp)-(cp-tmp));
-    cp += strlen(cp);
-  }
-  if (sizeof(tmp)-(cp-tmp) > 3)
-    memcpy(cp, "\r\n", 3);
-  else
-    tor_assert(0);
-  connection_write_to_buf(tmp, strlen(tmp), TO_CONN(conn));
-}
-
-/** As write_http_response_header_impl, but sets encoding and content-typed
- * based on whether the response will be <b>compressed</b> or not. */
-static void
-write_http_response_header(dir_connection_t *conn, ssize_t length,
-                           int compressed, long cache_lifetime)
-{
-  write_http_response_header_impl(conn, length,
-                          compressed?"application/octet-stream":"text/plain",
-                          compressed?"deflate":"identity",
-                             NULL,
-                             cache_lifetime);
-}
-
-/** Decide whether a client would accept the consensus we have.
- *
- * Clients can say they only want a consensus if it's signed by more
- * than half the authorities in a list.  They pass this list in
- * the url as "...consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>".
- *
- * <b>fpr</b> may be an abbreviated fingerprint, i.e. only a left substring
- * of the full authority identity digest. (Only strings of even length,
- * i.e. encodings of full bytes, are handled correctly.  In the case
- * of an odd number of hex digits the last one is silently ignored.)
- *
- * Returns 1 if more than half of the requested authorities signed the
- * consensus, 0 otherwise.
- */
-int
-client_likes_consensus(networkstatus_t *v, const char *want_url)
-{
-  smartlist_t *want_authorities = smartlist_new();
-  int need_at_least;
-  int have = 0;
-
-  dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0);
-  need_at_least = smartlist_len(want_authorities)/2+1;
-  SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, d) {
-    char want_digest[DIGEST_LEN];
-    size_t want_len = strlen(d)/2;
-    if (want_len > DIGEST_LEN)
-      want_len = DIGEST_LEN;
-
-    if (base16_decode(want_digest, DIGEST_LEN, d, want_len*2)
-                      != (int) want_len) {
-      log_fn(LOG_PROTOCOL_WARN, LD_DIR,
-             "Failed to decode requested authority digest %s.", escaped(d));
-      continue;
-    };
-
-    SMARTLIST_FOREACH_BEGIN(v->voters, networkstatus_voter_info_t *, vi) {
-      if (smartlist_len(vi->sigs) &&
-          tor_memeq(vi->identity_digest, want_digest, want_len)) {
-        have++;
-        break;
-      };
-    } SMARTLIST_FOREACH_END(vi);
-
-    /* early exit, if we already have enough */
-    if (have >= need_at_least)
-      break;
-  } SMARTLIST_FOREACH_END(d);
-
-  SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d));
-  smartlist_free(want_authorities);
-  return (have >= need_at_least);
-}
-
-/** Return the compression level we should use for sending a compressed
- * response of size <b>n_bytes</b>. */
-STATIC zlib_compression_level_t
-choose_compression_level(ssize_t n_bytes)
-{
-  if (! have_been_under_memory_pressure()) {
-    return HIGH_COMPRESSION; /* we have plenty of RAM. */
-  } else if (n_bytes < 0) {
-    return HIGH_COMPRESSION; /* unknown; might be big. */
-  } else if (n_bytes < 1024) {
-    return LOW_COMPRESSION;
-  } else if (n_bytes < 2048) {
-    return MEDIUM_COMPRESSION;
-  } else {
-    return HIGH_COMPRESSION;
-  }
-}
-
-/** Information passed to handle a GET request. */
-typedef struct get_handler_args_t {
-  /** True if the client asked for compressed data. */
-  int compressed;
-  /** If nonzero, the time included an if-modified-since header with this
-   * value. */
-  time_t if_modified_since;
-  /** String containing the requested URL or resource. */
-  const char *url;
-  /** String containing the HTTP headers */
-  const char *headers;
-} get_handler_args_t;
-
-/** Entry for handling an HTTP GET request.
- *
- * This entry matches a request if "string" is equal to the requested
- * resource, or if "is_prefix" is true and "string" is a prefix of the
- * requested resource.
- *
- * The 'handler' function is called to handle the request.  It receives
- * an arguments structure, and must return 0 on success or -1 if we should
- * close the connection.
- **/
-typedef struct url_table_ent_s {
-  const char *string;
-  int is_prefix;
-  int (*handler)(dir_connection_t *conn, const get_handler_args_t *args);
-} url_table_ent_t;
-
-static int handle_get_frontpage(dir_connection_t *conn,
-                                const get_handler_args_t *args);
-static int handle_get_current_consensus(dir_connection_t *conn,
-                                const get_handler_args_t *args);
-static int handle_get_status_vote(dir_connection_t *conn,
-                                const get_handler_args_t *args);
-static int handle_get_microdesc(dir_connection_t *conn,
-                                const get_handler_args_t *args);
-static int handle_get_descriptor(dir_connection_t *conn,
-                                const get_handler_args_t *args);
-static int handle_get_keys(dir_connection_t *conn,
-                                const get_handler_args_t *args);
-static int handle_get_rendezvous2(dir_connection_t *conn,
-                                const get_handler_args_t *args);
-static int handle_get_robots(dir_connection_t *conn,
-                                const get_handler_args_t *args);
-static int handle_get_networkstatus_bridges(dir_connection_t *conn,
-                                const get_handler_args_t *args);
-
-/** Table for handling GET requests. */
-static const url_table_ent_t url_table[] = {
-  { "/tor/", 0, handle_get_frontpage },
-  { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus },
-  { "/tor/status-vote/current/", 1, handle_get_status_vote },
-  { "/tor/status-vote/next/", 1, handle_get_status_vote },
-  { "/tor/micro/d/", 1, handle_get_microdesc },
-  { "/tor/server/", 1, handle_get_descriptor },
-  { "/tor/extra/", 1, handle_get_descriptor },
-  { "/tor/keys/", 1, handle_get_keys },
-  { "/tor/rendezvous2/", 1, handle_get_rendezvous2 },
-  { "/tor/robots.txt", 0, handle_get_robots },
-  { "/tor/networkstatus-bridges", 0, handle_get_networkstatus_bridges },
-  { NULL, 0, NULL },
-};
-
-/** Helper function: called when a dirserver gets a complete HTTP GET
- * request.  Look for a request for a directory or for a rendezvous
- * service descriptor.  On finding one, write a response into
- * conn-\>outbuf.  If the request is unrecognized, send a 404.
- * Return 0 if we handled this successfully, or -1 if we need to close
- * the connection. */
-STATIC int
-directory_handle_command_get(dir_connection_t *conn, const char *headers,
-                             const char *req_body, size_t req_body_len)
-{
-  char *url, *url_mem, *header;
-  time_t if_modified_since = 0;
-  int compressed;
-  size_t url_len;
-
-  /* We ignore the body of a GET request. */
-  (void)req_body;
-  (void)req_body_len;
-
-  log_debug(LD_DIRSERV,"Received GET command.");
-
-  conn->base_.state = DIR_CONN_STATE_SERVER_WRITING;
-
-  if (parse_http_url(headers, &url) < 0) {
-    write_http_status_line(conn, 400, "Bad request");
-    return 0;
-  }
-  if ((header = http_get_header(headers, "If-Modified-Since: "))) {
-    struct tm tm;
-    if (parse_http_time(header, &tm) == 0) {
-      if (tor_timegm(&tm, &if_modified_since)<0) {
-        if_modified_since = 0;
-      } else {
-        log_debug(LD_DIRSERV, "If-Modified-Since is '%s'.", escaped(header));
-      }
-    }
-    /* The correct behavior on a malformed If-Modified-Since header is to
-     * act as if no If-Modified-Since header had been given. */
-    tor_free(header);
-  }
-  log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url));
-
-  url_mem = url;
-  url_len = strlen(url);
-  compressed = url_len > 2 && !strcmp(url+url_len-2, ".z");
-  if (compressed) {
-    url[url_len-2] = '\0';
-    url_len -= 2;
-  }
-
-  get_handler_args_t args;
-  args.url = url;
-  args.headers = headers;
-  args.if_modified_since = if_modified_since;
-  args.compressed = compressed;
-
-  int i, result = -1;
-  for (i = 0; url_table[i].string; ++i) {
-    int match;
-    if (url_table[i].is_prefix) {
-      match = !strcmpstart(url, url_table[i].string);
-    } else {
-      match = !strcmp(url, url_table[i].string);
-    }
-    if (match) {
-      result = url_table[i].handler(conn, &args);
-      goto done;
-    }
-  }
-
-  /* we didn't recognize the url */
-  write_http_status_line(conn, 404, "Not found");
-  result = 0;
-
- done:
-  tor_free(url_mem);
-  return result;
-}
-
-/** Helper function for GET / or GET /tor/
- */
-static int
-handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args)
-{
-  (void) args; /* unused */
-  const char *frontpage = get_dirportfrontpage();
-
-  if (frontpage) {
-    size_t dlen;
-    dlen = strlen(frontpage);
-    /* Let's return a disclaimer page (users shouldn't use V1 anymore,
-       and caches don't fetch '/', so this is safe). */
-
-    /* [We don't check for write_bucket_low here, since we want to serve
-     *  this page no matter what.] */
-    write_http_response_header_impl(conn, dlen, "text/html", "identity",
-                                    NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME);
-    connection_write_to_buf(frontpage, dlen, TO_CONN(conn));
-  } else {
-    write_http_status_line(conn, 404, "Not found");
-  }
-  return 0;
-}
-
-/** Helper function for GET /tor/status-vote/current/consensus
- */
-static int
-handle_get_current_consensus(dir_connection_t *conn,
-                             const get_handler_args_t *args)
-{
-  const char *url = args->url;
-  const int compressed = args->compressed;
-  const time_t if_modified_since = args->if_modified_since;
-
-  {
-    /* v3 network status fetch. */
-    smartlist_t *dir_fps = smartlist_new();
-    long lifetime = NETWORKSTATUS_CACHE_LIFETIME;
-
-    if (1) {
-      networkstatus_t *v;
-      time_t now = time(NULL);
-      const char *want_fps = NULL;
-      char *flavor = NULL;
-      int flav = FLAV_NS;
-      #define CONSENSUS_URL_PREFIX "/tor/status-vote/current/consensus/"
-      #define CONSENSUS_FLAVORED_PREFIX "/tor/status-vote/current/consensus-"
-      /* figure out the flavor if any, and who we wanted to sign the thing */
-      if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) {
-        const char *f, *cp;
-        f = url + strlen(CONSENSUS_FLAVORED_PREFIX);
-        cp = strchr(f, '/');
-        if (cp) {
-          want_fps = cp+1;
-          flavor = tor_strndup(f, cp-f);
-        } else {
-          flavor = tor_strdup(f);
-        }
-        flav = networkstatus_parse_flavor_name(flavor);
-        if (flav < 0)
-          flav = FLAV_NS;
-      } else {
-        if (!strcmpstart(url, CONSENSUS_URL_PREFIX))
-          want_fps = url+strlen(CONSENSUS_URL_PREFIX);
-      }
-
-      v = networkstatus_get_latest_consensus_by_flavor(flav);
-
-      if (v && want_fps &&
-          !client_likes_consensus(v, want_fps)) {
-        write_http_status_line(conn, 404, "Consensus not signed by sufficient "
-                                          "number of requested authorities");
-        smartlist_free(dir_fps);
-        geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS);
-        tor_free(flavor);
-        goto done;
-      }
-
-      {
-        char *fp = tor_malloc_zero(DIGEST_LEN);
-        if (flavor)
-          strlcpy(fp, flavor, DIGEST_LEN);
-        tor_free(flavor);
-        smartlist_add(dir_fps, fp);
-      }
-      lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0;
-    }
-
-    if (!smartlist_len(dir_fps)) { /* we failed to create/cache cp */
-      write_http_status_line(conn, 503, "Network status object unavailable");
-      smartlist_free(dir_fps);
-      geoip_note_ns_response(GEOIP_REJECT_UNAVAILABLE);
-      goto done;
-    }
-
-    if (!dirserv_remove_old_statuses(dir_fps, if_modified_since)) {
-      write_http_status_line(conn, 404, "Not found");
-      SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp));
-      smartlist_free(dir_fps);
-      geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
-      goto done;
-    } else if (!smartlist_len(dir_fps)) {
-      write_http_status_line(conn, 304, "Not modified");
-      SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp));
-      smartlist_free(dir_fps);
-      geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED);
-      goto done;
-    }
-
-    size_t dlen = dirserv_estimate_data_size(dir_fps, 0, compressed);
-    if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) {
-      log_debug(LD_DIRSERV,
-               "Client asked for network status lists, but we've been "
-               "writing too many bytes lately. Sending 503 Dir busy.");
-      write_http_status_line(conn, 503, "Directory busy, try again later");
-      SMARTLIST_FOREACH(dir_fps, char *, fp, tor_free(fp));
-      smartlist_free(dir_fps);
-
-      geoip_note_ns_response(GEOIP_REJECT_BUSY);
-      goto done;
-    }
-
-    if (1) {
-      tor_addr_t addr;
-      if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) {
-        geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS,
-                               &addr, NULL,
-                               time(NULL));
-        geoip_note_ns_response(GEOIP_SUCCESS);
-        /* Note that a request for a network status has started, so that we
-         * can measure the download time later on. */
-        if (conn->dirreq_id)
-          geoip_start_dirreq(conn->dirreq_id, dlen, DIRREQ_TUNNELED);
-        else
-          geoip_start_dirreq(TO_CONN(conn)->global_identifier, dlen,
-                             DIRREQ_DIRECT);
-      }
-    }
-
-    write_http_response_header(conn, -1, compressed,
-                               smartlist_len(dir_fps) == 1 ? lifetime : 0);
-    conn->fingerprint_stack = dir_fps;
-    if (! compressed)
-      conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION);
-
-    /* Prime the connection with some data. */
-    conn->dir_spool_src = DIR_SPOOL_NETWORKSTATUS;
-    connection_dirserv_flushed_some(conn);
-    goto done;
-  }
-
- done:
-  return 0;
-}
-
-/** Helper function for GET /tor/status-vote/{current,next}/...
- */
-static int
-handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
-{
-  const char *url = args->url;
-  const int compressed = args->compressed;
-  {
-    int current;
-    ssize_t body_len = 0;
-    ssize_t estimated_len = 0;
-    smartlist_t *items = smartlist_new();
-    smartlist_t *dir_items = smartlist_new();
-    int lifetime = 60; /* XXXX?? should actually use vote intervals. */
-    url += strlen("/tor/status-vote/");
-    current = !strcmpstart(url, "current/");
-    url = strchr(url, '/');
-    tor_assert(url);
-    ++url;
-    if (!strcmp(url, "consensus")) {
-      const char *item;
-      tor_assert(!current); /* we handle current consensus specially above,
-                             * since it wants to be spooled. */
-      if ((item = dirvote_get_pending_consensus(FLAV_NS)))
-        smartlist_add(items, (char*)item);
-    } else if (!current && !strcmp(url, "consensus-signatures")) {
-      /* XXXX the spec says that we should implement
-       * current/consensus-signatures too.  It doesn't seem to be needed,
-       * though. */
-      const char *item;
-      if ((item=dirvote_get_pending_detached_signatures()))
-        smartlist_add(items, (char*)item);
-    } else if (!strcmp(url, "authority")) {
-      const cached_dir_t *d;
-      int flags = DGV_BY_ID |
-        (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING);
-      if ((d=dirvote_get_vote(NULL, flags)))
-        smartlist_add(dir_items, (cached_dir_t*)d);
-    } else {
-      const cached_dir_t *d;
-      smartlist_t *fps = smartlist_new();
-      int flags;
-      if (!strcmpstart(url, "d/")) {
-        url += 2;
-        flags = DGV_INCLUDE_PENDING | DGV_INCLUDE_PREVIOUS;
-      } else {
-        flags = DGV_BY_ID |
-          (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING);
-      }
-      dir_split_resource_into_fingerprints(url, fps, NULL,
-                                           DSR_HEX|DSR_SORT_UNIQ);
-      SMARTLIST_FOREACH(fps, char *, fp, {
-          if ((d = dirvote_get_vote(fp, flags)))
-            smartlist_add(dir_items, (cached_dir_t*)d);
-          tor_free(fp);
-        });
-      smartlist_free(fps);
-    }
-    if (!smartlist_len(dir_items) && !smartlist_len(items)) {
-      write_http_status_line(conn, 404, "Not found");
-      goto vote_done;
-    }
-    SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
-                      body_len += compressed ? d->dir_z_len : d->dir_len);
-    estimated_len += body_len;
-    SMARTLIST_FOREACH(items, const char *, item, {
-        size_t ln = strlen(item);
-        if (compressed) {
-          estimated_len += ln/2;
-        } else {
-          body_len += ln; estimated_len += ln;
-        }
-      });
-
-    if (global_write_bucket_low(TO_CONN(conn), estimated_len, 2)) {
-      write_http_status_line(conn, 503, "Directory busy, try again later");
-      goto vote_done;
-    }
-    write_http_response_header(conn, body_len ? body_len : -1, compressed,
-                 lifetime);
-
-    if (smartlist_len(items)) {
-      if (compressed) {
-        conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD,
-                                    choose_compression_level(estimated_len));
-        SMARTLIST_FOREACH(items, const char *, c,
-                 connection_write_to_buf_zlib(c, strlen(c), conn, 0));
-        connection_write_to_buf_zlib("", 0, conn, 1);
-      } else {
-        SMARTLIST_FOREACH(items, const char *, c,
-                         connection_write_to_buf(c, strlen(c), TO_CONN(conn)));
-      }
-    } else {
-      SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
-          connection_write_to_buf(compressed ? d->dir_z : d->dir,
-                                  compressed ? d->dir_z_len : d->dir_len,
-                                  TO_CONN(conn)));
-    }
-  vote_done:
-    smartlist_free(items);
-    smartlist_free(dir_items);
-    goto done;
-  }
- done:
-  return 0;
-}
-
-/** Helper function for GET /tor/micro/d/...
- */
-static int
-handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args)
-{
-  const char *url = args->url;
-  const int compressed = args->compressed;
-  {
-    smartlist_t *fps = smartlist_new();
-
-    dir_split_resource_into_fingerprints(url+strlen("/tor/micro/d/"),
-                                      fps, NULL,
-                                      DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ);
-
-    if (!dirserv_have_any_microdesc(fps)) {
-      write_http_status_line(conn, 404, "Not found");
-      SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp));
-      smartlist_free(fps);
-      goto done;
-    }
-    size_t dlen = dirserv_estimate_microdesc_size(fps, compressed);
-    if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) {
-      log_info(LD_DIRSERV,
-               "Client asked for server descriptors, but we've been "
-               "writing too many bytes lately. Sending 503 Dir busy.");
-      write_http_status_line(conn, 503, "Directory busy, try again later");
-      SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp));
-      smartlist_free(fps);
-      goto done;
-    }
-
-    write_http_response_header(conn, -1, compressed, MICRODESC_CACHE_LIFETIME);
-    conn->dir_spool_src = DIR_SPOOL_MICRODESC;
-    conn->fingerprint_stack = fps;
-
-    if (compressed)
-      conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD,
-                                      choose_compression_level(dlen));
-
-    connection_dirserv_flushed_some(conn);
-    goto done;
-  }
-
- done:
-  return 0;
-}
-
-/** Helper function for GET /tor/{server,extra}/...
- */
-static int
-handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args)
-{
-  const char *url = args->url;
-  const int compressed = args->compressed;
-  const or_options_t *options = get_options();
-  if (!strcmpstart(url,"/tor/server/") ||
-      (!options->BridgeAuthoritativeDir &&
-       !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) {
-    size_t dlen;
-    int res;
-    const char *msg;
-    int cache_lifetime = 0;
-    int is_extra = !strcmpstart(url,"/tor/extra/");
-    url += is_extra ? strlen("/tor/extra/") : strlen("/tor/server/");
-    conn->fingerprint_stack = smartlist_new();
-    res = dirserv_get_routerdesc_fingerprints(conn->fingerprint_stack, url,
-                                          &msg,
-                                          !connection_dir_is_encrypted(conn),
-                                          is_extra);
-
-    if (!strcmpstart(url, "fp/")) {
-      if (smartlist_len(conn->fingerprint_stack) == 1)
-        cache_lifetime = ROUTERDESC_CACHE_LIFETIME;
-    } else if (!strcmpstart(url, "authority")) {
-      cache_lifetime = ROUTERDESC_CACHE_LIFETIME;
-    } else if (!strcmpstart(url, "all")) {
-      cache_lifetime = FULL_DIR_CACHE_LIFETIME;
-    } else if (!strcmpstart(url, "d/")) {
-      if (smartlist_len(conn->fingerprint_stack) == 1)
-        cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME;
-    }
-    if (!strcmpstart(url, "d/"))
-      conn->dir_spool_src =
-        is_extra ? DIR_SPOOL_EXTRA_BY_DIGEST : DIR_SPOOL_SERVER_BY_DIGEST;
-    else
-      conn->dir_spool_src =
-        is_extra ? DIR_SPOOL_EXTRA_BY_FP : DIR_SPOOL_SERVER_BY_FP;
-
-    if (!dirserv_have_any_serverdesc(conn->fingerprint_stack,
-                                     conn->dir_spool_src)) {
-      res = -1;
-      msg = "Not found";
-    }
-
-    if (res < 0)
-      write_http_status_line(conn, 404, msg);
-    else {
-      dlen = dirserv_estimate_data_size(conn->fingerprint_stack,
-                                        1, compressed);
-      if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) {
-        log_info(LD_DIRSERV,
-                 "Client asked for server descriptors, but we've been "
-                 "writing too many bytes lately. Sending 503 Dir busy.");
-        write_http_status_line(conn, 503, "Directory busy, try again later");
-        conn->dir_spool_src = DIR_SPOOL_NONE;
-        goto done;
-      }
-      write_http_response_header(conn, -1, compressed, cache_lifetime);
-      if (compressed)
-        conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD,
-                                        choose_compression_level(dlen));
-      /* Prime the connection with some data. */
-      connection_dirserv_flushed_some(conn);
-    }
-    goto done;
-  }
- done:
- return 0;
-}
-
-/** Helper function for GET /tor/keys/...
- */
-static int
-handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
-{
-  const char *url = args->url;
-  const int compressed = args->compressed;
-  const time_t if_modified_since = args->if_modified_since;
-  {
-    smartlist_t *certs = smartlist_new();
-    ssize_t len = -1;
-    if (!strcmp(url, "/tor/keys/all")) {
-      authority_cert_get_all(certs);
-    } else if (!strcmp(url, "/tor/keys/authority")) {
-      authority_cert_t *cert = get_my_v3_authority_cert();
-      if (cert)
-        smartlist_add(certs, cert);
-    } else if (!strcmpstart(url, "/tor/keys/fp/")) {
-      smartlist_t *fps = smartlist_new();
-      dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"),
-                                           fps, NULL,
-                                           DSR_HEX|DSR_SORT_UNIQ);
-      SMARTLIST_FOREACH(fps, char *, d, {
-          authority_cert_t *c = authority_cert_get_newest_by_id(d);
-          if (c) smartlist_add(certs, c);
-          tor_free(d);
-      });
-      smartlist_free(fps);
-    } else if (!strcmpstart(url, "/tor/keys/sk/")) {
-      smartlist_t *fps = smartlist_new();
-      dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"),
-                                           fps, NULL,
-                                           DSR_HEX|DSR_SORT_UNIQ);
-      SMARTLIST_FOREACH(fps, char *, d, {
-          authority_cert_t *c = authority_cert_get_by_sk_digest(d);
-          if (c) smartlist_add(certs, c);
-          tor_free(d);
-      });
-      smartlist_free(fps);
-    } else if (!strcmpstart(url, "/tor/keys/fp-sk/")) {
-      smartlist_t *fp_sks = smartlist_new();
-      dir_split_resource_into_fingerprint_pairs(url+strlen("/tor/keys/fp-sk/"),
-                                                fp_sks);
-      SMARTLIST_FOREACH(fp_sks, fp_pair_t *, pair, {
-          authority_cert_t *c = authority_cert_get_by_digests(pair->first,
-                                                              pair->second);
-          if (c) smartlist_add(certs, c);
-          tor_free(pair);
-      });
-      smartlist_free(fp_sks);
-    } else {
-      write_http_status_line(conn, 400, "Bad request");
-      goto keys_done;
-    }
-    if (!smartlist_len(certs)) {
-      write_http_status_line(conn, 404, "Not found");
-      goto keys_done;
-    }
-    SMARTLIST_FOREACH(certs, authority_cert_t *, c,
-      if (c->cache_info.published_on < if_modified_since)
-        SMARTLIST_DEL_CURRENT(certs, c));
-    if (!smartlist_len(certs)) {
-      write_http_status_line(conn, 304, "Not modified");
-      goto keys_done;
-    }
-    len = 0;
-    SMARTLIST_FOREACH(certs, authority_cert_t *, c,
-                      len += c->cache_info.signed_descriptor_len);
-
-    if (global_write_bucket_low(TO_CONN(conn), compressed?len/2:len, 2)) {
-      write_http_status_line(conn, 503, "Directory busy, try again later");
-      goto keys_done;
-    }
-
-    write_http_response_header(conn, compressed?-1:len, compressed, 60*60);
-    if (compressed) {
-      conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD,
-                                      choose_compression_level(len));
-      SMARTLIST_FOREACH(certs, authority_cert_t *, c,
-            connection_write_to_buf_zlib(c->cache_info.signed_descriptor_body,
-                                         c->cache_info.signed_descriptor_len,
-                                         conn, 0));
-      connection_write_to_buf_zlib("", 0, conn, 1);
-    } else {
-      SMARTLIST_FOREACH(certs, authority_cert_t *, c,
-            connection_write_to_buf(c->cache_info.signed_descriptor_body,
-                                    c->cache_info.signed_descriptor_len,
-                                    TO_CONN(conn)));
-    }
-  keys_done:
-    smartlist_free(certs);
-    goto done;
-  }
- done:
-  return 0;
-}
-
-/** Helper function for GET /tor/rendezvous2/
- */
-static int
-handle_get_rendezvous2(dir_connection_t *conn, const get_handler_args_t *args)
-{
-  const char *url = args->url;
-  if (connection_dir_is_encrypted(conn)) {
-    /* Handle v2 rendezvous descriptor fetch request. */
-    const char *descp;
-    const char *query = url + strlen("/tor/rendezvous2/");
-    if (rend_valid_descriptor_id(query)) {
-      log_info(LD_REND, "Got a v2 rendezvous descriptor request for ID '%s'",
-               safe_str(escaped(query)));
-      switch (rend_cache_lookup_v2_desc_as_dir(query, &descp)) {
-        case 1: /* valid */
-          write_http_response_header(conn, strlen(descp), 0, 0);
-          connection_write_to_buf(descp, strlen(descp), TO_CONN(conn));
-          break;
-        case 0: /* well-formed but not present */
-          write_http_status_line(conn, 404, "Not found");
-          break;
-        case -1: /* not well-formed */
-          write_http_status_line(conn, 400, "Bad request");
-          break;
-      }
-    } else { /* not well-formed */
-      write_http_status_line(conn, 400, "Bad request");
-    }
-    goto done;
-  } else {
-    /* Not encrypted! */
-    write_http_status_line(conn, 404, "Not found");
-  }
- done:
-  return 0;
-}
-
-/** Helper function for GET /tor/networkstatus-bridges
- */
-static int
-handle_get_networkstatus_bridges(dir_connection_t *conn,
-                                 const get_handler_args_t *args)
-{
-  const char *headers = args->headers;
-
-  const or_options_t *options = get_options();
-  if (options->BridgeAuthoritativeDir &&
-      options->BridgePassword_AuthDigest_ &&
-      connection_dir_is_encrypted(conn)) {
-    char *status;
-    char digest[DIGEST256_LEN];
-
-    char *header = http_get_header(headers, "Authorization: Basic ");
-    if (header)
-      crypto_digest256(digest, header, strlen(header), DIGEST_SHA256);
-
-    /* now make sure the password is there and right */
-    if (!header ||
-        tor_memneq(digest,
-                   options->BridgePassword_AuthDigest_, DIGEST256_LEN)) {
-      write_http_status_line(conn, 404, "Not found");
-      tor_free(header);
-      goto done;
-    }
-    tor_free(header);
-
-    /* all happy now. send an answer. */
-    status = networkstatus_getinfo_by_purpose("bridge", time(NULL));
-    size_t dlen = strlen(status);
-    write_http_response_header(conn, dlen, 0, 0);
-    connection_write_to_buf(status, dlen, TO_CONN(conn));
-    tor_free(status);
-    goto done;
-  }
- done:
-  return 0;
-}
-
-/** Helper function for GET robots.txt or /tor/robots.txt */
-static int
-handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args)
-{
-  (void)args;
-  {
-    const char robots[] = "User-agent: *\r\nDisallow: /\r\n";
-    size_t len = strlen(robots);
-    write_http_response_header(conn, len, 0, ROBOTS_CACHE_LIFETIME);
-    connection_write_to_buf(robots, len, TO_CONN(conn));
-  }
-  return 0;
-}
-
-/** Helper function: called when a dirserver gets a complete HTTP POST
- * request.  Look for an uploaded server descriptor or rendezvous
- * service descriptor.  On finding one, process it and write a
- * response into conn-\>outbuf.  If the request is unrecognized, send a
- * 400.  Always return 0. */
-static int
-directory_handle_command_post(dir_connection_t *conn, const char *headers,
-                              const char *body, size_t body_len)
-{
-  char *url = NULL;
-  const or_options_t *options = get_options();
-
-  log_debug(LD_DIRSERV,"Received POST command.");
-
-  conn->base_.state = DIR_CONN_STATE_SERVER_WRITING;
-
-  if (!public_server_mode(options)) {
-    log_info(LD_DIR, "Rejected dir post request from %s "
-             "since we're not a public relay.", conn->base_.address);
-    write_http_status_line(conn, 503, "Not acting as a public relay");
-    goto done;
-  }
-
-  if (parse_http_url(headers, &url) < 0) {
-    write_http_status_line(conn, 400, "Bad request");
-    return 0;
-  }
-  log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url));
-
-  /* Handle v2 rendezvous service publish request. */
-  if (connection_dir_is_encrypted(conn) &&
-      !strcmpstart(url,"/tor/rendezvous2/publish")) {
-    if (rend_cache_store_v2_desc_as_dir(body) < 0) {
-      log_warn(LD_REND, "Rejected v2 rend descriptor (length %d) from %s.",
-               (int)body_len, conn->base_.address);
-      write_http_status_line(conn, 400,
-                             "Invalid v2 service descriptor rejected");
-    } else {
-      write_http_status_line(conn, 200, "Service descriptor (v2) stored");
-      log_info(LD_REND, "Handled v2 rendezvous descriptor post: accepted");
-    }
-    goto done;
-  }
-
-  if (!authdir_mode(options)) {
-    /* we just provide cached directories; we don't want to
-     * receive anything. */
-    write_http_status_line(conn, 400, "Nonauthoritative directory does not "
-                           "accept posted server descriptors");
-    goto done;
-  }
-
-  if (authdir_mode_handles_descs(options, -1) &&
-      !strcmp(url,"/tor/")) { /* server descriptor post */
-    const char *msg = "[None]";
-    uint8_t purpose = authdir_mode_bridge(options) ?
-                      ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL;
-    was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose,
-                                             conn->base_.address, &msg);
-    tor_assert(msg);
-
-    if (r == ROUTER_ADDED_NOTIFY_GENERATOR) {
-      /* Accepted with a message. */
-      log_info(LD_DIRSERV,
-               "Problematic router descriptor or extra-info from %s "
-               "(\"%s\").",
-               conn->base_.address, msg);
-      write_http_status_line(conn, 400, msg);
-    } else if (r == ROUTER_ADDED_SUCCESSFULLY) {
-      write_http_status_line(conn, 200, msg);
-    } else if (WRA_WAS_OUTDATED(r)) {
-      write_http_response_header_impl(conn, -1, NULL, NULL,
-                                      "X-Descriptor-Not-New: Yes\r\n", -1);
-    } else {
-      log_info(LD_DIRSERV,
-               "Rejected router descriptor or extra-info from %s "
-               "(\"%s\").",
-               conn->base_.address, msg);
-      write_http_status_line(conn, 400, msg);
-    }
-    goto done;
-  }
-
-  if (authdir_mode_v3(options) &&
-      !strcmp(url,"/tor/post/vote")) { /* v3 networkstatus vote */
-    const char *msg = "OK";
-    int status;
-    if (dirvote_add_vote(body, &msg, &status)) {
-      write_http_status_line(conn, status, "Vote stored");
-    } else {
-      tor_assert(msg);
-      log_warn(LD_DIRSERV, "Rejected vote from %s (\"%s\").",
-               conn->base_.address, msg);
-      write_http_status_line(conn, status, msg);
-    }
-    goto done;
-  }
-
-  if (authdir_mode_v3(options) &&
-      !strcmp(url,"/tor/post/consensus-signature")) { /* sigs on consensus. */
-    const char *msg = NULL;
-    if (dirvote_add_signatures(body, conn->base_.address, &msg)>=0) {
-      write_http_status_line(conn, 200, msg?msg:"Signatures stored");
-    } else {
-      log_warn(LD_DIR, "Unable to store signatures posted by %s: %s",
-               conn->base_.address, msg?msg:"???");
-      write_http_status_line(conn, 400, msg?msg:"Unable to store signatures");
-    }
-    goto done;
-  }
-
-  /* we didn't recognize the url */
-  write_http_status_line(conn, 404, "Not found");
-
- done:
-  tor_free(url);
-  return 0;
-}
-
-/** Called when a dirserver receives data on a directory connection;
- * looks for an HTTP request.  If the request is complete, remove it
- * from the inbuf, try to process it; otherwise, leave it on the
- * buffer.  Return a 0 on success, or -1 on error.
- */
-static int
-directory_handle_command(dir_connection_t *conn)
-{
-  char *headers=NULL, *body=NULL;
-  size_t body_len=0;
-  int r;
-
-  tor_assert(conn);
-  tor_assert(conn->base_.type == CONN_TYPE_DIR);
-
-  switch (connection_fetch_from_buf_http(TO_CONN(conn),
-                              &headers, MAX_HEADERS_SIZE,
-                              &body, &body_len, MAX_DIR_UL_SIZE, 0)) {
-    case -1: /* overflow */
-      log_warn(LD_DIRSERV,
-               "Request too large from address '%s' to DirPort. Closing.",
-               safe_str(conn->base_.address));
-      return -1;
-    case 0:
-      log_debug(LD_DIRSERV,"command not all here yet.");
-      return 0;
-    /* case 1, fall through */
-  }
-
-  http_set_address_origin(headers, TO_CONN(conn));
-  // we should escape headers here as well,
-  // but we can't call escaped() twice, as it uses the same buffer
-  //log_debug(LD_DIRSERV,"headers %s, body %s.", headers, escaped(body));
-
-  if (!strncasecmp(headers,"GET",3))
-    r = directory_handle_command_get(conn, headers, body, body_len);
-  else if (!strncasecmp(headers,"POST",4))
-    r = directory_handle_command_post(conn, headers, body, body_len);
-  else {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "Got headers %s with unknown command. Closing.",
-           escaped(headers));
-    r = -1;
-  }
-
-  tor_free(headers); tor_free(body);
-  return r;
-}
-
-/** Write handler for directory connections; called when all data has
- * been flushed.  Close the connection or wait for a response as
- * appropriate.
- */
-int
-connection_dir_finished_flushing(dir_connection_t *conn)
-{
-  tor_assert(conn);
-  tor_assert(conn->base_.type == CONN_TYPE_DIR);
-
-  /* Note that we have finished writing the directory response. For direct
-   * connections this means we're done; for tunneled connections it's only
-   * an intermediate step. */
-  if (conn->dirreq_id)
-    geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED,
-                              DIRREQ_FLUSHING_DIR_CONN_FINISHED);
-  else
-    geoip_change_dirreq_state(TO_CONN(conn)->global_identifier,
-                              DIRREQ_DIRECT,
-                              DIRREQ_FLUSHING_DIR_CONN_FINISHED);
-  switch (conn->base_.state) {
-    case DIR_CONN_STATE_CONNECTING:
-    case DIR_CONN_STATE_CLIENT_SENDING:
-      log_debug(LD_DIR,"client finished sending command.");
-      conn->base_.state = DIR_CONN_STATE_CLIENT_READING;
-      return 0;
-    case DIR_CONN_STATE_SERVER_WRITING:
-      if (conn->dir_spool_src != DIR_SPOOL_NONE) {
-        log_warn(LD_BUG, "Emptied a dirserv buffer, but it's still spooling!");
-        connection_mark_for_close(TO_CONN(conn));
-      } else {
-        log_debug(LD_DIRSERV, "Finished writing server response. Closing.");
-        connection_mark_for_close(TO_CONN(conn));
-      }
-      return 0;
-    default:
-      log_warn(LD_BUG,"called in unexpected state %d.",
-               conn->base_.state);
-      tor_fragile_assert();
-      return -1;
-  }
-  return 0;
-}
-
-/* We just got a new consensus! If there are other in-progress requests
- * for this consensus flavor (for example because we launched several in
- * parallel), cancel them.
- *
- * We do this check here (not just in
- * connection_ap_handshake_attach_circuit()) to handle the edge case where
- * a consensus fetch begins and ends before some other one tries to attach to
- * a circuit, in which case the other one won't know that we're all happy now.
- *
- * Don't mark the conn that just gave us the consensus -- otherwise we
- * would end up double-marking it when it cleans itself up.
- */
-static void
-connection_dir_close_consensus_fetches(dir_connection_t *except_this_one,
-                                       const char *resource)
-{
-  smartlist_t *conns_to_close =
-    connection_dir_list_by_purpose_and_resource(DIR_PURPOSE_FETCH_CONSENSUS,
-                                                resource);
-  SMARTLIST_FOREACH_BEGIN(conns_to_close, dir_connection_t *, d) {
-    if (d == except_this_one)
-      continue;
-    log_info(LD_DIR, "Closing consensus fetch (to %s) since one "
-             "has just arrived.", TO_CONN(d)->address);
-    connection_mark_for_close(TO_CONN(d));
-  } SMARTLIST_FOREACH_END(d);
-  smartlist_free(conns_to_close);
-}
-
-/** Connected handler for directory connections: begin sending data to the
- * server, and return 0.
- * Only used when connections don't immediately connect. */
-int
-connection_dir_finished_connecting(dir_connection_t *conn)
-{
-  tor_assert(conn);
-  tor_assert(conn->base_.type == CONN_TYPE_DIR);
-  tor_assert(conn->base_.state == DIR_CONN_STATE_CONNECTING);
-
-  log_debug(LD_HTTP,"Dir connection to router %s:%u established.",
-            conn->base_.address,conn->base_.port);
-
-  /* start flushing conn */
-  conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING;
-  return 0;
-}
-
-/** Decide which download schedule we want to use based on descriptor type
- * in <b>dls</b> and <b>options</b>.
- * Then return a list of int pointers defining download delays in seconds.
- * Helper function for download_status_increment_failure(),
- * download_status_reset(), and download_status_increment_attempt(). */
-STATIC const smartlist_t *
-find_dl_schedule(download_status_t *dls, const or_options_t *options)
-{
-  switch (dls->schedule) {
-    case DL_SCHED_GENERIC:
-      /* Any other directory document */
-      if (dir_server_mode(options)) {
-        /* A directory authority or directory mirror */
-        return options->TestingServerDownloadSchedule;
-      } else {
-        return options->TestingClientDownloadSchedule;
-      }
-    case DL_SCHED_CONSENSUS:
-      if (!networkstatus_consensus_can_use_multiple_directories(options)) {
-        /* A public relay */
-        return options->TestingServerConsensusDownloadSchedule;
-      } else {
-        /* A client or bridge */
-        if (networkstatus_consensus_is_bootstrapping(time(NULL))) {
-          /* During bootstrapping */
-          if (!networkstatus_consensus_can_use_extra_fallbacks(options)) {
-            /* A bootstrapping client without extra fallback directories */
-            return
-             options->ClientBootstrapConsensusAuthorityOnlyDownloadSchedule;
-          } else if (dls->want_authority) {
-            /* A bootstrapping client with extra fallback directories, but
-             * connecting to an authority */
-            return
-             options->ClientBootstrapConsensusAuthorityDownloadSchedule;
-          } else {
-            /* A bootstrapping client connecting to extra fallback directories
-             */
-            return
-              options->ClientBootstrapConsensusFallbackDownloadSchedule;
-          }
-        } else {
-          /* A client with a reasonably live consensus, with or without
-           * certificates */
-          return options->TestingClientConsensusDownloadSchedule;
-        }
-      }
-    case DL_SCHED_BRIDGE:
-      return options->TestingBridgeDownloadSchedule;
-    default:
-      tor_assert(0);
-  }
-
-  /* Impossible, but gcc will fail with -Werror without a `return`. */
-  return NULL;
-}
-
-/** Decide which minimum and maximum delay step we want to use based on
- * descriptor type in <b>dls</b> and <b>options</b>.
- * Helper function for download_status_schedule_get_delay(). */
-STATIC void
-find_dl_min_and_max_delay(download_status_t *dls, const or_options_t *options,
-                          int *min, int *max)
-{
-  tor_assert(dls);
-  tor_assert(options);
-  tor_assert(min);
-  tor_assert(max);
-
-  /*
-   * For now, just use the existing schedule config stuff and pick the
-   * first/last entries off to get min/max delay for backoff purposes
-   */
-  const smartlist_t *schedule = find_dl_schedule(dls, options);
-  tor_assert(schedule != NULL && smartlist_len(schedule) >= 2);
-  *min = *((int *)(smartlist_get(schedule, 0)));
-  if (dls->backoff == DL_SCHED_DETERMINISTIC)
-    *max = *((int *)((smartlist_get(schedule, smartlist_len(schedule) - 1))));
-  else
-    *max = INT_MAX;
-}
-
-/** Advance one delay step.  The algorithm is to use the previous delay to
- * compute an increment, we construct a value uniformly at random between
- * delay and MAX(delay*2,delay+1).  We then clamp that value to be no larger
- * than max_delay, and return it.
- *
- * Requires that delay is less than INT_MAX, and delay is in [0,max_delay].
- */
-STATIC int
-next_random_exponential_delay(int delay, int max_delay)
-{
-  /* Check preconditions */
-  if (BUG(max_delay < 0))
-    max_delay = 0;
-  if (BUG(delay > max_delay))
-    delay = max_delay;
-  if (delay == INT_MAX)
-    return INT_MAX; /* prevent overflow */
-  if (BUG(delay < 0))
-    delay = 0;
-
-  /* How much are we willing to add to the delay? */
-  int max_increment;
-  int multiplier = 3; /* no more than quadruple the previous delay */
-  if (get_options()->TestingTorNetwork) {
-    /* Decrease the multiplier in testing networks. This reduces the variance,
-     * so that bootstrap is more reliable. */
-    multiplier = 2; /* no more than triple the previous delay */
-  }
-
-  if (delay && delay < (INT_MAX-1) / multiplier) {
-    max_increment = delay * multiplier;
-  } else if (delay) {
-    max_increment = INT_MAX-1;
-  } else {
-    max_increment = 1;
-  }
-
-  if (BUG(max_increment < 1))
-    max_increment = 1;
-
-  /* the + 1 here is so that we always wait longer than last time. */
-  int increment = crypto_rand_int(max_increment)+1;
-
-  if (increment < max_delay - delay)
-    return delay + increment;
-  else
-    return max_delay;
-}
-
-/** Find the current delay for dls based on schedule or min_delay/
- * max_delay if we're using exponential backoff.  If dls->backoff is
- * DL_SCHED_RANDOM_EXPONENTIAL, we must have 0 <= min_delay <= max_delay <=
- * INT_MAX, but schedule may be set to NULL; otherwise schedule is required.
- * This function sets dls->next_attempt_at based on now, and returns the delay.
- * Helper for download_status_increment_failure and
- * download_status_increment_attempt. */
-STATIC int
-download_status_schedule_get_delay(download_status_t *dls,
-                                   const smartlist_t *schedule,
-                                   int min_delay, int max_delay,
-                                   time_t now)
-{
-  tor_assert(dls);
-  /* We don't need a schedule if we're using random exponential backoff */
-  tor_assert(dls->backoff == DL_SCHED_RANDOM_EXPONENTIAL ||
-             schedule != NULL);
-  /* If we're using random exponential backoff, we do need min/max delay */
-  tor_assert(dls->backoff != DL_SCHED_RANDOM_EXPONENTIAL ||
-             (min_delay >= 0 && max_delay >= min_delay));
-
-  int delay = INT_MAX;
-  uint8_t dls_schedule_position = (dls->increment_on
-                                   == DL_SCHED_INCREMENT_ATTEMPT
-                                   ? dls->n_download_attempts
-                                   : dls->n_download_failures);
-
-  if (dls->backoff == DL_SCHED_DETERMINISTIC) {
-    if (dls_schedule_position < smartlist_len(schedule))
-      delay = *(int *)smartlist_get(schedule, dls_schedule_position);
-    else if (dls_schedule_position == IMPOSSIBLE_TO_DOWNLOAD)
-      delay = INT_MAX;
-    else
-      delay = *(int *)smartlist_get(schedule, smartlist_len(schedule) - 1);
-  } else if (dls->backoff == DL_SCHED_RANDOM_EXPONENTIAL) {
-    /* Check if we missed a reset somehow */
-    if (dls->last_backoff_position > dls_schedule_position) {
-      dls->last_backoff_position = 0;
-      dls->last_delay_used = 0;
-    }
-
-    if (dls_schedule_position > 0) {
-      delay = dls->last_delay_used;
-
-      while (dls->last_backoff_position < dls_schedule_position) {
-        /* Do one increment step */
-        delay = next_random_exponential_delay(delay, max_delay);
-        /* Update our position */
-        ++(dls->last_backoff_position);
-      }
-    } else {
-      /* If we're just starting out, use the minimum delay */
-      delay = min_delay;
-    }
-
-    /* Clamp it within min/max if we have them */
-    if (min_delay >= 0 && delay < min_delay) delay = min_delay;
-    if (max_delay != INT_MAX && delay > max_delay) delay = max_delay;
-
-    /* Store it for next time */
-    dls->last_backoff_position = dls_schedule_position;
-    dls->last_delay_used = delay;
-  }
-
-  /* A negative delay makes no sense. Knowing that delay is
-   * non-negative allows us to safely do the wrapping check below. */
-  tor_assert(delay >= 0);
-
-  /* Avoid now+delay overflowing TIME_MAX, by comparing with a subtraction
-   * that won't overflow (since delay is non-negative). */
-  if (delay < INT_MAX && now <= TIME_MAX - delay) {
-    dls->next_attempt_at = now+delay;
-  } else {
-    dls->next_attempt_at = TIME_MAX;
-  }
-
-  return delay;
-}
-
-/* Log a debug message about item, which increments on increment_action, has
- * incremented dls_n_download_increments times. The message varies based on
- * was_schedule_incremented (if not, not_incremented_response is logged), and
- * the values of increment, dls_next_attempt_at, and now.
- * Helper for download_status_increment_failure and
- * download_status_increment_attempt. */
-static void
-download_status_log_helper(const char *item, int was_schedule_incremented,
-                           const char *increment_action,
-                           const char *not_incremented_response,
-                           uint8_t dls_n_download_increments, int increment,
-                           time_t dls_next_attempt_at, time_t now)
-{
-  if (item) {
-    if (!was_schedule_incremented)
-      log_debug(LD_DIR, "%s %s %d time(s); I'll try again %s.",
-                item, increment_action, (int)dls_n_download_increments,
-                not_incremented_response);
-    else if (increment == 0)
-      log_debug(LD_DIR, "%s %s %d time(s); I'll try again immediately.",
-                item, increment_action, (int)dls_n_download_increments);
-    else if (dls_next_attempt_at < TIME_MAX)
-      log_debug(LD_DIR, "%s %s %d time(s); I'll try again in %d seconds.",
-                item, increment_action, (int)dls_n_download_increments,
-                (int)(dls_next_attempt_at-now));
-    else
-      log_debug(LD_DIR, "%s %s %d time(s); Giving up for a while.",
-                item, increment_action, (int)dls_n_download_increments);
-  }
-}
-
-/** Determine when a failed download attempt should be retried.
- * Called when an attempt to download <b>dls</b> has failed with HTTP status
- * <b>status_code</b>.  Increment the failure count (if the code indicates a
- * real failure, or if we're a server) and set <b>dls</b>-\>next_attempt_at to
- * an appropriate time in the future and return it.
- * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_ATTEMPT, increment the
- * failure count, and return a time in the far future for the next attempt (to
- * avoid an immediate retry). */
-time_t
-download_status_increment_failure(download_status_t *dls, int status_code,
-                                  const char *item, int server, time_t now)
-{
-  (void) status_code; // XXXX no longer used.
-  (void) server; // XXXX no longer used.
-  int increment = -1;
-  int min_delay = 0, max_delay = INT_MAX;
-
-  tor_assert(dls);
-
-  /* count the failure */
-  if (dls->n_download_failures < IMPOSSIBLE_TO_DOWNLOAD-1) {
-    ++dls->n_download_failures;
-  }
-
-  if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) {
-    /* We don't find out that a failure-based schedule has attempted a
-     * connection until that connection fails.
-     * We'll never find out about successful connections, but this doesn't
-     * matter, because schedules are reset after a successful download.
-     */
-    if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1)
-      ++dls->n_download_attempts;
-
-    /* only return a failure retry time if this schedule increments on failures
-     */
-    const smartlist_t *schedule = find_dl_schedule(dls, get_options());
-    find_dl_min_and_max_delay(dls, get_options(), &min_delay, &max_delay);
-    increment = download_status_schedule_get_delay(dls, schedule,
-                                                   min_delay, max_delay, now);
-  }
-
-  download_status_log_helper(item, !dls->increment_on, "failed",
-                             "concurrently", dls->n_download_failures,
-                             increment, dls->next_attempt_at, now);
-
-  if (dls->increment_on == DL_SCHED_INCREMENT_ATTEMPT) {
-    /* stop this schedule retrying on failure, it will launch concurrent
-     * connections instead */
-    return TIME_MAX;
-  } else {
-    return dls->next_attempt_at;
-  }
-}
-
-/** Determine when the next download attempt should be made when using an
- * attempt-based (potentially concurrent) download schedule.
- * Called when an attempt to download <b>dls</b> is being initiated.
- * Increment the attempt count and set <b>dls</b>-\>next_attempt_at to an
- * appropriate time in the future and return it.
- * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_FAILURE, don't increment
- * the attempts, and return a time in the far future (to avoid launching a
- * concurrent attempt). */
-time_t
-download_status_increment_attempt(download_status_t *dls, const char *item,
-                                  time_t now)
-{
-  int delay = -1;
-  int min_delay = 0, max_delay = INT_MAX;
-
-  tor_assert(dls);
-
-  if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) {
-    /* this schedule should retry on failure, and not launch any concurrent
-     attempts */
-    log_warn(LD_BUG, "Tried to launch an attempt-based connection on a "
-             "failure-based schedule.");
-    return TIME_MAX;
-  }
-
-  if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1)
-    ++dls->n_download_attempts;
-
-  const smartlist_t *schedule = find_dl_schedule(dls, get_options());
-  find_dl_min_and_max_delay(dls, get_options(), &min_delay, &max_delay);
-  delay = download_status_schedule_get_delay(dls, schedule,
-                                             min_delay, max_delay, now);
-
-  download_status_log_helper(item, dls->increment_on, "attempted",
-                             "on failure", dls->n_download_attempts,
-                             delay, dls->next_attempt_at, now);
-
-  return dls->next_attempt_at;
-}
-
-/** Reset <b>dls</b> so that it will be considered downloadable
- * immediately, and/or to show that we don't need it anymore.
- *
- * Must be called to initialise a download schedule, otherwise the zeroth item
- * in the schedule will never be used.
- *
- * (We find the zeroth element of the download schedule, and set
- * next_attempt_at to be the appropriate offset from 'now'. In most
- * cases this means setting it to 'now', so the item will be immediately
- * downloadable; in the case of bridge descriptors, the zeroth element
- * is an hour from now.) */
-void
-download_status_reset(download_status_t *dls)
-{
-  if (dls->n_download_failures == IMPOSSIBLE_TO_DOWNLOAD
-      || dls->n_download_attempts == IMPOSSIBLE_TO_DOWNLOAD)
-    return; /* Don't reset this. */
-
-  const smartlist_t *schedule = find_dl_schedule(dls, get_options());
-
-  dls->n_download_failures = 0;
-  dls->n_download_attempts = 0;
-  dls->next_attempt_at = time(NULL) + *(int *)smartlist_get(schedule, 0);
-  dls->last_backoff_position = 0;
-  dls->last_delay_used = 0;
-  /* Don't reset dls->want_authority or dls->increment_on */
-}
-
-/** Return the number of failures on <b>dls</b> since the last success (if
- * any). */
-int
-download_status_get_n_failures(const download_status_t *dls)
-{
-  return dls->n_download_failures;
-}
-
-/** Return the number of attempts to download <b>dls</b> since the last success
- * (if any). This can differ from download_status_get_n_failures() due to
- * outstanding concurrent attempts. */
-int
-download_status_get_n_attempts(const download_status_t *dls)
-{
-  return dls->n_download_attempts;
-}
-
-/** Return the next time to attempt to download <b>dls</b>. */
-time_t
-download_status_get_next_attempt_at(const download_status_t *dls)
-{
-  return dls->next_attempt_at;
-}
-
-/** Called when one or more routerdesc (or extrainfo, if <b>was_extrainfo</b>)
- * fetches have failed (with uppercase fingerprints listed in <b>failed</b>,
- * either as descriptor digests or as identity digests based on
- * <b>was_descriptor_digests</b>).
- */
-static void
-dir_routerdesc_download_failed(smartlist_t *failed, int status_code,
-                               int router_purpose,
-                               int was_extrainfo, int was_descriptor_digests)
-{
-  char digest[DIGEST_LEN];
-  time_t now = time(NULL);
-  int server = directory_fetches_from_authorities(get_options());
-  if (!was_descriptor_digests) {
-    if (router_purpose == ROUTER_PURPOSE_BRIDGE) {
-      tor_assert(!was_extrainfo);
-      connection_dir_retry_bridges(failed);
-    }
-    return; /* FFFF should implement for other-than-router-purpose someday */
-  }
-  SMARTLIST_FOREACH_BEGIN(failed, const char *, cp) {
-    download_status_t *dls = NULL;
-    if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp)) != DIGEST_LEN) {
-      log_warn(LD_BUG, "Malformed fingerprint in list: %s", escaped(cp));
-      continue;
-    }
-    if (was_extrainfo) {
-      signed_descriptor_t *sd =
-        router_get_by_extrainfo_digest(digest);
-      if (sd)
-        dls = &sd->ei_dl_status;
-    } else {
-      dls = router_get_dl_status_by_descriptor_digest(digest);
-    }
-    if (!dls || dls->n_download_failures >=
-                get_options()->TestingDescriptorMaxDownloadTries)
-      continue;
-    download_status_increment_failure(dls, status_code, cp, server, now);
-  } SMARTLIST_FOREACH_END(cp);
-
-  /* No need to relaunch descriptor downloads here: we already do it
-   * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
-}
-
-/** Called when a connection to download microdescriptors has failed in whole
- * or in part. <b>failed</b> is a list of every microdesc digest we didn't
- * get. <b>status_code</b> is the http status code we received. Reschedule the
- * microdesc downloads as appropriate. */
-static void
-dir_microdesc_download_failed(smartlist_t *failed,
-                              int status_code)
-{
-  networkstatus_t *consensus
-    = networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
-  routerstatus_t *rs;
-  download_status_t *dls;
-  time_t now = time(NULL);
-  int server = directory_fetches_from_authorities(get_options());
-
-  if (! consensus)
-    return;
-  SMARTLIST_FOREACH_BEGIN(failed, const char *, d) {
-    rs = router_get_mutable_consensus_status_by_descriptor_digest(consensus,d);
-    if (!rs)
-      continue;
-    dls = &rs->dl_status;
-    if (dls->n_download_failures >=
-        get_options()->TestingMicrodescMaxDownloadTries)
-      continue;
-    {
-      char buf[BASE64_DIGEST256_LEN+1];
-      digest256_to_base64(buf, d);
-      download_status_increment_failure(dls, status_code, buf,
-                                        server, now);
-    }
-  } SMARTLIST_FOREACH_END(d);
-}
-
-/** Helper.  Compare two fp_pair_t objects, and return negative, 0, or
- * positive as appropriate. */
-static int
-compare_pairs_(const void **a, const void **b)
-{
-  const fp_pair_t *fp1 = *a, *fp2 = *b;
-  int r;
-  if ((r = fast_memcmp(fp1->first, fp2->first, DIGEST_LEN)))
-    return r;
-  else
-    return fast_memcmp(fp1->second, fp2->second, DIGEST_LEN);
-}
-
-/** Divide a string <b>res</b> of the form FP1-FP2+FP3-FP4...[.z], where each
- * FP is a hex-encoded fingerprint, into a sequence of distinct sorted
- * fp_pair_t. Skip malformed pairs. On success, return 0 and add those
- * fp_pair_t into <b>pairs_out</b>.  On failure, return -1. */
-int
-dir_split_resource_into_fingerprint_pairs(const char *res,
-                                          smartlist_t *pairs_out)
-{
-  smartlist_t *pairs_tmp = smartlist_new();
-  smartlist_t *pairs_result = smartlist_new();
-
-  smartlist_split_string(pairs_tmp, res, "+", 0, 0);
-  if (smartlist_len(pairs_tmp)) {
-    char *last = smartlist_get(pairs_tmp,smartlist_len(pairs_tmp)-1);
-    size_t last_len = strlen(last);
-    if (last_len > 2 && !strcmp(last+last_len-2, ".z")) {
-      last[last_len-2] = '\0';
-    }
-  }
-  SMARTLIST_FOREACH_BEGIN(pairs_tmp, char *, cp) {
-    if (strlen(cp) != HEX_DIGEST_LEN*2+1) {
-      log_info(LD_DIR,
-             "Skipping digest pair %s with non-standard length.", escaped(cp));
-    } else if (cp[HEX_DIGEST_LEN] != '-') {
-      log_info(LD_DIR,
-             "Skipping digest pair %s with missing dash.", escaped(cp));
-    } else {
-      fp_pair_t pair;
-      if (base16_decode(pair.first, DIGEST_LEN,
-                        cp, HEX_DIGEST_LEN) != DIGEST_LEN ||
-          base16_decode(pair.second,DIGEST_LEN,
-                        cp+HEX_DIGEST_LEN+1, HEX_DIGEST_LEN) != DIGEST_LEN) {
-        log_info(LD_DIR, "Skipping non-decodable digest pair %s", escaped(cp));
-      } else {
-        smartlist_add(pairs_result, tor_memdup(&pair, sizeof(pair)));
-      }
-    }
-    tor_free(cp);
-  } SMARTLIST_FOREACH_END(cp);
-  smartlist_free(pairs_tmp);
-
-  /* Uniq-and-sort */
-  smartlist_sort(pairs_result, compare_pairs_);
-  smartlist_uniq(pairs_result, compare_pairs_, tor_free_);
-
-  smartlist_add_all(pairs_out, pairs_result);
-  smartlist_free(pairs_result);
-  return 0;
-}
-
-/** Given a directory <b>resource</b> request, containing zero
- * or more strings separated by plus signs, followed optionally by ".z", store
- * the strings, in order, into <b>fp_out</b>.  If <b>compressed_out</b> is
- * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0.
- *
- * If (flags & DSR_HEX), then delete all elements that aren't hex digests, and
- * decode the rest.  If (flags & DSR_BASE64), then use "-" rather than "+" as
- * a separator, delete all the elements that aren't base64-encoded digests,
- * and decode the rest.  If (flags & DSR_DIGEST256), these digests should be
- * 256 bits long; else they should be 160.
- *
- * If (flags & DSR_SORT_UNIQ), then sort the list and remove all duplicates.
- */
-int
-dir_split_resource_into_fingerprints(const char *resource,
-                                     smartlist_t *fp_out, int *compressed_out,
-                                     int flags)
-{
-  const int decode_hex = flags & DSR_HEX;
-  const int decode_base64 = flags & DSR_BASE64;
-  const int digests_are_256 = flags & DSR_DIGEST256;
-  const int sort_uniq = flags & DSR_SORT_UNIQ;
-
-  const int digest_len = digests_are_256 ? DIGEST256_LEN : DIGEST_LEN;
-  const int hex_digest_len = digests_are_256 ?
-    HEX_DIGEST256_LEN : HEX_DIGEST_LEN;
-  const int base64_digest_len = digests_are_256 ?
-    BASE64_DIGEST256_LEN : BASE64_DIGEST_LEN;
-  smartlist_t *fp_tmp = smartlist_new();
-
-  tor_assert(!(decode_hex && decode_base64));
-  tor_assert(fp_out);
-
-  smartlist_split_string(fp_tmp, resource, decode_base64?"-":"+", 0, 0);
-  if (compressed_out)
-    *compressed_out = 0;
-  if (smartlist_len(fp_tmp)) {
-    char *last = smartlist_get(fp_tmp,smartlist_len(fp_tmp)-1);
-    size_t last_len = strlen(last);
-    if (last_len > 2 && !strcmp(last+last_len-2, ".z")) {
-      last[last_len-2] = '\0';
-      if (compressed_out)
-        *compressed_out = 1;
-    }
-  }
-  if (decode_hex || decode_base64) {
-    const size_t encoded_len = decode_hex ? hex_digest_len : base64_digest_len;
-    int i;
-    char *cp, *d = NULL;
-    for (i = 0; i < smartlist_len(fp_tmp); ++i) {
-      cp = smartlist_get(fp_tmp, i);
-      if (strlen(cp) != encoded_len) {
-        log_info(LD_DIR,
-                 "Skipping digest %s with non-standard length.", escaped(cp));
-        smartlist_del_keeporder(fp_tmp, i--);
-        goto again;
-      }
-      d = tor_malloc_zero(digest_len);
-      if (decode_hex ?
-          (base16_decode(d, digest_len, cp, hex_digest_len) != digest_len) :
-          (base64_decode(d, digest_len, cp, base64_digest_len)
-                         != digest_len)) {
-          log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp));
-          smartlist_del_keeporder(fp_tmp, i--);
-          goto again;
-      }
-      smartlist_set(fp_tmp, i, d);
-      d = NULL;
-    again:
-      tor_free(cp);
-      tor_free(d);
-    }
-  }
-  if (sort_uniq) {
-    if (decode_hex || decode_base64) {
-      if (digests_are_256) {
-        smartlist_sort_digests256(fp_tmp);
-        smartlist_uniq_digests256(fp_tmp);
-      } else {
-        smartlist_sort_digests(fp_tmp);
-        smartlist_uniq_digests(fp_tmp);
-      }
-    } else {
-      smartlist_sort_strings(fp_tmp);
-      smartlist_uniq_strings(fp_tmp);
-    }
-  }
-  smartlist_add_all(fp_out, fp_tmp);
-  smartlist_free(fp_tmp);
-  return 0;
-}
-
diff --git a/src/or/directory.h b/src/or/directory.h
deleted file mode 100644
index 629b3ead90..0000000000
--- a/src/or/directory.h
+++ /dev/null
@@ -1,175 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file directory.h
- * \brief Header file for directory.c.
- **/
-
-#ifndef TOR_DIRECTORY_H
-#define TOR_DIRECTORY_H
-
-int directories_have_accepted_server_descriptor(void);
-void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
-                                  dirinfo_type_t type, const char *payload,
-                                  size_t payload_len, size_t extrainfo_len);
-MOCK_DECL(void, directory_get_from_dirserver, (
-                          uint8_t dir_purpose,
-                          uint8_t router_purpose,
-                          const char *resource,
-                          int pds_flags,
-                          download_want_authority_t want_authority));
-void directory_get_from_all_authorities(uint8_t dir_purpose,
-                                        uint8_t router_purpose,
-                                        const char *resource);
-
-/** Enumeration of ways to connect to a directory server */
-typedef enum {
-  /** Default: connect over a one-hop Tor circuit. Relays fall back to direct
-   * DirPort connections, clients, onion services, and bridges do not */
-  DIRIND_ONEHOP=0,
-  /** Connect over a multi-hop anonymizing Tor circuit */
-  DIRIND_ANONYMOUS=1,
-  /** Connect to the DirPort directly */
-  DIRIND_DIRECT_CONN,
-  /** Connect over a multi-hop anonymizing Tor circuit to our dirport */
-  DIRIND_ANON_DIRPORT,
-} dir_indirection_t;
-
-int directory_must_use_begindir(const or_options_t *options);
-
-MOCK_DECL(void, directory_initiate_command_routerstatus,
-                (const routerstatus_t *status,
-                 uint8_t dir_purpose,
-                 uint8_t router_purpose,
-                 dir_indirection_t indirection,
-                 const char *resource,
-                 const char *payload,
-                 size_t payload_len,
-                 time_t if_modified_since));
-
-void directory_initiate_command_routerstatus_rend(const routerstatus_t *status,
-                                                  uint8_t dir_purpose,
-                                                  uint8_t router_purpose,
-                                                 dir_indirection_t indirection,
-                                                  const char *resource,
-                                                  const char *payload,
-                                                  size_t payload_len,
-                                                  time_t if_modified_since,
-                                                const rend_data_t *rend_query);
-
-int parse_http_response(const char *headers, int *code, time_t *date,
-                        compress_method_t *compression, char **response);
-
-int connection_dir_is_encrypted(dir_connection_t *conn);
-int connection_dir_reached_eof(dir_connection_t *conn);
-int connection_dir_process_inbuf(dir_connection_t *conn);
-int connection_dir_finished_flushing(dir_connection_t *conn);
-int connection_dir_finished_connecting(dir_connection_t *conn);
-void connection_dir_about_to_close(dir_connection_t *dir_conn);
-void directory_initiate_command(const tor_addr_t *or_addr, uint16_t or_port,
-                                const tor_addr_t *dir_addr, uint16_t dir_port,
-                                const char *digest,
-                                uint8_t dir_purpose, uint8_t router_purpose,
-                                dir_indirection_t indirection,
-                                const char *resource,
-                                const char *payload, size_t payload_len,
-                                time_t if_modified_since);
-
-#define DSR_HEX       (1<<0)
-#define DSR_BASE64    (1<<1)
-#define DSR_DIGEST256 (1<<2)
-#define DSR_SORT_UNIQ (1<<3)
-int dir_split_resource_into_fingerprints(const char *resource,
-                                     smartlist_t *fp_out, int *compressed_out,
-                                     int flags);
-
-int dir_split_resource_into_fingerprint_pairs(const char *res,
-                                              smartlist_t *pairs_out);
-char *directory_dump_request_log(void);
-void note_request(const char *key, size_t bytes);
-int router_supports_extrainfo(const char *identity_digest, int is_authority);
-
-time_t download_status_increment_failure(download_status_t *dls,
-                                         int status_code, const char *item,
-                                         int server, time_t now);
-time_t download_status_increment_attempt(download_status_t *dls,
-                                         const char *item,  time_t now);
-/** Increment the failure count of the download_status_t <b>dls</b>, with
- * the optional status code <b>sc</b>. */
-#define download_status_failed(dls, sc)                                 \
-  download_status_increment_failure((dls), (sc), NULL,                  \
-                                    dir_server_mode(get_options()), \
-                                    time(NULL))
-
-void download_status_reset(download_status_t *dls);
-static int download_status_is_ready(download_status_t *dls, time_t now,
-                                    int max_failures);
-/** Return true iff, as of <b>now</b>, the resource tracked by <b>dls</b> is
- * ready to get its download reattempted. */
-static inline int
-download_status_is_ready(download_status_t *dls, time_t now,
-                         int max_failures)
-{
-  if (dls->backoff == DL_SCHED_DETERMINISTIC) {
-    /* Deterministic schedules can hit an endpoint; exponential backoff
-     * schedules just wait longer and longer. */
-    int under_failure_limit = (dls->n_download_failures <= max_failures
-                               && dls->n_download_attempts <= max_failures);
-    if (!under_failure_limit)
-      return 0;
-  }
-  return dls->next_attempt_at <= now;
-}
-
-static void download_status_mark_impossible(download_status_t *dl);
-/** Mark <b>dl</b> as never downloadable. */
-static inline void
-download_status_mark_impossible(download_status_t *dl)
-{
-  dl->n_download_failures = IMPOSSIBLE_TO_DOWNLOAD;
-  dl->n_download_attempts = IMPOSSIBLE_TO_DOWNLOAD;
-}
-
-int download_status_get_n_failures(const download_status_t *dls);
-int download_status_get_n_attempts(const download_status_t *dls);
-time_t download_status_get_next_attempt_at(const download_status_t *dls);
-
-/* Yes, these two functions are confusingly similar.
- * Let's sort that out in #20077. */
-int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose);
-int is_sensitive_dir_purpose(uint8_t dir_purpose);
-
-#ifdef TOR_UNIT_TESTS
-/* Used only by directory.c and test_dir.c */
-
-STATIC int parse_http_url(const char *headers, char **url);
-STATIC dirinfo_type_t dir_fetch_type(int dir_purpose, int router_purpose,
-                                     const char *resource);
-STATIC int directory_handle_command_get(dir_connection_t *conn,
-                                        const char *headers,
-                                        const char *req_body,
-                                        size_t req_body_len);
-STATIC int download_status_schedule_get_delay(download_status_t *dls,
-                                              const smartlist_t *schedule,
-                                              int min_delay, int max_delay,
-                                              time_t now);
-
-STATIC char* authdir_type_to_string(dirinfo_type_t auth);
-STATIC const char * dir_conn_purpose_to_string(int purpose);
-STATIC int should_use_directory_guards(const or_options_t *options);
-STATIC zlib_compression_level_t choose_compression_level(ssize_t n_bytes);
-STATIC const smartlist_t *find_dl_schedule(download_status_t *dls,
-                                           const or_options_t *options);
-STATIC void find_dl_min_and_max_delay(download_status_t *dls,
-                                      const or_options_t *options,
-                                      int *min, int *max);
-STATIC int next_random_exponential_delay(int delay, int max_delay);
-
-#endif
-
-#endif
-
diff --git a/src/or/dirserv.c b/src/or/dirserv.c
deleted file mode 100644
index 177009208d..0000000000
--- a/src/or/dirserv.c
+++ /dev/null
@@ -1,3913 +0,0 @@
-/* Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#define DIRSERV_PRIVATE
-#include "or.h"
-#include "buffers.h"
-#include "config.h"
-#include "confparse.h"
-#include "channel.h"
-#include "channeltls.h"
-#include "command.h"
-#include "connection.h"
-#include "connection_or.h"
-#include "control.h"
-#include "directory.h"
-#include "dirserv.h"
-#include "dirvote.h"
-#include "hibernate.h"
-#include "keypin.h"
-#include "main.h"
-#include "microdesc.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "policies.h"
-#include "protover.h"
-#include "rephist.h"
-#include "router.h"
-#include "routerlist.h"
-#include "routerparse.h"
-#include "routerset.h"
-#include "torcert.h"
-
-/**
- * \file dirserv.c
- * \brief Directory server core implementation. Manages directory
- * contents and generates directories.
- *
- * This module implements most of directory cache functionality, and some of
- * the directory authority functionality.  The directory.c module delegates
- * here in order to handle incoming requests from clients, via
- * connection_dirserv_flushed_some() and its kin.  In order to save RAM, this
- * module is reponsible for spooling directory objects (in whole or in part)
- * onto buf_t instances, and then closing the dir_connection_t once the
- * objects are totally flushed.
- *
- * The directory.c module also delegates here for handling descriptor uploads
- * via dirserv_add_multiple_descriptors().
- *
- * Additionally, this module handles some aspects of voting, including:
- * deciding how to vote on individual flags (based on decisions reached in
- * rephist.c), of formatting routerstatus lines, and deciding what relays to
- * include in an authority's vote.  (TODO: Those functions could profitably be
- * split off.  They only live in this file because historically they were
- * shared among the v1, v2, and v3 directory code.)
- */
-
-/** How far in the future do we allow a router to get? (seconds) */
-#define ROUTER_ALLOW_SKEW (60*60*12)
-/** How many seconds do we wait before regenerating the directory? */
-#define DIR_REGEN_SLACK_TIME 30
-/** If we're a cache, keep this many networkstatuses around from non-trusted
- * directory authorities. */
-#define MAX_UNTRUSTED_NETWORKSTATUSES 16
-
-/** Total number of routers with measured bandwidth; this is set by
- * dirserv_count_measured_bws() before the loop in
- * dirserv_generate_networkstatus_vote_obj() and checked by
- * dirserv_get_credible_bandwidth() and
- * dirserv_compute_performance_thresholds() */
-static int routers_with_measured_bw = 0;
-
-static void directory_remove_invalid(void);
-struct authdir_config_t;
-static uint32_t
-dirserv_get_status_impl(const char *fp, const char *nickname,
-                        uint32_t addr, uint16_t or_port,
-                        const char *platform, const char **msg,
-                        int severity);
-static void clear_cached_dir(cached_dir_t *d);
-static const signed_descriptor_t *get_signed_descriptor_by_fp(
-                                                        const char *fp,
-                                                        int extrainfo,
-                                                        time_t publish_cutoff);
-static was_router_added_t dirserv_add_extrainfo(extrainfo_t *ei,
-                                                const char **msg);
-static uint32_t dirserv_get_bandwidth_for_router_kb(const routerinfo_t *ri);
-static uint32_t dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri);
-
-/************** Fingerprint handling code ************/
-
-/*                 1  Historically used to indicate Named */
-#define FP_INVALID 2  /**< Believed invalid. */
-#define FP_REJECT  4  /**< We will not publish this router. */
-/*                 8  Historically used to avoid using this as a dir. */
-#define FP_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */
-/*                 32 Historically used to indicade Unnamed */
-
-/** Target of status_by_digest map. */
-typedef uint32_t router_status_t;
-
-static void add_fingerprint_to_dir(const char *fp,
-                                   struct authdir_config_t *list,
-                                   router_status_t add_status);
-
-/** List of nickname-\>identity fingerprint mappings for all the routers
- * that we name.  Used to prevent router impersonation. */
-typedef struct authdir_config_t {
-  strmap_t *fp_by_name; /**< Map from lc nickname to fingerprint. */
-  digestmap_t *status_by_digest; /**< Map from digest to router_status_t. */
-} authdir_config_t;
-
-/** Should be static; exposed for testing. */
-static authdir_config_t *fingerprint_list = NULL;
-
-/** Allocate and return a new, empty, authdir_config_t. */
-static authdir_config_t *
-authdir_config_new(void)
-{
-  authdir_config_t *list = tor_malloc_zero(sizeof(authdir_config_t));
-  list->fp_by_name = strmap_new();
-  list->status_by_digest = digestmap_new();
-  return list;
-}
-
-/** Add the fingerprint <b>fp</b> to the smartlist of fingerprint_entry_t's
- * <b>list</b>, or-ing the currently set status flags with
- * <b>add_status</b>.
- */
-/* static */ void
-add_fingerprint_to_dir(const char *fp, authdir_config_t *list,
-                       router_status_t add_status)
-{
-  char *fingerprint;
-  char d[DIGEST_LEN];
-  router_status_t *status;
-  tor_assert(fp);
-  tor_assert(list);
-
-  fingerprint = tor_strdup(fp);
-  tor_strstrip(fingerprint, " ");
-  if (base16_decode(d, DIGEST_LEN,
-                    fingerprint, strlen(fingerprint)) != DIGEST_LEN) {
-    log_warn(LD_DIRSERV, "Couldn't decode fingerprint \"%s\"",
-             escaped(fp));
-    tor_free(fingerprint);
-    return;
-  }
-
-  status = digestmap_get(list->status_by_digest, d);
-  if (!status) {
-    status = tor_malloc_zero(sizeof(router_status_t));
-    digestmap_set(list->status_by_digest, d, status);
-  }
-
-  tor_free(fingerprint);
-  *status |= add_status;
-  return;
-}
-
-/** Add the fingerprint for this OR to the global list of recognized
- * identity key fingerprints. */
-int
-dirserv_add_own_fingerprint(crypto_pk_t *pk)
-{
-  char fp[FINGERPRINT_LEN+1];
-  if (crypto_pk_get_fingerprint(pk, fp, 0)<0) {
-    log_err(LD_BUG, "Error computing fingerprint");
-    return -1;
-  }
-  if (!fingerprint_list)
-    fingerprint_list = authdir_config_new();
-  add_fingerprint_to_dir(fp, fingerprint_list, 0);
-  return 0;
-}
-
-/** Load the nickname-\>fingerprint mappings stored in the approved-routers
- * file.  The file format is line-based, with each non-blank holding one
- * nickname, some space, and a fingerprint for that nickname.  On success,
- * replace the current fingerprint list with the new list and return 0.  On
- * failure, leave the current fingerprint list untouched, and return -1. */
-int
-dirserv_load_fingerprint_file(void)
-{
-  char *fname;
-  char *cf;
-  char *nickname, *fingerprint;
-  authdir_config_t *fingerprint_list_new;
-  int result;
-  config_line_t *front=NULL, *list;
-
-  fname = get_datadir_fname("approved-routers");
-  log_info(LD_GENERAL,
-           "Reloading approved fingerprints from \"%s\"...", fname);
-
-  cf = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
-  if (!cf) {
-    log_warn(LD_FS, "Cannot open fingerprint file '%s'. That's ok.", fname);
-    tor_free(fname);
-    return 0;
-  }
-  tor_free(fname);
-
-  result = config_get_lines(cf, &front, 0);
-  tor_free(cf);
-  if (result < 0) {
-    log_warn(LD_CONFIG, "Error reading from fingerprint file");
-    return -1;
-  }
-
-  fingerprint_list_new = authdir_config_new();
-
-  for (list=front; list; list=list->next) {
-    char digest_tmp[DIGEST_LEN];
-    router_status_t add_status = 0;
-    nickname = list->key; fingerprint = list->value;
-    tor_strstrip(fingerprint, " "); /* remove spaces */
-    if (strlen(fingerprint) != HEX_DIGEST_LEN ||
-        base16_decode(digest_tmp, sizeof(digest_tmp),
-                      fingerprint, HEX_DIGEST_LEN) != sizeof(digest_tmp)) {
-      log_notice(LD_CONFIG,
-                 "Invalid fingerprint (nickname '%s', "
-                 "fingerprint %s). Skipping.",
-                 nickname, fingerprint);
-      continue;
-    }
-    if (!strcasecmp(nickname, "!reject")) {
-        add_status = FP_REJECT;
-    } else if (!strcasecmp(nickname, "!badexit")) {
-        add_status = FP_BADEXIT;
-    } else if (!strcasecmp(nickname, "!invalid")) {
-        add_status = FP_INVALID;
-    }
-    add_fingerprint_to_dir(fingerprint, fingerprint_list_new, add_status);
-  }
-
-  config_free_lines(front);
-  dirserv_free_fingerprint_list();
-  fingerprint_list = fingerprint_list_new;
-  /* Delete any routers whose fingerprints we no longer recognize */
-  directory_remove_invalid();
-  return 0;
-}
-
-/* If this is set, then we don't allow routers that have advertised an Ed25519
- * identity to stop doing so.  This is going to be essential for good identity
- * security: otherwise anybody who can attack RSA-1024 but not Ed25519 could
- * just sign fake descriptors missing the Ed25519 key.  But we won't actually
- * be able to prevent that kind of thing until we're confident that there
- * isn't actually a legit reason to downgrade to 0.2.5.  So for now, we have
- * to leave this #undef.
- */
-#undef DISABLE_DISABLING_ED25519
-
-/** Check whether <b>router</b> has a nickname/identity key combination that
- * we recognize from the fingerprint list, or an IP we automatically act on
- * according to our configuration.  Return the appropriate router status.
- *
- * If the status is 'FP_REJECT' and <b>msg</b> is provided, set
- * *<b>msg</b> to an explanation of why. */
-uint32_t
-dirserv_router_get_status(const routerinfo_t *router, const char **msg,
-                          int severity)
-{
-  char d[DIGEST_LEN];
-  const int key_pinning = get_options()->AuthDirPinKeys;
-
-  if (crypto_pk_get_digest(router->identity_pkey, d)) {
-    log_warn(LD_BUG,"Error computing fingerprint");
-    if (msg)
-      *msg = "Bug: Error computing fingerprint";
-    return FP_REJECT;
-  }
-
-  /* dirserv_get_status_impl already rejects versions older than 0.2.4.18-rc,
-   * and onion_curve25519_pkey was introduced in 0.2.4.8-alpha.
-   * But just in case a relay doesn't provide or lies about its version, or
-   * doesn't include an ntor key in its descriptor, check that it exists,
-   * and is non-zero (clients check that it's non-zero before using it). */
-  if (!routerinfo_has_curve25519_onion_key(router)) {
-    log_fn(severity, LD_DIR,
-           "Descriptor from router %s is missing an ntor curve25519 onion "
-           "key.", router_describe(router));
-    if (msg)
-      *msg = "Missing ntor curve25519 onion key. Please upgrade!";
-    return FP_REJECT;
-  }
-
-  if (router->cache_info.signing_key_cert) {
-    /* This has an ed25519 identity key. */
-    if (KEYPIN_MISMATCH ==
-        keypin_check((const uint8_t*)router->cache_info.identity_digest,
-                   router->cache_info.signing_key_cert->signing_key.pubkey)) {
-      log_fn(severity, LD_DIR,
-             "Descriptor from router %s has an Ed25519 key, "
-               "but the <rsa,ed25519> keys don't match what they were before.",
-               router_describe(router));
-      if (key_pinning) {
-        if (msg) {
-          *msg = "Ed25519 identity key or RSA identity key has changed.";
-        }
-        return FP_REJECT;
-      }
-    }
-  } else {
-    /* No ed25519 key */
-    if (KEYPIN_MISMATCH == keypin_check_lone_rsa(
-                        (const uint8_t*)router->cache_info.identity_digest)) {
-      log_fn(severity, LD_DIR,
-               "Descriptor from router %s has no Ed25519 key, "
-               "when we previously knew an Ed25519 for it. Ignoring for now, "
-               "since Ed25519 keys are fairly new.",
-               router_describe(router));
-#ifdef DISABLE_DISABLING_ED25519
-      if (key_pinning) {
-        if (msg) {
-          *msg = "Ed25519 identity key has disappeared.";
-        }
-        return FP_REJECT;
-      }
-#endif
-    }
-  }
-
-  return dirserv_get_status_impl(d, router->nickname,
-                                 router->addr, router->or_port,
-                                 router->platform, msg, severity);
-}
-
-/** Return true if there is no point in downloading the router described by
- * <b>rs</b> because this directory would reject it. */
-int
-dirserv_would_reject_router(const routerstatus_t *rs)
-{
-  uint32_t res;
-
-  res = dirserv_get_status_impl(rs->identity_digest, rs->nickname,
-                                rs->addr, rs->or_port,
-                                NULL, NULL, LOG_DEBUG);
-
-  return (res & FP_REJECT) != 0;
-}
-
-/** Helper: As dirserv_router_get_status, but takes the router fingerprint
- * (hex, no spaces), nickname, address (used for logging only), IP address, OR
- * port and platform (logging only) as arguments.
- *
- * Log messages at 'severity'. (There's not much point in
- * logging that we're rejecting servers we'll not download.)
- */
-static uint32_t
-dirserv_get_status_impl(const char *id_digest, const char *nickname,
-                        uint32_t addr, uint16_t or_port,
-                        const char *platform, const char **msg, int severity)
-{
-  uint32_t result = 0;
-  router_status_t *status_by_digest;
-
-  if (!fingerprint_list)
-    fingerprint_list = authdir_config_new();
-
-  log_debug(LD_DIRSERV, "%d fingerprints, %d digests known.",
-            strmap_size(fingerprint_list->fp_by_name),
-            digestmap_size(fingerprint_list->status_by_digest));
-
-  if (platform) {
-    tor_version_t ver_tmp;
-    if (tor_version_parse_platform(platform, &ver_tmp, 1) < 0) {
-      if (msg) {
-        *msg = "Malformed platform string.";
-      }
-      return FP_REJECT;
-    }
-  }
-
-  /* Versions before Tor 0.2.4.18-rc are too old to support, and are
-   * missing some important security fixes too. Disable them. */
-  if (platform && !tor_version_as_new_as(platform,"0.2.4.18-rc")) {
-    if (msg)
-      *msg = "Tor version is insecure or unsupported. Please upgrade!";
-    return FP_REJECT;
-  }
-
-  /* Tor 0.2.9.x where x<5 suffers from bug #20499, where relays don't
-   * keep their consensus up to date so they make bad guards.
-   * The simple fix is to just drop them from the network. */
-  if (platform &&
-      tor_version_as_new_as(platform,"0.2.9.0-alpha") &&
-      !tor_version_as_new_as(platform,"0.2.9.5-alpha")) {
-    if (msg)
-      *msg = "Tor version contains bug 20499. Please upgrade!";
-    return FP_REJECT;
-  }
-
-  status_by_digest = digestmap_get(fingerprint_list->status_by_digest,
-                                   id_digest);
-  if (status_by_digest)
-    result |= *status_by_digest;
-
-  if (result & FP_REJECT) {
-    if (msg)
-      *msg = "Fingerprint is marked rejected -- if you think this is a "
-             "mistake please set a valid email address in ContactInfo and "
-             "send an email to bad-relays@lists.torproject.org mentioning "
-             "your fingerprint(s)?";
-    return FP_REJECT;
-  } else if (result & FP_INVALID) {
-    if (msg)
-      *msg = "Fingerprint is marked invalid";
-  }
-
-  if (authdir_policy_badexit_address(addr, or_port)) {
-    log_fn(severity, LD_DIRSERV,
-           "Marking '%s' as bad exit because of address '%s'",
-               nickname, fmt_addr32(addr));
-    result |= FP_BADEXIT;
-  }
-
-  if (!authdir_policy_permits_address(addr, or_port)) {
-    log_fn(severity, LD_DIRSERV, "Rejecting '%s' because of address '%s'",
-               nickname, fmt_addr32(addr));
-    if (msg)
-      *msg = "Suspicious relay address range -- if you think this is a "
-             "mistake please set a valid email address in ContactInfo and "
-             "send an email to bad-relays@lists.torproject.org mentioning "
-             "your address(es) and fingerprint(s)?";
-    return FP_REJECT;
-  }
-  if (!authdir_policy_valid_address(addr, or_port)) {
-    log_fn(severity, LD_DIRSERV,
-           "Not marking '%s' valid because of address '%s'",
-               nickname, fmt_addr32(addr));
-    result |= FP_INVALID;
-  }
-
-  return result;
-}
-
-/** Clear the current fingerprint list. */
-void
-dirserv_free_fingerprint_list(void)
-{
-  if (!fingerprint_list)
-    return;
-
-  strmap_free(fingerprint_list->fp_by_name, tor_free_);
-  digestmap_free(fingerprint_list->status_by_digest, tor_free_);
-  tor_free(fingerprint_list);
-}
-
-/*
- *    Descriptor list
- */
-
-/** Return -1 if <b>ri</b> has a private or otherwise bad address,
- * unless we're configured to not care. Return 0 if all ok. */
-static int
-dirserv_router_has_valid_address(routerinfo_t *ri)
-{
-  tor_addr_t addr;
-  if (get_options()->DirAllowPrivateAddresses)
-    return 0; /* whatever it is, we're fine with it */
-  tor_addr_from_ipv4h(&addr, ri->addr);
-
-  if (tor_addr_is_internal(&addr, 0)) {
-    log_info(LD_DIRSERV,
-             "Router %s published internal IP address. Refusing.",
-             router_describe(ri));
-    return -1; /* it's a private IP, we should reject it */
-  }
-  return 0;
-}
-
-/** Check whether we, as a directory server, want to accept <b>ri</b>.  If so,
- * set its is_valid,running fields and return 0.  Otherwise, return -1.
- *
- * If the router is rejected, set *<b>msg</b> to an explanation of why.
- *
- * If <b>complain</b> then explain at log-level 'notice' why we refused
- * a descriptor; else explain at log-level 'info'.
- */
-int
-authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
-                               int complain, int *valid_out)
-{
-  /* Okay.  Now check whether the fingerprint is recognized. */
-  time_t now;
-  int severity = (complain && ri->contact_info) ? LOG_NOTICE : LOG_INFO;
-  uint32_t status = dirserv_router_get_status(ri, msg, severity);
-  tor_assert(msg);
-  if (status & FP_REJECT)
-    return -1; /* msg is already set. */
-
-  /* Is there too much clock skew? */
-  now = time(NULL);
-  if (ri->cache_info.published_on > now+ROUTER_ALLOW_SKEW) {
-    log_fn(severity, LD_DIRSERV, "Publication time for %s is too "
-           "far (%d minutes) in the future; possible clock skew. Not adding "
-           "(%s)",
-           router_describe(ri),
-           (int)((ri->cache_info.published_on-now)/60),
-           esc_router_info(ri));
-    *msg = "Rejected: Your clock is set too far in the future, or your "
-      "timezone is not correct.";
-    return -1;
-  }
-  if (ri->cache_info.published_on < now-ROUTER_MAX_AGE_TO_PUBLISH) {
-    log_fn(severity, LD_DIRSERV,
-           "Publication time for %s is too far "
-           "(%d minutes) in the past. Not adding (%s)",
-           router_describe(ri),
-           (int)((now-ri->cache_info.published_on)/60),
-           esc_router_info(ri));
-    *msg = "Rejected: Server is expired, or your clock is too far in the past,"
-      " or your timezone is not correct.";
-    return -1;
-  }
-  if (dirserv_router_has_valid_address(ri) < 0) {
-    log_fn(severity, LD_DIRSERV,
-           "Router %s has invalid address. Not adding (%s).",
-           router_describe(ri),
-           esc_router_info(ri));
-    *msg = "Rejected: Address is a private address.";
-    return -1;
-  }
-
-  *valid_out = ! (status & FP_INVALID);
-
-  return 0;
-}
-
-/** Update the relevant flags of <b>node</b> based on our opinion as a
- * directory authority in <b>authstatus</b>, as returned by
- * dirserv_router_get_status or equivalent.  */
-void
-dirserv_set_node_flags_from_authoritative_status(node_t *node,
-                                                 uint32_t authstatus)
-{
-  node->is_valid = (authstatus & FP_INVALID) ? 0 : 1;
-  node->is_bad_exit = (authstatus & FP_BADEXIT) ? 1 : 0;
-}
-
-/** True iff <b>a</b> is more severe than <b>b</b>. */
-static int
-WRA_MORE_SEVERE(was_router_added_t a, was_router_added_t b)
-{
-  return a < b;
-}
-
-/** As for dirserv_add_descriptor(), but accepts multiple documents, and
- * returns the most severe error that occurred for any one of them. */
-was_router_added_t
-dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
-                                 const char *source,
-                                 const char **msg)
-{
-  was_router_added_t r, r_tmp;
-  const char *msg_out;
-  smartlist_t *list;
-  const char *s;
-  int n_parsed = 0;
-  time_t now = time(NULL);
-  char annotation_buf[ROUTER_ANNOTATION_BUF_LEN];
-  char time_buf[ISO_TIME_LEN+1];
-  int general = purpose == ROUTER_PURPOSE_GENERAL;
-  tor_assert(msg);
-
-  r=ROUTER_ADDED_SUCCESSFULLY; /*Least severe return value. */
-
-  format_iso_time(time_buf, now);
-  if (tor_snprintf(annotation_buf, sizeof(annotation_buf),
-                   "@uploaded-at %s\n"
-                   "@source %s\n"
-                   "%s%s%s", time_buf, escaped(source),
-                   !general ? "@purpose " : "",
-                   !general ? router_purpose_to_string(purpose) : "",
-                   !general ? "\n" : "")<0) {
-    *msg = "Couldn't format annotations";
-    return -1;
-  }
-
-  s = desc;
-  list = smartlist_new();
-  if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 0, 0,
-                                     annotation_buf, NULL)) {
-    SMARTLIST_FOREACH(list, routerinfo_t *, ri, {
-        msg_out = NULL;
-        tor_assert(ri->purpose == purpose);
-        r_tmp = dirserv_add_descriptor(ri, &msg_out, source);
-        if (WRA_MORE_SEVERE(r_tmp, r)) {
-          r = r_tmp;
-          *msg = msg_out;
-        }
-      });
-  }
-  n_parsed += smartlist_len(list);
-  smartlist_clear(list);
-
-  s = desc;
-  if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 1, 0,
-                                     NULL, NULL)) {
-    SMARTLIST_FOREACH(list, extrainfo_t *, ei, {
-        msg_out = NULL;
-
-        r_tmp = dirserv_add_extrainfo(ei, &msg_out);
-        if (WRA_MORE_SEVERE(r_tmp, r)) {
-          r = r_tmp;
-          *msg = msg_out;
-        }
-      });
-  }
-  n_parsed += smartlist_len(list);
-  smartlist_free(list);
-
-  if (! *msg) {
-    if (!n_parsed) {
-      *msg = "No descriptors found in your POST.";
-      if (WRA_WAS_ADDED(r))
-        r = ROUTER_IS_ALREADY_KNOWN;
-    } else {
-      *msg = "(no message)";
-    }
-  }
-
-  return r;
-}
-
-/** Examine the parsed server descriptor in <b>ri</b> and maybe insert it into
- * the list of server descriptors. Set *<b>msg</b> to a message that should be
- * passed back to the origin of this descriptor, or NULL if there is no such
- * message. Use <b>source</b> to produce better log messages.
- *
- * If <b>ri</b> is not added to the list of server descriptors, free it.
- * That means the caller must not access <b>ri</b> after this function
- * returns, since it might have been freed.
- *
- * Return the status of the operation.
- *
- * This function is only called when fresh descriptors are posted, not when
- * we re-load the cache.
- */
-was_router_added_t
-dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source)
-{
-  was_router_added_t r;
-  routerinfo_t *ri_old;
-  char *desc, *nickname;
-  const size_t desclen = ri->cache_info.signed_descriptor_len +
-      ri->cache_info.annotations_len;
-  const int key_pinning = get_options()->AuthDirPinKeys;
-  *msg = NULL;
-
-  /* If it's too big, refuse it now. Otherwise we'll cache it all over the
-   * network and it'll clog everything up. */
-  if (ri->cache_info.signed_descriptor_len > MAX_DESCRIPTOR_UPLOAD_SIZE) {
-    log_notice(LD_DIR, "Somebody attempted to publish a router descriptor '%s'"
-               " (source: %s) with size %d. Either this is an attack, or the "
-               "MAX_DESCRIPTOR_UPLOAD_SIZE (%d) constant is too low.",
-               ri->nickname, source, (int)ri->cache_info.signed_descriptor_len,
-               MAX_DESCRIPTOR_UPLOAD_SIZE);
-    *msg = "Router descriptor was too large.";
-    control_event_or_authdir_new_descriptor("REJECTED",
-               ri->cache_info.signed_descriptor_body,
-                                            desclen, *msg);
-    r = ROUTER_AUTHDIR_REJECTS;
-    goto fail;
-  }
-
-  /* Check whether this descriptor is semantically identical to the last one
-   * from this server.  (We do this here and not in router_add_to_routerlist
-   * because we want to be able to accept the newest router descriptor that
-   * another authority has, so we all converge on the same one.) */
-  ri_old = router_get_mutable_by_digest(ri->cache_info.identity_digest);
-  if (ri_old && ri_old->cache_info.published_on < ri->cache_info.published_on
-      && router_differences_are_cosmetic(ri_old, ri)
-      && !router_is_me(ri)) {
-    log_info(LD_DIRSERV,
-             "Not replacing descriptor from %s (source: %s); "
-             "differences are cosmetic.",
-             router_describe(ri), source);
-    *msg = "Not replacing router descriptor; no information has changed since "
-      "the last one with this identity.";
-    control_event_or_authdir_new_descriptor("DROPPED",
-                         ri->cache_info.signed_descriptor_body,
-                                            desclen, *msg);
-    r = ROUTER_IS_ALREADY_KNOWN;
-    goto fail;
-  }
-
-  /* Do keypinning again ... this time, to add the pin if appropriate */
-  int keypin_status;
-  if (ri->cache_info.signing_key_cert) {
-    keypin_status = keypin_check_and_add(
-      (const uint8_t*)ri->cache_info.identity_digest,
-      ri->cache_info.signing_key_cert->signing_key.pubkey,
-      ! key_pinning);
-  } else {
-    keypin_status = keypin_check_lone_rsa(
-      (const uint8_t*)ri->cache_info.identity_digest);
-#ifndef DISABLE_DISABLING_ED25519
-    if (keypin_status == KEYPIN_MISMATCH)
-      keypin_status = KEYPIN_NOT_FOUND;
-#endif
-  }
-  if (keypin_status == KEYPIN_MISMATCH && key_pinning) {
-    log_info(LD_DIRSERV, "Dropping descriptor from %s (source: %s) because "
-             "its key did not match an older RSA/Ed25519 keypair",
-             router_describe(ri), source);
-    *msg = "Looks like your keypair does not match its older value.";
-    r = ROUTER_AUTHDIR_REJECTS;
-    goto fail;
-  }
-
-  /* Make a copy of desc, since router_add_to_routerlist might free
-   * ri and its associated signed_descriptor_t. */
-  desc = tor_strndup(ri->cache_info.signed_descriptor_body, desclen);
-  nickname = tor_strdup(ri->nickname);
-
-  /* Tell if we're about to need to launch a test if we add this. */
-  ri->needs_retest_if_added =
-    dirserv_should_launch_reachability_test(ri, ri_old);
-
-  r = router_add_to_routerlist(ri, msg, 0, 0);
-  if (!WRA_WAS_ADDED(r)) {
-    /* unless the routerinfo was fine, just out-of-date */
-    if (WRA_WAS_REJECTED(r))
-      control_event_or_authdir_new_descriptor("REJECTED", desc, desclen, *msg);
-    log_info(LD_DIRSERV,
-             "Did not add descriptor from '%s' (source: %s): %s.",
-             nickname, source, *msg ? *msg : "(no message)");
-  } else {
-    smartlist_t *changed;
-    control_event_or_authdir_new_descriptor("ACCEPTED", desc, desclen, *msg);
-
-    changed = smartlist_new();
-    smartlist_add(changed, ri);
-    routerlist_descriptors_added(changed, 0);
-    smartlist_free(changed);
-    if (!*msg) {
-      *msg =  "Descriptor accepted";
-    }
-    log_info(LD_DIRSERV,
-             "Added descriptor from '%s' (source: %s): %s.",
-             nickname, source, *msg);
-  }
-  tor_free(desc);
-  tor_free(nickname);
-  return r;
- fail:
-  {
-    const char *desc_digest = ri->cache_info.signed_descriptor_digest;
-    download_status_t *dls =
-      router_get_dl_status_by_descriptor_digest(desc_digest);
-    if (dls) {
-      log_info(LD_GENERAL, "Marking router with descriptor %s as rejected, "
-               "and therefore undownloadable",
-               hex_str(desc_digest, DIGEST_LEN));
-      download_status_mark_impossible(dls);
-    }
-    routerinfo_free(ri);
-  }
-  return r;
-}
-
-/** As dirserv_add_descriptor, but for an extrainfo_t <b>ei</b>. */
-static was_router_added_t
-dirserv_add_extrainfo(extrainfo_t *ei, const char **msg)
-{
-  routerinfo_t *ri;
-  int r;
-  was_router_added_t rv;
-  tor_assert(msg);
-  *msg = NULL;
-
-  /* Needs to be mutable so routerinfo_incompatible_with_extrainfo
-   * can mess with some of the flags in ri->cache_info. */
-  ri = router_get_mutable_by_digest(ei->cache_info.identity_digest);
-  if (!ri) {
-    *msg = "No corresponding router descriptor for extra-info descriptor";
-    rv = ROUTER_BAD_EI;
-    goto fail;
-  }
-
-  /* If it's too big, refuse it now. Otherwise we'll cache it all over the
-   * network and it'll clog everything up. */
-  if (ei->cache_info.signed_descriptor_len > MAX_EXTRAINFO_UPLOAD_SIZE) {
-    log_notice(LD_DIR, "Somebody attempted to publish an extrainfo "
-               "with size %d. Either this is an attack, or the "
-               "MAX_EXTRAINFO_UPLOAD_SIZE (%d) constant is too low.",
-               (int)ei->cache_info.signed_descriptor_len,
-               MAX_EXTRAINFO_UPLOAD_SIZE);
-    *msg = "Extrainfo document was too large";
-    rv = ROUTER_BAD_EI;
-    goto fail;
-  }
-
-  if ((r = routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei,
-                                                  &ri->cache_info, msg))) {
-    if (r<0) {
-      extrainfo_free(ei);
-      return ROUTER_IS_ALREADY_KNOWN;
-    }
-    rv = ROUTER_BAD_EI;
-    goto fail;
-  }
-  router_add_extrainfo_to_routerlist(ei, msg, 0, 0);
-  return ROUTER_ADDED_SUCCESSFULLY;
- fail:
-  {
-    const char *d = ei->cache_info.signed_descriptor_digest;
-    signed_descriptor_t *sd = router_get_by_extrainfo_digest((char*)d);
-    if (sd) {
-      log_info(LD_GENERAL, "Marking extrainfo with descriptor %s as "
-               "rejected, and therefore undownloadable",
-               hex_str((char*)d,DIGEST_LEN));
-      download_status_mark_impossible(&sd->ei_dl_status);
-    }
-    extrainfo_free(ei);
-  }
-  return rv;
-}
-
-/** Remove all descriptors whose nicknames or fingerprints no longer
- * are allowed by our fingerprint list. (Descriptors that used to be
- * good can become bad when we reload the fingerprint list.)
- */
-static void
-directory_remove_invalid(void)
-{
-  routerlist_t *rl = router_get_routerlist();
-  smartlist_t *nodes = smartlist_new();
-  smartlist_add_all(nodes, nodelist_get_list());
-
-  SMARTLIST_FOREACH_BEGIN(nodes, node_t *, node) {
-    const char *msg = NULL;
-    routerinfo_t *ent = node->ri;
-    char description[NODE_DESC_BUF_LEN];
-    uint32_t r;
-    if (!ent)
-      continue;
-    r = dirserv_router_get_status(ent, &msg, LOG_INFO);
-    router_get_description(description, ent);
-    if (r & FP_REJECT) {
-      log_info(LD_DIRSERV, "Router %s is now rejected: %s",
-               description, msg?msg:"");
-      routerlist_remove(rl, ent, 0, time(NULL));
-      continue;
-    }
-    if (bool_neq((r & FP_INVALID), !node->is_valid)) {
-      log_info(LD_DIRSERV, "Router '%s' is now %svalid.", description,
-               (r&FP_INVALID) ? "in" : "");
-      node->is_valid = (r&FP_INVALID)?0:1;
-    }
-    if (bool_neq((r & FP_BADEXIT), node->is_bad_exit)) {
-      log_info(LD_DIRSERV, "Router '%s' is now a %s exit", description,
-               (r & FP_BADEXIT) ? "bad" : "good");
-      node->is_bad_exit = (r&FP_BADEXIT) ? 1: 0;
-    }
-  } SMARTLIST_FOREACH_END(node);
-
-  routerlist_assert_ok(rl);
-  smartlist_free(nodes);
-}
-
-/**
- * Allocate and return a description of the status of the server <b>desc</b>,
- * for use in a v1-style router-status line.  The server is listed
- * as running iff <b>is_live</b> is true.
- */
-static char *
-list_single_server_status(const routerinfo_t *desc, int is_live)
-{
-  char buf[MAX_NICKNAME_LEN+HEX_DIGEST_LEN+4]; /* !nickname=$hexdigest\0 */
-  char *cp;
-  const node_t *node;
-
-  tor_assert(desc);
-
-  cp = buf;
-  if (!is_live) {
-    *cp++ = '!';
-  }
-  node = node_get_by_id(desc->cache_info.identity_digest);
-  if (node && node->is_valid) {
-    strlcpy(cp, desc->nickname, sizeof(buf)-(cp-buf));
-    cp += strlen(cp);
-    *cp++ = '=';
-  }
-  *cp++ = '$';
-  base16_encode(cp, HEX_DIGEST_LEN+1, desc->cache_info.identity_digest,
-                DIGEST_LEN);
-  return tor_strdup(buf);
-}
-
-/* DOCDOC running_long_enough_to_decide_unreachable */
-static inline int
-running_long_enough_to_decide_unreachable(void)
-{
-  return time_of_process_start
-    + get_options()->TestingAuthDirTimeToLearnReachability < approx_time();
-}
-
-/** Each server needs to have passed a reachability test no more
- * than this number of seconds ago, or it is listed as down in
- * the directory. */
-#define REACHABLE_TIMEOUT (45*60)
-
-/** If we tested a router and found it reachable _at least this long_ after it
- * declared itself hibernating, it is probably done hibernating and we just
- * missed a descriptor from it. */
-#define HIBERNATION_PUBLICATION_SKEW (60*60)
-
-/** Treat a router as alive if
- *    - It's me, and I'm not hibernating.
- * or - We've found it reachable recently. */
-void
-dirserv_set_router_is_running(routerinfo_t *router, time_t now)
-{
-  /*XXXX This function is a mess.  Separate out the part that calculates
-    whether it's reachable and the part that tells rephist that the router was
-    unreachable.
-   */
-  int answer;
-  const or_options_t *options = get_options();
-  node_t *node = node_get_mutable_by_id(router->cache_info.identity_digest);
-  tor_assert(node);
-
-  if (router_is_me(router)) {
-    /* We always know if we are down ourselves. */
-    answer = ! we_are_hibernating();
-  } else if (router->is_hibernating &&
-             (router->cache_info.published_on +
-              HIBERNATION_PUBLICATION_SKEW) > node->last_reachable) {
-    /* A hibernating router is down unless we (somehow) had contact with it
-     * since it declared itself to be hibernating. */
-    answer = 0;
-  } else if (options->AssumeReachable) {
-    /* If AssumeReachable, everybody is up unless they say they are down! */
-    answer = 1;
-  } else {
-    /* Otherwise, a router counts as up if we found all announced OR
-       ports reachable in the last REACHABLE_TIMEOUT seconds.
-
-       XXX prop186 For now there's always one IPv4 and at most one
-       IPv6 OR port.
-
-       If we're not on IPv6, don't consider reachability of potential
-       IPv6 OR port since that'd kill all dual stack relays until a
-       majority of the dir auths have IPv6 connectivity. */
-    answer = (now < node->last_reachable + REACHABLE_TIMEOUT &&
-              (options->AuthDirHasIPv6Connectivity != 1 ||
-               tor_addr_is_null(&router->ipv6_addr) ||
-               now < node->last_reachable6 + REACHABLE_TIMEOUT));
-  }
-
-  if (!answer && running_long_enough_to_decide_unreachable()) {
-    /* Not considered reachable. tell rephist about that.
-
-       Because we launch a reachability test for each router every
-       REACHABILITY_TEST_CYCLE_PERIOD seconds, then the router has probably
-       been down since at least that time after we last successfully reached
-       it.
-
-       XXX ipv6
-     */
-    time_t when = now;
-    if (node->last_reachable &&
-        node->last_reachable + REACHABILITY_TEST_CYCLE_PERIOD < now)
-      when = node->last_reachable + REACHABILITY_TEST_CYCLE_PERIOD;
-    rep_hist_note_router_unreachable(router->cache_info.identity_digest, when);
-  }
-
-  node->is_running = answer;
-}
-
-/** Based on the routerinfo_ts in <b>routers</b>, allocate the
- * contents of a v1-style router-status line, and store it in
- * *<b>router_status_out</b>.  Return 0 on success, -1 on failure.
- *
- * If for_controller is true, include the routers with very old descriptors.
- */
-int
-list_server_status_v1(smartlist_t *routers, char **router_status_out,
-                      int for_controller)
-{
-  /* List of entries in a router-status style: An optional !, then an optional
-   * equals-suffixed nickname, then a dollar-prefixed hexdigest. */
-  smartlist_t *rs_entries;
-  time_t now = time(NULL);
-  time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
-  const or_options_t *options = get_options();
-  /* We include v2 dir auths here too, because they need to answer
-   * controllers. Eventually we'll deprecate this whole function;
-   * see also networkstatus_getinfo_by_purpose(). */
-  int authdir = authdir_mode_publishes_statuses(options);
-  tor_assert(router_status_out);
-
-  rs_entries = smartlist_new();
-
-  SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
-    const node_t *node = node_get_by_id(ri->cache_info.identity_digest);
-    tor_assert(node);
-    if (authdir) {
-      /* Update router status in routerinfo_t. */
-      dirserv_set_router_is_running(ri, now);
-    }
-    if (for_controller) {
-      char name_buf[MAX_VERBOSE_NICKNAME_LEN+2];
-      char *cp = name_buf;
-      if (!node->is_running)
-        *cp++ = '!';
-      router_get_verbose_nickname(cp, ri);
-      smartlist_add(rs_entries, tor_strdup(name_buf));
-    } else if (ri->cache_info.published_on >= cutoff) {
-      smartlist_add(rs_entries, list_single_server_status(ri,
-                                                          node->is_running));
-    }
-  } SMARTLIST_FOREACH_END(ri);
-
-  *router_status_out = smartlist_join_strings(rs_entries, " ", 0, NULL);
-
-  SMARTLIST_FOREACH(rs_entries, char *, cp, tor_free(cp));
-  smartlist_free(rs_entries);
-
-  return 0;
-}
-
-/** Given a (possibly empty) list of config_line_t, each line of which contains
- * a list of comma-separated version numbers surrounded by optional space,
- * allocate and return a new string containing the version numbers, in order,
- * separated by commas.  Used to generate Recommended(Client|Server)?Versions
- */
-char *
-format_recommended_version_list(const config_line_t *ln, int warn)
-{
-  smartlist_t *versions;
-  char *result;
-  versions = smartlist_new();
-  for ( ; ln; ln = ln->next) {
-    smartlist_split_string(versions, ln->value, ",",
-                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  }
-
-  /* Handle the case where a dirauth operator has accidentally made some
-   * versions space-separated instead of comma-separated. */
-  smartlist_t *more_versions = smartlist_new();
-  SMARTLIST_FOREACH_BEGIN(versions, char *, v) {
-    if (strchr(v, ' ')) {
-      if (warn)
-        log_warn(LD_DIRSERV, "Unexpected space in versions list member %s. "
-                 "(These are supposed to be comma-separated; I'll pretend you "
-                 "used commas instead.)", escaped(v));
-      SMARTLIST_DEL_CURRENT(versions, v);
-      smartlist_split_string(more_versions, v, NULL,
-                             SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-      tor_free(v);
-    }
-  } SMARTLIST_FOREACH_END(v);
-  smartlist_add_all(versions, more_versions);
-  smartlist_free(more_versions);
-
-  /* Check to make sure everything looks like a version. */
-  if (warn) {
-    SMARTLIST_FOREACH_BEGIN(versions, const char *, v) {
-      tor_version_t ver;
-      if (tor_version_parse(v, &ver) < 0) {
-        log_warn(LD_DIRSERV, "Recommended version %s does not look valid. "
-                 " (I'll include it anyway, since you told me to.)",
-                 escaped(v));
-      }
-    } SMARTLIST_FOREACH_END(v);
-  }
-
-  sort_version_list(versions, 1);
-  result = smartlist_join_strings(versions,",",0,NULL);
-  SMARTLIST_FOREACH(versions,char *,s,tor_free(s));
-  smartlist_free(versions);
-  return result;
-}
-
-/** Return 1 if <b>ri</b>'s descriptor is "active" -- running, valid,
- * not hibernating, having observed bw greater 0, and not too old. Else
- * return 0.
- */
-static int
-router_is_active(const routerinfo_t *ri, const node_t *node, time_t now)
-{
-  time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
-  if (ri->cache_info.published_on < cutoff) {
-    return 0;
-  }
-  if (!node->is_running || !node->is_valid || ri->is_hibernating) {
-    return 0;
-  }
-  /* Only require bandwith capacity in non-test networks, or
-   * if TestingTorNetwork, and TestingMinExitFlagThreshold is non-zero */
-  if (!ri->bandwidthcapacity) {
-    if (get_options()->TestingTorNetwork) {
-      if (get_options()->TestingMinExitFlagThreshold > 0) {
-        /* If we're in a TestingTorNetwork, and TestingMinExitFlagThreshold is,
-         * then require bandwidthcapacity */
-        return 0;
-      }
-    } else {
-      /* If we're not in a TestingTorNetwork, then require bandwidthcapacity */
-      return 0;
-    }
-  }
-  return 1;
-}
-
-/********************************************************************/
-
-/* A set of functions to answer questions about how we'd like to behave
- * as a directory mirror/client. */
-
-/** Return 1 if we fetch our directory material directly from the
- * authorities, rather than from a mirror. */
-int
-directory_fetches_from_authorities(const or_options_t *options)
-{
-  const routerinfo_t *me;
-  uint32_t addr;
-  int refuseunknown;
-  if (options->FetchDirInfoEarly)
-    return 1;
-  if (options->BridgeRelay == 1)
-    return 0;
-  if (server_mode(options) &&
-      router_pick_published_address(options, &addr, 1) < 0)
-    return 1; /* we don't know our IP address; ask an authority. */
-  refuseunknown = ! router_my_exit_policy_is_reject_star() &&
-    should_refuse_unknown_exits(options);
-  if (!dir_server_mode(options) && !refuseunknown)
-    return 0;
-  if (!server_mode(options) || !advertised_server_mode())
-    return 0;
-  me = router_get_my_routerinfo();
-  if (!me || (!me->supports_tunnelled_dir_requests && !refuseunknown))
-    return 0; /* if we don't service directory requests, return 0 too */
-  return 1;
-}
-
-/** Return 1 if we should fetch new networkstatuses, descriptors, etc
- * on the "mirror" schedule rather than the "client" schedule.
- */
-int
-directory_fetches_dir_info_early(const or_options_t *options)
-{
-  return directory_fetches_from_authorities(options);
-}
-
-/** Return 1 if we should fetch new networkstatuses, descriptors, etc
- * on a very passive schedule -- waiting long enough for ordinary clients
- * to probably have the info we want. These would include bridge users,
- * and maybe others in the future e.g. if a Tor client uses another Tor
- * client as a directory guard.
- */
-int
-directory_fetches_dir_info_later(const or_options_t *options)
-{
-  return options->UseBridges != 0;
-}
-
-/** Return true iff we want to fetch and keep certificates for authorities
- * that we don't acknowledge as authorities ourself.
- */
-int
-directory_caches_unknown_auth_certs(const or_options_t *options)
-{
-  return dir_server_mode(options) || options->BridgeRelay;
-}
-
-/** Return 1 if we want to keep descriptors, networkstatuses, etc around.
- * Else return 0.
- * Check options->DirPort_set and directory_permits_begindir_requests()
- * to see if we are willing to serve these directory documents to others via
- * the DirPort and begindir-over-ORPort, respectively.
- */
-int
-directory_caches_dir_info(const or_options_t *options)
-{
-  if (options->BridgeRelay || dir_server_mode(options))
-    return 1;
-  if (!server_mode(options) || !advertised_server_mode())
-    return 0;
-  /* We need an up-to-date view of network info if we're going to try to
-   * block exit attempts from unknown relays. */
-  return ! router_my_exit_policy_is_reject_star() &&
-    should_refuse_unknown_exits(options);
-}
-
-/** Return 1 if we want to allow remote people to ask us directory
- * requests via the "begin_dir" interface, which doesn't require
- * having any separate port open. */
-int
-directory_permits_begindir_requests(const or_options_t *options)
-{
-  return options->BridgeRelay != 0 || dir_server_mode(options);
-}
-
-/** Return 1 if we have no need to fetch new descriptors. This generally
- * happens when we're not a dir cache and we haven't built any circuits
- * lately.
- */
-int
-directory_too_idle_to_fetch_descriptors(const or_options_t *options,
-                                        time_t now)
-{
-  return !directory_caches_dir_info(options) &&
-         !options->FetchUselessDescriptors &&
-         rep_hist_circbuilding_dormant(now);
-}
-
-/********************************************************************/
-
-/** Map from flavor name to the cached_dir_t for the v3 consensuses that we're
- * currently serving. */
-static strmap_t *cached_consensuses = NULL;
-
-/** Decrement the reference count on <b>d</b>, and free it if it no longer has
- * any references. */
-void
-cached_dir_decref(cached_dir_t *d)
-{
-  if (!d || --d->refcnt > 0)
-    return;
-  clear_cached_dir(d);
-  tor_free(d);
-}
-
-/** Allocate and return a new cached_dir_t containing the string <b>s</b>,
- * published at <b>published</b>. */
-cached_dir_t *
-new_cached_dir(char *s, time_t published)
-{
-  cached_dir_t *d = tor_malloc_zero(sizeof(cached_dir_t));
-  d->refcnt = 1;
-  d->dir = s;
-  d->dir_len = strlen(s);
-  d->published = published;
-  if (tor_gzip_compress(&(d->dir_z), &(d->dir_z_len), d->dir, d->dir_len,
-                        ZLIB_METHOD)) {
-    log_warn(LD_BUG, "Error compressing directory");
-  }
-  return d;
-}
-
-/** Remove all storage held in <b>d</b>, but do not free <b>d</b> itself. */
-static void
-clear_cached_dir(cached_dir_t *d)
-{
-  tor_free(d->dir);
-  tor_free(d->dir_z);
-  memset(d, 0, sizeof(cached_dir_t));
-}
-
-/** Free all storage held by the cached_dir_t in <b>d</b>. */
-static void
-free_cached_dir_(void *_d)
-{
-  cached_dir_t *d;
-  if (!_d)
-    return;
-
-  d = (cached_dir_t *)_d;
-  cached_dir_decref(d);
-}
-
-/** Replace the v3 consensus networkstatus of type <b>flavor_name</b> that
- * we're serving with <b>networkstatus</b>, published at <b>published</b>.  No
- * validation is performed. */
-void
-dirserv_set_cached_consensus_networkstatus(const char *networkstatus,
-                                           const char *flavor_name,
-                                           const common_digests_t *digests,
-                                           time_t published)
-{
-  cached_dir_t *new_networkstatus;
-  cached_dir_t *old_networkstatus;
-  if (!cached_consensuses)
-    cached_consensuses = strmap_new();
-
-  new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published);
-  memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t));
-  old_networkstatus = strmap_set(cached_consensuses, flavor_name,
-                                 new_networkstatus);
-  if (old_networkstatus)
-    cached_dir_decref(old_networkstatus);
-}
-
-/** Return the latest downloaded consensus networkstatus in encoded, signed,
- * optionally compressed format, suitable for sending to clients. */
-cached_dir_t *
-dirserv_get_consensus(const char *flavor_name)
-{
-  if (!cached_consensuses)
-    return NULL;
-  return strmap_get(cached_consensuses, flavor_name);
-}
-
-/** If a router's uptime is at least this value, then it is always
- * considered stable, regardless of the rest of the network. This
- * way we resist attacks where an attacker doubles the size of the
- * network using allegedly high-uptime nodes, displacing all the
- * current guards. */
-#define UPTIME_TO_GUARANTEE_STABLE (3600*24*30)
-/** If a router's MTBF is at least this value, then it is always stable.
- * See above.  (Corresponds to about 7 days for current decay rates.) */
-#define MTBF_TO_GUARANTEE_STABLE (60*60*24*5)
-/** Similarly, every node with at least this much weighted time known can be
- * considered familiar enough to be a guard.  Corresponds to about 20 days for
- * current decay rates.
- */
-#define TIME_KNOWN_TO_GUARANTEE_FAMILIAR (8*24*60*60)
-/** Similarly, every node with sufficient WFU is around enough to be a guard.
- */
-#define WFU_TO_GUARANTEE_GUARD (0.98)
-
-/* Thresholds for server performance: set by
- * dirserv_compute_performance_thresholds, and used by
- * generate_v2_networkstatus */
-
-/** Any router with an uptime of at least this value is stable. */
-static uint32_t stable_uptime = 0; /* start at a safe value */
-/** Any router with an mtbf of at least this value is stable. */
-static double stable_mtbf = 0.0;
-/** If true, we have measured enough mtbf info to look at stable_mtbf rather
- * than stable_uptime. */
-static int enough_mtbf_info = 0;
-/** Any router with a weighted fractional uptime of at least this much might
- * be good as a guard. */
-static double guard_wfu = 0.0;
-/** Don't call a router a guard unless we've known about it for at least this
- * many seconds. */
-static long guard_tk = 0;
-/** Any router with a bandwidth at least this high is "Fast" */
-static uint32_t fast_bandwidth_kb = 0;
-/** If exits can be guards, then all guards must have a bandwidth this
- * high. */
-static uint32_t guard_bandwidth_including_exits_kb = 0;
-/** If exits can't be guards, then all guards must have a bandwidth this
- * high. */
-static uint32_t guard_bandwidth_excluding_exits_kb = 0;
-
-/** Helper: estimate the uptime of a router given its stated uptime and the
- * amount of time since it last stated its stated uptime. */
-static inline long
-real_uptime(const routerinfo_t *router, time_t now)
-{
-  if (now < router->cache_info.published_on)
-    return router->uptime;
-  else
-    return router->uptime + (now - router->cache_info.published_on);
-}
-
-/** Return 1 if <b>router</b> is not suitable for these parameters, else 0.
- * If <b>need_uptime</b> is non-zero, we require a minimum uptime.
- * If <b>need_capacity</b> is non-zero, we require a minimum advertised
- * bandwidth.
- */
-static int
-dirserv_thinks_router_is_unreliable(time_t now,
-                                    routerinfo_t *router,
-                                    int need_uptime, int need_capacity)
-{
-  if (need_uptime) {
-    if (!enough_mtbf_info) {
-      /* XXXX We should change the rule from
-       * "use uptime if we don't have mtbf data" to "don't advertise Stable on
-       * v3 if we don't have enough mtbf data."  Or maybe not, since if we ever
-       * hit a point where we need to reset a lot of authorities at once,
-       * none of them would be in a position to declare Stable.
-       */
-      long uptime = real_uptime(router, now);
-      if ((unsigned)uptime < stable_uptime &&
-          (unsigned)uptime < UPTIME_TO_GUARANTEE_STABLE)
-        return 1;
-    } else {
-      double mtbf =
-        rep_hist_get_stability(router->cache_info.identity_digest, now);
-      if (mtbf < stable_mtbf &&
-          mtbf < MTBF_TO_GUARANTEE_STABLE)
-        return 1;
-    }
-  }
-  if (need_capacity) {
-    uint32_t bw_kb = dirserv_get_credible_bandwidth_kb(router);
-    if (bw_kb < fast_bandwidth_kb)
-      return 1;
-  }
-  return 0;
-}
-
-/** Return true iff <b>router</b> should be assigned the "HSDir" flag.
- *
- * Right now this means it advertises support for it, it has a high uptime,
- * it's a directory cache, it has the Stable and Fast flags, and it's currently
- * considered Running.
- *
- * This function needs to be called after router-\>is_running has
- * been set.
- */
-static int
-dirserv_thinks_router_is_hs_dir(const routerinfo_t *router,
-                                const node_t *node, time_t now)
-{
-
-  long uptime;
-
-  /* If we haven't been running for at least
-   * get_options()->MinUptimeHidServDirectoryV2 seconds, we can't
-   * have accurate data telling us a relay has been up for at least
-   * that long. We also want to allow a bit of slack: Reachability
-   * tests aren't instant. If we haven't been running long enough,
-   * trust the relay. */
-
-  if (stats_n_seconds_working >
-      get_options()->MinUptimeHidServDirectoryV2 * 1.1)
-    uptime = MIN(rep_hist_get_uptime(router->cache_info.identity_digest, now),
-                 real_uptime(router, now));
-  else
-    uptime = real_uptime(router, now);
-
-  return (router->wants_to_be_hs_dir &&
-          router->supports_tunnelled_dir_requests &&
-          node->is_stable && node->is_fast &&
-          uptime >= get_options()->MinUptimeHidServDirectoryV2 &&
-          router_is_active(router, node, now));
-}
-
-/** Don't consider routers with less bandwidth than this when computing
- * thresholds. */
-#define ABSOLUTE_MIN_BW_VALUE_TO_CONSIDER_KB 4
-
-/** Helper for dirserv_compute_performance_thresholds(): Decide whether to
- * include a router in our calculations, and return true iff we should; the
- * require_mbw parameter is passed in by
- * dirserv_compute_performance_thresholds() and controls whether we ever
- * count routers with only advertised bandwidths */
-static int
-router_counts_toward_thresholds(const node_t *node, time_t now,
-                                const digestmap_t *omit_as_sybil,
-                                int require_mbw)
-{
-  /* Have measured bw? */
-  int have_mbw =
-    dirserv_has_measured_bw(node->identity);
-  uint64_t min_bw_kb = ABSOLUTE_MIN_BW_VALUE_TO_CONSIDER_KB;
-  const or_options_t *options = get_options();
-
-  if (options->TestingTorNetwork) {
-    min_bw_kb = (int64_t)options->TestingMinExitFlagThreshold / 1000;
-  }
-
-  return node->ri && router_is_active(node->ri, node, now) &&
-    !digestmap_get(omit_as_sybil, node->identity) &&
-    (dirserv_get_credible_bandwidth_kb(node->ri) >= min_bw_kb) &&
-    (have_mbw || !require_mbw);
-}
-
-/** Look through the routerlist, the Mean Time Between Failure history, and
- * the Weighted Fractional Uptime history, and use them to set thresholds for
- * the Stable, Fast, and Guard flags.  Update the fields stable_uptime,
- * stable_mtbf, enough_mtbf_info, guard_wfu, guard_tk, fast_bandwidth,
- * guard_bandwidth_including_exits, and guard_bandwidth_excluding_exits.
- *
- * Also, set the is_exit flag of each router appropriately. */
-static void
-dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil)
-{
-  int n_active, n_active_nonexit, n_familiar;
-  uint32_t *uptimes, *bandwidths_kb, *bandwidths_excluding_exits_kb;
-  long *tks;
-  double *mtbfs, *wfus;
-  smartlist_t *nodelist;
-  time_t now = time(NULL);
-  const or_options_t *options = get_options();
-
-  /* Require mbw? */
-  int require_mbw =
-    (routers_with_measured_bw >
-     options->MinMeasuredBWsForAuthToIgnoreAdvertised) ? 1 : 0;
-
-  /* initialize these all here, in case there are no routers */
-  stable_uptime = 0;
-  stable_mtbf = 0;
-  fast_bandwidth_kb = 0;
-  guard_bandwidth_including_exits_kb = 0;
-  guard_bandwidth_excluding_exits_kb = 0;
-  guard_tk = 0;
-  guard_wfu = 0;
-
-  nodelist_assert_ok();
-  nodelist = nodelist_get_list();
-
-  /* Initialize arrays that will hold values for each router.  We'll
-   * sort them and use that to compute thresholds. */
-  n_active = n_active_nonexit = 0;
-  /* Uptime for every active router. */
-  uptimes = tor_calloc(smartlist_len(nodelist), sizeof(uint32_t));
-  /* Bandwidth for every active router. */
-  bandwidths_kb = tor_calloc(smartlist_len(nodelist), sizeof(uint32_t));
-  /* Bandwidth for every active non-exit router. */
-  bandwidths_excluding_exits_kb =
-    tor_calloc(smartlist_len(nodelist), sizeof(uint32_t));
-  /* Weighted mean time between failure for each active router. */
-  mtbfs = tor_calloc(smartlist_len(nodelist), sizeof(double));
-  /* Time-known for each active router. */
-  tks = tor_calloc(smartlist_len(nodelist), sizeof(long));
-  /* Weighted fractional uptime for each active router. */
-  wfus = tor_calloc(smartlist_len(nodelist), sizeof(double));
-
-  /* Now, fill in the arrays. */
-  SMARTLIST_FOREACH_BEGIN(nodelist, node_t *, node) {
-    if (options->BridgeAuthoritativeDir &&
-        node->ri &&
-        node->ri->purpose != ROUTER_PURPOSE_BRIDGE)
-      continue;
-    if (router_counts_toward_thresholds(node, now, omit_as_sybil,
-                                        require_mbw)) {
-      routerinfo_t *ri = node->ri;
-      const char *id = node->identity;
-      uint32_t bw_kb;
-      /* resolve spurious clang shallow analysis null pointer errors */
-      tor_assert(ri);
-      node->is_exit = (!router_exit_policy_rejects_all(ri) &&
-                       exit_policy_is_general_exit(ri->exit_policy));
-      uptimes[n_active] = (uint32_t)real_uptime(ri, now);
-      mtbfs[n_active] = rep_hist_get_stability(id, now);
-      tks  [n_active] = rep_hist_get_weighted_time_known(id, now);
-      bandwidths_kb[n_active] = bw_kb = dirserv_get_credible_bandwidth_kb(ri);
-      if (!node->is_exit || node->is_bad_exit) {
-        bandwidths_excluding_exits_kb[n_active_nonexit] = bw_kb;
-        ++n_active_nonexit;
-      }
-      ++n_active;
-    }
-  } SMARTLIST_FOREACH_END(node);
-
-  /* Now, compute thresholds. */
-  if (n_active) {
-    /* The median uptime is stable. */
-    stable_uptime = median_uint32(uptimes, n_active);
-    /* The median mtbf is stable, if we have enough mtbf info */
-    stable_mtbf = median_double(mtbfs, n_active);
-    /* The 12.5th percentile bandwidth is fast. */
-    fast_bandwidth_kb = find_nth_uint32(bandwidths_kb, n_active, n_active/8);
-    /* (Now bandwidths is sorted.) */
-    if (fast_bandwidth_kb < RELAY_REQUIRED_MIN_BANDWIDTH/(2 * 1000))
-      fast_bandwidth_kb = bandwidths_kb[n_active/4];
-    guard_bandwidth_including_exits_kb =
-      third_quartile_uint32(bandwidths_kb, n_active);
-    guard_tk = find_nth_long(tks, n_active, n_active/8);
-  }
-
-  if (guard_tk > TIME_KNOWN_TO_GUARANTEE_FAMILIAR)
-    guard_tk = TIME_KNOWN_TO_GUARANTEE_FAMILIAR;
-
-  {
-    /* We can vote on a parameter for the minimum and maximum. */
-#define ABSOLUTE_MIN_VALUE_FOR_FAST_FLAG 4
-    int32_t min_fast_kb, max_fast_kb, min_fast, max_fast;
-    min_fast = networkstatus_get_param(NULL, "FastFlagMinThreshold",
-      ABSOLUTE_MIN_VALUE_FOR_FAST_FLAG,
-      ABSOLUTE_MIN_VALUE_FOR_FAST_FLAG,
-      INT32_MAX);
-    if (options->TestingTorNetwork) {
-      min_fast = (int32_t)options->TestingMinFastFlagThreshold;
-    }
-    max_fast = networkstatus_get_param(NULL, "FastFlagMaxThreshold",
-                                       INT32_MAX, min_fast, INT32_MAX);
-    min_fast_kb = min_fast / 1000;
-    max_fast_kb = max_fast / 1000;
-
-    if (fast_bandwidth_kb < (uint32_t)min_fast_kb)
-      fast_bandwidth_kb = min_fast_kb;
-    if (fast_bandwidth_kb > (uint32_t)max_fast_kb)
-      fast_bandwidth_kb = max_fast_kb;
-  }
-  /* Protect sufficiently fast nodes from being pushed out of the set
-   * of Fast nodes. */
-  if (options->AuthDirFastGuarantee &&
-      fast_bandwidth_kb > options->AuthDirFastGuarantee/1000)
-    fast_bandwidth_kb = (uint32_t)options->AuthDirFastGuarantee/1000;
-
-  /* Now that we have a time-known that 7/8 routers are known longer than,
-   * fill wfus with the wfu of every such "familiar" router. */
-  n_familiar = 0;
-
-  SMARTLIST_FOREACH_BEGIN(nodelist, node_t *, node) {
-      if (router_counts_toward_thresholds(node, now,
-                                          omit_as_sybil, require_mbw)) {
-        routerinfo_t *ri = node->ri;
-        const char *id = ri->cache_info.identity_digest;
-        long tk = rep_hist_get_weighted_time_known(id, now);
-        if (tk < guard_tk)
-          continue;
-        wfus[n_familiar++] = rep_hist_get_weighted_fractional_uptime(id, now);
-      }
-  } SMARTLIST_FOREACH_END(node);
-  if (n_familiar)
-    guard_wfu = median_double(wfus, n_familiar);
-  if (guard_wfu > WFU_TO_GUARANTEE_GUARD)
-    guard_wfu = WFU_TO_GUARANTEE_GUARD;
-
-  enough_mtbf_info = rep_hist_have_measured_enough_stability();
-
-  if (n_active_nonexit) {
-    guard_bandwidth_excluding_exits_kb =
-      find_nth_uint32(bandwidths_excluding_exits_kb,
-                      n_active_nonexit, n_active_nonexit*3/4);
-  }
-
-  log_info(LD_DIRSERV,
-      "Cutoffs: For Stable, %lu sec uptime, %lu sec MTBF. "
-      "For Fast: %lu kilobytes/sec. "
-      "For Guard: WFU %.03f%%, time-known %lu sec, "
-      "and bandwidth %lu or %lu kilobytes/sec. "
-      "We%s have enough stability data.",
-      (unsigned long)stable_uptime,
-      (unsigned long)stable_mtbf,
-      (unsigned long)fast_bandwidth_kb,
-      guard_wfu*100,
-      (unsigned long)guard_tk,
-      (unsigned long)guard_bandwidth_including_exits_kb,
-      (unsigned long)guard_bandwidth_excluding_exits_kb,
-      enough_mtbf_info ? "" : " don't");
-
-  tor_free(uptimes);
-  tor_free(mtbfs);
-  tor_free(bandwidths_kb);
-  tor_free(bandwidths_excluding_exits_kb);
-  tor_free(tks);
-  tor_free(wfus);
-}
-
-/* Use dirserv_compute_performance_thresholds() to compute the thresholds
- * for the status flags, specifically for bridges.
- *
- * This is only called by a Bridge Authority from
- * networkstatus_getinfo_by_purpose().
- */
-void
-dirserv_compute_bridge_flag_thresholds(void)
-{
-  digestmap_t *omit_as_sybil = digestmap_new();
-  dirserv_compute_performance_thresholds(omit_as_sybil);
-  digestmap_free(omit_as_sybil, NULL);
-}
-
-/** Measured bandwidth cache entry */
-typedef struct mbw_cache_entry_s {
-  long mbw_kb;
-  time_t as_of;
-} mbw_cache_entry_t;
-
-/** Measured bandwidth cache - keys are identity_digests, values are
- * mbw_cache_entry_t *. */
-static digestmap_t *mbw_cache = NULL;
-
-/** Store a measured bandwidth cache entry when reading the measured
- * bandwidths file. */
-STATIC void
-dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line,
-                          time_t as_of)
-{
-  mbw_cache_entry_t *e = NULL;
-
-  tor_assert(parsed_line);
-
-  /* Allocate a cache if we need */
-  if (!mbw_cache) mbw_cache = digestmap_new();
-
-  /* Check if we have an existing entry */
-  e = digestmap_get(mbw_cache, parsed_line->node_id);
-  /* If we do, we can re-use it */
-  if (e) {
-    /* Check that we really are newer, and update */
-    if (as_of > e->as_of) {
-      e->mbw_kb = parsed_line->bw_kb;
-      e->as_of = as_of;
-    }
-  } else {
-    /* We'll have to insert a new entry */
-    e = tor_malloc(sizeof(*e));
-    e->mbw_kb = parsed_line->bw_kb;
-    e->as_of = as_of;
-    digestmap_set(mbw_cache, parsed_line->node_id, e);
-  }
-}
-
-/** Clear and free the measured bandwidth cache */
-STATIC void
-dirserv_clear_measured_bw_cache(void)
-{
-  if (mbw_cache) {
-    /* Free the map and all entries */
-    digestmap_free(mbw_cache, tor_free_);
-    mbw_cache = NULL;
-  }
-}
-
-/** Scan the measured bandwidth cache and remove expired entries */
-STATIC void
-dirserv_expire_measured_bw_cache(time_t now)
-{
-
-  if (mbw_cache) {
-    /* Iterate through the cache and check each entry */
-    DIGESTMAP_FOREACH_MODIFY(mbw_cache, k, mbw_cache_entry_t *, e) {
-      if (now > e->as_of + MAX_MEASUREMENT_AGE) {
-        tor_free(e);
-        MAP_DEL_CURRENT(k);
-      }
-    } DIGESTMAP_FOREACH_END;
-
-    /* Check if we cleared the whole thing and free if so */
-    if (digestmap_size(mbw_cache) == 0) {
-      digestmap_free(mbw_cache, tor_free_);
-      mbw_cache = 0;
-    }
-  }
-}
-
-/** Get the current size of the measured bandwidth cache */
-STATIC int
-dirserv_get_measured_bw_cache_size(void)
-{
-  if (mbw_cache) return digestmap_size(mbw_cache);
-  else return 0;
-}
-
-/** Query the cache by identity digest, return value indicates whether
- * we found it. The bw_out and as_of_out pointers receive the cached
- * bandwidth value and the time it was cached if not NULL. */
-STATIC int
-dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_kb_out,
-                                   time_t *as_of_out)
-{
-  mbw_cache_entry_t *v = NULL;
-  int rv = 0;
-
-  if (mbw_cache && node_id) {
-    v = digestmap_get(mbw_cache, node_id);
-    if (v) {
-      /* Found something */
-      rv = 1;
-      if (bw_kb_out) *bw_kb_out = v->mbw_kb;
-      if (as_of_out) *as_of_out = v->as_of;
-    }
-  }
-
-  return rv;
-}
-
-/** Predicate wrapper for dirserv_query_measured_bw_cache() */
-STATIC int
-dirserv_has_measured_bw(const char *node_id)
-{
-  return dirserv_query_measured_bw_cache_kb(node_id, NULL, NULL);
-}
-
-/** Get the best estimate of a router's bandwidth for dirauth purposes,
- * preferring measured to advertised values if available. */
-
-static uint32_t
-dirserv_get_bandwidth_for_router_kb(const routerinfo_t *ri)
-{
-  uint32_t bw_kb = 0;
-  /*
-   * Yeah, measured bandwidths in measured_bw_line_t are (implicitly
-   * signed) longs and the ones router_get_advertised_bandwidth() returns
-   * are uint32_t.
-   */
-  long mbw_kb = 0;
-
-  if (ri) {
-    /*
-   * * First try to see if we have a measured bandwidth; don't bother with
-     * as_of_out here, on the theory that a stale measured bandwidth is still
-     * better to trust than an advertised one.
-     */
-    if (dirserv_query_measured_bw_cache_kb(ri->cache_info.identity_digest,
-                                        &mbw_kb, NULL)) {
-      /* Got one! */
-      bw_kb = (uint32_t)mbw_kb;
-    } else {
-      /* If not, fall back to advertised */
-      bw_kb = router_get_advertised_bandwidth(ri) / 1000;
-    }
-  }
-
-  return bw_kb;
-}
-
-/** Look through the routerlist, and using the measured bandwidth cache count
- * how many measured bandwidths we know.  This is used to decide whether we
- * ever trust advertised bandwidths for purposes of assigning flags. */
-static void
-dirserv_count_measured_bws(const smartlist_t *routers)
-{
-  /* Initialize this first */
-  routers_with_measured_bw = 0;
-
-  /* Iterate over the routerlist and count measured bandwidths */
-  SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) {
-    /* Check if we know a measured bandwidth for this one */
-    if (dirserv_has_measured_bw(ri->cache_info.identity_digest)) {
-      ++routers_with_measured_bw;
-    }
-  } SMARTLIST_FOREACH_END(ri);
-}
-
-/** Return the bandwidth we believe for assigning flags; prefer measured
- * over advertised, and if we have above a threshold quantity of measured
- * bandwidths, we don't want to ever give flags to unmeasured routers, so
- * return 0. */
-static uint32_t
-dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri)
-{
-  int threshold;
-  uint32_t bw_kb = 0;
-  long mbw_kb;
-
-  tor_assert(ri);
-  /* Check if we have a measured bandwidth, and check the threshold if not */
-  if (!(dirserv_query_measured_bw_cache_kb(ri->cache_info.identity_digest,
-                                       &mbw_kb, NULL))) {
-    threshold = get_options()->MinMeasuredBWsForAuthToIgnoreAdvertised;
-    if (routers_with_measured_bw > threshold) {
-      /* Return zero for unmeasured bandwidth if we are above threshold */
-      bw_kb = 0;
-    } else {
-      /* Return an advertised bandwidth otherwise */
-      bw_kb = router_get_advertised_bandwidth_capped(ri) / 1000;
-    }
-  } else {
-    /* We have the measured bandwidth in mbw */
-    bw_kb = (uint32_t)mbw_kb;
-  }
-
-  return bw_kb;
-}
-
-/** Give a statement of our current performance thresholds for inclusion
- * in a vote document. */
-char *
-dirserv_get_flag_thresholds_line(void)
-{
-  char *result=NULL;
-  const int measured_threshold =
-    get_options()->MinMeasuredBWsForAuthToIgnoreAdvertised;
-  const int enough_measured_bw = routers_with_measured_bw > measured_threshold;
-
-  tor_asprintf(&result,
-      "stable-uptime=%lu stable-mtbf=%lu "
-      "fast-speed=%lu "
-      "guard-wfu=%.03f%% guard-tk=%lu "
-      "guard-bw-inc-exits=%lu guard-bw-exc-exits=%lu "
-      "enough-mtbf=%d ignoring-advertised-bws=%d",
-      (unsigned long)stable_uptime,
-      (unsigned long)stable_mtbf,
-      (unsigned long)fast_bandwidth_kb*1000,
-      guard_wfu*100,
-      (unsigned long)guard_tk,
-      (unsigned long)guard_bandwidth_including_exits_kb*1000,
-      (unsigned long)guard_bandwidth_excluding_exits_kb*1000,
-      enough_mtbf_info ? 1 : 0,
-      enough_measured_bw ? 1 : 0);
-
-  return result;
-}
-
-/** Given a platform string as in a routerinfo_t (possibly null), return a
- * newly allocated version string for a networkstatus document, or NULL if the
- * platform doesn't give a Tor version. */
-static char *
-version_from_platform(const char *platform)
-{
-  if (platform && !strcmpstart(platform, "Tor ")) {
-    const char *eos = find_whitespace(platform+4);
-    if (eos && !strcmpstart(eos, " (r")) {
-      /* XXXX Unify this logic with the other version extraction
-       * logic in routerparse.c. */
-      eos = find_whitespace(eos+1);
-    }
-    if (eos) {
-      return tor_strndup(platform, eos-platform);
-    }
-  }
-  return NULL;
-}
-
-/** Helper: write the router-status information in <b>rs</b> into a newly
- * allocated character buffer.  Use the same format as in network-status
- * documents.  If <b>version</b> is non-NULL, add a "v" line for the platform.
- * Return 0 on success, -1 on failure.
- *
- * The format argument has one of the following values:
- *   NS_V2 - Output an entry suitable for a V2 NS opinion document
- *   NS_V3_CONSENSUS - Output the first portion of a V3 NS consensus entry
- *   NS_V3_CONSENSUS_MICRODESC - Output the first portion of a V3 microdesc
- *        consensus entry.
- *   NS_V3_VOTE - Output a complete V3 NS vote. If <b>vrs</b> is present,
- *        it contains additional information for the vote.
- *   NS_CONTROL_PORT - Output a NS document for the control port
- */
-char *
-routerstatus_format_entry(const routerstatus_t *rs, const char *version,
-                          const char *protocols,
-                          routerstatus_format_type_t format,
-                          const vote_routerstatus_t *vrs)
-{
-  char *summary;
-  char *result = NULL;
-
-  char published[ISO_TIME_LEN+1];
-  char identity64[BASE64_DIGEST_LEN+1];
-  char digest64[BASE64_DIGEST_LEN+1];
-  smartlist_t *chunks = smartlist_new();
-
-  format_iso_time(published, rs->published_on);
-  digest_to_base64(identity64, rs->identity_digest);
-  digest_to_base64(digest64, rs->descriptor_digest);
-
-  smartlist_add_asprintf(chunks,
-                   "r %s %s %s%s%s %s %d %d\n",
-                   rs->nickname,
-                   identity64,
-                   (format==NS_V3_CONSENSUS_MICRODESC)?"":digest64,
-                   (format==NS_V3_CONSENSUS_MICRODESC)?"":" ",
-                   published,
-                   fmt_addr32(rs->addr),
-                   (int)rs->or_port,
-                   (int)rs->dir_port);
-
-  /* TODO: Maybe we want to pass in what we need to build the rest of
-   * this here, instead of in the caller. Then we could use the
-   * networkstatus_type_t values, with an additional control port value
-   * added -MP */
-
-  /* V3 microdesc consensuses don't have "a" lines. */
-  if (format == NS_V3_CONSENSUS_MICRODESC)
-    goto done;
-
-  /* Possible "a" line. At most one for now. */
-  if (!tor_addr_is_null(&rs->ipv6_addr)) {
-    smartlist_add_asprintf(chunks, "a %s\n",
-                           fmt_addrport(&rs->ipv6_addr, rs->ipv6_orport));
-  }
-
-  if (format == NS_V3_CONSENSUS)
-    goto done;
-
-  smartlist_add_asprintf(chunks,
-                   "s%s%s%s%s%s%s%s%s%s%s\n",
-                  /* These must stay in alphabetical order. */
-                   rs->is_authority?" Authority":"",
-                   rs->is_bad_exit?" BadExit":"",
-                   rs->is_exit?" Exit":"",
-                   rs->is_fast?" Fast":"",
-                   rs->is_possible_guard?" Guard":"",
-                   rs->is_hs_dir?" HSDir":"",
-                   rs->is_flagged_running?" Running":"",
-                   rs->is_stable?" Stable":"",
-                   rs->is_v2_dir?" V2Dir":"",
-                   rs->is_valid?" Valid":"");
-
-  /* length of "opt v \n" */
-#define V_LINE_OVERHEAD 7
-  if (version && strlen(version) < MAX_V_LINE_LEN - V_LINE_OVERHEAD) {
-    smartlist_add_asprintf(chunks, "v %s\n", version);
-  }
-  if (protocols) {
-    smartlist_add_asprintf(chunks, "pr %s\n", protocols);
-  }
-
-  if (format != NS_V2) {
-    const routerinfo_t* desc = router_get_by_id_digest(rs->identity_digest);
-    uint32_t bw_kb;
-
-    if (format != NS_CONTROL_PORT) {
-      /* Blow up more or less nicely if we didn't get anything or not the
-       * thing we expected.
-       */
-      if (!desc) {
-        char id[HEX_DIGEST_LEN+1];
-        char dd[HEX_DIGEST_LEN+1];
-
-        base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN);
-        base16_encode(dd, sizeof(dd), rs->descriptor_digest, DIGEST_LEN);
-        log_warn(LD_BUG, "Cannot get any descriptor for %s "
-            "(wanted descriptor %s).",
-            id, dd);
-        goto err;
-      }
-
-      /* This assert could fire for the control port, because
-       * it can request NS documents before all descriptors
-       * have been fetched. Therefore, we only do this test when
-       * format != NS_CONTROL_PORT. */
-      if (tor_memneq(desc->cache_info.signed_descriptor_digest,
-            rs->descriptor_digest,
-            DIGEST_LEN)) {
-        char rl_d[HEX_DIGEST_LEN+1];
-        char rs_d[HEX_DIGEST_LEN+1];
-        char id[HEX_DIGEST_LEN+1];
-
-        base16_encode(rl_d, sizeof(rl_d),
-            desc->cache_info.signed_descriptor_digest, DIGEST_LEN);
-        base16_encode(rs_d, sizeof(rs_d), rs->descriptor_digest, DIGEST_LEN);
-        base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN);
-        log_err(LD_BUG, "descriptor digest in routerlist does not match "
-            "the one in routerstatus: %s vs %s "
-            "(router %s)\n",
-            rl_d, rs_d, id);
-
-        tor_assert(tor_memeq(desc->cache_info.signed_descriptor_digest,
-              rs->descriptor_digest,
-              DIGEST_LEN));
-      }
-    }
-
-    if (format == NS_CONTROL_PORT && rs->has_bandwidth) {
-      bw_kb = rs->bandwidth_kb;
-    } else {
-      tor_assert(desc);
-      bw_kb = router_get_advertised_bandwidth_capped(desc) / 1000;
-    }
-    smartlist_add_asprintf(chunks,
-                     "w Bandwidth=%d", bw_kb);
-
-    if (format == NS_V3_VOTE && vrs && vrs->has_measured_bw) {
-      smartlist_add_asprintf(chunks,
-                       " Measured=%d", vrs->measured_bw_kb);
-    }
-    /* Write down guardfraction information if we have it. */
-    if (format == NS_V3_VOTE && vrs && vrs->status.has_guardfraction) {
-      smartlist_add_asprintf(chunks,
-                             " GuardFraction=%d",
-                             vrs->status.guardfraction_percentage);
-    }
-
-    smartlist_add(chunks, tor_strdup("\n"));
-
-    if (desc) {
-      summary = policy_summarize(desc->exit_policy, AF_INET);
-      smartlist_add_asprintf(chunks, "p %s\n", summary);
-      tor_free(summary);
-    }
-
-    if (format == NS_V3_VOTE && vrs) {
-      if (tor_mem_is_zero((char*)vrs->ed25519_id, ED25519_PUBKEY_LEN)) {
-        smartlist_add(chunks, tor_strdup("id ed25519 none\n"));
-      } else {
-        char ed_b64[BASE64_DIGEST256_LEN+1];
-        digest256_to_base64(ed_b64, (const char*)vrs->ed25519_id);
-        smartlist_add_asprintf(chunks, "id ed25519 %s\n", ed_b64);
-      }
-    }
-  }
-
- done:
-  result = smartlist_join_strings(chunks, "", 0, NULL);
-
- err:
-  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
-  smartlist_free(chunks);
-
-  return result;
-}
-
-/** Helper for sorting: compares two routerinfos first by address, and then by
- * descending order of "usefulness".  (An authority is more useful than a
- * non-authority; a running router is more useful than a non-running router;
- * and a router with more bandwidth is more useful than one with less.)
- **/
-static int
-compare_routerinfo_by_ip_and_bw_(const void **a, const void **b)
-{
-  routerinfo_t *first = *(routerinfo_t **)a, *second = *(routerinfo_t **)b;
-  int first_is_auth, second_is_auth;
-  uint32_t bw_kb_first, bw_kb_second;
-  const node_t *node_first, *node_second;
-  int first_is_running, second_is_running;
-
-  /* we return -1 if first should appear before second... that is,
-   * if first is a better router. */
-  if (first->addr < second->addr)
-    return -1;
-  else if (first->addr > second->addr)
-    return 1;
-
-  /* Potentially, this next bit could cause k n lg n memeq calls.  But in
-   * reality, we will almost never get here, since addresses will usually be
-   * different. */
-
-  first_is_auth =
-    router_digest_is_trusted_dir(first->cache_info.identity_digest);
-  second_is_auth =
-    router_digest_is_trusted_dir(second->cache_info.identity_digest);
-
-  if (first_is_auth && !second_is_auth)
-    return -1;
-  else if (!first_is_auth && second_is_auth)
-    return 1;
-
-  node_first = node_get_by_id(first->cache_info.identity_digest);
-  node_second = node_get_by_id(second->cache_info.identity_digest);
-  first_is_running = node_first && node_first->is_running;
-  second_is_running = node_second && node_second->is_running;
-
-  if (first_is_running && !second_is_running)
-    return -1;
-  else if (!first_is_running && second_is_running)
-    return 1;
-
-  bw_kb_first = dirserv_get_bandwidth_for_router_kb(first);
-  bw_kb_second = dirserv_get_bandwidth_for_router_kb(second);
-
-  if (bw_kb_first > bw_kb_second)
-     return -1;
-  else if (bw_kb_first < bw_kb_second)
-    return 1;
-
-  /* They're equal! Compare by identity digest, so there's a
-   * deterministic order and we avoid flapping. */
-  return fast_memcmp(first->cache_info.identity_digest,
-                     second->cache_info.identity_digest,
-                     DIGEST_LEN);
-}
-
-/** Given a list of routerinfo_t in <b>routers</b>, return a new digestmap_t
- * whose keys are the identity digests of those routers that we're going to
- * exclude for Sybil-like appearance. */
-static digestmap_t *
-get_possible_sybil_list(const smartlist_t *routers)
-{
-  const or_options_t *options = get_options();
-  digestmap_t *omit_as_sybil;
-  smartlist_t *routers_by_ip = smartlist_new();
-  uint32_t last_addr;
-  int addr_count;
-  /* Allow at most this number of Tor servers on a single IP address, ... */
-  int max_with_same_addr = options->AuthDirMaxServersPerAddr;
-  /* ... unless it's a directory authority, in which case allow more. */
-  int max_with_same_addr_on_authority = options->AuthDirMaxServersPerAuthAddr;
-  if (max_with_same_addr <= 0)
-    max_with_same_addr = INT_MAX;
-  if (max_with_same_addr_on_authority <= 0)
-    max_with_same_addr_on_authority = INT_MAX;
-
-  smartlist_add_all(routers_by_ip, routers);
-  smartlist_sort(routers_by_ip, compare_routerinfo_by_ip_and_bw_);
-  omit_as_sybil = digestmap_new();
-
-  last_addr = 0;
-  addr_count = 0;
-  SMARTLIST_FOREACH_BEGIN(routers_by_ip, routerinfo_t *, ri) {
-      if (last_addr != ri->addr) {
-        last_addr = ri->addr;
-        addr_count = 1;
-      } else if (++addr_count > max_with_same_addr) {
-        if (!router_addr_is_trusted_dir(ri->addr) ||
-            addr_count > max_with_same_addr_on_authority)
-          digestmap_set(omit_as_sybil, ri->cache_info.identity_digest, ri);
-      }
-  } SMARTLIST_FOREACH_END(ri);
-
-  smartlist_free(routers_by_ip);
-  return omit_as_sybil;
-}
-
-/** If there are entries in <b>routers</b> with exactly the same ed25519 keys,
- * remove the older one.  If they are exactly the same age, remove the one
- * with the greater descriptor digest. May alter the order of the list. */
-static void
-routers_make_ed_keys_unique(smartlist_t *routers)
-{
-  routerinfo_t *ri2;
-  digest256map_t *by_ed_key = digest256map_new();
-
-  SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
-    ri->omit_from_vote = 0;
-    if (ri->cache_info.signing_key_cert == NULL)
-      continue; /* No ed key */
-    const uint8_t *pk = ri->cache_info.signing_key_cert->signing_key.pubkey;
-    if ((ri2 = digest256map_get(by_ed_key, pk))) {
-      /* Duplicate; must omit one.  Set the omit_from_vote flag in whichever
-       * one has the earlier published_on. */
-      const time_t ri_pub = ri->cache_info.published_on;
-      const time_t ri2_pub = ri2->cache_info.published_on;
-      if (ri2_pub < ri_pub ||
-          (ri2_pub == ri_pub &&
-           fast_memcmp(ri->cache_info.signed_descriptor_digest,
-                     ri2->cache_info.signed_descriptor_digest,DIGEST_LEN)<0)) {
-        digest256map_set(by_ed_key, pk, ri);
-        ri2->omit_from_vote = 1;
-      } else {
-        ri->omit_from_vote = 1;
-      }
-    } else {
-      /* Add to map */
-      digest256map_set(by_ed_key, pk, ri);
-    }
-  } SMARTLIST_FOREACH_END(ri);
-
-  digest256map_free(by_ed_key, NULL);
-
-  /* Now remove every router where the omit_from_vote flag got set. */
-  SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) {
-    if (ri->omit_from_vote) {
-      SMARTLIST_DEL_CURRENT(routers, ri);
-    }
-  } SMARTLIST_FOREACH_END(ri);
-}
-
-/** Extract status information from <b>ri</b> and from other authority
- * functions and store it in <b>rs</b>>.
- *
- * We assume that ri-\>is_running has already been set, e.g. by
- *   dirserv_set_router_is_running(ri, now);
- */
-void
-set_routerstatus_from_routerinfo(routerstatus_t *rs,
-                                 node_t *node,
-                                 routerinfo_t *ri,
-                                 time_t now,
-                                 int listbadexits)
-{
-  const or_options_t *options = get_options();
-  uint32_t routerbw_kb = dirserv_get_credible_bandwidth_kb(ri);
-
-  memset(rs, 0, sizeof(routerstatus_t));
-
-  rs->is_authority =
-    router_digest_is_trusted_dir(ri->cache_info.identity_digest);
-
-  /* Already set by compute_performance_thresholds. */
-  rs->is_exit = node->is_exit;
-  rs->is_stable = node->is_stable =
-    !dirserv_thinks_router_is_unreliable(now, ri, 1, 0);
-  rs->is_fast = node->is_fast =
-    !dirserv_thinks_router_is_unreliable(now, ri, 0, 1);
-  rs->is_flagged_running = node->is_running; /* computed above */
-
-  rs->is_valid = node->is_valid;
-
-  if (node->is_fast && node->is_stable &&
-      ((options->AuthDirGuardBWGuarantee &&
-        routerbw_kb >= options->AuthDirGuardBWGuarantee/1000) ||
-       routerbw_kb >= MIN(guard_bandwidth_including_exits_kb,
-                          guard_bandwidth_excluding_exits_kb))) {
-    long tk = rep_hist_get_weighted_time_known(
-                                      node->identity, now);
-    double wfu = rep_hist_get_weighted_fractional_uptime(
-                                      node->identity, now);
-    rs->is_possible_guard = (wfu >= guard_wfu && tk >= guard_tk) ? 1 : 0;
-  } else {
-    rs->is_possible_guard = 0;
-  }
-
-  rs->is_bad_exit = listbadexits && node->is_bad_exit;
-  rs->is_hs_dir = node->is_hs_dir =
-    dirserv_thinks_router_is_hs_dir(ri, node, now);
-
-  rs->is_named = rs->is_unnamed = 0;
-
-  rs->published_on = ri->cache_info.published_on;
-  memcpy(rs->identity_digest, node->identity, DIGEST_LEN);
-  memcpy(rs->descriptor_digest, ri->cache_info.signed_descriptor_digest,
-         DIGEST_LEN);
-  rs->addr = ri->addr;
-  strlcpy(rs->nickname, ri->nickname, sizeof(rs->nickname));
-  rs->or_port = ri->or_port;
-  rs->dir_port = ri->dir_port;
-  rs->is_v2_dir = ri->supports_tunnelled_dir_requests;
-  if (options->AuthDirHasIPv6Connectivity == 1 &&
-      !tor_addr_is_null(&ri->ipv6_addr) &&
-      node->last_reachable6 >= now - REACHABLE_TIMEOUT) {
-    /* We're configured as having IPv6 connectivity. There's an IPv6
-       OR port and it's reachable so copy it to the routerstatus.  */
-    tor_addr_copy(&rs->ipv6_addr, &ri->ipv6_addr);
-    rs->ipv6_orport = ri->ipv6_orport;
-  }
-
-  if (options->TestingTorNetwork) {
-    dirserv_set_routerstatus_testing(rs);
-  }
-}
-
-/** Use TestingDirAuthVoteExit, TestingDirAuthVoteGuard, and
- * TestingDirAuthVoteHSDir to give out the Exit, Guard, and HSDir flags,
- * respectively. But don't set the corresponding node flags.
- * Should only be called if TestingTorNetwork is set. */
-STATIC void
-dirserv_set_routerstatus_testing(routerstatus_t *rs)
-{
-  const or_options_t *options = get_options();
-
-  tor_assert(options->TestingTorNetwork);
-
-  if (routerset_contains_routerstatus(options->TestingDirAuthVoteExit,
-                                      rs, 0)) {
-    rs->is_exit = 1;
-  } else if (options->TestingDirAuthVoteExitIsStrict) {
-    rs->is_exit = 0;
-  }
-
-  if (routerset_contains_routerstatus(options->TestingDirAuthVoteGuard,
-                                      rs, 0)) {
-    rs->is_possible_guard = 1;
-  } else if (options->TestingDirAuthVoteGuardIsStrict) {
-    rs->is_possible_guard = 0;
-  }
-
-  if (routerset_contains_routerstatus(options->TestingDirAuthVoteHSDir,
-                                      rs, 0)) {
-    rs->is_hs_dir = 1;
-  } else if (options->TestingDirAuthVoteHSDirIsStrict) {
-    rs->is_hs_dir = 0;
-  }
-}
-
-/** Routerstatus <b>rs</b> is part of a group of routers that are on
- * too narrow an IP-space. Clear out its flags: we don't want people
- * using it.
- *
- * Leave its BadExit flag alone though, since if we think it's a bad exit,
- * we want to vote that way in case all the other authorities are voting
- * Running and Exit.
- */
-static void
-clear_status_flags_on_sybil(routerstatus_t *rs)
-{
-  rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast =
-    rs->is_flagged_running = rs->is_named = rs->is_valid =
-    rs->is_hs_dir = rs->is_v2_dir = rs->is_possible_guard = 0;
-  /* FFFF we might want some mechanism to check later on if we
-   * missed zeroing any flags: it's easy to add a new flag but
-   * forget to add it to this clause. */
-}
-
-/** The guardfraction of the guard with identity fingerprint <b>guard_id</b>
- *  is <b>guardfraction_percentage</b>. See if we have a vote routerstatus for
- *  this guard in <b>vote_routerstatuses</b>, and if we do, register the
- *  information to it.
- *
- *  Return 1 if we applied the information and 0 if we couldn't find a
- *  matching guard.
- *
- * Requires that <b>vote_routerstatuses</b> be sorted.
- */
-static int
-guardfraction_line_apply(const char *guard_id,
-                      uint32_t guardfraction_percentage,
-                      smartlist_t *vote_routerstatuses)
-{
-  vote_routerstatus_t *vrs = NULL;
-
-  tor_assert(vote_routerstatuses);
-
-  vrs = smartlist_bsearch(vote_routerstatuses, guard_id,
-                         compare_digest_to_vote_routerstatus_entry);
-
-  if (!vrs) {
-    return 0;
-  }
-
-  vrs->status.has_guardfraction = 1;
-  vrs->status.guardfraction_percentage = guardfraction_percentage;
-
-  return 1;
-}
-
-/* Given a guard line from a guardfraction file, parse it and register
- * its information to <b>vote_routerstatuses</b>.
- *
- * Return:
- * * 1 if the line was proper and its information got registered.
- * * 0 if the line was proper but no currently active guard was found
- *     to register the guardfraction information to.
- * * -1 if the line could not be parsed and set <b>err_msg</b> to a
-      newly allocated string containing the error message.
- */
-static int
-guardfraction_file_parse_guard_line(const char *guard_line,
-                                    smartlist_t *vote_routerstatuses,
-                                    char **err_msg)
-{
-  char guard_id[DIGEST_LEN];
-  uint32_t guardfraction;
-  char *inputs_tmp = NULL;
-  int num_ok = 1;
-
-  smartlist_t *sl = smartlist_new();
-  int retval = -1;
-
-  tor_assert(err_msg);
-
-  /* guard_line should contain something like this:
-     <hex digest> <guardfraction> <appearances> */
-  smartlist_split_string(sl, guard_line, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3);
-  if (smartlist_len(sl) < 3) {
-    tor_asprintf(err_msg, "bad line '%s'", guard_line);
-    goto done;
-  }
-
-  inputs_tmp = smartlist_get(sl, 0);
-  if (strlen(inputs_tmp) != HEX_DIGEST_LEN ||
-      base16_decode(guard_id, DIGEST_LEN,
-                    inputs_tmp, HEX_DIGEST_LEN) != DIGEST_LEN) {
-    tor_asprintf(err_msg, "bad digest '%s'", inputs_tmp);
-    goto done;
-  }
-
-  inputs_tmp = smartlist_get(sl, 1);
-  /* Guardfraction is an integer in [0, 100]. */
-  guardfraction =
-    (uint32_t) tor_parse_long(inputs_tmp, 10, 0, 100, &num_ok, NULL);
-  if (!num_ok) {
-    tor_asprintf(err_msg, "wrong percentage '%s'", inputs_tmp);
-    goto done;
-  }
-
-  /* If routerstatuses were provided, apply this info to actual routers. */
-  if (vote_routerstatuses) {
-    retval = guardfraction_line_apply(guard_id, guardfraction,
-                                      vote_routerstatuses);
-  } else {
-    retval = 0; /* If we got this far, line was correctly formatted. */
-  }
-
- done:
-
-  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
-  smartlist_free(sl);
-
-  return retval;
-}
-
-/** Given an inputs line from a guardfraction file, parse it and
- *  register its information to <b>total_consensuses</b> and
- *  <b>total_days</b>.
- *
- *  Return 0 if it parsed well. Return -1 if there was an error, and
- *  set <b>err_msg</b> to a newly allocated string containing the
- *  error message.
- */
-static int
-guardfraction_file_parse_inputs_line(const char *inputs_line,
-                                     int *total_consensuses,
-                                     int *total_days,
-                                     char **err_msg)
-{
-  int retval = -1;
-  char *inputs_tmp = NULL;
-  int num_ok = 1;
-  smartlist_t *sl = smartlist_new();
-
-  tor_assert(err_msg);
-
-  /* Second line is inputs information:
-   *   n-inputs <total_consensuses> <total_days>. */
-  smartlist_split_string(sl, inputs_line, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3);
-  if (smartlist_len(sl) < 2) {
-    tor_asprintf(err_msg, "incomplete line '%s'", inputs_line);
-    goto done;
-  }
-
-  inputs_tmp = smartlist_get(sl, 0);
-  *total_consensuses =
-    (int) tor_parse_long(inputs_tmp, 10, 0, INT_MAX, &num_ok, NULL);
-  if (!num_ok) {
-    tor_asprintf(err_msg, "unparseable consensus '%s'", inputs_tmp);
-    goto done;
-  }
-
-  inputs_tmp = smartlist_get(sl, 1);
-  *total_days =
-    (int) tor_parse_long(inputs_tmp, 10, 0, INT_MAX, &num_ok, NULL);
-  if (!num_ok) {
-    tor_asprintf(err_msg, "unparseable days '%s'", inputs_tmp);
-    goto done;
-  }
-
-  retval = 0;
-
- done:
-  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
-  smartlist_free(sl);
-
-  return retval;
-}
-
-/* Maximum age of a guardfraction file that we are willing to accept. */
-#define MAX_GUARDFRACTION_FILE_AGE (7*24*60*60) /* approx a week */
-
-/** Static strings of guardfraction files. */
-#define GUARDFRACTION_DATE_STR "written-at"
-#define GUARDFRACTION_INPUTS "n-inputs"
-#define GUARDFRACTION_GUARD "guard-seen"
-#define GUARDFRACTION_VERSION "guardfraction-file-version"
-
-/** Given a guardfraction file in a string, parse it and register the
- *  guardfraction information to the provided vote routerstatuses.
- *
- *  This is the rough format of the guardfraction file:
- *
- *      guardfraction-file-version 1
- *      written-at <date and time>
- *      n-inputs <number of consesuses parsed> <number of days considered>
- *
- *      guard-seen <fpr 1> <guardfraction percentage> <consensus appearances>
- *      guard-seen <fpr 2> <guardfraction percentage> <consensus appearances>
- *      guard-seen <fpr 3> <guardfraction percentage> <consensus appearances>
- *      guard-seen <fpr 4> <guardfraction percentage> <consensus appearances>
- *      guard-seen <fpr 5> <guardfraction percentage> <consensus appearances>
- *      ...
- *
- *  Return -1 if the parsing failed and 0 if it went smoothly. Parsing
- *  should tolerate errors in all lines but the written-at header.
- */
-STATIC int
-dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str,
-                                      smartlist_t *vote_routerstatuses)
-{
-  config_line_t *front=NULL, *line;
-  int ret_tmp;
-  int retval = -1;
-  int current_line_n = 0; /* line counter for better log messages */
-
-  /* Guardfraction info to be parsed */
-  int total_consensuses = 0;
-  int total_days = 0;
-
-  /* Stats */
-  int guards_read_n = 0;
-  int guards_applied_n = 0;
-
-  /* Parse file and split it in lines */
-  ret_tmp = config_get_lines(guardfraction_file_str, &front, 0);
-  if (ret_tmp < 0) {
-    log_warn(LD_CONFIG, "Error reading from guardfraction file");
-    goto done;
-  }
-
-  /* Sort routerstatuses (needed later when applying guardfraction info) */
-  if (vote_routerstatuses)
-    smartlist_sort(vote_routerstatuses, compare_vote_routerstatus_entries);
-
-  for (line = front; line; line=line->next) {
-    current_line_n++;
-
-    if (!strcmp(line->key, GUARDFRACTION_VERSION)) {
-      int num_ok = 1;
-      unsigned int version;
-
-      version =
-        (unsigned int) tor_parse_long(line->value,
-                                      10, 0, INT_MAX, &num_ok, NULL);
-
-      if (!num_ok || version != 1) {
-        log_warn(LD_GENERAL, "Got unknown guardfraction version %d.", version);
-        goto done;
-      }
-    } else if (!strcmp(line->key, GUARDFRACTION_DATE_STR)) {
-      time_t file_written_at;
-      time_t now = time(NULL);
-
-      /* First line is 'written-at <date>' */
-      if (parse_iso_time(line->value, &file_written_at) < 0) {
-        log_warn(LD_CONFIG, "Guardfraction:%d: Bad date '%s'. Ignoring",
-                 current_line_n, line->value);
-        goto done; /* don't tolerate failure here. */
-      }
-      if (file_written_at < now - MAX_GUARDFRACTION_FILE_AGE) {
-        log_warn(LD_CONFIG, "Guardfraction:%d: was written very long ago '%s'",
-                 current_line_n, line->value);
-        goto done; /* don't tolerate failure here. */
-      }
-    } else if (!strcmp(line->key, GUARDFRACTION_INPUTS)) {
-      char *err_msg = NULL;
-
-      if (guardfraction_file_parse_inputs_line(line->value,
-                                               &total_consensuses,
-                                               &total_days,
-                                               &err_msg) < 0) {
-        log_warn(LD_CONFIG, "Guardfraction:%d: %s",
-                 current_line_n, err_msg);
-        tor_free(err_msg);
-        continue;
-      }
-
-    } else if (!strcmp(line->key, GUARDFRACTION_GUARD)) {
-      char *err_msg = NULL;
-
-      ret_tmp = guardfraction_file_parse_guard_line(line->value,
-                                                    vote_routerstatuses,
-                                                    &err_msg);
-      if (ret_tmp < 0) { /* failed while parsing the guard line */
-        log_warn(LD_CONFIG, "Guardfraction:%d: %s",
-                 current_line_n, err_msg);
-        tor_free(err_msg);
-        continue;
-      }
-
-      /* Successfully parsed guard line. Check if it was applied properly. */
-      guards_read_n++;
-      if (ret_tmp > 0) {
-        guards_applied_n++;
-      }
-    } else {
-      log_warn(LD_CONFIG, "Unknown guardfraction line %d (%s %s)",
-               current_line_n, line->key, line->value);
-    }
-  }
-
-  retval = 0;
-
-  log_info(LD_CONFIG,
-           "Successfully parsed guardfraction file with %d consensuses over "
-           "%d days. Parsed %d nodes and applied %d of them%s.",
-           total_consensuses, total_days, guards_read_n, guards_applied_n,
-           vote_routerstatuses ? "" : " (no routerstatus provided)" );
-
- done:
-  config_free_lines(front);
-
-  if (retval < 0) {
-    return retval;
-  } else {
-    return guards_read_n;
-  }
-}
-
-/** Read a guardfraction file at <b>fname</b> and load all its
- *  information to <b>vote_routerstatuses</b>. */
-int
-dirserv_read_guardfraction_file(const char *fname,
-                             smartlist_t *vote_routerstatuses)
-{
-  char *guardfraction_file_str;
-
-  /* Read file to a string */
-  guardfraction_file_str = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
-  if (!guardfraction_file_str) {
-      log_warn(LD_FS, "Cannot open guardfraction file '%s'. Failing.", fname);
-      return -1;
-  }
-
-  return dirserv_read_guardfraction_file_from_str(guardfraction_file_str,
-                                               vote_routerstatuses);
-}
-
-/**
- * Helper function to parse out a line in the measured bandwidth file
- * into a measured_bw_line_t output structure. Returns -1 on failure
- * or 0 on success.
- */
-STATIC int
-measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line)
-{
-  char *line = tor_strdup(orig_line);
-  char *cp = line;
-  int got_bw = 0;
-  int got_node_id = 0;
-  char *strtok_state; /* lame sauce d'jour */
-  cp = tor_strtok_r(cp, " \t", &strtok_state);
-
-  if (!cp) {
-    log_warn(LD_DIRSERV, "Invalid line in bandwidth file: %s",
-             escaped(orig_line));
-    tor_free(line);
-    return -1;
-  }
-
-  if (orig_line[strlen(orig_line)-1] != '\n') {
-    log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s",
-             escaped(orig_line));
-    tor_free(line);
-    return -1;
-  }
-
-  do {
-    if (strcmpstart(cp, "bw=") == 0) {
-      int parse_ok = 0;
-      char *endptr;
-      if (got_bw) {
-        log_warn(LD_DIRSERV, "Double bw= in bandwidth file line: %s",
-                 escaped(orig_line));
-        tor_free(line);
-        return -1;
-      }
-      cp+=strlen("bw=");
-
-      out->bw_kb = tor_parse_long(cp, 0, 0, LONG_MAX, &parse_ok, &endptr);
-      if (!parse_ok || (*endptr && !TOR_ISSPACE(*endptr))) {
-        log_warn(LD_DIRSERV, "Invalid bandwidth in bandwidth file line: %s",
-                 escaped(orig_line));
-        tor_free(line);
-        return -1;
-      }
-      got_bw=1;
-    } else if (strcmpstart(cp, "node_id=$") == 0) {
-      if (got_node_id) {
-        log_warn(LD_DIRSERV, "Double node_id= in bandwidth file line: %s",
-                 escaped(orig_line));
-        tor_free(line);
-        return -1;
-      }
-      cp+=strlen("node_id=$");
-
-      if (strlen(cp) != HEX_DIGEST_LEN ||
-          base16_decode(out->node_id, DIGEST_LEN,
-                        cp, HEX_DIGEST_LEN) != DIGEST_LEN) {
-        log_warn(LD_DIRSERV, "Invalid node_id in bandwidth file line: %s",
-                 escaped(orig_line));
-        tor_free(line);
-        return -1;
-      }
-      strlcpy(out->node_hex, cp, sizeof(out->node_hex));
-      got_node_id=1;
-    }
-  } while ((cp = tor_strtok_r(NULL, " \t", &strtok_state)));
-
-  if (got_bw && got_node_id) {
-    tor_free(line);
-    return 0;
-  } else {
-    log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s",
-             escaped(orig_line));
-    tor_free(line);
-    return -1;
-  }
-}
-
-/**
- * Helper function to apply a parsed measurement line to a list
- * of bandwidth statuses. Returns true if a line is found,
- * false otherwise.
- */
-STATIC int
-measured_bw_line_apply(measured_bw_line_t *parsed_line,
-                       smartlist_t *routerstatuses)
-{
-  vote_routerstatus_t *rs = NULL;
-  if (!routerstatuses)
-    return 0;
-
-  rs = smartlist_bsearch(routerstatuses, parsed_line->node_id,
-                         compare_digest_to_vote_routerstatus_entry);
-
-  if (rs) {
-    rs->has_measured_bw = 1;
-    rs->measured_bw_kb = (uint32_t)parsed_line->bw_kb;
-  } else {
-    log_info(LD_DIRSERV, "Node ID %s not found in routerstatus list",
-             parsed_line->node_hex);
-  }
-
-  return rs != NULL;
-}
-
-/**
- * Read the measured bandwidth file and apply it to the list of
- * vote_routerstatus_t. Returns -1 on error, 0 otherwise.
- */
-int
-dirserv_read_measured_bandwidths(const char *from_file,
-                                 smartlist_t *routerstatuses)
-{
-  char line[512];
-  FILE *fp = tor_fopen_cloexec(from_file, "r");
-  int applied_lines = 0;
-  time_t file_time, now;
-  int ok;
-
-  /* Initialise line, so that we can't possibly run off the end. */
-  memset(line, 0, sizeof(line));
-
-  if (fp == NULL) {
-    log_warn(LD_CONFIG, "Can't open bandwidth file at configured location: %s",
-             from_file);
-    return -1;
-  }
-
-  /* If fgets fails, line is either unmodified, or indeterminate. */
-  if (!fgets(line, sizeof(line), fp)) {
-    log_warn(LD_DIRSERV, "Empty bandwidth file");
-    fclose(fp);
-    return -1;
-  }
-
-  if (!strlen(line) || line[strlen(line)-1] != '\n') {
-    log_warn(LD_DIRSERV, "Long or truncated time in bandwidth file: %s",
-             escaped(line));
-    fclose(fp);
-    return -1;
-  }
-
-  line[strlen(line)-1] = '\0';
-  file_time = (time_t)tor_parse_ulong(line, 10, 0, ULONG_MAX, &ok, NULL);
-  if (!ok) {
-    log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s",
-             escaped(line));
-    fclose(fp);
-    return -1;
-  }
-
-  now = time(NULL);
-  if ((now - file_time) > MAX_MEASUREMENT_AGE) {
-    log_warn(LD_DIRSERV, "Bandwidth measurement file stale. Age: %u",
-             (unsigned)(time(NULL) - file_time));
-    fclose(fp);
-    return -1;
-  }
-
-  if (routerstatuses)
-    smartlist_sort(routerstatuses, compare_vote_routerstatus_entries);
-
-  while (!feof(fp)) {
-    measured_bw_line_t parsed_line;
-    if (fgets(line, sizeof(line), fp) && strlen(line)) {
-      if (measured_bw_line_parse(&parsed_line, line) != -1) {
-        /* Also cache the line for dirserv_get_bandwidth_for_router() */
-        dirserv_cache_measured_bw(&parsed_line, file_time);
-        if (measured_bw_line_apply(&parsed_line, routerstatuses) > 0)
-          applied_lines++;
-      }
-    }
-  }
-
-  /* Now would be a nice time to clean the cache, too */
-  dirserv_expire_measured_bw_cache(now);
-
-  fclose(fp);
-  log_info(LD_DIRSERV,
-           "Bandwidth measurement file successfully read. "
-           "Applied %d measurements.", applied_lines);
-  return 0;
-}
-
-/** Return a new networkstatus_t* containing our current opinion. (For v3
- * authorities) */
-networkstatus_t *
-dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
-                                        authority_cert_t *cert)
-{
-  const or_options_t *options = get_options();
-  networkstatus_t *v3_out = NULL;
-  uint32_t addr;
-  char *hostname = NULL, *client_versions = NULL, *server_versions = NULL;
-  const char *contact;
-  smartlist_t *routers, *routerstatuses;
-  char identity_digest[DIGEST_LEN];
-  char signing_key_digest[DIGEST_LEN];
-  int listbadexits = options->AuthDirListBadExits;
-  routerlist_t *rl = router_get_routerlist();
-  time_t now = time(NULL);
-  time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
-  networkstatus_voter_info_t *voter = NULL;
-  vote_timing_t timing;
-  digestmap_t *omit_as_sybil = NULL;
-  const int vote_on_reachability = running_long_enough_to_decide_unreachable();
-  smartlist_t *microdescriptors = NULL;
-
-  tor_assert(private_key);
-  tor_assert(cert);
-
-  if (crypto_pk_get_digest(private_key, signing_key_digest)<0) {
-    log_err(LD_BUG, "Error computing signing key digest");
-    return NULL;
-  }
-  if (crypto_pk_get_digest(cert->identity_key, identity_digest)<0) {
-    log_err(LD_BUG, "Error computing identity key digest");
-    return NULL;
-  }
-  if (resolve_my_address(LOG_WARN, options, &addr, NULL, &hostname)<0) {
-    log_warn(LD_NET, "Couldn't resolve my hostname");
-    return NULL;
-  }
-  if (!hostname || !strchr(hostname, '.')) {
-    tor_free(hostname);
-    hostname = tor_dup_ip(addr);
-  }
-
-  if (options->VersioningAuthoritativeDir) {
-    client_versions =
-      format_recommended_version_list(options->RecommendedClientVersions, 0);
-    server_versions =
-      format_recommended_version_list(options->RecommendedServerVersions, 0);
-  }
-
-  contact = get_options()->ContactInfo;
-  if (!contact)
-    contact = "(none)";
-
-  /*
-   * Do this so dirserv_compute_performance_thresholds() and
-   * set_routerstatus_from_routerinfo() see up-to-date bandwidth info.
-   */
-  if (options->V3BandwidthsFile) {
-    dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL);
-  } else {
-    /*
-     * No bandwidths file; clear the measured bandwidth cache in case we had
-     * one last time around.
-     */
-    if (dirserv_get_measured_bw_cache_size() > 0) {
-      dirserv_clear_measured_bw_cache();
-    }
-  }
-
-  /* precompute this part, since we need it to decide what "stable"
-   * means. */
-  SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, {
-    dirserv_set_router_is_running(ri, now);
-  });
-
-  routers = smartlist_new();
-  smartlist_add_all(routers, rl->routers);
-  routers_make_ed_keys_unique(routers);
-  /* After this point, don't use rl->routers; use 'routers' instead. */
-  routers_sort_by_identity(routers);
-  omit_as_sybil = get_possible_sybil_list(routers);
-
-  DIGESTMAP_FOREACH(omit_as_sybil, sybil_id, void *, ignore) {
-    (void) ignore;
-    rep_hist_make_router_pessimal(sybil_id, now);
-  } DIGESTMAP_FOREACH_END;
-
-  /* Count how many have measured bandwidths so we know how to assign flags;
-   * this must come before dirserv_compute_performance_thresholds() */
-  dirserv_count_measured_bws(routers);
-
-  dirserv_compute_performance_thresholds(omit_as_sybil);
-
-  routerstatuses = smartlist_new();
-  microdescriptors = smartlist_new();
-
-  SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
-    if (ri->cache_info.published_on >= cutoff) {
-      routerstatus_t *rs;
-      vote_routerstatus_t *vrs;
-      node_t *node = node_get_mutable_by_id(ri->cache_info.identity_digest);
-      if (!node)
-        continue;
-
-      vrs = tor_malloc_zero(sizeof(vote_routerstatus_t));
-      rs = &vrs->status;
-      set_routerstatus_from_routerinfo(rs, node, ri, now,
-                                       listbadexits);
-
-      if (ri->cache_info.signing_key_cert) {
-        memcpy(vrs->ed25519_id,
-               ri->cache_info.signing_key_cert->signing_key.pubkey,
-               ED25519_PUBKEY_LEN);
-      }
-
-      if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest))
-        clear_status_flags_on_sybil(rs);
-
-      if (!vote_on_reachability)
-        rs->is_flagged_running = 0;
-
-      vrs->version = version_from_platform(ri->platform);
-      if (ri->protocol_list) {
-        vrs->protocols = tor_strdup(ri->protocol_list);
-      } else {
-        vrs->protocols = tor_strdup(
-                              protover_compute_for_old_tor(vrs->version));
-      }
-      vrs->microdesc = dirvote_format_all_microdesc_vote_lines(ri, now,
-                                                        microdescriptors);
-
-      smartlist_add(routerstatuses, vrs);
-    }
-  } SMARTLIST_FOREACH_END(ri);
-
-  {
-    smartlist_t *added =
-      microdescs_add_list_to_cache(get_microdesc_cache(),
-                                   microdescriptors, SAVED_NOWHERE, 0);
-    smartlist_free(added);
-    smartlist_free(microdescriptors);
-  }
-
-  smartlist_free(routers);
-  digestmap_free(omit_as_sybil, NULL);
-
-  /* Apply guardfraction information to routerstatuses. */
-  if (options->GuardfractionFile) {
-    dirserv_read_guardfraction_file(options->GuardfractionFile,
-                                    routerstatuses);
-  }
-
-  /* This pass through applies the measured bw lines to the routerstatuses */
-  if (options->V3BandwidthsFile) {
-    dirserv_read_measured_bandwidths(options->V3BandwidthsFile,
-                                     routerstatuses);
-  } else {
-    /*
-     * No bandwidths file; clear the measured bandwidth cache in case we had
-     * one last time around.
-     */
-    if (dirserv_get_measured_bw_cache_size() > 0) {
-      dirserv_clear_measured_bw_cache();
-    }
-  }
-
-  v3_out = tor_malloc_zero(sizeof(networkstatus_t));
-
-  v3_out->type = NS_TYPE_VOTE;
-  dirvote_get_preferred_voting_intervals(&timing);
-  v3_out->published = now;
-  {
-    char tbuf[ISO_TIME_LEN+1];
-    networkstatus_t *current_consensus =
-      networkstatus_get_live_consensus(now);
-    long last_consensus_interval; /* only used to pick a valid_after */
-    if (current_consensus)
-      last_consensus_interval = current_consensus->fresh_until -
-        current_consensus->valid_after;
-    else
-      last_consensus_interval = options->TestingV3AuthInitialVotingInterval;
-    v3_out->valid_after =
-      dirvote_get_start_of_next_interval(now, (int)last_consensus_interval,
-                                      options->TestingV3AuthVotingStartOffset);
-    format_iso_time(tbuf, v3_out->valid_after);
-    log_notice(LD_DIR,"Choosing valid-after time in vote as %s: "
-               "consensus_set=%d, last_interval=%d",
-               tbuf, current_consensus?1:0, (int)last_consensus_interval);
-  }
-  v3_out->fresh_until = v3_out->valid_after + timing.vote_interval;
-  v3_out->valid_until = v3_out->valid_after +
-    (timing.vote_interval * timing.n_intervals_valid);
-  v3_out->vote_seconds = timing.vote_delay;
-  v3_out->dist_seconds = timing.dist_delay;
-  tor_assert(v3_out->vote_seconds > 0);
-  tor_assert(v3_out->dist_seconds > 0);
-  tor_assert(timing.n_intervals_valid > 0);
-
-  v3_out->client_versions = client_versions;
-  v3_out->server_versions = server_versions;
-
-  /* These are hardwired, to avoid disaster. */
-  v3_out->recommended_relay_protocols =
-    tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
-               "Link=4 LinkAuth=1 Microdesc=1-2 Relay=2");
-  v3_out->recommended_client_protocols =
-    tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
-               "Link=4 LinkAuth=1 Microdesc=1-2 Relay=2");
-  v3_out->required_client_protocols =
-    tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
-               "Link=4 LinkAuth=1 Microdesc=1-2 Relay=2");
-  v3_out->required_relay_protocols =
-    tor_strdup("Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
-               "Link=3-4 LinkAuth=1 Microdesc=1 Relay=1-2");
-
-  /* We are not allowed to vote to require anything we don't have. */
-  tor_assert(protover_all_supported(v3_out->required_relay_protocols, NULL));
-  tor_assert(protover_all_supported(v3_out->required_client_protocols, NULL));
-
-  /* We should not recommend anything we don't have. */
-  tor_assert_nonfatal(protover_all_supported(
-                         v3_out->recommended_relay_protocols, NULL));
-  tor_assert_nonfatal(protover_all_supported(
-                         v3_out->recommended_client_protocols, NULL));
-
-  v3_out->package_lines = smartlist_new();
-  {
-    config_line_t *cl;
-    for (cl = get_options()->RecommendedPackages; cl; cl = cl->next) {
-      if (validate_recommended_package_line(cl->value))
-        smartlist_add(v3_out->package_lines, tor_strdup(cl->value));
-    }
-  }
-
-  v3_out->known_flags = smartlist_new();
-  smartlist_split_string(v3_out->known_flags,
-                "Authority Exit Fast Guard Stable V2Dir Valid HSDir",
-                0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  if (vote_on_reachability)
-    smartlist_add(v3_out->known_flags, tor_strdup("Running"));
-  if (listbadexits)
-    smartlist_add(v3_out->known_flags, tor_strdup("BadExit"));
-  smartlist_sort_strings(v3_out->known_flags);
-
-  if (options->ConsensusParams) {
-    v3_out->net_params = smartlist_new();
-    smartlist_split_string(v3_out->net_params,
-                           options->ConsensusParams, NULL, 0, 0);
-    smartlist_sort_strings(v3_out->net_params);
-  }
-
-  voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t));
-  voter->nickname = tor_strdup(options->Nickname);
-  memcpy(voter->identity_digest, identity_digest, DIGEST_LEN);
-  voter->sigs = smartlist_new();
-  voter->address = hostname;
-  voter->addr = addr;
-  voter->dir_port = router_get_advertised_dir_port(options, 0);
-  voter->or_port = router_get_advertised_or_port(options);
-  voter->contact = tor_strdup(contact);
-  if (options->V3AuthUseLegacyKey) {
-    authority_cert_t *c = get_my_v3_legacy_cert();
-    if (c) {
-      if (crypto_pk_get_digest(c->identity_key, voter->legacy_id_digest)) {
-        log_warn(LD_BUG, "Unable to compute digest of legacy v3 identity key");
-        memset(voter->legacy_id_digest, 0, DIGEST_LEN);
-      }
-    }
-  }
-
-  v3_out->voters = smartlist_new();
-  smartlist_add(v3_out->voters, voter);
-  v3_out->cert = authority_cert_dup(cert);
-  v3_out->routerstatus_list = routerstatuses;
-  /* Note: networkstatus_digest is unset; it won't get set until we actually
-   * format the vote. */
-
-  return v3_out;
-}
-
-/** As dirserv_get_routerdescs(), but instead of getting signed_descriptor_t
- * pointers, adds copies of digests to fps_out, and doesn't use the
- * /tor/server/ prefix.  For a /d/ request, adds descriptor digests; for other
- * requests, adds identity digests.
- */
-int
-dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key,
-                                    const char **msg, int for_unencrypted_conn,
-                                    int is_extrainfo)
-{
-  int by_id = 1;
-  *msg = NULL;
-
-  if (!strcmp(key, "all")) {
-    routerlist_t *rl = router_get_routerlist();
-    SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r,
-                      smartlist_add(fps_out,
-                      tor_memdup(r->cache_info.identity_digest, DIGEST_LEN)));
-    /* Treat "all" requests as if they were unencrypted */
-    for_unencrypted_conn = 1;
-  } else if (!strcmp(key, "authority")) {
-    const routerinfo_t *ri = router_get_my_routerinfo();
-    if (ri)
-      smartlist_add(fps_out,
-                    tor_memdup(ri->cache_info.identity_digest, DIGEST_LEN));
-  } else if (!strcmpstart(key, "d/")) {
-    by_id = 0;
-    key += strlen("d/");
-    dir_split_resource_into_fingerprints(key, fps_out, NULL,
-                                         DSR_HEX|DSR_SORT_UNIQ);
-  } else if (!strcmpstart(key, "fp/")) {
-    key += strlen("fp/");
-    dir_split_resource_into_fingerprints(key, fps_out, NULL,
-                                         DSR_HEX|DSR_SORT_UNIQ);
-  } else {
-    *msg = "Key not recognized";
-    return -1;
-  }
-
-  if (for_unencrypted_conn) {
-    /* Remove anything that insists it not be sent unencrypted. */
-    SMARTLIST_FOREACH_BEGIN(fps_out, char *, cp) {
-        const signed_descriptor_t *sd;
-        if (by_id)
-          sd = get_signed_descriptor_by_fp(cp,is_extrainfo,0);
-        else if (is_extrainfo)
-          sd = extrainfo_get_by_descriptor_digest(cp);
-        else
-          sd = router_get_by_descriptor_digest(cp);
-        if (sd && !sd->send_unencrypted) {
-          tor_free(cp);
-          SMARTLIST_DEL_CURRENT(fps_out, cp);
-        }
-    } SMARTLIST_FOREACH_END(cp);
-  }
-
-  if (!smartlist_len(fps_out)) {
-    *msg = "Servers unavailable";
-    return -1;
-  }
-  return 0;
-}
-
-/** Add a signed_descriptor_t to <b>descs_out</b> for each router matching
- * <b>key</b>.  The key should be either
- *   - "/tor/server/authority" for our own routerinfo;
- *   - "/tor/server/all" for all the routerinfos we have, concatenated;
- *   - "/tor/server/fp/FP" where FP is a plus-separated sequence of
- *     hex identity digests; or
- *   - "/tor/server/d/D" where D is a plus-separated sequence
- *     of server descriptor digests, in hex.
- *
- * Return 0 if we found some matching descriptors, or -1 if we do not
- * have any descriptors, no matching descriptors, or if we did not
- * recognize the key (URL).
- * If -1 is returned *<b>msg</b> will be set to an appropriate error
- * message.
- *
- * XXXX rename this function.  It's only called from the controller.
- * XXXX in fact, refactor this function, merging as much as possible.
- */
-int
-dirserv_get_routerdescs(smartlist_t *descs_out, const char *key,
-                        const char **msg)
-{
-  *msg = NULL;
-
-  if (!strcmp(key, "/tor/server/all")) {
-    routerlist_t *rl = router_get_routerlist();
-    SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r,
-                      smartlist_add(descs_out, &(r->cache_info)));
-  } else if (!strcmp(key, "/tor/server/authority")) {
-    const routerinfo_t *ri = router_get_my_routerinfo();
-    if (ri)
-      smartlist_add(descs_out, (void*) &(ri->cache_info));
-  } else if (!strcmpstart(key, "/tor/server/d/")) {
-    smartlist_t *digests = smartlist_new();
-    key += strlen("/tor/server/d/");
-    dir_split_resource_into_fingerprints(key, digests, NULL,
-                                         DSR_HEX|DSR_SORT_UNIQ);
-    SMARTLIST_FOREACH(digests, const char *, d,
-       {
-         signed_descriptor_t *sd = router_get_by_descriptor_digest(d);
-         if (sd)
-           smartlist_add(descs_out,sd);
-       });
-    SMARTLIST_FOREACH(digests, char *, d, tor_free(d));
-    smartlist_free(digests);
-  } else if (!strcmpstart(key, "/tor/server/fp/")) {
-    smartlist_t *digests = smartlist_new();
-    time_t cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH;
-    key += strlen("/tor/server/fp/");
-    dir_split_resource_into_fingerprints(key, digests, NULL,
-                                         DSR_HEX|DSR_SORT_UNIQ);
-    SMARTLIST_FOREACH_BEGIN(digests, const char *, d) {
-         if (router_digest_is_me(d)) {
-           /* calling router_get_my_routerinfo() to make sure it exists */
-           const routerinfo_t *ri = router_get_my_routerinfo();
-           if (ri)
-             smartlist_add(descs_out, (void*) &(ri->cache_info));
-         } else {
-           const routerinfo_t *ri = router_get_by_id_digest(d);
-           /* Don't actually serve a descriptor that everyone will think is
-            * expired.  This is an (ugly) workaround to keep buggy 0.1.1.10
-            * Tors from downloading descriptors that they will throw away.
-            */
-           if (ri && ri->cache_info.published_on > cutoff)
-             smartlist_add(descs_out, (void*) &(ri->cache_info));
-         }
-    } SMARTLIST_FOREACH_END(d);
-    SMARTLIST_FOREACH(digests, char *, d, tor_free(d));
-    smartlist_free(digests);
-  } else {
-    *msg = "Key not recognized";
-    return -1;
-  }
-
-  if (!smartlist_len(descs_out)) {
-    *msg = "Servers unavailable";
-    return -1;
-  }
-  return 0;
-}
-
-/** Called when a TLS handshake has completed successfully with a
- * router listening at <b>address</b>:<b>or_port</b>, and has yielded
- * a certificate with digest <b>digest_rcvd</b>.
- *
- * Inform the reachability checker that we could get to this relay.
- */
-void
-dirserv_orconn_tls_done(const tor_addr_t *addr,
-                        uint16_t or_port,
-                        const char *digest_rcvd)
-{
-  node_t *node = NULL;
-  tor_addr_port_t orport;
-  routerinfo_t *ri = NULL;
-  time_t now = time(NULL);
-  tor_assert(addr);
-  tor_assert(digest_rcvd);
-
-  node = node_get_mutable_by_id(digest_rcvd);
-  if (node == NULL || node->ri == NULL)
-    return;
-  ri = node->ri;
-
-  tor_addr_copy(&orport.addr, addr);
-  orport.port = or_port;
-  if (router_has_orport(ri, &orport)) {
-    /* Found the right router.  */
-    if (!authdir_mode_bridge(get_options()) ||
-        ri->purpose == ROUTER_PURPOSE_BRIDGE) {
-      char addrstr[TOR_ADDR_BUF_LEN];
-      /* This is a bridge or we're not a bridge authorititative --
-         mark it as reachable.  */
-      log_info(LD_DIRSERV, "Found router %s to be reachable at %s:%d. Yay.",
-               router_describe(ri),
-               tor_addr_to_str(addrstr, addr, sizeof(addrstr), 1),
-               ri->or_port);
-      if (tor_addr_family(addr) == AF_INET) {
-        rep_hist_note_router_reachable(digest_rcvd, addr, or_port, now);
-        node->last_reachable = now;
-      } else if (tor_addr_family(addr) == AF_INET6) {
-        /* No rephist for IPv6.  */
-        node->last_reachable6 = now;
-      }
-    }
-  }
-}
-
-/** Called when we, as an authority, receive a new router descriptor either as
- * an upload or a download.  Used to decide whether to relaunch reachability
- * testing for the server. */
-int
-dirserv_should_launch_reachability_test(const routerinfo_t *ri,
-                                        const routerinfo_t *ri_old)
-{
-  if (!authdir_mode_handles_descs(get_options(), ri->purpose))
-    return 0;
-  if (!ri_old) {
-    /* New router: Launch an immediate reachability test, so we will have an
-     * opinion soon in case we're generating a consensus soon */
-    return 1;
-  }
-  if (ri_old->is_hibernating && !ri->is_hibernating) {
-    /* It just came out of hibernation; launch a reachability test */
-    return 1;
-  }
-  if (! routers_have_same_or_addrs(ri, ri_old)) {
-    /* Address or port changed; launch a reachability test */
-    return 1;
-  }
-  return 0;
-}
-
-/** Helper function for dirserv_test_reachability(). Start a TLS
- * connection to <b>router</b>, and annotate it with when we started
- * the test. */
-void
-dirserv_single_reachability_test(time_t now, routerinfo_t *router)
-{
-  channel_t *chan = NULL;
-  node_t *node = NULL;
-  tor_addr_t router_addr;
-  (void) now;
-
-  tor_assert(router);
-  node = node_get_mutable_by_id(router->cache_info.identity_digest);
-  tor_assert(node);
-
-  /* IPv4. */
-  log_debug(LD_OR,"Testing reachability of %s at %s:%u.",
-            router->nickname, fmt_addr32(router->addr), router->or_port);
-  tor_addr_from_ipv4h(&router_addr, router->addr);
-  chan = channel_tls_connect(&router_addr, router->or_port,
-                             router->cache_info.identity_digest);
-  if (chan) command_setup_channel(chan);
-
-  /* Possible IPv6. */
-  if (get_options()->AuthDirHasIPv6Connectivity == 1 &&
-      !tor_addr_is_null(&router->ipv6_addr)) {
-    char addrstr[TOR_ADDR_BUF_LEN];
-    log_debug(LD_OR, "Testing reachability of %s at %s:%u.",
-              router->nickname,
-              tor_addr_to_str(addrstr, &router->ipv6_addr, sizeof(addrstr), 1),
-              router->ipv6_orport);
-    chan = channel_tls_connect(&router->ipv6_addr, router->ipv6_orport,
-                               router->cache_info.identity_digest);
-    if (chan) command_setup_channel(chan);
-  }
-}
-
-/** Auth dir server only: load balance such that we only
- * try a few connections per call.
- *
- * The load balancing is such that if we get called once every ten
- * seconds, we will cycle through all the tests in
- * REACHABILITY_TEST_CYCLE_PERIOD seconds (a bit over 20 minutes).
- */
-void
-dirserv_test_reachability(time_t now)
-{
-  /* XXX decide what to do here; see or-talk thread "purging old router
-   * information, revocation." -NM
-   * We can't afford to mess with this in 0.1.2.x. The reason is that
-   * if we stop doing reachability tests on some of routerlist, then
-   * we'll for-sure think they're down, which may have unexpected
-   * effects in other parts of the code. It doesn't hurt much to do
-   * the testing, and directory authorities are easy to upgrade. Let's
-   * wait til 0.2.0. -RD */
-//  time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
-  routerlist_t *rl = router_get_routerlist();
-  static char ctr = 0;
-  int bridge_auth = authdir_mode_bridge(get_options());
-
-  SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, router) {
-    const char *id_digest = router->cache_info.identity_digest;
-    if (router_is_me(router))
-      continue;
-    if (bridge_auth && router->purpose != ROUTER_PURPOSE_BRIDGE)
-      continue; /* bridge authorities only test reachability on bridges */
-//    if (router->cache_info.published_on > cutoff)
-//      continue;
-    if ((((uint8_t)id_digest[0]) % REACHABILITY_MODULO_PER_TEST) == ctr) {
-      dirserv_single_reachability_test(now, router);
-    }
-  } SMARTLIST_FOREACH_END(router);
-  ctr = (ctr + 1) % REACHABILITY_MODULO_PER_TEST; /* increment ctr */
-}
-
-/** Given a fingerprint <b>fp</b> which is either set if we're looking for a
- * v2 status, or zeroes if we're looking for a v3 status, or a NUL-padded
- * flavor name if we want a flavored v3 status, return a pointer to the
- * appropriate cached dir object, or NULL if there isn't one available. */
-static cached_dir_t *
-lookup_cached_dir_by_fp(const char *fp)
-{
-  cached_dir_t *d = NULL;
-  if (tor_digest_is_zero(fp) && cached_consensuses) {
-    d = strmap_get(cached_consensuses, "ns");
-  } else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses &&
-           (d = strmap_get(cached_consensuses, fp))) {
-    /* this here interface is a nasty hack XXXX */;
-  }
-  return d;
-}
-
-/** Remove from <b>fps</b> every networkstatus key where both
- * a) we have a networkstatus document and
- * b) it is not newer than <b>cutoff</b>.
- *
- * Return 1 if any items were present at all; else return 0.
- */
-int
-dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff)
-{
-  int found_any = 0;
-  SMARTLIST_FOREACH_BEGIN(fps, char *, digest) {
-    cached_dir_t *d = lookup_cached_dir_by_fp(digest);
-    if (!d)
-      continue;
-    found_any = 1;
-    if (d->published <= cutoff) {
-      tor_free(digest);
-      SMARTLIST_DEL_CURRENT(fps, digest);
-    }
-  } SMARTLIST_FOREACH_END(digest);
-
-  return found_any;
-}
-
-/** Return the cache-info for identity fingerprint <b>fp</b>, or
- * its extra-info document if <b>extrainfo</b> is true. Return
- * NULL if not found or if the descriptor is older than
- * <b>publish_cutoff</b>. */
-static const signed_descriptor_t *
-get_signed_descriptor_by_fp(const char *fp, int extrainfo,
-                            time_t publish_cutoff)
-{
-  if (router_digest_is_me(fp)) {
-    if (extrainfo)
-      return &(router_get_my_extrainfo()->cache_info);
-    else
-      return &(router_get_my_routerinfo()->cache_info);
-  } else {
-    const routerinfo_t *ri = router_get_by_id_digest(fp);
-    if (ri &&
-        ri->cache_info.published_on > publish_cutoff) {
-      if (extrainfo)
-        return extrainfo_get_by_descriptor_digest(
-                                     ri->cache_info.extra_info_digest);
-      else
-        return &ri->cache_info;
-    }
-  }
-  return NULL;
-}
-
-/** Return true iff we have any of the documents (extrainfo or routerdesc)
- * specified by the fingerprints in <b>fps</b> and <b>spool_src</b>.  Used to
- * decide whether to send a 404.  */
-int
-dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src)
-{
-  time_t publish_cutoff = time(NULL)-ROUTER_MAX_AGE_TO_PUBLISH;
-  SMARTLIST_FOREACH_BEGIN(fps, const char *, fp) {
-      switch (spool_src)
-      {
-        case DIR_SPOOL_EXTRA_BY_DIGEST:
-          if (extrainfo_get_by_descriptor_digest(fp)) return 1;
-          break;
-        case DIR_SPOOL_SERVER_BY_DIGEST:
-          if (router_get_by_descriptor_digest(fp)) return 1;
-          break;
-        case DIR_SPOOL_EXTRA_BY_FP:
-        case DIR_SPOOL_SERVER_BY_FP:
-          if (get_signed_descriptor_by_fp(fp,
-                spool_src == DIR_SPOOL_EXTRA_BY_FP, publish_cutoff))
-            return 1;
-          break;
-      }
-  } SMARTLIST_FOREACH_END(fp);
-  return 0;
-}
-
-/** Return true iff any of the 256-bit elements in <b>fps</b> is the digest of
- * a microdescriptor we have. */
-int
-dirserv_have_any_microdesc(const smartlist_t *fps)
-{
-  microdesc_cache_t *cache = get_microdesc_cache();
-  SMARTLIST_FOREACH(fps, const char *, fp,
-                    if (microdesc_cache_lookup_by_digest256(cache, fp))
-                      return 1);
-  return 0;
-}
-
-/** Return an approximate estimate of the number of bytes that will
- * be needed to transmit the server descriptors (if is_serverdescs --
- * they can be either d/ or fp/ queries) or networkstatus objects (if
- * !is_serverdescs) listed in <b>fps</b>.  If <b>compressed</b> is set,
- * we guess how large the data will be after compression.
- *
- * The return value is an estimate; it might be larger or smaller.
- **/
-size_t
-dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs,
-                           int compressed)
-{
-  size_t result;
-  tor_assert(fps);
-  if (is_serverdescs) {
-    int n = smartlist_len(fps);
-    const routerinfo_t *me = router_get_my_routerinfo();
-    result = (me?me->cache_info.signed_descriptor_len:2048) * n;
-    if (compressed)
-      result /= 2; /* observed compressibility is between 35 and 55%. */
-  } else {
-    result = 0;
-    SMARTLIST_FOREACH(fps, const char *, digest, {
-        cached_dir_t *dir = lookup_cached_dir_by_fp(digest);
-        if (dir)
-          result += compressed ? dir->dir_z_len : dir->dir_len;
-      });
-  }
-  return result;
-}
-
-/** Given a list of microdescriptor hashes, guess how many bytes will be
- * needed to transmit them, and return the guess. */
-size_t
-dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed)
-{
-  size_t result = smartlist_len(fps) * microdesc_average_size(NULL);
-  if (compressed)
-    result /= 2;
-  return result;
-}
-
-/** When we're spooling data onto our outbuf, add more whenever we dip
- * below this threshold. */
-#define DIRSERV_BUFFER_MIN 16384
-
-/** Spooling helper: called when we have no more data to spool to <b>conn</b>.
- * Flushes any remaining data to be (un)compressed, and changes the spool
- * source to NONE.  Returns 0 on success, negative on failure. */
-static int
-connection_dirserv_finish_spooling(dir_connection_t *conn)
-{
-  if (conn->zlib_state) {
-    connection_write_to_buf_zlib("", 0, conn, 1);
-    tor_zlib_free(conn->zlib_state);
-    conn->zlib_state = NULL;
-  }
-  conn->dir_spool_src = DIR_SPOOL_NONE;
-  return 0;
-}
-
-/** Spooling helper: called when we're sending a bunch of server descriptors,
- * and the outbuf has become too empty. Pulls some entries from
- * fingerprint_stack, and writes the corresponding servers onto outbuf.  If we
- * run out of entries, flushes the zlib state and sets the spool source to
- * NONE.  Returns 0 on success, negative on failure.
- */
-static int
-connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn)
-{
-  int by_fp = (conn->dir_spool_src == DIR_SPOOL_SERVER_BY_FP ||
-               conn->dir_spool_src == DIR_SPOOL_EXTRA_BY_FP);
-  int extra = (conn->dir_spool_src == DIR_SPOOL_EXTRA_BY_FP ||
-               conn->dir_spool_src == DIR_SPOOL_EXTRA_BY_DIGEST);
-  time_t publish_cutoff = time(NULL)-ROUTER_MAX_AGE_TO_PUBLISH;
-
-  const or_options_t *options = get_options();
-
-  while (smartlist_len(conn->fingerprint_stack) &&
-         connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN) {
-    const char *body;
-    char *fp = smartlist_pop_last(conn->fingerprint_stack);
-    const signed_descriptor_t *sd = NULL;
-    if (by_fp) {
-      sd = get_signed_descriptor_by_fp(fp, extra, publish_cutoff);
-    } else {
-      sd = extra ? extrainfo_get_by_descriptor_digest(fp)
-        : router_get_by_descriptor_digest(fp);
-    }
-    tor_free(fp);
-    if (!sd)
-      continue;
-    if (!connection_dir_is_encrypted(conn) && !sd->send_unencrypted) {
-      /* we did this check once before (so we could have an accurate size
-       * estimate and maybe send a 404 if somebody asked for only bridges on a
-       * connection), but we need to do it again in case a previously
-       * unknown bridge descriptor has shown up between then and now. */
-      continue;
-    }
-
-    /** If we are the bridge authority and the descriptor is a bridge
-     * descriptor, remember that we served this descriptor for desc stats. */
-    if (options->BridgeAuthoritativeDir && by_fp) {
-      const routerinfo_t *router =
-          router_get_by_id_digest(sd->identity_digest);
-      /* router can be NULL here when the bridge auth is asked for its own
-       * descriptor. */
-      if (router && router->purpose == ROUTER_PURPOSE_BRIDGE)
-        rep_hist_note_desc_served(sd->identity_digest);
-    }
-    body = signed_descriptor_get_body(sd);
-    if (conn->zlib_state) {
-      int last = ! smartlist_len(conn->fingerprint_stack);
-      connection_write_to_buf_zlib(body, sd->signed_descriptor_len, conn,
-                                   last);
-      if (last) {
-        tor_zlib_free(conn->zlib_state);
-        conn->zlib_state = NULL;
-      }
-    } else {
-      connection_write_to_buf(body,
-                              sd->signed_descriptor_len,
-                              TO_CONN(conn));
-    }
-  }
-
-  if (!smartlist_len(conn->fingerprint_stack)) {
-    /* We just wrote the last one; finish up. */
-    if (conn->zlib_state) {
-      connection_write_to_buf_zlib("", 0, conn, 1);
-      tor_zlib_free(conn->zlib_state);
-      conn->zlib_state = NULL;
-    }
-    conn->dir_spool_src = DIR_SPOOL_NONE;
-    smartlist_free(conn->fingerprint_stack);
-    conn->fingerprint_stack = NULL;
-  }
-  return 0;
-}
-
-/** Spooling helper: called when we're sending a bunch of microdescriptors,
- * and the outbuf has become too empty. Pulls some entries from
- * fingerprint_stack, and writes the corresponding microdescs onto outbuf.  If
- * we run out of entries, flushes the zlib state and sets the spool source to
- * NONE.  Returns 0 on success, negative on failure.
- */
-static int
-connection_dirserv_add_microdescs_to_outbuf(dir_connection_t *conn)
-{
-  microdesc_cache_t *cache = get_microdesc_cache();
-  while (smartlist_len(conn->fingerprint_stack) &&
-         connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN) {
-    char *fp256 = smartlist_pop_last(conn->fingerprint_stack);
-    microdesc_t *md = microdesc_cache_lookup_by_digest256(cache, fp256);
-    tor_free(fp256);
-    if (!md || !md->body)
-      continue;
-    if (conn->zlib_state) {
-      int last = !smartlist_len(conn->fingerprint_stack);
-      connection_write_to_buf_zlib(md->body, md->bodylen, conn, last);
-      if (last) {
-        tor_zlib_free(conn->zlib_state);
-        conn->zlib_state = NULL;
-      }
-    } else {
-      connection_write_to_buf(md->body, md->bodylen, TO_CONN(conn));
-    }
-  }
-  if (!smartlist_len(conn->fingerprint_stack)) {
-    if (conn->zlib_state) {
-      connection_write_to_buf_zlib("", 0, conn, 1);
-      tor_zlib_free(conn->zlib_state);
-      conn->zlib_state = NULL;
-    }
-    conn->dir_spool_src = DIR_SPOOL_NONE;
-    smartlist_free(conn->fingerprint_stack);
-    conn->fingerprint_stack = NULL;
-  }
-  return 0;
-}
-
-/** Spooling helper: Called when we're sending a directory or networkstatus,
- * and the outbuf has become too empty.  Pulls some bytes from
- * <b>conn</b>-\>cached_dir-\>dir_z, uncompresses them if appropriate, and
- * puts them on the outbuf.  If we run out of entries, flushes the zlib state
- * and sets the spool source to NONE.  Returns 0 on success, negative on
- * failure. */
-static int
-connection_dirserv_add_dir_bytes_to_outbuf(dir_connection_t *conn)
-{
-  ssize_t bytes;
-  int64_t remaining;
-
-  bytes = DIRSERV_BUFFER_MIN - connection_get_outbuf_len(TO_CONN(conn));
-  tor_assert(bytes > 0);
-  tor_assert(conn->cached_dir);
-  if (bytes < 8192)
-    bytes = 8192;
-  remaining = conn->cached_dir->dir_z_len - conn->cached_dir_offset;
-  if (bytes > remaining)
-    bytes = (ssize_t) remaining;
-
-  if (conn->zlib_state) {
-    connection_write_to_buf_zlib(
-                             conn->cached_dir->dir_z + conn->cached_dir_offset,
-                             bytes, conn, bytes == remaining);
-  } else {
-    connection_write_to_buf(conn->cached_dir->dir_z + conn->cached_dir_offset,
-                            bytes, TO_CONN(conn));
-  }
-  conn->cached_dir_offset += bytes;
-  if (conn->cached_dir_offset == (int)conn->cached_dir->dir_z_len) {
-    /* We just wrote the last one; finish up. */
-    connection_dirserv_finish_spooling(conn);
-    cached_dir_decref(conn->cached_dir);
-    conn->cached_dir = NULL;
-  }
-  return 0;
-}
-
-/** Spooling helper: Called when we're spooling networkstatus objects on
- * <b>conn</b>, and the outbuf has become too empty.  If the current
- * networkstatus object (in <b>conn</b>-\>cached_dir) has more data, pull data
- * from there.  Otherwise, pop the next fingerprint from fingerprint_stack,
- * and start spooling the next networkstatus.  (A digest of all 0 bytes is
- * treated as a request for the current consensus.) If we run out of entries,
- * flushes the zlib state and sets the spool source to NONE.  Returns 0 on
- * success, negative on failure. */
-static int
-connection_dirserv_add_networkstatus_bytes_to_outbuf(dir_connection_t *conn)
-{
-
-  while (connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN) {
-    if (conn->cached_dir) {
-      int uncompressing = (conn->zlib_state != NULL);
-      int r = connection_dirserv_add_dir_bytes_to_outbuf(conn);
-      if (conn->dir_spool_src == DIR_SPOOL_NONE) {
-        /* add_dir_bytes thinks we're done with the cached_dir.  But we
-         * may have more cached_dirs! */
-        conn->dir_spool_src = DIR_SPOOL_NETWORKSTATUS;
-        /* This bit is tricky.  If we were uncompressing the last
-         * networkstatus, we may need to make a new zlib object to
-         * uncompress the next one. */
-        if (uncompressing && ! conn->zlib_state &&
-            conn->fingerprint_stack &&
-            smartlist_len(conn->fingerprint_stack)) {
-          conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION);
-        }
-      }
-      if (r) return r;
-    } else if (conn->fingerprint_stack &&
-               smartlist_len(conn->fingerprint_stack)) {
-      /* Add another networkstatus; start serving it. */
-      char *fp = smartlist_pop_last(conn->fingerprint_stack);
-      cached_dir_t *d = lookup_cached_dir_by_fp(fp);
-      tor_free(fp);
-      if (d) {
-        ++d->refcnt;
-        conn->cached_dir = d;
-        conn->cached_dir_offset = 0;
-      }
-    } else {
-      connection_dirserv_finish_spooling(conn);
-      smartlist_free(conn->fingerprint_stack);
-      conn->fingerprint_stack = NULL;
-      return 0;
-    }
-  }
-  return 0;
-}
-
-/** Called whenever we have flushed some directory data in state
- * SERVER_WRITING. */
-int
-connection_dirserv_flushed_some(dir_connection_t *conn)
-{
-  tor_assert(conn->base_.state == DIR_CONN_STATE_SERVER_WRITING);
-
-  if (connection_get_outbuf_len(TO_CONN(conn)) >= DIRSERV_BUFFER_MIN)
-    return 0;
-
-  switch (conn->dir_spool_src) {
-    case DIR_SPOOL_EXTRA_BY_DIGEST:
-    case DIR_SPOOL_EXTRA_BY_FP:
-    case DIR_SPOOL_SERVER_BY_DIGEST:
-    case DIR_SPOOL_SERVER_BY_FP:
-      return connection_dirserv_add_servers_to_outbuf(conn);
-    case DIR_SPOOL_MICRODESC:
-      return connection_dirserv_add_microdescs_to_outbuf(conn);
-    case DIR_SPOOL_CACHED_DIR:
-      return connection_dirserv_add_dir_bytes_to_outbuf(conn);
-    case DIR_SPOOL_NETWORKSTATUS:
-      return connection_dirserv_add_networkstatus_bytes_to_outbuf(conn);
-    case DIR_SPOOL_NONE:
-    default:
-      return 0;
-  }
-}
-
-/** Return true iff <b>line</b> is a valid RecommendedPackages line.
- */
-/*
-  The grammar is:
-
-    "package" SP PACKAGENAME SP VERSION SP URL SP DIGESTS NL
-
-      PACKAGENAME = NONSPACE
-      VERSION = NONSPACE
-      URL = NONSPACE
-      DIGESTS = DIGEST | DIGESTS SP DIGEST
-      DIGEST = DIGESTTYPE "=" DIGESTVAL
-
-      NONSPACE = one or more non-space printing characters
-
-      DIGESTVAL = DIGESTTYPE = one or more non-=, non-" " characters.
-
-      SP = " "
-      NL = a newline
-
- */
-int
-validate_recommended_package_line(const char *line)
-{
-  const char *cp = line;
-
-#define WORD()                                  \
-  do {                                          \
-    if (*cp == ' ')                             \
-      return 0;                                 \
-    cp = strchr(cp, ' ');                       \
-    if (!cp)                                    \
-      return 0;                                 \
-  } while (0)
-
-  WORD(); /* skip packagename */
-  ++cp;
-  WORD(); /* skip version */
-  ++cp;
-  WORD(); /* Skip URL */
-  ++cp;
-
-  /* Skip digesttype=digestval + */
-  int n_entries = 0;
-  while (1) {
-    const char *start_of_word = cp;
-    const char *end_of_word = strchr(cp, ' ');
-    if (! end_of_word)
-      end_of_word = cp + strlen(cp);
-
-    if (start_of_word == end_of_word)
-      return 0;
-
-    const char *eq = memchr(start_of_word, '=', end_of_word - start_of_word);
-
-    if (!eq)
-      return 0;
-    if (eq == start_of_word)
-      return 0;
-    if (eq == end_of_word - 1)
-      return 0;
-    if (memchr(eq+1, '=', end_of_word - (eq+1)))
-      return 0;
-
-    ++n_entries;
-    if (0 == *end_of_word)
-      break;
-
-    cp = end_of_word + 1;
-  }
-
-  /* If we reach this point, we have at least 1 entry. */
-  tor_assert(n_entries > 0);
-  return 1;
-}
-
-/** Release all storage used by the directory server. */
-void
-dirserv_free_all(void)
-{
-  dirserv_free_fingerprint_list();
-
-  strmap_free(cached_consensuses, free_cached_dir_);
-  cached_consensuses = NULL;
-
-  dirserv_clear_measured_bw_cache();
-}
diff --git a/src/or/dirserv.h b/src/or/dirserv.h
deleted file mode 100644
index 624cd7e0b7..0000000000
--- a/src/or/dirserv.h
+++ /dev/null
@@ -1,143 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file dirserv.h
- * \brief Header file for dirserv.c.
- **/
-
-#ifndef TOR_DIRSERV_H
-#define TOR_DIRSERV_H
-
-#include "testsupport.h"
-
-/** What fraction (1 over this number) of the relay ID space do we
- * (as a directory authority) launch connections to at each reachability
- * test? */
-#define REACHABILITY_MODULO_PER_TEST 128
-
-/** How often (in seconds) do we launch reachability tests? */
-#define REACHABILITY_TEST_INTERVAL 10
-
-/** How many seconds apart are the reachability tests for a given relay? */
-#define REACHABILITY_TEST_CYCLE_PERIOD \
-  (REACHABILITY_TEST_INTERVAL*REACHABILITY_MODULO_PER_TEST)
-
-/** Maximum length of an exit policy summary. */
-#define MAX_EXITPOLICY_SUMMARY_LEN 1000
-
-/** Maximum allowable length of a version line in a networkstatus. */
-#define MAX_V_LINE_LEN 128
-
-int connection_dirserv_flushed_some(dir_connection_t *conn);
-
-int dirserv_add_own_fingerprint(crypto_pk_t *pk);
-int dirserv_load_fingerprint_file(void);
-void dirserv_free_fingerprint_list(void);
-enum was_router_added_t dirserv_add_multiple_descriptors(
-                                     const char *desc, uint8_t purpose,
-                                     const char *source,
-                                     const char **msg);
-enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri,
-                                               const char **msg,
-                                               const char *source);
-void dirserv_set_router_is_running(routerinfo_t *router, time_t now);
-int list_server_status_v1(smartlist_t *routers, char **router_status_out,
-                          int for_controller);
-char *dirserv_get_flag_thresholds_line(void);
-void dirserv_compute_bridge_flag_thresholds(void);
-
-int directory_fetches_from_authorities(const or_options_t *options);
-int directory_fetches_dir_info_early(const or_options_t *options);
-int directory_fetches_dir_info_later(const or_options_t *options);
-int directory_caches_unknown_auth_certs(const or_options_t *options);
-int directory_caches_dir_info(const or_options_t *options);
-int directory_permits_begindir_requests(const or_options_t *options);
-int directory_too_idle_to_fetch_descriptors(const or_options_t *options,
-                                            time_t now);
-
-cached_dir_t *dirserv_get_consensus(const char *flavor_name);
-void dirserv_set_cached_consensus_networkstatus(const char *consensus,
-                                              const char *flavor_name,
-                                              const common_digests_t *digests,
-                                              time_t published);
-void dirserv_clear_old_networkstatuses(time_t cutoff);
-int dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key,
-                                        const char **msg,
-                                        int for_unencrypted_conn,
-                                        int is_extrainfo);
-int dirserv_get_routerdescs(smartlist_t *descs_out, const char *key,
-                            const char **msg);
-void dirserv_orconn_tls_done(const tor_addr_t *addr,
-                             uint16_t or_port,
-                             const char *digest_rcvd);
-int dirserv_should_launch_reachability_test(const routerinfo_t *ri,
-                                            const routerinfo_t *ri_old);
-void dirserv_single_reachability_test(time_t now, routerinfo_t *router);
-void dirserv_test_reachability(time_t now);
-int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
-                                   int complain,
-                                   int *valid_out);
-uint32_t dirserv_router_get_status(const routerinfo_t *router,
-                                   const char **msg,
-                                   int severity);
-void dirserv_set_node_flags_from_authoritative_status(node_t *node,
-                                                      uint32_t authstatus);
-
-int dirserv_would_reject_router(const routerstatus_t *rs);
-int dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff);
-int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src);
-int dirserv_have_any_microdesc(const smartlist_t *fps);
-size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs,
-                                  int compressed);
-size_t dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed);
-
-char *routerstatus_format_entry(
-                              const routerstatus_t *rs,
-                              const char *version,
-                              const char *protocols,
-                              routerstatus_format_type_t format,
-                              const vote_routerstatus_t *vrs);
-void dirserv_free_all(void);
-void cached_dir_decref(cached_dir_t *d);
-cached_dir_t *new_cached_dir(char *s, time_t published);
-char *format_recommended_version_list(const config_line_t *line, int warn);
-int validate_recommended_package_line(const char *line);
-
-#ifdef DIRSERV_PRIVATE
-
-STATIC void dirserv_set_routerstatus_testing(routerstatus_t *rs);
-
-/* Put the MAX_MEASUREMENT_AGE #define here so unit tests can see it */
-#define MAX_MEASUREMENT_AGE (3*24*60*60) /* 3 days */
-
-STATIC int measured_bw_line_parse(measured_bw_line_t *out, const char *line);
-
-STATIC int measured_bw_line_apply(measured_bw_line_t *parsed_line,
-                           smartlist_t *routerstatuses);
-
-STATIC void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line,
-                               time_t as_of);
-STATIC void dirserv_clear_measured_bw_cache(void);
-STATIC void dirserv_expire_measured_bw_cache(time_t now);
-STATIC int dirserv_get_measured_bw_cache_size(void);
-STATIC int dirserv_query_measured_bw_cache_kb(const char *node_id,
-                                              long *bw_out,
-                                              time_t *as_of_out);
-STATIC int dirserv_has_measured_bw(const char *node_id);
-
-STATIC int
-dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str,
-                                      smartlist_t *vote_routerstatuses);
-#endif
-
-int dirserv_read_measured_bandwidths(const char *from_file,
-                                     smartlist_t *routerstatuses);
-
-int dirserv_read_guardfraction_file(const char *fname,
-                                 smartlist_t *vote_routerstatuses);
-
-#endif
diff --git a/src/or/entrynodes.c b/src/or/entrynodes.c
deleted file mode 100644
index 265b6dcda1..0000000000
--- a/src/or/entrynodes.c
+++ /dev/null
@@ -1,2561 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file entrynodes.c
- * \brief Code to manage our fixed first nodes for various functions.
- *
- * Entry nodes can be guards (for general use) or bridges (for censorship
- * circumvention).
- **/
-
-#define ENTRYNODES_PRIVATE
-
-#include "or.h"
-#include "circpathbias.h"
-#include "circuitbuild.h"
-#include "circuitstats.h"
-#include "config.h"
-#include "confparse.h"
-#include "connection.h"
-#include "connection_or.h"
-#include "control.h"
-#include "directory.h"
-#include "entrynodes.h"
-#include "main.h"
-#include "microdesc.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "policies.h"
-#include "router.h"
-#include "routerlist.h"
-#include "routerparse.h"
-#include "routerset.h"
-#include "transports.h"
-#include "statefile.h"
-
-/** Information about a configured bridge. Currently this just matches the
- * ones in the torrc file, but one day we may be able to learn about new
- * bridges on our own, and remember them in the state file. */
-typedef struct {
-  /** Address of the bridge. */
-  tor_addr_t addr;
-  /** TLS port for the bridge. */
-  uint16_t port;
-  /** Boolean: We are re-parsing our bridge list, and we are going to remove
-   * this one if we don't find it in the list of configured bridges. */
-  unsigned marked_for_removal : 1;
-  /** Expected identity digest, or all zero bytes if we don't know what the
-   * digest should be. */
-  char identity[DIGEST_LEN];
-
-  /** Name of pluggable transport protocol taken from its config line. */
-  char *transport_name;
-
-  /** When should we next try to fetch a descriptor for this bridge? */
-  download_status_t fetch_status;
-
-  /** A smartlist of k=v values to be passed to the SOCKS proxy, if
-      transports are used for this bridge. */
-  smartlist_t *socks_args;
-} bridge_info_t;
-
-/** A list of our chosen entry guards. */
-static smartlist_t *entry_guards = NULL;
-/** A value of 1 means that the entry_guards list has changed
- * and those changes need to be flushed to disk. */
-static int entry_guards_dirty = 0;
-
-static void bridge_free(bridge_info_t *bridge);
-static const node_t *choose_random_entry_impl(cpath_build_state_t *state,
-                                              int for_directory,
-                                              dirinfo_type_t dirtype,
-                                              int *n_options_out);
-static int num_bridges_usable(void);
-
-/* Default number of entry guards in the case where the NumEntryGuards
- * consensus parameter is not set */
-#define DEFAULT_N_GUARDS 1
-/* Minimum and maximum number of entry guards (in case the NumEntryGuards
- * consensus parameter is set). */
-#define MIN_N_GUARDS 1
-#define MAX_N_GUARDS 10
-
-/** Return the list of entry guards, creating it if necessary. */
-const smartlist_t *
-get_entry_guards(void)
-{
-  if (! entry_guards)
-    entry_guards = smartlist_new();
-  return entry_guards;
-}
-
-/** Check whether the entry guard <b>e</b> is usable, given the directory
- * authorities' opinion about the router (stored in <b>ri</b>) and the user's
- * configuration (in <b>options</b>). Set <b>e</b>->bad_since
- * accordingly. Return true iff the entry guard's status changes.
- *
- * If it's not usable, set *<b>reason</b> to a static string explaining why.
- */
-static int
-entry_guard_set_status(entry_guard_t *e, const node_t *node,
-                       time_t now, const or_options_t *options,
-                       const char **reason)
-{
-  char buf[HEX_DIGEST_LEN+1];
-  int changed = 0;
-
-  *reason = NULL;
-
-  /* Do we want to mark this guard as bad? */
-  if (!node)
-    *reason = "unlisted";
-  else if (!node->is_running)
-    *reason = "down";
-  else if (options->UseBridges && (!node->ri ||
-                                   node->ri->purpose != ROUTER_PURPOSE_BRIDGE))
-    *reason = "not a bridge";
-  else if (options->UseBridges && !node_is_a_configured_bridge(node))
-    *reason = "not a configured bridge";
-  else if (!options->UseBridges && !node->is_possible_guard &&
-           !routerset_contains_node(options->EntryNodes,node))
-    *reason = "not recommended as a guard";
-  else if (routerset_contains_node(options->ExcludeNodes, node))
-    *reason = "excluded";
-  /* We only care about OR connection connectivity for entry guards. */
-  else if (!fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION, 0))
-    *reason = "unreachable by config";
-  else if (e->path_bias_disabled)
-    *reason = "path-biased";
-
-  if (*reason && ! e->bad_since) {
-    /* Router is newly bad. */
-    base16_encode(buf, sizeof(buf), e->identity, DIGEST_LEN);
-    log_info(LD_CIRC, "Entry guard %s (%s) is %s: marking as unusable.",
-             e->nickname, buf, *reason);
-
-    e->bad_since = now;
-    control_event_guard(e->nickname, e->identity, "BAD");
-    changed = 1;
-  } else if (!*reason && e->bad_since) {
-    /* There's nothing wrong with the router any more. */
-    base16_encode(buf, sizeof(buf), e->identity, DIGEST_LEN);
-    log_info(LD_CIRC, "Entry guard %s (%s) is no longer unusable: "
-             "marking as ok.", e->nickname, buf);
-
-    e->bad_since = 0;
-    control_event_guard(e->nickname, e->identity, "GOOD");
-    changed = 1;
-  }
-
-  if (node) {
-    int is_dir = node_is_dir(node);
-    if (options->UseBridges && node_is_a_configured_bridge(node))
-      is_dir = 1;
-    if (e->is_dir_cache != is_dir) {
-      e->is_dir_cache = is_dir;
-      changed = 1;
-    }
-  }
-
-  return changed;
-}
-
-/** Return true iff enough time has passed since we last tried to connect
- * to the unreachable guard <b>e</b> that we're willing to try again. */
-STATIC int
-entry_is_time_to_retry(const entry_guard_t *e, time_t now)
-{
-  struct guard_retry_period_s {
-    time_t period_duration;
-    time_t interval_during_period;
-  };
-
-  struct guard_retry_period_s periods[] = {
-     {    6*60*60,    60*60 }, /* For first 6 hrs., retry hourly; */
-     { 3*24*60*60,  4*60*60 }, /* Then retry every 4 hrs. until the
-                                  3-day mark; */
-     { 7*24*60*60, 18*60*60 }, /* After 3 days, retry every 18 hours until
-                                  1 week mark. */
-     {   TIME_MAX, 36*60*60 }  /* After 1 week, retry every 36 hours. */
-  };
-
-  time_t ith_deadline_for_retry;
-  time_t unreachable_for;
-  unsigned i;
-
-  if (e->last_attempted < e->unreachable_since)
-    return 1;
-
-  unreachable_for = now - e->unreachable_since;
-
-  for (i = 0; i < ARRAY_LENGTH(periods); i++) {
-    if (unreachable_for <= periods[i].period_duration) {
-      ith_deadline_for_retry = e->last_attempted +
-                               periods[i].interval_during_period;
-
-      return (now > ith_deadline_for_retry);
-    }
-  }
-  return 0;
-}
-
-/** Return the node corresponding to <b>e</b>, if <b>e</b> is
- * working well enough that we are willing to use it as an entry
- * right now. (Else return NULL.) In particular, it must be
- * - Listed as either up or never yet contacted;
- * - Present in the routerlist;
- * - Listed as 'stable' or 'fast' by the current dirserver consensus,
- *   if demanded by <b>need_uptime</b> or <b>need_capacity</b>
- *   (unless it's a configured EntryNode);
- * - Allowed by our current ReachableORAddresses config option; and
- * - Currently thought to be reachable by us (unless <b>assume_reachable</b>
- *   is true).
- *
- * If the answer is no, set *<b>msg</b> to an explanation of why.
- *
- * If need_descriptor is true, only return the node if we currently have
- * a descriptor (routerinfo or microdesc) for it.
- */
-STATIC const node_t *
-entry_is_live(const entry_guard_t *e, entry_is_live_flags_t flags,
-              const char **msg)
-{
-  const node_t *node;
-  const or_options_t *options = get_options();
-  int need_uptime = (flags & ENTRY_NEED_UPTIME) != 0;
-  int need_capacity = (flags & ENTRY_NEED_CAPACITY) != 0;
-  const int assume_reachable = (flags & ENTRY_ASSUME_REACHABLE) != 0;
-  const int need_descriptor = (flags & ENTRY_NEED_DESCRIPTOR) != 0;
-
-  tor_assert(msg);
-
-  if (e->path_bias_disabled) {
-    *msg = "path-biased";
-    return NULL;
-  }
-  if (e->bad_since) {
-    *msg = "bad";
-    return NULL;
-  }
-  /* no good if it's unreachable, unless assume_unreachable or can_retry. */
-  if (!assume_reachable && !e->can_retry &&
-      e->unreachable_since && !entry_is_time_to_retry(e, time(NULL))) {
-    *msg = "unreachable";
-    return NULL;
-  }
-  node = node_get_by_id(e->identity);
-  if (!node) {
-    *msg = "no node info";
-    return NULL;
-  }
-  if (need_descriptor && !node_has_descriptor(node)) {
-    *msg = "no descriptor";
-    return NULL;
-  }
-  if (get_options()->UseBridges) {
-    if (node_get_purpose(node) != ROUTER_PURPOSE_BRIDGE) {
-      *msg = "not a bridge";
-      return NULL;
-    }
-    if (!node_is_a_configured_bridge(node)) {
-      *msg = "not a configured bridge";
-      return NULL;
-    }
-  } else { /* !get_options()->UseBridges */
-    if (node_get_purpose(node) != ROUTER_PURPOSE_GENERAL) {
-      *msg = "not general-purpose";
-      return NULL;
-    }
-  }
-  if (routerset_contains_node(options->EntryNodes, node)) {
-    /* they asked for it, they get it */
-    need_uptime = need_capacity = 0;
-  }
-  if (node_is_unreliable(node, need_uptime, need_capacity, 0)) {
-    *msg = "not fast/stable";
-    return NULL;
-  }
-  if (!fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION, 0)) {
-    *msg = "unreachable by config";
-    return NULL;
-  }
-  return node;
-}
-
-/** Return the number of entry guards that we think are usable. */
-int
-num_live_entry_guards(int for_directory)
-{
-  int n = 0;
-  const char *msg;
-  /* Set the entry node attributes we are interested in. */
-  entry_is_live_flags_t entry_flags = ENTRY_NEED_CAPACITY;
-  if (!for_directory) {
-    entry_flags |= ENTRY_NEED_DESCRIPTOR;
-  }
-
-  if (! entry_guards)
-    return 0;
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) {
-      if (for_directory && !entry->is_dir_cache)
-        continue;
-      if (entry_is_live(entry, entry_flags, &msg))
-        ++n;
-  } SMARTLIST_FOREACH_END(entry);
-  return n;
-}
-
-/** If <b>digest</b> matches the identity of any node in the
- * entry_guards list, return that node. Else return NULL. */
-entry_guard_t *
-entry_guard_get_by_id_digest(const char *digest)
-{
-  SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry,
-                    if (tor_memeq(digest, entry->identity, DIGEST_LEN))
-                      return entry;
-                   );
-  return NULL;
-}
-
-/** Dump a description of our list of entry guards to the log at level
- * <b>severity</b>. */
-static void
-log_entry_guards(int severity)
-{
-  smartlist_t *elements = smartlist_new();
-  char *s;
-
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e)
-    {
-      const char *msg = NULL;
-      if (entry_is_live(e, ENTRY_NEED_CAPACITY, &msg))
-        smartlist_add_asprintf(elements, "%s [%s] (up %s)",
-                     e->nickname,
-                     hex_str(e->identity, DIGEST_LEN),
-                     e->made_contact ? "made-contact" : "never-contacted");
-      else
-        smartlist_add_asprintf(elements, "%s [%s] (%s, %s)",
-                     e->nickname,
-                     hex_str(e->identity, DIGEST_LEN),
-                     msg,
-                     e->made_contact ? "made-contact" : "never-contacted");
-    }
-  SMARTLIST_FOREACH_END(e);
-
-  s = smartlist_join_strings(elements, ",", 0, NULL);
-  SMARTLIST_FOREACH(elements, char*, cp, tor_free(cp));
-  smartlist_free(elements);
-  log_fn(severity,LD_CIRC,"%s",s);
-  tor_free(s);
-}
-
-/** Called when one or more guards that we would previously have used for some
- * purpose are no longer in use because a higher-priority guard has become
- * usable again. */
-static void
-control_event_guard_deferred(void)
-{
-  /* XXXX We don't actually have a good way to figure out _how many_ entries
-   * are live for some purpose.  We need an entry_is_even_slightly_live()
-   * function for this to work right.  NumEntryGuards isn't reliable: if we
-   * need guards with weird properties, we can have more than that number
-   * live.
-   **/
-#if 0
-  int n = 0;
-  const char *msg;
-  const or_options_t *options = get_options();
-  if (!entry_guards)
-    return;
-  SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry,
-    {
-      if (entry_is_live(entry, 0, 1, 0, &msg)) {
-        if (n++ == options->NumEntryGuards) {
-          control_event_guard(entry->nickname, entry->identity, "DEFERRED");
-          return;
-        }
-      }
-    });
-#endif
-}
-
-/** Largest amount that we'll backdate chosen_on_date */
-#define CHOSEN_ON_DATE_SLOP (30*86400)
-
-/** Add a new (preferably stable and fast) router to our
- * entry_guards list. Return a pointer to the router if we succeed,
- * or NULL if we can't find any more suitable entries.
- *
- * If <b>chosen</b> is defined, use that one, and if it's not
- * already in our entry_guards list, put it at the *beginning*.
- * Else, put the one we pick at the end of the list. */
-STATIC const node_t *
-add_an_entry_guard(const node_t *chosen, int reset_status, int prepend,
-                   int for_discovery, int for_directory)
-{
-  const node_t *node;
-  entry_guard_t *entry;
-
-  if (chosen) {
-    node = chosen;
-    entry = entry_guard_get_by_id_digest(node->identity);
-    if (entry) {
-      if (reset_status) {
-        entry->bad_since = 0;
-        entry->can_retry = 1;
-      }
-      entry->is_dir_cache = node_is_dir(node);
-      if (get_options()->UseBridges && node_is_a_configured_bridge(node))
-        entry->is_dir_cache = 1;
-
-      return NULL;
-    }
-  } else if (!for_directory) {
-    node = choose_good_entry_server(CIRCUIT_PURPOSE_C_GENERAL, NULL);
-    if (!node)
-      return NULL;
-  } else {
-    const routerstatus_t *rs;
-    rs = router_pick_directory_server(MICRODESC_DIRINFO|V3_DIRINFO,
-                                      PDS_FOR_GUARD);
-    if (!rs)
-      return NULL;
-    node = node_get_by_id(rs->identity_digest);
-    if (!node)
-      return NULL;
-  }
-  if (node->using_as_guard)
-    return NULL;
-  if (entry_guard_get_by_id_digest(node->identity) != NULL) {
-    log_info(LD_CIRC, "I was about to add a duplicate entry guard.");
-    /* This can happen if we choose a guard, then the node goes away, then
-     * comes back. */
-    ((node_t*) node)->using_as_guard = 1;
-    return NULL;
-  }
-  entry = tor_malloc_zero(sizeof(entry_guard_t));
-  log_info(LD_CIRC, "Chose %s as new entry guard.",
-           node_describe(node));
-  strlcpy(entry->nickname, node_get_nickname(node), sizeof(entry->nickname));
-  memcpy(entry->identity, node->identity, DIGEST_LEN);
-  entry->is_dir_cache = node_is_dir(node);
-  if (get_options()->UseBridges && node_is_a_configured_bridge(node))
-    entry->is_dir_cache = 1;
-
-  /* Choose expiry time smudged over the past month. The goal here
-   * is to a) spread out when Tor clients rotate their guards, so they
-   * don't all select them on the same day, and b) avoid leaving a
-   * precise timestamp in the state file about when we first picked
-   * this guard. For details, see the Jan 2010 or-dev thread. */
-  time_t now = time(NULL);
-  entry->chosen_on_date = crypto_rand_time_range(now - 3600*24*30, now);
-  entry->chosen_by_version = tor_strdup(VERSION);
-
-  /* Are we picking this guard because all of our current guards are
-   * down so we need another one (for_discovery is 1), or because we
-   * decided we need more variety in our guard list (for_discovery is 0)?
-   *
-   * Currently we hack this behavior into place by setting "made_contact"
-   * for guards of the latter variety, so we'll be willing to use any of
-   * them right off the bat.
-   */
-  if (!for_discovery)
-    entry->made_contact = 1;
-
-  ((node_t*)node)->using_as_guard = 1;
-  if (prepend)
-    smartlist_insert(entry_guards, 0, entry);
-  else
-    smartlist_add(entry_guards, entry);
-  control_event_guard(entry->nickname, entry->identity, "NEW");
-  control_event_guard_deferred();
-  log_entry_guards(LOG_INFO);
-  return node;
-}
-
-/** Choose how many entry guards or directory guards we'll use. If
- * <b>for_directory</b> is true, we return how many directory guards to
- * use; else we return how many entry guards to use. */
-STATIC int
-decide_num_guards(const or_options_t *options, int for_directory)
-{
-  if (for_directory) {
-    int answer;
-    if (options->NumDirectoryGuards != 0)
-      return options->NumDirectoryGuards;
-    answer = networkstatus_get_param(NULL, "NumDirectoryGuards", 0, 0, 10);
-    if (answer) /* non-zero means use the consensus value */
-      return answer;
-  }
-
-  if (options->NumEntryGuards)
-    return options->NumEntryGuards;
-
-  /* Use the value from the consensus, or 3 if no guidance. */
-  return networkstatus_get_param(NULL, "NumEntryGuards", DEFAULT_N_GUARDS,
-                                 MIN_N_GUARDS, MAX_N_GUARDS);
-}
-
-/** If the use of entry guards is configured, choose more entry guards
- * until we have enough in the list. */
-static void
-pick_entry_guards(const or_options_t *options, int for_directory)
-{
-  int changed = 0;
-  const int num_needed = decide_num_guards(options, for_directory);
-
-  tor_assert(entry_guards);
-
-  while (num_live_entry_guards(for_directory) < num_needed) {
-    if (!add_an_entry_guard(NULL, 0, 0, 0, for_directory))
-      break;
-    changed = 1;
-  }
-  if (changed)
-    entry_guards_changed();
-}
-
-/** How long (in seconds) do we allow an entry guard to be nonfunctional,
- * unlisted, excluded, or otherwise nonusable before we give up on it? */
-#define ENTRY_GUARD_REMOVE_AFTER (30*24*60*60)
-
-/** Release all storage held by <b>e</b>. */
-static void
-entry_guard_free(entry_guard_t *e)
-{
-  if (!e)
-    return;
-  tor_free(e->chosen_by_version);
-  tor_free(e);
-}
-
-/**
- * Return the minimum lifetime of working entry guard, in seconds,
- * as given in the consensus networkstatus.  (Plus CHOSEN_ON_DATE_SLOP,
- * so that we can do the chosen_on_date randomization while achieving the
- * desired minimum lifetime.)
- */
-static int32_t
-guards_get_lifetime(void)
-{
-  const or_options_t *options = get_options();
-#define DFLT_GUARD_LIFETIME (86400 * 60)   /* Two months. */
-#define MIN_GUARD_LIFETIME  (86400 * 30)   /* One months. */
-#define MAX_GUARD_LIFETIME  (86400 * 1826) /* Five years. */
-
-  if (options->GuardLifetime >= 1) {
-    return CLAMP(MIN_GUARD_LIFETIME,
-                 options->GuardLifetime,
-                 MAX_GUARD_LIFETIME) + CHOSEN_ON_DATE_SLOP;
-  }
-
-  return networkstatus_get_param(NULL, "GuardLifetime",
-                                 DFLT_GUARD_LIFETIME,
-                                 MIN_GUARD_LIFETIME,
-                                 MAX_GUARD_LIFETIME) + CHOSEN_ON_DATE_SLOP;
-}
-
-/** Remove any entry guard which was selected by an unknown version of Tor,
- * or which was selected by a version of Tor that's known to select
- * entry guards badly, or which was selected more 2 months ago. */
-/* XXXX The "obsolete guards" and "chosen long ago guards" things should
- * probably be different functions. */
-static int
-remove_obsolete_entry_guards(time_t now)
-{
-  int changed = 0, i;
-  int32_t guard_lifetime = guards_get_lifetime();
-
-  for (i = 0; i < smartlist_len(entry_guards); ++i) {
-    entry_guard_t *entry = smartlist_get(entry_guards, i);
-    const char *ver = entry->chosen_by_version;
-    const char *msg = NULL;
-    tor_version_t v;
-    int version_is_bad = 0, date_is_bad = 0;
-    if (!ver) {
-      msg = "does not say what version of Tor it was selected by";
-      version_is_bad = 1;
-    } else if (tor_version_parse(ver, &v)) {
-      msg = "does not seem to be from any recognized version of Tor";
-      version_is_bad = 1;
-    }
-    if (!version_is_bad && entry->chosen_on_date + guard_lifetime < now) {
-      /* It's been too long since the date listed in our state file. */
-      msg = "was selected several months ago";
-      date_is_bad = 1;
-    }
-
-    if (version_is_bad || date_is_bad) { /* we need to drop it */
-      char dbuf[HEX_DIGEST_LEN+1];
-      tor_assert(msg);
-      base16_encode(dbuf, sizeof(dbuf), entry->identity, DIGEST_LEN);
-      log_fn(version_is_bad ? LOG_NOTICE : LOG_INFO, LD_CIRC,
-             "Entry guard '%s' (%s) %s. (Version=%s.) Replacing it.",
-             entry->nickname, dbuf, msg, ver?escaped(ver):"none");
-      control_event_guard(entry->nickname, entry->identity, "DROPPED");
-      entry_guard_free(entry);
-      smartlist_del_keeporder(entry_guards, i--);
-      log_entry_guards(LOG_INFO);
-      changed = 1;
-    }
-  }
-
-  return changed ? 1 : 0;
-}
-
-/** Remove all entry guards that have been down or unlisted for so
- * long that we don't think they'll come up again. Return 1 if we
- * removed any, or 0 if we did nothing. */
-static int
-remove_dead_entry_guards(time_t now)
-{
-  char dbuf[HEX_DIGEST_LEN+1];
-  char tbuf[ISO_TIME_LEN+1];
-  int i;
-  int changed = 0;
-
-  for (i = 0; i < smartlist_len(entry_guards); ) {
-    entry_guard_t *entry = smartlist_get(entry_guards, i);
-    if (entry->bad_since &&
-        ! entry->path_bias_disabled &&
-        entry->bad_since + ENTRY_GUARD_REMOVE_AFTER < now) {
-
-      base16_encode(dbuf, sizeof(dbuf), entry->identity, DIGEST_LEN);
-      format_local_iso_time(tbuf, entry->bad_since);
-      log_info(LD_CIRC, "Entry guard '%s' (%s) has been down or unlisted "
-               "since %s local time; removing.",
-               entry->nickname, dbuf, tbuf);
-      control_event_guard(entry->nickname, entry->identity, "DROPPED");
-      entry_guard_free(entry);
-      smartlist_del_keeporder(entry_guards, i);
-      log_entry_guards(LOG_INFO);
-      changed = 1;
-    } else
-      ++i;
-  }
-  return changed ? 1 : 0;
-}
-
-/** Remove all currently listed entry guards. So new ones will be chosen. */
-void
-remove_all_entry_guards(void)
-{
-  char dbuf[HEX_DIGEST_LEN+1];
-
-  while (smartlist_len(entry_guards)) {
-    entry_guard_t *entry = smartlist_get(entry_guards, 0);
-    base16_encode(dbuf, sizeof(dbuf), entry->identity, DIGEST_LEN);
-    log_info(LD_CIRC, "Entry guard '%s' (%s) has been dropped.",
-             entry->nickname, dbuf);
-    control_event_guard(entry->nickname, entry->identity, "DROPPED");
-    entry_guard_free(entry);
-    smartlist_del(entry_guards, 0);
-  }
-  log_entry_guards(LOG_INFO);
-  entry_guards_changed();
-}
-
-/** A new directory or router-status has arrived; update the down/listed
- * status of the entry guards.
- *
- * An entry is 'down' if the directory lists it as nonrunning.
- * An entry is 'unlisted' if the directory doesn't include it.
- *
- * Don't call this on startup; only on a fresh download. Otherwise we'll
- * think that things are unlisted.
- */
-void
-entry_guards_compute_status(const or_options_t *options, time_t now)
-{
-  int changed = 0;
-  digestmap_t *reasons;
-
-  if (! entry_guards)
-    return;
-
-  if (options->EntryNodes) /* reshuffle the entry guard list if needed */
-    entry_nodes_should_be_added();
-
-  reasons = digestmap_new();
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry)
-    {
-      const node_t *r = node_get_by_id(entry->identity);
-      const char *reason = NULL;
-      if (entry_guard_set_status(entry, r, now, options, &reason))
-        changed = 1;
-
-      if (entry->bad_since)
-        tor_assert(reason);
-      if (reason)
-        digestmap_set(reasons, entry->identity, (char*)reason);
-    }
-  SMARTLIST_FOREACH_END(entry);
-
-  if (remove_dead_entry_guards(now))
-    changed = 1;
-  if (remove_obsolete_entry_guards(now))
-    changed = 1;
-
-  if (changed) {
-    SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) {
-      const char *reason = digestmap_get(reasons, entry->identity);
-      const char *live_msg = "";
-      const node_t *r = entry_is_live(entry, ENTRY_NEED_CAPACITY, &live_msg);
-      log_info(LD_CIRC, "Summary: Entry %s [%s] is %s, %s%s%s, and %s%s.",
-               entry->nickname,
-               hex_str(entry->identity, DIGEST_LEN),
-               entry->unreachable_since ? "unreachable" : "reachable",
-               entry->bad_since ? "unusable" : "usable",
-               reason ? ", ": "",
-               reason ? reason : "",
-               r ? "live" : "not live / ",
-               r ? "" : live_msg);
-    } SMARTLIST_FOREACH_END(entry);
-    log_info(LD_CIRC, "    (%d/%d entry guards are usable/new)",
-             num_live_entry_guards(0), smartlist_len(entry_guards));
-    log_entry_guards(LOG_INFO);
-    entry_guards_changed();
-  }
-
-  digestmap_free(reasons, NULL);
-}
-
-/** Called when a connection to an OR with the identity digest <b>digest</b>
- * is established (<b>succeeded</b>==1) or has failed (<b>succeeded</b>==0).
- * If the OR is an entry, change that entry's up/down status.
- * Return 0 normally, or -1 if we want to tear down the new connection.
- *
- * If <b>mark_relay_status</b>, also call router_set_status() on this
- * relay.
- */
-/* XXX We could change succeeded and mark_relay_status into 'int flags'.
- * Too many boolean arguments is a recipe for confusion.
- */
-int
-entry_guard_register_connect_status(const char *digest, int succeeded,
-                                    int mark_relay_status, time_t now)
-{
-  int changed = 0;
-  int refuse_conn = 0;
-  int first_contact = 0;
-  entry_guard_t *entry = NULL;
-  int idx = -1;
-  char buf[HEX_DIGEST_LEN+1];
-
-  if (! entry_guards)
-    return 0;
-
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) {
-    tor_assert(e);
-    if (tor_memeq(e->identity, digest, DIGEST_LEN)) {
-      entry = e;
-      idx = e_sl_idx;
-      break;
-    }
-  } SMARTLIST_FOREACH_END(e);
-
-  if (!entry)
-    return 0;
-
-  base16_encode(buf, sizeof(buf), entry->identity, DIGEST_LEN);
-
-  if (succeeded) {
-    if (entry->unreachable_since) {
-      log_info(LD_CIRC, "Entry guard '%s' (%s) is now reachable again. Good.",
-               entry->nickname, buf);
-      entry->can_retry = 0;
-      entry->unreachable_since = 0;
-      entry->last_attempted = now;
-      control_event_guard(entry->nickname, entry->identity, "UP");
-      changed = 1;
-    }
-    if (!entry->made_contact) {
-      entry->made_contact = 1;
-      first_contact = changed = 1;
-    }
-  } else { /* ! succeeded */
-    if (!entry->made_contact) {
-      /* We've never connected to this one. */
-      log_info(LD_CIRC,
-               "Connection to never-contacted entry guard '%s' (%s) failed. "
-               "Removing from the list. %d/%d entry guards usable/new.",
-               entry->nickname, buf,
-               num_live_entry_guards(0)-1, smartlist_len(entry_guards)-1);
-      control_event_guard(entry->nickname, entry->identity, "DROPPED");
-      entry_guard_free(entry);
-      smartlist_del_keeporder(entry_guards, idx);
-      log_entry_guards(LOG_INFO);
-      changed = 1;
-    } else if (!entry->unreachable_since) {
-      log_info(LD_CIRC, "Unable to connect to entry guard '%s' (%s). "
-               "Marking as unreachable.", entry->nickname, buf);
-      entry->unreachable_since = entry->last_attempted = now;
-      control_event_guard(entry->nickname, entry->identity, "DOWN");
-      changed = 1;
-      entry->can_retry = 0; /* We gave it an early chance; no good. */
-    } else {
-      char tbuf[ISO_TIME_LEN+1];
-      format_iso_time(tbuf, entry->unreachable_since);
-      log_debug(LD_CIRC, "Failed to connect to unreachable entry guard "
-                "'%s' (%s).  It has been unreachable since %s.",
-                entry->nickname, buf, tbuf);
-      entry->last_attempted = now;
-      entry->can_retry = 0; /* We gave it an early chance; no good. */
-    }
-  }
-
-  /* if the caller asked us to, also update the is_running flags for this
-   * relay */
-  if (mark_relay_status)
-    router_set_status(digest, succeeded);
-
-  if (first_contact) {
-    /* We've just added a new long-term entry guard. Perhaps the network just
-     * came back? We should give our earlier entries another try too,
-     * and close this connection so we don't use it before we've given
-     * the others a shot. */
-    SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) {
-        if (e == entry)
-          break;
-        if (e->made_contact) {
-          const char *msg;
-          const node_t *r = entry_is_live(e,
-                     ENTRY_NEED_CAPACITY | ENTRY_ASSUME_REACHABLE,
-                     &msg);
-          if (r && e->unreachable_since) {
-            refuse_conn = 1;
-            e->can_retry = 1;
-          }
-        }
-    } SMARTLIST_FOREACH_END(e);
-    if (refuse_conn) {
-      log_info(LD_CIRC,
-               "Connected to new entry guard '%s' (%s). Marking earlier "
-               "entry guards up. %d/%d entry guards usable/new.",
-               entry->nickname, buf,
-               num_live_entry_guards(0), smartlist_len(entry_guards));
-      log_entry_guards(LOG_INFO);
-      changed = 1;
-    }
-  }
-
-  if (changed)
-    entry_guards_changed();
-  return refuse_conn ? -1 : 0;
-}
-
-/** When we try to choose an entry guard, should we parse and add
- * config's EntryNodes first? */
-static int should_add_entry_nodes = 0;
-
-/** Called when the value of EntryNodes changes in our configuration. */
-void
-entry_nodes_should_be_added(void)
-{
-  log_info(LD_CIRC, "EntryNodes config option set. Putting configured "
-           "relays at the front of the entry guard list.");
-  should_add_entry_nodes = 1;
-}
-
-/** Update the using_as_guard fields of all the nodes. We do this after we
- * remove entry guards from the list: This is the only function that clears
- * the using_as_guard field. */
-static void
-update_node_guard_status(void)
-{
-  smartlist_t *nodes = nodelist_get_list();
-  SMARTLIST_FOREACH(nodes, node_t *, node, node->using_as_guard = 0);
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) {
-    node_t *node = node_get_mutable_by_id(entry->identity);
-    if (node)
-      node->using_as_guard = 1;
-  } SMARTLIST_FOREACH_END(entry);
-}
-
-/** Adjust the entry guards list so that it only contains entries from
- * EntryNodes, adding new entries from EntryNodes to the list as needed. */
-STATIC void
-entry_guards_set_from_config(const or_options_t *options)
-{
-  smartlist_t *entry_nodes, *worse_entry_nodes, *entry_fps;
-  smartlist_t *old_entry_guards_on_list, *old_entry_guards_not_on_list;
-  const int numentryguards = decide_num_guards(options, 0);
-  tor_assert(entry_guards);
-
-  should_add_entry_nodes = 0;
-
-  if (!options->EntryNodes) {
-    /* It's possible that a controller set EntryNodes, thus making
-     * should_add_entry_nodes set, then cleared it again, all before the
-     * call to choose_random_entry() that triggered us. If so, just return.
-     */
-    return;
-  }
-
-  {
-    char *string = routerset_to_string(options->EntryNodes);
-    log_info(LD_CIRC,"Adding configured EntryNodes '%s'.", string);
-    tor_free(string);
-  }
-
-  entry_nodes = smartlist_new();
-  worse_entry_nodes = smartlist_new();
-  entry_fps = smartlist_new();
-  old_entry_guards_on_list = smartlist_new();
-  old_entry_guards_not_on_list = smartlist_new();
-
-  /* Split entry guards into those on the list and those not. */
-
-  routerset_get_all_nodes(entry_nodes, options->EntryNodes,
-                          options->ExcludeNodes, 0);
-  SMARTLIST_FOREACH(entry_nodes, const node_t *,node,
-                    smartlist_add(entry_fps, (void*)node->identity));
-
-  SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, {
-    if (smartlist_contains_digest(entry_fps, e->identity))
-      smartlist_add(old_entry_guards_on_list, e);
-    else
-      smartlist_add(old_entry_guards_not_on_list, e);
-  });
-
-  /* Remove all currently configured guard nodes, excluded nodes, unreachable
-   * nodes, or non-Guard nodes from entry_nodes. */
-  SMARTLIST_FOREACH_BEGIN(entry_nodes, const node_t *, node) {
-    if (entry_guard_get_by_id_digest(node->identity)) {
-      SMARTLIST_DEL_CURRENT(entry_nodes, node);
-      continue;
-    } else if (routerset_contains_node(options->ExcludeNodes, node)) {
-      SMARTLIST_DEL_CURRENT(entry_nodes, node);
-      continue;
-    } else if (!fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION,
-                                             0)) {
-      SMARTLIST_DEL_CURRENT(entry_nodes, node);
-      continue;
-    } else if (! node->is_possible_guard) {
-      smartlist_add(worse_entry_nodes, (node_t*)node);
-      SMARTLIST_DEL_CURRENT(entry_nodes, node);
-    }
-  } SMARTLIST_FOREACH_END(node);
-
-  /* Now build the new entry_guards list. */
-  smartlist_clear(entry_guards);
-  /* First, the previously configured guards that are in EntryNodes. */
-  smartlist_add_all(entry_guards, old_entry_guards_on_list);
-  /* Next, scramble the rest of EntryNodes, putting the guards first. */
-  smartlist_shuffle(entry_nodes);
-  smartlist_shuffle(worse_entry_nodes);
-  smartlist_add_all(entry_nodes, worse_entry_nodes);
-
-  /* Next, the rest of EntryNodes */
-  SMARTLIST_FOREACH_BEGIN(entry_nodes, const node_t *, node) {
-    add_an_entry_guard(node, 0, 0, 1, 0);
-    if (smartlist_len(entry_guards) > numentryguards * 10)
-      break;
-  } SMARTLIST_FOREACH_END(node);
-  log_notice(LD_GENERAL, "%d entries in guards", smartlist_len(entry_guards));
-  /* Finally, free the remaining previously configured guards that are not in
-   * EntryNodes. */
-  SMARTLIST_FOREACH(old_entry_guards_not_on_list, entry_guard_t *, e,
-                    entry_guard_free(e));
-
-  update_node_guard_status();
-
-  smartlist_free(entry_nodes);
-  smartlist_free(worse_entry_nodes);
-  smartlist_free(entry_fps);
-  smartlist_free(old_entry_guards_on_list);
-  smartlist_free(old_entry_guards_not_on_list);
-  entry_guards_changed();
-}
-
-/** Return 0 if we're fine adding arbitrary routers out of the
- * directory to our entry guard list, or return 1 if we have a
- * list already and we must stick to it.
- */
-int
-entry_list_is_constrained(const or_options_t *options)
-{
-  if (options->EntryNodes)
-    return 1;
-  if (options->UseBridges)
-    return 1;
-  return 0;
-}
-
-/** Pick a live (up and listed) entry guard from entry_guards. If
- * <b>state</b> is non-NULL, this is for a specific circuit --
- * make sure not to pick this circuit's exit or any node in the
- * exit's family. If <b>state</b> is NULL, we're looking for a random
- * guard (likely a bridge).  If <b>dirinfo</b> is not NO_DIRINFO (zero),
- * then only select from nodes that know how to answer directory questions
- * of that type. */
-const node_t *
-choose_random_entry(cpath_build_state_t *state)
-{
-  return choose_random_entry_impl(state, 0, NO_DIRINFO, NULL);
-}
-
-/** Pick a live (up and listed) directory guard from entry_guards for
- * downloading information of type <b>type</b>. */
-const node_t *
-choose_random_dirguard(dirinfo_type_t type)
-{
-  return choose_random_entry_impl(NULL, 1, type, NULL);
-}
-
-/** Filter <b>all_entry_guards</b> for usable entry guards and put them
- * in <b>live_entry_guards</b>. We filter based on whether the node is
- * currently alive, and on whether it satisfies the restrictions
- * imposed by the other arguments of this function.
- *
- * We don't place more guards than NumEntryGuards in <b>live_entry_guards</b>.
- *
- * If <b>chosen_exit</b> is set, it contains the exit node of this
- * circuit. Make sure to not use it or its family as an entry guard.
- *
- * If <b>need_uptime</b> is set, we are looking for a stable entry guard.
- * if <b>need_capacity</b> is set, we are looking for a fast entry guard.
- *
- * The rest of the arguments are the same as in choose_random_entry_impl().
- *
- * Return 1 if we should choose a guard right away. Return 0 if we
- * should try to add more nodes to our list before deciding on a
- * guard.
- */
-STATIC int
-populate_live_entry_guards(smartlist_t *live_entry_guards,
-                           const smartlist_t *all_entry_guards,
-                           const node_t *chosen_exit,
-                           dirinfo_type_t dirinfo_type,
-                           int for_directory,
-                           int need_uptime, int need_capacity)
-{
-  const or_options_t *options = get_options();
-  const node_t *node = NULL;
-  const int num_needed = decide_num_guards(options, for_directory);
-  smartlist_t *exit_family = smartlist_new();
-  int retval = 0;
-  entry_is_live_flags_t entry_flags = 0;
-
-  (void) dirinfo_type;
-
-  { /* Set the flags we want our entry node to have */
-    if (need_uptime) {
-      entry_flags |= ENTRY_NEED_UPTIME;
-    }
-    if (need_capacity) {
-      entry_flags |= ENTRY_NEED_CAPACITY;
-    }
-    if (!for_directory) {
-      entry_flags |= ENTRY_NEED_DESCRIPTOR;
-    }
-  }
-
-  tor_assert(all_entry_guards);
-
-  if (chosen_exit) {
-    nodelist_add_node_and_family(exit_family, chosen_exit);
-  }
-
-  SMARTLIST_FOREACH_BEGIN(all_entry_guards, const entry_guard_t *, entry) {
-      const char *msg;
-      node = entry_is_live(entry, entry_flags, &msg);
-      if (!node)
-        continue; /* down, no point */
-      if (for_directory) {
-        if (!entry->is_dir_cache)
-          continue; /* We need a directory and didn't get one. */
-      }
-      if (node == chosen_exit)
-        continue; /* don't pick the same node for entry and exit */
-      if (smartlist_contains(exit_family, node))
-        continue; /* avoid relays that are family members of our exit */
-      smartlist_add(live_entry_guards, (void*)node);
-      if (!entry->made_contact) {
-        /* Always start with the first not-yet-contacted entry
-         * guard. Otherwise we might add several new ones, pick
-         * the second new one, and now we've expanded our entry
-         * guard list without needing to. */
-        retval = 1;
-        goto done;
-      }
-      if (smartlist_len(live_entry_guards) >= num_needed) {
-        retval = 1;
-        goto done; /* We picked enough entry guards. Done! */
-      }
-  } SMARTLIST_FOREACH_END(entry);
-
- done:
-  smartlist_free(exit_family);
-
-  return retval;
-}
-
-/** Pick a node to be used as the entry guard of a circuit.
- *
- * If <b>state</b> is set, it contains the information we know about
- * the upcoming circuit.
- *
- * If <b>for_directory</b> is set, we are looking for a directory guard.
- *
- * <b>dirinfo_type</b> contains the kind of directory information we
- * are looking for in our node, or NO_DIRINFO (zero) if we are not
- * looking for any particular directory information (when set to
- * NO_DIRINFO, the <b>dirinfo_type</b> filter is ignored).
- *
- * If <b>n_options_out</b> is set, we set it to the number of
- * candidate guard nodes we had before picking a specific guard node.
- *
- * On success, return the node that should be used as the entry guard
- * of the circuit.  Return NULL if no such node could be found.
- *
- * Helper for choose_random{entry,dirguard}.
-*/
-static const node_t *
-choose_random_entry_impl(cpath_build_state_t *state, int for_directory,
-                         dirinfo_type_t dirinfo_type, int *n_options_out)
-{
-  const or_options_t *options = get_options();
-  smartlist_t *live_entry_guards = smartlist_new();
-  const node_t *chosen_exit =
-    state?build_state_get_exit_node(state) : NULL;
-  const node_t *node = NULL;
-  int need_uptime = state ? state->need_uptime : 0;
-  int need_capacity = state ? state->need_capacity : 0;
-  int preferred_min = 0;
-  const int num_needed = decide_num_guards(options, for_directory);
-  int retval = 0;
-
-  if (n_options_out)
-    *n_options_out = 0;
-
-  if (!entry_guards)
-    entry_guards = smartlist_new();
-
-  if (should_add_entry_nodes)
-    entry_guards_set_from_config(options);
-
-  if (!entry_list_is_constrained(options) &&
-      smartlist_len(entry_guards) < num_needed)
-    pick_entry_guards(options, for_directory);
-
- retry:
-  smartlist_clear(live_entry_guards);
-
-  /* Populate the list of live entry guards so that we pick one of
-     them. */
-  retval = populate_live_entry_guards(live_entry_guards,
-                                      entry_guards,
-                                      chosen_exit,
-                                      dirinfo_type,
-                                      for_directory,
-                                      need_uptime, need_capacity);
-
-  if (retval == 1) { /* We should choose a guard right now. */
-    goto choose_and_finish;
-  }
-
-  if (entry_list_is_constrained(options)) {
-    /* If we prefer the entry nodes we've got, and we have at least
-     * one choice, that's great. Use it. */
-    preferred_min = 1;
-  } else {
-    /* Try to have at least 2 choices available. This way we don't
-     * get stuck with a single live-but-crummy entry and just keep
-     * using it.
-     * (We might get 2 live-but-crummy entry guards, but so be it.) */
-    preferred_min = 2;
-  }
-
-  if (smartlist_len(live_entry_guards) < preferred_min) {
-    if (!entry_list_is_constrained(options)) {
-      /* still no? try adding a new entry then */
-      /* XXX if guard doesn't imply fast and stable, then we need
-       * to tell add_an_entry_guard below what we want, or it might
-       * be a long time til we get it. -RD */
-      node = add_an_entry_guard(NULL, 0, 0, 1, for_directory);
-      if (node) {
-        entry_guards_changed();
-        /* XXX we start over here in case the new node we added shares
-         * a family with our exit node. There's a chance that we'll just
-         * load up on entry guards here, if the network we're using is
-         * one big family. Perhaps we should teach add_an_entry_guard()
-         * to understand nodes-to-avoid-if-possible? -RD */
-        goto retry;
-      }
-    }
-    if (!node && need_uptime) {
-      need_uptime = 0; /* try without that requirement */
-      goto retry;
-    }
-    if (!node && need_capacity) {
-      /* still no? last attempt, try without requiring capacity */
-      need_capacity = 0;
-      goto retry;
-    }
-
-    /* live_entry_guards may be empty below. Oh well, we tried. */
-  }
-
- choose_and_finish:
-  if (entry_list_is_constrained(options)) {
-    /* We need to weight by bandwidth, because our bridges or entryguards
-     * were not already selected proportional to their bandwidth. */
-    node = node_sl_choose_by_bandwidth(live_entry_guards, WEIGHT_FOR_GUARD);
-  } else {
-    /* We choose uniformly at random here, because choose_good_entry_server()
-     * already weights its choices by bandwidth, so we don't want to
-     * *double*-weight our guard selection. */
-    node = smartlist_choose(live_entry_guards);
-  }
-  if (n_options_out)
-    *n_options_out = smartlist_len(live_entry_guards);
-  smartlist_free(live_entry_guards);
-  return node;
-}
-
-/** Parse <b>state</b> and learn about the entry guards it describes.
- * If <b>set</b> is true, and there are no errors, replace the global
- * entry_list with what we find.
- * On success, return 0. On failure, alloc into *<b>msg</b> a string
- * describing the error, and return -1.
- */
-int
-entry_guards_parse_state(or_state_t *state, int set, char **msg)
-{
-  entry_guard_t *node = NULL;
-  smartlist_t *new_entry_guards = smartlist_new();
-  config_line_t *line;
-  time_t now = time(NULL);
-  const char *state_version = state->TorVersion;
-  digestmap_t *added_by = digestmap_new();
-
-  *msg = NULL;
-  for (line = state->EntryGuards; line; line = line->next) {
-    if (!strcasecmp(line->key, "EntryGuard")) {
-      smartlist_t *args = smartlist_new();
-      node = tor_malloc_zero(sizeof(entry_guard_t));
-      /* all entry guards on disk have been contacted */
-      node->made_contact = 1;
-      smartlist_add(new_entry_guards, node);
-      smartlist_split_string(args, line->value, " ",
-                             SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-      if (smartlist_len(args)<2) {
-        *msg = tor_strdup("Unable to parse entry nodes: "
-                          "Too few arguments to EntryGuard");
-      } else if (!is_legal_nickname(smartlist_get(args,0))) {
-        *msg = tor_strdup("Unable to parse entry nodes: "
-                          "Bad nickname for EntryGuard");
-      } else {
-        strlcpy(node->nickname, smartlist_get(args,0), MAX_NICKNAME_LEN+1);
-        if (base16_decode(node->identity, DIGEST_LEN, smartlist_get(args,1),
-                          strlen(smartlist_get(args,1))) != DIGEST_LEN) {
-          *msg = tor_strdup("Unable to parse entry nodes: "
-                            "Bad hex digest for EntryGuard");
-        }
-      }
-      if (smartlist_len(args) >= 3) {
-        const char *is_cache = smartlist_get(args, 2);
-        if (!strcasecmp(is_cache, "DirCache")) {
-          node->is_dir_cache = 1;
-        } else if (!strcasecmp(is_cache, "NoDirCache")) {
-          node->is_dir_cache = 0;
-        } else {
-          log_warn(LD_CONFIG, "Bogus third argument to EntryGuard line: %s",
-                   escaped(is_cache));
-        }
-      }
-      SMARTLIST_FOREACH(args, char*, cp, tor_free(cp));
-      smartlist_free(args);
-      if (*msg)
-        break;
-    } else if (!strcasecmp(line->key, "EntryGuardDownSince") ||
-               !strcasecmp(line->key, "EntryGuardUnlistedSince")) {
-      time_t when;
-      time_t last_try = 0;
-      if (!node) {
-        *msg = tor_strdup("Unable to parse entry nodes: "
-               "EntryGuardDownSince/UnlistedSince without EntryGuard");
-        break;
-      }
-      if (parse_iso_time_(line->value, &when, 0)<0) {
-        *msg = tor_strdup("Unable to parse entry nodes: "
-                          "Bad time in EntryGuardDownSince/UnlistedSince");
-        break;
-      }
-      if (when > now) {
-        /* It's a bad idea to believe info in the future: you can wind
-         * up with timeouts that aren't allowed to happen for years. */
-        continue;
-      }
-      if (strlen(line->value) >= ISO_TIME_LEN+ISO_TIME_LEN+1) {
-        /* ignore failure */
-        (void) parse_iso_time(line->value+ISO_TIME_LEN+1, &last_try);
-      }
-      if (!strcasecmp(line->key, "EntryGuardDownSince")) {
-        node->unreachable_since = when;
-        node->last_attempted = last_try;
-      } else {
-        node->bad_since = when;
-      }
-    } else if (!strcasecmp(line->key, "EntryGuardAddedBy")) {
-      char d[DIGEST_LEN];
-      /* format is digest version date */
-      if (strlen(line->value) < HEX_DIGEST_LEN+1+1+1+ISO_TIME_LEN) {
-        log_warn(LD_BUG, "EntryGuardAddedBy line is not long enough.");
-        continue;
-      }
-      if (base16_decode(d, sizeof(d),
-                        line->value, HEX_DIGEST_LEN) != sizeof(d) ||
-                        line->value[HEX_DIGEST_LEN] != ' ') {
-        log_warn(LD_BUG, "EntryGuardAddedBy line %s does not begin with "
-                 "hex digest", escaped(line->value));
-        continue;
-      }
-      digestmap_set(added_by, d, tor_strdup(line->value+HEX_DIGEST_LEN+1));
-    } else if (!strcasecmp(line->key, "EntryGuardPathUseBias")) {
-      const or_options_t *options = get_options();
-      double use_cnt, success_cnt;
-
-      if (!node) {
-        *msg = tor_strdup("Unable to parse entry nodes: "
-               "EntryGuardPathUseBias without EntryGuard");
-        break;
-      }
-
-      if (tor_sscanf(line->value, "%lf %lf",
-                     &use_cnt, &success_cnt) != 2) {
-        log_info(LD_GENERAL, "Malformed path use bias line for node %s",
-                 node->nickname);
-        continue;
-      }
-
-      if (use_cnt < success_cnt) {
-        int severity = LOG_INFO;
-        /* If this state file was written by a Tor that would have
-         * already fixed it, then the overcounting bug is still there.. */
-        if (tor_version_as_new_as(state_version, "0.2.4.13-alpha")) {
-          severity = LOG_NOTICE;
-        }
-        log_fn(severity, LD_BUG,
-                   "State file contains unexpectedly high usage success "
-                   "counts %lf/%lf for Guard %s ($%s)",
-                   success_cnt, use_cnt,
-                   node->nickname, hex_str(node->identity, DIGEST_LEN));
-        success_cnt = use_cnt;
-      }
-
-      node->use_attempts = use_cnt;
-      node->use_successes = success_cnt;
-
-      log_info(LD_GENERAL, "Read %f/%f path use bias for node %s",
-               node->use_successes, node->use_attempts, node->nickname);
-
-      /* Note: We rely on the < comparison here to allow us to set a 0
-       * rate and disable the feature entirely. If refactoring, don't
-       * change to <= */
-      if (pathbias_get_use_success_count(node)/node->use_attempts
-            < pathbias_get_extreme_use_rate(options) &&
-          pathbias_get_dropguards(options)) {
-        node->path_bias_disabled = 1;
-        log_info(LD_GENERAL,
-                 "Path use bias is too high (%f/%f); disabling node %s",
-                 node->circ_successes, node->circ_attempts, node->nickname);
-      }
-    } else if (!strcasecmp(line->key, "EntryGuardPathBias")) {
-      const or_options_t *options = get_options();
-      double hop_cnt, success_cnt, timeouts, collapsed, successful_closed,
-               unusable;
-
-      if (!node) {
-        *msg = tor_strdup("Unable to parse entry nodes: "
-               "EntryGuardPathBias without EntryGuard");
-        break;
-      }
-
-      /* First try 3 params, then 2. */
-      /* In the long run: circuit_success ~= successful_circuit_close +
-       *                                     collapsed_circuits +
-       *                                     unusable_circuits */
-      if (tor_sscanf(line->value, "%lf %lf %lf %lf %lf %lf",
-                  &hop_cnt, &success_cnt, &successful_closed,
-                  &collapsed, &unusable, &timeouts) != 6) {
-        int old_success, old_hops;
-        if (tor_sscanf(line->value, "%u %u", &old_success, &old_hops) != 2) {
-          continue;
-        }
-        log_info(LD_GENERAL, "Reading old-style EntryGuardPathBias %s",
-                 escaped(line->value));
-
-        success_cnt = old_success;
-        successful_closed = old_success;
-        hop_cnt = old_hops;
-        timeouts = 0;
-        collapsed = 0;
-        unusable = 0;
-      }
-
-      if (hop_cnt < success_cnt) {
-        int severity = LOG_INFO;
-        /* If this state file was written by a Tor that would have
-         * already fixed it, then the overcounting bug is still there.. */
-        if (tor_version_as_new_as(state_version, "0.2.4.13-alpha")) {
-          severity = LOG_NOTICE;
-        }
-        log_fn(severity, LD_BUG,
-                "State file contains unexpectedly high success counts "
-                "%lf/%lf for Guard %s ($%s)",
-                success_cnt, hop_cnt,
-                node->nickname, hex_str(node->identity, DIGEST_LEN));
-        success_cnt = hop_cnt;
-      }
-
-      node->circ_attempts = hop_cnt;
-      node->circ_successes = success_cnt;
-
-      node->successful_circuits_closed = successful_closed;
-      node->timeouts = timeouts;
-      node->collapsed_circuits = collapsed;
-      node->unusable_circuits = unusable;
-
-      log_info(LD_GENERAL, "Read %f/%f path bias for node %s",
-               node->circ_successes, node->circ_attempts, node->nickname);
-      /* Note: We rely on the < comparison here to allow us to set a 0
-       * rate and disable the feature entirely. If refactoring, don't
-       * change to <= */
-      if (pathbias_get_close_success_count(node)/node->circ_attempts
-            < pathbias_get_extreme_rate(options) &&
-          pathbias_get_dropguards(options)) {
-        node->path_bias_disabled = 1;
-        log_info(LD_GENERAL,
-                 "Path bias is too high (%f/%f); disabling node %s",
-                 node->circ_successes, node->circ_attempts, node->nickname);
-      }
-
-    } else {
-      log_warn(LD_BUG, "Unexpected key %s", line->key);
-    }
-  }
-
-  SMARTLIST_FOREACH_BEGIN(new_entry_guards, entry_guard_t *, e) {
-     char *sp;
-     char *val = digestmap_get(added_by, e->identity);
-     if (val && (sp = strchr(val, ' '))) {
-       time_t when;
-       *sp++ = '\0';
-       if (parse_iso_time(sp, &when)<0) {
-         log_warn(LD_BUG, "Can't read time %s in EntryGuardAddedBy", sp);
-       } else {
-         e->chosen_by_version = tor_strdup(val);
-         e->chosen_on_date = when;
-       }
-     } else {
-       if (state_version) {
-         e->chosen_on_date = crypto_rand_time_range(now - 3600*24*30, now);
-         e->chosen_by_version = tor_strdup(state_version);
-       }
-     }
-     if (e->path_bias_disabled && !e->bad_since)
-       e->bad_since = time(NULL);
-    }
-  SMARTLIST_FOREACH_END(e);
-
-  if (*msg || !set) {
-    SMARTLIST_FOREACH(new_entry_guards, entry_guard_t *, e,
-                      entry_guard_free(e));
-    smartlist_free(new_entry_guards);
-  } else { /* !err && set */
-    if (entry_guards) {
-      SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e,
-                        entry_guard_free(e));
-      smartlist_free(entry_guards);
-    }
-    entry_guards = new_entry_guards;
-    entry_guards_dirty = 0;
-    /* XXX hand new_entry_guards to this func, and move it up a
-     * few lines, so we don't have to re-dirty it */
-    if (remove_obsolete_entry_guards(now))
-      entry_guards_dirty = 1;
-
-    update_node_guard_status();
-  }
-  digestmap_free(added_by, tor_free_);
-  return *msg ? -1 : 0;
-}
-
-/** How long will we let a change in our guard nodes stay un-saved
- * when we are trying to avoid disk writes? */
-#define SLOW_GUARD_STATE_FLUSH_TIME 600
-/** How long will we let a change in our guard nodes stay un-saved
- * when we are not trying to avoid disk writes? */
-#define FAST_GUARD_STATE_FLUSH_TIME 30
-
-/** Our list of entry guards has changed, or some element of one
- * of our entry guards has changed. Write the changes to disk within
- * the next few minutes.
- */
-void
-entry_guards_changed(void)
-{
-  time_t when;
-  entry_guards_dirty = 1;
-
-  if (get_options()->AvoidDiskWrites)
-    when = time(NULL) + SLOW_GUARD_STATE_FLUSH_TIME;
-  else
-    when = time(NULL) + FAST_GUARD_STATE_FLUSH_TIME;
-
-  /* or_state_save() will call entry_guards_update_state(). */
-  or_state_mark_dirty(get_or_state(), when);
-}
-
-/** If the entry guard info has not changed, do nothing and return.
- * Otherwise, free the EntryGuards piece of <b>state</b> and create
- * a new one out of the global entry_guards list, and then mark
- * <b>state</b> dirty so it will get saved to disk.
- */
-void
-entry_guards_update_state(or_state_t *state)
-{
-  config_line_t **next, *line;
-  if (! entry_guards_dirty)
-    return;
-
-  config_free_lines(state->EntryGuards);
-  next = &state->EntryGuards;
-  *next = NULL;
-  if (!entry_guards)
-    entry_guards = smartlist_new();
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) {
-      char dbuf[HEX_DIGEST_LEN+1];
-      if (!e->made_contact)
-        continue; /* don't write this one to disk */
-      *next = line = tor_malloc_zero(sizeof(config_line_t));
-      line->key = tor_strdup("EntryGuard");
-      base16_encode(dbuf, sizeof(dbuf), e->identity, DIGEST_LEN);
-      tor_asprintf(&line->value, "%s %s %sDirCache", e->nickname, dbuf,
-                   e->is_dir_cache ? "" : "No");
-      next = &(line->next);
-      if (e->unreachable_since) {
-        *next = line = tor_malloc_zero(sizeof(config_line_t));
-        line->key = tor_strdup("EntryGuardDownSince");
-        line->value = tor_malloc(ISO_TIME_LEN+1+ISO_TIME_LEN+1);
-        format_iso_time(line->value, e->unreachable_since);
-        if (e->last_attempted) {
-          line->value[ISO_TIME_LEN] = ' ';
-          format_iso_time(line->value+ISO_TIME_LEN+1, e->last_attempted);
-        }
-        next = &(line->next);
-      }
-      if (e->bad_since) {
-        *next = line = tor_malloc_zero(sizeof(config_line_t));
-        line->key = tor_strdup("EntryGuardUnlistedSince");
-        line->value = tor_malloc(ISO_TIME_LEN+1);
-        format_iso_time(line->value, e->bad_since);
-        next = &(line->next);
-      }
-      if (e->chosen_on_date && e->chosen_by_version &&
-          !strchr(e->chosen_by_version, ' ')) {
-        char d[HEX_DIGEST_LEN+1];
-        char t[ISO_TIME_LEN+1];
-        *next = line = tor_malloc_zero(sizeof(config_line_t));
-        line->key = tor_strdup("EntryGuardAddedBy");
-        base16_encode(d, sizeof(d), e->identity, DIGEST_LEN);
-        format_iso_time(t, e->chosen_on_date);
-        tor_asprintf(&line->value, "%s %s %s",
-                     d, e->chosen_by_version, t);
-        next = &(line->next);
-      }
-      if (e->circ_attempts > 0) {
-        *next = line = tor_malloc_zero(sizeof(config_line_t));
-        line->key = tor_strdup("EntryGuardPathBias");
-        /* In the long run: circuit_success ~= successful_circuit_close +
-         *                                     collapsed_circuits +
-         *                                     unusable_circuits */
-        tor_asprintf(&line->value, "%f %f %f %f %f %f",
-                     e->circ_attempts, e->circ_successes,
-                     pathbias_get_close_success_count(e),
-                     e->collapsed_circuits,
-                     e->unusable_circuits, e->timeouts);
-        next = &(line->next);
-      }
-      if (e->use_attempts > 0) {
-        *next = line = tor_malloc_zero(sizeof(config_line_t));
-        line->key = tor_strdup("EntryGuardPathUseBias");
-
-        tor_asprintf(&line->value, "%f %f",
-                     e->use_attempts,
-                     pathbias_get_use_success_count(e));
-        next = &(line->next);
-      }
-
-  } SMARTLIST_FOREACH_END(e);
-  if (!get_options()->AvoidDiskWrites)
-    or_state_mark_dirty(get_or_state(), 0);
-  entry_guards_dirty = 0;
-}
-
-/** If <b>question</b> is the string "entry-guards", then dump
- * to *<b>answer</b> a newly allocated string describing all of
- * the nodes in the global entry_guards list. See control-spec.txt
- * for details.
- * For backward compatibility, we also handle the string "helper-nodes".
- * */
-int
-getinfo_helper_entry_guards(control_connection_t *conn,
-                            const char *question, char **answer,
-                            const char **errmsg)
-{
-  (void) conn;
-  (void) errmsg;
-
-  if (!strcmp(question,"entry-guards") ||
-      !strcmp(question,"helper-nodes")) {
-    smartlist_t *sl = smartlist_new();
-    char tbuf[ISO_TIME_LEN+1];
-    char nbuf[MAX_VERBOSE_NICKNAME_LEN+1];
-    if (!entry_guards)
-      entry_guards = smartlist_new();
-    SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) {
-        const char *status = NULL;
-        time_t when = 0;
-        const node_t *node;
-
-        if (!e->made_contact) {
-          status = "never-connected";
-        } else if (e->bad_since) {
-          when = e->bad_since;
-          status = "unusable";
-        } else if (e->unreachable_since) {
-          when = e->unreachable_since;
-          status = "down";
-        } else {
-          status = "up";
-        }
-
-        node = node_get_by_id(e->identity);
-        if (node) {
-          node_get_verbose_nickname(node, nbuf);
-        } else {
-          nbuf[0] = '$';
-          base16_encode(nbuf+1, sizeof(nbuf)-1, e->identity, DIGEST_LEN);
-          /* e->nickname field is not very reliable if we don't know about
-           * this router any longer; don't include it. */
-        }
-
-        if (when) {
-          format_iso_time(tbuf, when);
-          smartlist_add_asprintf(sl, "%s %s %s\n", nbuf, status, tbuf);
-        } else {
-          smartlist_add_asprintf(sl, "%s %s\n", nbuf, status);
-        }
-    } SMARTLIST_FOREACH_END(e);
-    *answer = smartlist_join_strings(sl, "", 0, NULL);
-    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
-    smartlist_free(sl);
-  }
-  return 0;
-}
-
-/** Return 0 if we should apply guardfraction information found in the
- *  consensus. A specific consensus can be specified with the
- *  <b>ns</b> argument, if NULL the most recent one will be picked.*/
-int
-should_apply_guardfraction(const networkstatus_t *ns)
-{
-  /* We need to check the corresponding torrc option and the consensus
-   * parameter if we need to. */
-  const or_options_t *options = get_options();
-
-  /* If UseGuardFraction is 'auto' then check the same-named consensus
-   * parameter. If the consensus parameter is not present, default to
-   * "off". */
-  if (options->UseGuardFraction == -1) {
-    return networkstatus_get_param(ns, "UseGuardFraction",
-                                   0, /* default to "off" */
-                                   0, 1);
-  }
-
-  return options->UseGuardFraction;
-}
-
-/* Given the original bandwidth of a guard and its guardfraction,
- * calculate how much bandwidth the guard should have as a guard and
- * as a non-guard.
- *
- * Quoting from proposal236:
- *
- *   Let Wpf denote the weight from the 'bandwidth-weights' line a
- *   client would apply to N for position p if it had the guard
- *   flag, Wpn the weight if it did not have the guard flag, and B the
- *   measured bandwidth of N in the consensus.  Then instead of choosing
- *   N for position p proportionally to Wpf*B or Wpn*B, clients should
- *   choose N proportionally to F*Wpf*B + (1-F)*Wpn*B.
- *
- * This function fills the <b>guardfraction_bw</b> structure. It sets
- * <b>guard_bw</b> to F*B and <b>non_guard_bw</b> to (1-F)*B.
- */
-void
-guard_get_guardfraction_bandwidth(guardfraction_bandwidth_t *guardfraction_bw,
-                                  int orig_bandwidth,
-                                  uint32_t guardfraction_percentage)
-{
-  double guardfraction_fraction;
-
-  /* Turn the percentage into a fraction. */
-  tor_assert(guardfraction_percentage <= 100);
-  guardfraction_fraction = guardfraction_percentage / 100.0;
-
-  long guard_bw = tor_lround(guardfraction_fraction * orig_bandwidth);
-  tor_assert(guard_bw <= INT_MAX);
-
-  guardfraction_bw->guard_bw = (int) guard_bw;
-
-  guardfraction_bw->non_guard_bw = orig_bandwidth - (int) guard_bw;
-}
-
-/** A list of configured bridges. Whenever we actually get a descriptor
- * for one, we add it as an entry guard.  Note that the order of bridges
- * in this list does not necessarily correspond to the order of bridges
- * in the torrc. */
-static smartlist_t *bridge_list = NULL;
-
-/** Mark every entry of the bridge list to be removed on our next call to
- * sweep_bridge_list unless it has first been un-marked. */
-void
-mark_bridge_list(void)
-{
-  if (!bridge_list)
-    bridge_list = smartlist_new();
-  SMARTLIST_FOREACH(bridge_list, bridge_info_t *, b,
-                    b->marked_for_removal = 1);
-}
-
-/** Remove every entry of the bridge list that was marked with
- * mark_bridge_list if it has not subsequently been un-marked. */
-void
-sweep_bridge_list(void)
-{
-  if (!bridge_list)
-    bridge_list = smartlist_new();
-  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) {
-    if (b->marked_for_removal) {
-      SMARTLIST_DEL_CURRENT(bridge_list, b);
-      bridge_free(b);
-    }
-  } SMARTLIST_FOREACH_END(b);
-}
-
-/** Initialize the bridge list to empty, creating it if needed. */
-static void
-clear_bridge_list(void)
-{
-  if (!bridge_list)
-    bridge_list = smartlist_new();
-  SMARTLIST_FOREACH(bridge_list, bridge_info_t *, b, bridge_free(b));
-  smartlist_clear(bridge_list);
-}
-
-/** Free the bridge <b>bridge</b>. */
-static void
-bridge_free(bridge_info_t *bridge)
-{
-  if (!bridge)
-    return;
-
-  tor_free(bridge->transport_name);
-  if (bridge->socks_args) {
-    SMARTLIST_FOREACH(bridge->socks_args, char*, s, tor_free(s));
-    smartlist_free(bridge->socks_args);
-  }
-
-  tor_free(bridge);
-}
-
-/** If we have a bridge configured whose digest matches <b>digest</b>, or a
- * bridge with no known digest whose address matches any of the
- * tor_addr_port_t's in <b>orports</b>, return that bridge.  Else return
- * NULL. */
-static bridge_info_t *
-get_configured_bridge_by_orports_digest(const char *digest,
-                                        const smartlist_t *orports)
-{
-  if (!bridge_list)
-    return NULL;
-  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge)
-    {
-      if (tor_digest_is_zero(bridge->identity)) {
-        SMARTLIST_FOREACH_BEGIN(orports, tor_addr_port_t *, ap)
-          {
-            if (tor_addr_compare(&bridge->addr, &ap->addr, CMP_EXACT) == 0 &&
-                bridge->port == ap->port)
-              return bridge;
-          }
-        SMARTLIST_FOREACH_END(ap);
-      }
-      if (digest && tor_memeq(bridge->identity, digest, DIGEST_LEN))
-        return bridge;
-    }
-  SMARTLIST_FOREACH_END(bridge);
-  return NULL;
-}
-
-/** If we have a bridge configured whose digest matches <b>digest</b>, or a
- * bridge with no known digest whose address matches <b>addr</b>:<b>port</b>,
- * return that bridge.  Else return NULL. If <b>digest</b> is NULL, check for
- * address/port matches only. */
-static bridge_info_t *
-get_configured_bridge_by_addr_port_digest(const tor_addr_t *addr,
-                                          uint16_t port,
-                                          const char *digest)
-{
-  if (!bridge_list)
-    return NULL;
-  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge)
-    {
-      if ((tor_digest_is_zero(bridge->identity) || digest == NULL) &&
-          !tor_addr_compare(&bridge->addr, addr, CMP_EXACT) &&
-          bridge->port == port)
-        return bridge;
-      if (digest && tor_memeq(bridge->identity, digest, DIGEST_LEN))
-        return bridge;
-    }
-  SMARTLIST_FOREACH_END(bridge);
-  return NULL;
-}
-
-/** If we have a bridge configured whose digest matches <b>digest</b>, or a
- * bridge with no known digest whose address matches <b>addr</b>:<b>port</b>,
- * return 1.  Else return 0. If <b>digest</b> is NULL, check for
- * address/port matches only. */
-int
-addr_is_a_configured_bridge(const tor_addr_t *addr,
-                                uint16_t port,
-                                const char *digest)
-{
-  tor_assert(addr);
-  return get_configured_bridge_by_addr_port_digest(addr, port, digest) ? 1 : 0;
-}
-
-/** If we have a bridge configured whose digest matches
- * <b>ei->identity_digest</b>, or a bridge with no known digest whose address
- * matches <b>ei->addr</b>:<b>ei->port</b>, return 1.  Else return 0.
- * If <b>ei->onion_key</b> is NULL, check for address/port matches only. */
-int
-extend_info_is_a_configured_bridge(const extend_info_t *ei)
-{
-  const char *digest = ei->onion_key ? ei->identity_digest : NULL;
-  return addr_is_a_configured_bridge(&ei->addr, ei->port, digest);
-}
-
-/** Wrapper around get_configured_bridge_by_addr_port_digest() to look
- * it up via router descriptor <b>ri</b>. */
-static bridge_info_t *
-get_configured_bridge_by_routerinfo(const routerinfo_t *ri)
-{
-  bridge_info_t *bi = NULL;
-  smartlist_t *orports = router_get_all_orports(ri);
-  bi = get_configured_bridge_by_orports_digest(ri->cache_info.identity_digest,
-                                               orports);
-  SMARTLIST_FOREACH(orports, tor_addr_port_t *, p, tor_free(p));
-  smartlist_free(orports);
-  return bi;
-}
-
-/** Return 1 if <b>ri</b> is one of our known bridges, else 0. */
-int
-routerinfo_is_a_configured_bridge(const routerinfo_t *ri)
-{
-  return get_configured_bridge_by_routerinfo(ri) ? 1 : 0;
-}
-
-/** Return 1 if <b>node</b> is one of our configured bridges, else 0. */
-int
-node_is_a_configured_bridge(const node_t *node)
-{
-  int retval = 0;
-  smartlist_t *orports = node_get_all_orports(node);
-  retval = get_configured_bridge_by_orports_digest(node->identity,
-                                                   orports) != NULL;
-  SMARTLIST_FOREACH(orports, tor_addr_port_t *, p, tor_free(p));
-  smartlist_free(orports);
-  return retval;
-}
-
-/** We made a connection to a router at <b>addr</b>:<b>port</b>
- * without knowing its digest. Its digest turned out to be <b>digest</b>.
- * If it was a bridge, and we still don't know its digest, record it.
- */
-void
-learned_router_identity(const tor_addr_t *addr, uint16_t port,
-                        const char *digest)
-{
-  bridge_info_t *bridge =
-    get_configured_bridge_by_addr_port_digest(addr, port, digest);
-  if (bridge && tor_digest_is_zero(bridge->identity)) {
-    char *transport_info = NULL;
-    const char *transport_name =
-      find_transport_name_by_bridge_addrport(addr, port);
-    if (transport_name)
-      tor_asprintf(&transport_info, " (with transport '%s')", transport_name);
-
-    memcpy(bridge->identity, digest, DIGEST_LEN);
-    log_notice(LD_DIR, "Learned fingerprint %s for bridge %s%s.",
-               hex_str(digest, DIGEST_LEN), fmt_addrport(addr, port),
-               transport_info ? transport_info : "");
-    tor_free(transport_info);
-  }
-}
-
-/** Return true if <b>bridge</b> has the same identity digest as
- *  <b>digest</b>. If <b>digest</b> is NULL, it matches
- *  bridges with unspecified identity digests. */
-static int
-bridge_has_digest(const bridge_info_t *bridge, const char *digest)
-{
-  if (digest)
-    return tor_memeq(digest, bridge->identity, DIGEST_LEN);
-  else
-    return tor_digest_is_zero(bridge->identity);
-}
-
-/** We are about to add a new bridge at <b>addr</b>:<b>port</b>, with optional
- * <b>digest</b> and <b>transport_name</b>. Mark for removal any previously
- * existing bridge with the same address and port, and warn the user as
- * appropriate.
- */
-static void
-bridge_resolve_conflicts(const tor_addr_t *addr, uint16_t port,
-                         const char *digest, const char *transport_name)
-{
-  /* Iterate the already-registered bridge list:
-
-     If you find a bridge with the same adress and port, mark it for
-     removal. It doesn't make sense to have two active bridges with
-     the same IP:PORT. If the bridge in question has a different
-     digest or transport than <b>digest</b>/<b>transport_name</b>,
-     it's probably a misconfiguration and we should warn the user.
-  */
-  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge) {
-    if (bridge->marked_for_removal)
-      continue;
-
-    if (tor_addr_eq(&bridge->addr, addr) && (bridge->port == port)) {
-
-      bridge->marked_for_removal = 1;
-
-      if (!bridge_has_digest(bridge, digest) ||
-          strcmp_opt(bridge->transport_name, transport_name)) {
-        /* warn the user */
-        char *bridge_description_new, *bridge_description_old;
-        tor_asprintf(&bridge_description_new, "%s:%s:%s",
-                     fmt_addrport(addr, port),
-                     digest ? hex_str(digest, DIGEST_LEN) : "",
-                     transport_name ? transport_name : "");
-        tor_asprintf(&bridge_description_old, "%s:%s:%s",
-                     fmt_addrport(&bridge->addr, bridge->port),
-                     tor_digest_is_zero(bridge->identity) ?
-                     "" : hex_str(bridge->identity,DIGEST_LEN),
-                     bridge->transport_name ? bridge->transport_name : "");
-
-        log_warn(LD_GENERAL,"Tried to add bridge '%s', but we found a conflict"
-                 " with the already registered bridge '%s'. We will discard"
-                 " the old bridge and keep '%s'. If this is not what you"
-                 " wanted, please change your configuration file accordingly.",
-                 bridge_description_new, bridge_description_old,
-                 bridge_description_new);
-
-        tor_free(bridge_description_new);
-        tor_free(bridge_description_old);
-      }
-    }
-  } SMARTLIST_FOREACH_END(bridge);
-}
-
-/** Return True if we have a bridge that uses a transport with name
- *  <b>transport_name</b>. */
-MOCK_IMPL(int,
-transport_is_needed, (const char *transport_name))
-{
-  if (!bridge_list)
-    return 0;
-
-  SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) {
-    if (bridge->transport_name &&
-        !strcmp(bridge->transport_name, transport_name))
-      return 1;
-  } SMARTLIST_FOREACH_END(bridge);
-
-  return 0;
-}
-
-/** Register the bridge information in <b>bridge_line</b> to the
- *  bridge subsystem. Steals reference of <b>bridge_line</b>. */
-void
-bridge_add_from_config(bridge_line_t *bridge_line)
-{
-  bridge_info_t *b;
-
-  { /* Log the bridge we are about to register: */
-    log_debug(LD_GENERAL, "Registering bridge at %s (transport: %s) (%s)",
-              fmt_addrport(&bridge_line->addr, bridge_line->port),
-              bridge_line->transport_name ?
-              bridge_line->transport_name : "no transport",
-              tor_digest_is_zero(bridge_line->digest) ?
-              "no key listed" : hex_str(bridge_line->digest, DIGEST_LEN));
-
-    if (bridge_line->socks_args) { /* print socks arguments */
-      int i = 0;
-
-      tor_assert(smartlist_len(bridge_line->socks_args) > 0);
-
-      log_debug(LD_GENERAL, "Bridge uses %d SOCKS arguments:",
-                smartlist_len(bridge_line->socks_args));
-      SMARTLIST_FOREACH(bridge_line->socks_args, const char *, arg,
-                        log_debug(LD_CONFIG, "%d: %s", ++i, arg));
-    }
-  }
-
-  bridge_resolve_conflicts(&bridge_line->addr,
-                           bridge_line->port,
-                           bridge_line->digest,
-                           bridge_line->transport_name);
-
-  b = tor_malloc_zero(sizeof(bridge_info_t));
-  tor_addr_copy(&b->addr, &bridge_line->addr);
-  b->port = bridge_line->port;
-  memcpy(b->identity, bridge_line->digest, DIGEST_LEN);
-  if (bridge_line->transport_name)
-    b->transport_name = bridge_line->transport_name;
-  b->fetch_status.schedule = DL_SCHED_BRIDGE;
-  b->fetch_status.backoff = DL_SCHED_RANDOM_EXPONENTIAL;
-  b->socks_args = bridge_line->socks_args;
-  if (!bridge_list)
-    bridge_list = smartlist_new();
-
-  tor_free(bridge_line); /* Deallocate bridge_line now. */
-
-  smartlist_add(bridge_list, b);
-}
-
-/** Return true iff <b>routerset</b> contains the bridge <b>bridge</b>. */
-static int
-routerset_contains_bridge(const routerset_t *routerset,
-                          const bridge_info_t *bridge)
-{
-  int result;
-  extend_info_t *extinfo;
-  tor_assert(bridge);
-  if (!routerset)
-    return 0;
-
-  extinfo = extend_info_new(
-         NULL, bridge->identity, NULL, NULL, &bridge->addr, bridge->port);
-  result = routerset_contains_extendinfo(routerset, extinfo);
-  extend_info_free(extinfo);
-  return result;
-}
-
-/** If <b>digest</b> is one of our known bridges, return it. */
-static bridge_info_t *
-find_bridge_by_digest(const char *digest)
-{
-  SMARTLIST_FOREACH(bridge_list, bridge_info_t *, bridge,
-    {
-      if (tor_memeq(bridge->identity, digest, DIGEST_LEN))
-        return bridge;
-    });
-  return NULL;
-}
-
-/** Given the <b>addr</b> and <b>port</b> of a bridge, if that bridge
- *  supports a pluggable transport, return its name. Otherwise, return
- *  NULL. */
-const char *
-find_transport_name_by_bridge_addrport(const tor_addr_t *addr, uint16_t port)
-{
-  if (!bridge_list)
-    return NULL;
-
-  SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) {
-    if (tor_addr_eq(&bridge->addr, addr) &&
-        (bridge->port == port))
-      return bridge->transport_name;
-  } SMARTLIST_FOREACH_END(bridge);
-
-  return NULL;
-}
-
-/** If <b>addr</b> and <b>port</b> match the address and port of a
- * bridge of ours that uses pluggable transports, place its transport
- * in <b>transport</b>.
- *
- * Return 0 on success (found a transport, or found a bridge with no
- * transport, or found no bridge); return -1 if we should be using a
- * transport, but the transport could not be found.
- */
-int
-get_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port,
-                                  const transport_t **transport)
-{
-  *transport = NULL;
-  if (!bridge_list)
-    return 0;
-
-  SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) {
-    if (tor_addr_eq(&bridge->addr, addr) &&
-        (bridge->port == port)) { /* bridge matched */
-      if (bridge->transport_name) { /* it also uses pluggable transports */
-        *transport = transport_get_by_name(bridge->transport_name);
-        if (*transport == NULL) { /* it uses pluggable transports, but
-                                     the transport could not be found! */
-          return -1;
-        }
-        return 0;
-      } else { /* bridge matched, but it doesn't use transports. */
-        break;
-      }
-    }
-  } SMARTLIST_FOREACH_END(bridge);
-
-  *transport = NULL;
-  return 0;
-}
-
-/** Return a smartlist containing all the SOCKS arguments that we
- *  should pass to the SOCKS proxy. */
-const smartlist_t *
-get_socks_args_by_bridge_addrport(const tor_addr_t *addr, uint16_t port)
-{
-  bridge_info_t *bridge = get_configured_bridge_by_addr_port_digest(addr,
-                                                                    port,
-                                                                    NULL);
-  return bridge ? bridge->socks_args : NULL;
-}
-
-/** We need to ask <b>bridge</b> for its server descriptor. */
-static void
-launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge)
-{
-  const or_options_t *options = get_options();
-
-  if (connection_get_by_type_addr_port_purpose(
-      CONN_TYPE_DIR, &bridge->addr, bridge->port,
-      DIR_PURPOSE_FETCH_SERVERDESC))
-    return; /* it's already on the way */
-
-  if (routerset_contains_bridge(options->ExcludeNodes, bridge)) {
-    download_status_mark_impossible(&bridge->fetch_status);
-    log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.",
-             safe_str_client(fmt_and_decorate_addr(&bridge->addr)));
-    return;
-  }
-
-  /* Until we get a descriptor for the bridge, we only know one address for
-   * it. */
-  if (!fascist_firewall_allows_address_addr(&bridge->addr, bridge->port,
-                                            FIREWALL_OR_CONNECTION, 0, 0)) {
-    log_notice(LD_CONFIG, "Tried to fetch a descriptor directly from a "
-               "bridge, but that bridge is not reachable through our "
-               "firewall.");
-    return;
-  }
-
-  directory_initiate_command(&bridge->addr, bridge->port,
-                             NULL, 0, /*no dirport*/
-                             bridge->identity,
-                             DIR_PURPOSE_FETCH_SERVERDESC,
-                             ROUTER_PURPOSE_BRIDGE,
-                             DIRIND_ONEHOP, "authority.z", NULL, 0, 0);
-}
-
-/** Fetching the bridge descriptor from the bridge authority returned a
- * "not found". Fall back to trying a direct fetch. */
-void
-retry_bridge_descriptor_fetch_directly(const char *digest)
-{
-  bridge_info_t *bridge = find_bridge_by_digest(digest);
-  if (!bridge)
-    return; /* not found? oh well. */
-
-  launch_direct_bridge_descriptor_fetch(bridge);
-}
-
-/** For each bridge in our list for which we don't currently have a
- * descriptor, fetch a new copy of its descriptor -- either directly
- * from the bridge or via a bridge authority. */
-void
-fetch_bridge_descriptors(const or_options_t *options, time_t now)
-{
-  int num_bridge_auths = get_n_authorities(BRIDGE_DIRINFO);
-  int ask_bridge_directly;
-  int can_use_bridge_authority;
-
-  if (!bridge_list)
-    return;
-
-  /* If we still have unconfigured managed proxies, don't go and
-     connect to a bridge. */
-  if (pt_proxies_configuration_pending())
-    return;
-
-  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge)
-    {
-      if (!download_status_is_ready(&bridge->fetch_status, now,
-                                    IMPOSSIBLE_TO_DOWNLOAD))
-        continue; /* don't bother, no need to retry yet */
-      if (routerset_contains_bridge(options->ExcludeNodes, bridge)) {
-        download_status_mark_impossible(&bridge->fetch_status);
-        log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.",
-                 safe_str_client(fmt_and_decorate_addr(&bridge->addr)));
-        continue;
-      }
-
-      /* schedule another fetch as if this one will fail, in case it does */
-      download_status_failed(&bridge->fetch_status, 0);
-
-      can_use_bridge_authority = !tor_digest_is_zero(bridge->identity) &&
-                                 num_bridge_auths;
-      ask_bridge_directly = !can_use_bridge_authority ||
-                            !options->UpdateBridgesFromAuthority;
-      log_debug(LD_DIR, "ask_bridge_directly=%d (%d, %d, %d)",
-                ask_bridge_directly, tor_digest_is_zero(bridge->identity),
-                !options->UpdateBridgesFromAuthority, !num_bridge_auths);
-
-      if (ask_bridge_directly &&
-          !fascist_firewall_allows_address_addr(&bridge->addr, bridge->port,
-                                                FIREWALL_OR_CONNECTION, 0,
-                                                0)) {
-        log_notice(LD_DIR, "Bridge at '%s' isn't reachable by our "
-                   "firewall policy. %s.",
-                   fmt_addrport(&bridge->addr, bridge->port),
-                   can_use_bridge_authority ?
-                     "Asking bridge authority instead" : "Skipping");
-        if (can_use_bridge_authority)
-          ask_bridge_directly = 0;
-        else
-          continue;
-      }
-
-      if (ask_bridge_directly) {
-        /* we need to ask the bridge itself for its descriptor. */
-        launch_direct_bridge_descriptor_fetch(bridge);
-      } else {
-        /* We have a digest and we want to ask an authority. We could
-         * combine all the requests into one, but that may give more
-         * hints to the bridge authority than we want to give. */
-        char resource[10 + HEX_DIGEST_LEN];
-        memcpy(resource, "fp/", 3);
-        base16_encode(resource+3, HEX_DIGEST_LEN+1,
-                      bridge->identity, DIGEST_LEN);
-        memcpy(resource+3+HEX_DIGEST_LEN, ".z", 3);
-        log_info(LD_DIR, "Fetching bridge info '%s' from bridge authority.",
-                 resource);
-        directory_get_from_dirserver(DIR_PURPOSE_FETCH_SERVERDESC,
-                ROUTER_PURPOSE_BRIDGE, resource, 0, DL_WANT_AUTHORITY);
-      }
-    }
-  SMARTLIST_FOREACH_END(bridge);
-}
-
-/** If our <b>bridge</b> is configured to be a different address than
- * the bridge gives in <b>node</b>, rewrite the routerinfo
- * we received to use the address we meant to use. Now we handle
- * multihomed bridges better.
- */
-static void
-rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node)
-{
-  /* XXXX move this function. */
-  /* XXXX overridden addresses should really live in the node_t, so that the
-   *   routerinfo_t and the microdesc_t can be immutable.  But we can only
-   *   do that safely if we know that no function that connects to an OR
-   *   does so through an address from any source other than node_get_addr().
-   */
-  tor_addr_t addr;
-  const or_options_t *options = get_options();
-
-  if (node->ri) {
-    routerinfo_t *ri = node->ri;
-    tor_addr_from_ipv4h(&addr, ri->addr);
-
-    if ((!tor_addr_compare(&bridge->addr, &addr, CMP_EXACT) &&
-         bridge->port == ri->or_port) ||
-        (!tor_addr_compare(&bridge->addr, &ri->ipv6_addr, CMP_EXACT) &&
-         bridge->port == ri->ipv6_orport)) {
-      /* they match, so no need to do anything */
-    } else {
-      if (tor_addr_family(&bridge->addr) == AF_INET) {
-        ri->addr = tor_addr_to_ipv4h(&bridge->addr);
-        ri->or_port = bridge->port;
-        log_info(LD_DIR,
-                 "Adjusted bridge routerinfo for '%s' to match configured "
-                 "address %s:%d.",
-                 ri->nickname, fmt_addr32(ri->addr), ri->or_port);
-      } else if (tor_addr_family(&bridge->addr) == AF_INET6) {
-        tor_addr_copy(&ri->ipv6_addr, &bridge->addr);
-        ri->ipv6_orport = bridge->port;
-        log_info(LD_DIR,
-                 "Adjusted bridge routerinfo for '%s' to match configured "
-                 "address %s.",
-                 ri->nickname, fmt_addrport(&ri->ipv6_addr, ri->ipv6_orport));
-      } else {
-        log_err(LD_BUG, "Address family not supported: %d.",
-                tor_addr_family(&bridge->addr));
-        return;
-      }
-    }
-
-    if (options->ClientPreferIPv6ORPort == -1) {
-      /* Mark which address to use based on which bridge_t we got. */
-      node->ipv6_preferred = (tor_addr_family(&bridge->addr) == AF_INET6 &&
-                              !tor_addr_is_null(&node->ri->ipv6_addr));
-    } else {
-      /* Mark which address to use based on user preference */
-      node->ipv6_preferred = (fascist_firewall_prefer_ipv6_orport(options) &&
-                              !tor_addr_is_null(&node->ri->ipv6_addr));
-    }
-
-    /* XXXipv6 we lack support for falling back to another address for
-       the same relay, warn the user */
-    if (!tor_addr_is_null(&ri->ipv6_addr)) {
-      tor_addr_port_t ap;
-      node_get_pref_orport(node, &ap);
-      log_notice(LD_CONFIG,
-                 "Bridge '%s' has both an IPv4 and an IPv6 address.  "
-                 "Will prefer using its %s address (%s) based on %s.",
-                 ri->nickname,
-                 node->ipv6_preferred ? "IPv6" : "IPv4",
-                 fmt_addrport(&ap.addr, ap.port),
-                 options->ClientPreferIPv6ORPort == -1 ?
-                 "the configured Bridge address" :
-                 "ClientPreferIPv6ORPort");
-    }
-  }
-  if (node->rs) {
-    routerstatus_t *rs = node->rs;
-    tor_addr_from_ipv4h(&addr, rs->addr);
-
-    if (!tor_addr_compare(&bridge->addr, &addr, CMP_EXACT) &&
-        bridge->port == rs->or_port) {
-      /* they match, so no need to do anything */
-    } else {
-      rs->addr = tor_addr_to_ipv4h(&bridge->addr);
-      rs->or_port = bridge->port;
-      log_info(LD_DIR,
-               "Adjusted bridge routerstatus for '%s' to match "
-               "configured address %s.",
-               rs->nickname, fmt_addrport(&bridge->addr, rs->or_port));
-    }
-  }
-}
-
-/** We just learned a descriptor for a bridge. See if that
- * digest is in our entry guard list, and add it if not. */
-void
-learned_bridge_descriptor(routerinfo_t *ri, int from_cache)
-{
-  tor_assert(ri);
-  tor_assert(ri->purpose == ROUTER_PURPOSE_BRIDGE);
-  if (get_options()->UseBridges) {
-    int first = num_bridges_usable() <= 1;
-    bridge_info_t *bridge = get_configured_bridge_by_routerinfo(ri);
-    time_t now = time(NULL);
-    router_set_status(ri->cache_info.identity_digest, 1);
-
-    if (bridge) { /* if we actually want to use this one */
-      node_t *node;
-      /* it's here; schedule its re-fetch for a long time from now. */
-      if (!from_cache)
-        download_status_reset(&bridge->fetch_status);
-
-      node = node_get_mutable_by_id(ri->cache_info.identity_digest);
-      tor_assert(node);
-      rewrite_node_address_for_bridge(bridge, node);
-      if (tor_digest_is_zero(bridge->identity)) {
-        memcpy(bridge->identity,ri->cache_info.identity_digest, DIGEST_LEN);
-        log_notice(LD_DIR, "Learned identity %s for bridge at %s:%d",
-                   hex_str(bridge->identity, DIGEST_LEN),
-                   fmt_and_decorate_addr(&bridge->addr),
-                   (int) bridge->port);
-      }
-      add_an_entry_guard(node, 1, 1, 0, 0);
-
-      log_notice(LD_DIR, "new bridge descriptor '%s' (%s): %s", ri->nickname,
-                 from_cache ? "cached" : "fresh", router_describe(ri));
-      /* set entry->made_contact so if it goes down we don't drop it from
-       * our entry node list */
-      entry_guard_register_connect_status(ri->cache_info.identity_digest,
-                                          1, 0, now);
-      if (first) {
-        routerlist_retry_directory_downloads(now);
-      }
-    }
-  }
-}
-
-/** Return the number of bridges that have descriptors that
- * are marked with purpose 'bridge' and are running.
- *
- * We use this function to decide if we're ready to start building
- * circuits through our bridges, or if we need to wait until the
- * directory "server/authority" requests finish. */
-int
-any_bridge_descriptors_known(void)
-{
-  tor_assert(get_options()->UseBridges);
-  return choose_random_entry(NULL) != NULL;
-}
-
-/** Return the number of bridges that have descriptors that are marked with
- * purpose 'bridge' and are running.
- */
-static int
-num_bridges_usable(void)
-{
-  int n_options = 0;
-  tor_assert(get_options()->UseBridges);
-  (void) choose_random_entry_impl(NULL, 0, 0, &n_options);
-  return n_options;
-}
-
-/** Return a smartlist containing all bridge identity digests */
-MOCK_IMPL(smartlist_t *,
-list_bridge_identities, (void))
-{
-  smartlist_t *result = NULL;
-  char *digest_tmp;
-
-  if (get_options()->UseBridges && bridge_list) {
-    result = smartlist_new();
-
-    SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) {
-      digest_tmp = tor_malloc(DIGEST_LEN);
-      memcpy(digest_tmp, b->identity, DIGEST_LEN);
-      smartlist_add(result, digest_tmp);
-    } SMARTLIST_FOREACH_END(b);
-  }
-
-  return result;
-}
-
-/** Get the download status for a bridge descriptor given its identity */
-MOCK_IMPL(download_status_t *,
-get_bridge_dl_status_by_id, (const char *digest))
-{
-  download_status_t *dl = NULL;
-
-  if (digest && get_options()->UseBridges && bridge_list) {
-    SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) {
-      if (tor_memeq(digest, b->identity, DIGEST_LEN)) {
-        dl = &(b->fetch_status);
-        break;
-      }
-    } SMARTLIST_FOREACH_END(b);
-  }
-
-  return dl;
-}
-
-/** Return 1 if we have at least one descriptor for an entry guard
- * (bridge or member of EntryNodes) and all descriptors we know are
- * down. Else return 0. If <b>act</b> is 1, then mark the down guards
- * up; else just observe and report. */
-static int
-entries_retry_helper(const or_options_t *options, int act)
-{
-  const node_t *node;
-  int any_known = 0;
-  int any_running = 0;
-  int need_bridges = options->UseBridges != 0;
-  if (!entry_guards)
-    entry_guards = smartlist_new();
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) {
-      node = node_get_by_id(e->identity);
-      if (node && node_has_descriptor(node) &&
-          node_is_bridge(node) == need_bridges &&
-          (!need_bridges || (!e->bad_since &&
-                             node_is_a_configured_bridge(node)))) {
-        any_known = 1;
-        if (node->is_running)
-          any_running = 1; /* some entry is both known and running */
-        else if (act) {
-          /* Mark all current connections to this OR as unhealthy, since
-           * otherwise there could be one that started 30 seconds
-           * ago, and in 30 seconds it will time out, causing us to mark
-           * the node down and undermine the retry attempt. We mark even
-           * the established conns, since if the network just came back
-           * we'll want to attach circuits to fresh conns. */
-          connection_or_set_bad_connections(node->identity, 1);
-
-          /* mark this entry node for retry */
-          router_set_status(node->identity, 1);
-          e->can_retry = 1;
-          e->bad_since = 0;
-        }
-      }
-  } SMARTLIST_FOREACH_END(e);
-  log_debug(LD_DIR, "%d: any_known %d, any_running %d",
-            act, any_known, any_running);
-  return any_known && !any_running;
-}
-
-/** Do we know any descriptors for our bridges / entrynodes, and are
- * all the ones we have descriptors for down? */
-int
-entries_known_but_down(const or_options_t *options)
-{
-  tor_assert(entry_list_is_constrained(options));
-  return entries_retry_helper(options, 0);
-}
-
-/** Mark all down known bridges / entrynodes up. */
-void
-entries_retry_all(const or_options_t *options)
-{
-  tor_assert(entry_list_is_constrained(options));
-  entries_retry_helper(options, 1);
-}
-
-/** Return true if at least one of our bridges runs a Tor version that can
- * provide microdescriptors to us. If not, we'll fall back to asking for
- * full descriptors. */
-int
-any_bridge_supports_microdescriptors(void)
-{
-  const node_t *node;
-  if (!get_options()->UseBridges || !entry_guards)
-    return 0;
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) {
-    node = node_get_by_id(e->identity);
-    if (node && node->is_running &&
-        node_is_bridge(node) && node_is_a_configured_bridge(node)) {
-      /* This is one of our current bridges, and we know enough about
-       * it to know that it will be able to answer our questions. */
-       return 1;
-    }
-  } SMARTLIST_FOREACH_END(e);
-  return 0;
-}
-
-/** Release all storage held by the list of entry guards and related
- * memory structs. */
-void
-entry_guards_free_all(void)
-{
-  if (entry_guards) {
-    SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e,
-                      entry_guard_free(e));
-    smartlist_free(entry_guards);
-    entry_guards = NULL;
-  }
-  clear_bridge_list();
-  smartlist_free(bridge_list);
-  bridge_list = NULL;
-  circuit_build_times_free_timeouts(get_circuit_build_times_mutable());
-}
-
diff --git a/src/or/entrynodes.h b/src/or/entrynodes.h
deleted file mode 100644
index 1021e67d43..0000000000
--- a/src/or/entrynodes.h
+++ /dev/null
@@ -1,187 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file entrynodes.h
- * \brief Header file for circuitbuild.c.
- **/
-
-#ifndef TOR_ENTRYNODES_H
-#define TOR_ENTRYNODES_H
-
-#if 1
-/* XXXX NM I would prefer that all of this stuff be private to
- * entrynodes.c. */
-
-/** An entry_guard_t represents our information about a chosen long-term
- * first hop, known as a "helper" node in the literature. We can't just
- * use a node_t, since we want to remember these even when we
- * don't have any directory info. */
-typedef struct entry_guard_t {
-  char nickname[MAX_NICKNAME_LEN+1];
-  char identity[DIGEST_LEN];
-  time_t chosen_on_date; /**< Approximately when was this guard added?
-                          * "0" if we don't know. */
-  char *chosen_by_version; /**< What tor version added this guard? NULL
-                            * if we don't know. */
-  unsigned int made_contact : 1; /**< 0 if we have never connected to this
-                                  * router, 1 if we have. */
-  unsigned int can_retry : 1; /**< Should we retry connecting to this entry,
-                               * in spite of having it marked as unreachable?*/
-  unsigned int path_bias_noticed : 1; /**< Did we alert the user about path
-                                       * bias for this node already? */
-  unsigned int path_bias_warned : 1; /**< Did we alert the user about path bias
-                                      * for this node already? */
-  unsigned int path_bias_extreme : 1; /**< Did we alert the user about path
-                                       * bias for this node already? */
-  unsigned int path_bias_disabled : 1; /**< Have we disabled this node because
-                                        * of path bias issues? */
-  unsigned int path_bias_use_noticed : 1; /**< Did we alert the user about path
-                                       * use bias for this node already? */
-  unsigned int path_bias_use_extreme : 1; /**< Did we alert the user about path
-                                       * use bias for this node already? */
-  unsigned int is_dir_cache : 1; /**< Is this node a directory cache? */
-  time_t bad_since; /**< 0 if this guard is currently usable, or the time at
-                      * which it was observed to become (according to the
-                      * directory or the user configuration) unusable. */
-  time_t unreachable_since; /**< 0 if we can connect to this guard, or the
-                             * time at which we first noticed we couldn't
-                             * connect to it. */
-  time_t last_attempted; /**< 0 if we can connect to this guard, or the time
-                          * at which we last failed to connect to it. */
-
-  double circ_attempts; /**< Number of circuits this guard has "attempted" */
-  double circ_successes; /**< Number of successfully built circuits using
-                               * this guard as first hop. */
-  double successful_circuits_closed; /**< Number of circuits that carried
-                                        * streams successfully. */
-  double collapsed_circuits; /**< Number of fully built circuits that were
-                                 * remotely closed before any streams were
-                                 * attempted. */
-  double unusable_circuits; /**< Number of circuits for which streams were
-                                *  attempted, but none succeeded. */
-  double timeouts; /**< Number of 'right-censored' circuit timeouts for this
-                       * guard. */
-  double use_attempts; /**< Number of circuits we tried to use with streams */
-  double use_successes; /**< Number of successfully used circuits using
-                               * this guard as first hop. */
-} entry_guard_t;
-
-entry_guard_t *entry_guard_get_by_id_digest(const char *digest);
-void entry_guards_changed(void);
-const smartlist_t *get_entry_guards(void);
-int num_live_entry_guards(int for_directory);
-
-#endif
-
-#ifdef ENTRYNODES_PRIVATE
-STATIC const node_t *add_an_entry_guard(const node_t *chosen,
-                                        int reset_status, int prepend,
-                                        int for_discovery, int for_directory);
-
-STATIC int populate_live_entry_guards(smartlist_t *live_entry_guards,
-                                      const smartlist_t *all_entry_guards,
-                                      const node_t *chosen_exit,
-                                      dirinfo_type_t dirinfo_type,
-                                      int for_directory,
-                                      int need_uptime, int need_capacity);
-STATIC int decide_num_guards(const or_options_t *options, int for_directory);
-
-STATIC void entry_guards_set_from_config(const or_options_t *options);
-
-/** Flags to be passed to entry_is_live() to indicate what kind of
- * entry nodes we are looking for. */
-typedef enum {
-  ENTRY_NEED_UPTIME = 1<<0,
-  ENTRY_NEED_CAPACITY = 1<<1,
-  ENTRY_ASSUME_REACHABLE = 1<<2,
-  ENTRY_NEED_DESCRIPTOR = 1<<3,
-} entry_is_live_flags_t;
-
-STATIC const node_t *entry_is_live(const entry_guard_t *e,
-                                   entry_is_live_flags_t flags,
-                                   const char **msg);
-
-STATIC int entry_is_time_to_retry(const entry_guard_t *e, time_t now);
-
-#endif
-
-void remove_all_entry_guards(void);
-
-void entry_guards_compute_status(const or_options_t *options, time_t now);
-int entry_guard_register_connect_status(const char *digest, int succeeded,
-                                        int mark_relay_status, time_t now);
-void entry_nodes_should_be_added(void);
-int entry_list_is_constrained(const or_options_t *options);
-const node_t *choose_random_entry(cpath_build_state_t *state);
-const node_t *choose_random_dirguard(dirinfo_type_t t);
-int entry_guards_parse_state(or_state_t *state, int set, char **msg);
-void entry_guards_update_state(or_state_t *state);
-int getinfo_helper_entry_guards(control_connection_t *conn,
-                                const char *question, char **answer,
-                                const char **errmsg);
-
-void mark_bridge_list(void);
-void sweep_bridge_list(void);
-
-int addr_is_a_configured_bridge(const tor_addr_t *addr, uint16_t port,
-                                const char *digest);
-int extend_info_is_a_configured_bridge(const extend_info_t *ei);
-int routerinfo_is_a_configured_bridge(const routerinfo_t *ri);
-int node_is_a_configured_bridge(const node_t *node);
-void learned_router_identity(const tor_addr_t *addr, uint16_t port,
-                             const char *digest);
-struct bridge_line_t;
-void bridge_add_from_config(struct bridge_line_t *bridge_line);
-void retry_bridge_descriptor_fetch_directly(const char *digest);
-void fetch_bridge_descriptors(const or_options_t *options, time_t now);
-void learned_bridge_descriptor(routerinfo_t *ri, int from_cache);
-int any_bridge_descriptors_known(void);
-int entries_known_but_down(const or_options_t *options);
-void entries_retry_all(const or_options_t *options);
-
-int any_bridge_supports_microdescriptors(void);
-const smartlist_t *get_socks_args_by_bridge_addrport(const tor_addr_t *addr,
-                                                     uint16_t port);
-
-int any_bridges_dont_support_microdescriptors(void);
-
-void entry_guards_free_all(void);
-
-const char *find_transport_name_by_bridge_addrport(const tor_addr_t *addr,
-                                                   uint16_t port);
-struct transport_t;
-int get_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port,
-                                      const struct transport_t **transport);
-
-MOCK_DECL(int, transport_is_needed, (const char *transport_name));
-int validate_pluggable_transports_config(void);
-
-double pathbias_get_close_success_count(entry_guard_t *guard);
-double pathbias_get_use_success_count(entry_guard_t *guard);
-
-/** Contains the bandwidth of a relay as a guard and as a non-guard
- *  after the guardfraction has been considered. */
-typedef struct guardfraction_bandwidth_t {
-  /** Bandwidth as a guard after guardfraction has been considered. */
-  int guard_bw;
-  /** Bandwidth as a non-guard after guardfraction has been considered. */
-  int non_guard_bw;
-} guardfraction_bandwidth_t;
-
-int should_apply_guardfraction(const networkstatus_t *ns);
-
-void
-guard_get_guardfraction_bandwidth(guardfraction_bandwidth_t *guardfraction_bw,
-                                  int orig_bandwidth,
-                                  uint32_t guardfraction_percentage);
-
-MOCK_DECL(smartlist_t *, list_bridge_identities, (void));
-MOCK_DECL(download_status_t *, get_bridge_dl_status_by_id,
-          (const char *digest));
-
-#endif
-
diff --git a/src/or/include.am b/src/or/include.am
deleted file mode 100644
index 19cf00264b..0000000000
--- a/src/or/include.am
+++ /dev/null
@@ -1,222 +0,0 @@
-bin_PROGRAMS+= src/or/tor
-noinst_LIBRARIES += \
-	src/or/libtor.a
-if UNITTESTS_ENABLED
-noinst_LIBRARIES += \
-	src/or/libtor-testing.a
-endif
-if COVERAGE_ENABLED
-noinst_PROGRAMS+= src/or/tor-cov
-endif
-
-if BUILD_NT_SERVICES
-tor_platform_source=src/or/ntmain.c
-else
-tor_platform_source=
-endif
-
-EXTRA_DIST+= src/or/ntmain.c src/or/Makefile.nmake
-
-LIBTOR_A_SOURCES = \
-	src/or/addressmap.c				\
-	src/or/buffers.c				\
-	src/or/channel.c				\
-	src/or/channeltls.c				\
-	src/or/circpathbias.c				\
-	src/or/circuitbuild.c				\
-	src/or/circuitlist.c				\
-	src/or/circuitmux.c				\
-	src/or/circuitmux_ewma.c			\
-	src/or/circuitstats.c				\
-	src/or/circuituse.c				\
-	src/or/command.c				\
-	src/or/config.c					\
-	src/or/confparse.c				\
-	src/or/connection.c				\
-	src/or/connection_edge.c			\
-	src/or/connection_or.c				\
-	src/or/control.c				\
-	src/or/cpuworker.c				\
-	src/or/dircollate.c				\
-	src/or/directory.c				\
-	src/or/dirserv.c				\
-	src/or/dirvote.c				\
-	src/or/dns.c					\
-	src/or/dnsserv.c				\
-	src/or/dos.c					\
-	src/or/fp_pair.c				\
-	src/or/geoip.c					\
-	src/or/entrynodes.c				\
-	src/or/ext_orport.c				\
-	src/or/hibernate.c				\
-	src/or/keypin.c					\
-	src/or/main.c					\
-	src/or/microdesc.c				\
-	src/or/networkstatus.c				\
-	src/or/nodelist.c				\
-	src/or/onion.c					\
-	src/or/onion_fast.c				\
-	src/or/onion_tap.c				\
-	src/or/shared_random.c			\
-	src/or/shared_random_state.c		\
-	src/or/transports.c				\
-	src/or/periodic.c				\
-	src/or/protover.c				\
-	src/or/policies.c				\
-	src/or/reasons.c				\
-	src/or/relay.c					\
-	src/or/rendcache.c				\
-	src/or/rendclient.c				\
-	src/or/rendcommon.c				\
-	src/or/rendmid.c				\
-	src/or/rendservice.c				\
-	src/or/rephist.c				\
-	src/or/replaycache.c				\
-	src/or/router.c					\
-	src/or/routerkeys.c				\
-	src/or/routerlist.c				\
-	src/or/routerparse.c				\
-	src/or/routerset.c				\
-	src/or/scheduler.c				\
-	src/or/statefile.c				\
-	src/or/status.c					\
-	src/or/torcert.c				\
-	src/or/onion_ntor.c				\
-	$(tor_platform_source)
-
-src_or_libtor_a_SOURCES = $(LIBTOR_A_SOURCES)
-src_or_libtor_testing_a_SOURCES = $(LIBTOR_A_SOURCES)
-
-src_or_tor_SOURCES = src/or/tor_main.c
-AM_CPPFLAGS += -I$(srcdir)/src/or -Isrc/or
-
-src/or/tor_main.$(OBJEXT) \
-  src/or/src_or_tor_cov-tor_main.$(OBJEXT): micro-revision.i
-
-AM_CPPFLAGS += -DSHARE_DATADIR="\"$(datadir)\""		\
-		-DLOCALSTATEDIR="\"$(localstatedir)\""	\
-		-DBINDIR="\"$(bindir)\""
-
-src_or_libtor_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
-src_or_libtor_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
-
-# -L flags need to go in LDFLAGS. -l flags need to go in LDADD.
-# This seems to matter nowhere but on windows, but I assure you that it
-# matters a lot there, and is quite hard to debug if you forget to do it.
-
-
-src_or_tor_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@
-src_or_tor_LDADD = src/or/libtor.a src/common/libor.a src/common/libor-ctime.a \
-	src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \
-	src/common/libor-event.a src/trunnel/libor-trunnel.a \
-	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \
-	@TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@
-
-if COVERAGE_ENABLED
-src_or_tor_cov_SOURCES = src/or/tor_main.c
-src_or_tor_cov_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
-src_or_tor_cov_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
-src_or_tor_cov_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@
-src_or_tor_cov_LDADD = src/or/libtor-testing.a src/common/libor-testing.a \
-	src/common/libor-ctime-testing.a \
-	src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \
-	src/common/libor-event-testing.a src/trunnel/libor-trunnel-testing.a \
-	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \
-	@TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@
-endif
-
-ORHEADERS = \
-	src/or/addressmap.h				\
-	src/or/auth_dirs.inc				\
-	src/or/buffers.h				\
-	src/or/channel.h				\
-	src/or/channeltls.h				\
-	src/or/circpathbias.h				\
-	src/or/circuitbuild.h				\
-	src/or/circuitlist.h				\
-	src/or/circuitmux.h				\
-	src/or/circuitmux_ewma.h			\
-	src/or/circuitstats.h				\
-	src/or/circuituse.h				\
-	src/or/command.h				\
-	src/or/config.h					\
-	src/or/confparse.h				\
-	src/or/connection.h				\
-	src/or/connection_edge.h			\
-	src/or/connection_or.h				\
-	src/or/control.h				\
-	src/or/cpuworker.h				\
-	src/or/dircollate.h				\
-	src/or/directory.h				\
-	src/or/dirserv.h				\
-	src/or/dirvote.h				\
-	src/or/dns.h					\
-	src/or/dns_structs.h				\
-	src/or/dnsserv.h				\
-	src/or/dos.h					\
-	src/or/ext_orport.h				\
-	src/or/fallback_dirs.inc			\
-	src/or/fp_pair.h				\
-	src/or/geoip.h					\
-	src/or/entrynodes.h				\
-	src/or/hibernate.h				\
-	src/or/keypin.h					\
-	src/or/main.h					\
-	src/or/microdesc.h				\
-	src/or/networkstatus.h				\
-	src/or/nodelist.h				\
-	src/or/ntmain.h					\
-	src/or/onion.h					\
-	src/or/onion_fast.h				\
-	src/or/onion_ntor.h				\
-	src/or/onion_tap.h				\
-	src/or/or.h					\
-	src/or/shared_random.h			\
-	src/or/shared_random_state.h		\
-	src/or/transports.h				\
-	src/or/periodic.h				\
-	src/or/policies.h				\
-	src/or/protover.h				\
-	src/or/reasons.h				\
-	src/or/relay.h					\
-	src/or/rendcache.h				\
-	src/or/rendclient.h				\
-	src/or/rendcommon.h				\
-	src/or/rendmid.h				\
-	src/or/rendservice.h				\
-	src/or/rephist.h				\
-	src/or/replaycache.h				\
-	src/or/router.h					\
-	src/or/routerkeys.h				\
-	src/or/routerlist.h				\
-	src/or/routerkeys.h				\
-	src/or/routerset.h				\
-	src/or/routerparse.h				\
-	src/or/scheduler.h				\
-	src/or/statefile.h				\
-	src/or/status.h					\
-	src/or/torcert.h
-
-noinst_HEADERS+= $(ORHEADERS) micro-revision.i
-
-micro-revision.i: FORCE
-	$(AM_V_at)rm -f micro-revision.tmp; \
-	if test -r "$(top_srcdir)/.git" && \
-			test -x "`which git 2>&1;true`"; then \
-		HASH="`cd "$(top_srcdir)" && git rev-parse --short=16 HEAD`"; \
-		echo \"$$HASH\" > micro-revision.tmp; \
-	fi; \
-	if test ! -f micro-revision.tmp; then \
-		if test ! -f micro-revision.i; then \
-			echo '""' > micro-revision.i; \
-		fi; \
-	elif test ! -f micro-revision.i || \
-			test x"`cat micro-revision.tmp`" != x"`cat micro-revision.i`"; then \
-		mv micro-revision.tmp micro-revision.i; \
-	fi; \
-	rm -f micro-revision.tmp; \
-	true
-
-CLEANFILES+= micro-revision.i src/or/micro-revision.i micro-revision.tmp
-
-FORCE:
diff --git a/src/or/onion.c b/src/or/onion.c
deleted file mode 100644
index 4b803a785c..0000000000
--- a/src/or/onion.c
+++ /dev/null
@@ -1,1247 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file onion.c
- * \brief Functions to queue create cells, wrap the various onionskin types,
- * and parse and create the CREATE cell and its allies.
- *
- * This module has a few functions, all related to the CREATE/CREATED
- * handshake that we use on links in order to create a circuit, and the
- * related EXTEND/EXTENDED handshake that we use over circuits in order to
- * extend them an additional hop.
- *
- * In this module, we provide a set of abstractions to create a uniform
- * interface over the three circuit extension handshakes that Tor has used
- * over the years (TAP, CREATE_FAST, and ntor).  These handshakes are
- * implemented in onion_tap.c, onion_fast.c, and onion_ntor.c respectively.
- *
- * All[*] of these handshakes follow a similar pattern: a client, knowing
- * some key from the relay it wants to extend through, generates the
- * first part of a handshake. A relay receives that handshake, and sends
- * a reply.  Once the client handles the reply, it knows that it is
- * talking to the right relay, and it shares some freshly negotiated key
- * material with that relay.
- *
- * We sometimes call the client's part of the handshake an "onionskin".
- * We do this because historically, Onion Routing used a multi-layer
- * structure called an "onion" to construct circuits. Each layer of the
- * onion contained key material chosen by the client, the identity of
- * the next relay in the circuit, and a smaller onion, encrypted with
- * the key of the next relay.  When we changed Tor to use a telescoping
- * circuit extension design, it corresponded to sending each layer of the
- * onion separately -- as a series of onionskins.
- *
- * Clients invoke these functions when creating or extending a circuit,
- * from circuitbuild.c.
- *
- * Relays invoke these functions when they receive a CREATE or EXTEND
- * cell in command.c or relay.c, in order to queue the pending request.
- * They also invoke them from cpuworker.c, which handles dispatching
- * onionskin requests to different worker threads.
- *
- * <br>
- *
- * This module also handles:
- *  <ul>
- *  <li> Queueing incoming onionskins on the relay side before passing
- *      them to worker threads.
- *   <li>Expiring onionskins on the relay side if they have waited for
- *     too long.
- *   <li>Packaging private keys on the server side in order to pass
- *     them to worker threads.
- *   <li>Encoding and decoding CREATE, CREATED, CREATE2, and CREATED2 cells.
- *   <li>Encoding and decodign EXTEND, EXTENDED, EXTEND2, and EXTENDED2
- *    relay cells.
- * </ul>
- *
- * [*] The CREATE_FAST handshake is weaker than described here; see
- * onion_fast.c for more information.
- **/
-
-#include "or.h"
-#include "circuitbuild.h"
-#include "circuitlist.h"
-#include "config.h"
-#include "cpuworker.h"
-#include "networkstatus.h"
-#include "onion.h"
-#include "onion_fast.h"
-#include "onion_ntor.h"
-#include "onion_tap.h"
-#include "relay.h"
-#include "rephist.h"
-#include "router.h"
-
-/** Type for a linked list of circuits that are waiting for a free CPU worker
- * to process a waiting onion handshake. */
-typedef struct onion_queue_t {
-  TOR_TAILQ_ENTRY(onion_queue_t) next;
-  or_circuit_t *circ;
-  uint16_t handshake_type;
-  create_cell_t *onionskin;
-  time_t when_added;
-} onion_queue_t;
-
-/** 5 seconds on the onion queue til we just send back a destroy */
-#define ONIONQUEUE_WAIT_CUTOFF 5
-
-/** Array of queues of circuits waiting for CPU workers. An element is NULL
- * if that queue is empty.*/
-static TOR_TAILQ_HEAD(onion_queue_head_t, onion_queue_t)
-              ol_list[MAX_ONION_HANDSHAKE_TYPE+1] =
-{ TOR_TAILQ_HEAD_INITIALIZER(ol_list[0]), /* tap */
-  TOR_TAILQ_HEAD_INITIALIZER(ol_list[1]), /* fast */
-  TOR_TAILQ_HEAD_INITIALIZER(ol_list[2]), /* ntor */
-};
-
-/** Number of entries of each type currently in each element of ol_list[]. */
-static int ol_entries[MAX_ONION_HANDSHAKE_TYPE+1];
-
-static int num_ntors_per_tap(void);
-static void onion_queue_entry_remove(onion_queue_t *victim);
-
-/* XXXX Check lengths vs MAX_ONIONSKIN_{CHALLENGE,REPLY}_LEN.
- *
- * (By which I think I meant, "make sure that no
- * X_ONIONSKIN_CHALLENGE/REPLY_LEN is greater than
- * MAX_ONIONSKIN_CHALLENGE/REPLY_LEN."  Also, make sure that we can pass
- * over-large values via EXTEND2/EXTENDED2, for future-compatibility.*/
-
-/** Return true iff we have room to queue another onionskin of type
- * <b>type</b>. */
-static int
-have_room_for_onionskin(uint16_t type)
-{
-  const or_options_t *options = get_options();
-  int num_cpus;
-  uint64_t tap_usec, ntor_usec;
-  uint64_t ntor_during_tap_usec, tap_during_ntor_usec;
-
-  /* If we've got fewer than 50 entries, we always have room for one more. */
-  if (ol_entries[type] < 50)
-    return 1;
-  num_cpus = get_num_cpus(options);
-  /* Compute how many microseconds we'd expect to need to clear all
-   * onionskins in various combinations of the queues. */
-
-  /* How long would it take to process all the TAP cells in the queue? */
-  tap_usec  = estimated_usec_for_onionskins(
-                                    ol_entries[ONION_HANDSHAKE_TYPE_TAP],
-                                    ONION_HANDSHAKE_TYPE_TAP) / num_cpus;
-
-  /* How long would it take to process all the NTor cells in the queue? */
-  ntor_usec = estimated_usec_for_onionskins(
-                                    ol_entries[ONION_HANDSHAKE_TYPE_NTOR],
-                                    ONION_HANDSHAKE_TYPE_NTOR) / num_cpus;
-
-  /* How long would it take to process the tap cells that we expect to
-   * process while draining the ntor queue? */
-  tap_during_ntor_usec  = estimated_usec_for_onionskins(
-    MIN(ol_entries[ONION_HANDSHAKE_TYPE_TAP],
-        ol_entries[ONION_HANDSHAKE_TYPE_NTOR] / num_ntors_per_tap()),
-                                    ONION_HANDSHAKE_TYPE_TAP) / num_cpus;
-
-  /* How long would it take to process the ntor cells that we expect to
-   * process while draining the tap queue? */
-  ntor_during_tap_usec  = estimated_usec_for_onionskins(
-    MIN(ol_entries[ONION_HANDSHAKE_TYPE_NTOR],
-        ol_entries[ONION_HANDSHAKE_TYPE_TAP] * num_ntors_per_tap()),
-                                    ONION_HANDSHAKE_TYPE_NTOR) / num_cpus;
-
-  /* See whether that exceeds MaxOnionQueueDelay. If so, we can't queue
-   * this. */
-  if (type == ONION_HANDSHAKE_TYPE_NTOR &&
-      (ntor_usec + tap_during_ntor_usec) / 1000 >
-       (uint64_t)options->MaxOnionQueueDelay)
-    return 0;
-
-  if (type == ONION_HANDSHAKE_TYPE_TAP &&
-      (tap_usec + ntor_during_tap_usec) / 1000 >
-       (uint64_t)options->MaxOnionQueueDelay)
-    return 0;
-
-  /* If we support the ntor handshake, then don't let TAP handshakes use
-   * more than 2/3 of the space on the queue. */
-  if (type == ONION_HANDSHAKE_TYPE_TAP &&
-      tap_usec / 1000 > (uint64_t)options->MaxOnionQueueDelay * 2 / 3)
-    return 0;
-
-  return 1;
-}
-
-/** Add <b>circ</b> to the end of ol_list and return 0, except
- * if ol_list is too long, in which case do nothing and return -1.
- */
-int
-onion_pending_add(or_circuit_t *circ, create_cell_t *onionskin)
-{
-  onion_queue_t *tmp;
-  time_t now = time(NULL);
-
-  if (onionskin->handshake_type > MAX_ONION_HANDSHAKE_TYPE) {
-    /* LCOV_EXCL_START
-     * We should have rejected this far before this point */
-    log_warn(LD_BUG, "Handshake %d out of range! Dropping.",
-             onionskin->handshake_type);
-    return -1;
-    /* LCOV_EXCL_STOP */
-  }
-
-  tmp = tor_malloc_zero(sizeof(onion_queue_t));
-  tmp->circ = circ;
-  tmp->handshake_type = onionskin->handshake_type;
-  tmp->onionskin = onionskin;
-  tmp->when_added = now;
-
-  if (!have_room_for_onionskin(onionskin->handshake_type)) {
-#define WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL (60)
-    static ratelim_t last_warned =
-      RATELIM_INIT(WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL);
-    char *m;
-    if (onionskin->handshake_type == ONION_HANDSHAKE_TYPE_NTOR &&
-        (m = rate_limit_log(&last_warned, approx_time()))) {
-      log_warn(LD_GENERAL,
-               "Your computer is too slow to handle this many circuit "
-               "creation requests! Please consider using the "
-               "MaxAdvertisedBandwidth config option or choosing a more "
-               "restricted exit policy.%s",m);
-      tor_free(m);
-    }
-    tor_free(tmp);
-    return -1;
-  }
-
-  ++ol_entries[onionskin->handshake_type];
-  log_info(LD_OR, "New create (%s). Queues now ntor=%d and tap=%d.",
-    onionskin->handshake_type == ONION_HANDSHAKE_TYPE_NTOR ? "ntor" : "tap",
-    ol_entries[ONION_HANDSHAKE_TYPE_NTOR],
-    ol_entries[ONION_HANDSHAKE_TYPE_TAP]);
-
-  circ->onionqueue_entry = tmp;
-  TOR_TAILQ_INSERT_TAIL(&ol_list[onionskin->handshake_type], tmp, next);
-
-  /* cull elderly requests. */
-  while (1) {
-    onion_queue_t *head = TOR_TAILQ_FIRST(&ol_list[onionskin->handshake_type]);
-    if (now - head->when_added < (time_t)ONIONQUEUE_WAIT_CUTOFF)
-      break;
-
-    circ = head->circ;
-    circ->onionqueue_entry = NULL;
-    onion_queue_entry_remove(head);
-    log_info(LD_CIRC,
-             "Circuit create request is too old; canceling due to overload.");
-    if (! TO_CIRCUIT(circ)->marked_for_close) {
-      circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_RESOURCELIMIT);
-    }
-  }
-  return 0;
-}
-
-/** Return a fairness parameter, to prefer processing NTOR style
- * handshakes but still slowly drain the TAP queue so we don't starve
- * it entirely. */
-static int
-num_ntors_per_tap(void)
-{
-#define DEFAULT_NUM_NTORS_PER_TAP 10
-#define MIN_NUM_NTORS_PER_TAP 1
-#define MAX_NUM_NTORS_PER_TAP 100000
-
-  return networkstatus_get_param(NULL, "NumNTorsPerTAP",
-                                 DEFAULT_NUM_NTORS_PER_TAP,
-                                 MIN_NUM_NTORS_PER_TAP,
-                                 MAX_NUM_NTORS_PER_TAP);
-}
-
-/** Choose which onion queue we'll pull from next. If one is empty choose
- * the other; if they both have elements, load balance across them but
- * favoring NTOR. */
-static uint16_t
-decide_next_handshake_type(void)
-{
-  /* The number of times we've chosen ntor lately when both were available. */
-  static int recently_chosen_ntors = 0;
-
-  if (!ol_entries[ONION_HANDSHAKE_TYPE_NTOR])
-    return ONION_HANDSHAKE_TYPE_TAP; /* no ntors? try tap */
-
-  if (!ol_entries[ONION_HANDSHAKE_TYPE_TAP]) {
-
-    /* Nick wants us to prioritize new tap requests when there aren't
-     * any in the queue and we've processed k ntor cells since the last
-     * tap cell. This strategy is maybe a good idea, since it starves tap
-     * less in the case where tap is rare, or maybe a poor idea, since it
-     * makes the new tap cell unfairly jump in front of ntor cells that
-     * got here first. In any case this edge case will only become relevant
-     * once tap is rare. We should reevaluate whether we like this decision
-     * once tap gets more rare. */
-    if (ol_entries[ONION_HANDSHAKE_TYPE_NTOR] &&
-        recently_chosen_ntors <= num_ntors_per_tap())
-      ++recently_chosen_ntors;
-
-    return ONION_HANDSHAKE_TYPE_NTOR; /* no taps? try ntor */
-  }
-
-  /* They both have something queued. Pick ntor if we haven't done that
-   * too much lately. */
-  if (++recently_chosen_ntors <= num_ntors_per_tap()) {
-    return ONION_HANDSHAKE_TYPE_NTOR;
-  }
-
-  /* Else, it's time to let tap have its turn. */
-  recently_chosen_ntors = 0;
-  return ONION_HANDSHAKE_TYPE_TAP;
-}
-
-/** Remove the highest priority item from ol_list[] and return it, or
- * return NULL if the lists are empty.
- */
-or_circuit_t *
-onion_next_task(create_cell_t **onionskin_out)
-{
-  or_circuit_t *circ;
-  uint16_t handshake_to_choose = decide_next_handshake_type();
-  onion_queue_t *head = TOR_TAILQ_FIRST(&ol_list[handshake_to_choose]);
-
-  if (!head)
-    return NULL; /* no onions pending, we're done */
-
-  tor_assert(head->circ);
-  tor_assert(head->handshake_type <= MAX_ONION_HANDSHAKE_TYPE);
-//  tor_assert(head->circ->p_chan); /* make sure it's still valid */
-/* XXX I only commented out the above line to make the unit tests
- * more manageable. That's probably not good long-term. -RD */
-  circ = head->circ;
-  if (head->onionskin)
-    --ol_entries[head->handshake_type];
-  log_info(LD_OR, "Processing create (%s). Queues now ntor=%d and tap=%d.",
-    head->handshake_type == ONION_HANDSHAKE_TYPE_NTOR ? "ntor" : "tap",
-    ol_entries[ONION_HANDSHAKE_TYPE_NTOR],
-    ol_entries[ONION_HANDSHAKE_TYPE_TAP]);
-
-  *onionskin_out = head->onionskin;
-  head->onionskin = NULL; /* prevent free. */
-  circ->onionqueue_entry = NULL;
-  onion_queue_entry_remove(head);
-  return circ;
-}
-
-/** Return the number of <b>handshake_type</b>-style create requests pending.
- */
-int
-onion_num_pending(uint16_t handshake_type)
-{
-  return ol_entries[handshake_type];
-}
-
-/** Go through ol_list, find the onion_queue_t element which points to
- * circ, remove and free that element. Leave circ itself alone.
- */
-void
-onion_pending_remove(or_circuit_t *circ)
-{
-  onion_queue_t *victim;
-
-  if (!circ)
-    return;
-
-  victim = circ->onionqueue_entry;
-  if (victim)
-    onion_queue_entry_remove(victim);
-
-  cpuworker_cancel_circ_handshake(circ);
-}
-
-/** Remove a queue entry <b>victim</b> from the queue, unlinking it from
- * its circuit and freeing it and any structures it owns.*/
-static void
-onion_queue_entry_remove(onion_queue_t *victim)
-{
-  if (victim->handshake_type > MAX_ONION_HANDSHAKE_TYPE) {
-    /* LCOV_EXCL_START
-     * We should have rejected this far before this point */
-    log_warn(LD_BUG, "Handshake %d out of range! Dropping.",
-             victim->handshake_type);
-    /* XXX leaks */
-    return;
-    /* LCOV_EXCL_STOP */
-  }
-
-  TOR_TAILQ_REMOVE(&ol_list[victim->handshake_type], victim, next);
-
-  if (victim->circ)
-    victim->circ->onionqueue_entry = NULL;
-
-  if (victim->onionskin)
-    --ol_entries[victim->handshake_type];
-
-  tor_free(victim->onionskin);
-  tor_free(victim);
-}
-
-/** Remove all circuits from the pending list.  Called from tor_free_all. */
-void
-clear_pending_onions(void)
-{
-  onion_queue_t *victim, *next;
-  int i;
-  for (i=0; i<=MAX_ONION_HANDSHAKE_TYPE; i++) {
-    for (victim = TOR_TAILQ_FIRST(&ol_list[i]); victim; victim = next) {
-      next = TOR_TAILQ_NEXT(victim,next);
-      onion_queue_entry_remove(victim);
-    }
-    tor_assert(TOR_TAILQ_EMPTY(&ol_list[i]));
-  }
-  memset(ol_entries, 0, sizeof(ol_entries));
-}
-
-/* ============================================================ */
-
-/** Return a new server_onion_keys_t object with all of the keys
- * and other info we might need to do onion handshakes.  (We make a copy of
- * our keys for each cpuworker to avoid race conditions with the main thread,
- * and to avoid locking) */
-server_onion_keys_t *
-server_onion_keys_new(void)
-{
-  server_onion_keys_t *keys = tor_malloc_zero(sizeof(server_onion_keys_t));
-  memcpy(keys->my_identity, router_get_my_id_digest(), DIGEST_LEN);
-  dup_onion_keys(&keys->onion_key, &keys->last_onion_key);
-  keys->curve25519_key_map = construct_ntor_key_map();
-  keys->junk_keypair = tor_malloc_zero(sizeof(curve25519_keypair_t));
-  curve25519_keypair_generate(keys->junk_keypair, 0);
-  return keys;
-}
-
-/** Release all storage held in <b>keys</b>. */
-void
-server_onion_keys_free(server_onion_keys_t *keys)
-{
-  if (! keys)
-    return;
-
-  crypto_pk_free(keys->onion_key);
-  crypto_pk_free(keys->last_onion_key);
-  ntor_key_map_free(keys->curve25519_key_map);
-  tor_free(keys->junk_keypair);
-  memwipe(keys, 0, sizeof(server_onion_keys_t));
-  tor_free(keys);
-}
-
-/** Release whatever storage is held in <b>state</b>, depending on its
- * type, and clear its pointer. */
-void
-onion_handshake_state_release(onion_handshake_state_t *state)
-{
-  switch (state->tag) {
-  case ONION_HANDSHAKE_TYPE_TAP:
-    crypto_dh_free(state->u.tap);
-    state->u.tap = NULL;
-    break;
-  case ONION_HANDSHAKE_TYPE_FAST:
-    fast_handshake_state_free(state->u.fast);
-    state->u.fast = NULL;
-    break;
-  case ONION_HANDSHAKE_TYPE_NTOR:
-    ntor_handshake_state_free(state->u.ntor);
-    state->u.ntor = NULL;
-    break;
-  default:
-    /* LCOV_EXCL_START
-     * This state should not even exist. */
-    log_warn(LD_BUG, "called with unknown handshake state type %d",
-             (int)state->tag);
-    tor_fragile_assert();
-    /* LCOV_EXCL_STOP */
-  }
-}
-
-/** Perform the first step of a circuit-creation handshake of type <b>type</b>
- * (one of ONION_HANDSHAKE_TYPE_*): generate the initial "onion skin" in
- * <b>onion_skin_out</b>, and store any state information in <b>state_out</b>.
- * Return -1 on failure, and the length of the onionskin on acceptance.
- */
-int
-onion_skin_create(int type,
-                  const extend_info_t *node,
-                  onion_handshake_state_t *state_out,
-                  uint8_t *onion_skin_out)
-{
-  int r = -1;
-
-  switch (type) {
-  case ONION_HANDSHAKE_TYPE_TAP:
-    if (!node->onion_key)
-      return -1;
-
-    if (onion_skin_TAP_create(node->onion_key,
-                              &state_out->u.tap,
-                              (char*)onion_skin_out) < 0)
-      return -1;
-
-    r = TAP_ONIONSKIN_CHALLENGE_LEN;
-    break;
-  case ONION_HANDSHAKE_TYPE_FAST:
-    if (fast_onionskin_create(&state_out->u.fast, onion_skin_out) < 0)
-      return -1;
-
-    r = CREATE_FAST_LEN;
-    break;
-  case ONION_HANDSHAKE_TYPE_NTOR:
-    if (!extend_info_supports_ntor(node))
-      return -1;
-    if (onion_skin_ntor_create((const uint8_t*)node->identity_digest,
-                               &node->curve25519_onion_key,
-                               &state_out->u.ntor,
-                               onion_skin_out) < 0)
-      return -1;
-
-    r = NTOR_ONIONSKIN_LEN;
-    break;
-  default:
-    /* LCOV_EXCL_START
-     * We should never try to create an impossible handshake type. */
-    log_warn(LD_BUG, "called with unknown handshake state type %d", type);
-    tor_fragile_assert();
-    r = -1;
-    /* LCOV_EXCL_STOP */
-  }
-
-  if (r > 0)
-    state_out->tag = (uint16_t) type;
-
-  return r;
-}
-
-/** Perform the second (server-side) step of a circuit-creation handshake of
- * type <b>type</b>, responding to the client request in <b>onion_skin</b>
- * using the keys in <b>keys</b>.  On success, write our response into
- * <b>reply_out</b>, generate <b>keys_out_len</b> bytes worth of key material
- * in <b>keys_out_len</b>, a hidden service nonce to <b>rend_nonce_out</b>,
- * and return the length of the reply. On failure, return -1.
- */
-int
-onion_skin_server_handshake(int type,
-                      const uint8_t *onion_skin, size_t onionskin_len,
-                      const server_onion_keys_t *keys,
-                      uint8_t *reply_out,
-                      uint8_t *keys_out, size_t keys_out_len,
-                      uint8_t *rend_nonce_out)
-{
-  int r = -1;
-
-  switch (type) {
-  case ONION_HANDSHAKE_TYPE_TAP:
-    if (onionskin_len != TAP_ONIONSKIN_CHALLENGE_LEN)
-      return -1;
-    if (onion_skin_TAP_server_handshake((const char*)onion_skin,
-                                        keys->onion_key, keys->last_onion_key,
-                                        (char*)reply_out,
-                                        (char*)keys_out, keys_out_len)<0)
-      return -1;
-    r = TAP_ONIONSKIN_REPLY_LEN;
-    memcpy(rend_nonce_out, reply_out+DH_KEY_LEN, DIGEST_LEN);
-    break;
-  case ONION_HANDSHAKE_TYPE_FAST:
-    if (onionskin_len != CREATE_FAST_LEN)
-      return -1;
-    if (fast_server_handshake(onion_skin, reply_out, keys_out, keys_out_len)<0)
-      return -1;
-    r = CREATED_FAST_LEN;
-    memcpy(rend_nonce_out, reply_out+DIGEST_LEN, DIGEST_LEN);
-    break;
-  case ONION_HANDSHAKE_TYPE_NTOR:
-    if (onionskin_len < NTOR_ONIONSKIN_LEN)
-      return -1;
-    {
-      size_t keys_tmp_len = keys_out_len + DIGEST_LEN;
-      uint8_t *keys_tmp = tor_malloc(keys_out_len + DIGEST_LEN);
-
-      if (onion_skin_ntor_server_handshake(
-                                   onion_skin, keys->curve25519_key_map,
-                                   keys->junk_keypair,
-                                   keys->my_identity,
-                                   reply_out, keys_tmp, keys_tmp_len)<0) {
-        tor_free(keys_tmp);
-        return -1;
-      }
-      memcpy(keys_out, keys_tmp, keys_out_len);
-      memcpy(rend_nonce_out, keys_tmp+keys_out_len, DIGEST_LEN);
-      memwipe(keys_tmp, 0, keys_tmp_len);
-      tor_free(keys_tmp);
-      r = NTOR_REPLY_LEN;
-    }
-    break;
-  default:
-    /* LCOV_EXCL_START
-     * We should have rejected this far before this point */
-    log_warn(LD_BUG, "called with unknown handshake state type %d", type);
-    tor_fragile_assert();
-    return -1;
-    /* LCOV_EXCL_STOP */
-  }
-
-  return r;
-}
-
-/** Perform the final (client-side) step of a circuit-creation handshake of
- * type <b>type</b>, using our state in <b>handshake_state</b> and the
- * server's response in <b>reply</b>. On success, generate <b>keys_out_len</b>
- * bytes worth of key material in <b>keys_out_len</b>, set
- * <b>rend_authenticator_out</b> to the "KH" field that can be used to
- * establish introduction points at this hop, and return 0. On failure,
- * return -1, and set *msg_out to an error message if this is worth
- * complaining to the user about. */
-int
-onion_skin_client_handshake(int type,
-                      const onion_handshake_state_t *handshake_state,
-                      const uint8_t *reply, size_t reply_len,
-                      uint8_t *keys_out, size_t keys_out_len,
-                      uint8_t *rend_authenticator_out,
-                      const char **msg_out)
-{
-  if (handshake_state->tag != type)
-    return -1;
-
-  switch (type) {
-  case ONION_HANDSHAKE_TYPE_TAP:
-    if (reply_len != TAP_ONIONSKIN_REPLY_LEN) {
-      if (msg_out)
-        *msg_out = "TAP reply was not of the correct length.";
-      return -1;
-    }
-    if (onion_skin_TAP_client_handshake(handshake_state->u.tap,
-                                        (const char*)reply,
-                                        (char *)keys_out, keys_out_len,
-                                        msg_out) < 0)
-      return -1;
-
-    memcpy(rend_authenticator_out, reply+DH_KEY_LEN, DIGEST_LEN);
-
-    return 0;
-  case ONION_HANDSHAKE_TYPE_FAST:
-    if (reply_len != CREATED_FAST_LEN) {
-      if (msg_out)
-        *msg_out = "TAP reply was not of the correct length.";
-      return -1;
-    }
-    if (fast_client_handshake(handshake_state->u.fast, reply,
-                              keys_out, keys_out_len, msg_out) < 0)
-      return -1;
-
-    memcpy(rend_authenticator_out, reply+DIGEST_LEN, DIGEST_LEN);
-    return 0;
-  case ONION_HANDSHAKE_TYPE_NTOR:
-    if (reply_len < NTOR_REPLY_LEN) {
-      if (msg_out)
-        *msg_out = "ntor reply was not of the correct length.";
-      return -1;
-    }
-    {
-      size_t keys_tmp_len = keys_out_len + DIGEST_LEN;
-      uint8_t *keys_tmp = tor_malloc(keys_tmp_len);
-      if (onion_skin_ntor_client_handshake(handshake_state->u.ntor,
-                                        reply,
-                                        keys_tmp, keys_tmp_len, msg_out) < 0) {
-        tor_free(keys_tmp);
-        return -1;
-      }
-      memcpy(keys_out, keys_tmp, keys_out_len);
-      memcpy(rend_authenticator_out, keys_tmp + keys_out_len, DIGEST_LEN);
-      memwipe(keys_tmp, 0, keys_tmp_len);
-      tor_free(keys_tmp);
-    }
-    return 0;
-  default:
-    log_warn(LD_BUG, "called with unknown handshake state type %d", type);
-    tor_fragile_assert();
-    return -1;
-  }
-}
-
-/** Helper: return 0 if <b>cell</b> appears valid, -1 otherwise. If
- * <b>unknown_ok</b> is true, allow cells with handshake types we don't
- * recognize. */
-static int
-check_create_cell(const create_cell_t *cell, int unknown_ok)
-{
-  switch (cell->cell_type) {
-  case CELL_CREATE:
-    if (cell->handshake_type != ONION_HANDSHAKE_TYPE_TAP &&
-        cell->handshake_type != ONION_HANDSHAKE_TYPE_NTOR)
-      return -1;
-    break;
-  case CELL_CREATE_FAST:
-    if (cell->handshake_type != ONION_HANDSHAKE_TYPE_FAST)
-      return -1;
-    break;
-  case CELL_CREATE2:
-    break;
-  default:
-    return -1;
-  }
-
-  switch (cell->handshake_type) {
-  case ONION_HANDSHAKE_TYPE_TAP:
-    if (cell->handshake_len != TAP_ONIONSKIN_CHALLENGE_LEN)
-      return -1;
-    break;
-  case ONION_HANDSHAKE_TYPE_FAST:
-    if (cell->handshake_len != CREATE_FAST_LEN)
-      return -1;
-    break;
-  case ONION_HANDSHAKE_TYPE_NTOR:
-    if (cell->handshake_len != NTOR_ONIONSKIN_LEN)
-      return -1;
-    break;
-  default:
-    if (! unknown_ok)
-      return -1;
-  }
-
-  return 0;
-}
-
-/** Write the various parameters into the create cell. Separate from
- * create_cell_parse() to make unit testing easier.
- */
-void
-create_cell_init(create_cell_t *cell_out, uint8_t cell_type,
-                 uint16_t handshake_type, uint16_t handshake_len,
-                 const uint8_t *onionskin)
-{
-  memset(cell_out, 0, sizeof(*cell_out));
-
-  cell_out->cell_type = cell_type;
-  cell_out->handshake_type = handshake_type;
-  cell_out->handshake_len = handshake_len;
-  memcpy(cell_out->onionskin, onionskin, handshake_len);
-}
-
-/** Helper: parse the CREATE2 payload at <b>p</b>, which could be up to
- * <b>p_len</b> bytes long, and use it to fill the fields of
- * <b>cell_out</b>. Return 0 on success and -1 on failure.
- *
- * Note that part of the body of an EXTEND2 cell is a CREATE2 payload, so
- * this function is also used for parsing those.
- */
-static int
-parse_create2_payload(create_cell_t *cell_out, const uint8_t *p, size_t p_len)
-{
-  uint16_t handshake_type, handshake_len;
-
-  if (p_len < 4)
-    return -1;
-
-  handshake_type = ntohs(get_uint16(p));
-  handshake_len = ntohs(get_uint16(p+2));
-
-  if (handshake_len > CELL_PAYLOAD_SIZE - 4 || handshake_len > p_len - 4)
-    return -1;
-  if (handshake_type == ONION_HANDSHAKE_TYPE_FAST)
-    return -1;
-
-  create_cell_init(cell_out, CELL_CREATE2, handshake_type, handshake_len,
-                   p+4);
-  return 0;
-}
-
-/** Magic string which, in a CREATE or EXTEND cell, indicates that a seeming
- * TAP payload is really an ntor payload.  We'd do away with this if every
- * relay supported EXTEND2, but we want to be able to extend from A to B with
- * ntor even when A doesn't understand EXTEND2 and so can't generate a
- * CREATE2 cell.
- **/
-#define NTOR_CREATE_MAGIC "ntorNTORntorNTOR"
-
-/** Parse a CREATE, CREATE_FAST, or CREATE2 cell from <b>cell_in</b> into
- * <b>cell_out</b>. Return 0 on success, -1 on failure. (We reject some
- * syntactically valid CREATE2 cells that we can't generate or react to.) */
-int
-create_cell_parse(create_cell_t *cell_out, const cell_t *cell_in)
-{
-  switch (cell_in->command) {
-  case CELL_CREATE:
-    if (tor_memeq(cell_in->payload, NTOR_CREATE_MAGIC, 16)) {
-      create_cell_init(cell_out, CELL_CREATE, ONION_HANDSHAKE_TYPE_NTOR,
-                       NTOR_ONIONSKIN_LEN, cell_in->payload+16);
-    } else {
-      create_cell_init(cell_out, CELL_CREATE, ONION_HANDSHAKE_TYPE_TAP,
-                       TAP_ONIONSKIN_CHALLENGE_LEN, cell_in->payload);
-    }
-    break;
-  case CELL_CREATE_FAST:
-    create_cell_init(cell_out, CELL_CREATE_FAST, ONION_HANDSHAKE_TYPE_FAST,
-                     CREATE_FAST_LEN, cell_in->payload);
-    break;
-  case CELL_CREATE2:
-    if (parse_create2_payload(cell_out, cell_in->payload,
-                              CELL_PAYLOAD_SIZE) < 0)
-      return -1;
-    break;
-  default:
-    return -1;
-  }
-
-  return check_create_cell(cell_out, 0);
-}
-
-/** Helper: return 0 if <b>cell</b> appears valid, -1 otherwise. */
-static int
-check_created_cell(const created_cell_t *cell)
-{
-  switch (cell->cell_type) {
-  case CELL_CREATED:
-    if (cell->handshake_len != TAP_ONIONSKIN_REPLY_LEN &&
-        cell->handshake_len != NTOR_REPLY_LEN)
-      return -1;
-    break;
-  case CELL_CREATED_FAST:
-    if (cell->handshake_len != CREATED_FAST_LEN)
-      return -1;
-    break;
-  case CELL_CREATED2:
-    if (cell->handshake_len > RELAY_PAYLOAD_SIZE-2)
-      return -1;
-    break;
-  }
-
-  return 0;
-}
-
-/** Parse a CREATED, CREATED_FAST, or CREATED2 cell from <b>cell_in</b> into
- * <b>cell_out</b>. Return 0 on success, -1 on failure. */
-int
-created_cell_parse(created_cell_t *cell_out, const cell_t *cell_in)
-{
-  memset(cell_out, 0, sizeof(*cell_out));
-
-  switch (cell_in->command) {
-  case CELL_CREATED:
-    cell_out->cell_type = CELL_CREATED;
-    cell_out->handshake_len = TAP_ONIONSKIN_REPLY_LEN;
-    memcpy(cell_out->reply, cell_in->payload, TAP_ONIONSKIN_REPLY_LEN);
-    break;
-  case CELL_CREATED_FAST:
-    cell_out->cell_type = CELL_CREATED_FAST;
-    cell_out->handshake_len = CREATED_FAST_LEN;
-    memcpy(cell_out->reply, cell_in->payload, CREATED_FAST_LEN);
-    break;
-  case CELL_CREATED2:
-    {
-      const uint8_t *p = cell_in->payload;
-      cell_out->cell_type = CELL_CREATED2;
-      cell_out->handshake_len = ntohs(get_uint16(p));
-      if (cell_out->handshake_len > CELL_PAYLOAD_SIZE - 2)
-        return -1;
-      memcpy(cell_out->reply, p+2, cell_out->handshake_len);
-      break;
-    }
-  }
-
-  return check_created_cell(cell_out);
-}
-
-/** Helper: return 0 if <b>cell</b> appears valid, -1 otherwise. */
-static int
-check_extend_cell(const extend_cell_t *cell)
-{
-  if (tor_digest_is_zero((const char*)cell->node_id))
-    return -1;
-  /* We don't currently allow EXTEND2 cells without an IPv4 address */
-  if (tor_addr_family(&cell->orport_ipv4.addr) == AF_UNSPEC)
-    return -1;
-  if (cell->create_cell.cell_type == CELL_CREATE) {
-    if (cell->cell_type != RELAY_COMMAND_EXTEND)
-      return -1;
-  } else if (cell->create_cell.cell_type == CELL_CREATE2) {
-    if (cell->cell_type != RELAY_COMMAND_EXTEND2 &&
-        cell->cell_type != RELAY_COMMAND_EXTEND)
-      return -1;
-  } else {
-    /* In particular, no CREATE_FAST cells are allowed */
-    return -1;
-  }
-  if (cell->create_cell.handshake_type == ONION_HANDSHAKE_TYPE_FAST)
-    return -1;
-
-  return check_create_cell(&cell->create_cell, 1);
-}
-
-/** Protocol constants for specifier types in EXTEND2
- * @{
- */
-#define SPECTYPE_IPV4 0
-#define SPECTYPE_IPV6 1
-#define SPECTYPE_LEGACY_ID 2
-/** @} */
-
-/** Parse an EXTEND or EXTEND2 cell (according to <b>command</b>) from the
- * <b>payload_length</b> bytes of <b>payload</b> into <b>cell_out</b>. Return
- * 0 on success, -1 on failure. */
-int
-extend_cell_parse(extend_cell_t *cell_out, const uint8_t command,
-                  const uint8_t *payload, size_t payload_length)
-{
-  const uint8_t *eop;
-
-  memset(cell_out, 0, sizeof(*cell_out));
-  if (payload_length > RELAY_PAYLOAD_SIZE)
-    return -1;
-  eop = payload + payload_length;
-
-  switch (command) {
-  case RELAY_COMMAND_EXTEND:
-    {
-      if (payload_length != 6 + TAP_ONIONSKIN_CHALLENGE_LEN + DIGEST_LEN)
-        return -1;
-
-      cell_out->cell_type = RELAY_COMMAND_EXTEND;
-      tor_addr_from_ipv4n(&cell_out->orport_ipv4.addr, get_uint32(payload));
-      cell_out->orport_ipv4.port = ntohs(get_uint16(payload+4));
-      tor_addr_make_unspec(&cell_out->orport_ipv6.addr);
-      if (tor_memeq(payload + 6, NTOR_CREATE_MAGIC, 16)) {
-        cell_out->create_cell.cell_type = CELL_CREATE2;
-        cell_out->create_cell.handshake_type = ONION_HANDSHAKE_TYPE_NTOR;
-        cell_out->create_cell.handshake_len = NTOR_ONIONSKIN_LEN;
-        memcpy(cell_out->create_cell.onionskin, payload + 22,
-               NTOR_ONIONSKIN_LEN);
-      } else {
-        cell_out->create_cell.cell_type = CELL_CREATE;
-        cell_out->create_cell.handshake_type = ONION_HANDSHAKE_TYPE_TAP;
-        cell_out->create_cell.handshake_len = TAP_ONIONSKIN_CHALLENGE_LEN;
-        memcpy(cell_out->create_cell.onionskin, payload + 6,
-               TAP_ONIONSKIN_CHALLENGE_LEN);
-      }
-      memcpy(cell_out->node_id, payload + 6 + TAP_ONIONSKIN_CHALLENGE_LEN,
-             DIGEST_LEN);
-      break;
-    }
-  case RELAY_COMMAND_EXTEND2:
-    {
-      uint8_t n_specs, spectype, speclen;
-      int i;
-      int found_ipv4 = 0, found_ipv6 = 0, found_id = 0;
-      tor_addr_make_unspec(&cell_out->orport_ipv4.addr);
-      tor_addr_make_unspec(&cell_out->orport_ipv6.addr);
-
-      if (payload_length == 0)
-        return -1;
-
-      cell_out->cell_type = RELAY_COMMAND_EXTEND2;
-      n_specs = *payload++;
-      /* Parse the specifiers. We'll only take the first IPv4 and first IPv6
-       * address, and the node ID, and ignore everything else */
-      for (i = 0; i < n_specs; ++i) {
-        if (eop - payload < 2)
-          return -1;
-        spectype = payload[0];
-        speclen = payload[1];
-        payload += 2;
-        if (eop - payload < speclen)
-          return -1;
-        switch (spectype) {
-        case SPECTYPE_IPV4:
-          if (speclen != 6)
-            return -1;
-          if (!found_ipv4) {
-            tor_addr_from_ipv4n(&cell_out->orport_ipv4.addr,
-                                get_uint32(payload));
-            cell_out->orport_ipv4.port = ntohs(get_uint16(payload+4));
-            found_ipv4 = 1;
-          }
-          break;
-        case SPECTYPE_IPV6:
-          if (speclen != 18)
-            return -1;
-          if (!found_ipv6) {
-            tor_addr_from_ipv6_bytes(&cell_out->orport_ipv6.addr,
-                                     (const char*)payload);
-            cell_out->orport_ipv6.port = ntohs(get_uint16(payload+16));
-            found_ipv6 = 1;
-          }
-          break;
-        case SPECTYPE_LEGACY_ID:
-          if (speclen != 20)
-            return -1;
-          if (found_id)
-            return -1;
-          memcpy(cell_out->node_id, payload, 20);
-          found_id = 1;
-          break;
-        }
-        payload += speclen;
-      }
-      if (!found_id || !found_ipv4)
-        return -1;
-      if (parse_create2_payload(&cell_out->create_cell,payload,eop-payload)<0)
-        return -1;
-      break;
-    }
-  default:
-    return -1;
-  }
-
-  return check_extend_cell(cell_out);
-}
-
-/** Helper: return 0 if <b>cell</b> appears valid, -1 otherwise. */
-static int
-check_extended_cell(const extended_cell_t *cell)
-{
-  if (cell->created_cell.cell_type == CELL_CREATED) {
-    if (cell->cell_type != RELAY_COMMAND_EXTENDED)
-      return -1;
-  } else if (cell->created_cell.cell_type == CELL_CREATED2) {
-    if (cell->cell_type != RELAY_COMMAND_EXTENDED2)
-      return -1;
-  } else {
-    return -1;
-  }
-
-  return check_created_cell(&cell->created_cell);
-}
-
-/** Parse an EXTENDED or EXTENDED2 cell (according to <b>command</b>) from the
- * <b>payload_length</b> bytes of <b>payload</b> into <b>cell_out</b>. Return
- * 0 on success, -1 on failure. */
-int
-extended_cell_parse(extended_cell_t *cell_out,
-                    const uint8_t command, const uint8_t *payload,
-                    size_t payload_len)
-{
-  memset(cell_out, 0, sizeof(*cell_out));
-  if (payload_len > RELAY_PAYLOAD_SIZE)
-    return -1;
-
-  switch (command) {
-  case RELAY_COMMAND_EXTENDED:
-    if (payload_len != TAP_ONIONSKIN_REPLY_LEN)
-      return -1;
-    cell_out->cell_type = RELAY_COMMAND_EXTENDED;
-    cell_out->created_cell.cell_type = CELL_CREATED;
-    cell_out->created_cell.handshake_len = TAP_ONIONSKIN_REPLY_LEN;
-    memcpy(cell_out->created_cell.reply, payload, TAP_ONIONSKIN_REPLY_LEN);
-    break;
-  case RELAY_COMMAND_EXTENDED2:
-    {
-      cell_out->cell_type = RELAY_COMMAND_EXTENDED2;
-      cell_out->created_cell.cell_type = CELL_CREATED2;
-      cell_out->created_cell.handshake_len = ntohs(get_uint16(payload));
-      if (cell_out->created_cell.handshake_len > RELAY_PAYLOAD_SIZE - 2 ||
-          cell_out->created_cell.handshake_len > payload_len - 2)
-        return -1;
-      memcpy(cell_out->created_cell.reply, payload+2,
-             cell_out->created_cell.handshake_len);
-    }
-    break;
-  default:
-    return -1;
-  }
-
-  return check_extended_cell(cell_out);
-}
-
-/** Fill <b>cell_out</b> with a correctly formatted version of the
- * CREATE{,_FAST,2} cell in <b>cell_in</b>. Return 0 on success, -1 on
- * failure.  This is a cell we didn't originate if <b>relayed</b> is true. */
-static int
-create_cell_format_impl(cell_t *cell_out, const create_cell_t *cell_in,
-                        int relayed)
-{
-  uint8_t *p;
-  size_t space;
-  if (check_create_cell(cell_in, relayed) < 0)
-    return -1;
-
-  memset(cell_out->payload, 0, sizeof(cell_out->payload));
-  cell_out->command = cell_in->cell_type;
-
-  p = cell_out->payload;
-  space = sizeof(cell_out->payload);
-
-  switch (cell_in->cell_type) {
-  case CELL_CREATE:
-    if (cell_in->handshake_type == ONION_HANDSHAKE_TYPE_NTOR) {
-      memcpy(p, NTOR_CREATE_MAGIC, 16);
-      p += 16;
-      space -= 16;
-    }
-    /* Fall through */
-  case CELL_CREATE_FAST:
-    tor_assert(cell_in->handshake_len <= space);
-    memcpy(p, cell_in->onionskin, cell_in->handshake_len);
-    break;
-  case CELL_CREATE2:
-    tor_assert(cell_in->handshake_len <= sizeof(cell_out->payload)-4);
-    set_uint16(cell_out->payload, htons(cell_in->handshake_type));
-    set_uint16(cell_out->payload+2, htons(cell_in->handshake_len));
-    memcpy(cell_out->payload + 4, cell_in->onionskin, cell_in->handshake_len);
-    break;
-  default:
-    return -1;
-  }
-
-  return 0;
-}
-
-int
-create_cell_format(cell_t *cell_out, const create_cell_t *cell_in)
-{
-  return create_cell_format_impl(cell_out, cell_in, 0);
-}
-
-int
-create_cell_format_relayed(cell_t *cell_out, const create_cell_t *cell_in)
-{
-  return create_cell_format_impl(cell_out, cell_in, 1);
-}
-
-/** Fill <b>cell_out</b> with a correctly formatted version of the
- * CREATED{,_FAST,2} cell in <b>cell_in</b>. Return 0 on success, -1 on
- * failure. */
-int
-created_cell_format(cell_t *cell_out, const created_cell_t *cell_in)
-{
-  if (check_created_cell(cell_in) < 0)
-    return -1;
-
-  memset(cell_out->payload, 0, sizeof(cell_out->payload));
-  cell_out->command = cell_in->cell_type;
-
-  switch (cell_in->cell_type) {
-  case CELL_CREATED:
-  case CELL_CREATED_FAST:
-    tor_assert(cell_in->handshake_len <= sizeof(cell_out->payload));
-    memcpy(cell_out->payload, cell_in->reply, cell_in->handshake_len);
-    break;
-  case CELL_CREATED2:
-    tor_assert(cell_in->handshake_len <= sizeof(cell_out->payload)-2);
-    set_uint16(cell_out->payload, htons(cell_in->handshake_len));
-    memcpy(cell_out->payload + 2, cell_in->reply, cell_in->handshake_len);
-    break;
-  default:
-    return -1;
-  }
-  return 0;
-}
-
-/** Format the EXTEND{,2} cell in <b>cell_in</b>, storing its relay payload in
- * <b>payload_out</b>, the number of bytes used in *<b>len_out</b>, and the
- * relay command in *<b>command_out</b>. The <b>payload_out</b> must have
- * RELAY_PAYLOAD_SIZE bytes available.  Return 0 on success, -1 on failure. */
-int
-extend_cell_format(uint8_t *command_out, uint16_t *len_out,
-                   uint8_t *payload_out, const extend_cell_t *cell_in)
-{
-  uint8_t *p, *eop;
-  if (check_extend_cell(cell_in) < 0)
-    return -1;
-
-  p = payload_out;
-  eop = payload_out + RELAY_PAYLOAD_SIZE;
-
-  memset(p, 0, RELAY_PAYLOAD_SIZE);
-
-  switch (cell_in->cell_type) {
-  case RELAY_COMMAND_EXTEND:
-    {
-      *command_out = RELAY_COMMAND_EXTEND;
-      *len_out = 6 + TAP_ONIONSKIN_CHALLENGE_LEN + DIGEST_LEN;
-      set_uint32(p, tor_addr_to_ipv4n(&cell_in->orport_ipv4.addr));
-      set_uint16(p+4, ntohs(cell_in->orport_ipv4.port));
-      if (cell_in->create_cell.handshake_type == ONION_HANDSHAKE_TYPE_NTOR) {
-        memcpy(p+6, NTOR_CREATE_MAGIC, 16);
-        memcpy(p+22, cell_in->create_cell.onionskin, NTOR_ONIONSKIN_LEN);
-      } else {
-        memcpy(p+6, cell_in->create_cell.onionskin,
-               TAP_ONIONSKIN_CHALLENGE_LEN);
-      }
-      memcpy(p+6+TAP_ONIONSKIN_CHALLENGE_LEN, cell_in->node_id, DIGEST_LEN);
-    }
-    break;
-  case RELAY_COMMAND_EXTEND2:
-    {
-      uint8_t n = 2;
-      *command_out = RELAY_COMMAND_EXTEND2;
-
-      *p++ = n; /* 2 identifiers */
-      *p++ = SPECTYPE_IPV4; /* First is IPV4. */
-      *p++ = 6; /* It's 6 bytes long. */
-      set_uint32(p, tor_addr_to_ipv4n(&cell_in->orport_ipv4.addr));
-      set_uint16(p+4, htons(cell_in->orport_ipv4.port));
-      p += 6;
-      *p++ = SPECTYPE_LEGACY_ID; /* Next is an identity digest. */
-      *p++ = 20; /* It's 20 bytes long */
-      memcpy(p, cell_in->node_id, DIGEST_LEN);
-      p += 20;
-
-      /* Now we can send the handshake */
-      set_uint16(p, htons(cell_in->create_cell.handshake_type));
-      set_uint16(p+2, htons(cell_in->create_cell.handshake_len));
-      p += 4;
-
-      if (cell_in->create_cell.handshake_len > eop - p)
-        return -1;
-
-      memcpy(p, cell_in->create_cell.onionskin,
-             cell_in->create_cell.handshake_len);
-
-      p += cell_in->create_cell.handshake_len;
-      *len_out = p - payload_out;
-    }
-    break;
-  default:
-    return -1;
-  }
-
-  return 0;
-}
-
-/** Format the EXTENDED{,2} cell in <b>cell_in</b>, storing its relay payload
- * in <b>payload_out</b>, the number of bytes used in *<b>len_out</b>, and the
- * relay command in *<b>command_out</b>. The <b>payload_out</b> must have
- * RELAY_PAYLOAD_SIZE bytes available.  Return 0 on success, -1 on failure. */
-int
-extended_cell_format(uint8_t *command_out, uint16_t *len_out,
-                     uint8_t *payload_out, const extended_cell_t *cell_in)
-{
-  uint8_t *p;
-  if (check_extended_cell(cell_in) < 0)
-    return -1;
-
-  p = payload_out;
-  memset(p, 0, RELAY_PAYLOAD_SIZE);
-
-  switch (cell_in->cell_type) {
-  case RELAY_COMMAND_EXTENDED:
-    {
-      *command_out = RELAY_COMMAND_EXTENDED;
-      *len_out = TAP_ONIONSKIN_REPLY_LEN;
-      memcpy(payload_out, cell_in->created_cell.reply,
-             TAP_ONIONSKIN_REPLY_LEN);
-    }
-    break;
-  case RELAY_COMMAND_EXTENDED2:
-    {
-      *command_out = RELAY_COMMAND_EXTENDED2;
-      *len_out = 2 + cell_in->created_cell.handshake_len;
-      set_uint16(payload_out, htons(cell_in->created_cell.handshake_len));
-      if (2+cell_in->created_cell.handshake_len > RELAY_PAYLOAD_SIZE)
-        return -1;
-      memcpy(payload_out+2, cell_in->created_cell.reply,
-             cell_in->created_cell.handshake_len);
-    }
-    break;
-  default:
-    return -1;
-  }
-
-  return 0;
-}
-
diff --git a/src/or/or.h b/src/or/or.h
deleted file mode 100644
index 9f53c80644..0000000000
--- a/src/or/or.h
+++ /dev/null
@@ -1,5392 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file or.h
- * \brief Master header file for Tor-specific functionality.
- **/
-
-#ifndef TOR_OR_H
-#define TOR_OR_H
-
-#include "orconfig.h"
-
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SIGNAL_H
-#include <signal.h>
-#endif
-#ifdef HAVE_NETDB_H
-#include <netdb.h>
-#endif
-#ifdef HAVE_SYS_PARAM_H
-#include <sys/param.h> /* FreeBSD needs this to know what version it is */
-#endif
-#include "torint.h"
-#ifdef HAVE_SYS_FCNTL_H
-#include <sys/fcntl.h>
-#endif
-#ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif
-#ifdef HAVE_SYS_IOCTL_H
-#include <sys/ioctl.h>
-#endif
-#ifdef HAVE_SYS_UN_H
-#include <sys/un.h>
-#endif
-#ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
-#endif
-#ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
-#endif
-#ifdef HAVE_ARPA_INET_H
-#include <arpa/inet.h>
-#endif
-#ifdef HAVE_ERRNO_H
-#include <errno.h>
-#endif
-#ifdef HAVE_ASSERT_H
-#include <assert.h>
-#endif
-#ifdef HAVE_TIME_H
-#include <time.h>
-#endif
-
-#ifdef _WIN32
-#include <winsock2.h>
-#include <io.h>
-#include <process.h>
-#include <direct.h>
-#include <windows.h>
-#endif
-
-#include "crypto.h"
-#include "crypto_format.h"
-#include "tortls.h"
-#include "torlog.h"
-#include "container.h"
-#include "torgzip.h"
-#include "address.h"
-#include "compat_libevent.h"
-#include "ht.h"
-#include "replaycache.h"
-#include "crypto_curve25519.h"
-#include "crypto_ed25519.h"
-#include "tor_queue.h"
-#include "util_format.h"
-
-/* These signals are defined to help handle_control_signal work.
- */
-#ifndef SIGHUP
-#define SIGHUP 1
-#endif
-#ifndef SIGINT
-#define SIGINT 2
-#endif
-#ifndef SIGUSR1
-#define SIGUSR1 10
-#endif
-#ifndef SIGUSR2
-#define SIGUSR2 12
-#endif
-#ifndef SIGTERM
-#define SIGTERM 15
-#endif
-/* Controller signals start at a high number so we don't
- * conflict with system-defined signals. */
-#define SIGNEWNYM 129
-#define SIGCLEARDNSCACHE 130
-#define SIGHEARTBEAT 131
-
-#if (SIZEOF_CELL_T != 0)
-/* On Irix, stdlib.h defines a cell_t type, so we need to make sure
- * that our stuff always calls cell_t something different. */
-#define cell_t tor_cell_t
-#endif
-
-#ifdef ENABLE_TOR2WEB_MODE
-#define NON_ANONYMOUS_MODE_ENABLED 1
-#endif
-
-/** Length of longest allowable configured nickname. */
-#define MAX_NICKNAME_LEN 19
-/** Length of a router identity encoded as a hexadecimal digest, plus
- * possible dollar sign. */
-#define MAX_HEX_NICKNAME_LEN (HEX_DIGEST_LEN+1)
-/** Maximum length of verbose router identifier: dollar sign, hex ID digest,
- * equal sign or tilde, nickname. */
-#define MAX_VERBOSE_NICKNAME_LEN (1+HEX_DIGEST_LEN+1+MAX_NICKNAME_LEN)
-
-/** Maximum size, in bytes, for resized buffers. */
-#define MAX_BUF_SIZE ((1<<24)-1) /* 16MB-1 */
-/** Maximum size, in bytes, for any directory object that we've downloaded. */
-#define MAX_DIR_DL_SIZE MAX_BUF_SIZE
-
-/** For HTTP parsing: Maximum number of bytes we'll accept in the headers
- * of an HTTP request or response. */
-#define MAX_HEADERS_SIZE 50000
-/** Maximum size, in bytes, for any directory object that we're accepting
- * as an upload. */
-#define MAX_DIR_UL_SIZE MAX_BUF_SIZE
-
-/** Maximum size, in bytes, of a single router descriptor uploaded to us
- * as a directory authority. Caches and clients fetch whatever descriptors
- * the authorities tell them to fetch, and don't care about size. */
-#define MAX_DESCRIPTOR_UPLOAD_SIZE 20000
-
-/** Maximum size of a single extrainfo document, as above. */
-#define MAX_EXTRAINFO_UPLOAD_SIZE 50000
-
-/** How often do we rotate onion keys? */
-#define MIN_ONION_KEY_LIFETIME (7*24*60*60)
-/** How often do we rotate TLS contexts? */
-#define MAX_SSL_KEY_LIFETIME_INTERNAL (2*60*60)
-
-/** How old do we allow a router to get before removing it
- * from the router list? In seconds. */
-#define ROUTER_MAX_AGE (60*60*48)
-/** How old can a router get before we (as a server) will no longer
- * consider it live? In seconds. */
-#define ROUTER_MAX_AGE_TO_PUBLISH (60*60*24)
-/** How old do we let a saved descriptor get before force-removing it? */
-#define OLD_ROUTER_DESC_MAX_AGE (60*60*24*5)
-
-/** Possible rules for generating circuit IDs on an OR connection. */
-typedef enum {
-  CIRC_ID_TYPE_LOWER=0, /**< Pick from 0..1<<15-1. */
-  CIRC_ID_TYPE_HIGHER=1, /**< Pick from 1<<15..1<<16-1. */
-  /** The other side of a connection is an OP: never create circuits to it,
-   * and let it use any circuit ID it wants. */
-  CIRC_ID_TYPE_NEITHER=2
-} circ_id_type_t;
-#define circ_id_type_bitfield_t ENUM_BF(circ_id_type_t)
-
-#define CONN_TYPE_MIN_ 3
-/** Type for sockets listening for OR connections. */
-#define CONN_TYPE_OR_LISTENER 3
-/** A bidirectional TLS connection transmitting a sequence of cells.
- * May be from an OR to an OR, or from an OP to an OR. */
-#define CONN_TYPE_OR 4
-/** A TCP connection from an onion router to a stream's destination. */
-#define CONN_TYPE_EXIT 5
-/** Type for sockets listening for SOCKS connections. */
-#define CONN_TYPE_AP_LISTENER 6
-/** A SOCKS proxy connection from the user application to the onion
- * proxy. */
-#define CONN_TYPE_AP 7
-/** Type for sockets listening for HTTP connections to the directory server. */
-#define CONN_TYPE_DIR_LISTENER 8
-/** Type for HTTP connections to the directory server. */
-#define CONN_TYPE_DIR 9
-/* Type 10 is unused. */
-/** Type for listening for connections from user interface process. */
-#define CONN_TYPE_CONTROL_LISTENER 11
-/** Type for connections from user interface process. */
-#define CONN_TYPE_CONTROL 12
-/** Type for sockets listening for transparent connections redirected by pf or
- * netfilter. */
-#define CONN_TYPE_AP_TRANS_LISTENER 13
-/** Type for sockets listening for transparent connections redirected by
- * natd. */
-#define CONN_TYPE_AP_NATD_LISTENER 14
-/** Type for sockets listening for DNS requests. */
-#define CONN_TYPE_AP_DNS_LISTENER 15
-
-/** Type for connections from the Extended ORPort. */
-#define CONN_TYPE_EXT_OR 16
-/** Type for sockets listening for Extended ORPort connections. */
-#define CONN_TYPE_EXT_OR_LISTENER 17
-
-#define CONN_TYPE_MAX_ 17
-/* !!!! If _CONN_TYPE_MAX is ever over 31, we must grow the type field in
- * connection_t. */
-
-/* Proxy client types */
-#define PROXY_NONE 0
-#define PROXY_CONNECT 1
-#define PROXY_SOCKS4 2
-#define PROXY_SOCKS5 3
-/* !!!! If there is ever a PROXY_* type over 3, we must grow the proxy_type
- * field in or_connection_t */
-
-/* Pluggable transport proxy type. Don't use this in or_connection_t,
- * instead use the actual underlying proxy type (see above).  */
-#define PROXY_PLUGGABLE 4
-
-/* Proxy client handshake states */
-/* We use a proxy but we haven't even connected to it yet. */
-#define PROXY_INFANT 1
-/* We use an HTTP proxy and we've sent the CONNECT command. */
-#define PROXY_HTTPS_WANT_CONNECT_OK 2
-/* We use a SOCKS4 proxy and we've sent the CONNECT command. */
-#define PROXY_SOCKS4_WANT_CONNECT_OK 3
-/* We use a SOCKS5 proxy and we try to negotiate without
-   any authentication . */
-#define PROXY_SOCKS5_WANT_AUTH_METHOD_NONE 4
-/* We use a SOCKS5 proxy and we try to negotiate with
-   Username/Password authentication . */
-#define PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929 5
-/* We use a SOCKS5 proxy and we just sent our credentials. */
-#define PROXY_SOCKS5_WANT_AUTH_RFC1929_OK 6
-/* We use a SOCKS5 proxy and we just sent our CONNECT command. */
-#define PROXY_SOCKS5_WANT_CONNECT_OK 7
-/* We use a proxy and we CONNECTed successfully!. */
-#define PROXY_CONNECTED 8
-
-/** True iff <b>x</b> is an edge connection. */
-#define CONN_IS_EDGE(x) \
-  ((x)->type == CONN_TYPE_EXIT || (x)->type == CONN_TYPE_AP)
-
-/** State for any listener connection. */
-#define LISTENER_STATE_READY 0
-
-#define OR_CONN_STATE_MIN_ 1
-/** State for a connection to an OR: waiting for connect() to finish. */
-#define OR_CONN_STATE_CONNECTING 1
-/** State for a connection to an OR: waiting for proxy handshake to complete */
-#define OR_CONN_STATE_PROXY_HANDSHAKING 2
-/** State for an OR connection client: SSL is handshaking, not done
- * yet. */
-#define OR_CONN_STATE_TLS_HANDSHAKING 3
-/** State for a connection to an OR: We're doing a second SSL handshake for
- * renegotiation purposes. (V2 handshake only.) */
-#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4
-/** State for a connection at an OR: We're waiting for the client to
- * renegotiate (to indicate a v2 handshake) or send a versions cell (to
- * indicate a v3 handshake) */
-#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5
-/** State for an OR connection: We're done with our SSL handshake, we've done
- * renegotiation, but we haven't yet negotiated link protocol versions and
- * sent a netinfo cell. */
-#define OR_CONN_STATE_OR_HANDSHAKING_V2 6
-/** State for an OR connection: We're done with our SSL handshake, but we
- * haven't yet negotiated link protocol versions, done a V3 handshake, and
- * sent a netinfo cell. */
-#define OR_CONN_STATE_OR_HANDSHAKING_V3 7
-/** State for an OR connection: Ready to send/receive cells. */
-#define OR_CONN_STATE_OPEN 8
-#define OR_CONN_STATE_MAX_ 8
-
-/** States of the Extended ORPort protocol. Be careful before changing
- *  the numbers: they matter. */
-#define EXT_OR_CONN_STATE_MIN_ 1
-/** Extended ORPort authentication is waiting for the authentication
- *  type selected by the client. */
-#define EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE 1
-/** Extended ORPort authentication is waiting for the client nonce. */
-#define EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE 2
-/** Extended ORPort authentication is waiting for the client hash. */
-#define EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH 3
-#define EXT_OR_CONN_STATE_AUTH_MAX 3
-/** Authentication finished and the Extended ORPort is now accepting
- *  traffic. */
-#define EXT_OR_CONN_STATE_OPEN 4
-/** Extended ORPort is flushing its last messages and preparing to
- *  start accepting OR connections. */
-#define EXT_OR_CONN_STATE_FLUSHING 5
-#define EXT_OR_CONN_STATE_MAX_ 5
-
-#define EXIT_CONN_STATE_MIN_ 1
-/** State for an exit connection: waiting for response from DNS farm. */
-#define EXIT_CONN_STATE_RESOLVING 1
-/** State for an exit connection: waiting for connect() to finish. */
-#define EXIT_CONN_STATE_CONNECTING 2
-/** State for an exit connection: open and ready to transmit data. */
-#define EXIT_CONN_STATE_OPEN 3
-/** State for an exit connection: waiting to be removed. */
-#define EXIT_CONN_STATE_RESOLVEFAILED 4
-#define EXIT_CONN_STATE_MAX_ 4
-
-/* The AP state values must be disjoint from the EXIT state values. */
-#define AP_CONN_STATE_MIN_ 5
-/** State for a SOCKS connection: waiting for SOCKS request. */
-#define AP_CONN_STATE_SOCKS_WAIT 5
-/** State for a SOCKS connection: got a y.onion URL; waiting to receive
- * rendezvous descriptor. */
-#define AP_CONN_STATE_RENDDESC_WAIT 6
-/** The controller will attach this connection to a circuit; it isn't our
- * job to do so. */
-#define AP_CONN_STATE_CONTROLLER_WAIT 7
-/** State for a SOCKS connection: waiting for a completed circuit. */
-#define AP_CONN_STATE_CIRCUIT_WAIT 8
-/** State for a SOCKS connection: sent BEGIN, waiting for CONNECTED. */
-#define AP_CONN_STATE_CONNECT_WAIT 9
-/** State for a SOCKS connection: sent RESOLVE, waiting for RESOLVED. */
-#define AP_CONN_STATE_RESOLVE_WAIT 10
-/** State for a SOCKS connection: ready to send and receive. */
-#define AP_CONN_STATE_OPEN 11
-/** State for a transparent natd connection: waiting for original
- * destination. */
-#define AP_CONN_STATE_NATD_WAIT 12
-#define AP_CONN_STATE_MAX_ 12
-
-/** True iff the AP_CONN_STATE_* value <b>s</b> means that the corresponding
- * edge connection is not attached to any circuit. */
-#define AP_CONN_STATE_IS_UNATTACHED(s) \
-  ((s) <= AP_CONN_STATE_CIRCUIT_WAIT || (s) == AP_CONN_STATE_NATD_WAIT)
-
-#define DIR_CONN_STATE_MIN_ 1
-/** State for connection to directory server: waiting for connect(). */
-#define DIR_CONN_STATE_CONNECTING 1
-/** State for connection to directory server: sending HTTP request. */
-#define DIR_CONN_STATE_CLIENT_SENDING 2
-/** State for connection to directory server: reading HTTP response. */
-#define DIR_CONN_STATE_CLIENT_READING 3
-/** State for connection to directory server: happy and finished. */
-#define DIR_CONN_STATE_CLIENT_FINISHED 4
-/** State for connection at directory server: waiting for HTTP request. */
-#define DIR_CONN_STATE_SERVER_COMMAND_WAIT 5
-/** State for connection at directory server: sending HTTP response. */
-#define DIR_CONN_STATE_SERVER_WRITING 6
-#define DIR_CONN_STATE_MAX_ 6
-
-/** True iff the purpose of <b>conn</b> means that it's a server-side
- * directory connection. */
-#define DIR_CONN_IS_SERVER(conn) ((conn)->purpose == DIR_PURPOSE_SERVER)
-
-#define CONTROL_CONN_STATE_MIN_ 1
-/** State for a control connection: Authenticated and accepting v1 commands. */
-#define CONTROL_CONN_STATE_OPEN 1
-/** State for a control connection: Waiting for authentication; speaking
- * protocol v1. */
-#define CONTROL_CONN_STATE_NEEDAUTH 2
-#define CONTROL_CONN_STATE_MAX_ 2
-
-#define DIR_PURPOSE_MIN_ 4
-/** A connection to a directory server: set after a v2 rendezvous
- * descriptor is downloaded. */
-#define DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 4
-/** A connection to a directory server: download one or more server
- * descriptors. */
-#define DIR_PURPOSE_FETCH_SERVERDESC 6
-/** A connection to a directory server: download one or more extra-info
- * documents. */
-#define DIR_PURPOSE_FETCH_EXTRAINFO 7
-/** A connection to a directory server: upload a server descriptor. */
-#define DIR_PURPOSE_UPLOAD_DIR 8
-/** A connection to a directory server: upload a v3 networkstatus vote. */
-#define DIR_PURPOSE_UPLOAD_VOTE 10
-/** A connection to a directory server: upload a v3 consensus signature */
-#define DIR_PURPOSE_UPLOAD_SIGNATURES 11
-/** A connection to a directory server: download one or more v3 networkstatus
- * votes. */
-#define DIR_PURPOSE_FETCH_STATUS_VOTE 12
-/** A connection to a directory server: download a v3 detached signatures
- * object for a consensus. */
-#define DIR_PURPOSE_FETCH_DETACHED_SIGNATURES 13
-/** A connection to a directory server: download a v3 networkstatus
- * consensus. */
-#define DIR_PURPOSE_FETCH_CONSENSUS 14
-/** A connection to a directory server: download one or more directory
- * authority certificates. */
-#define DIR_PURPOSE_FETCH_CERTIFICATE 15
-
-/** Purpose for connection at a directory server. */
-#define DIR_PURPOSE_SERVER 16
-/** A connection to a hidden service directory server: upload a v2 rendezvous
- * descriptor. */
-#define DIR_PURPOSE_UPLOAD_RENDDESC_V2 17
-/** A connection to a hidden service directory server: download a v2 rendezvous
- * descriptor. */
-#define DIR_PURPOSE_FETCH_RENDDESC_V2 18
-/** A connection to a directory server: download a microdescriptor. */
-#define DIR_PURPOSE_FETCH_MICRODESC 19
-#define DIR_PURPOSE_MAX_ 19
-
-/** True iff <b>p</b> is a purpose corresponding to uploading data to a
- * directory server. */
-#define DIR_PURPOSE_IS_UPLOAD(p)                \
-  ((p)==DIR_PURPOSE_UPLOAD_DIR ||               \
-   (p)==DIR_PURPOSE_UPLOAD_VOTE ||              \
-   (p)==DIR_PURPOSE_UPLOAD_SIGNATURES)
-
-#define EXIT_PURPOSE_MIN_ 1
-/** This exit stream wants to do an ordinary connect. */
-#define EXIT_PURPOSE_CONNECT 1
-/** This exit stream wants to do a resolve (either normal or reverse). */
-#define EXIT_PURPOSE_RESOLVE 2
-#define EXIT_PURPOSE_MAX_ 2
-
-/* !!!! If any connection purpose is ever over 31, we must grow the type
- * field in connection_t. */
-
-/** Circuit state: I'm the origin, still haven't done all my handshakes. */
-#define CIRCUIT_STATE_BUILDING 0
-/** Circuit state: Waiting to process the onionskin. */
-#define CIRCUIT_STATE_ONIONSKIN_PENDING 1
-/** Circuit state: I'd like to deliver a create, but my n_chan is still
- * connecting. */
-#define CIRCUIT_STATE_CHAN_WAIT 2
-/** Circuit state: onionskin(s) processed, ready to send/receive cells. */
-#define CIRCUIT_STATE_OPEN 3
-
-#define CIRCUIT_PURPOSE_MIN_ 1
-
-/* these circuits were initiated elsewhere */
-#define CIRCUIT_PURPOSE_OR_MIN_ 1
-/** OR-side circuit purpose: normal circuit, at OR. */
-#define CIRCUIT_PURPOSE_OR 1
-/** OR-side circuit purpose: At OR, from the service, waiting for intro from
- * clients. */
-#define CIRCUIT_PURPOSE_INTRO_POINT 2
-/** OR-side circuit purpose: At OR, from the client, waiting for the service.
- */
-#define CIRCUIT_PURPOSE_REND_POINT_WAITING 3
-/** OR-side circuit purpose: At OR, both circuits have this purpose. */
-#define CIRCUIT_PURPOSE_REND_ESTABLISHED 4
-#define CIRCUIT_PURPOSE_OR_MAX_ 4
-
-/* these circuits originate at this node */
-
-/* here's how circ client-side purposes work:
- *   normal circuits are C_GENERAL.
- *   circuits that are c_introducing are either on their way to
- *     becoming open, or they are open and waiting for a
- *     suitable rendcirc before they send the intro.
- *   circuits that are c_introduce_ack_wait have sent the intro,
- *     but haven't gotten a response yet.
- *   circuits that are c_establish_rend are either on their way
- *     to becoming open, or they are open and have sent the
- *     establish_rendezvous cell but haven't received an ack.
- *   circuits that are c_rend_ready are open and have received a
- *     rend ack, but haven't heard from the service yet. if they have a
- *     buildstate->pending_final_cpath then they're expecting a
- *     cell from the service, else they're not.
- *   circuits that are c_rend_ready_intro_acked are open, and
- *     some intro circ has sent its intro and received an ack.
- *   circuits that are c_rend_joined are open, have heard from
- *     the service, and are talking to it.
- */
-/** Client-side circuit purpose: Normal circuit, with cpath. */
-#define CIRCUIT_PURPOSE_C_GENERAL 5
-/** Client-side circuit purpose: at the client, connecting to intro point. */
-#define CIRCUIT_PURPOSE_C_INTRODUCING 6
-/** Client-side circuit purpose: at the client, sent INTRODUCE1 to intro point,
- * waiting for ACK/NAK. */
-#define CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT 7
-/** Client-side circuit purpose: at the client, introduced and acked, closing.
- */
-#define CIRCUIT_PURPOSE_C_INTRODUCE_ACKED 8
-/** Client-side circuit purpose: at the client, waiting for ack. */
-#define CIRCUIT_PURPOSE_C_ESTABLISH_REND 9
-/** Client-side circuit purpose: at the client, waiting for the service. */
-#define CIRCUIT_PURPOSE_C_REND_READY 10
-/** Client-side circuit purpose: at the client, waiting for the service,
- * INTRODUCE has been acknowledged. */
-#define CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED 11
-/** Client-side circuit purpose: at the client, rendezvous established. */
-#define CIRCUIT_PURPOSE_C_REND_JOINED 12
-/** This circuit is used for build time measurement only */
-#define CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT 13
-#define CIRCUIT_PURPOSE_C_MAX_ 13
-/** Hidden-service-side circuit purpose: at the service, waiting for
- * introductions. */
-#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 14
-/** Hidden-service-side circuit purpose: at the service, successfully
- * established intro. */
-#define CIRCUIT_PURPOSE_S_INTRO 15
-/** Hidden-service-side circuit purpose: at the service, connecting to rend
- * point. */
-#define CIRCUIT_PURPOSE_S_CONNECT_REND 16
-/** Hidden-service-side circuit purpose: at the service, rendezvous
- * established. */
-#define CIRCUIT_PURPOSE_S_REND_JOINED 17
-/** A testing circuit; not meant to be used for actual traffic. */
-#define CIRCUIT_PURPOSE_TESTING 18
-/** A controller made this circuit and Tor should not use it. */
-#define CIRCUIT_PURPOSE_CONTROLLER 19
-/** This circuit is used for path bias probing only */
-#define CIRCUIT_PURPOSE_PATH_BIAS_TESTING 20
-#define CIRCUIT_PURPOSE_MAX_ 20
-/** A catch-all for unrecognized purposes. Currently we don't expect
- * to make or see any circuits with this purpose. */
-#define CIRCUIT_PURPOSE_UNKNOWN 255
-
-/** True iff the circuit purpose <b>p</b> is for a circuit that
- * originated at this node. */
-#define CIRCUIT_PURPOSE_IS_ORIGIN(p) ((p)>CIRCUIT_PURPOSE_OR_MAX_)
-/** True iff the circuit purpose <b>p</b> is for a circuit that originated
- * here to serve as a client.  (Hidden services don't count here.) */
-#define CIRCUIT_PURPOSE_IS_CLIENT(p)  \
-  ((p)> CIRCUIT_PURPOSE_OR_MAX_ &&    \
-   (p)<=CIRCUIT_PURPOSE_C_MAX_)
-/** True iff the circuit_t <b>c</b> is actually an origin_circuit_t. */
-#define CIRCUIT_IS_ORIGIN(c) (CIRCUIT_PURPOSE_IS_ORIGIN((c)->purpose))
-/** True iff the circuit purpose <b>p</b> is for an established rendezvous
- * circuit. */
-#define CIRCUIT_PURPOSE_IS_ESTABLISHED_REND(p) \
-  ((p) == CIRCUIT_PURPOSE_C_REND_JOINED ||     \
-   (p) == CIRCUIT_PURPOSE_S_REND_JOINED)
-/** True iff the circuit_t c is actually an or_circuit_t */
-#define CIRCUIT_IS_ORCIRC(c) (((circuit_t *)(c))->magic == OR_CIRCUIT_MAGIC)
-
-/** How many circuits do we want simultaneously in-progress to handle
- * a given stream? */
-#define MIN_CIRCUITS_HANDLING_STREAM 2
-
-/* These RELAY_COMMAND constants define values for relay cell commands, and
-* must match those defined in tor-spec.txt. */
-#define RELAY_COMMAND_BEGIN 1
-#define RELAY_COMMAND_DATA 2
-#define RELAY_COMMAND_END 3
-#define RELAY_COMMAND_CONNECTED 4
-#define RELAY_COMMAND_SENDME 5
-#define RELAY_COMMAND_EXTEND 6
-#define RELAY_COMMAND_EXTENDED 7
-#define RELAY_COMMAND_TRUNCATE 8
-#define RELAY_COMMAND_TRUNCATED 9
-#define RELAY_COMMAND_DROP 10
-#define RELAY_COMMAND_RESOLVE 11
-#define RELAY_COMMAND_RESOLVED 12
-#define RELAY_COMMAND_BEGIN_DIR 13
-#define RELAY_COMMAND_EXTEND2 14
-#define RELAY_COMMAND_EXTENDED2 15
-
-#define RELAY_COMMAND_ESTABLISH_INTRO 32
-#define RELAY_COMMAND_ESTABLISH_RENDEZVOUS 33
-#define RELAY_COMMAND_INTRODUCE1 34
-#define RELAY_COMMAND_INTRODUCE2 35
-#define RELAY_COMMAND_RENDEZVOUS1 36
-#define RELAY_COMMAND_RENDEZVOUS2 37
-#define RELAY_COMMAND_INTRO_ESTABLISHED 38
-#define RELAY_COMMAND_RENDEZVOUS_ESTABLISHED 39
-#define RELAY_COMMAND_INTRODUCE_ACK 40
-
-/* Reasons why an OR connection is closed. */
-#define END_OR_CONN_REASON_DONE           1
-#define END_OR_CONN_REASON_REFUSED        2 /* connection refused */
-#define END_OR_CONN_REASON_OR_IDENTITY    3
-#define END_OR_CONN_REASON_CONNRESET      4 /* connection reset by peer */
-#define END_OR_CONN_REASON_TIMEOUT        5
-#define END_OR_CONN_REASON_NO_ROUTE       6 /* no route to host/net */
-#define END_OR_CONN_REASON_IO_ERROR       7 /* read/write error */
-#define END_OR_CONN_REASON_RESOURCE_LIMIT 8 /* sockets, buffers, etc */
-#define END_OR_CONN_REASON_PT_MISSING     9 /* PT failed or not available */
-#define END_OR_CONN_REASON_MISC           10
-
-/* Reasons why we (or a remote OR) might close a stream. See tor-spec.txt for
- * documentation of these.  The values must match. */
-#define END_STREAM_REASON_MISC 1
-#define END_STREAM_REASON_RESOLVEFAILED 2
-#define END_STREAM_REASON_CONNECTREFUSED 3
-#define END_STREAM_REASON_EXITPOLICY 4
-#define END_STREAM_REASON_DESTROY 5
-#define END_STREAM_REASON_DONE 6
-#define END_STREAM_REASON_TIMEOUT 7
-#define END_STREAM_REASON_NOROUTE 8
-#define END_STREAM_REASON_HIBERNATING 9
-#define END_STREAM_REASON_INTERNAL 10
-#define END_STREAM_REASON_RESOURCELIMIT 11
-#define END_STREAM_REASON_CONNRESET 12
-#define END_STREAM_REASON_TORPROTOCOL 13
-#define END_STREAM_REASON_NOTDIRECTORY 14
-#define END_STREAM_REASON_ENTRYPOLICY 15
-
-/* These high-numbered end reasons are not part of the official spec,
- * and are not intended to be put in relay end cells. They are here
- * to be more informative when sending back socks replies to the
- * application. */
-/* XXXX 256 is no longer used; feel free to reuse it. */
-/** We were unable to attach the connection to any circuit at all. */
-/* XXXX the ways we use this one don't make a lot of sense. */
-#define END_STREAM_REASON_CANT_ATTACH 257
-/** We can't connect to any directories at all, so we killed our streams
- * before they can time out. */
-#define END_STREAM_REASON_NET_UNREACHABLE 258
-/** This is a SOCKS connection, and the client used (or misused) the SOCKS
- * protocol in a way we couldn't handle. */
-#define END_STREAM_REASON_SOCKSPROTOCOL 259
-/** This is a transparent proxy connection, but we can't extract the original
- * target address:port. */
-#define END_STREAM_REASON_CANT_FETCH_ORIG_DEST 260
-/** This is a connection on the NATD port, and the destination IP:Port was
- * either ill-formed or out-of-range. */
-#define END_STREAM_REASON_INVALID_NATD_DEST 261
-/** The target address is in a private network (like 127.0.0.1 or 10.0.0.1);
- * you don't want to do that over a randomly chosen exit */
-#define END_STREAM_REASON_PRIVATE_ADDR 262
-
-/** Bitwise-and this value with endreason to mask out all flags. */
-#define END_STREAM_REASON_MASK 511
-
-/** Bitwise-or this with the argument to control_event_stream_status
- * to indicate that the reason came from an END cell. */
-#define END_STREAM_REASON_FLAG_REMOTE 512
-/** Bitwise-or this with the argument to control_event_stream_status
- * to indicate that we already sent a CLOSED stream event. */
-#define END_STREAM_REASON_FLAG_ALREADY_SENT_CLOSED 1024
-/** Bitwise-or this with endreason to indicate that we already sent
- * a socks reply, and no further reply needs to be sent from
- * connection_mark_unattached_ap(). */
-#define END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED 2048
-
-/** Reason for remapping an AP connection's address: we have a cached
- * answer. */
-#define REMAP_STREAM_SOURCE_CACHE 1
-/** Reason for remapping an AP connection's address: the exit node told us an
- * answer. */
-#define REMAP_STREAM_SOURCE_EXIT 2
-
-/* 'type' values to use in RESOLVED cells.  Specified in tor-spec.txt. */
-#define RESOLVED_TYPE_HOSTNAME 0
-#define RESOLVED_TYPE_IPV4 4
-#define RESOLVED_TYPE_IPV6 6
-#define RESOLVED_TYPE_ERROR_TRANSIENT 0xF0
-#define RESOLVED_TYPE_ERROR 0xF1
-
-/* Negative reasons are internal: we never send them in a DESTROY or TRUNCATE
- * call; they only go to the controller for tracking  */
-
-/* Closing introduction point that were opened in parallel. */
-#define END_CIRC_REASON_IP_NOW_REDUNDANT -4
-
-/** Our post-timeout circuit time measurement period expired.
- * We must give up now */
-#define END_CIRC_REASON_MEASUREMENT_EXPIRED -3
-
-/** We couldn't build a path for this circuit. */
-#define END_CIRC_REASON_NOPATH          -2
-/** Catch-all "other" reason for closing origin circuits. */
-#define END_CIRC_AT_ORIGIN              -1
-
-/* Reasons why we (or a remote OR) might close a circuit. See tor-spec.txt for
- * documentation of these. */
-#define END_CIRC_REASON_MIN_            0
-#define END_CIRC_REASON_NONE            0
-#define END_CIRC_REASON_TORPROTOCOL     1
-#define END_CIRC_REASON_INTERNAL        2
-#define END_CIRC_REASON_REQUESTED       3
-#define END_CIRC_REASON_HIBERNATING     4
-#define END_CIRC_REASON_RESOURCELIMIT   5
-#define END_CIRC_REASON_CONNECTFAILED   6
-#define END_CIRC_REASON_OR_IDENTITY     7
-#define END_CIRC_REASON_CHANNEL_CLOSED  8
-#define END_CIRC_REASON_FINISHED        9
-#define END_CIRC_REASON_TIMEOUT         10
-#define END_CIRC_REASON_DESTROYED       11
-#define END_CIRC_REASON_NOSUCHSERVICE   12
-#define END_CIRC_REASON_MAX_            12
-
-/** Bitwise-OR this with the argument to circuit_mark_for_close() or
- * control_event_circuit_status() to indicate that the reason was
- * passed through from a destroy or truncate cell. */
-#define END_CIRC_REASON_FLAG_REMOTE     512
-
-/** Length of 'y' portion of 'y.onion' URL. */
-#define REND_SERVICE_ID_LEN_BASE32 16
-
-/** Length of 'y.onion' including '.onion' URL. */
-#define REND_SERVICE_ADDRESS_LEN (16+1+5)
-
-/** Length of a binary-encoded rendezvous service ID. */
-#define REND_SERVICE_ID_LEN 10
-
-/** Time period for which a v2 descriptor will be valid. */
-#define REND_TIME_PERIOD_V2_DESC_VALIDITY (24*60*60)
-
-/** Time period within which two sets of v2 descriptors will be uploaded in
- * parallel. */
-#define REND_TIME_PERIOD_OVERLAPPING_V2_DESCS (60*60)
-
-/** Number of non-consecutive replicas (i.e. distributed somewhere
- * in the ring) for a descriptor. */
-#define REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS 2
-
-/** Number of consecutive replicas for a descriptor. */
-#define REND_NUMBER_OF_CONSECUTIVE_REPLICAS 3
-
-/** Length of v2 descriptor ID (32 base32 chars = 160 bits). */
-#define REND_DESC_ID_V2_LEN_BASE32 32
-
-/** Length of the base32-encoded secret ID part of versioned hidden service
- * descriptors. */
-#define REND_SECRET_ID_PART_LEN_BASE32 32
-
-/** Length of the base32-encoded hash of an introduction point's
- * identity key. */
-#define REND_INTRO_POINT_ID_LEN_BASE32 32
-
-/** Length of the descriptor cookie that is used for client authorization
- * to hidden services. */
-#define REND_DESC_COOKIE_LEN 16
-
-/** Length of the base64-encoded descriptor cookie that is used for
- * exchanging client authorization between hidden service and client. */
-#define REND_DESC_COOKIE_LEN_BASE64 22
-
-/** Length of client identifier in encrypted introduction points for hidden
- * service authorization type 'basic'. */
-#define REND_BASIC_AUTH_CLIENT_ID_LEN 4
-
-/** Multiple of the number of clients to which the real number of clients
- * is padded with fake clients for hidden service authorization type
- * 'basic'. */
-#define REND_BASIC_AUTH_CLIENT_MULTIPLE 16
-
-/** Length of client entry consisting of client identifier and encrypted
- * session key for hidden service authorization type 'basic'. */
-#define REND_BASIC_AUTH_CLIENT_ENTRY_LEN (REND_BASIC_AUTH_CLIENT_ID_LEN \
-                                          + CIPHER_KEY_LEN)
-
-/** Maximum size of v2 hidden service descriptors. */
-#define REND_DESC_MAX_SIZE (20 * 1024)
-
-/** Legal characters for use in authorized client names for a hidden
- * service. */
-#define REND_LEGAL_CLIENTNAME_CHARACTERS \
-  "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+-_"
-
-/** Maximum length of authorized client names for a hidden service. */
-#define REND_CLIENTNAME_MAX_LEN 16
-
-/** Length of the rendezvous cookie that is used to connect circuits at the
- * rendezvous point. */
-#define REND_COOKIE_LEN DIGEST_LEN
-
-/** Client authorization type that a hidden service performs. */
-typedef enum rend_auth_type_t {
-  REND_NO_AUTH      = 0,
-  REND_BASIC_AUTH   = 1,
-  REND_STEALTH_AUTH = 2,
-} rend_auth_type_t;
-
-/** Client-side configuration of authorization for a hidden service. */
-typedef struct rend_service_authorization_t {
-  uint8_t descriptor_cookie[REND_DESC_COOKIE_LEN];
-  char onion_address[REND_SERVICE_ADDRESS_LEN+1];
-  rend_auth_type_t auth_type;
-} rend_service_authorization_t;
-
-/** Client- and server-side data that is used for hidden service connection
- * establishment. Not all fields contain data depending on where this struct
- * is used. */
-typedef struct rend_data_t {
-  /** Onion address (without the .onion part) that a client requests. */
-  char onion_address[REND_SERVICE_ID_LEN_BASE32+1];
-
-  /** Descriptor ID for each replicas computed from the onion address. If
-   * the onion address is empty, this array MUST be empty. We keep them so
-   * we know when to purge our entry in the last hsdir request table. */
-  char descriptor_id[REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS][DIGEST_LEN];
-
-  /** (Optional) descriptor cookie that is used by a client. */
-  char descriptor_cookie[REND_DESC_COOKIE_LEN];
-
-  /** Authorization type for accessing a service used by a client. */
-  rend_auth_type_t auth_type;
-
-  /** Descriptor ID for a client request. The control port command HSFETCH
-   * uses this. It's set if the descriptor query should only use this
-   * descriptor ID. */
-  char desc_id_fetch[DIGEST_LEN];
-
-  /** Hash of the hidden service's PK used by a service. */
-  char rend_pk_digest[DIGEST_LEN];
-
-  /** Rendezvous cookie used by both, client and service. */
-  char rend_cookie[REND_COOKIE_LEN];
-
-  /** List of HSDir fingerprints on which this request has been sent to.
-   * This contains binary identity digest of the directory. */
-  smartlist_t *hsdirs_fp;
-
-  /** Number of streams associated with this rendezvous circuit. */
-  int nr_streams;
-} rend_data_t;
-
-/** Time interval for tracking replays of DH public keys received in
- * INTRODUCE2 cells.  Used only to avoid launching multiple
- * simultaneous attempts to connect to the same rendezvous point. */
-#define REND_REPLAY_TIME_INTERVAL (5 * 60)
-
-/** Used to indicate which way a cell is going on a circuit. */
-typedef enum {
-  CELL_DIRECTION_IN=1, /**< The cell is moving towards the origin. */
-  CELL_DIRECTION_OUT=2, /**< The cell is moving away from the origin. */
-} cell_direction_t;
-
-/** Initial value for both sides of a circuit transmission window when the
- * circuit is initialized.  Measured in cells. */
-#define CIRCWINDOW_START 1000
-#define CIRCWINDOW_START_MIN 100
-#define CIRCWINDOW_START_MAX 1000
-/** Amount to increment a circuit window when we get a circuit SENDME. */
-#define CIRCWINDOW_INCREMENT 100
-/** Initial value on both sides of a stream transmission window when the
- * stream is initialized.  Measured in cells. */
-#define STREAMWINDOW_START 500
-/** Amount to increment a stream window when we get a stream SENDME. */
-#define STREAMWINDOW_INCREMENT 50
-
-/** Maximum number of queued cells on a circuit for which we are the
- * midpoint before we give up and kill it.  This must be >= circwindow
- * to avoid killing innocent circuits, and >= circwindow*2 to give
- * leaky-pipe a chance of working someday. The ORCIRC_MAX_MIDDLE_KILL_THRESH
- * ratio controls the margin of error between emitting a warning and
- * killing the circuit.
- */
-#define ORCIRC_MAX_MIDDLE_CELLS (CIRCWINDOW_START_MAX*2)
-/** Ratio of hard (circuit kill) to soft (warning) thresholds for the
- * ORCIRC_MAX_MIDDLE_CELLS tests.
- */
-#define ORCIRC_MAX_MIDDLE_KILL_THRESH (1.1f)
-
-/* Cell commands.  These values are defined in tor-spec.txt. */
-#define CELL_PADDING 0
-#define CELL_CREATE 1
-#define CELL_CREATED 2
-#define CELL_RELAY 3
-#define CELL_DESTROY 4
-#define CELL_CREATE_FAST 5
-#define CELL_CREATED_FAST 6
-#define CELL_VERSIONS 7
-#define CELL_NETINFO 8
-#define CELL_RELAY_EARLY 9
-#define CELL_CREATE2 10
-#define CELL_CREATED2 11
-
-#define CELL_VPADDING 128
-#define CELL_CERTS 129
-#define CELL_AUTH_CHALLENGE 130
-#define CELL_AUTHENTICATE 131
-#define CELL_AUTHORIZE 132
-#define CELL_COMMAND_MAX_ 132
-
-/** How long to test reachability before complaining to the user. */
-#define TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT (20*60)
-
-/** Legal characters in a nickname. */
-#define LEGAL_NICKNAME_CHARACTERS \
-  "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
-
-/** Name to use in client TLS certificates if no nickname is given. Once
- * Tor 0.1.2.x is obsolete, we can remove this. */
-#define DEFAULT_CLIENT_NICKNAME "client"
-
-/** Name chosen by routers that don't configure nicknames */
-#define UNNAMED_ROUTER_NICKNAME "Unnamed"
-
-/** Number of bytes in a SOCKS4 header. */
-#define SOCKS4_NETWORK_LEN 8
-
-/*
- * Relay payload:
- *         Relay command           [1 byte]
- *         Recognized              [2 bytes]
- *         Stream ID               [2 bytes]
- *         Partial SHA-1           [4 bytes]
- *         Length                  [2 bytes]
- *         Relay payload           [498 bytes]
- */
-
-/** Number of bytes in a cell, minus cell header. */
-#define CELL_PAYLOAD_SIZE 509
-/** Number of bytes in a cell transmitted over the network, in the longest
- * form */
-#define CELL_MAX_NETWORK_SIZE 514
-
-/** Maximum length of a header on a variable-length cell. */
-#define VAR_CELL_MAX_HEADER_SIZE 7
-
-static int get_cell_network_size(int wide_circ_ids);
-static inline int get_cell_network_size(int wide_circ_ids)
-{
-  return wide_circ_ids ? CELL_MAX_NETWORK_SIZE : CELL_MAX_NETWORK_SIZE - 2;
-}
-static int get_var_cell_header_size(int wide_circ_ids);
-static inline int get_var_cell_header_size(int wide_circ_ids)
-{
-  return wide_circ_ids ? VAR_CELL_MAX_HEADER_SIZE :
-    VAR_CELL_MAX_HEADER_SIZE - 2;
-}
-static int get_circ_id_size(int wide_circ_ids);
-static inline int get_circ_id_size(int wide_circ_ids)
-{
-  return wide_circ_ids ? 4 : 2;
-}
-
-/** Number of bytes in a relay cell's header (not including general cell
- * header). */
-#define RELAY_HEADER_SIZE (1+2+2+4+2)
-/** Largest number of bytes that can fit in a relay cell payload. */
-#define RELAY_PAYLOAD_SIZE (CELL_PAYLOAD_SIZE-RELAY_HEADER_SIZE)
-
-/** Identifies a circuit on an or_connection */
-typedef uint32_t circid_t;
-/** Identifies a stream on a circuit */
-typedef uint16_t streamid_t;
-
-/* channel_t typedef; struct channel_s is in channel.h */
-
-typedef struct channel_s channel_t;
-
-/* channel_listener_t typedef; struct channel_listener_s is in channel.h */
-
-typedef struct channel_listener_s channel_listener_t;
-
-/* channel states for channel_t */
-
-typedef enum {
-  /*
-   * Closed state - channel is inactive
-   *
-   * Permitted transitions from:
-   *   - CHANNEL_STATE_CLOSING
-   * Permitted transitions to:
-   *   - CHANNEL_STATE_OPENING
-   */
-  CHANNEL_STATE_CLOSED = 0,
-  /*
-   * Opening state - channel is trying to connect
-   *
-   * Permitted transitions from:
-   *   - CHANNEL_STATE_CLOSED
-   * Permitted transitions to:
-   *   - CHANNEL_STATE_CLOSING
-   *   - CHANNEL_STATE_ERROR
-   *   - CHANNEL_STATE_OPEN
-   */
-  CHANNEL_STATE_OPENING,
-  /*
-   * Open state - channel is active and ready for use
-   *
-   * Permitted transitions from:
-   *   - CHANNEL_STATE_MAINT
-   *   - CHANNEL_STATE_OPENING
-   * Permitted transitions to:
-   *   - CHANNEL_STATE_CLOSING
-   *   - CHANNEL_STATE_ERROR
-   *   - CHANNEL_STATE_MAINT
-   */
-  CHANNEL_STATE_OPEN,
-  /*
-   * Maintenance state - channel is temporarily offline for subclass specific
-   *   maintenance activities such as TLS renegotiation.
-   *
-   * Permitted transitions from:
-   *   - CHANNEL_STATE_OPEN
-   * Permitted transitions to:
-   *   - CHANNEL_STATE_CLOSING
-   *   - CHANNEL_STATE_ERROR
-   *   - CHANNEL_STATE_OPEN
-   */
-  CHANNEL_STATE_MAINT,
-  /*
-   * Closing state - channel is shutting down
-   *
-   * Permitted transitions from:
-   *   - CHANNEL_STATE_MAINT
-   *   - CHANNEL_STATE_OPEN
-   * Permitted transitions to:
-   *   - CHANNEL_STATE_CLOSED,
-   *   - CHANNEL_STATE_ERROR
-   */
-  CHANNEL_STATE_CLOSING,
-  /*
-   * Error state - channel has experienced a permanent error
-   *
-   * Permitted transitions from:
-   *   - CHANNEL_STATE_CLOSING
-   *   - CHANNEL_STATE_MAINT
-   *   - CHANNEL_STATE_OPENING
-   *   - CHANNEL_STATE_OPEN
-   * Permitted transitions to:
-   *   - None
-   */
-  CHANNEL_STATE_ERROR,
-  /*
-   * Placeholder for maximum state value
-   */
-  CHANNEL_STATE_LAST
-} channel_state_t;
-
-/* channel listener states for channel_listener_t */
-
-typedef enum {
-  /*
-   * Closed state - channel listener is inactive
-   *
-   * Permitted transitions from:
-   *   - CHANNEL_LISTENER_STATE_CLOSING
-   * Permitted transitions to:
-   *   - CHANNEL_LISTENER_STATE_LISTENING
-   */
-  CHANNEL_LISTENER_STATE_CLOSED = 0,
-  /*
-   * Listening state - channel listener is listening for incoming
-   * connections
-   *
-   * Permitted transitions from:
-   *   - CHANNEL_LISTENER_STATE_CLOSED
-   * Permitted transitions to:
-   *   - CHANNEL_LISTENER_STATE_CLOSING
-   *   - CHANNEL_LISTENER_STATE_ERROR
-   */
-  CHANNEL_LISTENER_STATE_LISTENING,
-  /*
-   * Closing state - channel listener is shutting down
-   *
-   * Permitted transitions from:
-   *   - CHANNEL_LISTENER_STATE_LISTENING
-   * Permitted transitions to:
-   *   - CHANNEL_LISTENER_STATE_CLOSED,
-   *   - CHANNEL_LISTENER_STATE_ERROR
-   */
-  CHANNEL_LISTENER_STATE_CLOSING,
-  /*
-   * Error state - channel listener has experienced a permanent error
-   *
-   * Permitted transitions from:
-   *   - CHANNEL_STATE_CLOSING
-   *   - CHANNEL_STATE_LISTENING
-   * Permitted transitions to:
-   *   - None
-   */
-  CHANNEL_LISTENER_STATE_ERROR,
-  /*
-   * Placeholder for maximum state value
-   */
-  CHANNEL_LISTENER_STATE_LAST
-} channel_listener_state_t;
-
-/* TLS channel stuff */
-
-typedef struct channel_tls_s channel_tls_t;
-
-/* circuitmux_t typedef; struct circuitmux_s is in circuitmux.h */
-
-typedef struct circuitmux_s circuitmux_t;
-
-/** Parsed onion routing cell.  All communication between nodes
- * is via cells. */
-typedef struct cell_t {
-  circid_t circ_id; /**< Circuit which received the cell. */
-  uint8_t command; /**< Type of the cell: one of CELL_PADDING, CELL_CREATE,
-                    * CELL_DESTROY, etc */
-  uint8_t payload[CELL_PAYLOAD_SIZE]; /**< Cell body. */
-} cell_t;
-
-/** Parsed variable-length onion routing cell. */
-typedef struct var_cell_t {
-  /** Type of the cell: CELL_VERSIONS, etc. */
-  uint8_t command;
-  /** Circuit thich received the cell */
-  circid_t circ_id;
-  /** Number of bytes actually stored in <b>payload</b> */
-  uint16_t payload_len;
-  /** Payload of this cell */
-  uint8_t payload[FLEXIBLE_ARRAY_MEMBER];
-} var_cell_t;
-
-/** A parsed Extended ORPort message. */
-typedef struct ext_or_cmd_t {
-  uint16_t cmd; /** Command type */
-  uint16_t len; /** Body length */
-  char body[FLEXIBLE_ARRAY_MEMBER]; /** Message body */
-} ext_or_cmd_t;
-
-/** A cell as packed for writing to the network. */
-typedef struct packed_cell_t {
-  /** Next cell queued on this circuit. */
-  TOR_SIMPLEQ_ENTRY(packed_cell_t) next;
-  char body[CELL_MAX_NETWORK_SIZE]; /**< Cell as packed for network. */
-  uint32_t inserted_time; /**< Time (in milliseconds since epoch, with high
-                           * bits truncated) when this cell was inserted. */
-} packed_cell_t;
-
-/** A queue of cells on a circuit, waiting to be added to the
- * or_connection_t's outbuf. */
-typedef struct cell_queue_t {
-  /** Linked list of packed_cell_t*/
-  TOR_SIMPLEQ_HEAD(cell_simpleq, packed_cell_t) head;
-  int n; /**< The number of cells in the queue. */
-} cell_queue_t;
-
-/** A single queued destroy cell. */
-typedef struct destroy_cell_t {
-  TOR_SIMPLEQ_ENTRY(destroy_cell_t) next;
-  circid_t circid;
-  uint32_t inserted_time; /** Timestamp when this was queued. */
-  uint8_t reason;
-} destroy_cell_t;
-
-/** A queue of destroy cells on a channel. */
-typedef struct destroy_cell_queue_t {
-  /** Linked list of packed_cell_t */
-  TOR_SIMPLEQ_HEAD(dcell_simpleq, destroy_cell_t) head;
-  int n; /**< The number of cells in the queue. */
-} destroy_cell_queue_t;
-
-/** Beginning of a RELAY cell payload. */
-typedef struct {
-  uint8_t command; /**< The end-to-end relay command. */
-  uint16_t recognized; /**< Used to tell whether cell is for us. */
-  streamid_t stream_id; /**< Which stream is this cell associated with? */
-  char integrity[4]; /**< Used to tell whether cell is corrupted. */
-  uint16_t length; /**< How long is the payload body? */
-} relay_header_t;
-
-typedef struct buf_t buf_t;
-typedef struct socks_request_t socks_request_t;
-
-#define buf_t buf_t
-
-typedef struct entry_port_cfg_t {
-  /* Client port types (socks, dns, trans, natd) only: */
-  uint8_t isolation_flags; /**< Zero or more isolation flags */
-  int session_group; /**< A session group, or -1 if this port is not in a
-                      * session group. */
-
-  /* Socks only: */
-  /** When both no-auth and user/pass are advertised by a SOCKS client, select
-   * no-auth. */
-  unsigned int socks_prefer_no_auth : 1;
-  /** When ISO_SOCKSAUTH is in use, Keep-Alive circuits indefinitely. */
-  unsigned int socks_iso_keep_alive : 1;
-
-  /* Client port types only: */
-  unsigned int ipv4_traffic : 1;
-  unsigned int ipv6_traffic : 1;
-  unsigned int prefer_ipv6 : 1;
-  unsigned int dns_request : 1;
-  unsigned int onion_traffic : 1;
-
-  /** For a socks listener: should we cache IPv4/IPv6 DNS information that
-   * exit nodes tell us?
-   *
-   * @{ */
-  unsigned int cache_ipv4_answers : 1;
-  unsigned int cache_ipv6_answers : 1;
-  /** @} */
-  /** For a socks listeners: if we find an answer in our client-side DNS cache,
-   * should we use it?
-   *
-   * @{ */
-  unsigned int use_cached_ipv4_answers : 1;
-  unsigned int use_cached_ipv6_answers : 1;
-  /** @} */
-  /** For socks listeners: When we can automap an address to IPv4 or IPv6,
-   * do we prefer IPv6? */
-  unsigned int prefer_ipv6_virtaddr : 1;
-
-} entry_port_cfg_t;
-
-typedef struct server_port_cfg_t {
-  /* Server port types (or, dir) only: */
-  unsigned int no_advertise : 1;
-  unsigned int no_listen : 1;
-  unsigned int all_addrs : 1;
-  unsigned int bind_ipv4_only : 1;
-  unsigned int bind_ipv6_only : 1;
-} server_port_cfg_t;
-
-/* Values for connection_t.magic: used to make sure that downcasts (casts from
-* connection_t to foo_connection_t) are safe. */
-#define BASE_CONNECTION_MAGIC 0x7C3C304Eu
-#define OR_CONNECTION_MAGIC 0x7D31FF03u
-#define EDGE_CONNECTION_MAGIC 0xF0374013u
-#define ENTRY_CONNECTION_MAGIC 0xbb4a5703
-#define DIR_CONNECTION_MAGIC 0x9988ffeeu
-#define CONTROL_CONNECTION_MAGIC 0x8abc765du
-#define LISTENER_CONNECTION_MAGIC 0x1a1ac741u
-
-/** Description of a connection to another host or process, and associated
- * data.
- *
- * A connection is named based on what it's connected to -- an "OR
- * connection" has a Tor node on the other end, an "exit
- * connection" has a website or other server on the other end, and an
- * "AP connection" has an application proxy (and thus a user) on the
- * other end.
- *
- * Every connection has a type and a state.  Connections never change
- * their type, but can go through many state changes in their lifetime.
- *
- * Every connection has two associated input and output buffers.
- * Listeners don't use them.  For non-listener connections, incoming
- * data is appended to conn->inbuf, and outgoing data is taken from
- * conn->outbuf.  Connections differ primarily in the functions called
- * to fill and drain these buffers.
- */
-typedef struct connection_t {
-  uint32_t magic; /**< For memory debugging: must equal one of
-                   * *_CONNECTION_MAGIC. */
-
-  uint8_t state; /**< Current state of this connection. */
-  unsigned int type:5; /**< What kind of connection is this? */
-  unsigned int purpose:5; /**< Only used for DIR and EXIT types currently. */
-
-  /* The next fields are all one-bit booleans. Some are only applicable to
-   * connection subtypes, but we hold them here anyway, to save space.
-   */
-  unsigned int read_blocked_on_bw:1; /**< Boolean: should we start reading
-                            * again once the bandwidth throttler allows it? */
-  unsigned int write_blocked_on_bw:1; /**< Boolean: should we start writing
-                             * again once the bandwidth throttler allows
-                             * writes? */
-  unsigned int hold_open_until_flushed:1; /**< Despite this connection's being
-                                      * marked for close, do we flush it
-                                      * before closing it? */
-  unsigned int inbuf_reached_eof:1; /**< Boolean: did read() return 0 on this
-                                     * conn? */
-  /** Set to 1 when we're inside connection_flushed_some to keep us from
-   * calling connection_handle_write() recursively. */
-  unsigned int in_flushed_some:1;
-  /** True if connection_handle_write is currently running on this connection.
-   */
-  unsigned int in_connection_handle_write:1;
-
-  /* For linked connections:
-   */
-  unsigned int linked:1; /**< True if there is, or has been, a linked_conn. */
-  /** True iff we'd like to be notified about read events from the
-   * linked conn. */
-  unsigned int reading_from_linked_conn:1;
-  /** True iff we're willing to write to the linked conn. */
-  unsigned int writing_to_linked_conn:1;
-  /** True iff we're currently able to read on the linked conn, and our
-   * read_event should be made active with libevent. */
-  unsigned int active_on_link:1;
-  /** True iff we've called connection_close_immediate() on this linked
-   * connection. */
-  unsigned int linked_conn_is_closed:1;
-
-  /** CONNECT/SOCKS proxy client handshake state (for outgoing connections). */
-  unsigned int proxy_state:4;
-
-  /** Our socket; set to TOR_INVALID_SOCKET if this connection is closed,
-   * or has no socket. */
-  tor_socket_t s;
-  int conn_array_index; /**< Index into the global connection array. */
-
-  struct event *read_event; /**< Libevent event structure. */
-  struct event *write_event; /**< Libevent event structure. */
-  buf_t *inbuf; /**< Buffer holding data read over this connection. */
-  buf_t *outbuf; /**< Buffer holding data to write over this connection. */
-  size_t outbuf_flushlen; /**< How much data should we try to flush from the
-                           * outbuf? */
-  time_t timestamp_lastread; /**< When was the last time libevent said we could
-                              * read? */
-  time_t timestamp_lastwritten; /**< When was the last time libevent said we
-                                 * could write? */
-
-  time_t timestamp_created; /**< When was this connection_t created? */
-
-  int socket_family; /**< Address family of this connection's socket.  Usually
-                      * AF_INET, but it can also be AF_UNIX, or AF_INET6 */
-  tor_addr_t addr; /**< IP that socket "s" is directly connected to;
-                    * may be the IP address for a proxy or pluggable transport,
-                    * see "address" for the address of the final destination.
-                    */
-  uint16_t port; /**< If non-zero, port that socket "s" is directly connected
-                  * to; may be the port for a proxy or pluggable transport,
-                  * see "address" for the port at the final destination. */
-  uint16_t marked_for_close; /**< Should we close this conn on the next
-                              * iteration of the main loop? (If true, holds
-                              * the line number where this connection was
-                              * marked.) */
-  const char *marked_for_close_file; /**< For debugging: in which file were
-                                      * we marked for close? */
-  char *address; /**< FQDN (or IP) and port of the final destination for this
-                  * connection; this is always the remote address, it is
-                  * passed to a proxy or pluggable transport if one in use.
-                  * See "addr" and "port" for the address that socket "s" is
-                  * directly connected to.
-                  * strdup into this, because free_connection() frees it. */
-  /** Another connection that's connected to this one in lieu of a socket. */
-  struct connection_t *linked_conn;
-
-  /** Unique identifier for this connection on this Tor instance. */
-  uint64_t global_identifier;
-
-  /** Bytes read since last call to control_event_conn_bandwidth_used().
-   * Only used if we're configured to emit CONN_BW events. */
-  uint32_t n_read_conn_bw;
-
-  /** Bytes written since last call to control_event_conn_bandwidth_used().
-   * Only used if we're configured to emit CONN_BW events. */
-  uint32_t n_written_conn_bw;
-} connection_t;
-
-/** Subtype of connection_t; used for a listener socket. */
-typedef struct listener_connection_t {
-  connection_t base_;
-
-  /** If the connection is a CONN_TYPE_AP_DNS_LISTENER, this field points
-   * to the evdns_server_port it uses to listen to and answer connections. */
-  struct evdns_server_port *dns_server_port;
-
-  entry_port_cfg_t entry_cfg;
-
-} listener_connection_t;
-
-/** Minimum length of the random part of an AUTH_CHALLENGE cell. */
-#define OR_AUTH_CHALLENGE_LEN 32
-
-/**
- * @name Certificate types for CERTS cells.
- *
- * These values are defined by the protocol, and affect how an X509
- * certificate in a CERTS cell is interpreted and used.
- *
- * @{ */
-/** A certificate that authenticates a TLS link key.  The subject key
- * must match the key used in the TLS handshake; it must be signed by
- * the identity key. */
-#define OR_CERT_TYPE_TLS_LINK 1
-/** A self-signed identity certificate. The subject key must be a
- * 1024-bit RSA key. */
-#define OR_CERT_TYPE_ID_1024 2
-/** A certificate that authenticates a key used in an AUTHENTICATE cell
- * in the v3 handshake.  The subject key must be a 1024-bit RSA key; it
- * must be signed by the identity key */
-#define OR_CERT_TYPE_AUTH_1024 3
-/* DOCDOC */
-#define OR_CERT_TYPE_RSA_ED_CROSSCERT 7
-/**@}*/
-
-/** The one currently supported type of AUTHENTICATE cell.  It contains
- * a bunch of structures signed with an RSA1024 key.  The signed
- * structures include a HMAC using negotiated TLS secrets, and a digest
- * of all cells sent or received before the AUTHENTICATE cell (including
- * the random server-generated AUTH_CHALLENGE cell).
- */
-#define AUTHTYPE_RSA_SHA256_TLSSECRET 1
-
-/** The length of the part of the AUTHENTICATE cell body that the client and
- * server can generate independently (when using RSA_SHA256_TLSSECRET). It
- * contains everything except the client's timestamp, the client's randomly
- * generated nonce, and the signature. */
-#define V3_AUTH_FIXED_PART_LEN (8+(32*6))
-/** The length of the part of the AUTHENTICATE cell body that the client
- * signs. */
-#define V3_AUTH_BODY_LEN (V3_AUTH_FIXED_PART_LEN + 8 + 16)
-
-/** Stores flags and information related to the portion of a v2/v3 Tor OR
- * connection handshake that happens after the TLS handshake is finished.
- */
-typedef struct or_handshake_state_t {
-  /** When was the VERSIONS cell sent on this connection?  Used to get
-   * an estimate of the skew in the returning NETINFO reply. */
-  time_t sent_versions_at;
-  /** True iff we originated this connection */
-  unsigned int started_here : 1;
-  /** True iff we have received and processed a VERSIONS cell. */
-  unsigned int received_versions : 1;
-  /** True iff we have received and processed an AUTH_CHALLENGE cell */
-  unsigned int received_auth_challenge : 1;
-  /** True iff we have received and processed a CERTS cell. */
-  unsigned int received_certs_cell : 1;
-  /** True iff we have received and processed an AUTHENTICATE cell */
-  unsigned int received_authenticate : 1;
-
-  /* True iff we've received valid authentication to some identity. */
-  unsigned int authenticated : 1;
-
-  /* True iff we have sent a netinfo cell */
-  unsigned int sent_netinfo : 1;
-
-  /** True iff we should feed outgoing cells into digest_sent and
-   * digest_received respectively.
-   *
-   * From the server's side of the v3 handshake, we want to capture everything
-   * from the VERSIONS cell through and including the AUTH_CHALLENGE cell.
-   * From the client's, we want to capture everything from the VERSIONS cell
-   * through but *not* including the AUTHENTICATE cell.
-   *
-   * @{ */
-  unsigned int digest_sent_data : 1;
-  unsigned int digest_received_data : 1;
-  /**@}*/
-
-  /** Identity digest that we have received and authenticated for our peer
-   * on this connection. */
-  uint8_t authenticated_peer_id[DIGEST_LEN];
-
-  /** Digests of the cells that we have sent or received as part of a V3
-   * handshake.  Used for making and checking AUTHENTICATE cells.
-   *
-   * @{
-   */
-  crypto_digest_t *digest_sent;
-  crypto_digest_t *digest_received;
-  /** @} */
-
-  /** Certificates that a connection initiator sent us in a CERTS cell; we're
-   * holding on to them until we get an AUTHENTICATE cell.
-   *
-   * @{
-   */
-  /** The cert for the key that's supposed to sign the AUTHENTICATE cell */
-  tor_x509_cert_t *auth_cert;
-  /** A self-signed identity certificate */
-  tor_x509_cert_t *id_cert;
-  /**@}*/
-} or_handshake_state_t;
-
-/** Length of Extended ORPort connection identifier. */
-#define EXT_OR_CONN_ID_LEN DIGEST_LEN /* 20 */
-/*
- * OR_CONN_HIGHWATER and OR_CONN_LOWWATER moved from connection_or.c so
- * channeltls.c can see them too.
- */
-
-/** When adding cells to an OR connection's outbuf, keep adding until the
- * outbuf is at least this long, or we run out of cells. */
-#define OR_CONN_HIGHWATER (32*1024)
-
-/** Add cells to an OR connection's outbuf whenever the outbuf's data length
- * drops below this size. */
-#define OR_CONN_LOWWATER (16*1024)
-
-/** Subtype of connection_t for an "OR connection" -- that is, one that speaks
- * cells over TLS. */
-typedef struct or_connection_t {
-  connection_t base_;
-
-  /** Hash of the public RSA key for the other side's identity key, or zeroes
-   * if the other side hasn't shown us a valid identity key. */
-  char identity_digest[DIGEST_LEN];
-
-  /** Extended ORPort connection identifier. */
-  char *ext_or_conn_id;
-  /** This is the ClientHash value we expect to receive from the
-   *  client during the Extended ORPort authentication protocol. We
-   *  compute it upon receiving the ClientNoce from the client, and we
-   *  compare it with the acual ClientHash value sent by the
-   *  client. */
-  char *ext_or_auth_correct_client_hash;
-  /** String carrying the name of the pluggable transport
-   *  (e.g. "obfs2") that is obfuscating this connection. If no
-   *  pluggable transports are used, it's NULL. */
-  char *ext_or_transport;
-
-  char *nickname; /**< Nickname of OR on other side (if any). */
-
-  tor_tls_t *tls; /**< TLS connection state. */
-  int tls_error; /**< Last tor_tls error code. */
-  /** When we last used this conn for any client traffic. If not
-   * recent, we can rate limit it further. */
-
-  /* Channel using this connection */
-  channel_tls_t *chan;
-
-  tor_addr_t real_addr; /**< The actual address that this connection came from
-                       * or went to.  The <b>addr</b> field is prone to
-                       * getting overridden by the address from the router
-                       * descriptor matching <b>identity_digest</b>. */
-
-  /** Should this connection be used for extending circuits to the server
-   * matching the <b>identity_digest</b> field?  Set to true if we're pretty
-   * sure we aren't getting MITMed, either because we're connected to an
-   * address listed in a server descriptor, or because an authenticated
-   * NETINFO cell listed the address we're connected to as recognized. */
-  unsigned int is_canonical:1;
-
-  /** True iff we have decided that the other end of this connection
-   * is a client.  Connections with this flag set should never be used
-   * to satisfy an EXTEND request.  */
-  unsigned int is_connection_with_client:1;
-  /** True iff this is an outgoing connection. */
-  unsigned int is_outgoing:1;
-  unsigned int proxy_type:2; /**< One of PROXY_NONE...PROXY_SOCKS5 */
-  unsigned int wide_circ_ids:1;
-  /** True iff this connection has had its bootstrap failure logged with
-   * control_event_bootstrap_problem. */
-  unsigned int have_noted_bootstrap_problem:1;
-  /** True iff this is a client connection and its address has been put in the
-   * geoip cache and handled by the DoS mitigation subsystem. We use this to
-   * insure we have a coherent count of concurrent connection. */
-  unsigned int tracked_for_dos_mitigation : 1;
-
-  uint16_t link_proto; /**< What protocol version are we using? 0 for
-                        * "none negotiated yet." */
-  uint16_t idle_timeout; /**< How long can this connection sit with no
-                          * circuits on it before we close it? Based on
-                          * IDLE_CIRCUIT_TIMEOUT_{NON,}CANONICAL and
-                          * on is_canonical, randomized. */
-  or_handshake_state_t *handshake_state; /**< If we are setting this connection
-                                          * up, state information to do so. */
-
-  time_t timestamp_lastempty; /**< When was the outbuf last completely empty?*/
-
-  /* bandwidth* and *_bucket only used by ORs in OPEN state: */
-  int bandwidthrate; /**< Bytes/s added to the bucket. (OPEN ORs only.) */
-  int bandwidthburst; /**< Max bucket size for this conn. (OPEN ORs only.) */
-  int read_bucket; /**< When this hits 0, stop receiving. Every second we
-                    * add 'bandwidthrate' to this, capping it at
-                    * bandwidthburst. (OPEN ORs only) */
-  int write_bucket; /**< When this hits 0, stop writing. Like read_bucket. */
-
-  struct or_connection_t *next_with_same_id; /**< Next connection with same
-                                              * identity digest as this one. */
-  /** Last emptied read token bucket in msec since midnight; only used if
-   * TB_EMPTY events are enabled. */
-  uint32_t read_emptied_time;
-  /** Last emptied write token bucket in msec since midnight; only used if
-   * TB_EMPTY events are enabled. */
-  uint32_t write_emptied_time;
-
-  /*
-   * Count the number of bytes flushed out on this orconn, and the number of
-   * bytes TLS actually sent - used for overhead estimation for scheduling.
-   */
-  uint64_t bytes_xmitted, bytes_xmitted_by_tls;
-} or_connection_t;
-
-/** Subtype of connection_t for an "edge connection" -- that is, an entry (ap)
- * connection, or an exit. */
-typedef struct edge_connection_t {
-  connection_t base_;
-
-  struct edge_connection_t *next_stream; /**< Points to the next stream at this
-                                          * edge, if any */
-  int package_window; /**< How many more relay cells can I send into the
-                       * circuit? */
-  int deliver_window; /**< How many more relay cells can end at me? */
-
-  struct circuit_t *on_circuit; /**< The circuit (if any) that this edge
-                                 * connection is using. */
-
-  /** A pointer to which node in the circ this conn exits at.  Set for AP
-   * connections and for hidden service exit connections. */
-  struct crypt_path_t *cpath_layer;
-  /** What rendezvous service are we querying for (if an AP) or providing (if
-   * an exit)? */
-  rend_data_t *rend_data;
-
-  uint32_t address_ttl; /**< TTL for address-to-addr mapping on exit
-                         * connection.  Exit connections only. */
-  uint32_t begincell_flags; /** Flags sent or received in the BEGIN cell
-                             * for this connection */
-
-  streamid_t stream_id; /**< The stream ID used for this edge connection on its
-                         * circuit */
-
-  /** The reason why this connection is closing; passed to the controller. */
-  uint16_t end_reason;
-
-  /** Bytes read since last call to control_event_stream_bandwidth_used() */
-  uint32_t n_read;
-
-  /** Bytes written since last call to control_event_stream_bandwidth_used() */
-  uint32_t n_written;
-
-  /** True iff this connection is for a DNS request only. */
-  unsigned int is_dns_request:1;
-  /** True iff this connection is for a PTR DNS request. (exit only) */
-  unsigned int is_reverse_dns_lookup:1;
-
-  unsigned int edge_has_sent_end:1; /**< For debugging; only used on edge
-                         * connections.  Set once we've set the stream end,
-                         * and check in connection_about_to_close_connection().
-                         */
-  /** True iff we've blocked reading until the circuit has fewer queued
-   * cells. */
-  unsigned int edge_blocked_on_circ:1;
-
-  /** Unique ID for directory requests; this used to be in connection_t, but
-   * that's going away and being used on channels instead.  We still tag
-   * edge connections with dirreq_id from circuits, so it's copied here. */
-  uint64_t dirreq_id;
-} edge_connection_t;
-
-/** Subtype of edge_connection_t for an "entry connection" -- that is, a SOCKS
- * connection, a DNS request, a TransPort connection or a NATD connection */
-typedef struct entry_connection_t {
-  edge_connection_t edge_;
-
-  /** Nickname of planned exit node -- used with .exit support. */
-  char *chosen_exit_name;
-
-  socks_request_t *socks_request; /**< SOCKS structure describing request (AP
-                                   * only.) */
-
-  /* === Isolation related, AP only. === */
-  entry_port_cfg_t entry_cfg;
-  /** AP only: The newnym epoch in which we created this connection. */
-  unsigned nym_epoch;
-
-  /** AP only: The original requested address before we rewrote it. */
-  char *original_dest_address;
-  /* Other fields to isolate on already exist.  The ClientAddr is addr.  The
-     ClientProtocol is a combination of type and socks_request->
-     socks_version.  SocksAuth is socks_request->username/password.
-     DestAddr is in socks_request->address. */
-
-  /** Number of times we've reassigned this application connection to
-   * a new circuit. We keep track because the timeout is longer if we've
-   * already retried several times. */
-  uint8_t num_socks_retries;
-
-  /** For AP connections only: buffer for data that we have sent
-   * optimistically, which we might need to re-send if we have to
-   * retry this connection. */
-  buf_t *pending_optimistic_data;
-  /* For AP connections only: buffer for data that we previously sent
-  * optimistically which we are currently re-sending as we retry this
-  * connection. */
-  buf_t *sending_optimistic_data;
-
-  /** If this is a DNSPort connection, this field holds the pending DNS
-   * request that we're going to try to answer.  */
-  struct evdns_server_request *dns_server_request;
-
-#define DEBUGGING_17659
-
-#ifdef DEBUGGING_17659
-  uint16_t marked_pending_circ_line;
-  const char *marked_pending_circ_file;
-#endif
-
-#define NUM_CIRCUITS_LAUNCHED_THRESHOLD 10
-  /** Number of times we've launched a circuit to handle this stream. If
-    * it gets too high, that could indicate an inconsistency between our
-    * "launch a circuit to handle this stream" logic and our "attach our
-    * stream to one of the available circuits" logic. */
-  unsigned int num_circuits_launched:4;
-
-  /** True iff this stream must attach to a one-hop circuit (e.g. for
-   * begin_dir). */
-  unsigned int want_onehop:1;
-  /** True iff this stream should use a BEGIN_DIR relay command to establish
-   * itself rather than BEGIN (either via onehop or via a whole circuit). */
-  unsigned int use_begindir:1;
-
-  /** For AP connections only. If 1, and we fail to reach the chosen exit,
-   * stop requiring it. */
-  unsigned int chosen_exit_optional:1;
-  /** For AP connections only. If non-zero, this exit node was picked as
-   * a result of the TrackHostExit, and the value decrements every time
-   * we fail to complete a circuit to our chosen exit -- if it reaches
-   * zero, abandon the associated mapaddress. */
-  unsigned int chosen_exit_retries:3;
-
-  /** True iff this is an AP connection that came from a transparent or
-   * NATd connection */
-  unsigned int is_transparent_ap:1;
-
-  /** For AP connections only: Set if this connection's target exit node
-   * allows optimistic data (that is, data sent on this stream before
-   * the exit has sent a CONNECTED cell) and we have chosen to use it.
-   */
-  unsigned int may_use_optimistic_data : 1;
-
-  /** Are we a socks SocksSocket listener? */
-  unsigned int is_socks_socket:1;
-} entry_connection_t;
-
-typedef enum {
-    DIR_SPOOL_NONE=0, DIR_SPOOL_SERVER_BY_DIGEST, DIR_SPOOL_SERVER_BY_FP,
-    DIR_SPOOL_EXTRA_BY_DIGEST, DIR_SPOOL_EXTRA_BY_FP,
-    DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS,
-    DIR_SPOOL_MICRODESC, /* NOTE: if we add another entry, add another bit. */
-} dir_spool_source_t;
-#define dir_spool_source_bitfield_t ENUM_BF(dir_spool_source_t)
-
-/** Subtype of connection_t for an "directory connection" -- that is, an HTTP
- * connection to retrieve or serve directory material. */
-typedef struct dir_connection_t {
-  connection_t base_;
-
- /** Which 'resource' did we ask the directory for? This is typically the part
-  * of the URL string that defines, relative to the directory conn purpose,
-  * what thing we want.  For example, in router descriptor downloads by
-  * descriptor digest, it contains "d/", then one ore more +-separated
-  * fingerprints.
-  **/
-  char *requested_resource;
-  unsigned int dirconn_direct:1; /**< Is this dirconn direct, or via Tor? */
-
-  /* Used only for server sides of some dir connections, to implement
-   * "spooling" of directory material to the outbuf.  Otherwise, we'd have
-   * to append everything to the outbuf in one enormous chunk. */
-  /** What exactly are we spooling right now? */
-  dir_spool_source_bitfield_t  dir_spool_src : 3;
-
-  /** If we're fetching descriptors, what router purpose shall we assign
-   * to them? */
-  uint8_t router_purpose;
-  /** List of fingerprints for networkstatuses or descriptors to be spooled. */
-  smartlist_t *fingerprint_stack;
-  /** A cached_dir_t object that we're currently spooling out */
-  struct cached_dir_t *cached_dir;
-  /** The current offset into cached_dir. */
-  off_t cached_dir_offset;
-  /** The zlib object doing on-the-fly compression for spooled data. */
-  tor_zlib_state_t *zlib_state;
-
-  /** What rendezvous service are we querying for? */
-  rend_data_t *rend_data;
-
-  char identity_digest[DIGEST_LEN]; /**< Hash of the public RSA key for
-                                     * the directory server's signing key. */
-
-  /** Unique ID for directory requests; this used to be in connection_t, but
-   * that's going away and being used on channels instead.  The dirserver still
-   * needs this for the incoming side, so it's moved here. */
-  uint64_t dirreq_id;
-} dir_connection_t;
-
-/** Subtype of connection_t for an connection to a controller. */
-typedef struct control_connection_t {
-  connection_t base_;
-
-  uint64_t event_mask; /**< Bitfield: which events does this controller
-                        * care about?
-                        * EVENT_MAX_ is >31, so we need a 64 bit mask */
-
-  /** True if we have sent a protocolinfo reply on this connection. */
-  unsigned int have_sent_protocolinfo:1;
-  /** True if we have received a takeownership command on this
-   * connection. */
-  unsigned int is_owning_control_connection:1;
-
-  /** List of ephemeral onion services belonging to this connection. */
-  smartlist_t *ephemeral_onion_services;
-
-  /** If we have sent an AUTHCHALLENGE reply on this connection and
-   * have not received a successful AUTHENTICATE command, points to
-   * the value which the client must send to authenticate itself;
-   * otherwise, NULL. */
-  char *safecookie_client_hash;
-
-  /** Amount of space allocated in incoming_cmd. */
-  uint32_t incoming_cmd_len;
-  /** Number of bytes currently stored in incoming_cmd. */
-  uint32_t incoming_cmd_cur_len;
-  /** A control command that we're reading from the inbuf, but which has not
-   * yet arrived completely. */
-  char *incoming_cmd;
-} control_connection_t;
-
-/** Cast a connection_t subtype pointer to a connection_t **/
-#define TO_CONN(c) (&(((c)->base_)))
-/** Helper macro: Given a pointer to to.base_, of type from*, return &to. */
-#define DOWNCAST(to, ptr) ((to*)SUBTYPE_P(ptr, to, base_))
-
-/** Cast a entry_connection_t subtype pointer to a edge_connection_t **/
-#define ENTRY_TO_EDGE_CONN(c) (&(((c))->edge_))
-/** Cast a entry_connection_t subtype pointer to a connection_t **/
-#define ENTRY_TO_CONN(c) (TO_CONN(ENTRY_TO_EDGE_CONN(c)))
-
-/** Convert a connection_t* to an or_connection_t*; assert if the cast is
- * invalid. */
-static or_connection_t *TO_OR_CONN(connection_t *);
-/** Convert a connection_t* to a dir_connection_t*; assert if the cast is
- * invalid. */
-static dir_connection_t *TO_DIR_CONN(connection_t *);
-/** Convert a connection_t* to an edge_connection_t*; assert if the cast is
- * invalid. */
-static edge_connection_t *TO_EDGE_CONN(connection_t *);
-/** Convert a connection_t* to an entry_connection_t*; assert if the cast is
- * invalid. */
-static entry_connection_t *TO_ENTRY_CONN(connection_t *);
-/** Convert a edge_connection_t* to an entry_connection_t*; assert if the cast
- * is invalid. */
-static entry_connection_t *EDGE_TO_ENTRY_CONN(edge_connection_t *);
-/** Convert a connection_t* to an control_connection_t*; assert if the cast is
- * invalid. */
-static control_connection_t *TO_CONTROL_CONN(connection_t *);
-/** Convert a connection_t* to an listener_connection_t*; assert if the cast is
- * invalid. */
-static listener_connection_t *TO_LISTENER_CONN(connection_t *);
-
-static inline or_connection_t *TO_OR_CONN(connection_t *c)
-{
-  tor_assert(c->magic == OR_CONNECTION_MAGIC);
-  return DOWNCAST(or_connection_t, c);
-}
-static inline dir_connection_t *TO_DIR_CONN(connection_t *c)
-{
-  tor_assert(c->magic == DIR_CONNECTION_MAGIC);
-  return DOWNCAST(dir_connection_t, c);
-}
-static inline edge_connection_t *TO_EDGE_CONN(connection_t *c)
-{
-  tor_assert(c->magic == EDGE_CONNECTION_MAGIC ||
-             c->magic == ENTRY_CONNECTION_MAGIC);
-  return DOWNCAST(edge_connection_t, c);
-}
-static inline entry_connection_t *TO_ENTRY_CONN(connection_t *c)
-{
-  tor_assert(c->magic == ENTRY_CONNECTION_MAGIC);
-  return (entry_connection_t*) SUBTYPE_P(c, entry_connection_t, edge_.base_);
-}
-static inline entry_connection_t *EDGE_TO_ENTRY_CONN(edge_connection_t *c)
-{
-  tor_assert(c->base_.magic == ENTRY_CONNECTION_MAGIC);
-  return (entry_connection_t*) SUBTYPE_P(c, entry_connection_t, edge_);
-}
-static inline control_connection_t *TO_CONTROL_CONN(connection_t *c)
-{
-  tor_assert(c->magic == CONTROL_CONNECTION_MAGIC);
-  return DOWNCAST(control_connection_t, c);
-}
-static inline listener_connection_t *TO_LISTENER_CONN(connection_t *c)
-{
-  tor_assert(c->magic == LISTENER_CONNECTION_MAGIC);
-  return DOWNCAST(listener_connection_t, c);
-}
-
-/** What action type does an address policy indicate: accept or reject? */
-typedef enum {
-  ADDR_POLICY_ACCEPT=1,
-  ADDR_POLICY_REJECT=2,
-} addr_policy_action_t;
-#define addr_policy_action_bitfield_t ENUM_BF(addr_policy_action_t)
-
-/** A reference-counted address policy rule. */
-typedef struct addr_policy_t {
-  int refcnt; /**< Reference count */
-  /** What to do when the policy matches.*/
-  addr_policy_action_bitfield_t policy_type:2;
-  unsigned int is_private:1; /**< True iff this is the pseudo-address,
-                              * "private". */
-  unsigned int is_canonical:1; /**< True iff this policy is the canonical
-                                * copy (stored in a hash table to avoid
-                                * duplication of common policies) */
-  maskbits_t maskbits; /**< Accept/reject all addresses <b>a</b> such that the
-                 * first <b>maskbits</b> bits of <b>a</b> match
-                 * <b>addr</b>. */
-  /** Base address to accept or reject.
-   *
-   * Note that wildcards are treated
-   * differntly depending on address family. An AF_UNSPEC address means
-   * "All addresses, IPv4 or IPv6." An AF_INET address with maskbits==0 means
-   * "All IPv4 addresses" and an AF_INET6 address with maskbits == 0 means
-   * "All IPv6 addresses".
-  **/
-  tor_addr_t addr;
-  uint16_t prt_min; /**< Lowest port number to accept/reject. */
-  uint16_t prt_max; /**< Highest port number to accept/reject. */
-} addr_policy_t;
-
-/** A cached_dir_t represents a cacheable directory object, along with its
- * compressed form. */
-typedef struct cached_dir_t {
-  char *dir; /**< Contents of this object, NUL-terminated. */
-  char *dir_z; /**< Compressed contents of this object. */
-  size_t dir_len; /**< Length of <b>dir</b> (not counting its NUL). */
-  size_t dir_z_len; /**< Length of <b>dir_z</b>. */
-  time_t published; /**< When was this object published. */
-  common_digests_t digests; /**< Digests of this object (networkstatus only) */
-  int refcnt; /**< Reference count for this cached_dir_t. */
-} cached_dir_t;
-
-/** Enum used to remember where a signed_descriptor_t is stored and how to
- * manage the memory for signed_descriptor_body.  */
-typedef enum {
-  /** The descriptor isn't stored on disk at all: the copy in memory is
-   * canonical; the saved_offset field is meaningless. */
-  SAVED_NOWHERE=0,
-  /** The descriptor is stored in the cached_routers file: the
-   * signed_descriptor_body is meaningless; the signed_descriptor_len and
-   * saved_offset are used to index into the mmaped cache file. */
-  SAVED_IN_CACHE,
-  /** The descriptor is stored in the cached_routers.new file: the
-   * signed_descriptor_body and saved_offset fields are both set. */
-  /* FFFF (We could also mmap the file and grow the mmap as needed, or
-   * lazy-load the descriptor text by using seek and read.  We don't, for
-   * now.)
-   */
-  SAVED_IN_JOURNAL
-} saved_location_t;
-#define saved_location_bitfield_t ENUM_BF(saved_location_t)
-
-/** Enumeration: what directory object is being downloaded?
- * This determines which schedule is selected to perform the download. */
-typedef enum {
-  DL_SCHED_GENERIC = 0,
-  DL_SCHED_CONSENSUS = 1,
-  DL_SCHED_BRIDGE = 2,
-} download_schedule_t;
-#define download_schedule_bitfield_t ENUM_BF(download_schedule_t)
-
-/** Enumeration: is the download schedule for downloading from an authority,
- * or from any available directory mirror?
- * During bootstrap, "any" means a fallback (or an authority, if there
- * are no fallbacks).
- * When we have a valid consensus, "any" means any directory server. */
-typedef enum {
-  DL_WANT_ANY_DIRSERVER = 0,
-  DL_WANT_AUTHORITY = 1,
-} download_want_authority_t;
-#define download_want_authority_bitfield_t \
-                                        ENUM_BF(download_want_authority_t)
-
-/** Enumeration: do we want to increment the schedule position each time a
- * connection is attempted (these attempts can be concurrent), or do we want
- * to increment the schedule position after a connection fails? */
-typedef enum {
-  DL_SCHED_INCREMENT_FAILURE = 0,
-  DL_SCHED_INCREMENT_ATTEMPT = 1,
-} download_schedule_increment_t;
-#define download_schedule_increment_bitfield_t \
-                                        ENUM_BF(download_schedule_increment_t)
-
-/** Enumeration: do we want to use the random exponential backoff
- * mechanism? */
-typedef enum {
-  DL_SCHED_DETERMINISTIC = 0,
-  DL_SCHED_RANDOM_EXPONENTIAL = 1,
-} download_schedule_backoff_t;
-#define download_schedule_backoff_bitfield_t \
-                                        ENUM_BF(download_schedule_backoff_t)
-
-/** Information about our plans for retrying downloads for a downloadable
- * directory object.
- * Each type of downloadable directory object has a corresponding retry
- * <b>schedule</b>, which can be different depending on whether the object is
- * being downloaded from an authority or a mirror (<b>want_authority</b>).
- * <b>next_attempt_at</b> contains the next time we will attempt to download
- * the object.
- * For schedules that <b>increment_on</b> failure, <b>n_download_failures</b>
- * is used to determine the position in the schedule. (Each schedule is a
- * smartlist of integer delays, parsed from a CSV option.) Every time a
- * connection attempt fails, <b>n_download_failures</b> is incremented,
- * the new delay value is looked up from the schedule, and
- * <b>next_attempt_at</b> is set delay seconds from the time the previous
- * connection failed. Therefore, at most one failure-based connection can be
- * in progress for each download_status_t.
- * For schedules that <b>increment_on</b> attempt, <b>n_download_attempts</b>
- * is used to determine the position in the schedule. Every time a
- * connection attempt is made, <b>n_download_attempts</b> is incremented,
- * the new delay value is looked up from the schedule, and
- * <b>next_attempt_at</b> is set delay seconds from the time the previous
- * connection was attempted. Therefore, multiple concurrent attempted-based
- * connections can be in progress for each download_status_t.
- * After an object is successfully downloaded, any other concurrent connections
- * are terminated. A new schedule which starts at position 0 is used for
- * subsequent downloads of the same object.
- */
-typedef struct download_status_t {
-  time_t next_attempt_at; /**< When should we try downloading this object
-                           * again? */
-  uint8_t n_download_failures; /**< Number of failed downloads of the most
-                                * recent object, since the last success. */
-  uint8_t n_download_attempts; /**< Number of (potentially concurrent) attempts
-                                * to download the most recent object, since
-                                * the last success. */
-  download_schedule_bitfield_t schedule : 8; /**< What kind of object is being
-                                              * downloaded? This determines the
-                                              * schedule used for the download.
-                                              */
-  download_want_authority_bitfield_t want_authority : 1; /**< Is the download
-                                              * happening from an authority
-                                              * or a mirror? This determines
-                                              * the schedule used for the
-                                              * download. */
-  download_schedule_increment_bitfield_t increment_on : 1; /**< does this
-                                        * schedule increment on each attempt,
-                                        * or after each failure? */
-  download_schedule_backoff_bitfield_t backoff : 1; /**< do we use the
-                                        * deterministic schedule, or random
-                                        * exponential backoffs? */
-  uint8_t last_backoff_position; /**< number of attempts/failures, depending
-                                  * on increment_on, when we last recalculated
-                                  * the delay.  Only updated if backoff
-                                  * == 1. */
-  int last_delay_used; /**< last delay used for random exponential backoff;
-                        * only updated if backoff == 1 */
-} download_status_t;
-
-/** If n_download_failures is this high, the download can never happen. */
-#define IMPOSSIBLE_TO_DOWNLOAD 255
-
-/** The max size we expect router descriptor annotations we create to
- * be. We'll accept larger ones if we see them on disk, but we won't
- * create any that are larger than this. */
-#define ROUTER_ANNOTATION_BUF_LEN 256
-
-/** Information need to cache an onion router's descriptor. */
-typedef struct signed_descriptor_t {
-  /** Pointer to the raw server descriptor, preceded by annotations.  Not
-   * necessarily NUL-terminated.  If saved_location is SAVED_IN_CACHE, this
-   * pointer is null. */
-  char *signed_descriptor_body;
-  /** Length of the annotations preceding the server descriptor. */
-  size_t annotations_len;
-  /** Length of the server descriptor. */
-  size_t signed_descriptor_len;
-  /** Digest of the server descriptor, computed as specified in
-   * dir-spec.txt. */
-  char signed_descriptor_digest[DIGEST_LEN];
-  /** Identity digest of the router. */
-  char identity_digest[DIGEST_LEN];
-  /** Declared publication time of the descriptor. */
-  time_t published_on;
-  /** For routerdescs only: digest of the corresponding extrainfo. */
-  char extra_info_digest[DIGEST_LEN];
-  /** For routerdescs only: A SHA256-digest of the extrainfo (if any) */
-  char extra_info_digest256[DIGEST256_LEN];
-  /** Certificate for ed25519 signing key. */
-  struct tor_cert_st *signing_key_cert;
-  /** For routerdescs only: Status of downloading the corresponding
-   * extrainfo. */
-  download_status_t ei_dl_status;
-  /** Where is the descriptor saved? */
-  saved_location_t saved_location;
-  /** If saved_location is SAVED_IN_CACHE or SAVED_IN_JOURNAL, the offset of
-   * this descriptor in the corresponding file. */
-  off_t saved_offset;
-  /** What position is this descriptor within routerlist->routers or
-   * routerlist->old_routers? -1 for none. */
-  int routerlist_index;
-  /** The valid-until time of the most recent consensus that listed this
-   * descriptor.  0 for "never listed in a consensus, so far as we know." */
-  time_t last_listed_as_valid_until;
-  /* If true, we do not ever try to save this object in the cache. */
-  unsigned int do_not_cache : 1;
-  /* If true, this item is meant to represent an extrainfo. */
-  unsigned int is_extrainfo : 1;
-  /* If true, we got an extrainfo for this item, and the digest was right,
-   * but it was incompatible. */
-  unsigned int extrainfo_is_bogus : 1;
-  /* If true, we are willing to transmit this item unencrypted. */
-  unsigned int send_unencrypted : 1;
-} signed_descriptor_t;
-
-/** A signed integer representing a country code. */
-typedef int16_t country_t;
-
-/** Information about another onion router in the network. */
-typedef struct {
-  signed_descriptor_t cache_info;
-  char *nickname; /**< Human-readable OR name. */
-
-  uint32_t addr; /**< IPv4 address of OR, in host order. */
-  uint16_t or_port; /**< Port for TLS connections. */
-  uint16_t dir_port; /**< Port for HTTP directory connections. */
-
-  /** A router's IPv6 address, if it has one. */
-  /* XXXXX187 Actually these should probably be part of a list of addresses,
-   * not just a special case.  Use abstractions to access these; don't do it
-   * directly. */
-  tor_addr_t ipv6_addr;
-  uint16_t ipv6_orport;
-
-  crypto_pk_t *onion_pkey; /**< Public RSA key for onions. */
-  crypto_pk_t *identity_pkey;  /**< Public RSA key for signing. */
-  /** Public curve25519 key for onions */
-  curve25519_public_key_t *onion_curve25519_pkey;
-  /** What's the earliest expiration time on all the certs in this
-   * routerinfo? */
-  time_t cert_expiration_time;
-
-  char *platform; /**< What software/operating system is this OR using? */
-
-  char *protocol_list; /**< Encoded list of subprotocol versions supported
-                        * by this OR */
-
-  /* link info */
-  uint32_t bandwidthrate; /**< How many bytes does this OR add to its token
-                           * bucket per second? */
-  uint32_t bandwidthburst; /**< How large is this OR's token bucket? */
-  /** How many bytes/s is this router known to handle? */
-  uint32_t bandwidthcapacity;
-  smartlist_t *exit_policy; /**< What streams will this OR permit
-                             * to exit on IPv4?  NULL for 'reject *:*'. */
-  /** What streams will this OR permit to exit on IPv6?
-   * NULL for 'reject *:*' */
-  struct short_policy_t *ipv6_exit_policy;
-  long uptime; /**< How many seconds the router claims to have been up */
-  smartlist_t *declared_family; /**< Nicknames of router which this router
-                                 * claims are its family. */
-  char *contact_info; /**< Declared contact info for this router. */
-  unsigned int is_hibernating:1; /**< Whether the router claims to be
-                                  * hibernating */
-  unsigned int caches_extra_info:1; /**< Whether the router says it caches and
-                                     * serves extrainfo documents. */
-  unsigned int allow_single_hop_exits:1;  /**< Whether the router says
-                                           * it allows single hop exits. */
-
-  unsigned int wants_to_be_hs_dir:1; /**< True iff this router claims to be
-                                      * a hidden service directory. */
-  unsigned int policy_is_reject_star:1; /**< True iff the exit policy for this
-                                         * router rejects everything. */
-  /** True if, after we have added this router, we should re-launch
-   * tests for it. */
-  unsigned int needs_retest_if_added:1;
-
-  /** True iff this router included "tunnelled-dir-server" in its descriptor,
-   * implying it accepts tunnelled directory requests, or it advertised
-   * dir_port > 0. */
-  unsigned int supports_tunnelled_dir_requests:1;
-
-  /** Used during voting to indicate that we should not include an entry for
-   * this routerinfo. Used only during voting. */
-  unsigned int omit_from_vote:1;
-
-/** Tor can use this router for general positions in circuits; we got it
- * from a directory server as usual, or we're an authority and a server
- * uploaded it. */
-#define ROUTER_PURPOSE_GENERAL 0
-/** Tor should avoid using this router for circuit-building: we got it
- * from a crontroller.  If the controller wants to use it, it'll have to
- * ask for it by identity. */
-#define ROUTER_PURPOSE_CONTROLLER 1
-/** Tor should use this router only for bridge positions in circuits: we got
- * it via a directory request from the bridge itself, or a bridge
- * authority. x*/
-#define ROUTER_PURPOSE_BRIDGE 2
-/** Tor should not use this router; it was marked in cached-descriptors with
- * a purpose we didn't recognize. */
-#define ROUTER_PURPOSE_UNKNOWN 255
-
-  /* In what way did we find out about this router?  One of ROUTER_PURPOSE_*.
-   * Routers of different purposes are kept segregated and used for different
-   * things; see notes on ROUTER_PURPOSE_* macros above.
-   */
-  uint8_t purpose;
-} routerinfo_t;
-
-/** Information needed to keep and cache a signed extra-info document. */
-typedef struct extrainfo_t {
-  signed_descriptor_t cache_info;
-  /** SHA256 digest of this document */
-  uint8_t digest256[DIGEST256_LEN];
-  /** The router's nickname. */
-  char nickname[MAX_NICKNAME_LEN+1];
-  /** True iff we found the right key for this extra-info, verified the
-   * signature, and found it to be bad. */
-  unsigned int bad_sig : 1;
-  /** If present, we didn't have the right key to verify this extra-info,
-   * so this is a copy of the signature in the document. */
-  char *pending_sig;
-  /** Length of pending_sig. */
-  size_t pending_sig_len;
-} extrainfo_t;
-
-/** Contents of a single router entry in a network status object.
- */
-typedef struct routerstatus_t {
-  time_t published_on; /**< When was this router published? */
-  char nickname[MAX_NICKNAME_LEN+1]; /**< The nickname this router says it
-                                      * has. */
-  char identity_digest[DIGEST_LEN]; /**< Digest of the router's identity
-                                     * key. */
-  /** Digest of the router's most recent descriptor or microdescriptor.
-   * If it's a descriptor, we only use the first DIGEST_LEN bytes. */
-  char descriptor_digest[DIGEST256_LEN];
-  uint32_t addr; /**< IPv4 address for this router, in host order. */
-  uint16_t or_port; /**< OR port for this router. */
-  uint16_t dir_port; /**< Directory port for this router. */
-  tor_addr_t ipv6_addr; /**< IPv6 address for this router. */
-  uint16_t ipv6_orport; /**<IPV6 OR port for this router. */
-  unsigned int is_authority:1; /**< True iff this router is an authority. */
-  unsigned int is_exit:1; /**< True iff this router is a good exit. */
-  unsigned int is_stable:1; /**< True iff this router stays up a long time. */
-  unsigned int is_fast:1; /**< True iff this router has good bandwidth. */
-  /** True iff this router is called 'running' in the consensus. We give it
-   * this funny name so that we don't accidentally use this bit as a view of
-   * whether we think the router is *currently* running.  If that's what you
-   * want to know, look at is_running in node_t. */
-  unsigned int is_flagged_running:1;
-  unsigned int is_named:1; /**< True iff "nickname" belongs to this router. */
-  unsigned int is_unnamed:1; /**< True iff "nickname" belongs to another
-                              * router. */
-  unsigned int is_valid:1; /**< True iff this router isn't invalid. */
-  unsigned int is_possible_guard:1; /**< True iff this router would be a good
-                                     * choice as an entry guard. */
-  unsigned int is_bad_exit:1; /**< True iff this node is a bad choice for
-                               * an exit node. */
-  unsigned int is_hs_dir:1; /**< True iff this router is a v2-or-later hidden
-                             * service directory. */
-  unsigned int is_v2_dir:1; /** True iff this router publishes an open DirPort
-                             * or it claims to accept tunnelled dir requests.
-                             */
-  /** True iff we have a proto line for this router, or a versions line
-   * from which we could infer the protocols. */
-  unsigned int protocols_known:1;
-
-  /** True iff this router has a version or protocol list that allows it to
-   * accept EXTEND2 cells */
-  unsigned int supports_extend2_cells:1;
-
-  unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */
-  unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */
-  unsigned int bw_is_unmeasured:1; /**< This is a consensus entry, with
-                                    * the Unmeasured flag set. */
-
-  uint32_t bandwidth_kb; /**< Bandwidth (capacity) of the router as reported in
-                       * the vote/consensus, in kilobytes/sec. */
-
-  /** The consensus has guardfraction information for this router. */
-  unsigned int has_guardfraction:1;
-  /** The guardfraction value of this router. */
-  uint32_t guardfraction_percentage;
-
-  char *exitsummary; /**< exit policy summary -
-                      * XXX weasel: this probably should not stay a string. */
-
-  /* ---- The fields below aren't derived from the networkstatus; they
-   * hold local information only. */
-
-  time_t last_dir_503_at; /**< When did this router last tell us that it
-                           * was too busy to serve directory info? */
-  download_status_t dl_status;
-
-} routerstatus_t;
-
-/** A single entry in a parsed policy summary, describing a range of ports. */
-typedef struct short_policy_entry_t {
-  uint16_t min_port, max_port;
-} short_policy_entry_t;
-
-/** A short_poliy_t is the parsed version of a policy summary. */
-typedef struct short_policy_t {
-  /** True if the members of 'entries' are port ranges to accept; false if
-   * they are port ranges to reject */
-  unsigned int is_accept : 1;
-  /** The actual number of values in 'entries'. */
-  unsigned int n_entries : 31;
-  /** An array of 0 or more short_policy_entry_t values, each describing a
-   * range of ports that this policy accepts or rejects (depending on the
-   * value of is_accept).
-   */
-  short_policy_entry_t entries[FLEXIBLE_ARRAY_MEMBER];
-} short_policy_t;
-
-/** A microdescriptor is the smallest amount of information needed to build a
- * circuit through a router.  They are generated by the directory authorities,
- * using information from the uploaded routerinfo documents.  They are not
- * self-signed, but are rather authenticated by having their hash in a signed
- * networkstatus document. */
-typedef struct microdesc_t {
-  /** Hashtable node, used to look up the microdesc by its digest. */
-  HT_ENTRY(microdesc_t) node;
-
-  /* Cache information */
-
-  /**  When was this microdescriptor last listed in a consensus document?
-   * Once a microdesc has been unlisted long enough, we can drop it.
-   */
-  time_t last_listed;
-  /** Where is this microdescriptor currently stored? */
-  saved_location_bitfield_t saved_location : 3;
-  /** If true, do not attempt to cache this microdescriptor on disk. */
-  unsigned int no_save : 1;
-  /** If true, this microdesc has an entry in the microdesc_map */
-  unsigned int held_in_map : 1;
-  /** Reference count: how many node_ts have a reference to this microdesc? */
-  unsigned int held_by_nodes;
-
-  /** If saved_location == SAVED_IN_CACHE, this field holds the offset of the
-   * microdescriptor in the cache. */
-  off_t off;
-
-  /* The string containing the microdesc. */
-
-  /** A pointer to the encoded body of the microdescriptor.  If the
-   * saved_location is SAVED_IN_CACHE, then the body is a pointer into an
-   * mmap'd region.  Otherwise, it is a malloc'd string.  The string might not
-   * be NUL-terminated; take the length from <b>bodylen</b>. */
-  char *body;
-  /** The length of the microdescriptor in <b>body</b>. */
-  size_t bodylen;
-  /** A SHA256-digest of the microdescriptor. */
-  char digest[DIGEST256_LEN];
-
-  /* Fields in the microdescriptor. */
-
-  /** As routerinfo_t.onion_pkey */
-  crypto_pk_t *onion_pkey;
-  /** As routerinfo_t.onion_curve25519_pkey */
-  curve25519_public_key_t *onion_curve25519_pkey;
-  /** Ed25519 identity key, if included. */
-  ed25519_public_key_t *ed25519_identity_pkey;
-  /** As routerinfo_t.ipv6_addr */
-  tor_addr_t ipv6_addr;
-  /** As routerinfo_t.ipv6_orport */
-  uint16_t ipv6_orport;
-  /** As routerinfo_t.family */
-  smartlist_t *family;
-  /** IPv4 exit policy summary */
-  short_policy_t *exit_policy;
-  /** IPv6 exit policy summary */
-  short_policy_t *ipv6_exit_policy;
-
-} microdesc_t;
-
-/** A node_t represents a Tor router.
- *
- * Specifically, a node_t is a Tor router as we are using it: a router that
- * we are considering for circuits, connections, and so on.  A node_t is a
- * thin wrapper around the routerstatus, routerinfo, and microdesc for a
- * single router, and provides a consistent interface for all of them.
- *
- * Also, a node_t has mutable state.  While a routerinfo, a routerstatus,
- * and a microdesc have[*] only the information read from a router
- * descriptor, a consensus entry, and a microdescriptor (respectively)...
- * a node_t has flags based on *our own current opinion* of the node.
- *
- * [*] Actually, there is some leftover information in each that is mutable.
- *  We should try to excise that.
- */
-typedef struct node_t {
-  /* Indexing information */
-
-  /** Used to look up the node_t by its identity digest. */
-  HT_ENTRY(node_t) ht_ent;
-  /** Position of the node within the list of nodes */
-  int nodelist_idx;
-
-  /** The identity digest of this node_t.  No more than one node_t per
-   * identity may exist at a time. */
-  char identity[DIGEST_LEN];
-
-  microdesc_t *md;
-  routerinfo_t *ri;
-  routerstatus_t *rs;
-
-  /* local info: copied from routerstatus, then possibly frobbed based
-   * on experience.  Authorities set this stuff directly.  Note that
-   * these reflect knowledge of the primary (IPv4) OR port only.  */
-
-  unsigned int is_running:1; /**< As far as we know, is this OR currently
-                              * running? */
-  unsigned int is_valid:1; /**< Has a trusted dirserver validated this OR?
-                            *  (For Authdir: Have we validated this OR?) */
-  unsigned int is_fast:1; /** Do we think this is a fast OR? */
-  unsigned int is_stable:1; /** Do we think this is a stable OR? */
-  unsigned int is_possible_guard:1; /**< Do we think this is an OK guard? */
-  unsigned int is_exit:1; /**< Do we think this is an OK exit? */
-  unsigned int is_bad_exit:1; /**< Do we think this exit is censored, borked,
-                               * or otherwise nasty? */
-  unsigned int is_hs_dir:1; /**< True iff this router is a hidden service
-                             * directory according to the authorities. */
-
-  /* Local info: warning state. */
-
-  unsigned int name_lookup_warned:1; /**< Have we warned the user for referring
-                                      * to this (unnamed) router by nickname?
-                                      */
-
-  /** Local info: we treat this node as if it rejects everything */
-  unsigned int rejects_all:1;
-
-  /** Local info: this node is in our list of guards */
-  unsigned int using_as_guard:1;
-
-  /* Local info: derived. */
-
-  /** True if the IPv6 OR port is preferred over the IPv4 OR port.
-   * XX/teor - can this become out of date if the torrc changes? */
-  unsigned int ipv6_preferred:1;
-
-  /** According to the geoip db what country is this router in? */
-  /* XXXprop186 what is this suppose to mean with multiple OR ports? */
-  country_t country;
-
-  /* The below items are used only by authdirservers for
-   * reachability testing. */
-
-  /** When was the last time we could reach this OR? */
-  time_t last_reachable;        /* IPv4. */
-  time_t last_reachable6;       /* IPv6. */
-
-} node_t;
-
-/** Linked list of microdesc hash lines for a single router in a directory
- * vote.
- */
-typedef struct vote_microdesc_hash_t {
-  /** Next element in the list, or NULL. */
-  struct vote_microdesc_hash_t *next;
-  /** The raw contents of the microdesc hash line, from the "m" through the
-   * newline. */
-  char *microdesc_hash_line;
-} vote_microdesc_hash_t;
-
-/** The claim about a single router, made in a vote. */
-typedef struct vote_routerstatus_t {
-  routerstatus_t status; /**< Underlying 'status' object for this router.
-                          * Flags are redundant. */
-  /** How many known-flags are allowed in a vote? This is the width of
-   * the flags field of vote_routerstatus_t */
-#define MAX_KNOWN_FLAGS_IN_VOTE 64
-  uint64_t flags; /**< Bit-field for all recognized flags; index into
-                   * networkstatus_t.known_flags. */
-  char *version; /**< The version that the authority says this router is
-                  * running. */
-  char *protocols; /**< The protocols that this authority says this router
-                    * provides. */
-  unsigned int has_measured_bw:1; /**< The vote had a measured bw */
-  /** True iff the vote included an entry for ed25519 ID, or included
-   * "id ed25519 none" to indicate that there was no ed25519 ID. */
-  unsigned int has_ed25519_listing:1;
-  /** True if the Ed25519 listing here is the consensus-opinion for the
-   * Ed25519 listing; false if there was no consensus on Ed25519 key status,
-   * or if this VRS doesn't reflect it. */
-  unsigned int ed25519_reflects_consensus:1;
-  uint32_t measured_bw_kb; /**< Measured bandwidth (capacity) of the router */
-  /** The hash or hashes that the authority claims this microdesc has. */
-  vote_microdesc_hash_t *microdesc;
-  /** Ed25519 identity for this router, or zero if it has none. */
-  uint8_t ed25519_id[ED25519_PUBKEY_LEN];
-} vote_routerstatus_t;
-
-/** A signature of some document by an authority. */
-typedef struct document_signature_t {
-  /** Declared SHA-1 digest of this voter's identity key */
-  char identity_digest[DIGEST_LEN];
-  /** Declared SHA-1 digest of signing key used by this voter. */
-  char signing_key_digest[DIGEST_LEN];
-  /** Algorithm used to compute the digest of the document. */
-  digest_algorithm_t alg;
-  /** Signature of the signed thing. */
-  char *signature;
-  /** Length of <b>signature</b> */
-  int signature_len;
-  unsigned int bad_signature : 1; /**< Set to true if we've tried to verify
-                                   * the sig, and we know it's bad. */
-  unsigned int good_signature : 1; /**< Set to true if we've verified the sig
-                                     * as good. */
-} document_signature_t;
-
-/** Information about a single voter in a vote or a consensus. */
-typedef struct networkstatus_voter_info_t {
-  /** Declared SHA-1 digest of this voter's identity key */
-  char identity_digest[DIGEST_LEN];
-  char *nickname; /**< Nickname of this voter */
-  /** Digest of this voter's "legacy" identity key, if any.  In vote only; for
-   * consensuses, we treat legacy keys as additional signers. */
-  char legacy_id_digest[DIGEST_LEN];
-  char *address; /**< Address of this voter, in string format. */
-  uint32_t addr; /**< Address of this voter, in IPv4, in host order. */
-  uint16_t dir_port; /**< Directory port of this voter */
-  uint16_t or_port; /**< OR port of this voter */
-  char *contact; /**< Contact information for this voter. */
-  char vote_digest[DIGEST_LEN]; /**< Digest of this voter's vote, as signed. */
-
-  /* Nothing from here on is signed. */
-  /** The signature of the document and the signature's status. */
-  smartlist_t *sigs;
-} networkstatus_voter_info_t;
-
-typedef struct networkstatus_sr_info_t {
-  /* Indicate if the dirauth partitipates in the SR protocol with its vote.
-   * This is tied to the SR flag in the vote. */
-  unsigned int participate:1;
-  /* Both vote and consensus: Current and previous SRV. If list is empty,
-   * this means none were found in either the consensus or vote. */
-  struct sr_srv_t *previous_srv;
-  struct sr_srv_t *current_srv;
-  /* Vote only: List of commitments. */
-  smartlist_t *commits;
-} networkstatus_sr_info_t;
-
-/** Enumerates the possible seriousness values of a networkstatus document. */
-typedef enum {
-  NS_TYPE_VOTE,
-  NS_TYPE_CONSENSUS,
-  NS_TYPE_OPINION,
-} networkstatus_type_t;
-
-/** Enumerates recognized flavors of a consensus networkstatus document.  All
- * flavors of a consensus are generated from the same set of votes, but they
- * present different types information to different versions of Tor. */
-typedef enum {
-  FLAV_NS = 0,
-  FLAV_MICRODESC = 1,
-} consensus_flavor_t;
-
-/** How many different consensus flavors are there? */
-#define N_CONSENSUS_FLAVORS ((int)(FLAV_MICRODESC)+1)
-
-/** A common structure to hold a v3 network status vote, or a v3 network
- * status consensus. */
-typedef struct networkstatus_t {
-  networkstatus_type_t type; /**< Vote, consensus, or opinion? */
-  consensus_flavor_t flavor; /**< If a consensus, what kind? */
-  unsigned int has_measured_bws : 1;/**< True iff this networkstatus contains
-                                     * measured= bandwidth values. */
-
-  time_t published; /**< Vote only: Time when vote was written. */
-  time_t valid_after; /**< Time after which this vote or consensus applies. */
-  time_t fresh_until; /**< Time before which this is the most recent vote or
-                       * consensus. */
-  time_t valid_until; /**< Time after which this vote or consensus should not
-                       * be used. */
-
-  /** Consensus only: what method was used to produce this consensus? */
-  int consensus_method;
-  /** Vote only: what methods is this voter willing to use? */
-  smartlist_t *supported_methods;
-
-  /** List of 'package' lines describing hashes of downloadable packages */
-  smartlist_t *package_lines;
-
-  /** How long does this vote/consensus claim that authorities take to
-   * distribute their votes to one another? */
-  int vote_seconds;
-  /** How long does this vote/consensus claim that authorities take to
-   * distribute their consensus signatures to one another? */
-  int dist_seconds;
-
-  /** Comma-separated list of recommended client software, or NULL if this
-   * voter has no opinion. */
-  char *client_versions;
-  char *server_versions;
-
-  /** Lists of subprotocol versions which are _recommended_ for relays and
-   * clients, or which are _require_ for relays and clients. Tor shouldn't
-   * make any more network connections if a required protocol is missing.
-   */
-  char *recommended_relay_protocols;
-  char *recommended_client_protocols;
-  char *required_relay_protocols;
-  char *required_client_protocols;
-
-  /** List of flags that this vote/consensus applies to routers.  If a flag is
-   * not listed here, the voter has no opinion on what its value should be. */
-  smartlist_t *known_flags;
-
-  /** List of key=value strings for the parameters in this vote or
-   * consensus, sorted by key. */
-  smartlist_t *net_params;
-
-  /** List of key=value strings for the bw weight parameters in the
-   * consensus. */
-  smartlist_t *weight_params;
-
-  /** List of networkstatus_voter_info_t.  For a vote, only one element
-   * is included.  For a consensus, one element is included for every voter
-   * whose vote contributed to the consensus. */
-  smartlist_t *voters;
-
-  struct authority_cert_t *cert; /**< Vote only: the voter's certificate. */
-
-  /** Digests of this document, as signed. */
-  common_digests_t digests;
-
-  /** List of router statuses, sorted by identity digest.  For a vote,
-   * the elements are vote_routerstatus_t; for a consensus, the elements
-   * are routerstatus_t. */
-  smartlist_t *routerstatus_list;
-
-  /** If present, a map from descriptor digest to elements of
-   * routerstatus_list. */
-  digestmap_t *desc_digest_map;
-
-  /** Contains the shared random protocol data from a vote or consensus. */
-  networkstatus_sr_info_t sr_info;
-} networkstatus_t;
-
-/** A set of signatures for a networkstatus consensus.  Unless otherwise
- * noted, all fields are as for networkstatus_t. */
-typedef struct ns_detached_signatures_t {
-  time_t valid_after;
-  time_t fresh_until;
-  time_t valid_until;
-  strmap_t *digests; /**< Map from flavor name to digestset_t */
-  strmap_t *signatures; /**< Map from flavor name to list of
-                         * document_signature_t */
-} ns_detached_signatures_t;
-
-/** Allowable types of desc_store_t. */
-typedef enum store_type_t {
-  ROUTER_STORE = 0,
-  EXTRAINFO_STORE = 1
-} store_type_t;
-
-/** A 'store' is a set of descriptors saved on disk, with accompanying
- * journal, mmaped as needed, rebuilt as needed. */
-typedef struct desc_store_t {
-  /** Filename (within DataDir) for the store.  We append .tmp to this
-   * filename for a temporary file when rebuilding the store, and .new to this
-   * filename for the journal. */
-  const char *fname_base;
-  /** Human-readable description of what this store contains. */
-  const char *description;
-
-  tor_mmap_t *mmap; /**< A mmap for the main file in the store. */
-
-  store_type_t type; /**< What's stored in this store? */
-
-  /** The size of the router log, in bytes. */
-  size_t journal_len;
-  /** The size of the router store, in bytes. */
-  size_t store_len;
-  /** Total bytes dropped since last rebuild: this is space currently
-   * used in the cache and the journal that could be freed by a rebuild. */
-  size_t bytes_dropped;
-} desc_store_t;
-
-/** Contents of a directory of onion routers. */
-typedef struct {
-  /** Map from server identity digest to a member of routers. */
-  struct digest_ri_map_t *identity_map;
-  /** Map from server descriptor digest to a signed_descriptor_t from
-   * routers or old_routers. */
-  struct digest_sd_map_t *desc_digest_map;
-  /** Map from extra-info digest to an extrainfo_t.  Only exists for
-   * routers in routers or old_routers. */
-  struct digest_ei_map_t *extra_info_map;
-  /** Map from extra-info digests to a signed_descriptor_t for a router
-   * descriptor having that extra-info digest.  Only exists for
-   * routers in routers or old_routers. */
-  struct digest_sd_map_t *desc_by_eid_map;
-  /** List of routerinfo_t for all currently live routers we know. */
-  smartlist_t *routers;
-  /** List of signed_descriptor_t for older router descriptors we're
-   * caching. */
-  smartlist_t *old_routers;
-  /** Store holding server descriptors.  If present, any router whose
-   * cache_info.saved_location == SAVED_IN_CACHE is stored in this file
-   * starting at cache_info.saved_offset */
-  desc_store_t desc_store;
-  /** Store holding extra-info documents. */
-  desc_store_t extrainfo_store;
-} routerlist_t;
-
-/** Information on router used when extending a circuit. We don't need a
- * full routerinfo_t to extend: we only need addr:port:keyid to build an OR
- * connection, and onion_key to create the onionskin. Note that for onehop
- * general-purpose tunnels, the onion_key is NULL. */
-typedef struct extend_info_t {
-  char nickname[MAX_HEX_NICKNAME_LEN+1]; /**< This router's nickname for
-                                          * display. */
-  char identity_digest[DIGEST_LEN]; /**< Hash of this router's identity key. */
-  uint16_t port; /**< OR port. */
-  tor_addr_t addr; /**< IP address. */
-  crypto_pk_t *onion_key; /**< Current onionskin key. */
-  curve25519_public_key_t curve25519_onion_key;
-} extend_info_t;
-
-/** Certificate for v3 directory protocol: binds long-term authority identity
- * keys to medium-term authority signing keys. */
-typedef struct authority_cert_t {
-  /** Information relating to caching this cert on disk and looking it up. */
-  signed_descriptor_t cache_info;
-  /** This authority's long-term authority identity key. */
-  crypto_pk_t *identity_key;
-  /** This authority's medium-term signing key. */
-  crypto_pk_t *signing_key;
-  /** The digest of <b>signing_key</b> */
-  char signing_key_digest[DIGEST_LEN];
-  /** The listed expiration time of this certificate. */
-  time_t expires;
-  /** This authority's IPv4 address, in host order. */
-  uint32_t addr;
-  /** This authority's directory port. */
-  uint16_t dir_port;
-} authority_cert_t;
-
-/** Bitfield enum type listing types of information that directory authorities
- * can be authoritative about, and that directory caches may or may not cache.
- *
- * Note that the granularity here is based on authority granularity and on
- * cache capabilities.  Thus, one particular bit may correspond in practice to
- * a few types of directory info, so long as every authority that pronounces
- * officially about one of the types prounounces officially about all of them,
- * and so long as every cache that caches one of them caches all of them.
- */
-typedef enum {
-  NO_DIRINFO      = 0,
-  /** Serves/signs v3 directory information: votes, consensuses, certs */
-  V3_DIRINFO      = 1 << 2,
-  /** Serves bridge descriptors. */
-  BRIDGE_DIRINFO  = 1 << 4,
-  /** Serves extrainfo documents. */
-  EXTRAINFO_DIRINFO=1 << 5,
-  /** Serves microdescriptors. */
-  MICRODESC_DIRINFO=1 << 6,
-} dirinfo_type_t;
-
-#define ALL_DIRINFO ((dirinfo_type_t)((1<<7)-1))
-
-#define CRYPT_PATH_MAGIC 0x70127012u
-
-struct fast_handshake_state_t;
-struct ntor_handshake_state_t;
-#define ONION_HANDSHAKE_TYPE_TAP  0x0000
-#define ONION_HANDSHAKE_TYPE_FAST 0x0001
-#define ONION_HANDSHAKE_TYPE_NTOR 0x0002
-#define MAX_ONION_HANDSHAKE_TYPE 0x0002
-typedef struct {
-  uint16_t tag;
-  union {
-    struct fast_handshake_state_t *fast;
-    crypto_dh_t *tap;
-    struct ntor_handshake_state_t *ntor;
-  } u;
-} onion_handshake_state_t;
-
-/** Holds accounting information for a single step in the layered encryption
- * performed by a circuit.  Used only at the client edge of a circuit. */
-typedef struct crypt_path_t {
-  uint32_t magic;
-
-  /* crypto environments */
-  /** Encryption key and counter for cells heading towards the OR at this
-   * step. */
-  crypto_cipher_t *f_crypto;
-  /** Encryption key and counter for cells heading back from the OR at this
-   * step. */
-  crypto_cipher_t *b_crypto;
-
-  /** Digest state for cells heading towards the OR at this step. */
-  crypto_digest_t *f_digest; /* for integrity checking */
-  /** Digest state for cells heading away from the OR at this step. */
-  crypto_digest_t *b_digest;
-
-  /** Current state of the handshake as performed with the OR at this
-   * step. */
-  onion_handshake_state_t handshake_state;
-  /** Diffie-hellman handshake state for performing an introduction
-   * operations */
-  crypto_dh_t *rend_dh_handshake_state;
-
-  /** Negotiated key material shared with the OR at this step. */
-  char rend_circ_nonce[DIGEST_LEN];/* KH in tor-spec.txt */
-
-  /** Information to extend to the OR at this step. */
-  extend_info_t *extend_info;
-
-  /** Is the circuit built to this step?  Must be one of:
-   *    - CPATH_STATE_CLOSED (The circuit has not been extended to this step)
-   *    - CPATH_STATE_AWAITING_KEYS (We have sent an EXTEND/CREATE to this step
-   *      and not received an EXTENDED/CREATED)
-   *    - CPATH_STATE_OPEN (The circuit has been extended to this step) */
-  uint8_t state;
-#define CPATH_STATE_CLOSED 0
-#define CPATH_STATE_AWAITING_KEYS 1
-#define CPATH_STATE_OPEN 2
-  struct crypt_path_t *next; /**< Link to next crypt_path_t in the circuit.
-                              * (The list is circular, so the last node
-                              * links to the first.) */
-  struct crypt_path_t *prev; /**< Link to previous crypt_path_t in the
-                              * circuit. */
-
-  int package_window; /**< How many cells are we allowed to originate ending
-                       * at this step? */
-  int deliver_window; /**< How many cells are we willing to deliver originating
-                       * at this step? */
-} crypt_path_t;
-
-/** A reference-counted pointer to a crypt_path_t, used only to share
- * the final rendezvous cpath to be used on a service-side rendezvous
- * circuit among multiple circuits built in parallel to the same
- * destination rendezvous point. */
-typedef struct {
-  /** The reference count. */
-  unsigned int refcount;
-  /** The pointer.  Set to NULL when the crypt_path_t is put into use
-   * on an opened rendezvous circuit. */
-  crypt_path_t *cpath;
-} crypt_path_reference_t;
-
-#define CPATH_KEY_MATERIAL_LEN (20*2+16*2)
-
-#define DH_KEY_LEN DH_BYTES
-
-/** Information used to build a circuit. */
-typedef struct {
-  /** Intended length of the final circuit. */
-  int desired_path_len;
-  /** How to extend to the planned exit node. */
-  extend_info_t *chosen_exit;
-  /** Whether every node in the circ must have adequate uptime. */
-  unsigned int need_uptime : 1;
-  /** Whether every node in the circ must have adequate capacity. */
-  unsigned int need_capacity : 1;
-  /** Whether the last hop was picked with exiting in mind. */
-  unsigned int is_internal : 1;
-  /** Did we pick this as a one-hop tunnel (not safe for other streams)?
-   * These are for encrypted dir conns that exit to this router, not
-   * for arbitrary exits from the circuit. */
-  unsigned int onehop_tunnel : 1;
-  /** The crypt_path_t to append after rendezvous: used for rendezvous. */
-  crypt_path_t *pending_final_cpath;
-  /** A ref-counted reference to the crypt_path_t to append after
-   * rendezvous; used on the service side. */
-  crypt_path_reference_t *service_pending_final_cpath_ref;
-  /** How many times has building a circuit for this task failed? */
-  int failure_count;
-  /** At what time should we give up on this task? */
-  time_t expiry_time;
-} cpath_build_state_t;
-
-/** "magic" value for an origin_circuit_t */
-#define ORIGIN_CIRCUIT_MAGIC 0x35315243u
-/** "magic" value for an or_circuit_t */
-#define OR_CIRCUIT_MAGIC 0x98ABC04Fu
-/** "magic" value for a circuit that would have been freed by circuit_free,
- * but which we're keeping around until a cpuworker reply arrives.  See
- * circuit_free() for more documentation. */
-#define DEAD_CIRCUIT_MAGIC 0xdeadc14c
-
-struct create_cell_t;
-
-/** Entry in the cell stats list of a circuit; used only if CELL_STATS
- * events are enabled. */
-typedef struct testing_cell_stats_entry_t {
-  uint8_t command; /**< cell command number. */
-  /** Waiting time in centiseconds if this event is for a removed cell,
-   * or 0 if this event is for adding a cell to the queue.  22 bits can
-   * store more than 11 hours, enough to assume that a circuit with this
-   * delay would long have been closed. */
-  unsigned int waiting_time:22;
-  unsigned int removed:1; /**< 0 for added to, 1 for removed from queue. */
-  unsigned int exitward:1; /**< 0 for app-ward, 1 for exit-ward. */
-} testing_cell_stats_entry_t;
-
-/**
- * An enum to allow us to specify which channel in a circuit
- * we're interested in.
- *
- * This is needed because our data structures and other fields
- * for channel delivery are disassociated from the channel.
- */
-typedef enum {
-  CIRCUIT_N_CHAN = 0,
-  CIRCUIT_P_CHAN = 1
-} circuit_channel_direction_t;
-
-/**
- * A circuit is a path over the onion routing
- * network. Applications can connect to one end of the circuit, and can
- * create exit connections at the other end of the circuit. AP and exit
- * connections have only one circuit associated with them (and thus these
- * connection types are closed when the circuit is closed), whereas
- * OR connections multiplex many circuits at once, and stay standing even
- * when there are no circuits running over them.
- *
- * A circuit_t structure can fill one of two roles.  First, a or_circuit_t
- * links two connections together: either an edge connection and an OR
- * connection, or two OR connections.  (When joined to an OR connection, a
- * circuit_t affects only cells sent to a particular circID on that
- * connection.  When joined to an edge connection, a circuit_t affects all
- * data.)
-
- * Second, an origin_circuit_t holds the cipher keys and state for sending data
- * along a given circuit.  At the OP, it has a sequence of ciphers, each
- * of which is shared with a single OR along the circuit.  Separate
- * ciphers are used for data going "forward" (away from the OP) and
- * "backward" (towards the OP).  At the OR, a circuit has only two stream
- * ciphers: one for data going forward, and one for data going backward.
- */
-typedef struct circuit_t {
-  uint32_t magic; /**< For memory and type debugging: must equal
-                   * ORIGIN_CIRCUIT_MAGIC or OR_CIRCUIT_MAGIC. */
-
-  /** The channel that is next in this circuit. */
-  channel_t *n_chan;
-
-  /**
-   * The circuit_id used in the next (forward) hop of this circuit;
-   * this is unique to n_chan, but this ordered pair is globally
-   * unique:
-   *
-   * (n_chan->global_identifier, n_circ_id)
-   */
-  circid_t n_circ_id;
-
-  /**
-   * Circuit mux associated with n_chan to which this circuit is attached;
-   * NULL if we have no n_chan.
-   */
-  circuitmux_t *n_mux;
-
-  /** Queue of cells waiting to be transmitted on n_chan */
-  cell_queue_t n_chan_cells;
-
-  /**
-   * The hop to which we want to extend this circuit.  Should be NULL if
-   * the circuit has attached to a channel.
-   */
-  extend_info_t *n_hop;
-
-  /** True iff we are waiting for n_chan_cells to become less full before
-   * allowing p_streams to add any more cells. (Origin circuit only.) */
-  unsigned int streams_blocked_on_n_chan : 1;
-  /** True iff we are waiting for p_chan_cells to become less full before
-   * allowing n_streams to add any more cells. (OR circuit only.) */
-  unsigned int streams_blocked_on_p_chan : 1;
-
-  /** True iff we have queued a delete backwards on this circuit, but not put
-   * it on the output buffer. */
-  unsigned int p_delete_pending : 1;
-  /** True iff we have queued a delete forwards on this circuit, but not put
-   * it on the output buffer. */
-  unsigned int n_delete_pending : 1;
-
-  /** True iff this circuit has received a DESTROY cell in either direction */
-  unsigned int received_destroy : 1;
-
-  uint8_t state; /**< Current status of this circuit. */
-  uint8_t purpose; /**< Why are we creating this circuit? */
-
-  /** How many relay data cells can we package (read from edge streams)
-   * on this circuit before we receive a circuit-level sendme cell asking
-   * for more? */
-  int package_window;
-  /** How many relay data cells will we deliver (write to edge streams)
-   * on this circuit? When deliver_window gets low, we send some
-   * circuit-level sendme cells to indicate that we're willing to accept
-   * more. */
-  int deliver_window;
-
-  /** Temporary field used during circuits_handle_oom. */
-  uint32_t age_tmp;
-
-  /** For storage while n_chan is pending (state CIRCUIT_STATE_CHAN_WAIT). */
-  struct create_cell_t *n_chan_create_cell;
-
-  /** When did circuit construction actually begin (ie send the
-   * CREATE cell or begin cannibalization).
-   *
-   * Note: This timer will get reset if we decide to cannibalize
-   * a circuit. It may also get reset during certain phases of hidden
-   * service circuit use.
-   *
-   * We keep this timestamp with a higher resolution than most so that the
-   * circuit-build-time tracking code can get millisecond resolution.
-   */
-  struct timeval timestamp_began;
-
-  /** This timestamp marks when the init_circuit_base constructor ran. */
-  struct timeval timestamp_created;
-
-  /** When the circuit was first used, or 0 if the circuit is clean.
-   *
-   * XXXX Note that some code will artifically adjust this value backward
-   * in time in order to indicate that a circuit shouldn't be used for new
-   * streams, but that it can stay alive as long as it has streams on it.
-   * That's a kludge we should fix.
-   *
-   * XXX The CBT code uses this field to record when HS-related
-   * circuits entered certain states.  This usage probably won't
-   * interfere with this field's primary purpose, but we should
-   * document it more thoroughly to make sure of that.
-   *
-   * XXX The SocksPort option KeepaliveIsolateSOCKSAuth will artificially
-   * adjust this value forward each time a suitable stream is attached to an
-   * already constructed circuit, potentially keeping the circuit alive
-   * indefinitely.
-   */
-  time_t timestamp_dirty;
-
-  uint16_t marked_for_close; /**< Should we close this circuit at the end of
-                              * the main loop? (If true, holds the line number
-                              * where this circuit was marked.) */
-  const char *marked_for_close_file; /**< For debugging: in which file was this
-                                      * circuit marked for close? */
-  /** For what reason (See END_CIRC_REASON...) is this circuit being closed?
-   * This field is set in circuit_mark_for_close and used later in
-   * circuit_about_to_free. */
-  int marked_for_close_reason;
-  /** As marked_for_close_reason, but reflects the underlying reason for
-   * closing this circuit.
-   */
-  int marked_for_close_orig_reason;
-
-  /** Unique ID for measuring tunneled network status requests. */
-  uint64_t dirreq_id;
-
-  /** Index in smartlist of all circuits (global_circuitlist). */
-  int global_circuitlist_idx;
-
-  /** Next circuit in the doubly-linked ring of circuits waiting to add
-   * cells to n_conn.  NULL if we have no cells pending, or if we're not
-   * linked to an OR connection. */
-  struct circuit_t *next_active_on_n_chan;
-  /** Previous circuit in the doubly-linked ring of circuits waiting to add
-   * cells to n_conn.  NULL if we have no cells pending, or if we're not
-   * linked to an OR connection. */
-  struct circuit_t *prev_active_on_n_chan;
-
-  /** Various statistics about cells being added to or removed from this
-   * circuit's queues; used only if CELL_STATS events are enabled and
-   * cleared after being sent to control port. */
-  smartlist_t *testing_cell_stats;
-} circuit_t;
-
-/** Largest number of relay_early cells that we can send on a given
- * circuit. */
-#define MAX_RELAY_EARLY_CELLS_PER_CIRCUIT 8
-
-/**
- * Describes the circuit building process in simplified terms based
- * on the path bias accounting state for a circuit.
- *
- * NOTE: These state values are enumerated in the order for which we
- * expect circuits to transition through them. If you add states,
- * you need to preserve this overall ordering. The various pathbias
- * state transition and accounting functions (pathbias_mark_* and
- * pathbias_count_*) contain ordinal comparisons to enforce proper
- * state transitions for corrections.
- *
- * This state machine and the associated logic was created to prevent
- * miscounting due to unknown cases of circuit reuse. See also tickets
- * #6475 and #7802.
- */
-typedef enum {
-    /** This circuit is "new". It has not yet completed a first hop
-     * or been counted by the path bias code. */
-    PATH_STATE_NEW_CIRC = 0,
-    /** This circuit has completed one/two hops, and has been counted by
-     * the path bias logic. */
-    PATH_STATE_BUILD_ATTEMPTED = 1,
-    /** This circuit has been completely built */
-    PATH_STATE_BUILD_SUCCEEDED = 2,
-    /** Did we try to attach any SOCKS streams or hidserv introductions to
-      * this circuit?
-      *
-      * Note: If we ever implement end-to-end stream timing through test
-      * stream probes (#5707), we must *not* set this for those probes
-      * (or any other automatic streams) because the adversary could
-      * just tag at a later point.
-      */
-    PATH_STATE_USE_ATTEMPTED = 3,
-    /** Did any SOCKS streams or hidserv introductions actually succeed on
-      * this circuit?
-      *
-      * If any streams detatch/fail from this circuit, the code transitions
-      * the circuit back to PATH_STATE_USE_ATTEMPTED to ensure we probe. See
-      * pathbias_mark_use_rollback() for that.
-      */
-    PATH_STATE_USE_SUCCEEDED = 4,
-
-    /**
-     * This is a special state to indicate that we got a corrupted
-     * relay cell on a circuit and we don't intend to probe it.
-     */
-    PATH_STATE_USE_FAILED = 5,
-
-    /**
-     * This is a special state to indicate that we already counted
-     * the circuit. Used to guard against potential state machine
-     * violations.
-     */
-    PATH_STATE_ALREADY_COUNTED = 6,
-} path_state_t;
-#define path_state_bitfield_t ENUM_BF(path_state_t)
-
-/** An origin_circuit_t holds data necessary to build and use a circuit.
- */
-typedef struct origin_circuit_t {
-  circuit_t base_;
-
-  /** Linked list of AP streams (or EXIT streams if hidden service)
-   * associated with this circuit. */
-  edge_connection_t *p_streams;
-
-  /** Bytes read from any attached stream since last call to
-   * control_event_circ_bandwidth_used().  Only used if we're configured
-   * to emit CIRC_BW events. */
-  uint32_t n_read_circ_bw;
-
-  /** Bytes written to any attached stream since last call to
-   * control_event_circ_bandwidth_used().  Only used if we're configured
-   * to emit CIRC_BW events. */
-  uint32_t n_written_circ_bw;
-
-  /** Build state for this circuit. It includes the intended path
-   * length, the chosen exit router, rendezvous information, etc.
-   */
-  cpath_build_state_t *build_state;
-  /** The doubly-linked list of crypt_path_t entries, one per hop,
-   * for this circuit. This includes ciphers for each hop,
-   * integrity-checking digests for each hop, and package/delivery
-   * windows for each hop.
-   */
-  crypt_path_t *cpath;
-
-  /** Holds all rendezvous data on either client or service side. */
-  rend_data_t *rend_data;
-
-  /** How many more relay_early cells can we send on this circuit, according
-   * to the specification? */
-  unsigned int remaining_relay_early_cells : 4;
-
-  /** Set if this circuit is insanely old and we already informed the user */
-  unsigned int is_ancient : 1;
-
-  /** Set if this circuit has already been opened. Used to detect
-   * cannibalized circuits. */
-  unsigned int has_opened : 1;
-
-  /**
-   * Path bias state machine. Used to ensure integrity of our
-   * circuit building and usage accounting. See path_state_t
-   * for more details.
-   */
-  path_state_bitfield_t path_state : 3;
-
-  /* If this flag is set, we should not consider attaching any more
-   * connections to this circuit. */
-  unsigned int unusable_for_new_conns : 1;
-
-  /**
-   * Tristate variable to guard against pathbias miscounting
-   * due to circuit purpose transitions changing the decision
-   * of pathbias_should_count(). This variable is informational
-   * only. The current results of pathbias_should_count() are
-   * the official decision for pathbias accounting.
-   */
-  uint8_t pathbias_shouldcount;
-#define PATHBIAS_SHOULDCOUNT_UNDECIDED 0
-#define PATHBIAS_SHOULDCOUNT_IGNORED   1
-#define PATHBIAS_SHOULDCOUNT_COUNTED   2
-
-  /** For path probing. Store the temporary probe stream ID
-   * for response comparison */
-  streamid_t pathbias_probe_id;
-
-  /** For path probing. Store the temporary probe address nonce
-   * (in host byte order) for response comparison. */
-  uint32_t pathbias_probe_nonce;
-
-  /** Set iff this is a hidden-service circuit which has timed out
-   * according to our current circuit-build timeout, but which has
-   * been kept around because it might still succeed in connecting to
-   * its destination, and which is not a fully-connected rendezvous
-   * circuit.
-   *
-   * (We clear this flag for client-side rendezvous circuits when they
-   * are 'joined' to the other side's rendezvous circuit, so that
-   * connection_ap_handshake_attach_circuit can put client streams on
-   * the circuit.  We also clear this flag for service-side rendezvous
-   * circuits when they are 'joined' to a client's rend circ, but only
-   * for symmetry with the client case.  Client-side introduction
-   * circuits are closed when we get a joined rend circ, and
-   * service-side introduction circuits never have this flag set.) */
-  unsigned int hs_circ_has_timed_out : 1;
-
-  /** Set iff this circuit has been given a relaxed timeout because
-   * no circuits have opened. Used to prevent spamming logs. */
-  unsigned int relaxed_timeout : 1;
-
-  /** Set iff this is a service-side rendezvous circuit for which a
-   * new connection attempt has been launched.  We consider launching
-   * a new service-side rend circ to a client when the previous one
-   * fails; now that we don't necessarily close a service-side rend
-   * circ when we launch a new one to the same client, this flag keeps
-   * us from launching two retries for the same failed rend circ. */
-  unsigned int hs_service_side_rend_circ_has_been_relaunched : 1;
-
-  /** What commands were sent over this circuit that decremented the
-   * RELAY_EARLY counter? This is for debugging task 878. */
-  uint8_t relay_early_commands[MAX_RELAY_EARLY_CELLS_PER_CIRCUIT];
-
-  /** How many RELAY_EARLY cells have been sent over this circuit? This is
-   * for debugging task 878, too. */
-  int relay_early_cells_sent;
-
-  /** The next stream_id that will be tried when we're attempting to
-   * construct a new AP stream originating at this circuit. */
-  streamid_t next_stream_id;
-
-  /* The intro key replaces the hidden service's public key if purpose is
-   * S_ESTABLISH_INTRO or S_INTRO, provided that no unversioned rendezvous
-   * descriptor is used. */
-  crypto_pk_t *intro_key;
-
-  /** Quasi-global identifier for this circuit; used for control.c */
-  /* XXXX NM This can get re-used after 2**32 circuits. */
-  uint32_t global_identifier;
-
-  /** True if we have associated one stream to this circuit, thereby setting
-   * the isolation paramaters for this circuit.  Note that this doesn't
-   * necessarily mean that we've <em>attached</em> any streams to the circuit:
-   * we may only have marked up this circuit during the launch process.
-   */
-  unsigned int isolation_values_set : 1;
-  /** True iff any stream has <em>ever</em> been attached to this circuit.
-   *
-   * In a better world we could use timestamp_dirty for this, but
-   * timestamp_dirty is far too overloaded at the moment.
-   */
-  unsigned int isolation_any_streams_attached : 1;
-
-  /** A bitfield of ISO_* flags for every isolation field such that this
-   * circuit has had streams with more than one value for that field
-   * attached to it. */
-  uint8_t isolation_flags_mixed;
-
-  /** @name Isolation parameters
-   *
-   * If any streams have been associated with this circ (isolation_values_set
-   * == 1), and all streams associated with the circuit have had the same
-   * value for some field ((isolation_flags_mixed & ISO_FOO) == 0), then these
-   * elements hold the value for that field.
-   *
-   * Note again that "associated" is not the same as "attached": we
-   * preliminarily associate streams with a circuit while the circuit is being
-   * launched, so that we can tell whether we need to launch more circuits.
-   *
-   * @{
-   */
-  uint8_t client_proto_type;
-  uint8_t client_proto_socksver;
-  uint16_t dest_port;
-  tor_addr_t client_addr;
-  char *dest_address;
-  int session_group;
-  unsigned nym_epoch;
-  size_t socks_username_len;
-  uint8_t socks_password_len;
-  /* Note that the next two values are NOT NUL-terminated; see
-     socks_username_len and socks_password_len for their lengths. */
-  char *socks_username;
-  char *socks_password;
-  /** Global identifier for the first stream attached here; used by
-   * ISO_STREAM. */
-  uint64_t associated_isolated_stream_global_id;
-  /**@}*/
-  /** A list of addr_policy_t for this circuit in particular. Used by
-   * adjust_exit_policy_from_exitpolicy_failure.
-   */
-  smartlist_t *prepend_policy;
-} origin_circuit_t;
-
-struct onion_queue_t;
-
-/** An or_circuit_t holds information needed to implement a circuit at an
- * OR. */
-typedef struct or_circuit_t {
-  circuit_t base_;
-
-  /** Next circuit in the doubly-linked ring of circuits waiting to add
-   * cells to p_chan.  NULL if we have no cells pending, or if we're not
-   * linked to an OR connection. */
-  struct circuit_t *next_active_on_p_chan;
-  /** Previous circuit in the doubly-linked ring of circuits waiting to add
-   * cells to p_chan.  NULL if we have no cells pending, or if we're not
-   * linked to an OR connection. */
-  struct circuit_t *prev_active_on_p_chan;
-  /** Pointer to an entry on the onion queue, if this circuit is waiting for a
-   * chance to give an onionskin to a cpuworker. Used only in onion.c */
-  struct onion_queue_t *onionqueue_entry;
-  /** Pointer to a workqueue entry, if this circuit has given an onionskin to
-   * a cpuworker and is waiting for a response. Used to decide whether it is
-   * safe to free a circuit or if it is still in use by a cpuworker. */
-  struct workqueue_entry_s *workqueue_entry;
-
-  /** The circuit_id used in the previous (backward) hop of this circuit. */
-  circid_t p_circ_id;
-  /** Queue of cells waiting to be transmitted on p_conn. */
-  cell_queue_t p_chan_cells;
-  /** The channel that is previous in this circuit. */
-  channel_t *p_chan;
-  /**
-   * Circuit mux associated with p_chan to which this circuit is attached;
-   * NULL if we have no p_chan.
-   */
-  circuitmux_t *p_mux;
-  /** Linked list of Exit streams associated with this circuit. */
-  edge_connection_t *n_streams;
-  /** Linked list of Exit streams associated with this circuit that are
-   * still being resolved. */
-  edge_connection_t *resolving_streams;
-  /** The cipher used by intermediate hops for cells heading toward the
-   * OP. */
-  crypto_cipher_t *p_crypto;
-  /** The cipher used by intermediate hops for cells heading away from
-   * the OP. */
-  crypto_cipher_t *n_crypto;
-
-  /** The integrity-checking digest used by intermediate hops, for
-   * cells packaged here and heading towards the OP.
-   */
-  crypto_digest_t *p_digest;
-  /** The integrity-checking digest used by intermediate hops, for
-   * cells packaged at the OP and arriving here.
-   */
-  crypto_digest_t *n_digest;
-
-  /** Points to spliced circuit if purpose is REND_ESTABLISHED, and circuit
-   * is not marked for close. */
-  struct or_circuit_t *rend_splice;
-
-  struct or_circuit_rendinfo_s *rendinfo;
-
-  /** Stores KH for the handshake. */
-  char rend_circ_nonce[DIGEST_LEN];/* KH in tor-spec.txt */
-
-  /** How many more relay_early cells can we send on this circuit, according
-   * to the specification? */
-  unsigned int remaining_relay_early_cells : 4;
-
-  /* We have already received an INTRODUCE1 cell on this circuit. */
-  unsigned int already_received_introduce1 : 1;
-
-  /** True iff this circuit was made with a CREATE_FAST cell. */
-  unsigned int is_first_hop : 1;
-
-  /** If set, this circuit carries HS traffic. Consider it in any HS
-   *  statistics. */
-  unsigned int circuit_carries_hs_traffic_stats : 1;
-
-  /** Number of cells that were removed from circuit queue; reset every
-   * time when writing buffer stats to disk. */
-  uint32_t processed_cells;
-
-  /** Total time in milliseconds that cells spent in both app-ward and
-   * exit-ward queues of this circuit; reset every time when writing
-   * buffer stats to disk. */
-  uint64_t total_cell_waiting_time;
-
-  /** Maximum cell queue size for a middle relay; this is stored per circuit
-   * so append_cell_to_circuit_queue() can adjust it if it changes.  If set
-   * to zero, it is initialized to the default value.
-   */
-  uint32_t max_middle_cells;
-} or_circuit_t;
-
-typedef struct or_circuit_rendinfo_s {
-
-#if REND_COOKIE_LEN != DIGEST_LEN
-#error "The REND_TOKEN_LEN macro assumes REND_COOKIE_LEN == DIGEST_LEN"
-#endif
-#define REND_TOKEN_LEN DIGEST_LEN
-
-  /** A hash of location-hidden service's PK if purpose is INTRO_POINT, or a
-   * rendezvous cookie if purpose is REND_POINT_WAITING. Filled with zeroes
-   * otherwise.
-   */
-  char rend_token[REND_TOKEN_LEN];
-
-  /** True if this is a rendezvous point circuit; false if this is an
-   * introduction point. */
-  unsigned is_rend_circ;
-
-} or_circuit_rendinfo_t;
-
-/** Convert a circuit subtype to a circuit_t. */
-#define TO_CIRCUIT(x)  (&((x)->base_))
-
-/** Convert a circuit_t* to a pointer to the enclosing or_circuit_t.  Assert
- * if the cast is impossible. */
-static or_circuit_t *TO_OR_CIRCUIT(circuit_t *);
-static const or_circuit_t *CONST_TO_OR_CIRCUIT(const circuit_t *);
-/** Convert a circuit_t* to a pointer to the enclosing origin_circuit_t.
- * Assert if the cast is impossible. */
-static origin_circuit_t *TO_ORIGIN_CIRCUIT(circuit_t *);
-static const origin_circuit_t *CONST_TO_ORIGIN_CIRCUIT(const circuit_t *);
-
-/** Return 1 iff <b>node</b> has Exit flag and no BadExit flag.
- * Otherwise, return 0.
- */
-static inline int node_is_good_exit(const node_t *node)
-{
-  return node->is_exit && ! node->is_bad_exit;
-}
-
-static inline or_circuit_t *TO_OR_CIRCUIT(circuit_t *x)
-{
-  tor_assert(x->magic == OR_CIRCUIT_MAGIC);
-  return DOWNCAST(or_circuit_t, x);
-}
-static inline const or_circuit_t *CONST_TO_OR_CIRCUIT(const circuit_t *x)
-{
-  tor_assert(x->magic == OR_CIRCUIT_MAGIC);
-  return DOWNCAST(or_circuit_t, x);
-}
-static inline origin_circuit_t *TO_ORIGIN_CIRCUIT(circuit_t *x)
-{
-  tor_assert(x->magic == ORIGIN_CIRCUIT_MAGIC);
-  return DOWNCAST(origin_circuit_t, x);
-}
-static inline const origin_circuit_t *CONST_TO_ORIGIN_CIRCUIT(
-    const circuit_t *x)
-{
-  tor_assert(x->magic == ORIGIN_CIRCUIT_MAGIC);
-  return DOWNCAST(origin_circuit_t, x);
-}
-
-/** Bitfield type: things that we're willing to use invalid routers for. */
-typedef enum invalid_router_usage_t {
-  ALLOW_INVALID_ENTRY       =1,
-  ALLOW_INVALID_EXIT        =2,
-  ALLOW_INVALID_MIDDLE      =4,
-  ALLOW_INVALID_RENDEZVOUS  =8,
-  ALLOW_INVALID_INTRODUCTION=16,
-} invalid_router_usage_t;
-
-/* limits for TCP send and recv buffer size used for constrained sockets */
-#define MIN_CONSTRAINED_TCP_BUFFER 2048
-#define MAX_CONSTRAINED_TCP_BUFFER 262144  /* 256k */
-
-/** @name Isolation flags
-
-    Ways to isolate client streams
-
-    @{
-*/
-/** Isolate based on destination port */
-#define ISO_DESTPORT    (1u<<0)
-/** Isolate based on destination address */
-#define ISO_DESTADDR    (1u<<1)
-/** Isolate based on SOCKS authentication */
-#define ISO_SOCKSAUTH   (1u<<2)
-/** Isolate based on client protocol choice */
-#define ISO_CLIENTPROTO (1u<<3)
-/** Isolate based on client address */
-#define ISO_CLIENTADDR  (1u<<4)
-/** Isolate based on session group (always on). */
-#define ISO_SESSIONGRP  (1u<<5)
-/** Isolate based on newnym epoch (always on). */
-#define ISO_NYM_EPOCH   (1u<<6)
-/** Isolate all streams (Internal only). */
-#define ISO_STREAM      (1u<<7)
-/**@}*/
-
-/** Default isolation level for ports. */
-#define ISO_DEFAULT (ISO_CLIENTADDR|ISO_SOCKSAUTH|ISO_SESSIONGRP|ISO_NYM_EPOCH)
-
-/** Indicates that we haven't yet set a session group on a port_cfg_t. */
-#define SESSION_GROUP_UNSET -1
-/** Session group reserved for directory connections */
-#define SESSION_GROUP_DIRCONN -2
-/** Session group reserved for resolve requests launched by a controller */
-#define SESSION_GROUP_CONTROL_RESOLVE -3
-/** First automatically allocated session group number */
-#define SESSION_GROUP_FIRST_AUTO -4
-
-/** Configuration for a single port that we're listening on. */
-typedef struct port_cfg_t {
-  tor_addr_t addr; /**< The actual IP to listen on, if !is_unix_addr. */
-  int port; /**< The configured port, or CFG_AUTO_PORT to tell Tor to pick its
-             * own port. */
-  uint8_t type; /**< One of CONN_TYPE_*_LISTENER */
-  unsigned is_unix_addr : 1; /**< True iff this is an AF_UNIX address. */
-
-  unsigned is_group_writable : 1;
-  unsigned is_world_writable : 1;
-  unsigned relax_dirmode_check : 1;
-
-  entry_port_cfg_t entry_cfg;
-
-  server_port_cfg_t server_cfg;
-
-  /* Unix sockets only: */
-  /** Path for an AF_UNIX address */
-  char unix_addr[FLEXIBLE_ARRAY_MEMBER];
-} port_cfg_t;
-
-/** Ordinary configuration line. */
-#define CONFIG_LINE_NORMAL 0
-/** Appends to previous configuration for the same option, even if we
- * would ordinary replace it. */
-#define CONFIG_LINE_APPEND 1
-/* Removes all previous configuration for an option. */
-#define CONFIG_LINE_CLEAR 2
-
-/** A linked list of lines in a config file. */
-typedef struct config_line_t {
-  char *key;
-  char *value;
-  struct config_line_t *next;
-  /** What special treatment (if any) does this line require? */
-  unsigned int command:2;
-  /** If true, subsequent assignments to this linelist should replace
-   * it, not extend it.  Set only on the first item in a linelist in an
-   * or_options_t. */
-  unsigned int fragile:1;
-} config_line_t;
-
-typedef struct routerset_t routerset_t;
-
-/** A magic value for the (Socks|OR|...)Port options below, telling Tor
- * to pick its own port. */
-#define CFG_AUTO_PORT 0xc4005e
-
-/** Configuration options for a Tor process. */
-typedef struct {
-  uint32_t magic_;
-
-  /** What should the tor process actually do? */
-  enum {
-    CMD_RUN_TOR=0, CMD_LIST_FINGERPRINT, CMD_HASH_PASSWORD,
-    CMD_VERIFY_CONFIG, CMD_RUN_UNITTESTS, CMD_DUMP_CONFIG,
-    CMD_KEYGEN
-  } command;
-  char *command_arg; /**< Argument for command-line option. */
-
-  config_line_t *Logs; /**< New-style list of configuration lines
-                        * for logs */
-  int LogTimeGranularity; /**< Log resolution in milliseconds. */
-
-  int LogMessageDomains; /**< Boolean: Should we log the domain(s) in which
-                          * each log message occurs? */
-  int TruncateLogFile; /**< Boolean: Should we truncate the log file
-                            before we start writing? */
-  char *SyslogIdentityTag; /**< Identity tag to add for syslog logging. */
-
-  char *DebugLogFile; /**< Where to send verbose log messages. */
-  char *DataDirectory; /**< OR only: where to store long-term data. */
-  int DataDirectoryGroupReadable; /**< Boolean: Is the DataDirectory g+r? */
-  char *Nickname; /**< OR only: nickname of this onion router. */
-  char *Address; /**< OR only: configured address for this onion router. */
-  char *PidFile; /**< Where to store PID of Tor process. */
-
-  routerset_t *ExitNodes; /**< Structure containing nicknames, digests,
-                           * country codes and IP address patterns of ORs to
-                           * consider as exits. */
-  routerset_t *EntryNodes;/**< Structure containing nicknames, digests,
-                           * country codes and IP address patterns of ORs to
-                           * consider as entry points. */
-  int StrictNodes; /**< Boolean: When none of our EntryNodes or ExitNodes
-                    * are up, or we need to access a node in ExcludeNodes,
-                    * do we just fail instead? */
-  routerset_t *ExcludeNodes;/**< Structure containing nicknames, digests,
-                             * country codes and IP address patterns of ORs
-                             * not to use in circuits. But see StrictNodes
-                             * above. */
-  routerset_t *ExcludeExitNodes;/**< Structure containing nicknames, digests,
-                                 * country codes and IP address patterns of
-                                 * ORs not to consider as exits. */
-
-  /** Union of ExcludeNodes and ExcludeExitNodes */
-  routerset_t *ExcludeExitNodesUnion_;
-
-  int DisableAllSwap; /**< Boolean: Attempt to call mlockall() on our
-                       * process for all current and future memory. */
-
-  /** List of "entry", "middle", "exit", "introduction", "rendezvous". */
-  smartlist_t *AllowInvalidNodes;
-  /** Bitmask; derived from AllowInvalidNodes. */
-  invalid_router_usage_t AllowInvalid_;
-  config_line_t *ExitPolicy; /**< Lists of exit policy components. */
-  int ExitPolicyRejectPrivate; /**< Should we not exit to reserved private
-                                * addresses, and our own published addresses?
-                                */
-  int ExitPolicyRejectLocalInterfaces; /**< Should we not exit to local
-                                        * interface addresses?
-                                        * Includes OutboundBindAddresses and
-                                        * configured ports. */
-  config_line_t *SocksPolicy; /**< Lists of socks policy components */
-  config_line_t *DirPolicy; /**< Lists of dir policy components */
-  /** Addresses to bind for listening for SOCKS connections. */
-  config_line_t *SocksListenAddress;
-  /** Addresses to bind for listening for transparent pf/netfilter
-   * connections. */
-  config_line_t *TransListenAddress;
-  /** Addresses to bind for listening for transparent natd connections */
-  config_line_t *NATDListenAddress;
-  /** Addresses to bind for listening for SOCKS connections. */
-  config_line_t *DNSListenAddress;
-  /** Addresses to bind for listening for OR connections. */
-  config_line_t *ORListenAddress;
-  /** Addresses to bind for listening for directory connections. */
-  config_line_t *DirListenAddress;
-  /** Addresses to bind for listening for control connections. */
-  config_line_t *ControlListenAddress;
-  /** Local address to bind outbound sockets */
-  config_line_t *OutboundBindAddress;
-  /** IPv4 address derived from OutboundBindAddress. */
-  tor_addr_t OutboundBindAddressIPv4_;
-  /** IPv6 address derived from OutboundBindAddress. */
-  tor_addr_t OutboundBindAddressIPv6_;
-  /** Directory server only: which versions of
-   * Tor should we tell users to run? */
-  config_line_t *RecommendedVersions;
-  config_line_t *RecommendedClientVersions;
-  config_line_t *RecommendedServerVersions;
-  config_line_t *RecommendedPackages;
-  /** Whether dirservers allow router descriptors with private IPs. */
-  int DirAllowPrivateAddresses;
-  /** Whether routers accept EXTEND cells to routers with private IPs. */
-  int ExtendAllowPrivateAddresses;
-  char *User; /**< Name of user to run Tor as. */
-  char *Group; /**< Name of group to run Tor as. */
-  config_line_t *ORPort_lines; /**< Ports to listen on for OR connections. */
-  /** Ports to listen on for extended OR connections. */
-  config_line_t *ExtORPort_lines;
-  /** Ports to listen on for SOCKS connections. */
-  config_line_t *SocksPort_lines;
-  /** Ports to listen on for transparent pf/netfilter connections. */
-  config_line_t *TransPort_lines;
-  const char *TransProxyType; /**< What kind of transparent proxy
-                               * implementation are we using? */
-  /** Parsed value of TransProxyType. */
-  enum {
-    TPT_DEFAULT,
-    TPT_PF_DIVERT,
-    TPT_IPFW,
-    TPT_TPROXY,
-  } TransProxyType_parsed;
-  config_line_t *NATDPort_lines; /**< Ports to listen on for transparent natd
-                            * connections. */
-  config_line_t *ControlPort_lines; /**< Ports to listen on for control
-                               * connections. */
-  config_line_t *ControlSocket; /**< List of Unix Domain Sockets to listen on
-                                 * for control connections. */
-
-  int ControlSocketsGroupWritable; /**< Boolean: Are control sockets g+rw? */
-  int SocksSocketsGroupWritable; /**< Boolean: Are SOCKS sockets g+rw? */
-  /** Ports to listen on for directory connections. */
-  config_line_t *DirPort_lines;
-  config_line_t *DNSPort_lines; /**< Ports to listen on for DNS requests. */
-
-  /* MaxMemInQueues value as input by the user. We clean this up to be
-   * MaxMemInQueues. */
-  uint64_t MaxMemInQueues_raw;
-  uint64_t MaxMemInQueues;/**< If we have more memory than this allocated
-                            * for queues and buffers, run the OOM handler */
-  /** Above this value, consider ourselves low on RAM. */
-  uint64_t MaxMemInQueues_low_threshold;
-
-  /** @name port booleans
-   *
-   * Derived booleans: For server ports and ControlPort, true iff there is a
-   * non-listener port on an AF_INET or AF_INET6 address of the given type
-   * configured in one of the _lines options above.
-   * For client ports, also true if there is a unix socket configured.
-   * If you are checking for client ports, you may want to use:
-   *   SocksPort_set || TransPort_set || NATDPort_set || DNSPort_set
-   * rather than SocksPort_set.
-   *
-   * @{
-   */
-  unsigned int ORPort_set : 1;
-  unsigned int SocksPort_set : 1;
-  unsigned int TransPort_set : 1;
-  unsigned int NATDPort_set : 1;
-  unsigned int ControlPort_set : 1;
-  unsigned int DirPort_set : 1;
-  unsigned int DNSPort_set : 1;
-  unsigned int ExtORPort_set : 1;
-  /**@}*/
-
-  int AssumeReachable; /**< Whether to publish our descriptor regardless. */
-  int AuthoritativeDir; /**< Boolean: is this an authoritative directory? */
-  int V3AuthoritativeDir; /**< Boolean: is this an authoritative directory
-                           * for version 3 directories? */
-  int VersioningAuthoritativeDir; /**< Boolean: is this an authoritative
-                                   * directory that's willing to recommend
-                                   * versions? */
-  int BridgeAuthoritativeDir; /**< Boolean: is this an authoritative directory
-                               * that aggregates bridge descriptors? */
-
-  /** If set on a bridge authority, it will answer requests on its dirport
-   * for bridge statuses -- but only if the requests use this password. */
-  char *BridgePassword;
-  /** If BridgePassword is set, this is a SHA256 digest of the basic http
-   * authenticator for it. Used so we can do a time-independent comparison. */
-  char *BridgePassword_AuthDigest_;
-
-  int UseBridges; /**< Boolean: should we start all circuits with a bridge? */
-  config_line_t *Bridges; /**< List of bootstrap bridge addresses. */
-
-  config_line_t *ClientTransportPlugin; /**< List of client
-                                           transport plugins. */
-
-  config_line_t *ServerTransportPlugin; /**< List of client
-                                           transport plugins. */
-
-  /** List of TCP/IP addresses that transports should listen at. */
-  config_line_t *ServerTransportListenAddr;
-
-  /** List of options that must be passed to pluggable transports. */
-  config_line_t *ServerTransportOptions;
-
-  int BridgeRelay; /**< Boolean: are we acting as a bridge relay? We make
-                    * this explicit so we can change how we behave in the
-                    * future. */
-
-  /** Boolean: if we know the bridge's digest, should we get new
-   * descriptors from the bridge authorities or from the bridge itself? */
-  int UpdateBridgesFromAuthority;
-
-  int AvoidDiskWrites; /**< Boolean: should we never cache things to disk?
-                        * Not used yet. */
-  int ClientOnly; /**< Boolean: should we never evolve into a server role? */
-  /** To what authority types do we publish our descriptor? Choices are
-   * "v1", "v2", "v3", "bridge", or "". */
-  smartlist_t *PublishServerDescriptor;
-  /** A bitfield of authority types, derived from PublishServerDescriptor. */
-  dirinfo_type_t PublishServerDescriptor_;
-  /** Boolean: do we publish hidden service descriptors to the HS auths? */
-  int PublishHidServDescriptors;
-  int FetchServerDescriptors; /**< Do we fetch server descriptors as normal? */
-  int FetchHidServDescriptors; /**< and hidden service descriptors? */
-
-  int MinUptimeHidServDirectoryV2; /**< As directory authority, accept hidden
-                                    * service directories after what time? */
-
-  int FetchUselessDescriptors; /**< Do we fetch non-running descriptors too? */
-  int AllDirActionsPrivate; /**< Should every directory action be sent
-                             * through a Tor circuit? */
-
-  /** Run in 'tor2web mode'? (I.e. only make client connections to hidden
-   * services, and use a single hop for all hidden-service-related
-   * circuits.) */
-  int Tor2webMode;
-
-  /** A routerset that should be used when picking RPs for HS circuits. */
-  routerset_t *Tor2webRendezvousPoints;
-
-  /** Close hidden service client circuits immediately when they reach
-   * the normal circuit-build timeout, even if they have already sent
-   * an INTRODUCE1 cell on its way to the service. */
-  int CloseHSClientCircuitsImmediatelyOnTimeout;
-
-  /** Close hidden-service-side rendezvous circuits immediately when
-   * they reach the normal circuit-build timeout. */
-  int CloseHSServiceRendCircuitsImmediatelyOnTimeout;
-
-  /** Onion Services in HiddenServiceSingleHopMode make one-hop (direct)
-   * circuits between the onion service server, and the introduction and
-   * rendezvous points. (Onion service descriptors are still posted using
-   * 3-hop paths, to avoid onion service directories blocking the service.)
-   * This option makes every hidden service instance hosted by
-   * this tor instance a Single Onion Service.
-   * HiddenServiceSingleHopMode requires HiddenServiceNonAnonymousMode to be
-   * set to 1.
-   * Use rend_service_allow_non_anonymous_connection() or
-   * rend_service_reveal_startup_time() instead of using this option directly.
-   */
-  int HiddenServiceSingleHopMode;
-  /* Makes hidden service clients and servers non-anonymous on this tor
-   * instance. Allows the non-anonymous HiddenServiceSingleHopMode. Enables
-   * non-anonymous behaviour in the hidden service protocol.
-   * Use rend_service_non_anonymous_mode_enabled() instead of using this option
-   * directly.
-   */
-  int HiddenServiceNonAnonymousMode;
-
-  int ConnLimit; /**< Demanded minimum number of simultaneous connections. */
-  int ConnLimit_; /**< Maximum allowed number of simultaneous connections. */
-  int ConnLimit_high_thresh; /**< start trying to lower socket usage if we
-                              *   have this many. */
-  int ConnLimit_low_thresh; /**< try to get down to here after socket
-                             *   exhaustion. */
-  int RunAsDaemon; /**< If true, run in the background. (Unix only) */
-  int FascistFirewall; /**< Whether to prefer ORs reachable on open ports. */
-  smartlist_t *FirewallPorts; /**< Which ports our firewall allows
-                               * (strings). */
-  config_line_t *ReachableAddresses; /**< IP:ports our firewall allows. */
-  config_line_t *ReachableORAddresses; /**< IP:ports for OR conns. */
-  config_line_t *ReachableDirAddresses; /**< IP:ports for Dir conns. */
-
-  int ConstrainedSockets; /**< Shrink xmit and recv socket buffers. */
-  uint64_t ConstrainedSockSize; /**< Size of constrained buffers. */
-
-  /** Whether we should drop exit streams from Tors that we don't know are
-   * relays.  One of "0" (never refuse), "1" (always refuse), or "-1" (do
-   * what the consensus says, defaulting to 'refuse' if the consensus says
-   * nothing). */
-  int RefuseUnknownExits;
-
-  /** Application ports that require all nodes in circ to have sufficient
-   * uptime. */
-  smartlist_t *LongLivedPorts;
-  /** Application ports that are likely to be unencrypted and
-   * unauthenticated; we reject requests for them to prevent the
-   * user from screwing up and leaking plaintext secrets to an
-   * observer somewhere on the Internet. */
-  smartlist_t *RejectPlaintextPorts;
-  /** Related to RejectPlaintextPorts above, except this config option
-   * controls whether we warn (in the log and via a controller status
-   * event) every time a risky connection is attempted. */
-  smartlist_t *WarnPlaintextPorts;
-  /** Should we try to reuse the same exit node for a given host */
-  smartlist_t *TrackHostExits;
-  int TrackHostExitsExpire; /**< Number of seconds until we expire an
-                             * addressmap */
-  config_line_t *AddressMap; /**< List of address map directives. */
-  int AutomapHostsOnResolve; /**< If true, when we get a resolve request for a
-                              * hostname ending with one of the suffixes in
-                              * <b>AutomapHostsSuffixes</b>, map it to a
-                              * virtual address. */
-  /** List of suffixes for <b>AutomapHostsOnResolve</b>.  The special value
-   * "." means "match everything." */
-  smartlist_t *AutomapHostsSuffixes;
-  int RendPostPeriod; /**< How often do we post each rendezvous service
-                       * descriptor? Remember to publish them independently. */
-  int KeepalivePeriod; /**< How often do we send padding cells to keep
-                        * connections alive? */
-  int SocksTimeout; /**< How long do we let a socks connection wait
-                     * unattached before we fail it? */
-  int LearnCircuitBuildTimeout; /**< If non-zero, we attempt to learn a value
-                                 * for CircuitBuildTimeout based on timeout
-                                 * history. Use circuit_build_times_disabled()
-                                 * rather than checking this value directly. */
-  int CircuitBuildTimeout; /**< Cull non-open circuits that were born at
-                            * least this many seconds ago. Used until
-                            * adaptive algorithm learns a new value. */
-  int CircuitIdleTimeout; /**< Cull open clean circuits that were born
-                           * at least this many seconds ago. */
-  int CircuitStreamTimeout; /**< If non-zero, detach streams from circuits
-                             * and try a new circuit if the stream has been
-                             * waiting for this many seconds. If zero, use
-                             * our default internal timeout schedule. */
-  int MaxOnionQueueDelay; /*< DOCDOC */
-  int NewCircuitPeriod; /**< How long do we use a circuit before building
-                         * a new one? */
-  int MaxCircuitDirtiness; /**< Never use circs that were first used more than
-                                this interval ago. */
-  int PredictedPortsRelevanceTime; /** How long after we've requested a
-                                    * connection for a given port, do we want
-                                    * to continue to pick exits that support
-                                    * that port?  */
-  uint64_t BandwidthRate; /**< How much bandwidth, on average, are we willing
-                           * to use in a second? */
-  uint64_t BandwidthBurst; /**< How much bandwidth, at maximum, are we willing
-                            * to use in a second? */
-  uint64_t MaxAdvertisedBandwidth; /**< How much bandwidth are we willing to
-                                    * tell people we have? */
-  uint64_t RelayBandwidthRate; /**< How much bandwidth, on average, are we
-                                 * willing to use for all relayed conns? */
-  uint64_t RelayBandwidthBurst; /**< How much bandwidth, at maximum, will we
-                                 * use in a second for all relayed conns? */
-  uint64_t PerConnBWRate; /**< Long-term bw on a single TLS conn, if set. */
-  uint64_t PerConnBWBurst; /**< Allowed burst on a single TLS conn, if set. */
-  int NumCPUs; /**< How many CPUs should we try to use? */
-//int RunTesting; /**< If true, create testing circuits to measure how well the
-//                 * other ORs are running. */
-  config_line_t *RendConfigLines; /**< List of configuration lines
-                                          * for rendezvous services. */
-  config_line_t *HidServAuth; /**< List of configuration lines for client-side
-                               * authorizations for hidden services */
-  char *ContactInfo; /**< Contact info to be published in the directory. */
-
-  int HeartbeatPeriod; /**< Log heartbeat messages after this many seconds
-                        * have passed. */
-
-  char *HTTPProxy; /**< hostname[:port] to use as http proxy, if any. */
-  tor_addr_t HTTPProxyAddr; /**< Parsed IPv4 addr for http proxy, if any. */
-  uint16_t HTTPProxyPort; /**< Parsed port for http proxy, if any. */
-  char *HTTPProxyAuthenticator; /**< username:password string, if any. */
-
-  char *HTTPSProxy; /**< hostname[:port] to use as https proxy, if any. */
-  tor_addr_t HTTPSProxyAddr; /**< Parsed addr for https proxy, if any. */
-  uint16_t HTTPSProxyPort; /**< Parsed port for https proxy, if any. */
-  char *HTTPSProxyAuthenticator; /**< username:password string, if any. */
-
-  char *Socks4Proxy; /**< hostname:port to use as a SOCKS4 proxy, if any. */
-  tor_addr_t Socks4ProxyAddr; /**< Derived from Socks4Proxy. */
-  uint16_t Socks4ProxyPort; /**< Derived from Socks4Proxy. */
-
-  char *Socks5Proxy; /**< hostname:port to use as a SOCKS5 proxy, if any. */
-  tor_addr_t Socks5ProxyAddr; /**< Derived from Sock5Proxy. */
-  uint16_t Socks5ProxyPort; /**< Derived from Socks5Proxy. */
-  char *Socks5ProxyUsername; /**< Username for SOCKS5 authentication, if any */
-  char *Socks5ProxyPassword; /**< Password for SOCKS5 authentication, if any */
-
-  /** List of configuration lines for replacement directory authorities.
-   * If you just want to replace one class of authority at a time,
-   * use the "Alternate*Authority" options below instead. */
-  config_line_t *DirAuthorities;
-
-  /** List of fallback directory servers */
-  config_line_t *FallbackDir;
-  /** Whether to use the default hard-coded FallbackDirs */
-  int UseDefaultFallbackDirs;
-
-  /** Weight to apply to all directory authority rates if considering them
-   * along with fallbackdirs */
-  double DirAuthorityFallbackRate;
-
-  /** If set, use these main (currently v3) directory authorities and
-   * not the default ones. */
-  config_line_t *AlternateDirAuthority;
-
-  /** If set, use these bridge authorities and not the default one. */
-  config_line_t *AlternateBridgeAuthority;
-
-  char *MyFamily; /**< Declared family for this OR. */
-  config_line_t *NodeFamilies; /**< List of config lines for
-                                * node families */
-  smartlist_t *NodeFamilySets; /**< List of parsed NodeFamilies values. */
-  config_line_t *AuthDirBadExit; /**< Address policy for descriptors to
-                                  * mark as bad exits. */
-  config_line_t *AuthDirReject; /**< Address policy for descriptors to
-                                 * reject. */
-  config_line_t *AuthDirInvalid; /**< Address policy for descriptors to
-                                  * never mark as valid. */
-  /** @name AuthDir...CC
-   *
-   * Lists of country codes to mark as BadExit, or Invalid, or to
-   * reject entirely.
-   *
-   * @{
-   */
-  smartlist_t *AuthDirBadExitCCs;
-  smartlist_t *AuthDirInvalidCCs;
-  smartlist_t *AuthDirRejectCCs;
-  /**@}*/
-
-  int AuthDirListBadExits; /**< True iff we should list bad exits,
-                            * and vote for all other exits as good. */
-  int AuthDirMaxServersPerAddr; /**< Do not permit more than this
-                                 * number of servers per IP address. */
-  int AuthDirMaxServersPerAuthAddr; /**< Do not permit more than this
-                                     * number of servers per IP address shared
-                                     * with an authority. */
-  int AuthDirHasIPv6Connectivity; /**< Boolean: are we on IPv6?  */
-  int AuthDirPinKeys; /**< Boolean: Do we enforce key-pinning? */
-
-  /** If non-zero, always vote the Fast flag for any relay advertising
-   * this amount of capacity or more. */
-  uint64_t AuthDirFastGuarantee;
-
-  /** If non-zero, this advertised capacity or more is always sufficient
-   * to satisfy the bandwidth requirement for the Guard flag. */
-  uint64_t AuthDirGuardBWGuarantee;
-
-  char *AccountingStart; /**< How long is the accounting interval, and when
-                          * does it start? */
-  uint64_t AccountingMax; /**< How many bytes do we allow per accounting
-                           * interval before hibernation?  0 for "never
-                           * hibernate." */
-  /** How do we determine when our AccountingMax has been reached?
-   * "max" for when in or out reaches AccountingMax
-   * "sum" for when in plus out reaches AccountingMax
-   * "in"  for when in reaches AccountingMax
-   * "out" for when out reaches AccountingMax */
-  char *AccountingRule_option;
-  enum { ACCT_MAX, ACCT_SUM, ACCT_IN, ACCT_OUT } AccountingRule;
-
-  /** Base64-encoded hash of accepted passwords for the control system. */
-  config_line_t *HashedControlPassword;
-  /** As HashedControlPassword, but not saved. */
-  config_line_t *HashedControlSessionPassword;
-
-  int CookieAuthentication; /**< Boolean: do we enable cookie-based auth for
-                             * the control system? */
-  char *CookieAuthFile; /**< Filesystem location of a ControlPort
-                         *   authentication cookie. */
-  char *ExtORPortCookieAuthFile; /**< Filesystem location of Extended
-                                 *   ORPort authentication cookie. */
-  int CookieAuthFileGroupReadable; /**< Boolean: Is the CookieAuthFile g+r? */
-  int ExtORPortCookieAuthFileGroupReadable; /**< Boolean: Is the
-                                             * ExtORPortCookieAuthFile g+r? */
-  int LeaveStreamsUnattached; /**< Boolean: Does Tor attach new streams to
-                          * circuits itself (0), or does it expect a controller
-                          * to cope? (1) */
-  int DisablePredictedCircuits; /**< Boolean: does Tor preemptively
-                                 * make circuits in the background (0),
-                                 * or not (1)? */
-
-  /** Process specifier for a controller that ‘owns’ this Tor
-   * instance.  Tor will terminate if its owning controller does. */
-  char *OwningControllerProcess;
-
-  int ShutdownWaitLength; /**< When we get a SIGINT and we're a server, how
-                           * long do we wait before exiting? */
-  char *SafeLogging; /**< Contains "relay", "1", "0" (meaning no scrubbing). */
-
-  /* Derived from SafeLogging */
-  enum {
-    SAFELOG_SCRUB_ALL, SAFELOG_SCRUB_RELAY, SAFELOG_SCRUB_NONE
-  } SafeLogging_;
-
-  int Sandbox; /**< Boolean: should sandboxing be enabled? */
-  int SafeSocks; /**< Boolean: should we outright refuse application
-                  * connections that use socks4 or socks5-with-local-dns? */
-#define LOG_PROTOCOL_WARN (get_options()->ProtocolWarnings ? \
-                           LOG_WARN : LOG_INFO)
-  int ProtocolWarnings; /**< Boolean: when other parties screw up the Tor
-                         * protocol, is it a warn or an info in our logs? */
-  int TestSocks; /**< Boolean: when we get a socks connection, do we loudly
-                  * log whether it was DNS-leaking or not? */
-  int HardwareAccel; /**< Boolean: Should we enable OpenSSL hardware
-                      * acceleration where available? */
-  /** Token Bucket Refill resolution in milliseconds. */
-  int TokenBucketRefillInterval;
-  char *AccelName; /**< Optional hardware acceleration engine name. */
-  char *AccelDir; /**< Optional hardware acceleration engine search dir. */
-
-  /** Boolean: Do we try to enter from a smallish number
-   * of fixed nodes? */
-  int UseEntryGuards_option;
-  /** Internal variable to remember whether we're actually acting on
-   * UseEntryGuards_option -- when we're a non-anonymous Tor2web client or
-   * Single Onion Service, it is alwasy false, otherwise we use the value of
-   * UseEntryGuards_option. */
-  int UseEntryGuards;
-
-  int NumEntryGuards; /**< How many entry guards do we try to establish? */
-  int UseEntryGuardsAsDirGuards; /** Boolean: Do we try to get directory info
-                                  * from a smallish number of fixed nodes? */
-
-  /** If 1, we use any guardfraction information we see in the
-   * consensus.  If 0, we don't.  If -1, let the consensus parameter
-   * decide. */
-  int UseGuardFraction;
-
-  int NumDirectoryGuards; /**< How many dir guards do we try to establish?
-                           * If 0, use value from NumEntryGuards. */
-  int RephistTrackTime; /**< How many seconds do we keep rephist info? */
-  int FastFirstHopPK; /**< If Tor believes it is safe, should we save a third
-                       * of our PK time by sending CREATE_FAST cells? */
-  /** Should we always fetch our dir info on the mirror schedule (which
-   * means directly from the authorities) no matter our other config? */
-  int FetchDirInfoEarly;
-
-  /** Should we fetch our dir info at the start of the consensus period? */
-  int FetchDirInfoExtraEarly;
-
-  int DirCache; /**< Cache all directory documents and accept requests via
-                 * tunnelled dir conns from clients. If 1, enabled (default);
-                 * If 0, disabled. */
-
-  char *VirtualAddrNetworkIPv4; /**< Address and mask to hand out for virtual
-                                 * MAPADDRESS requests for IPv4 addresses */
-  char *VirtualAddrNetworkIPv6; /**< Address and mask to hand out for virtual
-                                 * MAPADDRESS requests for IPv6 addresses */
-  int ServerDNSSearchDomains; /**< Boolean: If set, we don't force exit
-                      * addresses to be FQDNs, but rather search for them in
-                      * the local domains. */
-  int ServerDNSDetectHijacking; /**< Boolean: If true, check for DNS failure
-                                 * hijacking. */
-  int ServerDNSRandomizeCase; /**< Boolean: Use the 0x20-hack to prevent
-                               * DNS poisoning attacks. */
-  char *ServerDNSResolvConfFile; /**< If provided, we configure our internal
-                     * resolver from the file here rather than from
-                     * /etc/resolv.conf (Unix) or the registry (Windows). */
-  char *DirPortFrontPage; /**< This is a full path to a file with an html
-                    disclaimer. This allows a server administrator to show
-                    that they're running Tor and anyone visiting their server
-                    will know this without any specialized knowledge. */
-  int DisableDebuggerAttachment; /**< Currently Linux only specific attempt to
-                                      disable ptrace; needs BSD testing. */
-  /** Boolean: if set, we start even if our resolv.conf file is missing
-   * or broken. */
-  int ServerDNSAllowBrokenConfig;
-  /** Boolean: if set, then even connections to private addresses will get
-   * rate-limited. */
-  int CountPrivateBandwidth;
-  smartlist_t *ServerDNSTestAddresses; /**< A list of addresses that definitely
-                                        * should be resolvable. Used for
-                                        * testing our DNS server. */
-  int EnforceDistinctSubnets; /**< If true, don't allow multiple routers in the
-                               * same network zone in the same circuit. */
-  int PortForwarding; /**< If true, use NAT-PMP or UPnP to automatically
-                       * forward the DirPort and ORPort on the NAT device */
-  char *PortForwardingHelper; /** < Filename or full path of the port
-                                  forwarding helper executable */
-  int AllowNonRFC953Hostnames; /**< If true, we allow connections to hostnames
-                                * with weird characters. */
-  /** If true, we try resolving hostnames with weird characters. */
-  int ServerDNSAllowNonRFC953Hostnames;
-
-  /** If true, we try to download extra-info documents (and we serve them,
-   * if we are a cache).  For authorities, this is always true. */
-  int DownloadExtraInfo;
-
-  /** If true, and we are acting as a relay, allow exit circuits even when
-   * we are the first hop of a circuit. */
-  int AllowSingleHopExits;
-  /** If true, don't allow relays with AllowSingleHopExits=1 to be used in
-   * circuits that we build. */
-  int ExcludeSingleHopRelays;
-  /** If true, and the controller tells us to use a one-hop circuit, and the
-   * exit allows it, we use it. */
-  int AllowSingleHopCircuits;
-
-  /** If true, we convert "www.google.com.foo.exit" addresses on the
-   * socks/trans/natd ports into "www.google.com" addresses that
-   * exit from the node "foo". Disabled by default since attacking
-   * websites and exit relays can use it to manipulate your path
-   * selection. */
-  int AllowDotExit;
-
-  /** If true, we will warn if a user gives us only an IP address
-   * instead of a hostname. */
-  int WarnUnsafeSocks;
-
-  /** If true, we're configured to collect statistics on clients
-   * requesting network statuses from us as directory. */
-  int DirReqStatistics_option;
-  /** Internal variable to remember whether we're actually acting on
-   * DirReqStatistics_option -- yes if it's set and we're a server, else no. */
-  int DirReqStatistics;
-
-  /** If true, the user wants us to collect statistics on port usage. */
-  int ExitPortStatistics;
-
-  /** If true, the user wants us to collect connection statistics. */
-  int ConnDirectionStatistics;
-
-  /** If true, the user wants us to collect cell statistics. */
-  int CellStatistics;
-
-  /** If true, the user wants us to collect statistics as entry node. */
-  int EntryStatistics;
-
-  /** If true, the user wants us to collect statistics as hidden service
-   * directory, introduction point, or rendezvous point. */
-  int HiddenServiceStatistics;
-
-  /** If true, include statistics file contents in extra-info documents. */
-  int ExtraInfoStatistics;
-
-  /** If true, do not believe anybody who tells us that a domain resolves
-   * to an internal address, or that an internal address has a PTR mapping.
-   * Helps avoid some cross-site attacks. */
-  int ClientDNSRejectInternalAddresses;
-
-  /** If true, do not accept any requests to connect to internal addresses
-   * over randomly chosen exits. */
-  int ClientRejectInternalAddresses;
-
-  /** If true, clients may connect over IPv4. If false, they will avoid
-   * connecting over IPv4. We enforce this for OR and Dir connections. */
-  int ClientUseIPv4;
-  /** If true, clients may connect over IPv6. If false, they will avoid
-   * connecting over IPv4. We enforce this for OR and Dir connections.
-   * Use fascist_firewall_use_ipv6() instead of accessing this value
-   * directly. */
-  int ClientUseIPv6;
-  /** If true, prefer an IPv6 OR port over an IPv4 one for entry node
-   * connections. If auto, bridge clients prefer IPv6, and other clients
-   * prefer IPv4. Use node_ipv6_or_preferred() instead of accessing this value
-   * directly. */
-  int ClientPreferIPv6ORPort;
-  /** If true, prefer an IPv6 directory port over an IPv4 one for direct
-   * directory connections. If auto, bridge clients prefer IPv6, and other
-   * clients prefer IPv4. Use fascist_firewall_prefer_ipv6_dirport() instead of
-   * accessing this value directly.  */
-  int ClientPreferIPv6DirPort;
-
-  /** The length of time that we think a consensus should be fresh. */
-  int V3AuthVotingInterval;
-  /** The length of time we think it will take to distribute votes. */
-  int V3AuthVoteDelay;
-  /** The length of time we think it will take to distribute signatures. */
-  int V3AuthDistDelay;
-  /** The number of intervals we think a consensus should be valid. */
-  int V3AuthNIntervalsValid;
-
-  /** Should advertise and sign consensuses with a legacy key, for key
-   * migration purposes? */
-  int V3AuthUseLegacyKey;
-
-  /** Location of bandwidth measurement file */
-  char *V3BandwidthsFile;
-
-  /** Location of guardfraction file */
-  char *GuardfractionFile;
-
-  /** Authority only: key=value pairs that we add to our networkstatus
-   * consensus vote on the 'params' line. */
-  char *ConsensusParams;
-
-  /** Authority only: minimum number of measured bandwidths we must see
-   * before we only believe measured bandwidths to assign flags. */
-  int MinMeasuredBWsForAuthToIgnoreAdvertised;
-
-  /** The length of time that we think an initial consensus should be fresh.
-   * Only altered on testing networks. */
-  int TestingV3AuthInitialVotingInterval;
-
-  /** The length of time we think it will take to distribute initial votes.
-   * Only altered on testing networks. */
-  int TestingV3AuthInitialVoteDelay;
-
-  /** The length of time we think it will take to distribute initial
-   * signatures.  Only altered on testing networks.*/
-  int TestingV3AuthInitialDistDelay;
-
-  /** Offset in seconds added to the starting time for consensus
-      voting. Only altered on testing networks. */
-  int TestingV3AuthVotingStartOffset;
-
-  /** If an authority has been around for less than this amount of time, it
-   * does not believe its reachability information is accurate.  Only
-   * altered on testing networks. */
-  int TestingAuthDirTimeToLearnReachability;
-
-  /** Clients don't download any descriptor this recent, since it will
-   * probably not have propagated to enough caches.  Only altered on testing
-   * networks. */
-  int TestingEstimatedDescriptorPropagationTime;
-
-  /** Schedule for when servers should download things in general.  Only
-   * altered on testing networks. */
-  smartlist_t *TestingServerDownloadSchedule;
-
-  /** Schedule for when clients should download things in general.  Only
-   * altered on testing networks. */
-  smartlist_t *TestingClientDownloadSchedule;
-
-  /** Schedule for when servers should download consensuses.  Only altered
-   * on testing networks. */
-  smartlist_t *TestingServerConsensusDownloadSchedule;
-
-  /** Schedule for when clients should download consensuses.  Only altered
-   * on testing networks. */
-  smartlist_t *TestingClientConsensusDownloadSchedule;
-
-  /** Schedule for when clients should download consensuses from authorities
-   * if they are bootstrapping (that is, they don't have a usable, reasonably
-   * live consensus).  Only used by clients fetching from a list of fallback
-   * directory mirrors.
-   *
-   * This schedule is incremented by (potentially concurrent) connection
-   * attempts, unlike other schedules, which are incremented by connection
-   * failures.  Only altered on testing networks. */
-  smartlist_t *ClientBootstrapConsensusAuthorityDownloadSchedule;
-
-  /** Schedule for when clients should download consensuses from fallback
-   * directory mirrors if they are bootstrapping (that is, they don't have a
-   * usable, reasonably live consensus). Only used by clients fetching from a
-   * list of fallback directory mirrors.
-   *
-   * This schedule is incremented by (potentially concurrent) connection
-   * attempts, unlike other schedules, which are incremented by connection
-   * failures.  Only altered on testing networks. */
-  smartlist_t *ClientBootstrapConsensusFallbackDownloadSchedule;
-
-  /** Schedule for when clients should download consensuses from authorities
-   * if they are bootstrapping (that is, they don't have a usable, reasonably
-   * live consensus).  Only used by clients which don't have or won't fetch
-   * from a list of fallback directory mirrors.
-   *
-   * This schedule is incremented by (potentially concurrent) connection
-   * attempts, unlike other schedules, which are incremented by connection
-   * failures.  Only altered on testing networks. */
-  smartlist_t *ClientBootstrapConsensusAuthorityOnlyDownloadSchedule;
-
-  /** Schedule for when clients should download bridge descriptors.  Only
-   * altered on testing networks. */
-  smartlist_t *TestingBridgeDownloadSchedule;
-
-  /** When directory clients have only a few descriptors to request, they
-   * batch them until they have more, or until this amount of time has
-   * passed.  Only altered on testing networks. */
-  int TestingClientMaxIntervalWithoutRequest;
-
-  /** How long do we let a directory connection stall before expiring
-   * it?  Only altered on testing networks. */
-  int TestingDirConnectionMaxStall;
-
-  /** How many times will we try to fetch a consensus before we give
-   * up?  Only altered on testing networks. */
-  int TestingConsensusMaxDownloadTries;
-
-  /** How many times will a client try to fetch a consensus while
-   * bootstrapping using a list of fallback directories, before it gives up?
-   * Only altered on testing networks. */
-  int ClientBootstrapConsensusMaxDownloadTries;
-
-  /** How many times will a client try to fetch a consensus while
-   * bootstrapping using only a list of authorities, before it gives up?
-   * Only altered on testing networks. */
-  int ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries;
-
-  /** How many simultaneous in-progress connections will we make when trying
-   * to fetch a consensus before we wait for one to complete, timeout, or
-   * error out?  Only altered on testing networks. */
-  int ClientBootstrapConsensusMaxInProgressTries;
-
-  /** How many times will we try to download a router's descriptor before
-   * giving up?  Only altered on testing networks. */
-  int TestingDescriptorMaxDownloadTries;
-
-  /** How many times will we try to download a microdescriptor before
-   * giving up?  Only altered on testing networks. */
-  int TestingMicrodescMaxDownloadTries;
-
-  /** How many times will we try to fetch a certificate before giving
-   * up?  Only altered on testing networks. */
-  int TestingCertMaxDownloadTries;
-
-  /** If true, we take part in a testing network. Change the defaults of a
-   * couple of other configuration options and allow to change the values
-   * of certain configuration options. */
-  int TestingTorNetwork;
-
-  /** Minimum value for the Exit flag threshold on testing networks. */
-  uint64_t TestingMinExitFlagThreshold;
-
-  /** Minimum value for the Fast flag threshold on testing networks. */
-  uint64_t TestingMinFastFlagThreshold;
-
-  /** Relays in a testing network which should be voted Exit
-   * regardless of exit policy. */
-  routerset_t *TestingDirAuthVoteExit;
-  int TestingDirAuthVoteExitIsStrict;
-
-  /** Relays in a testing network which should be voted Guard
-   * regardless of uptime and bandwidth. */
-  routerset_t *TestingDirAuthVoteGuard;
-  int TestingDirAuthVoteGuardIsStrict;
-
-  /** Relays in a testing network which should be voted HSDir
-   * regardless of uptime and DirPort.
-   * Respects VoteOnHidServDirectoriesV2. */
-  routerset_t *TestingDirAuthVoteHSDir;
-  int TestingDirAuthVoteHSDirIsStrict;
-
-  /** Enable CONN_BW events.  Only altered on testing networks. */
-  int TestingEnableConnBwEvent;
-
-  /** Enable CELL_STATS events.  Only altered on testing networks. */
-  int TestingEnableCellStatsEvent;
-
-  /** Enable TB_EMPTY events.  Only altered on testing networks. */
-  int TestingEnableTbEmptyEvent;
-
-  /** If true, and we have GeoIP data, and we're a bridge, keep a per-country
-   * count of how many client addresses have contacted us so that we can help
-   * the bridge authority guess which countries have blocked access to us. */
-  int BridgeRecordUsageByCountry;
-
-  /** Optionally, IPv4 and IPv6 GeoIP data. */
-  char *GeoIPFile;
-  char *GeoIPv6File;
-
-  /** Autobool: if auto, then any attempt to Exclude{Exit,}Nodes a particular
-   * country code will exclude all nodes in ?? and A1.  If true, all nodes in
-   * ?? and A1 are excluded. Has no effect if we don't know any GeoIP data. */
-  int GeoIPExcludeUnknown;
-
-  /** If true, SIGHUP should reload the torrc.  Sometimes controllers want
-   * to make this false. */
-  int ReloadTorrcOnSIGHUP;
-
-  /* The main parameter for picking circuits within a connection.
-   *
-   * If this value is positive, when picking a cell to relay on a connection,
-   * we always relay from the circuit whose weighted cell count is lowest.
-   * Cells are weighted exponentially such that if one cell is sent
-   * 'CircuitPriorityHalflife' seconds before another, it counts for half as
-   * much.
-   *
-   * If this value is zero, we're disabling the cell-EWMA algorithm.
-   *
-   * If this value is negative, we're using the default approach
-   * according to either Tor or a parameter set in the consensus.
-   */
-  double CircuitPriorityHalflife;
-
-  /** Set to true if the TestingTorNetwork configuration option is set.
-   * This is used so that options_validate() has a chance to realize that
-   * the defaults have changed. */
-  int UsingTestNetworkDefaults_;
-
-  /** If 1, we try to use microdescriptors to build circuits.  If 0, we don't.
-   * If -1, Tor decides. */
-  int UseMicrodescriptors;
-
-  /** File where we should write the ControlPort. */
-  char *ControlPortWriteToFile;
-  /** Should that file be group-readable? */
-  int ControlPortFileGroupReadable;
-
-#define MAX_MAX_CLIENT_CIRCUITS_PENDING 1024
-  /** Maximum number of non-open general-purpose origin circuits to allow at
-   * once. */
-  int MaxClientCircuitsPending;
-
-  /** If 1, we always send optimistic data when it's supported.  If 0, we
-   * never use it.  If -1, we do what the consensus says. */
-  int OptimisticData;
-
-  /** If 1, we accept and launch no external network connections, except on
-   * control ports. */
-  int DisableNetwork;
-
-  /**
-   * Parameters for path-bias detection.
-   * @{
-   * These options override the default behavior of Tor's (**currently
-   * experimental**) path bias detection algorithm. To try to find broken or
-   * misbehaving guard nodes, Tor looks for nodes where more than a certain
-   * fraction of circuits through that guard fail to get built.
-   *
-   * The PathBiasCircThreshold option controls how many circuits we need to
-   * build through a guard before we make these checks.  The
-   * PathBiasNoticeRate, PathBiasWarnRate and PathBiasExtremeRate options
-   * control what fraction of circuits must succeed through a guard so we
-   * won't write log messages.  If less than PathBiasExtremeRate circuits
-   * succeed *and* PathBiasDropGuards is set to 1, we disable use of that
-   * guard.
-   *
-   * When we have seen more than PathBiasScaleThreshold circuits through a
-   * guard, we scale our observations by 0.5 (governed by the consensus) so
-   * that new observations don't get swamped by old ones.
-   *
-   * By default, or if a negative value is provided for one of these options,
-   * Tor uses reasonable defaults from the networkstatus consensus document.
-   * If no defaults are available there, these options default to 150, .70,
-   * .50, .30, 0, and 300 respectively.
-   */
-  int PathBiasCircThreshold;
-  double PathBiasNoticeRate;
-  double PathBiasWarnRate;
-  double PathBiasExtremeRate;
-  int PathBiasDropGuards;
-  int PathBiasScaleThreshold;
-  /** @} */
-
-  /**
-   * Parameters for path-bias use detection
-   * @{
-   * Similar to the above options, these options override the default behavior
-   * of Tor's (**currently experimental**) path use bias detection algorithm.
-   *
-   * Where as the path bias parameters govern thresholds for successfully
-   * building circuits, these four path use bias parameters govern thresholds
-   * only for circuit usage. Circuits which receive no stream usage are not
-   * counted by this detection algorithm. A used circuit is considered
-   * successful if it is capable of carrying streams or otherwise receiving
-   * well-formed responses to RELAY cells.
-   *
-   * By default, or if a negative value is provided for one of these options,
-   * Tor uses reasonable defaults from the networkstatus consensus document.
-   * If no defaults are available there, these options default to 20, .80,
-   * .60, and 100, respectively.
-   */
-  int PathBiasUseThreshold;
-  double PathBiasNoticeUseRate;
-  double PathBiasExtremeUseRate;
-  int PathBiasScaleUseThreshold;
-  /** @} */
-
-  int IPv6Exit; /**< Do we support exiting to IPv6 addresses? */
-
-  char *TLSECGroup; /**< One of "P256", "P224", or nil for auto */
-
-  /** Fraction: */
-  double PathsNeededToBuildCircuits;
-
-  /** What expiry time shall we place on our SSL certs? "0" means we
-   * should guess a suitable value. */
-  int SSLKeyLifetime;
-
-  /** How long (seconds) do we keep a guard before picking a new one? */
-  int GuardLifetime;
-
-  /** Low-water mark for global scheduler - start sending when estimated
-   * queued size falls below this threshold.
-   */
-  uint64_t SchedulerLowWaterMark__;
-  /** High-water mark for global scheduler - stop sending when estimated
-   * queued size exceeds this threshold.
-   */
-  uint64_t SchedulerHighWaterMark__;
-  /** Flush size for global scheduler - flush this many cells at a time
-   * when sending.
-   */
-  int SchedulerMaxFlushCells__;
-
-  /** Is this an exit node?  This is a tristate, where "1" means "yes, and use
-   * the default exit policy if none is given" and "0" means "no; exit policy
-   * is 'reject *'" and "auto" (-1) means "same as 1, but warn the user."
-   *
-   * XXXX Eventually, the default will be 0. */
-  int ExitRelay;
-
-  /** For how long (seconds) do we declare our singning keys to be valid? */
-  int SigningKeyLifetime;
-  /** For how long (seconds) do we declare our link keys to be valid? */
-  int TestingLinkCertLifetime;
-  /** For how long (seconds) do we declare our auth keys to be valid? */
-  int TestingAuthKeyLifetime;
-
-  /** How long before signing keys expire will we try to make a new one? */
-  int TestingSigningKeySlop;
-  /** How long before link keys expire will we try to make a new one? */
-  int TestingLinkKeySlop;
-  /** How long before auth keys expire will we try to make a new one? */
-  int TestingAuthKeySlop;
-
-  /** Force use of offline master key features: never generate a master
-   * ed25519 identity key except from tor --keygen */
-  int OfflineMasterKey;
-
-  enum {
-    FORCE_PASSPHRASE_AUTO=0,
-    FORCE_PASSPHRASE_ON,
-    FORCE_PASSPHRASE_OFF
-  } keygen_force_passphrase;
-  int use_keygen_passphrase_fd;
-  int keygen_passphrase_fd;
-  int change_key_passphrase;
-  char *master_key_fname;
-
-  /** Autobool: Do we try to retain capabilities if we can? */
-  int KeepBindCapabilities;
-
-  /** Maximum total size of unparseable descriptors to log during the
-   * lifetime of this Tor process.
-   */
-  uint64_t MaxUnparseableDescSizeToLog;
-
-  /** Bool (default: 1): Switch for the shared random protocol. Only
-   * relevant to a directory authority. If off, the authority won't
-   * participate in the protocol. If on (default), a flag is added to the
-   * vote indicating participation. */
-  int AuthDirSharedRandomness;
-
-  /** If 1, we skip all OOS checks. */
-  int DisableOOSCheck;
-
-  /** Autobool: Is the circuit creation DoS mitigation subsystem enabled? */
-  int DoSCircuitCreationEnabled;
-  /** Minimum concurrent connection needed from one single address before any
-   * defense is used. */
-  int DoSCircuitCreationMinConnections;
-  /** Circuit rate used to refill the token bucket. */
-  int DoSCircuitCreationRate;
-  /** Maximum allowed burst of circuits. Reaching that value, the address is
-   * detected as malicious and a defense might be used. */
-  int DoSCircuitCreationBurst;
-  /** When an address is marked as malicous, what defense should be used
-   * against it. See the dos_cc_defense_type_t enum. */
-  int DoSCircuitCreationDefenseType;
-  /** For how much time (in seconds) the defense is applicable for a malicious
-   * address. A random time delta is added to the defense time of an address
-   * which will be between 1 second and half of this value. */
-  int DoSCircuitCreationDefenseTimePeriod;
-
-  /** Autobool: Is the DoS connection mitigation subsystem enabled? */
-  int DoSConnectionEnabled;
-  /** Maximum concurrent connection allowed per address. */
-  int DoSConnectionMaxConcurrentCount;
-  /** When an address is reaches the maximum count, what defense should be
-   * used against it. See the dos_conn_defense_type_t enum. */
-  int DoSConnectionDefenseType;
-
-  /** Autobool: Do we refuse single hop client rendezvous? */
-  int DoSRefuseSingleHopClientRendezvous;
-} or_options_t;
-
-/** Persistent state for an onion router, as saved to disk. */
-typedef struct {
-  uint32_t magic_;
-  /** The time at which we next plan to write the state to the disk.  Equal to
-   * TIME_MAX if there are no savable changes, 0 if there are changes that
-   * should be saved right away. */
-  time_t next_write;
-
-  /** When was the state last written to disk? */
-  time_t LastWritten;
-
-  /** Fields for accounting bandwidth use. */
-  time_t AccountingIntervalStart;
-  uint64_t AccountingBytesReadInInterval;
-  uint64_t AccountingBytesWrittenInInterval;
-  int AccountingSecondsActive;
-  int AccountingSecondsToReachSoftLimit;
-  time_t AccountingSoftLimitHitAt;
-  uint64_t AccountingBytesAtSoftLimit;
-  uint64_t AccountingExpectedUsage;
-
-  /** A list of Entry Guard-related configuration lines. */
-  config_line_t *EntryGuards;
-
-  config_line_t *TransportProxies;
-
-  /** These fields hold information on the history of bandwidth usage for
-   * servers.  The "Ends" fields hold the time when we last updated the
-   * bandwidth usage. The "Interval" fields hold the granularity, in seconds,
-   * of the entries of Values.  The "Values" lists hold decimal string
-   * representations of the number of bytes read or written in each
-   * interval. The "Maxima" list holds decimal strings describing the highest
-   * rate achieved during the interval.
-   */
-  time_t      BWHistoryReadEnds;
-  int         BWHistoryReadInterval;
-  smartlist_t *BWHistoryReadValues;
-  smartlist_t *BWHistoryReadMaxima;
-  time_t      BWHistoryWriteEnds;
-  int         BWHistoryWriteInterval;
-  smartlist_t *BWHistoryWriteValues;
-  smartlist_t *BWHistoryWriteMaxima;
-  time_t      BWHistoryDirReadEnds;
-  int         BWHistoryDirReadInterval;
-  smartlist_t *BWHistoryDirReadValues;
-  smartlist_t *BWHistoryDirReadMaxima;
-  time_t      BWHistoryDirWriteEnds;
-  int         BWHistoryDirWriteInterval;
-  smartlist_t *BWHistoryDirWriteValues;
-  smartlist_t *BWHistoryDirWriteMaxima;
-
-  /** Build time histogram */
-  config_line_t * BuildtimeHistogram;
-  unsigned int TotalBuildTimes;
-  unsigned int CircuitBuildAbandonedCount;
-
-  /** What version of Tor wrote this state file? */
-  char *TorVersion;
-
-  /** Holds any unrecognized values we found in the state file, in the order
-   * in which we found them. */
-  config_line_t *ExtraLines;
-
-  /** When did we last rotate our onion key?  "0" for 'no idea'. */
-  time_t LastRotatedOnionKey;
-} or_state_t;
-
-/** Change the next_write time of <b>state</b> to <b>when</b>, unless the
- * state is already scheduled to be written to disk earlier than <b>when</b>.
- */
-static inline void or_state_mark_dirty(or_state_t *state, time_t when)
-{
-  if (state->next_write > when)
-    state->next_write = when;
-}
-
-#define MAX_SOCKS_REPLY_LEN 1024
-#define MAX_SOCKS_ADDR_LEN 256
-#define SOCKS_NO_AUTH 0x00
-#define SOCKS_USER_PASS 0x02
-
-/** Please open a TCP connection to this addr:port. */
-#define SOCKS_COMMAND_CONNECT       0x01
-/** Please turn this FQDN into an IP address, privately. */
-#define SOCKS_COMMAND_RESOLVE       0xF0
-/** Please turn this IP address into an FQDN, privately. */
-#define SOCKS_COMMAND_RESOLVE_PTR   0xF1
-
-/* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */
-#define SOCKS_COMMAND_IS_CONNECT(c) (((c)==SOCKS_COMMAND_CONNECT) || 0)
-#define SOCKS_COMMAND_IS_RESOLVE(c) ((c)==SOCKS_COMMAND_RESOLVE || \
-                                     (c)==SOCKS_COMMAND_RESOLVE_PTR)
-
-/** State of a SOCKS request from a user to an OP.  Also used to encode other
- * information for non-socks user request (such as those on TransPort and
- * DNSPort) */
-struct socks_request_t {
-  /** Which version of SOCKS did the client use? One of "0, 4, 5" -- where
-   * 0 means that no socks handshake ever took place, and this is just a
-   * stub connection (e.g. see connection_ap_make_link()). */
-  uint8_t socks_version;
-  /** If using socks5 authentication, which authentication type did we
-   * negotiate?  currently we support 0 (no authentication) and 2
-   * (username/password). */
-  uint8_t auth_type;
-  /** What is this stream's goal? One of the SOCKS_COMMAND_* values */
-  uint8_t command;
-  /** Which kind of listener created this stream? */
-  uint8_t listener_type;
-  size_t replylen; /**< Length of <b>reply</b>. */
-  uint8_t reply[MAX_SOCKS_REPLY_LEN]; /**< Write an entry into this string if
-                                    * we want to specify our own socks reply,
-                                    * rather than using the default socks4 or
-                                    * socks5 socks reply. We use this for the
-                                    * two-stage socks5 handshake.
-                                    */
-  char address[MAX_SOCKS_ADDR_LEN]; /**< What address did the client ask to
-                                       connect to/resolve? */
-  uint16_t port; /**< What port did the client ask to connect to? */
-  unsigned int has_finished : 1; /**< Has the SOCKS handshake finished? Used to
-                              * make sure we send back a socks reply for
-                              * every connection. */
-  unsigned int got_auth : 1; /**< Have we received any authentication data? */
-  /** If this is set, we will choose "no authentication" instead of
-   * "username/password" authentication if both are offered. Used as input to
-   * parse_socks. */
-  unsigned int socks_prefer_no_auth : 1;
-
-  /** Number of bytes in username; 0 if username is NULL */
-  size_t usernamelen;
-  /** Number of bytes in password; 0 if password is NULL */
-  uint8_t passwordlen;
-  /** The negotiated username value if any (for socks5), or the entire
-   * authentication string (for socks4).  This value is NOT nul-terminated;
-   * see usernamelen for its length. */
-  char *username;
-  /** The negotiated password value if any (for socks5). This value is NOT
-   * nul-terminated; see passwordlen for its length. */
-  char *password;
-};
-
-/********************************* circuitbuild.c **********************/
-
-/** How many hops does a general-purpose circuit have by default? */
-#define DEFAULT_ROUTE_LEN 3
-
-/* Circuit Build Timeout "public" structures. */
-
-/** Precision multiplier for the Bw weights */
-#define BW_WEIGHT_SCALE   10000
-#define BW_MIN_WEIGHT_SCALE 1
-#define BW_MAX_WEIGHT_SCALE INT32_MAX
-
-/** Total size of the circuit timeout history to accumulate.
- * 1000 is approx 2.5 days worth of continual-use circuits. */
-#define CBT_NCIRCUITS_TO_OBSERVE 1000
-
-/** Width of the histogram bins in milliseconds */
-#define CBT_BIN_WIDTH ((build_time_t)50)
-
-/** Number of modes to use in the weighted-avg computation of Xm */
-#define CBT_DEFAULT_NUM_XM_MODES 3
-#define CBT_MIN_NUM_XM_MODES 1
-#define CBT_MAX_NUM_XM_MODES 20
-
-/** A build_time_t is milliseconds */
-typedef uint32_t build_time_t;
-
-/**
- * CBT_BUILD_ABANDONED is our flag value to represent a force-closed
- * circuit (Aka a 'right-censored' pareto value).
- */
-#define CBT_BUILD_ABANDONED ((build_time_t)(INT32_MAX-1))
-#define CBT_BUILD_TIME_MAX ((build_time_t)(INT32_MAX))
-
-/** Save state every 10 circuits */
-#define CBT_SAVE_STATE_EVERY 10
-
-/* Circuit build times consensus parameters */
-
-/**
- * How long to wait before actually closing circuits that take too long to
- * build in terms of CDF quantile.
- */
-#define CBT_DEFAULT_CLOSE_QUANTILE 95
-#define CBT_MIN_CLOSE_QUANTILE CBT_MIN_QUANTILE_CUTOFF
-#define CBT_MAX_CLOSE_QUANTILE CBT_MAX_QUANTILE_CUTOFF
-
-/**
- * How many circuits count as recent when considering if the
- * connection has gone gimpy or changed.
- */
-#define CBT_DEFAULT_RECENT_CIRCUITS 20
-#define CBT_MIN_RECENT_CIRCUITS 3
-#define CBT_MAX_RECENT_CIRCUITS 1000
-
-/**
- * Maximum count of timeouts that finish the first hop in the past
- * RECENT_CIRCUITS before calculating a new timeout.
- *
- * This tells us whether to abandon timeout history and set
- * the timeout back to whatever circuit_build_times_get_initial_timeout()
- * gives us.
- */
-#define CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT (CBT_DEFAULT_RECENT_CIRCUITS*9/10)
-#define CBT_MIN_MAX_RECENT_TIMEOUT_COUNT 3
-#define CBT_MAX_MAX_RECENT_TIMEOUT_COUNT 10000
-
-/** Minimum circuits before estimating a timeout */
-#define CBT_DEFAULT_MIN_CIRCUITS_TO_OBSERVE 100
-#define CBT_MIN_MIN_CIRCUITS_TO_OBSERVE 1
-#define CBT_MAX_MIN_CIRCUITS_TO_OBSERVE 10000
-
-/** Cutoff percentile on the CDF for our timeout estimation. */
-#define CBT_DEFAULT_QUANTILE_CUTOFF 80
-#define CBT_MIN_QUANTILE_CUTOFF 10
-#define CBT_MAX_QUANTILE_CUTOFF 99
-double circuit_build_times_quantile_cutoff(void);
-
-/** How often in seconds should we build a test circuit */
-#define CBT_DEFAULT_TEST_FREQUENCY 60
-#define CBT_MIN_TEST_FREQUENCY 1
-#define CBT_MAX_TEST_FREQUENCY INT32_MAX
-
-/** Lowest allowable value for CircuitBuildTimeout in milliseconds */
-#define CBT_DEFAULT_TIMEOUT_MIN_VALUE (1500)
-#define CBT_MIN_TIMEOUT_MIN_VALUE 500
-#define CBT_MAX_TIMEOUT_MIN_VALUE INT32_MAX
-
-/** Initial circuit build timeout in milliseconds */
-#define CBT_DEFAULT_TIMEOUT_INITIAL_VALUE (60*1000)
-#define CBT_MIN_TIMEOUT_INITIAL_VALUE CBT_MIN_TIMEOUT_MIN_VALUE
-#define CBT_MAX_TIMEOUT_INITIAL_VALUE INT32_MAX
-int32_t circuit_build_times_initial_timeout(void);
-
-#if CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT < CBT_MIN_MAX_RECENT_TIMEOUT_COUNT
-#error "RECENT_CIRCUITS is set too low."
-#endif
-
-/** Information about the state of our local network connection */
-typedef struct {
-  /** The timestamp we last completed a TLS handshake or received a cell */
-  time_t network_last_live;
-  /** If the network is not live, how many timeouts has this caused? */
-  int nonlive_timeouts;
-  /** Circular array of circuits that have made it to the first hop. Slot is
-   * 1 if circuit timed out, 0 if circuit succeeded */
-  int8_t *timeouts_after_firsthop;
-  /** Number of elements allocated for the above array */
-  int num_recent_circs;
-  /** Index into circular array. */
-  int after_firsthop_idx;
-} network_liveness_t;
-
-typedef struct circuit_build_times_s circuit_build_times_t;
-
-/********************************* config.c ***************************/
-
-/** An error from options_trial_assign() or options_init_from_string(). */
-typedef enum setopt_err_t {
-  SETOPT_OK = 0,
-  SETOPT_ERR_MISC = -1,
-  SETOPT_ERR_PARSE = -2,
-  SETOPT_ERR_TRANSITION = -3,
-  SETOPT_ERR_SETTING = -4,
-} setopt_err_t;
-
-/********************************* connection_edge.c *************************/
-
-/** Enumerates possible origins of a client-side address mapping. */
-typedef enum {
-  /** We're remapping this address because the controller told us to. */
-  ADDRMAPSRC_CONTROLLER,
-  /** We're remapping this address because of an AutomapHostsOnResolve
-   * configuration. */
-  ADDRMAPSRC_AUTOMAP,
-  /** We're remapping this address because our configuration (via torrc, the
-   * command line, or a SETCONF command) told us to. */
-  ADDRMAPSRC_TORRC,
-  /** We're remapping this address because we have TrackHostExit configured,
-   * and we want to remember to use the same exit next time. */
-  ADDRMAPSRC_TRACKEXIT,
-  /** We're remapping this address because we got a DNS resolution from a
-   * Tor server that told us what its value was. */
-  ADDRMAPSRC_DNS,
-
-  /** No remapping has occurred.  This isn't a possible value for an
-   * addrmap_entry_t; it's used as a null value when we need to answer "Why
-   * did this remapping happen." */
-  ADDRMAPSRC_NONE
-} addressmap_entry_source_t;
-#define addressmap_entry_source_bitfield_t ENUM_BF(addressmap_entry_source_t)
-
-/********************************* control.c ***************************/
-
-/** Used to indicate the type of a circuit event passed to the controller.
- * The various types are defined in control-spec.txt */
-typedef enum circuit_status_event_t {
-  CIRC_EVENT_LAUNCHED = 0,
-  CIRC_EVENT_BUILT    = 1,
-  CIRC_EVENT_EXTENDED = 2,
-  CIRC_EVENT_FAILED   = 3,
-  CIRC_EVENT_CLOSED   = 4,
-} circuit_status_event_t;
-
-/** Used to indicate the type of a CIRC_MINOR event passed to the controller.
- * The various types are defined in control-spec.txt . */
-typedef enum circuit_status_minor_event_t {
-  CIRC_MINOR_EVENT_PURPOSE_CHANGED,
-  CIRC_MINOR_EVENT_CANNIBALIZED,
-} circuit_status_minor_event_t;
-
-/** Used to indicate the type of a stream event passed to the controller.
- * The various types are defined in control-spec.txt */
-typedef enum stream_status_event_t {
-  STREAM_EVENT_SENT_CONNECT = 0,
-  STREAM_EVENT_SENT_RESOLVE = 1,
-  STREAM_EVENT_SUCCEEDED    = 2,
-  STREAM_EVENT_FAILED       = 3,
-  STREAM_EVENT_CLOSED       = 4,
-  STREAM_EVENT_NEW          = 5,
-  STREAM_EVENT_NEW_RESOLVE  = 6,
-  STREAM_EVENT_FAILED_RETRIABLE = 7,
-  STREAM_EVENT_REMAP        = 8
-} stream_status_event_t;
-
-/** Used to indicate the type of an OR connection event passed to the
- * controller.  The various types are defined in control-spec.txt */
-typedef enum or_conn_status_event_t {
-  OR_CONN_EVENT_LAUNCHED     = 0,
-  OR_CONN_EVENT_CONNECTED    = 1,
-  OR_CONN_EVENT_FAILED       = 2,
-  OR_CONN_EVENT_CLOSED       = 3,
-  OR_CONN_EVENT_NEW          = 4,
-} or_conn_status_event_t;
-
-/** Used to indicate the type of a buildtime event */
-typedef enum buildtimeout_set_event_t {
-  BUILDTIMEOUT_SET_EVENT_COMPUTED  = 0,
-  BUILDTIMEOUT_SET_EVENT_RESET     = 1,
-  BUILDTIMEOUT_SET_EVENT_SUSPENDED = 2,
-  BUILDTIMEOUT_SET_EVENT_DISCARD = 3,
-  BUILDTIMEOUT_SET_EVENT_RESUME = 4
-} buildtimeout_set_event_t;
-
-/** Execute the statement <b>stmt</b>, which may log events concerning the
- * connection <b>conn</b>.  To prevent infinite loops, disable log messages
- * being sent to controllers if <b>conn</b> is a control connection.
- *
- * Stmt must not contain any return or goto statements.
- */
-#define CONN_LOG_PROTECT(conn, stmt)                                    \
-  STMT_BEGIN                                                            \
-    int _log_conn_is_control;                                           \
-    tor_assert(conn);                                                   \
-    _log_conn_is_control = (conn->type == CONN_TYPE_CONTROL);           \
-    if (_log_conn_is_control)                                           \
-      disable_control_logging();                                        \
-  STMT_BEGIN stmt; STMT_END;                                            \
-    if (_log_conn_is_control)                                           \
-      enable_control_logging();                                         \
-  STMT_END
-
-/** Enum describing various stages of bootstrapping, for use with controller
- * bootstrap status events. The values range from 0 to 100. */
-typedef enum {
-  BOOTSTRAP_STATUS_UNDEF=-1,
-  BOOTSTRAP_STATUS_STARTING=0,
-  BOOTSTRAP_STATUS_CONN_DIR=5,
-  BOOTSTRAP_STATUS_HANDSHAKE=-2,
-  BOOTSTRAP_STATUS_HANDSHAKE_DIR=10,
-  BOOTSTRAP_STATUS_ONEHOP_CREATE=15,
-  BOOTSTRAP_STATUS_REQUESTING_STATUS=20,
-  BOOTSTRAP_STATUS_LOADING_STATUS=25,
-  BOOTSTRAP_STATUS_LOADING_KEYS=40,
-  BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS=45,
-  BOOTSTRAP_STATUS_LOADING_DESCRIPTORS=50,
-  BOOTSTRAP_STATUS_CONN_OR=80,
-  BOOTSTRAP_STATUS_HANDSHAKE_OR=85,
-  BOOTSTRAP_STATUS_CIRCUIT_CREATE=90,
-  BOOTSTRAP_STATUS_DONE=100
-} bootstrap_status_t;
-
-/********************************* directory.c ***************************/
-
-/** A pair of digests created by dir_split_resource_info_fingerprint_pairs() */
-typedef struct {
-  char first[DIGEST_LEN];
-  char second[DIGEST_LEN];
-} fp_pair_t;
-
-/********************************* dirserv.c ***************************/
-
-/** An enum to describe what format we're generating a routerstatus line in.
- */
-typedef enum {
-  /** For use in a v2 opinion */
-  NS_V2,
-  /** For use in a consensus networkstatus document (ns flavor) */
-  NS_V3_CONSENSUS,
-  /** For use in a vote networkstatus document */
-  NS_V3_VOTE,
-  /** For passing to the controlport in response to a GETINFO request */
-  NS_CONTROL_PORT,
-  /** For use in a consensus networkstatus document (microdesc flavor) */
-  NS_V3_CONSENSUS_MICRODESC
-} routerstatus_format_type_t;
-
-#ifdef DIRSERV_PRIVATE
-typedef struct measured_bw_line_t {
-  char node_id[DIGEST_LEN];
-  char node_hex[MAX_HEX_NICKNAME_LEN+1];
-  long int bw_kb;
-} measured_bw_line_t;
-
-#endif
-
-/********************************* dirvote.c ************************/
-
-/** Describes the schedule by which votes should be generated. */
-typedef struct vote_timing_t {
-  /** Length in seconds between one consensus becoming valid and the next
-   * becoming valid. */
-  int vote_interval;
-  /** For how many intervals is a consensus valid? */
-  int n_intervals_valid;
-  /** Time in seconds allowed to propagate votes */
-  int vote_delay;
-  /** Time in seconds allowed to propagate signatures */
-  int dist_delay;
-} vote_timing_t;
-
-/********************************* geoip.c **************************/
-
-/** Indicates an action that we might be noting geoip statistics on.
- * Note that if we're noticing CONNECT, we're a bridge, and if we're noticing
- * the others, we're not.
- */
-typedef enum {
-  /** We've noticed a connection as a bridge relay or entry guard. */
-  GEOIP_CLIENT_CONNECT = 0,
-  /** We've served a networkstatus consensus as a directory server. */
-  GEOIP_CLIENT_NETWORKSTATUS = 1,
-} geoip_client_action_t;
-/** Indicates either a positive reply or a reason for rejectng a network
- * status request that will be included in geoip statistics. */
-typedef enum {
-  /** Request is answered successfully. */
-  GEOIP_SUCCESS = 0,
-  /** V3 network status is not signed by a sufficient number of requested
-   * authorities. */
-  GEOIP_REJECT_NOT_ENOUGH_SIGS = 1,
-  /** Requested network status object is unavailable. */
-  GEOIP_REJECT_UNAVAILABLE = 2,
-  /** Requested network status not found. */
-  GEOIP_REJECT_NOT_FOUND = 3,
-  /** Network status has not been modified since If-Modified-Since time. */
-  GEOIP_REJECT_NOT_MODIFIED = 4,
-  /** Directory is busy. */
-  GEOIP_REJECT_BUSY = 5,
-} geoip_ns_response_t;
-#define GEOIP_NS_RESPONSE_NUM 6
-
-/** Directory requests that we are measuring can be either direct or
- * tunneled. */
-typedef enum {
-  DIRREQ_DIRECT = 0,
-  DIRREQ_TUNNELED = 1,
-} dirreq_type_t;
-
-/** Possible states for either direct or tunneled directory requests that
- * are relevant for determining network status download times. */
-typedef enum {
-  /** Found that the client requests a network status; applies to both
-   * direct and tunneled requests; initial state of a request that we are
-   * measuring. */
-  DIRREQ_IS_FOR_NETWORK_STATUS = 0,
-  /** Finished writing a network status to the directory connection;
-   * applies to both direct and tunneled requests; completes a direct
-   * request. */
-  DIRREQ_FLUSHING_DIR_CONN_FINISHED = 1,
-  /** END cell sent to circuit that initiated a tunneled request. */
-  DIRREQ_END_CELL_SENT = 2,
-  /** Flushed last cell from queue of the circuit that initiated a
-    * tunneled request to the outbuf of the OR connection. */
-  DIRREQ_CIRC_QUEUE_FLUSHED = 3,
-  /** Flushed last byte from buffer of the channel belonging to the
-    * circuit that initiated a tunneled request; completes a tunneled
-    * request. */
-  DIRREQ_CHANNEL_BUFFER_FLUSHED = 4
-} dirreq_state_t;
-
-#define WRITE_STATS_INTERVAL (24*60*60)
-
-/********************************* microdesc.c *************************/
-
-typedef struct microdesc_cache_t microdesc_cache_t;
-
-/********************************* networkstatus.c *********************/
-
-/** Possible statuses of a version of Tor, given opinions from the directory
- * servers. */
-typedef enum version_status_t {
-  VS_RECOMMENDED=0, /**< This version is listed as recommended. */
-  VS_OLD=1, /**< This version is older than any recommended version. */
-  VS_NEW=2, /**< This version is newer than any recommended version. */
-  VS_NEW_IN_SERIES=3, /**< This version is newer than any recommended version
-                       * in its series, but later recommended versions exist.
-                       */
-  VS_UNRECOMMENDED=4, /**< This version is not recommended (general case). */
-  VS_EMPTY=5, /**< The version list was empty; no agreed-on versions. */
-  VS_UNKNOWN, /**< We have no idea. */
-} version_status_t;
-
-/********************************* policies.c ************************/
-
-/** Outcome of applying an address policy to an address. */
-typedef enum {
-  /** The address was accepted */
-  ADDR_POLICY_ACCEPTED=0,
-  /** The address was rejected */
-  ADDR_POLICY_REJECTED=-1,
-  /** Part of the address was unknown, but as far as we can tell, it was
-   * accepted. */
-  ADDR_POLICY_PROBABLY_ACCEPTED=1,
-  /** Part of the address was unknown, but as far as we can tell, it was
-   * rejected. */
-  ADDR_POLICY_PROBABLY_REJECTED=2,
-} addr_policy_result_t;
-
-/********************************* rephist.c ***************************/
-
-/** Possible public/private key operations in Tor: used to keep track of where
- * we're spending our time. */
-typedef enum {
-  SIGN_DIR, SIGN_RTR,
-  VERIFY_DIR, VERIFY_RTR,
-  ENC_ONIONSKIN, DEC_ONIONSKIN,
-  TLS_HANDSHAKE_C, TLS_HANDSHAKE_S,
-  REND_CLIENT, REND_MID, REND_SERVER,
-} pk_op_t;
-
-/********************************* rendcommon.c ***************************/
-
-/** Hidden-service side configuration of client authorization. */
-typedef struct rend_authorized_client_t {
-  char *client_name;
-  uint8_t descriptor_cookie[REND_DESC_COOKIE_LEN];
-  crypto_pk_t *client_key;
-} rend_authorized_client_t;
-
-/** ASCII-encoded v2 hidden service descriptor. */
-typedef struct rend_encoded_v2_service_descriptor_t {
-  char desc_id[DIGEST_LEN]; /**< Descriptor ID. */
-  char *desc_str; /**< Descriptor string. */
-} rend_encoded_v2_service_descriptor_t;
-
-/** The maximum number of non-circuit-build-timeout failures a hidden
- * service client will tolerate while trying to build a circuit to an
- * introduction point.  See also rend_intro_point_t.unreachable_count. */
-#define MAX_INTRO_POINT_REACHABILITY_FAILURES 5
-
-/** The minimum and maximum number of distinct INTRODUCE2 cells which a
- * hidden service's introduction point will receive before it begins to
- * expire. */
-#define INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS 16384
-/* Double the minimum value so the interval is [min, min * 2]. */
-#define INTRO_POINT_MAX_LIFETIME_INTRODUCTIONS \
-  (INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS * 2)
-
-/** The minimum number of seconds that an introduction point will last
- * before expiring due to old age.  (If it receives
- * INTRO_POINT_LIFETIME_INTRODUCTIONS INTRODUCE2 cells, it may expire
- * sooner.)
- *
- * XXX Should this be configurable? */
-#define INTRO_POINT_LIFETIME_MIN_SECONDS (18*60*60)
-/** The maximum number of seconds that an introduction point will last
- * before expiring due to old age.
- *
- * XXX Should this be configurable? */
-#define INTRO_POINT_LIFETIME_MAX_SECONDS (24*60*60)
-
-/** The maximum number of circuit creation retry we do to an intro point
- * before giving up. We try to reuse intro point that fails during their
- * lifetime so this is a hard limit on the amount of time we do that. */
-#define MAX_INTRO_POINT_CIRCUIT_RETRIES 3
-
-/** Introduction point information.  Used both in rend_service_t (on
- * the service side) and in rend_service_descriptor_t (on both the
- * client and service side). */
-typedef struct rend_intro_point_t {
-  extend_info_t *extend_info; /**< Extend info for connecting to this
-                               * introduction point via a multi-hop path. */
-  crypto_pk_t *intro_key; /**< Introduction key that replaces the service
-                               * key, if this descriptor is V2. */
-
-  /** (Client side only) Flag indicating that a timeout has occurred
-   * after sending an INTRODUCE cell to this intro point.  After a
-   * timeout, an intro point should not be tried again during the same
-   * hidden service connection attempt, but it may be tried again
-   * during a future connection attempt. */
-  unsigned int timed_out : 1;
-
-  /** (Client side only) The number of times we have failed to build a
-   * circuit to this intro point for some reason other than our
-   * circuit-build timeout.  See also MAX_INTRO_POINT_REACHABILITY_FAILURES. */
-  unsigned int unreachable_count : 3;
-
-  /** (Service side only) Flag indicating that this intro point was
-   * included in the last HS descriptor we generated. */
-  unsigned int listed_in_last_desc : 1;
-
-  /** (Service side only) A replay cache recording the RSA-encrypted parts
-   * of INTRODUCE2 cells this intro point's circuit has received.  This is
-   * used to prevent replay attacks. */
-  replaycache_t *accepted_intro_rsa_parts;
-
-  /** (Service side only) Count of INTRODUCE2 cells accepted from this
-   * intro point.
-   */
-  int accepted_introduce2_count;
-
-  /** (Service side only) Number of maximum INTRODUCE2 cells that this IP
-   * will accept. This is a random value between
-   * INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS and
-   * INTRO_POINT_MAX_LIFETIME_INTRODUCTIONS. */
-  int max_introductions;
-
-  /** (Service side only) The time at which this intro point was first
-   * published, or -1 if this intro point has not yet been
-   * published. */
-  time_t time_published;
-
-  /** (Service side only) The time at which this intro point should
-   * (start to) expire, or -1 if we haven't decided when this intro
-   * point should expire. */
-  time_t time_to_expire;
-
-  /** (Service side only) The amount of circuit creation we've made to this
-   * intro point. This is incremented every time we do a circuit relaunch on
-   * this object which is triggered when the circuit dies but the node is
-   * still in the consensus. After MAX_INTRO_POINT_CIRCUIT_RETRIES, we give
-   * up on it. */
-  unsigned int circuit_retries;
-
-  /** (Service side only) Set if this intro point has an established circuit
-   * and unset if it doesn't. */
-  unsigned int circuit_established:1;
-} rend_intro_point_t;
-
-#define REND_PROTOCOL_VERSION_BITMASK_WIDTH 16
-
-/** Information used to connect to a hidden service.  Used on both the
- * service side and the client side. */
-typedef struct rend_service_descriptor_t {
-  crypto_pk_t *pk; /**< This service's public key. */
-  int version; /**< Version of the descriptor format: 0 or 2. */
-  time_t timestamp; /**< Time when the descriptor was generated. */
-  /** Bitmask: which introduce/rendezvous protocols are supported?
-   * (We allow bits '0', '1', '2' and '3' to be set.) */
-  unsigned protocols : REND_PROTOCOL_VERSION_BITMASK_WIDTH;
-  /** List of the service's introduction points.  Elements are removed if
-   * introduction attempts fail. */
-  smartlist_t *intro_nodes;
-  /** Has descriptor been uploaded to all hidden service directories? */
-  int all_uploads_performed;
-  /** List of hidden service directories to which an upload request for
-   * this descriptor could be sent. Smartlist exists only when at least one
-   * of the previous upload requests failed (otherwise it's not important
-   * to know which uploads succeeded and which not). */
-  smartlist_t *successful_uploads;
-} rend_service_descriptor_t;
-
-/********************************* routerlist.c ***************************/
-
-/** Represents information about a single trusted or fallback directory
- * server. */
-typedef struct dir_server_t {
-  char *description;
-  char *nickname;
-  char *address; /**< Hostname. */
-  /* XX/teor - why do we duplicate the address and port fields here and in
-   *           fake_status? Surely we could just use fake_status (#17867). */
-  tor_addr_t ipv6_addr; /**< IPv6 address if present; AF_UNSPEC if not */
-  uint32_t addr; /**< IPv4 address. */
-  uint16_t dir_port; /**< Directory port. */
-  uint16_t or_port; /**< OR port: Used for tunneling connections. */
-  uint16_t ipv6_orport; /**< OR port corresponding to ipv6_addr. */
-  double weight; /** Weight used when selecting this node at random */
-  char digest[DIGEST_LEN]; /**< Digest of identity key. */
-  char v3_identity_digest[DIGEST_LEN]; /**< Digest of v3 (authority only,
-                                        * high-security) identity key. */
-
-  unsigned int is_running:1; /**< True iff we think this server is running. */
-  unsigned int is_authority:1; /**< True iff this is a directory authority
-                                * of some kind. */
-
-  /** True iff this server has accepted the most recent server descriptor
-   * we tried to upload to it. */
-  unsigned int has_accepted_serverdesc:1;
-
-  /** What kind of authority is this? (Bitfield.) */
-  dirinfo_type_t type;
-
-  time_t addr_current_at; /**< When was the document that we derived the
-                           * address information from published? */
-
-  routerstatus_t fake_status; /**< Used when we need to pass this trusted
-                               * dir_server_t to directory_initiate_command_*
-                               * as a routerstatus_t.  Not updated by the
-                               * router-status management code!
-                               **/
-} dir_server_t;
-
-#define RELAY_REQUIRED_MIN_BANDWIDTH (75*1024)
-#define BRIDGE_REQUIRED_MIN_BANDWIDTH (50*1024)
-
-#define ROUTER_MAX_DECLARED_BANDWIDTH INT32_MAX
-
-/* Flags for pick_directory_server() and pick_trusteddirserver(). */
-/** Flag to indicate that we should not automatically be willing to use
- * ourself to answer a directory request.
- * Passed to router_pick_directory_server (et al).*/
-#define PDS_ALLOW_SELF                 (1<<0)
-/** Flag to indicate that if no servers seem to be up, we should mark all
- * directory servers as up and try again.
- * Passed to router_pick_directory_server (et al).*/
-#define PDS_RETRY_IF_NO_SERVERS        (1<<1)
-/** Flag to indicate that we should not exclude directory servers that
- * our ReachableAddress settings would exclude.  This usually means that
- * we're going to connect to the server over Tor, and so we don't need to
- * worry about our firewall telling us we can't.
- * Passed to router_pick_directory_server (et al).*/
-#define PDS_IGNORE_FASCISTFIREWALL     (1<<2)
-/** Flag to indicate that we should not use any directory authority to which
- * we have an existing directory connection for downloading server descriptors
- * or extrainfo documents.
- *
- * Passed to router_pick_directory_server (et al)
- */
-#define PDS_NO_EXISTING_SERVERDESC_FETCH (1<<3)
-/** Flag to indicate that we should not use any directory authority to which
- * we have an existing directory connection for downloading microdescs.
- *
- * Passed to router_pick_directory_server (et al)
- */
-#define PDS_NO_EXISTING_MICRODESC_FETCH (1<<4)
-
-/** This node is to be chosen as a directory guard, so don't choose any
- * node that's currently a guard. */
-#define PDS_FOR_GUARD (1<<5)
-
-/** Possible ways to weight routers when choosing one randomly.  See
- * routerlist_sl_choose_by_bandwidth() for more information.*/
-typedef enum bandwidth_weight_rule_t {
-  NO_WEIGHTING, WEIGHT_FOR_EXIT, WEIGHT_FOR_MID, WEIGHT_FOR_GUARD,
-  WEIGHT_FOR_DIR
-} bandwidth_weight_rule_t;
-
-/** Flags to be passed to control router_choose_random_node() to indicate what
- * kind of nodes to pick according to what algorithm. */
-typedef enum {
-  CRN_NEED_UPTIME = 1<<0,
-  CRN_NEED_CAPACITY = 1<<1,
-  CRN_NEED_GUARD = 1<<2,
-  CRN_ALLOW_INVALID = 1<<3,
-  /* XXXX not used, apparently. */
-  CRN_WEIGHT_AS_EXIT = 1<<5,
-  CRN_NEED_DESC = 1<<6,
-  /* On clients, only provide nodes that satisfy ClientPreferIPv6OR */
-  CRN_PREF_ADDR = 1<<7,
-  /* On clients, only provide nodes that we can connect to directly, based on
-   * our firewall rules */
-  CRN_DIRECT_CONN = 1<<8
-} router_crn_flags_t;
-
-/** Return value for router_add_to_routerlist() and dirserv_add_descriptor() */
-typedef enum was_router_added_t {
-  /* Router was added successfully. */
-  ROUTER_ADDED_SUCCESSFULLY = 1,
-  /* Router descriptor was added with warnings to submitter. */
-  ROUTER_ADDED_NOTIFY_GENERATOR = 0,
-  /* Extrainfo document was rejected because no corresponding router
-   * descriptor was found OR router descriptor was rejected because
-   * it was incompatible with its extrainfo document. */
-  ROUTER_BAD_EI = -1,
-  /* Router descriptor was rejected because it is already known. */
-  ROUTER_IS_ALREADY_KNOWN = -2,
-  /* General purpose router was rejected, because it was not listed
-   * in consensus. */
-  ROUTER_NOT_IN_CONSENSUS = -3,
-  /* Router was neither in directory consensus nor in any of
-   * networkstatus documents. Caching it to access later.
-   * (Applies to fetched descriptors only.) */
-  ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS = -4,
-  /* Router was rejected by directory authority. */
-  ROUTER_AUTHDIR_REJECTS = -5,
-  /* Bridge descriptor was rejected because such bridge was not one
-   * of the bridges we have listed in our configuration. */
-  ROUTER_WAS_NOT_WANTED = -6,
-  /* Router descriptor was rejected because it was older than
-   * OLD_ROUTER_DESC_MAX_AGE. */
-  ROUTER_WAS_TOO_OLD = -7, /* note contrast with 'NOT_NEW' */
-  /* DOCDOC */
-  ROUTER_CERTS_EXPIRED = -8
-} was_router_added_t;
-
-/********************************* routerparse.c ************************/
-
-#define MAX_STATUS_TAG_LEN 32
-/** Structure to hold parsed Tor versions.  This is a little messier
- * than we would like it to be, because we changed version schemes with 0.1.0.
- *
- * See version-spec.txt for the whole business.
- */
-typedef struct tor_version_t {
-  int major;
-  int minor;
-  int micro;
-  /** Release status.  For version in the post-0.1 format, this is always
-   * VER_RELEASE. */
-  enum { VER_PRE=0, VER_RC=1, VER_RELEASE=2, } status;
-  int patchlevel;
-  char status_tag[MAX_STATUS_TAG_LEN];
-  int svn_revision;
-
-  int git_tag_len;
-  char git_tag[DIGEST_LEN];
-} tor_version_t;
-
-#endif
-
diff --git a/src/or/periodic.h b/src/or/periodic.h
deleted file mode 100644
index 021bb4ef5c..0000000000
--- a/src/or/periodic.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#ifndef TOR_PERIODIC_H
-#define TOR_PERIODIC_H
-
-#define PERIODIC_EVENT_NO_UPDATE (-1)
-
-/** Callback function for a periodic event to take action.  The return value
-* influences the next time the function will get called.  Return
-* PERIODIC_EVENT_NO_UPDATE to not update <b>last_action_time</b> and be polled
-* again in the next second. If a positive value is returned it will update the
-* interval time. */
-typedef int (*periodic_event_helper_t)(time_t now,
-                                      const or_options_t *options);
-
-struct event;
-
-/** A single item for the periodic-events-function table. */
-typedef struct periodic_event_item_t {
-  periodic_event_helper_t fn; /**< The function to run the event */
-  time_t last_action_time; /**< The last time the function did something */
-  struct event *ev; /**< Libevent callback we're using to implement this */
-  const char *name; /**< Name of the function -- for debug */
-} periodic_event_item_t;
-
-/** events will get their interval from first execution */
-#define PERIODIC_EVENT(fn) { fn##_callback, 0, NULL, #fn }
-#define END_OF_PERIODIC_EVENTS { NULL, 0, NULL, NULL }
-
-void periodic_event_launch(periodic_event_item_t *event);
-void periodic_event_setup(periodic_event_item_t *event);
-void periodic_event_destroy(periodic_event_item_t *event);
-void periodic_event_reschedule(periodic_event_item_t *event);
-
-#endif
-
diff --git a/src/or/router.h b/src/or/router.h
deleted file mode 100644
index c30a0301b7..0000000000
--- a/src/or/router.h
+++ /dev/null
@@ -1,163 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file router.h
- * \brief Header file for router.c.
- **/
-
-#ifndef TOR_ROUTER_H
-#define TOR_ROUTER_H
-
-#include "testsupport.h"
-
-crypto_pk_t *get_onion_key(void);
-time_t get_onion_key_set_at(void);
-void set_server_identity_key(crypto_pk_t *k);
-crypto_pk_t *get_server_identity_key(void);
-int server_identity_key_is_set(void);
-void set_client_identity_key(crypto_pk_t *k);
-crypto_pk_t *get_tlsclient_identity_key(void);
-int client_identity_key_is_set(void);
-MOCK_DECL(authority_cert_t *, get_my_v3_authority_cert, (void));
-crypto_pk_t *get_my_v3_authority_signing_key(void);
-authority_cert_t *get_my_v3_legacy_cert(void);
-crypto_pk_t *get_my_v3_legacy_signing_key(void);
-void dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last);
-void rotate_onion_key(void);
-crypto_pk_t *init_key_from_file(const char *fname, int generate,
-                                    int severity, int log_greeting);
-void v3_authority_check_key_expiry(void);
-
-di_digest256_map_t *construct_ntor_key_map(void);
-void ntor_key_map_free(di_digest256_map_t *map);
-
-int router_initialize_tls_context(void);
-int init_keys(void);
-int init_keys_client(void);
-
-int check_whether_orport_reachable(const or_options_t *options);
-int check_whether_dirport_reachable(const or_options_t *options);
-int dir_server_mode(const or_options_t *options);
-void consider_testing_reachability(int test_or, int test_dir);
-void router_orport_found_reachable(void);
-void router_dirport_found_reachable(void);
-void router_perform_bandwidth_test(int num_circs, time_t now);
-
-int net_is_disabled(void);
-
-int authdir_mode(const or_options_t *options);
-int authdir_mode_v3(const or_options_t *options);
-int authdir_mode_any_main(const or_options_t *options);
-int authdir_mode_any_nonhidserv(const or_options_t *options);
-int authdir_mode_handles_descs(const or_options_t *options, int purpose);
-int authdir_mode_publishes_statuses(const or_options_t *options);
-int authdir_mode_tests_reachability(const or_options_t *options);
-int authdir_mode_bridge(const or_options_t *options);
-
-uint16_t router_get_active_listener_port_by_type_af(int listener_type,
-                                                    sa_family_t family);
-uint16_t router_get_advertised_or_port(const or_options_t *options);
-uint16_t router_get_advertised_or_port_by_af(const or_options_t *options,
-                                             sa_family_t family);
-uint16_t router_get_advertised_dir_port(const or_options_t *options,
-                                        uint16_t dirport);
-
-MOCK_DECL(int, server_mode, (const or_options_t *options));
-MOCK_DECL(int, public_server_mode, (const or_options_t *options));
-MOCK_DECL(int, advertised_server_mode, (void));
-int proxy_mode(const or_options_t *options);
-void consider_publishable_server(int force);
-int should_refuse_unknown_exits(const or_options_t *options);
-
-void router_upload_dir_desc_to_dirservers(int force);
-void mark_my_descriptor_dirty_if_too_old(time_t now);
-void mark_my_descriptor_dirty(const char *reason);
-void check_descriptor_bandwidth_changed(time_t now);
-void check_descriptor_ipaddress_changed(time_t now);
-void router_new_address_suggestion(const char *suggestion,
-                                   const dir_connection_t *d_conn);
-int router_compare_to_my_exit_policy(const tor_addr_t *addr, uint16_t port);
-MOCK_DECL(int, router_my_exit_policy_is_reject_star,(void));
-MOCK_DECL(const routerinfo_t *, router_get_my_routerinfo, (void));
-extrainfo_t *router_get_my_extrainfo(void);
-const char *router_get_my_descriptor(void);
-const char *router_get_descriptor_gen_reason(void);
-int router_digest_is_me(const char *digest);
-const uint8_t *router_get_my_id_digest(void);
-int router_extrainfo_digest_is_me(const char *digest);
-int router_is_me(const routerinfo_t *router);
-MOCK_DECL(int,router_pick_published_address,(const or_options_t *options,
-                                             uint32_t *addr,
-                                             int cache_only));
-int router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e);
-int router_rebuild_descriptor(int force);
-char *router_dump_router_to_string(routerinfo_t *router,
-                                   const crypto_pk_t *ident_key,
-                                   const crypto_pk_t *tap_key,
-                                   const curve25519_keypair_t *ntor_keypair,
-                                   const ed25519_keypair_t *signing_keypair);
-char *router_dump_exit_policy_to_string(const routerinfo_t *router,
-                                         int include_ipv4,
-                                         int include_ipv6);
-void router_get_prim_orport(const routerinfo_t *router,
-                            tor_addr_port_t *addr_port_out);
-void router_get_pref_orport(const routerinfo_t *router,
-                            tor_addr_port_t *addr_port_out);
-void router_get_pref_ipv6_orport(const routerinfo_t *router,
-                                 tor_addr_port_t *addr_port_out);
-int router_ipv6_preferred(const routerinfo_t *router);
-int router_has_addr(const routerinfo_t *router, const tor_addr_t *addr);
-int router_has_orport(const routerinfo_t *router,
-                      const tor_addr_port_t *orport);
-int extrainfo_dump_to_string(char **s, extrainfo_t *extrainfo,
-                             crypto_pk_t *ident_key,
-                             const ed25519_keypair_t *signing_keypair);
-int is_legal_nickname(const char *s);
-int is_legal_nickname_or_hexdigest(const char *s);
-int is_legal_hexdigest(const char *s);
-
-/**
- * Longest allowed output of format_node_description, plus 1 character for
- * NUL.  This allows space for:
- * "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF~xxxxxxxxxxxxxxxxxxx at"
- * " [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]"
- * plus a terminating NUL.
- */
-#define NODE_DESC_BUF_LEN (MAX_VERBOSE_NICKNAME_LEN+4+TOR_ADDR_BUF_LEN)
-const char *format_node_description(char *buf,
-                                    const char *id_digest,
-                                    int is_named,
-                                    const char *nickname,
-                                    const tor_addr_t *addr,
-                                    uint32_t addr32h);
-const char *router_get_description(char *buf, const routerinfo_t *ri);
-const char *node_get_description(char *buf, const node_t *node);
-const char *routerstatus_get_description(char *buf, const routerstatus_t *rs);
-const char *extend_info_get_description(char *buf, const extend_info_t *ei);
-const char *router_describe(const routerinfo_t *ri);
-const char *node_describe(const node_t *node);
-const char *routerstatus_describe(const routerstatus_t *ri);
-const char *extend_info_describe(const extend_info_t *ei);
-
-void router_get_verbose_nickname(char *buf, const routerinfo_t *router);
-void router_reset_warnings(void);
-void router_reset_reachability(void);
-void router_free_all(void);
-
-const char *router_purpose_to_string(uint8_t p);
-uint8_t router_purpose_from_string(const char *s);
-
-smartlist_t *router_get_all_orports(const routerinfo_t *ri);
-
-#ifdef ROUTER_PRIVATE
-/* Used only by router.c and test.c */
-STATIC void get_platform_str(char *platform, size_t len);
-STATIC int router_write_fingerprint(int hashed);
-#endif
-
-#endif
-
diff --git a/src/or/routerparse.c b/src/or/routerparse.c
deleted file mode 100644
index f046cc39b4..0000000000
--- a/src/or/routerparse.c
+++ /dev/null
@@ -1,6364 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file routerparse.c
- * \brief Code to parse and validate router descriptors, consenus directories,
- *   and similar objects.
- *
- * The objects parsed by this module use a common text-based metaformat,
- * documented in dir-spec.txt in torspec.git.  This module is itself divided
- * into two major kinds of function: code to handle the metaformat, and code
- * to convert from particular instances of the metaformat into the
- * objects that Tor uses.
- *
- * The generic parsing code works by calling a table-based tokenizer on the
- * input string.  Each token corresponds to a single line with a token, plus
- * optional arguments on that line, plus an optional base-64 encoded object
- * after that line.  Each token has a definition in a table of token_rule_t
- * entries that describes how many arguments it can take, whether it takes an
- * object, how many times it may appear, whether it must appear first, and so
- * on.
- *
- * The tokenizer function tokenize_string() converts its string input into a
- * smartlist full of instances of directory_token_t, according to a provided
- * table of token_rule_t.
- *
- * The generic parts of this module additionally include functions for
- * finding the start and end of signed information inside a signed object, and
- * computing the digest that will be signed.
- *
- * There are also functions for saving objects to disk that have caused
- * parsing to fail.
- *
- * The specific parts of this module describe conversions between
- * particular lists of directory_token_t and particular objects.  The
- * kinds of objects that can be parsed here are:
- *  <ul>
- *  <li>router descriptors (managed from routerlist.c)
- *  <li>extra-info documents (managed from routerlist.c)
- *  <li>microdescriptors (managed from microdesc.c)
- *  <li>vote and consensus networkstatus documents, and the routerstatus_t
- *    objects that they comprise (managed from networkstatus.c)
- *  <li>detached-signature objects used by authorities for gathering
- *    signatures on the networkstatus consensus (managed from dirvote.c)
- *  <li>authority key certificates (managed from routerlist.c)
- *  <li>hidden service descriptors (managed from rendcommon.c and rendcache.c)
- * </ul>
- *
- * For no terribly good reason, the functions to <i>generate</i> signatures on
- * the above directory objects are also in this module.
- **/
-
-#define ROUTERPARSE_PRIVATE
-
-#include "or.h"
-#include "config.h"
-#include "circuitstats.h"
-#include "dirserv.h"
-#include "dirvote.h"
-#include "policies.h"
-#include "protover.h"
-#include "rendcommon.h"
-#include "router.h"
-#include "routerlist.h"
-#include "memarea.h"
-#include "microdesc.h"
-#include "networkstatus.h"
-#include "rephist.h"
-#include "routerkeys.h"
-#include "routerparse.h"
-#include "entrynodes.h"
-#include "torcert.h"
-#include "sandbox.h"
-#include "shared_random.h"
-
-#undef log
-#include <math.h>
-
-/****************************************************************************/
-
-/** Enumeration of possible token types.  The ones starting with K_ correspond
- * to directory 'keywords'. A_ is for an annotation, R or C is related to
- * hidden services, ERR_ is an error in the tokenizing process, EOF_ is an
- * end-of-file marker, and NIL_ is used to encode not-a-token.
- */
-typedef enum {
-  K_ACCEPT = 0,
-  K_ACCEPT6,
-  K_DIRECTORY_SIGNATURE,
-  K_RECOMMENDED_SOFTWARE,
-  K_REJECT,
-  K_REJECT6,
-  K_ROUTER,
-  K_SIGNED_DIRECTORY,
-  K_SIGNING_KEY,
-  K_ONION_KEY,
-  K_ONION_KEY_NTOR,
-  K_ROUTER_SIGNATURE,
-  K_PUBLISHED,
-  K_RUNNING_ROUTERS,
-  K_ROUTER_STATUS,
-  K_PLATFORM,
-  K_PROTO,
-  K_OPT,
-  K_BANDWIDTH,
-  K_CONTACT,
-  K_NETWORK_STATUS,
-  K_UPTIME,
-  K_DIR_SIGNING_KEY,
-  K_FAMILY,
-  K_FINGERPRINT,
-  K_HIBERNATING,
-  K_READ_HISTORY,
-  K_WRITE_HISTORY,
-  K_NETWORK_STATUS_VERSION,
-  K_DIR_SOURCE,
-  K_DIR_OPTIONS,
-  K_CLIENT_VERSIONS,
-  K_SERVER_VERSIONS,
-  K_RECOMMENDED_CLIENT_PROTOCOLS,
-  K_RECOMMENDED_RELAY_PROTOCOLS,
-  K_REQUIRED_CLIENT_PROTOCOLS,
-  K_REQUIRED_RELAY_PROTOCOLS,
-  K_OR_ADDRESS,
-  K_ID,
-  K_P,
-  K_P6,
-  K_R,
-  K_A,
-  K_S,
-  K_V,
-  K_W,
-  K_M,
-  K_EXTRA_INFO,
-  K_EXTRA_INFO_DIGEST,
-  K_CACHES_EXTRA_INFO,
-  K_HIDDEN_SERVICE_DIR,
-  K_ALLOW_SINGLE_HOP_EXITS,
-  K_IPV6_POLICY,
-  K_ROUTER_SIG_ED25519,
-  K_IDENTITY_ED25519,
-  K_MASTER_KEY_ED25519,
-  K_ONION_KEY_CROSSCERT,
-  K_NTOR_ONION_KEY_CROSSCERT,
-
-  K_DIRREQ_END,
-  K_DIRREQ_V2_IPS,
-  K_DIRREQ_V3_IPS,
-  K_DIRREQ_V2_REQS,
-  K_DIRREQ_V3_REQS,
-  K_DIRREQ_V2_SHARE,
-  K_DIRREQ_V3_SHARE,
-  K_DIRREQ_V2_RESP,
-  K_DIRREQ_V3_RESP,
-  K_DIRREQ_V2_DIR,
-  K_DIRREQ_V3_DIR,
-  K_DIRREQ_V2_TUN,
-  K_DIRREQ_V3_TUN,
-  K_ENTRY_END,
-  K_ENTRY_IPS,
-  K_CELL_END,
-  K_CELL_PROCESSED,
-  K_CELL_QUEUED,
-  K_CELL_TIME,
-  K_CELL_CIRCS,
-  K_EXIT_END,
-  K_EXIT_WRITTEN,
-  K_EXIT_READ,
-  K_EXIT_OPENED,
-
-  K_DIR_KEY_CERTIFICATE_VERSION,
-  K_DIR_IDENTITY_KEY,
-  K_DIR_KEY_PUBLISHED,
-  K_DIR_KEY_EXPIRES,
-  K_DIR_KEY_CERTIFICATION,
-  K_DIR_KEY_CROSSCERT,
-  K_DIR_ADDRESS,
-  K_DIR_TUNNELLED,
-
-  K_VOTE_STATUS,
-  K_VALID_AFTER,
-  K_FRESH_UNTIL,
-  K_VALID_UNTIL,
-  K_VOTING_DELAY,
-
-  K_KNOWN_FLAGS,
-  K_PARAMS,
-  K_BW_WEIGHTS,
-  K_VOTE_DIGEST,
-  K_CONSENSUS_DIGEST,
-  K_ADDITIONAL_DIGEST,
-  K_ADDITIONAL_SIGNATURE,
-  K_CONSENSUS_METHODS,
-  K_CONSENSUS_METHOD,
-  K_LEGACY_DIR_KEY,
-  K_DIRECTORY_FOOTER,
-  K_SIGNING_CERT_ED,
-  K_SR_FLAG,
-  K_COMMIT,
-  K_PREVIOUS_SRV,
-  K_CURRENT_SRV,
-  K_PACKAGE,
-
-  A_PURPOSE,
-  A_LAST_LISTED,
-  A_UNKNOWN_,
-
-  R_RENDEZVOUS_SERVICE_DESCRIPTOR,
-  R_VERSION,
-  R_PERMANENT_KEY,
-  R_SECRET_ID_PART,
-  R_PUBLICATION_TIME,
-  R_PROTOCOL_VERSIONS,
-  R_INTRODUCTION_POINTS,
-  R_SIGNATURE,
-
-  R_IPO_IDENTIFIER,
-  R_IPO_IP_ADDRESS,
-  R_IPO_ONION_PORT,
-  R_IPO_ONION_KEY,
-  R_IPO_SERVICE_KEY,
-
-  C_CLIENT_NAME,
-  C_DESCRIPTOR_COOKIE,
-  C_CLIENT_KEY,
-
-  ERR_,
-  EOF_,
-  NIL_
-} directory_keyword;
-
-#define MIN_ANNOTATION A_PURPOSE
-#define MAX_ANNOTATION A_UNKNOWN_
-
-/** Structure to hold a single directory token.
- *
- * We parse a directory by breaking it into "tokens", each consisting
- * of a keyword, a line full of arguments, and a binary object.  The
- * arguments and object are both optional, depending on the keyword
- * type.
- *
- * This structure is only allocated in memareas; do not allocate it on
- * the heap, or token_clear() won't work.
- */
-typedef struct directory_token_t {
-  directory_keyword tp;        /**< Type of the token. */
-  int n_args:30;               /**< Number of elements in args */
-  char **args;                 /**< Array of arguments from keyword line. */
-
-  char *object_type;           /**< -----BEGIN [object_type]-----*/
-  size_t object_size;          /**< Bytes in object_body */
-  char *object_body;           /**< Contents of object, base64-decoded. */
-
-  crypto_pk_t *key;        /**< For public keys only.  Heap-allocated. */
-
-  char *error;                 /**< For ERR_ tokens only. */
-} directory_token_t;
-
-/* ********************************************************************** */
-
-/** We use a table of rules to decide how to parse each token type. */
-
-/** Rules for whether the keyword needs an object. */
-typedef enum {
-  NO_OBJ,        /**< No object, ever. */
-  NEED_OBJ,      /**< Object is required. */
-  NEED_SKEY_1024,/**< Object is required, and must be a 1024 bit private key */
-  NEED_KEY_1024, /**< Object is required, and must be a 1024 bit public key */
-  NEED_KEY,      /**< Object is required, and must be a public key. */
-  OBJ_OK,        /**< Object is optional. */
-} obj_syntax;
-
-#define AT_START 1
-#define AT_END 2
-
-/** Determines the parsing rules for a single token type. */
-typedef struct token_rule_t {
-  /** The string value of the keyword identifying the type of item. */
-  const char *t;
-  /** The corresponding directory_keyword enum. */
-  directory_keyword v;
-  /** Minimum number of arguments for this item */
-  int min_args;
-  /** Maximum number of arguments for this item */
-  int max_args;
-  /** If true, we concatenate all arguments for this item into a single
-   * string. */
-  int concat_args;
-  /** Requirements on object syntax for this item. */
-  obj_syntax os;
-  /** Lowest number of times this item may appear in a document. */
-  int min_cnt;
-  /** Highest number of times this item may appear in a document. */
-  int max_cnt;
-  /** One or more of AT_START/AT_END to limit where the item may appear in a
-   * document. */
-  int pos;
-  /** True iff this token is an annotation. */
-  int is_annotation;
-} token_rule_t;
-
-/**
- * @name macros for defining token rules
- *
- * Helper macros to define token tables.  's' is a string, 't' is a
- * directory_keyword, 'a' is a trio of argument multiplicities, and 'o' is an
- * object syntax.
- */
-/**@{*/
-
-/** Appears to indicate the end of a table. */
-#define END_OF_TABLE { NULL, NIL_, 0,0,0, NO_OBJ, 0, INT_MAX, 0, 0 }
-/** An item with no restrictions: used for obsolete document types */
-#define T(s,t,a,o)    { s, t, a, o, 0, INT_MAX, 0, 0 }
-/** An item with no restrictions on multiplicity or location. */
-#define T0N(s,t,a,o)  { s, t, a, o, 0, INT_MAX, 0, 0 }
-/** An item that must appear exactly once */
-#define T1(s,t,a,o)   { s, t, a, o, 1, 1, 0, 0 }
-/** An item that must appear exactly once, at the start of the document */
-#define T1_START(s,t,a,o)   { s, t, a, o, 1, 1, AT_START, 0 }
-/** An item that must appear exactly once, at the end of the document */
-#define T1_END(s,t,a,o)   { s, t, a, o, 1, 1, AT_END, 0 }
-/** An item that must appear one or more times */
-#define T1N(s,t,a,o)  { s, t, a, o, 1, INT_MAX, 0, 0 }
-/** An item that must appear no more than once */
-#define T01(s,t,a,o)  { s, t, a, o, 0, 1, 0, 0 }
-/** An annotation that must appear no more than once */
-#define A01(s,t,a,o)  { s, t, a, o, 0, 1, 0, 1 }
-
-/** Argument multiplicity: any number of arguments. */
-#define ARGS        0,INT_MAX,0
-/** Argument multiplicity: no arguments. */
-#define NO_ARGS     0,0,0
-/** Argument multiplicity: concatenate all arguments. */
-#define CONCAT_ARGS 1,1,1
-/** Argument multiplicity: at least <b>n</b> arguments. */
-#define GE(n)       n,INT_MAX,0
-/** Argument multiplicity: exactly <b>n</b> arguments. */
-#define EQ(n)       n,n,0
-/**@}*/
-
-/** List of tokens recognized in router descriptors */
-static token_rule_t routerdesc_token_table[] = {
-  T0N("reject",              K_REJECT,              ARGS,    NO_OBJ ),
-  T0N("accept",              K_ACCEPT,              ARGS,    NO_OBJ ),
-  T0N("reject6",             K_REJECT6,             ARGS,    NO_OBJ ),
-  T0N("accept6",             K_ACCEPT6,             ARGS,    NO_OBJ ),
-  T1_START( "router",        K_ROUTER,              GE(5),   NO_OBJ ),
-  T01("ipv6-policy",         K_IPV6_POLICY,         CONCAT_ARGS, NO_OBJ),
-  T1( "signing-key",         K_SIGNING_KEY,         NO_ARGS, NEED_KEY_1024 ),
-  T1( "onion-key",           K_ONION_KEY,           NO_ARGS, NEED_KEY_1024 ),
-  T01("ntor-onion-key",      K_ONION_KEY_NTOR,      GE(1), NO_OBJ ),
-  T1_END( "router-signature",    K_ROUTER_SIGNATURE,    NO_ARGS, NEED_OBJ ),
-  T1( "published",           K_PUBLISHED,       CONCAT_ARGS, NO_OBJ ),
-  T01("uptime",              K_UPTIME,              GE(1),   NO_OBJ ),
-  T01("fingerprint",         K_FINGERPRINT,     CONCAT_ARGS, NO_OBJ ),
-  T01("hibernating",         K_HIBERNATING,         GE(1),   NO_OBJ ),
-  T01("platform",            K_PLATFORM,        CONCAT_ARGS, NO_OBJ ),
-  T01("proto",               K_PROTO,           CONCAT_ARGS, NO_OBJ ),
-  T01("contact",             K_CONTACT,         CONCAT_ARGS, NO_OBJ ),
-  T01("read-history",        K_READ_HISTORY,        ARGS,    NO_OBJ ),
-  T01("write-history",       K_WRITE_HISTORY,       ARGS,    NO_OBJ ),
-  T01("extra-info-digest",   K_EXTRA_INFO_DIGEST,   GE(1),   NO_OBJ ),
-  T01("hidden-service-dir",  K_HIDDEN_SERVICE_DIR,  NO_ARGS, NO_OBJ ),
-  T01("identity-ed25519",    K_IDENTITY_ED25519,    NO_ARGS, NEED_OBJ ),
-  T01("master-key-ed25519",  K_MASTER_KEY_ED25519,  GE(1),   NO_OBJ ),
-  T01("router-sig-ed25519",  K_ROUTER_SIG_ED25519,  GE(1),   NO_OBJ ),
-  T01("onion-key-crosscert", K_ONION_KEY_CROSSCERT, NO_ARGS, NEED_OBJ ),
-  T01("ntor-onion-key-crosscert", K_NTOR_ONION_KEY_CROSSCERT,
-                                                    EQ(1),   NEED_OBJ ),
-
-  T01("allow-single-hop-exits",K_ALLOW_SINGLE_HOP_EXITS,    NO_ARGS, NO_OBJ ),
-
-  T01("family",              K_FAMILY,              ARGS,    NO_OBJ ),
-  T01("caches-extra-info",   K_CACHES_EXTRA_INFO,   NO_ARGS, NO_OBJ ),
-  T0N("or-address",          K_OR_ADDRESS,          GE(1),   NO_OBJ ),
-
-  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
-  T1( "bandwidth",           K_BANDWIDTH,           GE(3),   NO_OBJ ),
-  A01("@purpose",            A_PURPOSE,             GE(1),   NO_OBJ ),
-  T01("tunnelled-dir-server",K_DIR_TUNNELLED,       NO_ARGS, NO_OBJ ),
-
-  END_OF_TABLE
-};
-
-/** List of tokens recognized in extra-info documents. */
-static token_rule_t extrainfo_token_table[] = {
-  T1_END( "router-signature",    K_ROUTER_SIGNATURE,    NO_ARGS, NEED_OBJ ),
-  T1( "published",           K_PUBLISHED,       CONCAT_ARGS, NO_OBJ ),
-  T01("identity-ed25519",    K_IDENTITY_ED25519,    NO_ARGS, NEED_OBJ ),
-  T01("router-sig-ed25519",  K_ROUTER_SIG_ED25519,  GE(1),   NO_OBJ ),
-  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
-  T01("read-history",        K_READ_HISTORY,        ARGS,    NO_OBJ ),
-  T01("write-history",       K_WRITE_HISTORY,       ARGS,    NO_OBJ ),
-  T01("dirreq-stats-end",    K_DIRREQ_END,          ARGS,    NO_OBJ ),
-  T01("dirreq-v2-ips",       K_DIRREQ_V2_IPS,       ARGS,    NO_OBJ ),
-  T01("dirreq-v3-ips",       K_DIRREQ_V3_IPS,       ARGS,    NO_OBJ ),
-  T01("dirreq-v2-reqs",      K_DIRREQ_V2_REQS,      ARGS,    NO_OBJ ),
-  T01("dirreq-v3-reqs",      K_DIRREQ_V3_REQS,      ARGS,    NO_OBJ ),
-  T01("dirreq-v2-share",     K_DIRREQ_V2_SHARE,     ARGS,    NO_OBJ ),
-  T01("dirreq-v3-share",     K_DIRREQ_V3_SHARE,     ARGS,    NO_OBJ ),
-  T01("dirreq-v2-resp",      K_DIRREQ_V2_RESP,      ARGS,    NO_OBJ ),
-  T01("dirreq-v3-resp",      K_DIRREQ_V3_RESP,      ARGS,    NO_OBJ ),
-  T01("dirreq-v2-direct-dl", K_DIRREQ_V2_DIR,       ARGS,    NO_OBJ ),
-  T01("dirreq-v3-direct-dl", K_DIRREQ_V3_DIR,       ARGS,    NO_OBJ ),
-  T01("dirreq-v2-tunneled-dl", K_DIRREQ_V2_TUN,     ARGS,    NO_OBJ ),
-  T01("dirreq-v3-tunneled-dl", K_DIRREQ_V3_TUN,     ARGS,    NO_OBJ ),
-  T01("entry-stats-end",     K_ENTRY_END,           ARGS,    NO_OBJ ),
-  T01("entry-ips",           K_ENTRY_IPS,           ARGS,    NO_OBJ ),
-  T01("cell-stats-end",      K_CELL_END,            ARGS,    NO_OBJ ),
-  T01("cell-processed-cells", K_CELL_PROCESSED,     ARGS,    NO_OBJ ),
-  T01("cell-queued-cells",   K_CELL_QUEUED,         ARGS,    NO_OBJ ),
-  T01("cell-time-in-queue",  K_CELL_TIME,           ARGS,    NO_OBJ ),
-  T01("cell-circuits-per-decile", K_CELL_CIRCS,     ARGS,    NO_OBJ ),
-  T01("exit-stats-end",      K_EXIT_END,            ARGS,    NO_OBJ ),
-  T01("exit-kibibytes-written", K_EXIT_WRITTEN,     ARGS,    NO_OBJ ),
-  T01("exit-kibibytes-read", K_EXIT_READ,           ARGS,    NO_OBJ ),
-  T01("exit-streams-opened", K_EXIT_OPENED,         ARGS,    NO_OBJ ),
-
-  T1_START( "extra-info",          K_EXTRA_INFO,          GE(2),   NO_OBJ ),
-
-  END_OF_TABLE
-};
-
-/** List of tokens recognized in the body part of v3 networkstatus
- * documents. */
-static token_rule_t rtrstatus_token_table[] = {
-  T01("p",                   K_P,               CONCAT_ARGS, NO_OBJ ),
-  T1( "r",                   K_R,                   GE(7),   NO_OBJ ),
-  T0N("a",                   K_A,                   GE(1),   NO_OBJ ),
-  T1( "s",                   K_S,                   ARGS,    NO_OBJ ),
-  T01("v",                   K_V,               CONCAT_ARGS, NO_OBJ ),
-  T01("w",                   K_W,                   ARGS,    NO_OBJ ),
-  T0N("m",                   K_M,               CONCAT_ARGS, NO_OBJ ),
-  T0N("id",                  K_ID,                  GE(2),   NO_OBJ ),
-  T01("pr",                  K_PROTO,           CONCAT_ARGS, NO_OBJ ),
-  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
-  END_OF_TABLE
-};
-
-/** List of tokens common to V3 authority certificates and V3 consensuses. */
-#define CERTIFICATE_MEMBERS                                                  \
-  T1("dir-key-certificate-version", K_DIR_KEY_CERTIFICATE_VERSION,           \
-                                                     GE(1),       NO_OBJ ),  \
-  T1("dir-identity-key", K_DIR_IDENTITY_KEY,         NO_ARGS,     NEED_KEY ),\
-  T1("dir-key-published",K_DIR_KEY_PUBLISHED,        CONCAT_ARGS, NO_OBJ),   \
-  T1("dir-key-expires",  K_DIR_KEY_EXPIRES,          CONCAT_ARGS, NO_OBJ),   \
-  T1("dir-signing-key",  K_DIR_SIGNING_KEY,          NO_ARGS,     NEED_KEY ),\
-  T1("dir-key-crosscert", K_DIR_KEY_CROSSCERT,       NO_ARGS,     NEED_OBJ ),\
-  T1("dir-key-certification", K_DIR_KEY_CERTIFICATION,                       \
-                                                     NO_ARGS,     NEED_OBJ), \
-  T01("dir-address",     K_DIR_ADDRESS,              GE(1),       NO_OBJ),
-
-/** List of tokens recognized in V3 authority certificates. */
-static token_rule_t dir_key_certificate_table[] = {
-  CERTIFICATE_MEMBERS
-  T1("fingerprint",      K_FINGERPRINT,              CONCAT_ARGS, NO_OBJ ),
-  END_OF_TABLE
-};
-
-/** List of tokens recognized in rendezvous service descriptors */
-static token_rule_t desc_token_table[] = {
-  T1_START("rendezvous-service-descriptor", R_RENDEZVOUS_SERVICE_DESCRIPTOR,
-           EQ(1), NO_OBJ),
-  T1("version", R_VERSION, EQ(1), NO_OBJ),
-  T1("permanent-key", R_PERMANENT_KEY, NO_ARGS, NEED_KEY_1024),
-  T1("secret-id-part", R_SECRET_ID_PART, EQ(1), NO_OBJ),
-  T1("publication-time", R_PUBLICATION_TIME, CONCAT_ARGS, NO_OBJ),
-  T1("protocol-versions", R_PROTOCOL_VERSIONS, EQ(1), NO_OBJ),
-  T01("introduction-points", R_INTRODUCTION_POINTS, NO_ARGS, NEED_OBJ),
-  T1_END("signature", R_SIGNATURE, NO_ARGS, NEED_OBJ),
-  END_OF_TABLE
-};
-
-/** List of tokens recognized in the (encrypted) list of introduction points of
- * rendezvous service descriptors */
-static token_rule_t ipo_token_table[] = {
-  T1_START("introduction-point", R_IPO_IDENTIFIER, EQ(1), NO_OBJ),
-  T1("ip-address", R_IPO_IP_ADDRESS, EQ(1), NO_OBJ),
-  T1("onion-port", R_IPO_ONION_PORT, EQ(1), NO_OBJ),
-  T1("onion-key", R_IPO_ONION_KEY, NO_ARGS, NEED_KEY_1024),
-  T1("service-key", R_IPO_SERVICE_KEY, NO_ARGS, NEED_KEY_1024),
-  END_OF_TABLE
-};
-
-/** List of tokens recognized in the (possibly encrypted) list of introduction
- * points of rendezvous service descriptors */
-static token_rule_t client_keys_token_table[] = {
-  T1_START("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ),
-  T1("descriptor-cookie", C_DESCRIPTOR_COOKIE, EQ(1), NO_OBJ),
-  T01("client-key", C_CLIENT_KEY, NO_ARGS, NEED_SKEY_1024),
-  END_OF_TABLE
-};
-
-/** List of tokens recognized in V3 networkstatus votes. */
-static token_rule_t networkstatus_token_table[] = {
-  T1_START("network-status-version", K_NETWORK_STATUS_VERSION,
-                                                   GE(1),       NO_OBJ ),
-  T1("vote-status",            K_VOTE_STATUS,      GE(1),       NO_OBJ ),
-  T1("published",              K_PUBLISHED,        CONCAT_ARGS, NO_OBJ ),
-  T1("valid-after",            K_VALID_AFTER,      CONCAT_ARGS, NO_OBJ ),
-  T1("fresh-until",            K_FRESH_UNTIL,      CONCAT_ARGS, NO_OBJ ),
-  T1("valid-until",            K_VALID_UNTIL,      CONCAT_ARGS, NO_OBJ ),
-  T1("voting-delay",           K_VOTING_DELAY,     GE(2),       NO_OBJ ),
-  T1("known-flags",            K_KNOWN_FLAGS,      ARGS,        NO_OBJ ),
-  T01("params",                K_PARAMS,           ARGS,        NO_OBJ ),
-  T( "fingerprint",            K_FINGERPRINT,      CONCAT_ARGS, NO_OBJ ),
-  T01("signing-ed25519",       K_SIGNING_CERT_ED,  NO_ARGS ,    NEED_OBJ ),
-  T01("shared-rand-participate",K_SR_FLAG,         NO_ARGS,     NO_OBJ ),
-  T0N("shared-rand-commit",    K_COMMIT,           GE(3),       NO_OBJ ),
-  T01("shared-rand-previous-value", K_PREVIOUS_SRV,EQ(2),       NO_OBJ ),
-  T01("shared-rand-current-value",  K_CURRENT_SRV, EQ(2),       NO_OBJ ),
-  T0N("package",               K_PACKAGE,          CONCAT_ARGS, NO_OBJ ),
-  T01("recommended-client-protocols", K_RECOMMENDED_CLIENT_PROTOCOLS,
-      CONCAT_ARGS, NO_OBJ ),
-  T01("recommended-relay-protocols", K_RECOMMENDED_RELAY_PROTOCOLS,
-      CONCAT_ARGS, NO_OBJ ),
-  T01("required-client-protocols",    K_REQUIRED_CLIENT_PROTOCOLS,
-      CONCAT_ARGS, NO_OBJ ),
-  T01("required-relay-protocols",    K_REQUIRED_RELAY_PROTOCOLS,
-      CONCAT_ARGS, NO_OBJ ),
-
-  CERTIFICATE_MEMBERS
-
-  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
-  T1( "contact",             K_CONTACT,         CONCAT_ARGS, NO_OBJ ),
-  T1( "dir-source",          K_DIR_SOURCE,      GE(6),       NO_OBJ ),
-  T01("legacy-dir-key",      K_LEGACY_DIR_KEY,  GE(1),       NO_OBJ ),
-  T1( "known-flags",         K_KNOWN_FLAGS,     CONCAT_ARGS, NO_OBJ ),
-  T01("client-versions",     K_CLIENT_VERSIONS, CONCAT_ARGS, NO_OBJ ),
-  T01("server-versions",     K_SERVER_VERSIONS, CONCAT_ARGS, NO_OBJ ),
-  T1( "consensus-methods",   K_CONSENSUS_METHODS, GE(1),     NO_OBJ ),
-
-  END_OF_TABLE
-};
-
-/** List of tokens recognized in V3 networkstatus consensuses. */
-static token_rule_t networkstatus_consensus_token_table[] = {
-  T1_START("network-status-version", K_NETWORK_STATUS_VERSION,
-                                                   GE(1),       NO_OBJ ),
-  T1("vote-status",            K_VOTE_STATUS,      GE(1),       NO_OBJ ),
-  T1("valid-after",            K_VALID_AFTER,      CONCAT_ARGS, NO_OBJ ),
-  T1("fresh-until",            K_FRESH_UNTIL,      CONCAT_ARGS, NO_OBJ ),
-  T1("valid-until",            K_VALID_UNTIL,      CONCAT_ARGS, NO_OBJ ),
-  T1("voting-delay",           K_VOTING_DELAY,     GE(2),       NO_OBJ ),
-
-  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
-
-  T1N("dir-source",          K_DIR_SOURCE,          GE(6),   NO_OBJ ),
-  T1N("contact",             K_CONTACT,         CONCAT_ARGS, NO_OBJ ),
-  T1N("vote-digest",         K_VOTE_DIGEST,         GE(1),   NO_OBJ ),
-
-  T1( "known-flags",         K_KNOWN_FLAGS,     CONCAT_ARGS, NO_OBJ ),
-
-  T01("client-versions",     K_CLIENT_VERSIONS, CONCAT_ARGS, NO_OBJ ),
-  T01("server-versions",     K_SERVER_VERSIONS, CONCAT_ARGS, NO_OBJ ),
-  T01("consensus-method",    K_CONSENSUS_METHOD,    EQ(1),   NO_OBJ),
-  T01("params",                K_PARAMS,           ARGS,        NO_OBJ ),
-
-  T01("shared-rand-previous-value", K_PREVIOUS_SRV, EQ(2),   NO_OBJ ),
-  T01("shared-rand-current-value",  K_CURRENT_SRV,  EQ(2),   NO_OBJ ),
-
-  T01("recommended-client-protocols", K_RECOMMENDED_CLIENT_PROTOCOLS,
-      CONCAT_ARGS, NO_OBJ ),
-  T01("recommended-relay-protocols", K_RECOMMENDED_RELAY_PROTOCOLS,
-      CONCAT_ARGS, NO_OBJ ),
-  T01("required-client-protocols",    K_REQUIRED_CLIENT_PROTOCOLS,
-      CONCAT_ARGS, NO_OBJ ),
-  T01("required-relay-protocols",    K_REQUIRED_RELAY_PROTOCOLS,
-      CONCAT_ARGS, NO_OBJ ),
-
-  END_OF_TABLE
-};
-
-/** List of tokens recognized in the footer of v1 directory footers. */
-static token_rule_t networkstatus_vote_footer_token_table[] = {
-  T01("directory-footer",    K_DIRECTORY_FOOTER,    NO_ARGS,   NO_OBJ ),
-  T01("bandwidth-weights",   K_BW_WEIGHTS,          ARGS,      NO_OBJ ),
-  T(  "directory-signature", K_DIRECTORY_SIGNATURE, GE(2),     NEED_OBJ ),
-  END_OF_TABLE
-};
-
-/** List of tokens recognized in detached networkstatus signature documents. */
-static token_rule_t networkstatus_detached_signature_token_table[] = {
-  T1_START("consensus-digest", K_CONSENSUS_DIGEST, GE(1),       NO_OBJ ),
-  T("additional-digest",       K_ADDITIONAL_DIGEST,GE(3),       NO_OBJ ),
-  T1("valid-after",            K_VALID_AFTER,      CONCAT_ARGS, NO_OBJ ),
-  T1("fresh-until",            K_FRESH_UNTIL,      CONCAT_ARGS, NO_OBJ ),
-  T1("valid-until",            K_VALID_UNTIL,      CONCAT_ARGS, NO_OBJ ),
-  T("additional-signature",  K_ADDITIONAL_SIGNATURE, GE(4),   NEED_OBJ ),
-  T1N("directory-signature", K_DIRECTORY_SIGNATURE,  GE(2),   NEED_OBJ ),
-  END_OF_TABLE
-};
-
-/** List of tokens recognized in microdescriptors */
-static token_rule_t microdesc_token_table[] = {
-  T1_START("onion-key",        K_ONION_KEY,        NO_ARGS,     NEED_KEY_1024),
-  T01("ntor-onion-key",        K_ONION_KEY_NTOR,   GE(1),       NO_OBJ ),
-  T0N("id",                    K_ID,               GE(2),       NO_OBJ ),
-  T0N("a",                     K_A,                GE(1),       NO_OBJ ),
-  T01("family",                K_FAMILY,           ARGS,        NO_OBJ ),
-  T01("p",                     K_P,                CONCAT_ARGS, NO_OBJ ),
-  T01("p6",                    K_P6,               CONCAT_ARGS, NO_OBJ ),
-  A01("@last-listed",          A_LAST_LISTED,      CONCAT_ARGS, NO_OBJ ),
-  END_OF_TABLE
-};
-
-#undef T
-
-/* static function prototypes */
-static int router_add_exit_policy(routerinfo_t *router,directory_token_t *tok);
-static addr_policy_t *router_parse_addr_policy(directory_token_t *tok,
-                                               unsigned fmt_flags);
-static addr_policy_t *router_parse_addr_policy_private(directory_token_t *tok);
-
-static int router_get_hash_impl_helper(const char *s, size_t s_len,
-                            const char *start_str,
-                            const char *end_str, char end_c,
-                            const char **start_out, const char **end_out);
-static int router_get_hash_impl(const char *s, size_t s_len, char *digest,
-                                const char *start_str, const char *end_str,
-                                char end_char,
-                                digest_algorithm_t alg);
-static int router_get_hashes_impl(const char *s, size_t s_len,
-                                  common_digests_t *digests,
-                                  const char *start_str, const char *end_str,
-                                  char end_char);
-static void token_clear(directory_token_t *tok);
-static smartlist_t *find_all_by_keyword(smartlist_t *s, directory_keyword k);
-static smartlist_t *find_all_exitpolicy(smartlist_t *s);
-static directory_token_t *find_by_keyword_(smartlist_t *s,
-                                           directory_keyword keyword,
-                                           const char *keyword_str);
-#define find_by_keyword(s, keyword) find_by_keyword_((s), (keyword), #keyword)
-static directory_token_t *find_opt_by_keyword(smartlist_t *s,
-                                              directory_keyword keyword);
-
-#define TS_ANNOTATIONS_OK 1
-#define TS_NOCHECK 2
-#define TS_NO_NEW_ANNOTATIONS 4
-static int tokenize_string(memarea_t *area,
-                           const char *start, const char *end,
-                           smartlist_t *out,
-                           token_rule_t *table,
-                           int flags);
-static directory_token_t *get_next_token(memarea_t *area,
-                                         const char **s,
-                                         const char *eos,
-                                         token_rule_t *table);
-#define CST_CHECK_AUTHORITY   (1<<0)
-#define CST_NO_CHECK_OBJTYPE  (1<<1)
-static int check_signature_token(const char *digest,
-                                 ssize_t digest_len,
-                                 directory_token_t *tok,
-                                 crypto_pk_t *pkey,
-                                 int flags,
-                                 const char *doctype);
-
-#undef DEBUG_AREA_ALLOC
-
-#ifdef DEBUG_AREA_ALLOC
-#define DUMP_AREA(a,name) STMT_BEGIN                              \
-  size_t alloc=0, used=0;                                         \
-  memarea_get_stats((a),&alloc,&used);                            \
-  log_debug(LD_MM, "Area for %s has %lu allocated; using %lu.",   \
-            name, (unsigned long)alloc, (unsigned long)used);     \
-  STMT_END
-#else
-#define DUMP_AREA(a,name) STMT_NIL
-#endif
-
-/* Dump mechanism for unparseable descriptors */
-
-/** List of dumped descriptors for FIFO cleanup purposes */
-STATIC smartlist_t *descs_dumped = NULL;
-/** Total size of dumped descriptors for FIFO cleanup */
-STATIC uint64_t len_descs_dumped = 0;
-/** Directory to stash dumps in */
-static int have_dump_desc_dir = 0;
-static int problem_with_dump_desc_dir = 0;
-
-#define DESC_DUMP_DATADIR_SUBDIR "unparseable-descs"
-#define DESC_DUMP_BASE_FILENAME "unparseable-desc"
-
-/** Find the dump directory and check if we'll be able to create it */
-static void
-dump_desc_init(void)
-{
-  char *dump_desc_dir;
-
-  dump_desc_dir = get_datadir_fname(DESC_DUMP_DATADIR_SUBDIR);
-
-  /*
-   * We just check for it, don't create it at this point; we'll
-   * create it when we need it if it isn't already there.
-   */
-  if (check_private_dir(dump_desc_dir, CPD_CHECK, get_options()->User) < 0) {
-    /* Error, log and flag it as having a problem */
-    log_notice(LD_DIR,
-               "Doesn't look like we'll be able to create descriptor dump "
-               "directory %s; dumps will be disabled.",
-               dump_desc_dir);
-    problem_with_dump_desc_dir = 1;
-    tor_free(dump_desc_dir);
-    return;
-  }
-
-  /* Check if it exists */
-  switch (file_status(dump_desc_dir)) {
-    case FN_DIR:
-      /* We already have a directory */
-      have_dump_desc_dir = 1;
-      break;
-    case FN_NOENT:
-      /* Nothing, we'll need to create it later */
-      have_dump_desc_dir = 0;
-      break;
-    case FN_ERROR:
-      /* Log and flag having a problem */
-      log_notice(LD_DIR,
-                 "Couldn't check whether descriptor dump directory %s already"
-                 " exists: %s",
-                 dump_desc_dir, strerror(errno));
-      problem_with_dump_desc_dir = 1;
-      break;
-    case FN_FILE:
-    case FN_EMPTY:
-    default:
-      /* Something else was here! */
-      log_notice(LD_DIR,
-                 "Descriptor dump directory %s already exists and isn't a "
-                 "directory",
-                 dump_desc_dir);
-      problem_with_dump_desc_dir = 1;
-  }
-
-  if (have_dump_desc_dir && !problem_with_dump_desc_dir) {
-    dump_desc_populate_fifo_from_directory(dump_desc_dir);
-  }
-
-  tor_free(dump_desc_dir);
-}
-
-/** Create the dump directory if needed and possible */
-static void
-dump_desc_create_dir(void)
-{
-  char *dump_desc_dir;
-
-  /* If the problem flag is set, skip it */
-  if (problem_with_dump_desc_dir) return;
-
-  /* Do we need it? */
-  if (!have_dump_desc_dir) {
-    dump_desc_dir = get_datadir_fname(DESC_DUMP_DATADIR_SUBDIR);
-
-    if (check_private_dir(dump_desc_dir, CPD_CREATE,
-                          get_options()->User) < 0) {
-      log_notice(LD_DIR,
-                 "Failed to create descriptor dump directory %s",
-                 dump_desc_dir);
-      problem_with_dump_desc_dir = 1;
-    }
-
-    /* Okay, we created it */
-    have_dump_desc_dir = 1;
-
-    tor_free(dump_desc_dir);
-  }
-}
-
-/** Dump desc FIFO/cleanup; take ownership of the given filename, add it to
- * the FIFO, and clean up the oldest entries to the extent they exceed the
- * configured cap.  If any old entries with a matching hash existed, they
- * just got overwritten right before this was called and we should adjust
- * the total size counter without deleting them.
- */
-static void
-dump_desc_fifo_add_and_clean(char *filename, const uint8_t *digest_sha256,
-                             size_t len)
-{
-  dumped_desc_t *ent = NULL, *tmp;
-  uint64_t max_len;
-
-  tor_assert(filename != NULL);
-  tor_assert(digest_sha256 != NULL);
-
-  if (descs_dumped == NULL) {
-    /* We better have no length, then */
-    tor_assert(len_descs_dumped == 0);
-    /* Make a smartlist */
-    descs_dumped = smartlist_new();
-  }
-
-  /* Make a new entry to put this one in */
-  ent = tor_malloc_zero(sizeof(*ent));
-  ent->filename = filename;
-  ent->len = len;
-  ent->when = time(NULL);
-  memcpy(ent->digest_sha256, digest_sha256, DIGEST256_LEN);
-
-  /* Do we need to do some cleanup? */
-  max_len = get_options()->MaxUnparseableDescSizeToLog;
-  /* Iterate over the list until we've freed enough space */
-  while (len > max_len - len_descs_dumped &&
-         smartlist_len(descs_dumped) > 0) {
-    /* Get the oldest thing on the list */
-    tmp = (dumped_desc_t *)(smartlist_get(descs_dumped, 0));
-
-    /*
-     * Check if it matches the filename we just added, so we don't delete
-     * something we just emitted if we get repeated identical descriptors.
-     */
-    if (strcmp(tmp->filename, filename) != 0) {
-      /* Delete it and adjust the length counter */
-      tor_unlink(tmp->filename);
-      tor_assert(len_descs_dumped >= tmp->len);
-      len_descs_dumped -= tmp->len;
-      log_info(LD_DIR,
-               "Deleting old unparseable descriptor dump %s due to "
-               "space limits",
-               tmp->filename);
-    } else {
-      /*
-       * Don't delete, but do adjust the counter since we will bump it
-       * later
-       */
-      tor_assert(len_descs_dumped >= tmp->len);
-      len_descs_dumped -= tmp->len;
-      log_info(LD_DIR,
-               "Replacing old descriptor dump %s with new identical one",
-               tmp->filename);
-    }
-
-    /* Free it and remove it from the list */
-    smartlist_del_keeporder(descs_dumped, 0);
-    tor_free(tmp->filename);
-    tor_free(tmp);
-  }
-
-  /* Append our entry to the end of the list and bump the counter */
-  smartlist_add(descs_dumped, ent);
-  len_descs_dumped += len;
-}
-
-/** Check if we already have a descriptor for this hash and move it to the
- * head of the queue if so.  Return 1 if one existed and 0 otherwise.
- */
-static int
-dump_desc_fifo_bump_hash(const uint8_t *digest_sha256)
-{
-  dumped_desc_t *match = NULL;
-
-  tor_assert(digest_sha256);
-
-  if (descs_dumped) {
-    /* Find a match if one exists */
-    SMARTLIST_FOREACH_BEGIN(descs_dumped, dumped_desc_t *, ent) {
-      if (ent &&
-          tor_memeq(ent->digest_sha256, digest_sha256, DIGEST256_LEN)) {
-        /*
-         * Save a pointer to the match and remove it from its current
-         * position.
-         */
-        match = ent;
-        SMARTLIST_DEL_CURRENT_KEEPORDER(descs_dumped, ent);
-        break;
-      }
-    } SMARTLIST_FOREACH_END(ent);
-
-    if (match) {
-      /* Update the timestamp */
-      match->when = time(NULL);
-      /* Add it back at the end of the list */
-      smartlist_add(descs_dumped, match);
-
-      /* Indicate we found one */
-      return 1;
-    }
-  }
-
-  return 0;
-}
-
-/** Clean up on exit; just memory, leave the dumps behind
- */
-STATIC void
-dump_desc_fifo_cleanup(void)
-{
-  if (descs_dumped) {
-    /* Free each descriptor */
-    SMARTLIST_FOREACH_BEGIN(descs_dumped, dumped_desc_t *, ent) {
-      tor_assert(ent);
-      tor_free(ent->filename);
-      tor_free(ent);
-    } SMARTLIST_FOREACH_END(ent);
-    /* Free the list */
-    smartlist_free(descs_dumped);
-    descs_dumped = NULL;
-    len_descs_dumped = 0;
-  }
-}
-
-/** Handle one file for dump_desc_populate_fifo_from_directory(); make sure
- * the filename is sensibly formed and matches the file content, and either
- * return a dumped_desc_t for it or remove the file and return NULL.
- */
-MOCK_IMPL(STATIC dumped_desc_t *,
-dump_desc_populate_one_file, (const char *dirname, const char *f))
-{
-  dumped_desc_t *ent = NULL;
-  char *path = NULL, *desc = NULL;
-  const char *digest_str;
-  char digest[DIGEST256_LEN], content_digest[DIGEST256_LEN];
-  /* Expected prefix before digest in filenames */
-  const char *f_pfx = DESC_DUMP_BASE_FILENAME ".";
-  /*
-   * Stat while reading; this is important in case the file
-   * contains a NUL character.
-   */
-  struct stat st;
-
-  /* Sanity-check args */
-  tor_assert(dirname != NULL);
-  tor_assert(f != NULL);
-
-  /* Form the full path */
-  tor_asprintf(&path, "%s" PATH_SEPARATOR "%s", dirname, f);
-
-  /* Check that f has the form DESC_DUMP_BASE_FILENAME.<digest256> */
-
-  if (!strcmpstart(f, f_pfx)) {
-    /* It matches the form, but is the digest parseable as such? */
-    digest_str = f + strlen(f_pfx);
-    if (base16_decode(digest, DIGEST256_LEN,
-                      digest_str, strlen(digest_str)) != DIGEST256_LEN) {
-      /* We failed to decode it */
-      digest_str = NULL;
-    }
-  } else {
-    /* No match */
-    digest_str = NULL;
-  }
-
-  if (!digest_str) {
-    /* We couldn't get a sensible digest */
-    log_notice(LD_DIR,
-               "Removing unrecognized filename %s from unparseable "
-               "descriptors directory", f);
-    tor_unlink(path);
-    /* We're done */
-    goto done;
-  }
-
-  /*
-   * The filename has the form DESC_DUMP_BASE_FILENAME "." <digest256> and
-   * we've decoded the digest.  Next, check that we can read it and the
-   * content matches this digest.  We are relying on the fact that if the
-   * file contains a '\0', read_file_to_str() will allocate space for and
-   * read the entire file and return the correct size in st.
-   */
-  desc = read_file_to_str(path, RFTS_IGNORE_MISSING|RFTS_BIN, &st);
-  if (!desc) {
-    /* We couldn't read it */
-    log_notice(LD_DIR,
-               "Failed to read %s from unparseable descriptors directory; "
-               "attempting to remove it.", f);
-    tor_unlink(path);
-    /* We're done */
-    goto done;
-  }
-
-#if SIZE_MAX > UINT64_MAX
-  if (BUG((uint64_t)st.st_size > (uint64_t)SIZE_MAX)) {
-    /* LCOV_EXCL_START
-     * Should be impossible since RFTS above should have failed to read the
-     * huge file into RAM. */
-    goto done;
-    /* LCOV_EXCL_STOP */
-  }
-#endif
-  if (BUG(st.st_size < 0)) {
-    /* LCOV_EXCL_START
-     * Should be impossible, since the OS isn't supposed to be b0rken. */
-    goto done;
-    /* LCOV_EXCL_STOP */
-  }
-  /* (Now we can be sure that st.st_size is safe to cast to a size_t.) */
-
-  /*
-   * We got one; now compute its digest and check that it matches the
-   * filename.
-   */
-  if (crypto_digest256((char *)content_digest, desc, (size_t) st.st_size,
-                       DIGEST_SHA256) != 0) {
-    /* Weird, but okay */
-    log_info(LD_DIR,
-             "Unable to hash content of %s from unparseable descriptors "
-             "directory", f);
-    tor_unlink(path);
-    /* We're done */
-    goto done;
-  }
-
-  /* Compare the digests */
-  if (tor_memneq(digest, content_digest, DIGEST256_LEN)) {
-    /* No match */
-    log_info(LD_DIR,
-             "Hash of %s from unparseable descriptors directory didn't "
-             "match its filename; removing it", f);
-    tor_unlink(path);
-    /* We're done */
-    goto done;
-  }
-
-  /* Okay, it's a match, we should prepare ent */
-  ent = tor_malloc_zero(sizeof(dumped_desc_t));
-  ent->filename = path;
-  memcpy(ent->digest_sha256, digest, DIGEST256_LEN);
-  ent->len = (size_t) st.st_size;
-  ent->when = st.st_mtime;
-  /* Null out path so we don't free it out from under ent */
-  path = NULL;
-
- done:
-  /* Free allocations if we had them */
-  tor_free(desc);
-  tor_free(path);
-
-  return ent;
-}
-
-/** Sort helper for dump_desc_populate_fifo_from_directory(); compares
- * the when field of dumped_desc_ts in a smartlist to put the FIFO in
- * the correct order after reconstructing it from the directory.
- */
-static int
-dump_desc_compare_fifo_entries(const void **a_v, const void **b_v)
-{
-  const dumped_desc_t **a = (const dumped_desc_t **)a_v;
-  const dumped_desc_t **b = (const dumped_desc_t **)b_v;
-
-  if ((a != NULL) && (*a != NULL)) {
-    if ((b != NULL) && (*b != NULL)) {
-      /* We have sensible dumped_desc_ts to compare */
-      if ((*a)->when < (*b)->when) {
-        return -1;
-      } else if ((*a)->when == (*b)->when) {
-        return 0;
-      } else {
-        return 1;
-      }
-    } else {
-      /*
-       * We shouldn't see this, but what the hell, NULLs precede everythin
-       * else
-       */
-      return 1;
-    }
-  } else {
-    return -1;
-  }
-}
-
-/** Scan the contents of the directory, and update FIFO/counters; this will
- * consistency-check descriptor dump filenames against hashes of descriptor
- * dump file content, and remove any inconsistent/unreadable dumps, and then
- * reconstruct the dump FIFO as closely as possible for the last time the
- * tor process shut down.  If a previous dump was repeated more than once and
- * moved ahead in the FIFO, the mtime will not have been updated and the
- * reconstructed order will be wrong, but will always be a permutation of
- * the original.
- */
-STATIC void
-dump_desc_populate_fifo_from_directory(const char *dirname)
-{
-  smartlist_t *files = NULL;
-  dumped_desc_t *ent = NULL;
-
-  tor_assert(dirname != NULL);
-
-  /* Get a list of files */
-  files = tor_listdir(dirname);
-  if (!files) {
-    log_notice(LD_DIR,
-               "Unable to get contents of unparseable descriptor dump "
-               "directory %s",
-               dirname);
-    return;
-  }
-
-  /*
-   * Iterate through the list and decide which files should go in the
-   * FIFO and which should be purged.
-   */
-
-  SMARTLIST_FOREACH_BEGIN(files, char *, f) {
-    /* Try to get a FIFO entry */
-    ent = dump_desc_populate_one_file(dirname, f);
-    if (ent) {
-      /*
-       * We got one; add it to the FIFO.  No need for duplicate checking
-       * here since we just verified the name and digest match.
-       */
-
-      /* Make sure we have a list to add it to */
-      if (!descs_dumped) {
-        descs_dumped = smartlist_new();
-        len_descs_dumped = 0;
-      }
-
-      /* Add it and adjust the counter */
-      smartlist_add(descs_dumped, ent);
-      len_descs_dumped += ent->len;
-    }
-    /*
-     * If we didn't, we will have unlinked the file if necessary and
-     * possible, and emitted a log message about it, so just go on to
-     * the next.
-     */
-  } SMARTLIST_FOREACH_END(f);
-
-  /* Did we get anything? */
-  if (descs_dumped != NULL) {
-    /* Sort the FIFO in order of increasing timestamp */
-    smartlist_sort(descs_dumped, dump_desc_compare_fifo_entries);
-
-    /* Log some stats */
-    log_info(LD_DIR,
-             "Reloaded unparseable descriptor dump FIFO with %d dump(s) "
-             "totaling " U64_FORMAT " bytes",
-             smartlist_len(descs_dumped), U64_PRINTF_ARG(len_descs_dumped));
-  }
-
-  /* Free the original list */
-  SMARTLIST_FOREACH(files, char *, f, tor_free(f));
-  smartlist_free(files);
-}
-
-/** For debugging purposes, dump unparseable descriptor *<b>desc</b> of
- * type *<b>type</b> to file $DATADIR/unparseable-desc. Do not write more
- * than one descriptor to disk per minute. If there is already such a
- * file in the data directory, overwrite it. */
-STATIC void
-dump_desc(const char *desc, const char *type)
-{
-  tor_assert(desc);
-  tor_assert(type);
-  size_t len;
-  /* The SHA256 of the string */
-  uint8_t digest_sha256[DIGEST256_LEN];
-  char digest_sha256_hex[HEX_DIGEST256_LEN+1];
-  /* Filename to log it to */
-  char *debugfile, *debugfile_base;
-
-  /* Get the hash for logging purposes anyway */
-  len = strlen(desc);
-  if (crypto_digest256((char *)digest_sha256, desc, len,
-                       DIGEST_SHA256) != 0) {
-    log_info(LD_DIR,
-             "Unable to parse descriptor of type %s, and unable to even hash"
-             " it!", type);
-    goto err;
-  }
-
-  base16_encode(digest_sha256_hex, sizeof(digest_sha256_hex),
-                (const char *)digest_sha256, sizeof(digest_sha256));
-
-  /*
-   * We mention type and hash in the main log; don't clutter up the files
-   * with anything but the exact dump.
-   */
-  tor_asprintf(&debugfile_base,
-               DESC_DUMP_BASE_FILENAME ".%s", digest_sha256_hex);
-  debugfile = get_datadir_fname2(DESC_DUMP_DATADIR_SUBDIR, debugfile_base);
-
-  /*
-   * Check if the sandbox is active or will become active; see comment
-   * below at the log message for why.
-   */
-  if (!(sandbox_is_active() || get_options()->Sandbox)) {
-    if (len <= get_options()->MaxUnparseableDescSizeToLog) {
-      if (!dump_desc_fifo_bump_hash(digest_sha256)) {
-        /* Create the directory if needed */
-        dump_desc_create_dir();
-        /* Make sure we've got it */
-        if (have_dump_desc_dir && !problem_with_dump_desc_dir) {
-          /* Write it, and tell the main log about it */
-          write_str_to_file(debugfile, desc, 1);
-          log_info(LD_DIR,
-                   "Unable to parse descriptor of type %s with hash %s and "
-                   "length %lu. See file %s in data directory for details.",
-                   type, digest_sha256_hex, (unsigned long)len,
-                   debugfile_base);
-          dump_desc_fifo_add_and_clean(debugfile, digest_sha256, len);
-          /* Since we handed ownership over, don't free debugfile later */
-          debugfile = NULL;
-        } else {
-          /* Problem with the subdirectory */
-          log_info(LD_DIR,
-                   "Unable to parse descriptor of type %s with hash %s and "
-                   "length %lu. Descriptor not dumped because we had a "
-                   "problem creating the " DESC_DUMP_DATADIR_SUBDIR
-                   " subdirectory",
-                   type, digest_sha256_hex, (unsigned long)len);
-          /* We do have to free debugfile in this case */
-        }
-      } else {
-        /* We already had one with this hash dumped */
-        log_info(LD_DIR,
-                 "Unable to parse descriptor of type %s with hash %s and "
-                 "length %lu. Descriptor not dumped because one with that "
-                 "hash has already been dumped.",
-                 type, digest_sha256_hex, (unsigned long)len);
-        /* We do have to free debugfile in this case */
-      }
-    } else {
-      /* Just log that it happened without dumping */
-      log_info(LD_DIR,
-               "Unable to parse descriptor of type %s with hash %s and "
-               "length %lu. Descriptor not dumped because it exceeds maximum"
-               " log size all by itself.",
-               type, digest_sha256_hex, (unsigned long)len);
-      /* We do have to free debugfile in this case */
-    }
-  } else {
-    /*
-     * Not logging because the sandbox is active and seccomp2 apparently
-     * doesn't have a sensible way to allow filenames according to a pattern
-     * match.  (If we ever figure out how to say "allow writes to /regex/",
-     * remove this checK).
-     */
-    log_info(LD_DIR,
-             "Unable to parse descriptor of type %s with hash %s and "
-             "length %lu. Descriptor not dumped because the sandbox is "
-             "configured",
-             type, digest_sha256_hex, (unsigned long)len);
-  }
-
-  tor_free(debugfile_base);
-  tor_free(debugfile);
-
- err:
-  return;
-}
-
-/** Set <b>digest</b> to the SHA-1 digest of the hash of the directory in
- * <b>s</b>.  Return 0 on success, -1 on failure.
- */
-int
-router_get_dir_hash(const char *s, char *digest)
-{
-  return router_get_hash_impl(s, strlen(s), digest,
-                              "signed-directory","\ndirectory-signature",'\n',
-                              DIGEST_SHA1);
-}
-
-/** Set <b>digest</b> to the SHA-1 digest of the hash of the first router in
- * <b>s</b>. Return 0 on success, -1 on failure.
- */
-int
-router_get_router_hash(const char *s, size_t s_len, char *digest)
-{
-  return router_get_hash_impl(s, s_len, digest,
-                              "router ","\nrouter-signature", '\n',
-                              DIGEST_SHA1);
-}
-
-/** Set <b>digests</b> to all the digests of the consensus document in
- * <b>s</b> */
-int
-router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests)
-{
-  return router_get_hashes_impl(s,strlen(s),digests,
-                                "network-status-version",
-                                "\ndirectory-signature",
-                                ' ');
-}
-
-/** Set <b>digest</b> to the SHA-1 digest of the hash of the <b>s_len</b>-byte
- * extrainfo string at <b>s</b>.  Return 0 on success, -1 on failure. */
-int
-router_get_extrainfo_hash(const char *s, size_t s_len, char *digest)
-{
-  return router_get_hash_impl(s, s_len, digest, "extra-info",
-                              "\nrouter-signature",'\n', DIGEST_SHA1);
-}
-
-/** Helper: used to generate signatures for routers, directories and
- * network-status objects.  Given a <b>digest_len</b>-byte digest in
- * <b>digest</b> and a secret <b>private_key</b>, generate an PKCS1-padded
- * signature, BASE64-encode it, surround it with -----BEGIN/END----- pairs,
- * and return the new signature on success or NULL on failure.
- */
-char *
-router_get_dirobj_signature(const char *digest,
-                            size_t digest_len,
-                            const crypto_pk_t *private_key)
-{
-  char *signature;
-  size_t i, keysize;
-  int siglen;
-  char *buf = NULL;
-  size_t buf_len;
-  /* overestimate of BEGIN/END lines total len. */
-#define BEGIN_END_OVERHEAD_LEN 64
-
-  keysize = crypto_pk_keysize(private_key);
-  signature = tor_malloc(keysize);
-  siglen = crypto_pk_private_sign(private_key, signature, keysize,
-                                  digest, digest_len);
-  if (siglen < 0) {
-    log_warn(LD_BUG,"Couldn't sign digest.");
-    goto err;
-  }
-
-  /* The *2 here is a ridiculous overestimate of base-64 overhead. */
-  buf_len = (siglen * 2) + BEGIN_END_OVERHEAD_LEN;
-  buf = tor_malloc(buf_len);
-
-  if (strlcpy(buf, "-----BEGIN SIGNATURE-----\n", buf_len) >= buf_len)
-    goto truncated;
-
-  i = strlen(buf);
-  if (base64_encode(buf+i, buf_len-i, signature, siglen,
-                    BASE64_ENCODE_MULTILINE) < 0) {
-    log_warn(LD_BUG,"couldn't base64-encode signature");
-    goto err;
-  }
-
-  if (strlcat(buf, "-----END SIGNATURE-----\n", buf_len) >= buf_len)
-    goto truncated;
-
-  tor_free(signature);
-  return buf;
-
- truncated:
-  log_warn(LD_BUG,"tried to exceed string length.");
- err:
-  tor_free(signature);
-  tor_free(buf);
-  return NULL;
-}
-
-/** Helper: used to generate signatures for routers, directories and
- * network-status objects.  Given a digest in <b>digest</b> and a secret
- * <b>private_key</b>, generate a PKCS1-padded signature, BASE64-encode it,
- * surround it with -----BEGIN/END----- pairs, and write it to the
- * <b>buf_len</b>-byte buffer at <b>buf</b>.  Return 0 on success, -1 on
- * failure.
- */
-int
-router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest,
-                               size_t digest_len, crypto_pk_t *private_key)
-{
-  size_t sig_len, s_len;
-  char *sig = router_get_dirobj_signature(digest, digest_len, private_key);
-  if (!sig) {
-    log_warn(LD_BUG, "No signature generated");
-    return -1;
-  }
-  sig_len = strlen(sig);
-  s_len = strlen(buf);
-  if (sig_len + s_len + 1 > buf_len) {
-    log_warn(LD_BUG, "Not enough room for signature");
-    tor_free(sig);
-    return -1;
-  }
-  memcpy(buf+s_len, sig, sig_len+1);
-  tor_free(sig);
-  return 0;
-}
-
-/** Return VS_RECOMMENDED if <b>myversion</b> is contained in
- * <b>versionlist</b>.  Else, return VS_EMPTY if versionlist has no
- * entries. Else, return VS_OLD if every member of
- * <b>versionlist</b> is newer than <b>myversion</b>.  Else, return
- * VS_NEW_IN_SERIES if there is at least one member of <b>versionlist</b> in
- * the same series (major.minor.micro) as <b>myversion</b>, but no such member
- * is newer than <b>myversion.</b>.  Else, return VS_NEW if every member of
- * <b>versionlist</b> is older than <b>myversion</b>.  Else, return
- * VS_UNRECOMMENDED.
- *
- * (versionlist is a comma-separated list of version strings,
- * optionally prefixed with "Tor".  Versions that can't be parsed are
- * ignored.)
- */
-version_status_t
-tor_version_is_obsolete(const char *myversion, const char *versionlist)
-{
-  tor_version_t mine, other;
-  int found_newer = 0, found_older = 0, found_newer_in_series = 0,
-    found_any_in_series = 0, r, same;
-  version_status_t ret = VS_UNRECOMMENDED;
-  smartlist_t *version_sl;
-
-  log_debug(LD_CONFIG,"Checking whether version '%s' is in '%s'",
-            myversion, versionlist);
-
-  if (tor_version_parse(myversion, &mine)) {
-    log_err(LD_BUG,"I couldn't parse my own version (%s)", myversion);
-    tor_assert(0);
-  }
-  version_sl = smartlist_new();
-  smartlist_split_string(version_sl, versionlist, ",", SPLIT_SKIP_SPACE, 0);
-
-  if (!strlen(versionlist)) { /* no authorities cared or agreed */
-    ret = VS_EMPTY;
-    goto done;
-  }
-
-  SMARTLIST_FOREACH_BEGIN(version_sl, const char *, cp) {
-    if (!strcmpstart(cp, "Tor "))
-      cp += 4;
-
-    if (tor_version_parse(cp, &other)) {
-      /* Couldn't parse other; it can't be a match. */
-    } else {
-      same = tor_version_same_series(&mine, &other);
-      if (same)
-        found_any_in_series = 1;
-      r = tor_version_compare(&mine, &other);
-      if (r==0) {
-        ret = VS_RECOMMENDED;
-        goto done;
-      } else if (r<0) {
-        found_newer = 1;
-        if (same)
-          found_newer_in_series = 1;
-      } else if (r>0) {
-        found_older = 1;
-      }
-    }
-  } SMARTLIST_FOREACH_END(cp);
-
-  /* We didn't find the listed version. Is it new or old? */
-  if (found_any_in_series && !found_newer_in_series && found_newer) {
-    ret = VS_NEW_IN_SERIES;
-  } else if (found_newer && !found_older) {
-    ret = VS_OLD;
-  } else if (found_older && !found_newer) {
-    ret = VS_NEW;
-  } else {
-    ret = VS_UNRECOMMENDED;
-  }
-
- done:
-  SMARTLIST_FOREACH(version_sl, char *, version, tor_free(version));
-  smartlist_free(version_sl);
-  return ret;
-}
-
-/** Return true iff <b>key</b> is allowed to sign directories.
- */
-static int
-dir_signing_key_is_trusted(crypto_pk_t *key)
-{
-  char digest[DIGEST_LEN];
-  if (!key) return 0;
-  if (crypto_pk_get_digest(key, digest) < 0) {
-    log_warn(LD_DIR, "Error computing dir-signing-key digest");
-    return 0;
-  }
-  if (!router_digest_is_trusted_dir(digest)) {
-    log_warn(LD_DIR, "Listed dir-signing-key is not trusted");
-    return 0;
-  }
-  return 1;
-}
-
-/** Check whether the object body of the token in <b>tok</b> has a good
- * signature for <b>digest</b> using key <b>pkey</b>.  If
- * <b>CST_CHECK_AUTHORITY</b> is set, make sure that <b>pkey</b> is the key of
- * a directory authority.  If <b>CST_NO_CHECK_OBJTYPE</b> is set, do not check
- * the object type of the signature object. Use <b>doctype</b> as the type of
- * the document when generating log messages.  Return 0 on success, negative
- * on failure.
- */
-static int
-check_signature_token(const char *digest,
-                      ssize_t digest_len,
-                      directory_token_t *tok,
-                      crypto_pk_t *pkey,
-                      int flags,
-                      const char *doctype)
-{
-  char *signed_digest;
-  size_t keysize;
-  const int check_authority = (flags & CST_CHECK_AUTHORITY);
-  const int check_objtype = ! (flags & CST_NO_CHECK_OBJTYPE);
-
-  tor_assert(pkey);
-  tor_assert(tok);
-  tor_assert(digest);
-  tor_assert(doctype);
-
-  if (check_authority && !dir_signing_key_is_trusted(pkey)) {
-    log_warn(LD_DIR, "Key on %s did not come from an authority; rejecting",
-             doctype);
-    return -1;
-  }
-
-  if (check_objtype) {
-    if (strcmp(tok->object_type, "SIGNATURE")) {
-      log_warn(LD_DIR, "Bad object type on %s signature", doctype);
-      return -1;
-    }
-  }
-
-  keysize = crypto_pk_keysize(pkey);
-  signed_digest = tor_malloc(keysize);
-  if (crypto_pk_public_checksig(pkey, signed_digest, keysize,
-                                tok->object_body, tok->object_size)
-      < digest_len) {
-    log_warn(LD_DIR, "Error reading %s: invalid signature.", doctype);
-    tor_free(signed_digest);
-    return -1;
-  }
-  //  log_debug(LD_DIR,"Signed %s hash starts %s", doctype,
-  //            hex_str(signed_digest,4));
-  if (tor_memneq(digest, signed_digest, digest_len)) {
-    log_warn(LD_DIR, "Error reading %s: signature does not match.", doctype);
-    tor_free(signed_digest);
-    return -1;
-  }
-  tor_free(signed_digest);
-  return 0;
-}
-
-/** Helper: move *<b>s_ptr</b> ahead to the next router, the next extra-info,
- * or to the first of the annotations proceeding the next router or
- * extra-info---whichever comes first.  Set <b>is_extrainfo_out</b> to true if
- * we found an extrainfo, or false if found a router. Do not scan beyond
- * <b>eos</b>.  Return -1 if we found nothing; 0 if we found something. */
-static int
-find_start_of_next_router_or_extrainfo(const char **s_ptr,
-                                       const char *eos,
-                                       int *is_extrainfo_out)
-{
-  const char *annotations = NULL;
-  const char *s = *s_ptr;
-
-  s = eat_whitespace_eos(s, eos);
-
-  while (s < eos-32) {  /* 32 gives enough room for a the first keyword. */
-    /* We're at the start of a line. */
-    tor_assert(*s != '\n');
-
-    if (*s == '@' && !annotations) {
-      annotations = s;
-    } else if (*s == 'r' && !strcmpstart(s, "router ")) {
-      *s_ptr = annotations ? annotations : s;
-      *is_extrainfo_out = 0;
-      return 0;
-    } else if (*s == 'e' && !strcmpstart(s, "extra-info ")) {
-      *s_ptr = annotations ? annotations : s;
-      *is_extrainfo_out = 1;
-      return 0;
-    }
-
-    if (!(s = memchr(s+1, '\n', eos-(s+1))))
-      break;
-    s = eat_whitespace_eos(s, eos);
-  }
-  return -1;
-}
-
-/** Given a string *<b>s</b> containing a concatenated sequence of router
- * descriptors (or extra-info documents if <b>is_extrainfo</b> is set), parses
- * them and stores the result in <b>dest</b>.  All routers are marked running
- * and valid.  Advances *s to a point immediately following the last router
- * entry.  Ignore any trailing router entries that are not complete.
- *
- * If <b>saved_location</b> isn't SAVED_IN_CACHE, make a local copy of each
- * descriptor in the signed_descriptor_body field of each routerinfo_t.  If it
- * isn't SAVED_NOWHERE, remember the offset of each descriptor.
- *
- * Returns 0 on success and -1 on failure.  Adds a digest to
- * <b>invalid_digests_out</b> for every entry that was unparseable or
- * invalid. (This may cause duplicate entries.)
- */
-int
-router_parse_list_from_string(const char **s, const char *eos,
-                              smartlist_t *dest,
-                              saved_location_t saved_location,
-                              int want_extrainfo,
-                              int allow_annotations,
-                              const char *prepend_annotations,
-                              smartlist_t *invalid_digests_out)
-{
-  routerinfo_t *router;
-  extrainfo_t *extrainfo;
-  signed_descriptor_t *signed_desc = NULL;
-  void *elt;
-  const char *end, *start;
-  int have_extrainfo;
-
-  tor_assert(s);
-  tor_assert(*s);
-  tor_assert(dest);
-
-  start = *s;
-  if (!eos)
-    eos = *s + strlen(*s);
-
-  tor_assert(eos >= *s);
-
-  while (1) {
-    char raw_digest[DIGEST_LEN];
-    int have_raw_digest = 0;
-    int dl_again = 0;
-    if (find_start_of_next_router_or_extrainfo(s, eos, &have_extrainfo) < 0)
-      break;
-
-    end = tor_memstr(*s, eos-*s, "\nrouter-signature");
-    if (end)
-      end = tor_memstr(end, eos-end, "\n-----END SIGNATURE-----\n");
-    if (end)
-      end += strlen("\n-----END SIGNATURE-----\n");
-
-    if (!end)
-      break;
-
-    elt = NULL;
-
-    if (have_extrainfo && want_extrainfo) {
-      routerlist_t *rl = router_get_routerlist();
-      have_raw_digest = router_get_extrainfo_hash(*s, end-*s, raw_digest) == 0;
-      extrainfo = extrainfo_parse_entry_from_string(*s, end,
-                                       saved_location != SAVED_IN_CACHE,
-                                       rl->identity_map, &dl_again);
-      if (extrainfo) {
-        signed_desc = &extrainfo->cache_info;
-        elt = extrainfo;
-      }
-    } else if (!have_extrainfo && !want_extrainfo) {
-      have_raw_digest = router_get_router_hash(*s, end-*s, raw_digest) == 0;
-      router = router_parse_entry_from_string(*s, end,
-                                              saved_location != SAVED_IN_CACHE,
-                                              allow_annotations,
-                                              prepend_annotations, &dl_again);
-      if (router) {
-        log_debug(LD_DIR, "Read router '%s', purpose '%s'",
-                  router_describe(router),
-                  router_purpose_to_string(router->purpose));
-        signed_desc = &router->cache_info;
-        elt = router;
-      }
-    }
-    if (! elt && ! dl_again && have_raw_digest && invalid_digests_out) {
-      smartlist_add(invalid_digests_out, tor_memdup(raw_digest, DIGEST_LEN));
-    }
-    if (!elt) {
-      *s = end;
-      continue;
-    }
-    if (saved_location != SAVED_NOWHERE) {
-      tor_assert(signed_desc);
-      signed_desc->saved_location = saved_location;
-      signed_desc->saved_offset = *s - start;
-    }
-    *s = end;
-    smartlist_add(dest, elt);
-  }
-
-  return 0;
-}
-
-/* For debugging: define to count every descriptor digest we've seen so we
- * know if we need to try harder to avoid duplicate verifies. */
-#undef COUNT_DISTINCT_DIGESTS
-
-#ifdef COUNT_DISTINCT_DIGESTS
-static digestmap_t *verified_digests = NULL;
-#endif
-
-/** Log the total count of the number of distinct router digests we've ever
- * verified.  When compared to the number of times we've verified routerdesc
- * signatures <i>in toto</i>, this will tell us if we're doing too much
- * multiple-verification. */
-void
-dump_distinct_digest_count(int severity)
-{
-#ifdef COUNT_DISTINCT_DIGESTS
-  if (!verified_digests)
-    verified_digests = digestmap_new();
-  tor_log(severity, LD_GENERAL, "%d *distinct* router digests verified",
-      digestmap_size(verified_digests));
-#else
-  (void)severity; /* suppress "unused parameter" warning */
-#endif
-}
-
-/** Try to find an IPv6 OR port in <b>list</b> of directory_token_t's
- * with at least one argument (use GE(1) in setup). If found, store
- * address and port number to <b>addr_out</b> and
- * <b>port_out</b>. Return number of OR ports found. */
-static int
-find_single_ipv6_orport(const smartlist_t *list,
-                        tor_addr_t *addr_out,
-                        uint16_t *port_out)
-{
-  int ret = 0;
-  tor_assert(list != NULL);
-  tor_assert(addr_out != NULL);
-  tor_assert(port_out != NULL);
-
-  SMARTLIST_FOREACH_BEGIN(list, directory_token_t *, t) {
-    tor_addr_t a;
-    maskbits_t bits;
-    uint16_t port_min, port_max;
-    tor_assert(t->n_args >= 1);
-    /* XXXX Prop186 the full spec allows much more than this. */
-    if (tor_addr_parse_mask_ports(t->args[0], 0,
-                                  &a, &bits, &port_min,
-                                  &port_max) == AF_INET6 &&
-        bits == 128 &&
-        port_min == port_max) {
-      /* Okay, this is one we can understand. Use it and ignore
-         any potential more addresses in list. */
-      tor_addr_copy(addr_out, &a);
-      *port_out = port_min;
-      ret = 1;
-      break;
-    }
-  } SMARTLIST_FOREACH_END(t);
-
-  return ret;
-}
-
-/** Helper function: reads a single router entry from *<b>s</b> ...
- * *<b>end</b>.  Mallocs a new router and returns it if all goes well, else
- * returns NULL.  If <b>cache_copy</b> is true, duplicate the contents of
- * s through end into the signed_descriptor_body of the resulting
- * routerinfo_t.
- *
- * If <b>end</b> is NULL, <b>s</b> must be properly NUL-terminated.
- *
- * If <b>allow_annotations</b>, it's okay to encounter annotations in <b>s</b>
- * before the router; if it's false, reject the router if it's annotated.  If
- * <b>prepend_annotations</b> is set, it should contain some annotations:
- * append them to the front of the router before parsing it, and keep them
- * around when caching the router.
- *
- * Only one of allow_annotations and prepend_annotations may be set.
- *
- * If <b>can_dl_again_out</b> is provided, set *<b>can_dl_again_out</b> to 1
- * if it's okay to try to download a descriptor with this same digest again,
- * and 0 if it isn't.  (It might not be okay to download it again if part of
- * the part covered by the digest is invalid.)
- */
-routerinfo_t *
-router_parse_entry_from_string(const char *s, const char *end,
-                               int cache_copy, int allow_annotations,
-                               const char *prepend_annotations,
-                               int *can_dl_again_out)
-{
-  routerinfo_t *router = NULL;
-  char digest[128];
-  smartlist_t *tokens = NULL, *exit_policy_tokens = NULL;
-  directory_token_t *tok;
-  struct in_addr in;
-  const char *start_of_annotations, *cp, *s_dup = s;
-  size_t prepend_len = prepend_annotations ? strlen(prepend_annotations) : 0;
-  int ok = 1;
-  memarea_t *area = NULL;
-  tor_cert_t *ntor_cc_cert = NULL;
-  /* Do not set this to '1' until we have parsed everything that we intend to
-   * parse that's covered by the hash. */
-  int can_dl_again = 0;
-
-  tor_assert(!allow_annotations || !prepend_annotations);
-
-  if (!end) {
-    end = s + strlen(s);
-  }
-
-  /* point 'end' to a point immediately after the final newline. */
-  while (end > s+2 && *(end-1) == '\n' && *(end-2) == '\n')
-    --end;
-
-  area = memarea_new();
-  tokens = smartlist_new();
-  if (prepend_annotations) {
-    if (tokenize_string(area,prepend_annotations,NULL,tokens,
-                        routerdesc_token_table,TS_NOCHECK)) {
-      log_warn(LD_DIR, "Error tokenizing router descriptor (annotations).");
-      goto err;
-    }
-  }
-
-  start_of_annotations = s;
-  cp = tor_memstr(s, end-s, "\nrouter ");
-  if (!cp) {
-    if (end-s < 7 || strcmpstart(s, "router ")) {
-      log_warn(LD_DIR, "No router keyword found.");
-      goto err;
-    }
-  } else {
-    s = cp+1;
-  }
-
-  if (start_of_annotations != s) { /* We have annotations */
-    if (allow_annotations) {
-      if (tokenize_string(area,start_of_annotations,s,tokens,
-                          routerdesc_token_table,TS_NOCHECK)) {
-        log_warn(LD_DIR, "Error tokenizing router descriptor (annotations).");
-        goto err;
-      }
-    } else {
-      log_warn(LD_DIR, "Found unexpected annotations on router descriptor not "
-               "loaded from disk.  Dropping it.");
-      goto err;
-    }
-  }
-
-  if (router_get_router_hash(s, end - s, digest) < 0) {
-    log_warn(LD_DIR, "Couldn't compute router hash.");
-    goto err;
-  }
-  {
-    int flags = 0;
-    if (allow_annotations)
-      flags |= TS_ANNOTATIONS_OK;
-    if (prepend_annotations)
-      flags |= TS_ANNOTATIONS_OK|TS_NO_NEW_ANNOTATIONS;
-
-    if (tokenize_string(area,s,end,tokens,routerdesc_token_table, flags)) {
-      log_warn(LD_DIR, "Error tokenizing router descriptor.");
-      goto err;
-    }
-  }
-
-  if (smartlist_len(tokens) < 2) {
-    log_warn(LD_DIR, "Impossibly short router descriptor.");
-    goto err;
-  }
-
-  tok = find_by_keyword(tokens, K_ROUTER);
-  const int router_token_pos = smartlist_pos(tokens, tok);
-  tor_assert(tok->n_args >= 5);
-
-  router = tor_malloc_zero(sizeof(routerinfo_t));
-  router->cert_expiration_time = TIME_MAX;
-  router->cache_info.routerlist_index = -1;
-  router->cache_info.annotations_len = s-start_of_annotations + prepend_len;
-  router->cache_info.signed_descriptor_len = end-s;
-  if (cache_copy) {
-    size_t len = router->cache_info.signed_descriptor_len +
-                 router->cache_info.annotations_len;
-    char *signed_body =
-      router->cache_info.signed_descriptor_body = tor_malloc(len+1);
-    if (prepend_annotations) {
-      memcpy(signed_body, prepend_annotations, prepend_len);
-      signed_body += prepend_len;
-    }
-    /* This assertion will always succeed.
-     * len == signed_desc_len + annotations_len
-     *     == end-s + s-start_of_annotations + prepend_len
-     *     == end-start_of_annotations + prepend_len
-     * We already wrote prepend_len bytes into the buffer; now we're
-     * writing end-start_of_annotations -NM. */
-    tor_assert(signed_body+(end-start_of_annotations) ==
-               router->cache_info.signed_descriptor_body+len);
-    memcpy(signed_body, start_of_annotations, end-start_of_annotations);
-    router->cache_info.signed_descriptor_body[len] = '\0';
-    tor_assert(strlen(router->cache_info.signed_descriptor_body) == len);
-  }
-  memcpy(router->cache_info.signed_descriptor_digest, digest, DIGEST_LEN);
-
-  router->nickname = tor_strdup(tok->args[0]);
-  if (!is_legal_nickname(router->nickname)) {
-    log_warn(LD_DIR,"Router nickname is invalid");
-    goto err;
-  }
-  if (!tor_inet_aton(tok->args[1], &in)) {
-    log_warn(LD_DIR,"Router address is not an IP address.");
-    goto err;
-  }
-  router->addr = ntohl(in.s_addr);
-
-  router->or_port =
-    (uint16_t) tor_parse_long(tok->args[2],10,0,65535,&ok,NULL);
-  if (!ok) {
-    log_warn(LD_DIR,"Invalid OR port %s", escaped(tok->args[2]));
-    goto err;
-  }
-  router->dir_port =
-    (uint16_t) tor_parse_long(tok->args[4],10,0,65535,&ok,NULL);
-  if (!ok) {
-    log_warn(LD_DIR,"Invalid dir port %s", escaped(tok->args[4]));
-    goto err;
-  }
-
-  tok = find_by_keyword(tokens, K_BANDWIDTH);
-  tor_assert(tok->n_args >= 3);
-  router->bandwidthrate = (int)
-    tor_parse_long(tok->args[0],10,1,INT_MAX,&ok,NULL);
-
-  if (!ok) {
-    log_warn(LD_DIR, "bandwidthrate %s unreadable or 0. Failing.",
-             escaped(tok->args[0]));
-    goto err;
-  }
-  router->bandwidthburst =
-    (int) tor_parse_long(tok->args[1],10,0,INT_MAX,&ok,NULL);
-  if (!ok) {
-    log_warn(LD_DIR, "Invalid bandwidthburst %s", escaped(tok->args[1]));
-    goto err;
-  }
-  router->bandwidthcapacity = (int)
-    tor_parse_long(tok->args[2],10,0,INT_MAX,&ok,NULL);
-  if (!ok) {
-    log_warn(LD_DIR, "Invalid bandwidthcapacity %s", escaped(tok->args[1]));
-    goto err;
-  }
-
-  if ((tok = find_opt_by_keyword(tokens, A_PURPOSE))) {
-    tor_assert(tok->n_args);
-    router->purpose = router_purpose_from_string(tok->args[0]);
-    if (router->purpose == ROUTER_PURPOSE_UNKNOWN) {
-      goto err;
-    }
-  } else {
-    router->purpose = ROUTER_PURPOSE_GENERAL;
-  }
-  router->cache_info.send_unencrypted =
-    (router->purpose == ROUTER_PURPOSE_GENERAL) ? 1 : 0;
-
-  if ((tok = find_opt_by_keyword(tokens, K_UPTIME))) {
-    tor_assert(tok->n_args >= 1);
-    router->uptime = tor_parse_long(tok->args[0],10,0,LONG_MAX,&ok,NULL);
-    if (!ok) {
-      log_warn(LD_DIR, "Invalid uptime %s", escaped(tok->args[0]));
-      goto err;
-    }
-  }
-
-  if ((tok = find_opt_by_keyword(tokens, K_HIBERNATING))) {
-    tor_assert(tok->n_args >= 1);
-    router->is_hibernating
-      = (tor_parse_long(tok->args[0],10,0,LONG_MAX,NULL,NULL) != 0);
-  }
-
-  tok = find_by_keyword(tokens, K_PUBLISHED);
-  tor_assert(tok->n_args == 1);
-  if (parse_iso_time(tok->args[0], &router->cache_info.published_on) < 0)
-    goto err;
-
-  tok = find_by_keyword(tokens, K_ONION_KEY);
-  if (!crypto_pk_public_exponent_ok(tok->key)) {
-    log_warn(LD_DIR,
-             "Relay's onion key had invalid exponent.");
-    goto err;
-  }
-  router->onion_pkey = tok->key;
-  tok->key = NULL; /* Prevent free */
-
-  if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) {
-    curve25519_public_key_t k;
-    tor_assert(tok->n_args >= 1);
-    if (curve25519_public_from_base64(&k, tok->args[0]) < 0) {
-      log_warn(LD_DIR, "Bogus ntor-onion-key in routerinfo");
-      goto err;
-    }
-    router->onion_curve25519_pkey =
-      tor_memdup(&k, sizeof(curve25519_public_key_t));
-  }
-
-  tok = find_by_keyword(tokens, K_SIGNING_KEY);
-  router->identity_pkey = tok->key;
-  tok->key = NULL; /* Prevent free */
-  if (crypto_pk_get_digest(router->identity_pkey,
-                           router->cache_info.identity_digest)) {
-    log_warn(LD_DIR, "Couldn't calculate key digest"); goto err;
-  }
-
-  {
-    directory_token_t *ed_sig_tok, *ed_cert_tok, *cc_tap_tok, *cc_ntor_tok,
-      *master_key_tok;
-    ed_sig_tok = find_opt_by_keyword(tokens, K_ROUTER_SIG_ED25519);
-    ed_cert_tok = find_opt_by_keyword(tokens, K_IDENTITY_ED25519);
-    master_key_tok = find_opt_by_keyword(tokens, K_MASTER_KEY_ED25519);
-    cc_tap_tok = find_opt_by_keyword(tokens, K_ONION_KEY_CROSSCERT);
-    cc_ntor_tok = find_opt_by_keyword(tokens, K_NTOR_ONION_KEY_CROSSCERT);
-    int n_ed_toks = !!ed_sig_tok + !!ed_cert_tok +
-      !!cc_tap_tok + !!cc_ntor_tok;
-    if ((n_ed_toks != 0 && n_ed_toks != 4) ||
-        (n_ed_toks == 4 && !router->onion_curve25519_pkey)) {
-      log_warn(LD_DIR, "Router descriptor with only partial ed25519/"
-               "cross-certification support");
-      goto err;
-    }
-    if (master_key_tok && !ed_sig_tok) {
-      log_warn(LD_DIR, "Router descriptor has ed25519 master key but no "
-               "certificate");
-      goto err;
-    }
-    if (ed_sig_tok) {
-      tor_assert(ed_cert_tok && cc_tap_tok && cc_ntor_tok);
-      const int ed_cert_token_pos = smartlist_pos(tokens, ed_cert_tok);
-      if (ed_cert_token_pos == -1 || router_token_pos == -1 ||
-          (ed_cert_token_pos != router_token_pos + 1 &&
-           ed_cert_token_pos != router_token_pos - 1)) {
-        log_warn(LD_DIR, "Ed25519 certificate in wrong position");
-        goto err;
-      }
-      if (ed_sig_tok != smartlist_get(tokens, smartlist_len(tokens)-2)) {
-        log_warn(LD_DIR, "Ed25519 signature in wrong position");
-        goto err;
-      }
-      if (strcmp(ed_cert_tok->object_type, "ED25519 CERT")) {
-        log_warn(LD_DIR, "Wrong object type on identity-ed25519 in decriptor");
-        goto err;
-      }
-      if (strcmp(cc_ntor_tok->object_type, "ED25519 CERT")) {
-        log_warn(LD_DIR, "Wrong object type on ntor-onion-key-crosscert "
-                 "in decriptor");
-        goto err;
-      }
-      if (strcmp(cc_tap_tok->object_type, "CROSSCERT")) {
-        log_warn(LD_DIR, "Wrong object type on onion-key-crosscert "
-                 "in decriptor");
-        goto err;
-      }
-      if (strcmp(cc_ntor_tok->args[0], "0") &&
-          strcmp(cc_ntor_tok->args[0], "1")) {
-        log_warn(LD_DIR, "Bad sign bit on ntor-onion-key-crosscert");
-        goto err;
-      }
-      int ntor_cc_sign_bit = !strcmp(cc_ntor_tok->args[0], "1");
-
-      uint8_t d256[DIGEST256_LEN];
-      const char *signed_start, *signed_end;
-      tor_cert_t *cert = tor_cert_parse(
-                       (const uint8_t*)ed_cert_tok->object_body,
-                       ed_cert_tok->object_size);
-      if (! cert) {
-        log_warn(LD_DIR, "Couldn't parse ed25519 cert");
-        goto err;
-      }
-      /* makes sure it gets freed. */
-      router->cache_info.signing_key_cert = cert;
-
-      if (cert->cert_type != CERT_TYPE_ID_SIGNING ||
-          ! cert->signing_key_included) {
-        log_warn(LD_DIR, "Invalid form for ed25519 cert");
-        goto err;
-      }
-
-      if (master_key_tok) {
-        /* This token is optional, but if it's present, it must match
-         * the signature in the signing cert, or supplant it. */
-        tor_assert(master_key_tok->n_args >= 1);
-        ed25519_public_key_t pkey;
-        if (ed25519_public_from_base64(&pkey, master_key_tok->args[0])<0) {
-          log_warn(LD_DIR, "Can't parse ed25519 master key");
-          goto err;
-        }
-
-        if (fast_memneq(&cert->signing_key.pubkey,
-                        pkey.pubkey, ED25519_PUBKEY_LEN)) {
-          log_warn(LD_DIR, "Ed25519 master key does not match "
-                   "key in certificate");
-          goto err;
-        }
-      }
-      ntor_cc_cert = tor_cert_parse((const uint8_t*)cc_ntor_tok->object_body,
-                                    cc_ntor_tok->object_size);
-      if (!ntor_cc_cert) {
-        log_warn(LD_DIR, "Couldn't parse ntor-onion-key-crosscert cert");
-        goto err;
-      }
-      if (ntor_cc_cert->cert_type != CERT_TYPE_ONION_ID ||
-          ! ed25519_pubkey_eq(&ntor_cc_cert->signed_key, &cert->signing_key)) {
-        log_warn(LD_DIR, "Invalid contents for ntor-onion-key-crosscert cert");
-        goto err;
-      }
-
-      ed25519_public_key_t ntor_cc_pk;
-      if (ed25519_public_key_from_curve25519_public_key(&ntor_cc_pk,
-                                            router->onion_curve25519_pkey,
-                                            ntor_cc_sign_bit)<0) {
-        log_warn(LD_DIR, "Error converting onion key to ed25519");
-        goto err;
-      }
-
-      if (router_get_hash_impl_helper(s, end-s, "router ",
-                                      "\nrouter-sig-ed25519",
-                                      ' ', &signed_start, &signed_end) < 0) {
-        log_warn(LD_DIR, "Can't find ed25519-signed portion of descriptor");
-        goto err;
-      }
-      crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA256);
-      crypto_digest_add_bytes(d, ED_DESC_SIGNATURE_PREFIX,
-        strlen(ED_DESC_SIGNATURE_PREFIX));
-      crypto_digest_add_bytes(d, signed_start, signed_end-signed_start);
-      crypto_digest_get_digest(d, (char*)d256, sizeof(d256));
-      crypto_digest_free(d);
-
-      ed25519_checkable_t check[3];
-      int check_ok[3];
-      if (tor_cert_get_checkable_sig(&check[0], cert, NULL) < 0) {
-        log_err(LD_BUG, "Couldn't create 'checkable' for cert.");
-        goto err;
-      }
-      if (tor_cert_get_checkable_sig(&check[1],
-                                     ntor_cc_cert, &ntor_cc_pk) < 0) {
-        log_err(LD_BUG, "Couldn't create 'checkable' for ntor_cc_cert.");
-        goto err;
-      }
-
-      if (ed25519_signature_from_base64(&check[2].signature,
-                                        ed_sig_tok->args[0])<0) {
-        log_warn(LD_DIR, "Couldn't decode ed25519 signature");
-        goto err;
-      }
-      check[2].pubkey = &cert->signed_key;
-      check[2].msg = d256;
-      check[2].len = DIGEST256_LEN;
-
-      if (ed25519_checksig_batch(check_ok, check, 3) < 0) {
-        log_warn(LD_DIR, "Incorrect ed25519 signature(s)");
-        goto err;
-      }
-
-      if (check_tap_onion_key_crosscert(
-                      (const uint8_t*)cc_tap_tok->object_body,
-                      (int)cc_tap_tok->object_size,
-                      router->onion_pkey,
-                      &cert->signing_key,
-                      (const uint8_t*)router->cache_info.identity_digest)<0) {
-        log_warn(LD_DIR, "Incorrect TAP cross-verification");
-        goto err;
-      }
-
-      /* We check this before adding it to the routerlist. */
-      if (cert->valid_until < ntor_cc_cert->valid_until)
-        router->cert_expiration_time = cert->valid_until;
-      else
-        router->cert_expiration_time = ntor_cc_cert->valid_until;
-    }
-  }
-
-  if ((tok = find_opt_by_keyword(tokens, K_FINGERPRINT))) {
-    /* If there's a fingerprint line, it must match the identity digest. */
-    char d[DIGEST_LEN];
-    tor_assert(tok->n_args == 1);
-    tor_strstrip(tok->args[0], " ");
-    if (base16_decode(d, DIGEST_LEN,
-                      tok->args[0], strlen(tok->args[0])) != DIGEST_LEN) {
-      log_warn(LD_DIR, "Couldn't decode router fingerprint %s",
-               escaped(tok->args[0]));
-      goto err;
-    }
-    if (tor_memneq(d,router->cache_info.identity_digest, DIGEST_LEN)) {
-      log_warn(LD_DIR, "Fingerprint '%s' does not match identity digest.",
-               tok->args[0]);
-      goto err;
-    }
-  }
-
-  if ((tok = find_opt_by_keyword(tokens, K_PLATFORM))) {
-    router->platform = tor_strdup(tok->args[0]);
-  }
-
-  if ((tok = find_opt_by_keyword(tokens, K_PROTO))) {
-    router->protocol_list = tor_strdup(tok->args[0]);
-  }
-
-  if ((tok = find_opt_by_keyword(tokens, K_CONTACT))) {
-    router->contact_info = tor_strdup(tok->args[0]);
-  }
-
-  if (find_opt_by_keyword(tokens, K_REJECT6) ||
-      find_opt_by_keyword(tokens, K_ACCEPT6)) {
-    log_warn(LD_DIR, "Rejecting router with reject6/accept6 line: they crash "
-             "older Tors.");
-    goto err;
-  }
-  {
-    smartlist_t *or_addresses = find_all_by_keyword(tokens, K_OR_ADDRESS);
-    if (or_addresses) {
-      find_single_ipv6_orport(or_addresses, &router->ipv6_addr,
-                              &router->ipv6_orport);
-      smartlist_free(or_addresses);
-    }
-  }
-  exit_policy_tokens = find_all_exitpolicy(tokens);
-  if (!smartlist_len(exit_policy_tokens)) {
-    log_warn(LD_DIR, "No exit policy tokens in descriptor.");
-    goto err;
-  }
-  SMARTLIST_FOREACH(exit_policy_tokens, directory_token_t *, t,
-                    if (router_add_exit_policy(router,t)<0) {
-                      log_warn(LD_DIR,"Error in exit policy");
-                      goto err;
-                    });
-  policy_expand_private(&router->exit_policy);
-
-  if ((tok = find_opt_by_keyword(tokens, K_IPV6_POLICY)) && tok->n_args) {
-    router->ipv6_exit_policy = parse_short_policy(tok->args[0]);
-    if (! router->ipv6_exit_policy) {
-      log_warn(LD_DIR , "Error in ipv6-policy %s", escaped(tok->args[0]));
-      goto err;
-    }
-  }
-
-  if (policy_is_reject_star(router->exit_policy, AF_INET, 1) &&
-      (!router->ipv6_exit_policy ||
-       short_policy_is_reject_star(router->ipv6_exit_policy)))
-    router->policy_is_reject_star = 1;
-
-  if ((tok = find_opt_by_keyword(tokens, K_FAMILY)) && tok->n_args) {
-    int i;
-    router->declared_family = smartlist_new();
-    for (i=0;i<tok->n_args;++i) {
-      if (!is_legal_nickname_or_hexdigest(tok->args[i])) {
-        log_warn(LD_DIR, "Illegal nickname %s in family line",
-                 escaped(tok->args[i]));
-        goto err;
-      }
-      smartlist_add(router->declared_family, tor_strdup(tok->args[i]));
-    }
-  }
-
-  if (find_opt_by_keyword(tokens, K_CACHES_EXTRA_INFO))
-    router->caches_extra_info = 1;
-
-  if (find_opt_by_keyword(tokens, K_ALLOW_SINGLE_HOP_EXITS))
-    router->allow_single_hop_exits = 1;
-
-  if ((tok = find_opt_by_keyword(tokens, K_EXTRA_INFO_DIGEST))) {
-    tor_assert(tok->n_args >= 1);
-    if (strlen(tok->args[0]) == HEX_DIGEST_LEN) {
-      if (base16_decode(router->cache_info.extra_info_digest, DIGEST_LEN,
-                        tok->args[0], HEX_DIGEST_LEN) != DIGEST_LEN) {
-          log_warn(LD_DIR,"Invalid extra info digest");
-      }
-    } else {
-      log_warn(LD_DIR, "Invalid extra info digest %s", escaped(tok->args[0]));
-    }
-
-    if (tok->n_args >= 2) {
-      if (digest256_from_base64(router->cache_info.extra_info_digest256,
-                                tok->args[1]) < 0) {
-        log_warn(LD_DIR, "Invalid extra info digest256 %s",
-                 escaped(tok->args[1]));
-      }
-    }
-  }
-
-  if (find_opt_by_keyword(tokens, K_HIDDEN_SERVICE_DIR)) {
-    router->wants_to_be_hs_dir = 1;
-  }
-
-  /* This router accepts tunnelled directory requests via begindir if it has
-   * an open dirport or it included "tunnelled-dir-server". */
-  if (find_opt_by_keyword(tokens, K_DIR_TUNNELLED) || router->dir_port > 0) {
-    router->supports_tunnelled_dir_requests = 1;
-  }
-
-  tok = find_by_keyword(tokens, K_ROUTER_SIGNATURE);
-  note_crypto_pk_op(VERIFY_RTR);
-#ifdef COUNT_DISTINCT_DIGESTS
-  if (!verified_digests)
-    verified_digests = digestmap_new();
-  digestmap_set(verified_digests, signed_digest, (void*)(uintptr_t)1);
-#endif
-
-  if (!router->or_port) {
-    log_warn(LD_DIR,"or_port unreadable or 0. Failing.");
-    goto err;
-  }
-
-  /* We've checked everything that's covered by the hash. */
-  can_dl_again = 1;
-  if (check_signature_token(digest, DIGEST_LEN, tok, router->identity_pkey, 0,
-                            "router descriptor") < 0)
-    goto err;
-
-  if (!router->platform) {
-    router->platform = tor_strdup("<unknown>");
-  }
-  goto done;
-
- err:
-  dump_desc(s_dup, "router descriptor");
-  routerinfo_free(router);
-  router = NULL;
- done:
-  tor_cert_free(ntor_cc_cert);
-  if (tokens) {
-    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
-    smartlist_free(tokens);
-  }
-  smartlist_free(exit_policy_tokens);
-  if (area) {
-    DUMP_AREA(area, "routerinfo");
-    memarea_drop_all(area);
-  }
-  if (can_dl_again_out)
-    *can_dl_again_out = can_dl_again;
-  return router;
-}
-
-/** Parse a single extrainfo entry from the string <b>s</b>, ending at
- * <b>end</b>.  (If <b>end</b> is NULL, parse up to the end of <b>s</b>.)  If
- * <b>cache_copy</b> is true, make a copy of the extra-info document in the
- * cache_info fields of the result.  If <b>routermap</b> is provided, use it
- * as a map from router identity to routerinfo_t when looking up signing keys.
- *
- * If <b>can_dl_again_out</b> is provided, set *<b>can_dl_again_out</b> to 1
- * if it's okay to try to download an extrainfo with this same digest again,
- * and 0 if it isn't.  (It might not be okay to download it again if part of
- * the part covered by the digest is invalid.)
- */
-extrainfo_t *
-extrainfo_parse_entry_from_string(const char *s, const char *end,
-                            int cache_copy, struct digest_ri_map_t *routermap,
-                            int *can_dl_again_out)
-{
-  extrainfo_t *extrainfo = NULL;
-  char digest[128];
-  smartlist_t *tokens = NULL;
-  directory_token_t *tok;
-  crypto_pk_t *key = NULL;
-  routerinfo_t *router = NULL;
-  memarea_t *area = NULL;
-  const char *s_dup = s;
-  /* Do not set this to '1' until we have parsed everything that we intend to
-   * parse that's covered by the hash. */
-  int can_dl_again = 0;
-
-  if (!end) {
-    end = s + strlen(s);
-  }
-
-  /* point 'end' to a point immediately after the final newline. */
-  while (end > s+2 && *(end-1) == '\n' && *(end-2) == '\n')
-    --end;
-
-  if (router_get_extrainfo_hash(s, end-s, digest) < 0) {
-    log_warn(LD_DIR, "Couldn't compute router hash.");
-    goto err;
-  }
-  tokens = smartlist_new();
-  area = memarea_new();
-  if (tokenize_string(area,s,end,tokens,extrainfo_token_table,0)) {
-    log_warn(LD_DIR, "Error tokenizing extra-info document.");
-    goto err;
-  }
-
-  if (smartlist_len(tokens) < 2) {
-    log_warn(LD_DIR, "Impossibly short extra-info document.");
-    goto err;
-  }
-
-  /* XXXX Accept this in position 1 too, and ed identity in position 0. */
-  tok = smartlist_get(tokens,0);
-  if (tok->tp != K_EXTRA_INFO) {
-    log_warn(LD_DIR,"Entry does not start with \"extra-info\"");
-    goto err;
-  }
-
-  extrainfo = tor_malloc_zero(sizeof(extrainfo_t));
-  extrainfo->cache_info.is_extrainfo = 1;
-  if (cache_copy)
-    extrainfo->cache_info.signed_descriptor_body = tor_memdup_nulterm(s,end-s);
-  extrainfo->cache_info.signed_descriptor_len = end-s;
-  memcpy(extrainfo->cache_info.signed_descriptor_digest, digest, DIGEST_LEN);
-  crypto_digest256((char*)extrainfo->digest256, s, end-s, DIGEST_SHA256);
-
-  tor_assert(tok->n_args >= 2);
-  if (!is_legal_nickname(tok->args[0])) {
-    log_warn(LD_DIR,"Bad nickname %s on \"extra-info\"",escaped(tok->args[0]));
-    goto err;
-  }
-  strlcpy(extrainfo->nickname, tok->args[0], sizeof(extrainfo->nickname));
-  if (strlen(tok->args[1]) != HEX_DIGEST_LEN ||
-      base16_decode(extrainfo->cache_info.identity_digest, DIGEST_LEN,
-                    tok->args[1], HEX_DIGEST_LEN) != DIGEST_LEN) {
-    log_warn(LD_DIR,"Invalid fingerprint %s on \"extra-info\"",
-             escaped(tok->args[1]));
-    goto err;
-  }
-
-  tok = find_by_keyword(tokens, K_PUBLISHED);
-  if (parse_iso_time(tok->args[0], &extrainfo->cache_info.published_on)) {
-    log_warn(LD_DIR,"Invalid published time %s on \"extra-info\"",
-             escaped(tok->args[0]));
-    goto err;
-  }
-
-  {
-    directory_token_t *ed_sig_tok, *ed_cert_tok;
-    ed_sig_tok = find_opt_by_keyword(tokens, K_ROUTER_SIG_ED25519);
-    ed_cert_tok = find_opt_by_keyword(tokens, K_IDENTITY_ED25519);
-    int n_ed_toks = !!ed_sig_tok + !!ed_cert_tok;
-    if (n_ed_toks != 0 && n_ed_toks != 2) {
-      log_warn(LD_DIR, "Router descriptor with only partial ed25519/"
-               "cross-certification support");
-      goto err;
-    }
-    if (ed_sig_tok) {
-      tor_assert(ed_cert_tok);
-      const int ed_cert_token_pos = smartlist_pos(tokens, ed_cert_tok);
-      if (ed_cert_token_pos != 1) {
-        /* Accept this in position 0 XXXX */
-        log_warn(LD_DIR, "Ed25519 certificate in wrong position");
-        goto err;
-      }
-      if (ed_sig_tok != smartlist_get(tokens, smartlist_len(tokens)-2)) {
-        log_warn(LD_DIR, "Ed25519 signature in wrong position");
-        goto err;
-      }
-      if (strcmp(ed_cert_tok->object_type, "ED25519 CERT")) {
-        log_warn(LD_DIR, "Wrong object type on identity-ed25519 in decriptor");
-        goto err;
-      }
-
-      uint8_t d256[DIGEST256_LEN];
-      const char *signed_start, *signed_end;
-      tor_cert_t *cert = tor_cert_parse(
-                       (const uint8_t*)ed_cert_tok->object_body,
-                       ed_cert_tok->object_size);
-      if (! cert) {
-        log_warn(LD_DIR, "Couldn't parse ed25519 cert");
-        goto err;
-      }
-      /* makes sure it gets freed. */
-      extrainfo->cache_info.signing_key_cert = cert;
-
-      if (cert->cert_type != CERT_TYPE_ID_SIGNING ||
-          ! cert->signing_key_included) {
-        log_warn(LD_DIR, "Invalid form for ed25519 cert");
-        goto err;
-      }
-
-      if (router_get_hash_impl_helper(s, end-s, "extra-info ",
-                                      "\nrouter-sig-ed25519",
-                                      ' ', &signed_start, &signed_end) < 0) {
-        log_warn(LD_DIR, "Can't find ed25519-signed portion of extrainfo");
-        goto err;
-      }
-      crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA256);
-      crypto_digest_add_bytes(d, ED_DESC_SIGNATURE_PREFIX,
-        strlen(ED_DESC_SIGNATURE_PREFIX));
-      crypto_digest_add_bytes(d, signed_start, signed_end-signed_start);
-      crypto_digest_get_digest(d, (char*)d256, sizeof(d256));
-      crypto_digest_free(d);
-
-      ed25519_checkable_t check[2];
-      int check_ok[2];
-      if (tor_cert_get_checkable_sig(&check[0], cert, NULL) < 0) {
-        log_err(LD_BUG, "Couldn't create 'checkable' for cert.");
-        goto err;
-      }
-
-      if (ed25519_signature_from_base64(&check[1].signature,
-                                        ed_sig_tok->args[0])<0) {
-        log_warn(LD_DIR, "Couldn't decode ed25519 signature");
-        goto err;
-      }
-      check[1].pubkey = &cert->signed_key;
-      check[1].msg = d256;
-      check[1].len = DIGEST256_LEN;
-
-      if (ed25519_checksig_batch(check_ok, check, 2) < 0) {
-        log_warn(LD_DIR, "Incorrect ed25519 signature(s)");
-        goto err;
-      }
-      /* We don't check the certificate expiration time: checking that it
-       * matches the cert in the router descriptor is adequate. */
-    }
-  }
-
-  /* We've checked everything that's covered by the hash. */
-  can_dl_again = 1;
-
-  if (routermap &&
-      (router = digestmap_get((digestmap_t*)routermap,
-                              extrainfo->cache_info.identity_digest))) {
-    key = router->identity_pkey;
-  }
-
-  tok = find_by_keyword(tokens, K_ROUTER_SIGNATURE);
-  if (strcmp(tok->object_type, "SIGNATURE") ||
-      tok->object_size < 128 || tok->object_size > 512) {
-    log_warn(LD_DIR, "Bad object type or length on extra-info signature");
-    goto err;
-  }
-
-  if (key) {
-    note_crypto_pk_op(VERIFY_RTR);
-    if (check_signature_token(digest, DIGEST_LEN, tok, key, 0,
-                              "extra-info") < 0)
-      goto err;
-
-    if (router)
-      extrainfo->cache_info.send_unencrypted =
-        router->cache_info.send_unencrypted;
-  } else {
-    extrainfo->pending_sig = tor_memdup(tok->object_body,
-                                        tok->object_size);
-    extrainfo->pending_sig_len = tok->object_size;
-  }
-
-  goto done;
- err:
-  dump_desc(s_dup, "extra-info descriptor");
-  extrainfo_free(extrainfo);
-  extrainfo = NULL;
- done:
-  if (tokens) {
-    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
-    smartlist_free(tokens);
-  }
-  if (area) {
-    DUMP_AREA(area, "extrainfo");
-    memarea_drop_all(area);
-  }
-  if (can_dl_again_out)
-    *can_dl_again_out = can_dl_again;
-  return extrainfo;
-}
-
-/** Parse a key certificate from <b>s</b>; point <b>end-of-string</b> to
- * the first character after the certificate. */
-authority_cert_t *
-authority_cert_parse_from_string(const char *s, const char **end_of_string)
-{
-  /** Reject any certificate at least this big; it is probably an overflow, an
-   * attack, a bug, or some other nonsense. */
-#define MAX_CERT_SIZE (128*1024)
-
-  authority_cert_t *cert = NULL, *old_cert;
-  smartlist_t *tokens = NULL;
-  char digest[DIGEST_LEN];
-  directory_token_t *tok;
-  char fp_declared[DIGEST_LEN];
-  char *eos;
-  size_t len;
-  int found;
-  memarea_t *area = NULL;
-  const char *s_dup = s;
-
-  s = eat_whitespace(s);
-  eos = strstr(s, "\ndir-key-certification");
-  if (! eos) {
-    log_warn(LD_DIR, "No signature found on key certificate");
-    return NULL;
-  }
-  eos = strstr(eos, "\n-----END SIGNATURE-----\n");
-  if (! eos) {
-    log_warn(LD_DIR, "No end-of-signature found on key certificate");
-    return NULL;
-  }
-  eos = strchr(eos+2, '\n');
-  tor_assert(eos);
-  ++eos;
-  len = eos - s;
-
-  if (len > MAX_CERT_SIZE) {
-    log_warn(LD_DIR, "Certificate is far too big (at %lu bytes long); "
-             "rejecting", (unsigned long)len);
-    return NULL;
-  }
-
-  tokens = smartlist_new();
-  area = memarea_new();
-  if (tokenize_string(area,s, eos, tokens, dir_key_certificate_table, 0) < 0) {
-    log_warn(LD_DIR, "Error tokenizing key certificate");
-    goto err;
-  }
-  if (router_get_hash_impl(s, strlen(s), digest, "dir-key-certificate-version",
-                           "\ndir-key-certification", '\n', DIGEST_SHA1) < 0)
-    goto err;
-  tok = smartlist_get(tokens, 0);
-  if (tok->tp != K_DIR_KEY_CERTIFICATE_VERSION || strcmp(tok->args[0], "3")) {
-    log_warn(LD_DIR,
-             "Key certificate does not begin with a recognized version (3).");
-    goto err;
-  }
-
-  cert = tor_malloc_zero(sizeof(authority_cert_t));
-  memcpy(cert->cache_info.signed_descriptor_digest, digest, DIGEST_LEN);
-
-  tok = find_by_keyword(tokens, K_DIR_SIGNING_KEY);
-  tor_assert(tok->key);
-  cert->signing_key = tok->key;
-  tok->key = NULL;
-  if (crypto_pk_get_digest(cert->signing_key, cert->signing_key_digest))
-    goto err;
-
-  tok = find_by_keyword(tokens, K_DIR_IDENTITY_KEY);
-  tor_assert(tok->key);
-  cert->identity_key = tok->key;
-  tok->key = NULL;
-
-  tok = find_by_keyword(tokens, K_FINGERPRINT);
-  tor_assert(tok->n_args);
-  if (base16_decode(fp_declared, DIGEST_LEN, tok->args[0],
-                    strlen(tok->args[0])) != DIGEST_LEN) {
-    log_warn(LD_DIR, "Couldn't decode key certificate fingerprint %s",
-             escaped(tok->args[0]));
-    goto err;
-  }
-
-  if (crypto_pk_get_digest(cert->identity_key,
-                           cert->cache_info.identity_digest))
-    goto err;
-
-  if (tor_memneq(cert->cache_info.identity_digest, fp_declared, DIGEST_LEN)) {
-    log_warn(LD_DIR, "Digest of certificate key didn't match declared "
-             "fingerprint");
-    goto err;
-  }
-
-  tok = find_opt_by_keyword(tokens, K_DIR_ADDRESS);
-  if (tok) {
-    struct in_addr in;
-    char *address = NULL;
-    tor_assert(tok->n_args);
-    /* XXX++ use some tor_addr parse function below instead. -RD */
-    if (tor_addr_port_split(LOG_WARN, tok->args[0], &address,
-                            &cert->dir_port) < 0 ||
-        tor_inet_aton(address, &in) == 0) {
-      log_warn(LD_DIR, "Couldn't parse dir-address in certificate");
-      tor_free(address);
-      goto err;
-    }
-    cert->addr = ntohl(in.s_addr);
-    tor_free(address);
-  }
-
-  tok = find_by_keyword(tokens, K_DIR_KEY_PUBLISHED);
-  if (parse_iso_time(tok->args[0], &cert->cache_info.published_on) < 0) {
-     goto err;
-  }
-  tok = find_by_keyword(tokens, K_DIR_KEY_EXPIRES);
-  if (parse_iso_time(tok->args[0], &cert->expires) < 0) {
-     goto err;
-  }
-
-  tok = smartlist_get(tokens, smartlist_len(tokens)-1);
-  if (tok->tp != K_DIR_KEY_CERTIFICATION) {
-    log_warn(LD_DIR, "Certificate didn't end with dir-key-certification.");
-    goto err;
-  }
-
-  /* If we already have this cert, don't bother checking the signature. */
-  old_cert = authority_cert_get_by_digests(
-                                     cert->cache_info.identity_digest,
-                                     cert->signing_key_digest);
-  found = 0;
-  if (old_cert) {
-    /* XXXX We could just compare signed_descriptor_digest, but that wouldn't
-     * buy us much. */
-    if (old_cert->cache_info.signed_descriptor_len == len &&
-        old_cert->cache_info.signed_descriptor_body &&
-        tor_memeq(s, old_cert->cache_info.signed_descriptor_body, len)) {
-      log_debug(LD_DIR, "We already checked the signature on this "
-                "certificate; no need to do so again.");
-      found = 1;
-    }
-  }
-  if (!found) {
-    if (check_signature_token(digest, DIGEST_LEN, tok, cert->identity_key, 0,
-                              "key certificate")) {
-      goto err;
-    }
-
-    tok = find_by_keyword(tokens, K_DIR_KEY_CROSSCERT);
-    if (check_signature_token(cert->cache_info.identity_digest,
-                              DIGEST_LEN,
-                              tok,
-                              cert->signing_key,
-                              CST_NO_CHECK_OBJTYPE,
-                              "key cross-certification")) {
-      goto err;
-    }
-  }
-
-  cert->cache_info.signed_descriptor_len = len;
-  cert->cache_info.signed_descriptor_body = tor_malloc(len+1);
-  memcpy(cert->cache_info.signed_descriptor_body, s, len);
-  cert->cache_info.signed_descriptor_body[len] = 0;
-  cert->cache_info.saved_location = SAVED_NOWHERE;
-
-  if (end_of_string) {
-    *end_of_string = eat_whitespace(eos);
-  }
-  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
-  smartlist_free(tokens);
-  if (area) {
-    DUMP_AREA(area, "authority cert");
-    memarea_drop_all(area);
-  }
-  return cert;
- err:
-  dump_desc(s_dup, "authority cert");
-  authority_cert_free(cert);
-  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
-  smartlist_free(tokens);
-  if (area) {
-    DUMP_AREA(area, "authority cert");
-    memarea_drop_all(area);
-  }
-  return NULL;
-}
-
-/** Helper: given a string <b>s</b>, return the start of the next router-status
- * object (starting with "r " at the start of a line).  If none is found,
- * return the start of the directory footer, or the next directory signature.
- * If none is found, return the end of the string. */
-static inline const char *
-find_start_of_next_routerstatus(const char *s)
-{
-  const char *eos, *footer, *sig;
-  if ((eos = strstr(s, "\nr ")))
-    ++eos;
-  else
-    eos = s + strlen(s);
-
-  footer = tor_memstr(s, eos-s, "\ndirectory-footer");
-  sig = tor_memstr(s, eos-s, "\ndirectory-signature");
-
-  if (footer && sig)
-    return MIN(footer, sig) + 1;
-  else if (footer)
-    return footer+1;
-  else if (sig)
-    return sig+1;
-  else
-    return eos;
-}
-
-/** Parse the GuardFraction string from a consensus or vote.
- *
- *  If <b>vote</b> or <b>vote_rs</b> are set the document getting
- *  parsed is a vote routerstatus. Otherwise it's a consensus. This is
- *  the same semantic as in routerstatus_parse_entry_from_string(). */
-STATIC int
-routerstatus_parse_guardfraction(const char *guardfraction_str,
-                                 networkstatus_t *vote,
-                                 vote_routerstatus_t *vote_rs,
-                                 routerstatus_t *rs)
-{
-  int ok;
-  const char *end_of_header = NULL;
-  int is_consensus = !vote_rs;
-  uint32_t guardfraction;
-
-  tor_assert(bool_eq(vote, vote_rs));
-
-  /* If this info comes from a consensus, but we should't apply
-     guardfraction, just exit. */
-  if (is_consensus && !should_apply_guardfraction(NULL)) {
-    return 0;
-  }
-
-  end_of_header = strchr(guardfraction_str, '=');
-  if (!end_of_header) {
-    return -1;
-  }
-
-  guardfraction = (uint32_t)tor_parse_ulong(end_of_header+1,
-                                            10, 0, 100, &ok, NULL);
-  if (!ok) {
-    log_warn(LD_DIR, "Invalid GuardFraction %s", escaped(guardfraction_str));
-    return -1;
-  }
-
-  log_debug(LD_GENERAL, "[*] Parsed %s guardfraction '%s' for '%s'.",
-            is_consensus ? "consensus" : "vote",
-            guardfraction_str, rs->nickname);
-
-  if (!is_consensus) { /* We are parsing a vote */
-    vote_rs->status.guardfraction_percentage = guardfraction;
-    vote_rs->status.has_guardfraction = 1;
-  } else {
-    /* We are parsing a consensus. Only apply guardfraction to guards. */
-    if (rs->is_possible_guard) {
-      rs->guardfraction_percentage = guardfraction;
-      rs->has_guardfraction = 1;
-    } else {
-      log_warn(LD_BUG, "Got GuardFraction for non-guard %s. "
-               "This is not supposed to happen. Not applying. ", rs->nickname);
-    }
-  }
-
-  return 0;
-}
-
-/** Given a string at *<b>s</b>, containing a routerstatus object, and an
- * empty smartlist at <b>tokens</b>, parse and return the first router status
- * object in the string, and advance *<b>s</b> to just after the end of the
- * router status.  Return NULL and advance *<b>s</b> on error.
- *
- * If <b>vote</b> and <b>vote_rs</b> are provided, don't allocate a fresh
- * routerstatus but use <b>vote_rs</b> instead.
- *
- * If <b>consensus_method</b> is nonzero, this routerstatus is part of a
- * consensus, and we should parse it according to the method used to
- * make that consensus.
- *
- * Parse according to the syntax used by the consensus flavor <b>flav</b>.
- **/
-STATIC routerstatus_t *
-routerstatus_parse_entry_from_string(memarea_t *area,
-                                     const char **s, smartlist_t *tokens,
-                                     networkstatus_t *vote,
-                                     vote_routerstatus_t *vote_rs,
-                                     int consensus_method,
-                                     consensus_flavor_t flav)
-{
-  const char *eos, *s_dup = *s;
-  routerstatus_t *rs = NULL;
-  directory_token_t *tok;
-  char timebuf[ISO_TIME_LEN+1];
-  struct in_addr in;
-  int offset = 0;
-  tor_assert(tokens);
-  tor_assert(bool_eq(vote, vote_rs));
-
-  if (!consensus_method)
-    flav = FLAV_NS;
-  tor_assert(flav == FLAV_NS || flav == FLAV_MICRODESC);
-
-  eos = find_start_of_next_routerstatus(*s);
-
-  if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) {
-    log_warn(LD_DIR, "Error tokenizing router status");
-    goto err;
-  }
-  if (smartlist_len(tokens) < 1) {
-    log_warn(LD_DIR, "Impossibly short router status");
-    goto err;
-  }
-  tok = find_by_keyword(tokens, K_R);
-  tor_assert(tok->n_args >= 7); /* guaranteed by GE(7) in K_R setup */
-  if (flav == FLAV_NS) {
-    if (tok->n_args < 8) {
-      log_warn(LD_DIR, "Too few arguments to r");
-      goto err;
-    }
-  } else if (flav == FLAV_MICRODESC) {
-    offset = -1; /* There is no identity digest */
-  }
-
-  if (vote_rs) {
-    rs = &vote_rs->status;
-  } else {
-    rs = tor_malloc_zero(sizeof(routerstatus_t));
-  }
-
-  if (!is_legal_nickname(tok->args[0])) {
-    log_warn(LD_DIR,
-             "Invalid nickname %s in router status; skipping.",
-             escaped(tok->args[0]));
-    goto err;
-  }
-  strlcpy(rs->nickname, tok->args[0], sizeof(rs->nickname));
-
-  if (digest_from_base64(rs->identity_digest, tok->args[1])) {
-    log_warn(LD_DIR, "Error decoding identity digest %s",
-             escaped(tok->args[1]));
-    goto err;
-  }
-
-  if (flav == FLAV_NS) {
-    if (digest_from_base64(rs->descriptor_digest, tok->args[2])) {
-      log_warn(LD_DIR, "Error decoding descriptor digest %s",
-               escaped(tok->args[2]));
-      goto err;
-    }
-  }
-
-  if (tor_snprintf(timebuf, sizeof(timebuf), "%s %s",
-                   tok->args[3+offset], tok->args[4+offset]) < 0 ||
-      parse_iso_time(timebuf, &rs->published_on)<0) {
-    log_warn(LD_DIR, "Error parsing time '%s %s' [%d %d]",
-             tok->args[3+offset], tok->args[4+offset],
-             offset, (int)flav);
-    goto err;
-  }
-
-  if (tor_inet_aton(tok->args[5+offset], &in) == 0) {
-    log_warn(LD_DIR, "Error parsing router address in network-status %s",
-             escaped(tok->args[5+offset]));
-    goto err;
-  }
-  rs->addr = ntohl(in.s_addr);
-
-  rs->or_port = (uint16_t) tor_parse_long(tok->args[6+offset],
-                                         10,0,65535,NULL,NULL);
-  rs->dir_port = (uint16_t) tor_parse_long(tok->args[7+offset],
-                                           10,0,65535,NULL,NULL);
-
-  {
-    smartlist_t *a_lines = find_all_by_keyword(tokens, K_A);
-    if (a_lines) {
-      find_single_ipv6_orport(a_lines, &rs->ipv6_addr, &rs->ipv6_orport);
-      smartlist_free(a_lines);
-    }
-  }
-
-  tok = find_opt_by_keyword(tokens, K_S);
-  if (tok && vote) {
-    int i;
-    vote_rs->flags = 0;
-    for (i=0; i < tok->n_args; ++i) {
-      int p = smartlist_string_pos(vote->known_flags, tok->args[i]);
-      if (p >= 0) {
-        vote_rs->flags |= (U64_LITERAL(1)<<p);
-      } else {
-        log_warn(LD_DIR, "Flags line had a flag %s not listed in known_flags.",
-                 escaped(tok->args[i]));
-        goto err;
-      }
-    }
-  } else if (tok) {
-    /* This is a consensus, not a vote. */
-    int i;
-    for (i=0; i < tok->n_args; ++i) {
-      if (!strcmp(tok->args[i], "Exit"))
-        rs->is_exit = 1;
-      else if (!strcmp(tok->args[i], "Stable"))
-        rs->is_stable = 1;
-      else if (!strcmp(tok->args[i], "Fast"))
-        rs->is_fast = 1;
-      else if (!strcmp(tok->args[i], "Running"))
-        rs->is_flagged_running = 1;
-      else if (!strcmp(tok->args[i], "Named"))
-        rs->is_named = 1;
-      else if (!strcmp(tok->args[i], "Valid"))
-        rs->is_valid = 1;
-      else if (!strcmp(tok->args[i], "Guard"))
-        rs->is_possible_guard = 1;
-      else if (!strcmp(tok->args[i], "BadExit"))
-        rs->is_bad_exit = 1;
-      else if (!strcmp(tok->args[i], "Authority"))
-        rs->is_authority = 1;
-      else if (!strcmp(tok->args[i], "Unnamed") &&
-               consensus_method >= 2) {
-        /* Unnamed is computed right by consensus method 2 and later. */
-        rs->is_unnamed = 1;
-      } else if (!strcmp(tok->args[i], "HSDir")) {
-        rs->is_hs_dir = 1;
-      } else if (!strcmp(tok->args[i], "V2Dir")) {
-        rs->is_v2_dir = 1;
-      }
-    }
-    /* These are implied true by having been included in a consensus made
-     * with a given method */
-    rs->is_flagged_running = 1; /* Starting with consensus method 4. */
-    if (consensus_method >= MIN_METHOD_FOR_EXCLUDING_INVALID_NODES)
-      rs->is_valid = 1;
-  }
-  int found_protocol_list = 0;
-  if ((tok = find_opt_by_keyword(tokens, K_PROTO))) {
-    found_protocol_list = 1;
-    rs->protocols_known = 1;
-    rs->supports_extend2_cells =
-      protocol_list_supports_protocol(tok->args[0], PRT_RELAY, 2);
-  }
-  if ((tok = find_opt_by_keyword(tokens, K_V))) {
-    tor_assert(tok->n_args == 1);
-    if (!strcmpstart(tok->args[0], "Tor ") && !found_protocol_list) {
-      /* We only do version checks like this in the case where
-       * the version is a "Tor" version, and where there is no
-       * list of protocol versions that we should be looking at instead. */
-      rs->supports_extend2_cells =
-        tor_version_as_new_as(tok->args[0], "0.2.4.8-alpha");
-      rs->protocols_known = 1;
-    }
-    if (vote_rs) {
-      vote_rs->version = tor_strdup(tok->args[0]);
-    }
-  }
-
-  /* handle weighting/bandwidth info */
-  if ((tok = find_opt_by_keyword(tokens, K_W))) {
-    int i;
-    for (i=0; i < tok->n_args; ++i) {
-      if (!strcmpstart(tok->args[i], "Bandwidth=")) {
-        int ok;
-        rs->bandwidth_kb =
-          (uint32_t)tor_parse_ulong(strchr(tok->args[i], '=')+1,
-                                    10, 0, UINT32_MAX,
-                                    &ok, NULL);
-        if (!ok) {
-          log_warn(LD_DIR, "Invalid Bandwidth %s", escaped(tok->args[i]));
-          goto err;
-        }
-        rs->has_bandwidth = 1;
-      } else if (!strcmpstart(tok->args[i], "Measured=") && vote_rs) {
-        int ok;
-        vote_rs->measured_bw_kb =
-            (uint32_t)tor_parse_ulong(strchr(tok->args[i], '=')+1,
-                                      10, 0, UINT32_MAX, &ok, NULL);
-        if (!ok) {
-          log_warn(LD_DIR, "Invalid Measured Bandwidth %s",
-                   escaped(tok->args[i]));
-          goto err;
-        }
-        vote_rs->has_measured_bw = 1;
-        vote->has_measured_bws = 1;
-      } else if (!strcmpstart(tok->args[i], "Unmeasured=1")) {
-        rs->bw_is_unmeasured = 1;
-      } else if (!strcmpstart(tok->args[i], "GuardFraction=")) {
-        if (routerstatus_parse_guardfraction(tok->args[i],
-                                             vote, vote_rs, rs) < 0) {
-          goto err;
-        }
-      }
-    }
-  }
-
-  /* parse exit policy summaries */
-  if ((tok = find_opt_by_keyword(tokens, K_P))) {
-    tor_assert(tok->n_args == 1);
-    if (strcmpstart(tok->args[0], "accept ") &&
-        strcmpstart(tok->args[0], "reject ")) {
-      log_warn(LD_DIR, "Unknown exit policy summary type %s.",
-               escaped(tok->args[0]));
-      goto err;
-    }
-    /* XXX weasel: parse this into ports and represent them somehow smart,
-     * maybe not here but somewhere on if we need it for the client.
-     * we should still parse it here to check it's valid tho.
-     */
-    rs->exitsummary = tor_strdup(tok->args[0]);
-    rs->has_exitsummary = 1;
-  }
-
-  if (vote_rs) {
-    SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, t) {
-      if (t->tp == K_M && t->n_args) {
-        vote_microdesc_hash_t *line =
-          tor_malloc(sizeof(vote_microdesc_hash_t));
-        line->next = vote_rs->microdesc;
-        line->microdesc_hash_line = tor_strdup(t->args[0]);
-        vote_rs->microdesc = line;
-      }
-      if (t->tp == K_ID) {
-        tor_assert(t->n_args >= 2);
-        if (!strcmp(t->args[0], "ed25519")) {
-          vote_rs->has_ed25519_listing = 1;
-          if (strcmp(t->args[1], "none") &&
-              digest256_from_base64((char*)vote_rs->ed25519_id,
-                                    t->args[1])<0) {
-            log_warn(LD_DIR, "Bogus ed25519 key in networkstatus vote");
-            goto err;
-          }
-        }
-      }
-      if (t->tp == K_PROTO) {
-        tor_assert(t->n_args == 1);
-        vote_rs->protocols = tor_strdup(t->args[0]);
-      }
-    } SMARTLIST_FOREACH_END(t);
-  } else if (flav == FLAV_MICRODESC) {
-    tok = find_opt_by_keyword(tokens, K_M);
-    if (tok) {
-      tor_assert(tok->n_args);
-      if (digest256_from_base64(rs->descriptor_digest, tok->args[0])) {
-        log_warn(LD_DIR, "Error decoding microdescriptor digest %s",
-                 escaped(tok->args[0]));
-        goto err;
-      }
-    } else {
-      log_info(LD_BUG, "Found an entry in networkstatus with no "
-               "microdescriptor digest. (Router %s ($%s) at %s:%d.)",
-               rs->nickname, hex_str(rs->identity_digest, DIGEST_LEN),
-               fmt_addr32(rs->addr), rs->or_port);
-    }
-  }
-
-  if (!strcasecmp(rs->nickname, UNNAMED_ROUTER_NICKNAME))
-    rs->is_named = 0;
-
-  goto done;
- err:
-  dump_desc(s_dup, "routerstatus entry");
-  if (rs && !vote_rs)
-    routerstatus_free(rs);
-  rs = NULL;
- done:
-  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
-  smartlist_clear(tokens);
-  if (area) {
-    DUMP_AREA(area, "routerstatus entry");
-    memarea_clear(area);
-  }
-  *s = eos;
-
-  return rs;
-}
-
-int
-compare_vote_routerstatus_entries(const void **_a, const void **_b)
-{
-  const vote_routerstatus_t *a = *_a, *b = *_b;
-  return fast_memcmp(a->status.identity_digest, b->status.identity_digest,
-                     DIGEST_LEN);
-}
-
-/** Verify the bandwidth weights of a network status document */
-int
-networkstatus_verify_bw_weights(networkstatus_t *ns, int consensus_method)
-{
-  int64_t weight_scale;
-  int64_t G=0, M=0, E=0, D=0, T=0;
-  double Wgg, Wgm, Wgd, Wmg, Wmm, Wme, Wmd, Weg, Wem, Wee, Wed;
-  double Gtotal=0, Mtotal=0, Etotal=0;
-  const char *casename = NULL;
-  int valid = 1;
-  (void) consensus_method;
-
-  weight_scale = networkstatus_get_weight_scale_param(ns);
-  Wgg = networkstatus_get_bw_weight(ns, "Wgg", -1);
-  Wgm = networkstatus_get_bw_weight(ns, "Wgm", -1);
-  Wgd = networkstatus_get_bw_weight(ns, "Wgd", -1);
-  Wmg = networkstatus_get_bw_weight(ns, "Wmg", -1);
-  Wmm = networkstatus_get_bw_weight(ns, "Wmm", -1);
-  Wme = networkstatus_get_bw_weight(ns, "Wme", -1);
-  Wmd = networkstatus_get_bw_weight(ns, "Wmd", -1);
-  Weg = networkstatus_get_bw_weight(ns, "Weg", -1);
-  Wem = networkstatus_get_bw_weight(ns, "Wem", -1);
-  Wee = networkstatus_get_bw_weight(ns, "Wee", -1);
-  Wed = networkstatus_get_bw_weight(ns, "Wed", -1);
-
-  if (Wgg<0 || Wgm<0 || Wgd<0 || Wmg<0 || Wmm<0 || Wme<0 || Wmd<0 || Weg<0
-          || Wem<0 || Wee<0 || Wed<0) {
-    log_warn(LD_BUG, "No bandwidth weights produced in consensus!");
-    return 0;
-  }
-
-  // First, sanity check basic summing properties that hold for all cases
-  // We use > 1 as the check for these because they are computed as integers.
-  // Sometimes there are rounding errors.
-  if (fabs(Wmm - weight_scale) > 1) {
-    log_warn(LD_BUG, "Wmm=%f != "I64_FORMAT,
-             Wmm, I64_PRINTF_ARG(weight_scale));
-    valid = 0;
-  }
-
-  if (fabs(Wem - Wee) > 1) {
-    log_warn(LD_BUG, "Wem=%f != Wee=%f", Wem, Wee);
-    valid = 0;
-  }
-
-  if (fabs(Wgm - Wgg) > 1) {
-    log_warn(LD_BUG, "Wgm=%f != Wgg=%f", Wgm, Wgg);
-    valid = 0;
-  }
-
-  if (fabs(Weg - Wed) > 1) {
-    log_warn(LD_BUG, "Wed=%f != Weg=%f", Wed, Weg);
-    valid = 0;
-  }
-
-  if (fabs(Wgg + Wmg - weight_scale) > 0.001*weight_scale) {
-    log_warn(LD_BUG, "Wgg=%f != "I64_FORMAT" - Wmg=%f", Wgg,
-             I64_PRINTF_ARG(weight_scale), Wmg);
-    valid = 0;
-  }
-
-  if (fabs(Wee + Wme - weight_scale) > 0.001*weight_scale) {
-    log_warn(LD_BUG, "Wee=%f != "I64_FORMAT" - Wme=%f", Wee,
-             I64_PRINTF_ARG(weight_scale), Wme);
-    valid = 0;
-  }
-
-  if (fabs(Wgd + Wmd + Wed - weight_scale) > 0.001*weight_scale) {
-    log_warn(LD_BUG, "Wgd=%f + Wmd=%f + Wed=%f != "I64_FORMAT,
-             Wgd, Wmd, Wed, I64_PRINTF_ARG(weight_scale));
-    valid = 0;
-  }
-
-  Wgg /= weight_scale;
-  Wgm /= weight_scale;
-  Wgd /= weight_scale;
-
-  Wmg /= weight_scale;
-  Wmm /= weight_scale;
-  Wme /= weight_scale;
-  Wmd /= weight_scale;
-
-  Weg /= weight_scale;
-  Wem /= weight_scale;
-  Wee /= weight_scale;
-  Wed /= weight_scale;
-
-  // Then, gather G, M, E, D, T to determine case
-  SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
-    int is_exit = 0;
-    /* Bug #2203: Don't count bad exits as exits for balancing */
-    is_exit = rs->is_exit && !rs->is_bad_exit;
-    if (rs->has_bandwidth) {
-      T += rs->bandwidth_kb;
-      if (is_exit && rs->is_possible_guard) {
-        D += rs->bandwidth_kb;
-        Gtotal += Wgd*rs->bandwidth_kb;
-        Mtotal += Wmd*rs->bandwidth_kb;
-        Etotal += Wed*rs->bandwidth_kb;
-      } else if (is_exit) {
-        E += rs->bandwidth_kb;
-        Mtotal += Wme*rs->bandwidth_kb;
-        Etotal += Wee*rs->bandwidth_kb;
-      } else if (rs->is_possible_guard) {
-        G += rs->bandwidth_kb;
-        Gtotal += Wgg*rs->bandwidth_kb;
-        Mtotal += Wmg*rs->bandwidth_kb;
-      } else {
-        M += rs->bandwidth_kb;
-        Mtotal += Wmm*rs->bandwidth_kb;
-      }
-    } else {
-      log_warn(LD_BUG, "Missing consensus bandwidth for router %s",
-               routerstatus_describe(rs));
-    }
-  } SMARTLIST_FOREACH_END(rs);
-
-  // Finally, check equality conditions depending upon case 1, 2 or 3
-  // Full equality cases: 1, 3b
-  // Partial equality cases: 2b (E=G), 3a (M=E)
-  // Fully unknown: 2a
-  if (3*E >= T && 3*G >= T) {
-    // Case 1: Neither are scarce
-    casename = "Case 1";
-    if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) {
-      log_warn(LD_DIR,
-               "Bw Weight Failure for %s: Etotal %f != Mtotal %f. "
-               "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT
-               " T="I64_FORMAT". "
-               "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
-               casename, Etotal, Mtotal,
-               I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-               I64_PRINTF_ARG(D), I64_PRINTF_ARG(T),
-               Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
-      valid = 0;
-    }
-    if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) {
-      log_warn(LD_DIR,
-               "Bw Weight Failure for %s: Etotal %f != Gtotal %f. "
-               "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT
-               " T="I64_FORMAT". "
-               "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
-               casename, Etotal, Gtotal,
-               I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-               I64_PRINTF_ARG(D), I64_PRINTF_ARG(T),
-               Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
-      valid = 0;
-    }
-    if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) {
-      log_warn(LD_DIR,
-               "Bw Weight Failure for %s: Mtotal %f != Gtotal %f. "
-               "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT
-               " T="I64_FORMAT". "
-               "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
-               casename, Mtotal, Gtotal,
-               I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-               I64_PRINTF_ARG(D), I64_PRINTF_ARG(T),
-               Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
-      valid = 0;
-    }
-  } else if (3*E < T && 3*G < T) {
-    int64_t R = MIN(E, G);
-    int64_t S = MAX(E, G);
-    /*
-     * Case 2: Both Guards and Exits are scarce
-     * Balance D between E and G, depending upon
-     * D capacity and scarcity. Devote no extra
-     * bandwidth to middle nodes.
-     */
-    if (R+D < S) { // Subcase a
-      double Rtotal, Stotal;
-      if (E < G) {
-        Rtotal = Etotal;
-        Stotal = Gtotal;
-      } else {
-        Rtotal = Gtotal;
-        Stotal = Etotal;
-      }
-      casename = "Case 2a";
-      // Rtotal < Stotal
-      if (Rtotal > Stotal) {
-        log_warn(LD_DIR,
-                   "Bw Weight Failure for %s: Rtotal %f > Stotal %f. "
-                   "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT
-                   " T="I64_FORMAT". "
-                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
-                   casename, Rtotal, Stotal,
-                   I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-                   I64_PRINTF_ARG(D), I64_PRINTF_ARG(T),
-                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
-        valid = 0;
-      }
-      // Rtotal < T/3
-      if (3*Rtotal > T) {
-        log_warn(LD_DIR,
-                   "Bw Weight Failure for %s: 3*Rtotal %f > T "
-                   I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT
-                   " D="I64_FORMAT" T="I64_FORMAT". "
-                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
-                   casename, Rtotal*3, I64_PRINTF_ARG(T),
-                   I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-                   I64_PRINTF_ARG(D), I64_PRINTF_ARG(T),
-                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
-        valid = 0;
-      }
-      // Stotal < T/3
-      if (3*Stotal > T) {
-        log_warn(LD_DIR,
-                   "Bw Weight Failure for %s: 3*Stotal %f > T "
-                   I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT
-                   " D="I64_FORMAT" T="I64_FORMAT". "
-                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
-                   casename, Stotal*3, I64_PRINTF_ARG(T),
-                   I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-                   I64_PRINTF_ARG(D), I64_PRINTF_ARG(T),
-                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
-        valid = 0;
-      }
-      // Mtotal > T/3
-      if (3*Mtotal < T) {
-        log_warn(LD_DIR,
-                   "Bw Weight Failure for %s: 3*Mtotal %f < T "
-                   I64_FORMAT". "
-                   "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT
-                   " T="I64_FORMAT". "
-                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
-                   casename, Mtotal*3, I64_PRINTF_ARG(T),
-                   I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-                   I64_PRINTF_ARG(D), I64_PRINTF_ARG(T),
-                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
-        valid = 0;
-      }
-    } else { // Subcase b: R+D > S
-      casename = "Case 2b";
-
-      /* Check the rare-M redirect case. */
-      if (D != 0 && 3*M < T) {
-        casename = "Case 2b (balanced)";
-        if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) {
-          log_warn(LD_DIR,
-                   "Bw Weight Failure for %s: Etotal %f != Mtotal %f. "
-                   "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT
-                   " T="I64_FORMAT". "
-                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
-                   casename, Etotal, Mtotal,
-                   I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-                   I64_PRINTF_ARG(D), I64_PRINTF_ARG(T),
-                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
-          valid = 0;
-        }
-        if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) {
-          log_warn(LD_DIR,
-                   "Bw Weight Failure for %s: Etotal %f != Gtotal %f. "
-                   "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT
-                   " T="I64_FORMAT". "
-                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
-                   casename, Etotal, Gtotal,
-                   I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-                   I64_PRINTF_ARG(D), I64_PRINTF_ARG(T),
-                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
-          valid = 0;
-        }
-        if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) {
-          log_warn(LD_DIR,
-                   "Bw Weight Failure for %s: Mtotal %f != Gtotal %f. "
-                   "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT
-                   " T="I64_FORMAT". "
-                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
-                   casename, Mtotal, Gtotal,
-                   I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-                   I64_PRINTF_ARG(D), I64_PRINTF_ARG(T),
-                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
-          valid = 0;
-        }
-      } else {
-        if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) {
-          log_warn(LD_DIR,
-                   "Bw Weight Failure for %s: Etotal %f != Gtotal %f. "
-                   "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT
-                   " T="I64_FORMAT". "
-                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
-                   casename, Etotal, Gtotal,
-                   I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-                   I64_PRINTF_ARG(D), I64_PRINTF_ARG(T),
-                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
-          valid = 0;
-        }
-      }
-    }
-  } else { // if (E < T/3 || G < T/3) {
-    int64_t S = MIN(E, G);
-    int64_t NS = MAX(E, G);
-    if (3*(S+D) < T) { // Subcase a:
-      double Stotal;
-      double NStotal;
-      if (G < E) {
-        casename = "Case 3a (G scarce)";
-        Stotal = Gtotal;
-        NStotal = Etotal;
-      } else { // if (G >= E) {
-        casename = "Case 3a (E scarce)";
-        NStotal = Gtotal;
-        Stotal = Etotal;
-      }
-      // Stotal < T/3
-      if (3*Stotal > T) {
-        log_warn(LD_DIR,
-                   "Bw Weight Failure for %s: 3*Stotal %f > T "
-                   I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT
-                   " D="I64_FORMAT" T="I64_FORMAT". "
-                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
-                   casename, Stotal*3, I64_PRINTF_ARG(T),
-                   I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-                   I64_PRINTF_ARG(D), I64_PRINTF_ARG(T),
-                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
-        valid = 0;
-      }
-      if (NS >= M) {
-        if (fabs(NStotal-Mtotal) > 0.01*MAX(NStotal,Mtotal)) {
-          log_warn(LD_DIR,
-                   "Bw Weight Failure for %s: NStotal %f != Mtotal %f. "
-                   "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT
-                   " T="I64_FORMAT". "
-                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
-                   casename, NStotal, Mtotal,
-                   I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-                   I64_PRINTF_ARG(D), I64_PRINTF_ARG(T),
-                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
-          valid = 0;
-        }
-      } else {
-        // if NS < M, NStotal > T/3 because only one of G or E is scarce
-        if (3*NStotal < T) {
-          log_warn(LD_DIR,
-                     "Bw Weight Failure for %s: 3*NStotal %f < T "
-                     I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT
-                     " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT". "
-                     "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
-                     casename, NStotal*3, I64_PRINTF_ARG(T),
-                     I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-                     I64_PRINTF_ARG(D), I64_PRINTF_ARG(T),
-                     Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
-          valid = 0;
-        }
-      }
-    } else { // Subcase b: S+D >= T/3
-      casename = "Case 3b";
-      if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) {
-        log_warn(LD_DIR,
-                 "Bw Weight Failure for %s: Etotal %f != Mtotal %f. "
-                 "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT
-                 " T="I64_FORMAT". "
-                 "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
-                 casename, Etotal, Mtotal,
-                 I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-                 I64_PRINTF_ARG(D), I64_PRINTF_ARG(T),
-                 Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
-        valid = 0;
-      }
-      if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) {
-        log_warn(LD_DIR,
-                 "Bw Weight Failure for %s: Etotal %f != Gtotal %f. "
-                 "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT
-                 " T="I64_FORMAT". "
-                 "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
-                 casename, Etotal, Gtotal,
-                 I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-                 I64_PRINTF_ARG(D), I64_PRINTF_ARG(T),
-                 Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
-        valid = 0;
-      }
-      if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) {
-        log_warn(LD_DIR,
-                 "Bw Weight Failure for %s: Mtotal %f != Gtotal %f. "
-                 "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT
-                 " T="I64_FORMAT". "
-                 "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
-                 casename, Mtotal, Gtotal,
-                 I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E),
-                 I64_PRINTF_ARG(D), I64_PRINTF_ARG(T),
-                 Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
-        valid = 0;
-      }
-    }
-  }
-
-  if (valid)
-    log_notice(LD_DIR, "Bandwidth-weight %s is verified and valid.",
-               casename);
-
-  return valid;
-}
-
-/** Parse and extract all SR commits from <b>tokens</b> and place them in
- *  <b>ns</b>. */
-static void
-extract_shared_random_commits(networkstatus_t *ns, smartlist_t *tokens)
-{
-  smartlist_t *chunks = NULL;
-
-  tor_assert(ns);
-  tor_assert(tokens);
-  /* Commits are only present in a vote. */
-  tor_assert(ns->type == NS_TYPE_VOTE);
-
-  ns->sr_info.commits = smartlist_new();
-
-  smartlist_t *commits = find_all_by_keyword(tokens, K_COMMIT);
-  /* It's normal that a vote might contain no commits even if it participates
-   * in the SR protocol. Don't treat it as an error. */
-  if (commits == NULL) {
-    goto end;
-  }
-
-  /* Parse the commit. We do NO validation of number of arguments or ordering
-   * for forward compatibility, it's the parse commit job to inform us if it's
-   * supported or not. */
-  chunks = smartlist_new();
-  SMARTLIST_FOREACH_BEGIN(commits, directory_token_t *, tok) {
-    /* Extract all arguments and put them in the chunks list. */
-    for (int i = 0; i < tok->n_args; i++) {
-      smartlist_add(chunks, tok->args[i]);
-    }
-    sr_commit_t *commit = sr_parse_commit(chunks);
-    smartlist_clear(chunks);
-    if (commit == NULL) {
-      /* Get voter identity so we can warn that this dirauth vote contains
-       * commit we can't parse. */
-      networkstatus_voter_info_t *voter = smartlist_get(ns->voters, 0);
-      tor_assert(voter);
-      log_warn(LD_DIR, "SR: Unable to parse commit %s from vote of voter %s.",
-               escaped(tok->object_body),
-               hex_str(voter->identity_digest,
-                       sizeof(voter->identity_digest)));
-      /* Commitment couldn't be parsed. Continue onto the next commit because
-       * this one could be unsupported for instance. */
-      continue;
-    }
-    /* Add newly created commit object to the vote. */
-    smartlist_add(ns->sr_info.commits, commit);
-  } SMARTLIST_FOREACH_END(tok);
-
- end:
-  smartlist_free(chunks);
-  smartlist_free(commits);
-}
-
-/** Check if a shared random value of type <b>srv_type</b> is in
- *  <b>tokens</b>. If there is, parse it and set it to <b>srv_out</b>. Return
- *  -1 on failure, 0 on success. The resulting srv is allocated on the heap and
- *  it's the responsibility of the caller to free it. */
-static int
-extract_one_srv(smartlist_t *tokens, directory_keyword srv_type,
-                sr_srv_t **srv_out)
-{
-  int ret = -1;
-  directory_token_t *tok;
-  sr_srv_t *srv = NULL;
-  smartlist_t *chunks;
-
-  tor_assert(tokens);
-
-  chunks = smartlist_new();
-  tok = find_opt_by_keyword(tokens, srv_type);
-  if (!tok) {
-    /* That's fine, no SRV is allowed. */
-    ret = 0;
-    goto end;
-  }
-  for (int i = 0; i < tok->n_args; i++) {
-    smartlist_add(chunks, tok->args[i]);
-  }
-  srv = sr_parse_srv(chunks);
-  if (srv == NULL) {
-    log_warn(LD_DIR, "SR: Unparseable SRV %s", escaped(tok->object_body));
-    goto end;
-  }
-  /* All is good. */
-  *srv_out = srv;
-  ret = 0;
- end:
-  smartlist_free(chunks);
-  return ret;
-}
-
-/** Extract any shared random values found in <b>tokens</b> and place them in
- *  the networkstatus <b>ns</b>. */
-static void
-extract_shared_random_srvs(networkstatus_t *ns, smartlist_t *tokens)
-{
-  const char *voter_identity;
-  networkstatus_voter_info_t *voter;
-
-  tor_assert(ns);
-  tor_assert(tokens);
-  /* Can be only one of them else code flow. */
-  tor_assert(ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_CONSENSUS);
-
-  if (ns->type == NS_TYPE_VOTE) {
-    voter = smartlist_get(ns->voters, 0);
-    tor_assert(voter);
-    voter_identity = hex_str(voter->identity_digest,
-                             sizeof(voter->identity_digest));
-  } else {
-    /* Consensus has multiple voters so no specific voter. */
-    voter_identity = "consensus";
-  }
-
-  /* We extract both and on error, everything is stopped because it means
-   * the votes is malformed for the shared random value(s). */
-  if (extract_one_srv(tokens, K_PREVIOUS_SRV, &ns->sr_info.previous_srv) < 0) {
-    log_warn(LD_DIR, "SR: Unable to parse previous SRV from %s",
-             voter_identity);
-    /* Maybe we have a chance with the current SRV so let's try it anyway. */
-  }
-  if (extract_one_srv(tokens, K_CURRENT_SRV, &ns->sr_info.current_srv) < 0) {
-    log_warn(LD_DIR, "SR: Unable to parse current SRV from %s",
-             voter_identity);
-  }
-}
-
-/** Parse a v3 networkstatus vote, opinion, or consensus (depending on
- * ns_type), from <b>s</b>, and return the result.  Return NULL on failure. */
-networkstatus_t *
-networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
-                                     networkstatus_type_t ns_type)
-{
-  smartlist_t *tokens = smartlist_new();
-  smartlist_t *rs_tokens = NULL, *footer_tokens = NULL;
-  networkstatus_voter_info_t *voter = NULL;
-  networkstatus_t *ns = NULL;
-  common_digests_t ns_digests;
-  const char *cert, *end_of_header, *end_of_footer, *s_dup = s;
-  directory_token_t *tok;
-  struct in_addr in;
-  int i, inorder, n_signatures = 0;
-  memarea_t *area = NULL, *rs_area = NULL;
-  consensus_flavor_t flav = FLAV_NS;
-  char *last_kwd=NULL;
-
-  tor_assert(s);
-
-  if (eos_out)
-    *eos_out = NULL;
-
-  if (router_get_networkstatus_v3_hashes(s, &ns_digests)) {
-    log_warn(LD_DIR, "Unable to compute digest of network-status");
-    goto err;
-  }
-
-  area = memarea_new();
-  end_of_header = find_start_of_next_routerstatus(s);
-  if (tokenize_string(area, s, end_of_header, tokens,
-                      (ns_type == NS_TYPE_CONSENSUS) ?
-                      networkstatus_consensus_token_table :
-                      networkstatus_token_table, 0)) {
-    log_warn(LD_DIR, "Error tokenizing network-status header");
-    goto err;
-  }
-
-  ns = tor_malloc_zero(sizeof(networkstatus_t));
-  memcpy(&ns->digests, &ns_digests, sizeof(ns_digests));
-
-  tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION);
-  tor_assert(tok);
-  if (tok->n_args > 1) {
-    int flavor = networkstatus_parse_flavor_name(tok->args[1]);
-    if (flavor < 0) {
-      log_warn(LD_DIR, "Can't parse document with unknown flavor %s",
-               escaped(tok->args[1]));
-      goto err;
-    }
-    ns->flavor = flav = flavor;
-  }
-  if (flav != FLAV_NS && ns_type != NS_TYPE_CONSENSUS) {
-    log_warn(LD_DIR, "Flavor found on non-consensus networkstatus.");
-    goto err;
-  }
-
-  if (ns_type != NS_TYPE_CONSENSUS) {
-    const char *end_of_cert = NULL;
-    if (!(cert = strstr(s, "\ndir-key-certificate-version")))
-      goto err;
-    ++cert;
-    ns->cert = authority_cert_parse_from_string(cert, &end_of_cert);
-    if (!ns->cert || !end_of_cert || end_of_cert > end_of_header)
-      goto err;
-  }
-
-  tok = find_by_keyword(tokens, K_VOTE_STATUS);
-  tor_assert(tok->n_args);
-  if (!strcmp(tok->args[0], "vote")) {
-    ns->type = NS_TYPE_VOTE;
-  } else if (!strcmp(tok->args[0], "consensus")) {
-    ns->type = NS_TYPE_CONSENSUS;
-  } else if (!strcmp(tok->args[0], "opinion")) {
-    ns->type = NS_TYPE_OPINION;
-  } else {
-    log_warn(LD_DIR, "Unrecognized vote status %s in network-status",
-             escaped(tok->args[0]));
-    goto err;
-  }
-  if (ns_type != ns->type) {
-    log_warn(LD_DIR, "Got the wrong kind of v3 networkstatus.");
-    goto err;
-  }
-
-  if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_OPINION) {
-    tok = find_by_keyword(tokens, K_PUBLISHED);
-    if (parse_iso_time(tok->args[0], &ns->published))
-      goto err;
-
-    ns->supported_methods = smartlist_new();
-    tok = find_opt_by_keyword(tokens, K_CONSENSUS_METHODS);
-    if (tok) {
-      for (i=0; i < tok->n_args; ++i)
-        smartlist_add(ns->supported_methods, tor_strdup(tok->args[i]));
-    } else {
-      smartlist_add(ns->supported_methods, tor_strdup("1"));
-    }
-  } else {
-    tok = find_opt_by_keyword(tokens, K_CONSENSUS_METHOD);
-    if (tok) {
-      int num_ok;
-      ns->consensus_method = (int)tor_parse_long(tok->args[0], 10, 1, INT_MAX,
-                                                 &num_ok, NULL);
-      if (!num_ok)
-        goto err;
-    } else {
-      ns->consensus_method = 1;
-    }
-  }
-
-  if ((tok = find_opt_by_keyword(tokens, K_RECOMMENDED_CLIENT_PROTOCOLS)))
-    ns->recommended_client_protocols = tor_strdup(tok->args[0]);
-  if ((tok = find_opt_by_keyword(tokens, K_RECOMMENDED_RELAY_PROTOCOLS)))
-    ns->recommended_relay_protocols = tor_strdup(tok->args[0]);
-  if ((tok = find_opt_by_keyword(tokens, K_REQUIRED_CLIENT_PROTOCOLS)))
-    ns->required_client_protocols = tor_strdup(tok->args[0]);
-  if ((tok = find_opt_by_keyword(tokens, K_REQUIRED_RELAY_PROTOCOLS)))
-    ns->required_relay_protocols = tor_strdup(tok->args[0]);
-
-  tok = find_by_keyword(tokens, K_VALID_AFTER);
-  if (parse_iso_time(tok->args[0], &ns->valid_after))
-    goto err;
-
-  tok = find_by_keyword(tokens, K_FRESH_UNTIL);
-  if (parse_iso_time(tok->args[0], &ns->fresh_until))
-    goto err;
-
-  tok = find_by_keyword(tokens, K_VALID_UNTIL);
-  if (parse_iso_time(tok->args[0], &ns->valid_until))
-    goto err;
-
-  tok = find_by_keyword(tokens, K_VOTING_DELAY);
-  tor_assert(tok->n_args >= 2);
-  {
-    int ok;
-    ns->vote_seconds =
-      (int) tor_parse_long(tok->args[0], 10, 0, INT_MAX, &ok, NULL);
-    if (!ok)
-      goto err;
-    ns->dist_seconds =
-      (int) tor_parse_long(tok->args[1], 10, 0, INT_MAX, &ok, NULL);
-    if (!ok)
-      goto err;
-  }
-  if (ns->valid_after +
-      (get_options()->TestingTorNetwork ?
-       MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL) > ns->fresh_until) {
-    log_warn(LD_DIR, "Vote/consensus freshness interval is too short");
-    goto err;
-  }
-  if (ns->valid_after +
-      (get_options()->TestingTorNetwork ?
-       MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL)*2 > ns->valid_until) {
-    log_warn(LD_DIR, "Vote/consensus liveness interval is too short");
-    goto err;
-  }
-  if (ns->vote_seconds < MIN_VOTE_SECONDS) {
-    log_warn(LD_DIR, "Vote seconds is too short");
-    goto err;
-  }
-  if (ns->dist_seconds < MIN_DIST_SECONDS) {
-    log_warn(LD_DIR, "Dist seconds is too short");
-    goto err;
-  }
-
-  if ((tok = find_opt_by_keyword(tokens, K_CLIENT_VERSIONS))) {
-    ns->client_versions = tor_strdup(tok->args[0]);
-  }
-  if ((tok = find_opt_by_keyword(tokens, K_SERVER_VERSIONS))) {
-    ns->server_versions = tor_strdup(tok->args[0]);
-  }
-
-  {
-    smartlist_t *package_lst = find_all_by_keyword(tokens, K_PACKAGE);
-    ns->package_lines = smartlist_new();
-    if (package_lst) {
-      SMARTLIST_FOREACH(package_lst, directory_token_t *, t,
-                    smartlist_add(ns->package_lines, tor_strdup(t->args[0])));
-    }
-    smartlist_free(package_lst);
-  }
-
-  tok = find_by_keyword(tokens, K_KNOWN_FLAGS);
-  ns->known_flags = smartlist_new();
-  inorder = 1;
-  for (i = 0; i < tok->n_args; ++i) {
-    smartlist_add(ns->known_flags, tor_strdup(tok->args[i]));
-    if (i>0 && strcmp(tok->args[i-1], tok->args[i])>= 0) {
-      log_warn(LD_DIR, "%s >= %s", tok->args[i-1], tok->args[i]);
-      inorder = 0;
-    }
-  }
-  if (!inorder) {
-    log_warn(LD_DIR, "known-flags not in order");
-    goto err;
-  }
-  if (ns->type != NS_TYPE_CONSENSUS &&
-      smartlist_len(ns->known_flags) > MAX_KNOWN_FLAGS_IN_VOTE) {
-    /* If we allowed more than 64 flags in votes, then parsing them would make
-     * us invoke undefined behavior whenever we used 1<<flagnum to do a
-     * bit-shift. This is only for votes and opinions: consensus users don't
-     * care about flags they don't recognize, and so don't build a bitfield
-     * for them. */
-    log_warn(LD_DIR, "Too many known-flags in consensus vote or opinion");
-    goto err;
-  }
-
-  tok = find_opt_by_keyword(tokens, K_PARAMS);
-  if (tok) {
-    int any_dups = 0;
-    inorder = 1;
-    ns->net_params = smartlist_new();
-    for (i = 0; i < tok->n_args; ++i) {
-      int ok=0;
-      char *eq = strchr(tok->args[i], '=');
-      size_t eq_pos;
-      if (!eq) {
-        log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i]));
-        goto err;
-      }
-      eq_pos = eq-tok->args[i];
-      tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL);
-      if (!ok) {
-        log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i]));
-        goto err;
-      }
-      if (i > 0 && strcmp(tok->args[i-1], tok->args[i]) >= 0) {
-        log_warn(LD_DIR, "%s >= %s", tok->args[i-1], tok->args[i]);
-        inorder = 0;
-      }
-      if (last_kwd && eq_pos == strlen(last_kwd) &&
-          fast_memeq(last_kwd, tok->args[i], eq_pos)) {
-        log_warn(LD_DIR, "Duplicate value for %s parameter",
-                 escaped(tok->args[i]));
-        any_dups = 1;
-      }
-      tor_free(last_kwd);
-      last_kwd = tor_strndup(tok->args[i], eq_pos);
-      smartlist_add(ns->net_params, tor_strdup(tok->args[i]));
-    }
-    if (!inorder) {
-      log_warn(LD_DIR, "params not in order");
-      goto err;
-    }
-    if (any_dups) {
-      log_warn(LD_DIR, "Duplicate in parameters");
-      goto err;
-    }
-  }
-
-  ns->voters = smartlist_new();
-
-  SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) {
-    tok = _tok;
-    if (tok->tp == K_DIR_SOURCE) {
-      tor_assert(tok->n_args >= 6);
-
-      if (voter)
-        smartlist_add(ns->voters, voter);
-      voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t));
-      voter->sigs = smartlist_new();
-      if (ns->type != NS_TYPE_CONSENSUS)
-        memcpy(voter->vote_digest, ns_digests.d[DIGEST_SHA1], DIGEST_LEN);
-
-      voter->nickname = tor_strdup(tok->args[0]);
-      if (strlen(tok->args[1]) != HEX_DIGEST_LEN ||
-          base16_decode(voter->identity_digest, sizeof(voter->identity_digest),
-                        tok->args[1], HEX_DIGEST_LEN)
-                        != sizeof(voter->identity_digest)) {
-        log_warn(LD_DIR, "Error decoding identity digest %s in "
-                 "network-status document.", escaped(tok->args[1]));
-        goto err;
-      }
-      if (ns->type != NS_TYPE_CONSENSUS &&
-          tor_memneq(ns->cert->cache_info.identity_digest,
-                 voter->identity_digest, DIGEST_LEN)) {
-        log_warn(LD_DIR,"Mismatch between identities in certificate and vote");
-        goto err;
-      }
-      if (ns->type != NS_TYPE_CONSENSUS) {
-        if (authority_cert_is_blacklisted(ns->cert)) {
-          log_warn(LD_DIR, "Rejecting vote signature made with blacklisted "
-                   "signing key %s",
-                   hex_str(ns->cert->signing_key_digest, DIGEST_LEN));
-          goto err;
-        }
-      }
-      voter->address = tor_strdup(tok->args[2]);
-      if (!tor_inet_aton(tok->args[3], &in)) {
-        log_warn(LD_DIR, "Error decoding IP address %s in network-status.",
-                 escaped(tok->args[3]));
-        goto err;
-      }
-      voter->addr = ntohl(in.s_addr);
-      int ok;
-      voter->dir_port = (uint16_t)
-        tor_parse_long(tok->args[4], 10, 0, 65535, &ok, NULL);
-      if (!ok)
-        goto err;
-      voter->or_port = (uint16_t)
-        tor_parse_long(tok->args[5], 10, 0, 65535, &ok, NULL);
-      if (!ok)
-        goto err;
-    } else if (tok->tp == K_CONTACT) {
-      if (!voter || voter->contact) {
-        log_warn(LD_DIR, "contact element is out of place.");
-        goto err;
-      }
-      voter->contact = tor_strdup(tok->args[0]);
-    } else if (tok->tp == K_VOTE_DIGEST) {
-      tor_assert(ns->type == NS_TYPE_CONSENSUS);
-      tor_assert(tok->n_args >= 1);
-      if (!voter || ! tor_digest_is_zero(voter->vote_digest)) {
-        log_warn(LD_DIR, "vote-digest element is out of place.");
-        goto err;
-      }
-      if (strlen(tok->args[0]) != HEX_DIGEST_LEN ||
-        base16_decode(voter->vote_digest, sizeof(voter->vote_digest),
-                      tok->args[0], HEX_DIGEST_LEN)
-                      != sizeof(voter->vote_digest)) {
-        log_warn(LD_DIR, "Error decoding vote digest %s in "
-                 "network-status consensus.", escaped(tok->args[0]));
-        goto err;
-      }
-    }
-  } SMARTLIST_FOREACH_END(_tok);
-  if (voter) {
-    smartlist_add(ns->voters, voter);
-    voter = NULL;
-  }
-  if (smartlist_len(ns->voters) == 0) {
-    log_warn(LD_DIR, "Missing dir-source elements in a networkstatus.");
-    goto err;
-  } else if (ns->type != NS_TYPE_CONSENSUS && smartlist_len(ns->voters) != 1) {
-    log_warn(LD_DIR, "Too many dir-source elements in a vote networkstatus.");
-    goto err;
-  }
-
-  if (ns->type != NS_TYPE_CONSENSUS &&
-      (tok = find_opt_by_keyword(tokens, K_LEGACY_DIR_KEY))) {
-    int bad = 1;
-    if (strlen(tok->args[0]) == HEX_DIGEST_LEN) {
-      networkstatus_voter_info_t *voter_0 = smartlist_get(ns->voters, 0);
-      if (base16_decode(voter_0->legacy_id_digest, DIGEST_LEN,
-                        tok->args[0], HEX_DIGEST_LEN) != DIGEST_LEN)
-        bad = 1;
-      else
-        bad = 0;
-    }
-    if (bad) {
-      log_warn(LD_DIR, "Invalid legacy key digest %s on vote.",
-               escaped(tok->args[0]));
-    }
-  }
-
-  /* If this is a vote document, check if information about the shared
-     randomness protocol is included, and extract it. */
-  if (ns->type == NS_TYPE_VOTE) {
-    /* Does this authority participates in the SR protocol? */
-    tok = find_opt_by_keyword(tokens, K_SR_FLAG);
-    if (tok) {
-      ns->sr_info.participate = 1;
-      /* Get the SR commitments and reveals from the vote. */
-      extract_shared_random_commits(ns, tokens);
-    }
-  }
-  /* For both a vote and consensus, extract the shared random values. */
-  if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_CONSENSUS) {
-    extract_shared_random_srvs(ns, tokens);
-  }
-
-  /* Parse routerstatus lines. */
-  rs_tokens = smartlist_new();
-  rs_area = memarea_new();
-  s = end_of_header;
-  ns->routerstatus_list = smartlist_new();
-
-  while (!strcmpstart(s, "r ")) {
-    if (ns->type != NS_TYPE_CONSENSUS) {
-      vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t));
-      if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns,
-                                               rs, 0, 0))
-        smartlist_add(ns->routerstatus_list, rs);
-      else {
-        tor_free(rs->version);
-        tor_free(rs);
-      }
-    } else {
-      routerstatus_t *rs;
-      if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens,
-                                                     NULL, NULL,
-                                                     ns->consensus_method,
-                                                     flav))) {
-        /* Use exponential-backoff scheduling when downloading microdescs */
-        rs->dl_status.backoff = DL_SCHED_RANDOM_EXPONENTIAL;
-        smartlist_add(ns->routerstatus_list, rs);
-      }
-    }
-  }
-  for (i = 1; i < smartlist_len(ns->routerstatus_list); ++i) {
-    routerstatus_t *rs1, *rs2;
-    if (ns->type != NS_TYPE_CONSENSUS) {
-      vote_routerstatus_t *a = smartlist_get(ns->routerstatus_list, i-1);
-      vote_routerstatus_t *b = smartlist_get(ns->routerstatus_list, i);
-      rs1 = &a->status; rs2 = &b->status;
-    } else {
-      rs1 = smartlist_get(ns->routerstatus_list, i-1);
-      rs2 = smartlist_get(ns->routerstatus_list, i);
-    }
-    if (fast_memcmp(rs1->identity_digest, rs2->identity_digest, DIGEST_LEN)
-        >= 0) {
-      log_warn(LD_DIR, "Networkstatus entries not sorted by identity digest");
-      goto err;
-    }
-  }
-  if (ns_type != NS_TYPE_CONSENSUS) {
-    digest256map_t *ed_id_map = digest256map_new();
-    SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, vote_routerstatus_t *,
-                            vrs) {
-      if (! vrs->has_ed25519_listing ||
-          tor_mem_is_zero((const char *)vrs->ed25519_id, DIGEST256_LEN))
-        continue;
-      if (digest256map_get(ed_id_map, vrs->ed25519_id) != NULL) {
-        log_warn(LD_DIR, "Vote networkstatus ed25519 identities were not "
-                 "unique");
-        digest256map_free(ed_id_map, NULL);
-        goto err;
-      }
-      digest256map_set(ed_id_map, vrs->ed25519_id, (void*)1);
-    } SMARTLIST_FOREACH_END(vrs);
-    digest256map_free(ed_id_map, NULL);
-  }
-
-  /* Parse footer; check signature. */
-  footer_tokens = smartlist_new();
-  if ((end_of_footer = strstr(s, "\nnetwork-status-version ")))
-    ++end_of_footer;
-  else
-    end_of_footer = s + strlen(s);
-  if (tokenize_string(area,s, end_of_footer, footer_tokens,
-                      networkstatus_vote_footer_token_table, 0)) {
-    log_warn(LD_DIR, "Error tokenizing network-status vote footer.");
-    goto err;
-  }
-
-  {
-    int found_sig = 0;
-    SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) {
-      tok = _tok;
-      if (tok->tp == K_DIRECTORY_SIGNATURE)
-        found_sig = 1;
-      else if (found_sig) {
-        log_warn(LD_DIR, "Extraneous token after first directory-signature");
-        goto err;
-      }
-    } SMARTLIST_FOREACH_END(_tok);
-  }
-
-  if ((tok = find_opt_by_keyword(footer_tokens, K_DIRECTORY_FOOTER))) {
-    if (tok != smartlist_get(footer_tokens, 0)) {
-      log_warn(LD_DIR, "Misplaced directory-footer token");
-      goto err;
-    }
-  }
-
-  tok = find_opt_by_keyword(footer_tokens, K_BW_WEIGHTS);
-  if (tok) {
-    ns->weight_params = smartlist_new();
-    for (i = 0; i < tok->n_args; ++i) {
-      int ok=0;
-      char *eq = strchr(tok->args[i], '=');
-      if (!eq) {
-        log_warn(LD_DIR, "Bad element '%s' in weight params",
-                 escaped(tok->args[i]));
-        goto err;
-      }
-      tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL);
-      if (!ok) {
-        log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i]));
-        goto err;
-      }
-      smartlist_add(ns->weight_params, tor_strdup(tok->args[i]));
-    }
-  }
-
-  SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) {
-    char declared_identity[DIGEST_LEN];
-    networkstatus_voter_info_t *v;
-    document_signature_t *sig;
-    const char *id_hexdigest = NULL;
-    const char *sk_hexdigest = NULL;
-    digest_algorithm_t alg = DIGEST_SHA1;
-    tok = _tok;
-    if (tok->tp != K_DIRECTORY_SIGNATURE)
-      continue;
-    tor_assert(tok->n_args >= 2);
-    if (tok->n_args == 2) {
-      id_hexdigest = tok->args[0];
-      sk_hexdigest = tok->args[1];
-    } else {
-      const char *algname = tok->args[0];
-      int a;
-      id_hexdigest = tok->args[1];
-      sk_hexdigest = tok->args[2];
-      a = crypto_digest_algorithm_parse_name(algname);
-      if (a<0) {
-        log_warn(LD_DIR, "Unknown digest algorithm %s; skipping",
-                 escaped(algname));
-        continue;
-      }
-      alg = a;
-    }
-
-    if (!tok->object_type ||
-        strcmp(tok->object_type, "SIGNATURE") ||
-        tok->object_size < 128 || tok->object_size > 512) {
-      log_warn(LD_DIR, "Bad object type or length on directory-signature");
-      goto err;
-    }
-
-    if (strlen(id_hexdigest) != HEX_DIGEST_LEN ||
-        base16_decode(declared_identity, sizeof(declared_identity),
-                      id_hexdigest, HEX_DIGEST_LEN)
-                      != sizeof(declared_identity)) {
-      log_warn(LD_DIR, "Error decoding declared identity %s in "
-               "network-status document.", escaped(id_hexdigest));
-      goto err;
-    }
-    if (!(v = networkstatus_get_voter_by_id(ns, declared_identity))) {
-      log_warn(LD_DIR, "ID on signature on network-status document does "
-               "not match any declared directory source.");
-      goto err;
-    }
-    sig = tor_malloc_zero(sizeof(document_signature_t));
-    memcpy(sig->identity_digest, v->identity_digest, DIGEST_LEN);
-    sig->alg = alg;
-    if (strlen(sk_hexdigest) != HEX_DIGEST_LEN ||
-        base16_decode(sig->signing_key_digest, sizeof(sig->signing_key_digest),
-                      sk_hexdigest, HEX_DIGEST_LEN)
-                      != sizeof(sig->signing_key_digest)) {
-      log_warn(LD_DIR, "Error decoding declared signing key digest %s in "
-               "network-status document.", escaped(sk_hexdigest));
-      tor_free(sig);
-      goto err;
-    }
-
-    if (ns->type != NS_TYPE_CONSENSUS) {
-      if (tor_memneq(declared_identity, ns->cert->cache_info.identity_digest,
-                 DIGEST_LEN)) {
-        log_warn(LD_DIR, "Digest mismatch between declared and actual on "
-                 "network-status vote.");
-        tor_free(sig);
-        goto err;
-      }
-    }
-
-    if (voter_get_sig_by_algorithm(v, sig->alg)) {
-      /* We already parsed a vote with this algorithm from this voter. Use the
-         first one. */
-      log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus "
-             "that contains two signatures from the same voter with the same "
-             "algorithm. Ignoring the second signature.");
-      tor_free(sig);
-      continue;
-    }
-
-    if (ns->type != NS_TYPE_CONSENSUS) {
-      if (check_signature_token(ns_digests.d[DIGEST_SHA1], DIGEST_LEN,
-                                tok, ns->cert->signing_key, 0,
-                                "network-status document")) {
-        tor_free(sig);
-        goto err;
-      }
-      sig->good_signature = 1;
-    } else {
-      if (tok->object_size >= INT_MAX || tok->object_size >= SIZE_T_CEILING) {
-        tor_free(sig);
-        goto err;
-      }
-      sig->signature = tor_memdup(tok->object_body, tok->object_size);
-      sig->signature_len = (int) tok->object_size;
-    }
-    smartlist_add(v->sigs, sig);
-
-    ++n_signatures;
-  } SMARTLIST_FOREACH_END(_tok);
-
-  if (! n_signatures) {
-    log_warn(LD_DIR, "No signatures on networkstatus document.");
-    goto err;
-  } else if (ns->type == NS_TYPE_VOTE && n_signatures != 1) {
-    log_warn(LD_DIR, "Received more than one signature on a "
-             "network-status vote.");
-    goto err;
-  }
-
-  if (eos_out)
-    *eos_out = end_of_footer;
-
-  goto done;
- err:
-  dump_desc(s_dup, "v3 networkstatus");
-  networkstatus_vote_free(ns);
-  ns = NULL;
- done:
-  if (tokens) {
-    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
-    smartlist_free(tokens);
-  }
-  if (voter) {
-    if (voter->sigs) {
-      SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig,
-                        document_signature_free(sig));
-      smartlist_free(voter->sigs);
-    }
-    tor_free(voter->nickname);
-    tor_free(voter->address);
-    tor_free(voter->contact);
-    tor_free(voter);
-  }
-  if (rs_tokens) {
-    SMARTLIST_FOREACH(rs_tokens, directory_token_t *, t, token_clear(t));
-    smartlist_free(rs_tokens);
-  }
-  if (footer_tokens) {
-    SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_clear(t));
-    smartlist_free(footer_tokens);
-  }
-  if (area) {
-    DUMP_AREA(area, "v3 networkstatus");
-    memarea_drop_all(area);
-  }
-  if (rs_area)
-    memarea_drop_all(rs_area);
-  tor_free(last_kwd);
-
-  return ns;
-}
-
-/** Return the common_digests_t that holds the digests of the
- * <b>flavor_name</b>-flavored networkstatus according to the detached
- * signatures document <b>sigs</b>, allocating a new common_digests_t as
- * neeeded. */
-static common_digests_t *
-detached_get_digests(ns_detached_signatures_t *sigs, const char *flavor_name)
-{
-  common_digests_t *d = strmap_get(sigs->digests, flavor_name);
-  if (!d) {
-    d = tor_malloc_zero(sizeof(common_digests_t));
-    strmap_set(sigs->digests, flavor_name, d);
-  }
-  return d;
-}
-
-/** Return the list of signatures of the <b>flavor_name</b>-flavored
- * networkstatus according to the detached signatures document <b>sigs</b>,
- * allocating a new common_digests_t as neeeded. */
-static smartlist_t *
-detached_get_signatures(ns_detached_signatures_t *sigs,
-                        const char *flavor_name)
-{
-  smartlist_t *sl = strmap_get(sigs->signatures, flavor_name);
-  if (!sl) {
-    sl = smartlist_new();
-    strmap_set(sigs->signatures, flavor_name, sl);
-  }
-  return sl;
-}
-
-/** Parse a detached v3 networkstatus signature document between <b>s</b> and
- * <b>eos</b> and return the result.  Return -1 on failure. */
-ns_detached_signatures_t *
-networkstatus_parse_detached_signatures(const char *s, const char *eos)
-{
-  /* XXXX there is too much duplicate shared between this function and
-   * networkstatus_parse_vote_from_string(). */
-  directory_token_t *tok;
-  memarea_t *area = NULL;
-  common_digests_t *digests;
-
-  smartlist_t *tokens = smartlist_new();
-  ns_detached_signatures_t *sigs =
-    tor_malloc_zero(sizeof(ns_detached_signatures_t));
-  sigs->digests = strmap_new();
-  sigs->signatures = strmap_new();
-
-  if (!eos)
-    eos = s + strlen(s);
-
-  area = memarea_new();
-  if (tokenize_string(area,s, eos, tokens,
-                      networkstatus_detached_signature_token_table, 0)) {
-    log_warn(LD_DIR, "Error tokenizing detached networkstatus signatures");
-    goto err;
-  }
-
-  /* Grab all the digest-like tokens. */
-  SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) {
-    const char *algname;
-    digest_algorithm_t alg;
-    const char *flavor;
-    const char *hexdigest;
-    size_t expected_length, digest_length;
-
-    tok = _tok;
-
-    if (tok->tp == K_CONSENSUS_DIGEST) {
-      algname = "sha1";
-      alg = DIGEST_SHA1;
-      flavor = "ns";
-      hexdigest = tok->args[0];
-    } else if (tok->tp == K_ADDITIONAL_DIGEST) {
-      int a = crypto_digest_algorithm_parse_name(tok->args[1]);
-      if (a<0) {
-        log_warn(LD_DIR, "Unrecognized algorithm name %s", tok->args[0]);
-        continue;
-      }
-      alg = (digest_algorithm_t) a;
-      flavor = tok->args[0];
-      algname = tok->args[1];
-      hexdigest = tok->args[2];
-    } else {
-      continue;
-    }
-
-    digest_length = crypto_digest_algorithm_get_length(alg);
-    expected_length = digest_length * 2; /* hex encoding */
-
-    if (strlen(hexdigest) != expected_length) {
-      log_warn(LD_DIR, "Wrong length on consensus-digest in detached "
-               "networkstatus signatures");
-      goto err;
-    }
-    digests = detached_get_digests(sigs, flavor);
-    tor_assert(digests);
-    if (!tor_mem_is_zero(digests->d[alg], digest_length)) {
-      log_warn(LD_DIR, "Multiple digests for %s with %s on detached "
-               "signatures document", flavor, algname);
-      continue;
-    }
-    if (base16_decode(digests->d[alg], digest_length,
-                      hexdigest, strlen(hexdigest)) != (int) digest_length) {
-      log_warn(LD_DIR, "Bad encoding on consensus-digest in detached "
-               "networkstatus signatures");
-      goto err;
-    }
-  } SMARTLIST_FOREACH_END(_tok);
-
-  tok = find_by_keyword(tokens, K_VALID_AFTER);
-  if (parse_iso_time(tok->args[0], &sigs->valid_after)) {
-    log_warn(LD_DIR, "Bad valid-after in detached networkstatus signatures");
-    goto err;
-  }
-
-  tok = find_by_keyword(tokens, K_FRESH_UNTIL);
-  if (parse_iso_time(tok->args[0], &sigs->fresh_until)) {
-    log_warn(LD_DIR, "Bad fresh-until in detached networkstatus signatures");
-    goto err;
-  }
-
-  tok = find_by_keyword(tokens, K_VALID_UNTIL);
-  if (parse_iso_time(tok->args[0], &sigs->valid_until)) {
-    log_warn(LD_DIR, "Bad valid-until in detached networkstatus signatures");
-    goto err;
-  }
-
-  SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) {
-    const char *id_hexdigest;
-    const char *sk_hexdigest;
-    const char *algname;
-    const char *flavor;
-    digest_algorithm_t alg;
-
-    char id_digest[DIGEST_LEN];
-    char sk_digest[DIGEST_LEN];
-    smartlist_t *siglist;
-    document_signature_t *sig;
-    int is_duplicate;
-
-    tok = _tok;
-    if (tok->tp == K_DIRECTORY_SIGNATURE) {
-      tor_assert(tok->n_args >= 2);
-      flavor = "ns";
-      algname = "sha1";
-      id_hexdigest = tok->args[0];
-      sk_hexdigest = tok->args[1];
-    } else if (tok->tp == K_ADDITIONAL_SIGNATURE) {
-      tor_assert(tok->n_args >= 4);
-      flavor = tok->args[0];
-      algname = tok->args[1];
-      id_hexdigest = tok->args[2];
-      sk_hexdigest = tok->args[3];
-    } else {
-      continue;
-    }
-
-    {
-      int a = crypto_digest_algorithm_parse_name(algname);
-      if (a<0) {
-        log_warn(LD_DIR, "Unrecognized algorithm name %s", algname);
-        continue;
-      }
-      alg = (digest_algorithm_t) a;
-    }
-
-    if (!tok->object_type ||
-        strcmp(tok->object_type, "SIGNATURE") ||
-        tok->object_size < 128 || tok->object_size > 512) {
-      log_warn(LD_DIR, "Bad object type or length on directory-signature");
-      goto err;
-    }
-
-    if (strlen(id_hexdigest) != HEX_DIGEST_LEN ||
-        base16_decode(id_digest, sizeof(id_digest),
-                      id_hexdigest, HEX_DIGEST_LEN) != sizeof(id_digest)) {
-      log_warn(LD_DIR, "Error decoding declared identity %s in "
-               "network-status vote.", escaped(id_hexdigest));
-      goto err;
-    }
-    if (strlen(sk_hexdigest) != HEX_DIGEST_LEN ||
-        base16_decode(sk_digest, sizeof(sk_digest),
-                      sk_hexdigest, HEX_DIGEST_LEN) != sizeof(sk_digest)) {
-      log_warn(LD_DIR, "Error decoding declared signing key digest %s in "
-               "network-status vote.", escaped(sk_hexdigest));
-      goto err;
-    }
-
-    siglist = detached_get_signatures(sigs, flavor);
-    is_duplicate = 0;
-    SMARTLIST_FOREACH(siglist, document_signature_t *, dsig, {
-      if (dsig->alg == alg &&
-          tor_memeq(id_digest, dsig->identity_digest, DIGEST_LEN) &&
-          tor_memeq(sk_digest, dsig->signing_key_digest, DIGEST_LEN)) {
-        is_duplicate = 1;
-      }
-    });
-    if (is_duplicate) {
-      log_warn(LD_DIR, "Two signatures with identical keys and algorithm "
-               "found.");
-      continue;
-    }
-
-    sig = tor_malloc_zero(sizeof(document_signature_t));
-    sig->alg = alg;
-    memcpy(sig->identity_digest, id_digest, DIGEST_LEN);
-    memcpy(sig->signing_key_digest, sk_digest, DIGEST_LEN);
-    if (tok->object_size >= INT_MAX || tok->object_size >= SIZE_T_CEILING) {
-      tor_free(sig);
-      goto err;
-    }
-    sig->signature = tor_memdup(tok->object_body, tok->object_size);
-    sig->signature_len = (int) tok->object_size;
-
-    smartlist_add(siglist, sig);
-  } SMARTLIST_FOREACH_END(_tok);
-
-  goto done;
- err:
-  ns_detached_signatures_free(sigs);
-  sigs = NULL;
- done:
-  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
-  smartlist_free(tokens);
-  if (area) {
-    DUMP_AREA(area, "detached signatures");
-    memarea_drop_all(area);
-  }
-  return sigs;
-}
-
-/** Parse the addr policy in the string <b>s</b> and return it.  If
- * assume_action is nonnegative, then insert its action (ADDR_POLICY_ACCEPT or
- * ADDR_POLICY_REJECT) for items that specify no action.
- *
- * Returns NULL on policy errors.
- *
- * Set *<b>malformed_list</b> to true if the entire policy list should be
- * discarded. Otherwise, set it to false, and only this item should be ignored
- * on error - the rest of the policy list can continue to be processed and
- * used.
- *
- * The addr_policy_t returned by this function can have its address set to
- * AF_UNSPEC for '*'.  Use policy_expand_unspec() to turn this into a pair
- * of AF_INET and AF_INET6 items.
- */
-MOCK_IMPL(addr_policy_t *,
-router_parse_addr_policy_item_from_string,(const char *s, int assume_action,
-                                           int *malformed_list))
-{
-  directory_token_t *tok = NULL;
-  const char *cp, *eos;
-  /* Longest possible policy is
-   * "accept6 [ffff:ffff:..255]/128:10000-65535",
-   * which contains a max-length IPv6 address, plus 26 characters.
-   * But note that there can be an arbitrary amount of space between the
-   * accept and the address:mask/port element.
-   * We don't need to multiply TOR_ADDR_BUF_LEN by 2, as there is only one
-   * IPv6 address. But making the buffer shorter might cause valid long lines,
-   * which parsed in previous versions, to fail to parse in new versions.
-   * (These lines would have to have excessive amounts of whitespace.) */
-  char line[TOR_ADDR_BUF_LEN*2 + 32];
-  addr_policy_t *r;
-  memarea_t *area = NULL;
-
-  tor_assert(malformed_list);
-  *malformed_list = 0;
-
-  s = eat_whitespace(s);
-  /* We can only do assume_action on []-quoted IPv6, as "a" (accept)
-   * and ":" (port separator) are ambiguous */
-  if ((*s == '*' || *s == '[' || TOR_ISDIGIT(*s)) && assume_action >= 0) {
-    if (tor_snprintf(line, sizeof(line), "%s %s",
-               assume_action == ADDR_POLICY_ACCEPT?"accept":"reject", s)<0) {
-      log_warn(LD_DIR, "Policy %s is too long.", escaped(s));
-      return NULL;
-    }
-    cp = line;
-    tor_strlower(line);
-  } else { /* assume an already well-formed address policy line */
-    cp = s;
-  }
-
-  eos = cp + strlen(cp);
-  area = memarea_new();
-  tok = get_next_token(area, &cp, eos, routerdesc_token_table);
-  if (tok->tp == ERR_) {
-    log_warn(LD_DIR, "Error reading address policy: %s", tok->error);
-    goto err;
-  }
-  if (tok->tp != K_ACCEPT && tok->tp != K_ACCEPT6 &&
-      tok->tp != K_REJECT && tok->tp != K_REJECT6) {
-    log_warn(LD_DIR, "Expected 'accept' or 'reject'.");
-    goto err;
-  }
-
-  /* Use the extended interpretation of accept/reject *,
-   * expanding it into an IPv4 wildcard and an IPv6 wildcard.
-   * Also permit *4 and *6 for IPv4 and IPv6 only wildcards. */
-  r = router_parse_addr_policy(tok, TAPMP_EXTENDED_STAR);
-  if (!r) {
-    goto err;
-  }
-
-  /* Ensure that accept6/reject6 fields are followed by IPv6 addresses.
-   * AF_UNSPEC addresses are only permitted on the accept/reject field type.
-   * Unlike descriptors, torrcs exit policy accept/reject can be followed by
-   * either an IPv4 or IPv6 address. */
-  if ((tok->tp == K_ACCEPT6 || tok->tp == K_REJECT6) &&
-       tor_addr_family(&r->addr) != AF_INET6) {
-    /* This is a non-fatal error, just ignore this one entry. */
-    *malformed_list = 0;
-    log_warn(LD_DIR, "IPv4 address '%s' with accept6/reject6 field type in "
-             "exit policy. Ignoring, but continuing to parse rules. (Use "
-             "accept/reject with IPv4 addresses.)",
-             tok->n_args == 1 ? tok->args[0] : "");
-    addr_policy_free(r);
-    r = NULL;
-    goto done;
-  }
-
-  goto done;
- err:
-  *malformed_list = 1;
-  r = NULL;
- done:
-  token_clear(tok);
-  if (area) {
-    DUMP_AREA(area, "policy item");
-    memarea_drop_all(area);
-  }
-  return r;
-}
-
-/** Add an exit policy stored in the token <b>tok</b> to the router info in
- * <b>router</b>.  Return 0 on success, -1 on failure. */
-static int
-router_add_exit_policy(routerinfo_t *router, directory_token_t *tok)
-{
-  addr_policy_t *newe;
-  /* Use the standard interpretation of accept/reject *, an IPv4 wildcard. */
-  newe = router_parse_addr_policy(tok, 0);
-  if (!newe)
-    return -1;
-  if (! router->exit_policy)
-    router->exit_policy = smartlist_new();
-
-  /* Ensure that in descriptors, accept/reject fields are followed by
-   * IPv4 addresses, and accept6/reject6 fields are followed by
-   * IPv6 addresses. Unlike torrcs, descriptor exit policies do not permit
-   * accept/reject followed by IPv6. */
-  if (((tok->tp == K_ACCEPT6 || tok->tp == K_REJECT6) &&
-       tor_addr_family(&newe->addr) == AF_INET)
-      ||
-      ((tok->tp == K_ACCEPT || tok->tp == K_REJECT) &&
-       tor_addr_family(&newe->addr) == AF_INET6)) {
-    /* There's nothing the user can do about other relays' descriptors,
-     * so we don't provide usage advice here. */
-    log_warn(LD_DIR, "Mismatch between field type and address type in exit "
-             "policy '%s'. Discarding entire router descriptor.",
-             tok->n_args == 1 ? tok->args[0] : "");
-    addr_policy_free(newe);
-    return -1;
-  }
-
-  smartlist_add(router->exit_policy, newe);
-
-  return 0;
-}
-
-/** Given a K_ACCEPT[6] or K_REJECT[6] token and a router, create and return
- * a new exit_policy_t corresponding to the token. If TAPMP_EXTENDED_STAR
- * is set in fmt_flags, K_ACCEPT6 and K_REJECT6 tokens followed by *
- * expand to IPv6-only policies, otherwise they expand to IPv4 and IPv6
- * policies */
-static addr_policy_t *
-router_parse_addr_policy(directory_token_t *tok, unsigned fmt_flags)
-{
-  addr_policy_t newe;
-  char *arg;
-
-  tor_assert(tok->tp == K_REJECT || tok->tp == K_REJECT6 ||
-             tok->tp == K_ACCEPT || tok->tp == K_ACCEPT6);
-
-  if (tok->n_args != 1)
-    return NULL;
-  arg = tok->args[0];
-
-  if (!strcmpstart(arg,"private"))
-    return router_parse_addr_policy_private(tok);
-
-  memset(&newe, 0, sizeof(newe));
-
-  if (tok->tp == K_REJECT || tok->tp == K_REJECT6)
-    newe.policy_type = ADDR_POLICY_REJECT;
-  else
-    newe.policy_type = ADDR_POLICY_ACCEPT;
-
-  /* accept6/reject6 * produces an IPv6 wildcard address only.
-   * (accept/reject * produces rules for IPv4 and IPv6 wildcard addresses.) */
-  if ((fmt_flags & TAPMP_EXTENDED_STAR)
-      && (tok->tp == K_ACCEPT6 || tok->tp == K_REJECT6)) {
-    fmt_flags |= TAPMP_STAR_IPV6_ONLY;
-  }
-
-  if (tor_addr_parse_mask_ports(arg, fmt_flags, &newe.addr, &newe.maskbits,
-                                &newe.prt_min, &newe.prt_max) < 0) {
-    log_warn(LD_DIR,"Couldn't parse line %s. Dropping", escaped(arg));
-    return NULL;
-  }
-
-  return addr_policy_get_canonical_entry(&newe);
-}
-
-/** Parse an exit policy line of the format "accept[6]/reject[6] private:...".
- * This didn't exist until Tor 0.1.1.15, so nobody should generate it in
- * router descriptors until earlier versions are obsolete.
- *
- * accept/reject and accept6/reject6 private all produce rules for both
- * IPv4 and IPv6 addresses.
- */
-static addr_policy_t *
-router_parse_addr_policy_private(directory_token_t *tok)
-{
-  const char *arg;
-  uint16_t port_min, port_max;
-  addr_policy_t result;
-
-  arg = tok->args[0];
-  if (strcmpstart(arg, "private"))
-    return NULL;
-
-  arg += strlen("private");
-  arg = (char*) eat_whitespace(arg);
-  if (!arg || *arg != ':')
-    return NULL;
-
-  if (parse_port_range(arg+1, &port_min, &port_max)<0)
-    return NULL;
-
-  memset(&result, 0, sizeof(result));
-  if (tok->tp == K_REJECT || tok->tp == K_REJECT6)
-    result.policy_type = ADDR_POLICY_REJECT;
-  else
-    result.policy_type = ADDR_POLICY_ACCEPT;
-  result.is_private = 1;
-  result.prt_min = port_min;
-  result.prt_max = port_max;
-
-  if (tok->tp == K_ACCEPT6 || tok->tp == K_REJECT6) {
-    log_warn(LD_GENERAL,
-             "'%s' expands into rules which apply to all private IPv4 and "
-             "IPv6 addresses. (Use accept/reject private:* for IPv4 and "
-             "IPv6.)", tok->n_args == 1 ? tok->args[0] : "");
-  }
-
-  return addr_policy_get_canonical_entry(&result);
-}
-
-/** Log and exit if <b>t</b> is malformed */
-void
-assert_addr_policy_ok(smartlist_t *lst)
-{
-  if (!lst) return;
-  SMARTLIST_FOREACH(lst, addr_policy_t *, t, {
-    tor_assert(t->policy_type == ADDR_POLICY_REJECT ||
-               t->policy_type == ADDR_POLICY_ACCEPT);
-    tor_assert(t->prt_min <= t->prt_max);
-  });
-}
-
-/*
- * Low-level tokenizer for router descriptors and directories.
- */
-
-/** Free all resources allocated for <b>tok</b> */
-static void
-token_clear(directory_token_t *tok)
-{
-  if (tok->key)
-    crypto_pk_free(tok->key);
-}
-
-#define ALLOC_ZERO(sz) memarea_alloc_zero(area,sz)
-#define ALLOC(sz) memarea_alloc(area,sz)
-#define STRDUP(str) memarea_strdup(area,str)
-#define STRNDUP(str,n) memarea_strndup(area,(str),(n))
-
-#define RET_ERR(msg)                                               \
-  STMT_BEGIN                                                       \
-    if (tok) token_clear(tok);                                      \
-    tok = ALLOC_ZERO(sizeof(directory_token_t));                   \
-    tok->tp = ERR_;                                                \
-    tok->error = STRDUP(msg);                                      \
-    goto done_tokenizing;                                          \
-  STMT_END
-
-/** Helper: make sure that the token <b>tok</b> with keyword <b>kwd</b> obeys
- * the object syntax of <b>o_syn</b>.  Allocate all storage in <b>area</b>.
- * Return <b>tok</b> on success, or a new ERR_ token if the token didn't
- * conform to the syntax we wanted.
- **/
-static inline directory_token_t *
-token_check_object(memarea_t *area, const char *kwd,
-                   directory_token_t *tok, obj_syntax o_syn)
-{
-  char ebuf[128];
-  switch (o_syn) {
-    case NO_OBJ:
-      /* No object is allowed for this token. */
-      if (tok->object_body) {
-        tor_snprintf(ebuf, sizeof(ebuf), "Unexpected object for %s", kwd);
-        RET_ERR(ebuf);
-      }
-      if (tok->key) {
-        tor_snprintf(ebuf, sizeof(ebuf), "Unexpected public key for %s", kwd);
-        RET_ERR(ebuf);
-      }
-      break;
-    case NEED_OBJ:
-      /* There must be a (non-key) object. */
-      if (!tok->object_body) {
-        tor_snprintf(ebuf, sizeof(ebuf), "Missing object for %s", kwd);
-        RET_ERR(ebuf);
-      }
-      break;
-    case NEED_KEY_1024: /* There must be a 1024-bit public key. */
-    case NEED_SKEY_1024: /* There must be a 1024-bit private key. */
-      if (tok->key && crypto_pk_num_bits(tok->key) != PK_BYTES*8) {
-        tor_snprintf(ebuf, sizeof(ebuf), "Wrong size on key for %s: %d bits",
-                     kwd, crypto_pk_num_bits(tok->key));
-        RET_ERR(ebuf);
-      }
-      /* fall through */
-    case NEED_KEY: /* There must be some kind of key. */
-      if (!tok->key) {
-        tor_snprintf(ebuf, sizeof(ebuf), "Missing public key for %s", kwd);
-        RET_ERR(ebuf);
-      }
-      if (o_syn != NEED_SKEY_1024) {
-        if (crypto_pk_key_is_private(tok->key)) {
-          tor_snprintf(ebuf, sizeof(ebuf),
-               "Private key given for %s, which wants a public key", kwd);
-          RET_ERR(ebuf);
-        }
-      } else { /* o_syn == NEED_SKEY_1024 */
-        if (!crypto_pk_key_is_private(tok->key)) {
-          tor_snprintf(ebuf, sizeof(ebuf),
-               "Public key given for %s, which wants a private key", kwd);
-          RET_ERR(ebuf);
-        }
-      }
-      break;
-    case OBJ_OK:
-      /* Anything goes with this token. */
-      break;
-  }
-
- done_tokenizing:
-  return tok;
-}
-
-/** Helper: parse space-separated arguments from the string <b>s</b> ending at
- * <b>eol</b>, and store them in the args field of <b>tok</b>.  Store the
- * number of parsed elements into the n_args field of <b>tok</b>.  Allocate
- * all storage in <b>area</b>.  Return the number of arguments parsed, or
- * return -1 if there was an insanely high number of arguments. */
-static inline int
-get_token_arguments(memarea_t *area, directory_token_t *tok,
-                    const char *s, const char *eol)
-{
-/** Largest number of arguments we'll accept to any token, ever. */
-#define MAX_ARGS 512
-  char *mem = memarea_strndup(area, s, eol-s);
-  char *cp = mem;
-  int j = 0;
-  char *args[MAX_ARGS];
-  while (*cp) {
-    if (j == MAX_ARGS)
-      return -1;
-    args[j++] = cp;
-    cp = (char*)find_whitespace(cp);
-    if (!cp || !*cp)
-      break; /* End of the line. */
-    *cp++ = '\0';
-    cp = (char*)eat_whitespace(cp);
-  }
-  tok->n_args = j;
-  tok->args = memarea_memdup(area, args, j*sizeof(char*));
-  return j;
-#undef MAX_ARGS
-}
-
-/** Helper function: read the next token from *s, advance *s to the end of the
- * token, and return the parsed token.  Parse *<b>s</b> according to the list
- * of tokens in <b>table</b>.
- */
-static directory_token_t *
-get_next_token(memarea_t *area,
-               const char **s, const char *eos, token_rule_t *table)
-{
-  /** Reject any object at least this big; it is probably an overflow, an
-   * attack, a bug, or some other nonsense. */
-#define MAX_UNPARSED_OBJECT_SIZE (128*1024)
-  /** Reject any line at least this big; it is probably an overflow, an
-   * attack, a bug, or some other nonsense. */
-#define MAX_LINE_LENGTH (128*1024)
-
-  const char *next, *eol, *obstart;
-  size_t obname_len;
-  int i;
-  directory_token_t *tok;
-  obj_syntax o_syn = NO_OBJ;
-  char ebuf[128];
-  const char *kwd = "";
-
-  tor_assert(area);
-  tok = ALLOC_ZERO(sizeof(directory_token_t));
-  tok->tp = ERR_;
-
-  /* Set *s to first token, eol to end-of-line, next to after first token */
-  *s = eat_whitespace_eos(*s, eos); /* eat multi-line whitespace */
-  tor_assert(eos >= *s);
-  eol = memchr(*s, '\n', eos-*s);
-  if (!eol)
-    eol = eos;
-  if (eol - *s > MAX_LINE_LENGTH) {
-    RET_ERR("Line far too long");
-  }
-
-  next = find_whitespace_eos(*s, eol);
-
-  if (!strcmp_len(*s, "opt", next-*s)) {
-    /* Skip past an "opt" at the start of the line. */
-    *s = eat_whitespace_eos_no_nl(next, eol);
-    next = find_whitespace_eos(*s, eol);
-  } else if (*s == eos) {  /* If no "opt", and end-of-line, line is invalid */
-    RET_ERR("Unexpected EOF");
-  }
-
-  /* Search the table for the appropriate entry.  (I tried a binary search
-   * instead, but it wasn't any faster.) */
-  for (i = 0; table[i].t ; ++i) {
-    if (!strcmp_len(*s, table[i].t, next-*s)) {
-      /* We've found the keyword. */
-      kwd = table[i].t;
-      tok->tp = table[i].v;
-      o_syn = table[i].os;
-      *s = eat_whitespace_eos_no_nl(next, eol);
-      /* We go ahead whether there are arguments or not, so that tok->args is
-       * always set if we want arguments. */
-      if (table[i].concat_args) {
-        /* The keyword takes the line as a single argument */
-        tok->args = ALLOC(sizeof(char*));
-        tok->args[0] = STRNDUP(*s,eol-*s); /* Grab everything on line */
-        tok->n_args = 1;
-      } else {
-        /* This keyword takes multiple arguments. */
-        if (get_token_arguments(area, tok, *s, eol)<0) {
-          tor_snprintf(ebuf, sizeof(ebuf),"Far too many arguments to %s", kwd);
-          RET_ERR(ebuf);
-        }
-        *s = eol;
-      }
-      if (tok->n_args < table[i].min_args) {
-        tor_snprintf(ebuf, sizeof(ebuf), "Too few arguments to %s", kwd);
-        RET_ERR(ebuf);
-      } else if (tok->n_args > table[i].max_args) {
-        tor_snprintf(ebuf, sizeof(ebuf), "Too many arguments to %s", kwd);
-        RET_ERR(ebuf);
-      }
-      break;
-    }
-  }
-
-  if (tok->tp == ERR_) {
-    /* No keyword matched; call it an "K_opt" or "A_unrecognized" */
-    if (*s < eol && **s == '@')
-      tok->tp = A_UNKNOWN_;
-    else
-      tok->tp = K_OPT;
-    tok->args = ALLOC(sizeof(char*));
-    tok->args[0] = STRNDUP(*s, eol-*s);
-    tok->n_args = 1;
-    o_syn = OBJ_OK;
-  }
-
-  /* Check whether there's an object present */
-  *s = eat_whitespace_eos(eol, eos);  /* Scan from end of first line */
-  tor_assert(eos >= *s);
-  eol = memchr(*s, '\n', eos-*s);
-  if (!eol || eol-*s<11 || strcmpstart(*s, "-----BEGIN ")) /* No object. */
-    goto check_object;
-
-  obstart = *s; /* Set obstart to start of object spec */
-  if (eol - *s <= 16 || memchr(*s+11,'\0',eol-*s-16) || /* no short lines, */
-      strcmp_len(eol-5, "-----", 5) ||           /* nuls or invalid endings */
-      (eol-*s) > MAX_UNPARSED_OBJECT_SIZE) {     /* name too long */
-    RET_ERR("Malformed object: bad begin line");
-  }
-  tok->object_type = STRNDUP(*s+11, eol-*s-16);
-  obname_len = eol-*s-16; /* store objname length here to avoid a strlen() */
-  *s = eol+1;    /* Set *s to possible start of object data (could be eos) */
-
-  /* Go to the end of the object */
-  next = tor_memstr(*s, eos-*s, "-----END ");
-  if (!next) {
-    RET_ERR("Malformed object: missing object end line");
-  }
-  tor_assert(eos >= next);
-  eol = memchr(next, '\n', eos-next);
-  if (!eol)  /* end-of-line marker, or eos if there's no '\n' */
-    eol = eos;
-  /* Validate the ending tag, which should be 9 + NAME + 5 + eol */
-  if ((size_t)(eol-next) != 9+obname_len+5 ||
-      strcmp_len(next+9, tok->object_type, obname_len) ||
-      strcmp_len(eol-5, "-----", 5)) {
-    tor_snprintf(ebuf, sizeof(ebuf), "Malformed object: mismatched end tag %s",
-             tok->object_type);
-    ebuf[sizeof(ebuf)-1] = '\0';
-    RET_ERR(ebuf);
-  }
-  if (next - *s > MAX_UNPARSED_OBJECT_SIZE)
-    RET_ERR("Couldn't parse object: missing footer or object much too big.");
-
-  if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) { /* If it's a public key */
-    tok->key = crypto_pk_new();
-    if (crypto_pk_read_public_key_from_string(tok->key, obstart, eol-obstart))
-      RET_ERR("Couldn't parse public key.");
-  } else if (!strcmp(tok->object_type, "RSA PRIVATE KEY")) { /* private key */
-    tok->key = crypto_pk_new();
-    if (crypto_pk_read_private_key_from_string(tok->key, obstart, eol-obstart))
-      RET_ERR("Couldn't parse private key.");
-  } else { /* If it's something else, try to base64-decode it */
-    int r;
-    tok->object_body = ALLOC(next-*s); /* really, this is too much RAM. */
-    r = base64_decode(tok->object_body, next-*s, *s, next-*s);
-    if (r<0)
-      RET_ERR("Malformed object: bad base64-encoded data");
-    tok->object_size = r;
-  }
-  *s = eol;
-
- check_object:
-  tok = token_check_object(area, kwd, tok, o_syn);
-
- done_tokenizing:
-  return tok;
-
-#undef RET_ERR
-#undef ALLOC
-#undef ALLOC_ZERO
-#undef STRDUP
-#undef STRNDUP
-}
-
-/** Read all tokens from a string between <b>start</b> and <b>end</b>, and add
- * them to <b>out</b>.  Parse according to the token rules in <b>table</b>.
- * Caller must free tokens in <b>out</b>.  If <b>end</b> is NULL, use the
- * entire string.
- */
-static int
-tokenize_string(memarea_t *area,
-                const char *start, const char *end, smartlist_t *out,
-                token_rule_t *table, int flags)
-{
-  const char **s;
-  directory_token_t *tok = NULL;
-  int counts[NIL_];
-  int i;
-  int first_nonannotation;
-  int prev_len = smartlist_len(out);
-  tor_assert(area);
-
-  s = &start;
-  if (!end) {
-    end = start+strlen(start);
-  } else {
-    /* it's only meaningful to check for nuls if we got an end-of-string ptr */
-    if (memchr(start, '\0', end-start)) {
-      log_warn(LD_DIR, "parse error: internal NUL character.");
-      return -1;
-    }
-  }
-  for (i = 0; i < NIL_; ++i)
-    counts[i] = 0;
-
-  SMARTLIST_FOREACH(out, const directory_token_t *, t, ++counts[t->tp]);
-
-  while (*s < end && (!tok || tok->tp != EOF_)) {
-    tok = get_next_token(area, s, end, table);
-    if (tok->tp == ERR_) {
-      log_warn(LD_DIR, "parse error: %s", tok->error);
-      token_clear(tok);
-      return -1;
-    }
-    ++counts[tok->tp];
-    smartlist_add(out, tok);
-    *s = eat_whitespace_eos(*s, end);
-  }
-
-  if (flags & TS_NOCHECK)
-    return 0;
-
-  if ((flags & TS_ANNOTATIONS_OK)) {
-    first_nonannotation = -1;
-    for (i = 0; i < smartlist_len(out); ++i) {
-      tok = smartlist_get(out, i);
-      if (tok->tp < MIN_ANNOTATION || tok->tp > MAX_ANNOTATION) {
-        first_nonannotation = i;
-        break;
-      }
-    }
-    if (first_nonannotation < 0) {
-      log_warn(LD_DIR, "parse error: item contains only annotations");
-      return -1;
-    }
-    for (i=first_nonannotation;  i < smartlist_len(out); ++i) {
-      tok = smartlist_get(out, i);
-      if (tok->tp >= MIN_ANNOTATION && tok->tp <= MAX_ANNOTATION) {
-        log_warn(LD_DIR, "parse error: Annotations mixed with keywords");
-        return -1;
-      }
-    }
-    if ((flags & TS_NO_NEW_ANNOTATIONS)) {
-      if (first_nonannotation != prev_len) {
-        log_warn(LD_DIR, "parse error: Unexpected annotations.");
-        return -1;
-      }
-    }
-  } else {
-    for (i=0;  i < smartlist_len(out); ++i) {
-      tok = smartlist_get(out, i);
-      if (tok->tp >= MIN_ANNOTATION && tok->tp <= MAX_ANNOTATION) {
-        log_warn(LD_DIR, "parse error: no annotations allowed.");
-        return -1;
-      }
-    }
-    first_nonannotation = 0;
-  }
-  for (i = 0; table[i].t; ++i) {
-    if (counts[table[i].v] < table[i].min_cnt) {
-      log_warn(LD_DIR, "Parse error: missing %s element.", table[i].t);
-      return -1;
-    }
-    if (counts[table[i].v] > table[i].max_cnt) {
-      log_warn(LD_DIR, "Parse error: too many %s elements.", table[i].t);
-      return -1;
-    }
-    if (table[i].pos & AT_START) {
-      if (smartlist_len(out) < 1 ||
-          (tok = smartlist_get(out, first_nonannotation))->tp != table[i].v) {
-        log_warn(LD_DIR, "Parse error: first item is not %s.", table[i].t);
-        return -1;
-      }
-    }
-    if (table[i].pos & AT_END) {
-      if (smartlist_len(out) < 1 ||
-          (tok = smartlist_get(out, smartlist_len(out)-1))->tp != table[i].v) {
-        log_warn(LD_DIR, "Parse error: last item is not %s.", table[i].t);
-        return -1;
-      }
-    }
-  }
-  return 0;
-}
-
-/** Find the first token in <b>s</b> whose keyword is <b>keyword</b>; return
- * NULL if no such keyword is found.
- */
-static directory_token_t *
-find_opt_by_keyword(smartlist_t *s, directory_keyword keyword)
-{
-  SMARTLIST_FOREACH(s, directory_token_t *, t, if (t->tp == keyword) return t);
-  return NULL;
-}
-
-/** Find the first token in <b>s</b> whose keyword is <b>keyword</b>; fail
- * with an assert if no such keyword is found.
- */
-static directory_token_t *
-find_by_keyword_(smartlist_t *s, directory_keyword keyword,
-                 const char *keyword_as_string)
-{
-  directory_token_t *tok = find_opt_by_keyword(s, keyword);
-  if (PREDICT_UNLIKELY(!tok)) {
-    log_err(LD_BUG, "Missing %s [%d] in directory object that should have "
-         "been validated. Internal error.", keyword_as_string, (int)keyword);
-    tor_assert(tok);
-  }
-  return tok;
-}
-
-/** If there are any directory_token_t entries in <b>s</b> whose keyword is
- * <b>k</b>, return a newly allocated smartlist_t containing all such entries,
- * in the same order in which they occur in <b>s</b>.  Otherwise return
- * NULL. */
-static smartlist_t *
-find_all_by_keyword(smartlist_t *s, directory_keyword k)
-{
-  smartlist_t *out = NULL;
-  SMARTLIST_FOREACH(s, directory_token_t *, t,
-                    if (t->tp == k) {
-                      if (!out)
-                        out = smartlist_new();
-                      smartlist_add(out, t);
-                    });
-  return out;
-}
-
-/** Return a newly allocated smartlist of all accept or reject tokens in
- * <b>s</b>.
- */
-static smartlist_t *
-find_all_exitpolicy(smartlist_t *s)
-{
-  smartlist_t *out = smartlist_new();
-  SMARTLIST_FOREACH(s, directory_token_t *, t,
-      if (t->tp == K_ACCEPT || t->tp == K_ACCEPT6 ||
-          t->tp == K_REJECT || t->tp == K_REJECT6)
-        smartlist_add(out,t));
-  return out;
-}
-
-/** Helper function for <b>router_get_hash_impl</b>: given <b>s</b>,
- * <b>s_len</b>, <b>start_str</b>, <b>end_str</b>, and <b>end_c</b> with the
- * same semantics as in that function, set *<b>start_out</b> (inclusive) and
- * *<b>end_out</b> (exclusive) to the boundaries of the string to be hashed.
- *
- * Return 0 on success and -1 on failure.
- */
-static int
-router_get_hash_impl_helper(const char *s, size_t s_len,
-                            const char *start_str,
-                            const char *end_str, char end_c,
-                            const char **start_out, const char **end_out)
-{
-  const char *start, *end;
-  start = tor_memstr(s, s_len, start_str);
-  if (!start) {
-    log_warn(LD_DIR,"couldn't find start of hashed material \"%s\"",start_str);
-    return -1;
-  }
-  if (start != s && *(start-1) != '\n') {
-    log_warn(LD_DIR,
-             "first occurrence of \"%s\" is not at the start of a line",
-             start_str);
-    return -1;
-  }
-  end = tor_memstr(start+strlen(start_str),
-                   s_len - (start-s) - strlen(start_str), end_str);
-  if (!end) {
-    log_warn(LD_DIR,"couldn't find end of hashed material \"%s\"",end_str);
-    return -1;
-  }
-  end = memchr(end+strlen(end_str), end_c, s_len - (end-s) - strlen(end_str));
-  if (!end) {
-    log_warn(LD_DIR,"couldn't find EOL");
-    return -1;
-  }
-  ++end;
-
-  *start_out = start;
-  *end_out = end;
-  return 0;
-}
-
-/** Compute the digest of the substring of <b>s</b> taken from the first
- * occurrence of <b>start_str</b> through the first instance of c after the
- * first subsequent occurrence of <b>end_str</b>; store the 20-byte or 32-byte
- * result in <b>digest</b>; return 0 on success.
- *
- * If no such substring exists, return -1.
- */
-static int
-router_get_hash_impl(const char *s, size_t s_len, char *digest,
-                     const char *start_str,
-                     const char *end_str, char end_c,
-                     digest_algorithm_t alg)
-{
-  const char *start=NULL, *end=NULL;
-  if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c,
-                                  &start,&end)<0)
-    return -1;
-
-  if (alg == DIGEST_SHA1) {
-    if (crypto_digest(digest, start, end-start)) {
-      log_warn(LD_BUG,"couldn't compute digest");
-      return -1;
-    }
-  } else {
-    if (crypto_digest256(digest, start, end-start, alg)) {
-      log_warn(LD_BUG,"couldn't compute digest");
-      return -1;
-    }
-  }
-
-  return 0;
-}
-
-/** As router_get_hash_impl, but compute all hashes. */
-static int
-router_get_hashes_impl(const char *s, size_t s_len, common_digests_t *digests,
-                       const char *start_str,
-                       const char *end_str, char end_c)
-{
-  const char *start=NULL, *end=NULL;
-  if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c,
-                                  &start,&end)<0)
-    return -1;
-
-  if (crypto_common_digests(digests, start, end-start)) {
-    log_warn(LD_BUG,"couldn't compute digests");
-    return -1;
-  }
-
-  return 0;
-}
-
-/** Assuming that s starts with a microdesc, return the start of the
- * *NEXT* one.  Return NULL on "not found." */
-static const char *
-find_start_of_next_microdesc(const char *s, const char *eos)
-{
-  int started_with_annotations;
-  s = eat_whitespace_eos(s, eos);
-  if (!s)
-    return NULL;
-
-#define CHECK_LENGTH() STMT_BEGIN \
-    if (eos - s < 32)             \
-      return NULL;                \
-  STMT_END
-
-#define NEXT_LINE() STMT_BEGIN            \
-    s = memchr(s, '\n', eos-s);           \
-    if (!s || eos - s <= 1)               \
-      return NULL;                        \
-    s++;                                  \
-  STMT_END
-
-  CHECK_LENGTH();
-
-  started_with_annotations = (*s == '@');
-
-  if (started_with_annotations) {
-    /* Start by advancing to the first non-annotation line. */
-    while (*s == '@')
-      NEXT_LINE();
-  }
-  CHECK_LENGTH();
-
-  /* Now we should be pointed at an onion-key line.  If we are, then skip
-   * it. */
-  if (!strcmpstart(s, "onion-key"))
-    NEXT_LINE();
-
-  /* Okay, now we're pointed at the first line of the microdescriptor which is
-     not an annotation or onion-key.  The next line that _is_ an annotation or
-     onion-key is the start of the next microdescriptor. */
-  while (eos - s > 32) {
-    if (*s == '@' || !strcmpstart(s, "onion-key"))
-      return s;
-    NEXT_LINE();
-  }
-  return NULL;
-
-#undef CHECK_LENGTH
-#undef NEXT_LINE
-}
-
-/** Parse as many microdescriptors as are found from the string starting at
- * <b>s</b> and ending at <b>eos</b>.  If allow_annotations is set, read any
- * annotations we recognize and ignore ones we don't.
- *
- * If <b>saved_location</b> isn't SAVED_IN_CACHE, make a local copy of each
- * descriptor in the body field of each microdesc_t.
- *
- * Return all newly parsed microdescriptors in a newly allocated
- * smartlist_t. If <b>invalid_disgests_out</b> is provided, add a SHA256
- * microdesc digest to it for every microdesc that we found to be badly
- * formed. (This may cause duplicates) */
-smartlist_t *
-microdescs_parse_from_string(const char *s, const char *eos,
-                             int allow_annotations,
-                             saved_location_t where,
-                             smartlist_t *invalid_digests_out)
-{
-  smartlist_t *tokens;
-  smartlist_t *result;
-  microdesc_t *md = NULL;
-  memarea_t *area;
-  const char *start = s;
-  const char *start_of_next_microdesc;
-  int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0;
-  const int copy_body = (where != SAVED_IN_CACHE);
-
-  directory_token_t *tok;
-
-  if (!eos)
-    eos = s + strlen(s);
-
-  s = eat_whitespace_eos(s, eos);
-  area = memarea_new();
-  result = smartlist_new();
-  tokens = smartlist_new();
-
-  while (s < eos) {
-    int okay = 0;
-
-    start_of_next_microdesc = find_start_of_next_microdesc(s, eos);
-    if (!start_of_next_microdesc)
-      start_of_next_microdesc = eos;
-
-    md = tor_malloc_zero(sizeof(microdesc_t));
-    {
-      const char *cp = tor_memstr(s, start_of_next_microdesc-s,
-                                  "onion-key");
-      const int no_onion_key = (cp == NULL);
-      if (no_onion_key) {
-        cp = s; /* So that we have *some* junk to put in the body */
-      }
-
-      md->bodylen = start_of_next_microdesc - cp;
-      md->saved_location = where;
-      if (copy_body)
-        md->body = tor_memdup_nulterm(cp, md->bodylen);
-      else
-        md->body = (char*)cp;
-      md->off = cp - start;
-      crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256);
-      if (no_onion_key) {
-        log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Malformed or truncated descriptor");
-        goto next;
-      }
-    }
-
-    if (tokenize_string(area, s, start_of_next_microdesc, tokens,
-                        microdesc_token_table, flags)) {
-      log_warn(LD_DIR, "Unparseable microdescriptor");
-      goto next;
-    }
-
-    if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) {
-      if (parse_iso_time(tok->args[0], &md->last_listed)) {
-        log_warn(LD_DIR, "Bad last-listed time in microdescriptor");
-        goto next;
-      }
-    }
-
-    tok = find_by_keyword(tokens, K_ONION_KEY);
-    if (!crypto_pk_public_exponent_ok(tok->key)) {
-      log_warn(LD_DIR,
-               "Relay's onion key had invalid exponent.");
-      goto next;
-    }
-    md->onion_pkey = tok->key;
-    tok->key = NULL;
-
-    if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) {
-      curve25519_public_key_t k;
-      tor_assert(tok->n_args >= 1);
-      if (curve25519_public_from_base64(&k, tok->args[0]) < 0) {
-        log_warn(LD_DIR, "Bogus ntor-onion-key in microdesc");
-        goto next;
-      }
-      md->onion_curve25519_pkey =
-        tor_memdup(&k, sizeof(curve25519_public_key_t));
-    }
-
-    smartlist_t *id_lines = find_all_by_keyword(tokens, K_ID);
-    if (id_lines) {
-      SMARTLIST_FOREACH_BEGIN(id_lines, directory_token_t *, t) {
-        tor_assert(t->n_args >= 2);
-        if (!strcmp(t->args[0], "ed25519")) {
-          if (md->ed25519_identity_pkey) {
-            log_warn(LD_DIR, "Extra ed25519 key in microdesc");
-            goto next;
-          }
-          ed25519_public_key_t k;
-          if (ed25519_public_from_base64(&k, t->args[1])<0) {
-            log_warn(LD_DIR, "Bogus ed25519 key in microdesc");
-            goto next;
-          }
-          md->ed25519_identity_pkey = tor_memdup(&k, sizeof(k));
-        }
-      } SMARTLIST_FOREACH_END(t);
-      smartlist_free(id_lines);
-    }
-
-    {
-      smartlist_t *a_lines = find_all_by_keyword(tokens, K_A);
-      if (a_lines) {
-        find_single_ipv6_orport(a_lines, &md->ipv6_addr, &md->ipv6_orport);
-        smartlist_free(a_lines);
-      }
-    }
-
-    if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) {
-      int i;
-      md->family = smartlist_new();
-      for (i=0;i<tok->n_args;++i) {
-        if (!is_legal_nickname_or_hexdigest(tok->args[i])) {
-          log_warn(LD_DIR, "Illegal nickname %s in family line",
-                   escaped(tok->args[i]));
-          goto next;
-        }
-        smartlist_add(md->family, tor_strdup(tok->args[i]));
-      }
-    }
-
-    if ((tok = find_opt_by_keyword(tokens, K_P))) {
-      md->exit_policy = parse_short_policy(tok->args[0]);
-    }
-    if ((tok = find_opt_by_keyword(tokens, K_P6))) {
-      md->ipv6_exit_policy = parse_short_policy(tok->args[0]);
-    }
-
-    smartlist_add(result, md);
-    okay = 1;
-
-    md = NULL;
-  next:
-    if (! okay && invalid_digests_out) {
-      smartlist_add(invalid_digests_out,
-                    tor_memdup(md->digest, DIGEST256_LEN));
-    }
-    microdesc_free(md);
-    md = NULL;
-
-    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
-    memarea_clear(area);
-    smartlist_clear(tokens);
-    s = start_of_next_microdesc;
-  }
-
-  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
-  memarea_drop_all(area);
-  smartlist_free(tokens);
-
-  return result;
-}
-
-/** Extract a Tor version from a <b>platform</b> line from a router
- * descriptor, and place the result in <b>router_version</b>.
- *
- * Return 1 on success, -1 on parsing failure, and 0 if the
- * platform line does not indicate some version of Tor.
- *
- * If <b>strict</b> is non-zero, finding any weird version components
- * (like negative numbers) counts as a parsing failure.
- */
-int
-tor_version_parse_platform(const char *platform,
-                           tor_version_t *router_version,
-                           int strict)
-{
-  char tmp[128];
-  char *s, *s2, *start;
-
-  if (strcmpstart(platform,"Tor ")) /* nonstandard Tor; say 0. */
-    return 0;
-
-  start = (char *)eat_whitespace(platform+3);
-  if (!*start) return -1;
-  s = (char *)find_whitespace(start); /* also finds '\0', which is fine */
-  s2 = (char*)eat_whitespace(s);
-  if (!strcmpstart(s2, "(r") || !strcmpstart(s2, "(git-"))
-    s = (char*)find_whitespace(s2);
-
-  if ((size_t)(s-start+1) >= sizeof(tmp)) /* too big, no */
-    return -1;
-  strlcpy(tmp, start, s-start+1);
-
-  if (tor_version_parse(tmp, router_version)<0) {
-    log_info(LD_DIR,"Router version '%s' unparseable.",tmp);
-    return -1;
-  }
-
-  if (strict) {
-    if (router_version->major < 0 ||
-        router_version->minor < 0 ||
-        router_version->micro < 0 ||
-        router_version->patchlevel < 0 ||
-        router_version->svn_revision < 0) {
-      return -1;
-    }
-  }
-
-  return 1;
-}
-
-/** Parse the Tor version of the platform string <b>platform</b>,
- * and compare it to the version in <b>cutoff</b>. Return 1 if
- * the router is at least as new as the cutoff, else return 0.
- */
-int
-tor_version_as_new_as(const char *platform, const char *cutoff)
-{
-  tor_version_t cutoff_version, router_version;
-  int r;
-  tor_assert(platform);
-
-  if (tor_version_parse(cutoff, &cutoff_version)<0) {
-    log_warn(LD_BUG,"cutoff version '%s' unparseable.",cutoff);
-    return 0;
-  }
-
-  r = tor_version_parse_platform(platform, &router_version, 0);
-  if (r == 0) {
-    /* nonstandard Tor; be safe and say yes */
-    return 1;
-  } else if (r < 0) {
-    /* unparseable version; be safe and say yes. */
-    return 1;
-  }
-
-  /* Here's why we don't need to do any special handling for svn revisions:
-   * - If neither has an svn revision, we're fine.
-   * - If the router doesn't have an svn revision, we can't assume that it
-   *   is "at least" any svn revision, so we need to return 0.
-   * - If the target version doesn't have an svn revision, any svn revision
-   *   (or none at all) is good enough, so return 1.
-   * - If both target and router have an svn revision, we compare them.
-   */
-
-  return tor_version_compare(&router_version, &cutoff_version) >= 0;
-}
-
-/** Parse a tor version from <b>s</b>, and store the result in <b>out</b>.
- * Return 0 on success, -1 on failure. */
-int
-tor_version_parse(const char *s, tor_version_t *out)
-{
-  char *eos=NULL;
-  const char *cp=NULL;
-  int ok = 1;
-  /* Format is:
-   *   "Tor " ? NUM dot NUM [ dot NUM [ ( pre | rc | dot ) NUM ] ] [ - tag ]
-   */
-  tor_assert(s);
-  tor_assert(out);
-
-  memset(out, 0, sizeof(tor_version_t));
-  out->status = VER_RELEASE;
-  if (!strcasecmpstart(s, "Tor "))
-    s += 4;
-
-  cp = s;
-
-#define NUMBER(m)                               \
-  do {                                          \
-    if (!cp || *cp < '0' || *cp > '9')          \
-      return -1;                                \
-    out->m = (int)tor_parse_uint64(cp, 10, 0, INT32_MAX, &ok, &eos);    \
-    if (!ok)                                    \
-      return -1;                                \
-    if (!eos || eos == cp)                      \
-      return -1;                                \
-    cp = eos;                                   \
-  } while (0)
-
-#define DOT()                                   \
-  do {                                          \
-    if (*cp != '.')                             \
-      return -1;                                \
-    ++cp;                                       \
-  } while (0)
-
-  NUMBER(major);
-  DOT();
-  NUMBER(minor);
-  if (*cp == 0)
-    return 0;
-  else if (*cp == '-')
-    goto status_tag;
-  DOT();
-  NUMBER(micro);
-
-  /* Get status */
-  if (*cp == 0) {
-    return 0;
-  } else if (*cp == '.') {
-    ++cp;
-  } else if (*cp == '-') {
-    goto status_tag;
-  } else if (0==strncmp(cp, "pre", 3)) {
-    out->status = VER_PRE;
-    cp += 3;
-  } else if (0==strncmp(cp, "rc", 2)) {
-    out->status = VER_RC;
-    cp += 2;
-  } else {
-    return -1;
-  }
-
-  NUMBER(patchlevel);
-
- status_tag:
-  /* Get status tag. */
-  if (*cp == '-' || *cp == '.')
-    ++cp;
-  eos = (char*) find_whitespace(cp);
-  if (eos-cp >= (int)sizeof(out->status_tag))
-    strlcpy(out->status_tag, cp, sizeof(out->status_tag));
-  else {
-    memcpy(out->status_tag, cp, eos-cp);
-    out->status_tag[eos-cp] = 0;
-  }
-  cp = eat_whitespace(eos);
-
-  if (!strcmpstart(cp, "(r")) {
-    cp += 2;
-    out->svn_revision = (int) strtol(cp,&eos,10);
-  } else if (!strcmpstart(cp, "(git-")) {
-    char *close_paren = strchr(cp, ')');
-    int hexlen;
-    char digest[DIGEST_LEN];
-    if (! close_paren)
-      return -1;
-    cp += 5;
-    if (close_paren-cp > HEX_DIGEST_LEN)
-      return -1;
-    hexlen = (int)(close_paren-cp);
-    memwipe(digest, 0, sizeof(digest));
-    if ( hexlen == 0 || (hexlen % 2) == 1)
-      return -1;
-    if (base16_decode(digest, hexlen/2, cp, hexlen) != hexlen/2)
-      return -1;
-    memcpy(out->git_tag, digest, hexlen/2);
-    out->git_tag_len = hexlen/2;
-  }
-
-  return 0;
-#undef NUMBER
-#undef DOT
-}
-
-/** Compare two tor versions; Return <0 if a < b; 0 if a ==b, >0 if a >
- * b. */
-int
-tor_version_compare(tor_version_t *a, tor_version_t *b)
-{
-  int i;
-  tor_assert(a);
-  tor_assert(b);
-
-  /* We take this approach to comparison to ensure the same (bogus!) behavior
-   * on all inputs as we would have seen before bug #21278 was fixed. The
-   * only important difference here is that this method doesn't cause
-   * a signed integer underflow.
-   */
-#define CMP(field) do {                               \
-    unsigned aval = (unsigned) a->field;              \
-    unsigned bval = (unsigned) b->field;              \
-    int result = (int) (aval - bval);                 \
-    if (result < 0)                                   \
-      return -1;                                      \
-    else if (result > 0)                              \
-      return 1;                                       \
-  } while (0)
-
-  CMP(major);
-  CMP(minor);
-  CMP(micro);
-  CMP(status);
-  CMP(patchlevel);
-  if ((i = strcmp(a->status_tag, b->status_tag)))
-     return i;
-  CMP(svn_revision);
-  CMP(git_tag_len);
-  if (a->git_tag_len)
-     return fast_memcmp(a->git_tag, b->git_tag, a->git_tag_len);
-  else
-     return 0;
-
-#undef CMP
-}
-
-/** Return true iff versions <b>a</b> and <b>b</b> belong to the same series.
- */
-int
-tor_version_same_series(tor_version_t *a, tor_version_t *b)
-{
-  tor_assert(a);
-  tor_assert(b);
-  return ((a->major == b->major) &&
-          (a->minor == b->minor) &&
-          (a->micro == b->micro));
-}
-
-/** Helper: Given pointers to two strings describing tor versions, return -1
- * if _a precedes _b, 1 if _b precedes _a, and 0 if they are equivalent.
- * Used to sort a list of versions. */
-static int
-compare_tor_version_str_ptr_(const void **_a, const void **_b)
-{
-  const char *a = *_a, *b = *_b;
-  int ca, cb;
-  tor_version_t va, vb;
-  ca = tor_version_parse(a, &va);
-  cb = tor_version_parse(b, &vb);
-  /* If they both parse, compare them. */
-  if (!ca && !cb)
-    return tor_version_compare(&va,&vb);
-  /* If one parses, it comes first. */
-  if (!ca && cb)
-    return -1;
-  if (ca && !cb)
-    return 1;
-  /* If neither parses, compare strings.  Also, the directory server admin
-  ** needs to be smacked upside the head.  But Tor is tolerant and gentle. */
-  return strcmp(a,b);
-}
-
-/** Sort a list of string-representations of versions in ascending order. */
-void
-sort_version_list(smartlist_t *versions, int remove_duplicates)
-{
-  smartlist_sort(versions, compare_tor_version_str_ptr_);
-
-  if (remove_duplicates)
-    smartlist_uniq(versions, compare_tor_version_str_ptr_, tor_free_);
-}
-
-/** Parse and validate the ASCII-encoded v2 descriptor in <b>desc</b>,
- * write the parsed descriptor to the newly allocated *<b>parsed_out</b>, the
- * binary descriptor ID of length DIGEST_LEN to <b>desc_id_out</b>, the
- * encrypted introduction points to the newly allocated
- * *<b>intro_points_encrypted_out</b>, their encrypted size to
- * *<b>intro_points_encrypted_size_out</b>, the size of the encoded descriptor
- * to *<b>encoded_size_out</b>, and a pointer to the possibly next
- * descriptor to *<b>next_out</b>; return 0 for success (including validation)
- * and -1 for failure.
- *
- * If <b>as_hsdir</b> is 1, we're parsing this as an HSDir, and we should
- * be strict about time formats.
- */
-int
-rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
-                                 char *desc_id_out,
-                                 char **intro_points_encrypted_out,
-                                 size_t *intro_points_encrypted_size_out,
-                                 size_t *encoded_size_out,
-                                 const char **next_out, const char *desc,
-                                 int as_hsdir)
-{
-  rend_service_descriptor_t *result =
-                            tor_malloc_zero(sizeof(rend_service_descriptor_t));
-  char desc_hash[DIGEST_LEN];
-  const char *eos;
-  smartlist_t *tokens = smartlist_new();
-  directory_token_t *tok;
-  char secret_id_part[DIGEST_LEN];
-  int i, version, num_ok=1;
-  smartlist_t *versions;
-  char public_key_hash[DIGEST_LEN];
-  char test_desc_id[DIGEST_LEN];
-  memarea_t *area = NULL;
-  const int strict_time_fmt = as_hsdir;
-
-  tor_assert(desc);
-  /* Check if desc starts correctly. */
-  if (strncmp(desc, "rendezvous-service-descriptor ",
-              strlen("rendezvous-service-descriptor "))) {
-    log_info(LD_REND, "Descriptor does not start correctly.");
-    goto err;
-  }
-  /* Compute descriptor hash for later validation. */
-  if (router_get_hash_impl(desc, strlen(desc), desc_hash,
-                           "rendezvous-service-descriptor ",
-                           "\nsignature", '\n', DIGEST_SHA1) < 0) {
-    log_warn(LD_REND, "Couldn't compute descriptor hash.");
-    goto err;
-  }
-  /* Determine end of string. */
-  eos = strstr(desc, "\nrendezvous-service-descriptor ");
-  if (!eos)
-    eos = desc + strlen(desc);
-  else
-    eos = eos + 1;
-  /* Check length. */
-  if (eos-desc > REND_DESC_MAX_SIZE) {
-    /* XXXX+ If we are parsing this descriptor as a server, this
-     * should be a protocol warning. */
-    log_warn(LD_REND, "Descriptor length is %d which exceeds "
-             "maximum rendezvous descriptor size of %d bytes.",
-             (int)(eos-desc), REND_DESC_MAX_SIZE);
-    goto err;
-  }
-  /* Tokenize descriptor. */
-  area = memarea_new();
-  if (tokenize_string(area, desc, eos, tokens, desc_token_table, 0)) {
-    log_warn(LD_REND, "Error tokenizing descriptor.");
-    goto err;
-  }
-  /* Set next to next descriptor, if available. */
-  *next_out = eos;
-  /* Set length of encoded descriptor. */
-  *encoded_size_out = eos - desc;
-  /* Check min allowed length of token list. */
-  if (smartlist_len(tokens) < 7) {
-    log_warn(LD_REND, "Impossibly short descriptor.");
-    goto err;
-  }
-  /* Parse base32-encoded descriptor ID. */
-  tok = find_by_keyword(tokens, R_RENDEZVOUS_SERVICE_DESCRIPTOR);
-  tor_assert(tok == smartlist_get(tokens, 0));
-  tor_assert(tok->n_args == 1);
-  if (!rend_valid_descriptor_id(tok->args[0])) {
-    log_warn(LD_REND, "Invalid descriptor ID: '%s'", tok->args[0]);
-    goto err;
-  }
-  if (base32_decode(desc_id_out, DIGEST_LEN,
-                    tok->args[0], REND_DESC_ID_V2_LEN_BASE32) < 0) {
-    log_warn(LD_REND, "Descriptor ID contains illegal characters: %s",
-             tok->args[0]);
-    goto err;
-  }
-  /* Parse descriptor version. */
-  tok = find_by_keyword(tokens, R_VERSION);
-  tor_assert(tok->n_args == 1);
-  result->version =
-    (int) tor_parse_long(tok->args[0], 10, 0, INT_MAX, &num_ok, NULL);
-  if (result->version != 2 || !num_ok) {
-    /* If it's <2, it shouldn't be under this format.  If the number
-     * is greater than 2, we bumped it because we broke backward
-     * compatibility.  See how version numbers in our other formats
-     * work. */
-    log_warn(LD_REND, "Unrecognized descriptor version: %s",
-             escaped(tok->args[0]));
-    goto err;
-  }
-  /* Parse public key. */
-  tok = find_by_keyword(tokens, R_PERMANENT_KEY);
-  result->pk = tok->key;
-  tok->key = NULL; /* Prevent free */
-  /* Parse secret ID part. */
-  tok = find_by_keyword(tokens, R_SECRET_ID_PART);
-  tor_assert(tok->n_args == 1);
-  if (strlen(tok->args[0]) != REND_SECRET_ID_PART_LEN_BASE32 ||
-      strspn(tok->args[0], BASE32_CHARS) != REND_SECRET_ID_PART_LEN_BASE32) {
-    log_warn(LD_REND, "Invalid secret ID part: '%s'", tok->args[0]);
-    goto err;
-  }
-  if (base32_decode(secret_id_part, DIGEST_LEN, tok->args[0], 32) < 0) {
-    log_warn(LD_REND, "Secret ID part contains illegal characters: %s",
-             tok->args[0]);
-    goto err;
-  }
-  /* Parse publication time -- up-to-date check is done when storing the
-   * descriptor. */
-  tok = find_by_keyword(tokens, R_PUBLICATION_TIME);
-  tor_assert(tok->n_args == 1);
-  if (parse_iso_time_(tok->args[0], &result->timestamp, strict_time_fmt) < 0) {
-    log_warn(LD_REND, "Invalid publication time: '%s'", tok->args[0]);
-    goto err;
-  }
-  /* Parse protocol versions. */
-  tok = find_by_keyword(tokens, R_PROTOCOL_VERSIONS);
-  tor_assert(tok->n_args == 1);
-  versions = smartlist_new();
-  smartlist_split_string(versions, tok->args[0], ",",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  for (i = 0; i < smartlist_len(versions); i++) {
-    version = (int) tor_parse_long(smartlist_get(versions, i),
-                                   10, 0, INT_MAX, &num_ok, NULL);
-    if (!num_ok) /* It's a string; let's ignore it. */
-      continue;
-    if (version >= REND_PROTOCOL_VERSION_BITMASK_WIDTH)
-      /* Avoid undefined left-shift behaviour. */
-      continue;
-    result->protocols |= 1 << version;
-  }
-  SMARTLIST_FOREACH(versions, char *, cp, tor_free(cp));
-  smartlist_free(versions);
-  /* Parse encrypted introduction points. Don't verify. */
-  tok = find_opt_by_keyword(tokens, R_INTRODUCTION_POINTS);
-  if (tok) {
-    if (strcmp(tok->object_type, "MESSAGE")) {
-      log_warn(LD_DIR, "Bad object type: introduction points should be of "
-               "type MESSAGE");
-      goto err;
-    }
-    *intro_points_encrypted_out = tor_memdup(tok->object_body,
-                                             tok->object_size);
-    *intro_points_encrypted_size_out = tok->object_size;
-  } else {
-    *intro_points_encrypted_out = NULL;
-    *intro_points_encrypted_size_out = 0;
-  }
-  /* Parse and verify signature. */
-  tok = find_by_keyword(tokens, R_SIGNATURE);
-  note_crypto_pk_op(VERIFY_RTR);
-  if (check_signature_token(desc_hash, DIGEST_LEN, tok, result->pk, 0,
-                            "v2 rendezvous service descriptor") < 0)
-    goto err;
-  /* Verify that descriptor ID belongs to public key and secret ID part. */
-  crypto_pk_get_digest(result->pk, public_key_hash);
-  rend_get_descriptor_id_bytes(test_desc_id, public_key_hash,
-                               secret_id_part);
-  if (tor_memneq(desc_id_out, test_desc_id, DIGEST_LEN)) {
-    log_warn(LD_REND, "Parsed descriptor ID does not match "
-             "computed descriptor ID.");
-    goto err;
-  }
-  goto done;
- err:
-  rend_service_descriptor_free(result);
-  result = NULL;
- done:
-  if (tokens) {
-    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
-    smartlist_free(tokens);
-  }
-  if (area)
-    memarea_drop_all(area);
-  *parsed_out = result;
-  if (result)
-    return 0;
-  return -1;
-}
-
-/** Decrypt the encrypted introduction points in <b>ipos_encrypted</b> of
- * length <b>ipos_encrypted_size</b> using <b>descriptor_cookie</b> and
- * write the result to a newly allocated string that is pointed to by
- * <b>ipos_decrypted</b> and its length to <b>ipos_decrypted_size</b>.
- * Return 0 if decryption was successful and -1 otherwise. */
-int
-rend_decrypt_introduction_points(char **ipos_decrypted,
-                                 size_t *ipos_decrypted_size,
-                                 const char *descriptor_cookie,
-                                 const char *ipos_encrypted,
-                                 size_t ipos_encrypted_size)
-{
-  tor_assert(ipos_encrypted);
-  tor_assert(descriptor_cookie);
-  if (ipos_encrypted_size < 2) {
-    log_warn(LD_REND, "Size of encrypted introduction points is too "
-                      "small.");
-    return -1;
-  }
-  if (ipos_encrypted[0] == (int)REND_BASIC_AUTH) {
-    char iv[CIPHER_IV_LEN], client_id[REND_BASIC_AUTH_CLIENT_ID_LEN],
-         session_key[CIPHER_KEY_LEN], *dec;
-    int declen, client_blocks;
-    size_t pos = 0, len, client_entries_len;
-    crypto_digest_t *digest;
-    crypto_cipher_t *cipher;
-    client_blocks = (int) ipos_encrypted[1];
-    client_entries_len = client_blocks * REND_BASIC_AUTH_CLIENT_MULTIPLE *
-                         REND_BASIC_AUTH_CLIENT_ENTRY_LEN;
-    if (ipos_encrypted_size < 2 + client_entries_len + CIPHER_IV_LEN + 1) {
-      log_warn(LD_REND, "Size of encrypted introduction points is too "
-                        "small.");
-      return -1;
-    }
-    memcpy(iv, ipos_encrypted + 2 + client_entries_len, CIPHER_IV_LEN);
-    digest = crypto_digest_new();
-    crypto_digest_add_bytes(digest, descriptor_cookie, REND_DESC_COOKIE_LEN);
-    crypto_digest_add_bytes(digest, iv, CIPHER_IV_LEN);
-    crypto_digest_get_digest(digest, client_id,
-                             REND_BASIC_AUTH_CLIENT_ID_LEN);
-    crypto_digest_free(digest);
-    for (pos = 2; pos < 2 + client_entries_len;
-         pos += REND_BASIC_AUTH_CLIENT_ENTRY_LEN) {
-      if (tor_memeq(ipos_encrypted + pos, client_id,
-                  REND_BASIC_AUTH_CLIENT_ID_LEN)) {
-        /* Attempt to decrypt introduction points. */
-        cipher = crypto_cipher_new(descriptor_cookie);
-        if (crypto_cipher_decrypt(cipher, session_key, ipos_encrypted
-                                  + pos + REND_BASIC_AUTH_CLIENT_ID_LEN,
-                                  CIPHER_KEY_LEN) < 0) {
-          log_warn(LD_REND, "Could not decrypt session key for client.");
-          crypto_cipher_free(cipher);
-          return -1;
-        }
-        crypto_cipher_free(cipher);
-
-        len = ipos_encrypted_size - 2 - client_entries_len - CIPHER_IV_LEN;
-        dec = tor_malloc_zero(len + 1);
-        declen = crypto_cipher_decrypt_with_iv(session_key, dec, len,
-            ipos_encrypted + 2 + client_entries_len,
-            ipos_encrypted_size - 2 - client_entries_len);
-
-        if (declen < 0) {
-          log_warn(LD_REND, "Could not decrypt introduction point string.");
-          tor_free(dec);
-          return -1;
-        }
-        if (fast_memcmpstart(dec, declen, "introduction-point ")) {
-          log_warn(LD_REND, "Decrypted introduction points don't "
-                            "look like we could parse them.");
-          tor_free(dec);
-          continue;
-        }
-        *ipos_decrypted = dec;
-        *ipos_decrypted_size = declen;
-        return 0;
-      }
-    }
-    log_warn(LD_REND, "Could not decrypt introduction points. Please "
-             "check your authorization for this service!");
-    return -1;
-  } else if (ipos_encrypted[0] == (int)REND_STEALTH_AUTH) {
-    char *dec;
-    int declen;
-    if (ipos_encrypted_size < CIPHER_IV_LEN + 2) {
-      log_warn(LD_REND, "Size of encrypted introduction points is too "
-                        "small.");
-      return -1;
-    }
-    dec = tor_malloc_zero(ipos_encrypted_size - CIPHER_IV_LEN - 1 + 1);
-
-    declen = crypto_cipher_decrypt_with_iv(descriptor_cookie, dec,
-                                           ipos_encrypted_size -
-                                               CIPHER_IV_LEN - 1,
-                                           ipos_encrypted + 1,
-                                           ipos_encrypted_size - 1);
-
-    if (declen < 0) {
-      log_warn(LD_REND, "Decrypting introduction points failed!");
-      tor_free(dec);
-      return -1;
-    }
-    *ipos_decrypted = dec;
-    *ipos_decrypted_size = declen;
-    return 0;
-  } else {
-    log_warn(LD_REND, "Unknown authorization type number: %d",
-             ipos_encrypted[0]);
-    return -1;
-  }
-}
-
-/** Parse the encoded introduction points in <b>intro_points_encoded</b> of
- * length <b>intro_points_encoded_size</b> and write the result to the
- * descriptor in <b>parsed</b>; return the number of successfully parsed
- * introduction points or -1 in case of a failure. */
-int
-rend_parse_introduction_points(rend_service_descriptor_t *parsed,
-                               const char *intro_points_encoded,
-                               size_t intro_points_encoded_size)
-{
-  const char *current_ipo, *end_of_intro_points;
-  smartlist_t *tokens = NULL;
-  directory_token_t *tok;
-  rend_intro_point_t *intro;
-  extend_info_t *info;
-  int result, num_ok=1;
-  memarea_t *area = NULL;
-  tor_assert(parsed);
-  /** Function may only be invoked once. */
-  tor_assert(!parsed->intro_nodes);
-  if (!intro_points_encoded || intro_points_encoded_size == 0) {
-    log_warn(LD_REND, "Empty or zero size introduction point list");
-    goto err;
-  }
-  /* Consider one intro point after the other. */
-  current_ipo = intro_points_encoded;
-  end_of_intro_points = intro_points_encoded + intro_points_encoded_size;
-  tokens = smartlist_new();
-  parsed->intro_nodes = smartlist_new();
-  area = memarea_new();
-
-  while (!fast_memcmpstart(current_ipo, end_of_intro_points-current_ipo,
-                      "introduction-point ")) {
-    /* Determine end of string. */
-    const char *eos = tor_memstr(current_ipo, end_of_intro_points-current_ipo,
-                                 "\nintroduction-point ");
-    if (!eos)
-      eos = end_of_intro_points;
-    else
-      eos = eos+1;
-    tor_assert(eos <= intro_points_encoded+intro_points_encoded_size);
-    /* Free tokens and clear token list. */
-    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
-    smartlist_clear(tokens);
-    memarea_clear(area);
-    /* Tokenize string. */
-    if (tokenize_string(area, current_ipo, eos, tokens, ipo_token_table, 0)) {
-      log_warn(LD_REND, "Error tokenizing introduction point");
-      goto err;
-    }
-    /* Advance to next introduction point, if available. */
-    current_ipo = eos;
-    /* Check minimum allowed length of introduction point. */
-    if (smartlist_len(tokens) < 5) {
-      log_warn(LD_REND, "Impossibly short introduction point.");
-      goto err;
-    }
-    /* Allocate new intro point and extend info. */
-    intro = tor_malloc_zero(sizeof(rend_intro_point_t));
-    info = intro->extend_info = tor_malloc_zero(sizeof(extend_info_t));
-    /* Parse identifier. */
-    tok = find_by_keyword(tokens, R_IPO_IDENTIFIER);
-    if (base32_decode(info->identity_digest, DIGEST_LEN,
-                      tok->args[0], REND_INTRO_POINT_ID_LEN_BASE32) < 0) {
-      log_warn(LD_REND, "Identity digest contains illegal characters: %s",
-               tok->args[0]);
-      rend_intro_point_free(intro);
-      goto err;
-    }
-    /* Write identifier to nickname. */
-    info->nickname[0] = '$';
-    base16_encode(info->nickname + 1, sizeof(info->nickname) - 1,
-                  info->identity_digest, DIGEST_LEN);
-    /* Parse IP address. */
-    tok = find_by_keyword(tokens, R_IPO_IP_ADDRESS);
-    if (tor_addr_parse(&info->addr, tok->args[0])<0) {
-      log_warn(LD_REND, "Could not parse introduction point address.");
-      rend_intro_point_free(intro);
-      goto err;
-    }
-    if (tor_addr_family(&info->addr) != AF_INET) {
-      log_warn(LD_REND, "Introduction point address was not ipv4.");
-      rend_intro_point_free(intro);
-      goto err;
-    }
-
-    /* Parse onion port. */
-    tok = find_by_keyword(tokens, R_IPO_ONION_PORT);
-    info->port = (uint16_t) tor_parse_long(tok->args[0],10,1,65535,
-                                           &num_ok,NULL);
-    if (!info->port || !num_ok) {
-      log_warn(LD_REND, "Introduction point onion port %s is invalid",
-               escaped(tok->args[0]));
-      rend_intro_point_free(intro);
-      goto err;
-    }
-    /* Parse onion key. */
-    tok = find_by_keyword(tokens, R_IPO_ONION_KEY);
-    if (!crypto_pk_public_exponent_ok(tok->key)) {
-      log_warn(LD_REND,
-               "Introduction point's onion key had invalid exponent.");
-      rend_intro_point_free(intro);
-      goto err;
-    }
-    info->onion_key = tok->key;
-    tok->key = NULL; /* Prevent free */
-    /* Parse service key. */
-    tok = find_by_keyword(tokens, R_IPO_SERVICE_KEY);
-    if (!crypto_pk_public_exponent_ok(tok->key)) {
-      log_warn(LD_REND,
-               "Introduction point key had invalid exponent.");
-      rend_intro_point_free(intro);
-      goto err;
-    }
-    intro->intro_key = tok->key;
-    tok->key = NULL; /* Prevent free */
-    /* Add extend info to list of introduction points. */
-    smartlist_add(parsed->intro_nodes, intro);
-  }
-  result = smartlist_len(parsed->intro_nodes);
-  goto done;
-
- err:
-  result = -1;
-
- done:
-  /* Free tokens and clear token list. */
-  if (tokens) {
-    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
-    smartlist_free(tokens);
-  }
-  if (area)
-    memarea_drop_all(area);
-
-  return result;
-}
-
-/** Parse the content of a client_key file in <b>ckstr</b> and add
- * rend_authorized_client_t's for each parsed client to
- * <b>parsed_clients</b>. Return the number of parsed clients as result
- * or -1 for failure. */
-int
-rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr)
-{
-  int result = -1;
-  smartlist_t *tokens;
-  directory_token_t *tok;
-  const char *current_entry = NULL;
-  memarea_t *area = NULL;
-  char *err_msg = NULL;
-  if (!ckstr || strlen(ckstr) == 0)
-    return -1;
-  tokens = smartlist_new();
-  /* Begin parsing with first entry, skipping comments or whitespace at the
-   * beginning. */
-  area = memarea_new();
-  current_entry = eat_whitespace(ckstr);
-  while (!strcmpstart(current_entry, "client-name ")) {
-    rend_authorized_client_t *parsed_entry;
-    /* Determine end of string. */
-    const char *eos = strstr(current_entry, "\nclient-name ");
-    if (!eos)
-      eos = current_entry + strlen(current_entry);
-    else
-      eos = eos + 1;
-    /* Free tokens and clear token list. */
-    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
-    smartlist_clear(tokens);
-    memarea_clear(area);
-    /* Tokenize string. */
-    if (tokenize_string(area, current_entry, eos, tokens,
-                        client_keys_token_table, 0)) {
-      log_warn(LD_REND, "Error tokenizing client keys file.");
-      goto err;
-    }
-    /* Advance to next entry, if available. */
-    current_entry = eos;
-    /* Check minimum allowed length of token list. */
-    if (smartlist_len(tokens) < 2) {
-      log_warn(LD_REND, "Impossibly short client key entry.");
-      goto err;
-    }
-    /* Parse client name. */
-    tok = find_by_keyword(tokens, C_CLIENT_NAME);
-    tor_assert(tok == smartlist_get(tokens, 0));
-    tor_assert(tok->n_args == 1);
-
-    if (!rend_valid_client_name(tok->args[0])) {
-      log_warn(LD_CONFIG, "Illegal client name: %s. (Length must be "
-               "between 1 and %d, and valid characters are "
-               "[A-Za-z0-9+-_].)", tok->args[0], REND_CLIENTNAME_MAX_LEN);
-      goto err;
-    }
-    /* Check if client name is duplicate. */
-    if (strmap_get(parsed_clients, tok->args[0])) {
-      log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains a "
-               "duplicate client name: '%s'. Ignoring.", tok->args[0]);
-      goto err;
-    }
-    parsed_entry = tor_malloc_zero(sizeof(rend_authorized_client_t));
-    parsed_entry->client_name = tor_strdup(tok->args[0]);
-    strmap_set(parsed_clients, parsed_entry->client_name, parsed_entry);
-    /* Parse client key. */
-    tok = find_opt_by_keyword(tokens, C_CLIENT_KEY);
-    if (tok) {
-      parsed_entry->client_key = tok->key;
-      tok->key = NULL; /* Prevent free */
-    }
-
-    /* Parse descriptor cookie. */
-    tok = find_by_keyword(tokens, C_DESCRIPTOR_COOKIE);
-    tor_assert(tok->n_args == 1);
-    if (rend_auth_decode_cookie(tok->args[0], parsed_entry->descriptor_cookie,
-                                NULL, &err_msg) < 0) {
-      tor_assert(err_msg);
-      log_warn(LD_REND, "%s", err_msg);
-      tor_free(err_msg);
-      goto err;
-    }
-  }
-  result = strmap_size(parsed_clients);
-  goto done;
- err:
-  result = -1;
- done:
-  /* Free tokens and clear token list. */
-  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
-  smartlist_free(tokens);
-  if (area)
-    memarea_drop_all(area);
-  return result;
-}
-
-/** Called on startup; right now we just handle scanning the unparseable
- * descriptor dumps, but hang anything else we might need to do in the
- * future here as well.
- */
-void
-routerparse_init(void)
-{
-  /*
-   * Check both if the sandbox is active and whether it's configured; no
-   * point in loading all that if we won't be able to use it after the
-   * sandbox becomes active.
-   */
-  if (!(sandbox_is_active() || get_options()->Sandbox)) {
-    dump_desc_init();
-  }
-}
-
-/** Clean up all data structures used by routerparse.c at exit */
-void
-routerparse_free_all(void)
-{
-  dump_desc_fifo_cleanup();
-}
diff --git a/src/or/routerparse.h b/src/or/routerparse.h
deleted file mode 100644
index 01a5de88e8..0000000000
--- a/src/or/routerparse.h
+++ /dev/null
@@ -1,131 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file routerparse.h
- * \brief Header file for routerparse.c.
- **/
-
-#ifndef TOR_ROUTERPARSE_H
-#define TOR_ROUTERPARSE_H
-
-int router_get_router_hash(const char *s, size_t s_len, char *digest);
-int router_get_dir_hash(const char *s, char *digest);
-int router_get_networkstatus_v3_hashes(const char *s,
-                                       common_digests_t *digests);
-int router_get_extrainfo_hash(const char *s, size_t s_len, char *digest);
-#define DIROBJ_MAX_SIG_LEN 256
-char *router_get_dirobj_signature(const char *digest,
-                                  size_t digest_len,
-                                  const crypto_pk_t *private_key);
-int router_append_dirobj_signature(char *buf, size_t buf_len,
-                                   const char *digest,
-                                   size_t digest_len,
-                                   crypto_pk_t *private_key);
-int router_parse_list_from_string(const char **s, const char *eos,
-                                  smartlist_t *dest,
-                                  saved_location_t saved_location,
-                                  int is_extrainfo,
-                                  int allow_annotations,
-                                  const char *prepend_annotations,
-                                  smartlist_t *invalid_digests_out);
-
-routerinfo_t *router_parse_entry_from_string(const char *s, const char *end,
-                                             int cache_copy,
-                                             int allow_annotations,
-                                             const char *prepend_annotations,
-                                             int *can_dl_again_out);
-extrainfo_t *extrainfo_parse_entry_from_string(const char *s, const char *end,
-                             int cache_copy, struct digest_ri_map_t *routermap,
-                             int *can_dl_again_out);
-MOCK_DECL(addr_policy_t *, router_parse_addr_policy_item_from_string,
-         (const char *s, int assume_action, int *malformed_list));
-version_status_t tor_version_is_obsolete(const char *myversion,
-                                         const char *versionlist);
-int tor_version_parse_platform(const char *platform,
-                               tor_version_t *version_out,
-                               int strict);
-int tor_version_as_new_as(const char *platform, const char *cutoff);
-int tor_version_parse(const char *s, tor_version_t *out);
-int tor_version_compare(tor_version_t *a, tor_version_t *b);
-int tor_version_same_series(tor_version_t *a, tor_version_t *b);
-void sort_version_list(smartlist_t *lst, int remove_duplicates);
-void assert_addr_policy_ok(smartlist_t *t);
-void dump_distinct_digest_count(int severity);
-
-int compare_vote_routerstatus_entries(const void **_a, const void **_b);
-int networkstatus_verify_bw_weights(networkstatus_t *ns, int);
-networkstatus_t *networkstatus_parse_vote_from_string(const char *s,
-                                                 const char **eos_out,
-                                                 networkstatus_type_t ns_type);
-ns_detached_signatures_t *networkstatus_parse_detached_signatures(
-                                          const char *s, const char *eos);
-
-smartlist_t *microdescs_parse_from_string(const char *s, const char *eos,
-                                          int allow_annotations,
-                                          saved_location_t where,
-                                          smartlist_t *invalid_digests_out);
-
-authority_cert_t *authority_cert_parse_from_string(const char *s,
-                                                   const char **end_of_string);
-int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
-                                     char *desc_id_out,
-                                     char **intro_points_encrypted_out,
-                                     size_t *intro_points_encrypted_size_out,
-                                     size_t *encoded_size_out,
-                                     const char **next_out, const char *desc,
-                                     int as_hsdir);
-int rend_decrypt_introduction_points(char **ipos_decrypted,
-                                     size_t *ipos_decrypted_size,
-                                     const char *descriptor_cookie,
-                                     const char *ipos_encrypted,
-                                     size_t ipos_encrypted_size);
-int rend_parse_introduction_points(rend_service_descriptor_t *parsed,
-                                   const char *intro_points_encoded,
-                                   size_t intro_points_encoded_size);
-int rend_parse_client_keys(strmap_t *parsed_clients, const char *str);
-
-void routerparse_init(void);
-void routerparse_free_all(void);
-
-#ifdef ROUTERPARSE_PRIVATE
-/*
- * One entry in the list of dumped descriptors; filename dumped to, length,
- * SHA-256 and timestamp.
- */
-
-typedef struct {
-  char *filename;
-  size_t len;
-  uint8_t digest_sha256[DIGEST256_LEN];
-  time_t when;
-} dumped_desc_t;
-
-EXTERN(uint64_t, len_descs_dumped)
-EXTERN(smartlist_t *, descs_dumped)
-STATIC int routerstatus_parse_guardfraction(const char *guardfraction_str,
-                                            networkstatus_t *vote,
-                                            vote_routerstatus_t *vote_rs,
-                                            routerstatus_t *rs);
-MOCK_DECL(STATIC dumped_desc_t *, dump_desc_populate_one_file,
-    (const char *dirname, const char *f));
-STATIC void dump_desc_populate_fifo_from_directory(const char *dirname);
-STATIC void dump_desc(const char *desc, const char *type);
-STATIC void dump_desc_fifo_cleanup(void);
-struct memarea_t;
-STATIC routerstatus_t *routerstatus_parse_entry_from_string(
-                                     struct memarea_t *area,
-                                     const char **s, smartlist_t *tokens,
-                                     networkstatus_t *vote,
-                                     vote_routerstatus_t *vote_rs,
-                                     int consensus_method,
-                                     consensus_flavor_t flav);
-#endif
-
-#define ED_DESC_SIGNATURE_PREFIX "Tor router descriptor signature v1"
-
-#endif
-
diff --git a/src/or/scheduler.c b/src/or/scheduler.c
deleted file mode 100644
index 49ac1b939a..0000000000
--- a/src/or/scheduler.c
+++ /dev/null
@@ -1,707 +0,0 @@
-/* * Copyright (c) 2013-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file scheduler.c
- * \brief Relay scheduling system
- **/
-
-#include "or.h"
-
-#define TOR_CHANNEL_INTERNAL_ /* For channel_flush_some_cells() */
-#include "channel.h"
-
-#include "compat_libevent.h"
-#define SCHEDULER_PRIVATE_
-#include "scheduler.h"
-
-#include <event2/event.h>
-
-/*
- * Scheduler high/low watermarks
- */
-
-static uint32_t sched_q_low_water = 16384;
-static uint32_t sched_q_high_water = 32768;
-
-/*
- * Maximum cells to flush in a single call to channel_flush_some_cells();
- * setting this low means more calls, but too high and we could overshoot
- * sched_q_high_water.
- */
-
-static uint32_t sched_max_flush_cells = 16;
-
-/*
- * Write scheduling works by keeping track of which channels can
- * accept cells, and have cells to write.  From the scheduler's perspective,
- * a channel can be in four possible states:
- *
- * 1.) Not open for writes, no cells to send
- *     - Not much to do here, and the channel will have scheduler_state ==
- *       SCHED_CHAN_IDLE
- *     - Transitions from:
- *       - Open for writes/has cells by simultaneously draining all circuit
- *         queues and filling the output buffer.
- *     - Transitions to:
- *       - Not open for writes/has cells by arrival of cells on an attached
- *         circuit (this would be driven from append_cell_to_circuit_queue())
- *       - Open for writes/no cells by a channel type specific path;
- *         driven from connection_or_flushed_some() for channel_tls_t.
- *
- * 2.) Open for writes, no cells to send
- *     - Not much here either; this will be the state an idle but open channel
- *       can be expected to settle in.  It will have scheduler_state ==
- *       SCHED_CHAN_WAITING_FOR_CELLS
- *     - Transitions from:
- *       - Not open for writes/no cells by flushing some of the output
- *         buffer.
- *       - Open for writes/has cells by the scheduler moving cells from
- *         circuit queues to channel output queue, but not having enough
- *         to fill the output queue.
- *     - Transitions to:
- *       - Open for writes/has cells by arrival of new cells on an attached
- *         circuit, in append_cell_to_circuit_queue()
- *
- * 3.) Not open for writes, cells to send
- *     - This is the state of a busy circuit limited by output bandwidth;
- *       cells have piled up in the circuit queues waiting to be relayed.
- *       The channel will have scheduler_state == SCHED_CHAN_WAITING_TO_WRITE.
- *     - Transitions from:
- *       - Not open for writes/no cells by arrival of cells on an attached
- *         circuit
- *       - Open for writes/has cells by filling an output buffer without
- *         draining all cells from attached circuits
- *    - Transitions to:
- *       - Opens for writes/has cells by draining some of the output buffer
- *         via the connection_or_flushed_some() path (for channel_tls_t).
- *
- * 4.) Open for writes, cells to send
- *     - This connection is ready to relay some cells and waiting for
- *       the scheduler to choose it.  The channel will have scheduler_state ==
- *       SCHED_CHAN_PENDING.
- *     - Transitions from:
- *       - Not open for writes/has cells by the connection_or_flushed_some()
- *         path
- *       - Open for writes/no cells by the append_cell_to_circuit_queue()
- *         path
- *     - Transitions to:
- *       - Not open for writes/no cells by draining all circuit queues and
- *         simultaneously filling the output buffer.
- *       - Not open for writes/has cells by writing enough cells to fill the
- *         output buffer
- *       - Open for writes/no cells by draining all attached circuit queues
- *         without also filling the output buffer
- *
- * Other event-driven parts of the code move channels between these scheduling
- * states by calling scheduler functions; the scheduler only runs on open-for-
- * writes/has-cells channels and is the only path for those to transition to
- * other states.  The scheduler_run() function gives us the opportunity to do
- * scheduling work, and is called from other scheduler functions whenever a
- * state transition occurs, and periodically from the main event loop.
- */
-
-/* Scheduler global data structures */
-
-/*
- * We keep a list of channels that are pending - i.e, have cells to write
- * and can accept them to send.  The enum scheduler_state in channel_t
- * is reserved for our use.
- */
-
-/* Pqueue of channels that can write and have cells (pending work) */
-STATIC smartlist_t *channels_pending = NULL;
-
-/*
- * This event runs the scheduler from its callback, and is manually
- * activated whenever a channel enters open for writes/cells to send.
- */
-
-STATIC struct event *run_sched_ev = NULL;
-
-/*
- * Queue heuristic; this is not the queue size, but an 'effective queuesize'
- * that ages out contributions from stalled channels.
- */
-
-STATIC uint64_t queue_heuristic = 0;
-
-/*
- * Timestamp for last queue heuristic update
- */
-
-STATIC time_t queue_heuristic_timestamp = 0;
-
-/* Scheduler static function declarations */
-
-static void scheduler_evt_callback(evutil_socket_t fd,
-                                   short events, void *arg);
-static int scheduler_more_work(void);
-static void scheduler_retrigger(void);
-#if 0
-static void scheduler_trigger(void);
-#endif
-
-/* Scheduler function implementations */
-
-/** Free everything and shut down the scheduling system */
-
-void
-scheduler_free_all(void)
-{
-  log_debug(LD_SCHED, "Shutting down scheduler");
-
-  if (run_sched_ev) {
-    if (event_del(run_sched_ev) < 0) {
-      log_warn(LD_BUG, "Problem deleting run_sched_ev");
-    }
-    tor_event_free(run_sched_ev);
-    run_sched_ev = NULL;
-  }
-
-  if (channels_pending) {
-    smartlist_free(channels_pending);
-    channels_pending = NULL;
-  }
-}
-
-/**
- * Comparison function to use when sorting pending channels
- */
-
-MOCK_IMPL(STATIC int,
-scheduler_compare_channels, (const void *c1_v, const void *c2_v))
-{
-  channel_t *c1 = NULL, *c2 = NULL;
-  /* These are a workaround for -Wbad-function-cast throwing a fit */
-  const circuitmux_policy_t *p1, *p2;
-  uintptr_t p1_i, p2_i;
-
-  tor_assert(c1_v);
-  tor_assert(c2_v);
-
-  c1 = (channel_t *)(c1_v);
-  c2 = (channel_t *)(c2_v);
-
-  tor_assert(c1);
-  tor_assert(c2);
-
-  if (c1 != c2) {
-    if (circuitmux_get_policy(c1->cmux) ==
-        circuitmux_get_policy(c2->cmux)) {
-      /* Same cmux policy, so use the mux comparison */
-      return circuitmux_compare_muxes(c1->cmux, c2->cmux);
-    } else {
-      /*
-       * Different policies; not important to get this edge case perfect
-       * because the current code never actually gives different channels
-       * different cmux policies anyway.  Just use this arbitrary but
-       * definite choice.
-       */
-      p1 = circuitmux_get_policy(c1->cmux);
-      p2 = circuitmux_get_policy(c2->cmux);
-      p1_i = (uintptr_t)p1;
-      p2_i = (uintptr_t)p2;
-
-      return (p1_i < p2_i) ? -1 : 1;
-    }
-  } else {
-    /* c1 == c2, so always equal */
-    return 0;
-  }
-}
-
-/*
- * Scheduler event callback; this should get triggered once per event loop
- * if any scheduling work was created during the event loop.
- */
-
-static void
-scheduler_evt_callback(evutil_socket_t fd, short events, void *arg)
-{
-  (void)fd;
-  (void)events;
-  (void)arg;
-  log_debug(LD_SCHED, "Scheduler event callback called");
-
-  tor_assert(run_sched_ev);
-
-  /* Run the scheduler */
-  scheduler_run();
-
-  /* Do we have more work to do? */
-  if (scheduler_more_work()) scheduler_retrigger();
-}
-
-/** Mark a channel as no longer ready to accept writes */
-
-MOCK_IMPL(void,
-scheduler_channel_doesnt_want_writes,(channel_t *chan))
-{
-  tor_assert(chan);
-
-  tor_assert(channels_pending);
-
-  /* If it's already in pending, we can put it in waiting_to_write */
-  if (chan->scheduler_state == SCHED_CHAN_PENDING) {
-    /*
-     * It's in channels_pending, so it shouldn't be in any of
-     * the other lists.  It can't write any more, so it goes to
-     * channels_waiting_to_write.
-     */
-    smartlist_pqueue_remove(channels_pending,
-                            scheduler_compare_channels,
-                            STRUCT_OFFSET(channel_t, sched_heap_idx),
-                            chan);
-    chan->scheduler_state = SCHED_CHAN_WAITING_TO_WRITE;
-    log_debug(LD_SCHED,
-              "Channel " U64_FORMAT " at %p went from pending "
-              "to waiting_to_write",
-              U64_PRINTF_ARG(chan->global_identifier), chan);
-  } else {
-    /*
-     * It's not in pending, so it can't become waiting_to_write; it's
-     * either not in any of the lists (nothing to do) or it's already in
-     * waiting_for_cells (remove it, can't write any more).
-     */
-    if (chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS) {
-      chan->scheduler_state = SCHED_CHAN_IDLE;
-      log_debug(LD_SCHED,
-                "Channel " U64_FORMAT " at %p left waiting_for_cells",
-                U64_PRINTF_ARG(chan->global_identifier), chan);
-    }
-  }
-}
-
-/** Mark a channel as having waiting cells */
-
-MOCK_IMPL(void,
-scheduler_channel_has_waiting_cells,(channel_t *chan))
-{
-  int became_pending = 0;
-
-  tor_assert(chan);
-  tor_assert(channels_pending);
-
-  /* First, check if this one also writeable */
-  if (chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS) {
-    /*
-     * It's in channels_waiting_for_cells, so it shouldn't be in any of
-     * the other lists.  It has waiting cells now, so it goes to
-     * channels_pending.
-     */
-    chan->scheduler_state = SCHED_CHAN_PENDING;
-    smartlist_pqueue_add(channels_pending,
-                         scheduler_compare_channels,
-                         STRUCT_OFFSET(channel_t, sched_heap_idx),
-                         chan);
-    log_debug(LD_SCHED,
-              "Channel " U64_FORMAT " at %p went from waiting_for_cells "
-              "to pending",
-              U64_PRINTF_ARG(chan->global_identifier), chan);
-    became_pending = 1;
-  } else {
-    /*
-     * It's not in waiting_for_cells, so it can't become pending; it's
-     * either not in any of the lists (we add it to waiting_to_write)
-     * or it's already in waiting_to_write or pending (we do nothing)
-     */
-    if (!(chan->scheduler_state == SCHED_CHAN_WAITING_TO_WRITE ||
-          chan->scheduler_state == SCHED_CHAN_PENDING)) {
-      chan->scheduler_state = SCHED_CHAN_WAITING_TO_WRITE;
-      log_debug(LD_SCHED,
-                "Channel " U64_FORMAT " at %p entered waiting_to_write",
-                U64_PRINTF_ARG(chan->global_identifier), chan);
-    }
-  }
-
-  /*
-   * If we made a channel pending, we potentially have scheduling work
-   * to do.
-   */
-  if (became_pending) scheduler_retrigger();
-}
-
-/** Set up the scheduling system */
-
-void
-scheduler_init(void)
-{
-  log_debug(LD_SCHED, "Initting scheduler");
-
-  tor_assert(!run_sched_ev);
-  run_sched_ev = tor_event_new(tor_libevent_get_base(), -1,
-                               0, scheduler_evt_callback, NULL);
-
-  channels_pending = smartlist_new();
-  queue_heuristic = 0;
-  queue_heuristic_timestamp = approx_time();
-}
-
-/** Check if there's more scheduling work */
-
-static int
-scheduler_more_work(void)
-{
-  tor_assert(channels_pending);
-
-  return ((scheduler_get_queue_heuristic() < sched_q_low_water) &&
-          ((smartlist_len(channels_pending) > 0))) ? 1 : 0;
-}
-
-/** Retrigger the scheduler in a way safe to use from the callback */
-
-static void
-scheduler_retrigger(void)
-{
-  tor_assert(run_sched_ev);
-  event_active(run_sched_ev, EV_TIMEOUT, 1);
-}
-
-/** Notify the scheduler of a channel being closed */
-
-MOCK_IMPL(void,
-scheduler_release_channel,(channel_t *chan))
-{
-  tor_assert(chan);
-  tor_assert(channels_pending);
-
-  if (chan->scheduler_state == SCHED_CHAN_PENDING) {
-    smartlist_pqueue_remove(channels_pending,
-                            scheduler_compare_channels,
-                            STRUCT_OFFSET(channel_t, sched_heap_idx),
-                            chan);
-  }
-
-  chan->scheduler_state = SCHED_CHAN_IDLE;
-}
-
-/** Run the scheduling algorithm if necessary */
-
-MOCK_IMPL(void,
-scheduler_run, (void))
-{
-  int n_cells, n_chans_before, n_chans_after;
-  uint64_t q_len_before, q_heur_before, q_len_after, q_heur_after;
-  ssize_t flushed, flushed_this_time;
-  smartlist_t *to_readd = NULL;
-  channel_t *chan = NULL;
-
-  log_debug(LD_SCHED, "We have a chance to run the scheduler");
-
-  if (scheduler_get_queue_heuristic() < sched_q_low_water) {
-    n_chans_before = smartlist_len(channels_pending);
-    q_len_before = channel_get_global_queue_estimate();
-    q_heur_before = scheduler_get_queue_heuristic();
-
-    while (scheduler_get_queue_heuristic() <= sched_q_high_water &&
-           smartlist_len(channels_pending) > 0) {
-      /* Pop off a channel */
-      chan = smartlist_pqueue_pop(channels_pending,
-                                  scheduler_compare_channels,
-                                  STRUCT_OFFSET(channel_t, sched_heap_idx));
-      tor_assert(chan);
-
-      /* Figure out how many cells we can write */
-      n_cells = channel_num_cells_writeable(chan);
-      if (n_cells > 0) {
-        log_debug(LD_SCHED,
-                  "Scheduler saw pending channel " U64_FORMAT " at %p with "
-                  "%d cells writeable",
-                  U64_PRINTF_ARG(chan->global_identifier), chan, n_cells);
-
-        flushed = 0;
-        while (flushed < n_cells &&
-               scheduler_get_queue_heuristic() <= sched_q_high_water) {
-          flushed_this_time =
-            channel_flush_some_cells(chan,
-                                     MIN(sched_max_flush_cells,
-                                         (size_t) n_cells - flushed));
-          if (flushed_this_time <= 0) break;
-          flushed += flushed_this_time;
-        }
-
-        if (flushed < n_cells) {
-          /* We ran out of cells to flush */
-          chan->scheduler_state = SCHED_CHAN_WAITING_FOR_CELLS;
-          log_debug(LD_SCHED,
-                    "Channel " U64_FORMAT " at %p "
-                    "entered waiting_for_cells from pending",
-                    U64_PRINTF_ARG(chan->global_identifier),
-                    chan);
-        } else {
-          /* The channel may still have some cells */
-          if (channel_more_to_flush(chan)) {
-          /* The channel goes to either pending or waiting_to_write */
-            if (channel_num_cells_writeable(chan) > 0) {
-              /* Add it back to pending later */
-              if (!to_readd) to_readd = smartlist_new();
-              smartlist_add(to_readd, chan);
-              log_debug(LD_SCHED,
-                        "Channel " U64_FORMAT " at %p "
-                        "is still pending",
-                        U64_PRINTF_ARG(chan->global_identifier),
-                        chan);
-            } else {
-              /* It's waiting to be able to write more */
-              chan->scheduler_state = SCHED_CHAN_WAITING_TO_WRITE;
-              log_debug(LD_SCHED,
-                        "Channel " U64_FORMAT " at %p "
-                        "entered waiting_to_write from pending",
-                        U64_PRINTF_ARG(chan->global_identifier),
-                        chan);
-            }
-          } else {
-            /* No cells left; it can go to idle or waiting_for_cells */
-            if (channel_num_cells_writeable(chan) > 0) {
-              /*
-               * It can still accept writes, so it goes to
-               * waiting_for_cells
-               */
-              chan->scheduler_state = SCHED_CHAN_WAITING_FOR_CELLS;
-              log_debug(LD_SCHED,
-                        "Channel " U64_FORMAT " at %p "
-                        "entered waiting_for_cells from pending",
-                        U64_PRINTF_ARG(chan->global_identifier),
-                        chan);
-            } else {
-              /*
-               * We exactly filled up the output queue with all available
-               * cells; go to idle.
-               */
-              chan->scheduler_state = SCHED_CHAN_IDLE;
-              log_debug(LD_SCHED,
-                        "Channel " U64_FORMAT " at %p "
-                        "become idle from pending",
-                        U64_PRINTF_ARG(chan->global_identifier),
-                        chan);
-            }
-          }
-        }
-
-        log_debug(LD_SCHED,
-                  "Scheduler flushed %d cells onto pending channel "
-                  U64_FORMAT " at %p",
-                  (int)flushed, U64_PRINTF_ARG(chan->global_identifier),
-                  chan);
-      } else {
-        log_info(LD_SCHED,
-                 "Scheduler saw pending channel " U64_FORMAT " at %p with "
-                 "no cells writeable",
-                 U64_PRINTF_ARG(chan->global_identifier), chan);
-        /* Put it back to WAITING_TO_WRITE */
-        chan->scheduler_state = SCHED_CHAN_WAITING_TO_WRITE;
-      }
-    }
-
-    /* Readd any channels we need to */
-    if (to_readd) {
-      SMARTLIST_FOREACH_BEGIN(to_readd, channel_t *, readd_chan) {
-        readd_chan->scheduler_state = SCHED_CHAN_PENDING;
-        smartlist_pqueue_add(channels_pending,
-                             scheduler_compare_channels,
-                             STRUCT_OFFSET(channel_t, sched_heap_idx),
-                             readd_chan);
-      } SMARTLIST_FOREACH_END(readd_chan);
-      smartlist_free(to_readd);
-    }
-
-    n_chans_after = smartlist_len(channels_pending);
-    q_len_after = channel_get_global_queue_estimate();
-    q_heur_after = scheduler_get_queue_heuristic();
-    log_debug(LD_SCHED,
-              "Scheduler handled %d of %d pending channels, queue size from "
-              U64_FORMAT " to " U64_FORMAT ", queue heuristic from "
-              U64_FORMAT " to " U64_FORMAT,
-              n_chans_before - n_chans_after, n_chans_before,
-              U64_PRINTF_ARG(q_len_before), U64_PRINTF_ARG(q_len_after),
-              U64_PRINTF_ARG(q_heur_before), U64_PRINTF_ARG(q_heur_after));
-  }
-}
-
-/** Trigger the scheduling event so we run the scheduler later */
-
-#if 0
-static void
-scheduler_trigger(void)
-{
-  log_debug(LD_SCHED, "Triggering scheduler event");
-
-  tor_assert(run_sched_ev);
-
-  event_add(run_sched_ev, EV_TIMEOUT, 1);
-}
-#endif
-
-/** Mark a channel as ready to accept writes */
-
-void
-scheduler_channel_wants_writes(channel_t *chan)
-{
-  int became_pending = 0;
-
-  tor_assert(chan);
-  tor_assert(channels_pending);
-
-  /* If it's already in waiting_to_write, we can put it in pending */
-  if (chan->scheduler_state == SCHED_CHAN_WAITING_TO_WRITE) {
-    /*
-     * It can write now, so it goes to channels_pending.
-     */
-    smartlist_pqueue_add(channels_pending,
-                         scheduler_compare_channels,
-                         STRUCT_OFFSET(channel_t, sched_heap_idx),
-                         chan);
-    chan->scheduler_state = SCHED_CHAN_PENDING;
-    log_debug(LD_SCHED,
-              "Channel " U64_FORMAT " at %p went from waiting_to_write "
-              "to pending",
-              U64_PRINTF_ARG(chan->global_identifier), chan);
-    became_pending = 1;
-  } else {
-    /*
-     * It's not in SCHED_CHAN_WAITING_TO_WRITE, so it can't become pending;
-     * it's either idle and goes to WAITING_FOR_CELLS, or it's a no-op.
-     */
-    if (!(chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS ||
-          chan->scheduler_state == SCHED_CHAN_PENDING)) {
-      chan->scheduler_state = SCHED_CHAN_WAITING_FOR_CELLS;
-      log_debug(LD_SCHED,
-                "Channel " U64_FORMAT " at %p entered waiting_for_cells",
-                U64_PRINTF_ARG(chan->global_identifier), chan);
-    }
-  }
-
-  /*
-   * If we made a channel pending, we potentially have scheduling work
-   * to do.
-   */
-  if (became_pending) scheduler_retrigger();
-}
-
-/**
- * Notify the scheduler that a channel's position in the pqueue may have
- * changed
- */
-
-void
-scheduler_touch_channel(channel_t *chan)
-{
-  tor_assert(chan);
-
-  if (chan->scheduler_state == SCHED_CHAN_PENDING) {
-    /* Remove and re-add it */
-    smartlist_pqueue_remove(channels_pending,
-                            scheduler_compare_channels,
-                            STRUCT_OFFSET(channel_t, sched_heap_idx),
-                            chan);
-    smartlist_pqueue_add(channels_pending,
-                         scheduler_compare_channels,
-                         STRUCT_OFFSET(channel_t, sched_heap_idx),
-                         chan);
-  }
-  /* else no-op, since it isn't in the queue */
-}
-
-/**
- * Notify the scheduler of a queue size adjustment, to recalculate the
- * queue heuristic.
- */
-
-void
-scheduler_adjust_queue_size(channel_t *chan, int dir, uint64_t adj)
-{
-  time_t now = approx_time();
-
-  log_debug(LD_SCHED,
-            "Queue size adjustment by %s" U64_FORMAT " for channel "
-            U64_FORMAT,
-            (dir >= 0) ? "+" : "-",
-            U64_PRINTF_ARG(adj),
-            U64_PRINTF_ARG(chan->global_identifier));
-
-  /* Get the queue heuristic up to date */
-  scheduler_update_queue_heuristic(now);
-
-  /* Adjust as appropriate */
-  if (dir >= 0) {
-    /* Increasing it */
-    queue_heuristic += adj;
-  } else {
-    /* Decreasing it */
-    if (queue_heuristic > adj) queue_heuristic -= adj;
-    else queue_heuristic = 0;
-  }
-
-  log_debug(LD_SCHED,
-            "Queue heuristic is now " U64_FORMAT,
-            U64_PRINTF_ARG(queue_heuristic));
-}
-
-/**
- * Query the current value of the queue heuristic
- */
-
-STATIC uint64_t
-scheduler_get_queue_heuristic(void)
-{
-  time_t now = approx_time();
-
-  scheduler_update_queue_heuristic(now);
-
-  return queue_heuristic;
-}
-
-/**
- * Adjust the queue heuristic value to the present time
- */
-
-STATIC void
-scheduler_update_queue_heuristic(time_t now)
-{
-  time_t diff;
-
-  if (queue_heuristic_timestamp == 0) {
-    /*
-     * Nothing we can sensibly do; must not have been initted properly.
-     * Oh well.
-     */
-    queue_heuristic_timestamp = now;
-  } else if (queue_heuristic_timestamp < now) {
-    diff = now - queue_heuristic_timestamp;
-    /*
-     * This is a simple exponential age-out; the other proposed alternative
-     * was a linear age-out using the bandwidth history in rephist.c; I'm
-     * going with this out of concern that if an adversary can jam the
-     * scheduler long enough, it would cause the bandwidth to drop to
-     * zero and render the aging mechanism ineffective thereafter.
-     */
-    if (0 <= diff && diff < 64) queue_heuristic >>= diff;
-    else queue_heuristic = 0;
-
-    queue_heuristic_timestamp = now;
-
-    log_debug(LD_SCHED,
-              "Queue heuristic is now " U64_FORMAT,
-              U64_PRINTF_ARG(queue_heuristic));
-  }
-  /* else no update needed, or time went backward */
-}
-
-/**
- * Set scheduler watermarks and flush size
- */
-
-void
-scheduler_set_watermarks(uint32_t lo, uint32_t hi, uint32_t max_flush)
-{
-  /* Sanity assertions - caller should ensure these are true */
-  tor_assert(lo > 0);
-  tor_assert(hi > lo);
-  tor_assert(max_flush > 0);
-
-  sched_q_low_water = lo;
-  sched_q_high_water = hi;
-  sched_max_flush_cells = max_flush;
-}
-
diff --git a/src/or/scheduler.h b/src/or/scheduler.h
deleted file mode 100644
index 3dcfd2faca..0000000000
--- a/src/or/scheduler.h
+++ /dev/null
@@ -1,57 +0,0 @@
-/* * Copyright (c) 2013-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file scheduler.h
- * \brief Header file for scheduler.c
- **/
-
-#ifndef TOR_SCHEDULER_H
-#define TOR_SCHEDULER_H
-
-#include "or.h"
-#include "channel.h"
-#include "testsupport.h"
-
-/* Global-visibility scheduler functions */
-
-/* Set up and shut down the scheduler from main.c */
-void scheduler_free_all(void);
-void scheduler_init(void);
-MOCK_DECL(void, scheduler_run, (void));
-
-/* Mark channels as having cells or wanting/not wanting writes */
-MOCK_DECL(void,scheduler_channel_doesnt_want_writes,(channel_t *chan));
-MOCK_DECL(void,scheduler_channel_has_waiting_cells,(channel_t *chan));
-void scheduler_channel_wants_writes(channel_t *chan);
-
-/* Notify the scheduler of a channel being closed */
-MOCK_DECL(void,scheduler_release_channel,(channel_t *chan));
-
-/* Notify scheduler of queue size adjustments */
-void scheduler_adjust_queue_size(channel_t *chan, int dir, uint64_t adj);
-
-/* Notify scheduler that a channel's queue position may have changed */
-void scheduler_touch_channel(channel_t *chan);
-
-/* Adjust the watermarks from config file*/
-void scheduler_set_watermarks(uint32_t lo, uint32_t hi, uint32_t max_flush);
-
-/* Things only scheduler.c and its test suite should see */
-
-#ifdef SCHEDULER_PRIVATE_
-MOCK_DECL(STATIC int, scheduler_compare_channels,
-          (const void *c1_v, const void *c2_v));
-STATIC uint64_t scheduler_get_queue_heuristic(void);
-STATIC void scheduler_update_queue_heuristic(time_t now);
-
-#ifdef TOR_UNIT_TESTS
-extern smartlist_t *channels_pending;
-extern struct event *run_sched_ev;
-extern uint64_t queue_heuristic;
-extern time_t queue_heuristic_timestamp;
-#endif
-#endif
-
-#endif /* !defined(TOR_SCHEDULER_H) */
-
diff --git a/src/or/torcert.c b/src/or/torcert.c
deleted file mode 100644
index a6a33c675a..0000000000
--- a/src/or/torcert.c
+++ /dev/null
@@ -1,297 +0,0 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file torcert.c
- *
- * \brief Implementation for ed25519-signed certificates as used in the Tor
- * protocol.
- */
-
-#include "crypto.h"
-#include "torcert.h"
-#include "ed25519_cert.h"
-#include "torlog.h"
-#include "util.h"
-#include "compat.h"
-#include "link_handshake.h"
-
-/** Helper for tor_cert_create(): signs any 32 bytes, not just an ed25519
- * key.
- */
-static tor_cert_t *
-tor_cert_sign_impl(const ed25519_keypair_t *signing_key,
-                      uint8_t cert_type,
-                      uint8_t signed_key_type,
-                      const uint8_t signed_key_info[32],
-                      time_t now, time_t lifetime,
-                      uint32_t flags)
-{
-  tor_cert_t *torcert = NULL;
-
-  ed25519_cert_t *cert = ed25519_cert_new();
-  cert->cert_type = cert_type;
-  cert->exp_field = (uint32_t) CEIL_DIV(now + lifetime, 3600);
-  cert->cert_key_type = signed_key_type;
-  memcpy(cert->certified_key, signed_key_info, 32);
-
-  if (flags & CERT_FLAG_INCLUDE_SIGNING_KEY) {
-    ed25519_cert_extension_t *ext = ed25519_cert_extension_new();
-    ext->ext_type = CERTEXT_SIGNED_WITH_KEY;
-    memcpy(ext->un_signing_key, signing_key->pubkey.pubkey, 32);
-    ed25519_cert_add_ext(cert, ext);
-    ++cert->n_extensions;
-  }
-
-  const ssize_t alloc_len = ed25519_cert_encoded_len(cert);
-  tor_assert(alloc_len > 0);
-  uint8_t *encoded = tor_malloc(alloc_len);
-  const ssize_t real_len = ed25519_cert_encode(encoded, alloc_len, cert);
-  if (real_len < 0)
-    goto err;
-  tor_assert(real_len == alloc_len);
-  tor_assert(real_len > ED25519_SIG_LEN);
-  uint8_t *sig = encoded + (real_len - ED25519_SIG_LEN);
-  tor_assert(tor_mem_is_zero((char*)sig, ED25519_SIG_LEN));
-
-  ed25519_signature_t signature;
-  if (ed25519_sign(&signature, encoded,
-                   real_len-ED25519_SIG_LEN, signing_key)<0) {
-    log_warn(LD_BUG, "Can't sign certificate");
-    goto err;
-  }
-  memcpy(sig, signature.sig, ED25519_SIG_LEN);
-
-  torcert = tor_cert_parse(encoded, real_len);
-  if (! torcert) {
-    log_warn(LD_BUG, "Generated a certificate we cannot parse");
-    goto err;
-  }
-
-  if (tor_cert_checksig(torcert, &signing_key->pubkey, now) < 0) {
-    log_warn(LD_BUG, "Generated a certificate whose signature we can't check");
-    goto err;
-  }
-
-  tor_free(encoded);
-
-  goto done;
-
- err:
-  tor_cert_free(torcert);
-  torcert = NULL;
- done:
-  ed25519_cert_free(cert);
-  tor_free(encoded);
-  return torcert;
-}
-
-/**
- * Create and return a new new certificate of type <b>cert_type</b> to
- * authenticate <b>signed_key</b> using the key <b>signing_key</b>.  The
- * certificate should remain valid for at least <b>lifetime</b> seconds after
- * <b>now</b>.
- *
- * If CERT_FLAG_INCLUDE_SIGNING_KEY is set in <b>flags</b>, embed
- * the public part of <b>signing_key</b> in the certificate.
- */
-tor_cert_t *
-tor_cert_create(const ed25519_keypair_t *signing_key,
-                uint8_t cert_type,
-                const ed25519_public_key_t *signed_key,
-                time_t now, time_t lifetime,
-                uint32_t flags)
-{
-  return tor_cert_sign_impl(signing_key, cert_type,
-                            SIGNED_KEY_TYPE_ED25519, signed_key->pubkey,
-                            now, lifetime, flags);
-}
-
-/** Release all storage held for <b>cert</b>. */
-void
-tor_cert_free(tor_cert_t *cert)
-{
-  if (! cert)
-    return;
-
-  if (cert->encoded)
-    memwipe(cert->encoded, 0, cert->encoded_len);
-  tor_free(cert->encoded);
-
-  memwipe(cert, 0, sizeof(tor_cert_t));
-  tor_free(cert);
-}
-
-/** Parse a certificate encoded with <b>len</b> bytes in <b>encoded</b>. */
-tor_cert_t *
-tor_cert_parse(const uint8_t *encoded, const size_t len)
-{
-  tor_cert_t *cert = NULL;
-  ed25519_cert_t *parsed = NULL;
-  ssize_t got_len = ed25519_cert_parse(&parsed, encoded, len);
-  if (got_len < 0 || (size_t) got_len != len)
-    goto err;
-
-  cert = tor_malloc_zero(sizeof(tor_cert_t));
-  cert->encoded = tor_memdup(encoded, len);
-  cert->encoded_len = len;
-
-  memcpy(cert->signed_key.pubkey, parsed->certified_key, 32);
-  cert->valid_until = parsed->exp_field * 3600;
-  cert->cert_type = parsed->cert_type;
-
-  for (unsigned i = 0; i < ed25519_cert_getlen_ext(parsed); ++i) {
-    ed25519_cert_extension_t *ext = ed25519_cert_get_ext(parsed, i);
-    if (ext->ext_type == CERTEXT_SIGNED_WITH_KEY) {
-      if (cert->signing_key_included)
-        goto err;
-
-      cert->signing_key_included = 1;
-      memcpy(cert->signing_key.pubkey, ext->un_signing_key, 32);
-    } else if (ext->ext_flags & CERTEXT_FLAG_AFFECTS_VALIDATION) {
-      /* Unrecognized extension with affects_validation set */
-      goto err;
-    }
-  }
-
-  goto done;
- err:
-  tor_cert_free(cert);
-  cert = NULL;
- done:
-  ed25519_cert_free(parsed);
-  return cert;
-}
-
-/** Fill in <b>checkable_out</b> with the information needed to check
- * the signature on <b>cert</b> with <b>pubkey</b>. */
-int
-tor_cert_get_checkable_sig(ed25519_checkable_t *checkable_out,
-                           const tor_cert_t *cert,
-                           const ed25519_public_key_t *pubkey)
-{
-  if (! pubkey) {
-    if (cert->signing_key_included)
-      pubkey = &cert->signing_key;
-    else
-      return -1;
-  }
-
-  checkable_out->msg = cert->encoded;
-  checkable_out->pubkey = pubkey;
-  tor_assert(cert->encoded_len > ED25519_SIG_LEN);
-  const size_t signed_len = cert->encoded_len - ED25519_SIG_LEN;
-  checkable_out->len = signed_len;
-  memcpy(checkable_out->signature.sig,
-         cert->encoded + signed_len, ED25519_SIG_LEN);
-
-  return 0;
-}
-
-/** Validates the signature on <b>cert</b> with <b>pubkey</b> relative to the
- * current time <b>now</b>.  (If <b>now</b> is 0, do not check the expiration
- * time.) Return 0 on success, -1 on failure.  Sets flags in <b>cert</b> as
- * appropriate.
- */
-int
-tor_cert_checksig(tor_cert_t *cert,
-                  const ed25519_public_key_t *pubkey, time_t now)
-{
-  ed25519_checkable_t checkable;
-  int okay;
-
-  if (now && now > cert->valid_until) {
-    cert->cert_expired = 1;
-    return -1;
-  }
-
-  if (tor_cert_get_checkable_sig(&checkable, cert, pubkey) < 0)
-    return -1;
-
-  if (ed25519_checksig_batch(&okay, &checkable, 1) < 0) {
-    cert->sig_bad = 1;
-    return -1;
-  } else {
-    cert->sig_ok = 1;
-    /* Only copy the checkable public key when it is different from the signing
-     * key of the certificate to avoid undefined behavior. */
-    if (cert->signing_key.pubkey != checkable.pubkey->pubkey) {
-      memcpy(cert->signing_key.pubkey, checkable.pubkey->pubkey, 32);
-    }
-    cert->cert_valid = 1;
-    return 0;
-  }
-}
-
-/** Return a new copy of <b>cert</b> */
-tor_cert_t *
-tor_cert_dup(const tor_cert_t *cert)
-{
-  tor_cert_t *newcert = tor_memdup(cert, sizeof(tor_cert_t));
-  if (cert->encoded)
-    newcert->encoded = tor_memdup(cert->encoded, cert->encoded_len);
-  return newcert;
-}
-
-/** Return true iff cert1 and cert2 are the same cert. */
-int
-tor_cert_eq(const tor_cert_t *cert1, const tor_cert_t *cert2)
-{
-  tor_assert(cert1);
-  tor_assert(cert2);
-  return cert1->encoded_len == cert2->encoded_len &&
-    tor_memeq(cert1->encoded, cert2->encoded, cert1->encoded_len);
-}
-
-/** Return true iff cert1 and cert2 are the same cert, or if they are both
- * NULL. */
-int
-tor_cert_opt_eq(const tor_cert_t *cert1, const tor_cert_t *cert2)
-{
-  if (cert1 == NULL && cert2 == NULL)
-    return 1;
-  if (!cert1 || !cert2)
-    return 0;
-  return tor_cert_eq(cert1, cert2);
-}
-
-/** Create new cross-certification object to certify <b>ed_key</b> as the
- * master ed25519 identity key for the RSA identity key <b>rsa_key</b>.
- * Allocates and stores the encoded certificate in *<b>cert</b>, and returns
- * the number of bytes stored. Returns negative on error.*/
-ssize_t
-tor_make_rsa_ed25519_crosscert(const ed25519_public_key_t *ed_key,
-                               const crypto_pk_t *rsa_key,
-                               time_t expires,
-                               uint8_t **cert)
-{
-  uint8_t *res;
-
-  rsa_ed_crosscert_t *cc = rsa_ed_crosscert_new();
-  memcpy(cc->ed_key, ed_key->pubkey, ED25519_PUBKEY_LEN);
-  cc->expiration = (uint32_t) CEIL_DIV(expires, 3600);
-  cc->sig_len = crypto_pk_keysize(rsa_key);
-  rsa_ed_crosscert_setlen_sig(cc, crypto_pk_keysize(rsa_key));
-
-  ssize_t alloc_sz = rsa_ed_crosscert_encoded_len(cc);
-  tor_assert(alloc_sz > 0);
-  res = tor_malloc_zero(alloc_sz);
-  ssize_t sz = rsa_ed_crosscert_encode(res, alloc_sz, cc);
-  tor_assert(sz > 0 && sz <= alloc_sz);
-
-  const int signed_part_len = 32 + 4;
-  int siglen = crypto_pk_private_sign(rsa_key,
-                                      (char*)rsa_ed_crosscert_getarray_sig(cc),
-                                      rsa_ed_crosscert_getlen_sig(cc),
-                                      (char*)res, signed_part_len);
-  tor_assert(siglen > 0 && siglen <= (int)crypto_pk_keysize(rsa_key));
-  tor_assert(siglen <= UINT8_MAX);
-  cc->sig_len = siglen;
-  rsa_ed_crosscert_setlen_sig(cc, siglen);
-
-  sz = rsa_ed_crosscert_encode(res, alloc_sz, cc);
-  rsa_ed_crosscert_free(cc);
-  *cert = res;
-  return sz;
-}
-
diff --git a/src/or/torcert.h b/src/or/torcert.h
deleted file mode 100644
index 9c819c0abb..0000000000
--- a/src/or/torcert.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#ifndef TORCERT_H_INCLUDED
-#define TORCERT_H_INCLUDED
-
-#include "crypto_ed25519.h"
-
-#define SIGNED_KEY_TYPE_ED25519 0x01
-
-#define CERT_TYPE_ID_SIGNING    0x04
-#define CERT_TYPE_SIGNING_LINK  0x05
-#define CERT_TYPE_SIGNING_AUTH  0x06
-#define CERT_TYPE_ONION_ID      0x0A
-
-#define CERT_FLAG_INCLUDE_SIGNING_KEY 0x1
-
-/** An ed25519-signed certificate as used throughout the Tor protocol.
- **/
-typedef struct tor_cert_st {
-  /** The key authenticated by this certificate */
-  ed25519_public_key_t signed_key;
-  /** The key that signed this certificate. This value may be unset if the
-   * certificate has never been checked, and didn't include its own key. */
-  ed25519_public_key_t signing_key;
-  /** A time after which this certificate will no longer be valid. */
-  time_t valid_until;
-
-  /** The encoded representation of this certificate */
-  uint8_t *encoded;
-  /** The length of <b>encoded</b> */
-  size_t encoded_len;
-
-  /** One of CERT_TYPE_... */
-  uint8_t cert_type;
-  /** True iff we received a signing key embedded in this certificate */
-  unsigned signing_key_included : 1;
-  /** True iff we checked the signature and found it bad */
-  unsigned sig_bad : 1;
-  /** True iff we checked the signature and found it correct */
-  unsigned sig_ok : 1;
-  /** True iff we checked the signature and first found that the cert
-   * had expired */
-  unsigned cert_expired : 1;
-  /** True iff we checked the signature and found the whole cert valid */
-  unsigned cert_valid : 1;
-} tor_cert_t;
-
-tor_cert_t *tor_cert_create(const ed25519_keypair_t *signing_key,
-                            uint8_t cert_type,
-                            const ed25519_public_key_t *signed_key,
-                            time_t now, time_t lifetime,
-                            uint32_t flags);
-
-tor_cert_t *tor_cert_parse(const uint8_t *cert, size_t certlen);
-
-void tor_cert_free(tor_cert_t *cert);
-
-int tor_cert_get_checkable_sig(ed25519_checkable_t *checkable_out,
-                                 const tor_cert_t *out,
-                                 const ed25519_public_key_t *pubkey);
-
-int tor_cert_checksig(tor_cert_t *cert,
-                      const ed25519_public_key_t *pubkey, time_t now);
-
-tor_cert_t *tor_cert_dup(const tor_cert_t *cert);
-int tor_cert_eq(const tor_cert_t *cert1, const tor_cert_t *cert2);
-int tor_cert_opt_eq(const tor_cert_t *cert1, const tor_cert_t *cert2);
-
-ssize_t tor_make_rsa_ed25519_crosscert(const ed25519_public_key_t *ed_key,
-                                       const crypto_pk_t *rsa_key,
-                                       time_t expires,
-                                       uint8_t **cert);
-
-#endif
-
diff --git a/src/rust/.cargo/config.in b/src/rust/.cargo/config.in
new file mode 100644
index 0000000000..6eddc75459
--- /dev/null
+++ b/src/rust/.cargo/config.in
@@ -0,0 +1,12 @@
+[source]
+
+@RUST_DL@ [source.crates-io]
+@RUST_DL@ registry = 'https://github.com/rust-lang/crates.io-index'
+@RUST_DL@ replace-with = 'vendored-sources'
+
+@RUST_DL@ [source.vendored-sources]
+@RUST_DL@ directory = '@TOR_RUST_DEPENDENCIES@'
+
+[build]
+@RUST_WARN@ rustflags = [ "-D", "warnings" ]
+@RUST_TARGET_PROP@
diff --git a/src/rust/.rustfmt.toml b/src/rust/.rustfmt.toml
new file mode 100644
index 0000000000..4ff839dcf3
--- /dev/null
+++ b/src/rust/.rustfmt.toml
@@ -0,0 +1,12 @@
+max_width = 100
+hard_tabs = false
+tab_spaces = 4
+newline_style = "Unix"
+#use_small_heuristics = "Default"
+reorder_imports = true
+reorder_modules = true
+remove_nested_parens = true
+merge_derives = true
+use_try_shorthand = false
+use_field_init_shorthand = false
+force_explicit_abi = true
diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock
new file mode 100644
index 0000000000..e2f24b0af7
--- /dev/null
+++ b/src/rust/Cargo.lock
@@ -0,0 +1,122 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+[[package]]
+name = "crypto"
+version = "0.0.1"
+dependencies = [
+ "digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "external 0.0.1",
+ "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.5.0-pre.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.2.0-pre.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smartlist 0.0.1",
+ "tor_allocate 0.0.1",
+ "tor_log 0.1.0",
+]
+
+[[package]]
+name = "digest"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "external"
+version = "0.0.1"
+dependencies = [
+ "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smartlist 0.0.1",
+ "tor_allocate 0.0.1",
+]
+
+[[package]]
+name = "generic-array"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "typenum 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.39"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "protover"
+version = "0.0.1"
+dependencies = [
+ "external 0.0.1",
+ "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smartlist 0.0.1",
+ "tor_allocate 0.0.1",
+ "tor_log 0.1.0",
+ "tor_util 0.0.1",
+]
+
+[[package]]
+name = "rand"
+version = "0.5.0-pre.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.2.0-pre.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.2.0-pre.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "smartlist"
+version = "0.0.1"
+dependencies = [
+ "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tor_allocate"
+version = "0.0.1"
+dependencies = [
+ "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tor_log"
+version = "0.1.0"
+dependencies = [
+ "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tor_allocate 0.0.1",
+]
+
+[[package]]
+name = "tor_rust"
+version = "0.1.0"
+dependencies = [
+ "protover 0.0.1",
+ "tor_util 0.0.1",
+]
+
+[[package]]
+name = "tor_util"
+version = "0.0.1"
+dependencies = [
+ "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tor_allocate 0.0.1",
+ "tor_log 0.1.0",
+]
+
+[[package]]
+name = "typenum"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[metadata]
+"checksum digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "00a49051fef47a72c9623101b19bd71924a45cca838826caae3eaa4d00772603"
+"checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d"
+"checksum libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)" = "f54263ad99207254cf58b5f701ecb432c717445ea2ee8af387334bdd1a03fdff"
+"checksum rand 0.5.0-pre.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3795e4701d9628a63a84d0289e66279883b40df165fca7caed7b87122447032a"
+"checksum rand_core 0.2.0-pre.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c7255ffbdb188d5be1a69b6f9f3cf187de4207430b9e79ed5b76458a6b20de9a"
+"checksum typenum 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "13a99dc6780ef33c78780b826cf9d2a78840b72cae9474de4bcaf9051e60ebbd"
diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml
new file mode 100644
index 0000000000..de8693ea33
--- /dev/null
+++ b/src/rust/Cargo.toml
@@ -0,0 +1,26 @@
+[workspace]
+members = [
+    "crypto",
+    "external",
+    "protover",
+    "smartlist",
+    "tor_allocate",
+    "tor_log",
+    "tor_rust",
+    "tor_util",
+]
+
+# Can remove panic="abort" when this issue is fixed:
+# https://github.com/rust-lang/rust/issues/52652
+[profile.dev]
+panic = "abort"
+
+[profile.release]
+debug = true
+panic = "abort"
+
+[profile.test]
+panic = "abort"
+
+[profile.bench]
+panic = "abort"
diff --git a/src/rust/build.rs b/src/rust/build.rs
new file mode 100644
index 0000000000..123d5c0682
--- /dev/null
+++ b/src/rust/build.rs
@@ -0,0 +1,190 @@
+//! Build script for Rust modules in Tor.
+//!
+//! We need to use this because some of our Rust tests need to use some
+//! of our C modules, which need to link some external libraries.
+//!
+//! This script works by looking at a "config.rust" file generated by our
+//! configure script, and then building a set of options for cargo to pass to
+//! the compiler.
+
+use std::collections::HashMap;
+use std::env;
+use std::fs::File;
+use std::io;
+use std::io::prelude::*;
+use std::path::PathBuf;
+
+/// Wrapper around a key-value map.
+struct Config(HashMap<String, String>);
+
+/// Locate a config.rust file generated by autoconf, starting in the OUT_DIR
+/// location provided by cargo and recursing up the directory tree.  Note that
+/// we need to look in the OUT_DIR, since autoconf will place generated files
+/// in the build directory.
+fn find_cfg() -> io::Result<String> {
+    let mut path = PathBuf::from(env::var("OUT_DIR").unwrap());
+    loop {
+        path.push("config.rust");
+        if path.exists() {
+            return Ok(path.to_str().unwrap().to_owned());
+        }
+        path.pop(); // remove config.rust
+        if !path.pop() {
+            // can't remove last part of directory
+            return Err(io::Error::new(io::ErrorKind::NotFound, "No config.rust"));
+        }
+    }
+}
+
+impl Config {
+    /// Find the config.rust file and try to parse it.
+    ///
+    /// The file format is a series of lines of the form KEY=VAL, with
+    /// any blank lines and lines starting with # ignored.
+    fn load() -> io::Result<Config> {
+        let path = find_cfg()?;
+        let f = File::open(&path)?;
+        let reader = io::BufReader::new(f);
+        let mut map = HashMap::new();
+        for line in reader.lines() {
+            let s = line?;
+            if s.trim().starts_with("#") || s.trim() == "" {
+                continue;
+            }
+            let idx = match s.find("=") {
+                None => {
+                    return Err(io::Error::new(io::ErrorKind::InvalidData, "missing ="));
+                }
+                Some(x) => x,
+            };
+            let (var, eq_val) = s.split_at(idx);
+            let val = &eq_val[1..];
+            map.insert(var.to_owned(), val.to_owned());
+        }
+        Ok(Config(map))
+    }
+
+    /// Return a reference to the value whose key is 'key'.
+    ///
+    /// Panics if 'key' is not found in the configuration.
+    fn get(&self, key: &str) -> &str {
+        self.0.get(key).unwrap()
+    }
+
+    /// Add a dependency on a static C library that is part of Tor, by name.
+    fn component(&self, s: &str) {
+        println!("cargo:rustc-link-lib=static={}", s);
+    }
+
+    /// Add a dependency on a native library that is not part of Tor, by name.
+    fn dependency(&self, s: &str) {
+        println!("cargo:rustc-link-lib={}", s);
+    }
+
+    /// Add a link path, relative to Tor's build directory.
+    fn link_relpath(&self, s: &str) {
+        let builddir = self.get("BUILDDIR");
+        println!("cargo:rustc-link-search=native={}/{}", builddir, s);
+    }
+
+    /// Add an absolute link path.
+    fn link_path(&self, s: &str) {
+        println!("cargo:rustc-link-search=native={}", s);
+    }
+
+    /// Parse the CFLAGS in s, looking for -l and -L items, and adding
+    /// rust configuration as appropriate.
+    fn from_cflags(&self, s: &str) {
+        let mut next_is_lib = false;
+        let mut next_is_path = false;
+        for ent in self.get(s).split_whitespace() {
+            if next_is_lib {
+                self.dependency(ent);
+                next_is_lib = false;
+            } else if next_is_path {
+                self.link_path(ent);
+                next_is_path = false;
+            } else if ent == "-l" {
+                next_is_lib = true;
+            } else if ent == "-L" {
+                next_is_path = true;
+            } else if ent.starts_with("-L") {
+                self.link_path(&ent[2..]);
+            } else if ent.starts_with("-l") {
+                self.dependency(&ent[2..]);
+            }
+        }
+    }
+}
+
+pub fn main() {
+    let cfg = Config::load().unwrap();
+    let package = env::var("CARGO_PKG_NAME").unwrap();
+
+    match package.as_ref() {
+        "crypto" => {
+            // Right now, I'm having a separate configuration for each Rust
+            // package, since I'm hoping we can trim them down.  Once we have a
+            // second Rust package that needs to use this build script, let's
+            // extract some of this stuff into a module.
+            //
+            // This is a ridiculous amount of code to be pulling in just
+            // to test our crypto library: modularity would be our
+            // friend here.
+            cfg.from_cflags("TOR_LDFLAGS_zlib");
+            cfg.from_cflags("TOR_LDFLAGS_openssl");
+            cfg.from_cflags("TOR_LDFLAGS_libevent");
+
+            cfg.link_relpath("src/lib");
+            cfg.link_relpath("src/ext/keccak-tiny");
+            cfg.link_relpath("src/ext/ed25519/ref10");
+            cfg.link_relpath("src/ext/ed25519/donna");
+            cfg.link_relpath("src/trunnel");
+
+            // Note that we can't pull in "libtor-testing", or else we
+            // will have dependencies on all the other rust packages that
+            // tor uses.  We must be careful with factoring and dependencies
+            // moving forward!
+            cfg.component("tor-crypt-ops-testing");
+            cfg.component("tor-sandbox-testing");
+            cfg.component("tor-encoding-testing");
+            cfg.component("tor-fs-testing");
+            cfg.component("tor-time-testing");
+            cfg.component("tor-net-testing");
+            cfg.component("tor-thread-testing");
+            cfg.component("tor-memarea-testing");
+            cfg.component("tor-log-testing");
+            cfg.component("tor-lock-testing");
+            cfg.component("tor-fdio-testing");
+            cfg.component("tor-container-testing");
+            cfg.component("tor-smartlist-core-testing");
+            cfg.component("tor-string-testing");
+            cfg.component("tor-malloc");
+            cfg.component("tor-wallclock");
+            cfg.component("tor-err-testing");
+            cfg.component("tor-intmath-testing");
+            cfg.component("tor-ctime-testing");
+            cfg.component("curve25519_donna");
+            cfg.component("keccak-tiny");
+            cfg.component("ed25519_ref10");
+            cfg.component("ed25519_donna");
+            cfg.component("or-trunnel-testing");
+
+            cfg.from_cflags("TOR_ZLIB_LIBS");
+            cfg.from_cflags("TOR_LIB_MATH");
+            cfg.from_cflags("NSS_LIBS");
+            cfg.from_cflags("TOR_OPENSSL_LIBS");
+            cfg.from_cflags("TOR_LIBEVENT_LIBS");
+            cfg.from_cflags("TOR_LIB_WS32");
+            cfg.from_cflags("TOR_LIB_GDI");
+            cfg.from_cflags("TOR_LIB_USERENV");
+            cfg.from_cflags("CURVE25519_LIBS");
+            cfg.from_cflags("TOR_LZMA_LIBS");
+            cfg.from_cflags("TOR_ZSTD_LIBS");
+            cfg.from_cflags("LIBS");
+        }
+        _ => {
+            panic!("No configuration in build.rs for package {}", package);
+        }
+    }
+}
diff --git a/src/rust/crypto/Cargo.toml b/src/rust/crypto/Cargo.toml
new file mode 100644
index 0000000000..a7ff7f78d9
--- /dev/null
+++ b/src/rust/crypto/Cargo.toml
@@ -0,0 +1,37 @@
+[package]
+authors = ["The Tor Project",
+           "Isis Lovecruft <isis@torproject.org>"]
+name = "crypto"
+version = "0.0.1"
+publish = false
+build = "../build.rs"
+
+[lib]
+name = "crypto"
+path = "lib.rs"
+
+[dependencies]
+libc = "=0.2.39"
+digest = "=0.7.2"
+rand_core = { version = "=0.2.0-pre.0", default-features = false }
+
+external = { path = "../external" }
+smartlist = { path = "../smartlist" }
+tor_allocate = { path = "../tor_allocate" }
+tor_log = { path = "../tor_log" }
+
+[dev-dependencies]
+rand = { version = "=0.5.0-pre.2", default-features = false }
+rand_core = { version = "=0.2.0-pre.0", default-features = false }
+
+[features]
+# If this feature is enabled, test code which calls Tor C code from Rust will
+# execute with `cargo test`.  Due to numerous linker issues (#25386), this is
+# currently disabled by default.
+test-c-from-rust = []
+
+# We have to define a feature here because doctests don't get cfg(test),
+# and we need to disable some C dependencies when running the doctests
+# because of the various linker issues.  See
+# https://github.com/rust-lang/rust/issues/45599
+test_linking_hack = []
diff --git a/src/rust/crypto/digests/mod.rs b/src/rust/crypto/digests/mod.rs
new file mode 100644
index 0000000000..58343b9ca7
--- /dev/null
+++ b/src/rust/crypto/digests/mod.rs
@@ -0,0 +1,7 @@
+// Copyright (c) 2018-2019, The Tor Project, Inc.
+// Copyright (c) 2018, isis agora lovecruft
+// See LICENSE for licensing information
+
+//! Hash Digests and eXtendible Output Functions (XOFs)
+
+pub mod sha2;
diff --git a/src/rust/crypto/digests/sha2.rs b/src/rust/crypto/digests/sha2.rs
new file mode 100644
index 0000000000..91e8b2b3c9
--- /dev/null
+++ b/src/rust/crypto/digests/sha2.rs
@@ -0,0 +1,234 @@
+// Copyright (c) 2018-2019, The Tor Project, Inc.
+// Copyright (c) 2018, isis agora lovecruft
+// See LICENSE for licensing information
+
+//! Hash Digests and eXtendible Output Functions (XOFs)
+
+pub use digest::Digest;
+
+use digest::generic_array::typenum::U32;
+use digest::generic_array::typenum::U64;
+use digest::generic_array::GenericArray;
+use digest::BlockInput;
+use digest::FixedOutput;
+use digest::Input;
+
+use external::crypto_digest::get_256_bit_digest;
+use external::crypto_digest::get_512_bit_digest;
+use external::crypto_digest::CryptoDigest;
+use external::crypto_digest::DigestAlgorithm;
+
+pub use external::crypto_digest::DIGEST256_LEN;
+pub use external::crypto_digest::DIGEST512_LEN;
+
+/// The block size for both SHA-256 and SHA-512 digests is 512 bits/64 bytes.
+///
+/// Unfortunately, we have to use the generic_array crate currently to express
+/// this at compile time.  Later, in the future, when Rust implements const
+/// generics, we'll be able to remove this dependency (actually, it will get
+/// removed from the digest crate, which is currently `pub use`ing it).
+type BlockSize = U64;
+
+/// A SHA2-256 digest.
+///
+/// # C_RUST_COUPLED
+///
+/// * `crypto_digest_dup`
+#[derive(Clone)]
+pub struct Sha256 {
+    engine: CryptoDigest,
+}
+
+/// Construct a new, default instance of a `Sha256` hash digest function.
+///
+/// # Examples
+///
+/// ```rust,no_run
+/// use crypto::digests::sha2::{Sha256, Digest};
+///
+/// let mut hasher: Sha256 = Sha256::default();
+/// ```
+///
+/// # Returns
+///
+/// A new `Sha256` digest.
+impl Default for Sha256 {
+    fn default() -> Sha256 {
+        Sha256 {
+            engine: CryptoDigest::new(Some(DigestAlgorithm::SHA2_256)),
+        }
+    }
+}
+
+impl BlockInput for Sha256 {
+    type BlockSize = BlockSize;
+}
+
+/// Input `msg` into the digest.
+///
+/// # Examples
+///
+/// ```rust,no_run
+/// use crypto::digests::sha2::{Sha256, Digest};
+///
+/// let mut hasher: Sha256 = Sha256::default();
+///
+/// hasher.input(b"foo");
+/// hasher.input(b"bar");
+/// ```
+impl Input for Sha256 {
+    fn process(&mut self, msg: &[u8]) {
+        self.engine.add_bytes(&msg);
+    }
+}
+
+/// Retrieve the output hash from everything which has been fed into this
+/// `Sha256` digest thus far.
+///
+//
+// FIXME: Once const generics land in Rust, we should genericise calling
+// crypto_digest_get_digest in external::crypto_digest.
+impl FixedOutput for Sha256 {
+    type OutputSize = U32;
+
+    fn fixed_result(self) -> GenericArray<u8, Self::OutputSize> {
+        let buffer: [u8; DIGEST256_LEN] = get_256_bit_digest(self.engine);
+
+        GenericArray::from(buffer)
+    }
+}
+
+/// A SHA2-512 digest.
+///
+/// # C_RUST_COUPLED
+///
+/// * `crypto_digest_dup`
+#[derive(Clone)]
+pub struct Sha512 {
+    engine: CryptoDigest,
+}
+
+/// Construct a new, default instance of a `Sha512` hash digest function.
+///
+/// # Examples
+///
+/// ```rust,no_run
+/// use crypto::digests::sha2::{Sha512, Digest};
+///
+/// let mut hasher: Sha512 = Sha512::default();
+/// ```
+///
+/// # Returns
+///
+/// A new `Sha512` digest.
+impl Default for Sha512 {
+    fn default() -> Sha512 {
+        Sha512 {
+            engine: CryptoDigest::new(Some(DigestAlgorithm::SHA2_512)),
+        }
+    }
+}
+
+impl BlockInput for Sha512 {
+    type BlockSize = BlockSize;
+}
+
+/// Input `msg` into the digest.
+///
+/// # Examples
+///
+/// ```rust,no_run
+/// use crypto::digests::sha2::{Sha512, Digest};
+///
+/// let mut hasher: Sha512 = Sha512::default();
+///
+/// hasher.input(b"foo");
+/// hasher.input(b"bar");
+/// ```
+impl Input for Sha512 {
+    fn process(&mut self, msg: &[u8]) {
+        self.engine.add_bytes(&msg);
+    }
+}
+
+/// Retrieve the output hash from everything which has been fed into this
+/// `Sha512` digest thus far.
+///
+//
+// FIXME: Once const generics land in Rust, we should genericise calling
+// crypto_digest_get_digest in external::crypto_digest.
+impl FixedOutput for Sha512 {
+    type OutputSize = U64;
+
+    fn fixed_result(self) -> GenericArray<u8, Self::OutputSize> {
+        let buffer: [u8; DIGEST512_LEN] = get_512_bit_digest(self.engine);
+
+        GenericArray::clone_from_slice(&buffer)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    #[cfg(feature = "test-c-from-rust")]
+    use digest::Digest;
+
+    #[cfg(feature = "test-c-from-rust")]
+    use super::*;
+
+    #[cfg(feature = "test-c-from-rust")]
+    #[test]
+    fn sha256_default() {
+        let _: Sha256 = Sha256::default();
+    }
+
+    #[cfg(feature = "test-c-from-rust")]
+    #[test]
+    fn sha256_digest() {
+        let mut h: Sha256 = Sha256::new();
+        let mut result: [u8; DIGEST256_LEN] = [0u8; DIGEST256_LEN];
+        let expected = [
+            151, 223, 53, 136, 181, 163, 242, 75, 171, 195, 133, 27, 55, 47, 11, 167, 26, 157, 205,
+            222, 212, 59, 20, 185, 208, 105, 97, 191, 193, 112, 125, 157,
+        ];
+
+        h.input(b"foo");
+        h.input(b"bar");
+        h.input(b"baz");
+
+        result.copy_from_slice(h.fixed_result().as_slice());
+
+        println!("{:?}", &result[..]);
+
+        assert_eq!(result, expected);
+    }
+
+    #[cfg(feature = "test-c-from-rust")]
+    #[test]
+    fn sha512_default() {
+        let _: Sha512 = Sha512::default();
+    }
+
+    #[cfg(feature = "test-c-from-rust")]
+    #[test]
+    fn sha512_digest() {
+        let mut h: Sha512 = Sha512::new();
+        let mut result: [u8; DIGEST512_LEN] = [0u8; DIGEST512_LEN];
+
+        let expected = [
+            203, 55, 124, 16, 176, 245, 166, 44, 128, 54, 37, 167, 153, 217, 233, 8, 190, 69, 231,
+            103, 245, 209, 71, 212, 116, 73, 7, 203, 5, 89, 122, 164, 237, 211, 41, 160, 175, 20,
+            122, 221, 12, 244, 24, 30, 211, 40, 250, 30, 121, 148, 38, 88, 38, 179, 237, 61, 126,
+            246, 240, 103, 202, 153, 24, 90,
+        ];
+
+        h.input(b"foo");
+        h.input(b"bar");
+        h.input(b"baz");
+
+        result.copy_from_slice(h.fixed_result().as_slice());
+
+        println!("{:?}", &result[..]);
+
+        assert_eq!(&result[..], &expected[..]);
+    }
+}
diff --git a/src/rust/crypto/lib.rs b/src/rust/crypto/lib.rs
new file mode 100644
index 0000000000..866ea93547
--- /dev/null
+++ b/src/rust/crypto/lib.rs
@@ -0,0 +1,46 @@
+// Copyright (c) 2018-2019, The Tor Project, Inc.
+// Copyright (c) 2018, isis agora lovecruft
+// See LICENSE for licensing information
+
+//! Common cryptographic functions and utilities.
+//!
+//! # Hash Digests and eXtendable Output Functions (XOFs)
+//!
+//! The `digests` module contains submodules for specific hash digests
+//! and extendable output functions.
+//!
+//! ```rust,no_run
+//! use crypto::digests::sha2::*;
+//!
+//! let mut hasher: Sha256 = Sha256::default();
+//! let mut result: [u8; 32] = [0u8; 32];
+//!
+//! hasher.input(b"foo");
+//! hasher.input(b"bar");
+//! hasher.input(b"baz");
+//!
+//! result.copy_from_slice(hasher.result().as_slice());
+//!
+//! assert!(result == [b'X'; DIGEST256_LEN]);
+//! ```
+
+// XXX: add missing docs
+//#![deny(missing_docs)]
+
+// External crates from cargo or TOR_RUST_DEPENDENCIES.
+extern crate digest;
+extern crate libc;
+extern crate rand_core;
+
+// External dependencies for tests.
+#[cfg(test)]
+extern crate rand as rand_crate;
+
+// Our local crates.
+extern crate external;
+#[cfg(not(test))]
+#[macro_use]
+extern crate tor_log;
+
+pub mod digests; // Unfortunately named "digests" plural to avoid name conflict with the digest crate
+pub mod rand;
diff --git a/src/rust/crypto/rand/mod.rs b/src/rust/crypto/rand/mod.rs
new file mode 100644
index 0000000000..da8b3bd8a5
--- /dev/null
+++ b/src/rust/crypto/rand/mod.rs
@@ -0,0 +1,6 @@
+// Copyright (c) 2018-2019, The Tor Project, Inc.
+// Copyright (c) 2018, isis agora lovecruft
+// See LICENSE for licensing information
+
+// Internal dependencies
+pub mod rng;
diff --git a/src/rust/crypto/rand/rng.rs b/src/rust/crypto/rand/rng.rs
new file mode 100644
index 0000000000..96e112799e
--- /dev/null
+++ b/src/rust/crypto/rand/rng.rs
@@ -0,0 +1,145 @@
+// Copyright (c) 2018-2019, The Tor Project, Inc.
+// Copyright (c) 2018, isis agora lovecruft
+// See LICENSE for licensing information
+
+//! Wrappers for Tor's random number generators to provide implementations of
+//! `rand_core` traits.
+
+// This is the real implementation, in use in production, which calls into our C
+// wrappers in /src/common/crypto_rand.c, which call into OpenSSL, system
+// libraries, and make syscalls.
+#[cfg(not(test))]
+mod internal {
+    use std::u64;
+
+    use rand_core::impls::next_u32_via_fill;
+    use rand_core::impls::next_u64_via_fill;
+    use rand_core::CryptoRng;
+    use rand_core::Error;
+    use rand_core::RngCore;
+
+    use external::c_tor_crypto_rand;
+    use external::c_tor_crypto_seed_rng;
+    use external::c_tor_crypto_strongest_rand;
+
+    use tor_log::LogDomain;
+    use tor_log::LogSeverity;
+
+    /// Largest strong entropy request permitted.
+    //
+    // C_RUST_COUPLED: `MAX_STRONGEST_RAND_SIZE` /src/common/crypto_rand.c
+    const MAX_STRONGEST_RAND_SIZE: usize = 256;
+
+    /// A wrapper around OpenSSL's RNG.
+    pub struct TorRng {
+        // This private, zero-length field forces the struct to be treated the
+        // same as its opaque C couterpart.
+        _unused: [u8; 0],
+    }
+
+    /// Mark `TorRng` as being suitable for cryptographic purposes.
+    impl CryptoRng for TorRng {}
+
+    impl TorRng {
+        // C_RUST_COUPLED: `crypto_seed_rng()` /src/common/crypto_rand.c
+        #[allow(dead_code)]
+        pub fn new() -> Self {
+            if !c_tor_crypto_seed_rng() {
+                tor_log_msg!(
+                    LogSeverity::Warn,
+                    LogDomain::General,
+                    "TorRng::from_seed()",
+                    "The RNG could not be seeded!"
+                );
+            }
+            // XXX also log success at info level —isis
+            TorRng { _unused: [0u8; 0] }
+        }
+    }
+
+    impl RngCore for TorRng {
+        // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c
+        fn next_u32(&mut self) -> u32 {
+            next_u32_via_fill(self)
+        }
+
+        // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c
+        fn next_u64(&mut self) -> u64 {
+            next_u64_via_fill(self)
+        }
+
+        // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c
+        fn fill_bytes(&mut self, dest: &mut [u8]) {
+            c_tor_crypto_rand(dest);
+        }
+
+        // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c
+        fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), Error> {
+            Ok(self.fill_bytes(dest))
+        }
+    }
+
+    /// A CSPRNG which hashes together randomness from OpenSSL's RNG and entropy
+    /// obtained from the operating system.
+    pub struct TorStrongestRng {
+        // This private, zero-length field forces the struct to be treated the
+        // same as its opaque C couterpart.
+        _unused: [u8; 0],
+    }
+
+    /// Mark `TorRng` as being suitable for cryptographic purposes.
+    impl CryptoRng for TorStrongestRng {}
+
+    impl TorStrongestRng {
+        // C_RUST_COUPLED: `crypto_seed_rng()` /src/common/crypto_rand.c
+        #[allow(dead_code)]
+        pub fn new() -> Self {
+            if !c_tor_crypto_seed_rng() {
+                tor_log_msg!(
+                    LogSeverity::Warn,
+                    LogDomain::General,
+                    "TorStrongestRng::from_seed()",
+                    "The RNG could not be seeded!"
+                );
+            }
+            // XXX also log success at info level —isis
+            TorStrongestRng { _unused: [0u8; 0] }
+        }
+    }
+
+    impl RngCore for TorStrongestRng {
+        // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c
+        fn next_u32(&mut self) -> u32 {
+            next_u32_via_fill(self)
+        }
+
+        // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c
+        fn next_u64(&mut self) -> u64 {
+            next_u64_via_fill(self)
+        }
+
+        // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c
+        fn fill_bytes(&mut self, dest: &mut [u8]) {
+            debug_assert!(dest.len() <= MAX_STRONGEST_RAND_SIZE);
+
+            c_tor_crypto_strongest_rand(dest);
+        }
+
+        // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c
+        fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), Error> {
+            Ok(self.fill_bytes(dest))
+        }
+    }
+}
+
+// For testing, we expose a pure-Rust implementation.
+#[cfg(test)]
+mod internal {
+    // It doesn't matter if we pretend ChaCha is a CSPRNG in tests.
+    pub use rand_crate::ChaChaRng as TorRng;
+    pub use rand_crate::ChaChaRng as TorStrongestRng;
+}
+
+// Finally, expose the public functionality of whichever appropriate internal
+// module.
+pub use self::internal::*;
diff --git a/src/rust/external/Cargo.toml b/src/rust/external/Cargo.toml
new file mode 100644
index 0000000000..5f443645bb
--- /dev/null
+++ b/src/rust/external/Cargo.toml
@@ -0,0 +1,20 @@
+[package]
+authors = ["The Tor Project"]
+version = "0.0.1"
+name = "external"
+
+[dependencies]
+libc = "=0.2.39"
+smartlist = { path = "../smartlist" }
+tor_allocate = { path = "../tor_allocate" }
+
+[lib]
+name = "external"
+path = "lib.rs"
+
+[features]
+# We have to define a feature here because doctests don't get cfg(test),
+# and we need to disable some C dependencies when running the doctests
+# because of the various linker issues.  See
+# https://github.com/rust-lang/rust/issues/45599
+test_linking_hack = []
diff --git a/src/rust/external/crypto_digest.rs b/src/rust/external/crypto_digest.rs
new file mode 100644
index 0000000000..454f836bad
--- /dev/null
+++ b/src/rust/external/crypto_digest.rs
@@ -0,0 +1,454 @@
+// Copyright (c) 2018-2019, The Tor Project, Inc.
+// Copyright (c) 2018, isis agora lovecruft
+// See LICENSE for licensing information
+
+//! Bindings to external digest and XOF functions which live within
+//! src/common/crypto_digest.[ch].
+//!
+//! We wrap our C implementations in src/common/crypto_digest.[ch] with more
+//! Rusty types and interfaces in src/rust/crypto/digest/.
+
+use std::process::abort;
+
+use libc::c_char;
+use libc::c_int;
+use libc::size_t;
+use libc::uint8_t;
+
+use smartlist::Stringlist;
+
+/// Length of the output of our message digest.
+pub const DIGEST_LEN: usize = 20;
+
+/// Length of the output of our second (improved) message digests.  (For now
+/// this is just sha256, but it could be any other 256-bit digest.)
+pub const DIGEST256_LEN: usize = 32;
+
+/// Length of the output of our 64-bit optimized message digests (SHA512).
+pub const DIGEST512_LEN: usize = 64;
+
+/// Length of a sha1 message digest when encoded in base32 with trailing = signs
+/// removed.
+pub const BASE32_DIGEST_LEN: usize = 32;
+
+/// Length of a sha1 message digest when encoded in base64 with trailing = signs
+/// removed.
+pub const BASE64_DIGEST_LEN: usize = 27;
+
+/// Length of a sha256 message digest when encoded in base64 with trailing =
+/// signs removed.
+pub const BASE64_DIGEST256_LEN: usize = 43;
+
+/// Length of a sha512 message digest when encoded in base64 with trailing =
+/// signs removed.
+pub const BASE64_DIGEST512_LEN: usize = 86;
+
+/// Length of hex encoding of SHA1 digest, not including final NUL.
+pub const HEX_DIGEST_LEN: usize = 40;
+
+/// Length of hex encoding of SHA256 digest, not including final NUL.
+pub const HEX_DIGEST256_LEN: usize = 64;
+
+/// Length of hex encoding of SHA512 digest, not including final NUL.
+pub const HEX_DIGEST512_LEN: usize = 128;
+
+/// Our C code uses an enum to declare the digest algorithm types which we know
+/// about.  However, because enums are implementation-defined in C, we can
+/// neither work with them directly nor translate them into Rust enums.
+/// Instead, we represent them as a u8 (under the assumption that we'll never
+/// support more than 256 hash functions).
+#[allow(non_camel_case_types)]
+type digest_algorithm_t = u8;
+
+const DIGEST_SHA1: digest_algorithm_t = 0;
+const DIGEST_SHA256: digest_algorithm_t = 1;
+const DIGEST_SHA512: digest_algorithm_t = 2;
+const DIGEST_SHA3_256: digest_algorithm_t = 3;
+const DIGEST_SHA3_512: digest_algorithm_t = 4;
+
+/// The number of hash digests we produce for a `common_digests_t`.
+///
+/// We can't access these from Rust, because their definitions in C require
+/// introspecting the `digest_algorithm_t` typedef, which is an enum, so we have
+/// to redefine them here.
+const N_COMMON_DIGEST_ALGORITHMS: usize = DIGEST_SHA256 as usize + 1;
+
+/// A digest function.
+#[repr(C)]
+#[derive(Debug, Copy, Clone)]
+#[allow(non_camel_case_types)]
+struct crypto_digest_t {
+    // This private, zero-length field forces the struct to be treated the same
+    // as its opaque C couterpart.
+    _unused: [u8; 0],
+}
+
+/// An eXtendible Output Function (XOF).
+#[repr(C)]
+#[derive(Debug, Copy, Clone)]
+#[allow(non_camel_case_types)]
+struct crypto_xof_t {
+    // This private, zero-length field forces the struct to be treated the same
+    // as its opaque C couterpart.
+    _unused: [u8; 0],
+}
+
+/// A set of all the digests we commonly compute, taken on a single
+/// string.  Any digests that are shorter than 512 bits are right-padded
+/// with 0 bits.
+///
+/// Note that this representation wastes 44 bytes for the SHA1 case, so
+/// don't use it for anything where we need to allocate a whole bunch at
+/// once.
+#[repr(C)]
+#[derive(Debug, Copy, Clone)]
+#[allow(non_camel_case_types)]
+struct common_digests_t {
+    pub d: [[c_char; N_COMMON_DIGEST_ALGORITHMS]; DIGEST256_LEN],
+}
+
+/// A `smartlist_t` is just an alias for the `#[repr(C)]` type `Stringlist`, to
+/// make it more clear that we're working with a smartlist which is owned by C.
+#[allow(non_camel_case_types)]
+// BINDGEN_GENERATED: This type isn't actually bindgen generated, but the code
+// below it which uses it is.  As such, this comes up as "dead code" as well.
+#[allow(dead_code)]
+type smartlist_t = Stringlist;
+
+/// All of the external functions from `src/common/crypto_digest.h`.
+///
+/// These are kept private because they should be wrapped with Rust to make their usage safer.
+//
+// BINDGEN_GENERATED: These definitions were generated with bindgen and cleaned
+// up manually.  As such, there are more bindings than are likely necessary or
+// which are in use.
+#[allow(dead_code)]
+extern "C" {
+    fn crypto_digest(digest: *mut c_char, m: *const c_char, len: size_t) -> c_int;
+    fn crypto_digest256(
+        digest: *mut c_char,
+        m: *const c_char,
+        len: size_t,
+        algorithm: digest_algorithm_t,
+    ) -> c_int;
+    fn crypto_digest512(
+        digest: *mut c_char,
+        m: *const c_char,
+        len: size_t,
+        algorithm: digest_algorithm_t,
+    ) -> c_int;
+    fn crypto_common_digests(ds_out: *mut common_digests_t, m: *const c_char, len: size_t)
+        -> c_int;
+    fn crypto_digest_smartlist_prefix(
+        digest_out: *mut c_char,
+        len_out: size_t,
+        prepend: *const c_char,
+        lst: *const smartlist_t,
+        append: *const c_char,
+        alg: digest_algorithm_t,
+    );
+    fn crypto_digest_smartlist(
+        digest_out: *mut c_char,
+        len_out: size_t,
+        lst: *const smartlist_t,
+        append: *const c_char,
+        alg: digest_algorithm_t,
+    );
+    fn crypto_digest_algorithm_get_name(alg: digest_algorithm_t) -> *const c_char;
+    fn crypto_digest_algorithm_get_length(alg: digest_algorithm_t) -> size_t;
+    fn crypto_digest_algorithm_parse_name(name: *const c_char) -> c_int;
+    fn crypto_digest_new() -> *mut crypto_digest_t;
+    fn crypto_digest256_new(algorithm: digest_algorithm_t) -> *mut crypto_digest_t;
+    fn crypto_digest512_new(algorithm: digest_algorithm_t) -> *mut crypto_digest_t;
+    fn crypto_digest_free_(digest: *mut crypto_digest_t);
+    fn crypto_digest_add_bytes(digest: *mut crypto_digest_t, data: *const c_char, len: size_t);
+    fn crypto_digest_get_digest(digest: *mut crypto_digest_t, out: *mut c_char, out_len: size_t);
+    fn crypto_digest_dup(digest: *const crypto_digest_t) -> *mut crypto_digest_t;
+    fn crypto_digest_assign(into: *mut crypto_digest_t, from: *const crypto_digest_t);
+    fn crypto_hmac_sha256(
+        hmac_out: *mut c_char,
+        key: *const c_char,
+        key_len: size_t,
+        msg: *const c_char,
+        msg_len: size_t,
+    );
+    fn crypto_mac_sha3_256(
+        mac_out: *mut uint8_t,
+        len_out: size_t,
+        key: *const uint8_t,
+        key_len: size_t,
+        msg: *const uint8_t,
+        msg_len: size_t,
+    );
+    fn crypto_xof_new() -> *mut crypto_xof_t;
+    fn crypto_xof_add_bytes(xof: *mut crypto_xof_t, data: *const uint8_t, len: size_t);
+    fn crypto_xof_squeeze_bytes(xof: *mut crypto_xof_t, out: *mut uint8_t, len: size_t);
+    fn crypto_xof_free(xof: *mut crypto_xof_t);
+}
+
+/// A wrapper around a `digest_algorithm_t`.
+pub enum DigestAlgorithm {
+    SHA2_256,
+    SHA2_512,
+    SHA3_256,
+    SHA3_512,
+}
+
+impl From<DigestAlgorithm> for digest_algorithm_t {
+    fn from(digest: DigestAlgorithm) -> digest_algorithm_t {
+        match digest {
+            DigestAlgorithm::SHA2_256 => DIGEST_SHA256,
+            DigestAlgorithm::SHA2_512 => DIGEST_SHA512,
+            DigestAlgorithm::SHA3_256 => DIGEST_SHA3_256,
+            DigestAlgorithm::SHA3_512 => DIGEST_SHA3_512,
+        }
+    }
+}
+
+/// A wrapper around a mutable pointer to a `crypto_digest_t`.
+pub struct CryptoDigest(*mut crypto_digest_t);
+
+/// Explicitly copy the state of a `CryptoDigest` hash digest context.
+///
+/// # C_RUST_COUPLED
+///
+/// * `crypto_digest_dup`
+impl Clone for CryptoDigest {
+    fn clone(&self) -> CryptoDigest {
+        let digest: *mut crypto_digest_t;
+
+        unsafe {
+            digest = crypto_digest_dup(self.0 as *const crypto_digest_t);
+        }
+
+        // See the note in the implementation of CryptoDigest for the
+        // reasoning for `abort()` here.
+        if digest.is_null() {
+            abort();
+        }
+
+        CryptoDigest(digest)
+    }
+}
+
+impl CryptoDigest {
+    /// A wrapper to call one of the C functions `crypto_digest_new`,
+    /// `crypto_digest256_new`, or `crypto_digest512_new`.
+    ///
+    /// # Warnings
+    ///
+    /// This function will `abort()` the entire process in an "abnormal" fashion,
+    /// i.e. not unwinding this or any other thread's stack, running any
+    /// destructors, or calling any panic/exit hooks) if `tor_malloc()` (called in
+    /// `crypto_digest256_new()`) is unable to allocate memory.
+    ///
+    /// # Returns
+    ///
+    /// A new `CryptoDigest`, which is a wrapper around a opaque representation
+    /// of a `crypto_digest_t`.  The underlying `crypto_digest_t` _MUST_ only
+    /// ever be handled via a raw pointer, and never introspected.
+    ///
+    /// # C_RUST_COUPLED
+    ///
+    /// * `crypto_digest_new`
+    /// * `crypto_digest256_new`
+    /// * `crypto_digest512_new`
+    /// * `tor_malloc` (called by `crypto_digest256_new`, but we make
+    ///    assumptions about its behvaiour and return values here)
+    pub fn new(algorithm: Option<DigestAlgorithm>) -> CryptoDigest {
+        let digest: *mut crypto_digest_t;
+
+        if algorithm.is_none() {
+            unsafe {
+                digest = crypto_digest_new();
+            }
+        } else {
+            let algo: digest_algorithm_t = algorithm.unwrap().into(); // can't fail because it's Some
+
+            unsafe {
+                // XXX This is a pretty awkward API to use from Rust...
+                digest = match algo {
+                    DIGEST_SHA1 => crypto_digest_new(),
+                    DIGEST_SHA256 => crypto_digest256_new(DIGEST_SHA256),
+                    DIGEST_SHA3_256 => crypto_digest256_new(DIGEST_SHA3_256),
+                    DIGEST_SHA512 => crypto_digest512_new(DIGEST_SHA512),
+                    DIGEST_SHA3_512 => crypto_digest512_new(DIGEST_SHA3_512),
+                    _ => abort(),
+                }
+            }
+        }
+
+        // In our C code, `crypto_digest*_new()` allocates memory with
+        // `tor_malloc()`.  In `tor_malloc()`, if the underlying malloc
+        // implementation fails to allocate the requested memory and returns a
+        // NULL pointer, we call `exit(1)`.  In the case that this `exit(1)` is
+        // called within a worker, be that a process or a thread, the inline
+        // comments within `tor_malloc()` mention "that's ok, since the parent
+        // will run out of memory soon anyway".  However, if it takes long
+        // enough for the worker to die, and it manages to return a NULL pointer
+        // to our Rust code, our Rust is now in an irreparably broken state and
+        // may exhibit undefined behaviour.  An even worse scenario, if/when we
+        // have parent/child processes/threads controlled by Rust, would be that
+        // the UB contagion in Rust manages to spread to other children before
+        // the entire process (hopefully terminates).
+        //
+        // However, following the assumptions made in `tor_malloc()` that
+        // calling `exit(1)` in a child is okay because the parent will
+        // eventually run into the same errors, and also to stymie any UB
+        // contagion in the meantime, we call abort!() here to terminate the
+        // entire program immediately.
+        if digest.is_null() {
+            abort();
+        }
+
+        CryptoDigest(digest)
+    }
+
+    /// A wrapper to call the C function `crypto_digest_add_bytes`.
+    ///
+    /// # Inputs
+    ///
+    /// * `bytes`: a byte slice of bytes to be added into this digest.
+    ///
+    /// # C_RUST_COUPLED
+    ///
+    /// * `crypto_digest_add_bytes`
+    pub fn add_bytes(&self, bytes: &[u8]) {
+        unsafe {
+            crypto_digest_add_bytes(
+                self.0 as *mut crypto_digest_t,
+                bytes.as_ptr() as *const c_char,
+                bytes.len() as size_t,
+            )
+        }
+    }
+}
+
+impl Drop for CryptoDigest {
+    fn drop(&mut self) {
+        unsafe {
+            crypto_digest_free_(self.0 as *mut crypto_digest_t);
+        }
+    }
+}
+
+/// Get the 256-bit digest output of a `crypto_digest_t`.
+///
+/// # Inputs
+///
+/// * `digest`: A `CryptoDigest` which wraps either a `DIGEST_SHA256` or a
+///   `DIGEST_SHA3_256`.
+///
+/// # Warning
+///
+/// Calling this function with a `CryptoDigest` which is neither SHA2-256 or
+/// SHA3-256 is a programming error.  Since we cannot introspect the opaque
+/// struct from Rust, however, there is no way for us to check that the correct
+/// one is being passed in.  That is up to you, dear programmer.  If you mess
+/// up, you will get a incorrectly-sized hash digest in return, and it will be
+/// your fault.  Don't do that.
+///
+/// # Returns
+///
+/// A 256-bit hash digest, as a `[u8; 32]`.
+///
+/// # C_RUST_COUPLED
+///
+/// * `crypto_digest_get_digest`
+/// * `DIGEST256_LEN`
+//
+// FIXME: Once const generics land in Rust, we should genericise calling
+// crypto_digest_get_digest w.r.t. output array size.
+pub fn get_256_bit_digest(digest: CryptoDigest) -> [u8; DIGEST256_LEN] {
+    let mut buffer: [u8; DIGEST256_LEN] = [0u8; DIGEST256_LEN];
+
+    unsafe {
+        crypto_digest_get_digest(
+            digest.0,
+            buffer.as_mut_ptr() as *mut c_char,
+            DIGEST256_LEN as size_t,
+        );
+
+        if buffer.as_ptr().is_null() {
+            abort();
+        }
+    }
+    buffer
+}
+
+/// Get the 512-bit digest output of a `crypto_digest_t`.
+///
+/// # Inputs
+///
+/// * `digest`: A `CryptoDigest` which wraps either a `DIGEST_SHA512` or a
+///   `DIGEST_SHA3_512`.
+///
+/// # Warning
+///
+/// Calling this function with a `CryptoDigest` which is neither SHA2-512 or
+/// SHA3-512 is a programming error.  Since we cannot introspect the opaque
+/// struct from Rust, however, there is no way for us to check that the correct
+/// one is being passed in.  That is up to you, dear programmer.  If you mess
+/// up, you will get a incorrectly-sized hash digest in return, and it will be
+/// your fault.  Don't do that.
+///
+/// # Returns
+///
+/// A 512-bit hash digest, as a `[u8; 64]`.
+///
+/// # C_RUST_COUPLED
+///
+/// * `crypto_digest_get_digest`
+/// * `DIGEST512_LEN`
+//
+// FIXME: Once const generics land in Rust, we should genericise calling
+// crypto_digest_get_digest w.r.t. output array size.
+pub fn get_512_bit_digest(digest: CryptoDigest) -> [u8; DIGEST512_LEN] {
+    let mut buffer: [u8; DIGEST512_LEN] = [0u8; DIGEST512_LEN];
+
+    unsafe {
+        crypto_digest_get_digest(
+            digest.0,
+            buffer.as_mut_ptr() as *mut c_char,
+            DIGEST512_LEN as size_t,
+        );
+
+        if buffer.as_ptr().is_null() {
+            abort();
+        }
+    }
+    buffer
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_layout_common_digests_t() {
+        assert_eq!(
+            ::std::mem::size_of::<common_digests_t>(),
+            64usize,
+            concat!("Size of: ", stringify!(common_digests_t))
+        );
+        assert_eq!(
+            ::std::mem::align_of::<common_digests_t>(),
+            1usize,
+            concat!("Alignment of ", stringify!(common_digests_t))
+        );
+    }
+
+    #[test]
+    fn test_layout_crypto_digest_t() {
+        assert_eq!(
+            ::std::mem::size_of::<crypto_digest_t>(),
+            0usize,
+            concat!("Size of: ", stringify!(crypto_digest_t))
+        );
+        assert_eq!(
+            ::std::mem::align_of::<crypto_digest_t>(),
+            1usize,
+            concat!("Alignment of ", stringify!(crypto_digest_t))
+        );
+    }
+}
diff --git a/src/rust/external/crypto_rand.rs b/src/rust/external/crypto_rand.rs
new file mode 100644
index 0000000000..703382093c
--- /dev/null
+++ b/src/rust/external/crypto_rand.rs
@@ -0,0 +1,84 @@
+// Copyright (c) 2018-2019, The Tor Project, Inc.
+// Copyright (c) 2018, isis agora lovecruft
+// See LICENSE for licensing information
+
+//! Bindings to external (P)RNG interfaces and utilities in
+//! src/common/crypto_rand.[ch].
+//!
+//! We wrap our C implementations in src/common/crypto_rand.[ch] here in order
+//! to provide wrappers with native Rust types, and then provide more Rusty
+//! types and and trait implementations in src/rust/crypto/rand/.
+
+use std::time::Duration;
+
+use libc::c_double;
+use libc::c_int;
+use libc::size_t;
+use libc::time_t;
+use libc::uint8_t;
+
+extern "C" {
+    fn crypto_seed_rng() -> c_int;
+    fn crypto_rand(out: *mut uint8_t, out_len: size_t);
+    fn crypto_strongest_rand(out: *mut uint8_t, out_len: size_t);
+    fn crypto_rand_time_range(min: time_t, max: time_t) -> time_t;
+    fn crypto_rand_double() -> c_double;
+}
+
+/// Seed OpenSSL's random number generator with bytes from the operating
+/// system.
+///
+/// # Returns
+///
+/// `true` on success; `false` on failure.
+pub fn c_tor_crypto_seed_rng() -> bool {
+    let ret: c_int;
+
+    unsafe {
+        ret = crypto_seed_rng();
+    }
+    match ret {
+        0 => return true,
+        _ => return false,
+    }
+}
+
+/// Fill the bytes of `dest` with random data.
+pub fn c_tor_crypto_rand(dest: &mut [u8]) {
+    unsafe {
+        crypto_rand(dest.as_mut_ptr(), dest.len() as size_t);
+    }
+}
+
+/// Fill the bytes of `dest` with "strong" random data by hashing
+/// together randomness obtained from OpenSSL's RNG and the operating
+/// system.
+pub fn c_tor_crypto_strongest_rand(dest: &mut [u8]) {
+    // We'll let the C side panic if the len is larger than
+    // MAX_STRONGEST_RAND_SIZE, rather than potentially panicking here.  A
+    // paranoid caller should assert on the length of dest *before* calling this
+    // function.
+    unsafe {
+        crypto_strongest_rand(dest.as_mut_ptr(), dest.len() as size_t);
+    }
+}
+
+/// Get a random time, in seconds since the Unix Epoch.
+///
+/// # Returns
+///
+/// A `std::time::Duration` of seconds since the Unix Epoch.
+pub fn c_tor_crypto_rand_time_range(min: &Duration, max: &Duration) -> Duration {
+    let ret: time_t;
+
+    unsafe {
+        ret = crypto_rand_time_range(min.as_secs() as time_t, max.as_secs() as time_t);
+    }
+
+    Duration::from_secs(ret as u64)
+}
+
+/// Return a pseudorandom 64-bit float, chosen uniformly from the range [0.0, 1.0).
+pub fn c_tor_crypto_rand_double() -> f64 {
+    unsafe { crypto_rand_double() }
+}
diff --git a/src/rust/external/external.rs b/src/rust/external/external.rs
new file mode 100644
index 0000000000..0d324c8820
--- /dev/null
+++ b/src/rust/external/external.rs
@@ -0,0 +1,37 @@
+// Copyright (c) 2016-2019, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+use libc::{c_char, c_int};
+use std::ffi::CString;
+
+extern "C" {
+    fn tor_version_as_new_as(platform: *const c_char, cutoff: *const c_char) -> c_int;
+}
+
+/// Wrap calls to tor_version_as_new_as, defined in routerparse.c
+pub fn c_tor_version_as_new_as(platform: &str, cutoff: &str) -> bool {
+    // CHK: These functions should log a warning if an error occurs. This
+    // can be added when integration with tor's logger is added to rust
+    let c_platform = match CString::new(platform) {
+        Ok(n) => n,
+        Err(_) => return false,
+    };
+
+    let c_cutoff = match CString::new(cutoff) {
+        Ok(n) => n,
+        Err(_) => return false,
+    };
+
+    let result: c_int = unsafe { tor_version_as_new_as(c_platform.as_ptr(), c_cutoff.as_ptr()) };
+
+    result == 1
+}
+
+extern "C" {
+    fn tor_is_using_nss() -> c_int;
+}
+
+/// Return true if Tor was built to use NSS.
+pub fn c_tor_is_using_nss() -> bool {
+    0 != unsafe { tor_is_using_nss() }
+}
diff --git a/src/rust/external/lib.rs b/src/rust/external/lib.rs
new file mode 100644
index 0000000000..2f50610a4d
--- /dev/null
+++ b/src/rust/external/lib.rs
@@ -0,0 +1,19 @@
+//! Copyright (c) 2016-2019, The Tor Project, Inc. */
+//! See LICENSE for licensing information */
+
+//! Interface for external calls to tor C ABI
+//!
+//! The purpose of this module is to provide a clean interface for when Rust
+//! modules need to interact with functionality in tor C code rather than each
+//! module implementing this functionality repeatedly.
+
+extern crate libc;
+extern crate tor_allocate;
+extern crate smartlist;
+
+pub mod crypto_digest;
+mod crypto_rand;
+mod external;
+
+pub use crypto_rand::*;
+pub use external::*;
diff --git a/src/rust/include.am b/src/rust/include.am
new file mode 100644
index 0000000000..5e5b0b3faf
--- /dev/null
+++ b/src/rust/include.am
@@ -0,0 +1,41 @@
+include src/rust/tor_rust/include.am
+
+EXTRA_DIST +=\
+	src/rust/build.rs \
+	src/rust/Cargo.toml \
+	src/rust/Cargo.lock \
+	src/rust/.cargo/config.in \
+	src/rust/crypto/Cargo.toml \
+	src/rust/crypto/lib.rs \
+	src/rust/crypto/digests/mod.rs \
+	src/rust/crypto/digests/sha2.rs \
+	src/rust/crypto/rand/mod.rs \
+	src/rust/crypto/rand/rng.rs \
+	src/rust/external/Cargo.toml \
+	src/rust/external/crypto_digest.rs \
+	src/rust/external/crypto_rand.rs \
+	src/rust/external/external.rs \
+	src/rust/external/lib.rs \
+	src/rust/protover/Cargo.toml \
+	src/rust/protover/errors.rs \
+	src/rust/protover/protoset.rs \
+	src/rust/protover/ffi.rs \
+	src/rust/protover/lib.rs \
+	src/rust/protover/protover.rs \
+	src/rust/protover/tests/protover.rs \
+	src/rust/smartlist/Cargo.toml \
+	src/rust/smartlist/lib.rs \
+	src/rust/smartlist/smartlist.rs \
+	src/rust/tor_allocate/Cargo.toml \
+	src/rust/tor_allocate/lib.rs \
+	src/rust/tor_allocate/tor_allocate.rs \
+	src/rust/tor_log/Cargo.toml \
+	src/rust/tor_log/lib.rs \
+	src/rust/tor_log/tor_log.rs \
+	src/rust/tor_rust/Cargo.toml \
+	src/rust/tor_rust/include.am \
+	src/rust/tor_rust/lib.rs \
+	src/rust/tor_util/Cargo.toml \
+	src/rust/tor_util/ffi.rs \
+	src/rust/tor_util/lib.rs \
+	src/rust/tor_util/strings.rs
diff --git a/src/rust/protover/Cargo.toml b/src/rust/protover/Cargo.toml
new file mode 100644
index 0000000000..84a7c71c1a
--- /dev/null
+++ b/src/rust/protover/Cargo.toml
@@ -0,0 +1,33 @@
+[package]
+authors = ["The Tor Project"]
+version = "0.0.1"
+name = "protover"
+
+[features]
+# We have to define a feature here because doctests don't get cfg(test),
+# and we need to disable some C dependencies when running the doctests
+# because of the various linker issues.  See
+# https://github.com/rust-lang/rust/issues/45599
+test_linking_hack = []
+
+[dependencies]
+libc = "=0.2.39"
+
+[dependencies.smartlist]
+path = "../smartlist"
+
+[dependencies.external]
+path = "../external"
+
+[dependencies.tor_util]
+path = "../tor_util"
+
+[dependencies.tor_allocate]
+path = "../tor_allocate"
+
+[dependencies.tor_log]
+path = "../tor_log"
+
+[lib]
+name = "protover"
+path = "lib.rs"
diff --git a/src/rust/protover/errors.rs b/src/rust/protover/errors.rs
new file mode 100644
index 0000000000..dc0d8735f4
--- /dev/null
+++ b/src/rust/protover/errors.rs
@@ -0,0 +1,57 @@
+// Copyright (c) 2018-2019, The Tor Project, Inc.
+// Copyright (c) 2018, isis agora lovecruft
+// See LICENSE for licensing information
+
+//! Various errors which may occur during protocol version parsing.
+
+use std::fmt;
+use std::fmt::Display;
+
+/// All errors which may occur during protover parsing routines.
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]
+#[allow(missing_docs)] // See Display impl for error descriptions
+pub enum ProtoverError {
+    Overlap,
+    LowGreaterThanHigh,
+    Unparseable,
+    ExceedsMax,
+    ExceedsExpansionLimit,
+    UnknownProtocol,
+    ExceedsNameLimit,
+    InvalidProtocol,
+}
+
+/// Descriptive error messages for `ProtoverError` variants.
+impl Display for ProtoverError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            ProtoverError::Overlap => write!(
+                f,
+                "Two or more (low, high) protover ranges would overlap once expanded."
+            ),
+            ProtoverError::LowGreaterThanHigh => write!(
+                f,
+                "The low in a (low, high) protover range was greater than high."
+            ),
+            ProtoverError::Unparseable => write!(f, "The protover string was unparseable."),
+            ProtoverError::ExceedsMax => write!(
+                f,
+                "The high in a (low, high) protover range exceeds u32::MAX."
+            ),
+            ProtoverError::ExceedsExpansionLimit => write!(
+                f,
+                "The protover string would exceed the maximum expansion limit."
+            ),
+            ProtoverError::UnknownProtocol => write!(
+                f,
+                "A protocol in the protover string we attempted to parse is unknown."
+            ),
+            ProtoverError::ExceedsNameLimit => {
+                write!(f, "An unrecognised protocol name was too long.")
+            }
+            ProtoverError::InvalidProtocol => {
+                write!(f, "A protocol name includes invalid characters.")
+            }
+        }
+    }
+}
diff --git a/src/rust/protover/ffi.rs b/src/rust/protover/ffi.rs
new file mode 100644
index 0000000000..6ee63adb10
--- /dev/null
+++ b/src/rust/protover/ffi.rs
@@ -0,0 +1,245 @@
+// Copyright (c) 2016-2019, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+//! FFI functions, only to be called from C.
+//!
+//! Equivalent C versions of this api are in `protover.c`
+
+use libc::{c_char, c_int, uint32_t};
+use std::ffi::CStr;
+
+use smartlist::*;
+use tor_allocate::allocate_and_copy_string;
+
+use errors::ProtoverError;
+use protover::*;
+
+/// Translate C enums to Rust Proto enums, using the integer value of the C
+/// enum to map to its associated Rust enum.
+///
+/// C_RUST_COUPLED: protover.h `protocol_type_t`
+fn translate_to_rust(c_proto: uint32_t) -> Result<Protocol, ProtoverError> {
+    match c_proto {
+        0 => Ok(Protocol::Link),
+        1 => Ok(Protocol::LinkAuth),
+        2 => Ok(Protocol::Relay),
+        3 => Ok(Protocol::DirCache),
+        4 => Ok(Protocol::HSDir),
+        5 => Ok(Protocol::HSIntro),
+        6 => Ok(Protocol::HSRend),
+        7 => Ok(Protocol::Desc),
+        8 => Ok(Protocol::Microdesc),
+        9 => Ok(Protocol::Cons),
+        _ => Err(ProtoverError::UnknownProtocol),
+    }
+}
+
+/// Provide an interface for C to translate arguments and return types for
+/// protover::all_supported
+#[no_mangle]
+pub extern "C" fn protover_all_supported(
+    c_relay_version: *const c_char,
+    missing_out: *mut *mut c_char,
+) -> c_int {
+    if c_relay_version.is_null() {
+        return 1;
+    }
+
+    // Require an unsafe block to read the version from a C string. The pointer
+    // is checked above to ensure it is not null.
+    let c_str: &CStr = unsafe { CStr::from_ptr(c_relay_version) };
+
+    let relay_version = match c_str.to_str() {
+        Ok(n) => n,
+        Err(_) => return 1,
+    };
+
+    let relay_proto_entry: UnvalidatedProtoEntry =
+        match UnvalidatedProtoEntry::from_str_any_len(relay_version) {
+            Ok(n) => n,
+            Err(_) => return 1,
+        };
+
+    if let Some(unsupported) = relay_proto_entry.all_supported() {
+        if missing_out.is_null() {
+            return 0;
+        }
+        let ptr = allocate_and_copy_string(&unsupported.to_string());
+        unsafe { *missing_out = ptr };
+
+        return 0;
+    }
+
+    1
+}
+
+/// Provide an interface for C to translate arguments and return types for
+/// protover::list_supports_protocol
+#[no_mangle]
+pub extern "C" fn protocol_list_supports_protocol(
+    c_protocol_list: *const c_char,
+    c_protocol: uint32_t,
+    version: uint32_t,
+) -> c_int {
+    if c_protocol_list.is_null() {
+        return 1;
+    }
+
+    // Require an unsafe block to read the version from a C string. The pointer
+    // is checked above to ensure it is not null.
+    let c_str: &CStr = unsafe { CStr::from_ptr(c_protocol_list) };
+
+    let protocol_list = match c_str.to_str() {
+        Ok(n) => n,
+        Err(_) => return 1,
+    };
+    let proto_entry: UnvalidatedProtoEntry = match protocol_list.parse() {
+        Ok(n) => n,
+        Err(_) => return 0,
+    };
+    let protocol: UnknownProtocol = match translate_to_rust(c_protocol) {
+        Ok(n) => n.into(),
+        Err(_) => return 0,
+    };
+    if proto_entry.supports_protocol(&protocol, &version) {
+        1
+    } else {
+        0
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn protover_contains_long_protocol_names_(c_protocol_list: *const c_char) -> c_int {
+    if c_protocol_list.is_null() {
+        return 1;
+    }
+
+    // Require an unsafe block to read the version from a C string. The pointer
+    // is checked above to ensure it is not null.
+    let c_str: &CStr = unsafe { CStr::from_ptr(c_protocol_list) };
+
+    let protocol_list = match c_str.to_str() {
+        Ok(n) => n,
+        Err(_) => return 1,
+    };
+
+    match protocol_list.parse::<UnvalidatedProtoEntry>() {
+        Ok(_) => 0,
+        Err(_) => 1,
+    }
+}
+
+/// Provide an interface for C to translate arguments and return types for
+/// protover::list_supports_protocol_or_later
+#[no_mangle]
+pub extern "C" fn protocol_list_supports_protocol_or_later(
+    c_protocol_list: *const c_char,
+    c_protocol: uint32_t,
+    version: uint32_t,
+) -> c_int {
+    if c_protocol_list.is_null() {
+        return 1;
+    }
+
+    // Require an unsafe block to read the version from a C string. The pointer
+    // is checked above to ensure it is not null.
+    let c_str: &CStr = unsafe { CStr::from_ptr(c_protocol_list) };
+
+    let protocol_list = match c_str.to_str() {
+        Ok(n) => n,
+        Err(_) => return 1,
+    };
+
+    let protocol = match translate_to_rust(c_protocol) {
+        Ok(n) => n,
+        Err(_) => return 0,
+    };
+
+    let proto_entry: UnvalidatedProtoEntry = match protocol_list.parse() {
+        Ok(n) => n,
+        Err(_) => return 1,
+    };
+
+    if proto_entry.supports_protocol_or_later(&protocol.into(), &version) {
+        return 1;
+    }
+    0
+}
+
+/// Provide an interface for C to translate arguments and return types for
+/// protover::get_supported_protocols
+#[no_mangle]
+pub extern "C" fn protover_get_supported_protocols() -> *const c_char {
+    let supported: &'static CStr;
+
+    supported = get_supported_protocols_cstr();
+    supported.as_ptr()
+}
+
+/// Provide an interface for C to translate arguments and return types for
+/// protover::compute_vote
+//
+// Why is the threshold a signed integer? —isis
+#[no_mangle]
+pub extern "C" fn protover_compute_vote(list: *const Stringlist, threshold: c_int) -> *mut c_char {
+    if list.is_null() {
+        return allocate_and_copy_string("");
+    }
+
+    // Dereference of raw pointer requires an unsafe block. The pointer is
+    // checked above to ensure it is not null.
+    let data: Vec<String> = unsafe { (*list).get_list() };
+    let hold: usize = threshold as usize;
+    let mut proto_entries: Vec<UnvalidatedProtoEntry> = Vec::new();
+
+    for datum in data {
+        let entry: UnvalidatedProtoEntry = match datum.parse() {
+            Ok(n) => n,
+            Err(_) => continue,
+        };
+        proto_entries.push(entry);
+    }
+    let vote: UnvalidatedProtoEntry = ProtoverVote::compute(&proto_entries, &hold);
+
+    allocate_and_copy_string(&vote.to_string())
+}
+
+/// Provide an interface for C to translate arguments and return types for
+/// protover::is_supported_here
+#[no_mangle]
+pub extern "C" fn protover_is_supported_here(c_protocol: uint32_t, version: uint32_t) -> c_int {
+    let protocol = match translate_to_rust(c_protocol) {
+        Ok(n) => n,
+        Err(_) => return 0,
+    };
+
+    let is_supported = is_supported_here(&protocol, &version);
+
+    return if is_supported { 1 } else { 0 };
+}
+
+/// Provide an interface for C to translate arguments and return types for
+/// protover::compute_for_old_tor
+#[no_mangle]
+pub extern "C" fn protover_compute_for_old_tor(version: *const c_char) -> *const c_char {
+    let supported: &'static CStr;
+    let empty: &'static CStr;
+
+    empty = cstr!("");
+
+    if version.is_null() {
+        return empty.as_ptr();
+    }
+
+    // Require an unsafe block to read the version from a C string. The pointer
+    // is checked above to ensure it is not null.
+    let c_str: &CStr = unsafe { CStr::from_ptr(version) };
+
+    let version = match c_str.to_str() {
+        Ok(n) => n,
+        Err(_) => return empty.as_ptr(),
+    };
+
+    supported = compute_for_old_tor_cstr(&version);
+    supported.as_ptr()
+}
diff --git a/src/rust/protover/lib.rs b/src/rust/protover/lib.rs
new file mode 100644
index 0000000000..35c4106ae5
--- /dev/null
+++ b/src/rust/protover/lib.rs
@@ -0,0 +1,40 @@
+//! Copyright (c) 2016-2019, The Tor Project, Inc. */
+//! See LICENSE for licensing information */
+
+//! Versioning information for different pieces of the Tor protocol.
+//!
+//! The below description is taken from src/rust/protover.c, which is currently
+//! enabled by default. We are in the process of experimenting with Rust in
+//! tor, and this protover module is implemented to help achieve this goal.
+//!
+//! Starting in version 0.2.9.3-alpha, Tor places separate version numbers on
+//! each of the different components of its protocol. Relays use these numbers
+//! to advertise what versions of the protocols they can support, and clients
+//! use them to find what they can ask a given relay to do.  Authorities vote
+//! on the supported protocol versions for each relay, and also vote on the
+//! which protocols you should have to support in order to be on the Tor
+//! network. All Tor instances use these required/recommended protocol versions
+//! to tell what level of support for recent protocols each relay has, and
+//! to decide whether they should be running given their current protocols.
+//!
+//! The main advantage of these protocol versions numbers over using Tor
+//! version numbers is that they allow different implementations of the Tor
+//! protocols to develop independently, without having to claim compatibility
+//! with specific versions of Tor.
+
+// XXX: add missing docs
+//#![deny(missing_docs)]
+
+extern crate external;
+extern crate libc;
+extern crate smartlist;
+extern crate tor_allocate;
+#[macro_use]
+extern crate tor_util;
+
+pub mod errors;
+pub mod ffi;
+pub mod protoset;
+mod protover;
+
+pub use protover::*;
diff --git a/src/rust/protover/protoset.rs b/src/rust/protover/protoset.rs
new file mode 100644
index 0000000000..3b283983c8
--- /dev/null
+++ b/src/rust/protover/protoset.rs
@@ -0,0 +1,689 @@
+// Copyright (c) 2018-2019, The Tor Project, Inc.
+// Copyright (c) 2018, isis agora lovecruft
+// See LICENSE for licensing information
+
+//! Sets for lazily storing ordered, non-overlapping ranges of integers.
+
+use std::cmp;
+use std::iter;
+use std::slice;
+use std::str::FromStr;
+use std::u32;
+
+use errors::ProtoverError;
+
+/// A single version number.
+pub type Version = u32;
+
+/// A `ProtoSet` stores an ordered `Vec<T>` of `(low, high)` pairs of ranges of
+/// non-overlapping protocol versions.
+///
+/// # Examples
+///
+/// ```
+/// use std::str::FromStr;
+///
+/// use protover::errors::ProtoverError;
+/// use protover::protoset::ProtoSet;
+/// use protover::protoset::Version;
+///
+/// # fn do_test() -> Result<ProtoSet, ProtoverError> {
+/// let protoset: ProtoSet = ProtoSet::from_str("3-5,8")?;
+///
+/// // We could also equivalently call:
+/// let protoset: ProtoSet = "3-5,8".parse()?;
+///
+/// assert!(protoset.contains(&4));
+/// assert!(!protoset.contains(&7));
+///
+/// let expanded: Vec<Version> = protoset.clone().into();
+///
+/// assert_eq!(&expanded[..], &[3, 4, 5, 8]);
+///
+/// let contracted: String = protoset.clone().to_string();
+///
+/// assert_eq!(contracted, "3-5,8".to_string());
+/// # Ok(protoset)
+/// # }
+/// # fn main() { do_test(); }  // wrap the test so we can use the ? operator
+#[derive(Clone, Debug, Eq, PartialEq, Hash)]
+pub struct ProtoSet {
+    pub(crate) pairs: Vec<(Version, Version)>,
+}
+
+impl Default for ProtoSet {
+    fn default() -> Self {
+        let pairs: Vec<(Version, Version)> = Vec::new();
+
+        ProtoSet { pairs }
+    }
+}
+
+impl<'a> ProtoSet {
+    /// Create a new `ProtoSet` from a slice of `(low, high)` pairs.
+    ///
+    /// # Inputs
+    ///
+    /// We do not assume the input pairs are deduplicated or ordered.
+    pub fn from_slice(low_high_pairs: &'a [(Version, Version)]) -> Result<Self, ProtoverError> {
+        let mut pairs: Vec<(Version, Version)> = Vec::with_capacity(low_high_pairs.len());
+
+        for &(low, high) in low_high_pairs {
+            pairs.push((low, high));
+        }
+        // Sort the pairs without reallocation and remove all duplicate pairs.
+        pairs.sort_unstable();
+        pairs.dedup();
+
+        ProtoSet { pairs }.is_ok()
+    }
+}
+
+/// Expand this `ProtoSet` to a `Vec` of all its `Version`s.
+///
+/// # Examples
+///
+/// ```
+/// use std::str::FromStr;
+/// use protover::protoset::ProtoSet;
+/// use protover::protoset::Version;
+/// # use protover::errors::ProtoverError;
+///
+/// # fn do_test() -> Result<Vec<Version>, ProtoverError> {
+/// let protoset: ProtoSet = ProtoSet::from_str("3-5,21")?;
+/// let versions: Vec<Version> = protoset.into();
+///
+/// assert_eq!(&versions[..], &[3, 4, 5, 21]);
+/// #
+/// # Ok(versions)
+/// # }
+/// # fn main() { do_test(); }  // wrap the test so we can use the ? operator
+/// ```
+impl Into<Vec<Version>> for ProtoSet {
+    fn into(self) -> Vec<Version> {
+        let mut versions: Vec<Version> = Vec::new();
+
+        for &(low, high) in self.iter() {
+            versions.extend(low..high + 1);
+        }
+        versions
+    }
+}
+
+impl ProtoSet {
+    /// Get an iterator over the `(low, high)` `pairs` in this `ProtoSet`.
+    pub fn iter(&self) -> slice::Iter<(Version, Version)> {
+        self.pairs.iter()
+    }
+
+    /// Expand this `ProtoSet` into a `Vec` of all its `Version`s.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use protover::errors::ProtoverError;
+    /// use protover::protoset::ProtoSet;
+    ///
+    /// # fn do_test() -> Result<bool, ProtoverError> {
+    /// let protoset: ProtoSet = "3-5,9".parse()?;
+    ///
+    /// assert_eq!(protoset.expand(), vec![3, 4, 5, 9]);
+    ///
+    /// let protoset: ProtoSet = "1,3,5-7".parse()?;
+    ///
+    /// assert_eq!(protoset.expand(), vec![1, 3, 5, 6, 7]);
+    /// #
+    /// # Ok(true)
+    /// # }
+    /// # fn main() { do_test(); }  // wrap the test so we can use the ? operator
+    /// ```
+    pub fn expand(self) -> Vec<Version> {
+        self.into()
+    }
+
+    pub fn len(&self) -> usize {
+        let mut length: usize = 0;
+
+        for &(low, high) in self.iter() {
+            length += (high as usize - low as usize) + 1;
+        }
+
+        length
+    }
+
+    /// Check that this `ProtoSet` is well-formed.
+    ///
+    /// This is automatically called in `ProtoSet::from_str()`.
+    ///
+    /// # Errors
+    ///
+    /// * `ProtoverError::LowGreaterThanHigh`: if its `pairs` were not
+    ///   well-formed, i.e. a `low` in a `(low, high)` was higher than the
+    ///   previous `high`,
+    /// * `ProtoverError::Overlap`: if one or more of the `pairs` are
+    ///   overlapping,
+    /// * `ProtoverError::ExceedsMax`: if the number of versions when expanded
+    ///   would exceed `MAX_PROTOCOLS_TO_EXPAND`, and
+    ///
+    /// # Returns
+    ///
+    /// A `Result` whose `Ok` is this `Protoset`, and whose `Err` is one of the
+    /// errors enumerated in the Errors section above.
+    fn is_ok(self) -> Result<ProtoSet, ProtoverError> {
+        let mut last_high: Version = 0;
+
+        for &(low, high) in self.iter() {
+            if low == u32::MAX || high == u32::MAX {
+                return Err(ProtoverError::ExceedsMax);
+            }
+            if low <= last_high {
+                return Err(ProtoverError::Overlap);
+            } else if low > high {
+                return Err(ProtoverError::LowGreaterThanHigh);
+            }
+            last_high = high;
+        }
+
+        Ok(self)
+    }
+
+    /// Determine if this `ProtoSet` contains no `Version`s.
+    ///
+    /// # Returns
+    ///
+    /// * `true` if this `ProtoSet`'s length is zero, and
+    /// * `false` otherwise.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use protover::protoset::ProtoSet;
+    ///
+    /// let protoset: ProtoSet = ProtoSet::default();
+    ///
+    /// assert!(protoset.is_empty());
+    /// ```
+    pub fn is_empty(&self) -> bool {
+        self.pairs.len() == 0
+    }
+
+    /// Determine if `version` is included within this `ProtoSet`.
+    ///
+    /// # Inputs
+    ///
+    /// * `version`: a `Version`.
+    ///
+    /// # Returns
+    ///
+    /// `true` if the `version` is contained within this set; `false` otherwise.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use protover::errors::ProtoverError;
+    /// use protover::protoset::ProtoSet;
+    ///
+    /// # fn do_test() -> Result<ProtoSet, ProtoverError> {
+    /// let protoset: ProtoSet = ProtoSet::from_slice(&[(0, 5), (7, 9), (13, 14)])?;
+    ///
+    /// assert!(protoset.contains(&5));
+    /// assert!(!protoset.contains(&10));
+    /// #
+    /// # Ok(protoset)
+    /// # }
+    /// # fn main() { do_test(); }  // wrap the test so we can use the ? operator
+    /// ```
+    pub fn contains(&self, version: &Version) -> bool {
+        for &(low, high) in self.iter() {
+            if low <= *version && *version <= high {
+                return true;
+            }
+        }
+        false
+    }
+
+    /// Returns all the `Version`s in `self` which are not also in the `other`
+    /// `ProtoSet`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use protover::errors::ProtoverError;
+    /// use protover::protoset::ProtoSet;
+    ///
+    /// # fn do_test() -> Result<bool, ProtoverError> {
+    /// let protoset: ProtoSet = "1,3-6,10-12,15-16".parse()?;
+    /// let other: ProtoSet = "2,5-7,9-11,14-20".parse()?;
+    ///
+    /// let subset: ProtoSet = protoset.and_not_in(&other);
+    ///
+    /// assert_eq!(subset.expand(), vec![1, 3, 4, 12]);
+    /// #
+    /// # Ok(true)
+    /// # }
+    /// # fn main() { do_test(); }  // wrap the test so we can use the ? operator
+    /// ```
+    pub fn and_not_in(&self, other: &Self) -> Self {
+        if self.is_empty() || other.is_empty() {
+            return self.clone();
+        }
+
+        let pairs = self.iter().flat_map(|&(lo, hi)| {
+            let the_end = (hi + 1, hi + 1); // special case to mark the end of the range.
+            let excluded_ranges = other
+                .iter()
+                .cloned() // have to be owned tuples, to match iter::once(the_end).
+                .skip_while(move|&(_, hi2)| hi2 < lo) // skip the non-overlapping ranges.
+                .take_while(move|&(lo2, _)| lo2 <= hi) // take all the overlapping ones.
+                .chain(iter::once(the_end));
+
+            let mut nextlo = lo;
+            excluded_ranges.filter_map(move |(excluded_lo, excluded_hi)| {
+                let pair = if nextlo < excluded_lo {
+                    Some((nextlo, excluded_lo - 1))
+                } else {
+                    None
+                };
+                nextlo = cmp::min(excluded_hi, u32::MAX - 1) + 1;
+                pair
+            })
+        });
+
+        let pairs = pairs.collect();
+        ProtoSet::is_ok(ProtoSet { pairs }).expect("should be already sorted")
+    }
+}
+
+impl FromStr for ProtoSet {
+    type Err = ProtoverError;
+
+    /// Parse the unique version numbers supported by a subprotocol from a string.
+    ///
+    /// # Inputs
+    ///
+    /// * `version_string`, a string comprised of "[0-9,-]"
+    ///
+    /// # Returns
+    ///
+    /// A `Result` whose `Ok` value is a `ProtoSet` holding all of the unique
+    /// version numbers.
+    ///
+    /// The returned `Result`'s `Err` value is an `ProtoverError` appropriate to
+    /// the error.
+    ///
+    /// # Errors
+    ///
+    /// This function will error if:
+    ///
+    /// * the `version_string` is an equals (`"="`) sign,
+    /// * the expansion of a version range produces an error (see
+    ///   `expand_version_range`),
+    /// * any single version number is not parseable as an `u32` in radix 10, or
+    /// * there are greater than 2^16 version numbers to expand.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use std::str::FromStr;
+    ///
+    /// use protover::errors::ProtoverError;
+    /// use protover::protoset::ProtoSet;
+    ///
+    /// # fn do_test() -> Result<ProtoSet, ProtoverError> {
+    /// let protoset: ProtoSet = ProtoSet::from_str("2-5,8")?;
+    ///
+    /// assert!(protoset.contains(&5));
+    /// assert!(!protoset.contains(&10));
+    ///
+    /// // We can also equivalently call `ProtoSet::from_str` by doing (all
+    /// // implementations of `FromStr` can be called this way, this one isn't
+    /// // special):
+    /// let protoset: ProtoSet = "4-6,12".parse()?;
+    ///
+    /// // Calling it (either way) can take really large ranges (up to `u32::MAX`):
+    /// let protoset: ProtoSet = "1-70000".parse()?;
+    /// let protoset: ProtoSet = "1-4294967296".parse()?;
+    ///
+    /// // There are lots of ways to get an `Err` from this function.  Here are
+    /// // a few:
+    /// assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("="));
+    /// assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("-"));
+    /// assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("not_an_int"));
+    /// assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("3-"));
+    /// assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1-,4"));
+    ///
+    /// // An empty string is, however, legal, and results in an
+    /// // empty `ProtoSet`:
+    /// assert_eq!(Ok(ProtoSet::default()), ProtoSet::from_str(""));
+    /// #
+    /// # Ok(protoset)
+    /// # }
+    /// # fn main() { do_test(); }  // wrap the test so we can use the ? operator
+    /// ```
+    fn from_str(version_string: &str) -> Result<Self, Self::Err> {
+        // If we were passed in an empty string, then return an empty ProtoSet.
+        if version_string.is_empty() {
+            return Ok(Self::default());
+        }
+
+        let mut pairs: Vec<(Version, Version)> = Vec::new();
+        let pieces: ::std::str::Split<char> = version_string.split(',');
+
+        for p in pieces {
+            if p.contains('-') {
+                let mut pair = p.splitn(2, '-');
+
+                let low = pair.next().ok_or(ProtoverError::Unparseable)?;
+                let high = pair.next().ok_or(ProtoverError::Unparseable)?;
+
+                let lo: Version = low.parse().or(Err(ProtoverError::Unparseable))?;
+                let hi: Version = high.parse().or(Err(ProtoverError::Unparseable))?;
+
+                pairs.push((lo, hi));
+            } else {
+                let v: u32 = p.parse().or(Err(ProtoverError::Unparseable))?;
+
+                pairs.push((v, v));
+            }
+        }
+
+        ProtoSet::from_slice(&pairs[..])
+    }
+}
+
+impl ToString for ProtoSet {
+    /// Contracts a `ProtoSet` of versions into a string.
+    ///
+    /// # Returns
+    ///
+    /// A `String` representation of this `ProtoSet` in ascending order.
+    fn to_string(&self) -> String {
+        let mut final_output: Vec<String> = Vec::new();
+
+        for &(lo, hi) in self.iter() {
+            if lo != hi {
+                debug_assert!(lo < hi);
+                final_output.push(format!("{}-{}", lo, hi));
+            } else {
+                final_output.push(format!("{}", lo));
+            }
+        }
+        final_output.join(",")
+    }
+}
+
+/// Checks to see if there is a continuous range of integers, starting at the
+/// first in the list. Returns the last integer in the range if a range exists.
+///
+/// # Inputs
+///
+/// `list`, an ordered  vector of `u32` integers of "[0-9,-]" representing the
+/// supported versions for a single protocol.
+///
+/// # Returns
+///
+/// A `bool` indicating whether the list contains a range, starting at the first
+/// in the list, a`Version` of the last integer in the range, and a `usize` of
+/// the index of that version.
+///
+/// For example, if given vec![1, 2, 3, 5], find_range will return true,
+/// as there is a continuous range, and 3, which is the last number in the
+/// continuous range, and 2 which is the index of 3.
+fn find_range(list: &Vec<Version>) -> (bool, Version, usize) {
+    if list.len() == 0 {
+        return (false, 0, 0);
+    }
+
+    let mut index: usize = 0;
+    let mut iterable = list.iter().peekable();
+    let mut range_end = match iterable.next() {
+        Some(n) => *n,
+        None => return (false, 0, 0),
+    };
+
+    let mut has_range = false;
+
+    while iterable.peek().is_some() {
+        let n = *iterable.next().unwrap();
+        if n != range_end + 1 {
+            break;
+        }
+
+        has_range = true;
+        range_end = n;
+        index += 1;
+    }
+
+    (has_range, range_end, index)
+}
+
+impl From<Vec<Version>> for ProtoSet {
+    fn from(mut v: Vec<Version>) -> ProtoSet {
+        let mut version_pairs: Vec<(Version, Version)> = Vec::new();
+
+        v.sort_unstable();
+        v.dedup();
+
+        'vector: while !v.is_empty() {
+            let (has_range, end, index): (bool, Version, usize) = find_range(&v);
+
+            if has_range {
+                let first: Version = match v.first() {
+                    Some(x) => *x,
+                    None => continue,
+                };
+                let last: Version = match v.get(index) {
+                    Some(x) => *x,
+                    None => continue,
+                };
+                debug_assert!(last == end, format!("last = {}, end = {}", last, end));
+
+                version_pairs.push((first, last));
+                v = v.split_off(index + 1);
+
+                if v.len() == 0 {
+                    break 'vector;
+                }
+            } else {
+                let last: Version = match v.get(index) {
+                    Some(x) => *x,
+                    None => continue,
+                };
+                version_pairs.push((last, last));
+                v.remove(index);
+            }
+        }
+        ProtoSet::from_slice(&version_pairs[..]).unwrap_or(ProtoSet::default())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_find_range() {
+        assert_eq!((false, 0, 0), find_range(&vec![]));
+        assert_eq!((false, 1, 0), find_range(&vec![1]));
+        assert_eq!((true, 2, 1), find_range(&vec![1, 2]));
+        assert_eq!((true, 3, 2), find_range(&vec![1, 2, 3]));
+        assert_eq!((true, 3, 2), find_range(&vec![1, 2, 3, 5]));
+    }
+
+    macro_rules! assert_contains_each {
+        ($protoset:expr, $versions:expr) => {
+            for version in $versions {
+                assert!($protoset.contains(version));
+            }
+        };
+    }
+
+    macro_rules! test_protoset_contains_versions {
+        ($list:expr, $str:expr) => {
+            let versions: &[Version] = $list;
+            let protoset: Result<ProtoSet, ProtoverError> = ProtoSet::from_str($str);
+
+            assert!(protoset.is_ok());
+            let p = protoset.unwrap();
+            assert_contains_each!(p, versions);
+        };
+    }
+
+    #[test]
+    fn test_versions_from_str() {
+        test_protoset_contains_versions!(&[], "");
+        test_protoset_contains_versions!(&[1], "1");
+        test_protoset_contains_versions!(&[1, 2], "1,2");
+        test_protoset_contains_versions!(&[1, 2, 3], "1-3");
+        test_protoset_contains_versions!(&[1, 2, 5], "1-2,5");
+        test_protoset_contains_versions!(&[1, 3, 4, 5], "1,3-5");
+        test_protoset_contains_versions!(&[42, 55, 56, 57, 58], "42,55-58");
+    }
+
+    #[test]
+    fn test_versions_from_str_ab() {
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("a,b"));
+    }
+
+    #[test]
+    fn test_versions_from_str_negative_1() {
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("-1"));
+    }
+
+    #[test]
+    fn test_versions_from_str_commas() {
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str(","));
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1,,2"));
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1,2,"));
+    }
+
+    #[test]
+    fn test_versions_from_str_hyphens() {
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("--1"));
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("-1-2"));
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1--2"));
+    }
+
+    #[test]
+    fn test_versions_from_str_triple() {
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1-2-3"));
+    }
+
+    #[test]
+    fn test_versions_from_str_1exclam() {
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1,!"));
+    }
+
+    #[test]
+    fn test_versions_from_str_percent_equal() {
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("%="));
+    }
+
+    #[test]
+    fn test_versions_from_str_whitespace() {
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1,2\n"));
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1\r,2"));
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1,\t2"));
+    }
+
+    #[test]
+    fn test_versions_from_str_overlap() {
+        assert_eq!(Err(ProtoverError::Overlap), ProtoSet::from_str("1-3,2-4"));
+    }
+
+    #[test]
+    fn test_versions_from_slice_overlap() {
+        assert_eq!(
+            Err(ProtoverError::Overlap),
+            ProtoSet::from_slice(&[(1, 3), (2, 4)])
+        );
+    }
+
+    #[test]
+    fn test_versions_from_str_max() {
+        assert_eq!(
+            Err(ProtoverError::ExceedsMax),
+            ProtoSet::from_str("4294967295")
+        );
+    }
+
+    #[test]
+    fn test_versions_from_slice_max() {
+        assert_eq!(
+            Err(ProtoverError::ExceedsMax),
+            ProtoSet::from_slice(&[(4294967295, 4294967295)])
+        );
+    }
+
+    #[test]
+    fn test_protoset_contains() {
+        let protoset: ProtoSet = ProtoSet::from_slice(&[(1, 5), (7, 9), (13, 14)]).unwrap();
+
+        for x in 1..6 {
+            assert!(protoset.contains(&x), format!("should contain {}", x));
+        }
+        for x in 7..10 {
+            assert!(protoset.contains(&x), format!("should contain {}", x));
+        }
+        for x in 13..15 {
+            assert!(protoset.contains(&x), format!("should contain {}", x));
+        }
+
+        for x in [6, 10, 11, 12, 15, 42, 43, 44, 45, 1234584].iter() {
+            assert!(!protoset.contains(&x), format!("should not contain {}", x));
+        }
+    }
+
+    #[test]
+    fn test_protoset_contains_1_3() {
+        let protoset: ProtoSet = ProtoSet::from_slice(&[(1, 3)]).unwrap();
+
+        for x in 1..4 {
+            assert!(protoset.contains(&x), format!("should contain {}", x));
+        }
+    }
+
+    macro_rules! assert_protoset_from_vec_contains_all {
+        ($($x:expr),*) => (
+            let vec: Vec<Version> = vec!($($x),*);
+            let protoset: ProtoSet = vec.clone().into();
+
+            for x in vec.iter() {
+                assert!(protoset.contains(&x));
+            }
+        )
+    }
+
+    #[test]
+    fn test_protoset_from_vec_123() {
+        assert_protoset_from_vec_contains_all!(1, 2, 3);
+    }
+
+    #[test]
+    fn test_protoset_from_vec_1_315() {
+        assert_protoset_from_vec_contains_all!(1, 2, 3, 15);
+    }
+
+    #[test]
+    fn test_protoset_from_vec_unordered() {
+        let v: Vec<Version> = vec![2, 3, 8, 4, 3, 9, 7, 2];
+        let ps: ProtoSet = v.into();
+
+        assert_eq!(ps.to_string(), "2-4,7-9");
+    }
+
+    #[test]
+    fn test_protoset_into_vec() {
+        let ps: ProtoSet = "1-13,42,9001,4294967294".parse().unwrap();
+        let v: Vec<Version> = ps.into();
+
+        assert!(v.contains(&7));
+        assert!(v.contains(&9001));
+        assert!(v.contains(&4294967294));
+    }
+}
+
+#[cfg(all(test, feature = "bench"))]
+mod bench {
+    use super::*;
+}
diff --git a/src/rust/protover/protover.rs b/src/rust/protover/protover.rs
new file mode 100644
index 0000000000..2661d811c4
--- /dev/null
+++ b/src/rust/protover/protover.rs
@@ -0,0 +1,971 @@
+// Copyright (c) 2016-2019, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+use std::collections::hash_map;
+use std::collections::HashMap;
+use std::ffi::CStr;
+use std::fmt;
+use std::str;
+use std::str::FromStr;
+use std::string::String;
+
+use external::c_tor_version_as_new_as;
+
+use errors::ProtoverError;
+use protoset::ProtoSet;
+use protoset::Version;
+
+/// The first version of Tor that included "proto" entries in its descriptors.
+/// Authorities should use this to decide whether to guess proto lines.
+///
+/// C_RUST_COUPLED:
+///     protover.h `FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS`
+const FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS: &'static str = "0.2.9.3-alpha";
+
+/// The maximum number of subprotocol version numbers we will attempt to expand
+/// before concluding that someone is trying to DoS us
+///
+/// C_RUST_COUPLED: protover.c `MAX_PROTOCOLS_TO_EXPAND`
+const MAX_PROTOCOLS_TO_EXPAND: usize = (1 << 16);
+
+/// The maximum size an `UnknownProtocol`'s name may be.
+pub(crate) const MAX_PROTOCOL_NAME_LENGTH: usize = 100;
+
+/// Known subprotocols in Tor. Indicates which subprotocol a relay supports.
+///
+/// C_RUST_COUPLED: protover.h `protocol_type_t`
+#[derive(Clone, Hash, Eq, PartialEq, Debug)]
+pub enum Protocol {
+    Cons,
+    Desc,
+    DirCache,
+    HSDir,
+    HSIntro,
+    HSRend,
+    Link,
+    LinkAuth,
+    Microdesc,
+    Relay,
+}
+
+impl fmt::Display for Protocol {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+/// Translates a string representation of a protocol into a Proto type.
+/// Error if the string is an unrecognized protocol name.
+///
+/// C_RUST_COUPLED: protover.c `PROTOCOL_NAMES`
+impl FromStr for Protocol {
+    type Err = ProtoverError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "Cons" => Ok(Protocol::Cons),
+            "Desc" => Ok(Protocol::Desc),
+            "DirCache" => Ok(Protocol::DirCache),
+            "HSDir" => Ok(Protocol::HSDir),
+            "HSIntro" => Ok(Protocol::HSIntro),
+            "HSRend" => Ok(Protocol::HSRend),
+            "Link" => Ok(Protocol::Link),
+            "LinkAuth" => Ok(Protocol::LinkAuth),
+            "Microdesc" => Ok(Protocol::Microdesc),
+            "Relay" => Ok(Protocol::Relay),
+            _ => Err(ProtoverError::UnknownProtocol),
+        }
+    }
+}
+
+/// A protocol string which is not one of the `Protocols` we currently know
+/// about.
+#[derive(Clone, Debug, Hash, Eq, PartialEq)]
+pub struct UnknownProtocol(String);
+
+impl fmt::Display for UnknownProtocol {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}
+
+fn is_valid_proto(s: &str) -> bool {
+    s.chars().all(|c| c.is_ascii_alphanumeric() || c == '-')
+}
+
+impl FromStr for UnknownProtocol {
+    type Err = ProtoverError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        if !is_valid_proto(s) {
+            Err(ProtoverError::InvalidProtocol)
+        } else if s.len() <= MAX_PROTOCOL_NAME_LENGTH {
+            Ok(UnknownProtocol(s.to_string()))
+        } else {
+            Err(ProtoverError::ExceedsNameLimit)
+        }
+    }
+}
+
+impl UnknownProtocol {
+    /// Create an `UnknownProtocol`, ignoring whether or not it
+    /// exceeds MAX_PROTOCOL_NAME_LENGTH.
+    fn from_str_any_len(s: &str) -> Result<Self, ProtoverError> {
+        if !is_valid_proto(s) {
+            return Err(ProtoverError::InvalidProtocol);
+        }
+        Ok(UnknownProtocol(s.to_string()))
+    }
+}
+
+impl From<Protocol> for UnknownProtocol {
+    fn from(p: Protocol) -> UnknownProtocol {
+        UnknownProtocol(p.to_string())
+    }
+}
+
+#[cfg(feature = "test_linking_hack")]
+fn have_linkauth_v1() -> bool {
+    true
+}
+
+#[cfg(not(feature = "test_linking_hack"))]
+fn have_linkauth_v1() -> bool {
+    use external::c_tor_is_using_nss;
+    !c_tor_is_using_nss()
+}
+
+/// Get a CStr representation of current supported protocols, for
+/// passing to C, or for converting to a `&str` for Rust.
+///
+/// # Returns
+///
+/// An `&'static CStr` whose value is the existing protocols supported by tor.
+/// Returned data is in the format as follows:
+///
+/// "HSDir=1-1 LinkAuth=1"
+///
+/// # Note
+///
+/// Rust code can use the `&'static CStr` as a normal `&'a str` by
+/// calling `protover::get_supported_protocols`.
+///
+//  C_RUST_COUPLED: protover.c `protover_get_supported_protocols`
+pub(crate) fn get_supported_protocols_cstr() -> &'static CStr {
+    if !have_linkauth_v1() {
+        cstr!(
+            "Cons=1-2 \
+             Desc=1-2 \
+             DirCache=1-2 \
+             HSDir=1-2 \
+             HSIntro=3-4 \
+             HSRend=1-2 \
+             Link=1-5 \
+             LinkAuth=3 \
+             Microdesc=1-2 \
+             Relay=1-2"
+        )
+    } else {
+        cstr!(
+            "Cons=1-2 \
+             Desc=1-2 \
+             DirCache=1-2 \
+             HSDir=1-2 \
+             HSIntro=3-4 \
+             HSRend=1-2 \
+             Link=1-5 \
+             LinkAuth=1,3 \
+             Microdesc=1-2 \
+             Relay=1-2"
+        )
+    }
+}
+
+/// A map of protocol names to the versions of them which are supported.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ProtoEntry(HashMap<Protocol, ProtoSet>);
+
+impl Default for ProtoEntry {
+    fn default() -> ProtoEntry {
+        ProtoEntry(HashMap::new())
+    }
+}
+
+impl ProtoEntry {
+    /// Get an iterator over the `Protocol`s and their `ProtoSet`s in this `ProtoEntry`.
+    pub fn iter(&self) -> hash_map::Iter<Protocol, ProtoSet> {
+        self.0.iter()
+    }
+
+    /// Translate the supported tor versions from a string into a
+    /// ProtoEntry, which is useful when looking up a specific
+    /// subprotocol.
+    pub fn supported() -> Result<Self, ProtoverError> {
+        let supported_cstr: &'static CStr = get_supported_protocols_cstr();
+        let supported: &str = supported_cstr.to_str().unwrap_or("");
+
+        supported.parse()
+    }
+
+    pub fn len(&self) -> usize {
+        self.0.len()
+    }
+
+    pub fn get(&self, protocol: &Protocol) -> Option<&ProtoSet> {
+        self.0.get(protocol)
+    }
+
+    pub fn insert(&mut self, key: Protocol, value: ProtoSet) {
+        self.0.insert(key, value);
+    }
+
+    pub fn remove(&mut self, key: &Protocol) -> Option<ProtoSet> {
+        self.0.remove(key)
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+}
+
+impl FromStr for ProtoEntry {
+    type Err = ProtoverError;
+
+    /// Parse a string of subprotocol types and their version numbers.
+    ///
+    /// # Inputs
+    ///
+    /// * A `protocol_entry` string, comprised of a keywords, an "=" sign, and
+    /// one or more version numbers, each separated by a space.  For example,
+    /// `"Cons=3-4 HSDir=1"`.
+    ///
+    /// # Returns
+    ///
+    /// A `Result` whose `Ok` value is a `ProtoEntry`.
+    /// Otherwise, the `Err` value of this `Result` is a `ProtoverError`.
+    fn from_str(protocol_entry: &str) -> Result<ProtoEntry, ProtoverError> {
+        let mut proto_entry: ProtoEntry = ProtoEntry::default();
+        let entries = protocol_entry.split(' ');
+
+        for entry in entries {
+            let mut parts = entry.splitn(2, '=');
+
+            let proto = match parts.next() {
+                Some(n) => n,
+                None => return Err(ProtoverError::Unparseable),
+            };
+
+            let vers = match parts.next() {
+                Some(n) => n,
+                None => return Err(ProtoverError::Unparseable),
+            };
+            let versions: ProtoSet = vers.parse()?;
+            let proto_name: Protocol = proto.parse()?;
+
+            proto_entry.insert(proto_name, versions);
+
+            if proto_entry.len() > MAX_PROTOCOLS_TO_EXPAND {
+                return Err(ProtoverError::ExceedsMax);
+            }
+        }
+        Ok(proto_entry)
+    }
+}
+
+/// Generate an implementation of `ToString` for either a `ProtoEntry` or an
+/// `UnvalidatedProtoEntry`.
+macro_rules! impl_to_string_for_proto_entry {
+    ($t:ty) => {
+        impl ToString for $t {
+            fn to_string(&self) -> String {
+                let mut parts: Vec<String> = Vec::new();
+
+                for (protocol, versions) in self.iter() {
+                    parts.push(format!("{}={}", protocol.to_string(), versions.to_string()));
+                }
+                parts.sort_unstable();
+                parts.join(" ")
+            }
+        }
+    };
+}
+
+impl_to_string_for_proto_entry!(ProtoEntry);
+impl_to_string_for_proto_entry!(UnvalidatedProtoEntry);
+
+/// A `ProtoEntry`, but whose `Protocols` can be any `UnknownProtocol`, not just
+/// the supported ones enumerated in `Protocols`.  The protocol versions are
+/// validated, however.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct UnvalidatedProtoEntry(HashMap<UnknownProtocol, ProtoSet>);
+
+impl Default for UnvalidatedProtoEntry {
+    fn default() -> UnvalidatedProtoEntry {
+        UnvalidatedProtoEntry(HashMap::new())
+    }
+}
+
+impl UnvalidatedProtoEntry {
+    /// Get an iterator over the `Protocol`s and their `ProtoSet`s in this `ProtoEntry`.
+    pub fn iter(&self) -> hash_map::Iter<UnknownProtocol, ProtoSet> {
+        self.0.iter()
+    }
+
+    pub fn get(&self, protocol: &UnknownProtocol) -> Option<&ProtoSet> {
+        self.0.get(protocol)
+    }
+
+    pub fn insert(&mut self, key: UnknownProtocol, value: ProtoSet) {
+        self.0.insert(key, value);
+    }
+
+    pub fn remove(&mut self, key: &UnknownProtocol) -> Option<ProtoSet> {
+        self.0.remove(key)
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+
+    pub fn len(&self) -> usize {
+        let mut total: usize = 0;
+
+        for (_, versions) in self.iter() {
+            total += versions.len();
+        }
+        total
+    }
+
+    /// Determine if we support every protocol a client supports, and if not,
+    /// determine which protocols we do not have support for.
+    ///
+    /// # Returns
+    ///
+    /// Optionally, return parameters which the client supports but which we do not.
+    ///
+    /// # Examples
+    /// ```
+    /// use protover::UnvalidatedProtoEntry;
+    ///
+    /// let protocols: UnvalidatedProtoEntry = "LinkAuth=1 Microdesc=1-2 Relay=2".parse().unwrap();
+    /// let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    /// assert_eq!(true, unsupported.is_none());
+    ///
+    /// let protocols: UnvalidatedProtoEntry = "Link=1-2 Wombat=9".parse().unwrap();
+    /// let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    /// assert_eq!(true, unsupported.is_some());
+    /// assert_eq!("Wombat=9", &unsupported.unwrap().to_string());
+    /// ```
+    pub fn all_supported(&self) -> Option<UnvalidatedProtoEntry> {
+        let mut unsupported: UnvalidatedProtoEntry = UnvalidatedProtoEntry::default();
+        let supported: ProtoEntry = match ProtoEntry::supported() {
+            Ok(x) => x,
+            Err(_) => return None,
+        };
+
+        for (protocol, versions) in self.iter() {
+            let is_supported: Result<Protocol, ProtoverError> = protocol.0.parse();
+            let supported_protocol: Protocol;
+
+            // If the protocol wasn't even in the enum, then we definitely don't
+            // know about it and don't support any of its versions.
+            if is_supported.is_err() {
+                if !versions.is_empty() {
+                    unsupported.insert(protocol.clone(), versions.clone());
+                }
+                continue;
+            } else {
+                supported_protocol = is_supported.unwrap();
+            }
+
+            let maybe_supported_versions: Option<&ProtoSet> = supported.get(&supported_protocol);
+            let supported_versions: &ProtoSet;
+
+            // If the protocol wasn't in the map, then we don't know about it
+            // and don't support any of its versions.  Add its versions to the
+            // map (if it has versions).
+            if maybe_supported_versions.is_none() {
+                if !versions.is_empty() {
+                    unsupported.insert(protocol.clone(), versions.clone());
+                }
+                continue;
+            } else {
+                supported_versions = maybe_supported_versions.unwrap();
+            }
+            let unsupported_versions = versions.and_not_in(supported_versions);
+
+            if !unsupported_versions.is_empty() {
+                unsupported.insert(protocol.clone(), unsupported_versions);
+            }
+        }
+
+        if unsupported.is_empty() {
+            return None;
+        }
+        Some(unsupported)
+    }
+
+    /// Determine if we have support for some protocol and version.
+    ///
+    /// # Inputs
+    ///
+    /// * `proto`, an `UnknownProtocol` to test support for
+    /// * `vers`, a `Version` which we will go on to determine whether the
+    /// specified protocol supports.
+    ///
+    /// # Return
+    ///
+    /// Returns `true` iff this `UnvalidatedProtoEntry` includes support for the
+    /// indicated protocol and version, and `false` otherwise.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use std::str::FromStr;
+    /// use protover::*;
+    /// # use protover::errors::ProtoverError;
+    ///
+    /// # fn do_test () -> Result<UnvalidatedProtoEntry, ProtoverError> {
+    /// let proto: UnvalidatedProtoEntry = "Link=3-4 Cons=1 Doggo=3-5".parse()?;
+    /// assert_eq!(true, proto.supports_protocol(&Protocol::Cons.into(), &1));
+    /// assert_eq!(false, proto.supports_protocol(&Protocol::Cons.into(), &5));
+    /// assert_eq!(true, proto.supports_protocol(&UnknownProtocol::from_str("Doggo")?, &4));
+    /// # Ok(proto)
+    /// # } fn main () { do_test(); }
+    /// ```
+    pub fn supports_protocol(&self, proto: &UnknownProtocol, vers: &Version) -> bool {
+        let supported_versions: &ProtoSet = match self.get(proto) {
+            Some(n) => n,
+            None => return false,
+        };
+        supported_versions.contains(&vers)
+    }
+
+    /// As `UnvalidatedProtoEntry::supports_protocol()`, but also returns `true`
+    /// if any later version of the protocol is supported.
+    ///
+    /// # Examples
+    /// ```
+    /// use protover::*;
+    /// # use protover::errors::ProtoverError;
+    ///
+    /// # fn do_test () -> Result<UnvalidatedProtoEntry, ProtoverError> {
+    /// let proto: UnvalidatedProtoEntry = "Link=3-4 Cons=5".parse()?;
+    ///
+    /// assert_eq!(true, proto.supports_protocol_or_later(&Protocol::Cons.into(), &5));
+    /// assert_eq!(true, proto.supports_protocol_or_later(&Protocol::Cons.into(), &4));
+    /// assert_eq!(false, proto.supports_protocol_or_later(&Protocol::Cons.into(), &6));
+    /// # Ok(proto)
+    /// # } fn main () { do_test(); }
+    /// ```
+    pub fn supports_protocol_or_later(&self, proto: &UnknownProtocol, vers: &Version) -> bool {
+        let supported_versions: &ProtoSet = match self.get(&proto) {
+            Some(n) => n,
+            None => return false,
+        };
+        supported_versions.iter().any(|v| v.1 >= *vers)
+    }
+
+    /// Split a string containing (potentially) several protocols and their
+    /// versions into a `Vec` of tuples of string in `(protocol, versions)`
+    /// form.
+    ///
+    /// # Inputs
+    ///
+    /// A &str in the form `"Link=3-4 Cons=5"`.
+    ///
+    /// # Returns
+    ///
+    /// A `Result` whose `Ok` variant is a `Vec<(&str, &str)>` of `(protocol,
+    /// versions)`, or whose `Err` variant is a `ProtoverError`.
+    ///
+    /// # Errors
+    ///
+    /// This will error with a `ProtoverError::Unparseable` if any of the
+    /// following are true:
+    ///
+    /// * If a protocol name is an empty string, e.g. `"Cons=1,3 =3-5"`.
+    /// * If an entry has no equals sign, e.g. `"Cons=1,3 Desc"`.
+    /// * If there is leading or trailing whitespace, e.g. `" Cons=1,3 Link=3"`.
+    /// * If there is any other extra whitespice, e.g. `"Cons=1,3  Link=3"`.
+    fn parse_protocol_and_version_str<'a>(
+        protocol_string: &'a str,
+    ) -> Result<Vec<(&'a str, &'a str)>, ProtoverError> {
+        let mut protovers: Vec<(&str, &str)> = Vec::new();
+
+        for subproto in protocol_string.split(' ') {
+            let mut parts = subproto.splitn(2, '=');
+
+            let name = match parts.next() {
+                Some("") => return Err(ProtoverError::Unparseable),
+                Some(n) => n,
+                None => return Err(ProtoverError::Unparseable),
+            };
+            let vers = match parts.next() {
+                Some(n) => n,
+                None => return Err(ProtoverError::Unparseable),
+            };
+            protovers.push((name, vers));
+        }
+        Ok(protovers)
+    }
+}
+
+impl FromStr for UnvalidatedProtoEntry {
+    type Err = ProtoverError;
+
+    /// Parses a protocol list without validating the protocol names.
+    ///
+    /// # Inputs
+    ///
+    /// * `protocol_string`, a string comprised of keys and values, both which are
+    /// strings. The keys are the protocol names while values are a string
+    /// representation of the supported versions.
+    ///
+    /// The input is _not_ expected to be a subset of the Protocol types
+    ///
+    /// # Returns
+    ///
+    /// A `Result` whose `Ok` value is an `UnvalidatedProtoEntry`.
+    ///
+    /// The returned `Result`'s `Err` value is an `ProtoverError`.
+    ///
+    /// # Errors
+    ///
+    /// This function will error if:
+    ///
+    /// * The protocol string does not follow the "protocol_name=version_list"
+    ///   expected format, or
+    /// * If the version string is malformed. See `impl FromStr for ProtoSet`.
+    fn from_str(protocol_string: &str) -> Result<UnvalidatedProtoEntry, ProtoverError> {
+        let mut parsed: UnvalidatedProtoEntry = UnvalidatedProtoEntry::default();
+        let parts: Vec<(&str, &str)> =
+            UnvalidatedProtoEntry::parse_protocol_and_version_str(protocol_string)?;
+
+        for &(name, vers) in parts.iter() {
+            let versions = ProtoSet::from_str(vers)?;
+            let protocol = UnknownProtocol::from_str(name)?;
+
+            parsed.insert(protocol, versions);
+        }
+        Ok(parsed)
+    }
+}
+
+impl UnvalidatedProtoEntry {
+    /// Create an `UnknownProtocol`, ignoring whether or not it
+    /// exceeds MAX_PROTOCOL_NAME_LENGTH.
+    pub(crate) fn from_str_any_len(
+        protocol_string: &str,
+    ) -> Result<UnvalidatedProtoEntry, ProtoverError> {
+        let mut parsed: UnvalidatedProtoEntry = UnvalidatedProtoEntry::default();
+        let parts: Vec<(&str, &str)> =
+            UnvalidatedProtoEntry::parse_protocol_and_version_str(protocol_string)?;
+
+        for &(name, vers) in parts.iter() {
+            let versions = ProtoSet::from_str(vers)?;
+            let protocol = UnknownProtocol::from_str_any_len(name)?;
+
+            parsed.insert(protocol, versions);
+        }
+        Ok(parsed)
+    }
+}
+
+/// Pretend a `ProtoEntry` is actually an `UnvalidatedProtoEntry`.
+impl From<ProtoEntry> for UnvalidatedProtoEntry {
+    fn from(proto_entry: ProtoEntry) -> UnvalidatedProtoEntry {
+        let mut unvalidated: UnvalidatedProtoEntry = UnvalidatedProtoEntry::default();
+
+        for (protocol, versions) in proto_entry.iter() {
+            unvalidated.insert(UnknownProtocol::from(protocol.clone()), versions.clone());
+        }
+        unvalidated
+    }
+}
+
+/// A mapping of protocols to a count of how many times each of their `Version`s
+/// were voted for or supported.
+///
+/// # Warning
+///
+/// The "protocols" are *not* guaranteed to be known/supported `Protocol`s, in
+/// order to allow new subprotocols to be introduced even if Directory
+/// Authorities don't yet know of them.
+pub struct ProtoverVote(HashMap<UnknownProtocol, HashMap<Version, usize>>);
+
+impl Default for ProtoverVote {
+    fn default() -> ProtoverVote {
+        ProtoverVote(HashMap::new())
+    }
+}
+
+impl IntoIterator for ProtoverVote {
+    type Item = (UnknownProtocol, HashMap<Version, usize>);
+    type IntoIter = hash_map::IntoIter<UnknownProtocol, HashMap<Version, usize>>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.0.into_iter()
+    }
+}
+
+impl ProtoverVote {
+    pub fn entry(
+        &mut self,
+        key: UnknownProtocol,
+    ) -> hash_map::Entry<UnknownProtocol, HashMap<Version, usize>> {
+        self.0.entry(key)
+    }
+
+    /// Protocol voting implementation.
+    ///
+    /// Given a slice of `UnvalidatedProtoEntry`s and a vote `threshold`, return
+    /// a new `UnvalidatedProtoEntry` encoding all of the protocols that are
+    /// listed by at least `threshold` of the inputs.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use protover::ProtoverVote;
+    /// use protover::UnvalidatedProtoEntry;
+    ///
+    /// let protos: &[UnvalidatedProtoEntry] = &["Link=3-4".parse().unwrap(),
+    ///                                          "Link=3".parse().unwrap()];
+    /// let vote = ProtoverVote::compute(protos, &2);
+    /// assert_eq!("Link=3", vote.to_string());
+    /// ```
+    // C_RUST_COUPLED: protover.c protover_compute_vote
+    pub fn compute(
+        proto_entries: &[UnvalidatedProtoEntry],
+        threshold: &usize,
+    ) -> UnvalidatedProtoEntry {
+        let mut all_count: ProtoverVote = ProtoverVote::default();
+        let mut final_output: UnvalidatedProtoEntry = UnvalidatedProtoEntry::default();
+
+        if proto_entries.is_empty() {
+            return final_output;
+        }
+
+        // parse and collect all of the protos and their versions and collect them
+        for vote in proto_entries {
+            // C_RUST_DIFFERS: This doesn't actually differ, bu this check on
+            // the total is here to make it match.  Because the C version calls
+            // expand_protocol_list() which checks if there would be too many
+            // subprotocols *or* individual version numbers, i.e. more than
+            // MAX_PROTOCOLS_TO_EXPAND, and does this *per vote*, we need to
+            // match it's behaviour and ensure we're not allowing more than it
+            // would.
+            if vote.len() > MAX_PROTOCOLS_TO_EXPAND {
+                continue;
+            }
+
+            for (protocol, versions) in vote.iter() {
+                let supported_vers: &mut HashMap<Version, usize> =
+                    all_count.entry(protocol.clone()).or_insert(HashMap::new());
+
+                for version in versions.clone().expand() {
+                    let counter: &mut usize = supported_vers.entry(version).or_insert(0);
+                    *counter += 1;
+                }
+            }
+        }
+
+        for (protocol, mut versions) in all_count {
+            // Go through and remove versions that are less than the threshold
+            versions.retain(|_, count| *count as usize >= *threshold);
+
+            if versions.len() > 0 {
+                let voted_versions: Vec<Version> = versions.keys().cloned().collect();
+                let voted_protoset: ProtoSet = ProtoSet::from(voted_versions);
+
+                final_output.insert(protocol, voted_protoset);
+            }
+        }
+        final_output
+    }
+}
+
+/// Returns a boolean indicating whether the given protocol and version is
+/// supported in any of the existing Tor protocols
+///
+/// # Examples
+/// ```
+/// use protover::is_supported_here;
+/// use protover::Protocol;
+///
+/// let is_supported = is_supported_here(&Protocol::Link, &10);
+/// assert_eq!(false, is_supported);
+///
+/// let is_supported = is_supported_here(&Protocol::Link, &1);
+/// assert_eq!(true, is_supported);
+/// ```
+pub fn is_supported_here(proto: &Protocol, vers: &Version) -> bool {
+    let currently_supported: ProtoEntry = match ProtoEntry::supported() {
+        Ok(result) => result,
+        Err(_) => return false,
+    };
+    let supported_versions = match currently_supported.get(proto) {
+        Some(n) => n,
+        None => return false,
+    };
+    supported_versions.contains(vers)
+}
+
+/// Since older versions of Tor cannot infer their own subprotocols,
+/// determine which subprotocols are supported by older Tor versions.
+///
+/// # Inputs
+///
+/// * `version`, a string comprised of "[0-9a-z.-]"
+///
+/// # Returns
+///
+/// A `&'static CStr` encoding a list of protocol names and supported
+/// versions. The string takes the following format:
+///
+/// "HSDir=1-1 LinkAuth=1"
+///
+/// This function returns the protocols that are supported by the version input,
+/// only for tor versions older than `FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS`
+/// (but not older than 0.2.4.19).  For newer tors (or older than 0.2.4.19), it
+/// returns an empty string.
+///
+/// # Note
+///
+/// This function is meant to be called for/within FFI code.  If you'd
+/// like to use this code in Rust, please see `compute_for_old_tor()`.
+//
+// C_RUST_COUPLED: src/rust/protover.c `compute_for_old_tor`
+pub(crate) fn compute_for_old_tor_cstr(version: &str) -> &'static CStr {
+    let empty: &'static CStr = cstr!("");
+
+    if c_tor_version_as_new_as(version, FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS) {
+        return empty;
+    }
+    if c_tor_version_as_new_as(version, "0.2.9.1-alpha") {
+        return cstr!(
+            "Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1-2 \
+             Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2"
+        );
+    }
+    if c_tor_version_as_new_as(version, "0.2.7.5") {
+        return cstr!(
+            "Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 \
+             Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2"
+        );
+    }
+    if c_tor_version_as_new_as(version, "0.2.4.19") {
+        return cstr!(
+            "Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 \
+             Link=1-4 LinkAuth=1 Microdesc=1 Relay=1-2"
+        );
+    }
+    empty
+}
+
+/// Since older versions of Tor cannot infer their own subprotocols,
+/// determine which subprotocols are supported by older Tor versions.
+///
+/// # Inputs
+///
+/// * `version`, a string comprised of "[0-9a-z.-]"
+///
+/// # Returns
+///
+/// A `Result` whose `Ok` value is an `&'static str` encoding a list of protocol
+/// names and supported versions. The string takes the following format:
+///
+/// "HSDir=1-1 LinkAuth=1"
+///
+/// This function returns the protocols that are supported by the version input,
+/// only for tor versions older than `FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS`.
+/// (but not older than 0.2.4.19).  For newer tors (or older than 0.2.4.19), its
+/// `Ok` `Result` contains an empty string.
+///
+/// Otherwise, its `Err` contains a `ProtoverError::Unparseable` if the
+/// `version` string was invalid utf-8.
+///
+/// # Note
+///
+/// This function is meant to be called for/within non-FFI Rust code.
+//
+// C_RUST_COUPLED: src/rust/protover.c `compute_for_old_tor`
+pub fn compute_for_old_tor(version: &str) -> Result<&'static str, ProtoverError> {
+    // .to_str() fails with a Utf8Error if it couldn't validate the
+    // utf-8, so convert that here into an Unparseable ProtoverError.
+    compute_for_old_tor_cstr(version)
+        .to_str()
+        .or(Err(ProtoverError::Unparseable))
+}
+
+#[cfg(test)]
+mod test {
+    use std::str::FromStr;
+    use std::string::ToString;
+
+    use super::*;
+
+    macro_rules! parse_proto {
+        ($e:expr) => {{
+            let proto: Result<UnknownProtocol, _> = $e.parse();
+            let proto2 = UnknownProtocol::from_str_any_len($e);
+            assert_eq!(proto, proto2);
+            proto
+        }};
+    }
+
+    #[test]
+    fn test_protocol_from_str() {
+        assert!(parse_proto!("Cons").is_ok());
+        assert!(parse_proto!("123").is_ok());
+        assert!(parse_proto!("1-2-3").is_ok());
+
+        let err = Err(ProtoverError::InvalidProtocol);
+        assert_eq!(err, parse_proto!("a_b_c"));
+        assert_eq!(err, parse_proto!("a b"));
+        assert_eq!(err, parse_proto!("a,"));
+        assert_eq!(err, parse_proto!("b."));
+        assert_eq!(err, parse_proto!("é"));
+    }
+
+    macro_rules! assert_protoentry_is_parseable {
+        ($e:expr) => {
+            let protoentry: Result<ProtoEntry, ProtoverError> = $e.parse();
+
+            assert!(protoentry.is_ok(), format!("{:?}", protoentry.err()));
+        };
+    }
+
+    macro_rules! assert_protoentry_is_unparseable {
+        ($e:expr) => {
+            let protoentry: Result<ProtoEntry, ProtoverError> = $e.parse();
+
+            assert!(protoentry.is_err());
+        };
+    }
+
+    #[test]
+    fn test_protoentry_from_str_multiple_protocols_multiple_versions() {
+        assert_protoentry_is_parseable!("Cons=3-4 Link=1,3-5");
+    }
+
+    #[test]
+    fn test_protoentry_from_str_empty() {
+        assert_protoentry_is_unparseable!("");
+    }
+
+    #[test]
+    fn test_protoentry_from_str_single_protocol_single_version() {
+        assert_protoentry_is_parseable!("HSDir=1");
+    }
+
+    #[test]
+    fn test_protoentry_from_str_unknown_protocol() {
+        assert_protoentry_is_unparseable!("Ducks=5-7,8");
+    }
+
+    #[test]
+    fn test_protoentry_from_str_allowed_number_of_versions() {
+        assert_protoentry_is_parseable!("Desc=1-4294967294");
+    }
+
+    #[test]
+    fn test_protoentry_from_str_too_many_versions() {
+        assert_protoentry_is_unparseable!("Desc=1-4294967295");
+    }
+
+    #[test]
+    fn test_protoentry_from_str_() {
+        assert_protoentry_is_unparseable!("");
+    }
+
+    #[test]
+    fn test_protoentry_all_supported_single_protocol_single_version() {
+        let protocol: UnvalidatedProtoEntry = "Cons=1".parse().unwrap();
+        let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported();
+        assert_eq!(true, unsupported.is_none());
+    }
+
+    #[test]
+    fn test_protoentry_all_supported_multiple_protocol_multiple_versions() {
+        let protocols: UnvalidatedProtoEntry = "Link=3-4 Desc=2".parse().unwrap();
+        let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+        assert_eq!(true, unsupported.is_none());
+    }
+
+    #[test]
+    fn test_protoentry_all_supported_three_values() {
+        let protocols: UnvalidatedProtoEntry = "LinkAuth=1 Microdesc=1-2 Relay=2".parse().unwrap();
+        let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+        assert_eq!(true, unsupported.is_none());
+    }
+
+    #[test]
+    fn test_protoentry_all_supported_unknown_protocol() {
+        let protocols: UnvalidatedProtoEntry = "Wombat=9".parse().unwrap();
+        let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+        assert_eq!(true, unsupported.is_some());
+        assert_eq!("Wombat=9", &unsupported.unwrap().to_string());
+    }
+
+    #[test]
+    fn test_protoentry_all_supported_unsupported_high_version() {
+        let protocols: UnvalidatedProtoEntry = "HSDir=12-100".parse().unwrap();
+        let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+        assert_eq!(true, unsupported.is_some());
+        assert_eq!("HSDir=12-100", &unsupported.unwrap().to_string());
+    }
+
+    #[test]
+    fn test_protoentry_all_supported_unsupported_low_version() {
+        let protocols: UnvalidatedProtoEntry = "HSIntro=2-3".parse().unwrap();
+        let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+        assert_eq!(true, unsupported.is_some());
+        assert_eq!("HSIntro=2", &unsupported.unwrap().to_string());
+    }
+
+    #[test]
+    fn test_contract_protocol_list() {
+        let mut versions = "";
+        assert_eq!(
+            String::from(versions),
+            ProtoSet::from_str(&versions).unwrap().to_string()
+        );
+
+        versions = "1";
+        assert_eq!(
+            String::from(versions),
+            ProtoSet::from_str(&versions).unwrap().to_string()
+        );
+
+        versions = "1-2";
+        assert_eq!(
+            String::from(versions),
+            ProtoSet::from_str(&versions).unwrap().to_string()
+        );
+
+        versions = "1,3";
+        assert_eq!(
+            String::from(versions),
+            ProtoSet::from_str(&versions).unwrap().to_string()
+        );
+
+        versions = "1-4";
+        assert_eq!(
+            String::from(versions),
+            ProtoSet::from_str(&versions).unwrap().to_string()
+        );
+
+        versions = "1,3,5-7";
+        assert_eq!(
+            String::from(versions),
+            ProtoSet::from_str(&versions).unwrap().to_string()
+        );
+
+        versions = "1-3,500";
+        assert_eq!(
+            String::from(versions),
+            ProtoSet::from_str(&versions).unwrap().to_string()
+        );
+    }
+}
diff --git a/src/rust/protover/tests/protover.rs b/src/rust/protover/tests/protover.rs
new file mode 100644
index 0000000000..942fe3c6ab
--- /dev/null
+++ b/src/rust/protover/tests/protover.rs
@@ -0,0 +1,404 @@
+// Copyright (c) 2016-2019, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+extern crate protover;
+
+use protover::errors::ProtoverError;
+use protover::ProtoEntry;
+use protover::ProtoverVote;
+use protover::UnvalidatedProtoEntry;
+
+#[test]
+fn parse_protocol_with_single_proto_and_single_version() {
+    let _: ProtoEntry = "Cons=1".parse().unwrap();
+}
+
+#[test]
+fn parse_protocol_with_single_protocol_and_multiple_versions() {
+    let _: ProtoEntry = "Cons=1-2".parse().unwrap();
+}
+
+#[test]
+fn parse_protocol_with_different_single_protocol_and_single_version() {
+    let _: ProtoEntry = "HSDir=1".parse().unwrap();
+}
+
+#[test]
+fn parse_protocol_with_single_protocol_and_supported_version() {
+    let _: ProtoEntry = "Desc=2".parse().unwrap();
+}
+
+#[test]
+fn parse_protocol_with_two_protocols_and_single_version() {
+    let _: ProtoEntry = "Cons=1 HSDir=1".parse().unwrap();
+}
+
+#[test]
+fn parse_protocol_with_single_protocol_and_two_sequential_versions() {
+    let _: ProtoEntry = "Desc=1-2".parse().unwrap();
+}
+
+#[test]
+fn parse_protocol_with_single_protocol_and_protocol_range() {
+    let _: ProtoEntry = "Link=1-4".parse().unwrap();
+}
+
+#[test]
+fn parse_protocol_with_single_protocol_and_protocol_set() {
+    let _: ProtoEntry = "Link=3-4 Desc=2".parse().unwrap();
+}
+
+#[test]
+fn protocol_all_supported_with_single_protocol_and_protocol_set() {
+    let protocols: UnvalidatedProtoEntry = "Link=3-4 Desc=2".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+fn protocol_all_supported_with_two_values() {
+    let protocols: UnvalidatedProtoEntry = "Microdesc=1-2 Relay=2".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+fn protocol_all_supported_with_one_value() {
+    let protocols: UnvalidatedProtoEntry = "Microdesc=1-2".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+#[should_panic]
+fn parse_protocol_unvalidated_with_empty() {
+    let _: UnvalidatedProtoEntry = "".parse().unwrap();
+}
+
+#[test]
+#[should_panic]
+fn parse_protocol_validated_with_empty() {
+    let _: UnvalidatedProtoEntry = "".parse().unwrap();
+}
+
+#[test]
+fn protocol_all_supported_with_three_values() {
+    let protocols: UnvalidatedProtoEntry = "LinkAuth=1 Microdesc=1-2 Relay=2".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+fn protocol_all_supported_with_unsupported_protocol() {
+    let protocols: UnvalidatedProtoEntry = "Wombat=9".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    assert_eq!(true, unsupported.is_some());
+    assert_eq!("Wombat=9", &unsupported.unwrap().to_string());
+}
+
+#[test]
+fn protocol_all_supported_with_unsupported_versions() {
+    let protocols: UnvalidatedProtoEntry = "Link=3-999".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    assert_eq!(true, unsupported.is_some());
+    assert_eq!("Link=6-999", &unsupported.unwrap().to_string());
+}
+
+#[test]
+fn protocol_all_supported_with_unsupported_low_version() {
+    let protocols: UnvalidatedProtoEntry = "HSIntro=2-3".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    assert_eq!(true, unsupported.is_some());
+    assert_eq!("HSIntro=2", &unsupported.unwrap().to_string());
+}
+
+#[test]
+fn protocol_all_supported_with_unsupported_high_version() {
+    let protocols: UnvalidatedProtoEntry = "Cons=1-2,999".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    assert_eq!(true, unsupported.is_some());
+    assert_eq!("Cons=999", &unsupported.unwrap().to_string());
+}
+
+#[test]
+fn protocol_all_supported_with_mix_of_supported_and_unsupproted() {
+    let protocols: UnvalidatedProtoEntry = "Link=3-4 Wombat=9".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    assert_eq!(true, unsupported.is_some());
+    assert_eq!("Wombat=9", &unsupported.unwrap().to_string());
+}
+
+#[test]
+fn protover_string_supports_protocol_returns_true_for_single_supported() {
+    let protocols: UnvalidatedProtoEntry = "Link=3-4 Cons=1".parse().unwrap();
+    let is_supported = protocols.supports_protocol(&protover::Protocol::Cons.into(), &1);
+    assert_eq!(true, is_supported);
+}
+
+#[test]
+fn protover_string_supports_protocol_returns_false_for_single_unsupported() {
+    let protocols: UnvalidatedProtoEntry = "Link=3-4 Cons=1".parse().unwrap();
+    let is_supported = protocols.supports_protocol(&protover::Protocol::Cons.into(), &2);
+    assert_eq!(false, is_supported);
+}
+
+#[test]
+fn protover_string_supports_protocol_returns_false_for_unsupported() {
+    let protocols: UnvalidatedProtoEntry = "Link=3-4".parse().unwrap();
+    let is_supported = protocols.supports_protocol(&protover::Protocol::Cons.into(), &2);
+    assert_eq!(false, is_supported);
+}
+
+#[test]
+#[should_panic]
+fn parse_protocol_with_unexpected_characters() {
+    let _: UnvalidatedProtoEntry = "Cons=*-%".parse().unwrap();
+}
+
+#[test]
+#[should_panic]
+fn protover_compute_vote_returns_empty_for_empty_string() {
+    let protocols: &[UnvalidatedProtoEntry] = &["".parse().unwrap()];
+    let listed = ProtoverVote::compute(protocols, &1);
+    assert_eq!("", listed.to_string());
+}
+
+#[test]
+fn protover_compute_vote_returns_single_protocol_for_matching() {
+    let protocols: &[UnvalidatedProtoEntry] = &["Cons=1".parse().unwrap()];
+    let listed = ProtoverVote::compute(protocols, &1);
+    assert_eq!("Cons=1", listed.to_string());
+}
+
+#[test]
+fn protover_compute_vote_returns_two_protocols_for_two_matching() {
+    let protocols: &[UnvalidatedProtoEntry] = &["Link=1 Cons=1".parse().unwrap()];
+    let listed = ProtoverVote::compute(protocols, &1);
+    assert_eq!("Cons=1 Link=1", listed.to_string());
+}
+
+#[test]
+fn protover_compute_vote_returns_one_protocol_when_one_out_of_two_matches() {
+    let protocols: &[UnvalidatedProtoEntry] =
+        &["Cons=1 Link=2".parse().unwrap(), "Cons=1".parse().unwrap()];
+    let listed = ProtoverVote::compute(protocols, &2);
+    assert_eq!("Cons=1", listed.to_string());
+}
+
+#[test]
+fn protover_compute_vote_returns_protocols_that_it_doesnt_currently_support() {
+    let protocols: &[UnvalidatedProtoEntry] =
+        &["Foo=1 Cons=2".parse().unwrap(), "Bar=1".parse().unwrap()];
+    let listed = ProtoverVote::compute(protocols, &1);
+    assert_eq!("Bar=1 Cons=2 Foo=1", listed.to_string());
+}
+
+#[test]
+fn protover_compute_vote_returns_matching_for_mix() {
+    let protocols: &[UnvalidatedProtoEntry] = &["Link=1-10,500 Cons=1,3-7,8".parse().unwrap()];
+    let listed = ProtoverVote::compute(protocols, &1);
+    assert_eq!("Cons=1,3-8 Link=1-10,500", listed.to_string());
+}
+
+#[test]
+fn protover_compute_vote_returns_matching_for_longer_mix() {
+    let protocols: &[UnvalidatedProtoEntry] = &[
+        "Desc=1-10,500 Cons=1,3-7,8".parse().unwrap(),
+        "Link=123-456,78 Cons=2-6,8 Desc=9".parse().unwrap(),
+    ];
+
+    let listed = ProtoverVote::compute(protocols, &1);
+    assert_eq!("Cons=1-8 Desc=1-10,500 Link=78,123-456", listed.to_string());
+}
+
+#[test]
+fn protover_compute_vote_returns_matching_for_longer_mix_with_threshold_two() {
+    let protocols: &[UnvalidatedProtoEntry] = &[
+        "Desc=1-10,500 Cons=1,3-7,8".parse().unwrap(),
+        "Link=123-456,78 Cons=2-6,8 Desc=9".parse().unwrap(),
+    ];
+
+    let listed = ProtoverVote::compute(protocols, &2);
+    assert_eq!("Cons=3-6,8 Desc=9", listed.to_string());
+}
+
+#[test]
+fn protover_compute_vote_handles_duplicated_versions() {
+    let protocols: &[UnvalidatedProtoEntry] =
+        &["Cons=1".parse().unwrap(), "Cons=1".parse().unwrap()];
+    assert_eq!("Cons=1", ProtoverVote::compute(protocols, &2).to_string());
+
+    let protocols: &[UnvalidatedProtoEntry] =
+        &["Cons=1-2".parse().unwrap(), "Cons=1-2".parse().unwrap()];
+    assert_eq!("Cons=1-2", ProtoverVote::compute(protocols, &2).to_string());
+}
+
+#[test]
+fn protover_compute_vote_handles_invalid_proto_entries() {
+    let protocols: &[UnvalidatedProtoEntry] = &[
+        "Cons=1".parse().unwrap(),
+        "Cons=1".parse().unwrap(),
+        "Dinosaur=1".parse().unwrap(),
+    ];
+    assert_eq!("Cons=1", ProtoverVote::compute(protocols, &2).to_string());
+}
+
+#[test]
+fn parse_protocol_with_single_protocol_and_two_nonsequential_versions() {
+    let _: ProtoEntry = "Desc=1,2".parse().unwrap();
+}
+
+#[test]
+fn protover_is_supported_here_returns_true_for_supported_protocol() {
+    assert_eq!(
+        true,
+        protover::is_supported_here(&protover::Protocol::Cons, &1)
+    );
+}
+
+#[test]
+fn protover_is_supported_here_returns_false_for_unsupported_protocol() {
+    assert_eq!(
+        false,
+        protover::is_supported_here(&protover::Protocol::Cons, &5)
+    );
+}
+
+#[test]
+fn protocol_all_supported_with_single_proto_and_single_version() {
+    let protocol: UnvalidatedProtoEntry = "Cons=1".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+fn protocol_all_supported_with_single_protocol_and_multiple_versions() {
+    let protocol: UnvalidatedProtoEntry = "Cons=1-2".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+fn protocol_all_supported_with_different_single_protocol_and_single_version() {
+    let protocol: UnvalidatedProtoEntry = "HSDir=1".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+fn protocol_all_supported_with_single_protocol_and_supported_version() {
+    let protocol: UnvalidatedProtoEntry = "Desc=2".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+fn protocol_all_supported_with_two_protocols_and_single_version() {
+    let protocols: UnvalidatedProtoEntry = "Cons=1 HSDir=1".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+fn protocol_all_supported_with_single_protocol_and_two_nonsequential_versions() {
+    let protocol: UnvalidatedProtoEntry = "Desc=1,2".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+fn protocol_all_supported_with_single_protocol_and_two_sequential_versions() {
+    let protocol: UnvalidatedProtoEntry = "Desc=1-2".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+fn protocol_all_supported_with_single_protocol_and_protocol_range() {
+    let protocol: UnvalidatedProtoEntry = "Link=1-4".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+// By allowing us to add to votes, the C implementation allows us to
+// exceed the limit.
+#[test]
+fn protover_compute_vote_may_exceed_limit() {
+    let proto1: UnvalidatedProtoEntry = "Sleen=1-65535".parse().unwrap();
+    let proto2: UnvalidatedProtoEntry = "Sleen=100000".parse().unwrap();
+
+    let _result: UnvalidatedProtoEntry = ProtoverVote::compute(&[proto1, proto2], &1);
+}
+
+#[test]
+fn protover_all_supported_should_exclude_versions_we_actually_do_support() {
+    let proto: UnvalidatedProtoEntry = "Link=3-999".parse().unwrap();
+    let result: String = proto.all_supported().unwrap().to_string();
+
+    assert_eq!(result, "Link=6-999".to_string());
+}
+
+#[test]
+fn protover_all_supported_should_exclude_versions_we_actually_do_support_complex1() {
+    let proto: UnvalidatedProtoEntry = "Link=1-3,345-666".parse().unwrap();
+    let result: String = proto.all_supported().unwrap().to_string();
+
+    assert_eq!(result, "Link=345-666".to_string());
+}
+
+#[test]
+fn protover_all_supported_should_exclude_versions_we_actually_do_support_complex2() {
+    let proto: UnvalidatedProtoEntry = "Link=1-3,5-12".parse().unwrap();
+    let result: String = proto.all_supported().unwrap().to_string();
+
+    assert_eq!(result, "Link=6-12".to_string());
+}
+
+#[test]
+fn protover_all_supported_should_exclude_some_versions_and_entire_protocols() {
+    let proto: UnvalidatedProtoEntry = "Link=1-3,5-12 Quokka=9000-9001".parse().unwrap();
+    let result: String = proto.all_supported().unwrap().to_string();
+
+    assert_eq!(result, "Link=6-12 Quokka=9000-9001".to_string());
+}
+
+#[test]
+fn protover_all_supported_should_not_dos_anyones_computer() {
+    let proto: UnvalidatedProtoEntry = "Link=1-2147483648".parse().unwrap();
+    let result: String = proto.all_supported().unwrap().to_string();
+
+    assert_eq!(result, "Link=6-2147483648".to_string());
+}
+
+#[test]
+fn protover_all_supported_should_not_dos_anyones_computer_max_versions() {
+    let proto: UnvalidatedProtoEntry = "Link=1-4294967294".parse().unwrap();
+    let result: String = proto.all_supported().unwrap().to_string();
+
+    assert_eq!(result, "Link=6-4294967294".to_string());
+}
+
+#[test]
+// C_RUST_DIFFERS: The C will return true (e.g. saying "yes, that's supported")
+// but set the msg to NULL (??? seems maybe potentially bad).  The Rust will
+// simply return a None.
+fn protover_all_supported_should_return_empty_string_for_weird_thing() {
+    let proto: UnvalidatedProtoEntry = "Fribble=".parse().unwrap();
+    let result: Option<UnvalidatedProtoEntry> = proto.all_supported();
+
+    assert!(result.is_none());
+}
+
+#[test]
+fn protover_unvalidatedprotoentry_should_err_entirely_unparseable_things() {
+    let proto: Result<UnvalidatedProtoEntry, ProtoverError> = "Fribble".parse();
+
+    assert_eq!(Err(ProtoverError::Unparseable), proto);
+}
+
+#[test]
+fn protover_all_supported_over_maximum_limit() {
+    let proto: Result<UnvalidatedProtoEntry, ProtoverError> = "Sleen=1-4294967295".parse();
+
+    assert_eq!(Err(ProtoverError::ExceedsMax), proto);
+}
diff --git a/src/rust/smartlist/Cargo.toml b/src/rust/smartlist/Cargo.toml
new file mode 100644
index 0000000000..a5afe7bf74
--- /dev/null
+++ b/src/rust/smartlist/Cargo.toml
@@ -0,0 +1,18 @@
+[package]
+authors = ["The Tor Project"]
+version = "0.0.1"
+name = "smartlist"
+
+[dependencies]
+libc = "0.2.39"
+
+[lib]
+name = "smartlist"
+path = "lib.rs"
+
+[features]
+# We have to define a feature here because doctests don't get cfg(test),
+# and we need to disable some C dependencies when running the doctests
+# because of the various linker issues.  See
+# https://github.com/rust-lang/rust/issues/45599
+test_linking_hack = []
diff --git a/src/rust/smartlist/lib.rs b/src/rust/smartlist/lib.rs
new file mode 100644
index 0000000000..23301f88c3
--- /dev/null
+++ b/src/rust/smartlist/lib.rs
@@ -0,0 +1,17 @@
+// Copyright (c) 2016-2019, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+extern crate libc;
+
+mod smartlist;
+
+pub use smartlist::*;
+
+// When testing we may be compiled with sanitizers which are incompatible with
+// Rust's default allocator, jemalloc (unsure why at this time). Most crates
+// link to `tor_allocate` which switches by default to a non-jemalloc allocator,
+// but we don't already depend on `tor_allocate` so make sure that while testing
+// we don't use jemalloc. (but rather malloc/free)
+#[global_allocator]
+#[cfg(test)]
+static A: std::alloc::System = std::alloc::System;
diff --git a/src/rust/smartlist/smartlist.rs b/src/rust/smartlist/smartlist.rs
new file mode 100644
index 0000000000..d8f8083dff
--- /dev/null
+++ b/src/rust/smartlist/smartlist.rs
@@ -0,0 +1,115 @@
+// Copyright (c) 2016-2019, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+use libc::{c_char, c_int};
+use std::ffi::CStr;
+use std::slice;
+
+/// Smartlists are a type used in C code in tor to define a collection of a
+/// generic type, which has a capacity and a number used. Each Smartlist
+/// defines how to extract the list of values from the underlying C structure
+///
+/// Implementations are required to have a C representation, as this module
+/// serves purely to translate smartlists as defined in tor to vectors in Rust.
+pub trait Smartlist<T> {
+    fn get_list(&self) -> Vec<T>;
+}
+
+#[repr(C)]
+pub struct Stringlist {
+    pub list: *const *const c_char,
+    pub num_used: c_int,
+    pub capacity: c_int,
+}
+
+impl Smartlist<String> for Stringlist {
+    fn get_list(&self) -> Vec<String> {
+        let empty: Vec<String> = Vec::new();
+        let mut rust_list: Vec<String> = Vec::new();
+
+        if self.list.is_null() || self.num_used == 0 {
+            return empty;
+        }
+
+        // unsafe, as we need to extract the smartlist list into a vector of
+        // pointers, and then transform each element into a Rust string.
+        let elems: &[*const c_char] =
+            unsafe { slice::from_raw_parts(self.list, self.num_used as usize) };
+
+        for elem in elems.iter() {
+            if elem.is_null() {
+                continue;
+            }
+
+            // unsafe, as we need to create a cstring from the referenced
+            // element
+            let c_string = unsafe { CStr::from_ptr(*elem) };
+
+            let r_string = match c_string.to_str() {
+                Ok(n) => n,
+                Err(_) => return empty,
+            };
+
+            rust_list.push(String::from(r_string));
+        }
+
+        rust_list
+    }
+}
+
+// TODO: CHK: this module maybe should be tested from a test in C with a
+// smartlist as defined in tor.
+#[cfg(test)]
+mod test {
+    #[test]
+    fn test_get_list_of_strings() {
+        extern crate libc;
+
+        use libc::c_char;
+        use std::ffi::CString;
+
+        use super::Smartlist;
+        use super::Stringlist;
+
+        {
+            // test to verify that null pointers are gracefully handled
+            use std::ptr;
+
+            let sl = Stringlist {
+                list: ptr::null(),
+                num_used: 0,
+                capacity: 0,
+            };
+
+            let data = sl.get_list();
+            assert_eq!(0, data.len());
+        }
+
+        {
+            let args = vec![String::from("a"), String::from("b")];
+
+            // for each string, transform  it into a CString
+            let c_strings: Vec<_> = args
+                .iter()
+                .map(|arg| CString::new(arg.as_str()).unwrap())
+                .collect();
+
+            // then, collect a pointer for each CString
+            let p_args: Vec<_> = c_strings.iter().map(|arg| arg.as_ptr()).collect();
+
+            let p: *const *const c_char = p_args.as_ptr();
+
+            // This is the representation that we expect when receiving a
+            // smartlist at the Rust/C FFI layer.
+            let sl = Stringlist {
+                list: p,
+                num_used: 2,
+                capacity: 2,
+            };
+
+            let data = sl.get_list();
+            assert_eq!("a", &data[0]);
+            assert_eq!("b", &data[1]);
+        }
+    }
+}
diff --git a/src/rust/tor_allocate/Cargo.toml b/src/rust/tor_allocate/Cargo.toml
new file mode 100644
index 0000000000..06ac605f17
--- /dev/null
+++ b/src/rust/tor_allocate/Cargo.toml
@@ -0,0 +1,18 @@
+[package]
+authors = ["The Tor Project"]
+version = "0.0.1"
+name = "tor_allocate"
+
+[dependencies]
+libc = "=0.2.39"
+
+[lib]
+name = "tor_allocate"
+path = "lib.rs"
+
+[features]
+# We have to define a feature here because doctests don't get cfg(test),
+# and we need to disable some C dependencies when running the doctests
+# because of the various linker issues.  See
+# https://github.com/rust-lang/rust/issues/45599
+test_linking_hack = []
diff --git a/src/rust/tor_allocate/lib.rs b/src/rust/tor_allocate/lib.rs
new file mode 100644
index 0000000000..fff8a08006
--- /dev/null
+++ b/src/rust/tor_allocate/lib.rs
@@ -0,0 +1,20 @@
+// Copyright (c) 2016-2019, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+//! Allocation helper functions that allow data to be allocated in Rust
+//! using tor's specified allocator. In doing so, this can be later freed
+//! from C.
+//!
+//! This is currently a temporary solution, we will later use tor's allocator
+//! by default for any allocation that occurs in Rust. However, as this will
+//! stabalize in 2018, we can use this as a temporary measure.
+
+extern crate libc;
+
+use std::alloc::System;
+
+mod tor_allocate;
+pub use tor_allocate::*;
+
+#[global_allocator]
+static A: System = System;
diff --git a/src/rust/tor_allocate/tor_allocate.rs b/src/rust/tor_allocate/tor_allocate.rs
new file mode 100644
index 0000000000..682a524ee7
--- /dev/null
+++ b/src/rust/tor_allocate/tor_allocate.rs
@@ -0,0 +1,104 @@
+// Copyright (c) 2016-2019, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+// No-op defined purely for testing at the module level
+use libc::c_char;
+
+use libc::c_void;
+#[cfg(not(feature = "testing"))]
+use std::{mem, ptr, slice};
+
+// Define a no-op implementation for testing Rust modules without linking to C
+#[cfg(feature = "testing")]
+pub fn allocate_and_copy_string(s: &str) -> *mut c_char {
+    use std::ffi::CString;
+    CString::new(s).unwrap().into_raw()
+}
+
+// Defined only for tests, used for testing purposes, so that we don't need
+// to link to tor C files. Uses the system allocator
+#[cfg(test)]
+unsafe extern "C" fn tor_malloc_(size: usize) -> *mut c_void {
+    use libc::malloc;
+    malloc(size)
+}
+
+#[cfg(all(not(test), not(feature = "testing")))]
+extern "C" {
+    fn tor_malloc_(size: usize) -> *mut c_void;
+}
+
+/// Allocate memory using tor_malloc_ and copy an existing string into the
+/// allocated buffer, returning a pointer that can later be called in C.
+///
+/// # Inputs
+///
+/// * `src`, a reference to a String.
+///
+/// # Returns
+///
+/// A `*mut c_char` that should be freed by tor_free in C
+///
+#[cfg(not(feature = "testing"))]
+pub fn allocate_and_copy_string(src: &str) -> *mut c_char {
+    let bytes: &[u8] = src.as_bytes();
+
+    let size = mem::size_of_val::<[u8]>(bytes);
+    let size_one_byte = mem::size_of::<u8>();
+
+    // handle integer overflow when adding one to the calculated length
+    let size_with_null_byte = match size.checked_add(size_one_byte) {
+        Some(n) => n,
+        None => return ptr::null_mut(),
+    };
+
+    let dest = unsafe { tor_malloc_(size_with_null_byte) as *mut u8 };
+
+    if dest.is_null() {
+        return ptr::null_mut();
+    }
+
+    unsafe { ptr::copy_nonoverlapping(bytes.as_ptr(), dest, size) };
+
+    // set the last byte as null, using the ability to index into a slice
+    // rather than doing pointer arithmatic
+    let slice = unsafe { slice::from_raw_parts_mut(dest, size_with_null_byte) };
+    slice[size] = 0; // add a null terminator
+
+    dest as *mut c_char
+}
+
+#[cfg(test)]
+mod test {
+
+    #[test]
+    fn test_allocate_and_copy_string_with_empty() {
+        use libc::{c_void, free};
+        use std::ffi::CStr;
+
+        use tor_allocate::allocate_and_copy_string;
+
+        let allocated_empty = allocate_and_copy_string("");
+
+        let allocated_empty_rust = unsafe { CStr::from_ptr(allocated_empty).to_str().unwrap() };
+
+        assert_eq!("", allocated_empty_rust);
+
+        unsafe { free(allocated_empty as *mut c_void) };
+    }
+
+    #[test]
+    fn test_allocate_and_copy_string_with_not_empty_string() {
+        use libc::{c_void, free};
+        use std::ffi::CStr;
+
+        use tor_allocate::allocate_and_copy_string;
+
+        let allocated_empty = allocate_and_copy_string("foo bar biz");
+
+        let allocated_empty_rust = unsafe { CStr::from_ptr(allocated_empty).to_str().unwrap() };
+
+        assert_eq!("foo bar biz", allocated_empty_rust);
+
+        unsafe { free(allocated_empty as *mut c_void) };
+    }
+}
diff --git a/src/rust/tor_log/Cargo.toml b/src/rust/tor_log/Cargo.toml
new file mode 100644
index 0000000000..14d9ae803a
--- /dev/null
+++ b/src/rust/tor_log/Cargo.toml
@@ -0,0 +1,21 @@
+[package]
+name = "tor_log"
+version = "0.1.0"
+authors = ["The Tor Project"]
+
+[lib]
+name = "tor_log"
+path = "lib.rs"
+
+[features]
+# We have to define a feature here because doctests don't get cfg(test),
+# and we need to disable some C dependencies when running the doctests
+# because of the various linker issues.  See
+# https://github.com/rust-lang/rust/issues/45599
+test_linking_hack = []
+
+[dependencies]
+libc = "0.2.39"
+
+[dependencies.tor_allocate]
+path = "../tor_allocate"
diff --git a/src/rust/tor_log/lib.rs b/src/rust/tor_log/lib.rs
new file mode 100644
index 0000000000..4aa658e35b
--- /dev/null
+++ b/src/rust/tor_log/lib.rs
@@ -0,0 +1,16 @@
+//! Copyright (c) 2016-2019, The Tor Project, Inc. */
+//! See LICENSE for licensing information */
+
+//! Logging wrapper for Rust to utilize Tor's logger, found at
+//! src/common/log.c and src/common/torlog.h
+//!
+//! Exposes different interfaces depending on whether we are running in test
+//! or non-test mode. When testing, we use a no-op implementation,
+//! otherwise we link directly to C.
+
+extern crate libc;
+extern crate tor_allocate;
+
+mod tor_log;
+
+pub use tor_log::*;
diff --git a/src/rust/tor_log/tor_log.rs b/src/rust/tor_log/tor_log.rs
new file mode 100644
index 0000000000..98fccba5a9
--- /dev/null
+++ b/src/rust/tor_log/tor_log.rs
@@ -0,0 +1,265 @@
+// Copyright (c) 2016-2019, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+// Note that these functions are untested due to the fact that there are no
+// return variables to test and they are calling into a C API.
+
+/// The related domain which the logging message is relevant. For example,
+/// log messages relevant to networking would use LogDomain::LdNet, whereas
+/// general messages can use LdGeneral.
+#[derive(Eq, PartialEq)]
+pub enum LogDomain {
+    Net,
+    General,
+}
+
+/// The severity level at which to log messages.
+#[derive(Eq, PartialEq)]
+pub enum LogSeverity {
+    Notice,
+    Warn,
+}
+
+/// Main entry point for Rust modules to log messages.
+///
+/// # Inputs
+///
+/// * A `severity` of type LogSeverity, which defines the level of severity the
+/// message will be logged.
+/// * A `domain` of type LogDomain, which defines the domain the log message
+/// will be associated with.
+/// * A `function` of type &str, which defines the name of the function where
+/// the message is being logged. There is a current RFC for a macro that
+/// defines function names. When it is, we should use it. See
+/// https://github.com/rust-lang/rfcs/pull/1719
+/// * A `message` of type &str, which is the log message itself.
+#[macro_export]
+macro_rules! tor_log_msg {
+    ($severity: path,
+     $domain: path,
+     $function: expr,
+     $($message:tt)*) =>
+    {
+        {
+            let msg = format!($($message)*);
+            $crate::tor_log_msg_impl($severity, $domain, $function, msg)
+        }
+    };
+}
+
+#[inline]
+pub fn tor_log_msg_impl(severity: LogSeverity, domain: LogDomain, function: &str, message: String) {
+    use std::ffi::CString;
+
+    /// Default function name to log in case of errors when converting
+    /// a function name to a CString
+    const ERR_LOG_FUNCTION: &str = "tor_log_msg";
+
+    /// Default message to log in case of errors when converting a log
+    /// message to a CString
+    const ERR_LOG_MSG: &str = "Unable to log message from Rust \
+                               module due to error when converting to CString";
+
+    let func = match CString::new(function) {
+        Ok(n) => n,
+        Err(_) => CString::new(ERR_LOG_FUNCTION).unwrap(),
+    };
+
+    let msg = match CString::new(message) {
+        Ok(n) => n,
+        Err(_) => CString::new(ERR_LOG_MSG).unwrap(),
+    };
+
+    // Bind to a local variable to preserve ownership. This is essential so
+    // that ownership is guaranteed until these local variables go out of scope
+    let func_ptr = func.as_ptr();
+    let msg_ptr = msg.as_ptr();
+
+    let c_severity = unsafe { log::translate_severity(severity) };
+    let c_domain = unsafe { log::translate_domain(domain) };
+
+    unsafe { log::tor_log_string(c_severity, c_domain, func_ptr, msg_ptr) }
+}
+
+/// This implementation is used when compiling for actual use, as opposed to
+/// testing.
+#[cfg(not(test))]
+pub mod log {
+    use super::LogDomain;
+    use super::LogSeverity;
+    use libc::{c_char, c_int};
+
+    /// Severity log types. These mirror definitions in src/lib/log/log.h
+    /// C_RUST_COUPLED: src/lib/log/log.c, log domain types
+    extern "C" {
+        static LOG_WARN_: c_int;
+        static LOG_NOTICE_: c_int;
+    }
+
+    /// Domain log types. These mirror definitions in src/lib/log/log.h
+    /// C_RUST_COUPLED: src/lib/log/log.c, log severity types
+    extern "C" {
+        static LD_NET_: u32;
+        static LD_GENERAL_: u32;
+    }
+
+    /// Translate Rust defintions of log domain levels to C. This exposes a 1:1
+    /// mapping between types.
+    #[inline]
+    pub unsafe fn translate_domain(domain: LogDomain) -> u32 {
+        match domain {
+            LogDomain::Net => LD_NET_,
+            LogDomain::General => LD_GENERAL_,
+        }
+    }
+
+    /// Translate Rust defintions of log severity levels to C. This exposes a
+    /// 1:1 mapping between types.
+    #[inline]
+    pub unsafe fn translate_severity(severity: LogSeverity) -> c_int {
+        match severity {
+            LogSeverity::Warn => LOG_WARN_,
+            LogSeverity::Notice => LOG_NOTICE_,
+        }
+    }
+
+    /// The main entry point into Tor's logger. When in non-test mode, this
+    /// will link directly with `tor_log_string` in torlog.c
+    extern "C" {
+        pub fn tor_log_string(
+            severity: c_int,
+            domain: u32,
+            function: *const c_char,
+            string: *const c_char,
+        );
+    }
+}
+
+/// This module exposes no-op functionality for testing other Rust modules
+/// without linking to C.
+#[cfg(test)]
+pub mod log {
+    use super::LogDomain;
+    use super::LogSeverity;
+    use libc::{c_char, c_int};
+
+    pub static mut LAST_LOGGED_FUNCTION: *mut String = 0 as *mut String;
+    pub static mut LAST_LOGGED_MESSAGE: *mut String = 0 as *mut String;
+
+    pub unsafe fn tor_log_string(
+        _severity: c_int,
+        _domain: u32,
+        function: *const c_char,
+        message: *const c_char,
+    ) {
+        use std::ffi::CStr;
+
+        let f = CStr::from_ptr(function);
+        let fct = match f.to_str() {
+            Ok(n) => n,
+            Err(_) => "",
+        };
+        LAST_LOGGED_FUNCTION = Box::into_raw(Box::new(String::from(fct)));
+
+        let m = CStr::from_ptr(message);
+        let msg = match m.to_str() {
+            Ok(n) => n,
+            Err(_) => "",
+        };
+        LAST_LOGGED_MESSAGE = Box::into_raw(Box::new(String::from(msg)));
+    }
+
+    pub unsafe fn translate_domain(_domain: LogDomain) -> u32 {
+        1
+    }
+
+    pub unsafe fn translate_severity(_severity: LogSeverity) -> c_int {
+        1
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use tor_log::log::{LAST_LOGGED_FUNCTION, LAST_LOGGED_MESSAGE};
+    use tor_log::*;
+
+    #[test]
+    fn test_get_log_message() {
+        {
+            fn test_macro() {
+                tor_log_msg!(
+                    LogSeverity::Warn,
+                    LogDomain::Net,
+                    "test_macro",
+                    "test log message {}",
+                    "a",
+                );
+            }
+
+            test_macro();
+
+            let function = unsafe { Box::from_raw(LAST_LOGGED_FUNCTION) };
+            assert_eq!("test_macro", *function);
+
+            let message = unsafe { Box::from_raw(LAST_LOGGED_MESSAGE) };
+            assert_eq!("test log message a", *message);
+        }
+
+        // test multiple inputs into the log message
+        {
+            fn test_macro() {
+                tor_log_msg!(
+                    LogSeverity::Warn,
+                    LogDomain::Net,
+                    "next_test_macro",
+                    "test log message {} {} {} {} {}",
+                    1,
+                    2,
+                    3,
+                    4,
+                    5
+                );
+            }
+
+            test_macro();
+
+            let function = unsafe { Box::from_raw(LAST_LOGGED_FUNCTION) };
+            assert_eq!("next_test_macro", *function);
+
+            let message = unsafe { Box::from_raw(LAST_LOGGED_MESSAGE) };
+            assert_eq!("test log message 1 2 3 4 5", *message);
+        }
+
+        // test how a long log message will be formatted
+        {
+            fn test_macro() {
+                tor_log_msg!(
+                    LogSeverity::Warn,
+                    LogDomain::Net,
+                    "test_macro",
+                    "{}",
+                    "All the world's a stage, and all the men and women \
+                     merely players: they have their exits and their \
+                     entrances; and one man in his time plays many parts, his \
+                     acts being seven ages."
+                );
+            }
+
+            test_macro();
+
+            let expected_string = "All the world's a \
+                                   stage, and all the men \
+                                   and women merely players: \
+                                   they have their exits and \
+                                   their entrances; and one man \
+                                   in his time plays many parts, \
+                                   his acts being seven ages.";
+
+            let function = unsafe { Box::from_raw(LAST_LOGGED_FUNCTION) };
+            assert_eq!("test_macro", *function);
+
+            let message = unsafe { Box::from_raw(LAST_LOGGED_MESSAGE) };
+            assert_eq!(expected_string, *message);
+        }
+    }
+}
diff --git a/src/rust/tor_rust/Cargo.toml b/src/rust/tor_rust/Cargo.toml
new file mode 100644
index 0000000000..35c629882e
--- /dev/null
+++ b/src/rust/tor_rust/Cargo.toml
@@ -0,0 +1,22 @@
+[package]
+authors = ["The Tor Project"]
+name = "tor_rust"
+version = "0.1.0"
+
+[lib]
+name = "tor_rust"
+path = "lib.rs"
+crate_type = ["staticlib"]
+
+[dependencies.tor_util]
+path = "../tor_util"
+
+[dependencies.protover]
+path = "../protover"
+
+[features]
+# We have to define a feature here because doctests don't get cfg(test),
+# and we need to disable some C dependencies when running the doctests
+# because of the various linker issues.  See
+# https://github.com/rust-lang/rust/issues/45599
+test_linking_hack = []
diff --git a/src/rust/tor_rust/include.am b/src/rust/tor_rust/include.am
new file mode 100644
index 0000000000..ce673abbee
--- /dev/null
+++ b/src/rust/tor_rust/include.am
@@ -0,0 +1,28 @@
+EXTRA_DIST +=\
+	src/rust/tor_rust/Cargo.toml \
+	src/rust/tor_rust/lib.rs
+
+EXTRA_CARGO_OPTIONS=
+
+@TOR_RUST_LIB_PATH@: FORCE
+	( cd "$(abs_top_builddir)/src/rust" ; \
+		CARGO_TARGET_DIR="$(abs_top_builddir)/src/rust/target" \
+		$(CARGO) build --release $(EXTRA_CARGO_OPTIONS) \
+		$(CARGO_ONLINE) \
+		--manifest-path "$(abs_top_srcdir)/src/rust/tor_rust/Cargo.toml" )
+
+distclean-rust:
+	( cd "$(abs_top_builddir)/src/rust" ; \
+		CARGO_TARGET_DIR="$(abs_top_builddir)/src/rust/target" \
+		$(CARGO) clean $(EXTRA_CARGO_OPTIONS) \
+		$(CARGO_ONLINE) \
+		--manifest-path "$(abs_top_srcdir)/src/rust/tor_rust/Cargo.toml" )
+	rm -rf "$(abs_top_builddir)/src/rust/registry"
+
+if USE_RUST
+build-rust: @TOR_RUST_LIB_PATH@
+else
+build-rust:
+endif
+
+FORCE:
diff --git a/src/rust/tor_rust/lib.rs b/src/rust/tor_rust/lib.rs
new file mode 100644
index 0000000000..18519f8497
--- /dev/null
+++ b/src/rust/tor_rust/lib.rs
@@ -0,0 +1,5 @@
+extern crate protover;
+extern crate tor_util;
+
+pub use protover::*;
+pub use tor_util::*;
diff --git a/src/rust/tor_util/Cargo.toml b/src/rust/tor_util/Cargo.toml
new file mode 100644
index 0000000000..9ffaeda8a6
--- /dev/null
+++ b/src/rust/tor_util/Cargo.toml
@@ -0,0 +1,24 @@
+[package]
+authors = ["The Tor Project"]
+name = "tor_util"
+version = "0.0.1"
+
+[lib]
+name = "tor_util"
+path = "lib.rs"
+
+[dependencies.tor_allocate]
+path = "../tor_allocate"
+
+[dependencies.tor_log]
+path = "../tor_log"
+
+[dependencies]
+libc = "=0.2.39"
+
+[features]
+# We have to define a feature here because doctests don't get cfg(test),
+# and we need to disable some C dependencies when running the doctests
+# because of the various linker issues.  See
+# https://github.com/rust-lang/rust/issues/45599
+test_linking_hack = []
diff --git a/src/rust/tor_util/ffi.rs b/src/rust/tor_util/ffi.rs
new file mode 100644
index 0000000000..b71b2bd093
--- /dev/null
+++ b/src/rust/tor_util/ffi.rs
@@ -0,0 +1,27 @@
+// Copyright (c) 2016-2019, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+//! FFI functions to announce Rust support during tor startup, only to be
+//! called from C.
+//!
+
+use tor_log::{LogDomain, LogSeverity};
+
+/// Returns a short string to announce Rust support during startup.
+///
+/// # Examples
+/// ```c
+/// char *rust_str = rust_welcome_string();
+/// printf("%s", rust_str);
+/// tor_free(rust_str);
+/// ```
+#[no_mangle]
+pub extern "C" fn rust_log_welcome_string() {
+    tor_log_msg!(
+        LogSeverity::Notice,
+        LogDomain::General,
+        "rust_log_welcome_string",
+        "Tor is running with Rust integration. Please report \
+         any bugs you encounter."
+    );
+}
diff --git a/src/rust/tor_util/lib.rs b/src/rust/tor_util/lib.rs
new file mode 100644
index 0000000000..8886767ede
--- /dev/null
+++ b/src/rust/tor_util/lib.rs
@@ -0,0 +1,14 @@
+// Copyright (c) 2016-2019, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+//! Small module to announce Rust support during startup for demonstration
+//! purposes.
+
+extern crate libc;
+extern crate tor_allocate;
+
+#[macro_use]
+extern crate tor_log;
+
+pub mod ffi;
+pub mod strings;
diff --git a/src/rust/tor_util/strings.rs b/src/rust/tor_util/strings.rs
new file mode 100644
index 0000000000..2a19458f46
--- /dev/null
+++ b/src/rust/tor_util/strings.rs
@@ -0,0 +1,140 @@
+// Copyright (c) 2016-2019, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+//! Utilities for working with static strings.
+
+/// Create a `CStr` from a literal byte slice, appending a NUL byte to it first.
+///
+/// # Warning
+///
+/// The literal byte slice which is taken as an argument *MUST NOT* have any NUL
+/// bytes (`b"\0"`) in it, anywhere, or else an empty string will be returned
+/// (`CStr::from_bytes_with_nul_unchecked(b"\0")`) so as to avoid `panic!()`ing.
+///
+/// # Examples
+///
+/// ```
+/// #[macro_use]
+/// extern crate tor_util;
+///
+/// use std::ffi::CStr;
+///
+/// # fn do_test() -> Result<&'static CStr, &'static str> {
+/// let message: &'static str = "This is a test of the tsunami warning system.";
+/// let tuesday: &'static CStr;
+/// let original: &str;
+///
+/// tuesday = cstr!("This is a test of the tsunami warning system.");
+/// original = tuesday.to_str().or(Err("Couldn't unwrap CStr!"))?;
+///
+/// assert!(original == message);
+/// #
+/// # Ok(tuesday)
+/// # }
+/// # fn main() {
+/// #     do_test();  // so that we can use the ? operator in the test
+/// # }
+/// ```
+/// It is also possible to pass several string literals to this macro.  They
+/// will be concatenated together in the order of the arguments, unmodified,
+/// before finally being suffixed with a NUL byte:
+///
+/// ```
+/// #[macro_use]
+/// extern crate tor_util;
+/// #
+/// # use std::ffi::CStr;
+/// #
+/// # fn do_test() -> Result<&'static CStr, &'static str> {
+///
+/// let quux: &'static CStr = cstr!("foo", "bar", "baz");
+/// let orig: &'static str = quux.to_str().or(Err("Couldn't unwrap CStr!"))?;
+///
+/// assert!(orig == "foobarbaz");
+/// # Ok(quux)
+/// # }
+/// # fn main() {
+/// #     do_test();  // so that we can use the ? operator in the test
+/// # }
+/// ```
+/// This is useful for passing static strings to C from Rust FFI code.  To do so
+/// so, use the `.as_ptr()` method on the resulting `&'static CStr` to convert
+/// it to the Rust equivalent of a C `const char*`:
+///
+/// ```
+/// #[macro_use]
+/// extern crate tor_util;
+///
+/// use std::ffi::CStr;
+/// use std::os::raw::c_char;
+///
+/// pub extern "C" fn give_static_borrowed_string_to_c() -> *const c_char {
+///     let hello: &'static CStr = cstr!("Hello, language my parents wrote.");
+///
+///     hello.as_ptr()
+/// }
+/// # fn main() {
+/// #     let greetings = give_static_borrowed_string_to_c();
+/// # }
+/// ```
+/// Note that the C code this static borrowed string is passed to *MUST NOT*
+/// attempt to free the memory for the string.
+///
+/// # Note
+///
+/// An unfortunate limitation of the rustc compiler (as of 1.25.0-nightly), is
+/// that the first example above compiles, but if we were to change the
+/// assignment of `tuesday` as follows, it will fail to compile, because Rust
+/// macros are expanded at parse time, and at parse time there is no symbol
+/// table available.
+///
+/// ```ignore
+/// tuesday = cstr!(message);
+/// ```
+/// with the error message `error: expected a literal`.
+///
+/// # Returns
+///
+/// If the string literals passed as arguments contain no NUL bytes anywhere,
+/// then an `&'static CStr` containing the (concatenated) bytes of the string
+/// literal(s) passed as arguments, with a NUL byte appended, is returned.
+/// Otherwise, an `&'static CStr` containing a single NUL byte is returned (an
+/// "empty" string in C).
+#[macro_export]
+macro_rules! cstr {
+    ($($bytes:expr),*) => (
+        ::std::ffi::CStr::from_bytes_with_nul(
+            concat!($($bytes),*, "\0").as_bytes()
+        ).unwrap_or(
+            unsafe{
+                ::std::ffi::CStr::from_bytes_with_nul_unchecked(b"\0")
+            }
+        )
+    )
+}
+
+#[cfg(test)]
+mod test {
+    use std::ffi::CStr;
+
+    #[test]
+    fn cstr_macro() {
+        let _: &'static CStr = cstr!("boo");
+    }
+
+    #[test]
+    fn cstr_macro_multi_input() {
+        let quux: &'static CStr = cstr!("foo", "bar", "baz");
+
+        assert!(quux.to_str().unwrap() == "foobarbaz");
+    }
+
+    #[test]
+    fn cstr_macro_bad_input() {
+        let waving: &'static CStr = cstr!("waving not drowning o/");
+        let drowning: &'static CStr = cstr!("\0 drowning not waving");
+
+        assert!(waving.to_str().unwrap() == "waving not drowning o/");
+        assert!(drowning.to_str().unwrap() == "")
+    }
+}
diff --git a/src/test/Makefile.nmake b/src/test/Makefile.nmake
index 0ba56d7036..cfbe281b94 100644
--- a/src/test/Makefile.nmake
+++ b/src/test/Makefile.nmake
@@ -12,11 +12,13 @@ LIBS = ..\..\..\build-alpha\lib\libevent.lib \
  crypt32.lib gdi32.lib user32.lib
 
 TEST_OBJECTS = test.obj test_addr.obj test_channel.obj test_channeltls.obj \
-        test_containers.obj \
+        test_consdiff.obj test_containers.obj \
 	test_controller_events.obj test_crypto.obj test_data.obj test_dir.obj \
 	test_checkdir.obj test_microdesc.obj test_pt.obj test_util.obj \
         test_config.obj test_connection.obj \
 	test_cell_formats.obj test_relay.obj test_replay.obj \
+	test_channelpadding.obj \
+	test_circuitstats.obj \
 	test_scheduler.obj test_introduce.obj test_hs.obj tinytest.obj
 
 tinytest.obj: ..\ext\tinytest.c
diff --git a/src/test/bench.c b/src/test/bench.c
index 30984fda70..06c616c3b0 100644
--- a/src/test/bench.c
+++ b/src/test/bench.c
@@ -1,13 +1,8 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-extern const char tor_git_revision[];
-/* Ordinarily defined in tor_main.c; this bit is just here to provide one
- * since we're not linking to tor_main.c */
-const char tor_git_revision[] = "";
-
 /**
  * \file bench.c
  * \brief Benchmarks for lower level Tor modules.
@@ -15,19 +10,33 @@ const char tor_git_revision[] = "";
 
 #include "orconfig.h"
 
-#include "or.h"
-#include "onion_tap.h"
-#include "relay.h"
+#include "core/or/or.h"
+#include "core/crypto/onion_tap.h"
+#include "core/crypto/relay_crypto.h"
+
+#ifdef ENABLE_OPENSSL
 #include <openssl/opensslv.h>
 #include <openssl/evp.h>
 #include <openssl/ec.h>
 #include <openssl/ecdh.h>
 #include <openssl/obj_mac.h>
+#endif
+
+#include "core/or/circuitlist.h"
+#include "app/config/config.h"
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "lib/crypt_ops/crypto_dh.h"
+#include "core/crypto/onion_ntor.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "feature/dircommon/consdiff.h"
+#include "lib/compress/compress.h"
 
-#include "config.h"
-#include "crypto_curve25519.h"
-#include "onion_ntor.h"
-#include "crypto_ed25519.h"
+#include "core/or/cell_st.h"
+#include "core/or/or_circuit_st.h"
+
+#include "lib/crypt_ops/digestset.h"
+#include "lib/crypt_ops/crypto_init.h"
 
 #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID)
 static uint64_t nanostart;
@@ -57,7 +66,7 @@ perftime(void)
   return timespec_to_nsec(&ts) - nanostart;
 }
 
-#else
+#else /* !(defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID)) */
 static struct timeval tv_start = { 0, 0 };
 static void
 reset_perftime(void)
@@ -72,7 +81,7 @@ perftime(void)
   timersub(&now, &tv_start, &out);
   return ((uint64_t)out.tv_sec)*1000000000 + out.tv_usec*1000;
 }
-#endif
+#endif /* defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID) */
 
 #define NANOCOUNT(start,end,iters) \
   ( ((double)((end)-(start))) / (iters) )
@@ -120,7 +129,7 @@ bench_onion_TAP(void)
   uint64_t start, end;
   char os[TAP_ONIONSKIN_CHALLENGE_LEN];
   char or[TAP_ONIONSKIN_REPLY_LEN];
-  crypto_dh_t *dh_out;
+  crypto_dh_t *dh_out = NULL;
 
   key = crypto_pk_new();
   key2 = crypto_pk_new();
@@ -175,6 +184,7 @@ bench_onion_TAP(void)
          NANOCOUNT(start, end, iters)/1e3);
 
  done:
+  crypto_dh_free(dh_out);
   crypto_pk_free(key);
   crypto_pk_free(key2);
 }
@@ -198,6 +208,7 @@ bench_onion_ntor_impl(void)
   curve25519_public_key_generate(&keypair2.pubkey, &keypair2.seckey);
   dimap_add_entry(&keymap, keypair1.pubkey.public_key, &keypair1);
   dimap_add_entry(&keymap, keypair2.pubkey.public_key, &keypair2);
+  crypto_rand((char *)nodeid, sizeof(nodeid));
 
   reset_perftime();
   start = perftime();
@@ -372,7 +383,7 @@ bench_dmap(void)
     crypto_rand(d, 20);
     smartlist_add(sl2, tor_memdup(d, 20));
   }
-  printf("nbits=%d\n", ds->mask+1);
+  //printf("nbits=%d\n", ds->mask+1);
 
   reset_perftime();
 
@@ -400,18 +411,20 @@ bench_dmap(void)
          NANOCOUNT(pt3, pt4, iters*elts));
 
   for (i = 0; i < iters; ++i) {
-    SMARTLIST_FOREACH(sl, const char *, cp, n += digestset_contains(ds, cp));
-    SMARTLIST_FOREACH(sl2, const char *, cp, n += digestset_contains(ds, cp));
+    SMARTLIST_FOREACH(sl, const char *, cp,
+                      n += digestset_probably_contains(ds, cp));
+    SMARTLIST_FOREACH(sl2, const char *, cp,
+                      n += digestset_probably_contains(ds, cp));
   }
   end = perftime();
-  printf("digestset_contains: %.2f ns per element.\n",
+  printf("digestset_probably_contains: %.2f ns per element.\n",
          NANOCOUNT(pt4, end, iters*elts*2));
   /* We need to use this, or else the whole loop gets optimized out. */
   printf("Hits == %d\n", n);
 
   for (i = 0; i < fpostests; ++i) {
     crypto_rand(d, 20);
-    if (digestset_contains(ds, d)) ++fp;
+    if (digestset_probably_contains(ds, d)) ++fp;
   }
   printf("False positive rate on digestset: %.2f%%\n",
          (fp/(double)fpostests)*100);
@@ -460,18 +473,19 @@ bench_digest(void)
     for (int i = 0; lens[i] > 0; ++i) {
       reset_perftime();
       start = perftime();
+      int failures = 0;
       for (int j = 0; j < N; ++j) {
         switch (alg) {
           case DIGEST_SHA1:
-            crypto_digest(out, buf, lens[i]);
+            failures += crypto_digest(out, buf, lens[i]) < 0;
             break;
           case DIGEST_SHA256:
           case DIGEST_SHA3_256:
-            crypto_digest256(out, buf, lens[i], alg);
+            failures += crypto_digest256(out, buf, lens[i], alg) < 0;
             break;
           case DIGEST_SHA512:
           case DIGEST_SHA3_512:
-            crypto_digest512(out, buf, lens[i], alg);
+            failures += crypto_digest512(out, buf, lens[i], alg) < 0;
             break;
           default:
             tor_assert(0);
@@ -481,6 +495,8 @@ bench_digest(void)
       printf("%s(%d): %.2f ns per call\n",
              crypto_digest_algorithm_get_name(alg),
              lens[i], NANOCOUNT(start,end,N));
+      if (failures)
+        printf("ERROR: crypto_digest failed %d times.\n", failures);
     }
   }
 }
@@ -507,10 +523,10 @@ bench_cell_ops(void)
   char key1[CIPHER_KEY_LEN], key2[CIPHER_KEY_LEN];
   crypto_rand(key1, sizeof(key1));
   crypto_rand(key2, sizeof(key2));
-  or_circ->p_crypto = crypto_cipher_new(key1);
-  or_circ->n_crypto = crypto_cipher_new(key2);
-  or_circ->p_digest = crypto_digest_new();
-  or_circ->n_digest = crypto_digest_new();
+  or_circ->crypto.f_crypto = crypto_cipher_new(key1);
+  or_circ->crypto.b_crypto = crypto_cipher_new(key2);
+  or_circ->crypto.f_digest = crypto_digest_new();
+  or_circ->crypto.b_digest = crypto_digest_new();
 
   reset_perftime();
 
@@ -520,7 +536,8 @@ bench_cell_ops(void)
     for (i = 0; i < iters; ++i) {
       char recognized = 0;
       crypt_path_t *layer_hint = NULL;
-      relay_crypt(TO_CIRCUIT(or_circ), cell, d, &layer_hint, &recognized);
+      relay_decrypt_cell(TO_CIRCUIT(or_circ), cell, d,
+                         &layer_hint, &recognized);
     }
     end = perftime();
     printf("%sbound cells: %.2f ns per cell. (%.2f ns per byte of payload)\n",
@@ -529,10 +546,7 @@ bench_cell_ops(void)
            NANOCOUNT(start,end,iters*CELL_PAYLOAD_SIZE));
   }
 
-  crypto_digest_free(or_circ->p_digest);
-  crypto_digest_free(or_circ->n_digest);
-  crypto_cipher_free(or_circ->p_crypto);
-  crypto_cipher_free(or_circ->n_crypto);
+  relay_crypto_clear(&or_circ->crypto);
   tor_free(or_circ);
   tor_free(cell);
 }
@@ -547,8 +561,8 @@ bench_dh(void)
   reset_perftime();
   start = perftime();
   for (i = 0; i < iters; ++i) {
-    char dh_pubkey_a[DH_BYTES], dh_pubkey_b[DH_BYTES];
-    char secret_a[DH_BYTES], secret_b[DH_BYTES];
+    char dh_pubkey_a[DH1024_KEY_LEN], dh_pubkey_b[DH1024_KEY_LEN];
+    char secret_a[DH1024_KEY_LEN], secret_b[DH1024_KEY_LEN];
     ssize_t slen_a, slen_b;
     crypto_dh_t *dh_a = crypto_dh_new(DH_TYPE_TLS);
     crypto_dh_t *dh_b = crypto_dh_new(DH_TYPE_TLS);
@@ -572,6 +586,7 @@ bench_dh(void)
          "      %f millisec each.\n", NANOCOUNT(start, end, iters)/1e6);
 }
 
+#ifdef ENABLE_OPENSSL
 static void
 bench_ecdh_impl(int nid, const char *name)
 {
@@ -582,7 +597,7 @@ bench_ecdh_impl(int nid, const char *name)
   reset_perftime();
   start = perftime();
   for (i = 0; i < iters; ++i) {
-    char secret_a[DH_BYTES], secret_b[DH_BYTES];
+    char secret_a[DH1024_KEY_LEN], secret_b[DH1024_KEY_LEN];
     ssize_t slen_a, slen_b;
     EC_KEY *dh_a = EC_KEY_new_by_curve_name(nid);
     EC_KEY *dh_b = EC_KEY_new_by_curve_name(nid);
@@ -593,10 +608,10 @@ bench_ecdh_impl(int nid, const char *name)
 
     EC_KEY_generate_key(dh_a);
     EC_KEY_generate_key(dh_b);
-    slen_a = ECDH_compute_key(secret_a, DH_BYTES,
+    slen_a = ECDH_compute_key(secret_a, DH1024_KEY_LEN,
                               EC_KEY_get0_public_key(dh_b), dh_a,
                               NULL);
-    slen_b = ECDH_compute_key(secret_b, DH_BYTES,
+    slen_b = ECDH_compute_key(secret_b, DH1024_KEY_LEN,
                               EC_KEY_get0_public_key(dh_a), dh_b,
                               NULL);
 
@@ -621,6 +636,7 @@ bench_ecdh_p224(void)
 {
   bench_ecdh_impl(NID_secp224r1, "P-224");
 }
+#endif
 
 typedef void (*bench_fn)(void);
 
@@ -644,8 +660,11 @@ static struct benchmark_t benchmarks[] = {
   ENT(cell_aes),
   ENT(cell_ops),
   ENT(dh),
+
+#ifdef ENABLE_OPENSSL
   ENT(ecdh_p256),
   ENT(ecdh_p224),
+#endif
   {NULL,NULL,0}
 };
 
@@ -672,6 +691,28 @@ main(int argc, const char **argv)
   or_options_t *options;
 
   tor_threads_init();
+  tor_compress_init();
+  init_logging(1);
+
+  if (argc == 4 && !strcmp(argv[1], "diff")) {
+    const int N = 200;
+    char *f1 = read_file_to_str(argv[2], RFTS_BIN, NULL);
+    char *f2 = read_file_to_str(argv[3], RFTS_BIN, NULL);
+    if (! f1 || ! f2) {
+      perror("X");
+      return 1;
+    }
+    for (i = 0; i < N; ++i) {
+      char *diff = consensus_diff_generate(f1, f2);
+      tor_free(diff);
+    }
+    char *diff = consensus_diff_generate(f1, f2);
+    printf("%s", diff);
+    tor_free(f1);
+    tor_free(f2);
+    tor_free(diff);
+    return 0;
+  }
 
   for (i = 1; i < argc; ++i) {
     if (!strcmp(argv[i], "--list")) {
@@ -689,15 +730,18 @@ main(int argc, const char **argv)
 
   reset_perftime();
 
-  if (crypto_seed_rng() < 0) {
+  if (crypto_global_init(0, NULL, NULL) < 0) {
     printf("Couldn't seed RNG; exiting.\n");
     return 1;
   }
-  crypto_init_siphash_key();
+
+  init_protocol_warning_severity_level();
   options = options_new();
   init_logging(1);
   options->command = CMD_RUN_UNITTESTS;
   options->DataDirectory = tor_strdup("");
+  options->KeyDirectory = tor_strdup("");
+  options->CacheDirectory = tor_strdup("");
   options_init(options);
   if (set_options(options, &errmsg) < 0) {
     printf("Failed to set initial options: %s\n", errmsg);
@@ -715,4 +759,3 @@ main(int argc, const char **argv)
 
   return 0;
 }
-
diff --git a/src/test/bt_test.py b/src/test/bt_test.py
index a1efca00fb..f9ca79efde 100755
--- a/src/test/bt_test.py
+++ b/src/test/bt_test.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2015, The Tor Project, Inc
+# Copyright 2013-2019, The Tor Project, Inc
 # See LICENSE for licensing information
 
 """
diff --git a/src/test/ed25519_exts_ref.py b/src/test/ed25519_exts_ref.py
index d5a3a79910..75562184b5 100644
--- a/src/test/ed25519_exts_ref.py
+++ b/src/test/ed25519_exts_ref.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright 2014-2015, The Tor Project, Inc
+# Copyright 2014-2019, The Tor Project, Inc
 # See LICENSE for licensing information
 
 """
@@ -32,8 +32,7 @@ def curve25519ToEd25519(c, sign):
     return encodepoint([x,y])
 
 def blindESK(esk, param):
-    h = H("Derive temporary signing key" + param)
-    mult = 2**(b-2) + sum(2**i * bit(h,i) for i in range(3,b-2))
+    mult = 2**(b-2) + sum(2**i * bit(param,i) for i in range(3,b-2))
     s = decodeint(esk[:32])
     s_prime = (s * mult) % ell
     k = esk[32:]
@@ -42,8 +41,7 @@ def blindESK(esk, param):
     return encodeint(s_prime) + k_prime
 
 def blindPK(pk, param):
-    h = H("Derive temporary signing key" + param)
-    mult = 2**(b-2) + sum(2**i * bit(h,i) for i in range(3,b-2))
+    mult = 2**(b-2) + sum(2**i * bit(param,i) for i in range(3,b-2))
     P = decodepoint(pk)
     return encodepoint(scalarmult(P, mult))
 
@@ -69,6 +67,11 @@ def signatureWithESK(m,h,pk):
 def newSK():
     return os.urandom(32)
 
+def random_scalar(entropy_f): # 0..L-1 inclusive
+    # reduce the bias to a safe level by generating 256 extra bits
+    oversized = int(binascii.hexlify(entropy_f(32+32)), 16)
+    return oversized % ell
+
 # ------------------------------------------------------------
 
 MSG = "This is extremely silly. But it is also incredibly serious business!"
@@ -126,6 +129,31 @@ class SelfTest(unittest.TestCase):
 
         self._testSignatures(besk, bpk)
 
+    def testIdentity(self):
+        # Base point:
+        # B is the unique point (x, 4/5) \in E for which x is positive
+        By = 4 * inv(5)
+        Bx = xrecover(By)
+        B = [Bx % q,By % q]
+
+        # Get identity E by doing: E = l*B, where l is the group order
+        identity = scalarmult(B, ell)
+
+        # Get identity E by doing: E = l*A, where A is a random point
+        sk = newSK()
+        pk = decodepoint(publickey(sk))
+        identity2 = scalarmult(pk, ell)
+
+        # Check that identities match
+        assert(identity == identity2)
+        # Check that identity is the point (0,1)
+        assert(identity == [0L,1L])
+
+        # Check identity element: a*E = E, where a is a random scalar
+        scalar = random_scalar(os.urandom)
+        result = scalarmult(identity, scalar)
+        assert(result == identity == identity2)
+
 # ------------------------------------------------------------
 
 # From pprint.pprint([ binascii.b2a_hex(os.urandom(32)) for _ in xrange(8) ])
diff --git a/src/test/ed25519_vectors.inc b/src/test/ed25519_vectors.inc
index 760bafb971..60c863beba 100644
--- a/src/test/ed25519_vectors.inc
+++ b/src/test/ed25519_vectors.inc
@@ -91,21 +91,21 @@ static const char *ED25519_BLINDING_PARAMS[] = {
  * blinding parameter.
  */
 static const char *ED25519_BLINDED_SECRET_KEYS[] = {
-  "014e83abadb2ca9a27e0ffe23920333d817729f48700e97656ec2823d694050e171d43"
+  "293c3acff4e902f6f63ddc5d5caa2a57e771db4f24de65d4c28df3232f47fa01171d43"
       "f24e3f53e70ec7ac280044ac77d4942dee5d6807118a59bdf3ee647e89",
-  "fad8cca0b4335847795288b1452508752b253e64e6c7c78d4a02dbbd7d46aa0eb8ceff"
+  "38b88f9f9440358da544504ee152fb475528f7c51c285bd1c68b14ade8e29a07b8ceff"
       "20dfcf53eb52b891fc078c934efbf0353af7242e7dc51bb32a093afa29",
-  "116eb0ae0a4a91763365bdf86db427b00862db448487808788cc339ac10e5e089217f5"
+  "4d03ce16a3f3249846aac9de0a0075061495c3b027248eeee47da4ddbaf9e0049217f5"
       "2e92797462bd890fc274672e05c98f2c82970d640084781334aae0f940",
-  "bd1fbb0ee5acddc4adbcf5f33e95d9445f40326ce579fdd764a24483a9ccb20f509ece"
+  "51d7db01aaa0d937a9fd7c8c7381445a14d8fa61f43347af5460d7cd8fda9904509ece"
       "e77082ce088f7c19d5a00e955eeef8df6fa41686abc1030c2d76807733",
-  "237f5345cefe8573ce9fa7e216381a1172796c9e3f70668ab503b1352952530fb57b95"
+  "1f76cab834e222bd2546efa7e073425680ab88df186ff41327d3e40770129b00b57b95"
       "a440570659a440a3e4771465022a8e67af86bdf2d0990c54e7bb87ff9a",
-  "ba8ff23bc4ad2b739e1ccffc9fbc7837053ea81cdfdb15073f56411cfbae1d0ec492fc"
+  "c23588c23ee76093419d07b27c6df5922a03ac58f96c53671456a7d1bdbf560ec492fc"
       "87d5ec2a1b185ca5a40541fdef0b1e128fd5c2380c888bfa924711bcab",
-  "0fa68f969de038c7a90a4a74ee6167c77582006f2dedecc1956501ba6b6fb10391b476"
+  "3ed249c6932d076e1a2f6916975914b14e8c739da00992358b8f37d3e790650691b476"
       "8f8e556d78f4bdcb9a13b6f6066fe81d3134ae965dc48cd0785b3af2b8",
-  "deaa3456d1c21944d5dcd361a646858c6cf9336b0a6851d925717eb1ae186902053d9c"
+  "288cbfd923cb286d48c084555b5bdd06c05e92fb81acdb45271367f57515380e053d9c"
       "00c81e1331c06ab50087be8cfc7dc11691b132614474f1aa9c2503cccd",
 };
 
@@ -115,14 +115,14 @@ static const char *ED25519_BLINDED_SECRET_KEYS[] = {
  * blinding parameter.
  */
 static const char *ED25519_BLINDED_PUBLIC_KEYS[] = {
-  "722d6da6348e618967ef782e71061e27163a8b35f21856475d9d2023f65b6495",
-  "1dffa0586da6cbfcff2024eedf4fc6c818242d9a82dbbe635d6da1b975a1160d",
-  "5ed81f98fed5a6acda4ea6da2c34fab0ab359d950c510c256473f1f33ff438b4",
-  "6e6f92a54fb282120c46d9603df41135f025bc1f58f283809d04be96aeb04040",
-  "cda236f28edc4c7e02d18007b8dab49d669265b0f7aefb1824d7cc8e73a2cd63",
-  "367b03b17b67ca7329b89a520bdab91782402a41cd67264e34b5541a4b3f875b",
-  "8d486b03ac4e3b486b7a1d563706c7fdac75aee789a7cf6f22789eedeff61a31",
-  "9f297ff0aa2ceda91c5ab1b6446f12533d145940de6d850dc323417afde0cb78",
+  "1fc1fa4465bd9d4956fdbdc9d3acb3c7019bb8d5606b951c2e1dfe0b42eaeb41",
+  "1cbbd4a88ce8f165447f159d9f628ada18674158c4f7c5ead44ce8eb0fa6eb7e",
+  "c5419ad133ffde7e0ac882055d942f582054132b092de377d587435722deb028",
+  "3e08d0dc291066272e313014bfac4d39ad84aa93c038478a58011f431648105f",
+  "59381f06acb6bf1389ba305f70874eed3e0f2ab57cdb7bc69ed59a9b8899ff4d",
+  "2b946a484344eb1c17c89dd8b04196a84f3b7222c876a07a4cece85f676f87d9",
+  "c6b585129b135f8769df2eba987e76e089e80ba3a2a6729134d3b28008ac098e",
+  "0eefdc795b59cabbc194c6174e34ba9451e8355108520554ec285acabebb34ac",
 };
 
 /**
diff --git a/src/test/fakechans.h b/src/test/fakechans.h
index fa0e37dbe6..4006e1bec4 100644
--- a/src/test/fakechans.h
+++ b/src/test/fakechans.h
@@ -1,4 +1,4 @@
- /* Copyright (c) 2014-2016, The Tor Project, Inc. */
+ /* Copyright (c) 2014-2019, The Tor Project, Inc. */
  /* See LICENSE for licensing information */
 
 #ifndef TOR_FAKECHANS_H
@@ -20,7 +20,6 @@ void scheduler_release_channel_mock(channel_t *ch);
 
 /* Query some counters used by the exposed mocks */
 int get_mock_scheduler_has_waiting_cells_count(void);
-int get_mock_scheduler_release_channel_count(void);
 
 #endif /* !defined(TOR_FAKECHANS_H) */
 
diff --git a/src/test/fuzz/dict/consensus b/src/test/fuzz/dict/consensus
new file mode 100644
index 0000000000..3fcd9ee7ff
--- /dev/null
+++ b/src/test/fuzz/dict/consensus
@@ -0,0 +1,52 @@
+"a"
+"additional-digest"
+"additional-signature"
+"bandwidth-weights"
+"client-versions"
+"consensus-digest"
+"consensus-method"
+"consensus-methods"
+"contact"
+"dir-address"
+"directory-footer"
+"directory-signature"
+"dir-identity-key"
+"dir-key-certificate-version"
+"dir-key-certification"
+"dir-key-crosscert"
+"dir-key-expires"
+"dir-key-published"
+"dir-signing-key"
+"dir-source"
+"fingerprint"
+"fresh-until"
+"id"
+"known-flags"
+"legacy-dir-key"
+"m"
+"network-status-version"
+"opt"
+"p"
+"package"
+"params"
+"pr"
+"published"
+"r"
+"recommended-client-protocols"
+"recommended-relay-protocols"
+"required-client-protocols"
+"required-relay-protocols"
+"s"
+"server-versions"
+"shared-rand-commit"
+"shared-rand-current-value"
+"shared-rand-participate"
+"shared-rand-previous-value"
+"signing-ed25519"
+"v"
+"valid-after"
+"valid-until"
+"vote-digest"
+"vote-status"
+"voting-delay"
+"w"
diff --git a/src/test/fuzz/dict/descriptor b/src/test/fuzz/dict/descriptor
new file mode 100644
index 0000000000..110ee3e820
--- /dev/null
+++ b/src/test/fuzz/dict/descriptor
@@ -0,0 +1,41 @@
+"reject"
+"accept"
+"reject6"
+"accept6"
+"router"
+"ipv6-policy"
+"signing-key"
+"onion-key"
+"ntor-onion-key"
+"router-signature"
+"published"
+"uptime"
+"fingerprint"
+"hibernating"
+"platform"
+"proto"
+"contact"
+"read-history"
+"write-history"
+"extra-info-digest"
+"hidden-service-dir"
+"identity-ed25519"
+"master-key-ed25519"
+"router-sig-ed25519"
+"onion-key-crosscert"
+"ntor-onion-key-crosscert"
+"allow-single-hop-exits"
+"family"
+"caches-extra-info"
+"or-address"
+"opt"
+ "bandwidth"
+"@purpose"
+"tunnelled-dir-server"
+"-----BEGIN"
+"-----END"
+"-----"
+"ED25519 CERT"
+"RSA PUBLIC KEY"
+"CROSSCERT"
+"SIGNATURE"
diff --git a/src/test/fuzz/dict/extrainfo b/src/test/fuzz/dict/extrainfo
new file mode 100644
index 0000000000..eba7a1e4ce
--- /dev/null
+++ b/src/test/fuzz/dict/extrainfo
@@ -0,0 +1,32 @@
+"cell-circuits-per-decile"
+"cell-processed-cells"
+"cell-queued-cells"
+"cell-stats-end"
+"cell-time-in-queue"
+"dirreq-stats-end"
+"dirreq-v2-direct-dl"
+"dirreq-v2-ips"
+"dirreq-v2-reqs"
+"dirreq-v2-resp"
+"dirreq-v2-share"
+"dirreq-v2-tunneled-dl"
+"dirreq-v3-direct-dl"
+"dirreq-v3-ips"
+"dirreq-v3-reqs"
+"dirreq-v3-resp"
+"dirreq-v3-share"
+"dirreq-v3-tunneled-dl"
+"entry-ips"
+"entry-stats-end"
+"exit-kibibytes-read"
+"exit-kibibytes-written"
+"exit-stats-end"
+"exit-streams-opened"
+"extra-info"
+"identity-ed25519"
+"opt"
+"published"
+"read-history"
+"router-sig-ed25519"
+"router-signature"
+"write-history"
diff --git a/src/test/fuzz/dict/hsdescv2 b/src/test/fuzz/dict/hsdescv2
new file mode 100644
index 0000000000..48788301dc
--- /dev/null
+++ b/src/test/fuzz/dict/hsdescv2
@@ -0,0 +1,8 @@
+"introduction-points"
+"permanent-key"
+"protocol-versions"
+"publication-time"
+"rendezvous-service-descriptor"
+"secret-id-part"
+"signature"
+"version"
diff --git a/src/test/fuzz/dict/hsdescv3 b/src/test/fuzz/dict/hsdescv3
new file mode 100644
index 0000000000..84e8db578a
--- /dev/null
+++ b/src/test/fuzz/dict/hsdescv3
@@ -0,0 +1,6 @@
+"hs-descriptor"
+"descriptor-lifetime"
+"descriptor-signing-key-cert"
+"revision-counter"
+"superencrypted"
+"signature"
diff --git a/src/test/fuzz/dict/http b/src/test/fuzz/dict/http
new file mode 100644
index 0000000000..aa3dec990d
--- /dev/null
+++ b/src/test/fuzz/dict/http
@@ -0,0 +1,24 @@
+#
+# AFL dictionary for the Tor Directory protocol's HTTP headers
+# ------------------------------------------------------------
+#
+# Extracted from directory_handle_command() in the tor source code
+#
+# Copyright (c) 2016-2019, The Tor Project, Inc.
+# See LICENSE for licensing information
+#
+# Usage:
+# Select the dictionaries relevant to the part of the directory protocol you
+# are fuzzing, and feed them to your fuzzer (if it supports dictionaries).
+
+http_header_body_delimiter = "\x0d\x0a\x0d\x0a"
+http_header_header_delimiter = "\x0d\x0a"
+# multi-character tokens only
+#http_header_value_delimiter = " "
+
+content_length_header = "Content-Length:"
+forwarded_for_header = "Forwarded-For:"
+x_forwarded_for_header = "X-Forwarded-For:"
+
+get_command = "GET"
+post_command = "POST"
diff --git a/src/test/fuzz/dict/iptsv2 b/src/test/fuzz/dict/iptsv2
new file mode 100644
index 0000000000..57791c5e3c
--- /dev/null
+++ b/src/test/fuzz/dict/iptsv2
@@ -0,0 +1,6 @@
+"introduction-point"
+"ip-address"
+"onion-port"
+"onion-key"
+"service-key"
+
diff --git a/src/test/fuzz/dict/microdesc b/src/test/fuzz/dict/microdesc
new file mode 100644
index 0000000000..fdd0567b65
--- /dev/null
+++ b/src/test/fuzz/dict/microdesc
@@ -0,0 +1,7 @@
+"onion-key"
+"ntor-onion-key"
+"id"
+"a"
+"family"
+"p"
+"p6"
diff --git a/src/test/fuzz/fixup_filenames.sh b/src/test/fuzz/fixup_filenames.sh
new file mode 100755
index 0000000000..68efc1abc5
--- /dev/null
+++ b/src/test/fuzz/fixup_filenames.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+set -e
+
+if [ ! -d "$1" ] ; then
+    echo "I need a directory"
+    exit 1
+fi
+
+for fn in "$1"/* ; do
+    prev=`basename "$fn"`
+    post=`sha256sum "$fn" | sed -e 's/ .*//;'`
+    if [ "$prev" == "$post" ] ; then
+      echo "OK $prev"
+    else
+      echo "mv $prev $post"
+      mv "$fn" "$1/$post"
+    fi
+done
diff --git a/src/test/fuzz/fuzz_consensus.c b/src/test/fuzz/fuzz_consensus.c
new file mode 100644
index 0000000000..5947a3f48c
--- /dev/null
+++ b/src/test/fuzz/fuzz_consensus.c
@@ -0,0 +1,81 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#define SIGCOMMON_PRIVATE
+#include "core/or/or.h"
+#include "feature/dirparse/ns_parse.h"
+#include "feature/dirparse/sigcommon.h"
+#include "feature/dirparse/unparseable.h"
+#include "feature/nodelist/networkstatus.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "test/fuzz/fuzzing.h"
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+static int
+mock_router_produce_hash_final__nohash(char *digest,
+                                       const char *start, size_t len,
+                                       digest_algorithm_t alg)
+{
+  (void)start;
+  (void)len;
+  /* we could look at start[..] */
+  if (alg == DIGEST_SHA1)
+    memset(digest, 0x01, 20);
+  else
+    memset(digest, 0x02, 32);
+  return 0;
+}
+
+static int
+mock_signed_digest_equals__yes(const uint8_t *d1, const uint8_t *d2,
+                               size_t len)
+{
+  (void) tor_memeq(d1, d2, len);
+  return 1;
+}
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  MOCK(router_compute_hash_final, mock_router_produce_hash_final__nohash);
+  MOCK(signed_digest_equals, mock_signed_digest_equals__yes);
+  ed25519_init();
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  networkstatus_t *ns;
+  char *str = tor_memdup_nulterm(data, sz);
+  const char *eos = NULL;
+  networkstatus_type_t tp = NS_TYPE_CONSENSUS;
+  if (tor_memstr(data, MIN(sz, 1024), "tus vote"))
+    tp = NS_TYPE_VOTE;
+  const char *what = (tp == NS_TYPE_CONSENSUS) ? "consensus" : "vote";
+  ns = networkstatus_parse_vote_from_string(str,
+                                            &eos,
+                                            tp);
+  if (ns) {
+    log_debug(LD_GENERAL, "Parsing as %s okay", what);
+    networkstatus_vote_free(ns);
+  } else {
+    log_debug(LD_GENERAL, "Parsing as %s failed", what);
+  }
+  tor_free(str);
+  return 0;
+}
diff --git a/src/test/fuzz/fuzz_descriptor.c b/src/test/fuzz/fuzz_descriptor.c
new file mode 100644
index 0000000000..58ee3dbc35
--- /dev/null
+++ b/src/test/fuzz/fuzz_descriptor.c
@@ -0,0 +1,81 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#define SIGCOMMON_PRIVATE
+#include "core/or/or.h"
+#include "feature/dirparse/routerparse.h"
+#include "feature/dirparse/sigcommon.h"
+#include "feature/dirparse/unparseable.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/torcert.h"
+#include "feature/keymgt/loadkey.h"
+#include "test/fuzz/fuzzing.h"
+
+static int
+mock_check_tap_onion_key_crosscert__nocheck(const uint8_t *crosscert,
+                                   int crosscert_len,
+                                   const crypto_pk_t *onion_pkey,
+                                   const ed25519_public_key_t *master_id_pkey,
+                                   const uint8_t *rsa_id_digest)
+{
+  tor_assert(crosscert && onion_pkey && master_id_pkey && rsa_id_digest);
+  /* we could look at crosscert[..] */
+  (void) crosscert_len;
+  return 0;
+}
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+static int
+mock_router_produce_hash_final__nohash(char *digest,
+                                       const char *start, size_t len,
+                                       digest_algorithm_t alg)
+{
+  (void)start;
+  (void)len;
+  /* we could look at start[..] */
+  if (alg == DIGEST_SHA1)
+    memset(digest, 0x01, 20);
+  else
+    memset(digest, 0x02, 32);
+  return 0;
+}
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(check_tap_onion_key_crosscert,
+       mock_check_tap_onion_key_crosscert__nocheck);
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  MOCK(router_compute_hash_final, mock_router_produce_hash_final__nohash);
+  ed25519_init();
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  routerinfo_t *ri;
+  const char *str = (const char*) data;
+  ri = router_parse_entry_from_string((const char *)str,
+                                      str+sz,
+                                      0, 0, 0, NULL);
+  if (ri) {
+    log_debug(LD_GENERAL, "Parsing okay");
+    routerinfo_free(ri);
+  } else {
+    log_debug(LD_GENERAL, "Parsing failed");
+  }
+  return 0;
+}
diff --git a/src/test/fuzz/fuzz_diff.c b/src/test/fuzz/fuzz_diff.c
new file mode 100644
index 0000000000..1bc60e50ee
--- /dev/null
+++ b/src/test/fuzz/fuzz_diff.c
@@ -0,0 +1,69 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CONSDIFF_PRIVATE
+
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "feature/dircommon/consdiff.h"
+
+#include "test/fuzz/fuzzing.h"
+
+static int
+mock_consensus_compute_digest_(const char *c, consensus_digest_t *d)
+{
+  (void)c;
+  memset(d->sha3_256, 3, sizeof(d->sha3_256));
+  return 0;
+}
+
+int
+fuzz_init(void)
+{
+  MOCK(consensus_compute_digest, mock_consensus_compute_digest_);
+  MOCK(consensus_compute_digest_as_signed, mock_consensus_compute_digest_);
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  UNMOCK(consensus_compute_digest);
+  UNMOCK(consensus_compute_digest_as_signed);
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *stdin_buf, size_t data_size)
+{
+#define SEP "=====\n"
+#define SEPLEN strlen(SEP)
+  const uint8_t *separator = tor_memmem(stdin_buf, data_size, SEP, SEPLEN);
+  if (! separator)
+    return 0;
+  size_t c1_len = separator - stdin_buf;
+  char *c1 = tor_memdup_nulterm(stdin_buf, c1_len);
+  size_t c2_len = data_size - c1_len - SEPLEN;
+  char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len);
+
+  char *c3 = consensus_diff_generate(c1, c2);
+
+  if (c3) {
+    char *c4 = consensus_diff_apply(c1, c3);
+    tor_assert(c4);
+    if (strcmp(c2, c4)) {
+      printf("%s\n", escaped(c1));
+      printf("%s\n", escaped(c2));
+      printf("%s\n", escaped(c3));
+      printf("%s\n", escaped(c4));
+    }
+    tor_assert(! strcmp(c2, c4));
+    tor_free(c3);
+    tor_free(c4);
+  }
+  tor_free(c1);
+  tor_free(c2);
+
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_diff_apply.c b/src/test/fuzz/fuzz_diff_apply.c
new file mode 100644
index 0000000000..9bd3cb0bf8
--- /dev/null
+++ b/src/test/fuzz/fuzz_diff_apply.c
@@ -0,0 +1,65 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CONSDIFF_PRIVATE
+
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "feature/dircommon/consdiff.h"
+
+#include "test/fuzz/fuzzing.h"
+
+static int
+mock_consensus_compute_digest_(const char *c, consensus_digest_t *d)
+{
+  (void)c;
+  memset(d->sha3_256, 3, sizeof(d->sha3_256));
+  return 0;
+}
+
+static int
+mock_consensus_digest_eq_(const uint8_t *a, const uint8_t *b)
+{
+  (void)a;
+  (void)b;
+  return 1;
+}
+
+int
+fuzz_init(void)
+{
+  MOCK(consensus_compute_digest, mock_consensus_compute_digest_);
+  MOCK(consensus_digest_eq, mock_consensus_digest_eq_);
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  UNMOCK(consensus_compute_digest);
+  UNMOCK(consensus_digest_eq);
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *stdin_buf, size_t data_size)
+{
+#define SEP "=====\n"
+#define SEPLEN strlen(SEP)
+  const uint8_t *separator = tor_memmem(stdin_buf, data_size, SEP, SEPLEN);
+  if (! separator)
+    return 0;
+  size_t c1_len = separator - stdin_buf;
+  char *c1 = tor_memdup_nulterm(stdin_buf, c1_len);
+  size_t c2_len = data_size - c1_len - SEPLEN;
+  char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len);
+
+  char *c3 = consensus_diff_apply(c1, c2);
+
+  tor_free(c1);
+  tor_free(c2);
+  tor_free(c3);
+
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_extrainfo.c b/src/test/fuzz/fuzz_extrainfo.c
new file mode 100644
index 0000000000..f18bd68d65
--- /dev/null
+++ b/src/test/fuzz/fuzz_extrainfo.c
@@ -0,0 +1,67 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#define SIGCOMMON_PRIVATE
+#include "core/or/or.h"
+#include "feature/dirparse/routerparse.h"
+#include "feature/dirparse/sigcommon.h"
+#include "feature/dirparse/unparseable.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/routerkeys.h"
+#include "test/fuzz/fuzzing.h"
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+static int
+mock_router_produce_hash_final__nohash(char *digest,
+                                       const char *start, size_t len,
+                                       digest_algorithm_t alg)
+{
+  (void)start;
+  (void)len;
+  /* we could look at start[..] */
+  if (alg == DIGEST_SHA1)
+    memset(digest, 0x01, 20);
+  else
+    memset(digest, 0x02, 32);
+  return 0;
+}
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  MOCK(router_compute_hash_final, mock_router_produce_hash_final__nohash);
+  ed25519_init();
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  extrainfo_t *ei;
+  const char *str = (const char*) data;
+  int again = 0;
+  ei = extrainfo_parse_entry_from_string((const char *)str,
+                                         str+sz,
+                                         0, NULL, &again);
+  if (ei) {
+    log_debug(LD_GENERAL, "Parsing okay");
+    extrainfo_free(ei);
+  } else {
+    log_debug(LD_GENERAL, "Parsing failed");
+  }
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_hsdescv2.c b/src/test/fuzz/fuzz_hsdescv2.c
new file mode 100644
index 0000000000..34639b237c
--- /dev/null
+++ b/src/test/fuzz/fuzz_hsdescv2.c
@@ -0,0 +1,52 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#include "core/or/or.h"
+#include "feature/dirparse/unparseable.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/rend/rendparse.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "test/fuzz/fuzzing.h"
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  ed25519_init();
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  rend_service_descriptor_t *desc = NULL;
+  char desc_id[64];
+  char *ipts = NULL;
+  size_t ipts_size, esize;
+  const char *next;
+  char *str = tor_memdup_nulterm(data, sz);
+  (void) rend_parse_v2_service_descriptor(&desc, desc_id, &ipts, &ipts_size,
+                                          &esize, &next, str, 1);
+  if (desc) {
+    log_debug(LD_GENERAL, "Parsing okay");
+    rend_service_descriptor_free(desc);
+  } else {
+    log_debug(LD_GENERAL, "Parsing failed");
+  }
+  tor_free(ipts);
+  tor_free(str);
+  return 0;
+}
diff --git a/src/test/fuzz/fuzz_hsdescv3.c b/src/test/fuzz/fuzz_hsdescv3.c
new file mode 100644
index 0000000000..2cbd655898
--- /dev/null
+++ b/src/test/fuzz/fuzz_hsdescv3.c
@@ -0,0 +1,99 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define HS_DESCRIPTOR_PRIVATE
+
+#include "core/or/or.h"
+#include "trunnel/ed25519_cert.h" /* Trunnel interface. */
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "feature/hs/hs_descriptor.h"
+#include "feature/dirparse/unparseable.h"
+
+#include "test/fuzz/fuzzing.h"
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+static int
+mock_rsa_ed25519_crosscert_check(const uint8_t *crosscert,
+                                 const size_t crosscert_len,
+                                 const crypto_pk_t *rsa_id_key,
+                                 const ed25519_public_key_t *master_key,
+                                 const time_t reject_if_expired_before)
+{
+  (void) crosscert;
+  (void) crosscert_len;
+  (void) rsa_id_key;
+  (void) master_key;
+  (void) reject_if_expired_before;
+  return 0;
+}
+
+static size_t
+mock_decrypt_desc_layer(const hs_descriptor_t *desc,
+                        const uint8_t *encrypted_blob,
+                        size_t encrypted_blob_size,
+                        const uint8_t *descriptor_cookie,
+                        int is_superencrypted_layer,
+                        char **decrypted_out)
+{
+  (void)is_superencrypted_layer;
+  (void)desc;
+  (void)descriptor_cookie;
+  const size_t overhead = HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN;
+  if (encrypted_blob_size < overhead)
+    return 0;
+  *decrypted_out = tor_memdup_nulterm(
+                   encrypted_blob + HS_DESC_ENCRYPTED_SALT_LEN,
+                   encrypted_blob_size - overhead);
+  size_t result = strlen(*decrypted_out);
+  if (result) {
+    return result;
+  } else {
+    tor_free(*decrypted_out);
+    return 0;
+  }
+}
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  MOCK(rsa_ed25519_crosscert_check, mock_rsa_ed25519_crosscert_check);
+  MOCK(decrypt_desc_layer, mock_decrypt_desc_layer);
+  ed25519_init();
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  hs_descriptor_t *desc = NULL;
+  uint8_t subcredential[DIGEST256_LEN];
+
+  char *fuzzing_data = tor_memdup_nulterm(data, sz);
+  memset(subcredential, 'A', sizeof(subcredential));
+
+  hs_desc_decode_descriptor(fuzzing_data, subcredential, NULL, &desc);
+  if (desc) {
+    log_debug(LD_GENERAL, "Decoding okay");
+    hs_descriptor_free(desc);
+  } else {
+    log_debug(LD_GENERAL, "Decoding failed");
+  }
+
+  tor_free(fuzzing_data);
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_http.c b/src/test/fuzz/fuzz_http.c
new file mode 100644
index 0000000000..2798c47d23
--- /dev/null
+++ b/src/test/fuzz/fuzz_http.c
@@ -0,0 +1,134 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#define BUFFERS_PRIVATE
+#define DIRCACHE_PRIVATE
+
+#include "core/or/or.h"
+#include "lib/err/backtrace.h"
+#include "lib/container/buffers.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "feature/dircache/dircache.h"
+#include "lib/log/log.h"
+
+#include "feature/dircommon/dir_connection_st.h"
+
+#include "test/fuzz/fuzzing.h"
+
+static void
+mock_connection_write_to_buf_impl_(const char *string, size_t len,
+                                   connection_t *conn, int compressed)
+{
+  log_debug(LD_GENERAL, "%sResponse:\n%u\nConnection: %p\n%s\n",
+            compressed ? "Compressed " : "", (unsigned)len, conn, string);
+}
+
+static int
+mock_directory_handle_command_get(dir_connection_t *conn,
+                                      const char *headers,
+                                      const char *body,
+                                      size_t body_len)
+{
+  (void)conn;
+
+  log_debug(LD_GENERAL, "Method:\nGET\n");
+
+  if (headers) {
+    log_debug(LD_GENERAL, "Header-Length:\n%u\n", (unsigned)strlen(headers));
+    log_debug(LD_GENERAL, "Headers:\n%s\n", headers);
+  }
+
+  log_debug(LD_GENERAL, "Body-Length:\n%u\n", (unsigned)body_len);
+  if (body) {
+    log_debug(LD_GENERAL, "Body:\n%s\n", body);
+  }
+
+  /* Always tell the caller we succeeded */
+  return 0;
+}
+
+static int
+mock_directory_handle_command_post(dir_connection_t *conn,
+                                       const char *headers,
+                                       const char *body,
+                                       size_t body_len)
+{
+  (void)conn;
+
+  log_debug(LD_GENERAL, "Method:\nPOST\n");
+
+  if (headers) {
+    log_debug(LD_GENERAL, "Header-Length:\n%u\n", (unsigned)strlen(headers));
+    log_debug(LD_GENERAL, "Headers:\n%s\n", headers);
+  }
+
+  log_debug(LD_GENERAL, "Body-Length:\n%u\n", (unsigned)body_len);
+  if (body) {
+    log_debug(LD_GENERAL, "Body:\n%s\n", body);
+  }
+
+  /* Always tell the caller we succeeded */
+  return 0;
+}
+
+int
+fuzz_init(void)
+{
+  /* Set up fake response handler */
+  MOCK(connection_write_to_buf_impl_, mock_connection_write_to_buf_impl_);
+  /* Set up the fake handler functions */
+  MOCK(directory_handle_command_get, mock_directory_handle_command_get);
+  MOCK(directory_handle_command_post, mock_directory_handle_command_post);
+
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  UNMOCK(connection_write_to_buf_impl_);
+  UNMOCK(directory_handle_command_get);
+  UNMOCK(directory_handle_command_post);
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *stdin_buf, size_t data_size)
+{
+  dir_connection_t dir_conn;
+
+  /* Set up the fake connection */
+  memset(&dir_conn, 0, sizeof(dir_connection_t));
+  dir_conn.base_.type = CONN_TYPE_DIR;
+  /* Apparently tor sets this before directory_handle_command() is called. */
+  dir_conn.base_.address = tor_strdup("replace-this-address.example.com");
+
+  dir_conn.base_.inbuf = buf_new_with_data((char*)stdin_buf, data_size);
+  if (!dir_conn.base_.inbuf) {
+    log_debug(LD_GENERAL, "Zero-Length-Input\n");
+    goto done;
+  }
+
+  /* Parse the headers */
+  int rv = directory_handle_command(&dir_conn);
+
+  /* TODO: check the output is correctly parsed based on the input */
+
+  /* Report the parsed origin address */
+  if (dir_conn.base_.address) {
+    log_debug(LD_GENERAL, "Address:\n%s\n", dir_conn.base_.address);
+  }
+
+  log_debug(LD_GENERAL, "Result:\n%d\n", rv);
+
+ done:
+  /* Reset. */
+  tor_free(dir_conn.base_.address);
+  buf_free(dir_conn.base_.inbuf);
+  dir_conn.base_.inbuf = NULL;
+
+  return 0;
+}
diff --git a/src/test/fuzz/fuzz_http_connect.c b/src/test/fuzz/fuzz_http_connect.c
new file mode 100644
index 0000000000..a60fc36804
--- /dev/null
+++ b/src/test/fuzz/fuzz_http_connect.c
@@ -0,0 +1,109 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#define BUFFERS_PRIVATE
+#define CONNECTION_EDGE_PRIVATE
+
+#include "core/or/or.h"
+#include "lib/err/backtrace.h"
+#include "lib/container/buffers.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/connection_edge.h"
+#include "core/proto/proto_socks.h"
+#include "lib/log/log.h"
+
+#include "core/or/entry_connection_st.h"
+#include "core/or/socks_request_st.h"
+
+#include "test/fuzz/fuzzing.h"
+
+static void
+mock_connection_write_to_buf_impl_(const char *string, size_t len,
+                                   connection_t *conn, int compressed)
+{
+  log_debug(LD_GENERAL, "%sResponse:\n%u\nConnection: %p\n%s\n",
+            compressed ? "Compressed " : "", (unsigned)len, conn, string);
+}
+
+static void
+mock_connection_mark_unattached_ap_(entry_connection_t *conn, int endreason,
+                                    int line, const char *file)
+{
+  (void)conn;
+  (void)endreason;
+  (void)line;
+  (void)file;
+}
+
+static int
+mock_connection_ap_rewrite_and_attach_if_allowed(entry_connection_t *conn,
+                                                 origin_circuit_t *circ,
+                                                 crypt_path_t *cpath)
+{
+  (void)conn;
+  (void)circ;
+  (void)cpath;
+  return 0;
+}
+
+int
+fuzz_init(void)
+{
+  /* Set up fake response handler */
+  MOCK(connection_write_to_buf_impl_, mock_connection_write_to_buf_impl_);
+  /* Set up the fake handler functions */
+  MOCK(connection_mark_unattached_ap_, mock_connection_mark_unattached_ap_);
+  MOCK(connection_ap_rewrite_and_attach_if_allowed,
+       mock_connection_ap_rewrite_and_attach_if_allowed);
+
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  UNMOCK(connection_write_to_buf_impl_);
+  UNMOCK(connection_mark_unattached_ap_);
+  UNMOCK(connection_ap_rewrite_and_attach_if_allowed);
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *stdin_buf, size_t data_size)
+{
+  entry_connection_t conn;
+
+  /* Set up the fake connection */
+  memset(&conn, 0, sizeof(conn));
+  conn.edge_.base_.type = CONN_TYPE_AP;
+  conn.edge_.base_.state = AP_CONN_STATE_HTTP_CONNECT_WAIT;
+  conn.socks_request = tor_malloc_zero(sizeof(socks_request_t));
+  conn.socks_request->listener_type = CONN_TYPE_AP_HTTP_CONNECT_LISTENER;
+
+  conn.edge_.base_.inbuf = buf_new_with_data((char*)stdin_buf, data_size);
+  if (!conn.edge_.base_.inbuf) {
+    log_debug(LD_GENERAL, "Zero-Length-Input\n");
+    goto done;
+  }
+
+  /* Parse the headers */
+  int rv = connection_ap_process_http_connect(&conn);
+
+  /* TODO: check the output is correctly parsed based on the input */
+
+  log_debug(LD_GENERAL, "Result:\n%d\n", rv);
+
+  goto done;
+
+ done:
+  /* Reset. */
+  socks_request_free(conn.socks_request);
+  buf_free(conn.edge_.base_.inbuf);
+  conn.edge_.base_.inbuf = NULL;
+
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_iptsv2.c b/src/test/fuzz/fuzz_iptsv2.c
new file mode 100644
index 0000000000..76fa3c164e
--- /dev/null
+++ b/src/test/fuzz/fuzz_iptsv2.c
@@ -0,0 +1,50 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "feature/dirparse/unparseable.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/rend/rendparse.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+#include "feature/rend/rend_service_descriptor_st.h"
+
+#include "test/fuzz/fuzzing.h"
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  ed25519_init();
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  rend_service_descriptor_t *desc =
+    tor_malloc_zero(sizeof(rend_service_descriptor_t));
+  const char *str = (const char*) data;
+  int r = rend_parse_introduction_points(desc, str, sz);
+  if (r >= 0) {
+    log_debug(LD_GENERAL, "Parsing okay: %d", r);
+  } else {
+    log_debug(LD_GENERAL, "Parsing failed");
+  }
+  rend_service_descriptor_free(desc);
+  return 0;
+}
diff --git a/src/test/fuzz/fuzz_microdesc.c b/src/test/fuzz/fuzz_microdesc.c
new file mode 100644
index 0000000000..28fdc5e24d
--- /dev/null
+++ b/src/test/fuzz/fuzz_microdesc.c
@@ -0,0 +1,49 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "feature/dirparse/microdesc_parse.h"
+#include "feature/dirparse/unparseable.h"
+#include "feature/nodelist/microdesc.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+#include "test/fuzz/fuzzing.h"
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  ed25519_init();
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  const char *str = (const char*) data;
+  smartlist_t *result = microdescs_parse_from_string((const char *)str,
+                                                     str+sz,
+                                                     0, SAVED_NOWHERE, NULL);
+  if (result) {
+    log_debug(LD_GENERAL, "Parsing okay: %d", smartlist_len(result));
+    SMARTLIST_FOREACH(result, microdesc_t *, md, microdesc_free(md));
+    smartlist_free(result);
+  } else {
+    log_debug(LD_GENERAL, "Parsing failed");
+  }
+  return 0;
+}
diff --git a/src/test/fuzz/fuzz_multi.sh b/src/test/fuzz/fuzz_multi.sh
new file mode 100755
index 0000000000..b4a17ed8cb
--- /dev/null
+++ b/src/test/fuzz/fuzz_multi.sh
@@ -0,0 +1,34 @@
+MEMLIMIT_BYTES=21990500990976
+
+N_CPUS=1
+if [ $# -ge 1 ]; then
+    N_CPUS="$1"
+    shift
+fi
+
+FILTER=echo
+
+for i in `seq -w "$N_CPUS"`; do
+    if [ "$i" -eq 1 ]; then
+        if [ "$N_CPUS" -eq 1 ]; then
+            INSTANCE=""
+            NUMBER=""
+        else
+            INSTANCE="-M"
+            NUMBER="$i"
+        fi
+    else
+        INSTANCE="-S"
+        NUMBER="$i"
+    fi
+    # use whatever remains on the command-line to prefix the fuzzer command
+    # you have to copy and paste and run these commands yourself
+    "$FILTER" "$@" \
+        ../afl/afl-fuzz \
+        -i src/test/fuzz/fuzz_dir_testcase \
+        -o src/test/fuzz/fuzz_dir_findings \
+        -x src/test/fuzz/fuzz_dir_dictionary/fuzz_dir_http_header.dct \
+        -m "$MEMLIMIT_BYTES" \
+        "$INSTANCE" "$NUMBER" \
+        -- src/test/fuzz_dir
+done
diff --git a/src/test/fuzz/fuzz_socks.c b/src/test/fuzz/fuzz_socks.c
new file mode 100644
index 0000000000..06cb08391e
--- /dev/null
+++ b/src/test/fuzz/fuzz_socks.c
@@ -0,0 +1,50 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#define BUFFERS_PRIVATE
+#include "core/or/or.h"
+
+#include "lib/container/buffers.h"
+#include "lib/err/backtrace.h"
+#include "lib/log/log.h"
+#include "core/proto/proto_socks.h"
+#include "feature/client/addressmap.h"
+
+#include "test/fuzz/fuzzing.h"
+
+int
+fuzz_init(void)
+{
+  addressmap_init();
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  addressmap_free_all();
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *stdin_buf, size_t data_size)
+{
+  buf_t *buffer = buf_new_with_data((char*)stdin_buf, data_size);
+  if (!buffer) {
+    tor_assert(data_size==0);
+    buffer = buf_new();
+  }
+
+  socks_request_t *request = socks_request_new();
+
+  int r = fetch_from_buf_socks(buffer, request, 0, 0);
+  log_info(LD_GENERAL, "Socks request status: %d", r);
+
+  /* Reset. */
+  buf_free(buffer);
+  socks_request_free(request);
+
+  return 0;
+}
diff --git a/src/test/fuzz/fuzz_vrs.c b/src/test/fuzz/fuzz_vrs.c
new file mode 100644
index 0000000000..967397d1af
--- /dev/null
+++ b/src/test/fuzz/fuzz_vrs.c
@@ -0,0 +1,87 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#define NS_PARSE_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+#include "core/or/or.h"
+#include "feature/dirparse/ns_parse.h"
+#include "feature/dirparse/unparseable.h"
+#include "lib/memarea/memarea.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/vote_routerstatus_st.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+#include "test/fuzz/fuzzing.h"
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+static networkstatus_t *dummy_vote = NULL;
+static memarea_t *area = NULL;
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  ed25519_init();
+  area = memarea_new();
+  dummy_vote = tor_malloc_zero(sizeof(*dummy_vote));
+  dummy_vote->known_flags = smartlist_new();
+  smartlist_split_string(dummy_vote->known_flags,
+                         "Authority BadExit Exit Fast Guard HSDir "
+                         "NoEdConsensus Running Stable V2Dir Valid",
+                         " ", 0, 0);
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  SMARTLIST_FOREACH(dummy_vote->known_flags, char *, cp, tor_free(cp));
+  smartlist_free(dummy_vote->known_flags);
+  tor_free(dummy_vote);
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  char *str = tor_memdup_nulterm(data, sz);
+  const char *s;
+  routerstatus_t *rs_ns = NULL, *rs_md = NULL, *rs_vote = NULL;
+  vote_routerstatus_t *vrs = tor_malloc_zero(sizeof(*vrs));
+  smartlist_t *tokens = smartlist_new();
+
+  s = str;
+  rs_ns = routerstatus_parse_entry_from_string(area, &s, tokens,
+                                               NULL, NULL, 26, FLAV_NS);
+  tor_assert(smartlist_len(tokens) == 0);
+
+  s = str;
+  rs_md = routerstatus_parse_entry_from_string(area, &s, tokens,
+                                               NULL, NULL, 26, FLAV_MICRODESC);
+  tor_assert(smartlist_len(tokens) == 0);
+
+  s = str;
+  rs_vote = routerstatus_parse_entry_from_string(area, &s, tokens,
+                                              dummy_vote, vrs, 26, FLAV_NS);
+  tor_assert(smartlist_len(tokens) == 0);
+
+  log_debug(LD_GENERAL,
+            "ns=%p, md=%p, vote=%p", rs_ns, rs_md, rs_vote);
+
+  routerstatus_free(rs_md);
+  routerstatus_free(rs_ns);
+  vote_routerstatus_free(vrs);
+  memarea_clear(area);
+  smartlist_free(tokens);
+  tor_free(str);
+  return 0;
+}
diff --git a/src/test/fuzz/fuzzing.h b/src/test/fuzz/fuzzing.h
new file mode 100644
index 0000000000..150ac4aa7d
--- /dev/null
+++ b/src/test/fuzz/fuzzing.h
@@ -0,0 +1,13 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#ifndef FUZZING_H
+#define FUZZING_H
+
+int fuzz_init(void);
+int fuzz_cleanup(void);
+int fuzz_main(const uint8_t *data, size_t sz);
+
+void disable_signature_checking(void);
+
+#endif /* FUZZING_H */
+
diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c
new file mode 100644
index 0000000000..8ea4898522
--- /dev/null
+++ b/src/test/fuzz/fuzzing_common.c
@@ -0,0 +1,197 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#define CRYPTO_ED25519_PRIVATE
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "lib/err/backtrace.h"
+#include "app/config/config.h"
+#include "test/fuzz/fuzzing.h"
+#include "lib/compress/compress.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "lib/crypt_ops/crypto_init.h"
+
+static or_options_t *mock_options = NULL;
+static const or_options_t *
+mock_get_options(void)
+{
+  return mock_options;
+}
+
+static int
+mock_crypto_pk_public_checksig__nocheck(const crypto_pk_t *env, char *to,
+                                        size_t tolen,
+                                        const char *from, size_t fromlen)
+{
+  tor_assert(env && to && from);
+  (void)fromlen;
+  /* We could look at from[0..fromlen-1] ... */
+  tor_assert(tolen >= crypto_pk_keysize(env));
+  size_t siglen = MIN(20, crypto_pk_keysize(env));
+  memset(to, 0x01, siglen);
+  return (int)siglen;
+}
+
+static int
+mock_crypto_pk_public_checksig_digest__nocheck(crypto_pk_t *env,
+                                               const char *data,
+                                               size_t datalen,
+                                               const char *sig,
+                                               size_t siglen)
+{
+  tor_assert(env && data && sig);
+  (void)datalen;
+  (void)siglen;
+  /* We could look at data[..] and sig[..] */
+  return 0;
+}
+
+static int
+mock_ed25519_checksig__nocheck(const ed25519_signature_t *signature,
+                      const uint8_t *msg, size_t len,
+                      const ed25519_public_key_t *pubkey)
+{
+  tor_assert(signature && msg && pubkey);
+  /* We could look at msg[0..len-1] ... */
+  (void)len;
+  return 0;
+}
+
+static int
+mock_ed25519_checksig_batch__nocheck(int *okay_out,
+                                     const ed25519_checkable_t *checkable,
+                                     int n_checkable)
+{
+  tor_assert(checkable);
+  int i;
+  for (i = 0; i < n_checkable; ++i) {
+    /* We could look at messages and signatures XXX */
+    tor_assert(checkable[i].pubkey);
+    tor_assert(checkable[i].msg);
+    if (okay_out)
+      okay_out[i] = 1;
+  }
+  return 0;
+}
+
+static int
+mock_ed25519_impl_spot_check__nocheck(void)
+{
+  return 0;
+}
+
+void
+disable_signature_checking(void)
+{
+  MOCK(crypto_pk_public_checksig,
+       mock_crypto_pk_public_checksig__nocheck);
+  MOCK(crypto_pk_public_checksig_digest,
+       mock_crypto_pk_public_checksig_digest__nocheck);
+  MOCK(ed25519_checksig, mock_ed25519_checksig__nocheck);
+  MOCK(ed25519_checksig_batch, mock_ed25519_checksig_batch__nocheck);
+  MOCK(ed25519_impl_spot_check, mock_ed25519_impl_spot_check__nocheck);
+}
+
+static void
+global_init(void)
+{
+  tor_threads_init();
+  tor_compress_init();
+
+  /* Initialise logging first */
+  init_logging(1);
+  configure_backtrace_handler(get_version());
+
+  if (crypto_global_init(0, NULL, NULL) < 0)
+    abort();
+
+  {
+    struct sipkey sipkey = { 1337, 7331 };
+    siphash_unset_global_key();
+    siphash_set_global_key(&sipkey);
+  }
+
+  /* set up the options. */
+  mock_options = tor_malloc_zero(sizeof(or_options_t));
+  MOCK(get_options, mock_get_options);
+
+  /* Make BUG() and nonfatal asserts crash */
+  tor_set_failed_assertion_callback(abort);
+
+  /* Make protocol warnings handled correctly. */
+  init_protocol_warning_severity_level();
+}
+
+#ifdef LLVM_FUZZ
+int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size);
+int
+LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
+{
+  static int initialized = 0;
+  if (!initialized) {
+    global_init();
+    if (fuzz_init() < 0)
+      abort();
+    initialized = 1;
+  }
+
+  return fuzz_main(Data, Size);
+}
+
+#else /* Not LLVM_FUZZ, so AFL. */
+
+int
+main(int argc, char **argv)
+{
+  size_t size;
+
+  global_init();
+
+  /* Disable logging by default to speed up fuzzing. */
+  int loglevel = LOG_ERR;
+
+  for (int i = 1; i < argc; ++i) {
+    if (!strcmp(argv[i], "--warn")) {
+      loglevel = LOG_WARN;
+    } else if (!strcmp(argv[i], "--notice")) {
+      loglevel = LOG_NOTICE;
+    } else if (!strcmp(argv[i], "--info")) {
+      loglevel = LOG_INFO;
+    } else if (!strcmp(argv[i], "--debug")) {
+      loglevel = LOG_DEBUG;
+    }
+  }
+
+  {
+    log_severity_list_t s;
+    memset(&s, 0, sizeof(s));
+    set_log_severity_config(loglevel, LOG_ERR, &s);
+    /* ALWAYS log bug warnings. */
+    s.masks[LOG_WARN-LOG_ERR] |= LD_BUG;
+    add_stream_log(&s, "", fileno(stdout));
+  }
+
+  if (fuzz_init() < 0)
+    abort();
+
+#ifdef __AFL_HAVE_MANUAL_CONTROL
+  /* Tell AFL to pause and fork here - ignored if not using AFL */
+  __AFL_INIT();
+#endif
+
+#define MAX_FUZZ_SIZE (128*1024)
+  char *input = read_file_to_str_until_eof(0, MAX_FUZZ_SIZE, &size);
+  tor_assert(input);
+  char *raw = tor_memdup(input, size); /* Because input is nul-terminated */
+  tor_free(input);
+  fuzz_main((const uint8_t*)raw, size);
+  tor_free(raw);
+
+  if (fuzz_cleanup() < 0)
+    abort();
+
+  tor_free(mock_options);
+  UNMOCK(get_options);
+  return 0;
+}
+
+#endif
diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am
new file mode 100644
index 0000000000..27eeced8c5
--- /dev/null
+++ b/src/test/fuzz/include.am
@@ -0,0 +1,440 @@
+# This file was generated by fuzzing_include_am.py; do not hand-edit unless
+# you enjoy having your changes erased.
+FUZZING_CPPFLAGS = \
+	$(src_test_AM_CPPFLAGS) $(TEST_CPPFLAGS)
+FUZZING_CFLAGS = \
+	$(AM_CFLAGS) $(TEST_CFLAGS)
+FUZZING_LDFLAG = \
+	@TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) @TOR_LDFLAGS_libevent@
+FUZZING_LIBS = \
+	$(TOR_INTERNAL_TESTING_LIBS) \
+	$(rust_ldadd) \
+	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
+	@TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \
+	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \
+	@TOR_SYSTEMD_LIBS@ \
+	@TOR_LZMA_LIBS@ \
+	@TOR_ZSTD_LIBS@
+
+oss-fuzz-prereqs: \
+    $(TOR_INTERNAL_TESTING_LIBS)
+
+noinst_HEADERS += \
+	src/test/fuzz/fuzzing.h
+
+LIBFUZZER = -lFuzzer
+LIBFUZZER_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ
+LIBFUZZER_CFLAGS = $(FUZZING_CFLAGS)
+LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG)
+LIBFUZZER_LIBS = $(FUZZING_LIBS) $(LIBFUZZER) -lstdc++
+
+LIBOSS_FUZZ_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ
+LIBOSS_FUZZ_CFLAGS = $(FUZZING_CFLAGS)
+
+# ===== AFL fuzzers
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_consensus_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_consensus.c
+src_test_fuzz_fuzz_consensus_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_consensus_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_consensus_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_consensus_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_descriptor_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_descriptor.c
+src_test_fuzz_fuzz_descriptor_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_descriptor_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_descriptor_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_descriptor_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_diff_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_diff.c
+src_test_fuzz_fuzz_diff_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_diff_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_diff_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_diff_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_diff_apply_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_diff_apply.c
+src_test_fuzz_fuzz_diff_apply_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_diff_apply_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_diff_apply_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_diff_apply_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_extrainfo_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_extrainfo.c
+src_test_fuzz_fuzz_extrainfo_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_extrainfo_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_extrainfo_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_extrainfo_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_hsdescv2_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_hsdescv2.c
+src_test_fuzz_fuzz_hsdescv2_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_hsdescv2_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_hsdescv2_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_hsdescv2_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_hsdescv3_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_hsdescv3.c
+src_test_fuzz_fuzz_hsdescv3_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_hsdescv3_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_hsdescv3_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_hsdescv3_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_http_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_http.c
+src_test_fuzz_fuzz_http_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_http_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_http_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_http_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_http_connect_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_http_connect.c
+src_test_fuzz_fuzz_http_connect_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_http_connect_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_http_connect_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_http_connect_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_iptsv2_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_iptsv2.c
+src_test_fuzz_fuzz_iptsv2_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_iptsv2_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_iptsv2_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_iptsv2_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_microdesc_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_microdesc.c
+src_test_fuzz_fuzz_microdesc_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_microdesc_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_microdesc_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_microdesc_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_socks_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_socks.c
+src_test_fuzz_fuzz_socks_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_socks_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_socks_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_socks_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_vrs_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_vrs.c
+src_test_fuzz_fuzz_vrs_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_vrs_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_vrs_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_vrs_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+FUZZERS = \
+	src/test/fuzz/fuzz-consensus \
+	src/test/fuzz/fuzz-descriptor \
+	src/test/fuzz/fuzz-diff \
+	src/test/fuzz/fuzz-diff-apply \
+	src/test/fuzz/fuzz-extrainfo \
+	src/test/fuzz/fuzz-hsdescv2 \
+	src/test/fuzz/fuzz-hsdescv3 \
+	src/test/fuzz/fuzz-http \
+	src/test/fuzz/fuzz-http-connect \
+	src/test/fuzz/fuzz-iptsv2 \
+	src/test/fuzz/fuzz-microdesc \
+	src/test/fuzz/fuzz-socks \
+	src/test/fuzz/fuzz-vrs
+endif
+
+# ===== libfuzzer
+
+if LIBFUZZER_ENABLED
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_consensus_SOURCES = \
+	$(src_test_fuzz_fuzz_consensus_SOURCES)
+src_test_fuzz_lf_fuzz_consensus_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_consensus_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_consensus_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_consensus_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_descriptor_SOURCES = \
+	$(src_test_fuzz_fuzz_descriptor_SOURCES)
+src_test_fuzz_lf_fuzz_descriptor_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_descriptor_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_descriptor_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_descriptor_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_diff_SOURCES = \
+	$(src_test_fuzz_fuzz_diff_SOURCES)
+src_test_fuzz_lf_fuzz_diff_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_diff_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_diff_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_diff_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_diff_apply_SOURCES = \
+	$(src_test_fuzz_fuzz_diff_apply_SOURCES)
+src_test_fuzz_lf_fuzz_diff_apply_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_diff_apply_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_diff_apply_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_diff_apply_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_extrainfo_SOURCES = \
+	$(src_test_fuzz_fuzz_extrainfo_SOURCES)
+src_test_fuzz_lf_fuzz_extrainfo_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_extrainfo_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_extrainfo_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_extrainfo_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_hsdescv2_SOURCES = \
+	$(src_test_fuzz_fuzz_hsdescv2_SOURCES)
+src_test_fuzz_lf_fuzz_hsdescv2_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_hsdescv2_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_hsdescv2_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_hsdescv2_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_hsdescv3_SOURCES = \
+	$(src_test_fuzz_fuzz_hsdescv3_SOURCES)
+src_test_fuzz_lf_fuzz_hsdescv3_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_hsdescv3_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_hsdescv3_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_hsdescv3_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_http_SOURCES = \
+	$(src_test_fuzz_fuzz_http_SOURCES)
+src_test_fuzz_lf_fuzz_http_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_http_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_http_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_http_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_http_connect_SOURCES = \
+	$(src_test_fuzz_fuzz_http_connect_SOURCES)
+src_test_fuzz_lf_fuzz_http_connect_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_http_connect_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_http_connect_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_http_connect_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_iptsv2_SOURCES = \
+	$(src_test_fuzz_fuzz_iptsv2_SOURCES)
+src_test_fuzz_lf_fuzz_iptsv2_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_iptsv2_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_iptsv2_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_iptsv2_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_microdesc_SOURCES = \
+	$(src_test_fuzz_fuzz_microdesc_SOURCES)
+src_test_fuzz_lf_fuzz_microdesc_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_microdesc_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_microdesc_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_microdesc_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_socks_SOURCES = \
+	$(src_test_fuzz_fuzz_socks_SOURCES)
+src_test_fuzz_lf_fuzz_socks_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_socks_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_socks_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_socks_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_vrs_SOURCES = \
+	$(src_test_fuzz_fuzz_vrs_SOURCES)
+src_test_fuzz_lf_fuzz_vrs_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_vrs_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_vrs_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_vrs_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+LIBFUZZER_FUZZERS = \
+	src/test/fuzz/lf-fuzz-consensus \
+	src/test/fuzz/lf-fuzz-descriptor \
+	src/test/fuzz/lf-fuzz-diff \
+	src/test/fuzz/lf-fuzz-diff-apply \
+	src/test/fuzz/lf-fuzz-extrainfo \
+	src/test/fuzz/lf-fuzz-hsdescv2 \
+	src/test/fuzz/lf-fuzz-hsdescv3 \
+	src/test/fuzz/lf-fuzz-http \
+	src/test/fuzz/lf-fuzz-http-connect \
+	src/test/fuzz/lf-fuzz-iptsv2 \
+	src/test/fuzz/lf-fuzz-microdesc \
+	src/test/fuzz/lf-fuzz-socks \
+	src/test/fuzz/lf-fuzz-vrs
+
+else
+LIBFUZZER_FUZZERS =
+endif
+
+# ===== oss-fuzz
+
+if OSS_FUZZ_ENABLED
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_consensus_a_SOURCES = \
+	$(src_test_fuzz_fuzz_consensus_SOURCES)
+src_test_fuzz_liboss_fuzz_consensus_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_consensus_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_descriptor_a_SOURCES = \
+	$(src_test_fuzz_fuzz_descriptor_SOURCES)
+src_test_fuzz_liboss_fuzz_descriptor_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_descriptor_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_diff_a_SOURCES = \
+	$(src_test_fuzz_fuzz_diff_SOURCES)
+src_test_fuzz_liboss_fuzz_diff_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_diff_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_diff_apply_a_SOURCES = \
+	$(src_test_fuzz_fuzz_diff_apply_SOURCES)
+src_test_fuzz_liboss_fuzz_diff_apply_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_diff_apply_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_extrainfo_a_SOURCES = \
+	$(src_test_fuzz_fuzz_extrainfo_SOURCES)
+src_test_fuzz_liboss_fuzz_extrainfo_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_extrainfo_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_hsdescv2_a_SOURCES = \
+	$(src_test_fuzz_fuzz_hsdescv2_SOURCES)
+src_test_fuzz_liboss_fuzz_hsdescv2_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_hsdescv2_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_hsdescv3_a_SOURCES = \
+	$(src_test_fuzz_fuzz_hsdescv3_SOURCES)
+src_test_fuzz_liboss_fuzz_hsdescv3_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_hsdescv3_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_http_a_SOURCES = \
+	$(src_test_fuzz_fuzz_http_SOURCES)
+src_test_fuzz_liboss_fuzz_http_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_http_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_http_connect_a_SOURCES = \
+	$(src_test_fuzz_fuzz_http_connect_SOURCES)
+src_test_fuzz_liboss_fuzz_http_connect_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_http_connect_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_iptsv2_a_SOURCES = \
+	$(src_test_fuzz_fuzz_iptsv2_SOURCES)
+src_test_fuzz_liboss_fuzz_iptsv2_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_iptsv2_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_microdesc_a_SOURCES = \
+	$(src_test_fuzz_fuzz_microdesc_SOURCES)
+src_test_fuzz_liboss_fuzz_microdesc_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_microdesc_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_socks_a_SOURCES = \
+	$(src_test_fuzz_fuzz_socks_SOURCES)
+src_test_fuzz_liboss_fuzz_socks_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_socks_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_vrs_a_SOURCES = \
+	$(src_test_fuzz_fuzz_vrs_SOURCES)
+src_test_fuzz_liboss_fuzz_vrs_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_vrs_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+OSS_FUZZ_FUZZERS = \
+	src/test/fuzz/liboss-fuzz-consensus.a \
+	src/test/fuzz/liboss-fuzz-descriptor.a \
+	src/test/fuzz/liboss-fuzz-diff.a \
+	src/test/fuzz/liboss-fuzz-diff-apply.a \
+	src/test/fuzz/liboss-fuzz-extrainfo.a \
+	src/test/fuzz/liboss-fuzz-hsdescv2.a \
+	src/test/fuzz/liboss-fuzz-hsdescv3.a \
+	src/test/fuzz/liboss-fuzz-http.a \
+	src/test/fuzz/liboss-fuzz-http-connect.a \
+	src/test/fuzz/liboss-fuzz-iptsv2.a \
+	src/test/fuzz/liboss-fuzz-microdesc.a \
+	src/test/fuzz/liboss-fuzz-socks.a \
+	src/test/fuzz/liboss-fuzz-vrs.a
+
+else
+OSS_FUZZ_FUZZERS =
+endif
+
+noinst_PROGRAMS += $(FUZZERS) $(LIBFUZZER_FUZZERS)
+noinst_LIBRARIES += $(OSS_FUZZ_FUZZERS)
+oss-fuzz-fuzzers:  oss-fuzz-prereqs $(OSS_FUZZ_FUZZERS)
+fuzzers: $(FUZZERS) $(LIBFUZZER_FUZZERS)
+
+test-fuzz-corpora: $(FUZZERS)
+	$(top_srcdir)/src/test/fuzz_static_testcases.sh
diff --git a/src/test/fuzz/minimize.sh b/src/test/fuzz/minimize.sh
new file mode 100755
index 0000000000..87d3dda13c
--- /dev/null
+++ b/src/test/fuzz/minimize.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -e
+
+if [ ! -d "$1" ] ; then
+    echo "I need a directory"
+    exit 1
+fi
+
+which=`basename "$1"`
+
+mkdir "$1.out"
+afl-cmin -i "$1" -o "$1.out" -m none "./src/test/fuzz/fuzz-${which}"
+
diff --git a/src/test/fuzz_static_testcases.sh b/src/test/fuzz_static_testcases.sh
new file mode 100755
index 0000000000..f7b3adffb1
--- /dev/null
+++ b/src/test/fuzz_static_testcases.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+# Copyright (c) 2016-2019, The Tor Project, Inc.
+# See LICENSE for licensing information
+
+set -e
+
+if [ -z "${TOR_FUZZ_CORPORA}" ] || [ ! -d "${TOR_FUZZ_CORPORA}" ] ; then
+    echo "You need to set TOR_FUZZ_CORPORA to point to a checkout of "
+    echo "the 'fuzzing-corpora' repository."
+    exit 77
+fi
+
+
+
+for fuzzer in "${builddir:-.}"/src/test/fuzz/fuzz-* ; do
+    f=`basename $fuzzer`
+    case="${f#fuzz-}"
+    if [ -d "${TOR_FUZZ_CORPORA}/${case}" ]; then
+        echo "Running tests for ${case}"
+        for entry in "${TOR_FUZZ_CORPORA}/${case}/"*; do
+	    "${fuzzer}" "--err" < "$entry"
+        done
+    else
+	echo "No tests found for ${case}"
+    fi
+done
diff --git a/src/test/hs_build_address.py b/src/test/hs_build_address.py
new file mode 100644
index 0000000000..7ff22c3a9a
--- /dev/null
+++ b/src/test/hs_build_address.py
@@ -0,0 +1,38 @@
+import sys
+import hashlib
+import struct
+import base64
+
+# Python 3.6+, the SHA3 is available in hashlib natively. Else this requires
+# the pysha3 package (pip install pysha3).
+if sys.version_info < (3, 6):
+    import sha3
+
+# Test vector to make sure the right sha3 version will be used. pysha3 < 1.0
+# used the old Keccak implementation. During the finalization of SHA3, NIST
+# changed the delimiter suffix from 0x01 to 0x06. The Keccak sponge function
+# stayed the same. pysha3 1.0 provides the previous Keccak hash, too.
+TEST_VALUE = "e167f68d6563d75bb25f3aa49c29ef612d41352dc00606de7cbd630bb2665f51"
+if TEST_VALUE != sha3.sha3_256(b"Hello World").hexdigest():
+    print("pysha3 version is < 1.0. Please install from:")
+    print("https://github.com/tiran/pysha3https://github.com/tiran/pysha3")
+    sys.exit(1)
+
+# Checksum is built like so:
+#   CHECKSUM = SHA3(".onion checksum" || PUBKEY || VERSION)
+PREFIX = ".onion checksum".encode()
+# 32 bytes ed25519 pubkey from first test vector of
+# https://tools.ietf.org/html/draft-josefsson-eddsa-ed25519-02#section-6
+PUBKEY = "d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a".decode('hex')
+# Version 3 is proposal224
+VERSION = 3
+
+data = struct.pack('15s32sb', PREFIX, PUBKEY, VERSION)
+checksum = hashlib.sha3_256(data).digest()
+
+# Onion address is built like so:
+#   onion_address = base32(PUBKEY || CHECKSUM || VERSION) + ".onion"
+address = struct.pack('!32s2sb', PUBKEY, checksum, VERSION)
+onion_addr = base64.b32encode(address).decode().lower()
+
+print("%s" % (onion_addr))
diff --git a/src/test/hs_indexes.py b/src/test/hs_indexes.py
new file mode 100644
index 0000000000..af0b81f8de
--- /dev/null
+++ b/src/test/hs_indexes.py
@@ -0,0 +1,70 @@
+#
+# The hidden service subsystem has two type of index. The first type is a
+# value that each node in the network gets assigned to using their identity
+# key which is their position in the hashring. (hs_build_hsdir_index()).
+#
+# The second type is a value that both the client and service computes to
+# store/fetch the descriptor on the hashring. (hs_build_hs_index()).
+#
+
+import sys
+import hashlib
+import struct
+import base64
+
+# Python 3.6+, the SHA3 is available in hashlib natively. Else this requires
+# the pysha3 package (pip install pysha3).
+if sys.version_info < (3, 6):
+    import sha3
+    # Test vector to make sure the right sha3 version will be used. pysha3 < 1.0
+    # used the old Keccak implementation. During the finalization of SHA3, NIST
+    # changed the delimiter suffix from 0x01 to 0x06. The Keccak sponge function
+    # stayed the same. pysha3 1.0 provides the previous Keccak hash, too.
+    TEST_VALUE = "e167f68d6563d75bb25f3aa49c29ef612d41352dc00606de7cbd630bb2665f51"
+    if TEST_VALUE != sha3.sha3_256(b"Hello World").hexdigest():
+        print("pysha3 version is < 1.0. Please install from:")
+        print("https://github.com/tiran/pysha3https://github.com/tiran/pysha3")
+        sys.exit(1)
+
+# The first index we'll build is the position index in the hashring that is
+# constructed by the hs_build_hsdir_index() function. Construction is:
+#   SHA3-256("node-idx" | node_identity |
+#            shared_random_value | INT_8(period_length) | INT_8(period_num) )
+
+PREFIX = "node-idx".encode()
+# 32 bytes ed25519 pubkey.
+IDENTITY = ("\x42" * 32).encode()
+# SRV is 32 bytes.
+SRV = ("\x43" * 32).encode()
+# Time period length is a 8 bytes value.
+PERIOD_LEN = 1440
+# Period number is a 8 bytes value.
+PERIOD_NUM = 42
+
+data = struct.pack('!8s32s32sQQ', PREFIX, IDENTITY, SRV, PERIOD_NUM,
+                                  PERIOD_LEN)
+hsdir_index = hashlib.sha3_256(data).hexdigest()
+
+print("[hs_build_hsdir_index] %s" % (hsdir_index))
+
+# The second index we'll build is where the HS stores and the client fetches
+# the descriptor on the hashring. It is constructed by the hs_build_hs_index()
+# function and the construction is:
+#   SHA3-256("store-at-idx" | blinded_public_key |
+#            INT_8(replicanum) | INT_8(period_num) | INT_8(period_length) )
+
+PREFIX = "store-at-idx".encode()
+# 32 bytes ed25519 pubkey.
+PUBKEY = ("\x42" * 32).encode()
+# Replica number is a 8 bytes value.
+REPLICA_NUM = 1
+# Time period length is a 8 bytes value.
+PERIOD_LEN = 1440
+# Period number is a 8 bytes value.
+PERIOD_NUM = 42
+
+data = struct.pack('!12s32sQQQ', PREFIX, PUBKEY, REPLICA_NUM, PERIOD_LEN,
+                                   PERIOD_NUM)
+hs_index = hashlib.sha3_256(data).hexdigest()
+
+print("[hs_build_hs_index]   %s" % (hs_index))
diff --git a/src/test/hs_ntor_ref.py b/src/test/hs_ntor_ref.py
new file mode 100644
index 0000000000..1b9772a5d6
--- /dev/null
+++ b/src/test/hs_ntor_ref.py
@@ -0,0 +1,428 @@
+#!/usr/bin/python
+# Copyright 2017-2019, The Tor Project, Inc
+# See LICENSE for licensing information
+
+"""
+hs_ntor_ref.py
+
+This module is a reference implementation of the modified ntor protocol
+proposed for Tor hidden services in proposal 224 (Next Generation Hidden
+Services) in section [NTOR-WITH-EXTRA-DATA].
+
+The modified ntor protocol is a single-round protocol, with three steps in total:
+
+    1: Client generates keys and sends them to service via INTRODUCE cell
+
+    2: Service computes key material based on client's keys, and sends its own
+       keys to client via RENDEZVOUS cell
+
+    3: Client computes key material as well.
+
+It's meant to be used to validate Tor's HS ntor implementation by conducting
+various integration tests. Specifically it conducts the following three tests:
+
+- Tests our Python implementation by running the whole protocol in Python and
+  making sure that results are consistent.
+
+- Tests little-t-tor ntor implementation. We use this Python code to instrument
+  little-t-tor and carry out the handshake by using little-t-tor code. The
+  small C wrapper at src/test/test-hs-ntor-cl is used for this Python module to
+  interface with little-t-tor.
+
+- Cross-tests Python and little-t-tor implementation by running half of the
+  protocol in Python code and the other in little-t-tor. This is actually two
+  tests so that all parts of the protocol are run both by little-t-tor and
+  Python.
+
+It requires the curve25519 python module from the curve25519-donna package.
+
+The whole logic and concept for this test suite was taken from ntor_ref.py.
+
+                *** DO NOT USE THIS IN PRODUCTION. ***
+"""
+
+import struct
+import os, sys
+import binascii
+import subprocess
+
+try:
+    import curve25519
+    curve25519mod = curve25519.keys
+except ImportError:
+    curve25519 = None
+    import slownacl_curve25519
+    curve25519mod = slownacl_curve25519
+
+import hashlib
+try:
+    import sha3
+except ImportError:
+    # In python 3.6, the sha3 functions are in hashlib whether we
+    # import sha3 or not.
+    sha3 = None
+
+try:
+    # Pull the sha3 functions in.
+    from hashlib import sha3_256, shake_256
+    shake_squeeze = shake_256.digest
+except ImportError:
+    if hasattr(sha3, "SHA3256"):
+        # If this happens, then we have the old "sha3" module which
+        # hashlib and pysha3 superseded.
+        sha3_256 = sha3.SHA3256
+        shake_256 = sha3.SHAKE256
+        shake_squeeze = shake_256.squeeze
+    else:
+        # error code 77 tells automake to skip this test
+        sys.exit(77)
+
+# Import Nick's ntor reference implementation in Python
+# We are gonna use a few of its utilities.
+from ntor_ref import hash_nil
+from ntor_ref import PrivateKey
+
+# String constants used in this protocol
+PROTOID = b"tor-hs-ntor-curve25519-sha3-256-1"
+T_HSENC    = PROTOID + b":hs_key_extract"
+T_HSVERIFY = PROTOID + b":hs_verify"
+T_HSMAC    = PROTOID + b":hs_mac"
+M_HSEXPAND = PROTOID + b":hs_key_expand"
+
+INTRO_SECRET_LEN = 161
+REND_SECRET_LEN = 225
+AUTH_INPUT_LEN = 199
+
+# Implements MAC(k,m) = H(htonll(len(k)) | k | m)
+def mac(k,m):
+    def htonll(num):
+        return struct.pack('!q', num)
+
+    s = sha3_256()
+    s.update(htonll(len(k)))
+    s.update(k)
+    s.update(m)
+    return s.digest()
+
+######################################################################
+
+# Functions that implement the modified HS ntor protocol
+
+"""As client compute key material for INTRODUCE cell as follows:
+
+  intro_secret_hs_input = EXP(B,x) | AUTH_KEY | X | B | PROTOID
+  info = m_hsexpand | subcredential
+  hs_keys = KDF(intro_secret_hs_input | t_hsenc | info, S_KEY_LEN+MAC_LEN)
+  ENC_KEY = hs_keys[0:S_KEY_LEN]
+  MAC_KEY = hs_keys[S_KEY_LEN:S_KEY_LEN+MAC_KEY_LEN]
+"""
+def intro2_ntor_client(intro_auth_pubkey_str, intro_enc_pubkey,
+                       client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey, subcredential):
+
+    dh_result = client_ephemeral_enc_privkey.get_shared_key(intro_enc_pubkey, hash_nil)
+    secret =  dh_result + intro_auth_pubkey_str + client_ephemeral_enc_pubkey.serialize() + intro_enc_pubkey.serialize() + PROTOID
+    assert(len(secret) == INTRO_SECRET_LEN)
+    info = M_HSEXPAND + subcredential
+
+    kdf = shake_256()
+    kdf.update(secret + T_HSENC + info)
+    key_material = shake_squeeze(kdf, 64*8)
+
+    enc_key = key_material[0:32]
+    mac_key = key_material[32:64]
+
+    return enc_key, mac_key
+
+"""Wrapper over intro2_ntor_client()"""
+def client_part1(intro_auth_pubkey_str, intro_enc_pubkey,
+                 client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey, subcredential):
+    enc_key, mac_key = intro2_ntor_client(intro_auth_pubkey_str, intro_enc_pubkey, client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey, subcredential)
+    assert(enc_key)
+    assert(mac_key)
+
+    return enc_key, mac_key
+
+"""As service compute key material for INTRODUCE cell as follows:
+
+  intro_secret_hs_input = EXP(X,b) | AUTH_KEY | X | B | PROTOID
+  info = m_hsexpand | subcredential
+  hs_keys = KDF(intro_secret_hs_input | t_hsenc | info, S_KEY_LEN+MAC_LEN)
+  HS_DEC_KEY = hs_keys[0:S_KEY_LEN]
+  HS_MAC_KEY = hs_keys[S_KEY_LEN:S_KEY_LEN+MAC_KEY_LEN]
+"""
+def intro2_ntor_service(intro_auth_pubkey_str, client_enc_pubkey, service_enc_privkey, service_enc_pubkey, subcredential):
+    dh_result = service_enc_privkey.get_shared_key(client_enc_pubkey, hash_nil)
+    secret = dh_result + intro_auth_pubkey_str + client_enc_pubkey.serialize() + service_enc_pubkey.serialize() + PROTOID
+    assert(len(secret) == INTRO_SECRET_LEN)
+    info = M_HSEXPAND + subcredential
+
+    kdf = shake_256()
+    kdf.update(secret + T_HSENC + info)
+    key_material = shake_squeeze(kdf, 64*8)
+
+    enc_key = key_material[0:32]
+    mac_key = key_material[32:64]
+
+    return enc_key, mac_key
+
+"""As service compute key material for INTRODUCE and REDNEZVOUS cells.
+
+  Use intro2_ntor_service() to calculate the INTRODUCE key material, and use
+  the following computations to do the RENDEZVOUS ones:
+
+      rend_secret_hs_input = EXP(X,y) | EXP(X,b) | AUTH_KEY | B | X | Y | PROTOID
+      NTOR_KEY_SEED = MAC(rend_secret_hs_input, t_hsenc)
+      verify = MAC(rend_secret_hs_input, t_hsverify)
+      auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server"
+      AUTH_INPUT_MAC = MAC(auth_input, t_hsmac)
+"""
+def service_part1(intro_auth_pubkey_str, client_enc_pubkey, intro_enc_privkey, intro_enc_pubkey, subcredential):
+    intro_enc_key, intro_mac_key = intro2_ntor_service(intro_auth_pubkey_str, client_enc_pubkey, intro_enc_privkey, intro_enc_pubkey, subcredential)
+    assert(intro_enc_key)
+    assert(intro_mac_key)
+
+    service_ephemeral_privkey = PrivateKey()
+    service_ephemeral_pubkey = service_ephemeral_privkey.get_public()
+
+    dh_result1 = service_ephemeral_privkey.get_shared_key(client_enc_pubkey, hash_nil)
+    dh_result2 = intro_enc_privkey.get_shared_key(client_enc_pubkey, hash_nil)
+    rend_secret_hs_input = dh_result1 + dh_result2 + intro_auth_pubkey_str + intro_enc_pubkey.serialize() + client_enc_pubkey.serialize() + service_ephemeral_pubkey.serialize() + PROTOID
+    assert(len(rend_secret_hs_input) == REND_SECRET_LEN)
+
+    ntor_key_seed = mac(rend_secret_hs_input, T_HSENC)
+    verify = mac(rend_secret_hs_input, T_HSVERIFY)
+    auth_input = verify + intro_auth_pubkey_str + intro_enc_pubkey.serialize() + service_ephemeral_pubkey.serialize() + client_enc_pubkey.serialize() + PROTOID + b"Server"
+    assert(len(auth_input) == AUTH_INPUT_LEN)
+    auth_input_mac = mac(auth_input, T_HSMAC)
+
+    assert(ntor_key_seed)
+    assert(auth_input_mac)
+    assert(service_ephemeral_pubkey)
+
+    return intro_enc_key, intro_mac_key, ntor_key_seed, auth_input_mac, service_ephemeral_pubkey
+
+"""As client compute key material for rendezvous cells as follows:
+
+  rend_secret_hs_input = EXP(Y,x) | EXP(B,x) | AUTH_KEY | B | X | Y | PROTOID
+  NTOR_KEY_SEED = MAC(ntor_secret_input, t_hsenc)
+  verify = MAC(ntor_secret_input, t_hsverify)
+  auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server"
+  AUTH_INPUT_MAC = MAC(auth_input, t_hsmac)
+"""
+def client_part2(intro_auth_pubkey_str, client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey,
+                 intro_enc_pubkey, service_ephemeral_rend_pubkey):
+    dh_result1 = client_ephemeral_enc_privkey.get_shared_key(service_ephemeral_rend_pubkey, hash_nil)
+    dh_result2 = client_ephemeral_enc_privkey.get_shared_key(intro_enc_pubkey, hash_nil)
+    rend_secret_hs_input = dh_result1 + dh_result2 + intro_auth_pubkey_str + intro_enc_pubkey.serialize() + client_ephemeral_enc_pubkey.serialize() + service_ephemeral_rend_pubkey.serialize() + PROTOID
+    assert(len(rend_secret_hs_input) == REND_SECRET_LEN)
+
+    ntor_key_seed = mac(rend_secret_hs_input, T_HSENC)
+    verify = mac(rend_secret_hs_input, T_HSVERIFY)
+    auth_input = verify + intro_auth_pubkey_str + intro_enc_pubkey.serialize() + service_ephemeral_rend_pubkey.serialize() + client_ephemeral_enc_pubkey.serialize() + PROTOID + b"Server"
+    assert(len(auth_input) == AUTH_INPUT_LEN)
+    auth_input_mac = mac(auth_input, T_HSMAC)
+
+    assert(ntor_key_seed)
+    assert(auth_input_mac)
+
+    return ntor_key_seed, auth_input_mac
+
+#################################################################################
+
+"""
+Utilities for communicating with the little-t-tor ntor wrapper to conduct the
+integration tests
+"""
+
+PROG = "./src/test/test-hs-ntor-cl"
+if sys.version_info[0] >= 3:
+    enhex=lambda s: binascii.b2a_hex(s).decode("ascii")
+else:
+    enhex=lambda s: binascii.b2a_hex(s)
+dehex=lambda s: binascii.a2b_hex(s.strip())
+
+def tor_client1(intro_auth_pubkey_str, intro_enc_pubkey,
+                client_ephemeral_enc_privkey, subcredential):
+    p = subprocess.Popen([PROG, "client1",
+                          enhex(intro_auth_pubkey_str),
+                          enhex(intro_enc_pubkey.serialize()),
+                          enhex(client_ephemeral_enc_privkey.serialize()),
+                          enhex(subcredential)],
+                         stdout=subprocess.PIPE)
+    return map(dehex, p.stdout.readlines())
+
+def tor_server1(intro_auth_pubkey_str, intro_enc_privkey,
+                client_ephemeral_enc_pubkey, subcredential):
+    p = subprocess.Popen([PROG, "server1",
+                          enhex(intro_auth_pubkey_str),
+                          enhex(intro_enc_privkey.serialize()),
+                          enhex(client_ephemeral_enc_pubkey.serialize()),
+                          enhex(subcredential)],
+                         stdout=subprocess.PIPE)
+    return map(dehex, p.stdout.readlines())
+
+def tor_client2(intro_auth_pubkey_str, client_ephemeral_enc_privkey,
+                intro_enc_pubkey, service_ephemeral_rend_pubkey, subcredential):
+    p = subprocess.Popen([PROG, "client2",
+                          enhex(intro_auth_pubkey_str),
+                          enhex(client_ephemeral_enc_privkey.serialize()),
+                          enhex(intro_enc_pubkey.serialize()),
+                          enhex(service_ephemeral_rend_pubkey.serialize()),
+                          enhex(subcredential)],
+                         stdout=subprocess.PIPE)
+    return map(dehex, p.stdout.readlines())
+
+##################################################################################
+
+# Perform a pure python ntor test
+def do_pure_python_ntor_test():
+    # Initialize all needed key material
+    client_ephemeral_enc_privkey = PrivateKey()
+    client_ephemeral_enc_pubkey = client_ephemeral_enc_privkey.get_public()
+    intro_enc_privkey = PrivateKey()
+    intro_enc_pubkey = intro_enc_privkey.get_public()
+    intro_auth_pubkey_str = os.urandom(32)
+    subcredential = os.urandom(32)
+
+    client_enc_key, client_mac_key = client_part1(intro_auth_pubkey_str, intro_enc_pubkey, client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey, subcredential)
+
+    service_enc_key, service_mac_key, service_ntor_key_seed, service_auth_input_mac, service_ephemeral_pubkey = service_part1(intro_auth_pubkey_str, client_ephemeral_enc_pubkey, intro_enc_privkey, intro_enc_pubkey, subcredential)
+
+    assert(client_enc_key == service_enc_key)
+    assert(client_mac_key == service_mac_key)
+
+    client_ntor_key_seed, client_auth_input_mac = client_part2(intro_auth_pubkey_str, client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey,
+                                                               intro_enc_pubkey, service_ephemeral_pubkey)
+
+    assert(client_ntor_key_seed == service_ntor_key_seed)
+    assert(client_auth_input_mac == service_auth_input_mac)
+
+    print("DONE: python dance [%s]" % repr(client_auth_input_mac))
+
+# Perform a pure little-t-tor integration test.
+def do_little_t_tor_ntor_test():
+    # Initialize all needed key material
+    subcredential = os.urandom(32)
+    client_ephemeral_enc_privkey = PrivateKey()
+    client_ephemeral_enc_pubkey = client_ephemeral_enc_privkey.get_public()
+    intro_enc_privkey = PrivateKey()
+    intro_enc_pubkey = intro_enc_privkey.get_public() # service-side enc key
+    intro_auth_pubkey_str = os.urandom(32)
+
+    client_enc_key, client_mac_key = tor_client1(intro_auth_pubkey_str, intro_enc_pubkey,
+                                                 client_ephemeral_enc_privkey, subcredential)
+    assert(client_enc_key)
+    assert(client_mac_key)
+
+    service_enc_key, service_mac_key, service_ntor_auth_mac, service_ntor_key_seed, service_eph_pubkey = tor_server1(intro_auth_pubkey_str,
+                                                                                                                     intro_enc_privkey,
+                                                                                                                     client_ephemeral_enc_pubkey,
+                                                                                                                     subcredential)
+    assert(service_enc_key)
+    assert(service_mac_key)
+    assert(service_ntor_auth_mac)
+    assert(service_ntor_key_seed)
+
+    assert(client_enc_key == service_enc_key)
+    assert(client_mac_key == service_mac_key)
+
+    # Turn from bytes to key
+    service_eph_pubkey = curve25519mod.Public(service_eph_pubkey)
+
+    client_ntor_auth_mac, client_ntor_key_seed  = tor_client2(intro_auth_pubkey_str, client_ephemeral_enc_privkey,
+                                                              intro_enc_pubkey, service_eph_pubkey, subcredential)
+    assert(client_ntor_auth_mac)
+    assert(client_ntor_key_seed)
+
+    assert(client_ntor_key_seed == service_ntor_key_seed)
+    assert(client_ntor_auth_mac == service_ntor_auth_mac)
+
+    print("DONE: tor dance [%s]" % repr(client_ntor_auth_mac))
+
+"""
+Do mixed test as follows:
+    1. C -> S (python mode)
+    2. C <- S (tor mode)
+    3. Client computes keys (python mode)
+"""
+def do_first_mixed_test():
+    subcredential = os.urandom(32)
+
+    client_ephemeral_enc_privkey = PrivateKey()
+    client_ephemeral_enc_pubkey = client_ephemeral_enc_privkey.get_public()
+    intro_enc_privkey = PrivateKey()
+    intro_enc_pubkey = intro_enc_privkey.get_public() # service-side enc key
+
+    intro_auth_pubkey_str = os.urandom(32)
+
+    # Let's do mixed
+    client_enc_key, client_mac_key = client_part1(intro_auth_pubkey_str, intro_enc_pubkey,
+                                                  client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey,
+                                                  subcredential)
+
+    service_enc_key, service_mac_key, service_ntor_auth_mac, service_ntor_key_seed, service_eph_pubkey = tor_server1(intro_auth_pubkey_str,
+                                                                                                                     intro_enc_privkey,
+                                                                                                                     client_ephemeral_enc_pubkey,
+                                                                                                                     subcredential)
+    assert(service_enc_key)
+    assert(service_mac_key)
+    assert(service_ntor_auth_mac)
+    assert(service_ntor_key_seed)
+    assert(service_eph_pubkey)
+
+    assert(client_enc_key == service_enc_key)
+    assert(client_mac_key == service_mac_key)
+
+    # Turn from bytes to key
+    service_eph_pubkey = curve25519mod.Public(service_eph_pubkey)
+
+    client_ntor_key_seed, client_auth_input_mac = client_part2(intro_auth_pubkey_str, client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey,
+                                                               intro_enc_pubkey, service_eph_pubkey)
+
+    assert(client_auth_input_mac == service_ntor_auth_mac)
+    assert(client_ntor_key_seed == service_ntor_key_seed)
+
+    print("DONE: 1st mixed dance [%s]" % repr(client_auth_input_mac))
+
+"""
+Do mixed test as follows:
+    1. C -> S (tor mode)
+    2. C <- S (python mode)
+    3. Client computes keys (tor mode)
+"""
+def do_second_mixed_test():
+    subcredential = os.urandom(32)
+
+    client_ephemeral_enc_privkey = PrivateKey()
+    client_ephemeral_enc_pubkey = client_ephemeral_enc_privkey.get_public()
+    intro_enc_privkey = PrivateKey()
+    intro_enc_pubkey = intro_enc_privkey.get_public() # service-side enc key
+
+    intro_auth_pubkey_str = os.urandom(32)
+
+    # Let's do mixed
+    client_enc_key, client_mac_key = tor_client1(intro_auth_pubkey_str, intro_enc_pubkey,
+                                                 client_ephemeral_enc_privkey, subcredential)
+    assert(client_enc_key)
+    assert(client_mac_key)
+
+    service_enc_key, service_mac_key, service_ntor_key_seed, service_ntor_auth_mac, service_ephemeral_pubkey = service_part1(intro_auth_pubkey_str, client_ephemeral_enc_pubkey, intro_enc_privkey, intro_enc_pubkey, subcredential)
+
+    client_ntor_auth_mac, client_ntor_key_seed  = tor_client2(intro_auth_pubkey_str, client_ephemeral_enc_privkey,
+                                                              intro_enc_pubkey, service_ephemeral_pubkey, subcredential)
+    assert(client_ntor_auth_mac)
+    assert(client_ntor_key_seed)
+
+    assert(client_ntor_key_seed == service_ntor_key_seed)
+    assert(client_ntor_auth_mac == service_ntor_auth_mac)
+
+    print("DONE: 2nd mixed dance [%s]" % repr(client_ntor_auth_mac))
+
+def do_mixed_tests():
+    do_first_mixed_test()
+    do_second_mixed_test()
+
+if __name__ == '__main__':
+    do_pure_python_ntor_test()
+    do_little_t_tor_ntor_test()
+    do_mixed_tests()
diff --git a/src/test/hs_test_helpers.c b/src/test/hs_test_helpers.c
new file mode 100644
index 0000000000..f2ae8398df
--- /dev/null
+++ b/src/test/hs_test_helpers.c
@@ -0,0 +1,325 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "test/test.h"
+#include "feature/nodelist/torcert.h"
+
+#include "feature/hs/hs_common.h"
+#include "test/hs_test_helpers.h"
+
+hs_desc_intro_point_t *
+hs_helper_build_intro_point(const ed25519_keypair_t *signing_kp, time_t now,
+                            const char *addr, int legacy)
+{
+  int ret;
+  ed25519_keypair_t auth_kp;
+  hs_desc_intro_point_t *intro_point = NULL;
+  hs_desc_intro_point_t *ip = hs_desc_intro_point_new();
+
+  /* For a usable intro point we need at least two link specifiers: One legacy
+   * keyid and one ipv4 */
+  {
+    hs_desc_link_specifier_t *ls_legacy = tor_malloc_zero(sizeof(*ls_legacy));
+    hs_desc_link_specifier_t *ls_v4 = tor_malloc_zero(sizeof(*ls_v4));
+    ls_legacy->type = LS_LEGACY_ID;
+    memcpy(ls_legacy->u.legacy_id, "0299F268FCA9D55CD157976D39AE92B4B455B3A8",
+           DIGEST_LEN);
+    ls_v4->u.ap.port = 9001;
+    int family = tor_addr_parse(&ls_v4->u.ap.addr, addr);
+    switch (family) {
+    case AF_INET:
+          ls_v4->type = LS_IPV4;
+          break;
+        case AF_INET6:
+          ls_v4->type = LS_IPV6;
+          break;
+        default:
+          /* Stop the test, not suppose to have an error. */
+          tt_int_op(family, OP_EQ, AF_INET);
+    }
+    smartlist_add(ip->link_specifiers, ls_legacy);
+    smartlist_add(ip->link_specifiers, ls_v4);
+  }
+
+  ret = ed25519_keypair_generate(&auth_kp, 0);
+  tt_int_op(ret, ==, 0);
+  ip->auth_key_cert = tor_cert_create(signing_kp, CERT_TYPE_AUTH_HS_IP_KEY,
+                                      &auth_kp.pubkey, now,
+                                      HS_DESC_CERT_LIFETIME,
+                                      CERT_FLAG_INCLUDE_SIGNING_KEY);
+  tt_assert(ip->auth_key_cert);
+
+  if (legacy) {
+    ip->legacy.key = crypto_pk_new();
+    tt_assert(ip->legacy.key);
+    ret = crypto_pk_generate_key(ip->legacy.key);
+    tt_int_op(ret, ==, 0);
+    ssize_t cert_len = tor_make_rsa_ed25519_crosscert(
+                                    &signing_kp->pubkey, ip->legacy.key,
+                                    now + HS_DESC_CERT_LIFETIME,
+                                    &ip->legacy.cert.encoded);
+    tt_assert(ip->legacy.cert.encoded);
+    tt_u64_op(cert_len, OP_GT, 0);
+    ip->legacy.cert.len = cert_len;
+  }
+
+  /* Encryption key. */
+  {
+    int signbit;
+    curve25519_keypair_t curve25519_kp;
+    ed25519_keypair_t ed25519_kp;
+    tor_cert_t *cross_cert;
+
+    ret = curve25519_keypair_generate(&curve25519_kp, 0);
+    tt_int_op(ret, ==, 0);
+    ed25519_keypair_from_curve25519_keypair(&ed25519_kp, &signbit,
+                                            &curve25519_kp);
+    cross_cert = tor_cert_create(signing_kp, CERT_TYPE_CROSS_HS_IP_KEYS,
+                                 &ed25519_kp.pubkey, time(NULL),
+                                 HS_DESC_CERT_LIFETIME,
+                                 CERT_FLAG_INCLUDE_SIGNING_KEY);
+    tt_assert(cross_cert);
+    ip->enc_key_cert = cross_cert;
+  }
+
+  intro_point = ip;
+ done:
+  if (intro_point == NULL)
+    tor_free(ip);
+
+  return intro_point;
+}
+
+/* Return a valid hs_descriptor_t object. If no_ip is set, no introduction
+ * points are added. */
+static hs_descriptor_t *
+hs_helper_build_hs_desc_impl(unsigned int no_ip,
+                             const ed25519_keypair_t *signing_kp)
+{
+  int ret;
+  int i;
+  time_t now = approx_time();
+  ed25519_keypair_t blinded_kp;
+  curve25519_keypair_t auth_ephemeral_kp;
+  hs_descriptor_t *descp = NULL, *desc = tor_malloc_zero(sizeof(*desc));
+
+  desc->plaintext_data.version = HS_DESC_SUPPORTED_FORMAT_VERSION_MAX;
+
+  /* Copy only the public key into the descriptor. */
+  memcpy(&desc->plaintext_data.signing_pubkey, &signing_kp->pubkey,
+         sizeof(ed25519_public_key_t));
+
+  uint64_t current_time_period = hs_get_time_period_num(0);
+  hs_build_blinded_keypair(signing_kp, NULL, 0,
+                           current_time_period, &blinded_kp);
+  /* Copy only the public key into the descriptor. */
+  memcpy(&desc->plaintext_data.blinded_pubkey, &blinded_kp.pubkey,
+         sizeof(ed25519_public_key_t));
+
+  desc->plaintext_data.signing_key_cert =
+    tor_cert_create(&blinded_kp, CERT_TYPE_SIGNING_HS_DESC,
+                    &signing_kp->pubkey, now, 3600,
+                    CERT_FLAG_INCLUDE_SIGNING_KEY);
+  tt_assert(desc->plaintext_data.signing_key_cert);
+  desc->plaintext_data.revision_counter = 42;
+  desc->plaintext_data.lifetime_sec = 3 * 60 * 60;
+
+  hs_get_subcredential(&signing_kp->pubkey, &blinded_kp.pubkey,
+                    desc->subcredential);
+
+  /* Setup superencrypted data section. */
+  ret = curve25519_keypair_generate(&auth_ephemeral_kp, 0);
+  tt_int_op(ret, ==, 0);
+  memcpy(&desc->superencrypted_data.auth_ephemeral_pubkey,
+         &auth_ephemeral_kp.pubkey,
+         sizeof(curve25519_public_key_t));
+
+  desc->superencrypted_data.clients = smartlist_new();
+  for (i = 0; i < HS_DESC_AUTH_CLIENT_MULTIPLE; i++) {
+    hs_desc_authorized_client_t *desc_client =
+      hs_desc_build_fake_authorized_client();
+    smartlist_add(desc->superencrypted_data.clients, desc_client);
+  }
+
+  /* Setup encrypted data section. */
+  desc->encrypted_data.create2_ntor = 1;
+  desc->encrypted_data.intro_auth_types = smartlist_new();
+  desc->encrypted_data.single_onion_service = 1;
+  smartlist_add(desc->encrypted_data.intro_auth_types, tor_strdup("ed25519"));
+  desc->encrypted_data.intro_points = smartlist_new();
+  if (!no_ip) {
+    /* Add four intro points. */
+    smartlist_add(desc->encrypted_data.intro_points,
+              hs_helper_build_intro_point(signing_kp, now, "1.2.3.4", 0));
+    smartlist_add(desc->encrypted_data.intro_points,
+              hs_helper_build_intro_point(signing_kp, now, "[2600::1]", 0));
+    smartlist_add(desc->encrypted_data.intro_points,
+              hs_helper_build_intro_point(signing_kp, now, "3.2.1.4", 1));
+    smartlist_add(desc->encrypted_data.intro_points,
+              hs_helper_build_intro_point(signing_kp, now, "5.6.7.8", 1));
+  }
+
+  descp = desc;
+ done:
+  if (descp == NULL)
+    tor_free(desc);
+
+  return descp;
+}
+
+/** Helper function to get the HS subcredential using the identity keypair of
+ *  an HS. Used to decrypt descriptors in unittests. */
+void
+hs_helper_get_subcred_from_identity_keypair(ed25519_keypair_t *signing_kp,
+                                            uint8_t *subcred_out)
+{
+  ed25519_keypair_t blinded_kp;
+  uint64_t current_time_period = hs_get_time_period_num(approx_time());
+  hs_build_blinded_keypair(signing_kp, NULL, 0,
+                           current_time_period, &blinded_kp);
+
+  hs_get_subcredential(&signing_kp->pubkey, &blinded_kp.pubkey,
+                       subcred_out);
+}
+
+/* Build a descriptor with introduction points. */
+hs_descriptor_t *
+hs_helper_build_hs_desc_with_ip(const ed25519_keypair_t *signing_kp)
+{
+  return hs_helper_build_hs_desc_impl(0, signing_kp);
+}
+
+/* Build a descriptor without any introduction points. */
+hs_descriptor_t *
+hs_helper_build_hs_desc_no_ip(const ed25519_keypair_t *signing_kp)
+{
+  return hs_helper_build_hs_desc_impl(1, signing_kp);
+}
+
+void
+hs_helper_desc_equal(const hs_descriptor_t *desc1,
+                     const hs_descriptor_t *desc2)
+{
+  char *addr1 = NULL, *addr2 = NULL;
+  /* Plaintext data section. */
+  tt_int_op(desc1->plaintext_data.version, OP_EQ,
+            desc2->plaintext_data.version);
+  tt_uint_op(desc1->plaintext_data.lifetime_sec, OP_EQ,
+             desc2->plaintext_data.lifetime_sec);
+  tt_assert(tor_cert_eq(desc1->plaintext_data.signing_key_cert,
+                        desc2->plaintext_data.signing_key_cert));
+  tt_mem_op(desc1->plaintext_data.signing_pubkey.pubkey, OP_EQ,
+            desc2->plaintext_data.signing_pubkey.pubkey,
+            ED25519_PUBKEY_LEN);
+  tt_mem_op(desc1->plaintext_data.blinded_pubkey.pubkey, OP_EQ,
+            desc2->plaintext_data.blinded_pubkey.pubkey,
+            ED25519_PUBKEY_LEN);
+  tt_u64_op(desc1->plaintext_data.revision_counter, ==,
+            desc2->plaintext_data.revision_counter);
+
+  /* NOTE: We can't compare the encrypted blob because when encoding the
+   * descriptor, the object is immutable thus we don't update it with the
+   * encrypted blob. As contrast to the decoding process where we populate a
+   * descriptor object. */
+
+  /* Superencrypted data section. */
+  tt_mem_op(desc1->superencrypted_data.auth_ephemeral_pubkey.public_key, OP_EQ,
+            desc2->superencrypted_data.auth_ephemeral_pubkey.public_key,
+            CURVE25519_PUBKEY_LEN);
+
+  /* Auth clients. */
+  {
+    tt_assert(desc1->superencrypted_data.clients);
+    tt_assert(desc2->superencrypted_data.clients);
+    tt_int_op(smartlist_len(desc1->superencrypted_data.clients), ==,
+              smartlist_len(desc2->superencrypted_data.clients));
+    for (int i=0;
+         i < smartlist_len(desc1->superencrypted_data.clients);
+         i++) {
+      hs_desc_authorized_client_t
+        *client1 = smartlist_get(desc1->superencrypted_data.clients, i),
+        *client2 = smartlist_get(desc2->superencrypted_data.clients, i);
+      tt_mem_op(client1->client_id, OP_EQ, client2->client_id,
+                sizeof(client1->client_id));
+      tt_mem_op(client1->iv, OP_EQ, client2->iv,
+                sizeof(client1->iv));
+      tt_mem_op(client1->encrypted_cookie, OP_EQ, client2->encrypted_cookie,
+                sizeof(client1->encrypted_cookie));
+    }
+  }
+
+  /* Encrypted data section. */
+  tt_uint_op(desc1->encrypted_data.create2_ntor, ==,
+             desc2->encrypted_data.create2_ntor);
+
+  /* Authentication type. */
+  tt_int_op(!!desc1->encrypted_data.intro_auth_types, ==,
+            !!desc2->encrypted_data.intro_auth_types);
+  if (desc1->encrypted_data.intro_auth_types &&
+      desc2->encrypted_data.intro_auth_types) {
+    tt_int_op(smartlist_len(desc1->encrypted_data.intro_auth_types), ==,
+              smartlist_len(desc2->encrypted_data.intro_auth_types));
+    for (int i = 0;
+         i < smartlist_len(desc1->encrypted_data.intro_auth_types);
+         i++) {
+      tt_str_op(smartlist_get(desc1->encrypted_data.intro_auth_types, i),OP_EQ,
+                smartlist_get(desc2->encrypted_data.intro_auth_types, i));
+    }
+  }
+
+  /* Introduction points. */
+  {
+    tt_assert(desc1->encrypted_data.intro_points);
+    tt_assert(desc2->encrypted_data.intro_points);
+    tt_int_op(smartlist_len(desc1->encrypted_data.intro_points), ==,
+              smartlist_len(desc2->encrypted_data.intro_points));
+    for (int i=0; i < smartlist_len(desc1->encrypted_data.intro_points); i++) {
+      hs_desc_intro_point_t *ip1 = smartlist_get(desc1->encrypted_data
+                                                 .intro_points, i),
+                            *ip2 = smartlist_get(desc2->encrypted_data
+                                                 .intro_points, i);
+      tt_assert(tor_cert_eq(ip1->auth_key_cert, ip2->auth_key_cert));
+      if (ip1->legacy.key) {
+        tt_int_op(crypto_pk_cmp_keys(ip1->legacy.key, ip2->legacy.key),
+                  OP_EQ, 0);
+      } else {
+        tt_mem_op(&ip1->enc_key, OP_EQ, &ip2->enc_key, CURVE25519_PUBKEY_LEN);
+      }
+
+      tt_int_op(smartlist_len(ip1->link_specifiers), ==,
+                smartlist_len(ip2->link_specifiers));
+      for (int j = 0; j < smartlist_len(ip1->link_specifiers); j++) {
+        hs_desc_link_specifier_t *ls1 = smartlist_get(ip1->link_specifiers, j),
+                                 *ls2 = smartlist_get(ip2->link_specifiers, j);
+        tt_int_op(ls1->type, ==, ls2->type);
+        switch (ls1->type) {
+          case LS_IPV4:
+          case LS_IPV6:
+            {
+              addr1 = tor_addr_to_str_dup(&ls1->u.ap.addr);
+              addr2 = tor_addr_to_str_dup(&ls2->u.ap.addr);
+              tt_str_op(addr1, OP_EQ, addr2);
+              tor_free(addr1);
+              tor_free(addr2);
+              tt_int_op(ls1->u.ap.port, ==, ls2->u.ap.port);
+            }
+            break;
+          case LS_LEGACY_ID:
+            tt_mem_op(ls1->u.legacy_id, OP_EQ, ls2->u.legacy_id,
+                      sizeof(ls1->u.legacy_id));
+            break;
+          default:
+            /* Unknown type, caught it and print its value. */
+            tt_int_op(ls1->type, OP_EQ, -1);
+        }
+      }
+    }
+  }
+
+ done:
+  tor_free(addr1);
+  tor_free(addr2);
+}
+
diff --git a/src/test/hs_test_helpers.h b/src/test/hs_test_helpers.h
new file mode 100644
index 0000000000..9662a83ba8
--- /dev/null
+++ b/src/test/hs_test_helpers.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_HS_TEST_HELPERS_H
+#define TOR_HS_TEST_HELPERS_H
+
+#include "trunnel/ed25519_cert.h"
+#include "feature/hs/hs_descriptor.h"
+
+/* Set of functions to help build and test descriptors. */
+hs_desc_intro_point_t *hs_helper_build_intro_point(
+                          const ed25519_keypair_t *signing_kp, time_t now,
+                          const char *addr, int legacy);
+hs_descriptor_t *hs_helper_build_hs_desc_no_ip(
+                                 const ed25519_keypair_t *signing_kp);
+hs_descriptor_t *hs_helper_build_hs_desc_with_ip(
+                                 const ed25519_keypair_t *signing_kp);
+void hs_helper_desc_equal(const hs_descriptor_t *desc1,
+                          const hs_descriptor_t *desc2);
+void
+hs_helper_get_subcred_from_identity_keypair(ed25519_keypair_t *signing_kp,
+                                            uint8_t *subcred_out);
+
+#endif /* !defined(TOR_HS_TEST_HELPERS_H) */
+
diff --git a/src/test/include.am b/src/test/include.am
index 7864d7d9fd..ecb7689579 100644
--- a/src/test/include.am
+++ b/src/test/include.am
@@ -5,10 +5,18 @@ TESTS_ENVIRONMENT = \
 	export PYTHON="$(PYTHON)"; \
 	export SHELL="$(SHELL)"; \
 	export abs_top_srcdir="$(abs_top_srcdir)"; \
+	export abs_top_builddir="$(abs_top_builddir)"; \
 	export builddir="$(builddir)"; \
-	export TESTING_TOR_BINARY="$(TESTING_TOR_BINARY)";
+	export TESTING_TOR_BINARY="$(TESTING_TOR_BINARY)"; \
+	export CARGO="$(CARGO)"; \
+	export EXTRA_CARGO_OPTIONS="$(EXTRA_CARGO_OPTIONS)"; \
+	export CARGO_ONLINE="$(CARGO_ONLINE)"; \
+	export CCLD="$(CCLD)"; \
+	export RUSTFLAGS="-C linker=`echo '$(CC)' | cut -d' ' -f 1` $(RUST_LINKER_OPTIONS)";
 
-TESTSCRIPTS = src/test/test_zero_length_keys.sh \
+TESTSCRIPTS = \
+	src/test/fuzz_static_testcases.sh \
+	src/test/test_zero_length_keys.sh \
 	src/test/test_workqueue_cancel.sh \
 	src/test/test_workqueue_efd.sh \
 	src/test/test_workqueue_efd2.sh \
@@ -17,23 +25,33 @@ TESTSCRIPTS = src/test/test_zero_length_keys.sh \
 	src/test/test_workqueue_socketpair.sh \
 	src/test/test_switch_id.sh
 
+if USE_RUST
+TESTSCRIPTS += \
+	src/test/test_rust.sh
+endif
+
 if USEPYTHON
-TESTSCRIPTS += src/test/test_ntor.sh src/test/test_bt.sh
+TESTSCRIPTS += src/test/test_ntor.sh src/test/test_hs_ntor.sh src/test/test_bt.sh
+TESTSCRIPTS += src/test/test_rebind.sh
 endif
 
 TESTS += src/test/test src/test/test-slow src/test/test-memwipe \
 	src/test/test_workqueue \
 	src/test/test_keygen.sh \
+	src/test/test_key_expiration.sh \
 	src/test/test-timers \
 	$(TESTSCRIPTS)
 
 # These flavors are run using automake's test-driver and test-network.sh
-TEST_CHUTNEY_FLAVORS = basic-min bridges-min hs-min single-onion
+TEST_CHUTNEY_FLAVORS = basic-min bridges-min hs-v2-min hs-v3-min \
+	single-onion-v23
 # only run if we can ping6 ::1 (localhost)
-TEST_CHUTNEY_FLAVORS_IPV6 = bridges+ipv6-min ipv6-exit-min hs-ipv6 \
-	single-onion-ipv6
+# IPv6-only v3 single onion services don't work yet, so we don't test the
+# single-onion-v23-ipv6-md flavor
+TEST_CHUTNEY_FLAVORS_IPV6 = bridges+ipv6-min ipv6-exit-min hs-v23-ipv6-md \
+	single-onion-ipv6-md
 # only run if we can find a stable (or simply another) version of tor
-TEST_CHUTNEY_FLAVORS_MIXED = mixed
+TEST_CHUTNEY_FLAVORS_MIXED = mixed+hs-v2
 
 ### This is a lovely feature, but it requires automake >= 1.12, and Tor
 ###  doesn't require that yet.
@@ -54,67 +72,95 @@ noinst_PROGRAMS+= \
 endif
 
 src_test_AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \
-        -DLOCALSTATEDIR="\"$(localstatedir)\"" \
-        -DBINDIR="\"$(bindir)\""	       \
-	-I"$(top_srcdir)/src/or" -I"$(top_srcdir)/src/ext" \
-	-I"$(top_srcdir)/src/trunnel" \
-	-I"$(top_srcdir)/src/ext/trunnel" \
-	-DTOR_UNIT_TESTS
+	-DLOCALSTATEDIR="\"$(localstatedir)\"" \
+	-DBINDIR="\"$(bindir)\""	       \
+	-DTOR_UNIT_TESTS                       \
+	$(AM_CPPFLAGS)
 
 # -L flags need to go in LDFLAGS. -l flags need to go in LDADD.
 # This seems to matter nowhere but on Windows, but I assure you that it
 # matters a lot there, and is quite hard to debug if you forget to do it.
 
-src_test_test_SOURCES = \
+src_test_test_SOURCES =
+
+if UNITTESTS_ENABLED
+src_test_test_SOURCES += \
 	src/test/log_test_helpers.c \
+	src/test/hs_test_helpers.c \
 	src/test/rend_test_helpers.c \
 	src/test/test.c \
 	src/test/test_accounting.c \
 	src/test/test_addr.c \
 	src/test/test_address.c \
 	src/test/test_address_set.c \
+	src/test/test_bridges.c \
 	src/test/test_buffers.c \
+	src/test/test_bwmgt.c \
 	src/test/test_cell_formats.c \
 	src/test/test_cell_queue.c \
 	src/test/test_channel.c \
+	src/test/test_channelpadding.c \
 	src/test/test_channeltls.c \
 	src/test/test_checkdir.c \
 	src/test/test_circuitlist.c \
 	src/test/test_circuitmux.c \
+	src/test/test_circuitbuild.c \
+	src/test/test_circuituse.c \
+	src/test/test_circuitstats.c \
 	src/test/test_compat_libevent.c \
 	src/test/test_config.c \
 	src/test/test_connection.c \
+	src/test/test_conscache.c \
+	src/test/test_consdiff.c \
+	src/test/test_consdiffmgr.c \
 	src/test/test_containers.c \
 	src/test/test_controller.c \
 	src/test/test_controller_events.c \
 	src/test/test_crypto.c \
-	src/test/test_dos.c \
+	src/test/test_crypto_ope.c \
 	src/test/test_data.c \
 	src/test/test_dir.c \
 	src/test/test_dir_common.c \
 	src/test/test_dir_handle_get.c \
+	src/test/test_dos.c \
 	src/test/test_entryconn.c \
 	src/test/test_entrynodes.c \
+	src/test/test_geoip.c \
 	src/test/test_guardfraction.c \
 	src/test/test_extorport.c \
 	src/test/test_hs.c \
+	src/test/test_hs_common.c \
+	src/test/test_hs_config.c \
+	src/test/test_hs_cell.c \
+	src/test/test_hs_ntor.c \
+	src/test/test_hs_service.c \
+	src/test/test_hs_client.c  \
+	src/test/test_hs_intropoint.c \
+	src/test/test_hs_control.c \
 	src/test/test_handles.c \
+	src/test/test_hs_cache.c \
+	src/test/test_hs_descriptor.c \
 	src/test/test_introduce.c \
 	src/test/test_keypin.c \
 	src/test/test_link_handshake.c \
 	src/test/test_logging.c \
+	src/test/test_mainloop.c \
 	src/test/test_microdesc.c \
 	src/test/test_nodelist.c \
 	src/test/test_oom.c \
 	src/test/test_oos.c \
 	src/test/test_options.c \
+	src/test/test_pem.c \
+	src/test/test_periodic_event.c \
 	src/test/test_policy.c \
 	src/test/test_procmon.c \
+	src/test/test_proto_http.c \
+	src/test/test_proto_misc.c \
 	src/test/test_protover.c \
 	src/test/test_pt.c \
-	src/test/test_pubsub.c \
 	src/test/test_relay.c \
 	src/test/test_relaycell.c \
+	src/test/test_relaycrypt.c \
 	src/test/test_rendcache.c \
 	src/test/test_replay.c \
 	src/test/test_router.c \
@@ -125,22 +171,40 @@ src_test_test_SOURCES = \
 	src/test/test_shared_random.c \
 	src/test/test_socks.c \
 	src/test/test_status.c \
+	src/test/test_storagedir.c \
 	src/test/test_threads.c \
 	src/test/test_tortls.c \
 	src/test/test_util.c \
 	src/test/test_util_format.c \
 	src/test/test_util_process.c \
+	src/test/test_voting_schedule.c \
+	src/test/test_x509.c \
 	src/test/test_helpers.c \
 	src/test/test_dns.c \
 	src/test/testing_common.c \
+	src/test/testing_rsakeys.c \
 	src/ext/tinytest.c
 
-src_test_test_slow_SOURCES = \
+if USE_NSS
+# ...
+else
+src_test_test_SOURCES += \
+	src/test/test_crypto_openssl.c \
+	src/test/test_tortls_openssl.c
+endif
+
+endif
+
+src_test_test_slow_SOURCES =
+if UNITTESTS_ENABLED
+src_test_test_slow_SOURCES += \
 	src/test/test_slow.c \
 	src/test/test_crypto_slow.c \
 	src/test/test_util_slow.c \
 	src/test/testing_common.c \
+	src/test/testing_rsakeys.c \
 	src/ext/tinytest.c
+endif
 
 src_test_test_memwipe_SOURCES = \
 	src/test/test-memwipe.c
@@ -166,23 +230,21 @@ src_test_test_switch_id_CPPFLAGS= $(src_test_AM_CPPFLAGS)
 src_test_test_switch_id_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 src_test_test_switch_id_LDFLAGS = @TOR_LDFLAGS_zlib@
 src_test_test_switch_id_LDADD = \
-	src/common/libor-testing.a \
-	src/common/libor-ctime-testing.a \
-	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@
-
-src_test_test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \
-        @TOR_LDFLAGS_libevent@
-src_test_test_LDADD = src/or/libtor-testing.a \
-	src/common/libor-crypto-testing.a \
-	$(LIBKECCAK_TINY) \
-	$(LIBDONNA) \
-	src/common/libor-testing.a \
-	src/common/libor-ctime-testing.a \
-	src/common/libor-event-testing.a \
-	src/trunnel/libor-trunnel-testing.a \
+	$(TOR_UTIL_TESTING_LIBS) \
+	$(rust_ldadd) \
+	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
+	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_USERENV@ \
+	@TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@
+
+src_test_test_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) \
+	@TOR_LDFLAGS_libevent@
+src_test_test_LDADD = \
+	$(TOR_INTERNAL_TESTING_LIBS) \
+	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \
-	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \
-	@TOR_SYSTEMD_LIBS@
+	$(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
+	@CURVE25519_LIBS@ \
+	@TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@
 
 src_test_test_slow_CPPFLAGS = $(src_test_test_CPPFLAGS)
 src_test_test_slow_CFLAGS = $(src_test_test_CFLAGS)
@@ -197,80 +259,110 @@ src_test_test_memwipe_LDADD = $(src_test_test_LDADD)
 # successfully with the libraries built with them.
 src_test_test_memwipe_LDFLAGS = $(src_test_test_LDFLAGS) @CFLAGS_BUGTRAP@
 
-src_test_bench_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \
-        @TOR_LDFLAGS_libevent@
-src_test_bench_LDADD = src/or/libtor.a src/common/libor.a \
-	src/common/libor-ctime.a \
-	src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \
-	src/common/libor-event.a src/trunnel/libor-trunnel.a \
+src_test_bench_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) \
+	@TOR_LDFLAGS_libevent@
+src_test_bench_LDADD = \
+	$(TOR_INTERNAL_LIBS) \
+	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \
-	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \
-	@TOR_SYSTEMD_LIBS@
-
-src_test_test_workqueue_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \
-        @TOR_LDFLAGS_libevent@
-src_test_test_workqueue_LDADD = src/or/libtor-testing.a \
-	src/common/libor-testing.a \
-	src/common/libor-ctime-testing.a \
-	src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \
-	src/common/libor-event-testing.a \
+	$(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
+	@CURVE25519_LIBS@ \
+	@TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@
+
+src_test_test_workqueue_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) \
+	@TOR_LDFLAGS_libevent@
+src_test_test_workqueue_LDADD = \
+	$(TOR_INTERNAL_TESTING_LIBS) \
+	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \
-	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
+	$(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
+	@CURVE25519_LIBS@ \
+	@TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@
 
 src_test_test_timers_CPPFLAGS = $(src_test_test_CPPFLAGS)
 src_test_test_timers_CFLAGS = $(src_test_test_CFLAGS)
 src_test_test_timers_LDADD = \
-	src/common/libor-testing.a \
-	src/common/libor-ctime-testing.a \
-	src/common/libor-event-testing.a \
-	src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \
+	src/lib/libtor-evloop-testing.a \
+	$(TOR_CRYPTO_TESTING_LIBS) \
+	$(TOR_UTIL_TESTING_LIBS) \
+	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \
-	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
+	$(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
+	@CURVE25519_LIBS@ \
+	@TOR_LZMA_LIBS@
 src_test_test_timers_LDFLAGS = $(src_test_test_LDFLAGS)
 
 noinst_HEADERS+= \
 	src/test/fakechans.h \
+	src/test/hs_test_helpers.h \
 	src/test/log_test_helpers.h \
 	src/test/rend_test_helpers.h \
 	src/test/test.h \
 	src/test/test_helpers.h \
 	src/test/test_dir_common.h \
+	src/test/test_connection.h \
+	src/test/test_tortls.h \
 	src/test/test_descriptors.inc \
 	src/test/example_extrainfo.inc \
 	src/test/failing_routerdescs.inc \
 	src/test/ed25519_vectors.inc \
 	src/test/test_descriptors.inc \
+	src/test/test_hs_descriptor.inc \
 	src/test/vote_descriptors.inc
 
 noinst_PROGRAMS+= src/test/test-ntor-cl
+noinst_PROGRAMS+= src/test/test-hs-ntor-cl
 src_test_test_ntor_cl_SOURCES  = src/test/test_ntor_cl.c
-src_test_test_ntor_cl_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@
-src_test_test_ntor_cl_LDADD = src/or/libtor.a src/common/libor.a \
-	src/common/libor-ctime.a \
-	src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \
+src_test_test_ntor_cl_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB)
+src_test_test_ntor_cl_LDADD = \
+	$(TOR_INTERNAL_LIBS) \
+	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
-	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
+	$(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
+	@CURVE25519_LIBS@ @TOR_LZMA_LIBS@
 src_test_test_ntor_cl_AM_CPPFLAGS =	       \
-	-I"$(top_srcdir)/src/or"
+	$(AM_CPPFLAGS)
 
+src_test_test_hs_ntor_cl_SOURCES  = src/test/test_hs_ntor_cl.c
+src_test_test_hs_ntor_cl_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB)
+src_test_test_hs_ntor_cl_LDADD = \
+	$(TOR_INTERNAL_LIBS) \
+	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
+	$(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
+src_test_test_hs_ntor_cl_AM_CPPFLAGS =	       \
+	$(AM_CPPFLAGS)
+
+
+if UNITTESTS_ENABLED
 noinst_PROGRAMS += src/test/test-bt-cl
 src_test_test_bt_cl_SOURCES = src/test/test_bt_cl.c
-src_test_test_bt_cl_LDADD = src/common/libor-testing.a \
-	src/common/libor-ctime-testing.a \
+src_test_test_bt_cl_LDADD = \
+	$(TOR_UTIL_TESTING_LIBS) \
+	$(rust_ldadd) \
 	@TOR_LIB_MATH@ \
-	@TOR_LIB_WS32@ @TOR_LIB_GDI@
+	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@
 src_test_test_bt_cl_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 src_test_test_bt_cl_CPPFLAGS= $(src_test_AM_CPPFLAGS) $(TEST_CPPFLAGS)
+endif
 
 EXTRA_DIST += \
 	src/test/bt_test.py \
 	src/test/ntor_ref.py \
+	src/test/hs_ntor_ref.py \
+	src/test/hs_build_address.py \
+	src/test/hs_indexes.py \
+	src/test/fuzz_static_testcases.sh \
 	src/test/slownacl_curve25519.py \
+	src/test/test_rebind.sh \
+	src/test/test_rebind.py \
 	src/test/zero_length_keys.sh \
+	src/test/rust_supp.txt \
 	src/test/test_keygen.sh \
+	src/test/test_key_expiration.sh \
 	src/test/test_zero_length_keys.sh \
-	src/test/test_ntor.sh src/test/test_bt.sh \
+	src/test/test_ntor.sh src/test/test_hs_ntor.sh src/test/test_bt.sh \
 	src/test/test-network.sh \
+	src/test/test_rust.sh \
 	src/test/test_switch_id.sh \
 	src/test/test_workqueue_cancel.sh \
 	src/test/test_workqueue_efd.sh \
@@ -279,3 +371,5 @@ EXTRA_DIST += \
 	src/test/test_workqueue_pipe2.sh \
 	src/test/test_workqueue_socketpair.sh
 
+test-rust:
+	$(TESTS_ENVIRONMENT) "$(abs_top_srcdir)/src/test/test_rust.sh"
diff --git a/src/test/log_test_helpers.c b/src/test/log_test_helpers.c
index c5368b6cbc..03c52dd6bd 100644
--- a/src/test/log_test_helpers.c
+++ b/src/test/log_test_helpers.c
@@ -1,8 +1,8 @@
-/* Copyright (c) 2015-2018, The Tor Project, Inc. */
+/* Copyright (c) 2015-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 #define LOG_PRIVATE
-#include "torlog.h"
-#include "log_test_helpers.h"
+#include "lib/log/log.h"
+#include "test/log_test_helpers.h"
 
 /**
  * \file log_test_helpers.c
diff --git a/src/test/log_test_helpers.h b/src/test/log_test_helpers.h
index a087b913f4..5d1c3c1914 100644
--- a/src/test/log_test_helpers.h
+++ b/src/test/log_test_helpers.h
@@ -1,7 +1,7 @@
-/* Copyright (c) 2014-2018, The Tor Project, Inc. */
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#include "or.h"
+#include "core/or/or.h"
 
 #ifndef TOR_LOG_TEST_HELPERS_H
 #define TOR_LOG_TEST_HELPERS_H
@@ -33,7 +33,7 @@ void mock_dump_saved_logs(void);
 #define assert_log_predicate(predicate, failure_msg)   \
   do {                                                 \
     if (!(predicate)) {                                \
-      tt_fail_msg((failure_msg));                      \
+      TT_FAIL(failure_msg);                            \
       mock_dump_saved_logs();                          \
       TT_EXIT_TEST_FUNCTION;                           \
     }                                                  \
@@ -41,70 +41,75 @@ void mock_dump_saved_logs(void);
 
 #define expect_log_msg(str)                             \
   assert_log_predicate(mock_saved_log_has_message(str), \
-                "expected log to contain " # str);
+                       ("expected log to contain \"%s\"", str));
 
 #define expect_log_msg_containing(str) \
   assert_log_predicate(mock_saved_log_has_message_containing(str), \
-                "expected log to contain " # str);
+                       ("expected log to contain \"%s\"", str));
 
 #define expect_log_msg_not_containing(str) \
   assert_log_predicate(mock_saved_log_has_message_not_containing(str), \
-                "expected log to not contain " # str);
+                       ("expected log to not contain \"%s\"", str));
 
 #define expect_log_msg_containing_either(str1, str2)                    \
   assert_log_predicate(mock_saved_log_has_message_containing(str1) ||   \
                        mock_saved_log_has_message_containing(str2),     \
-                       "expected log to contain " # str1 " or " # str2);
+           ("expected log to contain \"%s\" or \"%s\"", str1, str2));
 
 #define expect_log_msg_containing_either3(str1, str2, str3)             \
   assert_log_predicate(mock_saved_log_has_message_containing(str1) ||   \
                        mock_saved_log_has_message_containing(str2) ||   \
                        mock_saved_log_has_message_containing(str3),     \
-                       "expected log to contain " # str1 " or " # str2  \
-                       " or " # str3);
+           ("expected log to contain \"%s\" or \"%s\" or \"%s\"",       \
+                        str1, str2, str3))
 
 #define expect_log_msg_containing_either4(str1, str2, str3, str4)       \
   assert_log_predicate(mock_saved_log_has_message_containing(str1) ||   \
                        mock_saved_log_has_message_containing(str2) ||   \
                        mock_saved_log_has_message_containing(str3) ||   \
                        mock_saved_log_has_message_containing(str4),     \
-                       "expected log to contain " # str1 " or " # str2  \
-                       " or " # str3 " or " # str4);
+       ("expected log to contain \"%s\" or \"%s\" or \"%s\" or \"%s\"", \
+                        str1, str2, str3, str4))
 
 #define expect_single_log_msg(str) \
   do {                                                                  \
                                                                         \
     assert_log_predicate(mock_saved_log_has_message_containing(str) &&  \
                          mock_saved_log_n_entries() == 1,               \
-                  "expected log to contain exactly 1 message: " # str); \
+                         ("expected log to contain exactly 1 message \"%s\"", \
+                          str));                                        \
   } while (0);
 
 #define expect_single_log_msg_containing(str) \
   do {                                                                  \
     assert_log_predicate(mock_saved_log_has_message_containing(str)&&   \
                          mock_saved_log_n_entries() == 1 ,              \
-            "expected log to contain 1 message, containing" # str);     \
+                    ("expected log to contain 1 message, containing \"%s\"",\
+                     str));                                             \
   } while (0);
 
 #define expect_no_log_msg(str) \
   assert_log_predicate(!mock_saved_log_has_message(str), \
-                "expected log to not contain " # str);
+                       ("expected log to not contain \"%s\"",str));
+
+#define expect_no_log_msg_containing(str) \
+  assert_log_predicate(!mock_saved_log_has_message_containing(str), \
+                       ("expected log to not contain \"%s\"", str));
 
 #define expect_log_severity(severity) \
   assert_log_predicate(mock_saved_log_has_severity(severity), \
-                "expected log to contain severity " # severity);
+                       ("expected log to contain severity " # severity));
 
 #define expect_no_log_severity(severity) \
   assert_log_predicate(!mock_saved_log_has_severity(severity), \
-                "expected log to not contain severity " # severity);
+                       ("expected log to not contain severity " # severity));
 
 #define expect_log_entry() \
   assert_log_predicate(mock_saved_log_has_entry(), \
-                "expected log to contain entries");
+                       ("expected log to contain entries"));
 
 #define expect_no_log_entry() \
   assert_log_predicate(!mock_saved_log_has_entry(), \
-                "expected log to not contain entries");
-
-#endif
+                       ("expected log to not contain entries"));
 
+#endif /* !defined(TOR_LOG_TEST_HELPERS_H) */
diff --git a/src/test/ntor_ref.py b/src/test/ntor_ref.py
index 5ec117f2bd..204f05e2ad 100755
--- a/src/test/ntor_ref.py
+++ b/src/test/ntor_ref.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright 2012-2015, The Tor Project, Inc
+# Copyright 2012-2019, The Tor Project, Inc
 # See LICENSE for licensing information
 
 """
diff --git a/src/test/ope_ref.py b/src/test/ope_ref.py
new file mode 100644
index 0000000000..f9bd97c546
--- /dev/null
+++ b/src/test/ope_ref.py
@@ -0,0 +1,40 @@
+#!/usr/bin/python3
+# Copyright 2018-2019, The Tor Project, Inc. See LICENSE for licensing info.
+
+# Reference implementation for our rudimentary OPE code, used to
+# generate test vectors. See crypto_ope.c for more details.
+
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+from cryptography.hazmat.primitives.ciphers.algorithms import AES
+from cryptography.hazmat.backends import default_backend
+
+from binascii import a2b_hex
+
+#randomly generated and values.
+KEY = a2b_hex(
+    "19e05891d55232c08c2cad91d612fdb9cbd6691949a0742434a76c80bc6992fe")
+PTS = [ 121132, 82283, 72661, 72941, 123122, 12154, 121574, 11391, 65845,
+        86301, 61284, 70505, 30438, 60150, 114800, 109403, 21893, 123569,
+        95617, 48561, 53334, 92746, 7110, 9612, 106958, 46889, 87790, 68878,
+        47917, 121128, 108602, 28217, 69498, 63870, 57542, 122148, 46254,
+        42850, 92661, 57720]
+
+IV = b'\x00' * 16
+
+backend = default_backend()
+
+def words():
+    cipher = Cipher(algorithms.AES(KEY), modes.CTR(IV), backend=backend)
+    e = cipher.encryptor()
+    while True:
+        v = e.update(b'\x00\x00')
+        yield v[0] + 256 * v[1] + 1
+
+def encrypt(n):
+    return sum(w for w, _ in zip(words(), range(n)))
+
+def example(n):
+    return ' {{ {}, UINT64_C({}) }},'.format(n, encrypt(n))
+
+for v in PTS:
+    print(example(v))
diff --git a/src/test/rend_test_helpers.c b/src/test/rend_test_helpers.c
index 377337bcb9..f12d193cc5 100644
--- a/src/test/rend_test_helpers.c
+++ b/src/test/rend_test_helpers.c
@@ -1,10 +1,15 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#include "or.h"
-#include "test.h"
-#include "rendcommon.h"
-#include "rend_test_helpers.h"
+#include "core/or/or.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "test/test.h"
+#include "feature/rend/rendcommon.h"
+#include "test/rend_test_helpers.h"
+
+#include "core/or/extend_info_st.h"
+#include "feature/rend/rend_intro_point_st.h"
+#include "feature/rend/rend_service_descriptor_st.h"
 
 void
 generate_desc(int time_diff, rend_encoded_v2_service_descriptor_t **desc,
@@ -71,3 +76,19 @@ create_descriptor(rend_service_descriptor_t **generated, char **service_id,
   crypto_pk_free(pk2);
 }
 
+rend_data_t *
+mock_rend_data(const char *onion_address)
+{
+  rend_data_v2_t *v2_data = tor_malloc_zero(sizeof(*v2_data));
+  rend_data_t *rend_query = &v2_data->base_;
+  rend_query->version = 2;
+
+  strlcpy(v2_data->onion_address, onion_address,
+          sizeof(v2_data->onion_address));
+  v2_data->auth_type = REND_NO_AUTH;
+  rend_query->hsdirs_fp = smartlist_new();
+  smartlist_add(rend_query->hsdirs_fp, tor_memdup("aaaaaaaaaaaaaaaaaaaaaaaa",
+                                                 DIGEST_LEN));
+  return rend_query;
+}
+
diff --git a/src/test/rend_test_helpers.h b/src/test/rend_test_helpers.h
index 180a4e8fde..c10da52cd7 100644
--- a/src/test/rend_test_helpers.h
+++ b/src/test/rend_test_helpers.h
@@ -1,7 +1,7 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#include "or.h"
+#include "core/or/or.h"
 
 #ifndef TOR_REND_TEST_HELPERS_H
 #define TOR_REND_TEST_HELPERS_H
@@ -10,6 +10,7 @@ void generate_desc(int time_diff, rend_encoded_v2_service_descriptor_t **desc,
                    char **service_id, int intro_points);
 void create_descriptor(rend_service_descriptor_t **generated,
                        char **service_id, int intro_points);
+rend_data_t *mock_rend_data(const char *onion_address);
 
-#endif
+#endif /* !defined(TOR_REND_TEST_HELPERS_H) */
 
diff --git a/src/test/rust_supp.txt b/src/test/rust_supp.txt
new file mode 100644
index 0000000000..7fa50f3fb1
--- /dev/null
+++ b/src/test/rust_supp.txt
@@ -0,0 +1 @@
+leak:backtrace_alloc
diff --git a/src/test/test-child.c b/src/test/test-child.c
index fdf3ccec0a..11a1695cad 100644
--- a/src/test/test-child.c
+++ b/src/test/test-child.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2016, The Tor Project, Inc. */
+/* Copyright (c) 2011-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -8,7 +8,7 @@
 #include <windows.h>
 #else
 #include <unistd.h>
-#endif
+#endif /* defined(_WIN32) */
 #include <string.h>
 
 #ifdef _WIN32
diff --git a/src/test/test-memwipe.c b/src/test/test-memwipe.c
index c650b99c85..43754ed1c2 100644
--- a/src/test/test-memwipe.c
+++ b/src/test/test-memwipe.c
@@ -1,12 +1,20 @@
+/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
 #include "orconfig.h"
+#include "lib/crypt_ops/crypto_util.h"
+
+#include "lib/intmath/cmp.h"
+#include "lib/malloc/malloc.h"
+
 #include <string.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <stdlib.h>
 
-#include "crypto.h"
-#include "compat.h"
-#include "util.h"
+#ifdef HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
 
 static unsigned fill_a_buffer_memset(void) __attribute__((noinline));
 static unsigned fill_a_buffer_memwipe(void) __attribute__((noinline));
@@ -36,7 +44,7 @@ const char *s = NULL;
     sum += (unsigned char)buf[i];                                       \
   }
 
-#ifdef __OpenBSD__
+#ifdef OpenBSD
 /* Disable some of OpenBSD's malloc protections for this test. This helps
  * us do bad things, such as access freed buffers, without crashing. */
 extern const char *malloc_options;
@@ -213,4 +221,3 @@ main(int argc, char **argv)
     return 0;
   }
 }
-
diff --git a/src/test/test-network.sh b/src/test/test-network.sh
index 6e0f286573..b7a9f1b3c0 100755
--- a/src/test/test-network.sh
+++ b/src/test/test-network.sh
@@ -52,12 +52,12 @@ done
 #  - if $PWD looks like a tor build directory, set it to $PWD, or
 #  - unset $TOR_DIR, and let chutney fall back to finding tor binaries in $PATH
 if [ ! -d "$TOR_DIR" ]; then
-    if [ -d "$BUILDDIR/src/or" -a -d "$BUILDDIR/src/tools" ]; then
+    if [ -d "$BUILDDIR/src/core/or" -a -d "$BUILDDIR/src/tools" ]; then
         # Choose the build directory
         # But only if it looks like one
         $ECHO "$myname: \$TOR_DIR not set, trying \$BUILDDIR"
         TOR_DIR="$BUILDDIR"
-    elif [ -d "$PWD/src/or" -a -d "$PWD/src/tools" ]; then
+    elif [ -d "$PWD/src/core/or" -a -d "$PWD/src/tools" ]; then
         # Guess the tor directory is the current directory
         # But only if it looks like one
         $ECHO "$myname: \$TOR_DIR not set, trying \$PWD"
diff --git a/src/test/test-timers.c b/src/test/test-timers.c
index b5fcade7f8..c80fb1e305 100644
--- a/src/test/test-timers.c
+++ b/src/test/test-timers.c
@@ -1,4 +1,4 @@
-/* Copyright 2016, The Tor Project, Inc. */
+/* Copyright 2016-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -7,13 +7,13 @@
 #include <stdio.h>
 #include <string.h>
 
-#include <event2/event.h>
-
-#include "compat.h"
-#include "compat_libevent.h"
-#include "crypto.h"
-#include "timers.h"
-#include "util.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/evloop/timers.h"
+#include "lib/crypt_ops/crypto_init.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/log/util_bug.h"
+#include "lib/time/compat_time.h"
+#include "lib/wallclock/timeval.h"
 
 #define N_TIMERS 1000
 #define MAX_DURATION 30
@@ -50,7 +50,7 @@ timer_cb(tor_timer_t *t, void *arg, const monotime_t *now_mono)
 
   // printf("%d / %d\n",n_fired, N_TIMERS);
   if (n_fired == n_active_timers) {
-    event_base_loopbreak(tor_libevent_get_base());
+    tor_libevent_exit_loop_after_callback(tor_libevent_get_base());
   }
 }
 
@@ -63,6 +63,10 @@ main(int argc, char **argv)
   memset(&cfg, 0, sizeof(cfg));
   tor_libevent_initialize(&cfg);
   timers_initialize();
+  init_logging(1);
+
+  if (crypto_global_init(0, NULL, NULL) < 0)
+    return 1;
 
   int i;
   int ret;
@@ -90,7 +94,7 @@ main(int argc, char **argv)
     --n_active_timers;
   }
 
-  event_base_loop(tor_libevent_get_base(), 0);
+  tor_libevent_run_event_loop(tor_libevent_get_base(), 0);
 
   int64_t total_difference = 0;
   uint64_t total_square_difference = 0;
@@ -107,8 +111,8 @@ main(int argc, char **argv)
     total_square_difference += diff*diff;
   }
   const int64_t mean_diff = total_difference / n_active_timers;
-  printf("mean difference: "I64_FORMAT" usec\n",
-         I64_PRINTF_ARG(mean_diff));
+  printf("mean difference: %"PRId64" usec\n",
+         (mean_diff));
 
   const double mean_sq = ((double)total_square_difference)/ n_active_timers;
   const double sq_mean = mean_diff * mean_diff;
@@ -133,7 +137,7 @@ main(int argc, char **argv)
     ret = 0;
   }
 
-  timer_free(NULL);
+  timer_free_(NULL);
 
   for (i = 0; i < N_TIMERS; ++i) {
     timer_free(timers[i]);
@@ -141,4 +145,3 @@ main(int argc, char **argv)
   timers_shutdown();
   return ret;
 }
-
diff --git a/src/test/test.c b/src/test/test.c
index 4060ae1371..58b468775c 100644
--- a/src/test/test.c
+++ b/src/test/test.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,6 +9,9 @@
  **/
 
 #include "orconfig.h"
+#include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "app/config/or_state_st.h"
 
 #include <stdio.h>
 #ifdef HAVE_FCNTL_H
@@ -20,43 +23,47 @@
 #include <direct.h>
 #else
 #include <dirent.h>
-#endif
+#endif /* defined(_WIN32) */
 
 #include <math.h>
 
 /* These macros pull in declarations for some functions and structures that
  * are typically file-private. */
-#define GEOIP_PRIVATE
 #define ROUTER_PRIVATE
 #define CIRCUITSTATS_PRIVATE
 #define CIRCUITLIST_PRIVATE
-#define MAIN_PRIVATE
+#define MAINLOOP_PRIVATE
 #define STATEFILE_PRIVATE
 
-#include "or.h"
-#include "backtrace.h"
-#include "buffers.h"
-#include "circuitlist.h"
-#include "circuitstats.h"
-#include "config.h"
-#include "connection_edge.h"
-#include "geoip.h"
-#include "rendcommon.h"
-#include "rendcache.h"
-#include "test.h"
-#include "torgzip.h"
-#include "main.h"
-#include "memarea.h"
-#include "onion.h"
-#include "onion_ntor.h"
-#include "onion_fast.h"
-#include "onion_tap.h"
-#include "policies.h"
-#include "rephist.h"
-#include "routerparse.h"
-#include "statefile.h"
-#include "crypto_curve25519.h"
-#include "onion_ntor.h"
+#include "core/or/or.h"
+#include "lib/err/backtrace.h"
+#include "lib/container/buffers.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitstats.h"
+#include "lib/compress/compress.h"
+#include "app/config/config.h"
+#include "core/or/connection_edge.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/rend/rendcache.h"
+#include "feature/rend/rendparse.h"
+#include "test/test.h"
+#include "core/mainloop/mainloop.h"
+#include "lib/memarea/memarea.h"
+#include "core/or/onion.h"
+#include "core/crypto/onion_ntor.h"
+#include "core/crypto/onion_fast.h"
+#include "core/crypto/onion_tap.h"
+#include "core/or/policies.h"
+#include "feature/stats/rephist.h"
+#include "app/config/statefile.h"
+#include "lib/crypt_ops/crypto_curve25519.h"
+
+#include "core/or/extend_info_st.h"
+#include "core/or/or_circuit_st.h"
+#include "feature/rend/rend_encoded_v2_service_descriptor_st.h"
+#include "feature/rend/rend_intro_point_st.h"
+#include "feature/rend/rend_service_descriptor_st.h"
+#include "feature/relay/onion_queue.h"
 
 /** Run unit tests for the onion handshake code. */
 static void
@@ -136,8 +143,10 @@ test_bad_onion_handshake(void *arg)
 
   /* Server: Case 1: the encrypted data is degenerate. */
   memset(junk_buf, 0, sizeof(junk_buf));
-  crypto_pk_public_hybrid_encrypt(pk, junk_buf2, TAP_ONIONSKIN_CHALLENGE_LEN,
-                               junk_buf, DH_KEY_LEN, PK_PKCS1_OAEP_PADDING, 1);
+  crypto_pk_obsolete_public_hybrid_encrypt(pk,
+                               junk_buf2, TAP_ONIONSKIN_CHALLENGE_LEN,
+                               junk_buf, DH1024_KEY_LEN,
+                               PK_PKCS1_OAEP_PADDING, 1);
   tt_int_op(-1, OP_EQ,
             onion_skin_TAP_server_handshake(junk_buf2, pk, NULL,
                                             s_buf, s_keys, 40));
@@ -167,7 +176,7 @@ test_bad_onion_handshake(void *arg)
                                             s_buf, s_keys, 40));
   c_buf[64] ^= 33;
 
-  /* (Let the server procede) */
+  /* (Let the server proceed) */
   tt_int_op(0, OP_EQ,
             onion_skin_TAP_server_handshake(c_buf, pk, NULL,
                                             s_buf, s_keys, 40));
@@ -338,13 +347,25 @@ test_onion_queues(void *arg)
   tt_int_op(0,OP_EQ, onion_num_pending(ONION_HANDSHAKE_TYPE_NTOR));
 
  done:
-  circuit_free(TO_CIRCUIT(circ1));
-  circuit_free(TO_CIRCUIT(circ2));
+  circuit_free_(TO_CIRCUIT(circ1));
+  circuit_free_(TO_CIRCUIT(circ2));
   tor_free(create1);
   tor_free(create2);
   tor_free(onionskin);
 }
 
+static crypto_cipher_t *crypto_rand_aes_cipher = NULL;
+
+// Mock replacement for crypto_rand: Generates bytes from a provided AES_CTR
+// cipher in <b>crypto_rand_aes_cipher</b>.
+static void
+crypto_rand_deterministic_aes(char *out, size_t n)
+{
+  tor_assert(crypto_rand_aes_cipher);
+  memset(out, 0, n);
+  crypto_cipher_crypt_inplace(crypto_rand_aes_cipher, out, n);
+}
+
 static void
 test_circuit_timeout(void *arg)
 {
@@ -374,6 +395,11 @@ test_circuit_timeout(void *arg)
 
   state = or_state_new();
 
+  // Use a deterministic RNG here, or else we'll get nondeterministic
+  // coverage in some of the circuitstats functions.
+  MOCK(crypto_rand, crypto_rand_deterministic_aes);
+  crypto_rand_aes_cipher = crypto_cipher_new("xyzzyplughplover");
+
   circuitbuild_running_unit_tests();
 #define timeout0 (build_time_t)(30*1000.0)
   initial.Xm = 3000;
@@ -402,11 +428,11 @@ test_circuit_timeout(void *arg)
   } while (fabs(circuit_build_times_cdf(&initial, timeout0) -
                 circuit_build_times_cdf(&initial, timeout1)) > 0.02);
 
-  tt_assert(estimate.total_build_times <= CBT_NCIRCUITS_TO_OBSERVE);
+  tt_int_op(estimate.total_build_times, OP_LE, CBT_NCIRCUITS_TO_OBSERVE);
 
   circuit_build_times_update_state(&estimate, state);
   circuit_build_times_free_timeouts(&final);
-  tt_assert(circuit_build_times_parse_state(&final, state) == 0);
+  tt_int_op(circuit_build_times_parse_state(&final, state), OP_EQ, 0);
 
   circuit_build_times_update_alpha(&final);
   timeout2 = circuit_build_times_calculate_timeout(&final,
@@ -484,7 +510,7 @@ test_circuit_timeout(void *arg)
       }
     }
 
-    tt_assert(estimate.liveness.after_firsthop_idx == 0);
+    tt_int_op(estimate.liveness.after_firsthop_idx, OP_EQ, 0);
     tt_assert(final.liveness.after_firsthop_idx ==
                 CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT-1);
 
@@ -508,6 +534,8 @@ test_circuit_timeout(void *arg)
   circuit_build_times_free_timeouts(&final);
   or_state_free(state);
   teardown_periodic_events();
+  UNMOCK(crypto_rand);
+  crypto_cipher_free(crypto_rand_aes_cipher);
 }
 
 /** Test encoding and parsing of rendezvous service descriptors. */
@@ -527,25 +555,8 @@ test_rend_fns(void *arg)
   size_t intro_points_size;
   size_t encoded_size;
   int i;
-  char address1[] = "fooaddress.onion";
-  char address2[] = "aaaaaaaaaaaaaaaa.onion";
-  char address3[] = "fooaddress.exit";
-  char address4[] = "www.torproject.org";
-  char address5[] = "foo.abcdefghijklmnop.onion";
-  char address6[] = "foo.bar.abcdefghijklmnop.onion";
-  char address7[] = ".abcdefghijklmnop.onion";
 
   (void)arg;
-  tt_assert(BAD_HOSTNAME == parse_extended_hostname(address1));
-  tt_assert(ONION_HOSTNAME == parse_extended_hostname(address2));
-  tt_str_op(address2,OP_EQ, "aaaaaaaaaaaaaaaa");
-  tt_assert(EXIT_HOSTNAME == parse_extended_hostname(address3));
-  tt_assert(NORMAL_HOSTNAME == parse_extended_hostname(address4));
-  tt_assert(ONION_HOSTNAME == parse_extended_hostname(address5));
-  tt_str_op(address5,OP_EQ, "abcdefghijklmnop");
-  tt_assert(ONION_HOSTNAME == parse_extended_hostname(address6));
-  tt_str_op(address6,OP_EQ, "abcdefghijklmnop");
-  tt_assert(BAD_HOSTNAME == parse_extended_hostname(address7));
 
   /* Initialize the service cache. */
   rend_cache_init();
@@ -581,20 +592,21 @@ test_rend_fns(void *arg)
     intro->intro_key = crypto_pk_dup_key(pk2);
     smartlist_add(generated->intro_nodes, intro);
   }
-  tt_assert(rend_encode_v2_descriptors(descs, generated, now, 0,
-                                         REND_NO_AUTH, NULL, NULL) > 0);
-  tt_assert(rend_compute_v2_desc_id(computed_desc_id, service_id_base32,
-                                      NULL, now, 0) == 0);
+  int rv = rend_encode_v2_descriptors(descs, generated, now, 0,
+                                      REND_NO_AUTH, NULL, NULL);
+  tt_int_op(rv, OP_GT, 0);
+  rv = rend_compute_v2_desc_id(computed_desc_id, service_id_base32, NULL,
+                               now, 0);
+  tt_int_op(rv, OP_EQ, 0);
   tt_mem_op(((rend_encoded_v2_service_descriptor_t *)
              smartlist_get(descs, 0))->desc_id, OP_EQ,
             computed_desc_id, DIGEST_LEN);
-  tt_assert(rend_parse_v2_service_descriptor(&parsed, parsed_desc_id,
-                                             &intro_points_encrypted,
-                                             &intro_points_size,
-                                             &encoded_size,
-                                              &next_desc,
-                             ((rend_encoded_v2_service_descriptor_t *)
-                                 smartlist_get(descs, 0))->desc_str, 1) == 0);
+  rv = rend_parse_v2_service_descriptor(&parsed, parsed_desc_id,
+               &intro_points_encrypted, &intro_points_size, &encoded_size,
+               &next_desc,
+          ((rend_encoded_v2_service_descriptor_t *)smartlist_get(descs, 0))
+                                        ->desc_str, 1);
+  tt_int_op(rv, OP_EQ, 0);
   tt_assert(parsed);
   tt_mem_op(((rend_encoded_v2_service_descriptor_t *)
          smartlist_get(descs, 0))->desc_id,OP_EQ, parsed_desc_id, DIGEST_LEN);
@@ -625,7 +637,7 @@ test_rend_fns(void *arg)
  done:
   if (descs) {
     for (i = 0; i < smartlist_len(descs); i++)
-      rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i));
+      rend_encoded_v2_service_descriptor_free_(smartlist_get(descs, i));
     smartlist_free(descs);
   }
   if (parsed)
@@ -639,376 +651,6 @@ test_rend_fns(void *arg)
   tor_free(intro_points_encrypted);
 }
 
-  /* Record odd numbered fake-IPs using ipv6, even numbered fake-IPs
-   * using ipv4.  Since our fake geoip database is the same between
-   * ipv4 and ipv6, we should get the same result no matter which
-   * address family we pick for each IP. */
-#define SET_TEST_ADDRESS(i) do {                \
-    if ((i) & 1) {                              \
-      SET_TEST_IPV6(i);                         \
-      tor_addr_from_in6(&addr, &in6);           \
-    } else {                                    \
-      tor_addr_from_ipv4h(&addr, (uint32_t) i); \
-    }                                           \
-  } while (0)
-
-  /* Make sure that country ID actually works. */
-#define SET_TEST_IPV6(i) \
-  do {                                                          \
-    set_uint32(in6.s6_addr + 12, htonl((uint32_t) (i)));        \
-  } while (0)
-#define CHECK_COUNTRY(country, val) do {                                \
-    /* test ipv4 country lookup */                                      \
-    tt_str_op(country, OP_EQ,                                              \
-               geoip_get_country_name(geoip_get_country_by_ipv4(val))); \
-    /* test ipv6 country lookup */                                      \
-    SET_TEST_IPV6(val);                                                 \
-    tt_str_op(country, OP_EQ,                                              \
-               geoip_get_country_name(geoip_get_country_by_ipv6(&in6))); \
-  } while (0)
-
-/** Run unit tests for GeoIP code. */
-static void
-test_geoip(void *arg)
-{
-  int i, j;
-  time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */
-  char *s = NULL, *v = NULL;
-  const char *bridge_stats_1 =
-      "bridge-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-      "bridge-ips zz=24,xy=8\n"
-      "bridge-ip-versions v4=16,v6=16\n"
-      "bridge-ip-transports <OR>=24\n",
-  *dirreq_stats_1 =
-      "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-      "dirreq-v3-ips ab=8\n"
-      "dirreq-v3-reqs ab=8\n"
-      "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0,"
-          "not-modified=0,busy=0\n"
-      "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n"
-      "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n",
-  *dirreq_stats_2 =
-      "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-      "dirreq-v3-ips \n"
-      "dirreq-v3-reqs \n"
-      "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0,"
-          "not-modified=0,busy=0\n"
-      "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n"
-      "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n",
-  *dirreq_stats_3 =
-      "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-      "dirreq-v3-ips \n"
-      "dirreq-v3-reqs \n"
-      "dirreq-v3-resp ok=8,not-enough-sigs=0,unavailable=0,not-found=0,"
-          "not-modified=0,busy=0\n"
-      "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n"
-      "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n",
-  *dirreq_stats_4 =
-      "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-      "dirreq-v3-ips \n"
-      "dirreq-v3-reqs \n"
-      "dirreq-v3-resp ok=8,not-enough-sigs=0,unavailable=0,not-found=0,"
-          "not-modified=0,busy=0\n"
-      "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n"
-      "dirreq-v3-tunneled-dl complete=0,timeout=0,running=4\n",
-  *entry_stats_1 =
-      "entry-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-      "entry-ips ab=8\n",
-  *entry_stats_2 =
-      "entry-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-      "entry-ips \n";
-  tor_addr_t addr;
-  struct in6_addr in6;
-
-  /* Populate the DB a bit.  Add these in order, since we can't do the final
-   * 'sort' step.  These aren't very good IP addresses, but they're perfectly
-   * fine uint32_t values. */
-  (void)arg;
-  tt_int_op(0,OP_EQ, geoip_parse_entry("10,50,AB", AF_INET));
-  tt_int_op(0,OP_EQ, geoip_parse_entry("52,90,XY", AF_INET));
-  tt_int_op(0,OP_EQ, geoip_parse_entry("95,100,AB", AF_INET));
-  tt_int_op(0,OP_EQ, geoip_parse_entry("\"105\",\"140\",\"ZZ\"", AF_INET));
-  tt_int_op(0,OP_EQ, geoip_parse_entry("\"150\",\"190\",\"XY\"", AF_INET));
-  tt_int_op(0,OP_EQ, geoip_parse_entry("\"200\",\"250\",\"AB\"", AF_INET));
-
-  /* Populate the IPv6 DB equivalently with fake IPs in the same range */
-  tt_int_op(0,OP_EQ, geoip_parse_entry("::a,::32,AB", AF_INET6));
-  tt_int_op(0,OP_EQ, geoip_parse_entry("::34,::5a,XY", AF_INET6));
-  tt_int_op(0,OP_EQ, geoip_parse_entry("::5f,::64,AB", AF_INET6));
-  tt_int_op(0,OP_EQ, geoip_parse_entry("::69,::8c,ZZ", AF_INET6));
-  tt_int_op(0,OP_EQ, geoip_parse_entry("::96,::be,XY", AF_INET6));
-  tt_int_op(0,OP_EQ, geoip_parse_entry("::c8,::fa,AB", AF_INET6));
-
-  /* We should have 4 countries: ??, ab, xy, zz. */
-  tt_int_op(4,OP_EQ, geoip_get_n_countries());
-  memset(&in6, 0, sizeof(in6));
-
-  CHECK_COUNTRY("??", 3);
-  CHECK_COUNTRY("ab", 32);
-  CHECK_COUNTRY("??", 5);
-  CHECK_COUNTRY("??", 51);
-  CHECK_COUNTRY("xy", 150);
-  CHECK_COUNTRY("xy", 190);
-  CHECK_COUNTRY("??", 2000);
-
-  tt_int_op(0,OP_EQ, geoip_get_country_by_ipv4(3));
-  SET_TEST_IPV6(3);
-  tt_int_op(0,OP_EQ, geoip_get_country_by_ipv6(&in6));
-
-  get_options_mutable()->BridgeRelay = 1;
-  get_options_mutable()->BridgeRecordUsageByCountry = 1;
-  /* Put 9 observations in AB... */
-  for (i=32; i < 40; ++i) {
-    SET_TEST_ADDRESS(i);
-    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200);
-  }
-  SET_TEST_ADDRESS(225);
-  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200);
-  /* and 3 observations in XY, several times. */
-  for (j=0; j < 10; ++j)
-    for (i=52; i < 55; ++i) {
-      SET_TEST_ADDRESS(i);
-      geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-3600);
-    }
-  /* and 17 observations in ZZ... */
-  for (i=110; i < 127; ++i) {
-    SET_TEST_ADDRESS(i);
-    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now);
-  }
-  geoip_get_client_history(GEOIP_CLIENT_CONNECT, &s, &v);
-  tt_assert(s);
-  tt_assert(v);
-  tt_str_op("zz=24,ab=16,xy=8",OP_EQ, s);
-  tt_str_op("v4=16,v6=16",OP_EQ, v);
-  tor_free(s);
-  tor_free(v);
-
-  /* Now clear out all the AB observations. */
-  geoip_remove_old_clients(now-6000);
-  geoip_get_client_history(GEOIP_CLIENT_CONNECT, &s, &v);
-  tt_assert(s);
-  tt_assert(v);
-  tt_str_op("zz=24,xy=8",OP_EQ, s);
-  tt_str_op("v4=16,v6=16",OP_EQ, v);
-  tor_free(s);
-  tor_free(v);
-
-  /* Start testing bridge statistics by making sure that we don't output
-   * bridge stats without initializing them. */
-  s = geoip_format_bridge_stats(now + 86400);
-  tt_assert(!s);
-
-  /* Initialize stats and generate the bridge-stats history string out of
-   * the connecting clients added above. */
-  geoip_bridge_stats_init(now);
-  s = geoip_format_bridge_stats(now + 86400);
-  tt_assert(s);
-  tt_str_op(bridge_stats_1,OP_EQ, s);
-  tor_free(s);
-
-  /* Stop collecting bridge stats and make sure we don't write a history
-   * string anymore. */
-  geoip_bridge_stats_term();
-  s = geoip_format_bridge_stats(now + 86400);
-  tt_assert(!s);
-
-  /* Stop being a bridge and start being a directory mirror that gathers
-   * directory request statistics. */
-  geoip_bridge_stats_term();
-  get_options_mutable()->BridgeRelay = 0;
-  get_options_mutable()->BridgeRecordUsageByCountry = 0;
-  get_options_mutable()->DirReqStatistics = 1;
-
-  /* Start testing dirreq statistics by making sure that we don't collect
-   * dirreq stats without initializing them. */
-  SET_TEST_ADDRESS(100);
-  geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now);
-  s = geoip_format_dirreq_stats(now + 86400);
-  tt_assert(!s);
-
-  /* Initialize stats, note one connecting client, and generate the
-   * dirreq-stats history string. */
-  geoip_dirreq_stats_init(now);
-  SET_TEST_ADDRESS(100);
-  geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now);
-  s = geoip_format_dirreq_stats(now + 86400);
-  tt_str_op(dirreq_stats_1,OP_EQ, s);
-  tor_free(s);
-
-  /* Stop collecting stats, add another connecting client, and ensure we
-   * don't generate a history string. */
-  geoip_dirreq_stats_term();
-  SET_TEST_ADDRESS(101);
-  geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now);
-  s = geoip_format_dirreq_stats(now + 86400);
-  tt_assert(!s);
-
-  /* Re-start stats, add a connecting client, reset stats, and make sure
-   * that we get an all empty history string. */
-  geoip_dirreq_stats_init(now);
-  SET_TEST_ADDRESS(100);
-  geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now);
-  geoip_reset_dirreq_stats(now);
-  s = geoip_format_dirreq_stats(now + 86400);
-  tt_str_op(dirreq_stats_2,OP_EQ, s);
-  tor_free(s);
-
-  /* Note a successful network status response and make sure that it
-   * appears in the history string. */
-  geoip_note_ns_response(GEOIP_SUCCESS);
-  s = geoip_format_dirreq_stats(now + 86400);
-  tt_str_op(dirreq_stats_3,OP_EQ, s);
-  tor_free(s);
-
-  /* Start a tunneled directory request. */
-  geoip_start_dirreq((uint64_t) 1, 1024, DIRREQ_TUNNELED);
-  s = geoip_format_dirreq_stats(now + 86400);
-  tt_str_op(dirreq_stats_4,OP_EQ, s);
-  tor_free(s);
-
-  /* Stop collecting directory request statistics and start gathering
-   * entry stats. */
-  geoip_dirreq_stats_term();
-  get_options_mutable()->DirReqStatistics = 0;
-  get_options_mutable()->EntryStatistics = 1;
-
-  /* Start testing entry statistics by making sure that we don't collect
-   * anything without initializing entry stats. */
-  SET_TEST_ADDRESS(100);
-  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now);
-  s = geoip_format_entry_stats(now + 86400);
-  tt_assert(!s);
-
-  /* Initialize stats, note one connecting client, and generate the
-   * entry-stats history string. */
-  geoip_entry_stats_init(now);
-  SET_TEST_ADDRESS(100);
-  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now);
-  s = geoip_format_entry_stats(now + 86400);
-  tt_str_op(entry_stats_1,OP_EQ, s);
-  tor_free(s);
-
-  /* Stop collecting stats, add another connecting client, and ensure we
-   * don't generate a history string. */
-  geoip_entry_stats_term();
-  SET_TEST_ADDRESS(101);
-  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now);
-  s = geoip_format_entry_stats(now + 86400);
-  tt_assert(!s);
-
-  /* Re-start stats, add a connecting client, reset stats, and make sure
-   * that we get an all empty history string. */
-  geoip_entry_stats_init(now);
-  SET_TEST_ADDRESS(100);
-  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now);
-  geoip_reset_entry_stats(now);
-  s = geoip_format_entry_stats(now + 86400);
-  tt_str_op(entry_stats_2,OP_EQ, s);
-  tor_free(s);
-
-  /* Test the OOM handler. Add a client, run the OOM. */
-  geoip_entry_stats_init(now);
-  SET_TEST_ADDRESS(100);
-  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL,
-                         now - (12 * 60 * 60));
-  /* We've seen this 12 hours ago. Run the OOM, it should clean the entry
-   * because it is above the minimum cutoff of 4 hours. */
-  size_t bytes_removed = geoip_client_cache_handle_oom(now, 1000);
-  tt_size_op(bytes_removed, OP_GT, 0);
-
-  /* Do it again but this time with an entry with a lower cutoff. */
-  geoip_entry_stats_init(now);
-  SET_TEST_ADDRESS(100);
-  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL,
-                         now - (3 * 60 * 60));
-  bytes_removed = geoip_client_cache_handle_oom(now, 1000);
-  tt_size_op(bytes_removed, OP_EQ, 0);
-
-  /* Stop collecting entry statistics. */
-  geoip_entry_stats_term();
-  get_options_mutable()->EntryStatistics = 0;
-
- done:
-  tor_free(s);
-  tor_free(v);
-}
-
-static void
-test_geoip_with_pt(void *arg)
-{
-  time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */
-  char *s = NULL;
-  int i;
-  tor_addr_t addr;
-  struct in6_addr in6;
-
-  (void)arg;
-  get_options_mutable()->BridgeRelay = 1;
-  get_options_mutable()->BridgeRecordUsageByCountry = 1;
-
-  memset(&in6, 0, sizeof(in6));
-
-  /* No clients seen yet. */
-  s = geoip_get_transport_history();
-  tor_assert(!s);
-
-  /* 4 connections without a pluggable transport */
-  for (i=0; i < 4; ++i) {
-    SET_TEST_ADDRESS(i);
-    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200);
-  }
-
-  /* 9 connections with "alpha" */
-  for (i=4; i < 13; ++i) {
-    SET_TEST_ADDRESS(i);
-    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "alpha", now-7200);
-  }
-
-  /* one connection with "beta" */
-  SET_TEST_ADDRESS(13);
-  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "beta", now-7200);
-
-  /* 14 connections with "charlie" */
-  for (i=14; i < 28; ++i) {
-    SET_TEST_ADDRESS(i);
-    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "charlie", now-7200);
-  }
-
-  /* 131 connections with "ddr" */
-  for (i=28; i < 159; ++i) {
-    SET_TEST_ADDRESS(i);
-    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "ddr", now-7200);
-  }
-
-  /* 8 connections with "entropy" */
-  for (i=159; i < 167; ++i) {
-    SET_TEST_ADDRESS(i);
-    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "entropy", now-7200);
-  }
-
-  /* 2 connections from the same IP with two different transports. */
-  SET_TEST_ADDRESS(++i);
-  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "fire", now-7200);
-  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "google", now-7200);
-
-  /* Test the transport history string. */
-  s = geoip_get_transport_history();
-  tor_assert(s);
-  tt_str_op(s,OP_EQ, "<OR>=8,alpha=16,beta=8,charlie=16,ddr=136,"
-             "entropy=8,fire=8,google=8");
-
-  /* Stop collecting entry statistics. */
-  geoip_entry_stats_term();
-  get_options_mutable()->EntryStatistics = 0;
-
- done:
-  tor_free(s);
-}
-
-#undef SET_TEST_ADDRESS
-#undef SET_TEST_IPV6
-#undef CHECK_COUNTRY
-
 /** Run unit tests for stats code. */
 static void
 test_stats(void *arg)
@@ -1023,7 +665,7 @@ test_stats(void *arg)
   rep_hist_note_exit_stream_opened(80);
   rep_hist_note_exit_bytes(80, 100, 10000);
   s = rep_hist_format_exit_stats(now + 86400);
-  tt_assert(!s);
+  tt_ptr_op(s, OP_EQ, NULL);
 
   /* Initialize stats, note some streams and bytes, and generate history
    * string. */
@@ -1061,7 +703,7 @@ test_stats(void *arg)
   rep_hist_exit_stats_term();
   rep_hist_note_exit_bytes(80, 100, 10000);
   s = rep_hist_format_exit_stats(now + 86400);
-  tt_assert(!s);
+  tt_ptr_op(s, OP_EQ, NULL);
 
   /* Re-start stats, add some bytes, reset stats, and see what history we
    * get when observing no streams or bytes at all. */
@@ -1080,7 +722,7 @@ test_stats(void *arg)
    * conn stats without initializing them. */
   rep_hist_note_or_conn_bytes(1, 20, 400, now);
   s = rep_hist_format_conn_stats(now + 86400);
-  tt_assert(!s);
+  tt_ptr_op(s, OP_EQ, NULL);
 
   /* Initialize stats, note bytes, and generate history string. */
   rep_hist_conn_stats_init(now);
@@ -1097,7 +739,7 @@ test_stats(void *arg)
   rep_hist_conn_stats_term();
   rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 15);
   s = rep_hist_format_conn_stats(now + 86400);
-  tt_assert(!s);
+  tt_ptr_op(s, OP_EQ, NULL);
 
   /* Re-start stats, add some bytes, reset stats, and see what history we
    * get when observing no bytes at all. */
@@ -1115,7 +757,7 @@ test_stats(void *arg)
    * stats without initializing them. */
   rep_hist_add_buffer_stats(2.0, 2.0, 20);
   s = rep_hist_format_buffer_stats(now + 86400);
-  tt_assert(!s);
+  tt_ptr_op(s, OP_EQ, NULL);
 
   /* Initialize stats, add statistics for a single circuit, and generate
    * the history string. */
@@ -1150,7 +792,7 @@ test_stats(void *arg)
   rep_hist_buffer_stats_term();
   rep_hist_add_buffer_stats(2.0, 2.0, 20);
   s = rep_hist_format_buffer_stats(now + 86400);
-  tt_assert(!s);
+  tt_ptr_op(s, OP_EQ, NULL);
 
   /* Re-start stats, add statistics for one circuit, reset stats, and make
    * sure that the history has all zeros. */
@@ -1182,8 +824,6 @@ static struct testcase_t test_array[] = {
   { "fast_handshake", test_fast_handshake, 0, NULL, NULL },
   FORK(circuit_timeout),
   FORK(rend_fns),
-  ENT(geoip),
-  FORK(geoip_with_pt),
   FORK(stats),
 
   END_OF_TESTCASES
@@ -1195,43 +835,74 @@ struct testgroup_t testgroups[] = {
   { "addr/", addr_tests },
   { "address/", address_tests },
   { "address_set/", address_set_tests },
+  { "bridges/", bridges_tests },
   { "buffer/", buffer_tests },
+  { "bwmgt/", bwmgt_tests },
   { "cellfmt/", cell_format_tests },
   { "cellqueue/", cell_queue_tests },
   { "channel/", channel_tests },
+  { "channelpadding/", channelpadding_tests },
   { "channeltls/", channeltls_tests },
   { "checkdir/", checkdir_tests },
+  { "circuitbuild/", circuitbuild_tests },
   { "circuitlist/", circuitlist_tests },
   { "circuitmux/", circuitmux_tests },
+  { "circuituse/", circuituse_tests },
+  { "circuitstats/", circuitstats_tests },
   { "compat/libevent/", compat_libevent_tests },
   { "config/", config_tests },
   { "connection/", connection_tests },
+  { "conscache/", conscache_tests },
+  { "consdiff/", consdiff_tests },
+  { "consdiffmgr/", consdiffmgr_tests },
   { "container/", container_tests },
   { "control/", controller_tests },
   { "control/event/", controller_event_tests },
   { "crypto/", crypto_tests },
-  { "dos/", dos_tests },
+  { "crypto/ope/", crypto_ope_tests },
+#ifdef ENABLE_OPENSSL
+  { "crypto/openssl/", crypto_openssl_tests },
+#endif
+  { "crypto/pem/", pem_tests },
   { "dir/", dir_tests },
   { "dir_handle_get/", dir_handle_get_tests },
   { "dir/md/", microdesc_tests },
+  { "dir/voting-schedule/", voting_schedule_tests },
+  { "dos/", dos_tests },
   { "entryconn/", entryconn_tests },
   { "entrynodes/", entrynodes_tests },
   { "guardfraction/", guardfraction_tests },
   { "extorport/", extorport_tests },
-  { "hs/", hs_tests },
+  { "geoip/", geoip_tests },
+  { "legacy_hs/", hs_tests },
+  { "hs_cache/", hs_cache },
+  { "hs_cell/", hs_cell_tests },
+  { "hs_common/", hs_common_tests },
+  { "hs_config/", hs_config_tests },
+  { "hs_control/", hs_control_tests },
+  { "hs_descriptor/", hs_descriptor },
+  { "hs_ntor/", hs_ntor_tests },
+  { "hs_service/", hs_service_tests },
+  { "hs_client/", hs_client_tests },
+  { "hs_intropoint/", hs_intropoint_tests },
   { "introduce/", introduce_tests },
   { "keypin/", keypin_tests },
   { "link-handshake/", link_handshake_tests },
+  { "mainloop/", mainloop_tests },
   { "nodelist/", nodelist_tests },
   { "oom/", oom_tests },
   { "oos/", oos_tests },
   { "options/", options_tests },
+  { "periodic-event/" , periodic_event_tests },
   { "policy/" , policy_tests },
   { "procmon/", procmon_tests },
+  { "proto/http/", proto_http_tests },
+  { "proto/misc/", proto_misc_tests },
   { "protover/", protover_tests },
   { "pt/", pt_tests },
   { "relay/" , relay_tests },
   { "relaycell/", relaycell_tests },
+  { "relaycrypt/", relaycrypt_tests },
   { "rend_cache/", rend_cache_tests },
   { "replaycache/", replaycache_tests },
   { "router/", router_tests },
@@ -1242,15 +913,18 @@ struct testgroup_t testgroups[] = {
   { "socks/", socks_tests },
   { "shared-random/", sr_tests },
   { "status/" , status_tests },
+  { "storagedir/", storagedir_tests },
   { "tortls/", tortls_tests },
+#ifndef ENABLE_NSS
+  { "tortls/openssl/", tortls_openssl_tests },
+#endif
+  { "tortls/x509/", x509_tests },
   { "util/", util_tests },
   { "util/format/", util_format_tests },
   { "util/logging/", logging_tests },
   { "util/process/", util_process_tests },
-  { "util/pubsub/", pubsub_tests },
   { "util/thread/", thread_tests },
   { "util/handle/", handle_tests },
   { "dns/", dns_tests },
   END_OF_GROUPS
 };
-
diff --git a/src/test/test.h b/src/test/test.h
index 028082386e..aacc9dba87 100644
--- a/src/test/test.h
+++ b/src/test/test.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2003, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_TEST_H
@@ -11,7 +11,8 @@
  * \brief Macros and functions used by unit tests.
  */
 
-#include "compat.h"
+#define DEBUG_SMARTLIST 1
+
 #include "tinytest.h"
 #define TT_EXIT_TEST_FUNCTION STMT_BEGIN goto done; STMT_END
 #include "tinytest_macros.h"
@@ -45,36 +46,38 @@
  * you're doing. */
 #define tt_double_eq(a,b)     \
   STMT_BEGIN                  \
-  tt_double_op((a), >=, (b)); \
-  tt_double_op((a), <=, (b)); \
+  tt_double_op((a), OP_GE, (b)); \
+  tt_double_op((a), OP_LE, (b)); \
   STMT_END
 
-#ifdef _MSC_VER
-#define U64_PRINTF_TYPE uint64_t
-#define I64_PRINTF_TYPE int64_t
-#else
-#define U64_PRINTF_TYPE unsigned long long
-#define I64_PRINTF_TYPE long long
-#endif
-
 #define tt_size_op(a,op,b)                                              \
   tt_assert_test_fmt_type(a,b,#a" "#op" "#b,size_t,(val1_ op val2_),    \
-    U64_PRINTF_TYPE, U64_FORMAT,                                        \
-    {print_ = (U64_PRINTF_TYPE) value_;}, {}, TT_EXIT_TEST_FUNCTION)
+    size_t, "%"TOR_PRIuSZ,                                              \
+    {print_ = (size_t) value_;}, {}, TT_EXIT_TEST_FUNCTION)
 
 #define tt_u64_op(a,op,b)                                              \
   tt_assert_test_fmt_type(a,b,#a" "#op" "#b,uint64_t,(val1_ op val2_), \
-    U64_PRINTF_TYPE, U64_FORMAT,                                       \
-    {print_ = (U64_PRINTF_TYPE) value_;}, {}, TT_EXIT_TEST_FUNCTION)
+    uint64_t, "%"PRIu64,                                               \
+    {print_ = (uint64_t) value_;}, {}, TT_EXIT_TEST_FUNCTION)
 
 #define tt_i64_op(a,op,b)                                              \
-  tt_assert_test_fmt_type(a,b,#a" "#op" "#b,int64_t,(val1_ op val2_), \
-    I64_PRINTF_TYPE, I64_FORMAT,                                       \
-    {print_ = (I64_PRINTF_TYPE) value_;}, {}, TT_EXIT_TEST_FUNCTION)
+  tt_assert_test_fmt_type(a,b,#a" "#op" "#b,int64_t,(val1_ op val2_),  \
+    int64_t, "%"PRId64,                                                \
+    {print_ = (int64_t) value_;}, {}, TT_EXIT_TEST_FUNCTION)
+
+/**
+ * Declare that the test is done, even though no tt___op() calls were made.
+ *
+ * For use when you only want to test calling something, but not check
+ * any values/pointers/etc afterwards.
+ */
+#define tt_finished() TT_EXIT_TEST_FUNCTION
 
 const char *get_fname(const char *name);
 const char *get_fname_rnd(const char *name);
 struct crypto_pk_t *pk_generate(int idx);
+void init_pregenerated_keys(void);
+void free_pregenerated_keys(void);
 
 #define US2_CONCAT_2__(a, b) a ## __ ## b
 #define US_CONCAT_2__(a, b) a ## _ ## b
@@ -176,45 +179,72 @@ extern struct testcase_t accounting_tests[];
 extern struct testcase_t addr_tests[];
 extern struct testcase_t address_tests[];
 extern struct testcase_t address_set_tests[];
+extern struct testcase_t bridges_tests[];
+extern struct testcase_t bwmgt_tests[];
 extern struct testcase_t buffer_tests[];
 extern struct testcase_t cell_format_tests[];
 extern struct testcase_t cell_queue_tests[];
 extern struct testcase_t channel_tests[];
+extern struct testcase_t channelpadding_tests[];
 extern struct testcase_t channeltls_tests[];
 extern struct testcase_t checkdir_tests[];
+extern struct testcase_t circuitbuild_tests[];
 extern struct testcase_t circuitlist_tests[];
 extern struct testcase_t circuitmux_tests[];
+extern struct testcase_t circuituse_tests[];
+extern struct testcase_t circuitstats_tests[];
 extern struct testcase_t compat_libevent_tests[];
 extern struct testcase_t config_tests[];
 extern struct testcase_t connection_tests[];
+extern struct testcase_t conscache_tests[];
+extern struct testcase_t consdiff_tests[];
+extern struct testcase_t consdiffmgr_tests[];
 extern struct testcase_t container_tests[];
 extern struct testcase_t controller_tests[];
 extern struct testcase_t controller_event_tests[];
 extern struct testcase_t crypto_tests[];
-extern struct testcase_t dos_tests[];
+extern struct testcase_t crypto_ope_tests[];
+extern struct testcase_t crypto_openssl_tests[];
 extern struct testcase_t dir_tests[];
 extern struct testcase_t dir_handle_get_tests[];
+extern struct testcase_t dos_tests[];
 extern struct testcase_t entryconn_tests[];
 extern struct testcase_t entrynodes_tests[];
 extern struct testcase_t guardfraction_tests[];
 extern struct testcase_t extorport_tests[];
+extern struct testcase_t geoip_tests[];
 extern struct testcase_t hs_tests[];
+extern struct testcase_t hs_cache[];
+extern struct testcase_t hs_cell_tests[];
+extern struct testcase_t hs_common_tests[];
+extern struct testcase_t hs_config_tests[];
+extern struct testcase_t hs_control_tests[];
+extern struct testcase_t hs_descriptor[];
+extern struct testcase_t hs_ntor_tests[];
+extern struct testcase_t hs_service_tests[];
+extern struct testcase_t hs_client_tests[];
+extern struct testcase_t hs_intropoint_tests[];
 extern struct testcase_t introduce_tests[];
 extern struct testcase_t keypin_tests[];
 extern struct testcase_t link_handshake_tests[];
 extern struct testcase_t logging_tests[];
+extern struct testcase_t mainloop_tests[];
 extern struct testcase_t microdesc_tests[];
 extern struct testcase_t nodelist_tests[];
 extern struct testcase_t oom_tests[];
 extern struct testcase_t oos_tests[];
 extern struct testcase_t options_tests[];
+extern struct testcase_t pem_tests[];
+extern struct testcase_t periodic_event_tests[];
 extern struct testcase_t policy_tests[];
 extern struct testcase_t procmon_tests[];
+extern struct testcase_t proto_http_tests[];
+extern struct testcase_t proto_misc_tests[];
 extern struct testcase_t protover_tests[];
-extern struct testcase_t pubsub_tests[];
 extern struct testcase_t pt_tests[];
 extern struct testcase_t relay_tests[];
 extern struct testcase_t relaycell_tests[];
+extern struct testcase_t relaycrypt_tests[];
 extern struct testcase_t rend_cache_tests[];
 extern struct testcase_t replaycache_tests[];
 extern struct testcase_t router_tests[];
@@ -222,16 +252,20 @@ extern struct testcase_t routerkeys_tests[];
 extern struct testcase_t routerlist_tests[];
 extern struct testcase_t routerset_tests[];
 extern struct testcase_t scheduler_tests[];
+extern struct testcase_t storagedir_tests[];
 extern struct testcase_t socks_tests[];
 extern struct testcase_t status_tests[];
 extern struct testcase_t thread_tests[];
 extern struct testcase_t tortls_tests[];
+extern struct testcase_t tortls_openssl_tests[];
 extern struct testcase_t util_tests[];
 extern struct testcase_t util_format_tests[];
 extern struct testcase_t util_process_tests[];
+extern struct testcase_t voting_schedule_tests[];
 extern struct testcase_t dns_tests[];
 extern struct testcase_t handle_tests[];
 extern struct testcase_t sr_tests[];
+extern struct testcase_t x509_tests[];
 
 extern struct testcase_t slow_crypto_tests[];
 extern struct testcase_t slow_util_tests[];
@@ -251,5 +285,4 @@ extern const char AUTHORITY_SIGNKEY_3[];
 extern const char AUTHORITY_SIGNKEY_C_DIGEST[];
 extern const char AUTHORITY_SIGNKEY_C_DIGEST256[];
 
-#endif
-
+#endif /* !defined(TOR_TEST_H) */
diff --git a/src/test/test_accounting.c b/src/test/test_accounting.c
index 7edba988a6..8ae8fe4343 100644
--- a/src/test/test_accounting.c
+++ b/src/test/test_accounting.c
@@ -1,10 +1,15 @@
-#include "or.h"
-#include "test.h"
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "test/test.h"
 #define HIBERNATE_PRIVATE
-#include "hibernate.h"
-#include "config.h"
+#include "feature/hibernate/hibernate.h"
+#include "app/config/config.h"
 #define STATEFILE_PRIVATE
-#include "statefile.h"
+#include "app/config/statefile.h"
+
+#include "app/config/or_state_st.h"
 
 #define NS_MODULE accounting
 
@@ -99,4 +104,3 @@ struct testcase_t accounting_tests[] = {
   { "bwlimits", test_accounting_limits, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_addr.c b/src/test/test_addr.c
index 5c4b6449cd..8868edce25 100644
--- a/src/test/test_addr.c
+++ b/src/test/test_addr.c
@@ -1,53 +1,25 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define ADDRESSMAP_PRIVATE
 #include "orconfig.h"
-#include "or.h"
-#include "test.h"
-#include "addressmap.h"
+#include "core/or/or.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "test/test.h"
+#include "feature/client/addressmap.h"
+#include "test/log_test_helpers.h"
+#include "lib/net/resolve.h"
+
+#ifdef HAVE_SYS_UN_H
+#include <sys/un.h>
+#endif
 
 static void
 test_addr_basic(void *arg)
 {
-  uint32_t u32;
-  uint16_t u16;
-  char *cp;
-
-  /* Test addr_port_lookup */
-  (void)arg;
-  cp = NULL; u32 = 3; u16 = 3;
-  tt_assert(!addr_port_lookup(LOG_WARN, "1.2.3.4", &cp, &u32, &u16));
-  tt_str_op(cp,OP_EQ, "1.2.3.4");
-  tt_int_op(u32,OP_EQ, 0x01020304u);
-  tt_int_op(u16,OP_EQ, 0);
-  tor_free(cp);
-  tt_assert(!addr_port_lookup(LOG_WARN, "4.3.2.1:99", &cp, &u32, &u16));
-  tt_str_op(cp,OP_EQ, "4.3.2.1");
-  tt_int_op(u32,OP_EQ, 0x04030201u);
-  tt_int_op(u16,OP_EQ, 99);
-  tor_free(cp);
-  tt_assert(!addr_port_lookup(LOG_WARN, "nonexistent.address:4040",
-                               &cp, NULL, &u16));
-  tt_str_op(cp,OP_EQ, "nonexistent.address");
-  tt_int_op(u16,OP_EQ, 4040);
-  tor_free(cp);
-  tt_assert(!addr_port_lookup(LOG_WARN, "localhost:9999", &cp, &u32, &u16));
-  tt_str_op(cp,OP_EQ, "localhost");
-  tt_int_op(u32,OP_EQ, 0x7f000001u);
-  tt_int_op(u16,OP_EQ, 9999);
-  tor_free(cp);
-  u32 = 3;
-  tt_assert(!addr_port_lookup(LOG_WARN, "localhost", NULL, &u32, &u16));
-  tt_ptr_op(cp,OP_EQ, NULL);
-  tt_int_op(u32,OP_EQ, 0x7f000001u);
-  tt_int_op(u16,OP_EQ, 0);
-  tor_free(cp);
-
-  tt_assert(addr_port_lookup(LOG_WARN, "localhost:3", &cp, &u32, NULL));
-  tor_free(cp);
+  (void) arg;
 
   tt_int_op(0,OP_EQ, addr_mask_get_bits(0x0u));
   tt_int_op(32,OP_EQ, addr_mask_get_bits(0xFFFFFFFFu));
@@ -75,7 +47,7 @@ test_addr_basic(void *arg)
   }
 
  done:
-  tor_free(cp);
+  ;
 }
 
 #define test_op_ip6_(a,op,b,e1,e2)                               \
@@ -429,10 +401,10 @@ test_addr_ip6_helpers(void *arg)
                     "::ffff:6.0.0.0"); /* XXXX wrong. */
   tor_addr_parse_mask_ports("[::ffff:2.3.4.5]", 0, &t1, NULL, NULL, NULL);
   tor_addr_parse_mask_ports("2.3.4.5", 0, &t2, NULL, NULL, NULL);
-  tt_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) == 0);
+  tt_int_op(tor_addr_compare(&t1, &t2, CMP_SEMANTIC), OP_EQ, 0);
   tor_addr_parse_mask_ports("[::ffff:2.3.4.4]", 0, &t1, NULL, NULL, NULL);
   tor_addr_parse_mask_ports("2.3.4.5", 0, &t2, NULL, NULL, NULL);
-  tt_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) < 0);
+  tt_int_op(tor_addr_compare(&t1, &t2, CMP_SEMANTIC), OP_LT, 0);
 
   /* test compare_masked */
   test_addr_compare_masked("ffff::", OP_EQ, "ffff::0", 128);
@@ -615,7 +587,7 @@ test_addr_ip6_helpers(void *arg)
   /* Try some long addresses. */
   r=tor_addr_parse_mask_ports("[ffff:1111:1111:1111:1111:1111:1111:1111]",
                               0, &t1, NULL, NULL, NULL);
-  tt_assert(r == AF_INET6);
+  tt_int_op(r, OP_EQ, AF_INET6);
   r=tor_addr_parse_mask_ports("[ffff:1111:1111:1111:1111:1111:1111:11111]",
                               0, &t1, NULL, NULL, NULL);
   tt_int_op(r, OP_EQ, -1);
@@ -664,38 +636,38 @@ test_addr_ip6_helpers(void *arg)
   tt_int_op(r, OP_EQ, -1);
   r=tor_addr_parse_mask_ports("*6",0,&t1, &mask, NULL, NULL);
   tt_int_op(r, OP_EQ, -1);
-  tt_assert(r == -1);
+  tt_int_op(r, OP_EQ, -1);
   /* Try a mask with a wildcard. */
   r=tor_addr_parse_mask_ports("*/16",0,&t1, &mask, NULL, NULL);
-  tt_assert(r == -1);
+  tt_int_op(r, OP_EQ, -1);
   r=tor_addr_parse_mask_ports("*4/16",TAPMP_EXTENDED_STAR,
                               &t1, &mask, NULL, NULL);
-  tt_assert(r == -1);
+  tt_int_op(r, OP_EQ, -1);
   r=tor_addr_parse_mask_ports("*6/30",TAPMP_EXTENDED_STAR,
                               &t1, &mask, NULL, NULL);
-  tt_assert(r == -1);
+  tt_int_op(r, OP_EQ, -1);
   /* Basic mask tests*/
   r=tor_addr_parse_mask_ports("1.1.2.2/31",0,&t1, &mask, NULL, NULL);
-  tt_assert(r == AF_INET);
+  tt_int_op(r, OP_EQ, AF_INET);
   tt_int_op(mask,OP_EQ,31);
   tt_int_op(tor_addr_family(&t1),OP_EQ,AF_INET);
   tt_int_op(tor_addr_to_ipv4h(&t1),OP_EQ,0x01010202);
   r=tor_addr_parse_mask_ports("3.4.16.032:1-2",0,&t1, &mask, &port1, &port2);
-  tt_assert(r == AF_INET);
+  tt_int_op(r, OP_EQ, AF_INET);
   tt_int_op(mask,OP_EQ,32);
   tt_int_op(tor_addr_family(&t1),OP_EQ,AF_INET);
   tt_int_op(tor_addr_to_ipv4h(&t1),OP_EQ,0x03041020);
-  tt_assert(port1 == 1);
-  tt_assert(port2 == 2);
+  tt_uint_op(port1, OP_EQ, 1);
+  tt_uint_op(port2, OP_EQ, 2);
   r=tor_addr_parse_mask_ports("1.1.2.3/255.255.128.0",0,&t1, &mask,NULL,NULL);
-  tt_assert(r == AF_INET);
+  tt_int_op(r, OP_EQ, AF_INET);
   tt_int_op(mask,OP_EQ,17);
   tt_int_op(tor_addr_family(&t1),OP_EQ,AF_INET);
   tt_int_op(tor_addr_to_ipv4h(&t1),OP_EQ,0x01010203);
   r=tor_addr_parse_mask_ports("[efef::]/112",0,&t1, &mask, &port1, &port2);
-  tt_assert(r == AF_INET6);
-  tt_assert(port1 == 1);
-  tt_assert(port2 == 65535);
+  tt_int_op(r, OP_EQ, AF_INET6);
+  tt_uint_op(port1, OP_EQ, 1);
+  tt_uint_op(port2, OP_EQ, 65535);
   /* Try regular wildcard behavior without TAPMP_EXTENDED_STAR */
   r=tor_addr_parse_mask_ports("*:80-443",0,&t1,&mask,&port1,&port2);
   tt_int_op(r,OP_EQ,AF_INET); /* Old users of this always get inet */
@@ -730,15 +702,17 @@ test_addr_ip6_helpers(void *arg)
   tt_int_op(port2,OP_EQ,65535);
 
   /* make sure inet address lengths >= max */
-  tt_assert(INET_NTOA_BUF_LEN >= sizeof("255.255.255.255"));
-  tt_assert(TOR_ADDR_BUF_LEN >=
-              sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255"));
+  tt_int_op(INET_NTOA_BUF_LEN, OP_GE, sizeof("255.255.255.255"));
+  tt_int_op(TOR_ADDR_BUF_LEN, OP_GE,
+            sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255"));
 
   tt_assert(sizeof(tor_addr_t) >= sizeof(struct in6_addr));
 
   /* get interface addresses */
   r = get_interface_address6(LOG_DEBUG, AF_INET, &t1);
+  tt_int_op(r, OP_LE, 0); // "it worked or it didn't"
   i = get_interface_address6(LOG_DEBUG, AF_INET6, &t2);
+  tt_int_op(i, OP_LE, 0); // "it worked or it didn't"
 
   TT_BLATHER(("v4 address: %s (family=%d)", fmt_addr(&t1),
               tor_addr_family(&t1)));
@@ -917,6 +891,158 @@ test_virtaddrmap(void *data)
   ;
 }
 
+static const char *canned_data = NULL;
+static size_t canned_data_len = 0;
+
+/* Mock replacement for crypto_rand() that returns canned data from
+ * canned_data above. */
+static void
+crypto_canned(char *ptr, size_t n)
+{
+  if (canned_data_len) {
+    size_t to_copy = MIN(n, canned_data_len);
+    memcpy(ptr, canned_data, to_copy);
+    canned_data += to_copy;
+    canned_data_len -= to_copy;
+    n -= to_copy;
+    ptr += to_copy;
+  }
+  if (n) {
+    crypto_rand_unmocked(ptr, n);
+  }
+}
+
+static void
+test_virtaddrmap_persist(void *data)
+{
+  (void)data;
+  const char *a, *b, *c;
+  tor_addr_t addr;
+  char *ones = NULL;
+
+  addressmap_init();
+
+  // Try a hostname.
+  a = addressmap_register_virtual_address(RESOLVED_TYPE_HOSTNAME,
+                                          tor_strdup("foobar.baz"));
+  tt_assert(a);
+  tt_assert(!strcmpend(a, ".virtual"));
+
+  // mock crypto_rand to repeat the same result twice; make sure we get
+  // different outcomes.  (Because even though the odds for receiving the
+  // same 80-bit address twice is only 1/2^40, it could still happen for
+  // some user -- but running our test through 2^40 iterations isn't
+  // reasonable.)
+  canned_data = "1234567890" // the first call returns this.
+                "1234567890" // the second call returns this.
+                "abcdefghij"; // the third call returns this.
+  canned_data_len = 30;
+  MOCK(crypto_rand, crypto_canned);
+
+  a = addressmap_register_virtual_address(RESOLVED_TYPE_HOSTNAME,
+                                          tor_strdup("quuxit.baz"));
+  b = addressmap_register_virtual_address(RESOLVED_TYPE_HOSTNAME,
+                                          tor_strdup("nescio.baz"));
+  tt_assert(a);
+  tt_assert(b);
+  tt_str_op(a, OP_EQ, "gezdgnbvgy3tqojq.virtual");
+  tt_str_op(b, OP_EQ, "mfrggzdfmztwq2lk.virtual");
+
+  // Now try something to get us an ipv4 address
+  UNMOCK(crypto_rand);
+  tt_int_op(0,OP_EQ, parse_virtual_addr_network("192.168.0.0/16",
+                                                AF_INET, 0, NULL));
+  a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
+                                          tor_strdup("foobar.baz"));
+  tt_assert(a);
+  tt_assert(!strcmpstart(a, "192.168."));
+  tor_addr_parse(&addr, a);
+  tt_int_op(AF_INET, OP_EQ, tor_addr_family(&addr));
+
+  b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
+                                          tor_strdup("quuxit.baz"));
+  tt_str_op(b, OP_NE, a);
+  tt_assert(!strcmpstart(b, "192.168."));
+
+  // Try some canned entropy and verify all the we discard duplicates,
+  // addresses that end with 0, and addresses that end with 255.
+  MOCK(crypto_rand, crypto_canned);
+  canned_data = "\x01\x02\x03\x04" // okay
+                "\x01\x02\x03\x04" // duplicate
+                "\x03\x04\x00\x00" // bad ending 1
+                "\x05\x05\x00\xff" // bad ending 2
+                "\x05\x06\x07\xf0"; // okay
+  canned_data_len = 20;
+  a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
+                                          tor_strdup("wumble.onion"));
+  b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
+                                          tor_strdup("wumpus.onion"));
+  tt_str_op(a, OP_EQ, "192.168.3.4");
+  tt_str_op(b, OP_EQ, "192.168.7.240");
+
+  // Now try IPv6!
+  UNMOCK(crypto_rand);
+  tt_int_op(0,OP_EQ, parse_virtual_addr_network("1010:F000::/20",
+                                                AF_INET6, 0, NULL));
+  a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
+                                          tor_strdup("foobar.baz"));
+  tt_assert(a);
+  tt_assert(!strcmpstart(a, "[1010:f"));
+  tor_addr_parse(&addr, a);
+  tt_int_op(AF_INET6, OP_EQ, tor_addr_family(&addr));
+
+  b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
+                                          tor_strdup("quuxit.baz"));
+  tt_str_op(b, OP_NE, a);
+  tt_assert(!strcmpstart(b, "[1010:f"));
+
+  // Try IPv6 with canned entropy, to make sure we detect duplicates.
+  MOCK(crypto_rand, crypto_canned);
+  canned_data = "acanthopterygian" // okay
+                "cinematographist" // okay
+                "acanthopterygian" // duplicate
+                "acanthopterygian" // duplicate
+                "acanthopterygian" // duplicate
+                "cinematographist" // duplicate
+                "coadministration"; // okay
+  canned_data_len = 16 * 7;
+  a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
+                                          tor_strdup("wuffle.baz"));
+  b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
+                                          tor_strdup("gribble.baz"));
+  c = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
+                                      tor_strdup("surprisingly-legible.baz"));
+  tt_str_op(a, OP_EQ, "[1010:f16e:7468:6f70:7465:7279:6769:616e]");
+  tt_str_op(b, OP_EQ, "[1010:fe65:6d61:746f:6772:6170:6869:7374]");
+  tt_str_op(c, OP_EQ, "[1010:f164:6d69:6e69:7374:7261:7469:6f6e]");
+
+  // Try address exhaustion: make sure we can actually fail if we
+  // get too many already-existing addresses.
+  canned_data_len = 128*1024;
+  canned_data = ones = tor_malloc(canned_data_len);
+  memset(ones, 1, canned_data_len);
+  // There is some chance this one will fail if a previous random
+  // allocation gave out the address already.
+  a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
+                                          tor_strdup("might-work.onion"));
+  if (a) {
+    tt_str_op(a, OP_EQ, "192.168.1.1");
+  }
+  setup_capture_of_logs(LOG_WARN);
+  // This one will definitely fail, since we've set up the RNG to hand
+  // out "1" forever.
+  b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
+                                          tor_strdup("wont-work.onion"));
+  tt_assert(b == NULL);
+  expect_single_log_msg_containing("Ran out of virtual addresses!");
+
+ done:
+  UNMOCK(crypto_rand);
+  tor_free(ones);
+  addressmap_free_all();
+  teardown_capture_of_logs();
+}
+
 static void
 test_addr_localname(void *arg)
 {
@@ -988,7 +1114,7 @@ test_addr_sockaddr_to_str(void *arg)
   s_un.sun_family = AF_UNIX;
   strlcpy(s_un.sun_path, "/here/is/a/path", sizeof(s_un.sun_path));
   CHECK(s_un, "unix:/here/is/a/path");
-#endif
+#endif /* defined(HAVE_SYS_UN_H) */
 
   memset(&sin6,0,sizeof(sin6));
   sin6.sin6_family = AF_INET6;
@@ -1088,6 +1214,7 @@ struct testcase_t addr_tests[] = {
   ADDR_LEGACY(ip6_helpers),
   ADDR_LEGACY(parse),
   { "virtaddr", test_virtaddrmap, 0, NULL, NULL },
+  { "virtaddr_persist", test_virtaddrmap_persist, TT_FORK, NULL, NULL },
   { "localname", test_addr_localname, 0, NULL, NULL },
   { "dup_ip", test_addr_dup_ip, 0, NULL, NULL },
   { "sockaddr_to_str", test_addr_sockaddr_to_str, 0, NULL, NULL },
@@ -1096,4 +1223,3 @@ struct testcase_t addr_tests[] = {
   { "rfc6598", test_addr_rfc6598, 0, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_address.c b/src/test/test_address.c
index 0d142ad483..c33c30aee5 100644
--- a/src/test/test_address.c
+++ b/src/test/test_address.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define ADDRESS_PRIVATE
@@ -21,12 +21,13 @@
 #include <sys/ioctl.h>
 #endif
 #include <net/if.h>
-#endif
+#endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */
 
-#include "or.h"
-#include "address.h"
-#include "test.h"
-#include "log_test_helpers.h"
+#include "core/or/or.h"
+#include "feature/nodelist/nodelist.h"
+#include "lib/net/address.h"
+#include "test/test.h"
+#include "test/log_test_helpers.h"
 
 /** Return 1 iff <b>sockaddr1</b> and <b>sockaddr2</b> represent
  * the same IP address and port combination. Otherwise, return 0.
@@ -224,7 +225,7 @@ test_address_ifaddrs_to_smartlist(void *arg)
    smartlist = ifaddrs_to_smartlist(ifa, AF_UNSPEC);
 
    tt_assert(smartlist);
-   tt_assert(smartlist_len(smartlist) == 3);
+   tt_int_op(smartlist_len(smartlist), OP_EQ, 3);
 
    sockaddr_to_check = tor_malloc(sizeof(struct sockaddr_in6));
 
@@ -233,7 +234,7 @@ test_address_ifaddrs_to_smartlist(void *arg)
    tor_addr_to_sockaddr(tor_addr,0,sockaddr_to_check,
                         sizeof(struct sockaddr_in));
 
-   tt_int_op(addr_len,==,sizeof(struct sockaddr_in));
+   tt_int_op(addr_len,OP_EQ,sizeof(struct sockaddr_in));
    tt_assert(sockaddr_in_are_equal((struct sockaddr_in *)sockaddr_to_check,
                                    ipv4_sockaddr_local));
 
@@ -242,7 +243,7 @@ test_address_ifaddrs_to_smartlist(void *arg)
    tor_addr_to_sockaddr(tor_addr,0,sockaddr_to_check,
                         sizeof(struct sockaddr_in));
 
-   tt_int_op(addr_len,==,sizeof(struct sockaddr_in));
+   tt_int_op(addr_len,OP_EQ,sizeof(struct sockaddr_in));
    tt_assert(sockaddr_in_are_equal((struct sockaddr_in *)sockaddr_to_check,
                                    ipv4_sockaddr_remote));
 
@@ -251,7 +252,7 @@ test_address_ifaddrs_to_smartlist(void *arg)
    tor_addr_to_sockaddr(tor_addr,0,sockaddr_to_check,
                         sizeof(struct sockaddr_in6));
 
-   tt_int_op(addr_len,==,sizeof(struct sockaddr_in6));
+   tt_int_op(addr_len,OP_EQ,sizeof(struct sockaddr_in6));
    tt_assert(sockaddr_in6_are_equal((struct sockaddr_in6*)sockaddr_to_check,
                                     ipv6_sockaddr));
 
@@ -305,7 +306,7 @@ test_address_get_if_addrs_ifaddrs(void *arg)
   return;
 }
 
-#endif
+#endif /* defined(HAVE_IFADDRS_TO_SMARTLIST) */
 
 #ifdef HAVE_IP_ADAPTER_TO_SMARTLIST
 
@@ -319,7 +320,7 @@ test_address_get_if_addrs_win32(void *arg)
 
   results = get_interface_addresses_win32(LOG_ERR, AF_UNSPEC);
 
-  tt_int_op(smartlist_len(results),>=,1);
+  tt_int_op(smartlist_len(results),OP_GE,1);
   tt_assert(smartlist_contains_localhost_tor_addr(results));
   tt_assert(!smartlist_contains_null_tor_addr(results));
 
@@ -384,7 +385,7 @@ test_address_ip_adapter_addresses_to_smartlist(void *arg)
   result = ip_adapter_addresses_to_smartlist(addrs1);
 
   tt_assert(result);
-  tt_assert(smartlist_len(result) == 3);
+  tt_int_op(smartlist_len(result), OP_EQ, 3);
 
   tor_addr = smartlist_get(result,0);
 
@@ -421,7 +422,7 @@ test_address_ip_adapter_addresses_to_smartlist(void *arg)
   tor_free(sockaddr_to_check);
   return;
 }
-#endif
+#endif /* defined(HAVE_IP_ADAPTER_TO_SMARTLIST) */
 
 #ifdef HAVE_IFCONF_TO_SMARTLIST
 
@@ -456,14 +457,14 @@ test_address_ifreq_to_smartlist(void *arg)
   ifc->ifc_ifcu.ifcu_req = ifr;
 
   results = ifreq_to_smartlist(ifc->ifc_buf,ifc->ifc_len);
-  tt_int_op(smartlist_len(results),==,1);
+  tt_int_op(smartlist_len(results),OP_EQ,1);
 
   tor_addr = smartlist_get(results, 0);
   addr_len =
   tor_addr_to_sockaddr(tor_addr,0,(struct sockaddr *)sockaddr_to_check,
                        sizeof(struct sockaddr_in));
 
-  tt_int_op(addr_len,==,sizeof(struct sockaddr_in));
+  tt_int_op(addr_len,OP_EQ,sizeof(struct sockaddr_in));
   tt_assert(sockaddr_in_are_equal(sockaddr,sockaddr_to_check));
 
   ifr = tor_realloc(ifr,2*sizeof(struct ifreq));
@@ -479,14 +480,14 @@ test_address_ifreq_to_smartlist(void *arg)
   smartlist_free(results);
 
   results = ifreq_to_smartlist(ifc->ifc_buf,ifc->ifc_len);
-  tt_int_op(smartlist_len(results),==,2);
+  tt_int_op(smartlist_len(results),OP_EQ,2);
 
   tor_addr = smartlist_get(results, 0);
   addr_len =
   tor_addr_to_sockaddr(tor_addr,0,(struct sockaddr *)sockaddr_to_check,
                        sizeof(struct sockaddr_in));
 
-  tt_int_op(addr_len,==,sizeof(struct sockaddr_in));
+  tt_int_op(addr_len,OP_EQ,sizeof(struct sockaddr_in));
   tt_assert(sockaddr_in_are_equal(sockaddr,sockaddr_to_check));
 
   tor_addr = smartlist_get(results, 1);
@@ -494,7 +495,7 @@ test_address_ifreq_to_smartlist(void *arg)
   tor_addr_to_sockaddr(tor_addr,0,(struct sockaddr *)sockaddr_to_check,
                        sizeof(struct sockaddr_in));
 
-  tt_int_op(addr_len,==,sizeof(struct sockaddr_in));
+  tt_int_op(addr_len,OP_EQ,sizeof(struct sockaddr_in));
   tt_assert(sockaddr_in_are_equal(sockaddr_eth1,sockaddr_to_check));
 
   done:
@@ -543,7 +544,7 @@ test_address_get_if_addrs_ioctl(void *arg)
   return;
 }
 
-#endif
+#endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */
 
 #define FAKE_SOCKET_FD (42)
 
@@ -633,7 +634,7 @@ test_address_udp_socket_trick_whitebox(void *arg)
   get_interface_address6_via_udp_socket_hack(LOG_DEBUG,
                                              AF_INET, addr_from_hack);
 
-  tt_int_op(hack_retval,==,0);
+  tt_int_op(hack_retval,OP_EQ,0);
   tt_assert(tor_addr_eq_ipv4h(addr_from_hack, 0x1720f676));
 
   /* Now, lets do an IPv6 case. */
@@ -648,7 +649,7 @@ test_address_udp_socket_trick_whitebox(void *arg)
   get_interface_address6_via_udp_socket_hack(LOG_DEBUG,
                                              AF_INET6, addr_from_hack);
 
-  tt_int_op(hack_retval,==,0);
+  tt_int_op(hack_retval,OP_EQ,0);
 
   tor_addr_to_sockaddr(addr_from_hack,0,(struct sockaddr *)ipv6_to_check,
                        sizeof(struct sockaddr_in6));
@@ -693,7 +694,7 @@ test_address_udp_socket_trick_blackbox(void *arg)
                                                       AF_INET,
                                                       &addr4_to_check);
 
-  tt_int_op(retval,==,retval_reference);
+  tt_int_op(retval,OP_EQ,retval_reference);
   tt_assert( (retval == -1 && retval_reference == -1) ||
              (tor_addr_compare(&addr4,&addr4_to_check,CMP_EXACT) == 0) );
 
@@ -702,11 +703,11 @@ test_address_udp_socket_trick_blackbox(void *arg)
                                                       AF_INET6,
                                                       &addr6_to_check);
 
-  tt_int_op(retval,==,retval_reference);
+  tt_int_op(retval,OP_EQ,retval_reference);
   tt_assert( (retval == -1 && retval_reference == -1) ||
              (tor_addr_compare(&addr6,&addr6_to_check,CMP_EXACT) == 0) );
 
-#else
+#else /* !(0) */
   /* Both of the blackbox test cases fail horribly if:
    *  * The host has no external addreses.
    *  * There are multiple interfaces with either AF_INET or AF_INET6.
@@ -721,7 +722,7 @@ test_address_udp_socket_trick_blackbox(void *arg)
   (void)addr6_to_check;
   (void)addr6;
   (void) retval_reference;
-#endif
+#endif /* 0 */
 
   /* When family is neither AF_INET nor AF_INET6, we want _hack to
    * fail and return -1.
@@ -730,7 +731,7 @@ test_address_udp_socket_trick_blackbox(void *arg)
   retval = get_interface_address6_via_udp_socket_hack(LOG_DEBUG,
                                                       AF_INET+AF_INET6,&addr4);
 
-  tt_assert(retval == -1);
+  tt_int_op(retval, OP_EQ, -1);
 
   done:
   return;
@@ -743,13 +744,13 @@ test_address_get_if_addrs_list_internal(void *arg)
 
   (void)arg;
 
-  results = get_interface_address_list(LOG_ERR, 1);
+  results = get_interface_address_list(LOG_WARN, 1);
 
-  tt_assert(results != NULL);
+  tt_ptr_op(results, OP_NE, NULL);
   /* When the network is down, a system might not have any non-local
    * non-multicast addresseses, not even internal ones.
    * Unit tests shouldn't fail because of this. */
-  tt_int_op(smartlist_len(results),>=,0);
+  tt_int_op(smartlist_len(results),OP_GE,0);
 
   tt_assert(!smartlist_contains_localhost_tor_addr(results));
   tt_assert(!smartlist_contains_multicast_tor_addr(results));
@@ -763,7 +764,7 @@ test_address_get_if_addrs_list_internal(void *arg)
   tt_assert(!smartlist_contains_ipv6_tor_addr(results));
 
  done:
-  free_interface_address_list(results);
+  interface_address_list_free(results);
   return;
 }
 
@@ -774,11 +775,11 @@ test_address_get_if_addrs_list_no_internal(void *arg)
 
   (void)arg;
 
-  results = get_interface_address_list(LOG_ERR, 0);
+  results = get_interface_address_list(LOG_WARN, 0);
 
-  tt_assert(results != NULL);
+  tt_ptr_op(results, OP_NE, NULL);
   /* Work even on systems with only internal IPv4 addresses */
-  tt_int_op(smartlist_len(results),>=,0);
+  tt_int_op(smartlist_len(results),OP_GE,0);
 
   tt_assert(!smartlist_contains_localhost_tor_addr(results));
   tt_assert(!smartlist_contains_multicast_tor_addr(results));
@@ -792,7 +793,7 @@ test_address_get_if_addrs_list_no_internal(void *arg)
   tt_assert(!smartlist_contains_ipv6_tor_addr(results));
 
  done:
-  free_interface_address_list(results);
+  interface_address_list_free(results);
   return;
 }
 
@@ -818,9 +819,9 @@ test_address_get_if_addrs6_list_internal(void *arg)
   }
   teardown_capture_of_logs();
 
-  tt_assert(results != NULL);
+  tt_ptr_op(results, OP_NE, NULL);
   /* Work even on systems without IPv6 interfaces */
-  tt_int_op(smartlist_len(results),>=,0);
+  tt_int_op(smartlist_len(results),OP_GE,0);
 
   tt_assert(!smartlist_contains_localhost_tor_addr(results));
   tt_assert(!smartlist_contains_multicast_tor_addr(results));
@@ -834,7 +835,7 @@ test_address_get_if_addrs6_list_internal(void *arg)
   }
 
  done:
-  free_interface_address6_list(results);
+  interface_address6_list_free(results);
   teardown_capture_of_logs();
   return;
 }
@@ -861,9 +862,9 @@ test_address_get_if_addrs6_list_no_internal(void *arg)
   }
   teardown_capture_of_logs();
 
-  tt_assert(results != NULL);
+  tt_ptr_op(results, OP_NE, NULL);
   /* Work even on systems without IPv6 interfaces */
-  tt_int_op(smartlist_len(results),>=,0);
+  tt_int_op(smartlist_len(results),OP_GE,0);
 
   tt_assert(!smartlist_contains_localhost_tor_addr(results));
   tt_assert(!smartlist_contains_multicast_tor_addr(results));
@@ -878,7 +879,7 @@ test_address_get_if_addrs6_list_no_internal(void *arg)
 
  done:
   teardown_capture_of_logs();
-  free_interface_address6_list(results);
+  interface_address6_list_free(results);
   return;
 }
 
@@ -927,24 +928,24 @@ test_address_get_if_addrs_internal_fail(void *arg)
        mock_get_interface_address6_via_udp_socket_hack_fail);
 
   results1 = get_interface_address6_list(LOG_ERR, AF_INET6, 1);
-  tt_assert(results1 != NULL);
-  tt_int_op(smartlist_len(results1),==,0);
+  tt_ptr_op(results1, OP_NE, NULL);
+  tt_int_op(smartlist_len(results1),OP_EQ,0);
 
   results2 = get_interface_address_list(LOG_ERR, 1);
-  tt_assert(results2 != NULL);
-  tt_int_op(smartlist_len(results2),==,0);
+  tt_ptr_op(results2, OP_NE, NULL);
+  tt_int_op(smartlist_len(results2),OP_EQ,0);
 
   rv = get_interface_address6(LOG_ERR, AF_INET6, &ipv6_addr);
-  tt_assert(rv == -1);
+  tt_int_op(rv, OP_EQ, -1);
 
   rv = get_interface_address(LOG_ERR, &ipv4h_addr);
-  tt_assert(rv == -1);
+  tt_int_op(rv, OP_EQ, -1);
 
  done:
   UNMOCK(get_interface_addresses_raw);
   UNMOCK(get_interface_address6_via_udp_socket_hack);
-  free_interface_address6_list(results1);
-  free_interface_address6_list(results2);
+  interface_address6_list_free(results1);
+  interface_address6_list_free(results2);
   return;
 }
 
@@ -961,18 +962,18 @@ test_address_get_if_addrs_no_internal_fail(void *arg)
        mock_get_interface_address6_via_udp_socket_hack_fail);
 
   results1 = get_interface_address6_list(LOG_ERR, AF_INET6, 0);
-  tt_assert(results1 != NULL);
-  tt_int_op(smartlist_len(results1),==,0);
+  tt_ptr_op(results1, OP_NE, NULL);
+  tt_int_op(smartlist_len(results1),OP_EQ,0);
 
   results2 = get_interface_address_list(LOG_ERR, 0);
-  tt_assert(results2 != NULL);
-  tt_int_op(smartlist_len(results2),==,0);
+  tt_ptr_op(results2, OP_NE, NULL);
+  tt_int_op(smartlist_len(results2),OP_EQ,0);
 
  done:
   UNMOCK(get_interface_addresses_raw);
   UNMOCK(get_interface_address6_via_udp_socket_hack);
-  free_interface_address6_list(results1);
-  free_interface_address6_list(results2);
+  interface_address6_list_free(results1);
+  interface_address6_list_free(results2);
   return;
 }
 
@@ -985,7 +986,7 @@ test_address_get_if_addrs(void *arg)
 
   (void)arg;
 
-  rv = get_interface_address(LOG_ERR, &addr_h);
+  rv = get_interface_address(LOG_WARN, &addr_h);
 
   /* When the network is down, a system might not have any non-local
    * non-multicast IPv4 addresses, not even internal ones.
@@ -1012,7 +1013,7 @@ test_address_get_if_addrs6(void *arg)
 
   (void)arg;
 
-  rv = get_interface_address6(LOG_ERR, AF_INET6, &tor_addr);
+  rv = get_interface_address6(LOG_WARN, AF_INET6, &tor_addr);
 
   /* Work even on systems without IPv6 interfaces */
   if (rv == 0) {
@@ -1139,6 +1140,36 @@ test_address_tor_addr_eq_ipv4h(void *ignored)
   tor_free(a);
 }
 
+static void
+test_address_tor_addr_in_same_network_family(void *ignored)
+{
+  (void)ignored;
+  tor_addr_t a, b;
+
+  tor_addr_parse(&a, "8.8.8.8");
+  tor_addr_parse(&b, "8.8.4.4");
+  tt_int_op(addrs_in_same_network_family(&a, &b), OP_EQ, 1);
+
+  tor_addr_parse(&a, "8.8.8.8");
+  tor_addr_parse(&b, "1.1.1.1");
+  tt_int_op(addrs_in_same_network_family(&a, &b), OP_EQ, 0);
+
+  tor_addr_parse(&a, "8.8.8.8");
+  tor_addr_parse(&b, "2001:4860:4860::8844");
+  tt_int_op(addrs_in_same_network_family(&a, &b), OP_EQ, 0);
+
+  tor_addr_parse(&a, "2001:4860:4860::8888");
+  tor_addr_parse(&b, "2001:4860:4860::8844");
+  tt_int_op(addrs_in_same_network_family(&a, &b), OP_EQ, 1);
+
+  tor_addr_parse(&a, "2001:4860:4860::8888");
+  tor_addr_parse(&b, "2001:470:20::2");
+  tt_int_op(addrs_in_same_network_family(&a, &b), OP_EQ, 0);
+
+ done:
+  return;
+}
+
 #define ADDRESS_TEST(name, flags) \
   { #name, test_address_ ## name, flags, NULL, NULL }
 
@@ -1170,6 +1201,6 @@ struct testcase_t address_tests[] = {
   ADDRESS_TEST(tor_addr_to_ipv4n, 0),
   ADDRESS_TEST(tor_addr_to_mapped_ipv4h, 0),
   ADDRESS_TEST(tor_addr_eq_ipv4h, 0),
+  ADDRESS_TEST(tor_addr_in_same_network_family, 0),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_address_set.c b/src/test/test_address_set.c
index df022f539a..fb8408b3c3 100644
--- a/src/test/test_address_set.c
+++ b/src/test/test_address_set.c
@@ -1,15 +1,21 @@
-/* Copyright (c) 2017, The Tor Project, Inc. */
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#include "or.h"
-#include "address_set.h"
-#include "microdesc.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "routerlist.h"
-#include "torcert.h"
-
-#include "test.h"
+#include "core/or/or.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "core/or/address_set.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/torcert.h"
+
+#include "feature/nodelist/microdesc_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+#include "test/test.h"
 
 static networkstatus_t *dummy_ns = NULL;
 static networkstatus_t *
diff --git a/src/test/test_bridges.c b/src/test/test_bridges.c
new file mode 100644
index 0000000000..879ae6636b
--- /dev/null
+++ b/src/test/test_bridges.c
@@ -0,0 +1,704 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information     */
+
+/**
+ * \file test_bridges.c
+ * \brief Unittests for code in bridges.c
+ **/
+
+#define TOR_BRIDGES_PRIVATE
+#define PT_PRIVATE /* Only needed for the mock_* items below */
+
+#include <stdbool.h>
+
+#include "core/or/or.h"
+#include "lib/net/address.h"
+#include "feature/client/bridges.h"
+#include "app/config/config.h"
+#include "feature/client/transports.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+#include "feature/nodelist/microdesc_st.h"
+
+/* Test suite stuff */
+#include "test/test.h"
+
+/**
+ * A mocked transport_t, constructed via mock_transport_get_by_name().
+ */
+static transport_t *mock_transport = NULL;
+
+/**
+ * Mock transport_get_by_name() to simply return a transport_t for the
+ * transport name that was input to it.
+ */
+static transport_t *
+mock_transport_get_by_name(const char *name)
+{
+  tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t));
+  uint16_t port = 9999;
+  int socksv = 9;
+  char *args = tor_strdup("foo=bar");
+
+  if (!mock_transport) {
+    tor_addr_parse(addr, "99.99.99.99");
+    mock_transport = transport_new(addr, port, name, socksv, args);
+  }
+
+  tor_free(addr);
+  tor_free(args);
+
+  return mock_transport;
+}
+
+#undef PT_PRIVATE /* defined(PT_PRIVATE) */
+
+/**
+ * Test helper: Add a variety of bridges to our global bridgelist.
+ */
+static void
+helper_add_bridges_to_bridgelist(void *arg)
+{
+  /* Note: the two bridges which do not have specified fingerprints will be
+   * internally stored as both having the same fingerprint of all-zero bytes.
+   */
+
+  (void)arg;
+  char *bridge0 = tor_strdup("6.6.6.6:6666");
+  char *bridge1 = tor_strdup("6.6.6.7:6667 "
+                             "A10C4F666D27364036B562823E5830BC448E046A");
+  char *bridge2 = tor_strdup("obfs4 198.245.60.51:443 "
+                             "752CF7825B3B9EA6A98C83AC41F7099D67007EA5 "
+                  "cert=xpmQtKUqQ/6v5X7ijgYE/f03+l2/EuQ1dexjyUhh16wQlu/"
+                             "cpXUGalmhDIlhuiQPNEKmKw iat-mode=0");
+  char *bridge3 = tor_strdup("banana 5.5.5.5:5555 "
+                             "9D6AE1BD4FDF39721CE908966E79E16F9BFCCF2F");
+  char *bridge4 = tor_strdup("obfs4 1.2.3.4:1234 "
+                             "foo=abcdefghijklmnopqrstuvwxyz");
+  char *bridge5 = tor_strdup("apple 4.4.4.4:4444 "
+                             "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA "
+                             "foo=abcdefghijklmnopqrstuvwxyz");
+  char *bridge6 = tor_strdup("[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:6666");
+
+  mark_bridge_list();
+
+#define ADD_BRIDGE(bridge)                                          \
+  bridge_line_t *bridge_line_ ##bridge = parse_bridge_line(bridge); \
+  if (!bridge_line_ ##bridge) {                                     \
+    printf("Unparseable bridge line: '%s'", #bridge);               \
+  } else {                                                          \
+    bridge_add_from_config(bridge_line_ ##bridge);                  \
+  }                                                                 \
+  tor_free(bridge);
+
+  ADD_BRIDGE(bridge0);
+  ADD_BRIDGE(bridge1);
+  ADD_BRIDGE(bridge2);
+  ADD_BRIDGE(bridge3);
+  ADD_BRIDGE(bridge4);
+  ADD_BRIDGE(bridge5);
+  ADD_BRIDGE(bridge6);
+#undef ADD_BRIDGES
+
+  sweep_bridge_list();
+}
+
+/**
+ * Make sure our test helper works too.
+ */
+static void
+test_bridges_helper_func_add_bridges_to_bridgelist(void *arg)
+{
+  helper_add_bridges_to_bridgelist(arg);
+  tt_finished();
+
+ done:
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling bridge_list_get() should create a new bridgelist if we
+ * didn't have one before.
+ */
+static void
+test_bridges_bridge_list_get_creates_new_bridgelist(void *arg)
+{
+  const smartlist_t *bridgelist = bridge_list_get();
+
+  (void)arg;
+
+  tt_ptr_op(bridgelist, OP_NE, NULL);
+
+ done:
+  return;
+}
+
+/**
+ * Calling clear_bridge_list() should remove all bridges from the bridgelist.
+ */
+static void
+test_bridges_clear_bridge_list(void *arg)
+{
+  const smartlist_t *bridgelist;
+  const smartlist_t *bridgelist_after;
+  const bridge_info_t *bridge;
+
+  helper_add_bridges_to_bridgelist(arg);
+  bridgelist = bridge_list_get();
+  tt_ptr_op(bridgelist, OP_NE, NULL);
+
+  bridge = smartlist_get(bridgelist, 0);
+  tt_ptr_op(bridge, OP_NE, NULL);
+
+  clear_bridge_list();
+  bridgelist_after = bridge_list_get();
+  tt_ptr_op(bridgelist_after, OP_NE, NULL);
+  tt_int_op(smartlist_len(bridgelist_after), OP_EQ, 0);
+
+ done:
+  return;
+}
+
+/**
+ * Calling bridge_get_addrport() should give me the address and port
+ * of the bridge.  In this case, we sort the smartlist of bridges on
+ * fingerprints and choose the first one.
+ */
+static void
+test_bridges_bridge_get_addrport(void *arg)
+{
+  smartlist_t *bridgelist;
+  const bridge_info_t *bridge;
+  const tor_addr_port_t *addrport;
+
+  helper_add_bridges_to_bridgelist(arg);
+  bridgelist = (smartlist_t*)bridge_list_get();
+  tt_ptr_op(bridgelist, OP_NE, NULL);
+
+  // This should be the bridge at 6.6.6.6:6666 with fingerprint
+  // 0000000000000000000000000000000000000000
+  bridge = smartlist_get(bridgelist, 0);
+  tt_ptr_op(bridge, OP_NE, NULL);
+
+  addrport = bridge_get_addr_port(bridge);
+  tt_int_op(addrport->port, OP_EQ, 6666);
+
+ done:
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling get_configured_bridge_by_orports_digest() with two
+ * configured bridge orports and an invalid digest should return the
+ * bridge of the first addrport in the list.
+ */
+static void
+test_bridges_get_configured_bridge_by_orports_digest(void *arg)
+{
+  smartlist_t *orports = NULL;
+  const smartlist_t *bridgelist;
+  const bridge_info_t *bridge1;
+  const bridge_info_t *bridge2;
+  const bridge_info_t *ret;
+  tor_addr_port_t *addrport1;
+  tor_addr_port_t *addrport2;
+  const char *digest;
+
+  helper_add_bridges_to_bridgelist(arg);
+  bridgelist = bridge_list_get();
+  tt_ptr_op(bridgelist, OP_NE, NULL);
+
+  // This should be the bridge at 6.6.6.6:6666 with fingerprint
+  // 0000000000000000000000000000000000000000
+  bridge1 = smartlist_get(bridgelist, 0);
+  tt_ptr_op(bridge1, OP_NE, NULL);
+  // This should be the bridge at 6.6.6.7:6667 with fingerprint
+  // A10C4F666D27364036B562823E5830BC448E046A
+  bridge2 = smartlist_get(bridgelist, 1);
+  tt_ptr_op(bridge2, OP_NE, NULL);
+
+  addrport1 = (tor_addr_port_t*)bridge_get_addr_port(bridge1);
+  tt_int_op(addrport1->port, OP_EQ, 6666);
+  addrport2 = (tor_addr_port_t*)bridge_get_addr_port(bridge2);
+  tt_int_op(addrport2->port, OP_EQ, 6667);
+
+  orports = smartlist_new();
+  smartlist_add(orports, addrport1);
+  smartlist_add(orports, addrport2);
+
+  digest = "zzzzzzzzzzzzzzzz";
+
+  ret = get_configured_bridge_by_orports_digest(digest, orports);
+  tt_ptr_op(ret, OP_NE, NULL);
+
+  tt_assert(tor_addr_port_eq(addrport1, bridge_get_addr_port(ret)));
+
+ done:
+  smartlist_free(orports);
+
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling get_configured_bridge_by_addr_port_digest() with a digest that we do
+ * have and an addr:port pair we don't should return the bridge for that
+ * digest.
+ */
+static void
+test_bridges_get_configured_bridge_by_addr_port_digest_digest_only(void *arg)
+{
+  char digest[DIGEST_LEN];
+  bridge_info_t *bridge;
+  const char fingerprint[HEX_DIGEST_LEN] =
+    "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+  tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t));
+  char ret_addr[16];
+  uint16_t port = 11111;
+  int ret;
+
+  helper_add_bridges_to_bridgelist(arg);
+
+  // We don't actually have a bridge with this addr:port pair
+  base16_decode(digest, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN);
+  ret = tor_addr_parse(addr, "111.111.111.111");
+  tt_int_op(ret, OP_EQ, 2); // it returns the address family on success
+
+  bridge = get_configured_bridge_by_addr_port_digest(addr, port, digest);
+  tt_ptr_op(bridge, OP_NE, NULL);
+
+  tor_addr_to_str(ret_addr, &bridge_get_addr_port(bridge)->addr, 16, 0);
+  tt_str_op("4.4.4.4", OP_EQ, ret_addr);
+
+ done:
+  tor_free(addr);
+
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling get_configured_bridge_by_addr_port_digest() with only an
+ * addr:port (i.e. digest set to NULL) should return the bridge for
+ * that digest when there is such a bridge.
+ */
+static void
+test_bridges_get_configured_bridge_by_addr_port_digest_address_only(void *arg)
+{
+  bridge_info_t *bridge;
+  tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t));
+  char ret_addr[16];
+  uint16_t port = 6666;
+  int ret;
+
+  helper_add_bridges_to_bridgelist(arg);
+
+  ret = tor_addr_parse(addr, "6.6.6.6");
+  tt_int_op(ret, OP_EQ, 2); // it returns the address family on success
+
+  bridge = get_configured_bridge_by_addr_port_digest(addr, port, NULL);
+  tt_ptr_op(bridge, OP_NE, NULL);
+
+  tor_addr_to_str(ret_addr, &bridge_get_addr_port(bridge)->addr, 16, 0);
+  tt_str_op("6.6.6.6", OP_EQ, ret_addr);
+
+ done:
+  tor_free(addr);
+
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling get_configured_bridge_by_exact_addr_port_digest() with a digest that
+ * we do have, and an addr:port pair we don't have, should return NULL.
+ */
+static void
+test_bridges_get_configured_bridge_by_exact_addr_port_digest_donly(void *arg)
+{
+  char digest[DIGEST_LEN];
+  bridge_info_t *bridge;
+  const char fingerprint[HEX_DIGEST_LEN] =
+    "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+  tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t));
+  uint16_t port = 11111;
+  int ret;
+
+  helper_add_bridges_to_bridgelist(arg);
+
+  // We don't actually have a bridge with this addr:port pair
+  base16_decode(digest, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN);
+  ret = tor_addr_parse(addr, "111.111.111.111");
+  tt_int_op(ret, OP_EQ, 2); // it returns the address family on success
+
+  bridge = get_configured_bridge_by_exact_addr_port_digest(addr, port, digest);
+  tt_ptr_op(bridge, OP_EQ, NULL);
+
+ done:
+  tor_free(addr);
+
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling get_configured_bridge_by_exact_addr_port_digest() with a digest that
+ * we do have, and an addr:port pair we do have, should return the bridge.
+ */
+static void
+test_bridges_get_configured_bridge_by_exact_addr_port_digest_both(void *arg)
+{
+  char digest[DIGEST_LEN];
+  bridge_info_t *bridge;
+  const char fingerprint[HEX_DIGEST_LEN] =
+    "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+  tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t));
+  uint16_t port = 4444;
+  char ret_addr[16];
+  int ret;
+
+  helper_add_bridges_to_bridgelist(arg);
+
+  base16_decode(digest, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN);
+  ret = tor_addr_parse(addr, "4.4.4.4");
+  tt_int_op(ret, OP_EQ, 2); // it returns the address family on success
+
+  bridge = get_configured_bridge_by_exact_addr_port_digest(addr, port, digest);
+  tt_ptr_op(bridge, OP_NE, NULL);
+
+  tor_addr_to_str(ret_addr, &bridge_get_addr_port(bridge)->addr, 16, 0);
+  tt_str_op("4.4.4.4", OP_EQ, ret_addr);
+
+ done:
+  tor_free(addr);
+
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling get_configured_bridge_by_exact_addr_port_digest() with no digest,
+ * and an addr:port pair we do have, should return the bridge.
+ */
+static void
+test_bridges_get_configured_bridge_by_exact_addr_port_digest_aonly(void *arg)
+{
+  bridge_info_t *bridge;
+  tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t));
+  uint16_t port = 4444;
+  char ret_addr[16];
+  int ret;
+
+  helper_add_bridges_to_bridgelist(arg);
+
+  ret = tor_addr_parse(addr, "4.4.4.4");
+  tt_int_op(ret, OP_EQ, 2); // it returns the address family on success
+
+  bridge = get_configured_bridge_by_exact_addr_port_digest(addr, port, NULL);
+  tt_ptr_op(bridge, OP_NE, NULL);
+
+  tor_addr_to_str(ret_addr, &bridge_get_addr_port(bridge)->addr, 16, 0);
+  tt_str_op("4.4.4.4", OP_EQ, ret_addr);
+
+ done:
+  tor_free(addr);
+
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling find_bridge_by_digest() when we have a bridge with a known
+ * identity digest should return the bridge's information.
+ */
+static void
+test_bridges_find_bridge_by_digest_known(void *arg)
+{
+  char digest1[DIGEST_LEN];
+  bridge_info_t *bridge;
+  const char fingerprint[HEX_DIGEST_LEN] =
+    "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+
+  helper_add_bridges_to_bridgelist(arg);
+
+  base16_decode(digest1, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN);
+  bridge = find_bridge_by_digest(digest1);
+
+  tt_ptr_op(bridge, OP_NE, NULL);
+
+  /* We have to call bridge_get_rsa_id_digest() here because the bridge_info_t
+   * struct is opaquely defined in bridges.h. */
+  const uint8_t *digest2 = bridge_get_rsa_id_digest(bridge);
+
+  tt_mem_op((char*)digest2, OP_EQ, digest1, DIGEST_LEN);
+
+ done:
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling find_bridge_by_digest() when we do NOT have a bridge with that
+ * identity digest should return NULL.
+ */
+static void
+test_bridges_find_bridge_by_digest_unknown(void *arg)
+{
+  const char *fingerprint = "cccccccccccccccccccccccccccccccccccccccc";
+  bridge_info_t *bridge;
+
+  helper_add_bridges_to_bridgelist(arg);
+
+  bridge = find_bridge_by_digest(fingerprint);
+
+  tt_ptr_op(bridge, OP_EQ, NULL);
+
+ done:
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling bridge_resolve_conflicts() with an identical bridge to one we've
+ * already configure should mark the pre-configured bridge for removal.
+ */
+static void
+test_bridges_bridge_resolve_conflicts(void *arg)
+{
+  tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t));
+  uint16_t port = 4444;
+  const char *digest = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
+  const char *transport = "apple";
+  int ret;
+
+  helper_add_bridges_to_bridgelist(arg);
+
+  ret = tor_addr_parse(addr, "4.4.4.4");
+  tt_int_op(ret, OP_EQ, 2); // it returns the address family on success
+
+  bridge_resolve_conflicts((const tor_addr_t*)addr, port, digest, transport);
+
+  /* The bridge should now be marked for removal, and removed when we sweep the
+   * bridge_list */
+  sweep_bridge_list();
+  ret = addr_is_a_configured_bridge((const tor_addr_t*)addr, port, digest);
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  tor_free(addr);
+
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling transport_is_needed() with a transport we do need ("obfs4") and a
+ * bogus transport that we don't need should return 1 and 0, respectively.
+ */
+static void
+test_bridges_transport_is_needed(void *arg)
+{
+  int ret;
+
+  helper_add_bridges_to_bridgelist(arg);
+
+  ret = transport_is_needed("obfs4");
+  tt_int_op(ret, OP_EQ, 1);
+
+  ret = transport_is_needed("apowefjaoewpaief");
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling get_transport_by_bridge_addrport() with the address and port of a
+ * configured bridge which uses a pluggable transport when there is no global
+ * transport_list should return -1 and the transport_t should be NULL.
+ */
+static void
+test_bridges_get_transport_by_bridge_addrport_no_ptlist(void *arg)
+{
+  transport_t *transport = NULL;
+  tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t));
+  uint16_t port = 1234;
+  int ret;
+
+  helper_add_bridges_to_bridgelist(arg);
+
+  ret = tor_addr_parse(addr, "1.2.3.4");
+  tt_int_op(ret, OP_EQ, 2); // it returns the address family on success?
+
+  /* This will fail because the global transport_list has nothing in it, and so
+   * transport_get_by_name() has nothing to return, even the the bridge *did*
+   * say it had an obfs4 transport.
+   */
+  ret = get_transport_by_bridge_addrport((const tor_addr_t*)addr, port,
+                                         (const transport_t**)&transport);
+  tt_int_op(ret, OP_EQ, -1); // returns -1 on failure
+  tt_ptr_op(transport, OP_EQ, NULL);
+
+ done:
+  tor_free(addr);
+
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+#define PT_PRIVATE
+
+/**
+ * Calling get_transport_by_bridge_addrport() with the address and port of a
+ * configured bridge which uses a pluggable transport should return 0 and set
+ * appropriate transport_t.
+ */
+static void
+test_bridges_get_transport_by_bridge_addrport(void *arg)
+{
+  transport_t *transport = NULL;
+  tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t));
+  uint16_t port = 1234;
+  int ret;
+
+  helper_add_bridges_to_bridgelist(arg);
+  mark_transport_list(); // Also initialise our transport_list
+
+  ret = tor_addr_parse(addr, "1.2.3.4");
+  tt_int_op(ret, OP_EQ, 2); // it returns the address family on success?
+
+  /* After we mock transport_get_by_name() to return a bogus transport_t with
+   * the name it was asked for, the call should succeed.
+   */
+  MOCK(transport_get_by_name, mock_transport_get_by_name);
+  ret = get_transport_by_bridge_addrport((const tor_addr_t*)addr, port,
+                                         (const transport_t**)&transport);
+  tt_int_op(ret, OP_EQ, 0); // returns 0 on success
+  tt_ptr_op(transport, OP_NE, NULL);
+  tt_str_op(transport->name, OP_EQ, "obfs4");
+
+ done:
+  UNMOCK(transport_get_by_name);
+
+  tor_free(addr);
+  transport_free(transport);
+
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+static void
+test_bridges_node_is_a_configured_bridge(void *arg)
+{
+  routerinfo_t ri_ipv4 = { .addr = 0x06060606, .or_port = 6666 };
+  routerstatus_t rs_ipv4 = { .addr = 0x06060606, .or_port = 6666 };
+
+  routerinfo_t ri_ipv6 = { .ipv6_orport = 6666 };
+  tor_addr_parse(&(ri_ipv6.ipv6_addr),
+                 "2001:0db8:85a3:0000:0000:8a2e:0370:7334");
+
+  routerstatus_t rs_ipv6 = { .ipv6_orport = 6666 };
+  tor_addr_parse(&(rs_ipv6.ipv6_addr),
+                 "2001:0db8:85a3:0000:0000:8a2e:0370:7334");
+
+  microdesc_t md_ipv6 = { .ipv6_orport = 6666 };
+  tor_addr_parse(&(md_ipv6.ipv6_addr),
+                 "2001:0db8:85a3:0000:0000:8a2e:0370:7334");
+
+  helper_add_bridges_to_bridgelist(arg);
+
+  node_t node_with_digest;
+  memset(&node_with_digest, 0, sizeof(node_with_digest));
+
+  const char fingerprint[HEX_DIGEST_LEN] =
+    "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+
+  const char fingerprint2[HEX_DIGEST_LEN] =
+    "ffffffffffffffffffffffffffffffffffffffff";
+
+  base16_decode(node_with_digest.identity, DIGEST_LEN,
+                fingerprint, HEX_DIGEST_LEN);
+
+  node_t node_ri_ipv4 = { .ri = &ri_ipv4 };
+  base16_decode(node_ri_ipv4.identity, DIGEST_LEN,
+                fingerprint2, HEX_DIGEST_LEN);
+  tt_assert(node_is_a_configured_bridge(&node_ri_ipv4));
+
+  /* This will still match bridge0, since bridge0 has no digest set. */
+  memset(node_ri_ipv4.identity, 0x3f, DIGEST_LEN);
+  tt_assert(node_is_a_configured_bridge(&node_ri_ipv4));
+
+  /* It won't match bridge1, though, since bridge1 has a digest, and this
+     isn't it! */
+  node_ri_ipv4.ri->addr = 0x06060607;
+  node_ri_ipv4.ri->or_port = 6667;
+  tt_assert(! node_is_a_configured_bridge(&node_ri_ipv4));
+  /* If we set the fingerprint right, though, it will match. */
+  base16_decode(node_ri_ipv4.identity, DIGEST_LEN,
+                "A10C4F666D27364036B562823E5830BC448E046A", HEX_DIGEST_LEN);
+  tt_assert(node_is_a_configured_bridge(&node_ri_ipv4));
+
+  node_t node_rs_ipv4 = { .rs = &rs_ipv4 };
+  base16_decode(node_rs_ipv4.identity, DIGEST_LEN,
+                fingerprint2, HEX_DIGEST_LEN);
+  tt_assert(node_is_a_configured_bridge(&node_rs_ipv4));
+
+  node_t node_ri_ipv6 = { .ri = &ri_ipv6 };
+  base16_decode(node_ri_ipv6.identity, DIGEST_LEN,
+                fingerprint2, HEX_DIGEST_LEN);
+  tt_assert(node_is_a_configured_bridge(&node_ri_ipv6));
+
+  node_t node_rs_ipv6 = { .rs = &rs_ipv6 };
+  base16_decode(node_rs_ipv6.identity, DIGEST_LEN,
+                fingerprint2, HEX_DIGEST_LEN);
+  tt_assert(node_is_a_configured_bridge(&node_rs_ipv6));
+
+  node_t node_md_ipv6 = { .md = &md_ipv6 };
+  base16_decode(node_md_ipv6.identity, DIGEST_LEN,
+                fingerprint2, HEX_DIGEST_LEN);
+  tt_assert(node_is_a_configured_bridge(&node_md_ipv6));
+
+  mark_bridge_list();
+  sweep_bridge_list();
+
+  tt_assert(!node_is_a_configured_bridge(&node_with_digest));
+  tt_assert(!node_is_a_configured_bridge(&node_ri_ipv4));
+  tt_assert(!node_is_a_configured_bridge(&node_ri_ipv6));
+  tt_assert(!node_is_a_configured_bridge(&node_rs_ipv4));
+  tt_assert(!node_is_a_configured_bridge(&node_rs_ipv6));
+  tt_assert(!node_is_a_configured_bridge(&node_md_ipv6));
+
+ done:
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+#undef PT_PRIVATE /* defined(PT_PRIVATE) */
+
+#define B_TEST(name, flags) \
+  { #name, test_bridges_ ##name, (flags), NULL, NULL }
+
+struct testcase_t bridges_tests[] = {
+  B_TEST(helper_func_add_bridges_to_bridgelist, 0),
+  B_TEST(bridge_list_get_creates_new_bridgelist, 0),
+  B_TEST(clear_bridge_list, 0),
+  B_TEST(bridge_get_addrport, 0),
+  B_TEST(get_configured_bridge_by_orports_digest, 0),
+  B_TEST(get_configured_bridge_by_addr_port_digest_digest_only, 0),
+  B_TEST(get_configured_bridge_by_addr_port_digest_address_only, 0),
+  B_TEST(get_configured_bridge_by_exact_addr_port_digest_donly, 0),
+  B_TEST(get_configured_bridge_by_exact_addr_port_digest_both, 0),
+  B_TEST(get_configured_bridge_by_exact_addr_port_digest_aonly, 0),
+  B_TEST(find_bridge_by_digest_known, 0),
+  B_TEST(find_bridge_by_digest_unknown, 0),
+  B_TEST(bridge_resolve_conflicts, 0),
+  B_TEST(get_transport_by_bridge_addrport_no_ptlist, 0),
+  B_TEST(get_transport_by_bridge_addrport, 0),
+  B_TEST(transport_is_needed, 0),
+  B_TEST(node_is_a_configured_bridge, 0),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_bt_cl.c b/src/test/test_bt_cl.c
index 95b4f48f11..0c15a02ee4 100644
--- a/src/test/test_bt_cl.c
+++ b/src/test/test_bt_cl.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -7,10 +7,13 @@
 
 /* To prevent 'assert' from going away. */
 #undef TOR_COVERAGE
-#include "or.h"
-#include "util.h"
-#include "backtrace.h"
-#include "torlog.h"
+#include "core/or/or.h"
+#include "lib/err/backtrace.h"
+#include "lib/log/log.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
 
 /* -1: no crash.
  *  0: crash with a segmentation fault.
@@ -19,14 +22,12 @@ static int crashtype = 0;
 
 #ifdef __GNUC__
 #define NOINLINE __attribute__((noinline))
-#define NORETURN __attribute__((noreturn))
 #endif
 
 int crash(int x) NOINLINE;
 int oh_what(int x) NOINLINE;
 int a_tangled_web(int x) NOINLINE;
 int we_weave(int x) NOINLINE;
-static void abort_handler(int s) NORETURN;
 
 #ifdef HAVE_CFLAG_WNULL_DEREFERENCE
 DISABLE_GCC_WARNING(null-dereference)
@@ -40,7 +41,7 @@ crash(int x)
                          * don't need to see us dereference NULL. */
 #else
     *(volatile int *)0 = 0;
-#endif
+#endif /* defined(__clang_analyzer__) || defined(__COVERITY__) */
   } else if (crashtype == 1) {
     tor_assert(1 == 0);
   } else if (crashtype == -1) {
@@ -76,13 +77,6 @@ we_weave(int x)
   return a_tangled_web(x) + a_tangled_web(x+1);
 }
 
-static void
-abort_handler(int s)
-{
-  (void)s;
-  exit(0);
-}
-
 int
 main(int argc, char **argv)
 {
@@ -120,8 +114,6 @@ main(int argc, char **argv)
 
   configure_backtrace_handler(NULL);
 
-  signal(SIGABRT, abort_handler);
-
   printf("%d\n", we_weave(2));
 
   clean_up_backtrace_handler();
@@ -129,4 +121,3 @@ main(int argc, char **argv)
 
   return 0;
 }
-
diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c
index 3408da3aa9..2f9ad1fbe3 100644
--- a/src/test/test_buffers.c
+++ b/src/test/test_buffers.c
@@ -1,13 +1,19 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define BUFFERS_PRIVATE
-#include "or.h"
-#include "buffers.h"
-#include "ext_orport.h"
-#include "test.h"
+#define PROTO_HTTP_PRIVATE
+#include "core/or/or.h"
+#include "lib/container/buffers.h"
+#include "lib/tls/buffers_tls.h"
+#include "lib/tls/tortls.h"
+#include "lib/compress/compress.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "core/proto/proto_http.h"
+#include "core/proto/proto_socks.h"
+#include "test/test.h"
 
 /** Run unit tests for buffers.c */
 static void
@@ -38,15 +44,15 @@ test_buffers_basic(void *arg)
   for (j=0;j<256;++j) {
     str[j] = (char)j;
   }
-  write_to_buf(str, 256, buf);
-  write_to_buf(str, 256, buf);
+  buf_add(buf, str, 256);
+  buf_add(buf, str, 256);
   tt_int_op(buf_datalen(buf),OP_EQ, 512);
-  fetch_from_buf(str2, 200, buf);
+  buf_get_bytes(buf, str2, 200);
   tt_mem_op(str,OP_EQ, str2, 200);
   tt_int_op(buf_datalen(buf),OP_EQ, 312);
   memset(str2, 0, sizeof(str2));
 
-  fetch_from_buf(str2, 256, buf);
+  buf_get_bytes(buf, str2, 256);
   tt_mem_op(str+200,OP_EQ, str2, 56);
   tt_mem_op(str,OP_EQ, str2+56, 200);
   tt_int_op(buf_datalen(buf),OP_EQ, 56);
@@ -54,16 +60,16 @@ test_buffers_basic(void *arg)
   /* Okay, now we should be 512 bytes into the 4096-byte buffer.  If we add
    * another 3584 bytes, we hit the end. */
   for (j=0;j<15;++j) {
-    write_to_buf(str, 256, buf);
+    buf_add(buf, str, 256);
   }
-  assert_buf_ok(buf);
+  buf_assert_ok(buf);
   tt_int_op(buf_datalen(buf),OP_EQ, 3896);
-  fetch_from_buf(str2, 56, buf);
+  buf_get_bytes(buf, str2, 56);
   tt_int_op(buf_datalen(buf),OP_EQ, 3840);
   tt_mem_op(str+200,OP_EQ, str2, 56);
   for (j=0;j<15;++j) {
     memset(str2, 0, sizeof(str2));
-    fetch_from_buf(str2, 256, buf);
+    buf_get_bytes(buf, str2, 256);
     tt_mem_op(str,OP_EQ, str2, 256);
   }
   tt_int_op(buf_datalen(buf),OP_EQ, 0);
@@ -73,38 +79,38 @@ test_buffers_basic(void *arg)
   /* Okay, now make sure growing can work. */
   buf = buf_new_with_capacity(16);
   //test_eq(buf_capacity(buf), 16);
-  write_to_buf(str+1, 255, buf);
+  buf_add(buf, str+1, 255);
   //test_eq(buf_capacity(buf), 256);
-  fetch_from_buf(str2, 254, buf);
+  buf_get_bytes(buf, str2, 254);
   tt_mem_op(str+1,OP_EQ, str2, 254);
   //test_eq(buf_capacity(buf), 256);
-  assert_buf_ok(buf);
-  write_to_buf(str, 32, buf);
+  buf_assert_ok(buf);
+  buf_add(buf, str, 32);
   //test_eq(buf_capacity(buf), 256);
-  assert_buf_ok(buf);
-  write_to_buf(str, 256, buf);
-  assert_buf_ok(buf);
+  buf_assert_ok(buf);
+  buf_add(buf, str, 256);
+  buf_assert_ok(buf);
   //test_eq(buf_capacity(buf), 512);
   tt_int_op(buf_datalen(buf),OP_EQ, 33+256);
-  fetch_from_buf(str2, 33, buf);
+  buf_get_bytes(buf, str2, 33);
   tt_int_op(*str2,OP_EQ, str[255]);
 
   tt_mem_op(str2+1,OP_EQ, str, 32);
   //test_eq(buf_capacity(buf), 512);
   tt_int_op(buf_datalen(buf),OP_EQ, 256);
-  fetch_from_buf(str2, 256, buf);
+  buf_get_bytes(buf, str2, 256);
   tt_mem_op(str,OP_EQ, str2, 256);
 
   /* now try shrinking: case 1. */
   buf_free(buf);
   buf = buf_new_with_capacity(33668);
   for (j=0;j<67;++j) {
-    write_to_buf(str,255, buf);
+    buf_add(buf, str,255);
   }
   //test_eq(buf_capacity(buf), 33668);
   tt_int_op(buf_datalen(buf),OP_EQ, 17085);
   for (j=0; j < 40; ++j) {
-    fetch_from_buf(str2, 255,buf);
+    buf_get_bytes(buf, str2, 255);
     tt_mem_op(str2,OP_EQ, str, 255);
   }
 
@@ -112,18 +118,18 @@ test_buffers_basic(void *arg)
   buf_free(buf);
   buf = buf_new_with_capacity(33668);
   for (j=0;j<67;++j) {
-    write_to_buf(str,255, buf);
+    buf_add(buf, str, 255);
   }
   for (j=0; j < 20; ++j) {
-    fetch_from_buf(str2, 255,buf);
+    buf_get_bytes(buf, str2, 255);
     tt_mem_op(str2,OP_EQ, str, 255);
   }
   for (j=0;j<80;++j) {
-    write_to_buf(str,255, buf);
+    buf_add(buf, str, 255);
   }
   //test_eq(buf_capacity(buf),33668);
   for (j=0; j < 120; ++j) {
-    fetch_from_buf(str2, 255,buf);
+    buf_get_bytes(buf, str2, 255);
     tt_mem_op(str2,OP_EQ, str, 255);
   }
 
@@ -132,27 +138,27 @@ test_buffers_basic(void *arg)
   buf = buf_new_with_capacity(4096);
   buf2 = buf_new_with_capacity(4096);
   for (j=0;j<100;++j)
-    write_to_buf(str, 255, buf);
+    buf_add(buf, str, 255);
   tt_int_op(buf_datalen(buf),OP_EQ, 25500);
   for (j=0;j<100;++j) {
     r = 10;
-    move_buf_to_buf(buf2, buf, &r);
+    buf_move_to_buf(buf2, buf, &r);
     tt_int_op(r,OP_EQ, 0);
   }
   tt_int_op(buf_datalen(buf),OP_EQ, 24500);
   tt_int_op(buf_datalen(buf2),OP_EQ, 1000);
   for (j=0;j<3;++j) {
-    fetch_from_buf(str2, 255, buf2);
+    buf_get_bytes(buf2, str2, 255);
     tt_mem_op(str2,OP_EQ, str, 255);
   }
   r = 8192; /*big move*/
-  move_buf_to_buf(buf2, buf, &r);
+  buf_move_to_buf(buf2, buf, &r);
   tt_int_op(r,OP_EQ, 0);
   r = 30000; /* incomplete move */
-  move_buf_to_buf(buf2, buf, &r);
+  buf_move_to_buf(buf2, buf, &r);
   tt_int_op(r,OP_EQ, 13692);
   for (j=0;j<97;++j) {
-    fetch_from_buf(str2, 255, buf2);
+    buf_get_bytes(buf2, str2, 255);
     tt_mem_op(str2,OP_EQ, str, 255);
   }
   buf_free(buf);
@@ -162,7 +168,7 @@ test_buffers_basic(void *arg)
   buf = buf_new_with_capacity(5);
   cp = "Testing. This is a moderately long Testing string.";
   for (j = 0; cp[j]; j++)
-    write_to_buf(cp+j, 1, buf);
+    buf_add(buf, cp+j, 1);
   tt_int_op(0,OP_EQ, buf_find_string_offset(buf, "Testing", 7));
   tt_int_op(1,OP_EQ, buf_find_string_offset(buf, "esting", 6));
   tt_int_op(1,OP_EQ, buf_find_string_offset(buf, "est", 3));
@@ -180,7 +186,7 @@ test_buffers_basic(void *arg)
   {
     char *mem = tor_malloc_zero(65536);
     buf = buf_new();
-    write_to_buf(mem, 65536, buf);
+    buf_add(buf, mem, 65536);
     tor_free(mem);
 
     tt_int_op(buf_datalen(buf), OP_EQ, 65536);
@@ -215,8 +221,7 @@ test_buffer_pullup(void *arg)
 
   /* There are a bunch of cases for pullup.  One is the trivial case. Let's
      mess around with an empty buffer. */
-  buf_pullup(buf, 16);
-  buf_get_first_chunk_data(buf, &cp, &sz);
+  buf_pullup(buf, 16, &cp, &sz);
   tt_ptr_op(cp, OP_EQ, NULL);
   tt_uint_op(sz, OP_EQ, 0);
 
@@ -227,65 +232,62 @@ test_buffer_pullup(void *arg)
 
   /* Let's add some data. */
   crypto_rand(stuff, 16384);
-  write_to_buf(stuff, 3000, buf);
-  write_to_buf(stuff+3000, 3000, buf);
-  buf_get_first_chunk_data(buf, &cp, &sz);
+  buf_add(buf, stuff, 3000);
+  buf_add(buf, stuff+3000, 3000);
+  buf_pullup(buf, 0, &cp, &sz);
   tt_ptr_op(cp, OP_NE, NULL);
   tt_int_op(sz, OP_LE, 4096);
 
   /* Make room for 3000 bytes in the first chunk, so that the pullup-move code
    * can get tested. */
-  tt_int_op(fetch_from_buf(tmp, 3000, buf), OP_EQ, 3000);
+  tt_int_op(buf_get_bytes(buf, tmp, 3000), OP_EQ, 3000);
   tt_mem_op(tmp,OP_EQ, stuff, 3000);
-  buf_pullup(buf, 2048);
-  assert_buf_ok(buf);
-  buf_get_first_chunk_data(buf, &cp, &sz);
+  buf_pullup(buf, 2048, &cp, &sz);
+  buf_assert_ok(buf);
   tt_ptr_op(cp, OP_NE, NULL);
   tt_int_op(sz, OP_GE, 2048);
   tt_mem_op(cp,OP_EQ, stuff+3000, 2048);
   tt_int_op(3000, OP_EQ, buf_datalen(buf));
-  tt_int_op(fetch_from_buf(tmp, 3000, buf), OP_EQ, 0);
+  tt_int_op(buf_get_bytes(buf, tmp, 3000), OP_EQ, 0);
   tt_mem_op(tmp,OP_EQ, stuff+3000, 2048);
 
   buf_free(buf);
 
   /* Now try the large-chunk case. */
   buf = buf_new_with_capacity(3000); /* rounds up to next power of 2. */
-  write_to_buf(stuff, 4000, buf);
-  write_to_buf(stuff+4000, 4000, buf);
-  write_to_buf(stuff+8000, 4000, buf);
-  write_to_buf(stuff+12000, 4000, buf);
+  buf_add(buf, stuff, 4000);
+  buf_add(buf, stuff+4000, 4000);
+  buf_add(buf, stuff+8000, 4000);
+  buf_add(buf, stuff+12000, 4000);
   tt_int_op(buf_datalen(buf), OP_EQ, 16000);
-  buf_get_first_chunk_data(buf, &cp, &sz);
+  buf_pullup(buf, 0, &cp, &sz);
   tt_ptr_op(cp, OP_NE, NULL);
   tt_int_op(sz, OP_LE, 4096);
 
-  buf_pullup(buf, 12500);
-  assert_buf_ok(buf);
-  buf_get_first_chunk_data(buf, &cp, &sz);
+  buf_pullup(buf, 12500, &cp, &sz);
+  buf_assert_ok(buf);
   tt_ptr_op(cp, OP_NE, NULL);
   tt_int_op(sz, OP_GE, 12500);
   tt_mem_op(cp,OP_EQ, stuff, 12500);
   tt_int_op(buf_datalen(buf), OP_EQ, 16000);
 
-  fetch_from_buf(tmp, 12400, buf);
+  buf_get_bytes(buf, tmp, 12400);
   tt_mem_op(tmp,OP_EQ, stuff, 12400);
   tt_int_op(buf_datalen(buf), OP_EQ, 3600);
-  fetch_from_buf(tmp, 3500, buf);
+  buf_get_bytes(buf, tmp, 3500);
   tt_mem_op(tmp,OP_EQ, stuff+12400, 3500);
-  fetch_from_buf(tmp, 100, buf);
+  buf_get_bytes(buf, tmp, 100);
   tt_mem_op(tmp,OP_EQ, stuff+15900, 10);
 
   buf_free(buf);
 
   /* Make sure that the pull-up-whole-buffer case works */
   buf = buf_new_with_capacity(3000); /* rounds up to next power of 2. */
-  write_to_buf(stuff, 4000, buf);
-  write_to_buf(stuff+4000, 4000, buf);
-  fetch_from_buf(tmp, 100, buf); /* dump 100 bytes from first chunk */
-  buf_pullup(buf, 16000); /* Way too much. */
-  assert_buf_ok(buf);
-  buf_get_first_chunk_data(buf, &cp, &sz);
+  buf_add(buf, stuff, 4000);
+  buf_add(buf, stuff+4000, 4000);
+  buf_get_bytes(buf, tmp, 100); /* dump 100 bytes from first chunk */
+  buf_pullup(buf, 16000, &cp, &sz);
+  buf_assert_ok(buf);
   tt_ptr_op(cp, OP_NE, NULL);
   tt_int_op(sz, OP_EQ, 7900);
   tt_mem_op(cp,OP_EQ, stuff+100, 7900);
@@ -321,23 +323,23 @@ test_buffer_copy(void *arg)
   /* Now try with a short buffer. */
   s = "And now comes an act of enormous enormance!";
   len = strlen(s);
-  write_to_buf(s, len, buf);
+  buf_add(buf, s, len);
   tt_int_op(len, OP_EQ, buf_datalen(buf));
   /* Add junk to buf2 so we can test replacing.*/
-  write_to_buf("BLARG", 5, buf2);
+  buf_add(buf2, "BLARG", 5);
   tt_int_op(0, OP_EQ, buf_set_to_copy(&buf2, buf));
   tt_int_op(len, OP_EQ, buf_datalen(buf2));
-  fetch_from_buf(b, len, buf2);
+  buf_get_bytes(buf2, b, len);
   tt_mem_op(b, OP_EQ, s, len);
   /* Now free buf2 and retry so we can test allocating */
   buf_free(buf2);
   buf2 = NULL;
   tt_int_op(0, OP_EQ, buf_set_to_copy(&buf2, buf));
   tt_int_op(len, OP_EQ, buf_datalen(buf2));
-  fetch_from_buf(b, len, buf2);
+  buf_get_bytes(buf2, b, len);
   tt_mem_op(b, OP_EQ, s, len);
   /* Clear buf for next test */
-  fetch_from_buf(b, len, buf);
+  buf_get_bytes(buf, b, len);
   tt_int_op(buf_datalen(buf),OP_EQ,0);
 
   /* Okay, now let's try a bigger buffer. */
@@ -347,13 +349,13 @@ test_buffer_copy(void *arg)
   len = strlen(s);
   for (i = 0; i < 256; ++i) {
     b[0]=i;
-    write_to_buf(b, 1, buf);
-    write_to_buf(s, len, buf);
+    buf_add(buf, b, 1);
+    buf_add(buf, s, len);
   }
   tt_int_op(0, OP_EQ, buf_set_to_copy(&buf2, buf));
   tt_int_op(buf_datalen(buf2), OP_EQ, buf_datalen(buf));
   for (i = 0; i < 256; ++i) {
-    fetch_from_buf(b, len+1, buf2);
+    buf_get_bytes(buf2, b, len+1);
     tt_int_op((unsigned char)b[0],OP_EQ,i);
     tt_mem_op(b+1, OP_EQ, s, len);
   }
@@ -366,79 +368,6 @@ test_buffer_copy(void *arg)
 }
 
 static void
-test_buffer_ext_or_cmd(void *arg)
-{
-  ext_or_cmd_t *cmd = NULL;
-  buf_t *buf = buf_new();
-  char *tmp = NULL;
-  (void) arg;
-
-  /* Empty -- should give "not there. */
-  tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
-  tt_ptr_op(NULL, OP_EQ, cmd);
-
-  /* Three bytes: shouldn't work. */
-  write_to_buf("\x00\x20\x00", 3, buf);
-  tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
-  tt_ptr_op(NULL, OP_EQ, cmd);
-  tt_int_op(3, OP_EQ, buf_datalen(buf));
-
-  /* 0020 0000: That's a nil command. It should work. */
-  write_to_buf("\x00", 1, buf);
-  tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
-  tt_ptr_op(NULL, OP_NE, cmd);
-  tt_int_op(0x20, OP_EQ, cmd->cmd);
-  tt_int_op(0, OP_EQ, cmd->len);
-  tt_int_op(0, OP_EQ, buf_datalen(buf));
-  ext_or_cmd_free(cmd);
-  cmd = NULL;
-
-  /* Now try a length-6 command with one byte missing. */
-  write_to_buf("\x10\x21\x00\x06""abcde", 9, buf);
-  tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
-  tt_ptr_op(NULL, OP_EQ, cmd);
-  write_to_buf("f", 1, buf);
-  tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
-  tt_ptr_op(NULL, OP_NE, cmd);
-  tt_int_op(0x1021, OP_EQ, cmd->cmd);
-  tt_int_op(6, OP_EQ, cmd->len);
-  tt_mem_op("abcdef", OP_EQ, cmd->body, 6);
-  tt_int_op(0, OP_EQ, buf_datalen(buf));
-  ext_or_cmd_free(cmd);
-  cmd = NULL;
-
-  /* Now try a length-10 command with 4 extra bytes. */
-  write_to_buf("\xff\xff\x00\x0aloremipsum\x10\x00\xff\xff", 18, buf);
-  tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
-  tt_ptr_op(NULL, OP_NE, cmd);
-  tt_int_op(0xffff, OP_EQ, cmd->cmd);
-  tt_int_op(10, OP_EQ, cmd->len);
-  tt_mem_op("loremipsum", OP_EQ, cmd->body, 10);
-  tt_int_op(4, OP_EQ, buf_datalen(buf));
-  ext_or_cmd_free(cmd);
-  cmd = NULL;
-
-  /* Finally, let's try a maximum-length command. We already have the header
-   * waiting. */
-  tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
-  tmp = tor_malloc_zero(65535);
-  write_to_buf(tmp, 65535, buf);
-  tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
-  tt_ptr_op(NULL, OP_NE, cmd);
-  tt_int_op(0x1000, OP_EQ, cmd->cmd);
-  tt_int_op(0xffff, OP_EQ, cmd->len);
-  tt_mem_op(tmp, OP_EQ, cmd->body, 65535);
-  tt_int_op(0, OP_EQ, buf_datalen(buf));
-  ext_or_cmd_free(cmd);
-  cmd = NULL;
-
- done:
-  ext_or_cmd_free(cmd);
-  buf_free(buf);
-  tor_free(tmp);
-}
-
-static void
 test_buffer_allocation_tracking(void *arg)
 {
   char *junk = tor_malloc(16384);
@@ -458,36 +387,36 @@ test_buffer_allocation_tracking(void *arg)
   tt_int_op(buf_allocation(buf1), OP_EQ, 0);
   tt_int_op(buf_get_total_allocation(), OP_EQ, 0);
 
-  write_to_buf(junk, 4000, buf1);
-  write_to_buf(junk, 4000, buf1);
-  write_to_buf(junk, 4000, buf1);
-  write_to_buf(junk, 4000, buf1);
+  buf_add(buf1, junk, 4000);
+  buf_add(buf1, junk, 4000);
+  buf_add(buf1, junk, 4000);
+  buf_add(buf1, junk, 4000);
   tt_int_op(buf_allocation(buf1), OP_EQ, 16384);
-  fetch_from_buf(junk, 100, buf1);
+  buf_get_bytes(buf1, junk, 100);
   tt_int_op(buf_allocation(buf1), OP_EQ, 16384); /* still 4 4k chunks */
 
   tt_int_op(buf_get_total_allocation(), OP_EQ, 16384);
 
-  fetch_from_buf(junk, 4096, buf1); /* drop a 1k chunk... */
+  buf_get_bytes(buf1, junk, 4096); /* drop a 1k chunk... */
   tt_int_op(buf_allocation(buf1), OP_EQ, 3*4096); /* now 3 4k chunks */
 
   tt_int_op(buf_get_total_allocation(), OP_EQ, 12288); /* that chunk was really
                                                        freed. */
 
-  write_to_buf(junk, 4000, buf2);
+  buf_add(buf2, junk, 4000);
   tt_int_op(buf_allocation(buf2), OP_EQ, 4096); /* another 4k chunk. */
   /*
    * We bounce back up to 16384 by allocating a new chunk.
    */
   tt_int_op(buf_get_total_allocation(), OP_EQ, 16384);
-  write_to_buf(junk, 4000, buf2);
+  buf_add(buf2, junk, 4000);
   tt_int_op(buf_allocation(buf2), OP_EQ, 8192); /* another 4k chunk. */
   tt_int_op(buf_get_total_allocation(),
             OP_EQ, 5*4096); /* that chunk was new. */
 
   /* Make a really huge buffer */
   for (i = 0; i < 1000; ++i) {
-    write_to_buf(junk, 4000, buf2);
+    buf_add(buf2, junk, 4000);
   }
   tt_int_op(buf_allocation(buf2), OP_GE, 4008000);
   tt_int_op(buf_get_total_allocation(), OP_GE, 4008000);
@@ -524,52 +453,54 @@ test_buffer_time_tracking(void *arg)
   tt_assert(buf);
 
   monotime_coarse_set_mock_time_nsec(START_NSEC);
-  const uint32_t START_MSEC = (uint32_t)monotime_coarse_absolute_msec();
+  const uint32_t START_TS = monotime_coarse_get_stamp();
 
   /* Empty buffer means the timestamp is 0. */
-  tt_int_op(0, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC));
-  tt_int_op(0, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+1000));
+  tt_int_op(0, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_TS));
+  tt_int_op(0, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_TS+1000));
 
-  write_to_buf("ABCDEFG", 7, buf);
-  tt_int_op(1000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+1000));
+  buf_add(buf, "ABCDEFG", 7);
+  tt_int_op(1000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_TS+1000));
 
   buf2 = buf_copy(buf);
   tt_assert(buf2);
   tt_int_op(1234, OP_EQ,
-            buf_get_oldest_chunk_timestamp(buf2, START_MSEC+1234));
+            buf_get_oldest_chunk_timestamp(buf2, START_TS+1234));
 
   /* Now add more bytes; enough to overflow the first chunk. */
   monotime_coarse_set_mock_time_nsec(START_NSEC + 123 * (uint64_t)1000000);
+  const uint32_t TS2 = monotime_coarse_get_stamp();
   for (i = 0; i < 600; ++i)
-    write_to_buf("ABCDEFG", 7, buf);
+    buf_add(buf, "ABCDEFG", 7);
   tt_int_op(4207, OP_EQ, buf_datalen(buf));
 
   /* The oldest bytes are still in the front. */
-  tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2000));
+  tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_TS+2000));
 
   /* Once those bytes are dropped, the chunk is still on the first
    * timestamp. */
-  fetch_from_buf(tmp, 100, buf);
-  tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2000));
+  buf_get_bytes(buf, tmp, 100);
+  tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_TS+2000));
 
   /* But once we discard the whole first chunk, we get the data in the second
    * chunk. */
-  fetch_from_buf(tmp, 4000, buf);
+  buf_get_bytes(buf, tmp, 4000);
   tt_int_op(107, OP_EQ, buf_datalen(buf));
-  tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2123));
+  tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, TS2+2000));
 
   /* This time we'll be grabbing a chunk from the freelist, and making sure
      its time gets updated */
   monotime_coarse_set_mock_time_nsec(START_NSEC + 5617 * (uint64_t)1000000);
+  const uint32_t TS3 = monotime_coarse_get_stamp();
   for (i = 0; i < 600; ++i)
-    write_to_buf("ABCDEFG", 7, buf);
+    buf_add(buf, "ABCDEFG", 7);
   tt_int_op(4307, OP_EQ, buf_datalen(buf));
 
-  tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2123));
-  fetch_from_buf(tmp, 4000, buf);
-  fetch_from_buf(tmp, 306, buf);
-  tt_int_op(0, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+5617));
-  tt_int_op(383, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+6000));
+  tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, TS2+2000));
+  buf_get_bytes(buf, tmp, 4000);
+  buf_get_bytes(buf, tmp, 306);
+  tt_int_op(0, OP_EQ, buf_get_oldest_chunk_timestamp(buf, TS3));
+  tt_int_op(383, OP_EQ, buf_get_oldest_chunk_timestamp(buf, TS3+383));
 
  done:
   buf_free(buf);
@@ -578,120 +509,150 @@ test_buffer_time_tracking(void *arg)
 }
 
 static void
-test_buffers_zlib_impl(int finalize_with_nil)
+test_buffers_compress_fin_at_chunk_end_impl(compress_method_t method,
+                                            compression_level_t level)
 {
   char *msg = NULL;
   char *contents = NULL;
   char *expanded = NULL;
   buf_t *buf = NULL;
-  tor_zlib_state_t *zlib_state = NULL;
+  tor_compress_state_t *compress_state = NULL;
   size_t out_len, in_len;
-  int done;
+  size_t sz, headerjunk;
 
   buf = buf_new_with_capacity(128); /* will round up */
-  zlib_state = tor_zlib_new(1, ZLIB_METHOD, HIGH_COMPRESSION);
+  sz = buf_get_default_chunk_size(buf);
+  msg = tor_malloc_zero(sz);
 
-  msg = tor_malloc(512);
-  crypto_rand(msg, 512);
-  tt_int_op(write_to_buf_zlib(buf, zlib_state, msg, 128, 0), OP_EQ, 0);
-  tt_int_op(write_to_buf_zlib(buf, zlib_state, msg+128, 128, 0), OP_EQ, 0);
-  tt_int_op(write_to_buf_zlib(buf, zlib_state, msg+256, 256, 0), OP_EQ, 0);
-  done = !finalize_with_nil;
-  tt_int_op(write_to_buf_zlib(buf, zlib_state, "all done", 9, done), OP_EQ, 0);
-  if (finalize_with_nil) {
-    tt_int_op(write_to_buf_zlib(buf, zlib_state, "", 0, 1), OP_EQ, 0);
-  }
+  buf_add(buf, msg, 1);
+  tt_assert(buf->head);
+
+  /* Fill up the chunk so the compression stuff won't fit in one chunk. */
+  tt_uint_op(buf->head->memlen, OP_LT, sz);
+  headerjunk = buf->head->memlen - 7;
+  buf_add(buf, msg, headerjunk-1);
+  tt_uint_op(buf->head->datalen, OP_EQ, headerjunk);
+  tt_uint_op(buf_datalen(buf), OP_EQ, headerjunk);
+  /* Write an empty string, with finalization on. */
+  compress_state = tor_compress_new(1, method, level);
+  tt_int_op(buf_add_compress(buf, compress_state, "", 0, 1), OP_EQ, 0);
 
   in_len = buf_datalen(buf);
   contents = tor_malloc(in_len);
 
-  tt_int_op(fetch_from_buf(contents, in_len, buf), OP_EQ, 0);
+  tt_int_op(buf_get_bytes(buf, contents, in_len), OP_EQ, 0);
 
-  tt_int_op(0, OP_EQ, tor_gzip_uncompress(&expanded, &out_len,
-                                       contents, in_len,
-                                       ZLIB_METHOD, 1,
-                                       LOG_WARN));
+  if (method == NO_METHOD) {
+    tt_uint_op(in_len, OP_EQ, headerjunk);
+  } else {
+    tt_uint_op(in_len, OP_GT, headerjunk);
+  }
 
-  tt_int_op(out_len, OP_GE, 128);
-  tt_mem_op(msg, OP_EQ, expanded, 128);
-  tt_int_op(out_len, OP_GE, 512);
-  tt_mem_op(msg, OP_EQ, expanded, 512);
-  tt_int_op(out_len, OP_EQ, 512+9);
-  tt_mem_op("all done", OP_EQ, expanded+512, 9);
+  tt_int_op(0, OP_EQ, tor_uncompress(&expanded, &out_len,
+                                     contents + headerjunk,
+                                     in_len - headerjunk,
+                                     method, 1,
+                                     LOG_WARN));
+
+  tt_int_op(out_len, OP_EQ, 0);
+  tt_assert(expanded);
 
  done:
   buf_free(buf);
-  tor_zlib_free(zlib_state);
+  tor_compress_free(compress_state);
   tor_free(contents);
   tor_free(expanded);
   tor_free(msg);
 }
 
 static void
-test_buffers_zlib(void *arg)
-{
-  (void) arg;
-  test_buffers_zlib_impl(0);
-}
-static void
-test_buffers_zlib_fin_with_nil(void *arg)
-{
-  (void) arg;
-  test_buffers_zlib_impl(1);
-}
-
-static void
-test_buffers_zlib_fin_at_chunk_end(void *arg)
+test_buffers_compress_impl(compress_method_t method,
+                           compression_level_t level,
+                           int finalize_with_nil)
 {
   char *msg = NULL;
   char *contents = NULL;
   char *expanded = NULL;
   buf_t *buf = NULL;
-  tor_zlib_state_t *zlib_state = NULL;
+  tor_compress_state_t *compress_state = NULL;
   size_t out_len, in_len;
-  size_t sz, headerjunk;
-  (void) arg;
+  int done;
 
   buf = buf_new_with_capacity(128); /* will round up */
-  sz = buf_get_default_chunk_size(buf);
-  msg = tor_malloc_zero(sz);
-
-  write_to_buf(msg, 1, buf);
-  tt_assert(buf->head);
+  compress_state = tor_compress_new(1, method, level);
 
-  /* Fill up the chunk so the zlib stuff won't fit in one chunk. */
-  tt_uint_op(buf->head->memlen, OP_LT, sz);
-  headerjunk = buf->head->memlen - 7;
-  write_to_buf(msg, headerjunk-1, buf);
-  tt_uint_op(buf->head->datalen, OP_EQ, headerjunk);
-  tt_uint_op(buf_datalen(buf), OP_EQ, headerjunk);
-  /* Write an empty string, with finalization on. */
-  zlib_state = tor_zlib_new(1, ZLIB_METHOD, HIGH_COMPRESSION);
-  tt_int_op(write_to_buf_zlib(buf, zlib_state, "", 0, 1), OP_EQ, 0);
+  msg = tor_malloc(512);
+  crypto_rand(msg, 512);
+  tt_int_op(buf_add_compress(buf, compress_state,
+                                  msg, 128, 0), OP_EQ, 0);
+  tt_int_op(buf_add_compress(buf, compress_state,
+                                  msg+128, 128, 0), OP_EQ, 0);
+  tt_int_op(buf_add_compress(buf, compress_state,
+                                  msg+256, 256, 0), OP_EQ, 0);
+  done = !finalize_with_nil;
+  tt_int_op(buf_add_compress(buf, compress_state,
+                                  "all done", 9, done), OP_EQ, 0);
+  if (finalize_with_nil) {
+    tt_int_op(buf_add_compress(buf, compress_state, "", 0, 1), OP_EQ, 0);
+  }
 
   in_len = buf_datalen(buf);
   contents = tor_malloc(in_len);
 
-  tt_int_op(fetch_from_buf(contents, in_len, buf), OP_EQ, 0);
-
-  tt_uint_op(in_len, OP_GT, headerjunk);
+  tt_int_op(buf_get_bytes(buf, contents, in_len), OP_EQ, 0);
 
-  tt_int_op(0, OP_EQ, tor_gzip_uncompress(&expanded, &out_len,
-                                  contents + headerjunk, in_len - headerjunk,
-                                  ZLIB_METHOD, 1,
-                                  LOG_WARN));
+  tt_int_op(0, OP_EQ, tor_uncompress(&expanded, &out_len,
+                                     contents, in_len,
+                                     method, 1,
+                                     LOG_WARN));
 
-  tt_int_op(out_len, OP_EQ, 0);
-  tt_assert(expanded);
+  tt_int_op(out_len, OP_GE, 128);
+  tt_mem_op(msg, OP_EQ, expanded, 128);
+  tt_int_op(out_len, OP_GE, 512);
+  tt_mem_op(msg, OP_EQ, expanded, 512);
+  tt_int_op(out_len, OP_EQ, 512+9);
+  tt_mem_op("all done", OP_EQ, expanded+512, 9);
 
  done:
   buf_free(buf);
-  tor_zlib_free(zlib_state);
+  tor_compress_free(compress_state);
   tor_free(contents);
   tor_free(expanded);
   tor_free(msg);
 }
 
+static void
+test_buffers_compress(void *arg)
+{
+  const char *methodname = arg;
+  tt_assert(methodname);
+
+  compress_method_t method = compression_method_get_by_name(methodname);
+  tt_int_op(method, OP_NE, UNKNOWN_METHOD);
+
+  if (! tor_compress_supports_method(method)) {
+    tt_skip();
+  }
+
+  compression_level_t levels[] = {
+    BEST_COMPRESSION,
+    HIGH_COMPRESSION,
+    MEDIUM_COMPRESSION,
+    LOW_COMPRESSION
+  };
+
+  for (unsigned l = 0; l < ARRAY_LENGTH(levels); ++l) {
+    compression_level_t level = levels[l];
+
+    test_buffers_compress_impl(method, level, 0);
+    test_buffers_compress_impl(method, level, 1);
+    test_buffers_compress_fin_at_chunk_end_impl(method, level);
+  }
+
+ done:
+  ;
+}
+
 static const uint8_t *tls_read_ptr;
 static int n_remaining;
 static int next_reply_val[16];
@@ -732,11 +693,11 @@ test_buffers_tls_read_mocked(void *arg)
   buf = buf_new();
 
   next_reply_val[0] = 1024;
-  tt_int_op(128, ==, read_to_buf_tls(NULL, 128, buf));
+  tt_int_op(128, OP_EQ, buf_read_from_tls(buf, NULL, 128));
 
   next_reply_val[0] = 5000;
   next_reply_val[1] = 5000;
-  tt_int_op(6000, ==, read_to_buf_tls(NULL, 6000, buf));
+  tt_int_op(6000, OP_EQ, buf_read_from_tls(buf, NULL, 6000));
 
  done:
   UNMOCK(tor_tls_read);
@@ -750,36 +711,113 @@ test_buffers_chunk_size(void *arg)
   (void)arg;
   const int min = 256;
   const int max = 65536;
-  tt_uint_op(preferred_chunk_size(3), OP_EQ, min);
-  tt_uint_op(preferred_chunk_size(25), OP_EQ, min);
-  tt_uint_op(preferred_chunk_size(0), OP_EQ, min);
-  tt_uint_op(preferred_chunk_size(256), OP_EQ, 512);
-  tt_uint_op(preferred_chunk_size(65400), OP_EQ, max);
+  tt_uint_op(buf_preferred_chunk_size(3), OP_EQ, min);
+  tt_uint_op(buf_preferred_chunk_size(25), OP_EQ, min);
+  tt_uint_op(buf_preferred_chunk_size(0), OP_EQ, min);
+  tt_uint_op(buf_preferred_chunk_size(256), OP_EQ, 512);
+  tt_uint_op(buf_preferred_chunk_size(65400), OP_EQ, max);
   /* Here, we're implicitly saying that the chunk header overhead is
    * between 1 and 100 bytes. 24..48 would probably be more accurate. */
-  tt_uint_op(preferred_chunk_size(65536), OP_GT, 65536);
-  tt_uint_op(preferred_chunk_size(65536), OP_LT, 65536+100);
-  tt_uint_op(preferred_chunk_size(165536), OP_GT, 165536);
-  tt_uint_op(preferred_chunk_size(165536), OP_LT, 165536+100);
+  tt_uint_op(buf_preferred_chunk_size(65536), OP_GT, 65536);
+  tt_uint_op(buf_preferred_chunk_size(65536), OP_LT, 65536+100);
+  tt_uint_op(buf_preferred_chunk_size(165536), OP_GT, 165536);
+  tt_uint_op(buf_preferred_chunk_size(165536), OP_LT, 165536+100);
+ done:
+  ;
+}
+
+static void
+test_buffers_find_contentlen(void *arg)
+{
+  static const struct {
+    const char *headers;
+    int r;
+    int contentlen;
+  } results[] = {
+    { "Blah blah\r\nContent-Length: 1\r\n\r\n", 1, 1 },
+    { "Blah blah\r\n\r\n", 0, 0 }, /* no content-len */
+    { "Blah blah Content-Length: 1\r\n", 0, 0 }, /* no content-len. */
+    { "Blah blah\r\nContent-Length: 100000\r\n", 1, 100000},
+    { "Blah blah\r\nContent-Length: 1000000000000000000000000\r\n", -1, 0},
+    { "Blah blah\r\nContent-Length: 0\r\n", 1, 0},
+    { "Blah blah\r\nContent-Length: -1\r\n", -1, 0},
+    { "Blah blah\r\nContent-Length: 1x\r\n", -1, 0},
+    { "Blah blah\r\nContent-Length: 1 x\r\n", -1, 0},
+    { "Blah blah\r\nContent-Length: 1 \r\n", 1, 1},
+    { "Blah blah\r\nContent-Length:  \r\n", -1, 0},
+    { "Blah blah\r\nContent-Length: ", -1, 0},
+    { "Blah blah\r\nContent-Length: 5050", -1, 0},
+    { NULL, 0, 0 }
+  };
+  int i;
+
+  (void)arg;
+
+  for (i = 0; results[i].headers; ++i) {
+    int r;
+    size_t sz;
+    size_t headerlen = strlen(results[i].headers);
+    char * tmp = tor_memdup(results[i].headers, headerlen);/* ensure no eos */
+    sz = 999; /* to ensure it gets set */
+    r = buf_http_find_content_length(tmp, headerlen, &sz);
+    tor_free(tmp);
+    log_debug(LD_DIR, "%d: %s", i, escaped(results[i].headers));
+    tt_int_op(r, OP_EQ, results[i].r);
+    tt_int_op(sz, OP_EQ, results[i].contentlen);
+  }
  done:
   ;
 }
 
+static void
+test_buffer_peek_startswith(void *arg)
+{
+  (void)arg;
+  buf_t *buf;
+  buf = buf_new();
+  tt_ptr_op(buf, OP_NE, NULL);
+
+  tt_assert(buf_peek_startswith(buf, ""));
+  tt_assert(! buf_peek_startswith(buf, "X"));
+
+  buf_add(buf, "Tor", 3);
+
+  tt_assert(buf_peek_startswith(buf, ""));
+  tt_assert(buf_peek_startswith(buf, "T"));
+  tt_assert(buf_peek_startswith(buf, "To"));
+  tt_assert(buf_peek_startswith(buf, "Tor"));
+  tt_assert(! buf_peek_startswith(buf, "Top"));
+  tt_assert(! buf_peek_startswith(buf, "For"));
+  tt_assert(! buf_peek_startswith(buf, "Tork"));
+  tt_assert(! buf_peek_startswith(buf, "Torpor"));
+
+ done:
+  buf_free(buf);
+}
+
 struct testcase_t buffer_tests[] = {
   { "basic", test_buffers_basic, TT_FORK, NULL, NULL },
   { "copy", test_buffer_copy, TT_FORK, NULL, NULL },
   { "pullup", test_buffer_pullup, TT_FORK, NULL, NULL },
-  { "ext_or_cmd", test_buffer_ext_or_cmd, TT_FORK, NULL, NULL },
+  { "startswith", test_buffer_peek_startswith, 0, NULL, NULL },
   { "allocation_tracking", test_buffer_allocation_tracking, TT_FORK,
     NULL, NULL },
   { "time_tracking", test_buffer_time_tracking, TT_FORK, NULL, NULL },
-  { "zlib", test_buffers_zlib, TT_FORK, NULL, NULL },
-  { "zlib_fin_with_nil", test_buffers_zlib_fin_with_nil, TT_FORK, NULL, NULL },
-  { "zlib_fin_at_chunk_end", test_buffers_zlib_fin_at_chunk_end, TT_FORK,
-    NULL, NULL},
   { "tls_read_mocked", test_buffers_tls_read_mocked, 0,
     NULL, NULL },
   { "chunk_size", test_buffers_chunk_size, 0, NULL, NULL },
+  { "find_contentlen", test_buffers_find_contentlen, 0, NULL, NULL },
+
+  { "compress/zlib", test_buffers_compress, TT_FORK,
+    &passthrough_setup, (char*)"deflate" },
+  { "compress/gzip", test_buffers_compress, TT_FORK,
+    &passthrough_setup, (char*)"gzip" },
+  { "compress/zstd", test_buffers_compress, TT_FORK,
+    &passthrough_setup, (char*)"x-zstd" },
+  { "compress/lzma", test_buffers_compress, TT_FORK,
+    &passthrough_setup, (char*)"x-tor-lzma" },
+  { "compress/none", test_buffers_compress, TT_FORK,
+    &passthrough_setup, (char*)"identity" },
+
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_bwmgt.c b/src/test/test_bwmgt.c
new file mode 100644
index 0000000000..5a013aa268
--- /dev/null
+++ b/src/test/test_bwmgt.c
@@ -0,0 +1,233 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_bwmgt.c
+ * \brief tests for bandwidth management / token bucket functions
+ */
+
+#define TOKEN_BUCKET_PRIVATE
+
+#include "core/or/or.h"
+#include "test/test.h"
+
+#include "lib/evloop/token_bucket.h"
+
+// an imaginary time, in timestamp units. Chosen so it will roll over.
+static const uint32_t START_TS = UINT32_MAX-10;
+static const int32_t KB = 1024;
+static const uint32_t GB = (UINT64_C(1) << 30);
+
+static void
+test_bwmgt_token_buf_init(void *arg)
+{
+  (void)arg;
+  token_bucket_rw_t b;
+
+  token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS);
+  // Burst is correct
+  tt_uint_op(b.cfg.burst, OP_EQ, 64*KB);
+  // Rate is correct, within 1 percent.
+  {
+    uint32_t ticks_per_sec =
+      (uint32_t) monotime_msec_to_approx_coarse_stamp_units(1000);
+    uint32_t rate_per_sec = (b.cfg.rate * ticks_per_sec / TICKS_PER_STEP);
+
+    tt_uint_op(rate_per_sec, OP_GT, 16*KB-160);
+    tt_uint_op(rate_per_sec, OP_LT, 16*KB+160);
+  }
+  // Bucket starts out full:
+  tt_uint_op(b.last_refilled_at_timestamp, OP_EQ, START_TS);
+  tt_int_op(b.read_bucket.bucket, OP_EQ, 64*KB);
+
+ done:
+  ;
+}
+
+static void
+test_bwmgt_token_buf_adjust(void *arg)
+{
+  (void)arg;
+  token_bucket_rw_t b;
+
+  token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS);
+
+  uint32_t rate_orig = b.cfg.rate;
+  // Increasing burst
+  token_bucket_rw_adjust(&b, 16*KB, 128*KB);
+  tt_uint_op(b.cfg.rate, OP_EQ, rate_orig);
+  tt_uint_op(b.read_bucket.bucket, OP_EQ, 64*KB);
+  tt_uint_op(b.cfg.burst, OP_EQ, 128*KB);
+
+  // Decreasing burst but staying above bucket
+  token_bucket_rw_adjust(&b, 16*KB, 96*KB);
+  tt_uint_op(b.cfg.rate, OP_EQ, rate_orig);
+  tt_uint_op(b.read_bucket.bucket, OP_EQ, 64*KB);
+  tt_uint_op(b.cfg.burst, OP_EQ, 96*KB);
+
+  // Decreasing burst below bucket,
+  token_bucket_rw_adjust(&b, 16*KB, 48*KB);
+  tt_uint_op(b.cfg.rate, OP_EQ, rate_orig);
+  tt_uint_op(b.read_bucket.bucket, OP_EQ, 48*KB);
+  tt_uint_op(b.cfg.burst, OP_EQ, 48*KB);
+
+  // Changing rate.
+  token_bucket_rw_adjust(&b, 32*KB, 48*KB);
+  tt_uint_op(b.cfg.rate, OP_GE, rate_orig*2 - 10);
+  tt_uint_op(b.cfg.rate, OP_LE, rate_orig*2 + 10);
+  tt_uint_op(b.read_bucket.bucket, OP_EQ, 48*KB);
+  tt_uint_op(b.cfg.burst, OP_EQ, 48*KB);
+
+ done:
+  ;
+}
+
+static void
+test_bwmgt_token_buf_dec(void *arg)
+{
+  (void)arg;
+  token_bucket_rw_t b;
+  token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS);
+
+  // full-to-not-full.
+  tt_int_op(0, OP_EQ, token_bucket_rw_dec_read(&b, KB));
+  tt_int_op(b.read_bucket.bucket, OP_EQ, 63*KB);
+
+  // Full to almost-not-full
+  tt_int_op(0, OP_EQ, token_bucket_rw_dec_read(&b, 63*KB - 1));
+  tt_int_op(b.read_bucket.bucket, OP_EQ, 1);
+
+  // almost-not-full to empty.
+  tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, 1));
+  tt_int_op(b.read_bucket.bucket, OP_EQ, 0);
+
+  // reset bucket, try full-to-empty
+  token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS);
+  tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, 64*KB));
+  tt_int_op(b.read_bucket.bucket, OP_EQ, 0);
+
+  // reset bucket, try underflow.
+  token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS);
+  tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, 64*KB + 1));
+  tt_int_op(b.read_bucket.bucket, OP_EQ, -1);
+
+  // A second underflow does not make the bucket empty.
+  tt_int_op(0, OP_EQ, token_bucket_rw_dec_read(&b, 1000));
+  tt_int_op(b.read_bucket.bucket, OP_EQ, -1001);
+
+ done:
+  ;
+}
+
+static void
+test_bwmgt_token_buf_refill(void *arg)
+{
+  (void)arg;
+  token_bucket_rw_t b;
+  const uint32_t BW_SEC =
+    (uint32_t)monotime_msec_to_approx_coarse_stamp_units(1000);
+  token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS);
+
+  /* Make the buffer much emptier, then let one second elapse. */
+  token_bucket_rw_dec_read(&b, 48*KB);
+  tt_int_op(b.read_bucket.bucket, OP_EQ, 16*KB);
+  tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC));
+  tt_int_op(b.read_bucket.bucket, OP_GT, 32*KB - 300);
+  tt_int_op(b.read_bucket.bucket, OP_LT, 32*KB + 300);
+
+  /* Another half second. */
+  tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC*3/2));
+  tt_int_op(b.read_bucket.bucket, OP_GT, 40*KB - 400);
+  tt_int_op(b.read_bucket.bucket, OP_LT, 40*KB + 400);
+  tt_uint_op(b.last_refilled_at_timestamp, OP_EQ, START_TS + BW_SEC*3/2);
+
+  /* No time: nothing happens. */
+  {
+    const uint32_t bucket_orig = b.read_bucket.bucket;
+    tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC*3/2));
+    tt_int_op(b.read_bucket.bucket, OP_EQ, bucket_orig);
+  }
+
+  /* Another 30 seconds: fill the bucket. */
+  tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b,
+                                     START_TS + BW_SEC*3/2 + BW_SEC*30));
+  tt_int_op(b.read_bucket.bucket, OP_EQ, b.cfg.burst);
+  tt_uint_op(b.last_refilled_at_timestamp, OP_EQ,
+             START_TS + BW_SEC*3/2 + BW_SEC*30);
+
+  /* Another 30 seconds: nothing happens. */
+  tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b,
+                                     START_TS + BW_SEC*3/2 + BW_SEC*60));
+  tt_int_op(b.read_bucket.bucket, OP_EQ, b.cfg.burst);
+  tt_uint_op(b.last_refilled_at_timestamp, OP_EQ,
+             START_TS + BW_SEC*3/2 + BW_SEC*60);
+
+  /* Empty the bucket, let two seconds pass, and make sure that a refill is
+   * noticed. */
+  tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, b.cfg.burst));
+  tt_int_op(0, OP_EQ, b.read_bucket.bucket);
+  tt_int_op(1, OP_EQ, token_bucket_rw_refill(&b,
+                                     START_TS + BW_SEC*3/2 + BW_SEC*61));
+  tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b,
+                                     START_TS + BW_SEC*3/2 + BW_SEC*62));
+  tt_int_op(b.read_bucket.bucket, OP_GT, 32*KB-400);
+  tt_int_op(b.read_bucket.bucket, OP_LT, 32*KB+400);
+
+  /* Underflow the bucket, make sure we detect when it has tokens again. */
+  tt_int_op(1, OP_EQ,
+            token_bucket_rw_dec_read(&b, b.read_bucket.bucket+16*KB));
+  tt_int_op(-16*KB, OP_EQ, b.read_bucket.bucket);
+  // half a second passes...
+  tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC*64));
+  tt_int_op(b.read_bucket.bucket, OP_GT, -8*KB-300);
+  tt_int_op(b.read_bucket.bucket, OP_LT, -8*KB+300);
+  // a second passes
+  tt_int_op(1, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC*65));
+  tt_int_op(b.read_bucket.bucket, OP_GT, 8*KB-400);
+  tt_int_op(b.read_bucket.bucket, OP_LT, 8*KB+400);
+
+  // We step a second backwards, and nothing happens.
+  tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC*64));
+  tt_int_op(b.read_bucket.bucket, OP_GT, 8*KB-400);
+  tt_int_op(b.read_bucket.bucket, OP_LT, 8*KB+400);
+
+  // A ridiculous amount of time passes.
+  tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, INT32_MAX));
+  tt_int_op(b.read_bucket.bucket, OP_EQ, b.cfg.burst);
+
+ done:
+  ;
+}
+
+/* Test some helper functions we use within the token bucket interface. */
+static void
+test_bwmgt_token_buf_helpers(void *arg)
+{
+  uint32_t ret;
+
+  (void) arg;
+
+  /* The returned value will be OS specific but in any case, it should be
+   * greater than 1 since we are passing 1GB/sec rate. */
+  ret = rate_per_sec_to_rate_per_step(1 * GB);
+  tt_u64_op(ret, OP_GT, 1);
+
+  /* We default to 1 in case rate is 0. */
+  ret = rate_per_sec_to_rate_per_step(0);
+  tt_u64_op(ret, OP_EQ, 1);
+
+ done:
+  ;
+}
+
+#define BWMGT(name)                                          \
+  { #name, test_bwmgt_ ## name , 0, NULL, NULL }
+
+struct testcase_t bwmgt_tests[] = {
+  BWMGT(token_buf_init),
+  BWMGT(token_buf_adjust),
+  BWMGT(token_buf_dec),
+  BWMGT(token_buf_refill),
+  BWMGT(token_buf_helpers),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_cell_formats.c b/src/test/test_cell_formats.c
index f429f4291d..fc5367557d 100644
--- a/src/test/test_cell_formats.c
+++ b/src/test/test_cell_formats.c
@@ -1,22 +1,29 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 
 #define CONNECTION_EDGE_PRIVATE
 #define RELAY_PRIVATE
-#include "or.h"
-#include "channel.h"
-#include "connection_edge.h"
-#include "connection_or.h"
-#include "onion.h"
-#include "onion_tap.h"
-#include "onion_fast.h"
-#include "onion_ntor.h"
-#include "relay.h"
-#include "test.h"
+#include "core/or/or.h"
+#include "core/or/channel.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "app/config/config.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "core/or/onion.h"
+#include "core/crypto/onion_tap.h"
+#include "core/crypto/onion_fast.h"
+#include "core/crypto/onion_ntor.h"
+#include "core/or/relay.h"
+
+#include "core/or/cell_st.h"
+#include "core/or/cell_queue_st.h"
+#include "core/or/var_cell_st.h"
+
+#include "test/test.h"
 
 #include <stdlib.h>
 #include <string.h>
@@ -476,7 +483,7 @@ test_cfmt_create_cells(void *arg)
   cell.command = CELL_CREATED;
   tt_int_op(-1, OP_EQ, create_cell_parse(&cc, &cell));
 
-  /* You can't acutally make an unparseable CREATE or CREATE_FAST cell. */
+  /* You can't actually make an unparseable CREATE or CREATE_FAST cell. */
 
   /* Try some CREATE2 cells.  First with a bad type. */
   cell.command = CELL_CREATE2;
@@ -698,6 +705,7 @@ test_cfmt_extend_cells(void *arg)
   tt_int_op(61681, OP_EQ, ec.orport_ipv4.port);
   tt_str_op("2002::f0:c51e", OP_EQ, fmt_addr(&ec.orport_ipv6.addr));
   tt_int_op(4370, OP_EQ, ec.orport_ipv6.port);
+  tt_assert(ed25519_public_key_is_zero(&ec.ed_pubkey));
   tt_mem_op(ec.node_id,OP_EQ, "anthropomorphization", 20);
   tt_int_op(cc->cell_type, OP_EQ, CELL_CREATE2);
   tt_int_op(cc->handshake_type, OP_EQ, 0x105);
@@ -717,6 +725,37 @@ test_cfmt_extend_cells(void *arg)
   tt_mem_op(p2+1+8+22+4,OP_EQ, b, 99+20);
   tt_int_op(0, OP_EQ, create_cell_format_relayed(&cell, cc));
 
+  /* Now let's add an ed25519 key to that extend2 cell. */
+  memcpy(ec.ed_pubkey.pubkey,
+         "brownshoesdontmakeit/brownshoesd", 32);
+
+  /* As before, since we aren't extending by ed25519. */
+  get_options_mutable()->ExtendByEd25519ID = 0;
+  tt_int_op(0, OP_EQ, extend_cell_format(&p2_cmd, &p2_len, p2, &ec));
+  tt_int_op(p2_len, OP_EQ, 89+99-34-20);
+  test_memeq_hex(p2,
+                 "02000612F40001F0F1"
+                 "0214616e7468726f706f6d6f727068697a6174696f6e"
+                 "01050063");
+
+  /* Now try with the ed25519 ID. */
+  get_options_mutable()->ExtendByEd25519ID = 1;
+  tt_int_op(0, OP_EQ, extend_cell_format(&p2_cmd, &p2_len, p2, &ec));
+  tt_int_op(p2_len, OP_EQ, 89+99-34-20 + 34);
+  test_memeq_hex(p2,
+                 "03000612F40001F0F1"
+                 "0214616e7468726f706f6d6f727068697a6174696f6e"
+                 // ed digest follows:
+                 "0320" "62726f776e73686f6573646f6e746d616b656"
+                        "9742f62726f776e73686f657364"
+                 "01050063");
+  /* Can we parse that? Did the key come through right? */
+  memset(&ec, 0, sizeof(ec));
+  tt_int_op(0, OP_EQ, extend_cell_parse(&ec, RELAY_COMMAND_EXTEND2,
+                                        p2, p2_len));
+  tt_mem_op("brownshoesdontmakeit/brownshoesd", OP_EQ,
+            ec.ed_pubkey.pubkey, 32);
+
   /* == Now try parsing some junk */
 
   /* Try a too-long handshake */
@@ -1257,10 +1296,9 @@ struct testcase_t cell_format_tests[] = {
   TEST(connected_cells, 0),
   TEST(create_cells, 0),
   TEST(created_cells, 0),
-  TEST(extend_cells, 0),
+  TEST(extend_cells, TT_FORK),
   TEST(extended_cells, 0),
   TEST(resolved_cells, 0),
   TEST(is_destroy, 0),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_cell_queue.c b/src/test/test_cell_queue.c
index 93ac9854d8..8fc1da031e 100644
--- a/src/test/test_cell_queue.c
+++ b/src/test/test_cell_queue.c
@@ -1,12 +1,17 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CIRCUITLIST_PRIVATE
 #define RELAY_PRIVATE
-#include "or.h"
-#include "circuitlist.h"
-#include "relay.h"
-#include "test.h"
+#include "core/or/or.h"
+#include "core/or/circuitlist.h"
+#include "core/or/relay.h"
+#include "test/test.h"
+
+#include "core/or/cell_st.h"
+#include "core/or/cell_queue_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
 
 static void
 test_cq_manip(void *arg)
@@ -130,8 +135,8 @@ test_circuit_n_cells(void *arg)
   tt_int_op(n_cells_in_circ_queues(TO_CIRCUIT(origin_c)), OP_EQ, 2);
 
  done:
-  circuit_free(TO_CIRCUIT(or_c));
-  circuit_free(TO_CIRCUIT(origin_c));
+  circuit_free_(TO_CIRCUIT(or_c));
+  circuit_free_(TO_CIRCUIT(origin_c));
 }
 
 struct testcase_t cell_queue_tests[] = {
diff --git a/src/test/test_channel.c b/src/test/test_channel.c
index a9e0634d9e..e55b9b0750 100644
--- a/src/test/test_channel.c
+++ b/src/test/test_channel.c
@@ -1,24 +1,35 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define TOR_CHANNEL_INTERNAL_
 #define CHANNEL_PRIVATE_
-#include "or.h"
-#include "channel.h"
+#include "core/or/or.h"
+#include "core/or/channel.h"
 /* For channel_note_destroy_not_pending */
-#include "circuitlist.h"
-#include "circuitmux.h"
+#define CIRCUITLIST_PRIVATE
+#include "core/or/circuitlist.h"
+#include "core/or/circuitmux.h"
+#include "core/or/circuitmux_ewma.h"
 /* For var_cell_free */
-#include "connection_or.h"
+#include "core/or/connection_or.h"
+#include "lib/crypt_ops/crypto_rand.h"
 /* For packed_cell stuff */
 #define RELAY_PRIVATE
-#include "relay.h"
+#include "core/or/relay.h"
 /* For init/free stuff */
-#include "scheduler.h"
+#include "core/or/scheduler.h"
+#include "feature/nodelist/networkstatus.h"
+
+#include "core/or/cell_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+#include "core/or/var_cell_st.h"
 
 /* Test suite stuff */
-#include "test.h"
-#include "fakechans.h"
+#include "test/log_test_helpers.h"
+#include "test/test.h"
+#include "test/fakechans.h"
 
 static int test_chan_accept_cells = 0;
 static int test_chan_fixed_cells_recved = 0;
@@ -26,67 +37,23 @@ static cell_t * test_chan_last_seen_fixed_cell_ptr = NULL;
 static int test_chan_var_cells_recved = 0;
 static var_cell_t * test_chan_last_seen_var_cell_ptr = NULL;
 static int test_cells_written = 0;
-static int test_destroy_not_pending_calls = 0;
 static int test_doesnt_want_writes_count = 0;
 static int test_dumpstats_calls = 0;
 static int test_has_waiting_cells_count = 0;
-static double test_overhead_estimate = 1.0;
 static int test_releases_count = 0;
-static circuitmux_t *test_target_cmux = NULL;
-static unsigned int test_cmux_cells = 0;
 static channel_t *dump_statistics_mock_target = NULL;
 static int dump_statistics_mock_matches = 0;
-
-static void chan_test_channel_dump_statistics_mock(
-    channel_t *chan, int severity);
-static int chan_test_channel_flush_from_first_active_circuit_mock(
-    channel_t *chan, int max);
-static unsigned int chan_test_circuitmux_num_cells_mock(circuitmux_t *cmux);
-static void channel_note_destroy_not_pending_mock(channel_t *ch,
-                                                  circid_t circid);
-static void chan_test_cell_handler(channel_t *ch,
-                                   cell_t *cell);
-static const char * chan_test_describe_transport(channel_t *ch);
-static void chan_test_dumpstats(channel_t *ch, int severity);
-static void chan_test_var_cell_handler(channel_t *ch,
-                                       var_cell_t *var_cell);
-static void chan_test_close(channel_t *ch);
-static void chan_test_error(channel_t *ch);
-static void chan_test_finish_close(channel_t *ch);
-static const char * chan_test_get_remote_descr(channel_t *ch, int flags);
-static int chan_test_is_canonical(channel_t *ch, int req);
-static size_t chan_test_num_bytes_queued(channel_t *ch);
-static int chan_test_num_cells_writeable(channel_t *ch);
-static int chan_test_write_cell(channel_t *ch, cell_t *cell);
-static int chan_test_write_packed_cell(channel_t *ch,
-                                       packed_cell_t *packed_cell);
-static int chan_test_write_var_cell(channel_t *ch, var_cell_t *var_cell);
-static void scheduler_channel_doesnt_want_writes_mock(channel_t *ch);
-
-static void test_channel_dumpstats(void *arg);
-static void test_channel_flush(void *arg);
-static void test_channel_flushmux(void *arg);
-static void test_channel_incoming(void *arg);
-static void test_channel_lifecycle(void *arg);
-static void test_channel_multi(void *arg);
-static void test_channel_queue_incoming(void *arg);
-static void test_channel_queue_size(void *arg);
-static void test_channel_write(void *arg);
-
-static void
-channel_note_destroy_not_pending_mock(channel_t *ch,
-                                      circid_t circid)
-{
-  (void)ch;
-  (void)circid;
-
-  ++test_destroy_not_pending_calls;
-}
+static int test_close_called = 0;
+static int test_chan_should_be_canonical = 0;
+static int test_chan_should_match_target = 0;
+static int test_chan_canonical_should_be_reliable = 0;
+static int test_chan_listener_close_fn_called = 0;
+static int test_chan_listener_fn_called = 0;
 
 static const char *
 chan_test_describe_transport(channel_t *ch)
 {
-  tt_assert(ch != NULL);
+  tt_ptr_op(ch, OP_NE, NULL);
 
  done:
   return "Fake channel for unit tests";
@@ -100,7 +67,7 @@ chan_test_describe_transport(channel_t *ch)
 static void
 chan_test_channel_dump_statistics_mock(channel_t *chan, int severity)
 {
-  tt_assert(chan != NULL);
+  tt_ptr_op(chan, OP_NE, NULL);
 
   (void)severity;
 
@@ -112,71 +79,14 @@ chan_test_channel_dump_statistics_mock(channel_t *chan, int severity)
   return;
 }
 
-/**
- * If the target cmux is the cmux for chan, make fake cells up to the
- * target number of cells and write them to chan.  Otherwise, invoke
- * the real channel_flush_from_first_active_circuit().
- */
-
-static int
-chan_test_channel_flush_from_first_active_circuit_mock(channel_t *chan,
-                                                       int max)
-{
-  int result = 0, c = 0;
-  packed_cell_t *cell = NULL;
-
-  tt_assert(chan != NULL);
-  if (test_target_cmux != NULL &&
-      test_target_cmux == chan->cmux) {
-    while (c <= max && test_cmux_cells > 0) {
-      cell = packed_cell_new();
-      channel_write_packed_cell(chan, cell);
-      ++c;
-      --test_cmux_cells;
-    }
-    result = c;
-  } else {
-    result = channel_flush_from_first_active_circuit__real(chan, max);
-  }
-
- done:
-  return result;
-}
-
-/**
- * If we have a target cmux set and this matches it, lie about how
- * many cells we have according to the number indicated; otherwise
- * pass to the real circuitmux_num_cells().
- */
-
-static unsigned int
-chan_test_circuitmux_num_cells_mock(circuitmux_t *cmux)
-{
-  unsigned int result = 0;
-
-  tt_assert(cmux != NULL);
-  if (cmux != NULL) {
-    if (cmux == test_target_cmux) {
-      result = test_cmux_cells;
-    } else {
-      result = circuitmux_num_cells__real(cmux);
-    }
-  }
-
- done:
-
-  return result;
-}
-
 /*
  * Handle an incoming fixed-size cell for unit tests
  */
 
 static void
-chan_test_cell_handler(channel_t *ch,
-                       cell_t *cell)
+chan_test_cell_handler(channel_t *chan, cell_t *cell)
 {
-  tt_assert(ch);
+  tt_assert(chan);
   tt_assert(cell);
 
   test_chan_last_seen_fixed_cell_ptr = cell;
@@ -193,7 +103,7 @@ chan_test_cell_handler(channel_t *ch,
 static void
 chan_test_dumpstats(channel_t *ch, int severity)
 {
-  tt_assert(ch != NULL);
+  tt_ptr_op(ch, OP_NE, NULL);
 
   (void)severity;
 
@@ -226,6 +136,8 @@ chan_test_close(channel_t *ch)
 {
   tt_assert(ch);
 
+  ++test_close_called;
+
  done:
   return;
 }
@@ -268,41 +180,12 @@ static const char *
 chan_test_get_remote_descr(channel_t *ch, int flags)
 {
   tt_assert(ch);
-  tt_int_op(flags & ~(GRD_FLAG_ORIGINAL | GRD_FLAG_ADDR_ONLY), ==, 0);
+  tt_int_op(flags & ~(GRD_FLAG_ORIGINAL | GRD_FLAG_ADDR_ONLY), OP_EQ, 0);
 
  done:
   return "Fake channel for unit tests; no real endpoint";
 }
 
-static double
-chan_test_get_overhead_estimate(channel_t *ch)
-{
-  tt_assert(ch);
-
- done:
-  return test_overhead_estimate;
-}
-
-static int
-chan_test_is_canonical(channel_t *ch, int req)
-{
-  tt_assert(ch != NULL);
-  tt_assert(req == 0 || req == 1);
-
- done:
-  /* Fake channels are always canonical */
-  return 1;
-}
-
-static size_t
-chan_test_num_bytes_queued(channel_t *ch)
-{
-  tt_assert(ch);
-
- done:
-  return 0;
-}
-
 static int
 chan_test_num_cells_writeable(channel_t *ch)
 {
@@ -313,26 +196,6 @@ chan_test_num_cells_writeable(channel_t *ch)
 }
 
 static int
-chan_test_write_cell(channel_t *ch, cell_t *cell)
-{
-  int rv = 0;
-
-  tt_assert(ch);
-  tt_assert(cell);
-
-  if (test_chan_accept_cells) {
-    /* Free the cell and bump the counter */
-    tor_free(cell);
-    ++test_cells_written;
-    rv = 1;
-  }
-  /* else return 0, we didn't accept it */
-
- done:
-  return rv;
-}
-
-static int
 chan_test_write_packed_cell(channel_t *ch,
                             packed_cell_t *packed_cell)
 {
@@ -343,7 +206,6 @@ chan_test_write_packed_cell(channel_t *ch,
 
   if (test_chan_accept_cells) {
     /* Free the cell and bump the counter */
-    packed_cell_free(packed_cell);
     ++test_cells_written;
     rv = 1;
   }
@@ -380,7 +242,7 @@ chan_test_write_var_cell(channel_t *ch, var_cell_t *var_cell)
 void
 make_fake_cell(cell_t *c)
 {
-  tt_assert(c != NULL);
+  tt_ptr_op(c, OP_NE, NULL);
 
   c->circ_id = 1;
   c->command = CELL_RELAY;
@@ -397,7 +259,7 @@ make_fake_cell(cell_t *c)
 void
 make_fake_var_cell(var_cell_t *c)
 {
-  tt_assert(c != NULL);
+  tt_ptr_op(c, OP_NE, NULL);
 
   c->circ_id = 1;
   c->command = CELL_VERSIONS;
@@ -419,36 +281,27 @@ new_fake_channel(void)
   channel_init(chan);
 
   chan->close = chan_test_close;
-  chan->get_overhead_estimate = chan_test_get_overhead_estimate;
-  chan->get_remote_descr = chan_test_get_remote_descr;
-  chan->num_bytes_queued = chan_test_num_bytes_queued;
   chan->num_cells_writeable = chan_test_num_cells_writeable;
-  chan->write_cell = chan_test_write_cell;
+  chan->get_remote_descr = chan_test_get_remote_descr;
   chan->write_packed_cell = chan_test_write_packed_cell;
   chan->write_var_cell = chan_test_write_var_cell;
   chan->state = CHANNEL_STATE_OPEN;
 
+  chan->cmux = circuitmux_alloc();
+  circuitmux_set_policy(chan->cmux, &ewma_policy);
+
   return chan;
 }
 
 void
 free_fake_channel(channel_t *chan)
 {
-  cell_queue_entry_t *cell, *cell_tmp;
-
   if (! chan)
     return;
 
   if (chan->cmux)
     circuitmux_free(chan->cmux);
 
-  TOR_SIMPLEQ_FOREACH_SAFE(cell, &chan->incoming_queue, next, cell_tmp) {
-      cell_queue_entry_free(cell, 0);
-  }
-  TOR_SIMPLEQ_FOREACH_SAFE(cell, &chan->outgoing_queue, next, cell_tmp) {
-      cell_queue_entry_free(cell, 0);
-  }
-
   tor_free(chan);
 }
 
@@ -489,16 +342,6 @@ scheduler_channel_doesnt_want_writes_mock(channel_t *ch)
 }
 
 /**
- * Counter query for scheduler_release_channel_mock()
- */
-
-int
-get_mock_scheduler_release_channel_count(void)
-{
-  return test_releases_count;
-}
-
-/**
  * Mock for scheduler_release_channel()
  */
 
@@ -513,6 +356,58 @@ scheduler_release_channel_mock(channel_t *ch)
   return;
 }
 
+static int
+test_chan_is_canonical(channel_t *chan, int req)
+{
+  tor_assert(chan);
+
+  if (req && test_chan_canonical_should_be_reliable) {
+    return 1;
+  }
+
+  if (test_chan_should_be_canonical) {
+    return 1;
+  }
+  return 0;
+}
+
+static int
+test_chan_matches_target(channel_t *chan, const tor_addr_t *target)
+{
+  (void) chan;
+  (void) target;
+
+  if (test_chan_should_match_target) {
+    return 1;
+  }
+  return 0;
+}
+
+static void
+test_chan_listener_close(channel_listener_t *chan)
+{
+  (void) chan;
+  ++test_chan_listener_close_fn_called;
+  return;
+}
+
+static void
+test_chan_listener_fn(channel_listener_t *listener, channel_t *chan)
+{
+  (void) listener;
+  (void) chan;
+
+  ++test_chan_listener_fn_called;
+  return;
+}
+
+static const char *
+test_chan_listener_describe_transport(channel_listener_t *chan)
+{
+  (void) chan;
+  return "Fake listener channel.";
+}
+
 /**
  * Test for channel_dumpstats() and limited test for
  * channel_dump_statistics()
@@ -523,6 +418,7 @@ test_channel_dumpstats(void *arg)
 {
   channel_t *ch = NULL;
   cell_t *cell = NULL;
+  packed_cell_t *p_cell = NULL;
   int old_count;
 
   (void)arg;
@@ -536,7 +432,6 @@ test_channel_dumpstats(void *arg)
   /* Set up a new fake channel */
   ch = new_fake_channel();
   tt_assert(ch);
-  ch->cmux = circuitmux_alloc();
 
   /* Try to register it */
   channel_register(ch);
@@ -552,24 +447,24 @@ test_channel_dumpstats(void *arg)
   channel_dumpstats(LOG_DEBUG);
 
   /* Assert that we hit the mock */
-  tt_int_op(dump_statistics_mock_matches, ==, 1);
+  tt_int_op(dump_statistics_mock_matches, OP_EQ, 1);
 
   /* Close the channel */
   channel_mark_for_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSING);
   chan_test_finish_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSED);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSED);
 
   /* Try again and hit the finished channel */
   channel_dumpstats(LOG_DEBUG);
-  tt_int_op(dump_statistics_mock_matches, ==, 2);
+  tt_int_op(dump_statistics_mock_matches, OP_EQ, 2);
 
   channel_run_cleanup();
   ch = NULL;
 
   /* Now we should hit nothing */
   channel_dumpstats(LOG_DEBUG);
-  tt_int_op(dump_statistics_mock_matches, ==, 2);
+  tt_int_op(dump_statistics_mock_matches, OP_EQ, 2);
 
   /* Unmock */
   UNMOCK(channel_dump_statistics);
@@ -579,59 +474,56 @@ test_channel_dumpstats(void *arg)
   /* Now make another channel */
   ch = new_fake_channel();
   tt_assert(ch);
-  ch->cmux = circuitmux_alloc();
   channel_register(ch);
-  tt_assert(ch->registered);
+  tt_int_op(ch->registered, OP_EQ, 1);
   /* Lie about its age so dumpstats gets coverage for rate calculations */
   ch->timestamp_created = time(NULL) - 30;
-  tt_assert(ch->timestamp_created > 0);
-  tt_assert(time(NULL) > ch->timestamp_created);
+  tt_int_op(ch->timestamp_created, OP_GT, 0);
+  tt_int_op(time(NULL), OP_GT, ch->timestamp_created);
 
   /* Put cells through it both ways to make the counters non-zero */
-  cell = tor_malloc_zero(sizeof(*cell));
-  make_fake_cell(cell);
+  p_cell = packed_cell_new();
   test_chan_accept_cells = 1;
   old_count = test_cells_written;
-  channel_write_cell(ch, cell);
-  cell = NULL;
-  tt_int_op(test_cells_written, ==, old_count + 1);
-  tt_assert(ch->n_bytes_xmitted > 0);
-  tt_assert(ch->n_cells_xmitted > 0);
+  channel_write_packed_cell(ch, p_cell);
+  tt_int_op(test_cells_written, OP_EQ, old_count + 1);
+  tt_u64_op(ch->n_bytes_xmitted, OP_GT, 0);
+  tt_u64_op(ch->n_cells_xmitted, OP_GT, 0);
 
   /* Receive path */
   channel_set_cell_handlers(ch,
                             chan_test_cell_handler,
                             chan_test_var_cell_handler);
-  tt_ptr_op(channel_get_cell_handler(ch), ==, chan_test_cell_handler);
-  tt_ptr_op(channel_get_var_cell_handler(ch), ==, chan_test_var_cell_handler);
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
+  tt_ptr_op(channel_get_cell_handler(ch), OP_EQ, chan_test_cell_handler);
+  tt_ptr_op(channel_get_var_cell_handler(ch), OP_EQ,
+            chan_test_var_cell_handler);
+  cell = tor_malloc_zero(sizeof(*cell));
   old_count = test_chan_fixed_cells_recved;
-  channel_queue_cell(ch, cell);
-  tor_free(cell);
-  tt_int_op(test_chan_fixed_cells_recved, ==, old_count + 1);
-  tt_assert(ch->n_bytes_recved > 0);
-  tt_assert(ch->n_cells_recved > 0);
+  channel_process_cell(ch, cell);
+  tt_int_op(test_chan_fixed_cells_recved, OP_EQ, old_count + 1);
+  tt_u64_op(ch->n_bytes_recved, OP_GT, 0);
+  tt_u64_op(ch->n_cells_recved, OP_GT, 0);
 
   /* Test channel_dump_statistics */
   ch->describe_transport = chan_test_describe_transport;
   ch->dumpstats = chan_test_dumpstats;
-  ch->is_canonical = chan_test_is_canonical;
+  test_chan_should_be_canonical = 1;
+  ch->is_canonical = test_chan_is_canonical;
   old_count = test_dumpstats_calls;
   channel_dump_statistics(ch, LOG_DEBUG);
-  tt_int_op(test_dumpstats_calls, ==, old_count + 1);
+  tt_int_op(test_dumpstats_calls, OP_EQ, old_count + 1);
 
   /* Close the channel */
   channel_mark_for_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSING);
   chan_test_finish_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSED);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSED);
   channel_run_cleanup();
   ch = NULL;
 
  done:
-  tor_free(cell);
   free_fake_channel(ch);
+  tor_free(cell);
 
   UNMOCK(scheduler_channel_doesnt_want_writes);
   UNMOCK(scheduler_release_channel);
@@ -639,214 +531,235 @@ test_channel_dumpstats(void *arg)
   return;
 }
 
+/* Test outbound cell. The callstack is:
+ *  channel_flush_some_cells()
+ *   -> channel_flush_from_first_active_circuit()
+ *     -> channel_write_packed_cell()
+ *       -> write_packed_cell()
+ *         -> chan->write_packed_cell() fct ptr.
+ *
+ * This test goes from a cell in a circuit up to the channel write handler
+ * that should put them on the connection outbuf. */
 static void
-test_channel_flush(void *arg)
+test_channel_outbound_cell(void *arg)
 {
-  channel_t *ch = NULL;
-  cell_t *cell = NULL;
-  packed_cell_t *p_cell = NULL;
-  var_cell_t *v_cell = NULL;
-  int init_count;
+  int old_count;
+  channel_t *chan = NULL;
+  packed_cell_t *p_cell = NULL, *p_cell2 = NULL;
+  origin_circuit_t *circ = NULL;
+  cell_queue_t *queue;
 
-  (void)arg;
+  (void) arg;
 
-  ch = new_fake_channel();
-  tt_assert(ch);
+  /* Set the test time to be mocked, since this test assumes that no
+   * time will pass, ewma values will not need to be re-scaled, and so on */
+  monotime_enable_test_mocking();
+  monotime_set_mock_time_nsec(UINT64_C(1000000000) * 12345);
 
-  /* Cache the original count */
-  init_count = test_cells_written;
+  cmux_ewma_set_options(NULL,NULL);
 
-  /* Stop accepting so we can queue some */
-  test_chan_accept_cells = 0;
-
-  /* Queue a regular cell */
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  channel_write_cell(ch, cell);
-  /* It should be queued, so assert that we didn't write it */
-  tt_int_op(test_cells_written, ==, init_count);
-
-  /* Queue a var cell */
-  v_cell = tor_malloc_zero(sizeof(var_cell_t) + CELL_PAYLOAD_SIZE);
-  make_fake_var_cell(v_cell);
-  channel_write_var_cell(ch, v_cell);
-  /* It should be queued, so assert that we didn't write it */
-  tt_int_op(test_cells_written, ==, init_count);
-
-  /* Try a packed cell now */
-  p_cell = packed_cell_new();
-  tt_assert(p_cell);
-  channel_write_packed_cell(ch, p_cell);
-  /* It should be queued, so assert that we didn't write it */
-  tt_int_op(test_cells_written, ==, init_count);
+  /* The channel will be freed so we need to hijack this so the scheduler
+   * doesn't get confused. */
+  MOCK(scheduler_release_channel, scheduler_release_channel_mock);
 
-  /* Now allow writes through again */
+  /* Accept cells to lower layer */
   test_chan_accept_cells = 1;
 
-  /* ...and flush */
-  channel_flush_cells(ch);
-
-  /* All three should have gone through */
-  tt_int_op(test_cells_written, ==, init_count + 3);
-
- done:
-  tor_free(ch);
-
-  return;
-}
-
-/**
- * Channel flush tests that require cmux mocking
- */
-
-static void
-test_channel_flushmux(void *arg)
-{
-  channel_t *ch = NULL;
-  int old_count, q_len_before, q_len_after;
-  ssize_t result;
-
-  (void)arg;
-
-  /* Install mocks we need for this test */
-  MOCK(channel_flush_from_first_active_circuit,
-       chan_test_channel_flush_from_first_active_circuit_mock);
-  MOCK(circuitmux_num_cells,
-       chan_test_circuitmux_num_cells_mock);
-
-  ch = new_fake_channel();
-  tt_assert(ch);
-  ch->cmux = circuitmux_alloc();
-
+  /* Setup a valid circuit to queue a cell. */
+  circ = origin_circuit_new();
+  tt_assert(circ);
+  /* Circuit needs an origin purpose to be considered origin. */
+  TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  TO_CIRCUIT(circ)->n_circ_id = 42;
+  /* This is the outbound test so use the next channel queue. */
+  queue = &TO_CIRCUIT(circ)->n_chan_cells;
+  /* Setup packed cell to queue on the circuit. */
+  p_cell = packed_cell_new();
+  tt_assert(p_cell);
+  p_cell2 = packed_cell_new();
+  tt_assert(p_cell2);
+  /* Setup a channel to put the circuit on. */
+  chan = new_fake_channel();
+  tt_assert(chan);
+  chan->state = CHANNEL_STATE_OPENING;
+  channel_change_state_open(chan);
+  /* Outbound channel. */
+  channel_mark_outgoing(chan);
+  /* Try to register it so we can clean it through the channel cleanup
+   * process. */
+  channel_register(chan);
+  tt_int_op(chan->registered, OP_EQ, 1);
+  /* Set EWMA policy so we can pick it when flushing. */
+  circuitmux_set_policy(chan->cmux, &ewma_policy);
+  tt_ptr_op(circuitmux_get_policy(chan->cmux), OP_EQ, &ewma_policy);
+
+  /* Register circuit to the channel circid map which will attach the circuit
+   * to the channel's cmux as well. */
+  circuit_set_n_circid_chan(TO_CIRCUIT(circ), 42, chan);
+  tt_int_op(channel_num_circuits(chan), OP_EQ, 1);
+  /* Test the cmux state. */
+  tt_ptr_op(TO_CIRCUIT(circ)->n_mux, OP_EQ, chan->cmux);
+  tt_int_op(circuitmux_is_circuit_attached(chan->cmux, TO_CIRCUIT(circ)),
+            OP_EQ, 1);
+
+  /* Flush the channel without any cell on it. */
   old_count = test_cells_written;
-
-  test_target_cmux = ch->cmux;
-  test_cmux_cells = 1;
-
-  /* Enable cell acceptance */
-  test_chan_accept_cells = 1;
-
-  result = channel_flush_some_cells(ch, 1);
-
-  tt_int_op(result, ==, 1);
-  tt_int_op(test_cells_written, ==, old_count + 1);
-  tt_int_op(test_cmux_cells, ==, 0);
-
-  /* Now try it without accepting to force them into the queue */
-  test_chan_accept_cells = 0;
-  test_cmux_cells = 1;
-  q_len_before = chan_cell_queue_len(&(ch->outgoing_queue));
-
-  result = channel_flush_some_cells(ch, 1);
-
-  /* We should not have actually flushed any */
-  tt_int_op(result, ==, 0);
-  tt_int_op(test_cells_written, ==, old_count + 1);
-  /* But we should have gotten to the fake cellgen loop */
-  tt_int_op(test_cmux_cells, ==, 0);
-  /* ...and we should have a queued cell */
-  q_len_after = chan_cell_queue_len(&(ch->outgoing_queue));
-  tt_int_op(q_len_after, ==, q_len_before + 1);
-
-  /* Now accept cells again and drain the queue */
-  test_chan_accept_cells = 1;
-  channel_flush_cells(ch);
-  tt_int_op(test_cells_written, ==, old_count + 2);
-  tt_int_op(chan_cell_queue_len(&(ch->outgoing_queue)), ==, 0);
-
-  test_target_cmux = NULL;
-  test_cmux_cells = 0;
+  ssize_t flushed = channel_flush_some_cells(chan, 1);
+  tt_i64_op(flushed, OP_EQ, 0);
+  tt_int_op(test_cells_written, OP_EQ, old_count);
+  tt_int_op(channel_more_to_flush(chan), OP_EQ, 0);
+  tt_int_op(circuitmux_num_active_circuits(chan->cmux), OP_EQ, 0);
+  tt_int_op(circuitmux_num_cells(chan->cmux), OP_EQ, 0);
+  tt_int_op(circuitmux_is_circuit_active(chan->cmux, TO_CIRCUIT(circ)),
+            OP_EQ, 0);
+  tt_u64_op(chan->n_cells_xmitted, OP_EQ, 0);
+  tt_u64_op(chan->n_bytes_xmitted, OP_EQ, 0);
+
+  /* Queue cell onto the next queue that is the outbound direction. Than
+   * update its cmux so the circuit can be picked when flushing cells. */
+  cell_queue_append(queue, p_cell);
+  p_cell = NULL;
+  tt_int_op(queue->n, OP_EQ, 1);
+  cell_queue_append(queue, p_cell2);
+  p_cell2 = NULL;
+  tt_int_op(queue->n, OP_EQ, 2);
+
+  update_circuit_on_cmux(TO_CIRCUIT(circ), CELL_DIRECTION_OUT);
+  tt_int_op(circuitmux_num_active_circuits(chan->cmux), OP_EQ, 1);
+  tt_int_op(circuitmux_num_cells(chan->cmux), OP_EQ, 2);
+  tt_int_op(circuitmux_is_circuit_active(chan->cmux, TO_CIRCUIT(circ)),
+            OP_EQ, 1);
+
+  /* From this point on, we have a queued cell on an active circuit attached
+   * to the channel's cmux. */
+
+  /* Flush the first cell. This is going to go down the call stack. */
+  old_count = test_cells_written;
+  flushed = channel_flush_some_cells(chan, 1);
+  tt_i64_op(flushed, OP_EQ, 1);
+  tt_int_op(test_cells_written, OP_EQ, old_count + 1);
+  tt_int_op(circuitmux_num_cells(chan->cmux), OP_EQ, 1);
+  tt_int_op(channel_more_to_flush(chan), OP_EQ, 1);
+  /* Circuit should remain active because there is a second cell queued. */
+  tt_int_op(circuitmux_is_circuit_active(chan->cmux, TO_CIRCUIT(circ)),
+            OP_EQ, 1);
+  /* Should still be attached. */
+  tt_int_op(circuitmux_is_circuit_attached(chan->cmux, TO_CIRCUIT(circ)),
+            OP_EQ, 1);
+  tt_u64_op(chan->n_cells_xmitted, OP_EQ, 1);
+  tt_u64_op(chan->n_bytes_xmitted, OP_EQ, get_cell_network_size(0));
+
+  /* Flush second cell. This is going to go down the call stack. */
+  old_count = test_cells_written;
+  flushed = channel_flush_some_cells(chan, 1);
+  tt_i64_op(flushed, OP_EQ, 1);
+  tt_int_op(test_cells_written, OP_EQ, old_count + 1);
+  tt_int_op(circuitmux_num_cells(chan->cmux), OP_EQ, 0);
+  tt_int_op(channel_more_to_flush(chan), OP_EQ, 0);
+  /* No more cells should make the circuit inactive. */
+  tt_int_op(circuitmux_is_circuit_active(chan->cmux, TO_CIRCUIT(circ)),
+            OP_EQ, 0);
+  /* Should still be attached. */
+  tt_int_op(circuitmux_is_circuit_attached(chan->cmux, TO_CIRCUIT(circ)),
+            OP_EQ, 1);
+  tt_u64_op(chan->n_cells_xmitted, OP_EQ, 2);
+  tt_u64_op(chan->n_bytes_xmitted, OP_EQ, get_cell_network_size(0) * 2);
 
  done:
-  if (ch)
-    circuitmux_free(ch->cmux);
-  tor_free(ch);
-
-  UNMOCK(channel_flush_from_first_active_circuit);
-  UNMOCK(circuitmux_num_cells);
-
-  test_chan_accept_cells = 0;
-
-  return;
+  if (circ) {
+    circuit_free_(TO_CIRCUIT(circ));
+  }
+  tor_free(p_cell);
+  channel_free_all();
+  UNMOCK(scheduler_release_channel);
+  monotime_disable_test_mocking();
 }
 
+/* Test inbound cell. The callstack is:
+ *  channel_process_cell()
+ *    -> chan->cell_handler()
+ *
+ * This test is about checking if we can process an inbound cell down to the
+ * channel handler. */
 static void
-test_channel_incoming(void *arg)
+test_channel_inbound_cell(void *arg)
 {
-  channel_t *ch = NULL;
+  channel_t *chan = NULL;
   cell_t *cell = NULL;
-  var_cell_t *var_cell = NULL;
   int old_count;
 
-  (void)arg;
+  (void) arg;
 
-  /* Mock these for duration of the test */
-  MOCK(scheduler_channel_doesnt_want_writes,
-       scheduler_channel_doesnt_want_writes_mock);
-  MOCK(scheduler_release_channel,
-       scheduler_release_channel_mock);
+  /* The channel will be freed so we need to hijack this so the scheduler
+   * doesn't get confused. */
+  MOCK(scheduler_release_channel, scheduler_release_channel_mock);
 
   /* Accept cells to lower layer */
   test_chan_accept_cells = 1;
-  /* Use default overhead factor */
-  test_overhead_estimate = 1.0;
 
-  ch = new_fake_channel();
-  tt_assert(ch);
+  chan = new_fake_channel();
+  tt_assert(chan);
   /* Start it off in OPENING */
-  ch->state = CHANNEL_STATE_OPENING;
-  /* We'll need a cmux */
-  ch->cmux = circuitmux_alloc();
-
-  /* Install incoming cell handlers */
-  channel_set_cell_handlers(ch,
-                            chan_test_cell_handler,
-                            chan_test_var_cell_handler);
-  /* Test cell handler getters */
-  tt_ptr_op(channel_get_cell_handler(ch), ==, chan_test_cell_handler);
-  tt_ptr_op(channel_get_var_cell_handler(ch), ==, chan_test_var_cell_handler);
+  chan->state = CHANNEL_STATE_OPENING;
 
   /* Try to register it */
-  channel_register(ch);
-  tt_assert(ch->registered);
+  channel_register(chan);
+  tt_int_op(chan->registered, OP_EQ, 1);
 
   /* Open it */
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_OPEN);
+  channel_change_state_open(chan);
+  tt_int_op(chan->state, OP_EQ, CHANNEL_STATE_OPEN);
+  tt_int_op(chan->has_been_open, OP_EQ, 1);
 
-  /* Receive a fixed cell */
-  cell = tor_malloc_zero(sizeof(cell_t));
+  /* Receive a cell now. */
+  cell = tor_malloc_zero(sizeof(*cell));
   make_fake_cell(cell);
   old_count = test_chan_fixed_cells_recved;
-  channel_queue_cell(ch, cell);
-  tor_free(cell);
-  tt_int_op(test_chan_fixed_cells_recved, ==, old_count + 1);
-
-  /* Receive a variable-size cell */
-  var_cell = tor_malloc_zero(sizeof(var_cell_t) + CELL_PAYLOAD_SIZE);
-  make_fake_var_cell(var_cell);
-  old_count = test_chan_var_cells_recved;
-  channel_queue_var_cell(ch, var_cell);
-  tor_free(cell);
-  tt_int_op(test_chan_var_cells_recved, ==, old_count + 1);
+  channel_process_cell(chan, cell);
+  tt_int_op(test_chan_fixed_cells_recved, OP_EQ, old_count);
+  tt_assert(monotime_coarse_is_zero(&chan->timestamp_xfer));
+  tt_u64_op(chan->timestamp_active, OP_EQ, 0);
+  tt_u64_op(chan->timestamp_recv, OP_EQ, 0);
+
+  /* Setup incoming cell handlers. We don't care about var cell, the channel
+   * layers is not handling those. */
+  channel_set_cell_handlers(chan, chan_test_cell_handler, NULL);
+  tt_ptr_op(chan->cell_handler, OP_EQ, chan_test_cell_handler);
+  /* Now process the cell, we should see it. */
+  old_count = test_chan_fixed_cells_recved;
+  channel_process_cell(chan, cell);
+  tt_int_op(test_chan_fixed_cells_recved, OP_EQ, old_count + 1);
+  /* We should have a series of timestamp set. */
+  tt_assert(!monotime_coarse_is_zero(&chan->timestamp_xfer));
+  tt_u64_op(chan->timestamp_active, OP_NE, 0);
+  tt_u64_op(chan->timestamp_recv, OP_NE, 0);
+  tt_assert(monotime_coarse_is_zero(&chan->next_padding_time));
+  tt_u64_op(chan->n_cells_recved, OP_EQ, 1);
+  tt_u64_op(chan->n_bytes_recved, OP_EQ, get_cell_network_size(0));
 
   /* Close it */
-  channel_mark_for_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSING);
-  chan_test_finish_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSED);
+  old_count = test_close_called;
+  channel_mark_for_close(chan);
+  tt_int_op(chan->state, OP_EQ, CHANNEL_STATE_CLOSING);
+  tt_int_op(chan->reason_for_closing, OP_EQ, CHANNEL_CLOSE_REQUESTED);
+  tt_int_op(test_close_called, OP_EQ, old_count + 1);
+
+  /* This closes the channe so it calls in the scheduler, make sure of it. */
+  old_count = test_releases_count;
+  chan_test_finish_close(chan);
+  tt_int_op(test_releases_count, OP_EQ, old_count + 1);
+  tt_int_op(chan->state, OP_EQ, CHANNEL_STATE_CLOSED);
+
+  /* The channel will be free, lets make sure it is not accessible. */
+  uint64_t chan_id = chan->global_identifier;
+  tt_ptr_op(channel_find_by_global_id(chan_id), OP_EQ, chan);
   channel_run_cleanup();
-  ch = NULL;
+  chan = channel_find_by_global_id(chan_id);
+  tt_assert(chan == NULL);
 
  done:
-  free_fake_channel(ch);
   tor_free(cell);
-  tor_free(var_cell);
-
-  UNMOCK(scheduler_channel_doesnt_want_writes);
   UNMOCK(scheduler_release_channel);
-
-  return;
 }
 
 /**
@@ -859,7 +772,7 @@ static void
 test_channel_lifecycle(void *arg)
 {
   channel_t *ch1 = NULL, *ch2 = NULL;
-  cell_t *cell = NULL;
+  packed_cell_t *p_cell = NULL;
   int old_count, init_doesnt_want_writes_count;
   int init_releases_count;
 
@@ -877,79 +790,71 @@ test_channel_lifecycle(void *arg)
 
   /* Accept cells to lower layer */
   test_chan_accept_cells = 1;
-  /* Use default overhead factor */
-  test_overhead_estimate = 1.0;
 
   ch1 = new_fake_channel();
   tt_assert(ch1);
   /* Start it off in OPENING */
   ch1->state = CHANNEL_STATE_OPENING;
-  /* We'll need a cmux */
-  ch1->cmux = circuitmux_alloc();
 
   /* Try to register it */
   channel_register(ch1);
   tt_assert(ch1->registered);
 
   /* Try to write a cell through (should queue) */
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
+  p_cell = packed_cell_new();
   old_count = test_cells_written;
-  channel_write_cell(ch1, cell);
-  tt_int_op(old_count, ==, test_cells_written);
+  channel_write_packed_cell(ch1, p_cell);
+  tt_int_op(old_count, OP_EQ, test_cells_written);
 
   /* Move it to OPEN and flush */
-  channel_change_state(ch1, CHANNEL_STATE_OPEN);
-
-  /* Queue should drain */
-  tt_int_op(old_count + 1, ==, test_cells_written);
+  channel_change_state_open(ch1);
 
-  /* Get another one */
+/* Get another one */
   ch2 = new_fake_channel();
   tt_assert(ch2);
   ch2->state = CHANNEL_STATE_OPENING;
-  ch2->cmux = circuitmux_alloc();
 
   /* Register */
   channel_register(ch2);
   tt_assert(ch2->registered);
 
   /* Check counters */
-  tt_int_op(test_doesnt_want_writes_count, ==, init_doesnt_want_writes_count);
-  tt_int_op(test_releases_count, ==, init_releases_count);
+  tt_int_op(test_doesnt_want_writes_count, OP_EQ,
+            init_doesnt_want_writes_count);
+  tt_int_op(test_releases_count, OP_EQ, init_releases_count);
 
   /* Move ch1 to MAINT */
   channel_change_state(ch1, CHANNEL_STATE_MAINT);
-  tt_int_op(test_doesnt_want_writes_count, ==,
+  tt_int_op(test_doesnt_want_writes_count, OP_EQ,
             init_doesnt_want_writes_count + 1);
-  tt_int_op(test_releases_count, ==, init_releases_count);
+  tt_int_op(test_releases_count, OP_EQ, init_releases_count);
 
   /* Move ch2 to OPEN */
-  channel_change_state(ch2, CHANNEL_STATE_OPEN);
-  tt_int_op(test_doesnt_want_writes_count, ==,
+  channel_change_state_open(ch2);
+  tt_int_op(test_doesnt_want_writes_count, OP_EQ,
             init_doesnt_want_writes_count + 1);
-  tt_int_op(test_releases_count, ==, init_releases_count);
+  tt_int_op(test_releases_count, OP_EQ, init_releases_count);
 
   /* Move ch1 back to OPEN */
-  channel_change_state(ch1, CHANNEL_STATE_OPEN);
-  tt_int_op(test_doesnt_want_writes_count, ==,
+  channel_change_state_open(ch1);
+  tt_int_op(test_doesnt_want_writes_count, OP_EQ,
             init_doesnt_want_writes_count + 1);
-  tt_int_op(test_releases_count, ==, init_releases_count);
+  tt_int_op(test_releases_count, OP_EQ, init_releases_count);
 
   /* Mark ch2 for close */
   channel_mark_for_close(ch2);
-  tt_int_op(ch2->state, ==, CHANNEL_STATE_CLOSING);
-  tt_int_op(test_doesnt_want_writes_count, ==,
+  tt_int_op(ch2->state, OP_EQ, CHANNEL_STATE_CLOSING);
+  tt_int_op(test_doesnt_want_writes_count, OP_EQ,
             init_doesnt_want_writes_count + 1);
-  tt_int_op(test_releases_count, ==, init_releases_count + 1);
+  tt_int_op(test_releases_count, OP_EQ, init_releases_count + 1);
 
   /* Shut down channels */
   channel_free_all();
   ch1 = ch2 = NULL;
-  tt_int_op(test_doesnt_want_writes_count, ==,
+  tt_int_op(test_doesnt_want_writes_count, OP_EQ,
             init_doesnt_want_writes_count + 1);
   /* channel_free() calls scheduler_release_channel() */
-  tt_int_op(test_releases_count, ==, init_releases_count + 4);
+  tt_int_op(test_releases_count, OP_EQ, init_releases_count + 4);
 
  done:
   free_fake_channel(ch1);
@@ -957,8 +862,6 @@ test_channel_lifecycle(void *arg)
 
   UNMOCK(scheduler_channel_doesnt_want_writes);
   UNMOCK(scheduler_release_channel);
-
-  return;
 }
 
 /**
@@ -985,15 +888,11 @@ test_channel_lifecycle_2(void *arg)
 
   /* Accept cells to lower layer */
   test_chan_accept_cells = 1;
-  /* Use default overhead factor */
-  test_overhead_estimate = 1.0;
 
   ch = new_fake_channel();
   tt_assert(ch);
   /* Start it off in OPENING */
   ch->state = CHANNEL_STATE_OPENING;
-  /* The full lifecycle test needs a cmux */
-  ch->cmux = circuitmux_alloc();
 
   /* Try to register it */
   channel_register(ch);
@@ -1001,11 +900,11 @@ test_channel_lifecycle_2(void *arg)
 
   /* Try to close it */
   channel_mark_for_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSING);
 
   /* Finish closing it */
   chan_test_finish_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSED);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSED);
   channel_run_cleanup();
   ch = NULL;
 
@@ -1013,18 +912,17 @@ test_channel_lifecycle_2(void *arg)
   ch = new_fake_channel();
   tt_assert(ch);
   ch->state = CHANNEL_STATE_OPENING;
-  ch->cmux = circuitmux_alloc();
   channel_register(ch);
   tt_assert(ch->registered);
 
   /* Finish opening it */
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
+  channel_change_state_open(ch);
 
   /* Error exit from lower layer */
   chan_test_error(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSING);
   chan_test_finish_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_ERROR);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_ERROR);
   channel_run_cleanup();
   ch = NULL;
 
@@ -1032,25 +930,24 @@ test_channel_lifecycle_2(void *arg)
   ch = new_fake_channel();
   tt_assert(ch);
   ch->state = CHANNEL_STATE_OPENING;
-  ch->cmux = circuitmux_alloc();
   channel_register(ch);
   tt_assert(ch->registered);
 
   /* Finish opening it */
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_OPEN);
+  channel_change_state_open(ch);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_OPEN);
 
   /* Go to maintenance state */
   channel_change_state(ch, CHANNEL_STATE_MAINT);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_MAINT);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_MAINT);
 
   /* Lower layer close */
   channel_mark_for_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSING);
 
   /* Finish */
   chan_test_finish_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSED);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSED);
   channel_run_cleanup();
   ch = NULL;
 
@@ -1061,25 +958,24 @@ test_channel_lifecycle_2(void *arg)
   ch = new_fake_channel();
   tt_assert(ch);
   ch->state = CHANNEL_STATE_OPENING;
-  ch->cmux = circuitmux_alloc();
   channel_register(ch);
   tt_assert(ch->registered);
 
   /* Finish opening it */
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_OPEN);
+  channel_change_state_open(ch);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_OPEN);
 
   /* Go to maintenance state */
   channel_change_state(ch, CHANNEL_STATE_MAINT);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_MAINT);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_MAINT);
 
   /* Lower layer close */
   channel_close_from_lower_layer(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSING);
 
   /* Finish */
   chan_test_finish_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSED);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSED);
   channel_run_cleanup();
   ch = NULL;
 
@@ -1087,25 +983,24 @@ test_channel_lifecycle_2(void *arg)
   ch = new_fake_channel();
   tt_assert(ch);
   ch->state = CHANNEL_STATE_OPENING;
-  ch->cmux = circuitmux_alloc();
   channel_register(ch);
   tt_assert(ch->registered);
 
   /* Finish opening it */
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_OPEN);
+  channel_change_state_open(ch);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_OPEN);
 
   /* Go to maintenance state */
   channel_change_state(ch, CHANNEL_STATE_MAINT);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_MAINT);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_MAINT);
 
   /* Lower layer close */
   chan_test_error(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSING);
 
   /* Finish */
   chan_test_finish_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_ERROR);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_ERROR);
   channel_run_cleanup();
   ch = NULL;
 
@@ -1122,660 +1017,554 @@ test_channel_lifecycle_2(void *arg)
 }
 
 static void
-test_channel_multi(void *arg)
+test_channel_id_map(void *arg)
 {
-  channel_t *ch1 = NULL, *ch2 = NULL;
-  uint64_t global_queue_estimate;
-  cell_t *cell = NULL;
-
   (void)arg;
+#define N_CHAN 6
+  char rsa_id[N_CHAN][DIGEST_LEN];
+  ed25519_public_key_t *ed_id[N_CHAN];
+  channel_t *chan[N_CHAN];
+  int i;
+  ed25519_public_key_t ed_zero;
+  memset(&ed_zero, 0, sizeof(ed_zero));
+
+  tt_int_op(DIGEST_LEN, OP_EQ, sizeof(rsa_id[0])); // Do I remember C?
+
+  for (i = 0; i < N_CHAN; ++i) {
+    crypto_rand(rsa_id[i], DIGEST_LEN);
+    ed_id[i] = tor_malloc_zero(sizeof(*ed_id[i]));
+    crypto_rand((char*)ed_id[i]->pubkey, sizeof(ed_id[i]->pubkey));
+  }
 
-  /* Accept cells to lower layer */
-  test_chan_accept_cells = 1;
-  /* Use default overhead factor */
-  test_overhead_estimate = 1.0;
-
-  ch1 = new_fake_channel();
-  tt_assert(ch1);
-  ch2 = new_fake_channel();
-  tt_assert(ch2);
-
-  /* Initial queue size update */
-  channel_update_xmit_queue_size(ch1);
-  tt_u64_op(ch1->bytes_queued_for_xmit, ==, 0);
-  channel_update_xmit_queue_size(ch2);
-  tt_u64_op(ch2->bytes_queued_for_xmit, ==, 0);
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 0);
-
-  /* Queue some cells, check queue estimates */
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  channel_write_cell(ch1, cell);
-
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  channel_write_cell(ch2, cell);
-
-  channel_update_xmit_queue_size(ch1);
-  channel_update_xmit_queue_size(ch2);
-  tt_u64_op(ch1->bytes_queued_for_xmit, ==, 0);
-  tt_u64_op(ch2->bytes_queued_for_xmit, ==, 0);
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 0);
-
-  /* Stop accepting cells at lower layer */
-  test_chan_accept_cells = 0;
-
-  /* Queue some cells and check queue estimates */
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  channel_write_cell(ch1, cell);
-
-  channel_update_xmit_queue_size(ch1);
-  tt_u64_op(ch1->bytes_queued_for_xmit, ==, 512);
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 512);
-
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  channel_write_cell(ch2, cell);
-
-  channel_update_xmit_queue_size(ch2);
-  tt_u64_op(ch2->bytes_queued_for_xmit, ==, 512);
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 1024);
-
-  /* Allow cells through again */
-  test_chan_accept_cells = 1;
-
-  /* Flush chan 2 */
-  channel_flush_cells(ch2);
-
-  /* Update and check queue sizes */
-  channel_update_xmit_queue_size(ch1);
-  channel_update_xmit_queue_size(ch2);
-  tt_u64_op(ch1->bytes_queued_for_xmit, ==, 512);
-  tt_u64_op(ch2->bytes_queued_for_xmit, ==, 0);
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 512);
-
-  /* Flush chan 1 */
-  channel_flush_cells(ch1);
-
-  /* Update and check queue sizes */
-  channel_update_xmit_queue_size(ch1);
-  channel_update_xmit_queue_size(ch2);
-  tt_u64_op(ch1->bytes_queued_for_xmit, ==, 0);
-  tt_u64_op(ch2->bytes_queued_for_xmit, ==, 0);
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 0);
-
-  /* Now block again */
-  test_chan_accept_cells = 0;
-
-  /* Queue some cells */
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  channel_write_cell(ch1, cell);
+  /* For channel 3, have no Ed identity. */
+  tor_free(ed_id[3]);
 
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  channel_write_cell(ch2, cell);
-  cell = NULL;
-
-  /* Check the estimates */
-  channel_update_xmit_queue_size(ch1);
-  channel_update_xmit_queue_size(ch2);
-  tt_u64_op(ch1->bytes_queued_for_xmit, ==, 512);
-  tt_u64_op(ch2->bytes_queued_for_xmit, ==, 512);
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 1024);
-
-  /* Now close channel 2; it should be subtracted from the global queue */
-  MOCK(scheduler_release_channel, scheduler_release_channel_mock);
-  channel_mark_for_close(ch2);
-  UNMOCK(scheduler_release_channel);
+  /* Channel 2 and 4 have same ROSA identity */
+  memcpy(rsa_id[4], rsa_id[2], DIGEST_LEN);
 
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 512);
+  /* Channel 2 and 4 and 5 have same RSA identity */
+  memcpy(rsa_id[4], rsa_id[2], DIGEST_LEN);
+  memcpy(rsa_id[5], rsa_id[2], DIGEST_LEN);
 
-  /*
-   * Since the fake channels aren't registered, channel_free_all() can't
-   * see them properly.
-   */
-  MOCK(scheduler_release_channel, scheduler_release_channel_mock);
-  channel_mark_for_close(ch1);
-  UNMOCK(scheduler_release_channel);
+  /* Channels 2 and 5 have same Ed25519 identity */
+  memcpy(ed_id[5], ed_id[2], sizeof(*ed_id[2]));
 
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 0);
+  for (i = 0; i < N_CHAN; ++i) {
+    chan[i] = new_fake_channel();
+    channel_register(chan[i]);
+    channel_set_identity_digest(chan[i], rsa_id[i], ed_id[i]);
+  }
 
-  /* Now free everything */
-  MOCK(scheduler_release_channel, scheduler_release_channel_mock);
-  channel_free_all();
-  UNMOCK(scheduler_release_channel);
+  /* Lookup by RSA id only */
+  tt_ptr_op(chan[0], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[0], NULL));
+  tt_ptr_op(chan[1], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[1], NULL));
+  tt_ptr_op(chan[3], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[3], NULL));
+  channel_t *ch;
+  ch = channel_find_by_remote_identity(rsa_id[2], NULL);
+  tt_assert(ch == chan[2] || ch == chan[4] || ch == chan[5]);
+  ch = channel_next_with_rsa_identity(ch);
+  tt_assert(ch == chan[2] || ch == chan[4] || ch == chan[5]);
+  ch = channel_next_with_rsa_identity(ch);
+  tt_assert(ch == chan[2] || ch == chan[4] || ch == chan[5]);
+  ch = channel_next_with_rsa_identity(ch);
+  tt_ptr_op(ch, OP_EQ, NULL);
+
+  /* As above, but with zero Ed25519 ID (meaning "any ID") */
+  tt_ptr_op(chan[0], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[0], &ed_zero));
+  tt_ptr_op(chan[1], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[1], &ed_zero));
+  tt_ptr_op(chan[3], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[3], &ed_zero));
+  ch = channel_find_by_remote_identity(rsa_id[2], &ed_zero);
+  tt_assert(ch == chan[2] || ch == chan[4] || ch == chan[5]);
+  ch = channel_next_with_rsa_identity(ch);
+  tt_assert(ch == chan[2] || ch == chan[4] || ch == chan[5]);
+  ch = channel_next_with_rsa_identity(ch);
+  tt_assert(ch == chan[2] || ch == chan[4] || ch == chan[5]);
+  ch = channel_next_with_rsa_identity(ch);
+  tt_ptr_op(ch, OP_EQ, NULL);
+
+  /* Lookup nonexistent RSA identity */
+  tt_ptr_op(NULL, OP_EQ,
+            channel_find_by_remote_identity("!!!!!!!!!!!!!!!!!!!!", NULL));
+
+  /* Look up by full identity pair */
+  tt_ptr_op(chan[0], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[0], ed_id[0]));
+  tt_ptr_op(chan[1], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[1], ed_id[1]));
+  tt_ptr_op(chan[3], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[3], ed_id[3] /*NULL*/));
+  tt_ptr_op(chan[4], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[4], ed_id[4]));
+  ch = channel_find_by_remote_identity(rsa_id[2], ed_id[2]);
+  tt_assert(ch == chan[2] || ch == chan[5]);
+
+  /* Look up RSA identity with wrong ed25519 identity */
+  tt_ptr_op(NULL, OP_EQ,
+            channel_find_by_remote_identity(rsa_id[4], ed_id[0]));
+  tt_ptr_op(NULL, OP_EQ,
+            channel_find_by_remote_identity(rsa_id[2], ed_id[1]));
+  tt_ptr_op(NULL, OP_EQ,
+            channel_find_by_remote_identity(rsa_id[3], ed_id[1]));
 
  done:
-  free_fake_channel(ch1);
-  free_fake_channel(ch2);
-
-  return;
+  for (i = 0; i < N_CHAN; ++i) {
+    channel_clear_identity_digest(chan[i]);
+    channel_unregister(chan[i]);
+    free_fake_channel(chan[i]);
+    tor_free(ed_id[i]);
+  }
+#undef N_CHAN
 }
 
-/**
- * Check some hopefully-impossible edge cases in the channel queue we
- * can only trigger by doing evil things to the queue directly.
- */
-
 static void
-test_channel_queue_impossible(void *arg)
+test_channel_state(void *arg)
 {
-  channel_t *ch = NULL;
-  cell_t *cell = NULL;
-  packed_cell_t *packed_cell = NULL;
-  var_cell_t *var_cell = NULL;
-  int old_count;
-  cell_queue_entry_t *q = NULL;
-  uint64_t global_queue_estimate;
-  uintptr_t cellintptr;
-
-  /* Cache the global queue size (see below) */
-  global_queue_estimate = channel_get_global_queue_estimate();
-
-  (void)arg;
-
-  ch = new_fake_channel();
-  tt_assert(ch);
-
-  /* We test queueing here; tell it not to accept cells */
-  test_chan_accept_cells = 0;
-  /* ...and keep it from trying to flush the queue */
-  ch->state = CHANNEL_STATE_MAINT;
-
-  /* Cache the cell written count */
-  old_count = test_cells_written;
-
-  /* Assert that the queue is initially empty */
-  tt_int_op(chan_cell_queue_len(&(ch->outgoing_queue)), ==, 0);
-
-  /* Get a fresh cell and write it to the channel*/
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  cellintptr = (uintptr_t)(void*)cell;
-  channel_write_cell(ch, cell);
-
-  /* Now it should be queued */
-  tt_int_op(chan_cell_queue_len(&(ch->outgoing_queue)), ==, 1);
-  q = TOR_SIMPLEQ_FIRST(&(ch->outgoing_queue));
-  tt_assert(q);
-  if (q) {
-    tt_int_op(q->type, ==, CELL_QUEUE_FIXED);
-    tt_assert((uintptr_t)q->u.fixed.cell == cellintptr);
-  }
-  /* Do perverse things to it */
-  tor_free(q->u.fixed.cell);
-  q->u.fixed.cell = NULL;
-
-  /*
-   * Now change back to open with channel_change_state() and assert that it
-   * gets thrown away properly.
-   */
-  test_chan_accept_cells = 1;
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-  tt_assert(test_cells_written == old_count);
-  tt_int_op(chan_cell_queue_len(&(ch->outgoing_queue)), ==, 0);
-
-  /* Same thing but for a var_cell */
-
-  test_chan_accept_cells = 0;
-  ch->state = CHANNEL_STATE_MAINT;
-  var_cell = tor_malloc_zero(sizeof(var_cell_t) + CELL_PAYLOAD_SIZE);
-  make_fake_var_cell(var_cell);
-  cellintptr = (uintptr_t)(void*)var_cell;
-  channel_write_var_cell(ch, var_cell);
-
-  /* Check that it's queued */
-  tt_int_op(chan_cell_queue_len(&(ch->outgoing_queue)), ==, 1);
-  q = TOR_SIMPLEQ_FIRST(&(ch->outgoing_queue));
-  tt_assert(q);
-  if (q) {
-    tt_int_op(q->type, ==, CELL_QUEUE_VAR);
-    tt_assert((uintptr_t)q->u.var.var_cell == cellintptr);
-  }
-
-  /* Remove the cell from the queue entry */
-  tor_free(q->u.var.var_cell);
-  q->u.var.var_cell = NULL;
-
-  /* Let it drain and check that the bad entry is discarded */
-  test_chan_accept_cells = 1;
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-  tt_assert(test_cells_written == old_count);
-  tt_int_op(chan_cell_queue_len(&(ch->outgoing_queue)), ==, 0);
-
-  /* Same thing with a packed_cell */
-
-  test_chan_accept_cells = 0;
-  ch->state = CHANNEL_STATE_MAINT;
-  packed_cell = packed_cell_new();
-  tt_assert(packed_cell);
-  cellintptr = (uintptr_t)(void*)packed_cell;
-  channel_write_packed_cell(ch, packed_cell);
-
-  /* Check that it's queued */
-  tt_int_op(chan_cell_queue_len(&(ch->outgoing_queue)), ==, 1);
-  q = TOR_SIMPLEQ_FIRST(&(ch->outgoing_queue));
-  tt_assert(q);
-  if (q) {
-    tt_int_op(q->type, ==, CELL_QUEUE_PACKED);
-    tt_assert((uintptr_t)q->u.packed.packed_cell == cellintptr);
-  }
-
-  /* Remove the cell from the queue entry */
-  packed_cell_free(q->u.packed.packed_cell);
-  q->u.packed.packed_cell = NULL;
-
-  /* Let it drain and check that the bad entry is discarded */
-  test_chan_accept_cells = 1;
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-  tt_assert(test_cells_written == old_count);
-  tt_int_op(chan_cell_queue_len(&(ch->outgoing_queue)), ==, 0);
-
-  /* Unknown cell type case */
-  test_chan_accept_cells = 0;
-  ch->state = CHANNEL_STATE_MAINT;
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  cellintptr = (uintptr_t)(void*)cell;
-  channel_write_cell(ch, cell);
-
-  /* Check that it's queued */
-  tt_int_op(chan_cell_queue_len(&(ch->outgoing_queue)), ==, 1);
-  q = TOR_SIMPLEQ_FIRST(&(ch->outgoing_queue));
-  tt_assert(q);
-  if (q) {
-    tt_int_op(q->type, ==, CELL_QUEUE_FIXED);
-    tt_assert((uintptr_t)q->u.fixed.cell == cellintptr);
-  }
-  /* Clobber it, including the queue entry type */
-  tor_free(q->u.fixed.cell);
-  q->u.fixed.cell = NULL;
-  q->type = CELL_QUEUE_PACKED + 1;
-
-  /* Let it drain and check that the bad entry is discarded */
-  test_chan_accept_cells = 1;
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-  tt_assert(test_cells_written == old_count);
-  tt_int_op(chan_cell_queue_len(&(ch->outgoing_queue)), ==, 0);
+  (void) arg;
+
+  /* Test state validity. */
+  tt_int_op(channel_state_is_valid(CHANNEL_STATE_CLOSED), OP_EQ, 1);
+  tt_int_op(channel_state_is_valid(CHANNEL_STATE_CLOSING), OP_EQ, 1);
+  tt_int_op(channel_state_is_valid(CHANNEL_STATE_ERROR), OP_EQ, 1);
+  tt_int_op(channel_state_is_valid(CHANNEL_STATE_OPEN), OP_EQ, 1);
+  tt_int_op(channel_state_is_valid(CHANNEL_STATE_OPENING), OP_EQ, 1);
+  tt_int_op(channel_state_is_valid(CHANNEL_STATE_MAINT), OP_EQ, 1);
+  tt_int_op(channel_state_is_valid(CHANNEL_STATE_LAST), OP_EQ, 0);
+  tt_int_op(channel_state_is_valid(INT_MAX), OP_EQ, 0);
+
+  /* Test listener state validity. */
+  tt_int_op(channel_listener_state_is_valid(CHANNEL_LISTENER_STATE_CLOSED),
+            OP_EQ, 1);
+  tt_int_op(channel_listener_state_is_valid(CHANNEL_LISTENER_STATE_LISTENING),
+            OP_EQ, 1);
+  tt_int_op(channel_listener_state_is_valid(CHANNEL_LISTENER_STATE_CLOSING),
+            OP_EQ, 1);
+  tt_int_op(channel_listener_state_is_valid(CHANNEL_LISTENER_STATE_ERROR),
+            OP_EQ, 1);
+  tt_int_op(channel_listener_state_is_valid(CHANNEL_LISTENER_STATE_LAST),
+            OP_EQ, 0);
+  tt_int_op(channel_listener_state_is_valid(INT_MAX), OP_EQ, 0);
+
+  /* Test state transition. */
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_CLOSED,
+                                         CHANNEL_STATE_OPENING), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_CLOSED,
+                                         CHANNEL_STATE_ERROR), OP_EQ, 0);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_CLOSING,
+                                         CHANNEL_STATE_ERROR), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_CLOSING,
+                                         CHANNEL_STATE_CLOSED), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_CLOSING,
+                                         CHANNEL_STATE_OPEN), OP_EQ, 0);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_MAINT,
+                                         CHANNEL_STATE_CLOSING), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_MAINT,
+                                         CHANNEL_STATE_ERROR), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_MAINT,
+                                         CHANNEL_STATE_OPEN), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_MAINT,
+                                         CHANNEL_STATE_OPENING), OP_EQ, 0);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_OPENING,
+                                         CHANNEL_STATE_OPEN), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_OPENING,
+                                         CHANNEL_STATE_CLOSING), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_OPENING,
+                                         CHANNEL_STATE_ERROR), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_OPEN,
+                                         CHANNEL_STATE_ERROR), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_OPEN,
+                                         CHANNEL_STATE_CLOSING), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_OPEN,
+                                         CHANNEL_STATE_ERROR), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_OPEN,
+                                         CHANNEL_STATE_MAINT), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_LAST,
+                                         CHANNEL_STATE_MAINT), OP_EQ, 0);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_LAST, INT_MAX),
+            OP_EQ, 0);
+
+  /* Test listener state transition. */
+  tt_int_op(channel_listener_state_can_transition(
+                                       CHANNEL_LISTENER_STATE_CLOSED,
+                                       CHANNEL_LISTENER_STATE_LISTENING),
+            OP_EQ, 1);
+  tt_int_op(channel_listener_state_can_transition(
+                                       CHANNEL_LISTENER_STATE_CLOSED,
+                                       CHANNEL_LISTENER_STATE_ERROR),
+            OP_EQ, 0);
+
+  tt_int_op(channel_listener_state_can_transition(
+                                       CHANNEL_LISTENER_STATE_CLOSING,
+                                       CHANNEL_LISTENER_STATE_CLOSED),
+            OP_EQ, 1);
+
+  tt_int_op(channel_listener_state_can_transition(
+                                       CHANNEL_LISTENER_STATE_CLOSING,
+                                       CHANNEL_LISTENER_STATE_ERROR),
+            OP_EQ, 1);
+  tt_int_op(channel_listener_state_can_transition(
+                                       CHANNEL_LISTENER_STATE_ERROR,
+                                       CHANNEL_LISTENER_STATE_CLOSING),
+            OP_EQ, 0);
+
+  tt_int_op(channel_listener_state_can_transition(
+                                       CHANNEL_LISTENER_STATE_LISTENING,
+                                       CHANNEL_LISTENER_STATE_CLOSING),
+            OP_EQ, 1);
+  tt_int_op(channel_listener_state_can_transition(
+                                       CHANNEL_LISTENER_STATE_LISTENING,
+                                       CHANNEL_LISTENER_STATE_ERROR),
+            OP_EQ, 1);
+  tt_int_op(channel_listener_state_can_transition(
+                                       CHANNEL_LISTENER_STATE_LAST,
+                                       INT_MAX),
+            OP_EQ, 0);
+
+  /* Test state string. */
+  tt_str_op(channel_state_to_string(CHANNEL_STATE_CLOSING), OP_EQ,
+            "closing");
+  tt_str_op(channel_state_to_string(CHANNEL_STATE_ERROR), OP_EQ,
+            "channel error");
+  tt_str_op(channel_state_to_string(CHANNEL_STATE_CLOSED), OP_EQ,
+            "closed");
+  tt_str_op(channel_state_to_string(CHANNEL_STATE_OPEN), OP_EQ,
+            "open");
+  tt_str_op(channel_state_to_string(CHANNEL_STATE_OPENING), OP_EQ,
+            "opening");
+  tt_str_op(channel_state_to_string(CHANNEL_STATE_MAINT), OP_EQ,
+            "temporarily suspended for maintenance");
+  tt_str_op(channel_state_to_string(CHANNEL_STATE_LAST), OP_EQ,
+            "unknown or invalid channel state");
+  tt_str_op(channel_state_to_string(INT_MAX), OP_EQ,
+            "unknown or invalid channel state");
+
+  /* Test listener state string. */
+  tt_str_op(channel_listener_state_to_string(CHANNEL_LISTENER_STATE_CLOSING),
+            OP_EQ, "closing");
+  tt_str_op(channel_listener_state_to_string(CHANNEL_LISTENER_STATE_ERROR),
+            OP_EQ, "channel listener error");
+  tt_str_op(channel_listener_state_to_string(CHANNEL_LISTENER_STATE_LISTENING),
+            OP_EQ, "listening");
+  tt_str_op(channel_listener_state_to_string(CHANNEL_LISTENER_STATE_LAST),
+            OP_EQ, "unknown or invalid channel listener state");
+  tt_str_op(channel_listener_state_to_string(INT_MAX),
+            OP_EQ, "unknown or invalid channel listener state");
 
  done:
-  free_fake_channel(ch);
+  ;
+}
 
-  /*
-   * Doing that meant that we couldn't correctly adjust the queue size
-   * for the var cell, so manually reset the global queue size estimate
-   * so the next test doesn't break if we run with --no-fork.
-   */
-  estimated_total_queue_size = global_queue_estimate;
+static networkstatus_t *mock_ns = NULL;
 
-  return;
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus(void)
+{
+  return mock_ns;
 }
 
 static void
-test_channel_queue_incoming(void *arg)
+test_channel_duplicates(void *arg)
 {
-  channel_t *ch = NULL;
-  cell_t *cell = NULL;
-  var_cell_t *var_cell = NULL;
-  int old_fixed_count, old_var_count;
-
-  (void)arg;
-
-  /* Mock these for duration of the test */
-  MOCK(scheduler_channel_doesnt_want_writes,
-       scheduler_channel_doesnt_want_writes_mock);
-  MOCK(scheduler_release_channel,
-       scheduler_release_channel_mock);
-
-  /* Accept cells to lower layer */
-  test_chan_accept_cells = 1;
-  /* Use default overhead factor */
-  test_overhead_estimate = 1.0;
-
-  ch = new_fake_channel();
-  tt_assert(ch);
-  /* Start it off in OPENING */
-  ch->state = CHANNEL_STATE_OPENING;
-  /* We'll need a cmux */
-  ch->cmux = circuitmux_alloc();
-
-  /* Test cell handler getters */
-  tt_ptr_op(channel_get_cell_handler(ch), ==, NULL);
-  tt_ptr_op(channel_get_var_cell_handler(ch), ==, NULL);
-
-  /* Try to register it */
-  channel_register(ch);
-  tt_assert(ch->registered);
-
-  /* Open it */
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_OPEN);
-
-  /* Assert that the incoming queue is empty */
-  tt_assert(TOR_SIMPLEQ_EMPTY(&(ch->incoming_queue)));
-
-  /* Queue an incoming fixed-length cell */
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  channel_queue_cell(ch, cell);
-
-  /* Assert that the incoming queue has one entry */
-  tt_int_op(chan_cell_queue_len(&(ch->incoming_queue)), ==, 1);
-
-  /* Queue an incoming var cell */
-  var_cell = tor_malloc_zero(sizeof(var_cell_t) + CELL_PAYLOAD_SIZE);
-  make_fake_var_cell(var_cell);
-  channel_queue_var_cell(ch, var_cell);
-
-  /* Assert that the incoming queue has two entries */
-  tt_int_op(chan_cell_queue_len(&(ch->incoming_queue)), ==, 2);
-
-  /*
-   * Install cell handlers; this will drain the queue, so save the old
-   * cell counters first
-   */
-  old_fixed_count = test_chan_fixed_cells_recved;
-  old_var_count = test_chan_var_cells_recved;
-  channel_set_cell_handlers(ch,
-                            chan_test_cell_handler,
-                            chan_test_var_cell_handler);
-  tt_ptr_op(channel_get_cell_handler(ch), ==, chan_test_cell_handler);
-  tt_ptr_op(channel_get_var_cell_handler(ch), ==, chan_test_var_cell_handler);
-
-  /* Assert cells were received */
-  tt_int_op(test_chan_fixed_cells_recved, ==, old_fixed_count + 1);
-  tt_int_op(test_chan_var_cells_recved, ==, old_var_count + 1);
-
-  /*
-   * Assert that the pointers are different from the cells we allocated;
-   * when queueing cells with no incoming cell handlers installed, the
-   * channel layer should copy them to a new buffer, and free them after
-   * delivery.  These pointers will have already been freed by the time
-   * we get here, so don't dereference them.
-   */
-  tt_ptr_op(test_chan_last_seen_fixed_cell_ptr, !=, cell);
-  tt_ptr_op(test_chan_last_seen_var_cell_ptr, !=, var_cell);
-
-  /* Assert queue is now empty */
-  tt_assert(TOR_SIMPLEQ_EMPTY(&(ch->incoming_queue)));
+  channel_t *chan = NULL;
+  routerstatus_t rs;
+
+  (void) arg;
+
+  setup_full_capture_of_logs(LOG_INFO);
+  /* Try a flat call with channel nor connections. */
+  channel_check_for_duplicates();
+  expect_log_msg_containing(
+    "Found 0 connections to 0 relays. Found 0 current canonical "
+    "connections, in 0 of which we were a non-canonical peer. "
+    "0 relays had more than 1 connection, 0 had more than 2, and "
+    "0 had more than 4 connections.");
+
+  mock_ns = tor_malloc_zero(sizeof(*mock_ns));
+  mock_ns->routerstatus_list = smartlist_new();
+  MOCK(networkstatus_get_latest_consensus,
+       mock_networkstatus_get_latest_consensus);
+
+  chan = new_fake_channel();
+  tt_assert(chan);
+  chan->is_canonical = test_chan_is_canonical;
+  memset(chan->identity_digest, 'A', sizeof(chan->identity_digest));
+  channel_add_to_digest_map(chan);
+  tt_ptr_op(channel_find_by_remote_identity(chan->identity_digest, NULL),
+            OP_EQ, chan);
+
+  /* No relay has been associated with this channel. */
+  channel_check_for_duplicates();
+  expect_log_msg_containing(
+    "Found 0 connections to 0 relays. Found 0 current canonical "
+    "connections, in 0 of which we were a non-canonical peer. "
+    "0 relays had more than 1 connection, 0 had more than 2, and "
+    "0 had more than 4 connections.");
+
+  /* Associate relay to this connection in the consensus. */
+  memset(&rs, 0, sizeof(rs));
+  memset(rs.identity_digest, 'A', sizeof(rs.identity_digest));
+  smartlist_add(mock_ns->routerstatus_list, &rs);
+
+  /* Non opened channel. */
+  chan->state = CHANNEL_STATE_CLOSING;
+  channel_check_for_duplicates();
+  expect_log_msg_containing(
+    "Found 0 connections to 0 relays. Found 0 current canonical "
+    "connections, in 0 of which we were a non-canonical peer. "
+    "0 relays had more than 1 connection, 0 had more than 2, and "
+    "0 had more than 4 connections.");
+  chan->state = CHANNEL_STATE_OPEN;
 
-  /* Close it; this contains an assertion that the incoming queue is empty */
-  channel_mark_for_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSING);
-  chan_test_finish_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSED);
-  channel_run_cleanup();
-  ch = NULL;
+  channel_check_for_duplicates();
+  expect_log_msg_containing(
+    "Found 1 connections to 1 relays. Found 0 current canonical "
+    "connections, in 0 of which we were a non-canonical peer. "
+    "0 relays had more than 1 connection, 0 had more than 2, and "
+    "0 had more than 4 connections.");
+
+  test_chan_should_be_canonical = 1;
+  channel_check_for_duplicates();
+  expect_log_msg_containing(
+    "Found 1 connections to 1 relays. Found 1 current canonical "
+    "connections, in 1 of which we were a non-canonical peer. "
+    "0 relays had more than 1 connection, 0 had more than 2, and "
+    "0 had more than 4 connections.");
+  teardown_capture_of_logs();
 
  done:
-  free_fake_channel(ch);
-  tor_free(cell);
-  tor_free(var_cell);
-
-  UNMOCK(scheduler_channel_doesnt_want_writes);
-  UNMOCK(scheduler_release_channel);
-
-  return;
+  free_fake_channel(chan);
+  smartlist_clear(mock_ns->routerstatus_list);
+  networkstatus_vote_free(mock_ns);
+  UNMOCK(networkstatus_get_latest_consensus);
 }
 
 static void
-test_channel_queue_size(void *arg)
+test_channel_for_extend(void *arg)
 {
-  channel_t *ch = NULL;
-  cell_t *cell = NULL;
-  int n, old_count;
-  uint64_t global_queue_estimate;
-
-  (void)arg;
-
-  ch = new_fake_channel();
-  tt_assert(ch);
-
-  /* Initial queue size update */
-  channel_update_xmit_queue_size(ch);
-  tt_u64_op(ch->bytes_queued_for_xmit, ==, 0);
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 0);
-
-  /* Test the call-through to our fake lower layer */
-  n = channel_num_cells_writeable(ch);
-  /* chan_test_num_cells_writeable() always returns 32 */
-  tt_int_op(n, ==, 32);
-
-  /*
-   * Now we queue some cells and check that channel_num_cells_writeable()
-   * adjusts properly
-   */
-
-  /* tell it not to accept cells */
-  test_chan_accept_cells = 0;
-  /* ...and keep it from trying to flush the queue */
-  ch->state = CHANNEL_STATE_MAINT;
-
-  /* Get a fresh cell */
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-
-  old_count = test_cells_written;
-  channel_write_cell(ch, cell);
-  /* Assert that it got queued, not written through, correctly */
-  tt_int_op(test_cells_written, ==, old_count);
-
-  /* Now check chan_test_num_cells_writeable() again */
-  n = channel_num_cells_writeable(ch);
-  tt_int_op(n, ==, 0); /* Should return 0 since we're in CHANNEL_STATE_MAINT */
-
-  /* Update queue size estimates */
-  channel_update_xmit_queue_size(ch);
-  /* One cell, times an overhead factor of 1.0 */
-  tt_u64_op(ch->bytes_queued_for_xmit, ==, 512);
-  /* Try a different overhead factor */
-  test_overhead_estimate = 0.5;
-  /* This one should be ignored since it's below 1.0 */
-  channel_update_xmit_queue_size(ch);
-  tt_u64_op(ch->bytes_queued_for_xmit, ==, 512);
-  /* Now try a larger one */
-  test_overhead_estimate = 2.0;
-  channel_update_xmit_queue_size(ch);
-  tt_u64_op(ch->bytes_queued_for_xmit, ==, 1024);
-  /* Go back to 1.0 */
-  test_overhead_estimate = 1.0;
-  channel_update_xmit_queue_size(ch);
-  tt_u64_op(ch->bytes_queued_for_xmit, ==, 512);
-  /* Check the global estimate too */
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 512);
-
-  /* Go to open */
-  old_count = test_cells_written;
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-
-  /*
-   * It should try to write, but we aren't accepting cells right now, so
-   * it'll requeue
-   */
-  tt_int_op(test_cells_written, ==, old_count);
-
-  /* Check the queue size again */
-  channel_update_xmit_queue_size(ch);
-  tt_u64_op(ch->bytes_queued_for_xmit, ==, 512);
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 512);
-
-  /*
-   * Now the cell is in the queue, and we're open, so we should get 31
-   * writeable cells.
-   */
-  n = channel_num_cells_writeable(ch);
-  tt_int_op(n, ==, 31);
-
-  /* Accept cells again */
-  test_chan_accept_cells = 1;
-  /* ...and re-process the queue */
-  old_count = test_cells_written;
-  channel_flush_cells(ch);
-  tt_int_op(test_cells_written, ==, old_count + 1);
-
-  /* Should have 32 writeable now */
-  n = channel_num_cells_writeable(ch);
-  tt_int_op(n, ==, 32);
-
-  /* Should have queue size estimate of zero */
-  channel_update_xmit_queue_size(ch);
-  tt_u64_op(ch->bytes_queued_for_xmit, ==, 0);
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 0);
-
-  /* Okay, now we're done with this one */
-  MOCK(scheduler_release_channel, scheduler_release_channel_mock);
-  channel_mark_for_close(ch);
-  UNMOCK(scheduler_release_channel);
+  channel_t *chan1 = NULL, *chan2 = NULL;
+  channel_t *ret_chan = NULL;
+  char digest[DIGEST_LEN];
+  ed25519_public_key_t ed_id;
+  tor_addr_t addr;
+  const char *msg;
+  int launch;
+  time_t now = time(NULL);
+
+  (void) arg;
+
+  memset(digest, 'A', sizeof(digest));
+  memset(&ed_id, 'B', sizeof(ed_id));
+
+  chan1 = new_fake_channel();
+  tt_assert(chan1);
+  /* Need to be registered to get added to the id map. */
+  channel_register(chan1);
+  tt_int_op(chan1->registered, OP_EQ, 1);
+  /* We need those for the test. */
+  chan1->is_canonical = test_chan_is_canonical;
+  chan1->matches_target = test_chan_matches_target;
+  chan1->timestamp_created = now - 9;
+
+  chan2 = new_fake_channel();
+  tt_assert(chan2);
+  /* Need to be registered to get added to the id map. */
+  channel_register(chan2);
+  tt_int_op(chan2->registered, OP_EQ, 1);
+  /* We need those for the test. */
+  chan2->is_canonical = test_chan_is_canonical;
+  chan2->matches_target = test_chan_matches_target;
+  /* Make it older than chan1. */
+  chan2->timestamp_created = chan1->timestamp_created - 1;
+
+  /* Set channel identities and add it to the channel map. The last one to be
+   * added is made the first one in the list so the lookup will always return
+   * that one first. */
+  channel_set_identity_digest(chan2, digest, &ed_id);
+  channel_set_identity_digest(chan1, digest, &ed_id);
+  tt_ptr_op(channel_find_by_remote_identity(digest, NULL), OP_EQ, chan1);
+  tt_ptr_op(channel_find_by_remote_identity(digest, &ed_id), OP_EQ, chan1);
+
+  /* The expected result is chan2 because it is older than chan1. */
+  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  tt_assert(ret_chan);
+  tt_ptr_op(ret_chan, OP_EQ, chan2);
+  tt_int_op(launch, OP_EQ, 0);
+  tt_str_op(msg, OP_EQ, "Connection is fine; using it.");
+
+  /* Switch that around from previous test. */
+  chan2->timestamp_created = chan1->timestamp_created + 1;
+  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  tt_assert(ret_chan);
+  tt_ptr_op(ret_chan, OP_EQ, chan1);
+  tt_int_op(launch, OP_EQ, 0);
+  tt_str_op(msg, OP_EQ, "Connection is fine; using it.");
+
+  /* Same creation time, num circuits will be used and they both have 0 so the
+   * channel 2 should be picked due to how channel_is_better() work. */
+  chan2->timestamp_created = chan1->timestamp_created;
+  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  tt_assert(ret_chan);
+  tt_ptr_op(ret_chan, OP_EQ, chan1);
+  tt_int_op(launch, OP_EQ, 0);
+  tt_str_op(msg, OP_EQ, "Connection is fine; using it.");
+
+  /* For the rest of the tests, we need channel 1 to be the older. */
+  chan2->timestamp_created = chan1->timestamp_created + 1;
+
+  /* Condemned the older channel. */
+  chan1->state = CHANNEL_STATE_CLOSING;
+  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  tt_assert(ret_chan);
+  tt_ptr_op(ret_chan, OP_EQ, chan2);
+  tt_int_op(launch, OP_EQ, 0);
+  tt_str_op(msg, OP_EQ, "Connection is fine; using it.");
+  chan1->state = CHANNEL_STATE_OPEN;
+
+  /* Make the older channel a client one. */
+  channel_mark_client(chan1);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  tt_assert(ret_chan);
+  tt_ptr_op(ret_chan, OP_EQ, chan2);
+  tt_int_op(launch, OP_EQ, 0);
+  tt_str_op(msg, OP_EQ, "Connection is fine; using it.");
+  channel_clear_client(chan1);
+
+  /* Non matching ed identity with valid digest. */
+  ed25519_public_key_t dumb_ed_id;
+  memset(&dumb_ed_id, 0, sizeof(dumb_ed_id));
+  ret_chan = channel_get_for_extend(digest, &dumb_ed_id, &addr, &msg,
+                                    &launch);
+  tt_assert(!ret_chan);
+  tt_str_op(msg, OP_EQ, "Not connected. Connecting.");
+  tt_int_op(launch, OP_EQ, 1);
+
+  /* Opening channel, we'll check if the target address matches. */
+  test_chan_should_match_target = 1;
+  chan1->state = CHANNEL_STATE_OPENING;
+  chan2->state = CHANNEL_STATE_OPENING;
+  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  tt_assert(!ret_chan);
+  tt_str_op(msg, OP_EQ, "Connection in progress; waiting.");
+  tt_int_op(launch, OP_EQ, 0);
+  chan1->state = CHANNEL_STATE_OPEN;
+  chan2->state = CHANNEL_STATE_OPEN;
+
+  /* Mark channel 1 as bad for circuits. */
+  channel_mark_bad_for_new_circs(chan1);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  tt_assert(ret_chan);
+  tt_ptr_op(ret_chan, OP_EQ, chan2);
+  tt_int_op(launch, OP_EQ, 0);
+  tt_str_op(msg, OP_EQ, "Connection is fine; using it.");
+  chan1->is_bad_for_new_circs = 0;
+
+  /* Mark both channels as unusable. */
+  channel_mark_bad_for_new_circs(chan1);
+  channel_mark_bad_for_new_circs(chan2);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  tt_assert(!ret_chan);
+  tt_str_op(msg, OP_EQ, "Connections all too old, or too non-canonical. "
+                        " Launching a new one.");
+  tt_int_op(launch, OP_EQ, 1);
+  chan1->is_bad_for_new_circs = 0;
+  chan2->is_bad_for_new_circs = 0;
+
+  /* Non canonical channels. */
+  test_chan_should_match_target = 0;
+  test_chan_canonical_should_be_reliable = 1;
+  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  tt_assert(!ret_chan);
+  tt_str_op(msg, OP_EQ, "Connections all too old, or too non-canonical. "
+                        " Launching a new one.");
+  tt_int_op(launch, OP_EQ, 1);
 
  done:
-  free_fake_channel(ch);
-
-  return;
+  free_fake_channel(chan1);
+  free_fake_channel(chan2);
 }
 
 static void
-test_channel_write(void *arg)
+test_channel_listener(void *arg)
 {
-  channel_t *ch = NULL;
-  cell_t *cell = tor_malloc_zero(sizeof(cell_t));
-  packed_cell_t *packed_cell = NULL;
-  var_cell_t *var_cell =
-    tor_malloc_zero(sizeof(var_cell_t) + CELL_PAYLOAD_SIZE);
   int old_count;
-
-  (void)arg;
-
-  packed_cell = packed_cell_new();
-  tt_assert(packed_cell);
-
-  ch = new_fake_channel();
-  tt_assert(ch);
-  make_fake_cell(cell);
-  make_fake_var_cell(var_cell);
-
-  /* Tell it to accept cells */
-  test_chan_accept_cells = 1;
-
-  old_count = test_cells_written;
-  channel_write_cell(ch, cell);
-  cell = NULL;
-  tt_assert(test_cells_written == old_count + 1);
-
-  channel_write_var_cell(ch, var_cell);
-  var_cell = NULL;
-  tt_assert(test_cells_written == old_count + 2);
-
-  channel_write_packed_cell(ch, packed_cell);
-  packed_cell = NULL;
-  tt_assert(test_cells_written == old_count + 3);
-
-  /* Now we test queueing; tell it not to accept cells */
-  test_chan_accept_cells = 0;
-  /* ...and keep it from trying to flush the queue */
-  ch->state = CHANNEL_STATE_MAINT;
-
-  /* Get a fresh cell */
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-
-  old_count = test_cells_written;
-  channel_write_cell(ch, cell);
-  tt_assert(test_cells_written == old_count);
-
-  /*
-   * Now change back to open with channel_change_state() and assert that it
-   * gets drained from the queue.
-   */
-  test_chan_accept_cells = 1;
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-  tt_assert(test_cells_written == old_count + 1);
-
-  /*
-   * Check the note destroy case
-   */
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  cell->command = CELL_DESTROY;
-
-  /* Set up the mock */
-  MOCK(channel_note_destroy_not_pending,
-       channel_note_destroy_not_pending_mock);
-
-  old_count = test_destroy_not_pending_calls;
-  channel_write_cell(ch, cell);
-  tt_assert(test_destroy_not_pending_calls == old_count + 1);
-
-  /* Now send a non-destroy and check we don't call it */
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  channel_write_cell(ch, cell);
-  tt_assert(test_destroy_not_pending_calls == old_count + 1);
-
-  UNMOCK(channel_note_destroy_not_pending);
-
-  /*
-   * Now switch it to CLOSING so we can test the discard-cells case
-   * in the channel_write_*() functions.
-   */
-  MOCK(scheduler_release_channel, scheduler_release_channel_mock);
-  channel_mark_for_close(ch);
-  UNMOCK(scheduler_release_channel);
-
-  /* Send cells that will drop in the closing state */
-  old_count = test_cells_written;
-
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  channel_write_cell(ch, cell);
-  cell = NULL;
-  tt_assert(test_cells_written == old_count);
-
-  var_cell = tor_malloc_zero(sizeof(var_cell_t) + CELL_PAYLOAD_SIZE);
-  make_fake_var_cell(var_cell);
-  channel_write_var_cell(ch, var_cell);
-  var_cell = NULL;
-  tt_assert(test_cells_written == old_count);
-
-  packed_cell = packed_cell_new();
-  channel_write_packed_cell(ch, packed_cell);
-  packed_cell = NULL;
-  tt_assert(test_cells_written == old_count);
+  time_t now = time(NULL);
+  channel_listener_t *chan = NULL;
+
+  (void) arg;
+
+  chan = tor_malloc_zero(sizeof(*chan));
+  tt_assert(chan);
+  channel_init_listener(chan);
+  tt_u64_op(chan->global_identifier, OP_EQ, 1);
+  tt_int_op(chan->timestamp_created, OP_GE, now);
+  chan->close = test_chan_listener_close;
+
+  /* Register it. At this point, it is not open so it will be put in the
+   * finished list. */
+  channel_listener_register(chan);
+  tt_int_op(chan->registered, OP_EQ, 1);
+  channel_listener_unregister(chan);
+
+  /* Register it as listening now thus active. */
+  chan->state = CHANNEL_LISTENER_STATE_LISTENING;
+  channel_listener_register(chan);
+  tt_int_op(chan->registered, OP_EQ, 1);
+
+  /* Set the listener function. */
+  channel_listener_set_listener_fn(chan, test_chan_listener_fn);
+  tt_ptr_op(chan->listener, OP_EQ, test_chan_listener_fn);
+
+  /* Put a channel in the listener incoming list and queue it.
+   * function. By doing this, the listener() handler will be called. */
+  channel_t *in_chan = new_fake_channel();
+  old_count = test_chan_listener_fn_called;
+  channel_listener_queue_incoming(chan, in_chan);
+  free_fake_channel(in_chan);
+  tt_int_op(test_chan_listener_fn_called, OP_EQ, old_count + 1);
+
+  /* Put listener channel in CLOSING state. */
+  old_count = test_chan_listener_close_fn_called;
+  channel_listener_mark_for_close(chan);
+  tt_int_op(test_chan_listener_close_fn_called, OP_EQ, old_count + 1);
+  channel_listener_change_state(chan, CHANNEL_LISTENER_STATE_CLOSED);
+
+  /* Dump stats so we at least hit the code path. */
+  chan->describe_transport = test_chan_listener_describe_transport;
+  /* There is a check for "now > timestamp_created" when dumping the stats so
+   * make sure we go in. */
+  chan->timestamp_created = now - 10;
+  channel_listener_dump_statistics(chan, LOG_INFO);
 
  done:
-  free_fake_channel(ch);
-  tor_free(var_cell);
-  tor_free(cell);
-  packed_cell_free(packed_cell);
-  return;
+  channel_free_all();
 }
 
 struct testcase_t channel_tests[] = {
-  { "dumpstats", test_channel_dumpstats, TT_FORK, NULL, NULL },
-  { "flush", test_channel_flush, TT_FORK, NULL, NULL },
-  { "flushmux", test_channel_flushmux, TT_FORK, NULL, NULL },
-  { "incoming", test_channel_incoming, TT_FORK, NULL, NULL },
-  { "lifecycle", test_channel_lifecycle, TT_FORK, NULL, NULL },
-  { "lifecycle_2", test_channel_lifecycle_2, TT_FORK, NULL, NULL },
-  { "multi", test_channel_multi, TT_FORK, NULL, NULL },
-  { "queue_impossible", test_channel_queue_impossible, TT_FORK, NULL, NULL },
-  { "queue_incoming", test_channel_queue_incoming, TT_FORK, NULL, NULL },
-  { "queue_size", test_channel_queue_size, TT_FORK, NULL, NULL },
-  { "write", test_channel_write, TT_FORK, NULL, NULL },
+  { "inbound_cell", test_channel_inbound_cell, TT_FORK,
+    NULL, NULL },
+  { "outbound_cell", test_channel_outbound_cell, TT_FORK,
+    NULL, NULL },
+  { "id_map", test_channel_id_map, TT_FORK,
+    NULL, NULL },
+  { "lifecycle", test_channel_lifecycle, TT_FORK,
+    NULL, NULL },
+  { "lifecycle_2", test_channel_lifecycle_2, TT_FORK,
+    NULL, NULL },
+  { "dumpstats", test_channel_dumpstats, TT_FORK,
+    NULL, NULL },
+  { "state", test_channel_state, TT_FORK,
+    NULL, NULL },
+  { "duplicates", test_channel_duplicates, TT_FORK,
+    NULL, NULL },
+  { "get_channel_for_extend", test_channel_for_extend, TT_FORK,
+    NULL, NULL },
+  { "listener", test_channel_listener, TT_FORK,
+    NULL, NULL },
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_channelpadding.c b/src/test/test_channelpadding.c
new file mode 100644
index 0000000000..5da2d81377
--- /dev/null
+++ b/src/test/test_channelpadding.c
@@ -0,0 +1,1104 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define TOR_CHANNEL_INTERNAL_
+#define MAINLOOP_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+#define TOR_TIMERS_PRIVATE
+#include "core/or/or.h"
+#include "test/test.h"
+#include "lib/testsupport/testsupport.h"
+#include "core/mainloop/connection.h"
+#include "core/or/connection_or.h"
+#include "core/or/channel.h"
+#include "core/or/channeltls.h"
+#include "core/or/channelpadding.h"
+#include "lib/evloop/compat_libevent.h"
+#include "app/config/config.h"
+#include "lib/time/compat_time.h"
+#include "core/mainloop/mainloop.h"
+#include "feature/nodelist/networkstatus.h"
+#include "test/log_test_helpers.h"
+#include "lib/tls/tortls.h"
+#include "lib/evloop/timers.h"
+#include "lib/container/buffers.h"
+
+#include "core/or/cell_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "core/or/or_connection_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+int channelpadding_get_netflow_inactive_timeout_ms(channel_t *chan);
+int64_t channelpadding_compute_time_until_pad_for_netflow(channel_t *chan);
+int channelpadding_send_disable_command(channel_t*);
+int channelpadding_find_timerslot(channel_t *chan);
+
+void test_channelpadding_timers(void *arg);
+void test_channelpadding_consensus(void *arg);
+void test_channelpadding_negotiation(void *arg);
+void test_channelpadding_decide_to_pad_channel(void *arg);
+void test_channelpadding_killonehop(void *arg);
+
+void dummy_nop_timer(void);
+
+#define NSEC_PER_MSEC (1000*1000)
+
+/* Thing to cast to fake tor_tls_t * to appease assert_connection_ok() */
+static int fake_tortls = 0; /* Bleh... */
+
+static int dont_stop_libevent = 0;
+
+// From test_channel.c
+channel_t * new_fake_channel(void);
+void free_fake_channel(channel_t*);
+
+static int
+mock_channel_has_queued_writes(channel_t *chan)
+{
+  (void)chan;
+  return 0;
+}
+
+static int tried_to_write_cell = 0;
+
+static channel_t *relay1_relay2;
+static channel_t *relay2_relay1;
+static channel_t *relay3_client;
+static channel_t *client_relay3;
+
+static int
+mock_channel_write_cell_relay2(channel_t *chan, cell_t *cell)
+{
+  (void)chan;
+  tried_to_write_cell++;
+  channel_tls_handle_cell(cell, ((channel_tls_t*)relay1_relay2)->conn);
+  tor_libevent_exit_loop_after_callback(tor_libevent_get_base());
+  return 0;
+}
+
+static int
+mock_channel_write_cell_relay1(channel_t *chan, cell_t *cell)
+{
+  (void)chan;
+  tried_to_write_cell++;
+  channel_tls_handle_cell(cell, ((channel_tls_t*)relay2_relay1)->conn);
+  tor_libevent_exit_loop_after_callback(tor_libevent_get_base());
+  return 0;
+}
+
+static int
+mock_channel_write_cell_relay3(channel_t *chan, cell_t *cell)
+{
+  (void)chan;
+  tried_to_write_cell++;
+  channel_tls_handle_cell(cell, ((channel_tls_t*)client_relay3)->conn);
+  tor_libevent_exit_loop_after_callback(tor_libevent_get_base());
+  return 0;
+}
+
+static int
+mock_channel_write_cell_client(channel_t *chan, cell_t *cell)
+{
+  (void)chan;
+  tried_to_write_cell++;
+  channel_tls_handle_cell(cell, ((channel_tls_t*)relay3_client)->conn);
+  tor_libevent_exit_loop_after_callback(tor_libevent_get_base());
+  return 0;
+}
+
+static int
+mock_channel_write_cell(channel_t *chan, cell_t *cell)
+{
+  tried_to_write_cell++;
+  channel_tls_handle_cell(cell, ((channel_tls_t*)chan)->conn);
+  if (!dont_stop_libevent)
+    tor_libevent_exit_loop_after_callback(tor_libevent_get_base());
+  return 0;
+}
+
+static void
+setup_fake_connection_for_channel(channel_tls_t *chan)
+{
+  or_connection_t *conn = (or_connection_t*)connection_new(CONN_TYPE_OR,
+                                                           AF_INET);
+
+  conn->base_.conn_array_index = smartlist_len(connection_array);
+  smartlist_add(connection_array, conn);
+
+  conn->chan = chan;
+  chan->conn = conn;
+
+  conn->base_.magic = OR_CONNECTION_MAGIC;
+  conn->base_.state = OR_CONN_STATE_OPEN;
+  conn->base_.type = CONN_TYPE_OR;
+  conn->base_.socket_family = AF_INET;
+  conn->base_.address = tor_strdup("<fake>");
+
+  conn->base_.port = 4242;
+
+  conn->tls = (tor_tls_t *)((void *)(&fake_tortls));
+
+  conn->link_proto = MIN_LINK_PROTO_FOR_CHANNEL_PADDING;
+
+  connection_or_set_canonical(conn, 1);
+}
+
+static channel_tls_t *
+new_fake_channeltls(uint8_t id)
+{
+  channel_tls_t *chan = tor_realloc(new_fake_channel(), sizeof(channel_tls_t));
+  chan->base_.magic = TLS_CHAN_MAGIC;
+  setup_fake_connection_for_channel(chan);
+  chan->base_.channel_usage = CHANNEL_USED_FOR_FULL_CIRCS;
+  chan->base_.has_queued_writes = mock_channel_has_queued_writes;
+  chan->base_.write_cell = mock_channel_write_cell;
+  chan->base_.padding_enabled = 1;
+
+  chan->base_.identity_digest[0] = id;
+  channel_register(&chan->base_);
+
+  return chan;
+}
+
+static void
+free_fake_channeltls(channel_tls_t *chan)
+{
+  channel_unregister(&chan->base_);
+
+  tor_free(((channel_tls_t*)chan)->conn->base_.address);
+  buf_free(((channel_tls_t*)chan)->conn->base_.inbuf);
+  buf_free(((channel_tls_t*)chan)->conn->base_.outbuf);
+  tor_free(((channel_tls_t*)chan)->conn);
+
+  timer_free(chan->base_.padding_timer);
+  channel_handle_free(chan->base_.timer_handle);
+  channel_handles_clear(&chan->base_);
+
+  free_fake_channel(&chan->base_);
+
+  return;
+}
+
+static void
+setup_mock_consensus(void)
+{
+  current_md_consensus = current_ns_consensus
+        = tor_malloc_zero(sizeof(networkstatus_t));
+  current_md_consensus->net_params = smartlist_new();
+  current_md_consensus->routerstatus_list = smartlist_new();
+  channelpadding_new_consensus_params(current_md_consensus);
+}
+
+static void
+free_mock_consensus(void)
+{
+  SMARTLIST_FOREACH(current_md_consensus->routerstatus_list, void *, r,
+                    tor_free(r));
+  smartlist_free(current_md_consensus->routerstatus_list);
+  smartlist_free(current_ns_consensus->net_params);
+  tor_free(current_ns_consensus);
+}
+
+static void
+setup_mock_network(void)
+{
+  routerstatus_t *relay;
+  if (!connection_array)
+    connection_array = smartlist_new();
+
+  relay1_relay2 = (channel_t*)new_fake_channeltls(2);
+  relay1_relay2->write_cell = mock_channel_write_cell_relay1;
+  channel_timestamp_active(relay1_relay2);
+  relay = tor_malloc_zero(sizeof(routerstatus_t));
+  relay->identity_digest[0] = 1;
+  smartlist_add(current_md_consensus->routerstatus_list, relay);
+
+  relay2_relay1 = (channel_t*)new_fake_channeltls(1);
+  relay2_relay1->write_cell = mock_channel_write_cell_relay2;
+  channel_timestamp_active(relay2_relay1);
+  relay = tor_malloc_zero(sizeof(routerstatus_t));
+  relay->identity_digest[0] = 2;
+  smartlist_add(current_md_consensus->routerstatus_list, relay);
+
+  relay3_client = (channel_t*)new_fake_channeltls(0);
+  relay3_client->write_cell = mock_channel_write_cell_relay3;
+  relay3_client->is_client = 1;
+  channel_timestamp_active(relay3_client);
+  relay = tor_malloc_zero(sizeof(routerstatus_t));
+  relay->identity_digest[0] = 3;
+  smartlist_add(current_md_consensus->routerstatus_list, relay);
+
+  client_relay3 = (channel_t*)new_fake_channeltls(3);
+  client_relay3->write_cell = mock_channel_write_cell_client;
+  channel_timestamp_active(client_relay3);
+
+  channel_do_open_actions(relay1_relay2);
+  channel_do_open_actions(relay2_relay1);
+  channel_do_open_actions(relay3_client);
+  channel_do_open_actions(client_relay3);
+}
+
+static void
+free_mock_network(void)
+{
+  free_fake_channeltls((channel_tls_t*)relay1_relay2);
+  free_fake_channeltls((channel_tls_t*)relay2_relay1);
+  free_fake_channeltls((channel_tls_t*)relay3_client);
+  free_fake_channeltls((channel_tls_t*)client_relay3);
+
+  smartlist_free(connection_array);
+}
+
+static void
+dummy_timer_cb(tor_timer_t *t, void *arg, const monotime_t *now_mono)
+{
+  (void)t; (void)arg; (void)now_mono;
+  tor_libevent_exit_loop_after_callback(tor_libevent_get_base());
+  return;
+}
+
+// This hack adds a dummy timer so that the libevent base loop
+// actually returns when we don't expect any timers to fire. Otherwise,
+// the global_timer_event gets scheduled an hour from now, and the
+// base loop never returns.
+void
+dummy_nop_timer(void)
+{
+  tor_timer_t *dummy_timer = timer_new(dummy_timer_cb, NULL);
+  struct timeval timeout;
+  timeout.tv_sec = 1;
+  timeout.tv_usec = 0;
+
+  timer_schedule(dummy_timer, &timeout);
+
+  tor_libevent_run_event_loop(tor_libevent_get_base(), 0);
+
+  timer_free(dummy_timer);
+}
+
+#define CHANNELPADDING_MAX_TIMERS 25
+#define CHANNELS_TO_TEST (CHANNELPADDING_MAX_TIMERS*4)
+/**
+ * Tests to ensure that we handle more than the max number of pending
+ * timers properly.
+ */
+void
+test_channelpadding_timers(void *arg)
+{
+  channelpadding_decision_t decision;
+  channel_t *chans[CHANNELS_TO_TEST];
+  (void)arg;
+
+  tor_libevent_postfork();
+
+  if (!connection_array)
+    connection_array = smartlist_new();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  uint64_t nsec_mock = 1;
+  monotime_set_mock_time_nsec(nsec_mock);
+  monotime_coarse_set_mock_time_nsec(nsec_mock);
+
+  timers_initialize();
+  channelpadding_new_consensus_params(NULL);
+
+  for (int i = 0; i < CHANNELS_TO_TEST; i++) {
+    chans[i] = (channel_t*)new_fake_channeltls(0);
+    channel_timestamp_active(chans[i]);
+  }
+
+  for (int j = 0; j < 2; j++) {
+    tried_to_write_cell = 0;
+    int i = 0;
+
+    monotime_coarse_t now;
+    monotime_coarse_get(&now);
+
+    /* This loop fills our timerslot array with timers of increasing time
+     * until they fire */
+    for (; i < CHANNELPADDING_MAX_TIMERS; i++) {
+      monotime_coarse_add_msec(&chans[i]->next_padding_time,
+                        &now, 10 + i*4);
+      decision = channelpadding_decide_to_pad_channel(chans[i]);
+      tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+      tt_assert(chans[i]->pending_padding_callback);
+      tt_int_op(tried_to_write_cell, OP_EQ, 0);
+    }
+
+    /* This loop should add timers to the first position in the timerslot
+     * array, since its timeout is before all other timers. */
+    for (; i < CHANNELS_TO_TEST/3; i++) {
+      monotime_coarse_add_msec(&chans[i]->next_padding_time,
+                        &now, 1);
+      decision = channelpadding_decide_to_pad_channel(chans[i]);
+      tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+      tt_assert(chans[i]->pending_padding_callback);
+      tt_int_op(tried_to_write_cell, OP_EQ, 0);
+    }
+
+    /* This loop should add timers to our existing lists in a weak
+     * pseudorandom pattern.  It ensures that the lists can grow with multiple
+     * timers in them. */
+    for (; i < CHANNELS_TO_TEST/2; i++) {
+      monotime_coarse_add_msec(&chans[i]->next_padding_time,
+                        &now, 10 + i*3 % CHANNELPADDING_MAX_TIMERS);
+      decision = channelpadding_decide_to_pad_channel(chans[i]);
+      tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+      tt_assert(chans[i]->pending_padding_callback);
+      tt_int_op(tried_to_write_cell, OP_EQ, 0);
+    }
+
+    /* This loop should add timers to the last position in the timerslot
+     * array, since its timeout is after all other timers. */
+    for (; i < CHANNELS_TO_TEST; i++) {
+      monotime_coarse_add_msec(&chans[i]->next_padding_time,
+                               &now, 500 + i % CHANNELPADDING_MAX_TIMERS);
+      decision = channelpadding_decide_to_pad_channel(chans[i]);
+      tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+      tt_assert(chans[i]->pending_padding_callback);
+      tt_int_op(tried_to_write_cell, OP_EQ, 0);
+    }
+
+    // Wait for the timers and then kill the event loop.
+    nsec_mock += 1001 * NSEC_PER_MSEC;
+    monotime_coarse_set_mock_time_nsec(nsec_mock);
+    monotime_set_mock_time_nsec(nsec_mock);
+    timers_run_pending();
+
+    tt_int_op(tried_to_write_cell, OP_EQ, CHANNELS_TO_TEST);
+
+    // Test that we have no pending callbacks and all empty slots now
+    for (i = 0; i < CHANNELS_TO_TEST; i++) {
+      tt_assert(!chans[i]->pending_padding_callback);
+    }
+  }
+
+ done:
+  for (int i = 0; i < CHANNELS_TO_TEST; i++) {
+    free_fake_channeltls((channel_tls_t*)chans[i]);
+  }
+  smartlist_free(connection_array);
+
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  channel_free_all();
+
+  return;
+}
+
+void
+test_channelpadding_killonehop(void *arg)
+{
+  channelpadding_decision_t decision;
+  int64_t new_time;
+  (void)arg;
+  tor_libevent_postfork();
+
+  routerstatus_t *relay = tor_malloc_zero(sizeof(routerstatus_t));
+  monotime_init();
+  monotime_enable_test_mocking();
+  monotime_set_mock_time_nsec(1);
+  monotime_coarse_set_mock_time_nsec(1);
+  new_time = 1;
+
+  timers_initialize();
+  setup_mock_consensus();
+  setup_mock_network();
+
+  /* Do we disable padding if rsos is enabled, and the consensus says don't
+   * pad?  */
+
+  monotime_coarse_t now;
+  monotime_coarse_get(&now);
+
+  // First, test that padding works if either is enabled
+  smartlist_clear(current_md_consensus->net_params);
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  relay3_client->padding_enabled = 1;
+  client_relay3->padding_enabled = 1;
+
+  tried_to_write_cell = 0;
+  get_options_mutable()->ORPort_set = 0;
+  get_options_mutable()->HiddenServiceSingleHopMode = 1;
+  get_options_mutable()->HiddenServiceNonAnonymousMode = 1;
+
+  monotime_coarse_add_msec(&client_relay3->next_padding_time, &now, 100);
+  decision = channelpadding_decide_to_pad_channel(client_relay3);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(client_relay3->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  decision = channelpadding_decide_to_pad_channel(client_relay3);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_ALREADY_SCHEDULED);
+
+  // Wait for the timer
+  new_time += 101 * NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!client_relay3->pending_padding_callback);
+
+  // Then test disabling each via consensus param
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_pad_single_onion=0");
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  // Before the client tries to pad, the relay will still pad:
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&relay3_client->next_padding_time, &now, 100);
+  get_options_mutable()->ORPort_set = 1;
+  get_options_mutable()->HiddenServiceSingleHopMode = 0;
+  get_options_mutable()->HiddenServiceNonAnonymousMode = 0;
+  decision = channelpadding_decide_to_pad_channel(relay3_client);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(relay3_client->pending_padding_callback);
+
+  // Wait for the timer
+  new_time += 101 * NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!client_relay3->pending_padding_callback);
+
+  // Test client side (it should stop immediately)
+  get_options_mutable()->HiddenServiceSingleHopMode = 1;
+  get_options_mutable()->HiddenServiceNonAnonymousMode = 1;
+  /* For the relay to receive the negotiate: */
+  get_options_mutable()->ORPort_set = 1;
+  decision = channelpadding_decide_to_pad_channel(client_relay3);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_assert(!client_relay3->pending_padding_callback);
+
+  // Test relay side (it should have gotten the negotiation to disable)
+  get_options_mutable()->ORPort_set = 1;
+  get_options_mutable()->HiddenServiceSingleHopMode = 0;
+  get_options_mutable()->HiddenServiceNonAnonymousMode = 0;
+  tt_int_op(channelpadding_decide_to_pad_channel(relay3_client), OP_EQ,
+      CHANNELPADDING_WONTPAD);
+  tt_assert(!relay3_client->padding_enabled);
+
+ done:
+  free_mock_consensus();
+  free_mock_network();
+  tor_free(relay);
+
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  channel_free_all();
+}
+
+void
+test_channelpadding_consensus(void *arg)
+{
+  channelpadding_decision_t decision;
+  or_options_t *options = get_options_mutable();
+  int64_t val;
+  int64_t new_time;
+  (void)arg;
+
+  tor_libevent_postfork();
+
+  /*
+   * Params tested:
+   *   nf_pad_before_usage
+   *   nf_pad_relays
+   *   nf_ito_low
+   *   nf_ito_high
+   *
+   * Plan:
+   * 1. Padding can be completely disabled via consensus
+   * 2. Negotiation can't re-enable consensus-disabled padding
+   * 3. Negotiation can't increase padding from relays beyond
+   *    consensus defaults
+   * 4. Relay-to-relay padding can be enabled/disabled in consensus
+   * 5. Can enable/disable padding before actually using a connection
+   * 6. Can we control circ and TLS conn lifetime from the consensus?
+   */
+  channel_t *chan;
+  routerstatus_t *relay = tor_malloc_zero(sizeof(routerstatus_t));
+  monotime_enable_test_mocking();
+  monotime_set_mock_time_nsec(1);
+  monotime_coarse_set_mock_time_nsec(1);
+  new_time = 1;
+  monotime_coarse_t now;
+  monotime_coarse_get(&now);
+  timers_initialize();
+
+  if (!connection_array)
+    connection_array = smartlist_new();
+  chan = (channel_t*)new_fake_channeltls(0);
+  channel_timestamp_active(chan);
+
+  setup_mock_consensus();
+
+  get_options_mutable()->ORPort_set = 1;
+
+  /* Test 1: Padding can be completely disabled via consensus */
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now, 100);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(chan->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_ALREADY_SCHEDULED);
+
+  // Wait for the timer
+  new_time += 101*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!chan->pending_padding_callback);
+
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_ito_low=0");
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_ito_high=0");
+  get_options_mutable()->ConnectionPadding = 1;
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_assert(!chan->pending_padding_callback);
+  val = channelpadding_get_netflow_inactive_timeout_ms(chan);
+  tt_i64_op(val, OP_EQ, 0);
+  val = channelpadding_compute_time_until_pad_for_netflow(chan);
+  tt_i64_op(val, OP_EQ, -2);
+
+  /* Test 2: Negotiation can't re-enable consensus-disabled padding */
+  channelpadding_send_enable_command(chan, 100, 200);
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_assert(!chan->pending_padding_callback);
+  val = channelpadding_get_netflow_inactive_timeout_ms(chan);
+  tt_i64_op(val, OP_EQ, 0);
+  val = channelpadding_compute_time_until_pad_for_netflow(chan);
+  tt_i64_op(val, OP_EQ, -2);
+  tt_assert(monotime_coarse_is_zero(&chan->next_padding_time));
+
+  smartlist_clear(current_md_consensus->net_params);
+
+  /* Test 3: Negotiation can't increase padding from relays beyond consensus
+   * values */
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_ito_low=100");
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_ito_high=200");
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now, 100);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(chan->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  val = channelpadding_get_netflow_inactive_timeout_ms(chan);
+  tt_i64_op(val, OP_GE, 100);
+  tt_i64_op(val, OP_LE, 200);
+  val = channelpadding_compute_time_until_pad_for_netflow(chan);
+  tt_i64_op(val, OP_LE, 200);
+
+  // Wait for the timer
+  new_time += 201*NSEC_PER_MSEC;
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!chan->pending_padding_callback);
+
+  smartlist_clear(current_md_consensus->net_params);
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_ito_low=1500");
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_ito_high=4500");
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  channelpadding_send_enable_command(chan, 100, 200);
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADLATER);
+  tt_assert(!chan->pending_padding_callback);
+  val = channelpadding_get_netflow_inactive_timeout_ms(chan);
+  tt_i64_op(val, OP_GE, 1500);
+  tt_i64_op(val, OP_LE, 4500);
+  val = channelpadding_compute_time_until_pad_for_netflow(chan);
+  tt_i64_op(val, OP_LE, 4500);
+
+  /* Test 4: Relay-to-relay padding can be enabled/disabled in consensus */
+  /* Make this channel a relay's channel */
+  memcpy(relay->identity_digest,
+          ((channel_tls_t *)chan)->conn->identity_digest, DIGEST_LEN);
+  smartlist_add(current_md_consensus->routerstatus_list, relay);
+  relay = NULL; /* Prevent double-free */
+
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_assert(!chan->pending_padding_callback);
+
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_pad_relays=1");
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADLATER);
+  tt_assert(!chan->pending_padding_callback);
+  val = channelpadding_get_netflow_inactive_timeout_ms(chan);
+  tt_i64_op(val, OP_GE, 1500);
+  tt_i64_op(val, OP_LE, 4500);
+  val = channelpadding_compute_time_until_pad_for_netflow(chan);
+  tt_i64_op(val, OP_LE, 4500);
+
+  /* Test 5: If we disable padding before channel usage, does that work? */
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_pad_before_usage=0");
+  channelpadding_new_consensus_params(current_md_consensus);
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_assert(!chan->pending_padding_callback);
+
+  /* Test 6: Can we control circ and TLS conn lifetime from the consensus? */
+  val = channelpadding_get_channel_idle_timeout(NULL, 0);
+  tt_i64_op(val, OP_GE, 180);
+  tt_i64_op(val, OP_LE, 180+90);
+  val = channelpadding_get_channel_idle_timeout(chan, 0);
+  tt_i64_op(val, OP_GE, 180);
+  tt_i64_op(val, OP_LE, 180+90);
+  options->ReducedConnectionPadding = 1;
+  val = channelpadding_get_channel_idle_timeout(chan, 0);
+  tt_i64_op(val, OP_GE, 180/2);
+  tt_i64_op(val, OP_LE, (180+90)/2);
+
+  options->ReducedConnectionPadding = 0;
+  options->ORPort_set = 1;
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_conntimeout_relays=600");
+  channelpadding_new_consensus_params(current_md_consensus);
+  val = channelpadding_get_channel_idle_timeout(chan, 1);
+  tt_i64_op(val, OP_GE, 450);
+  tt_i64_op(val, OP_LE, 750);
+
+  val = channelpadding_get_circuits_available_timeout();
+  tt_i64_op(val, OP_GE, 30*60);
+  tt_i64_op(val, OP_LE, 30*60*2);
+
+  options->ReducedConnectionPadding = 1;
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_conntimeout_clients=600");
+  channelpadding_new_consensus_params(current_md_consensus);
+  val = channelpadding_get_circuits_available_timeout();
+  tt_i64_op(val, OP_GE, 600/2);
+  tt_i64_op(val, OP_LE, 600*2/2);
+
+  options->ReducedConnectionPadding = 0;
+  options->CircuitsAvailableTimeout = 24*60*60;
+  val = channelpadding_get_circuits_available_timeout();
+  tt_i64_op(val, OP_GE, 24*60*60);
+  tt_i64_op(val, OP_LE, 24*60*60*2);
+
+ done:
+  tor_free(relay);
+
+  free_mock_consensus();
+  free_fake_channeltls((channel_tls_t*)chan);
+  smartlist_free(connection_array);
+
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  channel_free_all();
+
+  return;
+}
+
+void
+test_channelpadding_negotiation(void *arg)
+{
+  channelpadding_negotiate_t disable;
+  cell_t cell;
+  channelpadding_decision_t decision;
+  int val;
+  (void)arg;
+
+  /* Plan:
+   * 1. Clients reject negotiation, relays accept it.
+   *    * Bridges accept negotiation from their clients,
+   *      but not from relays.
+   * 2. Torrc options can override client-side negotiation
+   * 3. Test a version issue in channelpadidng cell
+   * 4. Test channelpadding_reduced_padding
+   */
+  monotime_init();
+  monotime_enable_test_mocking();
+  monotime_set_mock_time_nsec(1);
+  monotime_coarse_set_mock_time_nsec(1);
+  timers_initialize();
+  setup_mock_consensus();
+  setup_mock_network();
+
+  /* Test case #1: Do the right things ignore negotiation? */
+  /* relay-to-client case: */
+  channelpadding_send_disable_command(relay3_client);
+  tt_assert(client_relay3->padding_enabled);
+
+  /* client-to-relay case: */
+  get_options_mutable()->ORPort_set = 1;
+  channelpadding_disable_padding_on_channel(client_relay3);
+  tt_int_op(channelpadding_decide_to_pad_channel(relay3_client), OP_EQ,
+      CHANNELPADDING_WONTPAD);
+  tt_assert(!relay3_client->padding_enabled);
+  relay3_client->padding_enabled = 1;
+  client_relay3->padding_enabled = 1;
+
+  /* Bridge case from relay */
+  get_options_mutable()->BridgeRelay = 1;
+  channelpadding_disable_padding_on_channel(relay2_relay1);
+  tt_assert(relay1_relay2->padding_enabled);
+
+  /* Bridge case from client */
+  channelpadding_disable_padding_on_channel(client_relay3);
+  tt_assert(!relay3_client->padding_enabled);
+  tt_int_op(channelpadding_decide_to_pad_channel(relay3_client), OP_EQ,
+      CHANNELPADDING_WONTPAD);
+  relay3_client->padding_enabled = 1;
+  client_relay3->padding_enabled = 1;
+  get_options_mutable()->BridgeRelay = 0;
+  get_options_mutable()->ORPort_set = 0;
+
+  /* Test case #2: Torrc options */
+  /* ConnectionPadding auto; Relay doesn't support us */
+  ((channel_tls_t*)relay3_client)->conn->link_proto = 4;
+  relay3_client->padding_enabled = 0;
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(relay3_client);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_assert(!relay3_client->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  ((channel_tls_t*)relay3_client)->conn->link_proto = 5;
+  relay3_client->padding_enabled = 1;
+
+  /* ConnectionPadding 1; Relay doesn't support us */
+  get_options_mutable()->ConnectionPadding = 1;
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(client_relay3);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADLATER);
+  tt_assert(!client_relay3->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  get_options_mutable()->ConnectionPadding = 0;
+
+  /* Test case #3: Test a version issue in channelpadding cell */
+  get_options_mutable()->ORPort_set = 1;
+  client_relay3->padding_enabled = 1;
+  relay3_client->padding_enabled = 1;
+  memset(&cell, 0, sizeof(cell_t));
+  memset(&disable, 0, sizeof(channelpadding_negotiate_t));
+  cell.command = CELL_PADDING_NEGOTIATE;
+
+  channelpadding_negotiate_set_command(&disable, CHANNELPADDING_COMMAND_STOP);
+  disable.version = 1;
+  channelpadding_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE, &disable);
+  client_relay3->write_cell(client_relay3, &cell);
+  tt_assert(relay3_client->padding_enabled);
+  tt_int_op(channelpadding_update_padding_for_channel(client_relay3, &disable),
+          OP_EQ, -1);
+  tt_assert(client_relay3->padding_enabled);
+
+  disable.version = 0;
+  channelpadding_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE, &disable);
+  client_relay3->write_cell(client_relay3, &cell);
+  tt_assert(!relay3_client->padding_enabled);
+
+  /* Test case 4: Reducing padding actually reduces it */
+  relay3_client->padding_enabled = 1;
+  client_relay3->padding_enabled = 1;
+
+  decision = channelpadding_decide_to_pad_channel(relay3_client);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADLATER);
+
+  channelpadding_reduce_padding_on_channel(client_relay3);
+
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(relay3_client);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+
+  get_options_mutable()->ORPort_set = 0;
+  decision = channelpadding_decide_to_pad_channel(client_relay3);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADLATER);
+
+  tt_assert(!client_relay3->pending_padding_callback);
+  val = channelpadding_get_netflow_inactive_timeout_ms(client_relay3);
+  tt_int_op(val, OP_GE, 9000);
+  tt_int_op(val, OP_LE, 14000);
+  int64_t val64 =
+    channelpadding_compute_time_until_pad_for_netflow(client_relay3);
+  tt_i64_op(val64, OP_LE, 14000);
+
+ done:
+  free_mock_network();
+  free_mock_consensus();
+
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  channel_free_all();
+
+  return;
+}
+
+void
+test_channelpadding_decide_to_pad_channel(void *arg)
+{
+  channelpadding_decision_t decision;
+  /**
+   * Test case plan:
+   *
+   * 1. Channel that has "sent a packet" before the timeout.
+   *    + We should decide to pad later
+   * 2. Channel that has not "sent a packet" before the timeout:
+   * 2a. Not within 1.1s of the timeout.
+   *    + We should decide to pad later
+   * 2b. Within 1.1s of the timemout.
+   *    + We should schedule padding
+   *    + We should get feedback that we wrote a cell
+   * 2c. Within 0.1s of the timeout.
+   *    + We should schedule padding
+   *    + We should get feedback that we wrote a cell
+   * 2d. Channel that asks to pad while timeout is scheduled
+   *    + We should schedule padding
+   *    + We should get feedback that we wrote a cell
+   * 2e. 0s of the timeout
+   *    + We should send padding immediately
+   *    + We should get feedback that we wrote a cell
+   * 2f. <0s of the timeout
+   *    + We should send padding immediately
+   *    + We should get feedback that we wrote a cell
+   * 3. Channel that sends a packet while timeout is scheduled
+   *    + We should not get feedback that we wrote a cell
+   * 4. Channel that closes while timeout is scheduled
+   *    + We should not get feedback that we wrote a cell
+   * 5. Make sure the channel still would work if repaired
+   *    + We should be able to schedule padding and resend
+   * 6. Channel is not used for full circuits
+   * 7. Channel that disappears while timeout is scheduled
+   *    + We should not send padding
+   */
+  channel_t *chan;
+  int64_t new_time;
+  if (!connection_array)
+    connection_array = smartlist_new();
+  (void)arg;
+
+  tor_libevent_postfork();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  monotime_set_mock_time_nsec(1);
+  monotime_coarse_set_mock_time_nsec(1);
+  new_time = 1;
+  monotime_coarse_t now;
+  monotime_coarse_get(&now);
+  timers_initialize();
+  setup_full_capture_of_logs(LOG_WARN);
+  channelpadding_new_consensus_params(NULL);
+
+  chan = (channel_t*)new_fake_channeltls(0);
+  channel_timestamp_active(chan);
+
+  /* Test case #1: Channel that has "sent a packet" before the timeout. */
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADLATER);
+  tt_assert(!chan->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  /* Test case #2a: > 1.1s until timeout */
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now, 1200);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADLATER);
+  tt_assert(!chan->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  /* Test case #2b: >= 1.0s until timeout */
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now, 1000);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(chan->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  // Set up a timer for the <0 case below.
+  monotime_coarse_t now_minus_100s;
+  monotime_coarse_add_msec(&now_minus_100s, &now, 900);
+  // Wait for the timer from case #2b
+  new_time += 1000*NSEC_PER_MSEC;
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!chan->pending_padding_callback);
+
+  /* Test case #2c: > 0.1s until timeout */
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now, 100);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(chan->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  /* Test case #2d: Channel that asks to pad while timeout is scheduled */
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_ALREADY_SCHEDULED);
+
+  // Wait for the timer
+  new_time += 101*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!chan->pending_padding_callback);
+
+  /* Test case #2e: 0s until timeout */
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now, 0);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SENT);
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!chan->pending_padding_callback);
+
+  /* Test case #2f: <0s until timeout */
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now_minus_100s, 0);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SENT);
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!chan->pending_padding_callback);
+
+  /* Test case #3: Channel that sends a packet while timeout is scheduled */
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now, 100);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  tt_assert(chan->pending_padding_callback);
+
+  // Pretend the channel sent a packet
+  channel_timestamp_active(chan);
+
+  // We don't expect any timer callbacks here. Make a dummy one to be sure.
+  // Wait for the timer
+  new_time += 101*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  tt_assert(!chan->pending_padding_callback);
+
+  /* Test case #4: Channel that closes while a timeout is scheduled */
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now, 100);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  tt_assert(chan->pending_padding_callback);
+
+  // Pretend the channel is temporarily down
+  chan->state = CHANNEL_STATE_MAINT;
+
+  // We don't expect any timer callbacks here. Make a dummy one to be sure.
+  new_time += 101*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  tt_assert(!chan->pending_padding_callback);
+  chan->state = CHANNEL_STATE_OPEN;
+
+  /* Test case #5: Make sure previous test case didn't break everything */
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now, 100);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(chan->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  // Wait for the timer
+  new_time += 101*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!chan->pending_padding_callback);
+
+  /* Test case #6. Channel is not used for full circuits */
+  chan->channel_usage = CHANNEL_USED_NOT_USED_FOR_FULL_CIRCS;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_assert(!chan->pending_padding_callback);
+  chan->channel_usage = CHANNEL_USED_FOR_FULL_CIRCS;
+
+  /* Test case #7. Channel is closed while timeout is scheduled.
+   *
+   * NOTE: This test deliberately breaks the channel callback mechanism.
+   * It must be last.
+   */
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now, 100);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  tt_assert(chan->pending_padding_callback);
+
+  // Close the connection while the timer is scheduled
+  free_fake_channeltls((channel_tls_t*)chan);
+
+  // We don't expect any timer callbacks here. Make a dummy one to be sure.
+  new_time = 101*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+ done:
+  smartlist_free(connection_array);
+
+  teardown_capture_of_logs();
+  monotime_disable_test_mocking();
+  timers_shutdown();
+  channel_free_all();
+
+  return;
+}
+
+#define TEST_CHANNELPADDING(name, flags) \
+    { #name, test_##name, (flags), NULL, NULL }
+
+struct testcase_t channelpadding_tests[] = {
+  //TEST_CHANNELPADDING(channelpadding_decide_to_pad_channel, 0),
+  TEST_CHANNELPADDING(channelpadding_decide_to_pad_channel, TT_FORK),
+  TEST_CHANNELPADDING(channelpadding_negotiation, TT_FORK),
+  TEST_CHANNELPADDING(channelpadding_consensus, TT_FORK),
+  TEST_CHANNELPADDING(channelpadding_killonehop, TT_FORK),
+  TEST_CHANNELPADDING(channelpadding_timers, TT_FORK),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_channeltls.c b/src/test/test_channeltls.c
index 08442e01b6..10513e451d 100644
--- a/src/test/test_channeltls.c
+++ b/src/test/test_channeltls.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -6,20 +6,23 @@
 #include <math.h>
 
 #define TOR_CHANNEL_INTERNAL_
-#include "or.h"
-#include "address.h"
-#include "buffers.h"
-#include "channel.h"
-#include "channeltls.h"
-#include "connection_or.h"
-#include "config.h"
+#include "core/or/or.h"
+#include "lib/net/address.h"
+#include "lib/container/buffers.h"
+#include "core/or/channel.h"
+#include "core/or/channeltls.h"
+#include "core/mainloop/connection.h"
+#include "core/or/connection_or.h"
+#include "app/config/config.h"
 /* For init/free stuff */
-#include "scheduler.h"
-#include "tortls.h"
+#include "core/or/scheduler.h"
+#include "lib/tls/tortls.h"
+
+#include "core/or/or_connection_st.h"
 
 /* Test suite stuff */
-#include "test.h"
-#include "fakechans.h"
+#include "test/test.h"
+#include "test/fakechans.h"
 
 /* The channeltls unit tests */
 static void test_channeltls_create(void *arg);
@@ -32,6 +35,7 @@ static or_connection_t * tlschan_connection_or_connect_mock(
     const tor_addr_t *addr,
     uint16_t port,
     const char *digest,
+    const ed25519_public_key_t *ed_id,
     channel_tls_t *tlschan);
 static int tlschan_is_local_addr_mock(const tor_addr_t *addr);
 
@@ -70,8 +74,8 @@ test_channeltls_create(void *arg)
   MOCK(connection_or_connect, tlschan_connection_or_connect_mock);
 
   /* Try connecting */
-  ch = channel_tls_connect(&test_addr, 567, test_digest);
-  tt_assert(ch != NULL);
+  ch = channel_tls_connect(&test_addr, 567, test_digest, NULL);
+  tt_ptr_op(ch, OP_NE, NULL);
 
  done:
   if (ch) {
@@ -119,8 +123,8 @@ test_channeltls_num_bytes_queued(void *arg)
   MOCK(connection_or_connect, tlschan_connection_or_connect_mock);
 
   /* Try connecting */
-  ch = channel_tls_connect(&test_addr, 567, test_digest);
-  tt_assert(ch != NULL);
+  ch = channel_tls_connect(&test_addr, 567, test_digest, NULL);
+  tt_ptr_op(ch, OP_NE, NULL);
 
   /*
    * Next, we have to test ch->num_bytes_queued, which is
@@ -131,7 +135,7 @@ test_channeltls_num_bytes_queued(void *arg)
 
   tt_assert(ch->num_bytes_queued != NULL);
   tlschan = BASE_CHAN_TO_TLS(ch);
-  tt_assert(tlschan != NULL);
+  tt_ptr_op(tlschan, OP_NE, NULL);
   if (TO_CONN(tlschan->conn)->outbuf == NULL) {
     /* We need an outbuf to make sure buf_datalen() gets called */
     fake_outbuf = 1;
@@ -141,7 +145,7 @@ test_channeltls_num_bytes_queued(void *arg)
   tlschan_buf_datalen_mock_size = 1024;
   MOCK(buf_datalen, tlschan_buf_datalen_mock);
   len = ch->num_bytes_queued(ch);
-  tt_int_op(len, ==, tlschan_buf_datalen_mock_size);
+  tt_int_op(len, OP_EQ, tlschan_buf_datalen_mock_size);
   /*
    * We also cover num_cells_writeable here; since wide_circ_ids = 0 on
    * the fake tlschans, cell_network_size returns 512, and so with
@@ -150,7 +154,7 @@ test_channeltls_num_bytes_queued(void *arg)
    * - 2 cells.
    */
   n = ch->num_cells_writeable(ch);
-  tt_int_op(n, ==, CEIL_DIV(OR_CONN_HIGHWATER, 512) - 2);
+  tt_int_op(n, OP_EQ, CEIL_DIV(OR_CONN_HIGHWATER, 512) - 2);
   UNMOCK(buf_datalen);
   tlschan_buf_datalen_mock_target = NULL;
   tlschan_buf_datalen_mock_size = 0;
@@ -204,12 +208,12 @@ test_channeltls_overhead_estimate(void *arg)
   MOCK(connection_or_connect, tlschan_connection_or_connect_mock);
 
   /* Try connecting */
-  ch = channel_tls_connect(&test_addr, 567, test_digest);
-  tt_assert(ch != NULL);
+  ch = channel_tls_connect(&test_addr, 567, test_digest, NULL);
+  tt_ptr_op(ch, OP_NE, NULL);
 
   /* First case: silly low ratios should get clamped to 1.0 */
   tlschan = BASE_CHAN_TO_TLS(ch);
-  tt_assert(tlschan != NULL);
+  tt_ptr_op(tlschan, OP_NE, NULL);
   tlschan->conn->bytes_xmitted = 128;
   tlschan->conn->bytes_xmitted_by_tls = 64;
   r = ch->get_overhead_estimate(ch);
@@ -266,14 +270,16 @@ static or_connection_t *
 tlschan_connection_or_connect_mock(const tor_addr_t *addr,
                                    uint16_t port,
                                    const char *digest,
+                                   const ed25519_public_key_t *ed_id,
                                    channel_tls_t *tlschan)
 {
   or_connection_t *result = NULL;
+  (void) ed_id; // XXXX Not yet used.
 
-  tt_assert(addr != NULL);
-  tt_assert(port != 0);
-  tt_assert(digest != NULL);
-  tt_assert(tlschan != NULL);
+  tt_ptr_op(addr, OP_NE, NULL);
+  tt_uint_op(port, OP_NE, 0);
+  tt_ptr_op(digest, OP_NE, NULL);
+  tt_ptr_op(tlschan, OP_NE, NULL);
 
   /* Make a fake orconn */
   result = tor_malloc_zero(sizeof(*result));
@@ -298,11 +304,11 @@ tlschan_fake_close_method(channel_t *chan)
 {
   channel_tls_t *tlschan = NULL;
 
-  tt_assert(chan != NULL);
-  tt_int_op(chan->magic, ==, TLS_CHAN_MAGIC);
+  tt_ptr_op(chan, OP_NE, NULL);
+  tt_int_op(chan->magic, OP_EQ, TLS_CHAN_MAGIC);
 
   tlschan = BASE_CHAN_TO_TLS(chan);
-  tt_assert(tlschan != NULL);
+  tt_ptr_op(tlschan, OP_NE, NULL);
 
   /* Just free the fake orconn */
   tor_free(tlschan->conn->base_.address);
@@ -317,7 +323,7 @@ tlschan_fake_close_method(channel_t *chan)
 static int
 tlschan_is_local_addr_mock(const tor_addr_t *addr)
 {
-  tt_assert(addr != NULL);
+  tt_ptr_op(addr, OP_NE, NULL);
 
  done:
   return tlschan_local;
@@ -331,4 +337,3 @@ struct testcase_t channeltls_tests[] = {
     TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_checkdir.c b/src/test/test_checkdir.c
index fbb33f87f6..1df74c390a 100644
--- a/src/test/test_checkdir.c
+++ b/src/test/test_checkdir.c
@@ -1,8 +1,8 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
-#include "or.h"
+#include "core/or/or.h"
 
 #ifdef _WIN32
 #include <direct.h>
@@ -10,9 +10,12 @@
 #include <dirent.h>
 #endif
 
-#include "config.h"
-#include "test.h"
-#include "util.h"
+#include "app/config/config.h"
+#include "test/test.h"
+
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
 
 #ifdef _WIN32
 #define mkdir(a,b) mkdir(a)
@@ -20,7 +23,7 @@
 #define umask(mask) ((void)0)
 #else
 #define tt_int_op_nowin(a,op,b) tt_int_op((a),op,(b))
-#endif
+#endif /* defined(_WIN32) */
 
 /** Run unit tests for private dir permission enforcement logic. */
 static void
@@ -146,4 +149,3 @@ struct testcase_t checkdir_tests[] = {
   CHECKDIR(perms, TT_FORK),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_circuitbuild.c b/src/test/test_circuitbuild.c
new file mode 100644
index 0000000000..538f20781f
--- /dev/null
+++ b/src/test/test_circuitbuild.c
@@ -0,0 +1,182 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CIRCUITBUILD_PRIVATE
+#define CIRCUITLIST_PRIVATE
+#define ENTRYNODES_PRIVATE
+
+#include "core/or/or.h"
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
+#include "app/config/config.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/extend_info_st.h"
+#include "core/or/origin_circuit_st.h"
+
+#include "feature/client/entrynodes.h"
+
+/* Dummy nodes smartlist for testing */
+static smartlist_t dummy_nodes;
+/* Dummy exit extend_info for testing */
+static extend_info_t dummy_ei;
+
+static int
+mock_count_acceptable_nodes(smartlist_t *nodes)
+{
+  (void)nodes;
+
+  return DEFAULT_ROUTE_LEN + 1;
+}
+
+/* Test route lengths when the caller of new_route_len() doesn't
+ * specify exit_ei. */
+static void
+test_new_route_len_noexit(void *arg)
+{
+  int r;
+
+  (void)arg;
+  MOCK(count_acceptable_nodes, mock_count_acceptable_nodes);
+
+  r = new_route_len(CIRCUIT_PURPOSE_C_GENERAL, NULL, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN, OP_EQ, r);
+
+  r = new_route_len(CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT, NULL, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN, OP_EQ, r);
+
+  r = new_route_len(CIRCUIT_PURPOSE_S_CONNECT_REND, NULL, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN, OP_EQ, r);
+
+ done:
+  UNMOCK(count_acceptable_nodes);
+}
+
+/* Test route lengths where someone else chose the "exit" node, which
+ * require an extra hop for safety. */
+static void
+test_new_route_len_unsafe_exit(void *arg)
+{
+  int r;
+
+  (void)arg;
+  MOCK(count_acceptable_nodes, mock_count_acceptable_nodes);
+
+  /* connecting to hidden service directory */
+  r = new_route_len(CIRCUIT_PURPOSE_C_GENERAL, &dummy_ei, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN + 1, OP_EQ, r);
+
+  /* client connecting to introduction point */
+  r = new_route_len(CIRCUIT_PURPOSE_C_INTRODUCING, &dummy_ei, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN + 1, OP_EQ, r);
+
+  /* hidden service connecting to rendezvous point */
+  r = new_route_len(CIRCUIT_PURPOSE_S_CONNECT_REND, &dummy_ei, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN + 1, OP_EQ, r);
+
+ done:
+  UNMOCK(count_acceptable_nodes);
+}
+
+/* Test route lengths where we chose the "exit" node, which don't
+ * require an extra hop for safety. */
+static void
+test_new_route_len_safe_exit(void *arg)
+{
+  int r;
+
+  (void)arg;
+  MOCK(count_acceptable_nodes, mock_count_acceptable_nodes);
+
+  /* hidden service connecting to introduction point */
+  r = new_route_len(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, &dummy_ei,
+                    &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN, OP_EQ, r);
+
+  /* router testing its own reachability */
+  r = new_route_len(CIRCUIT_PURPOSE_TESTING, &dummy_ei, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN, OP_EQ, r);
+
+ done:
+  UNMOCK(count_acceptable_nodes);
+}
+
+/* Make sure a non-fatal assertion fails when new_route_len() gets an
+ * unexpected circuit purpose. */
+static void
+test_new_route_len_unhandled_exit(void *arg)
+{
+  int r;
+
+  (void)arg;
+  MOCK(count_acceptable_nodes, mock_count_acceptable_nodes);
+
+  tor_capture_bugs_(1);
+  setup_full_capture_of_logs(LOG_WARN);
+  r = new_route_len(CIRCUIT_PURPOSE_CONTROLLER, &dummy_ei, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN + 1, OP_EQ, r);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(exit_ei && !known_purpose)");
+  expect_single_log_msg_containing("Unhandled purpose");
+  expect_single_log_msg_containing("with a chosen exit; assuming routelen");
+  teardown_capture_of_logs();
+  tor_end_capture_bugs_();
+
+ done:
+  UNMOCK(count_acceptable_nodes);
+}
+
+static void
+test_upgrade_from_guard_wait(void *arg)
+{
+  circuit_t *circ = NULL;
+  origin_circuit_t *orig_circ = NULL;
+  entry_guard_t *guard = NULL;
+  smartlist_t *list = NULL;
+
+  (void) arg;
+
+  circ = dummy_origin_circuit_new(0);
+  orig_circ = TO_ORIGIN_CIRCUIT(circ);
+  tt_assert(orig_circ);
+
+  orig_circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+
+  circuit_set_state(circ, CIRCUIT_STATE_GUARD_WAIT);
+
+  /* Put it in guard wait state. */
+  guard = tor_malloc_zero(sizeof(*guard));
+  guard->in_selection = get_guard_selection_info();
+
+  orig_circ->guard_state =
+    circuit_guard_state_new(guard, GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD,
+                            NULL);
+
+  /* Mark the circuit for close. */
+  circuit_mark_for_close(circ, END_CIRC_REASON_TORPROTOCOL);
+  tt_int_op(circ->marked_for_close, OP_NE, 0);
+
+  /* We shouldn't pick the mark for close circuit. */
+  list = circuit_find_circuits_to_upgrade_from_guard_wait();
+  tt_assert(!list);
+
+ done:
+  circuit_free(circ);
+  entry_guard_free_(guard);
+}
+
+struct testcase_t circuitbuild_tests[] = {
+  { "noexit", test_new_route_len_noexit, 0, NULL, NULL },
+  { "safe_exit", test_new_route_len_safe_exit, 0, NULL, NULL },
+  { "unsafe_exit", test_new_route_len_unsafe_exit, 0, NULL, NULL },
+  { "unhandled_exit", test_new_route_len_unhandled_exit, 0, NULL, NULL },
+  { "upgrade_from_guard_wait", test_upgrade_from_guard_wait, TT_FORK,
+    NULL, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_circuitlist.c b/src/test/test_circuitlist.c
index e996c42115..5cebdbeda0 100644
--- a/src/test/test_circuitlist.c
+++ b/src/test/test_circuitlist.c
@@ -1,15 +1,23 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define TOR_CHANNEL_INTERNAL_
 #define CIRCUITBUILD_PRIVATE
 #define CIRCUITLIST_PRIVATE
-#include "or.h"
-#include "channel.h"
-#include "circuitbuild.h"
-#include "circuitlist.h"
-#include "test.h"
-#include "log_test_helpers.h"
+#define HS_CIRCUITMAP_PRIVATE
+#include "core/or/or.h"
+#include "core/or/channel.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitmux_ewma.h"
+#include "feature/hs/hs_circuitmap.h"
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+
+#include "lib/container/bitarray.h"
 
 static channel_t *
 new_fake_channel(void)
@@ -139,7 +147,7 @@ test_clist_maps(void *arg)
   /* Okay, now free ch2 and make sure that the circuit ID is STILL not
    * usable, because we haven't declared the destroy to be nonpending */
   tt_int_op(cdm.ncalls, OP_EQ, 0);
-  circuit_free(TO_CIRCUIT(or_c2));
+  circuit_free_(TO_CIRCUIT(or_c2));
   or_c2 = NULL; /* prevent free */
   tt_int_op(cdm.ncalls, OP_EQ, 2);
   memset(&cdm, 0, sizeof(cdm));
@@ -158,9 +166,9 @@ test_clist_maps(void *arg)
 
  done:
   if (or_c1)
-    circuit_free(TO_CIRCUIT(or_c1));
+    circuit_free_(TO_CIRCUIT(or_c1));
   if (or_c2)
-    circuit_free(TO_CIRCUIT(or_c2));
+    circuit_free_(TO_CIRCUIT(or_c2));
   if (ch1)
     tor_free(ch1->cmux);
   if (ch2)
@@ -178,6 +186,7 @@ static void
 test_rend_token_maps(void *arg)
 {
   or_circuit_t *c1, *c2, *c3, *c4;
+  origin_circuit_t *c5;
   const uint8_t tok1[REND_TOKEN_LEN] = "The cat can't tell y";
   const uint8_t tok2[REND_TOKEN_LEN] = "ou its name, and it ";
   const uint8_t tok3[REND_TOKEN_LEN] = "doesn't really care.";
@@ -185,10 +194,14 @@ test_rend_token_maps(void *arg)
 
   (void)arg;
   (void)tok1; //xxxx
+
+  hs_circuitmap_init();
+
   c1 = or_circuit_new(0, NULL);
   c2 = or_circuit_new(0, NULL);
   c3 = or_circuit_new(0, NULL);
   c4 = or_circuit_new(0, NULL);
+  c5 = origin_circuit_new();
 
   /* Make sure we really filled up the tok* variables */
   tt_int_op(tok1[REND_TOKEN_LEN-1], OP_EQ, 'y');
@@ -196,78 +209,87 @@ test_rend_token_maps(void *arg)
   tt_int_op(tok3[REND_TOKEN_LEN-1], OP_EQ, '.');
 
   /* No maps; nothing there. */
-  tt_ptr_op(NULL, OP_EQ, circuit_get_rendezvous(tok1));
-  tt_ptr_op(NULL, OP_EQ, circuit_get_intro_point(tok1));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok1));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok1));
 
-  circuit_set_rendezvous_cookie(c1, tok1);
-  circuit_set_intro_point_digest(c2, tok2);
+  hs_circuitmap_register_rend_circ_relay_side(c1, tok1);
+  hs_circuitmap_register_intro_circ_v2_relay_side(c2, tok2);
 
-  tt_ptr_op(NULL, OP_EQ, circuit_get_rendezvous(tok3));
-  tt_ptr_op(NULL, OP_EQ, circuit_get_intro_point(tok3));
-  tt_ptr_op(NULL, OP_EQ, circuit_get_rendezvous(tok2));
-  tt_ptr_op(NULL, OP_EQ, circuit_get_intro_point(tok1));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok3));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok3));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok2));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok1));
 
   /* Without purpose set, we don't get the circuits */
-  tt_ptr_op(NULL, OP_EQ, circuit_get_rendezvous(tok1));
-  tt_ptr_op(NULL, OP_EQ, circuit_get_intro_point(tok2));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok1));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok2));
 
   c1->base_.purpose = CIRCUIT_PURPOSE_REND_POINT_WAITING;
   c2->base_.purpose = CIRCUIT_PURPOSE_INTRO_POINT;
 
   /* Okay, make sure they show up now. */
-  tt_ptr_op(c1, OP_EQ, circuit_get_rendezvous(tok1));
-  tt_ptr_op(c2, OP_EQ, circuit_get_intro_point(tok2));
+  tt_ptr_op(c1, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok1));
+  tt_ptr_op(c2, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok2));
 
   /* Two items at the same place with the same token. */
   c3->base_.purpose = CIRCUIT_PURPOSE_REND_POINT_WAITING;
-  circuit_set_rendezvous_cookie(c3, tok2);
-  tt_ptr_op(c2, OP_EQ, circuit_get_intro_point(tok2));
-  tt_ptr_op(c3, OP_EQ, circuit_get_rendezvous(tok2));
+  hs_circuitmap_register_rend_circ_relay_side(c3, tok2);
+  tt_ptr_op(c2, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok2));
+  tt_ptr_op(c3, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok2));
 
   /* Marking a circuit makes it not get returned any more */
   circuit_mark_for_close(TO_CIRCUIT(c1), END_CIRC_REASON_FINISHED);
-  tt_ptr_op(NULL, OP_EQ, circuit_get_rendezvous(tok1));
-  circuit_free(TO_CIRCUIT(c1));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok1));
+  circuit_free_(TO_CIRCUIT(c1));
   c1 = NULL;
 
   /* Freeing a circuit makes it not get returned any more. */
-  circuit_free(TO_CIRCUIT(c2));
+  circuit_free_(TO_CIRCUIT(c2));
   c2 = NULL;
-  tt_ptr_op(NULL, OP_EQ, circuit_get_intro_point(tok2));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok2));
 
   /* c3 -- are you still there? */
-  tt_ptr_op(c3, OP_EQ, circuit_get_rendezvous(tok2));
+  tt_ptr_op(c3, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok2));
   /* Change its cookie.  This never happens in Tor per se, but hey. */
   c3->base_.purpose = CIRCUIT_PURPOSE_INTRO_POINT;
-  circuit_set_intro_point_digest(c3, tok3);
+  hs_circuitmap_register_intro_circ_v2_relay_side(c3, tok3);
 
-  tt_ptr_op(NULL, OP_EQ, circuit_get_rendezvous(tok2));
-  tt_ptr_op(c3, OP_EQ, circuit_get_intro_point(tok3));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok2));
+  tt_ptr_op(c3, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok3));
 
   /* Now replace c3 with c4. */
   c4->base_.purpose = CIRCUIT_PURPOSE_INTRO_POINT;
-  circuit_set_intro_point_digest(c4, tok3);
+  hs_circuitmap_register_intro_circ_v2_relay_side(c4, tok3);
 
-  tt_ptr_op(c4, OP_EQ, circuit_get_intro_point(tok3));
+  tt_ptr_op(c4, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok3));
 
-  tt_ptr_op(c3->rendinfo, OP_EQ, NULL);
-  tt_ptr_op(c4->rendinfo, OP_NE, NULL);
-  tt_mem_op(c4->rendinfo, OP_EQ, tok3, REND_TOKEN_LEN);
+  tt_ptr_op(TO_CIRCUIT(c3)->hs_token, OP_EQ, NULL);
+  tt_ptr_op(TO_CIRCUIT(c4)->hs_token, OP_NE, NULL);
+  tt_mem_op(TO_CIRCUIT(c4)->hs_token->token, OP_EQ, tok3, REND_TOKEN_LEN);
 
   /* Now clear c4's cookie. */
-  circuit_set_intro_point_digest(c4, NULL);
-  tt_ptr_op(c4->rendinfo, OP_EQ, NULL);
-  tt_ptr_op(NULL, OP_EQ, circuit_get_intro_point(tok3));
+  hs_circuitmap_remove_circuit(TO_CIRCUIT(c4));
+  tt_ptr_op(TO_CIRCUIT(c4)->hs_token, OP_EQ, NULL);
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok3));
+
+  /* Now let's do a check for the client-side rend circuitmap */
+  c5->base_.purpose = CIRCUIT_PURPOSE_C_ESTABLISH_REND;
+  hs_circuitmap_register_rend_circ_client_side(c5, tok1);
+
+  tt_ptr_op(c5, OP_EQ, hs_circuitmap_get_rend_circ_client_side(tok1));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_client_side(tok2));
 
  done:
   if (c1)
-    circuit_free(TO_CIRCUIT(c1));
+    circuit_free_(TO_CIRCUIT(c1));
   if (c2)
-    circuit_free(TO_CIRCUIT(c2));
+    circuit_free_(TO_CIRCUIT(c2));
   if (c3)
-    circuit_free(TO_CIRCUIT(c3));
+    circuit_free_(TO_CIRCUIT(c3));
   if (c4)
-    circuit_free(TO_CIRCUIT(c4));
+    circuit_free_(TO_CIRCUIT(c4));
+  if (c5)
+    circuit_free_(TO_CIRCUIT(c5));
 }
 
 static void
@@ -365,10 +387,88 @@ test_pick_circid(void *arg)
   UNMOCK(channel_dump_statistics);
 }
 
+/** Test that the circuit pools of our HS circuitmap are isolated based on
+ *  their token type. */
+static void
+test_hs_circuitmap_isolation(void *arg)
+{
+  or_circuit_t *circ1 = NULL;
+  origin_circuit_t *circ2 = NULL;
+  or_circuit_t *circ3 = NULL;
+  origin_circuit_t *circ4 = NULL;
+
+  (void)arg;
+
+  hs_circuitmap_init();
+
+  {
+    const uint8_t tok1[REND_TOKEN_LEN] = "bet i got some of th";
+
+    circ1 = or_circuit_new(0, NULL);
+    tt_assert(circ1);
+    circ1->base_.purpose = CIRCUIT_PURPOSE_REND_POINT_WAITING;
+
+    /* check that circuitmap is empty right? */
+    tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok1));
+
+    /* Register circ1 with tok1 as relay-side rend circ */
+    hs_circuitmap_register_rend_circ_relay_side(circ1, tok1);
+
+    /* check that service-side getters don't work */
+    tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_service_side(tok1));
+    tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_intro_circ_v2_service_side(tok1));
+
+    /* Check that the right getter works. */
+    tt_ptr_op(circ1, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok1));
+  }
+
+  {
+    const uint8_t tok2[REND_TOKEN_LEN] = "you dont know anythi";
+
+    circ2 = origin_circuit_new();
+    tt_assert(circ2);
+    circ2->base_.purpose = CIRCUIT_PURPOSE_S_ESTABLISH_INTRO;
+    circ3 = or_circuit_new(0, NULL);
+    tt_assert(circ3);
+    circ3->base_.purpose = CIRCUIT_PURPOSE_INTRO_POINT;
+    circ4 = origin_circuit_new();
+    tt_assert(circ4);
+    circ4->base_.purpose = CIRCUIT_PURPOSE_S_ESTABLISH_INTRO;
+
+    /* Register circ2 with tok2 as service-side intro v2 circ */
+    hs_circuitmap_register_intro_circ_v2_service_side(circ2, tok2);
+    /* Register circ3 with tok2 again but for different purpose */
+    hs_circuitmap_register_intro_circ_v2_relay_side(circ3, tok2);
+
+    /* Check that the getters work */
+    tt_ptr_op(circ2, OP_EQ,
+              hs_circuitmap_get_intro_circ_v2_service_side(tok2));
+    tt_ptr_op(circ3, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok2));
+
+    /* Register circ4 with tok2: it should override circ2 */
+    hs_circuitmap_register_intro_circ_v2_service_side(circ4, tok2);
+
+    /* check that relay-side getters don't work */
+    tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok2));
+
+    /* Check that the getter returns circ4; the last circuit registered with
+     * that token. */
+    tt_ptr_op(circ4, OP_EQ,
+              hs_circuitmap_get_intro_circ_v2_service_side(tok2));
+  }
+
+ done:
+  circuit_free_(TO_CIRCUIT(circ1));
+  circuit_free_(TO_CIRCUIT(circ2));
+  circuit_free_(TO_CIRCUIT(circ3));
+  circuit_free_(TO_CIRCUIT(circ4));
+}
+
 struct testcase_t circuitlist_tests[] = {
   { "maps", test_clist_maps, TT_FORK, NULL, NULL },
   { "rend_token_maps", test_rend_token_maps, TT_FORK, NULL, NULL },
   { "pick_circid", test_pick_circid, TT_FORK, NULL, NULL },
+  { "hs_circuitmap_isolation", test_hs_circuitmap_isolation,
+    TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_circuitmux.c b/src/test/test_circuitmux.c
index 1ffa17247d..a2b3e62fe8 100644
--- a/src/test/test_circuitmux.c
+++ b/src/test/test_circuitmux.c
@@ -1,15 +1,21 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define TOR_CHANNEL_INTERNAL_
 #define CIRCUITMUX_PRIVATE
+#define CIRCUITMUX_EWMA_PRIVATE
 #define RELAY_PRIVATE
-#include "or.h"
-#include "channel.h"
-#include "circuitmux.h"
-#include "relay.h"
-#include "scheduler.h"
-#include "test.h"
+#include "core/or/or.h"
+#include "core/or/channel.h"
+#include "core/or/circuitmux.h"
+#include "core/or/circuitmux_ewma.h"
+#include "core/or/relay.h"
+#include "core/or/scheduler.h"
+#include "test/test.h"
+
+#include "core/or/destroy_cell_queue_st.h"
+
+#include <math.h>
 
 /* XXXX duplicated function from test_circuitlist.c */
 static channel_t *
@@ -45,12 +51,13 @@ test_cmux_destroy_cell_queue(void *arg)
   cmux = circuitmux_alloc();
   tt_assert(cmux);
   ch = new_fake_channel();
+  circuitmux_set_policy(cmux, &ewma_policy);
   ch->has_queued_writes = has_queued_writes;
   ch->wide_circ_ids = 1;
 
   circ = circuitmux_get_first_active_circuit(cmux, &cq);
-  tt_assert(!circ);
-  tt_assert(!cq);
+  tt_ptr_op(circ, OP_EQ, NULL);
+  tt_ptr_op(cq, OP_EQ, NULL);
 
   circuitmux_append_destroy_cell(ch, cmux, 100, 10);
   circuitmux_append_destroy_cell(ch, cmux, 190, 6);
@@ -59,7 +66,7 @@ test_cmux_destroy_cell_queue(void *arg)
   tt_int_op(circuitmux_num_cells(cmux), OP_EQ, 3);
 
   circ = circuitmux_get_first_active_circuit(cmux, &cq);
-  tt_assert(!circ);
+  tt_ptr_op(circ, OP_EQ, NULL);
   tt_assert(cq);
 
   tt_int_op(cq->n, OP_EQ, 3);
@@ -77,8 +84,50 @@ test_cmux_destroy_cell_queue(void *arg)
   tor_free(dc);
 }
 
+static void
+test_cmux_compute_ticks(void *arg)
+{
+  const int64_t NS_PER_S = 1000 * 1000 * 1000;
+  const int64_t START_NS = UINT64_C(1217709000)*NS_PER_S;
+  int64_t now;
+  double rem;
+  unsigned tick;
+  (void)arg;
+  circuitmux_ewma_free_all();
+  monotime_enable_test_mocking();
+
+  monotime_coarse_set_mock_time_nsec(START_NS);
+  cell_ewma_initialize_ticks();
+  const unsigned tick_zero = cell_ewma_get_current_tick_and_fraction(&rem);
+  tt_double_op(rem, OP_GT, -1e-9);
+  tt_double_op(rem, OP_LT, 1e-9);
+
+  /* 1.5 second later and we should still be in the same tick. */
+  now = START_NS + NS_PER_S + NS_PER_S/2;
+  monotime_coarse_set_mock_time_nsec(now);
+  tick = cell_ewma_get_current_tick_and_fraction(&rem);
+  tt_uint_op(tick, OP_EQ, tick_zero);
+#ifdef USING_32BIT_MSEC_HACK
+  const double tolerance = .0005;
+#else
+  const double tolerance = .00000001;
+#endif
+  tt_double_op(fabs(rem - .15), OP_LT, tolerance);
+
+  /* 25 second later and we should be in another tick. */
+  now = START_NS + NS_PER_S * 25;
+  monotime_coarse_set_mock_time_nsec(now);
+  tick = cell_ewma_get_current_tick_and_fraction(&rem);
+  tt_uint_op(tick, OP_EQ, tick_zero + 2);
+  tt_double_op(fabs(rem - .5), OP_LT, tolerance);
+
+ done:
+  ;
+}
+
 struct testcase_t circuitmux_tests[] = {
   { "destroy_cell_queue", test_cmux_destroy_cell_queue, TT_FORK, NULL, NULL },
+  { "compute_ticks", test_cmux_compute_ticks, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_circuitstats.c b/src/test/test_circuitstats.c
new file mode 100644
index 0000000000..1cbcb14f2b
--- /dev/null
+++ b/src/test/test_circuitstats.c
@@ -0,0 +1,206 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CIRCUITBUILD_PRIVATE
+#define CIRCUITSTATS_PRIVATE
+#define CIRCUITLIST_PRIVATE
+#define CHANNEL_PRIVATE_
+
+#include "core/or/or.h"
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
+#include "app/config/config.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitstats.h"
+#include "core/or/circuituse.h"
+#include "core/or/channel.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/crypt_path_st.h"
+#include "core/or/extend_info_st.h"
+#include "core/or/origin_circuit_st.h"
+
+void test_circuitstats_timeout(void *arg);
+void test_circuitstats_hoplen(void *arg);
+origin_circuit_t *subtest_fourhop_circuit(struct timeval, int);
+origin_circuit_t *add_opened_threehop(void);
+origin_circuit_t *build_unopened_fourhop(struct timeval);
+
+int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
+
+static int marked_for_close;
+/* Mock function because we are not trying to test the close circuit that does
+ * an awful lot of checks on the circuit object. */
+static void
+mock_circuit_mark_for_close(circuit_t *circ, int reason, int line,
+                            const char *file)
+{
+  (void) circ;
+  (void) reason;
+  (void) line;
+  (void) file;
+  marked_for_close = 1;
+  return;
+}
+
+origin_circuit_t *
+add_opened_threehop(void)
+{
+  origin_circuit_t *or_circ = origin_circuit_new();
+  extend_info_t fakehop;
+  memset(&fakehop, 0, sizeof(fakehop));
+
+  TO_CIRCUIT(or_circ)->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  or_circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  or_circ->build_state->desired_path_len = DEFAULT_ROUTE_LEN;
+
+  onion_append_hop(&or_circ->cpath, &fakehop);
+  onion_append_hop(&or_circ->cpath, &fakehop);
+  onion_append_hop(&or_circ->cpath, &fakehop);
+
+  or_circ->has_opened = 1;
+  TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN;
+  TO_CIRCUIT(or_circ)->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  return or_circ;
+}
+
+origin_circuit_t *
+build_unopened_fourhop(struct timeval circ_start_time)
+{
+  origin_circuit_t *or_circ = origin_circuit_new();
+  extend_info_t *fakehop = tor_malloc_zero(sizeof(extend_info_t));
+  memset(fakehop, 0, sizeof(extend_info_t));
+
+  TO_CIRCUIT(or_circ)->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  TO_CIRCUIT(or_circ)->timestamp_began = circ_start_time;
+  TO_CIRCUIT(or_circ)->timestamp_created = circ_start_time;
+
+  or_circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  or_circ->build_state->desired_path_len = 4;
+
+  onion_append_hop(&or_circ->cpath, fakehop);
+  onion_append_hop(&or_circ->cpath, fakehop);
+  onion_append_hop(&or_circ->cpath, fakehop);
+  onion_append_hop(&or_circ->cpath, fakehop);
+
+  tor_free(fakehop);
+
+  return or_circ;
+}
+
+origin_circuit_t *
+subtest_fourhop_circuit(struct timeval circ_start_time, int should_timeout)
+{
+  origin_circuit_t *or_circ = build_unopened_fourhop(circ_start_time);
+
+  // Now make them open one at a time and call
+  // circuit_build_times_handle_completed_hop();
+  or_circ->cpath->state = CPATH_STATE_OPEN;
+  circuit_build_times_handle_completed_hop(or_circ);
+  tt_int_op(get_circuit_build_times()->total_build_times, OP_EQ, 0);
+
+  or_circ->cpath->next->state = CPATH_STATE_OPEN;
+  circuit_build_times_handle_completed_hop(or_circ);
+  tt_int_op(get_circuit_build_times()->total_build_times, OP_EQ, 0);
+
+  // Third hop: We should count it now.
+  or_circ->cpath->next->next->state = CPATH_STATE_OPEN;
+  circuit_build_times_handle_completed_hop(or_circ);
+  tt_int_op(get_circuit_build_times()->total_build_times, OP_EQ,
+            !should_timeout); // 1 if counted, 0 otherwise
+
+  // Fourth hop: Don't double count
+  or_circ->cpath->next->next->next->state = CPATH_STATE_OPEN;
+  circuit_build_times_handle_completed_hop(or_circ);
+  tt_int_op(get_circuit_build_times()->total_build_times, OP_EQ,
+            !should_timeout);
+
+ done:
+  return or_circ;
+}
+
+void
+test_circuitstats_hoplen(void *arg)
+{
+  /* Plan:
+   *   0. Test no other opened circs (relaxed timeout)
+   *   1. Check >3 hop circ building w/o timeout
+   *   2. Check >3 hop circs w/ timeouts..
+   */
+  struct timeval circ_start_time;
+  origin_circuit_t *threehop = NULL;
+  origin_circuit_t *fourhop = NULL;
+  (void)arg;
+  MOCK(circuit_mark_for_close_, mock_circuit_mark_for_close);
+
+  circuit_build_times_init(get_circuit_build_times_mutable());
+
+  // Let's set a close_ms to 2X the initial timeout, so we can
+  // test relaxed functionality (which uses the close_ms timeout)
+  get_circuit_build_times_mutable()->close_ms *= 2;
+
+  tor_gettimeofday(&circ_start_time);
+  circ_start_time.tv_sec -= 119; // make us hit "relaxed" cutoff
+
+  // Test 1: Build a fourhop circuit that should get marked
+  // as relaxed and eventually counted by circuit_expire_building
+  // (but not before)
+  fourhop = subtest_fourhop_circuit(circ_start_time, 0);
+  tt_int_op(fourhop->relaxed_timeout, OP_EQ, 0);
+  tt_int_op(marked_for_close, OP_EQ, 0);
+  circuit_expire_building();
+  tt_int_op(marked_for_close, OP_EQ, 0);
+  tt_int_op(fourhop->relaxed_timeout, OP_EQ, 1);
+  TO_CIRCUIT(fourhop)->timestamp_began.tv_sec -= 119;
+  circuit_expire_building();
+  tt_int_op(get_circuit_build_times()->total_build_times, OP_EQ, 1);
+  tt_int_op(marked_for_close, OP_EQ, 1);
+
+  circuit_free_(TO_CIRCUIT(fourhop));
+  circuit_build_times_reset(get_circuit_build_times_mutable());
+
+  // Test 2: Add a threehop circuit for non-relaxed timeouts
+  threehop = add_opened_threehop();
+
+  /* This circuit should not timeout */
+  tor_gettimeofday(&circ_start_time);
+  circ_start_time.tv_sec -= 59;
+  fourhop = subtest_fourhop_circuit(circ_start_time, 0);
+  circuit_expire_building();
+  tt_int_op(get_circuit_build_times()->total_build_times, OP_EQ, 1);
+  tt_int_op(TO_CIRCUIT(fourhop)->purpose, OP_NE,
+            CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT);
+
+  circuit_free_((circuit_t *)fourhop);
+  circuit_build_times_reset(get_circuit_build_times_mutable());
+
+  /* Test 3: This circuit should now time out and get marked as a
+   * measurement circuit, but still get counted (and counted only once)
+   */
+  circ_start_time.tv_sec -= 2;
+  fourhop = subtest_fourhop_circuit(circ_start_time, 0);
+  tt_int_op(TO_CIRCUIT(fourhop)->purpose, OP_EQ,
+            CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT);
+  tt_int_op(get_circuit_build_times()->total_build_times, OP_EQ, 1);
+  circuit_expire_building();
+  tt_int_op(get_circuit_build_times()->total_build_times, OP_EQ, 1);
+
+ done:
+  UNMOCK(circuit_mark_for_close_);
+  circuit_free_(TO_CIRCUIT(threehop));
+  circuit_free_(TO_CIRCUIT(fourhop));
+  circuit_build_times_free_timeouts(get_circuit_build_times_mutable());
+}
+
+#define TEST_CIRCUITSTATS(name, flags) \
+    { #name, test_##name, (flags), NULL, NULL }
+
+struct testcase_t circuitstats_tests[] = {
+  TEST_CIRCUITSTATS(circuitstats_hoplen, TT_FORK),
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_circuituse.c b/src/test/test_circuituse.c
new file mode 100644
index 0000000000..3acfc12044
--- /dev/null
+++ b/src/test/test_circuituse.c
@@ -0,0 +1,310 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CIRCUITLIST_PRIVATE
+
+#include "core/or/or.h"
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "app/config/config.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/circuitbuild.h"
+#include "feature/nodelist/nodelist.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/origin_circuit_st.h"
+
+static void
+test_circuit_is_available_for_use_ret_false_when_marked_for_close(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = tor_malloc(sizeof(circuit_t));
+  circ->marked_for_close = 1;
+
+  tt_int_op(0, OP_EQ, circuit_is_available_for_use(circ));
+
+  done:
+    tor_free(circ);
+}
+
+static void
+test_circuit_is_available_for_use_ret_false_when_timestamp_dirty(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = tor_malloc(sizeof(circuit_t));
+  circ->timestamp_dirty = 1;
+
+  tt_int_op(0, OP_EQ, circuit_is_available_for_use(circ));
+
+  done:
+    tor_free(circ);
+}
+
+static void
+test_circuit_is_available_for_use_ret_false_for_non_general_purpose(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = tor_malloc(sizeof(circuit_t));
+  circ->purpose = CIRCUIT_PURPOSE_REND_POINT_WAITING;
+
+  tt_int_op(0, OP_EQ, circuit_is_available_for_use(circ));
+
+  done:
+    tor_free(circ);
+}
+
+static void
+test_circuit_is_available_for_use_ret_false_for_non_general_origin(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = tor_malloc(sizeof(circuit_t));
+  circ->purpose = CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT;
+
+  tt_int_op(0, OP_EQ, circuit_is_available_for_use(circ));
+
+  done:
+    tor_free(circ);
+}
+
+static void
+test_circuit_is_available_for_use_ret_false_for_non_origin_purpose(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = tor_malloc(sizeof(circuit_t));
+  circ->purpose = CIRCUIT_PURPOSE_OR;
+
+  tt_int_op(0, OP_EQ, circuit_is_available_for_use(circ));
+
+  done:
+    tor_free(circ);
+}
+
+static void
+test_circuit_is_available_for_use_ret_false_unusable_for_new_conns(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = dummy_origin_circuit_new(30);
+  mark_circuit_unusable_for_new_conns(TO_ORIGIN_CIRCUIT(circ));
+
+  tt_int_op(0, OP_EQ, circuit_is_available_for_use(circ));
+
+  done:
+    circuit_free(circ);
+}
+
+static void
+test_circuit_is_available_for_use_returns_false_for_onehop_tunnel(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = dummy_origin_circuit_new(30);
+  origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ);
+  oc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  oc->build_state->onehop_tunnel = 1;
+
+  tt_int_op(0, OP_EQ, circuit_is_available_for_use(circ));
+
+  done:
+    circuit_free(circ);
+}
+
+static void
+test_circuit_is_available_for_use_returns_true_for_clean_circuit(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = dummy_origin_circuit_new(30);
+  origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ);
+  oc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  oc->build_state->onehop_tunnel = 0;
+
+  tt_int_op(1, OP_EQ, circuit_is_available_for_use(circ));
+
+  done:
+    circuit_free(circ);
+}
+
+static int
+mock_circuit_all_predicted_ports_handled(time_t now,
+                                         int *need_uptime,
+                                         int *need_capacity)
+{
+  (void)now;
+
+  if (need_uptime && need_capacity)
+    return 0;
+  return 1;
+}
+
+static consensus_path_type_t
+mock_router_have_unknown_consensus_path(void)
+{
+  return CONSENSUS_PATH_UNKNOWN;
+}
+
+static consensus_path_type_t
+mock_router_have_exit_consensus_path(void)
+{
+  return CONSENSUS_PATH_EXIT;
+}
+
+static void
+test_needs_exit_circuits_ret_false_for_predicted_ports_and_path(void *arg)
+{
+  (void)arg;
+
+  MOCK(circuit_all_predicted_ports_handled,
+       mock_circuit_all_predicted_ports_handled);
+  int needs_uptime = 1;
+  int needs_capacity = 0;
+
+  time_t now = time(NULL);
+  tt_int_op(0, OP_EQ,
+            needs_exit_circuits(now, &needs_uptime, &needs_capacity));
+
+  done:
+    UNMOCK(circuit_all_predicted_ports_handled);
+}
+
+static void
+test_needs_exit_circuits_ret_false_for_non_exit_consensus_path(void *arg)
+{
+  (void)arg;
+
+  MOCK(circuit_all_predicted_ports_handled,
+       mock_circuit_all_predicted_ports_handled);
+  int needs_uptime = 1;
+  int needs_capacity = 1;
+  MOCK(router_have_consensus_path, mock_router_have_unknown_consensus_path);
+
+  time_t now = time(NULL);
+  tt_int_op(0, OP_EQ,
+            needs_exit_circuits(now, &needs_uptime, &needs_capacity));
+
+  done:
+    UNMOCK(circuit_all_predicted_ports_handled);
+    UNMOCK(router_have_consensus_path);
+}
+
+static void
+test_needs_exit_circuits_ret_true_for_predicted_ports_and_path(void *arg)
+{
+  (void)arg;
+
+  MOCK(circuit_all_predicted_ports_handled,
+       mock_circuit_all_predicted_ports_handled);
+  int needs_uptime = 1;
+  int needs_capacity = 1;
+  MOCK(router_have_consensus_path, mock_router_have_exit_consensus_path);
+
+  time_t now = time(NULL);
+  tt_int_op(1, OP_EQ,
+            needs_exit_circuits(now, &needs_uptime, &needs_capacity));
+
+  done:
+    UNMOCK(circuit_all_predicted_ports_handled);
+    UNMOCK(router_have_consensus_path);
+}
+
+static void
+test_needs_circuits_for_build_ret_false_consensus_path_unknown(void *arg)
+{
+  (void)arg;
+  MOCK(router_have_consensus_path, mock_router_have_unknown_consensus_path);
+  tt_int_op(0, OP_EQ, needs_circuits_for_build(0));
+  done: ;
+}
+
+static void
+test_needs_circuits_for_build_ret_false_if_num_less_than_max(void *arg)
+{
+  (void)arg;
+  MOCK(router_have_consensus_path, mock_router_have_exit_consensus_path);
+  tt_int_op(0, OP_EQ, needs_circuits_for_build(13));
+  done:
+    UNMOCK(router_have_consensus_path);
+}
+
+static void
+test_needs_circuits_for_build_returns_true_when_more_are_needed(void *arg)
+{
+  (void)arg;
+  MOCK(router_have_consensus_path, mock_router_have_exit_consensus_path);
+  tt_int_op(1, OP_EQ, needs_circuits_for_build(0));
+  done:
+    UNMOCK(router_have_consensus_path);
+}
+
+struct testcase_t circuituse_tests[] = {
+ { "marked",
+   test_circuit_is_available_for_use_ret_false_when_marked_for_close,
+   TT_FORK, NULL, NULL
+ },
+ { "timestamp",
+   test_circuit_is_available_for_use_ret_false_when_timestamp_dirty,
+   TT_FORK, NULL, NULL
+ },
+ { "non_general",
+   test_circuit_is_available_for_use_ret_false_for_non_general_purpose,
+   TT_FORK, NULL, NULL
+ },
+ { "non_general",
+  test_circuit_is_available_for_use_ret_false_for_non_general_origin,
+   TT_FORK, NULL, NULL
+ },
+ { "origin",
+   test_circuit_is_available_for_use_ret_false_for_non_origin_purpose,
+   TT_FORK, NULL, NULL
+ },
+ { "clean",
+   test_circuit_is_available_for_use_ret_false_unusable_for_new_conns,
+   TT_FORK, NULL, NULL
+ },
+ { "onehop",
+   test_circuit_is_available_for_use_returns_false_for_onehop_tunnel,
+   TT_FORK, NULL, NULL
+ },
+ { "clean_circ",
+   test_circuit_is_available_for_use_returns_true_for_clean_circuit,
+   TT_FORK, NULL, NULL
+ },
+ { "exit_f",
+   test_needs_exit_circuits_ret_false_for_predicted_ports_and_path,
+   TT_FORK, NULL, NULL
+ },
+ { "exit_t",
+   test_needs_exit_circuits_ret_true_for_predicted_ports_and_path,
+   TT_FORK, NULL, NULL
+ },
+ { "non_exit",
+   test_needs_exit_circuits_ret_false_for_non_exit_consensus_path,
+   TT_FORK, NULL, NULL
+ },
+ { "true",
+   test_needs_exit_circuits_ret_true_for_predicted_ports_and_path,
+   TT_FORK, NULL, NULL
+ },
+ { "consensus_path_unknown",
+   test_needs_circuits_for_build_ret_false_consensus_path_unknown,
+   TT_FORK, NULL, NULL
+ },
+ { "less_than_max",
+   test_needs_circuits_for_build_ret_false_if_num_less_than_max,
+   TT_FORK, NULL, NULL
+ },
+ { "more_needed",
+   test_needs_circuits_for_build_returns_true_when_more_are_needed,
+   TT_FORK, NULL, NULL
+ },
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_compat_libevent.c b/src/test/test_compat_libevent.c
index 0443cc0b1c..2f8646e897 100644
--- a/src/test/test_compat_libevent.c
+++ b/src/test/test_compat_libevent.c
@@ -1,18 +1,17 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define COMPAT_LIBEVENT_PRIVATE
 #include "orconfig.h"
-#include "or.h"
+#include "core/or/or.h"
 
-#include "test.h"
+#include "test/test.h"
 
-#include "compat_libevent.h"
+#include "lib/evloop/compat_libevent.h"
 
 #include <event2/event.h>
-#include <event2/thread.h>
 
-#include "log_test_helpers.h"
+#include "test/log_test_helpers.h"
 
 #define NS_MODULE compat_libevent
 
@@ -122,10 +121,70 @@ test_compat_libevent_header_version(void *ignored)
   (void)0;
 }
 
+/* Test for postloop events */
+
+/* Event callback to increment a counter. */
+static void
+increment_int_counter_cb(periodic_timer_t *timer, void *arg)
+{
+  (void)timer;
+  int *ctr = arg;
+  ++*ctr;
+}
+
+static int activated_counter = 0;
+
+/* Mainloop event callback to activate another mainloop event */
+static void
+activate_event_cb(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  mainloop_event_t **other_event = arg;
+  mainloop_event_activate(*other_event);
+  ++activated_counter;
+}
+
+static void
+test_compat_libevent_postloop_events(void *arg)
+{
+  (void)arg;
+  mainloop_event_t *a = NULL, *b = NULL;
+  periodic_timer_t *timed = NULL;
+
+  tor_libevent_postfork();
+
+  /* If postloop events don't work, then these events will activate one
+   * another ad infinitum and, and the periodic event will never occur. */
+  b = mainloop_event_postloop_new(activate_event_cb, &a);
+  a = mainloop_event_postloop_new(activate_event_cb, &b);
+
+  int counter = 0;
+  struct timeval fifty_ms = { 0, 10 * 1000 };
+  timed = periodic_timer_new(tor_libevent_get_base(), &fifty_ms,
+                             increment_int_counter_cb, &counter);
+
+  mainloop_event_activate(a);
+  int r;
+  do {
+    r = tor_libevent_run_event_loop(tor_libevent_get_base(), 0);
+    if (r == -1)
+      break;
+  } while (counter < 5);
+
+  tt_int_op(activated_counter, OP_GE, 2);
+
+ done:
+  mainloop_event_free(a);
+  mainloop_event_free(b);
+  periodic_timer_free(timed);
+}
+
 struct testcase_t compat_libevent_tests[] = {
   { "logging_callback", test_compat_libevent_logging_callback,
     TT_FORK, NULL, NULL },
   { "header_version", test_compat_libevent_header_version, 0, NULL, NULL },
+  { "postloop_events", test_compat_libevent_postloop_events,
+    TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_config.c b/src/test/test_config.c
index 89f9b3e2aa..8f011ce1f1 100644
--- a/src/test/test_config.c
+++ b/src/test/test_config.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -8,42 +8,59 @@
 #define CONFIG_PRIVATE
 #define PT_PRIVATE
 #define ROUTERSET_PRIVATE
-#include "or.h"
-#include "address.h"
-#include "addressmap.h"
-#include "circuitmux_ewma.h"
-#include "circuitbuild.h"
-#include "config.h"
-#include "confparse.h"
-#include "connection.h"
-#include "connection_edge.h"
-#include "test.h"
-#include "util.h"
-#include "address.h"
-#include "connection_or.h"
-#include "control.h"
-#include "cpuworker.h"
-#include "dirserv.h"
-#include "dirvote.h"
-#include "dns.h"
-#include "entrynodes.h"
-#include "transports.h"
-#include "ext_orport.h"
-#include "geoip.h"
-#include "hibernate.h"
-#include "main.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "policies.h"
-#include "rendclient.h"
-#include "rendservice.h"
-#include "router.h"
-#include "routerlist.h"
-#include "routerset.h"
-#include "statefile.h"
-#include "test.h"
-#include "transports.h"
-#include "util.h"
+#include "core/or/or.h"
+#include "lib/net/address.h"
+#include "lib/net/resolve.h"
+#include "feature/client/addressmap.h"
+#include "feature/client/bridges.h"
+#include "core/or/circuitmux_ewma.h"
+#include "core/or/circuitbuild.h"
+#include "app/config/config.h"
+#include "app/config/confparse.h"
+#include "core/mainloop/connection.h"
+#include "core/or/connection_edge.h"
+#include "test/test.h"
+#include "core/or/connection_or.h"
+#include "feature/control/control.h"
+#include "core/mainloop/cpuworker.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/relay/dns.h"
+#include "feature/client/entrynodes.h"
+#include "feature/client/transports.h"
+#include "feature/relay/ext_orport.h"
+#include "lib/geoip/geoip.h"
+#include "feature/hibernate/hibernate.h"
+#include "core/mainloop/mainloop.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "core/or/policies.h"
+#include "feature/rend/rendclient.h"
+#include "feature/rend/rendservice.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerset.h"
+#include "app/config/statefile.h"
+
+#include "test/test_helpers.h"
+
+#include "feature/dirclient/dir_server_st.h"
+#include "core/or/port_cfg_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+
+#include "lib/fs/conffile.h"
+#include "lib/meminfo/meminfo.h"
+#include "lib/net/gethostname.h"
+#include "lib/encoding/confline.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
 
 static void
 test_config_addressmap(void *arg)
@@ -276,7 +293,7 @@ test_config_check_or_create_data_subdir(void *arg)
   tt_assert(!is_private_dir(subpath));
   tt_assert(!check_or_create_data_subdir(subdir));
   tt_assert(is_private_dir(subpath));
-#endif
+#endif /* !defined (_WIN32) */
 
  done:
   rmdir(subpath);
@@ -320,7 +337,7 @@ test_config_write_to_data_subdir(void *arg)
   tt_int_op(mkdir(options->DataDirectory, 0700), OP_EQ, 0);
 #endif
 
-  // Write attempt shoudl fail, if subdirectory doesn't exist.
+  // Write attempt should fail, if subdirectory doesn't exist.
   tt_assert(write_to_data_subdir(subdir, fname, str, NULL));
   tt_assert(! check_or_create_data_subdir(subdir));
 
@@ -365,12 +382,12 @@ good_bridge_line_test(const char *string, const char *test_addrport,
   /* If we were asked to validate a digest, but we did not get a
      digest after parsing, we failed. */
   if (test_digest && tor_digest_is_zero(bridge_line->digest))
-    tt_assert(0);
+    tt_abort();
 
   /* If we were not asked to validate a digest, and we got a digest
      after parsing, we failed again. */
   if (!test_digest && !tor_digest_is_zero(bridge_line->digest))
-    tt_assert(0);
+    tt_abort();
 
   /* If we were asked to validate a digest, and we got a digest after
      parsing, make sure it's correct. */
@@ -384,17 +401,17 @@ good_bridge_line_test(const char *string, const char *test_addrport,
   /* If we were asked to validate a transport name, make sure tha it
      matches with the transport name that was parsed. */
   if (test_transport && !bridge_line->transport_name)
-    tt_assert(0);
+    tt_abort();
   if (!test_transport && bridge_line->transport_name)
-    tt_assert(0);
+    tt_abort();
   if (test_transport)
     tt_str_op(test_transport,OP_EQ, bridge_line->transport_name);
 
   /* Validate the SOCKS argument smartlist. */
   if (test_socks_args && !bridge_line->socks_args)
-    tt_assert(0);
+    tt_abort();
   if (!test_socks_args && bridge_line->socks_args)
-    tt_assert(0);
+    tt_abort();
   if (test_socks_args)
     tt_assert(smartlist_strings_eq(test_socks_args,
                                      bridge_line->socks_args));
@@ -412,7 +429,7 @@ bad_bridge_line_test(const char *string)
   bridge_line_t *bridge_line = parse_bridge_line(string);
   if (bridge_line)
     TT_FAIL(("%s was supposed to fail, but it didn't.", string));
-  tt_assert(!bridge_line);
+  tt_ptr_op(bridge_line, OP_EQ, NULL);
 
  done:
   bridge_line_free(bridge_line);
@@ -520,18 +537,18 @@ test_config_parse_transport_options_line(void *arg)
 
   { /* too small line */
     options_sl = get_options_from_transport_options_line("valley", NULL);
-    tt_assert(!options_sl);
+    tt_ptr_op(options_sl, OP_EQ, NULL);
   }
 
   { /* no k=v values */
     options_sl = get_options_from_transport_options_line("hit it!", NULL);
-    tt_assert(!options_sl);
+    tt_ptr_op(options_sl, OP_EQ, NULL);
   }
 
   { /* correct line, but wrong transport specified */
     options_sl =
       get_options_from_transport_options_line("trebuchet k=v", "rook");
-    tt_assert(!options_sl);
+    tt_ptr_op(options_sl, OP_EQ, NULL);
   }
 
   { /* correct -- no transport specified */
@@ -582,6 +599,22 @@ test_config_parse_transport_options_line(void *arg)
   }
 }
 
+/* Mocks needed for the compute_max_mem_in_queues test */
+static int get_total_system_memory_mock(size_t *mem_out);
+
+static size_t total_system_memory_output = 0;
+static int total_system_memory_return = 0;
+
+static int
+get_total_system_memory_mock(size_t *mem_out)
+{
+  if (! mem_out)
+    return -1;
+
+  *mem_out = total_system_memory_output;
+  return total_system_memory_return;
+}
+
 /* Mocks needed for the transport plugin line test */
 
 static void pt_kickstart_proxy_mock(const smartlist_t *transport_list,
@@ -655,84 +688,85 @@ test_config_parse_transport_plugin_line(void *arg)
 
   /* Bad transport lines - too short */
   r = parse_transport_line(options, "bad", 1, 0);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   r = parse_transport_line(options, "bad", 1, 1);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   r = parse_transport_line(options, "bad bad", 1, 0);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   r = parse_transport_line(options, "bad bad", 1, 1);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
 
   /* Test transport list parsing */
   r = parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 1, 0);
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   r = parse_transport_line(options,
    "transport_1 exec /usr/bin/fake-transport", 1, 1);
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   r = parse_transport_line(options,
       "transport_1,transport_2 exec /usr/bin/fake-transport", 1, 0);
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   r = parse_transport_line(options,
       "transport_1,transport_2 exec /usr/bin/fake-transport", 1, 1);
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   /* Bad transport identifiers */
   r = parse_transport_line(options,
       "transport_* exec /usr/bin/fake-transport", 1, 0);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   r = parse_transport_line(options,
       "transport_* exec /usr/bin/fake-transport", 1, 1);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
 
   /* Check SOCKS cases for client transport */
   r = parse_transport_line(options,
       "transport_1 socks4 1.2.3.4:567", 1, 0);
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   r = parse_transport_line(options,
       "transport_1 socks5 1.2.3.4:567", 1, 0);
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   /* Proxy case for server transport */
   r = parse_transport_line(options,
       "transport_1 proxy 1.2.3.4:567", 1, 1);
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   /* Multiple-transport error exit */
   r = parse_transport_line(options,
       "transport_1,transport_2 socks5 1.2.3.4:567", 1, 0);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   r = parse_transport_line(options,
       "transport_1,transport_2 proxy 1.2.3.4:567", 1, 1);
+  tt_int_op(r, OP_LT, 0);
   /* No port error exit */
   r = parse_transport_line(options,
       "transport_1 socks5 1.2.3.4", 1, 0);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   r = parse_transport_line(options,
      "transport_1 proxy 1.2.3.4", 1, 1);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   /* Unparsable address error exit */
   r = parse_transport_line(options,
       "transport_1 socks5 1.2.3:6x7", 1, 0);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   r = parse_transport_line(options,
       "transport_1 proxy 1.2.3:6x7", 1, 1);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
 
   /* "Strange {Client|Server}TransportPlugin field" error exit */
   r = parse_transport_line(options,
       "transport_1 foo bar", 1, 0);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   r = parse_transport_line(options,
       "transport_1 foo bar", 1, 1);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
 
   /* No sandbox mode error exit */
   tmp = options->Sandbox;
   options->Sandbox = 1;
   r = parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 1, 0);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   r = parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 1, 1);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   options->Sandbox = tmp;
 
   /*
@@ -744,7 +778,7 @@ test_config_parse_transport_plugin_line(void *arg)
     pt_kickstart_proxy_mock_call_count;
   r = parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 0, 1);
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   tt_assert(pt_kickstart_proxy_mock_call_count ==
       old_pt_kickstart_proxy_mock_call_count + 1);
   UNMOCK(pt_kickstart_proxy);
@@ -752,7 +786,7 @@ test_config_parse_transport_plugin_line(void *arg)
   /* This one hits a log line in the !validate_only case only */
   r = parse_transport_line(options,
       "transport_1 proxy 1.2.3.4:567", 0, 1);
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
 
   /* Check mocked client transport cases */
   MOCK(pt_kickstart_proxy, pt_kickstart_proxy_mock);
@@ -770,7 +804,7 @@ test_config_parse_transport_plugin_line(void *arg)
   r = parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 0, 0);
   /* Should have succeeded */
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   /* transport_is_needed() should have been called */
   tt_assert(transport_is_needed_mock_call_count ==
       old_transport_is_needed_mock_call_count + 1);
@@ -794,7 +828,7 @@ test_config_parse_transport_plugin_line(void *arg)
   r = parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 0, 0);
   /* Should have succeeded */
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   /*
    * transport_is_needed() and pt_kickstart_proxy() should have been
    * called.
@@ -818,7 +852,7 @@ test_config_parse_transport_plugin_line(void *arg)
   r = parse_transport_line(options,
       "transport_1 socks5 1.2.3.4:567", 0, 0);
   /* Should have succeeded */
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   /*
    * transport_is_needed() and transport_add_from_config() should have
    * been called.
@@ -851,9 +885,23 @@ static void
 test_config_fix_my_family(void *arg)
 {
   char *err = NULL;
-  const char *family = "$1111111111111111111111111111111111111111, "
-                       "1111111111111111111111111111111111111112, "
-                       "$1111111111111111111111111111111111111113";
+  config_line_t *family = tor_malloc_zero(sizeof(config_line_t));
+  family->key = tor_strdup("MyFamily");
+  family->value = tor_strdup("$1111111111111111111111111111111111111111, "
+                             "1111111111111111111111111111111111111112, "
+                             "$1111111111111111111111111111111111111113");
+
+  config_line_t *family2 = tor_malloc_zero(sizeof(config_line_t));
+  family2->key = tor_strdup("MyFamily");
+  family2->value = tor_strdup("1111111111111111111111111111111111111114");
+
+  config_line_t *family3 = tor_malloc_zero(sizeof(config_line_t));
+  family3->key = tor_strdup("MyFamily");
+  family3->value = tor_strdup("$1111111111111111111111111111111111111115");
+
+  family->next = family2;
+  family2->next = family3;
+  family3->next = NULL;
 
   or_options_t* options = options_new();
   or_options_t* defaults = options_new();
@@ -861,7 +909,7 @@ test_config_fix_my_family(void *arg)
 
   options_init(options);
   options_init(defaults);
-  options->MyFamily = tor_strdup(family);
+  options->MyFamily_lines = family;
 
   options_validate(NULL, options, defaults, 0, &err) ;
 
@@ -869,18 +917,23 @@ test_config_fix_my_family(void *arg)
     TT_FAIL(("options_validate failed: %s", err));
   }
 
-  tt_str_op(options->MyFamily,OP_EQ,
-                                "$1111111111111111111111111111111111111111, "
-                                "$1111111111111111111111111111111111111112, "
-                                "$1111111111111111111111111111111111111113");
-
-  done:
-    if (err != NULL) {
-      tor_free(err);
-    }
+  const char *valid[] = { "$1111111111111111111111111111111111111111",
+                          "$1111111111111111111111111111111111111112",
+                          "$1111111111111111111111111111111111111113",
+                          "$1111111111111111111111111111111111111114",
+                          "$1111111111111111111111111111111111111115" };
+  int ret_size = 0;
+  config_line_t *ret;
+  for (ret = options->MyFamily; ret && ret_size < 5; ret = ret->next) {
+    tt_str_op(ret->value, OP_EQ, valid[ret_size]);
+    ret_size++;
+  }
+  tt_int_op(ret_size, OP_EQ, 5);
 
-    or_options_free(options);
-    or_options_free(defaults);
+ done:
+  tor_free(err);
+  or_options_free(options);
+  or_options_free(defaults);
 }
 
 static int n_hostname_01010101 = 0;
@@ -1117,7 +1170,7 @@ test_config_resolve_my_address(void *arg)
                               &method_used,&hostname_out);
 
   tt_want(retval == 0);
-  tt_want_str_op(method_used,==,"CONFIGURED");
+  tt_want_str_op(method_used,OP_EQ,"CONFIGURED");
   tt_want(hostname_out == NULL);
   tt_assert(resolved_addr == 0x80348069);
 
@@ -1142,8 +1195,8 @@ test_config_resolve_my_address(void *arg)
 
   tt_want(retval == 0);
   tt_want(n_hostname_01010101 == prev_n_hostname_01010101 + 1);
-  tt_want_str_op(method_used,==,"RESOLVED");
-  tt_want_str_op(hostname_out,==,"www.torproject.org");
+  tt_want_str_op(method_used,OP_EQ,"RESOLVED");
+  tt_want_str_op(hostname_out,OP_EQ,"www.torproject.org");
   tt_assert(resolved_addr == 0x01010101);
 
   UNMOCK(tor_lookup_hostname);
@@ -1174,8 +1227,8 @@ test_config_resolve_my_address(void *arg)
   tt_want(retval == 0);
   tt_want(n_gethostname_replacement == prev_n_gethostname_replacement + 1);
   tt_want(n_hostname_01010101 == prev_n_hostname_01010101 + 1);
-  tt_want_str_op(method_used,==,"GETHOSTNAME");
-  tt_want_str_op(hostname_out,==,"onionrouter!");
+  tt_want_str_op(method_used,OP_EQ,"GETHOSTNAME");
+  tt_want_str_op(hostname_out,OP_EQ,"onionrouter!");
   tt_assert(resolved_addr == 0x01010101);
 
   UNMOCK(tor_gethostname);
@@ -1197,7 +1250,7 @@ test_config_resolve_my_address(void *arg)
                               &method_used,&hostname_out);
 
   tt_want(resolved_addr == 0);
-  tt_assert(retval == -1);
+  tt_int_op(retval, OP_EQ, -1);
 
   tor_free(options->Address);
   tor_free(hostname_out);
@@ -1219,7 +1272,7 @@ test_config_resolve_my_address(void *arg)
                               &method_used,&hostname_out);
 
   tt_want(n_hostname_failure == prev_n_hostname_failure + 1);
-  tt_assert(retval == -1);
+  tt_int_op(retval, OP_EQ, -1);
 
   UNMOCK(tor_lookup_hostname);
 
@@ -1240,7 +1293,7 @@ test_config_resolve_my_address(void *arg)
                               &method_used,&hostname_out);
 
   tt_want(n_gethostname_failure == prev_n_gethostname_failure + 1);
-  tt_assert(retval == -1);
+  tt_int_op(retval, OP_EQ, -1);
 
   UNMOCK(tor_gethostname);
   tor_free(hostname_out);
@@ -1263,11 +1316,11 @@ test_config_resolve_my_address(void *arg)
                               &method_used,&hostname_out);
 
   tt_want(retval == 0);
-  tt_want_int_op(n_gethostname_replacement, ==,
+  tt_want_int_op(n_gethostname_replacement, OP_EQ,
                  prev_n_gethostname_replacement + 1);
-  tt_want_int_op(n_get_interface_address, ==,
+  tt_want_int_op(n_get_interface_address, OP_EQ,
                  prev_n_get_interface_address + 1);
-  tt_want_str_op(method_used,==,"INTERFACE");
+  tt_want_str_op(method_used,OP_EQ,"INTERFACE");
   tt_want(hostname_out == NULL);
   tt_assert(resolved_addr == 0x08080808);
 
@@ -1293,7 +1346,7 @@ test_config_resolve_my_address(void *arg)
           prev_n_get_interface_address_failure + 1);
   tt_want(n_gethostname_replacement ==
           prev_n_gethostname_replacement + 1);
-  tt_assert(retval == -1);
+  tt_int_op(retval, OP_EQ, -1);
 
   UNMOCK(get_interface_address);
   tor_free(hostname_out);
@@ -1323,7 +1376,7 @@ test_config_resolve_my_address(void *arg)
   tt_want(n_hostname_failure == prev_n_hostname_failure + 1);
   tt_want(n_gethostname_replacement == prev_n_gethostname_replacement + 1);
   tt_want(retval == 0);
-  tt_want_str_op(method_used,==,"INTERFACE");
+  tt_want_str_op(method_used,OP_EQ,"INTERFACE");
   tt_assert(resolved_addr == 0x09090909);
 
   UNMOCK(tor_lookup_hostname);
@@ -1353,7 +1406,7 @@ test_config_resolve_my_address(void *arg)
                               &method_used,&hostname_out);
 
   tt_want(n_hostname_failure == prev_n_hostname_failure + 1);
-  tt_assert(retval == -1);
+  tt_int_op(retval, OP_EQ, -1);
 
   UNMOCK(tor_gethostname);
   UNMOCK(tor_lookup_hostname);
@@ -1368,7 +1421,7 @@ test_config_resolve_my_address(void *arg)
    *      if running on.
    *   3. Hostname from previous step cannot be converted to
    *      address by using tor_inet_aton() function.
-   *   4. However, tor_lookup_hostname() succeds in resolving the
+   *   4. However, tor_lookup_hostname() succeeds in resolving the
    *      hostname from step 2.
    *   5. Unfortunately, tor_addr_is_internal() deems this address
    *      to be internal.
@@ -1397,9 +1450,9 @@ test_config_resolve_my_address(void *arg)
   tt_want(n_hostname_localhost == prev_n_hostname_localhost + 1);
   tt_want(n_get_interface_address6 == prev_n_get_interface_address6 + 1);
 
-  tt_str_op(method_used,==,"INTERFACE");
-  tt_assert(!hostname_out);
-  tt_assert(retval == 0);
+  tt_str_op(method_used,OP_EQ,"INTERFACE");
+  tt_ptr_op(hostname_out, OP_EQ, NULL);
+  tt_int_op(retval, OP_EQ, 0);
 
   /*
    * CASE 11b:
@@ -1424,7 +1477,7 @@ test_config_resolve_my_address(void *arg)
   tt_want(n_get_interface_address6_failure ==
           prev_n_get_interface_address6_failure + 1);
 
-  tt_assert(retval == -1);
+  tt_int_op(retval, OP_EQ, -1);
 
   UNMOCK(tor_gethostname);
   UNMOCK(tor_lookup_hostname);
@@ -1453,7 +1506,7 @@ test_config_resolve_my_address(void *arg)
                               &method_used,&hostname_out);
 
   tt_want(n_gethostname_localhost == prev_n_gethostname_localhost + 1);
-  tt_assert(retval == -1);
+  tt_int_op(retval, OP_EQ, -1);
 
   UNMOCK(tor_gethostname);
 
@@ -1488,18 +1541,18 @@ test_config_adding_trusted_dir_server(void *arg)
                               NULL, V3_DIRINFO, 1.0);
   tt_assert(ds);
   dir_server_add(ds);
-  tt_assert(get_n_authorities(V3_DIRINFO) == 1);
-  tt_assert(smartlist_len(router_get_fallback_dir_servers()) == 1);
+  tt_int_op(get_n_authorities(V3_DIRINFO), OP_EQ, 1);
+  tt_int_op(smartlist_len(router_get_fallback_dir_servers()), OP_EQ, 1);
 
   /* create a trusted ds with an IPv6 address and port */
   rv = tor_addr_port_parse(LOG_WARN, "[::1]:9061", &ipv6.addr, &ipv6.port, -1);
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
   ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, &ipv6, digest,
                               NULL, V3_DIRINFO, 1.0);
   tt_assert(ds);
   dir_server_add(ds);
-  tt_assert(get_n_authorities(V3_DIRINFO) == 2);
-  tt_assert(smartlist_len(router_get_fallback_dir_servers()) == 2);
+  tt_int_op(get_n_authorities(V3_DIRINFO), OP_EQ, 2);
+  tt_int_op(smartlist_len(router_get_fallback_dir_servers()), OP_EQ, 2);
 
  done:
   clear_dir_servers();
@@ -1521,21 +1574,21 @@ test_config_adding_fallback_dir_server(void *arg)
   routerlist_free_all();
 
   rv = tor_addr_parse(&ipv4, "127.0.0.1");
-  tt_assert(rv == AF_INET);
+  tt_int_op(rv, OP_EQ, AF_INET);
 
   /* create a trusted ds without an IPv6 address and port */
   ds = fallback_dir_server_new(&ipv4, 9059, 9060, NULL, digest, 1.0);
   tt_assert(ds);
   dir_server_add(ds);
-  tt_assert(smartlist_len(router_get_fallback_dir_servers()) == 1);
+  tt_int_op(smartlist_len(router_get_fallback_dir_servers()), OP_EQ, 1);
 
   /* create a trusted ds with an IPv6 address and port */
   rv = tor_addr_port_parse(LOG_WARN, "[::1]:9061", &ipv6.addr, &ipv6.port, -1);
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
   ds = fallback_dir_server_new(&ipv4, 9059, 9060, &ipv6, digest, 1.0);
   tt_assert(ds);
   dir_server_add(ds);
-  tt_assert(smartlist_len(router_get_fallback_dir_servers()) == 2);
+  tt_int_op(smartlist_len(router_get_fallback_dir_servers()), OP_EQ, 2);
 
  done:
   clear_dir_servers();
@@ -1566,14 +1619,14 @@ test_config_parsing_trusted_dir_server(void *arg)
   rv = parse_dir_authority_line(TEST_DIR_AUTH_LINE_START
                                 TEST_DIR_AUTH_LINE_END,
                                 V3_DIRINFO, 1);
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
 
   /* parse a trusted dir server with an IPv6 address and port */
   rv = parse_dir_authority_line(TEST_DIR_AUTH_LINE_START
                                 TEST_DIR_AUTH_IPV6_FLAG
                                 TEST_DIR_AUTH_LINE_END,
                                 V3_DIRINFO, 1);
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
 
   /* Since we are only validating, there is no cleanup. */
  done:
@@ -1584,6 +1637,40 @@ test_config_parsing_trusted_dir_server(void *arg)
 #undef TEST_DIR_AUTH_LINE_END
 #undef TEST_DIR_AUTH_IPV6_FLAG
 
+#define TEST_DIR_AUTH_LINE_START                                        \
+                    "foobar orport=12345 "                              \
+                    "v3ident=14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 "
+#define TEST_DIR_AUTH_LINE_END_BAD_IP                                   \
+                    "0.256.3.4:54321 "                                  \
+                    "FDB2 FBD2 AAA5 25FA 2999 E617 5091 5A32 C777 3B17"
+#define TEST_DIR_AUTH_LINE_END_WITH_DNS_ADDR                            \
+                    "torproject.org:54321 "                             \
+                    "FDB2 FBD2 AAA5 25FA 2999 E617 5091 5A32 C777 3B17"
+
+static void
+test_config_parsing_invalid_dir_address(void *arg)
+{
+  (void)arg;
+  int rv;
+
+  rv = parse_dir_authority_line(TEST_DIR_AUTH_LINE_START
+                                TEST_DIR_AUTH_LINE_END_BAD_IP,
+                                V3_DIRINFO, 1);
+  tt_int_op(rv, OP_EQ, -1);
+
+  rv = parse_dir_authority_line(TEST_DIR_AUTH_LINE_START
+                                TEST_DIR_AUTH_LINE_END_WITH_DNS_ADDR,
+                                V3_DIRINFO, 1);
+  tt_int_op(rv, OP_EQ, -1);
+
+  done:
+  return;
+}
+
+#undef TEST_DIR_AUTH_LINE_START
+#undef TEST_DIR_AUTH_LINE_END_BAD_IP
+#undef TEST_DIR_AUTH_LINE_END_WITH_DNS_ADDR
+
 /* No secrets here:
  * id is `echo "syn-propanethial-S-oxide" | shasum | cut -d" " -f1`
  */
@@ -1601,13 +1688,13 @@ test_config_parsing_fallback_dir_server(void *arg)
 
   /* parse a trusted dir server without an IPv6 address and port */
   rv = parse_dir_fallback_line(TEST_DIR_FALLBACK_LINE, 1);
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
 
   /* parse a trusted dir server with an IPv6 address and port */
   rv = parse_dir_fallback_line(TEST_DIR_FALLBACK_LINE
                                TEST_DIR_FALLBACK_IPV6_FLAG,
                                1);
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
 
   /* Since we are only validating, there is no cleanup. */
  done:
@@ -1627,8 +1714,8 @@ test_config_adding_default_trusted_dir_servers(void *arg)
 
   /* Assume we only have one bridge authority */
   add_default_trusted_dir_authorities(BRIDGE_DIRINFO);
-  tt_assert(get_n_authorities(BRIDGE_DIRINFO) == 1);
-  tt_assert(smartlist_len(router_get_fallback_dir_servers()) == 1);
+  tt_int_op(get_n_authorities(BRIDGE_DIRINFO), OP_EQ, 1);
+  tt_int_op(smartlist_len(router_get_fallback_dir_servers()), OP_EQ, 1);
 
   /* Assume we have eight V3 authorities */
   add_default_trusted_dir_authorities(V3_DIRINFO);
@@ -1816,7 +1903,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 1);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 1);
 
     /* we have more fallbacks than just the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 1);
@@ -1835,7 +1922,7 @@ test_config_adding_dir_servers(void *arg)
                          1 : 0)
                         );
       /* If we have no default bridge authority, something has gone wrong */
-      tt_assert(n_default_alt_bridge_authority >= 1);
+      tt_int_op(n_default_alt_bridge_authority, OP_GE, 1);
 
       /* Count v3 Authorities */
       SMARTLIST_FOREACH(fallback_servers,
@@ -1847,7 +1934,7 @@ test_config_adding_dir_servers(void *arg)
                          1 : 0)
                         );
       /* If we have no default authorities, something has gone really wrong */
-      tt_assert(n_default_alt_dir_authority >= 1);
+      tt_int_op(n_default_alt_dir_authority, OP_GE, 1);
 
       /* Calculate Fallback Directory Count */
       n_default_fallback_dir = (smartlist_len(fallback_servers) -
@@ -1857,7 +1944,7 @@ test_config_adding_dir_servers(void *arg)
        * or some authorities aren't being added as fallback directories.
        * (networkstatus_consensus_can_use_extra_fallbacks depends on all
        * authorities being fallback directories.) */
-      tt_assert(n_default_fallback_dir >= 0);
+      tt_int_op(n_default_fallback_dir, OP_GE, 0);
     }
   }
 
@@ -1898,7 +1985,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must not have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 0);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 0);
 
     /* we have more fallbacks than just the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 1);
@@ -1907,7 +1994,7 @@ test_config_adding_dir_servers(void *arg)
       /* trusted_dir_servers */
       const smartlist_t *dir_servers = router_get_trusted_dir_servers();
       /* D0, (No B1), (No A2) */
-      tt_assert(smartlist_len(dir_servers) == 1);
+      tt_int_op(smartlist_len(dir_servers), OP_EQ, 1);
 
       /* DirAuthority - D0 - dir_port: 60090 */
       int found_D0 = 0;
@@ -1919,7 +2006,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 1);
+      tt_int_op(found_D0, OP_EQ, 1);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -1931,7 +2018,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -1943,14 +2030,14 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
     }
 
     {
       /* fallback_dir_servers */
       const smartlist_t *fallback_servers = router_get_fallback_dir_servers();
       /* D0, (No B1), (No A2), Custom Fallback */
-      tt_assert(smartlist_len(fallback_servers) == 2);
+      tt_int_op(smartlist_len(fallback_servers), OP_EQ, 2);
 
       /* DirAuthority - D0 - dir_port: 60090 */
       int found_D0 = 0;
@@ -1962,7 +2049,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 1);
+      tt_int_op(found_D0, OP_EQ, 1);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -1974,7 +2061,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -1986,7 +2073,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
 
       /* Custom FallbackDir - No Nickname - dir_port: 60093 */
       int found_non_default_fallback = 0;
@@ -1998,7 +2085,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60093 ?
                          1 : 0)
                         );
-      tt_assert(found_non_default_fallback == 1);
+      tt_int_op(found_non_default_fallback, OP_EQ, 1);
 
       /* (No Default FallbackDir) - No Nickname - dir_port: 60099 */
       int found_default_fallback = 0;
@@ -2010,7 +2097,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60099 ?
                          1 : 0)
                         );
-      tt_assert(found_default_fallback == 0);
+      tt_int_op(found_default_fallback, OP_EQ, 0);
     }
   }
 
@@ -2039,7 +2126,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must not have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 0);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 0);
 
     /* we just have the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 0);
@@ -2048,7 +2135,7 @@ test_config_adding_dir_servers(void *arg)
       /* trusted_dir_servers */
       const smartlist_t *dir_servers = router_get_trusted_dir_servers();
       /* D0, (No B1), (No A2) */
-      tt_assert(smartlist_len(dir_servers) == 1);
+      tt_int_op(smartlist_len(dir_servers), OP_EQ, 1);
 
       /* DirAuthority - D0 - dir_port: 60090 */
       int found_D0 = 0;
@@ -2060,7 +2147,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 1);
+      tt_int_op(found_D0, OP_EQ, 1);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2072,7 +2159,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2084,14 +2171,14 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
     }
 
     {
       /* fallback_dir_servers */
       const smartlist_t *fallback_servers = router_get_fallback_dir_servers();
       /* D0, (No B1), (No A2), (No Fallback) */
-      tt_assert(smartlist_len(fallback_servers) == 1);
+      tt_int_op(smartlist_len(fallback_servers), OP_EQ, 1);
 
       /* DirAuthority - D0 - dir_port: 60090 */
       int found_D0 = 0;
@@ -2103,7 +2190,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 1);
+      tt_int_op(found_D0, OP_EQ, 1);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2115,7 +2202,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2127,7 +2214,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
 
       /* (No Custom FallbackDir) - No Nickname - dir_port: 60093 */
       int found_non_default_fallback = 0;
@@ -2139,7 +2226,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60093 ?
                          1 : 0)
                         );
-      tt_assert(found_non_default_fallback == 0);
+      tt_int_op(found_non_default_fallback, OP_EQ, 0);
 
       /* (No Default FallbackDir) - No Nickname - dir_port: 60099 */
       int found_default_fallback = 0;
@@ -2151,7 +2238,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60099 ?
                          1 : 0)
                         );
-      tt_assert(found_default_fallback == 0);
+      tt_int_op(found_default_fallback, OP_EQ, 0);
     }
   }
 
@@ -2180,7 +2267,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must not have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 0);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 0);
 
     /* we have more fallbacks than just the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 1);
@@ -2189,7 +2276,7 @@ test_config_adding_dir_servers(void *arg)
       /* trusted_dir_servers */
       const smartlist_t *dir_servers = router_get_trusted_dir_servers();
       /* (No D0), B1, A2 */
-      tt_assert(smartlist_len(dir_servers) == 2);
+      tt_int_op(smartlist_len(dir_servers), OP_EQ, 2);
 
       /* (No DirAuthority) - D0 - dir_port: 60090 */
       int found_D0 = 0;
@@ -2201,7 +2288,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* AlternateBridgeAuthority - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2213,7 +2300,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 1);
+      tt_int_op(found_B1, OP_EQ, 1);
 
       /* AlternateDirAuthority - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2225,14 +2312,14 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 1);
+      tt_int_op(found_A2, OP_EQ, 1);
     }
 
     {
       /* fallback_dir_servers */
       const smartlist_t *fallback_servers = router_get_fallback_dir_servers();
       /* (No D0), B1, A2, Custom Fallback */
-      tt_assert(smartlist_len(fallback_servers) == 3);
+      tt_int_op(smartlist_len(fallback_servers), OP_EQ, 3);
 
       /* (No DirAuthority) - D0 - dir_port: 60090 */
       int found_D0 = 0;
@@ -2244,7 +2331,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* AlternateBridgeAuthority - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2256,7 +2343,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 1);
+      tt_int_op(found_B1, OP_EQ, 1);
 
       /* AlternateDirAuthority - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2268,7 +2355,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 1);
+      tt_int_op(found_A2, OP_EQ, 1);
 
       /* Custom FallbackDir - No Nickname - dir_port: 60093 */
       int found_non_default_fallback = 0;
@@ -2280,7 +2367,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60093 ?
                          1 : 0)
                         );
-      tt_assert(found_non_default_fallback == 1);
+      tt_int_op(found_non_default_fallback, OP_EQ, 1);
 
       /* (No Default FallbackDir) - No Nickname - dir_port: 60099 */
       int found_default_fallback = 0;
@@ -2292,7 +2379,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60099 ?
                          1 : 0)
                         );
-      tt_assert(found_default_fallback == 0);
+      tt_int_op(found_default_fallback, OP_EQ, 0);
     }
   }
 
@@ -2322,7 +2409,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must not have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 0);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 0);
 
     /* we have more fallbacks than just the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 0);
@@ -2331,7 +2418,7 @@ test_config_adding_dir_servers(void *arg)
       /* trusted_dir_servers */
       const smartlist_t *dir_servers = router_get_trusted_dir_servers();
       /* (No D0), B1, A2 */
-      tt_assert(smartlist_len(dir_servers) == 2);
+      tt_int_op(smartlist_len(dir_servers), OP_EQ, 2);
 
       /* (No DirAuthority) - D0 - dir_port: 60090 */
       int found_D0 = 0;
@@ -2343,7 +2430,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* AlternateBridgeAuthority - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2355,7 +2442,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 1);
+      tt_int_op(found_B1, OP_EQ, 1);
 
       /* AlternateDirAuthority - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2367,14 +2454,14 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 1);
+      tt_int_op(found_A2, OP_EQ, 1);
     }
 
     {
       /* fallback_dir_servers */
       const smartlist_t *fallback_servers = router_get_fallback_dir_servers();
       /* (No D0), B1, A2, (No Fallback) */
-      tt_assert(smartlist_len(fallback_servers) == 2);
+      tt_int_op(smartlist_len(fallback_servers), OP_EQ, 2);
 
       /* (No DirAuthority) - D0 - dir_port: 60090 */
       int found_D0 = 0;
@@ -2386,7 +2473,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* AlternateBridgeAuthority - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2398,7 +2485,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 1);
+      tt_int_op(found_B1, OP_EQ, 1);
 
       /* AlternateDirAuthority - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2410,7 +2497,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 1);
+      tt_int_op(found_A2, OP_EQ, 1);
 
       /* (No Custom FallbackDir) - No Nickname - dir_port: 60093 */
       int found_non_default_fallback = 0;
@@ -2422,7 +2509,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60093 ?
                          1 : 0)
                         );
-      tt_assert(found_non_default_fallback == 0);
+      tt_int_op(found_non_default_fallback, OP_EQ, 0);
 
       /* (No Default FallbackDir) - No Nickname - dir_port: 60099 */
       int found_default_fallback = 0;
@@ -2434,7 +2521,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60099 ?
                          1 : 0)
                         );
-      tt_assert(found_default_fallback == 0);
+      tt_int_op(found_default_fallback, OP_EQ, 0);
     }
   }
 
@@ -2474,7 +2561,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must not have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 0);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 0);
 
     /* we have more fallbacks than just the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 1);
@@ -2495,7 +2582,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* AlternateBridgeAuthority - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2507,7 +2594,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 1);
+      tt_int_op(found_B1, OP_EQ, 1);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2519,7 +2606,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
 
       /* There's no easy way of checking that we have included all the
        * default v3 non-Bridge directory authorities, so let's assume that
@@ -2545,7 +2632,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* AlternateBridgeAuthority - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2557,7 +2644,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 1);
+      tt_int_op(found_B1, OP_EQ, 1);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2569,7 +2656,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
 
       /* Custom FallbackDir - No Nickname - dir_port: 60093 */
       int found_non_default_fallback = 0;
@@ -2581,7 +2668,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60093 ?
                          1 : 0)
                         );
-      tt_assert(found_non_default_fallback == 1);
+      tt_int_op(found_non_default_fallback, OP_EQ, 1);
 
       /* (No Default FallbackDir) - No Nickname - dir_port: 60099 */
       int found_default_fallback = 0;
@@ -2593,7 +2680,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60099 ?
                          1 : 0)
                         );
-      tt_assert(found_default_fallback == 0);
+      tt_int_op(found_default_fallback, OP_EQ, 0);
 
       /* There's no easy way of checking that we have included all the
        * default v3 non-Bridge directory authorities, so let's assume that
@@ -2628,7 +2715,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 1);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 1);
 
     /* we have more fallbacks than just the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 1);
@@ -2649,7 +2736,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* AlternateBridgeAuthority - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2661,7 +2748,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 1);
+      tt_int_op(found_B1, OP_EQ, 1);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2673,7 +2760,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
 
       /* There's no easy way of checking that we have included all the
        * default v3 non-Bridge directory authorities, so let's assume that
@@ -2699,7 +2786,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* AlternateBridgeAuthority - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2711,7 +2798,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 1);
+      tt_int_op(found_B1, OP_EQ, 1);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2723,7 +2810,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
 
       /* (No Custom FallbackDir) - No Nickname - dir_port: 60093 */
       int found_non_default_fallback = 0;
@@ -2735,7 +2822,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60093 ?
                          1 : 0)
                         );
-      tt_assert(found_non_default_fallback == 0);
+      tt_int_op(found_non_default_fallback, OP_EQ, 0);
 
       /* Default FallbackDir - No Nickname - dir_port: 60099 */
       int found_default_fallback = 0;
@@ -2747,7 +2834,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60099 ?
                          1 : 0)
                         );
-      tt_assert(found_default_fallback == 1);
+      tt_int_op(found_default_fallback, OP_EQ, 1);
 
       /* There's no easy way of checking that we have included all the
        * default v3 non-Bridge directory authorities, so let's assume that
@@ -2791,7 +2878,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must not have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 0);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 0);
 
     /* we have more fallbacks than just the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 1);
@@ -2813,7 +2900,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2825,7 +2912,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* AlternateDirAuthority - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2837,7 +2924,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 1);
+      tt_int_op(found_A2, OP_EQ, 1);
 
       /* There's no easy way of checking that we have included all the
        * default Bridge authorities (except for hard-coding tonga's details),
@@ -2864,7 +2951,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2876,7 +2963,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* AlternateDirAuthority - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2888,7 +2975,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 1);
+      tt_int_op(found_A2, OP_EQ, 1);
 
       /* Custom FallbackDir - No Nickname - dir_port: 60093 */
       int found_non_default_fallback = 0;
@@ -2900,7 +2987,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60093 ?
                          1 : 0)
                         );
-      tt_assert(found_non_default_fallback == 1);
+      tt_int_op(found_non_default_fallback, OP_EQ, 1);
 
       /* (No Default FallbackDir) - No Nickname - dir_port: 60099 */
       int found_default_fallback = 0;
@@ -2912,7 +2999,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60099 ?
                          1 : 0)
                         );
-      tt_assert(found_default_fallback == 0);
+      tt_int_op(found_default_fallback, OP_EQ, 0);
 
       /* There's no easy way of checking that we have included all the
        * default Bridge authorities (except for hard-coding tonga's details),
@@ -2948,7 +3035,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must not have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 0);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 0);
 
     /* we just have the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 0);
@@ -2971,7 +3058,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2983,7 +3070,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* AlternateDirAuthority - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2995,7 +3082,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 1);
+      tt_int_op(found_A2, OP_EQ, 1);
 
       /* There's no easy way of checking that we have included all the
        * default Bridge authorities (except for hard-coding tonga's details),
@@ -3022,7 +3109,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -3034,7 +3121,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* AlternateDirAuthority - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -3046,7 +3133,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 1);
+      tt_int_op(found_A2, OP_EQ, 1);
 
       /* (No Custom FallbackDir) - No Nickname - dir_port: 60093 */
       int found_non_default_fallback = 0;
@@ -3058,7 +3145,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60093 ?
                          1 : 0)
                         );
-      tt_assert(found_non_default_fallback == 0);
+      tt_int_op(found_non_default_fallback, OP_EQ, 0);
 
       /* (No Default FallbackDir) - No Nickname - dir_port: 60099 */
       int found_default_fallback = 0;
@@ -3070,7 +3157,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60099 ?
                          1 : 0)
                         );
-      tt_assert(found_default_fallback == 0);
+      tt_int_op(found_default_fallback, OP_EQ, 0);
 
       /* There's no easy way of checking that we have included all the
        * default Bridge authorities (except for hard-coding tonga's details),
@@ -3114,7 +3201,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must not have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 0);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 0);
 
     /* we have more fallbacks than just the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 1);
@@ -3136,7 +3223,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -3148,7 +3235,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -3160,7 +3247,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
 
       /* There's no easy way of checking that we have included all the
        * default Bridge & V3 Directory authorities, so let's assume that
@@ -3187,7 +3274,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -3199,7 +3286,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -3211,7 +3298,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
 
       /* Custom FallbackDir - No Nickname - dir_port: 60093 */
       int found_non_default_fallback = 0;
@@ -3223,7 +3310,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60093 ?
                          1 : 0)
                         );
-      tt_assert(found_non_default_fallback == 1);
+      tt_int_op(found_non_default_fallback, OP_EQ, 1);
 
       /* (No Default FallbackDir) - No Nickname - dir_port: 60099 */
       int found_default_fallback = 0;
@@ -3235,7 +3322,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60099 ?
                          1 : 0)
                         );
-      tt_assert(found_default_fallback == 0);
+      tt_int_op(found_default_fallback, OP_EQ, 0);
 
       /* There's no easy way of checking that we have included all the
        * default Bridge & V3 Directory authorities, so let's assume that
@@ -3277,7 +3364,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 1);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 1);
 
     /* we have more fallbacks than just the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 1);
@@ -3299,7 +3386,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -3311,7 +3398,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -3323,7 +3410,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
 
       /* There's no easy way of checking that we have included all the
        * default Bridge & V3 Directory authorities, so let's assume that
@@ -3350,7 +3437,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -3362,7 +3449,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -3374,7 +3461,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
 
       /* Custom FallbackDir - No Nickname - dir_port: 60093 */
       int found_non_default_fallback = 0;
@@ -3386,7 +3473,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60093 ?
                          1 : 0)
                         );
-      tt_assert(found_non_default_fallback == 0);
+      tt_int_op(found_non_default_fallback, OP_EQ, 0);
 
       /* (No Default FallbackDir) - No Nickname - dir_port: 60099 */
       int found_default_fallback = 0;
@@ -3398,7 +3485,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60099 ?
                          1 : 0)
                         );
-      tt_assert(found_default_fallback == 1);
+      tt_int_op(found_default_fallback, OP_EQ, 1);
 
       /* There's no easy way of checking that we have included all the
        * default Bridge & V3 Directory authorities, and the default
@@ -3455,7 +3542,7 @@ test_config_default_dir_servers(void *arg)
   opts = NULL;
 
   /* assume a release will never go out with less than 7 authorities */
-  tt_assert(trusted_count >= 7);
+  tt_int_op(trusted_count, OP_GE, 7);
   /* if we disable the default fallbacks, there must not be any extra */
   tt_assert(fallback_count == trusted_count);
 
@@ -3468,7 +3555,7 @@ test_config_default_dir_servers(void *arg)
   opts = NULL;
 
   /* assume a release will never go out with less than 7 authorities */
-  tt_assert(trusted_count >= 7);
+  tt_int_op(trusted_count, OP_GE, 7);
   /* XX/teor - allow for default fallbacks to be added without breaking
    * the unit tests. Set a minimum fallback count once the list is stable. */
   tt_assert(fallback_count >= trusted_count);
@@ -3537,18 +3624,18 @@ test_config_directory_fetch(void *arg)
   options->ClientOnly = 1;
   tt_assert(server_mode(options) == 0);
   tt_assert(public_server_mode(options) == 0);
-  tt_assert(directory_fetches_from_authorities(options) == 0);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 1);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 1);
 
   /* Bridge Clients can use multiple directory mirrors for bootstrap */
   memset(options, 0, sizeof(or_options_t));
   options->UseBridges = 1;
   tt_assert(server_mode(options) == 0);
   tt_assert(public_server_mode(options) == 0);
-  tt_assert(directory_fetches_from_authorities(options) == 0);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 1);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 1);
 
   /* Bridge Relays (Bridges) must act like clients, and use multiple
    * directory mirrors for bootstrap */
@@ -3557,9 +3644,9 @@ test_config_directory_fetch(void *arg)
   options->ORPort_set = 1;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 0);
-  tt_assert(directory_fetches_from_authorities(options) == 0);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 1);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 1);
 
   /* Clients set to FetchDirInfoEarly must fetch it from the authorities,
    * but can use multiple authorities for bootstrap */
@@ -3567,9 +3654,9 @@ test_config_directory_fetch(void *arg)
   options->FetchDirInfoEarly = 1;
   tt_assert(server_mode(options) == 0);
   tt_assert(public_server_mode(options) == 0);
-  tt_assert(directory_fetches_from_authorities(options) == 1);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 1);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 1);
 
   /* OR servers only fetch the consensus from the authorities when they don't
    * know their own address, but never use multiple directories for bootstrap
@@ -3580,16 +3667,16 @@ test_config_directory_fetch(void *arg)
   mock_router_pick_published_address_result = -1;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_assert(directory_fetches_from_authorities(options) == 1);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 0);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 0);
 
   mock_router_pick_published_address_result = 0;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_assert(directory_fetches_from_authorities(options) == 0);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 0);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 0);
 
   /* Exit OR servers only fetch the consensus from the authorities when they
    * refuse unknown exits, but never use multiple directories for bootstrap
@@ -3607,17 +3694,17 @@ test_config_directory_fetch(void *arg)
   options->RefuseUnknownExits = 1;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_assert(directory_fetches_from_authorities(options) == 1);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 0);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 0);
 
   options->RefuseUnknownExits = 0;
   mock_router_pick_published_address_result = 0;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_assert(directory_fetches_from_authorities(options) == 0);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 0);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 0);
 
   /* Dir servers fetch the consensus from the authorities, unless they are not
    * advertising themselves (hibernating) or have no routerinfo or are not
@@ -3636,26 +3723,26 @@ test_config_directory_fetch(void *arg)
   mock_router_get_my_routerinfo_result = &routerinfo;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_assert(directory_fetches_from_authorities(options) == 1);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 0);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 0);
 
   mock_advertised_server_mode_result = 0;
   routerinfo.dir_port = 1;
   mock_router_get_my_routerinfo_result = &routerinfo;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_assert(directory_fetches_from_authorities(options) == 0);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 0);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 0);
 
   mock_advertised_server_mode_result = 1;
   mock_router_get_my_routerinfo_result = NULL;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_assert(directory_fetches_from_authorities(options) == 0);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 0);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 0);
 
   mock_advertised_server_mode_result = 1;
   routerinfo.dir_port = 0;
@@ -3663,9 +3750,9 @@ test_config_directory_fetch(void *arg)
   mock_router_get_my_routerinfo_result = &routerinfo;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_assert(directory_fetches_from_authorities(options) == 0);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 0);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 0);
 
   mock_advertised_server_mode_result = 1;
   routerinfo.dir_port = 1;
@@ -3673,9 +3760,9 @@ test_config_directory_fetch(void *arg)
   mock_router_get_my_routerinfo_result = &routerinfo;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_assert(directory_fetches_from_authorities(options) == 1);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 0);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 0);
 
  done:
   tor_free(options);
@@ -3689,7 +3776,7 @@ static void
 test_config_default_fallback_dirs(void *arg)
 {
   const char *fallback[] = {
-#include "../or/fallback_dirs.inc"
+#include "app/config/fallback_dirs.inc"
     NULL
   };
 
@@ -3722,119 +3809,119 @@ test_config_port_cfg_line_extract_addrport(void *arg)
 
   tt_int_op(port_cfg_line_extract_addrport("", &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 0);
-  tt_str_op(a, OP_EQ, "");;
+  tt_str_op(a, OP_EQ, "");
   tt_str_op(rest, OP_EQ, "");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("hello", &a, &unixy, &rest),
             OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 0);
-  tt_str_op(a, OP_EQ, "hello");;
+  tt_str_op(a, OP_EQ, "hello");
   tt_str_op(rest, OP_EQ, "");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport(" flipperwalt gersplut",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 0);
-  tt_str_op(a, OP_EQ, "flipperwalt");;
+  tt_str_op(a, OP_EQ, "flipperwalt");
   tt_str_op(rest, OP_EQ, "gersplut");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport(" flipperwalt \t gersplut",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 0);
-  tt_str_op(a, OP_EQ, "flipperwalt");;
+  tt_str_op(a, OP_EQ, "flipperwalt");
   tt_str_op(rest, OP_EQ, "gersplut");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("flipperwalt \t gersplut",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 0);
-  tt_str_op(a, OP_EQ, "flipperwalt");;
+  tt_str_op(a, OP_EQ, "flipperwalt");
   tt_str_op(rest, OP_EQ, "gersplut");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("unix:flipperwalt \t gersplut",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 1);
-  tt_str_op(a, OP_EQ, "flipperwalt");;
+  tt_str_op(a, OP_EQ, "flipperwalt");
   tt_str_op(rest, OP_EQ, "gersplut");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("lolol",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 0);
-  tt_str_op(a, OP_EQ, "lolol");;
+  tt_str_op(a, OP_EQ, "lolol");
   tt_str_op(rest, OP_EQ, "");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("unix:lolol",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 1);
-  tt_str_op(a, OP_EQ, "lolol");;
+  tt_str_op(a, OP_EQ, "lolol");
   tt_str_op(rest, OP_EQ, "");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("unix:lolol ",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 1);
-  tt_str_op(a, OP_EQ, "lolol");;
+  tt_str_op(a, OP_EQ, "lolol");
   tt_str_op(rest, OP_EQ, "");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport(" unix:lolol",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 1);
-  tt_str_op(a, OP_EQ, "lolol");;
+  tt_str_op(a, OP_EQ, "lolol");
   tt_str_op(rest, OP_EQ, "");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("foobar:lolol",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 0);
-  tt_str_op(a, OP_EQ, "foobar:lolol");;
+  tt_str_op(a, OP_EQ, "foobar:lolol");
   tt_str_op(rest, OP_EQ, "");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport(":lolol",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 0);
-  tt_str_op(a, OP_EQ, ":lolol");;
+  tt_str_op(a, OP_EQ, ":lolol");
   tt_str_op(rest, OP_EQ, "");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("unix:\"lolol\"",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 1);
-  tt_str_op(a, OP_EQ, "lolol");;
+  tt_str_op(a, OP_EQ, "lolol");
   tt_str_op(rest, OP_EQ, "");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("unix:\"lolol\" ",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 1);
-  tt_str_op(a, OP_EQ, "lolol");;
+  tt_str_op(a, OP_EQ, "lolol");
   tt_str_op(rest, OP_EQ, "");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("unix:\"lolol\" foo ",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 1);
-  tt_str_op(a, OP_EQ, "lolol");;
+  tt_str_op(a, OP_EQ, "lolol");
   tt_str_op(rest, OP_EQ, "foo ");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("unix:\"lol ol\" foo ",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 1);
-  tt_str_op(a, OP_EQ, "lol ol");;
+  tt_str_op(a, OP_EQ, "lol ol");
   tt_str_op(rest, OP_EQ, "foo ");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("unix:\"lol\\\" ol\" foo ",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 1);
-  tt_str_op(a, OP_EQ, "lol\" ol");;
+  tt_str_op(a, OP_EQ, "lol\" ol");
   tt_str_op(rest, OP_EQ, "foo ");
   tor_free(a);
 
@@ -3861,144 +3948,6 @@ mock_config_line(const char *key, const char *val)
 }
 
 static void
-test_config_parse_port_config__listenaddress(void *data)
-{
-  (void)data;
-  int ret;
-  config_line_t *config_listen_address = NULL, *config_listen_address2 = NULL,
-    *config_listen_address3 = NULL;
-  config_line_t *config_port1 = NULL, *config_port2 = NULL,
-    *config_port3 = NULL, *config_port4 = NULL, *config_port5 = NULL;
-  smartlist_t *slout = NULL;
-  port_cfg_t *port_cfg = NULL;
-
-  // Test basic invocation with no arguments
-  ret = parse_port_config(NULL, NULL, NULL, NULL, 0, NULL, 0, 0);
-  tt_int_op(ret, OP_EQ, 0);
-
-  // Setup some test data
-  config_listen_address = mock_config_line("DNSListenAddress", "127.0.0.1");
-  config_listen_address2 = mock_config_line("DNSListenAddress", "x$$$:::345");
-  config_listen_address3 = mock_config_line("DNSListenAddress",
-                                            "127.0.0.1:1442");
-  config_port1 = mock_config_line("DNSPort", "42");
-  config_port2 = mock_config_line("DNSPort", "43");
-  config_port1->next = config_port2;
-  config_port3 = mock_config_line("DNSPort", "auto");
-  config_port4 = mock_config_line("DNSPort", "55542");
-  config_port5 = mock_config_line("DNSPort", "666777");
-
-  // Test failure when we have a ListenAddress line and several
-  // Port lines for the same portname
-  ret = parse_port_config(NULL, config_port1, config_listen_address, "DNS", 0,
-                          NULL, 0, 0);
-
-  tt_int_op(ret, OP_EQ, -1);
-
-  // Test case when we have a listen address, no default port and allow
-  // spurious listen address lines
-  ret = parse_port_config(NULL, NULL, config_listen_address, "DNS", 0, NULL,
-                          0, CL_PORT_ALLOW_EXTRA_LISTENADDR);
-  tt_int_op(ret, OP_EQ, 1);
-
-  // Test case when we have a listen address, no default port but doesn't
-  // allow spurious listen address lines
-  ret = parse_port_config(NULL, NULL, config_listen_address, "DNS", 0, NULL,
-                          0, 0);
-  tt_int_op(ret, OP_EQ, -1);
-
-  // Test case when we have a listen address, and a port that points to auto,
-  // should use the AUTO port
-  slout = smartlist_new();
-  ret = parse_port_config(slout, config_port3, config_listen_address, "DNS",
-                          0, NULL, 0, 0);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(smartlist_len(slout), OP_EQ, 1);
-  port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
-  tt_int_op(port_cfg->port, OP_EQ, CFG_AUTO_PORT);
-
-  // Test when we have a listen address and a custom port
-  ret = parse_port_config(slout, config_port4, config_listen_address, "DNS",
-                          0, NULL, 0, 0);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(smartlist_len(slout), OP_EQ, 2);
-  port_cfg = (port_cfg_t *)smartlist_get(slout, 1);
-  tt_int_op(port_cfg->port, OP_EQ, 55542);
-
-  // Test when we have a listen address and an invalid custom port
-  ret = parse_port_config(slout, config_port5, config_listen_address, "DNS",
-                          0, NULL, 0, 0);
-  tt_int_op(ret, OP_EQ, -1);
-
-  // Test we get a server port configuration when asked for it
-  ret = parse_port_config(slout, NULL, config_listen_address, "DNS", 0, NULL,
-                          123, CL_PORT_SERVER_OPTIONS);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(smartlist_len(slout), OP_EQ, 4);
-  port_cfg = (port_cfg_t *)smartlist_get(slout, 2);
-  tt_int_op(port_cfg->port, OP_EQ, 123);
-  tt_int_op(port_cfg->server_cfg.no_listen, OP_EQ, 1);
-  tt_int_op(port_cfg->server_cfg.bind_ipv4_only, OP_EQ, 1);
-
-  // Test an invalid ListenAddress configuration
-  ret = parse_port_config(NULL, NULL, config_listen_address2, "DNS", 0, NULL,
-                          222, 0);
-  tt_int_op(ret, OP_EQ, -1);
-
-  // Test default to the port in the listen address if available
-  ret = parse_port_config(slout, config_port2, config_listen_address3, "DNS",
-                          0, NULL, 0, 0);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(smartlist_len(slout), OP_EQ, 5);
-  port_cfg = (port_cfg_t *)smartlist_get(slout, 4);
-  tt_int_op(port_cfg->port, OP_EQ, 1442);
-
-  // Test we work correctly without an out, but with a listen address
-  // and a port
-  ret = parse_port_config(NULL, config_port2, config_listen_address, "DNS",
-                          0, NULL, 0, 0);
-  tt_int_op(ret, OP_EQ, 0);
-
-  // Test warning nonlocal control
-  ret = parse_port_config(slout, config_port2, config_listen_address, "DNS",
-                          CONN_TYPE_CONTROL_LISTENER, NULL, 0,
-                          CL_PORT_WARN_NONLOCAL);
-  tt_int_op(ret, OP_EQ, 0);
-
-  // Test warning nonlocal ext or listener
-  ret = parse_port_config(slout, config_port2, config_listen_address, "DNS",
-                          CONN_TYPE_EXT_OR_LISTENER, NULL, 0,
-                          CL_PORT_WARN_NONLOCAL);
-  tt_int_op(ret, OP_EQ, 0);
-
-  // Test warning nonlocal other
-  SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
-  smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port2, config_listen_address, "DNS",
-                          0, NULL, 0, CL_PORT_WARN_NONLOCAL);
-  tt_int_op(ret, OP_EQ, 0);
-
-  // Test warning nonlocal control without an out
-  ret = parse_port_config(NULL, config_port2, config_listen_address, "DNS",
-                          CONN_TYPE_CONTROL_LISTENER, NULL, 0,
-                          CL_PORT_WARN_NONLOCAL);
-  tt_int_op(ret, OP_EQ, 0);
-
- done:
-  config_free_lines(config_listen_address);
-  config_free_lines(config_listen_address2);
-  config_free_lines(config_listen_address3);
-  config_free_lines(config_port1);
-  /* 2 was linked from 1. */
-  config_free_lines(config_port3);
-  config_free_lines(config_port4);
-  config_free_lines(config_port5);
-  if (slout)
-    SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
-  smartlist_free(slout);
-}
-
-static void
 test_config_parse_port_config__ports__no_ports_given(void *data)
 {
   (void)data;
@@ -4009,40 +3958,40 @@ test_config_parse_port_config__ports__no_ports_given(void *data)
   slout = smartlist_new();
 
   // Test no defaultport, no defaultaddress and no out
-  ret = parse_port_config(NULL, NULL, NULL, "DNS", 0, NULL, 0, 0);
+  ret = parse_port_config(NULL, NULL, "DNS", 0, NULL, 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test with defaultport, no defaultaddress and no out
-  ret = parse_port_config(NULL, NULL, NULL, "DNS", 0, NULL, 42, 0);
+  ret = parse_port_config(NULL, NULL, "DNS", 0, NULL, 42, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test no defaultport, with defaultaddress and no out
-  ret = parse_port_config(NULL, NULL, NULL, "DNS", 0, "127.0.0.2", 0, 0);
+  ret = parse_port_config(NULL, NULL, "DNS", 0, "127.0.0.2", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test with defaultport, with defaultaddress and no out
-  ret = parse_port_config(NULL, NULL, NULL, "DNS", 0, "127.0.0.2", 42, 0);
+  ret = parse_port_config(NULL, NULL, "DNS", 0, "127.0.0.2", 42, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test no defaultport, no defaultaddress and with out
-  ret = parse_port_config(slout, NULL, NULL, "DNS", 0, NULL, 0, 0);
+  ret = parse_port_config(slout, NULL, "DNS", 0, NULL, 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 0);
 
   // Test with defaultport, no defaultaddress and with out
-  ret = parse_port_config(slout, NULL, NULL, "DNS", 0, NULL, 42, 0);
+  ret = parse_port_config(slout, NULL, "DNS", 0, NULL, 42, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 0);
 
   // Test no defaultport, with defaultaddress and with out
-  ret = parse_port_config(slout, NULL, NULL, "DNS", 0, "127.0.0.2", 0, 0);
+  ret = parse_port_config(slout, NULL, "DNS", 0, "127.0.0.2", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 0);
 
   // Test with defaultport, with defaultaddress and out, adds a new port cfg
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, NULL, NULL, "DNS", 0, "127.0.0.2", 42, 0);
+  ret = parse_port_config(slout, NULL, "DNS", 0, "127.0.0.2", 42, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
@@ -4053,7 +4002,7 @@ test_config_parse_port_config__ports__no_ports_given(void *data)
   // for a unix address
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, NULL, NULL, "DNS", 0, "/foo/bar/unixdomain",
+  ret = parse_port_config(slout, NULL, "DNS", 0, "/foo/bar/unixdomain",
                           42, CL_PORT_IS_UNIXSOCKET);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4082,28 +4031,28 @@ test_config_parse_port_config__ports__ports_given(void *data)
 
   // Test error when encounters an invalid Port specification
   config_port_invalid = mock_config_line("DNSPort", "");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "DNS", 0, NULL,
+  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0, NULL,
                           0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test error when encounters an empty unix domain specification
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "unix:");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "DNS", 0, NULL,
+  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0, NULL,
                           0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test error when encounters a unix domain specification but the listener
   // doesn't support domain sockets
   config_port_valid = mock_config_line("DNSPort", "unix:/tmp/foo/bar");
-  ret = parse_port_config(NULL, config_port_valid, NULL, "DNS",
+  ret = parse_port_config(NULL, config_port_valid, "DNS",
                           CONN_TYPE_AP_DNS_LISTENER, NULL, 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test valid unix domain
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0, 0);
 #ifdef _WIN32
   tt_int_op(ret, OP_EQ, -1);
@@ -4118,16 +4067,17 @@ test_config_parse_port_config__ports__ports_given(void *data)
   tt_int_op(port_cfg->entry_cfg.dns_request, OP_EQ, 1);
   tt_int_op(port_cfg->entry_cfg.ipv4_traffic, OP_EQ, 1);
   tt_int_op(port_cfg->entry_cfg.onion_traffic, OP_EQ, 1);
-  tt_int_op(port_cfg->entry_cfg.cache_ipv4_answers, OP_EQ, 1);
+  tt_int_op(port_cfg->entry_cfg.cache_ipv4_answers, OP_EQ, 0);
   tt_int_op(port_cfg->entry_cfg.prefer_ipv6_virtaddr, OP_EQ, 1);
-#endif
+#endif /* defined(_WIN32) */
 
   // Test failure if we have no ipv4 and no ipv6 and no onion (DNS only)
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("SOCKSPort",
                                          "unix:/tmp/foo/bar NoIPv4Traffic "
+                                         "NoIPv6Traffic "
                                          "NoOnionTraffic");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "SOCKS",
+  ret = parse_port_config(NULL, config_port_invalid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4136,7 +4086,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort",
                                          "127.0.0.1:80 NoDNSRequest");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "DNS",
+  ret = parse_port_config(NULL, config_port_invalid, "DNS",
                           CONN_TYPE_AP_DNS_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4147,8 +4097,9 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.1:80 "
+                                       "NoIPv6Traffic "
                                        "NoIPv4Traffic NoOnionTraffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS",
+  ret = parse_port_config(slout, config_port_valid, "DNS",
                           CONN_TYPE_AP_DNS_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, 0);
@@ -4162,8 +4113,9 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test failure if we have DNS but no ipv4 and no ipv6
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("SOCKSPort",
+                                         "NoIPv6Traffic "
                                          "unix:/tmp/foo/bar NoIPv4Traffic");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "SOCKS",
+  ret = parse_port_config(NULL, config_port_invalid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4174,8 +4126,9 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/foo/bar "
+                                       "NoIPv6Traffic "
                                        "NoDNSRequest NoIPv4Traffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4188,15 +4141,16 @@ test_config_parse_port_config__ports__ports_given(void *data)
   tt_int_op(port_cfg->entry_cfg.ipv4_traffic, OP_EQ, 0);
   tt_int_op(port_cfg->entry_cfg.ipv6_traffic, OP_EQ, 0);
   tt_int_op(port_cfg->entry_cfg.onion_traffic, OP_EQ, 1);
-#endif
+#endif /* defined(_WIN32) */
 
   // Test success with quoted unix: address.
   config_free_lines(config_port_valid); config_port_valid = NULL;
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:\"/tmp/foo/ bar\" "
+                                       "NoIPv6Traffic "
                                        "NoDNSRequest NoIPv4Traffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4209,15 +4163,16 @@ test_config_parse_port_config__ports__ports_given(void *data)
   tt_int_op(port_cfg->entry_cfg.ipv4_traffic, OP_EQ, 0);
   tt_int_op(port_cfg->entry_cfg.ipv6_traffic, OP_EQ, 0);
   tt_int_op(port_cfg->entry_cfg.onion_traffic, OP_EQ, 1);
-#endif
+#endif /* defined(_WIN32) */
 
   // Test failure with broken quoted unix: address.
   config_free_lines(config_port_valid); config_port_valid = NULL;
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:\"/tmp/foo/ bar "
+                                       "NoIPv6Traffic "
                                        "NoDNSRequest NoIPv4Traffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4227,8 +4182,9 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:\"\" "
+                                       "NoIPv6Traffic "
                                        "NoDNSRequest NoIPv4Traffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4239,7 +4195,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/foo/bar "
                                        "OnionTrafficOnly");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4252,7 +4208,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   tt_int_op(port_cfg->entry_cfg.ipv4_traffic, OP_EQ, 0);
   tt_int_op(port_cfg->entry_cfg.ipv6_traffic, OP_EQ, 0);
   tt_int_op(port_cfg->entry_cfg.onion_traffic, OP_EQ, 1);
-#endif
+#endif /* defined(_WIN32) */
 
   // Test success with no ipv4 but take ipv6
   config_free_lines(config_port_valid); config_port_valid = NULL;
@@ -4260,7 +4216,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/foo/bar "
                                        "NoIPv4Traffic IPv6Traffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4271,7 +4227,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
   tt_int_op(port_cfg->entry_cfg.ipv4_traffic, OP_EQ, 0);
   tt_int_op(port_cfg->entry_cfg.ipv6_traffic, OP_EQ, 1);
-#endif
+#endif /* defined(_WIN32) */
 
   // Test success with both ipv4 and ipv6
   config_free_lines(config_port_valid); config_port_valid = NULL;
@@ -4279,7 +4235,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/foo/bar "
                                        "IPv4Traffic IPv6Traffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4290,33 +4246,33 @@ test_config_parse_port_config__ports__ports_given(void *data)
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
   tt_int_op(port_cfg->entry_cfg.ipv4_traffic, OP_EQ, 1);
   tt_int_op(port_cfg->entry_cfg.ipv6_traffic, OP_EQ, 1);
-#endif
+#endif /* defined(_WIN32) */
 
   // Test failure if we specify world writable for an IP Port
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "42 WorldWritable");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test failure if we specify group writable for an IP Port
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "42 GroupWritable");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test failure if we specify group writable for an IP Port
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "42 RelaxDirModeCheck");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test success with only a port (this will fail without a default address)
   config_free_lines(config_port_valid); config_port_valid = NULL;
   config_port_valid = mock_config_line("DNSPort", "42");
-  ret = parse_port_config(NULL, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(NULL, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
@@ -4325,7 +4281,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateDestPort");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4338,7 +4294,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 NoIsolateDestPorts");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4351,7 +4307,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateDestAddr");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4364,7 +4320,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateSOCKSAuth");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4377,7 +4333,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateClientProtocol");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4390,7 +4346,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateClientAddr");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4401,7 +4357,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test success with ignored unknown options
   config_free_lines(config_port_valid); config_port_valid = NULL;
   config_port_valid = mock_config_line("DNSPort", "42 ThisOptionDoesntExist");
-  ret = parse_port_config(NULL, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(NULL, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
@@ -4410,7 +4366,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 NoIsolateSOCKSAuth");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4423,7 +4379,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort",
                                        "42 IPv6Traffic PreferIPv6");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, "127.0.0.42", 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, 0);
@@ -4436,7 +4392,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 CacheIPv4DNS");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4449,12 +4405,12 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 CacheIPv6DNS");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
-  tt_int_op(port_cfg->entry_cfg.cache_ipv4_answers, OP_EQ, 1);
+  tt_int_op(port_cfg->entry_cfg.cache_ipv4_answers, OP_EQ, 0);
   tt_int_op(port_cfg->entry_cfg.cache_ipv6_answers, OP_EQ, 1);
 
   // Test success with no cache ipv4 DNS
@@ -4462,7 +4418,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 NoCacheIPv4DNS");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4475,7 +4431,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 CacheDNS");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4488,7 +4444,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 UseIPv4Cache");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4501,7 +4457,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 UseIPv6Cache");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4514,7 +4470,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 UseDNSCache");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4527,7 +4483,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 NoPreferIPv6Automap");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4539,7 +4495,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 PreferSOCKSNoAuth");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4554,14 +4510,14 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_port_invalid = mock_config_line("DNSPort", "0");
   config_port_valid = mock_config_line("DNSPort", "42");
   config_port_invalid->next = config_port_valid;
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test success with warn non-local control
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port_valid, NULL, "Control",
+  ret = parse_port_config(slout, config_port_valid, "Control",
                           CONN_TYPE_CONTROL_LISTENER, "127.0.0.42", 0,
                           CL_PORT_WARN_NONLOCAL);
   tt_int_op(ret, OP_EQ, 0);
@@ -4569,7 +4525,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test success with warn non-local listener
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port_valid, NULL, "ExtOR",
+  ret = parse_port_config(slout, config_port_valid, "ExtOR",
                           CONN_TYPE_EXT_OR_LISTENER, "127.0.0.42", 0,
                           CL_PORT_WARN_NONLOCAL);
   tt_int_op(ret, OP_EQ, 0);
@@ -4577,12 +4533,12 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test success with warn non-local other
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, CL_PORT_WARN_NONLOCAL);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test success with warn non-local other without out
-  ret = parse_port_config(NULL, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(NULL, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, CL_PORT_WARN_NONLOCAL);
   tt_int_op(ret, OP_EQ, 0);
 
@@ -4593,7 +4549,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IPv4Traffic "
                                        "IPv6Traffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.44", 0,
                           CL_PORT_TAKES_HOSTNAMES |
                           CL_PORT_NO_STREAM_OPTIONS);
@@ -4601,27 +4557,25 @@ test_config_parse_port_config__ports__ports_given(void *data)
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
   tt_int_op(port_cfg->entry_cfg.ipv4_traffic, OP_EQ, 1);
-  tt_int_op(port_cfg->entry_cfg.ipv6_traffic, OP_EQ, 0);
+  tt_int_op(port_cfg->entry_cfg.ipv6_traffic, OP_EQ, 1);
 
   // Test failure for a SessionGroup argument with invalid value
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "42 SessionGroup=invalid");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.44", 0, CL_PORT_NO_STREAM_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
-  // TODO: this seems wrong. Shouldn't it be the other way around?
-  // Potential bug.
-  // Test failure for a SessionGroup argument with valid value but with stream
-  // options allowed
+  // Test failure for a SessionGroup argument with valid value but with no
+  // stream options allowed
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "42 SessionGroup=123");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0,
-                          "127.0.0.44", 0, 0);
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
+                          "127.0.0.44", 0, CL_PORT_NO_STREAM_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test failure for more than one SessionGroup argument
@@ -4630,8 +4584,8 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "42 SessionGroup=123 "
                                          "SessionGroup=321");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0,
-                          "127.0.0.44", 0, CL_PORT_NO_STREAM_OPTIONS);
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
+                          "127.0.0.44", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test success with a sessiongroup options
@@ -4639,8 +4593,8 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 SessionGroup=1111122");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
-                          "127.0.0.44", 0, CL_PORT_NO_STREAM_OPTIONS);
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+                          "127.0.0.44", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
@@ -4651,7 +4605,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "0");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.45", 0, CL_PORT_IS_UNIXSOCKET);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 0);
@@ -4661,7 +4615,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "something");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.45", 0, CL_PORT_IS_UNIXSOCKET);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4674,7 +4628,21 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "auto");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+                          "127.0.0.46", 0, 0);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(smartlist_len(slout), OP_EQ, 1);
+  port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
+  tt_int_op(port_cfg->port, OP_EQ, CFG_AUTO_PORT);
+  tor_addr_parse(&addr, "127.0.0.46");
+  tt_assert(tor_addr_eq(&port_cfg->addr, &addr))
+
+  // Test success with a port of auto in mixed case
+  config_free_lines(config_port_valid); config_port_valid = NULL;
+  SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
+  smartlist_clear(slout);
+  config_port_valid = mock_config_line("DNSPort", "AuTo");
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4688,7 +4656,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.122:auto");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4700,8 +4668,10 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test failure when asked to parse an invalid address followed by auto
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "invalidstuff!!:auto");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "DNS", 0,
+  MOCK(tor_addr_lookup, mock_tor_addr_lookup__fail_on_bad_addrs);
+  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0,
                           "127.0.0.46", 0, 0);
+  UNMOCK(tor_addr_lookup);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test success with parsing both an address and a real port
@@ -4709,7 +4679,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.123:656");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4723,7 +4693,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "something wrong");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4732,7 +4702,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "127.0.1.0:123:auto");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4742,7 +4712,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/somewhere");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, "127.0.0.46", 0,
                           CL_PORT_DFLT_GROUP_WRITABLE);
 #ifdef _WIN32
@@ -4752,7 +4722,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
   tt_int_op(port_cfg->is_group_writable, OP_EQ, 1);
-#endif
+#endif /* defined(_WIN32) */
 
  done:
   if (slout)
@@ -4777,7 +4747,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   config_free_lines(config_port_valid); config_port_valid = NULL;
   config_port_valid = mock_config_line("DNSPort",
                                        "127.0.0.124:656 NoAdvertise");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0, NULL, 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4790,7 +4760,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.124:656 NoListen");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0, NULL, 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4804,7 +4774,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "127.0.0.124:656 NoListen "
                                          "NoAdvertise");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0, NULL,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4813,7 +4783,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.124:656 IPv4Only");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0, NULL, 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4826,7 +4796,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "[::1]:656 IPv6Only");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0, NULL, 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4840,7 +4810,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "127.0.0.124:656 IPv6Only "
                                          "IPv4Only");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0, NULL,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4849,7 +4819,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.124:656 unknown");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0, NULL, 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4860,7 +4830,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort",
                                          "127.0.0.124:656 IPv6Only");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0, NULL,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4869,7 +4839,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "[::1]:656 IPv4Only");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0, NULL,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4878,7 +4848,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("ORPort", "unix:\"\"");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "ORPort", 0, NULL,
+  ret = parse_port_config(slout, config_port_invalid, "ORPort", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4890,6 +4860,1024 @@ test_config_parse_port_config__ports__server_options(void *data)
   config_free_lines(config_port_valid); config_port_valid = NULL;
 }
 
+static void
+test_config_parse_log_severity(void *data)
+{
+  int ret;
+  const char *severity_log_lines[] = {
+    "debug file /tmp/debug.log",
+    "debug\tfile /tmp/debug.log",
+    "[handshake]debug [~net,~mm]info notice stdout",
+    "[handshake]debug\t[~net,~mm]info\tnotice\tstdout",
+    NULL
+  };
+  int i;
+  log_severity_list_t *severity;
+
+  (void) data;
+
+  severity = tor_malloc(sizeof(log_severity_list_t));
+  for (i = 0; severity_log_lines[i]; i++) {
+    memset(severity, 0, sizeof(log_severity_list_t));
+    ret = parse_log_severity_config(&severity_log_lines[i], severity);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+ done:
+  tor_free(severity);
+}
+
+static void
+test_config_include_limit(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *torrc_path = NULL;
+  char *dir = tor_strdup(get_fname("test_include_limit"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&torrc_path, "%s"PATH_SEPARATOR"torrc", dir);
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents), "%%include %s",
+               torrc_path);
+  tt_int_op(write_str_to_file(torrc_path, torrc_contents, 0), OP_EQ, 0);
+
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, NULL, NULL),
+            OP_EQ, -1);
+
+ done:
+  config_free_lines(result);
+  tor_free(torrc_path);
+  tor_free(dir);
+}
+
+static void
+test_config_include_does_not_exist(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *dir = tor_strdup(get_fname("test_include_does_not_exist"));
+  char *missing_path = NULL;
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&missing_path, "%s"PATH_SEPARATOR"missing", dir);
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents), "%%include %s",
+               missing_path);
+
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, NULL, NULL),
+            OP_EQ, -1);
+
+ done:
+  config_free_lines(result);
+  tor_free(dir);
+  tor_free(missing_path);
+}
+
+static void
+test_config_include_error_in_included_file(void *data)
+{
+  (void)data;
+  config_line_t *result = NULL;
+
+  char *dir = tor_strdup(get_fname("test_error_in_included_file"));
+  char *invalid_path = NULL;
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&invalid_path, "%s"PATH_SEPARATOR"invalid", dir);
+  tt_int_op(write_str_to_file(invalid_path, "unclosed \"", 0), OP_EQ, 0);
+
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents), "%%include %s",
+               invalid_path);
+
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, NULL, NULL),
+            OP_EQ, -1);
+
+ done:
+  config_free_lines(result);
+  tor_free(dir);
+  tor_free(invalid_path);
+}
+
+static void
+test_config_include_empty_file_folder(void *data)
+{
+  (void)data;
+  config_line_t *result = NULL;
+
+  char *folder_path = NULL;
+  char *file_path = NULL;
+  char *dir = tor_strdup(get_fname("test_include_empty_file_folder"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&folder_path, "%s"PATH_SEPARATOR"empty_dir", dir);
+#ifdef _WIN32
+  tt_int_op(mkdir(folder_path), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(folder_path, 0700), OP_EQ, 0);
+#endif
+  tor_asprintf(&file_path, "%s"PATH_SEPARATOR"empty_file", dir);
+  tt_int_op(write_str_to_file(file_path, "", 0), OP_EQ, 0);
+
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s\n"
+               "%%include %s\n",
+               folder_path, file_path);
+
+  int include_used;
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0,&include_used,
+            NULL), OP_EQ, 0);
+  tt_ptr_op(result, OP_EQ, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+ done:
+  config_free_lines(result);
+  tor_free(folder_path);
+  tor_free(file_path);
+  tor_free(dir);
+}
+
+#ifndef _WIN32
+static void
+test_config_include_no_permission(void *data)
+{
+  (void)data;
+  config_line_t *result = NULL;
+
+  char *folder_path = NULL;
+  char *dir = NULL;
+  if (geteuid() == 0)
+    tt_skip();
+
+  dir = tor_strdup(get_fname("test_include_forbidden_folder"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+
+  tor_asprintf(&folder_path, "%s"PATH_SEPARATOR"forbidden_dir", dir);
+  tt_int_op(mkdir(folder_path, 0100), OP_EQ, 0);
+
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s\n",
+               folder_path);
+
+  int include_used;
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0,
+                                     &include_used, NULL),
+            OP_EQ, -1);
+  tt_ptr_op(result, OP_EQ, NULL);
+
+ done:
+  config_free_lines(result);
+  tor_free(folder_path);
+  if (dir)
+    chmod(dir, 0700);
+  tor_free(dir);
+}
+#endif
+
+static void
+test_config_include_recursion_before_after(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *torrc_path = NULL;
+  char *dir = tor_strdup(get_fname("test_include_recursion_before_after"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&torrc_path, "%s"PATH_SEPARATOR"torrc", dir);
+
+  char file_contents[1000];
+  const int limit = MAX_INCLUDE_RECURSION_LEVEL;
+  int i;
+  // Loop backwards so file_contents has the contents of the first file by the
+  // end of the loop
+  for (i = limit; i > 0; i--) {
+    if (i < limit) {
+      tor_snprintf(file_contents, sizeof(file_contents),
+                   "Test %d\n"
+                   "%%include %s%d\n"
+                   "Test %d\n",
+                   i, torrc_path, i + 1, 2 * limit - i);
+    } else {
+      tor_snprintf(file_contents, sizeof(file_contents), "Test %d\n", i);
+    }
+
+    if (i > 1) {
+      char *file_path = NULL;
+      tor_asprintf(&file_path, "%s%d", torrc_path, i);
+      tt_int_op(write_str_to_file(file_path, file_contents, 0), OP_EQ, 0);
+      tor_free(file_path);
+    }
+  }
+
+  int include_used;
+  tt_int_op(config_get_lines_include(file_contents, &result, 0, &include_used,
+            NULL), OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+  int len = 0;
+  config_line_t *next;
+  for (next = result; next != NULL; next = next->next) {
+    char expected[10];
+    tor_snprintf(expected, sizeof(expected), "%d", len + 1);
+    tt_str_op(next->key, OP_EQ, "Test");
+    tt_str_op(next->value, OP_EQ, expected);
+    len++;
+  }
+  tt_int_op(len, OP_EQ, 2 * limit - 1);
+
+ done:
+  config_free_lines(result);
+  tor_free(dir);
+  tor_free(torrc_path);
+}
+
+static void
+test_config_include_recursion_after_only(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *torrc_path = NULL;
+  char *dir = tor_strdup(get_fname("test_include_recursion_after_only"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&torrc_path, "%s"PATH_SEPARATOR"torrc", dir);
+
+  char file_contents[1000];
+  const int limit = MAX_INCLUDE_RECURSION_LEVEL;
+  int i;
+  // Loop backwards so file_contents has the contents of the first file by the
+  // end of the loop
+  for (i = limit; i > 0; i--) {
+    int n = (i - limit - 1) * -1;
+    if (i < limit) {
+      tor_snprintf(file_contents, sizeof(file_contents),
+                   "%%include %s%d\n"
+                   "Test %d\n",
+                   torrc_path, i + 1, n);
+    } else {
+      tor_snprintf(file_contents, sizeof(file_contents), "Test %d\n", n);
+    }
+
+    if (i > 1) {
+      char *file_path = NULL;
+      tor_asprintf(&file_path, "%s%d", torrc_path, i);
+      tt_int_op(write_str_to_file(file_path, file_contents, 0), OP_EQ, 0);
+      tor_free(file_path);
+    }
+  }
+
+  int include_used;
+  tt_int_op(config_get_lines_include(file_contents, &result, 0, &include_used,
+            NULL), OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+  int len = 0;
+  config_line_t *next;
+  for (next = result; next != NULL; next = next->next) {
+    char expected[10];
+    tor_snprintf(expected, sizeof(expected), "%d", len + 1);
+    tt_str_op(next->key, OP_EQ, "Test");
+    tt_str_op(next->value, OP_EQ, expected);
+    len++;
+  }
+  tt_int_op(len, OP_EQ, limit);
+
+ done:
+  config_free_lines(result);
+  tor_free(dir);
+  tor_free(torrc_path);
+}
+
+static void
+test_config_include_folder_order(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *torrcd = NULL;
+  char *path = NULL;
+  char *path2 = NULL;
+  char *dir = tor_strdup(get_fname("test_include_folder_order"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&torrcd, "%s"PATH_SEPARATOR"%s", dir, "torrc.d");
+
+#ifdef _WIN32
+  tt_int_op(mkdir(torrcd), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(torrcd, 0700), OP_EQ, 0);
+#endif
+
+  // test that files in subfolders are ignored
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", torrcd, "subfolder");
+
+#ifdef _WIN32
+  tt_int_op(mkdir(path), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(path, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&path2, "%s"PATH_SEPARATOR"%s", path, "01_ignore");
+  tt_int_op(write_str_to_file(path2, "ShouldNotSee 1\n", 0), OP_EQ, 0);
+  tor_free(path);
+
+  // test that files starting with . are ignored
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", torrcd, ".dot");
+  tt_int_op(write_str_to_file(path, "ShouldNotSee 2\n", 0), OP_EQ, 0);
+  tor_free(path);
+
+  // test file order
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", torrcd, "01_1st");
+  tt_int_op(write_str_to_file(path, "Test 1\n", 0), OP_EQ, 0);
+  tor_free(path);
+
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", torrcd, "02_2nd");
+  tt_int_op(write_str_to_file(path, "Test 2\n", 0), OP_EQ, 0);
+  tor_free(path);
+
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", torrcd, "aa_3rd");
+  tt_int_op(write_str_to_file(path, "Test 3\n", 0), OP_EQ, 0);
+  tor_free(path);
+
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", torrcd, "ab_4th");
+  tt_int_op(write_str_to_file(path, "Test 4\n", 0), OP_EQ, 0);
+  tor_free(path);
+
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s\n",
+               torrcd);
+
+  int include_used;
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, &include_used,
+            NULL), OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+  int len = 0;
+  config_line_t *next;
+  for (next = result; next != NULL; next = next->next) {
+    char expected[10];
+    tor_snprintf(expected, sizeof(expected), "%d", len + 1);
+    tt_str_op(next->key, OP_EQ, "Test");
+    tt_str_op(next->value, OP_EQ, expected);
+    len++;
+  }
+  tt_int_op(len, OP_EQ, 4);
+
+ done:
+  config_free_lines(result);
+  tor_free(torrcd);
+  tor_free(path);
+  tor_free(path2);
+  tor_free(dir);
+}
+
+static void
+test_config_include_blank_file_last(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *torrcd = NULL;
+  char *path = NULL;
+  char *dir = tor_strdup(get_fname("test_include_blank_file_last"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&torrcd, "%s"PATH_SEPARATOR"%s", dir, "torrc.d");
+
+#ifdef _WIN32
+  tt_int_op(mkdir(torrcd), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(torrcd, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", torrcd, "aa_1st");
+  tt_int_op(write_str_to_file(path, "Test 1\n", 0), OP_EQ, 0);
+  tor_free(path);
+
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", torrcd, "bb_2nd");
+  tt_int_op(write_str_to_file(path, "Test 2\n", 0), OP_EQ, 0);
+  tor_free(path);
+
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", torrcd, "cc_comment");
+  tt_int_op(write_str_to_file(path, "# comment only\n", 0), OP_EQ, 0);
+  tor_free(path);
+
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s\n"
+               "Test 3\n",
+               torrcd);
+
+  int include_used;
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, &include_used,
+            NULL), OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+  int len = 0;
+  config_line_t *next;
+  for (next = result; next != NULL; next = next->next) {
+    char expected[10];
+    tor_snprintf(expected, sizeof(expected), "%d", len + 1);
+    tt_str_op(next->key, OP_EQ, "Test");
+    tt_str_op(next->value, OP_EQ, expected);
+    len++;
+  }
+  tt_int_op(len, OP_EQ, 3);
+
+ done:
+  config_free_lines(result);
+  tor_free(torrcd);
+  tor_free(path);
+  tor_free(dir);
+}
+
+static void
+test_config_include_path_syntax(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *dir = tor_strdup(get_fname("test_include_path_syntax"));
+  char *esc_dir = NULL, *dir_with_pathsep = NULL,
+    *esc_dir_with_pathsep = NULL, *torrc_contents = NULL;
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  esc_dir = esc_for_log(dir);
+  tor_asprintf(&dir_with_pathsep, "%s%s", dir, PATH_SEPARATOR);
+  esc_dir_with_pathsep = esc_for_log(dir_with_pathsep);
+
+  tor_asprintf(&torrc_contents,
+               "%%include %s\n"
+               "%%include %s%s \n" // space to avoid suppressing newline
+               "%%include %s\n",
+               esc_dir,
+               dir, PATH_SEPARATOR,
+               esc_dir_with_pathsep);
+
+  int include_used;
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0,&include_used,
+            NULL), OP_EQ, 0);
+  tt_ptr_op(result, OP_EQ, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+ done:
+  config_free_lines(result);
+  tor_free(dir);
+  tor_free(torrc_contents);
+  tor_free(esc_dir);
+  tor_free(dir_with_pathsep);
+  tor_free(esc_dir_with_pathsep);
+}
+
+static void
+test_config_include_not_processed(void *data)
+{
+  (void)data;
+
+  char torrc_contents[1000] = "%include does_not_exist\n";
+  config_line_t *result = NULL;
+  tt_int_op(config_get_lines(torrc_contents, &result, 0),OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+
+  int len = 0;
+  config_line_t *next;
+  for (next = result; next != NULL; next = next->next) {
+    tt_str_op(next->key, OP_EQ, "%include");
+    tt_str_op(next->value, OP_EQ, "does_not_exist");
+    len++;
+  }
+  tt_int_op(len, OP_EQ, 1);
+
+ done:
+  config_free_lines(result);
+}
+
+static void
+test_config_include_has_include(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *dir = tor_strdup(get_fname("test_include_has_include"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  char torrc_contents[1000] = "Test 1\n";
+  int include_used;
+
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0,&include_used,
+            NULL), OP_EQ, 0);
+  tt_int_op(include_used, OP_EQ, 0);
+  config_free_lines(result);
+
+  tor_snprintf(torrc_contents, sizeof(torrc_contents), "%%include %s\n", dir);
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0,&include_used,
+            NULL), OP_EQ, 0);
+  tt_int_op(include_used, OP_EQ, 1);
+
+ done:
+  config_free_lines(result);
+  tor_free(dir);
+}
+
+static void
+test_config_include_flag_both_without(void *data)
+{
+  (void)data;
+
+  char *errmsg = NULL;
+  char conf_empty[1000];
+  tor_snprintf(conf_empty, sizeof(conf_empty),
+               "DataDirectory %s\n",
+               get_fname(NULL));
+  // test with defaults-torrc and torrc without include
+  int ret = options_init_from_string(conf_empty, conf_empty, CMD_RUN_UNITTESTS,
+                                     NULL, &errmsg);
+  tt_int_op(ret, OP_EQ, 0);
+
+  const or_options_t *options = get_options();
+  tt_int_op(options->IncludeUsed, OP_EQ, 0);
+
+ done:
+  tor_free(errmsg);
+}
+
+static void
+test_config_include_flag_torrc_only(void *data)
+{
+  (void)data;
+
+  char *errmsg = NULL;
+  char *path = NULL;
+  char *dir = tor_strdup(get_fname("test_include_flag_torrc_only"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", dir, "dummy");
+  tt_int_op(write_str_to_file(path, "\n", 0), OP_EQ, 0);
+
+  char conf_empty[1000];
+  tor_snprintf(conf_empty, sizeof(conf_empty),
+               "DataDirectory %s\n",
+               get_fname(NULL));
+  char conf_include[1000];
+  tor_snprintf(conf_include, sizeof(conf_include), "%%include %s", path);
+
+  // test with defaults-torrc without include and torrc with include
+  int ret = options_init_from_string(conf_empty, conf_include,
+                                     CMD_RUN_UNITTESTS, NULL, &errmsg);
+  tt_int_op(ret, OP_EQ, 0);
+
+  const or_options_t *options = get_options();
+  tt_int_op(options->IncludeUsed, OP_EQ, 1);
+
+ done:
+  tor_free(errmsg);
+  tor_free(path);
+  tor_free(dir);
+}
+
+static void
+test_config_include_flag_defaults_only(void *data)
+{
+  (void)data;
+
+  char *errmsg = NULL;
+  char *path = NULL;
+  char *dir = tor_strdup(get_fname("test_include_flag_defaults_only"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", dir, "dummy");
+  tt_int_op(write_str_to_file(path, "\n", 0), OP_EQ, 0);
+
+  char conf_empty[1000];
+  tor_snprintf(conf_empty, sizeof(conf_empty),
+               "DataDirectory %s\n",
+               get_fname(NULL));
+  char conf_include[1000];
+  tor_snprintf(conf_include, sizeof(conf_include), "%%include %s", path);
+
+  // test with defaults-torrc with include and torrc without include
+  int ret = options_init_from_string(conf_include, conf_empty,
+                                     CMD_RUN_UNITTESTS, NULL, &errmsg);
+  tt_int_op(ret, OP_EQ, 0);
+
+  const or_options_t *options = get_options();
+  tt_int_op(options->IncludeUsed, OP_EQ, 0);
+
+ done:
+  tor_free(errmsg);
+  tor_free(path);
+  tor_free(dir);
+}
+
+static void
+test_config_dup_and_filter(void *arg)
+{
+  (void)arg;
+  /* Test normal input. */
+  config_line_t *line = NULL;
+  config_line_append(&line, "abc", "def");
+  config_line_append(&line, "ghi", "jkl");
+  config_line_append(&line, "ABCD", "mno");
+
+  config_line_t *line_dup = config_lines_dup_and_filter(line, "aBc");
+  tt_ptr_op(line_dup, OP_NE, NULL);
+  tt_ptr_op(line_dup->next, OP_NE, NULL);
+  tt_ptr_op(line_dup->next->next, OP_EQ, NULL);
+
+  tt_str_op(line_dup->key, OP_EQ, "abc");
+  tt_str_op(line_dup->value, OP_EQ, "def");
+  tt_str_op(line_dup->next->key, OP_EQ, "ABCD");
+  tt_str_op(line_dup->next->value, OP_EQ, "mno");
+
+  /* empty output */
+  config_free_lines(line_dup);
+  line_dup = config_lines_dup_and_filter(line, "skdjfsdkljf");
+  tt_ptr_op(line_dup, OP_EQ, NULL);
+
+  /* empty input */
+  config_free_lines(line_dup);
+  line_dup = config_lines_dup_and_filter(NULL, "abc");
+  tt_ptr_op(line_dup, OP_EQ, NULL);
+
+ done:
+  config_free_lines(line);
+  config_free_lines(line_dup);
+}
+
+/* If we're not configured to be a bridge, but we set
+ * BridgeDistribution, then options_validate () should return -1. */
+static void
+test_config_check_bridge_distribution_setting_not_a_bridge(void *arg)
+{
+  or_options_t* options = get_options_mutable();
+  or_options_t* old_options = options;
+  or_options_t* default_options = options;
+  char* message = NULL;
+  int ret;
+
+  (void)arg;
+
+  options->BridgeRelay = 0;
+  options->BridgeDistribution = (char*)("https");
+
+  ret = options_validate(old_options, options, default_options, 0, &message);
+
+  tt_int_op(ret, OP_EQ, -1);
+  tt_str_op(message, OP_EQ, "You set BridgeDistribution, but you "
+            "didn't set BridgeRelay!");
+ done:
+  tor_free(message);
+  options->BridgeDistribution = NULL;
+}
+
+/* If the BridgeDistribution setting was valid, 0 should be returned. */
+static void
+test_config_check_bridge_distribution_setting_valid(void *arg)
+{
+  int ret = check_bridge_distribution_setting("https");
+
+  (void)arg;
+
+  tt_int_op(ret, OP_EQ, 0);
+ done:
+  return;
+}
+
+/* If the BridgeDistribution setting was invalid, -1 should be returned. */
+static void
+test_config_check_bridge_distribution_setting_invalid(void *arg)
+{
+  int ret = check_bridge_distribution_setting("hyphens-are-allowed");
+
+  (void)arg;
+
+  tt_int_op(ret, OP_EQ, 0);
+
+  ret = check_bridge_distribution_setting("asterisks*are*forbidden");
+
+  tt_int_op(ret, OP_EQ, -1);
+ done:
+  return;
+}
+
+/* If the BridgeDistribution setting was unrecognised, a warning should be
+ * logged and 0 should be returned. */
+static void
+test_config_check_bridge_distribution_setting_unrecognised(void *arg)
+{
+  int ret = check_bridge_distribution_setting("unicorn");
+
+  (void)arg;
+
+  tt_int_op(ret, OP_EQ, 0);
+ done:
+  return;
+}
+
+static void
+test_config_include_opened_file_list(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  smartlist_t *opened_files = smartlist_new();
+  char *torrcd = NULL;
+  char *subfolder = NULL;
+  char *path = NULL;
+  char *empty = NULL;
+  char *file = NULL;
+  char *dot = NULL;
+  char *dir = tor_strdup(get_fname("test_include_opened_file_list"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&torrcd, "%s"PATH_SEPARATOR"%s", dir, "torrc.d");
+
+#ifdef _WIN32
+  tt_int_op(mkdir(torrcd), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(torrcd, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&subfolder, "%s"PATH_SEPARATOR"%s", torrcd, "subfolder");
+
+#ifdef _WIN32
+  tt_int_op(mkdir(subfolder), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(subfolder, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", subfolder,
+               "01_file_in_subfolder");
+  tt_int_op(write_str_to_file(path, "Test 1\n", 0), OP_EQ, 0);
+
+  tor_asprintf(&empty, "%s"PATH_SEPARATOR"%s", torrcd, "empty");
+  tt_int_op(write_str_to_file(empty, "", 0), OP_EQ, 0);
+
+  tor_asprintf(&file, "%s"PATH_SEPARATOR"%s", torrcd, "file");
+  tt_int_op(write_str_to_file(file, "Test 2\n", 0), OP_EQ, 0);
+
+  tor_asprintf(&dot, "%s"PATH_SEPARATOR"%s", torrcd, ".dot");
+  tt_int_op(write_str_to_file(dot, "Test 3\n", 0), OP_EQ, 0);
+
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s\n",
+               torrcd);
+
+  int include_used;
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, &include_used,
+            opened_files), OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+  tt_int_op(smartlist_len(opened_files), OP_EQ, 4);
+  tt_int_op(smartlist_contains_string(opened_files, torrcd), OP_EQ, 1);
+  tt_int_op(smartlist_contains_string(opened_files, subfolder), OP_EQ, 1);
+  // files inside subfolders are not opended, only the subfolder is opened
+  tt_int_op(smartlist_contains_string(opened_files, empty), OP_EQ, 1);
+  tt_int_op(smartlist_contains_string(opened_files, file), OP_EQ, 1);
+  // dot files are not opened as we ignore them when we get their name from
+  // their parent folder
+
+ done:
+  SMARTLIST_FOREACH(opened_files, char *, f, tor_free(f));
+  smartlist_free(opened_files);
+  config_free_lines(result);
+  tor_free(torrcd);
+  tor_free(subfolder);
+  tor_free(path);
+  tor_free(empty);
+  tor_free(file);
+  tor_free(dot);
+  tor_free(dir);
+}
+
+static void
+test_config_compute_max_mem_in_queues(void *data)
+{
+#define GIGABYTE(x) (UINT64_C(x) << 30)
+#define MEGABYTE(x) (UINT64_C(x) << 20)
+  (void)data;
+  MOCK(get_total_system_memory, get_total_system_memory_mock);
+
+  /* We are unable to detect the amount of memory on the system. Tor will try
+   * to use some sensible default values for 64-bit and 32-bit systems. */
+  total_system_memory_return = -1;
+
+#if SIZEOF_VOID_P >= 8
+  /* We are on a 64-bit system. */
+  tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, GIGABYTE(8));
+#else
+  /* We are on a 32-bit system. */
+  tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, GIGABYTE(1));
+#endif
+
+  /* We are able to detect the amount of RAM on the system. */
+  total_system_memory_return = 0;
+
+  /* We are running on a system with one gigabyte of RAM. */
+  total_system_memory_output = GIGABYTE(1);
+
+  /* We have 0.75 * RAM available. */
+  tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ,
+            3 * (GIGABYTE(1) / 4));
+
+  /* We are running on a tiny machine with 256 MB of RAM. */
+  total_system_memory_output = MEGABYTE(256);
+
+  /* We will now enforce a minimum of 256 MB of RAM available for the
+   * MaxMemInQueues here, even though we should only have had 0.75 * 256 = 192
+   * MB available. */
+  tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, MEGABYTE(256));
+
+#if SIZEOF_SIZE_T > 4
+  /* We are running on a machine with 8 GB of RAM. */
+  total_system_memory_output = GIGABYTE(8);
+
+  /* We will have 0.4 * RAM available. */
+  tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ,
+            2 * (GIGABYTE(8) / 5));
+
+  /* We are running on a machine with 16 GB of RAM. */
+  total_system_memory_output = GIGABYTE(16);
+
+  /* We will have 0.4 * RAM available. */
+  tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ,
+            2 * (GIGABYTE(16) / 5));
+
+  /* We are running on a machine with 32 GB of RAM. */
+  total_system_memory_output = GIGABYTE(32);
+
+  /* We will at maximum get MAX_DEFAULT_MEMORY_QUEUE_SIZE here. */
+  tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ,
+            MAX_DEFAULT_MEMORY_QUEUE_SIZE);
+#endif
+
+ done:
+  UNMOCK(get_total_system_memory);
+
+#undef GIGABYTE
+#undef MEGABYTE
+}
+
+static void
+test_config_extended_fmt(void *arg)
+{
+  (void)arg;
+  config_line_t *lines = NULL, *lp;
+  const char string1[] =
+    "thing1 is here\n"
+    "+thing2 is over here\n"
+    "/thing3\n"
+    "/thing4 is back here\n";
+
+  /* Try with the "extended" flag disabled. */
+  int r = config_get_lines(string1, &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  lp = lines;
+  tt_ptr_op(lp, OP_NE, NULL);
+  tt_str_op(lp->key, OP_EQ, "thing1");
+  tt_str_op(lp->value, OP_EQ, "is here");
+  tt_int_op(lp->command, OP_EQ, CONFIG_LINE_NORMAL);
+  lp = lp->next;
+  tt_ptr_op(lp, OP_NE, NULL);
+  tt_str_op(lp->key, OP_EQ, "+thing2");
+  tt_str_op(lp->value, OP_EQ, "is over here");
+  tt_int_op(lp->command, OP_EQ, CONFIG_LINE_NORMAL);
+  lp = lp->next;
+  tt_ptr_op(lp, OP_NE, NULL);
+  tt_str_op(lp->key, OP_EQ, "/thing3");
+  tt_str_op(lp->value, OP_EQ, "");
+  tt_int_op(lp->command, OP_EQ, CONFIG_LINE_NORMAL);
+  lp = lp->next;
+  tt_ptr_op(lp, OP_NE, NULL);
+  tt_str_op(lp->key, OP_EQ, "/thing4");
+  tt_str_op(lp->value, OP_EQ, "is back here");
+  tt_int_op(lp->command, OP_EQ, CONFIG_LINE_NORMAL);
+  lp = lp->next;
+  config_free_lines(lines);
+
+  /* Try with the "extended" flag enabled. */
+  r = config_get_lines(string1, &lines, 1);
+  tt_int_op(r, OP_EQ, 0);
+  lp = lines;
+  tt_ptr_op(lp, OP_NE, NULL);
+  tt_str_op(lp->key, OP_EQ, "thing1");
+  tt_str_op(lp->value, OP_EQ, "is here");
+  tt_int_op(lp->command, OP_EQ, CONFIG_LINE_NORMAL);
+  lp = lp->next;
+  tt_ptr_op(lp, OP_NE, NULL);
+  tt_str_op(lp->key, OP_EQ, "thing2");
+  tt_str_op(lp->value, OP_EQ, "is over here");
+  tt_int_op(lp->command, OP_EQ, CONFIG_LINE_APPEND);
+  lp = lp->next;
+  tt_ptr_op(lp, OP_NE, NULL);
+  tt_str_op(lp->key, OP_EQ, "thing3");
+  tt_str_op(lp->value, OP_EQ, "");
+  tt_int_op(lp->command, OP_EQ, CONFIG_LINE_CLEAR);
+  lp = lp->next;
+  tt_ptr_op(lp, OP_NE, NULL);
+  tt_str_op(lp->key, OP_EQ, "thing4");
+  tt_str_op(lp->value, OP_EQ, "");
+  tt_int_op(lp->command, OP_EQ, CONFIG_LINE_CLEAR);
+  lp = lp->next;
+
+ done:
+  config_free_lines(lines);
+}
+
 #define CONFIG_TEST(name, flags)                          \
   { #name, test_config_ ## name, flags, NULL, NULL }
 
@@ -4897,6 +5885,7 @@ struct testcase_t config_tests[] = {
   CONFIG_TEST(adding_trusted_dir_server, TT_FORK),
   CONFIG_TEST(adding_fallback_dir_server, TT_FORK),
   CONFIG_TEST(parsing_trusted_dir_server, 0),
+  CONFIG_TEST(parsing_invalid_dir_address, 0),
   CONFIG_TEST(parsing_fallback_dir_server, 0),
   CONFIG_TEST(adding_default_trusted_dir_servers, TT_FORK),
   CONFIG_TEST(adding_dir_servers, TT_FORK),
@@ -4912,10 +5901,34 @@ struct testcase_t config_tests[] = {
   CONFIG_TEST(fix_my_family, 0),
   CONFIG_TEST(directory_fetch, 0),
   CONFIG_TEST(port_cfg_line_extract_addrport, 0),
-  CONFIG_TEST(parse_port_config__listenaddress, 0),
   CONFIG_TEST(parse_port_config__ports__no_ports_given, 0),
   CONFIG_TEST(parse_port_config__ports__server_options, 0),
   CONFIG_TEST(parse_port_config__ports__ports_given, 0),
+  CONFIG_TEST(parse_log_severity, 0),
+  CONFIG_TEST(include_limit, 0),
+  CONFIG_TEST(include_does_not_exist, 0),
+  CONFIG_TEST(include_error_in_included_file, 0),
+  CONFIG_TEST(include_empty_file_folder, 0),
+#ifndef _WIN32
+  CONFIG_TEST(include_no_permission, 0),
+#endif
+  CONFIG_TEST(include_recursion_before_after, 0),
+  CONFIG_TEST(include_recursion_after_only, 0),
+  CONFIG_TEST(include_folder_order, 0),
+  CONFIG_TEST(include_blank_file_last, 0),
+  CONFIG_TEST(include_path_syntax, 0),
+  CONFIG_TEST(include_not_processed, 0),
+  CONFIG_TEST(include_has_include, 0),
+  CONFIG_TEST(include_flag_both_without, TT_FORK),
+  CONFIG_TEST(include_flag_torrc_only, TT_FORK),
+  CONFIG_TEST(include_flag_defaults_only, TT_FORK),
+  CONFIG_TEST(dup_and_filter, 0),
+  CONFIG_TEST(check_bridge_distribution_setting_not_a_bridge, TT_FORK),
+  CONFIG_TEST(check_bridge_distribution_setting_valid, 0),
+  CONFIG_TEST(check_bridge_distribution_setting_invalid, 0),
+  CONFIG_TEST(check_bridge_distribution_setting_unrecognised, 0),
+  CONFIG_TEST(include_opened_file_list, 0),
+  CONFIG_TEST(compute_max_mem_in_queues, 0),
+  CONFIG_TEST(extended_fmt, 0),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_connection.c b/src/test/test_connection.c
index d394fc9852..ebe7c6d36f 100644
--- a/src/test/test_connection.c
+++ b/src/test/test_connection.c
@@ -1,24 +1,36 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
+/* Copyright (c) 2015-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 
 #define CONNECTION_PRIVATE
-#define MAIN_PRIVATE
-
-#include "or.h"
-#include "test.h"
-
-#include "connection.h"
-#include "main.h"
-#include "microdesc.h"
-#include "networkstatus.h"
-#include "rendcache.h"
-#include "directory.h"
-
-static void test_conn_lookup_addr_helper(const char *address,
-                                         int family,
-                                         tor_addr_t *addr);
+#define MAINLOOP_PRIVATE
+#define CONNECTION_OR_PRIVATE
+
+#include "core/or/or.h"
+#include "test/test.h"
+
+#include "core/mainloop/connection.h"
+#include "core/or/connection_edge.h"
+#include "feature/hs/hs_common.h"
+#include "core/mainloop/mainloop.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/rend/rendcache.h"
+#include "feature/dircommon/directory.h"
+#include "core/or/connection_or.h"
+#include "lib/net/resolve.h"
+
+#include "test/test_connection.h"
+#include "test/test_helpers.h"
+
+#include "feature/dircommon/dir_connection_st.h"
+#include "core/or/entry_connection_st.h"
+#include "feature/nodelist/node_st.h"
+#include "core/or/or_connection_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "core/or/socks_request_st.h"
 
 static void * test_conn_get_basic_setup(const struct testcase_t *tc);
 static int test_conn_get_basic_teardown(const struct testcase_t *tc,
@@ -61,48 +73,7 @@ static int test_conn_get_rsrc_teardown(const struct testcase_t *tc,
 #define TEST_CONN_UNATTACHED_STATE (AP_CONN_STATE_CIRCUIT_WAIT)
 #define TEST_CONN_ATTACHED_STATE   (AP_CONN_STATE_CONNECT_WAIT)
 
-#define TEST_CONN_FD_INIT 50
-static int mock_connection_connect_sockaddr_called = 0;
-static int fake_socket_number = TEST_CONN_FD_INIT;
-
-static int
-mock_connection_connect_sockaddr(connection_t *conn,
-                                 const struct sockaddr *sa,
-                                 socklen_t sa_len,
-                                 const struct sockaddr *bindaddr,
-                                 socklen_t bindaddr_len,
-                                 int *socket_error)
-{
-  (void)sa_len;
-  (void)bindaddr;
-  (void)bindaddr_len;
-
-  tor_assert(conn);
-  tor_assert(sa);
-  tor_assert(socket_error);
-
-  mock_connection_connect_sockaddr_called++;
-
-  conn->s = fake_socket_number++;
-  tt_assert(SOCKET_OK(conn->s));
-  /* We really should call tor_libevent_initialize() here. Because we don't,
-   * we are relying on other parts of the code not checking if the_event_base
-   * (and therefore event->ev_base) is NULL.  */
-  tt_assert(connection_add_connecting(conn) == 0);
-
- done:
-  /* Fake "connected" status */
-  return 1;
-}
-
-static int
-fake_close_socket(evutil_socket_t sock)
-{
-  (void)sock;
-  return 0;
-}
-
-static void
+void
 test_conn_lookup_addr_helper(const char *address, int family, tor_addr_t *addr)
 {
   int rv = 0;
@@ -111,7 +82,7 @@ test_conn_lookup_addr_helper(const char *address, int family, tor_addr_t *addr)
 
   rv = tor_addr_lookup(address, family, addr);
   /* XXXX - should we retry on transient failure? */
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
   tt_assert(tor_addr_is_loopback(addr));
   tt_assert(tor_addr_is_v4(addr));
 
@@ -121,51 +92,6 @@ test_conn_lookup_addr_helper(const char *address, int family, tor_addr_t *addr)
   tor_addr_make_null(addr, TEST_CONN_FAMILY);
 }
 
-static connection_t *
-test_conn_get_connection(uint8_t state, uint8_t type, uint8_t purpose)
-{
-  connection_t *conn = NULL;
-  tor_addr_t addr;
-  int socket_err = 0;
-  int in_progress = 0;
-
-  MOCK(connection_connect_sockaddr,
-       mock_connection_connect_sockaddr);
-  MOCK(tor_close_socket, fake_close_socket);
-
-  init_connection_lists();
-
-  conn = connection_new(type, TEST_CONN_FAMILY);
-  tt_assert(conn);
-
-  test_conn_lookup_addr_helper(TEST_CONN_ADDRESS, TEST_CONN_FAMILY, &addr);
-  tt_assert(!tor_addr_is_null(&addr));
-
-  tor_addr_copy_tight(&conn->addr, &addr);
-  conn->port = TEST_CONN_PORT;
-  mock_connection_connect_sockaddr_called = 0;
-  in_progress = connection_connect(conn, TEST_CONN_ADDRESS_PORT, &addr,
-                                   TEST_CONN_PORT, &socket_err);
-  tt_assert(mock_connection_connect_sockaddr_called == 1);
-  tt_assert(!socket_err);
-  tt_assert(in_progress == 0 || in_progress == 1);
-
-  /* fake some of the attributes so the connection looks OK */
-  conn->state = state;
-  conn->purpose = purpose;
-  assert_connection_ok(conn, time(NULL));
-
-  UNMOCK(connection_connect_sockaddr);
-  UNMOCK(tor_close_socket);
-  return conn;
-
-  /* On failure */
- done:
-  UNMOCK(connection_connect_sockaddr);
-  UNMOCK(tor_close_socket);
-  return NULL;
-}
-
 static void *
 test_conn_get_basic_setup(const struct testcase_t *tc)
 {
@@ -264,14 +190,10 @@ test_conn_get_rend_setup(const struct testcase_t *tc)
 
   rend_cache_init();
 
-  /* TODO: use directory_initiate_command_rend() to do this - maybe? */
-  conn->rend_data = tor_malloc_zero(sizeof(rend_data_t));
+  /* TODO: use directory_initiate_request() to do this - maybe? */
   tor_assert(strlen(TEST_CONN_REND_ADDR) == REND_SERVICE_ID_LEN_BASE32);
-  memcpy(conn->rend_data->onion_address,
-         TEST_CONN_REND_ADDR,
-         REND_SERVICE_ID_LEN_BASE32+1);
-  conn->rend_data->hsdirs_fp = smartlist_new();
-
+  conn->rend_data = rend_data_client_create(TEST_CONN_REND_ADDR, NULL, NULL,
+                                            REND_NO_AUTH);
   assert_connection_ok(&conn->base_, time(NULL));
   return conn;
 
@@ -382,7 +304,7 @@ test_conn_download_status_teardown(const struct testcase_t *tc, void *arg)
 
       /* connection_free_() cleans up requested_resource */
       rv = test_conn_get_rsrc_teardown(tc, conn);
-      tt_assert(rv == 1);
+      tt_int_op(rv, OP_EQ, 1);
     }
   } SMARTLIST_FOREACH_END(conn);
 
@@ -456,12 +378,13 @@ test_conn_get_basic(void *arg)
    * its attributes, but get NULL when we supply a different value. */
 
   tt_assert(connection_get_by_global_id(conn->global_identifier) == conn);
-  tt_assert(connection_get_by_global_id(!conn->global_identifier) == NULL);
+  tt_ptr_op(connection_get_by_global_id(!conn->global_identifier), OP_EQ,
+            NULL);
 
   tt_assert(connection_get_by_type(conn->type) == conn);
   tt_assert(connection_get_by_type(TEST_CONN_TYPE) == conn);
-  tt_assert(connection_get_by_type(!conn->type) == NULL);
-  tt_assert(connection_get_by_type(!TEST_CONN_TYPE) == NULL);
+  tt_ptr_op(connection_get_by_type(!conn->type), OP_EQ, NULL);
+  tt_ptr_op(connection_get_by_type(!TEST_CONN_TYPE), OP_EQ, NULL);
 
   tt_assert(connection_get_by_type_state(conn->type, conn->state)
             == conn);
@@ -551,7 +474,8 @@ test_conn_get_rend(void *arg)
   tt_assert(connection_get_by_type_state_rendquery(
                                             conn->base_.type,
                                             conn->base_.state,
-                                            conn->rend_data->onion_address)
+                                            rend_data_get_address(
+                                                      conn->rend_data))
             == TO_CONN(conn));
   tt_assert(connection_get_by_type_state_rendquery(
                                             TEST_CONN_TYPE,
@@ -574,7 +498,7 @@ test_conn_get_rend(void *arg)
 #define sl_is_conn_assert(sl_input, conn) \
   do {                                               \
     the_sl = (sl_input);                             \
-    tt_assert(smartlist_len((the_sl)) == 1);         \
+    tt_int_op(smartlist_len((the_sl)), OP_EQ, 1);         \
     tt_assert(smartlist_get((the_sl), 0) == (conn)); \
     smartlist_free(the_sl); the_sl = NULL;           \
   } while (0)
@@ -582,7 +506,7 @@ test_conn_get_rend(void *arg)
 #define sl_no_conn_assert(sl_input)          \
   do {                                       \
     the_sl = (sl_input);                     \
-    tt_assert(smartlist_len((the_sl)) == 0); \
+    tt_int_op(smartlist_len((the_sl)), OP_EQ, 0); \
     smartlist_free(the_sl); the_sl = NULL;   \
   } while (0)
 
@@ -628,43 +552,32 @@ test_conn_get_rsrc(void *arg)
                                                     TEST_CONN_RSRC_2,
                                                     !TEST_CONN_STATE));
 
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                    conn->base_.purpose,
-                                                    conn->requested_resource)
-            == 1);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                    TEST_CONN_RSRC_PURPOSE,
-                                                    TEST_CONN_RSRC)
-            == 1);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                    !conn->base_.purpose,
-                                                    "")
-            == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                    !TEST_CONN_RSRC_PURPOSE,
-                                                    TEST_CONN_RSRC_2)
-            == 0);
-
-  tt_assert(connection_dir_count_by_purpose_resource_and_state(
-                                                    conn->base_.purpose,
-                                                    conn->requested_resource,
-                                                    conn->base_.state)
-            == 1);
-  tt_assert(connection_dir_count_by_purpose_resource_and_state(
-                                                    TEST_CONN_RSRC_PURPOSE,
-                                                    TEST_CONN_RSRC,
-                                                    TEST_CONN_STATE)
-            == 1);
-  tt_assert(connection_dir_count_by_purpose_resource_and_state(
-                                                    !conn->base_.purpose,
-                                                    "",
-                                                    !conn->base_.state)
-            == 0);
-  tt_assert(connection_dir_count_by_purpose_resource_and_state(
-                                                    !TEST_CONN_RSRC_PURPOSE,
-                                                    TEST_CONN_RSRC_2,
-                                                    !TEST_CONN_STATE)
-            == 0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                 conn->base_.purpose, conn->requested_resource),
+            OP_EQ, 1);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                 TEST_CONN_RSRC_PURPOSE, TEST_CONN_RSRC),
+            OP_EQ, 1);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                 !conn->base_.purpose, ""),
+            OP_EQ, 0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                 !TEST_CONN_RSRC_PURPOSE, TEST_CONN_RSRC_2),
+            OP_EQ, 0);
+
+  tt_int_op(connection_dir_count_by_purpose_resource_and_state(
+                 conn->base_.purpose, conn->requested_resource,
+                 conn->base_.state),
+            OP_EQ, 1);
+  tt_int_op(connection_dir_count_by_purpose_resource_and_state(
+                 TEST_CONN_RSRC_PURPOSE, TEST_CONN_RSRC, TEST_CONN_STATE),
+            OP_EQ, 1);
+  tt_int_op(connection_dir_count_by_purpose_resource_and_state(
+                 !conn->base_.purpose, "", !conn->base_.state),
+            OP_EQ, 0);
+  tt_int_op(connection_dir_count_by_purpose_resource_and_state(
+                 !TEST_CONN_RSRC_PURPOSE, TEST_CONN_RSRC_2, !TEST_CONN_STATE),
+            OP_EQ, 0);
 
  done:
   smartlist_free(the_sl);
@@ -690,117 +603,127 @@ test_conn_download_status(void *arg)
   const char *other_res = networkstatus_get_flavor_name(other_flavor);
 
   /* no connections */
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 0);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
 
   /* one connection, not downloading */
   conn = test_conn_download_status_add_a_connection(res);
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 0);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 1);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 1);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
 
   /* one connection, downloading but not linked (not possible on a client,
    * but possible on a relay) */
   conn->base_.state = TEST_CONN_DL_STATE;
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 0);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 1);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 1);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
 
   /* one connection, downloading and linked, but not yet attached */
   ap_conn = test_conn_get_linked_connection(TO_CONN(conn),
                                             TEST_CONN_UNATTACHED_STATE);
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 0);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 1);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 1);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
 
     /* one connection, downloading and linked and attached */
   ap_conn->state = TEST_CONN_ATTACHED_STATE;
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 1);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 1);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 1);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
 
   /* one connection, linked and attached but not downloading */
   conn->base_.state = TEST_CONN_STATE;
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 0);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 1);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 1);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
 
   /* two connections, both not downloading */
   conn2 = test_conn_download_status_add_a_connection(res);
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 0);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 2);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 2);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
 
   /* two connections, one downloading */
   conn->base_.state = TEST_CONN_DL_STATE;
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 1);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 2);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 2);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
   conn->base_.state = TEST_CONN_STATE;
 
   /* more connections, all not downloading */
   /* ignore the return value, it's free'd using the connection list */
   (void)test_conn_download_status_add_a_connection(res);
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 0);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 3);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 3);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
 
   /* more connections, one downloading */
   conn->base_.state = TEST_CONN_DL_STATE;
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 1);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 3);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 3);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
 
   /* more connections, two downloading (should never happen, but needs
    * to be tested for completeness) */
@@ -808,38 +731,41 @@ test_conn_download_status(void *arg)
   /* ignore the return value, it's free'd using the connection list */
   (void)test_conn_get_linked_connection(TO_CONN(conn2),
                                         TEST_CONN_ATTACHED_STATE);
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 1);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 3);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 3);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
   conn->base_.state = TEST_CONN_STATE;
 
   /* more connections, a different one downloading */
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 1);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 3);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 3);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
 
   /* a connection for the other flavor (could happen if a client is set to
    * cache directory documents), one preferred flavor downloading
    */
   conn4 = test_conn_download_status_add_a_connection(other_res);
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 1);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 3);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        other_res) == 1);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                   TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 3);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                   TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 1);
 
   /* a connection for the other flavor (could happen if a client is set to
    * cache directory documents), both flavors downloading
@@ -848,23 +774,117 @@ test_conn_download_status(void *arg)
   /* ignore the return value, it's free'd using the connection list */
   (void)test_conn_get_linked_connection(TO_CONN(conn4),
                                         TEST_CONN_ATTACHED_STATE);
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 1);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 1);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 3);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        other_res) == 1);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            1);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                   TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 3);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                   TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 1);
 
  done:
   /* the teardown function removes all the connections in the global list*/;
 }
 
+static node_t test_node;
+
+static node_t *
+mock_node_get_mutable_by_id(const char *digest)
+{
+  (void) digest;
+  static routerinfo_t node_ri;
+  memset(&node_ri, 0, sizeof(node_ri));
+
+  test_node.ri = &node_ri;
+  memset(test_node.identity, 'c', sizeof(test_node.identity));
+
+  tor_addr_t ipv4_addr;
+  tor_addr_parse(&ipv4_addr, "18.0.0.1");
+  node_ri.addr = tor_addr_to_ipv4h(&ipv4_addr);
+  node_ri.or_port = 1;
+
+  return &test_node;
+}
+
+static const node_t *
+mock_node_get_by_id(const char *digest)
+{
+  (void) digest;
+  memset(test_node.identity, 'c', sizeof(test_node.identity));
+  return &test_node;
+}
+
+/* Test whether we correctly track failed connections between relays. */
+static void
+test_failed_orconn_tracker(void *arg)
+{
+  (void) arg;
+
+  int can_connect;
+  time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */
+  (void) now;
+
+  update_approx_time(now);
+
+  /* Prepare the OR connection that will be used in this test */
+  or_connection_t or_conn;
+  tt_int_op(AF_INET,OP_EQ, tor_addr_parse(&or_conn.real_addr, "18.0.0.1"));
+  tt_int_op(AF_INET,OP_EQ, tor_addr_parse(&or_conn.base_.addr, "18.0.0.1"));
+  or_conn.base_.port = 1;
+  memset(or_conn.identity_digest, 'c', sizeof(or_conn.identity_digest));
+
+  /* Check whether we can connect with an empty failure cache:
+   * this should succeed */
+  can_connect = should_connect_to_relay(&or_conn);
+  tt_int_op(can_connect, OP_EQ, 1);
+
+  /* Now add the destination to the failure cache */
+  note_or_connect_failed(&or_conn);
+
+  /* Check again: now it shouldn't connect */
+  can_connect = should_connect_to_relay(&or_conn);
+  tt_int_op(can_connect, OP_EQ, 0);
+
+  /* Move time forward and check again: the cache should have been cleared and
+   * now it should connect */
+  now += 3600;
+  update_approx_time(now);
+  can_connect = should_connect_to_relay(&or_conn);
+  tt_int_op(can_connect, OP_EQ, 1);
+
+  /* Now mock the node_get_*by_id() functions to start using the node subsystem
+   * optimization. */
+  MOCK(node_get_by_id, mock_node_get_by_id);
+  MOCK(node_get_mutable_by_id, mock_node_get_mutable_by_id);
+
+  /* Since we just started using the node subsystem it will allow connections
+   * now */
+  can_connect = should_connect_to_relay(&or_conn);
+  tt_int_op(can_connect, OP_EQ, 1);
+
+  /* Mark it as failed */
+  note_or_connect_failed(&or_conn);
+
+  /* Check that it shouldn't connect now */
+  can_connect = should_connect_to_relay(&or_conn);
+  tt_int_op(can_connect, OP_EQ, 0);
+
+  /* Move time forward and check again: now it should connect  */
+  now += 3600;
+  update_approx_time(now);
+  can_connect = should_connect_to_relay(&or_conn);
+  tt_int_op(can_connect, OP_EQ, 1);
+
+ done:
+  ;
+}
+
 #define CONNECTION_TESTCASE(name, fork, setup)                           \
   { #name, test_conn_##name, fork, &setup, NULL }
 
-/* where arg is an expression (constant, varaible, compound expression) */
+/* where arg is an expression (constant, variable, compound expression) */
 #define CONNECTION_TESTCASE_ARG(name, fork, setup, arg)                  \
   { #name "_" #arg, test_conn_##name, fork, &setup, (void *)arg }
 
@@ -877,6 +897,6 @@ struct testcase_t connection_tests[] = {
   CONNECTION_TESTCASE_ARG(download_status,  TT_FORK,
                           test_conn_download_status_st, FLAV_NS),
 //CONNECTION_TESTCASE(func_suffix, TT_FORK, setup_func_pair),
+  { "failed_orconn_tracker", test_failed_orconn_tracker, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_connection.h b/src/test/test_connection.h
new file mode 100644
index 0000000000..47a5599e5f
--- /dev/null
+++ b/src/test/test_connection.h
@@ -0,0 +1,13 @@
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/** Some constants used by test_connection and helpers */
+#define TEST_CONN_FAMILY        (AF_INET)
+#define TEST_CONN_ADDRESS       "127.0.0.1"
+#define TEST_CONN_PORT          (12345)
+#define TEST_CONN_ADDRESS_PORT  "127.0.0.1:12345"
+#define TEST_CONN_FD_INIT 50
+
+void test_conn_lookup_addr_helper(const char *address,
+                                  int family, tor_addr_t *addr);
+
diff --git a/src/test/test_conscache.c b/src/test/test_conscache.c
new file mode 100644
index 0000000000..095ff09350
--- /dev/null
+++ b/src/test/test_conscache.c
@@ -0,0 +1,340 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "feature/dircache/conscache.h"
+#include "lib/encoding/confline.h"
+#include "test/test.h"
+
+#ifdef HAVE_UTIME_H
+#include <utime.h>
+#endif
+
+static void
+test_conscache_open_failure(void *arg)
+{
+  (void) arg;
+  /* Try opening a directory that doesn't exist and which we shouldn't be
+   * able to create. */
+  consensus_cache_t *cache = consensus_cache_open("a/b/c/d/e/f/g", 128);
+  tt_ptr_op(cache, OP_EQ, NULL);
+
+ done:
+  ;
+}
+
+static void
+test_conscache_simple_usage(void *arg)
+{
+  (void)arg;
+  consensus_cache_entry_t *ent = NULL, *ent2 = NULL;
+
+  /* Make a temporary datadir for these tests */
+  char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cache"));
+  tor_free(get_options_mutable()->CacheDirectory);
+  get_options_mutable()->CacheDirectory = tor_strdup(ddir_fname);
+  check_private_dir(ddir_fname, CPD_CREATE, NULL);
+  consensus_cache_t *cache = consensus_cache_open("cons", 128);
+
+  tt_assert(cache);
+
+  /* Create object; make sure it exists. */
+  config_line_t *labels = NULL;
+  config_line_append(&labels, "Hello", "world");
+  config_line_append(&labels, "Adios", "planetas");
+  ent = consensus_cache_add(cache,
+                            labels, (const uint8_t *)"A\0B\0C", 5);
+  config_free_lines(labels);
+  labels = NULL;
+  tt_assert(ent);
+
+  /* Make a second object */
+  config_line_append(&labels, "Hello", "mundo");
+  config_line_append(&labels, "Adios", "planets");
+  ent2 = consensus_cache_add(cache,
+                             labels, (const uint8_t *)"xyzzy", 5);
+  config_free_lines(labels);
+  labels = NULL;
+  tt_assert(ent2);
+  tt_assert(! consensus_cache_entry_is_mapped(ent2));
+  consensus_cache_entry_decref(ent2);
+  ent2 = NULL;
+
+  /* Check get_value */
+  tt_ptr_op(NULL, OP_EQ, consensus_cache_entry_get_value(ent, "hebbo"));
+  tt_str_op("world", OP_EQ, consensus_cache_entry_get_value(ent, "Hello"));
+
+  /* Check find_first */
+  ent2 = consensus_cache_find_first(cache, "Hello", "world!");
+  tt_ptr_op(ent2, OP_EQ, NULL);
+  ent2 = consensus_cache_find_first(cache, "Hello", "world");
+  tt_ptr_op(ent2, OP_EQ, ent);
+  ent2 = consensus_cache_find_first(cache, "Hello", "mundo");
+  tt_ptr_op(ent2, OP_NE, ent);
+
+  tt_assert(! consensus_cache_entry_is_mapped(ent));
+
+  /* Check get_body */
+  const uint8_t *bp = NULL;
+  size_t sz = 0;
+  int r = consensus_cache_entry_get_body(ent, &bp, &sz);
+  tt_int_op(r, OP_EQ, 0);
+  tt_u64_op(sz, OP_EQ, 5);
+  tt_mem_op(bp, OP_EQ, "A\0B\0C", 5);
+  tt_assert(consensus_cache_entry_is_mapped(ent));
+
+  /* Free and re-create the cache, to rescan the directory. */
+  consensus_cache_free(cache);
+  consensus_cache_entry_decref(ent);
+  cache = consensus_cache_open("cons", 128);
+
+  /* Make sure the entry is still there */
+  ent = consensus_cache_find_first(cache, "Hello", "mundo");
+  tt_assert(ent);
+  ent2 = consensus_cache_find_first(cache, "Adios", "planets");
+  tt_ptr_op(ent, OP_EQ, ent2);
+  consensus_cache_entry_incref(ent);
+  tt_assert(! consensus_cache_entry_is_mapped(ent));
+  r = consensus_cache_entry_get_body(ent, &bp, &sz);
+  tt_int_op(r, OP_EQ, 0);
+  tt_u64_op(sz, OP_EQ, 5);
+  tt_mem_op(bp, OP_EQ, "xyzzy", 5);
+  tt_assert(consensus_cache_entry_is_mapped(ent));
+
+  /* There should be two entries total. */
+  smartlist_t *entries = smartlist_new();
+  consensus_cache_find_all(entries, cache, NULL, NULL);
+  int n = smartlist_len(entries);
+  smartlist_free(entries);
+  tt_int_op(n, OP_EQ, 2);
+
+ done:
+  consensus_cache_entry_decref(ent);
+  tor_free(ddir_fname);
+  consensus_cache_free(cache);
+}
+
+static void
+test_conscache_cleanup(void *arg)
+{
+  (void)arg;
+  const int N = 20;
+  consensus_cache_entry_t **ents =
+    tor_calloc(N, sizeof(consensus_cache_entry_t*));
+
+  /* Make a temporary datadir for these tests */
+  char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cache"));
+  tor_free(get_options_mutable()->CacheDirectory);
+  get_options_mutable()->CacheDirectory = tor_strdup(ddir_fname);
+  check_private_dir(ddir_fname, CPD_CREATE, NULL);
+  consensus_cache_t *cache = consensus_cache_open("cons", 128);
+
+  tt_assert(cache);
+
+  /* Create a bunch of entries. */
+  int i;
+  for (i = 0; i < N; ++i) {
+    config_line_t *labels = NULL;
+    char num[8];
+    tor_snprintf(num, sizeof(num), "%d", i);
+    config_line_append(&labels, "test-id", "cleanup");
+    config_line_append(&labels, "index", num);
+    size_t bodylen = i * 3;
+    uint8_t *body = tor_malloc(bodylen);
+    memset(body, i, bodylen);
+    ents[i] = consensus_cache_add(cache, labels, body, bodylen);
+    tor_free(body);
+    config_free_lines(labels);
+    tt_assert(ents[i]);
+    /* We're still holding a reference to each entry at this point. */
+  }
+
+  /* Page all of the entries into RAM */
+  for (i = 0; i < N; ++i) {
+    const uint8_t *bp;
+    size_t sz;
+    tt_assert(! consensus_cache_entry_is_mapped(ents[i]));
+    consensus_cache_entry_get_body(ents[i], &bp, &sz);
+    tt_assert(consensus_cache_entry_is_mapped(ents[i]));
+  }
+
+  /* Mark some of the entries as deletable. */
+  for (i = 7; i < N; i += 7) {
+    consensus_cache_entry_mark_for_removal(ents[i]);
+    tt_assert(consensus_cache_entry_is_mapped(ents[i]));
+  }
+
+  /* Mark some of the entries as aggressively unpaged. */
+  for (i = 3; i < N; i += 3) {
+    consensus_cache_entry_mark_for_aggressive_release(ents[i]);
+    tt_assert(consensus_cache_entry_is_mapped(ents[i]));
+  }
+
+  /* Incref some of the entries again */
+  for (i = 0; i < N; i += 2) {
+    consensus_cache_entry_incref(ents[i]);
+  }
+
+  /* Now we're going to decref everything. We do so at a specific time.  I'm
+   * picking the moment when I was writing this test, at 2017-04-05 12:16:48
+   * UTC. */
+  const time_t example_time = 1491394608;
+  update_approx_time(example_time);
+  for (i = 0; i < N; ++i) {
+    consensus_cache_entry_decref(ents[i]);
+    if (i % 2) {
+      ents[i] = NULL; /* We're no longer holding any reference here. */
+    }
+  }
+
+  /* At this point, the aggressively-released items with refcount 1 should
+   * be unmapped. Nothing should be deleted. */
+  consensus_cache_entry_t *e_tmp;
+  e_tmp = consensus_cache_find_first(cache, "index", "3");
+  tt_assert(e_tmp);
+  tt_assert(! consensus_cache_entry_is_mapped(e_tmp));
+  e_tmp = consensus_cache_find_first(cache, "index", "5");
+  tt_assert(e_tmp);
+  tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+  e_tmp = consensus_cache_find_first(cache, "index", "6");
+  tt_assert(e_tmp);
+  tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+  e_tmp = consensus_cache_find_first(cache, "index", "7");
+  tt_ptr_op(e_tmp, OP_EQ, NULL); // not found because pending deletion.
+
+  /* Delete the pending-deletion items. */
+  consensus_cache_delete_pending(cache, 0);
+  {
+    smartlist_t *entries = smartlist_new();
+    consensus_cache_find_all(entries, cache, NULL, NULL);
+    int n = smartlist_len(entries);
+    smartlist_free(entries);
+    tt_int_op(n, OP_EQ, 20 - 2); /* 1 entry was deleted; 1 is not-found. */
+  }
+  e_tmp = consensus_cache_find_first(cache, "index", "7"); // refcnt == 1...
+  tt_ptr_op(e_tmp, OP_EQ, NULL); // so deleted.
+  e_tmp = consensus_cache_find_first(cache, "index", "14"); // refcnt == 2
+  tt_ptr_op(e_tmp, OP_EQ, NULL); // not deleted; but not found.
+
+  /* Now do lazy unmapping. */
+  // should do nothing.
+  consensus_cache_unmap_lazy(cache, example_time - 10);
+  e_tmp = consensus_cache_find_first(cache, "index", "11");
+  tt_assert(e_tmp);
+  tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+  // should actually unmap
+  consensus_cache_unmap_lazy(cache, example_time + 10);
+  e_tmp = consensus_cache_find_first(cache, "index", "11");
+  tt_assert(e_tmp);
+  tt_assert(! consensus_cache_entry_is_mapped(e_tmp));
+  // This one will still be mapped, since it has a reference.
+  e_tmp = consensus_cache_find_first(cache, "index", "16");
+  tt_assert(e_tmp);
+  tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+
+  for (i = 0; i < N; ++i) {
+    consensus_cache_entry_decref(ents[i]);
+    ents[i] = NULL;
+  }
+
+  /* Free and re-create the cache, to rescan the directory. Make sure the
+   * deleted thing is still deleted, along with the other deleted thing. */
+  consensus_cache_free(cache);
+  cache = consensus_cache_open("cons", 128);
+  {
+    smartlist_t *entries = smartlist_new();
+    consensus_cache_find_all(entries, cache, NULL, NULL);
+    int n = smartlist_len(entries);
+    smartlist_free(entries);
+    tt_int_op(n, OP_EQ, 18);
+  }
+
+ done:
+  for (i = 0; i < N; ++i) {
+    consensus_cache_entry_decref(ents[i]);
+  }
+  tor_free(ents);
+  tor_free(ddir_fname);
+  consensus_cache_free(cache);
+}
+
+static void
+test_conscache_filter(void *arg)
+{
+  (void)arg;
+  const int N = 30;
+  smartlist_t *lst = NULL;
+
+  /* Make a temporary datadir for these tests */
+  char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cache"));
+  tor_free(get_options_mutable()->CacheDirectory);
+  get_options_mutable()->CacheDirectory = tor_strdup(ddir_fname);
+  check_private_dir(ddir_fname, CPD_CREATE, NULL);
+  consensus_cache_t *cache = consensus_cache_open("cons", 128);
+
+  tt_assert(cache);
+
+  /* Create a bunch of entries with different labels */
+  int i;
+  for (i = 0; i < N; ++i) {
+    config_line_t *labels = NULL;
+    char num[8];
+    tor_snprintf(num, sizeof(num), "%d", i);
+    config_line_append(&labels, "test-id", "filter");
+    config_line_append(&labels, "index", num);
+    tor_snprintf(num, sizeof(num), "%d", i % 3);
+    config_line_append(&labels, "mod3", num);
+    tor_snprintf(num, sizeof(num), "%d", i % 5);
+    config_line_append(&labels, "mod5", num);
+
+    size_t bodylen = i * 3;
+    uint8_t *body = tor_malloc(bodylen);
+    memset(body, i, bodylen);
+    consensus_cache_entry_t *ent =
+      consensus_cache_add(cache, labels, body, bodylen);
+    tor_free(body);
+    config_free_lines(labels);
+    tt_assert(ent);
+    consensus_cache_entry_decref(ent);
+  }
+
+  lst = smartlist_new();
+  /* Find nothing. */
+  consensus_cache_find_all(lst, cache, "mod5", "5");
+  tt_int_op(smartlist_len(lst), OP_EQ, 0);
+  /* Find everything. */
+  consensus_cache_find_all(lst, cache, "test-id", "filter");
+  tt_int_op(smartlist_len(lst), OP_EQ, N);
+
+  /* Now filter to find the entries that have i%3 == 1 */
+  consensus_cache_filter_list(lst, "mod3", "1");
+  tt_int_op(smartlist_len(lst), OP_EQ, 10);
+  /* Now filter to find the entries that also have i%5 == 3 */
+  consensus_cache_filter_list(lst, "mod5", "3");
+  tt_int_op(smartlist_len(lst), OP_EQ, 2);
+  /* So now we have those entries for which i%15 == 13. */
+
+  consensus_cache_entry_t *ent1 = smartlist_get(lst, 0);
+  consensus_cache_entry_t *ent2 = smartlist_get(lst, 1);
+  const char *idx1 = consensus_cache_entry_get_value(ent1, "index");
+  const char *idx2 = consensus_cache_entry_get_value(ent2, "index");
+  tt_assert( (!strcmp(idx1, "28") && !strcmp(idx2, "13")) ||
+             (!strcmp(idx1, "13") && !strcmp(idx2, "28")) );
+
+ done:
+  tor_free(ddir_fname);
+  consensus_cache_free(cache);
+  smartlist_free(lst);
+}
+
+#define ENT(name)                                               \
+  { #name, test_conscache_ ## name, TT_FORK, NULL, NULL }
+
+struct testcase_t conscache_tests[] = {
+  ENT(open_failure),
+  ENT(simple_usage),
+  ENT(cleanup),
+  ENT(filter),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_consdiff.c b/src/test/test_consdiff.c
new file mode 100644
index 0000000000..682ba5b970
--- /dev/null
+++ b/src/test/test_consdiff.c
@@ -0,0 +1,1185 @@
+/* Copyright (c) 2014, Daniel Martí
+ * Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CONSDIFF_PRIVATE
+
+#include "core/or/or.h"
+#include "test/test.h"
+
+#include "feature/dircommon/consdiff.h"
+#include "lib/memarea/memarea.h"
+#include "test/log_test_helpers.h"
+
+#define tt_str_eq_line(a,b) \
+  tt_assert(line_str_eq((b),(a)))
+
+static void
+test_consdiff_smartlist_slice(void *arg)
+{
+  smartlist_t *sl = smartlist_new();
+  smartlist_slice_t *sls;
+  int items[6] = {0,0,0,0,0,0};
+
+  /* Create a regular smartlist. */
+  (void)arg;
+  smartlist_add(sl, &items[1]);
+  smartlist_add(sl, &items[2]);
+  smartlist_add(sl, &items[3]);
+  smartlist_add(sl, &items[4]);
+  smartlist_add(sl, &items[5]);
+
+  /* See if the slice was done correctly. */
+  sls = smartlist_slice(sl, 2, 5);
+  tt_ptr_op(sl, OP_EQ, sls->list);
+  tt_ptr_op(&items[3], OP_EQ, smartlist_get(sls->list, sls->offset));
+  tt_ptr_op(&items[5], OP_EQ,
+      smartlist_get(sls->list, sls->offset + (sls->len-1)));
+  tor_free(sls);
+
+  /* See that using -1 as the end does get to the last element. */
+  sls = smartlist_slice(sl, 2, -1);
+  tt_ptr_op(sl, OP_EQ, sls->list);
+  tt_ptr_op(&items[3], OP_EQ, smartlist_get(sls->list, sls->offset));
+  tt_ptr_op(&items[5], OP_EQ,
+      smartlist_get(sls->list, sls->offset + (sls->len-1)));
+
+ done:
+  tor_free(sls);
+  smartlist_free(sl);
+}
+
+static void
+test_consdiff_smartlist_slice_string_pos(void *arg)
+{
+  smartlist_t *sl = smartlist_new();
+  smartlist_slice_t *sls;
+  memarea_t *area = memarea_new();
+
+  /* Create a regular smartlist. */
+  (void)arg;
+  consensus_split_lines(sl, "a\nd\nc\na\nb\n", area);
+
+  /* See that smartlist_slice_string_pos respects the bounds of the slice. */
+  sls = smartlist_slice(sl, 2, 5);
+  cdline_t a_line = { "a", 1 };
+  tt_int_op(3, OP_EQ, smartlist_slice_string_pos(sls, &a_line));
+  cdline_t d_line = { "d", 1 };
+  tt_int_op(-1, OP_EQ, smartlist_slice_string_pos(sls, &d_line));
+
+ done:
+  tor_free(sls);
+  smartlist_free(sl);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_lcs_lengths(void *arg)
+{
+  smartlist_t *sl1 = smartlist_new();
+  smartlist_t *sl2 = smartlist_new();
+  smartlist_slice_t *sls1, *sls2;
+  int *lengths1, *lengths2;
+  memarea_t *area = memarea_new();
+
+  /* Expected lcs lengths in regular and reverse order. */
+  int e_lengths1[] = { 0, 1, 2, 3, 3, 4 };
+  int e_lengths2[] = { 0, 1, 1, 2, 3, 4 };
+
+  (void)arg;
+  consensus_split_lines(sl1, "a\nb\nc\nd\ne\n", area);
+  consensus_split_lines(sl2, "a\nc\nd\ni\ne\n", area);
+
+  sls1 = smartlist_slice(sl1, 0, -1);
+  sls2 = smartlist_slice(sl2, 0, -1);
+
+  lengths1 = lcs_lengths(sls1, sls2, 1);
+  lengths2 = lcs_lengths(sls1, sls2, -1);
+  tt_mem_op(e_lengths1, OP_EQ, lengths1, sizeof(int) * 6);
+  tt_mem_op(e_lengths2, OP_EQ, lengths2, sizeof(int) * 6);
+
+ done:
+  tor_free(lengths1);
+  tor_free(lengths2);
+  tor_free(sls1);
+  tor_free(sls2);
+  smartlist_free(sl1);
+  smartlist_free(sl2);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_trim_slices(void *arg)
+{
+  smartlist_t *sl1 = smartlist_new();
+  smartlist_t *sl2 = smartlist_new();
+  smartlist_t *sl3 = smartlist_new();
+  smartlist_t *sl4 = smartlist_new();
+  smartlist_slice_t *sls1, *sls2, *sls3, *sls4;
+  memarea_t *area = memarea_new();
+
+  (void)arg;
+  consensus_split_lines(sl1, "a\nb\nb\nb\nd\n", area);
+  consensus_split_lines(sl2, "a\nc\nc\nc\nd\n", area);
+  consensus_split_lines(sl3, "a\nb\nb\nb\na\n", area);
+  consensus_split_lines(sl4, "c\nb\nb\nb\nc\n", area);
+  sls1 = smartlist_slice(sl1, 0, -1);
+  sls2 = smartlist_slice(sl2, 0, -1);
+  sls3 = smartlist_slice(sl3, 0, -1);
+  sls4 = smartlist_slice(sl4, 0, -1);
+
+  /* They should be trimmed by one line at each end. */
+  tt_int_op(5, OP_EQ, sls1->len);
+  tt_int_op(5, OP_EQ, sls2->len);
+  trim_slices(sls1, sls2);
+  tt_int_op(3, OP_EQ, sls1->len);
+  tt_int_op(3, OP_EQ, sls2->len);
+
+  /* They should not be trimmed at all. */
+  tt_int_op(5, OP_EQ, sls3->len);
+  tt_int_op(5, OP_EQ, sls4->len);
+  trim_slices(sls3, sls4);
+  tt_int_op(5, OP_EQ, sls3->len);
+  tt_int_op(5, OP_EQ, sls4->len);
+
+ done:
+  tor_free(sls1);
+  tor_free(sls2);
+  tor_free(sls3);
+  tor_free(sls4);
+  smartlist_free(sl1);
+  smartlist_free(sl2);
+  smartlist_free(sl3);
+  smartlist_free(sl4);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_set_changed(void *arg)
+{
+  smartlist_t *sl1 = smartlist_new();
+  smartlist_t *sl2 = smartlist_new();
+  bitarray_t *changed1 = bitarray_init_zero(4);
+  bitarray_t *changed2 = bitarray_init_zero(4);
+  smartlist_slice_t *sls1, *sls2;
+  memarea_t *area = memarea_new();
+
+  (void)arg;
+  consensus_split_lines(sl1, "a\nb\na\na\n", area);
+  consensus_split_lines(sl2, "a\na\na\na\n", area);
+
+  /* Length of sls1 is 0. */
+  sls1 = smartlist_slice(sl1, 0, 0);
+  sls2 = smartlist_slice(sl2, 1, 3);
+  set_changed(changed1, changed2, sls1, sls2);
+
+  /* The former is not changed, the latter changes all of its elements. */
+  tt_assert(!bitarray_is_set(changed1, 0));
+  tt_assert(!bitarray_is_set(changed1, 1));
+  tt_assert(!bitarray_is_set(changed1, 2));
+  tt_assert(!bitarray_is_set(changed1, 3));
+
+  tt_assert(!bitarray_is_set(changed2, 0));
+  tt_assert(bitarray_is_set(changed2, 1));
+  tt_assert(bitarray_is_set(changed2, 2));
+  tt_assert(!bitarray_is_set(changed2, 3));
+  bitarray_clear(changed2, 1);
+  bitarray_clear(changed2, 2);
+
+  /* Length of sls1 is 1 and its element is in sls2. */
+  tor_free(sls1);
+  sls1 = smartlist_slice(sl1, 0, 1);
+  set_changed(changed1, changed2, sls1, sls2);
+
+  /* The latter changes all elements but the (first) common one. */
+  tt_assert(!bitarray_is_set(changed1, 0));
+  tt_assert(!bitarray_is_set(changed1, 1));
+  tt_assert(!bitarray_is_set(changed1, 2));
+  tt_assert(!bitarray_is_set(changed1, 3));
+
+  tt_assert(!bitarray_is_set(changed2, 0));
+  tt_assert(!bitarray_is_set(changed2, 1));
+  tt_assert(bitarray_is_set(changed2, 2));
+  tt_assert(!bitarray_is_set(changed2, 3));
+  bitarray_clear(changed2, 2);
+
+  /* Length of sls1 is 1 and its element is not in sls2. */
+  tor_free(sls1);
+  sls1 = smartlist_slice(sl1, 1, 2);
+  set_changed(changed1, changed2, sls1, sls2);
+
+  /* The former changes its element, the latter changes all elements. */
+  tt_assert(!bitarray_is_set(changed1, 0));
+  tt_assert(bitarray_is_set(changed1, 1));
+  tt_assert(!bitarray_is_set(changed1, 2));
+  tt_assert(!bitarray_is_set(changed1, 3));
+
+  tt_assert(!bitarray_is_set(changed2, 0));
+  tt_assert(bitarray_is_set(changed2, 1));
+  tt_assert(bitarray_is_set(changed2, 2));
+  tt_assert(!bitarray_is_set(changed2, 3));
+
+ done:
+  bitarray_free(changed1);
+  bitarray_free(changed2);
+  smartlist_free(sl1);
+  smartlist_free(sl2);
+  tor_free(sls1);
+  tor_free(sls2);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_calc_changes(void *arg)
+{
+  smartlist_t *sl1 = smartlist_new();
+  smartlist_t *sl2 = smartlist_new();
+  smartlist_slice_t *sls1, *sls2;
+  bitarray_t *changed1 = bitarray_init_zero(4);
+  bitarray_t *changed2 = bitarray_init_zero(4);
+  memarea_t *area = memarea_new();
+
+  (void)arg;
+  consensus_split_lines(sl1, "a\na\na\na\n", area);
+  consensus_split_lines(sl2, "a\na\na\na\n", area);
+
+  sls1 = smartlist_slice(sl1, 0, -1);
+  sls2 = smartlist_slice(sl2, 0, -1);
+  calc_changes(sls1, sls2, changed1, changed2);
+
+  /* Nothing should be set to changed. */
+  tt_assert(!bitarray_is_set(changed1, 0));
+  tt_assert(!bitarray_is_set(changed1, 1));
+  tt_assert(!bitarray_is_set(changed1, 2));
+  tt_assert(!bitarray_is_set(changed1, 3));
+
+  tt_assert(!bitarray_is_set(changed2, 0));
+  tt_assert(!bitarray_is_set(changed2, 1));
+  tt_assert(!bitarray_is_set(changed2, 2));
+  tt_assert(!bitarray_is_set(changed2, 3));
+
+  smartlist_clear(sl2);
+  consensus_split_lines(sl2, "a\nb\na\nb\n", area);
+  tor_free(sls1);
+  tor_free(sls2);
+  sls1 = smartlist_slice(sl1, 0, -1);
+  sls2 = smartlist_slice(sl2, 0, -1);
+  calc_changes(sls1, sls2, changed1, changed2);
+
+  /* Two elements are changed. */
+  tt_assert(!bitarray_is_set(changed1, 0));
+  tt_assert(bitarray_is_set(changed1, 1));
+  tt_assert(bitarray_is_set(changed1, 2));
+  tt_assert(!bitarray_is_set(changed1, 3));
+  bitarray_clear(changed1, 1);
+  bitarray_clear(changed1, 2);
+
+  tt_assert(!bitarray_is_set(changed2, 0));
+  tt_assert(bitarray_is_set(changed2, 1));
+  tt_assert(!bitarray_is_set(changed2, 2));
+  tt_assert(bitarray_is_set(changed2, 3));
+  bitarray_clear(changed1, 1);
+  bitarray_clear(changed1, 3);
+
+  smartlist_clear(sl2);
+  consensus_split_lines(sl2, "b\nb\nb\nb\n", area);
+  tor_free(sls1);
+  tor_free(sls2);
+  sls1 = smartlist_slice(sl1, 0, -1);
+  sls2 = smartlist_slice(sl2, 0, -1);
+  calc_changes(sls1, sls2, changed1, changed2);
+
+  /* All elements are changed. */
+  tt_assert(bitarray_is_set(changed1, 0));
+  tt_assert(bitarray_is_set(changed1, 1));
+  tt_assert(bitarray_is_set(changed1, 2));
+  tt_assert(bitarray_is_set(changed1, 3));
+
+  tt_assert(bitarray_is_set(changed2, 0));
+  tt_assert(bitarray_is_set(changed2, 1));
+  tt_assert(bitarray_is_set(changed2, 2));
+  tt_assert(bitarray_is_set(changed2, 3));
+
+ done:
+  bitarray_free(changed1);
+  bitarray_free(changed2);
+  smartlist_free(sl1);
+  smartlist_free(sl2);
+  tor_free(sls1);
+  tor_free(sls2);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_get_id_hash(void *arg)
+{
+  (void)arg;
+
+  cdline_t line1 = { "r name", 6 };
+  cdline_t line2 = { "r name _hash_isnt_base64 etc", 28 };
+  cdline_t line3 = { "r name hash+valid+base64 etc", 28 };
+  cdline_t tmp;
+
+  /* No hash. */
+  tt_int_op(-1, OP_EQ, get_id_hash(&line1, &tmp));
+  /* The hash contains characters that are not base64. */
+  tt_int_op(-1, OP_EQ, get_id_hash(&line2, &tmp));
+
+  /* valid hash. */
+  tt_int_op(0, OP_EQ, get_id_hash(&line3, &tmp));
+  tt_ptr_op(tmp.s, OP_EQ, line3.s + 7);
+  tt_uint_op(tmp.len, OP_EQ, line3.len - 11);
+
+ done:
+  ;
+}
+
+static void
+test_consdiff_is_valid_router_entry(void *arg)
+{
+  /* Doesn't start with "r ". */
+  (void)arg;
+  cdline_t line0 = { "foo", 3 };
+  tt_int_op(0, OP_EQ, is_valid_router_entry(&line0));
+
+  /* These are already tested with get_id_hash, but make sure it's run
+   * properly. */
+
+  cdline_t line1 = { "r name", 6 };
+  cdline_t line2 = { "r name _hash_isnt_base64 etc", 28 };
+  cdline_t line3 = { "r name hash+valid+base64 etc", 28 };
+  tt_int_op(0, OP_EQ, is_valid_router_entry(&line1));
+  tt_int_op(0, OP_EQ, is_valid_router_entry(&line2));
+  tt_int_op(1, OP_EQ, is_valid_router_entry(&line3));
+
+ done:
+  ;
+}
+
+static void
+test_consdiff_next_router(void *arg)
+{
+  smartlist_t *sl = smartlist_new();
+  memarea_t *area = memarea_new();
+  (void)arg;
+  smartlist_add_linecpy(sl, area, "foo");
+  smartlist_add_linecpy(sl, area,
+      "r name hash+longer+than+27+chars+and+valid+base64 etc");
+  smartlist_add_linecpy(sl, area, "foo");
+  smartlist_add_linecpy(sl, area, "foo");
+  smartlist_add_linecpy(sl, area,
+      "r name hash+longer+than+27+chars+and+valid+base64 etc");
+  smartlist_add_linecpy(sl, area, "foo");
+
+  /* Not currently on a router entry line, finding the next one. */
+  tt_int_op(1, OP_EQ, next_router(sl, 0));
+  tt_int_op(4, OP_EQ, next_router(sl, 2));
+
+  /* Already at the beginning of a router entry line, ignore it. */
+  tt_int_op(4, OP_EQ, next_router(sl, 1));
+
+  /* There are no more router entries, so return the line after the last. */
+  tt_int_op(6, OP_EQ, next_router(sl, 4));
+  tt_int_op(6, OP_EQ, next_router(sl, 5));
+
+ done:
+  smartlist_free(sl);
+  memarea_drop_all(area);
+}
+
+static int
+base64cmp_wrapper(const char *a, const char *b)
+{
+  cdline_t aa = { a, a ? (uint32_t) strlen(a) : 0 };
+  cdline_t bb = { b, b ? (uint32_t) strlen(b) : 0 };
+  return base64cmp(&aa, &bb);
+}
+
+static void
+test_consdiff_base64cmp(void *arg)
+{
+  /* NULL arguments. */
+  (void)arg;
+  tt_int_op(0, OP_EQ, base64cmp_wrapper(NULL, NULL));
+  tt_int_op(-1, OP_EQ, base64cmp_wrapper(NULL, "foo"));
+  tt_int_op(1, OP_EQ, base64cmp_wrapper("bar", NULL));
+
+  /* Nil base64 values. */
+  tt_int_op(0, OP_EQ, base64cmp_wrapper("", ""));
+  tt_int_op(0, OP_EQ, base64cmp_wrapper("_", "&"));
+
+  /* Exact same valid strings. */
+  tt_int_op(0, OP_EQ, base64cmp_wrapper("abcABC/+", "abcABC/+"));
+  /* Both end with an invalid base64 char other than '\0'. */
+  tt_int_op(0, OP_EQ, base64cmp_wrapper("abcABC/+ ", "abcABC/+ "));
+  /* Only one ends with an invalid base64 char other than '\0'. */
+  tt_int_op(-1, OP_EQ, base64cmp_wrapper("abcABC/+ ", "abcABC/+a"));
+
+  /* Comparisons that would return differently with strcmp(). */
+  tt_int_op(strcmp("/foo", "Afoo"), OP_LT, 0);
+  tt_int_op(base64cmp_wrapper("/foo", "Afoo"), OP_GT, 0);
+  tt_int_op(strcmp("Afoo", "0foo"), OP_GT, 0);
+  tt_int_op(base64cmp_wrapper("Afoo", "0foo"), OP_LT, 0);
+
+  /* Comparisons that would return the same as with strcmp(). */
+  tt_int_op(strcmp("afoo", "Afoo"), OP_GT, 0);
+  tt_int_op(base64cmp_wrapper("afoo", "Afoo"), OP_GT, 0);
+
+  /* Different lengths */
+  tt_int_op(base64cmp_wrapper("afoo", "afooo"), OP_LT, 0);
+  tt_int_op(base64cmp_wrapper("afooo", "afoo"), OP_GT, 0);
+
+ done:
+  ;
+}
+
+static void
+test_consdiff_gen_ed_diff(void *arg)
+{
+  smartlist_t *cons1=NULL, *cons2=NULL, *diff=NULL;
+  int i;
+  memarea_t *area = memarea_new();
+  setup_capture_of_logs(LOG_WARN);
+
+  (void)arg;
+  cons1 = smartlist_new();
+  cons2 = smartlist_new();
+
+  /* Identity hashes are not sorted properly, return NULL. */
+  smartlist_add_linecpy(cons1, area, "r name bbbbbbbbbbbbbbbbbbbbbbbbbbb etc");
+  smartlist_add_linecpy(cons1, area, "foo");
+  smartlist_add_linecpy(cons1, area, "r name aaaaaaaaaaaaaaaaaaaaaaaaaaa etc");
+  smartlist_add_linecpy(cons1, area, "bar");
+
+  smartlist_add_linecpy(cons2, area, "r name aaaaaaaaaaaaaaaaaaaaaaaaaaa etc");
+  smartlist_add_linecpy(cons2, area, "foo");
+  smartlist_add_linecpy(cons2, area, "r name ccccccccccccccccccccccccccc etc");
+  smartlist_add_linecpy(cons2, area, "bar");
+
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_EQ, diff);
+  expect_single_log_msg_containing("Refusing to generate consensus diff "
+         "because the base consensus doesn't have its router entries sorted "
+         "properly.");
+
+  /* Same, but now with the second consensus. */
+  mock_clean_saved_logs();
+  diff = gen_ed_diff(cons2, cons1, area);
+  tt_ptr_op(NULL, OP_EQ, diff);
+  expect_single_log_msg_containing("Refusing to generate consensus diff "
+         "because the target consensus doesn't have its router entries sorted "
+         "properly.");
+
+  /* Same as the two above, but with the reversed thing immediately after a
+     match. (The code handles this differently) */
+  smartlist_del(cons1, 0);
+  smartlist_add_linecpy(cons1, area, "r name aaaaaaaaaaaaaaaaaaaaaaaaaaa etc");
+
+  mock_clean_saved_logs();
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_EQ, diff);
+  expect_single_log_msg_containing("Refusing to generate consensus diff "
+         "because the base consensus doesn't have its router entries sorted "
+         "properly.");
+
+  mock_clean_saved_logs();
+  diff = gen_ed_diff(cons2, cons1, area);
+  tt_ptr_op(NULL, OP_EQ, diff);
+  expect_single_log_msg_containing("Refusing to generate consensus diff "
+         "because the target consensus doesn't have its router entries sorted "
+         "properly.");
+
+  /* Identity hashes are repeated, return NULL. */
+  smartlist_clear(cons1);
+
+  smartlist_add_linecpy(cons1, area, "r name bbbbbbbbbbbbbbbbbbbbbbbbbbb etc");
+  smartlist_add_linecpy(cons1, area, "foo");
+  smartlist_add_linecpy(cons1, area, "r name bbbbbbbbbbbbbbbbbbbbbbbbbbb etc");
+  smartlist_add_linecpy(cons1, area, "bar");
+
+  mock_clean_saved_logs();
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_EQ, diff);
+  expect_single_log_msg_containing("Refusing to generate consensus diff "
+         "because the base consensus doesn't have its router entries sorted "
+         "properly.");
+
+  /* We have to add a line that is just a dot, return NULL. */
+  smartlist_clear(cons1);
+  smartlist_clear(cons2);
+
+  smartlist_add_linecpy(cons1, area, "foo1");
+  smartlist_add_linecpy(cons1, area, "foo2");
+
+  smartlist_add_linecpy(cons2, area, "foo1");
+  smartlist_add_linecpy(cons2, area, ".");
+  smartlist_add_linecpy(cons2, area, "foo2");
+
+  mock_clean_saved_logs();
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_EQ, diff);
+  expect_single_log_msg_containing("Cannot generate consensus diff "
+         "because one of the lines to be added is \".\".");
+
+#define MAX_LINE_COUNT (10000)
+  /* Too many lines to be fed to the quadratic-time function. */
+  smartlist_clear(cons1);
+  smartlist_clear(cons2);
+
+  for (i=0; i < MAX_LINE_COUNT; ++i) smartlist_add_linecpy(cons1, area, "a");
+  for (i=0; i < MAX_LINE_COUNT; ++i) smartlist_add_linecpy(cons1, area, "b");
+
+  mock_clean_saved_logs();
+  diff = gen_ed_diff(cons1, cons2, area);
+
+  tt_ptr_op(NULL, OP_EQ, diff);
+  expect_single_log_msg_containing("Refusing to generate consensus diff "
+         "because we found too few common router ids.");
+
+  /* We have dot lines, but they don't interfere with the script format. */
+  smartlist_clear(cons1);
+  smartlist_clear(cons2);
+
+  smartlist_add_linecpy(cons1, area, "foo1");
+  smartlist_add_linecpy(cons1, area, ".");
+  smartlist_add_linecpy(cons1, area, ".");
+  smartlist_add_linecpy(cons1, area, "foo2");
+
+  smartlist_add_linecpy(cons2, area, "foo1");
+  smartlist_add_linecpy(cons2, area, ".");
+  smartlist_add_linecpy(cons2, area, "foo2");
+
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  smartlist_free(diff);
+
+  /* Empty diff tests. */
+  smartlist_clear(cons1);
+  smartlist_clear(cons2);
+
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(0, OP_EQ, smartlist_len(diff));
+  smartlist_free(diff);
+
+  smartlist_add_linecpy(cons1, area, "foo");
+  smartlist_add_linecpy(cons1, area, "bar");
+
+  smartlist_add_linecpy(cons2, area, "foo");
+  smartlist_add_linecpy(cons2, area, "bar");
+
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(0, OP_EQ, smartlist_len(diff));
+  smartlist_free(diff);
+
+  /* Everything is deleted. */
+  smartlist_clear(cons2);
+
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(1, OP_EQ, smartlist_len(diff));
+  tt_str_eq_line("1,2d", smartlist_get(diff, 0));
+
+  smartlist_free(diff);
+
+  /* Everything is added. */
+  diff = gen_ed_diff(cons2, cons1, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(4, OP_EQ, smartlist_len(diff));
+  tt_str_eq_line("0a", smartlist_get(diff, 0));
+  tt_str_eq_line("foo", smartlist_get(diff, 1));
+  tt_str_eq_line("bar", smartlist_get(diff, 2));
+  tt_str_eq_line(".", smartlist_get(diff, 3));
+
+  smartlist_free(diff);
+
+  /* Everything is changed. */
+  smartlist_add_linecpy(cons2, area, "foo2");
+  smartlist_add_linecpy(cons2, area, "bar2");
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(4, OP_EQ, smartlist_len(diff));
+  tt_str_eq_line("1,2c", smartlist_get(diff, 0));
+  tt_str_eq_line("foo2", smartlist_get(diff, 1));
+  tt_str_eq_line("bar2", smartlist_get(diff, 2));
+  tt_str_eq_line(".", smartlist_get(diff, 3));
+
+  smartlist_free(diff);
+
+  /* Test 'a', 'c' and 'd' together. See that it is done in reverse order. */
+  smartlist_clear(cons1);
+  smartlist_clear(cons2);
+  consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area);
+  consensus_split_lines(cons2, "A\nC\nO\nE\nU\n", area);
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(7, OP_EQ, smartlist_len(diff));
+  tt_str_eq_line("5a", smartlist_get(diff, 0));
+  tt_str_eq_line("U", smartlist_get(diff, 1));
+  tt_str_eq_line(".", smartlist_get(diff, 2));
+  tt_str_eq_line("4c", smartlist_get(diff, 3));
+  tt_str_eq_line("O", smartlist_get(diff, 4));
+  tt_str_eq_line(".", smartlist_get(diff, 5));
+  tt_str_eq_line("2d", smartlist_get(diff, 6));
+
+  smartlist_free(diff);
+
+  smartlist_clear(cons1);
+  smartlist_clear(cons2);
+  consensus_split_lines(cons1, "B\n", area);
+  consensus_split_lines(cons2, "A\nB\n", area);
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(3, OP_EQ, smartlist_len(diff));
+  tt_str_eq_line("0a", smartlist_get(diff, 0));
+  tt_str_eq_line("A", smartlist_get(diff, 1));
+  tt_str_eq_line(".", smartlist_get(diff, 2));
+
+  /* TODO: small real use-cases, i.e. consensuses. */
+
+ done:
+  teardown_capture_of_logs();
+  smartlist_free(cons1);
+  smartlist_free(cons2);
+  smartlist_free(diff);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_apply_ed_diff(void *arg)
+{
+  smartlist_t *cons1=NULL, *cons2=NULL, *diff=NULL;
+  memarea_t *area = memarea_new();
+  (void)arg;
+  cons1 = smartlist_new();
+  diff = smartlist_new();
+  setup_capture_of_logs(LOG_WARN);
+
+  consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area);
+
+  /* Command without range. */
+  smartlist_add_linecpy(diff, area, "a");
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  smartlist_clear(diff);
+  expect_single_log_msg_containing("an ed command was missing a line number");
+
+  /* Range without command. */
+  smartlist_add_linecpy(diff, area, "1");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("a line with no ed command was found");
+
+  smartlist_clear(diff);
+
+  /* Range without end. */
+  smartlist_add_linecpy(diff, area, "1,");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an ed command was missing a range "
+                                   "end line number.");
+
+  smartlist_clear(diff);
+
+  /* Incoherent ranges. */
+  smartlist_add_linecpy(diff, area, "1,1");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an invalid range was found");
+
+  smartlist_clear(diff);
+
+  smartlist_add_linecpy(diff, area, "3,2");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an invalid range was found");
+
+  smartlist_clear(diff);
+
+  /* Unexpected range for add command. */
+  smartlist_add_linecpy(diff, area, "1,2a");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("add lines after a range");
+
+  smartlist_clear(diff);
+
+  /* $ for a non-delete command. */
+  smartlist_add_linecpy(diff, area, "1,$c");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("it wanted to use $ with a command "
+                                   "other than delete");
+
+  smartlist_clear(diff);
+
+  /* Script is not in reverse order. */
+  smartlist_add_linecpy(diff, area, "1d");
+  smartlist_add_linecpy(diff, area, "3d");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("its commands are not properly sorted");
+
+  smartlist_clear(diff);
+
+  /* Script contains unrecognised commands longer than one char. */
+  smartlist_add_linecpy(diff, area, "1foo");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an ed command longer than one char was "
+                                   "found");
+
+  smartlist_clear(diff);
+
+  /* Script contains unrecognised commands. */
+  smartlist_add_linecpy(diff, area, "1e");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an unrecognised ed command was found");
+
+  smartlist_clear(diff);
+
+  /* Command that should be followed by at least one line and a ".", but
+   * isn't. */
+  smartlist_add_linecpy(diff, area, "0a");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("it has an ed command that tries to "
+                                   "insert zero lines.");
+
+  /* Now it is followed by a ".", but it inserts zero lines. */
+  smartlist_add_linecpy(diff, area, ".");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("it has an ed command that tries to "
+                                   "insert zero lines.");
+
+  smartlist_clear(diff);
+
+  /* Now it it inserts something, but has no terminator. */
+  smartlist_add_linecpy(diff, area, "0a");
+  smartlist_add_linecpy(diff, area, "hello");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("lines to be inserted that don't end with "
+                                   "a \".\".");
+
+  smartlist_clear(diff);
+
+  /* Ranges must be numeric only and cannot contain spaces. */
+  smartlist_add_linecpy(diff, area, "0, 4d");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an ed command was missing a range "
+                                   "end line number.");
+
+  smartlist_clear(diff);
+
+  /* '+' is not a number. */
+  smartlist_add_linecpy(diff, area, "+0,4d");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an ed command was missing a line number");
+
+  smartlist_clear(diff);
+
+  /* range duplication */
+  smartlist_add_linecpy(diff, area, "0,4d,5d");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an ed command longer than one char was "
+                                   "found");
+
+  smartlist_clear(diff);
+
+  /* space before command */
+  smartlist_add_linecpy(diff, area, "0,4 d");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an ed command longer than one char was "
+                                   "found");
+
+  smartlist_clear(diff);
+
+  /* space inside number */
+  smartlist_add_linecpy(diff, area, "0,4 5d");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an ed command longer than one char was "
+                                   "found");
+
+  smartlist_clear(diff);
+
+  /* Test appending text, 'a'. */
+  consensus_split_lines(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area);
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_NE, cons2);
+  tt_int_op(8, OP_EQ, smartlist_len(cons2));
+  tt_str_eq_line("V", smartlist_get(cons2, 0));
+  tt_str_eq_line("A", smartlist_get(cons2, 1));
+  tt_str_eq_line("B", smartlist_get(cons2, 2));
+  tt_str_eq_line("C", smartlist_get(cons2, 3));
+  tt_str_eq_line("U", smartlist_get(cons2, 4));
+  tt_str_eq_line("O", smartlist_get(cons2, 5));
+  tt_str_eq_line("D", smartlist_get(cons2, 6));
+  tt_str_eq_line("E", smartlist_get(cons2, 7));
+
+  smartlist_clear(diff);
+  smartlist_free(cons2);
+
+  /* Test deleting text, 'd'. */
+  consensus_split_lines(diff, "4d\n1,2d\n", area);
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_NE, cons2);
+  tt_int_op(2, OP_EQ, smartlist_len(cons2));
+  tt_str_eq_line("C", smartlist_get(cons2, 0));
+  tt_str_eq_line("E", smartlist_get(cons2, 1));
+
+  smartlist_clear(diff);
+  smartlist_free(cons2);
+
+  /* Test changing text, 'c'. */
+  consensus_split_lines(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area);
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_NE, cons2);
+  tt_int_op(5, OP_EQ, smartlist_len(cons2));
+  tt_str_eq_line("M", smartlist_get(cons2, 0));
+  tt_str_eq_line("C", smartlist_get(cons2, 1));
+  tt_str_eq_line("T", smartlist_get(cons2, 2));
+  tt_str_eq_line("X", smartlist_get(cons2, 3));
+  tt_str_eq_line("E", smartlist_get(cons2, 4));
+
+  smartlist_clear(diff);
+  smartlist_free(cons2);
+
+  /* Test 'a', 'd' and 'c' together. */
+  consensus_split_lines(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area);
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_NE, cons2);
+  tt_int_op(6, OP_EQ, smartlist_len(cons2));
+  tt_str_eq_line("M", smartlist_get(cons2, 0));
+  tt_str_eq_line("A", smartlist_get(cons2, 1));
+  tt_str_eq_line("C", smartlist_get(cons2, 2));
+  tt_str_eq_line("T", smartlist_get(cons2, 3));
+  tt_str_eq_line("X", smartlist_get(cons2, 4));
+  tt_str_eq_line("E", smartlist_get(cons2, 5));
+
+ done:
+  teardown_capture_of_logs();
+  smartlist_free(cons1);
+  smartlist_free(cons2);
+  smartlist_free(diff);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_gen_diff(void *arg)
+{
+  char *cons1_str=NULL, *cons2_str=NULL;
+  smartlist_t *cons1=NULL, *cons2=NULL, *diff=NULL;
+  consensus_digest_t digests1, digests2;
+  memarea_t *area = memarea_new();
+  (void)arg;
+  cons1 = smartlist_new();
+  cons2 = smartlist_new();
+
+  /* Identity hashes are not sorted properly, return NULL.
+   * Already tested in gen_ed_diff, but see that a NULL ed diff also makes
+   * gen_diff return NULL. */
+  cons1_str = tor_strdup(
+      "network-status-version foo\n"
+      "r name bbbbbbbbbbbbbbbbb etc\nfoo\n"
+      "r name aaaaaaaaaaaaaaaaa etc\nbar\n"
+      "directory-signature foo bar\nbar\n"
+      );
+  cons2_str = tor_strdup(
+      "network-status-version foo\n"
+      "r name aaaaaaaaaaaaaaaaa etc\nfoo\n"
+      "r name ccccccccccccccccc etc\nbar\n"
+      "directory-signature foo bar\nbar\n"
+      );
+
+  tt_int_op(0, OP_EQ,
+      consensus_compute_digest_as_signed(cons1_str, &digests1));
+  tt_int_op(0, OP_EQ,
+      consensus_compute_digest(cons2_str, &digests2));
+
+  consensus_split_lines(cons1, cons1_str, area);
+  consensus_split_lines(cons2, cons2_str, area);
+
+  diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area);
+  tt_ptr_op(NULL, OP_EQ, diff);
+
+  /* Check that the headers are done properly. */
+  tor_free(cons1_str);
+  cons1_str = tor_strdup(
+      "network-status-version foo\n"
+      "r name ccccccccccccccccc etc\nfoo\n"
+      "r name eeeeeeeeeeeeeeeee etc\nbar\n"
+      "directory-signature foo bar\nbar\n"
+      );
+  tt_int_op(0, OP_EQ,
+      consensus_compute_digest_as_signed(cons1_str, &digests1));
+  smartlist_clear(cons1);
+  consensus_split_lines(cons1, cons1_str, area);
+  diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(11, OP_EQ, smartlist_len(diff));
+  tt_assert(line_str_eq(smartlist_get(diff, 0),
+                        "network-status-diff-version 1"));
+  tt_assert(line_str_eq(smartlist_get(diff, 1), "hash "
+      "95D70F5A3CC65F920AA8B44C4563D7781A082674329661884E19E94B79D539C2 "
+      "7AFECEFA4599BA33D603653E3D2368F648DF4AC4723929B0F7CF39281596B0C1"));
+  tt_assert(line_str_eq(smartlist_get(diff, 2), "6,$d"));
+  tt_assert(line_str_eq(smartlist_get(diff, 3), "3,4c"));
+  tt_assert(line_str_eq(smartlist_get(diff, 4), "bar"));
+  tt_assert(line_str_eq(smartlist_get(diff, 5),
+                        "directory-signature foo bar"));
+  tt_assert(line_str_eq(smartlist_get(diff, 6),
+                        "."));
+  tt_assert(line_str_eq(smartlist_get(diff, 7), "1a"));
+  tt_assert(line_str_eq(smartlist_get(diff, 8),
+                        "r name aaaaaaaaaaaaaaaaa etc"));
+  tt_assert(line_str_eq(smartlist_get(diff, 9), "foo"));
+  tt_assert(line_str_eq(smartlist_get(diff, 10), "."));
+
+  /* TODO: small real use-cases, i.e. consensuses. */
+
+ done:
+  tor_free(cons1_str);
+  tor_free(cons2_str);
+  smartlist_free(cons1);
+  smartlist_free(cons2);
+  smartlist_free(diff);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_apply_diff(void *arg)
+{
+  smartlist_t *cons1=NULL, *diff=NULL;
+  char *cons1_str=NULL, *cons2 = NULL;
+  consensus_digest_t digests1;
+  (void)arg;
+  memarea_t *area = memarea_new();
+  cons1 = smartlist_new();
+  diff = smartlist_new();
+  setup_capture_of_logs(LOG_INFO);
+
+  cons1_str = tor_strdup(
+      "network-status-version foo\n"
+      "r name ccccccccccccccccc etc\nfoo\n"
+      "r name eeeeeeeeeeeeeeeee etc\nbar\n"
+      "directory-signature foo bar\nbar\n"
+      );
+  tt_int_op(0, OP_EQ,
+      consensus_compute_digest(cons1_str, &digests1));
+  consensus_split_lines(cons1, cons1_str, area);
+
+  /* diff doesn't have enough lines. */
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("too short")
+
+  /* first line doesn't match format-version string. */
+  smartlist_add_linecpy(diff, area, "foo-bar");
+  smartlist_add_linecpy(diff, area, "header-line");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("format is not known")
+
+  /* The first word of the second header line is not "hash". */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "word a b");
+  smartlist_add_linecpy(diff, area, "x");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("does not include the necessary digests")
+
+  /* Wrong number of words after "hash". */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash a b c");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("does not include the necessary digests")
+
+  /* base16 digests do not have the expected length. */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash aaa bbb");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("includes base16-encoded digests of "
+                                   "incorrect size")
+
+  /* base16 digests contain non-base16 characters. */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash"
+      " ????????????????????????????????????????????????????????????????"
+      " ----------------------------------------------------------------");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("includes malformed digests")
+
+  /* Invalid ed diff.
+   * As tested in apply_ed_diff, but check that apply_diff does return NULL if
+   * the ed diff can't be applied. */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash"
+      /* sha3 of cons1. */
+      " 06646D6CF563A41869D3B02E73254372AE3140046C5E7D83C9F71E54976AF9B4"
+      /* sha256 of cons2. */
+      " 635D34593020C08E5ECD865F9986E29D50028EFA62843766A8197AD228A7F6AA");
+  smartlist_add_linecpy(diff, area, "foobar");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("because an ed command was missing a line "
+                                   "number")
+
+  /* Base consensus doesn't match its digest as found in the diff. */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash"
+      /* bogus sha256. */
+      " 3333333333333333333333333333333333333333333333333333333333333333"
+      /* sha256 of cons2. */
+      " 635D34593020C08E5ECD865F9986E29D50028EFA62843766A8197AD228A7F6AA");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_log_msg_containing("base consensus doesn't match the digest "
+                            "as found");
+
+  /* Resulting consensus doesn't match its digest as found in the diff. */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash"
+      /* sha3 of cons1. */
+      " 06646D6CF563A41869D3B02E73254372AE3140046C5E7D83C9F71E54976AF9B4"
+      /* bogus sha3. */
+      " 3333333333333333333333333333333333333333333333333333333333333333");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_log_msg_containing("resulting consensus doesn't match the "
+                            "digest as found");
+
+#if 0
+  /* XXXX No longer possible, since we aren't using the other algorithm. */
+  /* Resulting consensus digest cannot be computed */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash"
+      /* sha3 of cons1. */
+      " 06646D6CF563A41869D3B02E73254372AE3140046C5E7D83C9F71E54976AF9B4"
+      /* bogus sha3. */
+      " 3333333333333333333333333333333333333333333333333333333333333333");
+  smartlist_add_linecpy(diff, area, "1,2d"); // remove starting line
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_log_msg_containing("Could not compute digests of the consensus "
+                            "resulting from applying a consensus diff.");
+#endif /* 0 */
+
+  /* Very simple test, only to see that nothing errors. */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash"
+      /* sha3 of cons1. */
+      " 06646D6CF563A41869D3B02E73254372AE3140046C5E7D83C9F71E54976AF9B4"
+      /* sha3 of cons2. */
+      " 90A418881B2FCAB3D9E60EE02E4D666D56CFA38F8A3B7AA3E0ADBA530DDA9353");
+  smartlist_add_linecpy(diff, area, "3c");
+  smartlist_add_linecpy(diff, area, "sample");
+  smartlist_add_linecpy(diff, area, ".");
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_NE, cons2);
+  tt_str_op(
+      "network-status-version foo\n"
+      "r name ccccccccccccccccc etc\nsample\n"
+      "r name eeeeeeeeeeeeeeeee etc\nbar\n"
+      "directory-signature foo bar\nbar\n", OP_EQ,
+      cons2);
+  tor_free(cons2);
+
+  /* Check that lowercase letters in base16-encoded digests work too. */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash"
+      /* sha3 of cons1. */
+      " 06646d6cf563a41869d3b02e73254372ae3140046c5e7d83c9f71e54976af9b4"
+      /* sha3 of cons2. */
+      " 90a418881b2fcab3d9e60ee02e4d666d56cfa38f8a3b7aa3e0adba530dda9353");
+  smartlist_add_linecpy(diff, area, "3c");
+  smartlist_add_linecpy(diff, area, "sample");
+  smartlist_add_linecpy(diff, area, ".");
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_NE, cons2);
+  tt_str_op(
+      "network-status-version foo\n"
+      "r name ccccccccccccccccc etc\nsample\n"
+      "r name eeeeeeeeeeeeeeeee etc\nbar\n"
+      "directory-signature foo bar\nbar\n", OP_EQ,
+      cons2);
+  tor_free(cons2);
+
+  smartlist_clear(diff);
+
+ done:
+  teardown_capture_of_logs();
+  tor_free(cons1_str);
+  smartlist_free(cons1);
+  smartlist_free(diff);
+  memarea_drop_all(area);
+}
+
+#define CONSDIFF_LEGACY(name)                                          \
+  { #name, test_consdiff_ ## name , 0, NULL, NULL }
+
+struct testcase_t consdiff_tests[] = {
+  CONSDIFF_LEGACY(smartlist_slice),
+  CONSDIFF_LEGACY(smartlist_slice_string_pos),
+  CONSDIFF_LEGACY(lcs_lengths),
+  CONSDIFF_LEGACY(trim_slices),
+  CONSDIFF_LEGACY(set_changed),
+  CONSDIFF_LEGACY(calc_changes),
+  CONSDIFF_LEGACY(get_id_hash),
+  CONSDIFF_LEGACY(is_valid_router_entry),
+  CONSDIFF_LEGACY(next_router),
+  CONSDIFF_LEGACY(base64cmp),
+  CONSDIFF_LEGACY(gen_ed_diff),
+  CONSDIFF_LEGACY(apply_ed_diff),
+  CONSDIFF_LEGACY(gen_diff),
+  CONSDIFF_LEGACY(apply_diff),
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c
new file mode 100644
index 0000000000..254a5ba5d0
--- /dev/null
+++ b/src/test/test_consdiffmgr.c
@@ -0,0 +1,900 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CONSDIFFMGR_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "feature/dircache/conscache.h"
+#include "feature/dircommon/consdiff.h"
+#include "feature/dircache/consdiffmgr.h"
+#include "core/mainloop/cpuworker.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/dirparse/ns_parse.h"
+#include "lib/evloop/workqueue.h"
+#include "lib/compress/compress.h"
+#include "lib/encoding/confline.h"
+
+#include "feature/nodelist/networkstatus_st.h"
+
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+
+// ============================== Setup/teardown the consdiffmgr
+// These functions get run before/after each test in this module
+
+static void *
+consdiffmgr_test_setup(const struct testcase_t *arg)
+{
+  (void)arg;
+  char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cdm"));
+  tor_free(get_options_mutable()->CacheDirectory);
+  get_options_mutable()->CacheDirectory = ddir_fname; // now owns the pointer.
+  check_private_dir(ddir_fname, CPD_CREATE, NULL);
+
+  consdiff_cfg_t consdiff_cfg = { 300 };
+  consdiffmgr_configure(&consdiff_cfg);
+  return (void *)1; // must return something non-null.
+}
+static int
+consdiffmgr_test_teardown(const struct testcase_t *arg, void *ignore)
+{
+  (void)arg;
+  (void)ignore;
+  consdiffmgr_free_all();
+  return 1;
+}
+static struct testcase_setup_t setup_diffmgr = {
+  consdiffmgr_test_setup,
+  consdiffmgr_test_teardown
+};
+
+// ============================== NS faking functions
+// These functions are for making quick fake consensus objects and
+// strings that are just good enough for consdiff and consdiffmgr.
+
+static networkstatus_t *
+fake_ns_new(consensus_flavor_t flav, time_t valid_after)
+{
+  networkstatus_t *ns = tor_malloc_zero(sizeof(networkstatus_t));
+  ns->type = NS_TYPE_CONSENSUS;
+  ns->flavor = flav;
+  ns->valid_after = valid_after;
+  return ns;
+}
+
+static char *
+fake_ns_body_new(consensus_flavor_t flav, time_t valid_after)
+{
+  const char *flavor_string = flav == FLAV_NS ? "" : " microdesc";
+  char valid_after_string[ISO_TIME_LEN+1];
+
+  format_iso_time(valid_after_string, valid_after);
+  char *random_stuff = crypto_random_hostname(3, 25, "junk ", "");
+  char *random_stuff2 = crypto_random_hostname(3, 10, "", "");
+
+  char *consensus;
+  tor_asprintf(&consensus,
+               "network-status-version 3%s\n"
+               "vote-status consensus\n"
+               "valid-after %s\n"
+               "r name ccccccccccccccccc etc\nsample\n"
+               "r name eeeeeeeeeeeeeeeee etc\nbar\n"
+               "%s\n"
+               "directory-signature hello-there\n"
+               "directory-signature %s\n",
+               flavor_string,
+               valid_after_string,
+               random_stuff,
+               random_stuff2);
+  tor_free(random_stuff);
+  tor_free(random_stuff2);
+  return consensus;
+}
+
+// ============================== Cpuworker mocking code
+// These mocking functions and types capture the cpuworker calls
+// so we can inspect them and run them in the main thread.
+static smartlist_t *fake_cpuworker_queue = NULL;
+typedef struct fake_work_queue_ent_t {
+  enum workqueue_reply_t (*fn)(void *, void *);
+  void (*reply_fn)(void *);
+  void *arg;
+} fake_work_queue_ent_t;
+static struct workqueue_entry_s *
+mock_cpuworker_queue_work(workqueue_priority_t prio,
+                          enum workqueue_reply_t (*fn)(void *, void *),
+                          void (*reply_fn)(void *),
+                          void *arg)
+{
+  (void) prio;
+
+  if (! fake_cpuworker_queue)
+    fake_cpuworker_queue = smartlist_new();
+
+  fake_work_queue_ent_t *ent = tor_malloc_zero(sizeof(*ent));
+  ent->fn = fn;
+  ent->reply_fn = reply_fn;
+  ent->arg = arg;
+  smartlist_add(fake_cpuworker_queue, ent);
+  return (struct workqueue_entry_s *)ent;
+}
+static int
+mock_cpuworker_run_work(void)
+{
+  if (! fake_cpuworker_queue)
+    return 0;
+  SMARTLIST_FOREACH(fake_cpuworker_queue, fake_work_queue_ent_t *, ent, {
+      enum workqueue_reply_t r = ent->fn(NULL, ent->arg);
+      if (r != WQ_RPL_REPLY)
+        return -1;
+  });
+  return 0;
+}
+static void
+mock_cpuworker_handle_replies(void)
+{
+  if (! fake_cpuworker_queue)
+    return;
+  SMARTLIST_FOREACH(fake_cpuworker_queue, fake_work_queue_ent_t *, ent, {
+      ent->reply_fn(ent->arg);
+      tor_free(ent);
+  });
+  smartlist_free(fake_cpuworker_queue);
+  fake_cpuworker_queue = NULL;
+}
+
+// ==============================  Other helpers
+
+static consdiff_status_t
+lookup_diff_from(consensus_cache_entry_t **out,
+                 consensus_flavor_t flav,
+                 const char *str1)
+{
+  uint8_t digest[DIGEST256_LEN];
+  if (router_get_networkstatus_v3_sha3_as_signed(digest, str1)<0) {
+    TT_FAIL(("Unable to compute sha3-as-signed"));
+    return CONSDIFF_NOT_FOUND;
+  }
+  return consdiffmgr_find_diff_from(out, flav,
+                                    DIGEST_SHA3_256, digest, sizeof(digest),
+                                    NO_METHOD);
+}
+
+static int
+lookup_apply_and_verify_diff(consensus_flavor_t flav,
+                             const char *str1,
+                             const char *str2)
+{
+  consensus_cache_entry_t *ent = NULL;
+  consdiff_status_t status = lookup_diff_from(&ent, flav, str1);
+  if (ent == NULL || status != CONSDIFF_AVAILABLE) {
+    return -1;
+  }
+
+  consensus_cache_entry_incref(ent);
+  size_t size;
+  char *diff_string = NULL;
+  int r = uncompress_or_copy(&diff_string, &size, ent);
+  consensus_cache_entry_decref(ent);
+  if (diff_string == NULL || r < 0)
+    return -1;
+
+  char *applied = consensus_diff_apply(str1, diff_string);
+  tor_free(diff_string);
+  if (applied == NULL)
+    return -1;
+
+  int match = !strcmp(applied, str2);
+  tor_free(applied);
+  return match ? 0 : -1;
+}
+
+static void
+cdm_reload(void)
+{
+  consdiffmgr_free_all();
+  cdm_cache_get();
+  consdiffmgr_rescan();
+}
+
+// ==============================  Beginning of tests
+
+#if 0
+static int got_failure = 0;
+static void
+got_assertion_failure(void)
+{
+  ++got_failure;
+}
+
+/* XXXX This test won't work, because there is currently no way to actually
+ * XXXX capture a real assertion failure. */
+static void
+test_consdiffmgr_init_failure(void *arg)
+{
+  (void)arg;
+  // Capture assertions and bugs.
+
+  /* As in ...test_setup, but do not create the datadir. The missing directory
+   * will cause a failure. */
+  char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cdm"));
+  tor_free(get_options_mutable()->CacheDirectory);
+  get_options_mutable()->CacheDirectory = ddir_fname; // now owns the pointer.
+
+  consdiff_cfg_t consdiff_cfg = { 7200, 300 };
+
+  tor_set_failed_assertion_callback(got_assertion_failure);
+  tor_capture_bugs_(1);
+  consdiffmgr_configure(&consdiff_cfg); // This should fail.
+  tt_int_op(got_failure, OP_EQ, 1);
+  const smartlist_t *bugs = tor_get_captured_bug_log_();
+  tt_int_op(smartlist_len(bugs), OP_EQ, 1);
+
+ done:
+  tor_end_capture_bugs_();
+}
+#endif /* 0 */
+
+static void
+test_consdiffmgr_sha3_helper(void *arg)
+{
+  (void) arg;
+  consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
+  config_line_t *lines = NULL;
+  char *mem_op_hex_tmp = NULL;
+  config_line_prepend(&lines, "good-sha",
+                      "F00DF00DF00DF00DF00DF00DF00DF00D"
+                      "F00DF00DF00DF00DF00DF00DF00DF00D");
+  config_line_prepend(&lines, "short-sha",
+                      "F00DF00DF00DF00DF00DF00DF00DF00D"
+                      "F00DF00DF00DF00DF00DF00DF00DF0");
+  config_line_prepend(&lines, "long-sha",
+                      "F00DF00DF00DF00DF00DF00DF00DF00D"
+                      "F00DF00DF00DF00DF00DF00DF00DF00DF00D");
+  config_line_prepend(&lines, "not-sha",
+                      "F00DF00DF00DF00DF00DF00DF00DF00D"
+                      "F00DF00DF00DF00DF00DF00DF00DXXXX");
+  consensus_cache_entry_t *ent =
+    consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
+
+  uint8_t buf[DIGEST256_LEN];
+  tt_int_op(-1, OP_EQ, cdm_entry_get_sha3_value(buf, NULL, "good-sha"));
+  tt_int_op(0, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "good-sha"));
+  test_memeq_hex(buf, "F00DF00DF00DF00DF00DF00DF00DF00D"
+                      "F00DF00DF00DF00DF00DF00DF00DF00D");
+
+  tt_int_op(-1, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "missing-sha"));
+  tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "short-sha"));
+  tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "long-sha"));
+  tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "not-sha"));
+
+ done:
+  consensus_cache_entry_decref(ent);
+  config_free_lines(lines);
+  tor_free(mem_op_hex_tmp);
+}
+
+static void
+test_consdiffmgr_add(void *arg)
+{
+  (void) arg;
+  time_t now = approx_time();
+
+  char *body = NULL;
+
+  consensus_cache_entry_t *ent = NULL;
+  networkstatus_t *ns_tmp = fake_ns_new(FLAV_NS, now);
+  const char *dummy = "foo";
+  int r = consdiffmgr_add_consensus(dummy, ns_tmp);
+  tt_int_op(r, OP_EQ, 0);
+
+  /* If we add it again, it won't work */
+  setup_capture_of_logs(LOG_INFO);
+  dummy = "bar";
+  r = consdiffmgr_add_consensus(dummy, ns_tmp);
+  tt_int_op(r, OP_EQ, -1);
+  expect_single_log_msg_containing("We already have a copy of that "
+                                   "consensus");
+  mock_clean_saved_logs();
+
+  /* But it will work fine if the flavor is different */
+  dummy = "baz";
+  ns_tmp->flavor = FLAV_MICRODESC;
+  r = consdiffmgr_add_consensus(dummy, ns_tmp);
+  tt_int_op(r, OP_EQ, 0);
+
+  /* And it will work fine if the time is different */
+  dummy = "quux";
+  ns_tmp->flavor = FLAV_NS;
+  ns_tmp->valid_after = now - 60;
+  r = consdiffmgr_add_consensus(dummy, ns_tmp);
+  tt_int_op(r, OP_EQ, 0);
+
+  /* If we add one a long long time ago, it will fail. */
+  dummy = "xyzzy";
+  ns_tmp->valid_after = 86400 * 100; /* A few months into 1970 */
+  r = consdiffmgr_add_consensus(dummy, ns_tmp);
+  tt_int_op(r, OP_EQ, -1);
+  expect_log_msg_containing("it's too old.");
+
+  /* Try looking up a consensuses. */
+  ent = cdm_cache_lookup_consensus(FLAV_NS, now-60);
+  tt_assert(ent);
+  consensus_cache_entry_incref(ent);
+  size_t s;
+  r = uncompress_or_copy(&body, &s, ent);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(s, OP_EQ, 4);
+  tt_mem_op(body, OP_EQ, "quux", 4);
+
+  /* Try looking up another entry, but fail */
+  tt_ptr_op(cdm_cache_lookup_consensus(FLAV_MICRODESC, now - 60), OP_EQ, NULL);
+  tt_ptr_op(cdm_cache_lookup_consensus(FLAV_NS, now - 61), OP_EQ, NULL);
+
+ done:
+  networkstatus_vote_free(ns_tmp);
+  teardown_capture_of_logs();
+  consensus_cache_entry_decref(ent);
+  tor_free(body);
+}
+
+static void
+test_consdiffmgr_make_diffs(void *arg)
+{
+  (void)arg;
+  networkstatus_t *ns = NULL;
+  char *ns_body = NULL, *md_ns_body = NULL, *md_ns_body_2 = NULL;
+  char *applied = NULL, *diff_text = NULL;
+  time_t now = approx_time();
+  int r;
+  consensus_cache_entry_t *diff = NULL;
+  uint8_t md_ns_sha3[DIGEST256_LEN];
+  consdiff_status_t diff_status;
+
+  MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
+
+  // Try rescan with no consensuses: shouldn't crash or queue work.
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
+
+  // Make two consensuses, 1 hour sec ago.
+  ns = fake_ns_new(FLAV_NS, now-3600);
+  ns_body = fake_ns_body_new(FLAV_NS, now-3600);
+  r = consdiffmgr_add_consensus(ns_body, ns);
+  networkstatus_vote_free(ns);
+  tor_free(ns_body);
+  tt_int_op(r, OP_EQ, 0);
+
+  ns = fake_ns_new(FLAV_MICRODESC, now-3600);
+  md_ns_body = fake_ns_body_new(FLAV_MICRODESC, now-3600);
+  r = consdiffmgr_add_consensus(md_ns_body, ns);
+  router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body);
+  networkstatus_vote_free(ns);
+  tt_int_op(r, OP_EQ, 0);
+
+  // No diffs will be generated.
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
+
+  // Add a MD consensus from 45 minutes ago. This should cause one diff
+  // worth of work to get queued.
+  ns = fake_ns_new(FLAV_MICRODESC, now-45*60);
+  md_ns_body_2 = fake_ns_body_new(FLAV_MICRODESC, now-45*60);
+  r = consdiffmgr_add_consensus(md_ns_body_2, ns);
+  networkstatus_vote_free(ns);
+  tt_int_op(r, OP_EQ, 0);
+
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
+  tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
+  diff_status = consdiffmgr_find_diff_from(&diff, FLAV_MICRODESC,
+                                           DIGEST_SHA3_256,
+                                           md_ns_sha3, DIGEST256_LEN,
+                                           NO_METHOD);
+  tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ, diff_status);
+
+  // Now run that process and get the diff.
+  r = mock_cpuworker_run_work();
+  tt_int_op(r, OP_EQ, 0);
+  mock_cpuworker_handle_replies();
+
+  // At this point we should be able to get that diff.
+  diff_status = consdiffmgr_find_diff_from(&diff, FLAV_MICRODESC,
+                                           DIGEST_SHA3_256,
+                                           md_ns_sha3, DIGEST256_LEN,
+                                           NO_METHOD);
+  tt_int_op(CONSDIFF_AVAILABLE, OP_EQ, diff_status);
+  tt_assert(diff);
+
+  /* Make sure applying the diff actually works */
+  const uint8_t *diff_body;
+  size_t diff_size;
+  r = consensus_cache_entry_get_body(diff, &diff_body, &diff_size);
+  tt_int_op(r, OP_EQ, 0);
+  diff_text = tor_memdup_nulterm(diff_body, diff_size);
+  applied = consensus_diff_apply(md_ns_body, diff_text);
+  tt_assert(applied);
+  tt_str_op(applied, OP_EQ, md_ns_body_2);
+
+  /* Rescan again: no more work to do. */
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
+
+ done:
+  tor_free(md_ns_body);
+  tor_free(md_ns_body_2);
+  tor_free(diff_text);
+  tor_free(applied);
+}
+
+static void
+test_consdiffmgr_diff_rules(void *arg)
+{
+  (void)arg;
+#define N 6
+  char *md_body[N], *ns_body[N];
+  networkstatus_t *md_ns[N], *ns_ns[N];
+  int i;
+
+  MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
+
+  /* Create a bunch of consensus things at 15-second intervals. */
+  time_t start = approx_time() - 120;
+  for (i = 0; i < N; ++i) {
+    time_t when = start + i * 15;
+    md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when);
+    ns_body[i] = fake_ns_body_new(FLAV_NS, when);
+    md_ns[i] = fake_ns_new(FLAV_MICRODESC, when);
+    ns_ns[i] = fake_ns_new(FLAV_NS, when);
+  }
+
+  /* For the MD consensuses: add 4 of them, and make sure that
+   * diffs are created to one consensus (the most recent) only. */
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[3], md_ns[3]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[4], md_ns[4]));
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
+  tt_int_op(3, OP_EQ, smartlist_len(fake_cpuworker_queue));
+  tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
+  mock_cpuworker_handle_replies();
+  tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
+
+  /* For the NS consensuses: add 3, generate, and add one older one and
+   * make sure that older one is the only one whose diff is generated */
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[0], ns_ns[0]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[1], ns_ns[1]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[5], ns_ns[5]));
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
+  tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue));
+  tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
+  mock_cpuworker_handle_replies();
+
+  /* At this point, we should actually have working diffs! */
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_NS, ns_body[0], ns_body[5]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_NS, ns_body[1], ns_body[5]));
+
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[4]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[2], md_body[4]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[3], md_body[4]));
+
+  /* Self-to-self diff won't be present */
+  consensus_cache_entry_t *ent;
+  tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
+       lookup_diff_from(&ent, FLAV_NS, ns_body[5]));
+  /* No diff from 2 has been added yet */
+  tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
+       lookup_diff_from(&ent, FLAV_NS, ns_body[2]));
+  /* No diff arriving at old things. */
+  tt_int_op(-1, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2]));
+  /* No backwards diff */
+  tt_int_op(-1, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[4], md_body[3]));
+
+  /* Now, an update: add number 2 and make sure it's the only one whose diff
+   * is regenerated. */
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[2], ns_ns[2]));
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
+  tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
+  tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
+  mock_cpuworker_handle_replies();
+
+  tt_int_op(0, OP_EQ,
+            lookup_apply_and_verify_diff(FLAV_NS, ns_body[2], ns_body[5]));
+
+  /* Finally: reload, and make sure that the information is still indexed */
+  cdm_reload();
+
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_NS, ns_body[0], ns_body[5]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_NS, ns_body[2], ns_body[5]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_NS, ns_body[1], ns_body[5]));
+
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[4]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[2], md_body[4]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[3], md_body[4]));
+
+ done:
+  for (i = 0; i < N; ++i) {
+    tor_free(md_body[i]);
+    tor_free(ns_body[i]);
+    networkstatus_vote_free(md_ns[i]);
+    networkstatus_vote_free(ns_ns[i]);
+  }
+  UNMOCK(cpuworker_queue_work);
+#undef N
+}
+
+static void
+test_consdiffmgr_diff_failure(void *arg)
+{
+  (void)arg;
+  MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
+
+  /* We're going to make sure that if we have a bogus request where
+   * we can't actually compute a diff, the world must not end. */
+  networkstatus_t *ns1 = NULL;
+  networkstatus_t *ns2 = NULL;
+  int r;
+
+  ns1 = fake_ns_new(FLAV_NS, approx_time()-100);
+  ns2 = fake_ns_new(FLAV_NS, approx_time()-50);
+  r = consdiffmgr_add_consensus("foo bar baz\n", ns1);
+  tt_int_op(r, OP_EQ, 0);
+  // We refuse to compute a diff to or from a line holding only a single dot.
+  // We can add it here, though.
+  r = consdiffmgr_add_consensus("foo bar baz\n.\n.\n", ns2);
+  tt_int_op(r, OP_EQ, 0);
+
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
+  setup_capture_of_logs(LOG_WARN);
+  tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
+  tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
+  expect_single_log_msg_containing("one of the lines to be added is \".\".");
+  mock_clean_saved_logs();
+  mock_cpuworker_handle_replies();
+  expect_single_log_msg_containing("Worker was unable to compute consensus "
+                                   "diff from ");
+
+  /* Make sure the diff is not present */
+  consensus_cache_entry_t *ent;
+  tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
+            lookup_diff_from(&ent, FLAV_NS, "foo bar baz\n"));
+
+ done:
+  teardown_capture_of_logs();
+  UNMOCK(cpuworker_queue_work);
+  networkstatus_vote_free(ns1);
+  networkstatus_vote_free(ns2);
+}
+
+static void
+test_consdiffmgr_diff_pending(void *arg)
+{
+#define N 3
+  (void)arg;
+  char *md_body[N];
+  networkstatus_t *md_ns[N];
+  time_t start = approx_time() - 120;
+  int i;
+  for (i = 0; i < N; ++i) {
+    time_t when = start + i * 30;
+    md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when);
+    md_ns[i] = fake_ns_new(FLAV_MICRODESC, when);
+  }
+
+  MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
+
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2]));
+  /* Make a diff */
+  consdiffmgr_rescan();
+  tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
+
+  /* Look it up.  Is it pending? */
+  consensus_cache_entry_t *ent = NULL;
+  consdiff_status_t diff_status;
+  diff_status = lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]);
+  tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ, diff_status);
+  tt_ptr_op(ent, OP_EQ, NULL);
+
+  /* Add another old consensus.  only one new diff should launch! */
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[0], md_ns[0]));
+  consdiffmgr_rescan();
+  tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue));
+
+  tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
+  mock_cpuworker_handle_replies();
+
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[0], md_body[2]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2]));
+
+ done:
+  UNMOCK(cpuworker_queue_work);
+  for (i = 0; i < N; ++i) {
+    tor_free(md_body[i]);
+    networkstatus_vote_free(md_ns[i]);
+  }
+#undef N
+}
+
+static void
+test_consdiffmgr_cleanup_old(void *arg)
+{
+  (void)arg;
+  config_line_t *labels = NULL;
+  consensus_cache_entry_t *ent = NULL;
+  consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
+
+  /* This item will be will be cleanable because it has a valid-after
+   * time far in the past. */
+  config_line_prepend(&labels, "document-type", "confribble-blarg");
+  config_line_prepend(&labels, "consensus-valid-after",
+                      "1980-10-10T10:10:10");
+  ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3);
+  tt_assert(ent);
+  consensus_cache_entry_decref(ent);
+
+  setup_capture_of_logs(LOG_DEBUG);
+  tt_int_op(1, OP_EQ, consdiffmgr_cleanup());
+  expect_log_msg_containing("Deleting entry because its consensus-valid-"
+                            "after value (1980-10-10T10:10:10) was too old");
+
+ done:
+  teardown_capture_of_logs();
+  config_free_lines(labels);
+}
+
+static void
+test_consdiffmgr_cleanup_bad_valid_after(void *arg)
+{
+  /* This will seem cleanable, but isn't, because its valid-after time is
+   * misformed. */
+
+  (void)arg;
+  config_line_t *labels = NULL;
+  consensus_cache_entry_t *ent = NULL;
+  consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
+
+  config_line_prepend(&labels, "document-type", "consensus");
+  config_line_prepend(&labels, "consensus-valid-after",
+                      "whan that aprille with his shoures soote"); // (~1385?)
+  ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3);
+  tt_assert(ent);
+  consensus_cache_entry_decref(ent);
+
+  setup_capture_of_logs(LOG_DEBUG);
+  tt_int_op(0, OP_EQ, consdiffmgr_cleanup());
+  expect_log_msg_containing("Ignoring entry because its consensus-valid-"
+                            "after value (\"whan that aprille with his "
+                            "shoures soote\") was unparseable");
+
+ done:
+  teardown_capture_of_logs();
+  config_free_lines(labels);
+}
+
+static void
+test_consdiffmgr_cleanup_no_valid_after(void *arg)
+{
+  (void)arg;
+  config_line_t *labels = NULL;
+  consensus_cache_entry_t *ent = NULL;
+  consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
+
+  /* This item will be will be uncleanable because it has no recognized
+   * valid-after. */
+  config_line_prepend(&labels, "document-type", "consensus");
+  config_line_prepend(&labels, "confrooble-voolid-oofter",
+                      "2010-10-10T09:08:07");
+  ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3);
+  tt_assert(ent);
+  consensus_cache_entry_decref(ent);
+
+  setup_capture_of_logs(LOG_DEBUG);
+  tt_int_op(0, OP_EQ, consdiffmgr_cleanup());
+  expect_log_msg_containing("Ignoring entry because it had no consensus-"
+                            "valid-after label");
+
+ done:
+  teardown_capture_of_logs();
+  config_free_lines(labels);
+}
+
+static void
+test_consdiffmgr_cleanup_old_diffs(void *arg)
+{
+  (void)arg;
+#define N 4
+  char *md_body[N];
+  networkstatus_t *md_ns[N];
+  int i;
+  consensus_cache_entry_t *hold_ent = NULL, *ent;
+
+  /* Make sure that the cleanup function removes diffs to the not-most-recent
+   * consensus. */
+
+  MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
+
+  /* Create a bunch of consensus things at 15-second intervals. */
+  time_t start = approx_time() - 120;
+  for (i = 0; i < N; ++i) {
+    time_t when = start + i * 15;
+    md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when);
+    md_ns[i] = fake_ns_new(FLAV_MICRODESC, when);
+  }
+
+  /* add the first 3. */
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[0], md_ns[0]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2]));
+  /* Make diffs. */
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
+  tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue));
+  tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
+  mock_cpuworker_handle_replies();
+  tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
+
+  /* Nothing is deletable now */
+  tt_int_op(0, OP_EQ, consdiffmgr_cleanup());
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[0], md_body[2]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2]));
+
+  tt_int_op(CONSDIFF_AVAILABLE, OP_EQ,
+            lookup_diff_from(&hold_ent, FLAV_MICRODESC, md_body[1]));
+  consensus_cache_entry_incref(hold_ent); // incref, so it is preserved.
+
+  /* Now add an even-more-recent consensus; this should make all previous
+   * diffs deletable, and make delete */
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[3], md_ns[3]));
+  tt_int_op(2 * n_diff_compression_methods() +
+            (n_consensus_compression_methods() - 1) , OP_EQ,
+            consdiffmgr_cleanup());
+
+  tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
+            lookup_diff_from(&ent, FLAV_MICRODESC, md_body[0]));
+  /* This one is marked deletable but still in the hashtable */
+  tt_int_op(CONSDIFF_AVAILABLE, OP_EQ,
+            lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]));
+  tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
+            lookup_diff_from(&ent, FLAV_MICRODESC, md_body[2]));
+
+  /* Everything should be valid at this point */
+  tt_int_op(0, OP_EQ, consdiffmgr_validate());
+
+  /* And if we recan NOW, we'll purge the hashtable of the entries,
+   * and launch attempts to generate new ones */
+  consdiffmgr_rescan();
+  tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ,
+            lookup_diff_from(&ent, FLAV_MICRODESC, md_body[0]));
+  tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ,
+            lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]));
+  tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ,
+            lookup_diff_from(&ent, FLAV_MICRODESC, md_body[2]));
+
+  /* We're still holding on to this, though, so we can still map it! */
+  const uint8_t *t1 = NULL;
+  size_t s;
+  int r = consensus_cache_entry_get_body(hold_ent, &t1, &s);
+  tt_int_op(r, OP_EQ, 0);
+  tt_assert(t1);
+
+ done:
+  for (i = 0; i < N; ++i) {
+    tor_free(md_body[i]);
+    networkstatus_vote_free(md_ns[i]);
+  }
+  consensus_cache_entry_decref(hold_ent);
+  UNMOCK(cpuworker_queue_work);
+#undef N
+}
+
+static void
+test_consdiffmgr_validate(void *arg)
+{
+  (void)arg;
+  config_line_t *lines = NULL;
+  consensus_cache_entry_t *ent = NULL;
+  consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
+  smartlist_t *vals = smartlist_new();
+
+  /* Put these: objects in the cache: one with a good sha3, one with bad sha3,
+   * one with a wrong sha3, and one with no sha3. */
+  config_line_prepend(&lines, "id", "wrong sha3");
+  config_line_prepend(&lines, "sha3-digest",
+                      "F00DF00DF00DF00DF00DF00DF00DF00D"
+                      "F00DF00DF00DF00DF00DF00DF00DF00D");
+  ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
+  consensus_cache_entry_decref(ent);
+  config_free_lines(lines);
+  lines = NULL;
+
+  config_line_prepend(&lines, "id", "bad sha3");
+  config_line_prepend(&lines, "sha3-digest",
+                      "now is the winter of our dicotheque");
+  ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
+  consensus_cache_entry_decref(ent);
+  config_free_lines(lines);
+  lines = NULL;
+
+  config_line_prepend(&lines, "id", "no sha3");
+  ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
+  consensus_cache_entry_decref(ent);
+  config_free_lines(lines);
+  lines = NULL;
+
+  config_line_prepend(&lines, "id", "good sha3");
+  config_line_prepend(&lines, "sha3-digest",
+                      "8d8b1998616cd6b4c4055da8d38728dc"
+                      "93c758d4131a53c7d81aa6337dee1c05");
+  ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
+  consensus_cache_entry_decref(ent);
+  config_free_lines(lines);
+  lines = NULL;
+
+  cdm_reload();
+  cache = cdm_cache_get();
+  tt_int_op(1, OP_EQ, consdiffmgr_validate());
+
+  consensus_cache_find_all(vals, cache, "id", "good sha3");
+  tt_int_op(smartlist_len(vals), OP_EQ, 1);
+  smartlist_clear(vals);
+
+  consensus_cache_find_all(vals, cache, "id", "no sha3");
+  tt_int_op(smartlist_len(vals), OP_EQ, 1);
+  smartlist_clear(vals);
+
+  consensus_cache_find_all(vals, cache, "id", "wrong sha3");
+  tt_int_op(smartlist_len(vals), OP_EQ, 0);
+  consensus_cache_find_all(vals, cache, "id", "bad sha3");
+  tt_int_op(smartlist_len(vals), OP_EQ, 0);
+
+ done:
+  smartlist_free(vals);
+}
+
+#define TEST(name)                                      \
+  { #name, test_consdiffmgr_ ## name , TT_FORK, &setup_diffmgr, NULL }
+
+struct testcase_t consdiffmgr_tests[] = {
+#if 0
+  { "init_failure", test_consdiffmgr_init_failure, TT_FORK, NULL, NULL },
+#endif
+  TEST(sha3_helper),
+  TEST(add),
+  TEST(make_diffs),
+  TEST(diff_rules),
+  TEST(diff_failure),
+  TEST(diff_pending),
+  TEST(cleanup_old),
+  TEST(cleanup_bad_valid_after),
+  TEST(cleanup_no_valid_after),
+  TEST(cleanup_old_diffs),
+  TEST(validate),
+
+  // XXXX Test: non-cacheing cases of replyfn().
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_containers.c b/src/test/test_containers.c
index d8b82e0661..aedd2f7a89 100644
--- a/src/test/test_containers.c
+++ b/src/test/test_containers.c
@@ -1,12 +1,17 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
-#include "or.h"
-#include "fp_pair.h"
-#include "test.h"
+#include "core/or/or.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "feature/dircommon/fp_pair.h"
+#include "test/test.h"
+
+#include "lib/container/bitarray.h"
+#include "lib/container/order.h"
+#include "lib/crypt_ops/digestset.h"
 
 /** Helper: return a tristate based on comparing the strings in *<b>a</b> and
  * *<b>b</b>. */
@@ -501,31 +506,31 @@ test_container_smartlist_pos(void *arg)
   (void) arg;
   smartlist_t *sl = smartlist_new();
 
-  smartlist_add(sl, tor_strdup("This"));
-  smartlist_add(sl, tor_strdup("is"));
-  smartlist_add(sl, tor_strdup("a"));
-  smartlist_add(sl, tor_strdup("test"));
-  smartlist_add(sl, tor_strdup("for"));
-  smartlist_add(sl, tor_strdup("a"));
-  smartlist_add(sl, tor_strdup("function"));
+  smartlist_add_strdup(sl, "This");
+  smartlist_add_strdup(sl, "is");
+  smartlist_add_strdup(sl, "a");
+  smartlist_add_strdup(sl, "test");
+  smartlist_add_strdup(sl, "for");
+  smartlist_add_strdup(sl, "a");
+  smartlist_add_strdup(sl, "function");
 
   /* Test string_pos */
-  tt_int_op(smartlist_string_pos(NULL, "Fred"), ==, -1);
-  tt_int_op(smartlist_string_pos(sl, "Fred"), ==, -1);
-  tt_int_op(smartlist_string_pos(sl, "This"), ==, 0);
-  tt_int_op(smartlist_string_pos(sl, "a"), ==, 2);
-  tt_int_op(smartlist_string_pos(sl, "function"), ==, 6);
+  tt_int_op(smartlist_string_pos(NULL, "Fred"), OP_EQ, -1);
+  tt_int_op(smartlist_string_pos(sl, "Fred"), OP_EQ, -1);
+  tt_int_op(smartlist_string_pos(sl, "This"), OP_EQ, 0);
+  tt_int_op(smartlist_string_pos(sl, "a"), OP_EQ, 2);
+  tt_int_op(smartlist_string_pos(sl, "function"), OP_EQ, 6);
 
   /* Test pos */
-  tt_int_op(smartlist_pos(NULL, "Fred"), ==, -1);
-  tt_int_op(smartlist_pos(sl, "Fred"), ==, -1);
-  tt_int_op(smartlist_pos(sl, "This"), ==, -1);
-  tt_int_op(smartlist_pos(sl, "a"), ==, -1);
-  tt_int_op(smartlist_pos(sl, "function"), ==, -1);
-  tt_int_op(smartlist_pos(sl, smartlist_get(sl,0)), ==, 0);
-  tt_int_op(smartlist_pos(sl, smartlist_get(sl,2)), ==, 2);
-  tt_int_op(smartlist_pos(sl, smartlist_get(sl,5)), ==, 5);
-  tt_int_op(smartlist_pos(sl, smartlist_get(sl,6)), ==, 6);
+  tt_int_op(smartlist_pos(NULL, "Fred"), OP_EQ, -1);
+  tt_int_op(smartlist_pos(sl, "Fred"), OP_EQ, -1);
+  tt_int_op(smartlist_pos(sl, "This"), OP_EQ, -1);
+  tt_int_op(smartlist_pos(sl, "a"), OP_EQ, -1);
+  tt_int_op(smartlist_pos(sl, "function"), OP_EQ, -1);
+  tt_int_op(smartlist_pos(sl, smartlist_get(sl,0)), OP_EQ, 0);
+  tt_int_op(smartlist_pos(sl, smartlist_get(sl,2)), OP_EQ, 2);
+  tt_int_op(smartlist_pos(sl, smartlist_get(sl,5)), OP_EQ, 5);
+  tt_int_op(smartlist_pos(sl, smartlist_get(sl,6)), OP_EQ, 6);
 
  done:
   SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
@@ -639,18 +644,18 @@ test_container_digestset(void *arg)
   }
   set = digestset_new(1000);
   SMARTLIST_FOREACH(included, const char *, cp,
-                    if (digestset_contains(set, cp))
+                    if (digestset_probably_contains(set, cp))
                       ok = 0);
   tt_assert(ok);
   SMARTLIST_FOREACH(included, const char *, cp,
                     digestset_add(set, cp));
   SMARTLIST_FOREACH(included, const char *, cp,
-                    if (!digestset_contains(set, cp))
+                    if (!digestset_probably_contains(set, cp))
                       ok = 0);
   tt_assert(ok);
   for (i = 0; i < 1000; ++i) {
     crypto_rand(d, DIGEST_LEN);
-    if (digestset_contains(set, d))
+    if (digestset_probably_contains(set, d))
       ++false_positives;
   }
   tt_int_op(50, OP_GT, false_positives); /* Should be far lower. */
@@ -681,7 +686,7 @@ test_container_pqueue(void *arg)
 {
   smartlist_t *sl = smartlist_new();
   int (*cmp)(const void *, const void*);
-  const int offset = STRUCT_OFFSET(pq_entry_t, idx);
+  const int offset = offsetof(pq_entry_t, idx);
 #define ENTRY(s) pq_entry_t s = { #s, -1 }
   ENTRY(cows);
   ENTRY(zebras);
@@ -830,7 +835,7 @@ test_container_strmap(void *arg)
   found_keys = smartlist_new();
   while (!strmap_iter_done(iter)) {
     strmap_iter_get(iter,&k,&v);
-    smartlist_add(found_keys, tor_strdup(k));
+    smartlist_add_strdup(found_keys, k);
     tt_ptr_op(v,OP_EQ, strmap_get(map, k));
 
     if (!strcmp(k, "K2")) {
@@ -882,6 +887,46 @@ test_container_strmap(void *arg)
   tor_free(v105);
 }
 
+static void
+test_container_smartlist_remove(void *arg)
+{
+  (void) arg;
+  int array[5];
+  smartlist_t *sl = smartlist_new();
+  int i,j;
+
+  for (j=0; j < 2; ++j)
+    for (i=0; i < 5; ++i)
+      smartlist_add(sl, &array[i]);
+
+  smartlist_remove(sl, &array[0]);
+  smartlist_remove(sl, &array[3]);
+  smartlist_remove(sl, &array[4]);
+  tt_assert(! smartlist_contains(sl, &array[0]));
+  tt_assert(smartlist_contains(sl, &array[1]));
+  tt_assert(smartlist_contains(sl, &array[2]));
+  tt_assert(! smartlist_contains(sl, &array[3]));
+  tt_assert(! smartlist_contains(sl, &array[4]));
+  tt_int_op(smartlist_len(sl), OP_EQ, 4);
+
+  smartlist_clear(sl);
+  for (j=0; j < 2; ++j)
+    for (i=0; i < 5; ++i)
+      smartlist_add(sl, &array[i]);
+
+  smartlist_remove_keeporder(sl, &array[0]);
+  smartlist_remove_keeporder(sl, &array[3]);
+  smartlist_remove_keeporder(sl, &array[4]);
+  tt_int_op(smartlist_len(sl), OP_EQ, 4);
+  tt_ptr_op(smartlist_get(sl, 0), OP_EQ, &array[1]);
+  tt_ptr_op(smartlist_get(sl, 1), OP_EQ, &array[2]);
+  tt_ptr_op(smartlist_get(sl, 2), OP_EQ, &array[1]);
+  tt_ptr_op(smartlist_get(sl, 3), OP_EQ, &array[2]);
+
+ done:
+  smartlist_free(sl);
+}
+
 /** Run unit tests for getting the median of a list. */
 static void
 test_container_order_functions(void *arg)
@@ -950,13 +995,13 @@ test_container_order_functions(void *arg)
   tt_assert(15 == median_time(times, 5));
 
   int32_t int32s[] = { -5, -10, -50, 100 };
-  tt_int_op(-5, ==, median_int32(int32s, 1));
-  tt_int_op(-10, ==, median_int32(int32s, 2));
-  tt_int_op(-10, ==, median_int32(int32s, 3));
-  tt_int_op(-10, ==, median_int32(int32s, 4));
+  tt_int_op(-5, OP_EQ, median_int32(int32s, 1));
+  tt_int_op(-10, OP_EQ, median_int32(int32s, 2));
+  tt_int_op(-10, OP_EQ, median_int32(int32s, 3));
+  tt_int_op(-10, OP_EQ, median_int32(int32s, 4));
 
   long longs[] = { -30, 30, 100, -100, 7 };
-  tt_int_op(7, ==, find_nth_long(longs, 5, 2));
+  tt_int_op(7, OP_EQ, find_nth_long(longs, 5, 2));
 
  done:
   ;
@@ -1066,7 +1111,7 @@ test_container_fp_pair_map(void *arg)
   tt_int_op(fp_pair_map_size(map),OP_EQ, 4);
   fp_pair_map_assert_ok(map);
   fp_pair_map_set(map, &fp5, v104);
-  fp_pair_map_set(map, &fp6, v105);
+  fp_pair_map_set_by_digests(map, fp6.first, fp6.second, v105);
   fp_pair_map_assert_ok(map);
 
   /* Test iterator. */
@@ -1084,7 +1129,8 @@ test_container_fp_pair_map(void *arg)
 
   /* Make sure we removed fp2, but not the others. */
   tt_ptr_op(fp_pair_map_get(map, &fp2),OP_EQ, NULL);
-  tt_ptr_op(fp_pair_map_get(map, &fp5),OP_EQ, v104);
+  tt_ptr_op(fp_pair_map_get_by_digests(map, fp5.first, fp5.second),
+            OP_EQ, v104);
 
   fp_pair_map_assert_ok(map);
   /* Clean up after ourselves. */
@@ -1113,31 +1159,31 @@ test_container_smartlist_most_frequent(void *arg)
   const char *cp;
 
   cp = smartlist_get_most_frequent_string_(sl, &count);
-  tt_int_op(count, ==, 0);
-  tt_ptr_op(cp, ==, NULL);
+  tt_int_op(count, OP_EQ, 0);
+  tt_ptr_op(cp, OP_EQ, NULL);
 
   /* String must be sorted before we call get_most_frequent */
   smartlist_split_string(sl, "abc:def:ghi", ":", 0, 0);
 
   cp = smartlist_get_most_frequent_string_(sl, &count);
-  tt_int_op(count, ==, 1);
-  tt_str_op(cp, ==, "ghi"); /* Ties broken in favor of later element */
+  tt_int_op(count, OP_EQ, 1);
+  tt_str_op(cp, OP_EQ, "ghi"); /* Ties broken in favor of later element */
 
   smartlist_split_string(sl, "def:ghi", ":", 0, 0);
   smartlist_sort_strings(sl);
 
   cp = smartlist_get_most_frequent_string_(sl, &count);
-  tt_int_op(count, ==, 2);
-  tt_ptr_op(cp, !=, NULL);
-  tt_str_op(cp, ==, "ghi"); /* Ties broken in favor of later element */
+  tt_int_op(count, OP_EQ, 2);
+  tt_ptr_op(cp, OP_NE, NULL);
+  tt_str_op(cp, OP_EQ, "ghi"); /* Ties broken in favor of later element */
 
   smartlist_split_string(sl, "def:abc:qwop", ":", 0, 0);
   smartlist_sort_strings(sl);
 
   cp = smartlist_get_most_frequent_string_(sl, &count);
-  tt_int_op(count, ==, 3);
-  tt_ptr_op(cp, !=, NULL);
-  tt_str_op(cp, ==, "def"); /* No tie */
+  tt_int_op(count, OP_EQ, 3);
+  tt_ptr_op(cp, OP_NE, NULL);
+  tt_str_op(cp, OP_EQ, "def"); /* No tie */
 
  done:
   SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
@@ -1166,7 +1212,7 @@ test_container_smartlist_sort_ptrs(void *arg)
     smartlist_shuffle(sl);
     smartlist_sort_pointers(sl);
     for (j = 0; j < ARRAY_LENGTH(arrayptrs); ++j) {
-      tt_ptr_op(smartlist_get(sl, j), ==, arrayptrs[j]);
+      tt_ptr_op(smartlist_get(sl, j), OP_EQ, arrayptrs[j]);
     }
   }
 
@@ -1192,11 +1238,11 @@ test_container_smartlist_strings_eq(void *arg)
   } while (0)
 
   /* Both NULL, so equal */
-  tt_int_op(1, ==, smartlist_strings_eq(NULL, NULL));
+  tt_int_op(1, OP_EQ, smartlist_strings_eq(NULL, NULL));
 
   /* One NULL, not equal. */
-  tt_int_op(0, ==, smartlist_strings_eq(NULL, sl1));
-  tt_int_op(0, ==, smartlist_strings_eq(sl1, NULL));
+  tt_int_op(0, OP_EQ, smartlist_strings_eq(NULL, sl1));
+  tt_int_op(0, OP_EQ, smartlist_strings_eq(sl1, NULL));
 
   /* Both empty, both equal. */
   EQ_SHOULD_SAY("", "", 1);
@@ -1239,6 +1285,7 @@ struct testcase_t container_tests[] = {
   CONTAINER_LEGACY(smartlist_digests),
   CONTAINER_LEGACY(smartlist_join),
   CONTAINER_LEGACY(smartlist_pos),
+  CONTAINER(smartlist_remove, 0),
   CONTAINER(smartlist_ints_eq, 0),
   CONTAINER_LEGACY(bitarray),
   CONTAINER_LEGACY(digestset),
@@ -1252,4 +1299,3 @@ struct testcase_t container_tests[] = {
   CONTAINER(smartlist_strings_eq, 0),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_controller.c b/src/test/test_controller.c
index f19c846144..5b406e159b 100644
--- a/src/test/test_controller.c
+++ b/src/test/test_controller.c
@@ -1,20 +1,108 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
+/* Copyright (c) 2015-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONTROL_PRIVATE
-#include "or.h"
-#include "control.h"
-#include "entrynodes.h"
-#include "networkstatus.h"
-#include "rendservice.h"
-#include "routerlist.h"
-#include "test.h"
+#include "core/or/or.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "feature/client/bridges.h"
+#include "feature/control/control.h"
+#include "feature/client/entrynodes.h"
+#include "feature/hs/hs_common.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/rend/rendservice.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/nodelist.h"
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "lib/net/resolve.h"
+
+#include "feature/control/control_connection_st.h"
+#include "feature/dirclient/download_status_st.h"
+#include "feature/nodelist/microdesc_st.h"
+#include "feature/nodelist/node_st.h"
 
 static void
-test_add_onion_helper_keyarg(void *arg)
+test_add_onion_helper_keyarg_v3(void *arg)
 {
-  crypto_pk_t *pk = NULL;
-  crypto_pk_t *pk2 = NULL;
+  int ret, hs_version;
+  add_onion_secret_key_t pk;
+  char *key_new_blob = NULL;
+  char *err_msg = NULL;
+  const char *key_new_alg = NULL;
+
+  (void) arg;
+
+  memset(&pk, 0, sizeof(pk));
+
+  /* Test explicit ED25519-V3 key generation. */
+  ret = add_onion_helper_keyarg("NEW:ED25519-V3", 0, &key_new_alg,
+                                &key_new_blob, &pk, &hs_version,
+                                &err_msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
+  tt_assert(pk.v3);
+  tt_str_op(key_new_alg, OP_EQ, "ED25519-V3");
+  tt_assert(key_new_blob);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tor_free(pk.v3); pk.v3 = NULL;
+  tor_free(key_new_blob);
+
+  /* Test discarding the private key. */
+  ret = add_onion_helper_keyarg("NEW:ED25519-V3", 1, &key_new_alg,
+                                &key_new_blob, &pk, &hs_version,
+                                &err_msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
+  tt_assert(pk.v3);
+  tt_ptr_op(key_new_alg, OP_EQ, NULL);
+  tt_ptr_op(key_new_blob, OP_EQ, NULL);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tor_free(pk.v3); pk.v3 = NULL;
+  tor_free(key_new_blob);
+
+  /* Test passing a key blob. */
+  {
+    /* The base64 key and hex key are the same. Hex key is 64 bytes long. The
+     * sk has been generated randomly using python3. */
+    const char *base64_sk =
+      "a9bT19PqGC9Y+BmOo1IQvCGjjwxMiaaxEXZ+FKMxpEQW"
+      "6AmSV5roThUGMRCaqQSCnR2jI1vL2QxHORzI4RxMmw==";
+    const char *hex_sk =
+      "\x6b\xd6\xd3\xd7\xd3\xea\x18\x2f\x58\xf8\x19\x8e\xa3\x52\x10\xbc"
+      "\x21\xa3\x8f\x0c\x4c\x89\xa6\xb1\x11\x76\x7e\x14\xa3\x31\xa4\x44"
+      "\x16\xe8\x09\x92\x57\x9a\xe8\x4e\x15\x06\x31\x10\x9a\xa9\x04\x82"
+      "\x9d\x1d\xa3\x23\x5b\xcb\xd9\x0c\x47\x39\x1c\xc8\xe1\x1c\x4c\x9b";
+    char *key_blob = NULL;
+
+    tor_asprintf(&key_blob, "ED25519-V3:%s", base64_sk);
+    tt_assert(key_blob);
+    ret = add_onion_helper_keyarg(key_blob, 1, &key_new_alg,
+                                  &key_new_blob, &pk, &hs_version,
+                                  &err_msg);
+    tor_free(key_blob);
+    tt_int_op(ret, OP_EQ, 0);
+    tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
+    tt_assert(pk.v3);
+    tt_mem_op(pk.v3, OP_EQ, hex_sk, 64);
+    tt_ptr_op(key_new_alg, OP_EQ, NULL);
+    tt_ptr_op(key_new_blob, OP_EQ, NULL);
+    tt_ptr_op(err_msg, OP_EQ, NULL);
+    tor_free(pk.v3); pk.v3 = NULL;
+    tor_free(key_new_blob);
+  }
+
+ done:
+  tor_free(pk.v3);
+  tor_free(key_new_blob);
+  tor_free(err_msg);
+}
+
+static void
+test_add_onion_helper_keyarg_v2(void *arg)
+{
+  int ret, hs_version;
+  add_onion_secret_key_t pk;
+  crypto_pk_t *pk1 = NULL;
   const char *key_new_alg = NULL;
   char *key_new_blob = NULL;
   char *err_msg = NULL;
@@ -23,83 +111,100 @@ test_add_onion_helper_keyarg(void *arg)
 
   (void) arg;
 
+  memset(&pk, 0, sizeof(pk));
+
   /* Test explicit RSA1024 key generation. */
-  pk = add_onion_helper_keyarg("NEW:RSA1024", 0, &key_new_alg, &key_new_blob,
-                               &err_msg);
-  tt_assert(pk);
+  ret = add_onion_helper_keyarg("NEW:RSA1024", 0, &key_new_alg, &key_new_blob,
+                                &pk, &hs_version, &err_msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
+  tt_assert(pk.v2);
   tt_str_op(key_new_alg, OP_EQ, "RSA1024");
   tt_assert(key_new_blob);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
 
   /* Test "BEST" key generation (Assumes BEST = RSA1024). */
-  crypto_pk_free(pk);
+  crypto_pk_free(pk.v2); pk.v2 = NULL;
   tor_free(key_new_blob);
-  pk = add_onion_helper_keyarg("NEW:BEST", 0, &key_new_alg, &key_new_blob,
-                               &err_msg);
-  tt_assert(pk);
+  ret = add_onion_helper_keyarg("NEW:BEST", 0, &key_new_alg, &key_new_blob,
+                                &pk, &hs_version, &err_msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
+  tt_assert(pk.v2);
   tt_str_op(key_new_alg, OP_EQ, "RSA1024");
   tt_assert(key_new_blob);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
 
   /* Test discarding the private key. */
-  crypto_pk_free(pk);
+  crypto_pk_free(pk.v2); pk.v2 = NULL;
   tor_free(key_new_blob);
-  pk = add_onion_helper_keyarg("NEW:BEST", 1, &key_new_alg, &key_new_blob,
-                               &err_msg);
-  tt_assert(pk);
-  tt_assert(!key_new_alg);
-  tt_assert(!key_new_blob);
-  tt_assert(!err_msg);
+  ret = add_onion_helper_keyarg("NEW:BEST", 1, &key_new_alg, &key_new_blob,
+                               &pk, &hs_version, &err_msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
+  tt_assert(pk.v2);
+  tt_ptr_op(key_new_alg, OP_EQ, NULL);
+  tt_ptr_op(key_new_blob, OP_EQ, NULL);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
 
   /* Test generating a invalid key type. */
-  crypto_pk_free(pk);
-  pk = add_onion_helper_keyarg("NEW:RSA512", 0, &key_new_alg, &key_new_blob,
-                               &err_msg);
-  tt_assert(!pk);
-  tt_assert(!key_new_alg);
-  tt_assert(!key_new_blob);
+  crypto_pk_free(pk.v2); pk.v2 = NULL;
+  ret = add_onion_helper_keyarg("NEW:RSA512", 0, &key_new_alg, &key_new_blob,
+                               &pk, &hs_version, &err_msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
+  tt_assert(!pk.v2);
+  tt_ptr_op(key_new_alg, OP_EQ, NULL);
+  tt_ptr_op(key_new_blob, OP_EQ, NULL);
   tt_assert(err_msg);
 
   /* Test loading a RSA1024 key. */
   tor_free(err_msg);
-  pk = pk_generate(0);
-  tt_int_op(0, OP_EQ, crypto_pk_base64_encode(pk, &encoded));
+  pk1 = pk_generate(0);
+  tt_int_op(0, OP_EQ, crypto_pk_base64_encode_private(pk1, &encoded));
   tor_asprintf(&arg_str, "RSA1024:%s", encoded);
-  pk2 = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
-                                &err_msg);
-  tt_assert(pk2);
-  tt_assert(!key_new_alg);
-  tt_assert(!key_new_blob);
-  tt_assert(!err_msg);
-  tt_assert(crypto_pk_cmp_keys(pk, pk2) == 0);
+  ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
+                                &pk, &hs_version, &err_msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
+  tt_assert(pk.v2);
+  tt_ptr_op(key_new_alg, OP_EQ, NULL);
+  tt_ptr_op(key_new_blob, OP_EQ, NULL);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_int_op(crypto_pk_cmp_keys(pk1, pk.v2), OP_EQ, 0);
 
   /* Test loading a invalid key type. */
   tor_free(arg_str);
-  crypto_pk_free(pk); pk = NULL;
+  crypto_pk_free(pk1); pk1 = NULL;
+  crypto_pk_free(pk.v2); pk.v2 = NULL;
   tor_asprintf(&arg_str, "RSA512:%s", encoded);
-  pk = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
-                               &err_msg);
-  tt_assert(!pk);
-  tt_assert(!key_new_alg);
-  tt_assert(!key_new_blob);
+  ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
+                                &pk, &hs_version, &err_msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
+  tt_assert(!pk.v2);
+  tt_ptr_op(key_new_alg, OP_EQ, NULL);
+  tt_ptr_op(key_new_blob, OP_EQ, NULL);
   tt_assert(err_msg);
 
   /* Test loading a invalid key. */
   tor_free(arg_str);
-  crypto_pk_free(pk); pk = NULL;
+  crypto_pk_free(pk.v2); pk.v2 = NULL;
   tor_free(err_msg);
   encoded[strlen(encoded)/2] = '\0';
   tor_asprintf(&arg_str, "RSA1024:%s", encoded);
-  pk = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
-                               &err_msg);
-  tt_assert(!pk);
-  tt_assert(!key_new_alg);
-  tt_assert(!key_new_blob);
+  ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
+                               &pk, &hs_version, &err_msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
+  tt_assert(!pk.v2);
+  tt_ptr_op(key_new_alg, OP_EQ, NULL);
+  tt_ptr_op(key_new_blob, OP_EQ, NULL);
   tt_assert(err_msg);
 
  done:
-  crypto_pk_free(pk);
-  crypto_pk_free(pk2);
+  crypto_pk_free(pk1);
+  crypto_pk_free(pk.v2);
   tor_free(key_new_blob);
   tor_free(err_msg);
   tor_free(encoded);
@@ -107,6 +212,45 @@ test_add_onion_helper_keyarg(void *arg)
 }
 
 static void
+test_getinfo_helper_onion(void *arg)
+{
+  (void)arg;
+  control_connection_t dummy;
+  /* Get results out */
+  char *answer = NULL;
+  const char *errmsg = NULL;
+  char *service_id = NULL;
+  int rt = 0;
+
+  dummy.ephemeral_onion_services = NULL;
+
+  /* successfully get an empty answer */
+  rt = getinfo_helper_onions(&dummy, "onions/current", &answer, &errmsg);
+  tt_int_op(rt, OP_EQ, 0);
+  tt_str_op(answer, OP_EQ, "");
+  tor_free(answer);
+
+  /* successfully get an empty answer */
+  rt = getinfo_helper_onions(&dummy, "onions/detached", &answer, &errmsg);
+  tt_int_op(rt, OP_EQ, 0);
+  tt_str_op(answer, OP_EQ, "");
+  tor_free(answer);
+
+  /* get an answer for one onion service */
+  service_id = tor_strdup("dummy_onion_id");
+  dummy.ephemeral_onion_services = smartlist_new();
+  smartlist_add(dummy.ephemeral_onion_services, service_id);
+  rt = getinfo_helper_onions(&dummy, "onions/current", &answer, &errmsg);
+  tt_int_op(rt, OP_EQ, 0);
+  tt_str_op(answer, OP_EQ, "dummy_onion_id");
+
+ done:
+  tor_free(answer);
+  tor_free(service_id);
+  smartlist_free(dummy.ephemeral_onion_services);
+}
+
+static void
 test_rend_service_parse_port_config(void *arg)
 {
   const char *sep = ",";
@@ -118,25 +262,25 @@ test_rend_service_parse_port_config(void *arg)
   /* Test "VIRTPORT" only. */
   cfg = rend_service_parse_port_config("80", sep, &err_msg);
   tt_assert(cfg);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
 
   /* Test "VIRTPORT,TARGET" (Target is port). */
   rend_service_port_config_free(cfg);
   cfg = rend_service_parse_port_config("80,8080", sep, &err_msg);
   tt_assert(cfg);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
 
   /* Test "VIRTPORT,TARGET" (Target is IPv4:port). */
   rend_service_port_config_free(cfg);
   cfg = rend_service_parse_port_config("80,192.0.2.1:8080", sep, &err_msg);
   tt_assert(cfg);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
 
   /* Test "VIRTPORT,TARGET" (Target is IPv6:port). */
   rend_service_port_config_free(cfg);
   cfg = rend_service_parse_port_config("80,[2001:db8::1]:8080", sep, &err_msg);
   tt_assert(cfg);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   rend_service_port_config_free(cfg);
   cfg = NULL;
 
@@ -145,13 +289,13 @@ test_rend_service_parse_port_config(void *arg)
   /* Test empty config. */
   rend_service_port_config_free(cfg);
   cfg = rend_service_parse_port_config("", sep, &err_msg);
-  tt_assert(!cfg);
+  tt_ptr_op(cfg, OP_EQ, NULL);
   tt_assert(err_msg);
 
   /* Test invalid port. */
   tor_free(err_msg);
   cfg = rend_service_parse_port_config("90001", sep, &err_msg);
-  tt_assert(!cfg);
+  tt_ptr_op(cfg, OP_EQ, NULL);
   tt_assert(err_msg);
   tor_free(err_msg);
 
@@ -163,7 +307,7 @@ test_rend_service_parse_port_config(void *arg)
   cfg = rend_service_parse_port_config("100 unix:\"/tmp/foo bar\"",
                                        " ", &err_msg);
   tt_assert(cfg);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   rend_service_port_config_free(cfg);
   cfg = NULL;
 
@@ -172,22 +316,24 @@ test_rend_service_parse_port_config(void *arg)
   cfg = rend_service_parse_port_config("100 unix:\"/tmp/foo bar\"",
                                        " ", &err_msg);
   tt_assert(cfg);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   rend_service_port_config_free(cfg);
   cfg = NULL;
 
   /* quoted unix port, missing end quote */
   cfg = rend_service_parse_port_config("100 unix:\"/tmp/foo bar",
                                        " ", &err_msg);
-  tt_assert(!cfg);
+  tt_ptr_op(cfg, OP_EQ, NULL);
   tt_str_op(err_msg, OP_EQ, "Couldn't process address <unix:\"/tmp/foo bar> "
             "from hidden service configuration");
   tor_free(err_msg);
 
   /* bogus IP address */
-  cfg = rend_service_parse_port_config("100 1.2.3.4.5:9000",
+  MOCK(tor_addr_lookup, mock_tor_addr_lookup__fail_on_bad_addrs);
+  cfg = rend_service_parse_port_config("100 foo!!.example.com:9000",
                                        " ", &err_msg);
-  tt_assert(!cfg);
+  UNMOCK(tor_addr_lookup);
+  tt_ptr_op(cfg, OP_EQ, NULL);
   tt_str_op(err_msg, OP_EQ, "Unparseable address in hidden service port "
             "configuration.");
   tor_free(err_msg);
@@ -195,11 +341,18 @@ test_rend_service_parse_port_config(void *arg)
   /* bogus port port */
   cfg = rend_service_parse_port_config("100 99999",
                                        " ", &err_msg);
-  tt_assert(!cfg);
+  tt_ptr_op(cfg, OP_EQ, NULL);
   tt_str_op(err_msg, OP_EQ, "Unparseable or out-of-range port \"99999\" "
             "in hidden service port configuration.");
   tor_free(err_msg);
 
+  /* Wrong target address and port separation */
+  cfg = rend_service_parse_port_config("80,127.0.0.1 1234", sep,
+                                       &err_msg);
+  tt_ptr_op(cfg, OP_EQ, NULL);
+  tt_assert(err_msg);
+  tor_free(err_msg);
+
  done:
   rend_service_port_config_free(cfg);
   tor_free(err_msg);
@@ -218,7 +371,7 @@ test_add_onion_helper_clientauth(void *arg)
   client = add_onion_helper_clientauth("alice", &created, &err_msg);
   tt_assert(client);
   tt_assert(created);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   rend_authorized_client_free(client);
 
   /* Test "ClientName:Blob" */
@@ -226,26 +379,26 @@ test_add_onion_helper_clientauth(void *arg)
                                        &created, &err_msg);
   tt_assert(client);
   tt_assert(!created);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   rend_authorized_client_free(client);
 
   /* Test invalid client names */
   client = add_onion_helper_clientauth("no*asterisks*allowed", &created,
                                        &err_msg);
-  tt_assert(!client);
+  tt_ptr_op(client, OP_EQ, NULL);
   tt_assert(err_msg);
   tor_free(err_msg);
 
   /* Test invalid auth cookie */
   client = add_onion_helper_clientauth("alice:12345", &created, &err_msg);
-  tt_assert(!client);
+  tt_ptr_op(client, OP_EQ, NULL);
   tt_assert(err_msg);
   tor_free(err_msg);
 
   /* Test invalid syntax */
   client = add_onion_helper_clientauth(":475hGBHPlq7Mc0cRZitK/B", &created,
                                        &err_msg);
-  tt_assert(!client);
+  tt_ptr_op(client, OP_EQ, NULL);
   tt_assert(err_msg);
   tor_free(err_msg);
 
@@ -258,7 +411,7 @@ test_add_onion_helper_clientauth(void *arg)
 
 static const download_status_t dl_status_default =
   { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER,
-    DL_SCHED_INCREMENT_FAILURE, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 };
+    DL_SCHED_INCREMENT_FAILURE, 0, 0 };
 static download_status_t ns_dl_status[N_CONSENSUS_FLAVORS];
 static download_status_t ns_dl_status_bootstrap[N_CONSENSUS_FLAVORS];
 static download_status_t ns_dl_status_running[N_CONSENSUS_FLAVORS];
@@ -269,7 +422,7 @@ static download_status_t ns_dl_status_running[N_CONSENSUS_FLAVORS];
  */
 static const download_status_t dls_sample_1 =
   { 1467163900, 0, 0, DL_SCHED_GENERIC, DL_WANT_ANY_DIRSERVER,
-    DL_SCHED_INCREMENT_FAILURE, DL_SCHED_DETERMINISTIC, 0, 0 };
+    DL_SCHED_INCREMENT_FAILURE, 0, 0 };
 static const char * dls_sample_1_str =
     "next-attempt-at 2016-06-29 01:31:40\n"
     "n-download-failures 0\n"
@@ -277,10 +430,12 @@ static const char * dls_sample_1_str =
     "schedule DL_SCHED_GENERIC\n"
     "want-authority DL_WANT_ANY_DIRSERVER\n"
     "increment-on DL_SCHED_INCREMENT_FAILURE\n"
-    "backoff DL_SCHED_DETERMINISTIC\n";
+    "backoff DL_SCHED_RANDOM_EXPONENTIAL\n"
+    "last-backoff-position 0\n"
+    "last-delay-used 0\n";
 static const download_status_t dls_sample_2 =
   { 1467164400, 1, 2, DL_SCHED_CONSENSUS, DL_WANT_AUTHORITY,
-    DL_SCHED_INCREMENT_FAILURE, DL_SCHED_DETERMINISTIC, 0, 0 };
+    DL_SCHED_INCREMENT_FAILURE, 0, 0 };
 static const char * dls_sample_2_str =
     "next-attempt-at 2016-06-29 01:40:00\n"
     "n-download-failures 1\n"
@@ -288,10 +443,12 @@ static const char * dls_sample_2_str =
     "schedule DL_SCHED_CONSENSUS\n"
     "want-authority DL_WANT_AUTHORITY\n"
     "increment-on DL_SCHED_INCREMENT_FAILURE\n"
-    "backoff DL_SCHED_DETERMINISTIC\n";
+    "backoff DL_SCHED_RANDOM_EXPONENTIAL\n"
+    "last-backoff-position 0\n"
+    "last-delay-used 0\n";
 static const download_status_t dls_sample_3 =
   { 1467154400, 12, 25, DL_SCHED_BRIDGE, DL_WANT_ANY_DIRSERVER,
-    DL_SCHED_INCREMENT_ATTEMPT, DL_SCHED_DETERMINISTIC, 0, 0 };
+    DL_SCHED_INCREMENT_ATTEMPT, 0, 0 };
 static const char * dls_sample_3_str =
     "next-attempt-at 2016-06-28 22:53:20\n"
     "n-download-failures 12\n"
@@ -299,10 +456,12 @@ static const char * dls_sample_3_str =
     "schedule DL_SCHED_BRIDGE\n"
     "want-authority DL_WANT_ANY_DIRSERVER\n"
     "increment-on DL_SCHED_INCREMENT_ATTEMPT\n"
-    "backoff DL_SCHED_DETERMINISTIC\n";
+    "backoff DL_SCHED_RANDOM_EXPONENTIAL\n"
+    "last-backoff-position 0\n"
+    "last-delay-used 0\n";
 static const download_status_t dls_sample_4 =
   { 1467166600, 3, 0, DL_SCHED_GENERIC, DL_WANT_ANY_DIRSERVER,
-    DL_SCHED_INCREMENT_FAILURE, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 };
+    DL_SCHED_INCREMENT_FAILURE, 0, 0 };
 static const char * dls_sample_4_str =
     "next-attempt-at 2016-06-29 02:16:40\n"
     "n-download-failures 3\n"
@@ -315,7 +474,7 @@ static const char * dls_sample_4_str =
     "last-delay-used 0\n";
 static const download_status_t dls_sample_5 =
   { 1467164600, 3, 7, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER,
-    DL_SCHED_INCREMENT_FAILURE, DL_SCHED_RANDOM_EXPONENTIAL, 1, 2112, };
+    DL_SCHED_INCREMENT_FAILURE, 1, 2112, };
 static const char * dls_sample_5_str =
     "next-attempt-at 2016-06-29 01:43:20\n"
     "n-download-failures 3\n"
@@ -328,7 +487,7 @@ static const char * dls_sample_5_str =
     "last-delay-used 2112\n";
 static const download_status_t dls_sample_6 =
   { 1467164200, 4, 9, DL_SCHED_CONSENSUS, DL_WANT_AUTHORITY,
-    DL_SCHED_INCREMENT_ATTEMPT, DL_SCHED_RANDOM_EXPONENTIAL, 3, 432 };
+    DL_SCHED_INCREMENT_ATTEMPT, 3, 432 };
 static const char * dls_sample_6_str =
     "next-attempt-at 2016-06-29 01:36:40\n"
     "n-download-failures 4\n"
@@ -501,7 +660,7 @@ cert_dl_status_def_for_auth_mock(const char *digest)
   download_status_t *dl = NULL;
   char digest_str[HEX_DIGEST_LEN+1];
 
-  tt_assert(digest != NULL);
+  tt_ptr_op(digest, OP_NE, NULL);
   base16_encode(digest_str, HEX_DIGEST_LEN + 1,
                 digest, DIGEST_LEN);
   digest_str[HEX_DIGEST_LEN] = '\0';
@@ -525,7 +684,7 @@ cert_dl_status_sks_for_auth_id_mock(const char *digest)
   char *tmp;
   int len;
 
-  tt_assert(digest != NULL);
+  tt_ptr_op(digest, OP_NE, NULL);
   base16_encode(digest_str, HEX_DIGEST_LEN + 1,
                 digest, DIGEST_LEN);
   digest_str[HEX_DIGEST_LEN] = '\0';
@@ -579,11 +738,11 @@ cert_dl_status_fp_sk_mock(const char *fp_digest, const char *sk_digest)
    * dl status we want.
    */
 
-  tt_assert(fp_digest != NULL);
+  tt_ptr_op(fp_digest, OP_NE, NULL);
   base16_encode(fp_digest_str, HEX_DIGEST_LEN + 1,
                 fp_digest, DIGEST_LEN);
   fp_digest_str[HEX_DIGEST_LEN] = '\0';
-  tt_assert(sk_digest != NULL);
+  tt_ptr_op(sk_digest, OP_NE, NULL);
   base16_encode(sk_digest_str, HEX_DIGEST_LEN + 1,
                 sk_digest, DIGEST_LEN);
   sk_digest_str[HEX_DIGEST_LEN] = '\0';
@@ -663,7 +822,7 @@ descbr_get_dl_by_digest_mock(const char *digest)
   char digest_str[HEX_DIGEST_LEN+1];
 
   if (!disable_descbr) {
-    tt_assert(digest != NULL);
+    tt_ptr_op(digest, OP_NE, NULL);
     base16_encode(digest_str, HEX_DIGEST_LEN + 1,
                   digest, DIGEST_LEN);
     digest_str[HEX_DIGEST_LEN] = '\0';
@@ -730,7 +889,7 @@ test_download_status_consensus(void *arg)
 
   /* Check that the unknown prefix case works; no mocks needed yet */
   getinfo_helper_downloads(&dummy, "downloads/foo", &answer, &errmsg);
-  tt_assert(answer == NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
   tt_str_op(errmsg, OP_EQ, "Unknown download status query");
 
   setup_ns_mocks();
@@ -745,8 +904,8 @@ test_download_status_consensus(void *arg)
          sizeof(download_status_t));
   getinfo_helper_downloads(&dummy, "downloads/networkstatus/ns",
                            &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_1_str);
   tor_free(answer);
   errmsg = NULL;
@@ -756,8 +915,8 @@ test_download_status_consensus(void *arg)
          sizeof(download_status_t));
   getinfo_helper_downloads(&dummy, "downloads/networkstatus/microdesc",
                            &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_2_str);
   tor_free(answer);
   errmsg = NULL;
@@ -767,8 +926,8 @@ test_download_status_consensus(void *arg)
          sizeof(download_status_t));
   getinfo_helper_downloads(&dummy, "downloads/networkstatus/ns/bootstrap",
                            &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_3_str);
   tor_free(answer);
   errmsg = NULL;
@@ -779,8 +938,8 @@ test_download_status_consensus(void *arg)
   getinfo_helper_downloads(&dummy,
                            "downloads/networkstatus/microdesc/bootstrap",
                            &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_4_str);
   tor_free(answer);
   errmsg = NULL;
@@ -791,8 +950,8 @@ test_download_status_consensus(void *arg)
   getinfo_helper_downloads(&dummy,
                            "downloads/networkstatus/ns/running",
                            &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_5_str);
   tor_free(answer);
   errmsg = NULL;
@@ -803,8 +962,8 @@ test_download_status_consensus(void *arg)
   getinfo_helper_downloads(&dummy,
                            "downloads/networkstatus/microdesc/running",
                            &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_6_str);
   tor_free(answer);
   errmsg = NULL;
@@ -812,8 +971,8 @@ test_download_status_consensus(void *arg)
   /* Now check the error case */
   getinfo_helper_downloads(&dummy, "downloads/networkstatus/foo",
                            &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "Unknown flavor");
   errmsg = NULL;
 
@@ -847,8 +1006,8 @@ test_download_status_cert(void *arg)
   getinfo_helper_downloads(&dummy,
                            "downloads/cert/fps",
                            &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, auth_id_digest_expected_list);
   tor_free(answer);
   errmsg = NULL;
@@ -857,10 +1016,10 @@ test_download_status_cert(void *arg)
   memcpy(&auth_def_cert_download_status_1, &dls_sample_1,
          sizeof(download_status_t));
   tor_asprintf(&question, "downloads/cert/fp/%s", auth_id_digest_1_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_1_str);
   tor_free(question);
   tor_free(answer);
@@ -870,10 +1029,10 @@ test_download_status_cert(void *arg)
   memcpy(&auth_def_cert_download_status_2, &dls_sample_2,
          sizeof(download_status_t));
   tor_asprintf(&question, "downloads/cert/fp/%s", auth_id_digest_2_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_2_str);
   tor_free(question);
   tor_free(answer);
@@ -881,10 +1040,10 @@ test_download_status_cert(void *arg)
 
   /* Case 4 - list of signing key digests for 1st auth id */
   tor_asprintf(&question, "downloads/cert/fp/%s/sks", auth_id_digest_1_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, auth_1_sk_digest_expected_list);
   tor_free(question);
   tor_free(answer);
@@ -892,10 +1051,10 @@ test_download_status_cert(void *arg)
 
   /* Case 5 - list of signing key digests for 2nd auth id */
   tor_asprintf(&question, "downloads/cert/fp/%s/sks", auth_id_digest_2_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, auth_2_sk_digest_expected_list);
   tor_free(question);
   tor_free(answer);
@@ -906,10 +1065,10 @@ test_download_status_cert(void *arg)
          sizeof(download_status_t));
   tor_asprintf(&question, "downloads/cert/fp/%s/%s",
                auth_id_digest_1_str, auth_1_sk_1_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_3_str);
   tor_free(question);
   tor_free(answer);
@@ -920,10 +1079,10 @@ test_download_status_cert(void *arg)
          sizeof(download_status_t));
   tor_asprintf(&question, "downloads/cert/fp/%s/%s",
                auth_id_digest_1_str, auth_1_sk_2_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_4_str);
   tor_free(question);
   tor_free(answer);
@@ -934,10 +1093,10 @@ test_download_status_cert(void *arg)
          sizeof(download_status_t));
   tor_asprintf(&question, "downloads/cert/fp/%s/%s",
                auth_id_digest_2_str, auth_2_sk_1_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_5_str);
   tor_free(question);
   tor_free(answer);
@@ -948,10 +1107,10 @@ test_download_status_cert(void *arg)
          sizeof(download_status_t));
   tor_asprintf(&question, "downloads/cert/fp/%s/%s",
                auth_id_digest_2_str, auth_2_sk_2_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_6_str);
   tor_free(question);
   tor_free(answer);
@@ -962,8 +1121,8 @@ test_download_status_cert(void *arg)
   /* Case 1 - query is garbage after downloads/cert/ part */
   getinfo_helper_downloads(&dummy, "downloads/cert/blahdeblah",
                            &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "Unknown certificate download status query");
   errmsg = NULL;
 
@@ -973,8 +1132,8 @@ test_download_status_cert(void *arg)
    */
   getinfo_helper_downloads(&dummy, "downloads/cert/fp/2B1D36D32B2942406",
                            &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "That didn't look like a digest");
   errmsg = NULL;
 
@@ -985,8 +1144,8 @@ test_download_status_cert(void *arg)
   getinfo_helper_downloads(&dummy,
       "downloads/cert/fp/82F52AF55D250115FE44D3GC81D49643241D56A1",
       &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "That didn't look like a digest");
   errmsg = NULL;
 
@@ -997,8 +1156,8 @@ test_download_status_cert(void *arg)
   getinfo_helper_downloads(&dummy,
       "downloads/cert/fp/AC4F23B5745BDD2A77997B85B1FD85D05C2E0F61",
       &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ,
       "Failed to get download status for this authority identity digest");
   errmsg = NULL;
@@ -1010,8 +1169,8 @@ test_download_status_cert(void *arg)
   getinfo_helper_downloads(&dummy,
       "downloads/cert/fp/82F52AF55D250115FE44D3GC81D49643241D56A1/blah",
       &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "That didn't look like an identity digest");
   errmsg = NULL;
 
@@ -1022,8 +1181,8 @@ test_download_status_cert(void *arg)
   getinfo_helper_downloads(&dummy,
       "downloads/cert/fp/82F52AF55D25/blah",
       &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "That didn't look like an identity digest");
   errmsg = NULL;
 
@@ -1034,8 +1193,8 @@ test_download_status_cert(void *arg)
   getinfo_helper_downloads(&dummy,
       "downloads/cert/fp/AC4F23B5745BDD2A77997B85B1FD85D05C2E0F61/sks",
       &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ,
       "Failed to get list of signing key digests for this authority "
       "identity digest");
@@ -1049,8 +1208,8 @@ test_download_status_cert(void *arg)
       "downloads/cert/fp/AC4F23B5745BDD2A77997B85B1FD85D05C2E0F61/"
       "82F52AF55D250115FE44D3GC81D49643241D56A1",
       &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "That didn't look like a signing key digest");
   errmsg = NULL;
 
@@ -1062,8 +1221,8 @@ test_download_status_cert(void *arg)
       "downloads/cert/fp/AC4F23B5745BDD2A77997B85B1FD85D05C2E0F61/"
       "82F52AF55D250115FE44D",
       &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "That didn't look like a signing key digest");
   errmsg = NULL;
 
@@ -1075,8 +1234,8 @@ test_download_status_cert(void *arg)
       "downloads/cert/fp/C6B05DF332F74DB9A13498EE3BBC7AA2F69FCB45/"
       "3A214FC21AE25B012C2ECCB5F4EC8A3602D0545D",
       &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ,
       "Failed to get download status for this identity/"
       "signing key digest pair");
@@ -1090,8 +1249,8 @@ test_download_status_cert(void *arg)
       "downloads/cert/fp/63CDD326DFEF0CA020BDD3FEB45A3286FE13A061/"
       "3A214FC21AE25B012C2ECCB5F4EC8A3602D0545D",
       &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ,
       "Failed to get download status for this identity/"
       "signing key digest pair");
@@ -1105,8 +1264,8 @@ test_download_status_cert(void *arg)
       "downloads/cert/fp/63CDD326DFEF0CA020BDD3FEB45A3286FE13A061/"
       "9451B8F1B10952384EB58B5F230C0BB701626C9B",
       &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ,
       "Failed to get download status for this identity/"
       "signing key digest pair");
@@ -1142,8 +1301,8 @@ test_download_status_desc(void *arg)
   getinfo_helper_downloads(&dummy,
                            "downloads/desc/descs",
                            &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, descbr_expected_list);
   tor_free(answer);
   errmsg = NULL;
@@ -1152,10 +1311,10 @@ test_download_status_desc(void *arg)
   memcpy(&descbr_digest_1_dl, &dls_sample_1,
          sizeof(download_status_t));
   tor_asprintf(&question, "downloads/desc/%s", descbr_digest_1_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_1_str);
   tor_free(question);
   tor_free(answer);
@@ -1165,10 +1324,10 @@ test_download_status_desc(void *arg)
   memcpy(&descbr_digest_2_dl, &dls_sample_2,
          sizeof(download_status_t));
   tor_asprintf(&question, "downloads/desc/%s", descbr_digest_2_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_2_str);
   tor_free(question);
   tor_free(answer);
@@ -1179,8 +1338,8 @@ test_download_status_desc(void *arg)
   /* Case 1 - non-digest-length garbage after downloads/desc */
   getinfo_helper_downloads(&dummy, "downloads/desc/blahdeblah",
                            &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "Unknown router descriptor download status query");
   errmsg = NULL;
 
@@ -1189,8 +1348,8 @@ test_download_status_desc(void *arg)
     &dummy,
     "downloads/desc/774EC52FD9A5B80A6FACZE536616E8022E3470AG",
     &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "That didn't look like a digest");
   errmsg = NULL;
 
@@ -1199,8 +1358,8 @@ test_download_status_desc(void *arg)
     &dummy,
     "downloads/desc/B05B46135B0B2C04EBE1DD6A6AE4B12D7CD2226A",
     &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "No such descriptor digest found");
   errmsg = NULL;
 
@@ -1209,8 +1368,8 @@ test_download_status_desc(void *arg)
   getinfo_helper_downloads(&dummy,
                            "downloads/desc/descs",
                            &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ,
             "We don't seem to have a networkstatus-flavored consensus");
   errmsg = NULL;
@@ -1246,8 +1405,8 @@ test_download_status_bridge(void *arg)
   getinfo_helper_downloads(&dummy,
                            "downloads/bridge/bridges",
                            &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, descbr_expected_list);
   tor_free(answer);
   errmsg = NULL;
@@ -1256,10 +1415,10 @@ test_download_status_bridge(void *arg)
   memcpy(&descbr_digest_1_dl, &dls_sample_3,
          sizeof(download_status_t));
   tor_asprintf(&question, "downloads/bridge/%s", descbr_digest_1_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_3_str);
   tor_free(question);
   tor_free(answer);
@@ -1269,10 +1428,10 @@ test_download_status_bridge(void *arg)
   memcpy(&descbr_digest_2_dl, &dls_sample_4,
          sizeof(download_status_t));
   tor_asprintf(&question, "downloads/bridge/%s", descbr_digest_2_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_4_str);
   tor_free(question);
   tor_free(answer);
@@ -1283,8 +1442,8 @@ test_download_status_bridge(void *arg)
   /* Case 1 - non-digest-length garbage after downloads/bridge */
   getinfo_helper_downloads(&dummy, "downloads/bridge/blahdeblah",
                            &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "Unknown bridge descriptor download status query");
   errmsg = NULL;
 
@@ -1293,8 +1452,8 @@ test_download_status_bridge(void *arg)
     &dummy,
     "downloads/bridge/774EC52FD9A5B80A6FACZE536616E8022E3470AG",
     &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "That didn't look like a digest");
   errmsg = NULL;
 
@@ -1303,8 +1462,8 @@ test_download_status_bridge(void *arg)
     &dummy,
     "downloads/bridge/B05B46135B0B2C04EBE1DD6A6AE4B12D7CD2226A",
     &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "No such bridge identity digest found");
   errmsg = NULL;
 
@@ -1313,8 +1472,8 @@ test_download_status_bridge(void *arg)
   getinfo_helper_downloads(&dummy,
                            "downloads/bridge/bridges",
                            &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "We don't seem to be using bridges");
   errmsg = NULL;
   disable_descbr = 0;
@@ -1326,8 +1485,141 @@ test_download_status_bridge(void *arg)
   return;
 }
 
+/** Set timeval to a mock date and time. This is necessary
+ * to make tor_gettimeofday() mockable. */
+static void
+mock_tor_gettimeofday(struct timeval *timeval)
+{
+  timeval->tv_sec = 1523405073;
+  timeval->tv_usec = 271645;
+}
+
+static void
+test_current_time(void *arg)
+{
+  /* We just need one of these to pass, it doesn't matter what's in it */
+  control_connection_t dummy;
+  /* Get results out */
+  char *answer = NULL;
+  const char *errmsg = NULL;
+
+  (void)arg;
+
+  /* We need these for storing the (mock) time. */
+  MOCK(tor_gettimeofday, mock_tor_gettimeofday);
+  struct timeval now;
+  tor_gettimeofday(&now);
+  char timebuf[ISO_TIME_LEN+1];
+
+  /* Case 1 - local time */
+  format_local_iso_time_nospace(timebuf, (time_t)now.tv_sec);
+  getinfo_helper_current_time(&dummy,
+                              "current-time/local",
+                              &answer, &errmsg);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
+  tt_str_op(answer, OP_EQ, timebuf);
+  tor_free(answer);
+  errmsg = NULL;
+
+  /* Case 2 - UTC time */
+  format_iso_time_nospace(timebuf, (time_t)now.tv_sec);
+  getinfo_helper_current_time(&dummy,
+                              "current-time/utc",
+                              &answer, &errmsg);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
+  tt_str_op(answer, OP_EQ, timebuf);
+  tor_free(answer);
+  errmsg = NULL;
+
+ done:
+  UNMOCK(tor_gettimeofday);
+  tor_free(answer);
+
+  return;
+}
+
+static size_t n_nodelist_get_list = 0;
+static smartlist_t *nodes = NULL;
+
+static smartlist_t *
+mock_nodelist_get_list(void)
+{
+  n_nodelist_get_list++;
+  tor_assert(nodes);
+
+  return nodes;
+}
+
+static void
+test_getinfo_md_all(void *arg)
+{
+  char *answer = NULL;
+  const char *errmsg = NULL;
+  int retval = 0;
+
+  (void)arg;
+
+  node_t *node1 = tor_malloc(sizeof(node_t));
+  memset(node1, 0, sizeof(node_t));
+  node1->md = tor_malloc(sizeof(microdesc_t));
+  memset(node1->md, 0, sizeof(microdesc_t));
+  node1->md->body = tor_strdup("md1\n");
+  node1->md->bodylen = 4;
+
+  node_t *node2 = tor_malloc(sizeof(node_t));
+  memset(node2, 0, sizeof(node_t));
+  node2->md = tor_malloc(sizeof(microdesc_t));
+  memset(node2->md, 0, sizeof(microdesc_t));
+  node2->md->body = tor_strdup("md2\n");
+  node2->md->bodylen = 4;
+
+  MOCK(nodelist_get_list, mock_nodelist_get_list);
+
+  nodes = smartlist_new();
+
+  retval = getinfo_helper_dir(NULL, "md/all", &answer, &errmsg);
+
+  tt_int_op(n_nodelist_get_list, OP_EQ, 1);
+  tt_int_op(retval, OP_EQ, 0);
+  tt_assert(answer != NULL);
+  tt_assert(errmsg == NULL);
+  tt_str_op(answer, OP_EQ, "");
+
+  tor_free(answer);
+
+  smartlist_add(nodes, node1);
+  smartlist_add(nodes, node2);
+
+  retval = getinfo_helper_dir(NULL, "md/all", &answer, &errmsg);
+
+  tt_int_op(n_nodelist_get_list, OP_EQ, 2);
+  tt_int_op(retval, OP_EQ, 0);
+  tt_assert(answer != NULL);
+  tt_assert(errmsg == NULL);
+
+  tt_str_op(answer, OP_EQ, "md1\nmd2\n");
+
+ done:
+  UNMOCK(nodelist_get_list);
+  tor_free(node1->md->body);
+  tor_free(node1->md);
+  tor_free(node1);
+  tor_free(node2->md->body);
+  tor_free(node2->md);
+  tor_free(node2);
+  tor_free(answer);
+  smartlist_free(nodes);
+  return;
+}
+
 struct testcase_t controller_tests[] = {
-  { "add_onion_helper_keyarg", test_add_onion_helper_keyarg, 0, NULL, NULL },
+  { "add_onion_helper_keyarg_v2", test_add_onion_helper_keyarg_v2, 0,
+    NULL, NULL },
+  { "add_onion_helper_keyarg_v3", test_add_onion_helper_keyarg_v3, 0,
+    NULL, NULL },
+  { "getinfo_helper_onion", test_getinfo_helper_onion, 0, NULL, NULL },
   { "rend_service_parse_port_config", test_rend_service_parse_port_config, 0,
     NULL, NULL },
   { "add_onion_helper_clientauth", test_add_onion_helper_clientauth, 0, NULL,
@@ -1338,6 +1630,7 @@ struct testcase_t controller_tests[] = {
     NULL },
   { "download_status_desc", test_download_status_desc, 0, NULL, NULL },
   { "download_status_bridge", test_download_status_bridge, 0, NULL, NULL },
+  { "current_time", test_current_time, 0, NULL, NULL },
+  { "getinfo_md_all", test_getinfo_md_all, 0, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c
index 11e1e3dc8f..70d36e53d4 100644
--- a/src/test/test_controller_events.c
+++ b/src/test/test_controller_events.c
@@ -1,88 +1,19 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONNECTION_PRIVATE
 #define TOR_CHANNEL_INTERNAL_
 #define CONTROL_PRIVATE
-#include "or.h"
-#include "channel.h"
-#include "channeltls.h"
-#include "connection.h"
-#include "control.h"
-#include "test.h"
+#include "core/or/or.h"
+#include "core/or/channel.h"
+#include "core/or/channeltls.h"
+#include "core/or/circuitlist.h"
+#include "core/mainloop/connection.h"
+#include "feature/control/control.h"
+#include "test/test.h"
 
-static void
-help_test_bucket_note_empty(uint32_t expected_msec_since_midnight,
-                            int tokens_before, size_t tokens_removed,
-                            uint32_t msec_since_epoch)
-{
-  uint32_t timestamp_var = 0;
-  struct timeval tvnow;
-  tvnow.tv_sec = msec_since_epoch / 1000;
-  tvnow.tv_usec = (msec_since_epoch % 1000) * 1000;
-  connection_buckets_note_empty_ts(&timestamp_var, tokens_before,
-                                   tokens_removed, &tvnow);
-  tt_int_op(expected_msec_since_midnight, OP_EQ, timestamp_var);
-
- done:
-  ;
-}
-
-static void
-test_cntev_bucket_note_empty(void *arg)
-{
-  (void)arg;
-
-  /* Two cases with nothing to note, because bucket was empty before;
-   * 86442200 == 1970-01-02 00:00:42.200000 */
-  help_test_bucket_note_empty(0, 0, 0, 86442200);
-  help_test_bucket_note_empty(0, -100, 100, 86442200);
-
-  /* Nothing to note, because bucket has not been emptied. */
-  help_test_bucket_note_empty(0, 101, 100, 86442200);
-
-  /* Bucket was emptied, note 42200 msec since midnight. */
-  help_test_bucket_note_empty(42200, 101, 101, 86442200);
-  help_test_bucket_note_empty(42200, 101, 102, 86442200);
-}
-
-static void
-test_cntev_bucket_millis_empty(void *arg)
-{
-  struct timeval tvnow;
-  (void)arg;
-
-  /* 1970-01-02 00:00:42.200000 */
-  tvnow.tv_sec = 86400 + 42;
-  tvnow.tv_usec = 200000;
-
-  /* Bucket has not been refilled. */
-  tt_int_op(0, OP_EQ, bucket_millis_empty(0, 42120, 0, 100, &tvnow));
-  tt_int_op(0, OP_EQ, bucket_millis_empty(-10, 42120, -10, 100, &tvnow));
-
-  /* Bucket was not empty. */
-  tt_int_op(0, OP_EQ, bucket_millis_empty(10, 42120, 20, 100, &tvnow));
-
-  /* Bucket has been emptied 80 msec ago and has just been refilled. */
-  tt_int_op(80, OP_EQ, bucket_millis_empty(-20, 42120, -10, 100, &tvnow));
-  tt_int_op(80, OP_EQ, bucket_millis_empty(-10, 42120, 0, 100, &tvnow));
-  tt_int_op(80, OP_EQ, bucket_millis_empty(0, 42120, 10, 100, &tvnow));
-
-  /* Bucket has been emptied 180 msec ago, last refill was 100 msec ago
-   * which was insufficient to make it positive, so cap msec at 100. */
-  tt_int_op(100, OP_EQ, bucket_millis_empty(0, 42020, 1, 100, &tvnow));
-
-  /* 1970-01-02 00:00:00:050000 */
-  tvnow.tv_sec = 86400;
-  tvnow.tv_usec = 50000;
-
-  /* Last emptied 30 msec before midnight, tvnow is 50 msec after
-   * midnight, that's 80 msec in total. */
-  tt_int_op(80, OP_EQ, bucket_millis_empty(0, 86400000 - 30, 1, 100, &tvnow));
-
- done:
-  ;
-}
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
 
 static void
 add_testing_cell_stats_entry(circuit_t *circ, uint8_t command,
@@ -391,16 +322,81 @@ test_cntev_event_mask(void *arg)
   ;
 }
 
+static char *saved_event_str = NULL;
+
+static void
+mock_queue_control_event_string(uint16_t event, char *msg)
+{
+  (void)event;
+
+  tor_free(saved_event_str);
+  saved_event_str = msg;
+}
+
+/* Helper macro for checking bootstrap control event strings */
+#define assert_bootmsg(s)                                               \
+  tt_ptr_op(strstr(saved_event_str, "650 STATUS_CLIENT NOTICE "         \
+                   "BOOTSTRAP PROGRESS=" s), OP_EQ, saved_event_str)
+
+/* Test deferral of directory bootstrap messages (requesting_descriptors) */
+static void
+test_cntev_dirboot_defer_desc(void *arg)
+{
+  (void)arg;
+
+  MOCK(queue_control_event_string, mock_queue_control_event_string);
+  control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT));
+  control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0);
+  assert_bootmsg("0 TAG=starting");
+  /* This event should get deferred */
+  control_event_boot_dir(BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, 0);
+  assert_bootmsg("0 TAG=starting");
+  control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0);
+  assert_bootmsg("5 TAG=conn_dir");
+  control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0);
+  assert_bootmsg("10 TAG=handshake_dir");
+  /* The deferred event should appear */
+  control_event_boot_first_orconn();
+  assert_bootmsg("45 TAG=requesting_descriptors");
+ done:
+  tor_free(saved_event_str);
+  UNMOCK(queue_control_event_string);
+}
+
+/* Test deferral of directory bootstrap messages (conn_or) */
+static void
+test_cntev_dirboot_defer_orconn(void *arg)
+{
+  (void)arg;
+
+  MOCK(queue_control_event_string, mock_queue_control_event_string);
+  control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT));
+  control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0);
+  assert_bootmsg("0 TAG=starting");
+  /* This event should get deferred */
+  control_event_boot_dir(BOOTSTRAP_STATUS_CONN_OR, 0);
+  assert_bootmsg("0 TAG=starting");
+  control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0);
+  assert_bootmsg("5 TAG=conn_dir");
+  control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0);
+  assert_bootmsg("10 TAG=handshake_dir");
+  /* The deferred event should appear */
+  control_event_boot_first_orconn();
+  assert_bootmsg("80 TAG=conn_or");
+ done:
+  tor_free(saved_event_str);
+  UNMOCK(queue_control_event_string);
+}
+
 #define TEST(name, flags)                                               \
   { #name, test_cntev_ ## name, flags, 0, NULL }
 
 struct testcase_t controller_event_tests[] = {
-  TEST(bucket_note_empty, TT_FORK),
-  TEST(bucket_millis_empty, TT_FORK),
   TEST(sum_up_cell_stats, TT_FORK),
   TEST(append_cell_stats, TT_FORK),
   TEST(format_cell_stats, TT_FORK),
   TEST(event_mask, TT_FORK),
+  TEST(dirboot_defer_desc, TT_FORK),
+  TEST(dirboot_defer_orconn, TT_FORK),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c
index 8fd9ca7671..fa79f4cc47 100644
--- a/src/test/test_crypto.c
+++ b/src/test/test_crypto.c
@@ -1,22 +1,38 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 #define CRYPTO_CURVE25519_PRIVATE
-#define CRYPTO_PRIVATE
-#include "or.h"
-#include "test.h"
-#include "aes.h"
-#include "util.h"
+#define CRYPTO_RAND_PRIVATE
+#include "core/or/or.h"
+#include "test/test.h"
+#include "lib/crypt_ops/aes.h"
 #include "siphash.h"
-#include "crypto_curve25519.h"
-#include "crypto_ed25519.h"
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_hkdf.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_init.h"
 #include "ed25519_vectors.inc"
+#include "test/log_test_helpers.h"
 
-#include <openssl/evp.h>
-#include <openssl/rand.h>
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#if defined(ENABLE_OPENSSL)
+#include "lib/crypt_ops/compat_openssl.h"
+DISABLE_GCC_WARNING(redundant-decls)
+#include <openssl/dh.h>
+ENABLE_GCC_WARNING(redundant-decls)
+#endif
 
 /** Run unit tests for Diffie-Hellman functionality. */
 static void
@@ -25,38 +41,45 @@ test_crypto_dh(void *arg)
   crypto_dh_t *dh1 = crypto_dh_new(DH_TYPE_CIRCUIT);
   crypto_dh_t *dh1_dup = NULL;
   crypto_dh_t *dh2 = crypto_dh_new(DH_TYPE_CIRCUIT);
-  char p1[DH_BYTES];
-  char p2[DH_BYTES];
-  char s1[DH_BYTES];
-  char s2[DH_BYTES];
+  char p1[DH1024_KEY_LEN];
+  char p2[DH1024_KEY_LEN];
+  char s1[DH1024_KEY_LEN];
+  char s2[DH1024_KEY_LEN];
   ssize_t s1len, s2len;
+#ifdef ENABLE_OPENSSL
+  crypto_dh_t *dh3 = NULL;
+  DH *dh4 = NULL;
+  BIGNUM *pubkey_tmp = NULL;
+#endif
 
   (void)arg;
-  tt_int_op(crypto_dh_get_bytes(dh1),OP_EQ, DH_BYTES);
-  tt_int_op(crypto_dh_get_bytes(dh2),OP_EQ, DH_BYTES);
+  tt_int_op(crypto_dh_get_bytes(dh1),OP_EQ, DH1024_KEY_LEN);
+  tt_int_op(crypto_dh_get_bytes(dh2),OP_EQ, DH1024_KEY_LEN);
 
-  memset(p1, 0, DH_BYTES);
-  memset(p2, 0, DH_BYTES);
-  tt_mem_op(p1,OP_EQ, p2, DH_BYTES);
+  memset(p1, 0, DH1024_KEY_LEN);
+  memset(p2, 0, DH1024_KEY_LEN);
+  tt_mem_op(p1,OP_EQ, p2, DH1024_KEY_LEN);
 
   tt_int_op(-1, OP_EQ, crypto_dh_get_public(dh1, p1, 6)); /* too short */
 
-  tt_assert(! crypto_dh_get_public(dh1, p1, DH_BYTES));
-  tt_mem_op(p1,OP_NE, p2, DH_BYTES);
-  tt_assert(! crypto_dh_get_public(dh2, p2, DH_BYTES));
-  tt_mem_op(p1,OP_NE, p2, DH_BYTES);
+  tt_assert(! crypto_dh_get_public(dh1, p1, DH1024_KEY_LEN));
+  tt_mem_op(p1,OP_NE, p2, DH1024_KEY_LEN);
+  tt_assert(! crypto_dh_get_public(dh2, p2, DH1024_KEY_LEN));
+  tt_mem_op(p1,OP_NE, p2, DH1024_KEY_LEN);
 
-  memset(s1, 0, DH_BYTES);
-  memset(s2, 0xFF, DH_BYTES);
-  s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p2, DH_BYTES, s1, 50);
-  s2len = crypto_dh_compute_secret(LOG_WARN, dh2, p1, DH_BYTES, s2, 50);
+  memset(s1, 0, DH1024_KEY_LEN);
+  memset(s2, 0xFF, DH1024_KEY_LEN);
+  s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p2, DH1024_KEY_LEN, s1, 50);
+  s2len = crypto_dh_compute_secret(LOG_WARN, dh2, p1, DH1024_KEY_LEN, s2, 50);
   tt_assert(s1len > 0);
   tt_int_op(s1len,OP_EQ, s2len);
   tt_mem_op(s1,OP_EQ, s2, s1len);
 
   /* test dh_dup; make sure it works the same. */
   dh1_dup = crypto_dh_dup(dh1);
-  s1len = crypto_dh_compute_secret(LOG_WARN, dh1_dup, p2, DH_BYTES, s1, 50);
+  s1len = crypto_dh_compute_secret(LOG_WARN, dh1_dup, p2, DH1024_KEY_LEN,
+                                   s1, 50);
+  tt_i64_op(s1len, OP_GE, 0);
   tt_mem_op(s1,OP_EQ, s2, s1len);
 
   {
@@ -69,18 +92,24 @@ test_crypto_dh(void *arg)
     s1len = crypto_dh_compute_secret(LOG_WARN, dh1, "\x00", 1, s1, 50);
     tt_int_op(-1, OP_EQ, s1len);
 
-    memset(p1, 0, DH_BYTES); /* 0 with padding. */
-    s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50);
+    memset(p1, 0, DH1024_KEY_LEN); /* 0 with padding. */
+    s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH1024_KEY_LEN,
+                                     s1, 50);
     tt_int_op(-1, OP_EQ, s1len);
 
-    p1[DH_BYTES-1] = 1; /* 1 with padding*/
-    s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50);
+    p1[DH1024_KEY_LEN-1] = 1; /* 1 with padding*/
+    s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH1024_KEY_LEN,
+                                     s1, 50);
     tt_int_op(-1, OP_EQ, s1len);
 
     /* 2 is okay, though weird. */
     s1len = crypto_dh_compute_secret(LOG_WARN, dh1, "\x02", 1, s1, 50);
     tt_int_op(50, OP_EQ, s1len);
 
+    /* 2 a second time is still okay, though weird. */
+    s1len = crypto_dh_compute_secret(LOG_WARN, dh1, "\x02", 1, s1, 50);
+    tt_int_op(50, OP_EQ, s1len);
+
     const char P[] =
       "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E08"
       "8A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B"
@@ -91,15 +120,18 @@ test_crypto_dh(void *arg)
     /* p-1, p, and so on are not okay. */
     base16_decode(p1, sizeof(p1), P, strlen(P));
 
-    s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50);
+    s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH1024_KEY_LEN,
+                                     s1, 50);
     tt_int_op(-1, OP_EQ, s1len);
 
-    p1[DH_BYTES-1] = 0xFE; /* p-1 */
-    s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50);
+    p1[DH1024_KEY_LEN-1] = 0xFE; /* p-1 */
+    s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH1024_KEY_LEN,
+                                     s1, 50);
     tt_int_op(-1, OP_EQ, s1len);
 
-    p1[DH_BYTES-1] = 0xFD; /* p-2 works fine */
-    s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50);
+    p1[DH1024_KEY_LEN-1] = 0xFD; /* p-2 works fine */
+    s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH1024_KEY_LEN,
+                                     s1, 50);
     tt_int_op(50, OP_EQ, s1len);
 
     const char P_plus_one[] =
@@ -111,51 +143,103 @@ test_crypto_dh(void *arg)
 
     base16_decode(p1, sizeof(p1), P_plus_one, strlen(P_plus_one));
 
-    s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50);
+    s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH1024_KEY_LEN,
+                                     s1, 50);
     tt_int_op(-1, OP_EQ, s1len);
 
-    p1[DH_BYTES-1] = 0x01; /* p+2 */
-    s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50);
+    p1[DH1024_KEY_LEN-1] = 0x01; /* p+2 */
+    s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH1024_KEY_LEN,
+                                     s1, 50);
     tt_int_op(-1, OP_EQ, s1len);
 
-    p1[DH_BYTES-1] = 0xff; /* p+256 */
-    s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50);
+    p1[DH1024_KEY_LEN-1] = 0xff; /* p+256 */
+    s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH1024_KEY_LEN,
+                                     s1, 50);
     tt_int_op(-1, OP_EQ, s1len);
 
-    memset(p1, 0xff, DH_BYTES), /* 2^1024-1 */
-    s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50);
+    memset(p1, 0xff, DH1024_KEY_LEN), /* 2^1024-1 */
+    s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH1024_KEY_LEN,
+                                     s1, 50);
     tt_int_op(-1, OP_EQ, s1len);
   }
 
   {
     /* provoke an error in the openssl DH_compute_key function; make sure we
      * survive. */
-    tt_assert(! crypto_dh_get_public(dh1, p1, DH_BYTES));
+    tt_assert(! crypto_dh_get_public(dh1, p1, DH1024_KEY_LEN));
 
     crypto_dh_free(dh2);
     dh2= crypto_dh_new(DH_TYPE_CIRCUIT); /* no private key set */
     s1len = crypto_dh_compute_secret(LOG_WARN, dh2,
-                                     p1, DH_BYTES,
+                                     p1, DH1024_KEY_LEN,
                                      s1, 50);
     tt_int_op(s1len, OP_EQ, -1);
   }
 
+#if defined(ENABLE_OPENSSL)
+  {
+    /* Make sure that our crypto library can handshake with openssl. */
+    dh3 = crypto_dh_new(DH_TYPE_TLS);
+    tt_assert(!crypto_dh_get_public(dh3, p1, DH1024_KEY_LEN));
+
+    dh4 = crypto_dh_new_openssl_tls();
+    tt_assert(DH_generate_key(dh4));
+    const BIGNUM *pk=NULL;
+#ifdef OPENSSL_1_1_API
+    const BIGNUM *sk=NULL;
+    DH_get0_key(dh4, &pk, &sk);
+#else
+    pk = dh4->pub_key;
+#endif
+    tt_assert(pk);
+    tt_int_op(BN_num_bytes(pk), OP_LE, DH1024_KEY_LEN);
+    tt_int_op(BN_num_bytes(pk), OP_GT, 0);
+    memset(p2, 0, sizeof(p2));
+    /* right-pad. */
+    BN_bn2bin(pk, (unsigned char *)(p2+DH1024_KEY_LEN-BN_num_bytes(pk)));
+
+    s1len = crypto_dh_handshake(LOG_WARN, dh3, p2, DH1024_KEY_LEN,
+                                (unsigned char *)s1, sizeof(s1));
+    pubkey_tmp = BN_bin2bn((unsigned char *)p1, DH1024_KEY_LEN, NULL);
+    s2len = DH_compute_key((unsigned char *)s2, pubkey_tmp, dh4);
+
+    tt_int_op(s1len, OP_EQ, s2len);
+    tt_int_op(s1len, OP_GT, 0);
+    tt_mem_op(s1, OP_EQ, s2, s1len);
+  }
+#endif
+
  done:
   crypto_dh_free(dh1);
   crypto_dh_free(dh2);
   crypto_dh_free(dh1_dup);
+#ifdef ENABLE_OPENSSL
+  crypto_dh_free(dh3);
+  if (dh4)
+    DH_free(dh4);
+  if (pubkey_tmp)
+    BN_free(pubkey_tmp);
+#endif
 }
 
 static void
 test_crypto_openssl_version(void *arg)
 {
   (void)arg;
+#ifdef ENABLE_NSS
+  tt_skip();
+#else
   const char *version = crypto_openssl_get_version_str();
   const char *h_version = crypto_openssl_get_header_version_str();
   tt_assert(version);
   tt_assert(h_version);
-  tt_assert(!strcmpstart(version, h_version)); /* "-fips" suffix, etc */
-  tt_assert(!strstr(version, "OpenSSL"));
+  if (strcmpstart(version, h_version)) { /* "-fips" suffix, etc */
+    TT_DIE(("OpenSSL library version %s did not begin with header version %s.",
+            version, h_version));
+  }
+  if (strstr(version, "OpenSSL")) {
+    TT_DIE(("assertion failed: !strstr(\"%s\", \"OpenSSL\")", version));
+  }
   int a=-1,b=-1,c=-1;
   if (!strcmpstart(version, "LibreSSL") || !strcmpstart(version, "BoringSSL"))
     return;
@@ -164,6 +248,7 @@ test_crypto_openssl_version(void *arg)
   tt_int_op(a, OP_GE, 0);
   tt_int_op(b, OP_GE, 0);
   tt_int_op(c, OP_GE, 0);
+#endif
 
  done:
   ;
@@ -192,10 +277,10 @@ test_crypto_rng(void *arg)
     j = crypto_rand_int(100);
     if (j < 0 || j >= 100)
       allok = 0;
-    big = crypto_rand_uint64(U64_LITERAL(1)<<40);
-    if (big >= (U64_LITERAL(1)<<40))
+    big = crypto_rand_uint64(UINT64_C(1)<<40);
+    if (big >= (UINT64_C(1)<<40))
       allok = 0;
-    big = crypto_rand_uint64(U64_LITERAL(5));
+    big = crypto_rand_uint64(UINT64_C(5));
     if (big >= 5)
       allok = 0;
     d = crypto_rand_double();
@@ -331,38 +416,6 @@ test_crypto_rng_strongest(void *arg)
 #undef N
 }
 
-/* Test for rectifying openssl RAND engine. */
-static void
-test_crypto_rng_engine(void *arg)
-{
-  (void)arg;
-  RAND_METHOD dummy_method;
-  memset(&dummy_method, 0, sizeof(dummy_method));
-
-  /* We should be a no-op if we're already on RAND_OpenSSL */
-  tt_int_op(0, ==, crypto_force_rand_ssleay());
-  tt_assert(RAND_get_rand_method() == RAND_OpenSSL());
-
-  /* We should correct the method if it's a dummy. */
-  RAND_set_rand_method(&dummy_method);
-#ifdef LIBRESSL_VERSION_NUMBER
-  /* On libressl, you can't override the RNG. */
-  tt_assert(RAND_get_rand_method() == RAND_OpenSSL());
-  tt_int_op(0, ==, crypto_force_rand_ssleay());
-#else
-  tt_assert(RAND_get_rand_method() == &dummy_method);
-  tt_int_op(1, ==, crypto_force_rand_ssleay());
-#endif
-  tt_assert(RAND_get_rand_method() == RAND_OpenSSL());
-
-  /* Make sure we aren't calling dummy_method */
-  crypto_rand((void *) &dummy_method, sizeof(dummy_method));
-  crypto_rand((void *) &dummy_method, sizeof(dummy_method));
-
- done:
-  ;
-}
-
 /** Run unit tests for our AES128 functionality */
 static void
 test_crypto_aes128(void *arg)
@@ -1135,6 +1188,54 @@ test_crypto_sha3_xof(void *arg)
   tor_free(mem_op_hex_tmp);
 }
 
+/* Test our MAC-SHA3 function. There are not actually any MAC-SHA3 test
+ * vectors out there for our H(len(k) || k || m) construction. Hence what we
+ * are gonna do is test our crypto_mac_sha3_256() function against manually
+ * doing H(len(k) || k||m).  If in the future the Keccak group decides to
+ * standarize an MAC construction and make test vectors, we should
+ * incorporate them here. */
+static void
+test_crypto_mac_sha3(void *arg)
+{
+  const char msg[] = "i am in a library somewhere using my computer";
+  const char key[] = "i'm from the past talking to the future.";
+
+  uint8_t hmac_test[DIGEST256_LEN];
+  char hmac_manual[DIGEST256_LEN];
+
+  (void) arg;
+
+  /* First let's use our nice HMAC-SHA3 function */
+  crypto_mac_sha3_256(hmac_test, sizeof(hmac_test),
+                      (uint8_t *) key, strlen(key),
+                      (uint8_t *) msg, strlen(msg));
+
+  /* Now let's try a manual H(len(k) || k || m) construction */
+  {
+    char *key_msg_concat = NULL, *all = NULL;
+    int result;
+    const uint64_t key_len_netorder = tor_htonll(strlen(key));
+    size_t all_len;
+
+    tor_asprintf(&key_msg_concat, "%s%s", key, msg);
+    all_len = sizeof(key_len_netorder) + strlen(key_msg_concat);
+    all = tor_malloc_zero(all_len);
+    memcpy(all, &key_len_netorder, sizeof(key_len_netorder));
+    memcpy(all + sizeof(key_len_netorder), key_msg_concat,
+           strlen(key_msg_concat));
+
+    result = crypto_digest256(hmac_manual, all, all_len, DIGEST_SHA3_256);
+    tor_free(key_msg_concat);
+    tor_free(all);
+    tt_int_op(result, OP_EQ, 0);
+  }
+
+  /* Now compare the two results */
+  tt_mem_op(hmac_test, OP_EQ, hmac_manual, DIGEST256_LEN);
+
+ done: ;
+}
+
 /** Run unit tests for our public key crypto functions */
 static void
 test_crypto_pk(void *arg)
@@ -1195,12 +1296,12 @@ test_crypto_pk(void *arg)
   tt_assert(! crypto_pk_write_private_key_to_filename(pk1,
                                                         get_fname("pkey1")));
   /* failing case for read: can't read. */
-  tt_assert(crypto_pk_read_private_key_from_filename(pk2,
-                                                   get_fname("xyzzy")) < 0);
+  tt_int_op(crypto_pk_read_private_key_from_filename(pk2, get_fname("xyzzy")),
+            OP_LT, 0);
   write_str_to_file(get_fname("xyzzy"), "foobar", 6);
   /* Failing case for read: no key. */
-  tt_assert(crypto_pk_read_private_key_from_filename(pk2,
-                                                   get_fname("xyzzy")) < 0);
+  tt_int_op(crypto_pk_read_private_key_from_filename(pk2, get_fname("xyzzy")),
+            OP_LT, 0);
   tt_assert(! crypto_pk_read_private_key_from_filename(pk2,
                                                          get_fname("pkey1")));
   tt_int_op(15,OP_EQ,
@@ -1232,17 +1333,17 @@ test_crypto_pk(void *arg)
   i = crypto_pk_asn1_encode(pk1, data1, 1024);
   tt_int_op(i, OP_GT, 0);
   pk2 = crypto_pk_asn1_decode(data1, i);
-  tt_assert(crypto_pk_cmp_keys(pk1,pk2) == 0);
+  tt_int_op(crypto_pk_cmp_keys(pk1, pk2), OP_EQ, 0);
 
   /* Try with hybrid encryption wrappers. */
   crypto_rand(data1, 1024);
   for (i = 85; i < 140; ++i) {
     memset(data2,0,1024);
     memset(data3,0,1024);
-    len = crypto_pk_public_hybrid_encrypt(pk1,data2,sizeof(data2),
+    len = crypto_pk_obsolete_public_hybrid_encrypt(pk1,data2,sizeof(data2),
                                           data1,i,PK_PKCS1_OAEP_PADDING,0);
     tt_int_op(len, OP_GE, 0);
-    len = crypto_pk_private_hybrid_decrypt(pk1,data3,sizeof(data3),
+    len = crypto_pk_obsolete_private_hybrid_decrypt(pk1,data3,sizeof(data3),
                                            data2,len,PK_PKCS1_OAEP_PADDING,1);
     tt_int_op(len,OP_EQ, i);
     tt_mem_op(data1,OP_EQ, data3,i);
@@ -1251,9 +1352,9 @@ test_crypto_pk(void *arg)
   /* Try copy_full */
   crypto_pk_free(pk2);
   pk2 = crypto_pk_copy_full(pk1);
-  tt_assert(pk2 != NULL);
+  tt_ptr_op(pk2, OP_NE, NULL);
   tt_ptr_op(pk1, OP_NE, pk2);
-  tt_assert(crypto_pk_cmp_keys(pk1,pk2) == 0);
+  tt_int_op(crypto_pk_cmp_keys(pk1, pk2), OP_EQ, 0);
 
  done:
   if (pk1)
@@ -1325,23 +1426,23 @@ test_crypto_pk_base64(void *arg)
   /* Test Base64 encoding a key. */
   pk1 = pk_generate(0);
   tt_assert(pk1);
-  tt_int_op(0, OP_EQ, crypto_pk_base64_encode(pk1, &encoded));
+  tt_int_op(0, OP_EQ, crypto_pk_base64_encode_private(pk1, &encoded));
   tt_assert(encoded);
 
   /* Test decoding a valid key. */
-  pk2 = crypto_pk_base64_decode(encoded, strlen(encoded));
+  pk2 = crypto_pk_base64_decode_private(encoded, strlen(encoded));
   tt_assert(pk2);
-  tt_assert(crypto_pk_cmp_keys(pk1,pk2) == 0);
+  tt_int_op(crypto_pk_cmp_keys(pk1, pk2), OP_EQ, 0);
   crypto_pk_free(pk2);
 
   /* Test decoding a invalid key (not Base64). */
   static const char *invalid_b64 = "The key is in another castle!";
-  pk2 = crypto_pk_base64_decode(invalid_b64, strlen(invalid_b64));
-  tt_assert(!pk2);
+  pk2 = crypto_pk_base64_decode_private(invalid_b64, strlen(invalid_b64));
+  tt_ptr_op(pk2, OP_EQ, NULL);
 
   /* Test decoding a truncated Base64 blob. */
-  pk2 = crypto_pk_base64_decode(encoded, strlen(encoded)/2);
-  tt_assert(!pk2);
+  pk2 = crypto_pk_base64_decode_private(encoded, strlen(encoded)/2);
+  tt_ptr_op(pk2, OP_EQ, NULL);
 
  done:
   crypto_pk_free(pk1);
@@ -1389,6 +1490,58 @@ test_crypto_pk_pem_encrypted(void *arg)
  done:
   crypto_pk_free(pk);
 }
+
+static void
+test_crypto_pk_invalid_private_key(void *arg)
+{
+  (void)arg;
+  /* Here is a simple invalid private key: it was produced by making
+   * a regular private key, and then adding 2 to the modulus. */
+  const char pem[] =
+    "-----BEGIN RSA PRIVATE KEY-----\n"
+    "MIIEpQIBAAKCAQEAskRyZrs+YAukvBmZlgo6/rCxyKF2xyUk073ap+2CgRUnSfGG\n"
+    "mflHlzqVq7tpH50DafpS+fFAbaEaNV/ac20QG0rUZi38HTB4qURWOu6n0Bws6E4l\n"
+    "UX/AkvDlWnuYH0pHHi2c3DGNFjwoJpjKuUTk+cRffVR8X3Kjr62SUDUaBNW0Kecz\n"
+    "3SYLbmgmZI16dFZ+g9sNM3znXZbhvb33WwPqpZSSPs37cPgF7eS6mAw/gUMx6zfE\n"
+    "HRmUnOQSzUdS05rvc/hsiCLhiIZ8hgfkD07XnTT1Ds8DwE55k7BUWY2wvwWCNLsH\n"
+    "qtqAxTr615XdkMxVkYgImpqPybarpfNYhFqkOwIDAQABAoIBACPC3VxEdbfYvhxJ\n"
+    "2mih9sG++nswAN7kUaX0cRe86rAwaShJPmJHApiQ1ROVTfpciiHJaLnhLraPWe2Z\n"
+    "I/6Bw3hmI4O399p3Lc1u+wlpdNqnvE6B1rSptx0DHE9xecvVH70rE0uM2Su7t6Y+\n"
+    "gnR2IKUGQs2mlCilm7aTUEWs0WJkkl4CG1dyxItuOSdNBjOEzXimJyiB10jEBFsp\n"
+    "SZeCF2FZ7AJbck5CVC42+oTsiDbZrHTHOn7v26rFGdONeHD1wOI1v7JwHFpCB923\n"
+    "aEHBzsPbMeq7DWG1rjzCYpcXHhTDBDBWSia4SEhyr2Nl7m7qxWWWwR+x4dqAj3rD\n"
+    "HeTmos0CgYEA6uf1CLpjPpOs5IaW1DQI8dJA/xFEAC/6GVgq4nFOGHZrm8G3L5o+\n"
+    "qvtQNMpDs2naWuZpqROFqv24o01DykHygR72GlPIY6uvmmf5tvJLoGnbFUay33L4\n"
+    "7b9dkNhuEIBNPzVDie0pgS77WgaPbYkVv5fnDwgPuVnkqfakEt7Pz2MCgYEAwkZ5\n"
+    "R1wLuTQEA2Poo6Gf4L8Bg6yNYI46LHDqDIs818iYLjtcnEEvbPfaoKNpOn7s7s4O\n"
+    "Pc+4HnT1aIQs0IKVLRTp+5a/9wfOkPZnobWOUHZk9UzBL3Hc1uy/qhp93iE3tSzx\n"
+    "v0O1pvR+hr3guTCZx8wZnDvaMgG3hlyPnVlHdrMCgYEAzQQxGbMC1ySv6quEjCP2\n"
+    "AogMbhE1lixJTUFj/EoDbNo9xKznIkauly/Lqqc1OysRhfA/G2+MY9YZBX1zwtyX\n"
+    "uBW7mPKynDrFgi9pBECnvJNmwET57Ic9ttIj6Tzbos83nAjyrzgr1zGX8dRz7ZeN\n"
+    "QbBj2vygLJbGOYinXkjUeh0CgYEAhN5aF9n2EqZmkEMGWtMxWy6HRJ0A3Cap1rcq\n"
+    "+4VHCXWhzwy+XAeg/e/N0MuyLlWcif7XcqLcE8h+BwtO8xQ8HmcNWApUJAls12wO\n"
+    "mGRpftJaXgIupdpD5aJpu1b++qrRRNIGTH9sf1D8L/8w8LcylZkbcuTkaAsQj45C\n"
+    "kqT64U0CgYEAq47IKS6xc3CDc17BqExR6t+1yRe+4ml+z1zcVbfUKony4pGvl1yo\n"
+    "rk0IYDN5Vd8h5xtXrkPdX9h+ywmohnelDKsayEuE+opyqEpSU4/96Bb22RZUoucb\n"
+    "LWkV5gZx5hFnDFtEd4vadMIiY4jVv/3JqiZDKwMVBJKlHRXJEEmIEBk=\n"
+    "-----END RSA PRIVATE KEY-----\n";
+
+  crypto_pk_t *pk = NULL;
+
+  pk = crypto_pk_new();
+  setup_capture_of_logs(LOG_WARN);
+  tt_int_op(-1, OP_EQ,
+            crypto_pk_read_private_key_from_string(pk, pem, strlen(pem)));
+#ifdef ENABLE_NSS
+  expect_single_log_msg_containing("received bad data");
+#else
+  expect_single_log_msg_containing("while checking RSA key");
+#endif
+ done:
+  teardown_capture_of_logs();
+  crypto_pk_free(pk);
+}
+
 #ifdef HAVE_TRUNCATE
 #define do_truncate truncate
 #else
@@ -1410,7 +1563,7 @@ do_truncate(const char *fname, size_t len)
   tor_free(bytes);
   return r;
 }
-#endif
+#endif /* defined(HAVE_TRUNCATE) */
 
 /** Sanity check for crypto pk digests  */
 static void
@@ -1424,7 +1577,8 @@ test_crypto_digests(void *arg)
   (void)arg;
   k = crypto_pk_new();
   tt_assert(k);
-  r = crypto_pk_read_private_key_from_string(k, AUTHORITY_SIGNKEY_3, -1);
+  r = crypto_pk_read_private_key_from_string(k, AUTHORITY_SIGNKEY_3,
+                                             strlen(AUTHORITY_SIGNKEY_3));
   tt_assert(!r);
 
   r = crypto_pk_get_digest(k, digest);
@@ -1433,6 +1587,7 @@ test_crypto_digests(void *arg)
              AUTHORITY_SIGNKEY_A_DIGEST, HEX_DIGEST_LEN);
 
   r = crypto_pk_get_common_digests(k, &pkey_digests);
+  tt_int_op(r, OP_EQ, 0);
 
   tt_mem_op(hex_str(pkey_digests.d[DIGEST_SHA1], DIGEST_LEN),OP_EQ,
              AUTHORITY_SIGNKEY_A_DIGEST, HEX_DIGEST_LEN);
@@ -1469,28 +1624,6 @@ test_crypto_digest_names(void *arg)
   ;
 }
 
-#ifndef OPENSSL_1_1_API
-#define EVP_ENCODE_CTX_new() tor_malloc_zero(sizeof(EVP_ENCODE_CTX))
-#define EVP_ENCODE_CTX_free(ctx) tor_free(ctx)
-#endif
-
-/** Encode src into dest with OpenSSL's EVP Encode interface, returning the
- * length of the encoded data in bytes.
- */
-static int
-base64_encode_evp(char *dest, char *src, size_t srclen)
-{
-  const unsigned char *s = (unsigned char*)src;
-  EVP_ENCODE_CTX *ctx = EVP_ENCODE_CTX_new();
-  int len, ret;
-
-  EVP_EncodeInit(ctx);
-  EVP_EncodeUpdate(ctx, (unsigned char *)dest, &len, s, (int)srclen);
-  EVP_EncodeFinal(ctx, (unsigned char *)(dest + len), &ret);
-  EVP_ENCODE_CTX_free(ctx);
-  return ret+ len;
-}
-
 /** Run unit tests for misc crypto formatting functionality (base64, base32,
  * fingerprints, etc) */
 static void
@@ -1519,7 +1652,7 @@ test_crypto_formats(void *arg)
     tt_int_op(i, OP_GE, 0);
     tt_int_op(i, OP_EQ, strlen(data2));
     tt_assert(! strchr(data2, '='));
-    j = base64_decode_nopad((uint8_t*)data3, 1024, data2, i);
+    j = base64_decode(data3, 1024, data2, i);
     tt_int_op(j, OP_EQ, idx);
     tt_mem_op(data3,OP_EQ, data1, idx);
   }
@@ -1544,21 +1677,7 @@ test_crypto_formats(void *arg)
   tt_mem_op(data1,OP_EQ, data3, DIGEST_LEN);
   tt_int_op(99,OP_EQ, data3[DIGEST_LEN+1]);
 
-  tt_assert(digest_from_base64(data3, "###") < 0);
-
-  for (i = 0; i < 256; i++) {
-    /* Test the multiline format Base64 encoder with 0 .. 256 bytes of
-     * output against OpenSSL.
-     */
-    const size_t enclen = base64_encode_size(i, BASE64_ENCODE_MULTILINE);
-    data1[i] = i;
-    j = base64_encode(data2, 1024, data1, i, BASE64_ENCODE_MULTILINE);
-    tt_int_op(j, OP_EQ, enclen);
-    j = base64_encode_evp(data3, data1, i);
-    tt_int_op(j, OP_EQ, enclen);
-    tt_mem_op(data2, OP_EQ, data3, enclen);
-    tt_int_op(j, OP_EQ, strlen(data2));
-  }
+  tt_int_op(digest_from_base64(data3, "###"), OP_LT, 0);
 
   /* Encoding SHA256 */
   crypto_rand(data2, DIGEST256_LEN);
@@ -1838,15 +1957,6 @@ test_crypto_hkdf_sha256(void *arg)
     key_material, 100)
 
   /* Test vectors generated with ntor_ref.py */
-  memset(key_material, 0, sizeof(key_material));
-  EXPAND("");
-  tt_int_op(r, OP_EQ, 0);
-  test_memeq_hex(key_material,
-                 "d3490ed48b12a48f9547861583573fe3f19aafe3f81dc7fc75"
-                 "eeed96d741b3290f941576c1f9f0b2d463d1ec7ab2c6bf71cd"
-                 "d7f826c6298c00dbfe6711635d7005f0269493edf6046cc7e7"
-                 "dcf6abe0d20c77cf363e8ffe358927817a3d3e73712cee28d8");
-
   EXPAND("Tor");
   tt_int_op(r, OP_EQ, 0);
   test_memeq_hex(key_material,
@@ -1969,7 +2079,7 @@ test_crypto_curve25519_impl(void *arg)
                                 "e0544770bc7de853b38f9100489e3e79";
   const char e1e2k_expected[] = "cd6e8269104eb5aaee886bd2071fba88"
                                 "bd13861475516bc2cd2b6e005e805064";
-#else
+#else /* !(defined(SLOW_CURVE25519_TEST)) */
   const int loop_max=200;
   const char e1_expected[]    = "bc7112cde03f97ef7008cad1bdc56be3"
                                 "c6a1037d74cceb3712e9206871dcf654";
@@ -1977,7 +2087,7 @@ test_crypto_curve25519_impl(void *arg)
                                 "8e3ee1a63c7d14274ea5d4c67f065467";
   const char e1e2k_expected[] = "7ddb98bd89025d2347776b33901b3e7e"
                                 "c0ee98cb2257a4545c0cfb2ca3e1812b";
-#endif
+#endif /* defined(SLOW_CURVE25519_TEST) */
 
   unsigned char e1k[32];
   unsigned char e2k[32];
@@ -2233,6 +2343,9 @@ test_crypto_ed25519_simple(void *arg)
   tt_int_op(0, OP_EQ, ed25519_public_key_generate(&pub1, &sec1));
   tt_int_op(0, OP_EQ, ed25519_public_key_generate(&pub2, &sec1));
 
+  tt_int_op(ed25519_validate_pubkey(&pub1), OP_EQ, 0);
+  tt_int_op(ed25519_validate_pubkey(&pub2), OP_EQ, 0);
+
   tt_mem_op(pub1.pubkey, OP_EQ, pub2.pubkey, sizeof(pub1.pubkey));
   tt_assert(ed25519_pubkey_eq(&pub1, &pub2));
   tt_assert(ed25519_pubkey_eq(&pub1, &pub1));
@@ -2604,6 +2717,39 @@ test_crypto_ed25519_blinding(void *arg)
   ;
 }
 
+/** Test that our blinding functions will fail if we pass them bad pubkeys */
+static void
+test_crypto_ed25519_blinding_fail(void *arg)
+{
+  int retval;
+  uint8_t param[32] = {2};
+  ed25519_public_key_t pub;
+  ed25519_public_key_t pub_blinded;
+
+  (void)arg;
+
+  /* This point is not on the curve: the blind routines should fail */
+  const char badkey[] =
+    "e19c65de75c68cf3b7643ea732ba9eb1a3d20d6d57ba223c2ece1df66feb5af0";
+  retval = base16_decode((char*)pub.pubkey, sizeof(pub.pubkey),
+                         badkey, strlen(badkey));
+  tt_int_op(retval, OP_EQ, sizeof(pub.pubkey));
+  retval = ed25519_public_blind(&pub_blinded, &pub, param);
+  tt_int_op(retval, OP_EQ, -1);
+
+  /* This point is legit: blind routines should be happy */
+  const char goodkey[] =
+    "4ba2e44760dff4c559ef3c38768c1c14a8a54740c782c8d70803e9d6e3ad8794";
+  retval = base16_decode((char*)pub.pubkey, sizeof(pub.pubkey),
+                         goodkey, strlen(goodkey));
+  tt_int_op(retval, OP_EQ, sizeof(pub.pubkey));
+  retval = ed25519_public_blind(&pub_blinded, &pub, param);
+  tt_int_op(retval, OP_EQ, 0);
+
+ done:
+  ;
+}
+
 static void
 test_crypto_ed25519_testvectors(void *arg)
 {
@@ -2621,6 +2767,8 @@ test_crypto_ed25519_testvectors(void *arg)
     ed25519_signature_t sig;
     int sign;
 
+    memset(&curvekp, 0xd0, sizeof(curvekp));
+
 #define DECODE(p,s) base16_decode((char*)(p),sizeof(p),(s),strlen(s))
 #define EQ(a,h) test_memeq_hex((const char*)(a), (h))
 
@@ -2702,8 +2850,8 @@ test_crypto_ed25519_storage(void *arg)
   tor_free(tag);
 
   /* whitebox test: truncated keys. */
-  tt_int_op(0, ==, do_truncate(fname_1, 40));
-  tt_int_op(0, ==, do_truncate(fname_2, 40));
+  tt_int_op(0, OP_EQ, do_truncate(fname_1, 40));
+  tt_int_op(0, OP_EQ, do_truncate(fname_2, 40));
   tt_int_op(-1, OP_EQ, ed25519_pubkey_read_from_file(&pub, &tag, fname_2));
   tt_ptr_op(tag, OP_EQ, NULL);
   tor_free(tag);
@@ -2793,8 +2941,8 @@ test_crypto_siphash(void *arg)
       { 0x72, 0x45, 0x06, 0xeb, 0x4c, 0x32, 0x8a, 0x95, }
     };
 
-  const struct sipkey K = { U64_LITERAL(0x0706050403020100),
-                            U64_LITERAL(0x0f0e0d0c0b0a0908) };
+  const struct sipkey K = { UINT64_C(0x0706050403020100),
+                            UINT64_C(0x0f0e0d0c0b0a0908) };
   uint8_t input[64];
   int i, j;
 
@@ -2849,12 +2997,12 @@ crypto_rand_check_failure_mode_identical(void)
 {
   /* just in case the buffer size isn't a multiple of sizeof(int64_t) */
 #define FAILURE_MODE_BUFFER_SIZE_I64 \
-  (FAILURE_MODE_BUFFER_SIZE/SIZEOF_INT64_T)
+  (FAILURE_MODE_BUFFER_SIZE/8)
 #define FAILURE_MODE_BUFFER_SIZE_I64_BYTES \
-  (FAILURE_MODE_BUFFER_SIZE_I64*SIZEOF_INT64_T)
+  (FAILURE_MODE_BUFFER_SIZE_I64*8)
 
 #if FAILURE_MODE_BUFFER_SIZE_I64 < 2
-#error FAILURE_MODE_BUFFER_SIZE needs to be at least 2*SIZEOF_INT64_T
+#error FAILURE_MODE_BUFFER_SIZE needs to be at least 2*8
 #endif
 
   int64_t buf[FAILURE_MODE_BUFFER_SIZE_I64];
@@ -2895,6 +3043,67 @@ crypto_rand_check_failure_mode_predict(void)
 
 #undef FAILURE_MODE_BUFFER_SIZE
 
+/** Test that our ed25519 validation function rejects evil public keys and
+ *  accepts good ones. */
+static void
+test_crypto_ed25519_validation(void *arg)
+{
+  (void) arg;
+
+  int retval;
+  ed25519_public_key_t pub1;
+
+  /* See https://lists.torproject.org/pipermail/tor-dev/2017-April/012230.html
+     for a list of points with torsion components in ed25519. */
+
+  { /* Point with torsion component (order 8l) */
+    const char badkey[] =
+      "300ef2e64e588e1df55b48e4da0416ffb64cc85d5b00af6463d5cc6c2b1c185e";
+    retval = base16_decode((char*)pub1.pubkey, sizeof(pub1.pubkey),
+                           badkey, strlen(badkey));
+    tt_int_op(retval, OP_EQ, sizeof(pub1.pubkey));
+    tt_int_op(ed25519_validate_pubkey(&pub1), OP_EQ, -1);
+  }
+
+  { /* Point with torsion component (order 4l) */
+    const char badkey[] =
+      "f43e3a046db8749164c6e69b193f1e942c7452e7d888736f40b98093d814d5e7";
+    retval = base16_decode((char*)pub1.pubkey, sizeof(pub1.pubkey),
+                           badkey, strlen(badkey));
+    tt_int_op(retval, OP_EQ, sizeof(pub1.pubkey));
+    tt_int_op(ed25519_validate_pubkey(&pub1), OP_EQ, -1);
+  }
+
+  { /* Point with torsion component (order 2l) */
+    const char badkey[] =
+      "c9fff3af0471c28e33e98c2043e44f779d0427b1e37c521a6bddc011ed1869af";
+    retval = base16_decode((char*)pub1.pubkey, sizeof(pub1.pubkey),
+                           badkey, strlen(badkey));
+    tt_int_op(retval, OP_EQ, sizeof(pub1.pubkey));
+    tt_int_op(ed25519_validate_pubkey(&pub1), OP_EQ, -1);
+  }
+
+  { /* This point is not even on the curve */
+    const char badkey[] =
+      "e19c65de75c68cf3b7643ea732ba9eb1a3d20d6d57ba223c2ece1df66feb5af0";
+    retval = base16_decode((char*)pub1.pubkey, sizeof(pub1.pubkey),
+                           badkey, strlen(badkey));
+    tt_int_op(retval, OP_EQ, sizeof(pub1.pubkey));
+    tt_int_op(ed25519_validate_pubkey(&pub1), OP_EQ, -1);
+  }
+
+  { /* This one is a good key */
+    const char goodkey[] =
+      "4ba2e44760dff4c559ef3c38768c1c14a8a54740c782c8d70803e9d6e3ad8794";
+    retval = base16_decode((char*)pub1.pubkey, sizeof(pub1.pubkey),
+                           goodkey, strlen(goodkey));
+    tt_int_op(retval, OP_EQ, sizeof(pub1.pubkey));
+    tt_int_op(ed25519_validate_pubkey(&pub1), OP_EQ, 0);
+  }
+
+ done: ;
+}
+
 static void
 test_crypto_failure_modes(void *arg)
 {
@@ -2902,17 +3111,17 @@ test_crypto_failure_modes(void *arg)
   (void)arg;
 
   rv = crypto_early_init();
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
 
   /* Check random works */
   rv = crypto_rand_check_failure_mode_zero();
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
 
   rv = crypto_rand_check_failure_mode_identical();
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
 
   rv = crypto_rand_check_failure_mode_predict();
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
 
  done:
   ;
@@ -2933,7 +3142,6 @@ struct testcase_t crypto_tests[] = {
   CRYPTO_LEGACY(formats),
   CRYPTO_LEGACY(rng),
   { "rng_range", test_crypto_rng_range, 0, NULL, NULL },
-  { "rng_engine", test_crypto_rng_engine, TT_FORK, NULL, NULL },
   { "rng_strongest", test_crypto_rng_strongest, TT_FORK, NULL, NULL },
   { "rng_strongest_nosyscall", test_crypto_rng_strongest, TT_FORK,
     &passthrough_setup, (void*)"nosyscall" },
@@ -2955,10 +3163,13 @@ struct testcase_t crypto_tests[] = {
   { "pk_fingerprints", test_crypto_pk_fingerprints, TT_FORK, NULL, NULL },
   { "pk_base64", test_crypto_pk_base64, TT_FORK, NULL, NULL },
   { "pk_pem_encrypted", test_crypto_pk_pem_encrypted, TT_FORK, NULL, NULL },
+  { "pk_invalid_private_key", test_crypto_pk_invalid_private_key, 0,
+    NULL, NULL },
   CRYPTO_LEGACY(digests),
   { "digest_names", test_crypto_digest_names, 0, NULL, NULL },
   { "sha3", test_crypto_sha3, TT_FORK, NULL, NULL},
   { "sha3_xof", test_crypto_sha3_xof, TT_FORK, NULL, NULL},
+  { "mac_sha3", test_crypto_mac_sha3, TT_FORK, NULL, NULL},
   CRYPTO_LEGACY(dh),
   { "aes_iv_AES", test_crypto_aes_iv, TT_FORK, &passthrough_setup,
     (void*)"aes" },
@@ -2981,10 +3192,11 @@ struct testcase_t crypto_tests[] = {
   ED25519_TEST(encode, 0),
   ED25519_TEST(convert, 0),
   ED25519_TEST(blinding, 0),
+  ED25519_TEST(blinding_fail, 0),
   ED25519_TEST(testvectors, 0),
+  ED25519_TEST(validation, 0),
   { "ed25519_storage", test_crypto_ed25519_storage, 0, NULL, NULL },
   { "siphash", test_crypto_siphash, 0, NULL, NULL },
   { "failure_modes", test_crypto_failure_modes, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_crypto_ope.c b/src/test/test_crypto_ope.c
new file mode 100644
index 0000000000..dc67c02676
--- /dev/null
+++ b/src/test/test_crypto_ope.c
@@ -0,0 +1,154 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#define CRYPTO_OPE_PRIVATE
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/crypt_ops/crypto_ope.h"
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "lib/encoding/binascii.h"
+#include "lib/malloc/malloc.h"
+#include "test/test.h"
+#include "tinytest.h"
+
+#include <stddef.h>
+#include <string.h>
+
+static void
+test_crypto_ope_consistency(void *arg)
+{
+  (void)arg;
+
+  crypto_ope_t *ope = NULL;
+  crypto_cipher_t *aes = NULL;
+  const int TEST_VALS[] = { 5, 500, 1023, 1024, 1025, 2046, 2047, 2048, 2049,
+                            10000, OPE_INPUT_MAX };
+  unsigned i;
+  const uint8_t key[32] = "A fixed key, chosen arbitrarily.";
+
+  ope = crypto_ope_new(key);
+  tt_assert(ope);
+
+  uint64_t last_val = 0;
+  for (i = 0; i < ARRAY_LENGTH(TEST_VALS); ++i) {
+    aes = ope_get_cipher(ope, 0);
+    int val = TEST_VALS[i];
+    uint64_t v1 = crypto_ope_encrypt(ope, val);
+    uint64_t v2 = sum_values_from_cipher(aes, val);
+    tt_u64_op(v1, OP_EQ, v2);
+    tt_u64_op(v2, OP_GT, last_val);
+    last_val = v2;
+    crypto_cipher_free(aes);
+  }
+
+ done:
+  crypto_cipher_free(aes);
+  crypto_ope_free(ope);
+}
+
+static void
+test_crypto_ope_oob(void *arg)
+{
+  (void)arg;
+
+  crypto_ope_t *ope = NULL;
+  const uint8_t key[32] = "A fixed key, chosen arbitrarily.";
+  ope = crypto_ope_new(key);
+
+  tt_u64_op(UINT64_MAX, OP_EQ, crypto_ope_encrypt(ope,INT_MIN));
+  tt_u64_op(UINT64_MAX, OP_EQ, crypto_ope_encrypt(ope,-100));
+  tt_u64_op(UINT64_MAX, OP_EQ, crypto_ope_encrypt(ope,0));
+  tt_u64_op(UINT64_MAX, OP_NE, crypto_ope_encrypt(ope,1));
+  tt_u64_op(UINT64_MAX, OP_NE, crypto_ope_encrypt(ope,7000));
+  tt_u64_op(UINT64_MAX, OP_NE, crypto_ope_encrypt(ope,OPE_INPUT_MAX));
+  tt_u64_op(UINT64_MAX, OP_EQ, crypto_ope_encrypt(ope,OPE_INPUT_MAX+1));
+  tt_u64_op(UINT64_MAX, OP_EQ, crypto_ope_encrypt(ope,INT_MAX));
+ done:
+  crypto_ope_free(ope);
+}
+
+static const char OPE_TEST_KEY[] =
+  "19e05891d55232c08c2cad91d612fdb9cbd6691949a0742434a76c80bc6992fe";
+
+/* generated by a separate python implementation. */
+static const struct {
+  int v;
+  uint64_t r;
+} OPE_TEST_VECTORS[] = {
+  { 121132, UINT64_C(3971694514) },
+  { 82283, UINT64_C(2695743564) },
+  { 72661, UINT64_C(2381548866) },
+  { 72941, UINT64_C(2390408421) },
+  { 123122, UINT64_C(4036781069) },
+  { 12154, UINT64_C(402067100) },
+  { 121574, UINT64_C(3986197593) },
+  { 11391, UINT64_C(376696838) },
+  { 65845, UINT64_C(2161801517) },
+  { 86301, UINT64_C(2828270975) },
+  { 61284, UINT64_C(2013616892) },
+  { 70505, UINT64_C(2313368870) },
+  { 30438, UINT64_C(1001394664) },
+  { 60150, UINT64_C(1977329668) },
+  { 114800, UINT64_C(3764946628) },
+  { 109403, UINT64_C(3585352477) },
+  { 21893, UINT64_C(721388468) },
+  { 123569, UINT64_C(4051780471) },
+  { 95617, UINT64_C(3134921876) },
+  { 48561, UINT64_C(1597596985) },
+  { 53334, UINT64_C(1753691710) },
+  { 92746, UINT64_C(3040874493) },
+  { 7110, UINT64_C(234966492) },
+  { 9612, UINT64_C(318326551) },
+  { 106958, UINT64_C(3506124249) },
+  { 46889, UINT64_C(1542219146) },
+  { 87790, UINT64_C(2877361609) },
+  { 68878, UINT64_C(2260369112) },
+  { 47917, UINT64_C(1576681737) },
+  { 121128, UINT64_C(3971553290) },
+  { 108602, UINT64_C(3559176081) },
+  { 28217, UINT64_C(929692460) },
+  { 69498, UINT64_C(2280554161) },
+  { 63870, UINT64_C(2098322675) },
+  { 57542, UINT64_C(1891698992) },
+  { 122148, UINT64_C(4004515805) },
+  { 46254, UINT64_C(1521227949) },
+  { 42850, UINT64_C(1408996941) },
+  { 92661, UINT64_C(3037901517) },
+  { 57720, UINT64_C(1897369989) },
+};
+
+static void
+test_crypto_ope_vectors(void *arg)
+{
+  (void)arg;
+  uint8_t key[32];
+  crypto_ope_t *ope = NULL, *ope2 = NULL;
+
+  base16_decode((char*)key, 32, OPE_TEST_KEY, strlen(OPE_TEST_KEY));
+
+  ope = crypto_ope_new(key);
+  key[8] += 1;
+  ope2 = crypto_ope_new(key);
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(OPE_TEST_VECTORS); ++i) {
+    int val = OPE_TEST_VECTORS[i].v;
+    uint64_t res = OPE_TEST_VECTORS[i].r;
+
+    tt_u64_op(crypto_ope_encrypt(ope, val), OP_EQ, res);
+    tt_u64_op(crypto_ope_encrypt(ope2, val), OP_NE, res);
+  }
+ done:
+  crypto_ope_free(ope);
+  crypto_ope_free(ope2);
+}
+
+struct testcase_t crypto_ope_tests[] = {
+  { "consistency", test_crypto_ope_consistency, 0, NULL, NULL },
+  { "oob", test_crypto_ope_oob, 0, NULL, NULL },
+  { "vectors", test_crypto_ope_vectors, 0, NULL, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_crypto_openssl.c b/src/test/test_crypto_openssl.c
new file mode 100644
index 0000000000..42dc3f6be2
--- /dev/null
+++ b/src/test/test_crypto_openssl.c
@@ -0,0 +1,106 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#define CRYPTO_RAND_PRIVATE
+
+#include "lib/crypt_ops/compat_openssl.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/encoding/binascii.h"
+#include "lib/malloc/malloc.h"
+#include "test/test.h"
+
+#include <openssl/evp.h>
+#include <openssl/rand.h>
+#include <string.h>
+
+/* Test for rectifying openssl RAND engine. */
+static void
+test_crypto_rng_engine(void *arg)
+{
+  (void)arg;
+  RAND_METHOD dummy_method;
+  memset(&dummy_method, 0, sizeof(dummy_method));
+
+  /* We should be a no-op if we're already on RAND_OpenSSL */
+  tt_int_op(0, OP_EQ, crypto_force_rand_ssleay());
+  tt_assert(RAND_get_rand_method() == RAND_OpenSSL());
+
+  /* We should correct the method if it's a dummy. */
+  RAND_set_rand_method(&dummy_method);
+#ifdef LIBRESSL_VERSION_NUMBER
+  /* On libressl, you can't override the RNG. */
+  tt_assert(RAND_get_rand_method() == RAND_OpenSSL());
+  tt_int_op(0, OP_EQ, crypto_force_rand_ssleay());
+#else
+  tt_assert(RAND_get_rand_method() == &dummy_method);
+  tt_int_op(1, OP_EQ, crypto_force_rand_ssleay());
+#endif /* defined(LIBRESSL_VERSION_NUMBER) */
+  tt_assert(RAND_get_rand_method() == RAND_OpenSSL());
+
+  /* Make sure we aren't calling dummy_method */
+  crypto_rand((void *) &dummy_method, sizeof(dummy_method));
+  crypto_rand((void *) &dummy_method, sizeof(dummy_method));
+
+ done:
+  ;
+}
+
+#ifndef OPENSSL_1_1_API
+#define EVP_ENCODE_CTX_new() tor_malloc_zero(sizeof(EVP_ENCODE_CTX))
+#define EVP_ENCODE_CTX_free(ctx) tor_free(ctx)
+#endif
+
+/** Encode src into dest with OpenSSL's EVP Encode interface, returning the
+ * length of the encoded data in bytes.
+ */
+static int
+base64_encode_evp(char *dest, char *src, size_t srclen)
+{
+  const unsigned char *s = (unsigned char*)src;
+  EVP_ENCODE_CTX *ctx = EVP_ENCODE_CTX_new();
+  int len, ret;
+
+  EVP_EncodeInit(ctx);
+  EVP_EncodeUpdate(ctx, (unsigned char *)dest, &len, s, (int)srclen);
+  EVP_EncodeFinal(ctx, (unsigned char *)(dest + len), &ret);
+  EVP_ENCODE_CTX_free(ctx);
+  return ret+ len;
+}
+
+static void
+test_crypto_base64_encode_matches(void *arg)
+{
+  (void)arg;
+  int i, j;
+  char data1[1024];
+  char data2[1024];
+  char data3[1024];
+
+  for (i = 0; i < 256; i++) {
+    /* Test the multiline format Base64 encoder with 0 .. 256 bytes of
+     * output against OpenSSL.
+     */
+    const size_t enclen = base64_encode_size(i, BASE64_ENCODE_MULTILINE);
+    data1[i] = i;
+    j = base64_encode(data2, 1024, data1, i, BASE64_ENCODE_MULTILINE);
+    tt_int_op(j, OP_EQ, enclen);
+    j = base64_encode_evp(data3, data1, i);
+    tt_int_op(j, OP_EQ, enclen);
+    tt_mem_op(data2, OP_EQ, data3, enclen);
+    tt_int_op(j, OP_EQ, strlen(data2));
+  }
+
+ done:
+  ;
+}
+
+struct testcase_t crypto_openssl_tests[] = {
+  { "rng_engine", test_crypto_rng_engine, TT_FORK, NULL, NULL },
+  { "base64_encode_match", test_crypto_base64_encode_matches,
+    TT_FORK, NULL, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_crypto_slow.c b/src/test/test_crypto_slow.c
index 0be58c9389..e24aee8930 100644
--- a/src/test/test_crypto_slow.c
+++ b/src/test/test_crypto_slow.c
@@ -1,21 +1,26 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 #define CRYPTO_S2K_PRIVATE
-#include "or.h"
-#include "test.h"
-#include "crypto_s2k.h"
-#include "crypto_pwbox.h"
+#include "core/or/or.h"
+#include "test/test.h"
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "lib/crypt_ops/crypto_s2k.h"
+#include "lib/crypt_ops/crypto_pwbox.h"
+#include "lib/crypt_ops/crypto_rand.h"
 
 #if defined(HAVE_LIBSCRYPT_H) && defined(HAVE_LIBSCRYPT_SCRYPT)
 #define HAVE_LIBSCRYPT
 #include <libscrypt.h>
 #endif
 
+#ifdef ENABLE_OPENSSL
 #include <openssl/evp.h>
+#endif
 
 /** Run unit tests for our secret-to-key passphrase hashing functionality. */
 static void
@@ -164,10 +169,10 @@ test_libscrypt_eq_openssl(void *arg)
   EVP_PBE_scrypt((const char *)"", 0, (const unsigned char *)"", 0,
                   N, r, p, maxmem, buf2, dk_len);
 
-  tt_int_op(libscrypt_retval, ==, 0);
-  tt_int_op(openssl_retval, ==, 1);
+  tt_int_op(libscrypt_retval, OP_EQ, 0);
+  tt_int_op(openssl_retval, OP_EQ, 1);
 
-  tt_mem_op(buf1, ==, buf2, 64);
+  tt_mem_op(buf1, OP_EQ, buf2, 64);
 
   memset(buf1,0,64);
   memset(buf2,0,64);
@@ -185,10 +190,10 @@ test_libscrypt_eq_openssl(void *arg)
                  (const unsigned char *)"NaCl", strlen("NaCl"),
                  N, r, p, maxmem, buf2, dk_len);
 
-  tt_int_op(libscrypt_retval, ==, 0);
-  tt_int_op(openssl_retval, ==, 1);
+  tt_int_op(libscrypt_retval, OP_EQ, 0);
+  tt_int_op(openssl_retval, OP_EQ, 1);
 
-  tt_mem_op(buf1, ==, buf2, 64);
+  tt_mem_op(buf1, OP_EQ, buf2, 64);
 
   memset(buf1,0,64);
   memset(buf2,0,64);
@@ -210,10 +215,10 @@ test_libscrypt_eq_openssl(void *arg)
                  strlen("SodiumChloride"),
                  N, r, p, maxmem, buf2, dk_len);
 
-  tt_int_op(libscrypt_retval, ==, 0);
-  tt_int_op(openssl_retval, ==, 1);
+  tt_int_op(libscrypt_retval, OP_EQ, 0);
+  tt_int_op(openssl_retval, OP_EQ, 1);
 
-  tt_mem_op(buf1, ==, buf2, 64);
+  tt_mem_op(buf1, OP_EQ, buf2, 64);
 
   memset(buf1,0,64);
   memset(buf2,0,64);
@@ -234,15 +239,15 @@ test_libscrypt_eq_openssl(void *arg)
                  strlen("SodiumChloride"),
                  N, r, p, maxmem, buf2, dk_len);
 
-  tt_int_op(libscrypt_retval, ==, 0);
-  tt_int_op(openssl_retval, ==, 1);
+  tt_int_op(libscrypt_retval, OP_EQ, 0);
+  tt_int_op(openssl_retval, OP_EQ, 1);
 
-  tt_mem_op(buf1, ==, buf2, 64);
+  tt_mem_op(buf1, OP_EQ, buf2, 64);
 
   done:
   return;
 }
-#endif
+#endif /* defined(HAVE_LIBSCRYPT) && defined(HAVE_EVP_PBE_SCRYPT) */
 
 static void
 test_crypto_s2k_errors(void *arg)
@@ -283,7 +288,7 @@ test_crypto_s2k_errors(void *arg)
                                                  "ABC", 3, 0));
   tt_int_op(S2K_TRUNCATED, OP_EQ, secret_to_key_new(buf, 50, &sz,
                                                  "ABC", 3, S2K_FLAG_LOW_MEM));
-#endif
+#endif /* defined(HAVE_LIBSCRYPT) */
   tt_int_op(S2K_TRUNCATED, OP_EQ, secret_to_key_new(buf, 37, &sz,
                                               "ABC", 3, S2K_FLAG_USE_PBKDF2));
   tt_int_op(S2K_TRUNCATED, OP_EQ, secret_to_key_new(buf, 29, &sz,
@@ -318,7 +323,7 @@ test_crypto_s2k_errors(void *arg)
   tt_int_op(S2K_BAD_PARAMS, OP_EQ,
             secret_to_key_derivekey(buf2, sizeof(buf2),
                                     buf, 19, "ABC", 3));
-#endif
+#endif /* defined(HAVE_LIBSCRYPT) */
 
  done:
   ;
@@ -516,7 +521,7 @@ test_crypto_ed25519_fuzz_donna(void *arg)
   unsigned i;
   (void)arg;
 
-  tt_assert(sizeof(msg) == iters);
+  tt_uint_op(iters, OP_EQ, sizeof(msg));
   crypto_rand((char*) msg, sizeof(msg));
 
   /* Fuzz Ed25519-donna vs ref10, alternating the implementation used to
@@ -600,7 +605,7 @@ struct testcase_t slow_crypto_tests[] = {
 #ifdef HAVE_EVP_PBE_SCRYPT
   { "libscrypt_eq_openssl", test_libscrypt_eq_openssl, 0, NULL, NULL },
 #endif
-#endif
+#endif /* defined(HAVE_LIBSCRYPT) */
   { "s2k_pbkdf2", test_crypto_s2k_general, 0, &passthrough_setup,
     (void*)"pbkdf2" },
   { "s2k_rfc2440_general", test_crypto_s2k_general, 0, &passthrough_setup,
@@ -614,4 +619,3 @@ struct testcase_t slow_crypto_tests[] = {
   ED25519_TEST(fuzz_donna, TT_FORK),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_data.c b/src/test/test_data.c
index 788489a097..fe1190ea77 100644
--- a/src/test/test_data.c
+++ b/src/test/test_data.c
@@ -1,9 +1,9 @@
 /* Copyright 2001-2004 Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#include "test.h"
+#include "test/test.h"
 
 /* Our unit test expect that the AUTHORITY_CERT_* public keys will sort
  * in this order. */
diff --git a/src/test/test_dir.c b/src/test/test_dir.c
index 4cdbfb4f84..0e44c47f3f 100644
--- a/src/test/test_dir.c
+++ b/src/test/test_dir.c
@@ -1,42 +1,96 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 #include <math.h>
 
+#define BWAUTH_PRIVATE
 #define CONFIG_PRIVATE
+#define CONTROL_PRIVATE
+#define DIRCACHE_PRIVATE
+#define DIRCLIENT_PRIVATE
 #define DIRSERV_PRIVATE
 #define DIRVOTE_PRIVATE
-#define ROUTER_PRIVATE
-#define ROUTERLIST_PRIVATE
-#define ROUTERPARSE_PRIVATE
+#define DLSTATUS_PRIVATE
 #define HIBERNATE_PRIVATE
 #define NETWORKSTATUS_PRIVATE
+#define NS_PARSE_PRIVATE
+#define NODE_SELECT_PRIVATE
 #define RELAY_PRIVATE
-
-#include "or.h"
-#include "confparse.h"
-#include "config.h"
-#include "crypto_ed25519.h"
-#include "directory.h"
-#include "dirserv.h"
-#include "dirvote.h"
-#include "hibernate.h"
-#include "memarea.h"
-#include "networkstatus.h"
-#include "router.h"
-#include "routerkeys.h"
-#include "routerlist.h"
-#include "routerparse.h"
-#include "routerset.h"
-#include "shared_random_state.h"
-#include "test.h"
-#include "test_dir_common.h"
-#include "torcert.h"
-#include "relay.h"
-#include "log_test_helpers.h"
+#define ROUTERLIST_PRIVATE
+#define ROUTER_PRIVATE
+#define UNPARSEABLE_PRIVATE
+#define VOTEFLAGS_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "app/config/confparse.h"
+#include "core/mainloop/connection.h"
+#include "core/or/relay.h"
+#include "core/or/versions.h"
+#include "feature/client/bridges.h"
+#include "feature/client/entrynodes.h"
+#include "feature/control/control.h"
+#include "feature/dirauth/bwauth.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/dsigs_parse.h"
+#include "feature/dirauth/process_descs.h"
+#include "feature/dirauth/recommend_pkg.h"
+#include "feature/dirauth/shared_random_state.h"
+#include "feature/dirauth/voteflags.h"
+#include "feature/dircache/dircache.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dircommon/fp_pair.h"
+#include "feature/dircommon/voting_schedule.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nickname.h"
+#include "feature/nodelist/node_select.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/dirparse/authcert_parse.h"
+#include "feature/dirparse/ns_parse.h"
+#include "feature/dirparse/routerparse.h"
+#include "feature/dirparse/unparseable.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/nodelist/torcert.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routerkeys.h"
+#include "feature/relay/routermode.h"
+#include "lib/compress/compress.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/encoding/confline.h"
+#include "lib/memarea/memarea.h"
+#include "lib/osinfo/uname.h"
+#include "test/log_test_helpers.h"
+#include "test/test.h"
+#include "test/test_dir_common.h"
+
+#include "core/or/addr_policy_st.h"
+#include "feature/nodelist/authority_cert_st.h"
+#include "feature/nodelist/document_signature_st.h"
+#include "feature/nodelist/extrainfo_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/networkstatus_voter_info_st.h"
+#include "feature/dirauth/ns_detached_signatures_st.h"
+#include "core/or/port_cfg_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerlist_st.h"
+#include "core/or/tor_version_st.h"
+#include "feature/dirauth/vote_microdesc_hash_st.h"
+#include "feature/nodelist/vote_routerstatus_st.h"
+
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
 
 #define NS_MODULE dir
 
@@ -136,7 +190,7 @@ test_dir_formats(void *arg)
   r1->supports_tunnelled_dir_requests = 1;
   tor_addr_parse(&r1->ipv6_addr, "1:2:3:4::");
   r1->ipv6_orport = 9999;
-  r1->onion_pkey = crypto_pk_dup_key(pk1);
+  router_set_rsa_onion_pkey(pk1, &r1->onion_pkey, &r1->onion_pkey_len);
   /* Fake just enough of an ntor key to get by */
   curve25519_keypair_t r1_onion_keypair;
   curve25519_keypair_generate(&r1_onion_keypair, 0);
@@ -179,7 +233,7 @@ test_dir_formats(void *arg)
   r2->or_port = 9005;
   r2->dir_port = 0;
   r2->supports_tunnelled_dir_requests = 1;
-  r2->onion_pkey = crypto_pk_dup_key(pk2);
+  router_set_rsa_onion_pkey(pk2, &r2->onion_pkey, &r2->onion_pkey_len);
   curve25519_keypair_t r2_onion_keypair;
   curve25519_keypair_generate(&r2_onion_keypair, 0);
   r2->onion_curve25519_pkey = tor_memdup(&r2_onion_keypair.pubkey,
@@ -272,8 +326,11 @@ test_dir_formats(void *arg)
   tt_int_op(rp1->bandwidthrate,OP_EQ, r1->bandwidthrate);
   tt_int_op(rp1->bandwidthburst,OP_EQ, r1->bandwidthburst);
   tt_int_op(rp1->bandwidthcapacity,OP_EQ, r1->bandwidthcapacity);
-  tt_assert(crypto_pk_cmp_keys(rp1->onion_pkey, pk1) == 0);
-  tt_assert(crypto_pk_cmp_keys(rp1->identity_pkey, pk2) == 0);
+  crypto_pk_t *onion_pkey = router_get_rsa_onion_pkey(rp1->onion_pkey,
+                                                      rp1->onion_pkey_len);
+  tt_int_op(crypto_pk_cmp_keys(onion_pkey, pk1), OP_EQ, 0);
+  crypto_pk_free(onion_pkey);
+  tt_int_op(crypto_pk_cmp_keys(rp1->identity_pkey, pk2), OP_EQ, 0);
   tt_assert(rp1->supports_tunnelled_dir_requests);
   //tt_assert(rp1->exit_policy == NULL);
   tor_free(buf);
@@ -291,9 +348,9 @@ test_dir_formats(void *arg)
   strlcat(buf2, "master-key-ed25519 ", sizeof(buf2));
   {
     char k[ED25519_BASE64_LEN+1];
-    tt_assert(ed25519_public_to_base64(k,
-                                &r2->cache_info.signing_key_cert->signing_key)
-              >= 0);
+    tt_int_op(ed25519_public_to_base64(k,
+                          &r2->cache_info.signing_key_cert->signing_key),
+              OP_GE, 0);
     strlcat(buf2, k, sizeof(buf2));
     strlcat(buf2, "\n", sizeof(buf2));
   }
@@ -328,7 +385,7 @@ test_dir_formats(void *arg)
     ntor_cc = make_ntor_onion_key_crosscert(&r2_onion_keypair,
                                           &kp1.pubkey,
                                           r2->cache_info.published_on,
-                                          MIN_ONION_KEY_LIFETIME,
+                                          get_onion_key_lifetime(),
                                           &ntor_cc_sign);
     tt_assert(ntor_cc);
     base64_encode(cert_buf, sizeof(cert_buf),
@@ -389,8 +446,11 @@ test_dir_formats(void *arg)
   tt_mem_op(rp2->onion_curve25519_pkey->public_key,OP_EQ,
              r2->onion_curve25519_pkey->public_key,
              CURVE25519_PUBKEY_LEN);
-  tt_assert(crypto_pk_cmp_keys(rp2->onion_pkey, pk2) == 0);
-  tt_assert(crypto_pk_cmp_keys(rp2->identity_pkey, pk1) == 0);
+  onion_pkey = router_get_rsa_onion_pkey(rp2->onion_pkey,
+                                         rp2->onion_pkey_len);
+  tt_int_op(crypto_pk_cmp_keys(onion_pkey, pk2), OP_EQ, 0);
+  crypto_pk_free(onion_pkey);
+  tt_int_op(crypto_pk_cmp_keys(rp2->identity_pkey, pk1), OP_EQ, 0);
   tt_assert(rp2->supports_tunnelled_dir_requests);
 
   tt_int_op(smartlist_len(rp2->exit_policy),OP_EQ, 2);
@@ -419,7 +479,7 @@ test_dir_formats(void *arg)
     add_fingerprint_to_dir(buf, fingerprint_list, 0);
   }
 
-#endif
+#endif /* 0 */
   dirserv_free_fingerprint_list();
 
  done:
@@ -475,34 +535,34 @@ test_dir_routerinfo_parsing(void *arg)
   routerinfo_free(ri);
   ri = router_parse_entry_from_string(EX_RI_MINIMAL, NULL, 0, 0,
                                       "@purpose bridge\n", NULL);
-  tt_assert(ri != NULL);
+  tt_ptr_op(ri, OP_NE, NULL);
   tt_assert(ri->purpose == ROUTER_PURPOSE_BRIDGE);
   routerinfo_free(ri);
 
   /* bad annotations prepended. */
   ri = router_parse_entry_from_string(EX_RI_MINIMAL,
                                       NULL, 0, 0, "@purpose\n", NULL);
-  tt_assert(ri == NULL);
+  tt_ptr_op(ri, OP_EQ, NULL);
 
   /* bad annotations on router. */
   ri = router_parse_entry_from_string("@purpose\nrouter x\n", NULL, 0, 1,
                                       NULL, NULL);
-  tt_assert(ri == NULL);
+  tt_ptr_op(ri, OP_EQ, NULL);
 
   /* unwanted annotations on router. */
   ri = router_parse_entry_from_string("@purpose foo\nrouter x\n", NULL, 0, 0,
                                       NULL, NULL);
-  tt_assert(ri == NULL);
+  tt_ptr_op(ri, OP_EQ, NULL);
 
   /* No signature. */
   ri = router_parse_entry_from_string("router x\n", NULL, 0, 0,
                                       NULL, NULL);
-  tt_assert(ri == NULL);
+  tt_ptr_op(ri, OP_EQ, NULL);
 
   /* Not a router */
   routerinfo_free(ri);
   ri = router_parse_entry_from_string("hello\n", NULL, 0, 0, NULL, NULL);
-  tt_assert(ri == NULL);
+  tt_ptr_op(ri, OP_EQ, NULL);
 
   CHECK_FAIL(EX_RI_BAD_SIG1, 1);
   CHECK_FAIL(EX_RI_BAD_SIG2, 1);
@@ -563,7 +623,7 @@ test_dir_routerinfo_parsing(void *arg)
 static void
 routerinfo_free_wrapper_(void *arg)
 {
-  routerinfo_free(arg);
+  routerinfo_free_(arg);
 }
 
 static void
@@ -629,11 +689,11 @@ test_dir_extrainfo_parsing(void *arg)
   ADD(EX_EI_ED_MISPLACED_SIG);
 
   CHECK_OK(EX_EI_MINIMAL);
-  tt_assert(!ei->pending_sig);
+  tt_ptr_op(ei->pending_sig, OP_EQ, NULL);
   CHECK_OK(EX_EI_MAXIMAL);
-  tt_assert(!ei->pending_sig);
+  tt_ptr_op(ei->pending_sig, OP_EQ, NULL);
   CHECK_OK(EX_EI_GOOD_ED_EI);
-  tt_assert(!ei->pending_sig);
+  tt_ptr_op(ei->pending_sig, OP_EQ, NULL);
 
   CHECK_FAIL(EX_EI_BAD_SIG1,1);
   CHECK_FAIL(EX_EI_BAD_SIG2,1);
@@ -660,7 +720,7 @@ test_dir_extrainfo_parsing(void *arg)
   escaped(NULL);
   extrainfo_free(ei);
   routerinfo_free(ri);
-  digestmap_free((digestmap_t*)map, routerinfo_free_wrapper_);
+  digestmap_free_((digestmap_t*)map, routerinfo_free_wrapper_);
 }
 
 static void
@@ -678,16 +738,16 @@ test_dir_parse_router_list(void *arg)
   routerinfo_t *ri = NULL;
   char d[DIGEST_LEN];
 
-  smartlist_add(chunks, tor_strdup(EX_RI_MINIMAL));     // ri 0
-  smartlist_add(chunks, tor_strdup(EX_RI_BAD_PORTS));   // bad ri 0
-  smartlist_add(chunks, tor_strdup(EX_EI_MAXIMAL));     // ei 0
-  smartlist_add(chunks, tor_strdup(EX_EI_BAD_SIG2));    // bad ei --
-  smartlist_add(chunks, tor_strdup(EX_EI_BAD_NICKNAME));// bad ei 0
-  smartlist_add(chunks, tor_strdup(EX_RI_BAD_SIG1));    // bad ri --
-  smartlist_add(chunks, tor_strdup(EX_EI_BAD_PUBLISHED));  // bad ei 1
-  smartlist_add(chunks, tor_strdup(EX_RI_MAXIMAL));     // ri 1
-  smartlist_add(chunks, tor_strdup(EX_RI_BAD_FAMILY));  // bad ri 1
-  smartlist_add(chunks, tor_strdup(EX_EI_MINIMAL));     // ei 1
+  smartlist_add_strdup(chunks, EX_RI_MINIMAL);     // ri 0
+  smartlist_add_strdup(chunks, EX_RI_BAD_PORTS);   // bad ri 0
+  smartlist_add_strdup(chunks, EX_EI_MAXIMAL);     // ei 0
+  smartlist_add_strdup(chunks, EX_EI_BAD_SIG2);    // bad ei --
+  smartlist_add_strdup(chunks, EX_EI_BAD_NICKNAME);// bad ei 0
+  smartlist_add_strdup(chunks, EX_RI_BAD_SIG1);    // bad ri --
+  smartlist_add_strdup(chunks, EX_EI_BAD_PUBLISHED);  // bad ei 1
+  smartlist_add_strdup(chunks, EX_RI_MAXIMAL);     // ri 1
+  smartlist_add_strdup(chunks, EX_RI_BAD_FAMILY);  // bad ri 1
+  smartlist_add_strdup(chunks, EX_EI_MINIMAL);     // ei 1
 
   list = smartlist_join_strings(chunks, "", 0, NULL);
 
@@ -756,7 +816,7 @@ test_dir_parse_router_list(void *arg)
   smartlist_free(chunks);
   routerinfo_free(ri);
   if (map) {
-    digestmap_free((digestmap_t*)map, routerinfo_free_wrapper_);
+    digestmap_free_((digestmap_t*)map, routerinfo_free_wrapper_);
     router_get_routerlist()->identity_map =
       (struct digest_ri_map_t*)digestmap_new();
   }
@@ -812,19 +872,19 @@ test_dir_load_routers(void *arg)
 #define ADD(str)                                                        \
   do {                                                                  \
     tt_int_op(0,OP_EQ,router_get_router_hash(str, strlen(str), buf));      \
-    smartlist_add(wanted, tor_strdup(hex_str(buf, DIGEST_LEN)));        \
+    smartlist_add_strdup(wanted, hex_str(buf, DIGEST_LEN));        \
   } while (0)
 
   MOCK(router_get_dl_status_by_descriptor_digest, mock_router_get_dl_status);
 
   update_approx_time(1412510400);
 
-  smartlist_add(chunks, tor_strdup(EX_RI_MINIMAL));
-  smartlist_add(chunks, tor_strdup(EX_RI_BAD_FINGERPRINT));
-  smartlist_add(chunks, tor_strdup(EX_RI_BAD_SIG2));
-  smartlist_add(chunks, tor_strdup(EX_RI_MAXIMAL));
-  smartlist_add(chunks, tor_strdup(EX_RI_BAD_PORTS));
-  smartlist_add(chunks, tor_strdup(EX_RI_BAD_TOKENS));
+  smartlist_add_strdup(chunks, EX_RI_MINIMAL);
+  smartlist_add_strdup(chunks, EX_RI_BAD_FINGERPRINT);
+  smartlist_add_strdup(chunks, EX_RI_BAD_SIG2);
+  smartlist_add_strdup(chunks, EX_RI_MAXIMAL);
+  smartlist_add_strdup(chunks, EX_RI_BAD_PORTS);
+  smartlist_add_strdup(chunks, EX_RI_BAD_TOKENS);
 
   /* not ADDing MINIMIAL */
   ADD(EX_RI_MAXIMAL);
@@ -909,6 +969,23 @@ mock_get_by_ei_desc_digest(const char *d)
   }
 }
 
+static signed_descriptor_t *
+mock_ei_get_by_ei_digest(const char *d)
+{
+  char hex[HEX_DIGEST_LEN+1];
+  base16_encode(hex, sizeof(hex), d, DIGEST_LEN);
+  signed_descriptor_t *sd = &sd_ei_minimal;
+
+  if (!strcmp(hex, "11E0EDF526950739F7769810FCACAB8C882FAEEE")) {
+    sd->signed_descriptor_body = (char *)EX_EI_MINIMAL;
+    sd->signed_descriptor_len = sizeof(EX_EI_MINIMAL);
+    sd->annotations_len = 0;
+    sd->saved_location = SAVED_NOWHERE;
+    return sd;
+  }
+  return NULL;
+}
+
 static smartlist_t *mock_ei_insert_list = NULL;
 static was_router_added_t
 mock_ei_insert(routerlist_t *rl, extrainfo_t *ei, int warn_if_incompatible)
@@ -932,18 +1009,18 @@ test_dir_load_extrainfo(void *arg)
 #define ADD(str)                                                        \
   do {                                                                  \
     tt_int_op(0,OP_EQ,router_get_extrainfo_hash(str, strlen(str), buf));   \
-    smartlist_add(wanted, tor_strdup(hex_str(buf, DIGEST_LEN)));        \
+    smartlist_add_strdup(wanted, hex_str(buf, DIGEST_LEN));        \
   } while (0)
 
   mock_ei_insert_list = smartlist_new();
   MOCK(router_get_by_extrainfo_digest, mock_get_by_ei_desc_digest);
   MOCK(extrainfo_insert, mock_ei_insert);
 
-  smartlist_add(chunks, tor_strdup(EX_EI_MINIMAL));
-  smartlist_add(chunks, tor_strdup(EX_EI_BAD_NICKNAME));
-  smartlist_add(chunks, tor_strdup(EX_EI_MAXIMAL));
-  smartlist_add(chunks, tor_strdup(EX_EI_BAD_PUBLISHED));
-  smartlist_add(chunks, tor_strdup(EX_EI_BAD_TOKENS));
+  smartlist_add_strdup(chunks, EX_EI_MINIMAL);
+  smartlist_add_strdup(chunks, EX_EI_BAD_NICKNAME);
+  smartlist_add_strdup(chunks, EX_EI_MAXIMAL);
+  smartlist_add_strdup(chunks, EX_EI_BAD_PUBLISHED);
+  smartlist_add_strdup(chunks, EX_EI_BAD_TOKENS);
 
   /* not ADDing MINIMIAL */
   ADD(EX_EI_MAXIMAL);
@@ -998,6 +1075,37 @@ test_dir_load_extrainfo(void *arg)
 }
 
 static void
+test_dir_getinfo_extra(void *arg)
+{
+  int r;
+  char *answer = NULL;
+  const char *errmsg = NULL;
+
+  (void)arg;
+  MOCK(extrainfo_get_by_descriptor_digest, mock_ei_get_by_ei_digest);
+  r = getinfo_helper_dir(NULL, "extra-info/digest/"
+                         "11E0EDF526950739F7769810FCACAB8C882FAEEE", &answer,
+                         &errmsg);
+  tt_int_op(0, OP_EQ, r);
+  tt_ptr_op(NULL, OP_EQ, errmsg);
+  tt_str_op(answer, OP_EQ, EX_EI_MINIMAL);
+  tor_free(answer);
+
+  answer = NULL;
+  r = getinfo_helper_dir(NULL, "extra-info/digest/"
+                         "NOTAVALIDHEXSTRINGNOTAVALIDHEXSTRINGNOTA", &answer,
+                         &errmsg);
+  tt_int_op(0, OP_EQ, r);
+  /* getinfo_helper_dir() should maybe return an error here but doesn't */
+  tt_ptr_op(NULL, OP_EQ, errmsg);
+  /* In any case, there should be no answer for an invalid hex string. */
+  tt_ptr_op(NULL, OP_EQ, answer);
+
+ done:
+  UNMOCK(extrainfo_get_by_descriptor_digest);
+}
+
+static void
 test_dir_versions(void *arg)
 {
   tor_version_t ver1;
@@ -1064,6 +1172,7 @@ test_dir_versions(void *arg)
   tt_int_op(0, OP_EQ, ver1.patchlevel);
   tt_int_op(VER_RELEASE, OP_EQ, ver1.status);
   tt_str_op("alpha", OP_EQ, ver1.status_tag);
+  /* Go through the full set of status tags */
   tt_int_op(0, OP_EQ, tor_version_parse("2.1.700-alpha", &ver1));
   tt_int_op(2, OP_EQ, ver1.major);
   tt_int_op(1, OP_EQ, ver1.minor);
@@ -1078,6 +1187,60 @@ test_dir_versions(void *arg)
   tt_int_op(0, OP_EQ, ver1.patchlevel);
   tt_int_op(VER_RELEASE, OP_EQ, ver1.status);
   tt_str_op("alpha-dev", OP_EQ, ver1.status_tag);
+  tt_int_op(0, OP_EQ, tor_version_parse("0.2.9.5-rc", &ver1));
+  tt_int_op(0, OP_EQ, ver1.major);
+  tt_int_op(2, OP_EQ, ver1.minor);
+  tt_int_op(9, OP_EQ, ver1.micro);
+  tt_int_op(5, OP_EQ, ver1.patchlevel);
+  tt_int_op(VER_RELEASE, OP_EQ, ver1.status);
+  tt_str_op("rc", OP_EQ, ver1.status_tag);
+  tt_int_op(0, OP_EQ, tor_version_parse("0.2.9.6-rc-dev", &ver1));
+  tt_int_op(0, OP_EQ, ver1.major);
+  tt_int_op(2, OP_EQ, ver1.minor);
+  tt_int_op(9, OP_EQ, ver1.micro);
+  tt_int_op(6, OP_EQ, ver1.patchlevel);
+  tt_int_op(VER_RELEASE, OP_EQ, ver1.status);
+  tt_str_op("rc-dev", OP_EQ, ver1.status_tag);
+  tt_int_op(0, OP_EQ, tor_version_parse("0.2.9.8", &ver1));
+  tt_int_op(0, OP_EQ, ver1.major);
+  tt_int_op(2, OP_EQ, ver1.minor);
+  tt_int_op(9, OP_EQ, ver1.micro);
+  tt_int_op(8, OP_EQ, ver1.patchlevel);
+  tt_int_op(VER_RELEASE, OP_EQ, ver1.status);
+  tt_str_op("", OP_EQ, ver1.status_tag);
+  tt_int_op(0, OP_EQ, tor_version_parse("0.2.9.9-dev", &ver1));
+  tt_int_op(0, OP_EQ, ver1.major);
+  tt_int_op(2, OP_EQ, ver1.minor);
+  tt_int_op(9, OP_EQ, ver1.micro);
+  tt_int_op(9, OP_EQ, ver1.patchlevel);
+  tt_int_op(VER_RELEASE, OP_EQ, ver1.status);
+  tt_str_op("dev", OP_EQ, ver1.status_tag);
+  /* In #21450, we fixed an inconsistency in parsing versions > INT32_MAX
+   * between i386 and x86_64, as we used tor_parse_long, and then cast to int
+   */
+  tt_int_op(0, OP_EQ, tor_version_parse("0.2147483647.0", &ver1));
+  tt_int_op(0, OP_EQ, ver1.major);
+  tt_int_op(2147483647, OP_EQ, ver1.minor);
+  tt_int_op(0, OP_EQ, ver1.micro);
+  tt_int_op(0, OP_EQ, ver1.patchlevel);
+  tt_int_op(VER_RELEASE, OP_EQ, ver1.status);
+  tt_str_op("", OP_EQ, ver1.status_tag);
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.2147483648.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.4294967295.0", &ver1));
+  /* In #21278, we reject negative version components */
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.-1.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.-2147483648.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.-4294967295.0", &ver1));
+  /* In #21507, we reject version components with non-numeric prefixes */
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.-0.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("+1.0.0", &ver1));
+  /* use the list in isspace() */
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.\t0.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.\n0.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.\v0.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.\f0.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.\r0.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0. 0.0", &ver1));
 
 #define tt_versionstatus_op(vs1, op, vs2)                               \
   tt_assert_test_type(vs1,vs2,#vs1" "#op" "#vs2,version_status_t,       \
@@ -1097,6 +1260,7 @@ test_dir_versions(void *arg)
   test_v_i_o(VS_RECOMMENDED, "0.0.7rc2", "0.0.7,Tor 0.0.7rc2,Tor 0.0.8");
   test_v_i_o(VS_OLD, "0.0.5.0", "0.0.5.1-cvs");
   test_v_i_o(VS_NEW_IN_SERIES, "0.0.5.1-cvs", "0.0.5, 0.0.6");
+  test_v_i_o(VS_NEW, "0.2.9.9-dev", "0.2.9.9");
   /* Not on list, but newer than any in same series. */
   test_v_i_o(VS_NEW_IN_SERIES, "0.1.0.3",
              "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0");
@@ -1135,6 +1299,70 @@ test_dir_versions(void *arg)
                                    "Tor 0.2.1.0-dev (r99)"));
   tt_int_op(1,OP_EQ, tor_version_as_new_as("Tor 0.2.1.1",
                                    "Tor 0.2.1.0-dev (r99)"));
+  /* And git revisions */
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                                        "Tor 0.2.9.9 (git-56788a2489127072)",
+                                        "Tor 0.2.9.9 (git-56788a2489127072)"));
+  /* a git revision is newer than no git revision */
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                                        "Tor 0.2.9.9 (git-56788a2489127072)",
+                                        "Tor 0.2.9.9"));
+  /* a longer git revision is newer than a shorter git revision
+   * this should be true if they prefix-match, but if they don't, they are
+   * incomparable, because hashes aren't ordered (but we compare their bytes
+   * anyway) */
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                  "Tor 0.2.9.9 (git-56788a2489127072d513cf4baf35a8ff475f3c7b)",
+                  "Tor 0.2.9.9 (git-56788a2489127072)"));
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                                        "Tor 0.2.9.9 (git-0102)",
+                                        "Tor 0.2.9.9 (git-03)"));
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                                        "Tor 0.2.9.9 (git-0102)",
+                                        "Tor 0.2.9.9 (git-00)"));
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                                           "Tor 0.2.9.9 (git-01)",
+                                           "Tor 0.2.9.9 (git-00)"));
+  tt_int_op(0,OP_EQ, tor_version_as_new_as(
+                                           "Tor 0.2.9.9 (git-00)",
+                                           "Tor 0.2.9.9 (git-01)"));
+  /* In #21278, we compare without integer overflows.
+   * But since #21450 limits version components to [0, INT32_MAX], it is no
+   * longer possible to cause an integer overflow in tor_version_compare() */
+  tt_int_op(0,OP_EQ, tor_version_as_new_as(
+                                           "Tor 0.0.0.0",
+                                           "Tor 2147483647.0.0.0"));
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                                           "Tor 2147483647.0.0.0",
+                                           "Tor 0.0.0.0"));
+  /* These versions used to cause an overflow, now they don't parse
+   * (and authorities reject their descriptors), and log a BUG message */
+  setup_full_capture_of_logs(LOG_WARN);
+  tt_int_op(0,OP_EQ, tor_version_as_new_as(
+                                           "Tor 0.0.0.0",
+                                           "Tor 0.-2147483648.0.0"));
+  expect_single_log_msg_containing("unparseable");
+  mock_clean_saved_logs();
+  tt_int_op(0,OP_EQ, tor_version_as_new_as(
+                                           "Tor 0.2147483647.0.0",
+                                           "Tor 0.-1.0.0"));
+  expect_single_log_msg_containing("unparseable");
+  mock_clean_saved_logs();
+  tt_int_op(0,OP_EQ, tor_version_as_new_as(
+                                           "Tor 0.2147483647.0.0",
+                                           "Tor 0.-2147483648.0.0"));
+  expect_single_log_msg_containing("unparseable");
+  mock_clean_saved_logs();
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                                           "Tor 4294967295.0.0.0",
+                                           "Tor 0.0.0.0"));
+  expect_no_log_entry();
+  tt_int_op(0,OP_EQ, tor_version_as_new_as(
+                                           "Tor 0.4294967295.0.0",
+                                           "Tor 0.-4294967295.0.0"));
+  expect_single_log_msg_containing("unparseable");
+  mock_clean_saved_logs();
+  teardown_capture_of_logs();
 
   /* Now try git revisions */
   tt_int_op(0,OP_EQ, tor_version_parse("0.5.6.7 (git-ff00ff)", &ver1));
@@ -1144,11 +1372,24 @@ test_dir_versions(void *arg)
   tt_int_op(7,OP_EQ, ver1.patchlevel);
   tt_int_op(3,OP_EQ, ver1.git_tag_len);
   tt_mem_op(ver1.git_tag,OP_EQ, "\xff\x00\xff", 3);
+  /* reject bad hex digits */
   tt_int_op(-1,OP_EQ, tor_version_parse("0.5.6.7 (git-ff00xx)", &ver1));
+  /* reject odd hex digit count */
   tt_int_op(-1,OP_EQ, tor_version_parse("0.5.6.7 (git-ff00fff)", &ver1));
+  /* ignore "git " */
   tt_int_op(0,OP_EQ, tor_version_parse("0.5.6.7 (git ff00fff)", &ver1));
+  /* standard length is 16 hex digits */
+  tt_int_op(0,OP_EQ, tor_version_parse("0.5.6.7 (git-0010203040506070)",
+                                       &ver1));
+  /* length limit is 40 hex digits */
+  tt_int_op(0,OP_EQ, tor_version_parse(
+                     "0.5.6.7 (git-000102030405060708090a0b0c0d0e0f10111213)",
+                     &ver1));
+  tt_int_op(-1,OP_EQ, tor_version_parse(
+                    "0.5.6.7 (git-000102030405060708090a0b0c0d0e0f1011121314)",
+                    &ver1));
  done:
-  ;
+  teardown_capture_of_logs();
 }
 
 /** Run unit tests for directory fp_pair functions. */
@@ -1314,6 +1555,26 @@ test_dir_measured_bw_kb(void *arg)
                 "bw=1024 junk=007\n",
     "misc=junk node_id=$557365204145532d32353620696e73746561642e  "
                 "bw=1024 junk=007\n",
+    /* check whether node_id can be at the end */
+    "bw=1024 node_id=$557365204145532d32353620696e73746561642e\n",
+    /* check whether node_id can be at the end and bw has something in front*/
+    "foo=bar bw=1024 node_id=$557365204145532d32353620696e73746561642e\n",
+    /* check whether node_id can be at the end and something in the
+     * in the middle of bw and node_id */
+    "bw=1024 foo=bar node_id=$557365204145532d32353620696e73746561642e\n",
+
+    /* Test that a line with vote=1 will pass. */
+    "node_id=$557365204145532d32353620696e73746561642e bw=1024 vote=1\n",
+    /* Test that a line with unmeasured=1 will pass. */
+    "node_id=$557365204145532d32353620696e73746561642e bw=1024 unmeasured=1\n",
+    /* Test that a line with vote=1 and unmeasured=1 will pass. */
+    "node_id=$557365204145532d32353620696e73746561642e bw=1024 vote=1"
+    "unmeasured=1\n",
+    /* Test that a line with unmeasured=0 will pass. */
+    "node_id=$557365204145532d32353620696e73746561642e bw=1024 unmeasured=0\n",
+    /* Test that a line with vote=1 and unmeasured=0 will pass. */
+    "node_id=$557365204145532d32353620696e73746561642e bw=1024 vote=1"
+    "unmeasured=0\n",
     "end"
   };
   const char *lines_fail[] = {
@@ -1347,18 +1608,30 @@ test_dir_measured_bw_kb(void *arg)
     "node_id=$55736520414552d32353620696e73746561642e bw=1024\n",
     "node_id=557365204145532d32353620696e73746561642e bw=1024\n",
     "node_id= $557365204145532d32353620696e73746561642e bw=0.23\n",
+
+    /* Test that a line with vote=0 will fail too, so that it is ignored. */
+    "node_id=$557365204145532d32353620696e73746561642e bw=1024 vote=0\n",
+    /* Test that a line with vote=0 will fail even if unmeasured=0. */
+    "node_id=$557365204145532d32353620696e73746561642e bw=1024 vote=0 "
+    "unmeasured=0\n",
     "end"
   };
 
   (void)arg;
   for (i = 0; strcmp(lines_fail[i], "end"); i++) {
     //fprintf(stderr, "Testing: %s\n", lines_fail[i]);
-    tt_assert(measured_bw_line_parse(&mbwl, lines_fail[i]) == -1);
+    /* Testing only with line_is_after_headers = 1. Tests with
+     * line_is_after_headers = 0 in
+     * test_dir_measured_bw_kb_line_is_after_headers */
+    tt_assert(measured_bw_line_parse(&mbwl, lines_fail[i], 1) == -1);
   }
 
   for (i = 0; strcmp(lines_pass[i], "end"); i++) {
     //fprintf(stderr, "Testing: %s %d\n", lines_pass[i], TOR_ISSPACE('\n'));
-    tt_assert(measured_bw_line_parse(&mbwl, lines_pass[i]) == 0);
+    /* Testing only with line_is_after_headers = 1. Tests with
+     * line_is_after_headers = 0 in
+     * test_dir_measured_bw_kb_line_is_after_headers */
+    tt_assert(measured_bw_line_parse(&mbwl, lines_pass[i], 1) == 0);
     tt_assert(mbwl.bw_kb == 1024);
     tt_assert(strcmp(mbwl.node_hex,
                 "557365204145532d32353620696e73746561642e") == 0);
@@ -1368,23 +1641,410 @@ test_dir_measured_bw_kb(void *arg)
   return;
 }
 
-/* Test dirserv_read_measured_bandwidths */
+/* Unit tests for measured_bw_line_parse using line_is_after_headers flag.
+ * When the end of the header is detected (a first complete bw line is parsed),
+ * incomplete lines fail and give warnings, but do not give warnings if
+ * the header is not ended, allowing to ignore additional header lines. */
 static void
-test_dir_dirserv_read_measured_bandwidths(void *arg)
+test_dir_measured_bw_kb_line_is_after_headers(void *arg)
 {
-  char *fname=NULL;
   (void)arg;
+  measured_bw_line_t mbwl;
+  const char *line_pass = \
+    "node_id=$557365204145532d32353620696e73746561642e bw=1024\n";
+  int i;
+  const char *lines_fail[] = {
+    "node_id=$557365204145532d32353620696e73746561642e \n",
+    "bw=1024\n",
+    "rtt=300\n",
+    "end"
+  };
+
+  setup_capture_of_logs(LOG_DEBUG);
+
+  /* Test bw lines when header has ended */
+  for (i = 0; strcmp(lines_fail[i], "end"); i++) {
+    tt_assert(measured_bw_line_parse(&mbwl, lines_fail[i], 1) == -1);
+    expect_log_msg_containing("Incomplete line in bandwidth file:");
+    mock_clean_saved_logs();
+  }
 
-  fname = tor_strdup(get_fname("V3BandwidthsFile"));
-  /* Test an empty file */
+  tt_assert(measured_bw_line_parse(&mbwl, line_pass, 1) == 0);
+
+  /* Test bw lines when header has not ended */
+  for (i = 0; strcmp(lines_fail[i], "end"); i++) {
+    tt_assert(measured_bw_line_parse(&mbwl, lines_fail[i], 0) == -1);
+    expect_log_msg_containing("Missing bw or node_id in bandwidth file line:");
+    mock_clean_saved_logs();
+  }
+
+  tt_assert(measured_bw_line_parse(&mbwl, line_pass, 0) == 0);
+
+ done:
+  teardown_capture_of_logs();
+}
+
+/* Test dirserv_read_measured_bandwidths with headers and complete files. */
+static void
+test_dir_dirserv_read_measured_bandwidths(void *arg)
+{
+  (void)arg;
+  char *content = NULL;
+  time_t timestamp = time(NULL);
+  char *fname = tor_strdup(get_fname("V3BandwidthsFile"));
+  smartlist_t *bw_file_headers = smartlist_new();
+  /* bw file strings in vote */
+  char *bw_file_headers_str = NULL;
+  char *bw_file_headers_str_v100 = NULL;
+  char *bw_file_headers_str_v110 = NULL;
+  char *bw_file_headers_str_bad = NULL;
+  char *bw_file_headers_str_extra = NULL;
+  char bw_file_headers_str_long[MAX_BW_FILE_HEADER_COUNT_IN_VOTE * 8 + 1] = "";
+  /* string header lines in bw file */
+  char *header_lines_v100 = NULL;
+  char *header_lines_v110_no_terminator = NULL;
+  char *header_lines_v110 = NULL;
+  char header_lines_long[MAX_BW_FILE_HEADER_COUNT_IN_VOTE * 8 + 1] = "";
+  int i;
+  const char *header_lines_v110_no_terminator_no_timestamp =
+    "version=1.1.0\n"
+    "software=sbws\n"
+    "software_version=0.1.0\n"
+    "earliest_bandwidth=2018-05-08T16:13:26\n"
+    "file_created=2018-04-16T21:49:18\n"
+    "generator_started=2018-05-08T16:13:25\n"
+    "latest_bandwidth=2018-04-16T20:49:18\n";
+  const char *bw_file_headers_str_v110_no_timestamp =
+    "version=1.1.0 software=sbws "
+    "software_version=0.1.0 "
+    "earliest_bandwidth=2018-05-08T16:13:26 "
+    "file_created=2018-04-16T21:49:18 "
+    "generator_started=2018-05-08T16:13:25 "
+    "latest_bandwidth=2018-04-16T20:49:18";
+  const char *relay_lines_v100 =
+    "node_id=$557365204145532d32353620696e73746561642e bw=1024 "
+    "nick=Test measured_at=1523911725 updated_at=1523911725 "
+    "pid_error=4.11374090719 pid_error_sum=4.11374090719 "
+    "pid_bw=57136645 pid_delta=2.12168374577 circ_fail=0.2 "
+    "scanner=/filepath\n";
+  const char *relay_lines_v110 =
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 "
+    "master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ "
+    "bw=760 nick=Test rtt=380 time=2018-05-08T16:13:26\n";
+  const char *relay_lines_bad =
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A\n";
+
+  tor_asprintf(&header_lines_v100, "%ld\n", (long)timestamp);
+  tor_asprintf(&header_lines_v110_no_terminator, "%ld\n%s", (long)timestamp,
+               header_lines_v110_no_terminator_no_timestamp);
+  tor_asprintf(&header_lines_v110, "%s%s",
+               header_lines_v110_no_terminator, BW_FILE_HEADERS_TERMINATOR);
+
+  tor_asprintf(&bw_file_headers_str_v100, "timestamp=%ld",(long)timestamp);
+  tor_asprintf(&bw_file_headers_str_v110, "timestamp=%ld %s",
+               (long)timestamp, bw_file_headers_str_v110_no_timestamp);
+  tor_asprintf(&bw_file_headers_str_bad, "%s "
+               "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A",
+               bw_file_headers_str_v110);
+
+  for (i=0; i<MAX_BW_FILE_HEADER_COUNT_IN_VOTE; i++) {
+    strlcat(header_lines_long, "foo=bar\n",
+            sizeof(header_lines_long));
+  }
+  /* 8 is the number of v110 lines in header_lines_v110 */
+  for (i=0; i<MAX_BW_FILE_HEADER_COUNT_IN_VOTE - 8 - 1; i++) {
+    strlcat(bw_file_headers_str_long, "foo=bar ",
+            sizeof(bw_file_headers_str_long));
+  }
+  strlcat(bw_file_headers_str_long, "foo=bar",
+          sizeof(bw_file_headers_str_long));
+  tor_asprintf(&bw_file_headers_str_extra,
+               "%s %s",
+               bw_file_headers_str_v110,
+               bw_file_headers_str_long);
+
+  /* Test an empty bandwidth file. bw_file_headers will be empty string */
   write_str_to_file(fname, "", 0);
   setup_capture_of_logs(LOG_WARN);
-  tt_int_op(-1, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL));
+  tt_int_op(-1, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
+                                                        bw_file_headers));
   expect_log_msg("Empty bandwidth file\n");
+  teardown_capture_of_logs();
+  bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
+  tt_str_op("", OP_EQ, bw_file_headers_str);
+  SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
+  smartlist_free(bw_file_headers);
+  tor_free(bw_file_headers_str);
+
+  /* Test bandwidth file with only timestamp.
+   * bw_file_headers will be empty string */
+  bw_file_headers = smartlist_new();
+  tor_asprintf(&content, "%ld", (long)timestamp);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+  tt_int_op(-1, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
+                                                        bw_file_headers));
+  bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
+  tt_str_op("", OP_EQ, bw_file_headers_str);
+  SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
+  smartlist_free(bw_file_headers);
+  tor_free(bw_file_headers_str);
+
+  /* Test v1.0.0 bandwidth file headers */
+  write_str_to_file(fname, header_lines_v100, 0);
+  bw_file_headers = smartlist_new();
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
+                                                       bw_file_headers));
+  bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
+  tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str);
+  SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
+  smartlist_free(bw_file_headers);
+  tor_free(bw_file_headers_str);
+
+  /* Test v1.0.0 complete bandwidth file */
+  bw_file_headers = smartlist_new();
+  tor_asprintf(&content, "%s%s", header_lines_v100, relay_lines_v100);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
+                                                       bw_file_headers));
+  bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
+  tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str);
+  SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
+  smartlist_free(bw_file_headers);
+  tor_free(bw_file_headers_str);
+
+  /* Test v1.0.0 complete bandwidth file with NULL bw_file_headers. */
+  tor_asprintf(&content, "%s%s", header_lines_v100, relay_lines_v100);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL));
+
+  /* Test bandwidth file including v1.1.0 bandwidth headers and
+   * v1.0.0 relay lines. bw_file_headers will contain the v1.1.0 headers. */
+  bw_file_headers = smartlist_new();
+  tor_asprintf(&content, "%s%s%s", header_lines_v100, header_lines_v110,
+               relay_lines_v100);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
+                                                       bw_file_headers));
+  bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
+  tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
+  SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
+  smartlist_free(bw_file_headers);
+  tor_free(bw_file_headers_str);
+
+  /* Test v1.0.0 complete bandwidth file with v1.1.0 headers at the end.
+   * bw_file_headers will contain only v1.0.0 headers and the additional
+   * headers will be interpreted as malformed relay lines. */
+  bw_file_headers = smartlist_new();
+  tor_asprintf(&content, "%s%s%s", header_lines_v100, relay_lines_v100,
+               header_lines_v110);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
+                                                       bw_file_headers));
+  bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
+  tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str);
+  SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
+  smartlist_free(bw_file_headers);
+  tor_free(bw_file_headers_str);
+
+  /* Test v1.0.0 complete bandwidth file, the v1.1.0 headers and more relay
+   * lines. bw_file_headers will contain only v1.0.0 headers, the additional
+   * headers will be interpreted as malformed relay lines and the last relay
+   * lines will be correctly interpreted as relay lines. */
+  bw_file_headers = smartlist_new();
+  tor_asprintf(&content, "%s%s%s%s", header_lines_v100, relay_lines_v100,
+               header_lines_v110, relay_lines_v100);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
+                                                       bw_file_headers));
+  bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
+  tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str);
+  SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
+  smartlist_free(bw_file_headers);
+  tor_free(bw_file_headers_str);
+
+  /* Test v1.1.0 bandwidth headers without terminator */
+  bw_file_headers = smartlist_new();
+  write_str_to_file(fname, header_lines_v110_no_terminator, 0);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
+                                                       bw_file_headers));
+  bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
+  tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
+  SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
+  smartlist_free(bw_file_headers);
+  tor_free(bw_file_headers_str);
+
+  /* Test v1.1.0 bandwidth headers with terminator */
+  bw_file_headers = smartlist_new();
+  write_str_to_file(fname, header_lines_v110, 0);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
+                                                       bw_file_headers));
+  bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
+  tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
+  SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
+  smartlist_free(bw_file_headers);
+  tor_free(bw_file_headers_str);
+
+  /* Test v1.1.0 bandwidth file without terminator, then relay lines.
+   * bw_file_headers will contain the v1.1.0 headers. */
+  bw_file_headers = smartlist_new();
+  tor_asprintf(&content, "%s%s",
+               header_lines_v110_no_terminator, relay_lines_v110);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
+                                                       bw_file_headers));
+  bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
+  tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
+  SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
+  smartlist_free(bw_file_headers);
+  tor_free(bw_file_headers_str);
+
+  /* Test v1.1.0 bandwidth headers with terminator, then relay lines
+   * bw_file_headers will contain the v1.1.0 headers. */
+  bw_file_headers = smartlist_new();
+  tor_asprintf(&content, "%s%s",
+               header_lines_v110, relay_lines_v110);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
+                                                       bw_file_headers));
+  bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
+  tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
+  SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
+  smartlist_free(bw_file_headers);
+  tor_free(bw_file_headers_str);
+
+  /* Test v1.1.0 bandwidth headers with terminator, then bad relay lines,
+   * then terminator, then relay_lines_bad.
+   * bw_file_headers will contain the v1.1.0 headers. */
+  bw_file_headers = smartlist_new();
+  tor_asprintf(&content, "%s%s%s%s", header_lines_v110, relay_lines_bad,
+               BW_FILE_HEADERS_TERMINATOR, relay_lines_bad);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
+                                                       bw_file_headers));
+  bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
+  tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
+  SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
+  smartlist_free(bw_file_headers);
+  tor_free(bw_file_headers_str);
+
+  /* Test v1.1.0 bandwidth headers without terminator, then bad relay lines,
+   * then relay lines. bw_file_headers will contain the v1.1.0 headers and
+   * the bad relay lines. */
+  bw_file_headers = smartlist_new();
+  tor_asprintf(&content, "%s%s%s",
+               header_lines_v110_no_terminator, relay_lines_bad,
+               relay_lines_v110);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
+                                                       bw_file_headers));
+  bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
+  tt_str_op(bw_file_headers_str_bad, OP_EQ, bw_file_headers_str);
+  SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
+  smartlist_free(bw_file_headers);
+  tor_free(bw_file_headers_str);
+
+  /* Test v1.1.0 bandwidth headers without terminator,
+   * then many bad relay lines, then relay lines.
+   * bw_file_headers will contain the v1.1.0 headers and the bad relay lines
+   * to a maximum of MAX_BW_FILE_HEADER_COUNT_IN_VOTE header lines. */
+  bw_file_headers = smartlist_new();
+  tor_asprintf(&content, "%s%s%s",
+               header_lines_v110_no_terminator, header_lines_long,
+               relay_lines_v110);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
+                                                       bw_file_headers));
+  tt_int_op(MAX_BW_FILE_HEADER_COUNT_IN_VOTE, OP_EQ,
+            smartlist_len(bw_file_headers));
+  bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
+  tt_str_op(bw_file_headers_str_extra, OP_EQ, bw_file_headers_str);
+  SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
+  smartlist_free(bw_file_headers);
+  tor_free(bw_file_headers_str);
+
+  /* Test v1.1.0 bandwidth headers without terminator,
+   * then many bad relay lines, then relay lines.
+   * bw_file_headers will contain the v1.1.0 headers and the bad relay lines.
+   * Force bw_file_headers to have more than MAX_BW_FILE_HEADER_COUNT_IN_VOTE
+   * This test is needed while there is not dirvote test. */
+  bw_file_headers = smartlist_new();
+  tor_asprintf(&content, "%s%s%s",
+               header_lines_v110_no_terminator, header_lines_long,
+               relay_lines_v110);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
+                                                       bw_file_headers));
+  tt_int_op(MAX_BW_FILE_HEADER_COUNT_IN_VOTE, OP_EQ,
+            smartlist_len(bw_file_headers));
+  /* force bw_file_headers to be bigger than
+   * MAX_BW_FILE_HEADER_COUNT_IN_VOTE */
+  char line[8] = "foo=bar\0";
+  smartlist_add_strdup(bw_file_headers, line);
+  tt_int_op(MAX_BW_FILE_HEADER_COUNT_IN_VOTE, OP_LT,
+            smartlist_len(bw_file_headers));
+  SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
+  smartlist_free(bw_file_headers);
+  tor_free(bw_file_headers_str);
+
+  /* Test v1.x.x bandwidth line with vote=0.
+   * It will be ignored it and logged it at debug level. */
+  const char *relay_lines_ignore =
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 bw=1024 vote=0\n"
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 bw=1024 vote=0"
+    "unmeasured=1\n"
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 bw=1024 vote=0"
+    "unmeasured=0\n";
+
+  /* Create the bandwidth file */
+  tor_asprintf(&content, "%ld\n%s", (long)timestamp, relay_lines_ignore);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+
+  /* Read the bandwidth file */
+  setup_full_capture_of_logs(LOG_DEBUG);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL));
+  expect_log_msg_containing("Ignoring bandwidth file line");
+  teardown_capture_of_logs();
+
+  /* Test v1.x.x bandwidth line with "vote=1" or "unmeasured=1" or
+   * "unmeasured=0".
+   * They will not be ignored. */
+  /* Create the bandwidth file */
+  const char *relay_lines_vote =
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 bw=1024 vote=1\n"
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 bw=1024 unmeasured=0\n"
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 bw=1024 unmeasured=1\n";
+  tor_asprintf(&content, "%ld\n%s", (long)timestamp, relay_lines_vote);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+
+  /* Read the bandwidth file */
+  setup_full_capture_of_logs(LOG_DEBUG);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL));
+  expect_log_msg_not_containing("Ignoring bandwidth file line");
+  teardown_capture_of_logs();
 
  done:
   tor_free(fname);
-  teardown_capture_of_logs();
+  tor_free(header_lines_v100);
+  tor_free(header_lines_v110_no_terminator);
+  tor_free(header_lines_v110);
+  tor_free(bw_file_headers_str_v100);
+  tor_free(bw_file_headers_str_v110);
+  tor_free(bw_file_headers_str_bad);
+  tor_free(bw_file_headers_str_extra);
 }
 
 #define MBWC_INIT_TIME 1000
@@ -1512,6 +2172,15 @@ test_dir_param_voting(void *arg)
   tt_int_op(-8,OP_EQ, networkstatus_get_param(&vote4, "ab", -12, -100, -8));
   tt_int_op(0,OP_EQ, networkstatus_get_param(&vote4, "foobar", 0, -100, 8));
 
+  tt_int_op(100,OP_EQ, networkstatus_get_overridable_param(
+                                        &vote4, -1, "x-yz", 50, 0, 300));
+  tt_int_op(30,OP_EQ, networkstatus_get_overridable_param(
+                                        &vote4, 30, "x-yz", 50, 0, 300));
+  tt_int_op(0,OP_EQ, networkstatus_get_overridable_param(
+                                        &vote4, -101, "foobar", 0, -100, 8));
+  tt_int_op(-99,OP_EQ, networkstatus_get_overridable_param(
+                                        &vote4, -99, "foobar", 0, -100, 8));
+
   smartlist_add(votes, &vote1);
 
   /* Do the first tests without adding all the other votes, for
@@ -1611,7 +2280,7 @@ test_dir_param_voting_lookup(void *arg)
             dirvote_get_intermediate_param_value(lst, "moomin", -100));
   tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
   tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
-            "!(n_found > 1)");
+            "n_found == 0");
   tor_end_capture_bugs_();
   /* There is no 'fred=', so that is treated as not existing. */
   tt_int_op(-100, OP_EQ,
@@ -1699,8 +2368,7 @@ vote_tweaks_for_v3ns(networkstatus_t *v, int voter, time_t now)
     measured_bw_line_t mbw;
     memset(mbw.node_id, 33, sizeof(mbw.node_id));
     mbw.bw_kb = 1024;
-    tt_assert(measured_bw_line_apply(&mbw,
-                v->routerstatus_list) == 1);
+    tt_int_op(measured_bw_line_apply(&mbw, v->routerstatus_list), OP_EQ, 1);
   } else if (voter == 2 || voter == 3) {
     /* Monkey around with the list a bit */
     vrs = smartlist_get(v->routerstatus_list, 2);
@@ -1756,7 +2424,7 @@ test_vrs_for_v3ns(vote_routerstatus_t *vrs, int voter, time_t now)
     tt_int_op(rs->or_port,OP_EQ, 443);
     tt_int_op(rs->dir_port,OP_EQ, 8000);
     /* no flags except "running" (16) and "v2dir" (64) and "valid" (128) */
-    tt_u64_op(vrs->flags, OP_EQ, U64_LITERAL(0xd0));
+    tt_u64_op(vrs->flags, OP_EQ, UINT64_C(0xd0));
   } else if (tor_memeq(rs->identity_digest,
                        "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5"
                        "\x5\x5\x5\x5",
@@ -1782,10 +2450,10 @@ test_vrs_for_v3ns(vote_routerstatus_t *vrs, int voter, time_t now)
     tt_int_op(rs->ipv6_orport,OP_EQ, 4711);
     if (voter == 1) {
       /* all except "authority" (1) */
-      tt_u64_op(vrs->flags, OP_EQ, U64_LITERAL(254));
+      tt_u64_op(vrs->flags, OP_EQ, UINT64_C(254));
     } else {
       /* 1023 - authority(1) - madeofcheese(16) - madeoftin(32) */
-      tt_u64_op(vrs->flags, OP_EQ, U64_LITERAL(974));
+      tt_u64_op(vrs->flags, OP_EQ, UINT64_C(974));
     }
   } else if (tor_memeq(rs->identity_digest,
                        "\x33\x33\x33\x33\x33\x33\x33\x33\x33\x33"
@@ -1816,7 +2484,7 @@ test_consensus_for_v3ns(networkstatus_t *con, time_t now)
   (void)now;
 
   tt_assert(con);
-  tt_assert(!con->cert);
+  tt_ptr_op(con->cert, OP_EQ, NULL);
   tt_int_op(2,OP_EQ, smartlist_len(con->routerstatus_list));
   /* There should be two listed routers: one with identity 3, one with
    * identity 5. */
@@ -1886,13 +2554,260 @@ test_routerstatus_for_v3ns(routerstatus_t *rs, time_t now)
     /* XXXX check version */
   } else {
     /* Weren't expecting this... */
-    tt_assert(0);
+    tt_abort();
   }
 
  done:
   return;
 }
 
+static void
+test_dir_networkstatus_compute_bw_weights_v10(void *arg)
+{
+  (void) arg;
+  smartlist_t *chunks = smartlist_new();
+  int64_t G, M, E, D, T, weight_scale;
+  int ret;
+  weight_scale = 10000;
+
+  /* no case. one or more of the values is 0 */
+  G = M = E = D = 0;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(smartlist_len(chunks), OP_EQ, 0);
+
+  /* case 1 */
+  /* XXX dir-spec not followed? See #20272. If it isn't closed, then this is
+   * testing current behavior, not spec. */
+  G = E = 10;
+  M = D = 1;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_int_op(smartlist_len(chunks), OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=3333 "
+    "Wbe=3000 Wbg=3000 Wbm=10000 Wdb=10000 Web=10000 Wed=3333 Wee=7000 "
+    "Weg=3333 Wem=7000 Wgb=10000 Wgd=3333 Wgg=7000 Wgm=7000 Wmb=10000 "
+    "Wmd=3333 Wme=3000 Wmg=3000 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 2a E scarce */
+  M = 100;
+  G = 20;
+  E = D = 5;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=0 Wbe=0 "
+    "Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=10000 Wee=10000 Weg=10000 "
+    "Wem=10000 Wgb=10000 Wgd=0 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=0 Wme=0 "
+    "Wmg=0 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 2a G scarce */
+  M = 100;
+  E = 20;
+  G = D = 5;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=0 Wbe=0 "
+    "Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=0 Wee=10000 Weg=0 Wem=10000 "
+    "Wgb=10000 Wgd=10000 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=0 Wme=0 Wmg=0 "
+    "Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 2b1 (Wgg=1, Wmd=Wgd) */
+  M = 10;
+  E = 30;
+  G = 10;
+  D = 100;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=4000 "
+    "Wbe=0 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=2000 Wee=10000 Weg=2000 "
+    "Wem=10000 Wgb=10000 Wgd=4000 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=4000 "
+    "Wme=0 Wmg=0 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 2b2 */
+  M = 60;
+  E = 30;
+  G = 10;
+  D = 100;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=666 Wbe=0 "
+    "Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=3666 Wee=10000 Weg=3666 "
+    "Wem=10000 Wgb=10000 Wgd=5668 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=666 "
+    "Wme=0 Wmg=0 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 2b3 */
+  /* XXX I can't get a combination of values that hits this case without error,
+   * so this just tests that it fails. See #20285. Also see #20284 as 2b3 does
+   * not follow dir-spec. */
+  /* (E < T/3 && G < T/3) && (E+D>=G || G+D>=E) && (M > T/3) */
+  M = 80;
+  E = 30;
+  G = 30;
+  D = 30;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 0);
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 3a G scarce */
+  M = 10;
+  E = 30;
+  G = 10;
+  D = 5;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=0 "
+    "Wbe=3333 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=0 Wee=6667 Weg=0 "
+    "Wem=6667 Wgb=10000 Wgd=10000 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=0 "
+    "Wme=3333 Wmg=0 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 3a E scarce */
+  M = 10;
+  E = 10;
+  G = 30;
+  D = 5;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=0 Wbe=0 "
+    "Wbg=3333 Wbm=10000 Wdb=10000 Web=10000 Wed=10000 Wee=10000 Weg=10000 "
+    "Wem=10000 Wgb=10000 Wgd=0 Wgg=6667 Wgm=6667 Wmb=10000 Wmd=0 Wme=0 "
+    "Wmg=3333 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 3bg */
+  M = 10;
+  E = 30;
+  G = 10;
+  D = 10;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=0 "
+    "Wbe=3334 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=0 Wee=6666 Weg=0 "
+    "Wem=6666 Wgb=10000 Wgd=10000 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=0 "
+    "Wme=3334 Wmg=0 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 3be */
+  M = 10;
+  E = 10;
+  G = 30;
+  D = 10;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=0 Wbe=0 "
+    "Wbg=3334 Wbm=10000 Wdb=10000 Web=10000 Wed=10000 Wee=10000 Weg=10000 "
+    "Wem=10000 Wgb=10000 Wgd=0 Wgg=6666 Wgm=6666 Wmb=10000 Wmd=0 Wme=0 "
+    "Wmg=3334 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case from 21 Jul 2013 (3be) */
+  G = 5483409;
+  M = 1455379;
+  E = 980834;
+  D = 3385803;
+  T = 11305425;
+  tt_i64_op(G+M+E+D, OP_EQ, T);
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_assert(ret);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=883 Wbe=0 "
+    "Wbg=3673 Wbm=10000 Wdb=10000 Web=10000 Wed=8233 Wee=10000 Weg=8233 "
+    "Wem=10000 Wgb=10000 Wgd=883 Wgg=6327 Wgm=6327 Wmb=10000 Wmd=883 Wme=0 "
+    "Wmg=3673 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case from 04 Oct 2016 (3a E scarce) */
+  G=29322240;
+  M=4721546;
+  E=1522058;
+  D=9273571;
+  T=44839415;
+  tt_i64_op(G+M+E+D, OP_EQ, T);
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_assert(ret);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=0 Wbe=0 "
+    "Wbg=4194 Wbm=10000 Wdb=10000 Web=10000 Wed=10000 Wee=10000 Weg=10000 "
+    "Wem=10000 Wgb=10000 Wgd=0 Wgg=5806 Wgm=5806 Wmb=10000 Wmd=0 Wme=0 "
+    "Wmg=4194 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case from 04 Sep 2013 (2b1) */
+  G=3091352;
+  M=1838837;
+  E=2109300;
+  D=2469369;
+  T=9508858;
+  tt_i64_op(G+M+E+D, OP_EQ, T);
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_assert(ret);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=317 "
+    "Wbe=5938 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=9366 Wee=4061 "
+    "Weg=9366 Wem=4061 Wgb=10000 Wgd=317 Wgg=10000 Wgm=10000 Wmb=10000 "
+    "Wmd=317 Wme=5938 Wmg=0 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* explicitly test initializing weights to 1*/
+  G=1;
+  M=1;
+  E=1;
+  D=1;
+  T=4;
+  tt_i64_op(G+M+E+D, OP_EQ, T);
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=3333 "
+    "Wbe=0 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=3333 Wee=10000 Weg=3333 "
+    "Wem=10000 Wgb=10000 Wgd=3333 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=3333 "
+    "Wme=0 Wmg=0 Wmm=10000\n");
+  tt_assert(ret);
+
+ done:
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_free(chunks);
+}
+
 static authority_cert_t *mock_cert;
 
 static authority_cert_t *
@@ -1958,6 +2873,7 @@ test_a_networkstatus(
   sign_skey_2 = crypto_pk_new();
   sign_skey_3 = crypto_pk_new();
   sign_skey_leg1 = pk_generate(4);
+  voting_schedule_recalculate_timing(get_options(), now);
   sr_state_init(0, 0);
 
   tt_assert(!crypto_pk_read_private_key_from_string(sign_skey_1,
@@ -2347,8 +3263,8 @@ test_dir_scale_bw(void *testdata)
   for (i=0; i<8; ++i) {
     total += vals_u64[i];
   }
-  tt_assert(total >= (U64_LITERAL(1)<<60));
-  tt_assert(total <= (U64_LITERAL(1)<<62));
+  tt_assert(total >= (UINT64_C(1)<<60));
+  tt_assert(total <= (UINT64_C(1)<<62));
 
   for (i=0; i<8; ++i) {
     /* vals[2].u64 is the scaled value of 1.0 */
@@ -2495,8 +3411,9 @@ gen_routerstatus_for_umbw(int idx, time_t now)
       rs->addr = 0x99008801;
       rs->or_port = 443;
       rs->dir_port = 8000;
-      /* all flags but running cleared */
+      /* all flags but running and valid cleared */
       rs->is_flagged_running = 1;
+      rs->is_valid = 1;
       /*
        * This one has measured bandwidth below the clip cutoff, and
        * so shouldn't be clipped; we'll have to test that it isn't
@@ -2569,8 +3486,9 @@ gen_routerstatus_for_umbw(int idx, time_t now)
       rs->addr = 0xC0000203;
       rs->or_port = 500;
       rs->dir_port = 1999;
-      /* all flags but running cleared */
+      /* all flags but running and valid cleared */
       rs->is_flagged_running = 1;
+      rs->is_valid = 1;
       /*
        * This one has unmeasured bandwidth below the clip cutoff, and
        * so shouldn't be clipped; we'll have to test that it isn't
@@ -2587,12 +3505,12 @@ gen_routerstatus_for_umbw(int idx, time_t now)
       break;
     default:
       /* Shouldn't happen */
-      tt_assert(0);
+      tt_abort();
   }
   if (vrs) {
     vrs->microdesc = tor_malloc_zero(sizeof(vote_microdesc_hash_t));
     tor_asprintf(&vrs->microdesc->microdesc_hash_line,
-                 "m 9,10,11,12,13,14,15,16,17 "
+                 "m 25,26,27,28 "
                  "sha256=xyzajkldsdsajdadlsdjaslsdksdjlsdjsdaskdaaa%d\n",
                  idx);
   }
@@ -2618,7 +3536,7 @@ vote_tweaks_for_umbw(networkstatus_t *v, int voter, time_t now)
   smartlist_clear(v->supported_methods);
   /* Method 17 is MIN_METHOD_TO_CLIP_UNMEASURED_BW_KB */
   smartlist_split_string(v->supported_methods,
-                         "1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17",
+                         "25 26 27 28",
                          NULL, 0, -1);
   /* If we're using a non-default clip bandwidth, add it to net_params */
   if (alternate_clip_bw > 0) {
@@ -2727,7 +3645,7 @@ test_vrs_for_umbw(vote_routerstatus_t *vrs, int voter, time_t now)
     tt_int_op(rs->bandwidth_kb,OP_EQ, max_unmeasured_bw_kb / 2);
     tt_int_op(vrs->measured_bw_kb,OP_EQ, 0);
   } else {
-    tt_assert(0);
+    tt_abort();
   }
 
  done:
@@ -2743,9 +3661,9 @@ test_consensus_for_umbw(networkstatus_t *con, time_t now)
   (void)now;
 
   tt_assert(con);
-  tt_assert(!con->cert);
+  tt_ptr_op(con->cert, OP_EQ, NULL);
   // tt_assert(con->consensus_method >= MIN_METHOD_TO_CLIP_UNMEASURED_BW_KB);
-  tt_assert(con->consensus_method >= 16);
+  tt_int_op(con->consensus_method, OP_GE, 16);
   tt_int_op(4,OP_EQ, smartlist_len(con->routerstatus_list));
   /* There should be four listed routers; all voters saw the same in this */
 
@@ -2780,9 +3698,9 @@ test_routerstatus_for_umbw(routerstatus_t *rs, time_t now)
     tt_assert(!rs->is_fast);
     tt_assert(!rs->is_possible_guard);
     tt_assert(!rs->is_stable);
-    /* (If it wasn't running it wouldn't be here) */
+    /* (If it wasn't running and valid it wouldn't be here) */
     tt_assert(rs->is_flagged_running);
-    tt_assert(!rs->is_valid);
+    tt_assert(rs->is_valid);
     tt_assert(!rs->is_named);
     /* This one should have measured bandwidth below the clip cutoff */
     tt_assert(rs->has_bandwidth);
@@ -2842,7 +3760,7 @@ test_routerstatus_for_umbw(routerstatus_t *rs, time_t now)
     tt_assert(rs->bw_is_unmeasured);
   } else {
     /* Weren't expecting this... */
-    tt_assert(0);
+    tt_abort();
   }
 
  done:
@@ -2949,7 +3867,7 @@ mock_get_options(void)
 static void
 reset_routerstatus(routerstatus_t *rs,
                    const char *hex_identity_digest,
-                   int32_t ipv4_addr)
+                   uint32_t ipv4_addr)
 {
   memset(rs, 0, sizeof(routerstatus_t));
   base16_decode(rs->identity_digest, sizeof(rs->identity_digest),
@@ -3010,15 +3928,17 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
    * Return values are {2, 3, 4} */
 
   /* We want 3 ("*" means match all addresses) */
-  tt_assert(routerset_contains_routerstatus(routerset_all,  rs_a, 0) == 3);
-  tt_assert(routerset_contains_routerstatus(routerset_all,  rs_b, 0) == 3);
+  tt_int_op(routerset_contains_routerstatus(routerset_all, rs_a, 0), OP_EQ, 3);
+  tt_int_op(routerset_contains_routerstatus(routerset_all, rs_b, 0), OP_EQ, 3);
 
   /* We want 4 (match id_digest [or nickname]) */
-  tt_assert(routerset_contains_routerstatus(routerset_a,    rs_a, 0) == 4);
-  tt_assert(routerset_contains_routerstatus(routerset_a,    rs_b, 0) == 0);
+  tt_int_op(routerset_contains_routerstatus(routerset_a, rs_a, 0), OP_EQ, 4);
+  tt_int_op(routerset_contains_routerstatus(routerset_a, rs_b, 0), OP_EQ, 0);
 
-  tt_assert(routerset_contains_routerstatus(routerset_none, rs_a, 0) == 0);
-  tt_assert(routerset_contains_routerstatus(routerset_none, rs_b, 0) == 0);
+  tt_int_op(routerset_contains_routerstatus(routerset_none, rs_a, 0), OP_EQ,
+            0);
+  tt_int_op(routerset_contains_routerstatus(routerset_none, rs_b, 0), OP_EQ,
+            0);
 
   /* Check that "*" sets flags on all routers: Exit
    * Check the flags aren't being confused with each other */
@@ -3030,17 +3950,17 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   mock_options->TestingDirAuthVoteExitIsStrict = 0;
 
   dirserv_set_routerstatus_testing(rs_a);
-  tt_assert(mock_get_options_calls == 1);
+  tt_int_op(mock_get_options_calls, OP_EQ, 1);
   dirserv_set_routerstatus_testing(rs_b);
-  tt_assert(mock_get_options_calls == 2);
+  tt_int_op(mock_get_options_calls, OP_EQ, 2);
 
-  tt_assert(rs_a->is_exit == 1);
-  tt_assert(rs_b->is_exit == 1);
+  tt_uint_op(rs_a->is_exit, OP_EQ, 1);
+  tt_uint_op(rs_b->is_exit, OP_EQ, 1);
   /* Be paranoid - check no other flags are set */
-  tt_assert(rs_a->is_possible_guard == 0);
-  tt_assert(rs_b->is_possible_guard == 0);
-  tt_assert(rs_a->is_hs_dir == 0);
-  tt_assert(rs_b->is_hs_dir == 0);
+  tt_uint_op(rs_a->is_possible_guard, OP_EQ, 0);
+  tt_uint_op(rs_b->is_possible_guard, OP_EQ, 0);
+  tt_uint_op(rs_a->is_hs_dir, OP_EQ, 0);
+  tt_uint_op(rs_b->is_hs_dir, OP_EQ, 0);
 
   /* Check that "*" sets flags on all routers: Guard & HSDir
    * Cover the remaining flags in one test */
@@ -3054,17 +3974,17 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   mock_options->TestingDirAuthVoteHSDirIsStrict = 0;
 
   dirserv_set_routerstatus_testing(rs_a);
-  tt_assert(mock_get_options_calls == 1);
+  tt_int_op(mock_get_options_calls, OP_EQ, 1);
   dirserv_set_routerstatus_testing(rs_b);
-  tt_assert(mock_get_options_calls == 2);
+  tt_int_op(mock_get_options_calls, OP_EQ, 2);
 
-  tt_assert(rs_a->is_possible_guard == 1);
-  tt_assert(rs_b->is_possible_guard == 1);
-  tt_assert(rs_a->is_hs_dir == 1);
-  tt_assert(rs_b->is_hs_dir == 1);
+  tt_uint_op(rs_a->is_possible_guard, OP_EQ, 1);
+  tt_uint_op(rs_b->is_possible_guard, OP_EQ, 1);
+  tt_uint_op(rs_a->is_hs_dir, OP_EQ, 1);
+  tt_uint_op(rs_b->is_hs_dir, OP_EQ, 1);
   /* Be paranoid - check exit isn't set */
-  tt_assert(rs_a->is_exit == 0);
-  tt_assert(rs_b->is_exit == 0);
+  tt_uint_op(rs_a->is_exit, OP_EQ, 0);
+  tt_uint_op(rs_b->is_exit, OP_EQ, 0);
 
   /* Check routerset A sets all flags on router A,
    * but leaves router B unmodified */
@@ -3080,16 +4000,16 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   mock_options->TestingDirAuthVoteHSDirIsStrict = 0;
 
   dirserv_set_routerstatus_testing(rs_a);
-  tt_assert(mock_get_options_calls == 1);
+  tt_int_op(mock_get_options_calls, OP_EQ, 1);
   dirserv_set_routerstatus_testing(rs_b);
-  tt_assert(mock_get_options_calls == 2);
+  tt_int_op(mock_get_options_calls, OP_EQ, 2);
 
-  tt_assert(rs_a->is_exit == 1);
-  tt_assert(rs_b->is_exit == 0);
-  tt_assert(rs_a->is_possible_guard == 1);
-  tt_assert(rs_b->is_possible_guard == 0);
-  tt_assert(rs_a->is_hs_dir == 1);
-  tt_assert(rs_b->is_hs_dir == 0);
+  tt_uint_op(rs_a->is_exit, OP_EQ, 1);
+  tt_uint_op(rs_b->is_exit, OP_EQ, 0);
+  tt_uint_op(rs_a->is_possible_guard, OP_EQ, 1);
+  tt_uint_op(rs_b->is_possible_guard, OP_EQ, 0);
+  tt_uint_op(rs_a->is_hs_dir, OP_EQ, 1);
+  tt_uint_op(rs_b->is_hs_dir, OP_EQ, 0);
 
   /* Check routerset A unsets all flags on router B when Strict is set */
   reset_options(mock_options, &mock_get_options_calls);
@@ -3107,11 +4027,11 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   rs_b->is_hs_dir = 1;
 
   dirserv_set_routerstatus_testing(rs_b);
-  tt_assert(mock_get_options_calls == 1);
+  tt_int_op(mock_get_options_calls, OP_EQ, 1);
 
-  tt_assert(rs_b->is_exit == 0);
-  tt_assert(rs_b->is_possible_guard == 0);
-  tt_assert(rs_b->is_hs_dir == 0);
+  tt_uint_op(rs_b->is_exit, OP_EQ, 0);
+  tt_uint_op(rs_b->is_possible_guard, OP_EQ, 0);
+  tt_uint_op(rs_b->is_hs_dir, OP_EQ, 0);
 
   /* Check routerset A doesn't modify flags on router B without Strict set */
   reset_options(mock_options, &mock_get_options_calls);
@@ -3129,11 +4049,11 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   rs_b->is_hs_dir = 1;
 
   dirserv_set_routerstatus_testing(rs_b);
-  tt_assert(mock_get_options_calls == 1);
+  tt_int_op(mock_get_options_calls, OP_EQ, 1);
 
-  tt_assert(rs_b->is_exit == 1);
-  tt_assert(rs_b->is_possible_guard == 1);
-  tt_assert(rs_b->is_hs_dir == 1);
+  tt_uint_op(rs_b->is_exit, OP_EQ, 1);
+  tt_uint_op(rs_b->is_possible_guard, OP_EQ, 1);
+  tt_uint_op(rs_b->is_hs_dir, OP_EQ, 1);
 
   /* Check the empty routerset zeroes all flags
    * on routers A & B with Strict set */
@@ -3152,11 +4072,11 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   rs_b->is_hs_dir = 1;
 
   dirserv_set_routerstatus_testing(rs_b);
-  tt_assert(mock_get_options_calls == 1);
+  tt_int_op(mock_get_options_calls, OP_EQ, 1);
 
-  tt_assert(rs_b->is_exit == 0);
-  tt_assert(rs_b->is_possible_guard == 0);
-  tt_assert(rs_b->is_hs_dir == 0);
+  tt_uint_op(rs_b->is_exit, OP_EQ, 0);
+  tt_uint_op(rs_b->is_possible_guard, OP_EQ, 0);
+  tt_uint_op(rs_b->is_hs_dir, OP_EQ, 0);
 
   /* Check the empty routerset doesn't modify any flags
    * on A or B without Strict set */
@@ -3176,16 +4096,16 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   rs_b->is_hs_dir = 1;
 
   dirserv_set_routerstatus_testing(rs_a);
-  tt_assert(mock_get_options_calls == 1);
+  tt_int_op(mock_get_options_calls, OP_EQ, 1);
   dirserv_set_routerstatus_testing(rs_b);
-  tt_assert(mock_get_options_calls == 2);
+  tt_int_op(mock_get_options_calls, OP_EQ, 2);
 
-  tt_assert(rs_a->is_exit == 0);
-  tt_assert(rs_a->is_possible_guard == 0);
-  tt_assert(rs_a->is_hs_dir == 0);
-  tt_assert(rs_b->is_exit == 1);
-  tt_assert(rs_b->is_possible_guard == 1);
-  tt_assert(rs_b->is_hs_dir == 1);
+  tt_uint_op(rs_a->is_exit, OP_EQ, 0);
+  tt_uint_op(rs_a->is_possible_guard, OP_EQ, 0);
+  tt_uint_op(rs_a->is_hs_dir, OP_EQ, 0);
+  tt_uint_op(rs_b->is_exit, OP_EQ, 1);
+  tt_uint_op(rs_b->is_possible_guard, OP_EQ, 1);
+  tt_uint_op(rs_b->is_hs_dir, OP_EQ, 1);
 
  done:
   tor_free(mock_options);
@@ -3241,7 +4161,7 @@ test_dir_http_handling(void *args)
                            "User-Agent: Mozilla/5.0 (Windows;"
                            " U; Windows NT 6.1; en-US; rv:1.9.1.5)\r\n",
                            &url),OP_EQ, -1);
-  tt_assert(!url);
+  tt_ptr_op(url, OP_EQ, NULL);
 
   /* Bad headers */
   tt_int_op(parse_http_url("GET /a/b/c.txt\r\n"
@@ -3249,40 +4169,113 @@ test_dir_http_handling(void *args)
                            "User-Agent: Mozilla/5.0 (Windows;"
                            " U; Windows NT 6.1; en-US; rv:1.9.1.5)\r\n",
                            &url),OP_EQ, -1);
-  tt_assert(!url);
+  tt_ptr_op(url, OP_EQ, NULL);
 
   tt_int_op(parse_http_url("GET /tor/a/b/c.txt", &url),OP_EQ, -1);
-  tt_assert(!url);
+  tt_ptr_op(url, OP_EQ, NULL);
 
   tt_int_op(parse_http_url("GET /tor/a/b/c.txt HTTP/1.1", &url),OP_EQ, -1);
-  tt_assert(!url);
+  tt_ptr_op(url, OP_EQ, NULL);
 
   tt_int_op(parse_http_url("GET /tor/a/b/c.txt HTTP/1.1x\r\n", &url),
             OP_EQ, -1);
-  tt_assert(!url);
+  tt_ptr_op(url, OP_EQ, NULL);
 
   tt_int_op(parse_http_url("GET /tor/a/b/c.txt HTTP/1.", &url),OP_EQ, -1);
-  tt_assert(!url);
+  tt_ptr_op(url, OP_EQ, NULL);
 
   tt_int_op(parse_http_url("GET /tor/a/b/c.txt HTTP/1.\r", &url),OP_EQ, -1);
-  tt_assert(!url);
+  tt_ptr_op(url, OP_EQ, NULL);
 
  done:
   tor_free(url);
 }
 
 static void
-test_dir_purpose_needs_anonymity(void *arg)
+test_dir_purpose_needs_anonymity_returns_true_by_default(void *arg)
+{
+  (void)arg;
+
+  tor_capture_bugs_(1);
+  setup_full_capture_of_logs(LOG_WARN);
+  tt_int_op(1, OP_EQ, purpose_needs_anonymity(0, 0, NULL));
+  tt_int_op(1, OP_EQ, smartlist_len(tor_get_captured_bug_log_()));
+  expect_single_log_msg_containing("Called with dir_purpose=0");
+
+  tor_end_capture_bugs_();
+ done:
+  tor_end_capture_bugs_();
+  teardown_capture_of_logs();
+}
+
+static void
+test_dir_purpose_needs_anonymity_returns_true_for_bridges(void *arg)
+{
+  (void)arg;
+
+  tt_int_op(1, OP_EQ, purpose_needs_anonymity(0, ROUTER_PURPOSE_BRIDGE, NULL));
+  tt_int_op(1, OP_EQ, purpose_needs_anonymity(0, ROUTER_PURPOSE_BRIDGE,
+                                           "foobar"));
+  tt_int_op(1, OP_EQ,
+            purpose_needs_anonymity(DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2,
+                                    ROUTER_PURPOSE_BRIDGE, NULL));
+ done: ;
+}
+
+static void
+test_dir_purpose_needs_anonymity_returns_false_for_own_bridge_desc(void *arg)
+{
+  (void)arg;
+  tt_int_op(0, OP_EQ, purpose_needs_anonymity(DIR_PURPOSE_FETCH_SERVERDESC,
+                                           ROUTER_PURPOSE_BRIDGE,
+                                           "authority.z"));
+ done: ;
+}
+
+static void
+test_dir_purpose_needs_anonymity_returns_true_for_sensitive_purpose(void *arg)
 {
   (void)arg;
-  tt_int_op(1, ==, purpose_needs_anonymity(0, ROUTER_PURPOSE_BRIDGE));
-  tt_int_op(1, ==, purpose_needs_anonymity(0, ROUTER_PURPOSE_GENERAL));
-  tt_int_op(0, ==, purpose_needs_anonymity(DIR_PURPOSE_FETCH_MICRODESC,
-                                            ROUTER_PURPOSE_GENERAL));
+
+  tt_int_op(1, OP_EQ, purpose_needs_anonymity(
+                    DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2,
+                    ROUTER_PURPOSE_GENERAL, NULL));
+  tt_int_op(1, OP_EQ, purpose_needs_anonymity(
+                    DIR_PURPOSE_UPLOAD_RENDDESC_V2, 0,  NULL));
+  tt_int_op(1, OP_EQ, purpose_needs_anonymity(
+                    DIR_PURPOSE_FETCH_RENDDESC_V2, 0, NULL));
  done: ;
 }
 
 static void
+test_dir_purpose_needs_anonymity_ret_false_for_non_sensitive_conn(void *arg)
+{
+  (void)arg;
+
+  tt_int_op(0, OP_EQ, purpose_needs_anonymity(DIR_PURPOSE_UPLOAD_DIR,
+                                           ROUTER_PURPOSE_GENERAL, NULL));
+  tt_int_op(0, OP_EQ,
+            purpose_needs_anonymity(DIR_PURPOSE_UPLOAD_VOTE, 0, NULL));
+  tt_int_op(0, OP_EQ,
+            purpose_needs_anonymity(DIR_PURPOSE_UPLOAD_SIGNATURES, 0, NULL));
+  tt_int_op(0, OP_EQ,
+            purpose_needs_anonymity(DIR_PURPOSE_FETCH_STATUS_VOTE, 0, NULL));
+  tt_int_op(0, OP_EQ, purpose_needs_anonymity(
+                    DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0, NULL));
+  tt_int_op(0, OP_EQ,
+            purpose_needs_anonymity(DIR_PURPOSE_FETCH_CONSENSUS, 0, NULL));
+  tt_int_op(0, OP_EQ,
+            purpose_needs_anonymity(DIR_PURPOSE_FETCH_CERTIFICATE, 0, NULL));
+  tt_int_op(0, OP_EQ,
+            purpose_needs_anonymity(DIR_PURPOSE_FETCH_SERVERDESC, 0, NULL));
+  tt_int_op(0, OP_EQ,
+            purpose_needs_anonymity(DIR_PURPOSE_FETCH_EXTRAINFO, 0, NULL));
+  tt_int_op(0, OP_EQ,
+            purpose_needs_anonymity(DIR_PURPOSE_FETCH_MICRODESC, 0, NULL));
+  done: ;
+}
+
+static void
 test_dir_fetch_type(void *arg)
 {
   (void)arg;
@@ -3330,9 +4323,9 @@ test_dir_packages(void *arg)
   (void)arg;
 
 #define BAD(s) \
-  tt_int_op(0, ==, validate_recommended_package_line(s));
+  tt_int_op(0, OP_EQ, validate_recommended_package_line(s));
 #define GOOD(s) \
-  tt_int_op(1, ==, validate_recommended_package_line(s));
+  tt_int_op(1, OP_EQ, validate_recommended_package_line(s));
   GOOD("tor 0.2.6.3-alpha "
        "http://torproject.example.com/dist/tor-0.2.6.3-alpha.tar.gz "
        "sha256=sssdlkfjdsklfjdskfljasdklfj");
@@ -3449,7 +4442,7 @@ test_dir_packages(void *arg)
 
   res = compute_consensus_package_lines(votes);
   tt_assert(res);
-  tt_str_op(res, ==,
+  tt_str_op(res, OP_EQ,
     "package cbc 99.1.11.1.1 http://example.com/cbc/ cubehash=ahooy sha512=m\n"
     "package clownshoes 22alpha3 http://quumble.example.com/ blake2=fooz\n"
     "package clownshoes 22alpha4 http://quumble.example.cam/ blake2=fooa\n"
@@ -3468,490 +4461,156 @@ test_dir_packages(void *arg)
 }
 
 static void
-test_dir_download_status_schedule(void *arg)
-{
-  (void)arg;
-  download_status_t dls_failure = { 0, 0, 0, DL_SCHED_GENERIC,
-                                             DL_WANT_AUTHORITY,
-                                             DL_SCHED_INCREMENT_FAILURE,
-                                             DL_SCHED_DETERMINISTIC, 0, 0 };
-  download_status_t dls_attempt = { 0, 0, 0, DL_SCHED_CONSENSUS,
-                                             DL_WANT_ANY_DIRSERVER,
-                                             DL_SCHED_INCREMENT_ATTEMPT,
-                                             DL_SCHED_DETERMINISTIC, 0, 0 };
-  download_status_t dls_bridge  = { 0, 0, 0, DL_SCHED_BRIDGE,
-                                             DL_WANT_AUTHORITY,
-                                             DL_SCHED_INCREMENT_FAILURE,
-                                             DL_SCHED_DETERMINISTIC, 0, 0 };
-  int increment = -1;
-  int expected_increment = -1;
-  time_t current_time = time(NULL);
-  int delay1 = -1;
-  int delay2 = -1;
-  smartlist_t *schedule = smartlist_new();
-
-  /* Make a dummy schedule */
-  smartlist_add(schedule, (void *)&delay1);
-  smartlist_add(schedule, (void *)&delay2);
-
-  /* check a range of values */
-  delay1 = 1000;
-  increment = download_status_schedule_get_delay(&dls_failure,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 TIME_MIN);
-  expected_increment = delay1;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_failure.next_attempt_at == TIME_MIN + expected_increment);
-
-  delay1 = INT_MAX;
-  increment =  download_status_schedule_get_delay(&dls_failure,
-                                                  schedule,
-                                                  0, INT_MAX,
-                                                  -1);
-  expected_increment = delay1;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_failure.next_attempt_at == TIME_MAX);
-
-  delay1 = 0;
-  increment = download_status_schedule_get_delay(&dls_attempt,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 0);
-  expected_increment = delay1;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_attempt.next_attempt_at == 0 + expected_increment);
-
-  delay1 = 1000;
-  increment = download_status_schedule_get_delay(&dls_attempt,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 1);
-  expected_increment = delay1;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_attempt.next_attempt_at == 1 + expected_increment);
-
-  delay1 = INT_MAX;
-  increment = download_status_schedule_get_delay(&dls_bridge,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 current_time);
-  expected_increment = delay1;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_bridge.next_attempt_at == TIME_MAX);
-
-  delay1 = 1;
-  increment = download_status_schedule_get_delay(&dls_bridge,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 TIME_MAX);
-  expected_increment = delay1;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_bridge.next_attempt_at == TIME_MAX);
-
-  /* see what happens when we reach the end */
-  dls_attempt.n_download_attempts++;
-  dls_bridge.n_download_failures++;
-
-  delay2 = 100;
-  increment = download_status_schedule_get_delay(&dls_attempt,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 current_time);
-  expected_increment = delay2;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_attempt.next_attempt_at == current_time + delay2);
-
-  delay2 = 1;
-  increment = download_status_schedule_get_delay(&dls_bridge,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 current_time);
-  expected_increment = delay2;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_bridge.next_attempt_at == current_time + delay2);
-
-  /* see what happens when we try to go off the end */
-  dls_attempt.n_download_attempts++;
-  dls_bridge.n_download_failures++;
-
-  delay2 = 5;
-  increment = download_status_schedule_get_delay(&dls_attempt,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 current_time);
-  expected_increment = delay2;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_attempt.next_attempt_at == current_time + delay2);
-
-  delay2 = 17;
-  increment = download_status_schedule_get_delay(&dls_bridge,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 current_time);
-  expected_increment = delay2;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_bridge.next_attempt_at == current_time + delay2);
-
-  /* see what happens when we reach IMPOSSIBLE_TO_DOWNLOAD */
-  dls_attempt.n_download_attempts = IMPOSSIBLE_TO_DOWNLOAD;
-  dls_bridge.n_download_failures = IMPOSSIBLE_TO_DOWNLOAD;
-
-  delay2 = 35;
-  increment = download_status_schedule_get_delay(&dls_attempt,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 current_time);
-  expected_increment = INT_MAX;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_attempt.next_attempt_at == TIME_MAX);
-
-  delay2 = 99;
-  increment = download_status_schedule_get_delay(&dls_bridge,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 current_time);
-  expected_increment = INT_MAX;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_bridge.next_attempt_at == TIME_MAX);
-
- done:
-  /* the pointers in schedule are allocated on the stack */
-  smartlist_free(schedule);
-}
-
-static void
-test_dir_download_status_random_backoff(void *arg)
+download_status_random_backoff_helper(int min_delay)
 {
   download_status_t dls_random =
     { 0, 0, 0, DL_SCHED_GENERIC, DL_WANT_AUTHORITY,
-               DL_SCHED_INCREMENT_FAILURE, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 };
+               DL_SCHED_INCREMENT_FAILURE, 0, 0 };
   int increment = -1;
-  int old_increment;
+  int old_increment = -1;
   time_t current_time = time(NULL);
-  const int min_delay = 0;
-  const int max_delay = 1000000;
-
-  (void)arg;
 
   /* Check the random backoff cases */
-  old_increment = 0;
+  int n_attempts = 0;
   do {
     increment = download_status_schedule_get_delay(&dls_random,
-                                                   NULL,
-                                                   min_delay, max_delay,
+                                                   min_delay,
                                                    current_time);
+
+    log_debug(LD_DIR, "Min: %d, Inc: %d, Old Inc: %d",
+              min_delay, increment, old_increment);
+
+    /* Regression test for 20534 and friends
+     * increment must always increase after the first */
+    if (dls_random.last_backoff_position > 0) {
+      /* Always increment the exponential backoff */
+      tt_int_op(increment, OP_GE, 1);
+    }
+
     /* Test */
     tt_int_op(increment, OP_GE, min_delay);
-    tt_int_op(increment, OP_LE, max_delay);
-    tt_int_op(increment, OP_GE, old_increment);
-    /* We at most quadruple, and maybe add one */
-    tt_int_op(increment, OP_LE, 4 * old_increment + 1);
 
     /* Advance */
-    current_time += increment;
-    ++(dls_random.n_download_attempts);
-    ++(dls_random.n_download_failures);
+    if (dls_random.n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD - 1) {
+      ++(dls_random.n_download_attempts);
+      ++(dls_random.n_download_failures);
+    }
 
     /* Try another maybe */
     old_increment = increment;
-  } while (increment < max_delay);
+  } while (++n_attempts < 1000);
 
  done:
   return;
 }
 
 static void
+test_dir_download_status_random_backoff(void *arg)
+{
+  (void)arg;
+
+  /* Do a standard test */
+  download_status_random_backoff_helper(0);
+  /* regression tests for 17750: initial delay */
+  download_status_random_backoff_helper(10);
+  download_status_random_backoff_helper(20);
+
+  /* Pathological cases */
+  download_status_random_backoff_helper(INT_MAX/2);
+}
+
+static void
+test_dir_download_status_random_backoff_ranges(void *arg)
+{
+  (void)arg;
+  int lo, hi;
+  next_random_exponential_delay_range(&lo, &hi, 0, 10);
+  tt_int_op(lo, OP_EQ, 10);
+  tt_int_op(hi, OP_EQ, 11);
+
+  next_random_exponential_delay_range(&lo, &hi, 6, 10);
+  tt_int_op(lo, OP_EQ, 10);
+  tt_int_op(hi, OP_EQ, 6*3);
+
+  next_random_exponential_delay_range(&lo, &hi, 13, 10);
+  tt_int_op(lo, OP_EQ, 10);
+  tt_int_op(hi, OP_EQ, 13 * 3);
+
+  next_random_exponential_delay_range(&lo, &hi, 37, 10);
+  tt_int_op(lo, OP_EQ, 10);
+  tt_int_op(hi, OP_EQ, 111);
+
+  next_random_exponential_delay_range(&lo, &hi, 123, 10);
+  tt_int_op(lo, OP_EQ, 10);
+  tt_int_op(hi, OP_EQ, 369);
+
+  next_random_exponential_delay_range(&lo, &hi, INT_MAX-5, 10);
+  tt_int_op(lo, OP_EQ, 10);
+  tt_int_op(hi, OP_EQ, INT_MAX);
+ done:
+  ;
+}
+
+static void
 test_dir_download_status_increment(void *arg)
 {
   (void)arg;
-  download_status_t dls_failure = { 0, 0, 0, DL_SCHED_GENERIC,
-    DL_WANT_AUTHORITY,
-    DL_SCHED_INCREMENT_FAILURE,
-    DL_SCHED_DETERMINISTIC, 0, 0 };
-  download_status_t dls_attempt = { 0, 0, 0, DL_SCHED_BRIDGE,
+  download_status_t dls_exp = { 0, 0, 0, DL_SCHED_GENERIC,
     DL_WANT_ANY_DIRSERVER,
     DL_SCHED_INCREMENT_ATTEMPT,
-    DL_SCHED_DETERMINISTIC, 0, 0 };
-  int delay0 = -1;
-  int delay1 = -1;
-  int delay2 = -1;
-  smartlist_t *schedule = smartlist_new();
+    0, 0 };
   or_options_t test_options;
-  time_t next_at = TIME_MAX;
   time_t current_time = time(NULL);
 
-  /* Provide some values for the schedule */
-  delay0 = 10;
-  delay1 = 99;
-  delay2 = 20;
-
-  /* Make the schedule */
-  smartlist_add(schedule, (void *)&delay0);
-  smartlist_add(schedule, (void *)&delay1);
-  smartlist_add(schedule, (void *)&delay2);
+  const int delay0 = 10;
+  const int no_delay = 0;
+  const int schedule = 10;
+  const int schedule_no_initial_delay = 0;
 
   /* Put it in the options */
   mock_options = &test_options;
   reset_options(mock_options, &mock_get_options_calls);
-  mock_options->TestingClientDownloadSchedule = schedule;
-  mock_options->TestingBridgeDownloadSchedule = schedule;
+  mock_options->TestingBridgeBootstrapDownloadInitialDelay = schedule;
+  mock_options->TestingClientDownloadInitialDelay = schedule;
 
   MOCK(get_options, mock_get_options);
 
-  /* Check that a failure reset works */
-  mock_get_options_calls = 0;
-  download_status_reset(&dls_failure);
-  /* we really want to test that it's equal to time(NULL) + delay0, but that's
-   * an unrealiable test, because time(NULL) might change. */
-  tt_assert(download_status_get_next_attempt_at(&dls_failure)
-            >= current_time + delay0);
-  tt_assert(download_status_get_next_attempt_at(&dls_failure)
-            != TIME_MAX);
-  tt_assert(download_status_get_n_failures(&dls_failure) == 0);
-  tt_assert(download_status_get_n_attempts(&dls_failure) == 0);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* avoid timing inconsistencies */
-  dls_failure.next_attempt_at = current_time + delay0;
-
-  /* check that a reset schedule becomes ready at the right time */
-  tt_assert(download_status_is_ready(&dls_failure,
-                                     current_time + delay0 - 1,
-                                     1) == 0);
-  tt_assert(download_status_is_ready(&dls_failure,
-                                     current_time + delay0,
-                                     1) == 1);
-  tt_assert(download_status_is_ready(&dls_failure,
-                                     current_time + delay0 + 1,
-                                     1) == 1);
-
-  /* Check that a failure increment works */
-  mock_get_options_calls = 0;
-  next_at = download_status_increment_failure(&dls_failure, 404, "test", 0,
-                                              current_time);
-  tt_assert(next_at == current_time + delay1);
-  tt_assert(download_status_get_n_failures(&dls_failure) == 1);
-  tt_assert(download_status_get_n_attempts(&dls_failure) == 1);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* check that an incremented schedule becomes ready at the right time */
-  tt_assert(download_status_is_ready(&dls_failure,
-                                     current_time + delay1 - 1,
-                                     1) == 0);
-  tt_assert(download_status_is_ready(&dls_failure,
-                                     current_time + delay1,
-                                     1) == 1);
-  tt_assert(download_status_is_ready(&dls_failure,
-                                     current_time + delay1 + 1,
-                                     1) == 1);
-
-  /* check that a schedule isn't ready if it's had too many failures */
-  tt_assert(download_status_is_ready(&dls_failure,
-                                     current_time + delay1 + 10,
-                                     0) == 0);
-
-  /* Check that failure increments do happen on 503 for clients, and
-   * attempt increments do too. */
-  mock_get_options_calls = 0;
-  next_at = download_status_increment_failure(&dls_failure, 503, "test", 0,
-                                              current_time);
-  tt_i64_op(next_at, ==, current_time + delay2);
-  tt_int_op(download_status_get_n_failures(&dls_failure), ==, 2);
-  tt_int_op(download_status_get_n_attempts(&dls_failure), ==, 2);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* Check that failure increments do happen on 503 for servers */
-  mock_get_options_calls = 0;
-  next_at = download_status_increment_failure(&dls_failure, 503, "test", 1,
-                                              current_time);
-  tt_assert(next_at == current_time + delay2);
-  tt_assert(download_status_get_n_failures(&dls_failure) == 3);
-  tt_assert(download_status_get_n_attempts(&dls_failure) == 3);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* Check what happens when we run off the end of the schedule */
-  mock_get_options_calls = 0;
-  next_at = download_status_increment_failure(&dls_failure, 404, "test", 0,
-                                              current_time);
-  tt_assert(next_at == current_time + delay2);
-  tt_assert(download_status_get_n_failures(&dls_failure) == 4);
-  tt_assert(download_status_get_n_attempts(&dls_failure) == 4);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* Check what happens when we hit the failure limit */
-  mock_get_options_calls = 0;
-  download_status_mark_impossible(&dls_failure);
-  next_at = download_status_increment_failure(&dls_failure, 404, "test", 0,
-                                              current_time);
-  tt_assert(next_at == TIME_MAX);
-  tt_assert(download_status_get_n_failures(&dls_failure)
-            == IMPOSSIBLE_TO_DOWNLOAD);
-  tt_assert(download_status_get_n_attempts(&dls_failure)
-            == IMPOSSIBLE_TO_DOWNLOAD);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* Check that a failure reset doesn't reset at the limit */
-  mock_get_options_calls = 0;
-  download_status_reset(&dls_failure);
-  tt_assert(download_status_get_next_attempt_at(&dls_failure)
-            == TIME_MAX);
-  tt_assert(download_status_get_n_failures(&dls_failure)
-            == IMPOSSIBLE_TO_DOWNLOAD);
-  tt_assert(download_status_get_n_attempts(&dls_failure)
-            == IMPOSSIBLE_TO_DOWNLOAD);
-  tt_assert(mock_get_options_calls == 0);
-
-  /* Check that a failure reset resets just before the limit */
+  /* Check that the initial value of the schedule is the first value used,
+   * whether or not it was reset before being used */
+
+  /* regression test for 17750: no initial delay */
+  mock_options->TestingClientDownloadInitialDelay = schedule_no_initial_delay;
   mock_get_options_calls = 0;
-  dls_failure.n_download_failures = IMPOSSIBLE_TO_DOWNLOAD - 1;
-  dls_failure.n_download_attempts = IMPOSSIBLE_TO_DOWNLOAD - 1;
-  download_status_reset(&dls_failure);
   /* we really want to test that it's equal to time(NULL) + delay0, but that's
    * an unrealiable test, because time(NULL) might change. */
-  tt_assert(download_status_get_next_attempt_at(&dls_failure)
-            >= current_time + delay0);
-  tt_assert(download_status_get_next_attempt_at(&dls_failure)
-            != TIME_MAX);
-  tt_assert(download_status_get_n_failures(&dls_failure) == 0);
-  tt_assert(download_status_get_n_attempts(&dls_failure) == 0);
-  tt_assert(mock_get_options_calls >= 1);
 
-  /* Check that failure increments do happen on attempt-based schedules,
-   * but that the retry is set at the end of time */
+  /* regression test for 17750: exponential, no initial delay */
+  mock_options->TestingClientDownloadInitialDelay = schedule_no_initial_delay;
   mock_get_options_calls = 0;
-  next_at = download_status_increment_failure(&dls_attempt, 404, "test", 0,
-                                              current_time);
-  tt_assert(next_at == TIME_MAX);
-  tt_assert(download_status_get_n_failures(&dls_attempt) == 1);
-  tt_assert(download_status_get_n_attempts(&dls_attempt) == 0);
-  tt_assert(mock_get_options_calls == 0);
-
-  /* Check that an attempt reset works */
-  mock_get_options_calls = 0;
-  download_status_reset(&dls_attempt);
   /* we really want to test that it's equal to time(NULL) + delay0, but that's
    * an unrealiable test, because time(NULL) might change. */
-  tt_assert(download_status_get_next_attempt_at(&dls_attempt)
-            >= current_time + delay0);
-  tt_assert(download_status_get_next_attempt_at(&dls_attempt)
+  tt_assert(download_status_get_next_attempt_at(&dls_exp)
+            >= current_time + no_delay);
+  tt_assert(download_status_get_next_attempt_at(&dls_exp)
             != TIME_MAX);
-  tt_assert(download_status_get_n_failures(&dls_attempt) == 0);
-  tt_assert(download_status_get_n_attempts(&dls_attempt) == 0);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* avoid timing inconsistencies */
-  dls_attempt.next_attempt_at = current_time + delay0;
-
-  /* check that a reset schedule becomes ready at the right time */
-  tt_assert(download_status_is_ready(&dls_attempt,
-                                     current_time + delay0 - 1,
-                                     1) == 0);
-  tt_assert(download_status_is_ready(&dls_attempt,
-                                     current_time + delay0,
-                                     1) == 1);
-  tt_assert(download_status_is_ready(&dls_attempt,
-                                     current_time + delay0 + 1,
-                                     1) == 1);
-
-  /* Check that an attempt increment works */
-  mock_get_options_calls = 0;
-  next_at = download_status_increment_attempt(&dls_attempt, "test",
-                                              current_time);
-  tt_assert(next_at == current_time + delay1);
-  tt_assert(download_status_get_n_failures(&dls_attempt) == 0);
-  tt_assert(download_status_get_n_attempts(&dls_attempt) == 1);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* check that an incremented schedule becomes ready at the right time */
-  tt_assert(download_status_is_ready(&dls_attempt,
-                                     current_time + delay1 - 1,
-                                     1) == 0);
-  tt_assert(download_status_is_ready(&dls_attempt,
-                                     current_time + delay1,
-                                     1) == 1);
-  tt_assert(download_status_is_ready(&dls_attempt,
-                                     current_time + delay1 + 1,
-                                     1) == 1);
-
-  /* check that a schedule isn't ready if it's had too many attempts */
-  tt_assert(download_status_is_ready(&dls_attempt,
-                                     current_time + delay1 + 10,
-                                     0) == 0);
-
-  /* Check what happens when we reach then run off the end of the schedule */
-  mock_get_options_calls = 0;
-  next_at = download_status_increment_attempt(&dls_attempt, "test",
-                                              current_time);
-  tt_assert(next_at == current_time + delay2);
-  tt_assert(download_status_get_n_failures(&dls_attempt) == 0);
-  tt_assert(download_status_get_n_attempts(&dls_attempt) == 2);
-  tt_assert(mock_get_options_calls >= 1);
+  tt_int_op(download_status_get_n_failures(&dls_exp), OP_EQ, 0);
+  tt_int_op(download_status_get_n_attempts(&dls_exp), OP_EQ, 0);
+  tt_int_op(mock_get_options_calls, OP_GE, 1);
 
+  /* regression test for 17750: exponential, initial delay */
+  mock_options->TestingClientDownloadInitialDelay = schedule;
   mock_get_options_calls = 0;
-  next_at = download_status_increment_attempt(&dls_attempt, "test",
-                                              current_time);
-  tt_assert(next_at == current_time + delay2);
-  tt_assert(download_status_get_n_failures(&dls_attempt) == 0);
-  tt_assert(download_status_get_n_attempts(&dls_attempt) == 3);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* Check what happens when we hit the attempt limit */
-  mock_get_options_calls = 0;
-  download_status_mark_impossible(&dls_attempt);
-  next_at = download_status_increment_attempt(&dls_attempt, "test",
-                                              current_time);
-  tt_assert(next_at == TIME_MAX);
-  tt_assert(download_status_get_n_failures(&dls_attempt)
-            == IMPOSSIBLE_TO_DOWNLOAD);
-  tt_assert(download_status_get_n_attempts(&dls_attempt)
-            == IMPOSSIBLE_TO_DOWNLOAD);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* Check that an attempt reset doesn't reset at the limit */
-  mock_get_options_calls = 0;
-  download_status_reset(&dls_attempt);
-  tt_assert(download_status_get_next_attempt_at(&dls_attempt)
-            == TIME_MAX);
-  tt_assert(download_status_get_n_failures(&dls_attempt)
-            == IMPOSSIBLE_TO_DOWNLOAD);
-  tt_assert(download_status_get_n_attempts(&dls_attempt)
-            == IMPOSSIBLE_TO_DOWNLOAD);
-  tt_assert(mock_get_options_calls == 0);
-
-  /* Check that an attempt reset resets just before the limit */
-  mock_get_options_calls = 0;
-  dls_attempt.n_download_failures = IMPOSSIBLE_TO_DOWNLOAD - 1;
-  dls_attempt.n_download_attempts = IMPOSSIBLE_TO_DOWNLOAD - 1;
-  download_status_reset(&dls_attempt);
   /* we really want to test that it's equal to time(NULL) + delay0, but that's
    * an unrealiable test, because time(NULL) might change. */
-  tt_assert(download_status_get_next_attempt_at(&dls_attempt)
+  tt_assert(download_status_get_next_attempt_at(&dls_exp)
             >= current_time + delay0);
-  tt_assert(download_status_get_next_attempt_at(&dls_attempt)
+  tt_assert(download_status_get_next_attempt_at(&dls_exp)
             != TIME_MAX);
-  tt_assert(download_status_get_n_failures(&dls_attempt) == 0);
-  tt_assert(download_status_get_n_attempts(&dls_attempt) == 0);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* Check that attempt increments don't happen on failure-based schedules,
-   * and that the attempt is set at the end of time */
-  mock_get_options_calls = 0;
-  setup_full_capture_of_logs(LOG_WARN);
-  next_at = download_status_increment_attempt(&dls_failure, "test",
-                                              current_time);
-  expect_single_log_msg_containing(
-    "Tried to launch an attempt-based connection on a failure-based "
-    "schedule.");
-  teardown_capture_of_logs();
-  tt_assert(next_at == TIME_MAX);
-  tt_assert(download_status_get_n_failures(&dls_failure) == 0);
-  tt_assert(download_status_get_n_attempts(&dls_failure) == 0);
-  tt_assert(mock_get_options_calls == 0);
+  tt_int_op(download_status_get_n_failures(&dls_exp), OP_EQ, 0);
+  tt_int_op(download_status_get_n_attempts(&dls_exp), OP_EQ, 0);
+  tt_int_op(mock_get_options_calls, OP_GE, 1);
 
  done:
-  /* the pointers in schedule are allocated on the stack */
-  smartlist_free(schedule);
   UNMOCK(get_options);
   mock_options = NULL;
   mock_get_options_calls = 0;
@@ -4054,7 +4713,6 @@ test_dir_should_use_directory_guards(void *data)
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
   tt_int_op(CALLED(public_server_mode), OP_EQ, 1);
 
-  options->UseEntryGuardsAsDirGuards = 1;
   options->UseEntryGuards = 1;
   options->DownloadExtraInfo = 0;
   options->FetchDirInfoEarly = 0;
@@ -4068,29 +4726,24 @@ test_dir_should_use_directory_guards(void *data)
   tt_int_op(CALLED(public_server_mode), OP_EQ, 3);
   options->UseEntryGuards = 1;
 
-  options->UseEntryGuardsAsDirGuards = 0;
-  tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 4);
-  options->UseEntryGuardsAsDirGuards = 1;
-
   options->DownloadExtraInfo = 1;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 5);
+  tt_int_op(CALLED(public_server_mode), OP_EQ, 4);
   options->DownloadExtraInfo = 0;
 
   options->FetchDirInfoEarly = 1;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 6);
+  tt_int_op(CALLED(public_server_mode), OP_EQ, 5);
   options->FetchDirInfoEarly = 0;
 
   options->FetchDirInfoExtraEarly = 1;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 7);
+  tt_int_op(CALLED(public_server_mode), OP_EQ, 6);
   options->FetchDirInfoExtraEarly = 0;
 
   options->FetchUselessDescriptors = 1;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 8);
+  tt_int_op(CALLED(public_server_mode), OP_EQ, 7);
   options->FetchUselessDescriptors = 0;
 
   done:
@@ -4100,14 +4753,7 @@ test_dir_should_use_directory_guards(void *data)
 }
 
 NS_DECL(void,
-directory_initiate_command_routerstatus, (const routerstatus_t *status,
-                                          uint8_t dir_purpose,
-                                          uint8_t router_purpose,
-                                          dir_indirection_t indirection,
-                                          const char *resource,
-                                          const char *payload,
-                                          size_t payload_len,
-                                          time_t if_modified_since));
+directory_initiate_request, (directory_request_t *req));
 
 static void
 test_dir_should_not_init_request_to_ourselves(void *data)
@@ -4117,7 +4763,7 @@ test_dir_should_not_init_request_to_ourselves(void *data)
   crypto_pk_t *key = pk_generate(2);
   (void) data;
 
-  NS_MOCK(directory_initiate_command_routerstatus);
+  NS_MOCK(directory_initiate_request);
 
   clear_dir_servers();
   routerlist_free_all();
@@ -4132,15 +4778,15 @@ test_dir_should_not_init_request_to_ourselves(void *data)
   dir_server_add(ourself);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_STATUS_VOTE, 0, NULL);
-  tt_int_op(CALLED(directory_initiate_command_routerstatus), OP_EQ, 0);
+  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 0);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0,
                                      NULL);
 
-  tt_int_op(CALLED(directory_initiate_command_routerstatus), OP_EQ, 0);
+  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 0);
 
   done:
-    NS_UNMOCK(directory_initiate_command_routerstatus);
+    NS_UNMOCK(directory_initiate_request);
     clear_dir_servers();
     routerlist_free_all();
     crypto_pk_free(key);
@@ -4154,7 +4800,7 @@ test_dir_should_not_init_request_to_dir_auths_without_v3_info(void *data)
                                 | MICRODESC_DIRINFO;
   (void) data;
 
-  NS_MOCK(directory_initiate_command_routerstatus);
+  NS_MOCK(directory_initiate_request);
 
   clear_dir_servers();
   routerlist_free_all();
@@ -4165,14 +4811,14 @@ test_dir_should_not_init_request_to_dir_auths_without_v3_info(void *data)
   dir_server_add(ds);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_STATUS_VOTE, 0, NULL);
-  tt_int_op(CALLED(directory_initiate_command_routerstatus), OP_EQ, 0);
+  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 0);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0,
                                      NULL);
-  tt_int_op(CALLED(directory_initiate_command_routerstatus), OP_EQ, 0);
+  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 0);
 
   done:
-    NS_UNMOCK(directory_initiate_command_routerstatus);
+    NS_UNMOCK(directory_initiate_request);
     clear_dir_servers();
     routerlist_free_all();
 }
@@ -4183,7 +4829,7 @@ test_dir_should_init_request_to_dir_auths(void *data)
   dir_server_t *ds = NULL;
   (void) data;
 
-  NS_MOCK(directory_initiate_command_routerstatus);
+  NS_MOCK(directory_initiate_request);
 
   clear_dir_servers();
   routerlist_free_all();
@@ -4194,37 +4840,23 @@ test_dir_should_init_request_to_dir_auths(void *data)
   dir_server_add(ds);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_STATUS_VOTE, 0, NULL);
-  tt_int_op(CALLED(directory_initiate_command_routerstatus), OP_EQ, 1);
+  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 1);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0,
                                      NULL);
-  tt_int_op(CALLED(directory_initiate_command_routerstatus), OP_EQ, 2);
+  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 2);
 
   done:
-    NS_UNMOCK(directory_initiate_command_routerstatus);
+    NS_UNMOCK(directory_initiate_request);
     clear_dir_servers();
     routerlist_free_all();
 }
 
 void
-NS(directory_initiate_command_routerstatus)(const routerstatus_t *status,
-                                            uint8_t dir_purpose,
-                                            uint8_t router_purpose,
-                                            dir_indirection_t indirection,
-                                            const char *resource,
-                                            const char *payload,
-                                            size_t payload_len,
-                                            time_t if_modified_since)
+NS(directory_initiate_request)(directory_request_t *req)
 {
-  (void)status;
-  (void)dir_purpose;
-  (void)router_purpose;
-  (void)indirection;
-  (void)resource;
-  (void)payload;
-  (void)payload_len;
-  (void)if_modified_since;
-  CALLED(directory_initiate_command_routerstatus)++;
+  (void)req;
+  CALLED(directory_initiate_request)++;
 }
 
 static void
@@ -4278,22 +4910,24 @@ mock_check_private_dir(const char *dirname, cpd_check_t check,
 
 static char *
 mock_get_datadir_fname(const or_options_t *options,
+                       directory_root_t roottype,
                        const char *sub1, const char *sub2,
                        const char *suffix)
 {
+  (void) roottype;
   char *rv = NULL;
 
   /*
    * Assert we were called like get_datadir_fname2() or get_datadir_fname(),
    * since that's all we implement here.
    */
-  tt_assert(options != NULL);
-  tt_assert(sub1 != NULL);
+  tt_ptr_op(options, OP_NE, NULL);
+  tt_ptr_op(sub1, OP_NE, NULL);
   /*
    * No particular assertions about sub2, since we could be in the
    * get_datadir_fname() or get_datadir_fname2() case.
    */
-  tt_assert(suffix == NULL);
+  tt_ptr_op(suffix, OP_EQ, NULL);
 
   /* Just duplicate the basename and return it for this mock */
   if (sub2) {
@@ -4320,7 +4954,7 @@ mock_unlink_reset(void)
 static int
 mock_unlink(const char *path)
 {
-  tt_assert(path != NULL);
+  tt_ptr_op(path, OP_NE, NULL);
 
   tor_free(last_unlinked_path);
   last_unlinked_path = tor_strdup(path);
@@ -4349,8 +4983,8 @@ mock_write_str_to_file(const char *path, const char *str, int bin)
 
   (void)bin;
 
-  tt_assert(path != NULL);
-  tt_assert(str != NULL);
+  tt_ptr_op(path, OP_NE, NULL);
+  tt_ptr_op(str, OP_NE, NULL);
 
   len = strlen(str);
   crypto_digest256((char *)hash, str, len, DIGEST_SHA256);
@@ -4437,7 +5071,7 @@ test_dir_dump_unparseable_descriptors(void *data)
   mock_options->MaxUnparseableDescSizeToLog = 1536;
   MOCK(get_options, mock_get_options);
   MOCK(check_private_dir, mock_check_private_dir);
-  MOCK(options_get_datadir_fname2_suffix,
+  MOCK(options_get_dir_fname2_suffix,
        mock_get_datadir_fname);
 
   /*
@@ -4476,7 +5110,7 @@ test_dir_dump_unparseable_descriptors(void *data)
    * Reset the FIFO and check its state
    */
   dump_desc_fifo_cleanup();
-  tt_u64_op(len_descs_dumped, ==, 0);
+  tt_u64_op(len_descs_dumped, OP_EQ, 0);
   tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0);
 
   /*
@@ -4489,21 +5123,21 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_1));
+  tt_u64_op(len_descs_dumped, OP_EQ, strlen(test_desc_1));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 1);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 1);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 1);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_1_hash, DIGEST_SHA256);
 
   /*
    * Reset the FIFO and check its state
    */
   dump_desc_fifo_cleanup();
-  tt_u64_op(len_descs_dumped, ==, 0);
+  tt_u64_op(len_descs_dumped, OP_EQ, 0);
   tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0);
 
   /*
@@ -4511,8 +5145,8 @@ test_dir_dump_unparseable_descriptors(void *data)
    */
   mock_unlink_reset();
   mock_write_str_to_file_reset();
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 0);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 0);
 
   /*
    * (2) Fire off dump_desc() twice; this still should trigger no cleanup.
@@ -4524,14 +5158,14 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_2));
+  tt_u64_op(len_descs_dumped, OP_EQ, strlen(test_desc_2));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 1);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 1);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 1);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_2_hash, DIGEST_SHA256);
 
   /* Second time */
@@ -4540,21 +5174,22 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_2) + strlen(test_desc_3));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_2) + strlen(test_desc_3));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 2);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 2);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_3_hash, DIGEST_SHA256);
 
   /*
    * Reset the FIFO and check its state
    */
   dump_desc_fifo_cleanup();
-  tt_u64_op(len_descs_dumped, ==, 0);
+  tt_u64_op(len_descs_dumped, OP_EQ, 0);
   tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0);
 
   /*
@@ -4562,8 +5197,8 @@ test_dir_dump_unparseable_descriptors(void *data)
    */
   mock_unlink_reset();
   mock_write_str_to_file_reset();
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 0);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 0);
 
   /*
    * (3) Three calls to dump_desc cause a FIFO cleanup
@@ -4575,14 +5210,14 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_4));
+  tt_u64_op(len_descs_dumped, OP_EQ, strlen(test_desc_4));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 1);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 1);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 1);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_4_hash, DIGEST_SHA256);
 
   /* Second time */
@@ -4591,14 +5226,15 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_4) + strlen(test_desc_1));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_4) + strlen(test_desc_1));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 2);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 2);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_1_hash, DIGEST_SHA256);
 
   /* Third time - we should unlink the dump of test_desc_4 here */
@@ -4607,21 +5243,22 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_1) + strlen(test_desc_2));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_1) + strlen(test_desc_2));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 1);
-  tt_int_op(write_str_count, ==, 3);
+  tt_int_op(unlinked_count, OP_EQ, 1);
+  tt_int_op(write_str_count, OP_EQ, 3);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_2_hash, DIGEST_SHA256);
 
   /*
    * Reset the FIFO and check its state
    */
   dump_desc_fifo_cleanup();
-  tt_u64_op(len_descs_dumped, ==, 0);
+  tt_u64_op(len_descs_dumped, OP_EQ, 0);
   tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0);
 
   /*
@@ -4629,8 +5266,8 @@ test_dir_dump_unparseable_descriptors(void *data)
    */
   mock_unlink_reset();
   mock_write_str_to_file_reset();
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 0);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 0);
 
   /*
    * (4) But repeating one (A B B) doesn't overflow and cleanup
@@ -4642,14 +5279,14 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_3));
+  tt_u64_op(len_descs_dumped, OP_EQ, strlen(test_desc_3));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 1);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 1);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 1);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_3_hash, DIGEST_SHA256);
 
   /* Second time */
@@ -4658,14 +5295,15 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_3) + strlen(test_desc_4));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_3) + strlen(test_desc_4));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 2);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 2);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_4_hash, DIGEST_SHA256);
 
   /* Third time */
@@ -4674,21 +5312,22 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_3) + strlen(test_desc_4));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_3) + strlen(test_desc_4));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 2);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 2);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_4_hash, DIGEST_SHA256);
 
   /*
    * Reset the FIFO and check its state
    */
   dump_desc_fifo_cleanup();
-  tt_u64_op(len_descs_dumped, ==, 0);
+  tt_u64_op(len_descs_dumped, OP_EQ, 0);
   tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0);
 
   /*
@@ -4696,8 +5335,8 @@ test_dir_dump_unparseable_descriptors(void *data)
    */
   mock_unlink_reset();
   mock_write_str_to_file_reset();
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 0);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 0);
 
   /*
    * (5) Same for the (A B A) repetition
@@ -4709,14 +5348,14 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_1));
+  tt_u64_op(len_descs_dumped, OP_EQ, strlen(test_desc_1));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 1);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 1);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 1);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_1_hash, DIGEST_SHA256);
 
   /* Second time */
@@ -4725,14 +5364,15 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_1) + strlen(test_desc_2));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_1) + strlen(test_desc_2));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 2);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 2);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_2_hash, DIGEST_SHA256);
 
   /* Third time */
@@ -4741,21 +5381,22 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_1) + strlen(test_desc_2));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_1) + strlen(test_desc_2));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 2);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 2);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_2_hash, DIGEST_SHA256);
 
   /*
    * Reset the FIFO and check its state
    */
   dump_desc_fifo_cleanup();
-  tt_u64_op(len_descs_dumped, ==, 0);
+  tt_u64_op(len_descs_dumped, OP_EQ, 0);
   tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0);
 
   /*
@@ -4763,8 +5404,8 @@ test_dir_dump_unparseable_descriptors(void *data)
    */
   mock_unlink_reset();
   mock_write_str_to_file_reset();
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 0);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 0);
 
   /*
    * (6) (A B B C) triggering overflow on C causes A, not B to be unlinked
@@ -4776,14 +5417,14 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_3));
+  tt_u64_op(len_descs_dumped, OP_EQ, strlen(test_desc_3));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 1);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 1);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 1);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_3_hash, DIGEST_SHA256);
 
   /* Second time */
@@ -4792,14 +5433,15 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_3) + strlen(test_desc_4));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_3) + strlen(test_desc_4));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 2);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 2);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_4_hash, DIGEST_SHA256);
 
   /* Third time */
@@ -4808,14 +5450,15 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_3) + strlen(test_desc_4));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_3) + strlen(test_desc_4));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 2);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 2);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_4_hash, DIGEST_SHA256);
 
   /* Fourth time - we should unlink the dump of test_desc_3 here */
@@ -4824,21 +5467,22 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_4) + strlen(test_desc_1));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_4) + strlen(test_desc_1));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 1);
-  tt_int_op(write_str_count, ==, 3);
+  tt_int_op(unlinked_count, OP_EQ, 1);
+  tt_int_op(write_str_count, OP_EQ, 3);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_1_hash, DIGEST_SHA256);
 
   /*
    * Reset the FIFO and check its state
    */
   dump_desc_fifo_cleanup();
-  tt_u64_op(len_descs_dumped, ==, 0);
+  tt_u64_op(len_descs_dumped, OP_EQ, 0);
   tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0);
 
   /*
@@ -4846,8 +5490,8 @@ test_dir_dump_unparseable_descriptors(void *data)
    */
   mock_unlink_reset();
   mock_write_str_to_file_reset();
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 0);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 0);
 
   /*
    * (7) (A B A C) triggering overflow on C causes B, not A to be unlinked
@@ -4859,14 +5503,14 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_2));
+  tt_u64_op(len_descs_dumped, OP_EQ, strlen(test_desc_2));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 1);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 1);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 1);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_2_hash, DIGEST_SHA256);
 
   /* Second time */
@@ -4875,14 +5519,15 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_2) + strlen(test_desc_3));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_2) + strlen(test_desc_3));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 2);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 2);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_3_hash, DIGEST_SHA256);
 
   /* Third time */
@@ -4891,14 +5536,15 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_2) + strlen(test_desc_3));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_2) + strlen(test_desc_3));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 2);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 2);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_3_hash, DIGEST_SHA256);
 
   /* Fourth time - we should unlink the dump of test_desc_3 here */
@@ -4907,21 +5553,22 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_2) + strlen(test_desc_4));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_2) + strlen(test_desc_4));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 1);
-  tt_int_op(write_str_count, ==, 3);
+  tt_int_op(unlinked_count, OP_EQ, 1);
+  tt_int_op(write_str_count, OP_EQ, 3);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_4_hash, DIGEST_SHA256);
 
   /*
    * Reset the FIFO and check its state
    */
   dump_desc_fifo_cleanup();
-  tt_u64_op(len_descs_dumped, ==, 0);
+  tt_u64_op(len_descs_dumped, OP_EQ, 0);
   tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0);
 
   /*
@@ -4929,8 +5576,8 @@ test_dir_dump_unparseable_descriptors(void *data)
    */
   mock_unlink_reset();
   mock_write_str_to_file_reset();
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 0);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 0);
 
  done:
 
@@ -4942,7 +5589,7 @@ test_dir_dump_unparseable_descriptors(void *data)
   mock_unlink_reset();
   UNMOCK(write_str_to_file);
   mock_write_str_to_file_reset();
-  UNMOCK(options_get_datadir_fname2_suffix);
+  UNMOCK(options_get_dir_fname2_suffix);
   UNMOCK(check_private_dir);
   UNMOCK(get_options);
   tor_free(mock_options);
@@ -4977,7 +5624,7 @@ read_file_to_str_mock(const char *filename, int flags,
   char *result = NULL;
 
   /* Insist we got a filename */
-  tt_assert(filename != NULL);
+  tt_ptr_op(filename, OP_NE, NULL);
 
   /* We ignore flags */
   (void)flags;
@@ -5040,53 +5687,53 @@ test_dir_populate_dump_desc_fifo(void *data)
   reset_read_file_to_str_mock();
 
   /* Check state of unlink mock */
-  tt_int_op(unlinked_count, ==, 0);
+  tt_int_op(unlinked_count, OP_EQ, 0);
 
   /* Some cases that should fail before trying to read the file */
   ent = dump_desc_populate_one_file(dirname, "bar");
-  tt_assert(ent == NULL);
-  tt_int_op(unlinked_count, ==, 1);
-  tt_int_op(read_count, ==, 0);
-  tt_int_op(read_call_count, ==, 0);
+  tt_ptr_op(ent, OP_EQ, NULL);
+  tt_int_op(unlinked_count, OP_EQ, 1);
+  tt_int_op(read_count, OP_EQ, 0);
+  tt_int_op(read_call_count, OP_EQ, 0);
 
   ent = dump_desc_populate_one_file(dirname, "unparseable-desc");
-  tt_assert(ent == NULL);
-  tt_int_op(unlinked_count, ==, 2);
-  tt_int_op(read_count, ==, 0);
-  tt_int_op(read_call_count, ==, 0);
+  tt_ptr_op(ent, OP_EQ, NULL);
+  tt_int_op(unlinked_count, OP_EQ, 2);
+  tt_int_op(read_count, OP_EQ, 0);
+  tt_int_op(read_call_count, OP_EQ, 0);
 
   ent = dump_desc_populate_one_file(dirname, "unparseable-desc.baz");
-  tt_assert(ent == NULL);
-  tt_int_op(unlinked_count, ==, 3);
-  tt_int_op(read_count, ==, 0);
-  tt_int_op(read_call_count, ==, 0);
+  tt_ptr_op(ent, OP_EQ, NULL);
+  tt_int_op(unlinked_count, OP_EQ, 3);
+  tt_int_op(read_count, OP_EQ, 0);
+  tt_int_op(read_call_count, OP_EQ, 0);
 
   ent = dump_desc_populate_one_file(
       dirname,
       "unparseable-desc.08AE85E90461F59E");
-  tt_assert(ent == NULL);
-  tt_int_op(unlinked_count, ==, 4);
-  tt_int_op(read_count, ==, 0);
-  tt_int_op(read_call_count, ==, 0);
+  tt_ptr_op(ent, OP_EQ, NULL);
+  tt_int_op(unlinked_count, OP_EQ, 4);
+  tt_int_op(read_count, OP_EQ, 0);
+  tt_int_op(read_call_count, OP_EQ, 0);
 
   ent = dump_desc_populate_one_file(
       dirname,
       "unparseable-desc.08AE85E90461F59EDF0981323F3A70D02B55AB54B44B04F"
       "287D72F7B72F242E85C8CB0EDA8854A99");
-  tt_assert(ent == NULL);
-  tt_int_op(unlinked_count, ==, 5);
-  tt_int_op(read_count, ==, 0);
-  tt_int_op(read_call_count, ==, 0);
+  tt_ptr_op(ent, OP_EQ, NULL);
+  tt_int_op(unlinked_count, OP_EQ, 5);
+  tt_int_op(read_count, OP_EQ, 0);
+  tt_int_op(read_call_count, OP_EQ, 0);
 
   /* This is a correct-length digest but base16_decode() will fail */
   ent = dump_desc_populate_one_file(
       dirname,
       "unparseable-desc.68219B8BGE64B705A6FFC728C069DC596216D60A7D7520C"
       "D5ECE250D912E686B");
-  tt_assert(ent == NULL);
-  tt_int_op(unlinked_count, ==, 6);
-  tt_int_op(read_count, ==, 0);
-  tt_int_op(read_call_count, ==, 0);
+  tt_ptr_op(ent, OP_EQ, NULL);
+  tt_int_op(unlinked_count, OP_EQ, 6);
+  tt_int_op(read_count, OP_EQ, 0);
+  tt_int_op(read_call_count, OP_EQ, 0);
 
   /* This one has a correctly formed filename and should try reading */
 
@@ -5095,10 +5742,10 @@ test_dir_populate_dump_desc_fifo(void *data)
       dirname,
       "unparseable-desc.DF0981323F3A70D02B55AB54B44B04F287D72F7B72F242E"
       "85C8CB0EDA8854A99");
-  tt_assert(ent == NULL);
-  tt_int_op(unlinked_count, ==, 7);
-  tt_int_op(read_count, ==, 0);
-  tt_int_op(read_call_count, ==, 1);
+  tt_ptr_op(ent, OP_EQ, NULL);
+  tt_int_op(unlinked_count, OP_EQ, 7);
+  tt_int_op(read_count, OP_EQ, 0);
+  tt_int_op(read_call_count, OP_EQ, 1);
 
   /* This read will succeed but the digest won't match the file content */
   fname =
@@ -5111,10 +5758,10 @@ test_dir_populate_dump_desc_fifo(void *data)
   file_stat.st_mtime = 123456;
   ent = dump_desc_populate_one_file(dirname, fname);
   enforce_expected_filename = 0;
-  tt_assert(ent == NULL);
-  tt_int_op(unlinked_count, ==, 8);
-  tt_int_op(read_count, ==, 1);
-  tt_int_op(read_call_count, ==, 2);
+  tt_ptr_op(ent, OP_EQ, NULL);
+  tt_int_op(unlinked_count, OP_EQ, 8);
+  tt_int_op(read_count, OP_EQ, 1);
+  tt_int_op(read_call_count, OP_EQ, 2);
   tor_free(expected_filename);
   tor_free(file_content);
 
@@ -5127,13 +5774,13 @@ test_dir_populate_dump_desc_fifo(void *data)
   file_content_len = strlen(file_content);
   file_stat.st_mtime = 789012;
   ent = dump_desc_populate_one_file(dirname, fname);
-  tt_assert(ent != NULL);
-  tt_int_op(unlinked_count, ==, 8);
-  tt_int_op(read_count, ==, 2);
-  tt_int_op(read_call_count, ==, 3);
+  tt_ptr_op(ent, OP_NE, NULL);
+  tt_int_op(unlinked_count, OP_EQ, 8);
+  tt_int_op(read_count, OP_EQ, 2);
+  tt_int_op(read_call_count, OP_EQ, 3);
   tt_str_op(ent->filename, OP_EQ, expected_filename);
-  tt_int_op(ent->len, ==, file_content_len);
-  tt_int_op(ent->when, ==, file_stat.st_mtime);
+  tt_int_op(ent->len, OP_EQ, file_content_len);
+  tt_int_op(ent->when, OP_EQ, file_stat.st_mtime);
   tor_free(ent->filename);
   tor_free(ent);
   tor_free(expected_filename);
@@ -5142,9 +5789,9 @@ test_dir_populate_dump_desc_fifo(void *data)
    * Reset the mocks and check their state
    */
   mock_unlink_reset();
-  tt_int_op(unlinked_count, ==, 0);
+  tt_int_op(unlinked_count, OP_EQ, 0);
   reset_read_file_to_str_mock();
-  tt_int_op(read_count, ==, 0);
+  tt_int_op(read_count, OP_EQ, 0);
 
  done:
 
@@ -5167,9 +5814,9 @@ listdir_mock(const char *dname)
   (void)dname;
 
   l = smartlist_new();
-  smartlist_add(l, tor_strdup("foo"));
-  smartlist_add(l, tor_strdup("bar"));
-  smartlist_add(l, tor_strdup("baz"));
+  smartlist_add_strdup(l, "foo");
+  smartlist_add_strdup(l, "bar");
+  smartlist_add_strdup(l, "baz");
 
   return l;
 }
@@ -5266,17 +5913,26 @@ mock_networkstatus_consensus_can_use_extra_fallbacks(
   return mock_networkstatus_consensus_can_use_extra_fallbacks_value;
 }
 
-/* data is a 2 character nul-terminated string.
+static int mock_num_bridges_usable_value = 0;
+static int
+mock_num_bridges_usable(int use_maybe_reachable)
+{
+  (void)use_maybe_reachable;
+  return mock_num_bridges_usable_value;
+}
+
+/* data is a 3 character nul-terminated string.
  * If data[0] is 'b', set bootstrapping, anything else means not bootstrapping
  * If data[1] is 'f', set extra fallbacks, anything else means no extra
+ * If data[2] is 'f', set running bridges, anything else means no extra
  * fallbacks.
  */
 static void
-test_dir_find_dl_schedule(void* data)
+test_dir_find_dl_min_delay(void* data)
 {
   const char *str = (const char *)data;
 
-  tt_assert(strlen(data) == 2);
+  tt_assert(strlen(data) == 3);
 
   if (str[0] == 'b') {
     mock_networkstatus_consensus_is_bootstrapping_value = 1;
@@ -5290,49 +5946,60 @@ test_dir_find_dl_schedule(void* data)
     mock_networkstatus_consensus_can_use_extra_fallbacks_value = 0;
   }
 
+  if (str[2] == 'r') {
+    /* Any positive, non-zero value should work */
+    mock_num_bridges_usable_value = 2;
+  } else {
+    mock_num_bridges_usable_value = 0;
+  }
+
   MOCK(networkstatus_consensus_is_bootstrapping,
        mock_networkstatus_consensus_is_bootstrapping);
   MOCK(networkstatus_consensus_can_use_extra_fallbacks,
        mock_networkstatus_consensus_can_use_extra_fallbacks);
+  MOCK(num_bridges_usable,
+       mock_num_bridges_usable);
 
   download_status_t dls;
-  smartlist_t server, client, server_cons, client_cons;
-  smartlist_t client_boot_auth_only_cons, client_boot_auth_cons;
-  smartlist_t client_boot_fallback_cons, bridge;
+
+  const int server=10, client=20, server_cons=30, client_cons=40;
+  const int client_boot_auth_only_cons=50, client_boot_auth_cons=60;
+  const int client_boot_fallback_cons=70, bridge=80, bridge_bootstrap=90;
 
   mock_options = tor_malloc(sizeof(or_options_t));
   reset_options(mock_options, &mock_get_options_calls);
   MOCK(get_options, mock_get_options);
 
-  mock_options->TestingServerDownloadSchedule = &server;
-  mock_options->TestingClientDownloadSchedule = &client;
-  mock_options->TestingServerConsensusDownloadSchedule = &server_cons;
-  mock_options->TestingClientConsensusDownloadSchedule = &client_cons;
-  mock_options->ClientBootstrapConsensusAuthorityOnlyDownloadSchedule =
-    &client_boot_auth_only_cons;
-  mock_options->ClientBootstrapConsensusAuthorityDownloadSchedule =
-    &client_boot_auth_cons;
-  mock_options->ClientBootstrapConsensusFallbackDownloadSchedule =
-    &client_boot_fallback_cons;
-  mock_options->TestingBridgeDownloadSchedule = &bridge;
+  mock_options->TestingServerDownloadInitialDelay = server;
+  mock_options->TestingClientDownloadInitialDelay = client;
+  mock_options->TestingServerConsensusDownloadInitialDelay = server_cons;
+  mock_options->TestingClientConsensusDownloadInitialDelay = client_cons;
+  mock_options->ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay =
+    client_boot_auth_only_cons;
+  mock_options->ClientBootstrapConsensusAuthorityDownloadInitialDelay =
+    client_boot_auth_cons;
+  mock_options->ClientBootstrapConsensusFallbackDownloadInitialDelay =
+    client_boot_fallback_cons;
+  mock_options->TestingBridgeDownloadInitialDelay = bridge;
+  mock_options->TestingBridgeBootstrapDownloadInitialDelay = bridge_bootstrap;
 
   dls.schedule = DL_SCHED_GENERIC;
   /* client */
   mock_options->ClientOnly = 1;
-  tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, &client);
+  tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, client);
   mock_options->ClientOnly = 0;
 
   /* dir mode */
   mock_options->DirPort_set = 1;
   mock_options->DirCache = 1;
-  tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, &server);
+  tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, server);
   mock_options->DirPort_set = 0;
   mock_options->DirCache = 0;
 
   dls.schedule = DL_SCHED_CONSENSUS;
   /* public server mode */
   mock_options->ORPort_set = 1;
-  tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, &server_cons);
+  tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, server_cons);
   mock_options->ORPort_set = 0;
 
   /* client and bridge modes */
@@ -5341,30 +6008,30 @@ test_dir_find_dl_schedule(void* data)
       dls.want_authority = 1;
       /* client */
       mock_options->ClientOnly = 1;
-      tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ,
-                &client_boot_auth_cons);
+      tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ,
+                client_boot_auth_cons);
       mock_options->ClientOnly = 0;
 
       /* bridge relay */
       mock_options->ORPort_set = 1;
       mock_options->BridgeRelay = 1;
-      tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ,
-                &client_boot_auth_cons);
+      tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ,
+                client_boot_auth_cons);
       mock_options->ORPort_set = 0;
       mock_options->BridgeRelay = 0;
 
       dls.want_authority = 0;
       /* client */
       mock_options->ClientOnly = 1;
-      tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ,
-                &client_boot_fallback_cons);
+      tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ,
+                client_boot_fallback_cons);
       mock_options->ClientOnly = 0;
 
       /* bridge relay */
       mock_options->ORPort_set = 1;
       mock_options->BridgeRelay = 1;
-      tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ,
-                &client_boot_fallback_cons);
+      tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ,
+                client_boot_fallback_cons);
       mock_options->ORPort_set = 0;
       mock_options->BridgeRelay = 0;
 
@@ -5372,30 +6039,30 @@ test_dir_find_dl_schedule(void* data)
       /* dls.want_authority is ignored */
       /* client */
       mock_options->ClientOnly = 1;
-      tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ,
-                &client_boot_auth_only_cons);
+      tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ,
+                client_boot_auth_only_cons);
       mock_options->ClientOnly = 0;
 
       /* bridge relay */
       mock_options->ORPort_set = 1;
       mock_options->BridgeRelay = 1;
-      tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ,
-                &client_boot_auth_only_cons);
+      tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ,
+                client_boot_auth_only_cons);
       mock_options->ORPort_set = 0;
       mock_options->BridgeRelay = 0;
     }
   } else {
     /* client */
     mock_options->ClientOnly = 1;
-    tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ,
-              &client_cons);
+    tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ,
+              client_cons);
     mock_options->ClientOnly = 0;
 
     /* bridge relay */
     mock_options->ORPort_set = 1;
     mock_options->BridgeRelay = 1;
-    tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ,
-              &client_cons);
+    tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ,
+              client_cons);
     mock_options->ORPort_set = 0;
     mock_options->BridgeRelay = 0;
   }
@@ -5403,11 +6070,17 @@ test_dir_find_dl_schedule(void* data)
   dls.schedule = DL_SCHED_BRIDGE;
   /* client */
   mock_options->ClientOnly = 1;
-  tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, &bridge);
+  mock_options->UseBridges = 1;
+  if (num_bridges_usable(0) > 0) {
+    tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, bridge);
+  } else {
+    tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, bridge_bootstrap);
+  }
 
  done:
   UNMOCK(networkstatus_consensus_is_bootstrapping);
   UNMOCK(networkstatus_consensus_can_use_extra_fallbacks);
+  UNMOCK(num_bridges_usable);
   UNMOCK(get_options);
   tor_free(mock_options);
   mock_options = NULL;
@@ -5421,9 +6094,8 @@ test_dir_assumed_flags(void *arg)
   memarea_t *area = memarea_new();
   routerstatus_t *rs = NULL;
 
-  /* First, we should always assume that the Running flag is set, even
-   * when it isn't listed, since the consensus method is always
-   * higher than 4. */
+  /* We can assume that consensus method is higher than 24, so Running and
+   * Valid are always implicitly set */
   const char *str1 =
     "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 "
        "192.168.0.1 9001 0\n"
@@ -5432,17 +6104,6 @@ test_dir_assumed_flags(void *arg)
 
   const char *cp = str1;
   rs = routerstatus_parse_entry_from_string(area, &cp, tokens, NULL, NULL,
-                                            23, FLAV_MICRODESC);
-  tt_assert(rs);
-  tt_assert(rs->is_flagged_running);
-  tt_assert(! rs->is_valid);
-  tt_assert(! rs->is_exit);
-  tt_assert(rs->is_fast);
-  routerstatus_free(rs);
-
-  /* With method 24 or later, we can assume "valid" is set. */
-  cp = str1;
-  rs = routerstatus_parse_entry_from_string(area, &cp, tokens, NULL, NULL,
                                             24, FLAV_MICRODESC);
   tt_assert(rs);
   tt_assert(rs->is_flagged_running);
@@ -5457,6 +6118,182 @@ test_dir_assumed_flags(void *arg)
 }
 
 static void
+test_dir_post_parsing(void *arg)
+{
+  (void) arg;
+
+  /* Test the version parsing from an HS descriptor publish request. */
+  {
+    const char *end;
+    const char *prefix = "/tor/hs/";
+    int version = parse_hs_version_from_post("/tor/hs//publish", prefix, &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+    version = parse_hs_version_from_post("/tor/hs/a/publish", prefix, &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+    version = parse_hs_version_from_post("/tor/hs/3/publish", prefix, &end);
+    tt_int_op(version, OP_EQ, 3);
+    tt_str_op(end, OP_EQ, "/publish");
+    version = parse_hs_version_from_post("/tor/hs/42/publish", prefix, &end);
+    tt_int_op(version, OP_EQ, 42);
+    tt_str_op(end, OP_EQ, "/publish");
+    version = parse_hs_version_from_post("/tor/hs/18163/publish",prefix, &end);
+    tt_int_op(version, OP_EQ, 18163);
+    tt_str_op(end, OP_EQ, "/publish");
+    version = parse_hs_version_from_post("JUNKJUNKJUNK", prefix, &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+    version = parse_hs_version_from_post("/tor/hs/3/publish", "blah", &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+    /* Missing the '/' at the end of the prefix. */
+    version = parse_hs_version_from_post("/tor/hs/3/publish", "/tor/hs", &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+    version = parse_hs_version_from_post("/random/blah/tor/hs/3/publish",
+                                         prefix, &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+    version = parse_hs_version_from_post("/tor/hs/3/publish/random/junk",
+                                         prefix, &end);
+    tt_int_op(version, OP_EQ, 3);
+    tt_str_op(end, OP_EQ, "/publish/random/junk");
+    version = parse_hs_version_from_post("/tor/hs/-1/publish", prefix, &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+    /* INT_MAX */
+    version = parse_hs_version_from_post("/tor/hs/2147483647/publish",
+                                         prefix, &end);
+    tt_int_op(version, OP_EQ, INT_MAX);
+    tt_str_op(end, OP_EQ, "/publish");
+    /* INT_MAX + 1*/
+    version = parse_hs_version_from_post("/tor/hs/2147483648/publish",
+                                         prefix, &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+  }
+
+ done:
+  ;
+}
+
+static void
+test_dir_platform_str(void *arg)
+{
+  char platform[256];
+  (void)arg;
+  platform[0] = 0;
+  get_platform_str(platform, sizeof(platform));
+  tt_int_op((int)strlen(platform), OP_GT, 0);
+  tt_assert(!strcmpstart(platform, "Tor "));
+
+  tor_version_t ver;
+  // make sure this is a tor version, a real actual tor version.
+  tt_int_op(tor_version_parse_platform(platform, &ver, 1), OP_EQ, 1);
+
+  TT_BLATHER(("%d.%d.%d.%d", ver.major, ver.minor, ver.micro, ver.patchlevel));
+
+  // Handle an example version.
+  tt_int_op(tor_version_parse_platform(
+        "Tor 0.3.3.3 (foo) (git-xyzzy) on a potato", &ver, 1), OP_EQ, 1);
+ done:
+  ;
+}
+
+static networkstatus_t *mock_networkstatus;
+
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f)
+{
+  (void)f;
+  return mock_networkstatus;
+}
+
+static void
+test_dir_networkstatus_consensus_has_ipv6(void *arg)
+{
+  (void)arg;
+
+  int has_ipv6 = 0;
+
+  /* Init options and networkstatus */
+  or_options_t our_options;
+  mock_options = &our_options;
+  reset_options(mock_options, &mock_get_options_calls);
+  MOCK(get_options, mock_get_options);
+
+  networkstatus_t our_networkstatus;
+  mock_networkstatus = &our_networkstatus;
+  memset(mock_networkstatus, 0, sizeof(*mock_networkstatus));
+  MOCK(networkstatus_get_latest_consensus_by_flavor,
+       mock_networkstatus_get_latest_consensus_by_flavor);
+
+  /* A live consensus */
+  mock_networkstatus->valid_after = time(NULL) - 3600;
+  mock_networkstatus->valid_until = time(NULL) + 3600;
+
+  /* Test the bounds for A lines in the NS consensus */
+  mock_options->UseMicrodescriptors = 0;
+
+  mock_networkstatus->consensus_method = MIN_SUPPORTED_CONSENSUS_METHOD;
+  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
+  tt_assert(has_ipv6);
+
+  /* Test the bounds for A lines in the microdesc consensus */
+  mock_options->UseMicrodescriptors = 1;
+
+  mock_networkstatus->consensus_method =
+      MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS;
+  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
+  tt_assert(has_ipv6);
+
+  mock_networkstatus->consensus_method = MAX_SUPPORTED_CONSENSUS_METHOD + 20;
+  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
+  tt_assert(has_ipv6);
+
+  mock_networkstatus->consensus_method =
+      MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS + 1;
+  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
+  tt_assert(has_ipv6);
+
+  mock_networkstatus->consensus_method =
+      MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS + 20;
+  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
+  tt_assert(has_ipv6);
+
+  mock_networkstatus->consensus_method =
+      MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS - 1;
+  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
+  tt_assert(!has_ipv6);
+
+  /* Test the edge cases */
+  mock_options->UseMicrodescriptors = 1;
+  mock_networkstatus->consensus_method =
+      MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS;
+
+  /* Reasonably live */
+  mock_networkstatus->valid_until = approx_time() - 60;
+  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
+  tt_assert(has_ipv6);
+
+  /* Not reasonably live */
+  mock_networkstatus->valid_after = approx_time() - 24*60*60 - 3600;
+  mock_networkstatus->valid_until = approx_time() - 24*60*60 - 60;
+  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
+  tt_assert(!has_ipv6);
+
+  /* NULL consensus */
+  mock_networkstatus = NULL;
+  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
+  tt_assert(!has_ipv6);
+
+ done:
+  UNMOCK(get_options);
+  UNMOCK(networkstatus_get_latest_consensus_by_flavor);
+}
+
+static void
 test_dir_format_versions_list(void *arg)
 {
   (void)arg;
@@ -5525,12 +6362,14 @@ struct testcase_t dir_tests[] = {
   DIR(parse_router_list, TT_FORK),
   DIR(load_routers, TT_FORK),
   DIR(load_extrainfo, TT_FORK),
+  DIR(getinfo_extra, 0),
   DIR_LEGACY(versions),
   DIR_LEGACY(fp_pairs),
   DIR(split_fps, 0),
-  DIR_LEGACY(dirserv_read_measured_bandwidths),
   DIR_LEGACY(measured_bw_kb),
+  DIR_LEGACY(measured_bw_kb_line_is_after_headers),
   DIR_LEGACY(measured_bw_kb_cache),
+  DIR_LEGACY(dirserv_read_measured_bandwidths),
   DIR_LEGACY(param_voting),
   DIR(param_voting_lookup, 0),
   DIR_LEGACY(v3_networkstatus),
@@ -5541,12 +6380,17 @@ struct testcase_t dir_tests[] = {
   DIR(fmt_control_ns, 0),
   DIR(dirserv_set_routerstatus_testing, 0),
   DIR(http_handling, 0),
-  DIR(purpose_needs_anonymity, 0),
+  DIR(purpose_needs_anonymity_returns_true_for_bridges, 0),
+  DIR(purpose_needs_anonymity_returns_false_for_own_bridge_desc, 0),
+  DIR(purpose_needs_anonymity_returns_true_by_default, 0),
+  DIR(purpose_needs_anonymity_returns_true_for_sensitive_purpose, 0),
+  DIR(purpose_needs_anonymity_ret_false_for_non_sensitive_conn, 0),
+  DIR(post_parsing, 0),
   DIR(fetch_type, 0),
   DIR(packages, 0),
-  DIR(download_status_schedule, 0),
   DIR(download_status_random_backoff, 0),
-  DIR(download_status_increment, 0),
+  DIR(download_status_random_backoff_ranges, 0),
+  DIR(download_status_increment, TT_FORK),
   DIR(authdir_type_to_string, 0),
   DIR(conn_purpose_to_string, 0),
   DIR(should_use_directory_guards, 0),
@@ -5557,11 +6401,18 @@ struct testcase_t dir_tests[] = {
   DIR(dump_unparseable_descriptors, 0),
   DIR(populate_dump_desc_fifo, 0),
   DIR(populate_dump_desc_fifo_2, 0),
-  DIR_ARG(find_dl_schedule, TT_FORK, "bf"),
-  DIR_ARG(find_dl_schedule, TT_FORK, "ba"),
-  DIR_ARG(find_dl_schedule, TT_FORK, "cf"),
-  DIR_ARG(find_dl_schedule, TT_FORK, "ca"),
+  DIR_ARG(find_dl_min_delay, TT_FORK, "bfd"),
+  DIR_ARG(find_dl_min_delay, TT_FORK, "bad"),
+  DIR_ARG(find_dl_min_delay, TT_FORK, "cfd"),
+  DIR_ARG(find_dl_min_delay, TT_FORK, "cad"),
+  DIR_ARG(find_dl_min_delay, TT_FORK, "bfr"),
+  DIR_ARG(find_dl_min_delay, TT_FORK, "bar"),
+  DIR_ARG(find_dl_min_delay, TT_FORK, "cfr"),
+  DIR_ARG(find_dl_min_delay, TT_FORK, "car"),
   DIR(assumed_flags, 0),
+  DIR(networkstatus_compute_bw_weights_v10, 0),
+  DIR(platform_str, 0),
+  DIR(networkstatus_consensus_has_ipv6, TT_FORK),
   DIR(format_versions_list, TT_FORK),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_dir_common.c b/src/test/test_dir_common.c
index ca43dd4c04..3723d6c31b 100644
--- a/src/test/test_dir_common.c
+++ b/src/test/test_dir_common.c
@@ -1,18 +1,26 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 #define DIRVOTE_PRIVATE
-#include "crypto.h"
-#include "test.h"
-#include "container.h"
-#include "or.h"
-#include "dirvote.h"
-#include "nodelist.h"
-#include "routerlist.h"
-#include "test_dir_common.h"
+#include "test/test.h"
+#include "core/or/or.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/dirparse/authcert_parse.h"
+#include "feature/dirparse/ns_parse.h"
+#include "test/test_dir_common.h"
+#include "feature/dircommon/voting_schedule.h"
+
+#include "feature/nodelist/authority_cert_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/networkstatus_voter_info_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/dirauth/vote_microdesc_hash_st.h"
+#include "feature/nodelist/vote_routerstatus_st.h"
 
 void dir_common_setup_vote(networkstatus_t **vote, time_t now);
 networkstatus_t * dir_common_add_rs_and_parse(networkstatus_t *vote,
@@ -146,7 +154,7 @@ dir_common_gen_routerstatus_for_v3ns(int idx, time_t now)
       break;
     default:
       /* Shouldn't happen */
-      tt_assert(0);
+      tt_abort();
   }
   if (vrs) {
     vrs->microdesc = tor_malloc_zero(sizeof(vote_microdesc_hash_t));
diff --git a/src/test/test_dir_common.h b/src/test/test_dir_common.h
index 9682b0db49..d6c5241b14 100644
--- a/src/test/test_dir_common.h
+++ b/src/test/test_dir_common.h
@@ -1,11 +1,10 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#include "or.h"
-#include "networkstatus.h"
-#include "routerparse.h"
+#include "core/or/or.h"
+#include "feature/nodelist/networkstatus.h"
 
 #define TEST_DIR_ROUTER_ID_1 3
 #define TEST_DIR_ROUTER_ID_2 5
diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c
index a0f22f1f0c..90691fff94 100644
--- a/src/test/test_dir_handle_get.c
+++ b/src/test/test_dir_handle_get.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define RENDCOMMON_PRIVATE
@@ -8,35 +8,50 @@
 #define CONNECTION_PRIVATE
 #define CONFIG_PRIVATE
 #define RENDCACHE_PRIVATE
-
-#include "or.h"
-#include "config.h"
-#include "connection.h"
-#include "directory.h"
-#include "test.h"
-#include "connection.h"
-#include "rendcommon.h"
-#include "rendcache.h"
-#include "router.h"
-#include "routerlist.h"
-#include "rend_test_helpers.h"
-#include "microdesc.h"
-#include "test_helpers.h"
-#include "nodelist.h"
-#include "entrynodes.h"
-#include "routerparse.h"
-#include "networkstatus.h"
-#include "geoip.h"
-#include "dirserv.h"
-#include "torgzip.h"
-#include "dirvote.h"
+#define DIRCACHE_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "feature/dircache/consdiffmgr.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dircache/dircache.h"
+#include "test/test.h"
+#include "lib/compress/compress.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/rend/rendcache.h"
+#include "feature/relay/router.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/routerlist.h"
+#include "test/rend_test_helpers.h"
+#include "feature/nodelist/microdesc.h"
+#include "test/test_helpers.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/client/entrynodes.h"
+#include "feature/dirparse/authcert_parse.h"
+#include "feature/nodelist/networkstatus.h"
+#include "core/proto/proto_http.h"
+#include "lib/geoip/geoip.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirauth/dirvote.h"
+#include "test/log_test_helpers.h"
+#include "feature/dircommon/voting_schedule.h"
+
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/dirclient/dir_server_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/rend/rend_encoded_v2_service_descriptor_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerlist_st.h"
 
 #ifdef _WIN32
 /* For mkdir() */
 #include <direct.h>
 #else
 #include <dirent.h>
-#endif
+#endif /* defined(_WIN32) */
 
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
 DISABLE_GCC_WARNING(overlength-strings)
@@ -50,22 +65,10 @@ ENABLE_GCC_WARNING(overlength-strings)
 
 #define NS_MODULE dir_handle_get
 
-static void
-connection_write_to_buf_mock(const char *string, size_t len,
-                             connection_t *conn, int zlib)
-{
-  (void) zlib;
-
-  tor_assert(string);
-  tor_assert(conn);
-
-  write_to_buf(string, len, conn->outbuf);
-}
-
-#define GET(path) "GET " path " HTTP/1.0\r\n\r\n"
 #define NOT_FOUND "HTTP/1.0 404 Not found\r\n\r\n"
 #define BAD_REQUEST "HTTP/1.0 400 Bad request\r\n\r\n"
 #define SERVER_BUSY "HTTP/1.0 503 Directory busy, try again later\r\n\r\n"
+#define TOO_OLD "HTTP/1.0 404 Consensus is too old\r\n\r\n"
 #define NOT_ENOUGH_CONSENSUS_SIGNATURES "HTTP/1.0 404 " \
   "Consensus not signed by sufficient number of requested authorities\r\n\r\n"
 
@@ -74,6 +77,7 @@ new_dir_conn(void)
 {
   dir_connection_t *conn = dir_connection_new(AF_INET);
   tor_addr_from_ipv4h(&conn->base_.addr, 0x7f000001);
+  TO_CONN(conn)->address = tor_strdup("127.0.0.1");
   return conn;
 }
 
@@ -96,7 +100,7 @@ test_dir_handle_get_bad_request(void *data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -125,7 +129,7 @@ test_dir_handle_get_v1_command_not_found(void *data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -172,7 +176,7 @@ test_dir_handle_get_v1_command(void *data)
   done:
     UNMOCK(connection_write_to_buf_impl_);
     UNMOCK(get_dirportfrontpage);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
 }
@@ -198,7 +202,7 @@ test_dir_handle_get_not_found(void *data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -233,7 +237,7 @@ test_dir_handle_get_robots_txt(void *data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
 }
@@ -262,7 +266,7 @@ test_dir_handle_get_rendezvous2_not_found_if_not_encrypted(void *data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -290,7 +294,7 @@ test_dir_handle_get_rendezvous2_on_encrypted_conn_with_invalid_desc_id(
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -323,7 +327,7 @@ test_dir_handle_get_rendezvous2_on_encrypted_conn_not_well_formed(void *data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -352,7 +356,7 @@ test_dir_handle_get_rendezvous2_not_found(void *data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     rend_cache_free_all();
 }
@@ -432,7 +436,7 @@ test_dir_handle_get_rendezvous2_on_encrypted_conn_success(void *data)
     UNMOCK(connection_write_to_buf_impl_);
     NS_UNMOCK(router_get_my_routerinfo);
 
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
     rend_encoded_v2_service_descriptor_free(desc_holder);
@@ -465,7 +469,7 @@ test_dir_handle_get_micro_d_not_found(void *data)
   done:
     UNMOCK(connection_write_to_buf_impl_);
 
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -476,6 +480,9 @@ init_mock_options(void)
   mock_options = tor_malloc(sizeof(or_options_t));
   memset(mock_options, 0, sizeof(or_options_t));
   mock_options->TestingTorNetwork = 1;
+  mock_options->DataDirectory = tor_strdup(get_fname_rnd("datadir_tmp"));
+  mock_options->CacheDirectory = tor_strdup(mock_options->DataDirectory);
+  check_private_dir(mock_options->DataDirectory, CPD_CREATE, NULL);
 }
 
 static const or_options_t *
@@ -512,14 +519,6 @@ test_dir_handle_get_micro_d(void *data)
 
   /* SETUP */
   init_mock_options();
-  const char *fn = get_fname("dir_handle_datadir_test1");
-  mock_options->DataDirectory = tor_strdup(fn);
-
-#ifdef _WIN32
-  tt_int_op(0, OP_EQ, mkdir(mock_options->DataDirectory));
-#else
-  tt_int_op(0, OP_EQ, mkdir(mock_options->DataDirectory, 0700));
-#endif
 
   /* Add microdesc to cache */
   crypto_digest256(digest, microdesc, strlen(microdesc), DIGEST_SHA256);
@@ -555,7 +554,7 @@ test_dir_handle_get_micro_d(void *data)
     UNMOCK(connection_write_to_buf_impl_);
 
     or_options_free(mock_options); mock_options = NULL;
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
     smartlist_free(list);
@@ -579,14 +578,6 @@ test_dir_handle_get_micro_d_server_busy(void *data)
 
   /* SETUP */
   init_mock_options();
-  const char *fn = get_fname("dir_handle_datadir_test2");
-  mock_options->DataDirectory = tor_strdup(fn);
-
-#ifdef _WIN32
-  tt_int_op(0, OP_EQ, mkdir(mock_options->DataDirectory));
-#else
-  tt_int_op(0, OP_EQ, mkdir(mock_options->DataDirectory, 0700));
-#endif
 
   /* Add microdesc to cache */
   crypto_digest256(digest, microdesc, strlen(microdesc), DIGEST_SHA256);
@@ -617,7 +608,7 @@ test_dir_handle_get_micro_d_server_busy(void *data)
     UNMOCK(connection_write_to_buf_impl_);
 
     or_options_free(mock_options); mock_options = NULL;
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     smartlist_free(list);
     microdesc_free_all();
@@ -654,7 +645,7 @@ test_dir_handle_get_networkstatus_bridges_not_found_without_auth(void *data)
     UNMOCK(get_options);
     UNMOCK(connection_write_to_buf_impl_);
     or_options_free(mock_options); mock_options = NULL;
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -694,7 +685,7 @@ test_dir_handle_get_networkstatus_bridges(void *data)
     UNMOCK(get_options);
     UNMOCK(connection_write_to_buf_impl_);
     or_options_free(mock_options); mock_options = NULL;
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -731,7 +722,7 @@ test_dir_handle_get_networkstatus_bridges_not_found_wrong_auth(void *data)
     UNMOCK(get_options);
     UNMOCK(connection_write_to_buf_impl_);
     or_options_free(mock_options); mock_options = NULL;
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -754,12 +745,12 @@ test_dir_handle_get_server_descriptors_not_found(void* data)
                       NULL, NULL, 1, 0);
 
   tt_str_op(NOT_FOUND, OP_EQ, header);
-  tt_int_op(conn->dir_spool_src, OP_EQ, DIR_SPOOL_SERVER_BY_FP);
+  tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
     or_options_free(mock_options); mock_options = NULL;
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -784,6 +775,7 @@ test_dir_handle_get_server_descriptors_all(void* data)
   tt_int_op(smartlist_len(our_routerlist->routers), OP_GE, 1);
   mock_routerinfo = smartlist_get(our_routerlist->routers, 0);
   set_server_identity_key(mock_routerinfo->identity_pkey);
+  mock_routerinfo->cache_info.published_on = time(NULL);
 
   /* Treat "all" requests as if they were unencrypted */
   mock_routerinfo->cache_info.send_unencrypted = 1;
@@ -798,7 +790,7 @@ test_dir_handle_get_server_descriptors_all(void* data)
   //which is smaller than that by annotation_len bytes
   fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
                       &body, &body_used,
-                      mock_routerinfo->cache_info.signed_descriptor_len+1, 0);
+                      1024*1024, 0);
 
   tt_assert(header);
   tt_assert(body);
@@ -814,12 +806,12 @@ test_dir_handle_get_server_descriptors_all(void* data)
 
   tt_str_op(body, OP_EQ, mock_routerinfo->cache_info.signed_descriptor_body +
                          mock_routerinfo->cache_info.annotations_len);
-  tt_int_op(conn->dir_spool_src, OP_EQ, DIR_SPOOL_NONE);
+  tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
     NS_UNMOCK(router_get_my_routerinfo);
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
 
@@ -891,8 +883,9 @@ test_dir_handle_get_server_descriptors_authority(void* data)
   mock_routerinfo->cache_info.signed_descriptor_body =
     tor_strdup(TEST_DESCRIPTOR);
   mock_routerinfo->cache_info.signed_descriptor_len =
-    strlen(TEST_DESCRIPTOR) - annotation_len;;
+    strlen(TEST_DESCRIPTOR) - annotation_len;
   mock_routerinfo->cache_info.annotations_len = annotation_len;
+  mock_routerinfo->cache_info.published_on = time(NULL);
 
   conn = new_dir_conn();
 
@@ -915,14 +908,14 @@ test_dir_handle_get_server_descriptors_authority(void* data)
   tt_int_op(body_used, OP_EQ, strlen(body));
 
   tt_str_op(body, OP_EQ, TEST_DESCRIPTOR + annotation_len);
-  tt_int_op(conn->dir_spool_src, OP_EQ, DIR_SPOOL_NONE);
+  tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
     NS_UNMOCK(router_get_my_routerinfo);
     UNMOCK(connection_write_to_buf_impl_);
     tor_free(mock_routerinfo->cache_info.signed_descriptor_body);
     tor_free(mock_routerinfo);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
     crypto_pk_free(identity_pkey);
@@ -957,6 +950,7 @@ test_dir_handle_get_server_descriptors_fp(void* data)
   mock_routerinfo->cache_info.signed_descriptor_len =
     strlen(TEST_DESCRIPTOR) - annotation_len;
   mock_routerinfo->cache_info.annotations_len = annotation_len;
+  mock_routerinfo->cache_info.published_on = time(NULL);
 
   conn = new_dir_conn();
 
@@ -986,14 +980,14 @@ test_dir_handle_get_server_descriptors_fp(void* data)
   tt_int_op(body_used, OP_EQ, strlen(body));
 
   tt_str_op(body, OP_EQ, TEST_DESCRIPTOR + annotation_len);
-  tt_int_op(conn->dir_spool_src, OP_EQ, DIR_SPOOL_NONE);
+  tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
     NS_UNMOCK(router_get_my_routerinfo);
     UNMOCK(connection_write_to_buf_impl_);
     tor_free(mock_routerinfo->cache_info.signed_descriptor_body);
     tor_free(mock_routerinfo);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
     crypto_pk_free(identity_pkey);
@@ -1052,12 +1046,12 @@ test_dir_handle_get_server_descriptors_d(void* data)
 
   tt_str_op(body, OP_EQ, router->cache_info.signed_descriptor_body +
                          router->cache_info.annotations_len);
-  tt_int_op(conn->dir_spool_src, OP_EQ, DIR_SPOOL_NONE);
+  tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
     tor_free(mock_routerinfo);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
     crypto_pk_free(identity_pkey);
@@ -1107,13 +1101,13 @@ test_dir_handle_get_server_descriptors_busy(void* data)
   tt_assert(header);
   tt_str_op(SERVER_BUSY, OP_EQ, header);
 
-  tt_int_op(conn->dir_spool_src, OP_EQ, DIR_SPOOL_NONE);
+  tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
     UNMOCK(get_options);
     UNMOCK(connection_write_to_buf_impl_);
     tor_free(mock_routerinfo);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     crypto_pk_free(identity_pkey);
 
@@ -1144,7 +1138,7 @@ test_dir_handle_get_server_keys_bad_req(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -1170,7 +1164,7 @@ test_dir_handle_get_server_keys_all_not_found(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -1229,7 +1223,7 @@ test_dir_handle_get_server_keys_all(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
 
@@ -1259,7 +1253,7 @@ test_dir_handle_get_server_keys_authority_not_found(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -1307,7 +1301,7 @@ test_dir_handle_get_server_keys_authority(void* data)
   done:
     UNMOCK(get_my_v3_authority_cert);
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
     authority_cert_free(mock_cert); mock_cert = NULL;
@@ -1335,7 +1329,7 @@ test_dir_handle_get_server_keys_fp_not_found(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -1389,7 +1383,7 @@ test_dir_handle_get_server_keys_fp(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
     clear_dir_servers();
@@ -1418,7 +1412,7 @@ test_dir_handle_get_server_keys_sk_not_found(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -1463,7 +1457,7 @@ test_dir_handle_get_server_keys_sk(void* data)
   done:
     UNMOCK(get_my_v3_authority_cert);
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     authority_cert_free(mock_cert); mock_cert = NULL;
     tor_free(header);
     tor_free(body);
@@ -1491,7 +1485,7 @@ test_dir_handle_get_server_keys_fpsk_not_found(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -1548,7 +1542,7 @@ test_dir_handle_get_server_keys_fpsk(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
 
@@ -1602,7 +1596,7 @@ test_dir_handle_get_server_keys_busy(void* data)
   done:
     UNMOCK(get_options);
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     or_options_free(mock_options); mock_options = NULL;
 
@@ -1629,7 +1623,13 @@ test_dir_handle_get_status_vote_current_consensus_ns_not_enough_sigs(void* d)
   /* init mock */
   mock_ns_val = tor_malloc_zero(sizeof(networkstatus_t));
   mock_ns_val->flavor = FLAV_NS;
+  mock_ns_val->type = NS_TYPE_CONSENSUS;
   mock_ns_val->voters = smartlist_new();
+  mock_ns_val->valid_after = time(NULL) - 1800;
+  mock_ns_val->valid_until = time(NULL) - 60;
+
+  #define NETWORK_STATUS "some network status string"
+  consdiffmgr_add_consensus(NETWORK_STATUS, mock_ns_val);
 
   /* init mock */
   init_mock_options();
@@ -1662,7 +1662,7 @@ test_dir_handle_get_status_vote_current_consensus_ns_not_enough_sigs(void* d)
     UNMOCK(connection_write_to_buf_impl_);
     UNMOCK(get_options);
 
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(stats);
     smartlist_free(mock_ns_val->voters);
@@ -1703,12 +1703,86 @@ test_dir_handle_get_status_vote_current_consensus_ns_not_found(void* data)
   done:
     UNMOCK(connection_write_to_buf_impl_);
     UNMOCK(get_options);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(stats);
     or_options_free(mock_options); mock_options = NULL;
 }
 
+static void
+test_dir_handle_get_status_vote_current_consensus_too_old(void *data)
+{
+  dir_connection_t *conn = NULL;
+  char *header = NULL;
+  (void)data;
+
+  mock_ns_val = tor_malloc_zero(sizeof(networkstatus_t));
+  mock_ns_val->type = NS_TYPE_CONSENSUS;
+  mock_ns_val->flavor = FLAV_MICRODESC;
+  mock_ns_val->valid_after = time(NULL) - (24 * 60 * 60 + 1800);
+  mock_ns_val->fresh_until = time(NULL) - (24 * 60 * 60 + 900);
+  mock_ns_val->valid_until = time(NULL) - (24 * 60 * 60 + 20);
+
+  #define NETWORK_STATUS "some network status string"
+  consdiffmgr_add_consensus(NETWORK_STATUS, mock_ns_val);
+
+  init_mock_options();
+
+  MOCK(get_options, mock_get_options);
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+  MOCK(networkstatus_get_latest_consensus_by_flavor, mock_ns_get_by_flavor);
+
+  conn = new_dir_conn();
+
+  setup_capture_of_logs(LOG_WARN);
+
+  tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
+    GET("/tor/status-vote/current/consensus-microdesc"), NULL, 0));
+
+  fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
+                      NULL, NULL, 1, 0);
+  tt_assert(header);
+  tt_str_op(TOO_OLD, OP_EQ, header);
+
+  expect_log_msg_containing("too old");
+
+  tor_free(header);
+  teardown_capture_of_logs();
+  tor_free(mock_ns_val);
+
+  mock_ns_val = tor_malloc_zero(sizeof(networkstatus_t));
+  mock_ns_val->type = NS_TYPE_CONSENSUS;
+  mock_ns_val->flavor = FLAV_NS;
+  mock_ns_val->valid_after = time(NULL) - (24 * 60 * 60 + 1800);
+  mock_ns_val->fresh_until = time(NULL) - (24 * 60 * 60 + 900);
+  mock_ns_val->valid_until = time(NULL) - (24 * 60 * 60 + 20);
+
+  #define NETWORK_STATUS "some network status string"
+  consdiffmgr_add_consensus(NETWORK_STATUS, mock_ns_val);
+
+  setup_capture_of_logs(LOG_WARN);
+
+  tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
+    GET("/tor/status-vote/current/consensus"), NULL, 0));
+
+  fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
+                      NULL, NULL, 1, 0);
+  tt_assert(header);
+  tt_str_op(TOO_OLD, OP_EQ, header);
+
+  expect_no_log_entry();
+
+  done:
+    teardown_capture_of_logs();
+    UNMOCK(networkstatus_get_latest_consensus_by_flavor);
+    UNMOCK(connection_write_to_buf_impl_);
+    UNMOCK(get_options);
+    connection_free_minimal(TO_CONN(conn));
+    tor_free(header);
+    tor_free(mock_ns_val);
+    or_options_free(mock_options); mock_options = NULL;
+}
+
 NS_DECL(int, geoip_get_country_by_addr, (const tor_addr_t *addr));
 
 int
@@ -1723,12 +1797,26 @@ static void
 status_vote_current_consensus_ns_test(char **header, char **body,
                                       size_t *body_len)
 {
-  common_digests_t digests;
   dir_connection_t *conn = NULL;
 
   #define NETWORK_STATUS "some network status string"
+#if 0
+  common_digests_t digests;
+  uint8_t sha3[DIGEST256_LEN];
+  memset(&digests, 0x60, sizeof(digests));
+  memset(sha3, 0x06, sizeof(sha3));
   dirserv_set_cached_consensus_networkstatus(NETWORK_STATUS, "ns", &digests,
+                                             sha3,
                                              time(NULL));
+#endif /* 0 */
+  networkstatus_t *ns = tor_malloc_zero(sizeof(networkstatus_t));
+  ns->type = NS_TYPE_CONSENSUS;
+  ns->flavor = FLAV_NS;
+  ns->valid_after = time(NULL) - 1800;
+  ns->fresh_until = time(NULL) - 900;
+  ns->valid_until = time(NULL) - 60;
+  consdiffmgr_add_consensus(NETWORK_STATUS, ns);
+  networkstatus_vote_free(ns);
 
   MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
 
@@ -1741,7 +1829,6 @@ status_vote_current_consensus_ns_test(char **header, char **body,
   tt_str_op("ab", OP_EQ, geoip_get_country_name(1));
 
   conn = new_dir_conn();
-  TO_CONN(conn)->address = tor_strdup("127.0.0.1");
 
   tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
     GET("/tor/status-vote/current/consensus-ns"), NULL, 0));
@@ -1751,7 +1838,7 @@ status_vote_current_consensus_ns_test(char **header, char **body,
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
 }
 
 static void
@@ -1783,8 +1870,8 @@ test_dir_handle_get_status_vote_current_consensus_ns(void* data)
                                                             comp_body_used);
   tt_int_op(ZLIB_METHOD, OP_EQ, compression);
 
-  tor_gzip_uncompress(&body, &body_used, comp_body, comp_body_used,
-                      compression, 0, LOG_PROTOCOL_WARN);
+  tor_uncompress(&body, &body_used, comp_body, comp_body_used,
+                 compression, 0, LOG_PROTOCOL_WARN);
 
   tt_str_op(NETWORK_STATUS, OP_EQ, body);
   tt_int_op(strlen(NETWORK_STATUS), OP_EQ, body_used);
@@ -1874,7 +1961,7 @@ test_dir_handle_get_status_vote_current_not_found(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -1897,7 +1984,7 @@ status_vote_current_d_test(char **header, char **body, size_t *body_l)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
 }
 
 static void
@@ -1917,7 +2004,7 @@ status_vote_next_d_test(char **header, char **body, size_t *body_l)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
 }
 
 static void
@@ -1982,7 +2069,7 @@ test_dir_handle_get_status_vote_d(void* data)
   mock_options->TestingV3AuthInitialDistDelay = 1;
 
   time_t now = 1441223455 -1;
-  dirvote_recalculate_timing(mock_options, now);
+  voting_schedule_recalculate_timing(mock_options, now);
 
   const char *msg_out = NULL;
   int status_out = 0;
@@ -2020,6 +2107,7 @@ test_dir_handle_get_status_vote_d(void* data)
 
     clear_dir_servers();
     dirvote_free_all();
+    routerlist_free_all();
 }
 
 static void
@@ -2042,7 +2130,7 @@ test_dir_handle_get_status_vote_next_not_found(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -2061,7 +2149,7 @@ status_vote_next_consensus_test(char **header, char **body, size_t *body_used)
                       body, body_used, 18, 0);
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
 }
 
 static void
@@ -2101,7 +2189,7 @@ test_dir_handle_get_status_vote_current_authority_not_found(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -2125,7 +2213,7 @@ test_dir_handle_get_status_vote_next_authority_not_found(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -2207,7 +2295,7 @@ status_vote_next_consensus_signatures_test(char **header, char **body,
                       body, body_used, 22, 0);
 
   done:
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     UNMOCK(connection_write_to_buf_impl_);
 }
 
@@ -2328,7 +2416,7 @@ test_dir_handle_get_status_vote_next_authority(void* data)
   mock_options->TestingV3AuthInitialDistDelay = 1;
 
   time_t now = 1441223455 -1;
-  dirvote_recalculate_timing(mock_options, now);
+  voting_schedule_recalculate_timing(mock_options, now);
 
   struct pending_vote_t *vote = dirvote_add_vote(VOTE_BODY_V3, &msg_out,
                                                  &status_out);
@@ -2355,7 +2443,7 @@ test_dir_handle_get_status_vote_next_authority(void* data)
   done:
     UNMOCK(connection_write_to_buf_impl_);
     UNMOCK(get_my_v3_authority_cert);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
     authority_cert_free(mock_cert); mock_cert = NULL;
@@ -2407,7 +2495,7 @@ test_dir_handle_get_status_vote_current_authority(void* data)
   mock_options->TestingV3AuthInitialDistDelay = 1;
 
   time_t now = 1441223455;
-  dirvote_recalculate_timing(mock_options, now-1);
+  voting_schedule_recalculate_timing(mock_options, now-1);
 
   struct pending_vote_t *vote = dirvote_add_vote(VOTE_BODY_V3, &msg_out,
                                                  &status_out);
@@ -2437,7 +2525,7 @@ test_dir_handle_get_status_vote_current_authority(void* data)
   done:
     UNMOCK(connection_write_to_buf_impl_);
     UNMOCK(get_my_v3_authority_cert);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
     authority_cert_free(mock_cert); mock_cert = NULL;
@@ -2448,6 +2536,53 @@ test_dir_handle_get_status_vote_current_authority(void* data)
     dirvote_free_all();
 }
 
+static void
+test_dir_handle_get_parse_accept_encoding(void *arg)
+{
+  (void)arg;
+  const unsigned B_NONE = 1u << NO_METHOD;
+  const unsigned B_ZLIB = 1u << ZLIB_METHOD;
+  const unsigned B_GZIP = 1u << GZIP_METHOD;
+  const unsigned B_LZMA = 1u << LZMA_METHOD;
+  const unsigned B_ZSTD = 1u << ZSTD_METHOD;
+
+  unsigned encodings;
+
+  encodings = parse_accept_encoding_header("");
+  tt_uint_op(B_NONE, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header("  ");
+  tt_uint_op(B_NONE, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header("dewey, cheatham, and howe ");
+  tt_uint_op(B_NONE, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header("dewey, cheatham, and gzip");
+  tt_uint_op(B_NONE, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header("dewey, cheatham, and, gzip");
+  tt_uint_op(B_NONE|B_GZIP, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header(" gzip");
+  tt_uint_op(B_NONE|B_GZIP, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header("gzip");
+  tt_uint_op(B_NONE|B_GZIP, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header("x-zstd, deflate, x-tor-lzma");
+  tt_uint_op(B_NONE|B_ZLIB|B_ZSTD|B_LZMA, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header(
+                                        "x-zstd, deflate, x-tor-lzma, gzip");
+  tt_uint_op(B_NONE|B_ZLIB|B_ZSTD|B_LZMA|B_GZIP, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header("x-zstd,deflate,x-tor-lzma,gzip");
+  tt_uint_op(B_NONE|B_ZLIB|B_ZSTD|B_LZMA|B_GZIP, OP_EQ, encodings);
+
+ done:
+  ;
+}
+
 #define DIR_HANDLE_CMD(name,flags) \
   { #name, test_dir_handle_get_##name, (flags), NULL, NULL }
 
@@ -2492,10 +2627,11 @@ struct testcase_t dir_handle_get_tests[] = {
   DIR_HANDLE_CMD(status_vote_current_authority, 0),
   DIR_HANDLE_CMD(status_vote_next_authority_not_found, 0),
   DIR_HANDLE_CMD(status_vote_next_authority, 0),
-  DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_enough_sigs, 0),
-  DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_found, 0),
-  DIR_HANDLE_CMD(status_vote_current_consensus_ns_busy, 0),
-  DIR_HANDLE_CMD(status_vote_current_consensus_ns, 0),
+  DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_enough_sigs, TT_FORK),
+  DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_found, TT_FORK),
+  DIR_HANDLE_CMD(status_vote_current_consensus_too_old, TT_FORK),
+  DIR_HANDLE_CMD(status_vote_current_consensus_ns_busy, TT_FORK),
+  DIR_HANDLE_CMD(status_vote_current_consensus_ns, TT_FORK),
   DIR_HANDLE_CMD(status_vote_current_d_not_found, 0),
   DIR_HANDLE_CMD(status_vote_next_d_not_found, 0),
   DIR_HANDLE_CMD(status_vote_d, 0),
@@ -2505,6 +2641,6 @@ struct testcase_t dir_handle_get_tests[] = {
   DIR_HANDLE_CMD(status_vote_next_consensus_signatures_not_found, 0),
   DIR_HANDLE_CMD(status_vote_next_consensus_signatures_busy, 0),
   DIR_HANDLE_CMD(status_vote_next_consensus_signatures, 0),
+  DIR_HANDLE_CMD(parse_accept_encoding, 0),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_dns.c b/src/test/test_dns.c
index 6a8e92cb47..41a56f65d8 100644
--- a/src/test/test_dns.c
+++ b/src/test/test_dns.c
@@ -1,11 +1,18 @@
-#include "or.h"
-#include "test.h"
+/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "test/test.h"
 
 #define DNS_PRIVATE
 
-#include "dns.h"
-#include "connection.h"
-#include "router.h"
+#include "feature/relay/dns.h"
+#include "core/mainloop/connection.h"
+#include "core/or/connection_edge.h"
+#include "feature/relay/router.h"
+
+#include "core/or/edge_connection_st.h"
+#include "core/or/or_circuit_st.h"
 
 #define NS_MODULE dns
 
@@ -18,9 +25,9 @@ NS(test_main)(void *arg)
 
   uint32_t ttl_mid = MIN_DNS_TTL_AT_EXIT / 2 + MAX_DNS_TTL_AT_EXIT / 2;
 
-  tt_int_op(dns_clip_ttl(MIN_DNS_TTL_AT_EXIT - 1),==,MIN_DNS_TTL_AT_EXIT);
-  tt_int_op(dns_clip_ttl(ttl_mid),==,MAX_DNS_TTL_AT_EXIT);
-  tt_int_op(dns_clip_ttl(MAX_DNS_TTL_AT_EXIT + 1),==,MAX_DNS_TTL_AT_EXIT);
+  tt_int_op(dns_clip_ttl(MIN_DNS_TTL_AT_EXIT - 1),OP_EQ,MIN_DNS_TTL_AT_EXIT);
+  tt_int_op(dns_clip_ttl(ttl_mid),OP_EQ,MAX_DNS_TTL_AT_EXIT);
+  tt_int_op(dns_clip_ttl(MAX_DNS_TTL_AT_EXIT + 1),OP_EQ,MAX_DNS_TTL_AT_EXIT);
 
   done:
   return;
@@ -121,7 +128,7 @@ static int n_connection_free = 0;
 static connection_t *last_freed_conn = NULL;
 
 static void
-NS(connection_free)(connection_t *conn)
+NS(connection_free_)(connection_t *conn)
 {
    n_connection_free++;
 
@@ -172,10 +179,10 @@ NS(test_main)(void *arg)
 
   retval = dns_resolve(exitconn);
 
-  tt_int_op(retval,==,1);
-  tt_str_op(resolved_name,==,last_resolved_hostname);
+  tt_int_op(retval,OP_EQ,1);
+  tt_str_op(resolved_name,OP_EQ,last_resolved_hostname);
   tt_assert(conn_for_resolved_cell == exitconn);
-  tt_int_op(n_send_resolved_hostname_cell_replacement,==,
+  tt_int_op(n_send_resolved_hostname_cell_replacement,OP_EQ,
             prev_n_send_resolved_hostname_cell_replacement + 1);
   tt_assert(exitconn->on_circuit == NULL);
 
@@ -201,12 +208,12 @@ NS(test_main)(void *arg)
 
   retval = dns_resolve(exitconn);
 
-  tt_int_op(retval,==,1);
+  tt_int_op(retval,OP_EQ,1);
   tt_assert(conn_for_resolved_cell == exitconn);
-  tt_int_op(n_send_resolved_cell_replacement,==,
+  tt_int_op(n_send_resolved_cell_replacement,OP_EQ,
             prev_n_send_resolved_cell_replacement + 1);
   tt_assert(last_resolved == fake_resolved);
-  tt_int_op(last_answer_type,==,0xff);
+  tt_int_op(last_answer_type,OP_EQ,0xff);
   tt_assert(exitconn->on_circuit == NULL);
 
   /* CASE 3: The purpose of exit connection is not EXIT_PURPOSE_RESOLVE
@@ -229,12 +236,12 @@ NS(test_main)(void *arg)
 
   retval = dns_resolve(exitconn);
 
-  tt_int_op(retval,==,1);
+  tt_int_op(retval,OP_EQ,1);
   tt_assert(on_circuit->n_streams == exitconn);
   tt_assert(exitconn->next_stream == nextconn);
-  tt_int_op(prev_n_send_resolved_cell_replacement,==,
+  tt_int_op(prev_n_send_resolved_cell_replacement,OP_EQ,
             n_send_resolved_cell_replacement);
-  tt_int_op(prev_n_send_resolved_hostname_cell_replacement,==,
+  tt_int_op(prev_n_send_resolved_hostname_cell_replacement,OP_EQ,
             n_send_resolved_hostname_cell_replacement);
 
   /* CASE 4: _impl returns 0.
@@ -253,8 +260,8 @@ NS(test_main)(void *arg)
 
   retval = dns_resolve(exitconn);
 
-  tt_int_op(retval,==,0);
-  tt_int_op(exitconn->base_.state,==,EXIT_CONN_STATE_RESOLVING);
+  tt_int_op(retval,OP_EQ,0);
+  tt_int_op(exitconn->base_.state,OP_EQ,EXIT_CONN_STATE_RESOLVING);
   tt_assert(on_circuit->resolving_streams == exitconn);
   tt_assert(exitconn->next_stream == nextconn);
 
@@ -264,7 +271,7 @@ NS(test_main)(void *arg)
    */
 
   NS_MOCK(dns_cancel_pending_resolve);
-  NS_MOCK(connection_free);
+  NS_MOCK(connection_free_);
 
   exitconn->on_circuit = &(on_circuit->base_);
   exitconn->base_.purpose = EXIT_PURPOSE_RESOLVE;
@@ -278,12 +285,12 @@ NS(test_main)(void *arg)
 
   retval = dns_resolve(exitconn);
 
-  tt_int_op(retval,==,-1);
-  tt_int_op(n_send_resolved_cell_replacement,==,
+  tt_int_op(retval,OP_EQ,-1);
+  tt_int_op(n_send_resolved_cell_replacement,OP_EQ,
             prev_n_send_resolved_cell_replacement + 1);
-  tt_int_op(last_answer_type,==,RESOLVED_TYPE_ERROR);
-  tt_int_op(n_dns_cancel_pending_resolve_replacement,==,1);
-  tt_int_op(n_connection_free,==,prev_n_connection_free + 1);
+  tt_int_op(last_answer_type,OP_EQ,RESOLVED_TYPE_ERROR);
+  tt_int_op(n_dns_cancel_pending_resolve_replacement,OP_EQ,1);
+  tt_int_op(n_connection_free,OP_EQ,prev_n_connection_free + 1);
   tt_assert(last_freed_conn == TO_CONN(exitconn));
 
   done:
@@ -291,7 +298,7 @@ NS(test_main)(void *arg)
   NS_UNMOCK(send_resolved_cell);
   NS_UNMOCK(send_resolved_hostname_cell);
   NS_UNMOCK(dns_cancel_pending_resolve);
-  NS_UNMOCK(connection_free);
+  NS_UNMOCK(connection_free_);
   tor_free(on_circuit);
   tor_free(exitconn);
   tor_free(nextconn);
@@ -351,9 +358,9 @@ NS(test_main)(void *arg)
 
   resolved_addr = &(exitconn->base_.addr);
 
-  tt_int_op(retval,==,1);
+  tt_int_op(retval,OP_EQ,1);
   tt_assert(tor_addr_eq(resolved_addr, (const tor_addr_t *)&addr_to_compare));
-  tt_int_op(exitconn->address_ttl,==,DEFAULT_DNS_TTL);
+  tt_int_op(exitconn->address_ttl,OP_EQ,DEFAULT_DNS_TTL);
 
   done:
   tor_free(on_circ);
@@ -393,7 +400,7 @@ NS(test_main)(void *arg)
   retval = dns_resolve_impl(exitconn, 1, on_circ, NULL, &made_pending,
                             NULL);
 
-  tt_int_op(retval,==,-1);
+  tt_int_op(retval,OP_EQ,-1);
 
   done:
   tor_free(TO_CONN(exitconn)->address);
@@ -436,7 +443,7 @@ NS(test_main)(void *arg)
   retval = dns_resolve_impl(exitconn, 1, on_circ, NULL, &made_pending,
                             NULL);
 
-  tt_int_op(retval,==,-1);
+  tt_int_op(retval,OP_EQ,-1);
 
   done:
   NS_UNMOCK(router_my_exit_policy_is_reject_star);
@@ -478,7 +485,7 @@ NS(test_main)(void *arg)
   retval = dns_resolve_impl(exitconn, 1, on_circ, NULL, &made_pending,
                             NULL);
 
-  tt_int_op(retval,==,-1);
+  tt_int_op(retval,OP_EQ,-1);
 
   tor_free(TO_CONN(exitconn)->address);
 
@@ -488,7 +495,7 @@ NS(test_main)(void *arg)
   retval = dns_resolve_impl(exitconn, 1, on_circ, NULL, &made_pending,
                             NULL);
 
-  tt_int_op(retval,==,-1);
+  tt_int_op(retval,OP_EQ,-1);
 
   done:
   NS_UNMOCK(router_my_exit_policy_is_reject_star);
@@ -546,8 +553,8 @@ NS(test_main)(void *arg)
   retval = dns_resolve_impl(exitconn, 1, on_circ, NULL, &made_pending,
                             NULL);
 
-  tt_int_op(retval,==,0);
-  tt_int_op(made_pending,==,1);
+  tt_int_op(retval,OP_EQ,0);
+  tt_int_op(made_pending,OP_EQ,1);
 
   pending_conn = cache_entry->pending_connections;
 
@@ -628,8 +635,8 @@ NS(test_main)(void *arg)
   retval = dns_resolve_impl(exitconn, 1, on_circ, NULL, &made_pending,
                             &resolve_out);
 
-  tt_int_op(retval,==,0);
-  tt_int_op(made_pending,==,0);
+  tt_int_op(retval,OP_EQ,0);
+  tt_int_op(made_pending,OP_EQ,0);
   tt_assert(resolve_out == cache_entry);
 
   tt_assert(last_exitconn == exitconn);
@@ -699,8 +706,8 @@ NS(test_main)(void *arg)
   retval = dns_resolve_impl(exitconn, 1, on_circ, NULL, &made_pending,
                             NULL);
 
-  tt_int_op(retval,==,0);
-  tt_int_op(made_pending,==,1);
+  tt_int_op(retval,OP_EQ,0);
+  tt_int_op(made_pending,OP_EQ,1);
 
   cache_entry = dns_get_cache_entry(&query);
 
@@ -712,7 +719,7 @@ NS(test_main)(void *arg)
   tt_assert(pending_conn->conn == exitconn);
 
   tt_assert(last_launched_resolve == cache_entry);
-  tt_str_op(cache_entry->address,==,TO_CONN(exitconn)->address);
+  tt_str_op(cache_entry->address,OP_EQ,TO_CONN(exitconn)->address);
 
   done:
   NS_UNMOCK(router_my_exit_policy_is_reject_star);
@@ -742,4 +749,3 @@ struct testcase_t dns_tests[] = {
 };
 
 #undef NS_MODULE
-
diff --git a/src/test/test_dos.c b/src/test/test_dos.c
index cb9d9e559c..4756c5014e 100644
--- a/src/test/test_dos.c
+++ b/src/test/test_dos.c
@@ -1,21 +1,27 @@
-/* Copyright (c) 2018, The Tor Project, Inc. */
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define DOS_PRIVATE
 #define TOR_CHANNEL_INTERNAL_
 #define CIRCUITLIST_PRIVATE
 
-#include "or.h"
-#include "dos.h"
-#include "circuitlist.h"
-#include "geoip.h"
-#include "channel.h"
-#include "microdesc.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "routerlist.h"
-#include "test.h"
-#include "log_test_helpers.h"
+#include "core/or/or.h"
+#include "core/or/dos.h"
+#include "core/or/circuitlist.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "feature/stats/geoip_stats.h"
+#include "core/or/channel.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+
+#include "feature/nodelist/networkstatus_st.h"
+#include "core/or/or_connection_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+#include "test/test.h"
+#include "test/log_test_helpers.h"
 
 static networkstatus_t *dummy_ns = NULL;
 static networkstatus_t *
@@ -494,4 +500,3 @@ struct testcase_t dos_tests[] = {
     NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_entryconn.c b/src/test/test_entryconn.c
index 9580a1fd3f..fc7c5d5800 100644
--- a/src/test/test_entryconn.c
+++ b/src/test/test_entryconn.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -6,14 +6,23 @@
 #define CONNECTION_PRIVATE
 #define CONNECTION_EDGE_PRIVATE
 
-#include "or.h"
-#include "test.h"
+#include "core/or/or.h"
+#include "test/test.h"
 
-#include "addressmap.h"
-#include "config.h"
-#include "confparse.h"
-#include "connection.h"
-#include "connection_edge.h"
+#include "feature/client/addressmap.h"
+#include "app/config/config.h"
+#include "app/config/confparse.h"
+#include "core/mainloop/connection.h"
+#include "core/or/connection_edge.h"
+#include "feature/nodelist/nodelist.h"
+
+#include "feature/hs/hs_cache.h"
+#include "feature/rend/rendcache.h"
+
+#include "core/or/entry_connection_st.h"
+#include "core/or/socks_request_st.h"
+
+#include "lib/encoding/confline.h"
 
 static void *
 entryconn_rewrite_setup(const struct testcase_t *tc)
@@ -30,7 +39,7 @@ entryconn_rewrite_teardown(const struct testcase_t *tc, void *arg)
   (void)tc;
   entry_connection_t *ec = arg;
   if (ec)
-    connection_free_(ENTRY_TO_CONN(ec));
+    connection_free_minimal(ENTRY_TO_CONN(ec));
   addressmap_free_all();
   return 1;
 }
@@ -72,7 +81,6 @@ test_entryconn_rewrite_bad_dotexit(void *arg)
   entry_connection_t *ec = arg;
   rewrite_result_t rr;
 
-  get_options_mutable()->AllowDotExit = 0;
   tt_assert(ec->socks_request);
   strlcpy(ec->socks_request->address, "www.TORproject.org.foo.exit",
           sizeof(ec->socks_request->address));
@@ -100,7 +108,7 @@ test_entryconn_rewrite_automap_ipv4(void *arg)
   ec3 = entry_connection_new(CONN_TYPE_AP, AF_INET);
 
   get_options_mutable()->AutomapHostsOnResolve = 1;
-  smartlist_add(get_options_mutable()->AutomapHostsSuffixes, tor_strdup("."));
+  smartlist_add_strdup(get_options_mutable()->AutomapHostsSuffixes, ".");
   parse_virtual_addr_network("127.202.0.0/16", AF_INET, 0, &msg);
 
   /* Automap this on resolve. */
@@ -153,8 +161,8 @@ test_entryconn_rewrite_automap_ipv4(void *arg)
             ec->socks_request->address);
 
  done:
-  connection_free_(ENTRY_TO_CONN(ec2));
-  connection_free_(ENTRY_TO_CONN(ec3));
+  connection_free_minimal(ENTRY_TO_CONN(ec2));
+  connection_free_minimal(ENTRY_TO_CONN(ec3));
 }
 
 /* Automap on resolve, connect to automapped address, resolve again and get
@@ -173,7 +181,7 @@ test_entryconn_rewrite_automap_ipv6(void *arg)
   ec3 = entry_connection_new(CONN_TYPE_AP, AF_INET6);
 
   get_options_mutable()->AutomapHostsOnResolve = 1;
-  smartlist_add(get_options_mutable()->AutomapHostsSuffixes, tor_strdup("."));
+  smartlist_add_strdup(get_options_mutable()->AutomapHostsSuffixes, ".");
   parse_virtual_addr_network("FE80::/32", AF_INET6, 0, &msg);
 
   /* Automap this on resolve. */
@@ -227,9 +235,9 @@ test_entryconn_rewrite_automap_ipv6(void *arg)
             ec->socks_request->address);
 
  done:
-  connection_free_(ENTRY_TO_CONN(ec));
-  connection_free_(ENTRY_TO_CONN(ec2));
-  connection_free_(ENTRY_TO_CONN(ec3));
+  connection_free_minimal(ENTRY_TO_CONN(ec));
+  connection_free_minimal(ENTRY_TO_CONN(ec2));
+  connection_free_minimal(ENTRY_TO_CONN(ec3));
 }
 
 #if 0
@@ -280,9 +288,9 @@ test_entryconn_rewrite_automap_reverse(void *arg)
   tt_int_op(rr.exit_source, OP_EQ, ADDRMAPSRC_NONE);
 
  done:
-  connection_free_(ENTRY_TO_CONN(ec2));
+  connection_free_minimal(ENTRY_TO_CONN(ec2));
 }
-#endif
+#endif /* 0 */
 
 /* Rewrite because of cached DNS entry. */
 static void
@@ -330,7 +338,7 @@ test_entryconn_rewrite_cached_dns_ipv4(void *arg)
   tt_str_op(ec2->socks_request->address, OP_EQ, "240.240.241.241");
 
  done:
-  connection_free_(ENTRY_TO_CONN(ec2));
+  connection_free_minimal(ENTRY_TO_CONN(ec2));
 }
 
 /* Rewrite because of cached DNS entry. */
@@ -382,8 +390,8 @@ test_entryconn_rewrite_cached_dns_ipv6(void *arg)
   tt_str_op(ec2->socks_request->address, OP_EQ, "[::f00f]");
 
  done:
-  connection_free_(ENTRY_TO_CONN(ec));
-  connection_free_(ENTRY_TO_CONN(ec2));
+  connection_free_minimal(ENTRY_TO_CONN(ec));
+  connection_free_minimal(ENTRY_TO_CONN(ec2));
 }
 
 /* Fail to connect to unmapped address in virtual range. */
@@ -423,7 +431,7 @@ test_entryconn_rewrite_unmapped_virtual(void *arg)
   tt_int_op(rr.exit_source, OP_EQ, ADDRMAPSRC_NONE);
 
  done:
-  connection_free_(ENTRY_TO_CONN(ec2));
+  connection_free_minimal(ENTRY_TO_CONN(ec2));
 }
 
 /* Rewrite because of mapaddress option */
@@ -476,7 +484,7 @@ test_entryconn_rewrite_reject_internal_reverse(void *arg)
   ;
 }
 
-/* Rewrite into .exit because of virtual address mapping */
+/* Rewrite into .exit because of virtual address mapping.  */
 static void
 test_entryconn_rewrite_automap_exit(void *arg)
 {
@@ -487,46 +495,24 @@ test_entryconn_rewrite_automap_exit(void *arg)
 
   ec2 = entry_connection_new(CONN_TYPE_AP, AF_INET);
 
-  get_options_mutable()->AutomapHostsOnResolve = 1;
-  get_options_mutable()->AllowDotExit = 1;
-  smartlist_add(get_options_mutable()->AutomapHostsSuffixes,
-                tor_strdup(".EXIT"));
+  smartlist_add_strdup(get_options_mutable()->AutomapHostsSuffixes,
+                ".EXIT");
   parse_virtual_addr_network("127.1.0.0/16", AF_INET, 0, &msg);
 
-  /* Automap this on resolve. */
+  /* Try to automap this on resolve. */
   strlcpy(ec->socks_request->address, "website.example.exit",
           sizeof(ec->socks_request->address));
   ec->socks_request->command = SOCKS_COMMAND_RESOLVE;
   connection_ap_handshake_rewrite(ec, &rr);
 
-  tt_int_op(rr.automap, OP_EQ, 1);
-  tt_int_op(rr.should_close, OP_EQ, 0);
-  tt_int_op(rr.end_reason, OP_EQ, 0);
-  tt_i64_op(rr.map_expires, OP_EQ, TIME_MAX);
-  tt_int_op(rr.exit_source, OP_EQ, ADDRMAPSRC_NONE);
-  tt_str_op(rr.orig_address, OP_EQ, "website.example.exit");
-  tt_str_op(ec->original_dest_address, OP_EQ, "website.example.exit");
-
-  tt_assert(!strcmpstart(ec->socks_request->address,"127.1."));
-
-  /* Connect to it and make sure we get the original address back. */
-  strlcpy(ec2->socks_request->address, ec->socks_request->address,
-          sizeof(ec2->socks_request->address));
-
-  ec2->socks_request->command = SOCKS_COMMAND_CONNECT;
-  connection_ap_handshake_rewrite(ec2, &rr);
-
+  /* Make sure it isn't allowed -- there is no longer an AllowDotExit
+   * option. */
   tt_int_op(rr.automap, OP_EQ, 0);
-  tt_int_op(rr.should_close, OP_EQ, 0);
-  tt_int_op(rr.end_reason, OP_EQ, 0);
-  tt_i64_op(rr.map_expires, OP_EQ, TIME_MAX);
-  tt_int_op(rr.exit_source, OP_EQ, ADDRMAPSRC_AUTOMAP);
-  tt_str_op(rr.orig_address, OP_EQ, ec->socks_request->address);
-  tt_str_op(ec2->original_dest_address, OP_EQ, ec->socks_request->address);
-  tt_str_op(ec2->socks_request->address, OP_EQ, "website.example.exit");
+  tt_int_op(rr.should_close, OP_EQ, 1);
+  tt_int_op(rr.end_reason, OP_EQ, END_STREAM_REASON_TORPROTOCOL);
 
  done:
-  connection_free_(ENTRY_TO_CONN(ec2));
+  connection_free_minimal(ENTRY_TO_CONN(ec2));
 }
 
 /* Rewrite into .exit because of mapaddress */
@@ -573,9 +559,8 @@ test_entryconn_rewrite_mapaddress_automap_onion(void *arg)
   ec4 = entry_connection_new(CONN_TYPE_AP, AF_INET);
 
   get_options_mutable()->AutomapHostsOnResolve = 1;
-  get_options_mutable()->AllowDotExit = 1;
-  smartlist_add(get_options_mutable()->AutomapHostsSuffixes,
-                tor_strdup(".onion"));
+  smartlist_add_strdup(get_options_mutable()->AutomapHostsSuffixes,
+                ".onion");
   parse_virtual_addr_network("192.168.0.0/16", AF_INET, 0, &msg);
   config_line_append(&get_options_mutable()->AddressMap,
                      "MapAddress", "foo.onion abcdefghijklmnop.onion");
@@ -638,9 +623,9 @@ test_entryconn_rewrite_mapaddress_automap_onion(void *arg)
   */
 
  done:
-  connection_free_(ENTRY_TO_CONN(ec2));
-  connection_free_(ENTRY_TO_CONN(ec3));
-  connection_free_(ENTRY_TO_CONN(ec4));
+  connection_free_minimal(ENTRY_TO_CONN(ec2));
+  connection_free_minimal(ENTRY_TO_CONN(ec3));
+  connection_free_minimal(ENTRY_TO_CONN(ec4));
 }
 
 static void
@@ -698,8 +683,8 @@ test_entryconn_rewrite_mapaddress_automap_onion_common(entry_connection_t *ec,
                            "abcdefghijklmnop.onion"));
 
  done:
-  connection_free_(ENTRY_TO_CONN(ec2));
-  connection_free_(ENTRY_TO_CONN(ec3));
+  connection_free_minimal(ENTRY_TO_CONN(ec2));
+  connection_free_minimal(ENTRY_TO_CONN(ec3));
 }
 
 /* This time is the same, but we start with a mapping from a non-onion
@@ -709,8 +694,8 @@ test_entryconn_rewrite_mapaddress_automap_onion2(void *arg)
 {
   char *msg = NULL;
   get_options_mutable()->AutomapHostsOnResolve = 1;
-  smartlist_add(get_options_mutable()->AutomapHostsSuffixes,
-                tor_strdup(".onion"));
+  smartlist_add_strdup(get_options_mutable()->AutomapHostsSuffixes,
+                ".onion");
   parse_virtual_addr_network("192.168.0.0/16", AF_INET, 0, &msg);
   config_line_append(&get_options_mutable()->AddressMap,
                      "MapAddress", "irc.example.com abcdefghijklmnop.onion");
@@ -736,13 +721,95 @@ test_entryconn_rewrite_mapaddress_automap_onion4(void *arg)
 {
   char *msg = NULL;
   get_options_mutable()->AutomapHostsOnResolve = 1;
-  smartlist_add(get_options_mutable()->AutomapHostsSuffixes,
-                tor_strdup(".onion"));
+  smartlist_add_strdup(get_options_mutable()->AutomapHostsSuffixes,
+                ".onion");
   parse_virtual_addr_network("192.168.0.0/16", AF_INET, 0, &msg);
 
   test_entryconn_rewrite_mapaddress_automap_onion_common(arg, 0, 1);
 }
 
+/** Test that rewrite functions can handle v2 addresses */
+static void
+test_entryconn_rewrite_onion_v2(void *arg)
+{
+  int retval;
+  entry_connection_t *conn = arg;
+
+  (void) arg;
+
+  rend_cache_init();
+
+  /* Make a SOCKS request */
+  conn->socks_request->command = SOCKS_COMMAND_CONNECT;
+  strlcpy(conn->socks_request->address,
+          "pqeed46efnwmfuid.onion",
+          sizeof(conn->socks_request->address));
+
+  /* Make an onion connection using the SOCKS request */
+  conn->entry_cfg.onion_traffic = 1;
+  ENTRY_TO_CONN(conn)->state = AP_CONN_STATE_SOCKS_WAIT;
+  tt_assert(!ENTRY_TO_EDGE_CONN(conn)->rend_data);
+
+  /* Handle SOCKS and rewrite! */
+  retval = connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check connection state after rewrite */
+  tt_int_op(ENTRY_TO_CONN(conn)->state, OP_EQ, AP_CONN_STATE_RENDDESC_WAIT);
+  /* check that the address got rewritten */
+  tt_str_op(conn->socks_request->address, OP_EQ,
+            "pqeed46efnwmfuid");
+  /* check that HS information got attached to the connection */
+  tt_assert(ENTRY_TO_EDGE_CONN(conn)->rend_data);
+  tt_assert(!ENTRY_TO_EDGE_CONN(conn)->hs_ident);
+
+ done:
+  rend_cache_free_all();
+  /* 'conn' is cleaned by handler */
+}
+
+/** Test that rewrite functions can handle v3 onion addresses */
+static void
+test_entryconn_rewrite_onion_v3(void *arg)
+{
+  int retval;
+  entry_connection_t *conn = arg;
+
+  (void) arg;
+
+  hs_cache_init();
+
+  /* Make a SOCKS request */
+  conn->socks_request->command = SOCKS_COMMAND_CONNECT;
+  strlcpy(conn->socks_request->address,
+          "git.25njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid.onion",
+          sizeof(conn->socks_request->address));
+
+  /* Make an onion connection using the SOCKS request */
+  conn->entry_cfg.onion_traffic = 1;
+  ENTRY_TO_CONN(conn)->state = AP_CONN_STATE_SOCKS_WAIT;
+  tt_assert(!ENTRY_TO_EDGE_CONN(conn)->rend_data);
+  tt_assert(!ENTRY_TO_EDGE_CONN(conn)->hs_ident);
+
+  /* Handle SOCKS and rewrite! */
+  retval = connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check connection state after rewrite. It should be in waiting for
+   * descriptor state. */
+  tt_int_op(ENTRY_TO_CONN(conn)->state, OP_EQ, AP_CONN_STATE_RENDDESC_WAIT);
+  /* check that the address got rewritten */
+  tt_str_op(conn->socks_request->address, OP_EQ,
+            "25njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid");
+  /* check that HS information got attached to the connection */
+  tt_assert(ENTRY_TO_EDGE_CONN(conn)->hs_ident);
+  tt_assert(!ENTRY_TO_EDGE_CONN(conn)->rend_data);
+
+ done:
+  hs_free_all();
+  /* 'conn' is cleaned by handler */
+}
+
 #define REWRITE(name)                           \
   { #name, test_entryconn_##name, TT_FORK, &test_rewrite_setup, NULL }
 
@@ -763,7 +830,8 @@ struct testcase_t entryconn_tests[] = {
   REWRITE(rewrite_mapaddress_automap_onion2),
   REWRITE(rewrite_mapaddress_automap_onion3),
   REWRITE(rewrite_mapaddress_automap_onion4),
+  REWRITE(rewrite_onion_v2),
+  REWRITE(rewrite_onion_v3),
 
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c
index b1c3accfab..a486b13ae1 100644
--- a/src/test/test_entrynodes.c
+++ b/src/test/test_entrynodes.c
@@ -1,26 +1,50 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 
+#define CIRCUITLIST_PRIVATE
+#define CIRCUITBUILD_PRIVATE
 #define STATEFILE_PRIVATE
 #define ENTRYNODES_PRIVATE
 #define ROUTERLIST_PRIVATE
-
-#include "or.h"
-#include "test.h"
-
-#include "config.h"
-#include "entrynodes.h"
-#include "nodelist.h"
-#include "policies.h"
-#include "routerlist.h"
-#include "routerparse.h"
-#include "routerset.h"
-#include "statefile.h"
-#include "util.h"
-
-#include "test_helpers.h"
+#define DIRCLIENT_PRIVATE
+
+#include "core/or/or.h"
+#include "test/test.h"
+
+#include "feature/client/bridges.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitbuild.h"
+#include "app/config/config.h"
+#include "app/config/confparse.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/client/entrynodes.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/networkstatus.h"
+#include "core/or/policies.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerset.h"
+#include "app/config/statefile.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/crypt_path_st.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/nodelist/microdesc_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "app/config/or_state_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
+
+#include "lib/container/bloomfilt.h"
+#include "lib/encoding/confline.h"
 
 /* TODO:
  * choose_random_entry() test with state set.
@@ -39,37 +63,167 @@ get_or_state_replacement(void)
   return dummy_state;
 }
 
+static networkstatus_t *dummy_consensus = NULL;
+
+static smartlist_t *big_fake_net_nodes = NULL;
+
+static smartlist_t *
+bfn_mock_nodelist_get_list(void)
+{
+  return big_fake_net_nodes;
+}
+
+static networkstatus_t *
+bfn_mock_networkstatus_get_reasonably_live_consensus(time_t now, int flavor)
+{
+  (void)now;
+  (void)flavor;
+  return dummy_consensus;
+}
+
+static const node_t *
+bfn_mock_node_get_by_id(const char *id)
+{
+  SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, n,
+                    if (fast_memeq(n->identity, id, 20))
+                      return n);
+
+  return NULL;
+}
+
+/* Helper function to free a test node. */
+static void
+test_node_free(node_t *n)
+{
+  tor_free(n->rs);
+  tor_free(n->md->onion_curve25519_pkey);
+  short_policy_free(n->md->exit_policy);
+  tor_free(n->md);
+  tor_free(n);
+}
+
 /* Unittest cleanup function: Cleanup the fake network. */
 static int
-fake_network_cleanup(const struct testcase_t *testcase, void *ptr)
+big_fake_network_cleanup(const struct testcase_t *testcase, void *ptr)
 {
   (void) testcase;
   (void) ptr;
 
-  routerlist_free_all();
-  nodelist_free_all();
-  entry_guards_free_all();
+  if (big_fake_net_nodes) {
+    SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, n, {
+      test_node_free(n);
+    });
+    smartlist_free(big_fake_net_nodes);
+  }
+
+  UNMOCK(nodelist_get_list);
+  UNMOCK(node_get_by_id);
+  UNMOCK(get_or_state);
+  UNMOCK(networkstatus_get_reasonably_live_consensus);
   or_state_free(dummy_state);
+  dummy_state = NULL;
+  tor_free(dummy_consensus);
 
   return 1; /* NOP */
 }
 
 /* Unittest setup function: Setup a fake network. */
 static void *
-fake_network_setup(const struct testcase_t *testcase)
+big_fake_network_setup(const struct testcase_t *testcase)
 {
-  (void) testcase;
+  int i;
+
+  /* These are minimal node_t objects that only contain the aspects of node_t
+   * that we need for entrynodes.c. */
+  const int N_NODES = 271;
+
+  const char *argument = testcase->setup_data;
+  int reasonably_live_consensus = 0;
+  if (argument) {
+    reasonably_live_consensus = strstr(argument, "reasonably-live") != NULL;
+  }
+
+  big_fake_net_nodes = smartlist_new();
+  for (i = 0; i < N_NODES; ++i) {
+    curve25519_secret_key_t curve25519_secret_key;
+
+    node_t *n = tor_malloc_zero(sizeof(node_t));
+    n->md = tor_malloc_zero(sizeof(microdesc_t));
+
+    /* Generate curve25519 key for this node */
+    n->md->onion_curve25519_pkey =
+      tor_malloc_zero(sizeof(curve25519_public_key_t));
+    curve25519_secret_key_generate(&curve25519_secret_key, 0);
+    curve25519_public_key_generate(n->md->onion_curve25519_pkey,
+                                   &curve25519_secret_key);
+
+    crypto_rand(n->identity, sizeof(n->identity));
+    n->rs = tor_malloc_zero(sizeof(routerstatus_t));
+
+    memcpy(n->rs->identity_digest, n->identity, DIGEST_LEN);
+
+    n->is_running = n->is_valid = n->is_fast = n->is_stable = 1;
+
+    /* Note: all these guards have the same address, so you'll need to
+     * disable EnforceDistinctSubnets when a restriction is applied. */
+    n->rs->addr = 0x04020202;
+    n->rs->or_port = 1234;
+    n->rs->is_v2_dir = 1;
+    n->rs->has_bandwidth = 1;
+    n->rs->bandwidth_kb = 30;
+
+    /* Make a random nickname for each node */
+    {
+      char nickname_binary[8];
+      crypto_rand(nickname_binary, sizeof(nickname_binary));
+      base32_encode(n->rs->nickname, sizeof(n->rs->nickname),
+                    nickname_binary, sizeof(nickname_binary));
+    }
+
+    /* Call half of the nodes a possible guard. */
+    if (i % 2 == 0) {
+      n->is_possible_guard = 1;
+      n->rs->guardfraction_percentage = 100;
+      n->rs->has_guardfraction = 1;
+      n->rs->is_possible_guard = 1;
+    }
+
+    /* Make some of these nodes a possible exit */
+    if (i % 7 == 0) {
+      n->md->exit_policy = parse_short_policy("accept 443");
+    }
+
+    smartlist_add(big_fake_net_nodes, n);
+  }
 
-  /* Setup fake state */
   dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_consensus = tor_malloc_zero(sizeof(networkstatus_t));
+  if (reasonably_live_consensus) {
+    /* Make the dummy consensus valid from 4 hours ago, but expired an hour
+     * ago. */
+    dummy_consensus->valid_after = approx_time() - 4*3600;
+    dummy_consensus->valid_until = approx_time() - 3600;
+  } else {
+    /* Make the dummy consensus valid for an hour either side of now. */
+    dummy_consensus->valid_after = approx_time() - 3600;
+    dummy_consensus->valid_until = approx_time() + 3600;
+  }
+
+  MOCK(nodelist_get_list, bfn_mock_nodelist_get_list);
+  MOCK(node_get_by_id, bfn_mock_node_get_by_id);
   MOCK(get_or_state,
        get_or_state_replacement);
-
-  /* Setup fake routerlist. */
-  helper_setup_fake_routerlist();
-
+  MOCK(networkstatus_get_reasonably_live_consensus,
+       bfn_mock_networkstatus_get_reasonably_live_consensus);
   /* Return anything but NULL (it's interpreted as test fail) */
-  return dummy_state;
+  return (void*)testcase;
+}
+
+static time_t
+mock_randomize_time_no_randomization(time_t a, time_t b)
+{
+  (void) b;
+  return a;
 }
 
 static or_options_t mocked_options;
@@ -80,796 +234,2876 @@ mock_get_options(void)
   return &mocked_options;
 }
 
-/** Test choose_random_entry() with none of our routers being guard nodes. */
+#define TEST_IPV4_ADDR "123.45.67.89"
+#define TEST_IPV6_ADDR "[1234:5678:90ab:cdef::]"
+
 static void
-test_choose_random_entry_no_guards(void *arg)
+test_node_preferred_orport(void *arg)
 {
-  const node_t *chosen_entry = NULL;
-
-  (void) arg;
+  (void)arg;
+  tor_addr_t ipv4_addr;
+  const uint16_t ipv4_port = 4444;
+  tor_addr_t ipv6_addr;
+  const uint16_t ipv6_port = 6666;
+  routerinfo_t node_ri;
+  node_t node;
+  tor_addr_port_t ap;
 
+  /* Setup options */
+  memset(&mocked_options, 0, sizeof(mocked_options));
+  /* We don't test ClientPreferIPv6ORPort here, because it's used in
+   * nodelist_set_consensus to setup node.ipv6_preferred, which we set
+   * directly. */
   MOCK(get_options, mock_get_options);
 
-  /* Check that we get a guard if it passes preferred
-   * address settings */
-  memset(&mocked_options, 0, sizeof(mocked_options));
+  /* Setup IP addresses */
+  tor_addr_parse(&ipv4_addr, TEST_IPV4_ADDR);
+  tor_addr_parse(&ipv6_addr, TEST_IPV6_ADDR);
+
+  /* Setup node_ri */
+  memset(&node_ri, 0, sizeof(node_ri));
+  node_ri.addr = tor_addr_to_ipv4h(&ipv4_addr);
+  node_ri.or_port = ipv4_port;
+  tor_addr_copy(&node_ri.ipv6_addr, &ipv6_addr);
+  node_ri.ipv6_orport = ipv6_port;
+
+  /* Setup node */
+  memset(&node, 0, sizeof(node));
+  node.ri = &node_ri;
+
+  /* Check the preferred address is IPv4 if we're only using IPv4, regardless
+   * of whether we prefer it or not */
   mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientPreferIPv6ORPort = 0;
+  mocked_options.ClientUseIPv6 = 0;
+  node.ipv6_preferred = 0;
+  node_get_pref_orport(&node, &ap);
+  tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
+  tt_assert(ap.port == ipv4_port);
 
-  /* Try to pick an entry even though none of our routers are guards. */
-  chosen_entry = choose_random_entry(NULL);
+  node.ipv6_preferred = 1;
+  node_get_pref_orport(&node, &ap);
+  tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
+  tt_assert(ap.port == ipv4_port);
 
-  /* Unintuitively, we actually pick a random node as our entry,
-     because router_choose_random_node() relaxes its constraints if it
-     can't find a proper entry guard. */
-  tt_assert(chosen_entry);
+  /* Check the preferred address is IPv4 if we're using IPv4 and IPv6, but
+   * don't prefer the IPv6 address */
+  mocked_options.ClientUseIPv4 = 1;
+  mocked_options.ClientUseIPv6 = 1;
+  node.ipv6_preferred = 0;
+  node_get_pref_orport(&node, &ap);
+  tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
+  tt_assert(ap.port == ipv4_port);
 
-  /* And with the other IP version active */
+  /* Check the preferred address is IPv6 if we prefer it and
+   * ClientUseIPv6 is 1, regardless of ClientUseIPv4 */
+  mocked_options.ClientUseIPv4 = 1;
   mocked_options.ClientUseIPv6 = 1;
-  chosen_entry = choose_random_entry(NULL);
-  tt_assert(chosen_entry);
+  node.ipv6_preferred = 1;
+  node_get_pref_orport(&node, &ap);
+  tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
+  tt_assert(ap.port == ipv6_port);
 
-  /* And with the preference on auto */
-  mocked_options.ClientPreferIPv6ORPort = -1;
-  chosen_entry = choose_random_entry(NULL);
-  tt_assert(chosen_entry);
+  mocked_options.ClientUseIPv4 = 0;
+  node_get_pref_orport(&node, &ap);
+  tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
+  tt_assert(ap.port == ipv6_port);
 
-  /* Check that we don't get a guard if it doesn't pass mandatory address
-   * settings */
-  memset(&mocked_options, 0, sizeof(mocked_options));
+  /* Check the preferred address is IPv6 if we don't prefer it, but
+   * ClientUseIPv4 is 0 */
   mocked_options.ClientUseIPv4 = 0;
-  mocked_options.ClientPreferIPv6ORPort = 0;
+  mocked_options.ClientUseIPv6 = 1;
+  node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport(&mocked_options);
+  node_get_pref_orport(&node, &ap);
+  tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
+  tt_assert(ap.port == ipv6_port);
 
-  chosen_entry = choose_random_entry(NULL);
+ done:
+  UNMOCK(get_options);
+}
 
-  /* If we don't allow IPv4 at all, we don't get a guard*/
-  tt_assert(!chosen_entry);
+static void
+test_entry_guard_describe(void *arg)
+{
+  (void)arg;
+  entry_guard_t g;
+  memset(&g, 0, sizeof(g));
+  strlcpy(g.nickname, "okefenokee", sizeof(g.nickname));
+  memcpy(g.identity, "theforestprimeval---", DIGEST_LEN);
 
-  /* Check that we get a guard if it passes allowed but not preferred address
-   * settings */
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 1;
-  mocked_options.ClientPreferIPv6ORPort = 1;
+  tt_str_op(entry_guard_describe(&g), OP_EQ,
+            "okefenokee ($746865666F726573747072696D6576616C2D2D2D)");
 
-  chosen_entry = choose_random_entry(NULL);
-  tt_assert(chosen_entry);
+ done:
+  ;
+}
 
-  /* Check that we get a guard if it passes preferred address settings when
-   * they're auto */
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientPreferIPv6ORPort = -1;
+static void
+test_entry_guard_randomize_time(void *arg)
+{
+  const time_t now = 1479153573;
+  const int delay = 86400;
+  const int N = 1000;
+  (void)arg;
 
-  chosen_entry = choose_random_entry(NULL);
-  tt_assert(chosen_entry);
+  time_t t;
+  int i;
+  for (i = 0; i < N; ++i) {
+    t = randomize_time(now, delay);
+    tt_int_op(t, OP_LE, now);
+    tt_int_op(t, OP_GE, now-delay);
+  }
 
-  /* And with IPv6 active */
-  mocked_options.ClientUseIPv6 = 1;
+  /* now try the corner cases */
+  for (i = 0; i < N; ++i) {
+    t = randomize_time(100, delay);
+    tt_int_op(t, OP_GE, 1);
+    tt_int_op(t, OP_LE, 100);
 
-  chosen_entry = choose_random_entry(NULL);
-  tt_assert(chosen_entry);
+    t = randomize_time(0, delay);
+    tt_int_op(t, OP_EQ, 1);
+  }
 
  done:
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  UNMOCK(get_options);
+  ;
 }
 
-/** Test choose_random_entry() with only one of our routers being a
-    guard node. */
 static void
-test_choose_random_entry_one_possible_guard(void *arg)
+test_entry_guard_encode_for_state_minimal(void *arg)
 {
-  const node_t *chosen_entry = NULL;
-  node_t *the_guard = NULL;
-  smartlist_t *our_nodelist = NULL;
+  (void) arg;
+  entry_guard_t *eg = tor_malloc_zero(sizeof(entry_guard_t));
+
+  eg->selection_name = tor_strdup("wubwub");
+  memcpy(eg->identity, "plurpyflurpyslurpydo", DIGEST_LEN);
+  eg->sampled_on_date = 1479081600;
+  eg->confirmed_idx = -1;
+
+  char *s = NULL;
+  s = entry_guard_encode_for_state(eg);
 
+  tt_str_op(s, OP_EQ,
+            "in=wubwub "
+            "rsa_id=706C75727079666C75727079736C75727079646F "
+            "sampled_on=2016-11-14T00:00:00 "
+            "listed=0");
+
+ done:
+  entry_guard_free(eg);
+  tor_free(s);
+}
+
+static void
+test_entry_guard_encode_for_state_maximal(void *arg)
+{
   (void) arg;
+  entry_guard_t *eg = tor_malloc_zero(sizeof(entry_guard_t));
+
+  strlcpy(eg->nickname, "Fred", sizeof(eg->nickname));
+  eg->selection_name = tor_strdup("default");
+  memcpy(eg->identity, "plurpyflurpyslurpydo", DIGEST_LEN);
+  eg->bridge_addr = tor_malloc_zero(sizeof(tor_addr_port_t));
+  tor_addr_from_ipv4h(&eg->bridge_addr->addr, 0x08080404);
+  eg->bridge_addr->port = 9999;
+  eg->sampled_on_date = 1479081600;
+  eg->sampled_by_version = tor_strdup("1.2.3");
+  eg->unlisted_since_date = 1479081645;
+  eg->currently_listed = 1;
+  eg->confirmed_on_date = 1479081690;
+  eg->confirmed_idx = 333;
+  eg->extra_state_fields = tor_strdup("and the green grass grew all around");
+
+  char *s = NULL;
+  s = entry_guard_encode_for_state(eg);
+
+  tt_str_op(s, OP_EQ,
+            "in=default "
+            "rsa_id=706C75727079666C75727079736C75727079646F "
+            "bridge_addr=8.8.4.4:9999 "
+            "nickname=Fred "
+            "sampled_on=2016-11-14T00:00:00 "
+            "sampled_by=1.2.3 "
+            "unlisted_since=2016-11-14T00:00:45 "
+            "listed=1 "
+            "confirmed_on=2016-11-14T00:01:30 "
+            "confirmed_idx=333 "
+            "and the green grass grew all around");
 
-  MOCK(get_options, mock_get_options);
+ done:
+  entry_guard_free(eg);
+  tor_free(s);
+}
 
-  /* Set one of the nodes to be a guard. */
-  our_nodelist = nodelist_get_list();
-  the_guard = smartlist_get(our_nodelist, 4); /* chosen by fair dice roll */
-  the_guard->is_possible_guard = 1;
+static void
+test_entry_guard_parse_from_state_minimal(void *arg)
+{
+  (void)arg;
+  char *mem_op_hex_tmp = NULL;
+  entry_guard_t *eg = NULL;
+  time_t t = approx_time();
+
+  eg = entry_guard_parse_from_state(
+                 "in=default_plus "
+                 "rsa_id=596f75206d6179206e656564206120686f626279");
+  tt_assert(eg);
+
+  tt_str_op(eg->selection_name, OP_EQ, "default_plus");
+  test_mem_op_hex(eg->identity, OP_EQ,
+                  "596f75206d6179206e656564206120686f626279");
+  tt_str_op(eg->nickname, OP_EQ, "$596F75206D6179206E656564206120686F626279");
+  tt_ptr_op(eg->bridge_addr, OP_EQ, NULL);
+  tt_i64_op(eg->sampled_on_date, OP_GE, t);
+  tt_i64_op(eg->sampled_on_date, OP_LE, t+86400);
+  tt_i64_op(eg->unlisted_since_date, OP_EQ, 0);
+  tt_ptr_op(eg->sampled_by_version, OP_EQ, NULL);
+  tt_int_op(eg->currently_listed, OP_EQ, 0);
+  tt_i64_op(eg->confirmed_on_date, OP_EQ, 0);
+  tt_int_op(eg->confirmed_idx, OP_EQ, -1);
+
+  tt_int_op(eg->last_tried_to_connect, OP_EQ, 0);
+  tt_int_op(eg->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
 
-  /* Check that we get the guard if it passes preferred
-   * address settings */
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientPreferIPv6ORPort = 0;
+ done:
+  entry_guard_free(eg);
+  tor_free(mem_op_hex_tmp);
+}
 
-  /* Pick an entry. Make sure we pick the node we marked as guard. */
-  chosen_entry = choose_random_entry(NULL);
-  tt_ptr_op(chosen_entry, OP_EQ, the_guard);
+static void
+test_entry_guard_parse_from_state_maximal(void *arg)
+{
+  (void)arg;
+  char *mem_op_hex_tmp = NULL;
+  entry_guard_t *eg = NULL;
+
+  eg = entry_guard_parse_from_state(
+            "in=fred "
+            "rsa_id=706C75727079666C75727079736C75727079646F "
+            "bridge_addr=[1::3]:9999 "
+            "nickname=Fred "
+            "sampled_on=2016-11-14T00:00:00 "
+            "sampled_by=1.2.3 "
+            "unlisted_since=2016-11-14T00:00:45 "
+            "listed=1 "
+            "confirmed_on=2016-11-14T00:01:30 "
+            "confirmed_idx=333 "
+            "and the green grass grew all around "
+            "rsa_id=all,around");
+  tt_assert(eg);
+
+  test_mem_op_hex(eg->identity, OP_EQ,
+                  "706C75727079666C75727079736C75727079646F");
+  tt_str_op(fmt_addr(&eg->bridge_addr->addr), OP_EQ, "1::3");
+  tt_int_op(eg->bridge_addr->port, OP_EQ, 9999);
+  tt_str_op(eg->nickname, OP_EQ, "Fred");
+  tt_i64_op(eg->sampled_on_date, OP_EQ, 1479081600);
+  tt_i64_op(eg->unlisted_since_date, OP_EQ, 1479081645);
+  tt_str_op(eg->sampled_by_version, OP_EQ, "1.2.3");
+  tt_int_op(eg->currently_listed, OP_EQ, 1);
+  tt_i64_op(eg->confirmed_on_date, OP_EQ, 1479081690);
+  tt_int_op(eg->confirmed_idx, OP_EQ, 333);
+  tt_str_op(eg->extra_state_fields, OP_EQ,
+            "and the green grass grew all around rsa_id=all,around");
+
+  tt_int_op(eg->last_tried_to_connect, OP_EQ, 0);
+  tt_int_op(eg->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
 
-  /* And with the other IP version active */
-  mocked_options.ClientUseIPv6 = 1;
-  chosen_entry = choose_random_entry(NULL);
-  tt_ptr_op(chosen_entry, OP_EQ, the_guard);
+ done:
+  entry_guard_free(eg);
+  tor_free(mem_op_hex_tmp);
+}
+
+static void
+test_entry_guard_parse_from_state_failure(void *arg)
+{
+  (void)arg;
+  entry_guard_t *eg = NULL;
+
+  /* no selection */
+  eg = entry_guard_parse_from_state(
+                 "rsa_id=596f75206d6179206e656564206120686f626270");
+  tt_ptr_op(eg, OP_EQ, NULL);
+
+  /* no RSA ID. */
+  eg = entry_guard_parse_from_state("in=default nickname=Fred");
+  tt_ptr_op(eg, OP_EQ, NULL);
+
+  /* Bad RSA ID: bad character. */
+  eg = entry_guard_parse_from_state(
+                 "in=default "
+                 "rsa_id=596f75206d6179206e656564206120686f62627q");
+  tt_ptr_op(eg, OP_EQ, NULL);
+
+  /* Bad RSA ID: too long.*/
+  eg = entry_guard_parse_from_state(
+                 "in=default "
+                 "rsa_id=596f75206d6179206e656564206120686f6262703");
+  tt_ptr_op(eg, OP_EQ, NULL);
+
+  /* Bad RSA ID: too short.*/
+  eg = entry_guard_parse_from_state(
+                 "in=default "
+                 "rsa_id=596f75206d6179206e65656420612");
+  tt_ptr_op(eg, OP_EQ, NULL);
 
-  /* And with the preference on auto */
-  mocked_options.ClientPreferIPv6ORPort = -1;
-  chosen_entry = choose_random_entry(NULL);
-  tt_ptr_op(chosen_entry, OP_EQ, the_guard);
+ done:
+  entry_guard_free(eg);
+}
 
-  /* Check that we don't get a guard if it doesn't pass mandatory address
-   * settings */
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  mocked_options.ClientUseIPv4 = 0;
-  mocked_options.ClientPreferIPv6ORPort = 0;
+static void
+test_entry_guard_parse_from_state_partial_failure(void *arg)
+{
+  (void)arg;
+  char *mem_op_hex_tmp = NULL;
+  entry_guard_t *eg = NULL;
+  time_t t = approx_time();
+
+  eg = entry_guard_parse_from_state(
+            "in=default "
+            "rsa_id=706C75727079666C75727079736C75727079646F "
+            "bridge_addr=1.2.3.3.4:5 "
+            "nickname=FredIsANodeWithAStrangeNicknameThatIsTooLong "
+            "sampled_on=2016-11-14T00:00:99 "
+            "sampled_by=1.2.3 stuff in the middle "
+            "unlisted_since=2016-xx-14T00:00:45 "
+            "listed=0 "
+            "confirmed_on=2016-11-14T00:01:30zz "
+            "confirmed_idx=idx "
+            "and the green grass grew all around "
+            "rsa_id=all,around");
+  tt_assert(eg);
+
+  test_mem_op_hex(eg->identity, OP_EQ,
+                  "706C75727079666C75727079736C75727079646F");
+  tt_str_op(eg->nickname, OP_EQ, "FredIsANodeWithAStrangeNicknameThatIsTooL");
+  tt_ptr_op(eg->bridge_addr, OP_EQ, NULL);
+  tt_i64_op(eg->sampled_on_date, OP_EQ, t);
+  tt_i64_op(eg->unlisted_since_date, OP_EQ, 0);
+  tt_str_op(eg->sampled_by_version, OP_EQ, "1.2.3");
+  tt_int_op(eg->currently_listed, OP_EQ, 0);
+  tt_i64_op(eg->confirmed_on_date, OP_EQ, 0);
+  tt_int_op(eg->confirmed_idx, OP_EQ, -1);
+  tt_str_op(eg->extra_state_fields, OP_EQ,
+            "stuff in the middle and the green grass grew all around "
+            "rsa_id=all,around");
+
+  tt_int_op(eg->last_tried_to_connect, OP_EQ, 0);
+  tt_int_op(eg->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
 
-  chosen_entry = choose_random_entry(NULL);
+ done:
+  entry_guard_free(eg);
+  tor_free(mem_op_hex_tmp);
+}
 
-  /* If we don't allow IPv4 at all, we don't get a guard*/
-  tt_assert(!chosen_entry);
+static int
+mock_entry_guard_is_listed(guard_selection_t *gs, const entry_guard_t *guard)
+{
+  (void)gs;
+  (void)guard;
+  return 1;
+}
 
-  /* Check that we get a node if it passes allowed but not preferred
-   * address settings */
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 1;
-  mocked_options.ClientPreferIPv6ORPort = 1;
+static void
+test_entry_guard_parse_from_state_full(void *arg)
+{
+  (void)arg;
+  /* Here's a state I made while testing.  The identities and locations for
+   * the bridges are redacted. */
+  const char STATE[] =
+  "Guard in=default rsa_id=214F44BD5B638E8C817D47FF7C97397790BF0345 "
+    "nickname=TotallyNinja sampled_on=2016-11-12T19:32:49 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1\n"
+  "Guard in=default rsa_id=052900AB0EA3ED54BAB84AE8A99E74E8693CE2B2 "
+    "nickname=5OfNovember sampled_on=2016-11-20T04:32:05 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1 confirmed_on=2016-11-22T08:13:28 confirmed_idx=0 "
+    "pb_circ_attempts=4.000000 pb_circ_successes=2.000000 "
+    "pb_successful_circuits_closed=2.000000\n"
+  "Guard in=default rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
+    "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1 confirmed_on=2016-11-24T08:45:30 confirmed_idx=4 "
+    "pb_circ_attempts=5.000000 pb_circ_successes=5.000000 "
+    "pb_successful_circuits_closed=5.000000\n"
+  "Guard in=wobblesome rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
+    "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1\n"
+  "Guard in=default rsa_id=E9025AD60D86875D5F11548D536CC6AF60F0EF5E "
+    "nickname=maibrunn sampled_on=2016-11-25T22:36:38 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1\n"
+  "Guard in=default rsa_id=DCD30B90BA3A792DA75DC54A327EF353FB84C38E "
+    "nickname=Unnamed sampled_on=2016-11-25T14:34:00 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1\n"
+  "Guard in=bridges rsa_id=8FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF2E "
+    "bridge_addr=24.1.1.1:443 sampled_on=2016-11-25T06:44:14 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1 "
+    "confirmed_on=2016-11-29T10:36:06 confirmed_idx=0 "
+    "pb_circ_attempts=8.000000 pb_circ_successes=8.000000 "
+    "pb_successful_circuits_closed=13.000000\n"
+  "Guard in=bridges rsa_id=5800000000000000000000000000000000000000 "
+    "bridge_addr=37.218.246.143:28366 "
+    "sampled_on=2016-11-18T15:07:34 sampled_by=0.3.0.0-alpha-dev listed=1\n";
+
+  config_line_t *lines = NULL;
+  or_state_t *state = tor_malloc_zero(sizeof(or_state_t));
+  int r = config_get_lines(STATE, &lines, 0);
+  char *msg = NULL;
+  smartlist_t *text = smartlist_new();
+  char *joined = NULL;
 
-  chosen_entry = choose_random_entry(NULL);
+  // So nodes aren't expired. This is Tue, 13 Dec 2016 09:37:14 GMT
+  update_approx_time(1481621834);
 
-  /* We disable the guard check and the preferred address check at the same
-   * time, so we can't be sure we get the guard */
-  tt_assert(chosen_entry);
+  MOCK(entry_guard_is_listed, mock_entry_guard_is_listed);
 
-  /* Check that we get a node if it is allowed but not preferred when settings
-   * are auto */
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientPreferIPv6ORPort = -1;
+  dummy_state = state;
+  MOCK(get_or_state,
+       get_or_state_replacement);
 
-  chosen_entry = choose_random_entry(NULL);
+  tt_int_op(r, OP_EQ, 0);
+  tt_assert(lines);
+
+  state->Guard = lines;
+
+  /* Try it first without setting the result. */
+  r = entry_guards_parse_state(state, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  guard_selection_t *gs_br =
+    get_guard_selection_by_name("bridges", GS_TYPE_BRIDGE, 0);
+  tt_ptr_op(gs_br, OP_EQ, NULL);
+
+  r = entry_guards_parse_state(state, 1, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  gs_br = get_guard_selection_by_name("bridges", GS_TYPE_BRIDGE, 0);
+  guard_selection_t *gs_df =
+    get_guard_selection_by_name("default", GS_TYPE_NORMAL, 0);
+  guard_selection_t *gs_wb =
+    get_guard_selection_by_name("wobblesome", GS_TYPE_NORMAL, 0);
+
+  tt_assert(gs_br);
+  tt_assert(gs_df);
+  tt_assert(gs_wb);
+
+  tt_int_op(smartlist_len(gs_df->sampled_entry_guards), OP_EQ, 5);
+  tt_int_op(smartlist_len(gs_br->sampled_entry_guards), OP_EQ, 2);
+  tt_int_op(smartlist_len(gs_wb->sampled_entry_guards), OP_EQ, 1);
+
+  /* Try again; make sure it doesn't double-add the guards. */
+  r = entry_guards_parse_state(state, 1, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  gs_br = get_guard_selection_by_name("bridges", GS_TYPE_BRIDGE, 0);
+  gs_df = get_guard_selection_by_name("default", GS_TYPE_NORMAL, 0);
+  tt_assert(gs_br);
+  tt_assert(gs_df);
+  tt_int_op(smartlist_len(gs_df->sampled_entry_guards), OP_EQ, 5);
+  tt_int_op(smartlist_len(gs_br->sampled_entry_guards), OP_EQ, 2);
+
+  /* Re-encode; it should be the same... almost. */
+  {
+    /* (Make a guard nonpersistent first) */
+    entry_guard_t *g = smartlist_get(gs_df->sampled_entry_guards, 0);
+    g->is_persistent = 0;
+  }
+  config_free_lines(lines);
+  lines = state->Guard = NULL; // to prevent double-free.
+  entry_guards_update_state(state);
+  tt_assert(state->Guard);
+  lines = state->Guard;
+
+  config_line_t *ln;
+  for (ln = lines; ln; ln = ln->next) {
+    smartlist_add_asprintf(text, "%s %s\n",ln->key, ln->value);
+  }
+  joined = smartlist_join_strings(text, "", 0, NULL);
+  tt_str_op(joined, OP_EQ,
+  "Guard in=default rsa_id=052900AB0EA3ED54BAB84AE8A99E74E8693CE2B2 "
+    "nickname=5OfNovember sampled_on=2016-11-20T04:32:05 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1 confirmed_on=2016-11-22T08:13:28 confirmed_idx=0 "
+    "pb_circ_attempts=4.000000 pb_circ_successes=2.000000 "
+    "pb_successful_circuits_closed=2.000000\n"
+  "Guard in=default rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
+    "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1 confirmed_on=2016-11-24T08:45:30 confirmed_idx=1 "
+    "pb_circ_attempts=5.000000 pb_circ_successes=5.000000 "
+    "pb_successful_circuits_closed=5.000000\n"
+  "Guard in=default rsa_id=E9025AD60D86875D5F11548D536CC6AF60F0EF5E "
+    "nickname=maibrunn sampled_on=2016-11-25T22:36:38 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1\n"
+  "Guard in=default rsa_id=DCD30B90BA3A792DA75DC54A327EF353FB84C38E "
+    "nickname=Unnamed sampled_on=2016-11-25T14:34:00 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1\n"
+  "Guard in=wobblesome rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
+    "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1\n"
+  "Guard in=bridges rsa_id=8FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF2E "
+    "bridge_addr=24.1.1.1:443 sampled_on=2016-11-25T06:44:14 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1 "
+    "confirmed_on=2016-11-29T10:36:06 confirmed_idx=0 "
+    "pb_circ_attempts=8.000000 pb_circ_successes=8.000000 "
+    "pb_successful_circuits_closed=13.000000\n"
+  "Guard in=bridges rsa_id=5800000000000000000000000000000000000000 "
+    "bridge_addr=37.218.246.143:28366 "
+    "sampled_on=2016-11-18T15:07:34 sampled_by=0.3.0.0-alpha-dev listed=1\n");
 
-  /* We disable the guard check and the preferred address check at the same
-   * time, so we can't be sure we get the guard */
-  tt_assert(chosen_entry);
+ done:
+  config_free_lines(lines);
+  tor_free(state);
+  tor_free(msg);
+  UNMOCK(get_or_state);
+  UNMOCK(entry_guard_is_listed);
+  SMARTLIST_FOREACH(text, char *, cp, tor_free(cp));
+  smartlist_free(text);
+  tor_free(joined);
+}
 
-  /* and with IPv6 active */
-  mocked_options.ClientUseIPv6 = 1;
+static void
+test_entry_guard_parse_from_state_broken(void *arg)
+{
+  (void)arg;
+  /* Here's a variation on the previous state. Every line but the first is
+   * busted somehow. */
+  const char STATE[] =
+  /* Okay. */
+  "Guard in=default rsa_id=214F44BD5B638E8C817D47FF7C97397790BF0345 "
+    "nickname=TotallyNinja sampled_on=2016-11-12T19:32:49 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1\n"
+  /* No selection listed. */
+  "Guard rsa_id=052900AB0EA3ED54BAB84AE8A99E74E8693CE2B2 "
+    "nickname=5OfNovember sampled_on=2016-11-20T04:32:05 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1 confirmed_on=2016-11-22T08:13:28 confirmed_idx=0 "
+    "pb_circ_attempts=4.000000 pb_circ_successes=2.000000 "
+    "pb_successful_circuits_closed=2.000000\n"
+  /* Selection is "legacy"!! */
+  "Guard in=legacy rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
+    "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1 confirmed_on=2016-11-24T08:45:30 confirmed_idx=4 "
+    "pb_circ_attempts=5.000000 pb_circ_successes=5.000000 "
+    "pb_successful_circuits_closed=5.000000\n";
+
+  config_line_t *lines = NULL;
+  or_state_t *state = tor_malloc_zero(sizeof(or_state_t));
+  int r = config_get_lines(STATE, &lines, 0);
+  char *msg = NULL;
+
+  dummy_state = state;
+  MOCK(get_or_state,
+       get_or_state_replacement);
+
+  tt_int_op(r, OP_EQ, 0);
+  tt_assert(lines);
+
+  state->Guard = lines;
+
+  /* First, no-set case. we should get an error. */
+  r = entry_guards_parse_state(state, 0, &msg);
+  tt_int_op(r, OP_LT, 0);
+  tt_ptr_op(msg, OP_NE, NULL);
+  /* And we shouldn't have made anything. */
+  guard_selection_t *gs_df =
+    get_guard_selection_by_name("default", GS_TYPE_NORMAL, 0);
+  tt_ptr_op(gs_df, OP_EQ, NULL);
+  tor_free(msg);
 
-  chosen_entry = choose_random_entry(NULL);
-  tt_assert(chosen_entry);
+  /* Now see about the set case (which shouldn't happen IRL) */
+  r = entry_guards_parse_state(state, 1, &msg);
+  tt_int_op(r, OP_LT, 0);
+  tt_ptr_op(msg, OP_NE, NULL);
+  gs_df = get_guard_selection_by_name("default", GS_TYPE_NORMAL, 0);
+  tt_ptr_op(gs_df, OP_NE, NULL);
+  tt_int_op(smartlist_len(gs_df->sampled_entry_guards), OP_EQ, 1);
 
  done:
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  UNMOCK(get_options);
+  config_free_lines(lines);
+  tor_free(state);
+  tor_free(msg);
+  UNMOCK(get_or_state);
 }
 
-/** Helper to conduct tests for populate_live_entry_guards().
+static void
+test_entry_guard_get_guard_selection_by_name(void *arg)
+{
+  (void)arg;
+  guard_selection_t *gs1, *gs2, *gs3;
+
+  gs1 = get_guard_selection_by_name("unlikely", GS_TYPE_NORMAL, 0);
+  tt_ptr_op(gs1, OP_EQ, NULL);
+  gs1 = get_guard_selection_by_name("unlikely", GS_TYPE_NORMAL, 1);
+  tt_ptr_op(gs1, OP_NE, NULL);
+  gs2 = get_guard_selection_by_name("unlikely", GS_TYPE_NORMAL, 1);
+  tt_assert(gs2 == gs1);
+  gs2 = get_guard_selection_by_name("unlikely", GS_TYPE_NORMAL, 0);
+  tt_assert(gs2 == gs1);
+
+  gs2 = get_guard_selection_by_name("implausible", GS_TYPE_NORMAL, 0);
+  tt_ptr_op(gs2, OP_EQ, NULL);
+  gs2 = get_guard_selection_by_name("implausible", GS_TYPE_NORMAL, 1);
+  tt_ptr_op(gs2, OP_NE, NULL);
+  tt_assert(gs2 != gs1);
+  gs3 = get_guard_selection_by_name("implausible", GS_TYPE_NORMAL, 0);
+  tt_assert(gs3 == gs2);
+
+  gs3 = get_guard_selection_by_name("default", GS_TYPE_NORMAL, 0);
+  tt_ptr_op(gs3, OP_EQ, NULL);
+  gs3 = get_guard_selection_by_name("default", GS_TYPE_NORMAL, 1);
+  tt_ptr_op(gs3, OP_NE, NULL);
+  tt_assert(gs3 != gs2);
+  tt_assert(gs3 != gs1);
+  tt_assert(gs3 == get_guard_selection_info());
 
-   This test adds some entry guards to our list, and then tests
-   populate_live_entry_guards() to mke sure it filters them correctly.
+ done:
+  entry_guards_free_all();
+}
 
-   <b>num_needed</b> is the number of guard nodes we support. It's
-   configurable to make sure we function properly with 1 or 3 guard
-   nodes configured.
-*/
 static void
-populate_live_entry_guards_test_helper(int num_needed)
+test_entry_guard_choose_selection_initial(void *arg)
 {
-  smartlist_t *our_nodelist = NULL;
-  smartlist_t *live_entry_guards = smartlist_new();
-  const smartlist_t *all_entry_guards = get_entry_guards();
-  or_options_t *options = get_options_mutable();
-  int retval;
+  /* Tests for picking our initial guard selection (based on having had
+   * no previous selection */
+  (void)arg;
+  guard_selection_type_t type = GS_TYPE_INFER;
+  const char *name = choose_guard_selection(get_options(),
+                                            dummy_consensus, NULL, &type);
+  tt_str_op(name, OP_EQ, "default");
+  tt_int_op(type, OP_EQ, GS_TYPE_NORMAL);
+
+  /* If we're using bridges, we get the bridge selection. */
+  get_options_mutable()->UseBridges = 1;
+  name = choose_guard_selection(get_options(),
+                                dummy_consensus, NULL, &type);
+  tt_str_op(name, OP_EQ, "bridges");
+  tt_int_op(type, OP_EQ, GS_TYPE_BRIDGE);
+  get_options_mutable()->UseBridges = 0;
+
+  /* If we discard >99% of our guards, though, we should be in the restricted
+   * set. */
+  tt_assert(get_options_mutable()->EntryNodes == NULL);
+  get_options_mutable()->EntryNodes = routerset_new();
+  routerset_parse(get_options_mutable()->EntryNodes, "1.0.0.0/8", "foo");
+  name = choose_guard_selection(get_options(),
+                                dummy_consensus, NULL, &type);
+  tt_str_op(name, OP_EQ, "restricted");
+  tt_int_op(type, OP_EQ, GS_TYPE_RESTRICTED);
 
-  /* Set NumEntryGuards to the provided number. */
-  options->NumEntryGuards = num_needed;
-  tt_int_op(num_needed, OP_EQ, decide_num_guards(options, 0));
-
-  /* The global entry guards smartlist should be empty now. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ, 0);
-
-  /* Walk the nodelist and add all nodes as entry guards. */
-  our_nodelist = nodelist_get_list();
-  tt_int_op(smartlist_len(our_nodelist), OP_EQ, HELPER_NUMBER_OF_DESCRIPTORS);
-
-  SMARTLIST_FOREACH_BEGIN(our_nodelist, const node_t *, node) {
-    const node_t *node_tmp;
-    node_tmp = add_an_entry_guard(node, 0, 1, 0, 0);
-    tt_assert(node_tmp);
-  } SMARTLIST_FOREACH_END(node);
-
-  /* Make sure the nodes were added as entry guards. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ,
-            HELPER_NUMBER_OF_DESCRIPTORS);
-
-  /* Ensure that all the possible entry guards are enough to satisfy us. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_GE, num_needed);
-
-  /* Walk the entry guard list for some sanity checking */
-  SMARTLIST_FOREACH_BEGIN(all_entry_guards, const entry_guard_t *, entry) {
-    /* Since we called add_an_entry_guard() with 'for_discovery' being
-       False, all guards should have made_contact enabled. */
-    tt_int_op(entry->made_contact, OP_EQ, 1);
-
-  } SMARTLIST_FOREACH_END(entry);
-
-  /* First, try to get some fast guards. This should fail. */
-  retval = populate_live_entry_guards(live_entry_guards,
-                                      all_entry_guards,
-                                      NULL,
-                                      NO_DIRINFO, /* Don't care about DIRINFO*/
-                                      0, 0,
-                                      1); /* We want fast guard! */
-  tt_int_op(retval, OP_EQ, 0);
-  tt_int_op(smartlist_len(live_entry_guards), OP_EQ, 0);
-
-  /* Now try to get some stable guards. This should fail too. */
-  retval = populate_live_entry_guards(live_entry_guards,
-                                      all_entry_guards,
-                                      NULL,
-                                      NO_DIRINFO,
-                                      0,
-                                      1, /* We want stable guard! */
-                                      0);
-  tt_int_op(retval, OP_EQ, 0);
-  tt_int_op(smartlist_len(live_entry_guards), OP_EQ, 0);
-
-  /* Now try to get any guard we can find. This should succeed. */
-  retval = populate_live_entry_guards(live_entry_guards,
-                                      all_entry_guards,
-                                      NULL,
-                                      NO_DIRINFO,
-                                      0, 0, 0); /* No restrictions! */
-
-  /* Since we had more than enough guards in 'all_entry_guards', we
-     should have added 'num_needed' of them to live_entry_guards.
-     'retval' should be 1 since we now have enough live entry guards
-     to pick one.  */
-  tt_int_op(retval, OP_EQ, 1);
-  tt_int_op(smartlist_len(live_entry_guards), OP_EQ, num_needed);
+ done:
+  ;
+}
+
+static void
+test_entry_guard_add_single_guard(void *arg)
+{
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+
+  /* 1: Add a single guard to the sample. */
+  node_t *n1 = smartlist_get(big_fake_net_nodes, 0);
+  time_t now = approx_time();
+  tt_assert(n1->is_possible_guard == 1);
+  entry_guard_t *g1 = entry_guard_add_to_sample(gs, n1);
+  tt_assert(g1);
+
+  /* Make sure its fields look right. */
+  tt_mem_op(n1->identity, OP_EQ, g1->identity, DIGEST_LEN);
+  tt_i64_op(g1->sampled_on_date, OP_GE, now - 12*86400);
+  tt_i64_op(g1->sampled_on_date, OP_LE, now);
+  tt_str_op(g1->sampled_by_version, OP_EQ, VERSION);
+  tt_uint_op(g1->currently_listed, OP_EQ, 1);
+  tt_i64_op(g1->confirmed_on_date, OP_EQ, 0);
+  tt_int_op(g1->confirmed_idx, OP_EQ, -1);
+  tt_int_op(g1->last_tried_to_connect, OP_EQ, 0);
+  tt_uint_op(g1->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+  tt_i64_op(g1->failing_since, OP_EQ, 0);
+  tt_uint_op(g1->is_filtered_guard, OP_EQ, 1);
+  tt_uint_op(g1->is_usable_filtered_guard, OP_EQ, 1);
+  tt_uint_op(g1->is_primary, OP_EQ, 0);
+  tt_ptr_op(g1->extra_state_fields, OP_EQ, NULL);
+
+  /* Make sure it got added. */
+  tt_int_op(1, OP_EQ, smartlist_len(gs->sampled_entry_guards));
+  tt_ptr_op(g1, OP_EQ, smartlist_get(gs->sampled_entry_guards, 0));
+  tt_ptr_op(g1, OP_EQ, get_sampled_guard_with_id(gs, (uint8_t*)n1->identity));
+  const uint8_t bad_id[20] = {0};
+  tt_ptr_op(NULL, OP_EQ, get_sampled_guard_with_id(gs, bad_id));
 
  done:
-  smartlist_free(live_entry_guards);
+  guard_selection_free(gs);
 }
 
-/* Test populate_live_entry_guards() for 1 guard node. */
 static void
-test_populate_live_entry_guards_1guard(void *arg)
+test_entry_guard_node_filter(void *arg)
 {
-  (void) arg;
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  bridge_line_t *bl = NULL;
+
+  /* Initialize a bunch of node objects that are all guards. */
+#define NUM 7
+  node_t *n[NUM];
+  entry_guard_t *g[NUM];
+  int i;
+  for (i=0; i < NUM; ++i) {
+    n[i] = smartlist_get(big_fake_net_nodes, i*2); // even ones are guards.
+    g[i] = entry_guard_add_to_sample(gs, n[i]);
+
+    // everything starts out filtered-in
+    tt_uint_op(g[i]->is_filtered_guard, OP_EQ, 1);
+    tt_uint_op(g[i]->is_usable_filtered_guard, OP_EQ, 1);
+  }
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, NUM);
+
+  /* Make sure refiltering doesn't hurt */
+  entry_guards_update_filtered_sets(gs);
+  for (i = 0; i < NUM; ++i) {
+    tt_uint_op(g[i]->is_filtered_guard, OP_EQ, 1);
+    tt_uint_op(g[i]->is_usable_filtered_guard, OP_EQ, 1);
+  }
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, NUM);
+
+  /* Now start doing things to make the guards get filtered out, 1 by 1. */
+
+  /* 0: Not listed. */
+  g[0]->currently_listed = 0;
+
+  /* 1: path bias says this guard is maybe eeeevil. */
+  g[1]->pb.path_bias_disabled = 1;
+
+  /* 2: Unreachable address. */
+  n[2]->rs->addr = 0;
+
+  /* 3: ExcludeNodes */
+  n[3]->rs->addr = 0x90902020;
+  routerset_free(get_options_mutable()->ExcludeNodes);
+  get_options_mutable()->ExcludeNodes = routerset_new();
+  routerset_parse(get_options_mutable()->ExcludeNodes, "144.144.0.0/16", "");
+
+  /* 4: Bridge. */
+  get_options_mutable()->UseBridges = 1;
+  sweep_bridge_list();
+  bl = tor_malloc_zero(sizeof(bridge_line_t));
+  tor_addr_from_ipv4h(&bl->addr, n[4]->rs->addr);
+  bl->port = n[4]->rs->or_port;
+  memcpy(bl->digest, n[4]->identity, 20);
+  bridge_add_from_config(bl);
+  bl = NULL; // prevent free.
+  get_options_mutable()->UseBridges = 0;
+
+  /* 5: Unreachable. This stays in the filter, but isn't in usable-filtered */
+  g[5]->last_tried_to_connect = approx_time(); // prevent retry.
+  g[5]->is_reachable = GUARD_REACHABLE_NO;
+
+  /* 6: no change. */
+
+  /* Now refilter and inspect. */
+  entry_guards_update_filtered_sets(gs);
+  for (i = 0; i < NUM; ++i) {
+    tt_assert(g[i]->is_filtered_guard == (i == 5 || i == 6));
+    tt_assert(g[i]->is_usable_filtered_guard == (i == 6));
+  }
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, 1);
+
+  /* Now make sure we have no live consensus, and no nodes.  Nothing should
+   * pass the filter any more. */
+  tor_free(dummy_consensus);
+  dummy_consensus = NULL;
+  SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, node, {
+    memset(node->identity, 0xff, 20);
+  });
+  entry_guards_update_filtered_sets(gs);
+  for (i = 0; i < NUM; ++i) {
+    tt_uint_op(g[i]->is_filtered_guard, OP_EQ, 0);
+    tt_uint_op(g[i]->is_usable_filtered_guard, OP_EQ, 0);
+  }
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, 0);
 
-  populate_live_entry_guards_test_helper(1);
+ done:
+  guard_selection_free(gs);
+  tor_free(bl);
+#undef NUM
 }
 
-/* Test populate_live_entry_guards() for 3 guard nodes. */
 static void
-test_populate_live_entry_guards_3guards(void *arg)
+test_entry_guard_expand_sample(void *arg)
 {
-  (void) arg;
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  digestmap_t *node_by_id = digestmap_new();
+
+  entry_guard_t *guard = entry_guards_expand_sample(gs);
+  tt_assert(guard); // the last guard returned.
+
+  // Every sampled guard here should be filtered and reachable for now.
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ,
+            num_reachable_filtered_guards(gs, NULL));
+
+  /* Make sure we got the right number. */
+  tt_int_op(DFLT_MIN_FILTERED_SAMPLE_SIZE, OP_EQ,
+            num_reachable_filtered_guards(gs, NULL));
+
+  // Make sure everything we got was from our fake node list, and everything
+  // was unique.
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, g) {
+    const node_t *n = bfn_mock_node_get_by_id(g->identity);
+    tt_assert(n);
+    tt_ptr_op(NULL, OP_EQ, digestmap_get(node_by_id, g->identity));
+    digestmap_set(node_by_id, g->identity, (void*) n);
+    int idx = smartlist_pos(big_fake_net_nodes, n);
+    // The even ones are the guards; make sure we got guards.
+    tt_int_op(idx & 1, OP_EQ, 0);
+  } SMARTLIST_FOREACH_END(g);
+
+  // Nothing became unusable/unfiltered, so a subsequent expand should
+  // make no changes.
+  guard = entry_guards_expand_sample(gs);
+  tt_ptr_op(guard, OP_EQ, NULL); // no guard was added.
+  tt_int_op(DFLT_MIN_FILTERED_SAMPLE_SIZE, OP_EQ,
+            num_reachable_filtered_guards(gs, NULL));
+
+  // Make a few guards unreachable.
+  guard = smartlist_get(gs->sampled_entry_guards, 0);
+  guard->is_usable_filtered_guard = 0;
+  guard = smartlist_get(gs->sampled_entry_guards, 1);
+  guard->is_usable_filtered_guard = 0;
+  guard = smartlist_get(gs->sampled_entry_guards, 2);
+  guard->is_usable_filtered_guard = 0;
+  tt_int_op(DFLT_MIN_FILTERED_SAMPLE_SIZE - 3, OP_EQ,
+            num_reachable_filtered_guards(gs, NULL));
+
+  // This time, expanding the sample will add some more guards.
+  guard = entry_guards_expand_sample(gs);
+  tt_assert(guard); // no guard was added.
+  tt_int_op(DFLT_MIN_FILTERED_SAMPLE_SIZE, OP_EQ,
+            num_reachable_filtered_guards(gs, NULL));
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ,
+            num_reachable_filtered_guards(gs, NULL)+3);
+
+  // Still idempotent.
+  guard = entry_guards_expand_sample(gs);
+  tt_ptr_op(guard, OP_EQ, NULL); // no guard was added.
+  tt_int_op(DFLT_MIN_FILTERED_SAMPLE_SIZE, OP_EQ,
+            num_reachable_filtered_guards(gs, NULL));
+
+  // Now, do a nasty trick: tell the filter to exclude 31/32 of the guards.
+  // This will cause the sample size to get reeeeally huge, while the
+  // filtered sample size grows only slowly.
+  routerset_free(get_options_mutable()->ExcludeNodes);
+  get_options_mutable()->ExcludeNodes = routerset_new();
+  routerset_parse(get_options_mutable()->ExcludeNodes, "144.144.0.0/16", "");
+  SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, n, {
+    if (n_sl_idx % 64 != 0) {
+      n->rs->addr = 0x90903030;
+    }
+  });
+  entry_guards_update_filtered_sets(gs);
+
+  // Surely (p ~ 1-2**-60), one of our guards has been excluded.
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_LT,
+            DFLT_MIN_FILTERED_SAMPLE_SIZE);
+
+  // Try to regenerate the guards.
+  guard = entry_guards_expand_sample(gs);
+  tt_assert(guard); // no guard was added.
+
+  /* this time, it's possible that we didn't add enough sampled guards. */
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_LE,
+            DFLT_MIN_FILTERED_SAMPLE_SIZE);
+  /* but we definitely didn't exceed the sample maximum. */
+  const int n_guards = 271 / 2;
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_LE,
+            (int)(n_guards * .3));
 
-  populate_live_entry_guards_test_helper(3);
+ done:
+  guard_selection_free(gs);
+  digestmap_free(node_by_id, NULL);
 }
 
-/** Append some EntryGuard lines to the Tor state at <b>state</b>.
+static void
+test_entry_guard_expand_sample_small_net(void *arg)
+{
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+
+  /* Fun corner case: not enough guards to make up our whole sample size. */
+  SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, n, {
+    if (n_sl_idx >= 15) {
+      test_node_free(n);
+      SMARTLIST_DEL_CURRENT(big_fake_net_nodes, n);
+    } else {
+      n->rs->addr = 0; // make the filter reject this.
+    }
+  });
+
+  entry_guard_t *guard = entry_guards_expand_sample(gs);
+  tt_assert(guard); // the last guard returned -- some guard was added.
+  // half the nodes are guards, so we have 8 guards left.  The set
+  // is small, so we sampled everything.
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, 8);
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, 0);
+ done:
+  guard_selection_free(gs);
+}
 
-   <b>entry_guard_lines</b> is a smartlist containing 2-tuple
-   smartlists that carry the key and values of the statefile.
-   As an example:
-   entry_guard_lines =
-     (("EntryGuard", "name 67E72FF33D7D41BF11C569646A0A7B4B188340DF DirCache"),
-      ("EntryGuardDownSince", "2014-06-07 16:02:46 2014-06-07 16:02:46"))
-*/
 static void
-state_insert_entry_guard_helper(or_state_t *state,
-                                smartlist_t *entry_guard_lines)
+test_entry_guard_update_from_consensus_status(void *arg)
 {
-  config_line_t **next, *line;
+  /* Here we're going to have some nodes become un-guardy, and say we got a
+   * new consensus. This should cause those nodes to get detected as
+   * unreachable. */
 
-  next = &state->EntryGuards;
-  *next = NULL;
+  (void)arg;
+  int i;
+  time_t start = approx_time();
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  networkstatus_t *ns_tmp = NULL;
+
+  /* Don't randomly backdate stuff; it will make correctness harder to check.*/
+  MOCK(randomize_time, mock_randomize_time_no_randomization);
+
+  /* First, sample some guards. */
+  entry_guards_expand_sample(gs);
+  int n_sampled_pre = smartlist_len(gs->sampled_entry_guards);
+  int n_filtered_pre = num_reachable_filtered_guards(gs, NULL);
+  tt_i64_op(n_sampled_pre, OP_EQ, n_filtered_pre);
+  tt_i64_op(n_sampled_pre, OP_GT, 10);
+
+  /* At this point, it should be a no-op to do this: */
+  sampled_guards_update_from_consensus(gs);
+
+  /* Now let's make some of our guards become unlisted.  The easiest way to
+   * do that would be to take away their guard flag. */
+  for (i = 0; i < 5; ++i) {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    node_t *n = (node_t*) bfn_mock_node_get_by_id(g->identity);
+    tt_assert(n);
+    n->is_possible_guard = 0;
+  }
 
-  /* Loop over all the state lines in the smartlist */
-  SMARTLIST_FOREACH_BEGIN(entry_guard_lines, const smartlist_t *,state_lines) {
-    /* Get key and value for each line */
-    const char *state_key = smartlist_get(state_lines, 0);
-    const char *state_value = smartlist_get(state_lines, 1);
+  update_approx_time(start + 30);
+  {
+    /* try this with no live networkstatus. Nothing should happen! */
+    ns_tmp = dummy_consensus;
+    dummy_consensus = NULL;
+    sampled_guards_update_from_consensus(gs);
+    tt_i64_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_sampled_pre);
+    tt_i64_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, n_filtered_pre);
+    /* put the networkstatus back. */
+    dummy_consensus = ns_tmp;
+    ns_tmp = NULL;
+  }
 
-    *next = line = tor_malloc_zero(sizeof(config_line_t));
-    line->key = tor_strdup(state_key);
-    tor_asprintf(&line->value, "%s", state_value);
-    next = &(line->next);
-  } SMARTLIST_FOREACH_END(state_lines);
+  /* Now those guards should become unlisted, and drop off the filter, but
+   * stay in the sample. */
+  update_approx_time(start + 60);
+  sampled_guards_update_from_consensus(gs);
+
+  tt_i64_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_sampled_pre);
+  tt_i64_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, n_filtered_pre-5);
+  for (i = 0; i < 5; ++i) {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    tt_assert(! g->currently_listed);
+    tt_i64_op(g->unlisted_since_date, OP_EQ, start+60);
+  }
+  for (i = 5; i < n_sampled_pre; ++i) {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    tt_assert(g->currently_listed);
+    tt_i64_op(g->unlisted_since_date, OP_EQ, 0);
+  }
+
+  /* Now re-list one, and remove one completely. */
+  {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 0);
+    node_t *n = (node_t*) bfn_mock_node_get_by_id(g->identity);
+    tt_assert(n);
+    n->is_possible_guard = 1;
+  }
+  {
+    /* try removing the node, to make sure we don't crash on an absent node
+     */
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 5);
+    node_t *n = (node_t*) bfn_mock_node_get_by_id(g->identity);
+    tt_assert(n);
+    smartlist_remove(big_fake_net_nodes, n);
+    test_node_free(n);
+  }
+  update_approx_time(start + 300);
+  sampled_guards_update_from_consensus(gs);
+
+  /* guards 1..5 are now unlisted; 0,6,7.. are listed. */
+  tt_i64_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_sampled_pre);
+  for (i = 1; i < 6; ++i) {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    tt_assert(! g->currently_listed);
+    if (i == 5)
+      tt_i64_op(g->unlisted_since_date, OP_EQ, start+300);
+    else
+      tt_i64_op(g->unlisted_since_date, OP_EQ, start+60);
+  }
+  for (i = 0; i < n_sampled_pre; i = (!i) ? 6 : i+1) { /* 0,6,7,8, ... */
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    tt_assert(g->currently_listed);
+    tt_i64_op(g->unlisted_since_date, OP_EQ, 0);
+  }
+
+ done:
+  tor_free(ns_tmp); /* in case we couldn't put it back */
+  guard_selection_free(gs);
+  UNMOCK(randomize_time);
 }
 
-/** Free memory occupied by <b>entry_guard_lines</b>. */
 static void
-state_lines_free(smartlist_t *entry_guard_lines)
+test_entry_guard_update_from_consensus_repair(void *arg)
 {
-  SMARTLIST_FOREACH_BEGIN(entry_guard_lines, smartlist_t *, state_lines) {
-    char *state_key = smartlist_get(state_lines, 0);
-    char *state_value = smartlist_get(state_lines, 1);
+  /* Here we'll make sure that our code to repair the unlisted-since
+   * times is correct. */
 
-    tor_free(state_key);
-    tor_free(state_value);
-    smartlist_free(state_lines);
-  } SMARTLIST_FOREACH_END(state_lines);
+  (void)arg;
+  int i;
+  time_t start = approx_time();
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+
+  /* Don't randomly backdate stuff; it will make correctness harder to check.*/
+  MOCK(randomize_time, mock_randomize_time_no_randomization);
+
+  /* First, sample some guards. */
+  entry_guards_expand_sample(gs);
+  int n_sampled_pre = smartlist_len(gs->sampled_entry_guards);
+  int n_filtered_pre = num_reachable_filtered_guards(gs, NULL);
+  tt_i64_op(n_sampled_pre, OP_EQ, n_filtered_pre);
+  tt_i64_op(n_sampled_pre, OP_GT, 10);
+
+  /* Now corrupt the list a bit.  Call some unlisted-since-never, and some
+   * listed-and-unlisted-since-a-time. */
+  update_approx_time(start + 300);
+  for (i = 0; i < 3; ++i) {
+    /* these will get a date. */
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    node_t *n = (node_t*) bfn_mock_node_get_by_id(g->identity);
+    tt_assert(n);
+    n->is_possible_guard = 0;
+    g->currently_listed = 0;
+  }
+  for (i = 3; i < 6; ++i) {
+    /* these will become listed. */
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    g->unlisted_since_date = start+100;
+  }
+  setup_full_capture_of_logs(LOG_WARN);
+  sampled_guards_update_from_consensus(gs);
+  expect_log_msg_containing(
+             "was listed, but with unlisted_since_date set");
+  expect_log_msg_containing(
+             "was unlisted, but with unlisted_since_date unset");
+  teardown_capture_of_logs();
+
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_sampled_pre);
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, n_filtered_pre-3);
+  for (i = 3; i < n_sampled_pre; ++i) {
+    /* these will become listed. */
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    if (i < 3) {
+      tt_assert(! g->currently_listed);
+      tt_i64_op(g->unlisted_since_date, OP_EQ, start+300);
+    } else {
+      tt_assert(g->currently_listed);
+      tt_i64_op(g->unlisted_since_date, OP_EQ, 0);
+    }
+  }
 
-  smartlist_free(entry_guard_lines);
+ done:
+  teardown_capture_of_logs();
+  guard_selection_free(gs);
+  UNMOCK(randomize_time);
 }
 
-/* Tests entry_guards_parse_state(). It creates a fake Tor state with
-   a saved entry guard and makes sure that Tor can parse it and
-   creates the right entry node out of it.
-*/
 static void
-test_entry_guards_parse_state_simple(void *arg)
+test_entry_guard_update_from_consensus_remove(void *arg)
 {
-  or_state_t *state = or_state_new();
-  const smartlist_t *all_entry_guards = get_entry_guards();
-  smartlist_t *entry_state_lines = smartlist_new();
-  char *msg = NULL;
-  int retval;
+  /* Now let's check the logic responsible for removing guards from the
+   * sample entirely. */
+
+  (void)arg;
+  //int i;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  smartlist_t *keep_ids = smartlist_new();
+  smartlist_t *remove_ids = smartlist_new();
+
+  /* Don't randomly backdate stuff; it will make correctness harder to check.*/
+  MOCK(randomize_time, mock_randomize_time_no_randomization);
+
+  /* First, sample some guards. */
+  entry_guards_expand_sample(gs);
+  int n_sampled_pre = smartlist_len(gs->sampled_entry_guards);
+  int n_filtered_pre = num_reachable_filtered_guards(gs, NULL);
+  tt_i64_op(n_sampled_pre, OP_EQ, n_filtered_pre);
+  tt_i64_op(n_sampled_pre, OP_GT, 10);
+
+  const time_t one_day_ago = approx_time() - 1*24*60*60;
+  const time_t one_year_ago = approx_time() - 365*24*60*60;
+  const time_t two_years_ago = approx_time() - 2*365*24*60*60;
+  /* 0: unlisted for a day. (keep this) */
+  {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 0);
+    node_t *n = (node_t*) bfn_mock_node_get_by_id(g->identity);
+    tt_assert(n);
+    n->is_possible_guard = 0;
+    g->currently_listed = 0;
+    g->unlisted_since_date = one_day_ago;
+    smartlist_add(keep_ids, tor_memdup(g->identity, 20));
+  }
+  /* 1: unlisted for a year. (remove this) */
+  {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 1);
+    node_t *n = (node_t*) bfn_mock_node_get_by_id(g->identity);
+    tt_assert(n);
+    n->is_possible_guard = 0;
+    g->currently_listed = 0;
+    g->unlisted_since_date = one_year_ago;
+    smartlist_add(remove_ids, tor_memdup(g->identity, 20));
+  }
+  /* 2: added a day ago, never confirmed. (keep this) */
+  {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 2);
+    g->sampled_on_date = one_day_ago;
+    smartlist_add(keep_ids, tor_memdup(g->identity, 20));
+  }
+  /* 3: added a year ago, never confirmed. (remove this) */
+  {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 3);
+    g->sampled_on_date = one_year_ago;
+    smartlist_add(remove_ids, tor_memdup(g->identity, 20));
+  }
+  /* 4: added two year ago, confirmed yesterday, primary. (keep this.) */
+  {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 4);
+    g->sampled_on_date = one_year_ago;
+    g->confirmed_on_date = one_day_ago;
+    g->confirmed_idx = 0;
+    g->is_primary = 1;
+    smartlist_add(gs->confirmed_entry_guards, g);
+    smartlist_add(gs->primary_entry_guards, g);
+    smartlist_add(keep_ids, tor_memdup(g->identity, 20));
+  }
+  /* 5: added two years ago, confirmed a year ago, primary. (remove this) */
+  {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 5);
+    g->sampled_on_date = two_years_ago;
+    g->confirmed_on_date = one_year_ago;
+    g->confirmed_idx = 1;
+    g->is_primary = 1;
+    smartlist_add(gs->confirmed_entry_guards, g);
+    smartlist_add(gs->primary_entry_guards, g);
+    smartlist_add(remove_ids, tor_memdup(g->identity, 20));
+  }
 
-  /* Details of our fake guard node */
-  const char *nickname = "hagbard";
-  const char *fpr = "B29D536DD1752D542E1FBB3C9CE4449D51298212";
-  const char *tor_version = "0.2.5.3-alpha-dev";
-  const char *added_at = get_yesterday_date_str();
-  const char *unlisted_since = "2014-06-08 16:16:50";
+  sampled_guards_update_from_consensus(gs);
 
-  (void) arg;
+  /* Did we remove the right ones? */
+  SMARTLIST_FOREACH(keep_ids, uint8_t *, id, {
+      tt_assert(get_sampled_guard_with_id(gs, id) != NULL);
+  });
+  SMARTLIST_FOREACH(remove_ids, uint8_t *, id, {
+    tt_want(get_sampled_guard_with_id(gs, id) == NULL);
+  });
+
+  /* Did we remove the right number? */
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_sampled_pre - 3);
 
-  /* The global entry guards smartlist should be empty now. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ, 0);
+ done:
+  guard_selection_free(gs);
+  UNMOCK(randomize_time);
+  SMARTLIST_FOREACH(keep_ids, char *, cp, tor_free(cp));
+  SMARTLIST_FOREACH(remove_ids, char *, cp, tor_free(cp));
+  smartlist_free(keep_ids);
+  smartlist_free(remove_ids);
+}
+
+static void
+test_entry_guard_confirming_guards(void *arg)
+{
+  (void)arg;
+  /* Now let's check the logic responsible for manipulating the list
+   * of confirmed guards */
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  MOCK(randomize_time, mock_randomize_time_no_randomization);
+
+  /* Create the sample. */
+  entry_guards_expand_sample(gs);
+
+  /* Confirm a few  guards. */
+  time_t start = approx_time();
+  entry_guard_t *g1 = smartlist_get(gs->sampled_entry_guards, 0);
+  entry_guard_t *g2 = smartlist_get(gs->sampled_entry_guards, 1);
+  entry_guard_t *g3 = smartlist_get(gs->sampled_entry_guards, 8);
+  make_guard_confirmed(gs, g2);
+  update_approx_time(start + 10);
+  make_guard_confirmed(gs, g1);
+  make_guard_confirmed(gs, g3);
+
+  /* Were the correct dates and indices fed in? */
+  tt_int_op(g1->confirmed_idx, OP_EQ, 1);
+  tt_int_op(g2->confirmed_idx, OP_EQ, 0);
+  tt_int_op(g3->confirmed_idx, OP_EQ, 2);
+  tt_i64_op(g1->confirmed_on_date, OP_EQ, start+10);
+  tt_i64_op(g2->confirmed_on_date, OP_EQ, start);
+  tt_i64_op(g3->confirmed_on_date, OP_EQ, start+10);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 0), OP_EQ, g2);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 1), OP_EQ, g1);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 2), OP_EQ, g3);
+
+  /* Now make sure we can regenerate the confirmed_entry_guards list. */
+  smartlist_clear(gs->confirmed_entry_guards);
+  g2->confirmed_idx = 0;
+  g1->confirmed_idx = 10;
+  g3->confirmed_idx = 100;
+  entry_guards_update_confirmed(gs);
+  tt_int_op(g1->confirmed_idx, OP_EQ, 1);
+  tt_int_op(g2->confirmed_idx, OP_EQ, 0);
+  tt_int_op(g3->confirmed_idx, OP_EQ, 2);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 0), OP_EQ, g2);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 1), OP_EQ, g1);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 2), OP_EQ, g3);
+
+  /* Now make sure we can regenerate the confirmed_entry_guards list if
+   * the indices are messed up. */
+  g1->confirmed_idx = g2->confirmed_idx = g3->confirmed_idx = 999;
+  smartlist_clear(gs->confirmed_entry_guards);
+  entry_guards_update_confirmed(gs);
+  tt_int_op(g1->confirmed_idx, OP_GE, 0);
+  tt_int_op(g2->confirmed_idx, OP_GE, 0);
+  tt_int_op(g3->confirmed_idx, OP_GE, 0);
+  tt_int_op(g1->confirmed_idx, OP_LE, 2);
+  tt_int_op(g2->confirmed_idx, OP_LE, 2);
+  tt_int_op(g3->confirmed_idx, OP_LE, 2);
+  g1 = smartlist_get(gs->confirmed_entry_guards, 0);
+  g2 = smartlist_get(gs->confirmed_entry_guards, 1);
+  g3 = smartlist_get(gs->confirmed_entry_guards, 2);
+  tt_int_op(g1->confirmed_idx, OP_EQ, 0);
+  tt_int_op(g2->confirmed_idx, OP_EQ, 1);
+  tt_int_op(g3->confirmed_idx, OP_EQ, 2);
+  tt_assert(g1 != g2);
+  tt_assert(g1 != g3);
+  tt_assert(g2 != g3);
+
+ done:
+  UNMOCK(randomize_time);
+  guard_selection_free(gs);
+}
+
+static void
+test_entry_guard_sample_reachable_filtered(void *arg)
+{
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  entry_guards_expand_sample(gs);
+  const int N = 10000;
+  bitarray_t *selected = NULL;
+  int i, j;
+
+  /* We've got a sampled list now; let's make one non-usable-filtered; some
+   * confirmed, some primary, some pending.
+   */
+  int n_guards = smartlist_len(gs->sampled_entry_guards);
+  tt_int_op(n_guards, OP_GT, 10);
+  entry_guard_t *g;
+  g = smartlist_get(gs->sampled_entry_guards, 0);
+  g->is_pending = 1;
+  g = smartlist_get(gs->sampled_entry_guards, 1);
+  make_guard_confirmed(gs, g);
+  g = smartlist_get(gs->sampled_entry_guards, 2);
+  g->is_primary = 1;
+  g = smartlist_get(gs->sampled_entry_guards, 3);
+  g->pb.path_bias_disabled = 1;
+
+  entry_guards_update_filtered_sets(gs);
+  gs->primary_guards_up_to_date = 1;
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, n_guards - 1);
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_guards);
+
+  // +1 since the one we made disabled will make  another one get added.
+  ++n_guards;
+
+  /* Try a bunch of selections. */
+  const struct {
+    int flag; int idx;
+  } tests[] = {
+    { 0, -1 },
+    { SAMPLE_EXCLUDE_CONFIRMED, 1 },
+    { SAMPLE_EXCLUDE_PRIMARY|SAMPLE_NO_UPDATE_PRIMARY, 2 },
+    { SAMPLE_EXCLUDE_PENDING, 0 },
+    { -1, -1},
+  };
+
+  for (j = 0; tests[j].flag >= 0; ++j) {
+    selected = bitarray_init_zero(n_guards);
+    const int excluded_flags = tests[j].flag;
+    const int excluded_idx = tests[j].idx;
+    for (i = 0; i < N; ++i) {
+      g = sample_reachable_filtered_entry_guards(gs, NULL, excluded_flags);
+      tor_assert(g);
+      int pos = smartlist_pos(gs->sampled_entry_guards, g);
+      tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_guards);
+      tt_int_op(pos, OP_GE, 0);
+      tt_int_op(pos, OP_LT, n_guards);
+      bitarray_set(selected, pos);
+    }
+    for (i = 0; i < n_guards; ++i) {
+      const int should_be_set = (i != excluded_idx &&
+                                 i != 3); // filtered out.
+      tt_int_op(!!bitarray_is_set(selected, i), OP_EQ, should_be_set);
+    }
+    bitarray_free(selected);
+    selected = NULL;
+  }
+
+ done:
+  guard_selection_free(gs);
+  bitarray_free(selected);
+}
 
-  { /* Prepare the state entry */
+static void
+test_entry_guard_sample_reachable_filtered_empty(void *arg)
+{
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  /* What if we try to sample from a set of 0? */
+  SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, n,
+                    n->is_possible_guard = 0);
 
-    /* Prepare the smartlist to hold the key/value of each line */
-    smartlist_t *state_line = smartlist_new();
-    smartlist_add_asprintf(state_line, "EntryGuard");
-    smartlist_add_asprintf(state_line, "%s %s %s", nickname, fpr, "DirCache");
-    smartlist_add(entry_state_lines, state_line);
+  entry_guard_t *g = sample_reachable_filtered_entry_guards(gs, NULL, 0);
+  tt_ptr_op(g, OP_EQ, NULL);
 
-    state_line = smartlist_new();
-    smartlist_add_asprintf(state_line, "EntryGuardAddedBy");
-    smartlist_add_asprintf(state_line, "%s %s %s", fpr, tor_version, added_at);
-    smartlist_add(entry_state_lines, state_line);
+ done:
+  guard_selection_free(gs);
+}
+
+static void
+test_entry_guard_retry_unreachable(void *arg)
+{
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+
+  entry_guards_expand_sample(gs);
+  /* Let's say that we have two guards, and they're down.
+   */
+  time_t start = approx_time();
+  entry_guard_t *g1 = smartlist_get(gs->sampled_entry_guards, 0);
+  entry_guard_t *g2 = smartlist_get(gs->sampled_entry_guards, 1);
+  entry_guard_t *g3 = smartlist_get(gs->sampled_entry_guards, 2);
+  g1->is_reachable = GUARD_REACHABLE_NO;
+  g2->is_reachable = GUARD_REACHABLE_NO;
+  g1->is_primary = 1;
+  g1->failing_since = g2->failing_since = start;
+  g1->last_tried_to_connect = g2->last_tried_to_connect = start;
+
+  /* Wait 5 minutes.  Nothing will get retried. */
+  update_approx_time(start + 5 * 60);
+  entry_guard_consider_retry(g1);
+  entry_guard_consider_retry(g2);
+  entry_guard_consider_retry(g3); // just to make sure this doesn't crash.
+  tt_int_op(g1->is_reachable, OP_EQ, GUARD_REACHABLE_NO);
+  tt_int_op(g2->is_reachable, OP_EQ, GUARD_REACHABLE_NO);
+  tt_int_op(g3->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+
+  /* After 30 min, the primary one gets retried */
+  update_approx_time(start + 35 * 60);
+  entry_guard_consider_retry(g1);
+  entry_guard_consider_retry(g2);
+  tt_int_op(g1->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+  tt_int_op(g2->is_reachable, OP_EQ, GUARD_REACHABLE_NO);
+
+  g1->is_reachable = GUARD_REACHABLE_NO;
+  g1->last_tried_to_connect = start + 55*60;
+
+  /* After 1 hour, we'll retry the nonprimary one. */
+  update_approx_time(start + 61 * 60);
+  entry_guard_consider_retry(g1);
+  entry_guard_consider_retry(g2);
+  tt_int_op(g1->is_reachable, OP_EQ, GUARD_REACHABLE_NO);
+  tt_int_op(g2->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+
+  g2->is_reachable = GUARD_REACHABLE_NO;
+  g2->last_tried_to_connect = start + 61*60;
+
+  /* And then the primary one again. */
+  update_approx_time(start + 66 * 60);
+  entry_guard_consider_retry(g1);
+  entry_guard_consider_retry(g2);
+  tt_int_op(g1->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+  tt_int_op(g2->is_reachable, OP_EQ, GUARD_REACHABLE_NO);
 
-    state_line = smartlist_new();
-    smartlist_add_asprintf(state_line, "EntryGuardUnlistedSince");
-    smartlist_add_asprintf(state_line, "%s", unlisted_since);
-    smartlist_add(entry_state_lines, state_line);
+ done:
+  guard_selection_free(gs);
+}
+
+static void
+test_entry_guard_manage_primary(void *arg)
+{
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  smartlist_t *prev_guards = smartlist_new();
+
+  /* If no guards are confirmed, we should pick a few reachable guards and
+   * call them all primary. But not confirmed.*/
+  entry_guards_update_primary(gs);
+  int n_primary = smartlist_len(gs->primary_entry_guards);
+  tt_int_op(n_primary, OP_GE, 1);
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, g, {
+    tt_assert(g->is_primary);
+    tt_assert(g->confirmed_idx == -1);
+  });
+
+  /* Calling it a second time should leave the guards unchanged. */
+  smartlist_add_all(prev_guards, gs->primary_entry_guards);
+  entry_guards_update_primary(gs);
+  tt_int_op(smartlist_len(gs->primary_entry_guards), OP_EQ, n_primary);
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, g, {
+    tt_ptr_op(g, OP_EQ, smartlist_get(prev_guards, g_sl_idx));
+  });
+
+  /* If we have one confirmed guard, that guards becomes the first primary
+   * guard, and the other primary guards get kept. */
+
+  /* find a non-primary guard... */
+  entry_guard_t *confirmed = NULL;
+  SMARTLIST_FOREACH(gs->sampled_entry_guards, entry_guard_t *, g, {
+    if (! g->is_primary) {
+      confirmed = g;
+      break;
+    }
+  });
+  tt_assert(confirmed);
+  /* make it confirmed. */
+  make_guard_confirmed(gs, confirmed);
+  /* update the list... */
+  smartlist_clear(prev_guards);
+  smartlist_add_all(prev_guards, gs->primary_entry_guards);
+  entry_guards_update_primary(gs);
+
+  /*  and see what's primary now! */
+  tt_int_op(smartlist_len(gs->primary_entry_guards), OP_EQ, n_primary);
+  tt_ptr_op(smartlist_get(gs->primary_entry_guards, 0), OP_EQ, confirmed);
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, g, {
+    tt_assert(g->is_primary);
+    if (g_sl_idx == 0)
+      continue;
+    tt_ptr_op(g, OP_EQ, smartlist_get(prev_guards, g_sl_idx - 1));
+  });
+  {
+    entry_guard_t *prev_last_guard = smartlist_get(prev_guards, n_primary-1);
+    tt_assert(! prev_last_guard->is_primary);
   }
 
-  /* Inject our lines in the state */
-  state_insert_entry_guard_helper(state, entry_state_lines);
+  /* Calling it a fourth time should leave the guards unchanged. */
+  smartlist_clear(prev_guards);
+  smartlist_add_all(prev_guards, gs->primary_entry_guards);
+  entry_guards_update_primary(gs);
+  tt_int_op(smartlist_len(gs->primary_entry_guards), OP_EQ, n_primary);
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, g, {
+    tt_ptr_op(g, OP_EQ, smartlist_get(prev_guards, g_sl_idx));
+  });
+
+  /* Do some dirinfo checks */
+  {
+    /* Check that we have all required dirinfo for the primaries (that's done
+     * in big_fake_network_setup()) */
+    char *dir_info_str =
+      guard_selection_get_err_str_if_dir_info_missing(gs, 0, 0, 0);
+    tt_assert(!dir_info_str);
+
+    /* Now artificially remove the first primary's descriptor and re-check */
+    entry_guard_t *first_primary;
+    first_primary = smartlist_get(gs->primary_entry_guards, 0);
+    /* Change the first primary's identity digest so that the mocked functions
+     * can't find its descriptor */
+    memset(first_primary->identity, 9, sizeof(first_primary->identity));
+    dir_info_str =guard_selection_get_err_str_if_dir_info_missing(gs, 1, 2, 3);
+    tt_str_op(dir_info_str, OP_EQ,
+              "We're missing descriptors for 1/2 of our primary entry guards "
+              "(total microdescriptors: 2/3). That's ok. We will try to fetch "
+              "missing descriptors soon.");
+    tor_free(dir_info_str);
+  }
 
-  /* Parse state */
-  retval = entry_guards_parse_state(state, 1, &msg);
-  tt_int_op(retval, OP_GE, 0);
+ done:
+  guard_selection_free(gs);
+  smartlist_free(prev_guards);
+}
+
+static void
+test_entry_guard_guard_preferred(void *arg)
+{
+  (void) arg;
+  entry_guard_t *g1 = tor_malloc_zero(sizeof(entry_guard_t));
+  entry_guard_t *g2 = tor_malloc_zero(sizeof(entry_guard_t));
 
-  /* Test that the guard was registered.
-     We need to re-get the entry guard list since its pointer was
-     overwritten in entry_guards_parse_state(). */
-  all_entry_guards = get_entry_guards();
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ, 1);
+  g1->confirmed_idx = g2->confirmed_idx = -1;
+  g1->last_tried_to_connect = approx_time();
+  g2->last_tried_to_connect = approx_time();
 
-  { /* Test the entry guard structure */
-    char hex_digest[1024];
-    char str_time[1024];
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g1, g1));
 
-    const entry_guard_t *e = smartlist_get(all_entry_guards, 0);
-    tt_str_op(e->nickname, OP_EQ, nickname); /* Verify nickname */
+  /* Neither is pending; priorities equal. */
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g2, g1));
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g1, g2));
 
-    base16_encode(hex_digest, sizeof(hex_digest),
-                  e->identity, DIGEST_LEN);
-    tt_str_op(hex_digest, OP_EQ, fpr); /* Verify fingerprint */
+  /* If one is pending, the pending one has higher priority */
+  g1->is_pending = 1;
+  tt_int_op(1, OP_EQ, entry_guard_has_higher_priority(g1, g2));
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g2, g1));
 
-    tt_assert(e->is_dir_cache); /* Verify dirness */
+  /* If both are pending, and last_tried_to_connect is equal:
+     priorities equal */
+  g2->is_pending = 1;
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g2, g1));
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g1, g2));
 
-    tt_str_op(e->chosen_by_version, OP_EQ, tor_version); /* Verify version */
+  /* One had a connection that startied earlier: it has higher priority. */
+  g2->last_tried_to_connect -= 10;
+  tt_int_op(1, OP_EQ, entry_guard_has_higher_priority(g2, g1));
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g1, g2));
 
-    tt_assert(e->made_contact); /* All saved guards have been contacted */
+  /* Now, say that g1 is confirmed. It will get higher priority. */
+  g1->confirmed_idx = 5;
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g2, g1));
+  tt_int_op(1, OP_EQ, entry_guard_has_higher_priority(g1, g2));
 
-    tt_assert(e->bad_since); /* Verify bad_since timestamp */
-    format_iso_time(str_time, e->bad_since);
-    tt_str_op(str_time, OP_EQ, unlisted_since);
+  /* But if g2 was confirmed first, it will get priority */
+  g2->confirmed_idx = 2;
+  tt_int_op(1, OP_EQ, entry_guard_has_higher_priority(g2, g1));
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g1, g2));
 
-    /* The rest should be unset */
-    tt_assert(!e->unreachable_since);
-    tt_assert(!e->can_retry);
-    tt_assert(!e->path_bias_noticed);
-    tt_assert(!e->path_bias_warned);
-    tt_assert(!e->path_bias_extreme);
-    tt_assert(!e->path_bias_disabled);
-    tt_assert(!e->path_bias_use_noticed);
-    tt_assert(!e->path_bias_use_extreme);
-    tt_assert(!e->last_attempted);
+ done:
+  tor_free(g1);
+  tor_free(g2);
+}
+
+static void
+test_entry_guard_select_for_circuit_no_confirmed(void *arg)
+{
+  /* Simpler cases: no gaurds are confirmed yet. */
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  entry_guard_restriction_t *rst = NULL;
+
+  /* simple starting configuration */
+  entry_guards_update_primary(gs);
+  unsigned state = 9999;
+
+  entry_guard_t *g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC,
+                                                    NULL, &state);
+
+  tt_assert(g);
+  tt_assert(g->is_primary);
+  tt_int_op(g->confirmed_idx, OP_EQ, -1);
+  tt_uint_op(g->is_pending, OP_EQ, 0); // primary implies non-pending.
+  tt_uint_op(state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  tt_i64_op(g->last_tried_to_connect, OP_EQ, approx_time());
+
+  // If we do that again, we should get the same guard.
+  entry_guard_t *g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC,
+                                                     NULL, &state);
+  tt_ptr_op(g2, OP_EQ, g);
+
+  // if we mark that guard down, we should get a different primary guard.
+  // auto-retry it.
+  g->is_reachable = GUARD_REACHABLE_NO;
+  g->failing_since = approx_time() - 10;
+  g->last_tried_to_connect = approx_time() - 10;
+  state = 9999;
+  g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_ptr_op(g2, OP_NE, g);
+  tt_assert(g2);
+  tt_assert(g2->is_primary);
+  tt_int_op(g2->confirmed_idx, OP_EQ, -1);
+  tt_uint_op(g2->is_pending, OP_EQ, 0); // primary implies non-pending.
+  tt_uint_op(state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  tt_i64_op(g2->last_tried_to_connect, OP_EQ, approx_time());
+
+  // If we say that the first primary guard was last tried a long time ago, we
+  // should get an automatic retry on it.
+  g->failing_since = approx_time() - 72*60*60;
+  g->last_tried_to_connect = approx_time() - 72*60*60;
+  state = 9999;
+  g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_ptr_op(g2, OP_EQ, g);
+  tt_assert(g2);
+  tt_uint_op(state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  tt_i64_op(g2->last_tried_to_connect, OP_EQ, approx_time());
+  tt_int_op(g2->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+
+  // And if we mark ALL the primary guards down, we should get another guard
+  // at random.
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, guard, {
+    guard->is_reachable = GUARD_REACHABLE_NO;
+    guard->last_tried_to_connect = approx_time() - 5;
+    guard->failing_since = approx_time() - 30;
+  });
+  state = 9999;
+  g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_assert(g2);
+  tt_assert(!g2->is_primary);
+  tt_int_op(g2->confirmed_idx, OP_EQ, -1);
+  tt_uint_op(g2->is_pending, OP_EQ, 1);
+  tt_uint_op(state, OP_EQ, GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+  tt_i64_op(g2->last_tried_to_connect, OP_EQ, approx_time());
+  tt_int_op(g2->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+
+  // As a bonus, maybe we should be retrying the primary guards. Let's say so.
+  mark_primary_guards_maybe_reachable(gs);
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, guard, {
+    tt_int_op(guard->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+    tt_assert(guard->is_usable_filtered_guard == 1);
+    // no change to these fields.
+    tt_i64_op(guard->last_tried_to_connect, OP_EQ, approx_time() - 5);
+    tt_i64_op(guard->failing_since, OP_EQ, approx_time() - 30);
+  });
+
+  /* Let's try again and we should get the first primary guard again */
+  g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_ptr_op(g, OP_EQ, smartlist_get(gs->primary_entry_guards, 0));
+  g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_ptr_op(g2, OP_EQ, g);
+
+  /* But if we impose a restriction, we don't get the same guard */
+  rst = guard_create_exit_restriction((uint8_t*)g->identity);
+  g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, rst, &state);
+  tt_ptr_op(g2, OP_NE, g);
+
+ done:
+  guard_selection_free(gs);
+  entry_guard_restriction_free(rst);
+}
+
+static void
+test_entry_guard_select_for_circuit_confirmed(void *arg)
+{
+  /* Case 2: if all the primary guards are down, and there are more confirmed
+     guards, we use a confirmed guard. */
+  (void)arg;
+  int i;
+  entry_guard_restriction_t *rst = NULL;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  const int N_CONFIRMED = 10;
+
+  /* slightly more complicated simple starting configuration */
+  entry_guards_update_primary(gs);
+  for (i = 0; i < N_CONFIRMED; ++i) {
+    entry_guard_t *guard = smartlist_get(gs->sampled_entry_guards, i);
+    make_guard_confirmed(gs, guard);
+  }
+  entry_guards_update_primary(gs); // rebuild the primary list.
+
+  unsigned state = 9999;
+
+  // As above, this gives us a primary guard.
+  entry_guard_t *g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC,
+                                                    NULL, &state);
+  tt_assert(g);
+  tt_assert(g->is_primary);
+  tt_int_op(g->confirmed_idx, OP_EQ, 0);
+  tt_uint_op(g->is_pending, OP_EQ, 0); // primary implies non-pending.
+  tt_uint_op(state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  tt_i64_op(g->last_tried_to_connect, OP_EQ, approx_time());
+  tt_ptr_op(g, OP_EQ, smartlist_get(gs->primary_entry_guards, 0));
+
+  // But if we mark all the primary guards down...
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, guard, {
+    guard->last_tried_to_connect = approx_time();
+    entry_guards_note_guard_failure(gs, guard);
+  });
+
+  // ... we should get a confirmed guard.
+  state = 9999;
+  g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_assert(g);
+  tt_assert(! g->is_primary);
+  tt_int_op(g->confirmed_idx, OP_EQ, smartlist_len(gs->primary_entry_guards));
+  tt_assert(g->is_pending);
+  tt_uint_op(state, OP_EQ, GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+  tt_i64_op(g->last_tried_to_connect, OP_EQ, approx_time());
+
+  // And if we try again, we should get a different confirmed guard, since
+  // that one is pending.
+  state = 9999;
+  entry_guard_t *g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC,
+                                                     NULL, &state);
+  tt_assert(g2);
+  tt_assert(! g2->is_primary);
+  tt_ptr_op(g2, OP_NE, g);
+  tt_int_op(g2->confirmed_idx, OP_EQ,
+            smartlist_len(gs->primary_entry_guards)+1);
+  tt_assert(g2->is_pending);
+  tt_uint_op(state, OP_EQ, GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+  tt_i64_op(g2->last_tried_to_connect, OP_EQ, approx_time());
+
+  // If we say that the next confirmed guard in order is excluded, and
+  // we disable EnforceDistinctSubnets, we get the guard AFTER the
+  // one we excluded.
+  get_options_mutable()->EnforceDistinctSubnets = 0;
+  g = smartlist_get(gs->confirmed_entry_guards,
+                     smartlist_len(gs->primary_entry_guards)+2);
+  rst = guard_create_exit_restriction((uint8_t*)g->identity);
+  g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, rst, &state);
+  tt_ptr_op(g2, OP_NE, NULL);
+  tt_ptr_op(g2, OP_NE, g);
+  tt_int_op(g2->confirmed_idx, OP_EQ,
+            smartlist_len(gs->primary_entry_guards)+3);
+
+  // If we make every confirmed guard become pending then we start poking
+  // other guards.
+  const int n_remaining_confirmed =
+    N_CONFIRMED - 3 - smartlist_len(gs->primary_entry_guards);
+  for (i = 0; i < n_remaining_confirmed; ++i) {
+    g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+    tt_int_op(g->confirmed_idx, OP_GE, 0);
+    tt_assert(g);
   }
+  state = 9999;
+  g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_assert(g);
+  tt_assert(g->is_pending);
+  tt_int_op(g->confirmed_idx, OP_EQ, -1);
+
+  // If we EnforceDistinctSubnets and apply a restriction, we get
+  // nothing, since we put all of the nodes in the same /16.
+  // Regression test for bug 22753/TROVE-2017-006.
+  get_options_mutable()->EnforceDistinctSubnets = 1;
+  g = smartlist_get(gs->confirmed_entry_guards, 0);
+  memcpy(rst->exclude_id, g->identity, DIGEST_LEN);
+  g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, rst, &state);
+  tt_ptr_op(g2, OP_EQ, NULL);
 
  done:
-  state_lines_free(entry_state_lines);
-  or_state_free(state);
-  tor_free(msg);
+  guard_selection_free(gs);
+  entry_guard_restriction_free(rst);
 }
 
-/** Similar to test_entry_guards_parse_state_simple() but aims to test
-    the PathBias-related details of the entry guard. */
 static void
-test_entry_guards_parse_state_pathbias(void *arg)
+test_entry_guard_select_for_circuit_highlevel_primary(void *arg)
 {
-  or_state_t *state = or_state_new();
-  const smartlist_t *all_entry_guards = get_entry_guards();
-  char *msg = NULL;
-  int retval;
-  smartlist_t *entry_state_lines = smartlist_new();
+  /* Play around with selecting primary guards for circuits and markign
+   * them up and down */
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+
+  time_t start = approx_time();
+
+  const node_t *node = NULL;
+  circuit_guard_state_t *guard = NULL;
+  entry_guard_t *g;
+  guard_usable_t u;
+  /*
+   * Make sure that the pick-for-circuit API basically works.  We'll get
+   * a primary guard, so it'll be usable on completion.
+   */
+  int r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                       &node, &guard);
+
+  tt_int_op(r, OP_EQ, 0);
+  tt_assert(node);
+  tt_assert(guard);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  g = entry_guard_handle_get(guard->guard);
+  tt_assert(g);
+  tt_mem_op(g->identity, OP_EQ, node->identity, DIGEST_LEN);
+  tt_int_op(g->is_primary, OP_EQ, 1);
+  tt_i64_op(g->last_tried_to_connect, OP_EQ, start);
+  tt_int_op(g->confirmed_idx, OP_EQ, -1);
+
+  /* Call that circuit successful. */
+  update_approx_time(start+15);
+  u = entry_guard_succeeded(&guard);
+  tt_int_op(u, OP_EQ, GUARD_USABLE_NOW); /* We can use it now. */
+  tt_assert(guard);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE);
+  g = entry_guard_handle_get(guard->guard);
+  tt_assert(g);
+  tt_int_op(g->is_reachable, OP_EQ, GUARD_REACHABLE_YES);
+  tt_int_op(g->confirmed_idx, OP_EQ, 0);
+
+  circuit_guard_state_free(guard);
+  guard = NULL;
+  node = NULL;
+  g = NULL;
+
+  /* Try again. We'll also get a primary guard this time. (The same one,
+     in fact.)  But this time, we'll say the connection has failed. */
+  update_approx_time(start+35);
+  r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                   &node, &guard);
+  tt_int_op(r, OP_EQ, 0);
+  tt_assert(node);
+  tt_assert(guard);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  tt_i64_op(guard->state_set_at, OP_EQ, start+35);
+  g = entry_guard_handle_get(guard->guard);
+  tt_assert(g);
+  tt_mem_op(g->identity, OP_EQ, node->identity, DIGEST_LEN);
+  tt_int_op(g->is_primary, OP_EQ, 1);
+  tt_i64_op(g->last_tried_to_connect, OP_EQ, start+35);
+  tt_int_op(g->confirmed_idx, OP_EQ, 0); // same one.
+
+  /* It's failed!  What will happen to our poor guard? */
+  update_approx_time(start+45);
+  entry_guard_failed(&guard);
+  tt_assert(guard);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_DEAD);
+  tt_i64_op(guard->state_set_at, OP_EQ, start+45);
+  g = entry_guard_handle_get(guard->guard);
+  tt_assert(g);
+  tt_int_op(g->is_reachable, OP_EQ, GUARD_REACHABLE_NO);
+  tt_i64_op(g->failing_since, OP_EQ, start+45);
+  tt_int_op(g->confirmed_idx, OP_EQ, 0); // still confirmed.
+
+  circuit_guard_state_free(guard);
+  guard = NULL;
+  node = NULL;
+  entry_guard_t *g_prev = g;
+  g = NULL;
+
+  /* Now try a third time. Since the other one is down, we'll get a different
+   * (still primary) guard.
+   */
+  update_approx_time(start+60);
+  r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                   &node, &guard);
+  tt_int_op(r, OP_EQ, 0);
+  tt_assert(node);
+  tt_assert(guard);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  g = entry_guard_handle_get(guard->guard);
+  tt_assert(g);
+  tt_ptr_op(g, OP_NE, g_prev);
+  tt_mem_op(g->identity, OP_EQ, node->identity, DIGEST_LEN);
+  tt_mem_op(g->identity, OP_NE, g_prev->identity, DIGEST_LEN);
+  tt_int_op(g->is_primary, OP_EQ, 1);
+  tt_i64_op(g->last_tried_to_connect, OP_EQ, start+60);
+  tt_int_op(g->confirmed_idx, OP_EQ, -1); // not confirmed now.
+
+  /* Call this one up; watch it get confirmed. */
+  update_approx_time(start+90);
+  u = entry_guard_succeeded(&guard);
+  tt_int_op(u, OP_EQ, GUARD_USABLE_NOW);
+  tt_assert(guard);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE);
+  g = entry_guard_handle_get(guard->guard);
+  tt_assert(g);
+  tt_int_op(g->is_reachable, OP_EQ, GUARD_REACHABLE_YES);
+  tt_int_op(g->confirmed_idx, OP_EQ, 1);
 
-  /* Path bias details of the fake guard */
-  const double circ_attempts = 9;
-  const double circ_successes = 8;
-  const double successful_closed = 4;
-  const double collapsed = 2;
-  const double unusable = 0;
-  const double timeouts = 1;
+ done:
+  guard_selection_free(gs);
+  circuit_guard_state_free(guard);
+}
 
+static void
+test_entry_guard_select_for_circuit_highlevel_confirm_other(void *arg)
+{
   (void) arg;
+  const int N_PRIMARY = DFLT_N_PRIMARY_GUARDS;
+
+  /* At the start, we have no confirmed guards.  We'll mark the primary guards
+   * down, then confirm something else.  As soon as we do, it should become
+   * primary, and we should get it next time. */
+
+  time_t start = approx_time();
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  circuit_guard_state_t *guard = NULL;
+  int i, r;
+  const node_t *node = NULL;
+  guard_usable_t u;
+
+  /* Declare that we're on the internet. */
+  entry_guards_note_internet_connectivity(gs);
+
+  /* Primary guards are down! */
+  for (i = 0; i < N_PRIMARY; ++i) {
+    r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                     &node, &guard);
+    tt_assert(node);
+    tt_assert(guard);
+    tt_int_op(r, OP_EQ, 0);
+    tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+    entry_guard_failed(&guard);
+    circuit_guard_state_free(guard);
+    guard = NULL;
+    node = NULL;
+  }
 
-  /* The global entry guards smartlist should be empty now. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ, 0);
+  /* Next guard should be non-primary. */
+  node = NULL;
+  r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                   &node, &guard);
+  tt_assert(node);
+  tt_assert(guard);
+  tt_int_op(r, OP_EQ, 0);
+  entry_guard_t *g = entry_guard_handle_get(guard->guard);
+  tt_assert(g);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+  tt_int_op(g->confirmed_idx, OP_EQ, -1);
+  tt_int_op(g->is_primary, OP_EQ, 0);
+  tt_int_op(g->is_pending, OP_EQ, 1);
+  (void)start;
+
+  u = entry_guard_succeeded(&guard);
+  /* We're on the internet (by fiat), so this guard will get called "confirmed"
+   * and should immediately become primary.
+   */
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE);
+  tt_assert(u == GUARD_USABLE_NOW);
+  tt_int_op(g->confirmed_idx, OP_EQ, 0);
+  tt_int_op(g->is_primary, OP_EQ, 1);
+  tt_int_op(g->is_pending, OP_EQ, 0);
 
-  { /* Prepare the state entry */
+ done:
+  guard_selection_free(gs);
+  circuit_guard_state_free(guard);
+}
 
-    /* Prepare the smartlist to hold the key/value of each line */
-    smartlist_t *state_line = smartlist_new();
-    smartlist_add_asprintf(state_line, "EntryGuard");
-    smartlist_add_asprintf(state_line,
-             "givethanks B29D536DD1752D542E1FBB3C9CE4449D51298212 NoDirCache");
-    smartlist_add(entry_state_lines, state_line);
+static void
+test_entry_guard_select_for_circuit_highlevel_primary_retry(void *arg)
+{
+  (void) arg;
+  const int N_PRIMARY = DFLT_N_PRIMARY_GUARDS;
+
+  /* At the start, we have no confirmed guards.  We'll mark the primary guards
+   * down, then confirm something else.  As soon as we do, it should become
+   * primary, and we should get it next time. */
+
+  time_t start = approx_time();
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  circuit_guard_state_t *guard = NULL, *guard2 = NULL;
+  int i, r;
+  const node_t *node = NULL;
+  entry_guard_t *g;
+  guard_usable_t u;
+
+  /* Declare that we're on the internet. */
+  entry_guards_note_internet_connectivity(gs);
+
+  /* Make primary guards confirmed (so they won't be superseded by a later
+   * guard), then mark them down. */
+  for (i = 0; i < N_PRIMARY; ++i) {
+    r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                     &node, &guard);
+    tt_assert(node);
+    tt_assert(guard);
+    tt_int_op(r, OP_EQ, 0);
+    tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+    g = entry_guard_handle_get(guard->guard);
+    make_guard_confirmed(gs, g);
+    tt_int_op(g->is_primary, OP_EQ, 1);
+    entry_guard_failed(&guard);
+    circuit_guard_state_free(guard);
+    tt_int_op(g->is_reachable, OP_EQ, GUARD_REACHABLE_NO);
+    guard = NULL;
+    node = NULL;
+  }
 
-    state_line = smartlist_new();
-    smartlist_add_asprintf(state_line, "EntryGuardAddedBy");
-    smartlist_add_asprintf(state_line,
-      "B29D536DD1752D542E1FBB3C9CE4449D51298212 0.2.5.3-alpha-dev "
-                           "%s", get_yesterday_date_str());
-    smartlist_add(entry_state_lines, state_line);
+  /* Get another guard that we might try. */
+  r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                   &node, &guard);
+  tt_assert(node);
+  tt_assert(guard);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+  g = entry_guard_handle_get(guard->guard);
+  tt_int_op(g->is_primary, OP_EQ, 0);
+
+  tt_assert(entry_guards_all_primary_guards_are_down(gs));
+
+  /* And an hour has passed ... */
+  update_approx_time(start + 3600);
+
+  /* Say that guard has succeeded! */
+  u = entry_guard_succeeded(&guard);
+  tt_int_op(u, OP_EQ, GUARD_MAYBE_USABLE_LATER);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD);
+  g = entry_guard_handle_get(guard->guard);
+
+  /* The primary guards should have been marked up! */
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, pg, {
+    tt_int_op(pg->is_primary, OP_EQ, 1);
+    tt_ptr_op(g, OP_NE, pg);
+    tt_int_op(pg->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+  });
+
+  /* Have a circuit to a primary guard succeed. */
+  r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                   &node, &guard2);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(guard2->state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  u = entry_guard_succeeded(&guard2);
+  tt_assert(u == GUARD_USABLE_NOW);
+  tt_int_op(guard2->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE);
+
+  tt_assert(! entry_guards_all_primary_guards_are_down(gs));
 
-    state_line = smartlist_new();
-    smartlist_add_asprintf(state_line, "EntryGuardUnlistedSince");
-    smartlist_add_asprintf(state_line, "2014-06-08 16:16:50");
-    smartlist_add(entry_state_lines, state_line);
+ done:
+  guard_selection_free(gs);
+  circuit_guard_state_free(guard);
+  circuit_guard_state_free(guard2);
+}
 
-    state_line = smartlist_new();
-    smartlist_add_asprintf(state_line, "EntryGuardPathBias");
-    smartlist_add_asprintf(state_line, "%f %f %f %f %f %f",
-                           circ_attempts, circ_successes, successful_closed,
-                           collapsed, unusable, timeouts);
-    smartlist_add(entry_state_lines, state_line);
+static void
+test_entry_guard_select_and_cancel(void *arg)
+{
+  (void) arg;
+  const int N_PRIMARY = DFLT_N_PRIMARY_GUARDS;
+  int i,r;
+  const node_t *node = NULL;
+  circuit_guard_state_t *guard;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  entry_guard_t *g;
+
+  /* Once more, we mark all the primary guards down. */
+  entry_guards_note_internet_connectivity(gs);
+  for (i = 0; i < N_PRIMARY; ++i) {
+    r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                     &node, &guard);
+    tt_int_op(r, OP_EQ, 0);
+    tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+    g = entry_guard_handle_get(guard->guard);
+    tt_int_op(g->is_primary, OP_EQ, 1);
+    tt_int_op(g->is_pending, OP_EQ, 0);
+    make_guard_confirmed(gs, g);
+    entry_guard_failed(&guard);
+    circuit_guard_state_free(guard);
+    guard = NULL;
+    node = NULL;
   }
 
-  /* Inject our lines in the state */
-  state_insert_entry_guard_helper(state, entry_state_lines);
+  tt_assert(entry_guards_all_primary_guards_are_down(gs));
+
+  /* Now get another guard we could try... */
+  r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                   &node, &guard);
+  tt_assert(node);
+  tt_assert(guard);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+  g = entry_guard_handle_get(guard->guard);
+  tt_int_op(g->is_primary, OP_EQ, 0);
+  tt_int_op(g->is_pending, OP_EQ, 1);
+
+  /* Whoops! We should never have asked for this guard. Cancel the request! */
+  entry_guard_cancel(&guard);
+  tt_ptr_op(guard, OP_EQ, NULL);
+  tt_int_op(g->is_primary, OP_EQ, 0);
+  tt_int_op(g->is_pending, OP_EQ, 0);
 
-  /* Parse state */
-  retval = entry_guards_parse_state(state, 1, &msg);
-  tt_int_op(retval, OP_GE, 0);
+ done:
+  guard_selection_free(gs);
+  circuit_guard_state_free(guard);
+}
 
-  /* Test that the guard was registered */
-  all_entry_guards = get_entry_guards();
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ, 1);
+static void
+test_entry_guard_drop_guards(void *arg)
+{
+  (void) arg;
+  int r;
+  const node_t *node = NULL;
+  circuit_guard_state_t *guard;
+  guard_selection_t *gs = get_guard_selection_info();
+
+  // Pick a guard, to get things set up.
+  r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                   &node, &guard);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_GE,
+            DFLT_MIN_FILTERED_SAMPLE_SIZE);
+  tt_ptr_op(gs, OP_EQ, get_guard_selection_info());
+
+  // Drop all the guards!  (This is a bad idea....)
+  remove_all_entry_guards_for_guard_selection(gs);
+  gs = get_guard_selection_info();
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, 0);
+  tt_int_op(smartlist_len(gs->primary_entry_guards), OP_EQ, 0);
+  tt_int_op(smartlist_len(gs->confirmed_entry_guards), OP_EQ, 0);
 
-  { /* Test the path bias of this guard */
-    const entry_guard_t *e = smartlist_get(all_entry_guards, 0);
+ done:
+  circuit_guard_state_free(guard);
+  guard_selection_free(gs);
+}
 
-    tt_assert(!e->is_dir_cache);
-    tt_assert(!e->can_retry);
+/* Unit test setup function: Create a fake network, and set everything up
+ * for testing the upgrade-a-waiting-circuit code. */
+typedef struct {
+  guard_selection_t *gs;
+  time_t start;
+  circuit_guard_state_t *guard1_state;
+  circuit_guard_state_t *guard2_state;
+  entry_guard_t *guard1;
+  entry_guard_t *guard2;
+  origin_circuit_t *circ1;
+  origin_circuit_t *circ2;
+  smartlist_t *all_origin_circuits;
+} upgrade_circuits_data_t;
+static void *
+upgrade_circuits_setup(const struct testcase_t *testcase)
+{
+  upgrade_circuits_data_t *data = tor_malloc_zero(sizeof(*data));
+  guard_selection_t *gs = data->gs =
+    guard_selection_new("default", GS_TYPE_NORMAL);
+  circuit_guard_state_t *guard;
+  const node_t *node;
+  entry_guard_t *g;
+  int i;
+  const int N_PRIMARY = DFLT_N_PRIMARY_GUARDS;
+  const char *argument = testcase->setup_data;
+  const int make_circ1_succeed = strstr(argument, "c1-done") != NULL;
+  const int make_circ2_succeed = strstr(argument, "c2-done") != NULL;
+
+  big_fake_network_setup(testcase);
+
+  /* We're going to set things up in a state where a circuit will be ready to
+   * be upgraded.  Each test can make a single change (or not) that should
+   * block the upgrade.
+   */
+
+  /* First, make all the primary guards confirmed, and down. */
+  data->start = approx_time();
+  entry_guards_note_internet_connectivity(gs);
+  for (i = 0; i < N_PRIMARY; ++i) {
+    entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &node, &guard);
+    g = entry_guard_handle_get(guard->guard);
+    make_guard_confirmed(gs, g);
+    entry_guard_failed(&guard);
+    circuit_guard_state_free(guard);
+  }
 
-    /* XXX tt_double_op doesn't support equality. Cast to int for now. */
-    tt_int_op((int)e->circ_attempts, OP_EQ, (int)circ_attempts);
-    tt_int_op((int)e->circ_successes, OP_EQ, (int)circ_successes);
-    tt_int_op((int)e->successful_circuits_closed, OP_EQ,
-              (int)successful_closed);
-    tt_int_op((int)e->timeouts, OP_EQ, (int)timeouts);
-    tt_int_op((int)e->collapsed_circuits, OP_EQ, (int)collapsed);
-    tt_int_op((int)e->unusable_circuits, OP_EQ, (int)unusable);
+  /* Grab another couple of guards */
+  data->all_origin_circuits = smartlist_new();
+
+  update_approx_time(data->start + 27);
+  entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                               &node, &data->guard1_state);
+  origin_circuit_t *circ;
+  data->circ1 = circ = origin_circuit_new();
+  circ->base_.purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  circ->guard_state = data->guard1_state;
+  smartlist_add(data->all_origin_circuits, circ);
+
+  update_approx_time(data->start + 30);
+  entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                               &node, &data->guard2_state);
+  data->circ2 = circ = origin_circuit_new();
+  circ->base_.purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  circ->guard_state = data->guard2_state;
+  smartlist_add(data->all_origin_circuits, circ);
+
+  data->guard1 = entry_guard_handle_get(data->guard1_state->guard);
+  data->guard2 = entry_guard_handle_get(data->guard2_state->guard);
+  tor_assert(data->guard1 != data->guard2);
+  tor_assert(data->guard1_state->state ==
+             GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+  tor_assert(data->guard2_state->state ==
+             GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+
+  guard_usable_t r;
+  update_approx_time(data->start + 32);
+  if (make_circ1_succeed) {
+    r = entry_guard_succeeded(&data->guard1_state);
+    tor_assert(r == GUARD_MAYBE_USABLE_LATER);
+    tor_assert(data->guard1_state->state ==
+               GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD);
+  }
+  update_approx_time(data->start + 33);
+  if (make_circ2_succeed) {
+    r = entry_guard_succeeded(&data->guard2_state);
+    tor_assert(r == GUARD_MAYBE_USABLE_LATER);
+    tor_assert(data->guard2_state->state ==
+               GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD);
   }
 
- done:
-  or_state_free(state);
-  state_lines_free(entry_state_lines);
-  tor_free(msg);
+  return data;
+}
+static int
+upgrade_circuits_cleanup(const struct testcase_t *testcase, void *ptr)
+{
+  upgrade_circuits_data_t *data = ptr;
+  // circuit_guard_state_free(data->guard1_state); // held in circ1
+  // circuit_guard_state_free(data->guard2_state); // held in circ2
+  guard_selection_free(data->gs);
+  smartlist_free(data->all_origin_circuits);
+  circuit_free_(TO_CIRCUIT(data->circ1));
+  circuit_free_(TO_CIRCUIT(data->circ2));
+  tor_free(data);
+  return big_fake_network_cleanup(testcase, NULL);
 }
 
-/* Simple test of entry_guards_set_from_config() by specifying a
-   particular EntryNode and making sure it gets picked. */
 static void
-test_entry_guards_set_from_config(void *arg)
+test_entry_guard_upgrade_a_circuit(void *arg)
 {
-  or_options_t *options = get_options_mutable();
-  const smartlist_t *all_entry_guards = get_entry_guards();
-  const char *entrynodes_str = "test003r";
-  const node_t *chosen_entry = NULL;
-  int retval;
+  upgrade_circuits_data_t *data = arg;
 
-  (void) arg;
+  /* This is the easy case: we have no COMPLETED circuits, all the
+   * primary guards are down, we have two WAITING circuits: one will
+   * get upgraded to COMPLETED!  (The one that started first.)
+   */
 
-  /* Prase EntryNodes as a routerset. */
-  options->EntryNodes = routerset_new();
-  retval = routerset_parse(options->EntryNodes,
-                           entrynodes_str,
-                           "test_entrynodes");
-  tt_int_op(retval, OP_GE, 0);
+  smartlist_t *result = smartlist_new();
+  int r;
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(smartlist_len(result), OP_EQ, 1);
+  origin_circuit_t *oc = smartlist_get(result, 0);
 
-  /* Read nodes from EntryNodes */
-  entry_guards_set_from_config(options);
+  /* circ1 was started first, so we'll get told to ugrade it... */
+  tt_ptr_op(oc, OP_EQ, data->circ1);
 
-  /* Test that only one guard was added. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ, 1);
+  /* And the guard state should be complete */
+  tt_ptr_op(data->guard1_state, OP_NE, NULL);
+  tt_int_op(data->guard1_state->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE);
 
-  /* Make sure it was the guard we specified. */
-  chosen_entry = choose_random_entry(NULL);
-  tt_str_op(chosen_entry->ri->nickname, OP_EQ, entrynodes_str);
+ done:
+  smartlist_free(result);
+}
+
+static void
+test_entry_guard_upgrade_blocked_by_live_primary_guards(void *arg)
+{
+  upgrade_circuits_data_t *data = arg;
+
+  /* If any primary guards might be up, we can't upgrade any waiting
+   * circuits.
+   */
+  mark_primary_guards_maybe_reachable(data->gs);
+
+  smartlist_t *result = smartlist_new();
+  int r;
+  setup_capture_of_logs(LOG_DEBUG);
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(result), OP_EQ, 0);
+  expect_log_msg_containing("not all primary guards were definitely down.");
 
  done:
-  routerset_free(options->EntryNodes);
+  teardown_capture_of_logs();
+  smartlist_free(result);
 }
 
 static void
-test_entry_is_time_to_retry(void *arg)
+test_entry_guard_upgrade_blocked_by_lack_of_waiting_circuits(void *arg)
 {
-  entry_guard_t *test_guard;
-  time_t now;
-  int retval;
-  (void)arg;
+  upgrade_circuits_data_t *data = arg;
+
+  /* If no circuits are waiting, we can't upgrade anything.  (The test
+   * setup in this case was told not to make any of the circuits "waiting".)
+   */
+  smartlist_t *result = smartlist_new();
+  int r;
+  setup_capture_of_logs(LOG_DEBUG);
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(result), OP_EQ, 0);
+  expect_log_msg_containing("Considered upgrading guard-stalled circuits, "
+                            "but didn't find any.");
 
-  now = time(NULL);
+ done:
+  teardown_capture_of_logs();
+  smartlist_free(result);
+}
 
-  test_guard = tor_malloc_zero(sizeof(entry_guard_t));
+static void
+test_entry_guard_upgrade_blocked_by_better_circ_complete(void *arg)
+{
+  upgrade_circuits_data_t *data = arg;
+
+  /* We'll run through the logic of upgrade_a_circuit below...
+   * and then try again to make sure that circ2 isn't also upgraded.
+   */
+
+  smartlist_t *result = smartlist_new();
+  int r;
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(smartlist_len(result), OP_EQ, 1);
+  origin_circuit_t *oc = smartlist_get(result, 0);
+  tt_ptr_op(oc, OP_EQ, data->circ1);
+  tt_ptr_op(data->guard1_state, OP_NE, NULL);
+  tt_int_op(data->guard1_state->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE);
+
+  /* Now, try again. Make sure that circ2 isn't upgraded. */
+  smartlist_clear(result);
+  setup_capture_of_logs(LOG_DEBUG);
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(result), OP_EQ, 0);
+  expect_log_msg_containing("At least one complete circuit had higher "
+                            "priority, so not upgrading.");
 
-  test_guard->last_attempted = now - 10;
-  test_guard->unreachable_since = now - 1;
+ done:
+  teardown_capture_of_logs();
+  smartlist_free(result);
+}
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+static void
+test_entry_guard_upgrade_not_blocked_by_restricted_circ_complete(void *arg)
+{
+  upgrade_circuits_data_t *data = arg;
+
+  /* Once more, let circ1 become complete. But this time, we'll claim
+   * that circ2 was restricted to not use the same guard as circ1. */
+  data->guard2_state->restrictions =
+    guard_create_exit_restriction((uint8_t*)data->guard1->identity);
+
+  smartlist_t *result = smartlist_new();
+  int r;
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(smartlist_len(result), OP_EQ, 1);
+  origin_circuit_t *oc = smartlist_get(result, 0);
+  tt_ptr_op(oc, OP_EQ, data->circ1);
+  tt_ptr_op(data->guard1_state, OP_NE, NULL);
+  tt_int_op(data->guard1_state->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE);
+
+  /* Now, we try again. Since circ2 has a restriction that circ1 doesn't obey,
+   * circ2 _is_ eligible for upgrade. */
+  smartlist_clear(result);
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(smartlist_len(result), OP_EQ, 1);
+  origin_circuit_t *oc2 = smartlist_get(result, 0);
+  tt_ptr_op(oc2, OP_EQ, data->circ2);
 
-  test_guard->unreachable_since = now - (6*60*60 - 1);
-  test_guard->last_attempted = now - (60*60 + 1);
+ done:
+  smartlist_free(result);
+}
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+static void
+test_entry_guard_upgrade_not_blocked_by_worse_circ_complete(void *arg)
+{
+  upgrade_circuits_data_t *data = arg;
+  smartlist_t *result = smartlist_new();
+  /* here we manually make circ2 COMPLETE, and make sure that circ1
+   * gets made complete anyway, since guard1 has higher priority
+   */
+  update_approx_time(data->start + 300);
+  data->guard2_state->state = GUARD_CIRC_STATE_COMPLETE;
+  data->guard2_state->state_set_at = approx_time();
+  update_approx_time(data->start + 301);
+
+  /* Now, try again. Make sure that circ1 is approved. */
+  int r;
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(smartlist_len(result), OP_EQ, 1);
+  origin_circuit_t *oc = smartlist_get(result, 0);
+  tt_ptr_op(oc, OP_EQ, data->circ1);
 
-  test_guard->last_attempted = now - (60*60 - 1);
+ done:
+  smartlist_free(result);
+}
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,0);
+static void
+test_entry_guard_upgrade_blocked_by_better_circ_pending(void *arg)
+{
+  upgrade_circuits_data_t *data = arg;
+
+  /* circ2 is done, but circ1 is still pending. Since circ1 is better,
+   * we won't upgrade circ2. */
+
+  /* XXXX Prop271 -- this is a kludge.  I'm making sure circ1 _is_ better,
+   * by messing with the guards' confirmed_idx */
+  make_guard_confirmed(data->gs, data->guard1);
+  {
+    int tmp;
+    tmp = data->guard1->confirmed_idx;
+    data->guard1->confirmed_idx = data->guard2->confirmed_idx;
+    data->guard2->confirmed_idx = tmp;
+  }
 
-  test_guard->unreachable_since = now - (6*60*60 + 1);
-  test_guard->last_attempted = now - (4*60*60 + 1);
+  smartlist_t *result = smartlist_new();
+  setup_capture_of_logs(LOG_DEBUG);
+  int r;
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(result), OP_EQ, 0);
+  expect_log_msg_containing("but 1 pending circuit(s) had higher guard "
+                            "priority, so not upgrading.");
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+ done:
+  teardown_capture_of_logs();
+  smartlist_free(result);
+}
 
-  test_guard->unreachable_since = now - (3*24*60*60 - 1);
-  test_guard->last_attempted = now - (4*60*60 + 1);
+static void
+test_entry_guard_upgrade_not_blocked_by_restricted_circ_pending(void *arg)
+{
+  upgrade_circuits_data_t *data = arg;
+  /* circ2 is done, but circ1 is still pending. But when there is a
+     restriction on circ2 that circ1 can't satisfy, circ1 can't block
+     circ2. */
+
+  /* XXXX Prop271 -- this is a kludge.  I'm making sure circ1 _is_ better,
+   * by messing with the guards' confirmed_idx */
+  make_guard_confirmed(data->gs, data->guard1);
+  {
+    int tmp;
+    tmp = data->guard1->confirmed_idx;
+    data->guard1->confirmed_idx = data->guard2->confirmed_idx;
+    data->guard2->confirmed_idx = tmp;
+  }
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+  data->guard2_state->restrictions =
+    guard_create_exit_restriction((uint8_t*)data->guard1->identity);
 
-  test_guard->unreachable_since = now - (3*24*60*60 + 1);
-  test_guard->last_attempted = now - (18*60*60 + 1);
+  smartlist_t *result = smartlist_new();
+  int r;
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(smartlist_len(result), OP_EQ, 1);
+  origin_circuit_t *oc = smartlist_get(result, 0);
+  tt_ptr_op(oc, OP_EQ, data->circ2);
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+ done:
+  smartlist_free(result);
+}
 
-  test_guard->unreachable_since = now - (7*24*60*60 - 1);
-  test_guard->last_attempted = now - (18*60*60 + 1);
+static void
+test_entry_guard_upgrade_not_blocked_by_worse_circ_pending(void *arg)
+{
+  upgrade_circuits_data_t *data = arg;
+
+  /* circ1 is done, but circ2 is still pending. Since circ1 is better,
+   * we will upgrade it. */
+  smartlist_t *result = smartlist_new();
+  int r;
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(smartlist_len(result), OP_EQ, 1);
+  origin_circuit_t *oc = smartlist_get(result, 0);
+  tt_ptr_op(oc, OP_EQ, data->circ1);
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+ done:
+  smartlist_free(result);
+}
 
-  test_guard->last_attempted = now - (18*60*60 - 1);
+static void
+test_entry_guard_should_expire_waiting(void *arg)
+{
+  (void)arg;
+  circuit_guard_state_t *fake_state = tor_malloc_zero(sizeof(*fake_state));
+  /* We'll leave "guard" unset -- it won't matter here. */
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,0);
+  /* No state? Can't expire. */
+  tt_assert(! entry_guard_state_should_expire(NULL));
 
-  test_guard->unreachable_since = now - (7*24*60*60 + 1);
-  test_guard->last_attempted = now - (36*60*60 + 1);
+  /* Let's try one that expires. */
+  fake_state->state = GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD;
+  fake_state->state_set_at =
+    approx_time() - DFLT_NONPRIMARY_GUARD_IDLE_TIMEOUT - 1;
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+  tt_assert(entry_guard_state_should_expire(fake_state));
 
-  test_guard->unreachable_since = now - (7*24*60*60 + 1);
-  test_guard->last_attempted = now - (36*60*60 + 1);
+  /* But it wouldn't expire if we changed the state. */
+  fake_state->state = GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD;
+  tt_assert(! entry_guard_state_should_expire(fake_state));
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+  /* And it wouldn't have expired a few seconds ago. */
+  fake_state->state = GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD;
+  fake_state->state_set_at =
+    approx_time() - DFLT_NONPRIMARY_GUARD_IDLE_TIMEOUT + 5;
+  tt_assert(! entry_guard_state_should_expire(fake_state));
 
  done:
-  tor_free(test_guard);
+  tor_free(fake_state);
 }
 
-/** XXX Do some tests that entry_is_live() */
+/** Test that the number of primary guards can be controlled using torrc */
 static void
-test_entry_is_live(void *arg)
+test_entry_guard_number_of_primaries(void *arg)
 {
-  smartlist_t *our_nodelist = NULL;
-  const smartlist_t *all_entry_guards = get_entry_guards();
-  const node_t *test_node = NULL;
-  const entry_guard_t *test_entry = NULL;
-  const char *msg;
-  int which_node;
-
   (void) arg;
 
-  /* The global entry guards smartlist should be empty now. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ, 0);
+  /* Get default value */
+  tt_int_op(get_n_primary_guards(), OP_EQ, DFLT_N_PRIMARY_GUARDS);
 
-  /* Walk the nodelist and add all nodes as entry guards. */
-  our_nodelist = nodelist_get_list();
-  tt_int_op(smartlist_len(our_nodelist), OP_EQ, HELPER_NUMBER_OF_DESCRIPTORS);
+  /* Set number of primaries using torrc */
+  get_options_mutable()->NumPrimaryGuards = 42;
+  tt_int_op(get_n_primary_guards(), OP_EQ, 42);
 
-  SMARTLIST_FOREACH_BEGIN(our_nodelist, const node_t *, node) {
-    const node_t *node_tmp;
-    node_tmp = add_an_entry_guard(node, 0, 1, 0, 0);
-    tt_assert(node_tmp);
+ done:
+  ;
+}
 
-    tt_int_op(node->is_stable, OP_EQ, 0);
-    tt_int_op(node->is_fast, OP_EQ, 0);
-  } SMARTLIST_FOREACH_END(node);
+static void
+mock_directory_initiate_request(directory_request_t *req)
+{
+  if (req->guard_state) {
+    circuit_guard_state_free(req->guard_state);
+  }
+}
 
-  /* Make sure the nodes were added as entry guards. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ,
-            HELPER_NUMBER_OF_DESCRIPTORS);
+static networkstatus_t *mock_ns_val = NULL;
+static networkstatus_t *
+mock_ns_get_by_flavor(consensus_flavor_t f)
+{
+  (void)f;
+  return mock_ns_val;
+}
 
-  /* Now get a random test entry that we will use for this unit test. */
-  which_node = 3;  /* (chosen by fair dice roll) */
-  test_entry = smartlist_get(all_entry_guards, which_node);
+/** Test that when we fetch microdescriptors we skip guards that have
+ *  previously failed to serve us needed microdescriptors. */
+static void
+test_entry_guard_outdated_dirserver_exclusion(void *arg)
+{
+  int retval;
+  response_handler_args_t *args = NULL;
+  dir_connection_t *conn = NULL;
+  (void) arg;
+
+  /* Test prep: Make a new guard selection */
+  guard_selection_t *gs = get_guard_selection_by_name("default",
+                                                      GS_TYPE_NORMAL, 1);
 
-  /* Let's do some entry_is_live() tests! */
+  /* ... we want to use entry guards */
+  or_options_t *options = get_options_mutable();
+  options->UseEntryGuards = 1;
+  options->UseBridges = 0;
+
+  /* ... prepare some md digests we want to download in the future */
+  smartlist_t *digests = smartlist_new();
+  const char *prose = "unhurried and wise, we perceive.";
+  for (int i = 0; i < 20; i++) {
+    smartlist_add(digests, (char*)prose);
+  }
 
-  /* Require the node to be stable, but it's not. Should fail.
-     Also enable 'assume_reachable' because why not. */
-  test_node = entry_is_live(test_entry,
-                            ENTRY_NEED_UPTIME | ENTRY_ASSUME_REACHABLE,
-                            &msg);
-  tt_assert(!test_node);
+  tt_int_op(smartlist_len(digests), OP_EQ, 20);
+
+  /* ... now mock some functions */
+  mock_ns_val = tor_malloc_zero(sizeof(networkstatus_t));
+  MOCK(networkstatus_get_latest_consensus_by_flavor, mock_ns_get_by_flavor);
+  MOCK(directory_initiate_request, mock_directory_initiate_request);
+
+  /* Test logic:
+   *  0. Create a proper guard set and primary guard list.
+   *  1. Pretend to fail microdescriptor fetches from all the primary guards.
+   *  2. Order another microdescriptor fetch and make sure that primary guards
+   *     get skipped since they failed previous fetches.
+   */
+
+  { /* Setup primary guard list */
+    int i;
+    entry_guards_update_primary(gs);
+    for (i = 0; i < DFLT_N_PRIMARY_GUARDS; ++i) {
+      entry_guard_t *guard = smartlist_get(gs->sampled_entry_guards, i);
+      make_guard_confirmed(gs, guard);
+    }
+    entry_guards_update_primary(gs);
+  }
 
-  /* Require the node to be fast, but it's not. Should fail. */
-  test_node = entry_is_live(test_entry,
-                            ENTRY_NEED_CAPACITY | ENTRY_ASSUME_REACHABLE,
-                            &msg);
-  tt_assert(!test_node);
+  {
+    /* Fail microdesc fetches with all the primary guards */
+    args = tor_malloc_zero(sizeof(response_handler_args_t));
+    args->status_code = 404;
+    args->reason = NULL;
+    args->body = NULL;
+    args->body_len = 0;
+
+    conn = tor_malloc_zero(sizeof(dir_connection_t));
+    conn->requested_resource = tor_strdup("d/jlinblackorigami");
+    conn->base_.purpose = DIR_PURPOSE_FETCH_MICRODESC;
+
+    /* Pretend to fail fetches with all primary guards */
+    SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards,const entry_guard_t *,g) {
+      memcpy(conn->identity_digest, g->identity, DIGEST_LEN);
+
+      retval = handle_response_fetch_microdesc(conn, args);
+      tt_int_op(retval, OP_EQ, 0);
+    } SMARTLIST_FOREACH_END(g);
+  }
 
-  /* Don't impose any restrictions on the node. Should succeed. */
-  test_node = entry_is_live(test_entry, 0, &msg);
-  tt_assert(test_node);
-  tt_ptr_op(test_node, OP_EQ, node_get_by_id(test_entry->identity));
+  {
+    /* Now order the final md download */
+    setup_full_capture_of_logs(LOG_INFO);
+    initiate_descriptor_downloads(NULL, DIR_PURPOSE_FETCH_MICRODESC,
+                                  digests, 3, 7, 0);
 
-  /* Require descriptor for this node. It has one so it should succeed. */
-  test_node = entry_is_live(test_entry, ENTRY_NEED_DESCRIPTOR, &msg);
-  tt_assert(test_node);
-  tt_ptr_op(test_node, OP_EQ, node_get_by_id(test_entry->identity));
+    /* ... and check that because we failed to fetch microdescs from all our
+     * primaries, we didn't end up selecting a primary for fetching dir info */
+    expect_log_msg_containing("No primary or confirmed guards available.");
+    teardown_capture_of_logs();
+  }
 
  done:
-  ; /* XXX */
+  UNMOCK(networkstatus_get_latest_consensus_by_flavor);
+  UNMOCK(directory_initiate_request);
+  smartlist_free(digests);
+  tor_free(mock_ns_val);
+  tor_free(args);
+  if (conn) {
+    tor_free(conn->requested_resource);
+    tor_free(conn);
+  }
 }
 
-#define TEST_IPV4_ADDR "123.45.67.89"
-#define TEST_IPV6_ADDR "[1234:5678:90ab:cdef::]"
+/** Test helper to extend the <b>oc</b> circuit path <b>n</b> times and then
+ *  ensure that the circuit is now complete. */
+static void
+helper_extend_circuit_path_n_times(origin_circuit_t *oc, int n)
+{
+  int retval;
+  int i;
+
+  /* Extend path n times */
+  for (i = 0 ; i < n ; i++) {
+    retval = onion_extend_cpath(oc);
+    tt_int_op(retval, OP_EQ, 0);
+    tt_int_op(circuit_get_cpath_len(oc), OP_EQ, i+1);
+  }
+
+  /* Now do it one last time and see that circ is complete */
+  retval = onion_extend_cpath(oc);
+  tt_int_op(retval, OP_EQ, 1);
+
+ done:
+  ;
+}
 
+/** Test for basic Tor path selection. Makes sure we build 3-hop circuits. */
 static void
-test_node_preferred_orport(void *arg)
+test_entry_guard_basic_path_selection(void *arg)
 {
-  (void)arg;
-  tor_addr_t ipv4_addr;
-  const uint16_t ipv4_port = 4444;
-  tor_addr_t ipv6_addr;
-  const uint16_t ipv6_port = 6666;
-  routerinfo_t node_ri;
-  node_t node;
-  tor_addr_port_t ap;
+  (void) arg;
 
-  /* Setup options */
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  /* We don't test ClientPreferIPv6ORPort here, because it's used in
-   * nodelist_set_consensus to setup node.ipv6_preferred, which we set
-   * directly. */
-  MOCK(get_options, mock_get_options);
+  int retval;
 
-  /* Setup IP addresses */
-  tor_addr_parse(&ipv4_addr, TEST_IPV4_ADDR);
-  tor_addr_parse(&ipv6_addr, TEST_IPV6_ADDR);
+  /* Enable entry guards */
+  or_options_t *options = get_options_mutable();
+  options->UseEntryGuards = 1;
 
-  /* Setup node_ri */
-  memset(&node_ri, 0, sizeof(node_ri));
-  node_ri.addr = tor_addr_to_ipv4h(&ipv4_addr);
-  node_ri.or_port = ipv4_port;
-  tor_addr_copy(&node_ri.ipv6_addr, &ipv6_addr);
-  node_ri.ipv6_orport = ipv6_port;
+  /* disables /16 check since all nodes have the same addr... */
+  options->EnforceDistinctSubnets = 0;
 
-  /* Setup node */
-  memset(&node, 0, sizeof(node));
-  node.ri = &node_ri;
+  /* Create our circuit */
+  circuit_t *circ = dummy_origin_circuit_new(30);
+  origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ);
+  oc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
 
-  /* Check the preferred address is IPv4 if we're only using IPv4, regardless
-   * of whether we prefer it or not */
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 0;
-  node.ipv6_preferred = 0;
-  node_get_pref_orport(&node, &ap);
-  tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
-  tt_assert(ap.port == ipv4_port);
+  /* First pick the exit and pin it on the build_state */
+  retval = onion_pick_cpath_exit(oc, NULL, 0);
+  tt_int_op(retval, OP_EQ, 0);
 
-  node.ipv6_preferred = 1;
-  node_get_pref_orport(&node, &ap);
-  tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
-  tt_assert(ap.port == ipv4_port);
+  /* Extend path 3 times. First we pick guard, then middle, then exit. */
+  helper_extend_circuit_path_n_times(oc, 3);
 
-  /* Check the preferred address is IPv4 if we're using IPv4 and IPv6, but
-   * don't prefer the IPv6 address */
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 1;
-  node.ipv6_preferred = 0;
-  node_get_pref_orport(&node, &ap);
-  tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
-  tt_assert(ap.port == ipv4_port);
+ done:
+  circuit_free_(circ);
+}
 
-  /* Check the preferred address is IPv6 if we prefer it and
-   * ClientUseIPv6 is 1, regardless of ClientUseIPv4 */
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 1;
-  node.ipv6_preferred = 1;
-  node_get_pref_orport(&node, &ap);
-  tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
-  tt_assert(ap.port == ipv6_port);
+/** Test helper to build an L2 and L3 vanguard list. The vanguard lists
+ *  produced should be completely disjoint. */
+static void
+helper_setup_vanguard_list(or_options_t *options)
+{
+  int i = 0;
+
+  /* Add some nodes to the vanguard L2 list */
+  options->HSLayer2Nodes = routerset_new();
+  for (i = 0; i < 10 ; i += 2) {
+    node_t *vanguard_node = smartlist_get(big_fake_net_nodes, i);
+    tt_assert(vanguard_node->is_possible_guard);
+    routerset_parse(options->HSLayer2Nodes, vanguard_node->rs->nickname, "l2");
+  }
+  /* also add some nodes to vanguard L3 list
+   * (L2 list and L3 list should be disjoint for this test to work) */
+  options->HSLayer3Nodes = routerset_new();
+  for (i = 10; i < 20 ; i += 2) {
+    node_t *vanguard_node = smartlist_get(big_fake_net_nodes, i);
+    tt_assert(vanguard_node->is_possible_guard);
+    routerset_parse(options->HSLayer3Nodes, vanguard_node->rs->nickname, "l3");
+  }
 
-  mocked_options.ClientUseIPv4 = 0;
-  node_get_pref_orport(&node, &ap);
-  tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
-  tt_assert(ap.port == ipv6_port);
+ done:
+  ;
+}
 
-  /* Check the preferred address is IPv6 if we don't prefer it, but
-   * ClientUseIPv4 is 0 */
-  mocked_options.ClientUseIPv4 = 0;
-  mocked_options.ClientUseIPv6 = 1;
-  node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport(&mocked_options);
-  node_get_pref_orport(&node, &ap);
-  tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
-  tt_assert(ap.port == ipv6_port);
+/** Test to ensure that vanguard path selection works properly.  Ensures that
+ *  default vanguard circuits are 4 hops, and that path selection works
+ *  correctly given the vanguard settings. */
+static void
+test_entry_guard_vanguard_path_selection(void *arg)
+{
+  (void) arg;
+
+  int retval;
+
+  /* Enable entry guards */
+  or_options_t *options = get_options_mutable();
+  options->UseEntryGuards = 1;
+
+  /* XXX disables /16 check */
+  options->EnforceDistinctSubnets = 0;
+
+  /* Setup our vanguard list */
+  helper_setup_vanguard_list(options);
+
+  /* Create our circuit */
+  circuit_t *circ = dummy_origin_circuit_new(30);
+  origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ);
+  oc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  oc->build_state->is_internal = 1;
+
+  /* Switch circuit purpose to vanguards */
+  circ->purpose = CIRCUIT_PURPOSE_HS_VANGUARDS;
+
+  /* First pick the exit and pin it on the build_state */
+  tt_int_op(oc->build_state->desired_path_len, OP_EQ, 0);
+  retval = onion_pick_cpath_exit(oc, NULL, 0);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Ensure that vanguards make 4-hop circuits by default */
+  tt_int_op(oc->build_state->desired_path_len, OP_EQ, 4);
+
+  /* Extend path as many times as needed to have complete circ. */
+  helper_extend_circuit_path_n_times(oc, oc->build_state->desired_path_len);
+
+  /* Test that the cpath linked list is set correctly. */
+  crypt_path_t *l1_node = oc->cpath;
+  crypt_path_t *l2_node = l1_node->next;
+  crypt_path_t *l3_node = l2_node->next;
+  crypt_path_t *l4_node = l3_node->next;
+  crypt_path_t *l1_node_again = l4_node->next;
+  tt_ptr_op(l1_node, OP_EQ, l1_node_again);
+
+  /* Test that L2 is indeed HSLayer2Node */
+  retval = routerset_contains_extendinfo(options->HSLayer2Nodes,
+                                         l2_node->extend_info);
+  tt_int_op(retval, OP_EQ, 4);
+  /* test that L3 node is _not_ contained in HSLayer2Node */
+  retval = routerset_contains_extendinfo(options->HSLayer2Nodes,
+                                         l3_node->extend_info);
+  tt_int_op(retval, OP_LT, 4);
+
+  /* Test that L3 is indeed HSLayer3Node */
+  retval = routerset_contains_extendinfo(options->HSLayer3Nodes,
+                                         l3_node->extend_info);
+  tt_int_op(retval, OP_EQ, 4);
+  /* test that L2 node is _not_ contained in HSLayer3Node */
+  retval = routerset_contains_extendinfo(options->HSLayer3Nodes,
+                                         l2_node->extend_info);
+  tt_int_op(retval, OP_LT, 4);
+
+  /* TODO: Test that L1 can be the same as exit. To test this we need start
+     enforcing EnforceDistinctSubnets again, which means that we need to give
+     each test node a different address which currently breaks some tests. */
 
  done:
-  UNMOCK(get_options);
+  circuit_free_(circ);
 }
 
-static const struct testcase_setup_t fake_network = {
-  fake_network_setup, fake_network_cleanup
+static const struct testcase_setup_t big_fake_network = {
+  big_fake_network_setup, big_fake_network_cleanup
+};
+
+static const struct testcase_setup_t upgrade_circuits = {
+  upgrade_circuits_setup, upgrade_circuits_cleanup
 };
 
+#define NO_PREFIX_TEST(name) \
+  { #name, test_ ## name, 0, NULL, NULL }
+
+#define EN_TEST_BASE(name, fork, setup, arg) \
+  { #name, test_entry_guard_ ## name, fork, setup, (void*)(arg) }
+
+#define EN_TEST(name)      EN_TEST_BASE(name, 0,       NULL, NULL)
+#define EN_TEST_FORK(name) EN_TEST_BASE(name, TT_FORK, NULL, NULL)
+
+#define BFN_TEST(name) \
+  EN_TEST_BASE(name, TT_FORK, &big_fake_network, NULL), \
+  { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \
+    &big_fake_network, (void*)("reasonably-live") }
+
+#define UPGRADE_TEST(name, arg) \
+  EN_TEST_BASE(name, TT_FORK, &upgrade_circuits, arg), \
+  { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \
+    &upgrade_circuits, (void*)(arg " reasonably-live") }
+
 struct testcase_t entrynodes_tests[] = {
-  { "entry_is_time_to_retry", test_entry_is_time_to_retry,
-    TT_FORK, NULL, NULL },
-  { "choose_random_entry_no_guards", test_choose_random_entry_no_guards,
-    TT_FORK, &fake_network, NULL },
-  { "choose_random_entry_one_possibleguard",
-    test_choose_random_entry_one_possible_guard,
-    TT_FORK, &fake_network, NULL },
-  { "populate_live_entry_guards_1guard",
-    test_populate_live_entry_guards_1guard,
-    TT_FORK, &fake_network, NULL },
-  { "populate_live_entry_guards_3guards",
-    test_populate_live_entry_guards_3guards,
-    TT_FORK, &fake_network, NULL },
-  { "entry_guards_parse_state_simple",
-    test_entry_guards_parse_state_simple,
-    TT_FORK, &fake_network, NULL },
-  { "entry_guards_parse_state_pathbias",
-    test_entry_guards_parse_state_pathbias,
-    TT_FORK, &fake_network, NULL },
-  { "entry_guards_set_from_config",
-    test_entry_guards_set_from_config,
-    TT_FORK, &fake_network, NULL },
-  { "entry_is_live",
-    test_entry_is_live,
-    TT_FORK, &fake_network, NULL },
-  { "node_preferred_orport",
-    test_node_preferred_orport,
-    0, NULL, NULL },
+  NO_PREFIX_TEST(node_preferred_orport),
+  NO_PREFIX_TEST(entry_guard_describe),
+
+  EN_TEST(randomize_time),
+  EN_TEST(encode_for_state_minimal),
+  EN_TEST(encode_for_state_maximal),
+  EN_TEST(parse_from_state_minimal),
+  EN_TEST(parse_from_state_maximal),
+  EN_TEST(parse_from_state_failure),
+  EN_TEST(parse_from_state_partial_failure),
+
+  EN_TEST_FORK(parse_from_state_full),
+  EN_TEST_FORK(parse_from_state_broken),
+  EN_TEST_FORK(get_guard_selection_by_name),
+  EN_TEST_FORK(number_of_primaries),
+
+  BFN_TEST(choose_selection_initial),
+  BFN_TEST(add_single_guard),
+  BFN_TEST(node_filter),
+  BFN_TEST(expand_sample),
+  BFN_TEST(expand_sample_small_net),
+  BFN_TEST(update_from_consensus_status),
+  BFN_TEST(update_from_consensus_repair),
+  BFN_TEST(update_from_consensus_remove),
+  BFN_TEST(confirming_guards),
+  BFN_TEST(sample_reachable_filtered),
+  BFN_TEST(sample_reachable_filtered_empty),
+  BFN_TEST(retry_unreachable),
+  BFN_TEST(manage_primary),
+
+  EN_TEST_FORK(guard_preferred),
+
+  BFN_TEST(select_for_circuit_no_confirmed),
+  BFN_TEST(select_for_circuit_confirmed),
+  BFN_TEST(select_for_circuit_highlevel_primary),
+  BFN_TEST(select_for_circuit_highlevel_confirm_other),
+  BFN_TEST(select_for_circuit_highlevel_primary_retry),
+  BFN_TEST(select_and_cancel),
+  BFN_TEST(drop_guards),
+  BFN_TEST(outdated_dirserver_exclusion),
+  BFN_TEST(basic_path_selection),
+  BFN_TEST(vanguard_path_selection),
+
+  UPGRADE_TEST(upgrade_a_circuit, "c1-done c2-done"),
+  UPGRADE_TEST(upgrade_blocked_by_live_primary_guards, "c1-done c2-done"),
+  UPGRADE_TEST(upgrade_blocked_by_lack_of_waiting_circuits, ""),
+  UPGRADE_TEST(upgrade_blocked_by_better_circ_complete, "c1-done c2-done"),
+  UPGRADE_TEST(upgrade_not_blocked_by_restricted_circ_complete,
+               "c1-done c2-done"),
+  UPGRADE_TEST(upgrade_not_blocked_by_worse_circ_complete, "c1-done c2-done"),
+  UPGRADE_TEST(upgrade_blocked_by_better_circ_pending, "c2-done"),
+  UPGRADE_TEST(upgrade_not_blocked_by_restricted_circ_pending,
+               "c2-done"),
+  UPGRADE_TEST(upgrade_not_blocked_by_worse_circ_pending, "c1-done"),
+
+  EN_TEST_FORK(should_expire_waiting),
+
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c
index 1f92780177..0c34d37a71 100644
--- a/src/test/test_extorport.c
+++ b/src/test/test_extorport.c
@@ -1,18 +1,27 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONNECTION_PRIVATE
 #define EXT_ORPORT_PRIVATE
-#define MAIN_PRIVATE
-#include "or.h"
-#include "buffers.h"
-#include "connection.h"
-#include "connection_or.h"
-#include "config.h"
-#include "control.h"
-#include "ext_orport.h"
-#include "main.h"
-#include "test.h"
+#define MAINLOOP_PRIVATE
+#include "core/or/or.h"
+#include "lib/container/buffers.h"
+#include "core/mainloop/connection.h"
+#include "core/or/connection_or.h"
+#include "app/config/config.h"
+#include "feature/control/control.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "feature/relay/ext_orport.h"
+#include "core/mainloop/mainloop.h"
+
+#include "core/or/or_connection_st.h"
+
+#include "test/test.h"
+#include "test/test_helpers.h"
+
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
 
 /* Test connection_or_remove_from_ext_or_id_map and
  * connection_or_set_ext_or_identifier */
@@ -58,11 +67,11 @@ test_ext_or_id_map(void *arg)
 
  done:
   if (c1)
-    connection_free_(TO_CONN(c1));
+    connection_free_minimal(TO_CONN(c1));
   if (c2)
-    connection_free_(TO_CONN(c2));
+    connection_free_minimal(TO_CONN(c2));
   if (c3)
-    connection_free_(TO_CONN(c3));
+    connection_free_minimal(TO_CONN(c3));
   tor_free(idp);
   tor_free(idp2);
   connection_or_clear_ext_or_id_map();
@@ -72,29 +81,13 @@ test_ext_or_id_map(void *arg)
  * writes to outbuf. */
 static void
 connection_write_to_buf_impl_replacement(const char *string, size_t len,
-                                         connection_t *conn, int zlib)
+                                         connection_t *conn, int compressed)
 {
-  (void) zlib;
+  (void) compressed;
 
   tor_assert(string);
   tor_assert(conn);
-  write_to_buf(string, len, conn->outbuf);
-}
-
-static char *
-buf_get_contents(buf_t *buf, size_t *sz_out)
-{
-  char *out;
-  *sz_out = buf_datalen(buf);
-  if (*sz_out >= ULONG_MAX)
-    return NULL; /* C'mon, really? */
-  out = tor_malloc(*sz_out + 1);
-  if (fetch_from_buf(out, (unsigned long)*sz_out, buf) != 0) {
-    tor_free(out);
-    return NULL;
-  }
-  out[*sz_out] = '\0'; /* Hopefully gratuitous. */
-  return out;
+  buf_add(conn->outbuf, string, len);
 }
 
 static void
@@ -145,7 +138,7 @@ test_ext_or_write_command(void *arg)
 
  done:
   if (c1)
-    connection_free_(TO_CONN(c1));
+    connection_free_minimal(TO_CONN(c1));
   tor_free(cp);
   tor_free(buf);
   UNMOCK(connection_write_to_buf_impl_);
@@ -399,14 +392,14 @@ handshake_start(or_connection_t *conn, int receiving)
 
 #define WRITE(s,n)                                                      \
   do {                                                                  \
-    write_to_buf((s), (n), TO_CONN(conn)->inbuf);                       \
+    buf_add(TO_CONN(conn)->inbuf, (s), (n));                           \
   } while (0)
 #define CONTAINS(s,n)                                           \
   do {                                                          \
     tt_int_op((n), OP_LE, sizeof(b));                              \
     tt_int_op(buf_datalen(TO_CONN(conn)->outbuf), OP_EQ, (n));     \
     if ((n)) {                                                  \
-      fetch_from_buf(b, (n), TO_CONN(conn)->outbuf);            \
+      buf_get_bytes(TO_CONN(conn)->outbuf, b, (n));                \
       tt_mem_op(b, OP_EQ, (s), (n));                               \
     }                                                           \
   } while (0)
@@ -455,7 +448,7 @@ test_ext_or_handshake(void *arg)
   memcpy(ext_or_auth_cookie, "Gliding wrapt in a brown mantle," , 32);
   ext_or_auth_cookie_is_set = 1;
 
-  init_connection_lists();
+  tor_init_connection_lists();
 
   conn = or_connection_new(CONN_TYPE_EXT_OR, AF_INET);
   tt_int_op(0, OP_EQ, connection_ext_or_start_auth(conn));
@@ -497,14 +490,14 @@ test_ext_or_handshake(void *arg)
            "te road There is always another ", 64);
   /* Send the wrong response. */
   WRITE("not with a bang but a whimper...", 32);
-  MOCK(control_event_bootstrap_problem, ignore_bootstrap_problem);
+  MOCK(control_event_bootstrap_prob_or, ignore_bootstrap_problem);
   tt_int_op(-1, OP_EQ, connection_ext_or_process_inbuf(conn));
   CONTAINS("\x00", 1);
   tt_assert(TO_CONN(conn)->marked_for_close);
   /* XXXX Hold-open-until-flushed. */
   close_closeable_connections();
   conn = NULL;
-  UNMOCK(control_event_bootstrap_problem);
+  UNMOCK(control_event_bootstrap_prob_or);
 
   MOCK(connection_start_reading, note_read_started);
   MOCK(connection_stop_reading, note_read_stopped);
@@ -552,26 +545,26 @@ test_ext_or_handshake(void *arg)
   do_ext_or_handshake(conn);
   /* USERADDR command with an extra NUL byte */
   WRITE("\x00\x01\x00\x0d""1.2.3.4:5678\x00", 17);
-  MOCK(control_event_bootstrap_problem, ignore_bootstrap_problem);
+  MOCK(control_event_bootstrap_prob_or, ignore_bootstrap_problem);
   tt_int_op(-1, OP_EQ, connection_ext_or_process_inbuf(conn));
   CONTAINS("", 0);
   tt_assert(TO_CONN(conn)->marked_for_close);
   close_closeable_connections();
   conn = NULL;
-  UNMOCK(control_event_bootstrap_problem);
+  UNMOCK(control_event_bootstrap_prob_or);
 
   /* Now fail the TRANSPORT command. */
   conn = or_connection_new(CONN_TYPE_EXT_OR, AF_INET);
   do_ext_or_handshake(conn);
   /* TRANSPORT command with an extra NUL byte */
   WRITE("\x00\x02\x00\x08""rfc1149\x00", 12);
-  MOCK(control_event_bootstrap_problem, ignore_bootstrap_problem);
+  MOCK(control_event_bootstrap_prob_or, ignore_bootstrap_problem);
   tt_int_op(-1, OP_EQ, connection_ext_or_process_inbuf(conn));
   CONTAINS("", 0);
   tt_assert(TO_CONN(conn)->marked_for_close);
   close_closeable_connections();
   conn = NULL;
-  UNMOCK(control_event_bootstrap_problem);
+  UNMOCK(control_event_bootstrap_prob_or);
 
   /* Now fail the TRANSPORT command. */
   conn = or_connection_new(CONN_TYPE_EXT_OR, AF_INET);
@@ -579,19 +572,19 @@ test_ext_or_handshake(void *arg)
   /* TRANSPORT command with transport name with symbols (not a
      C-identifier) */
   WRITE("\x00\x02\x00\x07""rf*1149", 11);
-  MOCK(control_event_bootstrap_problem, ignore_bootstrap_problem);
+  MOCK(control_event_bootstrap_prob_or, ignore_bootstrap_problem);
   tt_int_op(-1, OP_EQ, connection_ext_or_process_inbuf(conn));
   CONTAINS("", 0);
   tt_assert(TO_CONN(conn)->marked_for_close);
   close_closeable_connections();
   conn = NULL;
-  UNMOCK(control_event_bootstrap_problem);
+  UNMOCK(control_event_bootstrap_prob_or);
 
  done:
   UNMOCK(connection_write_to_buf_impl_);
   UNMOCK(crypto_rand);
   if (conn)
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
 #undef CONTAINS
 #undef WRITE
 }
@@ -606,4 +599,3 @@ struct testcase_t extorport_tests[] = {
   { "handshake", test_ext_or_handshake, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_geoip.c b/src/test/test_geoip.c
new file mode 100644
index 0000000000..16c566bdbc
--- /dev/null
+++ b/src/test/test_geoip.c
@@ -0,0 +1,580 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+/* These macros pull in declarations for some functions and structures that
+ * are typically file-private. */
+#define GEOIP_PRIVATE
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "lib/geoip/geoip.h"
+#include "feature/stats/geoip_stats.h"
+#include "test/test.h"
+
+  /* Record odd numbered fake-IPs using ipv6, even numbered fake-IPs
+   * using ipv4.  Since our fake geoip database is the same between
+   * ipv4 and ipv6, we should get the same result no matter which
+   * address family we pick for each IP. */
+#define SET_TEST_ADDRESS(i) do {                \
+    if ((i) & 1) {                              \
+      SET_TEST_IPV6(i);                         \
+      tor_addr_from_in6(&addr, &in6);           \
+    } else {                                    \
+      tor_addr_from_ipv4h(&addr, (uint32_t) i); \
+    }                                           \
+  } while (0)
+
+  /* Make sure that country ID actually works. */
+#define SET_TEST_IPV6(i) \
+  do {                                                          \
+    set_uint32(in6.s6_addr + 12, htonl((uint32_t) (i)));        \
+  } while (0)
+#define CHECK_COUNTRY(country, val) do {                                \
+    /* test ipv4 country lookup */                                      \
+    tt_str_op(country, OP_EQ,                                              \
+               geoip_get_country_name(geoip_get_country_by_ipv4(val))); \
+    /* test ipv6 country lookup */                                      \
+    SET_TEST_IPV6(val);                                                 \
+    tt_str_op(country, OP_EQ,                                              \
+               geoip_get_country_name(geoip_get_country_by_ipv6(&in6))); \
+  } while (0)
+
+/** Run unit tests for GeoIP code. */
+static void
+test_geoip(void *arg)
+{
+  int i, j;
+  time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */
+  char *s = NULL, *v = NULL;
+  const char *bridge_stats_1 =
+      "bridge-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+      "bridge-ips zz=24,xy=8\n"
+      "bridge-ip-versions v4=16,v6=16\n"
+      "bridge-ip-transports <OR>=24\n",
+  *dirreq_stats_1 =
+      "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+      "dirreq-v3-ips ab=8\n"
+      "dirreq-v3-reqs ab=8\n"
+      "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0,"
+          "not-modified=0,busy=0\n"
+      "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n"
+      "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n",
+  *dirreq_stats_2 =
+      "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+      "dirreq-v3-ips \n"
+      "dirreq-v3-reqs \n"
+      "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0,"
+          "not-modified=0,busy=0\n"
+      "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n"
+      "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n",
+  *dirreq_stats_3 =
+      "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+      "dirreq-v3-ips \n"
+      "dirreq-v3-reqs \n"
+      "dirreq-v3-resp ok=8,not-enough-sigs=0,unavailable=0,not-found=0,"
+          "not-modified=0,busy=0\n"
+      "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n"
+      "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n",
+  *dirreq_stats_4 =
+      "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+      "dirreq-v3-ips \n"
+      "dirreq-v3-reqs \n"
+      "dirreq-v3-resp ok=8,not-enough-sigs=0,unavailable=0,not-found=0,"
+          "not-modified=0,busy=0\n"
+      "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n"
+      "dirreq-v3-tunneled-dl complete=0,timeout=0,running=4\n",
+  *entry_stats_1 =
+      "entry-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+      "entry-ips ab=8\n",
+  *entry_stats_2 =
+      "entry-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+      "entry-ips \n";
+  tor_addr_t addr;
+  struct in6_addr in6;
+
+  /* Populate the DB a bit.  Add these in order, since we can't do the final
+   * 'sort' step.  These aren't very good IP addresses, but they're perfectly
+   * fine uint32_t values. */
+  (void)arg;
+  tt_int_op(0,OP_EQ, geoip_parse_entry("10,50,AB", AF_INET));
+  tt_int_op(0,OP_EQ, geoip_parse_entry("52,90,XY", AF_INET));
+  tt_int_op(0,OP_EQ, geoip_parse_entry("95,100,AB", AF_INET));
+  tt_int_op(0,OP_EQ, geoip_parse_entry("\"105\",\"140\",\"ZZ\"", AF_INET));
+  tt_int_op(0,OP_EQ, geoip_parse_entry("\"150\",\"190\",\"XY\"", AF_INET));
+  tt_int_op(0,OP_EQ, geoip_parse_entry("\"200\",\"250\",\"AB\"", AF_INET));
+
+  /* Populate the IPv6 DB equivalently with fake IPs in the same range */
+  tt_int_op(0,OP_EQ, geoip_parse_entry("::a,::32,AB", AF_INET6));
+  tt_int_op(0,OP_EQ, geoip_parse_entry("::34,::5a,XY", AF_INET6));
+  tt_int_op(0,OP_EQ, geoip_parse_entry("::5f,::64,AB", AF_INET6));
+  tt_int_op(0,OP_EQ, geoip_parse_entry("::69,::8c,ZZ", AF_INET6));
+  tt_int_op(0,OP_EQ, geoip_parse_entry("::96,::be,XY", AF_INET6));
+  tt_int_op(0,OP_EQ, geoip_parse_entry("::c8,::fa,AB", AF_INET6));
+
+  /* We should have 4 countries: ??, ab, xy, zz. */
+  tt_int_op(4,OP_EQ, geoip_get_n_countries());
+  memset(&in6, 0, sizeof(in6));
+
+  CHECK_COUNTRY("??", 3);
+  CHECK_COUNTRY("ab", 32);
+  CHECK_COUNTRY("??", 5);
+  CHECK_COUNTRY("??", 51);
+  CHECK_COUNTRY("xy", 150);
+  CHECK_COUNTRY("xy", 190);
+  CHECK_COUNTRY("??", 2000);
+
+  tt_int_op(0,OP_EQ, geoip_get_country_by_ipv4(3));
+  SET_TEST_IPV6(3);
+  tt_int_op(0,OP_EQ, geoip_get_country_by_ipv6(&in6));
+
+  get_options_mutable()->BridgeRelay = 1;
+  get_options_mutable()->BridgeRecordUsageByCountry = 1;
+  /* Put 9 observations in AB... */
+  for (i=32; i < 40; ++i) {
+    SET_TEST_ADDRESS(i);
+    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200);
+  }
+  SET_TEST_ADDRESS(225);
+  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200);
+  /* and 3 observations in XY, several times. */
+  for (j=0; j < 10; ++j)
+    for (i=52; i < 55; ++i) {
+      SET_TEST_ADDRESS(i);
+      geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-3600);
+    }
+  /* and 17 observations in ZZ... */
+  for (i=110; i < 127; ++i) {
+    SET_TEST_ADDRESS(i);
+    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now);
+  }
+  geoip_get_client_history(GEOIP_CLIENT_CONNECT, &s, &v);
+  tt_assert(s);
+  tt_assert(v);
+  tt_str_op("zz=24,ab=16,xy=8",OP_EQ, s);
+  tt_str_op("v4=16,v6=16",OP_EQ, v);
+  tor_free(s);
+  tor_free(v);
+
+  /* Now clear out all the AB observations. */
+  geoip_remove_old_clients(now-6000);
+  geoip_get_client_history(GEOIP_CLIENT_CONNECT, &s, &v);
+  tt_assert(s);
+  tt_assert(v);
+  tt_str_op("zz=24,xy=8",OP_EQ, s);
+  tt_str_op("v4=16,v6=16",OP_EQ, v);
+  tor_free(s);
+  tor_free(v);
+
+  /* Start testing bridge statistics by making sure that we don't output
+   * bridge stats without initializing them. */
+  s = geoip_format_bridge_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Initialize stats and generate the bridge-stats history string out of
+   * the connecting clients added above. */
+  geoip_bridge_stats_init(now);
+  s = geoip_format_bridge_stats(now + 86400);
+  tt_assert(s);
+  tt_str_op(bridge_stats_1,OP_EQ, s);
+  tor_free(s);
+
+  /* Stop collecting bridge stats and make sure we don't write a history
+   * string anymore. */
+  geoip_bridge_stats_term();
+  s = geoip_format_bridge_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Stop being a bridge and start being a directory mirror that gathers
+   * directory request statistics. */
+  geoip_bridge_stats_term();
+  get_options_mutable()->BridgeRelay = 0;
+  get_options_mutable()->BridgeRecordUsageByCountry = 0;
+  get_options_mutable()->DirReqStatistics = 1;
+
+  /* Start testing dirreq statistics by making sure that we don't collect
+   * dirreq stats without initializing them. */
+  SET_TEST_ADDRESS(100);
+  geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now);
+  s = geoip_format_dirreq_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Initialize stats, note one connecting client, and generate the
+   * dirreq-stats history string. */
+  geoip_dirreq_stats_init(now);
+  SET_TEST_ADDRESS(100);
+  geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now);
+  s = geoip_format_dirreq_stats(now + 86400);
+  tt_str_op(dirreq_stats_1,OP_EQ, s);
+  tor_free(s);
+
+  /* Stop collecting stats, add another connecting client, and ensure we
+   * don't generate a history string. */
+  geoip_dirreq_stats_term();
+  SET_TEST_ADDRESS(101);
+  geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now);
+  s = geoip_format_dirreq_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Re-start stats, add a connecting client, reset stats, and make sure
+   * that we get an all empty history string. */
+  geoip_dirreq_stats_init(now);
+  SET_TEST_ADDRESS(100);
+  geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now);
+  geoip_reset_dirreq_stats(now);
+  s = geoip_format_dirreq_stats(now + 86400);
+  tt_str_op(dirreq_stats_2,OP_EQ, s);
+  tor_free(s);
+
+  /* Note a successful network status response and make sure that it
+   * appears in the history string. */
+  geoip_note_ns_response(GEOIP_SUCCESS);
+  s = geoip_format_dirreq_stats(now + 86400);
+  tt_str_op(dirreq_stats_3,OP_EQ, s);
+  tor_free(s);
+
+  /* Start a tunneled directory request. */
+  geoip_start_dirreq((uint64_t) 1, 1024, DIRREQ_TUNNELED);
+  s = geoip_format_dirreq_stats(now + 86400);
+  tt_str_op(dirreq_stats_4,OP_EQ, s);
+  tor_free(s);
+
+  /* Stop collecting directory request statistics and start gathering
+   * entry stats. */
+  geoip_dirreq_stats_term();
+  get_options_mutable()->DirReqStatistics = 0;
+  get_options_mutable()->EntryStatistics = 1;
+
+  /* Start testing entry statistics by making sure that we don't collect
+   * anything without initializing entry stats. */
+  SET_TEST_ADDRESS(100);
+  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now);
+  s = geoip_format_entry_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Initialize stats, note one connecting client, and generate the
+   * entry-stats history string. */
+  geoip_entry_stats_init(now);
+  SET_TEST_ADDRESS(100);
+  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now);
+  s = geoip_format_entry_stats(now + 86400);
+  tt_str_op(entry_stats_1,OP_EQ, s);
+  tor_free(s);
+
+  /* Stop collecting stats, add another connecting client, and ensure we
+   * don't generate a history string. */
+  geoip_entry_stats_term();
+  SET_TEST_ADDRESS(101);
+  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now);
+  s = geoip_format_entry_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Re-start stats, add a connecting client, reset stats, and make sure
+   * that we get an all empty history string. */
+  geoip_entry_stats_init(now);
+  SET_TEST_ADDRESS(100);
+  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now);
+  geoip_reset_entry_stats(now);
+  s = geoip_format_entry_stats(now + 86400);
+  tt_str_op(entry_stats_2,OP_EQ, s);
+  tor_free(s);
+
+  /* Test the OOM handler. Add a client, run the OOM. */
+  geoip_entry_stats_init(now);
+  SET_TEST_ADDRESS(100);
+  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL,
+                         now - (12 * 60 * 60));
+  /* We've seen this 12 hours ago. Run the OOM, it should clean the entry
+   * because it is above the minimum cutoff of 4 hours. */
+  size_t bytes_removed = geoip_client_cache_handle_oom(now, 1000);
+  tt_size_op(bytes_removed, OP_GT, 0);
+
+  /* Do it again but this time with an entry with a lower cutoff. */
+  geoip_entry_stats_init(now);
+  SET_TEST_ADDRESS(100);
+  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL,
+                         now - (3 * 60 * 60));
+  bytes_removed = geoip_client_cache_handle_oom(now, 1000);
+  tt_size_op(bytes_removed, OP_EQ, 0);
+
+  /* Stop collecting entry statistics. */
+  geoip_entry_stats_term();
+  get_options_mutable()->EntryStatistics = 0;
+
+ done:
+  tor_free(s);
+  tor_free(v);
+}
+
+static void
+test_geoip_with_pt(void *arg)
+{
+  time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */
+  char *s = NULL;
+  int i;
+  tor_addr_t addr;
+  struct in6_addr in6;
+
+  (void)arg;
+  get_options_mutable()->BridgeRelay = 1;
+  get_options_mutable()->BridgeRecordUsageByCountry = 1;
+
+  memset(&in6, 0, sizeof(in6));
+
+  /* No clients seen yet. */
+  s = geoip_get_transport_history();
+  tor_assert(!s);
+
+  /* 4 connections without a pluggable transport */
+  for (i=0; i < 4; ++i) {
+    SET_TEST_ADDRESS(i);
+    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200);
+  }
+
+  /* 9 connections with "alpha" */
+  for (i=4; i < 13; ++i) {
+    SET_TEST_ADDRESS(i);
+    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "alpha", now-7200);
+  }
+
+  /* one connection with "beta" */
+  SET_TEST_ADDRESS(13);
+  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "beta", now-7200);
+
+  /* 14 connections with "charlie" */
+  for (i=14; i < 28; ++i) {
+    SET_TEST_ADDRESS(i);
+    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "charlie", now-7200);
+  }
+
+  /* 131 connections with "ddr" */
+  for (i=28; i < 159; ++i) {
+    SET_TEST_ADDRESS(i);
+    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "ddr", now-7200);
+  }
+
+  /* 8 connections with "entropy" */
+  for (i=159; i < 167; ++i) {
+    SET_TEST_ADDRESS(i);
+    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "entropy", now-7200);
+  }
+
+  /* 2 connections from the same IP with two different transports. */
+  SET_TEST_ADDRESS(++i);
+  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "fire", now-7200);
+  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "google", now-7200);
+
+  /* Test the transport history string. */
+  s = geoip_get_transport_history();
+  tor_assert(s);
+  tt_str_op(s,OP_EQ, "<OR>=8,alpha=16,beta=8,charlie=16,ddr=136,"
+             "entropy=8,fire=8,google=8");
+
+  /* Stop collecting entry statistics. */
+  geoip_entry_stats_term();
+  get_options_mutable()->EntryStatistics = 0;
+
+ done:
+  tor_free(s);
+}
+
+#undef SET_TEST_ADDRESS
+#undef SET_TEST_IPV6
+#undef CHECK_COUNTRY
+
+static const char GEOIP_CONTENT[] =
+  "134445936,134445939,MP\n"
+  "134445940,134447103,GU\n"
+  "134447104,134738943,US\n"
+  "134738944,134739199,CA\n"
+  "134739200,135192575,US\n"
+  "135192576,135200767,MX\n"
+  "135200768,135430143,US\n"
+  "135430144,135430399,CA\n"
+  "135430400,135432191,US\n";
+
+static void
+test_geoip_load_file(void *arg)
+{
+  (void)arg;
+  char *contents = NULL;
+  char *dhex = NULL;
+
+  /* A nonexistant filename should fail. */
+  tt_int_op(-1, OP_EQ,
+            geoip_load_file(AF_INET, "/you/did/not/put/a/file/here/I/hope",
+                            LOG_INFO));
+
+  /* We start out with only "Ningunpartia" in the database. */
+  tt_int_op(1, OP_EQ, geoip_get_n_countries());
+  tt_str_op("??", OP_EQ, geoip_get_country_name(0));
+  /* Any lookup attempt should say "-1" because we have no info */
+  tt_int_op(-1, OP_EQ, geoip_get_country_by_ipv4(0x01020304));
+  /* There should be no 'digest' for a nonexistant file */
+  tt_str_op("0000000000000000000000000000000000000000", OP_EQ,
+            geoip_db_digest(AF_INET));
+
+  const char *fname = get_fname("geoip");
+  tt_int_op(0, OP_EQ, write_str_to_file(fname, GEOIP_CONTENT, 1));
+
+  int rv = geoip_load_file(AF_INET, fname, LOG_WARN);
+  if (rv != 0) {
+    TT_GRIPE(("Unable to load geoip from %s", escaped(fname)));
+  }
+  tt_int_op(0, OP_EQ, rv);
+
+  /* Check that we loaded some countries; this will fail if there are ever
+   * fewer than 5 countries in our test above. */
+  tt_int_op(geoip_get_n_countries(), OP_GE, 5);
+
+  /* Let's see where 8.8.8.8 is. */
+  int country = geoip_get_country_by_ipv4(0x08080808);
+  tt_int_op(country, OP_GE, 1); /* It shouldn't be 'unknown' or 'nowhere' */
+  const char *cc = geoip_get_country_name(country);
+  tt_int_op(strlen(cc), OP_EQ, 2);
+
+  /* The digest should be set.... */
+  tt_str_op("0000000000000000000000000000000000000000", OP_NE,
+            geoip_db_digest(AF_INET));
+
+  /* And it should be set correctly */
+  contents = read_file_to_str(fname, RFTS_BIN, NULL);
+  uint8_t d[DIGEST_LEN];
+  crypto_digest((char*)d, contents, strlen(contents));
+  dhex = tor_strdup(hex_str((char*)d, DIGEST_LEN));
+  tt_str_op(dhex, OP_EQ, geoip_db_digest(AF_INET));
+
+  /* Make sure geoip_free_all() works. */
+  geoip_free_all();
+  tt_int_op(1, OP_EQ, geoip_get_n_countries());
+  tt_str_op("??", OP_EQ, geoip_get_country_name(0));
+  tt_int_op(-1, OP_EQ, geoip_get_country_by_ipv4(0x01020304));
+  tt_str_op("0000000000000000000000000000000000000000", OP_EQ,
+            geoip_db_digest(AF_INET)); // <--- nick bets this will fail.
+
+ done:
+  tor_free(contents);
+  tor_free(dhex);
+}
+
+static void
+test_geoip6_load_file(void *arg)
+{
+  (void)arg;
+  struct in6_addr iaddr6;
+  char *contents = NULL;
+  char *dhex = NULL;
+
+  /* A nonexistant filename should fail. */
+  tt_int_op(-1, OP_EQ,
+            geoip_load_file(AF_INET6, "/you/did/not/put/a/file/here/I/hope",
+                            LOG_INFO));
+
+  /* Any lookup attempt should say "-1" because we have no info */
+  tor_inet_pton(AF_INET6, "2001:4860:4860::8888", &iaddr6);
+  tt_int_op(-1, OP_EQ, geoip_get_country_by_ipv6(&iaddr6));
+
+  /* Load geiop6 file */
+  const char *fname6 = get_fname("geoip6");
+  const char CONTENT[] =
+    "2001:4830:6010::,2001:4830:601f:ffff:ffff:ffff:ffff:ffff,GB\n"
+    "2001:4830:6020::,2001:4830:ffff:ffff:ffff:ffff:ffff:ffff,US\n"
+    "2001:4838::,2001:4838:ffff:ffff:ffff:ffff:ffff:ffff,US\n"
+    "2001:4840::,2001:4840:ffff:ffff:ffff:ffff:ffff:ffff,XY\n"
+    "2001:4848::,2001:4848:ffff:ffff:ffff:ffff:ffff:ffff,ZD\n"
+    "2001:4850::,2001:4850:ffff:ffff:ffff:ffff:ffff:ffff,RO\n"
+    "2001:4858::,2001:4858:ffff:ffff:ffff:ffff:ffff:ffff,TC\n"
+    "2001:4860::,2001:4860:ffff:ffff:ffff:ffff:ffff:ffff,US\n"
+    "2001:4868::,2001:4868:ffff:ffff:ffff:ffff:ffff:ffff,US\n"
+    "2001:4870::,2001:4871:ffff:ffff:ffff:ffff:ffff:ffff,NB\n"
+    "2001:4878::,2001:4878:128:ffff:ffff:ffff:ffff:ffff,US\n"
+    "2001:4878:129::,2001:4878:129:ffff:ffff:ffff:ffff:ffff,CR\n"
+    "2001:4878:12a::,2001:4878:203:ffff:ffff:ffff:ffff:ffff,US\n"
+    "2001:4878:204::,2001:4878:204:ffff:ffff:ffff:ffff:ffff,DE\n"
+    "2001:4878:205::,2001:4878:214:ffff:ffff:ffff:ffff:ffff,US\n";
+  tt_int_op(0, OP_EQ, write_str_to_file(fname6, CONTENT, 1));
+
+  tt_int_op(0, OP_EQ, geoip_load_file(AF_INET6, fname6, LOG_WARN));
+
+  /* Check that we loaded some countries; this will fail if there are ever
+   * fewer than 5 countries in our test data above. */
+  tt_int_op(geoip_get_n_countries(), OP_GE, 5);
+
+  /* Let's see where 2001:4860:4860::8888 (google dns) is. */
+  const char *caddr6 = "2001:4860:4860::8888";
+  tor_inet_pton(AF_INET6, caddr6, &iaddr6);
+  int country6 = geoip_get_country_by_ipv6(&iaddr6);
+  tt_int_op(country6, OP_GE, 1);
+
+  const char *cc6 = geoip_get_country_name(country6);
+  tt_int_op(strlen(cc6), OP_EQ, 2);
+
+  /* The digest should be set.... */
+  tt_str_op("0000000000000000000000000000000000000000", OP_NE,
+            geoip_db_digest(AF_INET6));
+
+  /* And it should be set correctly */
+  contents = read_file_to_str(fname6, RFTS_BIN, NULL);
+  uint8_t d[DIGEST_LEN];
+  crypto_digest((char*)d, contents, strlen(contents));
+  dhex = tor_strdup(hex_str((char*)d, DIGEST_LEN));
+  tt_str_op(dhex, OP_EQ, geoip_db_digest(AF_INET6));
+
+  /* Make sure geoip_free_all() works. */
+  geoip_free_all();
+  tt_int_op(1, OP_EQ, geoip_get_n_countries());
+  tt_str_op("??", OP_EQ, geoip_get_country_name(0));
+  tor_inet_pton(AF_INET6, "::1:2:3:4", &iaddr6);
+  tt_int_op(-1, OP_EQ, geoip_get_country_by_ipv6(&iaddr6));
+  tt_str_op("0000000000000000000000000000000000000000", OP_EQ,
+            geoip_db_digest(AF_INET6));
+
+ done:
+  tor_free(contents);
+  tor_free(dhex);
+}
+
+static void
+test_geoip_load_2nd_file(void *arg)
+{
+  (void)arg;
+
+  char *fname_geoip = tor_strdup(get_fname("geoip_data"));
+  char *fname_empty = tor_strdup(get_fname("geoip_empty"));
+
+  tt_int_op(0, OP_EQ, write_str_to_file(fname_geoip, GEOIP_CONTENT, 1));
+  tt_int_op(0, OP_EQ, write_str_to_file(fname_empty, "\n", 1));
+
+  /* Load 1st geoip file */
+  tt_int_op(0, OP_EQ, geoip_load_file(AF_INET, fname_geoip, LOG_WARN));
+
+  /* Load 2nd geoip (empty) file */
+  /* It has to be the same IP address family */
+  tt_int_op(0, OP_EQ, geoip_load_file(AF_INET, fname_empty, LOG_WARN));
+
+  /* Check that there is no geoip information for 8.8.8.8, */
+  /* since loading the empty 2nd file should have delete it. */
+  int country = geoip_get_country_by_ipv4(0x08080808);
+  tt_int_op(country, OP_EQ, 0);
+
+ done:
+  tor_free(fname_geoip);
+  tor_free(fname_empty);
+}
+
+#define ENT(name)                                                       \
+  { #name, test_ ## name , 0, NULL, NULL }
+#define FORK(name)                                                      \
+  { #name, test_ ## name , TT_FORK, NULL, NULL }
+
+struct testcase_t geoip_tests[] = {
+  { "geoip", test_geoip, TT_FORK, NULL, NULL },
+  { "geoip_with_pt", test_geoip_with_pt, TT_FORK, NULL, NULL },
+  { "load_file", test_geoip_load_file, TT_FORK, NULL, NULL },
+  { "load_file6", test_geoip6_load_file, TT_FORK, NULL, NULL },
+  { "load_2nd_file", test_geoip_load_2nd_file, TT_FORK, NULL, NULL },
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_guardfraction.c b/src/test/test_guardfraction.c
index 8173e44d47..ac8bfbfded 100644
--- a/src/test/test_guardfraction.c
+++ b/src/test/test_guardfraction.c
@@ -1,23 +1,25 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#define DIRSERV_PRIVATE
-#define ROUTERPARSE_PRIVATE
+#define GUARDFRACTION_PRIVATE
 #define NETWORKSTATUS_PRIVATE
+#define NS_PARSE_PRIVATE
 
 #include "orconfig.h"
-#include "or.h"
-#include "config.h"
-#include "dirserv.h"
-#include "container.h"
-#include "entrynodes.h"
-#include "util.h"
-#include "routerparse.h"
-#include "networkstatus.h"
-
-#include "test.h"
-#include "test_helpers.h"
-#include "log_test_helpers.h"
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "feature/dirauth/guardfraction.h"
+#include "feature/client/entrynodes.h"
+#include "feature/dirparse/ns_parse.h"
+#include "feature/nodelist/networkstatus.h"
+
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/dirauth/vote_microdesc_hash_st.h"
+#include "feature/nodelist/vote_routerstatus_st.h"
+
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
 
 /** Generate a vote_routerstatus_t for a router with identity digest
  * <b>digest_in_hex</b>. */
@@ -38,10 +40,10 @@ gen_vote_routerstatus_for_tests(const char *digest_in_hex, int is_guard)
     rs->is_possible_guard = is_guard;
 
     /* Fill in the fpr */
-    tt_int_op(strlen(digest_in_hex), ==, HEX_DIGEST_LEN);
+    tt_int_op(strlen(digest_in_hex), OP_EQ, HEX_DIGEST_LEN);
     retval = base16_decode(digest_tmp, sizeof(digest_tmp),
                            digest_in_hex, HEX_DIGEST_LEN);
-    tt_int_op(retval, ==, sizeof(digest_tmp));
+    tt_int_op(retval, OP_EQ, sizeof(digest_tmp));
     memcpy(rs->identity_digest, digest_tmp, DIGEST_LEN);
   }
 
@@ -88,7 +90,7 @@ test_parse_guardfraction_file_bad(void *arg)
                yesterday_date_str);
 
   retval = dirserv_read_guardfraction_file_from_str(guardfraction_bad, NULL);
-  tt_int_op(retval, ==, -1);
+  tt_int_op(retval, OP_EQ, -1);
   tor_free(guardfraction_bad);
 
   /* This one does not have a date! Parsing should fail. */
@@ -100,7 +102,7 @@ test_parse_guardfraction_file_bad(void *arg)
                "guard-seen 07B5547026DF3E229806E135CFA8552D56AFBABC 5 420\n");
 
   retval = dirserv_read_guardfraction_file_from_str(guardfraction_bad, NULL);
-  tt_int_op(retval, ==, -1);
+  tt_int_op(retval, OP_EQ, -1);
   tor_free(guardfraction_bad);
 
   /* This one has an incomplete n-inputs line, but parsing should
@@ -114,7 +116,7 @@ test_parse_guardfraction_file_bad(void *arg)
                yesterday_date_str);
 
   retval = dirserv_read_guardfraction_file_from_str(guardfraction_bad, NULL);
-  tt_int_op(retval, ==, 2);
+  tt_int_op(retval, OP_EQ, 2);
   tor_free(guardfraction_bad);
 
   /* This one does not have a fingerprint in the guard line! */
@@ -126,7 +128,7 @@ test_parse_guardfraction_file_bad(void *arg)
                yesterday_date_str);
 
   retval = dirserv_read_guardfraction_file_from_str(guardfraction_bad, NULL);
-  tt_int_op(retval, ==, 0);
+  tt_int_op(retval, OP_EQ, 0);
   tor_free(guardfraction_bad);
 
   /* This one does not even have an integer guardfraction value. */
@@ -139,7 +141,7 @@ test_parse_guardfraction_file_bad(void *arg)
                yesterday_date_str);
 
   retval = dirserv_read_guardfraction_file_from_str(guardfraction_bad, NULL);
-  tt_int_op(retval, ==, 1);
+  tt_int_op(retval, OP_EQ, 1);
   tor_free(guardfraction_bad);
 
   /* This one is not a percentage (not in [0, 100]) */
@@ -152,7 +154,7 @@ test_parse_guardfraction_file_bad(void *arg)
                yesterday_date_str);
 
   retval = dirserv_read_guardfraction_file_from_str(guardfraction_bad, NULL);
-  tt_int_op(retval, ==, 1);
+  tt_int_op(retval, OP_EQ, 1);
   tor_free(guardfraction_bad);
 
   /* This one is not a percentage either (not in [0, 100]) */
@@ -164,7 +166,7 @@ test_parse_guardfraction_file_bad(void *arg)
                yesterday_date_str);
 
   retval = dirserv_read_guardfraction_file_from_str(guardfraction_bad, NULL);
-  tt_int_op(retval, ==, 0);
+  tt_int_op(retval, OP_EQ, 0);
 
  done:
   tor_free(guardfraction_bad);
@@ -216,14 +218,14 @@ test_parse_guardfraction_file_good(void *arg)
   /* Read the guardfraction file */
   retval = dirserv_read_guardfraction_file_from_str(guardfraction_good,
                                                     routerstatuses);
-  tt_int_op(retval, ==, 1);
+  tt_int_op(retval, OP_EQ, 1);
 
   { /* Test that routerstatus fields got filled properly */
 
     /* The guardfraction fields of the guard should be filled. */
     tt_assert(vrs_guard->status.has_guardfraction);
     tt_int_op(vrs_guard->status.guardfraction_percentage,
-              ==,
+              OP_EQ,
               guardfraction_value);
 
     /* The guard that was not in the guardfraction file should not have
@@ -252,12 +254,12 @@ test_get_guardfraction_bandwidth(void *arg)
   guard_get_guardfraction_bandwidth(&gf_bw,
                                     orig_bw, 25);
 
-  tt_int_op(gf_bw.guard_bw, ==, 250);
-  tt_int_op(gf_bw.non_guard_bw, ==, 750);
+  tt_int_op(gf_bw.guard_bw, OP_EQ, 250);
+  tt_int_op(gf_bw.non_guard_bw, OP_EQ, 750);
 
   /* Also check the 'guard_bw + non_guard_bw == original_bw'
    * invariant. */
-  tt_int_op(gf_bw.non_guard_bw + gf_bw.guard_bw, ==, orig_bw);
+  tt_int_op(gf_bw.non_guard_bw + gf_bw.guard_bw, OP_EQ, orig_bw);
 
  done:
   ;
@@ -295,9 +297,9 @@ test_parse_guardfraction_consensus(void *arg)
     retval = routerstatus_parse_guardfraction(guardfraction_str_good,
                                               NULL, NULL,
                                               &rs_good);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
     tt_assert(rs_good.has_guardfraction);
-    tt_int_op(rs_good.guardfraction_percentage, ==, 66);
+    tt_int_op(rs_good.guardfraction_percentage, OP_EQ, 66);
   }
 
   { /* Properly formatted GuardFraction but router is not a
@@ -309,7 +311,7 @@ test_parse_guardfraction_consensus(void *arg)
     retval = routerstatus_parse_guardfraction(guardfraction_str_good,
                                               NULL, NULL,
                                               &rs_no_guard);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
     tt_assert(!rs_no_guard.has_guardfraction);
     expect_single_log_msg_containing("Got GuardFraction for non-guard . "
                                      "This is not supposed to happen.");
@@ -323,7 +325,7 @@ test_parse_guardfraction_consensus(void *arg)
     retval = routerstatus_parse_guardfraction(guardfraction_str_bad1,
                                               NULL, NULL,
                                               &rs_bad1);
-    tt_int_op(retval, ==, -1);
+    tt_int_op(retval, OP_EQ, -1);
     tt_assert(!rs_bad1.has_guardfraction);
   }
 
@@ -334,7 +336,7 @@ test_parse_guardfraction_consensus(void *arg)
     retval = routerstatus_parse_guardfraction(guardfraction_str_bad2,
                                               NULL, NULL,
                                               &rs_bad2);
-    tt_int_op(retval, ==, -1);
+    tt_int_op(retval, OP_EQ, -1);
     tt_assert(!rs_bad2.has_guardfraction);
   }
 
@@ -375,27 +377,27 @@ test_should_apply_guardfraction(void *arg)
   /* If torrc option is set to yes, we should always use
    * guardfraction.*/
   options->UseGuardFraction = 1;
-  tt_int_op(should_apply_guardfraction(&vote_disabled), ==, 1);
+  tt_int_op(should_apply_guardfraction(&vote_disabled), OP_EQ, 1);
 
   /* If torrc option is set to no, we should never use
    * guardfraction.*/
   options->UseGuardFraction = 0;
-  tt_int_op(should_apply_guardfraction(&vote_enabled), ==, 0);
+  tt_int_op(should_apply_guardfraction(&vote_enabled), OP_EQ, 0);
 
   /* Now let's test torrc option set to auto. */
   options->UseGuardFraction = -1;
 
   /* If torrc option is set to auto, and consensus parameter is set to
    * yes, we should use guardfraction. */
-  tt_int_op(should_apply_guardfraction(&vote_enabled), ==, 1);
+  tt_int_op(should_apply_guardfraction(&vote_enabled), OP_EQ, 1);
 
   /* If torrc option is set to auto, and consensus parameter is set to
    * no, we should use guardfraction. */
-  tt_int_op(should_apply_guardfraction(&vote_disabled), ==, 0);
+  tt_int_op(should_apply_guardfraction(&vote_disabled), OP_EQ, 0);
 
   /* If torrc option is set to auto, and consensus parameter is not
    * set, we should fallback to "no". */
-  tt_int_op(should_apply_guardfraction(&vote_missing), ==, 0);
+  tt_int_op(should_apply_guardfraction(&vote_missing), OP_EQ, 0);
 
  done:
   SMARTLIST_FOREACH(vote_enabled.net_params, char *, cp, tor_free(cp));
@@ -420,4 +422,3 @@ struct testcase_t guardfraction_tests[] = {
 
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_handles.c b/src/test/test_handles.c
index 536a478689..7f1d6e1898 100644
--- a/src/test/test_handles.c
+++ b/src/test/test_handles.c
@@ -1,11 +1,13 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
-#include "test.h"
+#include "test/test.h"
 
-#include "util.h"
-#include "handles.h"
+#include "lib/container/handles.h"
+#include "lib/log/util_bug.h"
+
+#include <stdio.h>
 
 typedef struct demo_t {
   HANDLE_ENTRY(demo, demo_t);
@@ -13,6 +15,8 @@ typedef struct demo_t {
 } demo_t;
 
 HANDLE_DECL(demo, demo_t, static)
+#define demo_handle_free(h)    \
+  FREE_AND_NULL(demo_handle_t, demo_handle_free_, (h))
 HANDLE_IMPL(demo, demo_t, static)
 
 static demo_t *
@@ -92,4 +96,3 @@ struct testcase_t handle_tests[] = {
   HANDLE_TEST(basic, 0),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c
index ae9fc7a243..802d0a9ebe 100644
--- a/src/test/test_helpers.c
+++ b/src/test/test_helpers.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -7,14 +7,34 @@
  */
 
 #define ROUTERLIST_PRIVATE
+#define CONFIG_PRIVATE
+#define CONNECTION_PRIVATE
+#define MAINLOOP_PRIVATE
+
 #include "orconfig.h"
-#include "or.h"
+#include "core/or/or.h"
+
+#include "lib/container/buffers.h"
+#include "app/config/config.h"
+#include "app/config/confparse.h"
+#include "core/mainloop/connection.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "core/mainloop/mainloop.h"
+#include "feature/nodelist/nodelist.h"
+#include "core/or/relay.h"
+#include "feature/nodelist/routerlist.h"
+#include "lib/encoding/confline.h"
+#include "lib/net/resolve.h"
 
-#include "routerlist.h"
-#include "nodelist.h"
+#include "core/or/cell_st.h"
+#include "core/or/connection_st.h"
+#include "feature/nodelist/node_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "feature/nodelist/routerlist_st.h"
 
-#include "test.h"
-#include "test_helpers.h"
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/test_connection.h"
 
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
 DISABLE_GCC_WARNING(overlength-strings)
@@ -22,6 +42,7 @@ DISABLE_GCC_WARNING(overlength-strings)
  * at large. */
 #endif
 #include "test_descriptors.inc"
+#include "core/or/circuitlist.h"
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
 ENABLE_GCC_WARNING(overlength-strings)
 #endif
@@ -70,16 +91,16 @@ helper_setup_fake_routerlist(void)
   retval = router_load_routers_from_string(TEST_DESCRIPTORS,
                                            NULL, SAVED_IN_JOURNAL,
                                            NULL, 0, NULL);
-  tt_int_op(retval, ==, HELPER_NUMBER_OF_DESCRIPTORS);
+  tt_int_op(retval, OP_EQ, HELPER_NUMBER_OF_DESCRIPTORS);
 
   /* Sanity checking of routerlist and nodelist. */
   our_routerlist = router_get_routerlist();
-  tt_int_op(smartlist_len(our_routerlist->routers), ==,
+  tt_int_op(smartlist_len(our_routerlist->routers), OP_EQ,
               HELPER_NUMBER_OF_DESCRIPTORS);
   routerlist_assert_ok(our_routerlist);
 
   our_nodelist = nodelist_get_list();
-  tt_int_op(smartlist_len(our_nodelist), ==, HELPER_NUMBER_OF_DESCRIPTORS);
+  tt_int_op(smartlist_len(our_nodelist), OP_EQ, HELPER_NUMBER_OF_DESCRIPTORS);
 
   /* Mark all routers as non-guards but up and running! */
   SMARTLIST_FOREACH_BEGIN(our_nodelist, node_t *, node) {
@@ -92,3 +113,193 @@ helper_setup_fake_routerlist(void)
   UNMOCK(router_descriptor_is_older_than);
 }
 
+void
+connection_write_to_buf_mock(const char *string, size_t len,
+                             connection_t *conn, int compressed)
+{
+  (void) compressed;
+
+  tor_assert(string);
+  tor_assert(conn);
+
+  buf_add(conn->outbuf, string, len);
+}
+
+char *
+buf_get_contents(buf_t *buf, size_t *sz_out)
+{
+  tor_assert(buf);
+  tor_assert(sz_out);
+
+  char *out;
+  *sz_out = buf_datalen(buf);
+  if (*sz_out >= ULONG_MAX)
+    return NULL; /* C'mon, really? */
+  out = tor_malloc(*sz_out + 1);
+  if (buf_get_bytes(buf, out, (unsigned long)*sz_out) != 0) {
+    tor_free(out);
+    return NULL;
+  }
+  out[*sz_out] = '\0'; /* Hopefully gratuitous. */
+  return out;
+}
+
+/* Set up a fake origin circuit with the specified number of cells,
+ * Return a pointer to the newly-created dummy circuit */
+circuit_t *
+dummy_origin_circuit_new(int n_cells)
+{
+  origin_circuit_t *circ = origin_circuit_new();
+  int i;
+  cell_t cell;
+
+  for (i=0; i < n_cells; ++i) {
+    crypto_rand((void*)&cell, sizeof(cell));
+    cell_queue_append_packed_copy(TO_CIRCUIT(circ),
+                                  &TO_CIRCUIT(circ)->n_chan_cells,
+                                  1, &cell, 1, 0);
+  }
+
+  TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  return TO_CIRCUIT(circ);
+}
+
+/** Mock-replacement. As tor_addr_lookup, but always fails on any
+ * address containing a !.  This is necessary for running the unit tests
+ * on networks where DNS hijackers think it's helpful to give answers
+ * for things like 1.2.3.4.5 or "invalidstuff!!"
+ */
+int
+mock_tor_addr_lookup__fail_on_bad_addrs(const char *name,
+                                        uint16_t family, tor_addr_t *out)
+{
+  if (name && strchr(name, '!')) {
+    return -1;
+  }
+  return tor_addr_lookup__real(name, family, out);
+}
+
+/*********** Helper funcs for making new connections/streams *****************/
+
+/* Helper for test_conn_get_connection() */
+static int
+fake_close_socket(tor_socket_t sock)
+{
+  (void)sock;
+  return 0;
+}
+
+static int mock_connection_connect_sockaddr_called = 0;
+static int fake_socket_number = TEST_CONN_FD_INIT;
+
+/* Helper for test_conn_get_connection() */
+static int
+mock_connection_connect_sockaddr(connection_t *conn,
+                                 const struct sockaddr *sa,
+                                 socklen_t sa_len,
+                                 const struct sockaddr *bindaddr,
+                                 socklen_t bindaddr_len,
+                                 int *socket_error)
+{
+  (void)sa_len;
+  (void)bindaddr;
+  (void)bindaddr_len;
+
+  tor_assert(conn);
+  tor_assert(sa);
+  tor_assert(socket_error);
+
+  mock_connection_connect_sockaddr_called++;
+
+  conn->s = fake_socket_number++;
+  tt_assert(SOCKET_OK(conn->s));
+  /* We really should call tor_libevent_initialize() here. Because we don't,
+   * we are relying on other parts of the code not checking if the_event_base
+   * (and therefore event->ev_base) is NULL.  */
+  tt_int_op(connection_add_connecting(conn), OP_EQ, 0);
+
+ done:
+  /* Fake "connected" status */
+  return 1;
+}
+
+/** Create and return a new connection/stream */
+connection_t *
+test_conn_get_connection(uint8_t state, uint8_t type, uint8_t purpose)
+{
+  connection_t *conn = NULL;
+  tor_addr_t addr;
+  int socket_err = 0;
+  int in_progress = 0;
+
+  MOCK(connection_connect_sockaddr,
+       mock_connection_connect_sockaddr);
+  MOCK(tor_close_socket, fake_close_socket);
+
+  tor_init_connection_lists();
+
+  conn = connection_new(type, TEST_CONN_FAMILY);
+  tt_assert(conn);
+
+  test_conn_lookup_addr_helper(TEST_CONN_ADDRESS, TEST_CONN_FAMILY, &addr);
+  tt_assert(!tor_addr_is_null(&addr));
+
+  tor_addr_copy_tight(&conn->addr, &addr);
+  conn->port = TEST_CONN_PORT;
+  mock_connection_connect_sockaddr_called = 0;
+  in_progress = connection_connect(conn, TEST_CONN_ADDRESS_PORT, &addr,
+                                   TEST_CONN_PORT, &socket_err);
+  tt_int_op(mock_connection_connect_sockaddr_called, OP_EQ, 1);
+  tt_assert(!socket_err);
+  tt_assert(in_progress == 0 || in_progress == 1);
+
+  /* fake some of the attributes so the connection looks OK */
+  conn->state = state;
+  conn->purpose = purpose;
+  assert_connection_ok(conn, time(NULL));
+
+  UNMOCK(connection_connect_sockaddr);
+  UNMOCK(tor_close_socket);
+  return conn;
+
+  /* On failure */
+ done:
+  UNMOCK(connection_connect_sockaddr);
+  UNMOCK(tor_close_socket);
+  return NULL;
+}
+
+/* Helper function to parse a set of torrc options in a text format and return
+ * a newly allocated or_options_t object containing the configuration. On
+ * error, NULL is returned indicating that the conf couldn't be parsed
+ * properly. */
+or_options_t *
+helper_parse_options(const char *conf)
+{
+  int ret = 0;
+  char *msg = NULL;
+  or_options_t *opt = NULL;
+  config_line_t *line = NULL;
+
+  /* Kind of pointless to call this with a NULL value. */
+  tt_assert(conf);
+
+  opt = options_new();
+  tt_assert(opt);
+  ret = config_get_lines(conf, &line, 1);
+  if (ret != 0) {
+    goto done;
+  }
+  ret = config_assign(&options_format, opt, line, 0, &msg);
+  if (ret != 0) {
+    goto done;
+  }
+
+ done:
+  config_free_lines(line);
+  if (ret != 0) {
+    or_options_free(opt);
+    opt = NULL;
+  }
+  return opt;
+}
diff --git a/src/test/test_helpers.h b/src/test/test_helpers.h
index 684375e1b1..9e376a563d 100644
--- a/src/test/test_helpers.h
+++ b/src/test/test_helpers.h
@@ -1,17 +1,35 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_TEST_HELPERS_H
 #define TOR_TEST_HELPERS_H
 
+#define BUFFERS_PRIVATE
+
+#include "core/or/or.h"
+
 const char *get_yesterday_date_str(void);
 
+circuit_t * dummy_origin_circuit_new(int num_cells);
+
 /* Number of descriptors contained in test_descriptors.txt. */
 #define HELPER_NUMBER_OF_DESCRIPTORS 8
 
 void helper_setup_fake_routerlist(void);
 
+#define GET(path) "GET " path " HTTP/1.0\r\n\r\n"
+void connection_write_to_buf_mock(const char *string, size_t len,
+                                  connection_t *conn, int compressed);
+char *buf_get_contents(buf_t *buf, size_t *sz_out);
+
+int mock_tor_addr_lookup__fail_on_bad_addrs(const char *name,
+                                            uint16_t family, tor_addr_t *out);
+
+connection_t *test_conn_get_connection(uint8_t state,
+                                       uint8_t type, uint8_t purpose);
+or_options_t *helper_parse_options(const char *conf);
+
 extern const char TEST_DESCRIPTORS[];
 
-#endif
+#endif /* !defined(TOR_TEST_HELPERS_H) */
 
diff --git a/src/test/test_hs.c b/src/test/test_hs.c
index d572dd8d2e..a611b46ca6 100644
--- a/src/test/test_hs.c
+++ b/src/test/test_hs.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2016, The Tor Project, Inc. */
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -8,17 +8,31 @@
 
 #define CONTROL_PRIVATE
 #define CIRCUITBUILD_PRIVATE
+#define RENDCOMMON_PRIVATE
 #define RENDSERVICE_PRIVATE
-
-#include "or.h"
-#include "test.h"
-#include "control.h"
-#include "config.h"
-#include "rendcommon.h"
-#include "rendservice.h"
-#include "routerset.h"
-#include "circuitbuild.h"
-#include "test_helpers.h"
+#define HS_SERVICE_PRIVATE
+
+#include "core/or/or.h"
+#include "test/test.h"
+#include "feature/control/control.h"
+#include "app/config/config.h"
+#include "feature/hs/hs_common.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/rend/rendservice.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerset.h"
+#include "core/or/circuitbuild.h"
+
+#include "feature/nodelist/node_st.h"
+#include "feature/rend/rend_encoded_v2_service_descriptor_st.h"
+#include "feature/rend/rend_intro_point_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+
+#include "test/test_helpers.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
 
 /* mock ID digest and longname for node that's in nodelist */
 #define HSDIR_EXIST_ID "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA" \
@@ -31,8 +45,9 @@
 #define STR_HSDIR_NONE_EXIST_LONGNAME \
                        "$BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"
 
-/* DuckDuckGo descriptor as an example. */
-static const char *hs_desc_content = "\
+/* DuckDuckGo descriptor as an example. This one has extra "\r" at the end so
+ * the control port is happy. */
+static const char *hs_desc_content_control = "\
 rendezvous-service-descriptor g5ojobzupf275beh5ra72uyhb3dkpxwg\r\n\
 version 2\r\n\
 permanent-key\r\n\
@@ -93,6 +108,68 @@ PcftsZf2ztN0sbNCtPgDL3d0PqvxY3iHTQAI8EbaGq/IAJUZ8U4y963dD5+Bn6JQ\r\n\
 myE3ctmh0vy5+QxSiRjmQBkuEpCyks7LvWvHYrhnmcg=\r\n\
 -----END SIGNATURE-----";
 
+/* DuckDuckGo descriptor as an example. */
+static const char *hs_desc_content = "\
+rendezvous-service-descriptor g5ojobzupf275beh5ra72uyhb3dkpxwg\n\
+version 2\n\
+permanent-key\n\
+-----BEGIN RSA PUBLIC KEY-----\n\
+MIGJAoGBAJ/SzzgrXPxTlFrKVhXh3buCWv2QfcNgncUpDpKouLn3AtPH5Ocys0jE\n\
+aZSKdvaiQ62md2gOwj4x61cFNdi05tdQjS+2thHKEm/KsB9BGLSLBNJYY356bupg\n\
+I5gQozM65ENelfxYlysBjJ52xSDBd8C4f/p9umdzaaaCmzXG/nhzAgMBAAE=\n\
+-----END RSA PUBLIC KEY-----\n\
+secret-id-part anmjoxxwiupreyajjt5yasimfmwcnxlf\n\
+publication-time 2015-03-11 19:00:00\n\
+protocol-versions 2,3\n\
+introduction-points\n\
+-----BEGIN MESSAGE-----\n\
+aW50cm9kdWN0aW9uLXBvaW50IDd1bnd4cmg2dG5kNGh6eWt1Z3EzaGZzdHduc2ll\n\
+cmhyCmlwLWFkZHJlc3MgMTg4LjEzOC4xMjEuMTE4Cm9uaW9uLXBvcnQgOTAwMQpv\n\
+bmlvbi1rZXkKLS0tLS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tCk1JR0pBb0dC\n\
+QUxGRVVyeVpDbk9ROEhURmV5cDVjMTRObWVqL1BhekFLTTBxRENTNElKUWh0Y3g1\n\
+NXpRSFdOVWIKQ2hHZ0JqR1RjV3ZGRnA0N3FkdGF6WUZhVXE2c0lQKzVqeWZ5b0Q4\n\
+UmJ1bzBwQmFWclJjMmNhYUptWWM0RDh6Vgpuby9sZnhzOVVaQnZ1cWY4eHIrMDB2\n\
+S0JJNmFSMlA2OE1WeDhrMExqcUpUU2RKOE9idm9yQWdNQkFBRT0KLS0tLS1FTkQg\n\
+UlNBIFBVQkxJQyBLRVktLS0tLQpzZXJ2aWNlLWtleQotLS0tLUJFR0lOIFJTQSBQ\n\
+VUJMSUMgS0VZLS0tLS0KTUlHSkFvR0JBTnJHb0ozeTlHNXQzN2F2ekI1cTlwN1hG\n\
+VUplRUVYMUNOaExnWmJXWGJhVk5OcXpoZFhyL0xTUQppM1Z6dW5OaUs3cndUVnE2\n\
+K2QyZ1lRckhMMmIvMXBBY3ZKWjJiNSs0bTRRc0NibFpjRENXTktRbHJnRWN5WXRJ\n\
+CkdscXJTbFFEaXA0ZnNrUFMvNDVkWTI0QmJsQ3NGU1k3RzVLVkxJck4zZFpGbmJr\n\
+NEZIS1hBZ01CQUFFPQotLS0tLUVORCBSU0EgUFVCTElDIEtFWS0tLS0tCmludHJv\n\
+ZHVjdGlvbi1wb2ludCBiNGM3enlxNXNheGZzN2prNXFibG1wN3I1b3pwdHRvagpp\n\
+cC1hZGRyZXNzIDEwOS4xNjkuNDUuMjI2Cm9uaW9uLXBvcnQgOTAwMQpvbmlvbi1r\n\
+ZXkKLS0tLS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tCk1JR0pBb0dCQU8xSXpw\n\
+WFFUTUY3RXZUb1NEUXpzVnZiRVFRQUQrcGZ6NzczMVRXZzVaUEJZY1EyUkRaeVp4\n\
+OEQKNUVQSU1FeUE1RE83cGd0ak5LaXJvYXJGMC8yempjMkRXTUlSaXZyU29YUWVZ\n\
+ZXlMM1pzKzFIajJhMDlCdkYxZAp6MEswblRFdVhoNVR5V3lyMHdsbGI1SFBnTlI0\n\
+MS9oYkprZzkwZitPVCtIeGhKL1duUml2QWdNQkFBRT0KLS0tLS1FTkQgUlNBIFBV\n\
+QkxJQyBLRVktLS0tLQpzZXJ2aWNlLWtleQotLS0tLUJFR0lOIFJTQSBQVUJMSUMg\n\
+S0VZLS0tLS0KTUlHSkFvR0JBSzNWZEJ2ajFtQllLL3JrcHNwcm9Ub0llNUtHVmth\n\
+QkxvMW1tK1I2YUVJek1VZFE1SjkwNGtyRwpCd3k5NC8rV0lGNFpGYXh5Z2phejl1\n\
+N2pKY1k3ZGJhd1pFeG1hYXFCRlRwL2h2ZG9rcHQ4a1ByRVk4OTJPRHJ1CmJORUox\n\
+N1FPSmVMTVZZZk5Kcjl4TWZCQ3JQai8zOGh2RUdrbWVRNmRVWElvbVFNaUJGOVRB\n\
+Z01CQUFFPQotLS0tLUVORCBSU0EgUFVCTElDIEtFWS0tLS0tCmludHJvZHVjdGlv\n\
+bi1wb2ludCBhdjVtcWl0Y2Q3cjJkandsYmN0c2Jlc2R3eGt0ZWtvegppcC1hZGRy\n\
+ZXNzIDE0NC43Ni44LjczCm9uaW9uLXBvcnQgNDQzCm9uaW9uLWtleQotLS0tLUJF\n\
+R0lOIFJTQSBQVUJMSUMgS0VZLS0tLS0KTUlHSkFvR0JBTzVweVZzQmpZQmNmMXBE\n\
+dklHUlpmWXUzQ05nNldka0ZLMGlvdTBXTGZtejZRVDN0NWhzd3cyVwpjejlHMXhx\n\
+MmN0Nkd6VWkrNnVkTDlITTRVOUdHTi9BbW8wRG9GV1hKWHpBQkFXd2YyMVdsd1lW\n\
+eFJQMHRydi9WCkN6UDkzcHc5OG5vSmdGUGRUZ05iMjdKYmVUZENLVFBrTEtscXFt\n\
+b3NveUN2RitRa25vUS9BZ01CQUFFPQotLS0tLUVORCBSU0EgUFVCTElDIEtFWS0t\n\
+LS0tCnNlcnZpY2Uta2V5Ci0tLS0tQkVHSU4gUlNBIFBVQkxJQyBLRVktLS0tLQpN\n\
+SUdKQW9HQkFMVjNKSmtWN3lTNU9jc1lHMHNFYzFQOTVRclFRR3ZzbGJ6Wi9zRGxl\n\
+RlpKYXFSOUYvYjRUVERNClNGcFMxcU1GbldkZDgxVmRGMEdYRmN2WVpLamRJdHU2\n\
+SndBaTRJeEhxeXZtdTRKdUxrcXNaTEFLaXRLVkx4eGsKeERlMjlDNzRWMmJrOTRJ\n\
+MEgybTNKS2tzTHVwc3VxWWRVUmhOVXN0SElKZmgyZmNIalF0bEFnTUJBQUU9Ci0t\n\
+LS0tRU5EIFJTQSBQVUJMSUMgS0VZLS0tLS0KCg==\n\
+-----END MESSAGE-----\n\
+signature\n\
+-----BEGIN SIGNATURE-----\n\
+d4OuCE5OLAOnRB6cQN6WyMEmg/BHem144Vec+eYgeWoKwx3MxXFplUjFxgnMlmwN\n\
+PcftsZf2ztN0sbNCtPgDL3d0PqvxY3iHTQAI8EbaGq/IAJUZ8U4y963dD5+Bn6JQ\n\
+myE3ctmh0vy5+QxSiRjmQBkuEpCyks7LvWvHYrhnmcg=\n\
+-----END SIGNATURE-----";
+
 /* Helper global variable for hidden service descriptor event test.
  * It's used as a pointer to dynamically created message buffer in
  * send_control_event_string_replacement function, which mocks
@@ -124,6 +201,30 @@ node_describe_longname_by_id_replacement(const char *id_digest)
   }
 }
 
+/** Test that we can parse a hardcoded v2 HS desc. */
+static void
+test_hs_parse_static_v2_desc(void *arg)
+{
+  int ret;
+  rend_encoded_v2_service_descriptor_t desc;
+
+  (void) arg;
+
+  /* Test an obviously not parseable string */
+  desc.desc_str = tor_strdup("ceci n'est pas un HS descriptor");
+  ret = rend_desc_v2_is_parsable(&desc);
+  tor_free(desc.desc_str);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Test an actual descriptor */
+  desc.desc_str = tor_strdup(hs_desc_content);
+  ret = rend_desc_v2_is_parsable(&desc);
+  tor_free(desc.desc_str);
+  tt_int_op(ret, OP_EQ, 1);
+
+ done: ;
+}
+
 /** Make sure each hidden service descriptor async event generation
  *
  * function generates the message in expected format.
@@ -136,7 +237,7 @@ test_hs_desc_event(void *arg)
   #define STR_DESC_ID_BASE32 "hba3gmcgpfivzfhx5rtfqkfdhv65yrj3"
 
   int ret;
-  rend_data_t rend_query;
+  rend_data_v2_t rend_query;
   const char *expected_msg;
   char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
 
@@ -148,18 +249,19 @@ test_hs_desc_event(void *arg)
 
   /* setup rend_query struct */
   memset(&rend_query, 0, sizeof(rend_query));
+  rend_query.base_.version = 2;
   strncpy(rend_query.onion_address, STR_HS_ADDR,
           REND_SERVICE_ID_LEN_BASE32+1);
   rend_query.auth_type = REND_NO_AUTH;
-  rend_query.hsdirs_fp = smartlist_new();
-  smartlist_add(rend_query.hsdirs_fp, tor_memdup(HSDIR_EXIST_ID,
-                                                 DIGEST_LEN));
+  rend_query.base_.hsdirs_fp = smartlist_new();
+  smartlist_add(rend_query.base_.hsdirs_fp, tor_memdup(HSDIR_EXIST_ID,
+                                                       DIGEST_LEN));
 
   /* Compute descriptor ID for replica 0, should be STR_DESC_ID_BASE32. */
   ret = rend_compute_v2_desc_id(rend_query.descriptor_id[0],
                                 rend_query.onion_address,
                                 NULL, 0, 0);
-  tt_int_op(ret, ==, 0);
+  tt_int_op(ret, OP_EQ, 0);
   base32_encode(desc_id_base32, sizeof(desc_id_base32),
                 rend_query.descriptor_id[0], DIGEST_LEN);
   /* Make sure rend_compute_v2_desc_id works properly. */
@@ -167,8 +269,9 @@ test_hs_desc_event(void *arg)
             sizeof(desc_id_base32));
 
   /* test request event */
-  control_event_hs_descriptor_requested(&rend_query, HSDIR_EXIST_ID,
-                                        STR_DESC_ID_BASE32);
+  control_event_hs_descriptor_requested(rend_query.onion_address,
+                                        rend_query.auth_type, HSDIR_EXIST_ID,
+                                        STR_DESC_ID_BASE32, NULL);
   expected_msg = "650 HS_DESC REQUESTED "STR_HS_ADDR" NO_AUTH "\
                   STR_HSDIR_EXIST_LONGNAME " " STR_DESC_ID_BASE32 "\r\n";
   tt_assert(received_msg);
@@ -177,8 +280,8 @@ test_hs_desc_event(void *arg)
 
   /* test received event */
   rend_query.auth_type = REND_BASIC_AUTH;
-  control_event_hs_descriptor_received(rend_query.onion_address,
-                                       &rend_query, HSDIR_EXIST_ID);
+  control_event_hsv2_descriptor_received(rend_query.onion_address,
+                                         &rend_query.base_, HSDIR_EXIST_ID);
   expected_msg = "650 HS_DESC RECEIVED "STR_HS_ADDR" BASIC_AUTH "\
                   STR_HSDIR_EXIST_LONGNAME " " STR_DESC_ID_BASE32"\r\n";
   tt_assert(received_msg);
@@ -187,7 +290,7 @@ test_hs_desc_event(void *arg)
 
   /* test failed event */
   rend_query.auth_type = REND_STEALTH_AUTH;
-  control_event_hs_descriptor_failed(&rend_query,
+  control_event_hsv2_descriptor_failed(&rend_query.base_,
                                      HSDIR_NONE_EXIST_ID,
                                      "QUERY_REJECTED");
   expected_msg = "650 HS_DESC FAILED "STR_HS_ADDR" STEALTH_AUTH "\
@@ -198,7 +301,7 @@ test_hs_desc_event(void *arg)
 
   /* test invalid auth type */
   rend_query.auth_type = 999;
-  control_event_hs_descriptor_failed(&rend_query,
+  control_event_hsv2_descriptor_failed(&rend_query.base_,
                                      HSDIR_EXIST_ID,
                                      "QUERY_REJECTED");
   expected_msg = "650 HS_DESC FAILED "STR_HS_ADDR" UNKNOWN "\
@@ -208,21 +311,42 @@ test_hs_desc_event(void *arg)
   tt_str_op(received_msg,OP_EQ, expected_msg);
   tor_free(received_msg);
 
-  /* test valid content. */
+  /* test no HSDir fingerprint type */
+  rend_query.auth_type = REND_NO_AUTH;
+  control_event_hsv2_descriptor_failed(&rend_query.base_, NULL,
+                                     "QUERY_NO_HSDIR");
+  expected_msg = "650 HS_DESC FAILED "STR_HS_ADDR" NO_AUTH " \
+                 "UNKNOWN REASON=QUERY_NO_HSDIR\r\n";
+  tt_assert(received_msg);
+  tt_str_op(received_msg,OP_EQ, expected_msg);
+  tor_free(received_msg);
+
+  /* Test invalid content with no HSDir fingerprint. */
   char *exp_msg;
   control_event_hs_descriptor_content(rend_query.onion_address,
+                                      STR_HS_CONTENT_DESC_ID, NULL, NULL);
+  tor_asprintf(&exp_msg, "650+HS_DESC_CONTENT " STR_HS_ADDR " "\
+               STR_HS_CONTENT_DESC_ID " UNKNOWN" \
+               "\r\n\r\n.\r\n650 OK\r\n");
+  tt_assert(received_msg);
+  tt_str_op(received_msg, OP_EQ, exp_msg);
+  tor_free(received_msg);
+  tor_free(exp_msg);
+
+  /* test valid content. */
+  control_event_hs_descriptor_content(rend_query.onion_address,
                                       STR_HS_CONTENT_DESC_ID, HSDIR_EXIST_ID,
-                                      hs_desc_content);
+                                      hs_desc_content_control);
   tor_asprintf(&exp_msg, "650+HS_DESC_CONTENT " STR_HS_ADDR " "\
                STR_HS_CONTENT_DESC_ID " " STR_HSDIR_EXIST_LONGNAME\
-               "\r\n%s\r\n.\r\n650 OK\r\n", hs_desc_content);
+               "\r\n%s\r\n.\r\n650 OK\r\n", hs_desc_content_control);
 
   tt_assert(received_msg);
   tt_str_op(received_msg, OP_EQ, exp_msg);
   tor_free(received_msg);
   tor_free(exp_msg);
-  SMARTLIST_FOREACH(rend_query.hsdirs_fp, char *, d, tor_free(d));
-  smartlist_free(rend_query.hsdirs_fp);
+  SMARTLIST_FOREACH(rend_query.base_.hsdirs_fp, char *, d, tor_free(d));
+  smartlist_free(rend_query.base_.hsdirs_fp);
 
  done:
   UNMOCK(queue_control_event_string);
@@ -230,75 +354,6 @@ test_hs_desc_event(void *arg)
   tor_free(received_msg);
 }
 
-/* Make sure we always pick the right RP, given a well formatted
- * Tor2webRendezvousPoints value. */
-static void
-test_pick_tor2web_rendezvous_node(void *arg)
-{
-  or_options_t *options = get_options_mutable();
-  const node_t *chosen_rp = NULL;
-  router_crn_flags_t flags = CRN_NEED_DESC;
-  int retval, i;
-  const char *tor2web_rendezvous_str = "test003r";
-
-  (void) arg;
-
-  /* Setup fake routerlist. */
-  helper_setup_fake_routerlist();
-
-  /* Parse Tor2webRendezvousPoints as a routerset. */
-  options->Tor2webRendezvousPoints = routerset_new();
-  retval = routerset_parse(options->Tor2webRendezvousPoints,
-                           tor2web_rendezvous_str,
-                           "test_tor2web_rp");
-  tt_int_op(retval, >=, 0);
-
-  /* Pick rendezvous point. Make sure the correct one is
-     picked. Repeat many times to make sure it works properly. */
-  for (i = 0; i < 50 ; i++) {
-    chosen_rp = pick_tor2web_rendezvous_node(flags, options);
-    tt_assert(chosen_rp);
-    tt_str_op(chosen_rp->ri->nickname, ==, tor2web_rendezvous_str);
-  }
-
- done:
-  routerset_free(options->Tor2webRendezvousPoints);
-}
-
-/* Make sure we never pick an RP if Tor2webRendezvousPoints doesn't
- * correspond to an actual node. */
-static void
-test_pick_bad_tor2web_rendezvous_node(void *arg)
-{
-  or_options_t *options = get_options_mutable();
-  const node_t *chosen_rp = NULL;
-  router_crn_flags_t flags = CRN_NEED_DESC;
-  int retval, i;
-  const char *tor2web_rendezvous_str = "dummy";
-
-  (void) arg;
-
-  /* Setup fake routerlist. */
-  helper_setup_fake_routerlist();
-
-  /* Parse Tor2webRendezvousPoints as a routerset. */
-  options->Tor2webRendezvousPoints = routerset_new();
-  retval = routerset_parse(options->Tor2webRendezvousPoints,
-                           tor2web_rendezvous_str,
-                           "test_tor2web_rp");
-  tt_int_op(retval, >=, 0);
-
-  /* Pick rendezvous point. Since Tor2webRendezvousPoints was set to a
-     dummy value, we shouldn't find any eligible RPs. */
-  for (i = 0; i < 50 ; i++) {
-    chosen_rp = pick_tor2web_rendezvous_node(flags, options);
-    tt_assert(!chosen_rp);
-  }
-
- done:
-  routerset_free(options->Tor2webRendezvousPoints);
-}
-
 /* Make sure rend_data_t is valid at creation, destruction and when
  * duplicated. */
 static void
@@ -322,43 +377,47 @@ test_hs_rend_data(void *arg)
   client = rend_data_client_create(STR_HS_ADDR, desc_id, client_cookie,
                                    REND_NO_AUTH);
   tt_assert(client);
-  tt_int_op(client->auth_type, ==, REND_NO_AUTH);
-  tt_str_op(client->onion_address, OP_EQ, STR_HS_ADDR);
-  tt_mem_op(client->desc_id_fetch, OP_EQ, desc_id, sizeof(desc_id));
-  tt_mem_op(client->descriptor_cookie, OP_EQ, client_cookie,
+  rend_data_v2_t *client_v2 = TO_REND_DATA_V2(client);
+  tt_int_op(client_v2->auth_type, OP_EQ, REND_NO_AUTH);
+  tt_str_op(client_v2->onion_address, OP_EQ, STR_HS_ADDR);
+  tt_mem_op(client_v2->desc_id_fetch, OP_EQ, desc_id, sizeof(desc_id));
+  tt_mem_op(client_v2->descriptor_cookie, OP_EQ, client_cookie,
             sizeof(client_cookie));
   tt_assert(client->hsdirs_fp);
-  tt_int_op(smartlist_len(client->hsdirs_fp), ==, 0);
+  tt_int_op(smartlist_len(client->hsdirs_fp), OP_EQ, 0);
   for (rep = 0; rep < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; rep++) {
-    int ret = rend_compute_v2_desc_id(desc_id, client->onion_address,
-                                      client->descriptor_cookie, now, rep);
+    int ret = rend_compute_v2_desc_id(desc_id, client_v2->onion_address,
+                                      client_v2->descriptor_cookie, now, rep);
     /* That shouldn't never fail. */
-    tt_int_op(ret, ==, 0);
-    tt_mem_op(client->descriptor_id[rep], OP_EQ, desc_id, sizeof(desc_id));
+    tt_int_op(ret, OP_EQ, 0);
+    tt_mem_op(client_v2->descriptor_id[rep], OP_EQ, desc_id,
+              sizeof(desc_id));
   }
   /* The rest should be zeroed because this is a client request. */
-  tt_int_op(tor_digest_is_zero(client->rend_pk_digest), ==, 1);
-  tt_int_op(tor_digest_is_zero(client->rend_cookie), ==, 1);
+  tt_int_op(tor_digest_is_zero(client_v2->rend_pk_digest), OP_EQ, 1);
+  tt_int_op(tor_digest_is_zero(client->rend_cookie), OP_EQ, 1);
 
   /* Test dup(). */
   client_dup = rend_data_dup(client);
   tt_assert(client_dup);
-  tt_int_op(client_dup->auth_type, ==, client->auth_type);
-  tt_str_op(client_dup->onion_address, OP_EQ, client->onion_address);
-  tt_mem_op(client_dup->desc_id_fetch, OP_EQ, client->desc_id_fetch,
-            sizeof(client_dup->desc_id_fetch));
-  tt_mem_op(client_dup->descriptor_cookie, OP_EQ, client->descriptor_cookie,
-            sizeof(client_dup->descriptor_cookie));
+  rend_data_v2_t *client_dup_v2 = TO_REND_DATA_V2(client_dup);
+  tt_int_op(client_dup_v2->auth_type, OP_EQ, client_v2->auth_type);
+  tt_str_op(client_dup_v2->onion_address, OP_EQ, client_v2->onion_address);
+  tt_mem_op(client_dup_v2->desc_id_fetch, OP_EQ, client_v2->desc_id_fetch,
+            sizeof(client_dup_v2->desc_id_fetch));
+  tt_mem_op(client_dup_v2->descriptor_cookie, OP_EQ,
+            client_v2->descriptor_cookie,
+            sizeof(client_dup_v2->descriptor_cookie));
 
   tt_assert(client_dup->hsdirs_fp);
-  tt_int_op(smartlist_len(client_dup->hsdirs_fp), ==, 0);
+  tt_int_op(smartlist_len(client_dup->hsdirs_fp), OP_EQ, 0);
   for (rep = 0; rep < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; rep++) {
-    tt_mem_op(client_dup->descriptor_id[rep], OP_EQ,
-              client->descriptor_id[rep], DIGEST_LEN);
+    tt_mem_op(client_dup_v2->descriptor_id[rep], OP_EQ,
+              client_v2->descriptor_id[rep], DIGEST_LEN);
   }
   /* The rest should be zeroed because this is a client request. */
-  tt_int_op(tor_digest_is_zero(client_dup->rend_pk_digest), ==, 1);
-  tt_int_op(tor_digest_is_zero(client_dup->rend_cookie), ==, 1);
+  tt_int_op(tor_digest_is_zero(client_dup_v2->rend_pk_digest), OP_EQ, 1);
+  tt_int_op(tor_digest_is_zero(client_dup->rend_cookie), OP_EQ, 1);
   rend_data_free(client);
   client = NULL;
   rend_data_free(client_dup);
@@ -373,19 +432,20 @@ test_hs_rend_data(void *arg)
    * zeroed out. */
   client = rend_data_client_create(NULL, desc_id, NULL, REND_BASIC_AUTH);
   tt_assert(client);
-  tt_int_op(client->auth_type, ==, REND_BASIC_AUTH);
-  tt_int_op(strlen(client->onion_address), ==, 0);
-  tt_mem_op(client->desc_id_fetch, OP_EQ, desc_id, sizeof(desc_id));
-  tt_int_op(tor_mem_is_zero(client->descriptor_cookie,
-                            sizeof(client->descriptor_cookie)), ==, 1);
+  client_v2 = TO_REND_DATA_V2(client);
+  tt_int_op(client_v2->auth_type, OP_EQ, REND_BASIC_AUTH);
+  tt_int_op(strlen(client_v2->onion_address), OP_EQ, 0);
+  tt_mem_op(client_v2->desc_id_fetch, OP_EQ, desc_id, sizeof(desc_id));
+  tt_int_op(tor_mem_is_zero(client_v2->descriptor_cookie,
+                            sizeof(client_v2->descriptor_cookie)), OP_EQ, 1);
   tt_assert(client->hsdirs_fp);
-  tt_int_op(smartlist_len(client->hsdirs_fp), ==, 0);
+  tt_int_op(smartlist_len(client->hsdirs_fp), OP_EQ, 0);
   for (rep = 0; rep < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; rep++) {
-    tt_int_op(tor_digest_is_zero(client->descriptor_id[rep]), ==, 1);
+    tt_int_op(tor_digest_is_zero(client_v2->descriptor_id[rep]), OP_EQ, 1);
   }
   /* The rest should be zeroed because this is a client request. */
-  tt_int_op(tor_digest_is_zero(client->rend_pk_digest), ==, 1);
-  tt_int_op(tor_digest_is_zero(client->rend_cookie), ==, 1);
+  tt_int_op(tor_digest_is_zero(client_v2->rend_pk_digest), OP_EQ, 1);
+  tt_int_op(tor_digest_is_zero(client->rend_cookie), OP_EQ, 1);
   rend_data_free(client);
   client = NULL;
 
@@ -398,37 +458,39 @@ test_hs_rend_data(void *arg)
   service = rend_data_service_create(STR_HS_ADDR, rend_pk_digest,
                                      rend_cookie, REND_NO_AUTH);
   tt_assert(service);
-  tt_int_op(service->auth_type, ==, REND_NO_AUTH);
-  tt_str_op(service->onion_address, OP_EQ, STR_HS_ADDR);
-  tt_mem_op(service->rend_pk_digest, OP_EQ, rend_pk_digest,
+  rend_data_v2_t *service_v2 = TO_REND_DATA_V2(service);
+  tt_int_op(service_v2->auth_type, OP_EQ, REND_NO_AUTH);
+  tt_str_op(service_v2->onion_address, OP_EQ, STR_HS_ADDR);
+  tt_mem_op(service_v2->rend_pk_digest, OP_EQ, rend_pk_digest,
             sizeof(rend_pk_digest));
   tt_mem_op(service->rend_cookie, OP_EQ, rend_cookie, sizeof(rend_cookie));
   tt_assert(service->hsdirs_fp);
-  tt_int_op(smartlist_len(service->hsdirs_fp), ==, 0);
+  tt_int_op(smartlist_len(service->hsdirs_fp), OP_EQ, 0);
   for (rep = 0; rep < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; rep++) {
-    tt_int_op(tor_digest_is_zero(service->descriptor_id[rep]), ==, 1);
+    tt_int_op(tor_digest_is_zero(service_v2->descriptor_id[rep]), OP_EQ, 1);
   }
   /* The rest should be zeroed because this is a service request. */
-  tt_int_op(tor_digest_is_zero(service->descriptor_cookie), ==, 1);
-  tt_int_op(tor_digest_is_zero(service->desc_id_fetch), ==, 1);
+  tt_int_op(tor_digest_is_zero(service_v2->descriptor_cookie), OP_EQ, 1);
+  tt_int_op(tor_digest_is_zero(service_v2->desc_id_fetch), OP_EQ, 1);
 
   /* Test dup(). */
   service_dup = rend_data_dup(service);
+  rend_data_v2_t *service_dup_v2 = TO_REND_DATA_V2(service_dup);
   tt_assert(service_dup);
-  tt_int_op(service_dup->auth_type, ==, service->auth_type);
-  tt_str_op(service_dup->onion_address, OP_EQ, service->onion_address);
-  tt_mem_op(service_dup->rend_pk_digest, OP_EQ, service->rend_pk_digest,
-            sizeof(service_dup->rend_pk_digest));
+  tt_int_op(service_dup_v2->auth_type, OP_EQ, service_v2->auth_type);
+  tt_str_op(service_dup_v2->onion_address, OP_EQ, service_v2->onion_address);
+  tt_mem_op(service_dup_v2->rend_pk_digest, OP_EQ, service_v2->rend_pk_digest,
+            sizeof(service_dup_v2->rend_pk_digest));
   tt_mem_op(service_dup->rend_cookie, OP_EQ, service->rend_cookie,
             sizeof(service_dup->rend_cookie));
   tt_assert(service_dup->hsdirs_fp);
-  tt_int_op(smartlist_len(service_dup->hsdirs_fp), ==, 0);
+  tt_int_op(smartlist_len(service_dup->hsdirs_fp), OP_EQ, 0);
   for (rep = 0; rep < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; rep++) {
-    tt_int_op(tor_digest_is_zero(service_dup->descriptor_id[rep]), ==, 1);
+    tt_assert(tor_digest_is_zero(service_dup_v2->descriptor_id[rep]));
   }
   /* The rest should be zeroed because this is a service request. */
-  tt_int_op(tor_digest_is_zero(service_dup->descriptor_cookie), ==, 1);
-  tt_int_op(tor_digest_is_zero(service_dup->desc_id_fetch), ==, 1);
+  tt_int_op(tor_digest_is_zero(service_dup_v2->descriptor_cookie), OP_EQ, 1);
+  tt_int_op(tor_digest_is_zero(service_dup_v2->desc_id_fetch), OP_EQ, 1);
 
  done:
   rend_data_free(service);
@@ -467,7 +529,7 @@ test_hs_auth_cookies(void *arg)
   re = rend_auth_decode_cookie(TEST_COOKIE_ENCODED, raw_cookie, &auth_type,
                                &err_msg);
   tt_assert(!re);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   tt_mem_op(raw_cookie, OP_EQ, TEST_COOKIE_RAW, REND_DESC_COOKIE_LEN);
   tt_int_op(auth_type, OP_EQ, REND_BASIC_AUTH);
   memset(raw_cookie, 0, sizeof(raw_cookie));
@@ -475,7 +537,7 @@ test_hs_auth_cookies(void *arg)
   re = rend_auth_decode_cookie(TEST_COOKIE_ENCODED_STEALTH, raw_cookie,
                                &auth_type, &err_msg);
   tt_assert(!re);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   tt_mem_op(raw_cookie, OP_EQ, TEST_COOKIE_RAW, REND_DESC_COOKIE_LEN);
   tt_int_op(auth_type, OP_EQ, REND_STEALTH_AUTH);
   memset(raw_cookie, 0, sizeof(raw_cookie));
@@ -484,7 +546,7 @@ test_hs_auth_cookies(void *arg)
   re = rend_auth_decode_cookie(TEST_COOKIE_ENCODED "==", raw_cookie, NULL,
                                &err_msg);
   tt_assert(!re);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   tt_mem_op(raw_cookie, OP_EQ, TEST_COOKIE_RAW, REND_DESC_COOKIE_LEN);
 
   /* Decoding with an unknown type should fail */
@@ -545,32 +607,24 @@ test_single_onion_poisoning(void *arg)
   char *dir2 = tor_strdup(get_fname_rnd("test_hs_dir2"));
   smartlist_t *services = smartlist_new();
   char *poison_path = NULL;
+  char *err_msg = NULL;
 
-  /* No services, no service to verify, no problem! */
-  mock_options->HiddenServiceSingleHopMode = 0;
-  mock_options->HiddenServiceNonAnonymousMode = 0;
-  ret = rend_config_services(mock_options, 1);
-  tt_assert(ret == 0);
-
-  /* Either way, no problem. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
-  ret = rend_config_services(mock_options, 1);
-  tt_assert(ret == 0);
 
   /* Create the data directory, and, if the correct bit in arg is set,
    * create a directory for that service.
    * The data directory is required for the lockfile, which is used when
    * loading keys. */
   ret = check_private_dir(mock_options->DataDirectory, CPD_CREATE, NULL);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   if (create_dir_mask & CREATE_HS_DIR1) {
     ret = check_private_dir(dir1, CPD_CREATE, NULL);
-    tt_assert(ret == 0);
+    tt_int_op(ret, OP_EQ, 0);
   }
   if (create_dir_mask & CREATE_HS_DIR2) {
     ret = check_private_dir(dir2, CPD_CREATE, NULL);
-    tt_assert(ret == 0);
+    tt_int_op(ret, OP_EQ, 0);
   }
 
   service_1->directory = dir1;
@@ -580,194 +634,197 @@ test_single_onion_poisoning(void *arg)
   /* Add port to service 1 */
   service_1->ports = smartlist_new();
   service_2->ports = smartlist_new();
-  char *err_msg = NULL;
   rend_service_port_config_t *port1 = rend_service_parse_port_config("80", " ",
                                                                      &err_msg);
   tt_assert(port1);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   smartlist_add(service_1->ports, port1);
 
   rend_service_port_config_t *port2 = rend_service_parse_port_config("90", " ",
                                                                      &err_msg);
   /* Add port to service 2 */
   tt_assert(port2);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   smartlist_add(service_2->ports, port2);
 
   /* No services, a service to verify, no problem! */
   mock_options->HiddenServiceSingleHopMode = 0;
   mock_options->HiddenServiceNonAnonymousMode = 0;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Either way, no problem. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Add the first service */
-  ret = rend_service_check_dir_and_add(services, mock_options, service_1, 0);
-  tt_assert(ret == 0);
+  ret = hs_check_service_private_dir(mock_options->User, service_1->directory,
+                                     service_1->dir_group_readable, 1);
+  tt_int_op(ret, OP_EQ, 0);
+  smartlist_add(services, service_1);
   /* But don't add the second service yet. */
 
   /* Service directories, but no previous keys, no problem! */
   mock_options->HiddenServiceSingleHopMode = 0;
   mock_options->HiddenServiceNonAnonymousMode = 0;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Either way, no problem. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Poison! Poison! Poison!
    * This can only be done in HiddenServiceSingleHopMode. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
   ret = rend_service_poison_new_single_onion_dir(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   /* Poisoning twice is a no-op. */
   ret = rend_service_poison_new_single_onion_dir(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Poisoned service directories, but no previous keys, no problem! */
   mock_options->HiddenServiceSingleHopMode = 0;
   mock_options->HiddenServiceNonAnonymousMode = 0;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Either way, no problem. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Now add some keys, and we'll have a problem. */
   ret = rend_service_load_all_keys(services);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Poisoned service directories with previous keys are not allowed. */
   mock_options->HiddenServiceSingleHopMode = 0;
   mock_options->HiddenServiceNonAnonymousMode = 0;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret < 0);
+  tt_int_op(ret, OP_LT, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* But they are allowed if we're in non-anonymous mode. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Re-poisoning directories with existing keys is a no-op, because
    * directories with existing keys are ignored. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
   ret = rend_service_poison_new_single_onion_dir(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   /* And it keeps the poison. */
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Now add the second service: it has no key and no poison file */
-  ret = rend_service_check_dir_and_add(services, mock_options, service_2, 0);
-  tt_assert(ret == 0);
+  ret = hs_check_service_private_dir(mock_options->User, service_2->directory,
+                                     service_2->dir_group_readable, 1);
+  tt_int_op(ret, OP_EQ, 0);
+  smartlist_add(services, service_2);
 
   /* A new service, and an existing poisoned service. Not ok. */
   mock_options->HiddenServiceSingleHopMode = 0;
   mock_options->HiddenServiceNonAnonymousMode = 0;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret < 0);
+  tt_int_op(ret, OP_LT, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* But ok to add in non-anonymous mode. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Now remove the poisoning from the first service, and we have the opposite
    * problem. */
   poison_path = rend_service_sos_poison_path(service_1);
   tt_assert(poison_path);
   ret = unlink(poison_path);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Unpoisoned service directories with previous keys are ok, as are empty
    * directories. */
   mock_options->HiddenServiceSingleHopMode = 0;
   mock_options->HiddenServiceNonAnonymousMode = 0;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* But the existing unpoisoned key is not ok in non-anonymous mode, even if
    * there is an empty service. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret < 0);
+  tt_int_op(ret, OP_LT, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Poisoning directories with existing keys is a no-op, because directories
    * with existing keys are ignored. But the new directory should poison. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
   ret = rend_service_poison_new_single_onion_dir(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_poison_new_single_onion_dir(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   /* And the old directory remains unpoisoned. */
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret < 0);
+  tt_int_op(ret, OP_LT, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* And the new directory should be ignored, because it has no key. */
   mock_options->HiddenServiceSingleHopMode = 0;
   mock_options->HiddenServiceNonAnonymousMode = 0;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Re-poisoning directories without existing keys is a no-op. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
   ret = rend_service_poison_new_single_onion_dir(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_poison_new_single_onion_dir(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   /* And the old directory remains unpoisoned. */
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret < 0);
+  tt_int_op(ret, OP_LT, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
  done:
   /* The test harness deletes the directories at exit */
@@ -779,17 +836,145 @@ test_single_onion_poisoning(void *arg)
   rend_service_free(service_2);
   UNMOCK(get_options);
   tor_free(mock_options->DataDirectory);
+  tor_free(err_msg);
+}
+
+static rend_service_t *
+helper_create_rend_service(const char *path)
+{
+  rend_service_t *s = tor_malloc_zero(sizeof(rend_service_t));
+  s->ports = smartlist_new();
+  s->intro_nodes = smartlist_new();
+  s->expiring_nodes = smartlist_new();
+  if (path) {
+    s->directory = tor_strdup(path);
+  }
+  return s;
+}
+
+static void
+test_prune_services_on_reload(void *arg)
+{
+  smartlist_t *new = smartlist_new(), *old = smartlist_new();
+  /* Non ephemeral service. */
+  rend_service_t *s1 = helper_create_rend_service("SomePath");
+  /* Create a non ephemeral service with the _same_ path as so we can test the
+   * transfer of introduction point between the same services on reload. */
+  rend_service_t *s2 = helper_create_rend_service(s1->directory);
+  /* Ephemeral service (directory is NULL). */
+  rend_service_t *e1 = helper_create_rend_service(NULL);
+  rend_service_t *e2 = helper_create_rend_service(NULL);
+
+  (void) arg;
+
+  {
+    /* Add both services to the old list. */
+    smartlist_add(old, s1);
+    smartlist_add(old, e1);
+    /* Only put the non ephemeral in the new list. */
+    smartlist_add(new, s1);
+    set_rend_service_list(old);
+    set_rend_rend_service_staging_list(new);
+    rend_service_prune_list_impl_();
+    /* We expect that the ephemeral one is in the new list but removed from
+     * the old one. */
+    tt_int_op(smartlist_len(old), OP_EQ, 1);
+    tt_assert(smartlist_get(old, 0) == s1);
+    tt_int_op(smartlist_len(new), OP_EQ, 2);
+    tt_assert(smartlist_get(new, 0) == s1);
+    tt_assert(smartlist_get(new, 1) == e1);
+    /* Cleanup for next test. */
+    smartlist_clear(new);
+    smartlist_clear(old);
+  }
+
+  {
+    /* This test will make sure that only the ephemeral service is kept if the
+     * new list is empty. The old list should contain only the non ephemeral
+     * one. */
+    smartlist_add(old, s1);
+    smartlist_add(old, e1);
+    set_rend_service_list(old);
+    set_rend_rend_service_staging_list(new);
+    rend_service_prune_list_impl_();
+    tt_int_op(smartlist_len(old), OP_EQ, 1);
+    tt_assert(smartlist_get(old, 0) == s1);
+    tt_int_op(smartlist_len(new), OP_EQ, 1);
+    tt_assert(smartlist_get(new, 0) == e1);
+    /* Cleanup for next test. */
+    smartlist_clear(new);
+    smartlist_clear(old);
+  }
+
+  {
+    /* This test makes sure that the new list stays the same even from the old
+     * list being completely different. */
+    smartlist_add(new, s1);
+    smartlist_add(new, e1);
+    set_rend_service_list(old);
+    set_rend_rend_service_staging_list(new);
+    rend_service_prune_list_impl_();
+    tt_int_op(smartlist_len(old), OP_EQ, 0);
+    tt_int_op(smartlist_len(new), OP_EQ, 2);
+    tt_assert(smartlist_get(new, 0) == s1);
+    tt_assert(smartlist_get(new, 1) == e1);
+    /* Cleanup for next test. */
+    smartlist_clear(new);
+  }
+
+  {
+    rend_intro_point_t ip1;
+    /* This IP should be found in the s2 service after pruning. */
+    smartlist_add(s1->intro_nodes, &ip1);
+    /* Setup our list. */
+    smartlist_add(old, s1);
+    smartlist_add(new, s2);
+    set_rend_service_list(old);
+    set_rend_rend_service_staging_list(new);
+    rend_service_prune_list_impl_();
+    tt_int_op(smartlist_len(old), OP_EQ, 1);
+    /* Intro nodes have been moved to the s2 in theory so it must be empty. */
+    tt_int_op(smartlist_len(s1->intro_nodes), OP_EQ, 0);
+    tt_int_op(smartlist_len(new), OP_EQ, 1);
+    rend_service_t *elem = smartlist_get(new, 0);
+    tt_assert(elem);
+    tt_assert(elem == s2);
+    tt_int_op(smartlist_len(elem->intro_nodes), OP_EQ, 1);
+    tt_assert(smartlist_get(elem->intro_nodes, 0) == &ip1);
+    smartlist_clear(s1->intro_nodes);
+    smartlist_clear(s2->intro_nodes);
+    /* Cleanup for next test. */
+    smartlist_clear(new);
+    smartlist_clear(old);
+  }
+
+  {
+    /* Test two ephemeral services. */
+    smartlist_add(old, e1);
+    smartlist_add(old, e2);
+    set_rend_service_list(old);
+    set_rend_rend_service_staging_list(new);
+    rend_service_prune_list_impl_();
+    /* Check if they've all been transferred. */
+    tt_int_op(smartlist_len(old), OP_EQ, 0);
+    tt_int_op(smartlist_len(new), OP_EQ, 2);
+  }
+
+ done:
+  rend_service_free(s1);
+  rend_service_free(s2);
+  rend_service_free(e1);
+  rend_service_free(e2);
+  smartlist_free(new);
+  smartlist_free(old);
 }
 
 struct testcase_t hs_tests[] = {
   { "hs_rend_data", test_hs_rend_data, TT_FORK,
     NULL, NULL },
-  { "hs_desc_event", test_hs_desc_event, TT_FORK,
-    NULL, NULL },
-  { "pick_tor2web_rendezvous_node", test_pick_tor2web_rendezvous_node, TT_FORK,
+  { "hs_parse_static_v2_desc", test_hs_parse_static_v2_desc, TT_FORK,
     NULL, NULL },
-  { "pick_bad_tor2web_rendezvous_node",
-    test_pick_bad_tor2web_rendezvous_node, TT_FORK,
+  { "hs_desc_event", test_hs_desc_event, TT_FORK,
     NULL, NULL },
   { "hs_auth_cookies", test_hs_auth_cookies, TT_FORK,
     NULL, NULL },
@@ -801,6 +986,8 @@ struct testcase_t hs_tests[] = {
     TT_FORK, &passthrough_setup, (void*)(CREATE_HS_DIR2) },
   { "single_onion_poisoning_create_dir_both", test_single_onion_poisoning,
     TT_FORK, &passthrough_setup, (void*)(CREATE_HS_DIR1 | CREATE_HS_DIR2) },
+  { "prune_services_on_reload", test_prune_services_on_reload, TT_FORK,
+    NULL, NULL },
+
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_hs_cache.c b/src/test/test_hs_cache.c
new file mode 100644
index 0000000000..9182829116
--- /dev/null
+++ b/src/test/test_hs_cache.c
@@ -0,0 +1,566 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_cache.c
+ * \brief Test hidden service caches.
+ */
+
+#define CONNECTION_PRIVATE
+#define DIRCACHE_PRIVATE
+#define DIRCLIENT_PRIVATE
+#define HS_CACHE_PRIVATE
+
+#include "trunnel/ed25519_cert.h"
+#include "feature/hs/hs_cache.h"
+#include "feature/rend/rendcache.h"
+#include "feature/dircache/dircache.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/nodelist/networkstatus.h"
+#include "core/mainloop/connection.h"
+#include "core/proto/proto_http.h"
+#include "lib/crypt_ops/crypto_format.h"
+
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+
+#include "test/hs_test_helpers.h"
+#include "test/test_helpers.h"
+#include "test/test.h"
+
+/* Static variable used to encoded the HSDir query. */
+static char query_b64[256];
+
+/* Build an HSDir query using a ed25519 public key. */
+static const char *
+helper_get_hsdir_query(const hs_descriptor_t *desc)
+{
+  ed25519_public_to_base64(query_b64, &desc->plaintext_data.blinded_pubkey);
+  return query_b64;
+}
+
+static void
+init_test(void)
+{
+  /* Always needed. Initialize the subsystem. */
+  hs_cache_init();
+  /* We need the v2 cache since our OOM and cache cleanup does poke at it. */
+  rend_cache_init();
+}
+
+static void
+test_directory(void *arg)
+{
+  int ret;
+  size_t oom_size;
+  char *desc1_str = NULL;
+  const char *desc_out;
+  ed25519_keypair_t signing_kp1;
+  hs_descriptor_t *desc1 = NULL;
+
+  (void) arg;
+
+  init_test();
+  /* Generate a valid descriptor with normal values. */
+  ret = ed25519_keypair_generate(&signing_kp1, 0);
+  tt_int_op(ret, OP_EQ, 0);
+  desc1 = hs_helper_build_hs_desc_with_ip(&signing_kp1);
+  tt_assert(desc1);
+  ret = hs_desc_encode_descriptor(desc1, &signing_kp1, NULL, &desc1_str);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Very first basic test, should be able to be stored, survive a
+   * clean, found with a lookup and then cleaned by our OOM. */
+  {
+    ret = hs_cache_store_as_dir(desc1_str);
+    tt_int_op(ret, OP_EQ, 0);
+    /* Re-add, it should fail since we already have it. */
+    ret = hs_cache_store_as_dir(desc1_str);
+    tt_int_op(ret, OP_EQ, -1);
+    /* Try to clean now which should be fine, there is at worst few seconds
+     * between the store and this call. */
+    hs_cache_clean_as_dir(time(NULL));
+    /* We should find it in our cache. */
+    ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), &desc_out);
+    tt_int_op(ret, OP_EQ, 1);
+    tt_str_op(desc_out, OP_EQ, desc1_str);
+    /* Tell our OOM to run and to at least remove a byte which will result in
+     * removing the descriptor from our cache. */
+    oom_size = hs_cache_handle_oom(time(NULL), 1);
+    tt_int_op(oom_size, OP_GE, 1);
+    ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), NULL);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Store two descriptors and remove the expiring one only. */
+  {
+    ed25519_keypair_t signing_kp_zero;
+    ret = ed25519_keypair_generate(&signing_kp_zero, 0);
+    tt_int_op(ret, OP_EQ, 0);
+    hs_descriptor_t *desc_zero_lifetime;
+    desc_zero_lifetime = hs_helper_build_hs_desc_with_ip(&signing_kp_zero);
+    tt_assert(desc_zero_lifetime);
+    desc_zero_lifetime->plaintext_data.revision_counter = 1;
+    desc_zero_lifetime->plaintext_data.lifetime_sec = 0;
+    char *desc_zero_lifetime_str;
+    ret = hs_desc_encode_descriptor(desc_zero_lifetime, &signing_kp_zero,
+                                    NULL, &desc_zero_lifetime_str);
+    tt_int_op(ret, OP_EQ, 0);
+
+    ret = hs_cache_store_as_dir(desc1_str);
+    tt_int_op(ret, OP_EQ, 0);
+    ret = hs_cache_store_as_dir(desc_zero_lifetime_str);
+    tt_int_op(ret, OP_EQ, 0);
+    /* This one should clear out our zero lifetime desc. */
+    hs_cache_clean_as_dir(time(NULL));
+    /* We should find desc1 in our cache. */
+    ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), &desc_out);
+    tt_int_op(ret, OP_EQ, 1);
+    tt_str_op(desc_out, OP_EQ, desc1_str);
+    /* We should NOT find our zero lifetime desc in our cache. */
+    ret = hs_cache_lookup_as_dir(3,
+                                 helper_get_hsdir_query(desc_zero_lifetime),
+                                 NULL);
+    tt_int_op(ret, OP_EQ, 0);
+    /* Cleanup our entire cache. */
+    oom_size = hs_cache_handle_oom(time(NULL), 1);
+    tt_int_op(oom_size, OP_GE, 1);
+    hs_descriptor_free(desc_zero_lifetime);
+    tor_free(desc_zero_lifetime_str);
+  }
+
+  /* Throw junk at it. */
+  {
+    ret = hs_cache_store_as_dir("blah");
+    tt_int_op(ret, OP_EQ, -1);
+    /* Poor attempt at tricking the decoding. */
+    ret = hs_cache_store_as_dir("hs-descriptor 3\nJUNK");
+    tt_int_op(ret, OP_EQ, -1);
+    /* Undecodable base64 query. */
+    ret = hs_cache_lookup_as_dir(3, "blah", NULL);
+    tt_int_op(ret, OP_EQ, -1);
+    /* Decodable base64 query but wrong ed25519 size. */
+    ret = hs_cache_lookup_as_dir(3, "dW5pY29ybg==", NULL);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Test descriptor replacement with revision counter. */
+  {
+    char *new_desc_str;
+
+    /* Add a descriptor. */
+    ret = hs_cache_store_as_dir(desc1_str);
+    tt_int_op(ret, OP_EQ, 0);
+    ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), &desc_out);
+    tt_int_op(ret, OP_EQ, 1);
+    /* Bump revision counter. */
+    desc1->plaintext_data.revision_counter++;
+    ret = hs_desc_encode_descriptor(desc1, &signing_kp1, NULL, &new_desc_str);
+    tt_int_op(ret, OP_EQ, 0);
+    ret = hs_cache_store_as_dir(new_desc_str);
+    tt_int_op(ret, OP_EQ, 0);
+    /* Look it up, it should have been replaced. */
+    ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), &desc_out);
+    tt_int_op(ret, OP_EQ, 1);
+    tt_str_op(desc_out, OP_EQ, new_desc_str);
+    tor_free(new_desc_str);
+  }
+
+ done:
+  hs_descriptor_free(desc1);
+  tor_free(desc1_str);
+}
+
+static void
+test_clean_as_dir(void *arg)
+{
+  size_t ret;
+  char *desc1_str = NULL;
+  time_t now = time(NULL);
+  hs_descriptor_t *desc1 = NULL;
+  ed25519_keypair_t signing_kp1;
+
+  (void) arg;
+
+  init_test();
+
+  /* Generate a valid descriptor with values. */
+  ret = ed25519_keypair_generate(&signing_kp1, 0);
+  tt_int_op(ret, OP_EQ, 0);
+  desc1 = hs_helper_build_hs_desc_with_ip(&signing_kp1);
+  tt_assert(desc1);
+  ret = hs_desc_encode_descriptor(desc1, &signing_kp1, NULL, &desc1_str);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = hs_cache_store_as_dir(desc1_str);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* With the lifetime being 3 hours, a cleanup shouldn't remove it. */
+  ret = cache_clean_v3_as_dir(now, 0);
+  tt_int_op(ret, OP_EQ, 0);
+  /* Should be present after clean up. */
+  ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), NULL);
+  tt_int_op(ret, OP_EQ, 1);
+  /* Set a cutoff 100 seconds in the past. It should not remove the entry
+   * since the entry is still recent enough. */
+  ret = cache_clean_v3_as_dir(now, now - 100);
+  tt_int_op(ret, OP_EQ, 0);
+  /* Should be present after clean up. */
+  ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), NULL);
+  tt_int_op(ret, OP_EQ, 1);
+  /* Set a cutoff of 100 seconds in the future. It should remove the entry
+   * that we've just added since it's not too old for the cutoff. */
+  ret = cache_clean_v3_as_dir(now, now + 100);
+  tt_int_op(ret, OP_GT, 0);
+  /* Shouldn't be present after clean up. */
+  ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), NULL);
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  hs_descriptor_free(desc1);
+  tor_free(desc1_str);
+}
+
+/* Test helper: Fetch an HS descriptor from an HSDir (for the hidden service
+   with <b>blinded_key</b>. Return the received descriptor string. */
+static char *
+helper_fetch_desc_from_hsdir(const ed25519_public_key_t *blinded_key)
+{
+  int retval;
+
+  char *received_desc = NULL;
+  char *hsdir_query_str = NULL;
+
+  /* The dir conn we are going to simulate */
+  dir_connection_t *conn = NULL;
+
+  /* First extract the blinded public key that we are going to use in our
+     query, and then build the actual query string. */
+  {
+    char hsdir_cache_key[ED25519_BASE64_LEN+1];
+
+    retval = ed25519_public_to_base64(hsdir_cache_key,
+                                      blinded_key);
+    tt_int_op(retval, OP_EQ, 0);
+    tor_asprintf(&hsdir_query_str, GET("/tor/hs/3/%s"), hsdir_cache_key);
+  }
+
+  /* Simulate an HTTP GET request to the HSDir */
+  conn = dir_connection_new(AF_INET);
+  tor_addr_from_ipv4h(&conn->base_.addr, 0x7f000001);
+  TO_CONN(conn)->linked = 1;/* Pretend the conn is encrypted :) */
+  retval = directory_handle_command_get(conn, hsdir_query_str,
+                                        NULL, 0);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Read the descriptor that the HSDir just served us */
+  {
+    char *headers = NULL;
+    size_t body_used = 0;
+
+    fetch_from_buf_http(TO_CONN(conn)->outbuf, &headers, MAX_HEADERS_SIZE,
+                        &received_desc, &body_used, HS_DESC_MAX_LEN, 0);
+    tor_free(headers);
+  }
+
+ done:
+  tor_free(hsdir_query_str);
+  if (conn)
+    connection_free_minimal(TO_CONN(conn));
+
+  return received_desc;
+}
+
+/* Publish a descriptor to the HSDir, then fetch it. Check that the received
+   descriptor matches the published one. */
+static void
+test_upload_and_download_hs_desc(void *arg)
+{
+  int retval;
+  hs_descriptor_t *published_desc = NULL;
+
+  char *published_desc_str = NULL;
+  char *received_desc_str = NULL;
+
+  (void) arg;
+
+  /* Initialize HSDir cache subsystem */
+  init_test();
+
+  /* Test a descriptor not found in the directory cache. */
+  {
+    ed25519_public_key_t blinded_key;
+    memset(&blinded_key.pubkey, 'A', sizeof(blinded_key.pubkey));
+    received_desc_str = helper_fetch_desc_from_hsdir(&blinded_key);
+    tt_int_op(strlen(received_desc_str), OP_EQ, 0);
+    tor_free(received_desc_str);
+  }
+
+  /* Generate a valid descriptor with normal values. */
+  {
+    ed25519_keypair_t signing_kp;
+    retval = ed25519_keypair_generate(&signing_kp, 0);
+    tt_int_op(retval, OP_EQ, 0);
+    published_desc = hs_helper_build_hs_desc_with_ip(&signing_kp);
+    tt_assert(published_desc);
+    retval = hs_desc_encode_descriptor(published_desc, &signing_kp,
+                                       NULL, &published_desc_str);
+    tt_int_op(retval, OP_EQ, 0);
+  }
+
+  /* Publish descriptor to the HSDir */
+  {
+    retval = handle_post_hs_descriptor("/tor/hs/3/publish",published_desc_str);
+    tt_int_op(retval, OP_EQ, 200);
+  }
+
+  /* Simulate a fetch of the previously published descriptor */
+  {
+    const ed25519_public_key_t *blinded_key;
+    blinded_key = &published_desc->plaintext_data.blinded_pubkey;
+    received_desc_str = helper_fetch_desc_from_hsdir(blinded_key);
+  }
+
+  /* Verify we received the exact same descriptor we published earlier */
+  tt_str_op(received_desc_str, OP_EQ, published_desc_str);
+  tor_free(received_desc_str);
+
+  /* With a valid descriptor in the directory cache, try again an invalid. */
+  {
+    ed25519_public_key_t blinded_key;
+    memset(&blinded_key.pubkey, 'A', sizeof(blinded_key.pubkey));
+    received_desc_str = helper_fetch_desc_from_hsdir(&blinded_key);
+    tt_int_op(strlen(received_desc_str), OP_EQ, 0);
+  }
+
+ done:
+  tor_free(received_desc_str);
+  tor_free(published_desc_str);
+  hs_descriptor_free(published_desc);
+}
+
+/* Test that HSDirs reject outdated descriptors based on their revision
+ * counter. Also test that HSDirs correctly replace old descriptors with newer
+ * descriptors. */
+static void
+test_hsdir_revision_counter_check(void *arg)
+{
+  int retval;
+
+  ed25519_keypair_t signing_kp;
+
+  hs_descriptor_t *published_desc = NULL;
+  char *published_desc_str = NULL;
+
+  uint8_t subcredential[DIGEST256_LEN];
+  char *received_desc_str = NULL;
+  hs_descriptor_t *received_desc = NULL;
+
+  (void) arg;
+
+  /* Initialize HSDir cache subsystem */
+  init_test();
+
+  /* Generate a valid descriptor with normal values. */
+  {
+    retval = ed25519_keypair_generate(&signing_kp, 0);
+    tt_int_op(retval, OP_EQ, 0);
+    published_desc = hs_helper_build_hs_desc_with_ip(&signing_kp);
+    tt_assert(published_desc);
+    retval = hs_desc_encode_descriptor(published_desc, &signing_kp,
+                                       NULL, &published_desc_str);
+    tt_int_op(retval, OP_EQ, 0);
+  }
+
+  /* Publish descriptor to the HSDir */
+  {
+    retval = handle_post_hs_descriptor("/tor/hs/3/publish",published_desc_str);
+    tt_int_op(retval, OP_EQ, 200);
+  }
+
+  /* Try publishing again with the same revision counter: Should fail. */
+  {
+    retval = handle_post_hs_descriptor("/tor/hs/3/publish",published_desc_str);
+    tt_int_op(retval, OP_EQ, 400);
+  }
+
+  /* Fetch the published descriptor and validate the revision counter. */
+  {
+    const ed25519_public_key_t *blinded_key;
+
+    blinded_key = &published_desc->plaintext_data.blinded_pubkey;
+    hs_get_subcredential(&signing_kp.pubkey, blinded_key, subcredential);
+    received_desc_str = helper_fetch_desc_from_hsdir(blinded_key);
+
+    retval = hs_desc_decode_descriptor(received_desc_str,
+                                       subcredential, NULL, &received_desc);
+    tt_int_op(retval, OP_EQ, 0);
+    tt_assert(received_desc);
+
+    /* Check that the revision counter is correct */
+    tt_u64_op(received_desc->plaintext_data.revision_counter, OP_EQ, 42);
+
+    hs_descriptor_free(received_desc);
+    received_desc = NULL;
+    tor_free(received_desc_str);
+  }
+
+  /* Increment the revision counter and try again. Should work. */
+  {
+    published_desc->plaintext_data.revision_counter = 1313;
+    tor_free(published_desc_str);
+    retval = hs_desc_encode_descriptor(published_desc, &signing_kp,
+                                       NULL, &published_desc_str);
+    tt_int_op(retval, OP_EQ, 0);
+
+    retval = handle_post_hs_descriptor("/tor/hs/3/publish",published_desc_str);
+    tt_int_op(retval, OP_EQ, 200);
+  }
+
+  /* Again, fetch the published descriptor and perform the revision counter
+     validation. The revision counter must have changed. */
+  {
+    const ed25519_public_key_t *blinded_key;
+
+    blinded_key = &published_desc->plaintext_data.blinded_pubkey;
+    received_desc_str = helper_fetch_desc_from_hsdir(blinded_key);
+
+    retval = hs_desc_decode_descriptor(received_desc_str,
+                                       subcredential, NULL, &received_desc);
+    tt_int_op(retval, OP_EQ, 0);
+    tt_assert(received_desc);
+
+    /* Check that the revision counter is the latest */
+    tt_u64_op(received_desc->plaintext_data.revision_counter, OP_EQ, 1313);
+  }
+
+ done:
+  hs_descriptor_free(published_desc);
+  hs_descriptor_free(received_desc);
+  tor_free(received_desc_str);
+  tor_free(published_desc_str);
+}
+
+static networkstatus_t mock_ns;
+
+static networkstatus_t *
+mock_networkstatus_get_live_consensus(time_t now)
+{
+  (void) now;
+  return &mock_ns;
+}
+
+/** Test that we can store HS descriptors in the client HS cache. */
+static void
+test_client_cache(void *arg)
+{
+  int retval;
+  ed25519_keypair_t signing_kp;
+  hs_descriptor_t *published_desc = NULL;
+  char *published_desc_str = NULL;
+  uint8_t wanted_subcredential[DIGEST256_LEN];
+  response_handler_args_t *args = NULL;
+  dir_connection_t *conn = NULL;
+
+  (void) arg;
+
+  /* Initialize HSDir cache subsystem */
+  init_test();
+
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  /* Set consensus time */
+  parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                           &mock_ns.valid_after);
+  parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                           &mock_ns.fresh_until);
+  parse_rfc1123_time("Sat, 26 Oct 1985 16:00:00 UTC",
+                           &mock_ns.valid_until);
+
+  /* Generate a valid descriptor with normal values. */
+  {
+    retval = ed25519_keypair_generate(&signing_kp, 0);
+    tt_int_op(retval, OP_EQ, 0);
+    published_desc = hs_helper_build_hs_desc_with_ip(&signing_kp);
+    tt_assert(published_desc);
+    retval = hs_desc_encode_descriptor(published_desc, &signing_kp,
+                                       NULL, &published_desc_str);
+    tt_int_op(retval, OP_EQ, 0);
+    memcpy(wanted_subcredential, published_desc->subcredential, DIGEST256_LEN);
+    tt_assert(!tor_mem_is_zero((char*)wanted_subcredential, DIGEST256_LEN));
+  }
+
+  /* Test handle_response_fetch_hsdesc_v3() */
+  {
+    args = tor_malloc_zero(sizeof(response_handler_args_t));
+    args->status_code = 200;
+    args->reason = NULL;
+    args->body = published_desc_str;
+    args->body_len = strlen(published_desc_str);
+
+    conn = tor_malloc_zero(sizeof(dir_connection_t));
+    conn->hs_ident = tor_malloc_zero(sizeof(hs_ident_dir_conn_t));
+    ed25519_pubkey_copy(&conn->hs_ident->identity_pk, &signing_kp.pubkey);
+  }
+
+  /* store the descriptor! */
+  retval = handle_response_fetch_hsdesc_v3(conn, args);
+  tt_int_op(retval, == , 0);
+
+  /* Progress time a bit and attempt to clean cache: our desc should not be
+   * cleaned since we still in the same TP. */
+  {
+    parse_rfc1123_time("Sat, 27 Oct 1985 02:00:00 UTC",
+                       &mock_ns.valid_after);
+    parse_rfc1123_time("Sat, 27 Oct 1985 03:00:00 UTC",
+                       &mock_ns.fresh_until);
+    parse_rfc1123_time("Sat, 27 Oct 1985 05:00:00 UTC",
+                       &mock_ns.valid_until);
+
+    /* fetch the descriptor and make sure it's there */
+    const hs_descriptor_t *cached_desc = NULL;
+    cached_desc = hs_cache_lookup_as_client(&signing_kp.pubkey);
+    tt_assert(cached_desc);
+    tt_mem_op(cached_desc->subcredential, OP_EQ, wanted_subcredential,
+              DIGEST256_LEN);
+  }
+
+  /* Progress time to next TP and check that desc was cleaned */
+  {
+    parse_rfc1123_time("Sat, 27 Oct 1985 12:00:00 UTC",
+                       &mock_ns.valid_after);
+    parse_rfc1123_time("Sat, 27 Oct 1985 13:00:00 UTC",
+                       &mock_ns.fresh_until);
+    parse_rfc1123_time("Sat, 27 Oct 1985 15:00:00 UTC",
+                       &mock_ns.valid_until);
+
+    const hs_descriptor_t *cached_desc = NULL;
+    cached_desc = hs_cache_lookup_as_client(&signing_kp.pubkey);
+    tt_assert(!cached_desc);
+  }
+
+ done:
+  tor_free(args);
+  hs_descriptor_free(published_desc);
+  tor_free(published_desc_str);
+  if (conn) {
+    tor_free(conn->hs_ident);
+    tor_free(conn);
+  }
+}
+
+struct testcase_t hs_cache[] = {
+  /* Encoding tests. */
+  { "directory", test_directory, TT_FORK,
+    NULL, NULL },
+  { "clean_as_dir", test_clean_as_dir, TT_FORK,
+    NULL, NULL },
+  { "hsdir_revision_counter_check", test_hsdir_revision_counter_check, TT_FORK,
+    NULL, NULL },
+  { "upload_and_download_hs_desc", test_upload_and_download_hs_desc, TT_FORK,
+    NULL, NULL },
+  { "client_cache", test_client_cache, TT_FORK,
+    NULL, NULL },
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_hs_cell.c b/src/test/test_hs_cell.c
new file mode 100644
index 0000000000..f8af631c8b
--- /dev/null
+++ b/src/test/test_hs_cell.c
@@ -0,0 +1,131 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_cell.c
+ * \brief Test hidden service cell functionality.
+ */
+
+#define HS_INTROPOINT_PRIVATE
+#define HS_SERVICE_PRIVATE
+
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
+
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "feature/hs/hs_cell.h"
+#include "feature/hs/hs_intropoint.h"
+#include "feature/hs/hs_service.h"
+
+/* Trunnel. */
+#include "trunnel/hs/cell_establish_intro.h"
+
+/** We simulate the creation of an outgoing ESTABLISH_INTRO cell, and then we
+ *  parse it from the receiver side. */
+static void
+test_gen_establish_intro_cell(void *arg)
+{
+  (void) arg;
+  ssize_t ret;
+  char circ_nonce[DIGEST_LEN] = {0};
+  uint8_t buf[RELAY_PAYLOAD_SIZE];
+  trn_cell_establish_intro_t *cell_in = NULL;
+
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+     attempt to parse it. */
+  {
+    /* We only need the auth key pair here. */
+    hs_service_intro_point_t *ip = service_intro_point_new(NULL, 0, 0);
+    /* Auth key pair is generated in the constructor so we are all set for
+     * using this IP object. */
+    ret = hs_cell_build_establish_intro(circ_nonce, ip, buf);
+    service_intro_point_free(ip);
+    tt_u64_op(ret, OP_GT, 0);
+  }
+
+  /* Check the contents of the cell */
+  {
+    /* First byte is the auth key type: make sure its correct */
+    tt_int_op(buf[0], OP_EQ, TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_ED25519);
+    /* Next two bytes is auth key len */
+    tt_int_op(ntohs(get_uint16(buf+1)), OP_EQ, ED25519_PUBKEY_LEN);
+    /* Skip to the number of extensions: no extensions */
+    tt_int_op(buf[35], OP_EQ, 0);
+    /* Skip to the sig len. Make sure it's the size of an ed25519 sig */
+    tt_int_op(ntohs(get_uint16(buf+35+1+32)), OP_EQ, ED25519_SIG_LEN);
+  }
+
+  /* Parse it as the receiver */
+  {
+    ret = trn_cell_establish_intro_parse(&cell_in, buf, sizeof(buf));
+    tt_u64_op(ret, OP_GT, 0);
+
+    ret = verify_establish_intro_cell(cell_in,
+                                      (const uint8_t *) circ_nonce,
+                                      sizeof(circ_nonce));
+    tt_u64_op(ret, OP_EQ, 0);
+  }
+
+ done:
+  trn_cell_establish_intro_free(cell_in);
+}
+
+/* Mocked ed25519_sign_prefixed() function that always fails :) */
+static int
+mock_ed25519_sign_prefixed(ed25519_signature_t *signature_out,
+                           const uint8_t *msg, size_t msg_len,
+                           const char *prefix_str,
+                           const ed25519_keypair_t *keypair) {
+  (void) signature_out;
+  (void) msg;
+  (void) msg_len;
+  (void) prefix_str;
+  (void) keypair;
+  return -1;
+}
+
+/** We simulate a failure to create an ESTABLISH_INTRO cell */
+static void
+test_gen_establish_intro_cell_bad(void *arg)
+{
+  (void) arg;
+  ssize_t cell_len = 0;
+  trn_cell_establish_intro_t *cell = NULL;
+  char circ_nonce[DIGEST_LEN] = {0};
+  hs_service_intro_point_t *ip = NULL;
+
+  MOCK(ed25519_sign_prefixed, mock_ed25519_sign_prefixed);
+
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+
+  setup_full_capture_of_logs(LOG_WARN);
+  /* Easiest way to make that function fail is to mock the
+     ed25519_sign_prefixed() function and make it fail. */
+  cell = trn_cell_establish_intro_new();
+  tt_assert(cell);
+  ip = service_intro_point_new(NULL, 0, 0);
+  cell_len = hs_cell_build_establish_intro(circ_nonce, ip, NULL);
+  service_intro_point_free(ip);
+  expect_log_msg_containing("Unable to make signature for "
+                            "ESTABLISH_INTRO cell.");
+  teardown_capture_of_logs();
+  tt_i64_op(cell_len, OP_EQ, -1);
+
+ done:
+  trn_cell_establish_intro_free(cell);
+  UNMOCK(ed25519_sign_prefixed);
+}
+
+struct testcase_t hs_cell_tests[] = {
+  { "gen_establish_intro_cell", test_gen_establish_intro_cell, TT_FORK,
+    NULL, NULL },
+  { "gen_establish_intro_cell_bad", test_gen_establish_intro_cell_bad, TT_FORK,
+    NULL, NULL },
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c
new file mode 100644
index 0000000000..2f2bb45581
--- /dev/null
+++ b/src/test/test_hs_client.c
@@ -0,0 +1,1010 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_client.c
+ * \brief Test prop224 HS client functionality.
+ */
+
+#define CONFIG_PRIVATE
+#define CRYPTO_PRIVATE
+#define MAINLOOP_PRIVATE
+#define HS_CLIENT_PRIVATE
+#define TOR_CHANNEL_INTERNAL_
+#define CIRCUITBUILD_PRIVATE
+#define CIRCUITLIST_PRIVATE
+#define CONNECTION_PRIVATE
+
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
+#include "test/rend_test_helpers.h"
+#include "test/hs_test_helpers.h"
+
+#include "app/config/config.h"
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "lib/crypt_ops/crypto_dh.h"
+#include "core/or/channeltls.h"
+#include "feature/dircommon/directory.h"
+#include "core/mainloop/mainloop.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerset.h"
+
+#include "feature/hs/hs_circuit.h"
+#include "feature/hs/hs_circuitmap.h"
+#include "feature/hs/hs_client.h"
+#include "feature/hs/hs_config.h"
+#include "feature/hs/hs_ident.h"
+#include "feature/hs/hs_cache.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitbuild.h"
+#include "core/mainloop/connection.h"
+#include "core/or/connection_edge.h"
+#include "feature/nodelist/networkstatus.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/crypt_path_st.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/extend_info_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
+
+static int
+mock_connection_ap_handshake_send_begin(entry_connection_t *ap_conn)
+{
+  (void) ap_conn;
+  return 0;
+}
+
+static networkstatus_t mock_ns;
+
+/* Always return NULL. */
+static networkstatus_t *
+mock_networkstatus_get_live_consensus_false(time_t now)
+{
+  (void) now;
+  return NULL;
+}
+
+static networkstatus_t *
+mock_networkstatus_get_live_consensus(time_t now)
+{
+  (void) now;
+  return &mock_ns;
+}
+
+static int
+helper_config_client(const char *conf, int validate_only)
+{
+  int ret = 0;
+  or_options_t *options = NULL;
+  tt_assert(conf);
+  options = helper_parse_options(conf);
+  tt_assert(options);
+  ret = hs_config_client_auth_all(options, validate_only);
+ done:
+  or_options_free(options);
+  return ret;
+}
+
+/* Test helper function: Setup a circuit and a stream with the same hidden
+ * service destination, and put them in <b>circ_out</b> and
+ * <b>conn_out</b>. Make the stream wait for circuits to be established to the
+ * hidden service. */
+static int
+helper_get_circ_and_stream_for_test(origin_circuit_t **circ_out,
+                                    connection_t **conn_out,
+                                    int is_legacy)
+{
+  int retval;
+  channel_tls_t *n_chan=NULL;
+  rend_data_t *conn_rend_data = NULL;
+  origin_circuit_t *or_circ = NULL;
+  connection_t *conn = NULL;
+  ed25519_public_key_t service_pk;
+
+  /* Make a dummy connection stream and make it wait for our circuit */
+  conn = test_conn_get_connection(AP_CONN_STATE_CIRCUIT_WAIT,
+                                  CONN_TYPE_AP /* ??? */,
+                                  0);
+  if (is_legacy) {
+    /* Legacy: Setup rend_data of stream */
+    char service_id[REND_SERVICE_ID_LEN_BASE32+1] = {0};
+    TO_EDGE_CONN(conn)->rend_data = mock_rend_data(service_id);
+    conn_rend_data = TO_EDGE_CONN(conn)->rend_data;
+  } else {
+    /* prop224: Setup hs conn identifier on the stream */
+    ed25519_secret_key_t sk;
+    tt_int_op(0, OP_EQ, ed25519_secret_key_generate(&sk, 0));
+    tt_int_op(0, OP_EQ, ed25519_public_key_generate(&service_pk, &sk));
+
+    /* Setup hs_conn_identifier of stream */
+    TO_EDGE_CONN(conn)->hs_ident = hs_ident_edge_conn_new(&service_pk);
+  }
+
+  /* Make it wait for circuit */
+  connection_ap_mark_as_pending_circuit(TO_ENTRY_CONN(conn));
+
+  /* This is needed to silence a BUG warning from
+     connection_edge_update_circuit_isolation() */
+  TO_ENTRY_CONN(conn)->original_dest_address =
+    tor_strdup(TO_ENTRY_CONN(conn)->socks_request->address);
+
+  /****************************************************/
+
+  /* Now make dummy circuit */
+  or_circ = origin_circuit_new();
+
+  or_circ->base_.purpose = CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED;
+
+  or_circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  or_circ->build_state->is_internal = 1;
+
+  if (is_legacy) {
+    /* Legacy: Setup rend data and final cpath */
+    or_circ->build_state->pending_final_cpath =
+      tor_malloc_zero(sizeof(crypt_path_t));
+    or_circ->build_state->pending_final_cpath->magic = CRYPT_PATH_MAGIC;
+    or_circ->build_state->pending_final_cpath->rend_dh_handshake_state =
+      crypto_dh_new(DH_TYPE_REND);
+    tt_assert(
+         or_circ->build_state->pending_final_cpath->rend_dh_handshake_state);
+    retval = crypto_dh_generate_public(
+           or_circ->build_state->pending_final_cpath->rend_dh_handshake_state);
+    tt_int_op(retval, OP_EQ, 0);
+    or_circ->rend_data = rend_data_dup(conn_rend_data);
+  } else {
+    /* prop224: Setup hs ident on the circuit */
+    or_circ->hs_ident = hs_ident_circuit_new(&service_pk,
+                                             HS_IDENT_CIRCUIT_RENDEZVOUS);
+  }
+
+  TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN;
+
+  /* fake n_chan */
+  n_chan = tor_malloc_zero(sizeof(channel_tls_t));
+  n_chan->base_.global_identifier = 1;
+  or_circ->base_.n_chan = &(n_chan->base_);
+
+  *circ_out = or_circ;
+  *conn_out = conn;
+
+  return 0;
+
+ done:
+  /* something failed */
+  return -1;
+}
+
+/* Test: Ensure that setting up legacy e2e rendezvous circuits works
+ * correctly. */
+static void
+test_e2e_rend_circuit_setup_legacy(void *arg)
+{
+  ssize_t retval;
+  origin_circuit_t *or_circ = NULL;
+  connection_t *conn = NULL;
+
+  (void) arg;
+
+  /** In this test we create a v2 legacy HS stream and a circuit with the same
+   *  hidden service destination. We make the stream wait for circuits to be
+   *  established to the hidden service, and then we complete the circuit using
+   *  the hs_circuit_setup_e2e_rend_circ_legacy_client() function. We then
+   *  check that the end-to-end cpath was setup correctly and that the stream
+   *  was attached to the circuit as expected. */
+
+  MOCK(connection_ap_handshake_send_begin,
+       mock_connection_ap_handshake_send_begin);
+
+  /* Setup */
+  retval = helper_get_circ_and_stream_for_test( &or_circ, &conn, 1);
+  tt_int_op(retval, OP_EQ, 0);
+  tt_assert(or_circ);
+  tt_assert(conn);
+
+  /* Check number of hops */
+  retval = cpath_get_n_hops(&or_circ->cpath);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check that our stream is not attached on any circuits */
+  tt_ptr_op(TO_EDGE_CONN(conn)->on_circuit, OP_EQ, NULL);
+
+  /********************************************** */
+
+  /* Make a good RENDEZVOUS1 cell body because it needs to pass key exchange
+   * digest verification... */
+  uint8_t rend_cell_body[DH1024_KEY_LEN+DIGEST_LEN] = {2};
+  {
+    char keys[DIGEST_LEN+CPATH_KEY_MATERIAL_LEN];
+    crypto_dh_t *dh_state =
+      or_circ->build_state->pending_final_cpath->rend_dh_handshake_state;
+    /* compute and overwrite digest of cell body with the right value */
+    retval = crypto_dh_compute_secret(LOG_PROTOCOL_WARN, dh_state,
+                                      (char*)rend_cell_body, DH1024_KEY_LEN,
+                                      keys, DIGEST_LEN+CPATH_KEY_MATERIAL_LEN);
+    tt_int_op(retval, OP_GT, 0);
+    memcpy(rend_cell_body+DH1024_KEY_LEN, keys, DIGEST_LEN);
+  }
+
+  /* Setup the circuit */
+  retval = hs_circuit_setup_e2e_rend_circ_legacy_client(or_circ,
+                                                        rend_cell_body);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /**********************************************/
+
+  /* See that a hop was added to the circuit's cpath */
+  retval = cpath_get_n_hops(&or_circ->cpath);
+  tt_int_op(retval, OP_EQ, 1);
+
+  /* Check the digest algo */
+  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest),
+            OP_EQ, DIGEST_SHA1);
+  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest),
+            OP_EQ, DIGEST_SHA1);
+  tt_assert(or_circ->cpath->crypto.f_crypto);
+  tt_assert(or_circ->cpath->crypto.b_crypto);
+
+  /* Ensure that circ purpose was changed */
+  tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_C_REND_JOINED);
+
+  /* Test that stream got attached */
+  tt_ptr_op(TO_EDGE_CONN(conn)->on_circuit, OP_EQ, TO_CIRCUIT(or_circ));
+
+ done:
+  connection_free_minimal(conn);
+  if (or_circ)
+    tor_free(TO_CIRCUIT(or_circ)->n_chan);
+  circuit_free_(TO_CIRCUIT(or_circ));
+}
+
+/* Test: Ensure that setting up v3 rendezvous circuits works correctly. */
+static void
+test_e2e_rend_circuit_setup(void *arg)
+{
+  uint8_t ntor_key_seed[DIGEST256_LEN] = {0};
+  origin_circuit_t *or_circ = NULL;
+  int retval;
+  connection_t *conn = NULL;
+
+  (void) arg;
+
+  /** In this test we create a prop224 v3 HS stream and a circuit with the same
+   *  hidden service destination. We make the stream wait for circuits to be
+   *  established to the hidden service, and then we complete the circuit using
+   *  the hs_circuit_setup_e2e_rend_circ() function. We then check that the
+   *  end-to-end cpath was setup correctly and that the stream was attached to
+   *  the circuit as expected. */
+
+  MOCK(connection_ap_handshake_send_begin,
+       mock_connection_ap_handshake_send_begin);
+
+  /* Setup */
+  retval = helper_get_circ_and_stream_for_test( &or_circ, &conn, 0);
+  tt_int_op(retval, OP_EQ, 0);
+  tt_assert(or_circ);
+  tt_assert(conn);
+
+  /* Check number of hops: There should be no hops yet to this circ */
+  retval = cpath_get_n_hops(&or_circ->cpath);
+  tt_int_op(retval, OP_EQ, 0);
+  tt_ptr_op(or_circ->cpath, OP_EQ, NULL);
+
+  /* Check that our stream is not attached on any circuits */
+  tt_ptr_op(TO_EDGE_CONN(conn)->on_circuit, OP_EQ, NULL);
+
+  /**********************************************/
+
+  /* Setup the circuit */
+  retval = hs_circuit_setup_e2e_rend_circ(or_circ,
+                                          ntor_key_seed, sizeof(ntor_key_seed),
+                                          0);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /**********************************************/
+
+  /* See that a hop was added to the circuit's cpath */
+  retval = cpath_get_n_hops(&or_circ->cpath);
+  tt_int_op(retval, OP_EQ, 1);
+
+  /* Check that the crypt path has prop224 algorithm parameters */
+  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest),
+            OP_EQ, DIGEST_SHA3_256);
+  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest),
+            OP_EQ, DIGEST_SHA3_256);
+  tt_assert(or_circ->cpath->crypto.f_crypto);
+  tt_assert(or_circ->cpath->crypto.b_crypto);
+
+  /* Ensure that circ purpose was changed */
+  tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_C_REND_JOINED);
+
+  /* Test that stream got attached */
+  tt_ptr_op(TO_EDGE_CONN(conn)->on_circuit, OP_EQ, TO_CIRCUIT(or_circ));
+
+ done:
+  connection_free_minimal(conn);
+  if (or_circ)
+    tor_free(TO_CIRCUIT(or_circ)->n_chan);
+  circuit_free_(TO_CIRCUIT(or_circ));
+}
+
+/** Test client logic for picking intro points from a descriptor. Also test how
+ *  ExcludeNodes and intro point failures affect picking intro points. */
+static void
+test_client_pick_intro(void *arg)
+{
+  int ret;
+  ed25519_keypair_t service_kp;
+  hs_descriptor_t *desc = NULL;
+
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  (void) arg;
+
+  hs_init();
+
+  /* Generate service keypair */
+  tt_int_op(0, OP_EQ, ed25519_keypair_generate(&service_kp, 0));
+
+  /* Set time */
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                           &mock_ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                           &mock_ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+
+  update_approx_time(mock_ns.fresh_until-10);
+  time_t now = approx_time();
+
+  /* Test logic:
+   *
+   * 1) Add our desc with intro points to the HS cache.
+   *
+   * 2) Mark all descriptor intro points except _the chosen one_ as
+   *    failed. Then query the desc to get a random intro: check that we got
+   *    _the chosen one_. Then fail the chosen one as well, and see that no
+   *    intros are returned.
+   *
+   * 3) Then clean the intro state cache and get an intro point.
+   *
+   * 4) Try fetching an intro with the wrong service key: shouldn't work
+   *
+   * 5) Set StrictNodes and put all our intro points in ExcludeNodes: see that
+   *    nothing is returned.
+   */
+
+  /* 1) Add desc to HS cache */
+  {
+    char *encoded = NULL;
+    desc = hs_helper_build_hs_desc_with_ip(&service_kp);
+    ret = hs_desc_encode_descriptor(desc, &service_kp, NULL, &encoded);
+    tt_int_op(ret, OP_EQ, 0);
+    tt_assert(encoded);
+
+    /* store it */
+    hs_cache_store_as_client(encoded, &service_kp.pubkey);
+
+    /* fetch it to make sure it works */
+    const hs_descriptor_t *fetched_desc =
+      hs_cache_lookup_as_client(&service_kp.pubkey);
+    tt_assert(fetched_desc);
+    tt_mem_op(fetched_desc->subcredential, OP_EQ, desc->subcredential,
+              DIGEST256_LEN);
+    tt_assert(!tor_mem_is_zero((char*)fetched_desc->subcredential,
+                               DIGEST256_LEN));
+    tor_free(encoded);
+  }
+
+  /* 2) Mark all intro points except _the chosen one_ as failed. Then query the
+   *   desc and get a random intro: check that we got _the chosen one_. */
+  {
+    /* Pick the chosen intro point and get its ei */
+    hs_desc_intro_point_t *chosen_intro_point =
+      smartlist_get(desc->encrypted_data.intro_points, 0);
+    extend_info_t *chosen_intro_ei =
+      desc_intro_point_to_extend_info(chosen_intro_point);
+    tt_assert(chosen_intro_point);
+    tt_assert(chosen_intro_ei);
+
+    /* Now mark all other intro points as failed */
+    SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                            hs_desc_intro_point_t *, ip) {
+      /* Skip the chosen intro point */
+      if (ip == chosen_intro_point) {
+        continue;
+      }
+      ed25519_public_key_t *intro_auth_key = &ip->auth_key_cert->signed_key;
+      hs_cache_client_intro_state_note(&service_kp.pubkey,
+                                       intro_auth_key,
+                                       INTRO_POINT_FAILURE_GENERIC);
+    } SMARTLIST_FOREACH_END(ip);
+
+    /* Try to get a random intro: Should return the chosen one! */
+    /* (We try several times, to make sure this behavior is consistent, and to
+     * cover the different cases of client_get_random_intro().) */
+    for (int i = 0; i < 64; ++i) {
+      extend_info_t *ip = client_get_random_intro(&service_kp.pubkey);
+      tor_assert(ip);
+      tt_assert(!tor_mem_is_zero((char*)ip->identity_digest, DIGEST_LEN));
+      tt_mem_op(ip->identity_digest, OP_EQ, chosen_intro_ei->identity_digest,
+                DIGEST_LEN);
+      extend_info_free(ip);
+    }
+
+    extend_info_free(chosen_intro_ei);
+
+    /* Now also mark the chosen one as failed: See that we can't get any intro
+       points anymore. */
+    hs_cache_client_intro_state_note(&service_kp.pubkey,
+                                &chosen_intro_point->auth_key_cert->signed_key,
+                                     INTRO_POINT_FAILURE_TIMEOUT);
+    extend_info_t *ip = client_get_random_intro(&service_kp.pubkey);
+    tor_assert(!ip);
+  }
+
+  /* 3) Clean the intro state cache and get an intro point */
+  {
+    /* Pretend we are 5 mins in the future and order a cleanup of the intro
+     * state. This should clean up the intro point failures and allow us to get
+     * an intro. */
+    hs_cache_client_intro_state_clean(now + 5*60);
+
+    /* Get an intro. It should work! */
+    extend_info_t *ip = client_get_random_intro(&service_kp.pubkey);
+    tor_assert(ip);
+    extend_info_free(ip);
+  }
+
+  /* 4) Try fetching an intro with the wrong service key: shouldn't work */
+  {
+    ed25519_keypair_t dummy_kp;
+    tt_int_op(0, OP_EQ, ed25519_keypair_generate(&dummy_kp, 0));
+    extend_info_t *ip = client_get_random_intro(&dummy_kp.pubkey);
+    tor_assert(!ip);
+  }
+
+  /* 5) Set StrictNodes and put all our intro points in ExcludeNodes: see that
+   *    nothing is returned. */
+  {
+    get_options_mutable()->ExcludeNodes = routerset_new();
+    get_options_mutable()->StrictNodes = 1;
+    SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                            hs_desc_intro_point_t *, ip) {
+      extend_info_t *intro_ei = desc_intro_point_to_extend_info(ip);
+      if (intro_ei) {
+        const char *ptr;
+        char ip_addr[TOR_ADDR_BUF_LEN];
+        /* We need to decorate in case it is an IPv6 else routerset_parse()
+         * doesn't like it. */
+        ptr = tor_addr_to_str(ip_addr, &intro_ei->addr, sizeof(ip_addr), 1);
+        tt_assert(ptr == ip_addr);
+        ret = routerset_parse(get_options_mutable()->ExcludeNodes,
+                              ip_addr, "");
+        tt_int_op(ret, OP_EQ, 0);
+        extend_info_free(intro_ei);
+      }
+    } SMARTLIST_FOREACH_END(ip);
+
+    extend_info_t *ip = client_get_random_intro(&service_kp.pubkey);
+    tt_assert(!ip);
+  }
+
+ done:
+  hs_descriptor_free(desc);
+}
+
+static int
+mock_router_have_minimum_dir_info_false(void)
+{
+  return 0;
+}
+static int
+mock_router_have_minimum_dir_info_true(void)
+{
+  return 1;
+}
+
+static hs_client_fetch_status_t
+mock_fetch_v3_desc_error(const ed25519_public_key_t *key)
+{
+  (void) key;
+  return HS_CLIENT_FETCH_ERROR;
+}
+
+static void
+mock_connection_mark_unattached_ap_(entry_connection_t *conn, int endreason,
+                                    int line, const char *file)
+{
+  (void) line;
+  (void) file;
+  conn->edge_.end_reason = endreason;
+  /* This function ultimately will flag this so make sure we do also in the
+   * MOCK one so we can assess closed connections vs open ones. */
+  conn->edge_.base_.marked_for_close = 1;
+}
+
+static void
+test_descriptor_fetch(void *arg)
+{
+  int ret;
+  entry_connection_t *ec = NULL;
+  ed25519_public_key_t service_pk;
+  ed25519_secret_key_t service_sk;
+
+  (void) arg;
+
+  hs_init();
+  memset(&service_sk, 'A', sizeof(service_sk));
+  ret = ed25519_public_key_generate(&service_pk, &service_sk);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Initialize this so get_voting_interval() doesn't freak out. */
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                           &mock_ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                           &mock_ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+
+  ec = entry_connection_new(CONN_TYPE_AP, AF_INET);
+  tt_assert(ec);
+  ENTRY_TO_EDGE_CONN(ec)->hs_ident = hs_ident_edge_conn_new(&service_pk);
+  tt_assert(ENTRY_TO_EDGE_CONN(ec)->hs_ident);
+  TO_CONN(ENTRY_TO_EDGE_CONN(ec))->state = AP_CONN_STATE_RENDDESC_WAIT;
+  smartlist_add(get_connection_array(), &ec->edge_.base_);
+
+  /* 1. FetchHidServDescriptors is false so we shouldn't be able to fetch. */
+  get_options_mutable()->FetchHidServDescriptors = 0;
+  ret = hs_client_refetch_hsdesc(&service_pk);
+  tt_int_op(ret, OP_EQ, HS_CLIENT_FETCH_NOT_ALLOWED);
+  get_options_mutable()->FetchHidServDescriptors = 1;
+
+  /* 2. We don't have a live consensus. */
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus_false);
+  ret = hs_client_refetch_hsdesc(&service_pk);
+  UNMOCK(networkstatus_get_live_consensus);
+  tt_int_op(ret, OP_EQ, HS_CLIENT_FETCH_MISSING_INFO);
+
+  /* From now on, return a live consensus. */
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  /* 3. Not enough dir information. */
+  MOCK(router_have_minimum_dir_info,
+       mock_router_have_minimum_dir_info_false);
+  ret = hs_client_refetch_hsdesc(&service_pk);
+  UNMOCK(router_have_minimum_dir_info);
+  tt_int_op(ret, OP_EQ, HS_CLIENT_FETCH_MISSING_INFO);
+
+  /* From now on, we do have enough directory information. */
+  MOCK(router_have_minimum_dir_info,
+       mock_router_have_minimum_dir_info_true);
+
+  /* 4. We do have a pending directory request. */
+  {
+    dir_connection_t *dir_conn = dir_connection_new(AF_INET);
+    dir_conn->hs_ident = tor_malloc_zero(sizeof(hs_ident_dir_conn_t));
+    TO_CONN(dir_conn)->purpose = DIR_PURPOSE_FETCH_HSDESC;
+    ed25519_pubkey_copy(&dir_conn->hs_ident->identity_pk, &service_pk);
+    smartlist_add(get_connection_array(), TO_CONN(dir_conn));
+    ret = hs_client_refetch_hsdesc(&service_pk);
+    smartlist_remove(get_connection_array(), TO_CONN(dir_conn));
+    connection_free_minimal(TO_CONN(dir_conn));
+    tt_int_op(ret, OP_EQ, HS_CLIENT_FETCH_PENDING);
+  }
+
+  /* 5. We'll trigger an error on the fetch_desc_v3 and force to close all
+   *    pending SOCKS request. */
+  MOCK(router_have_minimum_dir_info,
+       mock_router_have_minimum_dir_info_true);
+  MOCK(fetch_v3_desc, mock_fetch_v3_desc_error);
+  MOCK(connection_mark_unattached_ap_,
+       mock_connection_mark_unattached_ap_);
+  ret = hs_client_refetch_hsdesc(&service_pk);
+  UNMOCK(fetch_v3_desc);
+  UNMOCK(connection_mark_unattached_ap_);
+  tt_int_op(ret, OP_EQ, HS_CLIENT_FETCH_ERROR);
+  /* The close waiting for descriptor function has been called. */
+  tt_int_op(ec->edge_.end_reason, OP_EQ, END_STREAM_REASON_RESOLVEFAILED);
+
+ done:
+  connection_free_minimal(ENTRY_TO_CONN(ec));
+  UNMOCK(networkstatus_get_live_consensus);
+  UNMOCK(router_have_minimum_dir_info);
+  hs_free_all();
+}
+
+static void
+test_auth_key_filename_is_valid(void *arg)
+{
+  (void) arg;
+
+  /* Valid file name. */
+  tt_assert(auth_key_filename_is_valid("a.auth_private"));
+  /* Valid file name with special character. */
+  tt_assert(auth_key_filename_is_valid("a-.auth_private"));
+  /* Invalid extension. */
+  tt_assert(!auth_key_filename_is_valid("a.ath_private"));
+  /* Nothing before the extension. */
+  tt_assert(!auth_key_filename_is_valid(".auth_private"));
+
+ done:
+  ;
+}
+
+static void
+test_parse_auth_file_content(void *arg)
+{
+  hs_client_service_authorization_t *auth = NULL;
+
+  (void) arg;
+
+  /* Valid authorized client. */
+  auth = parse_auth_file_content(
+      "4acth47i6kxnvkewtm6q7ib2s3ufpo5sqbsnzjpbi7utijcltosqemad:descriptor:"
+      "x25519:zdsyvn2jq534ugyiuzgjy4267jbtzcjbsgedhshzx5mforyxtryq");
+  tt_assert(auth);
+
+  /* Wrong number of fields. */
+  tt_assert(!parse_auth_file_content("a:b"));
+  /* Wrong auth type. */
+  tt_assert(!parse_auth_file_content(
+      "4acth47i6kxnvkewtm6q7ib2s3ufpo5sqbsnzjpbi7utijcltosqemad:x:"
+      "x25519:zdsyvn2jq534ugyiuzgjy4267jbtzcjbsgedhshzx5mforyxtryq"));
+  /* Wrong key type. */
+  tt_assert(!parse_auth_file_content(
+      "4acth47i6kxnvkewtm6q7ib2s3ufpo5sqbsnzjpbi7utijcltosqemad:descriptor:"
+      "x:zdsyvn2jq534ugyiuzgjy4267jbtzcjbsgedhshzx5mforyxtryq"));
+  /* Some malformed string. */
+  tt_assert(!parse_auth_file_content("xx:descriptor:x25519:aa=="));
+  /* Bigger key than it should be */
+  tt_assert(!parse_auth_file_content("xx:descriptor:x25519:"
+                     "vjqea4jbhwwc4hto7ekyvqfbeodghbaq6nxi45hz4wr3qvhqv3yqa"));
+ done:
+  tor_free(auth);
+}
+
+static char *
+mock_read_file_to_str(const char *filename, int flags, struct stat *stat_out)
+{
+  char *ret = NULL;
+
+  (void) flags;
+  (void) stat_out;
+
+  if (!strcmp(filename, get_fname("auth_keys" PATH_SEPARATOR
+                                              "client1.auth_private"))) {
+    ret = tor_strdup(
+        "4acth47i6kxnvkewtm6q7ib2s3ufpo5sqbsnzjpbi7utijcltosqemad:descriptor:"
+        "x25519:zdsyvn2jq534ugyiuzgjy4267jbtzcjbsgedhshzx5mforyxtryq");
+    goto done;
+  }
+
+  if (!strcmp(filename, get_fname("auth_keys" PATH_SEPARATOR "dummy.xxx"))) {
+    ret = tor_strdup(
+        "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx:descriptor:"
+        "x25519:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");
+    goto done;
+  }
+
+  if (!strcmp(filename, get_fname("auth_keys" PATH_SEPARATOR
+                                              "client2.auth_private"))) {
+    ret = tor_strdup(
+        "25njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid:descriptor:"
+        "x25519:fdreqzjqso7d2ac7qscrxfl5qfpamdvgy5d6cxejcgzc3hvhurmq");
+    goto done;
+  }
+
+ done:
+  return ret;
+}
+
+static int
+mock_check_private_dir(const char *dirname, cpd_check_t check,
+                       const char *effective_user)
+{
+  (void) dirname;
+  (void) check;
+  (void) effective_user;
+
+  return 0;
+}
+
+static smartlist_t *
+mock_tor_listdir(const char *dirname)
+{
+  smartlist_t *file_list = smartlist_new();
+
+  (void) dirname;
+
+  smartlist_add(file_list, tor_strdup("client1.auth_private"));
+  smartlist_add(file_list, tor_strdup("dummy.xxx"));
+  smartlist_add(file_list, tor_strdup("client2.auth_private"));
+
+  return file_list;
+}
+
+static void
+test_config_client_authorization(void *arg)
+{
+  int ret;
+  char *conf = NULL;
+  ed25519_public_key_t pk1, pk2;
+  digest256map_t *global_map = NULL;
+  char *key_dir = tor_strdup(get_fname("auth_keys"));
+
+  (void) arg;
+
+  MOCK(read_file_to_str, mock_read_file_to_str);
+  MOCK(tor_listdir, mock_tor_listdir);
+  MOCK(check_private_dir, mock_check_private_dir);
+
+#define conf_fmt \
+  "ClientOnionAuthDir %s\n"
+
+  tor_asprintf(&conf, conf_fmt, key_dir);
+  ret = helper_config_client(conf, 0);
+  tor_free(conf);
+  tt_int_op(ret, OP_EQ, 0);
+
+#undef conf_fmt
+
+  global_map = get_hs_client_auths_map();
+  tt_int_op(digest256map_size(global_map), OP_EQ, 2);
+
+  hs_parse_address("4acth47i6kxnvkewtm6q7ib2s3ufpo5sqbsnzjpbi7utijcltosqemad",
+                   &pk1, NULL, NULL);
+  hs_parse_address("25njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid",
+                   &pk2, NULL, NULL);
+
+  tt_assert(digest256map_get(global_map, pk1.pubkey));
+  tt_assert(digest256map_get(global_map, pk2.pubkey));
+
+ done:
+  tor_free(key_dir);
+  hs_free_all();
+  UNMOCK(read_file_to_str);
+  UNMOCK(tor_listdir);
+  UNMOCK(check_private_dir);
+}
+
+static entry_connection_t *
+helper_build_socks_connection(const ed25519_public_key_t *service_pk,
+                              int conn_state)
+{
+  entry_connection_t *socks = entry_connection_new(CONN_TYPE_AP, AF_INET);
+  ENTRY_TO_EDGE_CONN(socks)->hs_ident = hs_ident_edge_conn_new(service_pk);
+  TO_CONN(ENTRY_TO_EDGE_CONN(socks))->state = conn_state;
+  smartlist_add(get_connection_array(), &socks->edge_.base_);
+  return socks;
+}
+
+static void
+test_desc_has_arrived_cleanup(void *arg)
+{
+  /* The goal of this test is to make sure we clean up everything in between
+   * two descriptors from the same .onion. Because intro points can change
+   * from one descriptor to another, once we received a new descriptor, we
+   * need to cleanup the remaining circuits so they aren't used or selected
+   * when establishing a connection with the newly stored descriptor.
+   *
+   * This test was created because of #27410. */
+
+  int ret;
+  char *desc_str = NULL;
+  hs_descriptor_t *desc = NULL;
+  const hs_descriptor_t *cached_desc;
+  ed25519_keypair_t signing_kp;
+  entry_connection_t *socks1 = NULL, *socks2 = NULL;
+  hs_ident_dir_conn_t hs_dir_ident;
+
+  (void) arg;
+
+  hs_init();
+
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+  MOCK(connection_mark_unattached_ap_,
+       mock_connection_mark_unattached_ap_);
+  MOCK(router_have_minimum_dir_info,
+       mock_router_have_minimum_dir_info_true);
+
+  /* Set consensus time before our time so the cache lookup can always
+   * validate that the entry is not expired. */
+  parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC", &mock_ns.valid_after);
+  parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC", &mock_ns.fresh_until);
+  parse_rfc1123_time("Sat, 26 Oct 1985 16:00:00 UTC", &mock_ns.valid_until);
+
+  /* Build a descriptor for a specific .onion. */
+  ret = ed25519_keypair_generate(&signing_kp, 0);
+  tt_int_op(ret, OP_EQ, 0);
+  desc = hs_helper_build_hs_desc_with_ip(&signing_kp);
+  tt_assert(desc);
+  ret = hs_desc_encode_descriptor(desc, &signing_kp, NULL, &desc_str);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Store in the client cache. */
+  ret = hs_cache_store_as_client(desc_str, &signing_kp.pubkey);
+  tt_int_op(ret, OP_EQ, 0);
+  cached_desc = hs_cache_lookup_as_client(&signing_kp.pubkey);
+  tt_assert(cached_desc);
+  hs_helper_desc_equal(desc, cached_desc);
+
+  /* Create two SOCKS connection for the same .onion both in the waiting for a
+   * descriptor state. */
+  socks1 = helper_build_socks_connection(&signing_kp.pubkey,
+                                         AP_CONN_STATE_RENDDESC_WAIT);
+  tt_assert(socks1);
+  socks2 = helper_build_socks_connection(&signing_kp.pubkey,
+                                         AP_CONN_STATE_RENDDESC_WAIT);
+  tt_assert(socks2);
+
+  /* Now, we'll make the intro points in the current descriptor unusable so
+   * the hs_client_desc_has_arrived() will take the right code path that we
+   * want to test that is the fetched descriptor has bad intro points. */
+  SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                        hs_desc_intro_point_t *, ip) {
+    hs_cache_client_intro_state_note(&signing_kp.pubkey,
+                                     &ip->auth_key_cert->signed_key,
+                                     INTRO_POINT_FAILURE_GENERIC);
+  } SMARTLIST_FOREACH_END(ip);
+
+  /* Simulate that a new descriptor just arrived. We should have both of our
+   * SOCKS connection to be ended with a resolved failed. */
+  hs_ident_dir_conn_init(&signing_kp.pubkey,
+                         &desc->plaintext_data.blinded_pubkey, &hs_dir_ident);
+  hs_client_desc_has_arrived(&hs_dir_ident);
+  tt_int_op(socks1->edge_.end_reason, OP_EQ, END_STREAM_REASON_RESOLVEFAILED);
+  /* XXX: MUST work with OP_EQ. */
+  tt_int_op(socks2->edge_.end_reason, OP_EQ, END_STREAM_REASON_RESOLVEFAILED);
+
+  /* Now let say tor cleans up the intro state cache which resets all intro
+   * point failure count. */
+  hs_cache_client_intro_state_purge();
+
+  /* Retrying all SOCKS which should basically do nothing since we don't have
+   * any pending SOCKS connection in AP_CONN_STATE_RENDDESC_WAIT state. */
+  /* XXX: BUG() is triggered here, shouldn't if socks2 wasn't alive. */
+  retry_all_socks_conn_waiting_for_desc();
+
+ done:
+  connection_free_minimal(ENTRY_TO_CONN(socks1));
+  connection_free_minimal(ENTRY_TO_CONN(socks2));
+  hs_descriptor_free(desc);
+  tor_free(desc_str);
+  hs_free_all();
+
+  UNMOCK(networkstatus_get_live_consensus);
+  UNMOCK(connection_mark_unattached_ap_);
+  UNMOCK(router_have_minimum_dir_info);
+}
+
+static void
+test_close_intro_circuits_new_desc(void *arg)
+{
+  int ret;
+  ed25519_keypair_t service_kp;
+  circuit_t *circ = NULL;
+  origin_circuit_t *ocirc = NULL;
+  hs_descriptor_t *desc1 = NULL, *desc2 = NULL;
+
+  (void) arg;
+
+  hs_init();
+
+  /* This is needed because of the client cache expiration timestamp is based
+   * on having a consensus. See cached_client_descriptor_has_expired(). */
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  /* Set consensus time */
+  parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                           &mock_ns.valid_after);
+  parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                           &mock_ns.fresh_until);
+  parse_rfc1123_time("Sat, 26 Oct 1985 16:00:00 UTC",
+                           &mock_ns.valid_until);
+
+  /* Generate service keypair */
+  tt_int_op(0, OP_EQ, ed25519_keypair_generate(&service_kp, 0));
+
+  /* Create and add to the global list a dummy client introduction circuits.
+   * We'll then make sure the hs_ident is attached to a dummy descriptor. */
+  circ = dummy_origin_circuit_new(0);
+  tt_assert(circ);
+  circ->purpose = CIRCUIT_PURPOSE_C_INTRODUCING;
+  ocirc = TO_ORIGIN_CIRCUIT(circ);
+
+  /* Build the first descriptor and cache it. */
+  {
+    char *encoded;
+    desc1 = hs_helper_build_hs_desc_with_ip(&service_kp);
+    tt_assert(desc1);
+    ret = hs_desc_encode_descriptor(desc1, &service_kp, NULL, &encoded);
+    tt_int_op(ret, OP_EQ, 0);
+    tt_assert(encoded);
+
+    /* Store it */
+    ret = hs_cache_store_as_client(encoded, &service_kp.pubkey);
+    tt_int_op(ret, OP_EQ, 0);
+    tor_free(encoded);
+    tt_assert(hs_cache_lookup_as_client(&service_kp.pubkey));
+  }
+
+  /* We'll pick one introduction point and associate it with the circuit. */
+  {
+    const hs_desc_intro_point_t *ip =
+      smartlist_get(desc1->encrypted_data.intro_points, 0);
+    tt_assert(ip);
+    ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey,
+                                           HS_IDENT_CIRCUIT_INTRO);
+    ed25519_pubkey_copy(&ocirc->hs_ident->intro_auth_pk,
+                        &ip->auth_key_cert->signed_key);
+  }
+
+  /* Before we are about to clean up the intro circuits, make sure it is
+   * actually there. */
+  tt_assert(circuit_get_next_intro_circ(NULL, true));
+
+  /* Build the second descriptor for the same service and cache it. */
+  {
+    char *encoded;
+    desc2 = hs_helper_build_hs_desc_with_ip(&service_kp);
+    tt_assert(desc2);
+    tt_mem_op(&desc1->plaintext_data.signing_pubkey, OP_EQ,
+              &desc2->plaintext_data.signing_pubkey, ED25519_PUBKEY_LEN);
+    /* To replace the existing descriptor, the revision counter needs to be
+     * bigger. */
+    desc2->plaintext_data.revision_counter =
+      desc1->plaintext_data.revision_counter + 1;
+
+    ret = hs_desc_encode_descriptor(desc2, &service_kp, NULL, &encoded);
+    tt_int_op(ret, OP_EQ, 0);
+    tt_assert(encoded);
+
+    hs_cache_store_as_client(encoded, &service_kp.pubkey);
+    tt_int_op(ret, OP_EQ, 0);
+    tor_free(encoded);
+    tt_assert(hs_cache_lookup_as_client(&service_kp.pubkey));
+  }
+
+  /* Once stored, our intro circuit should be closed because it is related to
+   * an old introduction point that doesn't exists anymore. */
+  tt_assert(!circuit_get_next_intro_circ(NULL, true));
+
+ done:
+  circuit_free(circ);
+  hs_descriptor_free(desc1);
+  hs_descriptor_free(desc2);
+  hs_free_all();
+  UNMOCK(networkstatus_get_live_consensus);
+}
+
+struct testcase_t hs_client_tests[] = {
+  { "e2e_rend_circuit_setup_legacy", test_e2e_rend_circuit_setup_legacy,
+    TT_FORK, NULL, NULL },
+  { "e2e_rend_circuit_setup", test_e2e_rend_circuit_setup,
+    TT_FORK, NULL, NULL },
+  { "client_pick_intro", test_client_pick_intro,
+    TT_FORK, NULL, NULL },
+  { "descriptor_fetch", test_descriptor_fetch,
+    TT_FORK, NULL, NULL },
+  { "auth_key_filename_is_valid", test_auth_key_filename_is_valid, TT_FORK,
+    NULL, NULL },
+  { "parse_auth_file_content", test_parse_auth_file_content, TT_FORK,
+    NULL, NULL },
+  { "config_client_authorization", test_config_client_authorization,
+    TT_FORK, NULL, NULL },
+  { "desc_has_arrived_cleanup", test_desc_has_arrived_cleanup,
+    TT_FORK, NULL, NULL },
+  { "close_intro_circuits_new_desc", test_close_intro_circuits_new_desc,
+    TT_FORK, NULL, NULL },
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c
new file mode 100644
index 0000000000..2aff179687
--- /dev/null
+++ b/src/test/test_hs_common.c
@@ -0,0 +1,1839 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_common.c
+ * \brief Test hidden service common functionalities.
+ */
+
+#define HS_COMMON_PRIVATE
+#define HS_CLIENT_PRIVATE
+#define HS_SERVICE_PRIVATE
+#define NODELIST_PRIVATE
+
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
+#include "test/hs_test_helpers.h"
+
+#include "core/or/connection_edge.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_client.h"
+#include "feature/hs/hs_service.h"
+#include "app/config/config.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "app/config/statefile.h"
+#include "core/or/circuitlist.h"
+#include "feature/dirauth/shared_random.h"
+#include "feature/dircommon/voting_schedule.h"
+
+#include "feature/nodelist/microdesc_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "app/config/or_state_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+/** Test the validation of HS v3 addresses */
+static void
+test_validate_address(void *arg)
+{
+  int ret;
+
+  (void) arg;
+
+  /* Address too short and too long. */
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_address_is_valid("blah");
+  tt_int_op(ret, OP_EQ, 0);
+  expect_log_msg_containing("has an invalid length");
+  teardown_capture_of_logs();
+
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_address_is_valid(
+           "p3xnclpu4mu22dwaurjtsybyqk4xfjmcfz6z62yl24uwmhjatiwnlnadb");
+  tt_int_op(ret, OP_EQ, 0);
+  expect_log_msg_containing("has an invalid length");
+  teardown_capture_of_logs();
+
+  /* Invalid checksum (taken from prop224) */
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_address_is_valid(
+           "l5satjgud6gucryazcyvyvhuxhr74u6ygigiuyixe3a6ysis67ororad");
+  tt_int_op(ret, OP_EQ, 0);
+  expect_log_msg_containing("invalid checksum");
+  teardown_capture_of_logs();
+
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_address_is_valid(
+           "btojiu7nu5y5iwut64eufevogqdw4wmqzugnoluw232r4t3ecsfv37ad");
+  tt_int_op(ret, OP_EQ, 0);
+  expect_log_msg_containing("invalid checksum");
+  teardown_capture_of_logs();
+
+  /* Non base32 decodable string. */
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_address_is_valid(
+           "????????????????????????????????????????????????????????");
+  tt_int_op(ret, OP_EQ, 0);
+  expect_log_msg_containing("can't be decoded");
+  teardown_capture_of_logs();
+
+  /* Valid address. */
+  ret = hs_address_is_valid(
+           "25njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid");
+  tt_int_op(ret, OP_EQ, 1);
+
+ done:
+  ;
+}
+
+static int
+mock_write_str_to_file(const char *path, const char *str, int bin)
+{
+  (void)bin;
+  tt_str_op(path, OP_EQ, "/double/five"PATH_SEPARATOR"squared");
+  tt_str_op(str, OP_EQ,
+           "25njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid.onion\n");
+
+ done:
+  return 0;
+}
+
+/** Test building HS v3 onion addresses. Uses test vectors from the
+ *  ./hs_build_address.py script. */
+static void
+test_build_address(void *arg)
+{
+  int ret;
+  char onion_addr[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+  ed25519_public_key_t pubkey;
+  /* hex-encoded ed25519 pubkey used in hs_build_address.py */
+  char pubkey_hex[] =
+    "d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a";
+  hs_service_t *service = NULL;
+
+  (void) arg;
+
+  MOCK(write_str_to_file, mock_write_str_to_file);
+
+  /* The following has been created with hs_build_address.py script that
+   * follows proposal 224 specification to build an onion address. */
+  static const char *test_addr =
+    "25njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid";
+
+  /* Let's try to build the same onion address as the script */
+  base16_decode((char*)pubkey.pubkey, sizeof(pubkey.pubkey),
+                pubkey_hex, strlen(pubkey_hex));
+  hs_build_address(&pubkey, HS_VERSION_THREE, onion_addr);
+  tt_str_op(test_addr, OP_EQ, onion_addr);
+  /* Validate that address. */
+  ret = hs_address_is_valid(onion_addr);
+  tt_int_op(ret, OP_EQ, 1);
+
+  service = tor_malloc_zero(sizeof(hs_service_t));
+  memcpy(service->onion_address, onion_addr, sizeof(service->onion_address));
+  tor_asprintf(&service->config.directory_path, "/double/five");
+  ret = write_address_to_file(service, "squared");
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  hs_service_free(service);
+}
+
+/** Test that our HS time period calculation functions work properly */
+static void
+test_time_period(void *arg)
+{
+  (void) arg;
+  uint64_t tn;
+  int retval;
+  time_t fake_time, correct_time, start_time;
+
+  /* Let's do the example in prop224 section [TIME-PERIODS] */
+  retval = parse_rfc1123_time("Wed, 13 Apr 2016 11:00:00 UTC",
+                              &fake_time);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check that the time period number is right */
+  tn = hs_get_time_period_num(fake_time);
+  tt_u64_op(tn, OP_EQ, 16903);
+
+  /* Increase current time to 11:59:59 UTC and check that the time period
+     number is still the same */
+  fake_time += 3599;
+  tn = hs_get_time_period_num(fake_time);
+  tt_u64_op(tn, OP_EQ, 16903);
+
+  { /* Check start time of next time period */
+    retval = parse_rfc1123_time("Wed, 13 Apr 2016 12:00:00 UTC",
+                                &correct_time);
+    tt_int_op(retval, OP_EQ, 0);
+
+    start_time = hs_get_start_time_of_next_time_period(fake_time);
+    tt_int_op(start_time, OP_EQ, correct_time);
+  }
+
+  /* Now take time to 12:00:00 UTC and check that the time period rotated */
+  fake_time += 1;
+  tn = hs_get_time_period_num(fake_time);
+  tt_u64_op(tn, OP_EQ, 16904);
+
+  /* Now also check our hs_get_next_time_period_num() function */
+  tn = hs_get_next_time_period_num(fake_time);
+  tt_u64_op(tn, OP_EQ, 16905);
+
+  { /* Check start time of next time period again */
+    retval = parse_rfc1123_time("Wed, 14 Apr 2016 12:00:00 UTC",
+                                &correct_time);
+    tt_int_op(retval, OP_EQ, 0);
+
+    start_time = hs_get_start_time_of_next_time_period(fake_time);
+    tt_int_op(start_time, OP_EQ, correct_time);
+  }
+
+  /* Now do another sanity check: The time period number at the start of the
+   * next time period, must be the same time period number as the one returned
+   * from hs_get_next_time_period_num() */
+  {
+    time_t next_tp_start = hs_get_start_time_of_next_time_period(fake_time);
+    tt_u64_op(hs_get_time_period_num(next_tp_start), OP_EQ,
+              hs_get_next_time_period_num(fake_time));
+  }
+
+ done:
+  ;
+}
+
+/** Test that we can correctly find the start time of the next time period */
+static void
+test_start_time_of_next_time_period(void *arg)
+{
+  (void) arg;
+  int retval;
+  time_t fake_time;
+  char tbuf[ISO_TIME_LEN + 1];
+  time_t next_tp_start_time;
+
+  /* Do some basic tests */
+  retval = parse_rfc1123_time("Wed, 13 Apr 2016 11:00:00 UTC",
+                              &fake_time);
+  tt_int_op(retval, OP_EQ, 0);
+  next_tp_start_time = hs_get_start_time_of_next_time_period(fake_time);
+  /* Compare it with the correct result */
+  format_iso_time(tbuf, next_tp_start_time);
+  tt_str_op("2016-04-13 12:00:00", OP_EQ, tbuf);
+
+  /* Another test with an edge-case time (start of TP) */
+  retval = parse_rfc1123_time("Wed, 13 Apr 2016 12:00:00 UTC",
+                              &fake_time);
+  tt_int_op(retval, OP_EQ, 0);
+  next_tp_start_time = hs_get_start_time_of_next_time_period(fake_time);
+  format_iso_time(tbuf, next_tp_start_time);
+  tt_str_op("2016-04-14 12:00:00", OP_EQ, tbuf);
+
+  {
+    /* Now pretend we are on a testing network and alter the voting schedule to
+       be every 10 seconds. This means that a time period has length 10*24
+       seconds (4 minutes). It also means that we apply a rotational offset of
+       120 seconds to the time period, so that it starts at 00:02:00 instead of
+       00:00:00. */
+    or_options_t *options = get_options_mutable();
+    options->TestingTorNetwork = 1;
+    options->V3AuthVotingInterval = 10;
+    options->TestingV3AuthInitialVotingInterval = 10;
+
+    retval = parse_rfc1123_time("Wed, 13 Apr 2016 00:00:00 UTC",
+                                &fake_time);
+    tt_int_op(retval, OP_EQ, 0);
+    next_tp_start_time = hs_get_start_time_of_next_time_period(fake_time);
+    /* Compare it with the correct result */
+    format_iso_time(tbuf, next_tp_start_time);
+    tt_str_op("2016-04-13 00:02:00", OP_EQ, tbuf);
+
+    retval = parse_rfc1123_time("Wed, 13 Apr 2016 00:02:00 UTC",
+                                &fake_time);
+    tt_int_op(retval, OP_EQ, 0);
+    next_tp_start_time = hs_get_start_time_of_next_time_period(fake_time);
+    /* Compare it with the correct result */
+    format_iso_time(tbuf, next_tp_start_time);
+    tt_str_op("2016-04-13 00:06:00", OP_EQ, tbuf);
+  }
+
+ done:
+  ;
+}
+
+/* Cleanup the global nodelist. It also frees the "md" in the node_t because
+ * we allocate the memory in helper_add_hsdir_to_networkstatus(). */
+static void
+cleanup_nodelist(void)
+{
+  smartlist_t *nodelist = nodelist_get_list();
+  SMARTLIST_FOREACH_BEGIN(nodelist, node_t *, node) {
+    tor_free(node->md);
+    node->md = NULL;
+  } SMARTLIST_FOREACH_END(node);
+  nodelist_free_all();
+}
+
+static void
+helper_add_hsdir_to_networkstatus(networkstatus_t *ns,
+                                  int identity_idx,
+                                  const char *nickname,
+                                  int is_hsdir)
+{
+  routerstatus_t *rs = tor_malloc_zero(sizeof(routerstatus_t));
+  routerinfo_t *ri = tor_malloc_zero(sizeof(routerinfo_t));
+  uint8_t identity[DIGEST_LEN];
+  tor_addr_t ipv4_addr;
+  node_t *node = NULL;
+
+  memset(identity, identity_idx, sizeof(identity));
+
+  memcpy(rs->identity_digest, identity, DIGEST_LEN);
+  rs->is_hs_dir = is_hsdir;
+  rs->pv.supports_v3_hsdir = 1;
+  strlcpy(rs->nickname, nickname, sizeof(rs->nickname));
+  tor_addr_parse(&ipv4_addr, "1.2.3.4");
+  ri->addr = tor_addr_to_ipv4h(&ipv4_addr);
+  rs->addr = tor_addr_to_ipv4h(&ipv4_addr);
+  ri->nickname = tor_strdup(nickname);
+  ri->protocol_list = tor_strdup("HSDir=1-2 LinkAuth=3");
+  memcpy(ri->cache_info.identity_digest, identity, DIGEST_LEN);
+  ri->cache_info.signing_key_cert = tor_malloc_zero(sizeof(tor_cert_t));
+  /* Needed for the HSDir index computation. */
+  memset(&ri->cache_info.signing_key_cert->signing_key,
+         identity_idx, ED25519_PUBKEY_LEN);
+  tt_assert(nodelist_set_routerinfo(ri, NULL));
+
+  node = node_get_mutable_by_id(ri->cache_info.identity_digest);
+  tt_assert(node);
+  node->rs = rs;
+  /* We need this to exist for node_has_preferred_descriptor() to return
+   * true. */
+  node->md = tor_malloc_zero(sizeof(microdesc_t));
+  /* Do this now the nodelist_set_routerinfo() function needs a "rs" to set
+   * the indexes which it doesn't have when it is called. */
+  node_set_hsdir_index(node, ns);
+  node->ri = NULL;
+  smartlist_add(ns->routerstatus_list, rs);
+
+ done:
+  if (node == NULL)
+    routerstatus_free(rs);
+
+  routerinfo_free(ri);
+}
+
+static networkstatus_t *mock_ns = NULL;
+
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus(void)
+{
+  time_t now = approx_time();
+
+  /* If initialized, return it */
+  if (mock_ns) {
+    return mock_ns;
+  }
+
+  /* Initialize fake consensus */
+  mock_ns = tor_malloc_zero(sizeof(networkstatus_t));
+
+  /* This consensus is live */
+  mock_ns->valid_after = now-1;
+  mock_ns->fresh_until = now+1;
+  mock_ns->valid_until = now+2;
+  /* Create routerstatus list */
+  mock_ns->routerstatus_list = smartlist_new();
+  mock_ns->type = NS_TYPE_CONSENSUS;
+
+  return mock_ns;
+}
+
+static networkstatus_t *
+mock_networkstatus_get_live_consensus(time_t now)
+{
+  (void) now;
+
+  tt_assert(mock_ns);
+
+ done:
+  return mock_ns;
+}
+
+/** Test the responsible HSDirs calculation function */
+static void
+test_responsible_hsdirs(void *arg)
+{
+  smartlist_t *responsible_dirs = smartlist_new();
+  networkstatus_t *ns = NULL;
+  (void) arg;
+
+  hs_init();
+
+  MOCK(networkstatus_get_latest_consensus,
+       mock_networkstatus_get_latest_consensus);
+
+  ns = networkstatus_get_latest_consensus();
+
+  { /* First router: HSdir */
+    helper_add_hsdir_to_networkstatus(ns, 1, "igor", 1);
+  }
+
+  { /* Second HSDir */
+    helper_add_hsdir_to_networkstatus(ns, 2, "victor", 1);
+  }
+
+  { /* Third relay but not HSDir */
+    helper_add_hsdir_to_networkstatus(ns, 3, "spyro", 0);
+  }
+
+  /* Use a fixed time period and pub key so we always take the same path */
+  ed25519_public_key_t pubkey;
+  uint64_t time_period_num = 17653; // 2 May, 2018, 14:00.
+  memset(&pubkey, 42, sizeof(pubkey));
+
+  hs_get_responsible_hsdirs(&pubkey, time_period_num,
+                            0, 0, responsible_dirs);
+
+  /* Make sure that we only found 2 responsible HSDirs.
+   * The third relay was not an hsdir! */
+  tt_int_op(smartlist_len(responsible_dirs), OP_EQ, 2);
+
+  /** TODO: Build a bigger network and do more tests here */
+
+ done:
+  SMARTLIST_FOREACH(ns->routerstatus_list,
+                    routerstatus_t *, rs, routerstatus_free(rs));
+  smartlist_free(responsible_dirs);
+  smartlist_clear(ns->routerstatus_list);
+  networkstatus_vote_free(mock_ns);
+  cleanup_nodelist();
+}
+
+static void
+mock_directory_initiate_request(directory_request_t *req)
+{
+  (void)req;
+  return;
+}
+
+static int
+mock_hs_desc_encode_descriptor(const hs_descriptor_t *desc,
+                               const ed25519_keypair_t *signing_kp,
+                               const uint8_t *descriptor_cookie,
+                               char **encoded_out)
+{
+  (void)desc;
+  (void)signing_kp;
+  (void)descriptor_cookie;
+
+  tor_asprintf(encoded_out, "lulu");
+  return 0;
+}
+
+static or_state_t dummy_state;
+
+/* Mock function to get fake or state (used for rev counters) */
+static or_state_t *
+get_or_state_replacement(void)
+{
+  return &dummy_state;
+}
+
+static int
+mock_router_have_minimum_dir_info(void)
+{
+  return 1;
+}
+
+/** Test that we correctly detect when the HSDir hash ring changes so that we
+ *  reupload our descriptor. */
+static void
+test_desc_reupload_logic(void *arg)
+{
+  networkstatus_t *ns = NULL;
+
+  (void) arg;
+
+  hs_init();
+
+  MOCK(router_have_minimum_dir_info,
+       mock_router_have_minimum_dir_info);
+  MOCK(get_or_state,
+       get_or_state_replacement);
+  MOCK(networkstatus_get_latest_consensus,
+       mock_networkstatus_get_latest_consensus);
+  MOCK(directory_initiate_request,
+       mock_directory_initiate_request);
+  MOCK(hs_desc_encode_descriptor,
+       mock_hs_desc_encode_descriptor);
+
+  ns = networkstatus_get_latest_consensus();
+
+  /** Test logic:
+   *  1) Upload descriptor to HSDirs
+   *     CHECK that previous_hsdirs list was populated.
+   *  2) Then call router_dir_info_changed() without an HSDir set change.
+   *     CHECK that no reuplod occurs.
+   *  3) Now change the HSDir set, and call dir_info_changed() again.
+   *     CHECK that reupload occurs.
+   *  4) Finally call service_desc_schedule_upload().
+   *     CHECK that previous_hsdirs list was cleared.
+   **/
+
+  /* Let's start by building our descriptor and service */
+  hs_service_descriptor_t *desc = service_descriptor_new();
+  hs_service_t *service = NULL;
+  /* hex-encoded ed25519 pubkey used in hs_build_address.py */
+  char pubkey_hex[] =
+    "d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a";
+  char onion_addr[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+  ed25519_public_key_t pubkey;
+  base16_decode((char*)pubkey.pubkey, sizeof(pubkey.pubkey),
+                pubkey_hex, strlen(pubkey_hex));
+  hs_build_address(&pubkey, HS_VERSION_THREE, onion_addr);
+  service = tor_malloc_zero(sizeof(hs_service_t));
+  memcpy(service->onion_address, onion_addr, sizeof(service->onion_address));
+  ed25519_secret_key_generate(&service->keys.identity_sk, 0);
+  ed25519_public_key_generate(&service->keys.identity_pk,
+                              &service->keys.identity_sk);
+  service->desc_current = desc;
+  /* Also add service to service map */
+  hs_service_ht *service_map = get_hs_service_map();
+  tt_assert(service_map);
+  tt_int_op(hs_service_get_num_services(), OP_EQ, 0);
+  register_service(service_map, service);
+  tt_int_op(hs_service_get_num_services(), OP_EQ, 1);
+
+  /* Now let's create our hash ring: */
+  {
+    helper_add_hsdir_to_networkstatus(ns, 1, "dingus", 1);
+    helper_add_hsdir_to_networkstatus(ns, 2, "clive", 1);
+    helper_add_hsdir_to_networkstatus(ns, 3, "aaron", 1);
+    helper_add_hsdir_to_networkstatus(ns, 4, "lizzie", 1);
+    helper_add_hsdir_to_networkstatus(ns, 5, "daewon", 1);
+    helper_add_hsdir_to_networkstatus(ns, 6, "clarke", 1);
+  }
+
+  /* Now let's upload our desc to all hsdirs */
+  upload_descriptor_to_all(service, desc);
+  /* Check that previous hsdirs were populated */
+  tt_int_op(smartlist_len(desc->previous_hsdirs), OP_EQ, 6);
+
+  /* Poison next upload time so that we can see if it was changed by
+   * router_dir_info_changed(). No changes in hash ring so far, so the upload
+   * time should stay as is. */
+  desc->next_upload_time = 42;
+  router_dir_info_changed();
+  tt_int_op(desc->next_upload_time, OP_EQ, 42);
+
+  /* Now change the HSDir hash ring by swapping nora for aaron.
+   * Start by clearing the hash ring */
+  {
+    SMARTLIST_FOREACH(ns->routerstatus_list,
+                      routerstatus_t *, rs, routerstatus_free(rs));
+    smartlist_clear(ns->routerstatus_list);
+    cleanup_nodelist();
+    routerlist_free_all();
+  }
+
+  { /* Now add back all the nodes */
+    helper_add_hsdir_to_networkstatus(ns, 1, "dingus", 1);
+    helper_add_hsdir_to_networkstatus(ns, 2, "clive", 1);
+    helper_add_hsdir_to_networkstatus(ns, 4, "lizzie", 1);
+    helper_add_hsdir_to_networkstatus(ns, 5, "daewon", 1);
+    helper_add_hsdir_to_networkstatus(ns, 6, "clarke", 1);
+    helper_add_hsdir_to_networkstatus(ns, 7, "nora", 1);
+  }
+
+  /* Now call service_desc_hsdirs_changed() and see that it detected the hash
+     ring change */
+  time_t now = approx_time();
+  tt_assert(now);
+  tt_int_op(service_desc_hsdirs_changed(service, desc), OP_EQ, 1);
+  tt_int_op(smartlist_len(desc->previous_hsdirs), OP_EQ, 6);
+
+  /* Now order another upload and see that we keep having 6 prev hsdirs */
+  upload_descriptor_to_all(service, desc);
+  /* Check that previous hsdirs were populated */
+  tt_int_op(smartlist_len(desc->previous_hsdirs), OP_EQ, 6);
+
+  /* Now restore the HSDir hash ring to its original state by swapping back
+     aaron for nora */
+  /* First clear up the hash ring */
+  {
+    SMARTLIST_FOREACH(ns->routerstatus_list,
+                      routerstatus_t *, rs, routerstatus_free(rs));
+    smartlist_clear(ns->routerstatus_list);
+    cleanup_nodelist();
+    routerlist_free_all();
+  }
+
+  { /* Now populate the hash ring again */
+    helper_add_hsdir_to_networkstatus(ns, 1, "dingus", 1);
+    helper_add_hsdir_to_networkstatus(ns, 2, "clive", 1);
+    helper_add_hsdir_to_networkstatus(ns, 3, "aaron", 1);
+    helper_add_hsdir_to_networkstatus(ns, 4, "lizzie", 1);
+    helper_add_hsdir_to_networkstatus(ns, 5, "daewon", 1);
+    helper_add_hsdir_to_networkstatus(ns, 6, "clarke", 1);
+  }
+
+  /* Check that our algorithm catches this change of hsdirs */
+  tt_int_op(service_desc_hsdirs_changed(service, desc), OP_EQ, 1);
+
+  /* Now pretend that the descriptor changed, and order a reupload to all
+     HSDirs. Make sure that the set of previous HSDirs was cleared. */
+  service_desc_schedule_upload(desc, now, 1);
+  tt_int_op(smartlist_len(desc->previous_hsdirs), OP_EQ, 0);
+
+  /* Now reupload again: see that the prev hsdir set got populated again. */
+  upload_descriptor_to_all(service, desc);
+  tt_int_op(smartlist_len(desc->previous_hsdirs), OP_EQ, 6);
+
+ done:
+  SMARTLIST_FOREACH(ns->routerstatus_list,
+                    routerstatus_t *, rs, routerstatus_free(rs));
+  smartlist_clear(ns->routerstatus_list);
+  networkstatus_vote_free(ns);
+  cleanup_nodelist();
+  hs_free_all();
+}
+
+/** Test disaster SRV computation and caching */
+static void
+test_disaster_srv(void *arg)
+{
+  uint8_t *cached_disaster_srv_one = NULL;
+  uint8_t *cached_disaster_srv_two = NULL;
+  uint8_t srv_one[DIGEST256_LEN] = {0};
+  uint8_t srv_two[DIGEST256_LEN] = {0};
+  uint8_t srv_three[DIGEST256_LEN] = {0};
+  uint8_t srv_four[DIGEST256_LEN] = {0};
+  uint8_t srv_five[DIGEST256_LEN] = {0};
+
+  (void) arg;
+
+  /* Get the cached SRVs: we gonna use them later for verification */
+  cached_disaster_srv_one = get_first_cached_disaster_srv();
+  cached_disaster_srv_two = get_second_cached_disaster_srv();
+
+  /* Compute some srvs */
+  get_disaster_srv(1, srv_one);
+  get_disaster_srv(2, srv_two);
+
+  /* Check that the cached ones where updated */
+  tt_mem_op(cached_disaster_srv_one, OP_EQ, srv_one, DIGEST256_LEN);
+  tt_mem_op(cached_disaster_srv_two, OP_EQ, srv_two, DIGEST256_LEN);
+
+  /* Ask for an SRV that has already been computed */
+  get_disaster_srv(2, srv_two);
+  /* and check that the cache entries have not changed */
+  tt_mem_op(cached_disaster_srv_one, OP_EQ, srv_one, DIGEST256_LEN);
+  tt_mem_op(cached_disaster_srv_two, OP_EQ, srv_two, DIGEST256_LEN);
+
+  /* Ask for a new SRV */
+  get_disaster_srv(3, srv_three);
+  tt_mem_op(cached_disaster_srv_one, OP_EQ, srv_three, DIGEST256_LEN);
+  tt_mem_op(cached_disaster_srv_two, OP_EQ, srv_two, DIGEST256_LEN);
+
+  /* Ask for another SRV: none of the original SRVs should now be cached */
+  get_disaster_srv(4, srv_four);
+  tt_mem_op(cached_disaster_srv_one, OP_EQ, srv_three, DIGEST256_LEN);
+  tt_mem_op(cached_disaster_srv_two, OP_EQ, srv_four, DIGEST256_LEN);
+
+  /* Ask for yet another SRV */
+  get_disaster_srv(5, srv_five);
+  tt_mem_op(cached_disaster_srv_one, OP_EQ, srv_five, DIGEST256_LEN);
+  tt_mem_op(cached_disaster_srv_two, OP_EQ, srv_four, DIGEST256_LEN);
+
+ done:
+  ;
+}
+
+/** Test our HS descriptor request tracker by making various requests and
+ *  checking whether they get tracked properly. */
+static void
+test_hid_serv_request_tracker(void *arg)
+{
+  (void) arg;
+  time_t retval;
+  routerstatus_t *hsdir = NULL, *hsdir2 = NULL, *hsdir3 = NULL;
+  time_t now = approx_time();
+
+  const char *req_key_str_first =
+ "vd4zb6zesaubtrjvdqcr2w7x7lhw2up4Xnw4526ThUNbL5o1go+EdUuEqlKxHkNbnK41pRzizzs";
+  const char *req_key_str_second =
+ "g53o7iavcd62oihswhr24u6czmqws5kpXnw4526ThUNbL5o1go+EdUuEqlKxHkNbnK41pRzizzs";
+  const char *req_key_str_small = "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ";
+
+  /*************************** basic test *******************************/
+
+  /* Get request tracker and make sure it's empty */
+  strmap_t *request_tracker = get_last_hid_serv_requests();
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 0);
+
+  /* Let's register a hid serv request */
+  hsdir = tor_malloc_zero(sizeof(routerstatus_t));
+  memset(hsdir->identity_digest, 'Z', DIGEST_LEN);
+  retval = hs_lookup_last_hid_serv_request(hsdir, req_key_str_first,
+                                           now, 1);
+  tt_int_op(retval, OP_EQ, now);
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 1);
+
+  /* Let's lookup a non-existent hidserv request */
+  retval = hs_lookup_last_hid_serv_request(hsdir, req_key_str_second,
+                                           now+1, 0);
+  tt_int_op(retval, OP_EQ, 0);
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 1);
+
+  /* Let's lookup a real hidserv request */
+  retval = hs_lookup_last_hid_serv_request(hsdir, req_key_str_first,
+                                           now+2, 0);
+  tt_int_op(retval, OP_EQ, now); /* we got it */
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 1);
+
+  /**********************************************************************/
+
+  /* Let's add another request for the same HS but on a different HSDir. */
+  hsdir2 = tor_malloc_zero(sizeof(routerstatus_t));
+  memset(hsdir2->identity_digest, 2, DIGEST_LEN);
+  retval = hs_lookup_last_hid_serv_request(hsdir2, req_key_str_first,
+                                           now+3, 1);
+  tt_int_op(retval, OP_EQ, now+3);
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 2);
+
+  /* Check that we can clean the first request based on time */
+  hs_clean_last_hid_serv_requests(now+3+REND_HID_SERV_DIR_REQUERY_PERIOD);
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 1);
+  /* Check that it doesn't exist anymore */
+  retval = hs_lookup_last_hid_serv_request(hsdir, req_key_str_first,
+                                           now+2, 0);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Now let's add a smaller req key str */
+  hsdir3 = tor_malloc_zero(sizeof(routerstatus_t));
+  memset(hsdir3->identity_digest, 3, DIGEST_LEN);
+  retval = hs_lookup_last_hid_serv_request(hsdir3, req_key_str_small,
+                                           now+4, 1);
+  tt_int_op(retval, OP_EQ, now+4);
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 2);
+
+  /*************************** deleting entries **************************/
+
+  /* Add another request with very short key */
+  retval = hs_lookup_last_hid_serv_request(hsdir, "l",  now, 1);
+  tt_int_op(retval, OP_EQ, now);
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 3);
+
+  /* Try deleting entries with a dummy key. Check that our previous requests
+   * are still there */
+  tor_capture_bugs_(1);
+  hs_purge_hid_serv_from_last_hid_serv_requests("a");
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 3);
+  tor_end_capture_bugs_();
+
+  /* Try another dummy key. Check that requests are still there */
+  {
+    char dummy[2000];
+    memset(dummy, 'Z', 2000);
+    dummy[1999] = '\x00';
+    hs_purge_hid_serv_from_last_hid_serv_requests(dummy);
+    tt_int_op(strmap_size(request_tracker),OP_EQ, 3);
+  }
+
+  /* Another dummy key! */
+  hs_purge_hid_serv_from_last_hid_serv_requests(req_key_str_second);
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 3);
+
+  /* Now actually delete a request! */
+  hs_purge_hid_serv_from_last_hid_serv_requests(req_key_str_first);
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 2);
+
+  /* Purge it all! */
+  hs_purge_last_hid_serv_requests();
+  request_tracker = get_last_hid_serv_requests();
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 0);
+
+ done:
+  tor_free(hsdir);
+  tor_free(hsdir2);
+  tor_free(hsdir3);
+}
+
+static void
+test_parse_extended_hostname(void *arg)
+{
+  (void) arg;
+
+  char address1[] = "fooaddress.onion";
+  char address2[] = "aaaaaaaaaaaaaaaa.onion";
+  char address3[] = "fooaddress.exit";
+  char address4[] = "www.torproject.org";
+  char address5[] = "foo.abcdefghijklmnop.onion";
+  char address6[] = "foo.bar.abcdefghijklmnop.onion";
+  char address7[] = ".abcdefghijklmnop.onion";
+  char address8[] =
+    "www.25njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid.onion";
+
+  tt_assert(BAD_HOSTNAME == parse_extended_hostname(address1));
+  tt_assert(ONION_V2_HOSTNAME == parse_extended_hostname(address2));
+  tt_str_op(address2,OP_EQ, "aaaaaaaaaaaaaaaa");
+  tt_assert(EXIT_HOSTNAME == parse_extended_hostname(address3));
+  tt_assert(NORMAL_HOSTNAME == parse_extended_hostname(address4));
+  tt_assert(ONION_V2_HOSTNAME == parse_extended_hostname(address5));
+  tt_str_op(address5,OP_EQ, "abcdefghijklmnop");
+  tt_assert(ONION_V2_HOSTNAME == parse_extended_hostname(address6));
+  tt_str_op(address6,OP_EQ, "abcdefghijklmnop");
+  tt_assert(BAD_HOSTNAME == parse_extended_hostname(address7));
+  tt_assert(ONION_V3_HOSTNAME == parse_extended_hostname(address8));
+  tt_str_op(address8, OP_EQ,
+            "25njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid");
+
+ done: ;
+}
+
+static void
+test_time_between_tp_and_srv(void *arg)
+{
+  int ret;
+  networkstatus_t ns;
+  (void) arg;
+
+  /* This function should be returning true where "^" are:
+   *
+   *    +------------------------------------------------------------------+
+   *    |                                                                  |
+   *    | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *    | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *    |                                                                  |
+   *    |  $==========|-----------$===========|-----------$===========|    |
+   *    |             ^^^^^^^^^^^^            ^^^^^^^^^^^^                 |
+   *    |                                                                  |
+   *    +------------------------------------------------------------------+
+   */
+
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 00:00:00 UTC", &ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 01:00:00 UTC", &ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+  voting_schedule_recalculate_timing(get_options(), ns.valid_after);
+  ret = hs_in_period_between_tp_and_srv(&ns, 0);
+  tt_int_op(ret, OP_EQ, 0);
+
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 11:00:00 UTC", &ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 12:00:00 UTC", &ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+  voting_schedule_recalculate_timing(get_options(), ns.valid_after);
+  ret = hs_in_period_between_tp_and_srv(&ns, 0);
+  tt_int_op(ret, OP_EQ, 0);
+
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 12:00:00 UTC", &ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC", &ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+  voting_schedule_recalculate_timing(get_options(), ns.valid_after);
+  ret = hs_in_period_between_tp_and_srv(&ns, 0);
+  tt_int_op(ret, OP_EQ, 1);
+
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 23:00:00 UTC", &ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 27 Oct 1985 00:00:00 UTC", &ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+  voting_schedule_recalculate_timing(get_options(), ns.valid_after);
+  ret = hs_in_period_between_tp_and_srv(&ns, 0);
+  tt_int_op(ret, OP_EQ, 1);
+
+  ret = parse_rfc1123_time("Sat, 27 Oct 1985 00:00:00 UTC", &ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 27 Oct 1985 01:00:00 UTC", &ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+  voting_schedule_recalculate_timing(get_options(), ns.valid_after);
+  ret = hs_in_period_between_tp_and_srv(&ns, 0);
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  ;
+}
+
+/************ Reachability Test (it is huge) ****************/
+
+/* Simulate different consensus for client and service. Used by the
+ * reachability test. The SRV and responsible HSDir list are used by all
+ * reachability tests so make them common to simplify setup and teardown. */
+static networkstatus_t *mock_service_ns = NULL;
+static networkstatus_t *mock_client_ns = NULL;
+static sr_srv_t current_srv, previous_srv;
+static smartlist_t *service_responsible_hsdirs = NULL;
+static smartlist_t *client_responsible_hsdirs = NULL;
+
+static networkstatus_t *
+mock_networkstatus_get_live_consensus_service(time_t now)
+{
+  (void) now;
+
+  if (mock_service_ns) {
+    return mock_service_ns;
+  }
+
+  mock_service_ns = tor_malloc_zero(sizeof(networkstatus_t));
+  mock_service_ns->routerstatus_list = smartlist_new();
+  mock_service_ns->type = NS_TYPE_CONSENSUS;
+
+  return mock_service_ns;
+}
+
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus_service(void)
+{
+  return mock_networkstatus_get_live_consensus_service(0);
+}
+
+static networkstatus_t *
+mock_networkstatus_get_live_consensus_client(time_t now)
+{
+  (void) now;
+
+  if (mock_client_ns) {
+    return mock_client_ns;
+  }
+
+  mock_client_ns = tor_malloc_zero(sizeof(networkstatus_t));
+  mock_client_ns->routerstatus_list = smartlist_new();
+  mock_client_ns->type = NS_TYPE_CONSENSUS;
+
+  return mock_client_ns;
+}
+
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus_client(void)
+{
+  return mock_networkstatus_get_live_consensus_client(0);
+}
+
+/* Mock function because we are not trying to test the close circuit that does
+ * an awful lot of checks on the circuit object. */
+static void
+mock_circuit_mark_for_close(circuit_t *circ, int reason, int line,
+                            const char *file)
+{
+  (void) circ;
+  (void) reason;
+  (void) line;
+  (void) file;
+  return;
+}
+
+/* Initialize a big HSDir V3 hash ring. */
+static void
+helper_initialize_big_hash_ring(networkstatus_t *ns)
+{
+  int ret;
+
+  /* Generate 250 hsdirs! :) */
+  for (int counter = 1 ; counter < 251 ; counter++) {
+    /* Let's generate random nickname for each hsdir... */
+    char nickname_binary[8];
+    char nickname_str[13] = {0};
+    crypto_rand(nickname_binary, sizeof(nickname_binary));
+    ret = base64_encode(nickname_str, sizeof(nickname_str),
+                        nickname_binary, sizeof(nickname_binary), 0);
+    tt_int_op(ret, OP_EQ, 12);
+    helper_add_hsdir_to_networkstatus(ns, counter, nickname_str, 1);
+  }
+
+  /* Make sure we have 200 hsdirs in our list */
+  tt_int_op(smartlist_len(ns->routerstatus_list), OP_EQ, 250);
+
+ done:
+  ;
+}
+
+/** Initialize service and publish its descriptor as needed. Return the newly
+ *  allocated service object to the caller. */
+static hs_service_t *
+helper_init_service(time_t now)
+{
+  int retval;
+  hs_service_t *service = hs_service_new(get_options());
+  tt_assert(service);
+  service->config.version = HS_VERSION_THREE;
+  ed25519_secret_key_generate(&service->keys.identity_sk, 0);
+  ed25519_public_key_generate(&service->keys.identity_pk,
+                              &service->keys.identity_sk);
+  /* Register service to global map. */
+  retval = register_service(get_hs_service_map(), service);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Initialize service descriptor */
+  build_all_descriptors(now);
+  tt_assert(service->desc_current);
+  tt_assert(service->desc_next);
+
+ done:
+  return service;
+}
+
+/* Helper function to set the RFC 1123 time string into t. */
+static void
+set_consensus_times(const char *timestr, time_t *t)
+{
+  tt_assert(timestr);
+  tt_assert(t);
+
+  int ret = parse_rfc1123_time(timestr, t);
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  return;
+}
+
+/* Helper function to cleanup the mock consensus (client and service) */
+static void
+cleanup_mock_ns(void)
+{
+  if (mock_service_ns) {
+    SMARTLIST_FOREACH(mock_service_ns->routerstatus_list,
+                      routerstatus_t *, rs, routerstatus_free(rs));
+    smartlist_clear(mock_service_ns->routerstatus_list);
+    mock_service_ns->sr_info.current_srv = NULL;
+    mock_service_ns->sr_info.previous_srv = NULL;
+    networkstatus_vote_free(mock_service_ns);
+    mock_service_ns = NULL;
+  }
+
+  if (mock_client_ns) {
+    SMARTLIST_FOREACH(mock_client_ns->routerstatus_list,
+                      routerstatus_t *, rs, routerstatus_free(rs));
+    smartlist_clear(mock_client_ns->routerstatus_list);
+    mock_client_ns->sr_info.current_srv = NULL;
+    mock_client_ns->sr_info.previous_srv = NULL;
+    networkstatus_vote_free(mock_client_ns);
+    mock_client_ns = NULL;
+  }
+}
+
+/* Helper function to setup a reachability test. Once called, the
+ * cleanup_reachability_test MUST be called at the end. */
+static void
+setup_reachability_test(void)
+{
+  MOCK(circuit_mark_for_close_, mock_circuit_mark_for_close);
+  MOCK(get_or_state, get_or_state_replacement);
+
+  hs_init();
+
+  /* Baseline to start with. */
+  memset(&current_srv, 0, sizeof(current_srv));
+  memset(&previous_srv, 1, sizeof(previous_srv));
+
+  /* Initialize the consensuses. */
+  mock_networkstatus_get_latest_consensus_service();
+  mock_networkstatus_get_latest_consensus_client();
+
+  service_responsible_hsdirs = smartlist_new();
+  client_responsible_hsdirs = smartlist_new();
+}
+
+/* Helper function to cleanup a reachability test initial setup. */
+static void
+cleanup_reachability_test(void)
+{
+  smartlist_free(service_responsible_hsdirs);
+  service_responsible_hsdirs = NULL;
+  smartlist_free(client_responsible_hsdirs);
+  client_responsible_hsdirs = NULL;
+  hs_free_all();
+  cleanup_mock_ns();
+  UNMOCK(get_or_state);
+  UNMOCK(circuit_mark_for_close_);
+}
+
+/* A reachability test always check if the resulting service and client
+ * responsible HSDir for the given parameters are equal.
+ *
+ * Return true iff the same exact nodes are in both list. */
+static int
+are_responsible_hsdirs_equal(void)
+{
+  int count = 0;
+  tt_int_op(smartlist_len(client_responsible_hsdirs), OP_EQ, 6);
+  tt_int_op(smartlist_len(service_responsible_hsdirs), OP_EQ, 8);
+
+  SMARTLIST_FOREACH_BEGIN(client_responsible_hsdirs,
+                          const routerstatus_t *, c_rs) {
+    SMARTLIST_FOREACH_BEGIN(service_responsible_hsdirs,
+                            const routerstatus_t *, s_rs) {
+      if (tor_memeq(c_rs->identity_digest, s_rs->identity_digest,
+                    DIGEST_LEN)) {
+        count++;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(s_rs);
+  } SMARTLIST_FOREACH_END(c_rs);
+
+ done:
+  return (count == 6);
+}
+
+/* Tor doesn't use such a function to get the previous HSDir, it is only used
+ * in node_set_hsdir_index(). We need it here so we can test the reachability
+ * scenario 6 that requires the previous time period to compute the list of
+ * responsible HSDir because of the client state timing. */
+static uint64_t
+get_previous_time_period(time_t now)
+{
+  return hs_get_time_period_num(now) - 1;
+}
+
+/* Configuration of a reachability test scenario. */
+typedef struct reachability_cfg_t {
+  /* Consensus timings to be set. They have to be compliant with
+   * RFC 1123 time format. */
+  const char *service_valid_after;
+  const char *service_valid_until;
+  const char *client_valid_after;
+  const char *client_valid_until;
+
+  /* SRVs that the service and client should use. */
+  sr_srv_t *service_current_srv;
+  sr_srv_t *service_previous_srv;
+  sr_srv_t *client_current_srv;
+  sr_srv_t *client_previous_srv;
+
+  /* A time period function for the service to use for this scenario. For a
+   * successful reachability test, the client always use the current time
+   * period thus why no client function. */
+  uint64_t (*service_time_period_fn)(time_t);
+
+  /* Is the client and service expected to be in a new time period. After
+   * setting the consensus time, the reachability test checks
+   * hs_in_period_between_tp_and_srv() and test the returned value against
+   * this. */
+  unsigned int service_in_new_tp;
+  unsigned int client_in_new_tp;
+
+  /* Some scenario requires a hint that the client, because of its consensus
+   * time, will request the "next" service descriptor so this indicates if it
+   * is the case or not. */
+  unsigned int client_fetch_next_desc;
+} reachability_cfg_t;
+
+/* Some defines to help with semantic while reading a configuration below. */
+#define NOT_IN_NEW_TP 0
+#define IN_NEW_TP 1
+#define DONT_NEED_NEXT_DESC 0
+#define NEED_NEXT_DESC 1
+
+static reachability_cfg_t reachability_scenarios[] = {
+  /* Scenario 1
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|-----------$===========|    |
+   *  |              ^ ^                                                 |
+   *  |              S C                                                 |
+   *  +------------------------------------------------------------------+
+   *
+   *  S: Service, C: Client
+   *
+   *  Service consensus valid_after time is set to 13:00 and client to 15:00,
+   *  both are after TP#1 thus have access to SRV#1. Service and client should
+   *  be using TP#1.
+   */
+
+  { "Sat, 26 Oct 1985 13:00:00 UTC", /* Service valid_after */
+    "Sat, 26 Oct 1985 14:00:00 UTC", /* Service valid_until */
+    "Sat, 26 Oct 1985 15:00:00 UTC", /* Client valid_after */
+    "Sat, 26 Oct 1985 16:00:00 UTC", /* Client valid_until. */
+    &current_srv, NULL, /* Service current and previous SRV */
+    &current_srv, NULL, /* Client current and previous SRV */
+    hs_get_time_period_num, /* Service time period function. */
+    IN_NEW_TP, /* Is service in new TP? */
+    IN_NEW_TP, /* Is client in new TP? */
+    NEED_NEXT_DESC },
+
+  /* Scenario 2
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|-----------$===========|    |
+   *  |                        ^ ^                                       |
+   *  |                        S C                                       |
+   *  +------------------------------------------------------------------+
+   *
+   *  S: Service, C: Client
+   *
+   *  Service consensus valid_after time is set to 23:00 and client to 01:00,
+   *  which makes the client after the SRV#2 and the service just before. The
+   *  service should only be using TP#1. The client should be using TP#1.
+   */
+
+  { "Sat, 26 Oct 1985 23:00:00 UTC", /* Service valid_after */
+    "Sat, 27 Oct 1985 00:00:00 UTC", /* Service valid_until */
+    "Sat, 27 Oct 1985 01:00:00 UTC", /* Client valid_after */
+    "Sat, 27 Oct 1985 02:00:00 UTC", /* Client valid_until. */
+    &previous_srv, NULL, /* Service current and previous SRV */
+    &current_srv, &previous_srv, /* Client current and previous SRV */
+    hs_get_time_period_num, /* Service time period function. */
+    IN_NEW_TP, /* Is service in new TP? */
+    NOT_IN_NEW_TP, /* Is client in new TP? */
+    NEED_NEXT_DESC },
+
+  /* Scenario 3
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|----------$===========|     |
+   *  |                            ^ ^                                   |
+   *  |                            S C                                   |
+   *  +------------------------------------------------------------------+
+   *
+   *  S: Service, C: Client
+   *
+   *  Service consensus valid_after time is set to 03:00 and client to 05:00,
+   *  which makes both after SRV#2. The service should be using TP#1 as its
+   *  current time period. The client should be using TP#1.
+   */
+
+  { "Sat, 27 Oct 1985 03:00:00 UTC", /* Service valid_after */
+    "Sat, 27 Oct 1985 04:00:00 UTC", /* Service valid_until */
+    "Sat, 27 Oct 1985 05:00:00 UTC", /* Client valid_after */
+    "Sat, 27 Oct 1985 06:00:00 UTC", /* Client valid_until. */
+    &current_srv, &previous_srv, /* Service current and previous SRV */
+    &current_srv, &previous_srv, /* Client current and previous SRV */
+    hs_get_time_period_num, /* Service time period function. */
+    NOT_IN_NEW_TP, /* Is service in new TP? */
+    NOT_IN_NEW_TP, /* Is client in new TP? */
+    DONT_NEED_NEXT_DESC },
+
+  /* Scenario 4
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|-----------$===========|    |
+   *  |                                    ^ ^                           |
+   *  |                                    S C                           |
+   *  +------------------------------------------------------------------+
+   *
+   *  S: Service, C: Client
+   *
+   *  Service consensus valid_after time is set to 11:00 and client to 13:00,
+   *  which makes the service before TP#2 and the client just after. The
+   *  service should be using TP#1 as its current time period and TP#2 as the
+   *  next. The client should be using TP#2 time period.
+   */
+
+  { "Sat, 27 Oct 1985 11:00:00 UTC", /* Service valid_after */
+    "Sat, 27 Oct 1985 12:00:00 UTC", /* Service valid_until */
+    "Sat, 27 Oct 1985 13:00:00 UTC", /* Client valid_after */
+    "Sat, 27 Oct 1985 14:00:00 UTC", /* Client valid_until. */
+    &current_srv, &previous_srv, /* Service current and previous SRV */
+    &current_srv, &previous_srv, /* Client current and previous SRV */
+    hs_get_next_time_period_num, /* Service time period function. */
+    NOT_IN_NEW_TP, /* Is service in new TP? */
+    IN_NEW_TP, /* Is client in new TP? */
+    NEED_NEXT_DESC },
+
+  /* Scenario 5
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|-----------$===========|    |
+   *  |                        ^ ^                                       |
+   *  |                        C S                                       |
+   *  +------------------------------------------------------------------+
+   *
+   *  S: Service, C: Client
+   *
+   *  Service consensus valid_after time is set to 01:00 and client to 23:00,
+   *  which makes the service after SRV#2 and the client just before. The
+   *  service should be using TP#1 as its current time period and TP#2 as the
+   *  next. The client should be using TP#1 time period.
+   */
+
+  { "Sat, 27 Oct 1985 01:00:00 UTC", /* Service valid_after */
+    "Sat, 27 Oct 1985 02:00:00 UTC", /* Service valid_until */
+    "Sat, 26 Oct 1985 23:00:00 UTC", /* Client valid_after */
+    "Sat, 27 Oct 1985 00:00:00 UTC", /* Client valid_until. */
+    &current_srv, &previous_srv, /* Service current and previous SRV */
+    &previous_srv, NULL, /* Client current and previous SRV */
+    hs_get_time_period_num, /* Service time period function. */
+    NOT_IN_NEW_TP, /* Is service in new TP? */
+    IN_NEW_TP, /* Is client in new TP? */
+    DONT_NEED_NEXT_DESC },
+
+  /* Scenario 6
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|-----------$===========|    |
+   *  |                                    ^ ^                           |
+   *  |                                    C S                           |
+   *  +------------------------------------------------------------------+
+   *
+   *  S: Service, C: Client
+   *
+   *  Service consensus valid_after time is set to 13:00 and client to 11:00,
+   *  which makes the service outside after TP#2 and the client just before.
+   *  The service should be using TP#1 as its current time period and TP#2 as
+   *  its next. The client should be using TP#1 time period.
+   */
+
+  { "Sat, 27 Oct 1985 13:00:00 UTC", /* Service valid_after */
+    "Sat, 27 Oct 1985 14:00:00 UTC", /* Service valid_until */
+    "Sat, 27 Oct 1985 11:00:00 UTC", /* Client valid_after */
+    "Sat, 27 Oct 1985 12:00:00 UTC", /* Client valid_until. */
+    &current_srv, &previous_srv, /* Service current and previous SRV */
+    &current_srv, &previous_srv, /* Client current and previous SRV */
+    get_previous_time_period, /* Service time period function. */
+    IN_NEW_TP, /* Is service in new TP? */
+    NOT_IN_NEW_TP, /* Is client in new TP? */
+    DONT_NEED_NEXT_DESC },
+
+  /* End marker. */
+  { NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 0, 0, 0}
+};
+
+/* Run a single reachability scenario. num_scenario is the corresponding
+ * scenario number from the documentation. It is used to log it in case of
+ * failure so we know which scenario fails. */
+static int
+run_reachability_scenario(const reachability_cfg_t *cfg, int num_scenario)
+{
+  int ret = -1;
+  hs_service_t *service;
+  uint64_t service_tp, client_tp;
+  ed25519_public_key_t service_blinded_pk, client_blinded_pk;
+
+  setup_reachability_test();
+
+  tt_assert(cfg);
+
+  /* Set service consensus time. */
+  set_consensus_times(cfg->service_valid_after,
+                      &mock_service_ns->valid_after);
+  set_consensus_times(cfg->service_valid_until,
+                      &mock_service_ns->valid_until);
+  set_consensus_times(cfg->service_valid_until,
+                      &mock_service_ns->fresh_until);
+  voting_schedule_recalculate_timing(get_options(),
+                                     mock_service_ns->valid_after);
+  /* Check that service is in the right time period point */
+  tt_int_op(hs_in_period_between_tp_and_srv(mock_service_ns, 0), OP_EQ,
+            cfg->service_in_new_tp);
+
+  /* Set client consensus time. */
+  set_consensus_times(cfg->client_valid_after,
+                      &mock_client_ns->valid_after);
+  set_consensus_times(cfg->client_valid_until,
+                      &mock_client_ns->valid_until);
+  set_consensus_times(cfg->client_valid_until,
+                      &mock_client_ns->fresh_until);
+  voting_schedule_recalculate_timing(get_options(),
+                                     mock_client_ns->valid_after);
+  /* Check that client is in the right time period point */
+  tt_int_op(hs_in_period_between_tp_and_srv(mock_client_ns, 0), OP_EQ,
+            cfg->client_in_new_tp);
+
+  /* Set the SRVs for this scenario. */
+  mock_client_ns->sr_info.current_srv = cfg->client_current_srv;
+  mock_client_ns->sr_info.previous_srv = cfg->client_previous_srv;
+  mock_service_ns->sr_info.current_srv = cfg->service_current_srv;
+  mock_service_ns->sr_info.previous_srv = cfg->service_previous_srv;
+
+  /* Initialize a service to get keys. */
+  update_approx_time(mock_service_ns->valid_after);
+  service = helper_init_service(mock_service_ns->valid_after+1);
+
+  /*
+   * === Client setup ===
+   */
+
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus_client);
+  MOCK(networkstatus_get_latest_consensus,
+       mock_networkstatus_get_latest_consensus_client);
+
+  /* Make networkstatus_is_live() happy. */
+  update_approx_time(mock_client_ns->valid_after);
+  /* Initialize a big hashring for this consensus with the hsdir index set. */
+  helper_initialize_big_hash_ring(mock_client_ns);
+
+  /* Client ONLY use the current time period. This is the whole point of these
+   * reachability test that is to make sure the client can always reach the
+   * service using only its current time period. */
+  client_tp = hs_get_time_period_num(0);
+
+  hs_build_blinded_pubkey(&service->keys.identity_pk, NULL, 0,
+                          client_tp, &client_blinded_pk);
+  hs_get_responsible_hsdirs(&client_blinded_pk, client_tp, 0, 1,
+                            client_responsible_hsdirs);
+  /* Cleanup the nodelist so we can let the service computes its own set of
+   * node with its own hashring. */
+  cleanup_nodelist();
+  tt_int_op(smartlist_len(client_responsible_hsdirs), OP_EQ, 6);
+
+  UNMOCK(networkstatus_get_latest_consensus);
+  UNMOCK(networkstatus_get_live_consensus);
+
+  /*
+   * === Service setup ===
+   */
+
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus_service);
+  MOCK(networkstatus_get_latest_consensus,
+       mock_networkstatus_get_latest_consensus_service);
+
+  /* Make networkstatus_is_live() happy. */
+  update_approx_time(mock_service_ns->valid_after);
+  /* Initialize a big hashring for this consensus with the hsdir index set. */
+  helper_initialize_big_hash_ring(mock_service_ns);
+
+  service_tp = cfg->service_time_period_fn(0);
+
+  hs_build_blinded_pubkey(&service->keys.identity_pk, NULL, 0,
+                          service_tp, &service_blinded_pk);
+
+  /* A service builds two lists of responsible HSDir, for the current and the
+   * next descriptor. Depending on the scenario, the client timing indicate if
+   * it is fetching the current or the next descriptor so we use the
+   * "client_fetch_next_desc" to know which one the client is trying to get to
+   * confirm that the service computes the same hashring for the same blinded
+   * key and service time period function. */
+  hs_get_responsible_hsdirs(&service_blinded_pk, service_tp,
+                            cfg->client_fetch_next_desc, 0,
+                            service_responsible_hsdirs);
+  cleanup_nodelist();
+  tt_int_op(smartlist_len(service_responsible_hsdirs), OP_EQ, 8);
+
+  UNMOCK(networkstatus_get_latest_consensus);
+  UNMOCK(networkstatus_get_live_consensus);
+
+  /* Some testing of the values we just got from the client and service. */
+  tt_mem_op(&client_blinded_pk, OP_EQ, &service_blinded_pk,
+            ED25519_PUBKEY_LEN);
+  tt_int_op(are_responsible_hsdirs_equal(), OP_EQ, 1);
+
+  /* Everything went well. */
+  ret = 0;
+
+ done:
+  cleanup_reachability_test();
+  if (ret == -1) {
+    /* Do this so we can know which scenario failed. */
+    char msg[32];
+    tor_snprintf(msg, sizeof(msg), "Scenario %d failed", num_scenario);
+    tt_fail_msg(msg);
+  }
+  return ret;
+}
+
+static void
+test_reachability(void *arg)
+{
+  (void) arg;
+
+  /* NOTE: An important axiom to understand here is that SRV#N must only be
+   * used with TP#N value. For example, SRV#2 with TP#1 should NEVER be used
+   * together. The HSDir index computation is based on this axiom.*/
+
+  for (int i = 0; reachability_scenarios[i].service_valid_after; ++i) {
+    int ret = run_reachability_scenario(&reachability_scenarios[i], i + 1);
+    if (ret < 0) {
+      return;
+    }
+  }
+}
+
+/** Pick an HSDir for service with <b>onion_identity_pk</b> as a client. Put
+ *  its identity digest in <b>hsdir_digest_out</b>. */
+static void
+helper_client_pick_hsdir(const ed25519_public_key_t *onion_identity_pk,
+                        char *hsdir_digest_out)
+{
+  tt_assert(onion_identity_pk);
+
+  routerstatus_t *client_hsdir = pick_hsdir_v3(onion_identity_pk);
+  tt_assert(client_hsdir);
+  digest_to_base64(hsdir_digest_out, client_hsdir->identity_digest);
+
+ done:
+  ;
+}
+
+static void
+test_hs_indexes(void *arg)
+{
+  int ret;
+  uint64_t period_num = 42;
+  ed25519_public_key_t pubkey;
+
+  (void) arg;
+
+  /* Build the hs_index */
+  {
+    uint8_t hs_index[DIGEST256_LEN];
+    const char *b32_test_vector =
+      "37e5cbbd56a22823714f18f1623ece5983a0d64c78495a8cfab854245e5f9a8a";
+    char test_vector[DIGEST256_LEN];
+    ret = base16_decode(test_vector, sizeof(test_vector), b32_test_vector,
+                        strlen(b32_test_vector));
+    tt_int_op(ret, OP_EQ, sizeof(test_vector));
+    /* Our test vector uses a public key set to 32 bytes of \x42. */
+    memset(&pubkey, '\x42', sizeof(pubkey));
+    hs_build_hs_index(1, &pubkey, period_num, hs_index);
+    tt_mem_op(hs_index, OP_EQ, test_vector, sizeof(hs_index));
+  }
+
+  /* Build the hsdir_index */
+  {
+    uint8_t srv[DIGEST256_LEN];
+    uint8_t hsdir_index[DIGEST256_LEN];
+    const char *b32_test_vector =
+      "db475361014a09965e7e5e4d4a25b8f8d4b8f16cb1d8a7e95eed50249cc1a2d5";
+    char test_vector[DIGEST256_LEN];
+    ret = base16_decode(test_vector, sizeof(test_vector), b32_test_vector,
+                        strlen(b32_test_vector));
+    tt_int_op(ret, OP_EQ, sizeof(test_vector));
+    /* Our test vector uses a public key set to 32 bytes of \x42. */
+    memset(&pubkey, '\x42', sizeof(pubkey));
+    memset(srv, '\x43', sizeof(srv));
+    hs_build_hsdir_index(&pubkey, srv, period_num, hsdir_index);
+    tt_mem_op(hsdir_index, OP_EQ, test_vector, sizeof(hsdir_index));
+  }
+
+ done:
+  ;
+}
+
+#define EARLY_IN_SRV_TO_TP 0
+#define LATE_IN_SRV_TO_TP 1
+#define EARLY_IN_TP_TO_SRV 2
+#define LATE_IN_TP_TO_SRV 3
+
+/** Set the consensus and system time based on <b>position</b>. See the
+ *    following diagram for details:
+ *
+ *  +------------------------------------------------------------------+
+ *  |                                                                  |
+ *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+ *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+ *  |                                                                  |
+ *  |  $==========|-----------$===========|----------$===========|     |
+ *  |                                                                  |
+ *  |                                                                  |
+ *  +------------------------------------------------------------------+
+ */
+static time_t
+helper_set_consensus_and_system_time(networkstatus_t *ns, int position)
+{
+  time_t real_time = 0;
+
+  /* The period between SRV#N and TP#N is from 00:00 to 12:00 UTC. Consensus
+   * valid_after is what matters here, the rest is just to specify the voting
+   * period correctly. */
+  if (position == LATE_IN_SRV_TO_TP) {
+    parse_rfc1123_time("Wed, 13 Apr 2016 11:00:00 UTC", &ns->valid_after);
+    parse_rfc1123_time("Wed, 13 Apr 2016 12:00:00 UTC", &ns->fresh_until);
+    parse_rfc1123_time("Wed, 13 Apr 2016 14:00:00 UTC", &ns->valid_until);
+  } else if (position == EARLY_IN_TP_TO_SRV) {
+    parse_rfc1123_time("Wed, 13 Apr 2016 13:00:00 UTC", &ns->valid_after);
+    parse_rfc1123_time("Wed, 13 Apr 2016 14:00:00 UTC", &ns->fresh_until);
+    parse_rfc1123_time("Wed, 13 Apr 2016 16:00:00 UTC", &ns->valid_until);
+  } else if (position == LATE_IN_TP_TO_SRV) {
+    parse_rfc1123_time("Wed, 13 Apr 2016 23:00:00 UTC", &ns->valid_after);
+    parse_rfc1123_time("Wed, 14 Apr 2016 00:00:00 UTC", &ns->fresh_until);
+    parse_rfc1123_time("Wed, 14 Apr 2016 02:00:00 UTC", &ns->valid_until);
+  } else if (position == EARLY_IN_SRV_TO_TP) {
+    parse_rfc1123_time("Wed, 14 Apr 2016 01:00:00 UTC", &ns->valid_after);
+    parse_rfc1123_time("Wed, 14 Apr 2016 02:00:00 UTC", &ns->fresh_until);
+    parse_rfc1123_time("Wed, 14 Apr 2016 04:00:00 UTC", &ns->valid_until);
+  } else {
+    tt_assert(0);
+  }
+  voting_schedule_recalculate_timing(get_options(), ns->valid_after);
+
+  /* Set system time: pretend to be just 2 minutes before consensus expiry */
+  real_time = ns->valid_until - 120;
+  update_approx_time(real_time);
+
+ done:
+  return real_time;
+}
+
+/** Helper function that carries out the actual test for
+ *  test_client_service_sync() */
+static void
+helper_test_hsdir_sync(networkstatus_t *ns,
+                       int service_position, int client_position,
+                       int client_fetches_next_desc)
+{
+  hs_service_descriptor_t *desc;
+  int retval;
+
+  /** Test logic:
+   *   1) Initialize service time: consensus and system time.
+   *   1.1) Initialize service hash ring
+   *   2) Initialize service and publish descriptors.
+   *   3) Initialize client time: consensus and system time.
+   *   3.1) Initialize client hash ring
+   *   4) Try to fetch descriptor as client, and CHECK that the HSDir picked by
+   *      the client was also picked by service.
+   */
+
+  /* 1) Initialize service time: consensus and real time */
+  time_t now = helper_set_consensus_and_system_time(ns, service_position);
+  helper_initialize_big_hash_ring(ns);
+
+  /* 2) Initialize service */
+  hs_service_t *service = helper_init_service(now);
+  desc = client_fetches_next_desc ? service->desc_next : service->desc_current;
+
+  /* Now let's upload our desc to all hsdirs */
+  upload_descriptor_to_all(service, desc);
+  /* Cleanup right now so we don't memleak on error. */
+  cleanup_nodelist();
+  /* Check that previous hsdirs were populated */
+  tt_int_op(smartlist_len(desc->previous_hsdirs), OP_EQ, 8);
+
+  /* 3) Initialize client time */
+  helper_set_consensus_and_system_time(ns, client_position);
+
+  cleanup_nodelist();
+  SMARTLIST_FOREACH(ns->routerstatus_list,
+                    routerstatus_t *, rs, routerstatus_free(rs));
+  smartlist_clear(ns->routerstatus_list);
+  helper_initialize_big_hash_ring(ns);
+
+  /* 4) Pick 6 HSDirs as a client and check that they were also chosen by the
+        service. */
+  for (int y = 0 ; y < 6 ; y++) {
+    char client_hsdir_b64_digest[BASE64_DIGEST_LEN+1] = {0};
+    helper_client_pick_hsdir(&service->keys.identity_pk,
+                             client_hsdir_b64_digest);
+
+    /* CHECK: Go through the hsdirs chosen by the service and make sure that it
+     * contains the one picked by the client! */
+    retval = smartlist_contains_string(desc->previous_hsdirs,
+                                       client_hsdir_b64_digest);
+    tt_int_op(retval, OP_EQ, 1);
+  }
+
+  /* Finally, try to pick a 7th hsdir and see that NULL is returned since we
+   * exhausted all of them: */
+  tt_assert(!pick_hsdir_v3(&service->keys.identity_pk));
+
+ done:
+  /* At the end: free all services and initialize the subsystem again, we will
+   * need it for next scenario. */
+  cleanup_nodelist();
+  hs_service_free_all();
+  hs_service_init();
+  SMARTLIST_FOREACH(ns->routerstatus_list,
+                    routerstatus_t *, rs, routerstatus_free(rs));
+  smartlist_clear(ns->routerstatus_list);
+}
+
+/** This test ensures that client and service will pick the same HSDirs, under
+ *  various timing scenarios:
+ *  a) Scenario where both client and service are in the time segment between
+ *     SRV#N and TP#N:
+ *  b) Scenario where both client and service are in the time segment between
+ *     TP#N and SRV#N+1.
+ *  c) Scenario where service is between SRV#N and TP#N, but client is between
+ *     TP#N and SRV#N+1.
+ *  d) Scenario where service is between TP#N and SRV#N+1, but client is
+ *     between SRV#N and TP#N.
+ *
+ * This test is important because it tests that upload_descriptor_to_all() is
+ * in synch with pick_hsdir_v3(). That's not the case for the
+ * test_reachability() test which only compares the responsible hsdir sets.
+ */
+static void
+test_client_service_hsdir_set_sync(void *arg)
+{
+  networkstatus_t *ns = NULL;
+
+  (void) arg;
+
+  MOCK(networkstatus_get_latest_consensus,
+       mock_networkstatus_get_latest_consensus);
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+  MOCK(get_or_state,
+       get_or_state_replacement);
+  MOCK(hs_desc_encode_descriptor,
+       mock_hs_desc_encode_descriptor);
+  MOCK(directory_initiate_request,
+       mock_directory_initiate_request);
+
+  hs_init();
+
+  /* Initialize a big hash ring: we want it to be big so that client and
+   * service cannot accidentally select the same HSDirs */
+  ns = networkstatus_get_latest_consensus();
+  tt_assert(ns);
+
+  /** Now test the various synch scenarios. See the helper function for more
+      details: */
+
+  /*  a) Scenario where both client and service are in the time segment between
+   *     SRV#N and TP#N. At this time the client fetches the first HS desc:
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|----------$===========|     |
+   *  |                                  ^ ^                             |
+   *  |                                  S C                             |
+   *  +------------------------------------------------------------------+
+   */
+  helper_test_hsdir_sync(ns, LATE_IN_SRV_TO_TP, LATE_IN_SRV_TO_TP, 0);
+
+  /*  b) Scenario where both client and service are in the time segment between
+   *     TP#N and SRV#N+1. At this time the client fetches the second HS
+   *     desc:
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|-----------$===========|    |
+   *  |                      ^ ^                                         |
+   *  |                      S C                                         |
+   *  +------------------------------------------------------------------+
+   */
+  helper_test_hsdir_sync(ns, LATE_IN_TP_TO_SRV, LATE_IN_TP_TO_SRV, 1);
+
+  /*  c) Scenario where service is between SRV#N and TP#N, but client is
+   *     between TP#N and SRV#N+1. Client is forward in time so it fetches the
+   *     second HS desc.
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|-----------$===========|    |
+   *  |                                    ^ ^                           |
+   *  |                                    S C                           |
+   *  +------------------------------------------------------------------+
+   */
+  helper_test_hsdir_sync(ns, LATE_IN_SRV_TO_TP, EARLY_IN_TP_TO_SRV, 1);
+
+  /*  d) Scenario where service is between TP#N and SRV#N+1, but client is
+   *     between SRV#N and TP#N. Client is backwards in time so it fetches the
+   *     first HS desc.
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|-----------$===========|    |
+   *  |                                    ^ ^                           |
+   *  |                                    C S                           |
+   *  +------------------------------------------------------------------+
+   */
+  helper_test_hsdir_sync(ns, EARLY_IN_TP_TO_SRV, LATE_IN_SRV_TO_TP, 0);
+
+  /*  e) Scenario where service is between SRV#N and TP#N, but client is
+   *     between TP#N-1 and SRV#3. Client is backwards in time so it fetches
+   *     the first HS desc.
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|-----------$===========|    |
+   *  |                        ^ ^                                       |
+   *  |                        C S                                       |
+   *  +------------------------------------------------------------------+
+   */
+  helper_test_hsdir_sync(ns, EARLY_IN_SRV_TO_TP, LATE_IN_TP_TO_SRV, 0);
+
+  /*  f) Scenario where service is between TP#N and SRV#N+1, but client is
+   *     between SRV#N+1 and TP#N+1. Client is forward in time so it fetches
+   *     the second HS desc.
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|-----------$===========|    |
+   *  |                        ^ ^                                       |
+   *  |                        S C                                       |
+   *  +------------------------------------------------------------------+
+   */
+  helper_test_hsdir_sync(ns, LATE_IN_TP_TO_SRV, EARLY_IN_SRV_TO_TP, 1);
+
+ done:
+  networkstatus_vote_free(ns);
+  nodelist_free_all();
+  hs_free_all();
+}
+
+struct testcase_t hs_common_tests[] = {
+  { "build_address", test_build_address, TT_FORK,
+    NULL, NULL },
+  { "validate_address", test_validate_address, TT_FORK,
+    NULL, NULL },
+  { "time_period", test_time_period, TT_FORK,
+    NULL, NULL },
+  { "start_time_of_next_time_period", test_start_time_of_next_time_period,
+    TT_FORK, NULL, NULL },
+  { "responsible_hsdirs", test_responsible_hsdirs, TT_FORK,
+    NULL, NULL },
+  { "desc_reupload_logic", test_desc_reupload_logic, TT_FORK,
+    NULL, NULL },
+  { "disaster_srv", test_disaster_srv, TT_FORK,
+    NULL, NULL },
+  { "hid_serv_request_tracker", test_hid_serv_request_tracker, TT_FORK,
+    NULL, NULL },
+  { "parse_extended_hostname", test_parse_extended_hostname, TT_FORK,
+    NULL, NULL },
+  { "time_between_tp_and_srv", test_time_between_tp_and_srv, TT_FORK,
+    NULL, NULL },
+  { "reachability", test_reachability, TT_FORK,
+    NULL, NULL },
+  { "client_service_hsdir_set_sync", test_client_service_hsdir_set_sync,
+    TT_FORK, NULL, NULL },
+  { "hs_indexes", test_hs_indexes, TT_FORK,
+    NULL, NULL },
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_hs_config.c b/src/test/test_hs_config.c
new file mode 100644
index 0000000000..c2c556307d
--- /dev/null
+++ b/src/test/test_hs_config.c
@@ -0,0 +1,517 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_config.c
+ * \brief Test hidden service configuration functionality.
+ */
+
+#define CONFIG_PRIVATE
+#define HS_SERVICE_PRIVATE
+
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
+
+#include "app/config/config.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_config.h"
+#include "feature/hs/hs_service.h"
+#include "feature/rend/rendservice.h"
+
+static int
+helper_config_service(const char *conf, int validate_only)
+{
+  int ret = 0;
+  or_options_t *options = NULL;
+  tt_assert(conf);
+  options = helper_parse_options(conf);
+  tt_assert(options);
+  ret = hs_config_service_all(options, validate_only);
+ done:
+  or_options_free(options);
+  return ret;
+}
+
+static void
+test_invalid_service(void *arg)
+{
+  int ret;
+
+  (void) arg;
+
+  /* Try with a missing port configuration. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 1\n"; /* Wrong not supported version. */
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceVersion must be between 2 and 3");
+    teardown_capture_of_logs();
+  }
+
+  /* Bad value of HiddenServiceAllowUnknownPorts. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServiceAllowUnknownPorts 2\n"; /* Should be 0 or 1. */
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceAllowUnknownPorts must be "
+                              "between 0 and 1, not 2");
+    teardown_capture_of_logs();
+  }
+
+  /* Bad value of HiddenServiceDirGroupReadable */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServiceDirGroupReadable 2\n"; /* Should be 0 or 1. */
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceDirGroupReadable must be "
+                              "between 0 and 1, not 2");
+    teardown_capture_of_logs();
+  }
+
+  /* Bad value of HiddenServiceMaxStreamsCloseCircuit */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServiceMaxStreamsCloseCircuit 2\n"; /* Should be 0 or 1. */
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceMaxStreamsCloseCircuit must "
+                              "be between 0 and 1, not 2");
+    teardown_capture_of_logs();
+  }
+
+  /* Too much max streams. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 80\n"
+      "HiddenServiceMaxStreams 65536\n"; /* One too many. */
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceMaxStreams must be between "
+                              "0 and 65535, not 65536");
+    teardown_capture_of_logs();
+  }
+
+  /* Duplicate directory directive. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 80\n"
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 81\n";
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("Another hidden service is already "
+                              "configured for directory");
+    teardown_capture_of_logs();
+  }
+
+  /* Bad port. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 65536\n";
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("Missing or invalid port");
+    teardown_capture_of_logs();
+  }
+
+  /* Bad target addr:port separation. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 80 127.0.0.1 8000\n";
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServicePort parse error: "
+                              "invalid port mapping");
+    teardown_capture_of_logs();
+  }
+
+  /* Out of order directives. */
+  {
+    const char *conf =
+      "HiddenServiceVersion 2\n"
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServicePort 80\n";
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceVersion with no preceding "
+                              "HiddenServiceDir directive");
+    teardown_capture_of_logs();
+  }
+
+ done:
+  ;
+}
+
+static void
+test_valid_service(void *arg)
+{
+  int ret;
+
+  (void) arg;
+
+  /* Mix of v2 and v3. Still valid. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 80\n"
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs2\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 81\n"
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 82\n";
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+ done:
+  ;
+}
+
+static void
+test_invalid_service_v2(void *arg)
+{
+  int validate_only = 1, ret;
+
+  (void) arg;
+
+  /* Try with a missing port configuration. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n";
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, validate_only);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("with no ports configured.");
+    teardown_capture_of_logs();
+  }
+
+  /* Too many introduction points. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 80\n"
+      "HiddenServiceNumIntroductionPoints 11\n"; /* One too many. */
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, validate_only);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceNumIntroductionPoints should "
+                              "be between 0 and 10, not 11");
+    teardown_capture_of_logs();
+  }
+
+  /* Too little introduction points. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 80\n"
+      "HiddenServiceNumIntroductionPoints -1\n";
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, validate_only);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceNumIntroductionPoints should "
+                              "be between 0 and 10, not -1");
+    teardown_capture_of_logs();
+  }
+
+  /* Bad authorized client type. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 80\n"
+      "HiddenServiceAuthorizeClient blah alice,bob\n"; /* blah is no good. */
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, validate_only);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceAuthorizeClient contains "
+                              "unrecognized auth-type");
+    teardown_capture_of_logs();
+  }
+
+ done:
+  ;
+}
+
+static void
+test_valid_service_v2(void *arg)
+{
+  int ret;
+
+  (void) arg;
+
+  /* Valid complex configuration. Basic client authorization. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 80\n"
+      "HiddenServicePort 22 localhost:22\n"
+#ifdef HAVE_SYS_UN_H
+      "HiddenServicePort 42 unix:/path/to/socket\n"
+#endif
+      "HiddenServiceAuthorizeClient basic alice,bob,eve\n"
+      "HiddenServiceAllowUnknownPorts 1\n"
+      "HiddenServiceMaxStreams 42\n"
+      "HiddenServiceMaxStreamsCloseCircuit 0\n"
+      "HiddenServiceDirGroupReadable 1\n"
+      "HiddenServiceNumIntroductionPoints 7\n";
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Valid complex configuration. Stealth client authorization. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs2\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 65535\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+#ifdef HAVE_SYS_UN_H
+      "HiddenServicePort 9000 unix:/path/to/socket\n"
+#endif
+      "HiddenServiceAuthorizeClient stealth charlie,romeo\n"
+      "HiddenServiceAllowUnknownPorts 0\n"
+      "HiddenServiceMaxStreams 42\n"
+      "HiddenServiceMaxStreamsCloseCircuit 0\n"
+      "HiddenServiceDirGroupReadable 1\n"
+      "HiddenServiceNumIntroductionPoints 8\n";
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+ done:
+  ;
+}
+
+static void
+test_invalid_service_v3(void *arg)
+{
+  int validate_only = 1, ret;
+
+  (void) arg;
+
+  /* Try with a missing port configuration. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 3\n";
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, validate_only);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("with no ports configured.");
+    teardown_capture_of_logs();
+  }
+
+  /* Too many introduction points. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 80\n"
+      "HiddenServiceNumIntroductionPoints 21\n"; /* One too many. */
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, validate_only);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceNumIntroductionPoints must "
+                              "be between 3 and 20, not 21.");
+    teardown_capture_of_logs();
+  }
+
+  /* Too little introduction points. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 80\n"
+      "HiddenServiceNumIntroductionPoints 1\n";
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, validate_only);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceNumIntroductionPoints must "
+                              "be between 3 and 20, not 1.");
+    teardown_capture_of_logs();
+  }
+
+  /* v2-specific HiddenServiceAuthorizeClient set. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServiceAuthorizeClient stealth client1\n";
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, validate_only);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("Hidden service option "
+                              "HiddenServiceAuthorizeClient is incompatible "
+                              "with version 3 of service in "
+                              "/tmp/tor-test-hs-RANDOM/hs1");
+    teardown_capture_of_logs();
+  }
+
+ done:
+  ;
+}
+
+static void
+test_valid_service_v3(void *arg)
+{
+  int ret;
+
+  (void) arg;
+
+  /* Valid complex configuration. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 80\n"
+      "HiddenServicePort 22 localhost:22\n"
+#ifdef HAVE_SYS_UN_H
+      "HiddenServicePort 42 unix:/path/to/socket\n"
+#endif
+      "HiddenServiceAllowUnknownPorts 1\n"
+      "HiddenServiceMaxStreams 42\n"
+      "HiddenServiceMaxStreamsCloseCircuit 0\n"
+      "HiddenServiceDirGroupReadable 1\n"
+      "HiddenServiceNumIntroductionPoints 7\n";
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Valid complex configuration. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs2\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 65535\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+#ifdef HAVE_SYS_UN_H
+      "HiddenServicePort 9000 unix:/path/to/socket\n"
+#endif
+      "HiddenServiceAllowUnknownPorts 0\n"
+      "HiddenServiceMaxStreams 42\n"
+      "HiddenServiceMaxStreamsCloseCircuit 0\n"
+      "HiddenServiceDirGroupReadable 1\n"
+      "HiddenServiceNumIntroductionPoints 20\n";
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Mix of v2 and v3. Still valid. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 80\n"
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs2\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 81\n"
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 82\n";
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+ done:
+  ;
+}
+
+static void
+test_staging_service_v3(void *arg)
+{
+  int ret;
+
+  (void) arg;
+
+  /* We don't validate a service object, this is the service test that are in
+   * charge of doing so. We just check for the stable state after
+   * registration. */
+
+  hs_init();
+
+  /* Time for a valid v3 service that should get staged. */
+  const char *conf =
+    "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs2\n"
+    "HiddenServiceVersion 3\n"
+    "HiddenServicePort 65535\n"
+    "HiddenServicePort 22 1.1.1.1:22\n"
+#ifdef HAVE_SYS_UN_H
+    "HiddenServicePort 9000 unix:/path/to/socket\n"
+#endif
+    "HiddenServiceAllowUnknownPorts 0\n"
+    "HiddenServiceMaxStreams 42\n"
+    "HiddenServiceMaxStreamsCloseCircuit 0\n"
+    "HiddenServiceDirGroupReadable 1\n"
+    "HiddenServiceNumIntroductionPoints 20\n";
+  ret = helper_config_service(conf, 0);
+  tt_int_op(ret, OP_EQ, 0);
+  /* Ok, we have a service in our map! Registration went well. */
+  tt_int_op(get_hs_service_staging_list_size(), OP_EQ, 1);
+  /* Make sure we don't have a magic v2 service out of this. */
+  tt_int_op(rend_num_services(), OP_EQ, 0);
+
+ done:
+  hs_free_all();
+}
+
+struct testcase_t hs_config_tests[] = {
+  /* Invalid service not specific to any version. */
+  { "invalid_service", test_invalid_service, TT_FORK,
+    NULL, NULL },
+  { "valid_service", test_valid_service, TT_FORK,
+    NULL, NULL },
+
+  /* Test case only for version 2. */
+  { "invalid_service_v2", test_invalid_service_v2, TT_FORK,
+    NULL, NULL },
+  { "valid_service_v2", test_valid_service_v2, TT_FORK,
+    NULL, NULL },
+
+  /* Test case only for version 3. */
+  { "invalid_service_v3", test_invalid_service_v3, TT_FORK,
+    NULL, NULL },
+  { "valid_service_v3", test_valid_service_v3, TT_FORK,
+    NULL, NULL },
+
+  /* Test service staging. */
+  { "staging_service_v3", test_staging_service_v3, TT_FORK,
+    NULL, NULL },
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_hs_control.c b/src/test/test_hs_control.c
new file mode 100644
index 0000000000..ba67712f1b
--- /dev/null
+++ b/src/test/test_hs_control.c
@@ -0,0 +1,194 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_control.c
+ * \brief Unit tests for hidden service control port event and command.
+ **/
+
+#define CONTROL_PRIVATE
+
+#include "core/or/or.h"
+#include "test/test.h"
+#include "feature/control/control.h"
+#include "app/config/config.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_control.h"
+#include "feature/nodelist/nodelist.h"
+
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+#include "lib/crypt_ops/crypto_format.h"
+
+#include "test/test_helpers.h"
+
+/* mock ID digest and longname for node that's in nodelist */
+#define HSDIR_EXIST_ID \
+  "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA" \
+  "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+#define STR_HSDIR_EXIST_LONGNAME \
+  "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=TestDir"
+#define STR_HSDIR_NONE_EXIST_LONGNAME \
+  "$BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"
+
+/* Helper global variable for hidden service descriptor event test.
+ * It's used as a pointer to dynamically created message buffer in
+ * send_control_event_string_replacement function, which mocks
+ * send_control_event_string function.
+ *
+ * Always free it after use! */
+static char *received_msg = NULL;
+
+/** Mock function for send_control_event_string
+ */
+static void
+queue_control_event_string_replacement(uint16_t event, char *msg)
+{
+  (void) event;
+  tor_free(received_msg);
+  received_msg = msg;
+}
+
+/** Mock function for node_describe_longname_by_id, it returns either
+ * STR_HSDIR_EXIST_LONGNAME or STR_HSDIR_NONE_EXIST_LONGNAME
+ */
+static const char *
+node_describe_longname_by_id_replacement(const char *id_digest)
+{
+  if (!strcmp(id_digest, HSDIR_EXIST_ID)) {
+    return STR_HSDIR_EXIST_LONGNAME;
+  } else {
+    return STR_HSDIR_NONE_EXIST_LONGNAME;
+  }
+}
+
+/* HSDir fetch index is a series of 'D' */
+#define HSDIR_INDEX_FETCH_HEX \
+  "4343434343434343434343434343434343434343434343434343434343434343"
+#define HSDIR_INDEX_STORE_HEX \
+  "4444444444444444444444444444444444444444444444444444444444444444"
+
+static const node_t *
+mock_node_get_by_id(const char *digest)
+{
+  static node_t node;
+  memcpy(node.identity, digest, DIGEST_LEN);
+  memset(node.hsdir_index.fetch, 'C', DIGEST256_LEN);
+  memset(node.hsdir_index.store_first, 'D', DIGEST256_LEN);
+  return &node;
+}
+
+static void
+test_hs_desc_event(void *arg)
+{
+  int ret;
+  char *expected_msg = NULL;
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+  ed25519_keypair_t identity_kp;
+  ed25519_public_key_t blinded_pk;
+  char base64_blinded_pk[ED25519_BASE64_LEN + 1];
+  routerstatus_t hsdir_rs;
+  hs_ident_dir_conn_t ident;
+
+  (void) arg;
+  MOCK(queue_control_event_string,
+       queue_control_event_string_replacement);
+  MOCK(node_describe_longname_by_id,
+       node_describe_longname_by_id_replacement);
+  MOCK(node_get_by_id, mock_node_get_by_id);
+
+  /* Setup what we need for this test. */
+  ed25519_keypair_generate(&identity_kp, 0);
+  hs_build_address(&identity_kp.pubkey, HS_VERSION_THREE, onion_address);
+  ret = hs_address_is_valid(onion_address);
+  tt_int_op(ret, OP_EQ, 1);
+  memset(&blinded_pk, 'B', sizeof(blinded_pk));
+  memset(&hsdir_rs, 0, sizeof(hsdir_rs));
+  memcpy(hsdir_rs.identity_digest, HSDIR_EXIST_ID, DIGEST_LEN);
+  ret = ed25519_public_to_base64(base64_blinded_pk, &blinded_pk);
+  tt_int_op(ret, OP_EQ, 0);
+  memcpy(&ident.identity_pk, &identity_kp.pubkey,
+         sizeof(ed25519_public_key_t));
+  memcpy(&ident.blinded_pk, &blinded_pk, sizeof(blinded_pk));
+
+  /* HS_DESC REQUESTED ... */
+  hs_control_desc_event_requested(&identity_kp.pubkey, base64_blinded_pk,
+                                  &hsdir_rs);
+  tor_asprintf(&expected_msg, "650 HS_DESC REQUESTED %s NO_AUTH "
+               STR_HSDIR_EXIST_LONGNAME " %s HSDIR_INDEX="
+               HSDIR_INDEX_FETCH_HEX "\r\n",
+               onion_address, base64_blinded_pk);
+  tt_assert(received_msg);
+  tt_str_op(received_msg, OP_EQ, expected_msg);
+  tor_free(received_msg);
+  tor_free(expected_msg);
+
+  /* HS_DESC CREATED... */
+  hs_control_desc_event_created(onion_address, &blinded_pk);
+  tor_asprintf(&expected_msg, "650 HS_DESC CREATED %s UNKNOWN "
+                              "UNKNOWN %s\r\n",
+               onion_address, base64_blinded_pk);
+  tt_assert(received_msg);
+  tt_str_op(received_msg, OP_EQ, expected_msg);
+  tor_free(received_msg);
+  tor_free(expected_msg);
+
+  /* HS_DESC UPLOAD... */
+  uint8_t hsdir_index_store[DIGEST256_LEN];
+  memset(hsdir_index_store, 'D', sizeof(hsdir_index_store));
+  hs_control_desc_event_upload(onion_address, HSDIR_EXIST_ID,
+                               &blinded_pk, hsdir_index_store);
+  tor_asprintf(&expected_msg, "650 HS_DESC UPLOAD %s UNKNOWN "
+                              STR_HSDIR_EXIST_LONGNAME " %s "
+                              "HSDIR_INDEX=" HSDIR_INDEX_STORE_HEX "\r\n",
+               onion_address, base64_blinded_pk);
+  tt_assert(received_msg);
+  tt_str_op(received_msg, OP_EQ, expected_msg);
+  tor_free(received_msg);
+  tor_free(expected_msg);
+
+  /* HS_DESC FAILED... */
+  hs_control_desc_event_failed(&ident, HSDIR_EXIST_ID, "BAD_DESC");
+  tor_asprintf(&expected_msg, "650 HS_DESC FAILED %s NO_AUTH "
+                              STR_HSDIR_EXIST_LONGNAME " %s "
+                              "REASON=BAD_DESC\r\n",
+               onion_address, base64_blinded_pk);
+  tt_assert(received_msg);
+  tt_str_op(received_msg, OP_EQ, expected_msg);
+  tor_free(received_msg);
+  tor_free(expected_msg);
+
+  /* HS_DESC RECEIVED... */
+  hs_control_desc_event_received(&ident, HSDIR_EXIST_ID);
+  tor_asprintf(&expected_msg, "650 HS_DESC RECEIVED %s NO_AUTH "
+                              STR_HSDIR_EXIST_LONGNAME " %s\r\n",
+               onion_address, base64_blinded_pk);
+  tt_assert(received_msg);
+  tt_str_op(received_msg, OP_EQ, expected_msg);
+  tor_free(received_msg);
+  tor_free(expected_msg);
+
+  /* HS_DESC UPLOADED... */
+  hs_control_desc_event_uploaded(&ident, HSDIR_EXIST_ID);
+  tor_asprintf(&expected_msg, "650 HS_DESC UPLOADED %s UNKNOWN "
+                              STR_HSDIR_EXIST_LONGNAME "\r\n",
+               onion_address);
+  tt_assert(received_msg);
+  tt_str_op(received_msg, OP_EQ, expected_msg);
+  tor_free(received_msg);
+  tor_free(expected_msg);
+
+ done:
+  UNMOCK(queue_control_event_string);
+  UNMOCK(node_describe_longname_by_id);
+  UNMOCK(node_get_by_id);
+  tor_free(received_msg);
+  tor_free(expected_msg);
+}
+
+struct testcase_t hs_control_tests[] = {
+  { "hs_desc_event", test_hs_desc_event, TT_FORK,
+    NULL, NULL },
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_hs_descriptor.c b/src/test/test_hs_descriptor.c
new file mode 100644
index 0000000000..de584ed47a
--- /dev/null
+++ b/src/test/test_hs_descriptor.c
@@ -0,0 +1,965 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_descriptor.c
+ * \brief Test hidden service descriptor encoding and decoding.
+ */
+
+#define HS_DESCRIPTOR_PRIVATE
+
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "trunnel/ed25519_cert.h"
+#include "core/or/or.h"
+#include "feature/hs/hs_descriptor.h"
+#include "test/test.h"
+#include "feature/nodelist/torcert.h"
+
+#include "test/hs_test_helpers.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
+
+#ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
+DISABLE_GCC_WARNING(overlength-strings)
+/* We allow huge string constants in the unit tests, but not in the code
+ * at large. */
+#endif
+#include "test_hs_descriptor.inc"
+ENABLE_GCC_WARNING(overlength-strings)
+
+/* Mock function to fill all bytes with 1 */
+static void
+mock_crypto_strongest_rand(uint8_t *out, size_t out_len)
+{
+  memset(out, 1, out_len);
+}
+
+/* Test certificate encoding put in a descriptor. */
+static void
+test_cert_encoding(void *arg)
+{
+  int ret;
+  char *encoded = NULL;
+  time_t now = time(NULL);
+  ed25519_keypair_t kp;
+  ed25519_public_key_t signed_key;
+  ed25519_secret_key_t secret_key;
+  tor_cert_t *cert = NULL;
+
+  (void) arg;
+
+  ret = ed25519_keypair_generate(&kp, 0);
+  tt_int_op(ret, == , 0);
+  ret = ed25519_secret_key_generate(&secret_key, 0);
+  tt_int_op(ret, == , 0);
+  ret = ed25519_public_key_generate(&signed_key, &secret_key);
+  tt_int_op(ret, == , 0);
+
+  cert = tor_cert_create(&kp, CERT_TYPE_SIGNING_AUTH, &signed_key,
+                         now, 3600 * 2, CERT_FLAG_INCLUDE_SIGNING_KEY);
+  tt_assert(cert);
+
+  /* Test the certificate encoding function. */
+  ret = tor_cert_encode_ed22519(cert, &encoded);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Validated the certificate string. */
+  {
+    char *end, *pos = encoded;
+    char *b64_cert, buf[256];
+    size_t b64_cert_len;
+    tor_cert_t *parsed_cert;
+
+    tt_int_op(strcmpstart(pos, "-----BEGIN ED25519 CERT-----\n"), OP_EQ, 0);
+    pos += strlen("-----BEGIN ED25519 CERT-----\n");
+
+    /* Isolate the base64 encoded certificate and try to decode it. */
+    end = strstr(pos, "-----END ED25519 CERT-----");
+    tt_assert(end);
+    b64_cert = pos;
+    b64_cert_len = end - pos;
+    ret = base64_decode(buf, sizeof(buf), b64_cert, b64_cert_len);
+    tt_int_op(ret, OP_GT, 0);
+    /* Parseable? */
+    parsed_cert = tor_cert_parse((uint8_t *) buf, ret);
+    tt_assert(parsed_cert);
+    /* Signature is valid? */
+    ret = tor_cert_checksig(parsed_cert, &kp.pubkey, now + 10);
+    tt_int_op(ret, OP_EQ, 0);
+    ret = tor_cert_eq(cert, parsed_cert);
+    tt_int_op(ret, OP_EQ, 1);
+    /* The cert did have the signing key? */
+    ret= ed25519_pubkey_eq(&parsed_cert->signing_key, &kp.pubkey);
+    tt_int_op(ret, OP_EQ, 1);
+    tor_cert_free(parsed_cert);
+
+    /* Get to the end part of the certificate. */
+    pos += b64_cert_len;
+    tt_int_op(strcmpstart(pos, "-----END ED25519 CERT-----"), OP_EQ, 0);
+    pos += strlen("-----END ED25519 CERT-----");
+    tt_str_op(pos, OP_EQ, "");
+  }
+
+ done:
+  tor_cert_free(cert);
+  tor_free(encoded);
+}
+
+/* Test the descriptor padding. */
+static void
+test_descriptor_padding(void *arg)
+{
+  char *plaintext;
+  size_t plaintext_len, padded_len;
+  uint8_t *padded_plaintext = NULL;
+
+/* Example: if l = 129, the ceiled division gives 2 and then multiplied by 128
+ * to give 256. With l = 127, ceiled division gives 1 then times 128. */
+#define PADDING_EXPECTED_LEN(l) \
+  CEIL_DIV(l, HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE) * \
+  HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE
+
+  (void) arg;
+
+  { /* test #1: no padding */
+    plaintext_len = HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE;
+    plaintext = tor_malloc(plaintext_len);
+    padded_len = build_plaintext_padding(plaintext, plaintext_len,
+                                         &padded_plaintext);
+    tt_assert(padded_plaintext);
+    tor_free(plaintext);
+    /* Make sure our padding has been zeroed. */
+    tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len,
+                              padded_len - plaintext_len), OP_EQ, 1);
+    tor_free(padded_plaintext);
+    /* Never never have a padded length smaller than the plaintext. */
+    tt_int_op(padded_len, OP_GE, plaintext_len);
+    tt_int_op(padded_len, OP_EQ, PADDING_EXPECTED_LEN(plaintext_len));
+  }
+
+  { /* test #2: one byte padding? */
+    plaintext_len = HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE - 1;
+    plaintext = tor_malloc(plaintext_len);
+    padded_plaintext = NULL;
+    padded_len = build_plaintext_padding(plaintext, plaintext_len,
+                                         &padded_plaintext);
+    tt_assert(padded_plaintext);
+    tor_free(plaintext);
+    /* Make sure our padding has been zeroed. */
+    tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len,
+                              padded_len - plaintext_len), OP_EQ, 1);
+    tor_free(padded_plaintext);
+    /* Never never have a padded length smaller than the plaintext. */
+    tt_int_op(padded_len, OP_GE, plaintext_len);
+    tt_int_op(padded_len, OP_EQ, PADDING_EXPECTED_LEN(plaintext_len));
+  }
+
+  { /* test #3: Lots more bytes of padding? */
+    plaintext_len = HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE + 1;
+    plaintext = tor_malloc(plaintext_len);
+    padded_plaintext = NULL;
+    padded_len = build_plaintext_padding(plaintext, plaintext_len,
+                                         &padded_plaintext);
+    tt_assert(padded_plaintext);
+    tor_free(plaintext);
+    /* Make sure our padding has been zeroed. */
+    tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len,
+                              padded_len - plaintext_len), OP_EQ, 1);
+    tor_free(padded_plaintext);
+    /* Never never have a padded length smaller than the plaintext. */
+    tt_int_op(padded_len, OP_GE, plaintext_len);
+    tt_int_op(padded_len, OP_EQ, PADDING_EXPECTED_LEN(plaintext_len));
+  }
+
+ done:
+  return;
+}
+
+static void
+test_link_specifier(void *arg)
+{
+  ssize_t ret;
+  hs_desc_link_specifier_t spec;
+  smartlist_t *link_specifiers = smartlist_new();
+  char buf[256];
+  char *b64 = NULL;
+  link_specifier_t *ls = NULL;
+
+  (void) arg;
+
+  /* Always this port. */
+  spec.u.ap.port = 42;
+  smartlist_add(link_specifiers, &spec);
+
+  /* Test IPv4 for starter. */
+  {
+    uint32_t ipv4;
+
+    spec.type = LS_IPV4;
+    ret = tor_addr_parse(&spec.u.ap.addr, "1.2.3.4");
+    tt_int_op(ret, OP_EQ, AF_INET);
+    b64 = encode_link_specifiers(link_specifiers);
+    tt_assert(b64);
+
+    /* Decode it and validate the format. */
+    ret = base64_decode(buf, sizeof(buf), b64, strlen(b64));
+    tt_int_op(ret, OP_GT, 0);
+    /* First byte is the number of link specifier. */
+    tt_int_op(get_uint8(buf), OP_EQ, 1);
+    ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1);
+    tt_int_op(ret, OP_EQ, 8);
+    /* Should be 2 bytes for port and 4 bytes for IPv4. */
+    tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, 6);
+    ipv4 = link_specifier_get_un_ipv4_addr(ls);
+    tt_int_op(tor_addr_to_ipv4h(&spec.u.ap.addr), OP_EQ, ipv4);
+    tt_int_op(link_specifier_get_un_ipv4_port(ls), OP_EQ, spec.u.ap.port);
+
+    link_specifier_free(ls);
+    ls = NULL;
+    tor_free(b64);
+  }
+
+  /* Test IPv6. */
+  {
+    uint8_t ipv6[16];
+
+    spec.type = LS_IPV6;
+    ret = tor_addr_parse(&spec.u.ap.addr, "[1:2:3:4::]");
+    tt_int_op(ret, OP_EQ, AF_INET6);
+    b64 = encode_link_specifiers(link_specifiers);
+    tt_assert(b64);
+
+    /* Decode it and validate the format. */
+    ret = base64_decode(buf, sizeof(buf), b64, strlen(b64));
+    tt_int_op(ret, OP_GT, 0);
+    /* First byte is the number of link specifier. */
+    tt_int_op(get_uint8(buf), OP_EQ, 1);
+    ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1);
+    tt_int_op(ret, OP_EQ, 20);
+    /* Should be 2 bytes for port and 16 bytes for IPv6. */
+    tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, 18);
+    for (unsigned int i = 0; i < sizeof(ipv6); i++) {
+      ipv6[i] = link_specifier_get_un_ipv6_addr(ls, i);
+    }
+    tt_mem_op(tor_addr_to_in6_addr8(&spec.u.ap.addr), OP_EQ, ipv6,
+              sizeof(ipv6));
+    tt_int_op(link_specifier_get_un_ipv6_port(ls), OP_EQ, spec.u.ap.port);
+
+    link_specifier_free(ls);
+    ls = NULL;
+    tor_free(b64);
+  }
+
+  /* Test legacy. */
+  {
+    uint8_t *id;
+
+    spec.type = LS_LEGACY_ID;
+    memset(spec.u.legacy_id, 'Y', sizeof(spec.u.legacy_id));
+    b64 = encode_link_specifiers(link_specifiers);
+    tt_assert(b64);
+
+    /* Decode it and validate the format. */
+    ret = base64_decode(buf, sizeof(buf), b64, strlen(b64));
+    tt_int_op(ret, OP_GT, 0);
+    /* First byte is the number of link specifier. */
+    tt_int_op(get_uint8(buf), OP_EQ, 1);
+    ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1);
+    /* 20 bytes digest + 1 byte type + 1 byte len. */
+    tt_int_op(ret, OP_EQ, 22);
+    tt_int_op(link_specifier_getlen_un_legacy_id(ls), OP_EQ, DIGEST_LEN);
+    /* Digest length is 20 bytes. */
+    tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, DIGEST_LEN);
+    id = link_specifier_getarray_un_legacy_id(ls);
+    tt_mem_op(spec.u.legacy_id, OP_EQ, id, DIGEST_LEN);
+
+    link_specifier_free(ls);
+    ls = NULL;
+    tor_free(b64);
+  }
+
+ done:
+  link_specifier_free(ls);
+  tor_free(b64);
+  smartlist_free(link_specifiers);
+}
+
+static void
+test_encode_descriptor(void *arg)
+{
+  int ret;
+  ed25519_keypair_t signing_kp;
+  hs_descriptor_t *desc = NULL;
+
+  (void) arg;
+
+  ret = ed25519_keypair_generate(&signing_kp, 0);
+  tt_int_op(ret, OP_EQ, 0);
+  desc = hs_helper_build_hs_desc_with_ip(&signing_kp);
+
+  {
+    char *encoded = NULL;
+    ret = hs_desc_encode_descriptor(desc, &signing_kp, NULL, &encoded);
+    tt_int_op(ret, OP_EQ, 0);
+    tt_assert(encoded);
+
+    tor_free(encoded);
+  }
+
+  {
+    char *encoded = NULL;
+    uint8_t descriptor_cookie[HS_DESC_DESCRIPTOR_COOKIE_LEN];
+
+    crypto_strongest_rand(descriptor_cookie, sizeof(descriptor_cookie));
+
+    ret = hs_desc_encode_descriptor(desc, &signing_kp,
+                                   descriptor_cookie, &encoded);
+    tt_int_op(ret, OP_EQ, 0);
+    tt_assert(encoded);
+
+    tor_free(encoded);
+  }
+ done:
+  hs_descriptor_free(desc);
+}
+
+static void
+test_decode_descriptor(void *arg)
+{
+  int ret;
+  int i;
+  char *encoded = NULL;
+  ed25519_keypair_t signing_kp;
+  hs_descriptor_t *desc = NULL;
+  hs_descriptor_t *decoded = NULL;
+  hs_descriptor_t *desc_no_ip = NULL;
+  uint8_t subcredential[DIGEST256_LEN];
+
+  (void) arg;
+
+  ret = ed25519_keypair_generate(&signing_kp, 0);
+  tt_int_op(ret, OP_EQ, 0);
+  desc = hs_helper_build_hs_desc_with_ip(&signing_kp);
+
+  hs_helper_get_subcred_from_identity_keypair(&signing_kp,
+                                              subcredential);
+
+  /* Give some bad stuff to the decoding function. */
+  ret = hs_desc_decode_descriptor("hladfjlkjadf", subcredential,
+                                  NULL, &decoded);
+  tt_int_op(ret, OP_EQ, -1);
+
+  ret = hs_desc_encode_descriptor(desc, &signing_kp, NULL, &encoded);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_assert(encoded);
+
+  ret = hs_desc_decode_descriptor(encoded, subcredential, NULL, &decoded);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_assert(decoded);
+
+  hs_helper_desc_equal(desc, decoded);
+
+  /* Decode a descriptor with _no_ introduction points. */
+  {
+    ed25519_keypair_t signing_kp_no_ip;
+    ret = ed25519_keypair_generate(&signing_kp_no_ip, 0);
+    tt_int_op(ret, OP_EQ, 0);
+    hs_helper_get_subcred_from_identity_keypair(&signing_kp_no_ip,
+                                                subcredential);
+    desc_no_ip = hs_helper_build_hs_desc_no_ip(&signing_kp_no_ip);
+    tt_assert(desc_no_ip);
+    tor_free(encoded);
+    ret = hs_desc_encode_descriptor(desc_no_ip, &signing_kp_no_ip,
+                                    NULL, &encoded);
+    tt_int_op(ret, OP_EQ, 0);
+    tt_assert(encoded);
+    hs_descriptor_free(decoded);
+    ret = hs_desc_decode_descriptor(encoded, subcredential, NULL, &decoded);
+    tt_int_op(ret, OP_EQ, 0);
+    tt_assert(decoded);
+  }
+
+  /* Decode a descriptor with auth clients. */
+  {
+    uint8_t descriptor_cookie[HS_DESC_DESCRIPTOR_COOKIE_LEN];
+    curve25519_keypair_t auth_ephemeral_kp;
+    curve25519_keypair_t client_kp, invalid_client_kp;
+    smartlist_t *clients;
+    hs_desc_authorized_client_t *client, *fake_client;
+    client = tor_malloc_zero(sizeof(hs_desc_authorized_client_t));
+
+    /* Prepare all the keys needed to build the auth client. */
+    curve25519_keypair_generate(&auth_ephemeral_kp, 0);
+    curve25519_keypair_generate(&client_kp, 0);
+    curve25519_keypair_generate(&invalid_client_kp, 0);
+    crypto_strongest_rand(descriptor_cookie, HS_DESC_DESCRIPTOR_COOKIE_LEN);
+
+    memcpy(&desc->superencrypted_data.auth_ephemeral_pubkey,
+           &auth_ephemeral_kp.pubkey, CURVE25519_PUBKEY_LEN);
+
+    hs_helper_get_subcred_from_identity_keypair(&signing_kp,
+                                                subcredential);
+
+    /* Build and add the auth client to the descriptor. */
+    clients = desc->superencrypted_data.clients;
+    if (!clients) {
+      clients = smartlist_new();
+    }
+    hs_desc_build_authorized_client(subcredential,
+                                    &client_kp.pubkey,
+                                    &auth_ephemeral_kp.seckey,
+                                    descriptor_cookie, client);
+    smartlist_add(clients, client);
+
+    /* We need to add fake auth clients here. */
+    for (i=0; i < 15; ++i) {
+      fake_client = hs_desc_build_fake_authorized_client();
+      smartlist_add(clients, fake_client);
+    }
+    desc->superencrypted_data.clients = clients;
+
+    /* Test the encoding/decoding in the following lines. */
+    tor_free(encoded);
+    ret = hs_desc_encode_descriptor(desc, &signing_kp,
+                                    descriptor_cookie, &encoded);
+    tt_int_op(ret, OP_EQ, 0);
+    tt_assert(encoded);
+
+    /* If we do not have the client secret key, the decoding must fail. */
+    hs_descriptor_free(decoded);
+    ret = hs_desc_decode_descriptor(encoded, subcredential,
+                                    NULL, &decoded);
+    tt_int_op(ret, OP_LT, 0);
+    tt_assert(!decoded);
+
+    /* If we have an invalid client secret key, the decoding must fail. */
+    hs_descriptor_free(decoded);
+    ret = hs_desc_decode_descriptor(encoded, subcredential,
+                                    &invalid_client_kp.seckey, &decoded);
+    tt_int_op(ret, OP_LT, 0);
+    tt_assert(!decoded);
+
+    /* If we have the client secret key, the decoding must succeed and the
+     * decoded descriptor must be correct. */
+    ret = hs_desc_decode_descriptor(encoded, subcredential,
+                                    &client_kp.seckey, &decoded);
+    tt_int_op(ret, OP_EQ, 0);
+    tt_assert(decoded);
+
+    hs_helper_desc_equal(desc, decoded);
+  }
+
+ done:
+  hs_descriptor_free(desc);
+  hs_descriptor_free(desc_no_ip);
+  hs_descriptor_free(decoded);
+  tor_free(encoded);
+}
+
+static void
+test_supported_version(void *arg)
+{
+  int ret;
+
+  (void) arg;
+
+  /* Unsupported. */
+  ret = hs_desc_is_supported_version(42);
+  tt_int_op(ret, OP_EQ, 0);
+  /* To early. */
+  ret = hs_desc_is_supported_version(HS_DESC_SUPPORTED_FORMAT_VERSION_MIN - 1);
+  tt_int_op(ret, OP_EQ, 0);
+  /* One too new. */
+  ret = hs_desc_is_supported_version(HS_DESC_SUPPORTED_FORMAT_VERSION_MAX + 1);
+  tt_int_op(ret, OP_EQ, 0);
+  /* Valid version. */
+  ret = hs_desc_is_supported_version(3);
+  tt_int_op(ret, OP_EQ, 1);
+
+ done:
+  ;
+}
+
+static void
+test_encrypted_data_len(void *arg)
+{
+  int ret;
+  size_t value;
+
+  (void) arg;
+
+  /* No length, error. */
+  ret = encrypted_data_length_is_valid(0);
+  tt_int_op(ret, OP_EQ, 0);
+  /* Valid value. */
+  value = HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN + 1;
+  ret = encrypted_data_length_is_valid(value);
+  tt_int_op(ret, OP_EQ, 1);
+
+ done:
+  ;
+}
+
+static void
+test_decode_invalid_intro_point(void *arg)
+{
+  int ret;
+  char *encoded_ip = NULL;
+  size_t len_out;
+  hs_desc_intro_point_t *ip = NULL;
+  ed25519_keypair_t signing_kp;
+  hs_descriptor_t *desc = NULL;
+
+  (void) arg;
+
+  /* Separate pieces of a valid encoded introduction point. */
+  const char *intro_point =
+    "introduction-point AQIUMDI5OUYyNjhGQ0E5RDU1Q0QxNTc=";
+  const char *auth_key =
+    "auth-key\n"
+    "-----BEGIN ED25519 CERT-----\n"
+    "AQkACOhAAQW8ltYZMIWpyrfyE/b4Iyi8CNybCwYs6ADk7XfBaxsFAQAgBAD3/BE4\n"
+    "XojGE/N2bW/wgnS9r2qlrkydGyuCKIGayYx3haZ39LD4ZTmSMRxwmplMAqzG/XNP\n"
+    "0Kkpg4p2/VnLFJRdU1SMFo1lgQ4P0bqw7Tgx200fulZ4KUM5z5V7m+a/mgY=\n"
+    "-----END ED25519 CERT-----";
+  const char *enc_key =
+    "enc-key ntor bpZKLsuhxP6woDQ3yVyjm5gUKSk7RjfAijT2qrzbQk0=";
+  const char *enc_key_cert =
+    "enc-key-cert\n"
+    "-----BEGIN ED25519 CERT-----\n"
+    "AQsACOhZAUpNvCZ1aJaaR49lS6MCdsVkhVGVrRqoj0Y2T4SzroAtAQAgBABFOcGg\n"
+    "lbTt1DF5nKTE/gU3Fr8ZtlCIOhu1A+F5LM7fqCUupfesg0KTHwyIZOYQbJuM5/he\n"
+    "/jDNyLy9woPJdjkxywaY2RPUxGjLYtMQV0E8PUxWyICV+7y52fTCYaKpYQw=\n"
+    "-----END ED25519 CERT-----";
+
+  /* Try to decode a junk string. */
+  {
+    hs_descriptor_free(desc);
+    desc = NULL;
+    ret = ed25519_keypair_generate(&signing_kp, 0);
+    tt_int_op(ret, OP_EQ, 0);
+    desc = hs_helper_build_hs_desc_with_ip(&signing_kp);
+    const char *junk = "this is not a descriptor";
+    ip = decode_introduction_point(desc, junk);
+    tt_ptr_op(ip, OP_EQ, NULL);
+    hs_desc_intro_point_free(ip);
+    ip = NULL;
+  }
+
+  /* Invalid link specifiers. */
+  {
+    smartlist_t *lines = smartlist_new();
+    const char *bad_line = "introduction-point blah";
+    smartlist_add(lines, (char *) bad_line);
+    smartlist_add(lines, (char *) auth_key);
+    smartlist_add(lines, (char *) enc_key);
+    smartlist_add(lines, (char *) enc_key_cert);
+    encoded_ip = smartlist_join_strings(lines, "\n", 0, &len_out);
+    tt_assert(encoded_ip);
+    ip = decode_introduction_point(desc, encoded_ip);
+    tt_ptr_op(ip, OP_EQ, NULL);
+    tor_free(encoded_ip);
+    smartlist_free(lines);
+    hs_desc_intro_point_free(ip);
+    ip = NULL;
+  }
+
+  /* Invalid auth key type. */
+  {
+    smartlist_t *lines = smartlist_new();
+    /* Try to put a valid object that our tokenize function will be able to
+     * parse but that has nothing to do with the auth_key. */
+    const char *bad_line =
+      "auth-key\n"
+      "-----BEGIN UNICORN CERT-----\n"
+      "MIGJAoGBAO4bATcW8kW4h6RQQAKEgg+aXCpF4JwbcO6vGZtzXTDB+HdPVQzwqkbh\n"
+      "XzFM6VGArhYw4m31wcP1Z7IwULir7UMnAFd7Zi62aYfU6l+Y1yAoZ1wzu1XBaAMK\n"
+      "ejpwQinW9nzJn7c2f69fVke3pkhxpNdUZ+vplSA/l9iY+y+v+415AgMBAAE=\n"
+      "-----END UNICORN CERT-----";
+    /* Build intro point text. */
+    smartlist_add(lines, (char *) intro_point);
+    smartlist_add(lines, (char *) bad_line);
+    smartlist_add(lines, (char *) enc_key);
+    smartlist_add(lines, (char *) enc_key_cert);
+    encoded_ip = smartlist_join_strings(lines, "\n", 0, &len_out);
+    tt_assert(encoded_ip);
+    ip = decode_introduction_point(desc, encoded_ip);
+    tt_ptr_op(ip, OP_EQ, NULL);
+    tor_free(encoded_ip);
+    smartlist_free(lines);
+  }
+
+  /* Invalid enc-key. */
+  {
+    smartlist_t *lines = smartlist_new();
+    const char *bad_line =
+      "enc-key unicorn bpZKLsuhxP6woDQ3yVyjm5gUKSk7RjfAijT2qrzbQk0=";
+    /* Build intro point text. */
+    smartlist_add(lines, (char *) intro_point);
+    smartlist_add(lines, (char *) auth_key);
+    smartlist_add(lines, (char *) bad_line);
+    smartlist_add(lines, (char *) enc_key_cert);
+    encoded_ip = smartlist_join_strings(lines, "\n", 0, &len_out);
+    tt_assert(encoded_ip);
+    ip = decode_introduction_point(desc, encoded_ip);
+    tt_ptr_op(ip, OP_EQ, NULL);
+    tor_free(encoded_ip);
+    smartlist_free(lines);
+  }
+
+  /* Invalid enc-key object. */
+  {
+    smartlist_t *lines = smartlist_new();
+    const char *bad_line = "enc-key ntor";
+    /* Build intro point text. */
+    smartlist_add(lines, (char *) intro_point);
+    smartlist_add(lines, (char *) auth_key);
+    smartlist_add(lines, (char *) bad_line);
+    smartlist_add(lines, (char *) enc_key_cert);
+    encoded_ip = smartlist_join_strings(lines, "\n", 0, &len_out);
+    tt_assert(encoded_ip);
+    ip = decode_introduction_point(desc, encoded_ip);
+    tt_ptr_op(ip, OP_EQ, NULL);
+    tor_free(encoded_ip);
+    smartlist_free(lines);
+  }
+
+  /* Invalid enc-key base64 curv25519 key. */
+  {
+    smartlist_t *lines = smartlist_new();
+    const char *bad_line = "enc-key ntor blah===";
+    /* Build intro point text. */
+    smartlist_add(lines, (char *) intro_point);
+    smartlist_add(lines, (char *) auth_key);
+    smartlist_add(lines, (char *) bad_line);
+    smartlist_add(lines, (char *) enc_key_cert);
+    encoded_ip = smartlist_join_strings(lines, "\n", 0, &len_out);
+    tt_assert(encoded_ip);
+    ip = decode_introduction_point(desc, encoded_ip);
+    tt_ptr_op(ip, OP_EQ, NULL);
+    tor_free(encoded_ip);
+    smartlist_free(lines);
+  }
+
+  /* Invalid enc-key invalid legacy. */
+  {
+    smartlist_t *lines = smartlist_new();
+    const char *bad_line = "legacy-key blah===";
+    /* Build intro point text. */
+    smartlist_add(lines, (char *) intro_point);
+    smartlist_add(lines, (char *) auth_key);
+    smartlist_add(lines, (char *) bad_line);
+    smartlist_add(lines, (char *) enc_key_cert);
+    encoded_ip = smartlist_join_strings(lines, "\n", 0, &len_out);
+    tt_assert(encoded_ip);
+    ip = decode_introduction_point(desc, encoded_ip);
+    tt_ptr_op(ip, OP_EQ, NULL);
+    tor_free(encoded_ip);
+    smartlist_free(lines);
+  }
+
+ done:
+  hs_descriptor_free(desc);
+  hs_desc_intro_point_free(ip);
+}
+
+/** Make sure we fail gracefully when decoding the bad desc from #23233. */
+static void
+test_decode_bad_signature(void *arg)
+{
+  hs_desc_plaintext_data_t desc_plaintext;
+  int ret;
+
+  (void) arg;
+
+  memset(&desc_plaintext, 0, sizeof(desc_plaintext));
+
+  /* Update approx time to dodge cert expiration */
+  update_approx_time(1502661599);
+
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_desc_decode_plaintext(HS_DESC_BAD_SIG, &desc_plaintext);
+  tt_int_op(ret, OP_EQ, -1);
+  expect_log_msg_containing("Malformed signature line. Rejecting.");
+  teardown_capture_of_logs();
+
+ done:
+  hs_desc_plaintext_data_free_contents(&desc_plaintext);
+}
+
+static void
+test_decode_plaintext(void *arg)
+{
+  int ret;
+  hs_desc_plaintext_data_t desc_plaintext;
+  const char *bad_value = "unicorn";
+
+  (void) arg;
+
+#define template \
+    "hs-descriptor %s\n" \
+    "descriptor-lifetime %s\n" \
+    "descriptor-signing-key-cert\n" \
+    "-----BEGIN ED25519 CERT-----\n" \
+    "AQgABjvPAQaG3g+dc6oV/oJV4ODAtkvx56uBnPtBT9mYVuHVOhn7AQAgBABUg3mQ\n" \
+    "myBr4bu5LCr53wUEbW2EXui01CbUgU7pfo9LvJG3AcXRojj6HlfsUs9BkzYzYdjF\n" \
+    "A69Apikgu0ewHYkFFASt7Il+gB3w6J8YstQJZT7dtbtl+doM7ug8B68Qdg8=\n" \
+    "-----END ED25519 CERT-----\n" \
+    "revision-counter %s\n" \
+    "encrypted\n" \
+    "-----BEGIN %s-----\n" \
+    "UNICORN\n" \
+    "-----END MESSAGE-----\n" \
+    "signature m20WJH5agqvwhq7QeuEZ1mYyPWQDO+eJOZUjLhAiKu8DbL17DsDfJE6kXbWy" \
+    "HimbNj2we0enV3cCOOAsmPOaAw\n"
+
+  /* Invalid version. */
+  {
+    char *plaintext;
+    tor_asprintf(&plaintext, template, bad_value, "180", "42", "MESSAGE");
+    ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
+    tor_free(plaintext);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Missing fields. */
+  {
+    const char *plaintext = "hs-descriptor 3\n";
+    ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Max length. */
+  {
+    size_t big = 64000;
+    /* Must always be bigger than HS_DESC_MAX_LEN. */
+    tt_int_op(HS_DESC_MAX_LEN, OP_LT, big);
+    char *plaintext = tor_malloc_zero(big);
+    memset(plaintext, 'a', big);
+    plaintext[big - 1] = '\0';
+    ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
+    tor_free(plaintext);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Bad lifetime value. */
+  {
+    char *plaintext;
+    tor_asprintf(&plaintext, template, "3", bad_value, "42", "MESSAGE");
+    ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
+    tor_free(plaintext);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Huge lifetime value. */
+  {
+    char *plaintext;
+    tor_asprintf(&plaintext, template, "3", "7181615", "42", "MESSAGE");
+    ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
+    tor_free(plaintext);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Invalid encrypted section. */
+  {
+    char *plaintext;
+    tor_asprintf(&plaintext, template, "3", "180", "42", bad_value);
+    ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
+    tor_free(plaintext);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Invalid revision counter. */
+  {
+    char *plaintext;
+    tor_asprintf(&plaintext, template, "3", "180", bad_value, "MESSAGE");
+    ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
+    tor_free(plaintext);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+ done:
+  ;
+}
+
+static void
+test_validate_cert(void *arg)
+{
+  int ret;
+  time_t now = time(NULL);
+  ed25519_keypair_t kp;
+  tor_cert_t *cert = NULL;
+
+  (void) arg;
+
+  ret = ed25519_keypair_generate(&kp, 0);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Cert of type CERT_TYPE_AUTH_HS_IP_KEY. */
+  cert = tor_cert_create(&kp, CERT_TYPE_AUTH_HS_IP_KEY,
+                                     &kp.pubkey, now, 3600,
+                                     CERT_FLAG_INCLUDE_SIGNING_KEY);
+  tt_assert(cert);
+  /* Test with empty certificate. */
+  ret = cert_is_valid(NULL, CERT_TYPE_AUTH_HS_IP_KEY, "unicorn");
+  tt_int_op(ret, OP_EQ, 0);
+  /* Test with a bad type. */
+  ret = cert_is_valid(cert, CERT_TYPE_SIGNING_HS_DESC, "unicorn");
+  tt_int_op(ret, OP_EQ, 0);
+  /* Normal validation. */
+  ret = cert_is_valid(cert, CERT_TYPE_AUTH_HS_IP_KEY, "unicorn");
+  tt_int_op(ret, OP_EQ, 1);
+  /* Break signing key so signature verification will fails. */
+  memset(&cert->signing_key, 0, sizeof(cert->signing_key));
+  ret = cert_is_valid(cert, CERT_TYPE_AUTH_HS_IP_KEY, "unicorn");
+  tt_int_op(ret, OP_EQ, 0);
+  tor_cert_free(cert);
+
+  /* Try a cert without including the signing key. */
+  cert = tor_cert_create(&kp, CERT_TYPE_AUTH_HS_IP_KEY, &kp.pubkey, now,
+                         3600, 0);
+  tt_assert(cert);
+  /* Test with a bad type. */
+  ret = cert_is_valid(cert, CERT_TYPE_AUTH_HS_IP_KEY, "unicorn");
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  tor_cert_free(cert);
+}
+
+static void
+test_desc_signature(void *arg)
+{
+  int ret;
+  char *data = NULL, *desc = NULL;
+  char sig_b64[ED25519_SIG_BASE64_LEN + 1];
+  ed25519_keypair_t kp;
+  ed25519_signature_t sig;
+
+  (void) arg;
+
+  ed25519_keypair_generate(&kp, 0);
+  /* Setup a phoony descriptor but with a valid signature token that is the
+   * signature is verifiable. */
+  tor_asprintf(&data, "This is a signed descriptor\n");
+  ret = ed25519_sign_prefixed(&sig, (const uint8_t *) data, strlen(data),
+                              "Tor onion service descriptor sig v3", &kp);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = ed25519_signature_to_base64(sig_b64, &sig);
+  tt_int_op(ret, OP_EQ, 0);
+  /* Build the descriptor that should be valid. */
+  tor_asprintf(&desc, "%ssignature %s\n", data, sig_b64);
+  ret = desc_sig_is_valid(sig_b64, &kp.pubkey, desc, strlen(desc));
+  tt_int_op(ret, OP_EQ, 1);
+  /* Junk signature. */
+  ret = desc_sig_is_valid("JUNK", &kp.pubkey, desc, strlen(desc));
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  tor_free(desc);
+  tor_free(data);
+}
+
+static void
+test_build_authorized_client(void *arg)
+{
+  int ret;
+  hs_desc_authorized_client_t *desc_client = NULL;
+  uint8_t descriptor_cookie[HS_DESC_DESCRIPTOR_COOKIE_LEN];
+  curve25519_secret_key_t auth_ephemeral_sk;
+  curve25519_secret_key_t client_auth_sk;
+  curve25519_public_key_t client_auth_pk;
+  const char ephemeral_sk_b16[] =
+    "d023b674d993a5c8446bd2ca97e9961149b3c0e88c7dc14e8777744dd3468d6a";
+  const char descriptor_cookie_b16[] =
+    "07d087f1d8c68393721f6e70316d3b29";
+  const char client_pubkey_b16[] =
+    "8c1298fa6050e372f8598f6deca32e27b0ad457741422c2629ebb132cf7fae37";
+  uint8_t subcredential[DIGEST256_LEN];
+  char *mem_op_hex_tmp=NULL;
+
+  (void) arg;
+
+  ret = curve25519_secret_key_generate(&auth_ephemeral_sk, 0);
+  tt_int_op(ret, OP_EQ, 0);
+
+  ret = curve25519_secret_key_generate(&client_auth_sk, 0);
+  tt_int_op(ret, OP_EQ, 0);
+  curve25519_public_key_generate(&client_auth_pk, &client_auth_sk);
+
+  memset(subcredential, 42, sizeof(subcredential));
+
+  desc_client = tor_malloc_zero(sizeof(hs_desc_authorized_client_t));
+
+  base16_decode((char *) &auth_ephemeral_sk,
+                sizeof(auth_ephemeral_sk),
+                ephemeral_sk_b16,
+                strlen(ephemeral_sk_b16));
+
+  base16_decode((char *) descriptor_cookie,
+                sizeof(descriptor_cookie),
+                descriptor_cookie_b16,
+                strlen(descriptor_cookie_b16));
+
+  base16_decode((char *) &client_auth_pk,
+                sizeof(client_auth_pk),
+                client_pubkey_b16,
+                strlen(client_pubkey_b16));
+
+  MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand);
+
+  hs_desc_build_authorized_client(subcredential,
+                                  &client_auth_pk, &auth_ephemeral_sk,
+                                  descriptor_cookie, desc_client);
+
+  test_memeq_hex((char *) desc_client->client_id,
+                 "EC19B7FF4D2DDA13");
+  test_memeq_hex((char *) desc_client->iv,
+                "01010101010101010101010101010101");
+  test_memeq_hex((char *) desc_client->encrypted_cookie,
+                "B21222BE13F385F355BD07B2381F9F29");
+
+ done:
+  tor_free(desc_client);
+  tor_free(mem_op_hex_tmp);
+  UNMOCK(crypto_strongest_rand_);
+}
+
+struct testcase_t hs_descriptor[] = {
+  /* Encoding tests. */
+  { "cert_encoding", test_cert_encoding, TT_FORK,
+    NULL, NULL },
+  { "link_specifier", test_link_specifier, TT_FORK,
+    NULL, NULL },
+  { "encode_descriptor", test_encode_descriptor, TT_FORK,
+    NULL, NULL },
+  { "descriptor_padding", test_descriptor_padding, TT_FORK,
+    NULL, NULL },
+
+  /* Decoding tests. */
+  { "decode_descriptor", test_decode_descriptor, TT_FORK,
+    NULL, NULL },
+  { "encrypted_data_len", test_encrypted_data_len, TT_FORK,
+    NULL, NULL },
+  { "decode_invalid_intro_point", test_decode_invalid_intro_point, TT_FORK,
+    NULL, NULL },
+  { "decode_plaintext", test_decode_plaintext, TT_FORK,
+    NULL, NULL },
+  { "decode_bad_signature", test_decode_bad_signature, TT_FORK,
+    NULL, NULL },
+
+  /* Misc. */
+  { "version", test_supported_version, TT_FORK,
+    NULL, NULL },
+  { "validate_cert", test_validate_cert, TT_FORK,
+    NULL, NULL },
+  { "desc_signature", test_desc_signature, TT_FORK,
+    NULL, NULL },
+  { "build_authorized_client", test_build_authorized_client, TT_FORK,
+    NULL, NULL },
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_hs_descriptor.inc b/src/test/test_hs_descriptor.inc
new file mode 100644
index 0000000000..70a2c7c2f7
--- /dev/null
+++ b/src/test/test_hs_descriptor.inc
@@ -0,0 +1,224 @@
+static const char* HS_DESC_BAD_SIG =
+"hs-descriptor 3\n"
+"descriptor-lifetime 180\n"
+"descriptor-signing-key-cert\n"
+"-----BEGIN ED25519 CERT-----\n"
+"AQgABl5+AQoPXRnCGEOxIup3AcjQXb8npNiUFm2Qv7A6JKk/K+EuAQAgBAD18iUD\n"
+"nbkUblnUvTHzipq4bcr6aPyFVB42Ptobg4xr8s3VjHiJtjs9MDEdr6nXS7UlyhEl\n"
+"78vsuFEvLp7cvAgGxYY1xGXdn5RdHMCdi8W9yZLKMQX9OuJckmp1C6q+cA4=\n"
+"-----END ED25519 CERT-----\n"
+"revision-counter 42\n"
+"superencrypted\n"
+"-----BEGIN MESSAGE-----\n"
+"BxzghAOjM4De6Z6eGTvBrTP2SJDdQOYV/u9qtvlFsa2FRQWk20Adv3zJ/AI10CQO\n"
+"mUP4DNXM8FWQYGTvmD7wGz2/cXGjKwBXg1qO7zF5eP/D/My1sXsIfCcb41mkheNt\n"
+"xn1I5eKXcnghtd4lw7OkPVjSb/Z+VARUMmf+0qSNgmHLgEVnAoGJsn8W8B4qtIay\n"
+"4h4PuV0jPPlqJx6jMFOOEW72uqnfmqeNvClENXXW60xhnaxsf0up62fuW8ktu6Wf\n"
+"lnX/lvTstBFZZQ8/XI1+G+BPf8TZf7mxu0WYVg1s/KWYasYMSw46as59nkqdq2Ii\n"
+"qJnqHX/R20mWBhgpLse6wO0aNpky/rozEnikaPqyO1DShf6a6jXY8ADBg7spnK2/\n"
+"h7sf1+F1xfi2dy2WGxc1EUMP1kTVUmbft7kOo2nA7+3YZwQuSJHaN/66HrzU2x5z\n"
+"ayRUJ8+qDtfpEf17xthc/Uh253blFK96IoJJiqBfI6xt3IqOdHJq0OOC9zBbF6Rj\n"
+"vKMsaxmc/nc6uOB2WePYSgkZ0qs/dRKBJs6+Ahn1KdGkadyd8mDKL86Oe8lncHdB\n"
+"m/6sQjhKqFgngkCDOIlEJyWizqfN84AGqD5Zyxq0rbsN+9KLsHFfEbCRjgqjO5nS\n"
+"FYSFtuKgCZl2gaYEslL1pIEYE6BD2Whjn/HWTRyWiULJr6SuavgcbxeNEQDuVCC+\n"
+"fm0X7Z+qERaMAMR0vTMJK/NzT4GifrDpgmgbxc+34CtNBF5TriM8aXTNZZlsW00k\n"
+"d0XRxFbbbtiT5VOaEHbny7R3MdTVutEc9E/BhLBvjSSrGX7vrryh6Oj++nthIIzm\n"
+"F4M7I11S0TyA+UE06qF1C8rKmhcqU9MWy1SiccJ9KOWhJ5xwlsXBIID7wVygUhVl\n"
+"ovzfKkDDPfRoBch6NdVkxNJx3gb63CUmC2TzfwOMh973nntMVzqqw9A7jYkro9ln\n"
+"217kHUwMk3e83UgFL4nn7NCf3Kj0zhJ4jSfAsuQpV6e7dhzrlNya0lqrUsY2zFXP\n"
+"xv8wUtg6Vo1KewgVQas4oElkgFjDN8RJ7uBAwfuE/b9NnYJoQd76G8DHei/1PHbu\n"
+"tbtwN9I5RHaTvEOfetsJFnIAkCG6O4CQpzwHu1DdvEP4s6/el10b/4awBJ4VwOVZ\n"
+"YHSe4X0DStTV4Cu6aLh5OvrOmGbieRj6HdGQ6syYCaEBTuxbBUUpjIAfVlReAIph\n"
+"6aOrY6HNcCmeVmL5qm4dKr2XXOREsnUFuMqmfQuQd9pN3zlmS+RqCgSJuFrguFpd\n"
+"mjo6UxZvbjE7yJjtCih38HRe5BaigP5RDRkXmiXjqJ4koLJpyjQh19k3BYGcdxUC\n"
+"RCcYXydbGF7qHlnoaX9HnX7y6ZRsyKQpt91PMTGOUsB4fS8NhsqPpl2gdp4poLNs\n"
+"+hqjWZJ3uuLotXBcgM39Dtq9tqqu9vM12T80UAfWnVEHrBphmukh49EhEr2sx/la\n"
+"kAzRoTbLyTdlGVei8hI7/RtZIaIcOkzlhcFI5zmBlydyrv6/79vzt6WI/w9GVGpM\n"
+"OuSM0NS2CDJ7Iw412nz3CV1pEXB551ZBmbme6NHUe4EtEsDbgkP1Z201H4j51yVz\n"
+"wNoIksE5Bh5XRKuu4We5f9KZb+AEG9kxKJ5DbJk2YGJEQFTyfv0H68pl9urstPXD\n"
+"aMQF806COe2uhGm5gV/skvPVTeEvStE3K8DxZgcWNcTMVk8ZjrUHNfguVVToP8hT\n"
+"Fl4Iqo3r+JZEAGXnAbTpxUVC2Xxspf3jsT5xhUfB/NOexZxrXWnQZ+pscsbow0ba\n"
+"GATtakD3TF2WBqq5WscmOex+lrJcBCWVIzVWdwi5ngAtm1S7efkJlFUvmi4OuYnN\n"
+"RyZfxVIpoer8f2/xPXvxkOWFminDy5sFEvlh2/pnymfKOUV+CKih9ZApt+izlRJn\n"
+"+sMIOW6Jhf/WYyjeN6KQpwi6CDpclQJXA1SVoOVVL5A3lotLjs0x7ThIcBoxCZBq\n"
+"rFBhBu1gJgJ8guMySAHssIvhHHwXJsYEwzWCVAg/zIUXy4PLwIkgHApl+vGcldGv\n"
+"Br5HNCuqQ2pD9z2RvzNneB/LrYB214i+BP2piO5HbmeJBhby93blGXVfQewQT6aF\n"
+"dBlK8/jQM0rvb+LkmvQm2ypOttRpX2kyQXooJHYTTusaUr4jVmgngCvGtgqAQVqD\n"
+"HULXfHWvugZbAh6dXF7gKnnsyDOWwAgy4OJRi8i0jCaZ8aWSFRUjeGKT26dg/ayB\n"
+"U4QfMb8vL8tMdXVBfQLGcBgvrzQYrY69//pV6bX3SbLfUfWXV9eqUVWVPqVyPEwa\n"
+"Tz/aGVnGv/dY8h2cVnrgSXJGlOO+mCwSl+k9nk7VcEaKYuNlaOP3ZlKJvVj1LefM\n"
+"FODh4qTDBo5NkyfKu5fcZcOqDMBeGWXZzltE7CmvY7fOpDNMsuAoXYWI7q9gK82F\n"
+"w+nS0tVFCIWYa9DgGMv9GKTOk4Ia9elkbWypdRE/4oz4QxmHsArEsK4gDI+wmcp7\n"
+"/NsAZeuy96r2YDIUam4uASKOiAqrEfCv6B6cYctdYwZbAEXdo4fkGrCIjNRZmZGv\n"
+"kcZzHzIymnAmKRTkPt/LQ7Rx27Qd/Vt++B3zt2ORFuopqowOP0ocGZtkm0daK3Fc\n"
+"YDXMwIpf6Z8PwvvsG1bQHcSR+cUZi7vK7+hj/LGhMPafHM7HmFUbAxpJYr5CvR6y\n"
+"V1pZQYltT8xWayCeMHlLAAg10RgDkqCnY4dHnrY4GdwI2O7Wpxomni7qVHMjn+cN\n"
+"UTrd7EeVw+dxAIYosuqG7ua7ee3VGoOs+XMLrscAqHahfGbyYC+j+6Tow4qwWBdU\n"
+"/W3NJXnRWaHTXFHllpClnxggPRQx4yPtgTOmBBVl/O0T6i4Bv0ygsJeZAqC3VmAJ\n"
+"QodQTzGf2jwqsZf4uHKQa0EKGQvTGjFVgAFNpHmAuzyqh0b1pq5JeXiFERGsKC3j\n"
+"xcJilq1XeIx4SL38YNuCxi4pnyJyLnGGHpNjdjeFO5lvgCaKPegsPo4hpNpTvBJ1\n"
+"D7+o3E5CqxzjRt9kQmtwBbuH/SQX2T0x8aQ6vhwjj8ftDfw+FbjpMR9zfU0Lf8V7\n"
+"UjVGIl2yiVBGScBZu1nSD83PxjFy3XdFtBYoU5OrlXwBEYQs91jwK7UCiGtjI2Ao\n"
+"ZGkJaBd4AqP6voyJiGnC3LWFcmeMyzfExgiclQwfhFqqf762TX5JwG6xGqtdcNKS\n"
+"k54LlcI/RfvJw3ncSs9YsodZr6Jz5irpRTHX5WwCrX9mLukP96SXo29bIXEZAqEr\n"
+"ZxEcF0zlYE+km5bRfRCRcVVrScugCshSNLOdQp6fOAtHCl7rdQ/8Rz7oHuqieLVi\n"
+"UldRsAmpk9fIfRLphXj4j24jRP0VtL/LoJwakWTa0xO8K7eBAMVITI+HgFfN4wSO\n"
+"Yh1B+bGD5WKxFsWSgBMmW+YLF5ZtxVmmbg7wK2dIpJs4pjg2YO/MTO7SifJ9kjcb\n"
+"bCc74Tjs5mLLGGjGCIoXfda6WXbt2it40XhFk2zUAcPPsgjbctftkaWph7JSZpmZ\n"
+"fVcPqKdhmA1U0LA2XEOMTxGyCAeseH6pJXZm9LdBozc1CwyWP8XEDHHJf35vfPKY\n"
+"JDe2PanFepIOHaoRTgE7ZkGWKzOIKlS0Ucr1ezVfcxiFgQUNM+MYXXbUz51BVVq1\n"
+"Dulg4VvX104nt/ULijcfa/TsE+uklEnkyk1mhavH337NQg38XF4cAngNlUF4nSW/\n"
+"j0jizbAtaSx1f7q6xqPm3zPRlHrGQizHXLyl+SLzDUVPOXbPwcoeev97YeeyB6h5\n"
+"NBbIK9hmekNDmYIwI0bmlrg6IXhC5pyvRe8sQlV+9wBY2liF0M1mq5onW3a55afp\n"
+"+ynxXfQucb1HxZLXvRIGMBgWSQ7HfIPASqSE90Vu6qQCfkOW5PDqONr4BM65V4+g\n"
+"AYsVEgaosgHw9CF7yKgkvmZpToOtGpCHVcdUeeY2/rrQnAQeSy19gj/baJ+OKl6Q\n"
+"i1EGU8Yqo2r0d4XDFp/eKgC4sv57qp1PwkYQ/HKqoelJ09IAZL2sQWc05BGwt1A0\n"
+"11qDIEdkZBjzK3qUnY3QlOuoZtALZrnPg56SlF1RGDOPqbcF+3opqsvzBoiikh4V\n"
+"WV5OUYjRDMUDLQqf/OkuktdYf5N3RcbYP0XsAvY0ZWG3Gp068b3p8peCpkDzrF9p\n"
+"bQ2ZvS304tN7+p0hif3+JyZy5/sxl17RxTeg5I3mo2+J0ptQDYwF/WadONO8r7uU\n"
+"YlRltFtQfyMzyVzHON4NHGjZh7dDGtWp0MGeHRBHQsC8bEChhvWme19VXhgZoWpl\n"
+"dUIZkSuvRwiURXjhKbZrEdJbVmr9FX6zoyOahv3VnmcEARoR+umxzvo3hGQPbHyH\n"
+"jTsQtSBjs75/9fCxcYmBWkh3JHVDVsCbV+z+5KZpk3m50J4Y1hC8hvepC9CaBqOM\n"
+"DjfyXh58x1yKiueEbcjSWsRuF7CjcrYnFUBHOs9U1j9WytCI3fhOWPMgR4UZpGuU\n"
+"WlcR1BXg1wYxX273xOS/jYn9MLAVlbRpPTUMIH9VRP+sc8+XaxKpJSCl4C+vcwNY\n"
+"1YdKD2QiuoBJ3fXGtqMVRtn9eZvatSJuY9CnRKRbf0hWmFD4D5RkiwE0WkdtwoHR\n"
+"uEXJ47RlF0/JDU1fY1mXBkq3usvB4Absy78qL06vh45xkk9bHbdf+7Ao1RQKmqiB\n"
+"NL5XnjBu+YX535WG7t7Su3mTCJXYHvn72ATxry8yhSLgWqt81STkRwc14HmrOGG8\n"
+"Gw7bz7y5vikj/rnPyr7ry+QRgNNDDayAqenAu2vEAzWir0RQC/iZ9rc/r7YQWGgL\n"
+"Xrd4TQ6rTZePARhwB3VomnLDDvLvi2oq/jPzLKSYM2a7qj/vBSbJ/NnNaDW5Ccew\n"
+"RjMI1lIHeedqYTVAW/CKoSEPcFSAzi/Ija0gcWLgX5xsFDGIYBepAX0KS9426kMu\n"
+"0r/V66zmPMusMilqRTx7KW+jZMVxXVc2zClcdmohMmtjsbqLkczprfSbdGswMv9Q\n"
+"I1ktHJHIRD0vPeZXnvKZsRKZw3sKb0ltZi33ZxCJFQPeGGtM5aAFthj6awcXy6Tt\n"
+"DPUQdCU/vh1zmGRAX17/Xb0irfvN+GhQLEl42pzhigJXc/rCG3a4Na8wT+xAIZVf\n"
+"WUI7hMslx5wA+iB4lrAjCq0YIrjINI/lHYpotXUZGmz5wz0jOciTmXMSx9du4cpk\n"
+"fIQJfR+fr5tG3fjHMgSP+p+RewHkd/7RUAmHC2k3cuk5pCJvUVJrhUIqsi1fa0LG\n"
+"GA0UU6Nr9tpYdNr1WkbKQjxTg0D//AXe61jmUS5XUU4AQf6zQVfN0TMtmuYeacbK\n"
+"4r6Z1CSIRbsgcnL1BN8GSd4KddkCqSk941aJUCoX+77ou4t0btVSB9FnLKipigtE\n"
+"E/Rpmv+81lA4fLiIag62/pcJ3uppsZ9aaHdR10SMmuCjAVLYHqhJfrHHn32dyqLK\n"
+"UI8kEZJ6GQzHLUXcGbbdnk1Qm6JwO8TeF/oQvh9y9py+oAyFy0qzP2UeUMUI2yRQ\n"
+"mlWSy+wX1DbVDQ3UHwJjWp65CgyYXuW8eCB0AbyF0kF4KGf7/7Ae7tEGbmYSm5MA\n"
+"71z+Azxtv5gRyRb787V2dyo0wcmbRlL7iUBVXNM/czQo31tAZIwLc+lKNp0SPH6g\n"
+"gJ2yX/GeDSFNAeEVUZ/f4KZIa7QQsnGWrUr+agSnQFkySmIjWYjwC/abJwah0v0d\n"
+"ulwr3tECaaXtoWVdYXa3utEclBz9umBwMJ9MQCm4Kx7dTYUWFT3bMM/ESTkGPcfm\n"
+"m+C4FsqFBs80WY0ududu50vTDSdJt1RqZ7Sg6DNH6acBvWyXOpT5mPJKUjnSFwyG\n"
+"oVLgv0aDDx7lLZdCkhyz/Ff5LNmBgQsjGllPszJ2gTZxZ5LD68S4kUirQG/qtzlS\n"
+"PGfDOC79SMZGgsoAnr4wV3RUTxsTVFlxVHsBMB+EXOFHAr3wHTVxUGBbGzxBlQ9w\n"
+"I/jlu8LIIexXAU75HS5KCGGfg0Z7BLqEzqpMKqcBQC7BD7GnCXrDSQ2DCXnl7bLN\n"
+"lIrQ/z2Y8AgSdED46R40MqyyN6CPPNiOCjONHZ30fLEXuEgCp4R/+x0WWsWpjGk2\n"
+"Ydkc03cx/X6moUYxB5HTqTodBmAQuWMX0rxFDrnR0SWghWjdWth9gjd+dvZ82tt1\n"
+"UMUywDPhcYchtUi2lnqnYJm5p00GN9Mk14MC5ZC5qP57IJVqxu0ktOMpks+CLPnz\n"
+"qp9OBpI4sIzd0y0aUJC2Gd+E9aAhlREIiicyBDmxLdk1i37QeeCralI3eubLNmE+\n"
+"CjDjD8t8FUGPpKglSD3lfLTqbp2TUvyWfvJC6ulFPNsAbeLHTnPnpyPQmWxhMNGt\n"
+"h67B9tbYww2TvNwqIgmB4+YIR4/pSs15TpAqvuUvjpmRwGklqgiSmrQrlIxCxux/\n"
+"mfsaL3KE97wm8BsaMpMkjUL7ByTIFhFZ/gHPTxaFpbqTZ4G+lABLgp3bIsB9Dl/P\n"
+"ovoqX+qL2Mq9T0GrVJGfRBuA5hISw63hx5zdsj2Cj3A3khHPqR+GRN/rVYUuOpLm\n"
+"z3v5pU/74vZRmNMAIhyhmweSEPNtyVkgSdgbFErqvhxN0om2Cd/7cWh2g5BXHyUL\n"
+"PBr7ZkgfsE9TnuDH7Z0JoBqXJki+MO6nqz73oH2Mm86yxcXp6O/ieKTollrUJ3yQ\n"
+"P6hLcEbYPzUV99del7Va5Wi0nn0wbRXCGVQdwY+iWc7pT+VVlncyg0TvLXi0OtOt\n"
+"O8xbT2DAzVXxMwOsKV9ZgS/0dtwzwICpnTzBI/47V8GYhHbOUNTBPZ52GaXMeWlX\n"
+"cuRGb0+7OkKWuriyOQ5z5xaASCVfqgnOwSZYiAk0gcDoK+JHdr64/sMoJhH87R4i\n"
+"2TO90whkScgiGR7A06Ba42bT1nJtI6pxvzdB2b4BDAs2Lr2OdcB3BY1dtzKjFkw/\n"
+"qfIw3F55UQwcs84ZEFQDAB/tmfNHajblDFpXR4N5QvU/PdWVWJUub7oNyhIX6ruu\n"
+"ln4H7lpTUHJZ7jkr1qpnvkztZtHGlpJ0QdUHgyMYER1xU58Hg77yzIW3EdAa2PyK\n"
+"1t4udKbQKChShlShIMzwzj57ss/69QobrpYAHYi6IRMaMUGBfipGBACK3yeXsXz0\n"
+"c3Q2J5vI6QbxNsiJ5t7Ry1IqotbJcU7HND/yVUAUbEg5CpEDOSeSOW/ulyLuFxEV\n"
+"lRTwIO/68BoIoR7umlP23/1N5OYzaBHhH2nThILBovHeJRXnGXSgeFfwSj7LIYEV\n"
+"c1MdDSg/HzoADPXyEPLzqFzHRHeNiqEolmOPnFh0hRzbMZ0W5TQPDGWJdF21g816\n"
+"vA0WW4UQjLM+vnX9kKKLA1ut+9JWk1dGKsmWtdWUDfJjUP/L6dS4OYEl6O6+SjM9\n"
+"GcyGvHTiC5OpJllYpvELP/NjtTf9or8Bmruuga/axeOuS5ocYLK/sGRlmO6Z96da\n"
+"QSlyGWEQAnM2D1cDmdd4CetPslOVIcQ41+coWCi2xg3UjO/bFK1CA4R1rb4ekXfs\n"
+"s5U2XChyHhUPgl57y1r0ILXRXWJTJ0/F9hhu4aYQVFeIV/IuzJbmTKKkAcCOH6ys\n"
+"qnu2BXz8Pm2tU10JFfRcuZ8rHuUyUErA40ESsLijON98GMwL4Rat9ZSCNS5hlK7y\n"
+"yRJdr0ITp8oTbduAoulgWOvtcw1L87QBVojWz3cbhXra+WITirYuGNbzfmZn1WQM\n"
+"kukEZUEHSypGOrHr1XiuY4Rw/DBaJSLyZ+VybEOfXqXkDBh5s1ayypBvzrzFZCIn\n"
+"PJxIVsvrkhrpEbTJ9d7zLWjhOa9ZWw8lAubllbGm+7qCfdHmGsfBtvJdzx6zhB1Y\n"
+"otL/PCis2XVTBEDJeB8pGqKFOZjNz8PC5qP+ymtAfy2ktl/u4HsFlxV7CsEKGYPm\n"
+"p3LqnhPUy5M5gin4E4uPPyzzD2kcM3way49FKWUKlblQU0SyWtHRmMB3vcVmyT85\n"
+"BRULXF7jgog7XR/EMltwQyJI6GcUCrnWZu+G0BEwXG+CsgCzE7assDavc1NSGLZM\n"
+"rmzXiFFyfk7CE6lW2Lm+oWaFwKdvpmNZJFGGX8ZHRE9ZvkFMnfw9MYf2W7xa0jf7\n"
+"k3c6X5wMuk9mznVtq5itNFVXh1mT1ujeWOiiqyH5UhQQjj6O+ZXt4gqt/jT6dd1i\n"
+"jRuhhxaUGOlhpVBW/ySXhZ+HgOy9aCJ/bgjRGaqGixogk4f4rcgigHruwTpOQuDn\n"
+"xDZ3Xns70S40WtHSYN+Gbl9nIh4yl78aNnA4FVtTAuLlVKEKlMJi9OBFuP5TEczG\n"
+"+0HTwL/VPSCI+8FUZBhlz3YwecYq6dY5mS46+luPW+5Wl+5jtzb8V9oxVnRx2hQq\n"
+"B5HJsM5FOOhHDHMXoCsevj7N/ufK7cU7Wbr0DkgYRwvb0ZJB5WYgcaQ0W7aduhGb\n"
+"MQsandhP8Ajb2cmLobi3mHHPbcEkvjT8JP9Sim5xtfF+oCMMB5ByA5bI2aIFybZm\n"
+"jX9e/V8wNgtpDKDVKPjB3+9dj5gU1N5JsrjQwQDB0kVRMWdpJCtD4hZ2+T/QE3SI\n"
+"f8Rdk8pj8qBzRPbnhW6qsoWZdjMRC8qixZqHw4jol09UF7Ab9hjEF5ZDTfNGXwy8\n"
+"/hz8su+mr8hhrlCrOF2vBYUayAA96zhbDWfg3Pdxo9bTn3/DmyAngL4J5Gu679xK\n"
+"rWN4j7uQG4bzTa8WJb09/lW49UzWvmrz0c6/yexk3T//xDD067FafdnP5pYs4Cvp\n"
+"rCoHpXbKjxx99DJmb5iXW0JRLSpFSCbf1HPHbmzST3minSXap5FCWDJcSgExKIJp\n"
+"DXZ9rk0LMnQA74MWC5gjjM+5t0AHKuNRhJbQSwYWTKqeApXho53T/COlfDlSs2tb\n"
+"Vz1Ia5z7IOfu1QheE93huNAHT3Ob+mSmUq782SqFPr6uwud/l5uP3HpcuwugdlFm\n"
+"Jw8uBBOQ53W4lLbYfQYTVgieClVhmYMu7Ye0xYZ5B2jf714sjZRMa0LCbsyj58xH\n"
+"uzs8ddNN1fLMzb0JRBE8JWj5PbxhA/sTwMkD7SnEMBUTtP0obmuQ982aTfyvQCH/\n"
+"ve8OUPtYf5XWNv18mpR+h+riMt1Y8Eb6BJzTMFNWagMJAe3JV6A6upHroNFo2FxY\n"
+"1XPRM1Rt0zKo7GD+oXnixfpl1aG8yqZhYo1ZC9buaHwH6zvM+xoiGD0iujeDtpVy\n"
+"Vp6cAqqaGmrNwcPVBLc7hNKrJnbFKyhjL5/xp9j6jQov1aWQ8HsaNvh0p2ljmlwb\n"
+"daTYZcwLgSgPna7HhiqnOSAmXZ7St/qe/b9TqBtIVzwzmtevgMyG98QV0syFP5X6\n"
+"2Jc1g9733sTZp7njq4Cu07JhpICpinhLWR3nkODJbjk/mpLcQZgtV6W749AUo8oT\n"
+"jRVEJ8MpCo1h0bVDxsRnA3DrMneD88L8/b10aHs+bPm1HKbCmT+kJAFaUQNa8JvJ\n"
+"pReN37qTWvZCte7vaPAIP5cboATMu/J4t3izpm+YJoJlWcIegGx3kQ+17P4MbgDl\n"
+"S93U4sOLvTk9+MoyPo9yGWU/zHgzcQ6wCFdzWMDRswuh+/4TJ2+yg6maq3iBtj39\n"
+"gNLMR+sRgGGvYisqE9bfvNQy5IWrABBKcSBTXeTM1DmW6jv3TI8DoCzCbpjqcIwT\n"
+"u2J+7k8wJEHPcAwnBjlyWphVvwNwM0cXqOnlJZ/4z7OGgjiNEem7TMuvxk+YkiXK\n"
+"OzftdTjeIpzBwsGRP8/teMBpjS95M7GloKtxO+muBVxXbmsq8GBRC9vtNJ2Ma/xP\n"
+"bXvd+7caytD3ob6ZfOzCpi4ZS8uByEfIMxlgZ5Sn3jhgEkcIU+YW9b3teMZOuWdA\n"
+"QpDCoMpXaHVyRqwVV59JjmftiBnNBEo1/QzRj2UxRi7fHMfmNxL5LRM4CHSLUSCq\n"
+"Y3A3pkxvBHUzemhynSFvtCPa8GHiUpe9so0V/2hlgaENAVELPjMlWytaYufRllgy\n"
+"tUnCd32C5PrrmYzMKnxKRPXLcxLgziruJGSks9vIspoPk0pWgkZm+M9fRpJKlWHF\n"
+"yT9OOGBW2yynw/yvXssxJmdUDxVcWL4uS2bZc4s0Zc6RSL9uQPjZVX0JLj+cXfx4\n"
+"93Gn5bDhMgm+CGM6j3RiAAD7tT5V0sytNFjXd1A4U1u8yj3wzhKqOtZpDmuGUlMn\n"
+"EODu7I5KtWxOTPThy7TecI5r+F/6KL+2MOtRhj2PmlT/Xed6PaAmDkQeiXGps08x\n"
+"u0JIpuB61axvT4PAsKZNUd4ExbzNxRDAARUMgY8krpmyKZyHVFIQ19uHM2lGl9/i\n"
+"h3PKlLHYI8RsHutHElzq+F5tWd5AA99LVRZX4axAVIQNiqRg8IMSoCwUaCCbjUMz\n"
+"sJCo2t36GYk5S2BRnfrCqYoZRHw+ENYN0tDEMhXq1OqjvNHW3TzL3DsUhM6EZU5n\n"
+"cRR4ynUvPqqWFphLefRW10vCtaW9roJQZyFYf9kd8xgW/BhcDNbTTaQ1U6xCHgX+\n"
+"78DKee/NvY1WIEBR8X0iVk5XlSJb14eRtxNawXFyebVdmC/DiMNgnTBncMbePnZi\n"
+"KCl1r5xqo7tSIoJ6Z0l6qINd89T9fcg9mujTVwsfQ+5/kdEy0Iw7CQcTOGvMaoPX\n"
+"IAJlWSVeZ8eu8kmsD1Z8ewoPufMKiY4cPRAK5bCDgsrK6bAExOlCwPnNNM8Ym1Hz\n"
+"aYFeGs5sW468Qww+Nbl5xcNFKtwUKZ6EebRHjwttiyTgCdAhv9wL1u2WFydWWgkG\n"
+"rwUbNpSLKls+pijCeJAscvxzbZz96iOaYrY8IyzGBFwfgFAESfnzBc8SQjZzMzoO\n"
+"vmYIRon2m/5w5AZA2IjQ4VxXJDK6XExD/ZLsxNXzMnROD++hE+s8DvPlRPmN4egF\n"
+"gAzJs/9t7IyE/dDf7gSSBqzEBbwduD8ozzYHwELUc4ERdRzjEdBM0azT61g7Yilr\n"
+"iT5Hy+2iw/pNwiqVOYiAbj2lwcoMlFZmdxviD4IMXdsNVWsCAVJL0PqIh1UDDb3z\n"
+"Urv3idBJeSBuuFr6AFS6kAgvrwV/pEGoBoHuyii/rZxVugGKeuMynKEvSHuFNuQU\n"
+"qIHcNgqQR34v2Ut5pQ1R8s7K3Rae/AhE5GncJa6FJmB9TF8MYMu9PlSZV/eGv8UL\n"
+"IDWQ7sY3NdhZini//xtwPqIw29yOeZ0X6Aqsek9tfh21UwKSpHb7T+PwXYmoB+23\n"
+"p3FXkP/rv4AGRq1xJqFYzKJvwsXqTFuNFWP74yhTg6rC90w2p5TeH1rJMAnv4u0L\n"
+"hGtG/NL+D1Tzdf00TYAjno5Ia5dQJDd/eO+Ygqnhl6hAqGtS6r9JhIEXw1nQD7SC\n"
+"lj96ZuKdUWO8rpIiAtvHAsn++xvMVPm/S1SwA8oE049iVwS8/eNNiMKoSlTlYc7o\n"
+"pusBZQrVF4We4HHYFjysBbcXlvoXDd8LkZ8Nh63VQPnoIGNKH2U6aXCnQcJ8dZqO\n"
+"DNxL4uyM4A578FUUR6vxqt2asnLHQ0Z7pPE4uqtz/WgbiHI/i2oHS8oe1clsifCw\n"
+"3ZY33kflqLftkTNka1oiftDb0OqFLjkS7/AUorqHazw53gM3gqJY5EXA3Px9+nhu\n"
+"NzxSK/t41JoCfgQJHMkIWb3yUcO4OFZeGCeAxIJY95hv/brt6/WNielXjNaohYvc\n"
+"lsSUHEJRHwVxQmWK0LS+g13HAgOI7cNt3MA8sSkzTneHGFgEvmrSyb0wCEmushC9\n"
+"mjQThvaxfQk9douA/cR2bHr7axXqv9vjztmxUr0a30a7lvLMBQbJmFtJJylW+tJe\n"
+"v/vKNOB+9mK793cttr2JFnMhwUKFKWiFDQJtxw/eLQWY4BJ19Rs2x4BJgmV+u1jB\n"
+"zR8uvxuArG/cqVEJsoC6uuSzhAWSwdvumijO6yuyWF6nHY6aAcy8dyFQlDFHAd+/\n"
+"J05Lrbzj4N9lcI7hPalh0uMdERGvtUdT8QRm5ebP1zogYEkZk/1GOU29dMawkAt/\n"
+"SWhp2yWdjLt8f5HQKu72vUF/yyTfzfdqQqJwfthP7+vp+sHDO85AMF45uU9g3pxW\n"
+"IbXSbZ4fFGC1/41db/2GOHFgaheMXj0SIWHqQE1jtihr3BBBO4b3Ccz5QCnrn48J\n"
+"8L+QRdh4a/cAx4ty/oHEiXwpSBBSFRl5+y2NijC8GITA5dRjCRWP+Y0zuTrJ7j1a\n"
+"h+3kGs1kxqskhaEuhXnXyknGLjXrU+ewRGhHzP23o5betVhX+c1XjVqmJNZ5OPn/\n"
+"wrqx/XwoIl/3F5lMmGDG9mPtyg0E227nKl9Sy0Vbwx2tu1unjOlzSCa7lpoD4TIX\n"
+"PBJ5+Zb0CE6HEt3V0ec1m4uUe/xObAnzyr4UbzdqLaMy8vTcF/qsncXyPBjwqdjR\n"
+"ReDAtt99bAPY4roPKGt8dgKUPE0t/XoY+SlmUp75TkZDXrOIJXpEW0GpLPf53T+W\n"
+"Ex3KtfLAnZzrw8+dIageY7IgoQ85h3sYE7uEI8QlcO/o4udqUzTp4Sn4sWvdTLrx\n"
+"W7ImvK2rsU5ubVdsEaFKM7+7nxGn2JyMpIWFz0SbP34CkXHhrXxyRD+GhMIDHFxV\n"
+"uBnZnjJsw+ooIm1rL4I7/VMWEwmVegreT6w9Gsmb5igw+zu9v2YBgTOhysA9XZd4\n"
+"7O3VjqKkhTXcBqdpRWuz8gPQ+4rfwij28Gg2alG04Eh3G3868NOCFJhhaHVmwYR2\n"
+"ygRm6N9eDW1bHhYSN75HSEb6aIebk+1AT4S1QtJaPSH0EduIXO++JYAs+jIFKy2c\n"
+"jCVFlO/LbXl7iCdXurJHpSbMNmZFNUri6zEolENODLwke836jBOKiVrWzLnEMxHI\n"
+"WDDTpLTYhR3C7sEprpEQm9SX2Eik3WxVb4ZTb7SZFU1y1d4tWnjGu3U1D+vO9wVq\n"
+"Sss9lDipbkhQ9k4j1/Pqozaxvi8lYLbh3WEjK3Iwpr66Bk6Ai2oRg4b+7vzV4o+6\n"
+"L47JPJhajdHac0CIlmupyA4eejECS6OpoLDf5Wr/616k3dxM//3kAWGUnXVw9GSo\n"
+"UF5W8AaKlaGZ6EZk09NyGSFRjEs18z+g5ckviGF0EhZI7ZPWQQmlqWUsL9O0S4GO\n"
+"ZZ9f0UhNmHEspcugbs7e1yfjwGVyxIBkrmxpkmfHE4Gb47UGlJevg2OvZOPT3wMH\n"
+"vOds2BtqdT3tuss9k+7hsISGse7isEOb7TN5MHb6yyzqnCUZhp5m3Iag7TUkiyfU\n"
+"jKH5R13tHqKUoJ2rofWoLO2H5xSfp/lqF9sLd4rJ+Pbjhiuvfwz5copYsuTNL4kB\n"
+"SPUikHlTxSOgTBYNV77qxpsqOI3+iziCrSqHsxNdlaA1T3fiq6SeZBNdD822AYm9\n"
+"L5hbcgpDPEEwT/n5kWNbRNueerJkJwboaOnT1ZX1601Pwj5QDi+YM1NYy5PsdWxb\n"
+"bPGpQyZ+uf917q9gV7Ykr5cic10YD11khAghr0n6fYfb8Ijc22uP6m47KItDqQc1\n"
+"eFym149F56B0yg5FR85Arg==\n"
+"-----END MESSAGE-----\n"
+" signature Of+jvQKzH9ot2NV5twlDO2CFbzLSB4absWTwG58TCHb+TWgQi3z6SZIoTnGGY/uicJgEkCN++bZZR49GiyHyCQ\n";
diff --git a/src/test/test_hs_intropoint.c b/src/test/test_hs_intropoint.c
new file mode 100644
index 0000000000..558fc32c54
--- /dev/null
+++ b/src/test/test_hs_intropoint.c
@@ -0,0 +1,930 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_service.c
+ * \brief Test hidden service functionality.
+ */
+
+#define HS_SERVICE_PRIVATE
+#define HS_INTROPOINT_PRIVATE
+#define RENDSERVICE_PRIVATE
+#define CIRCUITLIST_PRIVATE
+
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "core/or/or.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "ht.h"
+#include "core/or/relay.h"
+#include "feature/rend/rendservice.h"
+
+#include "feature/hs/hs_cell.h"
+#include "feature/hs/hs_circuitmap.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_intropoint.h"
+#include "feature/hs/hs_service.h"
+
+#include "core/or/or_circuit_st.h"
+
+/* Trunnel. */
+#include "trunnel/hs/cell_establish_intro.h"
+#include "trunnel/hs/cell_introduce1.h"
+#include "trunnel/hs/cell_common.h"
+
+static size_t
+new_establish_intro_cell(const char *circ_nonce,
+                         trn_cell_establish_intro_t **cell_out)
+{
+  ssize_t cell_len = 0;
+  uint8_t buf[RELAY_PAYLOAD_SIZE] = {0};
+  trn_cell_establish_intro_t *cell = NULL;
+  hs_service_intro_point_t *ip = NULL;
+
+  /* Ensure that *cell_out is NULL such that we can use to check if we need to
+   * free `cell` in case of an error. */
+  *cell_out = NULL;
+
+  /* Auth key pair is generated in the constructor so we are all set for
+   * using this IP object. */
+  ip = service_intro_point_new(NULL, 0, 0);
+  tt_assert(ip);
+  cell_len = hs_cell_build_establish_intro(circ_nonce, ip, buf);
+  tt_i64_op(cell_len, OP_GT, 0);
+
+  cell_len = trn_cell_establish_intro_parse(&cell, buf, sizeof(buf));
+  tt_i64_op(cell_len, OP_GT, 0);
+  tt_assert(cell);
+  *cell_out = cell;
+
+ done:
+  if (*cell_out == NULL)
+    trn_cell_establish_intro_free(cell);
+
+  service_intro_point_free(ip);
+  return cell_len;
+}
+
+static ssize_t
+new_establish_intro_encoded_cell(const char *circ_nonce, uint8_t *cell_out)
+{
+  ssize_t cell_len = 0;
+  hs_service_intro_point_t *ip = NULL;
+
+  /* Auth key pair is generated in the constructor so we are all set for
+   * using this IP object. */
+  ip = service_intro_point_new(NULL, 0, 0);
+  tt_assert(ip);
+  cell_len = hs_cell_build_establish_intro(circ_nonce, ip, cell_out);
+  tt_i64_op(cell_len, OP_GT, 0);
+
+ done:
+  service_intro_point_free(ip);
+  return cell_len;
+}
+
+/* Mock function to avoid networking in unittests */
+static int
+mock_send_intro_established_cell(or_circuit_t *circ)
+{
+  (void) circ;
+  return 0;
+}
+
+static int
+mock_relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ,
+                                  uint8_t relay_command, const char *payload,
+                                  size_t payload_len,
+                                  crypt_path_t *cpath_layer,
+                                  const char *filename, int lineno)
+{
+  (void) stream_id;
+  (void) circ;
+  (void) relay_command;
+  (void) payload;
+  (void) payload_len;
+  (void) cpath_layer;
+  (void) filename;
+  (void) lineno;
+  return 0;
+}
+
+static or_circuit_t *
+helper_create_intro_circuit(void)
+{
+  or_circuit_t *circ = or_circuit_new(0, NULL);
+  tt_assert(circ);
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_OR);
+ done:
+  return circ;
+}
+
+static trn_cell_introduce1_t *
+helper_create_introduce1_cell(void)
+{
+  trn_cell_introduce1_t *cell = NULL;
+  ed25519_keypair_t auth_key_kp;
+
+  /* Generate the auth_key of the cell. */
+  if (ed25519_keypair_generate(&auth_key_kp, 0) < 0) {
+    goto err;
+  }
+
+  cell = trn_cell_introduce1_new();
+  tt_assert(cell);
+
+  /* Set the auth key. */
+  {
+    size_t auth_key_len = sizeof(auth_key_kp.pubkey);
+    trn_cell_introduce1_set_auth_key_type(cell,
+                                     TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_ED25519);
+    trn_cell_introduce1_set_auth_key_len(cell, auth_key_len);
+    trn_cell_introduce1_setlen_auth_key(cell, auth_key_len);
+    uint8_t *auth_key_ptr = trn_cell_introduce1_getarray_auth_key(cell);
+    memcpy(auth_key_ptr, auth_key_kp.pubkey.pubkey, auth_key_len);
+  }
+
+  /* Set the cell extensions to none. */
+  {
+    trn_cell_extension_t *ext = trn_cell_extension_new();
+    trn_cell_extension_set_num(ext, 0);
+    trn_cell_introduce1_set_extensions(cell, ext);
+  }
+
+  /* Set the encrypted section to some data. */
+  {
+    size_t enc_len = 128;
+    trn_cell_introduce1_setlen_encrypted(cell, enc_len);
+    uint8_t *enc_ptr = trn_cell_introduce1_getarray_encrypted(cell);
+    memset(enc_ptr, 'a', enc_len);
+  }
+
+  return cell;
+ err:
+ done:
+  trn_cell_introduce1_free(cell);
+  return NULL;
+}
+
+/* Try sending an ESTABLISH_INTRO cell on a circuit that is already an intro
+ * point. Should fail. */
+static void
+test_establish_intro_wrong_purpose(void *arg)
+{
+  int retval;
+  ssize_t cell_len = 0;
+  char circ_nonce[DIGEST_LEN] = {0};
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);
+
+  (void)arg;
+
+  /* Get the auth key of the intro point */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  memcpy(intro_circ->rend_circ_nonce, circ_nonce, DIGEST_LEN);
+
+  /* Set a bad circuit purpose!! :) */
+  circuit_change_purpose(TO_CIRCUIT(intro_circ), CIRCUIT_PURPOSE_INTRO_POINT);
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+     attempt to parse it. */
+  cell_len = new_establish_intro_encoded_cell(circ_nonce, cell_body);
+  tt_i64_op(cell_len, OP_GT, 0);
+
+  /* Receive the cell. Should fail. */
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_intro_received_establish_intro(intro_circ, cell_body, cell_len);
+  expect_log_msg_containing("Rejecting ESTABLISH_INTRO on non-OR circuit.");
+  teardown_capture_of_logs();
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  circuit_free_(TO_CIRCUIT(intro_circ));
+}
+
+/* Prepare a circuit for accepting an ESTABLISH_INTRO cell */
+static void
+helper_prepare_circ_for_intro(or_circuit_t *circ, const char *circ_nonce)
+{
+  /* Prepare the circuit for the incoming ESTABLISH_INTRO */
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_OR);
+  memcpy(circ->rend_circ_nonce, circ_nonce, DIGEST_LEN);
+}
+
+/* Send an empty ESTABLISH_INTRO cell. Should fail. */
+static void
+test_establish_intro_wrong_keytype(void *arg)
+{
+  int retval;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);
+  char circ_nonce[DIGEST_LEN] = {0};
+
+  (void) arg;
+
+  /* Get the auth key of the intro point */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Receive the cell. Should fail. */
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_intro_received_establish_intro(intro_circ, (uint8_t *) "", 0);
+  expect_log_msg_containing("Empty ESTABLISH_INTRO cell.");
+  teardown_capture_of_logs();
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  circuit_free_(TO_CIRCUIT(intro_circ));
+}
+
+/* Send an ESTABLISH_INTRO cell with an unknown auth key type. Should fail. */
+static void
+test_establish_intro_wrong_keytype2(void *arg)
+{
+  int retval;
+  char circ_nonce[DIGEST_LEN] = {0};
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  ssize_t cell_len = 0;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);
+
+  (void) arg;
+
+  /* Get the auth key of the intro point */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+   * attempt to parse it. */
+  cell_len = new_establish_intro_encoded_cell(circ_nonce, cell_body);
+  tt_i64_op(cell_len, OP_GT, 0);
+
+  /* Mutate the auth key type! :) */
+  cell_body[0] = 42;
+
+  /* Receive the cell. Should fail. */
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_intro_received_establish_intro(intro_circ, cell_body, cell_len);
+  expect_log_msg_containing("Unrecognized AUTH_KEY_TYPE 42.");
+  teardown_capture_of_logs();
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  circuit_free_(TO_CIRCUIT(intro_circ));
+}
+
+/* Send a legit ESTABLISH_INTRO cell but with a wrong MAC. Should fail. */
+static void
+test_establish_intro_wrong_mac(void *arg)
+{
+  int retval;
+  char circ_nonce[DIGEST_LEN] = {0};
+  ssize_t cell_len = 0;
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  trn_cell_establish_intro_t *cell = NULL;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);
+
+  (void) arg;
+
+  /* Get the auth key of the intro point */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+   * attempt to parse it. */
+  cell_len = new_establish_intro_cell(circ_nonce, &cell);
+  tt_i64_op(cell_len, OP_GT, 0);
+  tt_assert(cell);
+
+  /* Mangle one byte of the MAC. */
+  uint8_t *handshake_ptr =
+    trn_cell_establish_intro_getarray_handshake_mac(cell);
+  handshake_ptr[TRUNNEL_SHA3_256_LEN - 1]++;
+  /* We need to resign the payload with that change. */
+  {
+    ed25519_signature_t sig;
+    ed25519_keypair_t key_struct;
+    /* New keypair for the signature since we don't have access to the private
+     * key material generated earlier when creating the cell. */
+    retval = ed25519_keypair_generate(&key_struct, 0);
+    tt_int_op(retval, OP_EQ, 0);
+    uint8_t *auth_key_ptr =
+      trn_cell_establish_intro_getarray_auth_key(cell);
+    memcpy(auth_key_ptr, key_struct.pubkey.pubkey, ED25519_PUBKEY_LEN);
+    /* Encode payload so we can sign it. */
+    cell_len = trn_cell_establish_intro_encode(cell_body, sizeof(cell_body),
+                                               cell);
+    tt_i64_op(cell_len, OP_GT, 0);
+
+    retval = ed25519_sign_prefixed(&sig, cell_body,
+                                   cell_len -
+                                   (ED25519_SIG_LEN + sizeof(cell->sig_len)),
+                                   ESTABLISH_INTRO_SIG_PREFIX, &key_struct);
+    tt_int_op(retval, OP_EQ, 0);
+    /* And write the signature to the cell */
+    uint8_t *sig_ptr =
+      trn_cell_establish_intro_getarray_sig(cell);
+    memcpy(sig_ptr, sig.sig, cell->sig_len);
+    /* Re-encode with the new signature. */
+    cell_len = trn_cell_establish_intro_encode(cell_body, sizeof(cell_body),
+                                               cell);
+    tt_i64_op(cell_len, OP_GT, 0);
+  }
+
+  /* Receive the cell. Should fail because our MAC is wrong. */
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_intro_received_establish_intro(intro_circ, cell_body, cell_len);
+  expect_log_msg_containing("ESTABLISH_INTRO handshake_auth not as expected");
+  teardown_capture_of_logs();
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  trn_cell_establish_intro_free(cell);
+  circuit_free_(TO_CIRCUIT(intro_circ));
+}
+
+/* Send a legit ESTABLISH_INTRO cell but with a wrong auth key length. Should
+ * fail. */
+static void
+test_establish_intro_wrong_auth_key_len(void *arg)
+{
+  int retval;
+  char circ_nonce[DIGEST_LEN] = {0};
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  ssize_t cell_len = 0;
+  size_t bad_auth_key_len = ED25519_PUBKEY_LEN - 1;
+  trn_cell_establish_intro_t *cell = NULL;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);
+
+  (void) arg;
+
+  /* Get the auth key of the intro point */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+   * attempt to parse it. */
+  cell_len = new_establish_intro_cell(circ_nonce, &cell);
+  tt_i64_op(cell_len, OP_GT, 0);
+  tt_assert(cell);
+
+  /* Mangle the auth key length. */
+  trn_cell_establish_intro_set_auth_key_len(cell, bad_auth_key_len);
+  trn_cell_establish_intro_setlen_auth_key(cell, bad_auth_key_len);
+  /* Encode cell. */
+  cell_len = trn_cell_establish_intro_encode(cell_body, sizeof(cell_body),
+                                             cell);
+  tt_int_op(cell_len, OP_GT, 0);
+
+  /* Receive the cell. Should fail. */
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_intro_received_establish_intro(intro_circ, cell_body, cell_len);
+  expect_log_msg_containing("ESTABLISH_INTRO auth key length is invalid");
+  teardown_capture_of_logs();
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  trn_cell_establish_intro_free(cell);
+  circuit_free_(TO_CIRCUIT(intro_circ));
+}
+
+/* Send a legit ESTABLISH_INTRO cell but with a wrong sig length. Should
+ * fail. */
+static void
+test_establish_intro_wrong_sig_len(void *arg)
+{
+  int retval;
+  char circ_nonce[DIGEST_LEN] = {0};
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  ssize_t cell_len = 0;
+  size_t bad_sig_len = ED25519_SIG_LEN - 1;
+  trn_cell_establish_intro_t *cell = NULL;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);
+
+  (void) arg;
+
+  /* Get the auth key of the intro point */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+   * attempt to parse it. */
+  cell_len = new_establish_intro_cell(circ_nonce, &cell);
+  tt_i64_op(cell_len, OP_GT, 0);
+  tt_assert(cell);
+
+  /* Mangle the signature length. */
+  trn_cell_establish_intro_set_sig_len(cell, bad_sig_len);
+  trn_cell_establish_intro_setlen_sig(cell, bad_sig_len);
+  /* Encode cell. */
+  cell_len = trn_cell_establish_intro_encode(cell_body, sizeof(cell_body),
+                                             cell);
+  tt_int_op(cell_len, OP_GT, 0);
+
+  /* Receive the cell. Should fail. */
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_intro_received_establish_intro(intro_circ, cell_body, cell_len);
+  expect_log_msg_containing("ESTABLISH_INTRO sig len is invalid");
+  teardown_capture_of_logs();
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  trn_cell_establish_intro_free(cell);
+  circuit_free_(TO_CIRCUIT(intro_circ));
+}
+
+/* Send a legit ESTABLISH_INTRO cell but slightly change the signature. Should
+ * fail. */
+static void
+test_establish_intro_wrong_sig(void *arg)
+{
+  int retval;
+  char circ_nonce[DIGEST_LEN] = {0};
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  ssize_t cell_len = 0;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);
+
+  (void) arg;
+
+  /* Get the auth key of the intro point */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+     attempt to parse it. */
+  cell_len = new_establish_intro_encoded_cell(circ_nonce, cell_body);
+  tt_i64_op(cell_len, OP_GT, 0);
+
+  /* Mutate the last byte (signature)! :) */
+  cell_body[cell_len - 1]++;
+
+  /* Receive the cell. Should fail. */
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_intro_received_establish_intro(intro_circ, cell_body,
+                                             (size_t)cell_len);
+  expect_log_msg_containing("Failed to verify ESTABLISH_INTRO cell.");
+  teardown_capture_of_logs();
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  circuit_free_(TO_CIRCUIT(intro_circ));
+}
+
+/* Helper function: Send a well-formed v3 ESTABLISH_INTRO cell to
+ * <b>intro_circ</b>. Return the cell. */
+static trn_cell_establish_intro_t *
+helper_establish_intro_v3(or_circuit_t *intro_circ)
+{
+  int retval;
+  char circ_nonce[DIGEST_LEN] = {0};
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  ssize_t cell_len = 0;
+  trn_cell_establish_intro_t *cell = NULL;
+
+  tt_assert(intro_circ);
+
+  /* Prepare the circuit for the incoming ESTABLISH_INTRO */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+   * attempt to parse it. */
+  cell_len = new_establish_intro_cell(circ_nonce, &cell);
+  tt_i64_op(cell_len, OP_GT, 0);
+  tt_assert(cell);
+  cell_len = trn_cell_establish_intro_encode(cell_body, sizeof(cell_body),
+                                             cell);
+  tt_int_op(cell_len, OP_GT, 0);
+
+  /* Receive the cell */
+  retval = hs_intro_received_establish_intro(intro_circ, cell_body,
+                                             (size_t) cell_len);
+  tt_int_op(retval, OP_EQ, 0);
+
+ done:
+  return cell;
+}
+
+/* Helper function: Send a well-formed v2 ESTABLISH_INTRO cell to
+ * <b>intro_circ</b>. Return the public key advertised in the cell. */
+static crypto_pk_t *
+helper_establish_intro_v2(or_circuit_t *intro_circ)
+{
+  crypto_pk_t *key1 = NULL;
+  int retval;
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  ssize_t cell_len = 0;
+  char circ_nonce[DIGEST_LEN] = {0};
+
+  tt_assert(intro_circ);
+
+  /* Prepare the circuit for the incoming ESTABLISH_INTRO */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Send legacy establish_intro */
+  key1 = pk_generate(0);
+
+  /* Use old circ_nonce why not */
+  cell_len = rend_service_encode_establish_intro_cell(
+                                           (char*)cell_body,
+                                           sizeof(cell_body), key1,
+                                           circ_nonce);
+  tt_int_op(cell_len, OP_GT, 0);
+
+  /* Receive legacy establish_intro */
+  retval = hs_intro_received_establish_intro(intro_circ,
+                                             cell_body, (size_t) cell_len);
+  tt_int_op(retval, OP_EQ, 0);
+
+ done:
+  return key1;
+}
+
+/* Helper function: test circuitmap free_all function outside of
+ * test_intro_point_registration to prevent Coverity from seeing a
+ * double free if the assertion hypothetically fails.
+ */
+static void
+test_circuitmap_free_all(void)
+{
+  hs_circuitmap_ht *the_hs_circuitmap = NULL;
+
+  the_hs_circuitmap = get_hs_circuitmap();
+  tt_assert(the_hs_circuitmap);
+  hs_circuitmap_free_all();
+  the_hs_circuitmap = get_hs_circuitmap();
+  tt_ptr_op(the_hs_circuitmap, OP_EQ, NULL);
+ done:
+  ;
+}
+
+/** Successfully register a v2 intro point and a v3 intro point. Ensure that HS
+ *  circuitmap is maintained properly. */
+static void
+test_intro_point_registration(void *arg)
+{
+  int retval;
+  hs_circuitmap_ht *the_hs_circuitmap = NULL;
+
+  or_circuit_t *intro_circ = NULL;
+  trn_cell_establish_intro_t *establish_intro_cell = NULL;
+  ed25519_public_key_t auth_key;
+
+  crypto_pk_t *legacy_auth_key = NULL;
+  or_circuit_t *legacy_intro_circ = NULL;
+
+  or_circuit_t *returned_intro_circ = NULL;
+
+  (void) arg;
+
+  MOCK(hs_intro_send_intro_established_cell, mock_send_intro_established_cell);
+
+  hs_circuitmap_init();
+
+  /* Check that the circuitmap is currently empty */
+  {
+    the_hs_circuitmap = get_hs_circuitmap();
+    tt_assert(the_hs_circuitmap);
+    tt_int_op(0, OP_EQ, HT_SIZE(the_hs_circuitmap));
+    /* Do a circuitmap query in any case */
+    returned_intro_circ =hs_circuitmap_get_intro_circ_v3_relay_side(&auth_key);
+    tt_ptr_op(returned_intro_circ, OP_EQ, NULL);
+  }
+
+  /* Create a v3 intro point */
+  {
+    intro_circ = or_circuit_new(0, NULL);
+    tt_assert(intro_circ);
+    establish_intro_cell = helper_establish_intro_v3(intro_circ);
+
+    /* Check that the intro point was registered on the HS circuitmap */
+    the_hs_circuitmap = get_hs_circuitmap();
+    tt_assert(the_hs_circuitmap);
+    tt_int_op(1, OP_EQ, HT_SIZE(the_hs_circuitmap));
+    get_auth_key_from_cell(&auth_key, RELAY_COMMAND_ESTABLISH_INTRO,
+                           establish_intro_cell);
+    returned_intro_circ =
+      hs_circuitmap_get_intro_circ_v3_relay_side(&auth_key);
+    tt_ptr_op(intro_circ, OP_EQ, returned_intro_circ);
+  }
+
+  /* Create a v2 intro point */
+  {
+    char key_digest[DIGEST_LEN];
+
+    legacy_intro_circ = or_circuit_new(1, NULL);
+    tt_assert(legacy_intro_circ);
+    legacy_auth_key = helper_establish_intro_v2(legacy_intro_circ);
+    tt_assert(legacy_auth_key);
+
+    /* Check that the circuitmap now has two elements */
+    the_hs_circuitmap = get_hs_circuitmap();
+    tt_assert(the_hs_circuitmap);
+    tt_int_op(2, OP_EQ, HT_SIZE(the_hs_circuitmap));
+
+    /* Check that the new element is our legacy intro circuit. */
+    retval = crypto_pk_get_digest(legacy_auth_key, key_digest);
+    tt_int_op(retval, OP_EQ, 0);
+    returned_intro_circ =
+      hs_circuitmap_get_intro_circ_v2_relay_side((uint8_t*)key_digest);
+    tt_ptr_op(legacy_intro_circ, OP_EQ, returned_intro_circ);
+  }
+
+  /* XXX Continue test and try to register a second v3 intro point with the
+   * same auth key. Make sure that old intro circuit gets closed. */
+
+ done:
+  crypto_pk_free(legacy_auth_key);
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  circuit_free_(TO_CIRCUIT(legacy_intro_circ));
+  trn_cell_establish_intro_free(establish_intro_cell);
+  test_circuitmap_free_all();
+
+  UNMOCK(hs_intro_send_intro_established_cell);
+}
+
+static void
+test_introduce1_suitable_circuit(void *arg)
+{
+  int ret;
+  or_circuit_t *circ = NULL;
+
+  (void) arg;
+
+  /* Valid suitable circuit. */
+  {
+    circ = or_circuit_new(0, NULL);
+    circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_OR);
+    ret = circuit_is_suitable_for_introduce1(circ);
+    circuit_free_(TO_CIRCUIT(circ));
+    tt_int_op(ret, OP_EQ, 1);
+  }
+
+  /* Test if the circuit purpose safeguard works correctly. */
+  {
+    circ = or_circuit_new(0, NULL);
+    circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_INTRO_POINT);
+    ret = circuit_is_suitable_for_introduce1(circ);
+    circuit_free_(TO_CIRCUIT(circ));
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Test the non-edge circuit safeguard works correctly. */
+  {
+    circ = or_circuit_new(0, NULL);
+    circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_OR);
+    /* Bogus pointer, the check is against NULL on n_chan. */
+    circ->base_.n_chan = (channel_t *) circ;
+    ret = circuit_is_suitable_for_introduce1(circ);
+    circuit_free_(TO_CIRCUIT(circ));
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Mangle the circuit a bit more so see if our only one INTRODUCE1 cell
+   * limit works correctly. */
+  {
+    circ = or_circuit_new(0, NULL);
+    circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_OR);
+    circ->already_received_introduce1 = 1;
+    ret = circuit_is_suitable_for_introduce1(circ);
+    circuit_free_(TO_CIRCUIT(circ));
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+ done:
+  ;
+}
+
+static void
+test_introduce1_is_legacy(void *arg)
+{
+  int ret;
+  uint8_t request[256];
+
+  (void) arg;
+
+  /* For a cell to be considered legacy, according to the specification, the
+   * first 20 bytes MUST BE non-zero else it's a v3 cell. */
+  memset(request, 'a', DIGEST_LEN);
+  memset(request + DIGEST_LEN, 0, sizeof(request) - DIGEST_LEN);
+  ret = introduce1_cell_is_legacy(request);
+  tt_int_op(ret, OP_EQ, 1);
+
+  /* This is a NON legacy cell. */
+  memset(request, 0, DIGEST_LEN);
+  memset(request + DIGEST_LEN, 'a', sizeof(request) - DIGEST_LEN);
+  ret = introduce1_cell_is_legacy(request);
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  ;
+}
+
+static void
+test_introduce1_validation(void *arg)
+{
+  int ret;
+  trn_cell_introduce1_t *cell = NULL;
+
+  (void) arg;
+
+  /* Create our decoy cell that we'll modify as we go to test the validation
+   * function of that parsed cell. */
+  cell = helper_create_introduce1_cell();
+  tt_assert(cell);
+
+  /* It should NOT be a legacy cell which will trigger a BUG(). */
+  memset(cell->legacy_key_id, 'a', sizeof(cell->legacy_key_id));
+  tor_capture_bugs_(1);
+  ret = validate_introduce1_parsed_cell(cell);
+  tor_end_capture_bugs_();
+  tt_int_op(ret, OP_EQ, -1);
+  /* Reset legacy ID and make sure it's correct. */
+  memset(cell->legacy_key_id, 0, sizeof(cell->legacy_key_id));
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Non existing auth key type. */
+  cell->auth_key_type = 42;
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, -1);
+  /* Reset is to correct value and make sure it's correct. */
+  cell->auth_key_type = TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_ED25519;
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Really bad key length. */
+  cell->auth_key_len = 0;
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, -1);
+  cell->auth_key_len = UINT16_MAX;
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, -1);
+  /* Correct size, let's try that. */
+  cell->auth_key_len = sizeof(ed25519_public_key_t);
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, 0);
+  /* Set an invalid size of the auth key buffer. */
+  trn_cell_introduce1_setlen_auth_key(cell, 3);
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, -1);
+  /* Reset auth key buffer and make sure it works. */
+  trn_cell_introduce1_setlen_auth_key(cell, sizeof(ed25519_public_key_t));
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Empty encrypted section. */
+  trn_cell_introduce1_setlen_encrypted(cell, 0);
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, -1);
+  /* Reset it to some non zero bytes and validate. */
+  trn_cell_introduce1_setlen_encrypted(cell, 1);
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  trn_cell_introduce1_free(cell);
+}
+
+static void
+test_received_introduce1_handling(void *arg)
+{
+  int ret;
+  uint8_t *request = NULL, buf[128];
+  trn_cell_introduce1_t *cell = NULL;
+  or_circuit_t *circ = NULL;
+
+  (void) arg;
+
+  MOCK(relay_send_command_from_edge_, mock_relay_send_command_from_edge);
+
+  hs_circuitmap_init();
+
+  /* Too small request length. An INTRODUCE1 expect at the very least a
+   * DIGEST_LEN size. */
+  {
+    memset(buf, 0, sizeof(buf));
+    circ = helper_create_intro_circuit();
+    ret = hs_intro_received_introduce1(circ, buf, DIGEST_LEN - 1);
+    tt_int_op(ret, OP_EQ, -1);
+    circuit_free_(TO_CIRCUIT(circ));
+  }
+
+  /* We have a unit test only for the suitability of a circuit to receive an
+   * INTRODUCE1 cell so from now on we'll only test the handling of a cell. */
+
+  /* Bad request. */
+  {
+    circ = helper_create_intro_circuit();
+    uint8_t test[2]; /* Too small request. */
+    memset(test, 0, sizeof(test));
+    ret = handle_introduce1(circ, test, sizeof(test));
+    tor_free(circ->p_chan);
+    circuit_free_(TO_CIRCUIT(circ));
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Valid case. */
+  {
+    cell = helper_create_introduce1_cell();
+    ssize_t request_len = trn_cell_introduce1_encoded_len(cell);
+    tt_int_op((int)request_len, OP_GT, 0);
+    request = tor_malloc_zero(request_len);
+    ssize_t encoded_len =
+      trn_cell_introduce1_encode(request, request_len, cell);
+    tt_int_op((int)encoded_len, OP_GT, 0);
+
+    circ = helper_create_intro_circuit();
+    or_circuit_t *service_circ = helper_create_intro_circuit();
+    circuit_change_purpose(TO_CIRCUIT(service_circ),
+                           CIRCUIT_PURPOSE_INTRO_POINT);
+    /* Register the circuit in the map for the auth key of the cell. */
+    ed25519_public_key_t auth_key;
+    const uint8_t *cell_auth_key =
+      trn_cell_introduce1_getconstarray_auth_key(cell);
+    memcpy(auth_key.pubkey, cell_auth_key, ED25519_PUBKEY_LEN);
+    hs_circuitmap_register_intro_circ_v3_relay_side(service_circ, &auth_key);
+    ret = hs_intro_received_introduce1(circ, request, request_len);
+    circuit_free_(TO_CIRCUIT(circ));
+    circuit_free_(TO_CIRCUIT(service_circ));
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Valid legacy cell. */
+  {
+    tor_free(request);
+    trn_cell_introduce1_free(cell);
+    cell = helper_create_introduce1_cell();
+    uint8_t *legacy_key_id = trn_cell_introduce1_getarray_legacy_key_id(cell);
+    memset(legacy_key_id, 'a', DIGEST_LEN);
+    /* Add an arbitrary amount of data for the payload of a v2 cell. */
+    size_t request_len = trn_cell_introduce1_encoded_len(cell) + 256;
+    tt_size_op(request_len, OP_GT, 0);
+    request = tor_malloc_zero(request_len + 256);
+    ssize_t encoded_len =
+      trn_cell_introduce1_encode(request, request_len, cell);
+    tt_int_op((int)encoded_len, OP_GT, 0);
+
+    circ = helper_create_intro_circuit();
+    or_circuit_t *service_circ = helper_create_intro_circuit();
+    circuit_change_purpose(TO_CIRCUIT(service_circ),
+                           CIRCUIT_PURPOSE_INTRO_POINT);
+    /* Register the circuit in the map for the auth key of the cell. */
+    uint8_t token[REND_TOKEN_LEN];
+    memcpy(token, legacy_key_id, sizeof(token));
+    hs_circuitmap_register_intro_circ_v2_relay_side(service_circ, token);
+    ret = hs_intro_received_introduce1(circ, request, request_len);
+    circuit_free_(TO_CIRCUIT(circ));
+    circuit_free_(TO_CIRCUIT(service_circ));
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+ done:
+  trn_cell_introduce1_free(cell);
+  tor_free(request);
+  hs_circuitmap_free_all();
+  UNMOCK(relay_send_command_from_edge_);
+}
+
+struct testcase_t hs_intropoint_tests[] = {
+  { "intro_point_registration",
+    test_intro_point_registration, TT_FORK, NULL, NULL },
+
+  { "receive_establish_intro_wrong_keytype",
+    test_establish_intro_wrong_keytype, TT_FORK, NULL, NULL },
+
+  { "receive_establish_intro_wrong_keytype2",
+    test_establish_intro_wrong_keytype2, TT_FORK, NULL, NULL },
+
+  { "receive_establish_intro_wrong_purpose",
+    test_establish_intro_wrong_purpose, TT_FORK, NULL, NULL },
+
+  { "receive_establish_intro_wrong_sig",
+    test_establish_intro_wrong_sig, TT_FORK, NULL, NULL },
+
+  { "receive_establish_intro_wrong_sig_len",
+    test_establish_intro_wrong_sig_len, TT_FORK, NULL, NULL },
+
+  { "receive_establish_intro_wrong_auth_key_len",
+    test_establish_intro_wrong_auth_key_len, TT_FORK, NULL, NULL },
+
+  { "receive_establish_intro_wrong_mac",
+    test_establish_intro_wrong_mac, TT_FORK, NULL, NULL },
+
+  { "introduce1_suitable_circuit",
+    test_introduce1_suitable_circuit, TT_FORK, NULL, NULL },
+
+  { "introduce1_is_legacy",
+    test_introduce1_is_legacy, TT_FORK, NULL, NULL },
+
+  { "introduce1_validation",
+    test_introduce1_validation, TT_FORK, NULL, NULL },
+
+  { "received_introduce1_handling",
+    test_received_introduce1_handling, TT_FORK, NULL, NULL },
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_hs_ntor.c b/src/test/test_hs_ntor.c
new file mode 100644
index 0000000000..1c694e6040
--- /dev/null
+++ b/src/test/test_hs_ntor.c
@@ -0,0 +1,115 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_ntor.c
+ * \brief Test hidden service ntor functionality.
+ */
+
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+#include "core/crypto/hs_ntor.h"
+
+/* Test the HS ntor handshake. Simulate the sending of an encrypted INTRODUCE1
+ * cell, and verify the proper derivation of decryption keys on the other end.
+ * Then simulate the sending of an authenticated RENDEZVOUS1 cell and verify
+ * the proper verification on the other end. */
+static void
+test_hs_ntor(void *arg)
+{
+  int retval;
+
+  uint8_t subcredential[DIGEST256_LEN];
+
+  ed25519_keypair_t service_intro_auth_keypair;
+  curve25519_keypair_t service_intro_enc_keypair;
+  curve25519_keypair_t service_ephemeral_rend_keypair;
+
+  curve25519_keypair_t client_ephemeral_enc_keypair;
+
+  hs_ntor_intro_cell_keys_t client_hs_ntor_intro_cell_keys;
+  hs_ntor_intro_cell_keys_t service_hs_ntor_intro_cell_keys;
+
+  hs_ntor_rend_cell_keys_t service_hs_ntor_rend_cell_keys;
+  hs_ntor_rend_cell_keys_t client_hs_ntor_rend_cell_keys;
+
+  (void) arg;
+
+  /* Generate fake data for this unittest */
+  {
+    /* Generate fake subcredential */
+    memset(subcredential, 'Z', DIGEST256_LEN);
+
+    /* service */
+    curve25519_keypair_generate(&service_intro_enc_keypair, 0);
+    ed25519_keypair_generate(&service_intro_auth_keypair, 0);
+    curve25519_keypair_generate(&service_ephemeral_rend_keypair, 0);
+    /* client */
+    curve25519_keypair_generate(&client_ephemeral_enc_keypair, 0);
+  }
+
+  /* Client: Simulate the sending of an encrypted INTRODUCE1 cell */
+  retval =
+    hs_ntor_client_get_introduce1_keys(&service_intro_auth_keypair.pubkey,
+                                       &service_intro_enc_keypair.pubkey,
+                                       &client_ephemeral_enc_keypair,
+                                       subcredential,
+                                       &client_hs_ntor_intro_cell_keys);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Service: Simulate the decryption of the received INTRODUCE1 */
+  retval =
+    hs_ntor_service_get_introduce1_keys(&service_intro_auth_keypair.pubkey,
+                                        &service_intro_enc_keypair,
+                                        &client_ephemeral_enc_keypair.pubkey,
+                                        subcredential,
+                                        &service_hs_ntor_intro_cell_keys);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Test that the INTRODUCE1 encryption/mac keys match! */
+  tt_mem_op(client_hs_ntor_intro_cell_keys.enc_key, OP_EQ,
+            service_hs_ntor_intro_cell_keys.enc_key,
+            CIPHER256_KEY_LEN);
+  tt_mem_op(client_hs_ntor_intro_cell_keys.mac_key, OP_EQ,
+            service_hs_ntor_intro_cell_keys.mac_key,
+            DIGEST256_LEN);
+
+  /* Service: Simulate creation of RENDEZVOUS1 key material. */
+  retval =
+    hs_ntor_service_get_rendezvous1_keys(&service_intro_auth_keypair.pubkey,
+                                         &service_intro_enc_keypair,
+                                         &service_ephemeral_rend_keypair,
+                                         &client_ephemeral_enc_keypair.pubkey,
+                                         &service_hs_ntor_rend_cell_keys);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Client: Simulate the verification of a received RENDEZVOUS1 cell */
+  retval =
+    hs_ntor_client_get_rendezvous1_keys(&service_intro_auth_keypair.pubkey,
+                                        &client_ephemeral_enc_keypair,
+                                        &service_intro_enc_keypair.pubkey,
+                                        &service_ephemeral_rend_keypair.pubkey,
+                                        &client_hs_ntor_rend_cell_keys);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Test that the RENDEZVOUS1 key material match! */
+  tt_mem_op(client_hs_ntor_rend_cell_keys.rend_cell_auth_mac, OP_EQ,
+            service_hs_ntor_rend_cell_keys.rend_cell_auth_mac,
+            DIGEST256_LEN);
+  tt_mem_op(client_hs_ntor_rend_cell_keys.ntor_key_seed, OP_EQ,
+            service_hs_ntor_rend_cell_keys.ntor_key_seed,
+            DIGEST256_LEN);
+ done:
+  ;
+}
+
+struct testcase_t hs_ntor_tests[] = {
+  { "hs_ntor", test_hs_ntor, TT_FORK,
+    NULL, NULL },
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_hs_ntor.sh b/src/test/test_hs_ntor.sh
new file mode 100755
index 0000000000..8a0003d44a
--- /dev/null
+++ b/src/test/test_hs_ntor.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+# Validate Tor's ntor implementation.
+
+exitcode=0
+
+# Run the python integration test sand return the exitcode of the python
+# script.  The python script might ask the testsuite to skip it if not all
+# python dependencies are covered.
+"${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/hs_ntor_ref.py" || exitcode=$?
+
+exit ${exitcode}
diff --git a/src/test/test_hs_ntor_cl.c b/src/test/test_hs_ntor_cl.c
new file mode 100644
index 0000000000..6341b96d84
--- /dev/null
+++ b/src/test/test_hs_ntor_cl.c
@@ -0,0 +1,259 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/** This is a wrapper over the little-t-tor HS ntor functions. The wrapper is
+ *  used by src/test/hs_ntor_ref.py to conduct the HS ntor integration
+ *  tests.
+ *
+ *  The logic of this wrapper is basically copied from src/test/test_ntor_cl.c
+ */
+
+#include "orconfig.h"
+#include <stdio.h>
+#include <stdlib.h>
+
+#define ONION_NTOR_PRIVATE
+#include "core/or/or.h"
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_init.h"
+#include "core/crypto/hs_ntor.h"
+#include "core/crypto/onion_ntor.h"
+
+#define N_ARGS(n) STMT_BEGIN {                                  \
+    if (argc < (n)) {                                           \
+      fprintf(stderr, "%s needs %d arguments.\n",argv[1],n);    \
+      return 1;                                                 \
+    }                                                           \
+  } STMT_END
+#define BASE16(idx, var, n) STMT_BEGIN {                                \
+    const char *s = argv[(idx)];                                        \
+    if (base16_decode((char*)var, n, s, strlen(s)) < (int)n ) {              \
+      fprintf(stderr, "couldn't decode argument %d (%s)\n",idx,s);      \
+      return 1;                                                         \
+    }                                                                   \
+  } STMT_END
+#define INT(idx, var) STMT_BEGIN {                                      \
+    var = atoi(argv[(idx)]);                                            \
+    if (var <= 0) {                                                     \
+      fprintf(stderr, "bad integer argument %d (%s)\n",idx,argv[(idx)]); \
+    }                                                                   \
+  } STMT_END
+
+/** The first part of the HS ntor protocol. The client-side computes all
+    necessary key material and sends the appropriate message to the service. */
+static int
+client1(int argc, char **argv)
+{
+  int retval;
+
+  /* Inputs */
+  curve25519_public_key_t intro_enc_pubkey;
+  ed25519_public_key_t intro_auth_pubkey;
+  curve25519_keypair_t client_ephemeral_enc_keypair;
+  uint8_t subcredential[DIGEST256_LEN];
+
+  /* Output */
+  hs_ntor_intro_cell_keys_t hs_ntor_intro_cell_keys;
+
+  char buf[256];
+
+  N_ARGS(6);
+  BASE16(2, intro_auth_pubkey.pubkey, ED25519_PUBKEY_LEN);
+  BASE16(3, intro_enc_pubkey.public_key, CURVE25519_PUBKEY_LEN);
+  BASE16(4, client_ephemeral_enc_keypair.seckey.secret_key,
+         CURVE25519_SECKEY_LEN);
+  BASE16(5, subcredential, DIGEST256_LEN);
+
+  /* Generate keypair */
+  curve25519_public_key_generate(&client_ephemeral_enc_keypair.pubkey,
+                                 &client_ephemeral_enc_keypair.seckey);
+
+  retval = hs_ntor_client_get_introduce1_keys(&intro_auth_pubkey,
+                                              &intro_enc_pubkey,
+                                              &client_ephemeral_enc_keypair,
+                                              subcredential,
+                                              &hs_ntor_intro_cell_keys);
+  if (retval < 0) {
+    goto done;
+  }
+
+  /* Send ENC_KEY */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_intro_cell_keys.enc_key,
+                sizeof(hs_ntor_intro_cell_keys.enc_key));
+  printf("%s\n", buf);
+  /* Send MAC_KEY */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_intro_cell_keys.mac_key,
+                sizeof(hs_ntor_intro_cell_keys.mac_key));
+  printf("%s\n", buf);
+
+ done:
+  return retval;
+}
+
+/** The second part of the HS ntor protocol. The service-side computes all
+    necessary key material and sends the appropriate message to the client */
+static int
+server1(int argc, char **argv)
+{
+  int retval;
+
+  /* Inputs */
+  curve25519_keypair_t intro_enc_keypair;
+  ed25519_public_key_t intro_auth_pubkey;
+  curve25519_public_key_t client_ephemeral_enc_pubkey;
+  uint8_t subcredential[DIGEST256_LEN];
+
+  /* Output */
+  hs_ntor_intro_cell_keys_t hs_ntor_intro_cell_keys;
+  hs_ntor_rend_cell_keys_t hs_ntor_rend_cell_keys;
+  curve25519_keypair_t service_ephemeral_rend_keypair;
+
+  char buf[256];
+
+  N_ARGS(6);
+  BASE16(2, intro_auth_pubkey.pubkey, ED25519_PUBKEY_LEN);
+  BASE16(3, intro_enc_keypair.seckey.secret_key, CURVE25519_SECKEY_LEN);
+  BASE16(4, client_ephemeral_enc_pubkey.public_key, CURVE25519_PUBKEY_LEN);
+  BASE16(5, subcredential, DIGEST256_LEN);
+
+  /* Generate keypair */
+  curve25519_public_key_generate(&intro_enc_keypair.pubkey,
+                                 &intro_enc_keypair.seckey);
+  curve25519_keypair_generate(&service_ephemeral_rend_keypair, 0);
+
+  /* Get INTRODUCE1 keys */
+  retval = hs_ntor_service_get_introduce1_keys(&intro_auth_pubkey,
+                                               &intro_enc_keypair,
+                                               &client_ephemeral_enc_pubkey,
+                                               subcredential,
+                                               &hs_ntor_intro_cell_keys);
+  if (retval < 0) {
+    goto done;
+  }
+
+  /* Get RENDEZVOUS1 keys */
+  retval = hs_ntor_service_get_rendezvous1_keys(&intro_auth_pubkey,
+                                               &intro_enc_keypair,
+                                               &service_ephemeral_rend_keypair,
+                                               &client_ephemeral_enc_pubkey,
+                                               &hs_ntor_rend_cell_keys);
+  if (retval < 0) {
+    goto done;
+  }
+
+  /* Send ENC_KEY */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_intro_cell_keys.enc_key,
+                sizeof(hs_ntor_intro_cell_keys.enc_key));
+  printf("%s\n", buf);
+  /* Send MAC_KEY */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_intro_cell_keys.mac_key,
+                sizeof(hs_ntor_intro_cell_keys.mac_key));
+  printf("%s\n", buf);
+  /* Send AUTH_MAC */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_rend_cell_keys.rend_cell_auth_mac,
+                sizeof(hs_ntor_rend_cell_keys.rend_cell_auth_mac));
+  printf("%s\n", buf);
+  /* Send NTOR_KEY_SEED */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_rend_cell_keys.ntor_key_seed,
+                sizeof(hs_ntor_rend_cell_keys.ntor_key_seed));
+  printf("%s\n", buf);
+  /* Send service ephemeral pubkey (Y) */
+  base16_encode(buf, sizeof(buf),
+                (const char*)service_ephemeral_rend_keypair.pubkey.public_key,
+                sizeof(service_ephemeral_rend_keypair.pubkey.public_key));
+  printf("%s\n", buf);
+
+ done:
+  return retval;
+}
+
+/** The final step of the ntor protocol, the client computes and returns the
+ *  rendezvous key material. */
+static int
+client2(int argc, char **argv)
+{
+  int retval;
+
+  /* Inputs */
+  curve25519_public_key_t intro_enc_pubkey;
+  ed25519_public_key_t intro_auth_pubkey;
+  curve25519_keypair_t client_ephemeral_enc_keypair;
+  curve25519_public_key_t service_ephemeral_rend_pubkey;
+  uint8_t subcredential[DIGEST256_LEN];
+
+  /* Output */
+  hs_ntor_rend_cell_keys_t hs_ntor_rend_cell_keys;
+
+  char buf[256];
+
+  N_ARGS(7);
+  BASE16(2, intro_auth_pubkey.pubkey, ED25519_PUBKEY_LEN);
+  BASE16(3, client_ephemeral_enc_keypair.seckey.secret_key,
+         CURVE25519_SECKEY_LEN);
+  BASE16(4, intro_enc_pubkey.public_key, CURVE25519_PUBKEY_LEN);
+  BASE16(5, service_ephemeral_rend_pubkey.public_key, CURVE25519_PUBKEY_LEN);
+  BASE16(6, subcredential, DIGEST256_LEN);
+
+  /* Generate keypair */
+  curve25519_public_key_generate(&client_ephemeral_enc_keypair.pubkey,
+                                 &client_ephemeral_enc_keypair.seckey);
+
+  /* Get RENDEZVOUS1 keys */
+  retval = hs_ntor_client_get_rendezvous1_keys(&intro_auth_pubkey,
+                                               &client_ephemeral_enc_keypair,
+                                               &intro_enc_pubkey,
+                                               &service_ephemeral_rend_pubkey,
+                                               &hs_ntor_rend_cell_keys);
+  if (retval < 0) {
+    goto done;
+  }
+
+  /* Send AUTH_MAC */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_rend_cell_keys.rend_cell_auth_mac,
+                sizeof(hs_ntor_rend_cell_keys.rend_cell_auth_mac));
+  printf("%s\n", buf);
+  /* Send NTOR_KEY_SEED */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_rend_cell_keys.ntor_key_seed,
+                sizeof(hs_ntor_rend_cell_keys.ntor_key_seed));
+  printf("%s\n", buf);
+
+ done:
+  return 1;
+}
+
+/** Perform a different part of the protocol depdning on the argv used. */
+int
+main(int argc, char **argv)
+{
+  if (argc < 2) {
+    fprintf(stderr, "I need arguments. Read source for more info.\n");
+    return 1;
+  }
+
+  init_logging(1);
+  curve25519_init();
+  if (crypto_global_init(0, NULL, NULL) < 0)
+    return 1;
+
+  if (!strcmp(argv[1], "client1")) {
+    return client1(argc, argv);
+  } else if (!strcmp(argv[1], "server1")) {
+    return server1(argc, argv);
+  } else if (!strcmp(argv[1], "client2")) {
+    return client2(argc, argv);
+  } else {
+    fprintf(stderr, "What's a %s?\n", argv[1]);
+    return 1;
+  }
+}
diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c
new file mode 100644
index 0000000000..32b08ecf37
--- /dev/null
+++ b/src/test/test_hs_service.c
@@ -0,0 +1,2145 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_service.c
+ * \brief Test hidden service functionality.
+ */
+
+#define CIRCUITBUILD_PRIVATE
+#define CIRCUITLIST_PRIVATE
+#define CONFIG_PRIVATE
+#define CONNECTION_PRIVATE
+#define CONNECTION_EDGE_PRIVATE
+#define CRYPTO_PRIVATE
+#define HS_COMMON_PRIVATE
+#define HS_SERVICE_PRIVATE
+#define HS_INTROPOINT_PRIVATE
+#define HS_CIRCUIT_PRIVATE
+#define MAINLOOP_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+#define STATEFILE_PRIVATE
+#define TOR_CHANNEL_INTERNAL_
+#define HS_CLIENT_PRIVATE
+
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
+#include "test/rend_test_helpers.h"
+#include "test/hs_test_helpers.h"
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "app/config/statefile.h"
+#include "core/crypto/hs_ntor.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/connection_edge.h"
+#include "core/or/edge_connection_st.h"
+#include "core/or/relay.h"
+#include "core/or/versions.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/shared_random_state.h"
+#include "feature/dircommon/voting_schedule.h"
+#include "feature/hs/hs_circuit.h"
+#include "feature/hs/hs_circuitmap.h"
+#include "feature/hs/hs_client.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_config.h"
+#include "feature/hs/hs_ident.h"
+#include "feature/hs/hs_intropoint.h"
+#include "feature/hs/hs_service.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/rend/rendservice.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/fs/dir.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/crypt_path_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "app/config/or_state_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+
+/* Trunnel */
+#include "trunnel/hs/cell_establish_intro.h"
+
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+static networkstatus_t mock_ns;
+
+static networkstatus_t *
+mock_networkstatus_get_live_consensus(time_t now)
+{
+  (void) now;
+  return &mock_ns;
+}
+
+static or_state_t *dummy_state = NULL;
+
+/* Mock function to get fake or state (used for rev counters) */
+static or_state_t *
+get_or_state_replacement(void)
+{
+  return dummy_state;
+}
+
+/* Mock function because we are not trying to test the close circuit that does
+ * an awful lot of checks on the circuit object. */
+static void
+mock_circuit_mark_for_close(circuit_t *circ, int reason, int line,
+                            const char *file)
+{
+  (void) circ;
+  (void) reason;
+  (void) line;
+  (void) file;
+  return;
+}
+
+static int
+mock_relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ,
+                                  uint8_t relay_command, const char *payload,
+                                  size_t payload_len,
+                                  crypt_path_t *cpath_layer,
+                                  const char *filename, int lineno)
+{
+  (void) stream_id;
+  (void) circ;
+  (void) relay_command;
+  (void) payload;
+  (void) payload_len;
+  (void) cpath_layer;
+  (void) filename;
+  (void) lineno;
+  return 0;
+}
+
+/* Helper: from a set of options in conf, configure a service which will add
+ * it to the staging list of the HS subsytem. */
+static int
+helper_config_service(const char *conf)
+{
+  int ret = 0;
+  or_options_t *options = NULL;
+  tt_assert(conf);
+  options = helper_parse_options(conf);
+  tt_assert(options);
+  ret = hs_config_service_all(options, 0);
+ done:
+  or_options_free(options);
+  return ret;
+}
+
+/* Test: Ensure that setting up rendezvous circuits works correctly. */
+static void
+test_e2e_rend_circuit_setup(void *arg)
+{
+  ed25519_public_key_t service_pk;
+  origin_circuit_t *or_circ;
+  int retval;
+
+  /** In this test we create a v3 prop224 service-side rendezvous circuit.
+   *  We simulate an HS ntor key exchange with a client, and check that
+   *  the circuit was setup correctly and is ready to accept rendezvous data */
+
+  (void) arg;
+
+  /* Now make dummy circuit */
+  {
+    or_circ = origin_circuit_new();
+
+    or_circ->base_.purpose = CIRCUIT_PURPOSE_S_CONNECT_REND;
+
+    or_circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+    or_circ->build_state->is_internal = 1;
+
+    /* prop224: Setup hs conn identifier on the stream */
+    ed25519_secret_key_t sk;
+    tt_int_op(0, OP_EQ, ed25519_secret_key_generate(&sk, 0));
+    tt_int_op(0, OP_EQ, ed25519_public_key_generate(&service_pk, &sk));
+
+    or_circ->hs_ident = hs_ident_circuit_new(&service_pk,
+                                             HS_IDENT_CIRCUIT_RENDEZVOUS);
+
+    TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN;
+  }
+
+  /* Check number of hops */
+  retval = cpath_get_n_hops(&or_circ->cpath);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Setup the circuit: do the ntor key exchange */
+  {
+    uint8_t ntor_key_seed[DIGEST256_LEN] = {2};
+    retval = hs_circuit_setup_e2e_rend_circ(or_circ,
+                                       ntor_key_seed, sizeof(ntor_key_seed),
+                                       1);
+    tt_int_op(retval, OP_EQ, 0);
+  }
+
+  /* See that a hop was added to the circuit's cpath */
+  retval = cpath_get_n_hops(&or_circ->cpath);
+  tt_int_op(retval, OP_EQ, 1);
+
+  /* Check the digest algo */
+  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest),
+            OP_EQ, DIGEST_SHA3_256);
+  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest),
+            OP_EQ, DIGEST_SHA3_256);
+  tt_assert(or_circ->cpath->crypto.f_crypto);
+  tt_assert(or_circ->cpath->crypto.b_crypto);
+
+  /* Ensure that circ purpose was changed */
+  tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_S_REND_JOINED);
+
+ done:
+  circuit_free_(TO_CIRCUIT(or_circ));
+}
+
+/* Helper: Return a newly allocated and initialized origin circuit with
+ * purpose and flags. A default HS identifier is set to an ed25519
+ * authentication key for introduction point. */
+static origin_circuit_t *
+helper_create_origin_circuit(int purpose, int flags)
+{
+  origin_circuit_t *circ = NULL;
+
+  circ = origin_circuit_init(purpose, flags);
+  tor_assert(circ);
+  circ->cpath = tor_malloc_zero(sizeof(crypt_path_t));
+  circ->cpath->magic = CRYPT_PATH_MAGIC;
+  circ->cpath->state = CPATH_STATE_OPEN;
+  circ->cpath->package_window = circuit_initial_package_window();
+  circ->cpath->deliver_window = CIRCWINDOW_START;
+  circ->cpath->prev = circ->cpath;
+  /* Random nonce. */
+  crypto_rand(circ->cpath->prev->rend_circ_nonce, DIGEST_LEN);
+  /* Create a default HS identifier. */
+  circ->hs_ident = tor_malloc_zero(sizeof(hs_ident_circuit_t));
+
+  return circ;
+}
+
+/* Helper: Return a newly allocated authorized client object with
+ * and a newly generated public key. */
+static hs_service_authorized_client_t *
+helper_create_authorized_client(void)
+{
+  int ret;
+  hs_service_authorized_client_t *client;
+  curve25519_secret_key_t seckey;
+  client = tor_malloc_zero(sizeof(hs_service_authorized_client_t));
+
+  ret = curve25519_secret_key_generate(&seckey, 0);
+  tt_int_op(ret, OP_EQ, 0);
+  curve25519_public_key_generate(&client->client_pk, &seckey);
+
+ done:
+  return client;
+}
+
+/* Helper: Return a newly allocated authorized client object with the
+ * same client name and the same public key as the given client. */
+static hs_service_authorized_client_t *
+helper_clone_authorized_client(const hs_service_authorized_client_t *client)
+{
+  hs_service_authorized_client_t *client_out;
+
+  tor_assert(client);
+
+  client_out = tor_malloc_zero(sizeof(hs_service_authorized_client_t));
+  memcpy(client_out->client_pk.public_key,
+         client->client_pk.public_key, CURVE25519_PUBKEY_LEN);
+
+  return client_out;
+}
+
+/* Helper: Return a newly allocated service object with the identity keypair
+ * sets and the current descriptor. Then register it to the global map.
+ * Caller should us hs_free_all() to free this service or remove it from the
+ * global map before freeing. */
+static hs_service_t *
+helper_create_service(void)
+{
+  /* Set a service for this circuit. */
+  hs_service_t *service = hs_service_new(get_options());
+  tor_assert(service);
+  service->config.version = HS_VERSION_THREE;
+  ed25519_secret_key_generate(&service->keys.identity_sk, 0);
+  ed25519_public_key_generate(&service->keys.identity_pk,
+                              &service->keys.identity_sk);
+  service->desc_current = service_descriptor_new();
+  tt_assert(service->desc_current);
+  /* Register service to global map. */
+  int ret = register_service(get_hs_service_map(), service);
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  return service;
+}
+
+/* Helper: Return a newly allocated service object with clients. */
+static hs_service_t *
+helper_create_service_with_clients(int num_clients)
+{
+  int i;
+  hs_service_t *service = helper_create_service();
+  tt_assert(service);
+  service->config.is_client_auth_enabled = 1;
+  service->config.clients = smartlist_new();
+
+  for (i = 0; i < num_clients; i++) {
+    hs_service_authorized_client_t *client;
+    client = helper_create_authorized_client();
+    smartlist_add(service->config.clients, client);
+  }
+
+ done:
+  return service;
+}
+
+/* Helper: Return a newly allocated service intro point with two link
+ * specifiers, one IPv4 and one legacy ID set to As. */
+static hs_service_intro_point_t *
+helper_create_service_ip(void)
+{
+  hs_desc_link_specifier_t *ls;
+  hs_service_intro_point_t *ip = service_intro_point_new(NULL, 0, 0);
+  tor_assert(ip);
+  /* Add a first unused link specifier. */
+  ls = tor_malloc_zero(sizeof(*ls));
+  ls->type = LS_IPV4;
+  smartlist_add(ip->base.link_specifiers, ls);
+  /* Add a second link specifier used by a test. */
+  ls = tor_malloc_zero(sizeof(*ls));
+  ls->type = LS_LEGACY_ID;
+  memset(ls->u.legacy_id, 'A', sizeof(ls->u.legacy_id));
+  smartlist_add(ip->base.link_specifiers, ls);
+
+  return ip;
+}
+
+static void
+test_load_keys(void *arg)
+{
+  int ret;
+  char *conf = NULL;
+  char *hsdir_v2 = tor_strdup(get_fname("hs2"));
+  char *hsdir_v3 = tor_strdup(get_fname("hs3"));
+  char addr[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+
+  (void) arg;
+
+  /* We'll register two services, a v2 and a v3, then we'll load keys and
+   * validate that both are in a correct state. */
+
+  hs_init();
+
+#define conf_fmt \
+  "HiddenServiceDir %s\n" \
+  "HiddenServiceVersion %d\n" \
+  "HiddenServicePort 65535\n"
+
+  /* v2 service. */
+  tor_asprintf(&conf, conf_fmt, hsdir_v2, HS_VERSION_TWO);
+  ret = helper_config_service(conf);
+  tor_free(conf);
+  tt_int_op(ret, OP_EQ, 0);
+  /* This one should now be registered into the v2 list. */
+  tt_int_op(get_hs_service_staging_list_size(), OP_EQ, 0);
+  tt_int_op(rend_num_services(), OP_EQ, 1);
+
+  /* v3 service. */
+  tor_asprintf(&conf, conf_fmt, hsdir_v3, HS_VERSION_THREE);
+  ret = helper_config_service(conf);
+  tor_free(conf);
+  tt_int_op(ret, OP_EQ, 0);
+  /* It's in staging? */
+  tt_int_op(get_hs_service_staging_list_size(), OP_EQ, 1);
+
+#undef conf_fmt
+
+  /* Load the keys for these. After that, the v3 service should be registered
+   * in the global map. */
+  hs_service_load_all_keys();
+  tt_int_op(get_hs_service_map_size(), OP_EQ, 1);
+  hs_service_t *s = get_first_service();
+  tt_assert(s);
+
+  /* Ok we have the service object. Validate few things. */
+  tt_assert(!tor_mem_is_zero(s->onion_address, sizeof(s->onion_address)));
+  tt_int_op(hs_address_is_valid(s->onion_address), OP_EQ, 1);
+  tt_assert(!tor_mem_is_zero((char *) s->keys.identity_sk.seckey,
+                             ED25519_SECKEY_LEN));
+  tt_assert(!tor_mem_is_zero((char *) s->keys.identity_pk.pubkey,
+                             ED25519_PUBKEY_LEN));
+  /* Check onion address from identity key. */
+  hs_build_address(&s->keys.identity_pk, s->config.version, addr);
+  tt_int_op(hs_address_is_valid(addr), OP_EQ, 1);
+  tt_str_op(addr, OP_EQ, s->onion_address);
+
+  /* Check that the is_client_auth_enabled is not set. */
+  tt_assert(!s->config.is_client_auth_enabled);
+
+ done:
+  tor_free(hsdir_v2);
+  tor_free(hsdir_v3);
+  hs_free_all();
+}
+
+static void
+test_client_filename_is_valid(void *arg)
+{
+  (void) arg;
+
+  /* Valid file name. */
+  tt_assert(client_filename_is_valid("a.auth"));
+  /* Valid file name with special character. */
+  tt_assert(client_filename_is_valid("a-.auth"));
+  /* Invalid extension. */
+  tt_assert(!client_filename_is_valid("a.ath"));
+  /* Nothing before the extension. */
+  tt_assert(!client_filename_is_valid(".auth"));
+
+ done:
+  ;
+}
+
+static void
+test_parse_authorized_client(void *arg)
+{
+  hs_service_authorized_client_t *client = NULL;
+
+  (void) arg;
+
+  /* Valid authorized client. */
+  client = parse_authorized_client(
+    "descriptor:x25519:dz4q5xqlb4ldnbs72iarrml4ephk3du4i7o2cgiva5lwr6wkquja");
+  tt_assert(client);
+
+  /* Wrong number of fields. */
+  tt_assert(!parse_authorized_client("a:b:c:d:e"));
+  /* Wrong auth type. */
+  tt_assert(!parse_authorized_client(
+    "x:x25519:dz4q5xqlb4ldnbs72iarrml4ephk3du4i7o2cgiva5lwr6wkquja"));
+  /* Wrong key type. */
+  tt_assert(!parse_authorized_client(
+    "descriptor:x:dz4q5xqlb4ldnbs72iarrml4ephk3du4i7o2cgiva5lwr6wkquja"));
+  /* Some malformed string. */
+  tt_assert(!parse_authorized_client("descriptor:x25519:aa=="));
+  tt_assert(!parse_authorized_client("descriptor:"));
+  tt_assert(!parse_authorized_client("descriptor:x25519"));
+  tt_assert(!parse_authorized_client("descriptor:x25519:"));
+  tt_assert(!parse_authorized_client(""));
+
+ done:
+  service_authorized_client_free(client);
+}
+
+static char *
+mock_read_file_to_str(const char *filename, int flags, struct stat *stat_out)
+{
+  char *ret = NULL;
+
+  (void) flags;
+  (void) stat_out;
+
+  if (!strcmp(filename, get_fname("hs3" PATH_SEPARATOR
+                                  "authorized_clients" PATH_SEPARATOR
+                                  "client1.auth"))) {
+    ret = tor_strdup("descriptor:x25519:"
+                  "dz4q5xqlb4ldnbs72iarrml4ephk3du4i7o2cgiva5lwr6wkquja");
+    goto done;
+  }
+
+  if (!strcmp(filename, get_fname("hs3" PATH_SEPARATOR
+                                  "authorized_clients" PATH_SEPARATOR
+                                  "dummy.xxx"))) {
+    ret = tor_strdup("descriptor:x25519:"
+                  "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");
+    goto done;
+  }
+
+  if (!strcmp(filename, get_fname("hs3" PATH_SEPARATOR
+                                  "authorized_clients" PATH_SEPARATOR
+                                  "client2.auth"))) {
+    ret = tor_strdup("descriptor:x25519:"
+                  "okoi2gml3wd6x7jganlk5d66xxyjgg24sxw4y7javx4giqr66zta");
+    goto done;
+  }
+
+ done:
+  return ret;
+}
+
+static smartlist_t *
+mock_tor_listdir(const char *dirname)
+{
+  smartlist_t *file_list = smartlist_new();
+
+  (void) dirname;
+
+  smartlist_add(file_list, tor_strdup("client1.auth"));
+  smartlist_add(file_list, tor_strdup("dummy.xxx"));
+  smartlist_add(file_list, tor_strdup("client2.auth"));
+
+  return file_list;
+}
+
+static void
+test_load_keys_with_client_auth(void *arg)
+{
+  int ret;
+  char *conf = NULL;
+  smartlist_t *pubkey_b32_list = smartlist_new();
+  char *hsdir_v3 = tor_strdup(get_fname("hs3"));
+  hs_service_t *service;
+
+  (void) arg;
+
+  hs_init();
+  smartlist_add(pubkey_b32_list, tor_strdup(
+                "dz4q5xqlb4ldnbs72iarrml4ephk3du4i7o2cgiva5lwr6wkquja"));
+  smartlist_add(pubkey_b32_list, tor_strdup(
+                "okoi2gml3wd6x7jganlk5d66xxyjgg24sxw4y7javx4giqr66zta"));
+
+#define conf_fmt \
+  "HiddenServiceDir %s\n" \
+  "HiddenServiceVersion %d\n" \
+  "HiddenServicePort 65534\n"
+
+  tor_asprintf(&conf, conf_fmt, hsdir_v3, HS_VERSION_THREE);
+  ret = helper_config_service(conf);
+  tor_free(conf);
+  tt_int_op(ret, OP_EQ, 0);
+  /* It's in staging? */
+  tt_int_op(get_hs_service_staging_list_size(), OP_EQ, 1);
+
+#undef conf_fmt
+
+  MOCK(read_file_to_str, mock_read_file_to_str);
+  MOCK(tor_listdir, mock_tor_listdir);
+
+  /* Load the keys for these. After that, the v3 service should be registered
+   * in the global map. */
+  hs_service_load_all_keys();
+  tt_int_op(get_hs_service_map_size(), OP_EQ, 1);
+
+  service = get_first_service();
+  tt_assert(service);
+  tt_assert(service->config.clients);
+  tt_int_op(smartlist_len(service->config.clients), OP_EQ,
+            smartlist_len(pubkey_b32_list));
+
+  /* Test that the is_client_auth_enabled flag is set. */
+  tt_assert(service->config.is_client_auth_enabled);
+
+  /* Test that the keys in clients are correct. */
+  SMARTLIST_FOREACH_BEGIN(pubkey_b32_list, char *, pubkey_b32) {
+
+    curve25519_public_key_t pubkey;
+    /* This flag will be set if the key is found in clients. */
+    int is_found = 0;
+    base32_decode((char *) pubkey.public_key, sizeof(pubkey.public_key),
+                  pubkey_b32, strlen(pubkey_b32));
+
+    SMARTLIST_FOREACH_BEGIN(service->config.clients,
+                            hs_service_authorized_client_t *, client) {
+      if (tor_memeq(&pubkey, &client->client_pk, sizeof(pubkey))) {
+        is_found = 1;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(client);
+
+    tt_assert(is_found);
+
+  } SMARTLIST_FOREACH_END(pubkey_b32);
+
+ done:
+  SMARTLIST_FOREACH(pubkey_b32_list, char *, s, tor_free(s));
+  smartlist_free(pubkey_b32_list);
+  tor_free(hsdir_v3);
+  hs_free_all();
+  UNMOCK(read_file_to_str);
+  UNMOCK(tor_listdir);
+}
+
+static void
+test_access_service(void *arg)
+{
+  int ret;
+  char *conf = NULL;
+  char *hsdir_v3 = tor_strdup(get_fname("hs3"));
+  hs_service_ht *global_map;
+  hs_service_t *s = NULL;
+
+  (void) arg;
+
+  /* We'll register two services, a v2 and a v3, then we'll load keys and
+   * validate that both are in a correct state. */
+
+  hs_init();
+
+#define conf_fmt \
+  "HiddenServiceDir %s\n" \
+  "HiddenServiceVersion %d\n" \
+  "HiddenServicePort 65535\n"
+
+  /* v3 service. */
+  tor_asprintf(&conf, conf_fmt, hsdir_v3, HS_VERSION_THREE);
+  ret = helper_config_service(conf);
+  tor_free(conf);
+  tt_int_op(ret, OP_EQ, 0);
+  /* It's in staging? */
+  tt_int_op(get_hs_service_staging_list_size(), OP_EQ, 1);
+
+  /* Load the keys for these. After that, the v3 service should be registered
+   * in the global map. */
+  hs_service_load_all_keys();
+  tt_int_op(get_hs_service_map_size(), OP_EQ, 1);
+  s = get_first_service();
+  tt_assert(s);
+  global_map = get_hs_service_map();
+  tt_assert(global_map);
+
+  /* From here, we'll try the service accessors. */
+  hs_service_t *query = find_service(global_map, &s->keys.identity_pk);
+  tt_assert(query);
+  tt_mem_op(query, OP_EQ, s, sizeof(hs_service_t));
+  /* Remove service, check if it actually works and then put it back. */
+  remove_service(global_map, s);
+  tt_int_op(get_hs_service_map_size(), OP_EQ, 0);
+  query = find_service(global_map, &s->keys.identity_pk);
+  tt_ptr_op(query, OP_EQ, NULL);
+
+  /* Register back the service in the map. */
+  ret = register_service(global_map, s);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(get_hs_service_map_size(), OP_EQ, 1);
+  /* Twice should fail. */
+  ret = register_service(global_map, s);
+  tt_int_op(ret, OP_EQ, -1);
+  /* Remove service from map so we don't double free on cleanup. */
+  remove_service(global_map, s);
+  tt_int_op(get_hs_service_map_size(), OP_EQ, 0);
+  query = find_service(global_map, &s->keys.identity_pk);
+  tt_ptr_op(query, OP_EQ, NULL);
+  /* Let's try to remove twice for fun. */
+  setup_full_capture_of_logs(LOG_WARN);
+  remove_service(global_map, s);
+  expect_log_msg_containing("Could not find service in the global map");
+  teardown_capture_of_logs();
+
+ done:
+  hs_service_free(s);
+  tor_free(hsdir_v3);
+  hs_free_all();
+}
+
+/** Test that we can create intro point objects, index them and find them */
+static void
+test_service_intro_point(void *arg)
+{
+  hs_service_t *service = NULL;
+  hs_service_intro_point_t *ip = NULL;
+
+  (void) arg;
+
+  /* Test simple creation of an object. */
+  {
+    time_t now = time(NULL);
+    ip = helper_create_service_ip();
+    tt_assert(ip);
+    /* Make sure the authentication keypair is not zeroes. */
+    tt_int_op(tor_mem_is_zero((const char *) &ip->auth_key_kp,
+                              sizeof(ed25519_keypair_t)), OP_EQ, 0);
+    /* The introduce2_max MUST be in that range. */
+    tt_u64_op(ip->introduce2_max, OP_GE,
+              INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS);
+    tt_u64_op(ip->introduce2_max, OP_LE,
+              INTRO_POINT_MAX_LIFETIME_INTRODUCTIONS);
+    /* Time to expire MUST also be in that range. We subtract 500 seconds
+     * because there could be a gap between setting now and the time taken in
+     * service_intro_point_new. On ARM and other older CPUs, it can be
+     * surprisingly slow... */
+    tt_u64_op(ip->time_to_expire, OP_GE,
+              now + INTRO_POINT_LIFETIME_MIN_SECONDS - 500);
+    /* We add 500 seconds, because this time we're testing against the
+     * maximum allowed time. */
+    tt_u64_op(ip->time_to_expire, OP_LE,
+              now + INTRO_POINT_LIFETIME_MAX_SECONDS + 500);
+    tt_assert(ip->replay_cache);
+    tt_assert(ip->base.link_specifiers);
+    /* By default, this is NOT a legacy object. */
+    tt_int_op(ip->base.is_only_legacy, OP_EQ, 0);
+  }
+
+  /* Test functions that uses a service intropoints map with that previously
+   * created object (non legacy). */
+  {
+    ed25519_public_key_t garbage = { {0} };
+    hs_service_intro_point_t *query;
+
+    service = hs_service_new(get_options());
+    tt_assert(service);
+    service->desc_current = service_descriptor_new();
+    tt_assert(service->desc_current);
+    /* Add intropoint to descriptor map. */
+    service_intro_point_add(service->desc_current->intro_points.map, ip);
+    query = service_intro_point_find(service, &ip->auth_key_kp.pubkey);
+    tt_mem_op(query, OP_EQ, ip, sizeof(hs_service_intro_point_t));
+    query = service_intro_point_find(service, &garbage);
+    tt_ptr_op(query, OP_EQ, NULL);
+
+    /* While at it, can I find the descriptor with the intro point? */
+    hs_service_descriptor_t *desc_lookup =
+      service_desc_find_by_intro(service, ip);
+    tt_mem_op(service->desc_current, OP_EQ, desc_lookup,
+              sizeof(hs_service_descriptor_t));
+
+    /* Remove object from service descriptor and make sure it is out. */
+    service_intro_point_remove(service, ip);
+    query = service_intro_point_find(service, &ip->auth_key_kp.pubkey);
+    tt_ptr_op(query, OP_EQ, NULL);
+  }
+
+ done:
+  /* If the test succeed, this object is no longer referenced in the service
+   * so we can free it without use after free. Else, it might explode because
+   * it's still in the service descriptor map. */
+  service_intro_point_free(ip);
+  hs_service_free(service);
+}
+
+static node_t mock_node;
+static const node_t *
+mock_node_get_by_id(const char *digest)
+{
+  (void) digest;
+  memset(mock_node.identity, 'A', DIGEST_LEN);
+  /* Only return the matchin identity of As */
+  if (!tor_memcmp(mock_node.identity, digest, DIGEST_LEN)) {
+    return &mock_node;
+  }
+  return NULL;
+}
+
+static void
+test_helper_functions(void *arg)
+{
+  int ret;
+  hs_service_t *service = NULL;
+  hs_service_intro_point_t *ip = NULL;
+  hs_ident_circuit_t ident;
+
+  (void) arg;
+
+  MOCK(node_get_by_id, mock_node_get_by_id);
+
+  hs_service_init();
+  time_t now = time(NULL);
+  update_approx_time(now);
+
+  service = helper_create_service();
+
+  ip = helper_create_service_ip();
+  /* Immediately add the intro point to the service so the free service at the
+   * end cleans it as well. */
+  service_intro_point_add(service->desc_current->intro_points.map, ip);
+
+  /* Setup the circuit identifier. */
+  ed25519_pubkey_copy(&ident.intro_auth_pk, &ip->auth_key_kp.pubkey);
+  ed25519_pubkey_copy(&ident.identity_pk, &service->keys.identity_pk);
+
+  /* Testing get_objects_from_ident(). */
+  {
+    hs_service_t *s_lookup = NULL;
+    hs_service_intro_point_t *ip_lookup = NULL;
+    hs_service_descriptor_t *desc_lookup = NULL;
+
+    get_objects_from_ident(&ident, &s_lookup, &ip_lookup, &desc_lookup);
+    tt_mem_op(s_lookup, OP_EQ, service, sizeof(hs_service_t));
+    tt_mem_op(ip_lookup, OP_EQ, ip, sizeof(hs_service_intro_point_t));
+    tt_mem_op(desc_lookup, OP_EQ, service->desc_current,
+              sizeof(hs_service_descriptor_t));
+    /* Reset */
+    s_lookup = NULL; ip_lookup = NULL; desc_lookup = NULL;
+
+    /* NULL parameter should work. */
+    get_objects_from_ident(&ident, NULL, &ip_lookup, &desc_lookup);
+    tt_mem_op(ip_lookup, OP_EQ, ip, sizeof(hs_service_intro_point_t));
+    tt_mem_op(desc_lookup, OP_EQ, service->desc_current,
+              sizeof(hs_service_descriptor_t));
+    /* Reset. */
+    s_lookup = NULL; ip_lookup = NULL; desc_lookup = NULL;
+
+    /* Break the ident and we should find nothing. */
+    memset(&ident, 0, sizeof(ident));
+    get_objects_from_ident(&ident, &s_lookup, &ip_lookup, &desc_lookup);
+    tt_ptr_op(s_lookup, OP_EQ, NULL);
+    tt_ptr_op(ip_lookup, OP_EQ, NULL);
+    tt_ptr_op(desc_lookup, OP_EQ, NULL);
+  }
+
+  /* Testing get_node_from_intro_point() */
+  {
+    const node_t *node = get_node_from_intro_point(ip);
+    tt_ptr_op(node, OP_EQ, &mock_node);
+    SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
+                            hs_desc_link_specifier_t *, ls) {
+      if (ls->type == LS_LEGACY_ID) {
+        /* Change legacy id in link specifier which is not the mock node. */
+        memset(ls->u.legacy_id, 'B', sizeof(ls->u.legacy_id));
+      }
+    } SMARTLIST_FOREACH_END(ls);
+    node = get_node_from_intro_point(ip);
+    tt_ptr_op(node, OP_EQ, NULL);
+  }
+
+  /* Testing can_service_launch_intro_circuit() */
+  {
+    /* Put the start of the retry period back in time, we should be allowed.
+     * to launch intro circuit. */
+    service->state.num_intro_circ_launched = 2;
+    service->state.intro_circ_retry_started_time =
+      (now - INTRO_CIRC_RETRY_PERIOD - 1);
+    ret = can_service_launch_intro_circuit(service, now);
+    tt_int_op(ret, OP_EQ, 1);
+    tt_u64_op(service->state.intro_circ_retry_started_time, OP_EQ, now);
+    tt_u64_op(service->state.num_intro_circ_launched, OP_EQ, 0);
+    /* Call it again, we should still be allowed because we are under
+     * MAX_INTRO_CIRCS_PER_PERIOD which been set to 0 previously. */
+    ret = can_service_launch_intro_circuit(service, now);
+    tt_int_op(ret, OP_EQ, 1);
+    tt_u64_op(service->state.intro_circ_retry_started_time, OP_EQ, now);
+    tt_u64_op(service->state.num_intro_circ_launched, OP_EQ, 0);
+    /* Too many intro circuit launched means we are not allowed. */
+    service->state.num_intro_circ_launched = 20;
+    ret = can_service_launch_intro_circuit(service, now);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Testing intro_point_should_expire(). */
+  {
+    /* Just some basic test of the current state. */
+    tt_u64_op(ip->introduce2_max, OP_GE,
+              INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS);
+    tt_u64_op(ip->introduce2_max, OP_LE,
+              INTRO_POINT_MAX_LIFETIME_INTRODUCTIONS);
+    tt_u64_op(ip->time_to_expire, OP_GE,
+              now + INTRO_POINT_LIFETIME_MIN_SECONDS);
+    tt_u64_op(ip->time_to_expire, OP_LE,
+              now + INTRO_POINT_LIFETIME_MAX_SECONDS);
+
+    /* This newly created IP from above shouldn't expire now. */
+    ret = intro_point_should_expire(ip, now);
+    tt_int_op(ret, OP_EQ, 0);
+    /* Maximum number of INTRODUCE2 cell reached, it should expire. */
+    ip->introduce2_count = INTRO_POINT_MAX_LIFETIME_INTRODUCTIONS + 1;
+    ret = intro_point_should_expire(ip, now);
+    tt_int_op(ret, OP_EQ, 1);
+    ip->introduce2_count = 0;
+    /* It should expire if time to expire has been reached. */
+    ip->time_to_expire = now - 1000;
+    ret = intro_point_should_expire(ip, now);
+    tt_int_op(ret, OP_EQ, 1);
+  }
+
+ done:
+  /* This will free the service and all objects associated to it. */
+  hs_service_free_all();
+  UNMOCK(node_get_by_id);
+}
+
+/** Test that we do the right operations when an intro circuit opens */
+static void
+test_intro_circuit_opened(void *arg)
+{
+  int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
+  hs_service_t *service;
+  origin_circuit_t *circ = NULL;
+
+  (void) arg;
+
+  hs_init();
+  MOCK(circuit_mark_for_close_, mock_circuit_mark_for_close);
+  MOCK(relay_send_command_from_edge_, mock_relay_send_command_from_edge);
+
+  circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO,
+                                      flags);
+
+  /* No service associated with this circuit. */
+  setup_full_capture_of_logs(LOG_WARN);
+  hs_service_circuit_has_opened(circ);
+  expect_log_msg_containing("Unknown service identity key");
+  teardown_capture_of_logs();
+
+  /* Set a service for this circuit. */
+  {
+    service = helper_create_service();
+    ed25519_pubkey_copy(&circ->hs_ident->identity_pk,
+                        &service->keys.identity_pk);
+
+    /* No intro point associated with this circuit. */
+    setup_full_capture_of_logs(LOG_WARN);
+    hs_service_circuit_has_opened(circ);
+    expect_log_msg_containing("Unknown introduction point auth key");
+    teardown_capture_of_logs();
+  }
+
+  /* Set an IP object now for this circuit. */
+  {
+    hs_service_intro_point_t *ip = helper_create_service_ip();
+    service_intro_point_add(service->desc_current->intro_points.map, ip);
+    /* Update ident to contain the intro point auth key. */
+    ed25519_pubkey_copy(&circ->hs_ident->intro_auth_pk,
+                        &ip->auth_key_kp.pubkey);
+  }
+
+  /* This one should go all the way. */
+  setup_full_capture_of_logs(LOG_INFO);
+  hs_service_circuit_has_opened(circ);
+  expect_log_msg_containing("Introduction circuit 0 established for service");
+  teardown_capture_of_logs();
+
+ done:
+  circuit_free_(TO_CIRCUIT(circ));
+  hs_free_all();
+  UNMOCK(circuit_mark_for_close_);
+  UNMOCK(relay_send_command_from_edge_);
+}
+
+/** Test the operations we do on a circuit after we learn that we successfully
+ *  established an intro point on it */
+static void
+test_intro_established(void *arg)
+{
+  int ret;
+  int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
+  uint8_t payload[RELAY_PAYLOAD_SIZE] = {0};
+  origin_circuit_t *circ = NULL;
+  hs_service_t *service;
+  hs_service_intro_point_t *ip = NULL;
+
+  (void) arg;
+
+  hs_init();
+  MOCK(circuit_mark_for_close_, mock_circuit_mark_for_close);
+
+  circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO,
+                                      flags);
+  tt_assert(circ);
+
+  /* Test a wrong purpose. */
+  TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_S_INTRO;
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_service_receive_intro_established(circ, payload, sizeof(payload));
+  tt_int_op(ret, OP_EQ, -1);
+  expect_log_msg_containing("Received an INTRO_ESTABLISHED cell on a "
+                            "non introduction circuit of purpose");
+  teardown_capture_of_logs();
+
+  /* Back to normal. */
+  TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_S_ESTABLISH_INTRO;
+
+  /* No service associated to it. */
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_service_receive_intro_established(circ, payload, sizeof(payload));
+  tt_int_op(ret, OP_EQ, -1);
+  expect_log_msg_containing("Unknown service identity key");
+  teardown_capture_of_logs();
+
+  /* Set a service for this circuit. */
+  service = helper_create_service();
+  ed25519_pubkey_copy(&circ->hs_ident->identity_pk,
+                      &service->keys.identity_pk);
+  /* No introduction point associated to it. */
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_service_receive_intro_established(circ, payload, sizeof(payload));
+  tt_int_op(ret, OP_EQ, -1);
+  expect_log_msg_containing("Introduction circuit established without an "
+                            "intro point object on circuit");
+  teardown_capture_of_logs();
+
+  /* Set an IP object now for this circuit. */
+  {
+    ip = helper_create_service_ip();
+    service_intro_point_add(service->desc_current->intro_points.map, ip);
+    /* Update ident to contain the intro point auth key. */
+    ed25519_pubkey_copy(&circ->hs_ident->intro_auth_pk,
+                        &ip->auth_key_kp.pubkey);
+  }
+
+  /* Send an empty payload. INTRO_ESTABLISHED cells are basically zeroes. */
+  ret = hs_service_receive_intro_established(circ, payload, sizeof(payload));
+  tt_int_op(ret, OP_EQ, 0);
+  tt_u64_op(ip->circuit_established, OP_EQ, 1);
+  tt_int_op(TO_CIRCUIT(circ)->purpose, OP_EQ, CIRCUIT_PURPOSE_S_INTRO);
+
+ done:
+  if (circ)
+    circuit_free_(TO_CIRCUIT(circ));
+  hs_free_all();
+  UNMOCK(circuit_mark_for_close_);
+}
+
+/** Check the operations we do on a rendezvous circuit after we learn it's
+ *  open */
+static void
+test_rdv_circuit_opened(void *arg)
+{
+  int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
+  origin_circuit_t *circ = NULL;
+  hs_service_t *service;
+
+  (void) arg;
+
+  hs_init();
+  MOCK(circuit_mark_for_close_, mock_circuit_mark_for_close);
+  MOCK(relay_send_command_from_edge_, mock_relay_send_command_from_edge);
+
+  circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_S_CONNECT_REND, flags);
+  crypto_rand((char *) circ->hs_ident->rendezvous_cookie, REND_COOKIE_LEN);
+  crypto_rand((char *) circ->hs_ident->rendezvous_handshake_info,
+              sizeof(circ->hs_ident->rendezvous_handshake_info));
+
+  /* No service associated with this circuit. */
+  setup_full_capture_of_logs(LOG_WARN);
+  hs_service_circuit_has_opened(circ);
+  expect_log_msg_containing("Unknown service identity key");
+  teardown_capture_of_logs();
+  /* This should be set to a non zero timestamp. */
+  tt_u64_op(TO_CIRCUIT(circ)->timestamp_dirty, OP_NE, 0);
+
+  /* Set a service for this circuit. */
+  service = helper_create_service();
+  ed25519_pubkey_copy(&circ->hs_ident->identity_pk,
+                      &service->keys.identity_pk);
+  /* Should be all good. */
+  hs_service_circuit_has_opened(circ);
+  tt_int_op(TO_CIRCUIT(circ)->purpose, OP_EQ, CIRCUIT_PURPOSE_S_REND_JOINED);
+
+ done:
+  circuit_free_(TO_CIRCUIT(circ));
+  hs_free_all();
+  UNMOCK(circuit_mark_for_close_);
+  UNMOCK(relay_send_command_from_edge_);
+}
+
+static void
+mock_assert_circuit_ok(const circuit_t *c)
+{
+  (void) c;
+  return;
+}
+
+/** Test for the general mechanism for closing intro circs.
+ *  Also a way to identify that #23603 has been fixed. */
+static void
+test_closing_intro_circs(void *arg)
+{
+  hs_service_t *service = NULL;
+  hs_service_intro_point_t *ip = NULL, *entry = NULL;
+  origin_circuit_t *intro_circ = NULL, *tmp_circ;
+  int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
+
+  (void) arg;
+
+  MOCK(assert_circuit_ok, mock_assert_circuit_ok);
+
+  hs_init();
+
+  /* Initialize service */
+  service = helper_create_service();
+  /* Initialize intro point */
+  ip = helper_create_service_ip();
+  tt_assert(ip);
+  service_intro_point_add(service->desc_current->intro_points.map, ip);
+
+  /* Initialize intro circuit */
+  intro_circ = origin_circuit_init(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, flags);
+  intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk,
+                                              HS_IDENT_CIRCUIT_INTRO);
+  /* Register circuit in the circuitmap . */
+  hs_circuitmap_register_intro_circ_v3_service_side(intro_circ,
+                                                    &ip->auth_key_kp.pubkey);
+  tmp_circ =
+    hs_circuitmap_get_intro_circ_v3_service_side(&ip->auth_key_kp.pubkey);
+  tt_ptr_op(tmp_circ, OP_EQ, intro_circ);
+
+  /* Pretend that intro point has failed too much */
+  ip->circuit_retries = MAX_INTRO_POINT_CIRCUIT_RETRIES+1;
+
+  /* Now pretend we are freeing this intro circuit. We want to see that our
+   * destructor is not gonna kill our intro point structure since that's the
+   * job of the cleanup routine. */
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  intro_circ = NULL;
+  entry = service_intro_point_find(service, &ip->auth_key_kp.pubkey);
+  tt_assert(entry);
+  /* The free should also remove the circuit from the circuitmap. */
+  tmp_circ =
+    hs_circuitmap_get_intro_circ_v3_service_side(&ip->auth_key_kp.pubkey);
+  tt_assert(!tmp_circ);
+
+  /* Now pretend that a new intro point circ was launched and opened. Check
+   * that the intro point will be established correctly. */
+  intro_circ = origin_circuit_init(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, flags);
+  intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk,
+                                              HS_IDENT_CIRCUIT_INTRO);
+  ed25519_pubkey_copy(&intro_circ->hs_ident->intro_auth_pk,
+                      &ip->auth_key_kp.pubkey);
+  /* Register circuit in the circuitmap . */
+  hs_circuitmap_register_intro_circ_v3_service_side(intro_circ,
+                                                    &ip->auth_key_kp.pubkey);
+  tmp_circ =
+    hs_circuitmap_get_intro_circ_v3_service_side(&ip->auth_key_kp.pubkey);
+  tt_ptr_op(tmp_circ, OP_EQ, intro_circ);
+  tt_int_op(TO_CIRCUIT(intro_circ)->marked_for_close, OP_EQ, 0);
+  circuit_mark_for_close(TO_CIRCUIT(intro_circ), END_CIRC_REASON_INTERNAL);
+  tt_int_op(TO_CIRCUIT(intro_circ)->marked_for_close, OP_NE, 0);
+  /* At this point, we should not be able to find it in the circuitmap. */
+  tmp_circ =
+    hs_circuitmap_get_intro_circ_v3_service_side(&ip->auth_key_kp.pubkey);
+  tt_assert(!tmp_circ);
+
+ done:
+  if (intro_circ) {
+    circuit_free_(TO_CIRCUIT(intro_circ));
+  }
+  /* Frees the service object. */
+  hs_free_all();
+  UNMOCK(assert_circuit_ok);
+}
+
+/** Test sending and receiving introduce2 cells */
+static void
+test_introduce2(void *arg)
+{
+  int ret;
+  int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
+  uint8_t payload[RELAY_PAYLOAD_SIZE] = {0};
+  origin_circuit_t *circ = NULL;
+  hs_service_t *service;
+  hs_service_intro_point_t *ip = NULL;
+
+  (void) arg;
+
+  hs_init();
+  MOCK(circuit_mark_for_close_, mock_circuit_mark_for_close);
+  MOCK(get_or_state,
+       get_or_state_replacement);
+
+  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+
+  circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_S_INTRO, flags);
+  tt_assert(circ);
+
+  /* Test a wrong purpose. */
+  TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_S_ESTABLISH_INTRO;
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_service_receive_introduce2(circ, payload, sizeof(payload));
+  tt_int_op(ret, OP_EQ, -1);
+  expect_log_msg_containing("Received an INTRODUCE2 cell on a "
+                            "non introduction circuit of purpose");
+  teardown_capture_of_logs();
+
+  /* Back to normal. */
+  TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_S_INTRO;
+
+  /* No service associated to it. */
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_service_receive_introduce2(circ, payload, sizeof(payload));
+  tt_int_op(ret, OP_EQ, -1);
+  expect_log_msg_containing("Unknown service identity key");
+  teardown_capture_of_logs();
+
+  /* Set a service for this circuit. */
+  service = helper_create_service();
+  ed25519_pubkey_copy(&circ->hs_ident->identity_pk,
+                      &service->keys.identity_pk);
+  /* No introduction point associated to it. */
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_service_receive_introduce2(circ, payload, sizeof(payload));
+  tt_int_op(ret, OP_EQ, -1);
+  expect_log_msg_containing("Unknown introduction auth key when handling "
+                            "an INTRODUCE2 cell on circuit");
+  teardown_capture_of_logs();
+
+  /* Set an IP object now for this circuit. */
+  {
+    ip = helper_create_service_ip();
+    service_intro_point_add(service->desc_current->intro_points.map, ip);
+    /* Update ident to contain the intro point auth key. */
+    ed25519_pubkey_copy(&circ->hs_ident->intro_auth_pk,
+                        &ip->auth_key_kp.pubkey);
+  }
+
+  /* This will fail because receiving an INTRODUCE2 cell implies a valid cell
+   * and then launching circuits so let's not do that and instead test that
+   * behaviour differently. */
+  ret = hs_service_receive_introduce2(circ, payload, sizeof(payload));
+  tt_int_op(ret, OP_EQ, -1);
+  tt_u64_op(ip->introduce2_count, OP_EQ, 0);
+
+ done:
+  or_state_free(dummy_state);
+  dummy_state = NULL;
+  if (circ)
+    circuit_free_(TO_CIRCUIT(circ));
+  hs_free_all();
+  UNMOCK(circuit_mark_for_close_);
+}
+
+/** Test basic hidden service housekeeping operations (maintaining intro
+ *  points, etc) */
+static void
+test_service_event(void *arg)
+{
+  int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
+  time_t now = time(NULL);
+  hs_service_t *service;
+  origin_circuit_t *circ = NULL;
+
+  (void) arg;
+
+  hs_init();
+  MOCK(circuit_mark_for_close_, mock_circuit_mark_for_close);
+
+  circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_S_INTRO, flags);
+
+  /* Set a service for this circuit. */
+  service = helper_create_service();
+  ed25519_pubkey_copy(&circ->hs_ident->identity_pk,
+                      &service->keys.identity_pk);
+
+  /* Currently this consists of cleaning invalid intro points. So adding IPs
+   * here that should get cleaned up. */
+  {
+    hs_service_intro_point_t *ip = helper_create_service_ip();
+    service_intro_point_add(service->desc_current->intro_points.map, ip);
+    /* This run will remove the IP because we have no circuits nor node_t
+     * associated with it. */
+    run_housekeeping_event(now);
+    tt_int_op(digest256map_size(service->desc_current->intro_points.map),
+              OP_EQ, 0);
+    /* We'll trigger a removal because we've reached our maximum amount of
+     * times we should retry a circuit. For this, we need to have a node_t
+     * that matches the identity of this IP. */
+    routerinfo_t ri;
+    memset(&ri, 0, sizeof(ri));
+    ip = helper_create_service_ip();
+    service_intro_point_add(service->desc_current->intro_points.map, ip);
+    memset(ri.cache_info.identity_digest, 'A', DIGEST_LEN);
+    /* This triggers a node_t creation. */
+    tt_assert(nodelist_set_routerinfo(&ri, NULL));
+    ip->circuit_retries = MAX_INTRO_POINT_CIRCUIT_RETRIES + 1;
+    run_housekeeping_event(now);
+    tt_int_op(digest256map_size(service->desc_current->intro_points.map),
+              OP_EQ, 0);
+    /* No removal but no circuit so this means the IP object will stay in the
+     * descriptor map so we can retry it. */
+    ip = helper_create_service_ip();
+    service_intro_point_add(service->desc_current->intro_points.map, ip);
+    ip->circuit_established = 1;  /* We'll test that, it MUST be 0 after. */
+    run_housekeeping_event(now);
+    tt_int_op(digest256map_size(service->desc_current->intro_points.map),
+              OP_EQ, 1);
+    /* Remove the IP object at once for the next test. */
+    ip->circuit_retries = MAX_INTRO_POINT_CIRCUIT_RETRIES + 1;
+    run_housekeeping_event(now);
+    tt_int_op(digest256map_size(service->desc_current->intro_points.map),
+              OP_EQ, 0);
+    /* Now, we'll create an IP with a registered circuit. The IP object
+     * shouldn't go away. */
+    ip = helper_create_service_ip();
+    service_intro_point_add(service->desc_current->intro_points.map, ip);
+    ed25519_pubkey_copy(&circ->hs_ident->intro_auth_pk,
+                        &ip->auth_key_kp.pubkey);
+    hs_circuitmap_register_intro_circ_v3_service_side(
+                                         circ, &ip->auth_key_kp.pubkey);
+    run_housekeeping_event(now);
+    tt_int_op(digest256map_size(service->desc_current->intro_points.map),
+              OP_EQ, 1);
+    /* We'll mangle the IP object to expire. */
+    ip->time_to_expire = now;
+    run_housekeeping_event(now);
+    tt_int_op(digest256map_size(service->desc_current->intro_points.map),
+              OP_EQ, 0);
+  }
+
+ done:
+  hs_circuitmap_remove_circuit(TO_CIRCUIT(circ));
+  circuit_free_(TO_CIRCUIT(circ));
+  hs_free_all();
+  UNMOCK(circuit_mark_for_close_);
+}
+
+/** Test that we rotate descriptors correctly. */
+static void
+test_rotate_descriptors(void *arg)
+{
+  int ret;
+  time_t next_rotation_time, now;
+  hs_service_t *service;
+  hs_service_descriptor_t *desc_next;
+
+  (void) arg;
+
+  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+
+  hs_init();
+  MOCK(get_or_state, get_or_state_replacement);
+  MOCK(circuit_mark_for_close_, mock_circuit_mark_for_close);
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  /* Descriptor rotation happens with a consensus with a new SRV. */
+
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                           &mock_ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                           &mock_ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+  voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after);
+
+  update_approx_time(mock_ns.valid_after+1);
+  now = mock_ns.valid_after+1;
+
+  /* Create a service with a default descriptor and state. It's added to the
+   * global map. */
+  service = helper_create_service();
+  service_descriptor_free(service->desc_current);
+  service->desc_current = NULL;
+  /* This triggers a build for both descriptors. The time now is only used in
+   * the descriptor certificate which is important to be now else the decoding
+   * will complain that the cert has expired if we use valid_after. */
+  build_all_descriptors(now);
+  tt_assert(service->desc_current);
+  tt_assert(service->desc_next);
+
+  /* Tweak our service next rotation time so we can use a custom time. */
+  service->state.next_rotation_time = next_rotation_time =
+    mock_ns.valid_after + (11 * 60 * 60);
+
+  /* Nothing should happen, we are not at a new SRV. Our next rotation time
+   * should be untouched. */
+  rotate_all_descriptors(mock_ns.valid_after);
+  tt_u64_op(service->state.next_rotation_time, OP_EQ, next_rotation_time);
+  tt_assert(service->desc_current);
+  tt_assert(service->desc_next);
+  tt_u64_op(service->desc_current->time_period_num, OP_EQ,
+            hs_get_previous_time_period_num(0));
+  tt_u64_op(service->desc_next->time_period_num, OP_EQ,
+            hs_get_time_period_num(0));
+  /* Keep a reference so we can compare it after rotation to the current. */
+  desc_next = service->desc_next;
+
+  /* Going right after a new SRV. */
+  ret = parse_rfc1123_time("Sat, 27 Oct 1985 01:00:00 UTC",
+                           &mock_ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 27 Oct 1985 02:00:00 UTC",
+                           &mock_ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+  voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after);
+
+  update_approx_time(mock_ns.valid_after+1);
+  now = mock_ns.valid_after+1;
+
+  /* Note down what to expect for the next rotation time which is 01:00 + 23h
+   * meaning 00:00:00. */
+  next_rotation_time = mock_ns.valid_after + (23 * 60 * 60);
+  /* We should have our next rotation time modified, our current descriptor
+   * cleaned up and the next descriptor becoming the current. */
+  rotate_all_descriptors(mock_ns.valid_after);
+  tt_u64_op(service->state.next_rotation_time, OP_EQ, next_rotation_time);
+  tt_mem_op(service->desc_current, OP_EQ, desc_next, sizeof(*desc_next));
+  tt_assert(service->desc_next == NULL);
+
+  /* A second time should do nothing. */
+  rotate_all_descriptors(mock_ns.valid_after);
+  tt_u64_op(service->state.next_rotation_time, OP_EQ, next_rotation_time);
+  tt_mem_op(service->desc_current, OP_EQ, desc_next, sizeof(*desc_next));
+  tt_assert(service->desc_next == NULL);
+
+  build_all_descriptors(now);
+  tt_mem_op(service->desc_current, OP_EQ, desc_next, sizeof(*desc_next));
+  tt_u64_op(service->desc_current->time_period_num, OP_EQ,
+            hs_get_time_period_num(0));
+  tt_u64_op(service->desc_next->time_period_num, OP_EQ,
+            hs_get_next_time_period_num(0));
+  tt_assert(service->desc_next);
+
+ done:
+  hs_free_all();
+  UNMOCK(get_or_state);
+  UNMOCK(circuit_mark_for_close_);
+  UNMOCK(networkstatus_get_live_consensus);
+}
+
+/** Test building descriptors: picking intro points, setting up their link
+ *  specifiers, etc. */
+static void
+test_build_update_descriptors(void *arg)
+{
+  int ret;
+  time_t now = time(NULL);
+  node_t *node;
+  hs_service_t *service;
+  hs_service_intro_point_t *ip_cur, *ip_next;
+  routerinfo_t ri;
+
+  (void) arg;
+
+  hs_init();
+
+  MOCK(get_or_state,
+       get_or_state_replacement);
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 03:00:00 UTC",
+                           &mock_ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 04:00:00 UTC",
+                           &mock_ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+  voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after);
+
+  update_approx_time(mock_ns.valid_after+1);
+  now = mock_ns.valid_after+1;
+
+  /* Create a service without a current descriptor to trigger a build. */
+  service = helper_create_service();
+  tt_assert(service);
+  /* Unfortunately, the helper creates a dummy descriptor so get rid of it. */
+  service_descriptor_free(service->desc_current);
+  service->desc_current = NULL;
+
+  /* We have a fresh service so this should trigger a build for both
+   * descriptors for specific time period that we'll test. */
+  build_all_descriptors(now);
+  /* Check *current* descriptor. */
+  tt_assert(service->desc_current);
+  tt_assert(service->desc_current->desc);
+  tt_assert(service->desc_current->intro_points.map);
+  /* The current time period is the one expected when starting at 03:00. */
+  tt_u64_op(service->desc_current->time_period_num, OP_EQ,
+            hs_get_time_period_num(0));
+  /* This should be untouched, the update descriptor process changes it. */
+  tt_u64_op(service->desc_current->next_upload_time, OP_EQ, 0);
+
+  /* Check *next* descriptor. */
+  tt_assert(service->desc_next);
+  tt_assert(service->desc_next->desc);
+  tt_assert(service->desc_next->intro_points.map);
+  tt_assert(service->desc_current != service->desc_next);
+  tt_u64_op(service->desc_next->time_period_num, OP_EQ,
+            hs_get_next_time_period_num(0));
+  /* This should be untouched, the update descriptor process changes it. */
+  tt_u64_op(service->desc_next->next_upload_time, OP_EQ, 0);
+
+  /* Time to test the update of those descriptors. At first, we have no node
+   * in the routerlist so this will find NO suitable node for the IPs. */
+  setup_full_capture_of_logs(LOG_INFO);
+  update_all_descriptors_intro_points(now);
+  expect_log_msg_containing("Unable to find a suitable node to be an "
+                            "introduction point for service");
+  teardown_capture_of_logs();
+  tt_int_op(digest256map_size(service->desc_current->intro_points.map),
+            OP_EQ, 0);
+  tt_int_op(digest256map_size(service->desc_next->intro_points.map),
+            OP_EQ, 0);
+
+  /* Now, we'll setup a node_t. */
+  {
+    tor_addr_t ipv4_addr;
+    curve25519_secret_key_t curve25519_secret_key;
+
+    memset(&ri, 0, sizeof(routerinfo_t));
+
+    tor_addr_parse(&ipv4_addr, "127.0.0.1");
+    ri.addr = tor_addr_to_ipv4h(&ipv4_addr);
+    ri.or_port = 1337;
+    ri.purpose = ROUTER_PURPOSE_GENERAL;
+    /* Ugly yes but we never free the "ri" object so this just makes things
+     * easier. */
+    ri.protocol_list = (char *) "HSDir=1-2 LinkAuth=3";
+    summarize_protover_flags(&ri.pv, ri.protocol_list, NULL);
+    ret = curve25519_secret_key_generate(&curve25519_secret_key, 0);
+    tt_int_op(ret, OP_EQ, 0);
+    ri.onion_curve25519_pkey =
+      tor_malloc_zero(sizeof(curve25519_public_key_t));
+    ri.onion_pkey = tor_malloc_zero(140);
+    curve25519_public_key_generate(ri.onion_curve25519_pkey,
+                                   &curve25519_secret_key);
+    memset(ri.cache_info.identity_digest, 'A', DIGEST_LEN);
+    /* Setup ed25519 identity */
+    ed25519_keypair_t kp1;
+    ed25519_keypair_generate(&kp1, 0);
+    ri.cache_info.signing_key_cert = tor_malloc_zero(sizeof(tor_cert_t));
+    tt_assert(ri.cache_info.signing_key_cert);
+    ed25519_pubkey_copy(&ri.cache_info.signing_key_cert->signing_key,
+                        &kp1.pubkey);
+    nodelist_set_routerinfo(&ri, NULL);
+    node = node_get_mutable_by_id(ri.cache_info.identity_digest);
+    tt_assert(node);
+    node->is_running = node->is_valid = node->is_fast = node->is_stable = 1;
+  }
+
+  /* We have to set this, or the lack of microdescriptors for these
+   * nodes will make them unusable. */
+  get_options_mutable()->UseMicrodescriptors = 0;
+
+  /* We expect to pick only one intro point from the node above. */
+  setup_full_capture_of_logs(LOG_INFO);
+  update_all_descriptors_intro_points(now);
+  tor_free(node->ri->onion_curve25519_pkey); /* Avoid memleak. */
+  tor_free(node->ri->cache_info.signing_key_cert);
+  tor_free(node->ri->onion_pkey);
+  expect_log_msg_containing("just picked 1 intro points and wanted 3 for next "
+                            "descriptor. It currently has 0 intro points. "
+                            "Launching ESTABLISH_INTRO circuit shortly.");
+  teardown_capture_of_logs();
+  tt_int_op(digest256map_size(service->desc_current->intro_points.map),
+            OP_EQ, 1);
+  tt_int_op(digest256map_size(service->desc_next->intro_points.map),
+            OP_EQ, 1);
+  /* Get the IP object. Because we don't have the auth key of the IP, we can't
+   * query it so get the first element in the map. */
+  {
+    void *obj = NULL;
+    const uint8_t *key;
+    digest256map_iter_t *iter =
+      digest256map_iter_init(service->desc_current->intro_points.map);
+    digest256map_iter_get(iter, &key, &obj);
+    tt_assert(obj);
+    ip_cur = obj;
+    /* Get also the IP from the next descriptor. We'll make sure it's not the
+     * same object as in the current descriptor. */
+    iter = digest256map_iter_init(service->desc_next->intro_points.map);
+    digest256map_iter_get(iter, &key, &obj);
+    tt_assert(obj);
+    ip_next = obj;
+  }
+  tt_mem_op(ip_cur, OP_NE, ip_next, sizeof(hs_desc_intro_point_t));
+
+  /* We won't test the service IP object because there is a specific test
+   * already for this but we'll make sure that the state is coherent.*/
+
+  /* Three link specifiers are mandatoy so make sure we do have them. */
+  tt_int_op(smartlist_len(ip_cur->base.link_specifiers), OP_EQ, 3);
+  /* Make sure we have a valid encryption keypair generated when we pick an
+   * intro point in the update process. */
+  tt_assert(!tor_mem_is_zero((char *) ip_cur->enc_key_kp.seckey.secret_key,
+                             CURVE25519_SECKEY_LEN));
+  tt_assert(!tor_mem_is_zero((char *) ip_cur->enc_key_kp.pubkey.public_key,
+                             CURVE25519_PUBKEY_LEN));
+  tt_u64_op(ip_cur->time_to_expire, OP_GE, now +
+            INTRO_POINT_LIFETIME_MIN_SECONDS);
+  tt_u64_op(ip_cur->time_to_expire, OP_LE, now +
+            INTRO_POINT_LIFETIME_MAX_SECONDS);
+
+  /* Now, we will try to set up a service after a new time period has started
+   * and see if it behaves as expected. */
+
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                           &mock_ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                           &mock_ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+
+  update_approx_time(mock_ns.valid_after+1);
+  now = mock_ns.valid_after+1;
+
+  /* Create a service without a current descriptor to trigger a build. */
+  service = helper_create_service();
+  tt_assert(service);
+  /* Unfortunately, the helper creates a dummy descriptor so get rid of it. */
+  service_descriptor_free(service->desc_current);
+  service->desc_current = NULL;
+
+  /* We have a fresh service so this should trigger a build for both
+   * descriptors for specific time period that we'll test. */
+  build_all_descriptors(now);
+  /* Check *current* descriptor. */
+  tt_assert(service->desc_current);
+  tt_assert(service->desc_current->desc);
+  tt_assert(service->desc_current->intro_points.map);
+  /* This should be for the previous time period. */
+  tt_u64_op(service->desc_current->time_period_num, OP_EQ,
+            hs_get_previous_time_period_num(0));
+  /* This should be untouched, the update descriptor process changes it. */
+  tt_u64_op(service->desc_current->next_upload_time, OP_EQ, 0);
+
+  /* Check *next* descriptor. */
+  tt_assert(service->desc_next);
+  tt_assert(service->desc_next->desc);
+  tt_assert(service->desc_next->intro_points.map);
+  tt_assert(service->desc_current != service->desc_next);
+  tt_u64_op(service->desc_next->time_period_num, OP_EQ,
+            hs_get_time_period_num(0));
+  /* This should be untouched, the update descriptor process changes it. */
+  tt_u64_op(service->desc_next->next_upload_time, OP_EQ, 0);
+
+  /* Let's remove the next descriptor to simulate a rotation. */
+  service_descriptor_free(service->desc_next);
+  service->desc_next = NULL;
+
+  build_all_descriptors(now);
+  /* Check *next* descriptor. */
+  tt_assert(service->desc_next);
+  tt_assert(service->desc_next->desc);
+  tt_assert(service->desc_next->intro_points.map);
+  tt_assert(service->desc_current != service->desc_next);
+  tt_u64_op(service->desc_next->time_period_num, OP_EQ,
+            hs_get_next_time_period_num(0));
+  /* This should be untouched, the update descriptor process changes it. */
+  tt_u64_op(service->desc_next->next_upload_time, OP_EQ, 0);
+
+ done:
+  hs_free_all();
+  nodelist_free_all();
+}
+
+/** Test building descriptors. We use this separate function instead of
+ *  using test_build_update_descriptors because that function is too complex
+ *  and also too interactive. */
+static void
+test_build_descriptors(void *arg)
+{
+  int ret;
+  time_t now = time(NULL);
+
+  (void) arg;
+
+  hs_init();
+
+  MOCK(get_or_state,
+       get_or_state_replacement);
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 03:00:00 UTC",
+                           &mock_ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 04:00:00 UTC",
+                           &mock_ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+  voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after);
+
+  /* Generate a valid number of fake auth clients when a client authorization
+   * is disabled. */
+  {
+    hs_service_t *service = helper_create_service();
+    service_descriptor_free(service->desc_current);
+    service->desc_current = NULL;
+
+    build_all_descriptors(now);
+    hs_desc_superencrypted_data_t *superencrypted;
+    superencrypted = &service->desc_current->desc->superencrypted_data;
+    tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 16);
+  }
+
+  /* Generate a valid number of fake auth clients when the number of
+   * clients is zero. */
+  {
+    hs_service_t *service = helper_create_service_with_clients(0);
+    service_descriptor_free(service->desc_current);
+    service->desc_current = NULL;
+
+    build_all_descriptors(now);
+    hs_desc_superencrypted_data_t *superencrypted;
+    superencrypted = &service->desc_current->desc->superencrypted_data;
+    tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 16);
+  }
+
+  /* Generate a valid number of fake auth clients when the number of
+   * clients is not a multiple of 16. */
+  {
+    hs_service_t *service = helper_create_service_with_clients(20);
+    service_descriptor_free(service->desc_current);
+    service->desc_current = NULL;
+
+    build_all_descriptors(now);
+    hs_desc_superencrypted_data_t *superencrypted;
+    superencrypted = &service->desc_current->desc->superencrypted_data;
+    tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 32);
+  }
+
+  /* Do not generate any fake desc client when the number of clients is
+   * a multiple of 16 but not zero. */
+  {
+    hs_service_t *service = helper_create_service_with_clients(32);
+    service_descriptor_free(service->desc_current);
+    service->desc_current = NULL;
+
+    build_all_descriptors(now);
+    hs_desc_superencrypted_data_t *superencrypted;
+    superencrypted = &service->desc_current->desc->superencrypted_data;
+    tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 32);
+  }
+
+ done:
+  hs_free_all();
+}
+
+static void
+test_upload_descriptors(void *arg)
+{
+  int ret;
+  time_t now;
+  hs_service_t *service;
+
+  (void) arg;
+
+  hs_init();
+  MOCK(get_or_state,
+       get_or_state_replacement);
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                           &mock_ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                           &mock_ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+  voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after);
+
+  update_approx_time(mock_ns.valid_after+1);
+  now = mock_ns.valid_after+1;
+
+  /* Create a service with no descriptor. It's added to the global map. */
+  service = hs_service_new(get_options());
+  tt_assert(service);
+  service->config.version = HS_VERSION_THREE;
+  ed25519_secret_key_generate(&service->keys.identity_sk, 0);
+  ed25519_public_key_generate(&service->keys.identity_pk,
+                              &service->keys.identity_sk);
+  /* Register service to global map. */
+  ret = register_service(get_hs_service_map(), service);
+  tt_int_op(ret, OP_EQ, 0);
+  /* But first, build our descriptor. */
+  build_all_descriptors(now);
+
+  /* Nothing should happen because we have 0 introduction circuit established
+   * and we want (by default) 3 intro points. */
+  run_upload_descriptor_event(now);
+  /* If no upload happened, this should be untouched. */
+  tt_u64_op(service->desc_current->next_upload_time, OP_EQ, 0);
+  /* We'll simulate that we've opened our intro point circuit and that we only
+   * want one intro point. */
+  service->config.num_intro_points = 1;
+
+  /* Set our next upload time after now which will skip the upload. */
+  service->desc_current->next_upload_time = now + 1000;
+  run_upload_descriptor_event(now);
+  /* If no upload happened, this should be untouched. */
+  tt_u64_op(service->desc_current->next_upload_time, OP_EQ, now + 1000);
+
+ done:
+  hs_free_all();
+  UNMOCK(get_or_state);
+}
+
+/** Global vars used by test_rendezvous1_parsing() */
+static char rend1_payload[RELAY_PAYLOAD_SIZE];
+static size_t rend1_payload_len = 0;
+
+/** Mock for relay_send_command_from_edge() to send a RENDEZVOUS1 cell. Instead
+ *  of sending it to the network, instead save it to the global `rend1_payload`
+ *  variable so that we can inspect it in the test_rendezvous1_parsing()
+ *  test. */
+static int
+mock_relay_send_rendezvous1(streamid_t stream_id, circuit_t *circ,
+                            uint8_t relay_command, const char *payload,
+                            size_t payload_len,
+                            crypt_path_t *cpath_layer,
+                            const char *filename, int lineno)
+{
+  (void) stream_id;
+  (void) circ;
+  (void) relay_command;
+  (void) cpath_layer;
+  (void) filename;
+  (void) lineno;
+
+  memcpy(rend1_payload, payload, payload_len);
+  rend1_payload_len = payload_len;
+
+  return 0;
+}
+
+/** Send a RENDEZVOUS1 as a service, and parse it as a client. */
+static void
+test_rendezvous1_parsing(void *arg)
+{
+  int retval;
+  static const char *test_addr =
+    "4acth47i6kxnvkewtm6q7ib2s3ufpo5sqbsnzjpbi7utijcltosqemad.onion";
+  hs_service_t *service = NULL;
+  origin_circuit_t *service_circ = NULL;
+  origin_circuit_t *client_circ = NULL;
+  ed25519_keypair_t ip_auth_kp;
+  curve25519_keypair_t ephemeral_kp;
+  curve25519_keypair_t client_kp;
+  curve25519_keypair_t ip_enc_kp;
+  int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
+
+  (void) arg;
+
+  MOCK(relay_send_command_from_edge_, mock_relay_send_rendezvous1);
+
+  {
+    /* Let's start by setting up the service that will start the rend */
+    service = tor_malloc_zero(sizeof(hs_service_t));
+    ed25519_secret_key_generate(&service->keys.identity_sk, 0);
+    ed25519_public_key_generate(&service->keys.identity_pk,
+                                &service->keys.identity_sk);
+    memcpy(service->onion_address, test_addr, sizeof(service->onion_address));
+    tt_assert(service);
+  }
+
+  {
+    /* Now let's set up the service rendezvous circuit and its keys. */
+    service_circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_S_CONNECT_REND,
+                                                flags);
+    tor_free(service_circ->hs_ident);
+    hs_ntor_rend_cell_keys_t hs_ntor_rend_cell_keys;
+    uint8_t rendezvous_cookie[HS_REND_COOKIE_LEN];
+    curve25519_keypair_generate(&ip_enc_kp, 0);
+    curve25519_keypair_generate(&ephemeral_kp, 0);
+    curve25519_keypair_generate(&client_kp, 0);
+    ed25519_keypair_generate(&ip_auth_kp, 0);
+    retval = hs_ntor_service_get_rendezvous1_keys(&ip_auth_kp.pubkey,
+                                                  &ip_enc_kp,
+                                                  &ephemeral_kp,
+                                                  &client_kp.pubkey,
+                                                  &hs_ntor_rend_cell_keys);
+    tt_int_op(retval, OP_EQ, 0);
+
+    memset(rendezvous_cookie, 2, sizeof(rendezvous_cookie));
+    service_circ->hs_ident =
+      create_rp_circuit_identifier(service, rendezvous_cookie,
+                                   &ephemeral_kp.pubkey,
+                                   &hs_ntor_rend_cell_keys);
+  }
+
+  /* Send out the RENDEZVOUS1 and make sure that our mock func worked */
+  tt_assert(tor_mem_is_zero(rend1_payload, 32));
+  hs_circ_service_rp_has_opened(service, service_circ);
+  tt_assert(!tor_mem_is_zero(rend1_payload, 32));
+  tt_int_op(rend1_payload_len, OP_EQ, HS_LEGACY_RENDEZVOUS_CELL_SIZE);
+
+  /******************************/
+
+  /** Now let's create the client rendezvous circuit */
+  client_circ =
+    helper_create_origin_circuit(CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED,
+                                 flags);
+  /* fix up its circ ident */
+  ed25519_pubkey_copy(&client_circ->hs_ident->intro_auth_pk,
+                      &ip_auth_kp.pubkey);
+  memcpy(&client_circ->hs_ident->rendezvous_client_kp,
+         &client_kp, sizeof(client_circ->hs_ident->rendezvous_client_kp));
+  memcpy(&client_circ->hs_ident->intro_enc_pk.public_key,
+         &ip_enc_kp.pubkey.public_key,
+         sizeof(client_circ->hs_ident->intro_enc_pk.public_key));
+
+  /* Now parse the rendezvous2 circuit and make sure it was fine. We are
+   * skipping 20 bytes off its payload, since that's the rendezvous cookie
+   * which is only present in REND1. */
+  retval = handle_rendezvous2(client_circ,
+                              (uint8_t*)rend1_payload+20,
+                              rend1_payload_len-20);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* TODO: We are only simulating client/service here. We could also simulate
+   * the rendezvous point by plugging in rend_mid_establish_rendezvous(). We
+   * would need an extra circuit and some more stuff but it's doable. */
+
+ done:
+  circuit_free_(TO_CIRCUIT(service_circ));
+  circuit_free_(TO_CIRCUIT(client_circ));
+  hs_service_free(service);
+  hs_free_all();
+  UNMOCK(relay_send_command_from_edge_);
+}
+
+static void
+test_authorized_client_config_equal(void *arg)
+{
+  int ret;
+  hs_service_config_t *config1, *config2;
+
+  (void) arg;
+
+  config1 = tor_malloc_zero(sizeof(*config1));
+  config2 = tor_malloc_zero(sizeof(*config2));
+
+  /* Both configs are empty. */
+  {
+    config1->clients = smartlist_new();
+    config2->clients = smartlist_new();
+
+    ret = service_authorized_client_config_equal(config1, config2);
+    tt_int_op(ret, OP_EQ, 1);
+
+    service_clear_config(config1);
+    service_clear_config(config2);
+  }
+
+  /* Both configs have exactly the same client config. */
+  {
+    config1->clients = smartlist_new();
+    config2->clients = smartlist_new();
+
+    hs_service_authorized_client_t *client1, *client2;
+    client1 = helper_create_authorized_client();
+    client2 = helper_create_authorized_client();
+
+    smartlist_add(config1->clients, client1);
+    smartlist_add(config1->clients, client2);
+
+    /* We should swap the order of clients here to test that the order
+     * does not matter. */
+    smartlist_add(config2->clients, helper_clone_authorized_client(client2));
+    smartlist_add(config2->clients, helper_clone_authorized_client(client1));
+
+    ret = service_authorized_client_config_equal(config1, config2);
+    tt_int_op(ret, OP_EQ, 1);
+
+    service_clear_config(config1);
+    service_clear_config(config2);
+  }
+
+  /* The numbers of clients in both configs are not equal. */
+  {
+    config1->clients = smartlist_new();
+    config2->clients = smartlist_new();
+
+    hs_service_authorized_client_t *client1, *client2;
+    client1 = helper_create_authorized_client();
+    client2 = helper_create_authorized_client();
+
+    smartlist_add(config1->clients, client1);
+    smartlist_add(config1->clients, client2);
+
+    smartlist_add(config2->clients, helper_clone_authorized_client(client1));
+
+    ret = service_authorized_client_config_equal(config1, config2);
+    tt_int_op(ret, OP_EQ, 0);
+
+    service_clear_config(config1);
+    service_clear_config(config2);
+  }
+
+  /* The first config has two distinct clients while the second config
+   * has two clients but they are duplicate. */
+  {
+    config1->clients = smartlist_new();
+    config2->clients = smartlist_new();
+
+    hs_service_authorized_client_t *client1, *client2;
+    client1 = helper_create_authorized_client();
+    client2 = helper_create_authorized_client();
+
+    smartlist_add(config1->clients, client1);
+    smartlist_add(config1->clients, client2);
+
+    smartlist_add(config2->clients, helper_clone_authorized_client(client1));
+    smartlist_add(config2->clients, helper_clone_authorized_client(client1));
+
+    ret = service_authorized_client_config_equal(config1, config2);
+    tt_int_op(ret, OP_EQ, 0);
+
+    service_clear_config(config1);
+    service_clear_config(config2);
+  }
+
+  /* Both configs have totally distinct clients. */
+  {
+    config1->clients = smartlist_new();
+    config2->clients = smartlist_new();
+
+    hs_service_authorized_client_t *client1, *client2, *client3, *client4;
+    client1 = helper_create_authorized_client();
+    client2 = helper_create_authorized_client();
+    client3 = helper_create_authorized_client();
+    client4 = helper_create_authorized_client();
+
+    smartlist_add(config1->clients, client1);
+    smartlist_add(config1->clients, client2);
+
+    smartlist_add(config2->clients, client3);
+    smartlist_add(config2->clients, client4);
+
+    ret = service_authorized_client_config_equal(config1, config2);
+    tt_int_op(ret, OP_EQ, 0);
+
+    service_clear_config(config1);
+    service_clear_config(config2);
+  }
+
+ done:
+  tor_free(config1);
+  tor_free(config2);
+}
+
+/** Test that client circuit ID gets correctly exported */
+static void
+test_export_client_circuit_id(void *arg)
+{
+  origin_circuit_t *or_circ = NULL;
+  size_t sz;
+  char *cp1=NULL, *cp2=NULL;
+  connection_t *conn = NULL;
+
+  (void) arg;
+
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+
+  hs_service_init();
+
+  /* Create service */
+  hs_service_t *service = helper_create_service();
+  /* Check that export circuit ID detection works */
+  service->config.circuit_id_protocol = HS_CIRCUIT_ID_PROTOCOL_NONE;
+  tt_int_op(0, OP_EQ,
+            hs_service_exports_circuit_id(&service->keys.identity_pk));
+  service->config.circuit_id_protocol = HS_CIRCUIT_ID_PROTOCOL_HAPROXY;
+  tt_int_op(1, OP_EQ,
+            hs_service_exports_circuit_id(&service->keys.identity_pk));
+
+  /* Create client connection */
+  conn = test_conn_get_connection(AP_CONN_STATE_CIRCUIT_WAIT, CONN_TYPE_AP, 0);
+
+  /* Create client edge conn hs_ident */
+  edge_connection_t *edge_conn = TO_EDGE_CONN(conn);
+  edge_conn->hs_ident = hs_ident_edge_conn_new(&service->keys.identity_pk);
+  edge_conn->hs_ident->orig_virtual_port = 42;
+
+  /* Create rend circuit */
+  or_circ = origin_circuit_new();
+  or_circ->base_.purpose = CIRCUIT_PURPOSE_C_REND_JOINED;
+  edge_conn->on_circuit = TO_CIRCUIT(or_circ);
+  or_circ->global_identifier = 666;
+
+  /* Export circuit ID */
+  export_hs_client_circuit_id(edge_conn, service->config.circuit_id_protocol);
+
+  /* Check contents */
+  cp1 = buf_get_contents(conn->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ,
+            "PROXY TCP6 fc00:dead:beef:4dad::0:29a ::1 666 42\r\n");
+
+  /* Change circ GID and see that the reported circuit ID also changes */
+  or_circ->global_identifier = 22;
+
+  /* check changes */
+  export_hs_client_circuit_id(edge_conn, service->config.circuit_id_protocol);
+  cp2 = buf_get_contents(conn->outbuf, &sz);
+  tt_str_op(cp1, OP_NE, cp2);
+  tor_free(cp1);
+
+  /* Check that GID with UINT32_MAX works. */
+  or_circ->global_identifier = UINT32_MAX;
+
+  export_hs_client_circuit_id(edge_conn, service->config.circuit_id_protocol);
+  cp1 = buf_get_contents(conn->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ,
+            "PROXY TCP6 fc00:dead:beef:4dad::ffff:ffff ::1 65535 42\r\n");
+  tor_free(cp1);
+
+  /* Check that GID with UINT16_MAX works. */
+  or_circ->global_identifier = UINT16_MAX;
+
+  export_hs_client_circuit_id(edge_conn, service->config.circuit_id_protocol);
+  cp1 = buf_get_contents(conn->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ,
+            "PROXY TCP6 fc00:dead:beef:4dad::0:ffff ::1 65535 42\r\n");
+  tor_free(cp1);
+
+  /* Check that GID with UINT16_MAX + 7 works. */
+  or_circ->global_identifier = UINT16_MAX + 7;
+
+  export_hs_client_circuit_id(edge_conn, service->config.circuit_id_protocol);
+  cp1 = buf_get_contents(conn->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "PROXY TCP6 fc00:dead:beef:4dad::1:6 ::1 6 42\r\n");
+
+ done:
+  UNMOCK(connection_write_to_buf_impl_);
+  circuit_free_(TO_CIRCUIT(or_circ));
+  connection_free_minimal(conn);
+  hs_service_free(service);
+  tor_free(cp1);
+  tor_free(cp2);
+}
+
+struct testcase_t hs_service_tests[] = {
+  { "e2e_rend_circuit_setup", test_e2e_rend_circuit_setup, TT_FORK,
+    NULL, NULL },
+  { "load_keys", test_load_keys, TT_FORK,
+    NULL, NULL },
+  { "client_filename_is_valid", test_client_filename_is_valid, TT_FORK,
+    NULL, NULL },
+  { "parse_authorized_client", test_parse_authorized_client, TT_FORK,
+    NULL, NULL },
+  { "load_keys_with_client_auth", test_load_keys_with_client_auth, TT_FORK,
+    NULL, NULL },
+  { "access_service", test_access_service, TT_FORK,
+    NULL, NULL },
+  { "service_intro_point", test_service_intro_point, TT_FORK,
+    NULL, NULL },
+  { "helper_functions", test_helper_functions, TT_FORK,
+    NULL, NULL },
+  { "intro_circuit_opened", test_intro_circuit_opened, TT_FORK,
+    NULL, NULL },
+  { "intro_established", test_intro_established, TT_FORK,
+    NULL, NULL },
+  { "closing_intro_circs", test_closing_intro_circs, TT_FORK,
+    NULL, NULL },
+  { "rdv_circuit_opened", test_rdv_circuit_opened, TT_FORK,
+    NULL, NULL },
+  { "introduce2", test_introduce2, TT_FORK,
+    NULL, NULL },
+  { "service_event", test_service_event, TT_FORK,
+    NULL, NULL },
+  { "rotate_descriptors", test_rotate_descriptors, TT_FORK,
+    NULL, NULL },
+  { "build_update_descriptors", test_build_update_descriptors, TT_FORK,
+    NULL, NULL },
+  { "build_descriptors", test_build_descriptors, TT_FORK,
+    NULL, NULL },
+  { "upload_descriptors", test_upload_descriptors, TT_FORK,
+    NULL, NULL },
+  { "rendezvous1_parsing", test_rendezvous1_parsing, TT_FORK,
+    NULL, NULL },
+  { "authorized_client_config_equal", test_authorized_client_config_equal,
+    TT_FORK, NULL, NULL },
+  { "export_client_circuit_id", test_export_client_circuit_id, TT_FORK,
+    NULL, NULL },
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_introduce.c b/src/test/test_introduce.c
index 810b03c93d..4a6d90d97e 100644
--- a/src/test/test_introduce.c
+++ b/src/test/test_introduce.c
@@ -1,13 +1,13 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
-#include "crypto.h"
-#include "or.h"
-#include "test.h"
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "core/or/or.h"
+#include "test/test.h"
 
 #define RENDSERVICE_PRIVATE
-#include "rendservice.h"
+#include "feature/rend/rendservice.h"
 
 static uint8_t v0_test_plaintext[] =
     /* 20 bytes of rendezvous point nickname */
@@ -307,7 +307,7 @@ do_parse_test(uint8_t *plaintext, size_t plaintext_len, int phase)
   /* Do early parsing */
   parsed_req = rend_service_begin_parse_intro(cell, cell_len, 2, &err_msg);
   tt_assert(parsed_req);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   tt_mem_op(parsed_req->pk,OP_EQ, digest, DIGEST_LEN);
   tt_assert(parsed_req->ciphertext);
   tt_assert(parsed_req->ciphertext_len > 0);
@@ -318,7 +318,7 @@ do_parse_test(uint8_t *plaintext, size_t plaintext_len, int phase)
   /* Do decryption */
   r = rend_service_decrypt_intro(parsed_req, k, &err_msg);
   tt_assert(!r);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   tt_assert(parsed_req->plaintext);
   tt_assert(parsed_req->plaintext_len > 0);
 
@@ -328,7 +328,7 @@ do_parse_test(uint8_t *plaintext, size_t plaintext_len, int phase)
   /* Do late parsing */
   r = rend_service_parse_intro_plaintext(parsed_req, &err_msg);
   tt_assert(!r);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   tt_assert(parsed_req->parsed);
 
  done:
@@ -355,7 +355,7 @@ make_intro_from_plaintext(
 
   /*
    * Figure out an upper bound on how big the ciphertext will be
-   * (see crypto_pk_public_hybrid_encrypt())
+   * (see crypto_pk_obsolete_public_hybrid_encrypt())
    */
   ciphertext_size = PKCS1_OAEP_PADDING_OVERHEAD;
   ciphertext_size += crypto_pk_keysize(key);
@@ -372,7 +372,7 @@ make_intro_from_plaintext(
   tt_assert(r >= 0);
 
   /* Do encryption */
-  r = crypto_pk_public_hybrid_encrypt(
+  r = crypto_pk_obsolete_public_hybrid_encrypt(
       key, cell + DIGEST_LEN, ciphertext_size,
       buf, len,
       PK_PKCS1_OAEP_PADDING, 0);
diff --git a/src/test/test_key_expiration.sh b/src/test/test_key_expiration.sh
new file mode 100755
index 0000000000..3474210607
--- /dev/null
+++ b/src/test/test_key_expiration.sh
@@ -0,0 +1,138 @@
+#!/bin/sh
+
+# Note: some of this code is lifted from zero_length_keys.sh and
+# test_keygen.sh, and could be unified.
+
+umask 077
+set -e
+
+if [ $# -eq 0 ] || [ ! -f ${1} ] || [ ! -x ${1} ]; then
+  if [ "$TESTING_TOR_BINARY" = "" ] ; then
+    echo "Usage: ${0} PATH_TO_TOR [case-number]"
+    exit 1
+  fi
+fi
+
+UNAME_OS=`uname -s | cut -d_ -f1`
+if test "$UNAME_OS" = 'CYGWIN' || \
+   test "$UNAME_OS" = 'MSYS' || \
+   test "$UNAME_OS" = 'MINGW'; then
+  echo "This test is unreliable on Windows. See trac #26076. Skipping." >&2
+  exit 77
+fi
+
+if [ $# -ge 1 ]; then
+  TOR_BINARY="${1}"
+  shift
+else
+  TOR_BINARY="${TESTING_TOR_BINARY}"
+fi
+
+if [ $# -ge 1 ]; then
+  dflt=0
+else
+  dflt=1
+fi
+
+CASE1=$dflt
+CASE2=$dflt
+CASE3=$dflt
+
+if [ $# -ge 1 ]; then
+  eval "CASE${1}"=1
+fi
+
+
+dump() { xxd -p "$1" | tr -d '\n '; }
+die() { echo "$1" >&2 ; exit 5; }
+check_dir() { [ -d "$1" ] || die "$1 did not exist"; }
+check_file() { [ -e "$1" ] || die "$1 did not exist"; }
+check_no_file() { [ -e "$1" ] && die "$1 was not supposed to exist" || true; }
+check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: `dump $1` vs `dump $2`"; }
+check_keys_eq() { check_files_eq "${SRC}/keys/${1}" "${ME}/keys/${1}"; }
+
+DATA_DIR=`mktemp -d -t tor_key_expiration_tests.XXXXXX`
+if [ -z "$DATA_DIR" ]; then
+  echo "Failure: mktemp invocation returned empty string" >&2
+  exit 3
+fi
+if [ ! -d "$DATA_DIR" ]; then
+  echo "Failure: mktemp invocation result doesn't point to directory" >&2
+  exit 3
+fi
+trap "rm -rf '$DATA_DIR'" 0
+
+# Use an absolute path for this or Tor will complain
+DATA_DIR=`cd "${DATA_DIR}" && pwd`
+
+touch "${DATA_DIR}/empty_torrc"
+touch "${DATA_DIR}/empty_defaults_torrc"
+
+QUIETLY="--hush"
+SILENTLY="--quiet"
+TOR="${TOR_BINARY} --DisableNetwork 1 --ShutdownWaitLength 0 --ORPort 12345 --ExitRelay 0 --DataDirectory ${DATA_DIR} -f ${DATA_DIR}/empty_torrc --defaults-torrc ${DATA_DIR}/empty_defaults_torrc"
+
+##### SETUP
+#
+# Here we create a set of keys.
+
+# Step 1: Start Tor with --list-fingerprint --quiet.  Make sure everything is there.
+echo "Setup step #1"
+${TOR} --list-fingerprint ${SILENTLY} > /dev/null
+
+check_dir "${DATA_DIR}/keys"
+check_file "${DATA_DIR}/keys/ed25519_master_id_public_key"
+check_file "${DATA_DIR}/keys/ed25519_master_id_secret_key"
+check_file "${DATA_DIR}/keys/ed25519_signing_cert"
+check_file "${DATA_DIR}/keys/ed25519_signing_secret_key"
+check_file "${DATA_DIR}/keys/secret_id_key"
+check_file "${DATA_DIR}/keys/secret_onion_key"
+check_file "${DATA_DIR}/keys/secret_onion_key_ntor"
+
+##### TEST CASES
+
+echo "=== Starting key expiration tests."
+
+FN="${DATA_DIR}/stderr"
+
+if [ "$CASE1" = 1 ]; then
+  echo "==== Case 1: Test --key-expiration without argument and ensure usage"
+  echo "             instructions are printed."
+
+  ${TOR} ${QUIETLY} --key-expiration 2>"$FN" || true
+  grep "No valid argument to --key-expiration found!" "$FN" >/dev/null || \
+    die "Tor didn't mention supported --key-expiration argmuents"
+
+  echo "==== Case 1: ok"
+fi
+
+if [ "$CASE2" = 1 ]; then
+  echo "==== Case 2: Start Tor with --key-expiration 'sign' and make sure it prints an expiration."
+
+  ${TOR} ${QUIETLY} --key-expiration sign 2>"$FN"
+  grep "signing-cert-expiry:" "$FN" >/dev/null || \
+    die "Tor didn't print an expiration"
+
+  echo "==== Case 2: ok"
+fi
+
+if [ "$CASE3" = 1 ]; then
+  echo "==== Case 3: Start Tor with --key-expiration 'sign', when there is no"
+  echo "             signing key, and make sure that Tor generates a new key"
+  echo "             and prints its certificate's expiration."
+
+  mv "${DATA_DIR}/keys/ed25519_signing_cert" \
+     "${DATA_DIR}/keys/ed25519_signing_cert.bak"
+
+  ${TOR} --key-expiration sign > "$FN" 2>&1
+  grep "It looks like I need to generate and sign a new medium-term signing key" "$FN" >/dev/null || \
+    die "Tor didn't create a new signing key"
+  check_file "${DATA_DIR}/keys/ed25519_signing_cert"
+  grep "signing-cert-expiry:" "$FN" >/dev/null || \
+    die "Tor didn't print an expiration"
+
+  mv "${DATA_DIR}/keys/ed25519_signing_cert.bak" \
+     "${DATA_DIR}/keys/ed25519_signing_cert"
+
+  echo "==== Case 3: ok"
+fi
diff --git a/src/test/test_keygen.sh b/src/test/test_keygen.sh
index 4106425cb3..7afff271cb 100755
--- a/src/test/test_keygen.sh
+++ b/src/test/test_keygen.sh
@@ -48,6 +48,12 @@ fi
   CASE8=$dflt
   CASE9=$dflt
   CASE10=$dflt
+  CASE11A=$dflt
+  CASE11B=$dflt
+  CASE11C=$dflt
+  CASE11D=$dflt
+  CASE11E=$dflt
+  CASE11F=$dflt
 
   if [ $# -ge 1 ]; then
      eval "CASE${1}"=1
@@ -77,10 +83,11 @@ trap "rm -rf '$DATA_DIR'" 0
 DATA_DIR=`cd "${DATA_DIR}" && pwd`
 
 touch "${DATA_DIR}/empty_torrc"
+touch "${DATA_DIR}/empty_defaults_torrc"
 
 QUIETLY="--hush"
 SILENTLY="--quiet"
-TOR="${TOR_BINARY} ${QUIETLY} --DisableNetwork 1 --ShutdownWaitLength 0 --ORPort 12345 --ExitRelay 0 -f ${DATA_DIR}/empty_torrc"
+TOR="${TOR_BINARY} ${QUIETLY} --DisableNetwork 1 --ShutdownWaitLength 0 --ORPort 12345 --ExitRelay 0 -f ${DATA_DIR}/empty_torrc --defaults-torrc ${DATA_DIR}/empty_defaults_torrc"
 
 ##### SETUP
 #
@@ -371,6 +378,109 @@ echo "==== Case 10 ok"
 
 fi
 
+# Case 11a: -passphrase-fd without --keygen
+
+if [ "$CASE11A" = 1 ]; then
+
+ME="${DATA_DIR}/case11a"
+
+mkdir -p "${ME}/keys"
+
+${TOR} --DataDirectory "${ME}" --passphrase-fd 1 > "${ME}/stdout" && die "Successfully started with passphrase-fd but no keygen?" || true
+
+grep "passphrase-fd specified without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
+
+echo "==== Case 11A ok"
+
+fi
+
+# Case 11b: --no-passphrase without --keygen
+
+if [ "$CASE11B" = 1 ]; then
+
+ME="${DATA_DIR}/case11b"
+
+mkdir -p "${ME}/keys"
+
+${TOR} --DataDirectory "${ME}" --no-passphrase > "${ME}/stdout" && die "Successfully started with no-passphrase but no keygen?" || true
+
+grep "no-passphrase specified without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
+
+echo "==== Case 11B ok"
+
+fi
+
+# Case 11c: --newpass without --keygen
+
+if [ "$CASE11C" = 1 ]; then
+
+ME="${DATA_DIR}/case11C"
+
+mkdir -p "${ME}/keys"
+
+${TOR} --DataDirectory "${ME}" --newpass > "${ME}/stdout" && die "Successfully started with newpass but no keygen?" || true
+
+grep "newpass specified without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
+
+echo "==== Case 11C ok"
+
+fi
+
+######## --master-key does not work yet, but this will test the error case
+########  when it does.
+#
+# Case 11d: --master-key without --keygen
+#
+if [ "$CASE11D" = 1 ]; then
+#
+# ME="${DATA_DIR}/case11d"
+#
+# mkdir -p "${ME}/keys"
+#
+# ${TOR} --DataDirectory "${ME}" --master-key "${ME}/foobar" > "${ME}/stdout" && die "Successfully started with master-key but no keygen?" || true
+#
+# cat "${ME}/stdout"
+#
+# grep "master-key without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
+
+    echo "==== Case 11D skipped"
+
+fi
+
+
+# Case 11E: Silly passphrase-fd
+
+if [ "$CASE11E" = 1 ]; then
+
+ME="${DATA_DIR}/case11E"
+
+mkdir -p "${ME}/keys"
+
+${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd ewigeblumenkraft > "${ME}/stdout" && die "Successfully started with bogus passphrase-fd?" || true
+
+grep "Invalid --passphrase-fd value" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
+
+echo "==== Case 11E ok"
+
+fi
+
+
+# Case 11F: --no-passphrase with --passphrase-fd
+
+if [ "$CASE11F" = 1 ]; then
+
+ME="${DATA_DIR}/case11F"
+
+mkdir -p "${ME}/keys"
+
+${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd 1 --no-passphrase > "${ME}/stdout" && die "Successfully started with bogus passphrase-fd combination?" || true
+
+grep "no-passphrase specified with --passphrase-fd" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
+
+echo "==== Case 11F ok"
+
+fi
+
 
 # Check cert-only.
 
diff --git a/src/test/test_keypin.c b/src/test/test_keypin.c
index 95657349c6..e7beef8609 100644
--- a/src/test/test_keypin.c
+++ b/src/test/test_keypin.c
@@ -1,13 +1,12 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 #define KEYPIN_PRIVATE
-#include "or.h"
-#include "keypin.h"
-#include "util.h"
+#include "core/or/or.h"
+#include "feature/dirauth/keypin.h"
 
-#include "test.h"
+#include "test/test.h"
 
 static void
 test_keypin_parse_line(void *arg)
@@ -20,8 +19,8 @@ test_keypin_parse_line(void *arg)
                 "aGVyZSBpcyBhIGdvb2Qgc2hhMSE "
                 "VGhpcyBlZDI1NTE5IHNjb2ZmcyBhdCB0aGUgc2hhMS4");
   tt_assert(ent);
-  tt_mem_op(ent->rsa_id, ==, "here is a good sha1!", 20);
-  tt_mem_op(ent->ed25519_key, ==, "This ed25519 scoffs at the sha1.", 32);
+  tt_mem_op(ent->rsa_id, OP_EQ, "here is a good sha1!", 20);
+  tt_mem_op(ent->ed25519_key, OP_EQ, "This ed25519 scoffs at the sha1.", 32);
   tor_free(ent); ent = NULL;
 
   /* Good line with extra stuff we will ignore. */
@@ -29,27 +28,27 @@ test_keypin_parse_line(void *arg)
                 "aGVyZSBpcyBhIGdvb2Qgc2hhMSE "
                 "VGhpcyBlZDI1NTE5IHNjb2ZmcyBhdCB0aGUgc2hhMS4helloworld");
   tt_assert(ent);
-  tt_mem_op(ent->rsa_id, ==, "here is a good sha1!", 20);
-  tt_mem_op(ent->ed25519_key, ==, "This ed25519 scoffs at the sha1.", 32);
+  tt_mem_op(ent->rsa_id, OP_EQ, "here is a good sha1!", 20);
+  tt_mem_op(ent->ed25519_key, OP_EQ, "This ed25519 scoffs at the sha1.", 32);
   tor_free(ent); ent = NULL;
 
   /* Bad line: no space in the middle. */
   ent = keypin_parse_journal_line(
                 "aGVyZSBpcyBhIGdvb2Qgc2hhMSE?"
                 "VGhpcyBlZDI1NTE5IHNjb2ZmcyBhdCB0aGUgc2hhMS4");
-  tt_assert(! ent);
+  tt_ptr_op(ent, OP_EQ, NULL);
 
   /* Bad line: bad base64 in RSA ID */
   ent = keypin_parse_journal_line(
                 "aGVyZSBpcyBhIGdv!2Qgc2hhMSE "
                 "VGhpcyBlZDI1NTE5IHNjb2ZmcyBhdCB0aGUgc2hhMS4");
-  tt_assert(! ent);
+  tt_ptr_op(ent, OP_EQ, NULL);
 
   /* Bad line: bad base64 in Ed25519 */
   ent = keypin_parse_journal_line(
                 "aGVyZSBpcyBhIGdvb2Qgc2hhMSE "
                 "VGhpcyBlZDI1NTE5IHNjb2ZmcyB!dCB0aGUgc2hhMS4");
-  tt_assert(! ent);
+  tt_ptr_op(ent, OP_EQ, NULL);
 
  done:
   tor_free(ent);
@@ -82,11 +81,11 @@ test_keypin_parse_file(void *arg)
 "Z2dsZSBpbiBzd29tZWVzd2FucyA aW4gdm9sdXB0YXRlIGF4ZS1oYWNrZXIgZXNzZSByaXA\n"
 "cHVsdXMgY3J1bW1paSBldSBtb28 ZiBudWxsYSBzbnV2di5QTFVHSFBMT1ZFUlhZWlpZLi4\n";
 
-  tt_int_op(0, ==, keypin_load_journal_impl(data1, strlen(data1)));
-  tt_int_op(8, ==, smartlist_len(mock_addent_got));
+  tt_int_op(0, OP_EQ, keypin_load_journal_impl(data1, strlen(data1)));
+  tt_int_op(8, OP_EQ, smartlist_len(mock_addent_got));
   keypin_ent_t *ent = smartlist_get(mock_addent_got, 2);
-  tt_mem_op(ent->rsa_id, ==, "r lerkim, sed do bar", 20);
-  tt_mem_op(ent->ed25519_key, ==, "baloot tempor gluppitus ut labor", 32);
+  tt_mem_op(ent->rsa_id, OP_EQ, "r lerkim, sed do bar", 20);
+  tt_mem_op(ent->ed25519_key, OP_EQ, "baloot tempor gluppitus ut labor", 32);
 
   /* More complex example: weird lines, bogus lines,
      duplicate/conflicting lines */
@@ -107,24 +106,25 @@ test_keypin_parse_file(void *arg)
     "ZHMgc3BlYWsgdHJ1dGgsIGFuZCA aXQgd2FzIHRydaUgdGhhdCBhbGwgdGhlIG1hc3Rlcgo\n"
     ;
 
-  tt_int_op(0, ==, keypin_load_journal_impl(data2, strlen(data2)));
-  tt_int_op(13, ==, smartlist_len(mock_addent_got));
+  tt_int_op(0, OP_EQ, keypin_load_journal_impl(data2, strlen(data2)));
+  tt_int_op(13, OP_EQ, smartlist_len(mock_addent_got));
   ent = smartlist_get(mock_addent_got, 9);
-  tt_mem_op(ent->rsa_id, ==, "\"You have made a goo", 20);
-  tt_mem_op(ent->ed25519_key, ==, "d beginning.\" But no more. Wizar", 32);
+  tt_mem_op(ent->rsa_id, OP_EQ, "\"You have made a goo", 20);
+  tt_mem_op(ent->ed25519_key, OP_EQ, "d beginning.\" But no more. Wizar", 32);
 
   ent = smartlist_get(mock_addent_got, 12);
-  tt_mem_op(ent->rsa_id, ==, "ds speak truth, and ", 20);
-  tt_mem_op(ent->ed25519_key, ==, "it was tru\xa5 that all the master\n", 32);
+  tt_mem_op(ent->rsa_id, OP_EQ, "ds speak truth, and ", 20);
+  tt_mem_op(ent->ed25519_key, OP_EQ,
+            "it was tru\xa5 that all the master\n", 32);
 
   /* File truncated before NL */
   const char data3[] =
     "Tm8gZHJhZ29uIGNhbiByZXNpc3Q IHRoZSBmYXNjaW5hdGlvbiBvZiByaWRkbGluZyB0YWw";
-  tt_int_op(0, ==, keypin_load_journal_impl(data3, strlen(data3)));
-  tt_int_op(14, ==, smartlist_len(mock_addent_got));
+  tt_int_op(0, OP_EQ, keypin_load_journal_impl(data3, strlen(data3)));
+  tt_int_op(14, OP_EQ, smartlist_len(mock_addent_got));
   ent = smartlist_get(mock_addent_got, 13);
-  tt_mem_op(ent->rsa_id, ==, "No dragon can resist", 20);
-  tt_mem_op(ent->ed25519_key, ==, " the fascination of riddling tal", 32);
+  tt_mem_op(ent->rsa_id, OP_EQ, "No dragon can resist", 20);
+  tt_mem_op(ent->ed25519_key, OP_EQ, " the fascination of riddling tal", 32);
 
  done:
   keypin_clear();
@@ -141,32 +141,32 @@ test_keypin_add_entry(void *arg)
   (void)arg;
   keypin_clear();
 
-  tt_int_op(KEYPIN_ADDED, ==, ADD("ambassadors-at-large",
+  tt_int_op(KEYPIN_ADDED, OP_EQ, ADD("ambassadors-at-large",
                                   "bread-and-butter thing-in-itself"));
-  tt_int_op(KEYPIN_ADDED, ==, ADD("gentleman-adventurer",
+  tt_int_op(KEYPIN_ADDED, OP_EQ, ADD("gentleman-adventurer",
                                   "cloak-and-dagger what's-his-face"));
 
-  tt_int_op(KEYPIN_FOUND, ==, ADD("ambassadors-at-large",
+  tt_int_op(KEYPIN_FOUND, OP_EQ, ADD("ambassadors-at-large",
                                   "bread-and-butter thing-in-itself"));
-  tt_int_op(KEYPIN_FOUND, ==, ADD("ambassadors-at-large",
+  tt_int_op(KEYPIN_FOUND, OP_EQ, ADD("ambassadors-at-large",
                                   "bread-and-butter thing-in-itself"));
-  tt_int_op(KEYPIN_FOUND, ==, ADD("gentleman-adventurer",
+  tt_int_op(KEYPIN_FOUND, OP_EQ, ADD("gentleman-adventurer",
                                   "cloak-and-dagger what's-his-face"));
 
-  tt_int_op(KEYPIN_ADDED, ==, ADD("Johnnies-come-lately",
+  tt_int_op(KEYPIN_ADDED, OP_EQ, ADD("Johnnies-come-lately",
                                   "run-of-the-mill root-mean-square"));
 
-  tt_int_op(KEYPIN_MISMATCH, ==, ADD("gentleman-adventurer",
+  tt_int_op(KEYPIN_MISMATCH, OP_EQ, ADD("gentleman-adventurer",
                                      "hypersentimental closefistedness"));
 
-  tt_int_op(KEYPIN_MISMATCH, ==, ADD("disestablismentarian",
+  tt_int_op(KEYPIN_MISMATCH, OP_EQ, ADD("disestablismentarian",
                                      "cloak-and-dagger what's-his-face"));
 
-  tt_int_op(KEYPIN_FOUND, ==, ADD("gentleman-adventurer",
+  tt_int_op(KEYPIN_FOUND, OP_EQ, ADD("gentleman-adventurer",
                                   "cloak-and-dagger what's-his-face"));
 
-  tt_int_op(KEYPIN_NOT_FOUND, ==, LONE_RSA("Llanfairpwllgwyngyll"));
-  tt_int_op(KEYPIN_MISMATCH, ==, LONE_RSA("Johnnies-come-lately"));
+  tt_int_op(KEYPIN_NOT_FOUND, OP_EQ, LONE_RSA("Llanfairpwllgwyngyll"));
+  tt_int_op(KEYPIN_MISMATCH, OP_EQ, LONE_RSA("Johnnies-come-lately"));
 
  done:
   keypin_clear();
@@ -179,51 +179,51 @@ test_keypin_journal(void *arg)
   char *contents = NULL;
   const char *fname = get_fname("keypin-journal");
 
-  tt_int_op(0, ==, keypin_load_journal(fname)); /* ENOENT is okay */
+  tt_int_op(0, OP_EQ, keypin_load_journal(fname)); /* ENOENT is okay */
   update_approx_time(1217709000);
-  tt_int_op(0, ==, keypin_open_journal(fname));
+  tt_int_op(0, OP_EQ, keypin_open_journal(fname));
 
-  tt_int_op(KEYPIN_ADDED, ==, ADD("king-of-the-herrings",
+  tt_int_op(KEYPIN_ADDED, OP_EQ, ADD("king-of-the-herrings",
                                   "good-for-nothing attorney-at-law"));
-  tt_int_op(KEYPIN_ADDED, ==, ADD("yellowish-red-yellow",
+  tt_int_op(KEYPIN_ADDED, OP_EQ, ADD("yellowish-red-yellow",
                                   "salt-and-pepper high-muck-a-muck"));
-  tt_int_op(KEYPIN_FOUND, ==, ADD("yellowish-red-yellow",
+  tt_int_op(KEYPIN_FOUND, OP_EQ, ADD("yellowish-red-yellow",
                                   "salt-and-pepper high-muck-a-muck"));
   keypin_close_journal();
   keypin_clear();
 
-  tt_int_op(0, ==, keypin_load_journal(fname));
+  tt_int_op(0, OP_EQ, keypin_load_journal(fname));
   update_approx_time(1231041600);
-  tt_int_op(0, ==, keypin_open_journal(fname));
-  tt_int_op(KEYPIN_FOUND, ==, ADD("yellowish-red-yellow",
+  tt_int_op(0, OP_EQ, keypin_open_journal(fname));
+  tt_int_op(KEYPIN_FOUND, OP_EQ, ADD("yellowish-red-yellow",
                                   "salt-and-pepper high-muck-a-muck"));
-  tt_int_op(KEYPIN_ADDED, ==, ADD("theatre-in-the-round",
+  tt_int_op(KEYPIN_ADDED, OP_EQ, ADD("theatre-in-the-round",
                                   "holier-than-thou jack-in-the-box"));
-  tt_int_op(KEYPIN_ADDED, ==, ADD("no-deposit-no-return",
+  tt_int_op(KEYPIN_ADDED, OP_EQ, ADD("no-deposit-no-return",
                                   "across-the-board will-o-the-wisp"));
-  tt_int_op(KEYPIN_MISMATCH, ==, ADD("intellectualizations",
+  tt_int_op(KEYPIN_MISMATCH, OP_EQ, ADD("intellectualizations",
                                      "salt-and-pepper high-muck-a-muck"));
   keypin_close_journal();
   keypin_clear();
 
-  tt_int_op(0, ==, keypin_load_journal(fname));
+  tt_int_op(0, OP_EQ, keypin_load_journal(fname));
   update_approx_time(1412278354);
-  tt_int_op(0, ==, keypin_open_journal(fname));
-  tt_int_op(KEYPIN_FOUND, ==, ADD("yellowish-red-yellow",
+  tt_int_op(0, OP_EQ, keypin_open_journal(fname));
+  tt_int_op(KEYPIN_FOUND, OP_EQ, ADD("yellowish-red-yellow",
                                   "salt-and-pepper high-muck-a-muck"));
-  tt_int_op(KEYPIN_MISMATCH, ==, ADD("intellectualizations",
+  tt_int_op(KEYPIN_MISMATCH, OP_EQ, ADD("intellectualizations",
                                      "salt-and-pepper high-muck-a-muck"));
-  tt_int_op(KEYPIN_FOUND, ==, ADD("theatre-in-the-round",
+  tt_int_op(KEYPIN_FOUND, OP_EQ, ADD("theatre-in-the-round",
                                   "holier-than-thou jack-in-the-box"));
-  tt_int_op(KEYPIN_MISMATCH, ==, ADD("counterrevolutionary",
+  tt_int_op(KEYPIN_MISMATCH, OP_EQ, ADD("counterrevolutionary",
                                      "holier-than-thou jack-in-the-box"));
-  tt_int_op(KEYPIN_MISMATCH, ==, ADD("no-deposit-no-return",
+  tt_int_op(KEYPIN_MISMATCH, OP_EQ, ADD("no-deposit-no-return",
                                      "floccinaucinihilipilificationism"));
   keypin_close_journal();
 
   contents = read_file_to_str(fname, RFTS_BIN, NULL);
   tt_assert(contents);
-  tt_str_op(contents,==,
+  tt_str_op(contents,OP_EQ,
     "\n"
     "@opened-at 2008-08-02 20:30:00\n"
     "a2luZy1vZi10aGUtaGVycmluZ3M Z29vZC1mb3Itbm90aGluZyBhdHRvcm5leS1hdC1sYXc\n"
diff --git a/src/test/test_link_handshake.c b/src/test/test_link_handshake.c
index ddf66f4d34..34f59f26cd 100644
--- a/src/test/test_link_handshake.c
+++ b/src/test/test_link_handshake.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -6,16 +6,30 @@
 #define CHANNELTLS_PRIVATE
 #define CONNECTION_PRIVATE
 #define TOR_CHANNEL_INTERNAL_
-#include "or.h"
-#include "config.h"
-#include "connection.h"
-#include "connection_or.h"
-#include "channeltls.h"
-#include "link_handshake.h"
-#include "scheduler.h"
-
-#include "test.h"
-#include "log_test_helpers.h"
+#define TORTLS_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/connection_or.h"
+#include "core/or/channeltls.h"
+#include "trunnel/link_handshake.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routerkeys.h"
+#include "core/or/scheduler.h"
+#include "feature/nodelist/torcert.h"
+
+#include "core/or/or_connection_st.h"
+#include "core/or/or_handshake_certs_st.h"
+#include "core/or/or_handshake_state_st.h"
+#include "core/or/var_cell_st.h"
+
+#define TOR_X509_PRIVATE
+#include "lib/tls/tortls.h"
+#include "lib/tls/x509.h"
+
+#include "test/test.h"
+#include "test/log_test_helpers.h"
 
 static var_cell_t *mock_got_var_cell = NULL;
 
@@ -37,6 +51,16 @@ mock_tls_cert_matches_key(const tor_tls_t *tls, const tor_x509_cert_t *cert)
   (void) cert; // XXXX look at this.
   return 1;
 }
+static tor_tls_t *mock_peer_cert_expect_tortls = NULL;
+static tor_x509_cert_t *mock_peer_cert = NULL;
+static tor_x509_cert_t *
+mock_get_peer_cert(tor_tls_t *tls)
+{
+  if (mock_peer_cert_expect_tortls &&
+      mock_peer_cert_expect_tortls != tls)
+    return NULL;
+  return tor_x509_cert_dup(mock_peer_cert);
+}
 
 static int mock_send_netinfo_called = 0;
 static int
@@ -57,14 +81,29 @@ mock_close_for_err(or_connection_t *orconn, int flush)
 }
 
 static int mock_send_authenticate_called = 0;
+static int mock_send_authenticate_called_with_type = 0;
 static int
 mock_send_authenticate(or_connection_t *conn, int type)
 {
   (void) conn;
-  (void) type;
+  mock_send_authenticate_called_with_type = type;
   ++mock_send_authenticate_called;// XXX check_this
   return 0;
 }
+static int
+mock_export_key_material(tor_tls_t *tls, uint8_t *secrets_out,
+                         const uint8_t *context,
+                         size_t context_len,
+                         const char *label)
+{
+  (void) tls;
+  (void)secrets_out;
+  (void)context;
+  (void)context_len;
+  (void)label;
+  memcpy(secrets_out, "int getRandomNumber(){return 4;}", 32);
+  return 0;
+}
 
 static tor_x509_cert_t *mock_own_cert = NULL;
 static tor_x509_cert_t *
@@ -78,20 +117,23 @@ mock_get_own_cert(tor_tls_t *tls)
 static void
 test_link_handshake_certs_ok(void *arg)
 {
-  (void) arg;
-
   or_connection_t *c1 = or_connection_new(CONN_TYPE_OR, AF_INET);
   or_connection_t *c2 = or_connection_new(CONN_TYPE_OR, AF_INET);
   var_cell_t *cell1 = NULL, *cell2 = NULL;
   certs_cell_t *cc1 = NULL, *cc2 = NULL;
   channel_tls_t *chan1 = NULL, *chan2 = NULL;
   crypto_pk_t *key1 = NULL, *key2 = NULL;
+  const int with_ed = !strcmp((const char *)arg, "Ed25519");
+
+  tor_addr_from_ipv4h(&c1->base_.addr, 0x7f000001);
+  tor_addr_from_ipv4h(&c2->base_.addr, 0x7f000001);
 
   scheduler_init();
 
   MOCK(tor_tls_cert_matches_key, mock_tls_cert_matches_key);
   MOCK(connection_or_write_var_cell_to_buf, mock_write_var_cell);
   MOCK(connection_or_send_netinfo, mock_send_netinfo);
+  MOCK(tor_tls_get_peer_cert, mock_get_peer_cert);
   MOCK(tor_tls_get_own_cert, mock_get_own_cert);
 
   key1 = pk_generate(2);
@@ -101,8 +143,17 @@ test_link_handshake_certs_ok(void *arg)
    * actually generate a CERTS cell.
    */
   tt_int_op(tor_tls_context_init(TOR_TLS_CTX_IS_PUBLIC_SERVER,
-                                 key1, key2, 86400), ==, 0);
+                                 key1, key2, 86400), OP_EQ, 0);
+
+  if (with_ed) {
+    /* If we're making a CERTS cell for an ed handshake, let's make sure we
+     * have some Ed25519 certificates and keys. */
+    init_mock_ed_keys(key2);
+  } else {
+    certs_cell_ed25519_disabled_for_testing = 1;
+  }
 
+  /* c1 has started_here == 1 */
   {
     const tor_x509_cert_t *link_cert = NULL;
     tt_assert(!tor_tls_get_my_certs(1, &link_cert, NULL));
@@ -111,44 +162,66 @@ test_link_handshake_certs_ok(void *arg)
 
   c1->base_.state = OR_CONN_STATE_OR_HANDSHAKING_V3;
   c1->link_proto = 3;
-  tt_int_op(connection_init_or_handshake_state(c1, 1), ==, 0);
+  tt_int_op(connection_init_or_handshake_state(c1, 1), OP_EQ, 0);
 
+  /* c2 has started_here == 0 */
   c2->base_.state = OR_CONN_STATE_OR_HANDSHAKING_V3;
   c2->link_proto = 3;
-  tt_int_op(connection_init_or_handshake_state(c2, 0), ==, 0);
+  tt_int_op(connection_init_or_handshake_state(c2, 0), OP_EQ, 0);
 
-  tt_int_op(0, ==, connection_or_send_certs_cell(c1));
+  tt_int_op(0, OP_EQ, connection_or_send_certs_cell(c1));
   tt_assert(mock_got_var_cell);
   cell1 = mock_got_var_cell;
 
-  tt_int_op(0, ==, connection_or_send_certs_cell(c2));
+  tt_int_op(0, OP_EQ, connection_or_send_certs_cell(c2));
   tt_assert(mock_got_var_cell);
   cell2 = mock_got_var_cell;
 
-  tt_int_op(cell1->command, ==, CELL_CERTS);
-  tt_int_op(cell1->payload_len, >, 1);
+  tt_int_op(cell1->command, OP_EQ, CELL_CERTS);
+  tt_int_op(cell1->payload_len, OP_GT, 1);
 
-  tt_int_op(cell2->command, ==, CELL_CERTS);
-  tt_int_op(cell2->payload_len, >, 1);
+  tt_int_op(cell2->command, OP_EQ, CELL_CERTS);
+  tt_int_op(cell2->payload_len, OP_GT, 1);
 
-  tt_int_op(cell1->payload_len, ==,
+  tt_int_op(cell1->payload_len, OP_EQ,
             certs_cell_parse(&cc1, cell1->payload, cell1->payload_len));
-  tt_int_op(cell2->payload_len, ==,
+  tt_int_op(cell2->payload_len, OP_EQ,
             certs_cell_parse(&cc2, cell2->payload, cell2->payload_len));
 
-  tt_int_op(2, ==, cc1->n_certs);
-  tt_int_op(2, ==, cc2->n_certs);
+  if (with_ed) {
+    tt_int_op(5, OP_EQ, cc1->n_certs);
+    tt_int_op(5, OP_EQ, cc2->n_certs);
+  } else {
+    tt_int_op(2, OP_EQ, cc1->n_certs);
+    tt_int_op(2, OP_EQ, cc2->n_certs);
+  }
 
-  tt_int_op(certs_cell_get_certs(cc1, 0)->cert_type, ==,
+  tt_int_op(certs_cell_get_certs(cc1, 0)->cert_type, OP_EQ,
             CERTTYPE_RSA1024_ID_AUTH);
-  tt_int_op(certs_cell_get_certs(cc1, 1)->cert_type, ==,
+  tt_int_op(certs_cell_get_certs(cc1, 1)->cert_type, OP_EQ,
             CERTTYPE_RSA1024_ID_ID);
 
-  tt_int_op(certs_cell_get_certs(cc2, 0)->cert_type, ==,
+  tt_int_op(certs_cell_get_certs(cc2, 0)->cert_type, OP_EQ,
             CERTTYPE_RSA1024_ID_LINK);
-  tt_int_op(certs_cell_get_certs(cc2, 1)->cert_type, ==,
+  tt_int_op(certs_cell_get_certs(cc2, 1)->cert_type, OP_EQ,
             CERTTYPE_RSA1024_ID_ID);
 
+  if (with_ed) {
+    tt_int_op(certs_cell_get_certs(cc1, 2)->cert_type, OP_EQ,
+              CERTTYPE_ED_ID_SIGN);
+    tt_int_op(certs_cell_get_certs(cc1, 3)->cert_type, OP_EQ,
+              CERTTYPE_ED_SIGN_AUTH);
+    tt_int_op(certs_cell_get_certs(cc1, 4)->cert_type, OP_EQ,
+              CERTTYPE_RSA1024_ID_EDID);
+
+    tt_int_op(certs_cell_get_certs(cc2, 2)->cert_type, OP_EQ,
+              CERTTYPE_ED_ID_SIGN);
+    tt_int_op(certs_cell_get_certs(cc2, 3)->cert_type, OP_EQ,
+              CERTTYPE_ED_SIGN_LINK);
+    tt_int_op(certs_cell_get_certs(cc2, 4)->cert_type, OP_EQ,
+              CERTTYPE_RSA1024_ID_EDID);
+  }
+
   chan1 = tor_malloc_zero(sizeof(*chan1));
   channel_tls_common_init(chan1);
   c1->chan = chan1;
@@ -159,13 +232,39 @@ test_link_handshake_certs_ok(void *arg)
   c1->base_.conn_array_index = -1;
   crypto_pk_get_digest(key2, c1->identity_digest);
 
+  if (with_ed) {
+    const tor_x509_cert_t *linkc, *idc;
+    tor_tls_get_my_certs(1, &linkc, &idc);
+    mock_peer_cert_expect_tortls = c1->tls; /* We should see this tls... */
+    mock_peer_cert = tor_x509_cert_dup(linkc); /* and when we do, the peer's
+                                                *  cert is this... */
+  }
   channel_tls_process_certs_cell(cell2, chan1);
+  mock_peer_cert_expect_tortls = NULL;
+  tor_x509_cert_free(mock_peer_cert);
+  mock_peer_cert = NULL;
+
+  tor_assert(c1->handshake_state->authenticated);
 
   tt_assert(c1->handshake_state->received_certs_cell);
-  tt_assert(c1->handshake_state->auth_cert == NULL);
-  tt_assert(c1->handshake_state->id_cert);
+  tt_ptr_op(c1->handshake_state->certs->auth_cert, OP_EQ, NULL);
+  tt_ptr_op(c1->handshake_state->certs->ed_sign_auth, OP_EQ, NULL);
+  tt_assert(c1->handshake_state->certs->id_cert);
+  if (with_ed) {
+    tt_assert(c1->handshake_state->certs->ed_sign_link);
+    tt_assert(c1->handshake_state->certs->ed_rsa_crosscert);
+    tt_assert(c1->handshake_state->certs->ed_id_sign);
+    tt_assert(c1->handshake_state->authenticated_rsa);
+    tt_assert(c1->handshake_state->authenticated_ed25519);
+  } else {
+    tt_ptr_op(c1->handshake_state->certs->ed_sign_link, OP_EQ, NULL);
+    tt_ptr_op(c1->handshake_state->certs->ed_rsa_crosscert, OP_EQ, NULL);
+    tt_ptr_op(c1->handshake_state->certs->ed_id_sign, OP_EQ, NULL);
+    tt_assert(c1->handshake_state->authenticated_rsa);
+    tt_assert(! c1->handshake_state->authenticated_ed25519);
+  }
   tt_assert(! tor_mem_is_zero(
-                  (char*)c1->handshake_state->authenticated_peer_id, 20));
+                (char*)c1->handshake_state->authenticated_rsa_peer_id, 20));
 
   chan2 = tor_malloc_zero(sizeof(*chan2));
   channel_tls_common_init(chan2);
@@ -180,22 +279,38 @@ test_link_handshake_certs_ok(void *arg)
   channel_tls_process_certs_cell(cell1, chan2);
 
   tt_assert(c2->handshake_state->received_certs_cell);
-  tt_assert(c2->handshake_state->auth_cert);
-  tt_assert(c2->handshake_state->id_cert);
+  if (with_ed) {
+    tt_assert(c2->handshake_state->certs->ed_sign_auth);
+    tt_assert(c2->handshake_state->certs->ed_rsa_crosscert);
+    tt_assert(c2->handshake_state->certs->ed_id_sign);
+  } else {
+    tt_assert(c2->handshake_state->certs->auth_cert);
+    tt_ptr_op(c2->handshake_state->certs->ed_sign_auth, OP_EQ, NULL);
+    tt_ptr_op(c2->handshake_state->certs->ed_rsa_crosscert, OP_EQ, NULL);
+    tt_ptr_op(c2->handshake_state->certs->ed_id_sign, OP_EQ, NULL);
+  }
+  tt_assert(c2->handshake_state->certs->id_cert);
   tt_assert(tor_mem_is_zero(
-                (char*)c2->handshake_state->authenticated_peer_id, 20));
+              (char*)c2->handshake_state->authenticated_rsa_peer_id, 20));
+  /* no authentication has happened yet, since we haen't gotten an AUTH cell.
+   */
+  tt_assert(! c2->handshake_state->authenticated);
+  tt_assert(! c2->handshake_state->authenticated_rsa);
+  tt_assert(! c2->handshake_state->authenticated_ed25519);
 
  done:
   UNMOCK(tor_tls_cert_matches_key);
   UNMOCK(connection_or_write_var_cell_to_buf);
   UNMOCK(connection_or_send_netinfo);
+  UNMOCK(tor_tls_get_peer_cert);
   UNMOCK(tor_tls_get_own_cert);
   tor_x509_cert_free(mock_own_cert);
-  mock_own_cert = NULL;
+  tor_x509_cert_free(mock_peer_cert);
+  mock_own_cert = mock_peer_cert = NULL;
   memset(c1->identity_digest, 0, sizeof(c1->identity_digest));
   memset(c2->identity_digest, 0, sizeof(c2->identity_digest));
-  connection_free_(TO_CONN(c1));
-  connection_free_(TO_CONN(c2));
+  connection_free_minimal(TO_CONN(c1));
+  connection_free_minimal(TO_CONN(c2));
   tor_free(cell1);
   tor_free(cell2);
   certs_cell_free(cc1);
@@ -211,6 +326,8 @@ test_link_handshake_certs_ok(void *arg)
 }
 
 typedef struct certs_data_s {
+  int is_ed;
+  int is_link_cert;
   or_connection_t *c;
   channel_tls_t *chan;
   certs_cell_t *ccell;
@@ -226,18 +343,21 @@ recv_certs_cleanup(const struct testcase_t *test, void *obj)
   UNMOCK(tor_tls_cert_matches_key);
   UNMOCK(connection_or_send_netinfo);
   UNMOCK(connection_or_close_for_error);
+  UNMOCK(tor_tls_get_peer_cert);
+  UNMOCK(tor_tls_get_own_cert);
 
   if (d) {
     tor_free(d->cell);
     certs_cell_free(d->ccell);
-    connection_or_remove_from_identity_map(d->c);
-    connection_free_(TO_CONN(d->c));
+    connection_or_clear_identity(d->c);
+    connection_free_minimal(TO_CONN(d->c));
     circuitmux_free(d->chan->base_.cmux);
     tor_free(d->chan);
     crypto_pk_free(d->key1);
     crypto_pk_free(d->key2);
     tor_free(d);
   }
+  routerkeys_free_all();
   return 1;
 }
 
@@ -249,34 +369,47 @@ recv_certs_setup(const struct testcase_t *test)
   certs_cell_cert_t *ccc1 = NULL;
   certs_cell_cert_t *ccc2 = NULL;
   ssize_t n;
+  int is_ed = d->is_ed = !strcmpstart(test->setup_data, "Ed25519");
+  int is_rsa = !strcmpstart(test->setup_data, "RSA");
+  int is_link = d->is_link_cert = !strcmpend(test->setup_data, "-Link");
+  int is_auth = !strcmpend(test->setup_data, "-Auth");
+  tor_assert(is_ed != is_rsa);
+  tor_assert(is_link != is_auth);
 
   d->c = or_connection_new(CONN_TYPE_OR, AF_INET);
   d->chan = tor_malloc_zero(sizeof(*d->chan));
   d->c->chan = d->chan;
   d->c->base_.address = tor_strdup("HaveAnAddress");
+  tor_addr_from_ipv4h(&d->c->base_.addr, 0x801f0127);
   d->c->base_.state = OR_CONN_STATE_OR_HANDSHAKING_V3;
   d->chan->conn = d->c;
-  tt_int_op(connection_init_or_handshake_state(d->c, 1), ==, 0);
+  tt_int_op(connection_init_or_handshake_state(d->c, 1), OP_EQ, 0);
   d->c->link_proto = 4;
 
   d->key1 = pk_generate(2);
   d->key2 = pk_generate(3);
 
   tt_int_op(tor_tls_context_init(TOR_TLS_CTX_IS_PUBLIC_SERVER,
-                                 d->key1, d->key2, 86400), ==, 0);
+                                 d->key1, d->key2, 86400), OP_EQ, 0);
+  if (is_ed) {
+    init_mock_ed_keys(d->key2);
+  } else {
+    routerkeys_free_all();
+  }
+
   d->ccell = certs_cell_new();
   ccc1 = certs_cell_cert_new();
   certs_cell_add_certs(d->ccell, ccc1);
   ccc2 = certs_cell_cert_new();
   certs_cell_add_certs(d->ccell, ccc2);
   d->ccell->n_certs = 2;
-  ccc1->cert_type = 1;
+  ccc1->cert_type = is_link ? 1 : 3;
   ccc2->cert_type = 2;
 
   const tor_x509_cert_t *a,*b;
   const uint8_t *enca, *encb;
   size_t lena, lenb;
-  tor_tls_get_my_certs(1, &a, &b);
+  tor_tls_get_my_certs(is_link ? 1 : 0, &a, &b);
   tor_x509_cert_get_der(a, &enca, &lena);
   tor_x509_cert_get_der(b, &encb, &lenb);
   certs_cell_cert_setlen_body(ccc1, lena);
@@ -287,20 +420,61 @@ recv_certs_setup(const struct testcase_t *test)
   memcpy(certs_cell_cert_getarray_body(ccc1), enca, lena);
   memcpy(certs_cell_cert_getarray_body(ccc2), encb, lenb);
 
+  if (is_ed) {
+    certs_cell_cert_t *ccc3 = NULL; /* Id->Sign */
+    certs_cell_cert_t *ccc4 = NULL; /* Sign->Link or Sign->Auth. */
+    certs_cell_cert_t *ccc5 = NULL; /* RSAId->Ed Id. */
+    const tor_cert_t *id_sign = get_master_signing_key_cert();
+    const tor_cert_t *secondary =
+      is_link ? get_current_link_cert_cert() : get_current_auth_key_cert();
+    const uint8_t *cc = NULL;
+    size_t cc_sz;
+    get_master_rsa_crosscert(&cc, &cc_sz);
+
+    ccc3 = certs_cell_cert_new();
+    ccc4 = certs_cell_cert_new();
+    ccc5 = certs_cell_cert_new();
+    certs_cell_add_certs(d->ccell, ccc3);
+    certs_cell_add_certs(d->ccell, ccc4);
+    certs_cell_add_certs(d->ccell, ccc5);
+    ccc3->cert_len = id_sign->encoded_len;
+    ccc4->cert_len = secondary->encoded_len;
+    ccc5->cert_len = cc_sz;
+    certs_cell_cert_setlen_body(ccc3, ccc3->cert_len);
+    certs_cell_cert_setlen_body(ccc4, ccc4->cert_len);
+    certs_cell_cert_setlen_body(ccc5, ccc5->cert_len);
+    memcpy(certs_cell_cert_getarray_body(ccc3), id_sign->encoded,
+           ccc3->cert_len);
+    memcpy(certs_cell_cert_getarray_body(ccc4), secondary->encoded,
+           ccc4->cert_len);
+    memcpy(certs_cell_cert_getarray_body(ccc5), cc, ccc5->cert_len);
+    ccc3->cert_type = 4;
+    ccc4->cert_type = is_link ? 5 : 6;
+    ccc5->cert_type = 7;
+
+    d->ccell->n_certs = 5;
+  }
+
   d->cell = var_cell_new(4096);
   d->cell->command = CELL_CERTS;
 
   n = certs_cell_encode(d->cell->payload, 4096, d->ccell);
-  tt_int_op(n, >, 0);
+  tt_int_op(n, OP_GT, 0);
   d->cell->payload_len = n;
 
   MOCK(tor_tls_cert_matches_key, mock_tls_cert_matches_key);
   MOCK(connection_or_send_netinfo, mock_send_netinfo);
   MOCK(connection_or_close_for_error, mock_close_for_err);
+  MOCK(tor_tls_get_peer_cert, mock_get_peer_cert);
+
+  if (is_link) {
+    /* Say that this is the peer's certificate */
+    mock_peer_cert = tor_x509_cert_dup(a);
+  }
 
-  tt_int_op(0, ==, d->c->handshake_state->received_certs_cell);
-  tt_int_op(0, ==, mock_send_authenticate_called);
-  tt_int_op(0, ==, mock_send_netinfo_called);
+  tt_int_op(0, OP_EQ, d->c->handshake_state->received_certs_cell);
+  tt_int_op(0, OP_EQ, mock_send_authenticate_called);
+  tt_int_op(0, OP_EQ, mock_send_netinfo_called);
 
   return d;
  done:
@@ -318,11 +492,26 @@ test_link_handshake_recv_certs_ok(void *arg)
 {
   certs_data_t *d = arg;
   channel_tls_process_certs_cell(d->cell, d->chan);
-  tt_int_op(0, ==, mock_close_called);
-  tt_int_op(d->c->handshake_state->authenticated, ==, 1);
-  tt_int_op(d->c->handshake_state->received_certs_cell, ==, 1);
-  tt_assert(d->c->handshake_state->id_cert != NULL);
-  tt_assert(d->c->handshake_state->auth_cert == NULL);
+  tt_int_op(0, OP_EQ, mock_close_called);
+  tt_int_op(d->c->handshake_state->authenticated, OP_EQ, 1);
+  tt_int_op(d->c->handshake_state->authenticated_rsa, OP_EQ, 1);
+  tt_int_op(d->c->handshake_state->received_certs_cell, OP_EQ, 1);
+  tt_ptr_op(d->c->handshake_state->certs->id_cert, OP_NE, NULL);
+  tt_ptr_op(d->c->handshake_state->certs->auth_cert, OP_EQ, NULL);
+
+  if (d->is_ed) {
+    tt_ptr_op(d->c->handshake_state->certs->ed_id_sign, OP_NE, NULL);
+    tt_ptr_op(d->c->handshake_state->certs->ed_sign_link, OP_NE, NULL);
+    tt_ptr_op(d->c->handshake_state->certs->ed_sign_auth, OP_EQ, NULL);
+    tt_ptr_op(d->c->handshake_state->certs->ed_rsa_crosscert, OP_NE, NULL);
+    tt_int_op(d->c->handshake_state->authenticated_ed25519, OP_EQ, 1);
+  } else {
+    tt_ptr_op(d->c->handshake_state->certs->ed_id_sign, OP_EQ, NULL);
+    tt_ptr_op(d->c->handshake_state->certs->ed_sign_link, OP_EQ, NULL);
+    tt_ptr_op(d->c->handshake_state->certs->ed_sign_auth, OP_EQ, NULL);
+    tt_ptr_op(d->c->handshake_state->certs->ed_rsa_crosscert, OP_EQ, NULL);
+    tt_int_op(d->c->handshake_state->authenticated_ed25519, OP_EQ, 0);
+  }
 
  done:
   ;
@@ -333,17 +522,20 @@ test_link_handshake_recv_certs_ok_server(void *arg)
 {
   certs_data_t *d = arg;
   d->c->handshake_state->started_here = 0;
-  certs_cell_get_certs(d->ccell, 0)->cert_type = 3;
-  certs_cell_get_certs(d->ccell, 1)->cert_type = 2;
-  ssize_t n = certs_cell_encode(d->cell->payload, 2048, d->ccell);
-  tt_int_op(n, >, 0);
-  d->cell->payload_len = n;
+  d->c->handshake_state->certs->started_here = 0;
   channel_tls_process_certs_cell(d->cell, d->chan);
-  tt_int_op(0, ==, mock_close_called);
-  tt_int_op(d->c->handshake_state->authenticated, ==, 0);
-  tt_int_op(d->c->handshake_state->received_certs_cell, ==, 1);
-  tt_assert(d->c->handshake_state->id_cert != NULL);
-  tt_assert(d->c->handshake_state->auth_cert != NULL);
+  tt_int_op(0, OP_EQ, mock_close_called);
+  tt_int_op(d->c->handshake_state->authenticated, OP_EQ, 0);
+  tt_int_op(d->c->handshake_state->received_certs_cell, OP_EQ, 1);
+  tt_ptr_op(d->c->handshake_state->certs->id_cert, OP_NE, NULL);
+  tt_ptr_op(d->c->handshake_state->certs->link_cert, OP_EQ, NULL);
+  if (d->is_ed) {
+    tt_ptr_op(d->c->handshake_state->certs->ed_sign_auth, OP_NE, NULL);
+    tt_ptr_op(d->c->handshake_state->certs->auth_cert, OP_EQ, NULL);
+  } else {
+    tt_ptr_op(d->c->handshake_state->certs->ed_sign_auth, OP_EQ, NULL);
+    tt_ptr_op(d->c->handshake_state->certs->auth_cert, OP_NE, NULL);
+  }
 
  done:
   ;
@@ -358,9 +550,11 @@ test_link_handshake_recv_certs_ok_server(void *arg)
     setup_capture_of_logs(LOG_INFO);                                    \
     { code ; }                                                          \
     channel_tls_process_certs_cell(d->cell, d->chan);                   \
-    tt_int_op(1, ==, mock_close_called);                                \
-    tt_int_op(0, ==, mock_send_authenticate_called);                    \
-    tt_int_op(0, ==, mock_send_netinfo_called);                         \
+    tt_int_op(1, OP_EQ, mock_close_called);                                \
+    tt_int_op(0, OP_EQ, mock_send_authenticate_called);                    \
+    tt_int_op(0, OP_EQ, mock_send_netinfo_called);                         \
+    tt_int_op(0, OP_EQ, d->c->handshake_state->authenticated_rsa);         \
+    tt_int_op(0, OP_EQ, d->c->handshake_state->authenticated_ed25519);     \
     if (require_failure_message) {                                      \
       expect_log_msg_containing(require_failure_message);               \
     }                                                                   \
@@ -401,12 +595,41 @@ CERTS_FAIL(truncated_3,
              d->cell->payload_len = 7;
              memcpy(d->cell->payload, "\x01\x01\x00\x05""abc", 7);
            })
+CERTS_FAIL(truncated_4, /* ed25519 */
+           {
+             require_failure_message = "It couldn't be parsed";
+             d->cell->payload_len -= 10;
+           })
+CERTS_FAIL(truncated_5, /* ed25519 */
+           {
+             require_failure_message = "It couldn't be parsed";
+             d->cell->payload_len -= 100;
+           })
+
 #define REENCODE() do {                                                 \
+    const char *msg = certs_cell_check(d->ccell);                       \
+    if (msg) puts(msg);                                                 \
     ssize_t n = certs_cell_encode(d->cell->payload, 4096, d->ccell);    \
-    tt_int_op(n, >, 0);                                                 \
+    tt_int_op(n, OP_GT, 0);                                                 \
     d->cell->payload_len = n;                                           \
   } while (0)
 
+CERTS_FAIL(truncated_6, /* ed25519 */
+   {
+     /* truncate the link certificate */
+     require_failure_message = "undecodable Ed certificate";
+     certs_cell_cert_setlen_body(certs_cell_get_certs(d->ccell, 3), 7);
+     certs_cell_get_certs(d->ccell, 3)->cert_len = 7;
+     REENCODE();
+   })
+CERTS_FAIL(truncated_7, /* ed25519 */
+   {
+     /* truncate the crosscert */
+     require_failure_message = "Unparseable or overlong crosscert";
+     certs_cell_cert_setlen_body(certs_cell_get_certs(d->ccell, 4), 7);
+     certs_cell_get_certs(d->ccell, 4)->cert_len = 7;
+     REENCODE();
+   })
 CERTS_FAIL(not_x509,
   {
     require_failure_message = "Received undecodable certificate";
@@ -435,6 +658,219 @@ CERTS_FAIL(both_auth,
     certs_cell_get_certs(d->ccell, 1)->cert_type = 3;
     REENCODE();
   })
+CERTS_FAIL(duplicate_id, /* ed25519 */
+  {
+    require_failure_message = "Duplicate Ed25519 certificate";
+    certs_cell_get_certs(d->ccell, 2)->cert_type = 4;
+    certs_cell_get_certs(d->ccell, 3)->cert_type = 4;
+    REENCODE();
+  })
+CERTS_FAIL(duplicate_link, /* ed25519 */
+  {
+    require_failure_message = "Duplicate Ed25519 certificate";
+    certs_cell_get_certs(d->ccell, 2)->cert_type = 5;
+    certs_cell_get_certs(d->ccell, 3)->cert_type = 5;
+    REENCODE();
+  })
+CERTS_FAIL(duplicate_crosscert, /* ed25519 */
+  {
+    require_failure_message = "Duplicate RSA->Ed25519 crosscert";
+    certs_cell_get_certs(d->ccell, 2)->cert_type = 7;
+    certs_cell_get_certs(d->ccell, 3)->cert_type = 7;
+    REENCODE();
+  })
+static void
+test_link_handshake_recv_certs_missing_id(void *arg) /* ed25519 */
+{
+  certs_data_t *d = arg;
+  tt_int_op(certs_cell_getlen_certs(d->ccell), OP_EQ, 5);
+  certs_cell_set_certs(d->ccell, 2, certs_cell_get_certs(d->ccell, 4));
+  certs_cell_set0_certs(d->ccell, 4, NULL); /* prevent free */
+  certs_cell_setlen_certs(d->ccell, 4);
+  d->ccell->n_certs = 4;
+  REENCODE();
+
+  /* This handshake succeeds, but since we have no ID cert, we will
+   * just do the RSA handshake. */
+  channel_tls_process_certs_cell(d->cell, d->chan);
+  tt_int_op(0, OP_EQ, mock_close_called);
+  tt_int_op(0, OP_EQ, d->c->handshake_state->authenticated_ed25519);
+  tt_int_op(1, OP_EQ, d->c->handshake_state->authenticated_rsa);
+ done:
+  ;
+}
+CERTS_FAIL(missing_signing_key, /* ed25519 */
+  {
+    require_failure_message = "No Ed25519 signing key";
+    tt_int_op(certs_cell_getlen_certs(d->ccell), OP_EQ, 5);
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 2);
+    tt_int_op(cert->cert_type, OP_EQ, CERTTYPE_ED_ID_SIGN);
+    /* replace this with a valid master->signing cert, but with no
+     * signing key. */
+    const ed25519_keypair_t *mk = get_master_identity_keypair();
+    const ed25519_keypair_t *sk = get_master_signing_keypair();
+    tor_cert_t *bad_cert = tor_cert_create(mk, CERT_TYPE_ID_SIGNING,
+                                           &sk->pubkey, time(NULL), 86400,
+                                           0 /* don't include signer */);
+    certs_cell_cert_setlen_body(cert, bad_cert->encoded_len);
+    memcpy(certs_cell_cert_getarray_body(cert),
+           bad_cert->encoded, bad_cert->encoded_len);
+    cert->cert_len = bad_cert->encoded_len;
+    tor_cert_free(bad_cert);
+    REENCODE();
+  })
+CERTS_FAIL(missing_link, /* ed25519 */
+  {
+    require_failure_message = "No Ed25519 link key";
+    tt_int_op(certs_cell_getlen_certs(d->ccell), OP_EQ, 5);
+    certs_cell_set_certs(d->ccell, 3, certs_cell_get_certs(d->ccell, 4));
+    certs_cell_set0_certs(d->ccell, 4, NULL); /* prevent free */
+    certs_cell_setlen_certs(d->ccell, 4);
+    d->ccell->n_certs = 4;
+    REENCODE();
+  })
+CERTS_FAIL(missing_auth, /* ed25519 */
+  {
+    d->c->handshake_state->started_here = 0;
+    d->c->handshake_state->certs->started_here = 0;
+    require_failure_message = "No Ed25519 link authentication key";
+    tt_int_op(certs_cell_getlen_certs(d->ccell), OP_EQ, 5);
+    certs_cell_set_certs(d->ccell, 3, certs_cell_get_certs(d->ccell, 4));
+    certs_cell_set0_certs(d->ccell, 4, NULL); /* prevent free */
+    certs_cell_setlen_certs(d->ccell, 4);
+    d->ccell->n_certs = 4;
+    REENCODE();
+  })
+CERTS_FAIL(missing_crosscert, /* ed25519 */
+  {
+    require_failure_message = "Missing RSA->Ed25519 crosscert";
+    tt_int_op(certs_cell_getlen_certs(d->ccell), OP_EQ, 5);
+    certs_cell_setlen_certs(d->ccell, 4);
+    d->ccell->n_certs = 4;
+    REENCODE();
+  })
+CERTS_FAIL(missing_rsa_id, /* ed25519 */
+  {
+    require_failure_message = "Missing legacy RSA ID cert";
+    tt_int_op(certs_cell_getlen_certs(d->ccell), OP_EQ, 5);
+    certs_cell_set_certs(d->ccell, 1, certs_cell_get_certs(d->ccell, 4));
+    certs_cell_set0_certs(d->ccell, 4, NULL); /* prevent free */
+    certs_cell_setlen_certs(d->ccell, 4);
+    d->ccell->n_certs = 4;
+    REENCODE();
+  })
+CERTS_FAIL(link_mismatch, /* ed25519 */
+  {
+    require_failure_message = "Link certificate does not match "
+      "TLS certificate";
+    const tor_x509_cert_t *idc;
+    tor_tls_get_my_certs(1, NULL, &idc);
+    tor_x509_cert_free(mock_peer_cert);
+    /* Pretend that the peer cert was something else. */
+    mock_peer_cert = tor_x509_cert_dup(idc);
+    /* No reencode needed. */
+  })
+CERTS_FAIL(bad_ed_sig, /* ed25519 */
+  {
+    require_failure_message = "At least one Ed25519 certificate was "
+      "badly signed";
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 3);
+    uint8_t *body = certs_cell_cert_getarray_body(cert);
+    ssize_t body_len = certs_cell_cert_getlen_body(cert);
+    /* Frob a byte in the signature */
+    body[body_len - 13] ^= 7;
+    REENCODE();
+  })
+CERTS_FAIL(bad_crosscert, /*ed25519*/
+  {
+    require_failure_message = "Invalid RSA->Ed25519 crosscert";
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 4);
+    uint8_t *body = certs_cell_cert_getarray_body(cert);
+    ssize_t body_len = certs_cell_cert_getlen_body(cert);
+    /* Frob a byte in the signature */
+    body[body_len - 13] ^= 7;
+    REENCODE();
+  })
+CERTS_FAIL(bad_rsa_id_cert, /*ed25519*/
+  {
+    require_failure_message = "legacy RSA ID certificate was not valid";
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 1);
+    uint8_t *body;
+    /* Frob a byte in the signature, after making a new cert. (NSS won't let
+     * us just frob the old cert, since it will see that the issuer & serial
+     * number are the same, which will make it fail at an earlier stage than
+     * signature verification.) */
+    const tor_x509_cert_t *idc;
+    tor_x509_cert_t *newc;
+    tor_tls_get_my_certs(1, NULL, &idc);
+    time_t new_end = time(NULL) + 86400 * 10;
+    newc = tor_x509_cert_replace_expiration(idc, new_end, d->key2);
+    const uint8_t *encoded;
+    size_t encoded_len;
+    tor_x509_cert_get_der(newc, &encoded, &encoded_len);
+    certs_cell_cert_setlen_body(cert, encoded_len);
+    certs_cell_cert_set_cert_len(cert, encoded_len);
+    body = certs_cell_cert_getarray_body(cert);
+    memcpy(body, encoded, encoded_len);
+    body[encoded_len - 13] ^= 7;
+    REENCODE();
+    tor_x509_cert_free(newc);
+  })
+CERTS_FAIL(expired_rsa_id, /* both */
+  {
+    require_failure_message = "Certificate already expired";
+    /* we're going to replace the identity cert with an expired one. */
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 1);
+    const tor_x509_cert_t *idc;
+    tor_tls_get_my_certs(1, NULL, &idc);
+    tor_x509_cert_t *newc;
+    time_t new_end = time(NULL) - 86400 * 10;
+    newc = tor_x509_cert_replace_expiration(idc, new_end, d->key2);
+    const uint8_t *encoded;
+    size_t encoded_len;
+    tor_x509_cert_get_der(newc, &encoded, &encoded_len);
+    certs_cell_cert_setlen_body(cert, encoded_len);
+    certs_cell_cert_set_cert_len(cert, encoded_len);
+    memcpy(certs_cell_cert_getarray_body(cert), encoded, encoded_len);
+    REENCODE();
+    tor_x509_cert_free(newc);
+  })
+CERTS_FAIL(expired_ed_id, /* ed25519 */
+  {
+    /* we're going to replace the Ed Id->sign cert with an expired one. */
+    require_failure_message = "At least one certificate expired";
+    /* We don't need to re-sign, since we check for expiration first. */
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 2);
+    uint8_t *body = certs_cell_cert_getarray_body(cert);
+    /* The expiration field is bytes [2..5].  It is in HOURS since the
+     * epoch. */
+    set_uint32(body+2, htonl(24)); /* Back to jan 2, 1970. */
+    REENCODE();
+  })
+CERTS_FAIL(expired_ed_link, /* ed25519 */
+  {
+    /* we're going to replace the Ed Sign->link cert with an expired one. */
+    require_failure_message = "At least one certificate expired";
+    /* We don't need to re-sign, since we check for expiration first. */
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 3);
+    uint8_t *body = certs_cell_cert_getarray_body(cert);
+    /* The expiration field is bytes [2..5].  It is in HOURS since the
+     * epoch. */
+    set_uint32(body+2, htonl(24)); /* Back to jan 2, 1970. */
+    REENCODE();
+  })
+CERTS_FAIL(expired_crosscert, /* ed25519 */
+  {
+    /* we're going to replace the Ed Sign->link cert with an expired one. */
+    require_failure_message = "Crosscert is expired";
+    /* We don't need to re-sign, since we check for expiration first. */
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 4);
+    uint8_t *body = certs_cell_cert_getarray_body(cert);
+    /* The expiration field is bytes [32..35]. once again, HOURS. */
+    set_uint32(body+32, htonl(24)); /* Back to jan 2, 1970. */
+    REENCODE();
+  })
+
 CERTS_FAIL(wrong_labels_1,
   {
     require_failure_message = "The link certificate was not valid";
@@ -459,21 +895,26 @@ CERTS_FAIL(wrong_labels_2,
   })
 CERTS_FAIL(wrong_labels_3,
            {
-             require_failure_message = "The certs we wanted were missing";
+             require_failure_message =
+               "The certs we wanted (ID, Link) were missing";
              certs_cell_get_certs(d->ccell, 0)->cert_type = 2;
              certs_cell_get_certs(d->ccell, 1)->cert_type = 3;
              REENCODE();
            })
 CERTS_FAIL(server_missing_certs,
            {
-             require_failure_message = "The certs we wanted were missing";
+             require_failure_message =
+               "The certs we wanted (ID, Auth) were missing";
              d->c->handshake_state->started_here = 0;
+             d->c->handshake_state->certs->started_here = 0;
+
            })
 CERTS_FAIL(server_wrong_labels_1,
            {
              require_failure_message =
                "The authentication certificate was not valid";
              d->c->handshake_state->started_here = 0;
+             d->c->handshake_state->certs->started_here = 0;
              certs_cell_get_certs(d->ccell, 0)->cert_type = 2;
              certs_cell_get_certs(d->ccell, 1)->cert_type = 3;
              REENCODE();
@@ -487,31 +928,48 @@ test_link_handshake_send_authchallenge(void *arg)
   or_connection_t *c1 = or_connection_new(CONN_TYPE_OR, AF_INET);
   var_cell_t *cell1=NULL, *cell2=NULL;
 
+  crypto_pk_t *rsa0 = pk_generate(0), *rsa1 = pk_generate(1);
+  tt_int_op(tor_tls_context_init(TOR_TLS_CTX_IS_PUBLIC_SERVER,
+                                 rsa0, rsa1, 86400), OP_EQ, 0);
+  init_mock_ed_keys(rsa0);
+
   MOCK(connection_or_write_var_cell_to_buf, mock_write_var_cell);
 
-  tt_int_op(connection_init_or_handshake_state(c1, 0), ==, 0);
+  tt_int_op(connection_init_or_handshake_state(c1, 0), OP_EQ, 0);
   c1->base_.state = OR_CONN_STATE_OR_HANDSHAKING_V3;
-  tt_assert(! mock_got_var_cell);
-  tt_int_op(0, ==, connection_or_send_auth_challenge_cell(c1));
+  tt_ptr_op(mock_got_var_cell, OP_EQ, NULL);
+  tt_int_op(0, OP_EQ, connection_or_send_auth_challenge_cell(c1));
   cell1 = mock_got_var_cell;
-  tt_int_op(0, ==, connection_or_send_auth_challenge_cell(c1));
+  tt_int_op(0, OP_EQ, connection_or_send_auth_challenge_cell(c1));
   cell2 = mock_got_var_cell;
-  tt_int_op(36, ==, cell1->payload_len);
-  tt_int_op(36, ==, cell2->payload_len);
-  tt_int_op(0, ==, cell1->circ_id);
-  tt_int_op(0, ==, cell2->circ_id);
-  tt_int_op(CELL_AUTH_CHALLENGE, ==, cell1->command);
-  tt_int_op(CELL_AUTH_CHALLENGE, ==, cell2->command);
-
-  tt_mem_op("\x00\x01\x00\x01", ==, cell1->payload + 32, 4);
-  tt_mem_op("\x00\x01\x00\x01", ==, cell2->payload + 32, 4);
-  tt_mem_op(cell1->payload, !=, cell2->payload, 32);
+#ifdef HAVE_WORKING_TOR_TLS_GET_TLSSECRETS
+  tt_int_op(38, OP_EQ, cell1->payload_len);
+  tt_int_op(38, OP_EQ, cell2->payload_len);
+#else
+  tt_int_op(36, OP_EQ, cell1->payload_len);
+  tt_int_op(36, OP_EQ, cell2->payload_len);
+#endif
+  tt_int_op(0, OP_EQ, cell1->circ_id);
+  tt_int_op(0, OP_EQ, cell2->circ_id);
+  tt_int_op(CELL_AUTH_CHALLENGE, OP_EQ, cell1->command);
+  tt_int_op(CELL_AUTH_CHALLENGE, OP_EQ, cell2->command);
+
+#ifdef HAVE_WORKING_TOR_TLS_GET_TLSSECRETS
+  tt_mem_op("\x00\x02\x00\x01\x00\x03", OP_EQ, cell1->payload + 32, 6);
+  tt_mem_op("\x00\x02\x00\x01\x00\x03", OP_EQ, cell2->payload + 32, 6);
+#else
+  tt_mem_op("\x00\x01\x00\x03", OP_EQ, cell1->payload + 32, 4);
+  tt_mem_op("\x00\x01\x00\x03", OP_EQ, cell2->payload + 32, 4);
+#endif
+  tt_mem_op(cell1->payload, OP_NE, cell2->payload, 32);
 
  done:
   UNMOCK(connection_or_write_var_cell_to_buf);
-  connection_free_(TO_CONN(c1));
+  connection_free_minimal(TO_CONN(c1));
   tor_free(cell1);
   tor_free(cell2);
+  crypto_pk_free(rsa0);
+  crypto_pk_free(rsa1);
 }
 
 typedef struct authchallenge_data_s {
@@ -532,7 +990,7 @@ recv_authchallenge_cleanup(const struct testcase_t *test, void *obj)
 
   if (d) {
     tor_free(d->cell);
-    connection_free_(TO_CONN(d->c));
+    connection_free_minimal(TO_CONN(d->c));
     circuitmux_free(d->chan->base_.cmux);
     tor_free(d->chan);
     tor_free(d);
@@ -544,6 +1002,8 @@ static void *
 recv_authchallenge_setup(const struct testcase_t *test)
 {
   (void)test;
+
+  testing__connection_or_pretend_TLSSECRET_is_supported = 1;
   authchallenge_data_t *d = tor_malloc_zero(sizeof(*d));
   d->c = or_connection_new(CONN_TYPE_OR, AF_INET);
   d->chan = tor_malloc_zero(sizeof(*d->chan));
@@ -551,14 +1011,14 @@ recv_authchallenge_setup(const struct testcase_t *test)
   d->c->base_.address = tor_strdup("HaveAnAddress");
   d->c->base_.state = OR_CONN_STATE_OR_HANDSHAKING_V3;
   d->chan->conn = d->c;
-  tt_int_op(connection_init_or_handshake_state(d->c, 1), ==, 0);
+  tt_int_op(connection_init_or_handshake_state(d->c, 1), OP_EQ, 0);
   d->c->link_proto = 4;
   d->c->handshake_state->received_certs_cell = 1;
   d->cell = var_cell_new(128);
   d->cell->payload_len = 38;
-  d->cell->payload[33] = 2;
-  d->cell->payload[35] = 7;
-  d->cell->payload[37] = 1;
+  d->cell->payload[33] = 2; /* 2 methods */
+  d->cell->payload[35] = 7; /* This one isn't real */
+  d->cell->payload[37] = 1; /* This is the old RSA one. */
   d->cell->command = CELL_AUTH_CHALLENGE;
 
   get_options_mutable()->ORPort_set = 1;
@@ -566,10 +1026,9 @@ recv_authchallenge_setup(const struct testcase_t *test)
   MOCK(connection_or_close_for_error, mock_close_for_err);
   MOCK(connection_or_send_netinfo, mock_send_netinfo);
   MOCK(connection_or_send_authenticate_cell, mock_send_authenticate);
-
-  tt_int_op(0, ==, d->c->handshake_state->received_auth_challenge);
-  tt_int_op(0, ==, mock_send_authenticate_called);
-  tt_int_op(0, ==, mock_send_netinfo_called);
+  tt_int_op(0, OP_EQ, d->c->handshake_state->received_auth_challenge);
+  tt_int_op(0, OP_EQ, mock_send_authenticate_called);
+  tt_int_op(0, OP_EQ, mock_send_netinfo_called);
 
   return d;
  done:
@@ -588,10 +1047,30 @@ test_link_handshake_recv_authchallenge_ok(void *arg)
   authchallenge_data_t *d = arg;
 
   channel_tls_process_auth_challenge_cell(d->cell, d->chan);
-  tt_int_op(0, ==, mock_close_called);
-  tt_int_op(1, ==, d->c->handshake_state->received_auth_challenge);
-  tt_int_op(1, ==, mock_send_authenticate_called);
-  tt_int_op(1, ==, mock_send_netinfo_called);
+  tt_int_op(0, OP_EQ, mock_close_called);
+  tt_int_op(1, OP_EQ, d->c->handshake_state->received_auth_challenge);
+  tt_int_op(1, OP_EQ, mock_send_authenticate_called);
+  tt_int_op(1, OP_EQ, mock_send_netinfo_called);
+  tt_int_op(1, OP_EQ, mock_send_authenticate_called_with_type); /* RSA */
+ done:
+  ;
+}
+
+static void
+test_link_handshake_recv_authchallenge_ok_ed25519(void *arg)
+{
+  authchallenge_data_t *d = arg;
+
+  /* Add the ed25519 authentication mechanism here. */
+  d->cell->payload[33] = 3; /* 3 types are supported now. */
+  d->cell->payload[39] = 3;
+  d->cell->payload_len += 2;
+  channel_tls_process_auth_challenge_cell(d->cell, d->chan);
+  tt_int_op(0, OP_EQ, mock_close_called);
+  tt_int_op(1, OP_EQ, d->c->handshake_state->received_auth_challenge);
+  tt_int_op(1, OP_EQ, mock_send_authenticate_called);
+  tt_int_op(1, OP_EQ, mock_send_netinfo_called);
+  tt_int_op(3, OP_EQ, mock_send_authenticate_called_with_type); /* Ed25519 */
  done:
   ;
 }
@@ -603,10 +1082,10 @@ test_link_handshake_recv_authchallenge_ok_noserver(void *arg)
   get_options_mutable()->ORPort_set = 0;
 
   channel_tls_process_auth_challenge_cell(d->cell, d->chan);
-  tt_int_op(0, ==, mock_close_called);
-  tt_int_op(1, ==, d->c->handshake_state->received_auth_challenge);
-  tt_int_op(0, ==, mock_send_authenticate_called);
-  tt_int_op(0, ==, mock_send_netinfo_called);
+  tt_int_op(0, OP_EQ, mock_close_called);
+  tt_int_op(1, OP_EQ, d->c->handshake_state->received_auth_challenge);
+  tt_int_op(0, OP_EQ, mock_send_authenticate_called);
+  tt_int_op(0, OP_EQ, mock_send_netinfo_called);
  done:
   ;
 }
@@ -618,10 +1097,10 @@ test_link_handshake_recv_authchallenge_ok_unrecognized(void *arg)
   d->cell->payload[37] = 99;
 
   channel_tls_process_auth_challenge_cell(d->cell, d->chan);
-  tt_int_op(0, ==, mock_close_called);
-  tt_int_op(1, ==, d->c->handshake_state->received_auth_challenge);
-  tt_int_op(0, ==, mock_send_authenticate_called);
-  tt_int_op(1, ==, mock_send_netinfo_called);
+  tt_int_op(0, OP_EQ, mock_close_called);
+  tt_int_op(1, OP_EQ, d->c->handshake_state->received_auth_challenge);
+  tt_int_op(0, OP_EQ, mock_send_authenticate_called);
+  tt_int_op(1, OP_EQ, mock_send_netinfo_called);
  done:
   ;
 }
@@ -635,9 +1114,9 @@ test_link_handshake_recv_authchallenge_ok_unrecognized(void *arg)
     setup_capture_of_logs(LOG_INFO);                                    \
     { code ; }                                                          \
     channel_tls_process_auth_challenge_cell(d->cell, d->chan);          \
-    tt_int_op(1, ==, mock_close_called);                                \
-    tt_int_op(0, ==, mock_send_authenticate_called);                    \
-    tt_int_op(0, ==, mock_send_netinfo_called);                         \
+    tt_int_op(1, OP_EQ, mock_close_called);                                \
+    tt_int_op(0, OP_EQ, mock_send_authenticate_called);                    \
+    tt_int_op(0, OP_EQ, mock_send_netinfo_called);                         \
     if (require_failure_message) {                                      \
       expect_log_msg_containing(require_failure_message);               \
     }                                                                   \
@@ -655,7 +1134,8 @@ AUTHCHALLENGE_FAIL(badproto,
 AUTHCHALLENGE_FAIL(as_server,
                    require_failure_message = "We didn't originate this "
                      "connection";
-                   d->c->handshake_state->started_here = 0;)
+                   d->c->handshake_state->started_here = 0;
+                   d->c->handshake_state->certs->started_here = 0;)
 AUTHCHALLENGE_FAIL(duplicate,
                    require_failure_message = "We already received one";
                    d->c->handshake_state->received_auth_challenge = 1)
@@ -673,15 +1153,6 @@ AUTHCHALLENGE_FAIL(nonzero_circid,
                    require_failure_message = "It had a nonzero circuit ID";
                    d->cell->circ_id = 1337)
 
-static tor_x509_cert_t *mock_peer_cert = NULL;
-
-static tor_x509_cert_t *
-mock_get_peer_cert(tor_tls_t *tls)
-{
-  (void)tls;
-  return tor_x509_cert_dup(mock_peer_cert);
-}
-
 static int
 mock_get_tlssecrets(tor_tls_t *tls, uint8_t *secrets_out)
 {
@@ -701,6 +1172,7 @@ mock_set_circid_type(channel_t *chan,
 }
 
 typedef struct authenticate_data_s {
+  int is_ed;
   or_connection_t *c1, *c2;
   channel_tls_t *chan2;
   var_cell_t *cell;
@@ -717,13 +1189,14 @@ authenticate_data_cleanup(const struct testcase_t *test, void *arg)
   UNMOCK(tor_tls_get_tlssecrets);
   UNMOCK(connection_or_close_for_error);
   UNMOCK(channel_set_circid_type);
+  UNMOCK(tor_tls_export_key_material);
   authenticate_data_t *d = arg;
   if (d) {
     tor_free(d->cell);
-    connection_or_remove_from_identity_map(d->c1);
-    connection_or_remove_from_identity_map(d->c2);
-    connection_free_(TO_CONN(d->c1));
-    connection_free_(TO_CONN(d->c2));
+    connection_or_clear_identity(d->c1);
+    connection_or_clear_identity(d->c2);
+    connection_free_minimal(TO_CONN(d->c1));
+    connection_free_minimal(TO_CONN(d->c2));
     circuitmux_free(d->chan2->base_.cmux);
     tor_free(d->chan2);
     crypto_pk_free(d->key1);
@@ -742,6 +1215,9 @@ static void *
 authenticate_data_setup(const struct testcase_t *test)
 {
   authenticate_data_t *d = tor_malloc_zero(sizeof(*d));
+  int is_ed = d->is_ed = (test->setup_data == (void*)3);
+
+  testing__connection_or_pretend_TLSSECRET_is_supported = 1;
 
   scheduler_init();
 
@@ -751,6 +1227,7 @@ authenticate_data_setup(const struct testcase_t *test)
   MOCK(tor_tls_get_tlssecrets, mock_get_tlssecrets);
   MOCK(connection_or_close_for_error, mock_close_for_err);
   MOCK(channel_set_circid_type, mock_set_circid_type);
+  MOCK(tor_tls_export_key_material, mock_export_key_material);
   d->c1 = or_connection_new(CONN_TYPE_OR, AF_INET);
   d->c2 = or_connection_new(CONN_TYPE_OR, AF_INET);
   tor_addr_from_ipv4h(&d->c1->base_.addr, 0x01020304);
@@ -759,15 +1236,17 @@ authenticate_data_setup(const struct testcase_t *test)
   d->key1 = pk_generate(2);
   d->key2 = pk_generate(3);
   tt_int_op(tor_tls_context_init(TOR_TLS_CTX_IS_PUBLIC_SERVER,
-                                 d->key1, d->key2, 86400), ==, 0);
+                                 d->key1, d->key2, 86400), OP_EQ, 0);
+
+  init_mock_ed_keys(d->key2);
 
   d->c1->base_.state = OR_CONN_STATE_OR_HANDSHAKING_V3;
   d->c1->link_proto = 3;
-  tt_int_op(connection_init_or_handshake_state(d->c1, 1), ==, 0);
+  tt_int_op(connection_init_or_handshake_state(d->c1, 1), OP_EQ, 0);
 
   d->c2->base_.state = OR_CONN_STATE_OR_HANDSHAKING_V3;
   d->c2->link_proto = 3;
-  tt_int_op(connection_init_or_handshake_state(d->c2, 0), ==, 0);
+  tt_int_op(connection_init_or_handshake_state(d->c2, 0), OP_EQ, 0);
   var_cell_t *cell = var_cell_new(16);
   cell->command = CELL_CERTS;
   or_handshake_state_record_var_cell(d->c1, d->c1->handshake_state, cell, 1);
@@ -791,21 +1270,37 @@ authenticate_data_setup(const struct testcase_t *test)
   const uint8_t *der;
   size_t sz;
   tor_x509_cert_get_der(id_cert, &der, &sz);
-  d->c1->handshake_state->id_cert = tor_x509_cert_decode(der, sz);
-  d->c2->handshake_state->id_cert = tor_x509_cert_decode(der, sz);
+  d->c1->handshake_state->certs->id_cert = tor_x509_cert_decode(der, sz);
+  d->c2->handshake_state->certs->id_cert = tor_x509_cert_decode(der, sz);
+
+  if (is_ed) {
+    d->c1->handshake_state->certs->ed_id_sign =
+      tor_cert_dup(get_master_signing_key_cert());
+    d->c2->handshake_state->certs->ed_id_sign =
+      tor_cert_dup(get_master_signing_key_cert());
+    d->c2->handshake_state->certs->ed_sign_auth =
+      tor_cert_dup(get_current_auth_key_cert());
+  } else {
+    tt_assert(! tor_tls_get_my_certs(0, &auth_cert, &id_cert));
+    tor_x509_cert_get_der(auth_cert, &der, &sz);
+    d->c2->handshake_state->certs->auth_cert = tor_x509_cert_decode(der, sz);
+  }
 
   tor_x509_cert_get_der(link_cert, &der, &sz);
   mock_peer_cert = tor_x509_cert_decode(der, sz);
   tt_assert(mock_peer_cert);
+
   mock_own_cert = tor_x509_cert_decode(der, sz);
   tt_assert(mock_own_cert);
-  tt_assert(! tor_tls_get_my_certs(0, &auth_cert, &id_cert));
-  tor_x509_cert_get_der(auth_cert, &der, &sz);
-  d->c2->handshake_state->auth_cert = tor_x509_cert_decode(der, sz);
 
   /* Make an authenticate cell ... */
-  tt_int_op(0, ==, connection_or_send_authenticate_cell(d->c1,
-                                             AUTHTYPE_RSA_SHA256_TLSSECRET));
+  int authtype;
+  if (is_ed)
+    authtype = AUTHTYPE_ED25519_SHA256_RFC5705;
+  else
+    authtype = AUTHTYPE_RSA_SHA256_TLSSECRET;
+  tt_int_op(0, OP_EQ, connection_or_send_authenticate_cell(d->c1, authtype));
+
   tt_assert(mock_got_var_cell);
   d->cell = mock_got_var_cell;
   mock_got_var_cell = NULL;
@@ -829,44 +1324,66 @@ test_link_handshake_auth_cell(void *arg)
   crypto_pk_t *auth_pubkey = NULL;
 
   /* Is the cell well-formed on the outer layer? */
-  tt_int_op(d->cell->command, ==, CELL_AUTHENTICATE);
-  tt_int_op(d->cell->payload[0], ==, 0);
-  tt_int_op(d->cell->payload[1], ==, 1);
-  tt_int_op(ntohs(get_uint16(d->cell->payload + 2)), ==,
+  tt_int_op(d->cell->command, OP_EQ, CELL_AUTHENTICATE);
+  tt_int_op(d->cell->payload[0], OP_EQ, 0);
+  if (d->is_ed)
+    tt_int_op(d->cell->payload[1], OP_EQ, 3);
+  else
+    tt_int_op(d->cell->payload[1], OP_EQ, 1);
+  tt_int_op(ntohs(get_uint16(d->cell->payload + 2)), OP_EQ,
             d->cell->payload_len - 4);
 
   /* Check it out for plausibility... */
   auth_ctx_t ctx;
-  ctx.is_ed = 0;
-  tt_int_op(d->cell->payload_len-4, ==, auth1_parse(&auth1,
+  ctx.is_ed = d->is_ed;
+  tt_int_op(d->cell->payload_len-4, OP_EQ, auth1_parse(&auth1,
                                              d->cell->payload+4,
                                              d->cell->payload_len - 4, &ctx));
   tt_assert(auth1);
 
-  tt_mem_op(auth1->type, ==, "AUTH0001", 8);
-  tt_mem_op(auth1->tlssecrets, ==, "int getRandomNumber(){return 4;}", 32);
-  tt_int_op(auth1_getlen_sig(auth1), >, 120);
+  if (d->is_ed) {
+    tt_mem_op(auth1->type, OP_EQ, "AUTH0003", 8);
+  } else {
+    tt_mem_op(auth1->type, OP_EQ, "AUTH0001", 8);
+  }
+  tt_mem_op(auth1->tlssecrets, OP_EQ, "int getRandomNumber(){return 4;}", 32);
 
   /* Is the signature okay? */
-  uint8_t sig[128];
-  uint8_t digest[32];
-
-  auth_pubkey = tor_tls_cert_get_key(d->c2->handshake_state->auth_cert);
-  int n = crypto_pk_public_checksig(
+  const uint8_t *start = d->cell->payload+4, *end = auth1->end_of_signed;
+  if (d->is_ed) {
+    ed25519_signature_t sig;
+    tt_int_op(auth1_getlen_sig(auth1), OP_EQ, ED25519_SIG_LEN);
+    memcpy(&sig.sig, auth1_getarray_sig(auth1), ED25519_SIG_LEN);
+    tt_assert(!ed25519_checksig(&sig, start, end-start,
+                                &get_current_auth_keypair()->pubkey));
+  } else {
+    uint8_t sig[128];
+    uint8_t digest[32];
+    tt_int_op(auth1_getlen_sig(auth1), OP_GT, 120);
+    auth_pubkey = tor_tls_cert_get_key(
+                                d->c2->handshake_state->certs->auth_cert);
+    int n = crypto_pk_public_checksig(
               auth_pubkey,
               (char*)sig, sizeof(sig), (char*)auth1_getarray_sig(auth1),
               auth1_getlen_sig(auth1));
-  tt_int_op(n, ==, 32);
-  const uint8_t *start = d->cell->payload+4, *end = auth1->end_of_signed;
-  crypto_digest256((char*)digest,
-                   (const char*)start, end-start, DIGEST_SHA256);
-  tt_mem_op(sig, ==, digest, 32);
+    tt_int_op(n, OP_EQ, 32);
+    crypto_digest256((char*)digest,
+                     (const char*)start, end-start, DIGEST_SHA256);
+    tt_mem_op(sig, OP_EQ, digest, 32);
+  }
 
   /* Then feed it to c2. */
-  tt_int_op(d->c2->handshake_state->authenticated, ==, 0);
+  tt_int_op(d->c2->handshake_state->authenticated, OP_EQ, 0);
   channel_tls_process_authenticate_cell(d->cell, d->chan2);
-  tt_int_op(mock_close_called, ==, 0);
-  tt_int_op(d->c2->handshake_state->authenticated, ==, 1);
+  tt_int_op(mock_close_called, OP_EQ, 0);
+  tt_int_op(d->c2->handshake_state->authenticated, OP_EQ, 1);
+  if (d->is_ed) {
+    tt_int_op(d->c2->handshake_state->authenticated_ed25519, OP_EQ, 1);
+    tt_int_op(d->c2->handshake_state->authenticated_rsa, OP_EQ, 1);
+  } else {
+    tt_int_op(d->c2->handshake_state->authenticated_ed25519, OP_EQ, 0);
+    tt_int_op(d->c2->handshake_state->authenticated_rsa, OP_EQ, 1);
+  }
 
  done:
   auth1_free(auth1);
@@ -881,10 +1398,10 @@ test_link_handshake_auth_cell(void *arg)
     const char *require_failure_message = NULL;                 \
     setup_capture_of_logs(LOG_INFO);                            \
     { code ; }                                                  \
-    tt_int_op(d->c2->handshake_state->authenticated, ==, 0);    \
+    tt_int_op(d->c2->handshake_state->authenticated, OP_EQ, 0);    \
     channel_tls_process_authenticate_cell(d->cell, d->chan2);   \
-    tt_int_op(mock_close_called, ==, 1);                        \
-    tt_int_op(d->c2->handshake_state->authenticated, ==, 0);    \
+    tt_int_op(mock_close_called, OP_EQ, 1);                        \
+    tt_int_op(d->c2->handshake_state->authenticated, OP_EQ, 0);    \
     if (require_failure_message) {                              \
       expect_log_msg_containing(require_failure_message);       \
     }                                                           \
@@ -900,7 +1417,8 @@ AUTHENTICATE_FAIL(badproto,
                   d->c2->link_proto = 2)
 AUTHENTICATE_FAIL(atclient,
                   require_failure_message = "We originated this connection";
-                  d->c2->handshake_state->started_here = 1)
+                  d->c2->handshake_state->started_here = 1;
+                  d->c2->handshake_state->certs->started_here = 1;)
 AUTHENTICATE_FAIL(duplicate,
                   require_failure_message = "We already got one";
                   d->c2->handshake_state->received_authenticate = 1)
@@ -911,8 +1429,8 @@ test_link_handshake_auth_already_authenticated(void *arg)
   setup_capture_of_logs(LOG_INFO);
   d->c2->handshake_state->authenticated = 1;
   channel_tls_process_authenticate_cell(d->cell, d->chan2);
-  tt_int_op(mock_close_called, ==, 1);
-  tt_int_op(d->c2->handshake_state->authenticated, ==, 1);
+  tt_int_op(mock_close_called, OP_EQ, 1);
+  tt_int_op(d->c2->handshake_state->authenticated, OP_EQ, 1);
   expect_log_msg_containing("The peer is already authenticated");
  done:
   teardown_capture_of_logs();
@@ -924,13 +1442,13 @@ AUTHENTICATE_FAIL(nocerts,
 AUTHENTICATE_FAIL(noidcert,
                   require_failure_message = "We never got an identity "
                     "certificate";
-                  tor_x509_cert_free(d->c2->handshake_state->id_cert);
-                  d->c2->handshake_state->id_cert = NULL)
+                  tor_x509_cert_free(d->c2->handshake_state->certs->id_cert);
+                  d->c2->handshake_state->certs->id_cert = NULL)
 AUTHENTICATE_FAIL(noauthcert,
-                  require_failure_message = "We never got an authentication "
-                    "certificate";
-                  tor_x509_cert_free(d->c2->handshake_state->auth_cert);
-                  d->c2->handshake_state->auth_cert = NULL)
+                  require_failure_message = "We never got an RSA "
+                    "authentication certificate";
+                  tor_x509_cert_free(d->c2->handshake_state->certs->auth_cert);
+                  d->c2->handshake_state->certs->auth_cert = NULL)
 AUTHENTICATE_FAIL(tooshort,
                   require_failure_message = "Cell was way too short";
                   d->cell->payload_len = 3)
@@ -946,7 +1464,7 @@ AUTHENTICATE_FAIL(truncated_2,
                   d->cell->payload[3]++)
 AUTHENTICATE_FAIL(tooshort_1,
                   require_failure_message = "Authenticator was too short";
-                  tt_int_op(d->cell->payload_len, >=, 260);
+                  tt_int_op(d->cell->payload_len, OP_GE, 260);
                   d->cell->payload[2] -= 1;
                   d->cell->payload_len -= 256;)
 AUTHENTICATE_FAIL(badcontent,
@@ -954,11 +1472,33 @@ AUTHENTICATE_FAIL(badcontent,
                     "cell body was not as expected";
                   d->cell->payload[10] ^= 0xff)
 AUTHENTICATE_FAIL(badsig_1,
-                  require_failure_message = "Signature wasn't valid";
+                  if (d->is_ed)
+                    require_failure_message = "Ed25519 signature wasn't valid";
+                  else
+                    require_failure_message = "RSA signature wasn't valid";
                   d->cell->payload[d->cell->payload_len - 5] ^= 0xff)
-
-#define TEST(name, flags)                                       \
-  { #name , test_link_handshake_ ## name, (flags), NULL, NULL }
+AUTHENTICATE_FAIL(missing_ed_id,
+                {
+                  tor_cert_free(d->c2->handshake_state->certs->ed_id_sign);
+                  d->c2->handshake_state->certs->ed_id_sign = NULL;
+                  require_failure_message = "Ed authenticate without Ed ID "
+                    "cert from peer";
+                })
+AUTHENTICATE_FAIL(missing_ed_auth,
+                {
+                  tor_cert_free(d->c2->handshake_state->certs->ed_sign_auth);
+                  d->c2->handshake_state->certs->ed_sign_auth = NULL;
+                  require_failure_message = "We never got an Ed25519 "
+                    "authentication certificate";
+                })
+
+#define TEST_RSA(name, flags)                                           \
+  { #name , test_link_handshake_ ## name, (flags),                      \
+      &passthrough_setup, (void*)"RSA" }
+
+#define TEST_ED(name, flags)                                            \
+  { #name "_ed25519" , test_link_handshake_ ## name, (flags),           \
+      &passthrough_setup, (void*)"Ed25519" }
 
 #define TEST_RCV_AUTHCHALLENGE(name)                            \
   { "recv_authchallenge/" #name ,                               \
@@ -968,17 +1508,34 @@ AUTHENTICATE_FAIL(badsig_1,
 #define TEST_RCV_CERTS(name)                                    \
   { "recv_certs/" #name ,                                       \
       test_link_handshake_recv_certs_ ## name, TT_FORK,         \
-      &setup_recv_certs, NULL }
+      &setup_recv_certs, (void*)"RSA-Link" }
+
+#define TEST_RCV_CERTS_RSA(name,type)                           \
+  { "recv_certs/" #name ,                                       \
+      test_link_handshake_recv_certs_ ## name, TT_FORK,         \
+      &setup_recv_certs, (void*)type }
+
+#define TEST_RCV_CERTS_ED(name, type)                           \
+  { "recv_certs/" #name "_ed25519",                             \
+      test_link_handshake_recv_certs_ ## name, TT_FORK,         \
+      &setup_recv_certs, (void*)type }
 
 #define TEST_AUTHENTICATE(name)                                         \
   { "authenticate/" #name , test_link_handshake_auth_ ## name, TT_FORK, \
       &setup_authenticate, NULL }
 
+#define TEST_AUTHENTICATE_ED(name)                                      \
+  { "authenticate/" #name "_ed25519" , test_link_handshake_auth_ ## name, \
+      TT_FORK, &setup_authenticate, (void*)3 }
+
 struct testcase_t link_handshake_tests[] = {
-  TEST(certs_ok, TT_FORK),
-  //TEST(certs_bad, TT_FORK),
+  TEST_RSA(certs_ok, TT_FORK),
+  TEST_ED(certs_ok, TT_FORK),
+
   TEST_RCV_CERTS(ok),
-  TEST_RCV_CERTS(ok_server),
+  TEST_RCV_CERTS_ED(ok, "Ed25519-Link"),
+  TEST_RCV_CERTS_RSA(ok_server, "RSA-Auth"),
+  TEST_RCV_CERTS_ED(ok_server, "Ed25519-Auth"),
   TEST_RCV_CERTS(badstate),
   TEST_RCV_CERTS(badproto),
   TEST_RCV_CERTS(duplicate),
@@ -988,18 +1545,41 @@ struct testcase_t link_handshake_tests[] = {
   TEST_RCV_CERTS(truncated_1),
   TEST_RCV_CERTS(truncated_2),
   TEST_RCV_CERTS(truncated_3),
+  TEST_RCV_CERTS_ED(truncated_4, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(truncated_5, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(truncated_6, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(truncated_7, "Ed25519-Link"),
   TEST_RCV_CERTS(not_x509),
   TEST_RCV_CERTS(both_link),
   TEST_RCV_CERTS(both_id_rsa),
   TEST_RCV_CERTS(both_auth),
+  TEST_RCV_CERTS_ED(duplicate_id, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(duplicate_link, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(duplicate_crosscert, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(missing_crosscert, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(missing_id, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(missing_signing_key, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(missing_link, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(missing_auth, "Ed25519-Auth"),
+  TEST_RCV_CERTS_ED(missing_rsa_id, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(link_mismatch, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(bad_ed_sig, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(bad_rsa_id_cert, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(bad_crosscert, "Ed25519-Link"),
+  TEST_RCV_CERTS_RSA(expired_rsa_id, "RSA-Link"),
+  TEST_RCV_CERTS_ED(expired_rsa_id, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(expired_ed_id, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(expired_ed_link, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(expired_crosscert, "Ed25519-Link"),
   TEST_RCV_CERTS(wrong_labels_1),
   TEST_RCV_CERTS(wrong_labels_2),
   TEST_RCV_CERTS(wrong_labels_3),
   TEST_RCV_CERTS(server_missing_certs),
   TEST_RCV_CERTS(server_wrong_labels_1),
 
-  TEST(send_authchallenge, TT_FORK),
+  TEST_RSA(send_authchallenge, TT_FORK),
   TEST_RCV_AUTHCHALLENGE(ok),
+  TEST_RCV_AUTHCHALLENGE(ok_ed25519),
   TEST_RCV_AUTHCHALLENGE(ok_noserver),
   TEST_RCV_AUTHCHALLENGE(ok_unrecognized),
   TEST_RCV_AUTHCHALLENGE(badstate),
@@ -1012,6 +1592,7 @@ struct testcase_t link_handshake_tests[] = {
   TEST_RCV_AUTHCHALLENGE(nonzero_circid),
 
   TEST_AUTHENTICATE(cell),
+  TEST_AUTHENTICATE_ED(cell),
   TEST_AUTHENTICATE(badstate),
   TEST_AUTHENTICATE(badproto),
   TEST_AUTHENTICATE(atclient),
@@ -1027,8 +1608,10 @@ struct testcase_t link_handshake_tests[] = {
   TEST_AUTHENTICATE(tooshort_1),
   TEST_AUTHENTICATE(badcontent),
   TEST_AUTHENTICATE(badsig_1),
+  TEST_AUTHENTICATE_ED(badsig_1),
+  TEST_AUTHENTICATE_ED(missing_ed_id),
+  TEST_AUTHENTICATE_ED(missing_ed_auth),
   //TEST_AUTHENTICATE(),
 
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_logging.c b/src/test/test_logging.c
index 15471e46d0..95a2fce757 100644
--- a/src/test/test_logging.c
+++ b/src/test/test_logging.c
@@ -1,10 +1,19 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#define CONFIG_PRIVATE
+
 #include "orconfig.h"
-#include "or.h"
-#include "torlog.h"
-#include "test.h"
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "lib/err/torerr.h"
+#include "lib/log/log.h"
+#include "test/test.h"
+#include "lib/process/subprocess.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
 
 static void
 dummy_cb_fn(int severity, uint32_t domain, const char *msg)
@@ -89,7 +98,7 @@ test_sigsafe_err(void *arg)
 
   init_logging(1);
   mark_logs_temp();
-  add_file_log(&include_bug, fn, 0);
+  open_and_add_file_log(&include_bug, fn, 0);
   tor_log_update_sigsafe_err_fds();
   close_temp_logs();
 
@@ -107,7 +116,7 @@ test_sigsafe_err(void *arg)
   close(STDERR_FILENO);
   content = read_file_to_str(fn, 0, NULL);
 
-  tt_assert(content != NULL);
+  tt_ptr_op(content, OP_NE, NULL);
   tor_split_lines(lines, content, (int)strlen(content));
   tt_int_op(smartlist_len(lines), OP_GE, 5);
 
@@ -140,7 +149,7 @@ test_ratelim(void *arg)
   char *msg = NULL;
 
   msg = rate_limit_log(&ten_min, now);
-  tt_assert(msg != NULL);
+  tt_ptr_op(msg, OP_NE, NULL);
   tt_str_op(msg, OP_EQ, ""); /* nothing was suppressed. */
 
   tt_int_op(ten_min.last_allowed, OP_EQ, now);
@@ -150,14 +159,14 @@ test_ratelim(void *arg)
   for (i = 0; i < 9; ++i) {
     now += 60; /* one minute has passed. */
     msg = rate_limit_log(&ten_min, now);
-    tt_assert(msg == NULL);
+    tt_ptr_op(msg, OP_EQ, NULL);
     tt_int_op(ten_min.last_allowed, OP_EQ, start);
     tt_int_op(ten_min.n_calls_since_last_time, OP_EQ, i + 1);
   }
 
   now += 240; /* Okay, we can be done. */
   msg = rate_limit_log(&ten_min, now);
-  tt_assert(msg != NULL);
+  tt_ptr_op(msg, OP_NE, NULL);
   tt_str_op(msg, OP_EQ,
             " [9 similar message(s) suppressed in last 600 seconds]");
  done:
@@ -170,4 +179,3 @@ struct testcase_t logging_tests[] = {
   { "ratelim", test_ratelim, 0, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_mainloop.c b/src/test/test_mainloop.c
new file mode 100644
index 0000000000..089ea812cf
--- /dev/null
+++ b/src/test/test_mainloop.c
@@ -0,0 +1,142 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_mainloop.c
+ * \brief Tests for functions closely related to the Tor main loop
+ */
+
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+
+#include "core/or/or.h"
+#include "core/mainloop/mainloop.h"
+
+static const uint64_t BILLION = 1000000000;
+
+static void
+test_mainloop_update_time_normal(void *arg)
+{
+  (void)arg;
+
+  monotime_enable_test_mocking();
+  /* This is arbitrary */
+  uint64_t mt_now = UINT64_C(7493289274986);
+  /* This time is in the past as of when this test was written. */
+  time_t now = 1525272090;
+  monotime_coarse_set_mock_time_nsec(mt_now);
+  reset_uptime();
+  update_current_time(now);
+  tt_int_op(approx_time(), OP_EQ, now);
+  tt_int_op(get_uptime(), OP_EQ, 0);
+
+  update_current_time(now); // Same time as before is a no-op.
+  tt_int_op(get_uptime(), OP_EQ, 0);
+
+  now += 1;
+  mt_now += BILLION;
+  monotime_coarse_set_mock_time_nsec(mt_now);
+  update_current_time(now);
+  tt_int_op(approx_time(), OP_EQ, now);
+  tt_int_op(get_uptime(), OP_EQ, 1);
+
+  now += 2; // two-second jump is unremarkable.
+  mt_now += 2*BILLION;
+  update_current_time(now);
+  monotime_coarse_set_mock_time_nsec(mt_now);
+  tt_int_op(approx_time(), OP_EQ, now);
+  tt_int_op(get_uptime(), OP_EQ, 3);
+
+  now -= 1; // a one-second hop backwards is also unremarkable.
+  update_current_time(now);
+  tt_int_op(approx_time(), OP_EQ, now); // it changes the approx time...
+  tt_int_op(get_uptime(), OP_EQ, 3); // but it doesn't roll back our uptime
+
+ done:
+  monotime_disable_test_mocking();
+}
+
+static void
+test_mainloop_update_time_jumps(void *arg)
+{
+  (void)arg;
+
+  monotime_enable_test_mocking();
+  /* This is arbitrary */
+  uint64_t mt_now = UINT64_C(7493289274986);
+  /* This time is in the past as of when this test was written. */
+  time_t now = 220897152;
+  monotime_coarse_set_mock_time_nsec(mt_now);
+  reset_uptime();
+  update_current_time(now);
+  tt_int_op(approx_time(), OP_EQ, now);
+  tt_int_op(get_uptime(), OP_EQ, 0);
+
+  /* Put some uptime on the clock.. */
+  now += 3;
+  mt_now += 3*BILLION;
+  monotime_coarse_set_mock_time_nsec(mt_now);
+  update_current_time(now);
+  tt_int_op(approx_time(), OP_EQ, now);
+  tt_int_op(get_uptime(), OP_EQ, 3);
+
+  /* Now try jumping forward and backward, without updating the monotonic
+   * clock.  */
+  setup_capture_of_logs(LOG_NOTICE);
+  now += 1800;
+  update_current_time(now);
+  expect_single_log_msg_containing(
+               "Your system clock just jumped 1800 seconds forward");
+  tt_int_op(approx_time(), OP_EQ, now);
+  tt_int_op(get_uptime(), OP_EQ, 3); // no uptime change.
+  mock_clean_saved_logs();
+
+  now -= 600;
+  update_current_time(now);
+  expect_single_log_msg_containing(
+               "Your system clock just jumped 600 seconds backward");
+  tt_int_op(approx_time(), OP_EQ, now);
+  tt_int_op(get_uptime(), OP_EQ, 3); // no uptime change.
+  mock_clean_saved_logs();
+
+  /* uptime tracking should go normally now if the clock moves sensibly. */
+  now += 2;
+  mt_now += 2*BILLION;
+  update_current_time(now);
+  tt_int_op(approx_time(), OP_EQ, now);
+  tt_int_op(get_uptime(), OP_EQ, 5);
+
+  /* If we skip forward by a few minutes but the monotonic clock agrees,
+   * we've just been idle: that counts as not worth warning about. */
+  now += 1800;
+  mt_now += 1800*BILLION;
+  monotime_coarse_set_mock_time_nsec(mt_now);
+  update_current_time(now);
+  expect_no_log_entry();
+  tt_int_op(approx_time(), OP_EQ, now);
+  tt_int_op(get_uptime(), OP_EQ, 5); // this doesn't count to uptime, though.
+
+  /* If we skip forward by a long time, even if the clock agrees, it's
+   * idnless that counts. */
+  now += 4000;
+  mt_now += 4000*BILLION;
+  monotime_coarse_set_mock_time_nsec(mt_now);
+  update_current_time(now);
+  expect_single_log_msg_containing("Tor has been idle for 4000 seconds");
+  tt_int_op(approx_time(), OP_EQ, now);
+  tt_int_op(get_uptime(), OP_EQ, 5);
+
+ done:
+  teardown_capture_of_logs();
+  monotime_disable_test_mocking();
+}
+
+#define MAINLOOP_TEST(name) \
+  { #name, test_mainloop_## name , TT_FORK, NULL, NULL }
+
+struct testcase_t mainloop_tests[] = {
+  MAINLOOP_TEST(update_time_normal),
+  MAINLOOP_TEST(update_time_jumps),
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c
index 2ae605b8db..4c4317d81a 100644
--- a/src/test/test_microdesc.c
+++ b/src/test/test_microdesc.c
@@ -1,31 +1,36 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
-#include "or.h"
-
-#include "config.h"
-#include "dirvote.h"
-#include "microdesc.h"
-#include "networkstatus.h"
-#include "routerlist.h"
-#include "routerparse.h"
-#include "torcert.h"
-
-#include "test.h"
-
-DISABLE_GCC_WARNING(redundant-decls)
-#include <openssl/rsa.h>
-#include <openssl/bn.h>
-#include <openssl/pem.h>
-ENABLE_GCC_WARNING(redundant-decls)
+#include "core/or/or.h"
+
+#define DIRVOTE_PRIVATE
+#include "app/config/config.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirparse/microdesc_parse.h"
+#include "feature/dirparse/routerparse.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/torcert.h"
+
+#include "feature/nodelist/microdesc_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+#include "test/test.h"
+
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
 
 #ifdef _WIN32
 /* For mkdir() */
 #include <direct.h>
 #else
 #include <dirent.h>
-#endif
+#endif /* defined(_WIN32) */
 
 static const char test_md1[] =
   "onion-key\n"
@@ -75,12 +80,12 @@ test_md_cache(void *data)
   time3 = time(NULL) - 15*24*60*60;
 
   /* Possibly, turn this into a test setup/cleanup pair */
-  tor_free(options->DataDirectory);
-  options->DataDirectory = tor_strdup(get_fname("md_datadir_test"));
+  tor_free(options->CacheDirectory);
+  options->CacheDirectory = tor_strdup(get_fname("md_datadir_test"));
 #ifdef _WIN32
-  tt_int_op(0, OP_EQ, mkdir(options->DataDirectory));
+  tt_int_op(0, OP_EQ, mkdir(options->CacheDirectory));
 #else
-  tt_int_op(0, OP_EQ, mkdir(options->DataDirectory, 0700));
+  tt_int_op(0, OP_EQ, mkdir(options->CacheDirectory, 0700));
 #endif
 
   tt_assert(!strcmpstart(test_md3_noannotation, "onion-key"));
@@ -158,7 +163,7 @@ test_md_cache(void *data)
               strlen(test_md3_noannotation));
 
   tor_asprintf(&fn, "%s"PATH_SEPARATOR"cached-microdescs.new",
-               options->DataDirectory);
+               options->CacheDirectory);
   s = read_file_to_str(fn, RFTS_BIN, NULL);
   tt_assert(s);
   tt_mem_op(md1->body, OP_EQ, s + md1->off, md1->bodylen);
@@ -186,7 +191,7 @@ test_md_cache(void *data)
 
   /* read the cache. */
   tor_asprintf(&fn, "%s"PATH_SEPARATOR"cached-microdescs",
-               options->DataDirectory);
+               options->CacheDirectory);
   s = read_file_to_str(fn, RFTS_BIN, NULL);
   tt_mem_op(md1->body, OP_EQ, s + md1->off, strlen(test_md1));
   tt_mem_op(md2->body, OP_EQ, s + md2->off, strlen(test_md2));
@@ -240,7 +245,7 @@ test_md_cache(void *data)
 
  done:
   if (options)
-    tor_free(options->DataDirectory);
+    tor_free(options->CacheDirectory);
   microdesc_free_all();
 
   smartlist_free(added);
@@ -272,17 +277,17 @@ test_md_cache_broken(void *data)
 
   options = get_options_mutable();
   tt_assert(options);
-  tor_free(options->DataDirectory);
-  options->DataDirectory = tor_strdup(get_fname("md_datadir_test2"));
+  tor_free(options->CacheDirectory);
+  options->CacheDirectory = tor_strdup(get_fname("md_datadir_test2"));
 
 #ifdef _WIN32
-  tt_int_op(0, OP_EQ, mkdir(options->DataDirectory));
+  tt_int_op(0, OP_EQ, mkdir(options->CacheDirectory));
 #else
-  tt_int_op(0, OP_EQ, mkdir(options->DataDirectory, 0700));
+  tt_int_op(0, OP_EQ, mkdir(options->CacheDirectory, 0700));
 #endif
 
   tor_asprintf(&fn, "%s"PATH_SEPARATOR"cached-microdescs",
-               options->DataDirectory);
+               options->CacheDirectory);
 
   write_str_to_file(fn, truncated_md, 1);
 
@@ -291,7 +296,7 @@ test_md_cache_broken(void *data)
 
  done:
   if (options)
-    tor_free(options->DataDirectory);
+    tor_free(options->CacheDirectory);
   tor_free(fn);
   microdesc_free_all();
 }
@@ -391,25 +396,6 @@ static const char test_ri2[] =
   "cf34GXHv61XReJF3AlzNHFpbrPOYmowmhrTULKyMqow=\n"
   "-----END SIGNATURE-----\n";
 
-static const char test_md_8[] =
-  "onion-key\n"
-  "-----BEGIN RSA PUBLIC KEY-----\n"
-  "MIGJAoGBANBJz8Vldl12aFeSMPLiA4nOetLDN0oxU8bB1SDhO7Uu2zdWYVYAF5J0\n"
-  "st7WvrVy/jA9v/fsezNAPskBanecHRSkdMTpkcgRPMHE7CTGEwIy1Yp1X4bPgDlC\n"
-  "VCnbs5Pcts5HnWEYNK7qHDAUn+IlmjOO+pTUY8uyq+GQVz6H9wFlAgMBAAE=\n"
-  "-----END RSA PUBLIC KEY-----\n"
-  "p reject 25,119,135-139,445,563,1214,4661-4666,6346-6429,6699,6881-6999\n";
-
-static const char test_md_16[] =
-  "onion-key\n"
-  "-----BEGIN RSA PUBLIC KEY-----\n"
-  "MIGJAoGBANBJz8Vldl12aFeSMPLiA4nOetLDN0oxU8bB1SDhO7Uu2zdWYVYAF5J0\n"
-  "st7WvrVy/jA9v/fsezNAPskBanecHRSkdMTpkcgRPMHE7CTGEwIy1Yp1X4bPgDlC\n"
-  "VCnbs5Pcts5HnWEYNK7qHDAUn+IlmjOO+pTUY8uyq+GQVz6H9wFlAgMBAAE=\n"
-  "-----END RSA PUBLIC KEY-----\n"
-  "ntor-onion-key Gg73xH7+kTfT6bi1uNVx9gwQdQas9pROIfmc4NpAdC4=\n"
-  "p reject 25,119,135-139,445,563,1214,4661-4666,6346-6429,6699,6881-6999\n";
-
 static const char test_md_18[] =
   "onion-key\n"
   "-----BEGIN RSA PUBLIC KEY-----\n"
@@ -421,16 +407,6 @@ static const char test_md_18[] =
   "p reject 25,119,135-139,445,563,1214,4661-4666,6346-6429,6699,6881-6999\n"
   "id rsa1024 Cd47okjCHD83YGzThGBDptXs9Z4\n";
 
-static const char test_md2_18[] =
-  "onion-key\n"
-  "-----BEGIN RSA PUBLIC KEY-----\n"
-  "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n"
-  "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n"
-  "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n"
-  "-----END RSA PUBLIC KEY-----\n"
-  "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n"
-  "id rsa1024 t+J/EEITw28T5+mCkYKEXklZl6A\n";
-
 static const char test_md2_21[] =
   "onion-key\n"
   "-----BEGIN RSA PUBLIC KEY-----\n"
@@ -450,17 +426,6 @@ test_md_generate(void *arg)
 
   ri = router_parse_entry_from_string(test_ri, NULL, 0, 0, NULL, NULL);
   tt_assert(ri);
-  md = dirvote_create_microdescriptor(ri, 8);
-  tt_str_op(md->body, OP_EQ, test_md_8);
-
-  /* XXXX test family lines. */
-  /* XXXX test method 14 for A lines. */
-  /* XXXX test method 15 for P6 lines. */
-
-  microdesc_free(md);
-  md = NULL;
-  md = dirvote_create_microdescriptor(ri, 16);
-  tt_str_op(md->body, OP_EQ, test_md_16);
 
   microdesc_free(md);
   md = NULL;
@@ -470,20 +435,15 @@ test_md_generate(void *arg)
   microdesc_free(md);
   md = NULL;
   md = dirvote_create_microdescriptor(ri, 21);
-  tt_str_op(md->body, ==, test_md_18);
+  tt_str_op(md->body, OP_EQ, test_md_18);
 
   routerinfo_free(ri);
   ri = router_parse_entry_from_string(test_ri2, NULL, 0, 0, NULL, NULL);
 
   microdesc_free(md);
   md = NULL;
-  md = dirvote_create_microdescriptor(ri, 18);
-  tt_str_op(md->body, ==, test_md2_18);
-
-  microdesc_free(md);
-  md = NULL;
   md = dirvote_create_microdescriptor(ri, 21);
-  tt_str_op(md->body, ==, test_md2_21);
+  tt_str_op(md->body, OP_EQ, test_md2_21);
   tt_assert(ed25519_pubkey_eq(md->ed25519_identity_pkey,
                               &ri->cache_info.signing_key_cert->signing_key));
 
@@ -760,8 +720,8 @@ test_md_reject_cache(void *arg)
   or_options_t *options = get_options_mutable();
   char buf[DIGEST256_LEN];
 
-  tor_free(options->DataDirectory);
-  options->DataDirectory = tor_strdup(get_fname("md_datadir_test_rej"));
+  tor_free(options->CacheDirectory);
+  options->CacheDirectory = tor_strdup(get_fname("md_datadir_test_rej"));
   mock_rgsbd_val_a = tor_malloc_zero(sizeof(routerstatus_t));
   mock_rgsbd_val_b = tor_malloc_zero(sizeof(routerstatus_t));
   mock_ns_val = tor_malloc_zero(sizeof(networkstatus_t));
@@ -771,9 +731,9 @@ test_md_reject_cache(void *arg)
   mock_ns_val->flavor = FLAV_MICRODESC;
 
 #ifdef _WIN32
-  tt_int_op(0, OP_EQ, mkdir(options->DataDirectory));
+  tt_int_op(0, OP_EQ, mkdir(options->CacheDirectory));
 #else
-  tt_int_op(0, OP_EQ, mkdir(options->DataDirectory, 0700));
+  tt_int_op(0, OP_EQ, mkdir(options->CacheDirectory, 0700));
 #endif
 
   MOCK(router_get_mutable_consensus_status_by_descriptor_digest,
@@ -808,7 +768,7 @@ test_md_reject_cache(void *arg)
  done:
   UNMOCK(networkstatus_get_latest_consensus_by_flavor);
   UNMOCK(router_get_mutable_consensus_status_by_descriptor_digest);
-  tor_free(options->DataDirectory);
+  tor_free(options->CacheDirectory);
   microdesc_free_all();
   smartlist_free(added);
   SMARTLIST_FOREACH(wanted, char *, cp, tor_free(cp));
@@ -829,14 +789,14 @@ test_md_corrupt_desc(void *arg)
                                "@last-listed 2015-06-22 10:00:00\n"
                                "onion-k\n",
                                NULL, SAVED_IN_JOURNAL, 0, time(NULL), NULL);
-  tt_int_op(smartlist_len(sl), ==, 0);
+  tt_int_op(smartlist_len(sl), OP_EQ, 0);
   smartlist_free(sl);
 
   sl = microdescs_add_to_cache(get_microdesc_cache(),
                                "@last-listed 2015-06-22 10:00:00\n"
                                "wiggly\n",
                                NULL, SAVED_IN_JOURNAL, 0, time(NULL), NULL);
-  tt_int_op(smartlist_len(sl), ==, 0);
+  tt_int_op(smartlist_len(sl), OP_EQ, 0);
   smartlist_free(sl);
 
   tor_asprintf(&cp, "%s\n%s", test_md1, "@foobar\nonion-wobble\n");
@@ -844,7 +804,7 @@ test_md_corrupt_desc(void *arg)
   sl = microdescs_add_to_cache(get_microdesc_cache(),
                                cp, cp+strlen(cp),
                                SAVED_IN_JOURNAL, 0, time(NULL), NULL);
-  tt_int_op(smartlist_len(sl), ==, 0);
+  tt_int_op(smartlist_len(sl), OP_EQ, 0);
 
  done:
   tor_free(cp);
@@ -860,4 +820,3 @@ struct testcase_t microdesc_tests[] = {
   { "corrupt_desc", test_md_corrupt_desc, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c
index d58f8a7fca..53eb0413e5 100644
--- a/src/test/test_nodelist.c
+++ b/src/test/test_nodelist.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2016, The Tor Project, Inc. */
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -6,9 +6,20 @@
  * \brief Unit tests for nodelist related functions.
  **/
 
-#include "or.h"
-#include "nodelist.h"
-#include "test.h"
+#include "core/or/or.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/torcert.h"
+
+#include "feature/nodelist/microdesc_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+#include "test/test.h"
+#include "test/log_test_helpers.h"
 
 /** Test the case when node_get_by_id() returns NULL,
  * node_get_verbose_nickname_by_id should return the base 16 encoding
@@ -100,6 +111,126 @@ test_nodelist_node_is_dir(void *arg)
   return;
 }
 
+static networkstatus_t *dummy_ns = NULL;
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus(void)
+{
+  return dummy_ns;
+}
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f)
+{
+  tor_assert(f == FLAV_MICRODESC);
+  return dummy_ns;
+}
+
+static void
+test_nodelist_ed_id(void *arg)
+{
+#define N_NODES 5
+  routerstatus_t *rs[N_NODES];
+  microdesc_t *md[N_NODES];
+  routerinfo_t *ri[N_NODES];
+  networkstatus_t *ns;
+  int i;
+  (void)arg;
+
+  ns = tor_malloc_zero(sizeof(networkstatus_t));
+  ns->flavor = FLAV_MICRODESC;
+  ns->routerstatus_list = smartlist_new();
+  dummy_ns = ns;
+  MOCK(networkstatus_get_latest_consensus,
+       mock_networkstatus_get_latest_consensus);
+  MOCK(networkstatus_get_latest_consensus_by_flavor,
+       mock_networkstatus_get_latest_consensus_by_flavor);
+
+  /* Make a bunch of dummy objects that we can play around with.  Only set the
+     necessary fields */
+
+  for (i = 0; i < N_NODES; ++i) {
+    rs[i] = tor_malloc_zero(sizeof(*rs[i]));
+    md[i] = tor_malloc_zero(sizeof(*md[i]));
+    ri[i] = tor_malloc_zero(sizeof(*ri[i]));
+
+    crypto_rand(md[i]->digest, sizeof(md[i]->digest));
+    md[i]->ed25519_identity_pkey = tor_malloc(sizeof(ed25519_public_key_t));
+    crypto_rand((char*)md[i]->ed25519_identity_pkey,
+                sizeof(ed25519_public_key_t));
+    crypto_rand(rs[i]->identity_digest, sizeof(rs[i]->identity_digest));
+    memcpy(ri[i]->cache_info.identity_digest, rs[i]->identity_digest,
+           DIGEST_LEN);
+    memcpy(rs[i]->descriptor_digest, md[i]->digest, DIGEST256_LEN);
+    ri[i]->cache_info.signing_key_cert = tor_malloc_zero(sizeof(tor_cert_t));
+    memcpy(&ri[i]->cache_info.signing_key_cert->signing_key,
+           md[i]->ed25519_identity_pkey, sizeof(ed25519_public_key_t));
+
+    if (i < 3)
+      smartlist_add(ns->routerstatus_list, rs[i]);
+  }
+
+  tt_int_op(0, OP_EQ, smartlist_len(nodelist_get_list()));
+
+  nodelist_set_consensus(ns);
+
+  tt_int_op(3, OP_EQ, smartlist_len(nodelist_get_list()));
+
+  /* No Ed25519 info yet, so nothing has an ED id. */
+  tt_ptr_op(NULL, OP_EQ, node_get_by_ed25519_id(md[0]->ed25519_identity_pkey));
+
+  /* Register the first one by md, then look it up. */
+  node_t *n = nodelist_add_microdesc(md[0]);
+  tt_ptr_op(n, OP_EQ, node_get_by_ed25519_id(md[0]->ed25519_identity_pkey));
+
+  /* Register the second by ri, then look it up. */
+  routerinfo_t *ri_old = NULL;
+  n = nodelist_set_routerinfo(ri[1], &ri_old);
+  tt_ptr_op(n, OP_EQ, node_get_by_ed25519_id(md[1]->ed25519_identity_pkey));
+  tt_ptr_op(ri_old, OP_EQ, NULL);
+
+  /* Register it by md too. */
+  node_t *n2 = nodelist_add_microdesc(md[1]);
+  tt_ptr_op(n2, OP_EQ, n);
+  tt_ptr_op(n, OP_EQ, node_get_by_ed25519_id(md[1]->ed25519_identity_pkey));
+
+  /* Register the 4th by ri only -- we never put it into the networkstatus,
+   * so it has to be independent */
+  node_t *n3 = nodelist_set_routerinfo(ri[3], &ri_old);
+  tt_ptr_op(n3, OP_EQ, node_get_by_ed25519_id(md[3]->ed25519_identity_pkey));
+  tt_ptr_op(ri_old, OP_EQ, NULL);
+  tt_int_op(4, OP_EQ, smartlist_len(nodelist_get_list()));
+
+  /* Register the 5th by ri only, and rewrite its ed25519 pubkey to be
+   * the same as the 4th, to test the duplicate ed25519 key logging in
+   * nodelist.c */
+  memcpy(md[4]->ed25519_identity_pkey, md[3]->ed25519_identity_pkey,
+         sizeof(ed25519_public_key_t));
+  memcpy(&ri[4]->cache_info.signing_key_cert->signing_key,
+         md[3]->ed25519_identity_pkey, sizeof(ed25519_public_key_t));
+
+  setup_capture_of_logs(LOG_NOTICE);
+  node_t *n4 = nodelist_set_routerinfo(ri[4], &ri_old);
+  tt_ptr_op(ri_old, OP_EQ, NULL);
+  tt_int_op(5, OP_EQ, smartlist_len(nodelist_get_list()));
+  tt_ptr_op(n4, OP_NE, node_get_by_ed25519_id(md[3]->ed25519_identity_pkey));
+  tt_ptr_op(n3, OP_EQ, node_get_by_ed25519_id(md[3]->ed25519_identity_pkey));
+  expect_log_msg_containing("Reused ed25519_id");
+
+ done:
+  teardown_capture_of_logs();
+  for (i = 0; i < N_NODES; ++i) {
+    tor_free(rs[i]);
+    tor_free(md[i]->ed25519_identity_pkey);
+    tor_free(md[i]);
+    tor_free(ri[i]->cache_info.signing_key_cert);
+    tor_free(ri[i]);
+  }
+  smartlist_clear(ns->routerstatus_list);
+  networkstatus_vote_free(ns);
+  UNMOCK(networkstatus_get_latest_consensus);
+  UNMOCK(networkstatus_get_latest_consensus_by_flavor);
+#undef N_NODES
+}
+
 #define NODE(name, flags) \
   { #name, test_nodelist_##name, (flags), NULL, NULL }
 
@@ -107,6 +238,7 @@ struct testcase_t nodelist_tests[] = {
   NODE(node_get_verbose_nickname_by_id_null_node, TT_FORK),
   NODE(node_get_verbose_nickname_not_named, TT_FORK),
   NODE(node_is_dir, TT_FORK),
+  NODE(ed_id, TT_FORK),
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_ntor_cl.c b/src/test/test_ntor_cl.c
index a560e5fc5e..68b6927f56 100644
--- a/src/test/test_ntor_cl.c
+++ b/src/test/test_ntor_cl.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -6,12 +6,11 @@
 #include <stdlib.h>
 
 #define ONION_NTOR_PRIVATE
-#include "or.h"
-#include "util.h"
-#include "compat.h"
-#include "crypto.h"
-#include "crypto_curve25519.h"
-#include "onion_ntor.h"
+#include "core/or/or.h"
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "lib/crypt_ops/crypto_init.h"
+#include "core/crypto/onion_ntor.h"
 
 #define N_ARGS(n) STMT_BEGIN {                                  \
     if (argc < (n)) {                                           \
@@ -155,7 +154,11 @@ main(int argc, char **argv)
     return 1;
   }
 
+  init_logging(1);
   curve25519_init();
+  if (crypto_global_init(0, NULL, NULL) < 0)
+    return 1;
+
   if (!strcmp(argv[1], "client1")) {
     return client1(argc, argv);
   } else if (!strcmp(argv[1], "server1")) {
@@ -167,4 +170,3 @@ main(int argc, char **argv)
     return 1;
   }
 }
-
diff --git a/src/test/test_oom.c b/src/test/test_oom.c
index 6102af01f5..b813fa43a3 100644
--- a/src/test/test_oom.c
+++ b/src/test/test_oom.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /* Unit tests for OOM handling logic */
@@ -7,14 +7,21 @@
 #define BUFFERS_PRIVATE
 #define CIRCUITLIST_PRIVATE
 #define CONNECTION_PRIVATE
-#include "or.h"
-#include "buffers.h"
-#include "circuitlist.h"
-#include "compat_libevent.h"
-#include "connection.h"
-#include "config.h"
-#include "relay.h"
-#include "test.h"
+#include "core/or/or.h"
+#include "lib/container/buffers.h"
+#include "core/or/circuitlist.h"
+#include "lib/evloop/compat_libevent.h"
+#include "core/mainloop/connection.h"
+#include "app/config/config.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "core/or/relay.h"
+#include "test/test.h"
+#include "test/test_helpers.h"
+
+#include "core/or/cell_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
 
 /* small replacement mock for circuit_mark_for_close_ to avoid doing all
  * the other bookkeeping that comes with marking circuits. */
@@ -58,24 +65,6 @@ dummy_or_circuit_new(int n_p_cells, int n_n_cells)
   return TO_CIRCUIT(circ);
 }
 
-static circuit_t *
-dummy_origin_circuit_new(int n_cells)
-{
-  origin_circuit_t *circ = origin_circuit_new();
-  int i;
-  cell_t cell;
-
-  for (i=0; i < n_cells; ++i) {
-    crypto_rand((void*)&cell, sizeof(cell));
-    cell_queue_append_packed_copy(TO_CIRCUIT(circ),
-                                  &TO_CIRCUIT(circ)->n_chan_cells,
-                                  1, &cell, 1, 0);
-  }
-
-  TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_C_GENERAL;
-  return TO_CIRCUIT(circ);
-}
-
 static void
 add_bytes_to_buf(buf_t *buf, size_t n_bytes)
 {
@@ -84,7 +73,7 @@ add_bytes_to_buf(buf_t *buf, size_t n_bytes)
   while (n_bytes) {
     size_t this_add = n_bytes > sizeof(b) ? sizeof(b) : n_bytes;
     crypto_rand(b, this_add);
-    write_to_buf(b, this_add, buf);
+    buf_add(buf, b, this_add);
     n_bytes -= this_add;
   }
 }
@@ -219,7 +208,7 @@ test_oom_streambuf(void *arg)
 {
   or_options_t *options = get_options_mutable();
   circuit_t *c1 = NULL, *c2 = NULL, *c3 = NULL, *c4 = NULL, *c5 = NULL;
-  uint32_t tvms;
+  uint32_t tvts;
   int i;
   smartlist_t *edgeconns = smartlist_new();
   const uint64_t start_ns = 1389641159 * (uint64_t)1000000000;
@@ -287,22 +276,28 @@ test_oom_streambuf(void *arg)
   now_ns -= now_ns % 1000000000;
   now_ns += 1000000000;
   monotime_coarse_set_mock_time_nsec(now_ns);
-  tvms = (uint32_t) monotime_coarse_absolute_msec();
+  tvts = monotime_coarse_get_stamp();
+
+#define ts_is_approx(ts, val) do {                                   \
+    uint32_t x_ = (uint32_t) monotime_coarse_stamp_units_to_approx_msec(ts); \
+    tt_int_op(x_, OP_GE, val - 5);                                      \
+    tt_int_op(x_, OP_LE, val + 5);                                      \
+  } while (0)
 
-  tt_int_op(circuit_max_queued_cell_age(c1, tvms), OP_EQ, 500);
-  tt_int_op(circuit_max_queued_cell_age(c2, tvms), OP_EQ, 490);
-  tt_int_op(circuit_max_queued_cell_age(c3, tvms), OP_EQ, 480);
-  tt_int_op(circuit_max_queued_cell_age(c4, tvms), OP_EQ, 0);
+  ts_is_approx(circuit_max_queued_cell_age(c1, tvts), 500);
+  ts_is_approx(circuit_max_queued_cell_age(c2, tvts), 490);
+  ts_is_approx(circuit_max_queued_cell_age(c3, tvts), 480);
+  ts_is_approx(circuit_max_queued_cell_age(c4, tvts), 0);
 
-  tt_int_op(circuit_max_queued_data_age(c1, tvms), OP_EQ, 390);
-  tt_int_op(circuit_max_queued_data_age(c2, tvms), OP_EQ, 380);
-  tt_int_op(circuit_max_queued_data_age(c3, tvms), OP_EQ, 0);
-  tt_int_op(circuit_max_queued_data_age(c4, tvms), OP_EQ, 370);
+  ts_is_approx(circuit_max_queued_data_age(c1, tvts), 390);
+  ts_is_approx(circuit_max_queued_data_age(c2, tvts), 380);
+  ts_is_approx(circuit_max_queued_data_age(c3, tvts), 0);
+  ts_is_approx(circuit_max_queued_data_age(c4, tvts), 370);
 
-  tt_int_op(circuit_max_queued_item_age(c1, tvms), OP_EQ, 500);
-  tt_int_op(circuit_max_queued_item_age(c2, tvms), OP_EQ, 490);
-  tt_int_op(circuit_max_queued_item_age(c3, tvms), OP_EQ, 480);
-  tt_int_op(circuit_max_queued_item_age(c4, tvms), OP_EQ, 370);
+  ts_is_approx(circuit_max_queued_item_age(c1, tvts), 500);
+  ts_is_approx(circuit_max_queued_item_age(c2, tvts), 490);
+  ts_is_approx(circuit_max_queued_item_age(c3, tvts), 480);
+  ts_is_approx(circuit_max_queued_item_age(c4, tvts), 370);
 
   tt_int_op(cell_queues_get_total_allocation(), OP_EQ,
             packed_cell_mem_cost() * 80);
@@ -318,7 +313,7 @@ test_oom_streambuf(void *arg)
     smartlist_add(edgeconns, ec);
   }
   tt_int_op(buf_get_total_allocation(), OP_EQ, 4096*17*2);
-  tt_int_op(circuit_max_queued_item_age(c4, tvms), OP_EQ, 1000);
+  ts_is_approx(circuit_max_queued_item_age(c4, tvts), 1000);
 
   tt_int_op(cell_queues_check_size(), OP_EQ, 0);
 
@@ -352,7 +347,7 @@ test_oom_streambuf(void *arg)
   circuit_free(c5);
 
   SMARTLIST_FOREACH(edgeconns, edge_connection_t *, ec,
-                    connection_free_(TO_CONN(ec)));
+                    connection_free_minimal(TO_CONN(ec)));
   smartlist_free(edgeconns);
 
   UNMOCK(circuit_mark_for_close_);
diff --git a/src/test/test_oos.c b/src/test/test_oos.c
index db06625116..815feda7ce 100644
--- a/src/test/test_oos.c
+++ b/src/test/test_oos.c
@@ -1,16 +1,20 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /* Unit tests for OOS handler */
 
 #define CONNECTION_PRIVATE
 
-#include "or.h"
-#include "config.h"
-#include "connection.h"
-#include "connection_or.h"
-#include "main.h"
-#include "test.h"
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/connection_or.h"
+#include "feature/dircommon/directory.h"
+#include "core/mainloop/mainloop.h"
+#include "test/test.h"
+
+#include "feature/dircommon/dir_connection_st.h"
+#include "core/or/or_connection_st.h"
 
 static or_options_t mock_options;
 
@@ -52,7 +56,7 @@ kill_conn_list_mock(smartlist_t *conns)
 {
   ++kill_conn_list_calls;
 
-  tt_assert(conns != NULL);
+  tt_ptr_op(conns, OP_NE, NULL);
 
   kill_conn_list_killed += smartlist_len(conns);
 
@@ -248,7 +252,7 @@ close_for_error_mock(or_connection_t *orconn, int flush)
 {
   (void)flush;
 
-  tt_assert(orconn != NULL);
+  tt_ptr_op(orconn, OP_NE, NULL);
   ++cfe_calls;
 
  done:
@@ -264,7 +268,7 @@ mark_for_close_oos_mock(connection_t *conn,
   (void)line;
   (void)file;
 
-  tt_assert(conn != NULL);
+  tt_ptr_op(conn, OP_NE, NULL);
   ++mark_calls;
 
  done:
@@ -298,8 +302,8 @@ test_oos_kill_conn_list(void *arg)
   dir_c2->base_.purpose = DIR_PURPOSE_MIN_;
   c2 = TO_CONN(dir_c2);
 
-  tt_assert(c1 != NULL);
-  tt_assert(c2 != NULL);
+  tt_ptr_op(c1, OP_NE, NULL);
+  tt_ptr_op(c2, OP_NE, NULL);
 
   /* Make list */
   l = smartlist_new();
@@ -345,7 +349,7 @@ get_num_circuits_mock(or_connection_t *conn)
 {
   int circs = 0;
 
-  tt_assert(conn != NULL);
+  tt_ptr_op(conn, OP_NE, NULL);
 
   if (conns_with_circs &&
       smartlist_contains(conns_with_circs, TO_CONN(conn))) {
@@ -397,7 +401,7 @@ test_oos_pick_oos_victims(void *arg)
   /* Try picking one */
   picked = pick_oos_victims(1);
   /* It should be the one with circuits */
-  tt_assert(picked != NULL);
+  tt_ptr_op(picked, OP_NE, NULL);
   tt_int_op(smartlist_len(picked), OP_EQ, 1);
   tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0)));
   smartlist_free(picked);
@@ -405,14 +409,14 @@ test_oos_pick_oos_victims(void *arg)
   /* Try picking none */
   picked = pick_oos_victims(0);
   /* We should get an empty list */
-  tt_assert(picked != NULL);
+  tt_ptr_op(picked, OP_NE, NULL);
   tt_int_op(smartlist_len(picked), OP_EQ, 0);
   smartlist_free(picked);
 
   /* Try picking two */
   picked = pick_oos_victims(2);
   /* We should get both active orconns */
-  tt_assert(picked != NULL);
+  tt_ptr_op(picked, OP_NE, NULL);
   tt_int_op(smartlist_len(picked), OP_EQ, 2);
   tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0)));
   tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 1)));
@@ -453,4 +457,3 @@ struct testcase_t oos_tests[] = {
   { "pick_oos_victims", test_oos_pick_oos_victims, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_options.c b/src/test/test_options.c
index e85e11805b..66b0e7ef11 100644
--- a/src/test/test_options.c
+++ b/src/test/test_options.c
@@ -1,23 +1,31 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONFIG_PRIVATE
-#include "or.h"
-#include "confparse.h"
-#include "config.h"
-#include "test.h"
-#include "geoip.h"
+#include "core/or/or.h"
+#include "app/config/confparse.h"
+#include "app/config/config.h"
+#include "test/test.h"
+#include "lib/geoip/geoip.h"
 
 #define ROUTERSET_PRIVATE
-#include "routerset.h"
-#include "main.h"
-#include "log_test_helpers.h"
-
-#include "sandbox.h"
-#include "memarea.h"
-#include "policies.h"
+#include "feature/nodelist/routerset.h"
+#include "core/mainloop/mainloop.h"
+#include "test/log_test_helpers.h"
+
+#include "lib/sandbox/sandbox.h"
+#include "lib/memarea/memarea.h"
+#include "lib/osinfo/uname.h"
+#include "lib/encoding/confline.h"
+#include "core/or/policies.h"
+#include "test/test_helpers.h"
+#include "lib/net/resolve.h"
+
+#ifdef HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
 
 #define NS_MODULE test_options
 
@@ -104,11 +112,71 @@ clear_log_messages(void)
   "EDE6D711294FADF8E7951F4DE6CA56B58 194.109.206.212:80 7EA6 EAD6 FD83" \
   " 083C 538F 4403 8BBF A077 587D D755\n"
 
+static int
+test_options_checklog(const char *configuration, int expect_log_severity,
+                      const char *expect_log)
+{
+  int found = 0, ret = -1;
+  char *actual_log = NULL;
+
+  if (messages) {
+    SMARTLIST_FOREACH_BEGIN(messages, logmsg_t *, m) {
+      if (m->severity == expect_log_severity &&
+          strstr(m->msg, expect_log)) {
+        found = 1;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(m);
+  }
+  if (!found) {
+    actual_log = dump_logs();
+    TT_DIE(("Expected log message [%s] %s from <%s>, but got <%s>.",
+            log_level_to_string(expect_log_severity), expect_log,
+            configuration, actual_log));
+  }
+  ret = 0;
+
+ done:
+  tor_free(actual_log);
+  return ret;
+}
+
+static int
+test_options_checkmsgs(const char *configuration,
+                       const char *expect_errmsg,
+                       int expect_log_severity,
+                       const char *expect_log,
+                       char *msg)
+{
+  if (expect_errmsg && !msg) {
+    TT_DIE(("Expected error message <%s> from <%s>, but got none.",
+            expect_errmsg, configuration));
+  } else if (expect_errmsg && !strstr(msg, expect_errmsg)) {
+    TT_DIE(("Expected error message <%s> from <%s>, but got <%s>.",
+            expect_errmsg, configuration, msg));
+  } else if (!expect_errmsg && msg) {
+    TT_DIE(("Expected no error message from <%s> but got <%s>.",
+            configuration, msg));
+  }
+  if (expect_log) {
+    return test_options_checklog(configuration, expect_log_severity,
+                                 expect_log);
+  }
+  return 0;
+
+ done:
+  return -1;
+}
+
+/* Which phases of config parsing/validation to check for messages/logs */
+enum { PH_GETLINES, PH_ASSIGN, PH_VALIDATE };
+
 static void
 test_options_validate_impl(const char *configuration,
                            const char *expect_errmsg,
                            int expect_log_severity,
-                           const char *expect_log)
+                           const char *expect_log,
+                           int phase)
 {
   or_options_t *opt=NULL;
   or_options_t *dflt;
@@ -119,43 +187,34 @@ test_options_validate_impl(const char *configuration,
   setup_options(opt, dflt);
 
   r = config_get_lines(configuration, &cl, 1);
-  tt_int_op(r, OP_EQ, 0);
+  if (phase == PH_GETLINES) {
+    if (test_options_checkmsgs(configuration, expect_errmsg,
+                               expect_log_severity,
+                               expect_log, msg))
+      goto done;
+  }
+  if (r)
+    goto done;
 
   r = config_assign(&options_format, opt, cl, 0, &msg);
-  tt_int_op(r, OP_EQ, 0);
-
-  r = options_validate(NULL, opt, dflt, 0, &msg);
-  if (expect_errmsg && !msg) {
-    TT_DIE(("Expected error message <%s> from <%s>, but got none.",
-            expect_errmsg, configuration));
-  } else if (expect_errmsg && !strstr(msg, expect_errmsg)) {
-    TT_DIE(("Expected error message <%s> from <%s>, but got <%s>.",
-            expect_errmsg, configuration, msg));
-  } else if (!expect_errmsg && msg) {
-    TT_DIE(("Expected no error message from <%s> but got <%s>.",
-            configuration, msg));
+  if (phase == PH_ASSIGN) {
+    if (test_options_checkmsgs(configuration, expect_errmsg,
+                               expect_log_severity,
+                               expect_log, msg))
+      goto done;
   }
   tt_int_op((r == 0), OP_EQ, (msg == NULL));
+  if (r)
+    goto done;
 
-  if (expect_log) {
-    int found = 0;
-    if (messages) {
-      SMARTLIST_FOREACH_BEGIN(messages, logmsg_t *, m) {
-        if (m->severity == expect_log_severity &&
-            strstr(m->msg, expect_log)) {
-          found = 1;
-          break;
-        }
-      } SMARTLIST_FOREACH_END(m);
-    }
-    if (!found) {
-      tor_free(msg);
-      msg = dump_logs();
-      TT_DIE(("Expected log message [%s] %s from <%s>, but got <%s>.",
-              log_level_to_string(expect_log_severity), expect_log,
-              configuration, msg));
-    }
+  r = options_validate(NULL, opt, dflt, 0, &msg);
+  if (phase == PH_VALIDATE) {
+    if (test_options_checkmsgs(configuration, expect_errmsg,
+                               expect_log_severity,
+                               expect_log, msg))
+      goto done;
   }
+  tt_int_op((r == 0), OP_EQ, (msg == NULL));
 
  done:
   escaped(NULL);
@@ -167,14 +226,14 @@ test_options_validate_impl(const char *configuration,
   clear_log_messages();
 }
 
-#define WANT_ERR(config, msg)                           \
-  test_options_validate_impl((config), (msg), 0, NULL)
-#define WANT_LOG(config, severity, msg)                         \
-  test_options_validate_impl((config), NULL, (severity), (msg))
-#define WANT_ERR_LOG(config, msg, severity, logmsg)                     \
-  test_options_validate_impl((config), (msg), (severity), (logmsg))
-#define OK(config)                                      \
-  test_options_validate_impl((config), NULL, 0, NULL)
+#define WANT_ERR(config, msg, ph)                               \
+  test_options_validate_impl((config), (msg), 0, NULL, (ph))
+#define WANT_LOG(config, severity, msg, ph)                             \
+  test_options_validate_impl((config), NULL, (severity), (msg), (ph))
+#define WANT_ERR_LOG(config, msg, severity, logmsg, ph)                 \
+  test_options_validate_impl((config), (msg), (severity), (logmsg), (ph))
+#define OK(config, ph)                                          \
+  test_options_validate_impl((config), NULL, 0, NULL, (ph))
 
 static void
 test_options_validate(void *arg)
@@ -183,28 +242,46 @@ test_options_validate(void *arg)
   setup_log_callback();
   sandbox_disable_getaddrinfo_cache();
 
-  WANT_ERR("ExtORPort 500000", "Invalid ExtORPort");
+  WANT_ERR("ExtORPort 500000", "Invalid ExtORPort", PH_VALIDATE);
 
   WANT_ERR_LOG("ServerTransportOptions trebuchet",
                "ServerTransportOptions did not parse",
-               LOG_WARN, "Too few arguments");
-  OK("ServerTransportOptions trebuchet sling=snappy");
-  OK("ServerTransportOptions trebuchet sling=");
+               LOG_WARN, "Too few arguments", PH_VALIDATE);
+  OK("ServerTransportOptions trebuchet sling=snappy", PH_VALIDATE);
+  OK("ServerTransportOptions trebuchet sling=", PH_VALIDATE);
   WANT_ERR_LOG("ServerTransportOptions trebuchet slingsnappy",
                "ServerTransportOptions did not parse",
-               LOG_WARN, "\"slingsnappy\" is not a k=v");
+               LOG_WARN, "\"slingsnappy\" is not a k=v", PH_VALIDATE);
 
   WANT_ERR("DirPort 8080\nDirCache 0",
-           "DirPort configured but DirCache disabled.");
+           "DirPort configured but DirCache disabled.", PH_VALIDATE);
   WANT_ERR("BridgeRelay 1\nDirCache 0",
-           "We're a bridge but DirCache is disabled.");
+           "We're a bridge but DirCache is disabled.", PH_VALIDATE);
+
+  WANT_ERR_LOG("HeartbeatPeriod 21 snarks",
+               "Interval 'HeartbeatPeriod 21 snarks' is malformed or"
+               " out of bounds.", LOG_WARN, "Unknown unit 'snarks'.",
+               PH_ASSIGN);
+  WANT_ERR_LOG("LogTimeGranularity 21 snarks",
+               "Msec interval 'LogTimeGranularity 21 snarks' is malformed or"
+               " out of bounds.", LOG_WARN, "Unknown unit 'snarks'.",
+               PH_ASSIGN);
+  OK("HeartbeatPeriod 1 hour", PH_VALIDATE);
+  OK("LogTimeGranularity 100 milliseconds", PH_VALIDATE);
+
+  WANT_LOG("ControlSocket \"string with trailing garbage\" bogus", LOG_WARN,
+           "Error while parsing configuration: "
+           "Excess data after quoted string", PH_GETLINES);
+  WANT_LOG("ControlSocket \"bogus escape \\@\"", LOG_WARN,
+           "Error while parsing configuration: "
+           "Invalid escape sequence in quoted string", PH_GETLINES);
 
   close_temp_logs();
   clear_log_messages();
   return;
 }
 
-#define MEGABYTEIFY(mb) (U64_LITERAL(mb) << 20)
+#define MEGABYTEIFY(mb) (UINT64_C(mb) << 20)
 static void
 test_have_enough_mem_for_dircache(void *arg)
 {
@@ -212,7 +289,7 @@ test_have_enough_mem_for_dircache(void *arg)
   or_options_t *opt=NULL;
   or_options_t *dflt=NULL;
   config_line_t *cl=NULL;
-  char *msg=NULL;;
+  char *msg=NULL;
   int r;
   const char *configuration = "ORPort 8080\nDirCache 1", *expect_errmsg;
 
@@ -229,7 +306,7 @@ test_have_enough_mem_for_dircache(void *arg)
   /* 300 MB RAM available, DirCache enabled */
   r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(300), &msg);
   tt_int_op(r, OP_EQ, 0);
-  tt_assert(!msg);
+  tt_ptr_op(msg, OP_EQ, NULL);
 
   /* 200 MB RAM available, DirCache enabled */
   r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(200), &msg);
@@ -252,7 +329,7 @@ test_have_enough_mem_for_dircache(void *arg)
   /* 300 MB RAM available, DirCache enabled, Bridge */
   r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(300), &msg);
   tt_int_op(r, OP_EQ, 0);
-  tt_assert(!msg);
+  tt_ptr_op(msg, OP_EQ, NULL);
 
   /* 200 MB RAM available, DirCache enabled, Bridge */
   r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(200), &msg);
@@ -275,7 +352,7 @@ test_have_enough_mem_for_dircache(void *arg)
   /* 200 MB RAM available, DirCache disabled */
   r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(200), &msg);
   tt_int_op(r, OP_EQ, 0);
-  tt_assert(!msg);
+  tt_ptr_op(msg, OP_EQ, NULL);
 
   /* 300 MB RAM available, DirCache disabled */
   r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(300), &msg);
@@ -307,17 +384,11 @@ fixed_get_uname(void)
 }
 
 #define TEST_OPTIONS_OLD_VALUES   "TestingV3AuthInitialVotingInterval 1800\n" \
-  "ClientBootstrapConsensusMaxDownloadTries 7\n" \
-  "ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries 4\n" \
   "ClientBootstrapConsensusMaxInProgressTries 3\n" \
   "TestingV3AuthInitialVoteDelay 300\n"   \
   "TestingV3AuthInitialDistDelay 300\n" \
   "TestingClientMaxIntervalWithoutRequest 600\n" \
   "TestingDirConnectionMaxStall 600\n" \
-  "TestingConsensusMaxDownloadTries 8\n" \
-  "TestingDescriptorMaxDownloadTries 8\n" \
-  "TestingMicrodescMaxDownloadTries 8\n" \
-  "TestingCertMaxDownloadTries 8\n"
 
 #define TEST_OPTIONS_DEFAULT_VALUES TEST_OPTIONS_OLD_VALUES \
   "MaxClientCircuitsPending 1\n"                                        \
@@ -328,11 +399,12 @@ fixed_get_uname(void)
   "V3AuthVoteDelay 20\n"                                                \
   "V3AuthDistDelay 20\n"                                                \
   "V3AuthNIntervalsValid 3\n"                                           \
-  "ClientUseIPv4 1\n"                                                     \
+  "ClientUseIPv4 1\n"                                                   \
   "VirtualAddrNetworkIPv4 127.192.0.0/10\n"                             \
   "VirtualAddrNetworkIPv6 [FE80::]/10\n"                                \
-  "SchedulerHighWaterMark__ 42\n"                                       \
-  "SchedulerLowWaterMark__ 10\n"
+  "UseEntryGuards 1\n"                                                  \
+  "Schedulers Vanilla\n"                                                \
+  "ClientDNSRejectInternalAddresses 1\n"
 
 typedef struct {
   or_options_t *old_opt;
@@ -352,25 +424,31 @@ get_options_test_data(const char *conf)
   result->opt = options_new();
   result->old_opt = options_new();
   result->def_opt = options_new();
+
+  // XXX: Really, all of these options should be set to defaults
+  // with options_init(), but about a dozen tests break when I do that.
+  // Being kinda lame and just fixing the immedate breakage for now..
+  result->opt->ConnectionPadding = -1; // default must be "auto"
+
   rv = config_get_lines(conf, &cl, 1);
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
   rv = config_assign(&options_format, result->opt, cl, 0, &msg);
   if (msg) {
     /* Display the parse error message by comparing it with an empty string */
     tt_str_op(msg, OP_EQ, "");
   }
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
   config_free_lines(cl);
   result->opt->LogTimeGranularity = 1;
   result->opt->TokenBucketRefillInterval = 1;
   rv = config_get_lines(TEST_OPTIONS_OLD_VALUES, &cl, 1);
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
   rv = config_assign(&options_format, result->def_opt, cl, 0, &msg);
   if (msg) {
     /* Display the parse error message by comparing it with an empty string */
     tt_str_op(msg, OP_EQ, "");
   }
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
 
  done:
   config_free_lines(cl);
@@ -399,8 +477,15 @@ test_options_validate__uname_for_server(void *ignored)
 {
   (void)ignored;
   char *msg;
+
+#ifndef _WIN32
+  int unset_home_env = 0;
+  if (setenv("HOME", "/home/john", 0) == 0)
+    unset_home_env = 1;
+#endif
+
   options_test_data_t *tdata = get_options_test_data(
-                                      "ORListenAddress 127.0.0.1:5555");
+                                      "ORPort 127.0.0.1:5555");
   setup_capture_of_logs(LOG_WARN);
 
   MOCK(get_uname, fixed_get_uname);
@@ -430,7 +515,7 @@ test_options_validate__uname_for_server(void *ignored)
   fixed_get_uname_result = "Windows 2000";
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  expect_log_entry();
+  expect_no_log_entry();
   tor_free(msg);
 
  done:
@@ -438,6 +523,10 @@ test_options_validate__uname_for_server(void *ignored)
   free_options_test_data(tdata);
   tor_free(msg);
   teardown_capture_of_logs();
+#ifndef _WIN32
+  if (unset_home_env)
+    unsetenv("HOME");
+#endif
 }
 
 static void
@@ -451,6 +540,8 @@ test_options_validate__outbound_addresses(void *ignored)
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
+  tt_str_op(msg, OP_EQ, "Multiple outbound bind addresses configured: "
+                        "xxyy!!!sdfaf");
 
  done:
   free_options_test_data(tdata);
@@ -514,13 +605,14 @@ test_options_validate__nickname(void *ignored)
   tdata = get_options_test_data("Nickname AMoreValidNick");
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
-  tt_assert(!msg);
+  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
+  tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("DataDirectory /tmp/somewhere");
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
-  tt_assert(!msg);
+  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
 
  done:
   free_options_test_data(tdata);
@@ -534,7 +626,7 @@ test_options_validate__contactinfo(void *ignored)
   int ret;
   char *msg;
   options_test_data_t *tdata = get_options_test_data(
-                                "ORListenAddress 127.0.0.1:5555\nORPort 955");
+                                "ORPort 127.0.0.1:5555");
   setup_capture_of_logs(LOG_DEBUG);
   tdata->opt->ContactInfo = NULL;
 
@@ -547,7 +639,7 @@ test_options_validate__contactinfo(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("ORListenAddress 127.0.0.1:5555\nORPort 955\n"
+  tdata = get_options_test_data("ORPort 127.0.0.1:5555\n"
                                 "ContactInfo hella@example.org");
   mock_clean_saved_logs();
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
@@ -580,6 +672,7 @@ test_options_validate__logs(void *ignored)
   tt_str_op(tdata->opt->Logs->key, OP_EQ, "Log");
   tt_str_op(tdata->opt->Logs->value, OP_EQ, "notice stdout");
   tor_free(msg);
+  tt_int_op(ret, OP_EQ, -1);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
@@ -590,6 +683,7 @@ test_options_validate__logs(void *ignored)
   tt_str_op(tdata->opt->Logs->key, OP_EQ, "Log");
   tt_str_op(tdata->opt->Logs->value, OP_EQ, "warn stdout");
   tor_free(msg);
+  tt_int_op(ret, OP_EQ, -1);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
@@ -599,6 +693,7 @@ test_options_validate__logs(void *ignored)
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_assert(!tdata->opt->Logs);
   tor_free(msg);
+  tt_int_op(ret, OP_EQ, -1);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
@@ -607,6 +702,7 @@ test_options_validate__logs(void *ignored)
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 1, &msg);
   tt_assert(!tdata->opt->Logs);
   tor_free(msg);
+  tt_int_op(ret, OP_EQ, -1);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
@@ -615,6 +711,7 @@ test_options_validate__logs(void *ignored)
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_assert(!tdata->opt->Logs);
   tor_free(msg);
+  tt_int_op(ret, OP_EQ, -1);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
@@ -624,6 +721,7 @@ test_options_validate__logs(void *ignored)
   tdata->opt->Logs = cl;
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op((intptr_t)tdata->opt->Logs, OP_EQ, (intptr_t)cl);
+  tt_int_op(ret, OP_EQ, -1);
 
  done:
   quiet_level = orig_quiet_level;
@@ -650,16 +748,18 @@ test_options_validate__authdir(void *ignored)
   setup_capture_of_logs(LOG_INFO);
   options_test_data_t *tdata = get_options_test_data(
                                  "AuthoritativeDirectory 1\n"
-                                 "Address this.should.not_exist.example.org");
+                                 "Address this.should.not!exist!.example.org");
 
   sandbox_disable_getaddrinfo_cache();
 
+  MOCK(tor_addr_lookup, mock_tor_addr_lookup__fail_on_bad_addrs);
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  UNMOCK(tor_addr_lookup);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Failed to resolve/guess local address. See logs for"
             " details.");
   expect_log_msg("Could not resolve local Address "
-            "'this.should.not_exist.example.org'. Failing.\n");
+            "'this.should.not!exist!.example.org'. Failing.\n");
   tor_free(msg);
 
   free_options_test_data(tdata);
@@ -668,13 +768,13 @@ test_options_validate__authdir(void *ignored)
   mock_clean_saved_logs();
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
-  tt_assert(!msg);
+  tt_str_op(msg, OP_EQ, "Authoritative directory servers must set "
+                        "ContactInfo");
+  tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "Address 100.200.10.1\n");
   mock_clean_saved_logs();
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -685,9 +785,7 @@ test_options_validate__authdir(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("AuthoritativeDirectory 1\n"
                                 "Address 100.200.10.1\n"
-                                "TestingTorNetwork 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "TestingTorNetwork 1\n");
   mock_clean_saved_logs();
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -698,9 +796,7 @@ test_options_validate__authdir(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("AuthoritativeDirectory 1\n"
                                 "Address 100.200.10.1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -712,9 +808,7 @@ test_options_validate__authdir(void *ignored)
   tdata = get_options_test_data("AuthoritativeDirectory 1\n"
                                 "Address 100.200.10.1\n"
                                 "RecommendedVersions 1.2, 3.14\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_str_op(tdata->opt->RecommendedClientVersions->value, OP_EQ, "1.2, 3.14");
@@ -727,9 +821,7 @@ test_options_validate__authdir(void *ignored)
                                 "RecommendedVersions 1.2, 3.14\n"
                                 "RecommendedClientVersions 25\n"
                                 "RecommendedServerVersions 4.18\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_str_op(tdata->opt->RecommendedClientVersions->value, OP_EQ, "25");
@@ -743,9 +835,7 @@ test_options_validate__authdir(void *ignored)
                                 "RecommendedVersions 1.2, 3.14\n"
                                 "RecommendedClientVersions 25\n"
                                 "RecommendedServerVersions 4.18\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_str_op(msg, OP_EQ, "AuthoritativeDir is set, but none of (Bridge/V3)"
@@ -757,9 +847,7 @@ test_options_validate__authdir(void *ignored)
                                 "Address 100.200.10.1\n"
                                 "VersioningAuthoritativeDirectory 1\n"
                                 "RecommendedServerVersions 4.18\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_str_op(msg, OP_EQ, "Versioning authoritative dir servers must set "
@@ -771,9 +859,7 @@ test_options_validate__authdir(void *ignored)
                                 "Address 100.200.10.1\n"
                                 "VersioningAuthoritativeDirectory 1\n"
                                 "RecommendedClientVersions 4.18\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_str_op(msg, OP_EQ, "Versioning authoritative dir servers must set "
@@ -784,9 +870,7 @@ test_options_validate__authdir(void *ignored)
   tdata = get_options_test_data("AuthoritativeDirectory 1\n"
                                 "Address 100.200.10.1\n"
                                 "UseEntryGuards 1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   expect_log_msg("Authoritative directory servers "
@@ -798,9 +882,7 @@ test_options_validate__authdir(void *ignored)
   tdata = get_options_test_data("AuthoritativeDirectory 1\n"
                                 "Address 100.200.10.1\n"
                                 "V3AuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   expect_log_msg("Authoritative directories always try"
@@ -813,9 +895,7 @@ test_options_validate__authdir(void *ignored)
                                 "Address 100.200.10.1\n"
                                 "DownloadExtraInfo 1\n"
                                 "V3AuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   expect_no_log_msg("Authoritative directories always try"
@@ -826,9 +906,7 @@ test_options_validate__authdir(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("AuthoritativeDirectory 1\n"
                                 "Address 100.200.10.1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_str_op(msg, OP_EQ, "AuthoritativeDir is set, but none of (Bridge/V3)"
@@ -840,9 +918,7 @@ test_options_validate__authdir(void *ignored)
                                 "Address 100.200.10.1\n"
                                 "BridgeAuthoritativeDir 1\n"
                                 "ContactInfo hello@hello.com\n"
-                                "V3BandwidthsFile non-existant-file\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "V3BandwidthsFile non-existent-file\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_str_op(msg, OP_EQ,
@@ -854,9 +930,7 @@ test_options_validate__authdir(void *ignored)
                                 "Address 100.200.10.1\n"
                                 "BridgeAuthoritativeDir 1\n"
                                 "ContactInfo hello@hello.com\n"
-                                "V3BandwidthsFile non-existant-file\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "V3BandwidthsFile non-existent-file\n");
   mock_clean_saved_logs();
   options_validate(NULL, tdata->opt, tdata->def_opt, 0, &msg);
   tt_str_op(msg, OP_EQ,
@@ -868,9 +942,7 @@ test_options_validate__authdir(void *ignored)
                                 "Address 100.200.10.1\n"
                                 "BridgeAuthoritativeDir 1\n"
                                 "ContactInfo hello@hello.com\n"
-                                "GuardfractionFile non-existant-file\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "GuardfractionFile non-existent-file\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_str_op(msg, OP_EQ,
@@ -882,9 +954,7 @@ test_options_validate__authdir(void *ignored)
                                 "Address 100.200.10.1\n"
                                 "BridgeAuthoritativeDir 1\n"
                                 "ContactInfo hello@hello.com\n"
-                                "GuardfractionFile non-existant-file\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "GuardfractionFile non-existent-file\n");
   mock_clean_saved_logs();
   options_validate(NULL, tdata->opt, tdata->def_opt, 0, &msg);
   tt_str_op(msg, OP_EQ,
@@ -895,9 +965,7 @@ test_options_validate__authdir(void *ignored)
   tdata = get_options_test_data("AuthoritativeDirectory 1\n"
                                 "Address 100.200.10.1\n"
                                 "BridgeAuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -910,9 +978,7 @@ test_options_validate__authdir(void *ignored)
                                 "Address 100.200.10.1\n"
                                 "DirPort 999\n"
                                 "BridgeAuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -929,9 +995,7 @@ test_options_validate__authdir(void *ignored)
   /*                               "ORPort 888\n" */
   /*                               "ClientOnly 1\n" */
   /*                               "BridgeAuthoritativeDir 1\n" */
-  /*                               "ContactInfo hello@hello.com\n" */
-  /*                               "SchedulerHighWaterMark__ 42\n" */
-  /*                               "SchedulerLowWaterMark__ 10\n"); */
+  /*                               "ContactInfo hello@hello.com\n" ); */
   /* mock_clean_saved_logs(); */
   /* ret = options_validate(tdata->old_opt, tdata->opt, */
   /*                        tdata->def_opt, 0, &msg); */
@@ -953,8 +1017,7 @@ test_options_validate__relay_with_hidden_services(void *ignored)
   char *msg;
   setup_capture_of_logs(LOG_DEBUG);
   options_test_data_t *tdata = get_options_test_data(
-                                  "ORListenAddress 127.0.0.1:5555\n"
-                                  "ORPort 955\n"
+                                  "ORPort 127.0.0.1:5555\n"
                                   "HiddenServiceDir "
                                   "/Library/Tor/var/lib/tor/hidden_service/\n"
                                   "HiddenServicePort 80 127.0.0.1:8080\n"
@@ -1018,14 +1081,14 @@ test_options_validate__transproxy(void *ignored)
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
 
-#if !defined(__OpenBSD__) && !defined( DARWIN )
+#if !defined(OpenBSD) && !defined( DARWIN )
   tt_str_op(msg, OP_EQ,
           "pf-divert is a OpenBSD-specific and OS X/Darwin-specific feature.");
 #else
   tt_int_op(tdata->opt->TransProxyType_parsed, OP_EQ, TPT_PF_DIVERT);
   tt_str_op(msg, OP_EQ, "Cannot use TransProxyType without "
-            "any valid TransPort or TransListenAddress.");
-#endif
+            "any valid TransPort.");
+#endif /* !defined(OpenBSD) && !defined( DARWIN ) */
   tor_free(msg);
 
   // Test tproxy trans proxy
@@ -1039,8 +1102,8 @@ test_options_validate__transproxy(void *ignored)
 #else
   tt_int_op(tdata->opt->TransProxyType_parsed, OP_EQ, TPT_TPROXY);
   tt_str_op(msg, OP_EQ, "Cannot use TransProxyType without any valid "
-            "TransPort or TransListenAddress.");
-#endif
+            "TransPort.");
+#endif /* !defined(__linux__) */
   tor_free(msg);
 
   // Test ipfw trans proxy
@@ -1055,13 +1118,13 @@ test_options_validate__transproxy(void *ignored)
 #else
   tt_int_op(tdata->opt->TransProxyType_parsed, OP_EQ, TPT_IPFW);
   tt_str_op(msg, OP_EQ, "Cannot use TransProxyType without any valid "
-            "TransPort or TransListenAddress.");
-#endif
+            "TransPort.");
+#endif /* !defined(KERNEL_MAY_SUPPORT_IPFW) */
   tor_free(msg);
 
   // Test unknown trans proxy
   free_options_test_data(tdata);
-  tdata = get_options_test_data("TransProxyType non-existant\n");
+  tdata = get_options_test_data("TransProxyType non-existent\n");
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Unrecognized value for TransProxyType");
@@ -1076,47 +1139,41 @@ test_options_validate__transproxy(void *ignored)
                                 "TransPort 127.0.0.1:123\n");
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
-  if (msg) {
-    TT_DIE(("Expected NULL but got '%s'", msg));
-  }
+  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
 #elif defined(KERNEL_MAY_SUPPORT_IPFW)
   tdata = get_options_test_data("TransProxyType ipfw\n"
                                 "TransPort 127.0.0.1:123\n");
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
-  if (msg) {
-    TT_DIE(("Expected NULL but got '%s'", msg));
-  }
-#elif defined(__OpenBSD__)
+  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
+  tor_free(msg);
+#elif defined(OpenBSD)
   tdata = get_options_test_data("TransProxyType pf-divert\n"
                                 "TransPort 127.0.0.1:123\n");
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
-  if (msg) {
-    TT_DIE(("Expected NULL but got '%s'", msg));
-  }
+  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
+  tor_free(msg);
 #elif defined(__NetBSD__)
   tdata = get_options_test_data("TransProxyType default\n"
                                 "TransPort 127.0.0.1:123\n");
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
-  if (msg) {
-    TT_DIE(("Expected NULL but got '%s'", msg));
-  }
-#endif
+  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
+  tor_free(msg);
+#endif /* defined(__linux__) || ... */
 
   // Assert that a test has run for some TransProxyType
   tt_assert(tdata);
 
-#else
+#else /* !(defined(USE_TRANSPARENT)) */
   tdata = get_options_test_data("TransPort 127.0.0.1:555\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "TransPort and TransListenAddress are disabled in "
-            "this build.");
+  tt_str_op(msg, OP_EQ, "TransPort is disabled in this build.");
   tor_free(msg);
-#endif
+#endif /* defined(USE_TRANSPARENT) */
 
  done:
   free_options_test_data(tdata);
@@ -1181,9 +1238,7 @@ test_options_validate__exclude_nodes(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ExcludeNodes {cn}\n"
-                                "StrictNodes 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "StrictNodes 1\n");
   mock_clean_saved_logs();
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1194,9 +1249,7 @@ test_options_validate__exclude_nodes(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("ExcludeNodes {cn}\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+  tdata = get_options_test_data("ExcludeNodes {cn}\n");
   mock_clean_saved_logs();
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1214,49 +1267,6 @@ test_options_validate__exclude_nodes(void *ignored)
 }
 
 static void
-test_options_validate__scheduler(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  char *msg;
-  setup_capture_of_logs(LOG_DEBUG);
-  options_test_data_t *tdata = get_options_test_data(
-                                            "SchedulerLowWaterMark__ 0\n");
-
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  expect_log_msg("Bad SchedulerLowWaterMark__ option\n");
-  tor_free(msg);
-
-  // TODO: this test cannot run on platforms where UINT32_MAX == UINT64_MAX.
-  // I suspect it's unlikely this branch can actually happen
-  /* free_options_test_data(tdata); */
-  /* tdata = get_options_test_data( */
-  /*                      "SchedulerLowWaterMark 10000000000000000000\n"); */
-  /* tdata->opt->SchedulerLowWaterMark__ = (uint64_t)UINT32_MAX; */
-  /* tdata->opt->SchedulerLowWaterMark__++; */
-  /* mock_clean_saved_logs(); */
-  /* ret = options_validate(tdata->old_opt, tdata->opt, */
-  /*                        tdata->def_opt, 0, &msg); */
-  /* tt_int_op(ret, OP_EQ, -1); */
-  /* expect_log_msg("Bad SchedulerLowWaterMark__ option\n"); */
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data("SchedulerLowWaterMark__ 42\n"
-                                "SchedulerHighWaterMark__ 42\n");
-  mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  expect_log_msg("Bad SchedulerHighWaterMark option\n");
-  tor_free(msg);
-
- done:
-  teardown_capture_of_logs();
-  free_options_test_data(tdata);
-  tor_free(msg);
-}
-
-static void
 test_options_validate__node_families(void *ignored)
 {
   (void)ignored;
@@ -1264,9 +1274,7 @@ test_options_validate__node_families(void *ignored)
   char *msg;
   options_test_data_t *tdata = get_options_test_data(
                                      "NodeFamily flux, flax\n"
-                                     "NodeFamily somewhere\n"
-                                     "SchedulerHighWaterMark__ 42\n"
-                                     "SchedulerLowWaterMark__ 10\n");
+                                     "NodeFamily somewhere\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1284,8 +1292,7 @@ test_options_validate__node_families(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+  tdata = get_options_test_data("");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1293,9 +1300,7 @@ test_options_validate__node_families(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("NodeFamily !flux\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+  tdata = get_options_test_data("NodeFamily !flux\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1309,54 +1314,6 @@ test_options_validate__node_families(void *ignored)
 }
 
 static void
-test_options_validate__tlsec(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  char *msg;
-  setup_capture_of_logs(LOG_DEBUG);
-  options_test_data_t *tdata = get_options_test_data(
-                                 "TLSECGroup ed25519\n"
-                                 "SchedulerHighWaterMark__ 42\n"
-                                 "SchedulerLowWaterMark__ 10\n");
-
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  expect_log_msg("Unrecognized TLSECGroup: Falling back to the default.\n");
-  tt_assert(!tdata->opt->TLSECGroup);
-  tor_free(msg);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data("TLSECGroup P224\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
-  mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  expect_no_log_msg(
-            "Unrecognized TLSECGroup: Falling back to the default.\n");
-  tt_assert(tdata->opt->TLSECGroup);
-  tor_free(msg);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data("TLSECGroup P256\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
-  mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  expect_no_log_msg(
-            "Unrecognized TLSECGroup: Falling back to the default.\n");
-  tt_assert(tdata->opt->TLSECGroup);
-  tor_free(msg);
-
- done:
-  teardown_capture_of_logs();
-  free_options_test_data(tdata);
-  tor_free(msg);
-}
-
-static void
 test_options_validate__token_bucket(void *ignored)
 {
   (void)ignored;
@@ -1392,9 +1349,7 @@ test_options_validate__recommended_packages(void *ignored)
   setup_capture_of_logs(LOG_WARN);
   options_test_data_t *tdata = get_options_test_data(
             "RecommendedPackages foo 1.2 http://foo.com sha1=123123123123\n"
-            "RecommendedPackages invalid-package-line\n"
-            "SchedulerHighWaterMark__ 42\n"
-            "SchedulerLowWaterMark__ 10\n");
+            "RecommendedPackages invalid-package-line\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1416,9 +1371,7 @@ test_options_validate__fetch_dir(void *ignored)
   char *msg;
   options_test_data_t *tdata = get_options_test_data(
                                             "FetchDirInfoExtraEarly 1\n"
-                                            "FetchDirInfoEarly 0\n"
-                                            "SchedulerHighWaterMark__ 42\n"
-                                            "SchedulerLowWaterMark__ 10\n");
+                                            "FetchDirInfoEarly 0\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1428,9 +1381,7 @@ test_options_validate__fetch_dir(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("FetchDirInfoExtraEarly 1\n"
-                                "FetchDirInfoEarly 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "FetchDirInfoEarly 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1450,9 +1401,7 @@ test_options_validate__conn_limit(void *ignored)
   int ret;
   char *msg;
   options_test_data_t *tdata = get_options_test_data(
-                                            "ConnLimit 0\n"
-                                            "SchedulerHighWaterMark__ 42\n"
-                                            "SchedulerLowWaterMark__ 10\n");
+                                            "ConnLimit 0\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1460,9 +1409,7 @@ test_options_validate__conn_limit(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+  tdata = get_options_test_data("ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1481,12 +1428,17 @@ test_options_validate__paths_needed(void *ignored)
   (void)ignored;
   int ret;
   char *msg;
+
+#ifndef _WIN32
+  int unset_home_env = 0;
+  if (setenv("HOME", "/home/john", 0) == 0)
+    unset_home_env = 1;
+#endif
+
   setup_capture_of_logs(LOG_WARN);
   options_test_data_t *tdata = get_options_test_data(
                                       "PathsNeededToBuildCircuits 0.1\n"
-                                      "ConnLimit 1\n"
-                                      "SchedulerHighWaterMark__ 42\n"
-                                      "SchedulerLowWaterMark__ 10\n");
+                                      "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1499,9 +1451,7 @@ test_options_validate__paths_needed(void *ignored)
   free_options_test_data(tdata);
   mock_clean_saved_logs();
   tdata = get_options_test_data("PathsNeededToBuildCircuits 0.99\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1514,9 +1464,7 @@ test_options_validate__paths_needed(void *ignored)
   free_options_test_data(tdata);
   mock_clean_saved_logs();
   tdata = get_options_test_data("PathsNeededToBuildCircuits 0.91\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1529,6 +1477,10 @@ test_options_validate__paths_needed(void *ignored)
   teardown_capture_of_logs();
   free_options_test_data(tdata);
   tor_free(msg);
+#ifndef _WIN32
+  if (unset_home_env)
+    unsetenv("HOME");
+#endif
 }
 
 static void
@@ -1539,9 +1491,7 @@ test_options_validate__max_client_circuits(void *ignored)
   char *msg;
   options_test_data_t *tdata = get_options_test_data(
                                            "MaxClientCircuitsPending 0\n"
-                                           "ConnLimit 1\n"
-                                           "SchedulerHighWaterMark__ 42\n"
-                                           "SchedulerLowWaterMark__ 10\n");
+                                           "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1551,9 +1501,7 @@ test_options_validate__max_client_circuits(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("MaxClientCircuitsPending 1025\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1563,9 +1511,7 @@ test_options_validate__max_client_circuits(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1586,9 +1532,7 @@ test_options_validate__ports(void *ignored)
   options_test_data_t *tdata = get_options_test_data(
                                       "FirewallPorts 65537\n"
                                       "MaxClientCircuitsPending 1\n"
-                                      "ConnLimit 1\n"
-                                      "SchedulerHighWaterMark__ 42\n"
-                                      "SchedulerLowWaterMark__ 10\n");
+                                      "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1599,9 +1543,7 @@ test_options_validate__ports(void *ignored)
   tdata = get_options_test_data("FirewallPorts 1\n"
                                 "LongLivedPorts 124444\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1613,9 +1555,7 @@ test_options_validate__ports(void *ignored)
                                 "LongLivedPorts 2\n"
                                 "RejectPlaintextPorts 112233\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1628,9 +1568,7 @@ test_options_validate__ports(void *ignored)
                                 "RejectPlaintextPorts 3\n"
                                 "WarnPlaintextPorts 65536\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1643,9 +1581,7 @@ test_options_validate__ports(void *ignored)
                                 "RejectPlaintextPorts 3\n"
                                 "WarnPlaintextPorts 4\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1667,9 +1603,7 @@ test_options_validate__reachable_addresses(void *ignored)
   options_test_data_t *tdata = get_options_test_data(
                                      "FascistFirewall 1\n"
                                      "MaxClientCircuitsPending 1\n"
-                                     "ConnLimit 1\n"
-                                     "SchedulerHighWaterMark__ 42\n"
-                                     "SchedulerLowWaterMark__ 10\n");
+                                     "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1687,9 +1621,7 @@ test_options_validate__reachable_addresses(void *ignored)
                                 "ReachableDirAddresses *:81\n"
                                 "ReachableORAddresses *:444\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
   tdata->opt->FirewallPorts = smartlist_new();
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1703,9 +1635,7 @@ test_options_validate__reachable_addresses(void *ignored)
   tdata = get_options_test_data("FascistFirewall 1\n"
                                 "FirewallPort 123\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1722,9 +1652,7 @@ test_options_validate__reachable_addresses(void *ignored)
                                 "ReachableAddresses *:83\n"
                                 "ReachableAddresses reject *:*\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1732,18 +1660,27 @@ test_options_validate__reachable_addresses(void *ignored)
   tt_str_op(tdata->opt->ReachableAddresses->value, OP_EQ, "*:82");
   tor_free(msg);
 
+  free_options_test_data(tdata);
+  mock_clean_saved_logs();
+  tdata = get_options_test_data("FascistFirewall 1\n"
+                                "ReachableAddresses *:82\n"
+                                "MaxClientCircuitsPending 1\n"
+                                "ConnLimit 1\n");
+
+  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_ptr_op(tdata->opt->ReachableAddresses->next, OP_EQ, NULL);
+  tor_free(msg);
+
 #define SERVERS_REACHABLE_MSG "Servers must be able to freely connect to" \
   " the rest of the Internet, so they must not set Reachable*Addresses or" \
   " FascistFirewall or FirewallPorts or ClientUseIPv4 0."
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ReachableAddresses *:82\n"
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1752,12 +1689,9 @@ test_options_validate__reachable_addresses(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ReachableORAddresses *:82\n"
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1766,12 +1700,9 @@ test_options_validate__reachable_addresses(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ReachableDirAddresses *:82\n"
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1780,12 +1711,9 @@ test_options_validate__reachable_addresses(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ClientUseIPv4 0\n"
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1794,14 +1722,6 @@ test_options_validate__reachable_addresses(void *ignored)
 
   /* Test IPv4-only clients setting IPv6 preferences */
 
-#define WARN_PLEASE_USE_IPV6_OR_LOG_MSG \
-        "ClientPreferIPv6ORPort 1 is ignored unless tor is using IPv6. " \
-        "Please set ClientUseIPv6 1, ClientUseIPv4 0, or configure bridges.\n"
-
-#define WARN_PLEASE_USE_IPV6_DIR_LOG_MSG \
-        "ClientPreferIPv6DirPort 1 is ignored unless tor is using IPv6. " \
-        "Please set ClientUseIPv6 1, ClientUseIPv4 0, or configure bridges.\n"
-
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "ClientUseIPv4 1\n"
@@ -1811,7 +1731,6 @@ test_options_validate__reachable_addresses(void *ignored)
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg(WARN_PLEASE_USE_IPV6_OR_LOG_MSG);
   tor_free(msg);
 
   free_options_test_data(tdata);
@@ -1823,7 +1742,6 @@ test_options_validate__reachable_addresses(void *ignored)
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg(WARN_PLEASE_USE_IPV6_DIR_LOG_MSG);
   tor_free(msg);
 
   /* Now test an IPv4/IPv6 client setting IPv6 preferences */
@@ -1891,12 +1809,9 @@ test_options_validate__use_bridges(void *ignored)
   options_test_data_t *tdata = get_options_test_data(
                                    "UseBridges 1\n"
                                    "ClientUseIPv4 1\n"
-                                   "ORListenAddress 127.0.0.1:5555\n"
-                                   "ORPort 955\n"
+                                   "ORPort 127.0.0.1:5555\n"
                                    "MaxClientCircuitsPending 1\n"
-                                   "ConnLimit 1\n"
-                                   "SchedulerHighWaterMark__ 42\n"
-                                   "SchedulerLowWaterMark__ 10\n");
+                                   "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1907,9 +1822,7 @@ test_options_validate__use_bridges(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("UseBridges 1\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1922,9 +1835,7 @@ test_options_validate__use_bridges(void *ignored)
   tdata = get_options_test_data("UseBridges 1\n"
                                 "EntryNodes {cn}\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1945,6 +1856,19 @@ test_options_validate__use_bridges(void *ignored)
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "UseBridges 1\n"
                                 "Bridge 10.0.0.1\n"
+                                "UseEntryGuards 0\n"
+                                );
+
+  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_str_op(msg, OP_EQ,
+            "Setting UseBridges requires also setting UseEntryGuards.");
+  tor_free(msg);
+
+  free_options_test_data(tdata);
+  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+                                "UseBridges 1\n"
+                                "Bridge 10.0.0.1\n"
                                 "Bridge !!!\n"
                                 );
 
@@ -1971,9 +1895,7 @@ test_options_validate__entry_nodes(void *ignored)
                                          "EntryNodes {cn}\n"
                                          "UseEntryGuards 0\n"
                                          "MaxClientCircuitsPending 1\n"
-                                         "ConnLimit 1\n"
-                                         "SchedulerHighWaterMark__ 42\n"
-                                         "SchedulerLowWaterMark__ 10\n");
+                                         "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1985,9 +1907,7 @@ test_options_validate__entry_nodes(void *ignored)
   tdata = get_options_test_data("EntryNodes {cn}\n"
                                 "UseEntryGuards 1\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -2001,56 +1921,6 @@ test_options_validate__entry_nodes(void *ignored)
 }
 
 static void
-test_options_validate__invalid_nodes(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  char *msg;
-  options_test_data_t *tdata = get_options_test_data(
-                                  "AllowInvalidNodes something_stupid\n"
-                                  "MaxClientCircuitsPending 1\n"
-                                  "ConnLimit 1\n"
-                                  "SchedulerHighWaterMark__ 42\n"
-                                  "SchedulerLowWaterMark__ 10\n");
-
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ,
-            "Unrecognized value 'something_stupid' in AllowInvalidNodes");
-  tor_free(msg);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data("AllowInvalidNodes entry, middle, exit\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
-
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_int_op(tdata->opt->AllowInvalid_, OP_EQ, ALLOW_INVALID_ENTRY |
-            ALLOW_INVALID_EXIT | ALLOW_INVALID_MIDDLE);
-  tor_free(msg);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data("AllowInvalidNodes introduction, rendezvous\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
-
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_int_op(tdata->opt->AllowInvalid_, OP_EQ, ALLOW_INVALID_INTRODUCTION |
-            ALLOW_INVALID_RENDEZVOUS);
-  tor_free(msg);
-
- done:
-  free_options_test_data(tdata);
-  tor_free(msg);
-}
-
-static void
 test_options_validate__safe_logging(void *ignored)
 {
   (void)ignored;
@@ -2058,9 +1928,7 @@ test_options_validate__safe_logging(void *ignored)
   char *msg;
   options_test_data_t *tdata = get_options_test_data(
                                             "MaxClientCircuitsPending 1\n"
-                                            "ConnLimit 1\n"
-                                            "SchedulerHighWaterMark__ 42\n"
-                                            "SchedulerLowWaterMark__ 10\n");
+                                            "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -2070,9 +1938,7 @@ test_options_validate__safe_logging(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("SafeLogging 0\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -2082,9 +1948,7 @@ test_options_validate__safe_logging(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("SafeLogging Relay\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -2094,9 +1958,7 @@ test_options_validate__safe_logging(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("SafeLogging 1\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -2106,9 +1968,7 @@ test_options_validate__safe_logging(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("SafeLogging stuffy\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -2248,17 +2108,14 @@ test_options_validate__testing(void *ignored)
   ENSURE_DEFAULT(TestingV3AuthVotingStartOffset, 3000);
   ENSURE_DEFAULT(TestingAuthDirTimeToLearnReachability, 3000);
   ENSURE_DEFAULT(TestingEstimatedDescriptorPropagationTime, 3000);
-  ENSURE_DEFAULT(TestingServerDownloadSchedule, 3000);
-  ENSURE_DEFAULT(TestingClientDownloadSchedule, 3000);
-  ENSURE_DEFAULT(TestingServerConsensusDownloadSchedule, 3000);
-  ENSURE_DEFAULT(TestingClientConsensusDownloadSchedule, 3000);
-  ENSURE_DEFAULT(TestingBridgeDownloadSchedule, 3000);
+  ENSURE_DEFAULT(TestingServerDownloadInitialDelay, 3000);
+  ENSURE_DEFAULT(TestingClientDownloadInitialDelay, 3000);
+  ENSURE_DEFAULT(TestingServerConsensusDownloadInitialDelay, 3000);
+  ENSURE_DEFAULT(TestingClientConsensusDownloadInitialDelay, 3000);
+  ENSURE_DEFAULT(TestingBridgeDownloadInitialDelay, 3000);
+  ENSURE_DEFAULT(TestingBridgeBootstrapDownloadInitialDelay, 3000);
   ENSURE_DEFAULT(TestingClientMaxIntervalWithoutRequest, 3000);
   ENSURE_DEFAULT(TestingDirConnectionMaxStall, 3000);
-  ENSURE_DEFAULT(TestingConsensusMaxDownloadTries, 3000);
-  ENSURE_DEFAULT(TestingDescriptorMaxDownloadTries, 3000);
-  ENSURE_DEFAULT(TestingMicrodescMaxDownloadTries, 3000);
-  ENSURE_DEFAULT(TestingCertMaxDownloadTries, 3000);
   ENSURE_DEFAULT(TestingAuthKeyLifetime, 3000);
   ENSURE_DEFAULT(TestingLinkCertLifetime, 3000);
   ENSURE_DEFAULT(TestingSigningKeySlop, 3000);
@@ -2320,30 +2177,6 @@ test_options_validate__hidserv(void *ignored)
 }
 
 static void
-test_options_validate__predicted_ports(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  char *msg;
-  setup_capture_of_logs(LOG_WARN);
-
-  options_test_data_t *tdata = get_options_test_data(
-                                     "PredictedPortsRelevanceTime 100000000\n"
-                                     TEST_OPTIONS_DEFAULT_VALUES);
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg("PredictedPortsRelevanceTime is too "
-            "large; clipping to 3600s.\n");
-  tt_int_op(tdata->opt->PredictedPortsRelevanceTime, OP_EQ, 3600);
-
- done:
-  teardown_capture_of_logs();
-  policies_free_all();
-  free_options_test_data(tdata);
-  tor_free(msg);
-}
-
-static void
 test_options_validate__path_bias(void *ignored)
 {
   (void)ignored;
@@ -2489,8 +2322,7 @@ test_options_validate__bandwidth(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 1\n"
                                 );
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
@@ -2501,8 +2333,7 @@ test_options_validate__bandwidth(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76800\n"
                                 "MaxAdvertisedBandwidth 30000\n"
                                 );
@@ -2514,8 +2345,7 @@ test_options_validate__bandwidth(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76800\n"
                                 "RelayBandwidthRate 1\n"
                                 "MaxAdvertisedBandwidth 38400\n"
@@ -2528,8 +2358,7 @@ test_options_validate__bandwidth(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76800\n"
                                 "BandwidthBurst 76800\n"
                                 "RelayBandwidthRate 76800\n"
@@ -2634,67 +2463,6 @@ test_options_validate__circuits(void *ignored)
 }
 
 static void
-test_options_validate__port_forwarding(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  char *msg;
-  options_test_data_t *tdata = NULL;
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "PortForwarding 1\nSandbox 1\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "PortForwarding is not compatible with Sandbox;"
-            " at most one can be set");
-  tor_free(msg);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "PortForwarding 1\nSandbox 0\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_assert(!msg);
-  tor_free(msg);
-
- done:
-  free_options_test_data(tdata);
-  policies_free_all();
-  tor_free(msg);
-}
-
-static void
-test_options_validate__tor2web(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  char *msg;
-  options_test_data_t *tdata = NULL;
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Tor2webRendezvousPoints 1\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ,
-            "Tor2webRendezvousPoints cannot be set without Tor2webMode.");
-  tor_free(msg);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Tor2webRendezvousPoints 1\nTor2webMode 1\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, 0);
-  tor_free(msg);
-
- done:
-  policies_free_all();
-  free_options_test_data(tdata);
-  tor_free(msg);
-}
-
-static void
 test_options_validate__rend(void *ignored)
 {
   (void)ignored;
@@ -2807,20 +2575,18 @@ test_options_validate__single_onion(void *ignored)
   tt_ptr_op(msg, OP_EQ, NULL);
   free_options_test_data(tdata);
 
-  /* Test that SOCKSPort must come with Tor2webMode if
-   * HiddenServiceSingleHopMode is 1 */
+  /* Test that SOCKSPort if HiddenServiceSingleHopMode is 1 */
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "SOCKSPort 5000\n"
                                 "HiddenServiceSingleHopMode 1\n"
                                 "HiddenServiceNonAnonymousMode 1\n"
-                                "Tor2webMode 0\n"
                                 );
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "HiddenServiceNonAnonymousMode is incompatible with "
             "using Tor as an anonymous client. Please set "
-            "Socks/Trans/NATD/DNSPort to 0, or HiddenServiceNonAnonymousMode "
-            "to 0, or use the non-anonymous Tor2webMode.");
+            "Socks/Trans/NATD/DNSPort to 0, or revert "
+            "HiddenServiceNonAnonymousMode to 0.");
   tor_free(msg);
   free_options_test_data(tdata);
 
@@ -2828,7 +2594,6 @@ test_options_validate__single_onion(void *ignored)
                                 "SOCKSPort 0\n"
                                 "HiddenServiceSingleHopMode 1\n"
                                 "HiddenServiceNonAnonymousMode 1\n"
-                                "Tor2webMode 0\n"
                                 );
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, 0);
@@ -2838,27 +2603,13 @@ test_options_validate__single_onion(void *ignored)
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "SOCKSPort 5000\n"
                                 "HiddenServiceSingleHopMode 0\n"
-                                "Tor2webMode 0\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_ptr_op(msg, OP_EQ, NULL);
-  free_options_test_data(tdata);
-
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "SOCKSPort 5000\n"
-                                "HiddenServiceSingleHopMode 1\n"
-                                "HiddenServiceNonAnonymousMode 1\n"
-                                "Tor2webMode 1\n"
                                 );
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_ptr_op(msg, OP_EQ, NULL);
   free_options_test_data(tdata);
 
-  /* Test that a hidden service can't be run with Tor2web
-   * Use HiddenServiceNonAnonymousMode instead of Tor2webMode, because
-   * Tor2webMode requires a compilation #define */
+  /* Test that a hidden service can't be run in non anonymous mode. */
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                   "HiddenServiceNonAnonymousMode 1\n"
                   "HiddenServiceDir /Library/Tor/var/lib/tor/hidden_service/\n"
@@ -2967,8 +2718,7 @@ test_options_validate__accounting(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data(
            TEST_OPTIONS_DEFAULT_VALUES
-           "ORListenAddress 127.0.0.1:5555\n"
-           "ORPort 955\n"
+           "ORPort 127.0.0.1:5555\n"
            "BandwidthRate 76800\n"
            "BandwidthBurst 76800\n"
            "MaxAdvertisedBandwidth 38400\n"
@@ -3037,6 +2787,7 @@ test_options_validate__proxy(void *ignored)
   options_test_data_t *tdata = NULL;
   sandbox_disable_getaddrinfo_cache();
   setup_capture_of_logs(LOG_WARN);
+  MOCK(tor_addr_lookup, mock_tor_addr_lookup__fail_on_bad_addrs);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
@@ -3057,6 +2808,7 @@ test_options_validate__proxy(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
+
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "HttpProxy not_so_valid!\n"
                                 );
@@ -3357,6 +3109,7 @@ test_options_validate__proxy(void *ignored)
   policies_free_all();
   // sandbox_free_getaddrinfo_cache();
   tor_free(msg);
+  UNMOCK(tor_addr_lookup);
 }
 
 static void
@@ -3550,7 +3303,7 @@ test_options_validate__control(void *ignored)
             "can reconfigure your Tor.  That's bad!  You should upgrade your "
             "Tor controller as soon as possible.\n");
   tor_free(msg);
-#endif
+#endif /* defined(HAVE_SYS_UN_H) */
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
@@ -3599,8 +3352,7 @@ test_options_validate__families(void *ignored)
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "MyFamily home\n"
                                 "BridgeRelay 1\n"
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 51300\n"
                                 "BandwidthBurst 51300\n"
                                 "MaxAdvertisedBandwidth 25700\n"
@@ -3829,8 +3581,7 @@ test_options_validate__transport(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "ServerTransportPlugin foo exec bar\n"
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76900\n"
                                 "BandwidthBurst 76900\n"
                                 "MaxAdvertisedBandwidth 38500\n"
@@ -3872,8 +3623,7 @@ test_options_validate__transport(void *ignored)
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "ServerTransportListenAddr foo 127.0.0.42:55\n"
                                 "ServerTransportPlugin foo exec bar\n"
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76900\n"
                                 "BandwidthBurst 76900\n"
                                 "MaxAdvertisedBandwidth 38500\n"
@@ -4230,48 +3980,6 @@ test_options_validate__virtual_addr(void *ignored)
 }
 
 static void
-test_options_validate__exits(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  char *msg;
-  options_test_data_t *tdata = NULL;
-  setup_capture_of_logs(LOG_WARN);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AllowSingleHopExits 1"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg("You have set AllowSingleHopExits; "
-            "now your relay will allow others to make one-hop exits. However,"
-            " since by default most clients avoid relays that set this option,"
-            " most clients will ignore you.\n");
-  tor_free(msg);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AllowSingleHopExits 1\n"
-                                VALID_DIR_AUTH
-                                );
-  mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, 0);
-  expect_no_log_msg("You have set AllowSingleHopExits; "
-            "now your relay will allow others to make one-hop exits. However,"
-            " since by default most clients avoid relays that set this option,"
-            " most clients will ignore you.\n");
-  tor_free(msg);
-
- done:
-  policies_free_all();
-  teardown_capture_of_logs();
-  free_options_test_data(tdata);
-  tor_free(msg);
-}
-
-static void
 test_options_validate__testing_options(void *ignored)
 {
   (void)ignored;
@@ -4314,15 +4022,6 @@ test_options_validate__testing_options(void *ignored)
                       "is way too low.");
   TEST_TESTING_OPTION(TestingDirConnectionMaxStall, 1, 3601,
                       "is way too low.");
-  // TODO: I think this points to a bug/regression in options_validate
-  TEST_TESTING_OPTION(TestingConsensusMaxDownloadTries, 1, 801,
-                      "must be greater than 2.");
-  TEST_TESTING_OPTION(TestingDescriptorMaxDownloadTries, 1, 801,
-                      "must be greater than 1.");
-  TEST_TESTING_OPTION(TestingMicrodescMaxDownloadTries, 1, 801,
-                      "must be greater than 1.");
-  TEST_TESTING_OPTION(TestingCertMaxDownloadTries, 1, 801,
-                      "must be greater than 1.");
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
@@ -4399,16 +4098,6 @@ test_options_validate__testing_options(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "TestingEnableTbEmptyEvent 1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "TestingEnableTbEmptyEvent may only be changed "
-            "in testing Tor networks!");
-  tor_free(msg);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingEnableTbEmptyEvent 1\n"
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 1\n"
                                 "___UsingTestNetworkDefaults 0\n"
@@ -4507,9 +4196,7 @@ struct testcase_t options_tests[] = {
   LOCAL_VALIDATE_TEST(relay_with_hidden_services),
   LOCAL_VALIDATE_TEST(transproxy),
   LOCAL_VALIDATE_TEST(exclude_nodes),
-  LOCAL_VALIDATE_TEST(scheduler),
   LOCAL_VALIDATE_TEST(node_families),
-  LOCAL_VALIDATE_TEST(tlsec),
   LOCAL_VALIDATE_TEST(token_bucket),
   LOCAL_VALIDATE_TEST(recommended_packages),
   LOCAL_VALIDATE_TEST(fetch_dir),
@@ -4520,17 +4207,13 @@ struct testcase_t options_tests[] = {
   LOCAL_VALIDATE_TEST(reachable_addresses),
   LOCAL_VALIDATE_TEST(use_bridges),
   LOCAL_VALIDATE_TEST(entry_nodes),
-  LOCAL_VALIDATE_TEST(invalid_nodes),
   LOCAL_VALIDATE_TEST(safe_logging),
   LOCAL_VALIDATE_TEST(publish_server_descriptor),
   LOCAL_VALIDATE_TEST(testing),
   LOCAL_VALIDATE_TEST(hidserv),
-  LOCAL_VALIDATE_TEST(predicted_ports),
   LOCAL_VALIDATE_TEST(path_bias),
   LOCAL_VALIDATE_TEST(bandwidth),
   LOCAL_VALIDATE_TEST(circuits),
-  LOCAL_VALIDATE_TEST(port_forwarding),
-  LOCAL_VALIDATE_TEST(tor2web),
   LOCAL_VALIDATE_TEST(rend),
   LOCAL_VALIDATE_TEST(single_onion),
   LOCAL_VALIDATE_TEST(accounting),
@@ -4543,9 +4226,7 @@ struct testcase_t options_tests[] = {
   LOCAL_VALIDATE_TEST(constrained_sockets),
   LOCAL_VALIDATE_TEST(v3_auth),
   LOCAL_VALIDATE_TEST(virtual_addr),
-  LOCAL_VALIDATE_TEST(exits),
   LOCAL_VALIDATE_TEST(testing_options),
   LOCAL_VALIDATE_TEST(accel),
   END_OF_TESTCASES              /*  */
 };
-
diff --git a/src/test/test_pem.c b/src/test/test_pem.c
new file mode 100644
index 0000000000..865688b1a7
--- /dev/null
+++ b/src/test/test_pem.c
@@ -0,0 +1,122 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#include "lib/encoding/pem.h"
+#include "lib/cc/compat_compiler.h"
+#include "lib/malloc/malloc.h"
+
+#include "test/test.h"
+
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+static const char example_pre[] =
+    "Lest you get the wrong impression, we wombats "
+    "are not in the habit of tunneling madly about, without any supplies "
+    "or even a map."; /* -- Ursula Vernon, _Digger_ */
+static const char expected[] =
+    "-----BEGIN WOMBAT QUOTE-----\n"
+    "TGVzdCB5b3UgZ2V0IHRoZSB3cm9uZyBpbXByZXNzaW9uLCB3ZSB3b21iYXRzIGFy\n"
+    "ZSBub3QgaW4gdGhlIGhhYml0IG9mIHR1bm5lbGluZyBtYWRseSBhYm91dCwgd2l0\n"
+    "aG91dCBhbnkgc3VwcGxpZXMgb3IgZXZlbiBhIG1hcC4=\n"
+    "-----END WOMBAT QUOTE-----\n";
+
+static void
+test_crypto_pem_encode(void *arg)
+{
+  (void)arg;
+
+  char buf[4096];
+
+  int n = (int) pem_encoded_size(strlen(example_pre), "WOMBAT QUOTE");
+
+  int n2 = pem_encode(buf, sizeof(buf),
+                      (const unsigned char *)example_pre, strlen(example_pre),
+                      "WOMBAT QUOTE");
+  tt_int_op(strlen(buf)+1, OP_EQ, n);
+  tt_int_op(n2, OP_EQ, 0);
+  tt_str_op(buf, OP_EQ, expected);
+
+  /* Now make sure it succeeds if the buffer is exactly the length we want. */
+  memset(buf, 0, sizeof(buf));
+  n2 = pem_encode(buf, n, (const unsigned char *)example_pre,
+                      strlen(example_pre), "WOMBAT QUOTE");
+  tt_int_op(n2, OP_EQ, 0);
+  tt_str_op(buf, OP_EQ, expected);
+
+  /* Make sure it fails if the buffer is too short. */
+  memset(buf, 0, sizeof(buf));
+  n2 = pem_encode(buf, n - 1, (const unsigned char *)example_pre,
+                  strlen(example_pre), "WOMBAT QUOTE");
+  tt_int_op(n2, OP_EQ, -1);
+
+ done:
+  ;
+}
+
+static void
+test_crypto_pem_decode(void *arg)
+{
+  (void)arg;
+
+  unsigned char buf[4096];
+
+  /* Try a straightforward decoding. */
+  int n = pem_decode(buf, sizeof(buf),
+                     expected, strlen(expected),
+                     "WOMBAT QUOTE");
+  tt_int_op(n, OP_EQ, strlen(example_pre));
+  tt_mem_op(buf, OP_EQ, example_pre, n);
+
+  /* Succeed if the buffer is exactly the right size. */
+  memset(buf, 0xff, sizeof(buf));
+  n = pem_decode(buf, strlen(example_pre),
+                 expected, strlen(expected),
+                 "WOMBAT QUOTE");
+  tt_int_op(n, OP_EQ, strlen(example_pre));
+  tt_mem_op(buf, OP_EQ, example_pre, n);
+  tt_int_op(buf[n], OP_EQ, 0xff);
+
+  /* Verify that it fails if the buffer is too small. */
+  memset(buf, 0xff, sizeof(buf));
+  n = pem_decode(buf, strlen(example_pre) - 1,
+                 expected, strlen(expected),
+                 "WOMBAT QUOTE");
+  tt_int_op(n, OP_EQ, -1);
+
+  /* Verify that it fails with an incorrect tag. */
+  memset(buf, 0xff, sizeof(buf));
+  n = pem_decode(buf, sizeof(buf),
+                 expected, strlen(expected),
+                 "QUOKKA VOTE");
+  tt_int_op(n, OP_EQ, -1);
+
+  /* Try truncated buffers of different sizes. */
+  size_t i;
+  for (i = 0; i <= strlen(expected); ++i) {
+    char *truncated = tor_memdup(expected, i);
+    n = pem_decode(buf, sizeof(buf),
+                   truncated, i,
+                   "WOMBAT QUOTE");
+    tor_free(truncated);
+    if (i < strlen(expected) - 1) {
+      tt_int_op(n, OP_EQ, -1);
+    } else {
+      tt_int_op(n, OP_EQ, strlen(example_pre));
+    }
+  }
+
+ done:
+  ;
+}
+
+struct testcase_t pem_tests[] = {
+  { "encode", test_crypto_pem_encode, 0, NULL, NULL },
+  { "decode", test_crypto_pem_decode, 0, NULL, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c
new file mode 100644
index 0000000000..4a53639dad
--- /dev/null
+++ b/src/test/test_periodic_event.c
@@ -0,0 +1,333 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_periodic_event.c
+ * \brief Test the periodic events that Tor uses for different roles. They are
+ *        part of the libevent mainloop
+ */
+
+#define CONFIG_PRIVATE
+#define HS_SERVICE_PRIVATE
+#define MAINLOOP_PRIVATE
+
+#include "test/test.h"
+#include "test/test_helpers.h"
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/hs/hs_service.h"
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/periodic.h"
+
+/** Helper function: This is replaced in some tests for the event callbacks so
+ * we don't actually go into the code path of those callbacks. */
+static int
+dumb_event_fn(time_t now, const or_options_t *options)
+{
+  (void) now;
+  (void) options;
+
+  /* Will get rescheduled in 300 seconds. It just can't be 0. */
+  return 300;
+}
+
+static void
+register_dummy_hidden_service(hs_service_t *service)
+{
+  memset(service, 0, sizeof(hs_service_t));
+  memset(&service->keys.identity_pk, 'A', sizeof(service->keys.identity_pk));
+  (void) register_service(get_hs_service_map(), service);
+}
+
+static void
+test_pe_initialize(void *arg)
+{
+  (void) arg;
+
+  /* Initialize the events but the callback won't get called since we would
+   * need to run the main loop and then wait for a second delaying the unit
+   * tests. Instead, we'll test the callback work indepedently elsewhere. */
+  initialize_periodic_events();
+
+  /* Validate that all events have been set up. */
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+    tt_assert(item->ev);
+    tt_assert(item->fn);
+    tt_u64_op(item->last_action_time, OP_EQ, 0);
+    /* Every event must have role(s) assign to it. This is done statically. */
+    tt_u64_op(item->roles, OP_NE, 0);
+    tt_uint_op(periodic_event_is_enabled(item), OP_EQ, 0);
+  }
+
+ done:
+  teardown_periodic_events();
+}
+
+static void
+test_pe_launch(void *arg)
+{
+  hs_service_t service, *to_remove = NULL;
+  or_options_t *options;
+
+  (void) arg;
+
+  hs_init();
+  /* We need to put tor in hibernation live state so the events requiring
+   * network gets enabled. */
+  consider_hibernation(time(NULL));
+
+  /* Hack: We'll set a dumb fn() of each events so they don't get called when
+   * dispatching them. We just want to test the state of the callbacks, not
+   * the whole code path. */
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+    item->fn = dumb_event_fn;
+  }
+
+  options = get_options_mutable();
+  options->SocksPort_set = 1;
+  periodic_events_on_new_options(options);
+#if 0
+  /* Lets make sure that before intialization, we can't scan the periodic
+   * events list and launch them. Lets try by being a Client. */
+  /* XXXX We make sure these events are initialized now way earlier than we
+   * did before. */
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+    tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
+  }
+#endif
+
+  initialize_periodic_events();
+
+  /* Now that we've initialized, rescan the list to launch. */
+  periodic_events_on_new_options(options);
+
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+    if (item->roles & PERIODIC_EVENT_ROLE_CLIENT) {
+      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1);
+    } else {
+      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
+    }
+    // enabled or not, the event has not yet been run.
+    tt_u64_op(item->last_action_time, OP_EQ, 0);
+  }
+
+  /* Remove Client but become a Relay. */
+  options->SocksPort_set = 0;
+  options->ORPort_set = 1;
+  periodic_events_on_new_options(options);
+
+  unsigned roles = get_my_roles(options);
+  tt_uint_op(roles, OP_EQ,
+             PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER);
+
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+    /* Only Client role should be disabled. */
+    if (item->roles == PERIODIC_EVENT_ROLE_CLIENT) {
+      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
+    }
+    if (item->roles & PERIODIC_EVENT_ROLE_RELAY) {
+      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1);
+    }
+    /* Non Relay role should be disabled, except for Dirserver. */
+    if (!(item->roles & roles)) {
+      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
+    }
+  }
+
+  /* Disable everything and we'll enable them ALL. */
+  options->SocksPort_set = 0;
+  options->ORPort_set = 0;
+  periodic_events_on_new_options(options);
+
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+    tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
+  }
+
+  /* Enable everything. */
+  options->SocksPort_set = 1; options->ORPort_set = 1;
+  options->BridgeRelay = 1; options->AuthoritativeDir = 1;
+  options->V3AuthoritativeDir = 1; options->BridgeAuthoritativeDir = 1;
+  register_dummy_hidden_service(&service);
+  periodic_events_on_new_options(options);
+  /* Note down the reference because we need to remove this service from the
+   * global list before the hs_free_all() call so it doesn't try to free
+   * memory on the stack. Furthermore, we can't remove it now else it will
+   * trigger a rescan of the event disabling the HS service event. */
+  to_remove = &service;
+
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+    tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1);
+  }
+
+ done:
+  if (to_remove) {
+    remove_service(get_hs_service_map(), to_remove);
+  }
+  hs_free_all();
+}
+
+static void
+test_pe_get_roles(void *arg)
+{
+  int roles;
+
+  (void) arg;
+
+  /* Just so the HS global map exists. */
+  hs_init();
+
+  or_options_t *options = get_options_mutable();
+  tt_assert(options);
+
+  /* Nothing configured, should be no roles. */
+  roles = get_my_roles(options);
+  tt_int_op(roles, OP_EQ, 0);
+
+  /* Indicate we have a SocksPort, roles should be come Client. */
+  options->SocksPort_set = 1;
+  roles = get_my_roles(options);
+  tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT);
+
+  /* Now, we'll add a ORPort so should now be a Relay + Client. */
+  options->ORPort_set = 1;
+  roles = get_my_roles(options);
+  tt_int_op(roles, OP_EQ,
+            (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY |
+             PERIODIC_EVENT_ROLE_DIRSERVER));
+
+  /* Now add a Bridge. */
+  options->BridgeRelay = 1;
+  roles = get_my_roles(options);
+  tt_int_op(roles, OP_EQ,
+            (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY |
+             PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER));
+  tt_assert(roles & PERIODIC_EVENT_ROLE_ROUTER);
+  /* Unset client so we can solely test Router role. */
+  options->SocksPort_set = 0;
+  roles = get_my_roles(options);
+  tt_int_op(roles, OP_EQ,
+            PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER);
+
+  /* Reset options so we can test authorities. */
+  options->SocksPort_set = 0;
+  options->ORPort_set = 0;
+  options->BridgeRelay = 0;
+  roles = get_my_roles(options);
+  tt_int_op(roles, OP_EQ, 0);
+
+  /* Now upgrade to Dirauth. */
+  options->DirPort_set = 1;
+  options->AuthoritativeDir = 1;
+  options->V3AuthoritativeDir = 1;
+  roles = get_my_roles(options);
+  tt_int_op(roles, OP_EQ,
+            PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER);
+  tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
+
+  /* Now Bridge Authority. */
+  options->V3AuthoritativeDir = 0;
+  options->BridgeAuthoritativeDir = 1;
+  roles = get_my_roles(options);
+  tt_int_op(roles, OP_EQ,
+            PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER);
+  tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
+
+  /* Move that bridge auth to become a relay. */
+  options->ORPort_set = 1;
+  roles = get_my_roles(options);
+  tt_int_op(roles, OP_EQ,
+            (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY
+             | PERIODIC_EVENT_ROLE_DIRSERVER));
+  tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
+
+  /* And now an Hidden service. */
+  hs_service_t service;
+  register_dummy_hidden_service(&service);
+  roles = get_my_roles(options);
+  /* Remove it now so the hs_free_all() doesn't try to free stack memory. */
+  remove_service(get_hs_service_map(), &service);
+  tt_int_op(roles, OP_EQ,
+            (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY |
+             PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER));
+  tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
+
+ done:
+  hs_free_all();
+}
+
+static void
+test_pe_hs_service(void *arg)
+{
+  hs_service_t service, *to_remove = NULL;
+
+  (void) arg;
+
+  hs_init();
+  /* We need to put tor in hibernation live state so the events requiring
+   * network gets enabled. */
+  consider_hibernation(time(NULL));
+  /* Initialize the events so we can enable them */
+  initialize_periodic_events();
+
+  /* Hack: We'll set a dumb fn() of each events so they don't get called when
+   * dispatching them. We just want to test the state of the callbacks, not
+   * the whole code path. */
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+    item->fn = dumb_event_fn;
+  }
+
+  /* This should trigger a rescan of the list and enable the HS service
+   * events. */
+  register_dummy_hidden_service(&service);
+  /* Note down the reference because we need to remove this service from the
+   * global list before the hs_free_all() call so it doesn't try to free
+   * memory on the stack. Furthermore, we can't remove it now else it will
+   * trigger a rescan of the event disabling the HS service event. */
+  to_remove = &service;
+
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+    if (item->roles & PERIODIC_EVENT_ROLE_HS_SERVICE) {
+      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1);
+    }
+  }
+  to_remove = NULL;
+
+  /* Remove the service from the global map, it should trigger a rescan and
+   * disable the HS service events. */
+  remove_service(get_hs_service_map(), &service);
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+    if (item->roles & PERIODIC_EVENT_ROLE_HS_SERVICE) {
+      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
+    }
+  }
+
+ done:
+  if (to_remove) {
+    remove_service(get_hs_service_map(), to_remove);
+  }
+  hs_free_all();
+}
+
+#define PE_TEST(name) \
+  { #name, test_pe_## name , TT_FORK, NULL, NULL }
+
+struct testcase_t periodic_event_tests[] = {
+  PE_TEST(initialize),
+  PE_TEST(launch),
+  PE_TEST(get_roles),
+  PE_TEST(hs_service),
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_policy.c b/src/test/test_policy.c
index 1ffdc2cd51..9c001c294a 100644
--- a/src/test/test_policy.c
+++ b/src/test/test_policy.c
@@ -1,14 +1,22 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#include "or.h"
 #define CONFIG_PRIVATE
-#include "config.h"
-#include "router.h"
-#include "routerparse.h"
 #define POLICIES_PRIVATE
-#include "policies.h"
-#include "test.h"
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/or/policies.h"
+#include "feature/dirparse/policy_parse.h"
+#include "feature/relay/router.h"
+#include "lib/encoding/confline.h"
+#include "test/test.h"
+
+#include "core/or/addr_policy_st.h"
+#include "core/or/port_cfg_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
 
 /* Helper: assert that short_policy parses and writes back out as itself,
    or as <b>expected</b> if that's provided. */
@@ -59,7 +67,7 @@ test_policy_summary_helper_family_flags(const char *policy_str,
 
   summary = policy_summarize(policy, family);
 
-  tt_assert(summary != NULL);
+  tt_ptr_op(summary, OP_NE, NULL);
   tt_str_op(summary,OP_EQ, expected_summary);
 
   short_policy = parse_short_policy(summary);
@@ -147,7 +155,7 @@ test_policies_general(void *arg)
 
   p = router_parse_addr_policy_item_from_string("reject 192.168.0.0/16:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   tt_int_op(ADDR_POLICY_REJECT,OP_EQ, p->policy_type);
   tor_addr_from_ipv4h(&tar, 0xc0a80000u);
   tt_int_op(0,OP_EQ, tor_addr_compare(&p->addr, &tar, CMP_EXACT));
@@ -192,75 +200,75 @@ test_policies_general(void *arg)
   policy3 = smartlist_new();
   p = router_parse_addr_policy_item_from_string("reject *:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy3, p);
   p = router_parse_addr_policy_item_from_string("accept *:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy3, p);
 
   policy4 = smartlist_new();
   p = router_parse_addr_policy_item_from_string("accept *:443", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy4, p);
   p = router_parse_addr_policy_item_from_string("accept *:443", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy4, p);
 
   policy5 = smartlist_new();
   p = router_parse_addr_policy_item_from_string("reject 0.0.0.0/8:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy5, p);
   p = router_parse_addr_policy_item_from_string("reject 169.254.0.0/16:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy5, p);
   p = router_parse_addr_policy_item_from_string("reject 127.0.0.0/8:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy5, p);
   p = router_parse_addr_policy_item_from_string("reject 192.168.0.0/16:*",
                                                 -1, &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy5, p);
   p = router_parse_addr_policy_item_from_string("reject 10.0.0.0/8:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy5, p);
   p = router_parse_addr_policy_item_from_string("reject 172.16.0.0/12:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy5, p);
   p = router_parse_addr_policy_item_from_string("reject 80.190.250.90:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy5, p);
   p = router_parse_addr_policy_item_from_string("reject *:1-65534", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy5, p);
   p = router_parse_addr_policy_item_from_string("reject *:65535", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy5, p);
   p = router_parse_addr_policy_item_from_string("accept *:1-65535", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy5, p);
 
   policy6 = smartlist_new();
   p = router_parse_addr_policy_item_from_string("accept 43.3.0.0/9:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy6, p);
 
   policy7 = smartlist_new();
   p = router_parse_addr_policy_item_from_string("accept 0.0.0.0/8:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy7, p);
 
   tt_int_op(0, OP_EQ, policies_parse_exit_policy(NULL, &policy8,
@@ -282,13 +290,13 @@ test_policies_general(void *arg)
   policy10 = smartlist_new();
   p = router_parse_addr_policy_item_from_string("accept6 *:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy10, p);
 
   policy11 = smartlist_new();
   p = router_parse_addr_policy_item_from_string("reject6 *:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy11, p);
 
   tt_assert(!exit_policy_is_general_exit(policy));
@@ -392,21 +400,21 @@ test_policies_general(void *arg)
   p = router_parse_addr_policy_item_from_string("acce::abcd",
                                                 ADDR_POLICY_ACCEPT,
                                                 &malformed_list);
-  tt_assert(!p);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
   malformed_list = 0;
 
   p = router_parse_addr_policy_item_from_string("7:1234",
                                                 ADDR_POLICY_ACCEPT,
                                                 &malformed_list);
-  tt_assert(!p);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
   malformed_list = 0;
 
   p = router_parse_addr_policy_item_from_string("::",
                                                 ADDR_POLICY_ACCEPT,
                                                 &malformed_list);
-  tt_assert(!p);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
   malformed_list = 0;
 
@@ -968,73 +976,73 @@ test_policies_general(void *arg)
   /* Make sure that IPv4 addresses are ignored in accept6/reject6 lines. */
   p = router_parse_addr_policy_item_from_string("accept6 1.2.3.4:*", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(!malformed_list);
 
   p = router_parse_addr_policy_item_from_string("reject6 2.4.6.0/24:*", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(!malformed_list);
 
   p = router_parse_addr_policy_item_from_string("accept6 *4:*", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(!malformed_list);
 
   /* Make sure malformed policies are detected as such. */
   p = router_parse_addr_policy_item_from_string("bad_token *4:*", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
 
   p = router_parse_addr_policy_item_from_string("accept6 **:*", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
 
   p = router_parse_addr_policy_item_from_string("accept */15:*", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
 
   p = router_parse_addr_policy_item_from_string("reject6 */:*", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
 
   p = router_parse_addr_policy_item_from_string("accept 127.0.0.1/33:*", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
 
   p = router_parse_addr_policy_item_from_string("accept6 [::1]/129:*", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
 
   p = router_parse_addr_policy_item_from_string("reject 8.8.8.8/-1:*", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
 
   p = router_parse_addr_policy_item_from_string("reject 8.8.4.4:10-5", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
 
   p = router_parse_addr_policy_item_from_string("reject 1.2.3.4:-1", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
 
   /* Test a too-long policy. */
   {
     char *policy_strng = NULL;
     smartlist_t *chunks = smartlist_new();
-    smartlist_add(chunks, tor_strdup("accept "));
+    smartlist_add_strdup(chunks, "accept ");
     for (i=1; i<10000; ++i)
       smartlist_add_asprintf(chunks, "%d,", i);
-    smartlist_add(chunks, tor_strdup("20000"));
+    smartlist_add_strdup(chunks, "20000");
     policy_strng = smartlist_join_strings(chunks, "", 0, NULL);
     SMARTLIST_FOREACH(chunks, char *, ch, tor_free(ch));
     smartlist_free(chunks);
@@ -1048,9 +1056,9 @@ test_policies_general(void *arg)
   for (i=1; i<2000; i+=2) {
     char buf[POLICY_BUF_LEN];
     tor_snprintf(buf, sizeof(buf), "reject *:%d", i);
-    smartlist_add(sm, tor_strdup(buf));
+    smartlist_add_strdup(sm, buf);
   }
-  smartlist_add(sm, tor_strdup("accept *:*"));
+  smartlist_add_strdup(sm, "accept *:*");
   policy_str = smartlist_join_strings(sm, ",", 0, NULL);
   test_policy_summary_helper( policy_str,
     "accept 2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,"
@@ -1148,7 +1156,7 @@ test_policies_reject_exit_address(void *arg)
   /* test that IPv4 addresses are rejected on an IPv4-only exit */
   policies_parse_exit_policy_reject_private(&policy, 0, ipv4_list, 0, 0);
   tt_assert(policy);
-  tt_assert(smartlist_len(policy) == 1);
+  tt_int_op(smartlist_len(policy), OP_EQ, 1);
   tt_assert(test_policy_has_address_helper(policy, &ipv4_addr));
   addr_policy_list_free(policy);
   policy = NULL;
@@ -1158,12 +1166,12 @@ test_policies_reject_exit_address(void *arg)
    * on IPv4-only exits, so policies_parse_exit_policy_reject_private doesn't
    * need to do anything) */
   policies_parse_exit_policy_reject_private(&policy, 0, ipv6_list, 0, 0);
-  tt_assert(policy == NULL);
+  tt_ptr_op(policy, OP_EQ, NULL);
 
   /* test that only IPv4 addresses are rejected on an IPv4-only exit */
   policies_parse_exit_policy_reject_private(&policy, 0, both_list, 0, 0);
   tt_assert(policy);
-  tt_assert(smartlist_len(policy) == 1);
+  tt_int_op(smartlist_len(policy), OP_EQ, 1);
   tt_assert(test_policy_has_address_helper(policy, &ipv4_addr));
   addr_policy_list_free(policy);
   policy = NULL;
@@ -1171,7 +1179,7 @@ test_policies_reject_exit_address(void *arg)
   /* Test that lists with duplicate entries produce the same results */
   policies_parse_exit_policy_reject_private(&policy, 0, dupl_list, 0, 0);
   tt_assert(policy);
-  tt_assert(smartlist_len(policy) == 1);
+  tt_int_op(smartlist_len(policy), OP_EQ, 1);
   tt_assert(test_policy_has_address_helper(policy, &ipv4_addr));
   addr_policy_list_free(policy);
   policy = NULL;
@@ -1181,7 +1189,7 @@ test_policies_reject_exit_address(void *arg)
   /* test that IPv4 addresses are rejected on an IPv4/IPv6 exit */
   policies_parse_exit_policy_reject_private(&policy, 1, ipv4_list, 0, 0);
   tt_assert(policy);
-  tt_assert(smartlist_len(policy) == 1);
+  tt_int_op(smartlist_len(policy), OP_EQ, 1);
   tt_assert(test_policy_has_address_helper(policy, &ipv4_addr));
   addr_policy_list_free(policy);
   policy = NULL;
@@ -1189,7 +1197,7 @@ test_policies_reject_exit_address(void *arg)
   /* test that IPv6 addresses are rejected on an IPv4/IPv6 exit */
   policies_parse_exit_policy_reject_private(&policy, 1, ipv6_list,  0, 0);
   tt_assert(policy);
-  tt_assert(smartlist_len(policy) == 1);
+  tt_int_op(smartlist_len(policy), OP_EQ, 1);
   tt_assert(test_policy_has_address_helper(policy, &ipv6_addr));
   addr_policy_list_free(policy);
   policy = NULL;
@@ -1197,7 +1205,7 @@ test_policies_reject_exit_address(void *arg)
   /* test that IPv4 and IPv6 addresses are rejected on an IPv4/IPv6 exit */
   policies_parse_exit_policy_reject_private(&policy, 1, both_list,  0, 0);
   tt_assert(policy);
-  tt_assert(smartlist_len(policy) == 2);
+  tt_int_op(smartlist_len(policy), OP_EQ, 2);
   tt_assert(test_policy_has_address_helper(policy, &ipv4_addr));
   tt_assert(test_policy_has_address_helper(policy, &ipv6_addr));
   addr_policy_list_free(policy);
@@ -1206,7 +1214,7 @@ test_policies_reject_exit_address(void *arg)
   /* Test that lists with duplicate entries produce the same results */
   policies_parse_exit_policy_reject_private(&policy, 1, dupl_list,  0, 0);
   tt_assert(policy);
-  tt_assert(smartlist_len(policy) == 2);
+  tt_int_op(smartlist_len(policy), OP_EQ, 2);
   tt_assert(test_policy_has_address_helper(policy, &ipv4_addr));
   tt_assert(test_policy_has_address_helper(policy, &ipv6_addr));
   addr_policy_list_free(policy);
@@ -1258,7 +1266,7 @@ test_policies_reject_port_address(void *arg)
    * with IPv6 addresses on IPv4-only exits) */
   policies_parse_exit_policy_reject_private(&policy, 0, NULL, 0, 1);
   tt_assert(policy);
-  tt_assert(smartlist_len(policy) == 1);
+  tt_int_op(smartlist_len(policy), OP_EQ, 1);
   tt_assert(test_policy_has_address_helper(policy, &ipv4_port->addr));
   addr_policy_list_free(policy);
   policy = NULL;
@@ -1266,7 +1274,7 @@ test_policies_reject_port_address(void *arg)
   /* test that IPv4 and IPv6 ports are rejected on an IPv4/IPv6 exit */
   policies_parse_exit_policy_reject_private(&policy, 1, NULL, 0, 1);
   tt_assert(policy);
-  tt_assert(smartlist_len(policy) == 2);
+  tt_int_op(smartlist_len(policy), OP_EQ, 2);
   tt_assert(test_policy_has_address_helper(policy, &ipv4_port->addr));
   tt_assert(test_policy_has_address_helper(policy, &ipv6_port->addr));
   addr_policy_list_free(policy);
@@ -1318,7 +1326,7 @@ mock_get_interface_address6_list(int severity,
   return clone_list;
 
  done:
-  free_interface_address6_list(clone_list);
+  interface_address6_list_free(clone_list);
   return NULL;
 }
 
@@ -1337,7 +1345,7 @@ test_policies_reject_interface_address(void *arg)
 
   /* test that no addresses are rejected when none are supplied/requested */
   policies_parse_exit_policy_reject_private(&policy, 0, NULL, 0, 0);
-  tt_assert(policy == NULL);
+  tt_ptr_op(policy, OP_EQ, NULL);
 
   /* test that only IPv4 interface addresses are rejected on an IPv4-only exit
    * (and allow for duplicates)
@@ -1372,7 +1380,7 @@ test_policies_reject_interface_address(void *arg)
 
   /* test that no addresses are rejected when none are supplied/requested */
   policies_parse_exit_policy_reject_private(&policy, 0, NULL, 0, 0);
-  tt_assert(policy == NULL);
+  tt_ptr_op(policy, OP_EQ, NULL);
 
   /* test that only IPv4 interface addresses are rejected on an IPv4-only exit
    */
@@ -1393,11 +1401,11 @@ test_policies_reject_interface_address(void *arg)
 
  done:
   addr_policy_list_free(policy);
-  free_interface_address6_list(public_ipv4_addrs);
-  free_interface_address6_list(public_ipv6_addrs);
+  interface_address6_list_free(public_ipv4_addrs);
+  interface_address6_list_free(public_ipv6_addrs);
 
   UNMOCK(get_interface_address6_list);
-  /* we don't use free_interface_address6_list on these lists because their
+  /* we don't use interface_address6_list_free on these lists because their
    * address pointers are stack-based */
   smartlist_free(mock_ipv4_addrs);
   smartlist_free(mock_ipv6_addrs);
@@ -1496,9 +1504,21 @@ test_dump_exit_policy_to_string(void *arg)
 }
 
 static routerinfo_t *mock_desc_routerinfo = NULL;
+static int routerinfo_err;
+
 static const routerinfo_t *
-mock_router_get_my_routerinfo(void)
+mock_router_get_my_routerinfo_with_err(int *err)
 {
+  if (routerinfo_err) {
+    if (err)
+      *err = routerinfo_err;
+
+    return NULL;
+  }
+
+  if (err)
+    *err = 0;
+
   return mock_desc_routerinfo;
 }
 
@@ -1528,20 +1548,21 @@ test_policies_getinfo_helper_policies(void *arg)
   memset(&mock_my_routerinfo, 0, sizeof(mock_my_routerinfo));
 
   rv = getinfo_helper_policies(NULL, "exit-policy/default", &answer, &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   tt_assert(strlen(answer) > 0);
   tor_free(answer);
 
   rv = getinfo_helper_policies(NULL, "exit-policy/reject-private/default",
                                &answer, &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   tt_assert(strlen(answer) > 0);
   tor_free(answer);
 
   memset(&mock_my_routerinfo, 0, sizeof(routerinfo_t));
-  MOCK(router_get_my_routerinfo, mock_router_get_my_routerinfo);
+  MOCK(router_get_my_routerinfo_with_err,
+       mock_router_get_my_routerinfo_with_err);
   mock_my_routerinfo.exit_policy = smartlist_new();
   mock_desc_routerinfo = &mock_my_routerinfo;
 
@@ -1550,15 +1571,15 @@ test_policies_getinfo_helper_policies(void *arg)
 
   rv = getinfo_helper_policies(NULL, "exit-policy/reject-private/relay",
                                &answer, &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   tt_assert(strlen(answer) == 0);
   tor_free(answer);
 
   rv = getinfo_helper_policies(NULL, "exit-policy/ipv4", &answer,
                                &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   ipv4_len = strlen(answer);
   tt_assert(ipv4_len == 0 || ipv4_len == strlen(DEFAULT_POLICY_STRING));
   tt_assert(ipv4_len == 0 || !strcasecmp(answer, DEFAULT_POLICY_STRING));
@@ -1566,8 +1587,8 @@ test_policies_getinfo_helper_policies(void *arg)
 
   rv = getinfo_helper_policies(NULL, "exit-policy/ipv6", &answer,
                                &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   ipv6_len = strlen(answer);
   tt_assert(ipv6_len == 0 || ipv6_len == strlen(DEFAULT_POLICY_STRING));
   tt_assert(ipv6_len == 0 || !strcasecmp(answer, DEFAULT_POLICY_STRING));
@@ -1575,8 +1596,8 @@ test_policies_getinfo_helper_policies(void *arg)
 
   rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer,
                                &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   /* It's either empty or it's the default */
   tt_assert(strlen(answer) == 0 || !strcasecmp(answer, DEFAULT_POLICY_STRING));
   tor_free(answer);
@@ -1587,16 +1608,20 @@ test_policies_getinfo_helper_policies(void *arg)
   append_exit_policy_string(&mock_my_routerinfo.exit_policy, "reject *6:*");
 
   mock_options.IPv6Exit = 1;
-  tor_addr_from_ipv4h(&mock_options.OutboundBindAddressIPv4_, TEST_IPV4_ADDR);
-  tor_addr_parse(&mock_options.OutboundBindAddressIPv6_, TEST_IPV6_ADDR);
+  tor_addr_from_ipv4h(
+      &mock_options.OutboundBindAddresses[OUTBOUND_ADDR_EXIT][0],
+      TEST_IPV4_ADDR);
+  tor_addr_parse(
+      &mock_options.OutboundBindAddresses[OUTBOUND_ADDR_EXIT][1],
+      TEST_IPV6_ADDR);
 
   mock_options.ExitPolicyRejectPrivate = 1;
   mock_options.ExitPolicyRejectLocalInterfaces = 1;
 
   rv = getinfo_helper_policies(NULL, "exit-policy/reject-private/relay",
                                &answer, &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   tt_assert(strlen(answer) > 0);
   tor_free(answer);
 
@@ -1605,8 +1630,8 @@ test_policies_getinfo_helper_policies(void *arg)
 
   rv = getinfo_helper_policies(NULL, "exit-policy/reject-private/relay",
                                &answer, &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   tt_assert(strlen(answer) > 0);
   tor_free(answer);
 
@@ -1615,8 +1640,8 @@ test_policies_getinfo_helper_policies(void *arg)
 
   rv = getinfo_helper_policies(NULL, "exit-policy/reject-private/relay",
                                &answer, &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   tt_assert(strlen(answer) > 0);
   tor_free(answer);
 
@@ -1625,35 +1650,84 @@ test_policies_getinfo_helper_policies(void *arg)
 
   rv = getinfo_helper_policies(NULL, "exit-policy/reject-private/relay",
                                &answer, &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   tt_assert(strlen(answer) == 0);
   tor_free(answer);
 
   rv = getinfo_helper_policies(NULL, "exit-policy/ipv4", &answer,
                                &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   ipv4_len = strlen(answer);
   tt_assert(ipv4_len > 0);
   tor_free(answer);
 
   rv = getinfo_helper_policies(NULL, "exit-policy/ipv6", &answer,
                                &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   ipv6_len = strlen(answer);
   tt_assert(ipv6_len > 0);
   tor_free(answer);
 
   rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer,
                                &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   tt_assert(strlen(answer) > 0);
   tt_assert(strlen(answer) == ipv4_len + ipv6_len + 1);
   tor_free(answer);
 
+  routerinfo_err = TOR_ROUTERINFO_ERROR_NO_EXT_ADDR;
+  rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer,
+                               &errmsg);
+  tt_int_op(rv, OP_EQ, -1);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
+  tt_str_op(errmsg, OP_EQ, "No known exit address yet");
+
+  routerinfo_err = TOR_ROUTERINFO_ERROR_CANNOT_PARSE;
+  rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer,
+                               &errmsg);
+  tt_int_op(rv, OP_EQ, -1);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
+  tt_str_op(errmsg, OP_EQ, "Cannot parse descriptor");
+
+  routerinfo_err = TOR_ROUTERINFO_ERROR_NOT_A_SERVER;
+  rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer,
+                               &errmsg);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
+  tt_str_op(errmsg, OP_EQ, "Not running in server mode");
+
+  routerinfo_err = TOR_ROUTERINFO_ERROR_DIGEST_FAILED;
+  rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer,
+                               &errmsg);
+
+  tt_int_op(rv, OP_EQ, -1);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
+  tt_str_op(errmsg, OP_EQ, "Key digest failed");
+
+  routerinfo_err = TOR_ROUTERINFO_ERROR_CANNOT_GENERATE;
+  rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer,
+                               &errmsg);
+  tt_int_op(rv, OP_EQ, -1);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
+  tt_str_op(errmsg, OP_EQ, "Cannot generate descriptor");
+
+  routerinfo_err = TOR_ROUTERINFO_ERROR_DESC_REBUILDING;
+  rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer,
+                               &errmsg);
+  tt_int_op(rv, OP_EQ, -1);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
+  tt_str_op(errmsg, OP_EQ, "Descriptor still rebuilding - not ready yet");
+
  done:
   tor_free(answer);
   UNMOCK(get_options);
@@ -1742,34 +1816,34 @@ test_policies_fascist_firewall_allows_address(void *arg)
   mock_options.ClientUseIPv6 = 1;
   mock_options.UseBridges = 0;
 
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 0);
 
   /* Preferring IPv4 */
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, policy, 1, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, policy, 1, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 1, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 1, 0)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, policy, 1, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, policy, 1, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 1, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 1, 0),
+            OP_EQ, 0);
 
   /* Preferring IPv6 */
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, policy, 1, 1)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, policy, 1, 1)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 1, 1)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 1, 1)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, policy, 1, 1),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, policy, 1, 1),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 1, 1),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 1, 1),
+            OP_EQ, 0);
 
   /* Test the function's address matching with UseBridges on */
   memset(&mock_options, 0, sizeof(or_options_t));
@@ -1777,46 +1851,46 @@ test_policies_fascist_firewall_allows_address(void *arg)
   mock_options.ClientUseIPv6 = 1;
   mock_options.UseBridges = 1;
 
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 0);
 
   /* Preferring IPv4 */
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, policy, 1, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, policy, 1, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 1, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 1, 0)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, policy, 1, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, policy, 1, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 1, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 1, 0),
+            OP_EQ, 0);
 
   /* Preferring IPv6 */
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, policy, 1, 1)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, policy, 1, 1)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 1, 1)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 1, 1)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, policy, 1, 1),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, policy, 1, 1),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 1, 1),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 1, 1),
+            OP_EQ, 0);
 
   /* bridge clients always use IPv6, regardless of ClientUseIPv6 */
   mock_options.ClientUseIPv4 = 1;
   mock_options.ClientUseIPv6 = 0;
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 0);
 
   /* Test the function's address matching with IPv4 on */
   memset(&mock_options, 0, sizeof(or_options_t));
@@ -1824,14 +1898,14 @@ test_policies_fascist_firewall_allows_address(void *arg)
   mock_options.ClientUseIPv6 = 0;
   mock_options.UseBridges = 0;
 
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 0);
 
   /* Test the function's address matching with IPv6 on */
   memset(&mock_options, 0, sizeof(or_options_t));
@@ -1839,14 +1913,14 @@ test_policies_fascist_firewall_allows_address(void *arg)
   mock_options.ClientUseIPv6 = 1;
   mock_options.UseBridges = 0;
 
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 0);
 
   /* Test the function's address matching with ClientUseIPv4 0.
    * This means "use IPv6" regardless of the other settings. */
@@ -1855,14 +1929,14 @@ test_policies_fascist_firewall_allows_address(void *arg)
   mock_options.ClientUseIPv6 = 0;
   mock_options.UseBridges = 0;
 
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 0);
 
   /* Test the function's address matching for unusual inputs */
   memset(&mock_options, 0, sizeof(or_options_t));
@@ -1871,27 +1945,28 @@ test_policies_fascist_firewall_allows_address(void *arg)
   mock_options.UseBridges = 1;
 
   /* NULL and tor_addr_is_null addresses are rejected */
-  tt_assert(fascist_firewall_allows_address(NULL, port, policy, 0, 0) == 0);
-  tt_assert(fascist_firewall_allows_address(&n_ipv4_addr, port, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&n_ipv6_addr, port, policy, 0, 0)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(NULL, port, policy, 0, 0), OP_EQ,
+            0);
+  tt_int_op(fascist_firewall_allows_address(&n_ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&n_ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 0);
 
   /* zero ports are rejected */
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, 0, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, 0, policy, 0, 0)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, 0, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, 0, policy, 0, 0),
+            OP_EQ, 0);
 
   /* NULL and empty policies accept everything */
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, NULL, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, NULL, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, e_policy, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, e_policy, 0, 0)
-            == 1);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, NULL, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, NULL, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, e_policy, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, e_policy, 0, 0),
+            OP_EQ, 1);
 
  done:
   addr_policy_free(item);
@@ -1918,11 +1993,8 @@ test_policies_fascist_firewall_allows_address(void *arg)
     tor_addr_port_t chosen_rs_ap; \
     tor_addr_make_null(&chosen_rs_ap.addr, AF_INET); \
     chosen_rs_ap.port = 0; \
-    tt_int_op(fascist_firewall_choose_address_rs(&(fake_rs), \
-                                                 (fw_connection), \
-                                                 (pref_only), \
-                                                 &chosen_rs_ap), \
-              OP_EQ, (expect_rv)); \
+    fascist_firewall_choose_address_rs(&(fake_rs), (fw_connection), \
+                                       (pref_only), &chosen_rs_ap); \
     tt_assert(tor_addr_eq(&(expect_ap).addr, &chosen_rs_ap.addr)); \
     tt_int_op((expect_ap).port, OP_EQ, chosen_rs_ap.port); \
   STMT_END
@@ -1935,11 +2007,8 @@ test_policies_fascist_firewall_allows_address(void *arg)
     tor_addr_port_t chosen_node_ap; \
     tor_addr_make_null(&chosen_node_ap.addr, AF_INET); \
     chosen_node_ap.port = 0; \
-    tt_int_op(fascist_firewall_choose_address_node(&(fake_node), \
-                                                   (fw_connection), \
-                                                   (pref_only), \
-                                                   &chosen_node_ap), \
-              OP_EQ, (expect_rv)); \
+    fascist_firewall_choose_address_node(&(fake_node),(fw_connection), \
+                                         (pref_only), &chosen_node_ap); \
     tt_assert(tor_addr_eq(&(expect_ap).addr, &chosen_node_ap.addr)); \
     tt_int_op((expect_ap).port, OP_EQ, chosen_node_ap.port); \
   STMT_END
@@ -2028,12 +2097,12 @@ test_policies_fascist_firewall_choose_address(void *arg)
             == &ipv6_or_ap);
 
   /* null both OR addresses */
-  tt_assert(fascist_firewall_choose_address(&n_ipv4_ap, &n_ipv6_ap, 0,
-                                            FIREWALL_OR_CONNECTION, 0, 1)
-            == NULL);
-  tt_assert(fascist_firewall_choose_address(&n_ipv4_ap, &n_ipv6_ap, 1,
-                                            FIREWALL_OR_CONNECTION, 0, 0)
-            == NULL);
+  tt_ptr_op(fascist_firewall_choose_address(&n_ipv4_ap, &n_ipv6_ap, 0,
+                                            FIREWALL_OR_CONNECTION, 0, 1),
+            OP_EQ, NULL);
+  tt_ptr_op(fascist_firewall_choose_address(&n_ipv4_ap, &n_ipv6_ap, 1,
+                                            FIREWALL_OR_CONNECTION, 0, 0),
+            OP_EQ, NULL);
 
   /* null preferred Dir addresses */
   tt_assert(fascist_firewall_choose_address(&ipv4_dir_ap, &n_ipv6_ap, 0,
@@ -2044,12 +2113,12 @@ test_policies_fascist_firewall_choose_address(void *arg)
             == &ipv6_dir_ap);
 
   /* null both Dir addresses */
-  tt_assert(fascist_firewall_choose_address(&n_ipv4_ap, &n_ipv6_ap, 0,
-                                            FIREWALL_DIR_CONNECTION, 0, 1)
-            == NULL);
-  tt_assert(fascist_firewall_choose_address(&n_ipv4_ap, &n_ipv6_ap, 1,
-                                            FIREWALL_DIR_CONNECTION, 0, 0)
-            == NULL);
+  tt_ptr_op(fascist_firewall_choose_address(&n_ipv4_ap, &n_ipv6_ap, 0,
+                                            FIREWALL_DIR_CONNECTION, 0, 1),
+            OP_EQ, NULL);
+  tt_ptr_op(fascist_firewall_choose_address(&n_ipv4_ap, &n_ipv6_ap, 1,
+                                            FIREWALL_DIR_CONNECTION, 0, 0),
+            OP_EQ, NULL);
 
   /* Prefer IPv4 but want IPv6 (contradictory) */
   tt_assert(fascist_firewall_choose_address(&ipv4_or_ap, &ipv6_or_ap, 0,
@@ -2384,4 +2453,3 @@ struct testcase_t policy_tests[] = {
     test_policies_fascist_firewall_choose_address, 0, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_procmon.c b/src/test/test_procmon.c
index 9e63fc006d..e23578f4fd 100644
--- a/src/test/test_procmon.c
+++ b/src/test/test_procmon.c
@@ -1,14 +1,14 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define PROCMON_PRIVATE
 #include "orconfig.h"
-#include "or.h"
-#include "test.h"
+#include "core/or/or.h"
+#include "test/test.h"
 
-#include "procmon.h"
+#include "lib/evloop/procmon.h"
 
-#include "log_test_helpers.h"
+#include "test/log_test_helpers.h"
 
 #define NS_MODULE procmon
 
diff --git a/src/test/test_proto_http.c b/src/test/test_proto_http.c
new file mode 100644
index 0000000000..08990c0b6a
--- /dev/null
+++ b/src/test/test_proto_http.c
@@ -0,0 +1,213 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_proto_http.c
+ * \brief Tests for our HTTP protocol parser code
+ */
+
+#include "core/or/or.h"
+#include "test/test.h"
+#include "lib/container/buffers.h"
+#include "core/proto/proto_http.h"
+#include "test/log_test_helpers.h"
+
+#define S(str) str, sizeof(str)-1
+
+static void
+test_proto_http_peek(void *arg)
+{
+  (void) arg;
+  const struct {
+    int is_http;
+    const char *message;
+    size_t len;
+  } cases[] = {
+    { 1, S("GET /index HTTP/1.0\r\n") },
+    { 1, S("GET /index HTTP/1.1\r\n") },
+    { 1, S("GET ") },
+    { 0, S("GIT ") },
+    { 0, S("GET") },
+    { 0, S("get ") },
+    { 0, S("GETAWAY") },
+  };
+  unsigned i;
+  buf_t *buf = buf_new();
+  for (i = 0; i < ARRAY_LENGTH(cases); ++i) {
+    TT_BLATHER(("Trying case %u", i));
+    buf_add(buf, cases[i].message, cases[i].len);
+    tt_int_op(cases[i].is_http, OP_EQ, peek_buf_has_http_command(buf));
+    buf_clear(buf);
+  }
+ done:
+  buf_free(buf);
+}
+
+static void
+test_proto_http_valid(void *arg)
+{
+  (void) arg;
+  const struct {
+    const char *message;
+    size_t len;
+    const char *headers;
+    const char *body;
+    size_t bodylen;
+    int should_detect_truncated;
+    int bytes_left_over;
+  } cases[] = {
+    { S("GET /index.html HTTP/1.0\r\n\r\n"),
+      "GET /index.html HTTP/1.0\r\n\r\n",
+      S(""),
+      1, 0,
+    },
+    { S("PUT /tor/foo HTTP/1.1\r\n"
+        "Content-Length: 51\r\n\r\n"
+        "this is a test of the http parsing system . test te"),
+      "PUT /tor/foo HTTP/1.1\r\n" "Content-Length: 51\r\n\r\n",
+      S("this is a test of the http parsing system . test te"),
+      1, 0,
+    },
+    { S("PUT /tor/foo HTTP/1.1\r\n"
+        "Content-Length: 5\r\n\r\n"
+        "there are more than 5 characters in this body."),
+      "PUT /tor/foo HTTP/1.1\r\n" "Content-Length: 5\r\n\r\n",
+      S("there"),
+      0, 41,
+    },
+    { S("PUT /tor/bar HTTP/1.1\r\n\r\n"
+        "this is another \x00test"),
+      "PUT /tor/bar HTTP/1.1\r\n\r\n",
+      S("this is another \x00test"),
+      0, 0,
+    }
+  };
+  unsigned i;
+  buf_t *buf = buf_new();
+  char *h = NULL, *b = NULL;
+
+  for (i = 0; i < ARRAY_LENGTH(cases); ++i) {
+    TT_BLATHER(("Trying case %u", i));
+    size_t bl = 0;
+    // truncate by 2 chars
+    buf_add(buf, cases[i].message, cases[i].len - 2);
+
+    if (cases[i].should_detect_truncated) {
+      tt_int_op(0, OP_EQ, fetch_from_buf_http(buf, &h, 1024*16,
+                                              &b, &bl, 1024*16, 0));
+      tt_ptr_op(h, OP_EQ, NULL);
+      tt_ptr_op(b, OP_EQ, NULL);
+      tt_u64_op(bl, OP_EQ, 0);
+      tt_int_op(buf_datalen(buf), OP_EQ, cases[i].len - 2);
+    }
+
+    // add the rest.
+    buf_add(buf, cases[i].message+cases[i].len-2, 2);
+    tt_int_op(1, OP_EQ, fetch_from_buf_http(buf, &h, 1024*16,
+                                            &b, &bl, 1024*16, 0));
+    tt_str_op(h, OP_EQ, cases[i].headers);
+    tt_u64_op(bl, OP_EQ, cases[i].bodylen);
+    tt_mem_op(b, OP_EQ, cases[i].body, bl);
+    tt_int_op(buf_datalen(buf), OP_EQ, cases[i].bytes_left_over);
+
+    buf_clear(buf);
+    tor_free(h);
+    tor_free(b);
+  }
+ done:
+  tor_free(h);
+  tor_free(b);
+  buf_free(buf);
+}
+
+static void
+test_proto_http_invalid(void *arg)
+{
+  (void) arg;
+  const struct {
+    const char *message;
+    size_t len;
+    const char *expect;
+  } cases[] = {
+    /* Overlong headers, headers not finished. */
+    { S("GET /index.xhml HTTP/1.0\r\n"
+        "X-My-headers-are-too-long: yes indeed they are. They might be\r\n"
+        "X-My-headers-are-too-long: normal under other circumstances, but\r\n"
+        "X-My-headers-are-too-long: the 128-byte limit makes them bad\r\n"),
+      "headers too long." },
+    /* Overlong finished headers. */
+    { S("GET /index.xhml HTTP/1.0\r\n"
+        "X-My-headers-are-too-long: yes indeed they are. They might be\r\n"
+        "X-My-headers-are-too-long: normal under other circumstances, but\r\n"
+        "X-My-headers-are-too-long: the 128-byte limit makes them bad\r\n"
+        "\r\n"),
+        "headers too long." },
+    /* Exactly too long finished headers. */
+    { S("GET /index.xhml HTTP/1.0\r\n"
+        "X-My-headers-are-too-long: yes indeed they are. They might be\r\n"
+        "X-My-headers-are-too-long: normal un\r\n\r\n"),
+      "headerlen 129 larger than 127. Failing." },
+    /* Body too long, with content-length */
+    { S("GET /index.html HTTP/1.0\r\n"
+        "Content-Length: 129\r\n\r\n"
+        "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+        "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+        "xxxxxxxxxxxxxxxxxxx"),
+      "bodylen 129 larger than 127" },
+    /* Body too long, with content-length lying */
+    { S("GET /index.html HTTP/1.0\r\n"
+        "Content-Length: 99999\r\n\r\n"
+        "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+        "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+        "xxxxxxxxxxxxxxxxxxxxxxxxxxxx"),
+      "bodylen 138 larger than 127" },
+    /* Body too long, no content-length. */
+    { S("GET /index.html HTTP/1.0\r\n\r\n"
+        "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+        "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+        "xxxxxxxxxxxxxxxxxxxxxxxxxxxxz"),
+      "bodylen 139 larger than 127" },
+    /* Content-Length is junk. */
+    { S("GET /index.html HTTP/1.0\r\n"
+        "Content-Length: Cheese\r\n\r\n"
+        "foo"),
+      "Content-Length is bogus; maybe someone is trying to crash us." },
+    };
+  unsigned i;
+  buf_t *buf = buf_new();
+  char *h = NULL, *b = NULL;
+  setup_capture_of_logs(LOG_DEBUG);
+
+  for (i = 0; i < ARRAY_LENGTH(cases); ++i) {
+    TT_BLATHER(("Trying case %u", i));
+    size_t bl = 0;
+    buf_add(buf, cases[i].message, cases[i].len);
+
+    /* Use low body limits here so we can force over-sized object warnings */
+    tt_int_op(-1, OP_EQ, fetch_from_buf_http(buf, &h, 128,
+                                             &b, &bl, 128, 0));
+    tt_ptr_op(h, OP_EQ, NULL);
+    tt_ptr_op(b, OP_EQ, NULL);
+    tt_u64_op(bl, OP_EQ, 0);
+    expect_log_msg_containing(cases[i].expect);
+
+    buf_clear(buf);
+    tor_free(h);
+    tor_free(b);
+    mock_clean_saved_logs();
+  }
+ done:
+  tor_free(h);
+  tor_free(b);
+  buf_free(buf);
+  teardown_capture_of_logs();
+}
+
+struct testcase_t proto_http_tests[] = {
+  { "peek", test_proto_http_peek, 0, NULL, NULL },
+  { "valid", test_proto_http_valid, 0, NULL, NULL },
+  { "invalid", test_proto_http_invalid, 0, NULL, NULL },
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_proto_misc.c b/src/test/test_proto_misc.c
new file mode 100644
index 0000000000..af9cf7eee2
--- /dev/null
+++ b/src/test/test_proto_misc.c
@@ -0,0 +1,265 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_proto_misc.c
+ * \brief Test our smaller buffer-based protocol functions
+ */
+
+#include "core/or/or.h"
+#include "test/test.h"
+#include "lib/container/buffers.h"
+#include "core/or/connection_or.h"
+#include "feature/relay/ext_orport.h"
+#include "core/proto/proto_cell.h"
+#include "core/proto/proto_control0.h"
+#include "core/proto/proto_ext_or.h"
+
+#include "core/or/var_cell_st.h"
+
+static void
+test_proto_var_cell(void *arg)
+{
+  (void)arg;
+  char *mem_op_hex_tmp = NULL;
+  char tmp[1024];
+  buf_t *buf = NULL;
+  var_cell_t *cell = NULL;
+
+  buf = buf_new();
+  memset(tmp, 0xf0, sizeof(tmp));
+
+  /* Short little commands will make us say "no cell yet." */
+  tt_int_op(0, OP_EQ, fetch_var_cell_from_buf(buf, &cell, 4));
+  tt_ptr_op(cell, OP_EQ, NULL);
+  buf_add(buf, "\x01\x02\x02\0x2", 4);
+  tt_int_op(0, OP_EQ, fetch_var_cell_from_buf(buf, &cell, 4));
+  /* An incomplete fixed-length cell makes us say "no cell yet". */
+  buf_add(buf, "\x03", 1);
+  tt_int_op(0, OP_EQ, fetch_var_cell_from_buf(buf, &cell, 4));
+  /* A complete fixed length-cell makes us say "not a variable-length cell" */
+  buf_add(buf, tmp, 509);
+  tt_int_op(0, OP_EQ, fetch_var_cell_from_buf(buf, &cell, 4));
+  buf_clear(buf);
+
+  /* An incomplete versions cell is a variable-length cell that isn't ready
+   * yet. */
+  buf_add(buf,
+          "\x01\x02\x03\x04" /* circid */
+          "\x07" /* VERSIONS */
+          "\x00\x04" /* 4 bytes long */
+          "\x00" /* incomplete */, 8);
+  tt_int_op(1, OP_EQ, fetch_var_cell_from_buf(buf, &cell, 4));
+  tt_ptr_op(cell, OP_EQ, NULL);
+  /* Complete it, and it's a variable-length cell. Leave a byte on the end for
+   * fun. */
+  buf_add(buf, "\x09\x00\x25\ff", 4);
+  tt_int_op(1, OP_EQ, fetch_var_cell_from_buf(buf, &cell, 4));
+  tt_ptr_op(cell, OP_NE, NULL);
+  tt_int_op(cell->command, OP_EQ, CELL_VERSIONS);
+  tt_uint_op(cell->circ_id, OP_EQ, 0x01020304);
+  tt_int_op(cell->payload_len, OP_EQ, 4);
+  test_mem_op_hex(cell->payload, OP_EQ, "00090025");
+  var_cell_free(cell);
+  cell = NULL;
+  tt_int_op(buf_datalen(buf), OP_EQ, 1);
+  buf_clear(buf);
+
+  /* In link protocol 3 and earlier, circid fields were two bytes long. Let's
+   * ensure that gets handled correctly. */
+  buf_add(buf,
+          "\x23\x45\x81\x00\x06" /* command 81; 6 bytes long */
+          "coraje", 11);
+  tt_int_op(1, OP_EQ, fetch_var_cell_from_buf(buf, &cell, 3));
+  tt_ptr_op(cell, OP_NE, NULL);
+  tt_int_op(cell->command, OP_EQ, 129);
+  tt_uint_op(cell->circ_id, OP_EQ, 0x2345);
+  tt_int_op(cell->payload_len, OP_EQ, 6);
+  tt_mem_op(cell->payload, OP_EQ, "coraje", 6);
+  var_cell_free(cell);
+  cell = NULL;
+  tt_int_op(buf_datalen(buf), OP_EQ, 0);
+
+  /* In link protocol 2, only VERSIONS cells counted as variable-length */
+  buf_add(buf,
+          "\x23\x45\x81\x00\x06"
+          "coraje", 11); /* As above */
+  tt_int_op(0, OP_EQ, fetch_var_cell_from_buf(buf, &cell, 2));
+  buf_clear(buf);
+  buf_add(buf,
+          "\x23\x45\x07\x00\x06"
+          "futuro", 11);
+  tt_int_op(1, OP_EQ, fetch_var_cell_from_buf(buf, &cell, 2));
+  tt_ptr_op(cell, OP_NE, NULL);
+  tt_int_op(cell->command, OP_EQ, 7);
+  tt_uint_op(cell->circ_id, OP_EQ, 0x2345);
+  tt_int_op(cell->payload_len, OP_EQ, 6);
+  tt_mem_op(cell->payload, OP_EQ, "futuro", 6);
+  var_cell_free(cell);
+  cell = NULL;
+
+ done:
+  buf_free(buf);
+  var_cell_free(cell);
+  tor_free(mem_op_hex_tmp);
+}
+
+static void
+test_proto_control0(void *arg)
+{
+  (void)arg;
+  buf_t *buf = buf_new();
+
+  /* The only remaining function for the v0 control protocol is the function
+     that detects whether the user has stumbled across an old controller
+     that's using it.  The format was:
+        u16 length;
+        u16 command;
+        u8 body[length];
+  */
+
+  /* Empty buffer -- nothing to do. */
+  tt_int_op(0, OP_EQ, peek_buf_has_control0_command(buf));
+  /* 3 chars in buf -- can't tell */
+  buf_add(buf, "AUT", 3);
+  tt_int_op(0, OP_EQ, peek_buf_has_control0_command(buf));
+  /* command in buf -- easy to tell */
+  buf_add(buf, "HENTICATE ", 10);
+  tt_int_op(0, OP_EQ, peek_buf_has_control0_command(buf));
+
+  /* Control0 command header in buf: make sure we detect it. */
+  buf_clear(buf);
+  buf_add(buf, "\x09\x05" "\x00\x05" "blah", 8);
+  tt_int_op(1, OP_EQ, peek_buf_has_control0_command(buf));
+
+ done:
+  buf_free(buf);
+}
+
+static void
+test_proto_ext_or_cmd(void *arg)
+{
+  ext_or_cmd_t *cmd = NULL;
+  buf_t *buf = buf_new();
+  char *tmp = NULL;
+  (void) arg;
+
+  /* Empty -- should give "not there. */
+  tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
+  tt_ptr_op(NULL, OP_EQ, cmd);
+
+  /* Three bytes: shouldn't work. */
+  buf_add(buf, "\x00\x20\x00", 3);
+  tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
+  tt_ptr_op(NULL, OP_EQ, cmd);
+  tt_int_op(3, OP_EQ, buf_datalen(buf));
+
+  /* 0020 0000: That's a nil command. It should work. */
+  buf_add(buf, "\x00", 1);
+  tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
+  tt_ptr_op(NULL, OP_NE, cmd);
+  tt_int_op(0x20, OP_EQ, cmd->cmd);
+  tt_int_op(0, OP_EQ, cmd->len);
+  tt_int_op(0, OP_EQ, buf_datalen(buf));
+  ext_or_cmd_free(cmd);
+  cmd = NULL;
+
+  /* Now try a length-6 command with one byte missing. */
+  buf_add(buf, "\x10\x21\x00\x06""abcde", 9);
+  tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
+  tt_ptr_op(NULL, OP_EQ, cmd);
+  buf_add(buf, "f", 1);
+  tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
+  tt_ptr_op(NULL, OP_NE, cmd);
+  tt_int_op(0x1021, OP_EQ, cmd->cmd);
+  tt_int_op(6, OP_EQ, cmd->len);
+  tt_mem_op("abcdef", OP_EQ, cmd->body, 6);
+  tt_int_op(0, OP_EQ, buf_datalen(buf));
+  ext_or_cmd_free(cmd);
+  cmd = NULL;
+
+  /* Now try a length-10 command with 4 extra bytes. */
+  buf_add(buf, "\xff\xff\x00\x0aloremipsum\x10\x00\xff\xff", 18);
+  tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
+  tt_ptr_op(NULL, OP_NE, cmd);
+  tt_int_op(0xffff, OP_EQ, cmd->cmd);
+  tt_int_op(10, OP_EQ, cmd->len);
+  tt_mem_op("loremipsum", OP_EQ, cmd->body, 10);
+  tt_int_op(4, OP_EQ, buf_datalen(buf));
+  ext_or_cmd_free(cmd);
+  cmd = NULL;
+
+  /* Finally, let's try a maximum-length command. We already have the header
+   * waiting. */
+  tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
+  tmp = tor_malloc_zero(65535);
+  buf_add(buf, tmp, 65535);
+  tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
+  tt_ptr_op(NULL, OP_NE, cmd);
+  tt_int_op(0x1000, OP_EQ, cmd->cmd);
+  tt_int_op(0xffff, OP_EQ, cmd->len);
+  tt_mem_op(tmp, OP_EQ, cmd->body, 65535);
+  tt_int_op(0, OP_EQ, buf_datalen(buf));
+  ext_or_cmd_free(cmd);
+  cmd = NULL;
+
+ done:
+  ext_or_cmd_free(cmd);
+  buf_free(buf);
+  tor_free(tmp);
+}
+
+static void
+test_proto_line(void *arg)
+{
+  (void)arg;
+  char tmp[60];
+  buf_t *buf = buf_new();
+#define S(str) str, sizeof(str)-1
+  const struct {
+    const char *input;
+    size_t input_len;
+    size_t line_len;
+    const char *output;
+    int returnval;
+  } cases[] = {
+    { S("Hello world"), 0, NULL, 0 },
+    { S("Hello world\n"), 12, "Hello world\n", 1 },
+    { S("Hello world\nMore"), 12, "Hello world\n", 1 },
+    { S("\n oh hello world\nMore"), 1, "\n", 1 },
+    { S("Hello worpd\n\nMore"), 12, "Hello worpd\n", 1 },
+    { S("------------------------------------------------------------\n"), 0,
+      NULL, -1 },
+  };
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(cases); ++i) {
+    buf_add(buf, cases[i].input, cases[i].input_len);
+    memset(tmp, 0xfe, sizeof(tmp));
+    size_t sz = sizeof(tmp);
+    int rv = buf_get_line(buf, tmp, &sz);
+    tt_int_op(rv, OP_EQ, cases[i].returnval);
+    if (rv == 1) {
+      tt_int_op(sz, OP_LT, sizeof(tmp));
+      tt_mem_op(cases[i].output, OP_EQ, tmp, sz+1);
+      tt_int_op(buf_datalen(buf), OP_EQ, cases[i].input_len - strlen(tmp));
+      tt_int_op(sz, OP_EQ, cases[i].line_len);
+    } else {
+      tt_int_op(buf_datalen(buf), OP_EQ, cases[i].input_len);
+      // tt_int_op(sz, OP_EQ, sizeof(tmp));
+    }
+    buf_clear(buf);
+  }
+
+ done:
+  buf_free(buf);
+}
+
+struct testcase_t proto_misc_tests[] = {
+  { "var_cell", test_proto_var_cell, 0, NULL, NULL },
+  { "control0", test_proto_control0, 0, NULL, NULL },
+  { "ext_or_cmd", test_proto_ext_or_cmd, TT_FORK, NULL, NULL },
+  { "line", test_proto_line, 0, NULL, NULL },
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_protover.c b/src/test/test_protover.c
index c4379a15e1..63c508bd13 100644
--- a/src/test/test_protover.c
+++ b/src/test/test_protover.c
@@ -1,17 +1,28 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define PROTOVER_PRIVATE
 
 #include "orconfig.h"
-#include "test.h"
+#include "test/test.h"
 
-#include "protover.h"
+#include "core/or/protover.h"
+
+#include "core/or/or.h"
+#include "core/or/connection_or.h"
+#include "lib/tls/tortls.h"
 
 static void
 test_protover_parse(void *arg)
 {
   (void) arg;
+#ifdef HAVE_RUST
+  /** This test is disabled on rust builds, because it only exists to test
+   * internal C functions. */
+  tt_skip();
+ done:
+  ;
+#else
   char *re_encoded = NULL;
 
   const char *orig = "Foo=1,3 Bar=3 Baz= Quux=9-12,14,15-16,900";
@@ -78,38 +89,51 @@ test_protover_parse(void *arg)
     SMARTLIST_FOREACH(elts, proto_entry_t *, ent, proto_entry_free(ent));
   smartlist_free(elts);
   tor_free(re_encoded);
+#endif
 }
 
 static void
 test_protover_parse_fail(void *arg)
 {
   (void)arg;
+#ifdef HAVE_RUST
+  /** This test is disabled on rust builds, because it only exists to test
+   * internal C functions. */
+  tt_skip();
+#else
   smartlist_t *elts;
 
   /* random junk */
   elts = parse_protocol_list("!!3@*");
-  tt_assert(elts == NULL);
+  tt_ptr_op(elts, OP_EQ, NULL);
 
   /* Missing equals sign in an entry */
   elts = parse_protocol_list("Link=4 Haprauxymatyve Desc=9");
-  tt_assert(elts == NULL);
+  tt_ptr_op(elts, OP_EQ, NULL);
 
   /* Missing word. */
   elts = parse_protocol_list("Link=4 =3 Desc=9");
-  tt_assert(elts == NULL);
+  tt_ptr_op(elts, OP_EQ, NULL);
 
   /* Broken numbers */
   elts = parse_protocol_list("Link=fred");
-  tt_assert(elts == NULL);
+  tt_ptr_op(elts, OP_EQ, NULL);
   elts = parse_protocol_list("Link=1,fred");
-  tt_assert(elts == NULL);
+  tt_ptr_op(elts, OP_EQ, NULL);
   elts = parse_protocol_list("Link=1,fred,3");
-  tt_assert(elts == NULL);
+  tt_ptr_op(elts, OP_EQ, NULL);
 
   /* Broken range */
   elts = parse_protocol_list("Link=1,9-8,3");
-  tt_assert(elts == NULL);
+  tt_ptr_op(elts, OP_EQ, NULL);
+
+  /* Protocol name too long */
+  elts = parse_protocol_list("DoSaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+                           "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+                           "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+  tt_ptr_op(elts, OP_EQ, NULL);
 
+#endif
  done:
   ;
 }
@@ -144,6 +168,14 @@ test_protover_vote(void *arg)
   tt_str_op(result, OP_EQ, "");
   tor_free(result);
 
+  /* Don't count double-voting. */
+  smartlist_clear(lst);
+  smartlist_add(lst, (void*) "Foo=1 Foo=1");
+  smartlist_add(lst, (void*) "Bar=1-2,2-3");
+  result = protover_compute_vote(lst, 2);
+  tt_str_op(result, OP_EQ, "");
+  tor_free(result);
+
   /* Bad votes: the result must be empty */
   smartlist_clear(lst);
   smartlist_add(lst, (void*) "Faux=10-5");
@@ -203,6 +235,15 @@ test_protover_vote(void *arg)
   tt_str_op(result, OP_EQ, "");
   tor_free(result);
 
+  /* Protocol name too long */
+  smartlist_clear(lst);
+  smartlist_add(lst, (void*) "DoSaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+                         "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+                         "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+  result = protover_compute_vote(lst, 1);
+  tt_str_op(result, OP_EQ, "");
+  tor_free(result);
+
  done:
   tor_free(result);
   smartlist_free(lst);
@@ -215,18 +256,19 @@ test_protover_all_supported(void *arg)
   char *msg = NULL;
 
   tt_assert(protover_all_supported(NULL, &msg));
-  tt_assert(msg == NULL);
+  tt_ptr_op(msg, OP_EQ, NULL);
 
   tt_assert(protover_all_supported("", &msg));
-  tt_assert(msg == NULL);
+  tt_ptr_op(msg, OP_EQ, NULL);
 
   // Some things that we do support
   tt_assert(protover_all_supported("Link=3-4", &msg));
-  tt_assert(msg == NULL);
+  tt_ptr_op(msg, OP_EQ, NULL);
   tt_assert(protover_all_supported("Link=3-4 Desc=2", &msg));
-  tt_assert(msg == NULL);
+  tt_ptr_op(msg, OP_EQ, NULL);
 
   // Some things we don't support
+  tt_assert(! protover_all_supported("Wombat=9", NULL));
   tt_assert(! protover_all_supported("Wombat=9", &msg));
   tt_str_op(msg, OP_EQ, "Wombat=9");
   tor_free(msg);
@@ -238,35 +280,60 @@ test_protover_all_supported(void *arg)
   tt_assert(! protover_all_supported("Link=3-4 Wombat=9", &msg));
   tt_str_op(msg, OP_EQ, "Wombat=9");
   tor_free(msg);
+
+  /* Mix of things we support and don't support within a single protocol
+   * which we do support */
   tt_assert(! protover_all_supported("Link=3-999", &msg));
-  tt_str_op(msg, OP_EQ, "Link=3-999");
+  tt_str_op(msg, OP_EQ, "Link=6-999");
+  tor_free(msg);
+  tt_assert(! protover_all_supported("Link=1-3,345-666", &msg));
+  tt_str_op(msg, OP_EQ, "Link=345-666");
+  tor_free(msg);
+  tt_assert(! protover_all_supported("Link=1-3,5-12", &msg));
+  tt_str_op(msg, OP_EQ, "Link=6-12");
+  tor_free(msg);
+
+  /* Mix of protocols we do support and some we don't, where the protocols
+   * we do support have some versions we don't support. */
+  tt_assert(! protover_all_supported("Link=1-3,5-12 Quokka=9000-9001", &msg));
+  tt_str_op(msg, OP_EQ, "Link=6-12 Quokka=9000-9001");
   tor_free(msg);
 
-  /* CPU/RAM DoS loop: Rust only */
-  tt_assert(! protover_all_supported("Sleen=0-2147483648", &msg));
-  tt_str_op(msg, OP_EQ, "Sleen=0-2147483648");
+  /* We shouldn't be able to DoS ourselves parsing a large range. */
+  tt_assert(! protover_all_supported("Sleen=1-2147483648", &msg));
+  tt_str_op(msg, OP_EQ, "Sleen=1-2147483648");
   tor_free(msg);
 
   /* This case is allowed. */
-  tt_assert(! protover_all_supported("Sleen=0-4294967294", &msg));
-  tt_str_op(msg, OP_EQ, "Sleen=0-4294967294");
+  tt_assert(! protover_all_supported("Sleen=1-4294967294", &msg));
+  tt_str_op(msg, OP_EQ, "Sleen=1-4294967294");
   tor_free(msg);
 
-  /* If we get an unparseable list, we say "yes, that's supported." */
-#ifndef HAVE_RUST
-  // XXXX let's make this section unconditional: rust should behave the
-  // XXXX same as C here!
+  /* If we get a (barely) valid (but unsupported list, we say "yes, that's
+   * supported." */
+  tt_assert(protover_all_supported("Fribble=", &msg));
+  tt_ptr_op(msg, OP_EQ, NULL);
+
+  /* If we get a completely unparseable list, protover_all_supported should
+   * hit a fatal assertion for BUG(entries == NULL). */
   tor_capture_bugs_(1);
   tt_assert(protover_all_supported("Fribble", &msg));
-  tt_ptr_op(msg, OP_EQ, NULL);
   tor_end_capture_bugs_();
 
-  /* This case is forbidden. Since it came from a protover_all_supported,
-   * it can trigger a bug message.  */
+  /* If we get a completely unparseable list, protover_all_supported should
+   * hit a fatal assertion for BUG(entries == NULL). */
   tor_capture_bugs_(1);
-  tt_assert(protover_all_supported("Sleen=0-4294967295", &msg));
-  tt_ptr_op(msg, OP_EQ, NULL);
-  tor_free(msg);
+  tt_assert(protover_all_supported("Sleen=1-4294967295", &msg));
+  tor_end_capture_bugs_();
+
+  /* Protocol name too long */
+#ifndef HAVE_RUST // XXXXXX ?????
+  tor_capture_bugs_(1);
+  tt_assert(protover_all_supported(
+                 "DoSaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+                 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+                 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+                 "aaaaaaaaaaaa=1-65536", &msg));
   tor_end_capture_bugs_();
 #endif
 
@@ -276,6 +343,211 @@ test_protover_all_supported(void *arg)
 }
 
 static void
+test_protover_list_supports_protocol_returns_true(void *arg)
+{
+  (void)arg;
+
+  const char *protocols = "Link=1";
+  int is_supported = protocol_list_supports_protocol(protocols, PRT_LINK, 1);
+  tt_int_op(is_supported, OP_EQ, 1);
+
+ done:
+  ;
+}
+
+static void
+test_protover_list_supports_protocol_for_unsupported_returns_false(void *arg)
+{
+  (void)arg;
+
+  const char *protocols = "Link=1";
+  int is_supported = protocol_list_supports_protocol(protocols, PRT_LINK, 10);
+  tt_int_op(is_supported, OP_EQ, 0);
+
+ done:
+  ;
+}
+
+static void
+test_protover_supports_version(void *arg)
+{
+  (void)arg;
+
+  tt_assert(protocol_list_supports_protocol("Link=3-6", PRT_LINK, 3));
+  tt_assert(protocol_list_supports_protocol("Link=3-6", PRT_LINK, 6));
+  tt_assert(!protocol_list_supports_protocol("Link=3-6", PRT_LINK, 7));
+  tt_assert(!protocol_list_supports_protocol("Link=3-6", PRT_LINKAUTH, 3));
+
+  tt_assert(!protocol_list_supports_protocol("Link=4-6 LinkAuth=3",
+                                            PRT_LINKAUTH, 2));
+  tt_assert(protocol_list_supports_protocol("Link=4-6 LinkAuth=3",
+                                            PRT_LINKAUTH, 3));
+  tt_assert(!protocol_list_supports_protocol("Link=4-6 LinkAuth=3",
+                                             PRT_LINKAUTH, 4));
+  tt_assert(!protocol_list_supports_protocol_or_later("Link=4-6 LinkAuth=3",
+                                             PRT_LINKAUTH, 4));
+  tt_assert(protocol_list_supports_protocol_or_later("Link=4-6 LinkAuth=3",
+                                             PRT_LINKAUTH, 3));
+  tt_assert(protocol_list_supports_protocol_or_later("Link=4-6 LinkAuth=3",
+                                             PRT_LINKAUTH, 2));
+
+  tt_assert(!protocol_list_supports_protocol_or_later("Link=4-6 LinkAuth=3",
+                                                      PRT_DESC, 2));
+ done:
+ ;
+}
+
+/* This could be MAX_PROTOCOLS_TO_EXPAND, but that's not exposed by protover */
+#define MAX_PROTOCOLS_TO_TEST 1024
+
+/* LinkAuth and Relay protocol versions.
+ * Hard-coded here, because they are not in the code, or not exposed in the
+ * headers. */
+#define PROTOVER_LINKAUTH_V1 1
+#define PROTOVER_LINKAUTH_V3 3
+
+#define PROTOVER_RELAY_V1 1
+#define PROTOVER_RELAY_V2 2
+
+/* Highest supported HSv2 introduce protocol version.
+ * Hard-coded here, because it does not appear anywhere in the code.
+ * It's not clear if we actually support version 2, see #25068. */
+#define PROTOVER_HSINTRO_V2 3
+
+/* HSv2 Rend and HSDir protocol versions.
+ * Hard-coded here, because they do not appear anywhere in the code. */
+#define PROTOVER_HS_RENDEZVOUS_POINT_V2 1
+#define PROTOVER_HSDIR_V2 1
+
+/* DirCache, Desc, Microdesc, and Cons protocol versions.
+ * Hard-coded here, because they do not appear anywhere in the code. */
+#define PROTOVER_DIRCACHE_V1 1
+#define PROTOVER_DIRCACHE_V2 2
+
+#define PROTOVER_DESC_V1 1
+#define PROTOVER_DESC_V2 2
+
+#define PROTOVER_MICRODESC_V1 1
+#define PROTOVER_MICRODESC_V2 2
+
+#define PROTOVER_CONS_V1 1
+#define PROTOVER_CONS_V2 2
+
+/* Make sure we haven't forgotten any supported protocols */
+static void
+test_protover_supported_protocols(void *arg)
+{
+  (void)arg;
+
+  const char *supported_protocols = protover_get_supported_protocols();
+
+  /* Test for new Link in the code, that hasn't been added to supported
+   * protocols */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_LINK,
+                                            MAX_LINK_PROTO));
+  for (uint16_t i = 0; i < MAX_PROTOCOLS_TO_TEST; i++) {
+    if (is_or_protocol_version_known(i)) {
+      tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                                PRT_LINK,
+                                                i));
+    }
+  }
+
+#ifdef HAVE_WORKING_TOR_TLS_GET_TLSSECRETS
+  /* Legacy LinkAuth does not appear anywhere in the code. */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_LINKAUTH,
+                                            PROTOVER_LINKAUTH_V1));
+#endif
+  /* Latest LinkAuth is not exposed in the headers. */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_LINKAUTH,
+                                            PROTOVER_LINKAUTH_V3));
+  /* Is there any way to test for new LinkAuth? */
+
+  /* Relay protovers do not appear anywhere in the code. */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_RELAY,
+                                            PROTOVER_RELAY_V1));
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_RELAY,
+                                            PROTOVER_RELAY_V2));
+  /* Is there any way to test for new Relay? */
+
+  /* We could test legacy HSIntro by calling rend_service_update_descriptor(),
+   * and checking the protocols field. But that's unlikely to change, so
+   * we just use a hard-coded value. */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_HSINTRO,
+                                            PROTOVER_HSINTRO_V2));
+  /* Test for HSv3 HSIntro */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_HSINTRO,
+                                            PROTOVER_HS_INTRO_V3));
+  /* Is there any way to test for new HSIntro? */
+
+  /* Legacy HSRend does not appear anywhere in the code. */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_HSREND,
+                                            PROTOVER_HS_RENDEZVOUS_POINT_V2));
+  /* Test for HSv3 HSRend */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_HSREND,
+                                            PROTOVER_HS_RENDEZVOUS_POINT_V3));
+  /* Is there any way to test for new HSRend? */
+
+  /* Legacy HSDir does not appear anywhere in the code. */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_HSDIR,
+                                            PROTOVER_HSDIR_V2));
+  /* Test for HSv3 HSDir */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_HSDIR,
+                                            PROTOVER_HSDIR_V3));
+  /* Is there any way to test for new HSDir? */
+
+  /* No DirCache versions appear anywhere in the code. */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_DIRCACHE,
+                                            PROTOVER_DIRCACHE_V1));
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_DIRCACHE,
+                                            PROTOVER_DIRCACHE_V2));
+  /* Is there any way to test for new DirCache? */
+
+  /* No Desc versions appear anywhere in the code. */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_DESC,
+                                            PROTOVER_DESC_V1));
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_DESC,
+                                            PROTOVER_DESC_V2));
+  /* Is there any way to test for new Desc? */
+
+  /* No Microdesc versions appear anywhere in the code. */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_MICRODESC,
+                                            PROTOVER_MICRODESC_V1));
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_MICRODESC,
+                                            PROTOVER_MICRODESC_V2));
+  /* Is there any way to test for new Microdesc? */
+
+  /* No Cons versions appear anywhere in the code. */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_CONS,
+                                            PROTOVER_CONS_V1));
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_CONS,
+                                            PROTOVER_CONS_V2));
+  /* Is there any way to test for new Cons? */
+
+ done:
+ ;
+}
+
+static void
 test_protover_vote_roundtrip(void *args)
 {
   (void) args;
@@ -292,11 +564,11 @@ test_protover_vote_roundtrip(void *args)
     { "Zn=4294967295-1", NULL },
     { "Zn=4294967293-4294967295", NULL },
     /* Will fail because of 4294967295. */
-    { "Foo=1,3 Bar=3 Baz= Quux=9-12,14,15-16,900 Zn=0,4294967295",
+    { "Foo=1,3 Bar=3 Baz= Quux=9-12,14,15-16,900 Zn=1,4294967295",
        NULL },
-    { "Foo=1,3 Bar=3 Baz= Quux=9-12,14,15-16,900 Zn=0,4294967294",
-      "Bar=3 Foo=1,3 Quux=9-12,14-16,900 Zn=0,4294967294" },
-    { "Zu16=0,65536", "Zu16=0,65536" },
+    { "Foo=1,3 Bar=3 Baz= Quux=9-12,14,15-16,900 Zn=1,4294967294",
+      "Bar=3 Foo=1,3 Quux=9-12,14-16,900 Zn=1,4294967294" },
+    { "Zu16=1,65536", "Zu16=1,65536" },
     { "N-1=1,2", "N-1=1-2" },
     { "-1=4294967295", NULL },
     { "-1=3", "-1=3" },
@@ -316,20 +588,26 @@ test_protover_vote_roundtrip(void *args)
     { "Link=1,9-8,3", NULL },
     { "Faux=-0", NULL },
     { "Faux=0--0", NULL },
-    // "These fail at the splitting stage in Rust, but the number parsing
-    // stage in C."
     { "Faux=-1", NULL },
     { "Faux=-1-3", NULL },
     { "Faux=1--1", NULL },
+    { "Link=1-2-", NULL },
+    { "Link=1-2-3", NULL },
+    { "Faux=1-2-", NULL },
+    { "Faux=1-2-3", NULL },
+    { "Link=\t1,3", NULL },
+    { "Link=1\n,3", NULL },
+    { "Faux=1,\r3", NULL },
+    { "Faux=1,3\f", NULL },
     /* Large integers */
     { "Link=4294967296", NULL },
     /* Large range */
     { "Sleen=1-501", "Sleen=1-501" },
     { "Sleen=1-65537", NULL },
-    /* CPU/RAM DoS Loop: Rust only. */
-    { "Sleen=0-2147483648", NULL },
-    /* Rust seems to experience an internal error here. */
-    { "Sleen=0-4294967295", NULL },
+    /* Both C/Rust implementations should be able to handle this mild DoS. */
+    { "Sleen=1-2147483648", NULL },
+    /* Rust tests are built in debug mode, so ints are bounds-checked. */
+    { "Sleen=1-4294967295", NULL },
   };
   unsigned u;
   smartlist_t *votes = smartlist_new();
@@ -364,7 +642,10 @@ struct testcase_t protover_tests[] = {
   PV_TEST(parse_fail, 0),
   PV_TEST(vote, 0),
   PV_TEST(all_supported, 0),
+  PV_TEST(list_supports_protocol_for_unsupported_returns_false, 0),
+  PV_TEST(list_supports_protocol_returns_true, 0),
+  PV_TEST(supports_version, 0),
+  PV_TEST(supported_protocols, 0),
   PV_TEST(vote_roundtrip, 0),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_pt.c b/src/test/test_pt.c
index e5cdc5f3cd..1f9786648a 100644
--- a/src/test/test_pt.c
+++ b/src/test/test_pt.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -8,15 +8,20 @@
 #define UTIL_PRIVATE
 #define STATEFILE_PRIVATE
 #define CONTROL_PRIVATE
-#include "or.h"
-#include "config.h"
-#include "confparse.h"
-#include "control.h"
-#include "transports.h"
-#include "circuitbuild.h"
-#include "util.h"
-#include "statefile.h"
-#include "test.h"
+#define SUBPROCESS_PRIVATE
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "app/config/confparse.h"
+#include "feature/control/control.h"
+#include "feature/client/transports.h"
+#include "core/or/circuitbuild.h"
+#include "app/config/statefile.h"
+#include "test/test.h"
+#include "lib/process/subprocess.h"
+#include "lib/encoding/confline.h"
+#include "lib/net/resolve.h"
+
+#include "app/config/or_state_st.h"
 
 static void
 reset_mp(managed_proxy_t *mp)
@@ -40,34 +45,34 @@ test_pt_parsing(void *arg)
 
   /* incomplete cmethod */
   strlcpy(line,"CMETHOD trebuchet",sizeof(line));
-  tt_assert(parse_cmethod_line(line, mp) < 0);
+  tt_int_op(parse_cmethod_line(line, mp), OP_LT, 0);
 
   reset_mp(mp);
 
   /* wrong proxy type */
   strlcpy(line,"CMETHOD trebuchet dog 127.0.0.1:1999",sizeof(line));
-  tt_assert(parse_cmethod_line(line, mp) < 0);
+  tt_int_op(parse_cmethod_line(line, mp), OP_LT, 0);
 
   reset_mp(mp);
 
   /* wrong addrport */
   strlcpy(line,"CMETHOD trebuchet socks4 abcd",sizeof(line));
-  tt_assert(parse_cmethod_line(line, mp) < 0);
+  tt_int_op(parse_cmethod_line(line, mp), OP_LT, 0);
 
   reset_mp(mp);
 
   /* correct line */
   strlcpy(line,"CMETHOD trebuchet socks5 127.0.0.1:1999",sizeof(line));
-  tt_assert(parse_cmethod_line(line, mp) == 0);
-  tt_assert(smartlist_len(mp->transports) == 1);
+  tt_int_op(parse_cmethod_line(line, mp), OP_EQ, 0);
+  tt_int_op(smartlist_len(mp->transports), OP_EQ, 1);
   transport = smartlist_get(mp->transports, 0);
   /* test registered address of transport */
   tor_addr_parse(&test_addr, "127.0.0.1");
   tt_assert(tor_addr_eq(&test_addr, &transport->addr));
   /* test registered port of transport */
-  tt_assert(transport->port == 1999);
+  tt_uint_op(transport->port, OP_EQ, 1999);
   /* test registered SOCKS version of transport */
-  tt_assert(transport->socks_version == PROXY_SOCKS5);
+  tt_int_op(transport->socks_version, OP_EQ, PROXY_SOCKS5);
   /* test registered name of transport */
   tt_str_op(transport->name,OP_EQ, "trebuchet");
 
@@ -75,26 +80,26 @@ test_pt_parsing(void *arg)
 
   /* incomplete smethod */
   strlcpy(line,"SMETHOD trebuchet",sizeof(line));
-  tt_assert(parse_smethod_line(line, mp) < 0);
+  tt_int_op(parse_smethod_line(line, mp), OP_LT, 0);
 
   reset_mp(mp);
 
   /* wrong addr type */
   strlcpy(line,"SMETHOD trebuchet abcd",sizeof(line));
-  tt_assert(parse_smethod_line(line, mp) < 0);
+  tt_int_op(parse_smethod_line(line, mp), OP_LT, 0);
 
   reset_mp(mp);
 
   /* cowwect */
   strlcpy(line,"SMETHOD trebuchy 127.0.0.2:2999",sizeof(line));
-  tt_assert(parse_smethod_line(line, mp) == 0);
-  tt_assert(smartlist_len(mp->transports) == 1);
+  tt_int_op(parse_smethod_line(line, mp), OP_EQ, 0);
+  tt_int_op(smartlist_len(mp->transports), OP_EQ, 1);
   transport = smartlist_get(mp->transports, 0);
   /* test registered address of transport */
   tor_addr_parse(&test_addr, "127.0.0.2");
   tt_assert(tor_addr_eq(&test_addr, &transport->addr));
   /* test registered port of transport */
-  tt_assert(transport->port == 2999);
+  tt_uint_op(transport->port, OP_EQ, 2999);
   /* test registered name of transport */
   tt_str_op(transport->name,OP_EQ, "trebuchy");
 
@@ -104,7 +109,7 @@ test_pt_parsing(void *arg)
   strlcpy(line,"SMETHOD trebuchet 127.0.0.1:9999 "
           "ARGS:counterweight=3,sling=snappy",
           sizeof(line));
-  tt_assert(parse_smethod_line(line, mp) == 0);
+  tt_int_op(parse_smethod_line(line, mp), OP_EQ, 0);
   tt_int_op(1, OP_EQ, smartlist_len(mp->transports));
   {
     const transport_t *transport_ = smartlist_get(mp->transports, 0);
@@ -119,15 +124,15 @@ test_pt_parsing(void *arg)
 
   /* unsupported version */
   strlcpy(line,"VERSION 666",sizeof(line));
-  tt_assert(parse_version(line, mp) < 0);
+  tt_int_op(parse_version(line, mp), OP_LT, 0);
 
   /* incomplete VERSION */
   strlcpy(line,"VERSION ",sizeof(line));
-  tt_assert(parse_version(line, mp) < 0);
+  tt_int_op(parse_version(line, mp), OP_LT, 0);
 
   /* correct VERSION */
   strlcpy(line,"VERSION 1",sizeof(line));
-  tt_assert(parse_version(line, mp) == 0);
+  tt_int_op(parse_version(line, mp), OP_EQ, 0);
 
  done:
   reset_mp(mp);
@@ -155,9 +160,9 @@ test_pt_get_transport_options(void *arg)
   opt_str = get_transport_options_for_server_proxy(mp);
   tt_ptr_op(opt_str, OP_EQ, NULL);
 
-  smartlist_add(mp->transports_to_launch, tor_strdup("gruyere"));
-  smartlist_add(mp->transports_to_launch, tor_strdup("roquefort"));
-  smartlist_add(mp->transports_to_launch, tor_strdup("stnectaire"));
+  smartlist_add_strdup(mp->transports_to_launch, "gruyere");
+  smartlist_add_strdup(mp->transports_to_launch, "roquefort");
+  smartlist_add_strdup(mp->transports_to_launch, "stnectaire");
 
   tt_assert(options);
 
@@ -284,13 +289,13 @@ test_pt_get_extrainfo_string(void *arg)
 }
 
 #ifdef _WIN32
-#define STDIN_HANDLE HANDLE
+#define STDIN_HANDLE HANDLE*
 #else
-#define STDIN_HANDLE FILE
+#define STDIN_HANDLE int
 #endif
 
 static smartlist_t *
-tor_get_lines_from_handle_replacement(STDIN_HANDLE *handle,
+tor_get_lines_from_handle_replacement(STDIN_HANDLE handle,
                                       enum stream_status *stream_status_out)
 {
   static int times_called = 0;
@@ -305,7 +310,7 @@ tor_get_lines_from_handle_replacement(STDIN_HANDLE *handle,
     smartlist_add_asprintf(retval_sl, "SMETHOD mock%d 127.0.0.1:555%d",
                            times_called, times_called);
   } else {
-    smartlist_add(retval_sl, tor_strdup("SMETHODS DONE"));
+    smartlist_add_strdup(retval_sl, "SMETHODS DONE");
   }
 
   return retval_sl;
@@ -461,7 +466,7 @@ test_get_pt_proxy_uri(void *arg)
 
   /* Test with no proxy. */
   uri = get_pt_proxy_uri();
-  tt_assert(uri == NULL);
+  tt_ptr_op(uri, OP_EQ, NULL);
 
   /* Test with a SOCKS4 proxy. */
   options->Socks4Proxy = tor_strdup("192.0.2.1:1080");
@@ -544,4 +549,3 @@ struct testcase_t pt_tests[] = {
     NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_pubsub.c b/src/test/test_pubsub.c
deleted file mode 100644
index 547d6c6b32..0000000000
--- a/src/test/test_pubsub.c
+++ /dev/null
@@ -1,85 +0,0 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file test_pubsub.c
- * \brief Unit tests for publish-subscribe abstraction.
- **/
-
-#include "or.h"
-#include "test.h"
-#include "pubsub.h"
-
-DECLARE_PUBSUB_STRUCT_TYPES(foobar)
-DECLARE_PUBSUB_TOPIC(foobar)
-DECLARE_NOTIFY_PUBSUB_TOPIC(static, foobar)
-IMPLEMENT_PUBSUB_TOPIC(static, foobar)
-
-struct foobar_event_data_t {
-  unsigned u;
-  const char *s;
-};
-
-struct foobar_subscriber_data_t {
-  const char *name;
-  long l;
-};
-
-static int
-foobar_sub1(foobar_event_data_t *ev, foobar_subscriber_data_t *mine)
-{
-  ev->u += 10;
-  mine->l += 100;
-  return 0;
-}
-
-static int
-foobar_sub2(foobar_event_data_t *ev, foobar_subscriber_data_t *mine)
-{
-  ev->u += 5;
-  mine->l += 50;
-  return 0;
-}
-
-static void
-test_pubsub_basic(void *arg)
-{
-  (void)arg;
-  foobar_subscriber_data_t subdata1 = { "hi", 0 };
-  foobar_subscriber_data_t subdata2 = { "wow", 0 };
-  const foobar_subscriber_t *sub1;
-  const foobar_subscriber_t *sub2;
-  foobar_event_data_t ed = { 0, "x" };
-  foobar_event_data_t ed2 = { 0, "y" };
-  sub1 = foobar_subscribe(foobar_sub1, &subdata1, SUBSCRIBE_ATSTART, 100);
-  tt_assert(sub1);
-
-  foobar_notify(&ed, 0);
-  tt_int_op(subdata1.l, OP_EQ, 100);
-  tt_int_op(subdata2.l, OP_EQ, 0);
-  tt_int_op(ed.u, OP_EQ, 10);
-
-  sub2 = foobar_subscribe(foobar_sub2, &subdata2, 0, 5);
-  tt_assert(sub2);
-
-  foobar_notify(&ed2, 0);
-  tt_int_op(subdata1.l, OP_EQ, 200);
-  tt_int_op(subdata2.l, OP_EQ, 50);
-  tt_int_op(ed2.u, OP_EQ, 15);
-
-  foobar_unsubscribe(sub1);
-
-  foobar_notify(&ed, 0);
-  tt_int_op(subdata1.l, OP_EQ, 200);
-  tt_int_op(subdata2.l, OP_EQ, 100);
-  tt_int_op(ed.u, OP_EQ, 15);
-
- done:
-  foobar_clear();
-}
-
-struct testcase_t pubsub_tests[] = {
-  { "pubsub_basic", test_pubsub_basic, TT_FORK, NULL, NULL },
-  END_OF_TESTCASES
-};
-
diff --git a/src/test/test_rebind.py b/src/test/test_rebind.py
new file mode 100644
index 0000000000..30a587858f
--- /dev/null
+++ b/src/test/test_rebind.py
@@ -0,0 +1,147 @@
+from __future__ import print_function
+
+import errno
+import logging
+import os
+import random
+import socket
+import subprocess
+import sys
+import time
+
+LOG_TIMEOUT = 60.0
+LOG_WAIT = 0.1
+
+def fail(msg):
+    logging.error('FAIL')
+    sys.exit(msg)
+
+def skip(msg):
+    logging.warning('SKIP: {}'.format(msg))
+    sys.exit(77)
+
+def try_connecting_to_socksport():
+    socks_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    if socks_socket.connect_ex(('127.0.0.1', socks_port)):
+        tor_process.terminate()
+        fail('Cannot connect to SOCKSPort')
+    socks_socket.close()
+
+def wait_for_log(s):
+    cutoff = time.time() + LOG_TIMEOUT
+    while time.time() < cutoff:
+        l = tor_process.stdout.readline()
+        l = l.decode('utf8', 'backslashreplace')
+        if s in l:
+            logging.info('Tor logged: "{}"'.format(l.strip()))
+            return
+        # readline() returns a blank string when there is no output
+        # avoid busy-waiting
+        if len(l) == 0:
+            logging.debug('Tor has not logged anything, waiting for "{}"'.format(s))
+            time.sleep(LOG_WAIT)
+        else:
+            logging.info('Tor logged: "{}", waiting for "{}"'.format(l.strip(), s))
+    fail('Could not find "{}" in logs after {} seconds'.format(s, LOG_TIMEOUT))
+
+def pick_random_port():
+    port = 0
+    random.seed()
+
+    for i in range(8):
+        port = random.randint(10000, 60000)
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        if s.connect_ex(('127.0.0.1', port)) == 0:
+            s.close()
+        else:
+            break
+
+    if port == 0:
+        fail('Could not find a random free port between 10000 and 60000')
+
+    return port
+
+logging.basicConfig(level=logging.DEBUG,
+                    format='%(asctime)s.%(msecs)03d %(message)s',
+                    datefmt='%Y-%m-%d %H:%M:%S')
+
+if sys.hexversion < 0x02070000:
+    fail("ERROR: unsupported Python version (should be >= 2.7)")
+
+if sys.hexversion > 0x03000000 and sys.hexversion < 0x03010000:
+    fail("ERROR: unsupported Python3 version (should be >= 3.1)")
+
+if 'TOR_SKIP_TEST_REBIND' in os.environ:
+    skip('$TOR_SKIP_TEST_REBIND is set')
+
+control_port = pick_random_port()
+socks_port = pick_random_port()
+
+assert control_port != 0
+assert socks_port != 0
+
+if len(sys.argv) < 3:
+     fail('Usage: %s <path-to-tor> <data-dir>' % sys.argv[0])
+
+if not os.path.exists(sys.argv[1]):
+    fail('ERROR: cannot find tor at %s' % sys.argv[1])
+if not os.path.exists(sys.argv[2]):
+    fail('ERROR: cannot find datadir at %s' % sys.argv[2])
+
+tor_path = sys.argv[1]
+data_dir = sys.argv[2]
+
+empty_torrc_path = os.path.join(data_dir, 'empty_torrc')
+open(empty_torrc_path, 'w').close()
+empty_defaults_torrc_path = os.path.join(data_dir, 'empty_defaults_torrc')
+open(empty_defaults_torrc_path, 'w').close()
+
+tor_process = subprocess.Popen([tor_path,
+                               '-DataDirectory', data_dir,
+                               '-ControlPort', '127.0.0.1:{}'.format(control_port),
+                               '-SOCKSPort', '127.0.0.1:{}'.format(socks_port),
+                               '-Log', 'debug stdout',
+                               '-LogTimeGranularity', '1',
+                               '-FetchServerDescriptors', '0',
+                               '-f', empty_torrc_path,
+                               '--defaults-torrc', empty_defaults_torrc_path,
+                               ],
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+
+if tor_process == None:
+    fail('ERROR: running tor failed')
+
+wait_for_log('Opened Control listener on')
+
+try_connecting_to_socksport()
+
+control_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+if control_socket.connect_ex(('127.0.0.1', control_port)):
+    tor_process.terminate()
+    fail('Cannot connect to ControlPort')
+
+control_socket.sendall('AUTHENTICATE \r\n'.encode('ascii'))
+control_socket.sendall('SETCONF SOCKSPort=0.0.0.0:{}\r\n'.format(socks_port).encode('ascii'))
+wait_for_log('Opened Socks listener')
+
+try_connecting_to_socksport()
+
+control_socket.sendall('SETCONF SOCKSPort=127.0.0.1:{}\r\n'.format(socks_port).encode('ascii'))
+wait_for_log('Opened Socks listener')
+
+try_connecting_to_socksport()
+
+control_socket.sendall('SIGNAL HALT\r\n'.encode('ascii'))
+
+wait_for_log('exiting cleanly')
+logging.info('OK')
+
+try:
+    tor_process.terminate()
+except OSError as e:
+    if e.errno == errno.ESRCH: # errno 3: No such process
+        # assume tor has already exited due to SIGNAL HALT
+        logging.warn("Tor has already exited")
+    else:
+        raise
diff --git a/src/test/test_rebind.sh b/src/test/test_rebind.sh
new file mode 100755
index 0000000000..ea2012957e
--- /dev/null
+++ b/src/test/test_rebind.sh
@@ -0,0 +1,32 @@
+#!/bin/sh
+
+set -x
+
+UNAME_OS=$(uname -s | cut -d_ -f1)
+if test "$UNAME_OS" = 'CYGWIN' || \
+   test "$UNAME_OS" = 'MSYS' || \
+   test "$UNAME_OS" = 'MINGW'; then
+  if test "$APPVEYOR" = 'True'; then
+    echo "This test is disabled on Windows CI, as it requires firewall examptions. Skipping." >&2
+    exit 77
+  fi
+fi
+
+exitcode=0
+
+tmpdir=
+clean () { test -n "$tmpdir" && test -d "$tmpdir" && rm -rf "$tmpdir" || :; }
+trap clean EXIT HUP INT TERM
+
+tmpdir="`mktemp -d -t tor_rebind_test.XXXXXX`"
+if [ -z "$tmpdir" ]; then
+  echo >&2 mktemp failed
+  exit 2
+elif [ ! -d "$tmpdir" ]; then
+  echo >&2 mktemp failed to make a directory
+  exit 3
+fi
+
+"${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/test_rebind.py" "${TESTING_TOR_BINARY}" "$tmpdir"
+
+exit $?
diff --git a/src/test/test_relay.c b/src/test/test_relay.c
index d4ebb23382..0b7a7be332 100644
--- a/src/test/test_relay.c
+++ b/src/test/test_relay.c
@@ -1,30 +1,31 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#include "or.h"
 #define CIRCUITBUILD_PRIVATE
-#include "circuitbuild.h"
-#include "circuitlist.h"
-#include "rephist.h"
-#include "channeltls.h"
 #define RELAY_PRIVATE
-#include "relay.h"
+#define REPHIST_PRIVATE
+#include "core/or/or.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/channeltls.h"
+#include "feature/stats/rephist.h"
+#include "core/or/relay.h"
+#include "feature/stats/rephist.h"
+#include "lib/container/order.h"
 /* For init/free stuff */
-#include "scheduler.h"
+#include "core/or/scheduler.h"
+
+#include "core/or/cell_st.h"
+#include "core/or/or_circuit_st.h"
 
 /* Test suite stuff */
-#include "test.h"
-#include "fakechans.h"
+#include "test/test.h"
+#include "test/fakechans.h"
 
 static or_circuit_t * new_fake_orcirc(channel_t *nchan, channel_t *pchan);
 
 static void test_relay_append_cell_to_circuit_queue(void *arg);
 
-typedef struct bw_array_t bw_array_t;
-uint64_t find_largest_max(bw_array_t *b);
-void commit_max(bw_array_t *b);
-void advance_obs(bw_array_t *b);
-
 static or_circuit_t *
 new_fake_orcirc(channel_t *nchan, channel_t *pchan)
 {
@@ -50,12 +51,6 @@ new_fake_orcirc(channel_t *nchan, channel_t *pchan)
   circ->deliver_window = CIRCWINDOW_START_MAX;
   circ->n_chan_create_cell = NULL;
 
-  /* for assert_circ_ok */
-  orcirc->p_crypto = (void*)1;
-  orcirc->n_crypto = (void*)1;
-  orcirc->n_digest = (void*)1;
-  orcirc->p_digest = (void*)1;
-
   circuit_set_p_circid_chan(orcirc, get_unique_circ_id_by_chan(pchan), pchan);
   cell_queue_init(&(orcirc->p_chan_cells));
 
@@ -63,6 +58,13 @@ new_fake_orcirc(channel_t *nchan, channel_t *pchan)
 }
 
 static void
+assert_circuit_ok_mock(const circuit_t *c)
+{
+  (void) c;
+  return;
+}
+
+static void
 test_relay_close_circuit(void *arg)
 {
   channel_t *nchan = NULL, *pchan = NULL;
@@ -79,10 +81,6 @@ test_relay_close_circuit(void *arg)
   pchan = new_fake_channel();
   tt_assert(pchan);
 
-  /* We'll need chans with working cmuxes */
-  nchan->cmux = circuitmux_alloc();
-  pchan->cmux = circuitmux_alloc();
-
   /* Make a fake orcirc */
   orcirc = new_fake_orcirc(nchan, pchan);
   tt_assert(orcirc);
@@ -97,6 +95,8 @@ test_relay_close_circuit(void *arg)
 
   MOCK(scheduler_channel_has_waiting_cells,
        scheduler_channel_has_waiting_cells_mock);
+  MOCK(assert_circuit_ok,
+       assert_circuit_ok_mock);
 
   /* Append it */
   old_count = get_mock_scheduler_has_waiting_cells_count();
@@ -148,6 +148,7 @@ test_relay_close_circuit(void *arg)
   tor_free(orcirc);
   free_fake_channel(nchan);
   free_fake_channel(pchan);
+  UNMOCK(assert_circuit_ok);
 
   return;
 }
@@ -169,10 +170,6 @@ test_relay_append_cell_to_circuit_queue(void *arg)
   pchan = new_fake_channel();
   tt_assert(pchan);
 
-  /* We'll need chans with working cmuxes */
-  nchan->cmux = circuitmux_alloc();
-  pchan->cmux = circuitmux_alloc();
-
   /* Make a fake orcirc */
   orcirc = new_fake_orcirc(nchan, pchan);
   tt_assert(orcirc);
@@ -193,14 +190,14 @@ test_relay_append_cell_to_circuit_queue(void *arg)
   append_cell_to_circuit_queue(TO_CIRCUIT(orcirc), nchan, cell,
                                CELL_DIRECTION_OUT, 0);
   new_count = get_mock_scheduler_has_waiting_cells_count();
-  tt_int_op(new_count, ==, old_count + 1);
+  tt_int_op(new_count, OP_EQ, old_count + 1);
 
   /* Now try the reverse direction */
   old_count = get_mock_scheduler_has_waiting_cells_count();
   append_cell_to_circuit_queue(TO_CIRCUIT(orcirc), pchan, cell,
                                CELL_DIRECTION_IN, 0);
   new_count = get_mock_scheduler_has_waiting_cells_count();
-  tt_int_op(new_count, ==, old_count + 1);
+  tt_int_op(new_count, OP_EQ, old_count + 1);
 
   UNMOCK(scheduler_channel_has_waiting_cells);
 
diff --git a/src/test/test_relaycell.c b/src/test/test_relaycell.c
index fb6748965a..0623583511 100644
--- a/src/test/test_relaycell.c
+++ b/src/test/test_relaycell.c
@@ -1,15 +1,35 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /* Unit tests for handling different kinds of relay cell */
 
 #define RELAY_PRIVATE
-#include "or.h"
-#include "config.h"
-#include "connection.h"
-#include "connection_edge.h"
-#include "relay.h"
-#include "test.h"
+#define CIRCUITLIST_PRIVATE
+#define CONNECTION_EDGE_PRIVATE
+#define CONNECTION_PRIVATE
+
+#include "core/or/or.h"
+#include "core/mainloop/mainloop.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/connection_edge.h"
+#include "core/or/relay.h"
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+
+#include "core/or/cell_st.h"
+#include "core/or/crypt_path_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
+#include "core/or/half_edge_st.h"
+
+#include "feature/client/circpathbias.h"
+#include "core/or/connection_edge.h"
 
 static int srm_ncalls;
 static entry_connection_t *srm_conn;
@@ -60,6 +80,805 @@ mark_unattached_mock(entry_connection_t *conn, int endreason,
   (void) file;
 }
 
+/* Helper: Return a newly allocated and initialized origin circuit with
+ * purpose and flags. A default HS identifier is set to an ed25519
+ * authentication key for introduction point. */
+static origin_circuit_t *
+helper_create_origin_circuit(int purpose, int flags)
+{
+  origin_circuit_t *circ = NULL;
+
+  circ = origin_circuit_init(purpose, flags);
+  tor_assert(circ);
+  circ->cpath = tor_malloc_zero(sizeof(crypt_path_t));
+  circ->cpath->magic = CRYPT_PATH_MAGIC;
+  circ->cpath->state = CPATH_STATE_OPEN;
+  circ->cpath->package_window = circuit_initial_package_window();
+  circ->cpath->deliver_window = CIRCWINDOW_START;
+  circ->cpath->prev = circ->cpath;
+  /* Create a default HS identifier. */
+  circ->hs_ident = tor_malloc_zero(sizeof(hs_ident_circuit_t));
+
+  return circ;
+}
+
+static void
+mock_connection_mark_unattached_ap_(entry_connection_t *conn, int endreason,
+                                    int line, const char *file)
+{
+  (void) line;
+  (void) file;
+  conn->edge_.end_reason = endreason;
+}
+
+static void
+mock_mark_circ_for_close(circuit_t *circ, int reason, int line,
+                          const char *file)
+{
+  (void)reason; (void)line; (void)file;
+
+  circ->marked_for_close = 1;
+  return;
+}
+
+static void
+mock_mark_for_close(connection_t *conn,
+                        int line, const char *file)
+{
+  (void)line;
+  (void)file;
+
+  conn->marked_for_close = 1;
+  return;
+}
+
+static void
+mock_start_reading(connection_t *conn)
+{
+  (void)conn;
+  return;
+}
+
+static int
+mock_send_command(streamid_t stream_id, circuit_t *circ,
+                               uint8_t relay_command, const char *payload,
+                               size_t payload_len, crypt_path_t *cpath_layer,
+                               const char *filename, int lineno)
+{
+ (void)stream_id; (void)circ;
+ (void)relay_command; (void)payload;
+ (void)payload_len; (void)cpath_layer;
+ (void)filename; (void)lineno;
+
+ return 0;
+}
+
+static entry_connection_t *
+fake_entry_conn(origin_circuit_t *oncirc, streamid_t id)
+{
+  edge_connection_t *edgeconn;
+  entry_connection_t *entryconn;
+
+  entryconn = entry_connection_new(CONN_TYPE_AP, AF_INET);
+  edgeconn = ENTRY_TO_EDGE_CONN(entryconn);
+  edgeconn->base_.state = AP_CONN_STATE_CONNECT_WAIT;
+  edgeconn->deliver_window = STREAMWINDOW_START;
+  edgeconn->package_window = STREAMWINDOW_START;
+
+  edgeconn->stream_id = id;
+  edgeconn->on_circuit = TO_CIRCUIT(oncirc);
+  edgeconn->cpath_layer = oncirc->cpath;
+
+  return entryconn;
+}
+
+#define PACK_CELL(id, cmd, body_s) do {                                  \
+    memset(&cell, 0, sizeof(cell));                                     \
+    memset(&rh, 0, sizeof(rh));                                         \
+    memcpy(cell.payload+RELAY_HEADER_SIZE, (body_s), sizeof((body_s))-1); \
+    rh.length = sizeof((body_s))-1;                                     \
+    rh.command = (cmd);                                                 \
+    rh.stream_id = (id);                                                \
+    relay_header_pack((uint8_t*)&cell.payload, &rh);                    \
+  } while (0)
+#define ASSERT_COUNTED_BW() do { \
+    tt_int_op(circ->n_delivered_read_circ_bw, OP_EQ, delivered+rh.length); \
+    tt_int_op(circ->n_overhead_read_circ_bw, OP_EQ,                      \
+              overhead+RELAY_PAYLOAD_SIZE-rh.length);               \
+    delivered = circ->n_delivered_read_circ_bw;                          \
+    overhead = circ->n_overhead_read_circ_bw;                            \
+ } while (0)
+#define ASSERT_UNCOUNTED_BW() do { \
+    tt_int_op(circ->n_delivered_read_circ_bw, OP_EQ, delivered); \
+    tt_int_op(circ->n_overhead_read_circ_bw, OP_EQ, overhead); \
+ } while (0)
+
+static int
+subtest_circbw_halfclosed(origin_circuit_t *circ, streamid_t init_id)
+{
+  cell_t cell;
+  relay_header_t rh;
+  edge_connection_t *edgeconn;
+  entry_connection_t *entryconn2=NULL;
+  entry_connection_t *entryconn3=NULL;
+  entry_connection_t *entryconn4=NULL;
+  int delivered = circ->n_delivered_read_circ_bw;
+  int overhead = circ->n_overhead_read_circ_bw;
+
+  /* Make new entryconns */
+  entryconn2 = fake_entry_conn(circ, init_id);
+  entryconn2->socks_request->has_finished = 1;
+  entryconn3 = fake_entry_conn(circ, init_id+1);
+  entryconn3->socks_request->has_finished = 1;
+  entryconn4 = fake_entry_conn(circ, init_id+2);
+  entryconn4->socks_request->has_finished = 1;
+  edgeconn = ENTRY_TO_EDGE_CONN(entryconn2);
+  edgeconn->package_window = 23;
+  edgeconn->base_.state = AP_CONN_STATE_OPEN;
+
+  int data_cells = edgeconn->deliver_window;
+  int sendme_cells = (STREAMWINDOW_START-edgeconn->package_window)
+                             /STREAMWINDOW_INCREMENT;
+  ENTRY_TO_CONN(entryconn2)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn2)->outbuf_flushlen = 0;
+  connection_edge_reached_eof(edgeconn);
+
+  /* Data cell not in the half-opened list */
+  PACK_CELL(4000, RELAY_COMMAND_DATA, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                       circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Sendme cell not in the half-opened list */
+  PACK_CELL(4000, RELAY_COMMAND_SENDME, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Connected cell not in the half-opened list */
+  PACK_CELL(4000, RELAY_COMMAND_CONNECTED, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Resolved cell not in the half-opened list */
+  PACK_CELL(4000, RELAY_COMMAND_RESOLVED, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Connected cell: not counted -- we were open */
+  edgeconn = ENTRY_TO_EDGE_CONN(entryconn2);
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_CONNECTED, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* DATA cells up to limit */
+  while (data_cells > 0) {
+    ENTRY_TO_CONN(entryconn2)->marked_for_close = 0;
+    ENTRY_TO_CONN(entryconn2)->outbuf_flushlen = 0;
+    PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_DATA, "Data1234");
+    if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+      pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+    else
+      connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                       circ->cpath);
+    ASSERT_COUNTED_BW();
+    data_cells--;
+  }
+  ENTRY_TO_CONN(entryconn2)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn2)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_DATA, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* SENDME cells up to limit */
+  while (sendme_cells > 0) {
+    ENTRY_TO_CONN(entryconn2)->marked_for_close = 0;
+    ENTRY_TO_CONN(entryconn2)->outbuf_flushlen = 0;
+    PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_SENDME, "Data1234");
+    if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+      pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+    else
+      connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                       circ->cpath);
+    ASSERT_COUNTED_BW();
+    sendme_cells--;
+  }
+  ENTRY_TO_CONN(entryconn2)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn2)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_SENDME, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Only one END cell */
+  ENTRY_TO_CONN(entryconn2)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn2)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_END, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  ENTRY_TO_CONN(entryconn2)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn2)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_END, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  edgeconn = ENTRY_TO_EDGE_CONN(entryconn3);
+  edgeconn->base_.state = AP_CONN_STATE_OPEN;
+  ENTRY_TO_CONN(entryconn3)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn3)->outbuf_flushlen = 0;
+  /* sendme cell on open entryconn with full window */
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_SENDME, "Data1234");
+  int ret =
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  tt_int_op(ret, OP_EQ, -END_CIRC_REASON_TORPROTOCOL);
+  ASSERT_UNCOUNTED_BW();
+
+  /* connected cell on a after EOF */
+  ENTRY_TO_CONN(entryconn3)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn3)->outbuf_flushlen = 0;
+  edgeconn->base_.state = AP_CONN_STATE_CONNECT_WAIT;
+  connection_edge_reached_eof(edgeconn);
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_CONNECTED, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ),  NULL,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  ENTRY_TO_CONN(entryconn3)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn3)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_CONNECTED, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ),  NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* DATA and SENDME after END cell */
+  ENTRY_TO_CONN(entryconn3)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn3)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_END, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ),  NULL,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  ENTRY_TO_CONN(entryconn3)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn3)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_SENDME, "Data1234");
+  ret =
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  tt_int_op(ret, OP_NE, -END_CIRC_REASON_TORPROTOCOL);
+  ASSERT_UNCOUNTED_BW();
+
+  ENTRY_TO_CONN(entryconn3)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn3)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_DATA, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Resolved: 1 counted, more not */
+  edgeconn = ENTRY_TO_EDGE_CONN(entryconn4);
+  entryconn4->socks_request->command = SOCKS_COMMAND_RESOLVE;
+  edgeconn->base_.state = AP_CONN_STATE_RESOLVE_WAIT;
+  edgeconn->on_circuit = TO_CIRCUIT(circ);
+  ENTRY_TO_CONN(entryconn4)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn4)->outbuf_flushlen = 0;
+  connection_edge_reached_eof(edgeconn);
+
+  ENTRY_TO_CONN(entryconn4)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn4)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_RESOLVED,
+            "\x04\x04\x12\x00\x00\x01\x00\x00\x02\x00");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  ENTRY_TO_CONN(entryconn4)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn4)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_RESOLVED,
+            "\x04\x04\x12\x00\x00\x01\x00\x00\x02\x00");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Data not counted after resolved */
+  ENTRY_TO_CONN(entryconn4)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn4)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_DATA, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* End not counted after resolved */
+  ENTRY_TO_CONN(entryconn4)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn4)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_END, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  connection_free_minimal(ENTRY_TO_CONN(entryconn2));
+  connection_free_minimal(ENTRY_TO_CONN(entryconn3));
+  connection_free_minimal(ENTRY_TO_CONN(entryconn4));
+  return 1;
+ done:
+  connection_free_minimal(ENTRY_TO_CONN(entryconn2));
+  connection_free_minimal(ENTRY_TO_CONN(entryconn3));
+  connection_free_minimal(ENTRY_TO_CONN(entryconn4));
+  return 0;
+}
+
+static int
+halfstream_insert(origin_circuit_t *circ, edge_connection_t *edgeconn,
+                  streamid_t *streams, int num, int random)
+{
+  int inserted = 0;
+
+  /* Insert num random elements */
+  while (inserted < num) {
+    streamid_t id;
+
+    if (random)
+      id = (streamid_t)crypto_rand_int(65535)+1;
+    else
+      id = get_unique_stream_id_by_circ(circ);
+
+    edgeconn->stream_id = id;
+
+    /* Ensure it isn't there */
+    if (connection_half_edge_find_stream_id(circ->half_streams, id)) {
+      continue;
+    }
+
+    connection_half_edge_add(edgeconn, circ);
+    if (streams)
+      streams[inserted] = id;
+    inserted++;
+  }
+
+  return inserted;
+}
+
+static void
+subtest_halfstream_insertremove(int num)
+{
+  origin_circuit_t *circ =
+      helper_create_origin_circuit(CIRCUIT_PURPOSE_C_GENERAL, 0);
+  edge_connection_t *edgeconn;
+  entry_connection_t *entryconn;
+  streamid_t *streams = tor_malloc_zero(num*sizeof(streamid_t));
+  int i = 0;
+
+  circ->cpath->state = CPATH_STATE_AWAITING_KEYS;
+  circ->cpath->deliver_window = CIRCWINDOW_START;
+
+  entryconn = fake_entry_conn(circ, 23);
+  edgeconn = ENTRY_TO_EDGE_CONN(entryconn);
+
+  /* Explicity test all operations on an absent stream list */
+  tt_int_op(connection_half_edge_is_valid_data(circ->half_streams,
+            23), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_connected(circ->half_streams,
+            23), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_sendme(circ->half_streams,
+            23), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_resolved(circ->half_streams,
+            23), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_end(circ->half_streams,
+            23), OP_EQ, 0);
+
+  /* Insert a duplicate element; verify that other elements absent;
+   * ensure removing it once works */
+  edgeconn->stream_id = 23;
+  connection_half_edge_add(edgeconn, circ);
+  connection_half_edge_add(edgeconn, circ);
+  connection_half_edge_add(edgeconn, circ);
+
+  /* Verify that other elements absent */
+  tt_int_op(connection_half_edge_is_valid_data(circ->half_streams,
+            22), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_connected(circ->half_streams,
+            22), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_sendme(circ->half_streams,
+            22), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_resolved(circ->half_streams,
+            22), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_end(circ->half_streams,
+            22), OP_EQ, 0);
+
+  tt_int_op(connection_half_edge_is_valid_data(circ->half_streams,
+            24), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_connected(circ->half_streams,
+            24), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_sendme(circ->half_streams,
+            24), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_resolved(circ->half_streams,
+            24), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_end(circ->half_streams,
+            24), OP_EQ, 0);
+
+  /* Verify we only remove it once */
+  tt_int_op(connection_half_edge_is_valid_end(circ->half_streams,
+            23), OP_EQ, 1);
+  tt_int_op(connection_half_edge_is_valid_end(circ->half_streams,
+            23), OP_EQ, 0);
+
+  halfstream_insert(circ, edgeconn, streams, num, 1);
+
+  /* Remove half of them */
+  for (i = 0; i < num/2; i++) {
+    tt_int_op(connection_half_edge_is_valid_end(circ->half_streams,
+                                                streams[i]),
+              OP_EQ, 1);
+  }
+
+  /* Verify first half of list is gone */
+  for (i = 0; i < num/2; i++) {
+    tt_ptr_op(connection_half_edge_find_stream_id(circ->half_streams,
+                                                  streams[i]),
+              OP_EQ, NULL);
+  }
+
+  /* Verify second half of list is present */
+  for (; i < num; i++) {
+    tt_ptr_op(connection_half_edge_find_stream_id(circ->half_streams,
+                                                  streams[i]),
+              OP_NE, NULL);
+  }
+
+  /* Remove other half. Verify list is empty. */
+  for (i = num/2; i < num; i++) {
+    tt_int_op(connection_half_edge_is_valid_end(circ->half_streams,
+                                                streams[i]),
+              OP_EQ, 1);
+  }
+  tt_int_op(smartlist_len(circ->half_streams), OP_EQ, 0);
+
+  /* Explicity test all operations on an empty stream list */
+  tt_int_op(connection_half_edge_is_valid_data(circ->half_streams,
+            23), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_connected(circ->half_streams,
+            23), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_sendme(circ->half_streams,
+            23), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_resolved(circ->half_streams,
+            23), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_end(circ->half_streams,
+            23), OP_EQ, 0);
+
+  /* For valgrind, leave some around then free the circ */
+  halfstream_insert(circ, edgeconn, NULL, 10, 0);
+
+ done:
+  tor_free(streams);
+  circuit_free_(TO_CIRCUIT(circ));
+  connection_free_minimal(ENTRY_TO_CONN(entryconn));
+}
+
+static void
+test_halfstream_insertremove(void *arg)
+{
+  (void)arg;
+
+  /* Suppress the WARN message we generate in this test */
+  setup_full_capture_of_logs(LOG_WARN);
+
+  /* Test insertion and removal with a few different sizes */
+  subtest_halfstream_insertremove(10);
+  subtest_halfstream_insertremove(100);
+  subtest_halfstream_insertremove(1000);
+}
+
+static void
+test_halfstream_wrap(void *arg)
+{
+  origin_circuit_t *circ =
+      helper_create_origin_circuit(CIRCUIT_PURPOSE_C_GENERAL, 0);
+  edge_connection_t *edgeconn;
+  entry_connection_t *entryconn;
+
+  circ->cpath->state = CPATH_STATE_AWAITING_KEYS;
+  circ->cpath->deliver_window = CIRCWINDOW_START;
+
+  entryconn = fake_entry_conn(circ, 23);
+  edgeconn = ENTRY_TO_EDGE_CONN(entryconn);
+
+  (void)arg;
+
+  /* Suppress the WARN message we generate in this test */
+  setup_full_capture_of_logs(LOG_WARN);
+  MOCK(connection_mark_for_close_internal_, mock_mark_for_close);
+
+  /* Verify that get_unique_stream_id_by_circ() can wrap uint16_t */
+  circ->next_stream_id = 65530;
+  halfstream_insert(circ, edgeconn, NULL, 7, 0);
+  tt_int_op(circ->next_stream_id, OP_EQ, 2);
+  tt_int_op(smartlist_len(circ->half_streams), OP_EQ, 7);
+
+  /* Insert full-1 */
+  halfstream_insert(circ, edgeconn, NULL,
+                    65534-smartlist_len(circ->half_streams), 0);
+  tt_int_op(smartlist_len(circ->half_streams), OP_EQ, 65534);
+
+  /* Verify that we can get_unique_stream_id_by_circ() successfully */
+  edgeconn->stream_id = get_unique_stream_id_by_circ(circ);
+  tt_int_op(edgeconn->stream_id, OP_NE, 0); /* 0 is failure */
+
+  /* Insert an opened stream on the circ with that id */
+  ENTRY_TO_CONN(entryconn)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn)->outbuf_flushlen = 0;
+  edgeconn->base_.state = AP_CONN_STATE_CONNECT_WAIT;
+  circ->p_streams = edgeconn;
+
+  /* Verify that get_unique_stream_id_by_circ() fails */
+  tt_int_op(get_unique_stream_id_by_circ(circ), OP_EQ, 0); /* 0 is failure */
+
+  /* eof the one opened stream. Verify it is now in half-closed */
+  tt_int_op(smartlist_len(circ->half_streams), OP_EQ, 65534);
+  connection_edge_reached_eof(edgeconn);
+  tt_int_op(smartlist_len(circ->half_streams), OP_EQ, 65535);
+
+  /* Verify get_unique_stream_id_by_circ() fails due to full half-closed */
+  circ->p_streams = NULL;
+  tt_int_op(get_unique_stream_id_by_circ(circ), OP_EQ, 0); /* 0 is failure */
+
+ done:
+  circuit_free_(TO_CIRCUIT(circ));
+  connection_free_minimal(ENTRY_TO_CONN(entryconn));
+  UNMOCK(connection_mark_for_close_internal_);
+}
+
+static void
+test_circbw_relay(void *arg)
+{
+  cell_t cell;
+  relay_header_t rh;
+  tor_addr_t addr;
+  edge_connection_t *edgeconn;
+  entry_connection_t *entryconn1=NULL;
+  origin_circuit_t *circ;
+  int delivered = 0;
+  int overhead = 0;
+
+  (void)arg;
+
+  MOCK(connection_mark_unattached_ap_, mock_connection_mark_unattached_ap_);
+  MOCK(connection_start_reading, mock_start_reading);
+  MOCK(connection_mark_for_close_internal_, mock_mark_for_close);
+  MOCK(relay_send_command_from_edge_, mock_send_command);
+  MOCK(circuit_mark_for_close_, mock_mark_circ_for_close);
+
+  circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_C_GENERAL, 0);
+  circ->cpath->state = CPATH_STATE_AWAITING_KEYS;
+  circ->cpath->deliver_window = CIRCWINDOW_START;
+
+  entryconn1 = fake_entry_conn(circ, 1);
+  edgeconn = ENTRY_TO_EDGE_CONN(entryconn1);
+
+  /* Stream id 0: Not counted */
+  PACK_CELL(0, RELAY_COMMAND_END, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Stream id 1: Counted */
+  PACK_CELL(1, RELAY_COMMAND_END, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  /* Properly formatted connect cell: counted */
+  PACK_CELL(1, RELAY_COMMAND_CONNECTED, "Data1234");
+  tor_addr_parse(&addr, "30.40.50.60");
+  rh.length = connected_cell_format_payload(cell.payload+RELAY_HEADER_SIZE,
+                                            &addr, 1024);
+  relay_header_pack((uint8_t*)&cell.payload, &rh);                    \
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  /* Properly formatted resolved cell in correct state: counted */
+  edgeconn->base_.state = AP_CONN_STATE_RESOLVE_WAIT;
+  entryconn1->socks_request->command = SOCKS_COMMAND_RESOLVE;
+  edgeconn->on_circuit = TO_CIRCUIT(circ);
+  PACK_CELL(1, RELAY_COMMAND_RESOLVED,
+            "\x04\x04\x12\x00\x00\x01\x00\x00\x02\x00");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  edgeconn->base_.state = AP_CONN_STATE_OPEN;
+  entryconn1->socks_request->has_finished = 1;
+
+  /* Connected cell after open: not counted */
+  PACK_CELL(1, RELAY_COMMAND_CONNECTED, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Resolved cell after open: not counted */
+  PACK_CELL(1, RELAY_COMMAND_RESOLVED, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Drop cell: not counted */
+  PACK_CELL(1, RELAY_COMMAND_DROP, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Data cell on stream 0: not counted */
+  PACK_CELL(0, RELAY_COMMAND_DATA, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Data cell on open connection: counted */
+  ENTRY_TO_CONN(entryconn1)->marked_for_close = 0;
+  PACK_CELL(1, RELAY_COMMAND_DATA, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  /* Empty Data cell on open connection: not counted */
+  ENTRY_TO_CONN(entryconn1)->marked_for_close = 0;
+  PACK_CELL(1, RELAY_COMMAND_DATA, "");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Sendme on valid stream: counted */
+  edgeconn->package_window -= STREAMWINDOW_INCREMENT;
+  ENTRY_TO_CONN(entryconn1)->outbuf_flushlen = 0;
+  PACK_CELL(1, RELAY_COMMAND_SENDME, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  /* Sendme on valid stream with full window: not counted */
+  ENTRY_TO_CONN(entryconn1)->outbuf_flushlen = 0;
+  PACK_CELL(1, RELAY_COMMAND_SENDME, "Data1234");
+  edgeconn->package_window = STREAMWINDOW_START;
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Sendme on unknown stream: not counted */
+  ENTRY_TO_CONN(entryconn1)->outbuf_flushlen = 0;
+  PACK_CELL(1, RELAY_COMMAND_SENDME, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Sendme on circuit with full window: not counted */
+  PACK_CELL(0, RELAY_COMMAND_SENDME, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Sendme on circuit with non-full window: counted */
+  PACK_CELL(0, RELAY_COMMAND_SENDME, "Data1234");
+  circ->cpath->package_window = 900;
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  /* Invalid extended cell: not counted */
+  PACK_CELL(1, RELAY_COMMAND_EXTENDED2, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Invalid extended cell: not counted */
+  PACK_CELL(1, RELAY_COMMAND_EXTENDED, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Invalid HS cell: not counted */
+  PACK_CELL(1, RELAY_COMMAND_ESTABLISH_INTRO, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* "Valid" HS cell in expected state: counted */
+  TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_C_ESTABLISH_REND;
+  PACK_CELL(1, RELAY_COMMAND_RENDEZVOUS_ESTABLISHED, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  /* End cell on non-closed connection: counted */
+  PACK_CELL(1, RELAY_COMMAND_END, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  /* End cell on connection that already got one: not counted */
+  PACK_CELL(1, RELAY_COMMAND_END, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Simulate closed stream on entryconn, then test: */
+  if (!subtest_circbw_halfclosed(circ, 2))
+    goto done;
+
+  circ->base_.purpose = CIRCUIT_PURPOSE_PATH_BIAS_TESTING;
+  if (!subtest_circbw_halfclosed(circ, 6))
+    goto done;
+
+  /* Path bias: truncated */
+  tt_int_op(circ->base_.marked_for_close, OP_EQ, 0);
+  PACK_CELL(0, RELAY_COMMAND_TRUNCATED, "Data1234");
+  pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  tt_int_op(circ->base_.marked_for_close, OP_EQ, 1);
+
+ done:
+  UNMOCK(connection_start_reading);
+  UNMOCK(connection_mark_unattached_ap_);
+  UNMOCK(connection_mark_for_close_internal_);
+  UNMOCK(relay_send_command_from_edge_);
+  UNMOCK(circuit_mark_for_close_);
+  circuit_free_(TO_CIRCUIT(circ));
+  connection_free_minimal(ENTRY_TO_CONN(entryconn1));
+}
+
 /* Tests for connection_edge_process_resolved_cell().
 
    The point of ..process_resolved_cell() is to handle an incoming cell
@@ -244,6 +1063,8 @@ test_relaycell_resolved(void *arg)
 
 struct testcase_t relaycell_tests[] = {
   { "resolved", test_relaycell_resolved, TT_FORK, NULL, NULL },
+  { "circbw", test_circbw_relay, TT_FORK, NULL, NULL },
+  { "halfstream", test_halfstream_insertremove, TT_FORK, NULL, NULL },
+  { "streamwrap", test_halfstream_wrap, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_relaycrypt.c b/src/test/test_relaycrypt.c
new file mode 100644
index 0000000000..fe6889e521
--- /dev/null
+++ b/src/test/test_relaycrypt.c
@@ -0,0 +1,190 @@
+/* Copyright 2001-2004 Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "core/or/circuitbuild.h"
+#define CIRCUITLIST_PRIVATE
+#include "core/or/circuitlist.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "core/or/relay.h"
+#include "core/crypto/relay_crypto.h"
+
+#include "core/or/cell_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+
+#include "test/test.h"
+
+static const char KEY_MATERIAL[3][CPATH_KEY_MATERIAL_LEN] = {
+  "    'My public key is in this signed x509 object', said Tom assertively.",
+  "'Let's chart the pedal phlanges in the tomb', said Tom cryptographically",
+  "     'Segmentation fault bugs don't _just happen_', said Tom seethingly.",
+};
+
+typedef struct testing_circuitset_t {
+  or_circuit_t *or_circ[3];
+  origin_circuit_t *origin_circ;
+} testing_circuitset_t;
+
+static int testing_circuitset_teardown(const struct testcase_t *testcase,
+                                       void *ptr);
+
+static void *
+testing_circuitset_setup(const struct testcase_t *testcase)
+{
+  testing_circuitset_t *cs = tor_malloc_zero(sizeof(testing_circuitset_t));
+  int i;
+
+  for (i=0; i<3; ++i) {
+    cs->or_circ[i] = or_circuit_new(0, NULL);
+    tt_int_op(0, OP_EQ,
+              relay_crypto_init(&cs->or_circ[i]->crypto,
+                                KEY_MATERIAL[i], sizeof(KEY_MATERIAL[i]),
+                                0, 0));
+  }
+
+  cs->origin_circ = origin_circuit_new();
+  cs->origin_circ->base_.purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  for (i=0; i<3; ++i) {
+    crypt_path_t *hop = tor_malloc_zero(sizeof(*hop));
+    relay_crypto_init(&hop->crypto, KEY_MATERIAL[i], sizeof(KEY_MATERIAL[i]),
+                      0, 0);
+    hop->state = CPATH_STATE_OPEN;
+    onion_append_to_cpath(&cs->origin_circ->cpath, hop);
+    tt_ptr_op(hop, OP_EQ, cs->origin_circ->cpath->prev);
+  }
+
+  return cs;
+ done:
+  testing_circuitset_teardown(testcase, cs);
+  return NULL;
+}
+
+static int
+testing_circuitset_teardown(const struct testcase_t *testcase, void *ptr)
+{
+  (void)testcase;
+  testing_circuitset_t *cs = ptr;
+  int i;
+  for (i=0; i<3; ++i) {
+    circuit_free_(TO_CIRCUIT(cs->or_circ[i]));
+  }
+  circuit_free_(TO_CIRCUIT(cs->origin_circ));
+  tor_free(cs);
+  return 1;
+}
+
+static const struct testcase_setup_t relaycrypt_setup = {
+  testing_circuitset_setup, testing_circuitset_teardown
+};
+
+/* Test encrypting a cell to the final hop on a circuit, decrypting it
+ * at each hop, and recognizing it at the other end.  Then do it again
+ * and again as the state evolves. */
+static void
+test_relaycrypt_outbound(void *arg)
+{
+  testing_circuitset_t *cs = arg;
+  tt_assert(cs);
+
+  relay_header_t rh;
+  cell_t orig;
+  cell_t encrypted;
+  int i, j;
+
+  for (i = 0; i < 50; ++i) {
+    crypto_rand((char *)&orig, sizeof(orig));
+
+    relay_header_unpack(&rh, orig.payload);
+    rh.recognized = 0;
+    memset(rh.integrity, 0, sizeof(rh.integrity));
+    relay_header_pack(orig.payload, &rh);
+
+    memcpy(&encrypted, &orig, sizeof(orig));
+
+    /* Encrypt the cell to the last hop */
+    relay_encrypt_cell_outbound(&encrypted, cs->origin_circ,
+                                cs->origin_circ->cpath->prev);
+
+    for (j = 0; j < 3; ++j) {
+      crypt_path_t *layer_hint = NULL;
+      char recognized = 0;
+      int r = relay_decrypt_cell(TO_CIRCUIT(cs->or_circ[j]),
+                                 &encrypted,
+                                 CELL_DIRECTION_OUT,
+                                 &layer_hint, &recognized);
+      tt_int_op(r, OP_EQ, 0);
+      tt_ptr_op(layer_hint, OP_EQ, NULL);
+      tt_int_op(recognized != 0, OP_EQ, j == 2);
+    }
+
+    tt_mem_op(orig.payload, OP_EQ, encrypted.payload, CELL_PAYLOAD_SIZE);
+  }
+
+ done:
+  ;
+}
+
+/* As above, but simulate inbound cells from the last hop. */
+static void
+test_relaycrypt_inbound(void *arg)
+{
+  testing_circuitset_t *cs = arg;
+  tt_assert(cs);
+
+  relay_header_t rh;
+  cell_t orig;
+  cell_t encrypted;
+  int i, j;
+
+  for (i = 0; i < 50; ++i) {
+    crypto_rand((char *)&orig, sizeof(orig));
+
+    relay_header_unpack(&rh, orig.payload);
+    rh.recognized = 0;
+    memset(rh.integrity, 0, sizeof(rh.integrity));
+    relay_header_pack(orig.payload, &rh);
+
+    memcpy(&encrypted, &orig, sizeof(orig));
+
+    /* Encrypt the cell to the last hop */
+    relay_encrypt_cell_inbound(&encrypted, cs->or_circ[2]);
+
+    crypt_path_t *layer_hint = NULL;
+    char recognized = 0;
+    int r;
+    for (j = 1; j >= 0; --j) {
+      r = relay_decrypt_cell(TO_CIRCUIT(cs->or_circ[j]),
+                             &encrypted,
+                             CELL_DIRECTION_IN,
+                             &layer_hint, &recognized);
+      tt_int_op(r, OP_EQ, 0);
+      tt_ptr_op(layer_hint, OP_EQ, NULL);
+      tt_int_op(recognized, OP_EQ, 0);
+    }
+
+    relay_decrypt_cell(TO_CIRCUIT(cs->origin_circ),
+                       &encrypted,
+                       CELL_DIRECTION_IN,
+                       &layer_hint, &recognized);
+    tt_int_op(r, OP_EQ, 0);
+    tt_int_op(recognized, OP_EQ, 1);
+    tt_ptr_op(layer_hint, OP_EQ, cs->origin_circ->cpath->prev);
+
+    tt_mem_op(orig.payload, OP_EQ, encrypted.payload, CELL_PAYLOAD_SIZE);
+  }
+ done:
+  ;
+}
+
+#define TEST(name) \
+  { # name, test_relaycrypt_ ## name, 0, &relaycrypt_setup, NULL }
+
+struct testcase_t relaycrypt_tests[] = {
+  TEST(outbound),
+  TEST(inbound),
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_rendcache.c b/src/test/test_rendcache.c
index a5d3f351f8..4f544cf21c 100644
--- a/src/test/test_rendcache.c
+++ b/src/test/test_rendcache.c
@@ -1,18 +1,25 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
-#include "or.h"
+#include "core/or/or.h"
 
-#include "test.h"
+#include "test/test.h"
 #define RENDCACHE_PRIVATE
-#include "rendcache.h"
-#include "router.h"
-#include "routerlist.h"
-#include "config.h"
-#include <openssl/rsa.h>
-#include "rend_test_helpers.h"
-#include "log_test_helpers.h"
+#include "feature/rend/rendcache.h"
+#include "feature/relay/router.h"
+#include "feature/nodelist/routerlist.h"
+#include "app/config/config.h"
+#include "feature/hs/hs_common.h"
+
+#include "core/or/extend_info_st.h"
+#include "feature/rend/rend_encoded_v2_service_descriptor_st.h"
+#include "feature/rend/rend_intro_point_st.h"
+#include "feature/rend/rend_service_descriptor_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+
+#include "test/rend_test_helpers.h"
+#include "test/log_test_helpers.h"
 
 #define NS_MODULE rend_cache
 
@@ -21,21 +28,6 @@ static const int TIME_IN_THE_PAST = -(REND_CACHE_MAX_AGE + \
                                       REND_CACHE_MAX_SKEW + 60);
 static const int TIME_IN_THE_FUTURE = REND_CACHE_MAX_SKEW + 60;
 
-static rend_data_t *
-mock_rend_data(const char *onion_address)
-{
-  rend_data_t *rend_query = tor_malloc_zero(sizeof(rend_data_t));
-
-  strlcpy(rend_query->onion_address, onion_address,
-          sizeof(rend_query->onion_address));
-  rend_query->auth_type = REND_NO_AUTH;
-  rend_query->hsdirs_fp = smartlist_new();
-  smartlist_add(rend_query->hsdirs_fp, tor_memdup("aaaaaaaaaaaaaaaaaaaaaaaa",
-                                                 DIGEST_LEN));
-
-  return rend_query;
-}
-
 static void
 test_rend_cache_lookup_entry(void *data)
 {
@@ -73,6 +65,7 @@ test_rend_cache_lookup_entry(void *data)
   tt_int_op(ret, OP_EQ, 0);
 
   ret = rend_cache_lookup_entry(service_id, 2, &entry);
+  tt_int_op(ret, OP_EQ, 0);
   tt_assert(entry);
   tt_int_op(entry->len, OP_EQ, strlen(desc_holder->desc_str));
   tt_str_op(entry->desc, OP_EQ, desc_holder->desc_str);
@@ -144,7 +137,8 @@ test_rend_cache_store_v2_desc_as_client(void *data)
 
   // Test mismatch between service ID and onion address
   rend_cache_init();
-  strncpy(mock_rend_query->onion_address, "abc", REND_SERVICE_ID_LEN_BASE32+1);
+  strncpy(TO_REND_DATA_V2(mock_rend_query)->onion_address, "abc",
+          REND_SERVICE_ID_LEN_BASE32+1);
   ret = rend_cache_store_v2_desc_as_client(desc_holder->desc_str,
                                            desc_id_base32,
                                            mock_rend_query, NULL);
@@ -155,12 +149,16 @@ test_rend_cache_store_v2_desc_as_client(void *data)
   // Test incorrect descriptor ID
   rend_cache_init();
   mock_rend_query = mock_rend_data(service_id);
-  desc_id_base32[0]++;
+  char orig = desc_id_base32[0];
+  if (desc_id_base32[0] == 'a')
+    desc_id_base32[0] = 'b';
+  else
+    desc_id_base32[0] = 'a';
   ret = rend_cache_store_v2_desc_as_client(desc_holder->desc_str,
                                            desc_id_base32, mock_rend_query,
                                            NULL);
   tt_int_op(ret, OP_EQ, -1);
-  desc_id_base32[0]--;
+  desc_id_base32[0] = orig;
   rend_cache_free_all();
 
   // Test too old descriptor
@@ -230,9 +228,9 @@ test_rend_cache_store_v2_desc_as_client(void *data)
 
   generate_desc(RECENT_TIME, &desc_holder, &service_id, 3);
   mock_rend_query = mock_rend_data(service_id);
-  mock_rend_query->auth_type = REND_BASIC_AUTH;
+  TO_REND_DATA_V2(mock_rend_query)->auth_type = REND_BASIC_AUTH;
   client_cookie[0] = 'A';
-  memcpy(mock_rend_query->descriptor_cookie, client_cookie,
+  memcpy(TO_REND_DATA_V2(mock_rend_query)->descriptor_cookie, client_cookie,
          REND_DESC_COOKIE_LEN);
   base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_holder->desc_id,
                 DIGEST_LEN);
@@ -250,7 +248,7 @@ test_rend_cache_store_v2_desc_as_client(void *data)
 
   generate_desc(RECENT_TIME, &desc_holder, &service_id, 3);
   mock_rend_query = mock_rend_data(service_id);
-  mock_rend_query->auth_type = REND_BASIC_AUTH;
+  TO_REND_DATA_V2(mock_rend_query)->auth_type = REND_BASIC_AUTH;
   base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_holder->desc_id,
                 DIGEST_LEN);
   ret = rend_cache_store_v2_desc_as_client(desc_holder->desc_str,
@@ -790,7 +788,9 @@ test_rend_cache_clean(void *data)
   desc_two->pk = pk_generate(1);
 
   strmap_set_lc(rend_cache, "foo1", one);
+  rend_cache_increment_allocation(rend_cache_entry_allocation(one));
   strmap_set_lc(rend_cache, "foo2", two);
+  rend_cache_increment_allocation(rend_cache_entry_allocation(two));
 
   rend_cache_clean(time(NULL), REND_CACHE_TYPE_CLIENT);
   tt_int_op(strmap_size(rend_cache), OP_EQ, 0);
@@ -808,7 +808,9 @@ test_rend_cache_clean(void *data)
   desc_one->pk = pk_generate(0);
   desc_two->pk = pk_generate(1);
 
+  rend_cache_increment_allocation(rend_cache_entry_allocation(one));
   strmap_set_lc(rend_cache, "foo1", one);
+  rend_cache_increment_allocation(rend_cache_entry_allocation(two));
   strmap_set_lc(rend_cache, "foo2", two);
 
   rend_cache_clean(time(NULL), REND_CACHE_TYPE_CLIENT);
@@ -843,7 +845,7 @@ test_rend_cache_failure_entry_free(void *data)
   (void)data;
 
   // Test that it can deal with a NULL argument
-  rend_cache_failure_entry_free(NULL);
+  rend_cache_failure_entry_free_(NULL);
 
  /* done: */
  /*  (void)0; */
@@ -956,9 +958,9 @@ test_rend_cache_free_all(void *data)
 
   rend_cache_free_all();
 
-  tt_assert(!rend_cache);
-  tt_assert(!rend_cache_v2_dir);
-  tt_assert(!rend_cache_failure);
+  tt_ptr_op(rend_cache, OP_EQ, NULL);
+  tt_ptr_op(rend_cache_v2_dir, OP_EQ, NULL);
+  tt_ptr_op(rend_cache_failure, OP_EQ, NULL);
   tt_assert(!rend_cache_total_allocation);
 
  done:
@@ -972,7 +974,7 @@ test_rend_cache_entry_free(void *data)
   rend_cache_entry_t *e;
 
   // Handles NULL correctly
-  rend_cache_entry_free(NULL);
+  rend_cache_entry_free_(NULL);
 
   // Handles NULL descriptor correctly
   e = tor_malloc_zero(sizeof(rend_cache_entry_t));
@@ -1078,9 +1080,10 @@ static void
 test_rend_cache_clean_v2_descs_as_dir(void *data)
 {
   rend_cache_entry_t *e;
-  time_t now;
+  time_t now, cutoff;
   rend_service_descriptor_t *desc;
   now = time(NULL);
+  cutoff = now - (REND_CACHE_MAX_AGE + REND_CACHE_MAX_SKEW);
   const char key[DIGEST_LEN] = "abcde";
 
   (void)data;
@@ -1088,7 +1091,7 @@ test_rend_cache_clean_v2_descs_as_dir(void *data)
   rend_cache_init();
 
   // Test running with an empty cache
-  rend_cache_clean_v2_descs_as_dir(now, 0);
+  rend_cache_clean_v2_descs_as_dir(cutoff);
   tt_int_op(digestmap_size(rend_cache_v2_dir), OP_EQ, 0);
 
   // Test with only one new entry
@@ -1100,38 +1103,15 @@ test_rend_cache_clean_v2_descs_as_dir(void *data)
   e->parsed = desc;
   digestmap_set(rend_cache_v2_dir, key, e);
 
-  rend_cache_clean_v2_descs_as_dir(now, 0);
+  /* Set the cutoff to minus 10 seconds. */
+  rend_cache_clean_v2_descs_as_dir(cutoff - 10);
   tt_int_op(digestmap_size(rend_cache_v2_dir), OP_EQ, 1);
 
   // Test with one old entry
-  desc->timestamp = now - (REND_CACHE_MAX_AGE + REND_CACHE_MAX_SKEW + 1000);
-  rend_cache_clean_v2_descs_as_dir(now, 0);
-  tt_int_op(digestmap_size(rend_cache_v2_dir), OP_EQ, 0);
-
-  // Test with one entry that has an old last served
-  e = tor_malloc_zero(sizeof(rend_cache_entry_t));
-  e->last_served = now - (REND_CACHE_MAX_AGE + REND_CACHE_MAX_SKEW + 1000);
-  desc = tor_malloc_zero(sizeof(rend_service_descriptor_t));
-  desc->timestamp = now;
-  desc->pk = pk_generate(0);
-  e->parsed = desc;
-  digestmap_set(rend_cache_v2_dir, key, e);
-
-  rend_cache_clean_v2_descs_as_dir(now, 0);
+  desc->timestamp = cutoff - 1000;
+  rend_cache_clean_v2_descs_as_dir(cutoff);
   tt_int_op(digestmap_size(rend_cache_v2_dir), OP_EQ, 0);
 
-  // Test a run through asking for a large force_remove
-  e = tor_malloc_zero(sizeof(rend_cache_entry_t));
-  e->last_served = now;
-  desc = tor_malloc_zero(sizeof(rend_service_descriptor_t));
-  desc->timestamp = now;
-  desc->pk = pk_generate(0);
-  e->parsed = desc;
-  digestmap_set(rend_cache_v2_dir, key, e);
-
-  rend_cache_clean_v2_descs_as_dir(now, 20000);
-  tt_int_op(digestmap_size(rend_cache_v2_dir), OP_EQ, 1);
-
  done:
   rend_cache_free_all();
 }
@@ -1166,7 +1146,7 @@ test_rend_cache_failure_intro_entry_free(void *data)
   rend_cache_failure_intro_t *entry;
 
   // Handles a null argument
-  rend_cache_failure_intro_entry_free(NULL);
+  rend_cache_failure_intro_entry_free_(NULL);
 
   // Handles a non-null argument
   entry = rend_cache_failure_intro_entry_new(INTRO_POINT_FAILURE_TIMEOUT);
@@ -1179,7 +1159,7 @@ test_rend_cache_failure_purge(void *data)
   (void)data;
 
   // Handles a null failure cache
-  strmap_free(rend_cache_failure, rend_cache_failure_entry_free_);
+  strmap_free(rend_cache_failure, rend_cache_failure_entry_free_void);
   rend_cache_failure = NULL;
 
   rend_cache_failure_purge();
diff --git a/src/test/test_replay.c b/src/test/test_replay.c
index e882bc6164..28a508bf4d 100644
--- a/src/test/test_replay.c
+++ b/src/test/test_replay.c
@@ -1,12 +1,12 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define REPLAYCACHE_PRIVATE
 
 #include "orconfig.h"
-#include "or.h"
-#include "replaycache.h"
-#include "test.h"
+#include "core/or/or.h"
+#include "feature/hs_common/replaycache.h"
+#include "test/test.h"
 
 static const char *test_buffer =
   "Lorem ipsum dolor sit amet, consectetur adipisici elit, sed do eiusmod"
@@ -38,7 +38,7 @@ test_replaycache_alloc(void *arg)
 
   (void)arg;
   r = replaycache_new(600, 300);
-  tt_assert(r != NULL);
+  tt_ptr_op(r, OP_NE, NULL);
 
  done:
   if (r) replaycache_free(r);
@@ -54,15 +54,15 @@ test_replaycache_badalloc(void *arg)
   /* Negative horizon should fail */
   (void)arg;
   r = replaycache_new(-600, 300);
-  tt_assert(r == NULL);
+  tt_ptr_op(r, OP_EQ, NULL);
   /* Negative interval should get adjusted to zero */
   r = replaycache_new(600, -300);
-  tt_assert(r != NULL);
+  tt_ptr_op(r, OP_NE, NULL);
   tt_int_op(r->scrub_interval,OP_EQ, 0);
   replaycache_free(r);
   /* Negative horizon and negative interval should still fail */
   r = replaycache_new(-600, -300);
-  tt_assert(r == NULL);
+  tt_ptr_op(r, OP_EQ, NULL);
 
  done:
   if (r) replaycache_free(r);
@@ -74,7 +74,7 @@ static void
 test_replaycache_free_null(void *arg)
 {
   (void)arg;
-  replaycache_free(NULL);
+  replaycache_free_(NULL);
   /* Assert that we're here without horrible death */
   tt_assert(1);
 
@@ -90,7 +90,7 @@ test_replaycache_miss(void *arg)
 
   (void)arg;
   r = replaycache_new(600, 300);
-  tt_assert(r != NULL);
+  tt_ptr_op(r, OP_NE, NULL);
 
   result =
     replaycache_add_and_test_internal(1200, r, test_buffer,
@@ -123,7 +123,7 @@ test_replaycache_hit(void *arg)
 
   (void)arg;
   r = replaycache_new(600, 300);
-  tt_assert(r != NULL);
+  tt_ptr_op(r, OP_NE, NULL);
 
   result =
     replaycache_add_and_test_internal(1200, r, test_buffer,
@@ -161,7 +161,7 @@ test_replaycache_age(void *arg)
 
   (void)arg;
   r = replaycache_new(600, 300);
-  tt_assert(r != NULL);
+  tt_ptr_op(r, OP_NE, NULL);
 
   result =
     replaycache_add_and_test_internal(1200, r, test_buffer,
@@ -193,7 +193,7 @@ test_replaycache_elapsed(void *arg)
 
   (void)arg;
   r = replaycache_new(600, 300);
-  tt_assert(r != NULL);
+  tt_ptr_op(r, OP_NE, NULL);
 
   result =
     replaycache_add_and_test_internal(1200, r, test_buffer,
@@ -220,7 +220,7 @@ test_replaycache_noexpire(void *arg)
 
   (void)arg;
   r = replaycache_new(0, 0);
-  tt_assert(r != NULL);
+  tt_ptr_op(r, OP_NE, NULL);
 
   result =
     replaycache_add_and_test_internal(1200, r, test_buffer,
@@ -251,7 +251,7 @@ test_replaycache_scrub(void *arg)
 
   (void)arg;
   r = replaycache_new(600, 300);
-  tt_assert(r != NULL);
+  tt_ptr_op(r, OP_NE, NULL);
 
   /* Set up like in test_replaycache_hit() */
   result =
@@ -294,7 +294,7 @@ test_replaycache_future(void *arg)
 
   (void)arg;
   r = replaycache_new(600, 300);
-  tt_assert(r != NULL);
+  tt_ptr_op(r, OP_NE, NULL);
 
   /* Set up like in test_replaycache_hit() */
   result =
@@ -343,7 +343,7 @@ test_replaycache_realtime(void *arg)
   /* Test the realtime as well as *_internal() entry points */
   (void)arg;
   r = replaycache_new(600, 300);
-  tt_assert(r != NULL);
+  tt_ptr_op(r, OP_NE, NULL);
 
   /* This should miss */
   result =
diff --git a/src/test/test_router.c b/src/test/test_router.c
index 51055a3367..601881a124 100644
--- a/src/test/test_router.c
+++ b/src/test/test_router.c
@@ -1,18 +1,117 @@
-/* Copyright (c) 2018, The Tor Project, Inc. */
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017, isis agora lovecruft  */
 /* See LICENSE for licensing information     */
 
 /**
  * \file test_router.c
- * \brief Unittests for code in src/or/router.c
+ * \brief Unittests for code in router.c
  **/
 
-#include "or.h"
-#include "hibernate.h"
-#include "log_test_helpers.h"
-#include "main.h"
-#include "rephist.h"
-#include "router.h"
-#include "test.h"
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/mainloop.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/router.h"
+#include "feature/stats/rephist.h"
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+/* Test suite stuff */
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+
+NS_DECL(const routerinfo_t *, router_get_my_routerinfo, (void));
+
+static routerinfo_t* mock_routerinfo;
+
+static const routerinfo_t*
+NS(router_get_my_routerinfo)(void)
+{
+  crypto_pk_t* ident_key;
+  crypto_pk_t* tap_key;
+  time_t now;
+
+  if (!mock_routerinfo) {
+    /* Mock the published timestamp, otherwise router_dump_router_to_string()
+     * will poop its pants. */
+    time(&now);
+
+    /* We'll need keys, or router_dump_router_to_string() would return NULL. */
+    ident_key = pk_generate(0);
+    tap_key = pk_generate(0);
+
+    tor_assert(ident_key != NULL);
+    tor_assert(tap_key != NULL);
+
+    mock_routerinfo = tor_malloc_zero(sizeof(routerinfo_t));
+    mock_routerinfo->nickname = tor_strdup("ConlonNancarrow");
+    mock_routerinfo->addr = 123456789;
+    mock_routerinfo->or_port = 443;
+    mock_routerinfo->platform = tor_strdup("unittest");
+    mock_routerinfo->cache_info.published_on = now;
+    mock_routerinfo->identity_pkey = crypto_pk_dup_key(ident_key);
+    router_set_rsa_onion_pkey(tap_key, &mock_routerinfo->onion_pkey,
+                              &mock_routerinfo->onion_pkey_len);
+    mock_routerinfo->bandwidthrate = 9001;
+    mock_routerinfo->bandwidthburst = 9002;
+    crypto_pk_free(ident_key);
+    crypto_pk_free(tap_key);
+  }
+
+  return mock_routerinfo;
+}
+
+/* If no distribution option was set, then check_bridge_distribution_setting()
+ * should have set it to "any". */
+static void
+test_router_dump_router_to_string_no_bridge_distribution_method(void *arg)
+{
+  const char* needle = "bridge-distribution-request any";
+  or_options_t* options = get_options_mutable();
+  routerinfo_t* router = NULL;
+  curve25519_keypair_t ntor_keypair;
+  ed25519_keypair_t signing_keypair;
+  char* desc = NULL;
+  char* found = NULL;
+  (void)arg;
+
+  NS_MOCK(router_get_my_routerinfo);
+
+  options->ORPort_set = 1;
+  options->BridgeRelay = 1;
+
+  /* Generate keys which router_dump_router_to_string() expects to exist. */
+  tt_int_op(0, ==, curve25519_keypair_generate(&ntor_keypair, 0));
+  tt_int_op(0, ==, ed25519_keypair_generate(&signing_keypair, 0));
+
+  /* Set up part of our routerinfo_t so that we don't trigger any other
+   * assertions in router_dump_router_to_string(). */
+  router = (routerinfo_t*)router_get_my_routerinfo();
+  tt_ptr_op(router, !=, NULL);
+
+  router->onion_curve25519_pkey = &ntor_keypair.pubkey;
+
+  /* Generate our server descriptor and ensure that the substring
+   * "bridge-distribution-request any" occurs somewhere within it. */
+  crypto_pk_t *onion_pkey = router_get_rsa_onion_pkey(router->onion_pkey,
+                                                      router->onion_pkey_len);
+  desc = router_dump_router_to_string(router,
+                                      router->identity_pkey,
+                                      onion_pkey,
+                                      &ntor_keypair,
+                                      &signing_keypair);
+  crypto_pk_free(onion_pkey);
+  tt_ptr_op(desc, !=, NULL);
+  found = strstr(desc, needle);
+  tt_ptr_op(found, !=, NULL);
+
+ done:
+  NS_UNMOCK(router_get_my_routerinfo);
+
+  tor_free(desc);
+}
 
 static routerinfo_t *mock_router_get_my_routerinfo_result = NULL;
 
@@ -137,6 +236,6 @@ test_router_check_descriptor_bandwidth_changed(void *arg)
 
 struct testcase_t router_tests[] = {
   ROUTER_TEST(check_descriptor_bandwidth_changed, TT_FORK),
+  ROUTER_TEST(dump_router_to_string_no_bridge_distribution_method, TT_FORK),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_routerkeys.c b/src/test/test_routerkeys.c
index 24b0da1c46..727fa5660f 100644
--- a/src/test/test_routerkeys.c
+++ b/src/test/test_routerkeys.c
@@ -1,24 +1,32 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 #define ROUTER_PRIVATE
-#include "or.h"
-#include "config.h"
-#include "router.h"
-#include "routerkeys.h"
-#include "util.h"
-#include "crypto.h"
-#include "torcert.h"
-#include "test.h"
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routerkeys.h"
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "feature/keymgt/loadkey.h"
+#include "feature/nodelist/torcert.h"
+#include "test/test.h"
 
 #ifdef _WIN32
 /* For mkdir() */
 #include <direct.h>
 #endif
 
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
 static void
 test_routerkeys_write_fingerprint(void *arg)
 {
@@ -92,8 +100,8 @@ test_routerkeys_ed_certs(void *args)
   uint8_t *junk = NULL;
   char *base64 = NULL;
 
-  tt_int_op(0,==,ed25519_keypair_generate(&kp1, 0));
-  tt_int_op(0,==,ed25519_keypair_generate(&kp2, 0));
+  tt_int_op(0,OP_EQ,ed25519_keypair_generate(&kp1, 0));
+  tt_int_op(0,OP_EQ,ed25519_keypair_generate(&kp2, 0));
 
   for (int i = 0; i <= 1; ++i) {
     uint32_t flags = i ? CERT_FLAG_INCLUDE_SIGNING_KEY : 0;
@@ -101,45 +109,45 @@ test_routerkeys_ed_certs(void *args)
     cert[i] = tor_cert_create(&kp1, 5, &kp2.pubkey, now, 10000, flags);
     tt_assert(cert[i]);
 
-    tt_assert(cert[i]->sig_bad == 0);
-    tt_assert(cert[i]->sig_ok == 1);
-    tt_assert(cert[i]->cert_expired == 0);
-    tt_assert(cert[i]->cert_valid == 1);
-    tt_int_op(cert[i]->cert_type, ==, 5);
-    tt_mem_op(cert[i]->signed_key.pubkey, ==, &kp2.pubkey.pubkey, 32);
-    tt_mem_op(cert[i]->signing_key.pubkey, ==, &kp1.pubkey.pubkey, 32);
-    tt_int_op(cert[i]->signing_key_included, ==, i);
+    tt_uint_op(cert[i]->sig_bad, OP_EQ, 0);
+    tt_uint_op(cert[i]->sig_ok, OP_EQ, 1);
+    tt_uint_op(cert[i]->cert_expired, OP_EQ, 0);
+    tt_uint_op(cert[i]->cert_valid, OP_EQ, 1);
+    tt_int_op(cert[i]->cert_type, OP_EQ, 5);
+    tt_mem_op(cert[i]->signed_key.pubkey, OP_EQ, &kp2.pubkey.pubkey, 32);
+    tt_mem_op(cert[i]->signing_key.pubkey, OP_EQ, &kp1.pubkey.pubkey, 32);
+    tt_int_op(cert[i]->signing_key_included, OP_EQ, i);
 
     tt_assert(cert[i]->encoded);
-    tt_int_op(cert[i]->encoded_len, ==, 104 + 36 * i);
-    tt_int_op(cert[i]->encoded[0], ==, 1);
-    tt_int_op(cert[i]->encoded[1], ==, 5);
+    tt_int_op(cert[i]->encoded_len, OP_EQ, 104 + 36 * i);
+    tt_int_op(cert[i]->encoded[0], OP_EQ, 1);
+    tt_int_op(cert[i]->encoded[1], OP_EQ, 5);
 
     parsed_cert[i] = tor_cert_parse(cert[i]->encoded, cert[i]->encoded_len);
     tt_assert(parsed_cert[i]);
-    tt_int_op(cert[i]->encoded_len, ==, parsed_cert[i]->encoded_len);
-    tt_mem_op(cert[i]->encoded, ==, parsed_cert[i]->encoded,
+    tt_int_op(cert[i]->encoded_len, OP_EQ, parsed_cert[i]->encoded_len);
+    tt_mem_op(cert[i]->encoded, OP_EQ, parsed_cert[i]->encoded,
               cert[i]->encoded_len);
-    tt_assert(parsed_cert[i]->sig_bad == 0);
-    tt_assert(parsed_cert[i]->sig_ok == 0);
-    tt_assert(parsed_cert[i]->cert_expired == 0);
-    tt_assert(parsed_cert[i]->cert_valid == 0);
+    tt_uint_op(parsed_cert[i]->sig_bad, OP_EQ, 0);
+    tt_uint_op(parsed_cert[i]->sig_ok, OP_EQ, 0);
+    tt_uint_op(parsed_cert[i]->cert_expired, OP_EQ, 0);
+    tt_uint_op(parsed_cert[i]->cert_valid, OP_EQ, 0);
 
     /* Expired */
     tt_int_op(tor_cert_checksig(parsed_cert[i], &kp1.pubkey, now + 30000),
-              <, 0);
-    tt_assert(parsed_cert[i]->cert_expired == 1);
+              OP_LT, 0);
+    tt_uint_op(parsed_cert[i]->cert_expired, OP_EQ, 1);
     parsed_cert[i]->cert_expired = 0;
 
     /* Wrong key */
-    tt_int_op(tor_cert_checksig(parsed_cert[i], &kp2.pubkey, now), <, 0);
-    tt_assert(parsed_cert[i]->sig_bad== 1);
+    tt_int_op(tor_cert_checksig(parsed_cert[i], &kp2.pubkey, now), OP_LT, 0);
+    tt_uint_op(parsed_cert[i]->sig_bad, OP_EQ, 1);
     parsed_cert[i]->sig_bad = 0;
 
     /* Missing key */
     int ok = tor_cert_checksig(parsed_cert[i], NULL, now);
-    tt_int_op(ok < 0, ==, i == 0);
-    tt_assert(parsed_cert[i]->sig_bad == 0);
+    tt_int_op(ok < 0, OP_EQ, i == 0);
+    tt_uint_op(parsed_cert[i]->sig_bad, OP_EQ, 0);
     tt_assert(parsed_cert[i]->sig_ok == (i != 0));
     tt_assert(parsed_cert[i]->cert_valid == (i != 0));
     parsed_cert[i]->sig_bad = 0;
@@ -147,29 +155,29 @@ test_routerkeys_ed_certs(void *args)
     parsed_cert[i]->cert_valid = 0;
 
     /* Right key */
-    tt_int_op(tor_cert_checksig(parsed_cert[i], &kp1.pubkey, now), ==, 0);
-    tt_assert(parsed_cert[i]->sig_bad == 0);
-    tt_assert(parsed_cert[i]->sig_ok == 1);
-    tt_assert(parsed_cert[i]->cert_expired == 0);
-    tt_assert(parsed_cert[i]->cert_valid == 1);
+    tt_int_op(tor_cert_checksig(parsed_cert[i], &kp1.pubkey, now), OP_EQ, 0);
+    tt_uint_op(parsed_cert[i]->sig_bad, OP_EQ, 0);
+    tt_uint_op(parsed_cert[i]->sig_ok, OP_EQ, 1);
+    tt_uint_op(parsed_cert[i]->cert_expired, OP_EQ, 0);
+    tt_uint_op(parsed_cert[i]->cert_valid, OP_EQ, 1);
   }
 
   /* Now try some junky certs. */
   /* - Truncated */
   nocert = tor_cert_parse(cert[0]->encoded, cert[0]->encoded_len-1);
-  tt_ptr_op(NULL, ==, nocert);
+  tt_ptr_op(NULL, OP_EQ, nocert);
 
   /* - First byte modified */
   cert[0]->encoded[0] = 99;
   nocert = tor_cert_parse(cert[0]->encoded, cert[0]->encoded_len);
-  tt_ptr_op(NULL, ==, nocert);
+  tt_ptr_op(NULL, OP_EQ, nocert);
   cert[0]->encoded[0] = 1;
 
   /* - Extra byte at the end*/
   junk = tor_malloc_zero(cert[0]->encoded_len + 1);
   memcpy(junk, cert[0]->encoded, cert[0]->encoded_len);
   nocert = tor_cert_parse(junk, cert[0]->encoded_len+1);
-  tt_ptr_op(NULL, ==, nocert);
+  tt_ptr_op(NULL, OP_EQ, nocert);
 
   /* - Multiple signing key instances */
   tor_free(junk);
@@ -183,7 +191,7 @@ test_routerkeys_ed_certs(void *args)
   junk[77] = 32; /* extlen */
   junk[78] = 4; /* exttype */
   nocert = tor_cert_parse(junk, 104 + 36 * 2);
-  tt_ptr_op(NULL, ==, nocert);
+  tt_ptr_op(NULL, OP_EQ, nocert);
 
  done:
   tor_cert_free(cert[0]);
@@ -211,11 +219,12 @@ test_routerkeys_ed_key_create(void *arg)
   kp2 = ed_key_new(kp1, INIT_ED_KEY_NEEDCERT, now, 3600, 4, &cert);
   tt_assert(kp2);
   tt_assert(cert);
-  tt_mem_op(&cert->signed_key, ==, &kp2->pubkey, sizeof(ed25519_public_key_t));
+  tt_mem_op(&cert->signed_key, OP_EQ, &kp2->pubkey,
+            sizeof(ed25519_public_key_t));
   tt_assert(! cert->signing_key_included);
 
-  tt_int_op(cert->valid_until, >=, now);
-  tt_int_op(cert->valid_until, <=, now+7200);
+  tt_int_op(cert->valid_until, OP_GE, now);
+  tt_int_op(cert->valid_until, OP_LE, now+7200);
 
   /* Create a new key-including certificate signed by kp1 */
   ed25519_keypair_free(kp2);
@@ -227,8 +236,10 @@ test_routerkeys_ed_key_create(void *arg)
   tt_assert(kp2);
   tt_assert(cert);
   tt_assert(cert->signing_key_included);
-  tt_mem_op(&cert->signed_key, ==, &kp2->pubkey, sizeof(ed25519_public_key_t));
-  tt_mem_op(&cert->signing_key, ==, &kp1->pubkey,sizeof(ed25519_public_key_t));
+  tt_mem_op(&cert->signed_key, OP_EQ, &kp2->pubkey,
+            sizeof(ed25519_public_key_t));
+  tt_mem_op(&cert->signing_key, OP_EQ, &kp1->pubkey,
+            sizeof(ed25519_public_key_t));
 
  done:
   ed25519_keypair_free(kp1);
@@ -252,82 +263,83 @@ test_routerkeys_ed_key_init_basic(void *arg)
   unlink(fname2);
 
   /* Fail to load a key that isn't there. */
-  kp1 = ed_key_init_from_file(fname1, 0, LOG_INFO, NULL, now, 0, 7, &cert);
+  kp1 = ed_key_init_from_file(fname1, 0, LOG_INFO, NULL, now, 0, 7, &cert,
+                              NULL);
   tt_assert(kp1 == NULL);
   tt_assert(cert == NULL);
 
   /* Create the key if requested to do so. */
   kp1 = ed_key_init_from_file(fname1, INIT_ED_KEY_CREATE, LOG_INFO,
-                              NULL, now, 0, 7, &cert);
+                              NULL, now, 0, 7, &cert, NULL);
   tt_assert(kp1 != NULL);
   tt_assert(cert == NULL);
-  tt_int_op(stat(get_fname("test_ed_key_1_cert"), &st), <, 0);
-  tt_int_op(stat(get_fname("test_ed_key_1_secret_key"), &st), ==, 0);
+  tt_int_op(stat(get_fname("test_ed_key_1_cert"), &st), OP_LT, 0);
+  tt_int_op(stat(get_fname("test_ed_key_1_secret_key"), &st), OP_EQ, 0);
 
   /* Fail to load if we say we need a cert */
   kp2 = ed_key_init_from_file(fname1, INIT_ED_KEY_NEEDCERT, LOG_INFO,
-                              NULL, now, 0, 7, &cert);
+                              NULL, now, 0, 7, &cert, NULL);
   tt_assert(kp2 == NULL);
 
   /* Fail to load if we say the wrong key type */
   kp2 = ed_key_init_from_file(fname1, 0, LOG_INFO,
-                              NULL, now, 0, 6, &cert);
+                              NULL, now, 0, 6, &cert, NULL);
   tt_assert(kp2 == NULL);
 
   /* Load successfully if we're not picky, whether we say "create" or not. */
   kp2 = ed_key_init_from_file(fname1, INIT_ED_KEY_CREATE, LOG_INFO,
-                              NULL, now, 0, 7, &cert);
+                              NULL, now, 0, 7, &cert, NULL);
   tt_assert(kp2 != NULL);
   tt_assert(cert == NULL);
-  tt_mem_op(kp1, ==, kp2, sizeof(*kp1));
+  tt_mem_op(kp1, OP_EQ, kp2, sizeof(*kp1));
   ed25519_keypair_free(kp2); kp2 = NULL;
 
   kp2 = ed_key_init_from_file(fname1, 0, LOG_INFO,
-                              NULL, now, 0, 7, &cert);
+                              NULL, now, 0, 7, &cert, NULL);
   tt_assert(kp2 != NULL);
   tt_assert(cert == NULL);
-  tt_mem_op(kp1, ==, kp2, sizeof(*kp1));
+  tt_mem_op(kp1, OP_EQ, kp2, sizeof(*kp1));
   ed25519_keypair_free(kp2); kp2 = NULL;
 
   /* Now create a key with a cert. */
   kp2 = ed_key_init_from_file(fname2, (INIT_ED_KEY_CREATE|
                                        INIT_ED_KEY_NEEDCERT),
-                              LOG_INFO, kp1, now, 7200, 7, &cert);
+                              LOG_INFO, kp1, now, 7200, 7, &cert, NULL);
   tt_assert(kp2 != NULL);
   tt_assert(cert != NULL);
-  tt_mem_op(kp1, !=, kp2, sizeof(*kp1));
-  tt_int_op(stat(get_fname("test_ed_key_2_cert"), &st), ==, 0);
-  tt_int_op(stat(get_fname("test_ed_key_2_secret_key"), &st), ==, 0);
+  tt_mem_op(kp1, OP_NE, kp2, sizeof(*kp1));
+  tt_int_op(stat(get_fname("test_ed_key_2_cert"), &st), OP_EQ, 0);
+  tt_int_op(stat(get_fname("test_ed_key_2_secret_key"), &st), OP_EQ, 0);
 
   tt_assert(cert->cert_valid == 1);
-  tt_mem_op(&cert->signed_key, ==, &kp2->pubkey, 32);
+  tt_mem_op(&cert->signed_key, OP_EQ, &kp2->pubkey, 32);
 
   /* Now verify we can load the cert... */
   kp3 = ed_key_init_from_file(fname2, (INIT_ED_KEY_CREATE|
                                        INIT_ED_KEY_NEEDCERT),
-                              LOG_INFO, kp1, now, 7200, 7, &cert2);
-  tt_mem_op(kp2, ==, kp3, sizeof(*kp2));
-  tt_mem_op(cert2->encoded, ==, cert->encoded, cert->encoded_len);
+                              LOG_INFO, kp1, now, 7200, 7, &cert2, NULL);
+  tt_mem_op(kp2, OP_EQ, kp3, sizeof(*kp2));
+  tt_mem_op(cert2->encoded, OP_EQ, cert->encoded, cert->encoded_len);
   ed25519_keypair_free(kp3); kp3 = NULL;
   tor_cert_free(cert2); cert2 = NULL;
 
   /* ... even without create... */
   kp3 = ed_key_init_from_file(fname2, INIT_ED_KEY_NEEDCERT,
-                              LOG_INFO, kp1, now, 7200, 7, &cert2);
-  tt_mem_op(kp2, ==, kp3, sizeof(*kp2));
-  tt_mem_op(cert2->encoded, ==, cert->encoded, cert->encoded_len);
+                              LOG_INFO, kp1, now, 7200, 7, &cert2, NULL);
+  tt_mem_op(kp2, OP_EQ, kp3, sizeof(*kp2));
+  tt_mem_op(cert2->encoded, OP_EQ, cert->encoded, cert->encoded_len);
   ed25519_keypair_free(kp3); kp3 = NULL;
   tor_cert_free(cert2); cert2 = NULL;
 
   /* ... but that we don't crash or anything if we say we don't want it. */
   kp3 = ed_key_init_from_file(fname2, INIT_ED_KEY_NEEDCERT,
-                              LOG_INFO, kp1, now, 7200, 7, NULL);
-  tt_mem_op(kp2, ==, kp3, sizeof(*kp2));
+                              LOG_INFO, kp1, now, 7200, 7, NULL, NULL);
+  tt_mem_op(kp2, OP_EQ, kp3, sizeof(*kp2));
   ed25519_keypair_free(kp3); kp3 = NULL;
 
   /* Fail if we're told the wrong signing key */
   kp3 = ed_key_init_from_file(fname2, INIT_ED_KEY_NEEDCERT,
-                              LOG_INFO, kp2, now, 7200, 7, &cert2);
+                              LOG_INFO, kp2, now, 7200, 7, &cert2, NULL);
   tt_assert(kp3 == NULL);
   tt_assert(cert2 == NULL);
 
@@ -358,51 +370,52 @@ test_routerkeys_ed_key_init_split(void *arg)
   unlink(fname2);
 
   /* Can't load key that isn't there. */
-  kp1 = ed_key_init_from_file(fname1, flags, LOG_INFO, NULL, now, 0, 7, &cert);
+  kp1 = ed_key_init_from_file(fname1, flags, LOG_INFO, NULL, now, 0, 7, &cert,
+                              NULL);
   tt_assert(kp1 == NULL);
   tt_assert(cert == NULL);
 
   /* Create a split key */
   kp1 = ed_key_init_from_file(fname1, flags|INIT_ED_KEY_CREATE,
-                              LOG_INFO, NULL, now, 0, 7, &cert);
+                              LOG_INFO, NULL, now, 0, 7, &cert, NULL);
   tt_assert(kp1 != NULL);
   tt_assert(cert == NULL);
-  tt_int_op(stat(get_fname("test_ed_key_3_cert"), &st), <, 0);
-  tt_int_op(stat(get_fname("test_ed_key_3_secret_key"), &st), ==, 0);
-  tt_int_op(stat(get_fname("test_ed_key_3_public_key"), &st), ==, 0);
+  tt_int_op(stat(get_fname("test_ed_key_3_cert"), &st), OP_LT, 0);
+  tt_int_op(stat(get_fname("test_ed_key_3_secret_key"), &st), OP_EQ, 0);
+  tt_int_op(stat(get_fname("test_ed_key_3_public_key"), &st), OP_EQ, 0);
 
   /* Load it. */
   kp2 = ed_key_init_from_file(fname1, flags|INIT_ED_KEY_CREATE,
-                              LOG_INFO, NULL, now, 0, 7, &cert);
+                              LOG_INFO, NULL, now, 0, 7, &cert, NULL);
   tt_assert(kp2 != NULL);
   tt_assert(cert == NULL);
-  tt_mem_op(kp1, ==, kp2, sizeof(*kp2));
+  tt_mem_op(kp1, OP_EQ, kp2, sizeof(*kp2));
   ed25519_keypair_free(kp2); kp2 = NULL;
 
   /* Okay, try killing the secret key and loading it. */
   unlink(get_fname("test_ed_key_3_secret_key"));
   kp2 = ed_key_init_from_file(fname1, flags,
-                              LOG_INFO, NULL, now, 0, 7, &cert);
+                              LOG_INFO, NULL, now, 0, 7, &cert, NULL);
   tt_assert(kp2 != NULL);
   tt_assert(cert == NULL);
-  tt_mem_op(&kp1->pubkey, ==, &kp2->pubkey, sizeof(kp2->pubkey));
+  tt_mem_op(&kp1->pubkey, OP_EQ, &kp2->pubkey, sizeof(kp2->pubkey));
   tt_assert(tor_mem_is_zero((char*)kp2->seckey.seckey,
                             sizeof(kp2->seckey.seckey)));
   ed25519_keypair_free(kp2); kp2 = NULL;
 
   /* Even when we're told to "create", don't create if there's a public key */
   kp2 = ed_key_init_from_file(fname1, flags|INIT_ED_KEY_CREATE,
-                              LOG_INFO, NULL, now, 0, 7, &cert);
+                              LOG_INFO, NULL, now, 0, 7, &cert, NULL);
   tt_assert(kp2 != NULL);
   tt_assert(cert == NULL);
-  tt_mem_op(&kp1->pubkey, ==, &kp2->pubkey, sizeof(kp2->pubkey));
+  tt_mem_op(&kp1->pubkey, OP_EQ, &kp2->pubkey, sizeof(kp2->pubkey));
   tt_assert(tor_mem_is_zero((char*)kp2->seckey.seckey,
                             sizeof(kp2->seckey.seckey)));
   ed25519_keypair_free(kp2); kp2 = NULL;
 
   /* Make sure we fail on a tag mismatch, though */
   kp2 = ed_key_init_from_file(fname1, flags,
-                              LOG_INFO, NULL, now, 0, 99, &cert);
+                              LOG_INFO, NULL, now, 0, 99, &cert, NULL);
   tt_assert(kp2 == NULL);
 
  done:
@@ -418,6 +431,7 @@ test_routerkeys_ed_keys_init_all(void *arg)
 {
   (void)arg;
   char *dir = tor_strdup(get_fname("test_ed_keys_init_all"));
+  char *keydir = tor_strdup(get_fname("test_ed_keys_init_all/KEYS"));
   or_options_t *options = tor_malloc_zero(sizeof(or_options_t));
   time_t now = time(NULL);
   ed25519_public_key_t id;
@@ -442,16 +456,17 @@ test_routerkeys_ed_keys_init_all(void *arg)
 
 #ifdef _WIN32
   mkdir(dir);
-  mkdir(get_fname("test_ed_keys_init_all/keys"));
+  mkdir(keydir);
 #else
   mkdir(dir, 0700);
-  mkdir(get_fname("test_ed_keys_init_all/keys"), 0700);
-#endif
+  mkdir(keydir, 0700);
+#endif /* defined(_WIN32) */
 
   options->DataDirectory = dir;
+  options->KeyDirectory = keydir;
 
-  tt_int_op(0, ==, load_ed_keys(options, now));
-  tt_int_op(0, ==, generate_ed_link_cert(options, now));
+  tt_int_op(1, OP_EQ, load_ed_keys(options, now));
+  tt_int_op(0, OP_EQ, generate_ed_link_cert(options, now, 0));
   tt_assert(get_master_identity_key());
   tt_assert(get_master_identity_key());
   tt_assert(get_master_signing_keypair());
@@ -465,21 +480,21 @@ test_routerkeys_ed_keys_init_all(void *arg)
   link_cert = tor_cert_dup(get_current_link_cert_cert());
 
   /* Call load_ed_keys again, but nothing has changed. */
-  tt_int_op(0, ==, load_ed_keys(options, now));
-  tt_int_op(0, ==, generate_ed_link_cert(options, now));
-  tt_mem_op(&id, ==, get_master_identity_key(), sizeof(id));
-  tt_mem_op(&sign, ==, get_master_signing_keypair(), sizeof(sign));
-  tt_mem_op(&auth, ==, get_current_auth_keypair(), sizeof(auth));
+  tt_int_op(0, OP_EQ, load_ed_keys(options, now));
+  tt_int_op(0, OP_EQ, generate_ed_link_cert(options, now, 0));
+  tt_mem_op(&id, OP_EQ, get_master_identity_key(), sizeof(id));
+  tt_mem_op(&sign, OP_EQ, get_master_signing_keypair(), sizeof(sign));
+  tt_mem_op(&auth, OP_EQ, get_current_auth_keypair(), sizeof(auth));
   tt_assert(tor_cert_eq(link_cert, get_current_link_cert_cert()));
 
   /* Force a reload: we make new link/auth keys. */
   routerkeys_free_all();
-  tt_int_op(0, ==, load_ed_keys(options, now));
-  tt_int_op(0, ==, generate_ed_link_cert(options, now));
-  tt_mem_op(&id, ==, get_master_identity_key(), sizeof(id));
-  tt_mem_op(&sign, ==, get_master_signing_keypair(), sizeof(sign));
+  tt_int_op(1, OP_EQ, load_ed_keys(options, now));
+  tt_int_op(0, OP_EQ, generate_ed_link_cert(options, now, 0));
+  tt_mem_op(&id, OP_EQ, get_master_identity_key(), sizeof(id));
+  tt_mem_op(&sign, OP_EQ, get_master_signing_keypair(), sizeof(sign));
   tt_assert(tor_cert_eq(link_cert, get_current_link_cert_cert()));
-  tt_mem_op(&auth, !=, get_current_auth_keypair(), sizeof(auth));
+  tt_mem_op(&auth, OP_NE, get_current_auth_keypair(), sizeof(auth));
   tt_assert(get_master_signing_key_cert());
   tt_assert(get_current_link_cert_cert());
   tt_assert(get_current_auth_key_cert());
@@ -488,12 +503,12 @@ test_routerkeys_ed_keys_init_all(void *arg)
   memcpy(&auth, get_current_auth_keypair(), sizeof(auth));
 
   /* Force a link/auth-key regeneration by advancing time. */
-  tt_int_op(0, ==, load_ed_keys(options, now+3*86400));
-  tt_int_op(0, ==, generate_ed_link_cert(options, now+3*86400));
-  tt_mem_op(&id, ==, get_master_identity_key(), sizeof(id));
-  tt_mem_op(&sign, ==, get_master_signing_keypair(), sizeof(sign));
+  tt_int_op(0, OP_EQ, load_ed_keys(options, now+3*86400));
+  tt_int_op(0, OP_EQ, generate_ed_link_cert(options, now+3*86400, 0));
+  tt_mem_op(&id, OP_EQ, get_master_identity_key(), sizeof(id));
+  tt_mem_op(&sign, OP_EQ, get_master_signing_keypair(), sizeof(sign));
   tt_assert(! tor_cert_eq(link_cert, get_current_link_cert_cert()));
-  tt_mem_op(&auth, !=, get_current_auth_keypair(), sizeof(auth));
+  tt_mem_op(&auth, OP_NE, get_current_auth_keypair(), sizeof(auth));
   tt_assert(get_master_signing_key_cert());
   tt_assert(get_current_link_cert_cert());
   tt_assert(get_current_auth_key_cert());
@@ -502,12 +517,12 @@ test_routerkeys_ed_keys_init_all(void *arg)
   memcpy(&auth, get_current_auth_keypair(), sizeof(auth));
 
   /* Force a signing-key regeneration by advancing time. */
-  tt_int_op(0, ==, load_ed_keys(options, now+100*86400));
-  tt_int_op(0, ==, generate_ed_link_cert(options, now+100*86400));
-  tt_mem_op(&id, ==, get_master_identity_key(), sizeof(id));
-  tt_mem_op(&sign, !=, get_master_signing_keypair(), sizeof(sign));
+  tt_int_op(1, OP_EQ, load_ed_keys(options, now+100*86400));
+  tt_int_op(0, OP_EQ, generate_ed_link_cert(options, now+100*86400, 0));
+  tt_mem_op(&id, OP_EQ, get_master_identity_key(), sizeof(id));
+  tt_mem_op(&sign, OP_NE, get_master_signing_keypair(), sizeof(sign));
   tt_assert(! tor_cert_eq(link_cert, get_current_link_cert_cert()));
-  tt_mem_op(&auth, !=, get_current_auth_keypair(), sizeof(auth));
+  tt_mem_op(&auth, OP_NE, get_current_auth_keypair(), sizeof(auth));
   tt_assert(get_master_signing_key_cert());
   tt_assert(get_current_link_cert_cert());
   tt_assert(get_current_auth_key_cert());
@@ -518,14 +533,14 @@ test_routerkeys_ed_keys_init_all(void *arg)
 
   /* Demonstrate that we can start up with no secret identity key */
   routerkeys_free_all();
-  unlink(get_fname("test_ed_keys_init_all/keys/"
+  unlink(get_fname("test_ed_keys_init_all/KEYS/"
                    "ed25519_master_id_secret_key"));
-  tt_int_op(0, ==, load_ed_keys(options, now));
-  tt_int_op(0, ==, generate_ed_link_cert(options, now));
-  tt_mem_op(&id, ==, get_master_identity_key(), sizeof(id));
-  tt_mem_op(&sign, ==, get_master_signing_keypair(), sizeof(sign));
+  tt_int_op(1, OP_EQ, load_ed_keys(options, now));
+  tt_int_op(0, OP_EQ, generate_ed_link_cert(options, now, 0));
+  tt_mem_op(&id, OP_EQ, get_master_identity_key(), sizeof(id));
+  tt_mem_op(&sign, OP_EQ, get_master_signing_keypair(), sizeof(sign));
   tt_assert(! tor_cert_eq(link_cert, get_current_link_cert_cert()));
-  tt_mem_op(&auth, !=, get_current_auth_keypair(), sizeof(auth));
+  tt_mem_op(&auth, OP_NE, get_current_auth_keypair(), sizeof(auth));
   tt_assert(get_master_signing_key_cert());
   tt_assert(get_current_link_cert_cert());
   tt_assert(get_current_auth_key_cert());
@@ -535,10 +550,11 @@ test_routerkeys_ed_keys_init_all(void *arg)
   log_global_min_severity_ = LOG_ERR; /* Suppress warnings.
                                        * XXX (better way to do this)? */
   routerkeys_free_all();
-  tt_int_op(-1, ==, load_ed_keys(options, now+200*86400));
+  tt_int_op(-1, OP_EQ, load_ed_keys(options, now+200*86400));
 
  done:
   tor_free(dir);
+  tor_free(keydir);
   tor_free(options);
   tor_cert_free(link_cert);
   routerkeys_free_all();
@@ -556,20 +572,20 @@ test_routerkeys_cross_certify_ntor(void *args)
   time_t now = time(NULL);
   int sign;
 
-  tt_int_op(0, ==, ed25519_public_from_base64(&master_key,
+  tt_int_op(0, OP_EQ, ed25519_public_from_base64(&master_key,
                                "IamwritingthesetestsOnARainyAfternoonin2014"));
-  tt_int_op(0, ==, curve25519_keypair_generate(&onion_keys, 0));
+  tt_int_op(0, OP_EQ, curve25519_keypair_generate(&onion_keys, 0));
   cert = make_ntor_onion_key_crosscert(&onion_keys,
                                        &master_key,
                                        now, 10000,
                                        &sign);
   tt_assert(cert);
   tt_assert(sign == 0 || sign == 1);
-  tt_int_op(cert->cert_type, ==, CERT_TYPE_ONION_ID);
-  tt_int_op(1, ==, ed25519_pubkey_eq(&cert->signed_key, &master_key));
-  tt_int_op(0, ==, ed25519_public_key_from_curve25519_public_key(
+  tt_int_op(cert->cert_type, OP_EQ, CERT_TYPE_ONION_ID);
+  tt_int_op(1, OP_EQ, ed25519_pubkey_eq(&cert->signed_key, &master_key));
+  tt_int_op(0, OP_EQ, ed25519_public_key_from_curve25519_public_key(
                                &onion_check_key, &onion_keys.pubkey, sign));
-  tt_int_op(0, ==, tor_cert_checksig(cert, &onion_check_key, now));
+  tt_int_op(0, OP_EQ, tor_cert_checksig(cert, &onion_check_key, now));
 
  done:
   tor_cert_free(cert);
@@ -587,7 +603,7 @@ test_routerkeys_cross_certify_tap(void *args)
   char buf[128];
   int n;
 
-  tt_int_op(0, ==, ed25519_public_from_base64(&master_key,
+  tt_int_op(0, OP_EQ, ed25519_public_from_base64(&master_key,
                                "IAlreadyWroteTestsForRouterdescsUsingTheseX"));
 
   cc = make_tap_onion_key_crosscert(onion_key,
@@ -598,14 +614,14 @@ test_routerkeys_cross_certify_tap(void *args)
 
   n = crypto_pk_public_checksig(onion_key, buf, sizeof(buf),
                                 (char*)cc, cc_len);
-  tt_int_op(n,>,0);
-  tt_int_op(n,==,52);
+  tt_int_op(n,OP_GT,0);
+  tt_int_op(n,OP_EQ,52);
 
   crypto_pk_get_digest(id_key, digest);
-  tt_mem_op(buf,==,digest,20);
-  tt_mem_op(buf+20,==,master_key.pubkey,32);
+  tt_mem_op(buf,OP_EQ,digest,20);
+  tt_mem_op(buf+20,OP_EQ,master_key.pubkey,32);
 
-  tt_int_op(0, ==, check_tap_onion_key_crosscert(cc, cc_len,
+  tt_int_op(0, OP_EQ, check_tap_onion_key_crosscert(cc, cc_len,
                                     onion_key, &master_key, (uint8_t*)digest));
 
  done:
@@ -614,6 +630,66 @@ test_routerkeys_cross_certify_tap(void *args)
   crypto_pk_free(onion_key);
 }
 
+static void
+test_routerkeys_rsa_ed_crosscert(void *arg)
+{
+  (void)arg;
+  ed25519_public_key_t ed;
+  crypto_pk_t *rsa = pk_generate(2);
+
+  uint8_t *cc = NULL;
+  ssize_t cc_len;
+  time_t expires_in = 1470846177;
+
+  tt_int_op(0, OP_EQ, ed25519_public_from_base64(&ed,
+                        "ThisStringCanContainAnythingSoNoKeyHereNowX"));
+  cc_len = tor_make_rsa_ed25519_crosscert(&ed, rsa, expires_in, &cc);
+
+  tt_int_op(cc_len, OP_GT, 0);
+  tt_int_op(cc_len, OP_GT, 37); /* key, expires, siglen */
+  tt_mem_op(cc, OP_EQ, ed.pubkey, 32);
+  time_t expires_out = 3600 * ntohl(get_uint32(cc+32));
+  tt_int_op(expires_out, OP_GE, expires_in);
+  tt_int_op(expires_out, OP_LE, expires_in + 3600);
+
+  tt_int_op(cc_len, OP_EQ, 37 + get_uint8(cc+36));
+
+  tt_int_op(0, OP_EQ, rsa_ed25519_crosscert_check(cc, cc_len, rsa, &ed,
+                                                  expires_in - 10));
+
+  /* Now try after it has expired */
+  tt_int_op(-4, OP_EQ, rsa_ed25519_crosscert_check(cc, cc_len, rsa, &ed,
+                                                  expires_out + 1));
+
+  /* Truncated object */
+  tt_int_op(-2, OP_EQ, rsa_ed25519_crosscert_check(cc, cc_len - 2, rsa, &ed,
+                                                  expires_in - 10));
+
+  /* Key not as expected */
+  cc[0] ^= 3;
+  tt_int_op(-3, OP_EQ, rsa_ed25519_crosscert_check(cc, cc_len, rsa, &ed,
+                                                  expires_in - 10));
+  cc[0] ^= 3;
+
+  /* Bad signature */
+  cc[40] ^= 3;
+  tt_int_op(-5, OP_EQ, rsa_ed25519_crosscert_check(cc, cc_len, rsa, &ed,
+                                                   expires_in - 10));
+  cc[40] ^= 3;
+
+  /* Signature of wrong data */
+  cc[0] ^= 3;
+  ed.pubkey[0] ^= 3;
+  tt_int_op(-6, OP_EQ, rsa_ed25519_crosscert_check(cc, cc_len, rsa, &ed,
+                                                  expires_in - 10));
+  cc[0] ^= 3;
+  ed.pubkey[0] ^= 3;
+
+ done:
+  crypto_pk_free(rsa);
+  tor_free(cc);
+}
+
 #define TEST(name, flags)                                       \
   { #name , test_routerkeys_ ## name, (flags), NULL, NULL }
 
@@ -626,6 +702,6 @@ struct testcase_t routerkeys_tests[] = {
   TEST(ed_keys_init_all, TT_FORK),
   TEST(cross_certify_ntor, 0),
   TEST(cross_certify_tap, 0),
+  TEST(rsa_ed_crosscert, 0),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c
index 088bd257c3..95c9176faa 100644
--- a/src/test/test_routerlist.c
+++ b/src/test/test_routerlist.c
@@ -1,32 +1,58 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 #include <math.h>
 #include <time.h>
 
+#define CONNECTION_PRIVATE
+#define DIRCLIENT_PRIVATE
 #define DIRVOTE_PRIVATE
+#define ENTRYNODES_PRIVATE
+#define HIBERNATE_PRIVATE
 #define NETWORKSTATUS_PRIVATE
 #define ROUTERLIST_PRIVATE
+#define NODE_SELECT_PRIVATE
 #define TOR_UNIT_TESTING
-#include "or.h"
-#include "config.h"
-#include "connection.h"
-#include "container.h"
-#include "directory.h"
-#include "dirvote.h"
-#include "microdesc.h"
-#include "networkstatus.h"
-#include "nodelist.h"
-#include "policies.h"
-#include "router.h"
-#include "routerlist.h"
-#include "routerparse.h"
-#include "shared_random.h"
-#include "test.h"
-#include "test_dir_common.h"
-
-void construct_consensus(char **consensus_text_md);
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "feature/control/control.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/client/entrynodes.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "core/or/policies.h"
+#include "feature/relay/router.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/node_select.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/dirparse/authcert_parse.h"
+#include "feature/dirparse/ns_parse.h"
+#include "feature/dirauth/shared_random.h"
+#include "app/config/statefile.h"
+
+#include "feature/nodelist/authority_cert_st.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "app/config/or_state_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+#include "lib/encoding/confline.h"
+#include "lib/container/buffers.h"
+
+#include "test/test.h"
+#include "test/test_dir_common.h"
+#include "test/log_test_helpers.h"
+
+void construct_consensus(char **consensus_text_md, time_t now);
 
 static authority_cert_t *mock_cert;
 
@@ -116,7 +142,7 @@ test_routerlist_launch_descriptor_downloads(void *arg)
   MOCK(initiate_descriptor_downloads, mock_initiate_descriptor_downloads);
   launch_descriptor_downloads(DIR_PURPOSE_FETCH_MICRODESC, downloadable,
                               NULL, now);
-  tt_int_op(3, ==, count);
+  tt_int_op(3, OP_EQ, count);
   UNMOCK(initiate_descriptor_downloads);
 
  done:
@@ -125,7 +151,7 @@ test_routerlist_launch_descriptor_downloads(void *arg)
 }
 
 void
-construct_consensus(char **consensus_text_md)
+construct_consensus(char **consensus_text_md, time_t now)
 {
   networkstatus_t *vote = NULL;
   networkstatus_t *v1 = NULL, *v2 = NULL, *v3 = NULL;
@@ -133,7 +159,6 @@ construct_consensus(char **consensus_text_md)
   authority_cert_t *cert1=NULL, *cert2=NULL, *cert3=NULL;
   crypto_pk_t *sign_skey_1=NULL, *sign_skey_2=NULL, *sign_skey_3=NULL;
   crypto_pk_t *sign_skey_leg=NULL;
-  time_t now = time(NULL);
   smartlist_t *votes = NULL;
   int n_vrs;
 
@@ -147,24 +172,24 @@ construct_consensus(char **consensus_text_md)
                               &v1, &n_vrs, now, 1);
   networkstatus_vote_free(vote);
   tt_assert(v1);
-  tt_int_op(n_vrs, ==, 4);
-  tt_int_op(smartlist_len(v1->routerstatus_list), ==, 4);
+  tt_int_op(n_vrs, OP_EQ, 4);
+  tt_int_op(smartlist_len(v1->routerstatus_list), OP_EQ, 4);
 
   dir_common_construct_vote_2(&vote, cert2, sign_skey_2,
                               &dir_common_gen_routerstatus_for_v3ns,
                               &v2, &n_vrs, now, 1);
   networkstatus_vote_free(vote);
   tt_assert(v2);
-  tt_int_op(n_vrs, ==, 4);
-  tt_int_op(smartlist_len(v2->routerstatus_list), ==, 4);
+  tt_int_op(n_vrs, OP_EQ, 4);
+  tt_int_op(smartlist_len(v2->routerstatus_list), OP_EQ, 4);
 
   dir_common_construct_vote_3(&vote, cert3, sign_skey_3,
                               &dir_common_gen_routerstatus_for_v3ns,
                               &v3, &n_vrs, now, 1);
 
   tt_assert(v3);
-  tt_int_op(n_vrs, ==, 4);
-  tt_int_op(smartlist_len(v3->routerstatus_list), ==, 4);
+  tt_int_op(n_vrs, OP_EQ, 4);
+  tt_int_op(smartlist_len(v3->routerstatus_list), OP_EQ, 4);
   networkstatus_vote_free(vote);
   votes = smartlist_new();
   smartlist_add(votes, v1);
@@ -246,16 +271,16 @@ test_router_pick_directory_server_impl(void *arg)
 
   /* No consensus available, fail early */
   rs = router_pick_directory_server_impl(V3_DIRINFO, (const int) 0, NULL);
-  tt_assert(rs == NULL);
+  tt_ptr_op(rs, OP_EQ, NULL);
 
-  construct_consensus(&consensus_text_md);
+  construct_consensus(&consensus_text_md, now);
   tt_assert(consensus_text_md);
   con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL,
                                                 NS_TYPE_CONSENSUS);
   tt_assert(con_md);
-  tt_int_op(con_md->flavor,==, FLAV_MICRODESC);
+  tt_int_op(con_md->flavor,OP_EQ, FLAV_MICRODESC);
   tt_assert(con_md->routerstatus_list);
-  tt_int_op(smartlist_len(con_md->routerstatus_list), ==, 3);
+  tt_int_op(smartlist_len(con_md->routerstatus_list), OP_EQ, 3);
   tt_assert(!networkstatus_set_current_consensus_from_ns(con_md,
                                                  "microdesc"));
 
@@ -286,7 +311,7 @@ test_router_pick_directory_server_impl(void *arg)
   rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
   /* We should not fail now we have a consensus and routerstatus_list
    * and nodelist are populated. */
-  tt_assert(rs != NULL);
+  tt_ptr_op(rs, OP_NE, NULL);
 
   /* Manipulate the nodes so we get the dir server we expect */
   router1_id = tor_malloc(DIGEST_LEN);
@@ -305,7 +330,7 @@ test_router_pick_directory_server_impl(void *arg)
   node_router1->is_running = 0;
   node_router3->is_running = 0;
   rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
+  tt_ptr_op(rs, OP_NE, NULL);
   tt_assert(tor_memeq(rs->identity_digest, router2_id, DIGEST_LEN));
   rs = NULL;
   node_router1->is_running = 1;
@@ -318,7 +343,7 @@ test_router_pick_directory_server_impl(void *arg)
   node_router1->rs->dir_port = 0;
   node_router3->rs->dir_port = 0;
   rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
+  tt_ptr_op(rs, OP_NE, NULL);
   tt_assert(tor_memeq(rs->identity_digest, router2_id, DIGEST_LEN));
   rs = NULL;
   node_router1->rs->is_v2_dir = 1;
@@ -329,42 +354,18 @@ test_router_pick_directory_server_impl(void *arg)
   node_router1->is_valid = 0;
   node_router3->is_valid = 0;
   rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
+  tt_ptr_op(rs, OP_NE, NULL);
   tt_assert(tor_memeq(rs->identity_digest, router2_id, DIGEST_LEN));
   rs = NULL;
   node_router1->is_valid = 1;
   node_router3->is_valid = 1;
 
-  flags |= PDS_FOR_GUARD;
-  node_router1->using_as_guard = 1;
-  node_router2->using_as_guard = 1;
-  node_router3->using_as_guard = 1;
-  rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs == NULL);
-  node_router1->using_as_guard = 0;
-  rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
-  tt_assert(tor_memeq(rs->identity_digest, router1_id, DIGEST_LEN));
-  rs = NULL;
-  node_router2->using_as_guard = 0;
-  node_router3->using_as_guard = 0;
-
-  /* One not valid, one guard. This should leave one remaining */
-  node_router1->is_valid = 0;
-  node_router2->using_as_guard = 1;
-  rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
-  tt_assert(tor_memeq(rs->identity_digest, router3_id, DIGEST_LEN));
-  rs = NULL;
-  node_router1->is_valid = 1;
-  node_router2->using_as_guard = 0;
-
   /* Manipulate overloaded */
 
   node_router2->rs->last_dir_503_at = now;
   node_router3->rs->last_dir_503_at = now;
   rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
+  tt_ptr_op(rs, OP_NE, NULL);
   tt_assert(tor_memeq(rs->identity_digest, router1_id, DIGEST_LEN));
   node_router2->rs->last_dir_503_at = 0;
   node_router3->rs->last_dir_503_at = 0;
@@ -381,13 +382,13 @@ test_router_pick_directory_server_impl(void *arg)
   node_router2->rs->or_port = 443;
   node_router3->rs->or_port = 442;
   rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
+  tt_ptr_op(rs, OP_NE, NULL);
   tt_assert(tor_memeq(rs->identity_digest, router3_id, DIGEST_LEN));
   node_router1->rs->or_port = 442;
   node_router2->rs->or_port = 443;
   node_router3->rs->or_port = 444;
   rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
+  tt_ptr_op(rs, OP_NE, NULL);
   tt_assert(tor_memeq(rs->identity_digest, router1_id, DIGEST_LEN));
 
   /* Fascist firewall and overloaded */
@@ -396,7 +397,7 @@ test_router_pick_directory_server_impl(void *arg)
   node_router3->rs->or_port = 442;
   node_router3->rs->last_dir_503_at = now;
   rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
+  tt_ptr_op(rs, OP_NE, NULL);
   tt_assert(tor_memeq(rs->identity_digest, router1_id, DIGEST_LEN));
   node_router3->rs->last_dir_503_at = 0;
 
@@ -414,12 +415,13 @@ test_router_pick_directory_server_impl(void *arg)
   node_router3->rs->dir_port = 81;
   node_router1->rs->last_dir_503_at = now;
   rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
+  tt_ptr_op(rs, OP_NE, NULL);
   tt_assert(tor_memeq(rs->identity_digest, router1_id, DIGEST_LEN));
   node_router1->rs->last_dir_503_at = 0;
 
  done:
   UNMOCK(usable_consensus_flavor);
+
   if (router1_id)
     tor_free(router1_id);
   if (router2_id)
@@ -433,6 +435,112 @@ test_router_pick_directory_server_impl(void *arg)
   networkstatus_vote_free(con_md);
 }
 
+static or_state_t *dummy_state = NULL;
+static or_state_t *
+get_or_state_replacement(void)
+{
+  return dummy_state;
+}
+
+static void
+mock_directory_initiate_request(directory_request_t *req)
+{
+  (void)req;
+  return;
+}
+
+static circuit_guard_state_t *
+mock_circuit_guard_state_new(entry_guard_t *guard, unsigned state,
+                        entry_guard_restriction_t *rst)
+{
+  (void) guard;
+  (void) state;
+  (void) rst;
+  return NULL;
+}
+
+/** Test that we will use our directory guards to fetch mds even if we don't
+ *  have any dirinfo (tests bug #23862). */
+static void
+test_directory_guard_fetch_with_no_dirinfo(void *arg)
+{
+  int retval;
+  char *consensus_text_md = NULL;
+  or_options_t *options = get_options_mutable();
+  time_t now = time(NULL);
+
+  (void) arg;
+
+  hibernate_set_state_for_testing_(HIBERNATE_STATE_LIVE);
+
+  /* Initialize the SRV subsystem */
+  MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
+  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  sr_init(0);
+  UNMOCK(get_my_v3_authority_cert);
+
+  /* Initialize the entry node configuration from the ticket */
+  options->UseEntryGuards = 1;
+  options->StrictNodes = 1;
+  get_options_mutable()->EntryNodes = routerset_new();
+  routerset_parse(get_options_mutable()->EntryNodes,
+                  "2121212121212121212121212121212121212121", "foo");
+
+  /* Mock some functions */
+  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  MOCK(get_or_state, get_or_state_replacement);
+  MOCK(directory_initiate_request, mock_directory_initiate_request);
+  /* we need to mock this one to avoid memleaks */
+  MOCK(circuit_guard_state_new, mock_circuit_guard_state_new);
+
+  /* Call guards_update_all() to simulate loading our state file (see
+   * entry_guards_load_guards_from_state() and ticket #23989). */
+  guards_update_all();
+
+  /* Test logic: Simulate the arrival of a new consensus when we have no
+   * dirinfo at all. Tor will need to fetch the mds from the consensus. Make
+   * sure that Tor will use the specified entry guard instead of relying on the
+   * fallback directories. */
+
+  /* Fixup the dirconn that will deliver the consensus */
+  dir_connection_t *conn = dir_connection_new(AF_INET);
+  tor_addr_from_ipv4h(&conn->base_.addr, 0x7f000001);
+  conn->base_.port = 8800;
+  TO_CONN(conn)->address = tor_strdup("127.0.0.1");
+  conn->base_.purpose = DIR_PURPOSE_FETCH_CONSENSUS;
+  conn->requested_resource = tor_strdup("ns");
+
+  /* Construct a consensus */
+  construct_consensus(&consensus_text_md, now);
+  tt_assert(consensus_text_md);
+
+  /* Place the consensus in the dirconn */
+  response_handler_args_t args;
+  memset(&args, 0, sizeof(response_handler_args_t));
+  args.status_code = 200;
+  args.body = consensus_text_md;
+  args.body_len = strlen(consensus_text_md);
+
+  /* Update approx time so that the consensus is considered live */
+  update_approx_time(now+1010);
+
+  setup_capture_of_logs(LOG_DEBUG);
+
+  /* Now handle the consensus */
+  retval = handle_response_fetch_consensus(conn, &args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Make sure that our primary guard was chosen */
+  expect_log_msg_containing("Selected primary guard router3");
+
+ done:
+  tor_free(consensus_text_md);
+  tor_free(dummy_state);
+  connection_free_minimal(TO_CONN(conn));
+  entry_guards_free_all();
+  teardown_capture_of_logs();
+}
+
 static connection_t *mocked_connection = NULL;
 
 /* Mock connection_get_by_type_addr_port_purpose by returning
@@ -471,27 +579,27 @@ test_routerlist_router_is_already_dir_fetching(void *arg)
 
   /* Test that we never get 1 from a NULL connection */
   mocked_connection = NULL;
-  tt_assert(router_is_already_dir_fetching(&test_ap, 1, 1) == 0);
-  tt_assert(router_is_already_dir_fetching(&test_ap, 1, 0) == 0);
-  tt_assert(router_is_already_dir_fetching(&test_ap, 0, 1) == 0);
+  tt_int_op(router_is_already_dir_fetching(&test_ap, 1, 1), OP_EQ, 0);
+  tt_int_op(router_is_already_dir_fetching(&test_ap, 1, 0), OP_EQ, 0);
+  tt_int_op(router_is_already_dir_fetching(&test_ap, 0, 1), OP_EQ, 0);
   /* We always expect 0 in these cases */
-  tt_assert(router_is_already_dir_fetching(&test_ap, 0, 0) == 0);
-  tt_assert(router_is_already_dir_fetching(NULL, 1, 1) == 0);
-  tt_assert(router_is_already_dir_fetching(&null_addr_ap, 1, 1) == 0);
-  tt_assert(router_is_already_dir_fetching(&zero_port_ap, 1, 1) == 0);
+  tt_int_op(router_is_already_dir_fetching(&test_ap, 0, 0), OP_EQ, 0);
+  tt_int_op(router_is_already_dir_fetching(NULL, 1, 1), OP_EQ, 0);
+  tt_int_op(router_is_already_dir_fetching(&null_addr_ap, 1, 1), OP_EQ, 0);
+  tt_int_op(router_is_already_dir_fetching(&zero_port_ap, 1, 1), OP_EQ, 0);
 
   /* Test that we get 1 with a connection in the appropriate circumstances */
   mocked_connection = connection_new(CONN_TYPE_DIR, AF_INET);
-  tt_assert(router_is_already_dir_fetching(&test_ap, 1, 1) == 1);
-  tt_assert(router_is_already_dir_fetching(&test_ap, 1, 0) == 1);
-  tt_assert(router_is_already_dir_fetching(&test_ap, 0, 1) == 1);
+  tt_int_op(router_is_already_dir_fetching(&test_ap, 1, 1), OP_EQ, 1);
+  tt_int_op(router_is_already_dir_fetching(&test_ap, 1, 0), OP_EQ, 1);
+  tt_int_op(router_is_already_dir_fetching(&test_ap, 0, 1), OP_EQ, 1);
 
   /* Test that we get 0 even with a connection in the appropriate
    * circumstances */
-  tt_assert(router_is_already_dir_fetching(&test_ap, 0, 0) == 0);
-  tt_assert(router_is_already_dir_fetching(NULL, 1, 1) == 0);
-  tt_assert(router_is_already_dir_fetching(&null_addr_ap, 1, 1) == 0);
-  tt_assert(router_is_already_dir_fetching(&zero_port_ap, 1, 1) == 0);
+  tt_int_op(router_is_already_dir_fetching(&test_ap, 0, 0), OP_EQ, 0);
+  tt_int_op(router_is_already_dir_fetching(NULL, 1, 1), OP_EQ, 0);
+  tt_int_op(router_is_already_dir_fetching(&null_addr_ap, 1, 1), OP_EQ, 0);
+  tt_int_op(router_is_already_dir_fetching(&zero_port_ap, 1, 1), OP_EQ, 0);
 
  done:
   /* If a connection is never set up, connection_free chokes on it. */
@@ -506,16 +614,180 @@ test_routerlist_router_is_already_dir_fetching(void *arg)
 #undef TEST_ADDR_STR
 #undef TEST_DIR_PORT
 
+static long mock_apparent_skew = 0;
+
+/** Store apparent_skew and assert that the other arguments are as
+ * expected. */
+static void
+mock_clock_skew_warning(const connection_t *conn, long apparent_skew,
+                        int trusted, log_domain_mask_t domain,
+                        const char *received, const char *source)
+{
+  (void)conn;
+  mock_apparent_skew = apparent_skew;
+  tt_int_op(trusted, OP_EQ, 1);
+  tt_int_op(domain, OP_EQ, LD_GENERAL);
+  tt_str_op(received, OP_EQ, "microdesc flavor consensus");
+  tt_str_op(source, OP_EQ, "CONSENSUS");
+ done:
+  ;
+}
+
+/** Do common setup for test_timely_consensus() and
+ * test_early_consensus().  Call networkstatus_set_current_consensus()
+ * on a constructed consensus and with an appropriately-modified
+ * approx_time.  Callers expect presence or absence of appropriate log
+ * messages and control events. */
+static int
+test_skew_common(void *arg, time_t now, unsigned long *offset)
+{
+  char *consensus = NULL;
+  int retval = 0;
+
+  *offset = strtoul(arg, NULL, 10);
+
+  /* Initialize the SRV subsystem */
+  MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
+  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  sr_init(0);
+  UNMOCK(get_my_v3_authority_cert);
+
+  construct_consensus(&consensus, now);
+  tt_assert(consensus);
+
+  update_approx_time(now + *offset);
+
+  mock_apparent_skew = 0;
+  /* Caller will call UNMOCK() */
+  MOCK(clock_skew_warning, mock_clock_skew_warning);
+  /* Caller will call teardown_capture_of_logs() */
+  setup_capture_of_logs(LOG_WARN);
+  retval = networkstatus_set_current_consensus(consensus, "microdesc", 0,
+                                               NULL);
+
+ done:
+  tor_free(consensus);
+  return retval;
+}
+
+/** Test non-early consensus */
+static void
+test_timely_consensus(void *arg)
+{
+  time_t now = time(NULL);
+  unsigned long offset = 0;
+  int retval = 0;
+
+  retval = test_skew_common(arg, now, &offset);
+  (void)offset;
+  expect_no_log_msg_containing("behind the time published in the consensus");
+  tt_int_op(retval, OP_EQ, 0);
+  tt_int_op(mock_apparent_skew, OP_EQ, 0);
+ done:
+  teardown_capture_of_logs();
+  UNMOCK(clock_skew_warning);
+}
+
+/** Test early consensus  */
+static void
+test_early_consensus(void *arg)
+{
+  time_t now = time(NULL);
+  unsigned long offset = 0;
+  int retval = 0;
+
+  retval = test_skew_common(arg, now, &offset);
+  /* Can't use expect_single_log_msg() because of unrecognized authorities */
+  expect_log_msg_containing("behind the time published in the consensus");
+  tt_int_op(retval, OP_EQ, 0);
+  /* This depends on construct_consensus() setting valid_after=now+1000 */
+  tt_int_op(mock_apparent_skew, OP_EQ, offset - 1000);
+ done:
+  teardown_capture_of_logs();
+  UNMOCK(clock_skew_warning);
+}
+
+/** Test warn_early_consensus(), expecting no warning  */
+static void
+test_warn_early_consensus_no(const networkstatus_t *c, time_t now,
+                             long offset)
+{
+  mock_apparent_skew = 0;
+  setup_capture_of_logs(LOG_WARN);
+  warn_early_consensus(c, "microdesc", now + offset);
+  expect_no_log_msg_containing("behind the time published in the consensus");
+  tt_int_op(mock_apparent_skew, OP_EQ, 0);
+ done:
+  teardown_capture_of_logs();
+}
+
+/** Test warn_early_consensus(), expecting a warning */
+static void
+test_warn_early_consensus_yes(const networkstatus_t *c, time_t now,
+                              long offset)
+{
+  mock_apparent_skew = 0;
+  setup_capture_of_logs(LOG_WARN);
+  warn_early_consensus(c, "microdesc", now + offset);
+  /* Can't use expect_single_log_msg() because of unrecognized authorities */
+  expect_log_msg_containing("behind the time published in the consensus");
+  tt_int_op(mock_apparent_skew, OP_EQ, offset);
+ done:
+  teardown_capture_of_logs();
+}
+
+/**
+ * Test warn_early_consensus() directly, checking both the non-warning
+ * case (consensus is not early) and the warning case (consensus is
+ * early).  Depends on EARLY_CONSENSUS_NOTICE_SKEW=60.
+ */
+static void
+test_warn_early_consensus(void *arg)
+{
+  networkstatus_t *c = NULL;
+  time_t now = time(NULL);
+
+  (void)arg;
+  c = tor_malloc_zero(sizeof *c);
+  c->valid_after = now;
+  c->dist_seconds = 300;
+  mock_apparent_skew = 0;
+  MOCK(clock_skew_warning, mock_clock_skew_warning);
+  test_warn_early_consensus_no(c, now, 60);
+  test_warn_early_consensus_no(c, now, 0);
+  test_warn_early_consensus_no(c, now, -60);
+  test_warn_early_consensus_no(c, now, -360);
+  test_warn_early_consensus_yes(c, now, -361);
+  test_warn_early_consensus_yes(c, now, -600);
+  UNMOCK(clock_skew_warning);
+  tor_free(c);
+}
+
 #define NODE(name, flags) \
   { #name, test_routerlist_##name, (flags), NULL, NULL }
 #define ROUTER(name,flags) \
   { #name, test_router_##name, (flags), NULL, NULL }
 
+#define TIMELY(name, arg)                                     \
+  { name, test_timely_consensus, TT_FORK, &passthrough_setup, \
+    (char *)(arg) }
+#define EARLY(name, arg) \
+  { name, test_early_consensus, TT_FORK, &passthrough_setup, \
+    (char *)(arg) }
+
 struct testcase_t routerlist_tests[] = {
   NODE(initiate_descriptor_downloads, 0),
   NODE(launch_descriptor_downloads, 0),
   NODE(router_is_already_dir_fetching, TT_FORK),
   ROUTER(pick_directory_server_impl, TT_FORK),
+  { "directory_guard_fetch_with_no_dirinfo",
+    test_directory_guard_fetch_with_no_dirinfo, TT_FORK, NULL, NULL },
+  /* These depend on construct_consensus() setting
+   * valid_after=now+1000 and dist_seconds=250 */
+  TIMELY("timely_consensus1", "1010"),
+  TIMELY("timely_consensus2", "1000"),
+  TIMELY("timely_consensus3", "690"),
+  EARLY("early_consensus1", "689"),
+  { "warn_early_consensus", test_warn_early_consensus, 0, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_routerset.c b/src/test/test_routerset.c
index 1b526d430b..c45f0e1595 100644
--- a/src/test/test_routerset.c
+++ b/src/test/test_routerset.c
@@ -1,12 +1,22 @@
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
 #define ROUTERSET_PRIVATE
 
-#include "or.h"
-#include "geoip.h"
-#include "routerset.h"
-#include "routerparse.h"
-#include "policies.h"
-#include "nodelist.h"
-#include "test.h"
+#include "core/or/or.h"
+#include "core/or/policies.h"
+#include "feature/dirparse/policy_parse.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerset.h"
+#include "lib/geoip/geoip.h"
+
+#include "core/or/addr_policy_st.h"
+#include "core/or/extend_info_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+#include "test/test.h"
 
 #define NS_MODULE routerset
 
@@ -623,7 +633,7 @@ NS(test_main)(void *arg)
   (void)arg;
 
   tgt = routerset_new();
-  smartlist_add(src->list, tor_strdup("{xx}"));
+  smartlist_add_strdup(src->list, "{xx}");
   routerset_union(tgt, src);
 
   tt_int_op(smartlist_len(tgt->list), OP_NE, 0);
@@ -696,7 +706,7 @@ NS(test_main)(void *arg)
 static void
 NS(test_main)(void *arg)
 {
-  const routerset_t *set;
+  routerset_t *set;
   int needs_geoip;
   (void)arg;
 
@@ -706,14 +716,14 @@ NS(test_main)(void *arg)
 
   set = routerset_new();
   needs_geoip = routerset_needs_geoip(set);
-  routerset_free((routerset_t *)set);
+  routerset_free(set);
   tt_int_op(needs_geoip, OP_EQ, 0);
   set = NULL;
 
   set = routerset_new();
   smartlist_add(set->country_names, tor_strndup("xx", 2));
   needs_geoip = routerset_needs_geoip(set);
-  routerset_free((routerset_t *)set);
+  routerset_free(set);
   set = NULL;
   tt_int_op(needs_geoip, OP_NE, 0);
 
@@ -745,7 +755,7 @@ NS(test_main)(void *arg)
   tt_int_op(is_empty, OP_NE, 0);
 
   set = routerset_new();
-  smartlist_add(set->list, tor_strdup("{xx}"));
+  smartlist_add_strdup(set->list, "{xx}");
   is_empty = routerset_is_empty(set);
   routerset_free(set);
   set = NULL;
@@ -1486,6 +1496,7 @@ NS(test_main)(void *arg)
   int r;
   (void)arg;
 
+  memset(&NS(mock_node), 0, sizeof(NS(mock_node)));
   NS(mock_node).ri = NULL;
   NS(mock_node).rs = NULL;
 
@@ -1519,6 +1530,7 @@ NS(test_main)(void *arg)
 
   strncpy(rs.nickname, nickname, sizeof(rs.nickname) - 1);
   rs.nickname[sizeof(rs.nickname) - 1] = '\0';
+  memset(&NS(mock_node), 0, sizeof(NS(mock_node)));
   NS(mock_node).ri = NULL;
   NS(mock_node).rs = &rs;
 
@@ -1550,6 +1562,7 @@ NS(test_main)(void *arg)
   strmap_set_lc(set->names, nickname, (void *)1);
 
   ri.nickname = (char *)nickname;
+  memset(&mock_node, 0, sizeof(mock_node));
   mock_node.ri = &ri;
   mock_node.rs = NULL;
 
@@ -1602,7 +1615,7 @@ NS(test_main)(void *arg)
  */
 
 NS_DECL(const node_t *, node_get_by_nickname,
-    (const char *nickname, int warn_if_unused));
+    (const char *nickname, unsigned flags));
 static const char *NS(mock_nickname);
 
 static void
@@ -1616,7 +1629,7 @@ NS(test_main)(void *arg)
   NS_MOCK(node_get_by_nickname);
 
   NS(mock_nickname) = "foo";
-  smartlist_add(set->list, tor_strdup(NS(mock_nickname)));
+  smartlist_add_strdup(set->list, NS(mock_nickname));
 
   routerset_get_all_nodes(out, set, NULL, 0);
   out_len = smartlist_len(out);
@@ -1632,11 +1645,11 @@ NS(test_main)(void *arg)
 }
 
 const node_t *
-NS(node_get_by_nickname)(const char *nickname, int warn_if_unused)
+NS(node_get_by_nickname)(const char *nickname, unsigned flags)
 {
   CALLED(node_get_by_nickname)++;
   tt_str_op(nickname, OP_EQ, NS(mock_nickname));
-  tt_int_op(warn_if_unused, OP_EQ, 1);
+  tt_uint_op(flags, OP_EQ, 0);
 
   done:
     return NULL;
@@ -1651,7 +1664,7 @@ NS(node_get_by_nickname)(const char *nickname, int warn_if_unused)
  */
 
 NS_DECL(const node_t *, node_get_by_nickname,
-    (const char *nickname, int warn_if_unused));
+    (const char *nickname, unsigned flags));
 static const char *NS(mock_nickname);
 static node_t NS(mock_node);
 
@@ -1667,7 +1680,7 @@ NS(test_main)(void *arg)
 
   NS(mock_node).is_running = 0;
   NS(mock_nickname) = "foo";
-  smartlist_add(set->list, tor_strdup(NS(mock_nickname)));
+  smartlist_add_strdup(set->list, NS(mock_nickname));
 
   routerset_get_all_nodes(out, set, NULL, 1);
   out_len = smartlist_len(out);
@@ -1683,11 +1696,11 @@ NS(test_main)(void *arg)
 }
 
 const node_t *
-NS(node_get_by_nickname)(const char *nickname, int warn_if_unused)
+NS(node_get_by_nickname)(const char *nickname, unsigned flags)
 {
   CALLED(node_get_by_nickname)++;
   tt_str_op(nickname, OP_EQ, NS(mock_nickname));
-  tt_int_op(warn_if_unused, OP_EQ, 1);
+  tt_int_op(flags, OP_EQ, 0);
 
   done:
     return &NS(mock_node);
@@ -1701,7 +1714,7 @@ NS(node_get_by_nickname)(const char *nickname, int warn_if_unused)
  */
 
 NS_DECL(const node_t *, node_get_by_nickname,
-    (const char *nickname, int warn_if_unused));
+    (const char *nickname, unsigned flags));
 static char *NS(mock_nickname);
 static node_t NS(mock_node);
 
@@ -1735,11 +1748,11 @@ NS(test_main)(void *arg)
 }
 
 const node_t *
-NS(node_get_by_nickname)(const char *nickname, int warn_if_unused)
+NS(node_get_by_nickname)(const char *nickname, unsigned flags)
 {
   CALLED(node_get_by_nickname)++;
   tt_str_op(nickname, OP_EQ, NS(mock_nickname));
-  tt_int_op(warn_if_unused, OP_EQ, 1);
+  tt_int_op(flags, OP_EQ, 0);
 
   done:
     return &NS(mock_node);
@@ -1766,7 +1779,7 @@ NS(test_main)(void *arg)
 
   NS_MOCK(nodelist_get_list);
 
-  smartlist_add(set->country_names, tor_strdup("{xx}"));
+  smartlist_add_strdup(set->country_names, "{xx}");
   NS(mock_smartlist) = smartlist_new();
 
   routerset_get_all_nodes(out, set, NULL, 1);
@@ -1813,7 +1826,7 @@ NS(test_main)(void *arg)
 
   NS_MOCK(nodelist_get_list);
 
-  smartlist_add(set->country_names, tor_strdup("{xx}"));
+  smartlist_add_strdup(set->country_names, "{xx}");
   NS(mock_smartlist) = smartlist_new();
   NS(mock_node).is_running = 0;
   smartlist_add(NS(mock_smartlist), (void *)&NS(mock_node));
@@ -1944,7 +1957,7 @@ NS(test_main)(void *arg)
 
  done:
   tor_free(s);
-  routerset_free((routerset_t *)set);
+  routerset_free(set);
 }
 
 #undef NS_SUBMODULE
@@ -1985,7 +1998,7 @@ NS(test_main)(void *arg)
   int r;
   (void)arg;
 
-  smartlist_add(b->list, tor_strdup("{xx}"));
+  smartlist_add_strdup(b->list, "{xx}");
   r = routerset_equal(a, b);
   routerset_free(a);
   routerset_free(b);
@@ -2010,9 +2023,9 @@ NS(test_main)(void *arg)
   int r;
   (void)arg;
 
-  smartlist_add(a->list, tor_strdup("{aa}"));
-  smartlist_add(b->list, tor_strdup("{b1}"));
-  smartlist_add(b->list, tor_strdup("{b2}"));
+  smartlist_add_strdup(a->list, "{aa}");
+  smartlist_add_strdup(b->list, "{b1}");
+  smartlist_add_strdup(b->list, "{b2}");
   r = routerset_equal(a, b);
   routerset_free(a);
   routerset_free(b);
@@ -2037,8 +2050,8 @@ NS(test_main)(void *arg)
   int r;
   (void)arg;
 
-  smartlist_add(a->list, tor_strdup("foo"));
-  smartlist_add(b->list, tor_strdup("bar"));
+  smartlist_add_strdup(a->list, "foo");
+  smartlist_add_strdup(b->list, "bar");
   r = routerset_equal(a, b);
   routerset_free(a);
   routerset_free(b);
@@ -2063,8 +2076,8 @@ NS(test_main)(void *arg)
   int r;
   (void)arg;
 
-  smartlist_add(a->list, tor_strdup("foo"));
-  smartlist_add(b->list, tor_strdup("foo"));
+  smartlist_add_strdup(a->list, "foo");
+  smartlist_add_strdup(b->list, "foo");
   r = routerset_equal(a, b);
   routerset_free(a);
   routerset_free(b);
@@ -2081,28 +2094,28 @@ NS(test_main)(void *arg)
  * Structural test for routerset_free, where the routerset is NULL.
  */
 
-NS_DECL(void, smartlist_free, (smartlist_t *sl));
+NS_DECL(void, smartlist_free_, (smartlist_t *sl));
 
 static void
 NS(test_main)(void *arg)
 {
   (void)arg;
 
-  NS_MOCK(smartlist_free);
+  NS_MOCK(smartlist_free_);
 
-  routerset_free(NULL);
+  routerset_free_(NULL);
 
-  tt_int_op(CALLED(smartlist_free), OP_EQ, 0);
+  tt_int_op(CALLED(smartlist_free_), OP_EQ, 0);
 
   done:
     ;
 }
 
 void
-NS(smartlist_free)(smartlist_t *s)
+NS(smartlist_free_)(smartlist_t *s)
 {
   (void)s;
-  CALLED(smartlist_free)++;
+  CALLED(smartlist_free_)++;
 }
 
 #undef NS_SUBMODULE
@@ -2112,9 +2125,9 @@ NS(smartlist_free)(smartlist_t *s)
  * Structural test for routerset_free.
  */
 
-NS_DECL(void, smartlist_free, (smartlist_t *sl));
-NS_DECL(void, strmap_free,(strmap_t *map, void (*free_val)(void*)));
-NS_DECL(void, digestmap_free, (digestmap_t *map, void (*free_val)(void*)));
+NS_DECL(void, smartlist_free_, (smartlist_t *sl));
+NS_DECL(void, strmap_free_,(strmap_t *map, void (*free_val)(void*)));
+NS_DECL(void, digestmap_free_, (digestmap_t *map, void (*free_val)(void*)));
 
 static void
 NS(test_main)(void *arg)
@@ -2122,39 +2135,39 @@ NS(test_main)(void *arg)
   routerset_t *routerset = routerset_new();
   (void)arg;
 
-  NS_MOCK(smartlist_free);
-  NS_MOCK(strmap_free);
-  NS_MOCK(digestmap_free);
+  NS_MOCK(smartlist_free_);
+  NS_MOCK(strmap_free_);
+  NS_MOCK(digestmap_free_);
 
   routerset_free(routerset);
 
-  tt_int_op(CALLED(smartlist_free), OP_NE, 0);
-  tt_int_op(CALLED(strmap_free), OP_NE, 0);
-  tt_int_op(CALLED(digestmap_free), OP_NE, 0);
+  tt_int_op(CALLED(smartlist_free_), OP_NE, 0);
+  tt_int_op(CALLED(strmap_free_), OP_NE, 0);
+  tt_int_op(CALLED(digestmap_free_), OP_NE, 0);
 
   done:
     ;
 }
 
 void
-NS(smartlist_free)(smartlist_t *s)
+NS(smartlist_free_)(smartlist_t *s)
 {
-  CALLED(smartlist_free)++;
-  smartlist_free__real(s);
+  CALLED(smartlist_free_)++;
+  smartlist_free___real(s);
 }
 
 void
-NS(strmap_free)(strmap_t *map, void (*free_val)(void*))
+NS(strmap_free_)(strmap_t *map, void (*free_val)(void*))
 {
-  CALLED(strmap_free)++;
-  strmap_free__real(map, free_val);
+  CALLED(strmap_free_)++;
+  strmap_free___real(map, free_val);
 }
 
 void
-NS(digestmap_free)(digestmap_t *map, void (*free_val)(void*))
+NS(digestmap_free_)(digestmap_t *map, void (*free_val)(void*))
 {
-  CALLED(digestmap_free)++;
-  digestmap_free__real(map, free_val);
+  CALLED(digestmap_free_)++;
+  digestmap_free___real(map, free_val);
 }
 
 #undef NS_SUBMODULE
@@ -2218,4 +2231,3 @@ struct testcase_t routerset_tests[] = {
   TEST_CASE(routerset_free),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_rust.sh b/src/test/test_rust.sh
new file mode 100755
index 0000000000..00b3e88d37
--- /dev/null
+++ b/src/test/test_rust.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+# Test all Rust crates
+
+set -e
+
+export LSAN_OPTIONS=suppressions=${abs_top_srcdir:-../../..}/src/test/rust_supp.txt
+
+# When testing Cargo we pass a number of very specific linker flags down
+# through Cargo. We do not, however, want these flags to affect things like
+# build scripts, only the tests that we're compiling. To ensure this happens
+# we unconditionally pass `--target` into Cargo, ensuring that `RUSTFLAGS` in
+# the environment won't make their way into build scripts.
+rustc_host=$(rustc -vV | grep host | sed 's/host: //')
+
+for cargo_toml_dir in "${abs_top_srcdir:-../../..}"/src/rust/*; do
+    if [ -e "${cargo_toml_dir}/Cargo.toml" ]; then
+	cd "${abs_top_builddir:-../../..}/src/rust" && \
+	    CARGO_TARGET_DIR="${abs_top_builddir:-../../..}/src/rust/target" \
+	    "${CARGO:-cargo}" test ${CARGO_ONLINE-"--frozen"} \
+            --features "test_linking_hack" \
+            --target $rustc_host \
+	    ${EXTRA_CARGO_OPTIONS} \
+	    --manifest-path "${cargo_toml_dir}/Cargo.toml" || exitcode=1
+    fi
+done
+
+exit $exitcode
diff --git a/src/test/test_scheduler.c b/src/test/test_scheduler.c
index 05ea8e86e8..bf9c6a49cd 100644
--- a/src/test/test_scheduler.c
+++ b/src/test/test_scheduler.c
@@ -1,116 +1,177 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 
 #include <math.h>
-#include <event2/event.h>
 
+#define SCHEDULER_KIST_PRIVATE
 #define TOR_CHANNEL_INTERNAL_
 #define CHANNEL_PRIVATE_
-#include "or.h"
-#include "compat_libevent.h"
-#include "channel.h"
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "lib/evloop/compat_libevent.h"
+#include "core/or/channel.h"
+#include "core/or/channeltls.h"
+#include "core/mainloop/connection.h"
+#include "feature/nodelist/networkstatus.h"
 #define SCHEDULER_PRIVATE_
-#include "scheduler.h"
+#include "core/or/scheduler.h"
 
 /* Test suite stuff */
-#include "test.h"
-#include "fakechans.h"
+#include "test/test.h"
+#include "test/fakechans.h"
 
-/* Event base for scheduelr tests */
-static struct event_base *mock_event_base = NULL;
-
-/* Statics controlling mocks */
-static circuitmux_t *mock_ccm_tgt_1 = NULL;
-static circuitmux_t *mock_ccm_tgt_2 = NULL;
+/* Shamelessly stolen from compat_libevent.c */
+#define V(major, minor, patch) \
+  (((major) << 24) | ((minor) << 16) | ((patch) << 8))
 
-static circuitmux_t *mock_cgp_tgt_1 = NULL;
-static circuitmux_policy_t *mock_cgp_val_1 = NULL;
-static circuitmux_t *mock_cgp_tgt_2 = NULL;
-static circuitmux_policy_t *mock_cgp_val_2 = NULL;
+/******************************************************************************
+ * Statistical info
+ *****************************************************************************/
 static int scheduler_compare_channels_mock_ctr = 0;
 static int scheduler_run_mock_ctr = 0;
 
-static void channel_flush_some_cells_mock_free_all(void);
-static void channel_flush_some_cells_mock_set(channel_t *chan,
-                                              ssize_t num_cells);
-
-/* Setup for mock event stuff */
-static void mock_event_free_all(void);
-static void mock_event_init(void);
-
-/* Mocks used by scheduler tests */
-static ssize_t channel_flush_some_cells_mock(channel_t *chan,
-                                             ssize_t num_cells);
-static int circuitmux_compare_muxes_mock(circuitmux_t *cmux_1,
-                                         circuitmux_t *cmux_2);
-static const circuitmux_policy_t * circuitmux_get_policy_mock(
-    circuitmux_t *cmux);
-static int scheduler_compare_channels_mock(const void *c1_v,
-                                           const void *c2_v);
-static void scheduler_run_noop_mock(void);
-static struct event_base * tor_libevent_get_base_mock(void);
-
-/* Scheduler test cases */
-static void test_scheduler_channel_states(void *arg);
-static void test_scheduler_compare_channels(void *arg);
-static void test_scheduler_initfree(void *arg);
-static void test_scheduler_loop(void *arg);
-static void test_scheduler_queue_heuristic(void *arg);
-
-/* Mock event init/free */
+/******************************************************************************
+ * Utility functions and things we need to mock
+ *****************************************************************************/
+static or_options_t mocked_options;
+static const or_options_t *
+mock_get_options(void)
+{
+  return &mocked_options;
+}
 
-/* Shamelessly stolen from compat_libevent.c */
-#define V(major, minor, patch) \
-  (((major) << 24) | ((minor) << 16) | ((patch) << 8))
+static void
+cleanup_scheduler_options(void)
+{
+  if (mocked_options.SchedulerTypes_) {
+    SMARTLIST_FOREACH(mocked_options.SchedulerTypes_, int *, i, tor_free(i));
+    smartlist_free(mocked_options.SchedulerTypes_);
+    mocked_options.SchedulerTypes_ = NULL;
+  }
+}
 
 static void
-mock_event_free_all(void)
+set_scheduler_options(int val)
 {
-  tt_assert(mock_event_base != NULL);
+  int *type;
 
-  if (mock_event_base) {
-    event_base_free(mock_event_base);
-    mock_event_base = NULL;
+  if (mocked_options.SchedulerTypes_ == NULL) {
+    mocked_options.SchedulerTypes_ = smartlist_new();
   }
+  type = tor_malloc_zero(sizeof(int));
+  *type = val;
+  smartlist_add(mocked_options.SchedulerTypes_, type);
+}
+
+static void
+clear_options(void)
+{
+  cleanup_scheduler_options();
+  memset(&mocked_options, 0, sizeof(mocked_options));
+}
 
-  tt_ptr_op(mock_event_base, ==, NULL);
+static int32_t
+mock_vanilla_networkstatus_get_param(
+    const networkstatus_t *ns, const char *param_name, int32_t default_val,
+    int32_t min_val, int32_t max_val)
+{
+  (void)ns;
+  (void)default_val;
+  (void)min_val;
+  (void)max_val;
+  // only support KISTSchedRunInterval right now
+  tor_assert(strcmp(param_name, "KISTSchedRunInterval")==0);
+  return 0;
+}
 
- done:
-  return;
+static int32_t
+mock_kist_networkstatus_get_param(
+    const networkstatus_t *ns, const char *param_name, int32_t default_val,
+    int32_t min_val, int32_t max_val)
+{
+  (void)ns;
+  (void)default_val;
+  (void)min_val;
+  (void)max_val;
+  // only support KISTSchedRunInterval right now
+  tor_assert(strcmp(param_name, "KISTSchedRunInterval")==0);
+  return 12;
+}
+
+static int
+scheduler_compare_channels_mock(const void *c1_v,
+                                const void *c2_v)
+{
+  uintptr_t p1, p2;
+
+  p1 = (uintptr_t)(c1_v);
+  p2 = (uintptr_t)(c2_v);
+
+  ++scheduler_compare_channels_mock_ctr;
+
+  if (p1 == p2) return 0;
+  else if (p1 < p2) return 1;
+  else return -1;
 }
 
 static void
-mock_event_init(void)
+scheduler_run_noop_mock(void)
 {
-  struct event_config *cfg = NULL;
+  ++scheduler_run_mock_ctr;
+}
 
-  tt_ptr_op(mock_event_base, ==, NULL);
+static circuitmux_t *mock_ccm_tgt_1 = NULL;
+static circuitmux_t *mock_ccm_tgt_2 = NULL;
+static circuitmux_t *mock_cgp_tgt_1 = NULL;
+static circuitmux_policy_t *mock_cgp_val_1 = NULL;
+static circuitmux_t *mock_cgp_tgt_2 = NULL;
+static circuitmux_policy_t *mock_cgp_val_2 = NULL;
 
-  /*
-   * Really cut down from tor_libevent_initialize of
-   * src/common/compat_libevent.c to kill config dependencies
-   */
+static const circuitmux_policy_t *
+circuitmux_get_policy_mock(circuitmux_t *cmux)
+{
+  const circuitmux_policy_t *result = NULL;
 
-  if (!mock_event_base) {
-    cfg = event_config_new();
-#if LIBEVENT_VERSION_NUMBER >= V(2,0,9)
-    /* We can enable changelist support with epoll, since we don't give
-     * Libevent any dup'd fds.  This lets us avoid some syscalls. */
-    event_config_set_flag(cfg, EVENT_BASE_FLAG_EPOLL_USE_CHANGELIST);
-#endif
-    mock_event_base = event_base_new_with_config(cfg);
-    event_config_free(cfg);
+  tt_assert(cmux != NULL);
+  if (cmux) {
+    if (cmux == mock_cgp_tgt_1) result = mock_cgp_val_1;
+    else if (cmux == mock_cgp_tgt_2) result = mock_cgp_val_2;
+    else result = circuitmux_get_policy__real(cmux);
   }
 
-  tt_assert(mock_event_base != NULL);
-
  done:
-  return;
+  return result;
 }
 
-/* Mocks */
+static int
+circuitmux_compare_muxes_mock(circuitmux_t *cmux_1,
+                              circuitmux_t *cmux_2)
+{
+  int result = 0;
+
+  tt_assert(cmux_1 != NULL);
+  tt_assert(cmux_2 != NULL);
+
+  if (cmux_1 != cmux_2) {
+    if (cmux_1 == mock_ccm_tgt_1 && cmux_2 == mock_ccm_tgt_2) result = -1;
+    else if (cmux_1 == mock_ccm_tgt_2 && cmux_2 == mock_ccm_tgt_1) {
+      result = 1;
+    } else {
+      if (cmux_1 == mock_ccm_tgt_1 || cmux_1 == mock_ccm_tgt_2) result = -1;
+      else if (cmux_2 == mock_ccm_tgt_1 || cmux_2 == mock_ccm_tgt_2) {
+        result = 1;
+      } else {
+        result = circuitmux_compare_muxes__real(cmux_1, cmux_2);
+      }
+    }
+  }
+  /* else result = 0 always */
+
+ done:
+  return result;
+}
 
 typedef struct {
   const channel_t *chan;
@@ -174,6 +235,63 @@ channel_flush_some_cells_mock_set(channel_t *chan, ssize_t num_cells)
   }
 }
 
+static int
+channel_more_to_flush_mock(channel_t *chan)
+{
+  tor_assert(chan);
+
+  flush_mock_channel_t *found_mock_ch = NULL;
+
+  SMARTLIST_FOREACH_BEGIN(chans_for_flush_mock,
+                          flush_mock_channel_t *,
+                          flush_mock_ch) {
+    if (flush_mock_ch != NULL && flush_mock_ch->chan != NULL) {
+      if (flush_mock_ch->chan == chan) {
+        /* Found it */
+        found_mock_ch = flush_mock_ch;
+        break;
+      }
+    } else {
+      /* That shouldn't be there... */
+      SMARTLIST_DEL_CURRENT(chans_for_flush_mock, flush_mock_ch);
+      tor_free(flush_mock_ch);
+    }
+  } SMARTLIST_FOREACH_END(flush_mock_ch);
+
+  tor_assert(found_mock_ch);
+
+  /* Check if any circuits would like to queue some */
+  /* special for the mock: return the number of cells (instead of 1), or zero
+   * if nothing to flush */
+  return (found_mock_ch->cells > 0 ? (int)found_mock_ch->cells : 0 );
+}
+
+static void
+channel_write_to_kernel_mock(channel_t *chan)
+{
+  (void)chan;
+  //log_debug(LD_SCHED, "chan=%d writing to kernel",
+  //    (int)chan->global_identifier);
+}
+
+static int
+channel_should_write_to_kernel_mock(outbuf_table_t *ot, channel_t *chan)
+{
+  (void)ot;
+  (void)chan;
+  return 1;
+  /* We could make this more complicated if we wanted. But I don't think doing
+   * so tests much of anything */
+  //static int called_counter = 0;
+  //if (++called_counter >= 3) {
+  //  called_counter -= 3;
+  //  log_debug(LD_SCHED, "chan=%d should write to kernel",
+  //      (int)chan->global_identifier);
+  //  return 1;
+  //}
+  //return 0;
+}
+
 static ssize_t
 channel_flush_some_cells_mock(channel_t *chan, ssize_t num_cells)
 {
@@ -181,7 +299,7 @@ channel_flush_some_cells_mock(channel_t *chan, ssize_t num_cells)
   char unlimited = 0;
   flush_mock_channel_t *found = NULL;
 
-  tt_assert(chan != NULL);
+  tt_ptr_op(chan, OP_NE, NULL);
   if (chan) {
     if (num_cells < 0) {
       num_cells = 0;
@@ -215,11 +333,6 @@ channel_flush_some_cells_mock(channel_t *chan, ssize_t num_cells)
 
         flushed += max;
         found->cells -= max;
-
-        if (found->cells <= 0) {
-          smartlist_remove(chans_for_flush_mock, found);
-          tor_free(found);
-        }
       }
     }
   }
@@ -228,92 +341,26 @@ channel_flush_some_cells_mock(channel_t *chan, ssize_t num_cells)
   return flushed;
 }
 
-static int
-circuitmux_compare_muxes_mock(circuitmux_t *cmux_1,
-                              circuitmux_t *cmux_2)
-{
-  int result = 0;
-
-  tt_assert(cmux_1 != NULL);
-  tt_assert(cmux_2 != NULL);
-
-  if (cmux_1 != cmux_2) {
-    if (cmux_1 == mock_ccm_tgt_1 && cmux_2 == mock_ccm_tgt_2) result = -1;
-    else if (cmux_1 == mock_ccm_tgt_2 && cmux_2 == mock_ccm_tgt_1) {
-      result = 1;
-    } else {
-      if (cmux_1 == mock_ccm_tgt_1 || cmux_1 == mock_ccm_tgt_2) result = -1;
-      else if (cmux_2 == mock_ccm_tgt_1 || cmux_2 == mock_ccm_tgt_2) {
-        result = 1;
-      } else {
-        result = circuitmux_compare_muxes__real(cmux_1, cmux_2);
-      }
-    }
-  }
-  /* else result = 0 always */
-
- done:
-  return result;
-}
-
-static const circuitmux_policy_t *
-circuitmux_get_policy_mock(circuitmux_t *cmux)
-{
-  const circuitmux_policy_t *result = NULL;
-
-  tt_assert(cmux != NULL);
-  if (cmux) {
-    if (cmux == mock_cgp_tgt_1) result = mock_cgp_val_1;
-    else if (cmux == mock_cgp_tgt_2) result = mock_cgp_val_2;
-    else result = circuitmux_get_policy__real(cmux);
-  }
-
- done:
-  return result;
-}
-
-static int
-scheduler_compare_channels_mock(const void *c1_v,
-                                const void *c2_v)
-{
-  uintptr_t p1, p2;
-
-  p1 = (uintptr_t)(c1_v);
-  p2 = (uintptr_t)(c2_v);
-
-  ++scheduler_compare_channels_mock_ctr;
-
-  if (p1 == p2) return 0;
-  else if (p1 < p2) return 1;
-  else return -1;
-}
-
 static void
-scheduler_run_noop_mock(void)
+update_socket_info_impl_mock(socket_table_ent_t *ent)
 {
-  ++scheduler_run_mock_ctr;
+  ent->cwnd = ent->unacked = ent->mss = ent->notsent = 0;
+  ent->limit = INT_MAX;
 }
 
-static struct event_base *
-tor_libevent_get_base_mock(void)
-{
-  return mock_event_base;
-}
-
-/* Test cases */
-
 static void
-test_scheduler_channel_states(void *arg)
+perform_channel_state_tests(int KISTSchedRunInterval, int sched_type)
 {
   channel_t *ch1 = NULL, *ch2 = NULL;
   int old_count;
 
-  (void)arg;
-
-  /* Set up libevent and scheduler */
+  /* setup options so we're sure about what sched we are running */
+  MOCK(get_options, mock_get_options);
+  clear_options();
+  mocked_options.KISTSchedRunInterval = KISTSchedRunInterval;
+  set_scheduler_options(sched_type);
 
-  mock_event_init();
-  MOCK(tor_libevent_get_base, tor_libevent_get_base_mock);
+  /* Set up scheduler */
   scheduler_init();
   /*
    * Install the compare channels mock so we can test
@@ -324,9 +371,9 @@ test_scheduler_channel_states(void *arg)
    * Disable scheduler_run so we can just check the state transitions
    * without having to make everything it might call work too.
    */
-  MOCK(scheduler_run, scheduler_run_noop_mock);
+  ((scheduler_t *) the_scheduler)->run = scheduler_run_noop_mock;
 
-  tt_int_op(smartlist_len(channels_pending), ==, 0);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 0);
 
   /* Set up a fake channel */
   ch1 = new_fake_channel();
@@ -334,89 +381,97 @@ test_scheduler_channel_states(void *arg)
 
   /* Start it off in OPENING */
   ch1->state = CHANNEL_STATE_OPENING;
-  /* We'll need a cmux */
-  ch1->cmux = circuitmux_alloc();
   /* Try to register it */
   channel_register(ch1);
   tt_assert(ch1->registered);
 
   /* It should start off in SCHED_CHAN_IDLE */
-  tt_int_op(ch1->scheduler_state, ==, SCHED_CHAN_IDLE);
+  tt_int_op(ch1->scheduler_state, OP_EQ, SCHED_CHAN_IDLE);
 
   /* Now get another one */
   ch2 = new_fake_channel();
   tt_assert(ch2);
   ch2->state = CHANNEL_STATE_OPENING;
-  ch2->cmux = circuitmux_alloc();
   channel_register(ch2);
   tt_assert(ch2->registered);
 
-  /* Send it to SCHED_CHAN_WAITING_TO_WRITE */
+  /* Send ch1 to SCHED_CHAN_WAITING_TO_WRITE */
   scheduler_channel_has_waiting_cells(ch1);
-  tt_int_op(ch1->scheduler_state, ==, SCHED_CHAN_WAITING_TO_WRITE);
+  tt_int_op(ch1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_TO_WRITE);
 
   /* This should send it to SCHED_CHAN_PENDING */
   scheduler_channel_wants_writes(ch1);
-  tt_int_op(ch1->scheduler_state, ==, SCHED_CHAN_PENDING);
-  tt_int_op(smartlist_len(channels_pending), ==, 1);
+  tt_int_op(ch1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 1);
 
   /* Now send ch2 to SCHED_CHAN_WAITING_FOR_CELLS */
   scheduler_channel_wants_writes(ch2);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_WAITING_FOR_CELLS);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
 
   /* Drop ch2 back to idle */
   scheduler_channel_doesnt_want_writes(ch2);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_IDLE);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_IDLE);
 
   /* ...and back to SCHED_CHAN_WAITING_FOR_CELLS */
   scheduler_channel_wants_writes(ch2);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_WAITING_FOR_CELLS);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
 
   /* ...and this should kick ch2 into SCHED_CHAN_PENDING */
   scheduler_channel_has_waiting_cells(ch2);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_PENDING);
-  tt_int_op(smartlist_len(channels_pending), ==, 2);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 2);
 
   /* This should send ch2 to SCHED_CHAN_WAITING_TO_WRITE */
   scheduler_channel_doesnt_want_writes(ch2);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_WAITING_TO_WRITE);
-  tt_int_op(smartlist_len(channels_pending), ==, 1);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_TO_WRITE);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 1);
 
   /* ...and back to SCHED_CHAN_PENDING */
   scheduler_channel_wants_writes(ch2);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_PENDING);
-  tt_int_op(smartlist_len(channels_pending), ==, 2);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 2);
 
   /* Now we exercise scheduler_touch_channel */
   old_count = scheduler_compare_channels_mock_ctr;
   scheduler_touch_channel(ch1);
   tt_assert(scheduler_compare_channels_mock_ctr > old_count);
 
+  /* Release the ch2 and then do it another time to make sure it doesn't blow
+   * up and we are still in a quiescent state. */
+  scheduler_release_channel(ch2);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_IDLE);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 1);
+  /* Cheat a bit so make the release more confused but also will tells us if
+   * the release did put the channel in the right state. */
+  ch2->scheduler_state = SCHED_CHAN_PENDING;
+  scheduler_release_channel(ch2);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_IDLE);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 1);
+
   /* Close */
   channel_mark_for_close(ch1);
-  tt_int_op(ch1->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch1->state, OP_EQ, CHANNEL_STATE_CLOSING);
   channel_mark_for_close(ch2);
-  tt_int_op(ch2->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch2->state, OP_EQ, CHANNEL_STATE_CLOSING);
   channel_closed(ch1);
-  tt_int_op(ch1->state, ==, CHANNEL_STATE_CLOSED);
+  tt_int_op(ch1->state, OP_EQ, CHANNEL_STATE_CLOSED);
   ch1 = NULL;
   channel_closed(ch2);
-  tt_int_op(ch2->state, ==, CHANNEL_STATE_CLOSED);
+  tt_int_op(ch2->state, OP_EQ, CHANNEL_STATE_CLOSED);
   ch2 = NULL;
 
   /* Shut things down */
 
   channel_free_all();
   scheduler_free_all();
-  mock_event_free_all();
 
  done:
   tor_free(ch1);
   tor_free(ch2);
 
   UNMOCK(scheduler_compare_channels);
-  UNMOCK(scheduler_run);
-  UNMOCK(tor_libevent_get_base);
+  UNMOCK(get_options);
+  cleanup_scheduler_options();
 
   return;
 }
@@ -464,21 +519,21 @@ test_scheduler_compare_channels(void *arg)
 
   /* Equal-channel case */
   result = scheduler_compare_channels(&c1, &c1);
-  tt_int_op(result, ==, 0);
+  tt_int_op(result, OP_EQ, 0);
 
   /* Distinct channels, distinct policies */
   result = scheduler_compare_channels(&c1, &c2);
-  tt_int_op(result, ==, -1);
+  tt_int_op(result, OP_EQ, -1);
   result = scheduler_compare_channels(&c2, &c1);
-  tt_int_op(result, ==, 1);
+  tt_int_op(result, OP_EQ, 1);
 
   /* Distinct channels, same policy */
   tor_free(mock_cgp_val_2);
   mock_cgp_val_2 = mock_cgp_val_1;
   result = scheduler_compare_channels(&c1, &c2);
-  tt_int_op(result, ==, -1);
+  tt_int_op(result, OP_EQ, -1);
   result = scheduler_compare_channels(&c2, &c1);
-  tt_int_op(result, ==, 1);
+  tt_int_op(result, OP_EQ, 1);
 
  done:
 
@@ -502,45 +557,24 @@ test_scheduler_compare_channels(void *arg)
   return;
 }
 
-static void
-test_scheduler_initfree(void *arg)
-{
-  (void)arg;
-
-  tt_ptr_op(channels_pending, ==, NULL);
-  tt_ptr_op(run_sched_ev, ==, NULL);
-
-  mock_event_init();
-  MOCK(tor_libevent_get_base, tor_libevent_get_base_mock);
-
-  scheduler_init();
-
-  tt_assert(channels_pending != NULL);
-  tt_assert(run_sched_ev != NULL);
-
-  scheduler_free_all();
-
-  UNMOCK(tor_libevent_get_base);
-  mock_event_free_all();
-
-  tt_ptr_op(channels_pending, ==, NULL);
-  tt_ptr_op(run_sched_ev, ==, NULL);
-
- done:
-  return;
-}
+/******************************************************************************
+ * The actual tests!
+ *****************************************************************************/
 
 static void
-test_scheduler_loop(void *arg)
+test_scheduler_loop_vanilla(void *arg)
 {
-  channel_t *ch1 = NULL, *ch2 = NULL;
-
   (void)arg;
+  channel_t *ch1 = NULL, *ch2 = NULL;
+  void (*run_func_ptr)(void);
 
-  /* Set up libevent and scheduler */
+  /* setup options so we're sure about what sched we are running */
+  MOCK(get_options, mock_get_options);
+  clear_options();
+  set_scheduler_options(SCHEDULER_VANILLA);
+  mocked_options.KISTSchedRunInterval = 0;
 
-  mock_event_init();
-  MOCK(tor_libevent_get_base, tor_libevent_get_base_mock);
+  /* Set up scheduler */
   scheduler_init();
   /*
    * Install the compare channels mock so we can test
@@ -551,32 +585,32 @@ test_scheduler_loop(void *arg)
    * Disable scheduler_run so we can just check the state transitions
    * without having to make everything it might call work too.
    */
-  MOCK(scheduler_run, scheduler_run_noop_mock);
+  run_func_ptr = the_scheduler->run;
+  ((scheduler_t *) the_scheduler)->run = scheduler_run_noop_mock;
 
-  tt_int_op(smartlist_len(channels_pending), ==, 0);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 0);
 
   /* Set up a fake channel */
   ch1 = new_fake_channel();
+  ch1->magic = TLS_CHAN_MAGIC;
   tt_assert(ch1);
 
   /* Start it off in OPENING */
   ch1->state = CHANNEL_STATE_OPENING;
-  /* We'll need a cmux */
-  ch1->cmux = circuitmux_alloc();
   /* Try to register it */
   channel_register(ch1);
   tt_assert(ch1->registered);
   /* Finish opening it */
-  channel_change_state(ch1, CHANNEL_STATE_OPEN);
+  channel_change_state_open(ch1);
 
   /* It should start off in SCHED_CHAN_IDLE */
-  tt_int_op(ch1->scheduler_state, ==, SCHED_CHAN_IDLE);
+  tt_int_op(ch1->scheduler_state, OP_EQ, SCHED_CHAN_IDLE);
 
   /* Now get another one */
   ch2 = new_fake_channel();
+  ch2->magic = TLS_CHAN_MAGIC;
   tt_assert(ch2);
   ch2->state = CHANNEL_STATE_OPENING;
-  ch2->cmux = circuitmux_alloc();
   channel_register(ch2);
   tt_assert(ch2->registered);
   /*
@@ -584,68 +618,62 @@ test_scheduler_loop(void *arg)
    * zero and we'll get coverage of that exception case in scheduler_run()
    */
 
-  tt_int_op(ch1->state, ==, CHANNEL_STATE_OPEN);
-  tt_int_op(ch2->state, ==, CHANNEL_STATE_OPENING);
+  tt_int_op(ch1->state, OP_EQ, CHANNEL_STATE_OPEN);
+  tt_int_op(ch2->state, OP_EQ, CHANNEL_STATE_OPENING);
 
   /* Send it to SCHED_CHAN_WAITING_TO_WRITE */
   scheduler_channel_has_waiting_cells(ch1);
-  tt_int_op(ch1->scheduler_state, ==, SCHED_CHAN_WAITING_TO_WRITE);
+  tt_int_op(ch1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_TO_WRITE);
 
   /* This should send it to SCHED_CHAN_PENDING */
   scheduler_channel_wants_writes(ch1);
-  tt_int_op(ch1->scheduler_state, ==, SCHED_CHAN_PENDING);
-  tt_int_op(smartlist_len(channels_pending), ==, 1);
+  tt_int_op(ch1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 1);
 
   /* Now send ch2 to SCHED_CHAN_WAITING_FOR_CELLS */
   scheduler_channel_wants_writes(ch2);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_WAITING_FOR_CELLS);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
 
   /* Drop ch2 back to idle */
   scheduler_channel_doesnt_want_writes(ch2);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_IDLE);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_IDLE);
 
   /* ...and back to SCHED_CHAN_WAITING_FOR_CELLS */
   scheduler_channel_wants_writes(ch2);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_WAITING_FOR_CELLS);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
 
   /* ...and this should kick ch2 into SCHED_CHAN_PENDING */
   scheduler_channel_has_waiting_cells(ch2);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_PENDING);
-  tt_int_op(smartlist_len(channels_pending), ==, 2);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 2);
 
   /*
    * Now we've got two pending channels and need to fire off
-   * scheduler_run(); first, unmock it.
+   * the scheduler run() that we kept.
    */
-
-  UNMOCK(scheduler_run);
-
-  scheduler_run();
-
-  /* Now re-mock it */
-  MOCK(scheduler_run, scheduler_run_noop_mock);
+  run_func_ptr();
 
   /*
    * Assert that they're still in the states we left and aren't still
    * pending
    */
-  tt_int_op(ch1->state, ==, CHANNEL_STATE_OPEN);
-  tt_int_op(ch2->state, ==, CHANNEL_STATE_OPENING);
+  tt_int_op(ch1->state, OP_EQ, CHANNEL_STATE_OPEN);
+  tt_int_op(ch2->state, OP_EQ, CHANNEL_STATE_OPENING);
   tt_assert(ch1->scheduler_state != SCHED_CHAN_PENDING);
   tt_assert(ch2->scheduler_state != SCHED_CHAN_PENDING);
-  tt_int_op(smartlist_len(channels_pending), ==, 0);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 0);
 
   /* Now, finish opening ch2, and get both back to pending */
-  channel_change_state(ch2, CHANNEL_STATE_OPEN);
+  channel_change_state_open(ch2);
   scheduler_channel_wants_writes(ch1);
   scheduler_channel_wants_writes(ch2);
   scheduler_channel_has_waiting_cells(ch1);
   scheduler_channel_has_waiting_cells(ch2);
-  tt_int_op(ch1->state, ==, CHANNEL_STATE_OPEN);
-  tt_int_op(ch2->state, ==, CHANNEL_STATE_OPEN);
-  tt_int_op(ch1->scheduler_state, ==, SCHED_CHAN_PENDING);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_PENDING);
-  tt_int_op(smartlist_len(channels_pending), ==, 2);
+  tt_int_op(ch1->state, OP_EQ, CHANNEL_STATE_OPEN);
+  tt_int_op(ch2->state, OP_EQ, CHANNEL_STATE_OPEN);
+  tt_int_op(ch1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 2);
 
   /* Now, set up the channel_flush_some_cells() mock */
   MOCK(channel_flush_some_cells, channel_flush_some_cells_mock);
@@ -661,98 +689,593 @@ test_scheduler_loop(void *arg)
   channel_flush_some_cells_mock_set(ch2, 48);
 
   /*
-   * And re-run the scheduler_run() loop with non-zero returns from
+   * And re-run the scheduler run() loop with non-zero returns from
    * channel_flush_some_cells() this time.
    */
-  UNMOCK(scheduler_run);
-
-  scheduler_run();
-
-  /* Now re-mock it */
-  MOCK(scheduler_run, scheduler_run_noop_mock);
+  run_func_ptr();
 
   /*
    * ch1 should have gone to SCHED_CHAN_WAITING_FOR_CELLS, with 16 flushed
    * and 32 writeable.
    */
-  tt_int_op(ch1->scheduler_state, ==, SCHED_CHAN_WAITING_FOR_CELLS);
+  tt_int_op(ch1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
   /*
    * ...ch2 should also have gone to SCHED_CHAN_WAITING_FOR_CELLS, with
    * channel_more_to_flush() returning false and channel_num_cells_writeable()
    * > 0/
    */
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_WAITING_FOR_CELLS);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
 
   /* Close */
   channel_mark_for_close(ch1);
-  tt_int_op(ch1->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch1->state, OP_EQ, CHANNEL_STATE_CLOSING);
   channel_mark_for_close(ch2);
-  tt_int_op(ch2->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch2->state, OP_EQ, CHANNEL_STATE_CLOSING);
   channel_closed(ch1);
-  tt_int_op(ch1->state, ==, CHANNEL_STATE_CLOSED);
+  tt_int_op(ch1->state, OP_EQ, CHANNEL_STATE_CLOSED);
   ch1 = NULL;
   channel_closed(ch2);
-  tt_int_op(ch2->state, ==, CHANNEL_STATE_CLOSED);
+  tt_int_op(ch2->state, OP_EQ, CHANNEL_STATE_CLOSED);
   ch2 = NULL;
 
   /* Shut things down */
   channel_flush_some_cells_mock_free_all();
   channel_free_all();
   scheduler_free_all();
-  mock_event_free_all();
 
  done:
   tor_free(ch1);
   tor_free(ch2);
+  cleanup_scheduler_options();
 
   UNMOCK(channel_flush_some_cells);
   UNMOCK(scheduler_compare_channels);
-  UNMOCK(scheduler_run);
-  UNMOCK(tor_libevent_get_base);
+  UNMOCK(get_options);
 }
 
 static void
-test_scheduler_queue_heuristic(void *arg)
+test_scheduler_loop_kist(void *arg)
 {
-  time_t now = approx_time();
-  uint64_t qh;
+  (void) arg;
+
+#ifndef HAVE_KIST_SUPPORT
+  return;
+#endif
+
+  channel_t *ch1 = new_fake_channel(), *ch2 = new_fake_channel();
+  channel_t *ch3 = new_fake_channel();
+
+  /* setup options so we're sure about what sched we are running */
+  MOCK(get_options, mock_get_options);
+  MOCK(channel_flush_some_cells, channel_flush_some_cells_mock);
+  MOCK(channel_more_to_flush, channel_more_to_flush_mock);
+  MOCK(channel_write_to_kernel, channel_write_to_kernel_mock);
+  MOCK(channel_should_write_to_kernel, channel_should_write_to_kernel_mock);
+  MOCK(update_socket_info_impl, update_socket_info_impl_mock);
+  clear_options();
+  mocked_options.KISTSchedRunInterval = 11;
+  set_scheduler_options(SCHEDULER_KIST);
+  scheduler_init();
 
+  tt_assert(ch1);
+  ch1->magic = TLS_CHAN_MAGIC;
+  ch1->state = CHANNEL_STATE_OPENING;
+  channel_register(ch1);
+  tt_assert(ch1->registered);
+  channel_change_state_open(ch1);
+  scheduler_channel_has_waiting_cells(ch1);
+  scheduler_channel_wants_writes(ch1);
+  channel_flush_some_cells_mock_set(ch1, 5);
+
+  tt_assert(ch2);
+  ch2->magic = TLS_CHAN_MAGIC;
+  ch2->state = CHANNEL_STATE_OPENING;
+  channel_register(ch2);
+  tt_assert(ch2->registered);
+  channel_change_state_open(ch2);
+  scheduler_channel_has_waiting_cells(ch2);
+  scheduler_channel_wants_writes(ch2);
+  channel_flush_some_cells_mock_set(ch2, 5);
+
+  the_scheduler->run();
+
+  scheduler_channel_has_waiting_cells(ch1);
+  channel_flush_some_cells_mock_set(ch1, 5);
+
+  the_scheduler->run();
+
+  scheduler_channel_has_waiting_cells(ch1);
+  channel_flush_some_cells_mock_set(ch1, 5);
+  scheduler_channel_has_waiting_cells(ch2);
+  channel_flush_some_cells_mock_set(ch2, 5);
+
+  the_scheduler->run();
+
+  channel_flush_some_cells_mock_free_all();
+
+  /* We'll try to run this closed channel threw the scheduler loop and make
+   * sure it ends up in the right state. */
+  tt_assert(ch3);
+  ch3->magic = TLS_CHAN_MAGIC;
+  ch3->state = CHANNEL_STATE_OPEN;
+  circuitmux_free(ch3->cmux);
+  ch3->cmux = circuitmux_alloc();
+  channel_register(ch3);
+  tt_assert(ch3->registered);
+
+  ch3->scheduler_state = SCHED_CHAN_WAITING_FOR_CELLS;
+  scheduler_channel_has_waiting_cells(ch3);
+  /* Should be in the pending list now waiting to be handled. */
+  tt_int_op(ch3->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1);
+  /* By running the scheduler on a closed channel, it should end up in the
+   * IDLE state and not in the pending channel list. */
+  ch3->state = CHANNEL_STATE_CLOSED;
+  the_scheduler->run();
+  tt_int_op(ch3->scheduler_state, OP_EQ, SCHED_CHAN_IDLE);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0);
+
+ done:
+  /* Prep the channel so the free() function doesn't explode. */
+  ch1->state = ch2->state = ch3->state = CHANNEL_STATE_CLOSED;
+  ch1->registered = ch2->registered = ch3->registered = 0;
+  channel_free(ch1);
+  channel_free(ch2);
+  channel_free(ch3);
+  UNMOCK(update_socket_info_impl);
+  UNMOCK(channel_should_write_to_kernel);
+  UNMOCK(channel_write_to_kernel);
+  UNMOCK(channel_more_to_flush);
+  UNMOCK(channel_flush_some_cells);
+  UNMOCK(get_options);
+  scheduler_free_all();
+  return;
+}
+
+static void
+test_scheduler_channel_states(void *arg)
+{
+  (void)arg;
+  perform_channel_state_tests(-1, SCHEDULER_VANILLA);
+  perform_channel_state_tests(11, SCHEDULER_KIST_LITE);
+#ifdef HAVE_KIST_SUPPORT
+  perform_channel_state_tests(11, SCHEDULER_KIST);
+#endif
+}
+
+static void
+test_scheduler_initfree(void *arg)
+{
   (void)arg;
 
-  queue_heuristic = 0;
-  queue_heuristic_timestamp = 0;
+  tt_ptr_op(channels_pending, ==, NULL);
+  tt_ptr_op(run_sched_ev, ==, NULL);
+
+  MOCK(get_options, mock_get_options);
+  set_scheduler_options(SCHEDULER_KIST);
+  set_scheduler_options(SCHEDULER_KIST_LITE);
+  set_scheduler_options(SCHEDULER_VANILLA);
+
+  scheduler_init();
+
+  tt_ptr_op(channels_pending, !=, NULL);
+  tt_ptr_op(run_sched_ev, !=, NULL);
+  /* We have specified nothing in the torrc and there's no consensus so the
+   * KIST scheduler is what should be in use */
+  tt_ptr_op(the_scheduler, ==, get_kist_scheduler());
+  tt_int_op(sched_run_interval, ==, 10);
+
+  scheduler_free_all();
+
+  tt_ptr_op(channels_pending, ==, NULL);
+  tt_ptr_op(run_sched_ev, ==, NULL);
+
+ done:
+  UNMOCK(get_options);
+  cleanup_scheduler_options();
+  return;
+}
+
+static void
+test_scheduler_can_use_kist(void *arg)
+{
+  (void)arg;
+
+  int res_should, res_freq;
+  MOCK(get_options, mock_get_options);
+
+  /* Test force enabling of KIST */
+  clear_options();
+  mocked_options.KISTSchedRunInterval = 1234;
+  res_should = scheduler_can_use_kist();
+  res_freq = kist_scheduler_run_interval();
+#ifdef HAVE_KIST_SUPPORT
+  tt_int_op(res_should, ==, 1);
+#else /* HAVE_KIST_SUPPORT */
+  tt_int_op(res_should, ==, 0);
+#endif /* HAVE_KIST_SUPPORT */
+  tt_int_op(res_freq, ==, 1234);
+
+  /* Test defer to consensus, but no consensus available */
+  clear_options();
+  mocked_options.KISTSchedRunInterval = 0;
+  res_should = scheduler_can_use_kist();
+  res_freq = kist_scheduler_run_interval();
+#ifdef HAVE_KIST_SUPPORT
+  tt_int_op(res_should, ==, 1);
+#else /* HAVE_KIST_SUPPORT */
+  tt_int_op(res_should, ==, 0);
+#endif /* HAVE_KIST_SUPPORT */
+  tt_int_op(res_freq, ==, 10);
+
+  /* Test defer to consensus, and kist consensus available */
+  MOCK(networkstatus_get_param, mock_kist_networkstatus_get_param);
+  clear_options();
+  mocked_options.KISTSchedRunInterval = 0;
+  res_should = scheduler_can_use_kist();
+  res_freq = kist_scheduler_run_interval();
+#ifdef HAVE_KIST_SUPPORT
+  tt_int_op(res_should, ==, 1);
+#else /* HAVE_KIST_SUPPORT */
+  tt_int_op(res_should, ==, 0);
+#endif /* HAVE_KIST_SUPPORT */
+  tt_int_op(res_freq, ==, 12);
+  UNMOCK(networkstatus_get_param);
+
+  /* Test defer to consensus, and vanilla consensus available */
+  MOCK(networkstatus_get_param, mock_vanilla_networkstatus_get_param);
+  clear_options();
+  mocked_options.KISTSchedRunInterval = 0;
+  res_should = scheduler_can_use_kist();
+  res_freq = kist_scheduler_run_interval();
+  tt_int_op(res_should, ==, 0);
+  tt_int_op(res_freq, ==, 0);
+  UNMOCK(networkstatus_get_param);
+
+ done:
+  UNMOCK(get_options);
+  return;
+}
+
+static void
+test_scheduler_ns_changed(void *arg)
+{
+  (void) arg;
+
+  /*
+   * Currently no scheduler implementations use the old/new consensuses passed
+   * in scheduler_notify_networkstatus_changed, so it is okay to pass NULL.
+   *
+   * "But then what does test actually exercise???" It tests that
+   * scheduler_notify_networkstatus_changed fetches the correct value from the
+   * consensus, and then switches the scheduler if necessasry.
+   */
+
+  MOCK(get_options, mock_get_options);
+  clear_options();
+  set_scheduler_options(SCHEDULER_KIST);
+  set_scheduler_options(SCHEDULER_VANILLA);
+
+  tt_ptr_op(the_scheduler, ==, NULL);
+
+  /* Change from vanilla to kist via consensus */
+  the_scheduler = get_vanilla_scheduler();
+  MOCK(networkstatus_get_param, mock_kist_networkstatus_get_param);
+  scheduler_notify_networkstatus_changed();
+  UNMOCK(networkstatus_get_param);
+#ifdef HAVE_KIST_SUPPORT
+  tt_ptr_op(the_scheduler, ==, get_kist_scheduler());
+#else
+  tt_ptr_op(the_scheduler, ==, get_vanilla_scheduler());
+#endif
+
+  /* Change from kist to vanilla via consensus */
+  the_scheduler = get_kist_scheduler();
+  MOCK(networkstatus_get_param, mock_vanilla_networkstatus_get_param);
+  scheduler_notify_networkstatus_changed();
+  UNMOCK(networkstatus_get_param);
+  tt_ptr_op(the_scheduler, ==, get_vanilla_scheduler());
+
+  /* Doesn't change when using KIST */
+  the_scheduler = get_kist_scheduler();
+  MOCK(networkstatus_get_param, mock_kist_networkstatus_get_param);
+  scheduler_notify_networkstatus_changed();
+  UNMOCK(networkstatus_get_param);
+#ifdef HAVE_KIST_SUPPORT
+  tt_ptr_op(the_scheduler, ==, get_kist_scheduler());
+#else
+  tt_ptr_op(the_scheduler, ==, get_vanilla_scheduler());
+#endif
+
+  /* Doesn't change when using vanilla */
+  the_scheduler = get_vanilla_scheduler();
+  MOCK(networkstatus_get_param, mock_vanilla_networkstatus_get_param);
+  scheduler_notify_networkstatus_changed();
+  UNMOCK(networkstatus_get_param);
+  tt_ptr_op(the_scheduler, ==, get_vanilla_scheduler());
+
+ done:
+  UNMOCK(get_options);
+  cleanup_scheduler_options();
+  return;
+}
 
-  /* Not yet inited case */
-  scheduler_update_queue_heuristic(now - 180);
-  tt_u64_op(queue_heuristic, ==, 0);
-  tt_int_op(queue_heuristic_timestamp, ==, now - 180);
+/*
+ * Mocked functions for the kist_pending_list test.
+ */
 
-  queue_heuristic = 1000000000L;
-  queue_heuristic_timestamp = now - 120;
+static int mock_flush_some_cells_num = 1;
+static int mock_more_to_flush = 0;
+static int mock_update_socket_info_limit = 0;
 
-  scheduler_update_queue_heuristic(now - 119);
-  tt_u64_op(queue_heuristic, ==, 500000000L);
-  tt_int_op(queue_heuristic_timestamp, ==, now - 119);
+static ssize_t
+channel_flush_some_cells_mock_var(channel_t *chan, ssize_t num_cells)
+{
+  (void) chan;
+  (void) num_cells;
+  return mock_flush_some_cells_num;
+}
 
-  scheduler_update_queue_heuristic(now - 116);
-  tt_u64_op(queue_heuristic, ==, 62500000L);
-  tt_int_op(queue_heuristic_timestamp, ==, now - 116);
+/* Because when we flush cells, it is possible that the connection outbuf gets
+ * fully drained, the wants to write scheduler event is fired back while we
+ * are in the scheduler loop so this mock function does it for us.
+ * Furthermore, the socket limit is set to 0 so once this is triggered, it
+ * informs the scheduler that it can't write on the socket anymore. */
+static void
+channel_write_to_kernel_mock_trigger_24700(channel_t *chan)
+{
+  static int chan_id_seen[2] = {0};
+  if (++chan_id_seen[chan->global_identifier - 1] > 1) {
+    tt_assert(0);
+  }
 
-  qh = scheduler_get_queue_heuristic();
-  tt_u64_op(qh, ==, 0);
+  scheduler_channel_wants_writes(chan);
 
  done:
   return;
 }
 
+static int
+channel_more_to_flush_mock_var(channel_t *chan)
+{
+  (void) chan;
+  return mock_more_to_flush;
+}
+
+static void
+update_socket_info_impl_mock_var(socket_table_ent_t *ent)
+{
+  ent->cwnd = ent->unacked = ent->mss = ent->notsent = 0;
+  ent->limit = mock_update_socket_info_limit;
+}
+
+static void
+test_scheduler_kist_pending_list(void *arg)
+{
+  (void) arg;
+
+#ifndef HAVE_KIST_SUPPORT
+  return;
+#endif
+
+  /* This is for testing the channel flow with the pending list that is
+   * depending on the channel state, what will be the expected behavior of the
+   * scheduler with that list.
+   *
+   * For instance, we want to catch double channel add or removing a channel
+   * that doesn't exists, or putting a channel in the list in a wrong state.
+   * Essentially, this will articifically test cases of the KIST main loop and
+   * entry point in the channel subsystem.
+   *
+   * In part, this is to also catch things like #24700 and provide a test bed
+   * for more testing in the future like so. */
+
+  /* Mocking a series of scheduler function to control the flow of the
+   * scheduler loop to test every use cases and assess the pending list. */
+  MOCK(get_options, mock_get_options);
+  MOCK(channel_flush_some_cells, channel_flush_some_cells_mock_var);
+  MOCK(channel_more_to_flush, channel_more_to_flush_mock_var);
+  MOCK(update_socket_info_impl, update_socket_info_impl_mock_var);
+  MOCK(channel_write_to_kernel, channel_write_to_kernel_mock);
+  MOCK(channel_should_write_to_kernel, channel_should_write_to_kernel_mock);
+
+  /* Setup options so we're sure about what sched we are running */
+  mocked_options.KISTSchedRunInterval = 10;
+  set_scheduler_options(SCHEDULER_KIST);
+
+  /* Init scheduler. */
+  scheduler_init();
+
+  /* Initialize a channel. We'll need a second channel for the #24700 bug
+   * test. */
+  channel_t *chan1 = new_fake_channel();
+  channel_t *chan2 = new_fake_channel();
+  tt_assert(chan1);
+  tt_assert(chan2);
+  chan1->magic = chan2->magic = TLS_CHAN_MAGIC;
+  channel_register(chan1);
+  channel_register(chan2);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_IDLE);
+  tt_int_op(chan1->sched_heap_idx, OP_EQ, -1);
+  tt_int_op(chan2->scheduler_state, OP_EQ, SCHED_CHAN_IDLE);
+  tt_int_op(chan2->sched_heap_idx, OP_EQ, -1);
+
+  /* Once a channel becomes OPEN, it always have at least one cell in it so
+   * the scheduler is notified that the channel wants to write so this is the
+   * first step. Might not make sense to you but it is the way it is. */
+  scheduler_channel_wants_writes(chan1);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0);
+
+  /* Signal the scheduler that it has waiting cells which means the channel
+   * will get scheduled. */
+  scheduler_channel_has_waiting_cells(chan1);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1);
+  /* Subsequent call should not add it more times. It is possible we add many
+   * cells in rapid succession before the channel is scheduled. */
+  scheduler_channel_has_waiting_cells(chan1);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1);
+  scheduler_channel_has_waiting_cells(chan1);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1);
+
+  /* We'll flush one cell and make it that the socket can write but no more to
+   * flush else we end up in an infinite loop. We expect the channel to be put
+   * in waiting for cells state and the pending list empty. */
+  mock_update_socket_info_limit = INT_MAX;
+  mock_more_to_flush = 0;
+  the_scheduler->run();
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
+
+  /* Lets make believe that a cell is now in the channel but this time the
+   * channel can't write so obviously it has more to flush. We expect the
+   * channel to be back in the pending list. */
+  scheduler_channel_has_waiting_cells(chan1);
+  mock_update_socket_info_limit = 0;
+  mock_more_to_flush = 1;
+  the_scheduler->run();
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+
+  /* Channel is in the pending list now, during that time, we'll trigger a
+   * wants to write event because maybe the channel buffers were emptied in
+   * the meantime. This is possible because once the connection outbuf is
+   * flushed down the low watermark, the scheduler is notified.
+   *
+   * We expect the channel to NOT be added in the pending list again and stay
+   * in PENDING state. */
+  scheduler_channel_wants_writes(chan1);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+
+  /* Make it that the channel can write now but has nothing else to flush. We
+   * expect that it is removed from the pending list and waiting for cells. */
+  mock_update_socket_info_limit = INT_MAX;
+  mock_more_to_flush = 0;
+  the_scheduler->run();
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
+
+  /* While waiting for cells, lets say we were able to write more things on
+   * the connection outbuf (unlikely that this can happen but let say it
+   * does). We expect the channel to stay in waiting for cells. */
+  scheduler_channel_wants_writes(chan1);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
+
+  /* We'll not put it in the pending list and make the flush cell fail with 0
+   * cell flushed. We expect that it is put back in waiting for cells. */
+  scheduler_channel_has_waiting_cells(chan1);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  mock_flush_some_cells_num = 0;
+  the_scheduler->run();
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
+
+  /* Set the channel to a state where it doesn't want to write more. We expect
+   * that the channel becomes idle. */
+  scheduler_channel_doesnt_want_writes(chan1);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_IDLE);
+
+  /* Some cells arrive on the channel now. We expect it to go back in waiting
+   * to write. You might wonder why it is not put in the pending list? Because
+   * once the channel becomes OPEN again (the doesn't want to write event only
+   * occurs if the channel goes in MAINT mode), if there are cells in the
+   * channel, the wants to write event is triggered thus putting the channel
+   * in pending mode.
+   *
+   * Else, if no cells, it stays IDLE and then once a cell comes in, it should
+   * go in waiting to write which is a BUG itself because the channel can't be
+   * scheduled until a second cell comes in. Hopefully, #24554 will fix that
+   * for KIST. */
+  scheduler_channel_has_waiting_cells(chan1);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_TO_WRITE);
+
+  /* Second cell comes in, unfortunately, it won't get scheduled until a wants
+   * to write event occurs like described above. */
+  scheduler_channel_has_waiting_cells(chan1);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_TO_WRITE);
+
+  /* Unblock everything putting the channel in the pending list. */
+  scheduler_channel_wants_writes(chan1);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+
+  /* Testing bug #24700 which is the situation where we have at least two
+   * different channels in the pending list. The first one gets flushed and
+   * bytes are written on the wire which triggers a wants to write event
+   * because the outbuf is below the low watermark. The bug was that this
+   * exact channel was added back in the pending list because its state wasn't
+   * PENDING.
+   *
+   * The following does some ninja-tsu to try to make it happen. We need two
+   * different channels so we create a second one and add it to the pending
+   * list. Then, we have a custom function when we write to kernel that does
+   * two important things:
+   *
+   *  1) Calls scheduler_channel_wants_writes(chan) on the channel.
+   *  2) Keeps track of how many times it sees the channel going through. If
+   *     that limit goes > 1, it means we've added the channel twice in the
+   *     pending list.
+   *
+   * In the end, we expect both channels to be in the pending list after this
+   * scheduler run. */
+
+  /* Put the second channel in the pending list. */
+  scheduler_channel_wants_writes(chan2);
+  scheduler_channel_has_waiting_cells(chan2);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 2);
+  tt_int_op(chan2->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+
+  /* This makes it that the first pass on socket_can_write() will be true but
+   * then when a single cell is flushed (514 + 29 bytes), the second call to
+   * socket_can_write() will be false. If it wasn't sending back false on the
+   * second run, we end up in an infinite loop of the scheduler. */
+  mock_update_socket_info_limit = 600;
+  /* We want to hit "Case 3:" of the scheduler so channel_more_to_flush() is
+   * true but socket_can_write() has to be false on the second check on the
+   * channel. */
+  mock_more_to_flush = 1;
+  mock_flush_some_cells_num = 1;
+  MOCK(channel_write_to_kernel, channel_write_to_kernel_mock_trigger_24700);
+  the_scheduler->run();
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 2);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(chan2->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+
+ done:
+  chan1->state = chan2->state = CHANNEL_STATE_CLOSED;
+  chan1->registered = chan2->registered = 0;
+  channel_free(chan1);
+  channel_free(chan2);
+  scheduler_free_all();
+
+  UNMOCK(get_options);
+  UNMOCK(channel_flush_some_cells);
+  UNMOCK(channel_more_to_flush);
+  UNMOCK(update_socket_info_impl);
+  UNMOCK(channel_write_to_kernel);
+  UNMOCK(channel_should_write_to_kernel);
+}
+
 struct testcase_t scheduler_tests[] = {
-  { "channel_states", test_scheduler_channel_states, TT_FORK, NULL, NULL },
   { "compare_channels", test_scheduler_compare_channels,
     TT_FORK, NULL, NULL },
+  { "channel_states", test_scheduler_channel_states, TT_FORK, NULL, NULL },
   { "initfree", test_scheduler_initfree, TT_FORK, NULL, NULL },
-  { "loop", test_scheduler_loop, TT_FORK, NULL, NULL },
-  { "queue_heuristic", test_scheduler_queue_heuristic,
-    TT_FORK, NULL, NULL },
+  { "loop_vanilla", test_scheduler_loop_vanilla, TT_FORK, NULL, NULL },
+  { "loop_kist", test_scheduler_loop_kist, TT_FORK, NULL, NULL },
+  { "ns_changed", test_scheduler_ns_changed, TT_FORK, NULL, NULL},
+  { "should_use_kist", test_scheduler_can_use_kist, TT_FORK, NULL, NULL },
+  { "kist_pending_list", test_scheduler_kist_pending_list, TT_FORK,
+    NULL, NULL },
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c
index 526fb09b67..b4fe6eef64 100644
--- a/src/test/test_shared_random.c
+++ b/src/test/test_shared_random.c
@@ -1,20 +1,40 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
 #define SHARED_RANDOM_PRIVATE
 #define SHARED_RANDOM_STATE_PRIVATE
 #define CONFIG_PRIVATE
 #define DIRVOTE_PRIVATE
 
-#include "or.h"
-#include "test.h"
-#include "config.h"
-#include "dirvote.h"
-#include "shared_random.h"
-#include "shared_random_state.h"
-#include "routerkeys.h"
-#include "routerlist.h"
-#include "router.h"
-#include "routerparse.h"
-#include "networkstatus.h"
-#include "log_test_helpers.h"
+#include "core/or/or.h"
+#include "test/test.h"
+#include "app/config/config.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/shared_random.h"
+#include "feature/dirauth/shared_random_state.h"
+#include "test/log_test_helpers.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routerkeys.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/dirparse/authcert_parse.h"
+#include "feature/hs_common/shared_random_client.h"
+#include "feature/dircommon/voting_schedule.h"
+
+#include "feature/dirclient/dir_server_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "app/config/or_state_st.h"
+
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
+#ifdef _WIN32
+/* For mkdir */
+#include <direct.h>
+#endif
 
 static authority_cert_t *mock_cert;
 
@@ -48,7 +68,7 @@ init_authority_state(void)
   mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
   tt_assert(mock_cert);
   options->AuthoritativeDir = 1;
-  tt_int_op(0, ==, load_ed_keys(options, time(NULL)));
+  tt_int_op(load_ed_keys(options, time(NULL)), OP_GE, 0);
   sr_state_init(0, 0);
   /* It's possible a commit has been generated in our state depending on
    * the phase we are currently in which uses "now" as the starting
@@ -76,65 +96,73 @@ test_get_sr_protocol_phase(void *arg)
 
   {
     retval = parse_rfc1123_time("Wed, 20 Apr 2015 23:59:00 UTC", &the_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
 
     phase = get_sr_protocol_phase(the_time);
-    tt_int_op(phase, ==, SR_PHASE_REVEAL);
+    tt_int_op(phase, OP_EQ, SR_PHASE_REVEAL);
   }
 
   {
     retval = parse_rfc1123_time("Wed, 20 Apr 2015 00:00:00 UTC", &the_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
 
     phase = get_sr_protocol_phase(the_time);
-    tt_int_op(phase, ==, SR_PHASE_COMMIT);
+    tt_int_op(phase, OP_EQ, SR_PHASE_COMMIT);
   }
 
   {
     retval = parse_rfc1123_time("Wed, 20 Apr 2015 00:00:01 UTC", &the_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
 
     phase = get_sr_protocol_phase(the_time);
-    tt_int_op(phase, ==, SR_PHASE_COMMIT);
+    tt_int_op(phase, OP_EQ, SR_PHASE_COMMIT);
   }
 
   {
     retval = parse_rfc1123_time("Wed, 20 Apr 2015 11:59:00 UTC", &the_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
 
     phase = get_sr_protocol_phase(the_time);
-    tt_int_op(phase, ==, SR_PHASE_COMMIT);
+    tt_int_op(phase, OP_EQ, SR_PHASE_COMMIT);
   }
 
   {
     retval = parse_rfc1123_time("Wed, 20 Apr 2015 12:00:00 UTC", &the_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
 
     phase = get_sr_protocol_phase(the_time);
-    tt_int_op(phase, ==, SR_PHASE_REVEAL);
+    tt_int_op(phase, OP_EQ, SR_PHASE_REVEAL);
   }
 
   {
     retval = parse_rfc1123_time("Wed, 20 Apr 2015 12:00:01 UTC", &the_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
 
     phase = get_sr_protocol_phase(the_time);
-    tt_int_op(phase, ==, SR_PHASE_REVEAL);
+    tt_int_op(phase, OP_EQ, SR_PHASE_REVEAL);
   }
 
   {
     retval = parse_rfc1123_time("Wed, 20 Apr 2015 13:00:00 UTC", &the_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
 
     phase = get_sr_protocol_phase(the_time);
-    tt_int_op(phase, ==, SR_PHASE_REVEAL);
+    tt_int_op(phase, OP_EQ, SR_PHASE_REVEAL);
   }
 
  done:
   ;
 }
 
-static networkstatus_t *mock_consensus = NULL;
+static networkstatus_t mock_consensus;
+
+/* Mock function to immediately return our local 'mock_consensus'. */
+static networkstatus_t *
+mock_networkstatus_get_live_consensus(time_t now)
+{
+  (void) now;
+  return &mock_consensus;
+}
 
 static void
 test_get_state_valid_until_time(void *arg)
@@ -146,11 +174,23 @@ test_get_state_valid_until_time(void *arg)
 
   (void) arg;
 
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  retval = parse_rfc1123_time("Mon, 20 Apr 2015 01:00:00 UTC",
+                              &mock_consensus.fresh_until);
+  tt_int_op(retval, OP_EQ, 0);
+
+  retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC",
+                              &mock_consensus.valid_after);
+  tt_int_op(retval, OP_EQ, 0);
+
   {
     /* Get the valid until time if called at 00:00:01 */
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:01 UTC",
                                 &current_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
+    voting_schedule_recalculate_timing(get_options(), current_time);
     valid_until_time = get_state_valid_until_time(current_time);
 
     /* Compare it with the correct result */
@@ -161,7 +201,8 @@ test_get_state_valid_until_time(void *arg)
   {
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 19:22:00 UTC",
                                 &current_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
+    voting_schedule_recalculate_timing(get_options(), current_time);
     valid_until_time = get_state_valid_until_time(current_time);
 
     format_iso_time(tbuf, valid_until_time);
@@ -171,7 +212,8 @@ test_get_state_valid_until_time(void *arg)
   {
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 23:59:00 UTC",
                                 &current_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
+    voting_schedule_recalculate_timing(get_options(), current_time);
     valid_until_time = get_state_valid_until_time(current_time);
 
     format_iso_time(tbuf, valid_until_time);
@@ -181,7 +223,8 @@ test_get_state_valid_until_time(void *arg)
   {
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC",
                                 &current_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
+    voting_schedule_recalculate_timing(get_options(), current_time);
     valid_until_time = get_state_valid_until_time(current_time);
 
     format_iso_time(tbuf, valid_until_time);
@@ -189,82 +232,178 @@ test_get_state_valid_until_time(void *arg)
   }
 
  done:
-  ;
+  UNMOCK(networkstatus_get_live_consensus);
 }
 
-/* Mock function to immediately return our local 'mock_consensus'. */
-static networkstatus_t *
-mock_networkstatus_get_live_consensus(time_t now)
-{
-  (void) now;
-  return mock_consensus;
-}
-
-/** Test the get_next_valid_after_time() function. */
+/** Test the function that calculates the start time of the current SRV
+ *  protocol run. */
 static void
-test_get_next_valid_after_time(void *arg)
+test_get_start_time_of_current_run(void *arg)
 {
-  time_t current_time;
-  time_t valid_after_time;
-  char tbuf[ISO_TIME_LEN + 1];
   int retval;
+  char tbuf[ISO_TIME_LEN + 1];
+  time_t current_time, run_start_time;
 
   (void) arg;
 
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  retval = parse_rfc1123_time("Mon, 20 Apr 2015 01:00:00 UTC",
+                              &mock_consensus.fresh_until);
+  tt_int_op(retval, OP_EQ, 0);
+
+  retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC",
+                              &mock_consensus.valid_after);
+  tt_int_op(retval, OP_EQ, 0);
+
   {
-    /* Setup a fake consensus just to get the times out of it, since
-       get_next_valid_after_time() needs them. */
-    mock_consensus = tor_malloc_zero(sizeof(networkstatus_t));
+    /* Get start time if called at 00:00:01 */
+    retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:01 UTC",
+                                &current_time);
+    tt_int_op(retval, OP_EQ, 0);
+    voting_schedule_recalculate_timing(get_options(), current_time);
+    run_start_time = sr_state_get_start_time_of_current_protocol_run();
 
-    retval = parse_rfc1123_time("Mon, 13 Jan 2016 16:00:00 UTC",
-                                &mock_consensus->fresh_until);
-    tt_int_op(retval, ==, 0);
+    /* Compare it with the correct result */
+    format_iso_time(tbuf, run_start_time);
+    tt_str_op("2015-04-20 00:00:00", OP_EQ, tbuf);
+  }
 
-    retval = parse_rfc1123_time("Mon, 13 Jan 2016 15:00:00 UTC",
-                                &mock_consensus->valid_after);
-    tt_int_op(retval, ==, 0);
+  {
+    retval = parse_rfc1123_time("Mon, 20 Apr 2015 23:59:59 UTC",
+                                &current_time);
+    tt_int_op(retval, OP_EQ, 0);
+    voting_schedule_recalculate_timing(get_options(), current_time);
+    run_start_time = sr_state_get_start_time_of_current_protocol_run();
 
-    MOCK(networkstatus_get_live_consensus,
-         mock_networkstatus_get_live_consensus);
+    /* Compare it with the correct result */
+    format_iso_time(tbuf, run_start_time);
+    tt_str_op("2015-04-20 00:00:00", OP_EQ, tbuf);
   }
 
   {
-    /* Get the valid after time if called at 00:00:00 */
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC",
                                 &current_time);
-    tt_int_op(retval, ==, 0);
-    valid_after_time = get_next_valid_after_time(current_time);
+    tt_int_op(retval, OP_EQ, 0);
+    voting_schedule_recalculate_timing(get_options(), current_time);
+    run_start_time = sr_state_get_start_time_of_current_protocol_run();
 
     /* Compare it with the correct result */
-    format_iso_time(tbuf, valid_after_time);
-    tt_str_op("2015-04-20 01:00:00", OP_EQ, tbuf);
+    format_iso_time(tbuf, run_start_time);
+    tt_str_op("2015-04-20 00:00:00", OP_EQ, tbuf);
   }
 
   {
-    /* Get the valid until time if called at 00:00:01 */
-    retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:01 UTC",
+    /* We want the local time to be past midnight, but the current consensus to
+     * have valid-after 23:00 (e.g. this can happen if we fetch a new consensus
+     * at 00:08 before dircaches have a chance to get the midnight consensus).
+     *
+     * Basically, we want to cause a desynch between ns->valid_after (23:00)
+     * and the voting_schedule.interval_starts (01:00), to make sure that
+     * sr_state_get_start_time_of_current_protocol_run() handles it gracefully:
+     * It should actually follow the local consensus time and not the voting
+     * schedule (which is designed for authority voting purposes). */
+    retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC",
+                                &mock_consensus.fresh_until);
+    tt_int_op(retval, OP_EQ, 0);
+
+    retval = parse_rfc1123_time("Mon, 19 Apr 2015 23:00:00 UTC",
+                                &mock_consensus.valid_after);
+
+    retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:08:00 UTC",
                                 &current_time);
-    tt_int_op(retval, ==, 0);
-    valid_after_time = get_next_valid_after_time(current_time);
+    tt_int_op(retval, OP_EQ, 0);
+    update_approx_time(current_time);
+    voting_schedule_recalculate_timing(get_options(), current_time);
+
+    run_start_time = sr_state_get_start_time_of_current_protocol_run();
 
     /* Compare it with the correct result */
-    format_iso_time(tbuf, valid_after_time);
+    format_iso_time(tbuf, run_start_time);
+    tt_str_op("2015-04-19 00:00:00", OP_EQ, tbuf);
+    /* Check that voting_schedule.interval_starts is at 01:00 (see above) */
+    time_t interval_starts = voting_schedule_get_next_valid_after_time();
+    format_iso_time(tbuf, interval_starts);
     tt_str_op("2015-04-20 01:00:00", OP_EQ, tbuf);
- }
+  }
+
+  /* Next test is testing it without a consensus to use the testing voting
+   * interval . */
+  UNMOCK(networkstatus_get_live_consensus);
 
+  /* Now let's alter the voting schedule and check the correctness of the
+   * function. Voting interval of 10 seconds, means that an SRV protocol run
+   * takes 10 seconds * 24 rounds = 4 mins */
   {
-    retval = parse_rfc1123_time("Mon, 20 Apr 2015 23:30:01 UTC",
+    or_options_t *options = get_options_mutable();
+    options->V3AuthVotingInterval = 10;
+    options->TestingV3AuthInitialVotingInterval = 10;
+    retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:15:32 UTC",
                                 &current_time);
-    tt_int_op(retval, ==, 0);
-    valid_after_time = get_next_valid_after_time(current_time);
+    tt_int_op(retval, OP_EQ, 0);
+    voting_schedule_recalculate_timing(get_options(), current_time);
+    run_start_time = sr_state_get_start_time_of_current_protocol_run();
 
     /* Compare it with the correct result */
-    format_iso_time(tbuf, valid_after_time);
-    tt_str_op("2015-04-21 00:00:00", OP_EQ, tbuf);
- }
+    format_iso_time(tbuf, run_start_time);
+    tt_str_op("2015-04-20 00:12:00", OP_EQ, tbuf);
+  }
+
+ done:
+  ;
+}
+
+/** Do some rudimentary consistency checks between the functions that
+ *  understand the shared random protocol schedule */
+static void
+test_get_start_time_functions(void *arg)
+{
+  (void) arg;
+  int retval;
+
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  retval = parse_rfc1123_time("Mon, 20 Apr 2015 01:00:00 UTC",
+                              &mock_consensus.fresh_until);
+  tt_int_op(retval, OP_EQ, 0);
+
+  retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC",
+                              &mock_consensus.valid_after);
+  tt_int_op(retval, OP_EQ, 0);
+  time_t now = mock_consensus.valid_after;
+
+  voting_schedule_recalculate_timing(get_options(), now);
+  time_t start_time_of_protocol_run =
+    sr_state_get_start_time_of_current_protocol_run();
+  tt_assert(start_time_of_protocol_run);
+
+  /* Check that the round start time of the beginning of the run, is itself */
+  tt_int_op(get_start_time_of_current_round(), OP_EQ,
+            start_time_of_protocol_run);
 
  done:
-  networkstatus_vote_free(mock_consensus);
+  UNMOCK(networkstatus_get_live_consensus);
+}
+
+static void
+test_get_sr_protocol_duration(void *arg)
+{
+  (void) arg;
+
+  /* Check that by default an SR phase is 12 hours */
+  tt_int_op(sr_state_get_phase_duration(), OP_EQ, 12*60*60);
+  tt_int_op(sr_state_get_protocol_run_duration(), OP_EQ, 24*60*60);
+
+  /* Now alter the voting interval and check that the SR phase is 2 mins long
+   * if voting happens every 10 seconds (10*12 seconds = 2 mins) */
+  or_options_t *options = get_options_mutable();
+  options->V3AuthVotingInterval = 10;
+  tt_int_op(sr_state_get_phase_duration(), OP_EQ, 2*60);
+  tt_int_op(sr_state_get_protocol_run_duration(), OP_EQ, 4*60);
+
+ done: ;
 }
 
 /* In this test we are going to generate a sr_commit_t object and validate
@@ -289,7 +428,7 @@ test_sr_commit(void *arg)
     tt_assert(auth_cert);
 
     options->AuthoritativeDir = 1;
-    tt_int_op(0, ==, load_ed_keys(options, now));
+    tt_int_op(load_ed_keys(options, time(NULL)), OP_GE, 0);
   }
 
   /* Generate our commit object and validate it has the appropriate field
@@ -307,18 +446,18 @@ test_sr_commit(void *arg)
     tt_assert(!tor_mem_is_zero((char *) our_commit->random_number,
                                sizeof(our_commit->random_number)));
     /* Commit and reveal timestamp should be the same. */
-    tt_u64_op(our_commit->commit_ts, ==, our_commit->reveal_ts);
+    tt_u64_op(our_commit->commit_ts, OP_EQ, our_commit->reveal_ts);
     /* We should have a hashed reveal. */
     tt_assert(!tor_mem_is_zero(our_commit->hashed_reveal,
                                sizeof(our_commit->hashed_reveal)));
     /* Do we have a valid encoded commit and reveal. Note the following only
      * tests if the generated values are correct. Their could be a bug in
-     * the decode function but we test them seperately. */
-    tt_int_op(0, ==, reveal_decode(our_commit->encoded_reveal,
+     * the decode function but we test them separately. */
+    tt_int_op(0, OP_EQ, reveal_decode(our_commit->encoded_reveal,
                                    &test_commit));
-    tt_int_op(0, ==, commit_decode(our_commit->encoded_commit,
+    tt_int_op(0, OP_EQ, commit_decode(our_commit->encoded_commit,
                                    &test_commit));
-    tt_int_op(0, ==, verify_commit_and_reveal(our_commit));
+    tt_int_op(0, OP_EQ, verify_commit_and_reveal(our_commit));
   }
 
   /* Let's make sure our verify commit and reveal function works. We'll
@@ -331,32 +470,32 @@ test_sr_commit(void *arg)
     /* Timestamp MUST match. */
     test_commit.commit_ts = test_commit.reveal_ts - 42;
     setup_full_capture_of_logs(LOG_WARN);
-    tt_int_op(-1, ==, verify_commit_and_reveal(&test_commit));
+    tt_int_op(-1, OP_EQ, verify_commit_and_reveal(&test_commit));
     expect_log_msg_containing("doesn't match reveal timestamp");
     teardown_capture_of_logs();
     memcpy(&test_commit, our_commit, sizeof(test_commit));
-    tt_int_op(0, ==, verify_commit_and_reveal(&test_commit));
+    tt_int_op(0, OP_EQ, verify_commit_and_reveal(&test_commit));
 
     /* Hashed reveal must match the H(encoded_reveal). */
     memset(test_commit.hashed_reveal, 'X',
            sizeof(test_commit.hashed_reveal));
     setup_full_capture_of_logs(LOG_WARN);
-    tt_int_op(-1, ==, verify_commit_and_reveal(&test_commit));
+    tt_int_op(-1, OP_EQ, verify_commit_and_reveal(&test_commit));
     expect_single_log_msg_containing("doesn't match the commit value");
     teardown_capture_of_logs();
     memcpy(&test_commit, our_commit, sizeof(test_commit));
-    tt_int_op(0, ==, verify_commit_and_reveal(&test_commit));
+    tt_int_op(0, OP_EQ, verify_commit_and_reveal(&test_commit));
   }
 
   /* We'll build a list of values from our commit that our parsing function
    * takes from a vote line and see if we can parse it correctly. */
   {
-    smartlist_add(args, tor_strdup("1"));
-    smartlist_add(args,
-               tor_strdup(crypto_digest_algorithm_get_name(our_commit->alg)));
-    smartlist_add(args, tor_strdup(sr_commit_get_rsa_fpr(our_commit)));
-    smartlist_add(args, tor_strdup(our_commit->encoded_commit));
-    smartlist_add(args, tor_strdup(our_commit->encoded_reveal));
+    smartlist_add_strdup(args, "1");
+    smartlist_add_strdup(args,
+               crypto_digest_algorithm_get_name(our_commit->alg));
+    smartlist_add_strdup(args, sr_commit_get_rsa_fpr(our_commit));
+    smartlist_add_strdup(args, our_commit->encoded_commit);
+    smartlist_add_strdup(args, our_commit->encoded_reveal);
     parsed_commit = sr_parse_commit(args);
     tt_assert(parsed_commit);
     /* That parsed commit should be _EXACTLY_ like our original commit (we
@@ -399,26 +538,26 @@ test_encoding(void *arg)
   /* Hash random number because we don't expose bytes of the RNG. */
   ret = crypto_digest256(hashed_rand, raw_rand,
                          sizeof(raw_rand), SR_DIGEST_ALG);
-  tt_int_op(0, ==, ret);
+  tt_int_op(0, OP_EQ, ret);
   /* Hash reveal value. */
-  tt_int_op(SR_REVEAL_BASE64_LEN, ==, strlen(encoded_reveal));
+  tt_int_op(SR_REVEAL_BASE64_LEN, OP_EQ, strlen(encoded_reveal));
   ret = crypto_digest256(hashed_reveal, encoded_reveal,
                          strlen(encoded_reveal), SR_DIGEST_ALG);
-  tt_int_op(0, ==, ret);
-  tt_int_op(SR_COMMIT_BASE64_LEN, ==, strlen(encoded_commit));
+  tt_int_op(0, OP_EQ, ret);
+  tt_int_op(SR_COMMIT_BASE64_LEN, OP_EQ, strlen(encoded_commit));
 
   /* Test our commit/reveal decode functions. */
   {
     /* Test the reveal encoded value. */
-    tt_int_op(0, ==, reveal_decode(encoded_reveal, &parsed_commit));
-    tt_u64_op(ts, ==, parsed_commit.reveal_ts);
+    tt_int_op(0, OP_EQ, reveal_decode(encoded_reveal, &parsed_commit));
+    tt_u64_op(ts, OP_EQ, parsed_commit.reveal_ts);
     tt_mem_op(hashed_rand, OP_EQ, parsed_commit.random_number,
               sizeof(hashed_rand));
 
     /* Test the commit encoded value. */
     memset(&parsed_commit, 0, sizeof(parsed_commit));
-    tt_int_op(0, ==, commit_decode(encoded_commit, &parsed_commit));
-    tt_u64_op(ts, ==, parsed_commit.commit_ts);
+    tt_int_op(0, OP_EQ, commit_decode(encoded_commit, &parsed_commit));
+    tt_u64_op(ts, OP_EQ, parsed_commit.commit_ts);
     tt_mem_op(encoded_commit, OP_EQ, parsed_commit.encoded_commit,
               sizeof(parsed_commit.encoded_commit));
     tt_mem_op(hashed_reveal, OP_EQ, parsed_commit.hashed_reveal,
@@ -433,7 +572,7 @@ test_encoding(void *arg)
     memcpy(parsed_commit.random_number, hashed_rand,
            sizeof(parsed_commit.random_number));
     ret = reveal_encode(&parsed_commit, encoded, sizeof(encoded));
-    tt_int_op(SR_REVEAL_BASE64_LEN, ==, ret);
+    tt_int_op(SR_REVEAL_BASE64_LEN, OP_EQ, ret);
     tt_mem_op(encoded_reveal, OP_EQ, encoded, strlen(encoded_reveal));
   }
 
@@ -444,7 +583,7 @@ test_encoding(void *arg)
     memcpy(parsed_commit.hashed_reveal, hashed_reveal,
            sizeof(parsed_commit.hashed_reveal));
     ret = commit_encode(&parsed_commit, encoded, sizeof(encoded));
-    tt_int_op(SR_COMMIT_BASE64_LEN, ==, ret);
+    tt_int_op(SR_COMMIT_BASE64_LEN, OP_EQ, ret);
     tt_mem_op(encoded_commit, OP_EQ, encoded, strlen(encoded_commit));
   }
 
@@ -456,7 +595,7 @@ test_encoding(void *arg)
  *  If <b>also_current</b> is set, then set both current and previous SRVs.
  *  Otherwise, just set the previous SRV. (And clear the current SRV.)
  *
- * You must call sr_state_free() to free the state at the end of each test
+ * You must call sr_state_free_all() to free the state at the end of each test
  * function (on pass or fail). */
 static void
 test_sr_setup_srv(int also_current)
@@ -529,9 +668,9 @@ test_vote(void *arg)
     tt_assert(lines);
     /* Split the lines. We expect 2 here. */
     ret = smartlist_split_string(chunks, lines, "\n", SPLIT_IGNORE_BLANK, 0);
-    tt_int_op(ret, ==, 4);
+    tt_int_op(ret, OP_EQ, 4);
     tt_str_op(smartlist_get(chunks, 0), OP_EQ, "shared-rand-participate");
-    /* Get our commitment line and will validate it agains our commit. The
+    /* Get our commitment line and will validate it against our commit. The
      * format is as follow:
      * "shared-rand-commitment" SP version SP algname SP identity
      *                          SP COMMIT [SP REVEAL] NL
@@ -539,7 +678,7 @@ test_vote(void *arg)
     char *commit_line = smartlist_get(chunks, 1);
     tt_assert(commit_line);
     ret = smartlist_split_string(tokens, commit_line, " ", 0, 0);
-    tt_int_op(ret, ==, 6);
+    tt_int_op(ret, OP_EQ, 6);
     tt_str_op(smartlist_get(tokens, 0), OP_EQ, "shared-rand-commit");
     tt_str_op(smartlist_get(tokens, 1), OP_EQ, "1");
     tt_str_op(smartlist_get(tokens, 2), OP_EQ,
@@ -547,7 +686,7 @@ test_vote(void *arg)
     char digest[DIGEST_LEN];
     base16_decode(digest, sizeof(digest), smartlist_get(tokens, 3),
                   HEX_DIGEST_LEN);
-    tt_mem_op(digest, ==, our_commit->rsa_identity, sizeof(digest));
+    tt_mem_op(digest, OP_EQ, our_commit->rsa_identity, sizeof(digest));
     tt_str_op(smartlist_get(tokens, 4), OP_EQ, our_commit->encoded_commit);
     tt_str_op(smartlist_get(tokens, 5), OP_EQ, our_commit->encoded_reveal)
 ;
@@ -563,7 +702,7 @@ test_vote(void *arg)
     /* Set valid flag explicitly here to compare since it's not set by
      * simply parsing the commit. */
     parsed_commit->valid = 1;
-    tt_mem_op(parsed_commit, ==, our_commit, sizeof(*our_commit));
+    tt_mem_op(parsed_commit, OP_EQ, our_commit, sizeof(*our_commit));
 
     /* minor cleanup */
     SMARTLIST_FOREACH(tokens, char *, s, tor_free(s));
@@ -573,7 +712,7 @@ test_vote(void *arg)
     char *prev_srv_line = smartlist_get(chunks, 2);
     tt_assert(prev_srv_line);
     ret = smartlist_split_string(tokens, prev_srv_line, " ", 0, 0);
-    tt_int_op(ret, ==, 3);
+    tt_int_op(ret, OP_EQ, 3);
     tt_str_op(smartlist_get(tokens, 0), OP_EQ, "shared-rand-previous-value");
     tt_str_op(smartlist_get(tokens, 1), OP_EQ, "42");
     tt_str_op(smartlist_get(tokens, 2), OP_EQ,
@@ -587,7 +726,7 @@ test_vote(void *arg)
     char *current_srv_line = smartlist_get(chunks, 3);
     tt_assert(current_srv_line);
     ret = smartlist_split_string(tokens, current_srv_line, " ", 0, 0);
-    tt_int_op(ret, ==, 3);
+    tt_int_op(ret, OP_EQ, 3);
     tt_str_op(smartlist_get(tokens, 0), OP_EQ, "shared-rand-current-value");
     tt_str_op(smartlist_get(tokens, 1), OP_EQ, "128");
     tt_str_op(smartlist_get(tokens, 2), OP_EQ,
@@ -606,7 +745,7 @@ test_vote(void *arg)
 
  done:
   UNMOCK(trusteddirserver_get_by_v3_auth_digest);
-  sr_state_free();
+  sr_state_free_all();
 }
 
 static const char *sr_state_str = "Version 1\n"
@@ -640,7 +779,7 @@ test_state_load_from_disk(void *arg)
 
   /* First try with a nonexistent path. */
   ret = disk_state_load_from_disk_impl("NONEXISTENTNONEXISTENT");
-  tt_assert(ret == -ENOENT);
+  tt_int_op(ret, OP_EQ, -ENOENT);
 
   /* Now create a mock state directory and state file */
 #ifdef _WIN32
@@ -648,9 +787,9 @@ test_state_load_from_disk(void *arg)
 #else
   ret = mkdir(dir, 0700);
 #endif
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = write_str_to_file(sr_state_path, sr_state_str, 0);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Try to load the directory itself. Should fail. */
   ret = disk_state_load_from_disk_impl(dir);
@@ -658,11 +797,11 @@ test_state_load_from_disk(void *arg)
 
   /* State should be non-existent at this point. */
   the_sr_state = get_sr_state();
-  tt_assert(!the_sr_state);
+  tt_ptr_op(the_sr_state, OP_EQ, NULL);
 
   /* Now try to load the correct file! */
   ret = disk_state_load_from_disk_impl(sr_state_path);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Check the content of the state */
   /* XXX check more deeply!!! */
@@ -700,7 +839,7 @@ test_sr_setup_commits(void)
     tt_assert(auth_cert);
 
     options->AuthoritativeDir = 1;
-    tt_int_op(0, ==, load_ed_keys(options, now));
+    tt_int_op(0, OP_EQ, load_ed_keys(options, now));
   }
 
   /* Generate three dummy commits according to sr_srv_calc_ref.py .  Then
@@ -786,7 +925,7 @@ test_sr_setup_commits(void)
   save_commit_to_state(commit_b);
   save_commit_to_state(commit_c);
   save_commit_to_state(commit_d);
-  tt_int_op(digestmap_size(get_sr_state()->commits), ==, 4);
+  tt_int_op(digestmap_size(get_sr_state()->commits), OP_EQ, 4);
 
   /* Now during REVEAL phase save commit D by restoring its reveal. */
   set_sr_phase(SR_PHASE_REVEAL);
@@ -833,14 +972,14 @@ test_sr_compute_srv(void *arg)
   /* Check the result against the test vector */
   current_srv = sr_state_get_current_srv();
   tt_assert(current_srv);
-  tt_u64_op(current_srv->num_reveals, ==, 3);
+  tt_u64_op(current_srv->num_reveals, OP_EQ, 3);
   tt_str_op(hex_str((char*)current_srv->value, 32),
-            ==,
+            OP_EQ,
             SRV_TEST_VECTOR);
 
  done:
   UNMOCK(trusteddirserver_get_by_v3_auth_digest);
-  sr_state_free();
+  sr_state_free_all();
 }
 
 /** Return a minimal vote document with a current SRV value set to
@@ -892,7 +1031,7 @@ test_sr_get_majority_srv_from_votes(void *arg)
   /* Since it's only one vote with an SRV, it should not achieve majority and
      hence no SRV will be returned. */
   chosen_srv = get_majority_srv_from_votes(votes, 1);
-  tt_assert(!chosen_srv);
+  tt_ptr_op(chosen_srv, OP_EQ, NULL);
 
   { /* Now put in 8 more votes. Let SRV_1 have majority. */
     int i;
@@ -911,21 +1050,21 @@ test_sr_get_majority_srv_from_votes(void *arg)
       smartlist_add(votes, vote);
     }
 
-    tt_int_op(smartlist_len(votes), ==, 9);
+    tt_int_op(smartlist_len(votes), OP_EQ, 9);
   }
 
   /* Now we achieve majority for SRV_1, but not the AuthDirNumSRVAgreements
      requirement. So still not picking an SRV. */
   set_num_srv_agreements(8);
   chosen_srv = get_majority_srv_from_votes(votes, 1);
-  tt_assert(!chosen_srv);
+  tt_ptr_op(chosen_srv, OP_EQ, NULL);
 
   /* We will now lower the AuthDirNumSRVAgreements requirement by tweaking the
    * consensus parameter and we will try again. This time it should work. */
   set_num_srv_agreements(7);
   chosen_srv = get_majority_srv_from_votes(votes, 1);
   tt_assert(chosen_srv);
-  tt_u64_op(chosen_srv->num_reveals, ==, 42);
+  tt_u64_op(chosen_srv->num_reveals, OP_EQ, 42);
   tt_mem_op(chosen_srv->value, OP_EQ, SRV_1, sizeof(chosen_srv->value));
 
  done:
@@ -949,7 +1088,7 @@ test_utils(void *arg)
     memcpy(srv->value, srv_value, sizeof(srv->value));
     dup_srv = srv_dup(srv);
     tt_assert(dup_srv);
-    tt_u64_op(dup_srv->num_reveals, ==, srv->num_reveals);
+    tt_u64_op(dup_srv->num_reveals, OP_EQ, srv->num_reveals);
     tt_mem_op(dup_srv->value, OP_EQ, srv->value, sizeof(srv->value));
     tor_free(srv);
     tor_free(dup_srv);
@@ -969,10 +1108,10 @@ test_utils(void *arg)
     sr_commit_t commit1, commit2;
     memcpy(commit1.encoded_commit, payload, sizeof(commit1.encoded_commit));
     memcpy(commit2.encoded_commit, payload, sizeof(commit2.encoded_commit));
-    tt_int_op(commitments_are_the_same(&commit1, &commit2), ==, 1);
+    tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 1);
     /* Let's corrupt one of them. */
     memset(commit1.encoded_commit, 'A', sizeof(commit1.encoded_commit));
-    tt_int_op(commitments_are_the_same(&commit1, &commit2), ==, 0);
+    tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 0);
   }
 
   /* Testing commit_is_authoritative(). */
@@ -983,32 +1122,32 @@ test_utils(void *arg)
 
     tt_assert(!crypto_pk_generate_key(k));
 
-    tt_int_op(0, ==, crypto_pk_get_digest(k, digest));
+    tt_int_op(0, OP_EQ, crypto_pk_get_digest(k, digest));
     memcpy(commit.rsa_identity, digest, sizeof(commit.rsa_identity));
-    tt_int_op(commit_is_authoritative(&commit, digest), ==, 1);
+    tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 1);
     /* Change the pubkey. */
     memset(commit.rsa_identity, 0, sizeof(commit.rsa_identity));
-    tt_int_op(commit_is_authoritative(&commit, digest), ==, 0);
+    tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 0);
     crypto_pk_free(k);
   }
 
   /* Testing get_phase_str(). */
   {
-    tt_str_op(get_phase_str(SR_PHASE_REVEAL), ==, "reveal");
-    tt_str_op(get_phase_str(SR_PHASE_COMMIT), ==, "commit");
+    tt_str_op(get_phase_str(SR_PHASE_REVEAL), OP_EQ, "reveal");
+    tt_str_op(get_phase_str(SR_PHASE_COMMIT), OP_EQ, "commit");
   }
 
   /* Testing phase transition */
   {
     init_authority_state();
     set_sr_phase(SR_PHASE_COMMIT);
-    tt_int_op(is_phase_transition(SR_PHASE_REVEAL), ==, 1);
-    tt_int_op(is_phase_transition(SR_PHASE_COMMIT), ==, 0);
+    tt_int_op(is_phase_transition(SR_PHASE_REVEAL), OP_EQ, 1);
+    tt_int_op(is_phase_transition(SR_PHASE_COMMIT), OP_EQ, 0);
     set_sr_phase(SR_PHASE_REVEAL);
-    tt_int_op(is_phase_transition(SR_PHASE_REVEAL), ==, 0);
-    tt_int_op(is_phase_transition(SR_PHASE_COMMIT), ==, 1);
+    tt_int_op(is_phase_transition(SR_PHASE_REVEAL), OP_EQ, 0);
+    tt_int_op(is_phase_transition(SR_PHASE_COMMIT), OP_EQ, 1);
     /* Junk. */
-    tt_int_op(is_phase_transition(42), ==, 1);
+    tt_int_op(is_phase_transition(42), OP_EQ, 1);
   }
 
  done:
@@ -1036,24 +1175,24 @@ test_state_transition(void *arg)
     sr_commit_t *commit = sr_generate_our_commit(now, mock_cert);
     tt_assert(commit);
     sr_state_add_commit(commit);
-    tt_int_op(digestmap_size(state->commits), ==, 1);
+    tt_int_op(digestmap_size(state->commits), OP_EQ, 1);
     /* Let's test our delete feature. */
     sr_state_delete_commits();
-    tt_int_op(digestmap_size(state->commits), ==, 0);
+    tt_int_op(digestmap_size(state->commits), OP_EQ, 0);
     /* Add it back so we can continue the rest of the test because after
      * deletiong our commit will be freed so generate a new one. */
     commit = sr_generate_our_commit(now, mock_cert);
     tt_assert(commit);
     sr_state_add_commit(commit);
-    tt_int_op(digestmap_size(state->commits), ==, 1);
+    tt_int_op(digestmap_size(state->commits), OP_EQ, 1);
     state->n_reveal_rounds = 42;
     state->n_commit_rounds = 43;
     state->n_protocol_runs = 44;
     reset_state_for_new_protocol_run(now);
-    tt_int_op(state->n_reveal_rounds, ==, 0);
-    tt_int_op(state->n_commit_rounds, ==, 0);
-    tt_u64_op(state->n_protocol_runs, ==, 45);
-    tt_int_op(digestmap_size(state->commits), ==, 0);
+    tt_int_op(state->n_reveal_rounds, OP_EQ, 0);
+    tt_int_op(state->n_commit_rounds, OP_EQ, 0);
+    tt_u64_op(state->n_protocol_runs, OP_EQ, 45);
+    tt_int_op(digestmap_size(state->commits), OP_EQ, 0);
   }
 
   /* Test SRV rotation in our state. */
@@ -1066,7 +1205,7 @@ test_state_transition(void *arg)
     state_rotate_srv();
     prev = sr_state_get_previous_srv();
     tt_assert(prev == cur);
-    tt_assert(!sr_state_get_current_srv());
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
     sr_state_clean_srvs();
   }
 
@@ -1091,22 +1230,22 @@ test_state_transition(void *arg)
     /* Also, make sure we did change the current. */
     tt_assert(sr_state_get_current_srv() != cur);
     /* We should have our commitment alone. */
-    tt_int_op(digestmap_size(state->commits), ==, 1);
-    tt_int_op(state->n_reveal_rounds, ==, 0);
-    tt_int_op(state->n_commit_rounds, ==, 0);
+    tt_int_op(digestmap_size(state->commits), OP_EQ, 1);
+    tt_int_op(state->n_reveal_rounds, OP_EQ, 0);
+    tt_int_op(state->n_commit_rounds, OP_EQ, 0);
     /* 46 here since we were at 45 just before. */
-    tt_u64_op(state->n_protocol_runs, ==, 46);
+    tt_u64_op(state->n_protocol_runs, OP_EQ, 46);
   }
 
   /* Cleanup of SRVs. */
   {
     sr_state_clean_srvs();
-    tt_assert(!sr_state_get_current_srv());
-    tt_assert(!sr_state_get_previous_srv());
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
   }
 
  done:
-  sr_state_free();
+  sr_state_free_all();
 }
 
 static void
@@ -1131,6 +1270,8 @@ test_keep_commit(void *arg)
     state = get_sr_state();
   }
 
+  crypto_rand((char*)fp, sizeof(fp));
+
   /* Test this very important function that tells us if we should keep a
    * commit or not in our state. Most of it depends on the phase and what's
    * in the commit so we'll change the commit as we go. */
@@ -1139,21 +1280,21 @@ test_keep_commit(void *arg)
   /* Set us in COMMIT phase for starter. */
   set_sr_phase(SR_PHASE_COMMIT);
   /* We should never keep a commit from a non authoritative authority. */
-  tt_int_op(should_keep_commit(commit, fp, SR_PHASE_COMMIT), ==, 0);
+  tt_int_op(should_keep_commit(commit, fp, SR_PHASE_COMMIT), OP_EQ, 0);
   /* This should NOT be kept because it has a reveal value in it. */
   tt_assert(commit_has_reveal_value(commit));
   tt_int_op(should_keep_commit(commit, commit->rsa_identity,
-                               SR_PHASE_COMMIT), ==, 0);
+                               SR_PHASE_COMMIT), OP_EQ, 0);
   /* Add it to the state which should return to not keep it. */
   sr_state_add_commit(commit);
   tt_int_op(should_keep_commit(commit, commit->rsa_identity,
-                               SR_PHASE_COMMIT), ==, 0);
+                               SR_PHASE_COMMIT), OP_EQ, 0);
   /* Remove it from state so we can continue our testing. */
   digestmap_remove(state->commits, commit->rsa_identity);
   /* Let's remove our reveal value which should make it OK to keep it. */
   memset(commit->encoded_reveal, 0, sizeof(commit->encoded_reveal));
   tt_int_op(should_keep_commit(commit, commit->rsa_identity,
-                               SR_PHASE_COMMIT), ==, 1);
+                               SR_PHASE_COMMIT), OP_EQ, 1);
 
   /* Let's reset our commit and go into REVEAL phase. */
   sr_commit_free(commit);
@@ -1165,17 +1306,17 @@ test_keep_commit(void *arg)
   memset(dup_commit->encoded_reveal, 0, sizeof(dup_commit->encoded_reveal));
   set_sr_phase(SR_PHASE_REVEAL);
   /* We should never keep a commit from a non authoritative authority. */
-  tt_int_op(should_keep_commit(commit, fp, SR_PHASE_REVEAL), ==, 0);
+  tt_int_op(should_keep_commit(commit, fp, SR_PHASE_REVEAL), OP_EQ, 0);
   /* We shouldn't accept a commit that is not in our state. */
   tt_int_op(should_keep_commit(commit, commit->rsa_identity,
-                               SR_PHASE_REVEAL), ==, 0);
+                               SR_PHASE_REVEAL), OP_EQ, 0);
   /* Important to add the commit _without_ the reveal here. */
   sr_state_add_commit(dup_commit);
-  tt_int_op(digestmap_size(state->commits), ==, 1);
+  tt_int_op(digestmap_size(state->commits), OP_EQ, 1);
   /* Our commit should be valid that is authoritative, contains a reveal, be
    * in the state and commitment and reveal values match. */
   tt_int_op(should_keep_commit(commit, commit->rsa_identity,
-                               SR_PHASE_REVEAL), ==, 1);
+                               SR_PHASE_REVEAL), OP_EQ, 1);
   /* The commit shouldn't be kept if it's not verified that is no matchin
    * hashed reveal. */
   {
@@ -1186,22 +1327,22 @@ test_keep_commit(void *arg)
     memset(commit->hashed_reveal, 0, sizeof(commit->hashed_reveal));
     setup_full_capture_of_logs(LOG_WARN);
     tt_int_op(should_keep_commit(commit, commit->rsa_identity,
-                                 SR_PHASE_REVEAL), ==, 0);
+                                 SR_PHASE_REVEAL), OP_EQ, 0);
     expect_log_msg_containing("doesn't match the commit value.");
     expect_log_msg_containing("has an invalid reveal value.");
     assert_log_predicate(mock_saved_log_n_entries() == 2,
-                         "expected 2 log entries");
+                         ("expected 2 log entries"));
     teardown_capture_of_logs();
     memcpy(commit->hashed_reveal, place_holder.hashed_reveal,
            sizeof(commit->hashed_reveal));
   }
   /* We shouldn't keep a commit that has no reveal. */
   tt_int_op(should_keep_commit(dup_commit, dup_commit->rsa_identity,
-                               SR_PHASE_REVEAL), ==, 0);
+                               SR_PHASE_REVEAL), OP_EQ, 0);
   /* We must not keep a commit that is not the same from the commit phase. */
   memset(commit->encoded_commit, 0, sizeof(commit->encoded_commit));
   tt_int_op(should_keep_commit(commit, commit->rsa_identity,
-                               SR_PHASE_REVEAL), ==, 0);
+                               SR_PHASE_REVEAL), OP_EQ, 0);
 
  done:
   teardown_capture_of_logs();
@@ -1239,39 +1380,39 @@ test_state_update(void *arg)
   /* We are in COMMIT phase here and we'll trigger a state update but no
    * transition. */
   sr_state_update(commit_phase_time);
-  tt_int_op(state->valid_after, ==, commit_phase_time);
-  tt_int_op(state->n_commit_rounds, ==, 1);
-  tt_int_op(state->phase, ==, SR_PHASE_COMMIT);
-  tt_int_op(digestmap_size(state->commits), ==, 1);
+  tt_int_op(state->valid_after, OP_EQ, commit_phase_time);
+  tt_int_op(state->n_commit_rounds, OP_EQ, 1);
+  tt_int_op(state->phase, OP_EQ, SR_PHASE_COMMIT);
+  tt_int_op(digestmap_size(state->commits), OP_EQ, 1);
 
   /* We are still in the COMMIT phase here but we'll trigger a state
    * transition to the REVEAL phase. */
   sr_state_update(reveal_phase_time);
-  tt_int_op(state->phase, ==, SR_PHASE_REVEAL);
-  tt_int_op(state->valid_after, ==, reveal_phase_time);
+  tt_int_op(state->phase, OP_EQ, SR_PHASE_REVEAL);
+  tt_int_op(state->valid_after, OP_EQ, reveal_phase_time);
   /* Only our commit should be in there. */
-  tt_int_op(digestmap_size(state->commits), ==, 1);
-  tt_int_op(state->n_reveal_rounds, ==, 1);
+  tt_int_op(digestmap_size(state->commits), OP_EQ, 1);
+  tt_int_op(state->n_reveal_rounds, OP_EQ, 1);
 
   /* We can't update a state with a valid after _lower_ than the creation
    * time so here it is. */
   sr_state_update(commit_phase_time);
-  tt_int_op(state->valid_after, ==, reveal_phase_time);
+  tt_int_op(state->valid_after, OP_EQ, reveal_phase_time);
 
   /* Finally, let's go back in COMMIT phase so we can test the state update
    * of a new protocol run. */
   state->valid_after = 0;
   sr_state_update(commit_phase_time);
-  tt_int_op(state->valid_after, ==, commit_phase_time);
-  tt_int_op(state->n_commit_rounds, ==, 1);
-  tt_int_op(state->n_reveal_rounds, ==, 0);
-  tt_u64_op(state->n_protocol_runs, ==, 1);
-  tt_int_op(state->phase, ==, SR_PHASE_COMMIT);
-  tt_int_op(digestmap_size(state->commits), ==, 1);
+  tt_int_op(state->valid_after, OP_EQ, commit_phase_time);
+  tt_int_op(state->n_commit_rounds, OP_EQ, 1);
+  tt_int_op(state->n_reveal_rounds, OP_EQ, 0);
+  tt_u64_op(state->n_protocol_runs, OP_EQ, 1);
+  tt_int_op(state->phase, OP_EQ, SR_PHASE_COMMIT);
+  tt_int_op(digestmap_size(state->commits), OP_EQ, 1);
   tt_assert(state->current_srv);
 
  done:
-  sr_state_free();
+  sr_state_free_all();
   UNMOCK(get_my_v3_authority_cert);
 }
 
@@ -1284,7 +1425,11 @@ struct testcase_t sr_tests[] = {
     NULL, NULL },
   { "encoding", test_encoding, TT_FORK,
     NULL, NULL },
-  { "get_next_valid_after_time", test_get_next_valid_after_time, TT_FORK,
+  { "get_start_time_of_current_run", test_get_start_time_of_current_run,
+    TT_FORK, NULL, NULL },
+  { "get_start_time_functions", test_get_start_time_functions,
+    TT_FORK, NULL, NULL },
+  { "get_sr_protocol_duration", test_get_sr_protocol_duration, TT_FORK,
     NULL, NULL },
   { "get_state_valid_until_time", test_get_state_valid_until_time, TT_FORK,
     NULL, NULL },
@@ -1301,4 +1446,3 @@ struct testcase_t sr_tests[] = {
     NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_slow.c b/src/test/test_slow.c
index 7c9f0b1cc2..bda67b2d92 100644
--- a/src/test/test_slow.c
+++ b/src/test/test_slow.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,8 +15,8 @@
 #include <fcntl.h>
 #endif
 
-#include "or.h"
-#include "test.h"
+#include "core/or/or.h"
+#include "test/test.h"
 
 struct testgroup_t testgroups[] = {
   { "slow/crypto/", slow_crypto_tests },
diff --git a/src/test/test_socks.c b/src/test/test_socks.c
index 62ff12fe15..783f4726ee 100644
--- a/src/test/test_socks.c
+++ b/src/test/test_socks.c
@@ -1,12 +1,17 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#include "or.h"
-#include "buffers.h"
-#include "config.h"
-#include "test.h"
+#include "core/or/or.h"
+#include "lib/container/buffers.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/proto/proto_socks.h"
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+#include "core/or/socks_request_st.h"
+#include "lib/net/socks5_status.h"
 
 typedef struct socks_test_data_t {
   socks_request_t *req;
@@ -43,7 +48,7 @@ static const struct testcase_setup_t socks_setup = {
   buf_t *buf = testdata->buf;                   \
   socks_request_t *socks = testdata->req;
 #define ADD_DATA(buf, s)                                        \
-  write_to_buf(s, sizeof(s)-1, buf)
+  buf_add(buf, s, sizeof(s)-1)
 
 static void
 socks_request_clear(socks_request_t *socks)
@@ -61,8 +66,9 @@ test_socks_4_unsupported_commands(void *ptr)
 
   /* SOCKS 4 Send BIND [02] to IP address 2.2.2.2:4369 */
   ADD_DATA(buf, "\x04\x02\x11\x11\x02\x02\x02\x02\x00");
-  tt_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
-                                   get_options()->SafeSocks) == -1);
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, -1);
   tt_int_op(4,OP_EQ, socks->socks_version);
   tt_int_op(0,OP_EQ, socks->replylen); /* XXX: shouldn't tor reply? */
 
@@ -78,10 +84,11 @@ test_socks_4_supported_commands(void *ptr)
 
   tt_int_op(0,OP_EQ, buf_datalen(buf));
 
-  /* SOCKS 4 Send CONNECT [01] to IP address 2.2.2.2:4370 */
+  /* SOCKS 4 Send CONNECT [01] to IP address 2.2.2.3:4370 */
   ADD_DATA(buf, "\x04\x01\x11\x12\x02\x02\x02\x03\x00");
-  tt_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
-                                   get_options()->SafeSocks) == 1);
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, 1);
   tt_int_op(4,OP_EQ, socks->socks_version);
   tt_int_op(0,OP_EQ, socks->replylen); /* XXX: shouldn't tor reply? */
   tt_int_op(SOCKS_COMMAND_CONNECT,OP_EQ, socks->command);
@@ -93,10 +100,10 @@ test_socks_4_supported_commands(void *ptr)
   tt_int_op(0,OP_EQ, buf_datalen(buf));
   socks_request_clear(socks);
 
-  /* SOCKS 4 Send CONNECT [01] to IP address 2.2.2.2:4369 with userid*/
+  /* SOCKS 4 Send CONNECT [01] to IP address 2.2.2.4:4369 with userid*/
   ADD_DATA(buf, "\x04\x01\x11\x12\x02\x02\x02\x04me\x00");
-  tt_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
-                                   get_options()->SafeSocks) == 1);
+  tt_int_op(fetch_from_buf_socks(buf, socks, 1, 0),
+            OP_EQ, 1);
   tt_int_op(4,OP_EQ, socks->socks_version);
   tt_int_op(0,OP_EQ, socks->replylen); /* XXX: shouldn't tor reply? */
   tt_int_op(SOCKS_COMMAND_CONNECT,OP_EQ, socks->command);
@@ -112,8 +119,9 @@ test_socks_4_supported_commands(void *ptr)
 
   /* SOCKS 4a Send RESOLVE [F0] request for torproject.org */
   ADD_DATA(buf, "\x04\xF0\x01\x01\x00\x00\x00\x02me\x00torproject.org\x00");
-  tt_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
-                                   get_options()->SafeSocks) == 1);
+  tt_int_op(fetch_from_buf_socks(buf, socks, 1,
+                                 get_options()->SafeSocks),
+            OP_EQ, 1);
   tt_int_op(4,OP_EQ, socks->socks_version);
   tt_int_op(0,OP_EQ, socks->replylen); /* XXX: shouldn't tor reply? */
   tt_str_op("torproject.org",OP_EQ, socks->address);
@@ -124,6 +132,83 @@ test_socks_4_supported_commands(void *ptr)
   ;
 }
 
+static void
+test_socks_4_bad_arguments(void *ptr)
+{
+  SOCKS_TEST_INIT();
+  setup_capture_of_logs(LOG_DEBUG);
+
+  /* Try with 0 IPv4 address */
+  ADD_DATA(buf, "\x04\x01\x00\x50\x00\x00\x00\x00\x00");
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, -1);
+  buf_clear(buf);
+  expect_log_msg_containing("Port or DestIP is zero.");
+  mock_clean_saved_logs();
+
+  /* Try with 0 port */
+  ADD_DATA(buf, "\x04\x01\x00\x00\x01\x02\x03\x04\x00");
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, -1);
+  buf_clear(buf);
+  expect_log_msg_containing("Port or DestIP is zero.");
+  mock_clean_saved_logs();
+
+  /* Try with 2000-byte username (!) */
+  ADD_DATA(buf, "\x04\x01\x00\x50\x01\x02\x03\x04");
+  int i;
+  for (i = 0; i < 200; ++i) {
+    ADD_DATA(buf, "1234567890");
+  }
+  ADD_DATA(buf, "\x00");
+  tt_int_op(fetch_from_buf_socks(buf, socks, 1, 0),
+            OP_EQ, -1);
+  buf_clear(buf);
+  expect_log_msg_containing("socks4: parsing failed - invalid request.");
+  mock_clean_saved_logs();
+
+  /* Try with 2000-byte hostname */
+  ADD_DATA(buf, "\x04\x01\x00\x50\x00\x00\x00\x01\x00");
+  for (i = 0; i < 200; ++i) {
+    ADD_DATA(buf, "1234567890");
+  }
+  ADD_DATA(buf, "\x00");
+  {
+    const char *p;
+    size_t s;
+    buf_pullup(buf, 9999, &p, &s);
+  }
+  tt_int_op(fetch_from_buf_socks(buf, socks, 1, 0),
+            OP_EQ, -1);
+  buf_clear(buf);
+  expect_log_msg_containing("Destaddr too long. Rejecting.");
+  mock_clean_saved_logs();
+
+  /* Try with 2000-byte hostname, not terminated. */
+  ADD_DATA(buf, "\x04\x01\x00\x50\x00\x00\x00\x01\x00");
+  for (i = 0; i < 200; ++i) {
+    ADD_DATA(buf, "1234567890");
+  }
+  tt_int_op(fetch_from_buf_socks(buf, socks, 1, 0),
+            OP_EQ, -1);
+  buf_clear(buf);
+  expect_log_msg_containing("parsing failed - invalid request.");
+  mock_clean_saved_logs();
+
+  /* Socks4, bogus hostname */
+  ADD_DATA(buf, "\x04\x01\x00\x50\x00\x00\x00\x01\x00" "---\x00" );
+  tt_int_op(fetch_from_buf_socks(buf, socks, 1, 0), OP_EQ, -1);
+  buf_clear(buf);
+  expect_log_msg_containing("Your application (using socks4 to port 80) "
+                            "gave Tor a malformed hostname: ");
+  mock_clean_saved_logs();
+
+ done:
+  teardown_capture_of_logs();
+}
+
 /**  Perform unsupported SOCKS 5 commands */
 static void
 test_socks_5_unsupported_commands(void *ptr)
@@ -199,10 +284,28 @@ test_socks_5_supported_commands(void *ptr)
   tt_int_op(0,OP_EQ, buf_datalen(buf));
   socks_request_clear(socks);
 
+  /* SOCKS 5 Send CONNECT [01] to one of the ipv6 addresses for
+     torproject.org:80 */
+  ADD_DATA(buf, "\x05\x01\x00");
+  ADD_DATA(buf, "\x05\x01\x00\x04"
+           "\x20\x02\x41\xb8\x02\x02\x0d\xeb\x02\x13\x21\xff\xfe\x20\x14\x26"
+           "\x00\x50");
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                   get_options()->SafeSocks),OP_EQ, 1);
+  tt_int_op(5,OP_EQ, socks->socks_version);
+  tt_int_op(2,OP_EQ, socks->replylen);
+  tt_int_op(5,OP_EQ, socks->reply[0]);
+  tt_int_op(0,OP_EQ, socks->reply[1]);
+  tt_str_op("[2002:41b8:202:deb:213:21ff:fe20:1426]",OP_EQ, socks->address);
+  tt_int_op(80,OP_EQ, socks->port);
+
+  tt_int_op(0,OP_EQ, buf_datalen(buf));
+  socks_request_clear(socks);
+
   /* SOCKS 5 Send CONNECT [01] to FQDN torproject.org:4369 */
   ADD_DATA(buf, "\x05\x01\x00");
   ADD_DATA(buf, "\x05\x01\x00\x03\x0Etorproject.org\x11\x11");
-  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+  tt_int_op(fetch_from_buf_socks(buf, socks, 1,
                                    get_options()->SafeSocks),OP_EQ, 1);
 
   tt_int_op(5,OP_EQ, socks->socks_version);
@@ -218,8 +321,9 @@ test_socks_5_supported_commands(void *ptr)
   /* SOCKS 5 Send RESOLVE [F0] request for torproject.org:4369 */
   ADD_DATA(buf, "\x05\x01\x00");
   ADD_DATA(buf, "\x05\xF0\x00\x03\x0Etorproject.org\x01\x02");
-  tt_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
-                                   get_options()->SafeSocks) == 1);
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, 1);
   tt_int_op(5,OP_EQ, socks->socks_version);
   tt_int_op(2,OP_EQ, socks->replylen);
   tt_int_op(5,OP_EQ, socks->reply[0]);
@@ -229,47 +333,64 @@ test_socks_5_supported_commands(void *ptr)
   tt_int_op(0,OP_EQ, buf_datalen(buf));
   socks_request_clear(socks);
 
-  /* SOCKS 5 Should reject RESOLVE [F0] request for IPv4 address
+  /* SOCKS 5 Should NOT reject RESOLVE [F0] request for IPv4 address
    * string if SafeSocks is enabled. */
 
   ADD_DATA(buf, "\x05\x01\x00");
   ADD_DATA(buf, "\x05\xF0\x00\x03\x07");
   ADD_DATA(buf, "8.8.8.8");
-  ADD_DATA(buf, "\x01\x02");
-  tt_assert(fetch_from_buf_socks(buf,socks,get_options()->TestSocks,1)
-            == -1);
+  ADD_DATA(buf, "\x11\x11");
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, 1),
+            OP_EQ, 1);
 
-  tt_int_op(5,OP_EQ,socks->socks_version);
-  tt_int_op(10,OP_EQ,socks->replylen);
-  tt_int_op(5,OP_EQ,socks->reply[0]);
-  tt_int_op(SOCKS5_NOT_ALLOWED,OP_EQ,socks->reply[1]);
-  tt_int_op(1,OP_EQ,socks->reply[3]);
+  tt_str_op("8.8.8.8", OP_EQ, socks->address);
+  tt_int_op(4369, OP_EQ, socks->port);
+
+  tt_int_op(0, OP_EQ, buf_datalen(buf));
 
   socks_request_clear(socks);
 
-  /* SOCKS 5 should reject RESOLVE [F0] reject for IPv6 address
+  /* SOCKS 5 should NOT reject RESOLVE [F0] request for IPv6 address
    * string if SafeSocks is enabled. */
 
   ADD_DATA(buf, "\x05\x01\x00");
+  ADD_DATA(buf, "\x05\xF0\x00\x03\x29");
+  ADD_DATA(buf, "[2001:0db8:85a3:0000:0000:8a2e:0370:7334]");
+  ADD_DATA(buf, "\x01\x02");
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, 1),
+            OP_EQ, 1);
+
+  tt_str_op("[2001:0db8:85a3:0000:0000:8a2e:0370:7334]", OP_EQ,
+    socks->address);
+  tt_int_op(258, OP_EQ, socks->port);
+
+  tt_int_op(0, OP_EQ, buf_datalen(buf));
+
+  socks_request_clear(socks);
+
+  /* Also allow bracket-less form. */
+
+  ADD_DATA(buf, "\x05\x01\x00");
   ADD_DATA(buf, "\x05\xF0\x00\x03\x27");
   ADD_DATA(buf, "2001:0db8:85a3:0000:0000:8a2e:0370:7334");
   ADD_DATA(buf, "\x01\x02");
-  tt_assert(fetch_from_buf_socks(buf,socks,get_options()->TestSocks,1)
-            == -1);
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, 1),
+            OP_EQ, 1);
 
-  tt_int_op(5,OP_EQ,socks->socks_version);
-  tt_int_op(10,OP_EQ,socks->replylen);
-  tt_int_op(5,OP_EQ,socks->reply[0]);
-  tt_int_op(SOCKS5_NOT_ALLOWED,OP_EQ,socks->reply[1]);
-  tt_int_op(1,OP_EQ,socks->reply[3]);
+  tt_str_op("2001:0db8:85a3:0000:0000:8a2e:0370:7334", OP_EQ,
+    socks->address);
+  tt_int_op(258, OP_EQ, socks->port);
+
+  tt_int_op(0, OP_EQ, buf_datalen(buf));
 
   socks_request_clear(socks);
 
   /* SOCKS 5 Send RESOLVE_PTR [F1] for IP address 2.2.2.5 */
   ADD_DATA(buf, "\x05\x01\x00");
   ADD_DATA(buf, "\x05\xF1\x00\x01\x02\x02\x02\x05\x01\x03");
-  tt_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
-                                   get_options()->SafeSocks) == 1);
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, 1);
   tt_int_op(5,OP_EQ, socks->socks_version);
   tt_int_op(2,OP_EQ, socks->replylen);
   tt_int_op(5,OP_EQ, socks->reply[0]);
@@ -358,6 +479,44 @@ test_socks_5_authenticate(void *ptr)
   ;
 }
 
+/** Perform SOCKS 5 authentication with empty username/password fields.
+ * Technically this violates RfC 1929, but some client software will send
+ * this kind of message to Tor.
+ * */
+static void
+test_socks_5_authenticate_empty_user_pass(void *ptr)
+{
+  SOCKS_TEST_INIT();
+
+  /* SOCKS 5 Negotiate username/password authentication */
+  ADD_DATA(buf, "\x05\x01\x02");
+
+  tt_assert(!fetch_from_buf_socks(buf, socks,
+                                   get_options()->TestSocks,
+                                   get_options()->SafeSocks));
+  tt_int_op(2,OP_EQ, socks->replylen);
+  tt_int_op(5,OP_EQ, socks->reply[0]);
+  tt_int_op(SOCKS_USER_PASS,OP_EQ, socks->reply[1]);
+  tt_int_op(5,OP_EQ, socks->socks_version);
+
+  tt_int_op(0,OP_EQ, buf_datalen(buf));
+
+  /* SOCKS 5 Send username/password auth message with empty user/pass fields */
+  ADD_DATA(buf, "\x01\x00\x00");
+  tt_assert(!fetch_from_buf_socks(buf, socks,
+                                   get_options()->TestSocks,
+                                   get_options()->SafeSocks));
+  tt_int_op(5,OP_EQ, socks->socks_version);
+  tt_int_op(2,OP_EQ, socks->replylen);
+  tt_int_op(1,OP_EQ, socks->reply[0]);
+  tt_int_op(0,OP_EQ, socks->reply[1]);
+
+  tt_int_op(0,OP_EQ, socks->usernamelen);
+  tt_int_op(0,OP_EQ, socks->passwordlen);
+
+ done:
+  ;
+}
 /** Perform SOCKS 5 authentication and send data all in one go */
 static void
 test_socks_5_authenticate_with_data(void *ptr)
@@ -380,9 +539,9 @@ test_socks_5_authenticate_with_data(void *ptr)
   /* SOCKS 5 Send username/password */
   /* SOCKS 5 Send CONNECT [01] to IP address 2.2.2.2:4369 */
   ADD_DATA(buf, "\x01\x02me\x03you\x05\x01\x00\x01\x02\x02\x02\x02\x11\x11");
-  tt_assert(fetch_from_buf_socks(buf, socks,
-                                   get_options()->TestSocks,
-                                   get_options()->SafeSocks) == 1);
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, 1);
   tt_int_op(5,OP_EQ, socks->socks_version);
   tt_int_op(2,OP_EQ, socks->replylen);
   tt_int_op(1,OP_EQ, socks->reply[0]);
@@ -400,6 +559,48 @@ test_socks_5_authenticate_with_data(void *ptr)
   ;
 }
 
+/** Try to negotiate an unsupported authentication type */
+static void
+test_socks_5_auth_unsupported_type(void *ptr)
+{
+  SOCKS_TEST_INIT();
+
+  /* None of these authentication types are recognized. */
+  ADD_DATA(buf, "\x05\x03\x99\x21\x10");
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, -1);
+  tt_int_op(0,OP_EQ, socks->socks_version);
+  tt_int_op(2,OP_EQ, socks->replylen);
+  tt_int_op(5,OP_EQ, socks->reply[0]);
+  tt_int_op(0xff,OP_EQ, socks->reply[1]);
+
+ done:
+  ;
+}
+
+/** Try to negotiate an unsupported version of username/password auth. */
+static void
+test_socks_5_auth_unsupported_version(void *ptr)
+{
+  SOCKS_TEST_INIT();
+
+  /* Negotiate username/password */
+  ADD_DATA(buf, "\x05\x01\x02");
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, 0);
+  tt_int_op(0,OP_EQ, buf_datalen(buf)); /* buf should be drained */
+  /* Now, suggest an unrecognized username/password version */
+  ADD_DATA(buf, "\x02\x05" "hello" "\x05" "world");
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, -1);
+
+ done:
+  ;
+}
+
 /** Perform SOCKS 5 authentication before method negotiated */
 static void
 test_socks_5_auth_before_negotiation(void *ptr)
@@ -408,9 +609,9 @@ test_socks_5_auth_before_negotiation(void *ptr)
 
   /* SOCKS 5 Send username/password */
   ADD_DATA(buf, "\x01\x02me\x02me");
-  tt_assert(fetch_from_buf_socks(buf, socks,
-                                   get_options()->TestSocks,
-                                   get_options()->SafeSocks) == -1);
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, -1);
   tt_int_op(0,OP_EQ, socks->socks_version);
   tt_int_op(0,OP_EQ, socks->replylen);
   tt_int_op(0,OP_EQ, socks->reply[0]);
@@ -485,28 +686,406 @@ test_socks_5_malformed_commands(void *ptr)
   tt_int_op(5,OP_EQ,socks->socks_version);
   tt_int_op(10,OP_EQ,socks->replylen);
   tt_int_op(5,OP_EQ,socks->reply[0]);
-  tt_int_op(SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED,OP_EQ,socks->reply[1]);
+  /* trunnel parsing will fail with -1 */
+  tt_int_op(SOCKS5_GENERAL_ERROR,OP_EQ,socks->reply[1]);
   tt_int_op(1,OP_EQ,socks->reply[3]);
 
  done:
   ;
 }
 
+static void
+test_socks_5_bad_arguments(void *ptr)
+{
+  SOCKS_TEST_INIT();
+  setup_capture_of_logs(LOG_DEBUG);
+
+  /* Socks5, bogus hostname */
+  ADD_DATA(buf, "\x05\x01\x00" "\x05\x01\x00\x03\x03" "---" "\x00\x50" );
+  tt_int_op(fetch_from_buf_socks(buf, socks, 1, 0), OP_EQ, -1);
+  buf_clear(buf);
+  expect_log_msg_containing("Your application (using socks5 to port 80) "
+                            "gave Tor a malformed hostname: ");
+  mock_clean_saved_logs();
+  socks_request_clear(socks);
+
+ done:
+  teardown_capture_of_logs();
+}
+
+/** check for correct behavior when the socks command has not arrived. */
+static void
+test_socks_truncated(void *ptr)
+{
+  const struct {
+    enum { NONE, AUTH, ALL } setup;
+    const char *body;
+    size_t len;
+  } commands[] = {
+    /* SOCKS4 */
+    /* Connect, to an IP. */
+    { NONE, "\x04\x01\x05\x05\x01\x02\x03\x04\x00", 9},
+    /* Connect, to an IP, with authentication. */
+    { NONE, "\x04\x01\x05\x05\x01\x02\x03\x04hello\x00", 14},
+    /* SOCKS4A */
+    /* Connect, to a hostname */
+    { NONE, "\x04\x01\x09\x09\x00\x00\x00\x01\x00www.example.com\x00", 25},
+    /* Connect, to a hostname, with authentication */
+    { NONE, "\x04\x01\x09\x09\x00\x00\x00\x01hi\x00www.example.com\x00", 27},
+    /* SOCKS5 */
+    /* initial handshake */
+    { NONE, "\x05\x00", 2 },
+    /* no-auth handshake */
+    { NONE, "\x05\x03\x99\x21\x10", 5 },
+    /* SOCSK5, username-password, all empty. */
+    { AUTH, "\x01\x00\x00", 3 },
+    /* SOCSK5, username-password, 1 char each. */
+    { AUTH, "\x01\x01x\x01y", 5 },
+    /* SOCSK5, username-password, max length. */
+    { AUTH, "\x01\xff"
+      "Ogni tempo ha il suo fascismo: se ne notano i segni premonitori "
+      "dovunque la concentrazione di potere nega al cittadino la "
+      "possibilit\xc3\xa0 e la capacit\xc3\xa0 di esprimere ed attuare la "
+      "sua volont\xc3\xa0. A questo si arriva in molti modi, non "
+      "necessariamente col terror"
+      "\xff"
+      "e dell'intimidazione poliziesca, ma anche negando o distorcendo "
+      "l'informazione, inquinando la giustizia, paralizzando la scuola, "
+      "diffondendo in molti modi sottili la nostalgia per un mondo in cui "
+      "regnava sovrano l'ordine, ed in cui la sicurezza dei pochi "
+      /* privilegiati riposava sul lavoro forzato e sul silenzio forzato dei
+         molti. -- Primo Levi */ , 513 },
+    /* Socks5, IPv4 address */
+    { ALL, "\x05\x01\x00\x01\x01\x02\x03\x04\x20\x20", 10 },
+    /* Socks5, IPv6 address */
+    { ALL, "\x05\x01\x00\x04"
+      "\x49\x20\x48\x41\x5a\x20\x45\x41\x53\x54\x45\x52\x20\x45\x47\x47"
+      "\x20\x20", 22 },
+    /* Socks5, hostname, empty. */
+    { ALL, "\x05\x01\x00\x03" "\x00" "\x00\x50", 7 },
+    /* Socks5, hostname, moderate. */
+    { ALL, "\x05\x01\x00\x03" "\x11" "onion.example.com" "\x00\x50", 24 },
+    /* Socks5, hostname, maximum. */
+    { ALL, "\x05\x01\x00\x03" "\xff"
+      "whatsoever.I.shall.see.or.hear.in.the.course.of.my.profession.as.well."
+      "as.outside.my.profession.in.my.intercourse.with.men.if.it.be.what."
+      "should.not.be.published.abroad.I.will.never.divulge.holding.such."
+      "things.to.be.holy.secrets.x.hippocratic.oath.wikipedia"
+      "\x00\x50", 262 },
+  };
+  unsigned i, j;
+  SOCKS_TEST_INIT();
+  for (i = 0; i < ARRAY_LENGTH(commands); ++i) {
+    for (j = 0; j < commands[i].len; ++j) {
+      switch (commands[i].setup) {
+        default: /* Falls through */
+        case NONE:
+          /* This test calls for no setup on the socks state. */
+          break;
+        case AUTH:
+          /* This test calls for the socks state to be waiting for
+           * username/password authentication */
+          ADD_DATA(buf, "\x05\x01\x02");
+          tt_int_op(0, OP_EQ, fetch_from_buf_socks(buf, socks, 0, 0));
+          tt_int_op(0, OP_EQ, buf_datalen(buf));
+          break;
+        case ALL:
+          /* This test calls for the socks state to be waiting for
+           * the connection request */
+          ADD_DATA(buf, "\x05\x01\x00");
+          tt_int_op(0, OP_EQ, fetch_from_buf_socks(buf, socks, 0, 0));
+          tt_int_op(0, OP_EQ, buf_datalen(buf));
+      }
+
+      TT_BLATHER(("Checking command %u, length %u, omitting char %u", i, j,
+                  (unsigned)commands[i].body[j]));
+      buf_add(buf, commands[i].body, j);
+      /* This should return 0 meaning "not done yet" */
+      tt_int_op(0, OP_EQ, fetch_from_buf_socks(buf, socks, 0, 0));
+      tt_uint_op(j, OP_EQ, buf_datalen(buf)); /* Nothing was drained */
+      buf_clear(buf);
+      socks_request_free(testdata->req);
+      socks = testdata->req = socks_request_new();
+    }
+  }
+  done:
+  ;
+}
+
+static void
+test_socks_wrong_protocol(void *ptr)
+{
+  SOCKS_TEST_INIT();
+  setup_capture_of_logs(LOG_DEBUG);
+
+  /* HTTP request. */
+  ADD_DATA(buf, "GET /index.html HTTP/1.0" );
+  tt_int_op(fetch_from_buf_socks(buf, socks, 1, 0), OP_EQ, -1);
+  buf_clear(buf);
+  expect_log_msg_containing("Socks version 71 not recognized. "
+                            "(This port is not an HTTP proxy;");
+  mock_clean_saved_logs();
+  socks_request_clear(socks);
+
+ done:
+  teardown_capture_of_logs();
+}
+
+/* Check our client-side socks4 parsing (that is to say, our parsing of
+ * server responses).
+ */
+static void
+test_socks_client_v4(void *arg)
+{
+  (void)arg;
+  buf_t *buf = buf_new();
+  char *reason = NULL;
+
+  /* Legit socks4 response, success */
+  ADD_DATA(buf, "\x04\x5a\x20\x25\x01\x02\x03\x04");
+  tt_int_op(1, OP_EQ,
+            fetch_from_buf_socks_client(buf, PROXY_SOCKS4_WANT_CONNECT_OK,
+                                        &reason));
+  tt_ptr_op(reason, OP_EQ, NULL);
+  tt_int_op(buf_datalen(buf), OP_EQ, 0);
+
+  /* Legit socks4 response, failure. */
+  ADD_DATA(buf, "\x04\x5b\x20\x25\x01\x02\x03\x04");
+  tt_int_op(-1, OP_EQ,
+            fetch_from_buf_socks_client(buf, PROXY_SOCKS4_WANT_CONNECT_OK,
+                                        &reason));
+  tt_ptr_op(reason, OP_NE, NULL);
+  tt_str_op(reason, OP_EQ, "server rejected connection");
+
+ done:
+  buf_free(buf);
+  tor_free(reason);
+}
+
+/* Check our client-side socks5 authentication-negotiation parsing (that is to
+ * say, our parsing of server responses).
+ */
+static void
+test_socks_client_v5_auth(void *arg)
+{
+  (void)arg;
+  buf_t *buf = buf_new();
+  char *reason = NULL;
+
+  /* Legit socks5 responses, got a method we like. */
+  ADD_DATA(buf, "\x05\x00");
+  tt_int_op(1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_AUTH_METHOD_NONE,
+                                        &reason));
+  tt_ptr_op(reason, OP_EQ, NULL);
+  tt_int_op(buf_datalen(buf), OP_EQ, 0);
+
+  /* Same, but we wanted something else. */
+  ADD_DATA(buf, "\x05\x00");
+  tt_int_op(1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929,
+                                        &reason));
+  tt_ptr_op(reason, OP_EQ, NULL);
+  tt_int_op(buf_datalen(buf), OP_EQ, 0);
+
+  /* Same, and they offered a password. */
+  ADD_DATA(buf, "\x05\x02");
+  tt_int_op(2, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929,
+                                        &reason));
+  tt_ptr_op(reason, OP_EQ, NULL);
+  tt_int_op(buf_datalen(buf), OP_EQ, 0);
+
+  /* They rejected our method, or selected something we don't know. */
+  ADD_DATA(buf, "\x05\xff");
+  tt_int_op(-1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_AUTH_METHOD_NONE,
+                                        &reason));
+  tt_str_op(reason, OP_EQ, "server doesn't support any of our available "
+            "authentication methods");
+  buf_clear(buf);
+  tor_free(reason);
+  ADD_DATA(buf, "\x05\xff");
+  tt_int_op(-1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929,
+                                        &reason));
+  tt_str_op(reason, OP_EQ, "server doesn't support any of our available "
+            "authentication methods");
+  tor_free(reason);
+  buf_clear(buf);
+
+  /* Now check for authentication responses: check success and failure. */
+  ADD_DATA(buf, "\x01\x00");
+  tt_int_op(1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_AUTH_RFC1929_OK,
+                                        &reason));
+  tt_ptr_op(reason, OP_EQ, NULL);
+  tt_int_op(buf_datalen(buf), OP_EQ, 0);
+
+  ADD_DATA(buf, "\x01\xf0");
+  tt_int_op(-1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_AUTH_RFC1929_OK,
+                                        &reason));
+  tt_ptr_op(reason, OP_NE, NULL);
+  tt_str_op(reason, OP_EQ, "authentication failed");
+
+ done:
+  buf_free(buf);
+  tor_free(reason);
+}
+
+/* Check our client-side socks5 connect parsing (that is to say, our parsing
+ * of server responses).
+ */
+static void
+test_socks_client_v5_connect(void *arg)
+{
+  (void)arg;
+  buf_t *buf = buf_new();
+  char *reason = NULL;
+
+  /* Legit socks5 responses, success, ipv4. */
+  ADD_DATA(buf, "\x05\x00\x00\x01\x01\x02\x03\x04\x00\x05");
+  tt_int_op(1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_CONNECT_OK,
+                                        &reason));
+  tt_ptr_op(reason, OP_EQ, NULL);
+  tt_int_op(buf_datalen(buf), OP_EQ, 0);
+
+  /* Legit socks5 responses, success, ipv6. */
+  ADD_DATA(buf, "\x05\x00\x00\x04"
+           "abcdefghijklmnop"
+           "\x00\x05");
+  tt_int_op(1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_CONNECT_OK,
+                                        &reason));
+  tt_ptr_op(reason, OP_EQ, NULL);
+  tt_int_op(buf_datalen(buf), OP_EQ, 0);
+
+  /* Legit socks5 responses, success, hostname. */
+  ADD_DATA(buf, "\x05\x00\x00\x03\x12"
+           "gopher.example.com"
+           "\x00\x05");
+  tt_int_op(1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_CONNECT_OK,
+                                        &reason));
+  tt_ptr_op(reason, OP_EQ, NULL);
+  tt_int_op(buf_datalen(buf), OP_EQ, 0);
+
+  /* Legit socks5 responses, failure, hostname. */
+  ADD_DATA(buf, "\x05\x03\x00\x03\x12"
+           "gopher.example.com"
+           "\x00\x05");
+  tt_int_op(-1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_CONNECT_OK,
+                                        &reason));
+  tt_ptr_op(reason, OP_NE, NULL);
+  tt_str_op(reason, OP_EQ, "Network unreachable");
+  tor_free(reason);
+  buf_clear(buf);
+
+  /* Bogus socks5 responses: what is address type 0x17? */
+  ADD_DATA(buf, "\x05\x03\x00\x17\x12 blah blah");
+  tt_int_op(-1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_CONNECT_OK,
+                                        &reason));
+  tt_ptr_op(reason, OP_NE, NULL);
+  tt_str_op(reason, OP_EQ, "invalid response to connect request");
+  buf_clear(buf);
+
+ done:
+  buf_free(buf);
+  tor_free(reason);
+}
+
+static void
+test_socks_client_truncated(void *arg)
+{
+  (void)arg;
+  buf_t *buf = buf_new();
+  char *reason = NULL;
+
+#define S(str) str, (sizeof(str)-1)
+  const struct {
+    int state;
+    const char *body;
+    size_t len;
+  } replies[] = {
+    { PROXY_SOCKS4_WANT_CONNECT_OK, S("\x04\x5a\x20\x25\x01\x02\x03\x04") },
+    { PROXY_SOCKS4_WANT_CONNECT_OK, S("\x04\x5b\x20\x25\x01\x02\x03\x04") },
+    { PROXY_SOCKS5_WANT_AUTH_METHOD_NONE, S("\x05\x00") },
+    { PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929, S("\x05\x00") },
+    { PROXY_SOCKS5_WANT_AUTH_RFC1929_OK, S("\x01\x00") },
+    { PROXY_SOCKS5_WANT_CONNECT_OK,
+      S("\x05\x00\x00\x01\x01\x02\x03\x04\x00\x05") },
+    { PROXY_SOCKS5_WANT_CONNECT_OK,
+      S("\x05\x00\x00\x04" "abcdefghijklmnop" "\x00\x05") },
+    { PROXY_SOCKS5_WANT_CONNECT_OK,
+      S("\x05\x00\x00\x03\x12" "gopher.example.com" "\x00\x05") },
+    { PROXY_SOCKS5_WANT_CONNECT_OK,
+      S("\x05\x03\x00\x03\x12" "gopher.example.com""\x00\x05") },
+    { PROXY_SOCKS5_WANT_CONNECT_OK,
+      S("\x05\x03\x00\x17") },
+  };
+  unsigned i, j;
+  for (i = 0; i < ARRAY_LENGTH(replies); ++i) {
+    for (j = 0; j < replies[i].len; ++j) {
+      TT_BLATHER(("Checking command %u, length %u", i, j));
+      buf_add(buf, replies[i].body, j);
+      /* This should return 0 meaning "not done yet" */
+      tt_int_op(0, OP_EQ,
+                fetch_from_buf_socks_client(buf, replies[i].state, &reason));
+      tt_uint_op(j, OP_EQ, buf_datalen(buf)); /* Nothing was drained */
+      buf_clear(buf);
+      tt_ptr_op(reason, OP_EQ, NULL);
+    }
+  }
+
+ done:
+  tor_free(reason);
+  buf_free(buf);
+}
+
 #define SOCKSENT(name)                                  \
   { #name, test_socks_##name, TT_FORK, &socks_setup, NULL }
 
 struct testcase_t socks_tests[] = {
   SOCKSENT(4_unsupported_commands),
   SOCKSENT(4_supported_commands),
+  SOCKSENT(4_bad_arguments),
 
   SOCKSENT(5_unsupported_commands),
   SOCKSENT(5_supported_commands),
   SOCKSENT(5_no_authenticate),
+  SOCKSENT(5_auth_unsupported_type),
+  SOCKSENT(5_auth_unsupported_version),
   SOCKSENT(5_auth_before_negotiation),
   SOCKSENT(5_authenticate),
+  SOCKSENT(5_authenticate_empty_user_pass),
   SOCKSENT(5_authenticate_with_data),
   SOCKSENT(5_malformed_commands),
+  SOCKSENT(5_bad_arguments),
+
+  SOCKSENT(truncated),
+
+  SOCKSENT(wrong_protocol),
+
+  { "client/v4", test_socks_client_v4, TT_FORK, NULL, NULL },
+  { "client/v5_auth", test_socks_client_v5_auth, TT_FORK, NULL, NULL },
+  { "client/v5_connect", test_socks_client_v5_connect, TT_FORK, NULL, NULL },
+  { "client/truncated", test_socks_client_truncated, TT_FORK, NULL, NULL },
 
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_status.c b/src/test/test_status.c
index a3b1a2af87..9c47469975 100644
--- a/src/test/test_status.c
+++ b/src/test/test_status.c
@@ -1,3 +1,6 @@
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
 #define STATUS_PRIVATE
 #define HIBERNATE_PRIVATE
 #define LOG_PRIVATE
@@ -8,20 +11,27 @@
 #include <float.h>
 #include <math.h>
 
-#include "or.h"
-#include "torlog.h"
+#include "core/or/or.h"
+#include "lib/log/log.h"
 #include "tor_queue.h"
-#include "status.h"
-#include "circuitlist.h"
-#include "config.h"
-#include "hibernate.h"
-#include "rephist.h"
-#include "relay.h"
-#include "router.h"
-#include "main.h"
-#include "nodelist.h"
-#include "statefile.h"
-#include "test.h"
+#include "core/or/status.h"
+#include "core/or/circuitlist.h"
+#include "app/config/config.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/stats/rephist.h"
+#include "core/or/relay.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+#include "core/mainloop/mainloop.h"
+#include "feature/nodelist/nodelist.h"
+#include "app/config/statefile.h"
+#include "lib/tls/tortls.h"
+
+#include "core/or/origin_circuit_st.h"
+#include "app/config/or_state_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+
+#include "test/test.h"
 
 #define NS_MODULE status
 
@@ -223,7 +233,7 @@ NS(test_main)(void *arg)
   tor_free(actual);
 
   expected = "10.00 GB";
-  actual = bytes_to_usage((U64_LITERAL(1) << 30) * 10L);
+  actual = bytes_to_usage((UINT64_C(1) << 30) * 10L);
   tt_str_op(actual, OP_EQ, expected);
   tor_free(actual);
 
@@ -337,7 +347,7 @@ NS(test_main)(void *arg)
   actual = log_heartbeat(0);
 
   tt_int_op(actual, OP_EQ, expected);
-  tt_int_op(CALLED(logv), OP_EQ, 5);
+  tt_int_op(CALLED(logv), OP_EQ, 6);
 
   done:
     NS_UNMOCK(tls_get_write_overhead_ratio);
@@ -436,6 +446,16 @@ NS(logv)(int severity, log_domain_mask_t domain,
       tt_ptr_op(strstr(funcname, "rep_hist_log_link_protocol_counts"),
                 OP_NE, NULL);
       break;
+    case 5:
+      tt_int_op(severity, OP_EQ, LOG_NOTICE);
+      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_str_op(format, OP_EQ, "DoS mitigation since startup:%s%s%s%s");
+      tt_str_op(va_arg(ap, char *), OP_EQ,
+                " 0 circuits killed with too many cells.");
+      tt_str_op(va_arg(ap, char *), OP_EQ, " [cc not enabled]");
+      tt_str_op(va_arg(ap, char *), OP_EQ, " [conn not enabled]");
+      tt_str_op(va_arg(ap, char *), OP_EQ, "");
+      break;
     default:
       tt_abort_msg("unexpected call to logv()");  // TODO: prettyprint args
       break;
@@ -889,8 +909,8 @@ NS(logv)(int severity, log_domain_mask_t domain, const char *funcname,
       tt_str_op(format, OP_EQ,
           "Average packaged cell fullness: %2.3f%%. "
           "TLS write overhead: %.f%%");
-      tt_double_op(fabs(va_arg(ap, double) - 50.0), <=, DBL_EPSILON);
-      tt_double_op(fabs(va_arg(ap, double) - 0.0), <=, DBL_EPSILON);
+      tt_double_op(fabs(va_arg(ap, double) - 50.0), OP_LE, DBL_EPSILON);
+      tt_double_op(fabs(va_arg(ap, double) - 0.0), OP_LE, DBL_EPSILON);
       break;
     default:
       tt_abort_msg("unexpected call to logv()");  // TODO: prettyprint args
@@ -1039,7 +1059,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
           "Average packaged cell fullness: %2.3f%%. "
           "TLS write overhead: %.f%%");
       tt_int_op(fabs(va_arg(ap, double) - 100.0) <= DBL_EPSILON, OP_EQ, 1);
-      tt_double_op(fabs(va_arg(ap, double) - 100.0), <=, DBL_EPSILON);
+      tt_double_op(fabs(va_arg(ap, double) - 100.0), OP_LE, DBL_EPSILON);
       break;
     default:
       tt_abort_msg("unexpected call to logv()");  // TODO: prettyprint args
@@ -1080,4 +1100,3 @@ struct testcase_t status_tests[] = {
   TEST_CASE_ASPECT(log_heartbeat, tls_write_overhead),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_storagedir.c b/src/test/test_storagedir.c
new file mode 100644
index 0000000000..24e45c7428
--- /dev/null
+++ b/src/test/test_storagedir.c
@@ -0,0 +1,376 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/fs/storagedir.h"
+#include "lib/encoding/confline.h"
+#include "test/test.h"
+
+#ifdef HAVE_UTIME_H
+#include <utime.h>
+#endif
+
+static void
+test_storagedir_empty(void *arg)
+{
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  (void)arg;
+
+  tt_int_op(FN_NOENT, OP_EQ, file_status(dirname));
+
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+
+  tt_int_op(FN_DIR, OP_EQ, file_status(dirname));
+
+  tt_int_op(0, OP_EQ, smartlist_len(storage_dir_list(d)));
+  tt_u64_op(0, OP_EQ, storage_dir_get_usage(d));
+
+  storage_dir_free(d);
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+
+  tt_int_op(FN_DIR, OP_EQ, file_status(dirname));
+
+  tt_int_op(0, OP_EQ, smartlist_len(storage_dir_list(d)));
+  tt_u64_op(0, OP_EQ, storage_dir_get_usage(d));
+
+ done:
+  storage_dir_free(d);
+  tor_free(dirname);
+}
+
+static void
+test_storagedir_basic(void *arg)
+{
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  uint8_t *junk = NULL, *bytes = NULL;
+  const size_t junklen = 1024;
+  char *fname1 = NULL, *fname2 = NULL;
+  const char hello_str[] = "then what are we but cold, alone ... ?";
+  tor_mmap_t *mapping = NULL;
+  (void)arg;
+
+  junk = tor_malloc(junklen);
+  crypto_rand((void*)junk, junklen);
+
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+  tt_u64_op(0, OP_EQ, storage_dir_get_usage(d));
+
+  int r;
+  r = storage_dir_save_string_to_file(d, hello_str, 1, &fname1);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(fname1, OP_NE, NULL);
+  tt_u64_op(strlen(hello_str), OP_EQ, storage_dir_get_usage(d));
+
+  r = storage_dir_save_bytes_to_file(d, junk, junklen, 1, &fname2);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(fname2, OP_NE, NULL);
+
+  tt_str_op(fname1, OP_NE, fname2);
+
+  tt_int_op(2, OP_EQ, smartlist_len(storage_dir_list(d)));
+  tt_u64_op(junklen + strlen(hello_str), OP_EQ, storage_dir_get_usage(d));
+  tt_assert(smartlist_contains_string(storage_dir_list(d), fname1));
+  tt_assert(smartlist_contains_string(storage_dir_list(d), fname2));
+
+  storage_dir_free(d);
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+  tt_int_op(2, OP_EQ, smartlist_len(storage_dir_list(d)));
+  tt_u64_op(junklen + strlen(hello_str), OP_EQ, storage_dir_get_usage(d));
+  tt_assert(smartlist_contains_string(storage_dir_list(d), fname1));
+  tt_assert(smartlist_contains_string(storage_dir_list(d), fname2));
+
+  size_t n;
+  bytes = storage_dir_read(d, fname2, 1, &n);
+  tt_assert(bytes);
+  tt_u64_op(n, OP_EQ, junklen);
+  tt_mem_op(bytes, OP_EQ, junk, junklen);
+
+  mapping = storage_dir_map(d, fname1);
+  tt_assert(mapping);
+  tt_u64_op(mapping->size, OP_EQ, strlen(hello_str));
+  tt_mem_op(mapping->data, OP_EQ, hello_str, strlen(hello_str));
+
+ done:
+  tor_free(dirname);
+  tor_free(junk);
+  tor_free(bytes);
+  tor_munmap_file(mapping);
+  storage_dir_free(d);
+  tor_free(fname1);
+  tor_free(fname2);
+}
+
+static void
+test_storagedir_deletion(void *arg)
+{
+  (void)arg;
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  char *fn1 = NULL, *fn2 = NULL;
+  char *bytes = NULL;
+  int r;
+  const char str1[] = "There are nine and sixty ways to disguise communiques";
+  const char str2[] = "And rather more than one of them is right";
+
+  // Make sure the directory is there. */
+  d = storage_dir_new(dirname, 10);
+  storage_dir_free(d);
+  d = NULL;
+
+  tor_asprintf(&fn1, "%s/1007", dirname);
+  r = write_str_to_file(fn1, str1, 0);
+  tt_int_op(r, OP_EQ, 0);
+
+  tor_asprintf(&fn2, "%s/1003.tmp", dirname);
+  r = write_str_to_file(fn2, str2, 0);
+  tt_int_op(r, OP_EQ, 0);
+
+  // The tempfile should be deleted the next time we list the directory.
+  d = storage_dir_new(dirname, 10);
+  tt_int_op(1, OP_EQ, smartlist_len(storage_dir_list(d)));
+  tt_u64_op(strlen(str1), OP_EQ, storage_dir_get_usage(d));
+  tt_int_op(FN_FILE, OP_EQ, file_status(fn1));
+  tt_int_op(FN_NOENT, OP_EQ, file_status(fn2));
+
+  bytes = (char*) storage_dir_read(d, "1007", 1, NULL);
+  tt_str_op(bytes, OP_EQ, str1);
+
+  // Should have no effect; file already gone.
+  storage_dir_remove_file(d, "1003.tmp");
+  tt_int_op(1, OP_EQ, smartlist_len(storage_dir_list(d)));
+  tt_u64_op(strlen(str1), OP_EQ, storage_dir_get_usage(d));
+
+  // Actually remove a file.
+  storage_dir_remove_file(d, "1007");
+  tt_int_op(FN_NOENT, OP_EQ, file_status(fn1));
+  tt_int_op(0, OP_EQ, smartlist_len(storage_dir_list(d)));
+  tt_u64_op(0, OP_EQ, storage_dir_get_usage(d));
+
+ done:
+  tor_free(dirname);
+  tor_free(fn1);
+  tor_free(fn2);
+  storage_dir_free(d);
+  tor_free(bytes);
+}
+
+static void
+test_storagedir_full(void *arg)
+{
+  (void)arg;
+
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  const char str[] = "enemies of the peephole";
+  int r;
+
+  d = storage_dir_new(dirname, 3);
+  tt_assert(d);
+
+  r = storage_dir_save_string_to_file(d, str, 1, NULL);
+  tt_int_op(r, OP_EQ, 0);
+  r = storage_dir_save_string_to_file(d, str, 1, NULL);
+  tt_int_op(r, OP_EQ, 0);
+  r = storage_dir_save_string_to_file(d, str, 1, NULL);
+  tt_int_op(r, OP_EQ, 0);
+
+  // These should fail!
+  r = storage_dir_save_string_to_file(d, str, 1, NULL);
+  tt_int_op(r, OP_EQ, -1);
+  r = storage_dir_save_string_to_file(d, str, 1, NULL);
+  tt_int_op(r, OP_EQ, -1);
+
+  tt_u64_op(strlen(str) * 3, OP_EQ, storage_dir_get_usage(d));
+
+ done:
+  tor_free(dirname);
+  storage_dir_free(d);
+}
+
+static void
+test_storagedir_cleaning(void *arg)
+{
+  (void)arg;
+
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  const char str[] =
+    "On a mountain halfway between Reno and Rome / "
+    "We have a machine in a plexiglass dome / "
+    "Which listens and looks into everyone's home."
+    " -- Dr. Seuss";
+  char *fns[8];
+  int r, i;
+
+  memset(fns, 0, sizeof(fns));
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+
+  for (i = 0; i < 8; ++i) {
+    r = storage_dir_save_string_to_file(d, str+i*2, 1, &fns[i]);
+    tt_int_op(r, OP_EQ, 0);
+  }
+
+  /* Now we're going to make sure all the files have distinct mtimes. */
+  time_t now = time(NULL);
+  struct utimbuf ub;
+  ub.actime = now;
+  ub.modtime = now - 1000;
+  for (i = 0; i < 8; ++i) {
+    char *f = NULL;
+    tor_asprintf(&f, "%s/%s", dirname, fns[i]);
+    r = utime(f, &ub);
+    tor_free(f);
+    tt_int_op(r, OP_EQ, 0);
+    ub.modtime += 5;
+  }
+
+  const uint64_t usage_orig = storage_dir_get_usage(d);
+  /* No changes needed if we are already under target. */
+  storage_dir_shrink(d, 1024*1024, 0);
+  tt_u64_op(usage_orig, OP_EQ, storage_dir_get_usage(d));
+
+  /* Get rid of at least one byte.  This will delete fns[0]. */
+  storage_dir_shrink(d, usage_orig - 1, 0);
+  tt_u64_op(usage_orig, OP_GT, storage_dir_get_usage(d));
+  tt_u64_op(usage_orig - strlen(str), OP_EQ, storage_dir_get_usage(d));
+
+  /* Get rid of at least two files.  This will delete fns[1] and fns[2]. */
+  storage_dir_shrink(d, 1024*1024, 2);
+  tt_u64_op(usage_orig - strlen(str)*3 + 6, OP_EQ, storage_dir_get_usage(d));
+
+  /* Get rid of everything. */
+  storage_dir_remove_all(d);
+  tt_u64_op(0, OP_EQ, storage_dir_get_usage(d));
+
+ done:
+  tor_free(dirname);
+  storage_dir_free(d);
+  for (i = 0; i < 8; ++i) {
+    tor_free(fns[i]);
+  }
+}
+
+static void
+test_storagedir_save_labeled(void *arg)
+{
+  (void)arg;
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  uint8_t *inp = tor_malloc_zero(8192);
+  config_line_t *labels = NULL;
+  char *fname = NULL;
+  uint8_t *saved = NULL;
+
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+
+  crypto_rand((char *)inp, 8192);
+
+  config_line_append(&labels, "Foo", "bar baz");
+  config_line_append(&labels, "quux", "quuzXxz");
+  const char expected[] =
+    "Foo bar baz\n"
+    "quux quuzXxz\n";
+
+  int r = storage_dir_save_labeled_to_file(d, labels, inp, 8192, &fname);
+  tt_int_op(r, OP_EQ, 0);
+
+  size_t n = 0;
+  saved = storage_dir_read(d, fname, 1, &n);
+  tt_assert(memchr(saved, '\0', n));
+  tt_str_op((char*)saved, OP_EQ, expected); /* NUL guarantees strcmp works */
+  tt_mem_op(saved+strlen(expected)+1, OP_EQ, inp, 8192);
+
+ done:
+  storage_dir_free(d);
+  tor_free(dirname);
+  tor_free(inp);
+  tor_free(fname);
+  config_free_lines(labels);
+  tor_free(saved);
+}
+
+static void
+test_storagedir_read_labeled(void *arg)
+{
+  (void)arg;
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  uint8_t *inp = tor_malloc_zero(8192);
+  config_line_t *labels = NULL, *labels2 = NULL;
+  char *fname = NULL;
+  tor_mmap_t *map = NULL;
+  uint8_t *as_read = NULL;
+
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+
+  tor_snprintf((char*)inp, 8192,
+               "Hello world\n"
+               "This is a test\n"
+               "Yadda yadda.\n");
+  size_t bodylen = 8192 - strlen((char*)inp) - 1;
+  crypto_rand((char *)inp+strlen((char*)inp)+1, bodylen);
+
+  int r = storage_dir_save_bytes_to_file(d, inp, 8192, 1, &fname);
+  tt_int_op(r, OP_EQ, 0);
+
+  /* Try mapping */
+  const uint8_t *datap = NULL;
+  size_t sz = 0;
+  map = storage_dir_map_labeled(d, fname, &labels, &datap, &sz);
+  tt_assert(map);
+  tt_assert(datap);
+  tt_u64_op(sz, OP_EQ, bodylen);
+  tt_mem_op(datap, OP_EQ, inp+strlen((char*)inp)+1, bodylen);
+  tt_assert(labels);
+  tt_str_op(labels->key, OP_EQ, "Hello");
+  tt_str_op(labels->value, OP_EQ, "world");
+  tt_assert(labels->next);
+  tt_str_op(labels->next->key, OP_EQ, "This");
+  tt_str_op(labels->next->value, OP_EQ, "is a test");
+  tt_assert(labels->next->next);
+  tt_str_op(labels->next->next->key, OP_EQ, "Yadda");
+  tt_str_op(labels->next->next->value, OP_EQ, "yadda.");
+  tt_ptr_op(labels->next->next->next, OP_EQ, NULL);
+
+  /* Try reading this time. */
+  sz = 0;
+  as_read = storage_dir_read_labeled(d, fname, &labels2, &sz);
+  tt_assert(as_read);
+  tt_u64_op(sz, OP_EQ, bodylen);
+  tt_mem_op(as_read, OP_EQ, inp+strlen((char*)inp)+1, bodylen);
+  tt_assert(config_lines_eq(labels, labels2));
+
+ done:
+  storage_dir_free(d);
+  tor_free(dirname);
+  tor_free(inp);
+  tor_free(fname);
+  config_free_lines(labels);
+  config_free_lines(labels2);
+  tor_munmap_file(map);
+  tor_free(as_read);
+}
+
+#define ENT(name)                                               \
+  { #name, test_storagedir_ ## name, TT_FORK, NULL, NULL }
+
+struct testcase_t storagedir_tests[] = {
+  ENT(empty),
+  ENT(basic),
+  ENT(deletion),
+  ENT(full),
+  ENT(cleaning),
+  ENT(save_labeled),
+  ENT(read_labeled),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_switch_id.c b/src/test/test_switch_id.c
index e12205bb2e..baddf8d66e 100644
--- a/src/test/test_switch_id.c
+++ b/src/test/test_switch_id.c
@@ -1,11 +1,15 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
+/* Copyright (c) 2015-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#include "or.h"
+#include "core/or/or.h"
+#include "lib/process/setuid.h"
 
 #ifdef HAVE_SYS_CAPABILITY_H
 #include <sys/capability.h>
 #endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
 
 #define TEST_BUILT_WITH_CAPS         0
 #define TEST_HAVE_CAPS               1
@@ -71,7 +75,7 @@ check_can_bind_low_ports(void)
 
   return -1;
 }
-#endif
+#endif /* !defined(_WIN32) */
 
 int
 main(int argc, char **argv)
@@ -83,7 +87,7 @@ main(int argc, char **argv)
 
   fprintf(stderr, "This test is not supported on your OS.\n");
   return 77;
-#else
+#else /* !(defined(_WIN32)) */
   const char *username;
   const char *testname;
   if (argc != 3) {
@@ -174,7 +178,7 @@ main(int argc, char **argv)
         }
         cap_free(caps);
       }
-#endif
+#endif /* defined(HAVE_LINUX_CAPABILITIES) */
       break;
     default:
       fprintf(stderr, "Unsupported test '%s'\n", testname);
@@ -187,6 +191,5 @@ main(int argc, char **argv)
   }
 
   return (okay ? 0 : 1);
-#endif
+#endif /* defined(_WIN32) */
 }
-
diff --git a/src/test/test_threads.c b/src/test/test_threads.c
index 448ab2034b..4a5ecc6fae 100644
--- a/src/test/test_threads.c
+++ b/src/test/test_threads.c
@@ -1,12 +1,12 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
-#include "or.h"
-#include "compat_threads.h"
-#include "test.h"
+#include "core/or/or.h"
+#include "lib/thread/threads.h"
+#include "test/test.h"
 
 /** mutex for thread test to stop the threads hitting data at the same time. */
 static tor_mutex_t *thread_test_mutex_ = NULL;
@@ -139,8 +139,8 @@ test_threads_basic(void *arg)
               !strcmp(strmap_get(thread_test_strmap_, "thread 2"),
                       strmap_get(thread_test_strmap_, "last to run")));
 
-  tt_int_op(thread_fns_failed, ==, 0);
-  tt_int_op(thread_fn_tid1, !=, thread_fn_tid2);
+  tt_int_op(thread_fns_failed, OP_EQ, 0);
+  tt_int_op(thread_fn_tid1, OP_NE, thread_fn_tid2);
 
  done:
   tor_free(s1);
@@ -283,15 +283,15 @@ test_threads_conditionvar(void *arg)
   SPIN();
   tor_mutex_release(ti->mutex);
 
-  tt_int_op(ti->value, ==, 1337);
+  tt_int_op(ti->value, OP_EQ, 1337);
   if (!timeout) {
-    tt_int_op(ti->n_shutdown, ==, 4);
+    tt_int_op(ti->n_shutdown, OP_EQ, 4);
   } else {
     const int GIVE_UP_AFTER_SEC = 30;
     SPIN_UNTIL((ti->n_timeouts == 2 ||
                 time(NULL) >= started_at + GIVE_UP_AFTER_SEC), 10);
-    tt_int_op(ti->n_shutdown, ==, 2);
-    tt_int_op(ti->n_timeouts, ==, 2);
+    tt_int_op(ti->n_shutdown, OP_EQ, 2);
+    tt_int_op(ti->n_timeouts, OP_EQ, 2);
     tor_mutex_release(ti->mutex);
   }
 
diff --git a/src/test/test_tortls.c b/src/test/test_tortls.c
index 4febd82ddc..11e35be2fa 100644
--- a/src/test/test_tortls.c
+++ b/src/test/test_tortls.c
@@ -1,7 +1,8 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define TORTLS_PRIVATE
+#define TOR_X509_PRIVATE
 #define LOG_PRIVATE
 #include "orconfig.h"
 
@@ -9,60 +10,138 @@
 #include <winsock2.h>
 #endif
 #include <math.h>
+#include <stddef.h>
 
-#include "compat.h"
+#include "lib/cc/compat_compiler.h"
 
-/* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in
- * srtp.h. Suppress the GCC warning so we can build with -Wredundant-decl. */
-DISABLE_GCC_WARNING(redundant-decls)
+#include "core/or/or.h"
+#include "lib/log/log.h"
+#include "app/config/config.h"
+#include "lib/crypt_ops/compat_openssl.h"
+#include "lib/tls/x509.h"
+#include "lib/tls/x509_internal.h"
+#include "lib/tls/tortls.h"
+#include "lib/tls/tortls_st.h"
+#include "lib/tls/tortls_internal.h"
+#include "lib/encoding/pem.h"
+#include "app/config/or_state_st.h"
 
-#include <openssl/opensslv.h>
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+#include "test/test_tortls.h"
 
-#include <openssl/ssl.h>
-#include <openssl/ssl3.h>
-#include <openssl/err.h>
-#include <openssl/asn1t.h>
-#include <openssl/x509.h>
-#include <openssl/rsa.h>
-#include <openssl/evp.h>
-#include <openssl/bn.h>
+#include "tinytest.h"
 
-ENABLE_GCC_WARNING(redundant-decls)
-
-#include "or.h"
-#include "torlog.h"
-#include "config.h"
-#include "tortls.h"
-
-#include "test.h"
-#include "log_test_helpers.h"
-#define NS_MODULE tortls
+const char* notCompletelyValidCertString =
+  "-----BEGIN CERTIFICATE-----\n"
+  "MIICVjCCAb8CAg37MA0GCSqGSIb3DQEBBQUAMIGbMQswCQYDVQQGEwJKUDEOMAwG\n"
+  "A1UECBMFVG9reW8xEDAOBgNVBAcTB0NodW8ta3UxETAPBgNVBAoTCEZyYW5rNERE\n"
+  "MRgwFgYDVQQLEw9XZWJDZXJ0IFN1cHBvcnQxGDAWBgNVBAMTD0ZyYW5rNEREIFdl\n"
+  "YiBDQTEjMCEGCSqGSIb3DQEJARYUc3VwcG9ydEBmcmFuazRkZC5jb20wHhcNMTIw\n"
+  "ODIyMDUyNzIzWhcNMTcwODIxMDUyNzIzWjBKMQswCQYDVQQGEwJKUDEOMAwGA1UE\n"
+  "CAwFVG9reW8xETAPBgNVBAoMCEZyYW5rNEREMRgwFgYDVQQDDA93d3cuZXhhbXBs\n"
+  "ZS5jb20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMYBBrx5PlP0WNI/ZdzD\n"
+  "+6Pktmurn+F2kQYbtc7XQh8/LTBvCo+P6iZoLEmUA9e7EXLRxgU1CVqeAi7QcAn9\n"
+  "MwBlc8ksFJHB0rtf9pmf8Oza9E0Bynlq/4/Kb1x+d+AyhL7oK9tQwB24uHOueHi1\n"
+  "C/iVv8CSWKiYe6hzN1txYe8rAgMBAAEwDQYJKoZIhvcNAQEFBQADgYEAASPdjigJ\n"
+  "kXCqKWpnZ/Oc75EUcMi6HztaW8abUMlYXPIgkV2F7YanHOB7K4f7OOLjiz8DTPFf\n"
+  "jC9UeuErhaA/zzWi8ewMTFZW/WshOrm3fNvcMrMLKtH534JKvcdMg6qIdjTFINIr\n"
+  "evnAhf0cwULaebn+lMs8Pdl7y37+sfluVok=\n"
+  "-----END CERTIFICATE-----\n";
 
-#ifndef HAVE_SSL_STATE
-#define OPENSSL_OPAQUE
-#endif
+const char* validCertString = "-----BEGIN CERTIFICATE-----\n"
+  "MIIDpTCCAY0CAg3+MA0GCSqGSIb3DQEBBQUAMF4xCzAJBgNVBAYTAlVTMREwDwYD\n"
+  "VQQIDAhJbGxpbm9pczEQMA4GA1UEBwwHQ2hpY2FnbzEUMBIGA1UECgwLVG9yIFRl\n"
+  "c3RpbmcxFDASBgNVBAMMC1RvciBUZXN0aW5nMB4XDTE1MDkwNjEzMzk1OVoXDTQz\n"
+  "MDEyMjEzMzk1OVowVjELMAkGA1UEBhMCVVMxEDAOBgNVBAcMB0NoaWNhZ28xFDAS\n"
+  "BgNVBAoMC1RvciBUZXN0aW5nMR8wHQYDVQQDDBZ0ZXN0aW5nLnRvcnByb2plY3Qu\n"
+  "b3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDoT6uyVVhWyOF3wkHjjYbd\n"
+  "nKaykyRv4JVtKQdZ4OpEErmX1zw4MmyzpQNV6iR4bQnWiyLfzyVJMZDIC/WILBfX\n"
+  "w2Pza/yuLgUvDc3twMuhOACzOQVO8PrEF/aVv2+hbCCy2udXvKhnYn+CCXl3ozc8\n"
+  "XcKYvujTXDyvGWY3xwAjlQIDAQABMA0GCSqGSIb3DQEBBQUAA4ICAQCUvnhzQWuQ\n"
+  "MrN+pERkE+zcTI/9dGS90rUMMLgu8VDNqTa0TUQh8uO0EQ6uDvI8Js6e8tgwS0BR\n"
+  "UBahqb7ZHv+rejGCBr5OudqD+x4STiiuPNJVs86JTLN8SpM9CHjIBH5WCCN2KOy3\n"
+  "mevNoRcRRyYJzSFULCunIK6FGulszigMYGscrO4oiTkZiHPh9KvWT40IMiHfL+Lw\n"
+  "EtEWiLex6064LcA2YQ1AMuSZyCexks63lcfaFmQbkYOKqXa1oLkIRuDsOaSVjTfe\n"
+  "vec+X6jvf12cFTKS5WIeqkKF2Irt+dJoiHEGTe5RscUMN/f+gqHPzfFz5dR23sxo\n"
+  "g+HC6MZHlFkLAOx3wW6epPS8A/m1mw3zMPoTnb2U2YYt8T0dJMMlUn/7Y1sEAa+a\n"
+  "dSTMaeUf6VnJ//11m454EZl1to9Z7oJOgqmFffSrdD4BGIWe8f7hhW6L1Enmqe/J\n"
+  "BKL3wbzZh80O1W0bndAwhnEEhlzneFY84cbBo9pmVxpODHkUcStpr5Z7pBDrcL21\n"
+  "Ss/aB/1YrsVXhdvJdOGxl3Mnl9dUY57CympLGlT8f0pPS6GAKOelECOhFMHmJd8L\n"
+  "dj3XQSmKtYHevZ6IvuMXSlB/fJvSjSlkCuLo5+kJoaqPuRu+i/S1qxeRy3CBwmnE\n"
+  "LdSNdcX4N79GQJ996PA8+mUCQG7YRtK+WA==\n"
+  "-----END CERTIFICATE-----\n";
 
-#if defined(OPENSSL_OPAQUE) && !defined(LIBRESSL_VERSION_NUMBER)
-#define SSL_STATE_STR "before SSL initialization"
-#else
-#define SSL_STATE_STR "before/accept initialization"
-#endif
+const char* caCertString = "-----BEGIN CERTIFICATE-----\n"
+  "MIIFjzCCA3egAwIBAgIJAKd5WgyfPMYRMA0GCSqGSIb3DQEBCwUAMF4xCzAJBgNV\n"
+  "BAYTAlVTMREwDwYDVQQIDAhJbGxpbm9pczEQMA4GA1UEBwwHQ2hpY2FnbzEUMBIG\n"
+  "A1UECgwLVG9yIFRlc3RpbmcxFDASBgNVBAMMC1RvciBUZXN0aW5nMB4XDTE1MDkw\n"
+  "NjEzMzc0MVoXDTQzMDEyMjEzMzc0MVowXjELMAkGA1UEBhMCVVMxETAPBgNVBAgM\n"
+  "CElsbGlub2lzMRAwDgYDVQQHDAdDaGljYWdvMRQwEgYDVQQKDAtUb3IgVGVzdGlu\n"
+  "ZzEUMBIGA1UEAwwLVG9yIFRlc3RpbmcwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw\n"
+  "ggIKAoICAQCpLMUEiLW5leUgBZoEJms2V7lZRhIAjnJBhVMHD0e3UubNknmaQoxf\n"
+  "ARz3rvqOaRd0JlV+qM9qE0DjiYcCVP1cAfqAo9d83uS1vwY3YMVJzADlaIiHfyVW\n"
+  "uEgBy0vvkeUBqaua24dYlcwsemOiXYLu41yM1wkcGHW1AhBNHppY6cznb8TyLgNM\n"
+  "2x3SGUdzc5XMyAFx51faKGBA3wjs+Hg1PLY7d30nmCgEOBavpm5I1disM/0k+Mcy\n"
+  "YmAKEo/iHJX/rQzO4b9znP69juLlR8PDBUJEVIG/CYb6+uw8MjjUyiWXYoqfVmN2\n"
+  "hm/lH8b6rXw1a2Aa3VTeD0DxaWeacMYHY/i01fd5n7hCoDTRNdSw5KJ0L3Z0SKTu\n"
+  "0lzffKzDaIfyZGlpW5qdouACkWYzsaitQOePVE01PIdO30vUfzNTFDfy42ccx3Di\n"
+  "59UCu+IXB+eMtrBfsok0Qc63vtF1linJgjHW1z/8ujk8F7/qkOfODhk4l7wngc2A\n"
+  "EmwWFIFoGaiTEZHB9qteXr4unbXZ0AHpM02uGGwZEGohjFyebEb73M+J57WKKAFb\n"
+  "PqbLcGUksL1SHNBNAJcVLttX55sO4nbidOS/kA3m+F1R04MBTyQF9qA6YDDHqdI3\n"
+  "h/3pw0Z4fxVouTYT4/NfRnX4JTP4u+7Mpcoof28VME0qWqD1LnRhFQIDAQABo1Aw\n"
+  "TjAdBgNVHQ4EFgQUMoAgIXH7pZ3QMRwTjT+DM9Yo/v0wHwYDVR0jBBgwFoAUMoAg\n"
+  "IXH7pZ3QMRwTjT+DM9Yo/v0wDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOC\n"
+  "AgEAUJxacjXR9sT+Xs6ISFiUsyd0T6WVKMnV46xrYJHirGfx+krWHrjxMY+ZtxYD\n"
+  "DBDGlo11Qc4v6QrclNf5QUBfIiGQsP9Cm6hHcQ+Tpg9HHCgSqG1YNPwCPReCR4br\n"
+  "BLvLfrfkcBL2IWM0PdQdCze+59DBfipsULD2mEn9fjYRXQEwb2QWtQ9qRc20Yb/x\n"
+  "Q4b/+CvUodLkaq7B8MHz0BV8HHcBoph6DYaRmO/N+hPauIuSp6XyaGYcEefGKVKj\n"
+  "G2+fcsdyXsoijNdL8vNKwm4j2gVwCBnw16J00yfFoV46YcbfqEdJB2je0XSvwXqt\n"
+  "14AOTngxso2h9k9HLtrfpO1ZG/B5AcCMs1lzbZ2fp5DPHtjvvmvA2RJqgo3yjw4W\n"
+  "4DHAuTglYFlC3mDHNfNtcGP20JvepcQNzNP2UzwcpOc94hfKikOFw+gf9Vf1qd0y\n"
+  "h/Sk6OZHn2+JVUPiWHIQV98Vtoh4RmUZDJD+b55ia3fQGTGzt4z1XFzQYSva5sfs\n"
+  "wocS/papthqWldQU7x+3wofNd5CNU1x6WKXG/yw30IT/4F8ADJD6GeygNT8QJYvt\n"
+  "u/8lAkbOy6B9xGmSvr0Kk1oq9P2NshA6kalxp1Oz/DTNDdL4AeBXV3JmM6WWCjGn\n"
+  "Yy1RT69d0rwYc5u/vnqODz1IjvT90smsrkBumGt791FAFeg=\n"
+  "-----END CERTIFICATE-----\n";
 
-#ifndef OPENSSL_OPAQUE
-static SSL_METHOD *
-give_me_a_test_method(void)
+tor_x509_cert_impl_t *
+read_cert_from(const char *str)
 {
-  SSL_METHOD *method = tor_malloc_zero(sizeof(SSL_METHOD));
-  memcpy(method, TLSv1_method(), sizeof(SSL_METHOD));
-  return method;
+  size_t len = strlen(str);
+  uint8_t *raw_cert = tor_malloc(len);
+  ssize_t true_len = pem_decode(raw_cert, len, str, len, "CERTIFICATE");
+  if (true_len < 0) {
+    tor_free(raw_cert);
+    return NULL;
+  }
+  tor_x509_cert_t *cert = tor_x509_cert_decode(raw_cert, true_len);
+  tor_free(raw_cert);
+  if (! cert) {
+    return NULL;
+  }
+  tor_x509_cert_impl_t *res = tor_x509_cert_impl_dup_(cert->cert);
+  tor_x509_cert_free(cert);
+  return res;
 }
 
-static int
-fake_num_ciphers(void)
+static tor_x509_cert_impl_t *
+  fixed_try_to_extract_certs_from_tls_cert_out_result = NULL;
+static tor_x509_cert_impl_t *
+  fixed_try_to_extract_certs_from_tls_id_cert_out_result = NULL;
+
+static void
+fixed_try_to_extract_certs_from_tls(int severity, tor_tls_t *tls,
+                                    tor_x509_cert_impl_t **cert_out,
+                                    tor_x509_cert_impl_t **id_cert_out)
 {
-  return 0;
+  (void) severity;
+  (void) tls;
+  *cert_out = tor_x509_cert_impl_dup_(
+                      fixed_try_to_extract_certs_from_tls_cert_out_result);
+  *id_cert_out =  tor_x509_cert_impl_dup_(
+                      fixed_try_to_extract_certs_from_tls_id_cert_out_result);
 }
-#endif
 
 static void
 test_tortls_errno_to_tls_error(void *data)
@@ -106,6 +185,7 @@ test_tortls_err_to_string(void *data)
   (void)1;
 }
 
+#ifdef ENABLE_OPENSSL
 static int
 mock_tls_cert_matches_key(const tor_tls_t *tls, const tor_x509_cert_t *cert)
 {
@@ -115,66 +195,6 @@ mock_tls_cert_matches_key(const tor_tls_t *tls, const tor_x509_cert_t *cert)
 }
 
 static void
-test_tortls_tor_tls_new(void *data)
-{
-  (void) data;
-  MOCK(tor_tls_cert_matches_key, mock_tls_cert_matches_key);
-  crypto_pk_t *key1 = NULL, *key2 = NULL;
-  SSL_METHOD *method = NULL;
-
-  key1 = pk_generate(2);
-  key2 = pk_generate(3);
-
-  tor_tls_t *tls = NULL;
-  tt_int_op(tor_tls_context_init(TOR_TLS_CTX_IS_PUBLIC_SERVER,
-                                 key1, key2, 86400), OP_EQ, 0);
-  tls = tor_tls_new(-1, 0);
-  tt_want(tls);
-  tor_tls_free(tls); tls = NULL;
-
-  SSL_CTX_free(client_tls_context->ctx);
-  client_tls_context->ctx = NULL;
-  tls = tor_tls_new(-1, 0);
-  tt_assert(!tls);
-
-#ifndef OPENSSL_OPAQUE
-  method = give_me_a_test_method();
-  SSL_CTX *ctx = SSL_CTX_new(method);
-  method->num_ciphers = fake_num_ciphers;
-  client_tls_context->ctx = ctx;
-  tls = tor_tls_new(-1, 0);
-  tt_assert(!tls);
-#endif
-
- done:
-  UNMOCK(tor_tls_cert_matches_key);
-  crypto_pk_free(key1);
-  crypto_pk_free(key2);
-  tor_tls_free(tls);
-  tor_free(method);
-  tor_tls_free_all();
-}
-
-#define NS_MODULE tortls
-NS_DECL(void, logv, (int severity, log_domain_mask_t domain,
-                     const char *funcname, const char *suffix,
-                     const char *format, va_list ap));
-
-static void
-NS(logv)(int severity, log_domain_mask_t domain,
-         const char *funcname, const char *suffix, const char *format,
-         va_list ap)
-{
-  (void) severity;
-  (void) domain;
-  (void) funcname;
-  (void) suffix;
-  (void) format;
-  (void) ap; // XXXX look at this.
-  CALLED(logv)++;
-}
-
-static void
 test_tortls_tor_tls_get_error(void *data)
 {
   (void) data;
@@ -187,11 +207,10 @@ test_tortls_tor_tls_get_error(void *data)
   tt_int_op(tor_tls_context_init(TOR_TLS_CTX_IS_PUBLIC_SERVER,
                                  key1, key2, 86400), OP_EQ, 0);
   tls = tor_tls_new(-1, 0);
-  NS_MOCK(logv);
-  tt_int_op(CALLED(logv), OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
   tor_tls_get_error(tls, 0, 0,
-                    (const char *)"test", 0, 0);
-  tt_int_op(CALLED(logv), OP_EQ, 1);
+                    (const char *)"in unit test", LOG_WARN, LD_GENERAL);
+  expect_single_log_msg_containing("unexpected close while in unit test");
 
  done:
   UNMOCK(tor_tls_cert_matches_key);
@@ -200,325 +219,6 @@ test_tortls_tor_tls_get_error(void *data)
   crypto_pk_free(key2);
   tor_tls_free(tls);
 }
-
-static void
-test_tortls_get_state_description(void *ignored)
-{
-  (void)ignored;
-  tor_tls_t *tls;
-  char *buf;
-  SSL_CTX *ctx;
-
-  SSL_library_init();
-  SSL_load_error_strings();
-
-  ctx = SSL_CTX_new(SSLv23_method());
-
-  buf = tor_malloc_zero(1000);
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-
-  tor_tls_get_state_description(NULL, buf, 20);
-  tt_str_op(buf, OP_EQ, "(No SSL object)");
-
-  SSL_free(tls->ssl);
-  tls->ssl = NULL;
-  tor_tls_get_state_description(tls, buf, 20);
-  tt_str_op(buf, OP_EQ, "(No SSL object)");
-
-  tls->ssl = SSL_new(ctx);
-  tor_tls_get_state_description(tls, buf, 200);
-  tt_str_op(buf, OP_EQ, SSL_STATE_STR " in HANDSHAKE");
-
-  tls->state = TOR_TLS_ST_OPEN;
-  tor_tls_get_state_description(tls, buf, 200);
-  tt_str_op(buf, OP_EQ, SSL_STATE_STR " in OPEN");
-
-  tls->state = TOR_TLS_ST_GOTCLOSE;
-  tor_tls_get_state_description(tls, buf, 200);
-  tt_str_op(buf, OP_EQ, SSL_STATE_STR " in GOTCLOSE");
-
-  tls->state = TOR_TLS_ST_SENTCLOSE;
-  tor_tls_get_state_description(tls, buf, 200);
-  tt_str_op(buf, OP_EQ, SSL_STATE_STR " in SENTCLOSE");
-
-  tls->state = TOR_TLS_ST_CLOSED;
-  tor_tls_get_state_description(tls, buf, 200);
-  tt_str_op(buf, OP_EQ, SSL_STATE_STR " in CLOSED");
-
-  tls->state = TOR_TLS_ST_RENEGOTIATE;
-  tor_tls_get_state_description(tls, buf, 200);
-  tt_str_op(buf, OP_EQ, SSL_STATE_STR " in RENEGOTIATE");
-
-  tls->state = TOR_TLS_ST_BUFFEREVENT;
-  tor_tls_get_state_description(tls, buf, 200);
-  tt_str_op(buf, OP_EQ, SSL_STATE_STR);
-
-  tls->state = 7;
-  tor_tls_get_state_description(tls, buf, 200);
-  tt_str_op(buf, OP_EQ, SSL_STATE_STR " in unknown TLS state");
-
- done:
-  SSL_CTX_free(ctx);
-  SSL_free(tls->ssl);
-  tor_free(buf);
-  tor_free(tls);
-}
-
-static void
-test_tortls_get_by_ssl(void *ignored)
-{
-  (void)ignored;
-  tor_tls_t *tls;
-  tor_tls_t *res;
-  SSL_CTX *ctx;
-  SSL *ssl;
-
-  SSL_library_init();
-  SSL_load_error_strings();
-  tor_tls_allocate_tor_tls_object_ex_data_index();
-
-  ctx = SSL_CTX_new(SSLv23_method());
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  tls->magic = TOR_TLS_MAGIC;
-
-  ssl = SSL_new(ctx);
-
-  res = tor_tls_get_by_ssl(ssl);
-  tt_assert(!res);
-
-  SSL_set_ex_data(ssl, tor_tls_object_ex_data_index, tls);
-
-  res = tor_tls_get_by_ssl(ssl);
-  tt_assert(res == tls);
-
- done:
-  SSL_free(ssl);
-  SSL_CTX_free(ctx);
-  tor_free(tls);
-}
-
-static void
-test_tortls_allocate_tor_tls_object_ex_data_index(void *ignored)
-{
-  (void)ignored;
-  int first;
-
-  tor_tls_allocate_tor_tls_object_ex_data_index();
-
-  first = tor_tls_object_ex_data_index;
-  tor_tls_allocate_tor_tls_object_ex_data_index();
-  tt_int_op(first, OP_EQ, tor_tls_object_ex_data_index);
-
- done:
-  (void)0;
-}
-
-static void
-test_tortls_log_one_error(void *ignored)
-{
-  (void)ignored;
-  tor_tls_t *tls;
-  SSL_CTX *ctx;
-  SSL *ssl = NULL;
-
-  SSL_library_init();
-  SSL_load_error_strings();
-
-  ctx = SSL_CTX_new(SSLv23_method());
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  setup_capture_of_logs(LOG_INFO);
-
-  tor_tls_log_one_error(NULL, 0, LOG_WARN, 0, "something");
-  expect_log_msg("TLS error while something: "
-            "(null) (in (null):(null):---)\n");
-
-  mock_clean_saved_logs();
-  tor_tls_log_one_error(tls, 0, LOG_WARN, 0, NULL);
-  expect_log_msg("TLS error: (null) "
-            "(in (null):(null):---)\n");
-
-  mock_clean_saved_logs();
-  tls->address = tor_strdup("127.hello");
-  tor_tls_log_one_error(tls, 0, LOG_WARN, 0, NULL);
-  expect_log_msg("TLS error with 127.hello: "
-            "(null) (in (null):(null):---)\n");
-  tor_free(tls->address);
-
-  mock_clean_saved_logs();
-  tls->address = tor_strdup("127.hello");
-  tor_tls_log_one_error(tls, 0, LOG_WARN, 0, "blarg");
-  expect_log_msg("TLS error while blarg with "
-            "127.hello: (null) (in (null):(null):---)\n");
-
-  mock_clean_saved_logs();
-  tor_tls_log_one_error(tls, ERR_PACK(1, 2, 3), LOG_WARN, 0, NULL);
-  expect_log_msg("TLS error with 127.hello: "
-            "BN lib (in unknown library:(null):---)\n");
-
-  mock_clean_saved_logs();
-  tor_tls_log_one_error(tls, ERR_PACK(1, 2, SSL_R_HTTP_REQUEST),
-                        LOG_WARN, 0, NULL);
-  expect_log_severity(LOG_INFO);
-
-  mock_clean_saved_logs();
-  tor_tls_log_one_error(tls, ERR_PACK(1, 2, SSL_R_HTTPS_PROXY_REQUEST),
-                        LOG_WARN, 0, NULL);
-  expect_log_severity(LOG_INFO);
-
-  mock_clean_saved_logs();
-  tor_tls_log_one_error(tls, ERR_PACK(1, 2, SSL_R_RECORD_LENGTH_MISMATCH),
-                        LOG_WARN, 0, NULL);
-  expect_log_severity(LOG_INFO);
-
-#ifndef OPENSSL_1_1_API
-  mock_clean_saved_logs();
-  tor_tls_log_one_error(tls, ERR_PACK(1, 2, SSL_R_RECORD_TOO_LARGE),
-                        LOG_WARN, 0, NULL);
-  expect_log_severity(LOG_INFO);
-#endif
-
-  mock_clean_saved_logs();
-  tor_tls_log_one_error(tls, ERR_PACK(1, 2, SSL_R_UNKNOWN_PROTOCOL),
-                        LOG_WARN, 0, NULL);
-  expect_log_severity(LOG_INFO);
-
-  mock_clean_saved_logs();
-  tor_tls_log_one_error(tls, ERR_PACK(1, 2, SSL_R_UNSUPPORTED_PROTOCOL),
-                        LOG_WARN, 0, NULL);
-  expect_log_severity(LOG_INFO);
-
-  tls->ssl = SSL_new(ctx);
-
-  mock_clean_saved_logs();
-  tor_tls_log_one_error(tls, 0, LOG_WARN, 0, NULL);
-  expect_log_msg("TLS error with 127.hello: (null)"
-            " (in (null):(null):" SSL_STATE_STR ")\n");
-
- done:
-  teardown_capture_of_logs();
-  SSL_free(ssl);
-  SSL_CTX_free(ctx);
-  if (tls && tls->ssl)
-    SSL_free(tls->ssl);
-  if (tls)
-    tor_free(tls->address);
-  tor_free(tls);
-}
-
-#ifndef OPENSSL_OPAQUE
-static void
-test_tortls_get_error(void *ignored)
-{
-  (void)ignored;
-  tor_tls_t *tls;
-  int ret;
-  SSL_CTX *ctx;
-
-  SSL_library_init();
-  SSL_load_error_strings();
-
-  ctx = SSL_CTX_new(SSLv23_method());
-  setup_capture_of_logs(LOG_INFO);
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  tls->ssl = SSL_new(ctx);
-  SSL_set_bio(tls->ssl, BIO_new(BIO_s_mem()), NULL);
-
-  ret = tor_tls_get_error(tls, 0, 0, "something", LOG_WARN, 0);
-  tt_int_op(ret, OP_EQ, TOR_TLS_ERROR_IO);
-  expect_log_msg("TLS error: unexpected close while"
-            " something (before/accept initialization)\n");
-
-  mock_clean_saved_logs();
-  ret = tor_tls_get_error(tls, 2, 0, "something", LOG_WARN, 0);
-  tt_int_op(ret, OP_EQ, 0);
-  expect_no_log_entry();
-
-  mock_clean_saved_logs();
-  ret = tor_tls_get_error(tls, 0, 1, "something", LOG_WARN, 0);
-  tt_int_op(ret, OP_EQ, -11);
-  expect_no_log_entry();
-
-  mock_clean_saved_logs();
-  ERR_clear_error();
-  ERR_put_error(ERR_LIB_BN, 2, -1, "somewhere.c", 99);
-  ret = tor_tls_get_error(tls, 0, 0, "something", LOG_WARN, 0);
-  tt_int_op(ret, OP_EQ, TOR_TLS_ERROR_MISC);
-  expect_log_msg("TLS error while something: (null)"
-            " (in bignum routines:(null):before/accept initialization)\n");
-
-  mock_clean_saved_logs();
-  ERR_clear_error();
-  tls->ssl->rwstate = SSL_READING;
-  SSL_get_rbio(tls->ssl)->flags = BIO_FLAGS_READ;
-  ret = tor_tls_get_error(tls, -1, 0, "something", LOG_WARN, 0);
-  tt_int_op(ret, OP_EQ, TOR_TLS_WANTREAD);
-  expect_no_log_entry();
-
-  mock_clean_saved_logs();
-  ERR_clear_error();
-  tls->ssl->rwstate = SSL_READING;
-  SSL_get_rbio(tls->ssl)->flags = BIO_FLAGS_WRITE;
-  ret = tor_tls_get_error(tls, -1, 0, "something", LOG_WARN, 0);
-  tt_int_op(ret, OP_EQ, TOR_TLS_WANTWRITE);
-  expect_no_log_entry();
-
-  mock_clean_saved_logs();
-  ERR_clear_error();
-  tls->ssl->rwstate = 0;
-  tls->ssl->shutdown = SSL_RECEIVED_SHUTDOWN;
-  tls->ssl->s3->warn_alert =SSL_AD_CLOSE_NOTIFY;
-  ret = tor_tls_get_error(tls, 0, 0, "something", LOG_WARN, 0);
-  tt_int_op(ret, OP_EQ, TOR_TLS_CLOSE);
-  expect_log_entry();
-
-  mock_clean_saved_logs();
-  ret = tor_tls_get_error(tls, 0, 2, "something", LOG_WARN, 0);
-  tt_int_op(ret, OP_EQ, -10);
-  expect_no_log_entry();
-
-  mock_clean_saved_logs();
-  ERR_put_error(ERR_LIB_SYS, 2, -1, "somewhere.c", 99);
-  ret = tor_tls_get_error(tls, -1, 0, "something", LOG_WARN, 0);
-  tt_int_op(ret, OP_EQ, -9);
-  expect_log_msg("TLS error while something: (null) (in system library:"
-            "connect:before/accept initialization)\n");
-
- done:
-  teardown_capture_of_logs();
-  SSL_free(tls->ssl);
-  tor_free(tls);
-  SSL_CTX_free(ctx);
-}
-#endif
-
-static void
-test_tortls_always_accept_verify_cb(void *ignored)
-{
-  (void)ignored;
-  int ret;
-
-  ret = always_accept_verify_cb(0, NULL);
-  tt_int_op(ret, OP_EQ, 1);
-
- done:
-  (void)0;
-}
-
-#ifndef OPENSSL_OPAQUE
-static void
-test_tortls_x509_cert_free(void *ignored)
-{
-  (void)ignored;
-  tor_x509_cert_t *cert;
-
-  cert = tor_malloc_zero(sizeof(tor_x509_cert_t));
-  tor_x509_cert_free(cert);
-
-  cert = tor_malloc_zero(sizeof(tor_x509_cert_t));
-  cert->cert = tor_malloc_zero(sizeof(X509));
-  cert->encoded = tor_malloc_zero(1);
-  tor_x509_cert_free(cert);
-}
 #endif
 
 static void
@@ -538,6 +238,7 @@ test_tortls_x509_cert_get_id_digests(void *ignored)
   cert->pkey_digests_set = 1;
   cert->pkey_digests = *d;
   res = tor_x509_cert_get_id_digests(cert);
+  tt_assert(res);
   tt_int_op(res->d[0][0], OP_EQ, 42);
 
  done:
@@ -545,152 +246,6 @@ test_tortls_x509_cert_get_id_digests(void *ignored)
   tor_free(d);
 }
 
-#ifndef OPENSSL_OPAQUE
-static void
-fake_x509_free(X509 *cert)
-{
-  if (cert) {
-    if (cert->cert_info) {
-      if (cert->cert_info->key) {
-        if (cert->cert_info->key->pkey) {
-          tor_free(cert->cert_info->key->pkey);
-        }
-        tor_free(cert->cert_info->key);
-      }
-      tor_free(cert->cert_info);
-    }
-    tor_free(cert);
-  }
-}
-#endif
-
-static tor_x509_cert_t *fixed_x509_cert = NULL;
-static tor_x509_cert_t *
-get_peer_cert_mock_return_fixed(tor_tls_t *tls)
-{
-  (void)tls;
-  if (fixed_x509_cert)
-    return tor_x509_cert_dup(fixed_x509_cert);
-  else
-    return NULL;
-}
-
-static void
-test_tortls_cert_matches_key(void *ignored)
-{
-  (void)ignored;
-
-  X509 *cert1 = NULL, *cert2 = NULL, *cert3 = NULL, *cert4 = NULL;
-  tor_x509_cert_t *c1 = NULL, *c2 = NULL, *c3 = NULL, *c4 = NULL;
-  crypto_pk_t *k1 = NULL, *k2 = NULL, *k3 = NULL;
-
-  k1 = pk_generate(1);
-  k2 = pk_generate(2);
-  k3 = pk_generate(3);
-
-  cert1 = tor_tls_create_certificate(k1, k2, "A", "B", 1000);
-  cert2 = tor_tls_create_certificate(k1, k3, "C", "D", 1000);
-  cert3 = tor_tls_create_certificate(k2, k3, "C", "D", 1000);
-  cert4 = tor_tls_create_certificate(k3, k2, "E", "F", 1000);
-
-  tt_assert(cert1 && cert2 && cert3 && cert4);
-
-  c1 = tor_x509_cert_new(cert1); cert1 = NULL;
-  c2 = tor_x509_cert_new(cert2); cert2 = NULL;
-  c3 = tor_x509_cert_new(cert3); cert3 = NULL;
-  c4 = tor_x509_cert_new(cert4); cert4 = NULL;
-
-  tt_assert(c1 && c2 && c3 && c4);
-
-  MOCK(tor_tls_get_peer_cert, get_peer_cert_mock_return_fixed);
-
-  fixed_x509_cert = NULL;
-  /* If the peer has no certificate, it shouldn't match anything. */
-  tt_assert(! tor_tls_cert_matches_key(NULL, c1));
-  tt_assert(! tor_tls_cert_matches_key(NULL, c2));
-  tt_assert(! tor_tls_cert_matches_key(NULL, c3));
-  tt_assert(! tor_tls_cert_matches_key(NULL, c4));
-  fixed_x509_cert = c1;
-  /* If the peer has a certificate, it should match every cert with the same
-   * subject key. */
-  tt_assert(tor_tls_cert_matches_key(NULL, c1));
-  tt_assert(tor_tls_cert_matches_key(NULL, c2));
-  tt_assert(! tor_tls_cert_matches_key(NULL, c3));
-  tt_assert(! tor_tls_cert_matches_key(NULL, c4));
-
- done:
-  tor_x509_cert_free(c1);
-  tor_x509_cert_free(c2);
-  tor_x509_cert_free(c3);
-  tor_x509_cert_free(c4);
-  if (cert1) X509_free(cert1);
-  if (cert2) X509_free(cert2);
-  if (cert3) X509_free(cert3);
-  if (cert4) X509_free(cert4);
-  crypto_pk_free(k1);
-  crypto_pk_free(k2);
-  crypto_pk_free(k3);
-  UNMOCK(tor_tls_get_peer_cert);
-}
-
-#ifndef OPENSSL_OPAQUE
-static void
-test_tortls_cert_get_key(void *ignored)
-{
-  (void)ignored;
-  tor_x509_cert_t *cert = NULL;
-  crypto_pk_t *res = NULL;
-  cert = tor_malloc_zero(sizeof(tor_x509_cert_t));
-  X509 *key = NULL;
-  key = tor_malloc_zero(sizeof(X509));
-  key->references = 1;
-
-  res = tor_tls_cert_get_key(cert);
-  tt_assert(!res);
-
-  cert->cert = key;
-  key->cert_info = tor_malloc_zero(sizeof(X509_CINF));
-  key->cert_info->key = tor_malloc_zero(sizeof(X509_PUBKEY));
-  key->cert_info->key->pkey = tor_malloc_zero(sizeof(EVP_PKEY));
-  key->cert_info->key->pkey->references = 1;
-  key->cert_info->key->pkey->type = 2;
-  res = tor_tls_cert_get_key(cert);
-  tt_assert(!res);
-
- done:
-  fake_x509_free(key);
-  tor_free(cert);
-  crypto_pk_free(res);
-}
-#endif
-
-static void
-test_tortls_get_my_client_auth_key(void *ignored)
-{
-  (void)ignored;
-  crypto_pk_t *ret;
-  crypto_pk_t *expected;
-  tor_tls_context_t *ctx;
-  RSA *k = RSA_new();
-
-  ctx = tor_malloc_zero(sizeof(tor_tls_context_t));
-  expected = crypto_new_pk_from_rsa_(k);
-  ctx->auth_key = expected;
-
-  client_tls_context = NULL;
-  ret = tor_tls_get_my_client_auth_key();
-  tt_assert(!ret);
-
-  client_tls_context = ctx;
-  ret = tor_tls_get_my_client_auth_key();
-  tt_assert(ret == expected);
-
- done:
-  RSA_free(k);
-  tor_free(expected);
-  tor_free(ctx);
-}
-
 static void
 test_tortls_get_my_certs(void *ignored)
 {
@@ -726,435 +281,7 @@ test_tortls_get_my_certs(void *ignored)
   (void)1;
 }
 
-#ifndef HAVE_SSL_GET_CLIENT_CIPHERS
-static SSL_CIPHER *
-get_cipher_by_name(const char *name)
-{
-  int i;
-  const SSL_METHOD *method = SSLv23_method();
-  int num = method->num_ciphers();
-
-  for (i = 0; i < num; ++i) {
-    const SSL_CIPHER *cipher = method->get_cipher(i);
-    const char *ciphername = SSL_CIPHER_get_name(cipher);
-    if (!strcmp(ciphername, name)) {
-      return (SSL_CIPHER *)cipher;
-    }
-  }
-
-  return NULL;
-}
-#endif
-
-#ifndef OPENSSL_OPAQUE
-static void
-test_tortls_get_ciphersuite_name(void *ignored)
-{
-  (void)ignored;
-  const char *ret;
-  tor_tls_t *ctx;
-  ctx = tor_malloc_zero(sizeof(tor_tls_t));
-  ctx->ssl = tor_malloc_zero(sizeof(SSL));
-
-  ret = tor_tls_get_ciphersuite_name(ctx);
-  tt_str_op(ret, OP_EQ, "(NONE)");
-
- done:
-  tor_free(ctx->ssl);
-  tor_free(ctx);
-}
-
-static SSL_CIPHER *
-get_cipher_by_id(uint16_t id)
-{
-  int i;
-  const SSL_METHOD *method = SSLv23_method();
-  int num = method->num_ciphers();
-  for (i = 0; i < num; ++i) {
-    const SSL_CIPHER *cipher = method->get_cipher(i);
-    if (id == (SSL_CIPHER_get_id(cipher) & 0xffff)) {
-      return (SSL_CIPHER *)cipher;
-    }
-  }
-
-  return NULL;
-}
-
-static void
-test_tortls_classify_client_ciphers(void *ignored)
-{
-  (void)ignored;
-  int i;
-  int ret;
-  SSL_CTX *ctx;
-  SSL *ssl;
-  tor_tls_t *tls;
-  STACK_OF(SSL_CIPHER) *ciphers;
-  SSL_CIPHER *tmp_cipher;
-
-  SSL_library_init();
-  SSL_load_error_strings();
-  tor_tls_allocate_tor_tls_object_ex_data_index();
-
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  tls->magic = TOR_TLS_MAGIC;
-
-  ctx = SSL_CTX_new(TLSv1_method());
-  ssl = SSL_new(ctx);
-  tls->ssl = ssl;
-
-  ciphers = sk_SSL_CIPHER_new_null();
-
-  ret = tor_tls_classify_client_ciphers(ssl, NULL);
-  tt_int_op(ret, OP_EQ, -1);
-
-  SSL_set_ex_data(ssl, tor_tls_object_ex_data_index, tls);
-  tls->client_cipher_list_type = 42;
-
-  ret = tor_tls_classify_client_ciphers(ssl, NULL);
-  tt_int_op(ret, OP_EQ, 42);
-
-  tls->client_cipher_list_type = 0;
-  ret = tor_tls_classify_client_ciphers(ssl, ciphers);
-  tt_int_op(ret, OP_EQ, 1);
-  tt_int_op(tls->client_cipher_list_type, OP_EQ, 1);
-
-  tls->client_cipher_list_type = 0;
-  ret = tor_tls_classify_client_ciphers(ssl, SSL_get_ciphers(ssl));
-  tt_int_op(ret, OP_EQ, 3);
-  tt_int_op(tls->client_cipher_list_type, OP_EQ, 3);
-
-  SSL_CIPHER *one = get_cipher_by_name(TLS1_TXT_DHE_RSA_WITH_AES_128_SHA),
-    *two = get_cipher_by_name(TLS1_TXT_DHE_RSA_WITH_AES_256_SHA),
-    *three = get_cipher_by_name(SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA),
-    *four = NULL;
-  sk_SSL_CIPHER_push(ciphers, one);
-  sk_SSL_CIPHER_push(ciphers, two);
-  sk_SSL_CIPHER_push(ciphers, three);
-  sk_SSL_CIPHER_push(ciphers, four);
-
-  tls->client_cipher_list_type = 0;
-  ret = tor_tls_classify_client_ciphers(ssl, ciphers);
-  tt_int_op(ret, OP_EQ, 1);
-  tt_int_op(tls->client_cipher_list_type, OP_EQ, 1);
-
-  sk_SSL_CIPHER_zero(ciphers);
-
-  one = get_cipher_by_name("ECDHE-RSA-AES256-GCM-SHA384");
-  one->id = 0x00ff;
-  two = get_cipher_by_name("ECDHE-RSA-AES128-GCM-SHA256");
-  two->id = 0x0000;
-  sk_SSL_CIPHER_push(ciphers, one);
-  tls->client_cipher_list_type = 0;
-  ret = tor_tls_classify_client_ciphers(ssl, ciphers);
-  tt_int_op(ret, OP_EQ, 3);
-  tt_int_op(tls->client_cipher_list_type, OP_EQ, 3);
-
-  sk_SSL_CIPHER_push(ciphers, two);
-  tls->client_cipher_list_type = 0;
-  ret = tor_tls_classify_client_ciphers(ssl, ciphers);
-  tt_int_op(ret, OP_EQ, 3);
-  tt_int_op(tls->client_cipher_list_type, OP_EQ, 3);
-
-  one->id = 0xC00A;
-  tls->client_cipher_list_type = 0;
-  ret = tor_tls_classify_client_ciphers(ssl, ciphers);
-  tt_int_op(ret, OP_EQ, 3);
-  tt_int_op(tls->client_cipher_list_type, OP_EQ, 3);
-
-  sk_SSL_CIPHER_zero(ciphers);
-  for (i=0; v2_cipher_list[i]; i++) {
-    tmp_cipher = get_cipher_by_id(v2_cipher_list[i]);
-    tt_assert(tmp_cipher);
-    sk_SSL_CIPHER_push(ciphers, tmp_cipher);
-  }
-  tls->client_cipher_list_type = 0;
-  ret = tor_tls_classify_client_ciphers(ssl, ciphers);
-  tt_int_op(ret, OP_EQ, 2);
-  tt_int_op(tls->client_cipher_list_type, OP_EQ, 2);
-
- done:
-  sk_SSL_CIPHER_free(ciphers);
-  SSL_free(tls->ssl);
-  tor_free(tls);
-  SSL_CTX_free(ctx);
-}
-#endif
-
-static void
-test_tortls_client_is_using_v2_ciphers(void *ignored)
-{
-  (void)ignored;
-
-#ifdef HAVE_SSL_GET_CLIENT_CIPHERS
-  tt_skip();
- done:
-  (void)1;
-#else
-  int ret;
-  SSL_CTX *ctx;
-  SSL *ssl;
-  SSL_SESSION *sess;
-  STACK_OF(SSL_CIPHER) *ciphers;
-
-  SSL_library_init();
-  SSL_load_error_strings();
-
-  ctx = SSL_CTX_new(TLSv1_method());
-  ssl = SSL_new(ctx);
-  sess = SSL_SESSION_new();
-
-  ret = tor_tls_client_is_using_v2_ciphers(ssl);
-  tt_int_op(ret, OP_EQ, -1);
-
-  ssl->session = sess;
-  ret = tor_tls_client_is_using_v2_ciphers(ssl);
-  tt_int_op(ret, OP_EQ, 0);
-
-  ciphers = sk_SSL_CIPHER_new_null();
-  SSL_CIPHER *one = get_cipher_by_name("ECDHE-RSA-AES256-GCM-SHA384");
-  one->id = 0x00ff;
-  sk_SSL_CIPHER_push(ciphers, one);
-  sess->ciphers = ciphers;
-  ret = tor_tls_client_is_using_v2_ciphers(ssl);
-  tt_int_op(ret, OP_EQ, 1);
- done:
-  SSL_free(ssl);
-  SSL_CTX_free(ctx);
-#endif
-}
-
-#ifndef OPENSSL_OPAQUE
-static X509 *fixed_try_to_extract_certs_from_tls_cert_out_result = NULL;
-static X509 *fixed_try_to_extract_certs_from_tls_id_cert_out_result = NULL;
-
-static void
-fixed_try_to_extract_certs_from_tls(int severity, tor_tls_t *tls,
-                                    X509 **cert_out, X509 **id_cert_out)
-{
-  (void) severity;
-  (void) tls;
-  *cert_out = fixed_try_to_extract_certs_from_tls_cert_out_result;
-  *id_cert_out = fixed_try_to_extract_certs_from_tls_id_cert_out_result;
-}
-#endif
-
-#ifndef OPENSSL_OPAQUE
-static const char* notCompletelyValidCertString =
-  "-----BEGIN CERTIFICATE-----\n"
-  "MIICVjCCAb8CAg37MA0GCSqGSIb3DQEBBQUAMIGbMQswCQYDVQQGEwJKUDEOMAwG\n"
-  "A1UECBMFVG9reW8xEDAOBgNVBAcTB0NodW8ta3UxETAPBgNVBAoTCEZyYW5rNERE\n"
-  "MRgwFgYDVQQLEw9XZWJDZXJ0IFN1cHBvcnQxGDAWBgNVBAMTD0ZyYW5rNEREIFdl\n"
-  "YiBDQTEjMCEGCSqGSIb3DQEJARYUc3VwcG9ydEBmcmFuazRkZC5jb20wHhcNMTIw\n"
-  "ODIyMDUyNzIzWhcNMTcwODIxMDUyNzIzWjBKMQswCQYDVQQGEwJKUDEOMAwGA1UE\n"
-  "CAwFVG9reW8xETAPBgNVBAoMCEZyYW5rNEREMRgwFgYDVQQDDA93d3cuZXhhbXBs\n"
-  "ZS5jb20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMYBBrx5PlP0WNI/ZdzD\n"
-  "+6Pktmurn+F2kQYbtc7XQh8/LTBvCo+P6iZoLEmUA9e7EXLRxgU1CVqeAi7QcAn9\n"
-  "MwBlc8ksFJHB0rtf9pmf8Oza9E0Bynlq/4/Kb1x+d+AyhL7oK9tQwB24uHOueHi1\n"
-  "C/iVv8CSWKiYe6hzN1txYe8rAgMBAAEwDQYJKoZIhvcNAQEFBQADgYEAASPdjigJ\n"
-  "kXCqKWpnZ/Oc75EUcMi6HztaW8abUMlYXPIgkV2F7YanHOB7K4f7OOLjiz8DTPFf\n"
-  "jC9UeuErhaA/zzWi8ewMTFZW/WshOrm3fNvcMrMLKtH534JKvcdMg6qIdjTFINIr\n"
-  "evnAhf0cwULaebn+lMs8Pdl7y37+sfluVok=\n"
-  "-----END CERTIFICATE-----\n";
-#endif
-
-static const char* validCertString = "-----BEGIN CERTIFICATE-----\n"
-  "MIIDpTCCAY0CAg3+MA0GCSqGSIb3DQEBBQUAMF4xCzAJBgNVBAYTAlVTMREwDwYD\n"
-  "VQQIDAhJbGxpbm9pczEQMA4GA1UEBwwHQ2hpY2FnbzEUMBIGA1UECgwLVG9yIFRl\n"
-  "c3RpbmcxFDASBgNVBAMMC1RvciBUZXN0aW5nMB4XDTE1MDkwNjEzMzk1OVoXDTQz\n"
-  "MDEyMjEzMzk1OVowVjELMAkGA1UEBhMCVVMxEDAOBgNVBAcMB0NoaWNhZ28xFDAS\n"
-  "BgNVBAoMC1RvciBUZXN0aW5nMR8wHQYDVQQDDBZ0ZXN0aW5nLnRvcnByb2plY3Qu\n"
-  "b3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDoT6uyVVhWyOF3wkHjjYbd\n"
-  "nKaykyRv4JVtKQdZ4OpEErmX1zw4MmyzpQNV6iR4bQnWiyLfzyVJMZDIC/WILBfX\n"
-  "w2Pza/yuLgUvDc3twMuhOACzOQVO8PrEF/aVv2+hbCCy2udXvKhnYn+CCXl3ozc8\n"
-  "XcKYvujTXDyvGWY3xwAjlQIDAQABMA0GCSqGSIb3DQEBBQUAA4ICAQCUvnhzQWuQ\n"
-  "MrN+pERkE+zcTI/9dGS90rUMMLgu8VDNqTa0TUQh8uO0EQ6uDvI8Js6e8tgwS0BR\n"
-  "UBahqb7ZHv+rejGCBr5OudqD+x4STiiuPNJVs86JTLN8SpM9CHjIBH5WCCN2KOy3\n"
-  "mevNoRcRRyYJzSFULCunIK6FGulszigMYGscrO4oiTkZiHPh9KvWT40IMiHfL+Lw\n"
-  "EtEWiLex6064LcA2YQ1AMuSZyCexks63lcfaFmQbkYOKqXa1oLkIRuDsOaSVjTfe\n"
-  "vec+X6jvf12cFTKS5WIeqkKF2Irt+dJoiHEGTe5RscUMN/f+gqHPzfFz5dR23sxo\n"
-  "g+HC6MZHlFkLAOx3wW6epPS8A/m1mw3zMPoTnb2U2YYt8T0dJMMlUn/7Y1sEAa+a\n"
-  "dSTMaeUf6VnJ//11m454EZl1to9Z7oJOgqmFffSrdD4BGIWe8f7hhW6L1Enmqe/J\n"
-  "BKL3wbzZh80O1W0bndAwhnEEhlzneFY84cbBo9pmVxpODHkUcStpr5Z7pBDrcL21\n"
-  "Ss/aB/1YrsVXhdvJdOGxl3Mnl9dUY57CympLGlT8f0pPS6GAKOelECOhFMHmJd8L\n"
-  "dj3XQSmKtYHevZ6IvuMXSlB/fJvSjSlkCuLo5+kJoaqPuRu+i/S1qxeRy3CBwmnE\n"
-  "LdSNdcX4N79GQJ996PA8+mUCQG7YRtK+WA==\n"
-  "-----END CERTIFICATE-----\n";
-
-static const char* caCertString = "-----BEGIN CERTIFICATE-----\n"
-  "MIIFjzCCA3egAwIBAgIJAKd5WgyfPMYRMA0GCSqGSIb3DQEBCwUAMF4xCzAJBgNV\n"
-  "BAYTAlVTMREwDwYDVQQIDAhJbGxpbm9pczEQMA4GA1UEBwwHQ2hpY2FnbzEUMBIG\n"
-  "A1UECgwLVG9yIFRlc3RpbmcxFDASBgNVBAMMC1RvciBUZXN0aW5nMB4XDTE1MDkw\n"
-  "NjEzMzc0MVoXDTQzMDEyMjEzMzc0MVowXjELMAkGA1UEBhMCVVMxETAPBgNVBAgM\n"
-  "CElsbGlub2lzMRAwDgYDVQQHDAdDaGljYWdvMRQwEgYDVQQKDAtUb3IgVGVzdGlu\n"
-  "ZzEUMBIGA1UEAwwLVG9yIFRlc3RpbmcwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw\n"
-  "ggIKAoICAQCpLMUEiLW5leUgBZoEJms2V7lZRhIAjnJBhVMHD0e3UubNknmaQoxf\n"
-  "ARz3rvqOaRd0JlV+qM9qE0DjiYcCVP1cAfqAo9d83uS1vwY3YMVJzADlaIiHfyVW\n"
-  "uEgBy0vvkeUBqaua24dYlcwsemOiXYLu41yM1wkcGHW1AhBNHppY6cznb8TyLgNM\n"
-  "2x3SGUdzc5XMyAFx51faKGBA3wjs+Hg1PLY7d30nmCgEOBavpm5I1disM/0k+Mcy\n"
-  "YmAKEo/iHJX/rQzO4b9znP69juLlR8PDBUJEVIG/CYb6+uw8MjjUyiWXYoqfVmN2\n"
-  "hm/lH8b6rXw1a2Aa3VTeD0DxaWeacMYHY/i01fd5n7hCoDTRNdSw5KJ0L3Z0SKTu\n"
-  "0lzffKzDaIfyZGlpW5qdouACkWYzsaitQOePVE01PIdO30vUfzNTFDfy42ccx3Di\n"
-  "59UCu+IXB+eMtrBfsok0Qc63vtF1linJgjHW1z/8ujk8F7/qkOfODhk4l7wngc2A\n"
-  "EmwWFIFoGaiTEZHB9qteXr4unbXZ0AHpM02uGGwZEGohjFyebEb73M+J57WKKAFb\n"
-  "PqbLcGUksL1SHNBNAJcVLttX55sO4nbidOS/kA3m+F1R04MBTyQF9qA6YDDHqdI3\n"
-  "h/3pw0Z4fxVouTYT4/NfRnX4JTP4u+7Mpcoof28VME0qWqD1LnRhFQIDAQABo1Aw\n"
-  "TjAdBgNVHQ4EFgQUMoAgIXH7pZ3QMRwTjT+DM9Yo/v0wHwYDVR0jBBgwFoAUMoAg\n"
-  "IXH7pZ3QMRwTjT+DM9Yo/v0wDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOC\n"
-  "AgEAUJxacjXR9sT+Xs6ISFiUsyd0T6WVKMnV46xrYJHirGfx+krWHrjxMY+ZtxYD\n"
-  "DBDGlo11Qc4v6QrclNf5QUBfIiGQsP9Cm6hHcQ+Tpg9HHCgSqG1YNPwCPReCR4br\n"
-  "BLvLfrfkcBL2IWM0PdQdCze+59DBfipsULD2mEn9fjYRXQEwb2QWtQ9qRc20Yb/x\n"
-  "Q4b/+CvUodLkaq7B8MHz0BV8HHcBoph6DYaRmO/N+hPauIuSp6XyaGYcEefGKVKj\n"
-  "G2+fcsdyXsoijNdL8vNKwm4j2gVwCBnw16J00yfFoV46YcbfqEdJB2je0XSvwXqt\n"
-  "14AOTngxso2h9k9HLtrfpO1ZG/B5AcCMs1lzbZ2fp5DPHtjvvmvA2RJqgo3yjw4W\n"
-  "4DHAuTglYFlC3mDHNfNtcGP20JvepcQNzNP2UzwcpOc94hfKikOFw+gf9Vf1qd0y\n"
-  "h/Sk6OZHn2+JVUPiWHIQV98Vtoh4RmUZDJD+b55ia3fQGTGzt4z1XFzQYSva5sfs\n"
-  "wocS/papthqWldQU7x+3wofNd5CNU1x6WKXG/yw30IT/4F8ADJD6GeygNT8QJYvt\n"
-  "u/8lAkbOy6B9xGmSvr0Kk1oq9P2NshA6kalxp1Oz/DTNDdL4AeBXV3JmM6WWCjGn\n"
-  "Yy1RT69d0rwYc5u/vnqODz1IjvT90smsrkBumGt791FAFeg=\n"
-  "-----END CERTIFICATE-----\n";
-
-static X509 *
-read_cert_from(const char *str)
-{
-  BIO *bio = BIO_new(BIO_s_mem());
-  BIO_write(bio, str, (int) strlen(str));
-  X509 *res = PEM_read_bio_X509(bio, NULL, NULL, NULL);
-  BIO_free(bio);
-  return res;
-}
-
-#ifndef OPENSSL_OPAQUE
-static void
-test_tortls_verify(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  tor_tls_t *tls;
-  crypto_pk_t *k = NULL;
-  X509 *cert1 = NULL, *cert2 = NULL, *invalidCert = NULL,
-    *validCert = NULL, *caCert = NULL;
-
-  cert1 = tor_malloc_zero(sizeof(X509));
-  cert1->references = 10;
-
-  cert2 = tor_malloc_zero(sizeof(X509));
-  cert2->references = 10;
-
-  validCert = read_cert_from(validCertString);
-  caCert = read_cert_from(caCertString);
-  invalidCert = read_cert_from(notCompletelyValidCertString);
-
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  ret = tor_tls_verify(LOG_WARN, tls, &k);
-  tt_int_op(ret, OP_EQ, -1);
-
-  MOCK(try_to_extract_certs_from_tls, fixed_try_to_extract_certs_from_tls);
-
-  fixed_try_to_extract_certs_from_tls_cert_out_result = cert1;
-  ret = tor_tls_verify(LOG_WARN, tls, &k);
-  tt_int_op(ret, OP_EQ, -1);
-
-  fixed_try_to_extract_certs_from_tls_id_cert_out_result = cert2;
-  ret = tor_tls_verify(LOG_WARN, tls, &k);
-  tt_int_op(ret, OP_EQ, -1);
-
-  fixed_try_to_extract_certs_from_tls_cert_out_result = invalidCert;
-  fixed_try_to_extract_certs_from_tls_id_cert_out_result = invalidCert;
-
-  ret = tor_tls_verify(LOG_WARN, tls, &k);
-  tt_int_op(ret, OP_EQ, -1);
-
-  fixed_try_to_extract_certs_from_tls_cert_out_result = validCert;
-  fixed_try_to_extract_certs_from_tls_id_cert_out_result = caCert;
-
-  ret = tor_tls_verify(LOG_WARN, tls, &k);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_assert(k);
-
- done:
-  UNMOCK(try_to_extract_certs_from_tls);
-  tor_free(cert1);
-  tor_free(cert2);
-  tor_free(tls);
-  tor_free(k);
-}
-#endif
-
-#ifndef OPENSSL_OPAQUE
-static void
-test_tortls_check_lifetime(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  tor_tls_t *tls;
-  X509 *validCert = read_cert_from(validCertString);
-  time_t now = time(NULL);
-
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  ret = tor_tls_check_lifetime(LOG_WARN, tls, 0, 0);
-  tt_int_op(ret, OP_EQ, -1);
-
-  tls->ssl = tor_malloc_zero(sizeof(SSL));
-  tls->ssl->session = tor_malloc_zero(sizeof(SSL_SESSION));
-  tls->ssl->session->peer = validCert;
-  ret = tor_tls_check_lifetime(LOG_WARN, tls, 0, 0);
-  tt_int_op(ret, OP_EQ, 0);
-
-  ASN1_STRING_free(validCert->cert_info->validity->notBefore);
-  validCert->cert_info->validity->notBefore = ASN1_TIME_set(NULL, now-10);
-  ASN1_STRING_free(validCert->cert_info->validity->notAfter);
-  validCert->cert_info->validity->notAfter = ASN1_TIME_set(NULL, now+60);
-
-  ret = tor_tls_check_lifetime(LOG_WARN, tls, 0, -1000);
-  tt_int_op(ret, OP_EQ, -1);
-
-  ret = tor_tls_check_lifetime(LOG_WARN, tls, -1000, 0);
-  tt_int_op(ret, OP_EQ, -1);
-
- done:
-  tor_free(tls->ssl->session);
-  tor_free(tls->ssl);
-  tor_free(tls);
-  X509_free(validCert);
-}
-#endif
-
-#ifndef OPENSSL_OPAQUE
-static int fixed_ssl_pending_result = 0;
-
-static int
-fixed_ssl_pending(const SSL *ignored)
-{
-  (void)ignored;
-  return fixed_ssl_pending_result;
-}
-
-static void
-test_tortls_get_pending_bytes(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  tor_tls_t *tls;
-  SSL_METHOD *method;
-
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  tls->ssl = tor_malloc_zero(sizeof(SSL));
-  method = tor_malloc_zero(sizeof(SSL_METHOD));
-  method->ssl_pending = fixed_ssl_pending;
-  tls->ssl->method = method;
-
-  fixed_ssl_pending_result = 42;
-  ret = tor_tls_get_pending_bytes(tls);
-  tt_int_op(ret, OP_EQ, 42);
-
- done:
-  tor_free(method);
-  tor_free(tls->ssl);
-  tor_free(tls);
-}
-#endif
-
+#ifdef ENABLE_OPENSSL
 static void
 test_tortls_get_forced_write_size(void *ignored)
 {
@@ -1173,30 +300,6 @@ test_tortls_get_forced_write_size(void *ignored)
 }
 
 static void
-test_tortls_get_write_overhead_ratio(void *ignored)
-{
-  (void)ignored;
-  double ret;
-
-  total_bytes_written_over_tls = 0;
-  ret = tls_get_write_overhead_ratio();
-  tt_double_op(fabs(ret - 1.0), OP_LT, 1E-12);
-
-  total_bytes_written_by_tls = 10;
-  total_bytes_written_over_tls = 1;
-  ret = tls_get_write_overhead_ratio();
-  tt_double_op(fabs(ret - 10.0), OP_LT, 1E-12);
-
-  total_bytes_written_by_tls = 10;
-  total_bytes_written_over_tls = 2;
-  ret = tls_get_write_overhead_ratio();
-  tt_double_op(fabs(ret - 5.0), OP_LT, 1E-12);
-
- done:
-  (void)0;
-}
-
-static void
 test_tortls_used_v1_handshake(void *ignored)
 {
   (void)ignored;
@@ -1218,23 +321,6 @@ test_tortls_used_v1_handshake(void *ignored)
 }
 
 static void
-test_tortls_get_num_server_handshakes(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  tor_tls_t *tls;
-
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-
-  tls->server_handshake_count = 3;
-  ret = tor_tls_get_num_server_handshakes(tls);
-  tt_int_op(ret, OP_EQ, 3);
-
- done:
-  tor_free(tls);
-}
-
-static void
 test_tortls_server_got_renegotiate(void *ignored)
 {
   (void)ignored;
@@ -1250,115 +336,6 @@ test_tortls_server_got_renegotiate(void *ignored)
  done:
   tor_free(tls);
 }
-
-#ifndef OPENSSL_OPAQUE
-static void
-test_tortls_SSL_SESSION_get_master_key(void *ignored)
-{
-  (void)ignored;
-  size_t ret;
-  tor_tls_t *tls;
-  uint8_t *out;
-  out = tor_malloc_zero(1);
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  tls->ssl = tor_malloc_zero(sizeof(SSL));
-  tls->ssl->session = tor_malloc_zero(sizeof(SSL_SESSION));
-  tls->ssl->session->master_key_length = 1;
-
-#ifndef HAVE_SSL_SESSION_GET_MASTER_KEY
-  tls->ssl->session->master_key[0] = 43;
-  ret = SSL_SESSION_get_master_key(tls->ssl->session, out, 0);
-  tt_int_op(ret, OP_EQ, 1);
-  tt_int_op(out[0], OP_EQ, 0);
-
-  ret = SSL_SESSION_get_master_key(tls->ssl->session, out, 1);
-  tt_int_op(ret, OP_EQ, 1);
-  tt_int_op(out[0], OP_EQ, 43);
-
- done:
-#endif
-  tor_free(tls->ssl->session);
-  tor_free(tls->ssl);
-  tor_free(tls);
-  tor_free(out);
-}
-#endif
-
-#ifndef OPENSSL_OPAQUE
-static void
-test_tortls_get_tlssecrets(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  uint8_t *secret_out = tor_malloc_zero(DIGEST256_LEN);;
-  tor_tls_t *tls;
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  tls->ssl = tor_malloc_zero(sizeof(SSL));
-  tls->ssl->session = tor_malloc_zero(sizeof(SSL_SESSION));
-  tls->ssl->session->master_key_length = 1;
-  tls->ssl->s3 = tor_malloc_zero(sizeof(SSL3_STATE));
-
-  ret = tor_tls_get_tlssecrets(tls, secret_out);
-  tt_int_op(ret, OP_EQ, 0);
-
- done:
-  tor_free(secret_out);
-  tor_free(tls->ssl->s3);
-  tor_free(tls->ssl->session);
-  tor_free(tls->ssl);
-  tor_free(tls);
-}
-#endif
-
-#ifndef OPENSSL_OPAQUE
-static void
-test_tortls_get_buffer_sizes(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  tor_tls_t *tls;
-  size_t rbuf_c=-1, rbuf_b=-1, wbuf_c=-1, wbuf_b=-1;
-
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  tls->ssl = tor_malloc_zero(sizeof(SSL));
-  tls->ssl->s3 = tor_malloc_zero(sizeof(SSL3_STATE));
-
-  tls->ssl->s3->rbuf.buf = NULL;
-  tls->ssl->s3->rbuf.len = 1;
-  tls->ssl->s3->rbuf.offset = 0;
-  tls->ssl->s3->rbuf.left = 42;
-
-  tls->ssl->s3->wbuf.buf = NULL;
-  tls->ssl->s3->wbuf.len = 2;
-  tls->ssl->s3->wbuf.offset = 0;
-  tls->ssl->s3->wbuf.left = 43;
-
-  ret = tor_tls_get_buffer_sizes(tls, &rbuf_c, &rbuf_b, &wbuf_c, &wbuf_b);
-#if OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0)
-  tt_int_op(ret, OP_EQ, -1);
-#else
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(rbuf_c, OP_EQ, 0);
-  tt_int_op(wbuf_c, OP_EQ, 0);
-  tt_int_op(rbuf_b, OP_EQ, 42);
-  tt_int_op(wbuf_b, OP_EQ, 43);
-
-  tls->ssl->s3->rbuf.buf = tor_malloc_zero(1);
-  tls->ssl->s3->wbuf.buf = tor_malloc_zero(1);
-  ret = tor_tls_get_buffer_sizes(tls, &rbuf_c, &rbuf_b, &wbuf_c, &wbuf_b);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(rbuf_c, OP_EQ, 1);
-  tt_int_op(wbuf_c, OP_EQ, 2);
-
-#endif
-
- done:
-  tor_free(tls->ssl->s3->rbuf.buf);
-  tor_free(tls->ssl->s3->wbuf.buf);
-  tor_free(tls->ssl->s3);
-  tor_free(tls->ssl);
-  tor_free(tls);
-}
 #endif
 
 static void
@@ -1378,1448 +355,183 @@ test_tortls_evaluate_ecgroup_for_tls(void *ignored)
 
   ret = evaluate_ecgroup_for_tls("P224");
   //  tt_int_op(ret, OP_EQ, 1); This varies between machines
+  tt_assert(ret == 0 || ret == 1);
 
  done:
   (void)0;
 }
 
-#ifndef OPENSSL_OPAQUE
-typedef struct cert_pkey_st_local
-{
-  X509 *x509;
-  EVP_PKEY *privatekey;
-  const EVP_MD *digest;
-} CERT_PKEY_local;
-
-typedef struct sess_cert_st_local
-{
-  STACK_OF(X509) *cert_chain;
-  int peer_cert_type;
-  CERT_PKEY_local *peer_key;
-  CERT_PKEY_local peer_pkeys[8];
-  int references;
-} SESS_CERT_local;
-
-static void
-test_tortls_try_to_extract_certs_from_tls(void *ignored)
-{
-  (void)ignored;
-  tor_tls_t *tls;
-  X509 *cert = NULL, *id_cert = NULL, *c1 = NULL, *c2 = NULL;
-  SESS_CERT_local *sess = NULL;
-
-  c1 = read_cert_from(validCertString);
-  c2 = read_cert_from(caCertString);
-
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  tls->ssl = tor_malloc_zero(sizeof(SSL));
-  tls->ssl->session = tor_malloc_zero(sizeof(SSL_SESSION));
-  sess = tor_malloc_zero(sizeof(SESS_CERT_local));
-  tls->ssl->session->sess_cert = (void *)sess;
-
-  try_to_extract_certs_from_tls(LOG_WARN, tls, &cert, &id_cert);
-  tt_assert(!cert);
-  tt_assert(!id_cert);
-
-  tls->ssl->session->peer = c1;
-  try_to_extract_certs_from_tls(LOG_WARN, tls, &cert, &id_cert);
-  tt_assert(cert == c1);
-  tt_assert(!id_cert);
-  X509_free(cert); /* decrease refcnt */
-
-  sess->cert_chain = sk_X509_new_null();
-  try_to_extract_certs_from_tls(LOG_WARN, tls, &cert, &id_cert);
-  tt_assert(cert == c1);
-  tt_assert(!id_cert);
-  X509_free(cert); /* decrease refcnt */
-
-  sk_X509_push(sess->cert_chain, c1);
-  sk_X509_push(sess->cert_chain, c2);
-
-  try_to_extract_certs_from_tls(LOG_WARN, tls, &cert, &id_cert);
-  tt_assert(cert == c1);
-  tt_assert(id_cert);
-  X509_free(cert); /* decrease refcnt */
-
- done:
-  sk_X509_free(sess->cert_chain);
-  tor_free(sess);
-  tor_free(tls->ssl->session);
-  tor_free(tls->ssl);
-  tor_free(tls);
-  X509_free(c1);
-  X509_free(c2);
-}
-#endif
-
-#ifndef OPENSSL_OPAQUE
-static void
-test_tortls_get_peer_cert(void *ignored)
-{
-  (void)ignored;
-  tor_x509_cert_t *ret;
-  tor_tls_t *tls;
-  X509 *cert = NULL;
-
-  cert = read_cert_from(validCertString);
-
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  tls->ssl = tor_malloc_zero(sizeof(SSL));
-  tls->ssl->session = tor_malloc_zero(sizeof(SSL_SESSION));
-
-  ret = tor_tls_get_peer_cert(tls);
-  tt_assert(!ret);
-
-  tls->ssl->session->peer = cert;
-  ret = tor_tls_get_peer_cert(tls);
-  tt_assert(ret);
-  tt_assert(ret->cert == cert);
-
- done:
-  tor_x509_cert_free(ret);
-  tor_free(tls->ssl->session);
-  tor_free(tls->ssl);
-  tor_free(tls);
-  X509_free(cert);
-}
-#endif
-
-#ifndef OPENSSL_OPAQUE
-static void
-test_tortls_peer_has_cert(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  tor_tls_t *tls;
-  X509 *cert = NULL;
-
-  cert = read_cert_from(validCertString);
-
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  tls->ssl = tor_malloc_zero(sizeof(SSL));
-  tls->ssl->session = tor_malloc_zero(sizeof(SSL_SESSION));
-
-  ret = tor_tls_peer_has_cert(tls);
-  tt_assert(!ret);
-
-  tls->ssl->session->peer = cert;
-  ret = tor_tls_peer_has_cert(tls);
-  tt_assert(ret);
-
- done:
-  tor_free(tls->ssl->session);
-  tor_free(tls->ssl);
-  tor_free(tls);
-  X509_free(cert);
-}
-#endif
-
-static void
-test_tortls_is_server(void *ignored)
-{
-  (void)ignored;
-  tor_tls_t *tls;
-  int ret;
-
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  tls->isServer = 1;
-  ret = tor_tls_is_server(tls);
-  tt_int_op(ret, OP_EQ, 1);
-
- done:
-  tor_free(tls);
-}
-
-#ifndef OPENSSL_OPAQUE
-static void
-test_tortls_session_secret_cb(void *ignored)
-{
-  (void)ignored;
-  tor_tls_t *tls;
-  SSL_CTX *ctx;
-  STACK_OF(SSL_CIPHER) *ciphers = NULL;
-  SSL_CIPHER *one;
-
-  SSL_library_init();
-  SSL_load_error_strings();
-  tor_tls_allocate_tor_tls_object_ex_data_index();
-
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-
-  tls->magic = TOR_TLS_MAGIC;
-
-  ctx = SSL_CTX_new(TLSv1_method());
-  tls->ssl = SSL_new(ctx);
-  SSL_set_ex_data(tls->ssl, tor_tls_object_ex_data_index, tls);
-
-  SSL_set_session_secret_cb(tls->ssl, tor_tls_session_secret_cb, NULL);
-
-  tor_tls_session_secret_cb(tls->ssl, NULL, NULL, NULL, NULL, NULL);
-  tt_assert(!tls->ssl->tls_session_secret_cb);
-
-  one = get_cipher_by_name("ECDHE-RSA-AES256-GCM-SHA384");
-  one->id = 0x00ff;
-  ciphers = sk_SSL_CIPHER_new_null();
-  sk_SSL_CIPHER_push(ciphers, one);
-
-  tls->client_cipher_list_type = 0;
-  tor_tls_session_secret_cb(tls->ssl, NULL, NULL, ciphers, NULL, NULL);
-  tt_assert(!tls->ssl->tls_session_secret_cb);
-
- done:
-  sk_SSL_CIPHER_free(ciphers);
-  SSL_free(tls->ssl);
-  SSL_CTX_free(ctx);
-  tor_free(tls);
-}
-#endif
-
-#ifndef OPENSSL_OPAQUE
-/* TODO: It seems block_renegotiation and unblock_renegotiation and
- * using different blags. This might not be correct */
-static void
-test_tortls_block_renegotiation(void *ignored)
-{
-  (void)ignored;
-  tor_tls_t *tls;
-
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  tls->ssl = tor_malloc_zero(sizeof(SSL));
-  tls->ssl->s3 = tor_malloc_zero(sizeof(SSL3_STATE));
-#ifndef SUPPORT_UNSAFE_RENEGOTIATION_FLAG
-#define SSL3_FLAGS_ALLOW_UNSAFE_LEGACY_RENEGOTIATION 0
-#endif
-
-  tls->ssl->s3->flags = SSL3_FLAGS_ALLOW_UNSAFE_LEGACY_RENEGOTIATION;
-
-  tor_tls_block_renegotiation(tls);
-
-#ifndef OPENSSL_1_1_API
-  tt_assert(!(tls->ssl->s3->flags &
-              SSL3_FLAGS_ALLOW_UNSAFE_LEGACY_RENEGOTIATION));
-#endif
-
- done:
-  tor_free(tls->ssl->s3);
-  tor_free(tls->ssl);
-  tor_free(tls);
-}
-
 static void
-test_tortls_unblock_renegotiation(void *ignored)
+test_tortls_double_init(void *arg)
 {
-  (void)ignored;
-  tor_tls_t *tls;
+  (void) arg;
+  /* If we call tor_tls_context_init() a second time, nothing should go
+   * wrong.
+   */
+  crypto_pk_t *pk1 = NULL, *pk2 = NULL;
+  pk1 = pk_generate(2);
+  pk2 = pk_generate(0);
 
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  tls->ssl = tor_malloc_zero(sizeof(SSL));
-  tor_tls_unblock_renegotiation(tls);
+  int r = tor_tls_context_init(TOR_TLS_CTX_IS_PUBLIC_SERVER,
+                               pk1, pk2, 86400);
+  tt_int_op(r, OP_EQ, 0);
 
-  tt_uint_op(SSL_get_options(tls->ssl) &
-             SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION, OP_EQ,
-             SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION);
+  r = tor_tls_context_init(TOR_TLS_CTX_IS_PUBLIC_SERVER,
+                               pk2, pk1, 86400);
+  tt_int_op(r, OP_EQ, 0);
+  /* For a public server context, these are the same */
+  tt_ptr_op(tor_tls_context_get(0), OP_EQ, tor_tls_context_get(1));
 
  done:
-  tor_free(tls->ssl);
-  tor_free(tls);
-}
-#endif
-
-#ifndef OPENSSL_OPAQUE
-static void
-test_tortls_assert_renegotiation_unblocked(void *ignored)
-{
-  (void)ignored;
-  tor_tls_t *tls;
-
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  tls->ssl = tor_malloc_zero(sizeof(SSL));
-  tor_tls_unblock_renegotiation(tls);
-  tor_tls_assert_renegotiation_unblocked(tls);
-  /* No assertion here - this test will fail if tor_assert is turned on
-   * and things are bad. */
-
-  tor_free(tls->ssl);
-  tor_free(tls);
-}
-#endif
-
-static void
-test_tortls_set_logged_address(void *ignored)
-{
-  (void)ignored;
-  tor_tls_t *tls;
-
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-
-  tor_tls_set_logged_address(tls, "foo bar");
-
-  tt_str_op(tls->address, OP_EQ, "foo bar");
-
-  tor_tls_set_logged_address(tls, "foo bar 2");
-  tt_str_op(tls->address, OP_EQ, "foo bar 2");
-
- done:
-  tor_free(tls->address);
-  tor_free(tls);
+  crypto_pk_free(pk1);
+  crypto_pk_free(pk2);
 }
 
-#ifndef OPENSSL_OPAQUE
 static void
-example_cb(tor_tls_t *t, void *arg)
+test_tortls_bridge_init(void *arg)
 {
-  (void)t;
   (void)arg;
-}
-
-static void
-test_tortls_set_renegotiate_callback(void *ignored)
-{
-  (void)ignored;
-  tor_tls_t *tls;
-  const char *arg = "hello";
-
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  tls->ssl = tor_malloc_zero(sizeof(SSL));
-
-  tor_tls_set_renegotiate_callback(tls, example_cb, (void*)arg);
-  tt_assert(tls->negotiated_callback == example_cb);
-  tt_assert(tls->callback_arg == arg);
-  tt_assert(!tls->got_renegotiate);
-
-  /* Assumes V2_HANDSHAKE_SERVER */
-  tt_assert(tls->ssl->info_callback == tor_tls_server_info_callback);
-
-  tor_tls_set_renegotiate_callback(tls, NULL, (void*)arg);
-  tt_assert(tls->ssl->info_callback == tor_tls_debug_state_callback);
-
- done:
-  tor_free(tls->ssl);
-  tor_free(tls);
-}
-#endif
-
-#ifndef OPENSSL_OPAQUE
-static SSL_CIPHER *fixed_cipher1 = NULL;
-static SSL_CIPHER *fixed_cipher2 = NULL;
-static const SSL_CIPHER *
-fake_get_cipher(unsigned ncipher)
-{
-
-  switch (ncipher) {
-  case 1:
-    return fixed_cipher1;
-  case 2:
-    return fixed_cipher2;
-  default:
-    return NULL;
-  }
-}
-#endif
-
-#ifndef OPENSSL_OPAQUE
-static void
-test_tortls_find_cipher_by_id(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  SSL *ssl;
-  SSL_CTX *ctx;
-  const SSL_METHOD *m = TLSv1_method();
-  SSL_METHOD *empty_method = tor_malloc_zero(sizeof(SSL_METHOD));
-
-  fixed_cipher1 = tor_malloc_zero(sizeof(SSL_CIPHER));
-  fixed_cipher2 = tor_malloc_zero(sizeof(SSL_CIPHER));
-  fixed_cipher2->id = 0xC00A;
-
-  SSL_library_init();
-  SSL_load_error_strings();
-
-  ctx = SSL_CTX_new(m);
-  ssl = SSL_new(ctx);
-
-  ret = find_cipher_by_id(ssl, NULL, 0xC00A);
-  tt_int_op(ret, OP_EQ, 1);
-
-  ret = find_cipher_by_id(ssl, m, 0xC00A);
-  tt_int_op(ret, OP_EQ, 1);
-
-  ret = find_cipher_by_id(ssl, m, 0xFFFF);
-  tt_int_op(ret, OP_EQ, 0);
-
-  ret = find_cipher_by_id(ssl, empty_method, 0xC00A);
-  tt_int_op(ret, OP_EQ, 1);
-
-  ret = find_cipher_by_id(ssl, empty_method, 0xFFFF);
-#ifdef HAVE_SSL_CIPHER_FIND
-  tt_int_op(ret, OP_EQ, 0);
-#else
-  tt_int_op(ret, OP_EQ, 1);
-#endif
-
-  empty_method->get_cipher = fake_get_cipher;
-  ret = find_cipher_by_id(ssl, empty_method, 0xC00A);
-  tt_int_op(ret, OP_EQ, 1);
-
-  empty_method->get_cipher = m->get_cipher;
-  empty_method->num_ciphers = m->num_ciphers;
-  ret = find_cipher_by_id(ssl, empty_method, 0xC00A);
-  tt_int_op(ret, OP_EQ, 1);
-
-  empty_method->get_cipher = fake_get_cipher;
-  empty_method->num_ciphers = m->num_ciphers;
-  ret = find_cipher_by_id(ssl, empty_method, 0xC00A);
-  tt_int_op(ret, OP_EQ, 1);
-
-  empty_method->num_ciphers = fake_num_ciphers;
-  ret = find_cipher_by_id(ssl, empty_method, 0xC00A);
-#ifdef HAVE_SSL_CIPHER_FIND
-  tt_int_op(ret, OP_EQ, 1);
-#else
-  tt_int_op(ret, OP_EQ, 0);
-#endif
-
- done:
-  tor_free(empty_method);
-  SSL_free(ssl);
-  SSL_CTX_free(ctx);
-  tor_free(fixed_cipher1);
-}
-#endif
-
-#ifndef OPENSSL_OPAQUE
-static void
-test_tortls_debug_state_callback(void *ignored)
-{
-  (void)ignored;
-  SSL *ssl;
-  char *buf = tor_malloc_zero(1000);
-  int n;
-
-  setup_capture_of_logs(LOG_DEBUG);
-
-  ssl = tor_malloc_zero(sizeof(SSL));
-
-  tor_tls_debug_state_callback(ssl, 32, 45);
-
-  n = tor_snprintf(buf, 1000, "SSL %p is now in state unknown"
-               " state [type=32,val=45].\n", ssl);
-  /* tor's snprintf returns -1 on error */
-  tt_int_op(n, OP_NE, -1);
-  expect_log_msg(buf);
-
- done:
-  teardown_capture_of_logs();
-  tor_free(buf);
-  tor_free(ssl);
-}
-#endif
-
-#ifndef OPENSSL_OPAQUE
-static void
-test_tortls_server_info_callback(void *ignored)
-{
-  (void)ignored;
-  tor_tls_t *tls;
-  SSL_CTX *ctx;
-  SSL *ssl;
-
-  SSL_library_init();
-  SSL_load_error_strings();
+  crypto_pk_t *pk1 = NULL, *pk2 = NULL;
+  pk1 = pk_generate(2);
+  pk2 = pk_generate(0);
 
-  ctx = SSL_CTX_new(TLSv1_method());
-  ssl = SSL_new(ctx);
-
-  tor_tls_allocate_tor_tls_object_ex_data_index();
-
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  tls->magic = TOR_TLS_MAGIC;
-  tls->ssl = ssl;
-
-  setup_full_capture_of_logs(LOG_WARN);
-  SSL_set_state(ssl, SSL3_ST_SW_SRVR_HELLO_A);
-  mock_clean_saved_logs();
-  tor_tls_server_info_callback(ssl, SSL_CB_ACCEPT_LOOP, 0);
-  expect_single_log_msg("Couldn't look up the tls for an SSL*. How odd!\n");
-
-  SSL_set_state(ssl, SSL3_ST_SW_SRVR_HELLO_B);
-  mock_clean_saved_logs();
-  tor_tls_server_info_callback(ssl, SSL_CB_ACCEPT_LOOP, 0);
-  expect_single_log_msg("Couldn't look up the tls for an SSL*. How odd!\n");
-
-  SSL_set_state(ssl, 99);
-  mock_clean_saved_logs();
-  tor_tls_server_info_callback(ssl, SSL_CB_ACCEPT_LOOP, 0);
-  expect_no_log_entry();
-  teardown_capture_of_logs();
-
-  SSL_set_ex_data(tls->ssl, tor_tls_object_ex_data_index, tls);
-  SSL_set_state(ssl, SSL3_ST_SW_SRVR_HELLO_B);
-  tls->negotiated_callback = 0;
-  tls->server_handshake_count = 120;
-  tor_tls_server_info_callback(ssl, SSL_CB_ACCEPT_LOOP, 0);
-  tt_int_op(tls->server_handshake_count, OP_EQ, 121);
-
-  tls->server_handshake_count = 127;
-  tls->negotiated_callback = (void *)1;
-  tor_tls_server_info_callback(ssl, SSL_CB_ACCEPT_LOOP, 0);
-  tt_int_op(tls->server_handshake_count, OP_EQ, 127);
-  tt_int_op(tls->got_renegotiate, OP_EQ, 1);
-
-  tls->ssl->session = SSL_SESSION_new();
-  tls->wasV2Handshake = 0;
-  tor_tls_server_info_callback(ssl, SSL_CB_ACCEPT_LOOP, 0);
-  tt_int_op(tls->wasV2Handshake, OP_EQ, 0);
+  /* If we pass in a server identity key but not the
+     TOR_TLS_CTX_IS_PUBLIC_SERVER flag, we should get a bridge-style
+     configuration, with two distinct contexts. */
+  int r = tor_tls_context_init(0 /* flags */, pk1, pk2, 86400);
 
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(tor_tls_context_get(0), OP_NE, tor_tls_context_get(1));
  done:
-  teardown_capture_of_logs();
-  SSL_free(ssl);
-  SSL_CTX_free(ctx);
-  tor_free(tls);
-}
-#endif
-
-#ifndef OPENSSL_OPAQUE
-static int fixed_ssl_read_result_index;
-static int fixed_ssl_read_result[5];
-static int fixed_ssl_shutdown_result;
-
-static int
-fixed_ssl_read(SSL *s, void *buf, int len)
-{
-  (void)s;
-  (void)buf;
-  (void)len;
-  return fixed_ssl_read_result[fixed_ssl_read_result_index++];
-}
-
-static int
-fixed_ssl_shutdown(SSL *s)
-{
-  (void)s;
-  return fixed_ssl_shutdown_result;
-}
-
-#ifndef LIBRESSL_VERSION_NUMBER
-static int fixed_ssl_state_to_set;
-static tor_tls_t *fixed_tls;
-
-static int
-setting_version_ssl_shutdown(SSL *s)
-{
-  s->version = SSL2_VERSION;
-  return fixed_ssl_shutdown_result;
-}
-
-static int
-setting_version_and_state_ssl_shutdown(SSL *s)
-{
-  fixed_tls->state = fixed_ssl_state_to_set;
-  s->version = SSL2_VERSION;
-  return fixed_ssl_shutdown_result;
-}
-#endif
-
-static int
-dummy_handshake_func(SSL *s)
-{
-  (void)s;
-  return 1;
-}
-
-static void
-test_tortls_shutdown(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  tor_tls_t *tls;
-  SSL_METHOD *method = give_me_a_test_method();
-  setup_capture_of_logs(LOG_WARN);
-
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  tls->ssl = tor_malloc_zero(sizeof(SSL));
-  tls->ssl->method = method;
-  method->ssl_read = fixed_ssl_read;
-  method->ssl_shutdown = fixed_ssl_shutdown;
-
-  ret = tor_tls_shutdown(tls);
-  tt_int_op(ret, OP_EQ, -9);
-
-  tls->state = TOR_TLS_ST_SENTCLOSE;
-  fixed_ssl_read_result_index = 0;
-  fixed_ssl_read_result[0] = 10;
-  fixed_ssl_read_result[1] = -1;
-  ret = tor_tls_shutdown(tls);
-  tt_int_op(ret, OP_EQ, -9);
-
-#ifndef LIBRESSL_VERSION_NUMBER
-  tls->ssl->handshake_func = dummy_handshake_func;
-
-  fixed_ssl_read_result_index = 0;
-  fixed_ssl_read_result[0] = 10;
-  fixed_ssl_read_result[1] = 42;
-  fixed_ssl_read_result[2] = 0;
-  fixed_ssl_shutdown_result = 1;
-  ERR_clear_error();
-  tls->ssl->version = SSL2_VERSION;
-  ret = tor_tls_shutdown(tls);
-  tt_int_op(ret, OP_EQ, TOR_TLS_DONE);
-  tt_int_op(tls->state, OP_EQ, TOR_TLS_ST_CLOSED);
-
-  fixed_ssl_read_result_index = 0;
-  fixed_ssl_read_result[0] = 10;
-  fixed_ssl_read_result[1] = 42;
-  fixed_ssl_read_result[2] = 0;
-  fixed_ssl_shutdown_result = 0;
-  ERR_clear_error();
-  tls->ssl->version = 0;
-  ret = tor_tls_shutdown(tls);
-  tt_int_op(ret, OP_EQ, TOR_TLS_DONE);
-  tt_int_op(tls->state, OP_EQ, TOR_TLS_ST_CLOSED);
-
-  fixed_ssl_read_result_index = 0;
-  fixed_ssl_read_result[0] = 10;
-  fixed_ssl_read_result[1] = 42;
-  fixed_ssl_read_result[2] = 0;
-  fixed_ssl_shutdown_result = 0;
-  ERR_clear_error();
-  tls->ssl->version = 0;
-  method->ssl_shutdown = setting_version_ssl_shutdown;
-  ret = tor_tls_shutdown(tls);
-  tt_int_op(ret, OP_EQ, TOR_TLS_ERROR_MISC);
-
-  fixed_ssl_read_result_index = 0;
-  fixed_ssl_read_result[0] = 10;
-  fixed_ssl_read_result[1] = 42;
-  fixed_ssl_read_result[2] = 0;
-  fixed_ssl_shutdown_result = 0;
-  fixed_tls = tls;
-  fixed_ssl_state_to_set = TOR_TLS_ST_GOTCLOSE;
-  ERR_clear_error();
-  tls->ssl->version = 0;
-  method->ssl_shutdown = setting_version_and_state_ssl_shutdown;
-  ret = tor_tls_shutdown(tls);
-  tt_int_op(ret, OP_EQ, TOR_TLS_ERROR_MISC);
-
-  fixed_ssl_read_result_index = 0;
-  fixed_ssl_read_result[0] = 10;
-  fixed_ssl_read_result[1] = 42;
-  fixed_ssl_read_result[2] = 0;
-  fixed_ssl_read_result[3] = -1;
-  fixed_ssl_shutdown_result = 0;
-  fixed_tls = tls;
-  fixed_ssl_state_to_set = 0;
-  ERR_clear_error();
-  tls->ssl->version = 0;
-  method->ssl_shutdown = setting_version_and_state_ssl_shutdown;
-  ret = tor_tls_shutdown(tls);
-  tt_int_op(ret, OP_EQ, TOR_TLS_ERROR_MISC);
-#endif
-
- done:
-  teardown_capture_of_logs();
-  tor_free(method);
-  tor_free(tls->ssl);
-  tor_free(tls);
+  crypto_pk_free(pk1);
+  crypto_pk_free(pk2);
 }
 
-static int negotiated_callback_called;
-
 static void
-negotiated_callback_setter(tor_tls_t *t, void *arg)
+test_tortls_address(void *arg)
 {
-  (void)t;
   (void)arg;
-  negotiated_callback_called++;
-}
-
-static void
-test_tortls_read(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  tor_tls_t *tls;
-  char buf[100];
-  SSL_METHOD *method = give_me_a_test_method();
-  setup_capture_of_logs(LOG_WARN);
-
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  tls->ssl = tor_malloc_zero(sizeof(SSL));
-  tls->state = TOR_TLS_ST_OPEN;
-
-  ret = tor_tls_read(tls, buf, 10);
-  tt_int_op(ret, OP_EQ, -9);
-
-  /* These tests assume that V2_HANDSHAKE_SERVER is set */
-  tls->ssl->handshake_func = dummy_handshake_func;
-  tls->ssl->method = method;
-  method->ssl_read = fixed_ssl_read;
-  fixed_ssl_read_result_index = 0;
-  fixed_ssl_read_result[0] = 42;
-  tls->state = TOR_TLS_ST_OPEN;
-  ERR_clear_error();
-  ret = tor_tls_read(tls, buf, 10);
-  tt_int_op(ret, OP_EQ, 42);
-
-  tls->state = TOR_TLS_ST_OPEN;
-  tls->got_renegotiate = 1;
-  fixed_ssl_read_result_index = 0;
-  ERR_clear_error();
-  ret = tor_tls_read(tls, buf, 10);
-  tt_int_op(tls->got_renegotiate, OP_EQ, 0);
-
-  tls->state = TOR_TLS_ST_OPEN;
-  tls->got_renegotiate = 1;
-  negotiated_callback_called = 0;
-  tls->negotiated_callback = negotiated_callback_setter;
-  fixed_ssl_read_result_index = 0;
-  ERR_clear_error();
-  ret = tor_tls_read(tls, buf, 10);
-  tt_int_op(negotiated_callback_called, OP_EQ, 1);
-
-#ifndef LIBRESSL_VERSION_NUMBER
-  fixed_ssl_read_result_index = 0;
-  fixed_ssl_read_result[0] = 0;
-  tls->ssl->version = SSL2_VERSION;
-  ERR_clear_error();
-  ret = tor_tls_read(tls, buf, 10);
-  tt_int_op(ret, OP_EQ, TOR_TLS_CLOSE);
-  tt_int_op(tls->state, OP_EQ, TOR_TLS_ST_CLOSED);
-#endif
-  // TODO: fill up
-
- done:
-  teardown_capture_of_logs();
-  tor_free(tls->ssl);
-  tor_free(tls);
-  tor_free(method);
-}
-
-static int fixed_ssl_write_result;
-
-static int
-fixed_ssl_write(SSL *s, const void *buf, int len)
-{
-  (void)s;
-  (void)buf;
-  (void)len;
-  return fixed_ssl_write_result;
-}
+  tor_tls_t *tls = NULL;
+  crypto_pk_t *pk1=NULL, *pk2=NULL;
+  pk1 = pk_generate(2);
+  pk2 = pk_generate(0);
 
-static void
-test_tortls_write(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  tor_tls_t *tls;
-  SSL_METHOD *method = give_me_a_test_method();
-  char buf[100];
-  setup_capture_of_logs(LOG_WARN);
+  int r = tor_tls_context_init(TOR_TLS_CTX_IS_PUBLIC_SERVER,
+                               pk1, pk2, 86400);
+  tt_int_op(r, OP_EQ, 0);
 
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  tls->ssl = tor_malloc_zero(sizeof(SSL));
+  tls = tor_tls_new(-1, 0);
   tls->state = TOR_TLS_ST_OPEN;
+  tor_tls_set_logged_address(tls, "zombo.com");
 
-  ret = tor_tls_write(tls, buf, 0);
-  tt_int_op(ret, OP_EQ, 0);
-
-  ret = tor_tls_write(tls, buf, 10);
-  tt_int_op(ret, OP_EQ, -9);
-
-  tls->ssl->method = method;
-  tls->wantwrite_n = 1;
-  ret = tor_tls_write(tls, buf, 10);
-  tt_int_op(tls->wantwrite_n, OP_EQ, 0);
-
-  method->ssl_write = fixed_ssl_write;
-  tls->ssl->handshake_func = dummy_handshake_func;
-  fixed_ssl_write_result = 1;
-  ERR_clear_error();
-  ret = tor_tls_write(tls, buf, 10);
-  tt_int_op(ret, OP_EQ, 1);
-
-  fixed_ssl_write_result = -1;
-  ERR_clear_error();
-  tls->ssl->rwstate = SSL_READING;
-  SSL_set_bio(tls->ssl, BIO_new(BIO_s_mem()), NULL);
-  SSL_get_rbio(tls->ssl)->flags = BIO_FLAGS_READ;
-  ret = tor_tls_write(tls, buf, 10);
-  tt_int_op(ret, OP_EQ, TOR_TLS_WANTREAD);
-
-  ERR_clear_error();
-  tls->ssl->rwstate = SSL_READING;
-  SSL_set_bio(tls->ssl, BIO_new(BIO_s_mem()), NULL);
-  SSL_get_rbio(tls->ssl)->flags = BIO_FLAGS_WRITE;
-  ret = tor_tls_write(tls, buf, 10);
-  tt_int_op(ret, OP_EQ, TOR_TLS_WANTWRITE);
-
- done:
-  teardown_capture_of_logs();
-  BIO_free(tls->ssl->rbio);
-  tor_free(tls->ssl);
-  tor_free(tls);
-  tor_free(method);
-}
-#endif
-
-#ifndef OPENSSL_OPAQUE
-static int fixed_ssl_accept_result;
-static int fixed_ssl_connect_result;
-
-static int
-setting_error_ssl_accept(SSL *ssl)
-{
-  (void)ssl;
-  ERR_put_error(ERR_LIB_BN, 2, -1, "somewhere.c", 99);
-  ERR_put_error(ERR_LIB_SYS, 2, -1, "somewhere.c", 99);
-  return fixed_ssl_accept_result;
-}
-
-static int
-setting_error_ssl_connect(SSL *ssl)
-{
-  (void)ssl;
-  ERR_put_error(ERR_LIB_BN, 2, -1, "somewhere.c", 99);
-  ERR_put_error(ERR_LIB_SYS, 2, -1, "somewhere.c", 99);
-  return fixed_ssl_connect_result;
-}
-
-static int
-fixed_ssl_accept(SSL *ssl)
-{
-  (void) ssl;
-  return fixed_ssl_accept_result;
-}
-
-static void
-test_tortls_handshake(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  tor_tls_t *tls;
-  SSL_CTX *ctx;
-  SSL_METHOD *method = give_me_a_test_method();
+  /* This write should fail, since the fd is -1. */
   setup_capture_of_logs(LOG_INFO);
-
-  SSL_library_init();
-  SSL_load_error_strings();
-
-  ctx = SSL_CTX_new(TLSv1_method());
-
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  tls->ssl = SSL_new(ctx);
-  tls->state = TOR_TLS_ST_HANDSHAKE;
-
-  ret = tor_tls_handshake(tls);
-  tt_int_op(ret, OP_EQ, -9);
-
-  tls->isServer = 1;
-  tls->state = TOR_TLS_ST_HANDSHAKE;
-  ret = tor_tls_handshake(tls);
-  tt_int_op(ret, OP_EQ, -9);
-
-  tls->ssl->method = method;
-  method->ssl_accept = fixed_ssl_accept;
-  fixed_ssl_accept_result = 2;
-  ERR_clear_error();
-  tls->state = TOR_TLS_ST_HANDSHAKE;
-  ret = tor_tls_handshake(tls);
-  tt_int_op(tls->state, OP_EQ, TOR_TLS_ST_OPEN);
-
-  method->ssl_accept = setting_error_ssl_accept;
-  fixed_ssl_accept_result = 1;
-  ERR_clear_error();
-  mock_clean_saved_logs();
-  tls->state = TOR_TLS_ST_HANDSHAKE;
-  ret = tor_tls_handshake(tls);
-  tt_int_op(ret, OP_EQ, TOR_TLS_ERROR_MISC);
-  expect_log_entry();
-  /* This fails on jessie.  Investigate why! */
-#if 0
-  expect_log_msg("TLS error while handshaking: (null) (in bignum routines:"
-            "(null):SSLv3 write client hello B)\n");
-  expect_log_msg("TLS error while handshaking: (null) (in system library:"
-            "connect:SSLv3 write client hello B)\n");
-#endif
-  expect_log_severity(LOG_INFO);
-
-  tls->isServer = 0;
-  method->ssl_connect = setting_error_ssl_connect;
-  fixed_ssl_connect_result = 1;
-  ERR_clear_error();
-  mock_clean_saved_logs();
-  tls->state = TOR_TLS_ST_HANDSHAKE;
-  ret = tor_tls_handshake(tls);
-  tt_int_op(ret, OP_EQ, TOR_TLS_ERROR_MISC);
-  expect_log_entry();
-#if 0
-  /* See above */
-  expect_log_msg("TLS error while handshaking: "
-            "(null) (in bignum routines:(null):SSLv3 write client hello B)\n");
-  expect_log_msg("TLS error while handshaking: "
-            "(null) (in system library:connect:SSLv3 write client hello B)\n");
-#endif
-  expect_log_severity(LOG_WARN);
-
- done:
-  teardown_capture_of_logs();
-  SSL_free(tls->ssl);
-  SSL_CTX_free(ctx);
-  tor_free(tls);
-  tor_free(method);
-}
-#endif
-
-#ifndef OPENSSL_OPAQUE
-static void
-test_tortls_finish_handshake(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  tor_tls_t *tls;
-  SSL_CTX *ctx;
-  SSL_METHOD *method = give_me_a_test_method();
-  SSL_library_init();
-  SSL_load_error_strings();
-
-  X509 *c1 = read_cert_from(validCertString);
-  SESS_CERT_local *sess = NULL;
-
-  ctx = SSL_CTX_new(method);
-
-  tls = tor_malloc_zero(sizeof(tor_tls_t));
-  tls->ssl = SSL_new(ctx);
-  tls->state = TOR_TLS_ST_OPEN;
-
-  ret = tor_tls_finish_handshake(tls);
-  tt_int_op(ret, OP_EQ, 0);
-
-  tls->isServer = 1;
-  tls->wasV2Handshake = 0;
-  setup_full_capture_of_logs(LOG_WARN);
-  ret = tor_tls_finish_handshake(tls);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(tls->wasV2Handshake, OP_EQ, 1);
-  expect_single_log_msg_containing("For some reason, wasV2Handshake didn't "
-                                   "get set.");
-  teardown_capture_of_logs();
-
-  tls->wasV2Handshake = 1;
-  ret = tor_tls_finish_handshake(tls);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(tls->wasV2Handshake, OP_EQ, 1);
-
-  tls->wasV2Handshake = 1;
-  tls->ssl->session = SSL_SESSION_new();
-  ret = tor_tls_finish_handshake(tls);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(tls->wasV2Handshake, OP_EQ, 0);
-
-  tls->isServer = 0;
-
-  sess = tor_malloc_zero(sizeof(SESS_CERT_local));
-  tls->ssl->session->sess_cert = (void *)sess;
-  sess->cert_chain = sk_X509_new_null();
-  sk_X509_push(sess->cert_chain, c1);
-  tls->ssl->session->peer = c1;
-  tls->wasV2Handshake = 0;
-  ret = tor_tls_finish_handshake(tls);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(tls->wasV2Handshake, OP_EQ, 1);
-
-  method->num_ciphers = fake_num_ciphers;
-  ret = tor_tls_finish_handshake(tls);
-  tt_int_op(ret, OP_EQ, -9);
+  int n = tor_tls_write(tls, "welcome", 7);
+  tt_int_op(n, OP_LT, 0);
+  expect_log_msg_containing("with zombo.com");
 
  done:
-  if (sess)
-    sk_X509_free(sess->cert_chain);
-  if (tls->ssl && tls->ssl->session) {
-    tor_free(tls->ssl->session->sess_cert);
-  }
-  SSL_free(tls->ssl);
-  tor_free(tls);
-  SSL_CTX_free(ctx);
-  tor_free(method);
   teardown_capture_of_logs();
-}
-#endif
-
-static int fixed_crypto_pk_new_result_index;
-static crypto_pk_t *fixed_crypto_pk_new_result[5];
-
-static crypto_pk_t *
-fixed_crypto_pk_new(void)
-{
-  return fixed_crypto_pk_new_result[fixed_crypto_pk_new_result_index++];
-}
-
-#ifndef OPENSSL_OPAQUE
-static int fixed_crypto_pk_generate_key_with_bits_result_index;
-static int fixed_crypto_pk_generate_key_with_bits_result[5];
-static int fixed_tor_tls_create_certificate_result_index;
-static X509 *fixed_tor_tls_create_certificate_result[5];
-static int fixed_tor_x509_cert_new_result_index;
-static tor_x509_cert_t *fixed_tor_x509_cert_new_result[5];
-
-static int
-fixed_crypto_pk_generate_key_with_bits(crypto_pk_t *env, int bits)
-{
-  (void)env;
-  (void)bits;
-  return fixed_crypto_pk_generate_key_with_bits_result[
-                    fixed_crypto_pk_generate_key_with_bits_result_index++];
-}
-
-static X509 *
-fixed_tor_tls_create_certificate(crypto_pk_t *rsa,
-                                 crypto_pk_t *rsa_sign,
-                                 const char *cname,
-                                 const char *cname_sign,
-                                 unsigned int cert_lifetime)
-{
-  (void)rsa;
-  (void)rsa_sign;
-  (void)cname;
-  (void)cname_sign;
-  (void)cert_lifetime;
-  return fixed_tor_tls_create_certificate_result[
-                             fixed_tor_tls_create_certificate_result_index++];
-}
-
-static tor_x509_cert_t *
-fixed_tor_x509_cert_new(X509 *x509_cert)
-{
-  (void) x509_cert;
-  return fixed_tor_x509_cert_new_result[
-                                      fixed_tor_x509_cert_new_result_index++];
+  tor_tls_free(tls);
+  crypto_pk_free(pk1);
+  crypto_pk_free(pk2);
 }
 
 static void
-test_tortls_context_new(void *ignored)
+test_tortls_is_server(void *arg)
 {
-  (void)ignored;
-  tor_tls_context_t *ret;
-  crypto_pk_t *pk1, *pk2, *pk3, *pk4, *pk5, *pk6, *pk7, *pk8, *pk9, *pk10,
-    *pk11, *pk12, *pk13, *pk14, *pk15, *pk16, *pk17, *pk18;
-
-  pk1 = crypto_pk_new();
-  pk2 = crypto_pk_new();
-  pk3 = crypto_pk_new();
-  pk4 = crypto_pk_new();
-  pk5 = crypto_pk_new();
-  pk6 = crypto_pk_new();
-  pk7 = crypto_pk_new();
-  pk8 = crypto_pk_new();
-  pk9 = crypto_pk_new();
-  pk10 = crypto_pk_new();
-  pk11 = crypto_pk_new();
-  pk12 = crypto_pk_new();
-  pk13 = crypto_pk_new();
-  pk14 = crypto_pk_new();
-  pk15 = crypto_pk_new();
-  pk16 = crypto_pk_new();
-  pk17 = crypto_pk_new();
-  pk18 = crypto_pk_new();
-
-  fixed_crypto_pk_new_result_index = 0;
-  fixed_crypto_pk_new_result[0] = NULL;
-  MOCK(crypto_pk_new, fixed_crypto_pk_new);
-  ret = tor_tls_context_new(NULL, 0, 0, 0);
-  tt_assert(!ret);
-
-  /* note: we already override this in testing_common.c, so we
-   * run this unit test in a subprocess. */
-  MOCK(crypto_pk_generate_key_with_bits,
-       fixed_crypto_pk_generate_key_with_bits);
-  fixed_crypto_pk_new_result_index = 0;
-  fixed_crypto_pk_new_result[0] = pk1;
-  fixed_crypto_pk_new_result[1] = NULL;
-  fixed_crypto_pk_generate_key_with_bits_result[0] = -1;
-  fixed_crypto_pk_generate_key_with_bits_result_index = 0;
-  ret = tor_tls_context_new(NULL, 0, 0, 0);
-  tt_assert(!ret);
-
-  fixed_crypto_pk_new_result_index = 0;
-  fixed_crypto_pk_new_result[0] = pk2;
-  fixed_crypto_pk_new_result[1] = NULL;
-  fixed_crypto_pk_generate_key_with_bits_result[0] = 0;
-  fixed_crypto_pk_generate_key_with_bits_result_index = 0;
-  ret = tor_tls_context_new(NULL, 0, 0, 0);
-  tt_assert(!ret);
-
-  fixed_crypto_pk_new_result_index = 0;
-  fixed_crypto_pk_new_result[0] = pk3;
-  fixed_crypto_pk_new_result[1] = pk4;
-  fixed_crypto_pk_new_result[2] = NULL;
-  fixed_crypto_pk_generate_key_with_bits_result[0] = 0;
-  fixed_crypto_pk_generate_key_with_bits_result[1] = -1;
-  fixed_crypto_pk_generate_key_with_bits_result_index = 0;
-  ret = tor_tls_context_new(NULL, 0, 0, 0);
-  tt_assert(!ret);
-
-  MOCK(tor_tls_create_certificate, fixed_tor_tls_create_certificate);
-
-  fixed_crypto_pk_new_result_index = 0;
-  fixed_crypto_pk_new_result[0] = pk5;
-  fixed_crypto_pk_new_result[1] = pk6;
-  fixed_crypto_pk_new_result[2] = NULL;
-  fixed_crypto_pk_generate_key_with_bits_result_index = 0;
-  fixed_crypto_pk_generate_key_with_bits_result[1] = 0;
-  fixed_tor_tls_create_certificate_result_index = 0;
-  fixed_tor_tls_create_certificate_result[0] = NULL;
-  fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509));
-  ret = tor_tls_context_new(NULL, 0, 0, 0);
-  tt_assert(!ret);
-
-  fixed_crypto_pk_new_result_index = 0;
-  fixed_crypto_pk_new_result[0] = pk7;
-  fixed_crypto_pk_new_result[1] = pk8;
-  fixed_crypto_pk_new_result[2] = NULL;
-  fixed_crypto_pk_generate_key_with_bits_result_index = 0;
-  fixed_tor_tls_create_certificate_result_index = 0;
-  fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[1] = NULL;
-  fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509));
-  ret = tor_tls_context_new(NULL, 0, 0, 0);
-  tt_assert(!ret);
-
-  fixed_crypto_pk_new_result_index = 0;
-  fixed_crypto_pk_new_result[0] = pk9;
-  fixed_crypto_pk_new_result[1] = pk10;
-  fixed_crypto_pk_new_result[2] = NULL;
-  fixed_crypto_pk_generate_key_with_bits_result_index = 0;
-  fixed_tor_tls_create_certificate_result_index = 0;
-  fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[2] = NULL;
-  ret = tor_tls_context_new(NULL, 0, 0, 0);
-  tt_assert(!ret);
-
-  MOCK(tor_x509_cert_new, fixed_tor_x509_cert_new);
-  fixed_crypto_pk_new_result_index = 0;
-  fixed_crypto_pk_new_result[0] = pk11;
-  fixed_crypto_pk_new_result[1] = pk12;
-  fixed_crypto_pk_new_result[2] = NULL;
-  fixed_crypto_pk_generate_key_with_bits_result_index = 0;
-  fixed_tor_tls_create_certificate_result_index = 0;
-  fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_x509_cert_new_result_index = 0;
-  fixed_tor_x509_cert_new_result[0] = NULL;
-  fixed_tor_x509_cert_new_result[1] = NULL;
-  fixed_tor_x509_cert_new_result[2] = NULL;
-  ret = tor_tls_context_new(NULL, 0, 0, 0);
-  tt_assert(!ret);
-
-  fixed_crypto_pk_new_result_index = 0;
-  fixed_crypto_pk_new_result[0] = pk13;
-  fixed_crypto_pk_new_result[1] = pk14;
-  fixed_crypto_pk_new_result[2] = NULL;
-  fixed_crypto_pk_generate_key_with_bits_result_index = 0;
-  fixed_tor_tls_create_certificate_result_index = 0;
-  fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_x509_cert_new_result_index = 0;
-  fixed_tor_x509_cert_new_result[0] = tor_malloc_zero(sizeof(tor_x509_cert_t));
-  fixed_tor_x509_cert_new_result[1] = NULL;
-  fixed_tor_x509_cert_new_result[2] = NULL;
-  ret = tor_tls_context_new(NULL, 0, 0, 0);
-  tt_assert(!ret);
-
-  fixed_crypto_pk_new_result_index = 0;
-  fixed_crypto_pk_new_result[0] = pk15;
-  fixed_crypto_pk_new_result[1] = pk16;
-  fixed_crypto_pk_new_result[2] = NULL;
-  fixed_crypto_pk_generate_key_with_bits_result_index = 0;
-  fixed_tor_tls_create_certificate_result_index = 0;
-  fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_x509_cert_new_result_index = 0;
-  fixed_tor_x509_cert_new_result[0] = tor_malloc_zero(sizeof(tor_x509_cert_t));
-  fixed_tor_x509_cert_new_result[1] = tor_malloc_zero(sizeof(tor_x509_cert_t));
-  fixed_tor_x509_cert_new_result[2] = NULL;
-  ret = tor_tls_context_new(NULL, 0, 0, 0);
-  tt_assert(!ret);
-
-  fixed_crypto_pk_new_result_index = 0;
-  fixed_crypto_pk_new_result[0] = pk17;
-  fixed_crypto_pk_new_result[1] = pk18;
-  fixed_crypto_pk_new_result[2] = NULL;
-  fixed_crypto_pk_generate_key_with_bits_result_index = 0;
-  fixed_tor_tls_create_certificate_result_index = 0;
-  fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_x509_cert_new_result_index = 0;
-  fixed_tor_x509_cert_new_result[0] = tor_malloc_zero(sizeof(tor_x509_cert_t));
-  fixed_tor_x509_cert_new_result[1] = tor_malloc_zero(sizeof(tor_x509_cert_t));
-  fixed_tor_x509_cert_new_result[2] = tor_malloc_zero(sizeof(tor_x509_cert_t));
-  ret = tor_tls_context_new(NULL, 0, 0, 0);
-  tt_assert(!ret);
-
- done:
-  UNMOCK(tor_x509_cert_new);
-  UNMOCK(tor_tls_create_certificate);
-  UNMOCK(crypto_pk_generate_key_with_bits);
-  UNMOCK(crypto_pk_new);
-}
-#endif
-
-static int fixed_crypto_pk_get_evp_pkey_result_index = 0;
-static EVP_PKEY *fixed_crypto_pk_get_evp_pkey_result[5];
+  (void)arg;
+  crypto_pk_t *pk1=NULL, *pk2=NULL;
+  tor_tls_t *tls1=NULL, *tls2=NULL;
+  pk1 = pk_generate(2);
+  pk2 = pk_generate(0);
 
-static EVP_PKEY *
-fixed_crypto_pk_get_evp_pkey_(crypto_pk_t *env, int private)
-{
-  (void) env;
-  (void) private;
-  return fixed_crypto_pk_get_evp_pkey_result[
-                               fixed_crypto_pk_get_evp_pkey_result_index++];
-}
+  int r = tor_tls_context_init(TOR_TLS_CTX_IS_PUBLIC_SERVER,
+                               pk1, pk2, 86400);
+  tt_int_op(r, OP_EQ, 0);
+  tls1 = tor_tls_new(-1, 0);
+  tls2 = tor_tls_new(-1, 1);
 
-static void
-test_tortls_create_certificate(void *ignored)
-{
-  (void)ignored;
-  X509 *ret;
-  crypto_pk_t *pk1, *pk2;
-
-  pk1 = crypto_pk_new();
-  pk2 = crypto_pk_new();
-
-  MOCK(crypto_pk_get_evp_pkey_, fixed_crypto_pk_get_evp_pkey_);
-  fixed_crypto_pk_get_evp_pkey_result_index = 0;
-  fixed_crypto_pk_get_evp_pkey_result[0] = NULL;
-  ret = tor_tls_create_certificate(pk1, pk2, "hello", "hello2", 1);
-  tt_assert(!ret);
-
-  fixed_crypto_pk_get_evp_pkey_result_index = 0;
-  fixed_crypto_pk_get_evp_pkey_result[0] = EVP_PKEY_new();
-  fixed_crypto_pk_get_evp_pkey_result[1] = NULL;
-  ret = tor_tls_create_certificate(pk1, pk2, "hello", "hello2", 1);
-  tt_assert(!ret);
-
-  fixed_crypto_pk_get_evp_pkey_result_index = 0;
-  fixed_crypto_pk_get_evp_pkey_result[0] = EVP_PKEY_new();
-  fixed_crypto_pk_get_evp_pkey_result[1] = EVP_PKEY_new();
-  ret = tor_tls_create_certificate(pk1, pk2, "hello", "hello2", 1);
-  tt_assert(!ret);
+  tt_assert(! tor_tls_is_server(tls1));
+  tt_assert(tor_tls_is_server(tls2));
 
  done:
-  UNMOCK(crypto_pk_get_evp_pkey_);
+  tor_tls_free(tls1);
+  tor_tls_free(tls2);
   crypto_pk_free(pk1);
   crypto_pk_free(pk2);
 }
 
 static void
-test_tortls_cert_new(void *ignored)
-{
-  (void)ignored;
-  tor_x509_cert_t *ret;
-  X509 *cert = read_cert_from(validCertString);
-
-  ret = tor_x509_cert_new(NULL);
-  tt_assert(!ret);
-
-  ret = tor_x509_cert_new(cert);
-  tt_assert(ret);
-  tor_x509_cert_free(ret);
-  ret = NULL;
-
-#if 0
-  cert = read_cert_from(validCertString);
-  /* XXX this doesn't do what you think: it alters a copy of the pubkey. */
-  X509_get_pubkey(cert)->type = EVP_PKEY_DSA;
-  ret = tor_x509_cert_new(cert);
-  tt_assert(ret);
-#endif
-
-#ifndef OPENSSL_OPAQUE
-  cert = read_cert_from(validCertString);
-  X509_CINF_free(cert->cert_info);
-  cert->cert_info = NULL;
-  ret = tor_x509_cert_new(cert);
-  tt_assert(ret);
-#endif
-
- done:
-  tor_x509_cert_free(ret);
-}
-
-static void
-test_tortls_cert_is_valid(void *ignored)
+test_tortls_verify(void *ignored)
 {
   (void)ignored;
   int ret;
-  tor_x509_cert_t *cert = NULL, *scert = NULL;
+  tor_tls_t *tls;
+  crypto_pk_t *k = NULL;
+  tor_x509_cert_impl_t *cert1 = NULL, *cert2 = NULL, *invalidCert = NULL,
+    *validCert = NULL, *caCert = NULL;
 
-  scert = tor_malloc_zero(sizeof(tor_x509_cert_t));
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 0);
-  tt_int_op(ret, OP_EQ, 0);
+  validCert = read_cert_from(validCertString);
+  caCert = read_cert_from(caCertString);
+  invalidCert = read_cert_from(notCompletelyValidCertString);
 
-  cert = tor_malloc_zero(sizeof(tor_x509_cert_t));
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 0);
-  tt_int_op(ret, OP_EQ, 0);
-  tor_free(scert);
-  tor_free(cert);
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
 
-  cert = tor_x509_cert_new(read_cert_from(validCertString));
-  scert = tor_x509_cert_new(read_cert_from(caCertString));
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 0);
-  tt_int_op(ret, OP_EQ, 1);
+  MOCK(try_to_extract_certs_from_tls, fixed_try_to_extract_certs_from_tls);
 
-#ifndef OPENSSL_OPAQUE
-  tor_x509_cert_free(cert);
-  tor_x509_cert_free(scert);
-  cert = tor_x509_cert_new(read_cert_from(validCertString));
-  scert = tor_x509_cert_new(read_cert_from(caCertString));
-  ASN1_TIME_free(cert->cert->cert_info->validity->notAfter);
-  cert->cert->cert_info->validity->notAfter =
-    ASN1_TIME_set(NULL, time(NULL)-1000000);
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 0);
-  tt_int_op(ret, OP_EQ, 0);
+  fixed_try_to_extract_certs_from_tls_cert_out_result = cert1;
+  ret = tor_tls_verify(LOG_WARN, tls, &k);
+  tt_int_op(ret, OP_EQ, -1);
 
-  tor_x509_cert_free(cert);
-  tor_x509_cert_free(scert);
-  cert = tor_x509_cert_new(read_cert_from(validCertString));
-  scert = tor_x509_cert_new(read_cert_from(caCertString));
-  X509_PUBKEY_free(cert->cert->cert_info->key);
-  cert->cert->cert_info->key = NULL;
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 1);
-  tt_int_op(ret, OP_EQ, 0);
-#endif
+  fixed_try_to_extract_certs_from_tls_id_cert_out_result = cert2;
+  ret = tor_tls_verify(LOG_WARN, tls, &k);
+  tt_int_op(ret, OP_EQ, -1);
 
-#if 0
-  tor_x509_cert_free(cert);
-  tor_x509_cert_free(scert);
-  cert = tor_x509_cert_new(read_cert_from(validCertString));
-  scert = tor_x509_cert_new(read_cert_from(caCertString));
-  /* This doesn't actually change the key in the cert. XXXXXX */
-  BN_one(EVP_PKEY_get1_RSA(X509_get_pubkey(cert->cert))->n);
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 1);
-  tt_int_op(ret, OP_EQ, 0);
+  fixed_try_to_extract_certs_from_tls_cert_out_result = invalidCert;
+  fixed_try_to_extract_certs_from_tls_id_cert_out_result = invalidCert;
 
-  tor_x509_cert_free(cert);
-  tor_x509_cert_free(scert);
-  cert = tor_x509_cert_new(read_cert_from(validCertString));
-  scert = tor_x509_cert_new(read_cert_from(caCertString));
-  /* This doesn't actually change the key in the cert. XXXXXX */
-  X509_get_pubkey(cert->cert)->type = EVP_PKEY_EC;
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 1);
-  tt_int_op(ret, OP_EQ, 0);
+  ret = tor_tls_verify(LOG_WARN, tls, &k);
+  tt_int_op(ret, OP_EQ, -1);
 
-  tor_x509_cert_free(cert);
-  tor_x509_cert_free(scert);
-  cert = tor_x509_cert_new(read_cert_from(validCertString));
-  scert = tor_x509_cert_new(read_cert_from(caCertString));
-  /* This doesn't actually change the key in the cert. XXXXXX */
-  X509_get_pubkey(cert->cert)->type = EVP_PKEY_EC;
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 0);
-  tt_int_op(ret, OP_EQ, 1);
+  fixed_try_to_extract_certs_from_tls_cert_out_result = validCert;
+  fixed_try_to_extract_certs_from_tls_id_cert_out_result = caCert;
 
-  tor_x509_cert_free(cert);
-  tor_x509_cert_free(scert);
-  cert = tor_x509_cert_new(read_cert_from(validCertString));
-  scert = tor_x509_cert_new(read_cert_from(caCertString));
-  /* This doesn't actually change the key in the cert. XXXXXX */
-  X509_get_pubkey(cert->cert)->type = EVP_PKEY_EC;
-  X509_get_pubkey(cert->cert)->ameth = NULL;
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 0);
+  ret = tor_tls_verify(LOG_WARN, tls, &k);
   tt_int_op(ret, OP_EQ, 0);
-#endif
+  tt_assert(k);
 
  done:
-  tor_x509_cert_free(cert);
-  tor_x509_cert_free(scert);
-}
-
-static void
-test_tortls_context_init_one(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  tor_tls_context_t *old = NULL;
-
-  MOCK(crypto_pk_new, fixed_crypto_pk_new);
-
-  fixed_crypto_pk_new_result_index = 0;
-  fixed_crypto_pk_new_result[0] = NULL;
-  ret = tor_tls_context_init_one(&old, NULL, 0, 0, 0);
-  tt_int_op(ret, OP_EQ, -1);
+  UNMOCK(try_to_extract_certs_from_tls);
+  tor_x509_cert_impl_free(cert1);
+  tor_x509_cert_impl_free(cert2);
+  tor_x509_cert_impl_free(validCert);
+  tor_x509_cert_impl_free(invalidCert);
+  tor_x509_cert_impl_free(caCert);
 
- done:
-  UNMOCK(crypto_pk_new);
+  tor_free(tls);
+  crypto_pk_free(k);
 }
 
-#define LOCAL_TEST_CASE(name, flags)                    \
+#define LOCAL_TEST_CASE(name, flags)                            \
   { #name, test_tortls_##name, (flags|TT_FORK), NULL, NULL }
 
-#ifdef OPENSSL_OPAQUE
-#define INTRUSIVE_TEST_CASE(name, flags)        \
-  { #name, NULL, TT_SKIP, NULL, NULL }
-#else
-#define INTRUSIVE_TEST_CASE(name, flags) LOCAL_TEST_CASE(name, flags)
-#endif
-
 struct testcase_t tortls_tests[] = {
   LOCAL_TEST_CASE(errno_to_tls_error, 0),
   LOCAL_TEST_CASE(err_to_string, 0),
-  LOCAL_TEST_CASE(tor_tls_new, TT_FORK),
-  LOCAL_TEST_CASE(tor_tls_get_error, 0),
-  LOCAL_TEST_CASE(get_state_description, TT_FORK),
-  LOCAL_TEST_CASE(get_by_ssl, TT_FORK),
-  LOCAL_TEST_CASE(allocate_tor_tls_object_ex_data_index, TT_FORK),
-  LOCAL_TEST_CASE(log_one_error, TT_FORK),
-  INTRUSIVE_TEST_CASE(get_error, TT_FORK),
-  LOCAL_TEST_CASE(always_accept_verify_cb, 0),
-  INTRUSIVE_TEST_CASE(x509_cert_free, 0),
   LOCAL_TEST_CASE(x509_cert_get_id_digests, 0),
-  LOCAL_TEST_CASE(cert_matches_key, 0),
-  INTRUSIVE_TEST_CASE(cert_get_key, 0),
-  LOCAL_TEST_CASE(get_my_client_auth_key, TT_FORK),
   LOCAL_TEST_CASE(get_my_certs, TT_FORK),
-  INTRUSIVE_TEST_CASE(get_ciphersuite_name, 0),
-  INTRUSIVE_TEST_CASE(classify_client_ciphers, 0),
-  LOCAL_TEST_CASE(client_is_using_v2_ciphers, 0),
-  INTRUSIVE_TEST_CASE(verify, 0),
-  INTRUSIVE_TEST_CASE(check_lifetime, 0),
-  INTRUSIVE_TEST_CASE(get_pending_bytes, 0),
+#ifdef ENABLE_OPENSSL
+  LOCAL_TEST_CASE(tor_tls_get_error, 0),
   LOCAL_TEST_CASE(get_forced_write_size, 0),
-  LOCAL_TEST_CASE(get_write_overhead_ratio, TT_FORK),
   LOCAL_TEST_CASE(used_v1_handshake, TT_FORK),
-  LOCAL_TEST_CASE(get_num_server_handshakes, 0),
   LOCAL_TEST_CASE(server_got_renegotiate, 0),
-  INTRUSIVE_TEST_CASE(SSL_SESSION_get_master_key, 0),
-  INTRUSIVE_TEST_CASE(get_tlssecrets, 0),
-  INTRUSIVE_TEST_CASE(get_buffer_sizes, 0),
+#endif
   LOCAL_TEST_CASE(evaluate_ecgroup_for_tls, 0),
-  INTRUSIVE_TEST_CASE(try_to_extract_certs_from_tls, 0),
-  INTRUSIVE_TEST_CASE(get_peer_cert, 0),
-  INTRUSIVE_TEST_CASE(peer_has_cert, 0),
-  INTRUSIVE_TEST_CASE(shutdown, 0),
-  INTRUSIVE_TEST_CASE(finish_handshake, 0),
-  INTRUSIVE_TEST_CASE(handshake, 0),
-  INTRUSIVE_TEST_CASE(write, 0),
-  INTRUSIVE_TEST_CASE(read, 0),
-  INTRUSIVE_TEST_CASE(server_info_callback, 0),
+  LOCAL_TEST_CASE(double_init, TT_FORK),
+  LOCAL_TEST_CASE(address, TT_FORK),
   LOCAL_TEST_CASE(is_server, 0),
-  INTRUSIVE_TEST_CASE(assert_renegotiation_unblocked, 0),
-  INTRUSIVE_TEST_CASE(block_renegotiation, 0),
-  INTRUSIVE_TEST_CASE(unblock_renegotiation, 0),
-  INTRUSIVE_TEST_CASE(set_renegotiate_callback, 0),
-  LOCAL_TEST_CASE(set_logged_address, 0),
-  INTRUSIVE_TEST_CASE(find_cipher_by_id, 0),
-  INTRUSIVE_TEST_CASE(session_secret_cb, 0),
-  INTRUSIVE_TEST_CASE(debug_state_callback, 0),
-  INTRUSIVE_TEST_CASE(context_new, TT_FORK /* redundant */),
-  LOCAL_TEST_CASE(create_certificate, 0),
-  LOCAL_TEST_CASE(cert_new, 0),
-  LOCAL_TEST_CASE(cert_is_valid, 0),
-  LOCAL_TEST_CASE(context_init_one, 0),
+  LOCAL_TEST_CASE(bridge_init, TT_FORK),
+  LOCAL_TEST_CASE(verify, TT_FORK),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_tortls.h b/src/test/test_tortls.h
new file mode 100644
index 0000000000..1a8b117d0f
--- /dev/null
+++ b/src/test/test_tortls.h
@@ -0,0 +1,13 @@
+/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TEST_TORTLS_H
+#define TEST_TORTLS_H
+
+tor_x509_cert_impl_t *read_cert_from(const char *str);
+
+extern const char *notCompletelyValidCertString;
+extern const char *validCertString;
+extern const char *caCertString;
+
+#endif
diff --git a/src/test/test_tortls_openssl.c b/src/test/test_tortls_openssl.c
new file mode 100644
index 0000000000..73041a871c
--- /dev/null
+++ b/src/test/test_tortls_openssl.c
@@ -0,0 +1,2316 @@
+/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define TORTLS_PRIVATE
+#define TORTLS_OPENSSL_PRIVATE
+#define TOR_X509_PRIVATE
+#define LOG_PRIVATE
+#include "orconfig.h"
+
+#ifdef _WIN32
+#include <winsock2.h>
+#endif
+#include <math.h>
+
+#include "lib/cc/compat_compiler.h"
+
+/* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in
+ * srtp.h. Suppress the GCC warning so we can build with -Wredundant-decl. */
+DISABLE_GCC_WARNING(redundant-decls)
+
+#include <openssl/opensslv.h>
+
+#include <openssl/ssl.h>
+#include <openssl/ssl3.h>
+#include <openssl/err.h>
+#include <openssl/asn1t.h>
+#include <openssl/x509.h>
+#include <openssl/rsa.h>
+#include <openssl/evp.h>
+#include <openssl/bn.h>
+
+ENABLE_GCC_WARNING(redundant-decls)
+
+#include "core/or/or.h"
+#include "lib/log/log.h"
+#include "app/config/config.h"
+#include "lib/crypt_ops/compat_openssl.h"
+#include "lib/tls/x509.h"
+#include "lib/tls/x509_internal.h"
+#include "lib/tls/tortls.h"
+#include "lib/tls/tortls_st.h"
+#include "lib/tls/tortls_internal.h"
+#include "app/config/or_state_st.h"
+
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+#include "test/test_tortls.h"
+
+#define NS_MODULE tortls
+
+#ifndef HAVE_SSL_STATE
+#define OPENSSL_OPAQUE
+#endif
+
+#if defined(OPENSSL_OPAQUE) && !defined(LIBRESSL_VERSION_NUMBER)
+#define SSL_STATE_STR "before SSL initialization"
+#else
+#define SSL_STATE_STR "before/accept initialization"
+#endif
+
+#ifndef OPENSSL_OPAQUE
+static SSL_METHOD *
+give_me_a_test_method(void)
+{
+  SSL_METHOD *method = tor_malloc_zero(sizeof(SSL_METHOD));
+  memcpy(method, TLSv1_method(), sizeof(SSL_METHOD));
+  return method;
+}
+
+static int
+fake_num_ciphers(void)
+{
+  return 0;
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+static int
+mock_tls_cert_matches_key(const tor_tls_t *tls, const tor_x509_cert_t *cert)
+{
+  (void) tls;
+  (void) cert; // XXXX look at this.
+  return 1;
+}
+
+static void
+test_tortls_tor_tls_new(void *data)
+{
+  (void) data;
+  MOCK(tor_tls_cert_matches_key, mock_tls_cert_matches_key);
+  crypto_pk_t *key1 = NULL, *key2 = NULL;
+  SSL_METHOD *method = NULL;
+
+  key1 = pk_generate(2);
+  key2 = pk_generate(3);
+
+  tor_tls_t *tls = NULL;
+  tt_int_op(tor_tls_context_init(TOR_TLS_CTX_IS_PUBLIC_SERVER,
+                                 key1, key2, 86400), OP_EQ, 0);
+  tls = tor_tls_new(-1, 0);
+  tt_want(tls);
+  tor_tls_free(tls); tls = NULL;
+
+  SSL_CTX_free(client_tls_context->ctx);
+  client_tls_context->ctx = NULL;
+  tls = tor_tls_new(-1, 0);
+  tt_ptr_op(tls, OP_EQ, NULL);
+
+#ifndef OPENSSL_OPAQUE
+  method = give_me_a_test_method();
+  SSL_CTX *ctx = SSL_CTX_new(method);
+  method->num_ciphers = fake_num_ciphers;
+  client_tls_context->ctx = ctx;
+  tls = tor_tls_new(-1, 0);
+  tt_ptr_op(tls, OP_EQ, NULL);
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+ done:
+  UNMOCK(tor_tls_cert_matches_key);
+  crypto_pk_free(key1);
+  crypto_pk_free(key2);
+  tor_tls_free(tls);
+  tor_free(method);
+  tor_tls_free_all();
+}
+
+#define NS_MODULE tortls
+
+static void
+library_init(void)
+{
+#ifdef OPENSSL_1_1_API
+  OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS, NULL);
+#else
+  SSL_library_init();
+  SSL_load_error_strings();
+#endif
+}
+
+static void
+test_tortls_get_state_description(void *ignored)
+{
+  (void)ignored;
+  tor_tls_t *tls;
+  char *buf;
+  SSL_CTX *ctx;
+
+  library_init();
+  ctx = SSL_CTX_new(SSLv23_method());
+
+  buf = tor_malloc_zero(1000);
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+
+  tor_tls_get_state_description(NULL, buf, 20);
+  tt_str_op(buf, OP_EQ, "(No SSL object)");
+
+  SSL_free(tls->ssl);
+  tls->ssl = NULL;
+  tor_tls_get_state_description(tls, buf, 20);
+  tt_str_op(buf, OP_EQ, "(No SSL object)");
+
+  tls->ssl = SSL_new(ctx);
+  tor_tls_get_state_description(tls, buf, 200);
+  tt_str_op(buf, OP_EQ, SSL_STATE_STR " in HANDSHAKE");
+
+  tls->state = TOR_TLS_ST_OPEN;
+  tor_tls_get_state_description(tls, buf, 200);
+  tt_str_op(buf, OP_EQ, SSL_STATE_STR " in OPEN");
+
+  tls->state = TOR_TLS_ST_GOTCLOSE;
+  tor_tls_get_state_description(tls, buf, 200);
+  tt_str_op(buf, OP_EQ, SSL_STATE_STR " in GOTCLOSE");
+
+  tls->state = TOR_TLS_ST_SENTCLOSE;
+  tor_tls_get_state_description(tls, buf, 200);
+  tt_str_op(buf, OP_EQ, SSL_STATE_STR " in SENTCLOSE");
+
+  tls->state = TOR_TLS_ST_CLOSED;
+  tor_tls_get_state_description(tls, buf, 200);
+  tt_str_op(buf, OP_EQ, SSL_STATE_STR " in CLOSED");
+
+  tls->state = TOR_TLS_ST_RENEGOTIATE;
+  tor_tls_get_state_description(tls, buf, 200);
+  tt_str_op(buf, OP_EQ, SSL_STATE_STR " in RENEGOTIATE");
+
+  tls->state = TOR_TLS_ST_BUFFEREVENT;
+  tor_tls_get_state_description(tls, buf, 200);
+  tt_str_op(buf, OP_EQ, SSL_STATE_STR);
+
+  tls->state = 7;
+  tor_tls_get_state_description(tls, buf, 200);
+  tt_str_op(buf, OP_EQ, SSL_STATE_STR " in unknown TLS state");
+
+ done:
+  SSL_CTX_free(ctx);
+  SSL_free(tls->ssl);
+  tor_free(buf);
+  tor_free(tls);
+}
+
+static void
+test_tortls_get_by_ssl(void *ignored)
+{
+  (void)ignored;
+  tor_tls_t *tls;
+  tor_tls_t *res;
+  SSL_CTX *ctx;
+  SSL *ssl;
+
+  library_init();
+  tor_tls_allocate_tor_tls_object_ex_data_index();
+
+  ctx = SSL_CTX_new(SSLv23_method());
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+  tls->magic = TOR_TLS_MAGIC;
+
+  ssl = SSL_new(ctx);
+
+  res = tor_tls_get_by_ssl(ssl);
+  tt_assert(!res);
+
+  SSL_set_ex_data(ssl, tor_tls_object_ex_data_index, tls);
+
+  res = tor_tls_get_by_ssl(ssl);
+  tt_assert(res == tls);
+
+ done:
+  SSL_free(ssl);
+  SSL_CTX_free(ctx);
+  tor_free(tls);
+}
+
+static void
+test_tortls_allocate_tor_tls_object_ex_data_index(void *ignored)
+{
+  (void)ignored;
+  int first;
+
+  tor_tls_allocate_tor_tls_object_ex_data_index();
+
+  first = tor_tls_object_ex_data_index;
+  tor_tls_allocate_tor_tls_object_ex_data_index();
+  tt_int_op(first, OP_EQ, tor_tls_object_ex_data_index);
+
+ done:
+  (void)0;
+}
+
+static void
+test_tortls_log_one_error(void *ignored)
+{
+  (void)ignored;
+  tor_tls_t *tls;
+  SSL_CTX *ctx;
+  SSL *ssl = NULL;
+
+  library_init();
+
+  ctx = SSL_CTX_new(SSLv23_method());
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+  setup_capture_of_logs(LOG_INFO);
+
+  tor_tls_log_one_error(NULL, 0, LOG_WARN, 0, "something");
+  expect_log_msg("TLS error while something: "
+            "(null) (in (null):(null):---)\n");
+
+  mock_clean_saved_logs();
+  tor_tls_log_one_error(tls, 0, LOG_WARN, 0, NULL);
+  expect_log_msg("TLS error: (null) "
+            "(in (null):(null):---)\n");
+
+  mock_clean_saved_logs();
+  tls->address = tor_strdup("127.hello");
+  tor_tls_log_one_error(tls, 0, LOG_WARN, 0, NULL);
+  expect_log_msg("TLS error with 127.hello: "
+            "(null) (in (null):(null):---)\n");
+  tor_free(tls->address);
+
+  mock_clean_saved_logs();
+  tls->address = tor_strdup("127.hello");
+  tor_tls_log_one_error(tls, 0, LOG_WARN, 0, "blarg");
+  expect_log_msg("TLS error while blarg with "
+            "127.hello: (null) (in (null):(null):---)\n");
+
+  mock_clean_saved_logs();
+  tor_tls_log_one_error(tls, ERR_PACK(1, 2, 3), LOG_WARN, 0, NULL);
+  expect_log_msg("TLS error with 127.hello: "
+            "BN lib (in unknown library:(null):---)\n");
+
+  mock_clean_saved_logs();
+  tor_tls_log_one_error(tls, ERR_PACK(1, 2, SSL_R_HTTP_REQUEST),
+                        LOG_WARN, 0, NULL);
+  expect_log_severity(LOG_INFO);
+
+  mock_clean_saved_logs();
+  tor_tls_log_one_error(tls, ERR_PACK(1, 2, SSL_R_HTTPS_PROXY_REQUEST),
+                        LOG_WARN, 0, NULL);
+  expect_log_severity(LOG_INFO);
+
+  mock_clean_saved_logs();
+  tor_tls_log_one_error(tls, ERR_PACK(1, 2, SSL_R_RECORD_LENGTH_MISMATCH),
+                        LOG_WARN, 0, NULL);
+  expect_log_severity(LOG_INFO);
+
+#ifndef OPENSSL_1_1_API
+  mock_clean_saved_logs();
+  tor_tls_log_one_error(tls, ERR_PACK(1, 2, SSL_R_RECORD_TOO_LARGE),
+                        LOG_WARN, 0, NULL);
+  expect_log_severity(LOG_INFO);
+#endif /* !defined(OPENSSL_1_1_API) */
+
+  mock_clean_saved_logs();
+  tor_tls_log_one_error(tls, ERR_PACK(1, 2, SSL_R_UNKNOWN_PROTOCOL),
+                        LOG_WARN, 0, NULL);
+  expect_log_severity(LOG_INFO);
+
+  mock_clean_saved_logs();
+  tor_tls_log_one_error(tls, ERR_PACK(1, 2, SSL_R_UNSUPPORTED_PROTOCOL),
+                        LOG_WARN, 0, NULL);
+  expect_log_severity(LOG_INFO);
+
+  tls->ssl = SSL_new(ctx);
+
+  mock_clean_saved_logs();
+  tor_tls_log_one_error(tls, 0, LOG_WARN, 0, NULL);
+  expect_log_msg("TLS error with 127.hello: (null)"
+            " (in (null):(null):" SSL_STATE_STR ")\n");
+
+ done:
+  teardown_capture_of_logs();
+  SSL_free(ssl);
+  SSL_CTX_free(ctx);
+  if (tls && tls->ssl)
+    SSL_free(tls->ssl);
+  if (tls)
+    tor_free(tls->address);
+  tor_free(tls);
+}
+
+#ifndef OPENSSL_OPAQUE
+static void
+test_tortls_get_error(void *ignored)
+{
+  (void)ignored;
+  tor_tls_t *tls;
+  int ret;
+  SSL_CTX *ctx;
+
+  library_init();
+
+  ctx = SSL_CTX_new(SSLv23_method());
+  setup_capture_of_logs(LOG_INFO);
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+  tls->ssl = SSL_new(ctx);
+  SSL_set_bio(tls->ssl, BIO_new(BIO_s_mem()), NULL);
+
+  ret = tor_tls_get_error(tls, 0, 0, "something", LOG_WARN, 0);
+  tt_int_op(ret, OP_EQ, TOR_TLS_ERROR_IO);
+  expect_log_msg("TLS error: unexpected close while"
+            " something (before/accept initialization)\n");
+
+  mock_clean_saved_logs();
+  ret = tor_tls_get_error(tls, 2, 0, "something", LOG_WARN, 0);
+  tt_int_op(ret, OP_EQ, 0);
+  expect_no_log_entry();
+
+  mock_clean_saved_logs();
+  ret = tor_tls_get_error(tls, 0, 1, "something", LOG_WARN, 0);
+  tt_int_op(ret, OP_EQ, -11);
+  expect_no_log_entry();
+
+  mock_clean_saved_logs();
+  ERR_clear_error();
+  ERR_put_error(ERR_LIB_BN, 2, -1, "somewhere.c", 99);
+  ret = tor_tls_get_error(tls, 0, 0, "something", LOG_WARN, 0);
+  tt_int_op(ret, OP_EQ, TOR_TLS_ERROR_MISC);
+  expect_log_msg("TLS error while something: (null)"
+            " (in bignum routines:(null):before/accept initialization)\n");
+
+  mock_clean_saved_logs();
+  ERR_clear_error();
+  tls->ssl->rwstate = SSL_READING;
+  SSL_get_rbio(tls->ssl)->flags = BIO_FLAGS_READ;
+  ret = tor_tls_get_error(tls, -1, 0, "something", LOG_WARN, 0);
+  tt_int_op(ret, OP_EQ, TOR_TLS_WANTREAD);
+  expect_no_log_entry();
+
+  mock_clean_saved_logs();
+  ERR_clear_error();
+  tls->ssl->rwstate = SSL_READING;
+  SSL_get_rbio(tls->ssl)->flags = BIO_FLAGS_WRITE;
+  ret = tor_tls_get_error(tls, -1, 0, "something", LOG_WARN, 0);
+  tt_int_op(ret, OP_EQ, TOR_TLS_WANTWRITE);
+  expect_no_log_entry();
+
+  mock_clean_saved_logs();
+  ERR_clear_error();
+  tls->ssl->rwstate = 0;
+  tls->ssl->shutdown = SSL_RECEIVED_SHUTDOWN;
+  tls->ssl->s3->warn_alert =SSL_AD_CLOSE_NOTIFY;
+  ret = tor_tls_get_error(tls, 0, 0, "something", LOG_WARN, 0);
+  tt_int_op(ret, OP_EQ, TOR_TLS_CLOSE);
+  expect_log_entry();
+
+  mock_clean_saved_logs();
+  ret = tor_tls_get_error(tls, 0, 2, "something", LOG_WARN, 0);
+  tt_int_op(ret, OP_EQ, -10);
+  expect_no_log_entry();
+
+  mock_clean_saved_logs();
+  ERR_put_error(ERR_LIB_SYS, 2, -1, "somewhere.c", 99);
+  ret = tor_tls_get_error(tls, -1, 0, "something", LOG_WARN, 0);
+  tt_int_op(ret, OP_EQ, -9);
+  expect_log_msg("TLS error while something: (null) (in system library:"
+            "connect:before/accept initialization)\n");
+
+ done:
+  teardown_capture_of_logs();
+  SSL_free(tls->ssl);
+  tor_free(tls);
+  SSL_CTX_free(ctx);
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+static void
+test_tortls_always_accept_verify_cb(void *ignored)
+{
+  (void)ignored;
+  int ret;
+
+  ret = always_accept_verify_cb(0, NULL);
+  tt_int_op(ret, OP_EQ, 1);
+
+ done:
+  (void)0;
+}
+
+#ifndef OPENSSL_OPAQUE
+static void
+test_tortls_x509_cert_free(void *ignored)
+{
+  (void)ignored;
+  tor_x509_cert_t *cert;
+
+  cert = tor_malloc_zero(sizeof(tor_x509_cert_t));
+  tor_x509_cert_free(cert);
+
+  cert = tor_malloc_zero(sizeof(tor_x509_cert_t));
+  cert->cert = X509_new();
+  cert->encoded = tor_malloc_zero(1);
+  tor_x509_cert_free(cert);
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+#ifndef OPENSSL_OPAQUE
+/*
+ * Use only for the matching fake_x509_free() call
+ */
+static X509 *
+fake_x509_malloc(void)
+{
+  return tor_malloc_zero(sizeof(X509));
+}
+
+static void
+fake_x509_free(X509 *cert)
+{
+  if (cert) {
+    if (cert->cert_info) {
+      if (cert->cert_info->key) {
+        if (cert->cert_info->key->pkey) {
+          tor_free(cert->cert_info->key->pkey);
+        }
+        tor_free(cert->cert_info->key);
+      }
+      tor_free(cert->cert_info);
+    }
+    tor_free(cert);
+  }
+}
+#endif
+
+static tor_x509_cert_t *fixed_x509_cert = NULL;
+static tor_x509_cert_t *
+get_peer_cert_mock_return_fixed(tor_tls_t *tls)
+{
+  (void)tls;
+  if (fixed_x509_cert)
+    return tor_x509_cert_dup(fixed_x509_cert);
+  else
+    return NULL;
+}
+
+static void
+test_tortls_cert_matches_key(void *ignored)
+{
+  (void)ignored;
+
+  X509 *cert1 = NULL, *cert2 = NULL, *cert3 = NULL, *cert4 = NULL;
+  tor_x509_cert_t *c1 = NULL, *c2 = NULL, *c3 = NULL, *c4 = NULL;
+  crypto_pk_t *k1 = NULL, *k2 = NULL, *k3 = NULL;
+
+  k1 = pk_generate(1);
+  k2 = pk_generate(2);
+  k3 = pk_generate(3);
+
+  cert1 = tor_tls_create_certificate(k1, k2, "A", "B", 1000);
+  cert2 = tor_tls_create_certificate(k1, k3, "C", "D", 1000);
+  cert3 = tor_tls_create_certificate(k2, k3, "C", "D", 1000);
+  cert4 = tor_tls_create_certificate(k3, k2, "E", "F", 1000);
+
+  tt_assert(cert1 && cert2 && cert3 && cert4);
+
+  c1 = tor_x509_cert_new(cert1); cert1 = NULL;
+  c2 = tor_x509_cert_new(cert2); cert2 = NULL;
+  c3 = tor_x509_cert_new(cert3); cert3 = NULL;
+  c4 = tor_x509_cert_new(cert4); cert4 = NULL;
+
+  tt_assert(c1 && c2 && c3 && c4);
+
+  MOCK(tor_tls_get_peer_cert, get_peer_cert_mock_return_fixed);
+
+  fixed_x509_cert = NULL;
+  /* If the peer has no certificate, it shouldn't match anything. */
+  tt_assert(! tor_tls_cert_matches_key(NULL, c1));
+  tt_assert(! tor_tls_cert_matches_key(NULL, c2));
+  tt_assert(! tor_tls_cert_matches_key(NULL, c3));
+  tt_assert(! tor_tls_cert_matches_key(NULL, c4));
+  fixed_x509_cert = c1;
+  /* If the peer has a certificate, it should match every cert with the same
+   * subject key. */
+  tt_assert(tor_tls_cert_matches_key(NULL, c1));
+  tt_assert(tor_tls_cert_matches_key(NULL, c2));
+  tt_assert(! tor_tls_cert_matches_key(NULL, c3));
+  tt_assert(! tor_tls_cert_matches_key(NULL, c4));
+
+ done:
+  tor_x509_cert_free(c1);
+  tor_x509_cert_free(c2);
+  tor_x509_cert_free(c3);
+  tor_x509_cert_free(c4);
+  if (cert1) X509_free(cert1);
+  if (cert2) X509_free(cert2);
+  if (cert3) X509_free(cert3);
+  if (cert4) X509_free(cert4);
+  crypto_pk_free(k1);
+  crypto_pk_free(k2);
+  crypto_pk_free(k3);
+  UNMOCK(tor_tls_get_peer_cert);
+}
+
+#ifndef OPENSSL_OPAQUE
+static void
+test_tortls_cert_get_key(void *ignored)
+{
+  (void)ignored;
+  tor_x509_cert_t *cert = NULL;
+  crypto_pk_t *res = NULL;
+  cert = tor_malloc_zero(sizeof(tor_x509_cert_t));
+  X509 *key = NULL;
+  key = fake_x509_malloc();
+  key->references = 1;
+
+  res = tor_tls_cert_get_key(cert);
+  tt_assert(!res);
+
+  cert->cert = key;
+  key->cert_info = tor_malloc_zero(sizeof(X509_CINF));
+  key->cert_info->key = tor_malloc_zero(sizeof(X509_PUBKEY));
+  key->cert_info->key->pkey = tor_malloc_zero(sizeof(EVP_PKEY));
+  key->cert_info->key->pkey->references = 1;
+  key->cert_info->key->pkey->type = 2;
+  res = tor_tls_cert_get_key(cert);
+  tt_assert(!res);
+
+ done:
+  fake_x509_free(key);
+  tor_free(cert);
+  crypto_pk_free(res);
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+static void
+test_tortls_get_my_client_auth_key(void *ignored)
+{
+  (void)ignored;
+  crypto_pk_t *ret;
+  crypto_pk_t *expected;
+  tor_tls_context_t *ctx;
+  RSA *k = RSA_new();
+
+  ctx = tor_malloc_zero(sizeof(tor_tls_context_t));
+  expected = crypto_new_pk_from_openssl_rsa_(k);
+  ctx->auth_key = expected;
+
+  client_tls_context = NULL;
+  ret = tor_tls_get_my_client_auth_key();
+  tt_assert(!ret);
+
+  client_tls_context = ctx;
+  ret = tor_tls_get_my_client_auth_key();
+  tt_assert(ret == expected);
+
+ done:
+  crypto_pk_free(expected);
+  tor_free(ctx);
+}
+
+#ifndef HAVE_SSL_GET_CLIENT_CIPHERS
+static SSL_CIPHER *
+get_cipher_by_name(const char *name)
+{
+  int i;
+  const SSL_METHOD *method = SSLv23_method();
+  int num = method->num_ciphers();
+
+  for (i = 0; i < num; ++i) {
+    const SSL_CIPHER *cipher = method->get_cipher(i);
+    const char *ciphername = SSL_CIPHER_get_name(cipher);
+    if (!strcmp(ciphername, name)) {
+      return (SSL_CIPHER *)cipher;
+    }
+  }
+
+  return NULL;
+}
+#endif /* !defined(HAVE_SSL_GET_CLIENT_CIPHERS) */
+
+#ifndef OPENSSL_OPAQUE
+static void
+test_tortls_get_ciphersuite_name(void *ignored)
+{
+  (void)ignored;
+  const char *ret;
+  tor_tls_t *ctx;
+  ctx = tor_malloc_zero(sizeof(tor_tls_t));
+  ctx->ssl = tor_malloc_zero(sizeof(SSL));
+
+  ret = tor_tls_get_ciphersuite_name(ctx);
+  tt_str_op(ret, OP_EQ, "(NONE)");
+
+ done:
+  tor_free(ctx->ssl);
+  tor_free(ctx);
+}
+
+static SSL_CIPHER *
+get_cipher_by_id(uint16_t id)
+{
+  int i;
+  const SSL_METHOD *method = SSLv23_method();
+  int num = method->num_ciphers();
+  for (i = 0; i < num; ++i) {
+    const SSL_CIPHER *cipher = method->get_cipher(i);
+    if (id == (SSL_CIPHER_get_id(cipher) & 0xffff)) {
+      return (SSL_CIPHER *)cipher;
+    }
+  }
+
+  return NULL;
+}
+
+static void
+test_tortls_classify_client_ciphers(void *ignored)
+{
+  (void)ignored;
+  int i;
+  int ret;
+  SSL_CTX *ctx;
+  SSL *ssl;
+  tor_tls_t *tls;
+  STACK_OF(SSL_CIPHER) *ciphers;
+  SSL_CIPHER *tmp_cipher;
+
+  library_init();
+
+  tor_tls_allocate_tor_tls_object_ex_data_index();
+
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+  tls->magic = TOR_TLS_MAGIC;
+
+  ctx = SSL_CTX_new(TLSv1_method());
+  ssl = SSL_new(ctx);
+  tls->ssl = ssl;
+
+  ciphers = sk_SSL_CIPHER_new_null();
+
+  ret = tor_tls_classify_client_ciphers(ssl, NULL);
+  tt_int_op(ret, OP_EQ, -1);
+
+  SSL_set_ex_data(ssl, tor_tls_object_ex_data_index, tls);
+  tls->client_cipher_list_type = 42;
+
+  ret = tor_tls_classify_client_ciphers(ssl, NULL);
+  tt_int_op(ret, OP_EQ, 42);
+
+  tls->client_cipher_list_type = 0;
+  ret = tor_tls_classify_client_ciphers(ssl, ciphers);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_int_op(tls->client_cipher_list_type, OP_EQ, 1);
+
+  tls->client_cipher_list_type = 0;
+  ret = tor_tls_classify_client_ciphers(ssl, SSL_get_ciphers(ssl));
+  tt_int_op(ret, OP_EQ, 3);
+  tt_int_op(tls->client_cipher_list_type, OP_EQ, 3);
+
+  SSL_CIPHER *one = get_cipher_by_name(TLS1_TXT_DHE_RSA_WITH_AES_128_SHA),
+    *two = get_cipher_by_name(TLS1_TXT_DHE_RSA_WITH_AES_256_SHA),
+    *three = get_cipher_by_name(SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA),
+    *four = NULL;
+  sk_SSL_CIPHER_push(ciphers, one);
+  sk_SSL_CIPHER_push(ciphers, two);
+  sk_SSL_CIPHER_push(ciphers, three);
+  sk_SSL_CIPHER_push(ciphers, four);
+
+  tls->client_cipher_list_type = 0;
+  ret = tor_tls_classify_client_ciphers(ssl, ciphers);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_int_op(tls->client_cipher_list_type, OP_EQ, 1);
+
+  sk_SSL_CIPHER_zero(ciphers);
+
+  one = get_cipher_by_name("ECDHE-RSA-AES256-GCM-SHA384");
+  tt_assert(one);
+  one->id = 0x00ff;
+  two = get_cipher_by_name("ECDHE-RSA-AES128-GCM-SHA256");
+  tt_assert(two);
+  two->id = 0x0000;
+  sk_SSL_CIPHER_push(ciphers, one);
+  tls->client_cipher_list_type = 0;
+  ret = tor_tls_classify_client_ciphers(ssl, ciphers);
+  tt_int_op(ret, OP_EQ, 3);
+  tt_int_op(tls->client_cipher_list_type, OP_EQ, 3);
+
+  sk_SSL_CIPHER_push(ciphers, two);
+  tls->client_cipher_list_type = 0;
+  ret = tor_tls_classify_client_ciphers(ssl, ciphers);
+  tt_int_op(ret, OP_EQ, 3);
+  tt_int_op(tls->client_cipher_list_type, OP_EQ, 3);
+
+  one->id = 0xC00A;
+  tls->client_cipher_list_type = 0;
+  ret = tor_tls_classify_client_ciphers(ssl, ciphers);
+  tt_int_op(ret, OP_EQ, 3);
+  tt_int_op(tls->client_cipher_list_type, OP_EQ, 3);
+
+  sk_SSL_CIPHER_zero(ciphers);
+  for (i=0; v2_cipher_list[i]; i++) {
+    tmp_cipher = get_cipher_by_id(v2_cipher_list[i]);
+    tt_assert(tmp_cipher);
+    sk_SSL_CIPHER_push(ciphers, tmp_cipher);
+  }
+  tls->client_cipher_list_type = 0;
+  ret = tor_tls_classify_client_ciphers(ssl, ciphers);
+  tt_int_op(ret, OP_EQ, 2);
+  tt_int_op(tls->client_cipher_list_type, OP_EQ, 2);
+
+ done:
+  sk_SSL_CIPHER_free(ciphers);
+  SSL_free(tls->ssl);
+  tor_free(tls);
+  SSL_CTX_free(ctx);
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+static void
+test_tortls_client_is_using_v2_ciphers(void *ignored)
+{
+  (void)ignored;
+
+#ifdef HAVE_SSL_GET_CLIENT_CIPHERS
+  tt_skip();
+ done:
+  (void)1;
+#else
+  int ret;
+  SSL_CTX *ctx;
+  SSL *ssl;
+  SSL_SESSION *sess;
+  STACK_OF(SSL_CIPHER) *ciphers;
+
+  library_init();
+
+  ctx = SSL_CTX_new(TLSv1_method());
+  ssl = SSL_new(ctx);
+  sess = SSL_SESSION_new();
+
+  ret = tor_tls_client_is_using_v2_ciphers(ssl);
+  tt_int_op(ret, OP_EQ, -1);
+
+  ssl->session = sess;
+  ret = tor_tls_client_is_using_v2_ciphers(ssl);
+  tt_int_op(ret, OP_EQ, 0);
+
+  ciphers = sk_SSL_CIPHER_new_null();
+  SSL_CIPHER *one = get_cipher_by_name("ECDHE-RSA-AES256-GCM-SHA384");
+  tt_assert(one);
+  one->id = 0x00ff;
+  sk_SSL_CIPHER_push(ciphers, one);
+  sess->ciphers = ciphers;
+  ret = tor_tls_client_is_using_v2_ciphers(ssl);
+  tt_int_op(ret, OP_EQ, 1);
+ done:
+  SSL_free(ssl);
+  SSL_CTX_free(ctx);
+#endif /* defined(HAVE_SSL_GET_CLIENT_CIPHERS) */
+}
+
+#ifndef OPENSSL_OPAQUE
+static int fixed_ssl_pending_result = 0;
+
+static int
+fixed_ssl_pending(const SSL *ignored)
+{
+  (void)ignored;
+  return fixed_ssl_pending_result;
+}
+
+static void
+test_tortls_get_pending_bytes(void *ignored)
+{
+  (void)ignored;
+  int ret;
+  tor_tls_t *tls;
+  SSL_METHOD *method;
+
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+  tls->ssl = tor_malloc_zero(sizeof(SSL));
+  method = tor_malloc_zero(sizeof(SSL_METHOD));
+  method->ssl_pending = fixed_ssl_pending;
+  tls->ssl->method = method;
+
+  fixed_ssl_pending_result = 42;
+  ret = tor_tls_get_pending_bytes(tls);
+  tt_int_op(ret, OP_EQ, 42);
+
+ done:
+  tor_free(method);
+  tor_free(tls->ssl);
+  tor_free(tls);
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+#ifndef OPENSSL_OPAQUE
+static void
+test_tortls_SSL_SESSION_get_master_key(void *ignored)
+{
+  (void)ignored;
+  size_t ret;
+  tor_tls_t *tls;
+  uint8_t *out;
+  out = tor_malloc_zero(1);
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+  tls->ssl = tor_malloc_zero(sizeof(SSL));
+  tls->ssl->session = tor_malloc_zero(sizeof(SSL_SESSION));
+  tls->ssl->session->master_key_length = 1;
+
+#ifndef HAVE_SSL_SESSION_GET_MASTER_KEY
+  tls->ssl->session->master_key[0] = 43;
+  ret = SSL_SESSION_get_master_key(tls->ssl->session, out, 0);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_int_op(out[0], OP_EQ, 0);
+
+  ret = SSL_SESSION_get_master_key(tls->ssl->session, out, 1);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_int_op(out[0], OP_EQ, 43);
+
+ done:
+#endif /* !defined(HAVE_SSL_SESSION_GET_MASTER_KEY) */
+  tor_free(tls->ssl->session);
+  tor_free(tls->ssl);
+  tor_free(tls);
+  tor_free(out);
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+#ifndef OPENSSL_OPAQUE
+static void
+test_tortls_get_tlssecrets(void *ignored)
+{
+  (void)ignored;
+  int ret;
+  uint8_t *secret_out = tor_malloc_zero(DIGEST256_LEN);
+  tor_tls_t *tls;
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+  tls->ssl = tor_malloc_zero(sizeof(SSL));
+  tls->ssl->session = tor_malloc_zero(sizeof(SSL_SESSION));
+  tls->ssl->session->master_key_length = 1;
+  tls->ssl->s3 = tor_malloc_zero(sizeof(SSL3_STATE));
+
+  ret = tor_tls_get_tlssecrets(tls, secret_out);
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  tor_free(secret_out);
+  tor_free(tls->ssl->s3);
+  tor_free(tls->ssl->session);
+  tor_free(tls->ssl);
+  tor_free(tls);
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+#ifndef OPENSSL_OPAQUE
+static void
+test_tortls_get_buffer_sizes(void *ignored)
+{
+  (void)ignored;
+  int ret;
+  tor_tls_t *tls;
+  size_t rbuf_c=-1, rbuf_b=-1, wbuf_c=-1, wbuf_b=-1;
+
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+  tls->ssl = tor_malloc_zero(sizeof(SSL));
+  tls->ssl->s3 = tor_malloc_zero(sizeof(SSL3_STATE));
+
+  tls->ssl->s3->rbuf.buf = NULL;
+  tls->ssl->s3->rbuf.len = 1;
+  tls->ssl->s3->rbuf.offset = 0;
+  tls->ssl->s3->rbuf.left = 42;
+
+  tls->ssl->s3->wbuf.buf = NULL;
+  tls->ssl->s3->wbuf.len = 2;
+  tls->ssl->s3->wbuf.offset = 0;
+  tls->ssl->s3->wbuf.left = 43;
+
+  ret = tor_tls_get_buffer_sizes(tls, &rbuf_c, &rbuf_b, &wbuf_c, &wbuf_b);
+#if OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0)
+  tt_int_op(ret, OP_EQ, -1);
+#else
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(rbuf_c, OP_EQ, 0);
+  tt_int_op(wbuf_c, OP_EQ, 0);
+  tt_int_op(rbuf_b, OP_EQ, 42);
+  tt_int_op(wbuf_b, OP_EQ, 43);
+
+  tls->ssl->s3->rbuf.buf = tor_malloc_zero(1);
+  tls->ssl->s3->wbuf.buf = tor_malloc_zero(1);
+  ret = tor_tls_get_buffer_sizes(tls, &rbuf_c, &rbuf_b, &wbuf_c, &wbuf_b);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(rbuf_c, OP_EQ, 1);
+  tt_int_op(wbuf_c, OP_EQ, 2);
+
+#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) */
+
+ done:
+  tor_free(tls->ssl->s3->rbuf.buf);
+  tor_free(tls->ssl->s3->wbuf.buf);
+  tor_free(tls->ssl->s3);
+  tor_free(tls->ssl);
+  tor_free(tls);
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+#ifndef OPENSSL_OPAQUE
+typedef struct cert_pkey_st_local
+{
+  X509 *x509;
+  EVP_PKEY *privatekey;
+  const EVP_MD *digest;
+} CERT_PKEY_local;
+
+typedef struct sess_cert_st_local
+{
+  STACK_OF(X509) *cert_chain;
+  int peer_cert_type;
+  CERT_PKEY_local *peer_key;
+  CERT_PKEY_local peer_pkeys[8];
+  int references;
+} SESS_CERT_local;
+
+static void
+test_tortls_try_to_extract_certs_from_tls(void *ignored)
+{
+  (void)ignored;
+  tor_tls_t *tls;
+  X509 *cert = NULL, *id_cert = NULL, *c1 = NULL, *c2 = NULL;
+  SESS_CERT_local *sess = NULL;
+
+  c1 = read_cert_from(validCertString);
+  c2 = read_cert_from(caCertString);
+
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+  tls->ssl = tor_malloc_zero(sizeof(SSL));
+  tls->ssl->session = tor_malloc_zero(sizeof(SSL_SESSION));
+  sess = tor_malloc_zero(sizeof(SESS_CERT_local));
+  tls->ssl->session->sess_cert = (void *)sess;
+
+  try_to_extract_certs_from_tls(LOG_WARN, tls, &cert, &id_cert);
+  tt_assert(!cert);
+  tt_assert(!id_cert);
+
+  tls->ssl->session->peer = c1;
+  try_to_extract_certs_from_tls(LOG_WARN, tls, &cert, &id_cert);
+  tt_assert(cert == c1);
+  tt_assert(!id_cert);
+  X509_free(cert); /* decrease refcnt */
+
+  sess->cert_chain = sk_X509_new_null();
+  try_to_extract_certs_from_tls(LOG_WARN, tls, &cert, &id_cert);
+  tt_assert(cert == c1);
+  tt_assert(!id_cert);
+  X509_free(cert); /* decrease refcnt */
+
+  sk_X509_push(sess->cert_chain, c1);
+  sk_X509_push(sess->cert_chain, c2);
+
+  try_to_extract_certs_from_tls(LOG_WARN, tls, &cert, &id_cert);
+  tt_assert(cert == c1);
+  tt_assert(id_cert);
+  X509_free(cert); /* decrease refcnt */
+  X509_free(id_cert); /* decrease refcnt */
+
+ done:
+  sk_X509_free(sess->cert_chain);
+  tor_free(sess);
+  tor_free(tls->ssl->session);
+  tor_free(tls->ssl);
+  tor_free(tls);
+  X509_free(c1);
+  X509_free(c2);
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+#ifndef OPENSSL_OPAQUE
+static void
+test_tortls_get_peer_cert(void *ignored)
+{
+  (void)ignored;
+  tor_x509_cert_t *ret;
+  tor_tls_t *tls;
+  X509 *cert = NULL;
+
+  cert = read_cert_from(validCertString);
+
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+  tls->ssl = tor_malloc_zero(sizeof(SSL));
+  tls->ssl->session = tor_malloc_zero(sizeof(SSL_SESSION));
+
+  ret = tor_tls_get_peer_cert(tls);
+  tt_assert(!ret);
+
+  tls->ssl->session->peer = cert;
+  ret = tor_tls_get_peer_cert(tls);
+  tt_assert(ret);
+  tt_assert(ret->cert == cert);
+
+ done:
+  tor_x509_cert_free(ret);
+  tor_free(tls->ssl->session);
+  tor_free(tls->ssl);
+  tor_free(tls);
+  X509_free(cert);
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+#ifndef OPENSSL_OPAQUE
+static void
+test_tortls_peer_has_cert(void *ignored)
+{
+  (void)ignored;
+  int ret;
+  tor_tls_t *tls;
+  X509 *cert = NULL;
+
+  cert = read_cert_from(validCertString);
+
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+  tls->ssl = tor_malloc_zero(sizeof(SSL));
+  tls->ssl->session = tor_malloc_zero(sizeof(SSL_SESSION));
+
+  ret = tor_tls_peer_has_cert(tls);
+  tt_assert(!ret);
+
+  tls->ssl->session->peer = cert;
+  ret = tor_tls_peer_has_cert(tls);
+  tt_assert(ret);
+
+ done:
+  tor_free(tls->ssl->session);
+  tor_free(tls->ssl);
+  tor_free(tls);
+  X509_free(cert);
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+static void
+test_tortls_get_write_overhead_ratio(void *ignored)
+{
+  (void)ignored;
+  double ret;
+
+  total_bytes_written_over_tls = 0;
+  ret = tls_get_write_overhead_ratio();
+  tt_double_op(fabs(ret - 1.0), OP_LT, 1E-12);
+
+  total_bytes_written_by_tls = 10;
+  total_bytes_written_over_tls = 1;
+  ret = tls_get_write_overhead_ratio();
+  tt_double_op(fabs(ret - 10.0), OP_LT, 1E-12);
+
+  total_bytes_written_by_tls = 10;
+  total_bytes_written_over_tls = 2;
+  ret = tls_get_write_overhead_ratio();
+  tt_double_op(fabs(ret - 5.0), OP_LT, 1E-12);
+
+ done:
+  (void)0;
+}
+
+static void
+test_tortls_is_server(void *ignored)
+{
+  (void)ignored;
+  tor_tls_t *tls;
+  int ret;
+
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+  tls->isServer = 1;
+  ret = tor_tls_is_server(tls);
+  tt_int_op(ret, OP_EQ, 1);
+
+ done:
+  tor_free(tls);
+}
+
+#ifndef OPENSSL_OPAQUE
+static void
+test_tortls_session_secret_cb(void *ignored)
+{
+  (void)ignored;
+  tor_tls_t *tls;
+  SSL_CTX *ctx;
+  STACK_OF(SSL_CIPHER) *ciphers = NULL;
+  SSL_CIPHER *one;
+
+  library_init();
+
+  tor_tls_allocate_tor_tls_object_ex_data_index();
+
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+
+  tls->magic = TOR_TLS_MAGIC;
+
+  ctx = SSL_CTX_new(TLSv1_method());
+  tls->ssl = SSL_new(ctx);
+  SSL_set_ex_data(tls->ssl, tor_tls_object_ex_data_index, tls);
+
+  SSL_set_session_secret_cb(tls->ssl, tor_tls_session_secret_cb, NULL);
+
+  tor_tls_session_secret_cb(tls->ssl, NULL, NULL, NULL, NULL, NULL);
+  tt_assert(!tls->ssl->tls_session_secret_cb);
+
+  one = get_cipher_by_name("ECDHE-RSA-AES256-GCM-SHA384");
+  one->id = 0x00ff;
+  ciphers = sk_SSL_CIPHER_new_null();
+  sk_SSL_CIPHER_push(ciphers, one);
+
+  tls->client_cipher_list_type = 0;
+  tor_tls_session_secret_cb(tls->ssl, NULL, NULL, ciphers, NULL, NULL);
+  tt_assert(!tls->ssl->tls_session_secret_cb);
+
+ done:
+  sk_SSL_CIPHER_free(ciphers);
+  SSL_free(tls->ssl);
+  SSL_CTX_free(ctx);
+  tor_free(tls);
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+#ifndef OPENSSL_OPAQUE
+/* TODO: It seems block_renegotiation and unblock_renegotiation and
+ * using different blags. This might not be correct */
+static void
+test_tortls_block_renegotiation(void *ignored)
+{
+  (void)ignored;
+  tor_tls_t *tls;
+
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+  tls->ssl = tor_malloc_zero(sizeof(SSL));
+  tls->ssl->s3 = tor_malloc_zero(sizeof(SSL3_STATE));
+#ifndef SUPPORT_UNSAFE_RENEGOTIATION_FLAG
+#define SSL3_FLAGS_ALLOW_UNSAFE_LEGACY_RENEGOTIATION 0
+#endif
+
+  tls->ssl->s3->flags = SSL3_FLAGS_ALLOW_UNSAFE_LEGACY_RENEGOTIATION;
+
+  tor_tls_block_renegotiation(tls);
+
+#ifndef OPENSSL_1_1_API
+  tt_assert(!(tls->ssl->s3->flags &
+              SSL3_FLAGS_ALLOW_UNSAFE_LEGACY_RENEGOTIATION));
+#endif
+
+ done:
+  tor_free(tls->ssl->s3);
+  tor_free(tls->ssl);
+  tor_free(tls);
+}
+
+static void
+test_tortls_unblock_renegotiation(void *ignored)
+{
+  (void)ignored;
+  tor_tls_t *tls;
+
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+  tls->ssl = tor_malloc_zero(sizeof(SSL));
+  tor_tls_unblock_renegotiation(tls);
+
+  tt_uint_op(SSL_get_options(tls->ssl) &
+             SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION, OP_EQ,
+             SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION);
+
+ done:
+  tor_free(tls->ssl);
+  tor_free(tls);
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+#ifndef OPENSSL_OPAQUE
+static void
+test_tortls_assert_renegotiation_unblocked(void *ignored)
+{
+  (void)ignored;
+  tor_tls_t *tls;
+
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+  tls->ssl = tor_malloc_zero(sizeof(SSL));
+  tor_tls_unblock_renegotiation(tls);
+  tor_tls_assert_renegotiation_unblocked(tls);
+  /* No assertion here - this test will fail if tor_assert is turned on
+   * and things are bad. */
+
+  tor_free(tls->ssl);
+  tor_free(tls);
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+static void
+test_tortls_set_logged_address(void *ignored)
+{
+  (void)ignored;
+  tor_tls_t *tls;
+
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+
+  tor_tls_set_logged_address(tls, "foo bar");
+
+  tt_str_op(tls->address, OP_EQ, "foo bar");
+
+  tor_tls_set_logged_address(tls, "foo bar 2");
+  tt_str_op(tls->address, OP_EQ, "foo bar 2");
+
+ done:
+  tor_free(tls->address);
+  tor_free(tls);
+}
+
+#ifndef OPENSSL_OPAQUE
+static void
+example_cb(tor_tls_t *t, void *arg)
+{
+  (void)t;
+  (void)arg;
+}
+
+static void
+test_tortls_set_renegotiate_callback(void *ignored)
+{
+  (void)ignored;
+  tor_tls_t *tls;
+  const char *arg = "hello";
+
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+  tls->ssl = tor_malloc_zero(sizeof(SSL));
+
+  tor_tls_set_renegotiate_callback(tls, example_cb, (void*)arg);
+  tt_assert(tls->negotiated_callback == example_cb);
+  tt_assert(tls->callback_arg == arg);
+  tt_assert(!tls->got_renegotiate);
+
+  /* Assumes V2_HANDSHAKE_SERVER */
+  tt_assert(tls->ssl->info_callback == tor_tls_server_info_callback);
+
+  tor_tls_set_renegotiate_callback(tls, NULL, (void*)arg);
+  tt_assert(tls->ssl->info_callback == tor_tls_debug_state_callback);
+
+ done:
+  tor_free(tls->ssl);
+  tor_free(tls);
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+#ifndef OPENSSL_OPAQUE
+static SSL_CIPHER *fixed_cipher1 = NULL;
+static SSL_CIPHER *fixed_cipher2 = NULL;
+static const SSL_CIPHER *
+fake_get_cipher(unsigned ncipher)
+{
+
+  switch (ncipher) {
+  case 1:
+    return fixed_cipher1;
+  case 2:
+    return fixed_cipher2;
+  default:
+    return NULL;
+  }
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+#ifndef OPENSSL_OPAQUE
+static void
+test_tortls_find_cipher_by_id(void *ignored)
+{
+  (void)ignored;
+  int ret;
+  SSL *ssl;
+  SSL_CTX *ctx;
+  const SSL_METHOD *m = TLSv1_method();
+  SSL_METHOD *empty_method = tor_malloc_zero(sizeof(SSL_METHOD));
+
+  fixed_cipher1 = tor_malloc_zero(sizeof(SSL_CIPHER));
+  fixed_cipher2 = tor_malloc_zero(sizeof(SSL_CIPHER));
+  fixed_cipher2->id = 0xC00A;
+
+  library_init();
+
+  ctx = SSL_CTX_new(m);
+  ssl = SSL_new(ctx);
+
+  ret = find_cipher_by_id(ssl, NULL, 0xC00A);
+  tt_int_op(ret, OP_EQ, 1);
+
+  ret = find_cipher_by_id(ssl, m, 0xC00A);
+  tt_int_op(ret, OP_EQ, 1);
+
+  ret = find_cipher_by_id(ssl, m, 0xFFFF);
+  tt_int_op(ret, OP_EQ, 0);
+
+  ret = find_cipher_by_id(ssl, empty_method, 0xC00A);
+  tt_int_op(ret, OP_EQ, 1);
+
+  ret = find_cipher_by_id(ssl, empty_method, 0xFFFF);
+#ifdef HAVE_SSL_CIPHER_FIND
+  tt_int_op(ret, OP_EQ, 0);
+#else
+  tt_int_op(ret, OP_EQ, 1);
+#endif
+
+  empty_method->get_cipher = fake_get_cipher;
+  ret = find_cipher_by_id(ssl, empty_method, 0xC00A);
+  tt_int_op(ret, OP_EQ, 1);
+
+  empty_method->get_cipher = m->get_cipher;
+  empty_method->num_ciphers = m->num_ciphers;
+  ret = find_cipher_by_id(ssl, empty_method, 0xC00A);
+  tt_int_op(ret, OP_EQ, 1);
+
+  empty_method->get_cipher = fake_get_cipher;
+  empty_method->num_ciphers = m->num_ciphers;
+  ret = find_cipher_by_id(ssl, empty_method, 0xC00A);
+  tt_int_op(ret, OP_EQ, 1);
+
+  empty_method->num_ciphers = fake_num_ciphers;
+  ret = find_cipher_by_id(ssl, empty_method, 0xC00A);
+#ifdef HAVE_SSL_CIPHER_FIND
+  tt_int_op(ret, OP_EQ, 1);
+#else
+  tt_int_op(ret, OP_EQ, 0);
+#endif
+
+ done:
+  tor_free(empty_method);
+  SSL_free(ssl);
+  SSL_CTX_free(ctx);
+  tor_free(fixed_cipher1);
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+#ifndef OPENSSL_OPAQUE
+static void
+test_tortls_debug_state_callback(void *ignored)
+{
+  (void)ignored;
+  SSL *ssl;
+  char *buf = tor_malloc_zero(1000);
+  int n;
+
+  setup_capture_of_logs(LOG_DEBUG);
+
+  ssl = tor_malloc_zero(sizeof(SSL));
+
+  tor_tls_debug_state_callback(ssl, 32, 45);
+
+  n = tor_snprintf(buf, 1000, "SSL %p is now in state unknown"
+               " state [type=32,val=45].\n", ssl);
+  /* tor's snprintf returns -1 on error */
+  tt_int_op(n, OP_NE, -1);
+  expect_log_msg(buf);
+
+ done:
+  teardown_capture_of_logs();
+  tor_free(buf);
+  tor_free(ssl);
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+#ifndef OPENSSL_OPAQUE
+static void
+test_tortls_server_info_callback(void *ignored)
+{
+  (void)ignored;
+  tor_tls_t *tls;
+  SSL_CTX *ctx;
+  SSL *ssl;
+
+  library_init();
+
+  ctx = SSL_CTX_new(TLSv1_method());
+  ssl = SSL_new(ctx);
+
+  tor_tls_allocate_tor_tls_object_ex_data_index();
+
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+  tls->magic = TOR_TLS_MAGIC;
+  tls->ssl = ssl;
+
+  setup_full_capture_of_logs(LOG_WARN);
+  SSL_set_state(ssl, SSL3_ST_SW_SRVR_HELLO_A);
+  mock_clean_saved_logs();
+  tor_tls_server_info_callback(ssl, SSL_CB_ACCEPT_LOOP, 0);
+  expect_single_log_msg("Couldn't look up the tls for an SSL*. How odd!\n");
+
+  SSL_set_state(ssl, SSL3_ST_SW_SRVR_HELLO_B);
+  mock_clean_saved_logs();
+  tor_tls_server_info_callback(ssl, SSL_CB_ACCEPT_LOOP, 0);
+  expect_single_log_msg("Couldn't look up the tls for an SSL*. How odd!\n");
+
+  SSL_set_state(ssl, 99);
+  mock_clean_saved_logs();
+  tor_tls_server_info_callback(ssl, SSL_CB_ACCEPT_LOOP, 0);
+  expect_no_log_entry();
+  teardown_capture_of_logs();
+
+  SSL_set_ex_data(tls->ssl, tor_tls_object_ex_data_index, tls);
+  SSL_set_state(ssl, SSL3_ST_SW_SRVR_HELLO_B);
+  tls->negotiated_callback = 0;
+  //tls->server_handshake_count = 120;
+  tor_tls_server_info_callback(ssl, SSL_CB_ACCEPT_LOOP, 0);
+  //tt_int_op(tls->server_handshake_count, OP_EQ, 121);
+
+  //tls->server_handshake_count = 127;
+  tls->negotiated_callback = (void *)1;
+  tor_tls_server_info_callback(ssl, SSL_CB_ACCEPT_LOOP, 0);
+  //tt_int_op(tls->server_handshake_count, OP_EQ, 127);
+  tt_int_op(tls->got_renegotiate, OP_EQ, 1);
+
+  tls->ssl->session = SSL_SESSION_new();
+  tls->wasV2Handshake = 0;
+  tor_tls_server_info_callback(ssl, SSL_CB_ACCEPT_LOOP, 0);
+  tt_int_op(tls->wasV2Handshake, OP_EQ, 0);
+
+ done:
+  teardown_capture_of_logs();
+  SSL_free(ssl);
+  SSL_CTX_free(ctx);
+  tor_free(tls);
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+#ifndef OPENSSL_OPAQUE
+static int fixed_ssl_read_result_index;
+static int fixed_ssl_read_result[5];
+
+static int
+fixed_ssl_read(SSL *s, void *buf, int len)
+{
+  (void)s;
+  (void)buf;
+  (void)len;
+  return fixed_ssl_read_result[fixed_ssl_read_result_index++];
+}
+
+static int
+dummy_handshake_func(SSL *s)
+{
+  (void)s;
+  return 1;
+}
+
+static int negotiated_callback_called;
+
+static void
+negotiated_callback_setter(tor_tls_t *t, void *arg)
+{
+  (void)t;
+  (void)arg;
+  negotiated_callback_called++;
+}
+
+static void
+test_tortls_read(void *ignored)
+{
+  (void)ignored;
+  int ret;
+  tor_tls_t *tls;
+  char buf[100];
+  SSL_METHOD *method = give_me_a_test_method();
+  setup_capture_of_logs(LOG_WARN);
+
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+  tls->ssl = tor_malloc_zero(sizeof(SSL));
+  tls->state = TOR_TLS_ST_OPEN;
+
+  ret = tor_tls_read(tls, buf, 10);
+  tt_int_op(ret, OP_EQ, -9);
+
+  /* These tests assume that V2_HANDSHAKE_SERVER is set */
+  tls->ssl->handshake_func = dummy_handshake_func;
+  tls->ssl->method = method;
+  method->ssl_read = fixed_ssl_read;
+  fixed_ssl_read_result_index = 0;
+  fixed_ssl_read_result[0] = 42;
+  tls->state = TOR_TLS_ST_OPEN;
+  ERR_clear_error();
+  ret = tor_tls_read(tls, buf, 10);
+  tt_int_op(ret, OP_EQ, 42);
+
+  tls->state = TOR_TLS_ST_OPEN;
+  tls->got_renegotiate = 1;
+  fixed_ssl_read_result_index = 0;
+  ERR_clear_error();
+  ret = tor_tls_read(tls, buf, 10);
+  tt_int_op(tls->got_renegotiate, OP_EQ, 0);
+
+  tls->state = TOR_TLS_ST_OPEN;
+  tls->got_renegotiate = 1;
+  negotiated_callback_called = 0;
+  tls->negotiated_callback = negotiated_callback_setter;
+  fixed_ssl_read_result_index = 0;
+  ERR_clear_error();
+  ret = tor_tls_read(tls, buf, 10);
+  tt_int_op(negotiated_callback_called, OP_EQ, 1);
+
+#ifndef LIBRESSL_VERSION_NUMBER
+  fixed_ssl_read_result_index = 0;
+  fixed_ssl_read_result[0] = 0;
+  tls->ssl->version = SSL2_VERSION;
+  ERR_clear_error();
+  ret = tor_tls_read(tls, buf, 10);
+  tt_int_op(ret, OP_EQ, TOR_TLS_CLOSE);
+  tt_int_op(tls->state, OP_EQ, TOR_TLS_ST_CLOSED);
+#endif /* !defined(LIBRESSL_VERSION_NUMBER) */
+  // TODO: fill up
+
+ done:
+  teardown_capture_of_logs();
+  tor_free(tls->ssl);
+  tor_free(tls);
+  tor_free(method);
+}
+
+static int fixed_ssl_write_result;
+
+static int
+fixed_ssl_write(SSL *s, const void *buf, int len)
+{
+  (void)s;
+  (void)buf;
+  (void)len;
+  return fixed_ssl_write_result;
+}
+
+static void
+test_tortls_write(void *ignored)
+{
+  (void)ignored;
+  int ret;
+  tor_tls_t *tls;
+  SSL_METHOD *method = give_me_a_test_method();
+  char buf[100];
+  setup_capture_of_logs(LOG_WARN);
+
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+  tls->ssl = tor_malloc_zero(sizeof(SSL));
+  tls->state = TOR_TLS_ST_OPEN;
+
+  ret = tor_tls_write(tls, buf, 0);
+  tt_int_op(ret, OP_EQ, 0);
+
+  ret = tor_tls_write(tls, buf, 10);
+  tt_int_op(ret, OP_EQ, -9);
+
+  tls->ssl->method = method;
+  tls->wantwrite_n = 1;
+  ret = tor_tls_write(tls, buf, 10);
+  tt_int_op(tls->wantwrite_n, OP_EQ, 0);
+
+  method->ssl_write = fixed_ssl_write;
+  tls->ssl->handshake_func = dummy_handshake_func;
+  fixed_ssl_write_result = 1;
+  ERR_clear_error();
+  ret = tor_tls_write(tls, buf, 10);
+  tt_int_op(ret, OP_EQ, 1);
+
+  fixed_ssl_write_result = -1;
+  ERR_clear_error();
+  tls->ssl->rwstate = SSL_READING;
+  SSL_set_bio(tls->ssl, BIO_new(BIO_s_mem()), NULL);
+  SSL_get_rbio(tls->ssl)->flags = BIO_FLAGS_READ;
+  ret = tor_tls_write(tls, buf, 10);
+  tt_int_op(ret, OP_EQ, TOR_TLS_WANTREAD);
+
+  ERR_clear_error();
+  tls->ssl->rwstate = SSL_READING;
+  SSL_set_bio(tls->ssl, BIO_new(BIO_s_mem()), NULL);
+  SSL_get_rbio(tls->ssl)->flags = BIO_FLAGS_WRITE;
+  ret = tor_tls_write(tls, buf, 10);
+  tt_int_op(ret, OP_EQ, TOR_TLS_WANTWRITE);
+
+ done:
+  teardown_capture_of_logs();
+  BIO_free(tls->ssl->rbio);
+  tor_free(tls->ssl);
+  tor_free(tls);
+  tor_free(method);
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+#ifndef OPENSSL_OPAQUE
+static int fixed_ssl_accept_result;
+static int fixed_ssl_connect_result;
+
+static int
+setting_error_ssl_accept(SSL *ssl)
+{
+  (void)ssl;
+  ERR_put_error(ERR_LIB_BN, 2, -1, "somewhere.c", 99);
+  ERR_put_error(ERR_LIB_SYS, 2, -1, "somewhere.c", 99);
+  return fixed_ssl_accept_result;
+}
+
+static int
+setting_error_ssl_connect(SSL *ssl)
+{
+  (void)ssl;
+  ERR_put_error(ERR_LIB_BN, 2, -1, "somewhere.c", 99);
+  ERR_put_error(ERR_LIB_SYS, 2, -1, "somewhere.c", 99);
+  return fixed_ssl_connect_result;
+}
+
+static int
+fixed_ssl_accept(SSL *ssl)
+{
+  (void) ssl;
+  return fixed_ssl_accept_result;
+}
+
+static void
+test_tortls_handshake(void *ignored)
+{
+  (void)ignored;
+  int ret;
+  tor_tls_t *tls;
+  SSL_CTX *ctx;
+  SSL_METHOD *method = give_me_a_test_method();
+  setup_capture_of_logs(LOG_INFO);
+
+  SSL_library_init();
+  SSL_load_error_strings();
+
+  ctx = SSL_CTX_new(TLSv1_method());
+
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+  tls->ssl = SSL_new(ctx);
+  tls->state = TOR_TLS_ST_HANDSHAKE;
+
+  ret = tor_tls_handshake(tls);
+  tt_int_op(ret, OP_EQ, -9);
+
+  tls->isServer = 1;
+  tls->state = TOR_TLS_ST_HANDSHAKE;
+  ret = tor_tls_handshake(tls);
+  tt_int_op(ret, OP_EQ, -9);
+
+  tls->ssl->method = method;
+  method->ssl_accept = fixed_ssl_accept;
+  fixed_ssl_accept_result = 2;
+  ERR_clear_error();
+  tls->state = TOR_TLS_ST_HANDSHAKE;
+  ret = tor_tls_handshake(tls);
+  tt_int_op(tls->state, OP_EQ, TOR_TLS_ST_OPEN);
+
+  method->ssl_accept = setting_error_ssl_accept;
+  fixed_ssl_accept_result = 1;
+  ERR_clear_error();
+  mock_clean_saved_logs();
+  tls->state = TOR_TLS_ST_HANDSHAKE;
+  ret = tor_tls_handshake(tls);
+  tt_int_op(ret, OP_EQ, TOR_TLS_ERROR_MISC);
+  expect_log_entry();
+  /* This fails on jessie.  Investigate why! */
+#if 0
+  expect_log_msg("TLS error while handshaking: (null) (in bignum routines:"
+            "(null):SSLv3 write client hello B)\n");
+  expect_log_msg("TLS error while handshaking: (null) (in system library:"
+            "connect:SSLv3 write client hello B)\n");
+#endif /* 0 */
+  expect_log_severity(LOG_INFO);
+
+  tls->isServer = 0;
+  method->ssl_connect = setting_error_ssl_connect;
+  fixed_ssl_connect_result = 1;
+  ERR_clear_error();
+  mock_clean_saved_logs();
+  tls->state = TOR_TLS_ST_HANDSHAKE;
+  ret = tor_tls_handshake(tls);
+  tt_int_op(ret, OP_EQ, TOR_TLS_ERROR_MISC);
+  expect_log_entry();
+#if 0
+  /* See above */
+  expect_log_msg("TLS error while handshaking: "
+            "(null) (in bignum routines:(null):SSLv3 write client hello B)\n");
+  expect_log_msg("TLS error while handshaking: "
+            "(null) (in system library:connect:SSLv3 write client hello B)\n");
+#endif /* 0 */
+  expect_log_severity(LOG_WARN);
+
+ done:
+  teardown_capture_of_logs();
+  SSL_free(tls->ssl);
+  SSL_CTX_free(ctx);
+  tor_free(tls);
+  tor_free(method);
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+#ifndef OPENSSL_OPAQUE
+static void
+test_tortls_finish_handshake(void *ignored)
+{
+  (void)ignored;
+  int ret;
+  tor_tls_t *tls;
+  SSL_CTX *ctx;
+  SSL_METHOD *method = give_me_a_test_method();
+  SSL_library_init();
+  SSL_load_error_strings();
+
+  X509 *c1 = read_cert_from(validCertString);
+  SESS_CERT_local *sess = NULL;
+
+  ctx = SSL_CTX_new(method);
+
+  tls = tor_malloc_zero(sizeof(tor_tls_t));
+  tls->ssl = SSL_new(ctx);
+  tls->state = TOR_TLS_ST_OPEN;
+
+  ret = tor_tls_finish_handshake(tls);
+  tt_int_op(ret, OP_EQ, 0);
+
+  tls->isServer = 1;
+  tls->wasV2Handshake = 0;
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = tor_tls_finish_handshake(tls);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(tls->wasV2Handshake, OP_EQ, 1);
+  expect_single_log_msg_containing("For some reason, wasV2Handshake didn't "
+                                   "get set.");
+  teardown_capture_of_logs();
+
+  tls->wasV2Handshake = 1;
+  ret = tor_tls_finish_handshake(tls);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(tls->wasV2Handshake, OP_EQ, 1);
+
+  tls->wasV2Handshake = 1;
+  tls->ssl->session = SSL_SESSION_new();
+  ret = tor_tls_finish_handshake(tls);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(tls->wasV2Handshake, OP_EQ, 0);
+
+  tls->isServer = 0;
+
+  sess = tor_malloc_zero(sizeof(SESS_CERT_local));
+  tls->ssl->session->sess_cert = (void *)sess;
+  sess->cert_chain = sk_X509_new_null();
+  sk_X509_push(sess->cert_chain, c1);
+  tls->ssl->session->peer = c1;
+  tls->wasV2Handshake = 0;
+  ret = tor_tls_finish_handshake(tls);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(tls->wasV2Handshake, OP_EQ, 1);
+
+  method->num_ciphers = fake_num_ciphers;
+  ret = tor_tls_finish_handshake(tls);
+  tt_int_op(ret, OP_EQ, -9);
+
+ done:
+  if (sess)
+    sk_X509_free(sess->cert_chain);
+  if (tls->ssl && tls->ssl->session) {
+    tor_free(tls->ssl->session->sess_cert);
+  }
+  SSL_free(tls->ssl);
+  tor_free(tls);
+  SSL_CTX_free(ctx);
+  tor_free(method);
+  teardown_capture_of_logs();
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+static int fixed_crypto_pk_new_result_index;
+static crypto_pk_t *fixed_crypto_pk_new_result[5];
+
+static crypto_pk_t *
+fixed_crypto_pk_new(void)
+{
+  return fixed_crypto_pk_new_result[fixed_crypto_pk_new_result_index++];
+}
+
+#ifndef OPENSSL_OPAQUE
+static int fixed_crypto_pk_generate_key_with_bits_result_index;
+static int fixed_crypto_pk_generate_key_with_bits_result[5];
+static int fixed_tor_tls_create_certificate_result_index;
+static X509 *fixed_tor_tls_create_certificate_result[5];
+static int fixed_tor_x509_cert_new_result_index;
+static tor_x509_cert_t *fixed_tor_x509_cert_new_result[5];
+
+static int
+fixed_crypto_pk_generate_key_with_bits(crypto_pk_t *env, int bits)
+{
+  (void)env;
+  (void)bits;
+  return fixed_crypto_pk_generate_key_with_bits_result[
+                    fixed_crypto_pk_generate_key_with_bits_result_index++];
+}
+
+static X509 *
+fixed_tor_tls_create_certificate(crypto_pk_t *rsa,
+                                 crypto_pk_t *rsa_sign,
+                                 const char *cname,
+                                 const char *cname_sign,
+                                 unsigned int cert_lifetime)
+{
+  (void)rsa;
+  (void)rsa_sign;
+  (void)cname;
+  (void)cname_sign;
+  (void)cert_lifetime;
+  X509 *result = fixed_tor_tls_create_certificate_result[
+                             fixed_tor_tls_create_certificate_result_index++];
+  if (result)
+    return X509_dup(result);
+  else
+    return NULL;
+}
+
+static void
+fixed_tor_tls_create_certificate_results_free(void)
+{
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(fixed_tor_tls_create_certificate_result); ++i) {
+    X509 *cert = fixed_tor_tls_create_certificate_result[i];
+    if (cert)
+      X509_free(cert);
+    fixed_tor_tls_create_certificate_result[i] = NULL;
+  }
+}
+
+static void
+fixed_tor_x509_cert_new_results_free(void)
+{
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(fixed_tor_x509_cert_new_result); ++i) {
+    tor_x509_cert_free(fixed_tor_x509_cert_new_result[i]);
+  }
+}
+
+static tor_x509_cert_t *
+fixed_tor_x509_cert_new(tor_x509_cert_impl_t *x509_cert)
+{
+  (void) x509_cert;
+  tor_x509_cert_t **certp =
+    &fixed_tor_x509_cert_new_result[fixed_tor_x509_cert_new_result_index++];
+  tor_x509_cert_t *cert = *certp;
+  *certp = NULL;
+  return cert;
+}
+
+static void
+test_tortls_context_new(void *ignored)
+{
+  (void)ignored;
+  tor_tls_context_t *ret;
+  crypto_pk_t *pk1, *pk2, *pk3, *pk4, *pk5, *pk6, *pk7, *pk8, *pk9, *pk10,
+    *pk11, *pk12, *pk13, *pk14, *pk15, *pk16, *pk17, *pk18;
+
+  pk1 = crypto_pk_new();
+  pk2 = crypto_pk_new();
+  pk3 = crypto_pk_new();
+  pk4 = crypto_pk_new();
+  pk5 = crypto_pk_new();
+  pk6 = crypto_pk_new();
+  pk7 = crypto_pk_new();
+  pk8 = crypto_pk_new();
+  pk9 = crypto_pk_new();
+  pk10 = crypto_pk_new();
+  pk11 = crypto_pk_new();
+  pk12 = crypto_pk_new();
+  pk13 = crypto_pk_new();
+  pk14 = crypto_pk_new();
+  pk15 = crypto_pk_new();
+  pk16 = crypto_pk_new();
+  pk17 = crypto_pk_new();
+  pk18 = crypto_pk_new();
+
+  fixed_crypto_pk_new_result_index = 0;
+  fixed_crypto_pk_new_result[0] = NULL;
+  MOCK(crypto_pk_new, fixed_crypto_pk_new);
+  ret = tor_tls_context_new(NULL, 0, 0, 0);
+  tt_assert(!ret);
+
+  /* note: we already override this in testing_common.c, so we
+   * run this unit test in a subprocess. */
+  MOCK(crypto_pk_generate_key_with_bits,
+       fixed_crypto_pk_generate_key_with_bits);
+  fixed_crypto_pk_new_result_index = 0;
+  fixed_crypto_pk_new_result[0] = pk1;
+  fixed_crypto_pk_new_result[1] = NULL;
+  fixed_crypto_pk_generate_key_with_bits_result[0] = -1;
+  fixed_crypto_pk_generate_key_with_bits_result_index = 0;
+  ret = tor_tls_context_new(NULL, 0, 0, 0);
+  tt_assert(!ret);
+
+  fixed_crypto_pk_new_result_index = 0;
+  fixed_crypto_pk_new_result[0] = pk2;
+  fixed_crypto_pk_new_result[1] = NULL;
+  fixed_crypto_pk_generate_key_with_bits_result[0] = 0;
+  fixed_crypto_pk_generate_key_with_bits_result_index = 0;
+  ret = tor_tls_context_new(NULL, 0, 0, 0);
+  tt_assert(!ret);
+
+  fixed_crypto_pk_new_result_index = 0;
+  fixed_crypto_pk_new_result[0] = pk3;
+  fixed_crypto_pk_new_result[1] = pk4;
+  fixed_crypto_pk_new_result[2] = NULL;
+  fixed_crypto_pk_generate_key_with_bits_result[0] = 0;
+  fixed_crypto_pk_generate_key_with_bits_result[1] = -1;
+  fixed_crypto_pk_generate_key_with_bits_result_index = 0;
+  ret = tor_tls_context_new(NULL, 0, 0, 0);
+  tt_assert(!ret);
+
+  MOCK(tor_tls_create_certificate, fixed_tor_tls_create_certificate);
+
+  fixed_crypto_pk_new_result_index = 0;
+  fixed_crypto_pk_new_result[0] = pk5;
+  fixed_crypto_pk_new_result[1] = pk6;
+  fixed_crypto_pk_new_result[2] = NULL;
+  fixed_crypto_pk_generate_key_with_bits_result_index = 0;
+  fixed_crypto_pk_generate_key_with_bits_result[1] = 0;
+  fixed_tor_tls_create_certificate_result_index = 0;
+  fixed_tor_tls_create_certificate_result[0] = NULL;
+  fixed_tor_tls_create_certificate_result[1] = X509_new();
+  fixed_tor_tls_create_certificate_result[2] = X509_new();
+  ret = tor_tls_context_new(NULL, 0, 0, 0);
+  tt_assert(!ret);
+  fixed_tor_tls_create_certificate_results_free();
+
+  fixed_crypto_pk_new_result_index = 0;
+  fixed_crypto_pk_new_result[0] = pk7;
+  fixed_crypto_pk_new_result[1] = pk8;
+  fixed_crypto_pk_new_result[2] = NULL;
+  fixed_crypto_pk_generate_key_with_bits_result_index = 0;
+  fixed_tor_tls_create_certificate_result_index = 0;
+  fixed_tor_tls_create_certificate_result[0] = X509_new();
+  fixed_tor_tls_create_certificate_result[1] = NULL;
+  fixed_tor_tls_create_certificate_result[2] = X509_new();
+  ret = tor_tls_context_new(NULL, 0, 0, 0);
+  tt_assert(!ret);
+  fixed_tor_tls_create_certificate_results_free();
+
+  fixed_crypto_pk_new_result_index = 0;
+  fixed_crypto_pk_new_result[0] = pk9;
+  fixed_crypto_pk_new_result[1] = pk10;
+  fixed_crypto_pk_new_result[2] = NULL;
+  fixed_crypto_pk_generate_key_with_bits_result_index = 0;
+  fixed_tor_tls_create_certificate_result_index = 0;
+  fixed_tor_tls_create_certificate_result[0] = X509_new();
+  fixed_tor_tls_create_certificate_result[1] = X509_new();
+  fixed_tor_tls_create_certificate_result[2] = NULL;
+  ret = tor_tls_context_new(NULL, 0, 0, 0);
+  tt_assert(!ret);
+  fixed_tor_tls_create_certificate_results_free();
+
+  MOCK(tor_x509_cert_new, fixed_tor_x509_cert_new);
+  fixed_crypto_pk_new_result_index = 0;
+  fixed_crypto_pk_new_result[0] = pk11;
+  fixed_crypto_pk_new_result[1] = pk12;
+  fixed_crypto_pk_new_result[2] = NULL;
+  fixed_crypto_pk_generate_key_with_bits_result_index = 0;
+  fixed_tor_tls_create_certificate_result_index = 0;
+  fixed_tor_tls_create_certificate_result[0] = X509_new();
+  fixed_tor_tls_create_certificate_result[1] = X509_new();
+  fixed_tor_tls_create_certificate_result[2] = X509_new();
+  fixed_tor_x509_cert_new_result_index = 0;
+  fixed_tor_x509_cert_new_result[0] = NULL;
+  fixed_tor_x509_cert_new_result[1] = NULL;
+  fixed_tor_x509_cert_new_result[2] = NULL;
+  ret = tor_tls_context_new(NULL, 0, 0, 0);
+  tt_assert(!ret);
+  fixed_tor_tls_create_certificate_results_free();
+
+  fixed_crypto_pk_new_result_index = 0;
+  fixed_crypto_pk_new_result[0] = pk13;
+  fixed_crypto_pk_new_result[1] = pk14;
+  fixed_crypto_pk_new_result[2] = NULL;
+  fixed_crypto_pk_generate_key_with_bits_result_index = 0;
+  fixed_tor_tls_create_certificate_result_index = 0;
+  fixed_tor_tls_create_certificate_result[0] = X509_new();
+  fixed_tor_tls_create_certificate_result[1] = X509_new();
+  fixed_tor_tls_create_certificate_result[2] = X509_new();
+  fixed_tor_x509_cert_new_result_index = 0;
+  fixed_tor_x509_cert_new_result[0] = tor_malloc_zero(sizeof(tor_x509_cert_t));
+  fixed_tor_x509_cert_new_result[1] = NULL;
+  fixed_tor_x509_cert_new_result[2] = NULL;
+  ret = tor_tls_context_new(NULL, 0, 0, 0);
+  tt_assert(!ret);
+  fixed_tor_tls_create_certificate_results_free();
+  fixed_tor_x509_cert_new_results_free();
+
+  fixed_crypto_pk_new_result_index = 0;
+  fixed_crypto_pk_new_result[0] = pk15;
+  fixed_crypto_pk_new_result[1] = pk16;
+  fixed_crypto_pk_new_result[2] = NULL;
+  fixed_crypto_pk_generate_key_with_bits_result_index = 0;
+  fixed_tor_tls_create_certificate_result_index = 0;
+  fixed_tor_tls_create_certificate_result[0] = X509_new();
+  fixed_tor_tls_create_certificate_result[1] = X509_new();
+  fixed_tor_tls_create_certificate_result[2] = X509_new();
+  fixed_tor_x509_cert_new_result_index = 0;
+  fixed_tor_x509_cert_new_result[0] = tor_malloc_zero(sizeof(tor_x509_cert_t));
+  fixed_tor_x509_cert_new_result[1] = tor_malloc_zero(sizeof(tor_x509_cert_t));
+  fixed_tor_x509_cert_new_result[2] = NULL;
+  ret = tor_tls_context_new(NULL, 0, 0, 0);
+  tt_assert(!ret);
+  fixed_tor_tls_create_certificate_results_free();
+  fixed_tor_x509_cert_new_results_free();
+
+  fixed_crypto_pk_new_result_index = 0;
+  fixed_crypto_pk_new_result[0] = pk17;
+  fixed_crypto_pk_new_result[1] = pk18;
+  fixed_crypto_pk_new_result[2] = NULL;
+  fixed_crypto_pk_generate_key_with_bits_result_index = 0;
+  fixed_tor_tls_create_certificate_result_index = 0;
+  fixed_tor_tls_create_certificate_result[0] = X509_new();
+  fixed_tor_tls_create_certificate_result[1] = X509_new();
+  fixed_tor_tls_create_certificate_result[2] = X509_new();
+  fixed_tor_x509_cert_new_result_index = 0;
+  fixed_tor_x509_cert_new_result[0] = tor_malloc_zero(sizeof(tor_x509_cert_t));
+  fixed_tor_x509_cert_new_result[1] = tor_malloc_zero(sizeof(tor_x509_cert_t));
+  fixed_tor_x509_cert_new_result[2] = tor_malloc_zero(sizeof(tor_x509_cert_t));
+  ret = tor_tls_context_new(NULL, 0, 0, 0);
+  tt_assert(!ret);
+
+ done:
+  fixed_tor_tls_create_certificate_results_free();
+  fixed_tor_x509_cert_new_results_free();
+  UNMOCK(tor_x509_cert_new);
+  UNMOCK(tor_tls_create_certificate);
+  UNMOCK(crypto_pk_generate_key_with_bits);
+  UNMOCK(crypto_pk_new);
+}
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+static int fixed_crypto_pk_get_evp_pkey_result_index = 0;
+static EVP_PKEY *fixed_crypto_pk_get_evp_pkey_result[5];
+
+static EVP_PKEY *
+fixed_crypto_pk_get_evp_pkey_(crypto_pk_t *env, int private)
+{
+  (void) env;
+  (void) private;
+  return fixed_crypto_pk_get_evp_pkey_result[
+                               fixed_crypto_pk_get_evp_pkey_result_index++];
+}
+
+static void
+test_tortls_create_certificate(void *ignored)
+{
+  (void)ignored;
+  X509 *ret;
+  crypto_pk_t *pk1, *pk2;
+
+  pk1 = crypto_pk_new();
+  pk2 = crypto_pk_new();
+
+  MOCK(crypto_pk_get_openssl_evp_pkey_, fixed_crypto_pk_get_evp_pkey_);
+  fixed_crypto_pk_get_evp_pkey_result_index = 0;
+  fixed_crypto_pk_get_evp_pkey_result[0] = NULL;
+  ret = tor_tls_create_certificate(pk1, pk2, "hello", "hello2", 1);
+  tt_assert(!ret);
+
+  fixed_crypto_pk_get_evp_pkey_result_index = 0;
+  fixed_crypto_pk_get_evp_pkey_result[0] = EVP_PKEY_new();
+  fixed_crypto_pk_get_evp_pkey_result[1] = NULL;
+  ret = tor_tls_create_certificate(pk1, pk2, "hello", "hello2", 1);
+  tt_assert(!ret);
+
+  fixed_crypto_pk_get_evp_pkey_result_index = 0;
+  fixed_crypto_pk_get_evp_pkey_result[0] = EVP_PKEY_new();
+  fixed_crypto_pk_get_evp_pkey_result[1] = EVP_PKEY_new();
+  ret = tor_tls_create_certificate(pk1, pk2, "hello", "hello2", 1);
+  tt_assert(!ret);
+
+ done:
+  UNMOCK(crypto_pk_get_openssl_evp_pkey_);
+  crypto_pk_free(pk1);
+  crypto_pk_free(pk2);
+}
+
+static void
+test_tortls_cert_new(void *ignored)
+{
+  (void)ignored;
+  tor_x509_cert_t *ret;
+  X509 *cert = read_cert_from(validCertString);
+
+  ret = tor_x509_cert_new(NULL);
+  tt_assert(!ret);
+
+  ret = tor_x509_cert_new(cert);
+  tt_assert(ret);
+  tor_x509_cert_free(ret);
+  ret = NULL;
+
+#if 0
+  cert = read_cert_from(validCertString);
+  /* XXX this doesn't do what you think: it alters a copy of the pubkey. */
+  X509_get_pubkey(cert)->type = EVP_PKEY_DSA;
+  ret = tor_x509_cert_new(cert);
+  tt_assert(ret);
+#endif /* 0 */
+
+#ifndef OPENSSL_OPAQUE
+  cert = read_cert_from(validCertString);
+  X509_CINF_free(cert->cert_info);
+  cert->cert_info = NULL;
+  ret = tor_x509_cert_new(cert);
+  tt_assert(ret);
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+ done:
+  tor_x509_cert_free(ret);
+}
+
+static void
+test_tortls_cert_is_valid(void *ignored)
+{
+  (void)ignored;
+  int ret;
+  tor_x509_cert_t *cert = NULL, *scert = NULL;
+
+  scert = tor_malloc_zero(sizeof(tor_x509_cert_t));
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 0);
+  tt_int_op(ret, OP_EQ, 0);
+
+  cert = tor_malloc_zero(sizeof(tor_x509_cert_t));
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 0);
+  tt_int_op(ret, OP_EQ, 0);
+  tor_free(scert);
+  tor_free(cert);
+
+  cert = tor_x509_cert_new(read_cert_from(validCertString));
+  scert = tor_x509_cert_new(read_cert_from(caCertString));
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 0);
+  tt_int_op(ret, OP_EQ, 1);
+
+#ifndef OPENSSL_OPAQUE
+  tor_x509_cert_free(cert);
+  tor_x509_cert_free(scert);
+  cert = tor_x509_cert_new(read_cert_from(validCertString));
+  scert = tor_x509_cert_new(read_cert_from(caCertString));
+  ASN1_TIME_free(cert->cert->cert_info->validity->notAfter);
+  cert->cert->cert_info->validity->notAfter =
+    ASN1_TIME_set(NULL, time(NULL)-1000000);
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 0);
+  tt_int_op(ret, OP_EQ, 0);
+
+  tor_x509_cert_free(cert);
+  tor_x509_cert_free(scert);
+  cert = tor_x509_cert_new(read_cert_from(validCertString));
+  scert = tor_x509_cert_new(read_cert_from(caCertString));
+  X509_PUBKEY_free(cert->cert->cert_info->key);
+  cert->cert->cert_info->key = NULL;
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 1);
+  tt_int_op(ret, OP_EQ, 0);
+#endif /* !defined(OPENSSL_OPAQUE) */
+
+#if 0
+  tor_x509_cert_free(cert);
+  tor_x509_cert_free(scert);
+  cert = tor_x509_cert_new(read_cert_from(validCertString));
+  scert = tor_x509_cert_new(read_cert_from(caCertString));
+  /* This doesn't actually change the key in the cert. XXXXXX */
+  BN_one(EVP_PKEY_get1_RSA(X509_get_pubkey(cert->cert))->n);
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 1);
+  tt_int_op(ret, OP_EQ, 0);
+
+  tor_x509_cert_free(cert);
+  tor_x509_cert_free(scert);
+  cert = tor_x509_cert_new(read_cert_from(validCertString));
+  scert = tor_x509_cert_new(read_cert_from(caCertString));
+  /* This doesn't actually change the key in the cert. XXXXXX */
+  X509_get_pubkey(cert->cert)->type = EVP_PKEY_EC;
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 1);
+  tt_int_op(ret, OP_EQ, 0);
+
+  tor_x509_cert_free(cert);
+  tor_x509_cert_free(scert);
+  cert = tor_x509_cert_new(read_cert_from(validCertString));
+  scert = tor_x509_cert_new(read_cert_from(caCertString));
+  /* This doesn't actually change the key in the cert. XXXXXX */
+  X509_get_pubkey(cert->cert)->type = EVP_PKEY_EC;
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 0);
+  tt_int_op(ret, OP_EQ, 1);
+
+  tor_x509_cert_free(cert);
+  tor_x509_cert_free(scert);
+  cert = tor_x509_cert_new(read_cert_from(validCertString));
+  scert = tor_x509_cert_new(read_cert_from(caCertString));
+  /* This doesn't actually change the key in the cert. XXXXXX */
+  X509_get_pubkey(cert->cert)->type = EVP_PKEY_EC;
+  X509_get_pubkey(cert->cert)->ameth = NULL;
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 0);
+  tt_int_op(ret, OP_EQ, 0);
+#endif /* 0 */
+
+ done:
+  tor_x509_cert_free(cert);
+  tor_x509_cert_free(scert);
+}
+
+static void
+test_tortls_context_init_one(void *ignored)
+{
+  (void)ignored;
+  int ret;
+  tor_tls_context_t *old = NULL;
+
+  MOCK(crypto_pk_new, fixed_crypto_pk_new);
+
+  fixed_crypto_pk_new_result_index = 0;
+  fixed_crypto_pk_new_result[0] = NULL;
+  ret = tor_tls_context_init_one(&old, NULL, 0, 0, 0);
+  tt_int_op(ret, OP_EQ, -1);
+
+ done:
+  UNMOCK(crypto_pk_new);
+}
+
+#define LOCAL_TEST_CASE(name, flags)                    \
+  { #name, test_tortls_##name, (flags|TT_FORK), NULL, NULL }
+
+#ifdef OPENSSL_OPAQUE
+#define INTRUSIVE_TEST_CASE(name, flags)        \
+  { #name, NULL, TT_SKIP, NULL, NULL }
+#else
+#define INTRUSIVE_TEST_CASE(name, flags) LOCAL_TEST_CASE(name, flags)
+#endif /* defined(OPENSSL_OPAQUE) */
+
+struct testcase_t tortls_openssl_tests[] = {
+  LOCAL_TEST_CASE(tor_tls_new, TT_FORK),
+  LOCAL_TEST_CASE(get_state_description, TT_FORK),
+  LOCAL_TEST_CASE(get_by_ssl, TT_FORK),
+  LOCAL_TEST_CASE(allocate_tor_tls_object_ex_data_index, TT_FORK),
+  LOCAL_TEST_CASE(log_one_error, TT_FORK),
+  INTRUSIVE_TEST_CASE(get_error, TT_FORK),
+  LOCAL_TEST_CASE(always_accept_verify_cb, 0),
+  INTRUSIVE_TEST_CASE(x509_cert_free, 0),
+  LOCAL_TEST_CASE(cert_matches_key, 0),
+  INTRUSIVE_TEST_CASE(cert_get_key, 0),
+  LOCAL_TEST_CASE(get_my_client_auth_key, TT_FORK),
+  INTRUSIVE_TEST_CASE(get_ciphersuite_name, 0),
+  INTRUSIVE_TEST_CASE(classify_client_ciphers, 0),
+  LOCAL_TEST_CASE(client_is_using_v2_ciphers, 0),
+  INTRUSIVE_TEST_CASE(get_pending_bytes, 0),
+  INTRUSIVE_TEST_CASE(SSL_SESSION_get_master_key, 0),
+  INTRUSIVE_TEST_CASE(get_tlssecrets, 0),
+  INTRUSIVE_TEST_CASE(get_buffer_sizes, 0),
+  INTRUSIVE_TEST_CASE(try_to_extract_certs_from_tls, 0),
+  INTRUSIVE_TEST_CASE(get_peer_cert, 0),
+  INTRUSIVE_TEST_CASE(peer_has_cert, 0),
+  INTRUSIVE_TEST_CASE(finish_handshake, 0),
+  INTRUSIVE_TEST_CASE(handshake, 0),
+  INTRUSIVE_TEST_CASE(write, 0),
+  INTRUSIVE_TEST_CASE(read, 0),
+  INTRUSIVE_TEST_CASE(server_info_callback, 0),
+  LOCAL_TEST_CASE(get_write_overhead_ratio, TT_FORK),
+  LOCAL_TEST_CASE(is_server, 0),
+  INTRUSIVE_TEST_CASE(assert_renegotiation_unblocked, 0),
+  INTRUSIVE_TEST_CASE(block_renegotiation, 0),
+  INTRUSIVE_TEST_CASE(unblock_renegotiation, 0),
+  INTRUSIVE_TEST_CASE(set_renegotiate_callback, 0),
+  LOCAL_TEST_CASE(set_logged_address, 0),
+  INTRUSIVE_TEST_CASE(find_cipher_by_id, 0),
+  INTRUSIVE_TEST_CASE(session_secret_cb, 0),
+  INTRUSIVE_TEST_CASE(debug_state_callback, 0),
+  INTRUSIVE_TEST_CASE(context_new, TT_FORK /* redundant */),
+  LOCAL_TEST_CASE(create_certificate, 0),
+  LOCAL_TEST_CASE(cert_new, 0),
+  LOCAL_TEST_CASE(cert_is_valid, 0),
+  LOCAL_TEST_CASE(context_init_one, 0),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_util.c b/src/test/test_util.c
index dd122b250d..3a0eb15157 100644
--- a/src/test/test_util.c
+++ b/src/test/test_util.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -8,13 +8,37 @@
 #define COMPAT_TIME_PRIVATE
 #define CONTROL_PRIVATE
 #define UTIL_PRIVATE
-#include "or.h"
-#include "config.h"
-#include "control.h"
-#include "test.h"
-#include "memarea.h"
-#include "util_process.h"
-#include "log_test_helpers.h"
+#define UTIL_MALLOC_PRIVATE
+#define SOCKET_PRIVATE
+#define SUBPROCESS_PRIVATE
+#include "lib/testsupport/testsupport.h"
+#include "core/or/or.h"
+#include "lib/container/buffers.h"
+#include "app/config/config.h"
+#include "feature/control/control.h"
+#include "feature/client/transports.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "test/test.h"
+#include "lib/memarea/memarea.h"
+#include "lib/process/waitpid.h"
+#include "test/log_test_helpers.h"
+#include "lib/compress/compress.h"
+#include "lib/compress/compress_zstd.h"
+#include "lib/encoding/keyval.h"
+#include "lib/fdio/fdio.h"
+#include "lib/fs/winlib.h"
+#include "lib/process/env.h"
+#include "lib/process/pidfile.h"
+#include "lib/process/subprocess.h"
+#include "lib/intmath/weakrng.h"
+#include "lib/thread/numcpus.h"
+#include "lib/math/fp.h"
+#include "lib/math/laplace.h"
+#include "lib/meminfo/meminfo.h"
+#include "lib/time/tvdiff.h"
+#include "lib/encoding/confline.h"
+#include "lib/net/socketpair.h"
 
 #ifdef HAVE_PWD_H
 #include <pwd.h>
@@ -25,6 +49,16 @@
 #ifdef HAVE_UTIME_H
 #include <utime.h>
 #endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
 #ifdef _WIN32
 #include <tchar.h>
 #endif
@@ -69,7 +103,7 @@ test_util_read_until_eof_impl(const char *fname, size_t file_len,
   fd = open(fifo_name, O_RDONLY|O_BINARY);
   tt_int_op(fd, OP_GE, 0);
   str = read_file_to_str_until_eof(fd, read_limit, &sz);
-  tt_assert(str != NULL);
+  tt_ptr_op(str, OP_NE, NULL);
 
   if (read_limit < file_len)
     tt_int_op(sz, OP_EQ, read_limit);
@@ -366,7 +400,7 @@ test_util_time(void *arg)
    * calculations internally, then catches the overflow. */
 #define TV_SEC_MAX TIME_MAX
 #define TV_SEC_MIN TIME_MIN
-#endif
+#endif /* defined(_WIN32) */
 
 /* Assume tv_usec is an unsigned integer until proven otherwise */
 #define TV_USEC_MAX UINT_MAX
@@ -634,13 +668,16 @@ test_util_time(void *arg)
    * time_t */
   a_time.tm_year = 2039-1900;
 #if SIZEOF_TIME_T == 4
+  setup_full_capture_of_logs(LOG_WARN);
   tt_int_op((time_t) -1,OP_EQ, tor_timegm(&a_time));
+  expect_single_log_msg_containing("Result does not fit in tor_timegm");
+  teardown_capture_of_logs();
 #elif SIZEOF_TIME_T == 8
   t_res = 2178252895UL;
   tt_int_op(t_res, OP_EQ, tor_timegm(&a_time));
   tor_gmtime_r(&t_res, &b_time);
   TM_EQUAL(a_time, b_time);
-#endif
+#endif /* SIZEOF_TIME_T == 4 || ... */
 
   /* Test tor_timegm out of range */
 
@@ -650,10 +687,15 @@ test_util_time(void *arg)
     setup_full_capture_of_logs(LOG_WARN);                       \
   } while (0)
 #define CHECK_TIMEGM_WARNING(msg) do { \
-    expect_log_msg_containing(msg);                                     \
-    tt_int_op(1, OP_EQ, smartlist_len(mock_saved_logs()));              \
+    expect_single_log_msg_containing(msg);                              \
     teardown_capture_of_logs();                                         \
   } while (0)
+#define CHECK_POSSIBLE_EINVAL() do {                            \
+    if (mock_saved_log_n_entries()) {                           \
+      expect_single_log_msg_containing("Invalid argument");     \
+    }                                                           \
+    teardown_capture_of_logs();                                 \
+  } while (0)
 
 #define CHECK_TIMEGM_ARG_OUT_OF_RANGE(msg) \
     CHECK_TIMEGM_WARNING("Out-of-range argument to tor_timegm")
@@ -688,7 +730,7 @@ test_util_time(void *arg)
     CAPTURE();
     tt_int_op((time_t) -1,OP_EQ, tor_timegm(&a_time));
     CHECK_TIMEGM_ARG_OUT_OF_RANGE();
-#endif
+#endif /* SIZEOF_INT == 4 || SIZEOF_INT == 8 */
 
 #if SIZEOF_INT == 8
     a_time.tm_year = -1*(1 << 48);
@@ -700,7 +742,7 @@ test_util_time(void *arg)
      * a "correct" retrospective gregorian negative year value,
      * which I'm pretty sure is:
      * -1*(2^63)/60/60/24*2000/730485 + 1970 = -292277022657
-     * 730485 is the number of days in two millenia, including leap days */
+     * 730485 is the number of days in two millennia, including leap days */
     a_time.tm_year = -292277022657-1900;
     CAPTURE();
     tt_int_op((time_t) -1,OP_EQ, tor_timegm(&a_time));
@@ -710,7 +752,7 @@ test_util_time(void *arg)
     CAPTURE();
     tt_int_op((time_t) -1,OP_EQ, tor_timegm(&a_time));
     CHECK_TIMEGM_ARG_OUT_OF_RANGE();
-#endif
+#endif /* SIZEOF_INT == 8 */
 
   /* Wrong year >= INT32_MAX - 1900 */
 #if SIZEOF_INT == 4 || SIZEOF_INT == 8
@@ -723,7 +765,7 @@ test_util_time(void *arg)
     CAPTURE();
     tt_int_op((time_t) -1,OP_EQ, tor_timegm(&a_time));
     CHECK_TIMEGM_ARG_OUT_OF_RANGE();
-#endif
+#endif /* SIZEOF_INT == 4 || SIZEOF_INT == 8 */
 
 #if SIZEOF_INT == 8
     /* one of the largest tm_year values my 64 bit system supports */
@@ -736,7 +778,7 @@ test_util_time(void *arg)
      * a "correct" proleptic gregorian year value,
      * which I'm pretty sure is:
      * (2^63-1)/60/60/24*2000/730485 + 1970 = 292277026596
-     * 730485 is the number of days in two millenia, including leap days */
+     * 730485 is the number of days in two millennia, including leap days */
     a_time.tm_year = 292277026596-1900;
     CAPTURE();
     tt_int_op((time_t) -1,OP_EQ, tor_timegm(&a_time));
@@ -751,7 +793,7 @@ test_util_time(void *arg)
     CAPTURE();
     tt_int_op((time_t) -1,OP_EQ, tor_timegm(&a_time));
     CHECK_TIMEGM_ARG_OUT_OF_RANGE();
-#endif
+#endif /* SIZEOF_INT == 8 */
 
   /* month */
   a_time.tm_year = 2007-1900;  /* restore valid year */
@@ -843,18 +885,24 @@ test_util_time(void *arg)
    * depending on whether the implementation of the system gmtime(_r)
    * sets struct tm (1) or not (1970) */
   t_res = -1;
+  CAPTURE();
   tor_gmtime_r(&t_res, &b_time);
+  CHECK_POSSIBLE_EINVAL();
   tt_assert(b_time.tm_year == (1970-1900) ||
             b_time.tm_year == (1969-1900));
 
   if (sizeof(time_t) == 4 || sizeof(time_t) == 8) {
     t_res = -1*(1 << 30);
+    CAPTURE();
     tor_gmtime_r(&t_res, &b_time);
+    CHECK_POSSIBLE_EINVAL();
     tt_assert(b_time.tm_year == (1970-1900) ||
               b_time.tm_year == (1935-1900));
 
     t_res = INT32_MIN;
+    CAPTURE();
     tor_gmtime_r(&t_res, &b_time);
+    CHECK_POSSIBLE_EINVAL();
     tt_assert(b_time.tm_year == (1970-1900) ||
               b_time.tm_year == (1901-1900));
   }
@@ -864,7 +912,9 @@ test_util_time(void *arg)
     /* one of the smallest tm_year values my 64 bit system supports:
      * b_time.tm_year == (-292275055LL-1900LL) without clamping */
     t_res = -9223372036854775LL;
+    CAPTURE();
     tor_gmtime_r(&t_res, &b_time);
+    CHECK_POSSIBLE_EINVAL();
     tt_assert(b_time.tm_year == (1970-1900) ||
               b_time.tm_year == (1-1900));
 
@@ -872,7 +922,7 @@ test_util_time(void *arg)
      * a "correct" retrospective gregorian negative year value,
      * which I'm pretty sure is:
      * -1*(2^63)/60/60/24*2000/730485 + 1970 = -292277022657
-     * 730485 is the number of days in two millenia, including leap days
+     * 730485 is the number of days in two millennia, including leap days
      * (int64_t)b_time.tm_year == (-292277022657LL-1900LL) without clamping */
     t_res = INT64_MIN;
     CAPTURE();
@@ -887,7 +937,35 @@ test_util_time(void *arg)
       teardown_capture_of_logs();
     }
   }
-#endif
+  {
+    /* As above, but with localtime. */
+    t_res = -9223372036854775LL;
+    CAPTURE();
+    tor_localtime_r(&t_res, &b_time);
+    CHECK_POSSIBLE_EINVAL();
+    tt_assert(b_time.tm_year == (1970-1900) ||
+              b_time.tm_year == (1-1900));
+
+    /* while unlikely, the system's gmtime(_r) could return
+     * a "correct" retrospective gregorian negative year value,
+     * which I'm pretty sure is:
+     * -1*(2^63)/60/60/24*2000/730485 + 1970 = -292277022657
+     * 730485 is the number of days in two millennia, including leap days
+     * (int64_t)b_time.tm_year == (-292277022657LL-1900LL) without clamping */
+    t_res = INT64_MIN;
+    CAPTURE();
+    tor_localtime_r(&t_res, &b_time);
+    if (! (b_time.tm_year == (1970-1900) ||
+           b_time.tm_year == (1-1900))) {
+      tt_int_op(b_time.tm_year, OP_EQ, 1970-1900);
+    }
+    if (b_time.tm_year != 1970-1900) {
+      CHECK_TIMEGM_WARNING("Rounding up to ");
+    } else {
+      teardown_capture_of_logs();
+    }
+  }
+#endif /* SIZEOF_TIME_T == 8 */
 
   /* time_t >= INT_MAX yields a year clamped to 2037 or 9999,
    * depending on whether the implementation of the system gmtime(_r)
@@ -903,14 +981,27 @@ test_util_time(void *arg)
     tt_assert(b_time.tm_year == (2037-1900) ||
               b_time.tm_year == (2038-1900));
   }
-#endif
+  {
+    /* as above but with localtime. */
+    t_res = 3*(1 << 29);
+    tor_localtime_r(&t_res, &b_time);
+    tt_assert(b_time.tm_year == (2021-1900));
+
+    t_res = INT32_MAX;
+    tor_localtime_r(&t_res, &b_time);
+    tt_assert(b_time.tm_year == (2037-1900) ||
+              b_time.tm_year == (2038-1900));
+  }
+#endif /* SIZEOF_TIME_T == 4 || SIZEOF_TIME_T == 8 */
 
 #if SIZEOF_TIME_T == 8
   {
     /* one of the largest tm_year values my 64 bit system supports:
      * b_time.tm_year == (292278994L-1900L) without clamping */
     t_res = 9223372036854775LL;
+    CAPTURE();
     tor_gmtime_r(&t_res, &b_time);
+    CHECK_POSSIBLE_EINVAL();
     tt_assert(b_time.tm_year == (2037-1900) ||
               b_time.tm_year == (9999-1900));
 
@@ -918,7 +1009,7 @@ test_util_time(void *arg)
      * a "correct" proleptic gregorian year value,
      * which I'm pretty sure is:
      * (2^63-1)/60/60/24*2000/730485 + 1970 = 292277026596
-     * 730485 is the number of days in two millenia, including leap days
+     * 730485 is the number of days in two millennia, including leap days
      * (int64_t)b_time.tm_year == (292277026596L-1900L) without clamping */
     t_res = INT64_MAX;
     CAPTURE();
@@ -928,7 +1019,30 @@ test_util_time(void *arg)
     tt_assert(b_time.tm_year == (2037-1900) ||
               b_time.tm_year == (9999-1900));
   }
-#endif
+  {
+    /* As above but with localtime. */
+    t_res = 9223372036854775LL;
+    CAPTURE();
+    tor_localtime_r(&t_res, &b_time);
+    CHECK_POSSIBLE_EINVAL();
+    tt_assert(b_time.tm_year == (2037-1900) ||
+              b_time.tm_year == (9999-1900));
+
+    /* while unlikely, the system's gmtime(_r) could return
+     * a "correct" proleptic gregorian year value,
+     * which I'm pretty sure is:
+     * (2^63-1)/60/60/24*2000/730485 + 1970 = 292277026596
+     * 730485 is the number of days in two millennia, including leap days
+     * (int64_t)b_time.tm_year == (292277026596L-1900L) without clamping */
+    t_res = INT64_MAX;
+    CAPTURE();
+    tor_localtime_r(&t_res, &b_time);
+    CHECK_TIMEGM_WARNING("Rounding down to ");
+
+    tt_assert(b_time.tm_year == (2037-1900) ||
+              b_time.tm_year == (9999-1900));
+  }
+#endif /* SIZEOF_TIME_T == 8 */
 
   /* Test {format,parse}_rfc1123_time */
 
@@ -953,7 +1067,10 @@ test_util_time(void *arg)
 
   /* This value is out of range with 32 bit time_t, but in range for 64 bit
    * time_t */
+  CAPTURE();
   format_rfc1123_time(timestr, (time_t)2150000000UL);
+  CHECK_POSSIBLE_EINVAL();
+
 #if SIZEOF_TIME_T == 4
 #if 0
   /* Wrapping around will have made it this. */
@@ -964,7 +1081,9 @@ test_util_time(void *arg)
   strlcpy(timestr, "Wed, 17 Feb 2038 06:13:20 GMT", sizeof(timestr));
 
   t_res = 0;
+  CAPTURE();
   i = parse_rfc1123_time(timestr, &t_res);
+  CHECK_TIMEGM_WARNING("does not fit in tor_timegm");
   tt_int_op(-1,OP_EQ, i);
 #elif SIZEOF_TIME_T == 8
   tt_str_op("Wed, 17 Feb 2038 06:13:20 GMT",OP_EQ, timestr);
@@ -973,7 +1092,7 @@ test_util_time(void *arg)
   i = parse_rfc1123_time(timestr, &t_res);
   tt_int_op(0,OP_EQ, i);
   tt_int_op(t_res,OP_EQ, (time_t)2150000000UL);
-#endif
+#endif /* SIZEOF_TIME_T == 4 || ... */
 
   /* The timezone doesn't matter */
   t_res = 0;
@@ -1039,13 +1158,16 @@ test_util_time(void *arg)
   /* This value is out of range with 32 bit time_t, but in range for 64 bit
    * time_t */
   t_res = 0;
-  i = parse_iso_time("2038-02-17 06:13:20", &t_res);
 #if SIZEOF_TIME_T == 4
+  CAPTURE();
+  i = parse_iso_time("2038-02-17 06:13:20", &t_res);
   tt_int_op(-1,OP_EQ, i);
+  CHECK_TIMEGM_WARNING("does not fit in tor_timegm");
 #elif SIZEOF_TIME_T == 8
+  i = parse_iso_time("2038-02-17 06:13:20", &t_res);
   tt_int_op(0,OP_EQ, i);
   tt_int_op(t_res,OP_EQ, (time_t)2150000000UL);
-#endif
+#endif /* SIZEOF_TIME_T == 4 || ... */
 
   tt_int_op(-1,OP_EQ, parse_iso_time("2004-08-zz 99-99x99", &t_res));
   tt_int_op(-1,OP_EQ, parse_iso_time("2011-03-32 00:00:00", &t_res));
@@ -1059,6 +1181,23 @@ test_util_time(void *arg)
   tt_int_op(-1,OP_EQ, parse_iso_time("2004-08-04 00:48:22.100", &t_res));
   tt_int_op(-1,OP_EQ, parse_iso_time("2004-08-04 00:48:22XYZ", &t_res));
 
+  /* but... that _is_ acceptable if we aren't being strict. */
+  t_res = 0;
+  i = parse_iso_time_("2004-08-04 00:48:22XYZ", &t_res, 0, 0);
+  tt_int_op(0,OP_EQ, i);
+  tt_int_op(t_res,OP_EQ, (time_t)1091580502UL);
+
+  /* try nospace variant. */
+  t_res = 0;
+  i = parse_iso_time_nospace("2004-08-04T00:48:22", &t_res);
+  tt_int_op(0,OP_EQ, i);
+  tt_int_op(t_res,OP_EQ, (time_t)1091580502UL);
+
+  tt_int_op(-1,OP_EQ, parse_iso_time("2004-08-04T00:48:22", &t_res));
+  tt_int_op(-1,OP_EQ, parse_iso_time_nospace("2004-08-04 00:48:22", &t_res));
+  tt_int_op(-1,OP_EQ, parse_iso_time("2004-08-04x00:48:22", &t_res));
+  tt_int_op(-1,OP_EQ, parse_iso_time_nospace("2004-08-04x00:48:22", &t_res));
+
   /* Test tor_gettimeofday */
 
   end.tv_sec = 4;
@@ -1100,7 +1239,9 @@ test_util_time(void *arg)
   /* This value is out of range with 32 bit time_t, but in range for 64 bit
    * time_t */
   tv.tv_sec = (time_t)2150000000UL;
+  CAPTURE();
   format_iso_time(timestr, (time_t)tv.tv_sec);
+  CHECK_POSSIBLE_EINVAL();
 #if SIZEOF_TIME_T == 4
   /* format_iso_time should indicate failure on overflow, but it doesn't yet.
    * Hopefully #18480 will improve the failure semantics in this case.
@@ -1111,10 +1252,11 @@ test_util_time(void *arg)
   /* This SHOULD work on windows too; see bug #18665 */
   tt_str_op("2038-02-17 06:13:20",OP_EQ, timestr);
 #endif
-#endif
+#endif /* SIZEOF_TIME_T == 4 || ... */
 
 #undef CAPTURE
 #undef CHECK_TIMEGM_ARG_OUT_OF_RANGE
+#undef CHECK_POSSIBLE_EINVAL
 
  done:
   teardown_capture_of_logs();
@@ -1200,13 +1342,16 @@ test_util_parse_http_time(void *arg)
 #if SIZEOF_TIME_T == 4
   /* parse_http_time should indicate failure on overflow, but it doesn't yet.
    * Hopefully #18480 will improve the failure semantics in this case. */
+  setup_full_capture_of_logs(LOG_WARN);
   tt_int_op(0,OP_EQ,parse_http_time("Wed, 17 Feb 2038 06:13:20 GMT", &a_time));
   tt_int_op((time_t)-1,OP_EQ, tor_timegm(&a_time));
+  expect_single_log_msg_containing("does not fit in tor_timegm");
+  teardown_capture_of_logs();
 #elif SIZEOF_TIME_T == 8
   tt_int_op(0,OP_EQ,parse_http_time("Wed, 17 Feb 2038 06:13:20 GMT", &a_time));
   tt_int_op((time_t)2150000000UL,OP_EQ, tor_timegm(&a_time));
   T("2038-02-17 06:13:20");
-#endif
+#endif /* SIZEOF_TIME_T == 4 || ... */
 
   tt_int_op(-1,OP_EQ, parse_http_time("2004-08-zz 99-99x99 GMT", &a_time));
   tt_int_op(-1,OP_EQ, parse_http_time("2011-03-32 00:00:00 GMT", &a_time));
@@ -1219,7 +1364,7 @@ test_util_parse_http_time(void *arg)
 
 #undef T
  done:
-  ;
+  teardown_capture_of_logs();
 }
 
 static void
@@ -1453,7 +1598,7 @@ test_util_config_line_comment_character(void *arg)
   tor_free(k); tor_free(v);
 
   test_streq(str, "");
-#endif
+#endif /* 0 */
 
  done:
   tor_free(k);
@@ -1584,7 +1729,7 @@ test_util_config_line_escaped_content(void *arg)
   str = parse_config_line_from_str_verbose(str, &k, &v, NULL);
   tt_ptr_op(str, OP_EQ, NULL);
   tor_free(k); tor_free(v);
-#endif
+#endif /* 0 */
 
   str = buf6;
 
@@ -1656,14 +1801,14 @@ test_util_config_line_crlf(void *arg)
   tt_assert(str);
   tt_str_op(k,OP_EQ,"Hello");
   tt_str_op(v,OP_EQ,"world");
-  tt_assert(!err);
+  tt_ptr_op(err, OP_EQ, NULL);
   tor_free(k); tor_free(v);
 
   str = parse_config_line_from_str_verbose(str, &k, &v, &err);
   tt_assert(str);
   tt_str_op(k,OP_EQ,"Hello");
   tt_str_op(v,OP_EQ,"nice big world");
-  tt_assert(!err);
+  tt_ptr_op(err, OP_EQ, NULL);
   tor_free(k); tor_free(v);
   tt_str_op(str,OP_EQ, "");
 
@@ -1768,7 +1913,7 @@ test_util_expand_filename(void *arg)
  done:
   tor_free(str);
 }
-#endif
+#endif /* !defined(_WIN32) */
 
 /** Test tor_escape_str_for_pt_args(). */
 static void
@@ -1871,8 +2016,8 @@ test_util_strmisc(void *arg)
   tor_snprintf(buf, 10, "abcdef");
   tt_int_op(0,OP_EQ, buf[6]);
   /* uint64 */
-  tor_snprintf(buf, sizeof(buf), "x!"U64_FORMAT"!x",
-               U64_PRINTF_ARG(U64_LITERAL(12345678901)));
+  tor_snprintf(buf, sizeof(buf), "x!%"PRIu64"!x",
+               (UINT64_C(12345678901)));
   tt_str_op("x!12345678901!x",OP_EQ, buf);
 
   /* Test str{,case}cmpstart */
@@ -1923,7 +2068,7 @@ test_util_strmisc(void *arg)
   tt_assert(!tor_mem_is_zero(buf, 10));
 
   /* Test 'escaped' */
-  tt_assert(NULL == escaped(NULL));
+  tt_ptr_op(escaped(NULL), OP_EQ, NULL);
   tt_str_op("\"\"",OP_EQ, escaped(""));
   tt_str_op("\"abcd\"",OP_EQ, escaped("abcd"));
   tt_str_op("\"\\\\ \\n\\r\\t\\\"\\'\"",OP_EQ, escaped("\\ \n\r\t\"'"));
@@ -1981,23 +2126,23 @@ test_util_strmisc(void *arg)
   /* Test memmem and memstr */
   {
     const char *haystack = "abcde";
-    tt_assert(!tor_memmem(haystack, 5, "ef", 2));
+    tt_ptr_op(tor_memmem(haystack, 5, "ef", 2), OP_EQ, NULL);
     tt_ptr_op(tor_memmem(haystack, 5, "cd", 2),OP_EQ, haystack + 2);
     tt_ptr_op(tor_memmem(haystack, 5, "cde", 3),OP_EQ, haystack + 2);
-    tt_assert(!tor_memmem(haystack, 4, "cde", 3));
+    tt_ptr_op(tor_memmem(haystack, 4, "cde", 3), OP_EQ, NULL);
     haystack = "ababcad";
     tt_ptr_op(tor_memmem(haystack, 7, "abc", 3),OP_EQ, haystack + 2);
     tt_ptr_op(tor_memmem(haystack, 7, "ad", 2),OP_EQ, haystack + 5);
     tt_ptr_op(tor_memmem(haystack, 7, "cad", 3),OP_EQ, haystack + 4);
-    tt_assert(!tor_memmem(haystack, 7, "dadad", 5));
-    tt_assert(!tor_memmem(haystack, 7, "abcdefghij", 10));
+    tt_ptr_op(tor_memmem(haystack, 7, "dadad", 5), OP_EQ, NULL);
+    tt_ptr_op(tor_memmem(haystack, 7, "abcdefghij", 10), OP_EQ, NULL);
     /* memstr */
     tt_ptr_op(tor_memstr(haystack, 7, "abc"),OP_EQ, haystack + 2);
     tt_ptr_op(tor_memstr(haystack, 7, "cad"),OP_EQ, haystack + 4);
-    tt_assert(!tor_memstr(haystack, 6, "cad"));
-    tt_assert(!tor_memstr(haystack, 7, "cadd"));
-    tt_assert(!tor_memstr(haystack, 7, "fe"));
-    tt_assert(!tor_memstr(haystack, 7, "ababcade"));
+    tt_ptr_op(tor_memstr(haystack, 6, "cad"), OP_EQ, NULL);
+    tt_ptr_op(tor_memstr(haystack, 7, "cadd"), OP_EQ, NULL);
+    tt_ptr_op(tor_memstr(haystack, 7, "fe"), OP_EQ, NULL);
+    tt_ptr_op(tor_memstr(haystack, 7, "ababcade"), OP_EQ, NULL);
   }
 
   /* Test hex_str */
@@ -2085,20 +2230,10 @@ test_util_parse_integer(void *arg)
   tt_int_op(1,OP_EQ, i);
   tt_str_op(cp,OP_EQ, " plus garbage");
   /* Illogical min max */
-  tor_capture_bugs_(1);
   tt_int_op(0L,OP_EQ,  tor_parse_long("10",10,50,4,&i,NULL));
   tt_int_op(0,OP_EQ, i);
-  tt_int_op(1, OP_EQ, smartlist_len(tor_get_captured_bug_log_()));
-  tt_str_op("!(max < min)", OP_EQ,
-            smartlist_get(tor_get_captured_bug_log_(), 0));
-  tor_end_capture_bugs_();
-  tor_capture_bugs_(1);
   tt_int_op(0L,OP_EQ,   tor_parse_long("-50",10,100,-100,&i,NULL));
   tt_int_op(0,OP_EQ, i);
-  tt_int_op(1, OP_EQ, smartlist_len(tor_get_captured_bug_log_()));
-  tt_str_op("!(max < min)", OP_EQ,
-            smartlist_get(tor_get_captured_bug_log_(), 0));
-  tor_end_capture_bugs_();
   /* Out of bounds */
   tt_int_op(0L,OP_EQ,  tor_parse_long("10",10,50,100,&i,NULL));
   tt_int_op(0,OP_EQ, i);
@@ -2107,9 +2242,9 @@ test_util_parse_integer(void *arg)
   /* Base different than 10 */
   tt_int_op(2L,OP_EQ,   tor_parse_long("10",2,0,100,NULL,NULL));
   tt_int_op(0L,OP_EQ,   tor_parse_long("2",2,0,100,NULL,NULL));
-  tt_int_op(0L,OP_EQ,   tor_parse_long("10",-2,0,100,NULL,NULL));
   tt_int_op(68284L,OP_EQ, tor_parse_long("10abc",16,0,70000,NULL,NULL));
   tt_int_op(68284L,OP_EQ, tor_parse_long("10ABC",16,0,70000,NULL,NULL));
+  tt_int_op(0L,OP_EQ,   tor_parse_long("10",-2,0,100,NULL,NULL));
   tt_int_op(0,OP_EQ, tor_parse_long("10ABC",-1,0,70000,&i,NULL));
   tt_int_op(i,OP_EQ, 0);
 
@@ -2129,17 +2264,17 @@ test_util_parse_integer(void *arg)
   tt_int_op(0,OP_EQ, i);
 
   /* Test parse_uint64 */
-  tt_assert(U64_LITERAL(10) == tor_parse_uint64("10 x",10,0,100, &i, &cp));
+  tt_assert(UINT64_C(10) == tor_parse_uint64("10 x",10,0,100, &i, &cp));
   tt_int_op(1,OP_EQ, i);
   tt_str_op(cp,OP_EQ, " x");
-  tt_assert(U64_LITERAL(12345678901) ==
+  tt_assert(UINT64_C(12345678901) ==
               tor_parse_uint64("12345678901",10,0,UINT64_MAX, &i, &cp));
   tt_int_op(1,OP_EQ, i);
   tt_str_op(cp,OP_EQ, "");
-  tt_assert(U64_LITERAL(0) ==
+  tt_assert(UINT64_C(0) ==
               tor_parse_uint64("12345678901",10,500,INT32_MAX, &i, &cp));
   tt_int_op(0,OP_EQ, i);
-  tt_assert(U64_LITERAL(0) ==
+  tt_assert(UINT64_C(0) ==
               tor_parse_uint64("123",-1,0,INT32_MAX, &i, &cp));
   tt_int_op(0,OP_EQ, i);
 
@@ -2147,19 +2282,22 @@ test_util_parse_integer(void *arg)
   /* Test parse_double */
   double d = tor_parse_double("10", 0, (double)UINT64_MAX,&i,NULL);
   tt_int_op(1,OP_EQ, i);
-  tt_assert(DBL_TO_U64(d) == 10);
+  tt_assert(((uint64_t)d) == 10);
   d = tor_parse_double("0", 0, (double)UINT64_MAX,&i,NULL);
   tt_int_op(1,OP_EQ, i);
-  tt_assert(DBL_TO_U64(d) == 0);
+  tt_assert(((uint64_t)d) == 0);
   d = tor_parse_double(" ", 0, (double)UINT64_MAX,&i,NULL);
+  tt_double_op(fabs(d), OP_LT, 1e-10);
   tt_int_op(0,OP_EQ, i);
   d = tor_parse_double(".0a", 0, (double)UINT64_MAX,&i,NULL);
+  tt_double_op(fabs(d), OP_LT, 1e-10);
   tt_int_op(0,OP_EQ, i);
   d = tor_parse_double(".0a", 0, (double)UINT64_MAX,&i,&cp);
+  tt_double_op(fabs(d), OP_LT, 1e-10);
   tt_int_op(1,OP_EQ, i);
   d = tor_parse_double("-.0", 0, (double)UINT64_MAX,&i,NULL);
   tt_int_op(1,OP_EQ, i);
-  tt_assert(DBL_TO_U64(d) == 0);
+  tt_assert(((uint64_t)d) == 0);
   d = tor_parse_double("-10", -100.0, 100.0,&i,NULL);
   tt_int_op(1,OP_EQ, i);
   tt_double_op(fabs(d - -10.0),OP_LT, 1E-12);
@@ -2177,12 +2315,12 @@ test_util_parse_integer(void *arg)
     tt_int_op(i,OP_EQ, 0);
     tt_int_op(0UL,OP_EQ, tor_parse_ulong(TOOBIG, 10, 0, ULONG_MAX, &i, NULL));
     tt_int_op(i,OP_EQ, 0);
-    tt_u64_op(U64_LITERAL(0), OP_EQ, tor_parse_uint64(TOOBIG, 10,
+    tt_u64_op(UINT64_C(0), OP_EQ, tor_parse_uint64(TOOBIG, 10,
                                              0, UINT64_MAX, &i, NULL));
     tt_int_op(i,OP_EQ, 0);
   }
  done:
-  tor_end_capture_bugs_();
+  ;
 }
 
 static void
@@ -2200,17 +2338,17 @@ test_util_pow2(void *arg)
   tt_int_op(tor_log2(3),OP_EQ, 1);
   tt_int_op(tor_log2(4),OP_EQ, 2);
   tt_int_op(tor_log2(5),OP_EQ, 2);
-  tt_int_op(tor_log2(U64_LITERAL(40000000000000000)),OP_EQ, 55);
+  tt_int_op(tor_log2(UINT64_C(40000000000000000)),OP_EQ, 55);
   tt_int_op(tor_log2(UINT64_MAX),OP_EQ, 63);
 
   /* Test round_to_power_of_2 */
   tt_u64_op(round_to_power_of_2(120), OP_EQ, 128);
   tt_u64_op(round_to_power_of_2(128), OP_EQ, 128);
   tt_u64_op(round_to_power_of_2(130), OP_EQ, 128);
-  tt_u64_op(round_to_power_of_2(U64_LITERAL(40000000000000000)), OP_EQ,
-            U64_LITERAL(1)<<55);
-  tt_u64_op(round_to_power_of_2(U64_LITERAL(0xffffffffffffffff)), OP_EQ,
-          U64_LITERAL(1)<<63);
+  tt_u64_op(round_to_power_of_2(UINT64_C(40000000000000000)), OP_EQ,
+            UINT64_C(1)<<55);
+  tt_u64_op(round_to_power_of_2(UINT64_C(0xffffffffffffffff)), OP_EQ,
+          UINT64_C(1)<<63);
   tt_u64_op(round_to_power_of_2(0), OP_EQ, 1);
   tt_u64_op(round_to_power_of_2(1), OP_EQ, 1);
   tt_u64_op(round_to_power_of_2(2), OP_EQ, 2);
@@ -2224,114 +2362,371 @@ test_util_pow2(void *arg)
   ;
 }
 
-/** Run unit tests for compression functions */
 static void
-test_util_gzip(void *arg)
+test_util_compress_impl(compress_method_t method)
 {
-  char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2;
-  const char *ccp2;
+  char *buf1=NULL, *buf2=NULL, *buf3=NULL;
   size_t len1, len2;
-  tor_zlib_state_t *state = NULL;
 
-  (void)arg;
-  buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ");
-  tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD);
+  tt_assert(tor_compress_supports_method(method));
 
-  tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1,
-                               GZIP_METHOD));
-  tt_assert(buf2);
-  tt_assert(len1 < strlen(buf1));
-  tt_assert(detect_compression_method(buf2, len1) == GZIP_METHOD);
-
-  tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1,
-                                 GZIP_METHOD, 1, LOG_INFO));
-  tt_assert(buf3);
-  tt_int_op(strlen(buf1) + 1,OP_EQ, len2);
-  tt_str_op(buf1,OP_EQ, buf3);
+  if (method != NO_METHOD) {
+    tt_ptr_op(tor_compress_version_str(method), OP_NE, NULL);
+    tt_ptr_op(tor_compress_header_version_str(method), OP_NE, NULL);
+  }
 
-  tor_free(buf2);
-  tor_free(buf3);
+  buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ");
+  tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD);
 
-  tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1,
-                                 ZLIB_METHOD));
-  tt_assert(buf2);
-  tt_assert(detect_compression_method(buf2, len1) == ZLIB_METHOD);
+  tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, method));
+  tt_ptr_op(buf2, OP_NE, NULL);
+  if (method == NO_METHOD) {
+    // The identity transform doesn't actually compress, and it isn't
+    // detectable as "the identity transform."
+    tt_int_op(len1, OP_EQ, strlen(buf1)+1);
+    tt_int_op(detect_compression_method(buf2, len1), OP_EQ, UNKNOWN_METHOD);
+  } else {
+    tt_int_op(len1, OP_LT, strlen(buf1));
+    tt_int_op(detect_compression_method(buf2, len1), OP_EQ, method);
+  }
 
-  tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1,
-                                   ZLIB_METHOD, 1, LOG_INFO));
-  tt_assert(buf3);
-  tt_int_op(strlen(buf1) + 1,OP_EQ, len2);
-  tt_str_op(buf1,OP_EQ, buf3);
+  tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1, method, 1, LOG_INFO));
+  tt_ptr_op(buf3, OP_NE, NULL);
+  tt_int_op(strlen(buf1) + 1, OP_EQ, len2);
+  tt_str_op(buf1, OP_EQ, buf3);
+  tt_int_op(buf3[len2], OP_EQ, 0);
 
   /* Check whether we can uncompress concatenated, compressed strings. */
   tor_free(buf3);
   buf2 = tor_reallocarray(buf2, len1, 2);
   memcpy(buf2+len1, buf2, len1);
-  tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1*2,
-                                   ZLIB_METHOD, 1, LOG_INFO));
-  tt_int_op((strlen(buf1)+1)*2,OP_EQ, len2);
-  tt_mem_op(buf3,OP_EQ,
+  tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1*2, method, 1, LOG_INFO));
+  tt_int_op((strlen(buf1)+1)*2, OP_EQ, len2);
+  tt_mem_op(buf3, OP_EQ,
              "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0"
              "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0",
              (strlen(buf1)+1)*2);
+  tt_int_op(buf3[len2], OP_EQ, 0);
+
+  /* Check whether we can uncompress partial strings */
 
   tor_free(buf1);
   tor_free(buf2);
   tor_free(buf3);
 
-  /* Check whether we can uncompress partial strings. */
-  buf1 =
-    tor_strdup("String with low redundancy that won't be compressed much.");
-  tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1,
-                                 ZLIB_METHOD));
-  tt_assert(len1>16);
-  /* when we allow an incomplete string, we should succeed.*/
-  tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1-16,
-                                  ZLIB_METHOD, 0, LOG_INFO));
-  tt_assert(len2 > 5);
-  buf3[len2]='\0';
-  tt_assert(!strcmpstart(buf1, buf3));
-
-  /* when we demand a complete string, this must fail. */
+  size_t b1len = 1<<10;
+  if (method == ZSTD_METHOD) {
+    // zstd needs a big input before it starts generating output that it
+    // can partially decompress.
+    b1len = 1<<18;
+  }
+  buf1 = tor_malloc(b1len);
+  crypto_rand(buf1, b1len);
+  tt_assert(!tor_compress(&buf2, &len1, buf1, b1len, method));
+  tt_int_op(len1, OP_GT, 16);
+  /* when we allow an incomplete output we should succeed.*/
+  tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1-16,
+                            method, 0, LOG_INFO));
+  tt_int_op(len2, OP_GT, 5);
+  tt_int_op(len2, OP_LE, len1);
+  tt_assert(fast_memeq(buf1, buf3, len2));
+  tt_int_op(buf3[len2], OP_EQ, 0);
+
+  /* when we demand a complete output from a real compression method, this
+   * must fail. */
   tor_free(buf3);
-  tt_assert(tor_gzip_uncompress(&buf3, &len2, buf2, len1-16,
-                                 ZLIB_METHOD, 1, LOG_INFO));
-  tt_assert(!buf3);
+  if (method != NO_METHOD) {
+    tt_assert(tor_uncompress(&buf3, &len2, buf2, len1-16,
+                             method, 1, LOG_INFO));
+    tt_ptr_op(buf3, OP_EQ, NULL);
+  }
 
-  /* Now, try streaming compression. */
+ done:
   tor_free(buf1);
   tor_free(buf2);
   tor_free(buf3);
-  state = tor_zlib_new(1, ZLIB_METHOD, HIGH_COMPRESSION);
+}
+
+static void
+test_util_compress_stream_impl(compress_method_t method,
+                               compression_level_t level)
+{
+  char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2;
+  const char *ccp2;
+  size_t len1, len2;
+
+  tor_compress_state_t *state = NULL;
+  state = tor_compress_new(1, method, level);
   tt_assert(state);
   cp1 = buf1 = tor_malloc(1024);
   len1 = 1024;
   ccp2 = "ABCDEFGHIJABCDEFGHIJ";
   len2 = 21;
-  tt_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 0)
-              == TOR_ZLIB_OK);
-  tt_int_op(0,OP_EQ, len2); /* Make sure we compressed it all. */
+  tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 0),
+            OP_EQ, TOR_COMPRESS_OK);
+  tt_int_op(0, OP_EQ, len2); /* Make sure we compressed it all. */
   tt_assert(cp1 > buf1);
 
   len2 = 0;
   cp2 = cp1;
-  tt_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 1)
-              == TOR_ZLIB_DONE);
-  tt_int_op(0,OP_EQ, len2);
-  tt_assert(cp1 > cp2); /* Make sure we really added something. */
+  tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 1),
+            OP_EQ, TOR_COMPRESS_DONE);
+  tt_int_op(0, OP_EQ, len2);
+  if (method == NO_METHOD) {
+    tt_ptr_op(cp1, OP_EQ, cp2);
+  } else {
+    tt_assert(cp1 > cp2); /* Make sure we really added something. */
+  }
 
-  tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf1, 1024-len1,
-                                  ZLIB_METHOD, 1, LOG_WARN));
+  tt_int_op(tor_compress_state_size(state), OP_GT, 0);
+
+  tt_assert(!tor_uncompress(&buf3, &len2, buf1, 1024-len1,
+                            method, 1, LOG_WARN));
   /* Make sure it compressed right. */
   tt_str_op(buf3, OP_EQ, "ABCDEFGHIJABCDEFGHIJ");
-  tt_int_op(21,OP_EQ, len2);
+  tt_int_op(21, OP_EQ, len2);
 
  done:
   if (state)
-    tor_zlib_free(state);
+    tor_compress_free(state);
+  tor_free(buf1);
   tor_free(buf2);
   tor_free(buf3);
-  tor_free(buf1);
+}
+
+/** Setup function for compression tests: handles x-zstd:nostatic
+ */
+static void *
+compression_test_setup(const struct testcase_t *testcase)
+{
+  tor_assert(testcase->setup_data);
+  tor_assert(testcase->setup_data != (void*)TT_SKIP);
+  const char *methodname = testcase->setup_data;
+
+  if (!strcmp(methodname, "x-zstd:nostatic")) {
+    methodname = "x-zstd";
+    tor_zstd_set_static_apis_disabled_for_testing(1);
+  }
+
+  return (void *)methodname;
+}
+
+/** Cleanup for compression tests: disables nostatic */
+static int
+compression_test_cleanup(const struct testcase_t *testcase, void *ptr)
+{
+  (void)testcase;
+  (void)ptr;
+  tor_zstd_set_static_apis_disabled_for_testing(0);
+  return 1;
+}
+
+static const struct testcase_setup_t compress_setup = {
+  compression_test_setup, compression_test_cleanup
+};
+
+/** Run unit tests for compression functions */
+static void
+test_util_compress(void *arg)
+{
+  const char *methodname = arg;
+  tt_assert(methodname);
+
+  compress_method_t method = compression_method_get_by_name(methodname);
+  tt_int_op(method, OP_NE, UNKNOWN_METHOD);
+
+  if (! tor_compress_supports_method(method)) {
+    tt_skip();
+  }
+
+  compression_level_t levels[] = {
+    BEST_COMPRESSION,
+    HIGH_COMPRESSION,
+    MEDIUM_COMPRESSION,
+    LOW_COMPRESSION
+  };
+
+  test_util_compress_impl(method);
+
+  for (unsigned l = 0; l < ARRAY_LENGTH(levels); ++l) {
+    compression_level_t level = levels[l];
+    test_util_compress_stream_impl(method, level);
+  }
+ done:
+  ;
+}
+
+static void
+test_util_decompress_concatenated_impl(compress_method_t method)
+{
+  char input[4096];
+  char *c1 = NULL, *c2 = NULL, *c3 = NULL;
+  char *result = NULL;
+  size_t sz1, sz2, sz3, szr;
+  int r;
+
+  crypto_rand(input, sizeof(input));
+
+  /* Compress the input in two chunks. */
+  r = tor_compress(&c1, &sz1, input, 2048, method);
+  tt_int_op(r, OP_EQ, 0);
+  r = tor_compress(&c2, &sz2, input+2048, 2048, method);
+  tt_int_op(r, OP_EQ, 0);
+
+  /* concatenate the chunks. */
+  sz3 = sz1 + sz2;
+  c3 = tor_malloc(sz3);
+  memcpy(c3, c1, sz1);
+  memcpy(c3+sz1, c2, sz2);
+
+  /* decompress the concatenated result */
+  r = tor_uncompress(&result, &szr, c3, sz3, method, 0, LOG_WARN);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(szr, OP_EQ, sizeof(input));
+  tt_mem_op(result, OP_EQ, input, sizeof(input));
+
+ done:
+  tor_free(c1);
+  tor_free(c2);
+  tor_free(c3);
+  tor_free(result);
+}
+
+static void
+test_util_decompress_concatenated(void *arg)
+{
+  const char *methodname = arg;
+  tt_assert(methodname);
+
+  compress_method_t method = compression_method_get_by_name(methodname);
+  tt_int_op(method, OP_NE, UNKNOWN_METHOD);
+  if (! tor_compress_supports_method(method)) {
+    tt_skip();
+  }
+
+  test_util_decompress_concatenated_impl(method);
+ done:
+  ;
+}
+
+static void
+test_util_decompress_junk_impl(compress_method_t method)
+{
+  char input[4096];
+  char *result = NULL, *result2 = NULL;
+  size_t szr, szr2, sz;
+  int r;
+
+  /* This shouldn't be a compressed string according to any method. */
+  strlcpy(input, "This shouldn't be a compressed string by any means.",
+          sizeof(input));
+  sz = strlen(input);
+  setup_capture_of_logs(LOG_WARN);
+  r = tor_uncompress(&result, &szr, input, sz, method, 0, LOG_WARN);
+  tt_int_op(r, OP_EQ, -1);
+  tt_ptr_op(result, OP_EQ, NULL);
+  expect_log_msg_containing("Error while uncompressing data: bad input?");
+  mock_clean_saved_logs();
+
+  /* Now try again, with a compressed object that starts out good and turns to
+     junk. */
+  crypto_rand(input, sizeof(input));
+  r = tor_compress(&result, &szr, input, sizeof(input), method);
+  tt_int_op(r, OP_EQ, 0);
+  crypto_rand(result+szr/2, szr-(szr/2)); // trash the 2nd half of the result
+  r = tor_uncompress(&result2, &szr2, result, szr, method, 0, LOG_WARN);
+  tt_int_op(r, OP_EQ, -1);
+  expect_log_msg_containing("Error while uncompressing data: bad input?");
+
+ done:
+  teardown_capture_of_logs();
+  tor_free(result);
+  tor_free(result2);
+}
+
+static void
+test_util_decompress_junk(void *arg)
+{
+  const char *methodname = arg;
+  tt_assert(methodname);
+
+  compress_method_t method = compression_method_get_by_name(methodname);
+  tt_int_op(method, OP_NE, UNKNOWN_METHOD);
+  if (! tor_compress_supports_method(method)) {
+    tt_skip();
+  }
+
+  test_util_decompress_junk_impl(method);
+ done:
+  ;
+}
+
+/* mock replacement for tor_compress_is_compression_bomb that doesn't
+ * believe in compression bombs. */
+static int
+mock_is_never_compression_bomb(size_t in, size_t out)
+{
+  (void)in;
+  (void) out;
+  return 0;
+}
+
+static void
+test_util_decompress_dos_impl(compress_method_t method)
+{
+  char *input;
+  char *result = NULL, *result2 = NULL;
+  size_t szr, szr2;
+  int r;
+
+  const size_t big = 1024*1024;
+  /* one megabyte of 0s. */
+  input = tor_malloc_zero(big);
+
+  /* Compress it into "result": it should fail. */
+  setup_full_capture_of_logs(LOG_WARN);
+  r = tor_compress(&result, &szr, input, big, method);
+  tt_int_op(r, OP_EQ, -1);
+  expect_log_msg_containing(
+                 "other Tors would think this was a compression bomb");
+  teardown_capture_of_logs();
+
+  /* Try again, but this time suppress compression-bomb detection */
+  MOCK(tor_compress_is_compression_bomb, mock_is_never_compression_bomb);
+  r = tor_compress(&result, &szr, input, big, method);
+  UNMOCK(tor_compress_is_compression_bomb);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+
+  /* We should refuse to uncomrpess it again, since it looks like a
+   * compression bomb. */
+  setup_capture_of_logs(LOG_WARN);
+  r = tor_uncompress(&result2, &szr2, result, szr, method, 0, LOG_WARN);
+  tt_int_op(r, OP_EQ, -1);
+  expect_log_msg_containing("bomb; abandoning stream");
+
+ done:
+  teardown_capture_of_logs();
+  tor_free(input);
+  tor_free(result);
+  tor_free(result2);
+}
+
+static void
+test_util_decompress_dos(void *arg)
+{
+  const char *methodname = arg;
+  tt_assert(methodname);
+
+  compress_method_t method = compression_method_get_by_name(methodname);
+  tt_int_op(method, OP_NE, UNKNOWN_METHOD);
+  if (! tor_compress_supports_method(method)) {
+    tt_skip();
+  }
+
+  test_util_decompress_dos_impl(method);
+ done:
+  ;
 }
 
 static void
@@ -2346,44 +2741,44 @@ test_util_gzip_compression_bomb(void *arg)
   char *one_mb = tor_malloc_zero(one_million);
   char *result = NULL;
   size_t result_len = 0;
-  tor_zlib_state_t *state = NULL;
+  tor_compress_state_t *state = NULL;
 
   /* Make sure we can't produce a compression bomb */
   setup_full_capture_of_logs(LOG_WARN);
-  tt_int_op(-1, OP_EQ, tor_gzip_compress(&result, &result_len,
-                                         one_mb, one_million,
-                                         ZLIB_METHOD));
+  tt_int_op(-1, OP_EQ, tor_compress(&result, &result_len,
+                                    one_mb, one_million,
+                                    ZLIB_METHOD));
   expect_single_log_msg_containing(
          "We compressed something and got an insanely high "
          "compression factor; other Tors would think this "
-         "was a zlib bomb.");
+         "was a compression bomb.");
   teardown_capture_of_logs();
 
   /* Here's a compression bomb that we made manually. */
   const char compression_bomb[1039] =
     { 0x78, 0xDA, 0xED, 0xC1, 0x31, 0x01, 0x00, 0x00, 0x00, 0xC2,
       0xA0, 0xF5, 0x4F, 0x6D, 0x08, 0x5F, 0xA0 /* .... */ };
-  tt_int_op(-1, OP_EQ, tor_gzip_uncompress(&result, &result_len,
-                                           compression_bomb, 1039,
-                                           ZLIB_METHOD, 0, LOG_WARN));
+  tt_int_op(-1, OP_EQ, tor_uncompress(&result, &result_len,
+                                      compression_bomb, 1039,
+                                      ZLIB_METHOD, 0, LOG_WARN));
 
   /* Now try streaming that. */
-  state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION);
-  tor_zlib_output_t r;
+  state = tor_compress_new(0, ZLIB_METHOD, HIGH_COMPRESSION);
+  tor_compress_output_t r;
   const char *inp = compression_bomb;
   size_t inlen = 1039;
   do {
     char *outp = one_mb;
     size_t outleft = 4096; /* small on purpose */
-    r = tor_zlib_process(state, &outp, &outleft, &inp, &inlen, 0);
+    r = tor_compress_process(state, &outp, &outleft, &inp, &inlen, 0);
     tt_int_op(inlen, OP_NE, 0);
-  } while (r == TOR_ZLIB_BUF_FULL);
+  } while (r == TOR_COMPRESS_BUFFER_FULL);
 
-  tt_int_op(r, OP_EQ, TOR_ZLIB_ERR);
+  tt_int_op(r, OP_EQ, TOR_COMPRESS_ERROR);
 
  done:
   tor_free(one_mb);
-  tor_zlib_free(state);
+  tor_compress_free(state);
 }
 
 /** Run unit tests for mmap() wrapper functionality. */
@@ -2401,7 +2796,7 @@ test_util_mmap(void *arg)
   crypto_rand(buf, buflen);
 
   mapping = tor_mmap_file(fname1);
-  tt_assert(! mapping);
+  tt_ptr_op(mapping, OP_EQ, NULL);
 
   write_str_to_file(fname1, "Short file.", 1);
 
@@ -2419,7 +2814,7 @@ test_util_mmap(void *arg)
   tt_str_op(mapping->data,OP_EQ, "Short file.");
   tt_int_op(0, OP_EQ, tor_munmap_file(mapping));
   mapping = NULL;
-#endif
+#endif /* defined(_WIN32) */
 
   /* Now a zero-length file. */
   write_str_to_file(fname1, "", 1);
@@ -2430,7 +2825,7 @@ test_util_mmap(void *arg)
 
   /* Make sure that we fail to map a no-longer-existent file. */
   mapping = tor_mmap_file(fname1);
-  tt_assert(! mapping);
+  tt_ptr_op(mapping, OP_EQ, NULL);
 
   /* Now try a big file that stretches across a few pages and isn't aligned */
   write_bytes_to_file(fname2, buf, buflen, 1);
@@ -2748,7 +3143,7 @@ test_util_sscanf(void *arg)
   r = tor_sscanf("9223372036854775808. -9223372036854775809.",
                  "%d. %d.", &int1, &int2);
   tt_int_op(r,OP_EQ, 0);
-#endif
+#endif /* SIZEOF_INT == 4 || ... */
 
 #if SIZEOF_LONG == 4
   /* %lu */
@@ -2843,7 +3238,7 @@ test_util_sscanf(void *arg)
   r = tor_sscanf("9223372036854775808. -9223372036854775809.",
                  "%ld. %ld.", &lng1, &lng2);
   tt_int_op(r,OP_EQ, 0);
-#endif
+#endif /* SIZEOF_LONG == 4 || ... */
 
   r = tor_sscanf("123.456 .000007 -900123123.2000787 00003.2",
                  "%lf %lf %lf %lf", &d1,&d2,&d3,&d4);
@@ -2853,6 +3248,21 @@ test_util_sscanf(void *arg)
   test_feq(d3, -900123123.2000787);
   test_feq(d4, 3.2);
 
+  /* missing float */
+  r = tor_sscanf("3 ", "%d %lf", &int1, &d1);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(int1, OP_EQ, 3);
+
+  /* not a float */
+  r = tor_sscanf("999 notafloat", "%d %lf", &int1, &d1);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(int1, OP_EQ, 999);
+
+  /* %s but no buffer. */
+  char *nullbuf = NULL;
+  r = tor_sscanf("hello", "%3s", nullbuf);
+  tt_int_op(r, OP_EQ, 0);
+
  done:
   tor_free(huge);
 }
@@ -2870,7 +3280,7 @@ strnlen(const char *s, size_t len)
     return len;
   return p - s;
 }
-#endif
+#endif /* !defined(HAVE_STRNLEN) */
 
 static void
 test_util_format_time_interval(void *arg)
@@ -3237,7 +3647,7 @@ test_util_format_time_interval(void *arg)
   tt_ci_char_op(label_m[0],OP_EQ, 'm');
   /* and 7 or 8 seconds - ignored */
 
-#endif
+#endif /* SIZEOF_LONG == 4 || SIZEOF_LONG == 8 */
 
 #if SIZEOF_LONG == 8
 
@@ -3275,7 +3685,7 @@ test_util_format_time_interval(void *arg)
   tt_ci_char_op(label_m[0],OP_EQ, 'm');
   /* and 7 or 8 seconds - ignored */
 
-#endif
+#endif /* SIZEOF_LONG == 8 */
 
  done:
   ;
@@ -3318,7 +3728,7 @@ test_util_path_is_relative(void *arg)
   tt_int_op(0,OP_EQ, path_is_relative("\\dir"));
   tt_int_op(0,OP_EQ, path_is_relative("a:\\dir"));
   tt_int_op(0,OP_EQ, path_is_relative("z:\\dir"));
-#endif
+#endif /* defined(_WIN32) */
 
  done:
   ;
@@ -3333,6 +3743,13 @@ test_util_memarea(void *arg)
   void *malloced_ptr = NULL;
   int i;
 
+#ifdef DISABLE_MEMORY_SENTINELS
+  /* If memory sentinels are disabled, this whole module is just an alias for
+     malloc(), which is free to lay out memory most any way it wants. */
+  if (1)
+    tt_skip();
+#endif /* defined(DISABLE_MEMORY_SENTINELS) */
+
   (void)arg;
   tt_assert(area);
 
@@ -3502,8 +3919,8 @@ test_util_strtok(void *arg)
     }
     tor_snprintf(buf, sizeof(buf), "%s", pad1);
     tor_snprintf(buf2, sizeof(buf2), "%s", pad2);
-    tt_assert(NULL == tor_strtok_r_impl(buf, " ", &cp1));
-    tt_assert(NULL == tor_strtok_r_impl(buf2, ".!..;!", &cp2));
+    tt_ptr_op(tor_strtok_r_impl(buf, " ", &cp1), OP_EQ, NULL);
+    tt_ptr_op(tor_strtok_r_impl(buf2, ".!..;!", &cp2), OP_EQ, NULL);
 
     tor_snprintf(buf, sizeof(buf),
                  "%sGraved on the dark  in gestures of descent%s", pad1, pad1);
@@ -3622,6 +4039,53 @@ test_util_string_is_C_identifier(void *ptr)
 }
 
 static void
+test_util_string_is_utf8(void *ptr)
+{
+  (void)ptr;
+
+  tt_int_op(1, OP_EQ, string_is_utf8(NULL, 0));
+  tt_int_op(1, OP_EQ, string_is_utf8("", 1));
+  tt_int_op(1, OP_EQ, string_is_utf8("\uFEFF", 3));
+  tt_int_op(1, OP_EQ, string_is_utf8("\uFFFE", 3));
+  tt_int_op(1, OP_EQ, string_is_utf8("ascii\x7f\n", 7));
+  tt_int_op(1, OP_EQ, string_is_utf8("Risqu\u00e9=1", 9));
+
+  // Validate exactly 'len' bytes.
+  tt_int_op(0, OP_EQ, string_is_utf8("\0\x80", 2));
+  tt_int_op(0, OP_EQ, string_is_utf8("Risqu\u00e9=1", 6));
+
+  // Reject sequences with missing bytes.
+  tt_int_op(0, OP_EQ, string_is_utf8("\x80", 1));
+  tt_int_op(0, OP_EQ, string_is_utf8("\xc2", 1));
+  tt_int_op(0, OP_EQ, string_is_utf8("\xc2 ", 2));
+  tt_int_op(0, OP_EQ, string_is_utf8("\xe1\x80", 2));
+  tt_int_op(0, OP_EQ, string_is_utf8("\xe1\x80 ", 3));
+  tt_int_op(0, OP_EQ, string_is_utf8("\xf1\x80\x80", 3));
+  tt_int_op(0, OP_EQ, string_is_utf8("\xf1\x80\x80 ", 4));
+
+  // Reject encodings that are overly long.
+  tt_int_op(0, OP_EQ, string_is_utf8("\xc1\xbf", 2));
+  tt_int_op(1, OP_EQ, string_is_utf8("\xc2\x80", 2));
+  tt_int_op(0, OP_EQ, string_is_utf8("\xe0\x9f\xbf", 3));
+  tt_int_op(1, OP_EQ, string_is_utf8("\xe0\xa0\x80", 3));
+  tt_int_op(0, OP_EQ, string_is_utf8("\xf0\x8f\xbf\xbf", 4));
+  tt_int_op(1, OP_EQ, string_is_utf8("\xf0\x90\x80\x80", 4));
+
+  // Reject UTF-16 surrogate halves.
+  tt_int_op(1, OP_EQ, string_is_utf8("\xed\x9f\xbf", 3));
+  tt_int_op(0, OP_EQ, string_is_utf8("\xed\xa0\x80", 3));
+  tt_int_op(0, OP_EQ, string_is_utf8("\xed\xbf\xbf", 3));
+  tt_int_op(1, OP_EQ, string_is_utf8("\xee\x80\x80", 3));
+
+  // The maximum legal codepoint, 10FFFF.
+  tt_int_op(1, OP_EQ, string_is_utf8("\xf4\x8f\xbf\xbf", 4));
+  tt_int_op(0, OP_EQ, string_is_utf8("\xf4\x90\x80\x80", 4));
+
+ done:
+  ;
+}
+
+static void
 test_util_asprintf(void *ptr)
 {
 #define LOREMIPSUM                                              \
@@ -3796,7 +4260,8 @@ test_util_ftruncate(void *ptr)
   tt_int_op(fd, OP_GE, 0);
 
   /* Make the file be there. */
-  tt_int_op(strlen(message), OP_EQ, write_all(fd, message, strlen(message),0));
+  tt_int_op(strlen(message), OP_EQ,
+            write_all_to_fd(fd, message, strlen(message)));
   tt_int_op((int)tor_fd_getpos(fd), OP_EQ, strlen(message));
   tt_int_op(0, OP_EQ, fstat(fd, &st));
   tt_int_op((int)st.st_size, OP_EQ, strlen(message));
@@ -3809,7 +4274,7 @@ test_util_ftruncate(void *ptr)
 
   /* Replace, and see if it got replaced */
   tt_int_op(strlen(message2), OP_EQ,
-            write_all(fd, message2, strlen(message2), 0));
+            write_all_to_fd(fd, message2, strlen(message2)));
   tt_int_op((int)tor_fd_getpos(fd), OP_EQ, strlen(message2));
   tt_int_op(0, OP_EQ, fstat(fd, &st));
   tt_int_op((int)st.st_size, OP_EQ, strlen(message2));
@@ -3853,7 +4318,7 @@ test_util_load_win_lib(void *ptr)
   if (h)
     FreeLibrary(h);
 }
-#endif
+#endif /* defined(_WIN32) */
 
 #ifndef _WIN32
 static void
@@ -3905,7 +4370,7 @@ test_util_exit_status(void *ptr)
   tt_int_op(n,OP_EQ, strlen(hex_errno));
   tt_int_op(n,OP_EQ, HEX_ERRNO_SIZE);
 
-#endif
+#endif /* SIZEOF_INT == 4 || ... */
 
   clear_hex_errno(hex_errno);
   n = format_helper_exit_status(0x7F, 0, hex_errno);
@@ -3923,20 +4388,16 @@ test_util_exit_status(void *ptr)
  done:
   ;
 }
-#endif
+#endif /* !defined(_WIN32) */
 
 #ifndef _WIN32
-/* Check that fgets with a non-blocking pipe returns partial lines and sets
- * EAGAIN, returns full lines and sets no error, and returns NULL on EOF and
- * sets no error */
 static void
-test_util_fgets_eagain(void *ptr)
+test_util_string_from_pipe(void *ptr)
 {
   int test_pipe[2] = {-1, -1};
-  int retval;
+  int retval = 0;
+  enum stream_status status = IO_STREAM_TERM;
   ssize_t retlen;
-  char *retptr;
-  FILE *test_stream = NULL;
   char buf[4] = { 0 };
 
   (void)ptr;
@@ -3947,91 +4408,115 @@ test_util_fgets_eagain(void *ptr)
   retval = pipe(test_pipe);
   tt_int_op(retval, OP_EQ, 0);
 
-  /* Set up the read-end to be non-blocking */
-  retval = fcntl(test_pipe[0], F_SETFL, O_NONBLOCK);
-  tt_int_op(retval, OP_EQ, 0);
+  /* Send in a string. */
+  retlen = write(test_pipe[1], "ABC", 3);
+  tt_int_op(retlen, OP_EQ, 3);
 
-  /* Open it as a stdio stream */
-  test_stream = fdopen(test_pipe[0], "r");
-  tt_ptr_op(test_stream, OP_NE, NULL);
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
+  tt_int_op(errno, OP_EQ, 0);
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "ABC");
+  errno = 0;
+
+  /* Send in a string that contains a nul. */
+  retlen = write(test_pipe[1], "AB\0", 3);
+  tt_int_op(retlen, OP_EQ, 3);
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
+  tt_int_op(errno, OP_EQ, 0);
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "AB");
+  errno = 0;
 
-  /* Send in a partial line */
-  retlen = write(test_pipe[1], "A", 1);
+  /* Send in a string that contains a nul only. */
+  retlen = write(test_pipe[1], "\0", 1);
   tt_int_op(retlen, OP_EQ, 1);
-  retptr = fgets(buf, sizeof(buf), test_stream);
-  tt_int_op(errno, OP_EQ, EAGAIN);
-  tt_ptr_op(retptr, OP_EQ, buf);
-  tt_str_op(buf, OP_EQ, "A");
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
+  tt_int_op(errno, OP_EQ, 0);
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "");
   errno = 0;
 
-  /* Send in the rest */
-  retlen = write(test_pipe[1], "B\n", 2);
-  tt_int_op(retlen, OP_EQ, 2);
-  retptr = fgets(buf, sizeof(buf), test_stream);
+  /* Send in a string that contains a trailing newline. */
+  retlen = write(test_pipe[1], "AB\n", 3);
+  tt_int_op(retlen, OP_EQ, 3);
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
   tt_int_op(errno, OP_EQ, 0);
-  tt_ptr_op(retptr, OP_EQ, buf);
-  tt_str_op(buf, OP_EQ, "B\n");
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "AB");
   errno = 0;
 
-  /* Send in a full line */
-  retlen = write(test_pipe[1], "CD\n", 3);
+  /* Send in a string that contains a newline only. */
+  retlen = write(test_pipe[1], "\n", 1);
+  tt_int_op(retlen, OP_EQ, 1);
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
+  tt_int_op(errno, OP_EQ, 0);
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "");
+  errno = 0;
+
+  /* Send in a string and check that we nul terminate return values. */
+  retlen = write(test_pipe[1], "AAA", 3);
   tt_int_op(retlen, OP_EQ, 3);
-  retptr = fgets(buf, sizeof(buf), test_stream);
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
   tt_int_op(errno, OP_EQ, 0);
-  tt_ptr_op(retptr, OP_EQ, buf);
-  tt_str_op(buf, OP_EQ, "CD\n");
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "AAA");
+  tt_mem_op(buf, OP_EQ, "AAA\0", sizeof(buf));
   errno = 0;
 
-  /* Send in a partial line */
-  retlen = write(test_pipe[1], "E", 1);
+  retlen = write(test_pipe[1], "B", 1);
   tt_int_op(retlen, OP_EQ, 1);
-  retptr = fgets(buf, sizeof(buf), test_stream);
-  tt_int_op(errno, OP_EQ, EAGAIN);
-  tt_ptr_op(retptr, OP_EQ, buf);
-  tt_str_op(buf, OP_EQ, "E");
+
+  memset(buf, '\xff', sizeof(buf));
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
+  tt_int_op(errno, OP_EQ, 0);
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "B");
+  tt_mem_op(buf, OP_EQ, "B\0\xff\xff", sizeof(buf));
   errno = 0;
 
-  /* Send in the rest */
-  retlen = write(test_pipe[1], "F\n", 2);
-  tt_int_op(retlen, OP_EQ, 2);
-  retptr = fgets(buf, sizeof(buf), test_stream);
+  /* Send in multiple lines. */
+  retlen = write(test_pipe[1], "A\nB", 3);
+  tt_int_op(retlen, OP_EQ, 3);
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
   tt_int_op(errno, OP_EQ, 0);
-  tt_ptr_op(retptr, OP_EQ, buf);
-  tt_str_op(buf, OP_EQ, "F\n");
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "A\nB");
   errno = 0;
 
-  /* Send in a full line and close */
-  retlen = write(test_pipe[1], "GH", 2);
+  /* Send in a line and close */
+  retlen = write(test_pipe[1], "AB", 2);
   tt_int_op(retlen, OP_EQ, 2);
   retval = close(test_pipe[1]);
   tt_int_op(retval, OP_EQ, 0);
   test_pipe[1] = -1;
-  retptr = fgets(buf, sizeof(buf), test_stream);
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
   tt_int_op(errno, OP_EQ, 0);
-  tt_ptr_op(retptr, OP_EQ, buf);
-  tt_str_op(buf, OP_EQ, "GH");
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "AB");
   errno = 0;
 
   /* Check for EOF */
-  retptr = fgets(buf, sizeof(buf), test_stream);
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
   tt_int_op(errno, OP_EQ, 0);
-  tt_ptr_op(retptr, OP_EQ, NULL);
-  retval = feof(test_stream);
-  tt_int_op(retval, OP_NE, 0);
+  tt_int_op(status, OP_EQ, IO_STREAM_CLOSED);
   errno = 0;
 
-  /* Check that buf is unchanged according to C99 and C11 */
-  tt_str_op(buf, OP_EQ, "GH");
-
  done:
-  if (test_stream != NULL)
-    fclose(test_stream);
   if (test_pipe[0] != -1)
     close(test_pipe[0]);
   if (test_pipe[1] != -1)
     close(test_pipe[1]);
 }
-#endif
+
+#endif /* !defined(_WIN32) */
 
 /**
  * Test for format_hex_number_sigsafe()
@@ -4227,7 +4712,7 @@ test_util_split_lines(void *ptr)
       /* Check we have not got too many lines */
       tt_int_op(MAX_SPLIT_LINE_COUNT, OP_GT, j);
       /* Check that there actually should be a line here */
-      tt_assert(tests[i].split_line[j] != NULL);
+      tt_ptr_op(tests[i].split_line[j], OP_NE, NULL);
       log_info(LD_GENERAL, "Line %d of test %d, should be <%s>",
                j, i, tests[i].split_line[j]);
       /* Check that the line is as expected */
@@ -4343,11 +4828,11 @@ test_util_di_map(void *arg)
 
   char dflt_entry[] = "'You have made a good beginning', but no more";
 
-  tt_int_op(32, ==, sizeof(key1));
-  tt_int_op(32, ==, sizeof(key2));
-  tt_int_op(32, ==, sizeof(key3));
+  tt_int_op(32, OP_EQ, sizeof(key1));
+  tt_int_op(32, OP_EQ, sizeof(key2));
+  tt_int_op(32, OP_EQ, sizeof(key3));
 
-  tt_ptr_op(dflt_entry, ==, dimap_search(dimap, key1, dflt_entry));
+  tt_ptr_op(dflt_entry, OP_EQ, dimap_search(dimap, key1, dflt_entry));
 
   char *str1 = tor_strdup("You are precisely as big as what you love"
                           " and precisely as small as what you allow"
@@ -4365,10 +4850,10 @@ test_util_di_map(void *arg)
   dimap_add_entry(&dimap, key2, str2);
   dimap_add_entry(&dimap, key3, str3);
 
-  tt_ptr_op(str1, ==, dimap_search(dimap, key1, dflt_entry));
-  tt_ptr_op(str3, ==, dimap_search(dimap, key3, dflt_entry));
-  tt_ptr_op(str2, ==, dimap_search(dimap, key2, dflt_entry));
-  tt_ptr_op(dflt_entry, ==, dimap_search(dimap, key4, dflt_entry));
+  tt_ptr_op(str1, OP_EQ, dimap_search(dimap, key1, dflt_entry));
+  tt_ptr_op(str3, OP_EQ, dimap_search(dimap, key3, dflt_entry));
+  tt_ptr_op(str2, OP_EQ, dimap_search(dimap, key2, dflt_entry));
+  tt_ptr_op(dflt_entry, OP_EQ, dimap_search(dimap, key4, dflt_entry));
 
  done:
   dimap_free(dimap, tor_free_);
@@ -4835,34 +5320,34 @@ test_util_round_to_next_multiple_of(void *arg)
 {
   (void)arg;
 
-  tt_u64_op(round_uint64_to_next_multiple_of(0,1), ==, 0);
-  tt_u64_op(round_uint64_to_next_multiple_of(0,7), ==, 0);
+  tt_u64_op(round_uint64_to_next_multiple_of(0,1), OP_EQ, 0);
+  tt_u64_op(round_uint64_to_next_multiple_of(0,7), OP_EQ, 0);
 
-  tt_u64_op(round_uint64_to_next_multiple_of(99,1), ==, 99);
-  tt_u64_op(round_uint64_to_next_multiple_of(99,7), ==, 105);
-  tt_u64_op(round_uint64_to_next_multiple_of(99,9), ==, 99);
+  tt_u64_op(round_uint64_to_next_multiple_of(99,1), OP_EQ, 99);
+  tt_u64_op(round_uint64_to_next_multiple_of(99,7), OP_EQ, 105);
+  tt_u64_op(round_uint64_to_next_multiple_of(99,9), OP_EQ, 99);
 
-  tt_u64_op(round_uint64_to_next_multiple_of(UINT64_MAX,2), ==,
+  tt_u64_op(round_uint64_to_next_multiple_of(UINT64_MAX,2), OP_EQ,
             UINT64_MAX);
 
-  tt_int_op(round_uint32_to_next_multiple_of(0,1), ==, 0);
-  tt_int_op(round_uint32_to_next_multiple_of(0,7), ==, 0);
+  tt_int_op(round_uint32_to_next_multiple_of(0,1), OP_EQ, 0);
+  tt_int_op(round_uint32_to_next_multiple_of(0,7), OP_EQ, 0);
 
-  tt_int_op(round_uint32_to_next_multiple_of(99,1), ==, 99);
-  tt_int_op(round_uint32_to_next_multiple_of(99,7), ==, 105);
-  tt_int_op(round_uint32_to_next_multiple_of(99,9), ==, 99);
+  tt_int_op(round_uint32_to_next_multiple_of(99,1), OP_EQ, 99);
+  tt_int_op(round_uint32_to_next_multiple_of(99,7), OP_EQ, 105);
+  tt_int_op(round_uint32_to_next_multiple_of(99,9), OP_EQ, 99);
 
-  tt_int_op(round_uint32_to_next_multiple_of(UINT32_MAX,2), ==,
+  tt_int_op(round_uint32_to_next_multiple_of(UINT32_MAX,2), OP_EQ,
             UINT32_MAX);
 
-  tt_uint_op(round_to_next_multiple_of(0,1), ==, 0);
-  tt_uint_op(round_to_next_multiple_of(0,7), ==, 0);
+  tt_uint_op(round_to_next_multiple_of(0,1), OP_EQ, 0);
+  tt_uint_op(round_to_next_multiple_of(0,7), OP_EQ, 0);
 
-  tt_uint_op(round_to_next_multiple_of(99,1), ==, 99);
-  tt_uint_op(round_to_next_multiple_of(99,7), ==, 105);
-  tt_uint_op(round_to_next_multiple_of(99,9), ==, 99);
+  tt_uint_op(round_to_next_multiple_of(99,1), OP_EQ, 99);
+  tt_uint_op(round_to_next_multiple_of(99,7), OP_EQ, 105);
+  tt_uint_op(round_to_next_multiple_of(99,9), OP_EQ, 99);
 
-  tt_uint_op(round_to_next_multiple_of(UINT_MAX,2), ==,
+  tt_uint_op(round_to_next_multiple_of(UINT_MAX,2), OP_EQ,
             UINT_MAX);
  done:
   ;
@@ -4883,26 +5368,26 @@ test_util_laplace(void *arg)
   const double delta_f = 15.0, epsilon = 0.3; /* b = 15.0 / 0.3 = 50.0 */
   (void)arg;
 
-  tt_i64_op(INT64_MIN, ==, sample_laplace_distribution(mu, b, 0.0));
-  tt_i64_op(-69, ==, sample_laplace_distribution(mu, b, 0.01));
-  tt_i64_op(24, ==, sample_laplace_distribution(mu, b, 0.5));
-  tt_i64_op(24, ==, sample_laplace_distribution(mu, b, 0.51));
-  tt_i64_op(117, ==, sample_laplace_distribution(mu, b, 0.99));
+  tt_i64_op(INT64_MIN, OP_EQ, sample_laplace_distribution(mu, b, 0.0));
+  tt_i64_op(-69, OP_EQ, sample_laplace_distribution(mu, b, 0.01));
+  tt_i64_op(24, OP_EQ, sample_laplace_distribution(mu, b, 0.5));
+  tt_i64_op(24, OP_EQ, sample_laplace_distribution(mu, b, 0.51));
+  tt_i64_op(117, OP_EQ, sample_laplace_distribution(mu, b, 0.99));
 
   /* >>> laplace.ppf([0.0, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99],
    * ...             loc = 0, scale = 50)
    * array([         -inf,  -80.47189562,  -34.65735903,    0.        ,
    *          34.65735903,   80.47189562,  195.60115027])
    */
-  tt_i64_op(INT64_MIN + 20, ==,
+  tt_i64_op(INT64_MIN + 20, OP_EQ,
             add_laplace_noise(20, 0.0, delta_f, epsilon));
 
-  tt_i64_op(-60, ==, add_laplace_noise(20, 0.1, delta_f, epsilon));
-  tt_i64_op(-14, ==, add_laplace_noise(20, 0.25, delta_f, epsilon));
-  tt_i64_op(20, ==, add_laplace_noise(20, 0.5, delta_f, epsilon));
-  tt_i64_op(54, ==, add_laplace_noise(20, 0.75, delta_f, epsilon));
-  tt_i64_op(100, ==, add_laplace_noise(20, 0.9, delta_f, epsilon));
-  tt_i64_op(215, ==, add_laplace_noise(20, 0.99, delta_f, epsilon));
+  tt_i64_op(-60, OP_EQ, add_laplace_noise(20, 0.1, delta_f, epsilon));
+  tt_i64_op(-14, OP_EQ, add_laplace_noise(20, 0.25, delta_f, epsilon));
+  tt_i64_op(20, OP_EQ, add_laplace_noise(20, 0.5, delta_f, epsilon));
+  tt_i64_op(54, OP_EQ, add_laplace_noise(20, 0.75, delta_f, epsilon));
+  tt_i64_op(100, OP_EQ, add_laplace_noise(20, 0.9, delta_f, epsilon));
+  tt_i64_op(215, OP_EQ, add_laplace_noise(20, 0.99, delta_f, epsilon));
 
   /* Test extreme values of signal with maximally negative values of noise
    * 1.0000000000000002 is the smallest number > 1
@@ -4915,54 +5400,54 @@ test_util_laplace(void *arg)
    */
   const double noscale_df = 1.0, noscale_eps = 1.0;
 
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ,
             add_laplace_noise(0, 0.0, noscale_df, noscale_eps));
 
   /* is it clipped to INT64_MIN? */
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ,
             add_laplace_noise(-1, 0.0, noscale_df, noscale_eps));
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ,
             add_laplace_noise(INT64_MIN, 0.0,
                               noscale_df, noscale_eps));
   /* ... even when scaled? */
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ,
             add_laplace_noise(0, 0.0, delta_f, epsilon));
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ,
             add_laplace_noise(0, 0.0,
                               DBL_MAX, 1));
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ,
             add_laplace_noise(INT64_MIN, 0.0,
                               DBL_MAX, 1));
 
   /* does it play nice with INT64_MAX? */
-  tt_i64_op((INT64_MIN + INT64_MAX), ==,
+  tt_i64_op((INT64_MIN + INT64_MAX), OP_EQ,
             add_laplace_noise(INT64_MAX, 0.0,
                               noscale_df, noscale_eps));
 
   /* do near-zero fractional values work? */
   const double min_dbl_error = 0.0000000000000002;
 
-  tt_i64_op(-35, ==,
+  tt_i64_op(-35, OP_EQ,
             add_laplace_noise(0, min_dbl_error,
                               noscale_df, noscale_eps));
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ,
             add_laplace_noise(INT64_MIN, min_dbl_error,
                               noscale_df, noscale_eps));
-  tt_i64_op((-35 + INT64_MAX), ==,
+  tt_i64_op((-35 + INT64_MAX), OP_EQ,
             add_laplace_noise(INT64_MAX, min_dbl_error,
                               noscale_df, noscale_eps));
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ,
             add_laplace_noise(0, min_dbl_error,
                               DBL_MAX, 1));
-  tt_i64_op((INT64_MAX + INT64_MIN), ==,
+  tt_i64_op((INT64_MAX + INT64_MIN), OP_EQ,
             add_laplace_noise(INT64_MAX, min_dbl_error,
                               DBL_MAX, 1));
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ,
             add_laplace_noise(INT64_MIN, min_dbl_error,
                               DBL_MAX, 1));
 
   /* does it play nice with INT64_MAX? */
-  tt_i64_op((INT64_MAX - 35), ==,
+  tt_i64_op((INT64_MAX - 35), OP_EQ,
             add_laplace_noise(INT64_MAX, min_dbl_error,
                               noscale_df, noscale_eps));
 
@@ -4977,31 +5462,31 @@ test_util_laplace(void *arg)
   const double max_dbl_lt_one = 0.9999999999999998;
 
   /* do near-one fractional values work? */
-  tt_i64_op(35, ==,
+  tt_i64_op(35, OP_EQ,
             add_laplace_noise(0, max_dbl_lt_one, noscale_df, noscale_eps));
 
   /* is it clipped to INT64_MAX? */
-  tt_i64_op(INT64_MAX, ==,
+  tt_i64_op(INT64_MAX, OP_EQ,
             add_laplace_noise(INT64_MAX - 35, max_dbl_lt_one,
                               noscale_df, noscale_eps));
-  tt_i64_op(INT64_MAX, ==,
+  tt_i64_op(INT64_MAX, OP_EQ,
             add_laplace_noise(INT64_MAX - 34, max_dbl_lt_one,
                               noscale_df, noscale_eps));
-  tt_i64_op(INT64_MAX, ==,
+  tt_i64_op(INT64_MAX, OP_EQ,
             add_laplace_noise(INT64_MAX, max_dbl_lt_one,
                               noscale_df, noscale_eps));
   /* ... even when scaled? */
-  tt_i64_op(INT64_MAX, ==,
+  tt_i64_op(INT64_MAX, OP_EQ,
             add_laplace_noise(INT64_MAX, max_dbl_lt_one,
                               delta_f, epsilon));
-  tt_i64_op((INT64_MIN + INT64_MAX), ==,
+  tt_i64_op((INT64_MIN + INT64_MAX), OP_EQ,
             add_laplace_noise(INT64_MIN, max_dbl_lt_one,
                               DBL_MAX, 1));
-  tt_i64_op(INT64_MAX, ==,
+  tt_i64_op(INT64_MAX, OP_EQ,
             add_laplace_noise(INT64_MAX, max_dbl_lt_one,
                               DBL_MAX, 1));
   /* does it play nice with INT64_MIN? */
-  tt_i64_op((INT64_MIN + 35), ==,
+  tt_i64_op((INT64_MIN + 35), OP_EQ,
             add_laplace_noise(INT64_MIN, max_dbl_lt_one,
                               noscale_df, noscale_eps));
 
@@ -5014,32 +5499,32 @@ test_util_clamp_double_to_int64(void *arg)
 {
   (void)arg;
 
-  tt_i64_op(INT64_MIN, ==, clamp_double_to_int64(-INFINITY_DBL));
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ, clamp_double_to_int64(-INFINITY_DBL));
+  tt_i64_op(INT64_MIN, OP_EQ,
             clamp_double_to_int64(-1.0 * pow(2.0, 64.0) - 1.0));
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ,
             clamp_double_to_int64(-1.0 * pow(2.0, 63.0) - 1.0));
-  tt_i64_op(((uint64_t) -1) << 53, ==,
+  tt_i64_op(((uint64_t) -1) << 53, OP_EQ,
             clamp_double_to_int64(-1.0 * pow(2.0, 53.0)));
-  tt_i64_op((((uint64_t) -1) << 53) + 1, ==,
+  tt_i64_op((((uint64_t) -1) << 53) + 1, OP_EQ,
             clamp_double_to_int64(-1.0 * pow(2.0, 53.0) + 1.0));
-  tt_i64_op(-1, ==, clamp_double_to_int64(-1.0));
-  tt_i64_op(0, ==, clamp_double_to_int64(-0.9));
-  tt_i64_op(0, ==, clamp_double_to_int64(-0.1));
-  tt_i64_op(0, ==, clamp_double_to_int64(0.0));
-  tt_i64_op(0, ==, clamp_double_to_int64(NAN_DBL));
-  tt_i64_op(0, ==, clamp_double_to_int64(0.1));
-  tt_i64_op(0, ==, clamp_double_to_int64(0.9));
-  tt_i64_op(1, ==, clamp_double_to_int64(1.0));
-  tt_i64_op((((int64_t) 1) << 53) - 1, ==,
+  tt_i64_op(-1, OP_EQ, clamp_double_to_int64(-1.0));
+  tt_i64_op(0, OP_EQ, clamp_double_to_int64(-0.9));
+  tt_i64_op(0, OP_EQ, clamp_double_to_int64(-0.1));
+  tt_i64_op(0, OP_EQ, clamp_double_to_int64(0.0));
+  tt_i64_op(0, OP_EQ, clamp_double_to_int64(NAN_DBL));
+  tt_i64_op(0, OP_EQ, clamp_double_to_int64(0.1));
+  tt_i64_op(0, OP_EQ, clamp_double_to_int64(0.9));
+  tt_i64_op(1, OP_EQ, clamp_double_to_int64(1.0));
+  tt_i64_op((((int64_t) 1) << 53) - 1, OP_EQ,
             clamp_double_to_int64(pow(2.0, 53.0) - 1.0));
-  tt_i64_op(((int64_t) 1) << 53, ==,
+  tt_i64_op(((int64_t) 1) << 53, OP_EQ,
             clamp_double_to_int64(pow(2.0, 53.0)));
-  tt_i64_op(INT64_MAX, ==,
+  tt_i64_op(INT64_MAX, OP_EQ,
             clamp_double_to_int64(pow(2.0, 63.0)));
-  tt_i64_op(INT64_MAX, ==,
+  tt_i64_op(INT64_MAX, OP_EQ,
             clamp_double_to_int64(pow(2.0, 64.0)));
-  tt_i64_op(INT64_MAX, ==, clamp_double_to_int64(INFINITY_DBL));
+  tt_i64_op(INT64_MAX, OP_EQ, clamp_double_to_int64(INFINITY_DBL));
 
  done:
   ;
@@ -5053,7 +5538,7 @@ fd_is_cloexec(tor_socket_t fd)
   int flags = fcntl(fd, F_GETFD, 0);
   return (flags & FD_CLOEXEC) == FD_CLOEXEC;
 }
-#endif
+#endif /* defined(FD_CLOEXEC) */
 
 #ifndef _WIN32
 #define CAN_CHECK_NONBLOCK
@@ -5063,7 +5548,7 @@ fd_is_nonblocking(tor_socket_t fd)
   int flags = fcntl(fd, F_GETFL, 0);
   return (flags & O_NONBLOCK) == O_NONBLOCK;
 }
-#endif
+#endif /* !defined(_WIN32) */
 
 #define ERRNO_IS_EPROTO(e)    (e == SOCK_ERRNO(EPROTONOSUPPORT))
 #define SOCK_ERR_IS_EPROTO(s) ERRNO_IS_EPROTO(tor_socket_errno(s))
@@ -5085,7 +5570,8 @@ test_util_socket(void *arg)
 
   fd1 = tor_open_socket_with_extensions(domain, SOCK_STREAM, 0, 0, 0);
   int err = tor_socket_errno(fd1);
-  if (fd1 < 0 && err == SOCK_ERRNO(EPROTONOSUPPORT)) {
+  if (fd1 < 0 && (err == SOCK_ERRNO(EPROTONOSUPPORT) ||
+                  err == SOCK_ERRNO(EAFNOSUPPORT))) {
     /* Assume we're on an IPv4-only or IPv6-only system, and give up now. */
     goto done;
   }
@@ -5106,13 +5592,13 @@ test_util_socket(void *arg)
   tt_int_op(fd_is_cloexec(fd2), OP_EQ, 0);
   tt_int_op(fd_is_cloexec(fd3), OP_EQ, 1);
   tt_int_op(fd_is_cloexec(fd4), OP_EQ, 1);
-#endif
+#endif /* defined(CAN_CHECK_CLOEXEC) */
 #ifdef CAN_CHECK_NONBLOCK
   tt_int_op(fd_is_nonblocking(fd1), OP_EQ, 0);
   tt_int_op(fd_is_nonblocking(fd2), OP_EQ, 1);
   tt_int_op(fd_is_nonblocking(fd3), OP_EQ, 0);
   tt_int_op(fd_is_nonblocking(fd4), OP_EQ, 1);
-#endif
+#endif /* defined(CAN_CHECK_NONBLOCK) */
 
   tor_assert(tor_close_socket == tor_close_socket__real);
 
@@ -5168,10 +5654,10 @@ is_there_a_localhost(int family)
 
   return result;
 }
-#endif
+#endif /* 0 */
 
 /* Test for socketpair and ersatz_socketpair().  We test them both, since
- * the latter is a tolerably good way to exersize tor_accept_socket(). */
+ * the latter is a tolerably good way to exercise tor_accept_socket(). */
 static void
 test_util_socketpair(void *arg)
 {
@@ -5195,15 +5681,18 @@ test_util_socketpair(void *arg)
      * Assume we're on a machine without 127.0.0.1 or ::1 and give up now. */
     tt_skip();
   }
-#endif
+#endif /* defined(__FreeBSD__) */
   tt_int_op(0, OP_EQ, socketpair_result);
 
   tt_assert(SOCKET_OK(fds[0]));
   tt_assert(SOCKET_OK(fds[1]));
-  tt_int_op(get_n_open_sockets(), OP_EQ, n + 2);
+  if (ersatz)
+    tt_int_op(get_n_open_sockets(), OP_EQ, n);
+  else
+    tt_int_op(get_n_open_sockets(), OP_EQ, n + 2);
 #ifdef CAN_CHECK_CLOEXEC
-  tt_int_op(fd_is_cloexec(fds[0]), OP_EQ, 1);
-  tt_int_op(fd_is_cloexec(fds[1]), OP_EQ, 1);
+  tt_int_op(fd_is_cloexec(fds[0]), OP_EQ, !ersatz);
+  tt_int_op(fd_is_cloexec(fds[1]), OP_EQ, !ersatz);
 #endif
 #ifdef CAN_CHECK_NONBLOCK
   tt_int_op(fd_is_nonblocking(fds[0]), OP_EQ, 0);
@@ -5211,10 +5700,17 @@ test_util_socketpair(void *arg)
 #endif
 
  done:
-  if (SOCKET_OK(fds[0]))
-    tor_close_socket(fds[0]);
-  if (SOCKET_OK(fds[1]))
-    tor_close_socket(fds[1]);
+  if (ersatz) {
+    if (SOCKET_OK(fds[0]))
+      tor_close_socket_simple(fds[0]);
+    if (SOCKET_OK(fds[1]))
+      tor_close_socket_simple(fds[1]);
+  } else {
+    if (SOCKET_OK(fds[0]))
+      tor_close_socket(fds[0]);
+    if (SOCKET_OK(fds[1]))
+      tor_close_socket(fds[1]);
+  }
 }
 
 #undef SOCKET_EPROTO
@@ -5231,15 +5727,15 @@ test_util_max_mem(void *arg)
   tt_int_op(r, OP_EQ, r2);
   tt_uint_op(memory2, OP_EQ, memory1);
 
-  TT_BLATHER(("System memory: "U64_FORMAT, U64_PRINTF_ARG(memory1)));
+  TT_BLATHER(("System memory: %"TOR_PRIuSZ, (memory1)));
 
   if (r==0) {
     /* You have at least a megabyte. */
     tt_uint_op(memory1, OP_GT, (1<<20));
   } else {
     /* You do not have a petabyte. */
-#if SIZEOF_SIZE_T == SIZEOF_UINT64_T
-    tt_u64_op(memory1, OP_LT, (U64_LITERAL(1)<<50));
+#if SIZEOF_SIZE_T >= 8
+    tt_u64_op(memory1, OP_LT, (UINT64_C(1)<<50));
 #endif
   }
 
@@ -5248,47 +5744,74 @@ test_util_max_mem(void *arg)
 }
 
 static void
+test_util_dest_validation_edgecase(void *arg)
+{
+  (void)arg;
+
+  tt_assert(!string_is_valid_dest(NULL));
+  tt_assert(!string_is_valid_dest(""));
+
+  done:
+  return;
+}
+
+static void
 test_util_hostname_validation(void *arg)
 {
   (void)arg;
 
   // Lets try valid hostnames first.
-  tt_assert(string_is_valid_hostname("torproject.org"));
-  tt_assert(string_is_valid_hostname("ocw.mit.edu"));
-  tt_assert(string_is_valid_hostname("i.4cdn.org"));
-  tt_assert(string_is_valid_hostname("stanford.edu"));
-  tt_assert(string_is_valid_hostname("multiple-words-with-hypens.jp"));
+  tt_assert(string_is_valid_nonrfc_hostname("torproject.org"));
+  tt_assert(string_is_valid_nonrfc_hostname("ocw.mit.edu"));
+  tt_assert(string_is_valid_nonrfc_hostname("i.4cdn.org"));
+  tt_assert(string_is_valid_nonrfc_hostname("stanford.edu"));
+  tt_assert(string_is_valid_nonrfc_hostname("multiple-words-with-hypens.jp"));
 
   // Subdomain name cannot start with '-' or '_'.
-  tt_assert(!string_is_valid_hostname("-torproject.org"));
-  tt_assert(!string_is_valid_hostname("subdomain.-domain.org"));
-  tt_assert(!string_is_valid_hostname("-subdomain.domain.org"));
-  tt_assert(!string_is_valid_hostname("___abc.org"));
+  tt_assert(!string_is_valid_nonrfc_hostname("-torproject.org"));
+  tt_assert(!string_is_valid_nonrfc_hostname("subdomain.-domain.org"));
+  tt_assert(!string_is_valid_nonrfc_hostname("-subdomain.domain.org"));
+  tt_assert(!string_is_valid_nonrfc_hostname("___abc.org"));
 
   // Hostnames cannot contain non-alphanumeric characters.
-  tt_assert(!string_is_valid_hostname("%%domain.\\org."));
-  tt_assert(!string_is_valid_hostname("***x.net"));
-  tt_assert(!string_is_valid_hostname("\xff\xffxyz.org"));
-  tt_assert(!string_is_valid_hostname("word1 word2.net"));
+  tt_assert(!string_is_valid_nonrfc_hostname("%%domain.\\org."));
+  tt_assert(!string_is_valid_nonrfc_hostname("***x.net"));
+  tt_assert(!string_is_valid_nonrfc_hostname("\xff\xffxyz.org"));
+  tt_assert(!string_is_valid_nonrfc_hostname("word1 word2.net"));
 
   // Test workaround for nytimes.com stupidity, technically invalid,
   // but we allow it since they are big, even though they are failing to
   // comply with a ~30 year old standard.
-  tt_assert(string_is_valid_hostname("core3_euw1.fabrik.nytimes.com"));
+  tt_assert(string_is_valid_nonrfc_hostname("core3_euw1.fabrik.nytimes.com"));
 
   // Firefox passes FQDNs with trailing '.'s  directly to the SOCKS proxy,
   // which is redundant since the spec states DOMAINNAME addresses are fully
   // qualified.  While unusual, this should be tollerated.
-  tt_assert(string_is_valid_hostname("core9_euw1.fabrik.nytimes.com."));
-  tt_assert(!string_is_valid_hostname("..washingtonpost.is.better.com"));
-  tt_assert(!string_is_valid_hostname("so.is..ft.com"));
-  tt_assert(!string_is_valid_hostname("..."));
+  tt_assert(string_is_valid_nonrfc_hostname("core9_euw1.fabrik.nytimes.com."));
+  tt_assert(!string_is_valid_nonrfc_hostname(
+                                         "..washingtonpost.is.better.com"));
+  tt_assert(!string_is_valid_nonrfc_hostname("so.is..ft.com"));
+  tt_assert(!string_is_valid_nonrfc_hostname("..."));
 
   // XXX: do we allow single-label DNS names?
   // We shouldn't for SOCKS (spec says "contains a fully-qualified domain name"
   // but only test pathologically malformed traling '.' cases for now.
-  tt_assert(!string_is_valid_hostname("."));
-  tt_assert(!string_is_valid_hostname(".."));
+  tt_assert(!string_is_valid_nonrfc_hostname("."));
+  tt_assert(!string_is_valid_nonrfc_hostname(".."));
+
+  // IP address strings are not hostnames.
+  tt_assert(!string_is_valid_nonrfc_hostname("8.8.8.8"));
+  tt_assert(!string_is_valid_nonrfc_hostname("[2a00:1450:401b:800::200e]"));
+  tt_assert(!string_is_valid_nonrfc_hostname("2a00:1450:401b:800::200e"));
+
+  // We allow alphanumeric TLDs. For discussion, see ticket #25055.
+  tt_assert(string_is_valid_nonrfc_hostname("lucky.13"));
+  tt_assert(string_is_valid_nonrfc_hostname("luck.y13"));
+  tt_assert(string_is_valid_nonrfc_hostname("luck.y13."));
+
+  // We allow punycode TLDs. For examples, see
+  // http://data.iana.org/TLD/tlds-alpha-by-domain.txt
+  tt_assert(string_is_valid_nonrfc_hostname("example.xn--l1acc"));
 
   done:
   return;
@@ -5356,7 +5879,7 @@ test_util_get_avail_disk_space(void *arg)
 #else
   tt_i64_op(val, OP_GT, 0); /* You have some space. */
   tt_i64_op(val, OP_LT, ((int64_t)1)<<56); /* You don't have a zebibyte */
-#endif
+#endif /* !defined(HAVE_STATVFS) && !defined(_WIN32) */
 
  done:
   ;
@@ -5406,25 +5929,25 @@ test_util_pwdb(void *arg)
   /* Uncached case. */
   /* Let's assume that we exist. */
   me = tor_getpwuid(getuid());
-  tt_assert(me != NULL);
+  tt_ptr_op(me, OP_NE, NULL);
   name = tor_strdup(me->pw_name);
 
   /* Uncached case */
   me2 = tor_getpwnam(name);
-  tt_assert(me2 != NULL);
+  tt_ptr_op(me2, OP_NE, NULL);
   tt_int_op(me2->pw_uid, OP_EQ, getuid());
 
   /* Cached case */
   me3 = tor_getpwuid(getuid());
-  tt_assert(me3 != NULL);
+  tt_ptr_op(me3, OP_NE, NULL);
   tt_str_op(me3->pw_name, OP_EQ, name);
 
   me3 = tor_getpwnam(name);
-  tt_assert(me3 != NULL);
+  tt_ptr_op(me3, OP_NE, NULL);
   tt_int_op(me3->pw_uid, OP_EQ, getuid());
 
   dir = get_user_homedir(name);
-  tt_assert(dir != NULL);
+  tt_ptr_op(dir, OP_NE, NULL);
 
   /* Try failing cases.  First find a user that doesn't exist by name */
   char randbytes[4];
@@ -5444,7 +5967,7 @@ test_util_pwdb(void *arg)
   /* We should do a LOG_ERR */
   setup_full_capture_of_logs(LOG_ERR);
   dir = get_user_homedir(badname);
-  tt_assert(dir == NULL);
+  tt_ptr_op(dir, OP_EQ, NULL);
   expect_log_msg_containing("not found");
   tt_int_op(smartlist_len(mock_saved_logs()), OP_EQ, 1);
   teardown_capture_of_logs();
@@ -5466,33 +5989,33 @@ test_util_pwdb(void *arg)
   tor_free(dir);
   teardown_capture_of_logs();
 }
-#endif
+#endif /* !defined(_WIN32) */
 
 static void
 test_util_calloc_check(void *arg)
 {
   (void) arg;
   /* Easy cases that are good. */
-  tt_assert(size_mul_check__(0,0));
-  tt_assert(size_mul_check__(0,100));
-  tt_assert(size_mul_check__(100,0));
-  tt_assert(size_mul_check__(100,100));
+  tt_assert(size_mul_check(0,0));
+  tt_assert(size_mul_check(0,100));
+  tt_assert(size_mul_check(100,0));
+  tt_assert(size_mul_check(100,100));
 
   /* Harder cases that are still good. */
-  tt_assert(size_mul_check__(SIZE_MAX, 1));
-  tt_assert(size_mul_check__(1, SIZE_MAX));
-  tt_assert(size_mul_check__(SIZE_MAX / 10, 9));
-  tt_assert(size_mul_check__(11, SIZE_MAX / 12));
+  tt_assert(size_mul_check(SIZE_MAX, 1));
+  tt_assert(size_mul_check(1, SIZE_MAX));
+  tt_assert(size_mul_check(SIZE_MAX / 10, 9));
+  tt_assert(size_mul_check(11, SIZE_MAX / 12));
   const size_t sqrt_size_max_p1 = ((size_t)1) << (sizeof(size_t) * 4);
-  tt_assert(size_mul_check__(sqrt_size_max_p1, sqrt_size_max_p1 - 1));
+  tt_assert(size_mul_check(sqrt_size_max_p1, sqrt_size_max_p1 - 1));
 
   /* Cases that overflow */
-  tt_assert(! size_mul_check__(SIZE_MAX, 2));
-  tt_assert(! size_mul_check__(2, SIZE_MAX));
-  tt_assert(! size_mul_check__(SIZE_MAX / 10, 11));
-  tt_assert(! size_mul_check__(11, SIZE_MAX / 10));
-  tt_assert(! size_mul_check__(SIZE_MAX / 8, 9));
-  tt_assert(! size_mul_check__(sqrt_size_max_p1, sqrt_size_max_p1));
+  tt_assert(! size_mul_check(SIZE_MAX, 2));
+  tt_assert(! size_mul_check(2, SIZE_MAX));
+  tt_assert(! size_mul_check(SIZE_MAX / 10, 11));
+  tt_assert(! size_mul_check(11, SIZE_MAX / 10));
+  tt_assert(! size_mul_check(SIZE_MAX / 8, 9));
+  tt_assert(! size_mul_check(sqrt_size_max_p1, sqrt_size_max_p1));
 
  done:
   ;
@@ -5507,6 +6030,7 @@ test_util_monotonic_time(void *arg)
   monotime_coarse_t mtc1, mtc2;
   uint64_t nsec1, nsec2, usec1, msec1;
   uint64_t nsecc1, nsecc2, usecc1, msecc1;
+  uint32_t stamp1, stamp2;
 
   monotime_init();
 
@@ -5518,6 +6042,7 @@ test_util_monotonic_time(void *arg)
   nsecc1 = monotime_coarse_absolute_nsec();
   usecc1 = monotime_coarse_absolute_usec();
   msecc1 = monotime_coarse_absolute_msec();
+  stamp1 = monotime_coarse_to_stamp(&mtc1);
 
   tor_sleep_msec(200);
 
@@ -5525,6 +6050,7 @@ test_util_monotonic_time(void *arg)
   monotime_coarse_get(&mtc2);
   nsec2 = monotime_absolute_nsec();
   nsecc2 = monotime_coarse_absolute_nsec();
+  stamp2 = monotime_coarse_to_stamp(&mtc2);
 
   /* We need to be a little careful here since we don't know the system load.
    */
@@ -5546,6 +6072,18 @@ test_util_monotonic_time(void *arg)
   tt_u64_op(msecc1, OP_LE, nsecc1 / 1000000 + 10);
   tt_u64_op(usecc1, OP_LE, nsecc1 / 1000 + 10000);
 
+  uint64_t coarse_stamp_diff =
+    monotime_coarse_stamp_units_to_approx_msec(stamp2-stamp1);
+  tt_u64_op(coarse_stamp_diff, OP_GE, 120);
+  tt_u64_op(coarse_stamp_diff, OP_LE, 1200);
+
+  {
+    uint64_t units = monotime_msec_to_approx_coarse_stamp_units(5000);
+    uint64_t ms = monotime_coarse_stamp_units_to_approx_msec(units);
+    tt_u64_op(ms, OP_GE, 4950);
+    tt_u64_op(ms, OP_LT, 5050);
+  }
+
  done:
   ;
 }
@@ -5623,12 +6161,267 @@ test_util_monotonic_time_ratchet(void *arg)
   ;
 }
 
+static void
+test_util_monotonic_time_zero(void *arg)
+{
+  (void) arg;
+  monotime_t t1;
+  monotime_coarse_t ct1;
+  monotime_init();
+  /* Check 1: The current time is not zero. */
+  monotime_get(&t1);
+  monotime_coarse_get(&ct1);
+  tt_assert(!monotime_is_zero(&t1));
+  tt_assert(!monotime_coarse_is_zero(&ct1));
+
+  /* Check 2: The _zero() makes the time zero. */
+  monotime_zero(&t1);
+  monotime_coarse_zero(&ct1);
+  tt_assert(monotime_is_zero(&t1));
+  tt_assert(monotime_coarse_is_zero(&ct1));
+ done:
+  ;
+}
+
+static void
+test_util_monotonic_time_add_msec(void *arg)
+{
+  (void) arg;
+  monotime_t t1, t2;
+  monotime_coarse_t ct1, ct2;
+  monotime_init();
+
+  monotime_get(&t1);
+  monotime_coarse_get(&ct1);
+
+  /* adding zero does nothing */
+  monotime_add_msec(&t2, &t1, 0);
+  monotime_coarse_add_msec(&ct2, &ct1, 0);
+  tt_i64_op(monotime_diff_msec(&t1, &t2), OP_EQ, 0);
+  tt_i64_op(monotime_coarse_diff_msec(&ct1, &ct2), OP_EQ, 0);
+
+  /* Add 1337 msec; see if the diff function agree */
+  monotime_add_msec(&t2, &t1, 1337);
+  monotime_coarse_add_msec(&ct2, &ct1, 1337);
+  tt_i64_op(monotime_diff_msec(&t1, &t2), OP_EQ, 1337);
+  tt_i64_op(monotime_coarse_diff_msec(&ct1, &ct2), OP_EQ, 1337);
+  // The 32-bit variant must be within 1% of the regular one.
+  tt_int_op(monotime_coarse_diff_msec32_(&ct1, &ct2), OP_GT, 1323);
+  tt_int_op(monotime_coarse_diff_msec32_(&ct1, &ct2), OP_LT, 1350);
+
+  /* Add 1337 msec twice more; make sure that any second rollover issues
+   * worked. */
+  monotime_add_msec(&t2, &t2, 1337);
+  monotime_coarse_add_msec(&ct2, &ct2, 1337);
+  monotime_add_msec(&t2, &t2, 1337);
+  monotime_coarse_add_msec(&ct2, &ct2, 1337);
+  tt_i64_op(monotime_diff_msec(&t1, &t2), OP_EQ, 1337*3);
+  tt_i64_op(monotime_coarse_diff_msec(&ct1, &ct2), OP_EQ, 1337*3);
+  tt_int_op(monotime_coarse_diff_msec32_(&ct1, &ct2), OP_GT, 3970);
+  tt_int_op(monotime_coarse_diff_msec32_(&ct1, &ct2), OP_LT, 4051);
+
+ done:
+  ;
+}
+
+static void
+test_util_nowrap_math(void *arg)
+{
+  (void)arg;
+
+  tt_u64_op(0, OP_EQ, tor_add_u32_nowrap(0, 0));
+  tt_u64_op(1, OP_EQ, tor_add_u32_nowrap(0, 1));
+  tt_u64_op(1, OP_EQ, tor_add_u32_nowrap(1, 0));
+  tt_u64_op(4, OP_EQ, tor_add_u32_nowrap(2, 2));
+  tt_u64_op(UINT32_MAX, OP_EQ, tor_add_u32_nowrap(UINT32_MAX-1, 2));
+  tt_u64_op(UINT32_MAX, OP_EQ, tor_add_u32_nowrap(2, UINT32_MAX-1));
+  tt_u64_op(UINT32_MAX, OP_EQ, tor_add_u32_nowrap(UINT32_MAX, UINT32_MAX));
+
+ done:
+  ;
+}
+
+static void
+test_util_htonll(void *arg)
+{
+  (void)arg;
+#ifdef WORDS_BIGENDIAN
+  const uint64_t res_be = 0x8877665544332211;
+#else
+  const uint64_t res_le = 0x1122334455667788;
+#endif
+
+  tt_u64_op(0, OP_EQ, tor_htonll(0));
+  tt_u64_op(0, OP_EQ, tor_ntohll(0));
+  tt_u64_op(UINT64_MAX, OP_EQ, tor_htonll(UINT64_MAX));
+  tt_u64_op(UINT64_MAX, OP_EQ, tor_ntohll(UINT64_MAX));
+
+#ifdef WORDS_BIGENDIAN
+  tt_u64_op(res_be, OP_EQ, tor_htonll(0x8877665544332211));
+  tt_u64_op(res_be, OP_EQ, tor_ntohll(0x8877665544332211));
+#else
+  tt_u64_op(res_le, OP_EQ, tor_htonll(0x8877665544332211));
+  tt_u64_op(res_le, OP_EQ, tor_ntohll(0x8877665544332211));
+#endif /* defined(WORDS_BIGENDIAN) */
+
+ done:
+  ;
+}
+
+static void
+test_util_get_unquoted_path(void *arg)
+{
+  (void)arg;
+
+  char *r = NULL;
+
+  r = get_unquoted_path("\""); // "
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("\"\"\""); // """
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("\\\""); // \"
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("\\\"\\\""); // \"\"
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("A\\B\\C\""); // A\B\C"
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("\"A\\B\\C"); // "A\B\C
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("\"A\\B\"C\""); // "A\B"C"
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("A\\B\"C"); // A\B"C
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("");
+  tt_str_op(r, OP_EQ, "");
+  tor_free(r);
+
+  r = get_unquoted_path("\"\""); // ""
+  tt_str_op(r, OP_EQ, "");
+  tor_free(r);
+
+  r = get_unquoted_path("A\\B\\C"); // A\B\C
+  tt_str_op(r, OP_EQ, "A\\B\\C"); // A\B\C
+  tor_free(r);
+
+  r = get_unquoted_path("\"A\\B\\C\""); // "A\B\C"
+  tt_str_op(r, OP_EQ, "A\\B\\C"); // A\B\C
+  tor_free(r);
+
+  r = get_unquoted_path("\"\\\""); // "\"
+  tt_str_op(r, OP_EQ, "\\"); // \ /* comment to prevent line continuation */
+  tor_free(r);
+
+  r = get_unquoted_path("\"\\\"\""); // "\""
+  tt_str_op(r, OP_EQ, "\""); // "
+  tor_free(r);
+
+  r = get_unquoted_path("\"A\\B\\C\\\"\""); // "A\B\C\""
+  tt_str_op(r, OP_EQ, "A\\B\\C\""); // A\B\C"
+  tor_free(r);
+
+  r = get_unquoted_path("A\\B\\\"C"); // A\B\"C
+  tt_str_op(r, OP_EQ, "A\\B\"C"); // A\B"C
+  tor_free(r);
+
+  r = get_unquoted_path("\"A\\B\\\"C\""); // "A\B\"C"
+  tt_str_op(r, OP_EQ, "A\\B\"C"); // A\B"C
+
+ done:
+  tor_free(r);
+}
+
+static void
+test_util_log_mallinfo(void *arg)
+{
+  (void)arg;
+  char *log1 = NULL, *log2 = NULL, *mem = NULL;
+#ifdef HAVE_MALLINFO
+  setup_capture_of_logs(LOG_INFO);
+  tor_log_mallinfo(LOG_INFO);
+  expect_single_log_msg_containing("mallinfo() said: ");
+  mock_saved_log_entry_t *lg = smartlist_get(mock_saved_logs(), 0);
+  log1 = tor_strdup(lg->generated_msg);
+
+  mock_clean_saved_logs();
+  mem = tor_malloc(8192);
+  tor_log_mallinfo(LOG_INFO);
+  expect_single_log_msg_containing("mallinfo() said: ");
+  lg = smartlist_get(mock_saved_logs(), 0);
+  log2 = tor_strdup(lg->generated_msg);
+
+  /* Make sure that the amount of used memory increased. */
+  const char *used1 = strstr(log1, "uordblks=");
+  const char *used2 = strstr(log2, "uordblks=");
+  tt_assert(used1);
+  tt_assert(used2);
+  used1 += strlen("uordblks=");
+  used2 += strlen("uordblks=");
+
+  int ok1, ok2;
+  char *next1 = NULL, *next2 = NULL;
+  uint64_t mem1 = tor_parse_uint64(used1, 10, 0, UINT64_MAX, &ok1, &next1);
+  uint64_t mem2 = tor_parse_uint64(used2, 10, 0, UINT64_MAX, &ok2, &next2);
+  tt_assert(ok1);
+  tt_assert(ok2);
+  tt_assert(next1);
+  tt_assert(next2);
+  if (mem2 == 0) {
+    /* This is a fake mallinfo that doesn't actually fill in its outputs. */
+    tt_u64_op(mem1, OP_EQ, 0);
+  } else {
+    tt_u64_op(mem1, OP_LT, mem2);
+  }
+#else
+  tt_skip();
+#endif
+ done:
+  teardown_capture_of_logs();
+  tor_free(log1);
+  tor_free(log2);
+  tor_free(mem);
+}
+
 #define UTIL_LEGACY(name)                                               \
   { #name, test_util_ ## name , 0, NULL, NULL }
 
 #define UTIL_TEST(name, flags)                          \
   { #name, test_util_ ## name, flags, NULL, NULL }
 
+#define COMPRESS(name, identifier)              \
+  { "compress/" #name, test_util_compress, 0, &compress_setup,          \
+    (char*)(identifier) }
+
+#define COMPRESS_CONCAT(name, identifier)                               \
+  { "compress_concat/" #name, test_util_decompress_concatenated, 0,     \
+    &compress_setup,                                                    \
+    (char*)(identifier) }
+
+#define COMPRESS_JUNK(name, identifier)                                 \
+  { "compress_junk/" #name, test_util_decompress_junk, 0,               \
+    &compress_setup,                                                    \
+    (char*)(identifier) }
+
+#define COMPRESS_DOS(name, identifier)                                  \
+  { "compress_dos/" #name, test_util_decompress_dos, 0,                 \
+    &compress_setup,                                                    \
+    (char*)(identifier) }
+
 #ifdef _WIN32
 #define UTIL_TEST_NO_WIN(n, f) { #n, NULL, TT_SKIP, NULL, NULL }
 #define UTIL_TEST_WIN_ONLY(n, f) UTIL_TEST(n, (f))
@@ -5637,7 +6430,7 @@ test_util_monotonic_time_ratchet(void *arg)
 #define UTIL_TEST_NO_WIN(n, f) UTIL_TEST(n, (f))
 #define UTIL_TEST_WIN_ONLY(n, f) { #n, NULL, TT_SKIP, NULL, NULL }
 #define UTIL_LEGACY_NO_WIN(n) UTIL_LEGACY(n)
-#endif
+#endif /* defined(_WIN32) */
 
 struct testcase_t util_tests[] = {
   UTIL_LEGACY(time),
@@ -5653,7 +6446,26 @@ struct testcase_t util_tests[] = {
   UTIL_LEGACY(strmisc),
   UTIL_TEST(parse_integer, 0),
   UTIL_LEGACY(pow2),
-  UTIL_LEGACY(gzip),
+  COMPRESS(zlib, "deflate"),
+  COMPRESS(gzip, "gzip"),
+  COMPRESS(lzma, "x-tor-lzma"),
+  COMPRESS(zstd, "x-zstd"),
+  COMPRESS(zstd_nostatic, "x-zstd:nostatic"),
+  COMPRESS(none, "identity"),
+  COMPRESS_CONCAT(zlib, "deflate"),
+  COMPRESS_CONCAT(gzip, "gzip"),
+  COMPRESS_CONCAT(lzma, "x-tor-lzma"),
+  COMPRESS_CONCAT(zstd, "x-zstd"),
+  COMPRESS_CONCAT(zstd_nostatic, "x-zstd:nostatic"),
+  COMPRESS_CONCAT(none, "identity"),
+  COMPRESS_JUNK(zlib, "deflate"),
+  COMPRESS_JUNK(gzip, "gzip"),
+  COMPRESS_JUNK(lzma, "x-tor-lzma"),
+  COMPRESS_DOS(zlib, "deflate"),
+  COMPRESS_DOS(gzip, "gzip"),
+  COMPRESS_DOS(lzma, "x-tor-lzma"),
+  COMPRESS_DOS(zstd, "x-zstd"),
+  COMPRESS_DOS(zstd_nostatic, "x-zstd:nostatic"),
   UTIL_TEST(gzip_compression_bomb, TT_FORK),
   UTIL_LEGACY(datadir),
   UTIL_LEGACY(memarea),
@@ -5670,14 +6482,16 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(clamp_double_to_int64, 0),
   UTIL_TEST(find_str_at_start_of_line, 0),
   UTIL_TEST(string_is_C_identifier, 0),
+  UTIL_TEST(string_is_utf8, 0),
   UTIL_TEST(asprintf, 0),
   UTIL_TEST(listdir, 0),
   UTIL_TEST(parent_dir, 0),
   UTIL_TEST(ftruncate, 0),
+  UTIL_TEST(nowrap_math, 0),
   UTIL_TEST(num_cpus, 0),
   UTIL_TEST_WIN_ONLY(load_win_lib, 0),
   UTIL_TEST_NO_WIN(exit_status, 0),
-  UTIL_TEST_NO_WIN(fgets_eagain, 0),
+  UTIL_TEST_NO_WIN(string_from_pipe, 0),
   UTIL_TEST(format_hex_number, 0),
   UTIL_TEST(format_dec_number, 0),
   UTIL_TEST(join_win_cmdline, 0),
@@ -5708,6 +6522,7 @@ struct testcase_t util_tests[] = {
     &passthrough_setup, (void*)"1" },
   UTIL_TEST(max_mem, 0),
   UTIL_TEST(hostname_validation, 0),
+  UTIL_TEST(dest_validation_edgecase, 0),
   UTIL_TEST(ipv4_validation, 0),
   UTIL_TEST(writepid, 0),
   UTIL_TEST(get_avail_disk_space, 0),
@@ -5716,6 +6531,10 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(calloc_check, 0),
   UTIL_TEST(monotonic_time, 0),
   UTIL_TEST(monotonic_time_ratchet, TT_FORK),
+  UTIL_TEST(monotonic_time_zero, 0),
+  UTIL_TEST(monotonic_time_add_msec, 0),
+  UTIL_TEST(htonll, 0),
+  UTIL_TEST(get_unquoted_path, 0),
+  UTIL_TEST(log_mallinfo, 0),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_util_format.c b/src/test/test_util_format.c
index 63a668238c..d344d0e95c 100644
--- a/src/test/test_util_format.c
+++ b/src/test/test_util_format.c
@@ -1,35 +1,25 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
-#include "or.h"
+#include "core/or/or.h"
 
-#include "test.h"
+#include "test/test.h"
 
+#include "lib/crypt_ops/crypto_rand.h"
 #define UTIL_FORMAT_PRIVATE
-#include "util_format.h"
+#include "lib/encoding/binascii.h"
 
 #define NS_MODULE util_format
 
-#if !defined(HAVE_HTONLL) && !defined(htonll)
-#ifdef WORDS_BIGENDIAN
-#define htonll(x) (x)
-#else
-static uint64_t
-htonll(uint64_t a)
-{
-  return htonl((uint32_t)(a>>32)) | (((uint64_t)htonl((uint32_t)a))<<32);
-}
-#endif
-#endif
-
 static void
 test_util_format_unaligned_accessors(void *ignored)
 {
   (void)ignored;
   char buf[9] = "onionsoup"; // 6f6e696f6e736f7570
 
-  tt_u64_op(get_uint64(buf+1), OP_EQ, htonll(U64_LITERAL(0x6e696f6e736f7570)));
+  tt_u64_op(get_uint64(buf+1), OP_EQ,
+      tor_htonll(UINT64_C(0x6e696f6e736f7570)));
   tt_uint_op(get_uint32(buf+1), OP_EQ, htonl(0x6e696f6e));
   tt_uint_op(get_uint16(buf+1), OP_EQ, htons(0x6e69));
   tt_uint_op(get_uint8(buf+1), OP_EQ, 0x6e);
@@ -43,7 +33,7 @@ test_util_format_unaligned_accessors(void *ignored)
   set_uint32(buf+1, htonl(0x78696465));
   tt_mem_op(buf, OP_EQ, "oxidestop", 9);
 
-  set_uint64(buf+1, htonll(U64_LITERAL(0x6266757363617465)));
+  set_uint64(buf+1, tor_htonll(UINT64_C(0x6266757363617465)));
   tt_mem_op(buf, OP_EQ, "obfuscate", 9);
  done:
   ;
@@ -144,48 +134,54 @@ test_util_format_base64_encode(void *ignored)
 }
 
 static void
-test_util_format_base64_decode_nopad(void *ignored)
+test_util_format_base64_decode_oddsize(void *ignored)
 {
   (void)ignored;
   int res;
   int i;
   char *src;
-  uint8_t *dst, *real_dst;
-  uint8_t expected[] = {0x65, 0x78, 0x61, 0x6D, 0x70, 0x6C, 0x65};
+  char *dst, real_dst[7];
+  char expected[] = {0x65, 0x78, 0x61, 0x6D, 0x70, 0x6C, 0x65};
   char real_src[] = "ZXhhbXBsZQ";
+  char expected40[] = "testing40characteroddsizebase64encoding!";
+  char src40[] = "dGVzdGluZzQwY2hhcmFjdGVyb2Rkc2l6ZWJhc2U2NGVuY29kaW5nIQ";
+  char pad40[] = "dGVzdGluZzQwY2hhcmFjdGVyb2Rkc2l6ZWJhc2U2NGVuY29kaW5nIQ==";
 
   src = tor_malloc_zero(256);
   dst = tor_malloc_zero(1000);
-  real_dst = tor_malloc_zero(10);
 
   for (i=0;i<256;i++) {
     src[i] = (char)i;
   }
 
-  res = base64_decode_nopad(dst, 1, src, SIZE_T_CEILING);
-  tt_int_op(res, OP_EQ, -1);
-
-  res = base64_decode_nopad(dst, 1, src, 5);
+  res = base64_decode(dst, 1, src, 5);
   tt_int_op(res, OP_EQ, -1);
 
   const char *s = "SGVsbG8gd29ybGQ";
-  res = base64_decode_nopad(dst, 1000, s, strlen(s));
+  res = base64_decode(dst, 1000, s, strlen(s));
   tt_int_op(res, OP_EQ, 11);
   tt_mem_op(dst, OP_EQ, "Hello world", 11);
 
   s = "T3BhIG11bmRv";
-  res = base64_decode_nopad(dst, 9, s, strlen(s));
+  res = base64_decode(dst, 9, s, strlen(s));
   tt_int_op(res, OP_EQ, 9);
   tt_mem_op(dst, OP_EQ, "Opa mundo", 9);
 
-  res = base64_decode_nopad(real_dst, 10, real_src, 10);
+  res = base64_decode(real_dst, sizeof(real_dst), real_src, 10);
   tt_int_op(res, OP_EQ, 7);
   tt_mem_op(real_dst, OP_EQ, expected, 7);
 
+  res = base64_decode(dst, 40, src40, strlen(src40));
+  tt_int_op(res, OP_EQ, 40);
+  tt_mem_op(dst, OP_EQ, expected40, 40);
+
+  res = base64_decode(dst, 40, pad40, strlen(pad40));
+  tt_int_op(res, OP_EQ, 40);
+  tt_mem_op(dst, OP_EQ, expected40, 40);
+
  done:
   tor_free(src);
   tor_free(dst);
-  tor_free(real_dst);
 }
 
 static void
@@ -207,10 +203,10 @@ test_util_format_base64_decode(void *ignored)
     src[i] = (char)i;
   }
 
-  res = base64_decode(dst, 1, src, SIZE_T_CEILING);
+  res = base64_decode(dst, 1, src, 100);
   tt_int_op(res, OP_EQ, -1);
 
-  res = base64_decode(dst, SIZE_T_CEILING+1, src, 10);
+  res = base64_decode(dst, 1, real_src, 10);
   tt_int_op(res, OP_EQ, -1);
 
   const char *s = "T3BhIG11bmRv";
@@ -350,7 +346,7 @@ test_util_format_base32_decode(void *arg)
     const char *src = "mjwgc2dcnrswqmjs";
 
     ret = base32_decode(dst, strlen(expected), src, strlen(src));
-    tt_int_op(ret, ==, 0);
+    tt_int_op(ret, OP_EQ, 0);
     tt_str_op(expected, OP_EQ, dst);
   }
 
@@ -361,7 +357,7 @@ test_util_format_base32_decode(void *arg)
     const char *src = "mjwgc2dcnrswq";
 
     ret = base32_decode(dst, strlen(expected), src, strlen(src));
-    tt_int_op(ret, ==, 0);
+    tt_int_op(ret, OP_EQ, 0);
     tt_mem_op(expected, OP_EQ, dst, strlen(expected));
   }
 
@@ -369,20 +365,48 @@ test_util_format_base32_decode(void *arg)
   {
     /* Invalid character '#'. */
     ret = base32_decode(dst, real_dstlen, "#abcde", 6);
-    tt_int_op(ret, ==, -1);
+    tt_int_op(ret, OP_EQ, -1);
     /* Make sure the destination buffer has been zeroed even on error. */
-    tt_int_op(tor_mem_is_zero(dst, real_dstlen), ==, 1);
+    tt_int_op(tor_mem_is_zero(dst, real_dstlen), OP_EQ, 1);
   }
 
  done:
   tor_free(dst);
 }
 
+static void
+test_util_format_encoded_size(void *arg)
+{
+  (void)arg;
+  uint8_t inbuf[256];
+  char outbuf[1024];
+  unsigned i;
+
+  crypto_rand((char *)inbuf, sizeof(inbuf));
+  for (i = 0; i <= sizeof(inbuf); ++i) {
+    /* XXXX (Once the return values are consistent, check them too.) */
+
+    base32_encode(outbuf, sizeof(outbuf), (char *)inbuf, i);
+    /* The "+ 1" below is an API inconsistency. */
+    tt_int_op(strlen(outbuf) + 1, OP_EQ, base32_encoded_size(i));
+
+    base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i, 0);
+    tt_int_op(strlen(outbuf), OP_EQ, base64_encode_size(i, 0));
+    base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i,
+                  BASE64_ENCODE_MULTILINE);
+    tt_int_op(strlen(outbuf), OP_EQ,
+              base64_encode_size(i, BASE64_ENCODE_MULTILINE));
+  }
+
+ done:
+  ;
+}
+
 struct testcase_t util_format_tests[] = {
   { "unaligned_accessors", test_util_format_unaligned_accessors, 0,
     NULL, NULL },
   { "base64_encode", test_util_format_base64_encode, 0, NULL, NULL },
-  { "base64_decode_nopad", test_util_format_base64_decode_nopad, 0,
+  { "base64_decode_oddsize", test_util_format_base64_decode_oddsize, 0,
     NULL, NULL },
   { "base64_decode", test_util_format_base64_decode, 0, NULL, NULL },
   { "base16_decode", test_util_format_base16_decode, 0, NULL, NULL },
@@ -390,6 +414,7 @@ struct testcase_t util_format_tests[] = {
     NULL, NULL },
   { "base32_decode", test_util_format_base32_decode, 0,
     NULL, NULL },
+  { "encoded_size", test_util_format_encoded_size, 0, NULL, NULL },
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_util_process.c b/src/test/test_util_process.c
index 4e75b97f3d..4d04eb6dfc 100644
--- a/src/test/test_util_process.c
+++ b/src/test/test_util_process.c
@@ -1,15 +1,15 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define UTIL_PROCESS_PRIVATE
 #include "orconfig.h"
-#include "or.h"
+#include "core/or/or.h"
 
-#include "test.h"
+#include "test/test.h"
 
-#include "util_process.h"
+#include "lib/process/waitpid.h"
 
-#include "log_test_helpers.h"
+#include "test/log_test_helpers.h"
 
 #ifndef _WIN32
 #define NS_MODULE util_process
@@ -67,7 +67,7 @@ test_util_process_clear_waitpid_callback(void *ignored)
  done:
   teardown_capture_of_logs();
 }
-#endif /* _WIN32 */
+#endif /* !defined(_WIN32) */
 
 #ifndef _WIN32
 #define TEST(name) { #name, test_util_process_##name, 0, NULL, NULL }
diff --git a/src/test/test_util_slow.c b/src/test/test_util_slow.c
index 1e7160598c..29e30eaa11 100644
--- a/src/test/test_util_slow.c
+++ b/src/test/test_util_slow.c
@@ -1,15 +1,21 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 #define UTIL_PRIVATE
-#include "util.h"
-#include "util_process.h"
-#include "crypto.h"
-#include "torlog.h"
-#include "test.h"
+#define SUBPROCESS_PRIVATE
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "lib/log/log.h"
+#include "lib/process/subprocess.h"
+#include "lib/process/waitpid.h"
+#include "lib/string/printf.h"
+#include "lib/time/compat_time.h"
+#include "test/test.h"
+
+#include <errno.h>
+#include <string.h>
 
 #ifndef BUILDDIR
 #define BUILDDIR "."
@@ -22,14 +28,14 @@
 #else
 #define TEST_CHILD (BUILDDIR "/src/test/test-child")
 #define EOL "\n"
-#endif
+#endif /* defined(_WIN32) */
 
 #ifdef _WIN32
 /* I've assumed Windows doesn't have the gap between fork and exec
  * that causes the race condition on unix-like platforms */
 #define MATCH_PROCESS_STATUS(s1,s2)         ((s1) == (s2))
 
-#else
+#else /* !(defined(_WIN32)) */
 /* work around a race condition of the timing of SIGCHLD handler updates
  * to the process_handle's fields, and checks of those fields
  *
@@ -46,7 +52,7 @@
      ||((s2) == PROCESS_STATUS_RUNNING_OR_NOTRUNNING      \
         && IS_RUNNING_OR_NOTRUNNING(s1)))
 
-#endif // _WIN32
+#endif /* defined(_WIN32) */
 
 /** Helper function for testing tor_spawn_background */
 static void
@@ -78,7 +84,7 @@ run_util_spawn_background(const char *argv[], const char *expected_out,
     return;
   }
 
-  tt_assert(process_handle != NULL);
+  tt_ptr_op(process_handle, OP_NE, NULL);
 
   /* When a spawned process forks, fails, then exits very quickly,
    * (this typically occurs when exec fails)
@@ -102,7 +108,7 @@ run_util_spawn_background(const char *argv[], const char *expected_out,
    * that is, PROCESS_STATUS_RUNNING_OR_NOTRUNNING */
   tt_assert(process_handle->waitpid_cb != NULL
               || expected_status == PROCESS_STATUS_RUNNING_OR_NOTRUNNING);
-#endif
+#endif /* !defined(_WIN32) */
 
 #ifdef _WIN32
   tt_assert(process_handle->stdout_pipe != INVALID_HANDLE_VALUE);
@@ -112,7 +118,7 @@ run_util_spawn_background(const char *argv[], const char *expected_out,
   tt_assert(process_handle->stdout_pipe >= 0);
   tt_assert(process_handle->stderr_pipe >= 0);
   tt_assert(process_handle->stdin_pipe >= 0);
-#endif
+#endif /* defined(_WIN32) */
 
   /* Check stdout */
   pos = tor_read_all_from_process_stdout(process_handle, stdout_buf,
@@ -178,7 +184,7 @@ test_util_spawn_background_fail(void *ptr)
   /* TODO: Once we can signal failure to exec, set this to be
    * PROCESS_STATUS_RUNNING_OR_ERROR */
   const int expected_status = PROCESS_STATUS_RUNNING_OR_NOTRUNNING;
-#endif
+#endif /* defined(_WIN32) */
 
   memset(expected_out, 0xf0, sizeof(expected_out));
   memset(code, 0xf0, sizeof(code));
@@ -242,9 +248,9 @@ test_util_spawn_background_partial_read_impl(int exit_early)
 #else
     /* Check that we didn't read the end of file last time */
     tt_assert(!eof);
-    pos = tor_read_all_handle(process_handle->stdout_handle, stdout_buf,
+    pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
                               sizeof(stdout_buf) - 1, NULL, &eof);
-#endif
+#endif /* defined(_WIN32) */
     log_info(LD_GENERAL, "tor_read_all_handle() returned %d", (int)pos);
 
     /* We would have blocked, keep on trying */
@@ -270,17 +276,17 @@ test_util_spawn_background_partial_read_impl(int exit_early)
                             sizeof(stdout_buf) - 1,
                             process_handle);
   tt_int_op(0,OP_EQ, pos);
-#else
+#else /* !(defined(_WIN32)) */
   if (!eof) {
     /* We should have got all the data, but maybe not the EOF flag */
-    pos = tor_read_all_handle(process_handle->stdout_handle, stdout_buf,
+    pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
                               sizeof(stdout_buf) - 1,
                               process_handle, &eof);
     tt_int_op(0,OP_EQ, pos);
     tt_assert(eof);
   }
   /* Otherwise, we got the EOF on the last read */
-#endif
+#endif /* defined(_WIN32) */
 
   /* Check it terminated correctly */
   retval = tor_get_exit_code(process_handle, 1, &exit_code);
@@ -351,7 +357,7 @@ test_util_spawn_background_waitpid_notify(void *arg)
   }
   tt_int_op(ms_timer, OP_GT, 0);
   tt_ptr_op(process_handle->waitpid_cb, OP_EQ, NULL);
-#endif
+#endif /* !defined(_WIN32) */
 
   ms_timer = 30*1000;
   while (((retval = tor_get_exit_code(process_handle, 0, &exit_code))
@@ -388,4 +394,3 @@ struct testcase_t slow_util_tests[] = {
   UTIL_TEST(spawn_background_waitpid_notify, 0),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_voting_schedule.c b/src/test/test_voting_schedule.c
new file mode 100644
index 0000000000..ba4d53a4ae
--- /dev/null
+++ b/src/test/test_voting_schedule.c
@@ -0,0 +1,64 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#include "core/or/or.h"
+#include "feature/dircommon/voting_schedule.h"
+
+#include "test/test.h"
+
+static void
+test_voting_schedule_interval_start(void *arg)
+{
+#define next_interval voting_schedule_get_start_of_next_interval
+  (void)arg;
+  char buf[ISO_TIME_LEN+1];
+
+  // Midnight UTC tonight (as I am writing this test)
+  const time_t midnight = 1525651200;
+  format_iso_time(buf, midnight);
+  tt_str_op(buf, OP_EQ, "2018-05-07 00:00:00");
+
+  /* Some simple tests with a 50-minute voting interval */
+
+  tt_i64_op(next_interval(midnight, 3000, 0), OP_EQ,
+            midnight+3000);
+
+  tt_i64_op(next_interval(midnight+100, 3000, 0), OP_EQ,
+            midnight+3000);
+
+  tt_i64_op(next_interval(midnight+3000, 3000, 0), OP_EQ,
+            midnight+6000);
+
+  tt_i64_op(next_interval(midnight+3001, 3000, 0), OP_EQ,
+            midnight+6000);
+
+  /* Make sure that we roll around properly at midnight */
+  tt_i64_op(next_interval(midnight+83000, 3000, 0), OP_EQ,
+            midnight+84000);
+
+  /* We start fresh at midnight UTC, even if there are leftover seconds. */
+  tt_i64_op(next_interval(midnight+84005, 3000, 0), OP_EQ,
+            midnight+86400);
+
+  /* Now try with offsets.  (These are only used for test networks.) */
+  tt_i64_op(next_interval(midnight, 3000, 99), OP_EQ,
+            midnight+99);
+
+  tt_i64_op(next_interval(midnight+100, 3000, 99), OP_EQ,
+            midnight+3099);
+
+ done:
+  ;
+#undef next_interval
+}
+
+#define VS(name,flags)                                          \
+  { #name, test_voting_schedule_##name, (flags), NULL, NULL }
+
+struct testcase_t voting_schedule_tests[] = {
+  VS(interval_start, 0),
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_workqueue.c b/src/test/test_workqueue.c
index ccb8d0c8ca..c58634da5c 100644
--- a/src/test/test_workqueue.c
+++ b/src/test/test_workqueue.c
@@ -1,18 +1,20 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#include "or.h"
-#include "compat_threads.h"
-#include "onion.h"
-#include "workqueue.h"
-#include "crypto.h"
-#include "crypto_curve25519.h"
-#include "compat_libevent.h"
+#include "core/or/or.h"
+#include "lib/thread/threads.h"
+#include "core/or/onion.h"
+#include "lib/evloop/workqueue.h"
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/net/alertsock.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/intmath/weakrng.h"
+#include "lib/crypt_ops/crypto_init.h"
 
 #include <stdio.h>
-#include <event2/event.h>
 
 #define MAX_INFLIGHT (1<<16)
 
@@ -61,9 +63,9 @@ mark_handled(int serial)
   tor_assert(! bitarray_is_set(handled, serial));
   bitarray_set(handled, serial);
   tor_mutex_release(&bitmap_mutex);
-#else
+#else /* !(defined(TRACK_RESPONSES)) */
   (void)serial;
-#endif
+#endif /* defined(TRACK_RESPONSES) */
 }
 
 static workqueue_reply_t
@@ -159,6 +161,7 @@ static tor_weak_rng_t weak_rng;
 static int n_sent = 0;
 static int rsa_sent = 0;
 static int ecdh_sent = 0;
+static int n_received_previously = 0;
 static int n_received = 0;
 static int no_shutdown = 0;
 
@@ -200,7 +203,9 @@ add_work(threadpool_t *tp)
     crypto_rand((char*)w->msg, 20);
     w->msglen = 20;
     ++rsa_sent;
-    return threadpool_queue_work(tp, workqueue_do_rsa, handle_reply, w);
+    return threadpool_queue_work_priority(tp,
+                                          WQ_PRI_MED,
+                                          workqueue_do_rsa, handle_reply, w);
   } else {
     ecdh_work_t *w = tor_malloc_zero(sizeof(*w));
     w->serial = n_sent++;
@@ -222,18 +227,24 @@ add_n_work_items(threadpool_t *tp, int n)
   workqueue_entry_t **to_cancel;
   workqueue_entry_t *ent;
 
-  to_cancel = tor_malloc(sizeof(workqueue_entry_t*) * opt_n_cancel);
+  // We'll choose randomly which entries to cancel.
+  to_cancel = tor_calloc(opt_n_cancel, sizeof(workqueue_entry_t*));
 
   while (n_queued++ < n) {
     ent = add_work(tp);
     if (! ent) {
       puts("Z");
-      tor_event_base_loopexit(tor_libevent_get_base(), NULL);
+      tor_libevent_exit_loop_after_delay(tor_libevent_get_base(), NULL);
       return -1;
     }
-    if (n_try_cancel < opt_n_cancel &&
-        tor_weak_random_range(&weak_rng, n) < opt_n_cancel) {
+
+    if (n_try_cancel < opt_n_cancel) {
       to_cancel[n_try_cancel++] = ent;
+    } else {
+      int p = tor_weak_random_range(&weak_rng, n_queued);
+      if (p < n_try_cancel) {
+        to_cancel[p] = ent;
+      }
     }
   }
 
@@ -254,19 +265,13 @@ add_n_work_items(threadpool_t *tp, int n)
 static int shutting_down = 0;
 
 static void
-replysock_readable_cb(tor_socket_t sock, short what, void *arg)
+replysock_readable_cb(threadpool_t *tp)
 {
-  threadpool_t *tp = arg;
-  replyqueue_t *rq = threadpool_get_replyqueue(tp);
-
-  int old_r = n_received;
-  (void) sock;
-  (void) what;
-
-  replyqueue_process(rq);
-  if (old_r == n_received)
+  if (n_received_previously == n_received)
     return;
 
+  n_received_previously = n_received;
+
   if (opt_verbose) {
     printf("%d / %d", n_received, n_sent);
     if (opt_n_cancel)
@@ -286,7 +291,7 @@ replysock_readable_cb(tor_socket_t sock, short what, void *arg)
   }
   puts("");
   tor_mutex_release(&bitmap_mutex);
-#endif
+#endif /* defined(TRACK_RESPONSES) */
 
   if (n_sent - (n_received+n_successful_cancel) < opt_n_lowwater) {
     int n_to_send = n_received + opt_n_inflight - n_sent;
@@ -306,7 +311,7 @@ replysock_readable_cb(tor_socket_t sock, short what, void *arg)
                           handle_reply_shutdown, NULL);
     {
       struct timeval limit = { 2, 0 };
-      tor_event_base_loopexit(tor_libevent_get_base(), &limit);
+      tor_libevent_exit_loop_after_delay(tor_libevent_get_base(), &limit);
     }
   }
 }
@@ -335,7 +340,6 @@ main(int argc, char **argv)
   threadpool_t *tp;
   int i;
   tor_libevent_cfg evcfg;
-  struct event *ev;
   uint32_t as_flags = 0;
 
   for (i = 1; i < argc; ++i) {
@@ -409,18 +413,18 @@ main(int argc, char **argv)
   memset(&evcfg, 0, sizeof(evcfg));
   tor_libevent_initialize(&evcfg);
 
-  ev = tor_event_new(tor_libevent_get_base(),
-                     replyqueue_get_socket(rq), EV_READ|EV_PERSIST,
-                     replysock_readable_cb, tp);
-
-  event_add(ev, NULL);
+  {
+    int r = threadpool_register_reply_event(tp,
+                                            replysock_readable_cb);
+    tor_assert(r == 0);
+  }
 
 #ifdef TRACK_RESPONSES
   handled = bitarray_init_zero(opt_n_items);
   received = bitarray_init_zero(opt_n_items);
   tor_mutex_init(&bitmap_mutex);
   handled_len = opt_n_items;
-#endif
+#endif /* defined(TRACK_RESPONSES) */
 
   for (i = 0; i < opt_n_inflight; ++i) {
     if (! add_work(tp)) {
@@ -431,10 +435,10 @@ main(int argc, char **argv)
 
   {
     struct timeval limit = { 180, 0 };
-    tor_event_base_loopexit(tor_libevent_get_base(), &limit);
+    tor_libevent_exit_loop_after_delay(tor_libevent_get_base(), &limit);
   }
 
-  event_base_loop(tor_libevent_get_base(), 0);
+  tor_libevent_run_event_loop(tor_libevent_get_base(), 0);
 
   if (n_sent != opt_n_items || n_received+n_successful_cancel != n_sent) {
     printf("%d vs %d\n", n_sent, opt_n_items);
@@ -449,4 +453,3 @@ main(int argc, char **argv)
     return 0;
   }
 }
-
diff --git a/src/test/test_x509.c b/src/test/test_x509.c
new file mode 100644
index 0000000000..792849ae4b
--- /dev/null
+++ b/src/test/test_x509.c
@@ -0,0 +1,205 @@
+/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define TOR_X509_PRIVATE
+#include "orconfig.h"
+
+#ifdef _WIN32
+#include <winsock2.h>
+#endif
+#include <math.h>
+#include <stddef.h>
+
+#include "lib/cc/compat_compiler.h"
+
+#include "core/or/or.h"
+#include "lib/log/log.h"
+#include "app/config/config.h"
+#include "lib/tls/x509.h"
+#include "lib/tls/x509_internal.h"
+#include "app/config/or_state_st.h"
+
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+
+#include "tinytest.h"
+
+/* A mock replacement for crypto_digest that always fails. */
+static int
+mock_failing_digest(char *digest, const char *m, size_t len)
+{
+  (void)digest;
+  (void)m;
+  (void)len;
+  return -1;
+}
+
+static void
+test_x509_cert_new_failing_digest(void *arg)
+{
+  (void)arg;
+  crypto_pk_t *pk1=NULL, *pk2=NULL;
+  tor_x509_cert_impl_t *impl = NULL;
+  tor_x509_cert_t *cert = NULL;
+  pk1 = pk_generate(0);
+  pk2 = pk_generate(1);
+
+  impl = tor_tls_create_certificate(pk1, pk2, "hello", "world", 86400*100);
+  tt_assert(impl);
+  MOCK(crypto_digest, mock_failing_digest);
+
+  setup_full_capture_of_logs(LOG_WARN);
+  cert = tor_x509_cert_new(impl);
+  tt_assert(!cert);
+  expect_log_msg_containing("Couldn't wrap encoded X509 certificate");
+  expect_log_msg_containing("unable to compute digests of certificate key");
+
+ done:
+  crypto_pk_free(pk1);
+  crypto_pk_free(pk2);
+  UNMOCK(crypto_digest);
+  teardown_capture_of_logs();
+}
+
+static tor_x509_cert_t *
+cert_from_der64(const char *der64)
+{
+  size_t der64len = strlen(der64);
+  unsigned char *der = tor_malloc_zero(der64len);
+  int derlen;
+  tor_x509_cert_t *cert = NULL;
+
+  derlen = base64_decode((char*)der, der64len,
+                         der64, der64len);
+  if (derlen >= 0)
+    cert = tor_x509_cert_decode(der, derlen);
+  tor_free(der);
+  return cert;
+}
+
+static void
+test_x509_consume_ec_cert(void *arg)
+{
+  (void)arg;
+  /* This is a small self-signed EC certificate. */
+  const char certificate[] =
+    "MIIBEzCBugIJAIdl5svgOZ0OMAoGCCqGSM49BAMCMBIxEDAOBgNVBAMMB1Rlc3Rp\n"
+    "bmcwHhcNMTgwODIzMTcyMzI1WhcNMTkwODIzMTcyMzI1WjASMRAwDgYDVQQDDAdU\n"
+    "ZXN0aW5nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAExMDpnRc0Btic3tIyCKNE\n"
+    "iNY4j4gzcaYzS2sTYRoVK3RAukG29Qg6/c8e8XcnsSquU4fItYxDRbi/3nhYk4CP\n"
+    "GDAKBggqhkjOPQQDAgNIADBFAiA0h1q03C2xlONUgAOonJLrlV1SUtMeKDxNsxsU\n"
+    "+FSPvQIhAM7kY9Tlt0ELmyMnORPp1VJieXn/qhL5VoxGxSedTbny\n";
+  const time_t now = 1535045321; /* when I'm writing this test. */
+  tor_x509_cert_t *cert = cert_from_der64(certificate);
+  crypto_pk_t *key = NULL;
+  tt_assert(cert);
+
+  key = tor_tls_cert_get_key(cert);
+  tt_ptr_op(NULL, OP_EQ, key); // Can't get an RSA key out of an EC cert.
+
+  /* It's a self-signed cert -- make sure it signed itself. */
+  tt_assert(tor_tls_cert_is_valid(LOG_ERR, cert, cert, now, 0));
+
+  /* Make sure we detect its key as non-RSA1024 */
+  setup_capture_of_logs(LOG_INFO);
+  tt_assert(! tor_tls_cert_is_valid(LOG_INFO, cert, cert, now, 1));
+  expect_log_msg_containing("Key is not RSA1024");
+
+ done:
+  tor_x509_cert_free(cert);
+  crypto_pk_free(key);
+  teardown_capture_of_logs();
+}
+
+static void
+test_x509_reject_tiny_keys(void *arg)
+{
+  (void)arg;
+  const char *certificates[] = {
+    /* Self-signed RSA512 */
+   "MIIBXDCCAQYCCQDKikjJYZI5uDANBgkqhkiG9w0BAQsFADA1MRUwEwYDVQQHDAxE\n"
+   "ZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQgQ29tcGFueSBMdGQwHhcNMTgw\n"
+   "ODIzMTczNjQ4WhcNMTkwODIzMTczNjQ4WjA1MRUwEwYDVQQHDAxEZWZhdWx0IENp\n"
+   "dHkxHDAaBgNVBAoME0RlZmF1bHQgQ29tcGFueSBMdGQwXDANBgkqhkiG9w0BAQEF\n"
+   "AANLADBIAkEAqOvVKzrSpmKOTNqDzBG/iZrUdhCrMRsymFXyIScJcdsyn7jB8RMy\n"
+   "fbHqG8EqB8HHLU/eqt/+zhh2w08Lx3+5QwIDAQABMA0GCSqGSIb3DQEBCwUAA0EA\n"
+   "RSCq0sNbD9uWfcBqF0U4MtfFjU5x+RQQCeBVtAzwC9bggSILKZfB9XUvtGh6vqig\n",
+   /* Self-signed secp112r2 */
+   "MIIBLTCB+QIJAI0LtN9uWxy3MAoGCCqGSM49BAMCMEUxCzAJBgNVBAYTAkFVMRMw\n"
+   "EQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0\n"
+   "eSBMdGQwHhcNMTgwODIzMTc0MTQ4WhcNMTkwODIzMTc0MTQ4WjBFMQswCQYDVQQG\n"
+   "EwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lk\n"
+   "Z2l0cyBQdHkgTHRkMDIwEAYHKoZIzj0CAQYFK4EEAAcDHgAEf7dFHo7xhCtIcgyo\n"
+   "Px+IDcUUlntZCtar6V4O0zAKBggqhkjOPQQDAgMjADAgAg4yhBJMEmpkNbZU95Zf\n"
+   "uwIOJAan4J1ETxUII1RrGmw=\n"
+  };
+  const time_t now = 1535046182;
+  tor_x509_cert_t *cert = NULL;
+
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(certificates); ++i) {
+    cert = cert_from_der64(certificates[i]);
+    /* It might parse okay, depending on our version of NSS or OpenSSL. */
+    if (cert == NULL)
+      continue;
+    /* But it should not validate. */
+    tt_assert(! tor_tls_cert_is_valid(LOG_INFO, cert, cert, now, 0));
+    tor_x509_cert_free(cert);
+  }
+
+ done:
+  tor_x509_cert_free(cert);
+}
+
+static void
+test_x509_expiration(void *arg)
+{
+  (void)arg;
+  /* a 365-day RSA2048 cert, created between 0 and 60 minutes before "now" */
+  const char certificate[] =
+    "MIICzjCCAbYCCQDxIONWIQ9OGDANBgkqhkiG9w0BAQsFADApMQswCQYDVQQGEwJV\n"
+    "UzEaMBgGA1UEAwwRSW50ZXJlc3RpbmcgdGltZXMwHhcNMTgwODIzMTc1NTE4WhcN\n"
+    "MTkwODIzMTc1NTE4WjApMQswCQYDVQQGEwJVUzEaMBgGA1UEAwwRSW50ZXJlc3Rp\n"
+    "bmcgdGltZXMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQD0Blz1fBii\n"
+    "OffpFlzMrmfPah/vkPcNrwoyx5YiosbHErYUpqdCtfNb7rbBM5xcac1LmF9kjnOQ\n"
+    "uAw1jsCNE82QHwWMlXOqaZCEJsnttNo0Y7yaSR/ChbGJ54XCp+Lx2acyTeH9cBWU\n"
+    "de8/sKAQ4NqpbEP01pBH4+1mPu2MYWjVWVicUxmw0mJ3cfkJCWUzt0nC4ls8+Itk\n"
+    "7XliKb216Z9uQXu/zD/JGkxAljnFs1jXCX4NyWz46xnJFzXbYCeyQnBz0tUbAvgg\n"
+    "uRdryYtHzD46hd8LTXH6oK2gV64ILAhDnRb1aBjnCXxbex24XoW3hjSrKGTdNsXA\n"
+    "RMWU/8QZaoiBAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAFIYDBcbit2kOMrHECZK\n"
+    "ctem40A3s+0ZifzZ2KLhW8dTr/2Zb6DnlqVm2iUOV4cG/o1RAn/HzkQQuWEq+oBG\n"
+    "yOPVHudvCyGs+2ZQWudgAv9xq8N7KtZwJhnn42c2YSoreqRXDQgJqGFatyr+XdR7\n"
+    "gdQapLI4BFbZToeXp49Nl+q9330hKaSmIYmWEZ7R/33R64PU2el7X9/apYEcuZQT\n"
+    "+FjEqcO1lJ8/dTwM/2C1BJZqUeFTAu+ac1M+4//qyJRUUc6xSJLhiens8atWaxwL\n"
+    "eBCT8fCY8oPOwA1eImc/yWWmWXpv8bBWVe8OeLCMKM/OZoIdFqQpqSdcyGoh/kIW\n"
+    "Dws=\n";
+  const time_t now = 1535046996;
+
+  tor_x509_cert_t *cert = cert_from_der64(certificate);
+  tt_assert(cert);
+
+  tt_assert(tor_tls_cert_is_valid(LOG_ERR, cert, cert, now, 0));
+
+  tt_assert(tor_tls_cert_is_valid(LOG_ERR, cert, cert,
+                                 now-TOR_X509_FUTURE_SLOP, 0));
+  tt_assert(tor_tls_cert_is_valid(LOG_ERR, cert, cert,
+                                  now+365*86400+TOR_X509_PAST_SLOP - 3600, 0));
+
+  tt_assert(! tor_tls_cert_is_valid(LOG_INFO, cert, cert,
+                                    now-TOR_X509_FUTURE_SLOP - 3600, 0));
+  tt_assert(! tor_tls_cert_is_valid(LOG_INFO, cert, cert,
+                                    now+365*86400+TOR_X509_FUTURE_SLOP, 0));
+
+ done:
+  tor_x509_cert_free(cert);
+}
+
+#define TEST(name) { #name, test_x509_ ## name, TT_FORK, 0, NULL }
+
+struct testcase_t x509_tests[] = {
+  TEST(cert_new_failing_digest),
+  TEST(consume_ec_cert),
+  TEST(reject_tiny_keys),
+  TEST(expiration),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_zero_length_keys.sh b/src/test/test_zero_length_keys.sh
index f85edb68db..84ca513b0a 100755
--- a/src/test/test_zero_length_keys.sh
+++ b/src/test/test_zero_length_keys.sh
@@ -3,8 +3,8 @@
 
 exitcode=0
 
-"${SHELL:-sh}" "${abs_top_srcdir:-.}/src/test/zero_length_keys.sh" "${builddir:-.}/src/or/tor" -z || exitcode=1
-"${SHELL:-sh}" "${abs_top_srcdir:-.}/src/test/zero_length_keys.sh" "${builddir:-.}/src/or/tor" -d || exitcode=1
-"${SHELL:-sh}" "${abs_top_srcdir:-.}/src/test/zero_length_keys.sh" "${builddir:-.}/src/or/tor" -e || exitcode=1
+"${SHELL:-sh}" "${abs_top_srcdir:-.}/src/test/zero_length_keys.sh" "${builddir:-.}/src/app/tor" -z || exitcode=1
+"${SHELL:-sh}" "${abs_top_srcdir:-.}/src/test/zero_length_keys.sh" "${builddir:-.}/src/app/tor" -d || exitcode=1
+"${SHELL:-sh}" "${abs_top_srcdir:-.}/src/test/zero_length_keys.sh" "${builddir:-.}/src/app/tor" -e || exitcode=1
 
 exit ${exitcode}
diff --git a/src/test/testing_common.c b/src/test/testing_common.c
index 9c6580f788..62d40a42fa 100644
--- a/src/test/testing_common.c
+++ b/src/test/testing_common.c
@@ -1,46 +1,48 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-extern const char tor_git_revision[];
-
-/* Ordinarily defined in tor_main.c; this bit is just here to provide one
- * since we're not linking to tor_main.c */
-const char tor_git_revision[] = "";
-
 /**
  * \file test_common.c
  * \brief Common pieces to implement unit tests.
  **/
 
+#define MAINLOOP_PRIVATE
 #include "orconfig.h"
-#include "or.h"
-#include "control.h"
-#include "config.h"
-#include "rephist.h"
-#include "backtrace.h"
-#include "test.h"
+#include "core/or/or.h"
+#include "feature/control/control.h"
+#include "app/config/config.h"
+#include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "feature/stats/predict_ports.h"
+#include "feature/stats/rephist.h"
+#include "lib/err/backtrace.h"
+#include "test/test.h"
+#include "core/or/channelpadding.h"
+#include "core/mainloop/mainloop.h"
+#include "lib/compress/compress.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/crypt_ops/crypto_init.h"
 
 #include <stdio.h>
 #ifdef HAVE_FCNTL_H
 #include <fcntl.h>
 #endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
 
 #ifdef _WIN32
 /* For mkdir() */
 #include <direct.h>
 #else
 #include <dirent.h>
-#endif
-
-#include "or.h"
-
-#ifdef USE_DMALLOC
-#include <dmalloc.h>
-#include <openssl/crypto.h>
-#include "main.h"
-#endif
+#endif /* defined(_WIN32) */
 
 /** Temporary directory (set up by setup_directory) under which we store all
  * our files during testing. */
@@ -84,7 +86,7 @@ setup_directory(void)
                  (int)getpid(), rnd32);
     r = mkdir(temp_dir);
   }
-#else
+#else /* !(defined(_WIN32)) */
   tor_snprintf(temp_dir, sizeof(temp_dir), "/tmp/tor_test_%d_%s",
                (int) getpid(), rnd32);
   r = mkdir(temp_dir, 0700);
@@ -92,7 +94,7 @@ setup_directory(void)
     /* undo sticky bit so tests don't get confused. */
     r = chown(temp_dir, getuid(), getgid());
   }
-#endif
+#endif /* defined(_WIN32) */
   if (r) {
     fprintf(stderr, "Can't create directory %s:", temp_dir);
     perror("");
@@ -112,8 +114,8 @@ get_fname_suffix(const char *name, const char *suffix)
   setup_directory();
   if (!name)
     return temp_dir;
-  tor_snprintf(buf,sizeof(buf),"%s/%s%s%s",temp_dir,name,suffix ? "_" : "",
-               suffix ? suffix : "");
+  tor_snprintf(buf,sizeof(buf),"%s%s%s%s%s", temp_dir, PATH_SEPARATOR, name,
+               suffix ? "_" : "", suffix ? suffix : "");
   return buf;
 }
 
@@ -178,65 +180,6 @@ remove_directory(void)
   rm_rf(temp_dir);
 }
 
-/** Define this if unit tests spend too much time generating public keys*/
-#define CACHE_GENERATED_KEYS
-
-#define N_PREGEN_KEYS 11
-static crypto_pk_t *pregen_keys[N_PREGEN_KEYS];
-static int next_key_idx;
-
-/** Generate and return a new keypair for use in unit tests.  If we're using
- * the key cache optimization, we might reuse keys. "idx" is ignored.
- * Our only guarantee is that we won't reuse a key till this function has been
- * called several times. The order in which keys are returned is slightly
- * randomized, so that tests that depend on a particular order will not be
- * reliable. */
-crypto_pk_t *
-pk_generate(int idx)
-{
-  (void) idx;
-#ifdef CACHE_GENERATED_KEYS
-  /* Either skip 1 or 2 keys. */
-  next_key_idx += crypto_rand_int_range(1,3);
-  next_key_idx %= N_PREGEN_KEYS;
-  return crypto_pk_dup_key(pregen_keys[next_key_idx]);
-#else
-  crypto_pk_t *result;
-  int res;
-  result = crypto_pk_new();
-  res = crypto_pk_generate_key__real(result);
-  tor_assert(!res);
-  return result;
-#endif
-}
-
-#ifdef CACHE_GENERATED_KEYS
-static int
-crypto_pk_generate_key_with_bits__get_cached(crypto_pk_t *env, int bits)
-{
-  if (bits != 1024)
-    return crypto_pk_generate_key_with_bits__real(env, bits);
-
-  crypto_pk_t *newkey = pk_generate(0);
-  crypto_pk_assign_(env, newkey);
-  crypto_pk_free(newkey);
-  return 0;
-}
-#endif
-
-/** Free all storage used for the cached key optimization. */
-static void
-free_pregenerated_keys(void)
-{
-  unsigned idx;
-  for (idx = 0; idx < N_PREGEN_KEYS; ++idx) {
-    if (pregen_keys[idx]) {
-      crypto_pk_free(pregen_keys[idx]);
-      pregen_keys[idx] = NULL;
-    }
-  }
-}
-
 static void *
 passthrough_test_setup(const struct testcase_t *testcase)
 {
@@ -281,6 +224,30 @@ an_assertion_failed(void)
   tinytest_set_test_failed_();
 }
 
+void tinytest_prefork(void);
+void tinytest_postfork(void);
+void
+tinytest_prefork(void)
+{
+  free_pregenerated_keys();
+  crypto_prefork();
+}
+void
+tinytest_postfork(void)
+{
+  crypto_postfork();
+  init_pregenerated_keys();
+}
+
+static void
+log_callback_failure(int severity, uint32_t domain, const char *msg)
+{
+  (void)msg;
+  if (severity == LOG_ERR || (domain & LD_BUG)) {
+    tinytest_set_test_failed_();
+  }
+}
+
 /** Main entry point for unit test code: parse the command line, and run
  * some unit tests. */
 int
@@ -295,16 +262,10 @@ main(int c, const char **v)
   /* We must initialise logs before we call tor_assert() */
   init_logging(1);
 
-#ifdef USE_DMALLOC
-  {
-    int r = CRYPTO_set_mem_ex_functions(tor_malloc_, tor_realloc_, tor_free_);
-    tor_assert(r);
-  }
-#endif
-
   update_approx_time(time(NULL));
   options = options_new();
   tor_threads_init();
+  tor_compress_init();
 
   network_init();
 
@@ -335,6 +296,7 @@ main(int c, const char **v)
   c = i_out;
 
   {
+    /* setup logs to stdout */
     log_severity_list_t s;
     memset(&s, 0, sizeof(s));
     set_log_severity_config(loglevel, LOG_ERR, &s);
@@ -342,48 +304,54 @@ main(int c, const char **v)
     s.masks[LOG_WARN-LOG_ERR] |= LD_BUG;
     add_stream_log(&s, "", fileno(stdout));
   }
+  {
+    /* Setup logs that cause failure. */
+    log_severity_list_t s;
+    memset(&s, 0, sizeof(s));
+    set_log_severity_config(LOG_ERR, LOG_ERR, &s);
+    s.masks[LOG_WARN-LOG_ERR] |= LD_BUG;
+    add_callback_log(&s, log_callback_failure);
+  }
+  init_protocol_warning_severity_level();
 
   options->command = CMD_RUN_UNITTESTS;
   if (crypto_global_init(accel_crypto, NULL, NULL)) {
     printf("Can't initialize crypto subsystem; exiting.\n");
     return 1;
   }
-  crypto_set_tls_dh_prime();
   if (crypto_seed_rng() < 0) {
     printf("Couldn't seed RNG; exiting.\n");
     return 1;
   }
   rep_hist_init();
   setup_directory();
+  initialize_mainloop_events();
   options_init(options);
   options->DataDirectory = tor_strdup(temp_dir);
+  tor_asprintf(&options->KeyDirectory, "%s"PATH_SEPARATOR"keys",
+               options->DataDirectory);
+  options->CacheDirectory = tor_strdup(temp_dir);
   options->EntryStatistics = 1;
   if (set_options(options, &errmsg) < 0) {
     printf("Failed to set initial options: %s\n", errmsg);
     tor_free(errmsg);
     return 1;
   }
+
   tor_set_failed_assertion_callback(an_assertion_failed);
 
-#ifdef CACHE_GENERATED_KEYS
-  for (i = 0; i < N_PREGEN_KEYS; ++i) {
-    pregen_keys[i] = crypto_pk_new();
-    int r = crypto_pk_generate_key(pregen_keys[i]);
-    tor_assert(r == 0);
-  }
-  MOCK(crypto_pk_generate_key_with_bits,
-       crypto_pk_generate_key_with_bits__get_cached);
-#endif
+  init_pregenerated_keys();
+
+  channelpadding_new_consensus_params(NULL);
+
+  predicted_ports_init();
 
   atexit(remove_directory);
 
   int have_failed = (tinytest_main(c, v, testgroups) != 0);
 
   free_pregenerated_keys();
-#ifdef USE_DMALLOC
-  tor_free_all(0);
-  dmalloc_log_unfreed();
-#endif
+
   crypto_global_cleanup();
 
   if (have_failed)
@@ -391,4 +359,3 @@ main(int c, const char **v)
   else
     return 0;
 }
-
diff --git a/src/test/testing_rsakeys.c b/src/test/testing_rsakeys.c
new file mode 100644
index 0000000000..0f22d4e01b
--- /dev/null
+++ b/src/test/testing_rsakeys.c
@@ -0,0 +1,546 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "lib/crypt_ops/crypto_rand.h"
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "test/test.h"
+
+/** Define this if unit tests spend too much time generating public keys.
+ * This module is meant to save time by using a bunch of pregenerated RSA
+keys among */
+#define USE_PREGENERATED_RSA_KEYS
+
+#ifdef USE_PREGENERATED_RSA_KEYS
+
+static const char *PREGEN_KEYS_1024[] = {
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICWwIBAAKBgQCZa39BCgq7KWBWFSjGYHhqmTCHvQ7WNEFAb9Mujb6Xn/Zy01fu\n"
+"WIpVvqmAKeLNEziItUm/gB8GwAN+/ZLwL9pufjIp2Ar+yqVXKySioZQxuCgTP2wm\n"
+"Ku0OfmAra1Xbtrkc2OCJllxkyNPrJ/kxfwjWR96UP0+VMbOlkBoEH1FtvwIDAQAB\n"
+"AoGAUXoygeMIYe+OdwkTt48CRHKIwH3aRE5KHSOGPyIOB05vvvmYqD8jcHgqYqNc\n"
+"DNdZXdkRin9LevU8phObFq4DTXp08XggUx4Kk4AdsFKubQtJ8gHm3xlSKbZXX2m/\n"
+"ZF0GRaZtVDQ3TRGh+OBLILt/2jT+BaFKGAyJ7al76F2nprECQQDJyLlteLDFBmrd\n"
+"0kAjNBE50S5YskBCQeQACROfyTKW8lG1J57UBeYjXvbrDFBR4alIS9DEexGai9Gz\n"
+"wxpgKg2nAkEAwqQmPstjHxvqGQRi41uXO026MLxY7dhEqs1aSw3tuT8v17pW3OEa\n"
+"Qxv7JINePZ3+sNN+Ic+3RXBR0QuD7lSSKQJAZjVSF21GvMXfY7SX4D0DbLHUNAE2\n"
+"I1mUz5/JXOpgwazETmpfPS4vwELd93kpRhBz2rbsbFmaNRoVgmSU+5jRiQJAZ1bV\n"
+"g2NilgKxEGU2x3U6Xt8Oqo9lO6omEvUCKnUTsNWuZf/l3FGbKuQxO5qPr3Ex5tny\n"
+"zqrEqBZRKgbOHfxCuQJAbJY5C3Nm5koemr031r00MY2YD1b6+hyKZyPdZ21HpyY8\n"
+"z1kWShL0POjYPX/BnKE1FkpklWcKBb7wkK7dvAKkEQ==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXQIBAAKBgQCyqMM2TfFGV5tVBTVabxLVln8146nDavIdR6q78DCUMh8Zfzkk\n"
+"h9Lbl1NX4RU+AmrCZMPq21/EjIRxRQyRdgPYJVLdp96eGeYnEzmMkqvXiswXvDg/\n"
+"tXqsjyJeYsoHMQWDTpCLfjYo4K1ol1sg8VIs4wQeq5og6QSdmhBoz7MyqQIDAQAB\n"
+"AoGBAIJekey7nZeV8Bxva4ptSRIg+v0I/2VBUiG5nUX9NIW/uV/yrXERx/VDjKaw\n"
+"8b5JJzxpKWnk4RJc83xwRYaT1qMYHiQfybxEI0K9SjhtaThAjtXkQGtZgLJILl3t\n"
+"yh3LPTh1ocwafsKjU6eGYAe/DYn9/QwYHbtyaimcigu4etp9AkEA2DgC+HndoP1i\n"
+"np26Lx+4TG0vAfrVYGSLT9FXwf2iBV3oJvdKqu6wr8ipb1SbshRPcOQd31/mCh6+\n"
+"2BR+d4ddcwJBANOHrlBbGZdHnoEu6kKbPwwkc31IZYqyfSpkqm0Lb2oWZ9SInKfc\n"
+"cz0qpH91p610XUpYmycaJr4K+N8jgrz86HMCQQCoqGBg1Ca2OpCf66bctWB8dTqS\n"
+"z8d7rlIhC8npr1+f0hWRt5pN5Wx7YgoQpq3gZgllpPtMT7DQOhVh1fKkaDnTAkA4\n"
+"XuskPPLX7t0dvhvtviOSH9CrLXTp/mD+wC7uumJpmij3aaSd01DelxOZaAhUYDNQ\n"
+"UcafKAf1E0V5aaQ4qwljAkA9NVN6CtpzzcLrstTKxrx5P1Ylt/0UYQDo1lIaqwrT\n"
+"aOFbXmOungiC9+p/4U7RbX0MEzjFDHCWlaHASviGVgta\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICWwIBAAKBgQDDt2V63APj3JSqaRgofUzhtB+prm0wII4uHyxfOxnpYIELOW5z\n"
+"3UHmkr+B4D+Nif5jIp0i6W4OS4S+YHewKsDsXvXKRIW78KzOt6Le4JI9rSarNjy5\n"
+"aJKksWQRALLCmxP/BdolaBFqF3fIPD5+Zxu8ESgxhkEQI4p7awUp3E730QIDAQAB\n"
+"AoGAZktfAR4p8lkCYydW9yK2ommQ+xEuBK+fYL/uYz/yxSYpjIJSFsEYhrlA21Mo\n"
+"JIRxr8MRuoOjgFk8YnztUeimuHpslDlZDaCBzjRjBRFCMepZNG9xqSEL0u7C+SH6\n"
+"KU5f2x2P6PneBj6WaHZM+6Lf2xHlOoeuaVSUfq2Pk2VBF9kCQQDtawWWNwP0+xea\n"
+"oCAQpanaLzYPjlqZfHJQ1AAI5eSkdf1qmlypIHwOtjAEa6XuEO/Or8RNkNy4nQdw\n"
+"qhcQ7PXDAkEA0wjT6Z+Lrt67FnwPgoSvl4Nukcqw4OWHbBKhaQPsO9+oc3PAXLdD\n"
+"SclUUqDF6NX1yONTV1KrPdz4zElmEua+2wJABm4inZnp2oW+cuqpU6oY+pbSwQMb\n"
+"AxMyyWukgJkxYx7q+SsrHU2K7p8Sl9wOh28f/5oVGAC3aayfGfcRXtz8HwJAIqeO\n"
+"dQzYGU1GF7kjquEzHIRewd4xEZ1fkaW1j9MvFd3ygZL+gbsud41yJWd1WHjaNbTu\n"
+"2KYgrLX+vT1IX844hQJAbg0V7iHlttQqXL7yN09jIjQLprqVhDZCUHS9s9Dxe7fz\n"
+"Ac0ZZD0D6EVNmSmBB71q7kLUWX/W/10d447TLnnfew==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXgIBAAKBgQDhCAjPEockl4lqkvoIb5O3NJJG8NWD31c63e/cPWY6MX5nOM/q\n"
+"avof2eWJxFOk0HQ2BRVwIgNex6kLxtsdw7XE0A5uZorTp9DbRCGMqUqHNhHH9ci2\n"
+"mMPP9jptq3ieWg310bH4Tad8h3WE2npSCDBvxyV6EmuH2rlQW9ZlHNoiRQIDAQAB\n"
+"AoGBAI4PgWggPTqng7PJF5mNvsYQpSutzE0VCL977nmuNUQVjMPjRLarVD4ZU+QW\n"
+"EevhQQv9R5xjjJcgGqL5pchzjeKDm0/LA+AygnZoDMs2O68Neieqvr7cPqr5ALGs\n"
+"WuZvSn+bRJTenvV9sUh2ii0/u3GQbL1v7GWDkIdD7itDbmRhAkEA8iijuEY+W67w\n"
+"7JusjY2MQ2Cm6xxxR0YcnYPzT6UDm+Z7NNJwKscQ6AjayNmxmXGpbUdukzLzXf8y\n"
+"fccI9t6iHQJBAO3kx9nZay0Ktl51QP5o2gwoqRIbnogGfR06KJOlzIPGR0aPn8cg\n"
+"uKq2SiyjewEaSBM6S/4UlxYUmvc3VKnxCEkCQQDpTjg2YQ7RPGIIRA/iLV7Wx3bq\n"
+"C/QjjCwjoi44LK6mdE9928WPoUzrkSRg4EQYpwZqL6kcDrmkdSuLPMipOGQNAkA3\n"
+"KtzlujPOiDNuiEaAORSHyU4b8ue6p7aP9pK+Wq6oyGxzAo+NABuTCx78ZxT5Vnzs\n"
+"aJKC44d+CV0+g0hQ+KJxAkEAqFYzNWIzTHX8DVDdK9BpUaBg1DFxIeP5Kk+/X3FF\n"
+"5BafG08B6OiLf8qIGGsxLXNRjIE0GVp3Sy23FUKtUymP+A==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXQIBAAKBgQDMDk01VwPxQq/BAwOBmfGUP/x5BQn+uxI0Aat6bdWuz/2CsjbS\n"
+"CWD/YLCaPm+DpHp9RMwk4HONJaw4B2XOw3ELPx7y9DEgdC1wZ9wRkJmqr2IJZoZR\n"
+"C7x43nNv+/IXTiRkkljCcMpoL1Tld+L2VbmWR29PdZwvspWRILkEZu1mNwIDAQAB\n"
+"AoGANvFK3KfXSei4xfF3yjeXEmHAKx2uOUZJenNQpqBYPr+F9ODjXd5knZ59LqrM\n"
+"/9cTnBMgHHXK5yBTpKppQSjikLeQ2BF04Ktff9oGqVcS9x/rKo0CREuxsEfawZOW\n"
+"OzOWENp4YcDKGP1I/Ctr185QzStaWrXVQftxmYQ53T77ShECQQDnhabwtqW7rfe4\n"
+"+MfkWEJ9Y2s6iMs3JWnwPOX9G9R39PiAD4vAghHJyHHttS9Ipxmvp0hThu0x7a4g\n"
+"8BfUpqgjAkEA4aFAmzarWKigREAACVTYH2RHpXbuk05vF9WqfMPiEvQUd5a1q6vc\n"
+"xkGZsE3v/TExLjPRZP4FeUNV5sD7THzA3QJBAJxPoRlNx3GCEAlDdfnWGPX9JI09\n"
+"hC40RWUcSI7ttjJTI1+an1kWuBnLChhaRpU/tFjikTNLmmMmPHUihIRfDI8CQG7g\n"
+"3WzpKr8A7vFbOilbxnF2yDaqAYfmTXW7DHMPl/OUetJh/5kDdhT/e9VGF5+nIvH/\n"
+"iPFGW85Bpt8lCtmFnQkCQQDjpp9iy2qesE7KKX4Kv3++QfCJ2w3g7lwg4iyncoDd\n"
+"JrM53p29HROM21R6eekvqeWIe9tEX754b+E/N60ZjpGm\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXgIBAAKBgQDdDn3H+Eu0AW5GKohqDBntw6ubnd3VaJwZGzZyga4J2kLg8peP\n"
+"RAW6GDD6pcHzW+KZbFWHtRk70FSwvmyGcf+DY0r5tfyCHyDGmbJyPR0o6OVCgSFl\n"
+"ccf4eDvbyszzMdlx3uL05ABIpCShoKtEUqvyIQla3Jon+QBwuVkizMzyVwIDAQAB\n"
+"AoGACoKh4Fwh3VEkGRn0mnYw1Wk0Q5Xh8j+jDF6K3C7mQ3mpLGDca+dkDlEQIxq2\n"
+"egeoYnsQJf+qT3m8TRsAtfO9nj7+7IX4BfCtdIi4RNcorbs5YMWtFyaywnM6SQjS\n"
+"+1qf74aL4On9WRO2FtvnTMjFAAkiWNbQp7mWwTmB59i620ECQQDwde6/PwhUzvZh\n"
+"dyslKJdna5RjkDQyDIuh0zD/tFZ0Iko7Luec8q6n52ev/n0OiTLGetUh8goePsPP\n"
+"HVZHidNJAkEA61eMCmmu+GCAg2vJRtL5sDakAXsbP5M9Bf/QVHXtc4EVXHC6T2ld\n"
+"bldOJriNbBThBuPNmlQbssn9FApkyWT4nwJBAIuHIv3+CUuMvBJaH8L0BsaP+g67\n"
+"wk24Ud2Yujnl3rSMoR4uXV8IwqfS8quAs/gXTEs3QyzrUUuzh9NKZqIkK2ECQQCz\n"
+"vivBEDKIlPvSZBJYO25kfXcJgoKvLb9fw5/TwjXXD/HGpnpFiI3JZnjT7gRlVhT/\n"
+"9CDmC/MTvF3EXqPXhXy1AkEAo3a2me23Ljmub21jycSKaCk09dK85QTRRMe9c/hs\n"
+"i+pcGi9ZZW0Mm7cyQo47oXjNurkkv0fEvXIobVTEXAGU7w==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXAIBAAKBgQCv8R1IbfYnE3R3kNeezJ7m02XnyCBDDy0YfrQldQ+urdg1CFye\n"
+"bO0iPniJb8fmV8NW7x6nUZTDznCg+igroKXtK/w0WYmJJiH4A7Oi5xNjAfRIPvJ/\n"
+"J5GI8szS8rH8tp8pW1h8k/kNg2pnBjwQ2U9omhp95RGaHDQSRYzzH/fEFQIDAQAB\n"
+"AoGAcy7+BcH/iZuB/xjzIIJDcUhqibCJ9n0D/+pLU85sYuZrCmUcBZe4M1gEn61v\n"
+"iExilRJc1hthskL/l1POYql8lk+aqeeDuh38fWJj60TCV/sENiuXOsTmoFVA5pNn\n"
+"lwlG8JlpBMsgr1fGqg1C/WLFfMmvXdKVGvpRqI06j7AYUa0CQQDfZ5rI+FhXBlxo\n"
+"PR5CM1LB90DuHUMW+Kqoj0c9d2esXEQM7UqQ/9BiBQbL6Py7Z3VwCxibOqyz7+V7\n"
+"2aGUMAKnAkEAyZy5Mu2tHs6YBBxPYam7huzMUYjddN7ixAZUyGwxQp9kTIF2NbSQ\n"
+"yVDjKrco3s2lO4qj4pSumwVe3GGlsi6G4wJAOOS3pIqqZK84BUvbUtyjLMZ9AKbv\n"
+"GQCG5ZpneB3ahyiQJAKiRL8BIJVLH87b3hYA8GHDCHUu2jwz4xCPd5+qbQJAV0TP\n"
+"pYvb9AnZI25drhiaY7z8dA6aTYxs/A0Bhf/PEteLwtIHKRgP1BR/QG4n8slxTGSm\n"
+"q91P9ypL9XkPECGzoQJBAIMvGEM7ZGevQHBjJ8HhU8IsgT4cYH/XEYb8jRy4F+Ui\n"
+"jKxHPxLuFK4urAZunNUNrqhT0PxbB7hRjtHZrmFkrcc=\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXQIBAAKBgQDSpmV8ncLwc8gXzdFsZGPDtMO7C/IN9jKCIK13WIseMg1APlMt\n"
+"PB5lMQ9fa3m9ZRU0L8HzRo+u/Xdos3yIBI38X2Avy0laGKnQxiOKaDT/5ZHeiBBh\n"
+"nMZjP2WY5V1sgqNP9RD8enE6WaSvq1j0BM++mn9KEe//5+dWD8tboBKF4QIDAQAB\n"
+"AoGBALgVoerdE1Z+WAY1XyaSNHz6o3H6ZnW9CTaex/jb7/dbVikmThnhx842qXCB\n"
+"w8m3ZGhOs/edWkNaTde5wsI6+LhVGco/PWxN4v61jokxUU+5KvUvGacXhXIjzKwG\n"
+"DrNCYmle62QCI1z4+TLQW/Lq+jw2Wzk70NWEvoP58gt5SJoBAkEA9wubRKRs49LW\n"
+"5JNQZ9hjc+mAfP9YK/sMe4jkdloMMWXjSMlF3Z4mI9XQSpfbBqwWIBXsjU/15LIS\n"
+"ftmujZsMKQJBANpJEZI7UFoRdSP7AlM0YJuXWnVGyn/K+VIeEso5AlZdKXCTpxqp\n"
+"9blWq0UVC6jLesZ5UNPuBiAnrBaVwDA8YvkCQF+FQVfdK607TJO80g4VAP9EfcXX\n"
+"BUScIUtytsN8NdKzzpnKGRWDnMOmXI87ABkoWLW3RGuvSyhOIhCiInfmR2ECQASc\n"
+"FmroJcJBLCAeZOYs7P1cLOTdIdmhB7LcP7lVit8YCJAADj9Z536KfgNvdleSNH2M\n"
+"glB3blmvfMrdTrm2DMECQQDj6GJ/Tc2rCsq534xknasVjrgtJMQFxmQCTVgBx9pc\n"
+"gTflJAHAmNDvstacVqeObLCF2ZIvya8fSXGbDOJYeGDv\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXAIBAAKBgQDGgUJAm7vf/3focNGwzv4TkzYF2XwpAirnb61dyxvfug1zKv2k\n"
+"AUg3qACiurR7JrI+kAbmxEnNaKV7ts7uO763wP9KE8YAuFZsp7NFA295rEZhw38T\n"
+"rUlWHMCeaZ3mqW2q8gA14C/ZJCG4gS91SIHLjNGsbHwr2Jvri2ItwIP8FQIDAQAB\n"
+"AoGAONceb32oiHWQkkBr6uL6ogRPPdGO2fdC7c5uqCLWsnOGEmpHAsVTNoym0fIA\n"
+"aBsmgv+e2klukKDccdZg3prA+z7lHcc2a4bIFguF6ei80hLIis/dds66fFXofCzy\n"
+"DMlkncSbJwIvQHG9gblxp9qSKElZF7XjABZEImarfUlakGkCQQD//msGy5N0ZhMI\n"
+"yGMXkwXRJXfmRrIrOqHx6u1eUp4OuqDW+hBz4KCHnWfuRJkNGQIammSf18jPasP5\n"
+"YHyr/LifAkEAxoJ8R8Vusexo9ZjuU44qXCSvJQ26UBV7mn6TGEAn2DRK1RWKDaHv\n"
+"j2vnRjt3CO9WPDQL7SB/1HNAy+dIMPyqywJBAIB6tESIz8zPniX+TJ18UKMTZwXP\n"
+"3YQMvVKpUdDRLjq+OBMtFizSRD9MJOlUzGvibUfkzTPcHRDcyNbUMj4vbIkCQBx4\n"
+"6sqAjvgGKKfRX52sbnb47AYsieSisC/gp8h6qzxfg7w8cqix6WJw36M7ND+b1Iqe\n"
+"DHfeiXc3cLvOWJRuKTECQCEYkujtSjXWb26xaESFWGtUI/nEvCyqYPQAFBpaGzQ3\n"
+"tiTDeKHzypesWYoTxOiNQWCQMLrFGuUbDpYOuDOVNjw=\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXQIBAAKBgQCcwSAfytnspSSDX/sKmCPOMnpuCYeWA4wbz1wLyb63a8/KXhhG\n"
+"6o2W0kt3x1vnGZkeWwZOeBFUqwoc+xHhoNcZFsMOyqbqA3UMZW5cx27MsexRTQHs\n"
+"Go1newu/E+8NNCohY51G7z1Hdo0L6mi/Tldh7puuGsMwKqNG/Vvo/GQDgwIDAQAB\n"
+"AoGBAIUdpBAbjXDe1OET0vYuOMnUKA/l29RS8tpy/zGrg1/0GCM8QNWIPfEEaL4w\n"
+"+CSKonMazYI5iE4kaZQuygKXOdFqKxX8nrGK2hR0DIEUHhhiqyGMUKrf4ELkAJzK\n"
+"tHtcO64OFEU2EGa72wCmyk2MhqhLxWxA7E00x24uvW6pen6xAkEAzHhbzlRgLZ+K\n"
+"QuXmQHEqkGaS2Ccf6c9TA5Bf5S2/5zBl+OqVyJJQH0yrbPYR6Nn1NeSv3R4IDJYg\n"
+"fSZLaVzWHQJBAMRCU6QtTnZoQ97pLvXCSKRYKJF+CnE3zDFTyoJrpK0W1FSnb1EE\n"
+"DWjjdSdMLynf/InX+VOaLk3Gxwjme4NKjh8CQQCg2b4/HplayrsVzY3I/D2jw02Z\n"
+"xY2RfYusrhMCU284DBbsLn8OfiuRs9rXqOyF5ZDFiNXgeROT8zYzvcBtbp7xAkBU\n"
+"ZET9IvJLXjhZISItUXbVHIeNUIqC9sBaMbKx9EGioF97a2gliT2O7cgRtuPM+ODq\n"
+"ETHILlNc5G3vuNRBt4x3AkBV98Y1SZA3TQlUVTsjGraxkFTfU1IlomiOdOwTQ+xZ\n"
+"x+JxhhgZwZ+kgI3PidEufFCTZJ3WO6Wk9gk18Bx7CLjm\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXgIBAAKBgQDq/K7wNW3fcTbaRTjNZlM4W0G7tKeO+X0bca4+9uin3ML3ogNJ\n"
+"6qT/B0QAZB6Vyi9kKa3E8plQkjmPuX8Q27zj2QjEuDZ12RGFnikeOosUhOYiDh3Z\n"
+"T9CHnr6stozzgk79Xd6VI7bqRcgRwbY0uc9QVr6vwddyIfSploSpVcgspQIDAQAB\n"
+"AoGBAJfUpo/sZc6uzxtfCKGmkPTj+ef3hSBbUZuu60AhtxfnC06HrwpOg0eJAUYj\n"
+"aqOsHMziJTYQ7kDiCjE0UMaqxDNS5hueumznq2xM2mSN0nYoktU00kpANVkW4VPA\n"
+"33TB16DyqlKq2/21Rs1g8/8+IKkKDbRLTC//1WqNHASQVoGNAkEA/+z4hxTVXZkr\n"
+"9hz29tAHKURlqzxUEKLnS0eL+XGJRNfGJ+65eXL+gFiIbTnpVeidL1+lKWkZyYzl\n"
+"75cNRdUHhwJBAOsOJ9mUOqTbLW5tzh18ewZGOa1JcxhOvf2E1d56N8tDK6lvoqkF\n"
+"oUUb8kIweDxPLCVLCl8qFrbjn619fxDInXMCQAfEZGKNIlCd5nSoumIRPDZnagKB\n"
+"aTe8CfMB7+CZLoZVWiE6IIzsDYdNqI5QFKHT1nlqmLOiCfNRAGV+GxwEdB8CQQDE\n"
+"sHu4HclU2fMSTOAE3H01qt3om2WsGXfyBI3SNQMrG3IVvkymkwd4BQKbUGPMU5Pl\n"
+"QP3U1CtdruuXCUSijrzxAkEAoqYub6+0zM8fakSQZcZ01TG9Fuo2xVFDCQsvqR3m\n"
+"ZhRT/oinIvOxSh4fQs40bmt1RBmc2L1Is6YB2NTVQEBZDQ==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICWwIBAAKBgQCrf0rPvHYaGYQrc1ciRwaONs8TUvSVmUU98HMYXoFEkBL4CAGH\n"
+"4oNHFk8kXHEOsBED0eccSYegWhqKHSz7PbjmJaXloExWrtx5ea3Twf8VTgcfDWQP\n"
+"0TzD3G1TYjAFPQ1/LAZCpQFmwpMmTGGxegUhOzkpEWXdLVEVc9Uw4C4L2QIDAQAB\n"
+"AoGAZXAJZA5pHM7y6nBynYe9TOkGWru6h7H8zsImkcd0VoWRcrvpi+JjG+0KKsuy\n"
+"46kop0XEmWq0mhgxknfnX0QG1MKTqGMIUGN4qCaezOabIpCOdA4d/pr/mWoNgOWw\n"
+"9Kc/tNCrKxPKsQMAlWP6ktHN30XRSlHgAjSeUVUiNHztvTECQQDUNin2nyIvj8ZA\n"
+"QAsFW9qW+TiTkeUK6yiZ9Gvgf20gwZRWOe5/xnMxVvtN6v7Av1ew/l4VhBoj/w5g\n"
+"ydIZk+2LAkEAzuJwdt+ccllG19qmEcbo9XFafgi2PvlEjPJmT1rHV2ns/7HIMu27\n"
+"PJY36GgExSfFco6VmicaoOt+RKg+5acgqwJBAKQxAEjcGWQ5VsgRhTVxO3DChX7Q\n"
+"TColhrWPwwPhM/s7K92HVzwvvKL5TNmdr9xMb7n3Ja56FouxZVuH6/J0XT8CQAat\n"
+"Mhnz/3WFQg8HRGLAe5YoMVZt64u+uaKe1ARtlo9QoNBjqWVTXL6IzocWjEjcjrey\n"
+"uEtARdC5qNqIX3dD3H8CP3pVCPvpHOTxkUaktmLYowSA1HSfO9wkE6bMCHhkLwXF\n"
+"yTIJ+N7c5u5YN1B6hhVqpKbdnSv+K0MQ0xbfwOWNMw==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXAIBAAKBgQDGQmrKfO3WovoXkOTSh/shO9qjbX4izhg4pccVU3Tp45v/dgAE\n"
+"uDUuaa/clToyH5AhOtuazO/asC3ZNajg1ia5VPzmQU3gtqiIZIEXFaOovPlOrXru\n"
+"wyQnxaGORndJwfDXicG6bUwI+PDpNq8c4VOTujReeF0r74qMSc7TQLVlUQIDAQAB\n"
+"AoGAakR/aTm9YibJVohbnl00xoOGlcLCsXU2lmaFZ3DsYdGWdD+TkvQJzW7ozJtQ\n"
+"Lj2sy6L4wujGR7nXWW3hr2IaLpoc1UoyJpieAZM5os6bMN+N4MCqdcZMlazMtSWV\n"
+"UDO7O7xQGFpcvvZmnfKCyluFaJ5K/tWxP+2TnS1/m0BDRIECQQD5DYvToA0eKBt+\n"
+"7K4eEI8pzDot9NlcL21D86kNgpmuY4pifALU7GvXr299JpFFiYa2A1JVRfpQaoI3\n"
+"hZzz0ze1AkEAy8opWJP+T2q4reD5Qq5UjjrHUXFID23KeJEjh5YF40/bHqyVpWVR\n"
+"UMntNgAzs+13vRij48Zn6I8GRhStaQ3ArQJASPyFS8GN1paeaDXoWPs1WWR2cF1f\n"
+"DbsAZHeVxVXOv+J//ZimI8wdVpodLCoPTLee+NxEVqUpVEPCYY8QjgwKOQJAATmj\n"
+"6f5pxvxzQ8hYd0gpBfngfOLbdgxI7VSiDAyg2G8AeDy9YZMsW/n6zRpPNUO2NpLR\n"
+"WWs18LX7aaxyJnGIuQJBAPPfy9pd4XEFsRBIIe3N23Gua1XkS/407RJtAGm73Vrt\n"
+"QhtWh3i6D5gfpEApMoaE8aaQQ7H0z+0Uh1t8SWesy10=\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICWwIBAAKBgQCc/M/X8etUqrxnmH3PyuAYLIPZhwNySch8qz9NB47izYjxzuBG\n"
+"GSls6H7WeKIrB8UJY1gW8TLkdOLcrI/0hTANNHEPaueOE0xdABFj7tAaiiGPIM25\n"
+"N0wc76me0ZAMYJrZTHk8JZK153y9wInYBwVZreXCVSVf11RuVwe+iFQa5QIDAQAB\n"
+"AoGAQC4XJtivdhDLL6snHFF7pkZkrQTGgu3pOhakrXA+mTigGQOTqvTUe8LdP/9X\n"
+"hTIK+tiTheWcAcxLhx5BSB0/VDKjYhS0ROpTc33Iq9KalOQaTJbBYGA4eagpQjwU\n"
+"jGwr9u2sUsM9WI/Jg0VvLSKhfnNwYIUzLpK3BbWb2qAdh+0CQQDQ2s/8DlibFSBK\n"
+"UsFK7lLpV8UgMk9CkaNM2BPzI8Hsjpp6s3pULVRd36m4YTSg15EEHv7bZ1N/+krX\n"
+"mXb9xUULAkEAwGy5wHsUSjTK+kntkNXjlCU/+9R+HFpzg9Bwm/PqXTBwEWeU24hV\n"
+"iRjPvqPtWFZrWi/nfcviuMaqtdliw1I1zwJAZ2mQxhtMYC2LuYFUWAe9YfClmJWQ\n"
+"jUOTef8bka5I3RqW/t5TWc7AEWMnpDXtWx6hnUrDolt9Cschu7MvKeQ9lQJAL18U\n"
+"46PpPNN+XNuyVoOxgRkihVasrUI/SeYYsuv7eHGiRUagyOLpW9T139LvbV3pE8zT\n"
+"So7VA/Q0towL2lX01QJAGcoBNNouSpum9+5NvGQK1XXsZweawE+pFR2BE5XcjG+n\n"
+"FnaLEUBX7nTxhTU2cSQET1PKRNp568a281NEna0nxw==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXAIBAAKBgQDFOqqGG/VtIScxayZYZ+BT+hcs5W1bD5qRxunbG9O36UVT18UE\n"
+"CWw9HUf0Q5sDMGvVmBxwZ4GjbR5FDPfhIXaRCzobnejJXq/0k+O5NAVkcSPtJvhK\n"
+"AaUqBrWA41vnjKOtJudTsZLfufKafzYwVonze7fXGyVsBRjVwHNS4iqq2QIDAQAB\n"
+"AoGAJCoStI6R3RXUKvKb0GATuTJFZ50WBTmCPTK9FMkwdCuY47vPy2Ky7y3cUMTI\n"
+"urf5PewrYs0H72CFyWGMXkKVi8aOYshsATEXMfGSqOcqXn+UDssRzvabZFlpnAUa\n"
+"WDVt/iN092AdakXNna7/DxrLisDpq8HHJfjtlWGPfkXRg4ECQQDpHeKimTvwJcPc\n"
+"iDa6Qb/n9gwLeRckfzhYtfX1luJYLIOHh+J9vjQN75thenBLQB/B6qlKtOn9ejxg\n"
+"5z+3zIOpAkEA2JbxXVTCOA802p9khvHxDtLHdKi3w/BjjJiC7Mgqo69ZI+s3PB9E\n"
+"F2HJA69kZqpGqvybWHDapjWsq7rcMlxrsQJBAME2yvR3y00VEAyGPc4M1vF8ZqlP\n"
+"uRW/+ETWtEDUyU/JvU6lGt2bu2tdkEyv/cjxIiFIzP4litdT7B1pLc+6S9kCQBwE\n"
+"usiWFGHoJbA6emiyl7qRLdg7kzo3uMkRWa6D3nA6WM+6t/SBHu/faH+fit91G5s2\n"
+"/mmcf8yMmP/GNoIVTqECQFl4Pt6yGiz/YVoYSp35ljY5n3JB6T8o2pOmIrRLuPmT\n"
+"6kgyygtJBAmx5nnQoeG8n08tl9QakWznKzkNJ0DIFKI=\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXQIBAAKBgQDCaOqJ0lsSAEBcnNB6X7BvVcEcol+evi/nJsPe0uT1SbtW50Ch\n"
+"vYOHwK6aQR2C5x9VSs47cLynTL7tNt5d8oeryF3NpI8VTPLImDJCcvUZhS7p4bxn\n"
+"JO+Wm+D/e3TWfyjreuWtdL+Mfimw2gzwWuBEtmj51GzQ89eYm7fh11SB6QIDAQAB\n"
+"AoGAWaakMbZNxPlUtOCjyysBY/Y5vYira7rswD3CKak7aFn+CE9QIMYSN7IFUqEg\n"
+"iNMoQd7jR8nvVX8wtJeO5+gF48W13C3n8FZSrW7c5N3bmfMIgo0xa/TGfeXHP98o\n"
+"7vhH0I58j3ZZt0Q+3wTm7t7WPE/nJzgrCk30TqmoaEmstTkCQQDtV6YZ6juEK2Lp\n"
+"LGUiqohcS/WJxvFrF5+LNpk86Xdgomf6FphZlkq42KYkvl7qibKDcfDqLKTbHHle\n"
+"vQQeCgZ7AkEA0bFHi7F8o4iHtKleBvt4QCj1neA0q3CRDypCI5EqFSrNpxY4Krhh\n"
+"WYSVX+xT00QYaCpKKWfYQztCw7Anylv96wJACl86Mwe5ch0zRV1bThiFvQLUyCCZ\n"
+"jESMBFlueOr6/I4cXSF/puqaeVl+aTyoiTdbRcNE8/bffXPRGgLIm0d04QJBAJSY\n"
+"lmTN789Lby99Xh6AkaSV4ghw26Ip8QHYJmph8npxjK69Niw/4Oy44cnKBVUPSmR2\n"
+"o3tYFY7/Lb7S1D+4lOUCQQDbMQUGVsZT+ZjuOG1bAjIuXoAOfOd3mgH5VgQHjSgJ\n"
+"ourZtlJ4OUpNrq9IfWqPkM+zSE8+0Dk8/9MS5ngBA/SJ\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXQIBAAKBgQDNbHjwg+7tVNr9erMLowXRnIcttp4pUJbr3B7Jo/u+kD/Yo3F3\n"
+"4rIKhHpJl1uEHP1QmvAD+4ApFFI2hNG54xYI8dGflxL5HOs5xxyOPpkrwzQ8Qvnv\n"
+"LPg7Gf6PAW9zF4McG4wK0TkrV28G6NhqcPs5VFY6UyvfZ0fEdWAeoWTIfQIDAQAB\n"
+"AoGBAKOmkMp7MLLd8QAS6eSRYSdWHdLrMyES1MjduaFGBF4SKOr7en/Zl6ENXSaX\n"
+"cA7V0XCPnjpt9/HCAKTyNupx4LCeFWiqdu8VGXhlzX8bdb896OSR2brKbxgRY5tF\n"
+"36uL8akrZdrYgocykQCxmRARMB7/rHwDusiamjL6RUZ3+c45AkEA6UPTVmKZQRMr\n"
+"A7Qgg5nXrXo9117Lpqf3FdZ1wdni9V59Ptf5xrx9oGZNZzctJPXSAH4M4cumSJrV\n"
+"sZ1V8qE7AwJBAOFx+5luLrVKrdlG7MyOhTAdhKYUvKIvL4wvVSY6y+L2nNEx/cTx\n"
+"KYbxGC+H1RJbkCS09rYir3VfDRWQ3W1c1n8CQH+X4hn2hO3blkPIW6CgniD+JKWR\n"
+"7MOUTMtdK7yFemfM76VYbgAPSohabSxwOfllnSE30cQQqTw9tXYaIdE98BECQG+M\n"
+"QWxSS0QillB6unIgVqBPCrJOcmNhK4qWZPBMiVNcqI0Nyj2nAeAl7MyfzfqOWY0A\n"
+"CU5nbR+LD2NLUXRqSisCQQCN3IGv1WOWInmA5xhU6vCFDX5u48Dcji7VLJO/Nv/i\n"
+"b/zHKAgjHk5Js7bi5ZWEGaUgA4Jt6cKmGdERheqTMKxx\n"
+"-----END RSA PRIVATE KEY-----\n"
+};
+
+static const char *PREGEN_KEYS_2048[] = {
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIIEpAIBAAKCAQEAoksI1qIuIaFCqT4QbgDvOQCmr9Z9F0E7ku+U5Ep/5dWNANqB\n"
+"bSzAOq0+cxiisfF+H4desoqiWDUwlOwXH74qD3ZsbChhvFUD78cQBWQkF+whLVHb\n"
+"296QmF0LZqosqz9HMS9CdoMUc1brZb78Hb25QIOOjrg25KYHLZHaqcet1wfhHow6\n"
+"Uehc6QTuWgOWFhJnfiXzYgen2o8lnLixxZozhk7Lm7Aix9ur2ckXdQ2Wgny4xw70\n"
+"JW84Hapnd8oFUD98XXrExk4VFuIcA8qo7r7y18II6wx4Cw1suKru6bhW65cM/y51\n"
+"KC4lB7VkvuoJCelRFdM1PfKZLv2tJP63oAqJrQIDAQABAoIBAQCWc38PEqw3avqU\n"
+"UMAEaoNa0bq1Gd8/Nq8WqVnbRSFKHO2pk+cWIb1W6BITuwvgcGKesezdEV4s7apK\n"
+"9I7/U1hEm2Ep50mrwRh0KZM1nD9Fmharn851Bt//D4qpMytT2caS1yADI8NKpZJ1\n"
+"8VZh7+cT4qG+txHUaAIRgbw3VrBWvTIMu6SOSOZm+e3eOr5UU3du1KvjdJHJ2c2k\n"
+"TceHvUdKxV7OYt+BBSN1oBOhs3ajUSRge1v3twRDg3cmbwG0DeXvwHNhGUTcF8IH\n"
+"JO1RF5njbkFvyqdAi3ltjU41zYd4OMuPtrwzFOtxUjKT62Soz109HUXXE2CGKFPZ\n"
+"PVi5/BIhAoGBANN1xqS5BgHszIB0nXbw5ImYpTRmyhO0KsTblBT9+8Q/B7BCK7bM\n"
+"zl+dOPeyvEadSwE7RSMMt6CAlTakWIf3Quw/VZajvXy9C9/LHf52pEKXjxMFMPKE\n"
+"aGLHpQnwMtDi8/H8AEAXxI3hpxB2KVR7sAYHWihSGjRJ6oPGvEmKEkb5AoGBAMR6\n"
+"G2PKz0xk1vFrjfjSY+y13gH/t7xHaXUggjggUSGKaknQh2BDUllXjadeI0fi1eLW\n"
+"r98ZImZZgntAgjaIZ4bAlooTDk4gRHaz9jI+z8lsRwOKnWdiigM7txiXZTMVwMqj\n"
+"o5mMNGMA+A+ACkTViRHmkDI7S/9FqAvnbOqVwgFVAoGBALUcY6WDvwx5B3Jh7tgH\n"
+"XIYpEh3+h8c2gYcX1g3gtvkPTwN8uToY0gz8eOVV1YHZiHsmi4GIi+HRH3usaRMT\n"
+"COOVHzYlSc8Dj57+tdLTRL6wVl9hC9o647ju64DGlI9qQquYPZKniLZIdbFYsu9j\n"
+"/JA9Tc/I+h6czFpPJccKlbrpAoGAAPWXrKUQ3g6f/g3IY66jTkSVEO1uuDyhBzFh\n"
+"cWS3ALLsUe/yuUWa4VTMHEUZZwB0iucBdNVqlZVaTb/C4wFHgCDwmzv8leUScIHw\n"
+"cc5ctV8R+bJzkk2o3tsrybLzi4xPpK2n3tgQaWtXyruVUUC5qpy1l4kylcyBRY2b\n"
+"uomAqQECgYAiCNWtuWIDlRBcvtIB+kHguzcoFT3vTCCNhalTEn0zi/tbi+voQgVJ\n"
+"SDJNptZv+6vRwQ/HfcQtljKIPO6hUZPYaFWRNhgbh7Ay85lRXYXQOottE8ayReBk\n"
+"zZb0fl853Qah4DPsaOugAvhjjKeBmKg6bFWO1z6hj18I3UpDf2YnVQ==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIIEpQIBAAKCAQEAssO0r37mSJNAkc/ISwXBsu9JjyLeWlsHPAhylQGkSAdp2rjz\n"
+"E6AT0Eh3wrocNO31I4pvHReAuh1QedGY6T1cQwO/WAAhQtRCBQDK12qWRgfbC11y\n"
+"Xu7zNYPd1Z7YIRy+FxhbL5f+lv3rEUv0HUG5c3CWhLtbANKg+jOieIDzA4Yp1s55\n"
+"ynodQBUkTZrwQiT0P8yDSjiasf+clgJRfA1k2XK12KSAMRgyDuPTE4OtBxBvUM3L\n"
+"Zvxs81PsmcOuAG4DLaFTg2a/QkCjt2VC1SYYuh/LVxpL41FFh3eMoK5g5deHkgRe\n"
+"tlywKjAHIDJu/qgNzNgNW7ymwn2CfBvry9h0/wIDAQABAoIBAEMZ4wDdCWPEokAZ\n"
+"Vn2Ss5qO53WrCPuxn42RPjFgZGIFJl7LfbKoK8fK6+lUIrJbf+DPXdX1tIQn7MVN\n"
+"P7CNL8yX44MMyW9kbUOjgIBLqgyvdjFV6lBoMTKtRN+iuE31lATnR5Md4pqaxVnA\n"
+"wOkaepoycM1x5j7w0SwZparF/HIdkYv0y/MysqT9ByupPA4Fqp/iRSrosHXahNtI\n"
+"KZYj1TyERYtuDXq91P4dr/pWq3FmDNI8O3upblkL0YouvG/ZlFLdiNy77XbAyWcX\n"
+"ps3YDddM+vECnXO3+sa3ZxgBYvXJdWrrIzM5A+jCkDRZQGsFAzK5I5/S7C2ljt6i\n"
+"SmzqvMECgYEA16bGy2XTi6KBPb8aev/OBgK9XuGLwUqK1m15mS9Y2qPHmuc22qaZ\n"
+"hw6zginPFrxAEtQWKanhZy4aVqlLkDPLwRnyeuMo1EZAc5B1gZ5ViSAKxBq99hA9\n"
+"eqyakdb+IUQsEnRDxSc2gqUQ0EagksUyw5wGG5Q/CVEALmS/r1SU3KUCgYEA1DYf\n"
+"6JYdzuRtule3vYeWXKf8sOJpdplgWV7tvLrKkQhdE564uwMCYB23HvYfwWqEdDYG\n"
+"fsYg/ur/stk9MDZ3wZKffTEM8V3sX1t1JXnC3ogSAgMGhLZ3ILOLqkoO4BEZJnsS\n"
+"dMdiNijlAtQkqs/BO/UVUAKysCtKP3v/+1775dMCgYEAvLjGFjApfnSbV/cK7IM6\n"
+"wEXbhdIqZOCgOeEaXjVyM/zKbMRVW+oaR3hVHd8KzSG3jQKv1oxFpu9Qu3ByoWLC\n"
+"uF3Ft0debs6ADuJoAyQWROeWpGGmxlUWCGpO5rxYL7KiQxAeUsXrTU+5NBvq4CbV\n"
+"MxwyuCX3OGb7mp4upfiGQcUCgYEAuhVsDYv1P4LXJVvd5viKRV2ZG5KuYC1Ga5fu\n"
+"aFxzXJI07At2eaa94oKsHR494mEBHNZzA5/BN0fiSHZuTWS1xqxH5oOokc6Gg2ez\n"
+"ZdVLp88x20nD4YQPGkHW6tBeEuVrZG7vVC+yU0Ow7bYRISdkjqrusWZsQkbzqI+X\n"
+"fFliEbkCgYEAu8x+47M1ordbI7NmbBGyiyP0r7nMRCZ+KEvGeCNYracWmsnCNnfV\n"
+"zR2UzmwtSainw3Ho8Jv/rWDC8RIDauyBRYEi2VqOnUzT2ca0iymQyLeBCudAQuio\n"
+"drOu4JU8RzZ3Ad6V3DNFnaqmX/7GA9Pa2GI8NJMyb8p1GAGv7Gi8nxc=\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIIEowIBAAKCAQEAt01S8JuEwWy/Hzb90yO2O7oGWq3GfvfDpFOF4OQnwG3kQ/BP\n"
+"4MoPDCYHdqb3iI9aD3vykZA6Q8zpdfGwjm4+bHrgRdiSmZWv8NvRwuQ5Ji9xbiGn\n"
+"hA1XwqH9hvgFTiy6tRvirWSJ7kzH3Q/bEGpCbHUQkwMog4v6yCNKNrjlwjN++eCi\n"
+"gFK/0RMOJMLOs8BD3zY+lKjd/pd8LBRujkMyUF5SryeRueAFjD2sq4OXq8DPABGt\n"
+"zdR6vbTcsi4JwP1Q6y4x0/LIWEprzzewNU63I5E2zj0WnoRGAIM4aF+VuqcHjWUx\n"
+"VWnyLZldSen6lScZ4xj4seitiDbSFvtFkDF6VwIDAQABAoIBAGTP9im2ntDyyjqU\n"
+"uA0DuxomOZBtupniEouyFBOX5/UBe2WSKZxsBNKdp8UuFz3X+aRCeyprtF/NtyjT\n"
+"AFOVdmebPPWtIxOtK9LAUyFo+7VwqmXzxHnwDLBS/2jXx7MzDozFBWpvvRx+xf1i\n"
+"1wy0JEwaJj90oTeYKRkhr5NhJZwkX8zCNYaemBd3kHB3aGWGJasI1Y81UezeRKCn\n"
+"hSbn2CrWalI7pyJ4lsavM11nIq1Eu2ZthJiNCMghbYrHoBHd+iVWiCYchP2rNEWV\n"
+"sdHtaVHtQ9zdZ43bao3OzPu7lAjd6UAbxsuhUe+a2YdDz/+Up+6+BvQf1FCfYIjW\n"
+"KFUdCoECgYEA4t5O+u0V9gkMUhKsevYb0zgc7O/mo8ivN+V++EpAtL0mhiwxeO8p\n"
+"oef0szLyhdULQeLN9pJQDCeAbkGdwIe3L+AKU8o8BFGEWLFysZjMg9In/UTrp5MN\n"
+"mMDy2SRKKu5BqsvdYH302xpZfHq1T2cMNDWE8lrZffduH06Cgq/XEtECgYEAztbj\n"
+"bhFneADnrvk609VnOQvoQEjySeCQKFQFRRI6k/FguqMisL2IRXnMaWammosdeCAg\n"
+"m7eZchnszHIst9cwZUKXUFqmAqeDuWSNdTI7uKZH6nT/A6IDlgdjaHsqhvpK0Ac9\n"
+"ngycdHONitOZh0ZG74pdWjf828Dwzf+CuYjl9KcCgYEAmIvI6ZqvkJ8m5Kzfw1Jn\n"
+"BVCOypbJK8oOX3R2Orea6KzjEYb3wQx3nwFcHX6danYFOskpmqlpH7MT/Y8rZsEa\n"
+"4RsxdoPedTzm08iFiXtn0R9nejp0hlov402iPXXUVSedih3IflBTa1w9XaEY9wog\n"
+"P57ZBSknYzcTmgNtaDiaUnECgYA5sWauhNw/dMEq5QmrnJK2LsQRakdqo+CR3x25\n"
+"LmR4b5Nze51pfvRLrLV/kMpXwQXvQ8bUqFl8og6S2CXxAWzWUcSy/RXhF6h+RbXP\n"
+"Qru1vWvB0fBvqvklF9p6giBSle3YKKzfMNVTBggs+OiR+uA+YHG5gHRfN2nzi5mC\n"
+"9tRtcQKBgBnDSi4lRCjRe9pPnyAYaa4iyBUGhjPysScSLY9orel89+qmTBQ/Py6J\n"
+"0+sefL4ZJaOsuaR2mSSPP/lbSkF9DMFs4tHbBqY+WkVNYLshAkauHwqv26HTVCSd\n"
+"QKzeb7uZw9lNaRIzDvy/3wfCLvXfdDozPFrOUgkyaBN5pJSA/4sv\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIIEogIBAAKCAQEA9qtiDoJWqU/eSlpj381eG6UcDzfMguFh/q4e4s7QVdRYj5J0\n"
+"Msv0PCkti8JHuvQUyncRpOPccBkhNbVjNbjIgw1pHaIZNdVotUDhP0kseRyJ6z3M\n"
+"qbZ5qKn+0mHjVjPNItVDDe6tebYMT1BZpVyRrCOqY2v5z1ecLC+ReygmHgDpzg+L\n"
+"0rWfIxGT10IPZ8pAlcdEn6xt5aEhi7mPCX/xwqfQChPIJz6zVLEC8UaPtvDBohPR\n"
+"6NQTBTeZZAAtzrQ7+oNxfz1v6Fz6RwMei7Q+qOBnMiwpQmbcDBKABM2RnXSpD0LA\n"
+"1GR7/+CiV1HQoShWVvEwrSIlM6jVAJo6iqF6WQIDAQABAoIBAHqwcdxPnfUm4aTP\n"
+"4r9NcZKEhDlZgqJSoiA/0OL1BRC7xrTanmspoLhPrvTF1FG715+Aq8j9AQbMqQUC\n"
+"zG7LEwiEIhV4K9vn4uXMeHy206UFud/E5EhBl695pmJUB/Q3XcAGnQyP+77++o50\n"
+"o7IpIdeiAbzj1uP3aplbq5u7M4JV7fUZWA/368G4HolqFTxcAfBJ05GXlp97BBwY\n"
+"AnY3/pNrKMz0NiPf3nsJHYWK18up0JCLPL3tomc94wuNZ66spIazHIL9aaKY0q3V\n"
+"LkBrelndfYM1m4xRTnSOy6STu0qKTPOpX0C8XBLYs6uiXjRsChqSYwndCCeASaH3\n"
+"LGNIcbUCgYEA/m4qvt8tdT4wEvnE+QUxEELmBtT4UFa3NnQISrzNlhNeI0Zd2xlp\n"
+"SG0/pcw83mG2uX+V5xSaWL5LYfLBkvy83Y0yIWgYbbIkyyCOUZnTpwaDGU/FjWip\n"
+"3TfXf5qpAgiez94sV+MsFpKfG05yxJh5u+3sIyGTVUAxp0HPx4LVgbMCgYEA+DD1\n"
+"fu6ttpuV1UMrsFdjuk6gBvSbyJ9OilY2jT+yE7hSRc/yP3O9ikuR74tNlVrWTnO2\n"
+"0kcYbyLJXE2cGUC2q5e4r8TDGiozNfQ7/OC2M3XaJ+xJk4zMf/8PuDDpWr+18ZXA\n"
+"Pf+ibXWTFvZ6ZeUmpbrrfCrXdvmIZnwVuOI0FcMCgYAZn26emksxq3mb75tumJ9A\n"
+"S/xuY7Q+Iv2Adl7/Z9QscPbiBowdLIn1yUrHn7Hhk2WbeMXX57NDjKZ6zr+/1cQP\n"
+"a9DInHsZUP9zlWu/vAYcpAM/4VC71PaGWMFTEHhExCl6NZ2xnCcsfseXMGdOdSyN\n"
+"SICnaRI1W6mkdnQ+W2a1EQKBgGEKA3KVr6XuPy8bDEHuaTe29irCCQbwAq1j+ABS\n"
+"HzZGoyRYocbdYgZoda7LMJJs6c3SwHCHC66oU0KbtaTKAKImuDdBH2djiJJX4/yD\n"
+"f7mvIpTpdfsS2gJRn7vMo/CvdFv4ySl0gfV6OwCHbmPYrLuv0dLCjWwfNI2dhoC7\n"
+"MNIxAoGAIPSIG4BrShzbeX4c2L18iwIg+NlOcUbtl0Ccr1t6uLGI+ge/6I6T/5XH\n"
+"DPKqYIf0IRYV8suxpfQNKiz/C0NPffA1d1M2hvuAg2v09o2cSwvdcQwdmakKZ5bl\n"
+"sdCuYKdCIwomEUOz/4XgQrJl4XDUqxftJT6/egAjWvcIYvfNCsY=\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIIEowIBAAKCAQEA1yHZMsgRLckL+v6rgpGq9qmxVBNDxeuul1V/QlFyOlcAk5n/\n"
+"uduTalSqGQhc4NEePMxq6nFui4ucpkZOozmcEnhV0N9jld9IB9rLGt4erdg7RKl9\n"
+"+gQ+zTn69j69U36E2I47H4dM69uxeSOyWP2Odxpw+biisa3o8mMz1zCmuj4GMDtG\n"
+"DlnSpthFzgQR6N1pbvxLXrWg5F16GqFiJOD7kXDfy4/l6kB/mDs1T/3r8kav6DqR\n"
+"c/t3aQZxgWGIpI7hc9Qgvp7coZRMey5dNOZEna3tqS8dn2tZlhkpYV5uyFUjmxjG\n"
+"TERSULQ7hvUqW+eshGGsnxFtL7ANnTSc4xECowIDAQABAoIBAFhJJMhpQFuIySjd\n"
+"AGeZ/g4x/3rgWQzNNp4WUR5XLEhy0eLA7ShJywp06kVRoEQGraEHxsyldldAGS5H\n"
+"ZhgoGTufNKB+PHER646FpJpHE1IGjfQUloVW3qr8I1iQ0MOGBWCVpf+/V7rnMsLi\n"
+"+lr421FXgYuJ0QKXuyRVv72M0q9U6i+ml3aVAhgW/19oFg+dW7YccX+9iVyD05Q5\n"
+"KR64tX8xd4wrAqfAgYA3erbbE6GTyHYD5K54kIgfRr/+pIU4qc1L7XOCblnqc/rI\n"
+"BilFysEC634r2MNe66uQvNui4oQTfBcFFlXg0zAmp7d5QE0ApOL6HpCsmbImm2uJ\n"
+"sdFNYyECgYEA716kfEv7HfnF0P3pAP2AOuEsW6t8q0UtWvnHrwRQXQw8Yv90g7kD\n"
+"pUV3/BjD9VQgsQZosbdSn5wbT4j7dypRdrzYk+8m/hBk4Q8M/tWoRGVOn46NudvK\n"
+"/KX0A4ODLuulj8yAZVc7CM5Cdy4GCGJBVO+oVvBUAnHxfZziOyqBw9MCgYEA5hQg\n"
+"HEORzdxvbbfAx1ggvH1Eg1lqRhmpI43PpRkaoqb8jLwXb2CyBeuv3RBft/X2Tr6F\n"
+"mHpe0U1kN/5YEjii/Q/jUX8azIHaUNNSAjrriEeMQZOqFxmhCdiyeXuqg2fbFbhe\n"
+"K3Q6/fsB1xj9OOSwyPMqm/M5U0LsoGjmg8TFE/ECgYAlImKUIdlwOgp1NJ7MF4eo\n"
+"Gryd8AmkLFQv8+YFgb7R4I8RsJ2rva0SG6fUhScJTSbRL7RYNZ9swXP/L7oLL5Z5\n"
+"vCxBLu22pmZv/7y9X/n9ulWrLRtRhQaFkV08mk9knQwPNeOJVTIEWLM49/vZmxyV\n"
+"h6Ru8FOoGXMkUI1MLnj5HwKBgGJLkNhiacVYeuaWDa9c0EeXARFYvxWJ2wAMkvzG\n"
+"9+ErlFQP+7ciyYvMAItidnJii8NilDLrfNzQwpNFf5zxQ3j4M7bapblfdMT5M10u\n"
+"jPfhEWPm0VEjKvDI+p76HYQcd7YU2W6ZLqbZeRTLYUvQMFL5yGduBzyyJ+P0TR9Y\n"
+"jpYRAoGBAM7vYGTprw4w2tTZPFICXVk1bQ0LO06oNRtwkiQTUT6UqPjWMFyvHnmN\n"
+"11SVVBmRZ0RAk6e5eZLFX8WelJ4J4nSOGRcJheCtoEFlO7D1ewAUSbqWJ0pBqp2T\n"
+"gV4oCS8LYe8zReVoYZJjuLwoHvxZzs/hUjc3SI2HRW2W/HQRPC25\n"
+"-----END RSA PRIVATE KEY-----\n"
+};
+
+#define N_PREGEN_KEYS_1024 ARRAY_LENGTH(PREGEN_KEYS_1024)
+static crypto_pk_t *pregen_keys_1024[N_PREGEN_KEYS_1024];
+static int next_key_idx_1024;
+#define N_PREGEN_KEYS_2048 ARRAY_LENGTH(PREGEN_KEYS_2048)
+static crypto_pk_t *pregen_keys_2048[N_PREGEN_KEYS_2048];
+static int next_key_idx_2048;
+#endif /* defined(USE_PREGENERATED_RSA_KEYS) */
+
+/** Generate and return a new keypair for use in unit tests.  If we're using
+ * the key cache optimization, we might reuse keys. "idx" is ignored.
+ * Our only guarantee is that we won't reuse a key till this function has been
+ * called several times. The order in which keys are returned is slightly
+ * randomized, so that tests that depend on a particular order will not be
+ * reliable. */
+static crypto_pk_t *
+pk_generate_internal(int bits)
+{
+  tor_assert(bits == 2048 || bits == 1024);
+
+#ifdef USE_PREGENERATED_RSA_KEYS
+  int *idxp;
+  int n_pregen;
+  crypto_pk_t **pregen_array;
+  if (bits == 2048) {
+    idxp = &next_key_idx_2048;
+    n_pregen = N_PREGEN_KEYS_2048;
+    pregen_array = pregen_keys_2048;
+  } else {
+    idxp = &next_key_idx_1024;
+    n_pregen = N_PREGEN_KEYS_1024;
+    pregen_array = pregen_keys_1024;
+  }
+  /* Either skip 1 or 2 keys. */
+  *idxp += crypto_rand_int_range(1,3);
+  *idxp %= n_pregen;
+  return crypto_pk_dup_key(pregen_array[*idxp]);
+#else /* !(defined(USE_PREGENERATED_RSA_KEYS)) */
+  crypto_pk_t *result;
+  int res;
+  result = crypto_pk_new();
+  res = crypto_pk_generate_key_with_bits__real(result, bits);
+  tor_assert(!res);
+  return result;
+#endif /* defined(USE_PREGENERATED_RSA_KEYS) */
+}
+
+crypto_pk_t *
+pk_generate(int idx)
+{
+  (void) idx;
+  return pk_generate_internal(1024);
+}
+
+#ifdef USE_PREGENERATED_RSA_KEYS
+static int
+crypto_pk_generate_key_with_bits__get_cached(crypto_pk_t *env, int bits)
+{
+  if (bits == 1024 || bits == 2048)  {
+    crypto_pk_t *newkey = pk_generate_internal(bits);
+    crypto_pk_assign_private(env, newkey);
+    crypto_pk_free(newkey);
+  } else {
+    return crypto_pk_generate_key_with_bits__real(env, bits);
+  }
+  return 0;
+}
+#endif /* defined(USE_PREGENERATED_RSA_KEYS) */
+
+/** Free all storage used for the cached key optimization. */
+void
+free_pregenerated_keys(void)
+{
+#ifdef USE_PREGENERATED_RSA_KEYS
+  unsigned idx;
+  for (idx = 0; idx < N_PREGEN_KEYS_1024; ++idx) {
+    if (pregen_keys_1024[idx]) {
+      crypto_pk_free(pregen_keys_1024[idx]);
+      pregen_keys_1024[idx] = NULL;
+    }
+  }
+  for (idx = 0; idx < N_PREGEN_KEYS_2048; ++idx) {
+    if (pregen_keys_2048[idx]) {
+      crypto_pk_free(pregen_keys_2048[idx]);
+      pregen_keys_2048[idx] = NULL;
+    }
+  }
+#endif /* defined(USE_PREGENERATED_RSA_KEYS) */
+}
+
+void
+init_pregenerated_keys(void)
+{
+#ifdef USE_PREGENERATED_RSA_KEYS
+  const char *s;
+  crypto_pk_t *pk;
+  unsigned i;
+  for (i = 0; i < N_PREGEN_KEYS_1024; ++i) {
+    pk = pregen_keys_1024[i] = crypto_pk_new();
+    s = PREGEN_KEYS_1024[i];
+    int r = crypto_pk_read_private_key_from_string(pk, s, strlen(s));
+    tor_assert(r == 0);
+  }
+  for (i = 0; i < N_PREGEN_KEYS_2048; ++i) {
+    pk = pregen_keys_2048[i] = crypto_pk_new();
+    s = PREGEN_KEYS_2048[i];
+    int r = crypto_pk_read_private_key_from_string(pk, s, strlen(s));
+    tor_assert(r == 0);
+  }
+
+  MOCK(crypto_pk_generate_key_with_bits,
+       crypto_pk_generate_key_with_bits__get_cached);
+#endif /* defined(USE_PREGENERATED_RSA_KEYS) */
+}
diff --git a/src/test/zero_length_keys.sh b/src/test/zero_length_keys.sh
index 3c61f8d465..5635bdfd89 100755
--- a/src/test/zero_length_keys.sh
+++ b/src/test/zero_length_keys.sh
@@ -43,10 +43,11 @@ fi
 trap "rm -rf '$DATA_DIR'" 0
 
 touch "$DATA_DIR"/empty_torrc
+touch "$DATA_DIR"/empty_defaults_torrc
 
 # DisableNetwork means that the ORPort won't actually be opened.
 # 'ExitRelay 0' suppresses a warning.
-TOR="${TOR_BINARY} --hush --DisableNetwork 1 --ShutdownWaitLength 0 --ORPort 12345 --ExitRelay 0 -f $DATA_DIR/empty_torrc"
+TOR="${TOR_BINARY} --hush --DisableNetwork 1 --ShutdownWaitLength 0 --ORPort 12345 --ExitRelay 0 -f $DATA_DIR/empty_torrc --defaults-torrc $DATA_DIR/empty_defaults_torrc"
 
 if [ -s "$DATA_DIR"/keys/secret_id_key ] && [ -s "$DATA_DIR"/keys/secret_onion_key ] &&
    [ -s "$DATA_DIR"/keys/secret_onion_key_ntor ]; then
diff --git a/src/tools/Makefile.nmake b/src/tools/Makefile.nmake
index fda1990e0b..e223d9b135 100644
--- a/src/tools/Makefile.nmake
+++ b/src/tools/Makefile.nmake
@@ -1,4 +1,4 @@
-all: tor-resolve.exe tor-gencert.exe
+all: tor-resolve.exe tor-gencert.exe tor-print-ed-signing-cert.exe
 
 CFLAGS = /I ..\win32 /I ..\..\..\build-alpha\include /I ..\common /I ..\or
 
@@ -15,5 +15,8 @@ tor-gencert.exe: tor-gencert.obj
 tor-resolve.exe: tor-resolve.obj
 	$(CC) $(CFLAGS) $(LIBS) ..\common\*.lib tor-resolve.obj
 
+tor-print-ed-signing-cert.exe: tor-print-ed-signing-cert.obj
+	$(CC) $(CFLAGS) $(LIBS) ..\common\*.lib tor-print-ed-signing-cert.obj
+
 clean:
 	del *.obj *.lib *.exe
diff --git a/src/tools/include.am b/src/tools/include.am
index d0185b5887..f7aa7e0d1e 100644
--- a/src/tools/include.am
+++ b/src/tools/include.am
@@ -1,56 +1,68 @@
-bin_PROGRAMS+= src/tools/tor-resolve src/tools/tor-gencert
-noinst_PROGRAMS+=  src/tools/tor-checkkey
+bin_PROGRAMS+= src/tools/tor-resolve src/tools/tor-print-ed-signing-cert
 
 if COVERAGE_ENABLED
-noinst_PROGRAMS+= src/tools/tor-cov-resolve src/tools/tor-cov-gencert
+noinst_PROGRAMS+= src/tools/tor-cov-resolve
 endif
 
 src_tools_tor_resolve_SOURCES = src/tools/tor-resolve.c
 src_tools_tor_resolve_LDFLAGS =
-src_tools_tor_resolve_LDADD = src/common/libor.a \
-	src/common/libor-ctime.a \
-	@TOR_LIB_MATH@ @TOR_LIB_WS32@
+src_tools_tor_resolve_LDADD = \
+	$(TOR_UTIL_LIBS) \
+	$(rust_ldadd) \
+	@TOR_LIB_MATH@ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_USERENV@
 
 if COVERAGE_ENABLED
 src_tools_tor_cov_resolve_SOURCES = src/tools/tor-resolve.c
 src_tools_tor_cov_resolve_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_tools_tor_cov_resolve_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
-src_tools_tor_cov_resolve_LDADD = src/common/libor-testing.a \
-	src/common/libor-ctime-testing.a \
+src_tools_tor_cov_resolve_LDADD = \
+	$(TOR_UTIL_TESTING_LIBS) \
 	@TOR_LIB_MATH@ @TOR_LIB_WS32@
 endif
 
+if USE_NSS
+# ...
+else
+bin_PROGRAMS += src/tools/tor-gencert
 src_tools_tor_gencert_SOURCES = src/tools/tor-gencert.c
-src_tools_tor_gencert_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@
-src_tools_tor_gencert_LDADD = src/common/libor.a src/common/libor-crypto.a \
-    src/common/libor-ctime.a \
-    $(LIBKECCAK_TINY) \
-    $(LIBDONNA) \
-    @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \
-    @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
+src_tools_tor_gencert_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB)
+src_tools_tor_gencert_LDADD = \
+	$(TOR_CRYPTO_LIBS) \
+	$(TOR_UTIL_LIBS) \
+	$(rust_ldadd) \
+	@TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ $(TOR_LIBS_CRYPTLIB) \
+	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@
+endif
+
+src_tools_tor_print_ed_signing_cert_SOURCES = src/tools/tor-print-ed-signing-cert.c
+src_tools_tor_print_ed_signing_cert_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@
+src_tools_tor_print_ed_signing_cert_LDADD = \
+	src/trunnel/libor-trunnel.a \
+        $(TOR_CRYPTO_LIBS) \
+        $(TOR_UTIL_LIBS) \
+	@TOR_LIB_MATH@ $(TOR_LIBS_CRYPTLIB) \
+	@TOR_LIB_WS32@ @TOR_LIB_USERENV@ @TOR_LIB_GDI@
 
+if USE_NSS
+# ...
+else
 if COVERAGE_ENABLED
+noinst_PROGRAMS += src/tools/tor-cov-gencert
 src_tools_tor_cov_gencert_SOURCES = src/tools/tor-gencert.c
 src_tools_tor_cov_gencert_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_tools_tor_cov_gencert_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
-src_tools_tor_cov_gencert_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@
-src_tools_tor_cov_gencert_LDADD = src/common/libor-testing.a \
-    src/common/libor-crypto-testing.a \
-    src/common/libor-ctime-testing.a \
-    $(LIBKECCAK_TINY) \
-    $(LIBDONNA) \
-    @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \
-    @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
+src_tools_tor_cov_gencert_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB)
+src_tools_tor_cov_gencert_LDADD = \
+    $(TOR_CRYPTO_TESTING_LIBS) \
+    $(TOR_UTIL_TESTING_LIBS) \
+    @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ $(TOR_LIBS_CRYPTLIB) \
+    @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
+endif
 endif
 
-src_tools_tor_checkkey_SOURCES = src/tools/tor-checkkey.c
-src_tools_tor_checkkey_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@
-src_tools_tor_checkkey_LDADD = src/common/libor.a \
-    src/common/libor-ctime.a \
-    src/common/libor-crypto.a \
-    $(LIBKECCAK_TINY) \
-    $(LIBDONNA) \
-    @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \
-    @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
-
-EXTRA_DIST += src/tools/tor-fw-helper/README
+if BUILD_LIBTORRUNNER
+noinst_LIBRARIES += src/tools/libtorrunner.a
+src_tools_libtorrunner_a_SOURCES = \
+        src/tools/tor_runner.c \
+        src/feature/api/tor_api.c
+endif
diff --git a/src/tools/tor-checkkey.c b/src/tools/tor-checkkey.c
deleted file mode 100644
index 3e16fd0336..0000000000
--- a/src/tools/tor-checkkey.c
+++ /dev/null
@@ -1,89 +0,0 @@
-/* Copyright (c) 2008-2015, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#include "orconfig.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include "crypto.h"
-#include "torlog.h"
-#include "util.h"
-#include "compat.h"
-#include "compat_openssl.h"
-#include <openssl/bn.h>
-#include <openssl/rsa.h>
-
-int
-main(int c, char **v)
-{
-  crypto_pk_t *env;
-  char *str;
-  RSA *rsa;
-  int wantdigest=0;
-  int fname_idx;
-  char *fname=NULL;
-  init_logging(1);
-
-  if (c < 2) {
-    fprintf(stderr, "Hi. I'm tor-checkkey.  Tell me a filename that "
-            "has a PEM-encoded RSA public key (like in a cert) and I'll "
-            "dump the modulus.  Use the --digest option too and I'll "
-            "dump the digest.\n");
-    return 1;
-  }
-
-  if (crypto_global_init(0, NULL, NULL)) {
-    fprintf(stderr, "Couldn't initialize crypto library.\n");
-    return 1;
-  }
-
-  if (!strcmp(v[1], "--digest")) {
-    wantdigest = 1;
-    fname_idx = 2;
-    if (c<3) {
-      fprintf(stderr, "too few arguments");
-      return 1;
-    }
-  } else {
-    wantdigest = 0;
-    fname_idx = 1;
-  }
-
-  fname = expand_filename(v[fname_idx]);
-  str = read_file_to_str(fname, 0, NULL);
-  tor_free(fname);
-  if (!str) {
-    fprintf(stderr, "Couldn't read %s\n", v[fname_idx]);
-    return 1;
-  }
-
-  env = crypto_pk_new();
-  if (crypto_pk_read_public_key_from_string(env, str, strlen(str))<0) {
-    fprintf(stderr, "Couldn't parse key.\n");
-    return 1;
-  }
-  tor_free(str);
-
-  if (wantdigest) {
-    char digest[HEX_DIGEST_LEN+1];
-    if (crypto_pk_get_fingerprint(env, digest, 0)<0)
-      return 1;
-    printf("%s\n",digest);
-  } else {
-    rsa = crypto_pk_get_rsa_(env);
-
-    const BIGNUM *rsa_n;
-#ifdef OPENSSL_1_1_API
-    const BIGNUM *rsa_e, *rsa_d;
-    RSA_get0_key(rsa, &rsa_n, &rsa_e, &rsa_d);
-#else
-    rsa_n = rsa->n;
-#endif
-    str = BN_bn2hex(rsa_n);
-
-    printf("%s\n", str);
-  }
-
-  return 0;
-}
-
diff --git a/src/tools/tor-fw-helper/README b/src/tools/tor-fw-helper/README
deleted file mode 100644
index 6a1ecaa1e4..0000000000
--- a/src/tools/tor-fw-helper/README
+++ /dev/null
@@ -1,10 +0,0 @@
-
-We no longer recommend the use of this tool.  Instead, please use the
-pure-Go version of tor-fw-helper available at
-     https://gitweb.torproject.org/tor-fw-helper.git
-
-Why?
-
-The C code here was fine, but frankly: we don't trust the underlying
-libraries. They don't seem to have been written with network security
-in mind, and we have very little faith in their safety.
diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c
index db308485e6..25113420df 100644
--- a/src/tools/tor-gencert.c
+++ b/src/tools/tor-gencert.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2015, The Tor Project, Inc. */
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -13,8 +13,11 @@
 #include <unistd.h>
 #endif
 
-#include "compat.h"
+#include "lib/cc/compat_compiler.h"
+#include "lib/crypt_ops/crypto_init.h"
+#include "lib/crypt_ops/crypto_openssl_mgt.h"
 
+#ifdef ENABLE_OPENSSL
 /* Some versions of OpenSSL declare X509_STORE_CTX_set_verify_cb twice in
  * x509.h and x509_vfy.h. Suppress the GCC warning so we can build with
  * -Wredundant-decl. */
@@ -28,20 +31,24 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/err.h>
 
 ENABLE_GCC_WARNING(redundant-decls)
+#endif
 
 #include <errno.h>
-#if 0
-#include <stdlib.h>
-#include <stdarg.h>
-#include <assert.h>
-#endif
 
-#include "compat.h"
-#include "util.h"
-#include "torlog.h"
-#include "crypto.h"
-#include "address.h"
-#include "util_format.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_rsa.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/encoding/binascii.h"
+#include "lib/encoding/time_fmt.h"
+#include "lib/fs/files.h"
+#include "lib/log/log.h"
+#include "lib/malloc/malloc.h"
+#include "lib/net/address.h"
+#include "lib/net/inaddr.h"
+#include "lib/net/resolve.h"
+#include "lib/string/compat_string.h"
+#include "lib/string/printf.h"
 
 #define IDENTITY_KEY_BITS 3072
 #define SIGNING_KEY_BITS 2048
@@ -76,29 +83,6 @@ show_help(void)
           "[--passphrase-fd <fd>]\n");
 }
 
-/* XXXX copied from crypto.c */
-static void
-crypto_log_errors(int severity, const char *doing)
-{
-  unsigned long err;
-  const char *msg, *lib, *func;
-  while ((err = ERR_get_error()) != 0) {
-    msg = (const char*)ERR_reason_error_string(err);
-    lib = (const char*)ERR_lib_error_string(err);
-    func = (const char*)ERR_func_error_string(err);
-    if (!msg) msg = "(null)";
-    if (!lib) lib = "(null)";
-    if (!func) func = "(null)";
-    if (doing) {
-      tor_log(severity, LD_CRYPTO, "crypto error while %s: %s (in %s:%s)",
-              doing, msg, lib, func);
-    } else {
-      tor_log(severity, LD_CRYPTO, "crypto error: %s (in %s:%s)",
-              msg, lib, func);
-    }
-  }
-}
-
 /** Read the passphrase from the passphrase fd. */
 static int
 load_passphrase(void)
@@ -106,7 +90,7 @@ load_passphrase(void)
   char *cp;
   char buf[1024]; /* "Ought to be enough for anybody." */
   memset(buf, 0, sizeof(buf)); /* should be needless */
-  ssize_t n = read_all(passphrase_fd, buf, sizeof(buf), 0);
+  ssize_t n = read_all_from_fd(passphrase_fd, buf, sizeof(buf));
   if (n < 0) {
     log_err(LD_GENERAL, "Couldn't read from passphrase fd: %s",
             strerror(errno));
@@ -191,19 +175,23 @@ parse_commandline(int argc, char **argv)
     } else if (!strcmp(argv[i], "-v")) {
       verbose = 1;
     } else if (!strcmp(argv[i], "-a")) {
-      uint32_t addr;
+      tor_addr_t addr;
       uint16_t port;
-      char b[INET_NTOA_BUF_LEN];
-      struct in_addr in;
       if (i+1>=argc) {
         fprintf(stderr, "No argument to -a\n");
         return 1;
       }
-      if (addr_port_lookup(LOG_ERR, argv[++i], NULL, &addr, &port)<0)
+      const char *addr_arg = argv[++i];
+      if (tor_addr_port_lookup(addr_arg, &addr, &port)<0) {
+        fprintf(stderr, "Can't resolve address/port for %s", addr_arg);
         return 1;
-      in.s_addr = htonl(addr);
-      tor_inet_ntoa(&in, b, sizeof(b));
-      tor_asprintf(&address, "%s:%d", b, (int)port);
+      }
+      if (tor_addr_family(&addr) != AF_INET) {
+        fprintf(stderr, "%s must resolve to an IPv4 address", addr_arg);
+        return 1;
+      }
+      tor_free(address);
+      address = tor_strdup(fmt_addrport(&addr, port));
     } else if (!strcmp(argv[i], "--create-identity-key")) {
       make_new_id = 1;
     } else if (!strcmp(argv[i], "--passphrase-fd")) {
@@ -254,8 +242,7 @@ generate_key(int bits)
   crypto_pk_t *env = crypto_pk_new();
   if (crypto_pk_generate_key_with_bits(env,bits)<0)
     goto done;
-  rsa = crypto_pk_get_rsa_(env);
-  rsa = RSAPrivateKey_dup(rsa);
+  rsa = crypto_pk_get_openssl_rsa_(env);
  done:
   crypto_pk_free(env);
   return rsa;
@@ -283,7 +270,7 @@ load_identity_key(void)
                IDENTITY_KEY_BITS);
     if (!(key = generate_key(IDENTITY_KEY_BITS))) {
       log_err(LD_GENERAL, "Couldn't generate identity key.");
-      crypto_log_errors(LOG_ERR, "Generating identity key");
+      crypto_openssl_log_errors(LOG_ERR, "Generating identity key");
       return 1;
     }
     identity_key = EVP_PKEY_new();
@@ -305,7 +292,7 @@ load_identity_key(void)
                                        NULL, NULL)) {
       log_err(LD_GENERAL, "Couldn't write identity key to %s",
               identity_key_file);
-      crypto_log_errors(LOG_ERR, "Writing identity key");
+      crypto_openssl_log_errors(LOG_ERR, "Writing identity key");
       abort_writing_to_file(open_file);
       return 1;
     }
@@ -370,7 +357,7 @@ generate_signing_key(void)
              SIGNING_KEY_BITS);
   if (!(key = generate_key(SIGNING_KEY_BITS))) {
     log_err(LD_GENERAL, "Couldn't generate signing key.");
-    crypto_log_errors(LOG_ERR, "Generating signing key");
+    crypto_openssl_log_errors(LOG_ERR, "Generating signing key");
     return 1;
   }
   signing_key = EVP_PKEY_new();
@@ -386,7 +373,7 @@ generate_signing_key(void)
 
   /* Write signing key with no encryption. */
   if (!PEM_write_RSAPrivateKey(f, key, NULL, NULL, 0, NULL, NULL)) {
-    crypto_log_errors(LOG_WARN, "writing signing key");
+    crypto_openssl_log_errors(LOG_WARN, "writing signing key");
     abort_writing_to_file(open_file);
     return 1;
   }
@@ -410,7 +397,7 @@ key_to_string(EVP_PKEY *key)
 
   b = BIO_new(BIO_s_mem());
   if (!PEM_write_bio_RSAPublicKey(b, rsa)) {
-    crypto_log_errors(LOG_WARN, "writing public key to string");
+    crypto_openssl_log_errors(LOG_WARN, "writing public key to string");
     RSA_free(rsa);
     return NULL;
   }
@@ -430,8 +417,8 @@ key_to_string(EVP_PKEY *key)
 static int
 get_fingerprint(EVP_PKEY *pkey, char *out)
 {
-  int r = 1;
-  crypto_pk_t *pk = crypto_new_pk_from_rsa_(EVP_PKEY_get1_RSA(pkey));
+  int r = -1;
+  crypto_pk_t *pk = crypto_new_pk_from_openssl_rsa_(EVP_PKEY_get1_RSA(pkey));
   if (pk) {
     r = crypto_pk_get_fingerprint(pk, out, 0);
     crypto_pk_free(pk);
@@ -443,8 +430,8 @@ get_fingerprint(EVP_PKEY *pkey, char *out)
 static int
 get_digest(EVP_PKEY *pkey, char *out)
 {
-  int r = 1;
-  crypto_pk_t *pk = crypto_new_pk_from_rsa_(EVP_PKEY_get1_RSA(pkey));
+  int r = -1;
+  crypto_pk_t *pk = crypto_new_pk_from_openssl_rsa_(EVP_PKEY_get1_RSA(pkey));
   if (pk) {
     r = crypto_pk_get_digest(pk, out);
     crypto_pk_free(pk);
@@ -464,16 +451,20 @@ generate_certificate(void)
   char expires[ISO_TIME_LEN+1];
   char id_digest[DIGEST_LEN];
   char fingerprint[FINGERPRINT_LEN+1];
-  char *ident = key_to_string(identity_key);
-  char *signing = key_to_string(signing_key);
   FILE *f;
   size_t signed_len;
   char digest[DIGEST_LEN];
   char signature[1024]; /* handles up to 8192-bit keys. */
   int r;
 
-  get_fingerprint(identity_key, fingerprint);
-  get_digest(identity_key, id_digest);
+  if (get_fingerprint(identity_key, fingerprint) < 0) {
+    return -1;
+  }
+  if (get_digest(identity_key, id_digest)) {
+    return -1;
+  }
+  char *ident = key_to_string(identity_key);
+  char *signing = key_to_string(signing_key);
 
   tor_localtime_r(&now, &tm);
   tm.tm_mon += months_lifetime;
@@ -593,4 +584,3 @@ main(int argc, char **argv)
   crypto_global_cleanup();
   return r;
 }
-
diff --git a/src/tools/tor-print-ed-signing-cert.c b/src/tools/tor-print-ed-signing-cert.c
new file mode 100644
index 0000000000..1f1a01ab5c
--- /dev/null
+++ b/src/tools/tor-print-ed-signing-cert.c
@@ -0,0 +1,65 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+
+#include "trunnel/ed25519_cert.h"
+#include "lib/cc/torint.h"  /* TOR_PRIdSZ */
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/malloc/malloc.h"
+
+int
+main(int argc, char **argv)
+{
+  ed25519_cert_t *cert = NULL;
+
+  if (argc != 2) {
+    fprintf(stderr, "Usage:\n");
+    fprintf(stderr, "%s <path to ed25519_signing_cert file>\n", argv[0]);
+    return -1;
+  }
+
+  const char *filepath = argv[1];
+  char *got_tag = NULL;
+
+  uint8_t certbuf[256];
+  ssize_t cert_body_len = crypto_read_tagged_contents_from_file(
+                 filepath, "ed25519v1-cert",
+                 &got_tag, certbuf, sizeof(certbuf));
+
+  if (cert_body_len <= 0) {
+    fprintf(stderr, "crypto_read_tagged_contents_from_file failed with "
+                    "error: %s\n", strerror(errno));
+    return -2;
+  }
+
+  if (!got_tag) {
+    fprintf(stderr, "Found no tag\n");
+    return -3;
+  }
+
+  if (strcmp(got_tag, "type4") != 0) {
+    fprintf(stderr, "Wrong tag: %s\n", got_tag);
+    return -4;
+  }
+
+  tor_free(got_tag);
+
+  ssize_t parsed = ed25519_cert_parse(&cert, certbuf, cert_body_len);
+  if (parsed <= 0) {
+    fprintf(stderr, "ed25519_cert_parse failed with return value %" TOR_PRIdSZ
+                    "\n", parsed);
+    return -5;
+  }
+
+  time_t expires_at = (time_t)cert->exp_field * 60 * 60;
+
+  printf("Expires at: %s", ctime(&expires_at));
+
+  ed25519_cert_free(cert);
+
+  return 0;
+}
diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c
index 6ac866d3c0..6a84abe557 100644
--- a/src/tools/tor-resolve.c
+++ b/src/tools/tor-resolve.c
@@ -1,20 +1,25 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
- * Copyright (c) 2007-2015, The Tor Project, Inc.
+ * Copyright (c) 2007-2019, The Tor Project, Inc.
  */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
-#include "compat.h"
-#include "util.h"
-#include "address.h"
-#include "torlog.h"
-#include "sandbox.h"
+
+#include "lib/arch/bytes.h"
+#include "lib/log/log.h"
+#include "lib/malloc/malloc.h"
+#include "lib/net/address.h"
+#include "lib/net/resolve.h"
+#include "lib/net/socket.h"
+#include "lib/sandbox/sandbox.h"
+#include "lib/string/util_string.h"
+
+#include "lib/net/socks5_status.h"
 
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdarg.h>
 #include <string.h>
-#include <assert.h>
 
 #ifdef HAVE_NETINET_IN_H
 #include <netinet/in.h>
@@ -192,12 +197,14 @@ socks5_reason_to_string(char reason)
  * address (in host order) into *<b>result_addr</b>.
  */
 static int
-do_resolve(const char *hostname, uint32_t sockshost, uint16_t socksport,
+do_resolve(const char *hostname,
+           const tor_addr_t *sockshost, uint16_t socksport,
            int reverse, int version,
            tor_addr_t *result_addr, char **result_hostname)
 {
   int s = -1;
-  struct sockaddr_in socksaddr;
+  struct sockaddr_storage ss;
+  socklen_t socklen;
   char *req = NULL;
   ssize_t len = 0;
 
@@ -214,22 +221,21 @@ do_resolve(const char *hostname, uint32_t sockshost, uint16_t socksport,
     return -1;
   }
 
-  memset(&socksaddr, 0, sizeof(socksaddr));
-  socksaddr.sin_family = AF_INET;
-  socksaddr.sin_port = htons(socksport);
-  socksaddr.sin_addr.s_addr = htonl(sockshost);
-  if (connect(s, (struct sockaddr*)&socksaddr, sizeof(socksaddr))) {
+  socklen = tor_addr_to_sockaddr(sockshost, socksport,
+                                 (struct sockaddr *)&ss, sizeof(ss));
+
+  if (connect(s, (struct sockaddr*)&ss, socklen)) {
     log_sock_error("connecting to SOCKS host", s);
     goto err;
   }
 
   if (version == 5) {
     char method_buf[2];
-    if (write_all(s, "\x05\x01\x00", 3, 1) != 3) {
+    if (write_all_to_socket(s, "\x05\x01\x00", 3) != 3) {
       log_err(LD_NET, "Error sending SOCKS5 method list.");
       goto err;
     }
-    if (read_all(s, method_buf, 2, 1) != 2) {
+    if (read_all_from_socket(s, method_buf, 2) != 2) {
       log_err(LD_NET, "Error reading SOCKS5 methods.");
       goto err;
     }
@@ -251,7 +257,7 @@ do_resolve(const char *hostname, uint32_t sockshost, uint16_t socksport,
     tor_assert(!req);
     goto err;
   }
-  if (write_all(s, req, len, 1) != len) {
+  if (write_all_to_socket(s, req, len) != len) {
     log_sock_error("sending SOCKS request", s);
     tor_free(req);
     goto err;
@@ -260,7 +266,7 @@ do_resolve(const char *hostname, uint32_t sockshost, uint16_t socksport,
 
   if (version == 4) {
     char reply_buf[RESPONSE_LEN_4];
-    if (read_all(s, reply_buf, RESPONSE_LEN_4, 1) != RESPONSE_LEN_4) {
+    if (read_all_from_socket(s, reply_buf, RESPONSE_LEN_4) != RESPONSE_LEN_4) {
       log_err(LD_NET, "Error reading SOCKS4 response.");
       goto err;
     }
@@ -271,7 +277,7 @@ do_resolve(const char *hostname, uint32_t sockshost, uint16_t socksport,
     }
   } else {
     char reply_buf[16];
-    if (read_all(s, reply_buf, 4, 1) != 4) {
+    if (read_all_from_socket(s, reply_buf, 4) != 4) {
       log_err(LD_NET, "Error reading SOCKS5 response.");
       goto err;
     }
@@ -291,14 +297,14 @@ do_resolve(const char *hostname, uint32_t sockshost, uint16_t socksport,
     }
     if (reply_buf[3] == 1) {
       /* IPv4 address */
-      if (read_all(s, reply_buf, 4, 1) != 4) {
+      if (read_all_from_socket(s, reply_buf, 4) != 4) {
         log_err(LD_NET, "Error reading address in socks5 response.");
         goto err;
       }
       tor_addr_from_ipv4n(result_addr, get_uint32(reply_buf));
     } else if (reply_buf[3] == 4) {
       /* IPv6 address */
-      if (read_all(s, reply_buf, 16, 1) != 16) {
+      if (read_all_from_socket(s, reply_buf, 16) != 16) {
         log_err(LD_NET, "Error reading address in socks5 response.");
         goto err;
       }
@@ -306,13 +312,14 @@ do_resolve(const char *hostname, uint32_t sockshost, uint16_t socksport,
     } else if (reply_buf[3] == 3) {
       /* Domain name */
       size_t result_len;
-      if (read_all(s, reply_buf, 1, 1) != 1) {
+      if (read_all_from_socket(s, reply_buf, 1) != 1) {
         log_err(LD_NET, "Error reading address_length in socks5 response.");
         goto err;
       }
       result_len = *(uint8_t*)(reply_buf);
       *result_hostname = tor_malloc(result_len+1);
-      if (read_all(s, *result_hostname, result_len, 1) != (int) result_len) {
+      if (read_all_from_socket(s, *result_hostname, result_len)
+          != (int) result_len) {
         log_err(LD_NET, "Error reading hostname in socks5 response.");
         goto err;
       }
@@ -340,14 +347,13 @@ usage(void)
 int
 main(int argc, char **argv)
 {
-  uint32_t sockshost;
+  tor_addr_t sockshost;
   uint16_t socksport = 0, port_option = 0;
   int isSocks4 = 0, isVerbose = 0, isReverse = 0;
   char **arg;
   int n_args;
   tor_addr_t result;
   char *result_hostname = NULL;
-  log_severity_list_t *s = tor_malloc_zero(sizeof(log_severity_list_t));
 
   init_logging(1);
   sandbox_disable_getaddrinfo_cache();
@@ -398,15 +404,18 @@ main(int argc, char **argv)
     usage();
   }
 
+  log_severity_list_t *severities =
+    tor_malloc_zero(sizeof(log_severity_list_t));
   if (isVerbose)
-    set_log_severity_config(LOG_DEBUG, LOG_ERR, s);
+    set_log_severity_config(LOG_DEBUG, LOG_ERR, severities);
   else
-    set_log_severity_config(LOG_WARN, LOG_ERR, s);
-  add_stream_log(s, "<stderr>", fileno(stderr));
+    set_log_severity_config(LOG_WARN, LOG_ERR, severities);
+  add_stream_log(severities, "<stderr>", fileno(stderr));
+  tor_free(severities);
 
   if (n_args == 1) {
     log_debug(LD_CONFIG, "defaulting to localhost");
-    sockshost = 0x7f000001u; /* localhost */
+    tor_addr_from_ipv4h(&sockshost, 0x7f000001u); /* localhost */
     if (port_option) {
       log_debug(LD_CONFIG, "Using port %d", (int)port_option);
       socksport = port_option;
@@ -415,7 +424,7 @@ main(int argc, char **argv)
       socksport = 9050; /* 9050 */
     }
   } else if (n_args == 2) {
-    if (addr_port_lookup(LOG_WARN, arg[1], NULL, &sockshost, &socksport)<0) {
+    if (tor_addr_port_lookup(arg[1], &sockshost, &socksport)<0) {
       fprintf(stderr, "Couldn't parse/resolve address %s", arg[1]);
       return 1;
     }
@@ -437,7 +446,7 @@ main(int argc, char **argv)
     return 1;
   }
 
-  if (do_resolve(arg[0], sockshost, socksport, isReverse,
+  if (do_resolve(arg[0], &sockshost, socksport, isReverse,
                  isSocks4 ? 4 : 5, &result,
                  &result_hostname))
     return 1;
@@ -449,4 +458,3 @@ main(int argc, char **argv)
   }
   return 0;
 }
-
diff --git a/src/tools/tor_runner.c b/src/tools/tor_runner.c
new file mode 100644
index 0000000000..3c6ade91d9
--- /dev/null
+++ b/src/tools/tor_runner.c
@@ -0,0 +1,112 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file tor_runner.c
+ * @brief Experimental module to emulate tor_run_main() API with fork+exec
+ *
+ * The functions here are meant to allow the application developer to
+ * use the tor_run_main() API without having to care whether Tor is
+ * running in-process or out-of-process.  For in-process usage, the
+ * developer can link Tor as a library and call tor_run_main(); for
+ * out-of-process usage, the developer can link this library instead.
+ *
+ * This interface is EXPERIMENTAL; please let us know if you would like
+ * to depend on it.  We don't know yet whether it will be reliable in
+ * practice.
+ */
+
+/* NOTE: This module is supposed to work without the standard Tor utility
+ * functions.  Don't add more dependencies!
+ */
+
+#include "feature/api/tor_api.h"
+#include "feature/api/tor_api_internal.h"
+
+#include "orconfig.h"
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#include <stdlib.h>
+#include <string.h>
+
+#ifndef __GNUC__
+#define __attribute__(x)
+#endif
+
+static void child(const tor_main_configuration_t *cfg)
+  __attribute__((noreturn));
+
+const char *
+tor_api_get_provider_version(void)
+{
+  return "libtorrunner " VERSION;
+}
+
+int
+tor_run_main(const tor_main_configuration_t *cfg)
+{
+  pid_t pid = fork();
+  if (pid == 0) {
+    child(cfg);
+    exit(0); /* Unreachable */
+  }
+
+  pid_t stopped_pid;
+  int status = 0;
+  do {
+    stopped_pid = waitpid(pid, &status, 0);
+  } while (stopped_pid == -1);
+
+  /* Note: these return values are not documented.  No return value is
+   * documented! */
+
+  if (stopped_pid != pid) {
+    return -99999;
+  }
+  if (WIFSTOPPED(status)) {
+    return WEXITSTATUS(status);
+  }
+  if (WIFSIGNALED(status)) {
+    return -WTERMSIG(status);
+  }
+
+  return -999988;
+}
+
+/* circumlocution to avoid getting warned about calling calloc instead of
+ * tor_calloc. */
+#define real_calloc calloc
+#define real_free free
+
+static void
+child(const tor_main_configuration_t *cfg)
+{
+  /* XXXX Close unused file descriptors. */
+
+  char **args = real_calloc(cfg->argc + cfg->argc_owned+1, sizeof(char *));
+  memcpy(args, cfg->argv, cfg->argc * sizeof(char *));
+  if (cfg->argc_owned)
+    memcpy(args + cfg->argc, cfg->argv_owned,
+           cfg->argc_owned * sizeof(char *));
+
+  args[cfg->argc + cfg->argc_owned] = NULL;
+
+  int rv = execv(BINDIR "/tor", args);
+
+  if (rv < 0) {
+    real_free(args);
+    exit(254);
+  } else {
+    abort(); /* Unreachable */
+  }
+}
diff --git a/src/trunnel/channelpadding_negotiation.c b/src/trunnel/channelpadding_negotiation.c
new file mode 100644
index 0000000000..59e6b38384
--- /dev/null
+++ b/src/trunnel/channelpadding_negotiation.c
@@ -0,0 +1,281 @@
+/* channelpadding_negotiation.c -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "channelpadding_negotiation.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int channelpaddingnegotiation_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || channelpaddingnegotiation_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+channelpadding_negotiate_t *
+channelpadding_negotiate_new(void)
+{
+  channelpadding_negotiate_t *val = trunnel_calloc(1, sizeof(channelpadding_negotiate_t));
+  if (NULL == val)
+    return NULL;
+  val->command = CHANNELPADDING_COMMAND_START;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+channelpadding_negotiate_clear(channelpadding_negotiate_t *obj)
+{
+  (void) obj;
+}
+
+void
+channelpadding_negotiate_free(channelpadding_negotiate_t *obj)
+{
+  if (obj == NULL)
+    return;
+  channelpadding_negotiate_clear(obj);
+  trunnel_memwipe(obj, sizeof(channelpadding_negotiate_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+channelpadding_negotiate_get_version(const channelpadding_negotiate_t *inp)
+{
+  return inp->version;
+}
+int
+channelpadding_negotiate_set_version(channelpadding_negotiate_t *inp, uint8_t val)
+{
+  if (! ((val == 0))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint8_t
+channelpadding_negotiate_get_command(const channelpadding_negotiate_t *inp)
+{
+  return inp->command;
+}
+int
+channelpadding_negotiate_set_command(channelpadding_negotiate_t *inp, uint8_t val)
+{
+  if (! ((val == CHANNELPADDING_COMMAND_START || val == CHANNELPADDING_COMMAND_STOP))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->command = val;
+  return 0;
+}
+uint16_t
+channelpadding_negotiate_get_ito_low_ms(const channelpadding_negotiate_t *inp)
+{
+  return inp->ito_low_ms;
+}
+int
+channelpadding_negotiate_set_ito_low_ms(channelpadding_negotiate_t *inp, uint16_t val)
+{
+  inp->ito_low_ms = val;
+  return 0;
+}
+uint16_t
+channelpadding_negotiate_get_ito_high_ms(const channelpadding_negotiate_t *inp)
+{
+  return inp->ito_high_ms;
+}
+int
+channelpadding_negotiate_set_ito_high_ms(channelpadding_negotiate_t *inp, uint16_t val)
+{
+  inp->ito_high_ms = val;
+  return 0;
+}
+const char *
+channelpadding_negotiate_check(const channelpadding_negotiate_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 0))
+    return "Integer out of bounds";
+  if (! (obj->command == CHANNELPADDING_COMMAND_START || obj->command == CHANNELPADDING_COMMAND_STOP))
+    return "Integer out of bounds";
+  return NULL;
+}
+
+ssize_t
+channelpadding_negotiate_encoded_len(const channelpadding_negotiate_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != channelpadding_negotiate_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [0] */
+  result += 1;
+
+  /* Length of u8 command IN [CHANNELPADDING_COMMAND_START, CHANNELPADDING_COMMAND_STOP] */
+  result += 1;
+
+  /* Length of u16 ito_low_ms */
+  result += 2;
+
+  /* Length of u16 ito_high_ms */
+  result += 2;
+  return result;
+}
+int
+channelpadding_negotiate_clear_errors(channelpadding_negotiate_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+channelpadding_negotiate_encode(uint8_t *output, const size_t avail, const channelpadding_negotiate_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = channelpadding_negotiate_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = channelpadding_negotiate_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [0] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u8 command IN [CHANNELPADDING_COMMAND_START, CHANNELPADDING_COMMAND_STOP] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->command));
+  written += 1; ptr += 1;
+
+  /* Encode u16 ito_low_ms */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->ito_low_ms));
+  written += 2; ptr += 2;
+
+  /* Encode u16 ito_high_ms */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->ito_high_ms));
+  written += 2; ptr += 2;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As channelpadding_negotiate_parse(), but do not allocate the
+ * output object.
+ */
+static ssize_t
+channelpadding_negotiate_parse_into(channelpadding_negotiate_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [0] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 0))
+    goto fail;
+
+  /* Parse u8 command IN [CHANNELPADDING_COMMAND_START, CHANNELPADDING_COMMAND_STOP] */
+  CHECK_REMAINING(1, truncated);
+  obj->command = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->command == CHANNELPADDING_COMMAND_START || obj->command == CHANNELPADDING_COMMAND_STOP))
+    goto fail;
+
+  /* Parse u16 ito_low_ms */
+  CHECK_REMAINING(2, truncated);
+  obj->ito_low_ms = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u16 ito_high_ms */
+  CHECK_REMAINING(2, truncated);
+  obj->ito_high_ms = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+channelpadding_negotiate_parse(channelpadding_negotiate_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = channelpadding_negotiate_new();
+  if (NULL == *output)
+    return -1;
+  result = channelpadding_negotiate_parse_into(*output, input, len_in);
+  if (result < 0) {
+    channelpadding_negotiate_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/channelpadding_negotiation.h b/src/trunnel/channelpadding_negotiation.h
new file mode 100644
index 0000000000..fcfc232fea
--- /dev/null
+++ b/src/trunnel/channelpadding_negotiation.h
@@ -0,0 +1,98 @@
+/* channelpadding_negotiation.h -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_CHANNELPADDING_NEGOTIATION_H
+#define TRUNNEL_CHANNELPADDING_NEGOTIATION_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+#define CHANNELPADDING_COMMAND_STOP 1
+#define CHANNELPADDING_COMMAND_START 2
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_CHANNELPADDING_NEGOTIATE)
+struct channelpadding_negotiate_st {
+  uint8_t version;
+  uint8_t command;
+  uint16_t ito_low_ms;
+  uint16_t ito_high_ms;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct channelpadding_negotiate_st channelpadding_negotiate_t;
+/** Return a newly allocated channelpadding_negotiate with all
+ * elements set to zero.
+ */
+channelpadding_negotiate_t *channelpadding_negotiate_new(void);
+/** Release all storage held by the channelpadding_negotiate in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void channelpadding_negotiate_free(channelpadding_negotiate_t *victim);
+/** Try to parse a channelpadding_negotiate from the buffer in
+ * 'input', using up to 'len_in' bytes from the input buffer. On
+ * success, return the number of bytes consumed and set *output to the
+ * newly allocated channelpadding_negotiate_t. On failure, return -2
+ * if the input appears truncated, and -1 if the input is otherwise
+ * invalid.
+ */
+ssize_t channelpadding_negotiate_parse(channelpadding_negotiate_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * channelpadding_negotiate in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t channelpadding_negotiate_encoded_len(const channelpadding_negotiate_t *obj);
+/** Try to encode the channelpadding_negotiate from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t channelpadding_negotiate_encode(uint8_t *output, size_t avail, const channelpadding_negotiate_t *input);
+/** Check whether the internal state of the channelpadding_negotiate
+ * in 'obj' is consistent. Return NULL if it is, and a short message
+ * if it is not.
+ */
+const char *channelpadding_negotiate_check(const channelpadding_negotiate_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int channelpadding_negotiate_clear_errors(channelpadding_negotiate_t *obj);
+/** Return the value of the version field of the
+ * channelpadding_negotiate_t in 'inp'
+ */
+uint8_t channelpadding_negotiate_get_version(const channelpadding_negotiate_t *inp);
+/** Set the value of the version field of the
+ * channelpadding_negotiate_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int channelpadding_negotiate_set_version(channelpadding_negotiate_t *inp, uint8_t val);
+/** Return the value of the command field of the
+ * channelpadding_negotiate_t in 'inp'
+ */
+uint8_t channelpadding_negotiate_get_command(const channelpadding_negotiate_t *inp);
+/** Set the value of the command field of the
+ * channelpadding_negotiate_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int channelpadding_negotiate_set_command(channelpadding_negotiate_t *inp, uint8_t val);
+/** Return the value of the ito_low_ms field of the
+ * channelpadding_negotiate_t in 'inp'
+ */
+uint16_t channelpadding_negotiate_get_ito_low_ms(const channelpadding_negotiate_t *inp);
+/** Set the value of the ito_low_ms field of the
+ * channelpadding_negotiate_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int channelpadding_negotiate_set_ito_low_ms(channelpadding_negotiate_t *inp, uint16_t val);
+/** Return the value of the ito_high_ms field of the
+ * channelpadding_negotiate_t in 'inp'
+ */
+uint16_t channelpadding_negotiate_get_ito_high_ms(const channelpadding_negotiate_t *inp);
+/** Set the value of the ito_high_ms field of the
+ * channelpadding_negotiate_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int channelpadding_negotiate_set_ito_high_ms(channelpadding_negotiate_t *inp, uint16_t val);
+
+
+#endif
diff --git a/src/trunnel/channelpadding_negotiation.trunnel b/src/trunnel/channelpadding_negotiation.trunnel
new file mode 100644
index 0000000000..7f2d4795b0
--- /dev/null
+++ b/src/trunnel/channelpadding_negotiation.trunnel
@@ -0,0 +1,17 @@
+const CHANNELPADDING_COMMAND_STOP = 1;
+const CHANNELPADDING_COMMAND_START = 2;
+
+/* This command tells the relay to alter its min and max netflow
+   timeout range values, and send padding at that rate (resuming
+   if stopped). */
+struct channelpadding_negotiate {
+  u8 version IN [0];
+  u8 command IN [CHANNELPADDING_COMMAND_START, CHANNELPADDING_COMMAND_STOP];
+
+  /* Min must not be lower than the current consensus parameter
+     nf_ito_low. */
+  u16 ito_low_ms;
+
+  /* Max must not be lower than ito_low_ms */
+  u16 ito_high_ms;
+};
diff --git a/src/trunnel/ed25519_cert.c b/src/trunnel/ed25519_cert.c
index 24988d510b..1276c7a505 100644
--- a/src/trunnel/ed25519_cert.c
+++ b/src/trunnel/ed25519_cert.c
@@ -1,4 +1,4 @@
-/* ed25519_cert.c -- generated by Trunnel v1.4.6.
+/* ed25519_cert.c -- generated by Trunnel v1.5.2.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -13,7 +13,7 @@
   } while (0)
 
 #if defined(__COVERITY__) || defined(__clang_analyzer__)
-/* If we're runnning a static analysis tool, we don't want it to complain
+/* If we're running a static analysis tool, we don't want it to complain
  * that some of our remaining-bytes checks are dead-code. */
 int edcert_deadcode_dummy__ = 0;
 #define OR_DEADCODE_DUMMY || edcert_deadcode_dummy__
@@ -28,6 +28,281 @@ int edcert_deadcode_dummy__ = 0;
     }                                                            \
   } while (0)
 
+create2_cell_body_t *
+create2_cell_body_new(void)
+{
+  create2_cell_body_t *val = trunnel_calloc(1, sizeof(create2_cell_body_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+create2_cell_body_clear(create2_cell_body_t *obj)
+{
+  (void) obj;
+  TRUNNEL_DYNARRAY_WIPE(&obj->handshake_data);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->handshake_data);
+}
+
+void
+create2_cell_body_free(create2_cell_body_t *obj)
+{
+  if (obj == NULL)
+    return;
+  create2_cell_body_clear(obj);
+  trunnel_memwipe(obj, sizeof(create2_cell_body_t));
+  trunnel_free_(obj);
+}
+
+uint16_t
+create2_cell_body_get_handshake_type(const create2_cell_body_t *inp)
+{
+  return inp->handshake_type;
+}
+int
+create2_cell_body_set_handshake_type(create2_cell_body_t *inp, uint16_t val)
+{
+  inp->handshake_type = val;
+  return 0;
+}
+uint16_t
+create2_cell_body_get_handshake_len(const create2_cell_body_t *inp)
+{
+  return inp->handshake_len;
+}
+int
+create2_cell_body_set_handshake_len(create2_cell_body_t *inp, uint16_t val)
+{
+  inp->handshake_len = val;
+  return 0;
+}
+size_t
+create2_cell_body_getlen_handshake_data(const create2_cell_body_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->handshake_data);
+}
+
+uint8_t
+create2_cell_body_get_handshake_data(create2_cell_body_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->handshake_data, idx);
+}
+
+uint8_t
+create2_cell_body_getconst_handshake_data(const create2_cell_body_t *inp, size_t idx)
+{
+  return create2_cell_body_get_handshake_data((create2_cell_body_t*)inp, idx);
+}
+int
+create2_cell_body_set_handshake_data(create2_cell_body_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->handshake_data, idx, elt);
+  return 0;
+}
+int
+create2_cell_body_add_handshake_data(create2_cell_body_t *inp, uint8_t elt)
+{
+#if SIZE_MAX >= UINT16_MAX
+  if (inp->handshake_data.n_ == UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->handshake_data, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+create2_cell_body_getarray_handshake_data(create2_cell_body_t *inp)
+{
+  return inp->handshake_data.elts_;
+}
+const uint8_t  *
+create2_cell_body_getconstarray_handshake_data(const create2_cell_body_t *inp)
+{
+  return (const uint8_t  *)create2_cell_body_getarray_handshake_data((create2_cell_body_t*)inp);
+}
+int
+create2_cell_body_setlen_handshake_data(create2_cell_body_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+#if UINT16_MAX < SIZE_MAX
+  if (newlen > UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->handshake_data.allocated_,
+                 &inp->handshake_data.n_, inp->handshake_data.elts_, newlen,
+                 sizeof(inp->handshake_data.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->handshake_data.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+create2_cell_body_check(const create2_cell_body_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (TRUNNEL_DYNARRAY_LEN(&obj->handshake_data) != obj->handshake_len)
+    return "Length mismatch for handshake_data";
+  return NULL;
+}
+
+ssize_t
+create2_cell_body_encoded_len(const create2_cell_body_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != create2_cell_body_check(obj))
+     return -1;
+
+
+  /* Length of u16 handshake_type */
+  result += 2;
+
+  /* Length of u16 handshake_len */
+  result += 2;
+
+  /* Length of u8 handshake_data[handshake_len] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->handshake_data);
+  return result;
+}
+int
+create2_cell_body_clear_errors(create2_cell_body_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+create2_cell_body_encode(uint8_t *output, const size_t avail, const create2_cell_body_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = create2_cell_body_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = create2_cell_body_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u16 handshake_type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->handshake_type));
+  written += 2; ptr += 2;
+
+  /* Encode u16 handshake_len */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->handshake_len));
+  written += 2; ptr += 2;
+
+  /* Encode u8 handshake_data[handshake_len] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->handshake_data);
+    trunnel_assert(obj->handshake_len == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->handshake_data.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As create2_cell_body_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+create2_cell_body_parse_into(create2_cell_body_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u16 handshake_type */
+  CHECK_REMAINING(2, truncated);
+  obj->handshake_type = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u16 handshake_len */
+  CHECK_REMAINING(2, truncated);
+  obj->handshake_len = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u8 handshake_data[handshake_len] */
+  CHECK_REMAINING(obj->handshake_len, truncated);
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->handshake_data, obj->handshake_len, {});
+  obj->handshake_data.n_ = obj->handshake_len;
+  if (obj->handshake_len)
+    memcpy(obj->handshake_data.elts_, ptr, obj->handshake_len);
+  ptr += obj->handshake_len; remaining -= obj->handshake_len;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+create2_cell_body_parse(create2_cell_body_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = create2_cell_body_new();
+  if (NULL == *output)
+    return -1;
+  result = create2_cell_body_parse_into(*output, input, len_in);
+  if (result < 0) {
+    create2_cell_body_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
 ed25519_cert_extension_t *
 ed25519_cert_extension_new(void)
 {
@@ -58,7 +333,7 @@ ed25519_cert_extension_free(ed25519_cert_extension_t *obj)
 }
 
 uint16_t
-ed25519_cert_extension_get_ext_length(ed25519_cert_extension_t *inp)
+ed25519_cert_extension_get_ext_length(const ed25519_cert_extension_t *inp)
 {
   return inp->ext_length;
 }
@@ -69,7 +344,7 @@ ed25519_cert_extension_set_ext_length(ed25519_cert_extension_t *inp, uint16_t va
   return 0;
 }
 uint8_t
-ed25519_cert_extension_get_ext_type(ed25519_cert_extension_t *inp)
+ed25519_cert_extension_get_ext_type(const ed25519_cert_extension_t *inp)
 {
   return inp->ext_type;
 }
@@ -80,7 +355,7 @@ ed25519_cert_extension_set_ext_type(ed25519_cert_extension_t *inp, uint8_t val)
   return 0;
 }
 uint8_t
-ed25519_cert_extension_get_ext_flags(ed25519_cert_extension_t *inp)
+ed25519_cert_extension_get_ext_flags(const ed25519_cert_extension_t *inp)
 {
   return inp->ext_flags;
 }
@@ -97,12 +372,17 @@ ed25519_cert_extension_getlen_un_signing_key(const ed25519_cert_extension_t *inp
 }
 
 uint8_t
-ed25519_cert_extension_get_un_signing_key(const ed25519_cert_extension_t *inp, size_t idx)
+ed25519_cert_extension_get_un_signing_key(ed25519_cert_extension_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->un_signing_key[idx];
 }
 
+uint8_t
+ed25519_cert_extension_getconst_un_signing_key(const ed25519_cert_extension_t *inp, size_t idx)
+{
+  return ed25519_cert_extension_get_un_signing_key((ed25519_cert_extension_t*)inp, idx);
+}
 int
 ed25519_cert_extension_set_un_signing_key(ed25519_cert_extension_t *inp, size_t idx, uint8_t elt)
 {
@@ -116,6 +396,11 @@ ed25519_cert_extension_getarray_un_signing_key(ed25519_cert_extension_t *inp)
 {
   return inp->un_signing_key;
 }
+const uint8_t  *
+ed25519_cert_extension_getconstarray_un_signing_key(const ed25519_cert_extension_t *inp)
+{
+  return (const uint8_t  *)ed25519_cert_extension_getarray_un_signing_key((ed25519_cert_extension_t*)inp);
+}
 size_t
 ed25519_cert_extension_getlen_un_unparsed(const ed25519_cert_extension_t *inp)
 {
@@ -128,6 +413,11 @@ ed25519_cert_extension_get_un_unparsed(ed25519_cert_extension_t *inp, size_t idx
   return TRUNNEL_DYNARRAY_GET(&inp->un_unparsed, idx);
 }
 
+uint8_t
+ed25519_cert_extension_getconst_un_unparsed(const ed25519_cert_extension_t *inp, size_t idx)
+{
+  return ed25519_cert_extension_get_un_unparsed((ed25519_cert_extension_t*)inp, idx);
+}
 int
 ed25519_cert_extension_set_un_unparsed(ed25519_cert_extension_t *inp, size_t idx, uint8_t elt)
 {
@@ -149,6 +439,11 @@ ed25519_cert_extension_getarray_un_unparsed(ed25519_cert_extension_t *inp)
 {
   return inp->un_unparsed.elts_;
 }
+const uint8_t  *
+ed25519_cert_extension_getconstarray_un_unparsed(const ed25519_cert_extension_t *inp)
+{
+  return (const uint8_t  *)ed25519_cert_extension_getarray_un_unparsed((ed25519_cert_extension_t*)inp);
+}
 int
 ed25519_cert_extension_setlen_un_unparsed(ed25519_cert_extension_t *inp, size_t newlen)
 {
@@ -410,6 +705,878 @@ ed25519_cert_extension_parse(ed25519_cert_extension_t **output, const uint8_t *i
   }
   return result;
 }
+extend1_cell_body_t *
+extend1_cell_body_new(void)
+{
+  extend1_cell_body_t *val = trunnel_calloc(1, sizeof(extend1_cell_body_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+extend1_cell_body_clear(extend1_cell_body_t *obj)
+{
+  (void) obj;
+}
+
+void
+extend1_cell_body_free(extend1_cell_body_t *obj)
+{
+  if (obj == NULL)
+    return;
+  extend1_cell_body_clear(obj);
+  trunnel_memwipe(obj, sizeof(extend1_cell_body_t));
+  trunnel_free_(obj);
+}
+
+uint32_t
+extend1_cell_body_get_ipv4addr(const extend1_cell_body_t *inp)
+{
+  return inp->ipv4addr;
+}
+int
+extend1_cell_body_set_ipv4addr(extend1_cell_body_t *inp, uint32_t val)
+{
+  inp->ipv4addr = val;
+  return 0;
+}
+uint16_t
+extend1_cell_body_get_port(const extend1_cell_body_t *inp)
+{
+  return inp->port;
+}
+int
+extend1_cell_body_set_port(extend1_cell_body_t *inp, uint16_t val)
+{
+  inp->port = val;
+  return 0;
+}
+size_t
+extend1_cell_body_getlen_onionskin(const extend1_cell_body_t *inp)
+{
+  (void)inp;  return 186;
+}
+
+uint8_t
+extend1_cell_body_get_onionskin(extend1_cell_body_t *inp, size_t idx)
+{
+  trunnel_assert(idx < 186);
+  return inp->onionskin[idx];
+}
+
+uint8_t
+extend1_cell_body_getconst_onionskin(const extend1_cell_body_t *inp, size_t idx)
+{
+  return extend1_cell_body_get_onionskin((extend1_cell_body_t*)inp, idx);
+}
+int
+extend1_cell_body_set_onionskin(extend1_cell_body_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < 186);
+  inp->onionskin[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+extend1_cell_body_getarray_onionskin(extend1_cell_body_t *inp)
+{
+  return inp->onionskin;
+}
+const uint8_t  *
+extend1_cell_body_getconstarray_onionskin(const extend1_cell_body_t *inp)
+{
+  return (const uint8_t  *)extend1_cell_body_getarray_onionskin((extend1_cell_body_t*)inp);
+}
+size_t
+extend1_cell_body_getlen_identity(const extend1_cell_body_t *inp)
+{
+  (void)inp;  return 20;
+}
+
+uint8_t
+extend1_cell_body_get_identity(extend1_cell_body_t *inp, size_t idx)
+{
+  trunnel_assert(idx < 20);
+  return inp->identity[idx];
+}
+
+uint8_t
+extend1_cell_body_getconst_identity(const extend1_cell_body_t *inp, size_t idx)
+{
+  return extend1_cell_body_get_identity((extend1_cell_body_t*)inp, idx);
+}
+int
+extend1_cell_body_set_identity(extend1_cell_body_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < 20);
+  inp->identity[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+extend1_cell_body_getarray_identity(extend1_cell_body_t *inp)
+{
+  return inp->identity;
+}
+const uint8_t  *
+extend1_cell_body_getconstarray_identity(const extend1_cell_body_t *inp)
+{
+  return (const uint8_t  *)extend1_cell_body_getarray_identity((extend1_cell_body_t*)inp);
+}
+const char *
+extend1_cell_body_check(const extend1_cell_body_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  return NULL;
+}
+
+ssize_t
+extend1_cell_body_encoded_len(const extend1_cell_body_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != extend1_cell_body_check(obj))
+     return -1;
+
+
+  /* Length of u32 ipv4addr */
+  result += 4;
+
+  /* Length of u16 port */
+  result += 2;
+
+  /* Length of u8 onionskin[186] */
+  result += 186;
+
+  /* Length of u8 identity[20] */
+  result += 20;
+  return result;
+}
+int
+extend1_cell_body_clear_errors(extend1_cell_body_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+extend1_cell_body_encode(uint8_t *output, const size_t avail, const extend1_cell_body_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = extend1_cell_body_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = extend1_cell_body_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u32 ipv4addr */
+  trunnel_assert(written <= avail);
+  if (avail - written < 4)
+    goto truncated;
+  trunnel_set_uint32(ptr, trunnel_htonl(obj->ipv4addr));
+  written += 4; ptr += 4;
+
+  /* Encode u16 port */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->port));
+  written += 2; ptr += 2;
+
+  /* Encode u8 onionskin[186] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 186)
+    goto truncated;
+  memcpy(ptr, obj->onionskin, 186);
+  written += 186; ptr += 186;
+
+  /* Encode u8 identity[20] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 20)
+    goto truncated;
+  memcpy(ptr, obj->identity, 20);
+  written += 20; ptr += 20;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As extend1_cell_body_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+extend1_cell_body_parse_into(extend1_cell_body_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u32 ipv4addr */
+  CHECK_REMAINING(4, truncated);
+  obj->ipv4addr = trunnel_ntohl(trunnel_get_uint32(ptr));
+  remaining -= 4; ptr += 4;
+
+  /* Parse u16 port */
+  CHECK_REMAINING(2, truncated);
+  obj->port = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u8 onionskin[186] */
+  CHECK_REMAINING(186, truncated);
+  memcpy(obj->onionskin, ptr, 186);
+  remaining -= 186; ptr += 186;
+
+  /* Parse u8 identity[20] */
+  CHECK_REMAINING(20, truncated);
+  memcpy(obj->identity, ptr, 20);
+  remaining -= 20; ptr += 20;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+}
+
+ssize_t
+extend1_cell_body_parse(extend1_cell_body_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = extend1_cell_body_new();
+  if (NULL == *output)
+    return -1;
+  result = extend1_cell_body_parse_into(*output, input, len_in);
+  if (result < 0) {
+    extend1_cell_body_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+link_specifier_t *
+link_specifier_new(void)
+{
+  link_specifier_t *val = trunnel_calloc(1, sizeof(link_specifier_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+link_specifier_clear(link_specifier_t *obj)
+{
+  (void) obj;
+  TRUNNEL_DYNARRAY_WIPE(&obj->un_unrecognized);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->un_unrecognized);
+}
+
+void
+link_specifier_free(link_specifier_t *obj)
+{
+  if (obj == NULL)
+    return;
+  link_specifier_clear(obj);
+  trunnel_memwipe(obj, sizeof(link_specifier_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+link_specifier_get_ls_type(const link_specifier_t *inp)
+{
+  return inp->ls_type;
+}
+int
+link_specifier_set_ls_type(link_specifier_t *inp, uint8_t val)
+{
+  inp->ls_type = val;
+  return 0;
+}
+uint8_t
+link_specifier_get_ls_len(const link_specifier_t *inp)
+{
+  return inp->ls_len;
+}
+int
+link_specifier_set_ls_len(link_specifier_t *inp, uint8_t val)
+{
+  inp->ls_len = val;
+  return 0;
+}
+uint32_t
+link_specifier_get_un_ipv4_addr(const link_specifier_t *inp)
+{
+  return inp->un_ipv4_addr;
+}
+int
+link_specifier_set_un_ipv4_addr(link_specifier_t *inp, uint32_t val)
+{
+  inp->un_ipv4_addr = val;
+  return 0;
+}
+uint16_t
+link_specifier_get_un_ipv4_port(const link_specifier_t *inp)
+{
+  return inp->un_ipv4_port;
+}
+int
+link_specifier_set_un_ipv4_port(link_specifier_t *inp, uint16_t val)
+{
+  inp->un_ipv4_port = val;
+  return 0;
+}
+size_t
+link_specifier_getlen_un_ipv6_addr(const link_specifier_t *inp)
+{
+  (void)inp;  return 16;
+}
+
+uint8_t
+link_specifier_get_un_ipv6_addr(link_specifier_t *inp, size_t idx)
+{
+  trunnel_assert(idx < 16);
+  return inp->un_ipv6_addr[idx];
+}
+
+uint8_t
+link_specifier_getconst_un_ipv6_addr(const link_specifier_t *inp, size_t idx)
+{
+  return link_specifier_get_un_ipv6_addr((link_specifier_t*)inp, idx);
+}
+int
+link_specifier_set_un_ipv6_addr(link_specifier_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < 16);
+  inp->un_ipv6_addr[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+link_specifier_getarray_un_ipv6_addr(link_specifier_t *inp)
+{
+  return inp->un_ipv6_addr;
+}
+const uint8_t  *
+link_specifier_getconstarray_un_ipv6_addr(const link_specifier_t *inp)
+{
+  return (const uint8_t  *)link_specifier_getarray_un_ipv6_addr((link_specifier_t*)inp);
+}
+uint16_t
+link_specifier_get_un_ipv6_port(const link_specifier_t *inp)
+{
+  return inp->un_ipv6_port;
+}
+int
+link_specifier_set_un_ipv6_port(link_specifier_t *inp, uint16_t val)
+{
+  inp->un_ipv6_port = val;
+  return 0;
+}
+size_t
+link_specifier_getlen_un_legacy_id(const link_specifier_t *inp)
+{
+  (void)inp;  return 20;
+}
+
+uint8_t
+link_specifier_get_un_legacy_id(link_specifier_t *inp, size_t idx)
+{
+  trunnel_assert(idx < 20);
+  return inp->un_legacy_id[idx];
+}
+
+uint8_t
+link_specifier_getconst_un_legacy_id(const link_specifier_t *inp, size_t idx)
+{
+  return link_specifier_get_un_legacy_id((link_specifier_t*)inp, idx);
+}
+int
+link_specifier_set_un_legacy_id(link_specifier_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < 20);
+  inp->un_legacy_id[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+link_specifier_getarray_un_legacy_id(link_specifier_t *inp)
+{
+  return inp->un_legacy_id;
+}
+const uint8_t  *
+link_specifier_getconstarray_un_legacy_id(const link_specifier_t *inp)
+{
+  return (const uint8_t  *)link_specifier_getarray_un_legacy_id((link_specifier_t*)inp);
+}
+size_t
+link_specifier_getlen_un_ed25519_id(const link_specifier_t *inp)
+{
+  (void)inp;  return 32;
+}
+
+uint8_t
+link_specifier_get_un_ed25519_id(link_specifier_t *inp, size_t idx)
+{
+  trunnel_assert(idx < 32);
+  return inp->un_ed25519_id[idx];
+}
+
+uint8_t
+link_specifier_getconst_un_ed25519_id(const link_specifier_t *inp, size_t idx)
+{
+  return link_specifier_get_un_ed25519_id((link_specifier_t*)inp, idx);
+}
+int
+link_specifier_set_un_ed25519_id(link_specifier_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < 32);
+  inp->un_ed25519_id[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+link_specifier_getarray_un_ed25519_id(link_specifier_t *inp)
+{
+  return inp->un_ed25519_id;
+}
+const uint8_t  *
+link_specifier_getconstarray_un_ed25519_id(const link_specifier_t *inp)
+{
+  return (const uint8_t  *)link_specifier_getarray_un_ed25519_id((link_specifier_t*)inp);
+}
+size_t
+link_specifier_getlen_un_unrecognized(const link_specifier_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->un_unrecognized);
+}
+
+uint8_t
+link_specifier_get_un_unrecognized(link_specifier_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->un_unrecognized, idx);
+}
+
+uint8_t
+link_specifier_getconst_un_unrecognized(const link_specifier_t *inp, size_t idx)
+{
+  return link_specifier_get_un_unrecognized((link_specifier_t*)inp, idx);
+}
+int
+link_specifier_set_un_unrecognized(link_specifier_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->un_unrecognized, idx, elt);
+  return 0;
+}
+int
+link_specifier_add_un_unrecognized(link_specifier_t *inp, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->un_unrecognized, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+link_specifier_getarray_un_unrecognized(link_specifier_t *inp)
+{
+  return inp->un_unrecognized.elts_;
+}
+const uint8_t  *
+link_specifier_getconstarray_un_unrecognized(const link_specifier_t *inp)
+{
+  return (const uint8_t  *)link_specifier_getarray_un_unrecognized((link_specifier_t*)inp);
+}
+int
+link_specifier_setlen_un_unrecognized(link_specifier_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+  newptr = trunnel_dynarray_setlen(&inp->un_unrecognized.allocated_,
+                 &inp->un_unrecognized.n_, inp->un_unrecognized.elts_, newlen,
+                 sizeof(inp->un_unrecognized.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->un_unrecognized.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+link_specifier_check(const link_specifier_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  switch (obj->ls_type) {
+
+    case LS_IPV4:
+      break;
+
+    case LS_IPV6:
+      break;
+
+    case LS_LEGACY_ID:
+      break;
+
+    case LS_ED25519_ID:
+      break;
+
+    default:
+      break;
+  }
+  return NULL;
+}
+
+ssize_t
+link_specifier_encoded_len(const link_specifier_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != link_specifier_check(obj))
+     return -1;
+
+
+  /* Length of u8 ls_type */
+  result += 1;
+
+  /* Length of u8 ls_len */
+  result += 1;
+  switch (obj->ls_type) {
+
+    case LS_IPV4:
+
+      /* Length of u32 un_ipv4_addr */
+      result += 4;
+
+      /* Length of u16 un_ipv4_port */
+      result += 2;
+      break;
+
+    case LS_IPV6:
+
+      /* Length of u8 un_ipv6_addr[16] */
+      result += 16;
+
+      /* Length of u16 un_ipv6_port */
+      result += 2;
+      break;
+
+    case LS_LEGACY_ID:
+
+      /* Length of u8 un_legacy_id[20] */
+      result += 20;
+      break;
+
+    case LS_ED25519_ID:
+
+      /* Length of u8 un_ed25519_id[32] */
+      result += 32;
+      break;
+
+    default:
+
+      /* Length of u8 un_unrecognized[] */
+      result += TRUNNEL_DYNARRAY_LEN(&obj->un_unrecognized);
+      break;
+  }
+  return result;
+}
+int
+link_specifier_clear_errors(link_specifier_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+link_specifier_encode(uint8_t *output, const size_t avail, const link_specifier_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = link_specifier_encoded_len(obj);
+#endif
+
+  uint8_t *backptr_ls_len = NULL;
+
+  if (NULL != (msg = link_specifier_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 ls_type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->ls_type));
+  written += 1; ptr += 1;
+
+  /* Encode u8 ls_len */
+  backptr_ls_len = ptr;
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->ls_len));
+  written += 1; ptr += 1;
+  {
+    size_t written_before_union = written;
+
+    /* Encode union un[ls_type] */
+    trunnel_assert(written <= avail);
+    switch (obj->ls_type) {
+
+      case LS_IPV4:
+
+        /* Encode u32 un_ipv4_addr */
+        trunnel_assert(written <= avail);
+        if (avail - written < 4)
+          goto truncated;
+        trunnel_set_uint32(ptr, trunnel_htonl(obj->un_ipv4_addr));
+        written += 4; ptr += 4;
+
+        /* Encode u16 un_ipv4_port */
+        trunnel_assert(written <= avail);
+        if (avail - written < 2)
+          goto truncated;
+        trunnel_set_uint16(ptr, trunnel_htons(obj->un_ipv4_port));
+        written += 2; ptr += 2;
+        break;
+
+      case LS_IPV6:
+
+        /* Encode u8 un_ipv6_addr[16] */
+        trunnel_assert(written <= avail);
+        if (avail - written < 16)
+          goto truncated;
+        memcpy(ptr, obj->un_ipv6_addr, 16);
+        written += 16; ptr += 16;
+
+        /* Encode u16 un_ipv6_port */
+        trunnel_assert(written <= avail);
+        if (avail - written < 2)
+          goto truncated;
+        trunnel_set_uint16(ptr, trunnel_htons(obj->un_ipv6_port));
+        written += 2; ptr += 2;
+        break;
+
+      case LS_LEGACY_ID:
+
+        /* Encode u8 un_legacy_id[20] */
+        trunnel_assert(written <= avail);
+        if (avail - written < 20)
+          goto truncated;
+        memcpy(ptr, obj->un_legacy_id, 20);
+        written += 20; ptr += 20;
+        break;
+
+      case LS_ED25519_ID:
+
+        /* Encode u8 un_ed25519_id[32] */
+        trunnel_assert(written <= avail);
+        if (avail - written < 32)
+          goto truncated;
+        memcpy(ptr, obj->un_ed25519_id, 32);
+        written += 32; ptr += 32;
+        break;
+
+      default:
+
+        /* Encode u8 un_unrecognized[] */
+        {
+          size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->un_unrecognized);
+          trunnel_assert(written <= avail);
+          if (avail - written < elt_len)
+            goto truncated;
+          if (elt_len)
+            memcpy(ptr, obj->un_unrecognized.elts_, elt_len);
+          written += elt_len; ptr += elt_len;
+        }
+        break;
+    }
+    /* Write the length field back to ls_len */
+    trunnel_assert(written >= written_before_union);
+#if UINT8_MAX < SIZE_MAX
+    if (written - written_before_union > UINT8_MAX)
+      goto check_failed;
+#endif
+    trunnel_set_uint8(backptr_ls_len, (written - written_before_union));
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As link_specifier_parse(), but do not allocate the output object.
+ */
+static ssize_t
+link_specifier_parse_into(link_specifier_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 ls_type */
+  CHECK_REMAINING(1, truncated);
+  obj->ls_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u8 ls_len */
+  CHECK_REMAINING(1, truncated);
+  obj->ls_len = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  {
+    size_t remaining_after;
+    CHECK_REMAINING(obj->ls_len, truncated);
+    remaining_after = remaining - obj->ls_len;
+    remaining = obj->ls_len;
+
+    /* Parse union un[ls_type] */
+    switch (obj->ls_type) {
+
+      case LS_IPV4:
+
+        /* Parse u32 un_ipv4_addr */
+        CHECK_REMAINING(4, fail);
+        obj->un_ipv4_addr = trunnel_ntohl(trunnel_get_uint32(ptr));
+        remaining -= 4; ptr += 4;
+
+        /* Parse u16 un_ipv4_port */
+        CHECK_REMAINING(2, fail);
+        obj->un_ipv4_port = trunnel_ntohs(trunnel_get_uint16(ptr));
+        remaining -= 2; ptr += 2;
+        break;
+
+      case LS_IPV6:
+
+        /* Parse u8 un_ipv6_addr[16] */
+        CHECK_REMAINING(16, fail);
+        memcpy(obj->un_ipv6_addr, ptr, 16);
+        remaining -= 16; ptr += 16;
+
+        /* Parse u16 un_ipv6_port */
+        CHECK_REMAINING(2, fail);
+        obj->un_ipv6_port = trunnel_ntohs(trunnel_get_uint16(ptr));
+        remaining -= 2; ptr += 2;
+        break;
+
+      case LS_LEGACY_ID:
+
+        /* Parse u8 un_legacy_id[20] */
+        CHECK_REMAINING(20, fail);
+        memcpy(obj->un_legacy_id, ptr, 20);
+        remaining -= 20; ptr += 20;
+        break;
+
+      case LS_ED25519_ID:
+
+        /* Parse u8 un_ed25519_id[32] */
+        CHECK_REMAINING(32, fail);
+        memcpy(obj->un_ed25519_id, ptr, 32);
+        remaining -= 32; ptr += 32;
+        break;
+
+      default:
+
+        /* Parse u8 un_unrecognized[] */
+        TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->un_unrecognized, remaining, {});
+        obj->un_unrecognized.n_ = remaining;
+        if (remaining)
+          memcpy(obj->un_unrecognized.elts_, ptr, remaining);
+        ptr += remaining; remaining -= remaining;
+        break;
+    }
+    if (remaining != 0)
+      goto fail;
+    remaining = remaining_after;
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ trunnel_alloc_failed:
+  return -1;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+link_specifier_parse(link_specifier_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = link_specifier_new();
+  if (NULL == *output)
+    return -1;
+  result = link_specifier_parse_into(*output, input, len_in);
+  if (result < 0) {
+    link_specifier_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
 ed25519_cert_t *
 ed25519_cert_new(void)
 {
@@ -448,7 +1615,7 @@ ed25519_cert_free(ed25519_cert_t *obj)
 }
 
 uint8_t
-ed25519_cert_get_version(ed25519_cert_t *inp)
+ed25519_cert_get_version(const ed25519_cert_t *inp)
 {
   return inp->version;
 }
@@ -463,7 +1630,7 @@ ed25519_cert_set_version(ed25519_cert_t *inp, uint8_t val)
   return 0;
 }
 uint8_t
-ed25519_cert_get_cert_type(ed25519_cert_t *inp)
+ed25519_cert_get_cert_type(const ed25519_cert_t *inp)
 {
   return inp->cert_type;
 }
@@ -474,7 +1641,7 @@ ed25519_cert_set_cert_type(ed25519_cert_t *inp, uint8_t val)
   return 0;
 }
 uint32_t
-ed25519_cert_get_exp_field(ed25519_cert_t *inp)
+ed25519_cert_get_exp_field(const ed25519_cert_t *inp)
 {
   return inp->exp_field;
 }
@@ -485,7 +1652,7 @@ ed25519_cert_set_exp_field(ed25519_cert_t *inp, uint32_t val)
   return 0;
 }
 uint8_t
-ed25519_cert_get_cert_key_type(ed25519_cert_t *inp)
+ed25519_cert_get_cert_key_type(const ed25519_cert_t *inp)
 {
   return inp->cert_key_type;
 }
@@ -502,12 +1669,17 @@ ed25519_cert_getlen_certified_key(const ed25519_cert_t *inp)
 }
 
 uint8_t
-ed25519_cert_get_certified_key(const ed25519_cert_t *inp, size_t idx)
+ed25519_cert_get_certified_key(ed25519_cert_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->certified_key[idx];
 }
 
+uint8_t
+ed25519_cert_getconst_certified_key(const ed25519_cert_t *inp, size_t idx)
+{
+  return ed25519_cert_get_certified_key((ed25519_cert_t*)inp, idx);
+}
 int
 ed25519_cert_set_certified_key(ed25519_cert_t *inp, size_t idx, uint8_t elt)
 {
@@ -521,8 +1693,13 @@ ed25519_cert_getarray_certified_key(ed25519_cert_t *inp)
 {
   return inp->certified_key;
 }
+const uint8_t  *
+ed25519_cert_getconstarray_certified_key(const ed25519_cert_t *inp)
+{
+  return (const uint8_t  *)ed25519_cert_getarray_certified_key((ed25519_cert_t*)inp);
+}
 uint8_t
-ed25519_cert_get_n_extensions(ed25519_cert_t *inp)
+ed25519_cert_get_n_extensions(const ed25519_cert_t *inp)
 {
   return inp->n_extensions;
 }
@@ -544,6 +1721,11 @@ ed25519_cert_get_ext(ed25519_cert_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->ext, idx);
 }
 
+ const struct ed25519_cert_extension_st *
+ed25519_cert_getconst_ext(const ed25519_cert_t *inp, size_t idx)
+{
+  return ed25519_cert_get_ext((ed25519_cert_t*)inp, idx);
+}
 int
 ed25519_cert_set_ext(ed25519_cert_t *inp, size_t idx, struct ed25519_cert_extension_st * elt)
 {
@@ -577,6 +1759,11 @@ ed25519_cert_getarray_ext(ed25519_cert_t *inp)
 {
   return inp->ext.elts_;
 }
+const struct ed25519_cert_extension_st *  const  *
+ed25519_cert_getconstarray_ext(const ed25519_cert_t *inp)
+{
+  return (const struct ed25519_cert_extension_st *  const  *)ed25519_cert_getarray_ext((ed25519_cert_t*)inp);
+}
 int
 ed25519_cert_setlen_ext(ed25519_cert_t *inp, size_t newlen)
 {
@@ -604,12 +1791,17 @@ ed25519_cert_getlen_signature(const ed25519_cert_t *inp)
 }
 
 uint8_t
-ed25519_cert_get_signature(const ed25519_cert_t *inp, size_t idx)
+ed25519_cert_get_signature(ed25519_cert_t *inp, size_t idx)
 {
   trunnel_assert(idx < 64);
   return inp->signature[idx];
 }
 
+uint8_t
+ed25519_cert_getconst_signature(const ed25519_cert_t *inp, size_t idx)
+{
+  return ed25519_cert_get_signature((ed25519_cert_t*)inp, idx);
+}
 int
 ed25519_cert_set_signature(ed25519_cert_t *inp, size_t idx, uint8_t elt)
 {
@@ -623,6 +1815,11 @@ ed25519_cert_getarray_signature(ed25519_cert_t *inp)
 {
   return inp->signature;
 }
+const uint8_t  *
+ed25519_cert_getconstarray_signature(const ed25519_cert_t *inp)
+{
+  return (const uint8_t  *)ed25519_cert_getarray_signature((ed25519_cert_t*)inp);
+}
 const char *
 ed25519_cert_check(const ed25519_cert_t *obj)
 {
@@ -887,3 +2084,630 @@ ed25519_cert_parse(ed25519_cert_t **output, const uint8_t *input, const size_t l
   }
   return result;
 }
+extend2_cell_body_t *
+extend2_cell_body_new(void)
+{
+  extend2_cell_body_t *val = trunnel_calloc(1, sizeof(extend2_cell_body_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+extend2_cell_body_clear(extend2_cell_body_t *obj)
+{
+  (void) obj;
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->ls); ++idx) {
+      link_specifier_free(TRUNNEL_DYNARRAY_GET(&obj->ls, idx));
+    }
+  }
+  TRUNNEL_DYNARRAY_WIPE(&obj->ls);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->ls);
+  create2_cell_body_free(obj->create2);
+  obj->create2 = NULL;
+}
+
+void
+extend2_cell_body_free(extend2_cell_body_t *obj)
+{
+  if (obj == NULL)
+    return;
+  extend2_cell_body_clear(obj);
+  trunnel_memwipe(obj, sizeof(extend2_cell_body_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+extend2_cell_body_get_n_spec(const extend2_cell_body_t *inp)
+{
+  return inp->n_spec;
+}
+int
+extend2_cell_body_set_n_spec(extend2_cell_body_t *inp, uint8_t val)
+{
+  inp->n_spec = val;
+  return 0;
+}
+size_t
+extend2_cell_body_getlen_ls(const extend2_cell_body_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->ls);
+}
+
+struct link_specifier_st *
+extend2_cell_body_get_ls(extend2_cell_body_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->ls, idx);
+}
+
+ const struct link_specifier_st *
+extend2_cell_body_getconst_ls(const extend2_cell_body_t *inp, size_t idx)
+{
+  return extend2_cell_body_get_ls((extend2_cell_body_t*)inp, idx);
+}
+int
+extend2_cell_body_set_ls(extend2_cell_body_t *inp, size_t idx, struct link_specifier_st * elt)
+{
+  link_specifier_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->ls, idx);
+  if (oldval && oldval != elt)
+    link_specifier_free(oldval);
+  return extend2_cell_body_set0_ls(inp, idx, elt);
+}
+int
+extend2_cell_body_set0_ls(extend2_cell_body_t *inp, size_t idx, struct link_specifier_st * elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->ls, idx, elt);
+  return 0;
+}
+int
+extend2_cell_body_add_ls(extend2_cell_body_t *inp, struct link_specifier_st * elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->ls.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(struct link_specifier_st *, &inp->ls, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+struct link_specifier_st * *
+extend2_cell_body_getarray_ls(extend2_cell_body_t *inp)
+{
+  return inp->ls.elts_;
+}
+const struct link_specifier_st *  const  *
+extend2_cell_body_getconstarray_ls(const extend2_cell_body_t *inp)
+{
+  return (const struct link_specifier_st *  const  *)extend2_cell_body_getarray_ls((extend2_cell_body_t*)inp);
+}
+int
+extend2_cell_body_setlen_ls(extend2_cell_body_t *inp, size_t newlen)
+{
+  struct link_specifier_st * *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->ls.allocated_,
+                 &inp->ls.n_, inp->ls.elts_, newlen,
+                 sizeof(inp->ls.elts_[0]), (trunnel_free_fn_t) link_specifier_free,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->ls.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+struct create2_cell_body_st *
+extend2_cell_body_get_create2(extend2_cell_body_t *inp)
+{
+  return inp->create2;
+}
+const struct create2_cell_body_st *
+extend2_cell_body_getconst_create2(const extend2_cell_body_t *inp)
+{
+  return extend2_cell_body_get_create2((extend2_cell_body_t*) inp);
+}
+int
+extend2_cell_body_set_create2(extend2_cell_body_t *inp, struct create2_cell_body_st *val)
+{
+  if (inp->create2 && inp->create2 != val)
+    create2_cell_body_free(inp->create2);
+  return extend2_cell_body_set0_create2(inp, val);
+}
+int
+extend2_cell_body_set0_create2(extend2_cell_body_t *inp, struct create2_cell_body_st *val)
+{
+  inp->create2 = val;
+  return 0;
+}
+const char *
+extend2_cell_body_check(const extend2_cell_body_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->ls); ++idx) {
+      if (NULL != (msg = link_specifier_check(TRUNNEL_DYNARRAY_GET(&obj->ls, idx))))
+        return msg;
+    }
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->ls) != obj->n_spec)
+    return "Length mismatch for ls";
+  {
+    const char *msg;
+    if (NULL != (msg = create2_cell_body_check(obj->create2)))
+      return msg;
+  }
+  return NULL;
+}
+
+ssize_t
+extend2_cell_body_encoded_len(const extend2_cell_body_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != extend2_cell_body_check(obj))
+     return -1;
+
+
+  /* Length of u8 n_spec */
+  result += 1;
+
+  /* Length of struct link_specifier ls[n_spec] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->ls); ++idx) {
+      result += link_specifier_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->ls, idx));
+    }
+  }
+
+  /* Length of struct create2_cell_body create2 */
+  result += create2_cell_body_encoded_len(obj->create2);
+  return result;
+}
+int
+extend2_cell_body_clear_errors(extend2_cell_body_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+extend2_cell_body_encode(uint8_t *output, const size_t avail, const extend2_cell_body_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = extend2_cell_body_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = extend2_cell_body_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 n_spec */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->n_spec));
+  written += 1; ptr += 1;
+
+  /* Encode struct link_specifier ls[n_spec] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->ls); ++idx) {
+      trunnel_assert(written <= avail);
+      result = link_specifier_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->ls, idx));
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+    }
+  }
+
+  /* Encode struct create2_cell_body create2 */
+  trunnel_assert(written <= avail);
+  result = create2_cell_body_encode(ptr, avail - written, obj->create2);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As extend2_cell_body_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+extend2_cell_body_parse_into(extend2_cell_body_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 n_spec */
+  CHECK_REMAINING(1, truncated);
+  obj->n_spec = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse struct link_specifier ls[n_spec] */
+  TRUNNEL_DYNARRAY_EXPAND(link_specifier_t *, &obj->ls, obj->n_spec, {});
+  {
+    link_specifier_t * elt;
+    unsigned idx;
+    for (idx = 0; idx < obj->n_spec; ++idx) {
+      result = link_specifier_parse(&elt, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      TRUNNEL_DYNARRAY_ADD(link_specifier_t *, &obj->ls, elt, {link_specifier_free(elt);});
+    }
+  }
+
+  /* Parse struct create2_cell_body create2 */
+  result = create2_cell_body_parse(&obj->create2, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+extend2_cell_body_parse(extend2_cell_body_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = extend2_cell_body_new();
+  if (NULL == *output)
+    return -1;
+  result = extend2_cell_body_parse_into(*output, input, len_in);
+  if (result < 0) {
+    extend2_cell_body_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+link_specifier_list_t *
+link_specifier_list_new(void)
+{
+  link_specifier_list_t *val = trunnel_calloc(1, sizeof(link_specifier_list_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+link_specifier_list_clear(link_specifier_list_t *obj)
+{
+  (void) obj;
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->spec); ++idx) {
+      link_specifier_free(TRUNNEL_DYNARRAY_GET(&obj->spec, idx));
+    }
+  }
+  TRUNNEL_DYNARRAY_WIPE(&obj->spec);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->spec);
+}
+
+void
+link_specifier_list_free(link_specifier_list_t *obj)
+{
+  if (obj == NULL)
+    return;
+  link_specifier_list_clear(obj);
+  trunnel_memwipe(obj, sizeof(link_specifier_list_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+link_specifier_list_get_n_spec(const link_specifier_list_t *inp)
+{
+  return inp->n_spec;
+}
+int
+link_specifier_list_set_n_spec(link_specifier_list_t *inp, uint8_t val)
+{
+  inp->n_spec = val;
+  return 0;
+}
+size_t
+link_specifier_list_getlen_spec(const link_specifier_list_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->spec);
+}
+
+struct link_specifier_st *
+link_specifier_list_get_spec(link_specifier_list_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->spec, idx);
+}
+
+ const struct link_specifier_st *
+link_specifier_list_getconst_spec(const link_specifier_list_t *inp, size_t idx)
+{
+  return link_specifier_list_get_spec((link_specifier_list_t*)inp, idx);
+}
+int
+link_specifier_list_set_spec(link_specifier_list_t *inp, size_t idx, struct link_specifier_st * elt)
+{
+  link_specifier_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->spec, idx);
+  if (oldval && oldval != elt)
+    link_specifier_free(oldval);
+  return link_specifier_list_set0_spec(inp, idx, elt);
+}
+int
+link_specifier_list_set0_spec(link_specifier_list_t *inp, size_t idx, struct link_specifier_st * elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->spec, idx, elt);
+  return 0;
+}
+int
+link_specifier_list_add_spec(link_specifier_list_t *inp, struct link_specifier_st * elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->spec.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(struct link_specifier_st *, &inp->spec, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+struct link_specifier_st * *
+link_specifier_list_getarray_spec(link_specifier_list_t *inp)
+{
+  return inp->spec.elts_;
+}
+const struct link_specifier_st *  const  *
+link_specifier_list_getconstarray_spec(const link_specifier_list_t *inp)
+{
+  return (const struct link_specifier_st *  const  *)link_specifier_list_getarray_spec((link_specifier_list_t*)inp);
+}
+int
+link_specifier_list_setlen_spec(link_specifier_list_t *inp, size_t newlen)
+{
+  struct link_specifier_st * *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->spec.allocated_,
+                 &inp->spec.n_, inp->spec.elts_, newlen,
+                 sizeof(inp->spec.elts_[0]), (trunnel_free_fn_t) link_specifier_free,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->spec.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+link_specifier_list_check(const link_specifier_list_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->spec); ++idx) {
+      if (NULL != (msg = link_specifier_check(TRUNNEL_DYNARRAY_GET(&obj->spec, idx))))
+        return msg;
+    }
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->spec) != obj->n_spec)
+    return "Length mismatch for spec";
+  return NULL;
+}
+
+ssize_t
+link_specifier_list_encoded_len(const link_specifier_list_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != link_specifier_list_check(obj))
+     return -1;
+
+
+  /* Length of u8 n_spec */
+  result += 1;
+
+  /* Length of struct link_specifier spec[n_spec] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->spec); ++idx) {
+      result += link_specifier_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->spec, idx));
+    }
+  }
+  return result;
+}
+int
+link_specifier_list_clear_errors(link_specifier_list_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+link_specifier_list_encode(uint8_t *output, const size_t avail, const link_specifier_list_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = link_specifier_list_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = link_specifier_list_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 n_spec */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->n_spec));
+  written += 1; ptr += 1;
+
+  /* Encode struct link_specifier spec[n_spec] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->spec); ++idx) {
+      trunnel_assert(written <= avail);
+      result = link_specifier_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->spec, idx));
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+    }
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As link_specifier_list_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+link_specifier_list_parse_into(link_specifier_list_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 n_spec */
+  CHECK_REMAINING(1, truncated);
+  obj->n_spec = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse struct link_specifier spec[n_spec] */
+  TRUNNEL_DYNARRAY_EXPAND(link_specifier_t *, &obj->spec, obj->n_spec, {});
+  {
+    link_specifier_t * elt;
+    unsigned idx;
+    for (idx = 0; idx < obj->n_spec; ++idx) {
+      result = link_specifier_parse(&elt, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      TRUNNEL_DYNARRAY_ADD(link_specifier_t *, &obj->spec, elt, {link_specifier_free(elt);});
+    }
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+link_specifier_list_parse(link_specifier_list_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = link_specifier_list_new();
+  if (NULL == *output)
+    return -1;
+  result = link_specifier_list_parse_into(*output, input, len_in);
+  if (result < 0) {
+    link_specifier_list_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/ed25519_cert.h b/src/trunnel/ed25519_cert.h
index 28f6feef31..e086c6fced 100644
--- a/src/trunnel/ed25519_cert.h
+++ b/src/trunnel/ed25519_cert.h
@@ -1,4 +1,4 @@
-/* ed25519_cert.h -- generated by by Trunnel v1.4.6.
+/* ed25519_cert.h -- generated by Trunnel v1.5.2.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -10,6 +10,19 @@
 
 #define CERTEXT_SIGNED_WITH_KEY 4
 #define CERTEXT_FLAG_AFFECTS_VALIDATION 1
+#define LS_IPV4 0
+#define LS_IPV6 1
+#define LS_LEGACY_ID 2
+#define LS_ED25519_ID 3
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_CREATE2_CELL_BODY)
+struct create2_cell_body_st {
+  uint16_t handshake_type;
+  uint16_t handshake_len;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) handshake_data;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct create2_cell_body_st create2_cell_body_t;
 #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_ED25519_CERT_EXTENSION)
 struct ed25519_cert_extension_st {
   uint16_t ext_length;
@@ -21,6 +34,31 @@ struct ed25519_cert_extension_st {
 };
 #endif
 typedef struct ed25519_cert_extension_st ed25519_cert_extension_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_EXTEND1_CELL_BODY)
+struct extend1_cell_body_st {
+  uint32_t ipv4addr;
+  uint16_t port;
+  uint8_t onionskin[186];
+  uint8_t identity[20];
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct extend1_cell_body_st extend1_cell_body_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_LINK_SPECIFIER)
+struct link_specifier_st {
+  uint8_t ls_type;
+  uint8_t ls_len;
+  uint32_t un_ipv4_addr;
+  uint16_t un_ipv4_port;
+  uint8_t un_ipv6_addr[16];
+  uint16_t un_ipv6_port;
+  uint8_t un_legacy_id[20];
+  uint8_t un_ed25519_id[32];
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) un_unrecognized;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct link_specifier_st link_specifier_t;
 #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_ED25519_CERT)
 struct ed25519_cert_st {
   uint8_t version;
@@ -35,6 +73,112 @@ struct ed25519_cert_st {
 };
 #endif
 typedef struct ed25519_cert_st ed25519_cert_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_EXTEND2_CELL_BODY)
+struct extend2_cell_body_st {
+  uint8_t n_spec;
+  TRUNNEL_DYNARRAY_HEAD(, struct link_specifier_st *) ls;
+  struct create2_cell_body_st *create2;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct extend2_cell_body_st extend2_cell_body_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_LINK_SPECIFIER_LIST)
+struct link_specifier_list_st {
+  uint8_t n_spec;
+  TRUNNEL_DYNARRAY_HEAD(, struct link_specifier_st *) spec;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct link_specifier_list_st link_specifier_list_t;
+/** Return a newly allocated create2_cell_body with all elements set
+ * to zero.
+ */
+create2_cell_body_t *create2_cell_body_new(void);
+/** Release all storage held by the create2_cell_body in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void create2_cell_body_free(create2_cell_body_t *victim);
+/** Try to parse a create2_cell_body from the buffer in 'input', using
+ * up to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * create2_cell_body_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t create2_cell_body_parse(create2_cell_body_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * create2_cell_body in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t create2_cell_body_encoded_len(const create2_cell_body_t *obj);
+/** Try to encode the create2_cell_body from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t create2_cell_body_encode(uint8_t *output, size_t avail, const create2_cell_body_t *input);
+/** Check whether the internal state of the create2_cell_body in 'obj'
+ * is consistent. Return NULL if it is, and a short message if it is
+ * not.
+ */
+const char *create2_cell_body_check(const create2_cell_body_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int create2_cell_body_clear_errors(create2_cell_body_t *obj);
+/** Return the value of the handshake_type field of the
+ * create2_cell_body_t in 'inp'
+ */
+uint16_t create2_cell_body_get_handshake_type(const create2_cell_body_t *inp);
+/** Set the value of the handshake_type field of the
+ * create2_cell_body_t in 'inp' to 'val'. Return 0 on success; return
+ * -1 and set the error code on 'inp' on failure.
+ */
+int create2_cell_body_set_handshake_type(create2_cell_body_t *inp, uint16_t val);
+/** Return the value of the handshake_len field of the
+ * create2_cell_body_t in 'inp'
+ */
+uint16_t create2_cell_body_get_handshake_len(const create2_cell_body_t *inp);
+/** Set the value of the handshake_len field of the
+ * create2_cell_body_t in 'inp' to 'val'. Return 0 on success; return
+ * -1 and set the error code on 'inp' on failure.
+ */
+int create2_cell_body_set_handshake_len(create2_cell_body_t *inp, uint16_t val);
+/** Return the length of the dynamic array holding the handshake_data
+ * field of the create2_cell_body_t in 'inp'.
+ */
+size_t create2_cell_body_getlen_handshake_data(const create2_cell_body_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * handshake_data of the create2_cell_body_t in 'inp'.
+ */
+uint8_t create2_cell_body_get_handshake_data(create2_cell_body_t *inp, size_t idx);
+/** As create2_cell_body_get_handshake_data, but take and return a
+ * const pointer
+ */
+uint8_t create2_cell_body_getconst_handshake_data(const create2_cell_body_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * handshake_data of the create2_cell_body_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int create2_cell_body_set_handshake_data(create2_cell_body_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field
+ * handshake_data of the create2_cell_body_t in 'inp'.
+ */
+int create2_cell_body_add_handshake_data(create2_cell_body_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field handshake_data
+ * of 'inp'.
+ */
+uint8_t * create2_cell_body_getarray_handshake_data(create2_cell_body_t *inp);
+/** As create2_cell_body_get_handshake_data, but take and return a
+ * const pointer
+ */
+const uint8_t  * create2_cell_body_getconstarray_handshake_data(const create2_cell_body_t *inp);
+/** Change the length of the variable-length array field
+ * handshake_data of 'inp' to 'newlen'.Fill extra elements with 0.
+ * Return 0 on success; return -1 and set the error code on 'inp' on
+ * failure.
+ */
+int create2_cell_body_setlen_handshake_data(create2_cell_body_t *inp, size_t newlen);
 /** Return a newly allocated ed25519_cert_extension with all elements
  * set to zero.
  */
@@ -74,7 +218,7 @@ int ed25519_cert_extension_clear_errors(ed25519_cert_extension_t *obj);
 /** Return the value of the ext_length field of the
  * ed25519_cert_extension_t in 'inp'
  */
-uint16_t ed25519_cert_extension_get_ext_length(ed25519_cert_extension_t *inp);
+uint16_t ed25519_cert_extension_get_ext_length(const ed25519_cert_extension_t *inp);
 /** Set the value of the ext_length field of the
  * ed25519_cert_extension_t in 'inp' to 'val'. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
@@ -83,7 +227,7 @@ int ed25519_cert_extension_set_ext_length(ed25519_cert_extension_t *inp, uint16_
 /** Return the value of the ext_type field of the
  * ed25519_cert_extension_t in 'inp'
  */
-uint8_t ed25519_cert_extension_get_ext_type(ed25519_cert_extension_t *inp);
+uint8_t ed25519_cert_extension_get_ext_type(const ed25519_cert_extension_t *inp);
 /** Set the value of the ext_type field of the
  * ed25519_cert_extension_t in 'inp' to 'val'. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
@@ -92,7 +236,7 @@ int ed25519_cert_extension_set_ext_type(ed25519_cert_extension_t *inp, uint8_t v
 /** Return the value of the ext_flags field of the
  * ed25519_cert_extension_t in 'inp'
  */
-uint8_t ed25519_cert_extension_get_ext_flags(ed25519_cert_extension_t *inp);
+uint8_t ed25519_cert_extension_get_ext_flags(const ed25519_cert_extension_t *inp);
 /** Set the value of the ext_flags field of the
  * ed25519_cert_extension_t in 'inp' to 'val'. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
@@ -105,7 +249,11 @@ size_t ed25519_cert_extension_getlen_un_signing_key(const ed25519_cert_extension
 /** Return the element at position 'idx' of the fixed array field
  * un_signing_key of the ed25519_cert_extension_t in 'inp'.
  */
-uint8_t ed25519_cert_extension_get_un_signing_key(const ed25519_cert_extension_t *inp, size_t idx);
+uint8_t ed25519_cert_extension_get_un_signing_key(ed25519_cert_extension_t *inp, size_t idx);
+/** As ed25519_cert_extension_get_un_signing_key, but take and return
+ * a const pointer
+ */
+uint8_t ed25519_cert_extension_getconst_un_signing_key(const ed25519_cert_extension_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * un_signing_key of the ed25519_cert_extension_t in 'inp', so that it
  * will hold the value 'elt'.
@@ -115,6 +263,10 @@ int ed25519_cert_extension_set_un_signing_key(ed25519_cert_extension_t *inp, siz
  * 'inp'.
  */
 uint8_t * ed25519_cert_extension_getarray_un_signing_key(ed25519_cert_extension_t *inp);
+/** As ed25519_cert_extension_get_un_signing_key, but take and return
+ * a const pointer
+ */
+const uint8_t  * ed25519_cert_extension_getconstarray_un_signing_key(const ed25519_cert_extension_t *inp);
 /** Return the length of the dynamic array holding the un_unparsed
  * field of the ed25519_cert_extension_t in 'inp'.
  */
@@ -123,6 +275,10 @@ size_t ed25519_cert_extension_getlen_un_unparsed(const ed25519_cert_extension_t
  * un_unparsed of the ed25519_cert_extension_t in 'inp'.
  */
 uint8_t ed25519_cert_extension_get_un_unparsed(ed25519_cert_extension_t *inp, size_t idx);
+/** As ed25519_cert_extension_get_un_unparsed, but take and return a
+ * const pointer
+ */
+uint8_t ed25519_cert_extension_getconst_un_unparsed(const ed25519_cert_extension_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * un_unparsed of the ed25519_cert_extension_t in 'inp', so that it
  * will hold the value 'elt'.
@@ -136,11 +292,308 @@ int ed25519_cert_extension_add_un_unparsed(ed25519_cert_extension_t *inp, uint8_
  * 'inp'.
  */
 uint8_t * ed25519_cert_extension_getarray_un_unparsed(ed25519_cert_extension_t *inp);
+/** As ed25519_cert_extension_get_un_unparsed, but take and return a
+ * const pointer
+ */
+const uint8_t  * ed25519_cert_extension_getconstarray_un_unparsed(const ed25519_cert_extension_t *inp);
 /** Change the length of the variable-length array field un_unparsed
  * of 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on
  * success; return -1 and set the error code on 'inp' on failure.
  */
 int ed25519_cert_extension_setlen_un_unparsed(ed25519_cert_extension_t *inp, size_t newlen);
+/** Return a newly allocated extend1_cell_body with all elements set
+ * to zero.
+ */
+extend1_cell_body_t *extend1_cell_body_new(void);
+/** Release all storage held by the extend1_cell_body in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void extend1_cell_body_free(extend1_cell_body_t *victim);
+/** Try to parse a extend1_cell_body from the buffer in 'input', using
+ * up to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * extend1_cell_body_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t extend1_cell_body_parse(extend1_cell_body_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * extend1_cell_body in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t extend1_cell_body_encoded_len(const extend1_cell_body_t *obj);
+/** Try to encode the extend1_cell_body from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t extend1_cell_body_encode(uint8_t *output, size_t avail, const extend1_cell_body_t *input);
+/** Check whether the internal state of the extend1_cell_body in 'obj'
+ * is consistent. Return NULL if it is, and a short message if it is
+ * not.
+ */
+const char *extend1_cell_body_check(const extend1_cell_body_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int extend1_cell_body_clear_errors(extend1_cell_body_t *obj);
+/** Return the value of the ipv4addr field of the extend1_cell_body_t
+ * in 'inp'
+ */
+uint32_t extend1_cell_body_get_ipv4addr(const extend1_cell_body_t *inp);
+/** Set the value of the ipv4addr field of the extend1_cell_body_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int extend1_cell_body_set_ipv4addr(extend1_cell_body_t *inp, uint32_t val);
+/** Return the value of the port field of the extend1_cell_body_t in
+ * 'inp'
+ */
+uint16_t extend1_cell_body_get_port(const extend1_cell_body_t *inp);
+/** Set the value of the port field of the extend1_cell_body_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int extend1_cell_body_set_port(extend1_cell_body_t *inp, uint16_t val);
+/** Return the (constant) length of the array holding the onionskin
+ * field of the extend1_cell_body_t in 'inp'.
+ */
+size_t extend1_cell_body_getlen_onionskin(const extend1_cell_body_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * onionskin of the extend1_cell_body_t in 'inp'.
+ */
+uint8_t extend1_cell_body_get_onionskin(extend1_cell_body_t *inp, size_t idx);
+/** As extend1_cell_body_get_onionskin, but take and return a const
+ * pointer
+ */
+uint8_t extend1_cell_body_getconst_onionskin(const extend1_cell_body_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * onionskin of the extend1_cell_body_t in 'inp', so that it will hold
+ * the value 'elt'.
+ */
+int extend1_cell_body_set_onionskin(extend1_cell_body_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the 186-element array field onionskin of
+ * 'inp'.
+ */
+uint8_t * extend1_cell_body_getarray_onionskin(extend1_cell_body_t *inp);
+/** As extend1_cell_body_get_onionskin, but take and return a const
+ * pointer
+ */
+const uint8_t  * extend1_cell_body_getconstarray_onionskin(const extend1_cell_body_t *inp);
+/** Return the (constant) length of the array holding the identity
+ * field of the extend1_cell_body_t in 'inp'.
+ */
+size_t extend1_cell_body_getlen_identity(const extend1_cell_body_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * identity of the extend1_cell_body_t in 'inp'.
+ */
+uint8_t extend1_cell_body_get_identity(extend1_cell_body_t *inp, size_t idx);
+/** As extend1_cell_body_get_identity, but take and return a const
+ * pointer
+ */
+uint8_t extend1_cell_body_getconst_identity(const extend1_cell_body_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * identity of the extend1_cell_body_t in 'inp', so that it will hold
+ * the value 'elt'.
+ */
+int extend1_cell_body_set_identity(extend1_cell_body_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the 20-element array field identity of 'inp'.
+ */
+uint8_t * extend1_cell_body_getarray_identity(extend1_cell_body_t *inp);
+/** As extend1_cell_body_get_identity, but take and return a const
+ * pointer
+ */
+const uint8_t  * extend1_cell_body_getconstarray_identity(const extend1_cell_body_t *inp);
+/** Return a newly allocated link_specifier with all elements set to
+ * zero.
+ */
+link_specifier_t *link_specifier_new(void);
+/** Release all storage held by the link_specifier in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void link_specifier_free(link_specifier_t *victim);
+/** Try to parse a link_specifier from the buffer in 'input', using up
+ * to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * link_specifier_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t link_specifier_parse(link_specifier_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * link_specifier in 'obj'. On failure, return a negative value. Note
+ * that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t link_specifier_encoded_len(const link_specifier_t *obj);
+/** Try to encode the link_specifier from 'input' into the buffer at
+ * 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t link_specifier_encode(uint8_t *output, size_t avail, const link_specifier_t *input);
+/** Check whether the internal state of the link_specifier in 'obj' is
+ * consistent. Return NULL if it is, and a short message if it is not.
+ */
+const char *link_specifier_check(const link_specifier_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int link_specifier_clear_errors(link_specifier_t *obj);
+/** Return the value of the ls_type field of the link_specifier_t in
+ * 'inp'
+ */
+uint8_t link_specifier_get_ls_type(const link_specifier_t *inp);
+/** Set the value of the ls_type field of the link_specifier_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int link_specifier_set_ls_type(link_specifier_t *inp, uint8_t val);
+/** Return the value of the ls_len field of the link_specifier_t in
+ * 'inp'
+ */
+uint8_t link_specifier_get_ls_len(const link_specifier_t *inp);
+/** Set the value of the ls_len field of the link_specifier_t in 'inp'
+ * to 'val'. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int link_specifier_set_ls_len(link_specifier_t *inp, uint8_t val);
+/** Return the value of the un_ipv4_addr field of the link_specifier_t
+ * in 'inp'
+ */
+uint32_t link_specifier_get_un_ipv4_addr(const link_specifier_t *inp);
+/** Set the value of the un_ipv4_addr field of the link_specifier_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int link_specifier_set_un_ipv4_addr(link_specifier_t *inp, uint32_t val);
+/** Return the value of the un_ipv4_port field of the link_specifier_t
+ * in 'inp'
+ */
+uint16_t link_specifier_get_un_ipv4_port(const link_specifier_t *inp);
+/** Set the value of the un_ipv4_port field of the link_specifier_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int link_specifier_set_un_ipv4_port(link_specifier_t *inp, uint16_t val);
+/** Return the (constant) length of the array holding the un_ipv6_addr
+ * field of the link_specifier_t in 'inp'.
+ */
+size_t link_specifier_getlen_un_ipv6_addr(const link_specifier_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * un_ipv6_addr of the link_specifier_t in 'inp'.
+ */
+uint8_t link_specifier_get_un_ipv6_addr(link_specifier_t *inp, size_t idx);
+/** As link_specifier_get_un_ipv6_addr, but take and return a const
+ * pointer
+ */
+uint8_t link_specifier_getconst_un_ipv6_addr(const link_specifier_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * un_ipv6_addr of the link_specifier_t in 'inp', so that it will hold
+ * the value 'elt'.
+ */
+int link_specifier_set_un_ipv6_addr(link_specifier_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the 16-element array field un_ipv6_addr of
+ * 'inp'.
+ */
+uint8_t * link_specifier_getarray_un_ipv6_addr(link_specifier_t *inp);
+/** As link_specifier_get_un_ipv6_addr, but take and return a const
+ * pointer
+ */
+const uint8_t  * link_specifier_getconstarray_un_ipv6_addr(const link_specifier_t *inp);
+/** Return the value of the un_ipv6_port field of the link_specifier_t
+ * in 'inp'
+ */
+uint16_t link_specifier_get_un_ipv6_port(const link_specifier_t *inp);
+/** Set the value of the un_ipv6_port field of the link_specifier_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int link_specifier_set_un_ipv6_port(link_specifier_t *inp, uint16_t val);
+/** Return the (constant) length of the array holding the un_legacy_id
+ * field of the link_specifier_t in 'inp'.
+ */
+size_t link_specifier_getlen_un_legacy_id(const link_specifier_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * un_legacy_id of the link_specifier_t in 'inp'.
+ */
+uint8_t link_specifier_get_un_legacy_id(link_specifier_t *inp, size_t idx);
+/** As link_specifier_get_un_legacy_id, but take and return a const
+ * pointer
+ */
+uint8_t link_specifier_getconst_un_legacy_id(const link_specifier_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * un_legacy_id of the link_specifier_t in 'inp', so that it will hold
+ * the value 'elt'.
+ */
+int link_specifier_set_un_legacy_id(link_specifier_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the 20-element array field un_legacy_id of
+ * 'inp'.
+ */
+uint8_t * link_specifier_getarray_un_legacy_id(link_specifier_t *inp);
+/** As link_specifier_get_un_legacy_id, but take and return a const
+ * pointer
+ */
+const uint8_t  * link_specifier_getconstarray_un_legacy_id(const link_specifier_t *inp);
+/** Return the (constant) length of the array holding the
+ * un_ed25519_id field of the link_specifier_t in 'inp'.
+ */
+size_t link_specifier_getlen_un_ed25519_id(const link_specifier_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * un_ed25519_id of the link_specifier_t in 'inp'.
+ */
+uint8_t link_specifier_get_un_ed25519_id(link_specifier_t *inp, size_t idx);
+/** As link_specifier_get_un_ed25519_id, but take and return a const
+ * pointer
+ */
+uint8_t link_specifier_getconst_un_ed25519_id(const link_specifier_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * un_ed25519_id of the link_specifier_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int link_specifier_set_un_ed25519_id(link_specifier_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the 32-element array field un_ed25519_id of
+ * 'inp'.
+ */
+uint8_t * link_specifier_getarray_un_ed25519_id(link_specifier_t *inp);
+/** As link_specifier_get_un_ed25519_id, but take and return a const
+ * pointer
+ */
+const uint8_t  * link_specifier_getconstarray_un_ed25519_id(const link_specifier_t *inp);
+/** Return the length of the dynamic array holding the un_unrecognized
+ * field of the link_specifier_t in 'inp'.
+ */
+size_t link_specifier_getlen_un_unrecognized(const link_specifier_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * un_unrecognized of the link_specifier_t in 'inp'.
+ */
+uint8_t link_specifier_get_un_unrecognized(link_specifier_t *inp, size_t idx);
+/** As link_specifier_get_un_unrecognized, but take and return a const
+ * pointer
+ */
+uint8_t link_specifier_getconst_un_unrecognized(const link_specifier_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * un_unrecognized of the link_specifier_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int link_specifier_set_un_unrecognized(link_specifier_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field
+ * un_unrecognized of the link_specifier_t in 'inp'.
+ */
+int link_specifier_add_un_unrecognized(link_specifier_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field
+ * un_unrecognized of 'inp'.
+ */
+uint8_t * link_specifier_getarray_un_unrecognized(link_specifier_t *inp);
+/** As link_specifier_get_un_unrecognized, but take and return a const
+ * pointer
+ */
+const uint8_t  * link_specifier_getconstarray_un_unrecognized(const link_specifier_t *inp);
+/** Change the length of the variable-length array field
+ * un_unrecognized of 'inp' to 'newlen'.Fill extra elements with 0.
+ * Return 0 on success; return -1 and set the error code on 'inp' on
+ * failure.
+ */
+int link_specifier_setlen_un_unrecognized(link_specifier_t *inp, size_t newlen);
 /** Return a newly allocated ed25519_cert with all elements set to
  * zero.
  */
@@ -179,7 +632,7 @@ int ed25519_cert_clear_errors(ed25519_cert_t *obj);
 /** Return the value of the version field of the ed25519_cert_t in
  * 'inp'
  */
-uint8_t ed25519_cert_get_version(ed25519_cert_t *inp);
+uint8_t ed25519_cert_get_version(const ed25519_cert_t *inp);
 /** Set the value of the version field of the ed25519_cert_t in 'inp'
  * to 'val'. Return 0 on success; return -1 and set the error code on
  * 'inp' on failure.
@@ -188,7 +641,7 @@ int ed25519_cert_set_version(ed25519_cert_t *inp, uint8_t val);
 /** Return the value of the cert_type field of the ed25519_cert_t in
  * 'inp'
  */
-uint8_t ed25519_cert_get_cert_type(ed25519_cert_t *inp);
+uint8_t ed25519_cert_get_cert_type(const ed25519_cert_t *inp);
 /** Set the value of the cert_type field of the ed25519_cert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -197,7 +650,7 @@ int ed25519_cert_set_cert_type(ed25519_cert_t *inp, uint8_t val);
 /** Return the value of the exp_field field of the ed25519_cert_t in
  * 'inp'
  */
-uint32_t ed25519_cert_get_exp_field(ed25519_cert_t *inp);
+uint32_t ed25519_cert_get_exp_field(const ed25519_cert_t *inp);
 /** Set the value of the exp_field field of the ed25519_cert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -206,7 +659,7 @@ int ed25519_cert_set_exp_field(ed25519_cert_t *inp, uint32_t val);
 /** Return the value of the cert_key_type field of the ed25519_cert_t
  * in 'inp'
  */
-uint8_t ed25519_cert_get_cert_key_type(ed25519_cert_t *inp);
+uint8_t ed25519_cert_get_cert_key_type(const ed25519_cert_t *inp);
 /** Set the value of the cert_key_type field of the ed25519_cert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -219,7 +672,11 @@ size_t ed25519_cert_getlen_certified_key(const ed25519_cert_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * certified_key of the ed25519_cert_t in 'inp'.
  */
-uint8_t ed25519_cert_get_certified_key(const ed25519_cert_t *inp, size_t idx);
+uint8_t ed25519_cert_get_certified_key(ed25519_cert_t *inp, size_t idx);
+/** As ed25519_cert_get_certified_key, but take and return a const
+ * pointer
+ */
+uint8_t ed25519_cert_getconst_certified_key(const ed25519_cert_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * certified_key of the ed25519_cert_t in 'inp', so that it will hold
  * the value 'elt'.
@@ -229,10 +686,14 @@ int ed25519_cert_set_certified_key(ed25519_cert_t *inp, size_t idx, uint8_t elt)
  * 'inp'.
  */
 uint8_t * ed25519_cert_getarray_certified_key(ed25519_cert_t *inp);
+/** As ed25519_cert_get_certified_key, but take and return a const
+ * pointer
+ */
+const uint8_t  * ed25519_cert_getconstarray_certified_key(const ed25519_cert_t *inp);
 /** Return the value of the n_extensions field of the ed25519_cert_t
  * in 'inp'
  */
-uint8_t ed25519_cert_get_n_extensions(ed25519_cert_t *inp);
+uint8_t ed25519_cert_get_n_extensions(const ed25519_cert_t *inp);
 /** Set the value of the n_extensions field of the ed25519_cert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -246,6 +707,9 @@ size_t ed25519_cert_getlen_ext(const ed25519_cert_t *inp);
  * ext of the ed25519_cert_t in 'inp'.
  */
 struct ed25519_cert_extension_st * ed25519_cert_get_ext(ed25519_cert_t *inp, size_t idx);
+/** As ed25519_cert_get_ext, but take and return a const pointer
+ */
+ const struct ed25519_cert_extension_st * ed25519_cert_getconst_ext(const ed25519_cert_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * ext of the ed25519_cert_t in 'inp', so that it will hold the value
  * 'elt'. Free the previous value, if any.
@@ -261,6 +725,9 @@ int ed25519_cert_add_ext(ed25519_cert_t *inp, struct ed25519_cert_extension_st *
 /** Return a pointer to the variable-length array field ext of 'inp'.
  */
 struct ed25519_cert_extension_st * * ed25519_cert_getarray_ext(ed25519_cert_t *inp);
+/** As ed25519_cert_get_ext, but take and return a const pointer
+ */
+const struct ed25519_cert_extension_st *  const  * ed25519_cert_getconstarray_ext(const ed25519_cert_t *inp);
 /** Change the length of the variable-length array field ext of 'inp'
  * to 'newlen'.Fill extra elements with NULL; free removed elements.
  * Return 0 on success; return -1 and set the error code on 'inp' on
@@ -274,7 +741,10 @@ size_t ed25519_cert_getlen_signature(const ed25519_cert_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * signature of the ed25519_cert_t in 'inp'.
  */
-uint8_t ed25519_cert_get_signature(const ed25519_cert_t *inp, size_t idx);
+uint8_t ed25519_cert_get_signature(ed25519_cert_t *inp, size_t idx);
+/** As ed25519_cert_get_signature, but take and return a const pointer
+ */
+uint8_t ed25519_cert_getconst_signature(const ed25519_cert_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * signature of the ed25519_cert_t in 'inp', so that it will hold the
  * value 'elt'.
@@ -283,6 +753,190 @@ int ed25519_cert_set_signature(ed25519_cert_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 64-element array field signature of 'inp'.
  */
 uint8_t * ed25519_cert_getarray_signature(ed25519_cert_t *inp);
+/** As ed25519_cert_get_signature, but take and return a const pointer
+ */
+const uint8_t  * ed25519_cert_getconstarray_signature(const ed25519_cert_t *inp);
+/** Return a newly allocated extend2_cell_body with all elements set
+ * to zero.
+ */
+extend2_cell_body_t *extend2_cell_body_new(void);
+/** Release all storage held by the extend2_cell_body in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void extend2_cell_body_free(extend2_cell_body_t *victim);
+/** Try to parse a extend2_cell_body from the buffer in 'input', using
+ * up to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * extend2_cell_body_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t extend2_cell_body_parse(extend2_cell_body_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * extend2_cell_body in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t extend2_cell_body_encoded_len(const extend2_cell_body_t *obj);
+/** Try to encode the extend2_cell_body from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t extend2_cell_body_encode(uint8_t *output, size_t avail, const extend2_cell_body_t *input);
+/** Check whether the internal state of the extend2_cell_body in 'obj'
+ * is consistent. Return NULL if it is, and a short message if it is
+ * not.
+ */
+const char *extend2_cell_body_check(const extend2_cell_body_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int extend2_cell_body_clear_errors(extend2_cell_body_t *obj);
+/** Return the value of the n_spec field of the extend2_cell_body_t in
+ * 'inp'
+ */
+uint8_t extend2_cell_body_get_n_spec(const extend2_cell_body_t *inp);
+/** Set the value of the n_spec field of the extend2_cell_body_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int extend2_cell_body_set_n_spec(extend2_cell_body_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the ls field of the
+ * extend2_cell_body_t in 'inp'.
+ */
+size_t extend2_cell_body_getlen_ls(const extend2_cell_body_t *inp);
+/** Return the element at position 'idx' of the dynamic array field ls
+ * of the extend2_cell_body_t in 'inp'.
+ */
+struct link_specifier_st * extend2_cell_body_get_ls(extend2_cell_body_t *inp, size_t idx);
+/** As extend2_cell_body_get_ls, but take and return a const pointer
+ */
+ const struct link_specifier_st * extend2_cell_body_getconst_ls(const extend2_cell_body_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field ls
+ * of the extend2_cell_body_t in 'inp', so that it will hold the value
+ * 'elt'. Free the previous value, if any.
+ */
+int extend2_cell_body_set_ls(extend2_cell_body_t *inp, size_t idx, struct link_specifier_st * elt);
+/** As extend2_cell_body_set_ls, but does not free the previous value.
+ */
+int extend2_cell_body_set0_ls(extend2_cell_body_t *inp, size_t idx, struct link_specifier_st * elt);
+/** Append a new element 'elt' to the dynamic array field ls of the
+ * extend2_cell_body_t in 'inp'.
+ */
+int extend2_cell_body_add_ls(extend2_cell_body_t *inp, struct link_specifier_st * elt);
+/** Return a pointer to the variable-length array field ls of 'inp'.
+ */
+struct link_specifier_st * * extend2_cell_body_getarray_ls(extend2_cell_body_t *inp);
+/** As extend2_cell_body_get_ls, but take and return a const pointer
+ */
+const struct link_specifier_st *  const  * extend2_cell_body_getconstarray_ls(const extend2_cell_body_t *inp);
+/** Change the length of the variable-length array field ls of 'inp'
+ * to 'newlen'.Fill extra elements with NULL; free removed elements.
+ * Return 0 on success; return -1 and set the error code on 'inp' on
+ * failure.
+ */
+int extend2_cell_body_setlen_ls(extend2_cell_body_t *inp, size_t newlen);
+/** Return the value of the create2 field of the extend2_cell_body_t
+ * in 'inp'
+ */
+struct create2_cell_body_st * extend2_cell_body_get_create2(extend2_cell_body_t *inp);
+/** As extend2_cell_body_get_create2, but take and return a const
+ * pointer
+ */
+const struct create2_cell_body_st * extend2_cell_body_getconst_create2(const extend2_cell_body_t *inp);
+/** Set the value of the create2 field of the extend2_cell_body_t in
+ * 'inp' to 'val'. Free the old value if any. Steals the referenceto
+ * 'val'.Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int extend2_cell_body_set_create2(extend2_cell_body_t *inp, struct create2_cell_body_st *val);
+/** As extend2_cell_body_set_create2, but does not free the previous
+ * value.
+ */
+int extend2_cell_body_set0_create2(extend2_cell_body_t *inp, struct create2_cell_body_st *val);
+/** Return a newly allocated link_specifier_list with all elements set
+ * to zero.
+ */
+link_specifier_list_t *link_specifier_list_new(void);
+/** Release all storage held by the link_specifier_list in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void link_specifier_list_free(link_specifier_list_t *victim);
+/** Try to parse a link_specifier_list from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated link_specifier_list_t. On failure, return -2 if the input
+ * appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t link_specifier_list_parse(link_specifier_list_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * link_specifier_list in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t link_specifier_list_encoded_len(const link_specifier_list_t *obj);
+/** Try to encode the link_specifier_list from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t link_specifier_list_encode(uint8_t *output, size_t avail, const link_specifier_list_t *input);
+/** Check whether the internal state of the link_specifier_list in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *link_specifier_list_check(const link_specifier_list_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int link_specifier_list_clear_errors(link_specifier_list_t *obj);
+/** Return the value of the n_spec field of the link_specifier_list_t
+ * in 'inp'
+ */
+uint8_t link_specifier_list_get_n_spec(const link_specifier_list_t *inp);
+/** Set the value of the n_spec field of the link_specifier_list_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int link_specifier_list_set_n_spec(link_specifier_list_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the spec field of
+ * the link_specifier_list_t in 'inp'.
+ */
+size_t link_specifier_list_getlen_spec(const link_specifier_list_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * spec of the link_specifier_list_t in 'inp'.
+ */
+struct link_specifier_st * link_specifier_list_get_spec(link_specifier_list_t *inp, size_t idx);
+/** As link_specifier_list_get_spec, but take and return a const
+ * pointer
+ */
+ const struct link_specifier_st * link_specifier_list_getconst_spec(const link_specifier_list_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * spec of the link_specifier_list_t in 'inp', so that it will hold
+ * the value 'elt'. Free the previous value, if any.
+ */
+int link_specifier_list_set_spec(link_specifier_list_t *inp, size_t idx, struct link_specifier_st * elt);
+/** As link_specifier_list_set_spec, but does not free the previous
+ * value.
+ */
+int link_specifier_list_set0_spec(link_specifier_list_t *inp, size_t idx, struct link_specifier_st * elt);
+/** Append a new element 'elt' to the dynamic array field spec of the
+ * link_specifier_list_t in 'inp'.
+ */
+int link_specifier_list_add_spec(link_specifier_list_t *inp, struct link_specifier_st * elt);
+/** Return a pointer to the variable-length array field spec of 'inp'.
+ */
+struct link_specifier_st * * link_specifier_list_getarray_spec(link_specifier_list_t *inp);
+/** As link_specifier_list_get_spec, but take and return a const
+ * pointer
+ */
+const struct link_specifier_st *  const  * link_specifier_list_getconstarray_spec(const link_specifier_list_t *inp);
+/** Change the length of the variable-length array field spec of 'inp'
+ * to 'newlen'.Fill extra elements with NULL; free removed elements.
+ * Return 0 on success; return -1 and set the error code on 'inp' on
+ * failure.
+ */
+int link_specifier_list_setlen_spec(link_specifier_list_t *inp, size_t newlen);
 
 
 #endif
diff --git a/src/trunnel/ed25519_cert.trunnel b/src/trunnel/ed25519_cert.trunnel
index c46f1b6c6b..8d6483d558 100644
--- a/src/trunnel/ed25519_cert.trunnel
+++ b/src/trunnel/ed25519_cert.trunnel
@@ -23,44 +23,17 @@ struct ed25519_cert_extension {
   };
 }
 
-/*
-struct cert_revocation {
-  u8 prefix[8];
-  u8 version IN [1];
-  u8 keytype;
-  u8 identity_key[32];
-  u8 revoked_key[32];
-  u64 published;
-  u8 n_extensions;
-  struct cert_extension ext[n_extensions];
-  u8 signature[64];
-}
-
-struct crosscert_ed_rsa {
-  u8 ed_key[32];
-  u32 expiration_date;
-  u8 signature[128];
-}
-
-struct auth02_cell {
-  u8 type[8];
-  u8 cid[32];
-  u8 sid[32];
-  u8 cid_ed[32];
-  u8 sid_ed[32];
-  u8 slog[32];
-  u8 clog[32];
-  u8 scert[32];
-  u8 tlssecrets[32];
-  u8 rand[24];
-  u8 sig[64];
-}
-
 const LS_IPV4 = 0x00;
 const LS_IPV6 = 0x01;
 const LS_LEGACY_ID = 0x02;
 const LS_ED25519_ID = 0x03;
 
+// XXX hs_link_specifier_dup() violates the opaqueness of link_specifier_t by
+//  taking its sizeof(). If we ever want to turn on TRUNNEL_OPAQUE, or
+//  if we ever make link_specifier contain other types, we will
+//  need to refactor that function to do the copy by encoding and decoding the
+//  object.
+
 // amended from tor.trunnel
 struct link_specifier {
   u8 ls_type;
@@ -73,4 +46,27 @@ struct link_specifier {
     default: u8 unrecognized[];
   };
 }
-*/
\ No newline at end of file
+
+struct link_specifier_list {
+  u8 n_spec;
+  struct link_specifier spec[n_spec];
+}
+
+struct extend1_cell_body {
+  u32 ipv4addr;
+  u16 port;
+  u8 onionskin[186];
+  u8 identity[20];
+}
+
+struct create2_cell_body {
+  u16 handshake_type;
+  u16 handshake_len;
+  u8 handshake_data[handshake_len];
+}
+
+struct extend2_cell_body {
+  u8 n_spec;
+  struct link_specifier ls[n_spec];
+  struct create2_cell_body create2;
+}
diff --git a/src/trunnel/hs/cell_common.c b/src/trunnel/hs/cell_common.c
new file mode 100644
index 0000000000..af223560c1
--- /dev/null
+++ b/src/trunnel/hs/cell_common.c
@@ -0,0 +1,595 @@
+/* cell_common.c -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "cell_common.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int cellcommon_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || cellcommon_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+trn_cell_extension_fields_t *
+trn_cell_extension_fields_new(void)
+{
+  trn_cell_extension_fields_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_fields_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_extension_fields_clear(trn_cell_extension_fields_t *obj)
+{
+  (void) obj;
+  TRUNNEL_DYNARRAY_WIPE(&obj->field);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->field);
+}
+
+void
+trn_cell_extension_fields_free(trn_cell_extension_fields_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_extension_fields_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_extension_fields_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+trn_cell_extension_fields_get_field_type(const trn_cell_extension_fields_t *inp)
+{
+  return inp->field_type;
+}
+int
+trn_cell_extension_fields_set_field_type(trn_cell_extension_fields_t *inp, uint8_t val)
+{
+  inp->field_type = val;
+  return 0;
+}
+uint8_t
+trn_cell_extension_fields_get_field_len(const trn_cell_extension_fields_t *inp)
+{
+  return inp->field_len;
+}
+int
+trn_cell_extension_fields_set_field_len(trn_cell_extension_fields_t *inp, uint8_t val)
+{
+  inp->field_len = val;
+  return 0;
+}
+size_t
+trn_cell_extension_fields_getlen_field(const trn_cell_extension_fields_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->field);
+}
+
+uint8_t
+trn_cell_extension_fields_get_field(trn_cell_extension_fields_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->field, idx);
+}
+
+uint8_t
+trn_cell_extension_fields_getconst_field(const trn_cell_extension_fields_t *inp, size_t idx)
+{
+  return trn_cell_extension_fields_get_field((trn_cell_extension_fields_t*)inp, idx);
+}
+int
+trn_cell_extension_fields_set_field(trn_cell_extension_fields_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->field, idx, elt);
+  return 0;
+}
+int
+trn_cell_extension_fields_add_field(trn_cell_extension_fields_t *inp, uint8_t elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->field.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->field, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_extension_fields_getarray_field(trn_cell_extension_fields_t *inp)
+{
+  return inp->field.elts_;
+}
+const uint8_t  *
+trn_cell_extension_fields_getconstarray_field(const trn_cell_extension_fields_t *inp)
+{
+  return (const uint8_t  *)trn_cell_extension_fields_getarray_field((trn_cell_extension_fields_t*)inp);
+}
+int
+trn_cell_extension_fields_setlen_field(trn_cell_extension_fields_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->field.allocated_,
+                 &inp->field.n_, inp->field.elts_, newlen,
+                 sizeof(inp->field.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->field.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+trn_cell_extension_fields_check(const trn_cell_extension_fields_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (TRUNNEL_DYNARRAY_LEN(&obj->field) != obj->field_len)
+    return "Length mismatch for field";
+  return NULL;
+}
+
+ssize_t
+trn_cell_extension_fields_encoded_len(const trn_cell_extension_fields_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_extension_fields_check(obj))
+     return -1;
+
+
+  /* Length of u8 field_type */
+  result += 1;
+
+  /* Length of u8 field_len */
+  result += 1;
+
+  /* Length of u8 field[field_len] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->field);
+  return result;
+}
+int
+trn_cell_extension_fields_clear_errors(trn_cell_extension_fields_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_extension_fields_encode(uint8_t *output, const size_t avail, const trn_cell_extension_fields_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_extension_fields_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_extension_fields_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 field_type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->field_type));
+  written += 1; ptr += 1;
+
+  /* Encode u8 field_len */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->field_len));
+  written += 1; ptr += 1;
+
+  /* Encode u8 field[field_len] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->field);
+    trunnel_assert(obj->field_len == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->field.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_extension_fields_parse(), but do not allocate the
+ * output object.
+ */
+static ssize_t
+trn_cell_extension_fields_parse_into(trn_cell_extension_fields_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 field_type */
+  CHECK_REMAINING(1, truncated);
+  obj->field_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u8 field_len */
+  CHECK_REMAINING(1, truncated);
+  obj->field_len = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u8 field[field_len] */
+  CHECK_REMAINING(obj->field_len, truncated);
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->field, obj->field_len, {});
+  obj->field.n_ = obj->field_len;
+  if (obj->field_len)
+    memcpy(obj->field.elts_, ptr, obj->field_len);
+  ptr += obj->field_len; remaining -= obj->field_len;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+trn_cell_extension_fields_parse(trn_cell_extension_fields_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_extension_fields_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_extension_fields_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_extension_fields_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+trn_cell_extension_t *
+trn_cell_extension_new(void)
+{
+  trn_cell_extension_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_extension_clear(trn_cell_extension_t *obj)
+{
+  (void) obj;
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
+      trn_cell_extension_fields_free(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
+    }
+  }
+  TRUNNEL_DYNARRAY_WIPE(&obj->fields);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->fields);
+}
+
+void
+trn_cell_extension_free(trn_cell_extension_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_extension_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_extension_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+trn_cell_extension_get_num(const trn_cell_extension_t *inp)
+{
+  return inp->num;
+}
+int
+trn_cell_extension_set_num(trn_cell_extension_t *inp, uint8_t val)
+{
+  inp->num = val;
+  return 0;
+}
+size_t
+trn_cell_extension_getlen_fields(const trn_cell_extension_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->fields);
+}
+
+struct trn_cell_extension_fields_st *
+trn_cell_extension_get_fields(trn_cell_extension_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->fields, idx);
+}
+
+ const struct trn_cell_extension_fields_st *
+trn_cell_extension_getconst_fields(const trn_cell_extension_t *inp, size_t idx)
+{
+  return trn_cell_extension_get_fields((trn_cell_extension_t*)inp, idx);
+}
+int
+trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt)
+{
+  trn_cell_extension_fields_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->fields, idx);
+  if (oldval && oldval != elt)
+    trn_cell_extension_fields_free(oldval);
+  return trn_cell_extension_set0_fields(inp, idx, elt);
+}
+int
+trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->fields, idx, elt);
+  return 0;
+}
+int
+trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_fields_st * elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->fields.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(struct trn_cell_extension_fields_st *, &inp->fields, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+struct trn_cell_extension_fields_st * *
+trn_cell_extension_getarray_fields(trn_cell_extension_t *inp)
+{
+  return inp->fields.elts_;
+}
+const struct trn_cell_extension_fields_st *  const  *
+trn_cell_extension_getconstarray_fields(const trn_cell_extension_t *inp)
+{
+  return (const struct trn_cell_extension_fields_st *  const  *)trn_cell_extension_getarray_fields((trn_cell_extension_t*)inp);
+}
+int
+trn_cell_extension_setlen_fields(trn_cell_extension_t *inp, size_t newlen)
+{
+  struct trn_cell_extension_fields_st * *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->fields.allocated_,
+                 &inp->fields.n_, inp->fields.elts_, newlen,
+                 sizeof(inp->fields.elts_[0]), (trunnel_free_fn_t) trn_cell_extension_fields_free,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->fields.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+trn_cell_extension_check(const trn_cell_extension_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
+      if (NULL != (msg = trn_cell_extension_fields_check(TRUNNEL_DYNARRAY_GET(&obj->fields, idx))))
+        return msg;
+    }
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->fields) != obj->num)
+    return "Length mismatch for fields";
+  return NULL;
+}
+
+ssize_t
+trn_cell_extension_encoded_len(const trn_cell_extension_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_extension_check(obj))
+     return -1;
+
+
+  /* Length of u8 num */
+  result += 1;
+
+  /* Length of struct trn_cell_extension_fields fields[num] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
+      result += trn_cell_extension_fields_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
+    }
+  }
+  return result;
+}
+int
+trn_cell_extension_clear_errors(trn_cell_extension_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_extension_encode(uint8_t *output, const size_t avail, const trn_cell_extension_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_extension_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_extension_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 num */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->num));
+  written += 1; ptr += 1;
+
+  /* Encode struct trn_cell_extension_fields fields[num] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
+      trunnel_assert(written <= avail);
+      result = trn_cell_extension_fields_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+    }
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_extension_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+trn_cell_extension_parse_into(trn_cell_extension_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 num */
+  CHECK_REMAINING(1, truncated);
+  obj->num = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse struct trn_cell_extension_fields fields[num] */
+  TRUNNEL_DYNARRAY_EXPAND(trn_cell_extension_fields_t *, &obj->fields, obj->num, {});
+  {
+    trn_cell_extension_fields_t * elt;
+    unsigned idx;
+    for (idx = 0; idx < obj->num; ++idx) {
+      result = trn_cell_extension_fields_parse(&elt, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      TRUNNEL_DYNARRAY_ADD(trn_cell_extension_fields_t *, &obj->fields, elt, {trn_cell_extension_fields_free(elt);});
+    }
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+trn_cell_extension_parse(trn_cell_extension_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_extension_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_extension_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_extension_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/hs/cell_common.h b/src/trunnel/hs/cell_common.h
new file mode 100644
index 0000000000..e08eedfdb3
--- /dev/null
+++ b/src/trunnel/hs/cell_common.h
@@ -0,0 +1,203 @@
+/* cell_common.h -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_CELL_COMMON_H
+#define TRUNNEL_CELL_COMMON_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_FIELDS)
+struct trn_cell_extension_fields_st {
+  uint8_t field_type;
+  uint8_t field_len;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) field;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_extension_fields_st trn_cell_extension_fields_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION)
+struct trn_cell_extension_st {
+  uint8_t num;
+  TRUNNEL_DYNARRAY_HEAD(, struct trn_cell_extension_fields_st *) fields;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_extension_st trn_cell_extension_t;
+/** Return a newly allocated trn_cell_extension_fields with all
+ * elements set to zero.
+ */
+trn_cell_extension_fields_t *trn_cell_extension_fields_new(void);
+/** Release all storage held by the trn_cell_extension_fields in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_extension_fields_free(trn_cell_extension_fields_t *victim);
+/** Try to parse a trn_cell_extension_fields from the buffer in
+ * 'input', using up to 'len_in' bytes from the input buffer. On
+ * success, return the number of bytes consumed and set *output to the
+ * newly allocated trn_cell_extension_fields_t. On failure, return -2
+ * if the input appears truncated, and -1 if the input is otherwise
+ * invalid.
+ */
+ssize_t trn_cell_extension_fields_parse(trn_cell_extension_fields_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_extension_fields in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_extension_fields_encoded_len(const trn_cell_extension_fields_t *obj);
+/** Try to encode the trn_cell_extension_fields from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_extension_fields_encode(uint8_t *output, size_t avail, const trn_cell_extension_fields_t *input);
+/** Check whether the internal state of the trn_cell_extension_fields
+ * in 'obj' is consistent. Return NULL if it is, and a short message
+ * if it is not.
+ */
+const char *trn_cell_extension_fields_check(const trn_cell_extension_fields_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_extension_fields_clear_errors(trn_cell_extension_fields_t *obj);
+/** Return the value of the field_type field of the
+ * trn_cell_extension_fields_t in 'inp'
+ */
+uint8_t trn_cell_extension_fields_get_field_type(const trn_cell_extension_fields_t *inp);
+/** Set the value of the field_type field of the
+ * trn_cell_extension_fields_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_extension_fields_set_field_type(trn_cell_extension_fields_t *inp, uint8_t val);
+/** Return the value of the field_len field of the
+ * trn_cell_extension_fields_t in 'inp'
+ */
+uint8_t trn_cell_extension_fields_get_field_len(const trn_cell_extension_fields_t *inp);
+/** Set the value of the field_len field of the
+ * trn_cell_extension_fields_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_extension_fields_set_field_len(trn_cell_extension_fields_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the field field of
+ * the trn_cell_extension_fields_t in 'inp'.
+ */
+size_t trn_cell_extension_fields_getlen_field(const trn_cell_extension_fields_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * field of the trn_cell_extension_fields_t in 'inp'.
+ */
+uint8_t trn_cell_extension_fields_get_field(trn_cell_extension_fields_t *inp, size_t idx);
+/** As trn_cell_extension_fields_get_field, but take and return a
+ * const pointer
+ */
+uint8_t trn_cell_extension_fields_getconst_field(const trn_cell_extension_fields_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * field of the trn_cell_extension_fields_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int trn_cell_extension_fields_set_field(trn_cell_extension_fields_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field field of the
+ * trn_cell_extension_fields_t in 'inp'.
+ */
+int trn_cell_extension_fields_add_field(trn_cell_extension_fields_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field field of
+ * 'inp'.
+ */
+uint8_t * trn_cell_extension_fields_getarray_field(trn_cell_extension_fields_t *inp);
+/** As trn_cell_extension_fields_get_field, but take and return a
+ * const pointer
+ */
+const uint8_t  * trn_cell_extension_fields_getconstarray_field(const trn_cell_extension_fields_t *inp);
+/** Change the length of the variable-length array field field of
+ * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_extension_fields_setlen_field(trn_cell_extension_fields_t *inp, size_t newlen);
+/** Return a newly allocated trn_cell_extension with all elements set
+ * to zero.
+ */
+trn_cell_extension_t *trn_cell_extension_new(void);
+/** Release all storage held by the trn_cell_extension in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_extension_free(trn_cell_extension_t *victim);
+/** Try to parse a trn_cell_extension from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated trn_cell_extension_t. On failure, return -2 if the input
+ * appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t trn_cell_extension_parse(trn_cell_extension_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_extension in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_extension_encoded_len(const trn_cell_extension_t *obj);
+/** Try to encode the trn_cell_extension from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_extension_encode(uint8_t *output, size_t avail, const trn_cell_extension_t *input);
+/** Check whether the internal state of the trn_cell_extension in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *trn_cell_extension_check(const trn_cell_extension_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_extension_clear_errors(trn_cell_extension_t *obj);
+/** Return the value of the num field of the trn_cell_extension_t in
+ * 'inp'
+ */
+uint8_t trn_cell_extension_get_num(const trn_cell_extension_t *inp);
+/** Set the value of the num field of the trn_cell_extension_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int trn_cell_extension_set_num(trn_cell_extension_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the fields field of
+ * the trn_cell_extension_t in 'inp'.
+ */
+size_t trn_cell_extension_getlen_fields(const trn_cell_extension_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * fields of the trn_cell_extension_t in 'inp'.
+ */
+struct trn_cell_extension_fields_st * trn_cell_extension_get_fields(trn_cell_extension_t *inp, size_t idx);
+/** As trn_cell_extension_get_fields, but take and return a const
+ * pointer
+ */
+ const struct trn_cell_extension_fields_st * trn_cell_extension_getconst_fields(const trn_cell_extension_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * fields of the trn_cell_extension_t in 'inp', so that it will hold
+ * the value 'elt'. Free the previous value, if any.
+ */
+int trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt);
+/** As trn_cell_extension_set_fields, but does not free the previous
+ * value.
+ */
+int trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt);
+/** Append a new element 'elt' to the dynamic array field fields of
+ * the trn_cell_extension_t in 'inp'.
+ */
+int trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_fields_st * elt);
+/** Return a pointer to the variable-length array field fields of
+ * 'inp'.
+ */
+struct trn_cell_extension_fields_st * * trn_cell_extension_getarray_fields(trn_cell_extension_t *inp);
+/** As trn_cell_extension_get_fields, but take and return a const
+ * pointer
+ */
+const struct trn_cell_extension_fields_st *  const  * trn_cell_extension_getconstarray_fields(const trn_cell_extension_t *inp);
+/** Change the length of the variable-length array field fields of
+ * 'inp' to 'newlen'.Fill extra elements with NULL; free removed
+ * elements. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int trn_cell_extension_setlen_fields(trn_cell_extension_t *inp, size_t newlen);
+
+
+#endif
diff --git a/src/trunnel/hs/cell_common.trunnel b/src/trunnel/hs/cell_common.trunnel
new file mode 100644
index 0000000000..1aa6999de7
--- /dev/null
+++ b/src/trunnel/hs/cell_common.trunnel
@@ -0,0 +1,12 @@
+/* This file contains common data structure that cells use. */
+
+struct trn_cell_extension_fields {
+  u8 field_type;
+  u8 field_len;
+  u8 field[field_len];
+};
+
+struct trn_cell_extension {
+  u8 num;
+  struct trn_cell_extension_fields fields[num];
+};
diff --git a/src/trunnel/hs/cell_establish_intro.c b/src/trunnel/hs/cell_establish_intro.c
new file mode 100644
index 0000000000..ae3b7b1bc8
--- /dev/null
+++ b/src/trunnel/hs/cell_establish_intro.c
@@ -0,0 +1,735 @@
+/* cell_establish_intro.c -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "cell_establish_intro.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int cellestablishintro_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || cellestablishintro_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+typedef struct trn_cell_extension_st trn_cell_extension_t;
+trn_cell_extension_t *trn_cell_extension_new(void);
+void trn_cell_extension_free(trn_cell_extension_t *victim);
+ssize_t trn_cell_extension_parse(trn_cell_extension_t **output, const uint8_t *input, const size_t len_in);
+ssize_t trn_cell_extension_encoded_len(const trn_cell_extension_t *obj);
+ssize_t trn_cell_extension_encode(uint8_t *output, size_t avail, const trn_cell_extension_t *input);
+const char *trn_cell_extension_check(const trn_cell_extension_t *obj);
+int trn_cell_extension_clear_errors(trn_cell_extension_t *obj);
+trn_cell_establish_intro_t *
+trn_cell_establish_intro_new(void)
+{
+  trn_cell_establish_intro_t *val = trunnel_calloc(1, sizeof(trn_cell_establish_intro_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_establish_intro_clear(trn_cell_establish_intro_t *obj)
+{
+  (void) obj;
+  TRUNNEL_DYNARRAY_WIPE(&obj->auth_key);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->auth_key);
+  trn_cell_extension_free(obj->extensions);
+  obj->extensions = NULL;
+  TRUNNEL_DYNARRAY_WIPE(&obj->sig);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->sig);
+}
+
+void
+trn_cell_establish_intro_free(trn_cell_establish_intro_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_establish_intro_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_establish_intro_t));
+  trunnel_free_(obj);
+}
+
+const uint8_t *
+trn_cell_establish_intro_get_start_cell(const trn_cell_establish_intro_t *inp)
+{
+  return inp->start_cell;
+}
+uint8_t
+trn_cell_establish_intro_get_auth_key_type(const trn_cell_establish_intro_t *inp)
+{
+  return inp->auth_key_type;
+}
+int
+trn_cell_establish_intro_set_auth_key_type(trn_cell_establish_intro_t *inp, uint8_t val)
+{
+  if (! ((val == 0 || val == 1 || val == 2))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->auth_key_type = val;
+  return 0;
+}
+uint16_t
+trn_cell_establish_intro_get_auth_key_len(const trn_cell_establish_intro_t *inp)
+{
+  return inp->auth_key_len;
+}
+int
+trn_cell_establish_intro_set_auth_key_len(trn_cell_establish_intro_t *inp, uint16_t val)
+{
+  inp->auth_key_len = val;
+  return 0;
+}
+size_t
+trn_cell_establish_intro_getlen_auth_key(const trn_cell_establish_intro_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->auth_key);
+}
+
+uint8_t
+trn_cell_establish_intro_get_auth_key(trn_cell_establish_intro_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->auth_key, idx);
+}
+
+uint8_t
+trn_cell_establish_intro_getconst_auth_key(const trn_cell_establish_intro_t *inp, size_t idx)
+{
+  return trn_cell_establish_intro_get_auth_key((trn_cell_establish_intro_t*)inp, idx);
+}
+int
+trn_cell_establish_intro_set_auth_key(trn_cell_establish_intro_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->auth_key, idx, elt);
+  return 0;
+}
+int
+trn_cell_establish_intro_add_auth_key(trn_cell_establish_intro_t *inp, uint8_t elt)
+{
+#if SIZE_MAX >= UINT16_MAX
+  if (inp->auth_key.n_ == UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->auth_key, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_establish_intro_getarray_auth_key(trn_cell_establish_intro_t *inp)
+{
+  return inp->auth_key.elts_;
+}
+const uint8_t  *
+trn_cell_establish_intro_getconstarray_auth_key(const trn_cell_establish_intro_t *inp)
+{
+  return (const uint8_t  *)trn_cell_establish_intro_getarray_auth_key((trn_cell_establish_intro_t*)inp);
+}
+int
+trn_cell_establish_intro_setlen_auth_key(trn_cell_establish_intro_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+#if UINT16_MAX < SIZE_MAX
+  if (newlen > UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->auth_key.allocated_,
+                 &inp->auth_key.n_, inp->auth_key.elts_, newlen,
+                 sizeof(inp->auth_key.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->auth_key.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+struct trn_cell_extension_st *
+trn_cell_establish_intro_get_extensions(trn_cell_establish_intro_t *inp)
+{
+  return inp->extensions;
+}
+const struct trn_cell_extension_st *
+trn_cell_establish_intro_getconst_extensions(const trn_cell_establish_intro_t *inp)
+{
+  return trn_cell_establish_intro_get_extensions((trn_cell_establish_intro_t*) inp);
+}
+int
+trn_cell_establish_intro_set_extensions(trn_cell_establish_intro_t *inp, struct trn_cell_extension_st *val)
+{
+  if (inp->extensions && inp->extensions != val)
+    trn_cell_extension_free(inp->extensions);
+  return trn_cell_establish_intro_set0_extensions(inp, val);
+}
+int
+trn_cell_establish_intro_set0_extensions(trn_cell_establish_intro_t *inp, struct trn_cell_extension_st *val)
+{
+  inp->extensions = val;
+  return 0;
+}
+const uint8_t *
+trn_cell_establish_intro_get_end_mac_fields(const trn_cell_establish_intro_t *inp)
+{
+  return inp->end_mac_fields;
+}
+size_t
+trn_cell_establish_intro_getlen_handshake_mac(const trn_cell_establish_intro_t *inp)
+{
+  (void)inp;  return TRUNNEL_SHA3_256_LEN;
+}
+
+uint8_t
+trn_cell_establish_intro_get_handshake_mac(trn_cell_establish_intro_t *inp, size_t idx)
+{
+  trunnel_assert(idx < TRUNNEL_SHA3_256_LEN);
+  return inp->handshake_mac[idx];
+}
+
+uint8_t
+trn_cell_establish_intro_getconst_handshake_mac(const trn_cell_establish_intro_t *inp, size_t idx)
+{
+  return trn_cell_establish_intro_get_handshake_mac((trn_cell_establish_intro_t*)inp, idx);
+}
+int
+trn_cell_establish_intro_set_handshake_mac(trn_cell_establish_intro_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < TRUNNEL_SHA3_256_LEN);
+  inp->handshake_mac[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+trn_cell_establish_intro_getarray_handshake_mac(trn_cell_establish_intro_t *inp)
+{
+  return inp->handshake_mac;
+}
+const uint8_t  *
+trn_cell_establish_intro_getconstarray_handshake_mac(const trn_cell_establish_intro_t *inp)
+{
+  return (const uint8_t  *)trn_cell_establish_intro_getarray_handshake_mac((trn_cell_establish_intro_t*)inp);
+}
+const uint8_t *
+trn_cell_establish_intro_get_end_sig_fields(const trn_cell_establish_intro_t *inp)
+{
+  return inp->end_sig_fields;
+}
+uint16_t
+trn_cell_establish_intro_get_sig_len(const trn_cell_establish_intro_t *inp)
+{
+  return inp->sig_len;
+}
+int
+trn_cell_establish_intro_set_sig_len(trn_cell_establish_intro_t *inp, uint16_t val)
+{
+  inp->sig_len = val;
+  return 0;
+}
+size_t
+trn_cell_establish_intro_getlen_sig(const trn_cell_establish_intro_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->sig);
+}
+
+uint8_t
+trn_cell_establish_intro_get_sig(trn_cell_establish_intro_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->sig, idx);
+}
+
+uint8_t
+trn_cell_establish_intro_getconst_sig(const trn_cell_establish_intro_t *inp, size_t idx)
+{
+  return trn_cell_establish_intro_get_sig((trn_cell_establish_intro_t*)inp, idx);
+}
+int
+trn_cell_establish_intro_set_sig(trn_cell_establish_intro_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->sig, idx, elt);
+  return 0;
+}
+int
+trn_cell_establish_intro_add_sig(trn_cell_establish_intro_t *inp, uint8_t elt)
+{
+#if SIZE_MAX >= UINT16_MAX
+  if (inp->sig.n_ == UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->sig, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_establish_intro_getarray_sig(trn_cell_establish_intro_t *inp)
+{
+  return inp->sig.elts_;
+}
+const uint8_t  *
+trn_cell_establish_intro_getconstarray_sig(const trn_cell_establish_intro_t *inp)
+{
+  return (const uint8_t  *)trn_cell_establish_intro_getarray_sig((trn_cell_establish_intro_t*)inp);
+}
+int
+trn_cell_establish_intro_setlen_sig(trn_cell_establish_intro_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+#if UINT16_MAX < SIZE_MAX
+  if (newlen > UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->sig.allocated_,
+                 &inp->sig.n_, inp->sig.elts_, newlen,
+                 sizeof(inp->sig.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->sig.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+trn_cell_establish_intro_check(const trn_cell_establish_intro_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->auth_key_type == 0 || obj->auth_key_type == 1 || obj->auth_key_type == 2))
+    return "Integer out of bounds";
+  if (TRUNNEL_DYNARRAY_LEN(&obj->auth_key) != obj->auth_key_len)
+    return "Length mismatch for auth_key";
+  {
+    const char *msg;
+    if (NULL != (msg = trn_cell_extension_check(obj->extensions)))
+      return msg;
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->sig) != obj->sig_len)
+    return "Length mismatch for sig";
+  return NULL;
+}
+
+ssize_t
+trn_cell_establish_intro_encoded_len(const trn_cell_establish_intro_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_establish_intro_check(obj))
+     return -1;
+
+
+  /* Length of u8 auth_key_type IN [0, 1, 2] */
+  result += 1;
+
+  /* Length of u16 auth_key_len */
+  result += 2;
+
+  /* Length of u8 auth_key[auth_key_len] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->auth_key);
+
+  /* Length of struct trn_cell_extension extensions */
+  result += trn_cell_extension_encoded_len(obj->extensions);
+
+  /* Length of u8 handshake_mac[TRUNNEL_SHA3_256_LEN] */
+  result += TRUNNEL_SHA3_256_LEN;
+
+  /* Length of u16 sig_len */
+  result += 2;
+
+  /* Length of u8 sig[sig_len] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->sig);
+  return result;
+}
+int
+trn_cell_establish_intro_clear_errors(trn_cell_establish_intro_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_establish_intro_encode(uint8_t *output, const size_t avail, const trn_cell_establish_intro_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_establish_intro_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_establish_intro_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 auth_key_type IN [0, 1, 2] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->auth_key_type));
+  written += 1; ptr += 1;
+
+  /* Encode u16 auth_key_len */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->auth_key_len));
+  written += 2; ptr += 2;
+
+  /* Encode u8 auth_key[auth_key_len] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->auth_key);
+    trunnel_assert(obj->auth_key_len == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->auth_key.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+  /* Encode struct trn_cell_extension extensions */
+  trunnel_assert(written <= avail);
+  result = trn_cell_extension_encode(ptr, avail - written, obj->extensions);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+  /* Encode u8 handshake_mac[TRUNNEL_SHA3_256_LEN] */
+  trunnel_assert(written <= avail);
+  if (avail - written < TRUNNEL_SHA3_256_LEN)
+    goto truncated;
+  memcpy(ptr, obj->handshake_mac, TRUNNEL_SHA3_256_LEN);
+  written += TRUNNEL_SHA3_256_LEN; ptr += TRUNNEL_SHA3_256_LEN;
+
+  /* Encode u16 sig_len */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->sig_len));
+  written += 2; ptr += 2;
+
+  /* Encode u8 sig[sig_len] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->sig);
+    trunnel_assert(obj->sig_len == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->sig.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_establish_intro_parse(), but do not allocate the
+ * output object.
+ */
+static ssize_t
+trn_cell_establish_intro_parse_into(trn_cell_establish_intro_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+  obj->start_cell = ptr;
+
+  /* Parse u8 auth_key_type IN [0, 1, 2] */
+  CHECK_REMAINING(1, truncated);
+  obj->auth_key_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->auth_key_type == 0 || obj->auth_key_type == 1 || obj->auth_key_type == 2))
+    goto fail;
+
+  /* Parse u16 auth_key_len */
+  CHECK_REMAINING(2, truncated);
+  obj->auth_key_len = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u8 auth_key[auth_key_len] */
+  CHECK_REMAINING(obj->auth_key_len, truncated);
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->auth_key, obj->auth_key_len, {});
+  obj->auth_key.n_ = obj->auth_key_len;
+  if (obj->auth_key_len)
+    memcpy(obj->auth_key.elts_, ptr, obj->auth_key_len);
+  ptr += obj->auth_key_len; remaining -= obj->auth_key_len;
+
+  /* Parse struct trn_cell_extension extensions */
+  result = trn_cell_extension_parse(&obj->extensions, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+  obj->end_mac_fields = ptr;
+
+  /* Parse u8 handshake_mac[TRUNNEL_SHA3_256_LEN] */
+  CHECK_REMAINING(TRUNNEL_SHA3_256_LEN, truncated);
+  memcpy(obj->handshake_mac, ptr, TRUNNEL_SHA3_256_LEN);
+  remaining -= TRUNNEL_SHA3_256_LEN; ptr += TRUNNEL_SHA3_256_LEN;
+  obj->end_sig_fields = ptr;
+
+  /* Parse u16 sig_len */
+  CHECK_REMAINING(2, truncated);
+  obj->sig_len = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u8 sig[sig_len] */
+  CHECK_REMAINING(obj->sig_len, truncated);
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->sig, obj->sig_len, {});
+  obj->sig.n_ = obj->sig_len;
+  if (obj->sig_len)
+    memcpy(obj->sig.elts_, ptr, obj->sig_len);
+  ptr += obj->sig_len; remaining -= obj->sig_len;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+trn_cell_establish_intro_parse(trn_cell_establish_intro_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_establish_intro_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_establish_intro_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_establish_intro_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+trn_cell_intro_established_t *
+trn_cell_intro_established_new(void)
+{
+  trn_cell_intro_established_t *val = trunnel_calloc(1, sizeof(trn_cell_intro_established_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_intro_established_clear(trn_cell_intro_established_t *obj)
+{
+  (void) obj;
+  trn_cell_extension_free(obj->extensions);
+  obj->extensions = NULL;
+}
+
+void
+trn_cell_intro_established_free(trn_cell_intro_established_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_intro_established_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_intro_established_t));
+  trunnel_free_(obj);
+}
+
+struct trn_cell_extension_st *
+trn_cell_intro_established_get_extensions(trn_cell_intro_established_t *inp)
+{
+  return inp->extensions;
+}
+const struct trn_cell_extension_st *
+trn_cell_intro_established_getconst_extensions(const trn_cell_intro_established_t *inp)
+{
+  return trn_cell_intro_established_get_extensions((trn_cell_intro_established_t*) inp);
+}
+int
+trn_cell_intro_established_set_extensions(trn_cell_intro_established_t *inp, struct trn_cell_extension_st *val)
+{
+  if (inp->extensions && inp->extensions != val)
+    trn_cell_extension_free(inp->extensions);
+  return trn_cell_intro_established_set0_extensions(inp, val);
+}
+int
+trn_cell_intro_established_set0_extensions(trn_cell_intro_established_t *inp, struct trn_cell_extension_st *val)
+{
+  inp->extensions = val;
+  return 0;
+}
+const char *
+trn_cell_intro_established_check(const trn_cell_intro_established_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+    if (NULL != (msg = trn_cell_extension_check(obj->extensions)))
+      return msg;
+  }
+  return NULL;
+}
+
+ssize_t
+trn_cell_intro_established_encoded_len(const trn_cell_intro_established_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_intro_established_check(obj))
+     return -1;
+
+
+  /* Length of struct trn_cell_extension extensions */
+  result += trn_cell_extension_encoded_len(obj->extensions);
+  return result;
+}
+int
+trn_cell_intro_established_clear_errors(trn_cell_intro_established_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_intro_established_encode(uint8_t *output, const size_t avail, const trn_cell_intro_established_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_intro_established_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_intro_established_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode struct trn_cell_extension extensions */
+  trunnel_assert(written <= avail);
+  result = trn_cell_extension_encode(ptr, avail - written, obj->extensions);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_intro_established_parse(), but do not allocate the
+ * output object.
+ */
+static ssize_t
+trn_cell_intro_established_parse_into(trn_cell_intro_established_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse struct trn_cell_extension extensions */
+  result = trn_cell_extension_parse(&obj->extensions, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+ssize_t
+trn_cell_intro_established_parse(trn_cell_intro_established_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_intro_established_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_intro_established_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_intro_established_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/hs/cell_establish_intro.h b/src/trunnel/hs/cell_establish_intro.h
new file mode 100644
index 0000000000..ccaef5488c
--- /dev/null
+++ b/src/trunnel/hs/cell_establish_intro.h
@@ -0,0 +1,276 @@
+/* cell_establish_intro.h -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_CELL_ESTABLISH_INTRO_H
+#define TRUNNEL_CELL_ESTABLISH_INTRO_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+struct trn_cell_extension_st;
+#define TRUNNEL_SHA3_256_LEN 32
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_ESTABLISH_INTRO)
+struct trn_cell_establish_intro_st {
+  const uint8_t *start_cell;
+  uint8_t auth_key_type;
+  uint16_t auth_key_len;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) auth_key;
+  struct trn_cell_extension_st *extensions;
+  const uint8_t *end_mac_fields;
+  uint8_t handshake_mac[TRUNNEL_SHA3_256_LEN];
+  const uint8_t *end_sig_fields;
+  uint16_t sig_len;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) sig;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_establish_intro_st trn_cell_establish_intro_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_INTRO_ESTABLISHED)
+struct trn_cell_intro_established_st {
+  struct trn_cell_extension_st *extensions;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_intro_established_st trn_cell_intro_established_t;
+/** Return a newly allocated trn_cell_establish_intro with all
+ * elements set to zero.
+ */
+trn_cell_establish_intro_t *trn_cell_establish_intro_new(void);
+/** Release all storage held by the trn_cell_establish_intro in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_establish_intro_free(trn_cell_establish_intro_t *victim);
+/** Try to parse a trn_cell_establish_intro from the buffer in
+ * 'input', using up to 'len_in' bytes from the input buffer. On
+ * success, return the number of bytes consumed and set *output to the
+ * newly allocated trn_cell_establish_intro_t. On failure, return -2
+ * if the input appears truncated, and -1 if the input is otherwise
+ * invalid.
+ */
+ssize_t trn_cell_establish_intro_parse(trn_cell_establish_intro_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_establish_intro in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_establish_intro_encoded_len(const trn_cell_establish_intro_t *obj);
+/** Try to encode the trn_cell_establish_intro from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_establish_intro_encode(uint8_t *output, size_t avail, const trn_cell_establish_intro_t *input);
+/** Check whether the internal state of the trn_cell_establish_intro
+ * in 'obj' is consistent. Return NULL if it is, and a short message
+ * if it is not.
+ */
+const char *trn_cell_establish_intro_check(const trn_cell_establish_intro_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_establish_intro_clear_errors(trn_cell_establish_intro_t *obj);
+/** Return the position for start_cell when we parsed this object
+ */
+const uint8_t * trn_cell_establish_intro_get_start_cell(const trn_cell_establish_intro_t *inp);
+/** Return the value of the auth_key_type field of the
+ * trn_cell_establish_intro_t in 'inp'
+ */
+uint8_t trn_cell_establish_intro_get_auth_key_type(const trn_cell_establish_intro_t *inp);
+/** Set the value of the auth_key_type field of the
+ * trn_cell_establish_intro_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_establish_intro_set_auth_key_type(trn_cell_establish_intro_t *inp, uint8_t val);
+/** Return the value of the auth_key_len field of the
+ * trn_cell_establish_intro_t in 'inp'
+ */
+uint16_t trn_cell_establish_intro_get_auth_key_len(const trn_cell_establish_intro_t *inp);
+/** Set the value of the auth_key_len field of the
+ * trn_cell_establish_intro_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_establish_intro_set_auth_key_len(trn_cell_establish_intro_t *inp, uint16_t val);
+/** Return the length of the dynamic array holding the auth_key field
+ * of the trn_cell_establish_intro_t in 'inp'.
+ */
+size_t trn_cell_establish_intro_getlen_auth_key(const trn_cell_establish_intro_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * auth_key of the trn_cell_establish_intro_t in 'inp'.
+ */
+uint8_t trn_cell_establish_intro_get_auth_key(trn_cell_establish_intro_t *inp, size_t idx);
+/** As trn_cell_establish_intro_get_auth_key, but take and return a
+ * const pointer
+ */
+uint8_t trn_cell_establish_intro_getconst_auth_key(const trn_cell_establish_intro_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * auth_key of the trn_cell_establish_intro_t in 'inp', so that it
+ * will hold the value 'elt'.
+ */
+int trn_cell_establish_intro_set_auth_key(trn_cell_establish_intro_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field auth_key of
+ * the trn_cell_establish_intro_t in 'inp'.
+ */
+int trn_cell_establish_intro_add_auth_key(trn_cell_establish_intro_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field auth_key of
+ * 'inp'.
+ */
+uint8_t * trn_cell_establish_intro_getarray_auth_key(trn_cell_establish_intro_t *inp);
+/** As trn_cell_establish_intro_get_auth_key, but take and return a
+ * const pointer
+ */
+const uint8_t  * trn_cell_establish_intro_getconstarray_auth_key(const trn_cell_establish_intro_t *inp);
+/** Change the length of the variable-length array field auth_key of
+ * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_establish_intro_setlen_auth_key(trn_cell_establish_intro_t *inp, size_t newlen);
+/** Return the value of the extensions field of the
+ * trn_cell_establish_intro_t in 'inp'
+ */
+struct trn_cell_extension_st * trn_cell_establish_intro_get_extensions(trn_cell_establish_intro_t *inp);
+/** As trn_cell_establish_intro_get_extensions, but take and return a
+ * const pointer
+ */
+const struct trn_cell_extension_st * trn_cell_establish_intro_getconst_extensions(const trn_cell_establish_intro_t *inp);
+/** Set the value of the extensions field of the
+ * trn_cell_establish_intro_t in 'inp' to 'val'. Free the old value if
+ * any. Steals the referenceto 'val'.Return 0 on success; return -1
+ * and set the error code on 'inp' on failure.
+ */
+int trn_cell_establish_intro_set_extensions(trn_cell_establish_intro_t *inp, struct trn_cell_extension_st *val);
+/** As trn_cell_establish_intro_set_extensions, but does not free the
+ * previous value.
+ */
+int trn_cell_establish_intro_set0_extensions(trn_cell_establish_intro_t *inp, struct trn_cell_extension_st *val);
+/** Return the position for end_mac_fields when we parsed this object
+ */
+const uint8_t * trn_cell_establish_intro_get_end_mac_fields(const trn_cell_establish_intro_t *inp);
+/** Return the (constant) length of the array holding the
+ * handshake_mac field of the trn_cell_establish_intro_t in 'inp'.
+ */
+size_t trn_cell_establish_intro_getlen_handshake_mac(const trn_cell_establish_intro_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * handshake_mac of the trn_cell_establish_intro_t in 'inp'.
+ */
+uint8_t trn_cell_establish_intro_get_handshake_mac(trn_cell_establish_intro_t *inp, size_t idx);
+/** As trn_cell_establish_intro_get_handshake_mac, but take and return
+ * a const pointer
+ */
+uint8_t trn_cell_establish_intro_getconst_handshake_mac(const trn_cell_establish_intro_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * handshake_mac of the trn_cell_establish_intro_t in 'inp', so that
+ * it will hold the value 'elt'.
+ */
+int trn_cell_establish_intro_set_handshake_mac(trn_cell_establish_intro_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the TRUNNEL_SHA3_256_LEN-element array field
+ * handshake_mac of 'inp'.
+ */
+uint8_t * trn_cell_establish_intro_getarray_handshake_mac(trn_cell_establish_intro_t *inp);
+/** As trn_cell_establish_intro_get_handshake_mac, but take and return
+ * a const pointer
+ */
+const uint8_t  * trn_cell_establish_intro_getconstarray_handshake_mac(const trn_cell_establish_intro_t *inp);
+/** Return the position for end_sig_fields when we parsed this object
+ */
+const uint8_t * trn_cell_establish_intro_get_end_sig_fields(const trn_cell_establish_intro_t *inp);
+/** Return the value of the sig_len field of the
+ * trn_cell_establish_intro_t in 'inp'
+ */
+uint16_t trn_cell_establish_intro_get_sig_len(const trn_cell_establish_intro_t *inp);
+/** Set the value of the sig_len field of the
+ * trn_cell_establish_intro_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_establish_intro_set_sig_len(trn_cell_establish_intro_t *inp, uint16_t val);
+/** Return the length of the dynamic array holding the sig field of
+ * the trn_cell_establish_intro_t in 'inp'.
+ */
+size_t trn_cell_establish_intro_getlen_sig(const trn_cell_establish_intro_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * sig of the trn_cell_establish_intro_t in 'inp'.
+ */
+uint8_t trn_cell_establish_intro_get_sig(trn_cell_establish_intro_t *inp, size_t idx);
+/** As trn_cell_establish_intro_get_sig, but take and return a const
+ * pointer
+ */
+uint8_t trn_cell_establish_intro_getconst_sig(const trn_cell_establish_intro_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * sig of the trn_cell_establish_intro_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int trn_cell_establish_intro_set_sig(trn_cell_establish_intro_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field sig of the
+ * trn_cell_establish_intro_t in 'inp'.
+ */
+int trn_cell_establish_intro_add_sig(trn_cell_establish_intro_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field sig of 'inp'.
+ */
+uint8_t * trn_cell_establish_intro_getarray_sig(trn_cell_establish_intro_t *inp);
+/** As trn_cell_establish_intro_get_sig, but take and return a const
+ * pointer
+ */
+const uint8_t  * trn_cell_establish_intro_getconstarray_sig(const trn_cell_establish_intro_t *inp);
+/** Change the length of the variable-length array field sig of 'inp'
+ * to 'newlen'.Fill extra elements with 0. Return 0 on success; return
+ * -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_establish_intro_setlen_sig(trn_cell_establish_intro_t *inp, size_t newlen);
+/** Return a newly allocated trn_cell_intro_established with all
+ * elements set to zero.
+ */
+trn_cell_intro_established_t *trn_cell_intro_established_new(void);
+/** Release all storage held by the trn_cell_intro_established in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_intro_established_free(trn_cell_intro_established_t *victim);
+/** Try to parse a trn_cell_intro_established from the buffer in
+ * 'input', using up to 'len_in' bytes from the input buffer. On
+ * success, return the number of bytes consumed and set *output to the
+ * newly allocated trn_cell_intro_established_t. On failure, return -2
+ * if the input appears truncated, and -1 if the input is otherwise
+ * invalid.
+ */
+ssize_t trn_cell_intro_established_parse(trn_cell_intro_established_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_intro_established in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_intro_established_encoded_len(const trn_cell_intro_established_t *obj);
+/** Try to encode the trn_cell_intro_established from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_intro_established_encode(uint8_t *output, size_t avail, const trn_cell_intro_established_t *input);
+/** Check whether the internal state of the trn_cell_intro_established
+ * in 'obj' is consistent. Return NULL if it is, and a short message
+ * if it is not.
+ */
+const char *trn_cell_intro_established_check(const trn_cell_intro_established_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_intro_established_clear_errors(trn_cell_intro_established_t *obj);
+/** Return the value of the extensions field of the
+ * trn_cell_intro_established_t in 'inp'
+ */
+struct trn_cell_extension_st * trn_cell_intro_established_get_extensions(trn_cell_intro_established_t *inp);
+/** As trn_cell_intro_established_get_extensions, but take and return
+ * a const pointer
+ */
+const struct trn_cell_extension_st * trn_cell_intro_established_getconst_extensions(const trn_cell_intro_established_t *inp);
+/** Set the value of the extensions field of the
+ * trn_cell_intro_established_t in 'inp' to 'val'. Free the old value
+ * if any. Steals the referenceto 'val'.Return 0 on success; return -1
+ * and set the error code on 'inp' on failure.
+ */
+int trn_cell_intro_established_set_extensions(trn_cell_intro_established_t *inp, struct trn_cell_extension_st *val);
+/** As trn_cell_intro_established_set_extensions, but does not free
+ * the previous value.
+ */
+int trn_cell_intro_established_set0_extensions(trn_cell_intro_established_t *inp, struct trn_cell_extension_st *val);
+
+
+#endif
diff --git a/src/trunnel/hs/cell_establish_intro.trunnel b/src/trunnel/hs/cell_establish_intro.trunnel
new file mode 100644
index 0000000000..011ee62a15
--- /dev/null
+++ b/src/trunnel/hs/cell_establish_intro.trunnel
@@ -0,0 +1,41 @@
+/*
+ * This contains the definition of the ESTABLISH_INTRO and INTRO_ESTABLISHED
+ * cell for onion service version 3 and onward. The following format is
+ * specified in proposal 224 section 3.1.
+ */
+
+extern struct trn_cell_extension;
+
+const TRUNNEL_SHA3_256_LEN = 32;
+
+/* ESTABLISH_INTRO payload. See details in section 3.1.1 */
+struct trn_cell_establish_intro {
+  /* Indicate the start of the handshake authentication data. */
+  @ptr start_cell;
+
+  /* Authentication key material. */
+  u8 auth_key_type IN [0x00, 0x01, 0x02];
+  u16 auth_key_len;
+  u8 auth_key[auth_key_len];
+
+  /* Extension(s). Reserved fields. */
+  struct trn_cell_extension extensions;
+  @ptr end_mac_fields;
+
+  /* Handshake MAC. */
+  u8 handshake_mac[TRUNNEL_SHA3_256_LEN];
+
+  /* Signature */
+  /* Indicate the end of the handshake authentication data. */
+  @ptr end_sig_fields;
+  u16 sig_len;
+  u8 sig[sig_len];
+};
+
+/* INTRO_ESTABLISHED payload which is an acknowledge of the ESTABLISH_INTRO
+ * cell. For legacy node, this payload is empty so the following only applies
+ * to version >= 3. */
+struct trn_cell_intro_established {
+  /* Extension(s). Reserved fields. */
+  struct trn_cell_extension extensions;
+};
diff --git a/src/trunnel/hs/cell_introduce1.c b/src/trunnel/hs/cell_introduce1.c
new file mode 100644
index 0000000000..53b3d299f2
--- /dev/null
+++ b/src/trunnel/hs/cell_introduce1.c
@@ -0,0 +1,1347 @@
+/* cell_introduce1.c -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "cell_introduce1.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int cellintroduce_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || cellintroduce_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+typedef struct trn_cell_extension_st trn_cell_extension_t;
+trn_cell_extension_t *trn_cell_extension_new(void);
+void trn_cell_extension_free(trn_cell_extension_t *victim);
+ssize_t trn_cell_extension_parse(trn_cell_extension_t **output, const uint8_t *input, const size_t len_in);
+ssize_t trn_cell_extension_encoded_len(const trn_cell_extension_t *obj);
+ssize_t trn_cell_extension_encode(uint8_t *output, size_t avail, const trn_cell_extension_t *input);
+const char *trn_cell_extension_check(const trn_cell_extension_t *obj);
+int trn_cell_extension_clear_errors(trn_cell_extension_t *obj);
+typedef struct link_specifier_st link_specifier_t;
+link_specifier_t *link_specifier_new(void);
+void link_specifier_free(link_specifier_t *victim);
+ssize_t link_specifier_parse(link_specifier_t **output, const uint8_t *input, const size_t len_in);
+ssize_t link_specifier_encoded_len(const link_specifier_t *obj);
+ssize_t link_specifier_encode(uint8_t *output, size_t avail, const link_specifier_t *input);
+const char *link_specifier_check(const link_specifier_t *obj);
+int link_specifier_clear_errors(link_specifier_t *obj);
+trn_cell_introduce1_t *
+trn_cell_introduce1_new(void)
+{
+  trn_cell_introduce1_t *val = trunnel_calloc(1, sizeof(trn_cell_introduce1_t));
+  if (NULL == val)
+    return NULL;
+  val->auth_key_type = TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_ED25519;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_introduce1_clear(trn_cell_introduce1_t *obj)
+{
+  (void) obj;
+  TRUNNEL_DYNARRAY_WIPE(&obj->auth_key);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->auth_key);
+  trn_cell_extension_free(obj->extensions);
+  obj->extensions = NULL;
+  TRUNNEL_DYNARRAY_WIPE(&obj->encrypted);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->encrypted);
+}
+
+void
+trn_cell_introduce1_free(trn_cell_introduce1_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_introduce1_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_introduce1_t));
+  trunnel_free_(obj);
+}
+
+size_t
+trn_cell_introduce1_getlen_legacy_key_id(const trn_cell_introduce1_t *inp)
+{
+  (void)inp;  return TRUNNEL_SHA1_LEN;
+}
+
+uint8_t
+trn_cell_introduce1_get_legacy_key_id(trn_cell_introduce1_t *inp, size_t idx)
+{
+  trunnel_assert(idx < TRUNNEL_SHA1_LEN);
+  return inp->legacy_key_id[idx];
+}
+
+uint8_t
+trn_cell_introduce1_getconst_legacy_key_id(const trn_cell_introduce1_t *inp, size_t idx)
+{
+  return trn_cell_introduce1_get_legacy_key_id((trn_cell_introduce1_t*)inp, idx);
+}
+int
+trn_cell_introduce1_set_legacy_key_id(trn_cell_introduce1_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < TRUNNEL_SHA1_LEN);
+  inp->legacy_key_id[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+trn_cell_introduce1_getarray_legacy_key_id(trn_cell_introduce1_t *inp)
+{
+  return inp->legacy_key_id;
+}
+const uint8_t  *
+trn_cell_introduce1_getconstarray_legacy_key_id(const trn_cell_introduce1_t *inp)
+{
+  return (const uint8_t  *)trn_cell_introduce1_getarray_legacy_key_id((trn_cell_introduce1_t*)inp);
+}
+uint8_t
+trn_cell_introduce1_get_auth_key_type(const trn_cell_introduce1_t *inp)
+{
+  return inp->auth_key_type;
+}
+int
+trn_cell_introduce1_set_auth_key_type(trn_cell_introduce1_t *inp, uint8_t val)
+{
+  if (! ((val == TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_ED25519 || val == TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_LEGACY0 || val == TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_LEGACY1))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->auth_key_type = val;
+  return 0;
+}
+uint16_t
+trn_cell_introduce1_get_auth_key_len(const trn_cell_introduce1_t *inp)
+{
+  return inp->auth_key_len;
+}
+int
+trn_cell_introduce1_set_auth_key_len(trn_cell_introduce1_t *inp, uint16_t val)
+{
+  inp->auth_key_len = val;
+  return 0;
+}
+size_t
+trn_cell_introduce1_getlen_auth_key(const trn_cell_introduce1_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->auth_key);
+}
+
+uint8_t
+trn_cell_introduce1_get_auth_key(trn_cell_introduce1_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->auth_key, idx);
+}
+
+uint8_t
+trn_cell_introduce1_getconst_auth_key(const trn_cell_introduce1_t *inp, size_t idx)
+{
+  return trn_cell_introduce1_get_auth_key((trn_cell_introduce1_t*)inp, idx);
+}
+int
+trn_cell_introduce1_set_auth_key(trn_cell_introduce1_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->auth_key, idx, elt);
+  return 0;
+}
+int
+trn_cell_introduce1_add_auth_key(trn_cell_introduce1_t *inp, uint8_t elt)
+{
+#if SIZE_MAX >= UINT16_MAX
+  if (inp->auth_key.n_ == UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->auth_key, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_introduce1_getarray_auth_key(trn_cell_introduce1_t *inp)
+{
+  return inp->auth_key.elts_;
+}
+const uint8_t  *
+trn_cell_introduce1_getconstarray_auth_key(const trn_cell_introduce1_t *inp)
+{
+  return (const uint8_t  *)trn_cell_introduce1_getarray_auth_key((trn_cell_introduce1_t*)inp);
+}
+int
+trn_cell_introduce1_setlen_auth_key(trn_cell_introduce1_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+#if UINT16_MAX < SIZE_MAX
+  if (newlen > UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->auth_key.allocated_,
+                 &inp->auth_key.n_, inp->auth_key.elts_, newlen,
+                 sizeof(inp->auth_key.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->auth_key.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+struct trn_cell_extension_st *
+trn_cell_introduce1_get_extensions(trn_cell_introduce1_t *inp)
+{
+  return inp->extensions;
+}
+const struct trn_cell_extension_st *
+trn_cell_introduce1_getconst_extensions(const trn_cell_introduce1_t *inp)
+{
+  return trn_cell_introduce1_get_extensions((trn_cell_introduce1_t*) inp);
+}
+int
+trn_cell_introduce1_set_extensions(trn_cell_introduce1_t *inp, struct trn_cell_extension_st *val)
+{
+  if (inp->extensions && inp->extensions != val)
+    trn_cell_extension_free(inp->extensions);
+  return trn_cell_introduce1_set0_extensions(inp, val);
+}
+int
+trn_cell_introduce1_set0_extensions(trn_cell_introduce1_t *inp, struct trn_cell_extension_st *val)
+{
+  inp->extensions = val;
+  return 0;
+}
+size_t
+trn_cell_introduce1_getlen_encrypted(const trn_cell_introduce1_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->encrypted);
+}
+
+uint8_t
+trn_cell_introduce1_get_encrypted(trn_cell_introduce1_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->encrypted, idx);
+}
+
+uint8_t
+trn_cell_introduce1_getconst_encrypted(const trn_cell_introduce1_t *inp, size_t idx)
+{
+  return trn_cell_introduce1_get_encrypted((trn_cell_introduce1_t*)inp, idx);
+}
+int
+trn_cell_introduce1_set_encrypted(trn_cell_introduce1_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->encrypted, idx, elt);
+  return 0;
+}
+int
+trn_cell_introduce1_add_encrypted(trn_cell_introduce1_t *inp, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->encrypted, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_introduce1_getarray_encrypted(trn_cell_introduce1_t *inp)
+{
+  return inp->encrypted.elts_;
+}
+const uint8_t  *
+trn_cell_introduce1_getconstarray_encrypted(const trn_cell_introduce1_t *inp)
+{
+  return (const uint8_t  *)trn_cell_introduce1_getarray_encrypted((trn_cell_introduce1_t*)inp);
+}
+int
+trn_cell_introduce1_setlen_encrypted(trn_cell_introduce1_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+  newptr = trunnel_dynarray_setlen(&inp->encrypted.allocated_,
+                 &inp->encrypted.n_, inp->encrypted.elts_, newlen,
+                 sizeof(inp->encrypted.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->encrypted.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+trn_cell_introduce1_check(const trn_cell_introduce1_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->auth_key_type == TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_ED25519 || obj->auth_key_type == TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_LEGACY0 || obj->auth_key_type == TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_LEGACY1))
+    return "Integer out of bounds";
+  if (TRUNNEL_DYNARRAY_LEN(&obj->auth_key) != obj->auth_key_len)
+    return "Length mismatch for auth_key";
+  {
+    const char *msg;
+    if (NULL != (msg = trn_cell_extension_check(obj->extensions)))
+      return msg;
+  }
+  return NULL;
+}
+
+ssize_t
+trn_cell_introduce1_encoded_len(const trn_cell_introduce1_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_introduce1_check(obj))
+     return -1;
+
+
+  /* Length of u8 legacy_key_id[TRUNNEL_SHA1_LEN] */
+  result += TRUNNEL_SHA1_LEN;
+
+  /* Length of u8 auth_key_type IN [TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_ED25519, TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_LEGACY0, TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_LEGACY1] */
+  result += 1;
+
+  /* Length of u16 auth_key_len */
+  result += 2;
+
+  /* Length of u8 auth_key[auth_key_len] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->auth_key);
+
+  /* Length of struct trn_cell_extension extensions */
+  result += trn_cell_extension_encoded_len(obj->extensions);
+
+  /* Length of u8 encrypted[] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->encrypted);
+  return result;
+}
+int
+trn_cell_introduce1_clear_errors(trn_cell_introduce1_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_introduce1_encode(uint8_t *output, const size_t avail, const trn_cell_introduce1_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_introduce1_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_introduce1_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 legacy_key_id[TRUNNEL_SHA1_LEN] */
+  trunnel_assert(written <= avail);
+  if (avail - written < TRUNNEL_SHA1_LEN)
+    goto truncated;
+  memcpy(ptr, obj->legacy_key_id, TRUNNEL_SHA1_LEN);
+  written += TRUNNEL_SHA1_LEN; ptr += TRUNNEL_SHA1_LEN;
+
+  /* Encode u8 auth_key_type IN [TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_ED25519, TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_LEGACY0, TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_LEGACY1] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->auth_key_type));
+  written += 1; ptr += 1;
+
+  /* Encode u16 auth_key_len */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->auth_key_len));
+  written += 2; ptr += 2;
+
+  /* Encode u8 auth_key[auth_key_len] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->auth_key);
+    trunnel_assert(obj->auth_key_len == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->auth_key.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+  /* Encode struct trn_cell_extension extensions */
+  trunnel_assert(written <= avail);
+  result = trn_cell_extension_encode(ptr, avail - written, obj->extensions);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+  /* Encode u8 encrypted[] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->encrypted);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->encrypted.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_introduce1_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+trn_cell_introduce1_parse_into(trn_cell_introduce1_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 legacy_key_id[TRUNNEL_SHA1_LEN] */
+  CHECK_REMAINING(TRUNNEL_SHA1_LEN, truncated);
+  memcpy(obj->legacy_key_id, ptr, TRUNNEL_SHA1_LEN);
+  remaining -= TRUNNEL_SHA1_LEN; ptr += TRUNNEL_SHA1_LEN;
+
+  /* Parse u8 auth_key_type IN [TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_ED25519, TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_LEGACY0, TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_LEGACY1] */
+  CHECK_REMAINING(1, truncated);
+  obj->auth_key_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->auth_key_type == TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_ED25519 || obj->auth_key_type == TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_LEGACY0 || obj->auth_key_type == TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_LEGACY1))
+    goto fail;
+
+  /* Parse u16 auth_key_len */
+  CHECK_REMAINING(2, truncated);
+  obj->auth_key_len = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u8 auth_key[auth_key_len] */
+  CHECK_REMAINING(obj->auth_key_len, truncated);
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->auth_key, obj->auth_key_len, {});
+  obj->auth_key.n_ = obj->auth_key_len;
+  if (obj->auth_key_len)
+    memcpy(obj->auth_key.elts_, ptr, obj->auth_key_len);
+  ptr += obj->auth_key_len; remaining -= obj->auth_key_len;
+
+  /* Parse struct trn_cell_extension extensions */
+  result = trn_cell_extension_parse(&obj->extensions, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+
+  /* Parse u8 encrypted[] */
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->encrypted, remaining, {});
+  obj->encrypted.n_ = remaining;
+  if (remaining)
+    memcpy(obj->encrypted.elts_, ptr, remaining);
+  ptr += remaining; remaining -= remaining;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+trn_cell_introduce1_parse(trn_cell_introduce1_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_introduce1_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_introduce1_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_introduce1_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+trn_cell_introduce_ack_t *
+trn_cell_introduce_ack_new(void)
+{
+  trn_cell_introduce_ack_t *val = trunnel_calloc(1, sizeof(trn_cell_introduce_ack_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_introduce_ack_clear(trn_cell_introduce_ack_t *obj)
+{
+  (void) obj;
+  trn_cell_extension_free(obj->extensions);
+  obj->extensions = NULL;
+}
+
+void
+trn_cell_introduce_ack_free(trn_cell_introduce_ack_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_introduce_ack_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_introduce_ack_t));
+  trunnel_free_(obj);
+}
+
+uint16_t
+trn_cell_introduce_ack_get_status(const trn_cell_introduce_ack_t *inp)
+{
+  return inp->status;
+}
+int
+trn_cell_introduce_ack_set_status(trn_cell_introduce_ack_t *inp, uint16_t val)
+{
+  inp->status = val;
+  return 0;
+}
+struct trn_cell_extension_st *
+trn_cell_introduce_ack_get_extensions(trn_cell_introduce_ack_t *inp)
+{
+  return inp->extensions;
+}
+const struct trn_cell_extension_st *
+trn_cell_introduce_ack_getconst_extensions(const trn_cell_introduce_ack_t *inp)
+{
+  return trn_cell_introduce_ack_get_extensions((trn_cell_introduce_ack_t*) inp);
+}
+int
+trn_cell_introduce_ack_set_extensions(trn_cell_introduce_ack_t *inp, struct trn_cell_extension_st *val)
+{
+  if (inp->extensions && inp->extensions != val)
+    trn_cell_extension_free(inp->extensions);
+  return trn_cell_introduce_ack_set0_extensions(inp, val);
+}
+int
+trn_cell_introduce_ack_set0_extensions(trn_cell_introduce_ack_t *inp, struct trn_cell_extension_st *val)
+{
+  inp->extensions = val;
+  return 0;
+}
+const char *
+trn_cell_introduce_ack_check(const trn_cell_introduce_ack_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+    if (NULL != (msg = trn_cell_extension_check(obj->extensions)))
+      return msg;
+  }
+  return NULL;
+}
+
+ssize_t
+trn_cell_introduce_ack_encoded_len(const trn_cell_introduce_ack_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_introduce_ack_check(obj))
+     return -1;
+
+
+  /* Length of u16 status */
+  result += 2;
+
+  /* Length of struct trn_cell_extension extensions */
+  result += trn_cell_extension_encoded_len(obj->extensions);
+  return result;
+}
+int
+trn_cell_introduce_ack_clear_errors(trn_cell_introduce_ack_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_introduce_ack_encode(uint8_t *output, const size_t avail, const trn_cell_introduce_ack_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_introduce_ack_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_introduce_ack_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u16 status */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->status));
+  written += 2; ptr += 2;
+
+  /* Encode struct trn_cell_extension extensions */
+  trunnel_assert(written <= avail);
+  result = trn_cell_extension_encode(ptr, avail - written, obj->extensions);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_introduce_ack_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+trn_cell_introduce_ack_parse_into(trn_cell_introduce_ack_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u16 status */
+  CHECK_REMAINING(2, truncated);
+  obj->status = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse struct trn_cell_extension extensions */
+  result = trn_cell_extension_parse(&obj->extensions, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+ssize_t
+trn_cell_introduce_ack_parse(trn_cell_introduce_ack_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_introduce_ack_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_introduce_ack_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_introduce_ack_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+trn_cell_introduce_encrypted_t *
+trn_cell_introduce_encrypted_new(void)
+{
+  trn_cell_introduce_encrypted_t *val = trunnel_calloc(1, sizeof(trn_cell_introduce_encrypted_t));
+  if (NULL == val)
+    return NULL;
+  val->onion_key_type = TRUNNEL_HS_INTRO_ONION_KEY_TYPE_NTOR;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_introduce_encrypted_clear(trn_cell_introduce_encrypted_t *obj)
+{
+  (void) obj;
+  trn_cell_extension_free(obj->extensions);
+  obj->extensions = NULL;
+  TRUNNEL_DYNARRAY_WIPE(&obj->onion_key);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->onion_key);
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->nspecs); ++idx) {
+      link_specifier_free(TRUNNEL_DYNARRAY_GET(&obj->nspecs, idx));
+    }
+  }
+  TRUNNEL_DYNARRAY_WIPE(&obj->nspecs);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->nspecs);
+  TRUNNEL_DYNARRAY_WIPE(&obj->pad);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->pad);
+}
+
+void
+trn_cell_introduce_encrypted_free(trn_cell_introduce_encrypted_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_introduce_encrypted_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_introduce_encrypted_t));
+  trunnel_free_(obj);
+}
+
+size_t
+trn_cell_introduce_encrypted_getlen_rend_cookie(const trn_cell_introduce_encrypted_t *inp)
+{
+  (void)inp;  return TRUNNEL_REND_COOKIE_LEN;
+}
+
+uint8_t
+trn_cell_introduce_encrypted_get_rend_cookie(trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  trunnel_assert(idx < TRUNNEL_REND_COOKIE_LEN);
+  return inp->rend_cookie[idx];
+}
+
+uint8_t
+trn_cell_introduce_encrypted_getconst_rend_cookie(const trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  return trn_cell_introduce_encrypted_get_rend_cookie((trn_cell_introduce_encrypted_t*)inp, idx);
+}
+int
+trn_cell_introduce_encrypted_set_rend_cookie(trn_cell_introduce_encrypted_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < TRUNNEL_REND_COOKIE_LEN);
+  inp->rend_cookie[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+trn_cell_introduce_encrypted_getarray_rend_cookie(trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->rend_cookie;
+}
+const uint8_t  *
+trn_cell_introduce_encrypted_getconstarray_rend_cookie(const trn_cell_introduce_encrypted_t *inp)
+{
+  return (const uint8_t  *)trn_cell_introduce_encrypted_getarray_rend_cookie((trn_cell_introduce_encrypted_t*)inp);
+}
+struct trn_cell_extension_st *
+trn_cell_introduce_encrypted_get_extensions(trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->extensions;
+}
+const struct trn_cell_extension_st *
+trn_cell_introduce_encrypted_getconst_extensions(const trn_cell_introduce_encrypted_t *inp)
+{
+  return trn_cell_introduce_encrypted_get_extensions((trn_cell_introduce_encrypted_t*) inp);
+}
+int
+trn_cell_introduce_encrypted_set_extensions(trn_cell_introduce_encrypted_t *inp, struct trn_cell_extension_st *val)
+{
+  if (inp->extensions && inp->extensions != val)
+    trn_cell_extension_free(inp->extensions);
+  return trn_cell_introduce_encrypted_set0_extensions(inp, val);
+}
+int
+trn_cell_introduce_encrypted_set0_extensions(trn_cell_introduce_encrypted_t *inp, struct trn_cell_extension_st *val)
+{
+  inp->extensions = val;
+  return 0;
+}
+uint8_t
+trn_cell_introduce_encrypted_get_onion_key_type(const trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->onion_key_type;
+}
+int
+trn_cell_introduce_encrypted_set_onion_key_type(trn_cell_introduce_encrypted_t *inp, uint8_t val)
+{
+  if (! ((val == TRUNNEL_HS_INTRO_ONION_KEY_TYPE_NTOR))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->onion_key_type = val;
+  return 0;
+}
+uint16_t
+trn_cell_introduce_encrypted_get_onion_key_len(const trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->onion_key_len;
+}
+int
+trn_cell_introduce_encrypted_set_onion_key_len(trn_cell_introduce_encrypted_t *inp, uint16_t val)
+{
+  inp->onion_key_len = val;
+  return 0;
+}
+size_t
+trn_cell_introduce_encrypted_getlen_onion_key(const trn_cell_introduce_encrypted_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->onion_key);
+}
+
+uint8_t
+trn_cell_introduce_encrypted_get_onion_key(trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->onion_key, idx);
+}
+
+uint8_t
+trn_cell_introduce_encrypted_getconst_onion_key(const trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  return trn_cell_introduce_encrypted_get_onion_key((trn_cell_introduce_encrypted_t*)inp, idx);
+}
+int
+trn_cell_introduce_encrypted_set_onion_key(trn_cell_introduce_encrypted_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->onion_key, idx, elt);
+  return 0;
+}
+int
+trn_cell_introduce_encrypted_add_onion_key(trn_cell_introduce_encrypted_t *inp, uint8_t elt)
+{
+#if SIZE_MAX >= UINT16_MAX
+  if (inp->onion_key.n_ == UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->onion_key, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_introduce_encrypted_getarray_onion_key(trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->onion_key.elts_;
+}
+const uint8_t  *
+trn_cell_introduce_encrypted_getconstarray_onion_key(const trn_cell_introduce_encrypted_t *inp)
+{
+  return (const uint8_t  *)trn_cell_introduce_encrypted_getarray_onion_key((trn_cell_introduce_encrypted_t*)inp);
+}
+int
+trn_cell_introduce_encrypted_setlen_onion_key(trn_cell_introduce_encrypted_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+#if UINT16_MAX < SIZE_MAX
+  if (newlen > UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->onion_key.allocated_,
+                 &inp->onion_key.n_, inp->onion_key.elts_, newlen,
+                 sizeof(inp->onion_key.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->onion_key.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+uint8_t
+trn_cell_introduce_encrypted_get_nspec(const trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->nspec;
+}
+int
+trn_cell_introduce_encrypted_set_nspec(trn_cell_introduce_encrypted_t *inp, uint8_t val)
+{
+  inp->nspec = val;
+  return 0;
+}
+size_t
+trn_cell_introduce_encrypted_getlen_nspecs(const trn_cell_introduce_encrypted_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->nspecs);
+}
+
+struct link_specifier_st *
+trn_cell_introduce_encrypted_get_nspecs(trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->nspecs, idx);
+}
+
+ const struct link_specifier_st *
+trn_cell_introduce_encrypted_getconst_nspecs(const trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  return trn_cell_introduce_encrypted_get_nspecs((trn_cell_introduce_encrypted_t*)inp, idx);
+}
+int
+trn_cell_introduce_encrypted_set_nspecs(trn_cell_introduce_encrypted_t *inp, size_t idx, struct link_specifier_st * elt)
+{
+  link_specifier_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->nspecs, idx);
+  if (oldval && oldval != elt)
+    link_specifier_free(oldval);
+  return trn_cell_introduce_encrypted_set0_nspecs(inp, idx, elt);
+}
+int
+trn_cell_introduce_encrypted_set0_nspecs(trn_cell_introduce_encrypted_t *inp, size_t idx, struct link_specifier_st * elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->nspecs, idx, elt);
+  return 0;
+}
+int
+trn_cell_introduce_encrypted_add_nspecs(trn_cell_introduce_encrypted_t *inp, struct link_specifier_st * elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->nspecs.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(struct link_specifier_st *, &inp->nspecs, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+struct link_specifier_st * *
+trn_cell_introduce_encrypted_getarray_nspecs(trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->nspecs.elts_;
+}
+const struct link_specifier_st *  const  *
+trn_cell_introduce_encrypted_getconstarray_nspecs(const trn_cell_introduce_encrypted_t *inp)
+{
+  return (const struct link_specifier_st *  const  *)trn_cell_introduce_encrypted_getarray_nspecs((trn_cell_introduce_encrypted_t*)inp);
+}
+int
+trn_cell_introduce_encrypted_setlen_nspecs(trn_cell_introduce_encrypted_t *inp, size_t newlen)
+{
+  struct link_specifier_st * *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->nspecs.allocated_,
+                 &inp->nspecs.n_, inp->nspecs.elts_, newlen,
+                 sizeof(inp->nspecs.elts_[0]), (trunnel_free_fn_t) link_specifier_free,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->nspecs.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+size_t
+trn_cell_introduce_encrypted_getlen_pad(const trn_cell_introduce_encrypted_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->pad);
+}
+
+uint8_t
+trn_cell_introduce_encrypted_get_pad(trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->pad, idx);
+}
+
+uint8_t
+trn_cell_introduce_encrypted_getconst_pad(const trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  return trn_cell_introduce_encrypted_get_pad((trn_cell_introduce_encrypted_t*)inp, idx);
+}
+int
+trn_cell_introduce_encrypted_set_pad(trn_cell_introduce_encrypted_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->pad, idx, elt);
+  return 0;
+}
+int
+trn_cell_introduce_encrypted_add_pad(trn_cell_introduce_encrypted_t *inp, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->pad, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_introduce_encrypted_getarray_pad(trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->pad.elts_;
+}
+const uint8_t  *
+trn_cell_introduce_encrypted_getconstarray_pad(const trn_cell_introduce_encrypted_t *inp)
+{
+  return (const uint8_t  *)trn_cell_introduce_encrypted_getarray_pad((trn_cell_introduce_encrypted_t*)inp);
+}
+int
+trn_cell_introduce_encrypted_setlen_pad(trn_cell_introduce_encrypted_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+  newptr = trunnel_dynarray_setlen(&inp->pad.allocated_,
+                 &inp->pad.n_, inp->pad.elts_, newlen,
+                 sizeof(inp->pad.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->pad.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+trn_cell_introduce_encrypted_check(const trn_cell_introduce_encrypted_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+    if (NULL != (msg = trn_cell_extension_check(obj->extensions)))
+      return msg;
+  }
+  if (! (obj->onion_key_type == TRUNNEL_HS_INTRO_ONION_KEY_TYPE_NTOR))
+    return "Integer out of bounds";
+  if (TRUNNEL_DYNARRAY_LEN(&obj->onion_key) != obj->onion_key_len)
+    return "Length mismatch for onion_key";
+  {
+    const char *msg;
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->nspecs); ++idx) {
+      if (NULL != (msg = link_specifier_check(TRUNNEL_DYNARRAY_GET(&obj->nspecs, idx))))
+        return msg;
+    }
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->nspecs) != obj->nspec)
+    return "Length mismatch for nspecs";
+  return NULL;
+}
+
+ssize_t
+trn_cell_introduce_encrypted_encoded_len(const trn_cell_introduce_encrypted_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_introduce_encrypted_check(obj))
+     return -1;
+
+
+  /* Length of u8 rend_cookie[TRUNNEL_REND_COOKIE_LEN] */
+  result += TRUNNEL_REND_COOKIE_LEN;
+
+  /* Length of struct trn_cell_extension extensions */
+  result += trn_cell_extension_encoded_len(obj->extensions);
+
+  /* Length of u8 onion_key_type IN [TRUNNEL_HS_INTRO_ONION_KEY_TYPE_NTOR] */
+  result += 1;
+
+  /* Length of u16 onion_key_len */
+  result += 2;
+
+  /* Length of u8 onion_key[onion_key_len] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->onion_key);
+
+  /* Length of u8 nspec */
+  result += 1;
+
+  /* Length of struct link_specifier nspecs[nspec] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->nspecs); ++idx) {
+      result += link_specifier_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->nspecs, idx));
+    }
+  }
+
+  /* Length of u8 pad[] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->pad);
+  return result;
+}
+int
+trn_cell_introduce_encrypted_clear_errors(trn_cell_introduce_encrypted_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_introduce_encrypted_encode(uint8_t *output, const size_t avail, const trn_cell_introduce_encrypted_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_introduce_encrypted_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_introduce_encrypted_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 rend_cookie[TRUNNEL_REND_COOKIE_LEN] */
+  trunnel_assert(written <= avail);
+  if (avail - written < TRUNNEL_REND_COOKIE_LEN)
+    goto truncated;
+  memcpy(ptr, obj->rend_cookie, TRUNNEL_REND_COOKIE_LEN);
+  written += TRUNNEL_REND_COOKIE_LEN; ptr += TRUNNEL_REND_COOKIE_LEN;
+
+  /* Encode struct trn_cell_extension extensions */
+  trunnel_assert(written <= avail);
+  result = trn_cell_extension_encode(ptr, avail - written, obj->extensions);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+  /* Encode u8 onion_key_type IN [TRUNNEL_HS_INTRO_ONION_KEY_TYPE_NTOR] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->onion_key_type));
+  written += 1; ptr += 1;
+
+  /* Encode u16 onion_key_len */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->onion_key_len));
+  written += 2; ptr += 2;
+
+  /* Encode u8 onion_key[onion_key_len] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->onion_key);
+    trunnel_assert(obj->onion_key_len == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->onion_key.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+  /* Encode u8 nspec */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->nspec));
+  written += 1; ptr += 1;
+
+  /* Encode struct link_specifier nspecs[nspec] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->nspecs); ++idx) {
+      trunnel_assert(written <= avail);
+      result = link_specifier_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->nspecs, idx));
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+    }
+  }
+
+  /* Encode u8 pad[] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->pad);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->pad.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_introduce_encrypted_parse(), but do not allocate the
+ * output object.
+ */
+static ssize_t
+trn_cell_introduce_encrypted_parse_into(trn_cell_introduce_encrypted_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 rend_cookie[TRUNNEL_REND_COOKIE_LEN] */
+  CHECK_REMAINING(TRUNNEL_REND_COOKIE_LEN, truncated);
+  memcpy(obj->rend_cookie, ptr, TRUNNEL_REND_COOKIE_LEN);
+  remaining -= TRUNNEL_REND_COOKIE_LEN; ptr += TRUNNEL_REND_COOKIE_LEN;
+
+  /* Parse struct trn_cell_extension extensions */
+  result = trn_cell_extension_parse(&obj->extensions, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+
+  /* Parse u8 onion_key_type IN [TRUNNEL_HS_INTRO_ONION_KEY_TYPE_NTOR] */
+  CHECK_REMAINING(1, truncated);
+  obj->onion_key_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->onion_key_type == TRUNNEL_HS_INTRO_ONION_KEY_TYPE_NTOR))
+    goto fail;
+
+  /* Parse u16 onion_key_len */
+  CHECK_REMAINING(2, truncated);
+  obj->onion_key_len = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u8 onion_key[onion_key_len] */
+  CHECK_REMAINING(obj->onion_key_len, truncated);
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->onion_key, obj->onion_key_len, {});
+  obj->onion_key.n_ = obj->onion_key_len;
+  if (obj->onion_key_len)
+    memcpy(obj->onion_key.elts_, ptr, obj->onion_key_len);
+  ptr += obj->onion_key_len; remaining -= obj->onion_key_len;
+
+  /* Parse u8 nspec */
+  CHECK_REMAINING(1, truncated);
+  obj->nspec = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse struct link_specifier nspecs[nspec] */
+  TRUNNEL_DYNARRAY_EXPAND(link_specifier_t *, &obj->nspecs, obj->nspec, {});
+  {
+    link_specifier_t * elt;
+    unsigned idx;
+    for (idx = 0; idx < obj->nspec; ++idx) {
+      result = link_specifier_parse(&elt, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      TRUNNEL_DYNARRAY_ADD(link_specifier_t *, &obj->nspecs, elt, {link_specifier_free(elt);});
+    }
+  }
+
+  /* Parse u8 pad[] */
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->pad, remaining, {});
+  obj->pad.n_ = remaining;
+  if (remaining)
+    memcpy(obj->pad.elts_, ptr, remaining);
+  ptr += remaining; remaining -= remaining;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+trn_cell_introduce_encrypted_parse(trn_cell_introduce_encrypted_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_introduce_encrypted_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_introduce_encrypted_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_introduce_encrypted_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/hs/cell_introduce1.h b/src/trunnel/hs/cell_introduce1.h
new file mode 100644
index 0000000000..986a531ca7
--- /dev/null
+++ b/src/trunnel/hs/cell_introduce1.h
@@ -0,0 +1,500 @@
+/* cell_introduce1.h -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_CELL_INTRODUCE1_H
+#define TRUNNEL_CELL_INTRODUCE1_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+struct trn_cell_extension_st;
+struct link_specifier_st;
+#define TRUNNEL_SHA1_LEN 20
+#define TRUNNEL_REND_COOKIE_LEN 20
+#define TRUNNEL_HS_INTRO_ACK_STATUS_SUCCESS 0
+#define TRUNNEL_HS_INTRO_ACK_STATUS_UNKNOWN_ID 1
+#define TRUNNEL_HS_INTRO_ACK_STATUS_BAD_FORMAT 2
+#define TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_LEGACY0 0
+#define TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_LEGACY1 1
+#define TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_ED25519 2
+#define TRUNNEL_HS_INTRO_ONION_KEY_TYPE_NTOR 1
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_INTRODUCE1)
+struct trn_cell_introduce1_st {
+  uint8_t legacy_key_id[TRUNNEL_SHA1_LEN];
+  uint8_t auth_key_type;
+  uint16_t auth_key_len;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) auth_key;
+  struct trn_cell_extension_st *extensions;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) encrypted;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_introduce1_st trn_cell_introduce1_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_INTRODUCE_ACK)
+struct trn_cell_introduce_ack_st {
+  uint16_t status;
+  struct trn_cell_extension_st *extensions;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_introduce_ack_st trn_cell_introduce_ack_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_INTRODUCE_ENCRYPTED)
+struct trn_cell_introduce_encrypted_st {
+  uint8_t rend_cookie[TRUNNEL_REND_COOKIE_LEN];
+  struct trn_cell_extension_st *extensions;
+  uint8_t onion_key_type;
+  uint16_t onion_key_len;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) onion_key;
+  uint8_t nspec;
+  TRUNNEL_DYNARRAY_HEAD(, struct link_specifier_st *) nspecs;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) pad;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_introduce_encrypted_st trn_cell_introduce_encrypted_t;
+/** Return a newly allocated trn_cell_introduce1 with all elements set
+ * to zero.
+ */
+trn_cell_introduce1_t *trn_cell_introduce1_new(void);
+/** Release all storage held by the trn_cell_introduce1 in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_introduce1_free(trn_cell_introduce1_t *victim);
+/** Try to parse a trn_cell_introduce1 from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated trn_cell_introduce1_t. On failure, return -2 if the input
+ * appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t trn_cell_introduce1_parse(trn_cell_introduce1_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_introduce1 in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_introduce1_encoded_len(const trn_cell_introduce1_t *obj);
+/** Try to encode the trn_cell_introduce1 from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_introduce1_encode(uint8_t *output, size_t avail, const trn_cell_introduce1_t *input);
+/** Check whether the internal state of the trn_cell_introduce1 in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *trn_cell_introduce1_check(const trn_cell_introduce1_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_introduce1_clear_errors(trn_cell_introduce1_t *obj);
+/** Return the (constant) length of the array holding the
+ * legacy_key_id field of the trn_cell_introduce1_t in 'inp'.
+ */
+size_t trn_cell_introduce1_getlen_legacy_key_id(const trn_cell_introduce1_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * legacy_key_id of the trn_cell_introduce1_t in 'inp'.
+ */
+uint8_t trn_cell_introduce1_get_legacy_key_id(trn_cell_introduce1_t *inp, size_t idx);
+/** As trn_cell_introduce1_get_legacy_key_id, but take and return a
+ * const pointer
+ */
+uint8_t trn_cell_introduce1_getconst_legacy_key_id(const trn_cell_introduce1_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * legacy_key_id of the trn_cell_introduce1_t in 'inp', so that it
+ * will hold the value 'elt'.
+ */
+int trn_cell_introduce1_set_legacy_key_id(trn_cell_introduce1_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the TRUNNEL_SHA1_LEN-element array field
+ * legacy_key_id of 'inp'.
+ */
+uint8_t * trn_cell_introduce1_getarray_legacy_key_id(trn_cell_introduce1_t *inp);
+/** As trn_cell_introduce1_get_legacy_key_id, but take and return a
+ * const pointer
+ */
+const uint8_t  * trn_cell_introduce1_getconstarray_legacy_key_id(const trn_cell_introduce1_t *inp);
+/** Return the value of the auth_key_type field of the
+ * trn_cell_introduce1_t in 'inp'
+ */
+uint8_t trn_cell_introduce1_get_auth_key_type(const trn_cell_introduce1_t *inp);
+/** Set the value of the auth_key_type field of the
+ * trn_cell_introduce1_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce1_set_auth_key_type(trn_cell_introduce1_t *inp, uint8_t val);
+/** Return the value of the auth_key_len field of the
+ * trn_cell_introduce1_t in 'inp'
+ */
+uint16_t trn_cell_introduce1_get_auth_key_len(const trn_cell_introduce1_t *inp);
+/** Set the value of the auth_key_len field of the
+ * trn_cell_introduce1_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce1_set_auth_key_len(trn_cell_introduce1_t *inp, uint16_t val);
+/** Return the length of the dynamic array holding the auth_key field
+ * of the trn_cell_introduce1_t in 'inp'.
+ */
+size_t trn_cell_introduce1_getlen_auth_key(const trn_cell_introduce1_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * auth_key of the trn_cell_introduce1_t in 'inp'.
+ */
+uint8_t trn_cell_introduce1_get_auth_key(trn_cell_introduce1_t *inp, size_t idx);
+/** As trn_cell_introduce1_get_auth_key, but take and return a const
+ * pointer
+ */
+uint8_t trn_cell_introduce1_getconst_auth_key(const trn_cell_introduce1_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * auth_key of the trn_cell_introduce1_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int trn_cell_introduce1_set_auth_key(trn_cell_introduce1_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field auth_key of
+ * the trn_cell_introduce1_t in 'inp'.
+ */
+int trn_cell_introduce1_add_auth_key(trn_cell_introduce1_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field auth_key of
+ * 'inp'.
+ */
+uint8_t * trn_cell_introduce1_getarray_auth_key(trn_cell_introduce1_t *inp);
+/** As trn_cell_introduce1_get_auth_key, but take and return a const
+ * pointer
+ */
+const uint8_t  * trn_cell_introduce1_getconstarray_auth_key(const trn_cell_introduce1_t *inp);
+/** Change the length of the variable-length array field auth_key of
+ * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce1_setlen_auth_key(trn_cell_introduce1_t *inp, size_t newlen);
+/** Return the value of the extensions field of the
+ * trn_cell_introduce1_t in 'inp'
+ */
+struct trn_cell_extension_st * trn_cell_introduce1_get_extensions(trn_cell_introduce1_t *inp);
+/** As trn_cell_introduce1_get_extensions, but take and return a const
+ * pointer
+ */
+const struct trn_cell_extension_st * trn_cell_introduce1_getconst_extensions(const trn_cell_introduce1_t *inp);
+/** Set the value of the extensions field of the trn_cell_introduce1_t
+ * in 'inp' to 'val'. Free the old value if any. Steals the
+ * referenceto 'val'.Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int trn_cell_introduce1_set_extensions(trn_cell_introduce1_t *inp, struct trn_cell_extension_st *val);
+/** As trn_cell_introduce1_set_extensions, but does not free the
+ * previous value.
+ */
+int trn_cell_introduce1_set0_extensions(trn_cell_introduce1_t *inp, struct trn_cell_extension_st *val);
+/** Return the length of the dynamic array holding the encrypted field
+ * of the trn_cell_introduce1_t in 'inp'.
+ */
+size_t trn_cell_introduce1_getlen_encrypted(const trn_cell_introduce1_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * encrypted of the trn_cell_introduce1_t in 'inp'.
+ */
+uint8_t trn_cell_introduce1_get_encrypted(trn_cell_introduce1_t *inp, size_t idx);
+/** As trn_cell_introduce1_get_encrypted, but take and return a const
+ * pointer
+ */
+uint8_t trn_cell_introduce1_getconst_encrypted(const trn_cell_introduce1_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * encrypted of the trn_cell_introduce1_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int trn_cell_introduce1_set_encrypted(trn_cell_introduce1_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field encrypted of
+ * the trn_cell_introduce1_t in 'inp'.
+ */
+int trn_cell_introduce1_add_encrypted(trn_cell_introduce1_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field encrypted of
+ * 'inp'.
+ */
+uint8_t * trn_cell_introduce1_getarray_encrypted(trn_cell_introduce1_t *inp);
+/** As trn_cell_introduce1_get_encrypted, but take and return a const
+ * pointer
+ */
+const uint8_t  * trn_cell_introduce1_getconstarray_encrypted(const trn_cell_introduce1_t *inp);
+/** Change the length of the variable-length array field encrypted of
+ * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce1_setlen_encrypted(trn_cell_introduce1_t *inp, size_t newlen);
+/** Return a newly allocated trn_cell_introduce_ack with all elements
+ * set to zero.
+ */
+trn_cell_introduce_ack_t *trn_cell_introduce_ack_new(void);
+/** Release all storage held by the trn_cell_introduce_ack in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_introduce_ack_free(trn_cell_introduce_ack_t *victim);
+/** Try to parse a trn_cell_introduce_ack from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated trn_cell_introduce_ack_t. On failure, return -2 if the
+ * input appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t trn_cell_introduce_ack_parse(trn_cell_introduce_ack_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_introduce_ack in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_introduce_ack_encoded_len(const trn_cell_introduce_ack_t *obj);
+/** Try to encode the trn_cell_introduce_ack from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_introduce_ack_encode(uint8_t *output, size_t avail, const trn_cell_introduce_ack_t *input);
+/** Check whether the internal state of the trn_cell_introduce_ack in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *trn_cell_introduce_ack_check(const trn_cell_introduce_ack_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_introduce_ack_clear_errors(trn_cell_introduce_ack_t *obj);
+/** Return the value of the status field of the
+ * trn_cell_introduce_ack_t in 'inp'
+ */
+uint16_t trn_cell_introduce_ack_get_status(const trn_cell_introduce_ack_t *inp);
+/** Set the value of the status field of the trn_cell_introduce_ack_t
+ * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int trn_cell_introduce_ack_set_status(trn_cell_introduce_ack_t *inp, uint16_t val);
+/** Return the value of the extensions field of the
+ * trn_cell_introduce_ack_t in 'inp'
+ */
+struct trn_cell_extension_st * trn_cell_introduce_ack_get_extensions(trn_cell_introduce_ack_t *inp);
+/** As trn_cell_introduce_ack_get_extensions, but take and return a
+ * const pointer
+ */
+const struct trn_cell_extension_st * trn_cell_introduce_ack_getconst_extensions(const trn_cell_introduce_ack_t *inp);
+/** Set the value of the extensions field of the
+ * trn_cell_introduce_ack_t in 'inp' to 'val'. Free the old value if
+ * any. Steals the referenceto 'val'.Return 0 on success; return -1
+ * and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce_ack_set_extensions(trn_cell_introduce_ack_t *inp, struct trn_cell_extension_st *val);
+/** As trn_cell_introduce_ack_set_extensions, but does not free the
+ * previous value.
+ */
+int trn_cell_introduce_ack_set0_extensions(trn_cell_introduce_ack_t *inp, struct trn_cell_extension_st *val);
+/** Return a newly allocated trn_cell_introduce_encrypted with all
+ * elements set to zero.
+ */
+trn_cell_introduce_encrypted_t *trn_cell_introduce_encrypted_new(void);
+/** Release all storage held by the trn_cell_introduce_encrypted in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_introduce_encrypted_free(trn_cell_introduce_encrypted_t *victim);
+/** Try to parse a trn_cell_introduce_encrypted from the buffer in
+ * 'input', using up to 'len_in' bytes from the input buffer. On
+ * success, return the number of bytes consumed and set *output to the
+ * newly allocated trn_cell_introduce_encrypted_t. On failure, return
+ * -2 if the input appears truncated, and -1 if the input is otherwise
+ * invalid.
+ */
+ssize_t trn_cell_introduce_encrypted_parse(trn_cell_introduce_encrypted_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_introduce_encrypted in 'obj'. On failure, return a
+ * negative value. Note that this value may be an overestimate, and
+ * can even be an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_introduce_encrypted_encoded_len(const trn_cell_introduce_encrypted_t *obj);
+/** Try to encode the trn_cell_introduce_encrypted from 'input' into
+ * the buffer at 'output', using up to 'avail' bytes of the output
+ * buffer. On success, return the number of bytes used. On failure,
+ * return -2 if the buffer was not long enough, and -1 if the input
+ * was invalid.
+ */
+ssize_t trn_cell_introduce_encrypted_encode(uint8_t *output, size_t avail, const trn_cell_introduce_encrypted_t *input);
+/** Check whether the internal state of the
+ * trn_cell_introduce_encrypted in 'obj' is consistent. Return NULL if
+ * it is, and a short message if it is not.
+ */
+const char *trn_cell_introduce_encrypted_check(const trn_cell_introduce_encrypted_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_introduce_encrypted_clear_errors(trn_cell_introduce_encrypted_t *obj);
+/** Return the (constant) length of the array holding the rend_cookie
+ * field of the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+size_t trn_cell_introduce_encrypted_getlen_rend_cookie(const trn_cell_introduce_encrypted_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * rend_cookie of the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+uint8_t trn_cell_introduce_encrypted_get_rend_cookie(trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** As trn_cell_introduce_encrypted_get_rend_cookie, but take and
+ * return a const pointer
+ */
+uint8_t trn_cell_introduce_encrypted_getconst_rend_cookie(const trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * rend_cookie of the trn_cell_introduce_encrypted_t in 'inp', so that
+ * it will hold the value 'elt'.
+ */
+int trn_cell_introduce_encrypted_set_rend_cookie(trn_cell_introduce_encrypted_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the TRUNNEL_REND_COOKIE_LEN-element array
+ * field rend_cookie of 'inp'.
+ */
+uint8_t * trn_cell_introduce_encrypted_getarray_rend_cookie(trn_cell_introduce_encrypted_t *inp);
+/** As trn_cell_introduce_encrypted_get_rend_cookie, but take and
+ * return a const pointer
+ */
+const uint8_t  * trn_cell_introduce_encrypted_getconstarray_rend_cookie(const trn_cell_introduce_encrypted_t *inp);
+/** Return the value of the extensions field of the
+ * trn_cell_introduce_encrypted_t in 'inp'
+ */
+struct trn_cell_extension_st * trn_cell_introduce_encrypted_get_extensions(trn_cell_introduce_encrypted_t *inp);
+/** As trn_cell_introduce_encrypted_get_extensions, but take and
+ * return a const pointer
+ */
+const struct trn_cell_extension_st * trn_cell_introduce_encrypted_getconst_extensions(const trn_cell_introduce_encrypted_t *inp);
+/** Set the value of the extensions field of the
+ * trn_cell_introduce_encrypted_t in 'inp' to 'val'. Free the old
+ * value if any. Steals the referenceto 'val'.Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce_encrypted_set_extensions(trn_cell_introduce_encrypted_t *inp, struct trn_cell_extension_st *val);
+/** As trn_cell_introduce_encrypted_set_extensions, but does not free
+ * the previous value.
+ */
+int trn_cell_introduce_encrypted_set0_extensions(trn_cell_introduce_encrypted_t *inp, struct trn_cell_extension_st *val);
+/** Return the value of the onion_key_type field of the
+ * trn_cell_introduce_encrypted_t in 'inp'
+ */
+uint8_t trn_cell_introduce_encrypted_get_onion_key_type(const trn_cell_introduce_encrypted_t *inp);
+/** Set the value of the onion_key_type field of the
+ * trn_cell_introduce_encrypted_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce_encrypted_set_onion_key_type(trn_cell_introduce_encrypted_t *inp, uint8_t val);
+/** Return the value of the onion_key_len field of the
+ * trn_cell_introduce_encrypted_t in 'inp'
+ */
+uint16_t trn_cell_introduce_encrypted_get_onion_key_len(const trn_cell_introduce_encrypted_t *inp);
+/** Set the value of the onion_key_len field of the
+ * trn_cell_introduce_encrypted_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce_encrypted_set_onion_key_len(trn_cell_introduce_encrypted_t *inp, uint16_t val);
+/** Return the length of the dynamic array holding the onion_key field
+ * of the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+size_t trn_cell_introduce_encrypted_getlen_onion_key(const trn_cell_introduce_encrypted_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * onion_key of the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+uint8_t trn_cell_introduce_encrypted_get_onion_key(trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** As trn_cell_introduce_encrypted_get_onion_key, but take and return
+ * a const pointer
+ */
+uint8_t trn_cell_introduce_encrypted_getconst_onion_key(const trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * onion_key of the trn_cell_introduce_encrypted_t in 'inp', so that
+ * it will hold the value 'elt'.
+ */
+int trn_cell_introduce_encrypted_set_onion_key(trn_cell_introduce_encrypted_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field onion_key of
+ * the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+int trn_cell_introduce_encrypted_add_onion_key(trn_cell_introduce_encrypted_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field onion_key of
+ * 'inp'.
+ */
+uint8_t * trn_cell_introduce_encrypted_getarray_onion_key(trn_cell_introduce_encrypted_t *inp);
+/** As trn_cell_introduce_encrypted_get_onion_key, but take and return
+ * a const pointer
+ */
+const uint8_t  * trn_cell_introduce_encrypted_getconstarray_onion_key(const trn_cell_introduce_encrypted_t *inp);
+/** Change the length of the variable-length array field onion_key of
+ * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce_encrypted_setlen_onion_key(trn_cell_introduce_encrypted_t *inp, size_t newlen);
+/** Return the value of the nspec field of the
+ * trn_cell_introduce_encrypted_t in 'inp'
+ */
+uint8_t trn_cell_introduce_encrypted_get_nspec(const trn_cell_introduce_encrypted_t *inp);
+/** Set the value of the nspec field of the
+ * trn_cell_introduce_encrypted_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce_encrypted_set_nspec(trn_cell_introduce_encrypted_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the nspecs field of
+ * the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+size_t trn_cell_introduce_encrypted_getlen_nspecs(const trn_cell_introduce_encrypted_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * nspecs of the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+struct link_specifier_st * trn_cell_introduce_encrypted_get_nspecs(trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** As trn_cell_introduce_encrypted_get_nspecs, but take and return a
+ * const pointer
+ */
+ const struct link_specifier_st * trn_cell_introduce_encrypted_getconst_nspecs(const trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * nspecs of the trn_cell_introduce_encrypted_t in 'inp', so that it
+ * will hold the value 'elt'. Free the previous value, if any.
+ */
+int trn_cell_introduce_encrypted_set_nspecs(trn_cell_introduce_encrypted_t *inp, size_t idx, struct link_specifier_st * elt);
+/** As trn_cell_introduce_encrypted_set_nspecs, but does not free the
+ * previous value.
+ */
+int trn_cell_introduce_encrypted_set0_nspecs(trn_cell_introduce_encrypted_t *inp, size_t idx, struct link_specifier_st * elt);
+/** Append a new element 'elt' to the dynamic array field nspecs of
+ * the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+int trn_cell_introduce_encrypted_add_nspecs(trn_cell_introduce_encrypted_t *inp, struct link_specifier_st * elt);
+/** Return a pointer to the variable-length array field nspecs of
+ * 'inp'.
+ */
+struct link_specifier_st * * trn_cell_introduce_encrypted_getarray_nspecs(trn_cell_introduce_encrypted_t *inp);
+/** As trn_cell_introduce_encrypted_get_nspecs, but take and return a
+ * const pointer
+ */
+const struct link_specifier_st *  const  * trn_cell_introduce_encrypted_getconstarray_nspecs(const trn_cell_introduce_encrypted_t *inp);
+/** Change the length of the variable-length array field nspecs of
+ * 'inp' to 'newlen'.Fill extra elements with NULL; free removed
+ * elements. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int trn_cell_introduce_encrypted_setlen_nspecs(trn_cell_introduce_encrypted_t *inp, size_t newlen);
+/** Return the length of the dynamic array holding the pad field of
+ * the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+size_t trn_cell_introduce_encrypted_getlen_pad(const trn_cell_introduce_encrypted_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * pad of the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+uint8_t trn_cell_introduce_encrypted_get_pad(trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** As trn_cell_introduce_encrypted_get_pad, but take and return a
+ * const pointer
+ */
+uint8_t trn_cell_introduce_encrypted_getconst_pad(const trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * pad of the trn_cell_introduce_encrypted_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int trn_cell_introduce_encrypted_set_pad(trn_cell_introduce_encrypted_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field pad of the
+ * trn_cell_introduce_encrypted_t in 'inp'.
+ */
+int trn_cell_introduce_encrypted_add_pad(trn_cell_introduce_encrypted_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field pad of 'inp'.
+ */
+uint8_t * trn_cell_introduce_encrypted_getarray_pad(trn_cell_introduce_encrypted_t *inp);
+/** As trn_cell_introduce_encrypted_get_pad, but take and return a
+ * const pointer
+ */
+const uint8_t  * trn_cell_introduce_encrypted_getconstarray_pad(const trn_cell_introduce_encrypted_t *inp);
+/** Change the length of the variable-length array field pad of 'inp'
+ * to 'newlen'.Fill extra elements with 0. Return 0 on success; return
+ * -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce_encrypted_setlen_pad(trn_cell_introduce_encrypted_t *inp, size_t newlen);
+
+
+#endif
diff --git a/src/trunnel/hs/cell_introduce1.trunnel b/src/trunnel/hs/cell_introduce1.trunnel
new file mode 100644
index 0000000000..5911c695a2
--- /dev/null
+++ b/src/trunnel/hs/cell_introduce1.trunnel
@@ -0,0 +1,75 @@
+/*
+ * This contains the definition of the INTRODUCE1 and INTRODUCE_ACK cell for
+ * onion service version 3 and onward. The following format is specified in
+ * proposal 224 section 3.2.
+ */
+
+/* From cell_common.trunnel. */
+extern struct trn_cell_extension;
+/* From ed25519_cert.trunnel. */
+extern struct link_specifier;
+
+const TRUNNEL_SHA1_LEN = 20;
+const TRUNNEL_REND_COOKIE_LEN = 20;
+
+/* Introduce ACK status code. */
+const TRUNNEL_HS_INTRO_ACK_STATUS_SUCCESS    = 0x0000;
+const TRUNNEL_HS_INTRO_ACK_STATUS_UNKNOWN_ID = 0x0001;
+const TRUNNEL_HS_INTRO_ACK_STATUS_BAD_FORMAT = 0x0002;
+
+/* Authentication key type. */
+const TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_LEGACY0 = 0x00;
+const TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_LEGACY1 = 0x01;
+const TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_ED25519 = 0x02;
+
+/* Onion key type. */
+const TRUNNEL_HS_INTRO_ONION_KEY_TYPE_NTOR = 0x01;
+
+/* INTRODUCE1 payload. See details in section 3.2.1. */
+struct trn_cell_introduce1 {
+  /* Always zeroed. MUST be checked explicitly by the caller. */
+  u8 legacy_key_id[TRUNNEL_SHA1_LEN];
+
+  /* Authentication key material. */
+  u8 auth_key_type IN [TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_LEGACY0,
+                       TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_LEGACY1,
+                       TRUNNEL_HS_INTRO_AUTH_KEY_TYPE_ED25519];
+  u16 auth_key_len;
+  u8 auth_key[auth_key_len];
+
+  /* Extension(s). Reserved fields. */
+  struct trn_cell_extension extensions;
+
+  /* Variable length, up to the end of cell. */
+  u8 encrypted[];
+};
+
+/* INTRODUCE_ACK payload. See details in section 3.2.2. */
+struct trn_cell_introduce_ack {
+  /* Status of introduction. */
+  u16 status;
+
+  /* Extension(s). Reserved fields. */
+  struct trn_cell_extension extensions;
+};
+
+/* Encrypted section of the INTRODUCE1/INTRODUCE2 cell. */
+struct trn_cell_introduce_encrypted {
+  /* Rendezvous cookie. */
+  u8 rend_cookie[TRUNNEL_REND_COOKIE_LEN];
+
+  /* Extension(s). Reserved fields. */
+  struct trn_cell_extension extensions;
+
+  /* Onion key material. */
+  u8 onion_key_type IN [TRUNNEL_HS_INTRO_ONION_KEY_TYPE_NTOR];
+  u16 onion_key_len;
+  u8 onion_key[onion_key_len];
+
+  /* Link specifiers(s) */
+  u8 nspec;
+  struct link_specifier nspecs[nspec];
+
+  /* Optional padding. This might be empty or not. */
+  u8 pad[];
+};
diff --git a/src/trunnel/hs/cell_rendezvous.c b/src/trunnel/hs/cell_rendezvous.c
new file mode 100644
index 0000000000..53cb609138
--- /dev/null
+++ b/src/trunnel/hs/cell_rendezvous.c
@@ -0,0 +1,470 @@
+/* cell_rendezvous.c -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "cell_rendezvous.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int cellrendezvous_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || cellrendezvous_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+trn_cell_rendezvous1_t *
+trn_cell_rendezvous1_new(void)
+{
+  trn_cell_rendezvous1_t *val = trunnel_calloc(1, sizeof(trn_cell_rendezvous1_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_rendezvous1_clear(trn_cell_rendezvous1_t *obj)
+{
+  (void) obj;
+  TRUNNEL_DYNARRAY_WIPE(&obj->handshake_info);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->handshake_info);
+}
+
+void
+trn_cell_rendezvous1_free(trn_cell_rendezvous1_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_rendezvous1_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_rendezvous1_t));
+  trunnel_free_(obj);
+}
+
+size_t
+trn_cell_rendezvous1_getlen_rendezvous_cookie(const trn_cell_rendezvous1_t *inp)
+{
+  (void)inp;  return TRUNNEL_REND_COOKIE_LEN;
+}
+
+uint8_t
+trn_cell_rendezvous1_get_rendezvous_cookie(trn_cell_rendezvous1_t *inp, size_t idx)
+{
+  trunnel_assert(idx < TRUNNEL_REND_COOKIE_LEN);
+  return inp->rendezvous_cookie[idx];
+}
+
+uint8_t
+trn_cell_rendezvous1_getconst_rendezvous_cookie(const trn_cell_rendezvous1_t *inp, size_t idx)
+{
+  return trn_cell_rendezvous1_get_rendezvous_cookie((trn_cell_rendezvous1_t*)inp, idx);
+}
+int
+trn_cell_rendezvous1_set_rendezvous_cookie(trn_cell_rendezvous1_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < TRUNNEL_REND_COOKIE_LEN);
+  inp->rendezvous_cookie[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+trn_cell_rendezvous1_getarray_rendezvous_cookie(trn_cell_rendezvous1_t *inp)
+{
+  return inp->rendezvous_cookie;
+}
+const uint8_t  *
+trn_cell_rendezvous1_getconstarray_rendezvous_cookie(const trn_cell_rendezvous1_t *inp)
+{
+  return (const uint8_t  *)trn_cell_rendezvous1_getarray_rendezvous_cookie((trn_cell_rendezvous1_t*)inp);
+}
+size_t
+trn_cell_rendezvous1_getlen_handshake_info(const trn_cell_rendezvous1_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->handshake_info);
+}
+
+uint8_t
+trn_cell_rendezvous1_get_handshake_info(trn_cell_rendezvous1_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->handshake_info, idx);
+}
+
+uint8_t
+trn_cell_rendezvous1_getconst_handshake_info(const trn_cell_rendezvous1_t *inp, size_t idx)
+{
+  return trn_cell_rendezvous1_get_handshake_info((trn_cell_rendezvous1_t*)inp, idx);
+}
+int
+trn_cell_rendezvous1_set_handshake_info(trn_cell_rendezvous1_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->handshake_info, idx, elt);
+  return 0;
+}
+int
+trn_cell_rendezvous1_add_handshake_info(trn_cell_rendezvous1_t *inp, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->handshake_info, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_rendezvous1_getarray_handshake_info(trn_cell_rendezvous1_t *inp)
+{
+  return inp->handshake_info.elts_;
+}
+const uint8_t  *
+trn_cell_rendezvous1_getconstarray_handshake_info(const trn_cell_rendezvous1_t *inp)
+{
+  return (const uint8_t  *)trn_cell_rendezvous1_getarray_handshake_info((trn_cell_rendezvous1_t*)inp);
+}
+int
+trn_cell_rendezvous1_setlen_handshake_info(trn_cell_rendezvous1_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+  newptr = trunnel_dynarray_setlen(&inp->handshake_info.allocated_,
+                 &inp->handshake_info.n_, inp->handshake_info.elts_, newlen,
+                 sizeof(inp->handshake_info.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->handshake_info.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+trn_cell_rendezvous1_check(const trn_cell_rendezvous1_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  return NULL;
+}
+
+ssize_t
+trn_cell_rendezvous1_encoded_len(const trn_cell_rendezvous1_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_rendezvous1_check(obj))
+     return -1;
+
+
+  /* Length of u8 rendezvous_cookie[TRUNNEL_REND_COOKIE_LEN] */
+  result += TRUNNEL_REND_COOKIE_LEN;
+
+  /* Length of u8 handshake_info[] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->handshake_info);
+  return result;
+}
+int
+trn_cell_rendezvous1_clear_errors(trn_cell_rendezvous1_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_rendezvous1_encode(uint8_t *output, const size_t avail, const trn_cell_rendezvous1_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_rendezvous1_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_rendezvous1_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 rendezvous_cookie[TRUNNEL_REND_COOKIE_LEN] */
+  trunnel_assert(written <= avail);
+  if (avail - written < TRUNNEL_REND_COOKIE_LEN)
+    goto truncated;
+  memcpy(ptr, obj->rendezvous_cookie, TRUNNEL_REND_COOKIE_LEN);
+  written += TRUNNEL_REND_COOKIE_LEN; ptr += TRUNNEL_REND_COOKIE_LEN;
+
+  /* Encode u8 handshake_info[] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->handshake_info);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->handshake_info.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_rendezvous1_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+trn_cell_rendezvous1_parse_into(trn_cell_rendezvous1_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 rendezvous_cookie[TRUNNEL_REND_COOKIE_LEN] */
+  CHECK_REMAINING(TRUNNEL_REND_COOKIE_LEN, truncated);
+  memcpy(obj->rendezvous_cookie, ptr, TRUNNEL_REND_COOKIE_LEN);
+  remaining -= TRUNNEL_REND_COOKIE_LEN; ptr += TRUNNEL_REND_COOKIE_LEN;
+
+  /* Parse u8 handshake_info[] */
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->handshake_info, remaining, {});
+  obj->handshake_info.n_ = remaining;
+  if (remaining)
+    memcpy(obj->handshake_info.elts_, ptr, remaining);
+  ptr += remaining; remaining -= remaining;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+trn_cell_rendezvous1_parse(trn_cell_rendezvous1_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_rendezvous1_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_rendezvous1_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_rendezvous1_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+trn_cell_rendezvous2_t *
+trn_cell_rendezvous2_new(void)
+{
+  trn_cell_rendezvous2_t *val = trunnel_calloc(1, sizeof(trn_cell_rendezvous2_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_rendezvous2_clear(trn_cell_rendezvous2_t *obj)
+{
+  (void) obj;
+}
+
+void
+trn_cell_rendezvous2_free(trn_cell_rendezvous2_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_rendezvous2_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_rendezvous2_t));
+  trunnel_free_(obj);
+}
+
+size_t
+trn_cell_rendezvous2_getlen_handshake_info(const trn_cell_rendezvous2_t *inp)
+{
+  (void)inp;  return TRUNNEL_HANDSHAKE_INFO_LEN;
+}
+
+uint8_t
+trn_cell_rendezvous2_get_handshake_info(trn_cell_rendezvous2_t *inp, size_t idx)
+{
+  trunnel_assert(idx < TRUNNEL_HANDSHAKE_INFO_LEN);
+  return inp->handshake_info[idx];
+}
+
+uint8_t
+trn_cell_rendezvous2_getconst_handshake_info(const trn_cell_rendezvous2_t *inp, size_t idx)
+{
+  return trn_cell_rendezvous2_get_handshake_info((trn_cell_rendezvous2_t*)inp, idx);
+}
+int
+trn_cell_rendezvous2_set_handshake_info(trn_cell_rendezvous2_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < TRUNNEL_HANDSHAKE_INFO_LEN);
+  inp->handshake_info[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+trn_cell_rendezvous2_getarray_handshake_info(trn_cell_rendezvous2_t *inp)
+{
+  return inp->handshake_info;
+}
+const uint8_t  *
+trn_cell_rendezvous2_getconstarray_handshake_info(const trn_cell_rendezvous2_t *inp)
+{
+  return (const uint8_t  *)trn_cell_rendezvous2_getarray_handshake_info((trn_cell_rendezvous2_t*)inp);
+}
+const char *
+trn_cell_rendezvous2_check(const trn_cell_rendezvous2_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  return NULL;
+}
+
+ssize_t
+trn_cell_rendezvous2_encoded_len(const trn_cell_rendezvous2_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_rendezvous2_check(obj))
+     return -1;
+
+
+  /* Length of u8 handshake_info[TRUNNEL_HANDSHAKE_INFO_LEN] */
+  result += TRUNNEL_HANDSHAKE_INFO_LEN;
+  return result;
+}
+int
+trn_cell_rendezvous2_clear_errors(trn_cell_rendezvous2_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_rendezvous2_encode(uint8_t *output, const size_t avail, const trn_cell_rendezvous2_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_rendezvous2_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_rendezvous2_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 handshake_info[TRUNNEL_HANDSHAKE_INFO_LEN] */
+  trunnel_assert(written <= avail);
+  if (avail - written < TRUNNEL_HANDSHAKE_INFO_LEN)
+    goto truncated;
+  memcpy(ptr, obj->handshake_info, TRUNNEL_HANDSHAKE_INFO_LEN);
+  written += TRUNNEL_HANDSHAKE_INFO_LEN; ptr += TRUNNEL_HANDSHAKE_INFO_LEN;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_rendezvous2_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+trn_cell_rendezvous2_parse_into(trn_cell_rendezvous2_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 handshake_info[TRUNNEL_HANDSHAKE_INFO_LEN] */
+  CHECK_REMAINING(TRUNNEL_HANDSHAKE_INFO_LEN, truncated);
+  memcpy(obj->handshake_info, ptr, TRUNNEL_HANDSHAKE_INFO_LEN);
+  remaining -= TRUNNEL_HANDSHAKE_INFO_LEN; ptr += TRUNNEL_HANDSHAKE_INFO_LEN;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+}
+
+ssize_t
+trn_cell_rendezvous2_parse(trn_cell_rendezvous2_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_rendezvous2_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_rendezvous2_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_rendezvous2_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/hs/cell_rendezvous.h b/src/trunnel/hs/cell_rendezvous.h
new file mode 100644
index 0000000000..39e14da25b
--- /dev/null
+++ b/src/trunnel/hs/cell_rendezvous.h
@@ -0,0 +1,187 @@
+/* cell_rendezvous.h -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_CELL_RENDEZVOUS_H
+#define TRUNNEL_CELL_RENDEZVOUS_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+#define TRUNNEL_REND_COOKIE_LEN 20
+#define TRUNNEL_HANDSHAKE_INFO_LEN 64
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_RENDEZVOUS1)
+struct trn_cell_rendezvous1_st {
+  uint8_t rendezvous_cookie[TRUNNEL_REND_COOKIE_LEN];
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) handshake_info;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_rendezvous1_st trn_cell_rendezvous1_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_RENDEZVOUS2)
+struct trn_cell_rendezvous2_st {
+  uint8_t handshake_info[TRUNNEL_HANDSHAKE_INFO_LEN];
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_rendezvous2_st trn_cell_rendezvous2_t;
+/** Return a newly allocated trn_cell_rendezvous1 with all elements
+ * set to zero.
+ */
+trn_cell_rendezvous1_t *trn_cell_rendezvous1_new(void);
+/** Release all storage held by the trn_cell_rendezvous1 in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_rendezvous1_free(trn_cell_rendezvous1_t *victim);
+/** Try to parse a trn_cell_rendezvous1 from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated trn_cell_rendezvous1_t. On failure, return -2 if the
+ * input appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t trn_cell_rendezvous1_parse(trn_cell_rendezvous1_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_rendezvous1 in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_rendezvous1_encoded_len(const trn_cell_rendezvous1_t *obj);
+/** Try to encode the trn_cell_rendezvous1 from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_rendezvous1_encode(uint8_t *output, size_t avail, const trn_cell_rendezvous1_t *input);
+/** Check whether the internal state of the trn_cell_rendezvous1 in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *trn_cell_rendezvous1_check(const trn_cell_rendezvous1_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_rendezvous1_clear_errors(trn_cell_rendezvous1_t *obj);
+/** Return the (constant) length of the array holding the
+ * rendezvous_cookie field of the trn_cell_rendezvous1_t in 'inp'.
+ */
+size_t trn_cell_rendezvous1_getlen_rendezvous_cookie(const trn_cell_rendezvous1_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * rendezvous_cookie of the trn_cell_rendezvous1_t in 'inp'.
+ */
+uint8_t trn_cell_rendezvous1_get_rendezvous_cookie(trn_cell_rendezvous1_t *inp, size_t idx);
+/** As trn_cell_rendezvous1_get_rendezvous_cookie, but take and return
+ * a const pointer
+ */
+uint8_t trn_cell_rendezvous1_getconst_rendezvous_cookie(const trn_cell_rendezvous1_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * rendezvous_cookie of the trn_cell_rendezvous1_t in 'inp', so that
+ * it will hold the value 'elt'.
+ */
+int trn_cell_rendezvous1_set_rendezvous_cookie(trn_cell_rendezvous1_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the TRUNNEL_REND_COOKIE_LEN-element array
+ * field rendezvous_cookie of 'inp'.
+ */
+uint8_t * trn_cell_rendezvous1_getarray_rendezvous_cookie(trn_cell_rendezvous1_t *inp);
+/** As trn_cell_rendezvous1_get_rendezvous_cookie, but take and return
+ * a const pointer
+ */
+const uint8_t  * trn_cell_rendezvous1_getconstarray_rendezvous_cookie(const trn_cell_rendezvous1_t *inp);
+/** Return the length of the dynamic array holding the handshake_info
+ * field of the trn_cell_rendezvous1_t in 'inp'.
+ */
+size_t trn_cell_rendezvous1_getlen_handshake_info(const trn_cell_rendezvous1_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * handshake_info of the trn_cell_rendezvous1_t in 'inp'.
+ */
+uint8_t trn_cell_rendezvous1_get_handshake_info(trn_cell_rendezvous1_t *inp, size_t idx);
+/** As trn_cell_rendezvous1_get_handshake_info, but take and return a
+ * const pointer
+ */
+uint8_t trn_cell_rendezvous1_getconst_handshake_info(const trn_cell_rendezvous1_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * handshake_info of the trn_cell_rendezvous1_t in 'inp', so that it
+ * will hold the value 'elt'.
+ */
+int trn_cell_rendezvous1_set_handshake_info(trn_cell_rendezvous1_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field
+ * handshake_info of the trn_cell_rendezvous1_t in 'inp'.
+ */
+int trn_cell_rendezvous1_add_handshake_info(trn_cell_rendezvous1_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field handshake_info
+ * of 'inp'.
+ */
+uint8_t * trn_cell_rendezvous1_getarray_handshake_info(trn_cell_rendezvous1_t *inp);
+/** As trn_cell_rendezvous1_get_handshake_info, but take and return a
+ * const pointer
+ */
+const uint8_t  * trn_cell_rendezvous1_getconstarray_handshake_info(const trn_cell_rendezvous1_t *inp);
+/** Change the length of the variable-length array field
+ * handshake_info of 'inp' to 'newlen'.Fill extra elements with 0.
+ * Return 0 on success; return -1 and set the error code on 'inp' on
+ * failure.
+ */
+int trn_cell_rendezvous1_setlen_handshake_info(trn_cell_rendezvous1_t *inp, size_t newlen);
+/** Return a newly allocated trn_cell_rendezvous2 with all elements
+ * set to zero.
+ */
+trn_cell_rendezvous2_t *trn_cell_rendezvous2_new(void);
+/** Release all storage held by the trn_cell_rendezvous2 in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_rendezvous2_free(trn_cell_rendezvous2_t *victim);
+/** Try to parse a trn_cell_rendezvous2 from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated trn_cell_rendezvous2_t. On failure, return -2 if the
+ * input appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t trn_cell_rendezvous2_parse(trn_cell_rendezvous2_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_rendezvous2 in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_rendezvous2_encoded_len(const trn_cell_rendezvous2_t *obj);
+/** Try to encode the trn_cell_rendezvous2 from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_rendezvous2_encode(uint8_t *output, size_t avail, const trn_cell_rendezvous2_t *input);
+/** Check whether the internal state of the trn_cell_rendezvous2 in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *trn_cell_rendezvous2_check(const trn_cell_rendezvous2_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_rendezvous2_clear_errors(trn_cell_rendezvous2_t *obj);
+/** Return the (constant) length of the array holding the
+ * handshake_info field of the trn_cell_rendezvous2_t in 'inp'.
+ */
+size_t trn_cell_rendezvous2_getlen_handshake_info(const trn_cell_rendezvous2_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * handshake_info of the trn_cell_rendezvous2_t in 'inp'.
+ */
+uint8_t trn_cell_rendezvous2_get_handshake_info(trn_cell_rendezvous2_t *inp, size_t idx);
+/** As trn_cell_rendezvous2_get_handshake_info, but take and return a
+ * const pointer
+ */
+uint8_t trn_cell_rendezvous2_getconst_handshake_info(const trn_cell_rendezvous2_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * handshake_info of the trn_cell_rendezvous2_t in 'inp', so that it
+ * will hold the value 'elt'.
+ */
+int trn_cell_rendezvous2_set_handshake_info(trn_cell_rendezvous2_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the TRUNNEL_HANDSHAKE_INFO_LEN-element array
+ * field handshake_info of 'inp'.
+ */
+uint8_t * trn_cell_rendezvous2_getarray_handshake_info(trn_cell_rendezvous2_t *inp);
+/** As trn_cell_rendezvous2_get_handshake_info, but take and return a
+ * const pointer
+ */
+const uint8_t  * trn_cell_rendezvous2_getconstarray_handshake_info(const trn_cell_rendezvous2_t *inp);
+
+
+#endif
diff --git a/src/trunnel/hs/cell_rendezvous.trunnel b/src/trunnel/hs/cell_rendezvous.trunnel
new file mode 100644
index 0000000000..2128b4d126
--- /dev/null
+++ b/src/trunnel/hs/cell_rendezvous.trunnel
@@ -0,0 +1,29 @@
+/*
+ * This contains the definition of the RENDEZVOUS1/2 cell for onion service
+ * version 3 and onward. The following format is specified in proposal 224
+ * section 4.2.
+ */
+
+/* Rendezvous cookie length. */
+const TRUNNEL_REND_COOKIE_LEN = 20;
+/* The HANDSHAKE_INFO field layout is as follow:
+ *    SERVER_PK   [PK_PUBKEY_LEN bytes]
+ *    AUTH        [MAC_LEN bytes]
+ * This means, the size is 32 bytes + 32 bytes. */
+const TRUNNEL_HANDSHAKE_INFO_LEN = 64;
+
+/* RENDEZVOUS1 payload. See details in section 4.2. */
+struct trn_cell_rendezvous1 {
+  /* The RENDEZVOUS_COOKIE field. */
+  u8 rendezvous_cookie[TRUNNEL_REND_COOKIE_LEN];
+  
+  /* The HANDSHAKE_INFO field which has a variable length depending on the
+   * handshake type used. */
+  u8 handshake_info[];
+};
+
+/* RENDEZVOUS2 payload. See details in section 4.2. */
+struct trn_cell_rendezvous2 {
+  /* The HANDSHAKE_INFO field. */
+  u8 handshake_info[TRUNNEL_HANDSHAKE_INFO_LEN];
+};
diff --git a/src/trunnel/include.am b/src/trunnel/include.am
index b1448b7cb2..03c1753e96 100644
--- a/src/trunnel/include.am
+++ b/src/trunnel/include.am
@@ -1,4 +1,3 @@
-
 noinst_LIBRARIES += \
 	src/trunnel/libor-trunnel.a
 
@@ -7,31 +6,49 @@ noinst_LIBRARIES += \
 	src/trunnel/libor-trunnel-testing.a
 endif
 
-AM_CPPFLAGS += -I$(srcdir)/src/ext/trunnel -I$(srcdir)/src/trunnel
-
 TRUNNELINPUTS = \
 	src/trunnel/ed25519_cert.trunnel \
 	src/trunnel/link_handshake.trunnel \
-	src/trunnel/pwbox.trunnel
+	src/trunnel/pwbox.trunnel \
+	src/trunnel/channelpadding_negotiation.trunnel \
+	src/trunner/socks5.trunnel
 
 TRUNNELSOURCES = \
 	src/ext/trunnel/trunnel.c \
 	src/trunnel/ed25519_cert.c \
 	src/trunnel/link_handshake.c \
-	src/trunnel/pwbox.c
+	src/trunnel/pwbox.c			\
+	src/trunnel/hs/cell_common.c            \
+	src/trunnel/hs/cell_establish_intro.c	\
+	src/trunnel/hs/cell_introduce1.c \
+	src/trunnel/hs/cell_rendezvous.c \
+	src/trunnel/channelpadding_negotiation.c \
+	src/trunnel/socks5.c
 
 TRUNNELHEADERS = \
 	src/ext/trunnel/trunnel.h		\
 	src/ext/trunnel/trunnel-impl.h		\
-	src/trunnel/trunnel-local.h 		\
+	src/trunnel/trunnel-local.h		\
 	src/trunnel/ed25519_cert.h		\
-	src/trunnel/link_handshake.h	 	\
-	src/trunnel/pwbox.h
+	src/trunnel/link_handshake.h		\
+	src/trunnel/pwbox.h			\
+	src/trunnel/hs/cell_common.h            \
+	src/trunnel/hs/cell_establish_intro.h	\
+	src/trunnel/hs/cell_introduce1.h \
+	src/trunnel/hs/cell_rendezvous.h \
+	src/trunnel/channelpadding_negotiation.h \
+	src/trunnel/socks5.h
 
 src_trunnel_libor_trunnel_a_SOURCES = $(TRUNNELSOURCES)
-src_trunnel_libor_trunnel_a_CPPFLAGS = -DTRUNNEL_LOCAL_H $(AM_CPPFLAGS)
+src_trunnel_libor_trunnel_a_CPPFLAGS = \
+	-DTRUNNEL_LOCAL_H $(AM_CPPFLAGS) -I$(top_srcdir)/src/trunnel
 
+if UNITTESTS_ENABLED
 src_trunnel_libor_trunnel_testing_a_SOURCES = $(TRUNNELSOURCES)
+else
+src_trunnel_libor_trunnel_testing_a_SOURCES =
+endif
+
 src_trunnel_libor_trunnel_testing_a_CPPFLAGS = -DTRUNNEL_LOCAL_H $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_trunnel_libor_trunnel_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
@@ -39,4 +56,3 @@ noinst_HEADERS+= $(TRUNNELHEADERS)
 
 EXTRA_DIST += \
 	src/trunnel/README
-
diff --git a/src/trunnel/link_handshake.c b/src/trunnel/link_handshake.c
index c2717f36bf..03ead31c62 100644
--- a/src/trunnel/link_handshake.c
+++ b/src/trunnel/link_handshake.c
@@ -1,4 +1,4 @@
-/* link_handshake.c -- generated by Trunnel v1.4.6.
+/* link_handshake.c -- generated by Trunnel v1.5.2.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -13,7 +13,7 @@
   } while (0)
 
 #if defined(__COVERITY__) || defined(__clang_analyzer__)
-/* If we're runnning a static analysis tool, we don't want it to complain
+/* If we're running a static analysis tool, we don't want it to complain
  * that some of our remaining-bytes checks are dead-code. */
 int linkhandshake_deadcode_dummy__ = 0;
 #define OR_DEADCODE_DUMMY || linkhandshake_deadcode_dummy__
@@ -64,12 +64,17 @@ auth_challenge_cell_getlen_challenge(const auth_challenge_cell_t *inp)
 }
 
 uint8_t
-auth_challenge_cell_get_challenge(const auth_challenge_cell_t *inp, size_t idx)
+auth_challenge_cell_get_challenge(auth_challenge_cell_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->challenge[idx];
 }
 
+uint8_t
+auth_challenge_cell_getconst_challenge(const auth_challenge_cell_t *inp, size_t idx)
+{
+  return auth_challenge_cell_get_challenge((auth_challenge_cell_t*)inp, idx);
+}
 int
 auth_challenge_cell_set_challenge(auth_challenge_cell_t *inp, size_t idx, uint8_t elt)
 {
@@ -83,8 +88,13 @@ auth_challenge_cell_getarray_challenge(auth_challenge_cell_t *inp)
 {
   return inp->challenge;
 }
+const uint8_t  *
+auth_challenge_cell_getconstarray_challenge(const auth_challenge_cell_t *inp)
+{
+  return (const uint8_t  *)auth_challenge_cell_getarray_challenge((auth_challenge_cell_t*)inp);
+}
 uint16_t
-auth_challenge_cell_get_n_methods(auth_challenge_cell_t *inp)
+auth_challenge_cell_get_n_methods(const auth_challenge_cell_t *inp)
 {
   return inp->n_methods;
 }
@@ -106,6 +116,11 @@ auth_challenge_cell_get_methods(auth_challenge_cell_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->methods, idx);
 }
 
+uint16_t
+auth_challenge_cell_getconst_methods(const auth_challenge_cell_t *inp, size_t idx)
+{
+  return auth_challenge_cell_get_methods((auth_challenge_cell_t*)inp, idx);
+}
 int
 auth_challenge_cell_set_methods(auth_challenge_cell_t *inp, size_t idx, uint16_t elt)
 {
@@ -131,6 +146,11 @@ auth_challenge_cell_getarray_methods(auth_challenge_cell_t *inp)
 {
   return inp->methods.elts_;
 }
+const uint16_t  *
+auth_challenge_cell_getconstarray_methods(const auth_challenge_cell_t *inp)
+{
+  return (const uint16_t  *)auth_challenge_cell_getarray_methods((auth_challenge_cell_t*)inp);
+}
 int
 auth_challenge_cell_setlen_methods(auth_challenge_cell_t *inp, size_t newlen)
 {
@@ -342,7 +362,7 @@ auth_ctx_free(auth_ctx_t *obj)
 }
 
 uint8_t
-auth_ctx_get_is_ed(auth_ctx_t *inp)
+auth_ctx_get_is_ed(const auth_ctx_t *inp)
 {
   return inp->is_ed;
 }
@@ -382,7 +402,7 @@ certs_cell_cert_free(certs_cell_cert_t *obj)
 }
 
 uint8_t
-certs_cell_cert_get_cert_type(certs_cell_cert_t *inp)
+certs_cell_cert_get_cert_type(const certs_cell_cert_t *inp)
 {
   return inp->cert_type;
 }
@@ -393,7 +413,7 @@ certs_cell_cert_set_cert_type(certs_cell_cert_t *inp, uint8_t val)
   return 0;
 }
 uint16_t
-certs_cell_cert_get_cert_len(certs_cell_cert_t *inp)
+certs_cell_cert_get_cert_len(const certs_cell_cert_t *inp)
 {
   return inp->cert_len;
 }
@@ -415,6 +435,11 @@ certs_cell_cert_get_body(certs_cell_cert_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->body, idx);
 }
 
+uint8_t
+certs_cell_cert_getconst_body(const certs_cell_cert_t *inp, size_t idx)
+{
+  return certs_cell_cert_get_body((certs_cell_cert_t*)inp, idx);
+}
 int
 certs_cell_cert_set_body(certs_cell_cert_t *inp, size_t idx, uint8_t elt)
 {
@@ -440,6 +465,11 @@ certs_cell_cert_getarray_body(certs_cell_cert_t *inp)
 {
   return inp->body.elts_;
 }
+const uint8_t  *
+certs_cell_cert_getconstarray_body(const certs_cell_cert_t *inp)
+{
+  return (const uint8_t  *)certs_cell_cert_getarray_body((certs_cell_cert_t*)inp);
+}
 int
 certs_cell_cert_setlen_body(certs_cell_cert_t *inp, size_t newlen)
 {
@@ -652,12 +682,17 @@ rsa_ed_crosscert_getlen_ed_key(const rsa_ed_crosscert_t *inp)
 }
 
 uint8_t
-rsa_ed_crosscert_get_ed_key(const rsa_ed_crosscert_t *inp, size_t idx)
+rsa_ed_crosscert_get_ed_key(rsa_ed_crosscert_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->ed_key[idx];
 }
 
+uint8_t
+rsa_ed_crosscert_getconst_ed_key(const rsa_ed_crosscert_t *inp, size_t idx)
+{
+  return rsa_ed_crosscert_get_ed_key((rsa_ed_crosscert_t*)inp, idx);
+}
 int
 rsa_ed_crosscert_set_ed_key(rsa_ed_crosscert_t *inp, size_t idx, uint8_t elt)
 {
@@ -671,8 +706,13 @@ rsa_ed_crosscert_getarray_ed_key(rsa_ed_crosscert_t *inp)
 {
   return inp->ed_key;
 }
+const uint8_t  *
+rsa_ed_crosscert_getconstarray_ed_key(const rsa_ed_crosscert_t *inp)
+{
+  return (const uint8_t  *)rsa_ed_crosscert_getarray_ed_key((rsa_ed_crosscert_t*)inp);
+}
 uint32_t
-rsa_ed_crosscert_get_expiration(rsa_ed_crosscert_t *inp)
+rsa_ed_crosscert_get_expiration(const rsa_ed_crosscert_t *inp)
 {
   return inp->expiration;
 }
@@ -688,7 +728,7 @@ rsa_ed_crosscert_get_end_of_signed(const rsa_ed_crosscert_t *inp)
   return inp->end_of_signed;
 }
 uint8_t
-rsa_ed_crosscert_get_sig_len(rsa_ed_crosscert_t *inp)
+rsa_ed_crosscert_get_sig_len(const rsa_ed_crosscert_t *inp)
 {
   return inp->sig_len;
 }
@@ -710,6 +750,11 @@ rsa_ed_crosscert_get_sig(rsa_ed_crosscert_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->sig, idx);
 }
 
+uint8_t
+rsa_ed_crosscert_getconst_sig(const rsa_ed_crosscert_t *inp, size_t idx)
+{
+  return rsa_ed_crosscert_get_sig((rsa_ed_crosscert_t*)inp, idx);
+}
 int
 rsa_ed_crosscert_set_sig(rsa_ed_crosscert_t *inp, size_t idx, uint8_t elt)
 {
@@ -735,6 +780,11 @@ rsa_ed_crosscert_getarray_sig(rsa_ed_crosscert_t *inp)
 {
   return inp->sig.elts_;
 }
+const uint8_t  *
+rsa_ed_crosscert_getconstarray_sig(const rsa_ed_crosscert_t *inp)
+{
+  return (const uint8_t  *)rsa_ed_crosscert_getarray_sig((rsa_ed_crosscert_t*)inp);
+}
 int
 rsa_ed_crosscert_setlen_sig(rsa_ed_crosscert_t *inp, size_t newlen)
 {
@@ -964,12 +1014,17 @@ auth1_getlen_type(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_type(const auth1_t *inp, size_t idx)
+auth1_get_type(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 8);
   return inp->type[idx];
 }
 
+uint8_t
+auth1_getconst_type(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_type((auth1_t*)inp, idx);
+}
 int
 auth1_set_type(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -983,6 +1038,11 @@ auth1_getarray_type(auth1_t *inp)
 {
   return inp->type;
 }
+const uint8_t  *
+auth1_getconstarray_type(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_type((auth1_t*)inp);
+}
 size_t
 auth1_getlen_cid(const auth1_t *inp)
 {
@@ -990,12 +1050,17 @@ auth1_getlen_cid(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_cid(const auth1_t *inp, size_t idx)
+auth1_get_cid(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->cid[idx];
 }
 
+uint8_t
+auth1_getconst_cid(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_cid((auth1_t*)inp, idx);
+}
 int
 auth1_set_cid(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1009,6 +1074,11 @@ auth1_getarray_cid(auth1_t *inp)
 {
   return inp->cid;
 }
+const uint8_t  *
+auth1_getconstarray_cid(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_cid((auth1_t*)inp);
+}
 size_t
 auth1_getlen_sid(const auth1_t *inp)
 {
@@ -1016,12 +1086,17 @@ auth1_getlen_sid(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_sid(const auth1_t *inp, size_t idx)
+auth1_get_sid(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->sid[idx];
 }
 
+uint8_t
+auth1_getconst_sid(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_sid((auth1_t*)inp, idx);
+}
 int
 auth1_set_sid(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1035,6 +1110,11 @@ auth1_getarray_sid(auth1_t *inp)
 {
   return inp->sid;
 }
+const uint8_t  *
+auth1_getconstarray_sid(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_sid((auth1_t*)inp);
+}
 size_t
 auth1_getlen_u1_cid_ed(const auth1_t *inp)
 {
@@ -1042,12 +1122,17 @@ auth1_getlen_u1_cid_ed(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_u1_cid_ed(const auth1_t *inp, size_t idx)
+auth1_get_u1_cid_ed(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->u1_cid_ed[idx];
 }
 
+uint8_t
+auth1_getconst_u1_cid_ed(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_u1_cid_ed((auth1_t*)inp, idx);
+}
 int
 auth1_set_u1_cid_ed(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1061,6 +1146,11 @@ auth1_getarray_u1_cid_ed(auth1_t *inp)
 {
   return inp->u1_cid_ed;
 }
+const uint8_t  *
+auth1_getconstarray_u1_cid_ed(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_u1_cid_ed((auth1_t*)inp);
+}
 size_t
 auth1_getlen_u1_sid_ed(const auth1_t *inp)
 {
@@ -1068,12 +1158,17 @@ auth1_getlen_u1_sid_ed(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_u1_sid_ed(const auth1_t *inp, size_t idx)
+auth1_get_u1_sid_ed(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->u1_sid_ed[idx];
 }
 
+uint8_t
+auth1_getconst_u1_sid_ed(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_u1_sid_ed((auth1_t*)inp, idx);
+}
 int
 auth1_set_u1_sid_ed(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1087,6 +1182,11 @@ auth1_getarray_u1_sid_ed(auth1_t *inp)
 {
   return inp->u1_sid_ed;
 }
+const uint8_t  *
+auth1_getconstarray_u1_sid_ed(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_u1_sid_ed((auth1_t*)inp);
+}
 size_t
 auth1_getlen_slog(const auth1_t *inp)
 {
@@ -1094,12 +1194,17 @@ auth1_getlen_slog(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_slog(const auth1_t *inp, size_t idx)
+auth1_get_slog(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->slog[idx];
 }
 
+uint8_t
+auth1_getconst_slog(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_slog((auth1_t*)inp, idx);
+}
 int
 auth1_set_slog(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1113,6 +1218,11 @@ auth1_getarray_slog(auth1_t *inp)
 {
   return inp->slog;
 }
+const uint8_t  *
+auth1_getconstarray_slog(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_slog((auth1_t*)inp);
+}
 size_t
 auth1_getlen_clog(const auth1_t *inp)
 {
@@ -1120,12 +1230,17 @@ auth1_getlen_clog(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_clog(const auth1_t *inp, size_t idx)
+auth1_get_clog(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->clog[idx];
 }
 
+uint8_t
+auth1_getconst_clog(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_clog((auth1_t*)inp, idx);
+}
 int
 auth1_set_clog(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1139,6 +1254,11 @@ auth1_getarray_clog(auth1_t *inp)
 {
   return inp->clog;
 }
+const uint8_t  *
+auth1_getconstarray_clog(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_clog((auth1_t*)inp);
+}
 size_t
 auth1_getlen_scert(const auth1_t *inp)
 {
@@ -1146,12 +1266,17 @@ auth1_getlen_scert(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_scert(const auth1_t *inp, size_t idx)
+auth1_get_scert(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->scert[idx];
 }
 
+uint8_t
+auth1_getconst_scert(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_scert((auth1_t*)inp, idx);
+}
 int
 auth1_set_scert(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1165,6 +1290,11 @@ auth1_getarray_scert(auth1_t *inp)
 {
   return inp->scert;
 }
+const uint8_t  *
+auth1_getconstarray_scert(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_scert((auth1_t*)inp);
+}
 size_t
 auth1_getlen_tlssecrets(const auth1_t *inp)
 {
@@ -1172,12 +1302,17 @@ auth1_getlen_tlssecrets(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_tlssecrets(const auth1_t *inp, size_t idx)
+auth1_get_tlssecrets(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->tlssecrets[idx];
 }
 
+uint8_t
+auth1_getconst_tlssecrets(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_tlssecrets((auth1_t*)inp, idx);
+}
 int
 auth1_set_tlssecrets(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1191,6 +1326,11 @@ auth1_getarray_tlssecrets(auth1_t *inp)
 {
   return inp->tlssecrets;
 }
+const uint8_t  *
+auth1_getconstarray_tlssecrets(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_tlssecrets((auth1_t*)inp);
+}
 const uint8_t *
 auth1_get_end_of_fixed_part(const auth1_t *inp)
 {
@@ -1203,12 +1343,17 @@ auth1_getlen_rand(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_rand(const auth1_t *inp, size_t idx)
+auth1_get_rand(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 24);
   return inp->rand[idx];
 }
 
+uint8_t
+auth1_getconst_rand(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_rand((auth1_t*)inp, idx);
+}
 int
 auth1_set_rand(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1222,6 +1367,11 @@ auth1_getarray_rand(auth1_t *inp)
 {
   return inp->rand;
 }
+const uint8_t  *
+auth1_getconstarray_rand(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_rand((auth1_t*)inp);
+}
 const uint8_t *
 auth1_get_end_of_signed(const auth1_t *inp)
 {
@@ -1239,6 +1389,11 @@ auth1_get_sig(auth1_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->sig, idx);
 }
 
+uint8_t
+auth1_getconst_sig(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_sig((auth1_t*)inp, idx);
+}
 int
 auth1_set_sig(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1260,6 +1415,11 @@ auth1_getarray_sig(auth1_t *inp)
 {
   return inp->sig.elts_;
 }
+const uint8_t  *
+auth1_getconstarray_sig(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_sig((auth1_t*)inp);
+}
 int
 auth1_setlen_sig(auth1_t *inp, size_t newlen)
 {
@@ -1647,7 +1807,7 @@ certs_cell_free(certs_cell_t *obj)
 }
 
 uint8_t
-certs_cell_get_n_certs(certs_cell_t *inp)
+certs_cell_get_n_certs(const certs_cell_t *inp)
 {
   return inp->n_certs;
 }
@@ -1669,6 +1829,11 @@ certs_cell_get_certs(certs_cell_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->certs, idx);
 }
 
+ const struct certs_cell_cert_st *
+certs_cell_getconst_certs(const certs_cell_t *inp, size_t idx)
+{
+  return certs_cell_get_certs((certs_cell_t*)inp, idx);
+}
 int
 certs_cell_set_certs(certs_cell_t *inp, size_t idx, struct certs_cell_cert_st * elt)
 {
@@ -1702,6 +1867,11 @@ certs_cell_getarray_certs(certs_cell_t *inp)
 {
   return inp->certs.elts_;
 }
+const struct certs_cell_cert_st *  const  *
+certs_cell_getconstarray_certs(const certs_cell_t *inp)
+{
+  return (const struct certs_cell_cert_st *  const  *)certs_cell_getarray_certs((certs_cell_t*)inp);
+}
 int
 certs_cell_setlen_certs(certs_cell_t *inp, size_t newlen)
 {
diff --git a/src/trunnel/link_handshake.h b/src/trunnel/link_handshake.h
index 54611b96e8..6a23483adc 100644
--- a/src/trunnel/link_handshake.h
+++ b/src/trunnel/link_handshake.h
@@ -1,4 +1,4 @@
-/* link_handshake.h -- generated by by Trunnel v1.4.6.
+/* link_handshake.h -- generated by Trunnel v1.5.2.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -121,7 +121,11 @@ size_t auth_challenge_cell_getlen_challenge(const auth_challenge_cell_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * challenge of the auth_challenge_cell_t in 'inp'.
  */
-uint8_t auth_challenge_cell_get_challenge(const auth_challenge_cell_t *inp, size_t idx);
+uint8_t auth_challenge_cell_get_challenge(auth_challenge_cell_t *inp, size_t idx);
+/** As auth_challenge_cell_get_challenge, but take and return a const
+ * pointer
+ */
+uint8_t auth_challenge_cell_getconst_challenge(const auth_challenge_cell_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * challenge of the auth_challenge_cell_t in 'inp', so that it will
  * hold the value 'elt'.
@@ -130,10 +134,14 @@ int auth_challenge_cell_set_challenge(auth_challenge_cell_t *inp, size_t idx, ui
 /** Return a pointer to the 32-element array field challenge of 'inp'.
  */
 uint8_t * auth_challenge_cell_getarray_challenge(auth_challenge_cell_t *inp);
+/** As auth_challenge_cell_get_challenge, but take and return a const
+ * pointer
+ */
+const uint8_t  * auth_challenge_cell_getconstarray_challenge(const auth_challenge_cell_t *inp);
 /** Return the value of the n_methods field of the
  * auth_challenge_cell_t in 'inp'
  */
-uint16_t auth_challenge_cell_get_n_methods(auth_challenge_cell_t *inp);
+uint16_t auth_challenge_cell_get_n_methods(const auth_challenge_cell_t *inp);
 /** Set the value of the n_methods field of the auth_challenge_cell_t
  * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -147,6 +155,10 @@ size_t auth_challenge_cell_getlen_methods(const auth_challenge_cell_t *inp);
  * methods of the auth_challenge_cell_t in 'inp'.
  */
 uint16_t auth_challenge_cell_get_methods(auth_challenge_cell_t *inp, size_t idx);
+/** As auth_challenge_cell_get_methods, but take and return a const
+ * pointer
+ */
+uint16_t auth_challenge_cell_getconst_methods(const auth_challenge_cell_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * methods of the auth_challenge_cell_t in 'inp', so that it will hold
  * the value 'elt'.
@@ -160,6 +172,10 @@ int auth_challenge_cell_add_methods(auth_challenge_cell_t *inp, uint16_t elt);
  * 'inp'.
  */
 uint16_t * auth_challenge_cell_getarray_methods(auth_challenge_cell_t *inp);
+/** As auth_challenge_cell_get_methods, but take and return a const
+ * pointer
+ */
+const uint16_t  * auth_challenge_cell_getconstarray_methods(const auth_challenge_cell_t *inp);
 /** Change the length of the variable-length array field methods of
  * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
@@ -174,7 +190,7 @@ auth_ctx_t *auth_ctx_new(void);
 void auth_ctx_free(auth_ctx_t *victim);
 /** Return the value of the is_ed field of the auth_ctx_t in 'inp'
  */
-uint8_t auth_ctx_get_is_ed(auth_ctx_t *inp);
+uint8_t auth_ctx_get_is_ed(const auth_ctx_t *inp);
 /** Set the value of the is_ed field of the auth_ctx_t in 'inp' to
  * 'val'. Return 0 on success; return -1 and set the error code on
  * 'inp' on failure.
@@ -219,7 +235,7 @@ int certs_cell_cert_clear_errors(certs_cell_cert_t *obj);
 /** Return the value of the cert_type field of the certs_cell_cert_t
  * in 'inp'
  */
-uint8_t certs_cell_cert_get_cert_type(certs_cell_cert_t *inp);
+uint8_t certs_cell_cert_get_cert_type(const certs_cell_cert_t *inp);
 /** Set the value of the cert_type field of the certs_cell_cert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -228,7 +244,7 @@ int certs_cell_cert_set_cert_type(certs_cell_cert_t *inp, uint8_t val);
 /** Return the value of the cert_len field of the certs_cell_cert_t in
  * 'inp'
  */
-uint16_t certs_cell_cert_get_cert_len(certs_cell_cert_t *inp);
+uint16_t certs_cell_cert_get_cert_len(const certs_cell_cert_t *inp);
 /** Set the value of the cert_len field of the certs_cell_cert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -242,6 +258,9 @@ size_t certs_cell_cert_getlen_body(const certs_cell_cert_t *inp);
  * body of the certs_cell_cert_t in 'inp'.
  */
 uint8_t certs_cell_cert_get_body(certs_cell_cert_t *inp, size_t idx);
+/** As certs_cell_cert_get_body, but take and return a const pointer
+ */
+uint8_t certs_cell_cert_getconst_body(const certs_cell_cert_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * body of the certs_cell_cert_t in 'inp', so that it will hold the
  * value 'elt'.
@@ -254,6 +273,9 @@ int certs_cell_cert_add_body(certs_cell_cert_t *inp, uint8_t elt);
 /** Return a pointer to the variable-length array field body of 'inp'.
  */
 uint8_t * certs_cell_cert_getarray_body(certs_cell_cert_t *inp);
+/** As certs_cell_cert_get_body, but take and return a const pointer
+ */
+const uint8_t  * certs_cell_cert_getconstarray_body(const certs_cell_cert_t *inp);
 /** Change the length of the variable-length array field body of 'inp'
  * to 'newlen'.Fill extra elements with 0. Return 0 on success; return
  * -1 and set the error code on 'inp' on failure.
@@ -302,7 +324,11 @@ size_t rsa_ed_crosscert_getlen_ed_key(const rsa_ed_crosscert_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * ed_key of the rsa_ed_crosscert_t in 'inp'.
  */
-uint8_t rsa_ed_crosscert_get_ed_key(const rsa_ed_crosscert_t *inp, size_t idx);
+uint8_t rsa_ed_crosscert_get_ed_key(rsa_ed_crosscert_t *inp, size_t idx);
+/** As rsa_ed_crosscert_get_ed_key, but take and return a const
+ * pointer
+ */
+uint8_t rsa_ed_crosscert_getconst_ed_key(const rsa_ed_crosscert_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * ed_key of the rsa_ed_crosscert_t in 'inp', so that it will hold the
  * value 'elt'.
@@ -311,10 +337,14 @@ int rsa_ed_crosscert_set_ed_key(rsa_ed_crosscert_t *inp, size_t idx, uint8_t elt
 /** Return a pointer to the 32-element array field ed_key of 'inp'.
  */
 uint8_t * rsa_ed_crosscert_getarray_ed_key(rsa_ed_crosscert_t *inp);
+/** As rsa_ed_crosscert_get_ed_key, but take and return a const
+ * pointer
+ */
+const uint8_t  * rsa_ed_crosscert_getconstarray_ed_key(const rsa_ed_crosscert_t *inp);
 /** Return the value of the expiration field of the rsa_ed_crosscert_t
  * in 'inp'
  */
-uint32_t rsa_ed_crosscert_get_expiration(rsa_ed_crosscert_t *inp);
+uint32_t rsa_ed_crosscert_get_expiration(const rsa_ed_crosscert_t *inp);
 /** Set the value of the expiration field of the rsa_ed_crosscert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -326,7 +356,7 @@ const uint8_t * rsa_ed_crosscert_get_end_of_signed(const rsa_ed_crosscert_t *inp
 /** Return the value of the sig_len field of the rsa_ed_crosscert_t in
  * 'inp'
  */
-uint8_t rsa_ed_crosscert_get_sig_len(rsa_ed_crosscert_t *inp);
+uint8_t rsa_ed_crosscert_get_sig_len(const rsa_ed_crosscert_t *inp);
 /** Set the value of the sig_len field of the rsa_ed_crosscert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -340,6 +370,9 @@ size_t rsa_ed_crosscert_getlen_sig(const rsa_ed_crosscert_t *inp);
  * sig of the rsa_ed_crosscert_t in 'inp'.
  */
 uint8_t rsa_ed_crosscert_get_sig(rsa_ed_crosscert_t *inp, size_t idx);
+/** As rsa_ed_crosscert_get_sig, but take and return a const pointer
+ */
+uint8_t rsa_ed_crosscert_getconst_sig(const rsa_ed_crosscert_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * sig of the rsa_ed_crosscert_t in 'inp', so that it will hold the
  * value 'elt'.
@@ -352,6 +385,9 @@ int rsa_ed_crosscert_add_sig(rsa_ed_crosscert_t *inp, uint8_t elt);
 /** Return a pointer to the variable-length array field sig of 'inp'.
  */
 uint8_t * rsa_ed_crosscert_getarray_sig(rsa_ed_crosscert_t *inp);
+/** As rsa_ed_crosscert_get_sig, but take and return a const pointer
+ */
+const uint8_t  * rsa_ed_crosscert_getconstarray_sig(const rsa_ed_crosscert_t *inp);
 /** Change the length of the variable-length array field sig of 'inp'
  * to 'newlen'.Fill extra elements with 0. Return 0 on success; return
  * -1 and set the error code on 'inp' on failure.
@@ -398,7 +434,10 @@ size_t auth1_getlen_type(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field type
  * of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_type(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_type(auth1_t *inp, size_t idx);
+/** As auth1_get_type, but take and return a const pointer
+ */
+uint8_t auth1_getconst_type(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field type
  * of the auth1_t in 'inp', so that it will hold the value 'elt'.
  */
@@ -406,6 +445,9 @@ int auth1_set_type(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 8-element array field type of 'inp'.
  */
 uint8_t * auth1_getarray_type(auth1_t *inp);
+/** As auth1_get_type, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_type(const auth1_t *inp);
 /** Return the (constant) length of the array holding the cid field of
  * the auth1_t in 'inp'.
  */
@@ -413,7 +455,10 @@ size_t auth1_getlen_cid(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field cid
  * of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_cid(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_cid(auth1_t *inp, size_t idx);
+/** As auth1_get_cid, but take and return a const pointer
+ */
+uint8_t auth1_getconst_cid(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field cid
  * of the auth1_t in 'inp', so that it will hold the value 'elt'.
  */
@@ -421,6 +466,9 @@ int auth1_set_cid(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field cid of 'inp'.
  */
 uint8_t * auth1_getarray_cid(auth1_t *inp);
+/** As auth1_get_cid, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_cid(const auth1_t *inp);
 /** Return the (constant) length of the array holding the sid field of
  * the auth1_t in 'inp'.
  */
@@ -428,7 +476,10 @@ size_t auth1_getlen_sid(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field sid
  * of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_sid(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_sid(auth1_t *inp, size_t idx);
+/** As auth1_get_sid, but take and return a const pointer
+ */
+uint8_t auth1_getconst_sid(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field sid
  * of the auth1_t in 'inp', so that it will hold the value 'elt'.
  */
@@ -436,6 +487,9 @@ int auth1_set_sid(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field sid of 'inp'.
  */
 uint8_t * auth1_getarray_sid(auth1_t *inp);
+/** As auth1_get_sid, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_sid(const auth1_t *inp);
 /** Return the (constant) length of the array holding the u1_cid_ed
  * field of the auth1_t in 'inp'.
  */
@@ -443,7 +497,10 @@ size_t auth1_getlen_u1_cid_ed(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * u1_cid_ed of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_u1_cid_ed(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_u1_cid_ed(auth1_t *inp, size_t idx);
+/** As auth1_get_u1_cid_ed, but take and return a const pointer
+ */
+uint8_t auth1_getconst_u1_cid_ed(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * u1_cid_ed of the auth1_t in 'inp', so that it will hold the value
  * 'elt'.
@@ -452,6 +509,9 @@ int auth1_set_u1_cid_ed(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field u1_cid_ed of 'inp'.
  */
 uint8_t * auth1_getarray_u1_cid_ed(auth1_t *inp);
+/** As auth1_get_u1_cid_ed, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_u1_cid_ed(const auth1_t *inp);
 /** Return the (constant) length of the array holding the u1_sid_ed
  * field of the auth1_t in 'inp'.
  */
@@ -459,7 +519,10 @@ size_t auth1_getlen_u1_sid_ed(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * u1_sid_ed of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_u1_sid_ed(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_u1_sid_ed(auth1_t *inp, size_t idx);
+/** As auth1_get_u1_sid_ed, but take and return a const pointer
+ */
+uint8_t auth1_getconst_u1_sid_ed(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * u1_sid_ed of the auth1_t in 'inp', so that it will hold the value
  * 'elt'.
@@ -468,6 +531,9 @@ int auth1_set_u1_sid_ed(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field u1_sid_ed of 'inp'.
  */
 uint8_t * auth1_getarray_u1_sid_ed(auth1_t *inp);
+/** As auth1_get_u1_sid_ed, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_u1_sid_ed(const auth1_t *inp);
 /** Return the (constant) length of the array holding the slog field
  * of the auth1_t in 'inp'.
  */
@@ -475,7 +541,10 @@ size_t auth1_getlen_slog(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field slog
  * of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_slog(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_slog(auth1_t *inp, size_t idx);
+/** As auth1_get_slog, but take and return a const pointer
+ */
+uint8_t auth1_getconst_slog(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field slog
  * of the auth1_t in 'inp', so that it will hold the value 'elt'.
  */
@@ -483,6 +552,9 @@ int auth1_set_slog(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field slog of 'inp'.
  */
 uint8_t * auth1_getarray_slog(auth1_t *inp);
+/** As auth1_get_slog, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_slog(const auth1_t *inp);
 /** Return the (constant) length of the array holding the clog field
  * of the auth1_t in 'inp'.
  */
@@ -490,7 +562,10 @@ size_t auth1_getlen_clog(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field clog
  * of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_clog(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_clog(auth1_t *inp, size_t idx);
+/** As auth1_get_clog, but take and return a const pointer
+ */
+uint8_t auth1_getconst_clog(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field clog
  * of the auth1_t in 'inp', so that it will hold the value 'elt'.
  */
@@ -498,6 +573,9 @@ int auth1_set_clog(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field clog of 'inp'.
  */
 uint8_t * auth1_getarray_clog(auth1_t *inp);
+/** As auth1_get_clog, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_clog(const auth1_t *inp);
 /** Return the (constant) length of the array holding the scert field
  * of the auth1_t in 'inp'.
  */
@@ -505,7 +583,10 @@ size_t auth1_getlen_scert(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * scert of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_scert(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_scert(auth1_t *inp, size_t idx);
+/** As auth1_get_scert, but take and return a const pointer
+ */
+uint8_t auth1_getconst_scert(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * scert of the auth1_t in 'inp', so that it will hold the value
  * 'elt'.
@@ -514,6 +595,9 @@ int auth1_set_scert(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field scert of 'inp'.
  */
 uint8_t * auth1_getarray_scert(auth1_t *inp);
+/** As auth1_get_scert, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_scert(const auth1_t *inp);
 /** Return the (constant) length of the array holding the tlssecrets
  * field of the auth1_t in 'inp'.
  */
@@ -521,7 +605,10 @@ size_t auth1_getlen_tlssecrets(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * tlssecrets of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_tlssecrets(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_tlssecrets(auth1_t *inp, size_t idx);
+/** As auth1_get_tlssecrets, but take and return a const pointer
+ */
+uint8_t auth1_getconst_tlssecrets(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * tlssecrets of the auth1_t in 'inp', so that it will hold the value
  * 'elt'.
@@ -531,6 +618,9 @@ int auth1_set_tlssecrets(auth1_t *inp, size_t idx, uint8_t elt);
  * 'inp'.
  */
 uint8_t * auth1_getarray_tlssecrets(auth1_t *inp);
+/** As auth1_get_tlssecrets, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_tlssecrets(const auth1_t *inp);
 /** Return the position for end_of_fixed_part when we parsed this
  * object
  */
@@ -542,7 +632,10 @@ size_t auth1_getlen_rand(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field rand
  * of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_rand(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_rand(auth1_t *inp, size_t idx);
+/** As auth1_get_rand, but take and return a const pointer
+ */
+uint8_t auth1_getconst_rand(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field rand
  * of the auth1_t in 'inp', so that it will hold the value 'elt'.
  */
@@ -550,6 +643,9 @@ int auth1_set_rand(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 24-element array field rand of 'inp'.
  */
 uint8_t * auth1_getarray_rand(auth1_t *inp);
+/** As auth1_get_rand, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_rand(const auth1_t *inp);
 /** Return the position for end_of_signed when we parsed this object
  */
 const uint8_t * auth1_get_end_of_signed(const auth1_t *inp);
@@ -561,6 +657,9 @@ size_t auth1_getlen_sig(const auth1_t *inp);
  * sig of the auth1_t in 'inp'.
  */
 uint8_t auth1_get_sig(auth1_t *inp, size_t idx);
+/** As auth1_get_sig, but take and return a const pointer
+ */
+uint8_t auth1_getconst_sig(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * sig of the auth1_t in 'inp', so that it will hold the value 'elt'.
  */
@@ -572,6 +671,9 @@ int auth1_add_sig(auth1_t *inp, uint8_t elt);
 /** Return a pointer to the variable-length array field sig of 'inp'.
  */
 uint8_t * auth1_getarray_sig(auth1_t *inp);
+/** As auth1_get_sig, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_sig(const auth1_t *inp);
 /** Change the length of the variable-length array field sig of 'inp'
  * to 'newlen'.Fill extra elements with 0. Return 0 on success; return
  * -1 and set the error code on 'inp' on failure.
@@ -613,7 +715,7 @@ const char *certs_cell_check(const certs_cell_t *obj);
 int certs_cell_clear_errors(certs_cell_t *obj);
 /** Return the value of the n_certs field of the certs_cell_t in 'inp'
  */
-uint8_t certs_cell_get_n_certs(certs_cell_t *inp);
+uint8_t certs_cell_get_n_certs(const certs_cell_t *inp);
 /** Set the value of the n_certs field of the certs_cell_t in 'inp' to
  * 'val'. Return 0 on success; return -1 and set the error code on
  * 'inp' on failure.
@@ -627,6 +729,9 @@ size_t certs_cell_getlen_certs(const certs_cell_t *inp);
  * certs of the certs_cell_t in 'inp'.
  */
 struct certs_cell_cert_st * certs_cell_get_certs(certs_cell_t *inp, size_t idx);
+/** As certs_cell_get_certs, but take and return a const pointer
+ */
+ const struct certs_cell_cert_st * certs_cell_getconst_certs(const certs_cell_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * certs of the certs_cell_t in 'inp', so that it will hold the value
  * 'elt'. Free the previous value, if any.
@@ -643,6 +748,9 @@ int certs_cell_add_certs(certs_cell_t *inp, struct certs_cell_cert_st * elt);
  * 'inp'.
  */
 struct certs_cell_cert_st * * certs_cell_getarray_certs(certs_cell_t *inp);
+/** As certs_cell_get_certs, but take and return a const pointer
+ */
+const struct certs_cell_cert_st *  const  * certs_cell_getconstarray_certs(const certs_cell_t *inp);
 /** Change the length of the variable-length array field certs of
  * 'inp' to 'newlen'.Fill extra elements with NULL; free removed
  * elements. Return 0 on success; return -1 and set the error code on
diff --git a/src/trunnel/pwbox.c b/src/trunnel/pwbox.c
index 62662a9369..c356515d36 100644
--- a/src/trunnel/pwbox.c
+++ b/src/trunnel/pwbox.c
@@ -1,4 +1,4 @@
-/* pwbox.c -- generated by Trunnel v1.4.6.
+/* pwbox.c -- generated by Trunnel v1.5.2.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -13,7 +13,7 @@
   } while (0)
 
 #if defined(__COVERITY__) || defined(__clang_analyzer__)
-/* If we're runnning a static analysis tool, we don't want it to complain
+/* If we're running a static analysis tool, we don't want it to complain
  * that some of our remaining-bytes checks are dead-code. */
 int pwbox_deadcode_dummy__ = 0;
 #define OR_DEADCODE_DUMMY || pwbox_deadcode_dummy__
@@ -62,7 +62,7 @@ pwbox_encoded_free(pwbox_encoded_t *obj)
 }
 
 uint32_t
-pwbox_encoded_get_fixedbytes0(pwbox_encoded_t *inp)
+pwbox_encoded_get_fixedbytes0(const pwbox_encoded_t *inp)
 {
   return inp->fixedbytes0;
 }
@@ -77,7 +77,7 @@ pwbox_encoded_set_fixedbytes0(pwbox_encoded_t *inp, uint32_t val)
   return 0;
 }
 uint32_t
-pwbox_encoded_get_fixedbytes1(pwbox_encoded_t *inp)
+pwbox_encoded_get_fixedbytes1(const pwbox_encoded_t *inp)
 {
   return inp->fixedbytes1;
 }
@@ -92,7 +92,7 @@ pwbox_encoded_set_fixedbytes1(pwbox_encoded_t *inp, uint32_t val)
   return 0;
 }
 uint8_t
-pwbox_encoded_get_header_len(pwbox_encoded_t *inp)
+pwbox_encoded_get_header_len(const pwbox_encoded_t *inp)
 {
   return inp->header_len;
 }
@@ -114,6 +114,11 @@ pwbox_encoded_get_skey_header(pwbox_encoded_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->skey_header, idx);
 }
 
+uint8_t
+pwbox_encoded_getconst_skey_header(const pwbox_encoded_t *inp, size_t idx)
+{
+  return pwbox_encoded_get_skey_header((pwbox_encoded_t*)inp, idx);
+}
 int
 pwbox_encoded_set_skey_header(pwbox_encoded_t *inp, size_t idx, uint8_t elt)
 {
@@ -139,6 +144,11 @@ pwbox_encoded_getarray_skey_header(pwbox_encoded_t *inp)
 {
   return inp->skey_header.elts_;
 }
+const uint8_t  *
+pwbox_encoded_getconstarray_skey_header(const pwbox_encoded_t *inp)
+{
+  return (const uint8_t  *)pwbox_encoded_getarray_skey_header((pwbox_encoded_t*)inp);
+}
 int
 pwbox_encoded_setlen_skey_header(pwbox_encoded_t *inp, size_t newlen)
 {
@@ -166,12 +176,17 @@ pwbox_encoded_getlen_iv(const pwbox_encoded_t *inp)
 }
 
 uint8_t
-pwbox_encoded_get_iv(const pwbox_encoded_t *inp, size_t idx)
+pwbox_encoded_get_iv(pwbox_encoded_t *inp, size_t idx)
 {
   trunnel_assert(idx < 16);
   return inp->iv[idx];
 }
 
+uint8_t
+pwbox_encoded_getconst_iv(const pwbox_encoded_t *inp, size_t idx)
+{
+  return pwbox_encoded_get_iv((pwbox_encoded_t*)inp, idx);
+}
 int
 pwbox_encoded_set_iv(pwbox_encoded_t *inp, size_t idx, uint8_t elt)
 {
@@ -185,6 +200,11 @@ pwbox_encoded_getarray_iv(pwbox_encoded_t *inp)
 {
   return inp->iv;
 }
+const uint8_t  *
+pwbox_encoded_getconstarray_iv(const pwbox_encoded_t *inp)
+{
+  return (const uint8_t  *)pwbox_encoded_getarray_iv((pwbox_encoded_t*)inp);
+}
 size_t
 pwbox_encoded_getlen_data(const pwbox_encoded_t *inp)
 {
@@ -197,6 +217,11 @@ pwbox_encoded_get_data(pwbox_encoded_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->data, idx);
 }
 
+uint8_t
+pwbox_encoded_getconst_data(const pwbox_encoded_t *inp, size_t idx)
+{
+  return pwbox_encoded_get_data((pwbox_encoded_t*)inp, idx);
+}
 int
 pwbox_encoded_set_data(pwbox_encoded_t *inp, size_t idx, uint8_t elt)
 {
@@ -218,6 +243,11 @@ pwbox_encoded_getarray_data(pwbox_encoded_t *inp)
 {
   return inp->data.elts_;
 }
+const uint8_t  *
+pwbox_encoded_getconstarray_data(const pwbox_encoded_t *inp)
+{
+  return (const uint8_t  *)pwbox_encoded_getarray_data((pwbox_encoded_t*)inp);
+}
 int
 pwbox_encoded_setlen_data(pwbox_encoded_t *inp, size_t newlen)
 {
@@ -241,12 +271,17 @@ pwbox_encoded_getlen_hmac(const pwbox_encoded_t *inp)
 }
 
 uint8_t
-pwbox_encoded_get_hmac(const pwbox_encoded_t *inp, size_t idx)
+pwbox_encoded_get_hmac(pwbox_encoded_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->hmac[idx];
 }
 
+uint8_t
+pwbox_encoded_getconst_hmac(const pwbox_encoded_t *inp, size_t idx)
+{
+  return pwbox_encoded_get_hmac((pwbox_encoded_t*)inp, idx);
+}
 int
 pwbox_encoded_set_hmac(pwbox_encoded_t *inp, size_t idx, uint8_t elt)
 {
@@ -260,6 +295,11 @@ pwbox_encoded_getarray_hmac(pwbox_encoded_t *inp)
 {
   return inp->hmac;
 }
+const uint8_t  *
+pwbox_encoded_getconstarray_hmac(const pwbox_encoded_t *inp)
+{
+  return (const uint8_t  *)pwbox_encoded_getarray_hmac((pwbox_encoded_t*)inp);
+}
 const char *
 pwbox_encoded_check(const pwbox_encoded_t *obj)
 {
diff --git a/src/trunnel/pwbox.h b/src/trunnel/pwbox.h
index 77a813d123..a9a421408a 100644
--- a/src/trunnel/pwbox.h
+++ b/src/trunnel/pwbox.h
@@ -1,4 +1,4 @@
-/* pwbox.h -- generated by by Trunnel v1.4.6.
+/* pwbox.h -- generated by Trunnel v1.5.2.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -61,7 +61,7 @@ int pwbox_encoded_clear_errors(pwbox_encoded_t *obj);
 /** Return the value of the fixedbytes0 field of the pwbox_encoded_t
  * in 'inp'
  */
-uint32_t pwbox_encoded_get_fixedbytes0(pwbox_encoded_t *inp);
+uint32_t pwbox_encoded_get_fixedbytes0(const pwbox_encoded_t *inp);
 /** Set the value of the fixedbytes0 field of the pwbox_encoded_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -70,7 +70,7 @@ int pwbox_encoded_set_fixedbytes0(pwbox_encoded_t *inp, uint32_t val);
 /** Return the value of the fixedbytes1 field of the pwbox_encoded_t
  * in 'inp'
  */
-uint32_t pwbox_encoded_get_fixedbytes1(pwbox_encoded_t *inp);
+uint32_t pwbox_encoded_get_fixedbytes1(const pwbox_encoded_t *inp);
 /** Set the value of the fixedbytes1 field of the pwbox_encoded_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -79,7 +79,7 @@ int pwbox_encoded_set_fixedbytes1(pwbox_encoded_t *inp, uint32_t val);
 /** Return the value of the header_len field of the pwbox_encoded_t in
  * 'inp'
  */
-uint8_t pwbox_encoded_get_header_len(pwbox_encoded_t *inp);
+uint8_t pwbox_encoded_get_header_len(const pwbox_encoded_t *inp);
 /** Set the value of the header_len field of the pwbox_encoded_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -93,6 +93,10 @@ size_t pwbox_encoded_getlen_skey_header(const pwbox_encoded_t *inp);
  * skey_header of the pwbox_encoded_t in 'inp'.
  */
 uint8_t pwbox_encoded_get_skey_header(pwbox_encoded_t *inp, size_t idx);
+/** As pwbox_encoded_get_skey_header, but take and return a const
+ * pointer
+ */
+uint8_t pwbox_encoded_getconst_skey_header(const pwbox_encoded_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * skey_header of the pwbox_encoded_t in 'inp', so that it will hold
  * the value 'elt'.
@@ -106,6 +110,10 @@ int pwbox_encoded_add_skey_header(pwbox_encoded_t *inp, uint8_t elt);
  * 'inp'.
  */
 uint8_t * pwbox_encoded_getarray_skey_header(pwbox_encoded_t *inp);
+/** As pwbox_encoded_get_skey_header, but take and return a const
+ * pointer
+ */
+const uint8_t  * pwbox_encoded_getconstarray_skey_header(const pwbox_encoded_t *inp);
 /** Change the length of the variable-length array field skey_header
  * of 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on
  * success; return -1 and set the error code on 'inp' on failure.
@@ -118,7 +126,10 @@ size_t pwbox_encoded_getlen_iv(const pwbox_encoded_t *inp);
 /** Return the element at position 'idx' of the fixed array field iv
  * of the pwbox_encoded_t in 'inp'.
  */
-uint8_t pwbox_encoded_get_iv(const pwbox_encoded_t *inp, size_t idx);
+uint8_t pwbox_encoded_get_iv(pwbox_encoded_t *inp, size_t idx);
+/** As pwbox_encoded_get_iv, but take and return a const pointer
+ */
+uint8_t pwbox_encoded_getconst_iv(const pwbox_encoded_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field iv
  * of the pwbox_encoded_t in 'inp', so that it will hold the value
  * 'elt'.
@@ -127,6 +138,9 @@ int pwbox_encoded_set_iv(pwbox_encoded_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 16-element array field iv of 'inp'.
  */
 uint8_t * pwbox_encoded_getarray_iv(pwbox_encoded_t *inp);
+/** As pwbox_encoded_get_iv, but take and return a const pointer
+ */
+const uint8_t  * pwbox_encoded_getconstarray_iv(const pwbox_encoded_t *inp);
 /** Return the length of the dynamic array holding the data field of
  * the pwbox_encoded_t in 'inp'.
  */
@@ -135,6 +149,9 @@ size_t pwbox_encoded_getlen_data(const pwbox_encoded_t *inp);
  * data of the pwbox_encoded_t in 'inp'.
  */
 uint8_t pwbox_encoded_get_data(pwbox_encoded_t *inp, size_t idx);
+/** As pwbox_encoded_get_data, but take and return a const pointer
+ */
+uint8_t pwbox_encoded_getconst_data(const pwbox_encoded_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * data of the pwbox_encoded_t in 'inp', so that it will hold the
  * value 'elt'.
@@ -147,6 +164,9 @@ int pwbox_encoded_add_data(pwbox_encoded_t *inp, uint8_t elt);
 /** Return a pointer to the variable-length array field data of 'inp'.
  */
 uint8_t * pwbox_encoded_getarray_data(pwbox_encoded_t *inp);
+/** As pwbox_encoded_get_data, but take and return a const pointer
+ */
+const uint8_t  * pwbox_encoded_getconstarray_data(const pwbox_encoded_t *inp);
 /** Change the length of the variable-length array field data of 'inp'
  * to 'newlen'.Fill extra elements with 0. Return 0 on success; return
  * -1 and set the error code on 'inp' on failure.
@@ -159,7 +179,10 @@ size_t pwbox_encoded_getlen_hmac(const pwbox_encoded_t *inp);
 /** Return the element at position 'idx' of the fixed array field hmac
  * of the pwbox_encoded_t in 'inp'.
  */
-uint8_t pwbox_encoded_get_hmac(const pwbox_encoded_t *inp, size_t idx);
+uint8_t pwbox_encoded_get_hmac(pwbox_encoded_t *inp, size_t idx);
+/** As pwbox_encoded_get_hmac, but take and return a const pointer
+ */
+uint8_t pwbox_encoded_getconst_hmac(const pwbox_encoded_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field hmac
  * of the pwbox_encoded_t in 'inp', so that it will hold the value
  * 'elt'.
@@ -168,6 +191,9 @@ int pwbox_encoded_set_hmac(pwbox_encoded_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field hmac of 'inp'.
  */
 uint8_t * pwbox_encoded_getarray_hmac(pwbox_encoded_t *inp);
+/** As pwbox_encoded_get_hmac, but take and return a const pointer
+ */
+const uint8_t  * pwbox_encoded_getconstarray_hmac(const pwbox_encoded_t *inp);
 
 
 #endif
diff --git a/src/trunnel/socks5.c b/src/trunnel/socks5.c
new file mode 100644
index 0000000000..9e5f6fcfed
--- /dev/null
+++ b/src/trunnel/socks5.c
@@ -0,0 +1,3978 @@
+/* socks5.c -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "socks5.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int socks_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || socks_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+domainname_t *
+domainname_new(void)
+{
+  domainname_t *val = trunnel_calloc(1, sizeof(domainname_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+domainname_clear(domainname_t *obj)
+{
+  (void) obj;
+  TRUNNEL_DYNARRAY_WIPE(&obj->name);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->name);
+}
+
+void
+domainname_free(domainname_t *obj)
+{
+  if (obj == NULL)
+    return;
+  domainname_clear(obj);
+  trunnel_memwipe(obj, sizeof(domainname_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+domainname_get_len(const domainname_t *inp)
+{
+  return inp->len;
+}
+int
+domainname_set_len(domainname_t *inp, uint8_t val)
+{
+  inp->len = val;
+  return 0;
+}
+size_t
+domainname_getlen_name(const domainname_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->name);
+}
+
+char
+domainname_get_name(domainname_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->name, idx);
+}
+
+char
+domainname_getconst_name(const domainname_t *inp, size_t idx)
+{
+  return domainname_get_name((domainname_t*)inp, idx);
+}
+int
+domainname_set_name(domainname_t *inp, size_t idx, char elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->name, idx, elt);
+  return 0;
+}
+int
+domainname_add_name(domainname_t *inp, char elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->name.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(char, &inp->name, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+char *
+domainname_getarray_name(domainname_t *inp)
+{
+  return inp->name.elts_;
+}
+const char  *
+domainname_getconstarray_name(const domainname_t *inp)
+{
+  return (const char  *)domainname_getarray_name((domainname_t*)inp);
+}
+int
+domainname_setlen_name(domainname_t *inp, size_t newlen)
+{
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  return trunnel_string_setlen(&inp->name, newlen,
+            &inp->trunnel_error_code_);
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+domainname_getstr_name(domainname_t *inp)
+{
+  return trunnel_string_getstr(&inp->name);
+}
+int
+domainname_setstr0_name(domainname_t *inp, const char *val, size_t len)
+{
+#if UINT8_MAX < SIZE_MAX
+  if (len > UINT8_MAX) {
+    TRUNNEL_SET_ERROR_CODE(inp);
+    return -1;
+  }
+#endif
+  return trunnel_string_setstr0(&inp->name, val, len, &inp->trunnel_error_code_);
+}
+int
+domainname_setstr_name(domainname_t *inp, const char *val)
+{
+  return domainname_setstr0_name(inp, val, strlen(val));
+}
+const char *
+domainname_check(const domainname_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (TRUNNEL_DYNARRAY_LEN(&obj->name) != obj->len)
+    return "Length mismatch for name";
+  return NULL;
+}
+
+ssize_t
+domainname_encoded_len(const domainname_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != domainname_check(obj))
+     return -1;
+
+
+  /* Length of u8 len */
+  result += 1;
+
+  /* Length of char name[len] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->name);
+  return result;
+}
+int
+domainname_clear_errors(domainname_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+domainname_encode(uint8_t *output, const size_t avail, const domainname_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = domainname_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = domainname_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 len */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->len));
+  written += 1; ptr += 1;
+
+  /* Encode char name[len] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->name);
+    trunnel_assert(obj->len == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->name.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As domainname_parse(), but do not allocate the output object.
+ */
+static ssize_t
+domainname_parse_into(domainname_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 len */
+  CHECK_REMAINING(1, truncated);
+  obj->len = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse char name[len] */
+  CHECK_REMAINING(obj->len, truncated);
+  if (domainname_setstr0_name(obj, (const char*)ptr, obj->len))
+    goto fail;
+  ptr += obj->len; remaining -= obj->len;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+domainname_parse(domainname_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = domainname_new();
+  if (NULL == *output)
+    return -1;
+  result = domainname_parse_into(*output, input, len_in);
+  if (result < 0) {
+    domainname_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+socks4_client_request_t *
+socks4_client_request_new(void)
+{
+  socks4_client_request_t *val = trunnel_calloc(1, sizeof(socks4_client_request_t));
+  if (NULL == val)
+    return NULL;
+  val->version = 4;
+  val->command = CMD_BIND;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+socks4_client_request_clear(socks4_client_request_t *obj)
+{
+  (void) obj;
+  trunnel_wipestr(obj->username);
+  trunnel_free(obj->username);
+  trunnel_wipestr(obj->socks4a_addr_hostname);
+  trunnel_free(obj->socks4a_addr_hostname);
+}
+
+void
+socks4_client_request_free(socks4_client_request_t *obj)
+{
+  if (obj == NULL)
+    return;
+  socks4_client_request_clear(obj);
+  trunnel_memwipe(obj, sizeof(socks4_client_request_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+socks4_client_request_get_version(const socks4_client_request_t *inp)
+{
+  return inp->version;
+}
+int
+socks4_client_request_set_version(socks4_client_request_t *inp, uint8_t val)
+{
+  if (! ((val == 4))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint8_t
+socks4_client_request_get_command(const socks4_client_request_t *inp)
+{
+  return inp->command;
+}
+int
+socks4_client_request_set_command(socks4_client_request_t *inp, uint8_t val)
+{
+  if (! ((val == CMD_BIND || val == CMD_CONNECT || val == CMD_RESOLVE || val == CMD_RESOLVE_PTR))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->command = val;
+  return 0;
+}
+uint16_t
+socks4_client_request_get_port(const socks4_client_request_t *inp)
+{
+  return inp->port;
+}
+int
+socks4_client_request_set_port(socks4_client_request_t *inp, uint16_t val)
+{
+  inp->port = val;
+  return 0;
+}
+uint32_t
+socks4_client_request_get_addr(const socks4_client_request_t *inp)
+{
+  return inp->addr;
+}
+int
+socks4_client_request_set_addr(socks4_client_request_t *inp, uint32_t val)
+{
+  inp->addr = val;
+  return 0;
+}
+const char *
+socks4_client_request_get_username(const socks4_client_request_t *inp)
+{
+  return inp->username;
+}
+int
+socks4_client_request_set_username(socks4_client_request_t *inp, const char *val)
+{
+  trunnel_free(inp->username);
+  if (NULL == (inp->username = trunnel_strdup(val))) {
+    TRUNNEL_SET_ERROR_CODE(inp);
+    return -1;
+  }
+  return 0;
+}
+const char *
+socks4_client_request_get_socks4a_addr_hostname(const socks4_client_request_t *inp)
+{
+  return inp->socks4a_addr_hostname;
+}
+int
+socks4_client_request_set_socks4a_addr_hostname(socks4_client_request_t *inp, const char *val)
+{
+  trunnel_free(inp->socks4a_addr_hostname);
+  if (NULL == (inp->socks4a_addr_hostname = trunnel_strdup(val))) {
+    TRUNNEL_SET_ERROR_CODE(inp);
+    return -1;
+  }
+  return 0;
+}
+const char *
+socks4_client_request_check(const socks4_client_request_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 4))
+    return "Integer out of bounds";
+  if (! (obj->command == CMD_BIND || obj->command == CMD_CONNECT || obj->command == CMD_RESOLVE || obj->command == CMD_RESOLVE_PTR))
+    return "Integer out of bounds";
+  if (NULL == obj->username)
+    return "Missing username";
+  switch (obj->addr) {
+
+    case 1:
+    case 2:
+    case 3:
+    case 4:
+    case 5:
+    case 6:
+    case 7:
+    case 8:
+    case 9:
+    case 10:
+    case 11:
+    case 12:
+    case 13:
+    case 14:
+    case 15:
+    case 16:
+    case 17:
+    case 18:
+    case 19:
+    case 20:
+    case 21:
+    case 22:
+    case 23:
+    case 24:
+    case 25:
+    case 26:
+    case 27:
+    case 28:
+    case 29:
+    case 30:
+    case 31:
+    case 32:
+    case 33:
+    case 34:
+    case 35:
+    case 36:
+    case 37:
+    case 38:
+    case 39:
+    case 40:
+    case 41:
+    case 42:
+    case 43:
+    case 44:
+    case 45:
+    case 46:
+    case 47:
+    case 48:
+    case 49:
+    case 50:
+    case 51:
+    case 52:
+    case 53:
+    case 54:
+    case 55:
+    case 56:
+    case 57:
+    case 58:
+    case 59:
+    case 60:
+    case 61:
+    case 62:
+    case 63:
+    case 64:
+    case 65:
+    case 66:
+    case 67:
+    case 68:
+    case 69:
+    case 70:
+    case 71:
+    case 72:
+    case 73:
+    case 74:
+    case 75:
+    case 76:
+    case 77:
+    case 78:
+    case 79:
+    case 80:
+    case 81:
+    case 82:
+    case 83:
+    case 84:
+    case 85:
+    case 86:
+    case 87:
+    case 88:
+    case 89:
+    case 90:
+    case 91:
+    case 92:
+    case 93:
+    case 94:
+    case 95:
+    case 96:
+    case 97:
+    case 98:
+    case 99:
+    case 100:
+    case 101:
+    case 102:
+    case 103:
+    case 104:
+    case 105:
+    case 106:
+    case 107:
+    case 108:
+    case 109:
+    case 110:
+    case 111:
+    case 112:
+    case 113:
+    case 114:
+    case 115:
+    case 116:
+    case 117:
+    case 118:
+    case 119:
+    case 120:
+    case 121:
+    case 122:
+    case 123:
+    case 124:
+    case 125:
+    case 126:
+    case 127:
+    case 128:
+    case 129:
+    case 130:
+    case 131:
+    case 132:
+    case 133:
+    case 134:
+    case 135:
+    case 136:
+    case 137:
+    case 138:
+    case 139:
+    case 140:
+    case 141:
+    case 142:
+    case 143:
+    case 144:
+    case 145:
+    case 146:
+    case 147:
+    case 148:
+    case 149:
+    case 150:
+    case 151:
+    case 152:
+    case 153:
+    case 154:
+    case 155:
+    case 156:
+    case 157:
+    case 158:
+    case 159:
+    case 160:
+    case 161:
+    case 162:
+    case 163:
+    case 164:
+    case 165:
+    case 166:
+    case 167:
+    case 168:
+    case 169:
+    case 170:
+    case 171:
+    case 172:
+    case 173:
+    case 174:
+    case 175:
+    case 176:
+    case 177:
+    case 178:
+    case 179:
+    case 180:
+    case 181:
+    case 182:
+    case 183:
+    case 184:
+    case 185:
+    case 186:
+    case 187:
+    case 188:
+    case 189:
+    case 190:
+    case 191:
+    case 192:
+    case 193:
+    case 194:
+    case 195:
+    case 196:
+    case 197:
+    case 198:
+    case 199:
+    case 200:
+    case 201:
+    case 202:
+    case 203:
+    case 204:
+    case 205:
+    case 206:
+    case 207:
+    case 208:
+    case 209:
+    case 210:
+    case 211:
+    case 212:
+    case 213:
+    case 214:
+    case 215:
+    case 216:
+    case 217:
+    case 218:
+    case 219:
+    case 220:
+    case 221:
+    case 222:
+    case 223:
+    case 224:
+    case 225:
+    case 226:
+    case 227:
+    case 228:
+    case 229:
+    case 230:
+    case 231:
+    case 232:
+    case 233:
+    case 234:
+    case 235:
+    case 236:
+    case 237:
+    case 238:
+    case 239:
+    case 240:
+    case 241:
+    case 242:
+    case 243:
+    case 244:
+    case 245:
+    case 246:
+    case 247:
+    case 248:
+    case 249:
+    case 250:
+    case 251:
+    case 252:
+    case 253:
+    case 254:
+    case 255:
+      if (NULL == obj->socks4a_addr_hostname)
+        return "Missing socks4a_addr_hostname";
+      break;
+
+    default:
+      break;
+  }
+  return NULL;
+}
+
+ssize_t
+socks4_client_request_encoded_len(const socks4_client_request_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != socks4_client_request_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [4] */
+  result += 1;
+
+  /* Length of u8 command IN [CMD_BIND, CMD_CONNECT, CMD_RESOLVE, CMD_RESOLVE_PTR] */
+  result += 1;
+
+  /* Length of u16 port */
+  result += 2;
+
+  /* Length of u32 addr */
+  result += 4;
+
+  /* Length of nulterm username */
+  result += strlen(obj->username) + 1;
+  switch (obj->addr) {
+
+    case 1:
+    case 2:
+    case 3:
+    case 4:
+    case 5:
+    case 6:
+    case 7:
+    case 8:
+    case 9:
+    case 10:
+    case 11:
+    case 12:
+    case 13:
+    case 14:
+    case 15:
+    case 16:
+    case 17:
+    case 18:
+    case 19:
+    case 20:
+    case 21:
+    case 22:
+    case 23:
+    case 24:
+    case 25:
+    case 26:
+    case 27:
+    case 28:
+    case 29:
+    case 30:
+    case 31:
+    case 32:
+    case 33:
+    case 34:
+    case 35:
+    case 36:
+    case 37:
+    case 38:
+    case 39:
+    case 40:
+    case 41:
+    case 42:
+    case 43:
+    case 44:
+    case 45:
+    case 46:
+    case 47:
+    case 48:
+    case 49:
+    case 50:
+    case 51:
+    case 52:
+    case 53:
+    case 54:
+    case 55:
+    case 56:
+    case 57:
+    case 58:
+    case 59:
+    case 60:
+    case 61:
+    case 62:
+    case 63:
+    case 64:
+    case 65:
+    case 66:
+    case 67:
+    case 68:
+    case 69:
+    case 70:
+    case 71:
+    case 72:
+    case 73:
+    case 74:
+    case 75:
+    case 76:
+    case 77:
+    case 78:
+    case 79:
+    case 80:
+    case 81:
+    case 82:
+    case 83:
+    case 84:
+    case 85:
+    case 86:
+    case 87:
+    case 88:
+    case 89:
+    case 90:
+    case 91:
+    case 92:
+    case 93:
+    case 94:
+    case 95:
+    case 96:
+    case 97:
+    case 98:
+    case 99:
+    case 100:
+    case 101:
+    case 102:
+    case 103:
+    case 104:
+    case 105:
+    case 106:
+    case 107:
+    case 108:
+    case 109:
+    case 110:
+    case 111:
+    case 112:
+    case 113:
+    case 114:
+    case 115:
+    case 116:
+    case 117:
+    case 118:
+    case 119:
+    case 120:
+    case 121:
+    case 122:
+    case 123:
+    case 124:
+    case 125:
+    case 126:
+    case 127:
+    case 128:
+    case 129:
+    case 130:
+    case 131:
+    case 132:
+    case 133:
+    case 134:
+    case 135:
+    case 136:
+    case 137:
+    case 138:
+    case 139:
+    case 140:
+    case 141:
+    case 142:
+    case 143:
+    case 144:
+    case 145:
+    case 146:
+    case 147:
+    case 148:
+    case 149:
+    case 150:
+    case 151:
+    case 152:
+    case 153:
+    case 154:
+    case 155:
+    case 156:
+    case 157:
+    case 158:
+    case 159:
+    case 160:
+    case 161:
+    case 162:
+    case 163:
+    case 164:
+    case 165:
+    case 166:
+    case 167:
+    case 168:
+    case 169:
+    case 170:
+    case 171:
+    case 172:
+    case 173:
+    case 174:
+    case 175:
+    case 176:
+    case 177:
+    case 178:
+    case 179:
+    case 180:
+    case 181:
+    case 182:
+    case 183:
+    case 184:
+    case 185:
+    case 186:
+    case 187:
+    case 188:
+    case 189:
+    case 190:
+    case 191:
+    case 192:
+    case 193:
+    case 194:
+    case 195:
+    case 196:
+    case 197:
+    case 198:
+    case 199:
+    case 200:
+    case 201:
+    case 202:
+    case 203:
+    case 204:
+    case 205:
+    case 206:
+    case 207:
+    case 208:
+    case 209:
+    case 210:
+    case 211:
+    case 212:
+    case 213:
+    case 214:
+    case 215:
+    case 216:
+    case 217:
+    case 218:
+    case 219:
+    case 220:
+    case 221:
+    case 222:
+    case 223:
+    case 224:
+    case 225:
+    case 226:
+    case 227:
+    case 228:
+    case 229:
+    case 230:
+    case 231:
+    case 232:
+    case 233:
+    case 234:
+    case 235:
+    case 236:
+    case 237:
+    case 238:
+    case 239:
+    case 240:
+    case 241:
+    case 242:
+    case 243:
+    case 244:
+    case 245:
+    case 246:
+    case 247:
+    case 248:
+    case 249:
+    case 250:
+    case 251:
+    case 252:
+    case 253:
+    case 254:
+    case 255:
+
+      /* Length of nulterm socks4a_addr_hostname */
+      result += strlen(obj->socks4a_addr_hostname) + 1;
+      break;
+
+    default:
+      break;
+  }
+  return result;
+}
+int
+socks4_client_request_clear_errors(socks4_client_request_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+socks4_client_request_encode(uint8_t *output, const size_t avail, const socks4_client_request_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = socks4_client_request_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = socks4_client_request_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [4] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u8 command IN [CMD_BIND, CMD_CONNECT, CMD_RESOLVE, CMD_RESOLVE_PTR] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->command));
+  written += 1; ptr += 1;
+
+  /* Encode u16 port */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->port));
+  written += 2; ptr += 2;
+
+  /* Encode u32 addr */
+  trunnel_assert(written <= avail);
+  if (avail - written < 4)
+    goto truncated;
+  trunnel_set_uint32(ptr, trunnel_htonl(obj->addr));
+  written += 4; ptr += 4;
+
+  /* Encode nulterm username */
+  {
+    size_t len = strlen(obj->username);
+    trunnel_assert(written <= avail);
+    if (avail - written < len + 1)
+      goto truncated;
+    memcpy(ptr, obj->username, len + 1);
+    ptr += len + 1; written += len + 1;
+  }
+
+  /* Encode union socks4a_addr[addr] */
+  trunnel_assert(written <= avail);
+  switch (obj->addr) {
+
+    case 1:
+    case 2:
+    case 3:
+    case 4:
+    case 5:
+    case 6:
+    case 7:
+    case 8:
+    case 9:
+    case 10:
+    case 11:
+    case 12:
+    case 13:
+    case 14:
+    case 15:
+    case 16:
+    case 17:
+    case 18:
+    case 19:
+    case 20:
+    case 21:
+    case 22:
+    case 23:
+    case 24:
+    case 25:
+    case 26:
+    case 27:
+    case 28:
+    case 29:
+    case 30:
+    case 31:
+    case 32:
+    case 33:
+    case 34:
+    case 35:
+    case 36:
+    case 37:
+    case 38:
+    case 39:
+    case 40:
+    case 41:
+    case 42:
+    case 43:
+    case 44:
+    case 45:
+    case 46:
+    case 47:
+    case 48:
+    case 49:
+    case 50:
+    case 51:
+    case 52:
+    case 53:
+    case 54:
+    case 55:
+    case 56:
+    case 57:
+    case 58:
+    case 59:
+    case 60:
+    case 61:
+    case 62:
+    case 63:
+    case 64:
+    case 65:
+    case 66:
+    case 67:
+    case 68:
+    case 69:
+    case 70:
+    case 71:
+    case 72:
+    case 73:
+    case 74:
+    case 75:
+    case 76:
+    case 77:
+    case 78:
+    case 79:
+    case 80:
+    case 81:
+    case 82:
+    case 83:
+    case 84:
+    case 85:
+    case 86:
+    case 87:
+    case 88:
+    case 89:
+    case 90:
+    case 91:
+    case 92:
+    case 93:
+    case 94:
+    case 95:
+    case 96:
+    case 97:
+    case 98:
+    case 99:
+    case 100:
+    case 101:
+    case 102:
+    case 103:
+    case 104:
+    case 105:
+    case 106:
+    case 107:
+    case 108:
+    case 109:
+    case 110:
+    case 111:
+    case 112:
+    case 113:
+    case 114:
+    case 115:
+    case 116:
+    case 117:
+    case 118:
+    case 119:
+    case 120:
+    case 121:
+    case 122:
+    case 123:
+    case 124:
+    case 125:
+    case 126:
+    case 127:
+    case 128:
+    case 129:
+    case 130:
+    case 131:
+    case 132:
+    case 133:
+    case 134:
+    case 135:
+    case 136:
+    case 137:
+    case 138:
+    case 139:
+    case 140:
+    case 141:
+    case 142:
+    case 143:
+    case 144:
+    case 145:
+    case 146:
+    case 147:
+    case 148:
+    case 149:
+    case 150:
+    case 151:
+    case 152:
+    case 153:
+    case 154:
+    case 155:
+    case 156:
+    case 157:
+    case 158:
+    case 159:
+    case 160:
+    case 161:
+    case 162:
+    case 163:
+    case 164:
+    case 165:
+    case 166:
+    case 167:
+    case 168:
+    case 169:
+    case 170:
+    case 171:
+    case 172:
+    case 173:
+    case 174:
+    case 175:
+    case 176:
+    case 177:
+    case 178:
+    case 179:
+    case 180:
+    case 181:
+    case 182:
+    case 183:
+    case 184:
+    case 185:
+    case 186:
+    case 187:
+    case 188:
+    case 189:
+    case 190:
+    case 191:
+    case 192:
+    case 193:
+    case 194:
+    case 195:
+    case 196:
+    case 197:
+    case 198:
+    case 199:
+    case 200:
+    case 201:
+    case 202:
+    case 203:
+    case 204:
+    case 205:
+    case 206:
+    case 207:
+    case 208:
+    case 209:
+    case 210:
+    case 211:
+    case 212:
+    case 213:
+    case 214:
+    case 215:
+    case 216:
+    case 217:
+    case 218:
+    case 219:
+    case 220:
+    case 221:
+    case 222:
+    case 223:
+    case 224:
+    case 225:
+    case 226:
+    case 227:
+    case 228:
+    case 229:
+    case 230:
+    case 231:
+    case 232:
+    case 233:
+    case 234:
+    case 235:
+    case 236:
+    case 237:
+    case 238:
+    case 239:
+    case 240:
+    case 241:
+    case 242:
+    case 243:
+    case 244:
+    case 245:
+    case 246:
+    case 247:
+    case 248:
+    case 249:
+    case 250:
+    case 251:
+    case 252:
+    case 253:
+    case 254:
+    case 255:
+
+      /* Encode nulterm socks4a_addr_hostname */
+      {
+        size_t len = strlen(obj->socks4a_addr_hostname);
+        trunnel_assert(written <= avail);
+        if (avail - written < len + 1)
+          goto truncated;
+        memcpy(ptr, obj->socks4a_addr_hostname, len + 1);
+        ptr += len + 1; written += len + 1;
+      }
+      break;
+
+    default:
+      break;
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As socks4_client_request_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+socks4_client_request_parse_into(socks4_client_request_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [4] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 4))
+    goto fail;
+
+  /* Parse u8 command IN [CMD_BIND, CMD_CONNECT, CMD_RESOLVE, CMD_RESOLVE_PTR] */
+  CHECK_REMAINING(1, truncated);
+  obj->command = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->command == CMD_BIND || obj->command == CMD_CONNECT || obj->command == CMD_RESOLVE || obj->command == CMD_RESOLVE_PTR))
+    goto fail;
+
+  /* Parse u16 port */
+  CHECK_REMAINING(2, truncated);
+  obj->port = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u32 addr */
+  CHECK_REMAINING(4, truncated);
+  obj->addr = trunnel_ntohl(trunnel_get_uint32(ptr));
+  remaining -= 4; ptr += 4;
+
+  /* Parse nulterm username */
+  {
+    uint8_t *eos = (uint8_t*)memchr(ptr, 0, remaining);
+    size_t memlen;
+    if (eos == NULL)
+      goto truncated;
+    trunnel_assert(eos >= ptr);
+    trunnel_assert((size_t)(eos - ptr) < SIZE_MAX - 1);
+    memlen = ((size_t)(eos - ptr)) + 1;
+    if (!(obj->username = trunnel_malloc(memlen)))
+      goto fail;
+    memcpy(obj->username, ptr, memlen);
+    remaining -= memlen; ptr += memlen;
+  }
+
+  /* Parse union socks4a_addr[addr] */
+  switch (obj->addr) {
+
+    case 1:
+    case 2:
+    case 3:
+    case 4:
+    case 5:
+    case 6:
+    case 7:
+    case 8:
+    case 9:
+    case 10:
+    case 11:
+    case 12:
+    case 13:
+    case 14:
+    case 15:
+    case 16:
+    case 17:
+    case 18:
+    case 19:
+    case 20:
+    case 21:
+    case 22:
+    case 23:
+    case 24:
+    case 25:
+    case 26:
+    case 27:
+    case 28:
+    case 29:
+    case 30:
+    case 31:
+    case 32:
+    case 33:
+    case 34:
+    case 35:
+    case 36:
+    case 37:
+    case 38:
+    case 39:
+    case 40:
+    case 41:
+    case 42:
+    case 43:
+    case 44:
+    case 45:
+    case 46:
+    case 47:
+    case 48:
+    case 49:
+    case 50:
+    case 51:
+    case 52:
+    case 53:
+    case 54:
+    case 55:
+    case 56:
+    case 57:
+    case 58:
+    case 59:
+    case 60:
+    case 61:
+    case 62:
+    case 63:
+    case 64:
+    case 65:
+    case 66:
+    case 67:
+    case 68:
+    case 69:
+    case 70:
+    case 71:
+    case 72:
+    case 73:
+    case 74:
+    case 75:
+    case 76:
+    case 77:
+    case 78:
+    case 79:
+    case 80:
+    case 81:
+    case 82:
+    case 83:
+    case 84:
+    case 85:
+    case 86:
+    case 87:
+    case 88:
+    case 89:
+    case 90:
+    case 91:
+    case 92:
+    case 93:
+    case 94:
+    case 95:
+    case 96:
+    case 97:
+    case 98:
+    case 99:
+    case 100:
+    case 101:
+    case 102:
+    case 103:
+    case 104:
+    case 105:
+    case 106:
+    case 107:
+    case 108:
+    case 109:
+    case 110:
+    case 111:
+    case 112:
+    case 113:
+    case 114:
+    case 115:
+    case 116:
+    case 117:
+    case 118:
+    case 119:
+    case 120:
+    case 121:
+    case 122:
+    case 123:
+    case 124:
+    case 125:
+    case 126:
+    case 127:
+    case 128:
+    case 129:
+    case 130:
+    case 131:
+    case 132:
+    case 133:
+    case 134:
+    case 135:
+    case 136:
+    case 137:
+    case 138:
+    case 139:
+    case 140:
+    case 141:
+    case 142:
+    case 143:
+    case 144:
+    case 145:
+    case 146:
+    case 147:
+    case 148:
+    case 149:
+    case 150:
+    case 151:
+    case 152:
+    case 153:
+    case 154:
+    case 155:
+    case 156:
+    case 157:
+    case 158:
+    case 159:
+    case 160:
+    case 161:
+    case 162:
+    case 163:
+    case 164:
+    case 165:
+    case 166:
+    case 167:
+    case 168:
+    case 169:
+    case 170:
+    case 171:
+    case 172:
+    case 173:
+    case 174:
+    case 175:
+    case 176:
+    case 177:
+    case 178:
+    case 179:
+    case 180:
+    case 181:
+    case 182:
+    case 183:
+    case 184:
+    case 185:
+    case 186:
+    case 187:
+    case 188:
+    case 189:
+    case 190:
+    case 191:
+    case 192:
+    case 193:
+    case 194:
+    case 195:
+    case 196:
+    case 197:
+    case 198:
+    case 199:
+    case 200:
+    case 201:
+    case 202:
+    case 203:
+    case 204:
+    case 205:
+    case 206:
+    case 207:
+    case 208:
+    case 209:
+    case 210:
+    case 211:
+    case 212:
+    case 213:
+    case 214:
+    case 215:
+    case 216:
+    case 217:
+    case 218:
+    case 219:
+    case 220:
+    case 221:
+    case 222:
+    case 223:
+    case 224:
+    case 225:
+    case 226:
+    case 227:
+    case 228:
+    case 229:
+    case 230:
+    case 231:
+    case 232:
+    case 233:
+    case 234:
+    case 235:
+    case 236:
+    case 237:
+    case 238:
+    case 239:
+    case 240:
+    case 241:
+    case 242:
+    case 243:
+    case 244:
+    case 245:
+    case 246:
+    case 247:
+    case 248:
+    case 249:
+    case 250:
+    case 251:
+    case 252:
+    case 253:
+    case 254:
+    case 255:
+
+      /* Parse nulterm socks4a_addr_hostname */
+      {
+        uint8_t *eos = (uint8_t*)memchr(ptr, 0, remaining);
+        size_t memlen;
+        if (eos == NULL)
+          goto truncated;
+        trunnel_assert(eos >= ptr);
+        trunnel_assert((size_t)(eos - ptr) < SIZE_MAX - 1);
+        memlen = ((size_t)(eos - ptr)) + 1;
+        if (!(obj->socks4a_addr_hostname = trunnel_malloc(memlen)))
+          goto fail;
+        memcpy(obj->socks4a_addr_hostname, ptr, memlen);
+        remaining -= memlen; ptr += memlen;
+      }
+      break;
+
+    default:
+      break;
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+socks4_client_request_parse(socks4_client_request_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = socks4_client_request_new();
+  if (NULL == *output)
+    return -1;
+  result = socks4_client_request_parse_into(*output, input, len_in);
+  if (result < 0) {
+    socks4_client_request_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+socks4_server_reply_t *
+socks4_server_reply_new(void)
+{
+  socks4_server_reply_t *val = trunnel_calloc(1, sizeof(socks4_server_reply_t));
+  if (NULL == val)
+    return NULL;
+  val->version = 4;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+socks4_server_reply_clear(socks4_server_reply_t *obj)
+{
+  (void) obj;
+}
+
+void
+socks4_server_reply_free(socks4_server_reply_t *obj)
+{
+  if (obj == NULL)
+    return;
+  socks4_server_reply_clear(obj);
+  trunnel_memwipe(obj, sizeof(socks4_server_reply_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+socks4_server_reply_get_version(const socks4_server_reply_t *inp)
+{
+  return inp->version;
+}
+int
+socks4_server_reply_set_version(socks4_server_reply_t *inp, uint8_t val)
+{
+  if (! ((val == 4))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint8_t
+socks4_server_reply_get_status(const socks4_server_reply_t *inp)
+{
+  return inp->status;
+}
+int
+socks4_server_reply_set_status(socks4_server_reply_t *inp, uint8_t val)
+{
+  inp->status = val;
+  return 0;
+}
+uint16_t
+socks4_server_reply_get_port(const socks4_server_reply_t *inp)
+{
+  return inp->port;
+}
+int
+socks4_server_reply_set_port(socks4_server_reply_t *inp, uint16_t val)
+{
+  inp->port = val;
+  return 0;
+}
+uint32_t
+socks4_server_reply_get_addr(const socks4_server_reply_t *inp)
+{
+  return inp->addr;
+}
+int
+socks4_server_reply_set_addr(socks4_server_reply_t *inp, uint32_t val)
+{
+  inp->addr = val;
+  return 0;
+}
+const char *
+socks4_server_reply_check(const socks4_server_reply_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 4))
+    return "Integer out of bounds";
+  return NULL;
+}
+
+ssize_t
+socks4_server_reply_encoded_len(const socks4_server_reply_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != socks4_server_reply_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [4] */
+  result += 1;
+
+  /* Length of u8 status */
+  result += 1;
+
+  /* Length of u16 port */
+  result += 2;
+
+  /* Length of u32 addr */
+  result += 4;
+  return result;
+}
+int
+socks4_server_reply_clear_errors(socks4_server_reply_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+socks4_server_reply_encode(uint8_t *output, const size_t avail, const socks4_server_reply_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = socks4_server_reply_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = socks4_server_reply_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [4] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u8 status */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->status));
+  written += 1; ptr += 1;
+
+  /* Encode u16 port */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->port));
+  written += 2; ptr += 2;
+
+  /* Encode u32 addr */
+  trunnel_assert(written <= avail);
+  if (avail - written < 4)
+    goto truncated;
+  trunnel_set_uint32(ptr, trunnel_htonl(obj->addr));
+  written += 4; ptr += 4;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As socks4_server_reply_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+socks4_server_reply_parse_into(socks4_server_reply_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [4] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 4))
+    goto fail;
+
+  /* Parse u8 status */
+  CHECK_REMAINING(1, truncated);
+  obj->status = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u16 port */
+  CHECK_REMAINING(2, truncated);
+  obj->port = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u32 addr */
+  CHECK_REMAINING(4, truncated);
+  obj->addr = trunnel_ntohl(trunnel_get_uint32(ptr));
+  remaining -= 4; ptr += 4;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+socks4_server_reply_parse(socks4_server_reply_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = socks4_server_reply_new();
+  if (NULL == *output)
+    return -1;
+  result = socks4_server_reply_parse_into(*output, input, len_in);
+  if (result < 0) {
+    socks4_server_reply_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+socks5_client_userpass_auth_t *
+socks5_client_userpass_auth_new(void)
+{
+  socks5_client_userpass_auth_t *val = trunnel_calloc(1, sizeof(socks5_client_userpass_auth_t));
+  if (NULL == val)
+    return NULL;
+  val->version = 1;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+socks5_client_userpass_auth_clear(socks5_client_userpass_auth_t *obj)
+{
+  (void) obj;
+  TRUNNEL_DYNARRAY_WIPE(&obj->username);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->username);
+  TRUNNEL_DYNARRAY_WIPE(&obj->passwd);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->passwd);
+}
+
+void
+socks5_client_userpass_auth_free(socks5_client_userpass_auth_t *obj)
+{
+  if (obj == NULL)
+    return;
+  socks5_client_userpass_auth_clear(obj);
+  trunnel_memwipe(obj, sizeof(socks5_client_userpass_auth_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+socks5_client_userpass_auth_get_version(const socks5_client_userpass_auth_t *inp)
+{
+  return inp->version;
+}
+int
+socks5_client_userpass_auth_set_version(socks5_client_userpass_auth_t *inp, uint8_t val)
+{
+  if (! ((val == 1))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint8_t
+socks5_client_userpass_auth_get_username_len(const socks5_client_userpass_auth_t *inp)
+{
+  return inp->username_len;
+}
+int
+socks5_client_userpass_auth_set_username_len(socks5_client_userpass_auth_t *inp, uint8_t val)
+{
+  inp->username_len = val;
+  return 0;
+}
+size_t
+socks5_client_userpass_auth_getlen_username(const socks5_client_userpass_auth_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->username);
+}
+
+char
+socks5_client_userpass_auth_get_username(socks5_client_userpass_auth_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->username, idx);
+}
+
+char
+socks5_client_userpass_auth_getconst_username(const socks5_client_userpass_auth_t *inp, size_t idx)
+{
+  return socks5_client_userpass_auth_get_username((socks5_client_userpass_auth_t*)inp, idx);
+}
+int
+socks5_client_userpass_auth_set_username(socks5_client_userpass_auth_t *inp, size_t idx, char elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->username, idx, elt);
+  return 0;
+}
+int
+socks5_client_userpass_auth_add_username(socks5_client_userpass_auth_t *inp, char elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->username.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(char, &inp->username, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+char *
+socks5_client_userpass_auth_getarray_username(socks5_client_userpass_auth_t *inp)
+{
+  return inp->username.elts_;
+}
+const char  *
+socks5_client_userpass_auth_getconstarray_username(const socks5_client_userpass_auth_t *inp)
+{
+  return (const char  *)socks5_client_userpass_auth_getarray_username((socks5_client_userpass_auth_t*)inp);
+}
+int
+socks5_client_userpass_auth_setlen_username(socks5_client_userpass_auth_t *inp, size_t newlen)
+{
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  return trunnel_string_setlen(&inp->username, newlen,
+            &inp->trunnel_error_code_);
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+socks5_client_userpass_auth_getstr_username(socks5_client_userpass_auth_t *inp)
+{
+  return trunnel_string_getstr(&inp->username);
+}
+int
+socks5_client_userpass_auth_setstr0_username(socks5_client_userpass_auth_t *inp, const char *val, size_t len)
+{
+#if UINT8_MAX < SIZE_MAX
+  if (len > UINT8_MAX) {
+    TRUNNEL_SET_ERROR_CODE(inp);
+    return -1;
+  }
+#endif
+  return trunnel_string_setstr0(&inp->username, val, len, &inp->trunnel_error_code_);
+}
+int
+socks5_client_userpass_auth_setstr_username(socks5_client_userpass_auth_t *inp, const char *val)
+{
+  return socks5_client_userpass_auth_setstr0_username(inp, val, strlen(val));
+}
+uint8_t
+socks5_client_userpass_auth_get_passwd_len(const socks5_client_userpass_auth_t *inp)
+{
+  return inp->passwd_len;
+}
+int
+socks5_client_userpass_auth_set_passwd_len(socks5_client_userpass_auth_t *inp, uint8_t val)
+{
+  inp->passwd_len = val;
+  return 0;
+}
+size_t
+socks5_client_userpass_auth_getlen_passwd(const socks5_client_userpass_auth_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->passwd);
+}
+
+char
+socks5_client_userpass_auth_get_passwd(socks5_client_userpass_auth_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->passwd, idx);
+}
+
+char
+socks5_client_userpass_auth_getconst_passwd(const socks5_client_userpass_auth_t *inp, size_t idx)
+{
+  return socks5_client_userpass_auth_get_passwd((socks5_client_userpass_auth_t*)inp, idx);
+}
+int
+socks5_client_userpass_auth_set_passwd(socks5_client_userpass_auth_t *inp, size_t idx, char elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->passwd, idx, elt);
+  return 0;
+}
+int
+socks5_client_userpass_auth_add_passwd(socks5_client_userpass_auth_t *inp, char elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->passwd.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(char, &inp->passwd, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+char *
+socks5_client_userpass_auth_getarray_passwd(socks5_client_userpass_auth_t *inp)
+{
+  return inp->passwd.elts_;
+}
+const char  *
+socks5_client_userpass_auth_getconstarray_passwd(const socks5_client_userpass_auth_t *inp)
+{
+  return (const char  *)socks5_client_userpass_auth_getarray_passwd((socks5_client_userpass_auth_t*)inp);
+}
+int
+socks5_client_userpass_auth_setlen_passwd(socks5_client_userpass_auth_t *inp, size_t newlen)
+{
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  return trunnel_string_setlen(&inp->passwd, newlen,
+            &inp->trunnel_error_code_);
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+socks5_client_userpass_auth_getstr_passwd(socks5_client_userpass_auth_t *inp)
+{
+  return trunnel_string_getstr(&inp->passwd);
+}
+int
+socks5_client_userpass_auth_setstr0_passwd(socks5_client_userpass_auth_t *inp, const char *val, size_t len)
+{
+#if UINT8_MAX < SIZE_MAX
+  if (len > UINT8_MAX) {
+    TRUNNEL_SET_ERROR_CODE(inp);
+    return -1;
+  }
+#endif
+  return trunnel_string_setstr0(&inp->passwd, val, len, &inp->trunnel_error_code_);
+}
+int
+socks5_client_userpass_auth_setstr_passwd(socks5_client_userpass_auth_t *inp, const char *val)
+{
+  return socks5_client_userpass_auth_setstr0_passwd(inp, val, strlen(val));
+}
+const char *
+socks5_client_userpass_auth_check(const socks5_client_userpass_auth_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 1))
+    return "Integer out of bounds";
+  if (TRUNNEL_DYNARRAY_LEN(&obj->username) != obj->username_len)
+    return "Length mismatch for username";
+  if (TRUNNEL_DYNARRAY_LEN(&obj->passwd) != obj->passwd_len)
+    return "Length mismatch for passwd";
+  return NULL;
+}
+
+ssize_t
+socks5_client_userpass_auth_encoded_len(const socks5_client_userpass_auth_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != socks5_client_userpass_auth_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [1] */
+  result += 1;
+
+  /* Length of u8 username_len */
+  result += 1;
+
+  /* Length of char username[username_len] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->username);
+
+  /* Length of u8 passwd_len */
+  result += 1;
+
+  /* Length of char passwd[passwd_len] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->passwd);
+  return result;
+}
+int
+socks5_client_userpass_auth_clear_errors(socks5_client_userpass_auth_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+socks5_client_userpass_auth_encode(uint8_t *output, const size_t avail, const socks5_client_userpass_auth_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = socks5_client_userpass_auth_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = socks5_client_userpass_auth_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [1] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u8 username_len */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->username_len));
+  written += 1; ptr += 1;
+
+  /* Encode char username[username_len] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->username);
+    trunnel_assert(obj->username_len == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->username.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+  /* Encode u8 passwd_len */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->passwd_len));
+  written += 1; ptr += 1;
+
+  /* Encode char passwd[passwd_len] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->passwd);
+    trunnel_assert(obj->passwd_len == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->passwd.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As socks5_client_userpass_auth_parse(), but do not allocate the
+ * output object.
+ */
+static ssize_t
+socks5_client_userpass_auth_parse_into(socks5_client_userpass_auth_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [1] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 1))
+    goto fail;
+
+  /* Parse u8 username_len */
+  CHECK_REMAINING(1, truncated);
+  obj->username_len = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse char username[username_len] */
+  CHECK_REMAINING(obj->username_len, truncated);
+  if (socks5_client_userpass_auth_setstr0_username(obj, (const char*)ptr, obj->username_len))
+    goto fail;
+  ptr += obj->username_len; remaining -= obj->username_len;
+
+  /* Parse u8 passwd_len */
+  CHECK_REMAINING(1, truncated);
+  obj->passwd_len = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse char passwd[passwd_len] */
+  CHECK_REMAINING(obj->passwd_len, truncated);
+  if (socks5_client_userpass_auth_setstr0_passwd(obj, (const char*)ptr, obj->passwd_len))
+    goto fail;
+  ptr += obj->passwd_len; remaining -= obj->passwd_len;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+socks5_client_userpass_auth_parse(socks5_client_userpass_auth_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = socks5_client_userpass_auth_new();
+  if (NULL == *output)
+    return -1;
+  result = socks5_client_userpass_auth_parse_into(*output, input, len_in);
+  if (result < 0) {
+    socks5_client_userpass_auth_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+socks5_client_version_t *
+socks5_client_version_new(void)
+{
+  socks5_client_version_t *val = trunnel_calloc(1, sizeof(socks5_client_version_t));
+  if (NULL == val)
+    return NULL;
+  val->version = 5;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+socks5_client_version_clear(socks5_client_version_t *obj)
+{
+  (void) obj;
+  TRUNNEL_DYNARRAY_WIPE(&obj->methods);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->methods);
+}
+
+void
+socks5_client_version_free(socks5_client_version_t *obj)
+{
+  if (obj == NULL)
+    return;
+  socks5_client_version_clear(obj);
+  trunnel_memwipe(obj, sizeof(socks5_client_version_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+socks5_client_version_get_version(const socks5_client_version_t *inp)
+{
+  return inp->version;
+}
+int
+socks5_client_version_set_version(socks5_client_version_t *inp, uint8_t val)
+{
+  if (! ((val == 5))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint8_t
+socks5_client_version_get_n_methods(const socks5_client_version_t *inp)
+{
+  return inp->n_methods;
+}
+int
+socks5_client_version_set_n_methods(socks5_client_version_t *inp, uint8_t val)
+{
+  inp->n_methods = val;
+  return 0;
+}
+size_t
+socks5_client_version_getlen_methods(const socks5_client_version_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->methods);
+}
+
+uint8_t
+socks5_client_version_get_methods(socks5_client_version_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->methods, idx);
+}
+
+uint8_t
+socks5_client_version_getconst_methods(const socks5_client_version_t *inp, size_t idx)
+{
+  return socks5_client_version_get_methods((socks5_client_version_t*)inp, idx);
+}
+int
+socks5_client_version_set_methods(socks5_client_version_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->methods, idx, elt);
+  return 0;
+}
+int
+socks5_client_version_add_methods(socks5_client_version_t *inp, uint8_t elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->methods.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->methods, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+socks5_client_version_getarray_methods(socks5_client_version_t *inp)
+{
+  return inp->methods.elts_;
+}
+const uint8_t  *
+socks5_client_version_getconstarray_methods(const socks5_client_version_t *inp)
+{
+  return (const uint8_t  *)socks5_client_version_getarray_methods((socks5_client_version_t*)inp);
+}
+int
+socks5_client_version_setlen_methods(socks5_client_version_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->methods.allocated_,
+                 &inp->methods.n_, inp->methods.elts_, newlen,
+                 sizeof(inp->methods.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->methods.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+socks5_client_version_check(const socks5_client_version_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 5))
+    return "Integer out of bounds";
+  if (TRUNNEL_DYNARRAY_LEN(&obj->methods) != obj->n_methods)
+    return "Length mismatch for methods";
+  return NULL;
+}
+
+ssize_t
+socks5_client_version_encoded_len(const socks5_client_version_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != socks5_client_version_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [5] */
+  result += 1;
+
+  /* Length of u8 n_methods */
+  result += 1;
+
+  /* Length of u8 methods[n_methods] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->methods);
+  return result;
+}
+int
+socks5_client_version_clear_errors(socks5_client_version_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+socks5_client_version_encode(uint8_t *output, const size_t avail, const socks5_client_version_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = socks5_client_version_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = socks5_client_version_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [5] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u8 n_methods */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->n_methods));
+  written += 1; ptr += 1;
+
+  /* Encode u8 methods[n_methods] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->methods);
+    trunnel_assert(obj->n_methods == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->methods.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As socks5_client_version_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+socks5_client_version_parse_into(socks5_client_version_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [5] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 5))
+    goto fail;
+
+  /* Parse u8 n_methods */
+  CHECK_REMAINING(1, truncated);
+  obj->n_methods = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u8 methods[n_methods] */
+  CHECK_REMAINING(obj->n_methods, truncated);
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->methods, obj->n_methods, {});
+  obj->methods.n_ = obj->n_methods;
+  if (obj->n_methods)
+    memcpy(obj->methods.elts_, ptr, obj->n_methods);
+  ptr += obj->n_methods; remaining -= obj->n_methods;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ trunnel_alloc_failed:
+  return -1;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+socks5_client_version_parse(socks5_client_version_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = socks5_client_version_new();
+  if (NULL == *output)
+    return -1;
+  result = socks5_client_version_parse_into(*output, input, len_in);
+  if (result < 0) {
+    socks5_client_version_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+socks5_server_method_t *
+socks5_server_method_new(void)
+{
+  socks5_server_method_t *val = trunnel_calloc(1, sizeof(socks5_server_method_t));
+  if (NULL == val)
+    return NULL;
+  val->version = 5;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+socks5_server_method_clear(socks5_server_method_t *obj)
+{
+  (void) obj;
+}
+
+void
+socks5_server_method_free(socks5_server_method_t *obj)
+{
+  if (obj == NULL)
+    return;
+  socks5_server_method_clear(obj);
+  trunnel_memwipe(obj, sizeof(socks5_server_method_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+socks5_server_method_get_version(const socks5_server_method_t *inp)
+{
+  return inp->version;
+}
+int
+socks5_server_method_set_version(socks5_server_method_t *inp, uint8_t val)
+{
+  if (! ((val == 5))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint8_t
+socks5_server_method_get_method(const socks5_server_method_t *inp)
+{
+  return inp->method;
+}
+int
+socks5_server_method_set_method(socks5_server_method_t *inp, uint8_t val)
+{
+  inp->method = val;
+  return 0;
+}
+const char *
+socks5_server_method_check(const socks5_server_method_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 5))
+    return "Integer out of bounds";
+  return NULL;
+}
+
+ssize_t
+socks5_server_method_encoded_len(const socks5_server_method_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != socks5_server_method_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [5] */
+  result += 1;
+
+  /* Length of u8 method */
+  result += 1;
+  return result;
+}
+int
+socks5_server_method_clear_errors(socks5_server_method_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+socks5_server_method_encode(uint8_t *output, const size_t avail, const socks5_server_method_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = socks5_server_method_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = socks5_server_method_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [5] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u8 method */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->method));
+  written += 1; ptr += 1;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As socks5_server_method_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+socks5_server_method_parse_into(socks5_server_method_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [5] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 5))
+    goto fail;
+
+  /* Parse u8 method */
+  CHECK_REMAINING(1, truncated);
+  obj->method = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+socks5_server_method_parse(socks5_server_method_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = socks5_server_method_new();
+  if (NULL == *output)
+    return -1;
+  result = socks5_server_method_parse_into(*output, input, len_in);
+  if (result < 0) {
+    socks5_server_method_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+socks5_server_userpass_auth_t *
+socks5_server_userpass_auth_new(void)
+{
+  socks5_server_userpass_auth_t *val = trunnel_calloc(1, sizeof(socks5_server_userpass_auth_t));
+  if (NULL == val)
+    return NULL;
+  val->version = 1;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+socks5_server_userpass_auth_clear(socks5_server_userpass_auth_t *obj)
+{
+  (void) obj;
+}
+
+void
+socks5_server_userpass_auth_free(socks5_server_userpass_auth_t *obj)
+{
+  if (obj == NULL)
+    return;
+  socks5_server_userpass_auth_clear(obj);
+  trunnel_memwipe(obj, sizeof(socks5_server_userpass_auth_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+socks5_server_userpass_auth_get_version(const socks5_server_userpass_auth_t *inp)
+{
+  return inp->version;
+}
+int
+socks5_server_userpass_auth_set_version(socks5_server_userpass_auth_t *inp, uint8_t val)
+{
+  if (! ((val == 1))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint8_t
+socks5_server_userpass_auth_get_status(const socks5_server_userpass_auth_t *inp)
+{
+  return inp->status;
+}
+int
+socks5_server_userpass_auth_set_status(socks5_server_userpass_auth_t *inp, uint8_t val)
+{
+  inp->status = val;
+  return 0;
+}
+const char *
+socks5_server_userpass_auth_check(const socks5_server_userpass_auth_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 1))
+    return "Integer out of bounds";
+  return NULL;
+}
+
+ssize_t
+socks5_server_userpass_auth_encoded_len(const socks5_server_userpass_auth_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != socks5_server_userpass_auth_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [1] */
+  result += 1;
+
+  /* Length of u8 status */
+  result += 1;
+  return result;
+}
+int
+socks5_server_userpass_auth_clear_errors(socks5_server_userpass_auth_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+socks5_server_userpass_auth_encode(uint8_t *output, const size_t avail, const socks5_server_userpass_auth_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = socks5_server_userpass_auth_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = socks5_server_userpass_auth_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [1] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u8 status */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->status));
+  written += 1; ptr += 1;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As socks5_server_userpass_auth_parse(), but do not allocate the
+ * output object.
+ */
+static ssize_t
+socks5_server_userpass_auth_parse_into(socks5_server_userpass_auth_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [1] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 1))
+    goto fail;
+
+  /* Parse u8 status */
+  CHECK_REMAINING(1, truncated);
+  obj->status = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+socks5_server_userpass_auth_parse(socks5_server_userpass_auth_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = socks5_server_userpass_auth_new();
+  if (NULL == *output)
+    return -1;
+  result = socks5_server_userpass_auth_parse_into(*output, input, len_in);
+  if (result < 0) {
+    socks5_server_userpass_auth_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+socks5_client_request_t *
+socks5_client_request_new(void)
+{
+  socks5_client_request_t *val = trunnel_calloc(1, sizeof(socks5_client_request_t));
+  if (NULL == val)
+    return NULL;
+  val->version = 5;
+  val->command = CMD_BIND;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+socks5_client_request_clear(socks5_client_request_t *obj)
+{
+  (void) obj;
+  domainname_free(obj->dest_addr_domainname);
+  obj->dest_addr_domainname = NULL;
+}
+
+void
+socks5_client_request_free(socks5_client_request_t *obj)
+{
+  if (obj == NULL)
+    return;
+  socks5_client_request_clear(obj);
+  trunnel_memwipe(obj, sizeof(socks5_client_request_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+socks5_client_request_get_version(const socks5_client_request_t *inp)
+{
+  return inp->version;
+}
+int
+socks5_client_request_set_version(socks5_client_request_t *inp, uint8_t val)
+{
+  if (! ((val == 5))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint8_t
+socks5_client_request_get_command(const socks5_client_request_t *inp)
+{
+  return inp->command;
+}
+int
+socks5_client_request_set_command(socks5_client_request_t *inp, uint8_t val)
+{
+  if (! ((val == CMD_BIND || val == CMD_CONNECT || val == CMD_RESOLVE || val == CMD_RESOLVE_PTR || val == CMD_UDP_ASSOCIATE))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->command = val;
+  return 0;
+}
+uint8_t
+socks5_client_request_get_reserved(const socks5_client_request_t *inp)
+{
+  return inp->reserved;
+}
+int
+socks5_client_request_set_reserved(socks5_client_request_t *inp, uint8_t val)
+{
+  if (! ((val == 0))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->reserved = val;
+  return 0;
+}
+uint8_t
+socks5_client_request_get_atype(const socks5_client_request_t *inp)
+{
+  return inp->atype;
+}
+int
+socks5_client_request_set_atype(socks5_client_request_t *inp, uint8_t val)
+{
+  inp->atype = val;
+  return 0;
+}
+uint32_t
+socks5_client_request_get_dest_addr_ipv4(const socks5_client_request_t *inp)
+{
+  return inp->dest_addr_ipv4;
+}
+int
+socks5_client_request_set_dest_addr_ipv4(socks5_client_request_t *inp, uint32_t val)
+{
+  inp->dest_addr_ipv4 = val;
+  return 0;
+}
+size_t
+socks5_client_request_getlen_dest_addr_ipv6(const socks5_client_request_t *inp)
+{
+  (void)inp;  return 16;
+}
+
+uint8_t
+socks5_client_request_get_dest_addr_ipv6(socks5_client_request_t *inp, size_t idx)
+{
+  trunnel_assert(idx < 16);
+  return inp->dest_addr_ipv6[idx];
+}
+
+uint8_t
+socks5_client_request_getconst_dest_addr_ipv6(const socks5_client_request_t *inp, size_t idx)
+{
+  return socks5_client_request_get_dest_addr_ipv6((socks5_client_request_t*)inp, idx);
+}
+int
+socks5_client_request_set_dest_addr_ipv6(socks5_client_request_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < 16);
+  inp->dest_addr_ipv6[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+socks5_client_request_getarray_dest_addr_ipv6(socks5_client_request_t *inp)
+{
+  return inp->dest_addr_ipv6;
+}
+const uint8_t  *
+socks5_client_request_getconstarray_dest_addr_ipv6(const socks5_client_request_t *inp)
+{
+  return (const uint8_t  *)socks5_client_request_getarray_dest_addr_ipv6((socks5_client_request_t*)inp);
+}
+struct domainname_st *
+socks5_client_request_get_dest_addr_domainname(socks5_client_request_t *inp)
+{
+  return inp->dest_addr_domainname;
+}
+const struct domainname_st *
+socks5_client_request_getconst_dest_addr_domainname(const socks5_client_request_t *inp)
+{
+  return socks5_client_request_get_dest_addr_domainname((socks5_client_request_t*) inp);
+}
+int
+socks5_client_request_set_dest_addr_domainname(socks5_client_request_t *inp, struct domainname_st *val)
+{
+  if (inp->dest_addr_domainname && inp->dest_addr_domainname != val)
+    domainname_free(inp->dest_addr_domainname);
+  return socks5_client_request_set0_dest_addr_domainname(inp, val);
+}
+int
+socks5_client_request_set0_dest_addr_domainname(socks5_client_request_t *inp, struct domainname_st *val)
+{
+  inp->dest_addr_domainname = val;
+  return 0;
+}
+uint16_t
+socks5_client_request_get_dest_port(const socks5_client_request_t *inp)
+{
+  return inp->dest_port;
+}
+int
+socks5_client_request_set_dest_port(socks5_client_request_t *inp, uint16_t val)
+{
+  inp->dest_port = val;
+  return 0;
+}
+const char *
+socks5_client_request_check(const socks5_client_request_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 5))
+    return "Integer out of bounds";
+  if (! (obj->command == CMD_BIND || obj->command == CMD_CONNECT || obj->command == CMD_RESOLVE || obj->command == CMD_RESOLVE_PTR || obj->command == CMD_UDP_ASSOCIATE))
+    return "Integer out of bounds";
+  if (! (obj->reserved == 0))
+    return "Integer out of bounds";
+  switch (obj->atype) {
+
+    case ATYPE_IPV4:
+      break;
+
+    case ATYPE_IPV6:
+      break;
+
+    case ATYPE_DOMAINNAME:
+      {
+        const char *msg;
+        if (NULL != (msg = domainname_check(obj->dest_addr_domainname)))
+          return msg;
+      }
+      break;
+
+    default:
+        return "Bad tag for union";
+      break;
+  }
+  return NULL;
+}
+
+ssize_t
+socks5_client_request_encoded_len(const socks5_client_request_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != socks5_client_request_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [5] */
+  result += 1;
+
+  /* Length of u8 command IN [CMD_BIND, CMD_CONNECT, CMD_RESOLVE, CMD_RESOLVE_PTR, CMD_UDP_ASSOCIATE] */
+  result += 1;
+
+  /* Length of u8 reserved IN [0] */
+  result += 1;
+
+  /* Length of u8 atype */
+  result += 1;
+  switch (obj->atype) {
+
+    case ATYPE_IPV4:
+
+      /* Length of u32 dest_addr_ipv4 */
+      result += 4;
+      break;
+
+    case ATYPE_IPV6:
+
+      /* Length of u8 dest_addr_ipv6[16] */
+      result += 16;
+      break;
+
+    case ATYPE_DOMAINNAME:
+
+      /* Length of struct domainname dest_addr_domainname */
+      result += domainname_encoded_len(obj->dest_addr_domainname);
+      break;
+
+    default:
+      trunnel_assert(0);
+      break;
+  }
+
+  /* Length of u16 dest_port */
+  result += 2;
+  return result;
+}
+int
+socks5_client_request_clear_errors(socks5_client_request_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+socks5_client_request_encode(uint8_t *output, const size_t avail, const socks5_client_request_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = socks5_client_request_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = socks5_client_request_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [5] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u8 command IN [CMD_BIND, CMD_CONNECT, CMD_RESOLVE, CMD_RESOLVE_PTR, CMD_UDP_ASSOCIATE] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->command));
+  written += 1; ptr += 1;
+
+  /* Encode u8 reserved IN [0] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->reserved));
+  written += 1; ptr += 1;
+
+  /* Encode u8 atype */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->atype));
+  written += 1; ptr += 1;
+
+  /* Encode union dest_addr[atype] */
+  trunnel_assert(written <= avail);
+  switch (obj->atype) {
+
+    case ATYPE_IPV4:
+
+      /* Encode u32 dest_addr_ipv4 */
+      trunnel_assert(written <= avail);
+      if (avail - written < 4)
+        goto truncated;
+      trunnel_set_uint32(ptr, trunnel_htonl(obj->dest_addr_ipv4));
+      written += 4; ptr += 4;
+      break;
+
+    case ATYPE_IPV6:
+
+      /* Encode u8 dest_addr_ipv6[16] */
+      trunnel_assert(written <= avail);
+      if (avail - written < 16)
+        goto truncated;
+      memcpy(ptr, obj->dest_addr_ipv6, 16);
+      written += 16; ptr += 16;
+      break;
+
+    case ATYPE_DOMAINNAME:
+
+      /* Encode struct domainname dest_addr_domainname */
+      trunnel_assert(written <= avail);
+      result = domainname_encode(ptr, avail - written, obj->dest_addr_domainname);
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+      break;
+
+    default:
+      trunnel_assert(0);
+      break;
+  }
+
+  /* Encode u16 dest_port */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->dest_port));
+  written += 2; ptr += 2;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As socks5_client_request_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+socks5_client_request_parse_into(socks5_client_request_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [5] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 5))
+    goto fail;
+
+  /* Parse u8 command IN [CMD_BIND, CMD_CONNECT, CMD_RESOLVE, CMD_RESOLVE_PTR, CMD_UDP_ASSOCIATE] */
+  CHECK_REMAINING(1, truncated);
+  obj->command = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->command == CMD_BIND || obj->command == CMD_CONNECT || obj->command == CMD_RESOLVE || obj->command == CMD_RESOLVE_PTR || obj->command == CMD_UDP_ASSOCIATE))
+    goto fail;
+
+  /* Parse u8 reserved IN [0] */
+  CHECK_REMAINING(1, truncated);
+  obj->reserved = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->reserved == 0))
+    goto fail;
+
+  /* Parse u8 atype */
+  CHECK_REMAINING(1, truncated);
+  obj->atype = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse union dest_addr[atype] */
+  switch (obj->atype) {
+
+    case ATYPE_IPV4:
+
+      /* Parse u32 dest_addr_ipv4 */
+      CHECK_REMAINING(4, truncated);
+      obj->dest_addr_ipv4 = trunnel_ntohl(trunnel_get_uint32(ptr));
+      remaining -= 4; ptr += 4;
+      break;
+
+    case ATYPE_IPV6:
+
+      /* Parse u8 dest_addr_ipv6[16] */
+      CHECK_REMAINING(16, truncated);
+      memcpy(obj->dest_addr_ipv6, ptr, 16);
+      remaining -= 16; ptr += 16;
+      break;
+
+    case ATYPE_DOMAINNAME:
+
+      /* Parse struct domainname dest_addr_domainname */
+      result = domainname_parse(&obj->dest_addr_domainname, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      break;
+
+    default:
+      goto fail;
+      break;
+  }
+
+  /* Parse u16 dest_port */
+  CHECK_REMAINING(2, truncated);
+  obj->dest_port = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+socks5_client_request_parse(socks5_client_request_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = socks5_client_request_new();
+  if (NULL == *output)
+    return -1;
+  result = socks5_client_request_parse_into(*output, input, len_in);
+  if (result < 0) {
+    socks5_client_request_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+socks5_server_reply_t *
+socks5_server_reply_new(void)
+{
+  socks5_server_reply_t *val = trunnel_calloc(1, sizeof(socks5_server_reply_t));
+  if (NULL == val)
+    return NULL;
+  val->version = 5;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+socks5_server_reply_clear(socks5_server_reply_t *obj)
+{
+  (void) obj;
+  domainname_free(obj->bind_addr_domainname);
+  obj->bind_addr_domainname = NULL;
+}
+
+void
+socks5_server_reply_free(socks5_server_reply_t *obj)
+{
+  if (obj == NULL)
+    return;
+  socks5_server_reply_clear(obj);
+  trunnel_memwipe(obj, sizeof(socks5_server_reply_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+socks5_server_reply_get_version(const socks5_server_reply_t *inp)
+{
+  return inp->version;
+}
+int
+socks5_server_reply_set_version(socks5_server_reply_t *inp, uint8_t val)
+{
+  if (! ((val == 5))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint8_t
+socks5_server_reply_get_reply(const socks5_server_reply_t *inp)
+{
+  return inp->reply;
+}
+int
+socks5_server_reply_set_reply(socks5_server_reply_t *inp, uint8_t val)
+{
+  inp->reply = val;
+  return 0;
+}
+uint8_t
+socks5_server_reply_get_reserved(const socks5_server_reply_t *inp)
+{
+  return inp->reserved;
+}
+int
+socks5_server_reply_set_reserved(socks5_server_reply_t *inp, uint8_t val)
+{
+  if (! ((val == 0))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->reserved = val;
+  return 0;
+}
+uint8_t
+socks5_server_reply_get_atype(const socks5_server_reply_t *inp)
+{
+  return inp->atype;
+}
+int
+socks5_server_reply_set_atype(socks5_server_reply_t *inp, uint8_t val)
+{
+  inp->atype = val;
+  return 0;
+}
+uint32_t
+socks5_server_reply_get_bind_addr_ipv4(const socks5_server_reply_t *inp)
+{
+  return inp->bind_addr_ipv4;
+}
+int
+socks5_server_reply_set_bind_addr_ipv4(socks5_server_reply_t *inp, uint32_t val)
+{
+  inp->bind_addr_ipv4 = val;
+  return 0;
+}
+size_t
+socks5_server_reply_getlen_bind_addr_ipv6(const socks5_server_reply_t *inp)
+{
+  (void)inp;  return 16;
+}
+
+uint8_t
+socks5_server_reply_get_bind_addr_ipv6(socks5_server_reply_t *inp, size_t idx)
+{
+  trunnel_assert(idx < 16);
+  return inp->bind_addr_ipv6[idx];
+}
+
+uint8_t
+socks5_server_reply_getconst_bind_addr_ipv6(const socks5_server_reply_t *inp, size_t idx)
+{
+  return socks5_server_reply_get_bind_addr_ipv6((socks5_server_reply_t*)inp, idx);
+}
+int
+socks5_server_reply_set_bind_addr_ipv6(socks5_server_reply_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < 16);
+  inp->bind_addr_ipv6[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+socks5_server_reply_getarray_bind_addr_ipv6(socks5_server_reply_t *inp)
+{
+  return inp->bind_addr_ipv6;
+}
+const uint8_t  *
+socks5_server_reply_getconstarray_bind_addr_ipv6(const socks5_server_reply_t *inp)
+{
+  return (const uint8_t  *)socks5_server_reply_getarray_bind_addr_ipv6((socks5_server_reply_t*)inp);
+}
+struct domainname_st *
+socks5_server_reply_get_bind_addr_domainname(socks5_server_reply_t *inp)
+{
+  return inp->bind_addr_domainname;
+}
+const struct domainname_st *
+socks5_server_reply_getconst_bind_addr_domainname(const socks5_server_reply_t *inp)
+{
+  return socks5_server_reply_get_bind_addr_domainname((socks5_server_reply_t*) inp);
+}
+int
+socks5_server_reply_set_bind_addr_domainname(socks5_server_reply_t *inp, struct domainname_st *val)
+{
+  if (inp->bind_addr_domainname && inp->bind_addr_domainname != val)
+    domainname_free(inp->bind_addr_domainname);
+  return socks5_server_reply_set0_bind_addr_domainname(inp, val);
+}
+int
+socks5_server_reply_set0_bind_addr_domainname(socks5_server_reply_t *inp, struct domainname_st *val)
+{
+  inp->bind_addr_domainname = val;
+  return 0;
+}
+uint16_t
+socks5_server_reply_get_bind_port(const socks5_server_reply_t *inp)
+{
+  return inp->bind_port;
+}
+int
+socks5_server_reply_set_bind_port(socks5_server_reply_t *inp, uint16_t val)
+{
+  inp->bind_port = val;
+  return 0;
+}
+const char *
+socks5_server_reply_check(const socks5_server_reply_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 5))
+    return "Integer out of bounds";
+  if (! (obj->reserved == 0))
+    return "Integer out of bounds";
+  switch (obj->atype) {
+
+    case ATYPE_IPV4:
+      break;
+
+    case ATYPE_IPV6:
+      break;
+
+    case ATYPE_DOMAINNAME:
+      {
+        const char *msg;
+        if (NULL != (msg = domainname_check(obj->bind_addr_domainname)))
+          return msg;
+      }
+      break;
+
+    default:
+        return "Bad tag for union";
+      break;
+  }
+  return NULL;
+}
+
+ssize_t
+socks5_server_reply_encoded_len(const socks5_server_reply_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != socks5_server_reply_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [5] */
+  result += 1;
+
+  /* Length of u8 reply */
+  result += 1;
+
+  /* Length of u8 reserved IN [0] */
+  result += 1;
+
+  /* Length of u8 atype */
+  result += 1;
+  switch (obj->atype) {
+
+    case ATYPE_IPV4:
+
+      /* Length of u32 bind_addr_ipv4 */
+      result += 4;
+      break;
+
+    case ATYPE_IPV6:
+
+      /* Length of u8 bind_addr_ipv6[16] */
+      result += 16;
+      break;
+
+    case ATYPE_DOMAINNAME:
+
+      /* Length of struct domainname bind_addr_domainname */
+      result += domainname_encoded_len(obj->bind_addr_domainname);
+      break;
+
+    default:
+      trunnel_assert(0);
+      break;
+  }
+
+  /* Length of u16 bind_port */
+  result += 2;
+  return result;
+}
+int
+socks5_server_reply_clear_errors(socks5_server_reply_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+socks5_server_reply_encode(uint8_t *output, const size_t avail, const socks5_server_reply_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = socks5_server_reply_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = socks5_server_reply_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [5] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u8 reply */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->reply));
+  written += 1; ptr += 1;
+
+  /* Encode u8 reserved IN [0] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->reserved));
+  written += 1; ptr += 1;
+
+  /* Encode u8 atype */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->atype));
+  written += 1; ptr += 1;
+
+  /* Encode union bind_addr[atype] */
+  trunnel_assert(written <= avail);
+  switch (obj->atype) {
+
+    case ATYPE_IPV4:
+
+      /* Encode u32 bind_addr_ipv4 */
+      trunnel_assert(written <= avail);
+      if (avail - written < 4)
+        goto truncated;
+      trunnel_set_uint32(ptr, trunnel_htonl(obj->bind_addr_ipv4));
+      written += 4; ptr += 4;
+      break;
+
+    case ATYPE_IPV6:
+
+      /* Encode u8 bind_addr_ipv6[16] */
+      trunnel_assert(written <= avail);
+      if (avail - written < 16)
+        goto truncated;
+      memcpy(ptr, obj->bind_addr_ipv6, 16);
+      written += 16; ptr += 16;
+      break;
+
+    case ATYPE_DOMAINNAME:
+
+      /* Encode struct domainname bind_addr_domainname */
+      trunnel_assert(written <= avail);
+      result = domainname_encode(ptr, avail - written, obj->bind_addr_domainname);
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+      break;
+
+    default:
+      trunnel_assert(0);
+      break;
+  }
+
+  /* Encode u16 bind_port */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->bind_port));
+  written += 2; ptr += 2;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As socks5_server_reply_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+socks5_server_reply_parse_into(socks5_server_reply_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [5] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 5))
+    goto fail;
+
+  /* Parse u8 reply */
+  CHECK_REMAINING(1, truncated);
+  obj->reply = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u8 reserved IN [0] */
+  CHECK_REMAINING(1, truncated);
+  obj->reserved = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->reserved == 0))
+    goto fail;
+
+  /* Parse u8 atype */
+  CHECK_REMAINING(1, truncated);
+  obj->atype = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse union bind_addr[atype] */
+  switch (obj->atype) {
+
+    case ATYPE_IPV4:
+
+      /* Parse u32 bind_addr_ipv4 */
+      CHECK_REMAINING(4, truncated);
+      obj->bind_addr_ipv4 = trunnel_ntohl(trunnel_get_uint32(ptr));
+      remaining -= 4; ptr += 4;
+      break;
+
+    case ATYPE_IPV6:
+
+      /* Parse u8 bind_addr_ipv6[16] */
+      CHECK_REMAINING(16, truncated);
+      memcpy(obj->bind_addr_ipv6, ptr, 16);
+      remaining -= 16; ptr += 16;
+      break;
+
+    case ATYPE_DOMAINNAME:
+
+      /* Parse struct domainname bind_addr_domainname */
+      result = domainname_parse(&obj->bind_addr_domainname, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      break;
+
+    default:
+      goto fail;
+      break;
+  }
+
+  /* Parse u16 bind_port */
+  CHECK_REMAINING(2, truncated);
+  obj->bind_port = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+socks5_server_reply_parse(socks5_server_reply_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = socks5_server_reply_new();
+  if (NULL == *output)
+    return -1;
+  result = socks5_server_reply_parse_into(*output, input, len_in);
+  if (result < 0) {
+    socks5_server_reply_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/socks5.h b/src/trunnel/socks5.h
new file mode 100644
index 0000000000..d3bea152e7
--- /dev/null
+++ b/src/trunnel/socks5.h
@@ -0,0 +1,995 @@
+/* socks5.h -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_SOCKS5_H
+#define TRUNNEL_SOCKS5_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+#define CMD_CONNECT 1
+#define CMD_BIND 2
+#define CMD_UDP_ASSOCIATE 3
+#define CMD_RESOLVE 240
+#define CMD_RESOLVE_PTR 241
+#define ATYPE_IPV4 1
+#define ATYPE_IPV6 4
+#define ATYPE_DOMAINNAME 3
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_DOMAINNAME)
+struct domainname_st {
+  uint8_t len;
+  trunnel_string_t name;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct domainname_st domainname_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_SOCKS4_CLIENT_REQUEST)
+struct socks4_client_request_st {
+  uint8_t version;
+  uint8_t command;
+  uint16_t port;
+  uint32_t addr;
+  char *username;
+  char *socks4a_addr_hostname;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct socks4_client_request_st socks4_client_request_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_SOCKS4_SERVER_REPLY)
+struct socks4_server_reply_st {
+  uint8_t version;
+  uint8_t status;
+  uint16_t port;
+  uint32_t addr;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct socks4_server_reply_st socks4_server_reply_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_SOCKS5_CLIENT_USERPASS_AUTH)
+struct socks5_client_userpass_auth_st {
+  uint8_t version;
+  uint8_t username_len;
+  trunnel_string_t username;
+  uint8_t passwd_len;
+  trunnel_string_t passwd;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct socks5_client_userpass_auth_st socks5_client_userpass_auth_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_SOCKS5_CLIENT_VERSION)
+struct socks5_client_version_st {
+  uint8_t version;
+  uint8_t n_methods;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) methods;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct socks5_client_version_st socks5_client_version_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_SOCKS5_SERVER_METHOD)
+struct socks5_server_method_st {
+  uint8_t version;
+  uint8_t method;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct socks5_server_method_st socks5_server_method_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_SOCKS5_SERVER_USERPASS_AUTH)
+struct socks5_server_userpass_auth_st {
+  uint8_t version;
+  uint8_t status;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct socks5_server_userpass_auth_st socks5_server_userpass_auth_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_SOCKS5_CLIENT_REQUEST)
+struct socks5_client_request_st {
+  uint8_t version;
+  uint8_t command;
+  uint8_t reserved;
+  uint8_t atype;
+  uint32_t dest_addr_ipv4;
+  uint8_t dest_addr_ipv6[16];
+  struct domainname_st *dest_addr_domainname;
+  uint16_t dest_port;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct socks5_client_request_st socks5_client_request_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_SOCKS5_SERVER_REPLY)
+struct socks5_server_reply_st {
+  uint8_t version;
+  uint8_t reply;
+  uint8_t reserved;
+  uint8_t atype;
+  uint32_t bind_addr_ipv4;
+  uint8_t bind_addr_ipv6[16];
+  struct domainname_st *bind_addr_domainname;
+  uint16_t bind_port;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct socks5_server_reply_st socks5_server_reply_t;
+/** Return a newly allocated domainname with all elements set to zero.
+ */
+domainname_t *domainname_new(void);
+/** Release all storage held by the domainname in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void domainname_free(domainname_t *victim);
+/** Try to parse a domainname from the buffer in 'input', using up to
+ * 'len_in' bytes from the input buffer. On success, return the number
+ * of bytes consumed and set *output to the newly allocated
+ * domainname_t. On failure, return -2 if the input appears truncated,
+ * and -1 if the input is otherwise invalid.
+ */
+ssize_t domainname_parse(domainname_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * domainname in 'obj'. On failure, return a negative value. Note that
+ * this value may be an overestimate, and can even be an underestimate
+ * for certain unencodeable objects.
+ */
+ssize_t domainname_encoded_len(const domainname_t *obj);
+/** Try to encode the domainname from 'input' into the buffer at
+ * 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t domainname_encode(uint8_t *output, size_t avail, const domainname_t *input);
+/** Check whether the internal state of the domainname in 'obj' is
+ * consistent. Return NULL if it is, and a short message if it is not.
+ */
+const char *domainname_check(const domainname_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int domainname_clear_errors(domainname_t *obj);
+/** Return the value of the len field of the domainname_t in 'inp'
+ */
+uint8_t domainname_get_len(const domainname_t *inp);
+/** Set the value of the len field of the domainname_t in 'inp' to
+ * 'val'. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int domainname_set_len(domainname_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the name field of
+ * the domainname_t in 'inp'.
+ */
+size_t domainname_getlen_name(const domainname_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * name of the domainname_t in 'inp'.
+ */
+char domainname_get_name(domainname_t *inp, size_t idx);
+/** As domainname_get_name, but take and return a const pointer
+ */
+char domainname_getconst_name(const domainname_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * name of the domainname_t in 'inp', so that it will hold the value
+ * 'elt'.
+ */
+int domainname_set_name(domainname_t *inp, size_t idx, char elt);
+/** Append a new element 'elt' to the dynamic array field name of the
+ * domainname_t in 'inp'.
+ */
+int domainname_add_name(domainname_t *inp, char elt);
+/** Return a pointer to the variable-length array field name of 'inp'.
+ */
+char * domainname_getarray_name(domainname_t *inp);
+/** As domainname_get_name, but take and return a const pointer
+ */
+const char  * domainname_getconstarray_name(const domainname_t *inp);
+/** Change the length of the variable-length array field name of 'inp'
+ * to 'newlen'.Fill extra elements with 0. Return 0 on success; return
+ * -1 and set the error code on 'inp' on failure.
+ */
+int domainname_setlen_name(domainname_t *inp, size_t newlen);
+/** Return the value of the name field of a domainname_t as a NUL-
+ * terminated string.
+ */
+const char * domainname_getstr_name(domainname_t *inp);
+/** Set the value of the name field of a domainname_t to a given
+ * string of length 'len'. Return 0 on success; return -1 and set the
+ * error code on 'inp' on failure.
+ */
+int domainname_setstr0_name(domainname_t *inp, const char *val, size_t len);
+/** Set the value of the name field of a domainname_t to a given NUL-
+ * terminated string. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int domainname_setstr_name(domainname_t *inp, const char *val);
+/** Return a newly allocated socks4_client_request with all elements
+ * set to zero.
+ */
+socks4_client_request_t *socks4_client_request_new(void);
+/** Release all storage held by the socks4_client_request in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void socks4_client_request_free(socks4_client_request_t *victim);
+/** Try to parse a socks4_client_request from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated socks4_client_request_t. On failure, return -2 if the
+ * input appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t socks4_client_request_parse(socks4_client_request_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * socks4_client_request in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t socks4_client_request_encoded_len(const socks4_client_request_t *obj);
+/** Try to encode the socks4_client_request from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t socks4_client_request_encode(uint8_t *output, size_t avail, const socks4_client_request_t *input);
+/** Check whether the internal state of the socks4_client_request in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *socks4_client_request_check(const socks4_client_request_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int socks4_client_request_clear_errors(socks4_client_request_t *obj);
+/** Return the value of the version field of the
+ * socks4_client_request_t in 'inp'
+ */
+uint8_t socks4_client_request_get_version(const socks4_client_request_t *inp);
+/** Set the value of the version field of the socks4_client_request_t
+ * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int socks4_client_request_set_version(socks4_client_request_t *inp, uint8_t val);
+/** Return the value of the command field of the
+ * socks4_client_request_t in 'inp'
+ */
+uint8_t socks4_client_request_get_command(const socks4_client_request_t *inp);
+/** Set the value of the command field of the socks4_client_request_t
+ * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int socks4_client_request_set_command(socks4_client_request_t *inp, uint8_t val);
+/** Return the value of the port field of the socks4_client_request_t
+ * in 'inp'
+ */
+uint16_t socks4_client_request_get_port(const socks4_client_request_t *inp);
+/** Set the value of the port field of the socks4_client_request_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int socks4_client_request_set_port(socks4_client_request_t *inp, uint16_t val);
+/** Return the value of the addr field of the socks4_client_request_t
+ * in 'inp'
+ */
+uint32_t socks4_client_request_get_addr(const socks4_client_request_t *inp);
+/** Set the value of the addr field of the socks4_client_request_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int socks4_client_request_set_addr(socks4_client_request_t *inp, uint32_t val);
+/** Return the value of the username field of the
+ * socks4_client_request_t in 'inp'
+ */
+const char * socks4_client_request_get_username(const socks4_client_request_t *inp);
+/** Set the value of the username field of the socks4_client_request_t
+ * in 'inp' to 'val'. Free the old value if any. Does not steal the
+ * reference to 'val'.Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int socks4_client_request_set_username(socks4_client_request_t *inp, const char *val);
+/** Return the value of the socks4a_addr_hostname field of the
+ * socks4_client_request_t in 'inp'
+ */
+const char * socks4_client_request_get_socks4a_addr_hostname(const socks4_client_request_t *inp);
+/** Set the value of the socks4a_addr_hostname field of the
+ * socks4_client_request_t in 'inp' to 'val'. Free the old value if
+ * any. Does not steal the reference to 'val'.Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int socks4_client_request_set_socks4a_addr_hostname(socks4_client_request_t *inp, const char *val);
+/** Return a newly allocated socks4_server_reply with all elements set
+ * to zero.
+ */
+socks4_server_reply_t *socks4_server_reply_new(void);
+/** Release all storage held by the socks4_server_reply in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void socks4_server_reply_free(socks4_server_reply_t *victim);
+/** Try to parse a socks4_server_reply from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated socks4_server_reply_t. On failure, return -2 if the input
+ * appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t socks4_server_reply_parse(socks4_server_reply_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * socks4_server_reply in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t socks4_server_reply_encoded_len(const socks4_server_reply_t *obj);
+/** Try to encode the socks4_server_reply from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t socks4_server_reply_encode(uint8_t *output, size_t avail, const socks4_server_reply_t *input);
+/** Check whether the internal state of the socks4_server_reply in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *socks4_server_reply_check(const socks4_server_reply_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int socks4_server_reply_clear_errors(socks4_server_reply_t *obj);
+/** Return the value of the version field of the socks4_server_reply_t
+ * in 'inp'
+ */
+uint8_t socks4_server_reply_get_version(const socks4_server_reply_t *inp);
+/** Set the value of the version field of the socks4_server_reply_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int socks4_server_reply_set_version(socks4_server_reply_t *inp, uint8_t val);
+/** Return the value of the status field of the socks4_server_reply_t
+ * in 'inp'
+ */
+uint8_t socks4_server_reply_get_status(const socks4_server_reply_t *inp);
+/** Set the value of the status field of the socks4_server_reply_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int socks4_server_reply_set_status(socks4_server_reply_t *inp, uint8_t val);
+/** Return the value of the port field of the socks4_server_reply_t in
+ * 'inp'
+ */
+uint16_t socks4_server_reply_get_port(const socks4_server_reply_t *inp);
+/** Set the value of the port field of the socks4_server_reply_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int socks4_server_reply_set_port(socks4_server_reply_t *inp, uint16_t val);
+/** Return the value of the addr field of the socks4_server_reply_t in
+ * 'inp'
+ */
+uint32_t socks4_server_reply_get_addr(const socks4_server_reply_t *inp);
+/** Set the value of the addr field of the socks4_server_reply_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int socks4_server_reply_set_addr(socks4_server_reply_t *inp, uint32_t val);
+/** Return a newly allocated socks5_client_userpass_auth with all
+ * elements set to zero.
+ */
+socks5_client_userpass_auth_t *socks5_client_userpass_auth_new(void);
+/** Release all storage held by the socks5_client_userpass_auth in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void socks5_client_userpass_auth_free(socks5_client_userpass_auth_t *victim);
+/** Try to parse a socks5_client_userpass_auth from the buffer in
+ * 'input', using up to 'len_in' bytes from the input buffer. On
+ * success, return the number of bytes consumed and set *output to the
+ * newly allocated socks5_client_userpass_auth_t. On failure, return
+ * -2 if the input appears truncated, and -1 if the input is otherwise
+ * invalid.
+ */
+ssize_t socks5_client_userpass_auth_parse(socks5_client_userpass_auth_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * socks5_client_userpass_auth in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t socks5_client_userpass_auth_encoded_len(const socks5_client_userpass_auth_t *obj);
+/** Try to encode the socks5_client_userpass_auth from 'input' into
+ * the buffer at 'output', using up to 'avail' bytes of the output
+ * buffer. On success, return the number of bytes used. On failure,
+ * return -2 if the buffer was not long enough, and -1 if the input
+ * was invalid.
+ */
+ssize_t socks5_client_userpass_auth_encode(uint8_t *output, size_t avail, const socks5_client_userpass_auth_t *input);
+/** Check whether the internal state of the
+ * socks5_client_userpass_auth in 'obj' is consistent. Return NULL if
+ * it is, and a short message if it is not.
+ */
+const char *socks5_client_userpass_auth_check(const socks5_client_userpass_auth_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int socks5_client_userpass_auth_clear_errors(socks5_client_userpass_auth_t *obj);
+/** Return the value of the version field of the
+ * socks5_client_userpass_auth_t in 'inp'
+ */
+uint8_t socks5_client_userpass_auth_get_version(const socks5_client_userpass_auth_t *inp);
+/** Set the value of the version field of the
+ * socks5_client_userpass_auth_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int socks5_client_userpass_auth_set_version(socks5_client_userpass_auth_t *inp, uint8_t val);
+/** Return the value of the username_len field of the
+ * socks5_client_userpass_auth_t in 'inp'
+ */
+uint8_t socks5_client_userpass_auth_get_username_len(const socks5_client_userpass_auth_t *inp);
+/** Set the value of the username_len field of the
+ * socks5_client_userpass_auth_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int socks5_client_userpass_auth_set_username_len(socks5_client_userpass_auth_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the username field
+ * of the socks5_client_userpass_auth_t in 'inp'.
+ */
+size_t socks5_client_userpass_auth_getlen_username(const socks5_client_userpass_auth_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * username of the socks5_client_userpass_auth_t in 'inp'.
+ */
+char socks5_client_userpass_auth_get_username(socks5_client_userpass_auth_t *inp, size_t idx);
+/** As socks5_client_userpass_auth_get_username, but take and return a
+ * const pointer
+ */
+char socks5_client_userpass_auth_getconst_username(const socks5_client_userpass_auth_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * username of the socks5_client_userpass_auth_t in 'inp', so that it
+ * will hold the value 'elt'.
+ */
+int socks5_client_userpass_auth_set_username(socks5_client_userpass_auth_t *inp, size_t idx, char elt);
+/** Append a new element 'elt' to the dynamic array field username of
+ * the socks5_client_userpass_auth_t in 'inp'.
+ */
+int socks5_client_userpass_auth_add_username(socks5_client_userpass_auth_t *inp, char elt);
+/** Return a pointer to the variable-length array field username of
+ * 'inp'.
+ */
+char * socks5_client_userpass_auth_getarray_username(socks5_client_userpass_auth_t *inp);
+/** As socks5_client_userpass_auth_get_username, but take and return a
+ * const pointer
+ */
+const char  * socks5_client_userpass_auth_getconstarray_username(const socks5_client_userpass_auth_t *inp);
+/** Change the length of the variable-length array field username of
+ * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int socks5_client_userpass_auth_setlen_username(socks5_client_userpass_auth_t *inp, size_t newlen);
+/** Return the value of the username field of a
+ * socks5_client_userpass_auth_t as a NUL-terminated string.
+ */
+const char * socks5_client_userpass_auth_getstr_username(socks5_client_userpass_auth_t *inp);
+/** Set the value of the username field of a
+ * socks5_client_userpass_auth_t to a given string of length 'len'.
+ * Return 0 on success; return -1 and set the error code on 'inp' on
+ * failure.
+ */
+int socks5_client_userpass_auth_setstr0_username(socks5_client_userpass_auth_t *inp, const char *val, size_t len);
+/** Set the value of the username field of a
+ * socks5_client_userpass_auth_t to a given NUL-terminated string.
+ * Return 0 on success; return -1 and set the error code on 'inp' on
+ * failure.
+ */
+int socks5_client_userpass_auth_setstr_username(socks5_client_userpass_auth_t *inp, const char *val);
+/** Return the value of the passwd_len field of the
+ * socks5_client_userpass_auth_t in 'inp'
+ */
+uint8_t socks5_client_userpass_auth_get_passwd_len(const socks5_client_userpass_auth_t *inp);
+/** Set the value of the passwd_len field of the
+ * socks5_client_userpass_auth_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int socks5_client_userpass_auth_set_passwd_len(socks5_client_userpass_auth_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the passwd field of
+ * the socks5_client_userpass_auth_t in 'inp'.
+ */
+size_t socks5_client_userpass_auth_getlen_passwd(const socks5_client_userpass_auth_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * passwd of the socks5_client_userpass_auth_t in 'inp'.
+ */
+char socks5_client_userpass_auth_get_passwd(socks5_client_userpass_auth_t *inp, size_t idx);
+/** As socks5_client_userpass_auth_get_passwd, but take and return a
+ * const pointer
+ */
+char socks5_client_userpass_auth_getconst_passwd(const socks5_client_userpass_auth_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * passwd of the socks5_client_userpass_auth_t in 'inp', so that it
+ * will hold the value 'elt'.
+ */
+int socks5_client_userpass_auth_set_passwd(socks5_client_userpass_auth_t *inp, size_t idx, char elt);
+/** Append a new element 'elt' to the dynamic array field passwd of
+ * the socks5_client_userpass_auth_t in 'inp'.
+ */
+int socks5_client_userpass_auth_add_passwd(socks5_client_userpass_auth_t *inp, char elt);
+/** Return a pointer to the variable-length array field passwd of
+ * 'inp'.
+ */
+char * socks5_client_userpass_auth_getarray_passwd(socks5_client_userpass_auth_t *inp);
+/** As socks5_client_userpass_auth_get_passwd, but take and return a
+ * const pointer
+ */
+const char  * socks5_client_userpass_auth_getconstarray_passwd(const socks5_client_userpass_auth_t *inp);
+/** Change the length of the variable-length array field passwd of
+ * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int socks5_client_userpass_auth_setlen_passwd(socks5_client_userpass_auth_t *inp, size_t newlen);
+/** Return the value of the passwd field of a
+ * socks5_client_userpass_auth_t as a NUL-terminated string.
+ */
+const char * socks5_client_userpass_auth_getstr_passwd(socks5_client_userpass_auth_t *inp);
+/** Set the value of the passwd field of a
+ * socks5_client_userpass_auth_t to a given string of length 'len'.
+ * Return 0 on success; return -1 and set the error code on 'inp' on
+ * failure.
+ */
+int socks5_client_userpass_auth_setstr0_passwd(socks5_client_userpass_auth_t *inp, const char *val, size_t len);
+/** Set the value of the passwd field of a
+ * socks5_client_userpass_auth_t to a given NUL-terminated string.
+ * Return 0 on success; return -1 and set the error code on 'inp' on
+ * failure.
+ */
+int socks5_client_userpass_auth_setstr_passwd(socks5_client_userpass_auth_t *inp, const char *val);
+/** Return a newly allocated socks5_client_version with all elements
+ * set to zero.
+ */
+socks5_client_version_t *socks5_client_version_new(void);
+/** Release all storage held by the socks5_client_version in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void socks5_client_version_free(socks5_client_version_t *victim);
+/** Try to parse a socks5_client_version from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated socks5_client_version_t. On failure, return -2 if the
+ * input appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t socks5_client_version_parse(socks5_client_version_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * socks5_client_version in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t socks5_client_version_encoded_len(const socks5_client_version_t *obj);
+/** Try to encode the socks5_client_version from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t socks5_client_version_encode(uint8_t *output, size_t avail, const socks5_client_version_t *input);
+/** Check whether the internal state of the socks5_client_version in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *socks5_client_version_check(const socks5_client_version_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int socks5_client_version_clear_errors(socks5_client_version_t *obj);
+/** Return the value of the version field of the
+ * socks5_client_version_t in 'inp'
+ */
+uint8_t socks5_client_version_get_version(const socks5_client_version_t *inp);
+/** Set the value of the version field of the socks5_client_version_t
+ * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int socks5_client_version_set_version(socks5_client_version_t *inp, uint8_t val);
+/** Return the value of the n_methods field of the
+ * socks5_client_version_t in 'inp'
+ */
+uint8_t socks5_client_version_get_n_methods(const socks5_client_version_t *inp);
+/** Set the value of the n_methods field of the
+ * socks5_client_version_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int socks5_client_version_set_n_methods(socks5_client_version_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the methods field
+ * of the socks5_client_version_t in 'inp'.
+ */
+size_t socks5_client_version_getlen_methods(const socks5_client_version_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * methods of the socks5_client_version_t in 'inp'.
+ */
+uint8_t socks5_client_version_get_methods(socks5_client_version_t *inp, size_t idx);
+/** As socks5_client_version_get_methods, but take and return a const
+ * pointer
+ */
+uint8_t socks5_client_version_getconst_methods(const socks5_client_version_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * methods of the socks5_client_version_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int socks5_client_version_set_methods(socks5_client_version_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field methods of
+ * the socks5_client_version_t in 'inp'.
+ */
+int socks5_client_version_add_methods(socks5_client_version_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field methods of
+ * 'inp'.
+ */
+uint8_t * socks5_client_version_getarray_methods(socks5_client_version_t *inp);
+/** As socks5_client_version_get_methods, but take and return a const
+ * pointer
+ */
+const uint8_t  * socks5_client_version_getconstarray_methods(const socks5_client_version_t *inp);
+/** Change the length of the variable-length array field methods of
+ * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int socks5_client_version_setlen_methods(socks5_client_version_t *inp, size_t newlen);
+/** Return a newly allocated socks5_server_method with all elements
+ * set to zero.
+ */
+socks5_server_method_t *socks5_server_method_new(void);
+/** Release all storage held by the socks5_server_method in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void socks5_server_method_free(socks5_server_method_t *victim);
+/** Try to parse a socks5_server_method from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated socks5_server_method_t. On failure, return -2 if the
+ * input appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t socks5_server_method_parse(socks5_server_method_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * socks5_server_method in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t socks5_server_method_encoded_len(const socks5_server_method_t *obj);
+/** Try to encode the socks5_server_method from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t socks5_server_method_encode(uint8_t *output, size_t avail, const socks5_server_method_t *input);
+/** Check whether the internal state of the socks5_server_method in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *socks5_server_method_check(const socks5_server_method_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int socks5_server_method_clear_errors(socks5_server_method_t *obj);
+/** Return the value of the version field of the
+ * socks5_server_method_t in 'inp'
+ */
+uint8_t socks5_server_method_get_version(const socks5_server_method_t *inp);
+/** Set the value of the version field of the socks5_server_method_t
+ * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int socks5_server_method_set_version(socks5_server_method_t *inp, uint8_t val);
+/** Return the value of the method field of the socks5_server_method_t
+ * in 'inp'
+ */
+uint8_t socks5_server_method_get_method(const socks5_server_method_t *inp);
+/** Set the value of the method field of the socks5_server_method_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int socks5_server_method_set_method(socks5_server_method_t *inp, uint8_t val);
+/** Return a newly allocated socks5_server_userpass_auth with all
+ * elements set to zero.
+ */
+socks5_server_userpass_auth_t *socks5_server_userpass_auth_new(void);
+/** Release all storage held by the socks5_server_userpass_auth in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void socks5_server_userpass_auth_free(socks5_server_userpass_auth_t *victim);
+/** Try to parse a socks5_server_userpass_auth from the buffer in
+ * 'input', using up to 'len_in' bytes from the input buffer. On
+ * success, return the number of bytes consumed and set *output to the
+ * newly allocated socks5_server_userpass_auth_t. On failure, return
+ * -2 if the input appears truncated, and -1 if the input is otherwise
+ * invalid.
+ */
+ssize_t socks5_server_userpass_auth_parse(socks5_server_userpass_auth_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * socks5_server_userpass_auth in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t socks5_server_userpass_auth_encoded_len(const socks5_server_userpass_auth_t *obj);
+/** Try to encode the socks5_server_userpass_auth from 'input' into
+ * the buffer at 'output', using up to 'avail' bytes of the output
+ * buffer. On success, return the number of bytes used. On failure,
+ * return -2 if the buffer was not long enough, and -1 if the input
+ * was invalid.
+ */
+ssize_t socks5_server_userpass_auth_encode(uint8_t *output, size_t avail, const socks5_server_userpass_auth_t *input);
+/** Check whether the internal state of the
+ * socks5_server_userpass_auth in 'obj' is consistent. Return NULL if
+ * it is, and a short message if it is not.
+ */
+const char *socks5_server_userpass_auth_check(const socks5_server_userpass_auth_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int socks5_server_userpass_auth_clear_errors(socks5_server_userpass_auth_t *obj);
+/** Return the value of the version field of the
+ * socks5_server_userpass_auth_t in 'inp'
+ */
+uint8_t socks5_server_userpass_auth_get_version(const socks5_server_userpass_auth_t *inp);
+/** Set the value of the version field of the
+ * socks5_server_userpass_auth_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int socks5_server_userpass_auth_set_version(socks5_server_userpass_auth_t *inp, uint8_t val);
+/** Return the value of the status field of the
+ * socks5_server_userpass_auth_t in 'inp'
+ */
+uint8_t socks5_server_userpass_auth_get_status(const socks5_server_userpass_auth_t *inp);
+/** Set the value of the status field of the
+ * socks5_server_userpass_auth_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int socks5_server_userpass_auth_set_status(socks5_server_userpass_auth_t *inp, uint8_t val);
+/** Return a newly allocated socks5_client_request with all elements
+ * set to zero.
+ */
+socks5_client_request_t *socks5_client_request_new(void);
+/** Release all storage held by the socks5_client_request in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void socks5_client_request_free(socks5_client_request_t *victim);
+/** Try to parse a socks5_client_request from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated socks5_client_request_t. On failure, return -2 if the
+ * input appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t socks5_client_request_parse(socks5_client_request_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * socks5_client_request in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t socks5_client_request_encoded_len(const socks5_client_request_t *obj);
+/** Try to encode the socks5_client_request from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t socks5_client_request_encode(uint8_t *output, size_t avail, const socks5_client_request_t *input);
+/** Check whether the internal state of the socks5_client_request in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *socks5_client_request_check(const socks5_client_request_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int socks5_client_request_clear_errors(socks5_client_request_t *obj);
+/** Return the value of the version field of the
+ * socks5_client_request_t in 'inp'
+ */
+uint8_t socks5_client_request_get_version(const socks5_client_request_t *inp);
+/** Set the value of the version field of the socks5_client_request_t
+ * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int socks5_client_request_set_version(socks5_client_request_t *inp, uint8_t val);
+/** Return the value of the command field of the
+ * socks5_client_request_t in 'inp'
+ */
+uint8_t socks5_client_request_get_command(const socks5_client_request_t *inp);
+/** Set the value of the command field of the socks5_client_request_t
+ * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int socks5_client_request_set_command(socks5_client_request_t *inp, uint8_t val);
+/** Return the value of the reserved field of the
+ * socks5_client_request_t in 'inp'
+ */
+uint8_t socks5_client_request_get_reserved(const socks5_client_request_t *inp);
+/** Set the value of the reserved field of the socks5_client_request_t
+ * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int socks5_client_request_set_reserved(socks5_client_request_t *inp, uint8_t val);
+/** Return the value of the atype field of the socks5_client_request_t
+ * in 'inp'
+ */
+uint8_t socks5_client_request_get_atype(const socks5_client_request_t *inp);
+/** Set the value of the atype field of the socks5_client_request_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int socks5_client_request_set_atype(socks5_client_request_t *inp, uint8_t val);
+/** Return the value of the dest_addr_ipv4 field of the
+ * socks5_client_request_t in 'inp'
+ */
+uint32_t socks5_client_request_get_dest_addr_ipv4(const socks5_client_request_t *inp);
+/** Set the value of the dest_addr_ipv4 field of the
+ * socks5_client_request_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int socks5_client_request_set_dest_addr_ipv4(socks5_client_request_t *inp, uint32_t val);
+/** Return the (constant) length of the array holding the
+ * dest_addr_ipv6 field of the socks5_client_request_t in 'inp'.
+ */
+size_t socks5_client_request_getlen_dest_addr_ipv6(const socks5_client_request_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * dest_addr_ipv6 of the socks5_client_request_t in 'inp'.
+ */
+uint8_t socks5_client_request_get_dest_addr_ipv6(socks5_client_request_t *inp, size_t idx);
+/** As socks5_client_request_get_dest_addr_ipv6, but take and return a
+ * const pointer
+ */
+uint8_t socks5_client_request_getconst_dest_addr_ipv6(const socks5_client_request_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * dest_addr_ipv6 of the socks5_client_request_t in 'inp', so that it
+ * will hold the value 'elt'.
+ */
+int socks5_client_request_set_dest_addr_ipv6(socks5_client_request_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the 16-element array field dest_addr_ipv6 of
+ * 'inp'.
+ */
+uint8_t * socks5_client_request_getarray_dest_addr_ipv6(socks5_client_request_t *inp);
+/** As socks5_client_request_get_dest_addr_ipv6, but take and return a
+ * const pointer
+ */
+const uint8_t  * socks5_client_request_getconstarray_dest_addr_ipv6(const socks5_client_request_t *inp);
+/** Return the value of the dest_addr_domainname field of the
+ * socks5_client_request_t in 'inp'
+ */
+struct domainname_st * socks5_client_request_get_dest_addr_domainname(socks5_client_request_t *inp);
+/** As socks5_client_request_get_dest_addr_domainname, but take and
+ * return a const pointer
+ */
+const struct domainname_st * socks5_client_request_getconst_dest_addr_domainname(const socks5_client_request_t *inp);
+/** Set the value of the dest_addr_domainname field of the
+ * socks5_client_request_t in 'inp' to 'val'. Free the old value if
+ * any. Steals the referenceto 'val'.Return 0 on success; return -1
+ * and set the error code on 'inp' on failure.
+ */
+int socks5_client_request_set_dest_addr_domainname(socks5_client_request_t *inp, struct domainname_st *val);
+/** As socks5_client_request_set_dest_addr_domainname, but does not
+ * free the previous value.
+ */
+int socks5_client_request_set0_dest_addr_domainname(socks5_client_request_t *inp, struct domainname_st *val);
+/** Return the value of the dest_port field of the
+ * socks5_client_request_t in 'inp'
+ */
+uint16_t socks5_client_request_get_dest_port(const socks5_client_request_t *inp);
+/** Set the value of the dest_port field of the
+ * socks5_client_request_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int socks5_client_request_set_dest_port(socks5_client_request_t *inp, uint16_t val);
+/** Return a newly allocated socks5_server_reply with all elements set
+ * to zero.
+ */
+socks5_server_reply_t *socks5_server_reply_new(void);
+/** Release all storage held by the socks5_server_reply in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void socks5_server_reply_free(socks5_server_reply_t *victim);
+/** Try to parse a socks5_server_reply from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated socks5_server_reply_t. On failure, return -2 if the input
+ * appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t socks5_server_reply_parse(socks5_server_reply_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * socks5_server_reply in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t socks5_server_reply_encoded_len(const socks5_server_reply_t *obj);
+/** Try to encode the socks5_server_reply from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t socks5_server_reply_encode(uint8_t *output, size_t avail, const socks5_server_reply_t *input);
+/** Check whether the internal state of the socks5_server_reply in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *socks5_server_reply_check(const socks5_server_reply_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int socks5_server_reply_clear_errors(socks5_server_reply_t *obj);
+/** Return the value of the version field of the socks5_server_reply_t
+ * in 'inp'
+ */
+uint8_t socks5_server_reply_get_version(const socks5_server_reply_t *inp);
+/** Set the value of the version field of the socks5_server_reply_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int socks5_server_reply_set_version(socks5_server_reply_t *inp, uint8_t val);
+/** Return the value of the reply field of the socks5_server_reply_t
+ * in 'inp'
+ */
+uint8_t socks5_server_reply_get_reply(const socks5_server_reply_t *inp);
+/** Set the value of the reply field of the socks5_server_reply_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int socks5_server_reply_set_reply(socks5_server_reply_t *inp, uint8_t val);
+/** Return the value of the reserved field of the
+ * socks5_server_reply_t in 'inp'
+ */
+uint8_t socks5_server_reply_get_reserved(const socks5_server_reply_t *inp);
+/** Set the value of the reserved field of the socks5_server_reply_t
+ * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int socks5_server_reply_set_reserved(socks5_server_reply_t *inp, uint8_t val);
+/** Return the value of the atype field of the socks5_server_reply_t
+ * in 'inp'
+ */
+uint8_t socks5_server_reply_get_atype(const socks5_server_reply_t *inp);
+/** Set the value of the atype field of the socks5_server_reply_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int socks5_server_reply_set_atype(socks5_server_reply_t *inp, uint8_t val);
+/** Return the value of the bind_addr_ipv4 field of the
+ * socks5_server_reply_t in 'inp'
+ */
+uint32_t socks5_server_reply_get_bind_addr_ipv4(const socks5_server_reply_t *inp);
+/** Set the value of the bind_addr_ipv4 field of the
+ * socks5_server_reply_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int socks5_server_reply_set_bind_addr_ipv4(socks5_server_reply_t *inp, uint32_t val);
+/** Return the (constant) length of the array holding the
+ * bind_addr_ipv6 field of the socks5_server_reply_t in 'inp'.
+ */
+size_t socks5_server_reply_getlen_bind_addr_ipv6(const socks5_server_reply_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * bind_addr_ipv6 of the socks5_server_reply_t in 'inp'.
+ */
+uint8_t socks5_server_reply_get_bind_addr_ipv6(socks5_server_reply_t *inp, size_t idx);
+/** As socks5_server_reply_get_bind_addr_ipv6, but take and return a
+ * const pointer
+ */
+uint8_t socks5_server_reply_getconst_bind_addr_ipv6(const socks5_server_reply_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * bind_addr_ipv6 of the socks5_server_reply_t in 'inp', so that it
+ * will hold the value 'elt'.
+ */
+int socks5_server_reply_set_bind_addr_ipv6(socks5_server_reply_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the 16-element array field bind_addr_ipv6 of
+ * 'inp'.
+ */
+uint8_t * socks5_server_reply_getarray_bind_addr_ipv6(socks5_server_reply_t *inp);
+/** As socks5_server_reply_get_bind_addr_ipv6, but take and return a
+ * const pointer
+ */
+const uint8_t  * socks5_server_reply_getconstarray_bind_addr_ipv6(const socks5_server_reply_t *inp);
+/** Return the value of the bind_addr_domainname field of the
+ * socks5_server_reply_t in 'inp'
+ */
+struct domainname_st * socks5_server_reply_get_bind_addr_domainname(socks5_server_reply_t *inp);
+/** As socks5_server_reply_get_bind_addr_domainname, but take and
+ * return a const pointer
+ */
+const struct domainname_st * socks5_server_reply_getconst_bind_addr_domainname(const socks5_server_reply_t *inp);
+/** Set the value of the bind_addr_domainname field of the
+ * socks5_server_reply_t in 'inp' to 'val'. Free the old value if any.
+ * Steals the referenceto 'val'.Return 0 on success; return -1 and set
+ * the error code on 'inp' on failure.
+ */
+int socks5_server_reply_set_bind_addr_domainname(socks5_server_reply_t *inp, struct domainname_st *val);
+/** As socks5_server_reply_set_bind_addr_domainname, but does not free
+ * the previous value.
+ */
+int socks5_server_reply_set0_bind_addr_domainname(socks5_server_reply_t *inp, struct domainname_st *val);
+/** Return the value of the bind_port field of the
+ * socks5_server_reply_t in 'inp'
+ */
+uint16_t socks5_server_reply_get_bind_port(const socks5_server_reply_t *inp);
+/** Set the value of the bind_port field of the socks5_server_reply_t
+ * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int socks5_server_reply_set_bind_port(socks5_server_reply_t *inp, uint16_t val);
+
+
+#endif
diff --git a/src/trunnel/socks5.trunnel b/src/trunnel/socks5.trunnel
new file mode 100644
index 0000000000..b86ec03b9d
--- /dev/null
+++ b/src/trunnel/socks5.trunnel
@@ -0,0 +1,94 @@
+// Example: here's a quickie implementation of the messages in the
+// socks5 protocol.
+
+struct socks5_client_version {
+   u8 version IN [5];
+   u8 n_methods;
+   u8 methods[n_methods];
+}
+
+struct socks5_server_method {
+   u8 version IN [5];
+   u8 method;
+}
+
+const CMD_CONNECT = 1;
+const CMD_BIND = 2;
+const CMD_UDP_ASSOCIATE = 3;
+// This is a tor extension
+const CMD_RESOLVE = 0xF0;
+const CMD_RESOLVE_PTR = 0xF1;
+
+const ATYPE_IPV4 = 1;
+const ATYPE_IPV6 = 4;
+const ATYPE_DOMAINNAME = 3;
+
+struct domainname {
+   u8 len;
+   char name[len];
+}
+
+struct socks5_client_request {
+   u8 version IN [5];
+   u8 command IN [CMD_CONNECT, CMD_BIND, CMD_UDP_ASSOCIATE, CMD_RESOLVE, CMD_RESOLVE_PTR];
+   u8 reserved IN [0];
+   u8 atype;
+   union dest_addr[atype] {
+     ATYPE_IPV4: u32 ipv4;
+     ATYPE_IPV6: u8 ipv6[16];
+     ATYPE_DOMAINNAME: struct domainname domainname;
+     default: fail;
+   };
+   u16 dest_port;
+}
+
+struct socks5_server_reply {
+   u8 version IN [5];
+   u8 reply;
+   u8 reserved IN [0];
+   u8 atype;
+   union bind_addr[atype] {
+     ATYPE_IPV4: u32 ipv4;
+     ATYPE_IPV6: u8 ipv6[16];
+     ATYPE_DOMAINNAME: struct domainname domainname;
+     default: fail;
+   };
+   u16 bind_port;
+}
+
+struct socks5_client_userpass_auth {
+   u8 version IN [1];
+   u8 username_len;
+   char username[username_len];
+   u8 passwd_len;
+   char passwd[passwd_len];
+}
+
+struct socks5_server_userpass_auth {
+   u8 version IN [1];
+   u8 status;
+}
+
+// Oh why not.  Here's socks4 and socks4a.
+
+struct socks4_client_request {
+   u8 version IN [4];
+   u8 command IN [CMD_CONNECT,CMD_BIND,CMD_RESOLVE,CMD_RESOLVE_PTR];
+   u16 port;
+   u32 addr;
+   nulterm username;
+   union socks4a_addr[addr] {
+      1..255:
+               nulterm hostname;
+      default:
+               ;
+   };
+}
+
+struct socks4_server_reply {
+   u8 version IN [4];
+   u8 status;
+   u16 port;
+   u32 addr;
+}
+
diff --git a/src/trunnel/trunnel-local.h b/src/trunnel/trunnel-local.h
index b7c2ab98ef..c4118fce4c 100644
--- a/src/trunnel/trunnel-local.h
+++ b/src/trunnel/trunnel-local.h
@@ -2,9 +2,9 @@
 #ifndef TRUNNEL_LOCAL_H_INCLUDED
 #define TRUNNEL_LOCAL_H_INCLUDED
 
-#include "util.h"
-#include "compat.h"
-#include "crypto.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/malloc/malloc.h"
+#include "lib/log/util_bug.h"
 
 #define trunnel_malloc tor_malloc
 #define trunnel_calloc tor_calloc
diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h
index f6d0e7d684..baad733d0f 100644
--- a/src/win32/orconfig.h
+++ b/src/win32/orconfig.h
@@ -218,7 +218,7 @@
 #define USING_TWOS_COMPLEMENT
 
 /* Version number of package */
-#define VERSION "0.2.9.17-dev"
+#define VERSION "0.3.5.9-dev"
 
 
 
diff --git a/warning_flags.in b/warning_flags.in
new file mode 100644
index 0000000000..e8ebe6e9a4
--- /dev/null
+++ b/warning_flags.in
@@ -0,0 +1 @@
+@TOR_WARNING_FLAGS@
\ No newline at end of file